Merge pull request #63 from marinak-ebi/slurm-migration

Migrate all pipelines from LSF to SLURM

Author: marinak-ebi

Extract details from StatPackets/ExtractPVals.R

@@ -733,13 +733,13 @@ makejobs = function(path = getwd()) {
     bf = basename(file)
     n = length(parameters)
     jobs = paste0 (
-      'bsub -J IMPC_stats_pipeline_lsf_jobs -M 16000 -e err/err',
+      'sbatch --job-name=impc_stats_pipeline_job --mem=16G --time=2-00 -e err/err',
       bf,
       1:n,
       ' -o out/out',
       bf,
       1:n,
-      ' "Rscript ExtractPVals.R ',
+      ' --wrap="Rscript ExtractPVals.R ',
       parameters,
       ' ',
       file,

Extract details from StatPackets/ExtractPValsPhenStat.R

@@ -637,13 +637,13 @@ makejobs = function(path = getwd()) {
     bf = basename(file)
     n = length(parameters)
     jobs = paste0 (
-      'bsub -J IMPC_stats_pipeline_lsf_jobs -M 16000 -e err/err',
+      'sbatch --job-name=impc_stats_pipeline_job --mem=16G --time=2-00 -e err/err',
       bf,
       1:n,
       ' -o out/out',
       bf,
       1:n,
-      ' "Rscript ExtractPVals.R ',
+      ' --wrap="Rscript ExtractPVals.R ',
       parameters,
       ' ',
       file,

IMPC annotation pipeline/UsefulCommands.txt

@@ -1,23 +1,19 @@
 # Split a text files based on #lines
 split -8500 Index_DR101_V1.txt split_index_
 
-
-# for creating LSF jobs in R
+# for creating jobs in R
 DRrequiredAgeing:::annotationIndexCreator()
 
 ### Logs
 find ./*/*_RawData/ClusterOut/ -name *ClusterOut -type f  |xargs cp --backup=numbered -t ~/NoBckDir/DR10/logs/
 find ./*/*_RawData/ClusterErr/ -name *ClusterErr -type f  |xargs cp --backup=numbered -t ~/NoBckDir/DR10/logs/
 
-
 ### Logs
 find ./*/*_RawData/ClusterOut/ -name *ClusterOut -type f  |xargs rm
 find ./*/*_RawData/ClusterErr/ -name *ClusterErr -type f  |xargs rm
 
-
 ### Merge files
 cat *.tsv >> all.tsv
 
-
-### Compress a directory
-bsub -J IMPC_stats_pipeline_lsf_jobs -q research-rh74  "tar -zcvf ~/NoBckDir/DR9.2/jo
+### Compress a directory: don't forget to specify memory and time
+sbatch --job-name=impc_stats_pipeline_job --mem=1G --time=xx:xx:xx --wrap="tar -zcvf ~/NoBckDir/DR9.2/jo"

Late adults stats pipeline/DRrequiredAgeing/DRrequiredAgeingPackage/R/main.R

@@ -51,7 +51,8 @@ mainAgeing = function(file = NULL                                    ,
                       # Only for Batch generator
                       BatchProducer          =  FALSE      ,
                       cpu = 1                              ,
-                      memory = 8000                        ,
+                      memory = "7G"                        ,
+                      time = "08:00:00"                    ,
                       nMax                   = 10000       ,
                       ChunkSize              = 24          ,
                       MinColoniesInChunks    = 32          ,
@@ -299,6 +300,7 @@ mainAgeing = function(file = NULL                                    ,
                         center               = center           ,
                         cpu                  = cpu              ,
                         memory               = memory           ,
+                        time                 = time             ,
                         extraBatchParameters =  extraBatchParameters
                       )
                       write(