Skip to content

Commit d90f054

Browse files
marinak-ebimarinak-ebi
authored
Merge pull request #67 from marinak-ebi/annotation-pipeline
Fix logs of annotation pipeline
2 parents d4a1144 + b6dc4d6 commit d90f054

File tree

1 file changed

+29
-27
lines changed
  • Late adults stats pipeline/DRrequiredAgeing/DRrequiredAgeingPackage/R

1 file changed

+29
-27
lines changed

‎Late adults stats pipeline/DRrequiredAgeing/DRrequiredAgeingPackage/R/sideFunctions.R‎

Lines changed: 29 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -4476,11 +4476,13 @@ list.dirsDepth = function(path = getwd(),
44764476

44774477
minijobsCreator = function(path = getwd(),
44784478
depth = 2,
4479-
fname = 'minijobs.txt',
4479+
fname = 'minijobs.bch',
44804480
type = '*.tsv') {
44814481
lf = list.dirsDepth(path = path, depth = depth)
44824482
a = paste0(
4483-
'sbatch --job-name=impc_stats_pipeline_job --mem=1G --time=2-00 -e error.err -o output.out --wrap="find ',
4483+
'sbatch --job-name=impc_stats_pipeline_job --mem=1G --time=2-00',
4484+
' -e ', basename(lf), '_error.err',
4485+
' -o ', basename(lf), '_output.log --wrap="find ',
44844486
lf,
44854487
' -type f -name "',
44864488
type,
@@ -4920,11 +4922,11 @@ IMPC_statspipelinePostProcess = function(SP.results = getwd(),
49204922
DRrequiredAgeing:::message0('Indexing the results ...')
49214923
setwd(file.path(SP.results))
49224924

4923-
system('rm -f minijobs.txt', wait = TRUE)
4925+
system('rm -f minijobs.bch', wait = TRUE)
49244926
system('rm -f *.Ind', wait = TRUE)
49254927
DRrequiredAgeing:::minijobsCreator()
4926-
system('chmod 775 minijobs.txt', wait = TRUE)
4927-
system('./minijobs.txt', wait = TRUE)
4928+
system('chmod 775 minijobs.bch', wait = TRUE)
4929+
system('./minijobs.bch', wait = TRUE)
49284930
DRrequiredAgeing:::waitTillCommandFinish(
49294931
WaitIfTheOutputContains = waitUntillSee,
49304932
ignoreline = ignoreThisLineInWaitingCheck
@@ -5096,15 +5098,15 @@ IMPC_annotationPostProcess = function(SP.results = getwd(),
50965098
DRrequiredAgeing:::message0('Indexing the results ...')
50975099
setwd(file.path(SP.results))
50985100

5099-
system('rm -f minijobs.txt', wait = TRUE)
5101+
system('rm -f minijobs.bch', wait = TRUE)
51005102
system('rm -f error.err', wait = TRUE)
51015103
system('rm -f output.out', wait = TRUE)
51025104
system('rm -f *.Ind', wait = TRUE)
51035105
system('rm -rf AnnotationExtractor/', wait = TRUE)
51045106

51055107
DRrequiredAgeing:::minijobsCreator()
5106-
system('chmod 775 minijobs.txt', wait = TRUE)
5107-
system('./minijobs.txt', wait = TRUE)
5108+
system('chmod 775 minijobs.bch', wait = TRUE)
5109+
system('./minijobs.bch', wait = TRUE)
51085110
DRrequiredAgeing:::waitTillCommandFinish(
51095111
WaitIfTheOutputContains = waitUntillSee,
51105112
ignoreline = ignoreThisLineInWaitingCheck
@@ -5128,7 +5130,7 @@ IMPC_annotationPostProcess = function(SP.results = getwd(),
51285130
system('split -15000 AllResultsIndeces.txt split_index_', wait = TRUE)
51295131

51305132

5131-
system('rm -f jobs.bch', wait = TRUE)
5133+
system('rm -f annotation_jobs.bch', wait = TRUE)
51325134
system('rm -f config.Rdata', wait = TRUE)
51335135
system('rm -f logs.zip' , wait = TRUE)
51345136
system('rm -f loader.R', wait = TRUE)
@@ -5145,9 +5147,9 @@ IMPC_annotationPostProcess = function(SP.results = getwd(),
51455147
pattern = 'split_index',
51465148
mem = "12G",
51475149
time = "2-00",
5148-
outputfile = 'jobs.bch'
5150+
outputfile = 'annotation_jobs.bch'
51495151
)
5150-
system('chmod 775 jobs.bch', wait = TRUE)
5152+
system('chmod 775 annotation_jobs.bch', wait = TRUE)
51515153

51525154

51535155
if (is.null(mp_chooser_file) || !file.exists(file.path(DRrequiredAgeing:::local(), 'annotation', mp_chooser_file))) {
@@ -5168,7 +5170,7 @@ IMPC_annotationPostProcess = function(SP.results = getwd(),
51685170
wait = TRUE
51695171
)
51705172

5171-
system('sbatch --job-name=impc_stats_pipeline_job --time=01:00:00 --mem=1G -o ../annotation_postprocess_job_id.txt --wrap="bash ./jobs.bch"', wait = TRUE)
5173+
system('sbatch --job-name=impc_stats_pipeline_job --time=01:00:00 --mem=1G -o ../annotation_postprocess_job_id.txt --wrap="bash ./annotation_jobs.bch"', wait = TRUE)
51725174
DRrequiredAgeing:::waitTillCommandFinish(
51735175
WaitIfTheOutputContains = waitUntillSee,
51745176
ignoreline = ignoreThisLineInWaitingCheck
@@ -6318,7 +6320,7 @@ annotationIndexCreator = function(path = getwd(),
63186320
pattern = 'split_index',
63196321
mem = "12G",
63206322
time = "2-00",
6321-
outputfile = 'jobs.bch') {
6323+
outputfile = 'annotation_jobs.bch') {
63226324
lf = list.files(
63236325
path = path,
63246326
pattern = pattern,
@@ -6453,26 +6455,26 @@ IMPC_HadoopLoad = function(SP.results = getwd(),
64536455
)
64546456

64556457
DRrequiredAgeing:::message0('Step 1: Clean ups and creating the global index for results')
6456-
DRrequiredAgeing:::message0('Zipping the logs ...')
6457-
setwd(file.path(SP.results, '..','..', 'logs'))
6458-
system(command = 'zip -rm logs.zip *.ClusterErr *.ClusterOut', wait = TRUE)
6459-
64606458
DRrequiredAgeing:::message0('Indexing the results ...')
64616459
setwd(file.path(SP.results))
64626460

6463-
system('rm -f minijobs.txt', wait = TRUE)
6461+
system('rm -f minijobs.bch', wait = TRUE)
64646462
system('rm -f error.err', wait = TRUE)
64656463
system('rm -f output.out', wait = TRUE)
64666464
system('rm -f *.Ind', wait = TRUE)
64676465
system('rm -rf AnnotationExtractorAndHadoopLoader/', wait = TRUE)
64686466

64696467
DRrequiredAgeing:::minijobsCreator()
6470-
system('chmod 775 minijobs.txt', wait = TRUE)
6471-
system('./minijobs.txt', wait = TRUE)
6468+
system('chmod 775 minijobs.bch', wait = TRUE)
6469+
DRrequiredAgeing:::submit_limit_jobs(bch_file="minijobs.bch", job_id_logfile="../../../compressed_logs/minijobs_job_id.txt")
64726470
DRrequiredAgeing:::waitTillCommandFinish(
64736471
WaitIfTheOutputContains = waitUntillSee,
64746472
ignoreline = ignoreThisLineInWaitingCheck
64756473
)
6474+
6475+
system('mv minijobs.bch ../../../compressed_logs', wait = TRUE)
6476+
system(command = "find . -type f -name '*_output.log' -exec zip -m ../../../compressed_logs/minijobs_logs.zip {} +", wait = TRUE)
6477+
system(command = "find . -type f -name '*_error.err' -exec zip -m ../../../compressed_logs/minijobs_logs.zip {} +", wait = TRUE)
64766478

64776479
DRrequiredAgeing:::message0('Moving single indeces into a separate directory called AnnotationExtractorAndHadoopLoader ...')
64786480
system('rm -rf AnnotationExtractorAndHadoopLoader/', wait = TRUE)
@@ -6484,15 +6486,15 @@ IMPC_HadoopLoad = function(SP.results = getwd(),
64846486
DRrequiredAgeing:::message0('Concating single index files to create a global index for the results ...')
64856487
system('cat *.Ind >> AllResultsIndeces.txt', wait = TRUE)
64866488

6487-
DRrequiredAgeing:::message0('zipping the single indeces ...')
6489+
DRrequiredAgeing:::message0('Zipping the single indeces ...')
64886490
system('zip -rm allsingleindeces.zip *.Ind', wait = TRUE)
64896491

64906492
system('rm -f split_index_*', wait = TRUE)
64916493
system('rm -f splits.zip', wait = TRUE)
64926494
system('split -50 AllResultsIndeces.txt split_index_', wait = TRUE)
64936495

64946496

6495-
system('rm -f jobs.bch', wait = TRUE)
6497+
system('rm -f annotation_jobs.bch', wait = TRUE)
64966498
system('rm -f config.Rdata', wait = TRUE)
64976499
system('rm -f configHadoop.Rdata', wait = TRUE)
64986500
system('rm -f logs.zip' , wait = TRUE)
@@ -6510,9 +6512,9 @@ IMPC_HadoopLoad = function(SP.results = getwd(),
65106512
pattern = 'split_index',
65116513
mem = "5G",
65126514
time = "2-00",
6513-
outputfile = 'jobs.bch'
6515+
outputfile = 'annotation_jobs.bch'
65146516
)
6515-
system('chmod 775 jobs.bch', wait = TRUE)
6517+
system('chmod 775 annotation_jobs.bch', wait = TRUE)
65166518

65176519

65186520
if (is.null(mp_chooser_file) || !file.exists(file.path(DRrequiredAgeing:::local(), 'annotation', mp_chooser_file))) {
@@ -6533,8 +6535,7 @@ IMPC_HadoopLoad = function(SP.results = getwd(),
65336535
wait = TRUE
65346536
)
65356537

6536-
system('./jobs.bch', wait = TRUE)
6537-
system('sbatch --job-name=impc_stats_pipeline_job --time=03:00:00 --mem=1G -o ../../../../compressed_logs/hadoop_load_job_id.txt --wrap="bash ./jobs.bch"', wait = TRUE)
6538+
DRrequiredAgeing:::submit_limit_jobs(bch_file="annotation_jobs.bch", job_id_logfile="../../../../compressed_logs/annotation_job_id.txt")
65386539
DRrequiredAgeing:::waitTillCommandFinish(
65396540
WaitIfTheOutputContains = waitUntillSee,
65406541
ignoreline = ignoreThisLineInWaitingCheck
@@ -6547,7 +6548,8 @@ IMPC_HadoopLoad = function(SP.results = getwd(),
65476548

65486549
DRrequiredAgeing:::message0('Zipping logs ...')
65496550
setwd(file.path(SP.results, 'AnnotationExtractorAndHadoopLoader'))
6550-
system('zip -rm logs.zip log/* err/* out/*', wait = TRUE)
6551+
system('mv annotation_jobs.bch ../../../../compressed_logs', wait = TRUE)
6552+
system('zip -rm ../../../../compressed_logs/annotation_logs.zip log/* err/* out/*', wait = TRUE)
65516553
system('zip -rm splits.zip split_index_*', wait = TRUE)
65526554

65536555
DRrequiredAgeing:::message0('Job done.')

0 commit comments

Comments
 (0)