This repository has been archived by the owner on Feb 18, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 1
/
multimd.sh
executable file
·1028 lines (832 loc) · 35.5 KB
/
multimd.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/bash
### error codes
E_SCRIPT=255
### script directory
SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
### global functions
source "${SCRIPTDIR}/global.sh" 2> /dev/null || { echo "ERROR: library file global.sh not found! Exiting"; exit ${E_SCRIPT}; }
### perform some checks
check_bash ${L2_PRINT_INT}
# print header
print_header ${L2_PRINT_INT} "Lomonosov-2 batch wrapper v${L2_MMD_VER}" "Written by Viktor Drobot"
echo
echo
# check for the rest of necessary tools
check_exec ${L2_PRINT_INT} "awk"
check_exec ${L2_PRINT_INT} "sbatch"
### usage help
if [[ "$#" -ne 2 ]]
then
echo "Usage: $0 engine taskfile"
exit ${E_MMD_POS_ARGS}
fi
### list of known keywords
KEYWORDS="DATAROOT AMBERROOT NAMDROOT GAUSSIANROOT CP2KROOT RUNTIME PARTITION NUMNODES BIN TASK"
### supported engines and their wrappers
ENG_AMBER=1
ENG_NAMD=2
ENG_GAUSSIAN=3
ENG_CP2K=4
AMBERWRAPPER="${SCRIPTDIR}/amber-wrapper.sh"
NAMDWRAPPER="${SCRIPTDIR}/namd-wrapper.sh"
GAUSSIANWRAPPER="${SCRIPTDIR}/gaussian-wrapper.sh"
CP2KWRAPPER="${SCRIPTDIR}/cp2k-wrapper.sh"
### some defaults
JOBID=$$
declare -i ENGINE
ENGINE=${ENG_AMBER}
BIN="sander"
DATAROOT=''
AMBERROOT=''
NAMDROOT=''
GAUSSIANROOT=''
CP2KROOT=''
RUNTIME='05:00'
PARTITION='test'
declare -i NUMNODES
NUMNODES=1
declare -i NUMTASKS
NUMTASKS=0
### here we will store our configurations
declare -a T_DIRS
declare -a T_BASENAMES
declare -a T_NODES
declare -a T_THREADS
declare -a T_BINS
declare -a T_CONFIGS
declare -a T_OUTPUTS
declare -a T_AMB_PRMTOPS
declare -a T_AMB_COORDS
declare -a T_AMB_RESTARTS
declare -a T_AMB_REFCS
declare -a T_AMB_TRAJS
declare -a T_AMB_VELS
declare -a T_AMB_ENS
declare -a T_AMB_INFOS
declare -a T_AMB_CPINS
declare -a T_AMB_CPOUTS
declare -a T_AMB_CPRESTRTS
declare -a T_AMB_GROUPFILES
declare -a T_AMB_NGS
declare -a T_AMB_REMS
### parse TASK keyword
task () {
declare -i idx
idx="$1"
shift
# ugly hack - rebuild positional parameters list from passed argument string
declare -a p
eval p=($@)
set -- "${p[@]}"
T_DIRS[${idx}]="$1" # store directory name for current task
T_BASENAMES[${idx}]=`basename "$1"` # get basename for all files
shift
# apply default parameters
T_NODES[${idx}]="${NUMNODES}"
T_THREADS[${idx}]=0
T_BINS[${idx}]="${BIN}"
EXT=''
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
EXT='in'
elif [[ "${ENGINE}" -eq "${ENG_NAMD}" ]]
then
EXT='conf'
elif [[ "${ENGINE}" -eq "${ENG_GAUSSIAN}" ]]
then
EXT='gin'
elif [[ "${ENGINE}" -eq "${ENG_CP2K}" ]]
then
EXT='inp'
fi
T_CONFIGS[${idx}]="${T_BASENAMES[${idx}]}.${EXT}"
T_OUTPUTS[${idx}]="${T_BASENAMES[${idx}]}.out"
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
T_AMB_PRMTOPS[${idx}]="${T_BASENAMES[${idx}]}.prmtop"
T_AMB_COORDS[${idx}]="${T_BASENAMES[${idx}]}.ncrst"
T_AMB_RESTARTS[${idx}]="${T_BASENAMES[${idx}]}.ncrst"
T_AMB_REFCS[${idx}]=""
T_AMB_TRAJS[${idx}]="${T_BASENAMES[${idx}]}.nc"
T_AMB_VELS[${idx}]=""
T_AMB_ENS[${idx}]=""
T_AMB_INFOS[${idx}]="${T_BASENAMES[${idx}]}.mdinfo"
T_AMB_CPINS[${idx}]=""
T_AMB_CPOUTS[${idx}]=""
T_AMB_CPRESTRTS[${idx}]=""
T_AMB_GROUPFILES[${idx}]=""
T_AMB_NGS[${idx}]=""
T_AMB_REMS[${idx}]=""
fi
# parse remaining positional parameters
while [[ "$#" -gt 0 ]]
do
local token="$1"
case "${token}" in
-N|--nodes)
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
if [[ "$2" -lt 1 ]]
then
echo -e "${C_RED}ERROR:${C_NC} number of nodes for the task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC} is less than 1! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_NODES[${idx}]="$2"
shift 2
;;
-T|--threads)
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
if [[ "${ENGINE}" -eq "${ENG_NAMD}" || "${ENGINE}" -eq "${ENG_GAUSSIAN}" ]]
then
echo -e "${C_RED}ERROR:${C_NC} selected engine can't be used in custom threaded mode (task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC})! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
if [[ "$2" -lt 1 ]]
then
echo -e "${C_RED}ERROR:${C_NC} number of threads for the task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC} is less than 1! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_THREADS[${idx}]="$2"
shift 2
;;
-b|--bin)
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_BINS[${idx}]="$2"
shift 2
;;
-i|--cfg)
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_CONFIGS[${idx}]="$2"
shift 2
;;
-o|--out)
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_OUTPUTS[${idx}]="$2"
shift 2
;;
# AMBER-specific options are here
-p|--prmtop)
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_AMB_PRMTOPS[${idx}]="$2"
else
echo -e "${C_RED}WARNING:${C_NC} skipping AMBER-specific parameter ${C_YELLOW}[${token}]${C_NC} in task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}" >&2
fi
shift 2
;;
-c|--inpcrd)
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_AMB_COORDS[${idx}]="$2"
else
echo -e "${C_RED}WARNING:${C_NC} skipping AMBER-specific parameter ${C_YELLOW}[${token}]${C_NC} in task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}" >&2
fi
shift 2
;;
-r|--restrt)
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_AMB_RESTARTS[${idx}]="$2"
else
echo -e "${C_RED}WARNING:${C_NC} skipping AMBER-specific parameter ${C_YELLOW}[${token}]${C_NC} in task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}" >&2
fi
shift 2
;;
-ref|--refc)
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_AMB_REFCS[${idx}]="$2"
else
echo -e "${C_RED}WARNING:${C_NC} skipping AMBER-specific parameter ${C_YELLOW}[${token}]${C_NC} in task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}" >&2
fi
shift 2
;;
-x|--mdcrd)
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_AMB_TRAJS[${idx}]="$2"
else
echo -e "${C_RED}WARNING:${C_NC} skipping AMBER-specific parameter ${C_YELLOW}[${token}]${C_NC} in task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}" >&2
fi
shift 2
;;
-v|--mdvel)
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_AMB_VELS[${idx}]="$2"
else
echo -e "${C_RED}WARNING:${C_NC} skipping AMBER-specific parameter ${C_YELLOW}[${token}]${C_NC} in task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}" >&2
fi
shift 2
;;
-e|--mden)
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_AMB_ENS[${idx}]="$2"
else
echo -e "${C_RED}WARNING:${C_NC} skipping AMBER-specific parameter ${C_YELLOW}[${token}]${C_NC} in task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}" >&2
fi
shift 2
;;
-inf|--mdinfo)
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_AMB_INFOS[${idx}]="$2"
else
echo -e "${C_RED}WARNING:${C_NC} skipping AMBER-specific parameter ${C_YELLOW}[${token}]${C_NC} in task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}" >&2
fi
shift 2
;;
-cpin)
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_AMB_CPINS[${idx}]="$2"
else
echo -e "${C_RED}WARNING:${C_NC} skipping AMBER-specific parameter ${C_YELLOW}[${token}]${C_NC} in task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}" >&2
fi
shift 2
;;
-cpout)
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_AMB_CPOUTS[${idx}]="$2"
else
echo -e "${C_RED}WARNING:${C_NC} skipping AMBER-specific parameter ${C_YELLOW}[${token}]${C_NC} in task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}" >&2
fi
shift 2
;;
-cprestrt)
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_AMB_CPRESTRTS[${idx}]="$2"
else
echo -e "${C_RED}WARNING:${C_NC} skipping AMBER-specific parameter ${C_YELLOW}[${token}]${C_NC} in task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}" >&2
fi
shift 2
;;
-groupfile)
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_AMB_GROUPFILES[${idx}]="$2"
else
echo -e "${C_RED}WARNING:${C_NC} skipping AMBER-specific parameter ${C_YELLOW}[${token}]${C_NC} in task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}" >&2
fi
shift 2
;;
-ng)
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_AMB_NGS[${idx}]="$2"
else
echo -e "${C_RED}WARNING:${C_NC} skipping AMBER-specific parameter ${C_YELLOW}[${token}]${C_NC} in task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}" >&2
fi
shift 2
;;
-rem)
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
if [[ "$#" -lt 2 ]]
then
echo -e "${C_RED}ERROR:${C_NC} parameters string is messed up for task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
T_AMB_REMS[${idx}]="$2"
else
echo -e "${C_RED}WARNING:${C_NC} skipping AMBER-specific parameter ${C_YELLOW}[${token}]${C_NC} in task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}" >&2
fi
shift 2
;;
*)
echo -e "${C_RED}WARNING:${C_NC} skipping unknown parameter ${C_YELLOW}[${token}]${C_NC} in task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC}" >&2
shift
;;
esac
done
# check for consistency between executable names and requested number of nodes/threads for AMBER engine
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
case "${T_BINS[${idx}]}" in
sander|pmemd|pmemd.cuda.MPI)
if [[ "${T_THREADS[${idx}]}" -ne 0 ]]
then
echo -e "${C_RED}ERROR:${C_NC} executable ${C_YELLOW}[${T_BINS[${idx}]}]${C_NC} can't be run in custom threaded mode (task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC})! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
if [[ "${T_NODES[${idx}]}" -ne 1 && "${T_BINS[${idx}]}" != "pmemd.cuda.MPI" ]]
then
echo -e "${C_RED}ERROR:${C_NC} executable ${C_YELLOW}[${T_BINS[${idx}]}]${C_NC} can be run only on 1 node, but requested number is ${C_YELLOW}[${T_NODES[${idx}]}]${C_NC} (task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC})! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
;;
pmemd.cuda)
if [[ "${T_NODES[${idx}]}" -ne 1 ]]
then
echo -e "${C_RED}ERROR:${C_NC} executable ${C_YELLOW}[${T_BINS[${idx}]}]${C_NC} can be run only on 1 node, but requested number is ${C_YELLOW}[${T_NODES[${idx}]}]${C_NC} (task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC})! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
;;
esac
fi
# check for consistency between executable and requested number of nodes for Gaussian engine
if [[ "${ENGINE}" -eq "${ENG_GAUSSIAN}" ]]
then
case "${T_BINS[${idx}]}" in
g03|g09|g16)
if [[ "${T_NODES[${idx}]}" -ne 1 ]]
then
echo -e "${C_RED}ERROR:${C_NC} executable ${C_YELLOW}[${T_BINS[${idx}]}]${C_NC} can be run only on 1 node, but requested number is ${C_YELLOW}[${T_NODES[${idx}]}]${C_NC} (task ${C_YELLOW}#$((idx + 1))${C_NC}, line ${C_YELLOW}#${lineno}${C_NC})! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
;;
esac
fi
}
### main script starts here
# determine which engine should we use
case "${1^^}" in
"AMBER")
ENGINE=${ENG_AMBER}
;;
"NAMD")
ENGINE=${ENG_NAMD}
;;
"GAUSSIAN")
ENGINE=${ENG_GAUSSIAN}
;;
"CP2K")
ENGINE=${ENG_CP2K}
;;
*)
echo -e "${C_RED}ERROR:${C_NC} unsupported engine ${C_YELLOW}[$1]${C_NC}! Exiting" >&2
exit ${E_MMD_UNK_ENGN}
;;
esac
# drop engine argument
shift
# some helpful variables
declare -i lineno
lineno=0
declare -i task_idx
task_idx=0
# process given taskfile
# TODO reading the whole file into while loop may cause sync errors depending on filesystem
while IFS='' read -r line || [[ -n "${line}" ]]; do
# prepare line for parsing
let lineno++
line=$(chomp "${line}")
# ignore comments and empty lines
if [[ "${line}" == \#* || -z "${line}" ]]
then
continue
fi
# get keyword from line
KEYWORD=`echo "${line}" | awk '{print $1}'`
# check if our keyword is supported
if ! echo "${KEYWORDS}" | grep -i -q -P "(^|[[:space:]])${KEYWORD}(\$|[[:space:]])"
then
echo -e "${C_RED}WARNING:${C_NC} ignoring unknown keyword ${C_YELLOW}[${KEYWORD}${C_NC}] (line ${C_YELLOW}#${lineno}${C_NC})" >&2
continue
fi
# extract remaining parameters and store needed data
PARAMS=$(chomp "`echo "${line}" | awk '{$1 = ""; print $0}'`")
case "${KEYWORD^^}" in
"DATAROOT")
DATAROOT="${PARAMS}"
;;
"AMBERROOT")
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
AMBERROOT="${PARAMS}"
else
echo -e "${C_RED}WARNING:${C_NC} ignoring AMBER-specific keyword ${C_YELLOW}[${KEYWORD}]${C_NC} (line ${C_YELLOW}#${lineno}${C_NC})" >&2
fi
;;
"NAMDROOT")
if [[ "${ENGINE}" -eq "${ENG_NAMD}" ]]
then
NAMDROOT="${PARAMS}"
else
echo -e "${C_RED}WARNING:${C_NC} ignoring NAMD-specific keyword ${C_YELLOW}[${KEYWORD}]${C_NC} (line ${C_YELLOW}#${lineno}${C_NC})" >&2
fi
;;
"GAUSSIANROOT")
if [[ "${ENGINE}" -eq "${ENG_GAUSSIAN}" ]]
then
GAUSSIANROOT="${PARAMS}"
else
echo -e "${C_RED}WARNING:${C_NC} ignoring Gaussian-specific keyword ${C_YELLOW}[${KEYWORD}]${C_NC} (line ${C_YELLOW}#${lineno}${C_NC})" >&2
fi
;;
"CP2KROOT")
if [[ "${ENGINE}" -eq "${ENG_CP2K}" ]]
then
CP2KROOT="${PARAMS}"
else
echo -e "${C_RED}WARNING:${C_NC} ignoring CP2K-specific keyword ${C_YELLOW}[${KEYWORD}]${C_NC} (line ${C_YELLOW}#${lineno}${C_NC})" >&2
fi
;;
"RUNTIME")
RUNTIME=`echo "${PARAMS}" | awk '{print $1}'`
;;
"PARTITION")
PARTITION=`echo "${PARAMS}" | awk '{print $1}'`
;;
"NUMNODES")
NUMNODES=`echo "${PARAMS}" | awk '{print $1}'`
;;
"BIN")
BIN="${PARAMS}"
;;
"TASK")
task ${task_idx} "${PARAMS}"
let task_idx++
;;
# actually we shouldn't reach that place
*)
echo "${KEYWORD} -- ${PARAMS}"
;;
esac
done < "$1"
# total number of tasks to run
declare -i NUMTASKS
NUMTASKS="${task_idx}"
# check if something wrong with given taskfile, e. g. necessary keywords are omitted or no tasks to run
if [[ -z "${DATAROOT}" || -z "${AMBERROOT}${NAMDROOT}${GAUSSIANROOT}${CP2KROOT}" || -z "${RUNTIME}" || -z "${PARTITION}" || "${NUMTASKS}" -eq 0 ]]
then
echo -e "${C_RED}ERROR:${C_NC} something wrong with taskfile (check DATAROOT, AMBERROOT/NAMDROOT/GAUSSIANROOT/CP2KROOT, RUNTIME, PARTITION directives and the number of tasks given)! Exiting" >&2
exit ${E_MMD_INV_CONF}
fi
# print short summary about requested job and prepare command lines
echo -e "${C_BLUE}===========${C_NC}"
echo -e "${C_BLUE}JOB SUMMARY${C_NC}"
echo -e "${C_BLUE}===========${C_NC}"
echo -e "Base data directory is ${C_YELLOW}[${DATAROOT}]${C_NC}"
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
echo -e "Will use AMBER engine. AMBER is installed into ${C_YELLOW}[${AMBERROOT}]${C_NC}"
elif [[ "${ENGINE}" -eq "${ENG_NAMD}" ]]
then
echo -e "Will use NAMD engine. NAMD is installed into ${C_YELLOW}[${NAMDROOT}]${C_NC}"
elif [[ "${ENGINE}" -eq "${ENG_GAUSSIAN}" ]]
then
echo -e "Will use Gaussian engine. Gaussian is installed into ${C_YELLOW}[${GAUSSIANROOT}]${C_NC}"
elif [[ "${ENGINE}" -eq "${ENG_CP2K}" ]]
then
echo -e "Will use CP2K engine. CP2K is installed into ${C_YELLOW}[${CP2KROOT}]${C_NC}"
fi
echo -e "Time limit for the whole job is ${C_YELLOW}[${RUNTIME}]${C_NC}"
echo -e "We will use ${C_YELLOW}[${PARTITION}]${C_NC} partition to run our tasks"
echo -e "One task will consume ${C_YELLOW}[${NUMNODES}]${C_NC} nodes by default"
echo -e "Default executable is ${C_YELLOW}[${BIN}]${C_NC}"
echo -e "Will run ${C_YELLOW}[${NUMTASKS}]${C_NC} tasks"
echo -e "Internal job ID is ${C_YELLOW}[${JOBID}]${C_NC}"
echo
echo
echo -e "${C_BLUE}===========================${C_NC}"
echo -e "${C_BLUE}TASKS CONFIGURATION DETAILS${C_NC}"
echo -e "${C_BLUE}===========================${C_NC}"
# error counter
declare -i NUMERRORS
NUMERRORS=0
# total number of nodes to be requested
declare -i TOTALNODES
TOTALNODES=0
# file with final list of directories to be processed
RUNLIST="${DATAROOT}/runlist.${JOBID}"
:> "${RUNLIST}"
source "${SCRIPTDIR}/partitions.sh" 2> /dev/null || { echo "ERROR: library file partitions.sh not found! Exiting"; exit ${E_SCRIPT}; }
for ((task_idx=0; task_idx < NUMTASKS; task_idx++))
do
# recalculate number of nodes for special cases
# partition parameters imported above
if [[ ("${T_THREADS[${task_idx}]}" -ne 0) && ("${ENGINE}" -eq "${ENG_AMBER}") ]]
then
declare -i NUMTHREADS
NUMTHREADS=${T_THREADS[${task_idx}]}
case "${T_BINS[${task_idx}]}" in
sander.MPI|pmemd.MPI)
T_NODES[${task_idx}]=$((1 + (NUMTHREADS - 1) / NUMCORES))
;;
pmemd.cuda)
T_NODES[${task_idx}]=1
if [[ "${NUMTHREADS}" -gt "${NUMGPUS}" ]]
then
echo -e "${C_RED}ERROR:${C_NC} Unsupported thread configuration! You asked more tasks per node then there are GPUs available! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
if [[ "${task_idx}" > 0 ]]
then
if [[ ${T_THREADS[${task_idx}]} -ne ${T_THREADS[$((task_idx-1))]} ]]
then
echo -e "${C_RED}ERROR:${C_NC} Unsupported thread configuration! All threading configuration must be the same! Exiting" >&2
exit ${E_MMD_INV_TASK}
fi
fi
;;
*)
;;
esac
fi
echo -e "${C_PURPLE}>> Task #$((task_idx + 1)) <<${C_NC}"
echo -e "Data directory is ${C_YELLOW}[${T_DIRS[${task_idx}]}]${C_NC}"
echo -e "Will use ${C_YELLOW}[${T_NODES[${task_idx}]}]${C_NC} nodes"
if [[ "${T_THREADS[${task_idx}]}" -ne 0 ]]
then
echo -e "Will run with ${C_YELLOW}[${T_THREADS[${task_idx}]}]${C_NC} threads"
fi
echo -e "Executable binary is ${C_YELLOW}[${T_BINS[${task_idx}]}]${C_NC}"
echo -e "Config file is ${C_YELLOW}[${T_CONFIGS[${task_idx}]}]${C_NC}"
echo -e "Output file is ${C_YELLOW}[${T_OUTPUTS[${task_idx}]}]${C_NC}"
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
echo -e "Topology file is ${C_YELLOW}[${T_AMB_PRMTOPS[${task_idx}]}]${C_NC}"
echo -e "Start coordinates are in file ${C_YELLOW}[${T_AMB_COORDS[${task_idx}]}]${C_NC}"
echo -e "Restart will be written to file ${C_YELLOW}[${T_AMB_RESTARTS[${task_idx}]}]${C_NC}"
if [[ -n "${T_AMB_REFCS[${task_idx}]}" ]]
then
echo -e "Positional restraints are in file ${C_YELLOW}[${T_AMB_REFCS[${task_idx}]}]${C_NC}"
fi
echo -e "Trajectories will be written to file ${C_YELLOW}[${T_AMB_TRAJS[${task_idx}]}]${C_NC}"
if [[ -n "${T_AMB_VELS[${task_idx}]}" ]]
then
echo -e "Velocities will be written to file ${C_YELLOW}[${T_AMB_VELS[${task_idx}]}]${C_NC}"
fi
if [[ -n "${T_AMB_ENS[${task_idx}]}" ]]
then
echo -e "Energies will be written to file ${C_YELLOW}[${T_AMB_ENS[${task_idx}]}]${C_NC}"
fi
echo -e "MD information will be available in file ${C_YELLOW}[${T_AMB_INFOS[${task_idx}]}]${C_NC}"
if [[ -n "${T_AMB_CPINS[${task_idx}]}" ]]
then
echo -e "Protonation states are in file ${C_YELLOW}[${T_AMB_CPINS[${task_idx}]}]${C_NC}"
fi
if [[ -n "${T_AMB_CPOUTS[${task_idx}]}" ]]
then
echo -e "Protonation states will be written to file ${C_YELLOW}[${T_AMB_CPOUTS[${task_idx}]}]${C_NC}"
fi
if [[ -n "${T_AMB_CPRESTRTS[${task_idx}]}" ]]
then
echo -e "Protonation states for restart will be written to file ${C_YELLOW}[${T_AMB_CPRESTRTS[${task_idx}]}]${C_NC}"
fi
if [[ -n "${T_AMB_GROUPFILES[${task_idx}]}" ]]
then
echo -e "Reference groupfile is ${C_YELLOW}[${T_AMB_GROUPFILES[${task_idx}]}]${C_NC}"
fi
if [[ -n "${T_AMB_NGS[${task_idx}]}" ]]
then
echo -e "Number of replicas is ${C_YELLOW}[${T_AMB_NGS[${task_idx}]}]${C_NC}"
fi
if [[ -n "${T_AMB_REMS[${task_idx}]}" ]]
then
echo -e "Replica exchange type is ${C_YELLOW}[${T_AMB_REMS[${task_idx}]}]${C_NC}"
fi
if [[ "${T_AMB_COORDS[${task_idx}]}" == "${T_AMB_RESTARTS[${task_idx}]}" ]]
then
echo -e "${C_RED}WARNING:${C_NC} coordinates and restart files are the same! Original coordinates will be overwritten!" >&2
fi
fi
echo -e "${C_BLUE}------${C_NC}"
echo -n -e "Trying to save prepared command to ${C_YELLOW}[${DATAROOT%/}/${T_DIRS[${task_idx}]}/runcmd.${JOBID}]${C_NC}... "
# now we'll build final execution line...
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
COMMAND="\"${AMBERROOT}/bin/${T_BINS[${task_idx}]}\" -O -i \"${T_CONFIGS[${task_idx}]}\" -o \"${T_OUTPUTS[${task_idx}]}\" -p \"${T_AMB_PRMTOPS[${task_idx}]}\" -c \"${T_AMB_COORDS[${task_idx}]}\" -r \"${T_AMB_RESTARTS[${task_idx}]}\" -x \"${T_AMB_TRAJS[${task_idx}]}\" -inf \"${T_AMB_INFOS[${task_idx}]}\""
if [[ -n "${T_AMB_REFCS[${task_idx}]}" ]]
then
COMMAND="${COMMAND} -ref \"${T_AMB_REFCS[${task_idx}]}\""
fi
if [[ -n "${T_AMB_VELS[${task_idx}]}" ]]
then
COMMAND="${COMMAND} -v \"${T_AMB_VELS[${task_idx}]}\""
fi
if [[ -n "${T_AMB_ENS[${task_idx}]}" ]]
then
COMMAND="${COMMAND} -e \"${T_AMB_ENS[${task_idx}]}\""
fi
if [[ -n "${T_AMB_CPINS[${task_idx}]}" ]]
then
COMMAND="${COMMAND} -cpin \"${T_AMB_CPINS[${task_idx}]}\""
fi
if [[ -n "${T_AMB_CPOUTS[${task_idx}]}" ]]
then
COMMAND="${COMMAND} -cpout \"${T_AMB_CPOUTS[${task_idx}]}\""
fi
if [[ -n "${T_AMB_CPRESTRTS[${task_idx}]}" ]]
then
COMMAND="${COMMAND} -cprestrt \"${T_AMB_CPRESTRTS[${task_idx}]}\""
fi
if [[ -n "${T_AMB_GROUPFILES[${task_idx}]}" ]]
then
COMMAND="${COMMAND} -groupfile \"${T_AMB_GROUPFILES[${task_idx}]}\""
fi
if [[ -n "${T_AMB_NGS[${task_idx}]}" ]]
then
COMMAND="${COMMAND} -ng \"${T_AMB_NGS[${task_idx}]}\""
fi
if [[ -n "${T_AMB_REMS[${task_idx}]}" ]]
then
COMMAND="${COMMAND} -rem \"${T_AMB_REMS[${task_idx}]}\""
fi
elif [[ "${ENGINE}" -eq "${ENG_NAMD}" ]]
then
COMMAND="\"${NAMDROOT}/namd-runscript.sh\" \"${NAMDROOT}/${T_BINS[${task_idx}]}\" +isomalloc_sync +idlepoll \"${T_CONFIGS[${task_idx}]}\" > \"${T_OUTPUTS[${task_idx}]}\""
elif [[ "${ENGINE}" -eq "${ENG_GAUSSIAN}" ]]
then
COMMAND="\"${GAUSSIANROOT}/${T_BINS[${task_idx}]}/${T_BINS[${task_idx}]}\" < \"${T_CONFIGS[${task_idx}]}\" > \"${T_OUTPUTS[${task_idx}]}\""
elif [[ "${ENGINE}" -eq "${ENG_CP2K}" ]]
then
COMMAND="\"${CP2KROOT}/${T_BINS[${task_idx}]}\" -o \"${T_OUTPUTS[${task_idx}]}\" \"${T_CONFIGS[${task_idx}]}\""
fi
# ...and store it in appropriate place
echo "${COMMAND}" 2> /dev/null > "${DATAROOT%/}/${T_DIRS[${task_idx}]}/runcmd.${JOBID}"
if [[ "$?" -eq 0 ]]
then
# add number of nodes, threads and data directory for that task to runlist and increment total nodes counter
echo "${T_NODES[${task_idx}]} ${T_THREADS[${task_idx}]} ${DATAROOT%/}/${T_DIRS[${task_idx}]}" >> "${RUNLIST}"
let "TOTALNODES += ${T_NODES[${task_idx}]}"
echo -e "${C_GREEN}ok${C_NC}"
else
echo -e "${C_RED}fail${C_NC}"
let NUMERRORS++
fi
echo
done
# Special case - pmemd.cuda on multi-GPU node
if [[ ("${ENGINE}" -eq "${ENG_AMBER}") && ("${T_BINS[0]}" == "pmemd.cuda") ]]
then
ALL_TASKS_PMEMD_CUDA=true
for TASK_BIN in "${T_BINS[@]}"
do
if [[ "${TASK_BIN}" != "pmemd.cuda" ]]
then
ALL_TASKS_PMEMD_CUDA=false
echo -e "${C_RED}WARNING:${C_NC} Detected pmemd.cuda in combination with ${TASK_BIN}. Cannot distribute between GPUs"
break
fi
done
fi
if [[ ("${ENGINE}" -eq "${ENG_AMBER}") && ("${ALL_TASKS_PMEMD_CUDA}" == true) && ("${NUMGPUS}" -gt 1) ]]
then
echo -e "Detected pmemd.cuda on multi-GPU partition. Distributing tasks to increase efficency"
declare -i NUM_TASKS_PER_NODE
declare -i TASKS_REMAINDER
declare -i TOTAL_MULTITASK_NODES
# We already checked that NUMTHREADS <= NUMGPUS
if [[ "${T_THREADS[0]}" -ne 0 ]]
then
echo -e "You requested custom number of tasks per node: ${C_YELLOW}[${T_THREADS[0]}]${C_NC}"
NUM_TASKS_PER_NODE="${T_THREADS[0]}"
else
NUM_TASKS_PER_NODE=NUMGPUS
fi
let "TOTAL_MULTITASK_NODES = NUMTASKS / NUM_TASKS_PER_NODE"
let "TASKS_REMAINDER = NUMTASKS % NUM_TASKS_PER_NODE"
echo -e "Placed ${C_YELLOW}[${NUM_TASKS_PER_NODE}]${C_NC} tasks per node"
if [[ "${TASKS_REMAINDER}" -gt 0 ]]
then
let "TOTAL_MULTITASK_NODES += 1"
echo -e "${C_RED}WARNING:${C_NC} Inefficient configuration: you have unused GPUs. Provide number of tasks divisible by number of GPUs per node" >&2
fi
if [[ "${TOTAL_MULTITASK_NODES}" -le "${TOTALNODES}" ]]
then
echo -e "Will use ${C_YELLOW}[${TOTAL_MULTITASK_NODES}]${C_NC} nodes in total"
TOTALNODES=${TOTAL_MULTITASK_NODES}
else
echo -e "${C_RED}ERROR:${C_NC} Cannot distribute pmemd.cuda tasks between multi-GPU nodes! Exiting" >&2
exit ${E_MMD_INV_CONF}
fi
fi
# prepare SLURM command
WRAPPER=''
if [[ "${ENGINE}" -eq "${ENG_AMBER}" ]]
then
WRAPPER="${AMBERWRAPPER}"
elif [[ "${ENGINE}" -eq "${ENG_NAMD}" ]]
then
WRAPPER="${NAMDWRAPPER}"
elif [[ "${ENGINE}" -eq "${ENG_GAUSSIAN}" ]]
then
WRAPPER="${GAUSSIANWRAPPER}"
elif [[ "${ENGINE}" -eq "${ENG_CP2K}" ]]
then
WRAPPER="${CP2KWRAPPER}"
fi
# we should enclose paths in quotes to protect ourself from space-containing parameters
CMD="sbatch -N ${TOTALNODES} -p ${PARTITION} -t ${RUNTIME} ${WRAPPER} ${JOBID} ${RUNTIME} ${PARTITION} $((NUMTASKS - NUMERRORS)) \"${L2_ROOT}\" \"${DATAROOT}\""
# give user the last chance to check for possible errors and exit if none of the tasks have been prepared successfully
echo
echo
if [[ "${NUMERRORS}" -eq "${NUMTASKS}" ]]
then
echo -e "${C_RED}ERROR:${C_NC} none of the requested tasks have been prepared successfully! Please re-check your config. Exiting" >&2
exit ${E_MMD_PREP_FAIL}
fi
C_TMP_TYPE="${C_GREEN}"
if [[ "${NUMERRORS}" -gt 0 ]]
then
echo -e "${C_RED}WARNING:${C_NC} there was some problems with ${C_YELLOW}${NUMERRORS}${C_NC} task(s). Please review job summary and your config with extra attention." >&2
C_TMP_TYPE="${C_RED}"
fi