-
Notifications
You must be signed in to change notification settings - Fork 2
Expand file tree
/
Copy pathcompile.sh
More file actions
1415 lines (1277 loc) · 63.2 KB
/
compile.sh
File metadata and controls
1415 lines (1277 loc) · 63.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# define machine
# 0 : local (Mac)
# 1 : local
# 2 : Marconi A1
# 3 : VSC3 (VSC)
# 4 : Vesta (ANL)
# 5 : Marconi A2 KNL
# 6 : Theta (ANL)
# 7 : Bridges (PSC)
# 8 : Adelaide
# 9 : VSC4 (VSC)
#10 : Joliot Curie - Irene (KNL)
#11 : Davide (CINECA)
#12 : HAWK (HLRS - AMD Epyc Rome)
#13 : G100 (CINECA)
#14 : Tersicore o Zonker (Uniud)
#15 : M100 (CINECA) - no GPU
#16 : M100 (CINECA) - GPU (openACC)
#17 : Discoverer (Sofiatech)
#18 : LUMI-C (CSC)
#19 : VSC-5 (VSC) - CPU Partition
#20 : VSC-5 (VSC) - GPU Partition
#21 : Leonardo - CPU Partition
#22 : Leoanrdo - GPU (Booster)
#23 : Marenostrum5 - CPU Partition (GPP)
machine="1"
echo ""
echo "=============================================================================="
echo "= Running on ="
if [ "$machine" == "0" ]; then
echo "= Local machine (Mac) ="
cp ./Local_Mac/makefile ./makefile
cp ./Local_Mac/go.sh ./go.sh
# save recovery files in modal space (1) or physical space (0)
savespectral="1"
openacc_flag="0"
elif [ "$machine" == "1" ]; then
# assume openMPI is employed
echo "= Local machine ="
cp ./Local/makefile ./makefile
cp ./Local/go.sh ./go.sh
savespectral="1"
openacc_flag="0"
elif [ "$machine" == "2" ]; then
echo "= Marconi A1 Broadwell ="
cp ./Marconi/makefile ./makefile
cp ./Marconi/go.sh ./go.sh
module purge
module load profile/global
# load modules
module load intel/pe-xe-2017--binary
module load intelmpi/2017--binary
module load fftw/3.3.5--intelmpi--2017--binary
# or
#module load gnu/6.1.0
#module load openmpi/1-10.3--gnu--6.1.0
#module load fftw/3.3.4--openmpi--1-10.3--gnu--6.1.0
savespectral="0"
openacc_flag="0"
elif [ "$machine" == "3" ]; then
echo "= VSC-3 ="
cp ./VSC-3/makefile ./makefile
cp ./VSC-3/go.sh ./go.sh
module purge
# load modules
module load intel/16 intel-mpi/5.1.3 fftw/3.3.4-DP
# or (but problem with .mod files)
#module load gcc/5.3 intel-mpi/5.1.3 fftw/3.3.4-DP
savespectral="0"
openacc_flag="0"
elif [ "$machine" == "4" ]; then
echo "= Vesta ="
cp ./Vesta/makefile ./makefile
cp ./Vesta/go.sh ./go.sh
savespectral="1"
openacc_flag="0"
elif [ "$machine" == "5" ]; then
echo "= Marconi A2 KNL ="
module purge
module load env-knl
module load profile/global
# load modules
module load intel/pe-xe-2017--binary
module load intelmpi/2017--binary
module load fftw/3.3.5--intelmpi--2017--binary
cp ./Marconi_KNL/makefile ./makefile
cp ./Marconi_KNL/go.sh ./go.sh
savespectral="0"
openacc_flag="0"
elif [ "$machine" == "6" ]; then
echo "= Theta ="
# load modules
#module load gcc
module load fftw
module load craype-hugepages16M
cp ./Theta/makefile ./makefile
cp ./Theta/go.sh ./go.sh
savespectral="0"
openacc_flag="0"
elif [ "$machine" == "7" ]; then
echo "= Bridges ="
module purge
# load modules
module load pgi/19.4
module load mpi/pgi_openmpi/19.4
#module load intel
#module load mpi/intel_mpi
module load fftw3/3.3.4
cp ./Bridges/makefile ./makefile
cp ./Bridges/go.sh ./go.sh
savespectral="0"
openacc_flag="0"
elif [ "$machine" == "8" ]; then
echo "= Adelaide ="
module purge
# load modules
# IMPORTANT: if OpenMPI line 946 must be replaced with: if [[ "$machine" == "7" || "$machine" == "8"]]; then
# Intel version
module load intel
module load mpich
# GCC version
#module load gcc
#module load #some version of MPI/OpenMPI for gcc
module load FFTW
# change to makefile_intel/makefile_gnu according to needs
cp ./Adelaide/makefile_intel ./makefile
cp ./Adelaide/go.sh ./go.sh
savespectral="0"
openacc_flag="0"
elif [ "$machine" == "9" ]; then
echo "= VSC-4 ="
#GCC version (openMPI is used, check MPI_GET_ADDRESS LINES)
#cp ./VSC-4/makefile_gcc ./makefile
#cp ./VSC-4/go_gcc.sh ./go.sh
#module purge
#module load gcc
#module load fftw
#Intel version (FFTW)
cp ./VSC-4/makefile_intel ./makefile
cp ./VSC-4/go_intel.sh ./go.sh
module purge
module load intel/19.1.3
module load intel-mpi/2019.10.317-intel-19.1.3.304-x276qb5
module load fftw
module list
savespectral="0"
openacc_flag="0"
elif [ "$machine" == "10" ]; then
echo "= Joliot-Curie (Irene-KNL) ="
module purge
module load mpi
module load fftw3
cp ./Irene_KNL/makefile ./makefile
cp ./Irene_KNL/go.sh ./go.sh
savespectral="0"
openacc_flag="0"
elif [ "$machine" == "11" ]; then
echo "= Davide (no GPU) ="
module purge
# load modules
module load gnu
module load openmpi
module load fftw
cp ./Davide/makefile ./makefile
cp ./Davide/go.sh ./go.sh
savespectral="0"
openacc_flag="0"
elif [ "$machine" == "12" ]; then
echo "= HAWK Epyc Rome ="
module purge
# load modules
module load gcc/9.2.0
module load openmpi/4.0.4
# check aocl for AMD-optimized FFTW, see https://kb.hlrs.de/platforms/index.php/Libraries(Hawk)
module load fftw/3.3.8
cp ./HAWK/makefile ./makefile
cp ./HAWK/go.sh ./go.sh
savespectral="0"
openacc_flag="0"
elif [ "$machine" == "13" ]; then
echo "= Galileo-100 ="
module purge
# load modules
module load profile/advanced
module load gnu
# intel-mpi
module load intel/oneapi-2021--binary
module load intelmpi/oneapi-2021--binary
module load fftw/3.3.9--intelmpi--oneapi-2021--binary
cp ./Galileo_100/makefile ./makefile
cp ./Galileo_100/go.sh ./go.sh
savespectral="0"
openacc_flag="0"
elif [ "$machine" == "14" ]; then
echo "= Tersicore (2 x GPUs) ="
#must compile with nvfortran a
cp ./Tersicore_gpu/makefile ./makefile
cp ./Tersicore_gpu/go.sh ./go.sh
#Setup the environment (otherwise gfortran is used)
NVARCH=Linux_x86_64; export NVARCH
NVCOMPILERS=/opt/nvidia/hpc_sdk; export NVCOMPILERS
MANPATH=$MANPATH:$NVCOMPILERS/$NVARCH/25.7/compilers/man; export MANPATH
PATH=$NVCOMPILERS/$NVARCH/25.7/compilers/bin:$PATH; export PATH
export PATH=$NVCOMPILERS/$NVARCH/25.7/comm_libs/mpi/bin:$PATH
export MANPATH=$MANPATH:$NVCOMPILERS/$NVARCH/25.7/comm_libs/mpi/man
savespectral="0"
openacc_flag="1"
elif [ "$machine" == "15" ]; then
echo "= Marconi-100 (CPU only) ="
module purge
# load modules
module load profile/advanced
module load gnu
module load cuda/10.1
#module load openmpi/4.0.3--gnu--8.4.0
#module load fftw/3.3.8--gnu--8.4.0
# MPI spectrum implementation (please swap also go.sh and makefile)
module load spectrum_mpi/10.3.1--binary
module load fftw/3.3.8--spectrum_mpi--10.3.1--binary
cp ./Marconi_100/makefile_cpu_spectrum ./makefile
cp ./Marconi_100/go_cpu_spectrum.sh ./go.sh
savespectral="0"
openacc_flag="0"
elif [ "$machine" == "16" ]; then
echo "= Marconi-100 (GPU) ="
module purge
# HPC-SDK + OPENMPI (slower)
# module load hpc-sdk/2023--binary
# HPC-SDK + MPI SPECTRUM (faster, M100 optimized)
module load hpc-sdk/2023--binary
module load spectrum_mpi/10.4.0--binary
#cp ./Marconi_100/makefile_gpu_openmpi ./makefile
#cp ./Marconi_100/go_gpu_openmpi.sh ./go.sh
cp ./Marconi_100/makefile_gpu_spectrum ./makefile
cp ./Marconi_100/go_gpu_spectrum.sh ./go.sh
savespectral="0"
openacc_flag="1"
elif [ "$machine" == "17" ]; then
echo "= Discoverer (Sofiatech) ="
echo "= Do not run in home ! ="
# ---- Gcc + mpich + FFTW (CPU) ----
#module load gcc/11/latest
#module load mpich/3/gcc/latest
#module load fftw/3/latest-gcc-mpich
#cp ./Discoverer/makefile_gcc ./makefile
#cp ./Discoverer/go_gcc.sh ./go.sh
# ---- Nvidia HPC-SDK ----
module load nvidia
module load nvhpc-nompi/latest
module load gcc/11/latest
module load openmpi/4/nvidia/latest
module load fftw/3/3.3.10-nvidia-openmpi
cp ./Discoverer/makefile_nv ./makefile
cp ./Discoverer/go_nv.sh ./go.sh
# ---- AMD AOCC ="
#module purge
#module load amd/aocc/3/latest
#module load openmpi/4/aocc/latest
#module load fftw/3/3.3.10-aocc-openmpi
#cp ./Discoverer/makefile_amd ./makefile
#cp ./Discoverer/go_amd.sh ./go.sh
# ---- Intel (problems with MPI_Open) ----
#module load intel compiler-rt/latest
#module load compiler/latest
#module load mpi/latest
#module load mkl/latest
#module load fftw/3/latest-intelmpi
#cp ./Discoverer/makefile_intel ./makefile
#cp ./Discoverer/go_intel.sh ./go.sh
savespectral="0"
openacc_flag="0"
elif [ "$machine" == "18" ]; then
echo "= LUMI-C (CSC) ="
module --force purge
# Cray version (problem with more than 1 node)
# module load PrgEnv-cray
# module load craype-x86-milan
# module load cray-fftw
# GNU version
module load PrgEnv-gnu
module load craype-x86-milan
module load cray-fftw
module list
cp ./Lumic/makefile_gnu ./makefile
cp ./Lumic/go_gnu.sh ./go.sh
savespectral="0"
openacc_flag="0"
elif [ "$machine" == "19" ]; then
echo "= VSC-5 (CPU) ="
# load modules (SPACK)
spack unload --all
spack unload --all
spack load gcc@12.2.0
spack load mpich@4.1.1
spack load /xgvooar
cp ./VSC-5/makefile_cpu ./makefile
cp ./VSC-5/go_cpu.sh ./go.sh
savespectral="0"
openacc_flag="0"
elif [ "$machine" == "20" ]; then
echo "= VSC-5 (GPU) ="
# load modules (SPACK)
spack unload --all
spack load nvhpc
cp ./VSC-5/makefile_gpu ./makefile
cp ./VSC-5/go_gpu.sh ./go.sh
savespectral="0"
openacc_flag="1"
elif [ "$machine" == "21" ]; then
echo "= Leonardo (CPU) ="
module purge
module load gcc/11.3.0
module load openmpi/4.1.4--gcc--11.3.0-cuda-11.8
module load fftw/3.3.10--openmpi--4.1.4--gcc--11.3.0
cp ./Leonardo/makefile_cpu ./makefile
cp ./Leonardo/go_cpu.sh ./go.sh
savespectral="0"
openacc_flag="0"
elif [ "$machine" == "22" ]; then
echo "= Leonardo (GPU) ="
module purge
module load nvhpc/24.3
module load openmpi/4.1.6--nvhpc--24.3
module list
cp ./Leonardo/makefile_gpu ./makefile
cp ./Leonardo/go_gpu.sh ./go.sh
cp ./Leonardo/binder.sh ./binder.sh
cp ./Leonardo/go_binder.sh ./go_bind.sh
savespectral="0"
openacc_flag="1"
elif [ "$machine" == "23" ]; then
echo "= Marenostrum5 (CPU) ="
module purge
module load gcc/13.2.0
module load openmpi/4.1.5-gcc
module load fftw/3.3.10-gcc-ompi
cp ./Marenostrum_5/makefile_cpu ./makefile
cp ./Marenostrum_5/go_cpu.sh ./go.sh
savespectral="0"
openacc_flag="0"
fi
echo "=============================================================================="
echo ""
################################################################################
# define simulation parameters
# FFTW plan craation flag (only for CPU - FFTW library, ignored when GPUs are used)
# 0: FFTW_ESTIMATE, faster plan creation, transforms may be slower
# 1: FFTW_PATIENT, try several algorithms, choose the best one, slower plan creation, transforms may be faster
fftw_flag="0"
# PAY ATTENTION TO VARIABLE TIPE #
# number of grid points (edit only exponent)
ix="8" # integer
iy="8" # integer
iz="8" # integer
# dual grid for surfactant, expansion factors:
exp_x="1" # integer, (2**ix)*exp_x
exp_y="1" # integer, (2**iy)*exp_y
exp_z="1" # integer, (2**iz)*exp_z+1
# parallelization strategy
NYCPU="1" # integer
NZCPU="4" # integer
# running on single shared memory environment (0) or on many (1)
multinode="0" # integer
# number of MPI processes per node
nodesize="68" # integer
# REMARKS on GPUs and Acceleration strategy
# On Machines 14, 16 and 20 GPUs are used by default (openacc_flag=1).
# OpenACC directives are used to accelerate the code w/ GPUs
################################################################################
# restart flag: 1 restart, 0 new simulation
restart="0" # integer
nt_restart="0" # integer
# initial condition
# 0 : initialize zero velocity
# 1 : laminar Poiseuille flow in x direction
# 2 : laminar Poiseuille flow in y direction
# 3 : read input from file (parallel read)
# 4 : read input from file (serial read)
# 5 : shear flow y direction
# 6 : shear flow x direction
# always keep list of initial conditions updated
incond="0" # integer
# Reynolds number
Re="150.0" # real (double)
# Courant number
Co="0.2" # real (double)
# mean pressure gradient (x and y), defined ad (p_out-p_in)/L
gradpx="-1.0" # real (double)
gradpy="0.0" # real (double)
# Constant power input approach (adaptive gradpx)
cpi_flag="0" #if activated, gradpx should be set to -1.
repow="100.0" #B*Re_pi - re used to control the pressure gradient
# domain size, divided by pi (z size is always 2, between -1 and 1)
lx="4.0" # real (double)
ly="2.0" # real (double)
# initial time step
nstart="0" # integer
# final time step
nend="10" #integer (up to 8 digits)
# frequency of solution saving in physical space
dump="500" # integer
# frequency of solution saving in spectral space
sdump="-1" # integer
# frequency of solution saving in spectral space, needed to restart the simulation
# from a checkpoint if it crashes (those files are not kept)
failure_dump="10" # integer
# Run time statistics calculation parameters
# frequency of statistics calculation (leave -1 to skip statistics calculation)
st_dump="-1" # integer
# timestep from which begin statistics calculation
start_stats="0" # integer
# flag for mean, rms, skewness, flatness calculation
mean_flag="0" # 0 to skip mean, rms, skewness and flatness calculation, 1 to do it
# flag for budget calculation
budget_flag="0" # 0 to skip budget calculation, 1 to do it
# flag for power spectra calculation
spectra_flag="0" # 0 to skip power spectra calculation, 1 to do it
# dt
dt="0.5e-4" # real (exponential)
# 0: no-slip
# 1: free-slip
# 2: y shear flow (+1 at z=1, -1 at z=-1)
# 3: x shear flow (+1 at z=1, -1 at z=-1)
# boundary condition, z=1
bc_upb="0" # integer
# boundary condition, z=-1
bc_lb="0" # integer
################################################################################
# Phase field only
# phase field flag, 0: phase field deactivated, 1: phase field activated
phi_flag="0" # integer
# Models selection for phi, different formulations used to improve mass conservation
# Models 0-6 Cahn-Hilliard is solved (4th order), Model 7-8 Allen-Cahn is solved (2nd order)
# Models Summary
# 0: OFF
# 1: profile-corrected
# 2: flux-corrected
# 3: profile-corrected turned off at the walls
# 4: profile-corrected kill the gradients (filter on gradients lower than threshold 1/(50*Ch)
# 5: flux-corrected kill the gradients (filter on gradients lower than threshold 1/(50*Ch)
# 6: Kwakkel model (A redefined energy functional to prevent mass loss in phase-field methods) Work in progess, do not use in production
# 7: Conservative Allen-Cahn, Second-order phase-field model (Mirjalili), read the respective article before using it, pe and ch have very different meanings.!!
# 8: Conservative Allen-Cahn, Second-order phase-field model (Jain), read the respective article before using it, pe and ch have very different meanings.!!
phicor_flag="0" # integer
# Value of the parameter lambda used to correct the phi profile (only for phicor_flag=1,2,3,4,5)
# Lam=0.0625/Ch (check for spotting at the wall when using refined grids, otherwise relax lambda)
lamcorphi="2.5" # real (double)
# matched densities: 1 for matched densities, 0 for rhor < 1, 2 for rhor > 1
matchedrho="1" # integer
# density ratio, phase +1 over phase -1
rhor="1.0" # real (double)
#matched dynamic viscosities: 1 for matched viscosities, 0 for visr < 1 (or non-newtonian), 2 for visr > 1
matchedvis="1" # integer
# dynamic viscosity ratio, phase +1 over phase -1 (not considered when non-newtonian is enabled)
visr="1.0" # real (double)
#non-newtonian phase, 0 deactivaed, 1 phase=+1 is non-newtonian (Carreau model)
non_newtonian="0" # integer
#Ratio between the viscosity at zero and infinity shear rate (Non-newtonian-Carreau model)
muinfmuzero="0.1"
#Exponent for the non-newtonian fluid (phi=+1)-n<1 pseudoplastic
exp_non_new="0.9"
# Weber number
We="1.0" # real (double)
# Cahn number
Ch="0.02" # real (double)
# Peclet number
Pe="150.0" # real (double)
# Froud number
Fr="0.1" # real (double)
# Body force flag, 0: deactivated, 1: activated
body_flag="0" # integer
# Body force coefficient
Bd="4.0" # real (double)
# Body force direction
# 1: positive x direction
# -1: negative x direction
# 2: positive z direction
# -2: negative z direction
# 3: positive y direction
# -3: negative y direction
bodydir="2" # integer
# S-Shaped pressure gradient force (counter-current), 0: deactivated, 1: activated
sgradp_flag="0" # integer
# Body force direction
# 1: phi=+1 along x+ and phi=-1 x-.
# -1: phi=+1 along x- and phi=-1 x+..
# 3: phi=+1 along y+ and phi=-1 y-.
# -3: phi=+1 along y- and phi=-1 y+..
sgradpdir="1" # integer
# electric force flag, 0: deactivated, 1: activated
ele_flag="0" # integer
# Stuart number
stuart="1.0" # real (double)
# initial conditions on phi
# 0: only phase -1
# 1: read input from file (parallel read)
# 2: read input from file (serial read)
# 3: 2D drop (radius,height)
# 4: 3D drop (radius,height)
# 5: stratified flow (mean height, sine wave amplitude, sine wave frequency, perturbation amplitude)
# 6: 3D drop array (radius, height, number of drops x direction, number of drops y direction, number of drops z direction),
# x, y and z drop centers distance must be at least 2*(radius+5*sqrt(2)*Ch)
# otherwise number of drops will be reduced
# 7: Drop attached to the bottom wall z_c=-1 (radius)
# 8: 2x 2D Droplets in kissing mode. (radius, ygap , zgap)
# 9: Layer of phi=+1 (mean height, thickness)
in_condphi="3" # integer
radius="0.5" # real (double)
height="0.0" # real (double)
wave_amp_x="0.0" # real (double)
wave_freq_x="0.0" # real (double)
wave_amp_y="0.0" # real (double)
wave_freq_y="0.0" # real (double)
pert_amp="0.0" # real (double)
num_x="5" # integer
num_y="2" # integer
num_z="3" # integer
ygap="1.0" # real(double)
zgap="0.25" # real (double)
thickness="0.1086" # real (double)
# gravity direction
# 1: positive x direction
# -1: negative x direction
# 2: positive z direction
# -2: negative z direction
# 3: positive y direction
# -3: negative y direction
gravdir="-2" # integer
# buoyancy type
# 0: no buoyancy and gravity
# 1: buoyancy and gravity (rho*g)
# 2: only buoyancy (Delta rho*g)
buoyancy="0" # integer
################################################################################
# Surfactant only
# surfactant flag, 0 : surfactant deactivated, 1 : surfactant activated
psi_flag="0" # integer
# surfactant Peclet number
Pe_psi="100.0" # real (double)
# Ex number
Ex="0.117" # real (double)
# Pi number
PI="1.35" # real (double)
# Elasticity number
El="0.5" # real (double)
# Initial conditions on the surfactant
# 0: Initialize constant value (psi_mean)
# 1: Read input from file (parallel read)
# 2: Initialize equilibrium profile (psi_bulk)
# 3: Equilibrium profile multiplied with Y gradient
# 4: Equilibrium profile multiplied with Z gradient
# 5: Diffusion Test, angular distribution
# 6: Read input from file (parallel read, fine grid)
in_condpsi="2"
psi_mean="0.01" # real (double)
psi_bulk="0.01" # real (double)
################################################################################
# Temperature only
# temperature flag, 0 : temperature deactivated, 1 : temperature activated
temp_flag="0" # integer
# Rayleigh number
# for Rayleigh-Benard choose Re=sqrt(Ra/Pr)/4
Ra="10000.0" # real (double)
# Prandtl number
Pr="1.0" # real (double)
# boundary conditions
# A*T+B*dT/dz=C at z=-1
# D*T+E*dT/dz=F at z=+1
A="1.0" # real (double)
B="0.0" # real (double)
C="1.0" # real (double)
D="1.0" # real (double)
E="0.0" # real (double)
F="-1.0" # real (double)
# initial conditons for the temperature
# 0 : initialize constant temperature (mean_t)
# 1 : read from data file (parallel read)
in_cond_temp="0" # integer
temp_mean="0.0" # real (double)
# 1 activate buoyancy term in N-S, 0 deactivate it (Boussinnesq approximation)
# uses same gravity array as defined in the phase field part
boussinnesq="0" # integer
################################################################################
# Lagrangian Particle Tracking only
part_flag="0" # integer
part_number="4" # integer
# 1 use tracer particles (implies 1-way coupling), 0 use inertial particles
tracer="1" # integer
# number of sets of particles run at the same time (one-way coupling only)
# for each set specify Stokes and density ratio (array). Each case corresponds to
# index of array: case 0: stokes(0),dens_part(0), case 1: stokes(1),dens_part(1), ...
nset="2" # integer
# stokes number (in wall units), array declared as stokes=(1.0 10.0 25.0)
stokes=(0.0 1.0) # real (double, array)
# drag type, 1 Stokes drag, 0 Schiller-Naumann drag
stokes_drag="1" # integer
# density ratio particle/fluid, array declared as dens_part=(1.0 10.0 25.0)
dens_part=(1.0 1000.0) # real (double, array)
# 1 to activate gravity and buoyancy force on particle tracking
part_gravity="1" # integer
# 1 activate two-way coupling, 0 deactivate it
twoway="0" # integer
# frequency of saving particle data
part_dump="1000" # integer
# number of substep for particle time integration within one (flow field) time step
subiterations="10" # integer
# initial conditions for the particle position
# 0 : initialize random position
# 1 : read from input file (parallel read, binary file)
# 2 : initialize random position on a x-y plane at height par_plane (part_height)
# 3 : initialize random position on N x-y planes, cubic distribution betweeen +-level (n_planes,level)
in_cond_part_pos="3" # integer
part_height="0.0" # real (double) between -1 and +1
n_planes="11" # integer (even number to have a plane at the channel centre)
level="0.95" # real (double) between 0 and +1
# initial conditions for the particle velocity
# 0 : zero velocity
# 1 : fluid velocity at particle position
# 2 : read from input file (parallel read, binary file)
in_cond_part_vel="0" # integer
# end of parameters declaration
################################################################################
echo ""
echo " FFFFFFF L OOO W W 333 666"
echo " F L O O W W W 3 3 6 6"
echo " F L O O W W W W 3 6"
echo " FFFFF L O O W W W W 3 6666"
echo " F L O O W W W W 3 6 6"
echo " F L O O W W W W 3 3 6 6"
echo " F LLLLLL OOO W W 333 666"
echo ""
echo " +hhy/ "
echo " :: -ddddd "
echo " ydy- :syo- "
echo " +hdho//:/+osyhhdhhyo+:."
echo " :oyhddddddddd+//+oshddo"
echo " sdddddd- ."
echo " /ddddddo "
echo " +hddddhssooshddy -/osh/"
echo " ydds:. hddo+oyddddh+"
echo " hdd: .-:/+osyhddddddddds:"
echo " -sddddddddddddddddddho:"
echo " oddddddddddddddddys+:."
echo " ./+osdddo++//:-."
echo " .yho."
echo " :."
echo ""
echo "=============================================================================="
echo "= START OF NEW RUN ="
echo "= END OF PARAMETER DECLARATION ="
echo "=============================================================================="
echo ""
if [ "$machine" == "0" ]; then
echo ""
echo "============================= OS X Version ==================================="
echo ""
echo " ### "
echo " #### "
echo " ### "
echo " ####### ####### "
echo " ##################### "
echo " ##################### "
echo " #################### "
echo " #################### "
echo " ##################### "
echo " ###################### "
echo " ################### "
echo " ############### "
echo " #### ##### "
echo ""
echo "=============================================================================="
fi
if [ "$openacc_flag" == "1" ]; then
echo ""
echo "=============================== GPU Version =================================="
echo ""
echo " .^!?Y555555J?!: "
echo " .~J5PP5YJJJ??JJJY5PP5? "
echo " :YGPYJ???????!~7?777??J5PP?. "
echo " :5G5??????????!~~!???????7?JPGJ. "
echo " ?B5???????????7~::~7??????????JPG~ "
echo " YBY7???????????7~::~!????????????5B7 "
echo " ?BY7????????????7~^^~!?????????????5B~ "
echo " :B57?????????????7~~~~7??????????????GP "
echo " JBJ7???????????7!~~~~~~!7???????????75B^ "
echo " 5GJ77!!!!7???7?7777~~7777!~~~~7?7777?YB! "
echo " ?BJ^^^^^^^7~^^^!?77::?777^^^^^^~^^^^~5B^ "
echo " :B5:::....::::::^:^..^:::.....::::::^GP "
echo " ?B7. ... .......... . .YB^ "
echo " JB! :::... ......!.:^^:^^. ?B! "
echo " 7BJ. !.!~^~~~~~:^7Y?Y:!J~~. :5G^ "
echo " :5G7....:...... .: :.^:.:: :JGJ. "
echo " :JPY~. .:75P7. "
echo " ^?55J7~::....:^~7J5Y7: "
echo " .:!?JYY55YYJ7~ "
echo ""
echo "=============================================================================="
echo ""
fi
mkdir -p set_run
rm -r ./set_run/go.sh
rm -r ./set_run/sc_compiled
rm -r ./set_run/paraview_output_fg
rm -r ./set_run/stats_calc
rm -r ./set_run/*.out
rm -r ./set_run/*.err
rm -r ./set_run/*.log
if [ "$restart" == "0" ]; then
rm -r ./set_run/initial_fields/*
rm -r ./set_run/results
mkdir ./set_run/results
mkdir ./set_run/results/backup
cp -r ./initial_fields ./set_run/
fi
cp ./restart_copy.sh ./set_run/results/backup/
mkdir ./set_run/sc_compiled
# grid size
NX="$((2**$ix))"
NY="$((2**$iy))"
NZ="$(((2**$iz)+1))"
# set total number of MPI processes requested
if [[ "$multinode" == "1" && "$part_flag" == 1 ]]; then
NNT="$(($NYCPU*$NZCPU+$nodesize))"
elif [[ "$multinode" == "0" && "$part_flag" == 1 ]]; then
NNT="$(($NYCPU*$NZCPU))"
elif [ "$part_flag" == 0 ]; then
NNT="$(($NYCPU*$NZCPU))"
fi
# copy executable and edit it
cp ./go.sh ./set_run
if [ "$machine" == "0" ]; then
sed -i "" "s/NUMTASKS/$NNT/g" ./set_run/go.sh
else
sed -i "s/NUMTASKS/$NNT/g" ./set_run/go.sh
fi
# if tracer force to 0ne-way coupled
if [ "$tracer" == "1" ]; then
twoway="0"
fi
# copy input file and edit it
cp ./input.f90 ./set_run/sc_compiled
if [ "$machine" == "0" ]; then
sed -i "" "s/restartflag/$restart/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/restart_iteration/$nt_restart/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/initialcondition/$incond/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/nxxxxxx/$NX/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/nyyyyyy/$NY/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/nzzzzzz/$NZ/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/Renum/$Re/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/courantnum/$Co/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/gradpx/$gradpx/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/gradpy/$gradpy/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/cpiflag/$cpi_flag/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/repower/$repow/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/len_x/$lx/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/len_y/$ly/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/nstart/$nstart/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/nend/$nend/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/nfdump/$dump/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/nsdump/$sdump/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/faildump/$failure_dump/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/stats_dump/$st_dump/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/stats_start/$start_stats/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/delta_t/$dt/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/bc_upbound/$bc_upb/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/bc_lowbound/$bc_lb/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/phasephiflag/$phi_flag/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/phaseprofflag/$phicor_flag/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/lamcorphi/$lamcorphi/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/matcheddens/$matchedrho/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/densratio/$rhor/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/matchedvisc/$matchedvis/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/viscratio/$visr/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/nonnewtonian/$non_newtonian/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/muinfmuzero/$muinfmuzero/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/expnonnew/$exp_non_new/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/webernumber/$We/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/cahnnumber/$Ch/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/pecletnumber/$Pe/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/froudnumber/$Fr/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/phinitial_condition/$in_condphi/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/gravitydir/$gravdir/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/gravitytype/$buoyancy/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/bodyforce/$body_flag/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/bodyfcoeff/$Bd/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/bodydirection/$bodydir/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/sgradpforce/$sgradp_flag/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/sgradpdirection/$sgradpdir/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/eleforce/$ele_flag/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/elefcoeff/$stuart/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/surfactantflag/$psi_flag/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/surfpeclet/$Pe_psi/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/exnumber/$Ex/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/pinumber/$PI/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/surfelasticity/$El/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/psinitial_condition/$in_condpsi/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/temperatureflag/$temp_flag/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/rayleighnumb/$Ra/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/prandtlnumb/$Pr/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/Aboundary/$A/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/Bboundary/$B/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/Cboundary/$C/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/Dboundary/$D/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/Eboundary/$E/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/Fboundary/$F/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/tempinitial_condition/$in_cond_temp/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/particleflag/$part_flag/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/particlenumber/$part_number/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/npartset/$nset/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/incondpartpos/$in_cond_part_pos/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/incondpartvel/$in_cond_part_vel/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/particledump/$part_dump/g" ./set_run/sc_compiled/input.f90
sed -i "" "s/numsubiteration/$subiterations/g" ./set_run/sc_compiled/input.f90
else
sed -i "s/restartflag/$restart/g" ./set_run/sc_compiled/input.f90
sed -i "s/restart_iteration/$nt_restart/g" ./set_run/sc_compiled/input.f90
sed -i "s/initialcondition/$incond/g" ./set_run/sc_compiled/input.f90
sed -i "s/nxxxxxx/$NX/g" ./set_run/sc_compiled/input.f90
sed -i "s/nyyyyyy/$NY/g" ./set_run/sc_compiled/input.f90
sed -i "s/nzzzzzz/$NZ/g" ./set_run/sc_compiled/input.f90
sed -i "s/Renum/$Re/g" ./set_run/sc_compiled/input.f90
sed -i "s/courantnum/$Co/g" ./set_run/sc_compiled/input.f90
sed -i "s/gradpx/$gradpx/g" ./set_run/sc_compiled/input.f90
sed -i "s/gradpy/$gradpy/g" ./set_run/sc_compiled/input.f90
sed -i "s/cpiflag/$cpi_flag/g" ./set_run/sc_compiled/input.f90
sed -i "s/repower/$repow/g" ./set_run/sc_compiled/input.f90
sed -i "s/len_x/$lx/g" ./set_run/sc_compiled/input.f90
sed -i "s/len_y/$ly/g" ./set_run/sc_compiled/input.f90
sed -i "s/nstart/$nstart/g" ./set_run/sc_compiled/input.f90
sed -i "s/nend/$nend/g" ./set_run/sc_compiled/input.f90
sed -i "s/nfdump/$dump/g" ./set_run/sc_compiled/input.f90
sed -i "s/nsdump/$sdump/g" ./set_run/sc_compiled/input.f90
sed -i "s/faildump/$failure_dump/g" ./set_run/sc_compiled/input.f90
sed -i "s/stats_dump/$st_dump/g" ./set_run/sc_compiled/input.f90
sed -i "s/stats_start/$start_stats/g" ./set_run/sc_compiled/input.f90
sed -i "s/delta_t/$dt/g" ./set_run/sc_compiled/input.f90
sed -i "s/bc_upbound/$bc_upb/g" ./set_run/sc_compiled/input.f90
sed -i "s/bc_lowbound/$bc_lb/g" ./set_run/sc_compiled/input.f90
sed -i "s/phasephiflag/$phi_flag/g" ./set_run/sc_compiled/input.f90
sed -i "s/phaseprofflag/$phicor_flag/g" ./set_run/sc_compiled/input.f90
sed -i "s/lamcorphi/$lamcorphi/g" ./set_run/sc_compiled/input.f90
sed -i "s/matcheddens/$matchedrho/g" ./set_run/sc_compiled/input.f90
sed -i "s/densratio/$rhor/g" ./set_run/sc_compiled/input.f90
sed -i "s/matchedvisc/$matchedvis/g" ./set_run/sc_compiled/input.f90
sed -i "s/viscratio/$visr/g" ./set_run/sc_compiled/input.f90
sed -i "s/nonnewtonian/$non_newtonian/g" ./set_run/sc_compiled/input.f90
sed -i "s/muinfmuzero/$muinfmuzero/g" ./set_run/sc_compiled/input.f90
sed -i "s/expnonnew/$exp_non_new/g" ./set_run/sc_compiled/input.f90
sed -i "s/webernumber/$We/g" ./set_run/sc_compiled/input.f90
sed -i "s/cahnnumber/$Ch/g" ./set_run/sc_compiled/input.f90
sed -i "s/pecletnumber/$Pe/g" ./set_run/sc_compiled/input.f90
sed -i "s/froudnumber/$Fr/g" ./set_run/sc_compiled/input.f90
sed -i "s/phinitial_condition/$in_condphi/g" ./set_run/sc_compiled/input.f90
sed -i "s/gravitydir/$gravdir/g" ./set_run/sc_compiled/input.f90
sed -i "s/gravitytype/$buoyancy/g" ./set_run/sc_compiled/input.f90
sed -i "s/bodyforce/$body_flag/g" ./set_run/sc_compiled/input.f90
sed -i "s/bodyfcoeff/$Bd/g" ./set_run/sc_compiled/input.f90
sed -i "s/bodydirection/$bodydir/g" ./set_run/sc_compiled/input.f90
sed -i "s/sgradpforce/$sgradp_flag/g" ./set_run/sc_compiled/input.f90
sed -i "s/sgradpdirection/$sgradpdir/g" ./set_run/sc_compiled/input.f90
sed -i "s/eleforce/$ele_flag/g" ./set_run/sc_compiled/input.f90
sed -i "s/elefcoeff/$stuart/g" ./set_run/sc_compiled/input.f90
sed -i "s/surfactantflag/$psi_flag/g" ./set_run/sc_compiled/input.f90
sed -i "s/surfpeclet/$Pe_psi/g" ./set_run/sc_compiled/input.f90
sed -i "s/exnumber/$Ex/g" ./set_run/sc_compiled/input.f90
sed -i "s/pinumber/$PI/g" ./set_run/sc_compiled/input.f90
sed -i "s/surfelasticity/$El/g" ./set_run/sc_compiled/input.f90
sed -i "s/psinitial_condition/$in_condpsi/g" ./set_run/sc_compiled/input.f90
sed -i "s/temperatureflag/$temp_flag/g" ./set_run/sc_compiled/input.f90
sed -i "s/rayleighnumb/$Ra/g" ./set_run/sc_compiled/input.f90
sed -i "s/prandtlnumb/$Pr/g" ./set_run/sc_compiled/input.f90
sed -i "s/Aboundary/$A/g" ./set_run/sc_compiled/input.f90
sed -i "s/Bboundary/$B/g" ./set_run/sc_compiled/input.f90
sed -i "s/Cboundary/$C/g" ./set_run/sc_compiled/input.f90
sed -i "s/Dboundary/$D/g" ./set_run/sc_compiled/input.f90
sed -i "s/Eboundary/$E/g" ./set_run/sc_compiled/input.f90
sed -i "s/Fboundary/$F/g" ./set_run/sc_compiled/input.f90
sed -i "s/tempinitial_condition/$in_cond_temp/g" ./set_run/sc_compiled/input.f90
sed -i "s/particleflag/$part_flag/g" ./set_run/sc_compiled/input.f90
sed -i "s/particlenumber/$part_number/g" ./set_run/sc_compiled/input.f90
sed -i "s/npartset/$nset/g" ./set_run/sc_compiled/input.f90
sed -i "s/incondpartpos/$in_cond_part_pos/g" ./set_run/sc_compiled/input.f90
sed -i "s/incondpartvel/$in_cond_part_vel/g" ./set_run/sc_compiled/input.f90
sed -i "s/particledump/$part_dump/g" ./set_run/sc_compiled/input.f90
sed -i "s/numsubiteration/$subiterations/g" ./set_run/sc_compiled/input.f90
fi
# end of input file editing
# copy source files