From 660ad49185683dc8bf4809d7f3e1d51551edb9eb Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 13 Sep 2022 09:15:17 +0200 Subject: [PATCH 01/20] Added cores_per_node to config --- perf_metrics.config | 4 +++- src/Job_Creator.py | 14 +++++++++----- src/functions.bash | 13 +++++++------ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/perf_metrics.config b/perf_metrics.config index 575f178..39125fa 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -13,11 +13,13 @@ Nemo_input_data="../DATA/ORCA2_ICE_v4.2.0/" Nemo_cores=( 4 24 48 96 192) # Jobs_n_cores: nºcores used for other jobs like compiling nemo. +# Jobs_cores_per_node: define the number of cores per node. # Jobs_scheduler: Available (slurm/lsf/torque). # Jobs_time: Max duration of the job in min. # Jobs_queue: Queue used. -Jobs_n_cores=96 +Jobs_n_cores=96 +Jobs_cores_per_node=0 Jobs_scheduler="slurm" Jobs_time=60 Jobs_queue=debug diff --git a/src/Job_Creator.py b/src/Job_Creator.py index dc70f90..97cc727 100644 --- a/src/Job_Creator.py +++ b/src/Job_Creator.py @@ -22,7 +22,7 @@ def get_command_line_arguments(): help="Set slurm time, in minutes") parser.add_argument("--set-core", default=1, type=int, help="Set number of cores to be used for the job") - parser.add_argument("--set-core-per-node", default=48, type=int, + parser.add_argument("--set-core-per-node", default=0, type=int, help="Set number of cores to be used for the job") parser.add_argument("-j", "--job-name", default=None, help="Name of the job you want to create or modify") @@ -65,7 +65,7 @@ def create_job_slurm(args): if cores is not None: file.append("#SBATCH --ntasks " + str(cores) + "\n") - if cores_per_node is not None: + if cores_per_node is not None and not 0: file.append("#SBATCH --ntasks-per-node " + str(cores_per_node) + "\n") if time is not None and not 0: file.append("#SBATCH --time " + str(time) + "\n") @@ -125,9 +125,13 @@ def create_job_torque(args): cores_per_node = args.set_core_per_node queue = args.set_queue workload = args.set_workload - nodes = (cores//cores_per_node)+1 - hours = time // 60 - minutes = time % 60 + if cores_per_node is not None: + nodes = (cores//cores_per_node)+1 + + if time is not None: + hours = time // 60 + minutes = time % 60 + file = ["#!/bin/bash \n", "############################################################################### \n", "#PBS -o "+str(name)+".out \n#PBS -e "+str(name)+".err \n" diff --git a/src/functions.bash b/src/functions.bash index 7d7cb1d..e3832aa 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -115,7 +115,7 @@ Compile_extrae() echo "Output of the compilation in compile.err and compile.out" printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" - python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" state1=$("$job" --wait compile."$Jobs_scheduler") echo @@ -202,7 +202,7 @@ Compile_gprof() echo "Output of the compilation in compile.err and compile.out" printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg" - python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" state1=$("$job" --wait compile."$Jobs_scheduler") echo @@ -257,6 +257,7 @@ Init() Nemo_input_data="${Nemo_input_data=:-"./"}" Nemo_cores="${Nemo_cores:-( 48 )}" Jobs_n_cores="${Jobs_n_cores:-48}" + Jobs_cores_per_node="${Jobs_cores_per_node:-0}" Jobs_scheduler="${Jobs_scheduler:-"slurm"}" time="${Jobs_time:-0}" queue="${Jobs_queue:-""}" @@ -391,7 +392,7 @@ Gprof_functions() rm gmon* 2> /dev/null echo "Runing Nemo with 2 cores to obtain function data..." echo - python3 ./../src/Job_Creator.py -f "run" -j "run" --set-core 4 -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np 2 ./nemo" + python3 ./../src/Job_Creator.py -f "run" -j "run" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np 2 ./nemo" state2=$("$job" --wait run."$Jobs_scheduler") Job_completed "$state2" @@ -406,7 +407,7 @@ Gprof_functions() fi echo "Gthrottling functions ..." echo - python3 ./../src/Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./../src/gthrottling.sh nemo" + python3 ./../src/Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./../src/gthrottling.sh nemo" state3=$("$job" --wait gthrottling."$Jobs_scheduler") Job_completed "$state3" if [ $Completed == false ]; then @@ -446,7 +447,7 @@ Compile_extrae echo "Creating trace with $core cores..." echo - python3 ./../src/Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core ./../src/trace.sh ./nemo" + python3 ./../src/Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core ./../src/trace.sh ./nemo" state4=$("$job" --wait run_extrae."$Jobs_scheduler") Job_completed "$state4" @@ -488,7 +489,7 @@ Create_metrics() # Create performance metrics echo "Creating metrics and storing theme in Metrics folder" echo - python3 ./src/Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./../src/modelfactors.py -ms 100000 *" + python3 ./src/Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./../src/modelfactors.py -ms 100000 *" mv analysis."$Jobs_scheduler" Metrics cd Metrics||(echo "Error Metrics folder doesn't exists"; exit 1) state5=$("$job" --wait analysis."$Jobs_scheduler") -- GitLab From 1b93fa7175e234ebe686f78072af0ae823f457ba Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 13 Sep 2022 09:30:05 +0200 Subject: [PATCH 02/20] Fixed mistakes --- src/functions.bash | 1 - src/trace.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/functions.bash b/src/functions.bash index e3832aa..8564d5e 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -136,7 +136,6 @@ Compile_extrae() #Copy all the EXP00 data but don't overwrite namelist just the executable cd "$dir" ||( echo "Error original dir doesn't exist"; exit 1 ) - echow "" cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* Run_NEMO cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo Run_NEMO cd Run_NEMO||( echo "Error Run_NEMO folder doesn't exists"; exit 1 ) diff --git a/src/trace.sh b/src/trace.sh index 12a53dc..f81f6d5 100755 --- a/src/trace.sh +++ b/src/trace.sh @@ -1,7 +1,7 @@ #!/bin/sh # Configure Extrae -export EXTRAE_CONFIG_FILE=./src/extrae.xml +export EXTRAE_CONFIG_FILE=./../src/extrae.xml # Load the tracing library (choose C/Fortran) #export LD_PRELOAD=${EXTRAE_HOME}/lib/libmpitrace.so export EXTRAE_SKIP_AUTO_LIBRARY_INITIALIZE=1 -- GitLab From 52ba0e2bb4e5a5c86039860ba1112ee51dcbd33e Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 13 Sep 2022 12:11:11 +0200 Subject: [PATCH 03/20] Fixed some errors related with absolute path --- perf_metrics.config | 40 +++++++++++++++++----------------- src/Job_Creator.py | 8 +++---- src/functions.bash | 52 +++++++++++++++++++++++++-------------------- 3 files changed, 54 insertions(+), 46 deletions(-) diff --git a/perf_metrics.config b/perf_metrics.config index 39125fa..0d14865 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -3,41 +3,43 @@ ################################################################################# -# Nemo_path: Path to nemo installation folder containing the cfgs and arch dirs. -# Nemo_input_data: Path to the input data needed to run the nemo cfg. -# Nemo_cores: List of nºcores used for executing Nemo, ( 4 48 ) makes the script execute and +# Nemo_path (string) : Path to nemo installation folder containing the cfgs and arch dirs. +# Nemo_input_data (string): Path to the input data needed to run the nemo cfg. +# Nemo_run (string): Path where the folder Run_NEMO will be created. +# Nemo_cores (array): List of nºcores used for executing Nemo, ( 4 48 ) makes the script execute and # get Nemo traces with 4 and 48 cores. 2 different nºcores are needed to obtain scalability data. -Nemo_path="../NEMO/" -Nemo_input_data="../DATA/ORCA2_ICE_v4.2.0/" +Nemo_path="Nemo_installation_path" +Nemo_input_data="Nemo_input_data_path" +Nemo_run="." Nemo_cores=( 4 24 48 96 192) -# Jobs_n_cores: nºcores used for other jobs like compiling nemo. -# Jobs_cores_per_node: define the number of cores per node. -# Jobs_scheduler: Available (slurm/lsf/torque). -# Jobs_time: Max duration of the job in min. -# Jobs_queue: Queue used. +# Jobs_n_cores (integer): nºcores used for other jobs like compiling nemo. +# Jobs_cores_per_node (integer): define the number of cores per node. +# Jobs_scheduler (string): Available (slurm/lsf/torque). +# Jobs_time (integer): Max duration of the job in min. +# Jobs_queue (string): Queue used. -Jobs_n_cores=96 +Jobs_n_cores= 4 Jobs_cores_per_node=0 Jobs_scheduler="slurm" Jobs_time=60 -Jobs_queue=debug +Jobs_queue="debug" -# Compilation_compile: When false only compiles NEMO if arch file lacks the needed flags, when true always compiles NEMO. -# Compilation_ref: Reference configuration. -# Compilation_arch: Architecture used (without the -arch sufix and the .fcm). -# Compilation_name: Name of the new configutation (Important to not be an existing one). -# Compilation_sub: Add or remove subcomponents. +# Compilation_compile (string): When false only compiles NEMO if arch file lacks the needed flags, when true always compiles NEMO. +# Compilation_ref (string): Reference configuration. +# Compilation_arch (string): Architecture used (without the -arch sufix and the .fcm). +# Compilation_name (string): Name of the new configutation (Important to not be an existing one). +# Compilation_sub (string): Add or remove subcomponents. Compilation_compile="false" Compilation_ref="ORCA2_ICE_PISCES" -Compilation_arch="Your-arch-file" +Compilation_arch="Your-arch-file" Compilation_name="ORCA2_EXTRAE" Compilation_sub="OCE del_key 'key_si3 key_top'" -# List of modules loaded. +# Modules (string): List of modules loaded. # Required: # - Perl interpreter # - Fortran compiler (ifort, gfortran, pgfortran, ftn, …) diff --git a/src/Job_Creator.py b/src/Job_Creator.py index 97cc727..94180fd 100644 --- a/src/Job_Creator.py +++ b/src/Job_Creator.py @@ -65,9 +65,9 @@ def create_job_slurm(args): if cores is not None: file.append("#SBATCH --ntasks " + str(cores) + "\n") - if cores_per_node is not None and not 0: + if cores_per_node is not None and cores_per_node != 0: file.append("#SBATCH --ntasks-per-node " + str(cores_per_node) + "\n") - if time is not None and not 0: + if time is not None and time!= 0: file.append("#SBATCH --time " + str(time) + "\n") if name is not None: file.append("#SBATCH -J " + name + "\n") @@ -100,7 +100,7 @@ def create_job_lsf(args): if cores is not None: file.append("#BSUB -n " + str(cores) + "\n") - if time is not None and not 0: + if time is not None and time!= 0: file.append("#BSUB -W " + str(time) + "\n") if name is not None: file.append("#BSUB-J " + name + "\n") @@ -139,7 +139,7 @@ def create_job_torque(args): if cores is not None: file.append("#PBS -l nodes" + str(nodes) + ":ppn=" + str(cores) + "\n") - if time is not None and not 0: + if time is not None and time != 0: file.append("#PBS -l cput=" + str(hours) + ":"+str(minutes)+ ":00\n") if name is not None: file.append("#PBS -N " + name + "\n") diff --git a/src/functions.bash b/src/functions.bash index 8564d5e..3afa808 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -115,7 +115,7 @@ Compile_extrae() echo "Output of the compilation in compile.err and compile.out" printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" - python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + python3 src/./Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" state1=$("$job" --wait compile."$Jobs_scheduler") echo @@ -135,9 +135,10 @@ Compile_extrae() fi #Copy all the EXP00 data but don't overwrite namelist just the executable - cd "$dir" ||( echo "Error original dir doesn't exist"; exit 1 ) + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* Run_NEMO cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo Run_NEMO + cd Run_NEMO||( echo "Error Run_NEMO folder doesn't exists"; exit 1 ) } @@ -201,7 +202,7 @@ Compile_gprof() echo "Output of the compilation in compile.err and compile.out" printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg" - python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + python3 src/./Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" state1=$("$job" --wait compile."$Jobs_scheduler") echo @@ -278,7 +279,7 @@ Test_arguments() { # Nemo path correct? if ! test -d "${Nemo_path}"; then - echo "Nemo relative path: ${Nemo_path} is not found" + echo "Nemo relative path: ${Nemo_path} is not found, modify it in perf_metrics.config with the correct path" echo exit 1 fi @@ -286,7 +287,7 @@ Test_arguments() # Is Nemo data path correct? if ! test -d "${Nemo_input_data}"; then - echo "Nemo input data path: ${Nemo_input_data} is not found" + echo "Nemo input data path: ${Nemo_input_data} is not found, modify it in perf_metrics.config with the correct path" echo exit 1 @@ -294,12 +295,12 @@ Test_arguments() if ! test -d "Run_NEMO"; then mkdir Run_NEMO fi - cp -P "${Nemo_input_data}"* Run_NEMO + cp -P "${Nemo_input_data}"/* Run_NEMO fi #Nemo_cores is array? if [[ ! "$(declare -p Nemo_cores)" =~ "declare -a" ]]; then - echo "Error Nemo_cores has to be a bash array like ( 4 24 48 )" + echo "Error Nemo_cores has to be a bash array like ( 4 24 48 ), modify it in perf_metrics.config with a valid value" echo exit 1 fi @@ -312,7 +313,7 @@ Test_arguments() do if ! [[ $core =~ $re ]] ; then - echo "Error Nemo_cores has to contain integer values" + echo "Error Nemo_cores has to contain integer values, modify it in perf_metrics.config with a valid value" echo exit 1 fi @@ -320,31 +321,31 @@ Test_arguments() # cfg exists? if ! test -d "${Nemo_path}/cfgs/${cfg}"; then - echo "configuration: ${cfg} doesn't exists in ${Nemo_path}/cfgs dir" + echo "configuration: ${cfg} doesn't exists in ${Nemo_path}/cfgs dir, modify it in perf_metrics.config with a valid cfg" echo exit 1 fi # arch exists? if ! test -f "${Nemo_path}/arch/arch-${arch}.fcm"; then - echo "architecture: arch-${arch}.fcm doesn't exists in ${Nemo_path}/arch dir" + echo "architecture: arch-${arch}.fcm doesn't exists in ${Nemo_path}/arch dir, modify it in perf_metrics.config with a valid arch" echo exit 1 fi # scheduler correct? if [ "$Jobs_scheduler" != "slurm" ] && [ "$Jobs_scheduler" != "lsf" ] && [ "$Jobs_scheduler" != "torque" ]; then - echo "$Jobs_scheduler is not a valid scheduler" + echo "$Jobs_scheduler is not a valid scheduler, modify it in perf_metrics.config with a valid scheduler (slurm/lsf/torque)" echo exit 1 fi #Nemo_cores is array? if [[ ! "$(declare -p Nemo_cores)" =~ "declare -a" ]]; then - echo "Error, variable Nemo_cores has to be an array" + echo "Error, variable Nemo_cores has to be an array, modify it in perf_metrics.config with a proper value" echo exit 1 fi #Modules available if ! module load $Modules;then - echo "Error loading modules aborting" + echo "Error loading modules aborting, modify the modules loaded in perf_metrics.config" echo exit 1 fi @@ -357,7 +358,7 @@ Test_arguments() else sed -i 's|home=.*|home="'"$EXTRAE_HOME"'"|g' src/extrae.xml fi - + # Adding -d to variable if not empty if [ -n "$comp_cfg" ]; then comp_cfg="-d $comp_cfg" @@ -372,6 +373,10 @@ Test_arguments() job="qsub" fi + # Changing trace.sh reference to extrae.xml + + sed -i "s|export EXTRAE_CONFIG_FILE=.*|export EXTRAE_CONFIG_FILE=$dir/src/./extrae.xml|g" src/trace.sh + } @@ -382,6 +387,7 @@ Gprof_functions() { cd Run_NEMO||(echo "Error Run_NEMO folder doesn't exists"; exit 1) + #Generating function list in case of missing if ! test -f "extrae_functions.txt"; then @@ -391,7 +397,7 @@ Gprof_functions() rm gmon* 2> /dev/null echo "Runing Nemo with 2 cores to obtain function data..." echo - python3 ./../src/Job_Creator.py -f "run" -j "run" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np 2 ./nemo" + python3 "$dir"/src/./Job_Creator.py -f "run" -j "run" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np 2 ./nemo" state2=$("$job" --wait run."$Jobs_scheduler") Job_completed "$state2" @@ -406,7 +412,7 @@ Gprof_functions() fi echo "Gthrottling functions ..." echo - python3 ./../src/Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./../src/gthrottling.sh nemo" + python3 "$dir"/src/./Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./gthrottling.sh nemo" state3=$("$job" --wait gthrottling."$Jobs_scheduler") Job_completed "$state3" if [ $Completed == false ]; then @@ -438,7 +444,7 @@ Get_trace() Compile_extrae # Run nemo with extrae - ./../src/extraf.sh nemo extrae_functions.txt + "$dir"/src/./extraf.sh nemo extrae_functions.txt sed -i "s|list=.*|list=\"${dir}/Run_NEMO/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" ../src/extrae.xml for core in "${Nemo_cores[@]}" @@ -446,7 +452,7 @@ Compile_extrae echo "Creating trace with $core cores..." echo - python3 ./../src/Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core ./../src/trace.sh ./nemo" + python3 "$dir"/src/./Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core "$dir"/src/./trace.sh ./nemo" state4=$("$job" --wait run_extrae."$Jobs_scheduler") Job_completed "$state4" @@ -469,10 +475,10 @@ Compile_extrae echo # Creating folder - if ! test -d "../Metrics"; then - mkdir ../Metrics + if ! test -d "$dir/Metrics"; then + mkdir "$dir"/Metrics fi - cp nemo_"$core".best_cut.* ../Metrics + cp nemo_"$core".best_cut.* "$dir"/Metrics @@ -488,7 +494,7 @@ Create_metrics() # Create performance metrics echo "Creating metrics and storing theme in Metrics folder" echo - python3 ./src/Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./../src/modelfactors.py -ms 100000 *" + python3 src/./Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./modelfactors.py -ms 100000 *" mv analysis."$Jobs_scheduler" Metrics cd Metrics||(echo "Error Metrics folder doesn't exists"; exit 1) state5=$("$job" --wait analysis."$Jobs_scheduler") @@ -506,4 +512,4 @@ Create_metrics() echo "------------------------------------------------------------------------------" echo "------------------------------------------------------------------------------" echo -} \ No newline at end of file +} -- GitLab From 53249f7c805335985efc1833c2147919c4aa0fc0 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 13 Sep 2022 12:32:03 +0200 Subject: [PATCH 04/20] Now exits if sbatch fails submitting a job --- src/functions.bash | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/functions.bash b/src/functions.bash index 3afa808..6362da4 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -117,7 +117,11 @@ Compile_extrae() printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" python3 src/./Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" - state1=$("$job" --wait compile."$Jobs_scheduler") + + if ! state1=$("$job" --wait compile."$Jobs_scheduler"); then + exit 1: + fi + echo Job_completed "$state1" if [ $Completed == false ]; then @@ -204,7 +208,10 @@ Compile_gprof() printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg" python3 src/./Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" - state1=$("$job" --wait compile."$Jobs_scheduler") + if ! state1=$("$job" --wait compile."$Jobs_scheduler"); then + exit 1 + fi + echo Job_completed "$state1" if [ "$Completed" == false ]; then @@ -399,7 +406,9 @@ Gprof_functions() echo python3 "$dir"/src/./Job_Creator.py -f "run" -j "run" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np 2 ./nemo" - state2=$("$job" --wait run."$Jobs_scheduler") + if ! state2=$("$job" --wait run."$Jobs_scheduler"); then + exit 1 + fi Job_completed "$state2" if [ $Completed == false ]; then echo "Nemo execution failed look at Run_NEMO/run.err and Run_NEMO/ocean.output for more info" @@ -413,7 +422,9 @@ Gprof_functions() echo "Gthrottling functions ..." echo python3 "$dir"/src/./Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./gthrottling.sh nemo" - state3=$("$job" --wait gthrottling."$Jobs_scheduler") + if ! state3=$("$job" --wait gthrottling."$Jobs_scheduler"); then + exit 1 + fi Job_completed "$state3" if [ $Completed == false ]; then echo "Error listing functions, look at Run_NEMO/gthrottling.err for more info" @@ -454,7 +465,9 @@ Compile_extrae echo python3 "$dir"/src/./Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core "$dir"/src/./trace.sh ./nemo" - state4=$("$job" --wait run_extrae."$Jobs_scheduler") + if ! state4=$("$job" --wait run_extrae."$Jobs_scheduler") ; then + exit 1 + fi Job_completed "$state4" if [ $Completed == false ]; then echo "Nemo execution failed, no traces files generated more info inside Run_NEMO/run_extrae.err" @@ -497,7 +510,9 @@ Create_metrics() python3 src/./Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./modelfactors.py -ms 100000 *" mv analysis."$Jobs_scheduler" Metrics cd Metrics||(echo "Error Metrics folder doesn't exists"; exit 1) - state5=$("$job" --wait analysis."$Jobs_scheduler") + if ! state5=$("$job" --wait analysis."$Jobs_scheduler"); then + exit 1 + fi Job_completed "$state5" if [ $Completed == false ]; then echo "Error, metrics have not generated check Metrics/analysis.err to get more details" -- GitLab From 490b654fd7e90877083886f9d35a39b531e6212d Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 13 Sep 2022 12:54:28 +0200 Subject: [PATCH 05/20] Logs are now stored in logs dir --- src/functions.bash | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/functions.bash b/src/functions.bash index 6362da4..d6f64dd 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -125,12 +125,13 @@ Compile_extrae() echo Job_completed "$state1" if [ $Completed == false ]; then - echo "Nemo compilation failed, remember to load all the needed modules. Check the details in compile.err" + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in logs/compile.err" echo exit 1 else echo "Nemo compilation successful" echo + mv compile.* "$dir"/logs fi else @@ -215,12 +216,13 @@ Compile_gprof() echo Job_completed "$state1" if [ "$Completed" == false ]; then - echo "Nemo compilation failed, remember to load all the needed modules. Check the details in compile.err" + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in logs/compile.err" echo exit 1 else echo "Nemo compilation successful" echo + mv compile.* "$dir"/logs fi else @@ -384,6 +386,10 @@ Test_arguments() sed -i "s|export EXTRAE_CONFIG_FILE=.*|export EXTRAE_CONFIG_FILE=$dir/src/./extrae.xml|g" src/trace.sh + # Creating logs dir + if ! test -d "logs"; then + mkdir logs + fi } @@ -411,13 +417,14 @@ Gprof_functions() fi Job_completed "$state2" if [ $Completed == false ]; then - echo "Nemo execution failed look at Run_NEMO/run.err and Run_NEMO/ocean.output for more info" + echo "Nemo execution failed look at logs/run.err and Run_NEMO/ocean.output for more info" echo "Remember that the namelist files copied are the default ones, change theme in order to fit with the input files in the dir " echo exit 1 else echo "Gprof files generated " echo + mv run.* "$dir"/logs fi echo "Gthrottling functions ..." echo @@ -427,12 +434,13 @@ Gprof_functions() fi Job_completed "$state3" if [ $Completed == false ]; then - echo "Error listing functions, look at Run_NEMO/gthrottling.err for more info" + echo "Error listing functions, look at logs/gthrottling.err for more info" echo exit 1 else echo "Functions listed correctly" echo + mv gthrottling.* "$dir"/logs fi @@ -470,10 +478,13 @@ Compile_extrae fi Job_completed "$state4" if [ $Completed == false ]; then - echo "Nemo execution failed, no traces files generated more info inside Run_NEMO/run_extrae.err" + echo "Nemo execution failed, no traces files generated more info inside logs/run_extrae.err" echo exit 1 fi + + mv run_extrae.* "$dir"/logs + mv nemo.prv nemo_"$core".prv mv nemo.pcf nemo_"$core".pcf mv nemo.row nemo_"$core".row @@ -513,14 +524,16 @@ Create_metrics() if ! state5=$("$job" --wait analysis."$Jobs_scheduler"); then exit 1 fi + Job_completed "$state5" if [ $Completed == false ]; then - echo "Error, metrics have not generated check Metrics/analysis.err to get more details" + echo "Error, metrics have not generated check logs/analysis.err to get more details" echo exit 1 fi - + cp analysis.out overview.txt + mv analysis.* "$dir"/logs echo "------------------------------------------------------------------------------" echo "------------------------- Script Completed -----------------------------------" echo "----------------------- Data in Metrics folder -------------------------------" -- GitLab From c5e99533b58f912a678ace1fd5159890e9f0149b Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 13 Sep 2022 15:11:18 +0200 Subject: [PATCH 06/20] Now NEMO run dir can be defined in config --- README.md | 2 +- perf_metrics.config | 2 +- src/functions.bash | 65 ++++++++++++++++++++++++++------------------- 3 files changed, 39 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 58941d3..abf8159 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Nemo modelfactors -The project was developed tested using NEMO 4.2 version. +The project was developed and tested using NEMO 4.2 version. This project is intended to compute important performance metrics for a NEMO run. diff --git a/perf_metrics.config b/perf_metrics.config index 0d14865..86c051c 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -21,7 +21,7 @@ Nemo_cores=( 4 24 48 96 192) # Jobs_queue (string): Queue used. Jobs_n_cores= 4 -Jobs_cores_per_node=0 +Jobs_cores_per_node= Jobs_scheduler="slurm" Jobs_time=60 Jobs_queue="debug" diff --git a/src/functions.bash b/src/functions.bash index d6f64dd..ebf0faa 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -141,10 +141,10 @@ Compile_extrae() #Copy all the EXP00 data but don't overwrite namelist just the executable - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* Run_NEMO - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo Run_NEMO + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* "${Nemo_run}"/Run_NEMO + cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Nemo_run}"/Run_NEMO - cd Run_NEMO||( echo "Error Run_NEMO folder doesn't exists"; exit 1 ) + cd "${Nemo_run}"/Run_NEMO||( echo "Error ${Nemo_run}/Run_NEMO folder doesn't exists"; exit 1 ) } #Check if Nemo is compiled for using Gprof @@ -233,25 +233,25 @@ Compile_gprof() #Copy all the EXP00 data in RUN_NEMO folder but don't overwrite the namelist, just the executable. - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/* Run_NEMO - cp "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/nemo Run_NEMO + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/* "${Nemo_run}"/Run_NEMO + cp "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/nemo "${Nemo_run}"/Run_NEMO if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then - sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' Run_NEMO/context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) + sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' "${Nemo_run}"/Run_NEMO/context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) fi #Solving NEMO input file common errors - if test -f "Run_NEMO/weights_core_orca2_bicubic_noc.nc"; then - mv Run_NEMO/weights_core_orca2_bicubic_noc.nc Run_NEMO/weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES + if test -f "${Nemo_run}/Run_NEMO/weights_core_orca2_bicubic_noc.nc"; then + mv "${Nemo_run}"/Run_NEMO/weights_core_orca2_bicubic_noc.nc "${Nemo_run}"/Run_NEMO/weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES fi - if test -f "weights_core_orca2_bilinear_noc.nc"; then - mv Run_NEMO/weights_core_orca2_bilinear_noc.nc Run_NEMO/weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES + if test -f "${Nemo_run}/Run_NEMO/weights_core_orca2_bilinear_noc.nc"; then + mv "${Nemo_run}"/Run_NEMO/weights_core_orca2_bilinear_noc.nc "${Nemo_run}"/Run_NEMO/weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES fi - cd Run_NEMO||(echo "Error Run_NEMO folder doesn't exists"; exit 1) + cd "${Nemo_run}"/Run_NEMO||(echo "Error ${Nemo_run}/Run_NEMO folder doesn't exists"; exit 1) } @@ -264,6 +264,7 @@ Init() { Nemo_path="${Nemo_path:-"./"}" Nemo_input_data="${Nemo_input_data=:-"./"}" + Nemo_run="${Nemo_run=:-"./"}" Nemo_cores="${Nemo_cores:-( 48 )}" Jobs_n_cores="${Jobs_n_cores:-48}" Jobs_cores_per_node="${Jobs_cores_per_node:-0}" @@ -299,14 +300,21 @@ Test_arguments() echo "Nemo input data path: ${Nemo_input_data} is not found, modify it in perf_metrics.config with the correct path" echo exit 1 - - else - if ! test -d "Run_NEMO"; then - mkdir Run_NEMO - fi - cp -P "${Nemo_input_data}"/* Run_NEMO + fi + # Is Nemo Run path correct? + if ! test -d "${Nemo_run}"; then + echo "Nemo run path: ${Nemo_run} is not found, modify it in perf_metrics.config with the correct path" + echo + exit 1 fi + # Create Run_Nemo dir after comprovations + if ! test -d "${Nemo_run}/Run_NEMO"; then + mkdir "${Nemo_run}"/Run_NEMO + fi + cp -P "${Nemo_input_data}"/* "${Nemo_run}"/Run_NEMO + + #Nemo_cores is array? if [[ ! "$(declare -p Nemo_cores)" =~ "declare -a" ]]; then echo "Error Nemo_cores has to be a bash array like ( 4 24 48 ), modify it in perf_metrics.config with a valid value" @@ -387,9 +395,10 @@ Test_arguments() sed -i "s|export EXTRAE_CONFIG_FILE=.*|export EXTRAE_CONFIG_FILE=$dir/src/./extrae.xml|g" src/trace.sh # Creating logs dir - if ! test -d "logs"; then - mkdir logs - fi + if ! test -d "logs"; then + mkdir logs + + fi } @@ -399,7 +408,7 @@ Test_arguments() Gprof_functions() { - cd Run_NEMO||(echo "Error Run_NEMO folder doesn't exists"; exit 1) + cd "${Nemo_run}"/Run_NEMO||(echo "Error ${Nemo_run}/Run_NEMO folder doesn't exists"; exit 1) #Generating function list in case of missing if ! test -f "extrae_functions.txt"; then @@ -417,8 +426,8 @@ Gprof_functions() fi Job_completed "$state2" if [ $Completed == false ]; then - echo "Nemo execution failed look at logs/run.err and Run_NEMO/ocean.output for more info" - echo "Remember that the namelist files copied are the default ones, change theme in order to fit with the input files in the dir " + echo "Nemo execution failed look at logs/run.err and ${Nemo_run}/Run_NEMO/ocean.output for more info" + echo "Remember that the namelist files copied are the default ones, change them in order to fit with the input files in the dir " echo exit 1 else @@ -447,7 +456,7 @@ Gprof_functions() else - echo "Functions already listed, file Run_NEMO/extrae_functions.txt does exist" + echo "Functions already listed, file ${Nemo_run}/Run_NEMO/extrae_functions.txt does exist" echo fi @@ -464,7 +473,7 @@ Compile_extrae # Run nemo with extrae "$dir"/src/./extraf.sh nemo extrae_functions.txt - sed -i "s|list=.*|list=\"${dir}/Run_NEMO/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" ../src/extrae.xml + sed -i "s|list=.*|list=\"${Nemo_run}/Run_NEMO/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" "$dir"/src/extrae.xml for core in "${Nemo_cores[@]}" do @@ -490,9 +499,9 @@ Compile_extrae mv nemo.row nemo_"$core".row echo "Cutting best iteration" echo - ./../src/magiccut/magicCut nemo_"${core}".prv "$Nemo_iterations" > cut_"$core".out 2>&1 + "$dir"/src/magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > "$dir"/logs/cut_"$core".out 2>&1 if ! ls nemo_"$core".best_cut.prv; then - echo "Cut failed, look at Run_NEMO/cut_$core.out for more info." + echo "Cut failed, look at ${dir}/logs/cut_$core.out for more info." echo exit 1 fi @@ -516,7 +525,7 @@ Create_metrics() cd "$dir" || (echo "Error original dir doesn't exist"; exit 1) # Create performance metrics - echo "Creating metrics and storing theme in Metrics folder" + echo "Creating metrics and storing them in Metrics folder" echo python3 src/./Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./modelfactors.py -ms 100000 *" mv analysis."$Jobs_scheduler" Metrics -- GitLab From fd9d7ae3e9223a437d602b77035c9fb4d4624f36 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Wed, 14 Sep 2022 09:17:03 +0200 Subject: [PATCH 07/20] Now each execution to obtain a trace has logs --- src/functions.bash | 55 +++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/src/functions.bash b/src/functions.bash index ebf0faa..37090cb 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -112,28 +112,29 @@ Compile_extrae() if [ "$compile" == true ] || [ "$compile_ext" == true ]; then echo "Compiling Nemo for EXTRAE" - echo "Output of the compilation in compile.err and compile.out" + printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" - python3 src/./Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + python3 src/./Job_Creator.py -f "compile_extrae" -j "compile_extrae" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" - if ! state1=$("$job" --wait compile."$Jobs_scheduler"); then + if ! state1=$("$job" --wait compile_extrae."$Jobs_scheduler"); then exit 1: fi echo Job_completed "$state1" if [ $Completed == false ]; then - echo "Nemo compilation failed, remember to load all the needed modules. Check the details in logs/compile.err" + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in logs/compile_extrae.err" echo exit 1 else echo "Nemo compilation successful" echo - mv compile.* "$dir"/logs + fi - + mv compile_extrae.* "$dir"/logs + else echo "Compilation not needed" echo @@ -172,7 +173,7 @@ Compile_gprof() if ! echo "${line}"|grep -q "\-g\b"; then echo "-g flag not found in arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " sed -i '/^%FCFLAGS/ s/$/ -g /' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm - compile_gprof=true + comp_gprof=true fi # If -pg is not there recompilation is requiered and -pg added @@ -180,51 +181,50 @@ Compile_gprof() if ! echo "${line}"|grep -q "\-pg\b"; then echo "-pg flag not found in FCFLAGS arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " sed -i '/^%FCFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm - compile_gprof=true + comp_gprof=true fi if ! echo "${line2}"|grep -q "\-pg\b"; then echo "-pg flag not found in FPPFLAGS arch-${arch}_GPROF.fcm : editing arch-${arch}_GPROF.fcm " sed -i '/^%FPPFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm - compile_gprof=true + comp_gprof=true fi if ! echo "${line3}"|grep -q "\-pg\b"; then echo "-pg flag not found in LDFLAGS arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " sed -i '/^%LDFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm - compile_gprof=true + comp_gprof=true fi # If nemo executable is not on the run file compile if ! test -f "${Nemo_path}/cfgs/${name_cfg}_GPROF/EXP00/nemo"; then echo "nemo executable not found in ${name_cfg}_GPROF" - compile_gprof=true + comp_gprof=true fi - if [ "$compile" == true ] || [ "$compile_gprof" == true ]; then + if [ "$compile" == true ] || [ "$comp_gprof" == true ]; then echo "Compiling Nemo for GPROF" - echo "Output of the compilation in compile.err and compile.out" - printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg" - python3 src/./Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + python3 src/./Job_Creator.py -f "compile_gprof" -j "compile_gprof" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" - if ! state1=$("$job" --wait compile."$Jobs_scheduler"); then + if ! state1=$("$job" --wait compile_gprof."$Jobs_scheduler"); then exit 1 fi echo Job_completed "$state1" if [ "$Completed" == false ]; then - echo "Nemo compilation failed, remember to load all the needed modules. Check the details in logs/compile.err" + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in logs/compile_gprof.err" echo exit 1 else echo "Nemo compilation successful" echo - mv compile.* "$dir"/logs + fi - + mv compile_gprof.* "$dir"/logs + else echo "Compilation not needed" echo @@ -433,8 +433,9 @@ Gprof_functions() else echo "Gprof files generated " echo - mv run.* "$dir"/logs fi + mv run.* "$dir"/logs + echo "Gthrottling functions ..." echo python3 "$dir"/src/./Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./gthrottling.sh nemo" @@ -449,12 +450,10 @@ Gprof_functions() else echo "Functions listed correctly" echo - mv gthrottling.* "$dir"/logs + fi - + mv gthrottling.* "$dir"/logs - - else echo "Functions already listed, file ${Nemo_run}/Run_NEMO/extrae_functions.txt does exist" echo @@ -480,19 +479,19 @@ Compile_extrae echo "Creating trace with $core cores..." echo - python3 "$dir"/src/./Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core "$dir"/src/./trace.sh ./nemo" + python3 "$dir"/src/./Job_Creator.py -f "run_extrae_$core" -j "run_extrae_$core" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core "$dir"/src/./trace.sh ./nemo" - if ! state4=$("$job" --wait run_extrae."$Jobs_scheduler") ; then + if ! state4=$("$job" --wait run_extrae_$core."$Jobs_scheduler") ; then exit 1 fi Job_completed "$state4" if [ $Completed == false ]; then - echo "Nemo execution failed, no traces files generated more info inside logs/run_extrae.err" + echo "Nemo execution failed, no traces files generated more info inside logs/run_extrae_$core.err" echo exit 1 fi - mv run_extrae.* "$dir"/logs + mv run_extrae_"$core".* "$dir"/logs mv nemo.prv nemo_"$core".prv mv nemo.pcf nemo_"$core".pcf -- GitLab From 585b91f42fbaf18e3115d653e9a446b2058b51aa Mon Sep 17 00:00:00 2001 From: cpenadep Date: Wed, 14 Sep 2022 09:44:54 +0200 Subject: [PATCH 08/20] Fixed spelling errors --- perf_metrics.config | 6 +++--- src/functions.bash | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/perf_metrics.config b/perf_metrics.config index 86c051c..cccfd7a 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -28,9 +28,9 @@ Jobs_queue="debug" # Compilation_compile (string): When false only compiles NEMO if arch file lacks the needed flags, when true always compiles NEMO. # Compilation_ref (string): Reference configuration. -# Compilation_arch (string): Architecture used (without the -arch sufix and the .fcm). -# Compilation_name (string): Name of the new configutation (Important to not be an existing one). -# Compilation_sub (string): Add or remove subcomponents. +# Compilation_arch (string): Architecture used (without the -arch suffix and the .fcm). +# Compilation_name (string): Name of the new configuration (Important to not be an existing one). +# Compilation_sub (string): Add or remove sub-components. Compilation_compile="false" diff --git a/src/functions.bash b/src/functions.bash index 37090cb..34ded5d 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -496,7 +496,7 @@ Compile_extrae mv nemo.prv nemo_"$core".prv mv nemo.pcf nemo_"$core".pcf mv nemo.row nemo_"$core".row - echo "Cutting best iteration" + echo "Cutting up best iteration" echo "$dir"/src/magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > "$dir"/logs/cut_"$core".out 2>&1 if ! ls nemo_"$core".best_cut.prv; then -- GitLab From a8f9412a9180812b2857239f4d57802126452451 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Wed, 14 Sep 2022 11:50:42 +0200 Subject: [PATCH 09/20] Now all the output is located in Output dir --- perf_metrics.config | 13 +++-- src/functions.bash | 113 +++++++++++++++++++++++++------------------- 2 files changed, 73 insertions(+), 53 deletions(-) diff --git a/perf_metrics.config b/perf_metrics.config index cccfd7a..16fa533 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -3,15 +3,18 @@ ################################################################################# +# Output (string): Path where the Output dir, containing all the output files, will be created. + +Output=".." + # Nemo_path (string) : Path to nemo installation folder containing the cfgs and arch dirs. # Nemo_input_data (string): Path to the input data needed to run the nemo cfg. # Nemo_run (string): Path where the folder Run_NEMO will be created. # Nemo_cores (array): List of nºcores used for executing Nemo, ( 4 48 ) makes the script execute and # get Nemo traces with 4 and 48 cores. 2 different nºcores are needed to obtain scalability data. -Nemo_path="Nemo_installation_path" -Nemo_input_data="Nemo_input_data_path" -Nemo_run="." +Nemo_path="NEMO_INSTALLATION_PATH" +Nemo_input_data="NEMO_INPUT_DATA_PATH" Nemo_cores=( 4 24 48 96 192) # Jobs_n_cores (integer): nºcores used for other jobs like compiling nemo. @@ -20,7 +23,7 @@ Nemo_cores=( 4 24 48 96 192) # Jobs_time (integer): Max duration of the job in min. # Jobs_queue (string): Queue used. -Jobs_n_cores= 4 +Jobs_n_cores=4 Jobs_cores_per_node= Jobs_scheduler="slurm" Jobs_time=60 @@ -35,7 +38,7 @@ Jobs_queue="debug" Compilation_compile="false" Compilation_ref="ORCA2_ICE_PISCES" -Compilation_arch="Your-arch-file" +Compilation_arch="YOUR_ARCH_FILE" Compilation_name="ORCA2_EXTRAE" Compilation_sub="OCE del_key 'key_si3 key_top'" diff --git a/src/functions.bash b/src/functions.bash index 34ded5d..a73d802 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -125,7 +125,7 @@ Compile_extrae() echo Job_completed "$state1" if [ $Completed == false ]; then - echo "Nemo compilation failed, remember to load all the needed modules. Check the details in logs/compile_extrae.err" + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in ${logs_path}/compile_extrae.err" echo exit 1 else @@ -133,7 +133,7 @@ Compile_extrae() echo fi - mv compile_extrae.* "$dir"/logs + mv compile_extrae.* "${logs_path}" else echo "Compilation not needed" @@ -142,10 +142,10 @@ Compile_extrae() #Copy all the EXP00 data but don't overwrite namelist just the executable - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* "${Nemo_run}"/Run_NEMO - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Nemo_run}"/Run_NEMO + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* "${Run_path}" + cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Run_path}" - cd "${Nemo_run}"/Run_NEMO||( echo "Error ${Nemo_run}/Run_NEMO folder doesn't exists"; exit 1 ) + cd "${Run_path}"||( echo "Error ${Run_path} folder doesn't exists"; exit 1 ) } #Check if Nemo is compiled for using Gprof @@ -215,7 +215,7 @@ Compile_gprof() echo Job_completed "$state1" if [ "$Completed" == false ]; then - echo "Nemo compilation failed, remember to load all the needed modules. Check the details in logs/compile_gprof.err" + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in ${logs_path}/compile_gprof.err" echo exit 1 else @@ -223,7 +223,7 @@ Compile_gprof() echo fi - mv compile_gprof.* "$dir"/logs + mv compile_gprof.* "${logs_path}" else echo "Compilation not needed" @@ -233,25 +233,25 @@ Compile_gprof() #Copy all the EXP00 data in RUN_NEMO folder but don't overwrite the namelist, just the executable. - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/* "${Nemo_run}"/Run_NEMO - cp "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/nemo "${Nemo_run}"/Run_NEMO + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/* "${Run_path}" + cp "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/nemo "${Run_path}" if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then - sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' "${Nemo_run}"/Run_NEMO/context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) + sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' "${Run_path}"/context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) fi #Solving NEMO input file common errors - if test -f "${Nemo_run}/Run_NEMO/weights_core_orca2_bicubic_noc.nc"; then - mv "${Nemo_run}"/Run_NEMO/weights_core_orca2_bicubic_noc.nc "${Nemo_run}"/Run_NEMO/weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES + if test -f "${Run_path}/weights_core_orca2_bicubic_noc.nc"; then + mv "${Run_path}"/weights_core_orca2_bicubic_noc.nc "${Run_path}"/weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES fi - if test -f "${Nemo_run}/Run_NEMO/weights_core_orca2_bilinear_noc.nc"; then - mv "${Nemo_run}"/Run_NEMO/weights_core_orca2_bilinear_noc.nc "${Nemo_run}"/Run_NEMO/weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES + if test -f "${Run_path}/weights_core_orca2_bilinear_noc.nc"; then + mv "${Run_path}"/weights_core_orca2_bilinear_noc.nc "${Run_path}"/weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES fi - cd "${Nemo_run}"/Run_NEMO||(echo "Error ${Nemo_run}/Run_NEMO folder doesn't exists"; exit 1) + cd "${Run_path}"||(echo "Error ${Run_path} folder doesn't exists"; exit 1) } @@ -262,9 +262,9 @@ Compile_gprof() Init() { + Output="${Output=:-"../"}" Nemo_path="${Nemo_path:-"./"}" Nemo_input_data="${Nemo_input_data=:-"./"}" - Nemo_run="${Nemo_run=:-"./"}" Nemo_cores="${Nemo_cores:-( 48 )}" Jobs_n_cores="${Jobs_n_cores:-48}" Jobs_cores_per_node="${Jobs_cores_per_node:-0}" @@ -287,6 +287,23 @@ Init() Test_arguments() { + + # Does Output path exist? + if ! test -d "${Output}"; then + echo "Output path: ${Output} is not found, modify it in perf_metrics.config with the correct path" + echo + exit 1 + fi + + Output_dir="${Output}/Output" + + if ! test -d "${Output_dir}"; then + + mkdir "${Output_dir}" + + fi + + # Nemo path correct? if ! test -d "${Nemo_path}"; then echo "Nemo relative path: ${Nemo_path} is not found, modify it in perf_metrics.config with the correct path" @@ -301,18 +318,14 @@ Test_arguments() echo exit 1 fi - # Is Nemo Run path correct? - if ! test -d "${Nemo_run}"; then - echo "Nemo run path: ${Nemo_run} is not found, modify it in perf_metrics.config with the correct path" - echo - exit 1 - fi - + + Run_path="${Output_dir}/Run_NEMO" # Create Run_Nemo dir after comprovations - if ! test -d "${Nemo_run}/Run_NEMO"; then - mkdir "${Nemo_run}"/Run_NEMO + if ! test -d "${Run_path}"; then + + mkdir "${Run_path}" fi - cp -P "${Nemo_input_data}"/* "${Nemo_run}"/Run_NEMO + cp -P "${Nemo_input_data}"/* "${Run_path}" #Nemo_cores is array? @@ -394,9 +407,11 @@ Test_arguments() sed -i "s|export EXTRAE_CONFIG_FILE=.*|export EXTRAE_CONFIG_FILE=$dir/src/./extrae.xml|g" src/trace.sh + logs_path="${Output_dir}/logs" # Creating logs dir - if ! test -d "logs"; then - mkdir logs + if ! test -d "${logs_path}"; then + + mkdir "$logs_path" fi @@ -408,7 +423,7 @@ Test_arguments() Gprof_functions() { - cd "${Nemo_run}"/Run_NEMO||(echo "Error ${Nemo_run}/Run_NEMO folder doesn't exists"; exit 1) + cd "${Run_path}"||(echo "Error ${Run_path} folder doesn't exists"; exit 1) #Generating function list in case of missing if ! test -f "extrae_functions.txt"; then @@ -426,7 +441,7 @@ Gprof_functions() fi Job_completed "$state2" if [ $Completed == false ]; then - echo "Nemo execution failed look at logs/run.err and ${Nemo_run}/Run_NEMO/ocean.output for more info" + echo "Nemo execution failed look at ${logs_path}/run.err and ${Run_path}/ocean.output for more info" echo "Remember that the namelist files copied are the default ones, change them in order to fit with the input files in the dir " echo exit 1 @@ -434,7 +449,7 @@ Gprof_functions() echo "Gprof files generated " echo fi - mv run.* "$dir"/logs + mv run.* "${logs_path}" echo "Gthrottling functions ..." echo @@ -444,7 +459,7 @@ Gprof_functions() fi Job_completed "$state3" if [ $Completed == false ]; then - echo "Error listing functions, look at logs/gthrottling.err for more info" + echo "Error listing functions, look at ${logs_path}/gthrottling.err for more info" echo exit 1 else @@ -452,10 +467,10 @@ Gprof_functions() echo fi - mv gthrottling.* "$dir"/logs + mv gthrottling.* "${logs_path}" else - echo "Functions already listed, file ${Nemo_run}/Run_NEMO/extrae_functions.txt does exist" + echo "Functions already listed, file ${Run_path}/extrae_functions.txt does exist" echo fi @@ -472,7 +487,7 @@ Compile_extrae # Run nemo with extrae "$dir"/src/./extraf.sh nemo extrae_functions.txt - sed -i "s|list=.*|list=\"${Nemo_run}/Run_NEMO/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" "$dir"/src/extrae.xml + sed -i "s|list=.*|list=\"${Run_path}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" "$dir"/src/extrae.xml for core in "${Nemo_cores[@]}" do @@ -486,31 +501,33 @@ Compile_extrae fi Job_completed "$state4" if [ $Completed == false ]; then - echo "Nemo execution failed, no traces files generated more info inside logs/run_extrae_$core.err" + echo "Nemo execution failed, no traces files generated more info inside ${logs_path}/run_extrae_$core.err" echo exit 1 fi - mv run_extrae_"$core".* "$dir"/logs + mv run_extrae_"$core".* "${logs_path}" mv nemo.prv nemo_"$core".prv mv nemo.pcf nemo_"$core".pcf mv nemo.row nemo_"$core".row echo "Cutting up best iteration" echo - "$dir"/src/magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > "$dir"/logs/cut_"$core".out 2>&1 + "$dir"/src/magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > "${logs_path}"/cut_"$core".out 2>&1 if ! ls nemo_"$core".best_cut.prv; then - echo "Cut failed, look at ${dir}/logs/cut_$core.out for more info." + echo "Cut failed, look at "${logs_path}"/cut_$core.out for more info." echo exit 1 fi echo # Creating folder - if ! test -d "$dir/Metrics"; then - mkdir "$dir"/Metrics + Metrics_path="${Output_dir}/Metrics" + + if ! test -d "${Metrics_path}"; then + mkdir "${Metrics_path}" fi - cp nemo_"$core".best_cut.* "$dir"/Metrics + cp nemo_"$core".best_cut.* "${Metrics_path}" @@ -524,27 +541,27 @@ Create_metrics() cd "$dir" || (echo "Error original dir doesn't exist"; exit 1) # Create performance metrics - echo "Creating metrics and storing them in Metrics folder" + echo "Creating metrics and storing them in ${Metrics_path} folder" echo python3 src/./Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./modelfactors.py -ms 100000 *" - mv analysis."$Jobs_scheduler" Metrics - cd Metrics||(echo "Error Metrics folder doesn't exists"; exit 1) + mv analysis."$Jobs_scheduler" "${Metrics_path}" + cd "${Metrics_path}"||(echo "Error ${Metrics_path} folder doesn't exists"; exit 1) if ! state5=$("$job" --wait analysis."$Jobs_scheduler"); then exit 1 fi Job_completed "$state5" if [ $Completed == false ]; then - echo "Error, metrics have not generated check logs/analysis.err to get more details" + echo "Error, metrics have not generated check ${logs_path}/analysis.err to get more details" echo exit 1 fi cp analysis.out overview.txt - mv analysis.* "$dir"/logs + mv analysis.* "${logs_path}" echo "------------------------------------------------------------------------------" echo "------------------------- Script Completed -----------------------------------" - echo "----------------------- Data in Metrics folder -------------------------------" + echo "----------------------- Data in ${Metrics_path} folder -------------------------------" echo "------------------------------------------------------------------------------" echo "------------------------------------------------------------------------------" echo -- GitLab From 0d1bf7997d40ebfd316e86999d5e442e35781807 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Thu, 15 Sep 2022 09:39:15 +0200 Subject: [PATCH 10/20] Code cleaned, relative path now work, fixed bugs --- perf_metrics.bash | 3 ++ src/functions.bash | 105 ++++++++++++++++++++++++--------------------- 2 files changed, 60 insertions(+), 48 deletions(-) diff --git a/perf_metrics.bash b/perf_metrics.bash index 6f28d98..b427da1 100755 --- a/perf_metrics.bash +++ b/perf_metrics.bash @@ -33,12 +33,15 @@ Init #Test if parameters are valid Test_arguments +cd "${Run_path}"||(echo "Error ${Run_path} folder doesn't exists"; exit 1) + #Create the list of important functions from NEMO Gprof_functions #Get the traces of the executions and cut 1 timestep Get_trace +cd "${Metrics_path}"||(echo "Error ${Metrics_path} folder doesn't exists"; exit 1) #Generate the performance metrics Create_metrics diff --git a/src/functions.bash b/src/functions.bash index a73d802..a4a442a 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -40,8 +40,7 @@ Job_completed() Compile_extrae() { - cd "$dir" || (echo "Error original dir doesn't exist"; exit 1) - + #Get flag lines line=$(sed -n '/^%FCFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) @@ -114,8 +113,8 @@ Compile_extrae() echo "Compiling Nemo for EXTRAE" - printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" - python3 src/./Job_Creator.py -f "compile_extrae" -j "compile_extrae" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + printf -v workload1 "cd ${Nemo_path}; ./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg; cd ${Run_path}" + python3 "$dir"/src/./Job_Creator.py -f "compile_extrae" -j "compile_extrae" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" if ! state1=$("$job" --wait compile_extrae."$Jobs_scheduler"); then @@ -145,7 +144,7 @@ Compile_extrae() cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* "${Run_path}" cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Run_path}" - cd "${Run_path}"||( echo "Error ${Run_path} folder doesn't exists"; exit 1 ) + } #Check if Nemo is compiled for using Gprof @@ -153,8 +152,6 @@ Compile_extrae() Compile_gprof() { - cd "$dir" || (echo "Error original dir doesn't exist"; exit 1) - if [ "$compile" == true ]; then echo 'compile parameter is inicialized true' fi @@ -205,8 +202,8 @@ Compile_gprof() if [ "$compile" == true ] || [ "$comp_gprof" == true ]; then echo "Compiling Nemo for GPROF" - printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg" - python3 src/./Job_Creator.py -f "compile_gprof" -j "compile_gprof" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + printf -v workload1 "cd ${Nemo_path}; ./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg; cd ${Run_path};" + python3 "$dir"/src/./Job_Creator.py -f "compile_gprof" -j "compile_gprof" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" if ! state1=$("$job" --wait compile_gprof."$Jobs_scheduler"); then exit 1 @@ -251,7 +248,7 @@ Compile_gprof() mv "${Run_path}"/weights_core_orca2_bilinear_noc.nc "${Run_path}"/weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES fi - cd "${Run_path}"||(echo "Error ${Run_path} folder doesn't exists"; exit 1) + } @@ -288,44 +285,43 @@ Init() Test_arguments() { + # Does Output path exist? if ! test -d "${Output}"; then echo "Output path: ${Output} is not found, modify it in perf_metrics.config with the correct path" echo exit 1 + # Transform to absolute path + else + cd "${Output}" + Output=$(pwd) + cd "$dir" fi - Output_dir="${Output}/Output" - - if ! test -d "${Output_dir}"; then - - mkdir "${Output_dir}" - - fi - - # Nemo path correct? if ! test -d "${Nemo_path}"; then echo "Nemo relative path: ${Nemo_path} is not found, modify it in perf_metrics.config with the correct path" echo exit 1 + # Transform to absolute path + else + cd "${Nemo_path}" + Nemo_path=$(pwd) + cd "$dir" fi - - # Is Nemo data path correct? if ! test -d "${Nemo_input_data}"; then echo "Nemo input data path: ${Nemo_input_data} is not found, modify it in perf_metrics.config with the correct path" echo exit 1 + # Transform to absolute path + else + cd "${Nemo_input_data}" + Nemo_input_data=$(pwd) + cd "$dir" fi - Run_path="${Output_dir}/Run_NEMO" - # Create Run_Nemo dir after comprovations - if ! test -d "${Run_path}"; then - - mkdir "${Run_path}" - fi - cp -P "${Nemo_input_data}"/* "${Run_path}" + #Nemo_cores is array? @@ -407,24 +403,43 @@ Test_arguments() sed -i "s|export EXTRAE_CONFIG_FILE=.*|export EXTRAE_CONFIG_FILE=$dir/src/./extrae.xml|g" src/trace.sh + + #Creating output folder after comprovations + + Output_dir="${Output}/Output" + mkdir -p "${Output_dir}" + + + # Creating logs folder + logs_path="${Output_dir}/logs" - # Creating logs dir - if ! test -d "${logs_path}"; then - - mkdir "$logs_path" + mkdir -p "$logs_path" - fi + + + + # Create Run_Nemo folder + + Run_path="${Output_dir}/Run_NEMO" + mkdir -p "${Run_path}" + cp -P "${Nemo_input_data}"/* "${Run_path}" + + + # Creating Metricsfolder + + Metrics_path="${Output_dir}/Metrics" + mkdir -p "${Metrics_path}" } + + # Generates a list of Nemo functions Gprof_functions() { - cd "${Run_path}"||(echo "Error ${Run_path} folder doesn't exists"; exit 1) - #Generating function list in case of missing if ! test -f "extrae_functions.txt"; then @@ -486,6 +501,7 @@ Get_trace() Compile_extrae # Run nemo with extrae + "$dir"/src/./extraf.sh nemo extrae_functions.txt sed -i "s|list=.*|list=\"${Run_path}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" "$dir"/src/extrae.xml for core in "${Nemo_cores[@]}" @@ -515,18 +531,13 @@ Compile_extrae echo "$dir"/src/magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > "${logs_path}"/cut_"$core".out 2>&1 if ! ls nemo_"$core".best_cut.prv; then - echo "Cut failed, look at "${logs_path}"/cut_$core.out for more info." + echo "Cut failed, look at ${logs_path}/cut_$core.out for more info." echo exit 1 fi echo - # Creating folder - Metrics_path="${Output_dir}/Metrics" - - if ! test -d "${Metrics_path}"; then - mkdir "${Metrics_path}" - fi + cp nemo_"$core".best_cut.* "${Metrics_path}" @@ -538,14 +549,12 @@ Compile_extrae Create_metrics() { - - cd "$dir" || (echo "Error original dir doesn't exist"; exit 1) # Create performance metrics echo "Creating metrics and storing them in ${Metrics_path} folder" echo - python3 src/./Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./modelfactors.py -ms 100000 *" - mv analysis."$Jobs_scheduler" "${Metrics_path}" - cd "${Metrics_path}"||(echo "Error ${Metrics_path} folder doesn't exists"; exit 1) + python3 "$dir"/src/./Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./modelfactors.py -ms 100000 *" + + if ! state5=$("$job" --wait analysis."$Jobs_scheduler"); then exit 1 fi @@ -561,7 +570,7 @@ Create_metrics() mv analysis.* "${logs_path}" echo "------------------------------------------------------------------------------" echo "------------------------- Script Completed -----------------------------------" - echo "----------------------- Data in ${Metrics_path} folder -------------------------------" + echo "--- Data in ${Metrics_path} folder ---" echo "------------------------------------------------------------------------------" echo "------------------------------------------------------------------------------" echo -- GitLab From 2cd586d4ca0dde58e0a97773e7a70db445e2606f Mon Sep 17 00:00:00 2001 From: cpenadep Date: Thu, 15 Sep 2022 09:50:17 +0200 Subject: [PATCH 11/20] Now gthrottling forks the for to go quicker. --- src/gthrottling.sh | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/src/gthrottling.sh b/src/gthrottling.sh index 27c4f50..e91d785 100755 --- a/src/gthrottling.sh +++ b/src/gthrottling.sh @@ -3,27 +3,26 @@ # Usage: ./extract_gprof.sh path/to/executable/executable # Output file: extrae_functions.txt -rm gprof_functions -rm suspected_functions_names_only -rm suspected_functions -rm extrae_functions.txt # nm tool lists the symbols from objects and we select the ones with type T|t which the type is in the text section nm $1 | grep -i " T " | awk '{print $3}' > function_names.txt echo "See the function names from the binary in the file function_names.txt" dir=$(dirname $1) -for i in `ls $dir/gmon*`; -do +analyze_gmon() + + { + local n=$(echo "$2" | sed 's/[^0-9]*//g') + local i=$2 echo -e "Analyzing "$i"\n" - gprof $1 $i >gprof_temp + gprof $1 $i >gprof_temp_"$n" #We extract from each gprof file only the part about the functions, number of calls and durations, the call-paths ar enot needed - cat gprof_temp | grep -v ":" | awk 'BEGIN{k=0}{if($1=="%") {k=k+1};if(k>0 && k<2 && $1==$1+0) print $0}' > temp + cat gprof_temp_"$n" | grep -v ":" | awk 'BEGIN{k=0}{if($1=="%") {k=k+1};if(k>0 && k<2 && $1==$1+0) print $0}' > temp_"$n" #We save the name of the functions - cat temp | awk '{if($7~/^$/) print $4;else print $7}' > gprof_functions + cat temp_"$n" | awk '{if($7~/^$/) print $4;else print $7}' > gprof_functions_"$n" #From the initial list we save only the ones that gprof files include - cat function_names.txt | grep -w -f gprof_functions > extrae_new_list + cat function_names.txt | grep -w -f gprof_functions_"$n" > extrae_new_list_"$n" #We apply the throttling rule: # 1) If there is no information about each call of a function is suspected if the total duration is less than 0.1% of the total execution time @@ -31,24 +30,34 @@ do # 2.1) If its duration is less or equal to 5% ($1<=5, you can change them according to your application) of the total execution time # 2.2) If the duration of each call is less than 0.001s, then exclude it # 3) If the total execution time of this function is 0.0%, then remove it - cat temp | awk '{if($7~/^$/ && $1<0.1) print $4" "$1; else if(NF==7 && $4>10000 && (($1<=5 || $5<=0.001)) || $1==0.0) print $7" "$4}' >> suspected_functions - awk '{print $1}' suspected_functions >> suspected_functions_names_only + cat temp_"$n" | awk '{if($7~/^$/ && $1<0.1) print $4" "$1; else if(NF==7 && $4>10000 && (($1<=5 || $5<=0.001)) || $1==0.0) print $7" "$4}' >> suspected_functions_"$n" + awk '{print $1}' suspected_functions_"$n" >> suspected_functions_names_only_"$n" # Sort and remove any double functions from the list with suspected functions - cat suspected_functions_names_only | sort | uniq > temp_file - mv temp_file suspected_functions_names_only + cat suspected_functions_names_only_"$n" | sort | uniq > temp_file_"$n" + mv temp_file_"$n" suspected_functions_names_only_"$n" # Create a new fucntion list with the non suspected functions - cat extrae_new_list | grep -w -v -f suspected_functions_names_only >> extrae_functions.txt -done + cat extrae_new_list_"$n" | grep -w -v -f suspected_functions_names_only_"$n" >> extrae_functions.txt + + rm extrae_new_list_"$n" + rm suspected_functions_names_only_"$n" + rm suspected_functions_"$n" + rm gprof_temp_"$n" + rm temp_"$n" + rm gprof_functions_"$n" + +} +for i in `ls $dir/gmon*`; do analyze_gmon "$1" "$i" & done + +wait #Sort and uniw the useful functions because are called from many processors and can by include twice cat extrae_functions.txt | sort | uniq > temp2 mv temp2 extrae_functions.txt -rm temp echo -e "Input function list: "function_names.txt" "`wc -l function_names.txt | awk '{print $1}'`" functions" echo -e "New function list: extrae_functions.txt "`wc -l extrae_functions.txt | awk '{print $1}'`" functions" - +rm function_names.txt exit -- GitLab From ea4a59a7bfeb8451c25f1671c212cde2fadf1000 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Thu, 15 Sep 2022 10:55:32 +0200 Subject: [PATCH 12/20] Extra clarification --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index abf8159..f98650c 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,9 @@ Here the list of the modules that need to be loaded before the script execution. # Usage * Clone this repository wherever you please. -* Edit the file perf_metrics.config and replace the parameters values with the suited information. -* MINIMUM CHANGES perf_metrics.config : +* Inside the cloned repository use `git submodule update --init` to load the submodules. +* ***Edit the file perf_metrics.config and replace the parameters values with the suited information.*** +* ***MINIMUM CHANGES perf_metrics.config:*** * Nemo_path, change the value to the path were NEMO is installed in your machine. * Nemo_input_data, change the value to the path were the input data for the configuration is downloaded. * Compilation_arch, replace the value with the name of the arch file that you use to compile NEMO. @@ -38,4 +39,4 @@ Here the list of the modules that need to be loaded before the script execution. ``` ./perf_metrics.bash ``` -* If the script executes without problems the data will be ready in the Metrics folder. +* If the script executes without problems the data will be by default ready inside ../Output/Metrics folder. The Output dir path can be changed at perf_metrics.config. -- GitLab From e182f5aa477fdf0c65842bdd291e63ff1f9c0e9a Mon Sep 17 00:00:00 2001 From: cpenadep Date: Fri, 16 Sep 2022 10:35:11 +0200 Subject: [PATCH 13/20] Made the for to obtain traces parallel. Now submodules load at the beginning of the script --- README.md | 3 +-- perf_metrics.bash | 3 +++ src/functions.bash | 7 +++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f98650c..006aed1 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,6 @@ Here the list of the modules that need to be loaded before the script execution. # Usage * Clone this repository wherever you please. -* Inside the cloned repository use `git submodule update --init` to load the submodules. * ***Edit the file perf_metrics.config and replace the parameters values with the suited information.*** * ***MINIMUM CHANGES perf_metrics.config:*** * Nemo_path, change the value to the path were NEMO is installed in your machine. @@ -35,7 +34,7 @@ Here the list of the modules that need to be loaded before the script execution. * Compilation_arch, replace the value with the name of the arch file that you use to compile NEMO. * Modules, change the value to suit the name of the modules you need to load. * Jobs_scheduler, replace the value with the name of the scheduler installed in your machine (currently supports slurm, lsf and torque) -* Execute perf_metrics.bash +* Execute perf_metrics.bash inside the git repository ``` ./perf_metrics.bash ``` diff --git a/perf_metrics.bash b/perf_metrics.bash index b427da1..3520439 100755 --- a/perf_metrics.bash +++ b/perf_metrics.bash @@ -10,6 +10,9 @@ if [ $# -gt 0 ]; then fi + +#Load submodules +git submodule update --init #Get script directory dir=$(pwd) diff --git a/src/functions.bash b/src/functions.bash index a4a442a..03d17d5 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -507,7 +507,7 @@ Compile_extrae for core in "${Nemo_cores[@]}" do - + ( echo "Creating trace with $core cores..." echo python3 "$dir"/src/./Job_Creator.py -f "run_extrae_$core" -j "run_extrae_$core" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core "$dir"/src/./trace.sh ./nemo" @@ -536,14 +536,13 @@ Compile_extrae exit 1 fi echo - - cp nemo_"$core".best_cut.* "${Metrics_path}" - + )& done + wait } Create_metrics() -- GitLab From d711dab43daa2f17ad276cf87afe7a932ad84ed9 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Fri, 16 Sep 2022 10:47:32 +0200 Subject: [PATCH 14/20] Expanded explanation --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 006aed1..ade9efd 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Here the list of the modules that need to be loaded before the script execution. # Usage * Clone this repository wherever you please. +* Don't move the content of the repository outside , the sub-modules won't load and the script will fail. If you want you can init the sub-modules manually with `git submodule update --init` and then move the content. * ***Edit the file perf_metrics.config and replace the parameters values with the suited information.*** * ***MINIMUM CHANGES perf_metrics.config:*** * Nemo_path, change the value to the path were NEMO is installed in your machine. @@ -34,7 +35,7 @@ Here the list of the modules that need to be loaded before the script execution. * Compilation_arch, replace the value with the name of the arch file that you use to compile NEMO. * Modules, change the value to suit the name of the modules you need to load. * Jobs_scheduler, replace the value with the name of the scheduler installed in your machine (currently supports slurm, lsf and torque) -* Execute perf_metrics.bash inside the git repository +* Execute perf_metrics.bash ``` ./perf_metrics.bash ``` -- GitLab From 36743e5351ac938d8440433ef01084a53916e8f1 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Mon, 19 Sep 2022 12:01:43 +0200 Subject: [PATCH 15/20] link instead of copy, dir for gprof, deleted outputs --- perf_metrics.bash | 5 ++- src/functions.bash | 105 ++++++++++++++++++++++++++------------------- 2 files changed, 65 insertions(+), 45 deletions(-) diff --git a/perf_metrics.bash b/perf_metrics.bash index 3520439..f9dd8cb 100755 --- a/perf_metrics.bash +++ b/perf_metrics.bash @@ -36,11 +36,12 @@ Init #Test if parameters are valid Test_arguments -cd "${Run_path}"||(echo "Error ${Run_path} folder doesn't exists"; exit 1) +cd "${Gprof_path}"||(echo "Error ${Gprof_path} folder doesn't exists"; exit 1) #Create the list of important functions from NEMO -Gprof_functions +Gprof_functions & +cd "${Run_path}"||(echo "Error ${Run_path} folder doesn't exists"; exit 1) #Get the traces of the executions and cut 1 timestep Get_trace diff --git a/src/functions.bash b/src/functions.bash index 03d17d5..d5b8259 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -6,7 +6,10 @@ Job_completed() if [ "$Jobs_scheduler" == "slurm" ]; then local id1 id1=${1##* } - sleep 5 + until ! scontrol show job "$id1" | grep -q 'JobState=COMPLETING' + do + sleep 1 + done if ! scontrol show job "$id1" | grep -q 'JobState=COMPLETED'; then Completed=false else @@ -16,7 +19,6 @@ Job_completed() elif [ "$Jobs_scheduler" == "lsf" ]; then local id2 id2=$(head -n1 "$1" | cut -d'<' -f2 | cut -d'>' -f1) - sleep 5 if ! bjobs -l "$id2" | grep -q 'Status '; then Completed=false else @@ -25,7 +27,6 @@ Job_completed() elif [ "$Jobs_scheduler" == "torque" ]; then local id3 id3=$(head -n1 "$1" | awk '{ print $3 }') - sleep 5 if ! qstat f "$id3" | grep -q 'exit_status = 0'; then Completed=false else @@ -117,22 +118,23 @@ Compile_extrae() python3 "$dir"/src/./Job_Creator.py -f "compile_extrae" -j "compile_extrae" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" - if ! state1=$("$job" --wait compile_extrae."$Jobs_scheduler"); then + if ! state1=$("$job" "$wait" compile_extrae."$Jobs_scheduler"); then exit 1: fi echo Job_completed "$state1" + mv compile_extrae.* "${logs_path}" if [ $Completed == false ]; then echo "Nemo compilation failed, remember to load all the needed modules. Check the details in ${logs_path}/compile_extrae.err" echo exit 1 else - echo "Nemo compilation successful" + echo "Nemo Extrae compilation successful" echo fi - mv compile_extrae.* "${logs_path}" + else echo "Compilation not needed" @@ -142,9 +144,17 @@ Compile_extrae() #Copy all the EXP00 data but don't overwrite namelist just the executable cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* "${Run_path}" - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Run_path}" + cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Run_path}" + + + + if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then + sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' "${Run_path}"/context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) + fi + + sed -i '/|g" "$dir"/src/extrae.xml + for core in "${Nemo_cores[@]}" do @@ -512,24 +527,27 @@ Compile_extrae echo python3 "$dir"/src/./Job_Creator.py -f "run_extrae_$core" -j "run_extrae_$core" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core "$dir"/src/./trace.sh ./nemo" - if ! state4=$("$job" --wait run_extrae_$core."$Jobs_scheduler") ; then + if ! state4=$("$job" "$wait" run_extrae_$core."$Jobs_scheduler") ; then exit 1 fi Job_completed "$state4" + mv run_extrae_"$core".* "${logs_path}" + if [ $Completed == false ]; then echo "Nemo execution failed, no traces files generated more info inside ${logs_path}/run_extrae_$core.err" echo exit 1 fi - mv run_extrae_"$core".* "${logs_path}" - + mv nemo.prv nemo_"$core".prv mv nemo.pcf nemo_"$core".pcf mv nemo.row nemo_"$core".row echo "Cutting up best iteration" echo "$dir"/src/magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > "${logs_path}"/cut_"$core".out 2>&1 + + if ! ls nemo_"$core".best_cut.prv; then echo "Cut failed, look at ${logs_path}/cut_$core.out for more info." echo @@ -554,19 +572,20 @@ Create_metrics() python3 "$dir"/src/./Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./modelfactors.py -ms 100000 *" - if ! state5=$("$job" --wait analysis."$Jobs_scheduler"); then + if ! state5=$("$job" "$wait" analysis."$Jobs_scheduler"); then exit 1 fi Job_completed "$state5" + cp analysis.out overview.txt + mv analysis.* "${logs_path}" if [ $Completed == false ]; then echo "Error, metrics have not generated check ${logs_path}/analysis.err to get more details" echo exit 1 fi - cp analysis.out overview.txt - mv analysis.* "${logs_path}" + echo "------------------------------------------------------------------------------" echo "------------------------- Script Completed -----------------------------------" echo "--- Data in ${Metrics_path} folder ---" -- GitLab From fe9888e1327305bbb5f2e95b1d40c54880009d37 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 20 Sep 2022 12:12:49 +0200 Subject: [PATCH 16/20] Now dir for every execution are created, run files are deleted after finishing --- perf_metrics.config | 12 ++++-- src/functions.bash | 90 ++++++++++++++++++++++----------------------- 2 files changed, 52 insertions(+), 50 deletions(-) diff --git a/perf_metrics.config b/perf_metrics.config index 16fa533..59ee9cc 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -27,21 +27,25 @@ Jobs_n_cores=4 Jobs_cores_per_node= Jobs_scheduler="slurm" Jobs_time=60 -Jobs_queue="debug" +Jobs_queue= -# Compilation_compile (string): When false only compiles NEMO if arch file lacks the needed flags, when true always compiles NEMO. +# Compilation_compile (boolean): When false only compiles NEMO if arch file lacks the needed flags, when true always compiles NEMO. # Compilation_ref (string): Reference configuration. # Compilation_arch (string): Architecture used (without the -arch suffix and the .fcm). # Compilation_name (string): Name of the new configuration (Important to not be an existing one). # Compilation_sub (string): Add or remove sub-components. -Compilation_compile="false" +Compilation_compile=false Compilation_ref="ORCA2_ICE_PISCES" Compilation_arch="YOUR_ARCH_FILE" Compilation_name="ORCA2_EXTRAE" Compilation_sub="OCE del_key 'key_si3 key_top'" - + +# Clean (boolean): If true, at the end of the script, all residual files from NEMO executions (data, outputs, executable, folders) are deleted. + +Clean=true + # Modules (string): List of modules loaded. # Required: # - Perl interpreter diff --git a/src/functions.bash b/src/functions.bash index d5b8259..e7a1c47 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -141,19 +141,7 @@ Compile_extrae() echo fi - #Copy all the EXP00 data but don't overwrite namelist just the executable - - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* "${Run_path}" - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Run_path}" - - - - if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then - sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' "${Run_path}"/context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) - fi - - sed -i '/ /dev/null sed -i "s|list=.*|list=\"${Run_path}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" "$dir"/src/extrae.xml + # Change iterations + sed -i "s|nn_itend * =.*|nn_itend = $Nemo_iterations ! last time step (std 5475)|g" "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/namelist_cfg + if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then + sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) + fi + sed -i '/ "${logs_path}"/cut_"$core".out 2>&1 - if ! ls nemo_"$core".best_cut.prv; then echo "Cut failed, look at ${logs_path}/cut_$core.out for more info." echo @@ -555,9 +545,9 @@ Get_trace() fi echo cp nemo_"$core".best_cut.* "${Metrics_path}" + cd "$Run_path" )& - - + done wait @@ -586,6 +576,14 @@ Create_metrics() fi + # Removing run folders + if [ $Clean == true ]; then + rm -r -f "$Run_path" + mv "${Gprof_path}"/extrae_functions* "$dir" + rm -r -f "$Gprof_path"/* + mv "${dir}"/extrae_functions* "${Gprof_path}" + fi + echo "------------------------------------------------------------------------------" echo "------------------------- Script Completed -----------------------------------" echo "--- Data in ${Metrics_path} folder ---" -- GitLab From 5e96722bb6a86e9e803ad378db064f8a55c21844 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 20 Sep 2022 12:33:39 +0200 Subject: [PATCH 17/20] Fixed naming mistake --- src/functions.bash | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/functions.bash b/src/functions.bash index e7a1c47..f21ba9e 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -427,7 +427,7 @@ Gprof_functions() { #Generating function list in case of missing - if ! test -f "extrae_functions_${cfg}.txt"; then + if ! test -f "extrae_functions_${name_cfg}.txt"; then Compile_gprof #Changing iterations, big traces generate problems. @@ -466,14 +466,14 @@ Gprof_functions() echo exit 1 else - mv extrae_functions.txt extrae_functions_"${cfg}".txt + mv extrae_functions.txt extrae_functions_"${name_cfg}".txt echo "Functions listed correctly" echo fi else - echo "Functions already listed, file ${Gprof_path}/extrae_functions_${cfg}.txt does exist." + echo "Functions already listed, file ${Gprof_path}/extrae_functions_${name_cfg}.txt does exist." echo fi @@ -488,7 +488,7 @@ Get_trace() Compile_extrae wait - "$dir"/src/./extraf.sh "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Gprof_path}"/extrae_functions_"${cfg}".txt > /dev/null + "$dir"/src/./extraf.sh "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Gprof_path}"/extrae_functions_"${name_cfg}".txt > /dev/null sed -i "s|list=.*|list=\"${Run_path}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" "$dir"/src/extrae.xml # Change iterations -- GitLab From 44cb3e215ddd61c403810f70da90144d1abad834 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 20 Sep 2022 14:41:28 +0200 Subject: [PATCH 18/20] Now exit terminates all the processes --- perf_metrics.bash | 2 ++ src/functions.bash | 14 ++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/perf_metrics.bash b/perf_metrics.bash index f9dd8cb..9470d4a 100755 --- a/perf_metrics.bash +++ b/perf_metrics.bash @@ -52,3 +52,5 @@ Create_metrics } main "$@"; exit + + diff --git a/src/functions.bash b/src/functions.bash index f21ba9e..95792cc 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -1,3 +1,5 @@ + + # Functions #Checks if the job submission ended correctly. @@ -41,7 +43,8 @@ Job_completed() Compile_extrae() { - + trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT + #Get flag lines line=$(sed -n '/^%FCFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) @@ -149,6 +152,7 @@ Compile_extrae() Compile_gprof() { + trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT if [ "$compile" == true ]; then echo 'compile parameter is inicialized true' @@ -425,7 +429,7 @@ Test_arguments() Gprof_functions() { - + trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT #Generating function list in case of missing if ! test -f "extrae_functions_${name_cfg}.txt"; then @@ -484,7 +488,7 @@ Gprof_functions() Get_trace() { - + trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT Compile_extrae wait @@ -504,6 +508,7 @@ Get_trace() do ( + trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT # Create folder for n cores mkdir -p "$core"_cores @@ -528,19 +533,16 @@ Get_trace() if [ $Completed == false ]; then echo "Nemo execution failed, no traces files generated more info inside ${logs_path}/run_extrae_$core.err" - echo exit 1 fi mv nemo.prv nemo_"$core".prv mv nemo.pcf nemo_"$core".pcf mv nemo.row nemo_"$core".row echo "Cutting up best iteration" - echo "$dir"/src/magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > "${logs_path}"/cut_"$core".out 2>&1 if ! ls nemo_"$core".best_cut.prv; then echo "Cut failed, look at ${logs_path}/cut_$core.out for more info." - echo exit 1 fi echo -- GitLab From bdf053d16c451b5cfdb1656291dd797eadc1a45c Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 20 Sep 2022 15:34:24 +0200 Subject: [PATCH 19/20] Fixed mistake where exits loops instead script --- src/functions.bash | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/functions.bash b/src/functions.bash index 95792cc..996ec0d 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -508,8 +508,7 @@ Get_trace() do ( - trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT - + # Create folder for n cores mkdir -p "$core"_cores cd "$core"_cores @@ -526,6 +525,7 @@ Get_trace() python3 "$dir"/src/./Job_Creator.py -f "run_extrae_$core" -j "run_extrae_$core" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core "$dir"/src/./trace.sh ./nemo" if ! state4=$("$job" "$wait" run_extrae_$core."$Jobs_scheduler") ; then + kill -- -$$ exit 1 fi Job_completed "$state4" @@ -533,6 +533,7 @@ Get_trace() if [ $Completed == false ]; then echo "Nemo execution failed, no traces files generated more info inside ${logs_path}/run_extrae_$core.err" + kill -- -$$ exit 1 fi mv nemo.prv nemo_"$core".prv @@ -543,6 +544,7 @@ Get_trace() if ! ls nemo_"$core".best_cut.prv; then echo "Cut failed, look at ${logs_path}/cut_$core.out for more info." + kill -- -$$ exit 1 fi echo -- GitLab From c2b6653f1107df9beee45198b154f595472262ae Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 20 Sep 2022 15:35:21 +0200 Subject: [PATCH 20/20] Fixed mistake exiting script --- src/functions.bash | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/functions.bash b/src/functions.bash index 996ec0d..fd3de97 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -525,7 +525,7 @@ Get_trace() python3 "$dir"/src/./Job_Creator.py -f "run_extrae_$core" -j "run_extrae_$core" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core "$dir"/src/./trace.sh ./nemo" if ! state4=$("$job" "$wait" run_extrae_$core."$Jobs_scheduler") ; then - kill -- -$$ + kill 0 exit 1 fi Job_completed "$state4" @@ -533,7 +533,7 @@ Get_trace() if [ $Completed == false ]; then echo "Nemo execution failed, no traces files generated more info inside ${logs_path}/run_extrae_$core.err" - kill -- -$$ + kill 0 exit 1 fi mv nemo.prv nemo_"$core".prv @@ -544,7 +544,7 @@ Get_trace() if ! ls nemo_"$core".best_cut.prv; then echo "Cut failed, look at ${logs_path}/cut_$core.out for more info." - kill -- -$$ + kill 0 exit 1 fi echo -- GitLab