From 9a51b6bb1add619440ec2f83ecebc36812d609ae Mon Sep 17 00:00:00 2001 From: cpenadep Date: Thu, 1 Sep 2022 10:51:38 +0200 Subject: [PATCH 01/40] =?UTF-8?q?Deleted=20=C2=BA=20typo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- script.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/script.sh b/script.sh index c88b0cc..82a4254 100755 --- a/script.sh +++ b/script.sh @@ -278,7 +278,6 @@ Create_metrics() #Generating function list in case of missing if ! test -f "extrae_functions_for_xml.txt"; then - º rm gmon* 2> /dev/null echo "Runing Nemo with 2 cores to obtain function data..." echo -- GitLab From fe4efd0d06527d0cc5867aa1ad3c130603c3bf1e Mon Sep 17 00:00:00 2001 From: cpenadep Date: Thu, 1 Sep 2022 15:30:31 +0200 Subject: [PATCH 02/40] Initial Torque manager added --- Job_Creator.py | 40 +++++++++++++++++++++++++++++++++++++++- script.sh | 27 +++++++++++++++++++-------- 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/Job_Creator.py b/Job_Creator.py index 15330c8..4d9f46e 100644 --- a/Job_Creator.py +++ b/Job_Creator.py @@ -117,6 +117,41 @@ def create_job_lsf(args): lsf_job.write(line) +def create_job_torque(args): + file_name = args.file_name + time = args.set_time + cores = args.set_core + name = args.job_name + cores_per_node = args.set_core_per_node + queue = args.set_queue + workload = args.set_workload + nodes = (cores//cores_per_node)+1 + hours = time // 60 + minutes = time % 60 + file = ["#!/bin/bash \n", + "############################################################################### \n", + "#PBS -o "+str(name)+".out \n#PBS -e "+str(name)+".err \n" + "#PBS --constraint=perfparanoid \n"] + + if cores is not None: + file.append("#PBS -l nodes"+str(nodes)":ppn=" + str(cores) + "\n") + if time is not None and not 0: + file.append("#PBS -l cput=" + str(hours)+":"+str(minutes)+ ":00\n") + if name is not None: + file.append("#PBS -N " + name + "\n") + if queue is not None and not len(queue) == 0: + file.append("#PBS -q " + queue + "\n") + + if workload is not None: + file.append("\n") + for work in workload: + file.append(str(work) + "") + + with open(file_name, "w") as torque_job: + for line in file: + torque_job.write(line) + + def modify_job(args): file_name = args.file_name time = args.set_time @@ -148,6 +183,8 @@ if __name__ == "__main__": args.file_name = args.file_name if args.file_name.count(".slurm") else args.file_name + ".slurm" elif args.scheduler == "lsf": args.file_name = args.file_name if args.file_name.count(".lsf") else args.file_name + ".lsf" + elif args.scheduler == "torque": + args.file_name = args.file_name if args.file_name.count(".torque") else args.file_name + ".torque" if os.path.exists(str(args.file_name)): os.remove(str(args.file_name)) @@ -156,4 +193,5 @@ if __name__ == "__main__": create_job_slurm(args) elif args.scheduler == "lsf": create_job_lsf(args) - + elif args.scheduler == "torque": + create_job_torque(args) diff --git a/script.sh b/script.sh index 82a4254..9a4911f 100755 --- a/script.sh +++ b/script.sh @@ -32,22 +32,31 @@ Job_completed() { if [ "$Jobs_scheduler" == "slurm" ]; then local id1=${1##* } - sleep .5 + sleep 5 if ! scontrol show job $id1 | grep -q 'JobState=COMPLETED'; then Completed=false else Completed=true fi - else - local id1=${head -n1 1 | cut -d'<' -f2 | cut -d'>' -f1} - sleep .5 + elif [ "$Jobs_scheduler" == "lsf" ]; then + local id1=$(head -n1 1 | cut -d'<' -f2 | cut -d'>' -f1) + sleep 5 if ! bjobs -l $id | grep -q 'Status '; then Completed=false else Completed=true fi - fi + elif [ "$Jobs_scheduler" == "torque" ]; then + local id1=$(head -n1 $1 | awk '{ print $3 }') + sleep 5 + if ! qstat f $id | grep -q 'exit_status = 0'; then + Completed=false + else + Completed=true + fi + + } @@ -98,7 +107,7 @@ Test_arguments() exit 1 fi # scheduler correct? - if [ "$Jobs_scheduler" != "slurm" ] && [ "$Jobs_scheduler" != "lsf" ]; then + if [ "$Jobs_scheduler" != "slurm" ] && [ "$Jobs_scheduler" != "lsf" ] && [ "$Jobs_scheduler" != "torque" ]; then echo "$Jobs_scheduler is not a valid scheduler" echo exit 1 @@ -133,8 +142,10 @@ Test_arguments() # Creating auxiliar vars for submiting jobs if [ "$Jobs_scheduler" == "slurm" ]; then job="sbatch" - else - job="bsub" + elif [ "$Jobs_scheduler" == "lsf" ] + job="qsub" + elif [ "$Jobs_scheduler" == "torque" ] + job="" fi -- GitLab From a313905c0cd2c3a98660e2753450b3feba9620ff Mon Sep 17 00:00:00 2001 From: cpenadep Date: Thu, 1 Sep 2022 15:52:03 +0200 Subject: [PATCH 03/40] Fixed mistakes in principal files --- Job_Creator.py | 6 +++--- script.sh | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Job_Creator.py b/Job_Creator.py index 4d9f46e..dc70f90 100644 --- a/Job_Creator.py +++ b/Job_Creator.py @@ -134,9 +134,9 @@ def create_job_torque(args): "#PBS --constraint=perfparanoid \n"] if cores is not None: - file.append("#PBS -l nodes"+str(nodes)":ppn=" + str(cores) + "\n") + file.append("#PBS -l nodes" + str(nodes) + ":ppn=" + str(cores) + "\n") if time is not None and not 0: - file.append("#PBS -l cput=" + str(hours)+":"+str(minutes)+ ":00\n") + file.append("#PBS -l cput=" + str(hours) + ":"+str(minutes)+ ":00\n") if name is not None: file.append("#PBS -N " + name + "\n") if queue is not None and not len(queue) == 0: @@ -194,4 +194,4 @@ if __name__ == "__main__": elif args.scheduler == "lsf": create_job_lsf(args) elif args.scheduler == "torque": - create_job_torque(args) + create_job_torque(args) diff --git a/script.sh b/script.sh index 9a4911f..65193fd 100755 --- a/script.sh +++ b/script.sh @@ -55,7 +55,7 @@ Job_completed() else Completed=true fi - + fi } @@ -142,10 +142,10 @@ Test_arguments() # Creating auxiliar vars for submiting jobs if [ "$Jobs_scheduler" == "slurm" ]; then job="sbatch" - elif [ "$Jobs_scheduler" == "lsf" ] + elif [ "$Jobs_scheduler" == "lsf" ]; then + job="bsub" + elif [ "$Jobs_scheduler" == "torque" ]; then job="qsub" - elif [ "$Jobs_scheduler" == "torque" ] - job="" fi -- GitLab From 063b8b5afc70ffded273c14317b382a1264658b0 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Mon, 5 Sep 2022 13:08:19 +0200 Subject: [PATCH 04/40] Now namelist are not overwritten with the copy only the nemo binary --- config.bash | 2 +- script.sh | 24 ++++++++++-------------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/config.bash b/config.bash index 8d0f4c8..aa4ad29 100644 --- a/config.bash +++ b/config.bash @@ -19,7 +19,7 @@ Jobs_queue=debug # Compilation_compile: When false only compiles NEMO if arch file lacks the needed flags, when true always compiles NEMO. # Compilation_ref: Reference configuration # Compilation_arch: Architecture used (without the -arch sufix and the .fcm) -# Compilation_name: Name of the new configutation +# Compilation_name: Name of the new configutation (Important to not be an existing one) # Compilation_sub: Add or remove subcomponents diff --git a/script.sh b/script.sh index 65193fd..8f6d240 100755 --- a/script.sh +++ b/script.sh @@ -252,30 +252,25 @@ Test_Comp() fi - #Rename the namelist_cfg if exists in order to not overwrite it - if test -f "namelist_cfg"; then - mv namelist_cfg namelist_cfg_old - cd "$dir" || echo "Error original dir doesn't exist" exit - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . - rm namelist_cfg - mv namelist_cfg_old namelist_cfg - else - cd "$dir" || echo "Error original dir doesn't exist" exit - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . - fi + #Copy all the EXP00 data but don't overwrite namelist just the executable + cd "$dir" || echo "Error original dir doesn't exist" exit + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . + cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo . if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) fi - #Solving NEMO 4.2 Errors - sed -i 's|ln_zdfiwm * = .true.|ln_zdfiwm = .false.|g' namelist_cfg #CHANGE DUE TO NON EXISTING FILES + #Solving NEMO input file common errors + if test -f "weights_core_orca2_bicubic_noc.nc"; then mv weights_core_orca2_bicubic_noc.nc weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES fi + if test -f "weights_core_orca2_bilinear_noc.nc"; then mv weights_core_orca2_bilinear_noc.nc weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES fi + } @@ -297,7 +292,8 @@ Create_metrics() state2=$("$job" --wait run."$Jobs_scheduler") Job_completed "$state2" if [ $Completed == false ]; then - echo "Nemo execution failed no gprof files generated look at run.err for more info" + echo "Nemo execution failed look at run.err and ocean.output for more info" + echo "Remember that the namelist files are now the default, change theme in order to fit with the input files in the dir " echo exit 1 else -- GitLab From cd592bc928ce616661fb04e96cd5f586d42fed09 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Mon, 5 Sep 2022 15:43:10 +0200 Subject: [PATCH 05/40] Revert "Merge branch 'main' into 'origin/#1'" This reverts merge request !4 --- Job_Creator.py | 40 +--------------------------------------- config.bash | 2 +- script.sh | 44 +++++++++++++++++++------------------------- 3 files changed, 21 insertions(+), 65 deletions(-) diff --git a/Job_Creator.py b/Job_Creator.py index dc70f90..15330c8 100644 --- a/Job_Creator.py +++ b/Job_Creator.py @@ -117,41 +117,6 @@ def create_job_lsf(args): lsf_job.write(line) -def create_job_torque(args): - file_name = args.file_name - time = args.set_time - cores = args.set_core - name = args.job_name - cores_per_node = args.set_core_per_node - queue = args.set_queue - workload = args.set_workload - nodes = (cores//cores_per_node)+1 - hours = time // 60 - minutes = time % 60 - file = ["#!/bin/bash \n", - "############################################################################### \n", - "#PBS -o "+str(name)+".out \n#PBS -e "+str(name)+".err \n" - "#PBS --constraint=perfparanoid \n"] - - if cores is not None: - file.append("#PBS -l nodes" + str(nodes) + ":ppn=" + str(cores) + "\n") - if time is not None and not 0: - file.append("#PBS -l cput=" + str(hours) + ":"+str(minutes)+ ":00\n") - if name is not None: - file.append("#PBS -N " + name + "\n") - if queue is not None and not len(queue) == 0: - file.append("#PBS -q " + queue + "\n") - - if workload is not None: - file.append("\n") - for work in workload: - file.append(str(work) + "") - - with open(file_name, "w") as torque_job: - for line in file: - torque_job.write(line) - - def modify_job(args): file_name = args.file_name time = args.set_time @@ -183,8 +148,6 @@ if __name__ == "__main__": args.file_name = args.file_name if args.file_name.count(".slurm") else args.file_name + ".slurm" elif args.scheduler == "lsf": args.file_name = args.file_name if args.file_name.count(".lsf") else args.file_name + ".lsf" - elif args.scheduler == "torque": - args.file_name = args.file_name if args.file_name.count(".torque") else args.file_name + ".torque" if os.path.exists(str(args.file_name)): os.remove(str(args.file_name)) @@ -193,5 +156,4 @@ if __name__ == "__main__": create_job_slurm(args) elif args.scheduler == "lsf": create_job_lsf(args) - elif args.scheduler == "torque": - create_job_torque(args) + diff --git a/config.bash b/config.bash index aa4ad29..8d0f4c8 100644 --- a/config.bash +++ b/config.bash @@ -19,7 +19,7 @@ Jobs_queue=debug # Compilation_compile: When false only compiles NEMO if arch file lacks the needed flags, when true always compiles NEMO. # Compilation_ref: Reference configuration # Compilation_arch: Architecture used (without the -arch sufix and the .fcm) -# Compilation_name: Name of the new configutation (Important to not be an existing one) +# Compilation_name: Name of the new configutation # Compilation_sub: Add or remove subcomponents diff --git a/script.sh b/script.sh index b28e5b8..4a97f5c 100755 --- a/script.sh +++ b/script.sh @@ -39,24 +39,15 @@ Job_completed() Completed=true fi - elif [ "$Jobs_scheduler" == "lsf" ]; then - local id1=$(head -n1 1 | cut -d'<' -f2 | cut -d'>' -f1) + else + local id1=${head -n1 1 | cut -d'<' -f2 | cut -d'>' -f1} sleep 5 if ! bjobs -l $id | grep -q 'Status '; then Completed=false else Completed=true fi - elif [ "$Jobs_scheduler" == "torque" ]; then - local id1=$(head -n1 $1 | awk '{ print $3 }') - sleep 5 - if ! qstat f $id | grep -q 'exit_status = 0'; then - Completed=false - else - Completed=true - fi fi - } @@ -107,7 +98,7 @@ Test_arguments() exit 1 fi # scheduler correct? - if [ "$Jobs_scheduler" != "slurm" ] && [ "$Jobs_scheduler" != "lsf" ] && [ "$Jobs_scheduler" != "torque" ]; then + if [ "$Jobs_scheduler" != "slurm" ] && [ "$Jobs_scheduler" != "lsf" ]; then echo "$Jobs_scheduler is not a valid scheduler" echo exit 1 @@ -142,10 +133,8 @@ Test_arguments() # Creating auxiliar vars for submiting jobs if [ "$Jobs_scheduler" == "slurm" ]; then job="sbatch" - elif [ "$Jobs_scheduler" == "lsf" ]; then + else job="bsub" - elif [ "$Jobs_scheduler" == "torque" ]; then - job="qsub" fi @@ -252,25 +241,30 @@ Test_Comp() fi - #Copy all the EXP00 data but don't overwrite namelist just the executable - cd "$dir" || echo "Error original dir doesn't exist" exit - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo . + #Rename the namelist_cfg if exists in order to not overwrite it + if test -f "namelist_cfg"; then + mv namelist_cfg namelist_cfg_old + cd "$dir" || echo "Error original dir doesn't exist" exit + cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . + rm namelist_cfg + mv namelist_cfg_old namelist_cfg + else + cd "$dir" || echo "Error original dir doesn't exist" exit + cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . + fi if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) fi - #Solving NEMO input file common errors - + #Solving NEMO 4.2 Errors + sed -i 's|ln_zdfiwm * = .true.|ln_zdfiwm = .false.|g' namelist_cfg #CHANGE DUE TO NON EXISTING FILES if test -f "weights_core_orca2_bicubic_noc.nc"; then mv weights_core_orca2_bicubic_noc.nc weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES fi - if test -f "weights_core_orca2_bilinear_noc.nc"; then mv weights_core_orca2_bilinear_noc.nc weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES fi - } @@ -284,6 +278,7 @@ Create_metrics() #Generating function list in case of missing if ! test -f "extrae_functions_for_xml.txt"; then + rm gmon* 2> /dev/null echo "Runing Nemo with 2 cores to obtain function data..." echo @@ -292,8 +287,7 @@ Create_metrics() state2=$("$job" --wait run."$Jobs_scheduler") Job_completed "$state2" if [ $Completed == false ]; then - echo "Nemo execution failed look at run.err and ocean.output for more info" - echo "Remember that the namelist files are now the default, change theme in order to fit with the input files in the dir " + echo "Nemo execution failed no gprof files generated look at run.err for more info" echo exit 1 else -- GitLab From 87c7e5975e3c1661a9959a5d7391ee4aa6fa03e3 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Mon, 5 Sep 2022 15:44:54 +0200 Subject: [PATCH 06/40] Revert "Merge branch 'revert-6eb0eafd' into 'main'" This reverts merge request !5 --- Job_Creator.py | 40 +++++++++++++++++++++++++++++++++++++++- config.bash | 2 +- script.sh | 44 +++++++++++++++++++++++++------------------- 3 files changed, 65 insertions(+), 21 deletions(-) diff --git a/Job_Creator.py b/Job_Creator.py index 15330c8..dc70f90 100644 --- a/Job_Creator.py +++ b/Job_Creator.py @@ -117,6 +117,41 @@ def create_job_lsf(args): lsf_job.write(line) +def create_job_torque(args): + file_name = args.file_name + time = args.set_time + cores = args.set_core + name = args.job_name + cores_per_node = args.set_core_per_node + queue = args.set_queue + workload = args.set_workload + nodes = (cores//cores_per_node)+1 + hours = time // 60 + minutes = time % 60 + file = ["#!/bin/bash \n", + "############################################################################### \n", + "#PBS -o "+str(name)+".out \n#PBS -e "+str(name)+".err \n" + "#PBS --constraint=perfparanoid \n"] + + if cores is not None: + file.append("#PBS -l nodes" + str(nodes) + ":ppn=" + str(cores) + "\n") + if time is not None and not 0: + file.append("#PBS -l cput=" + str(hours) + ":"+str(minutes)+ ":00\n") + if name is not None: + file.append("#PBS -N " + name + "\n") + if queue is not None and not len(queue) == 0: + file.append("#PBS -q " + queue + "\n") + + if workload is not None: + file.append("\n") + for work in workload: + file.append(str(work) + "") + + with open(file_name, "w") as torque_job: + for line in file: + torque_job.write(line) + + def modify_job(args): file_name = args.file_name time = args.set_time @@ -148,6 +183,8 @@ if __name__ == "__main__": args.file_name = args.file_name if args.file_name.count(".slurm") else args.file_name + ".slurm" elif args.scheduler == "lsf": args.file_name = args.file_name if args.file_name.count(".lsf") else args.file_name + ".lsf" + elif args.scheduler == "torque": + args.file_name = args.file_name if args.file_name.count(".torque") else args.file_name + ".torque" if os.path.exists(str(args.file_name)): os.remove(str(args.file_name)) @@ -156,4 +193,5 @@ if __name__ == "__main__": create_job_slurm(args) elif args.scheduler == "lsf": create_job_lsf(args) - + elif args.scheduler == "torque": + create_job_torque(args) diff --git a/config.bash b/config.bash index 8d0f4c8..aa4ad29 100644 --- a/config.bash +++ b/config.bash @@ -19,7 +19,7 @@ Jobs_queue=debug # Compilation_compile: When false only compiles NEMO if arch file lacks the needed flags, when true always compiles NEMO. # Compilation_ref: Reference configuration # Compilation_arch: Architecture used (without the -arch sufix and the .fcm) -# Compilation_name: Name of the new configutation +# Compilation_name: Name of the new configutation (Important to not be an existing one) # Compilation_sub: Add or remove subcomponents diff --git a/script.sh b/script.sh index 4a97f5c..b28e5b8 100755 --- a/script.sh +++ b/script.sh @@ -39,15 +39,24 @@ Job_completed() Completed=true fi - else - local id1=${head -n1 1 | cut -d'<' -f2 | cut -d'>' -f1} + elif [ "$Jobs_scheduler" == "lsf" ]; then + local id1=$(head -n1 1 | cut -d'<' -f2 | cut -d'>' -f1) sleep 5 if ! bjobs -l $id | grep -q 'Status '; then Completed=false else Completed=true fi + elif [ "$Jobs_scheduler" == "torque" ]; then + local id1=$(head -n1 $1 | awk '{ print $3 }') + sleep 5 + if ! qstat f $id | grep -q 'exit_status = 0'; then + Completed=false + else + Completed=true + fi fi + } @@ -98,7 +107,7 @@ Test_arguments() exit 1 fi # scheduler correct? - if [ "$Jobs_scheduler" != "slurm" ] && [ "$Jobs_scheduler" != "lsf" ]; then + if [ "$Jobs_scheduler" != "slurm" ] && [ "$Jobs_scheduler" != "lsf" ] && [ "$Jobs_scheduler" != "torque" ]; then echo "$Jobs_scheduler is not a valid scheduler" echo exit 1 @@ -133,8 +142,10 @@ Test_arguments() # Creating auxiliar vars for submiting jobs if [ "$Jobs_scheduler" == "slurm" ]; then job="sbatch" - else + elif [ "$Jobs_scheduler" == "lsf" ]; then job="bsub" + elif [ "$Jobs_scheduler" == "torque" ]; then + job="qsub" fi @@ -241,30 +252,25 @@ Test_Comp() fi - #Rename the namelist_cfg if exists in order to not overwrite it - if test -f "namelist_cfg"; then - mv namelist_cfg namelist_cfg_old - cd "$dir" || echo "Error original dir doesn't exist" exit - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . - rm namelist_cfg - mv namelist_cfg_old namelist_cfg - else - cd "$dir" || echo "Error original dir doesn't exist" exit - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . - fi + #Copy all the EXP00 data but don't overwrite namelist just the executable + cd "$dir" || echo "Error original dir doesn't exist" exit + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . + cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo . if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) fi - #Solving NEMO 4.2 Errors - sed -i 's|ln_zdfiwm * = .true.|ln_zdfiwm = .false.|g' namelist_cfg #CHANGE DUE TO NON EXISTING FILES + #Solving NEMO input file common errors + if test -f "weights_core_orca2_bicubic_noc.nc"; then mv weights_core_orca2_bicubic_noc.nc weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES fi + if test -f "weights_core_orca2_bilinear_noc.nc"; then mv weights_core_orca2_bilinear_noc.nc weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES fi + } @@ -278,7 +284,6 @@ Create_metrics() #Generating function list in case of missing if ! test -f "extrae_functions_for_xml.txt"; then - rm gmon* 2> /dev/null echo "Runing Nemo with 2 cores to obtain function data..." echo @@ -287,7 +292,8 @@ Create_metrics() state2=$("$job" --wait run."$Jobs_scheduler") Job_completed "$state2" if [ $Completed == false ]; then - echo "Nemo execution failed no gprof files generated look at run.err for more info" + echo "Nemo execution failed look at run.err and ocean.output for more info" + echo "Remember that the namelist files are now the default, change theme in order to fit with the input files in the dir " echo exit 1 else -- GitLab From b74f9189a22d34dc8137587d7ad4f125eb6d880e Mon Sep 17 00:00:00 2001 From: cpenadep Date: Mon, 5 Sep 2022 15:47:36 +0200 Subject: [PATCH 07/40] Removed files added by error --- INSTALL.md | 28 - TODO.md | 14 - cfgs/.directory | 4 - cfgs/2dh_BurstEfficiency.cfg | 221 ----- cfgs/barrier-syncr-time.cfg | 80 -- cfgs/burst_duration.cfg | 77 -- cfgs/burst_useful.cfg | 150 ---- cfgs/cycles.cfg | 141 --- cfgs/dimemas.collectives | 14 - cfgs/dimemas_ideal.cfg | 159 ---- cfgs/efficiency_table-global.gp | 31 - cfgs/efficiency_table-hybrid.gp | 30 - cfgs/efficiency_table.gp | 31 - cfgs/flushing-cycles.cfg | 148 --- cfgs/flushing-inst.cfg | 148 --- cfgs/flushing.cfg | 76 -- cfgs/instructions.cfg | 141 --- cfgs/io-call-cycles.cfg | 148 --- cfgs/io-call-instructions.cfg | 148 --- cfgs/io-call-reverse.cfg | 76 -- cfgs/modelfactors-all.gp | 36 - cfgs/modelfactors-comm.gp | 26 - cfgs/modelfactors-hybrid.gp | 37 - cfgs/modelfactors-mpi-hybrid.gp | 32 - cfgs/modelfactors-onlydata.gp | 31 - cfgs/modelfactors-scale.gp | 27 - cfgs/mpi-call-outside.cfg | 77 -- cfgs/mpi-io-cycles.cfg | 149 --- cfgs/mpi-io-instructions.cfg | 149 --- cfgs/mpi-io-reverse.cfg | 77 -- cfgs/mpi-io.cfg | 77 -- cfgs/mpi-master-thread.cfg | 79 -- cfgs/runtime.cfg | 72 -- cfgs/runtime_app.cfg | 74 -- cfgs/time_computing.cfg | 114 --- cfgs/timings.cfg | 72 -- hybridmetrics.py | 1496 ------------------------------- modelfactors.py | 192 ---- plots.py | 738 --------------- rawdata.py | 857 ------------------ simplemetrics.py | 1249 -------------------------- tracemetadata.py | 376 -------- utils.py | 186 ---- 43 files changed, 8088 deletions(-) delete mode 100644 INSTALL.md delete mode 100644 TODO.md delete mode 100644 cfgs/.directory delete mode 100644 cfgs/2dh_BurstEfficiency.cfg delete mode 100644 cfgs/barrier-syncr-time.cfg delete mode 100644 cfgs/burst_duration.cfg delete mode 100644 cfgs/burst_useful.cfg delete mode 100644 cfgs/cycles.cfg delete mode 100644 cfgs/dimemas.collectives delete mode 100644 cfgs/dimemas_ideal.cfg delete mode 100644 cfgs/efficiency_table-global.gp delete mode 100644 cfgs/efficiency_table-hybrid.gp delete mode 100644 cfgs/efficiency_table.gp delete mode 100644 cfgs/flushing-cycles.cfg delete mode 100644 cfgs/flushing-inst.cfg delete mode 100644 cfgs/flushing.cfg delete mode 100644 cfgs/instructions.cfg delete mode 100644 cfgs/io-call-cycles.cfg delete mode 100644 cfgs/io-call-instructions.cfg delete mode 100644 cfgs/io-call-reverse.cfg delete mode 100644 cfgs/modelfactors-all.gp delete mode 100644 cfgs/modelfactors-comm.gp delete mode 100644 cfgs/modelfactors-hybrid.gp delete mode 100644 cfgs/modelfactors-mpi-hybrid.gp delete mode 100644 cfgs/modelfactors-onlydata.gp delete mode 100644 cfgs/modelfactors-scale.gp delete mode 100644 cfgs/mpi-call-outside.cfg delete mode 100644 cfgs/mpi-io-cycles.cfg delete mode 100644 cfgs/mpi-io-instructions.cfg delete mode 100644 cfgs/mpi-io-reverse.cfg delete mode 100644 cfgs/mpi-io.cfg delete mode 100644 cfgs/mpi-master-thread.cfg delete mode 100644 cfgs/runtime.cfg delete mode 100644 cfgs/runtime_app.cfg delete mode 100644 cfgs/time_computing.cfg delete mode 100644 cfgs/timings.cfg delete mode 100644 hybridmetrics.py delete mode 100755 modelfactors.py delete mode 100644 plots.py delete mode 100644 rawdata.py delete mode 100644 simplemetrics.py delete mode 100644 tracemetadata.py delete mode 100644 utils.py diff --git a/INSTALL.md b/INSTALL.md deleted file mode 100644 index 5fedce4..0000000 --- a/INSTALL.md +++ /dev/null @@ -1,28 +0,0 @@ -# Installation - -There is no installation required. Just copy the content of this folder to your -preferred location and add the directory to the PATH environment variable. - -## Prerequisites - -Basicanalysis requires Python 3 and relies on -*paramedir* and *Dimemas* being installed and available -through the PATH environment variable. - -* *paramedir* available at https://tools.bsc.es/paraver -* *Dimemas* available at https://tools.bsc.es/dimemas - -If not already done, install both tools and add them to the PATH environment -variable with: - -``` -export PATH=/bin:$PATH -export PARAVER_HOME= -export PATH=/bin:$PATH -export DIMEMAS_HOME= - -``` - -Additionally, plotting relies on the according SciPy(>= 0.17.0), -NumPy, pandas, searborn and matplotlib (>= 3.x) modules for Python 3. -Furthermore, the gnuplot output requires gnuplot version 5.0 or higher. diff --git a/TODO.md b/TODO.md deleted file mode 100644 index b8b4b81..0000000 --- a/TODO.md +++ /dev/null @@ -1,14 +0,0 @@ -# Open Issues and Future Features - -* Add some sanity checks: - * Test if simulated time is less or equal to original time to detect Dimemas errors. - * Correctly track return values of external calls and handle errors. - -* Add support for compressed Paraver traces. - -* Add basic threading support to run some analyses in parallel. - -* Add support PyCompSs - * Encapsulate main routines in functions - * Provide stable switches for systems without PyCompSs - diff --git a/cfgs/.directory b/cfgs/.directory deleted file mode 100644 index 5ff3922..0000000 --- a/cfgs/.directory +++ /dev/null @@ -1,4 +0,0 @@ -[Dolphin] -Timestamp=2019,6,19,17,11,21 -Version=4 -ViewMode=1 diff --git a/cfgs/2dh_BurstEfficiency.cfg b/cfgs/2dh_BurstEfficiency.cfg deleted file mode 100644 index 9429c78..0000000 --- a/cfgs/2dh_BurstEfficiency.cfg +++ /dev/null @@ -1,221 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 5 -ConfigFile.BeginDescription -ConfigFile.EndDescription - -################################################################################ -< NEW DISPLAYING WINDOW Burst Region duration.c1.c1 > -################################################################################ -window_name Burst Region duration.c1.c1 -window_type single -window_id 1 -window_position_x 484 -window_position_y 126 -window_width 600 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 39435569.853000000119 -window_minimum_y 1.377000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode 1 -window_drawmode_rows 1 -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Int. Between Evt}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_filter_module evt_type 1 54000006 -window_filter_module evt_type_label 1 "Elapsed time in MPI" - -################################################################################ -< NEW DISPLAYING WINDOW MPI ellapsed time In burst Region.c3.c1 > -################################################################################ -window_name MPI ellapsed time In burst Region.c3.c1 -window_type single -window_id 2 -window_position_x 502 -window_position_y 144 -window_width 600 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 39435569.853000000119 -window_minimum_y 1.377000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode 1 -window_drawmode_rows 1 -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Div}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_semantic_module compose_thread Div { 1, { 1 1000.000000000000 } } -window_filter_module evt_type 1 54000006 -window_filter_module evt_type_label 1 "Elapsed time in MPI" - -################################################################################ -< NEW DISPLAYING WINDOW Computing time in Region.c1 > -################################################################################ -window_name Computing time in Region.c1 -window_type composed -window_id 3 -window_factors 1.000000000000 1.000000000000 -window_operation substract -window_identifiers 1 2 -window_position_x 538 -window_position_y 180 -window_width 600 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_units Microseconds -window_maximum_y 39435569.853000000119 -window_minimum_y 1.377000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode 1 -window_drawmode_rows 1 -window_pixel_size 1 -window_labels_to_draw 0 -window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } - -################################################################################ -< NEW DISPLAYING WINDOW Burst Region duration.c2 > -################################################################################ -window_name Burst Region duration.c2 -window_type single -window_id 4 -window_position_x 556 -window_position_y 198 -window_width 600 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 39435569.853000000119 -window_minimum_y 1.377000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode 1 -window_drawmode_rows 1 -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Int. Between Evt}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_filter_module evt_type 1 54000006 -window_filter_module evt_type_label 1 "Elapsed time in MPI" - -################################################################################ -< NEW DISPLAYING WINDOW Burst efficiency > -################################################################################ -window_name Burst efficiency -window_type composed -window_id 5 -window_factors 1.000000000000 1.000000000000 -window_operation divide -window_identifiers 3 4 -window_position_x 475 -window_position_y 145 -window_width 600 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_units Microseconds -window_maximum_y 1.100000000000 -window_minimum_y 0.000000004381 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode 3 -window_drawmode_rows 3 -window_pixel_size 1 -window_labels_to_draw 0 -window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } - -< NEW ANALYZER2D > -Analyzer2D.Name: 2DH - efficiency -Analyzer2D.X: 130 -Analyzer2D.Y: 233 -Analyzer2D.Width: 365 -Analyzer2D.Height: 218 -Analyzer2D.ControlWindow: 5 -Analyzer2D.DataWindow: 5 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Average value -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: False -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Enabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: False -Analyzer2D.Minimum: 0.000000000000 -Analyzer2D.Maximum: 1.000000000000 -Analyzer2D.Delta: 2.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 0.142980245071 -Analyzer2D.MaximumGradient: 0.184636349192 -Analyzer2D.PixelSize: 1 -Analyzer2D.CodeColor: False -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/cfgs/barrier-syncr-time.cfg b/cfgs/barrier-syncr-time.cfg deleted file mode 100644 index ee7b63d..0000000 --- a/cfgs/barrier-syncr-time.cfg +++ /dev/null @@ -1,80 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 1 - - -################################################################################ -< NEW DISPLAYING WINDOW MPI call.SerAndTransfer > -################################################################################ -window_name MPI call.SerAndTransfer -window_type single -window_id 1 -window_position_x 3024 -window_position_y 486 -window_width 729 -window_height 320 -window_comm_lines_enabled true -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 70.000000000000 -window_minimum_y 0.000000000000 -window_compute_y_max true -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_random -window_drawmode_rows draw_random -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_semantic_module task Thread i { 1, { 1 0.000000000000 } } -window_filter_module evt_type 2 50000001 50000002 -window_filter_module evt_type_label 2 "MPI Point-to-point" "MPI Collective Comm" -window_filter_module evt_value 3 5.000000000000 6.000000000000 8.000000000000 -window_filter_module evt_value_label 3 "MPI_Wait" "MPI_Waitall" "MPI_Barrier" -window_synchronize 1 - -< NEW ANALYZER2D > -Analyzer2D.Name: MPI-Ser-Transfer -Analyzer2D.X: 2810 -Analyzer2D.Y: 48 -Analyzer2D.Width: 868 -Analyzer2D.Height: 491 -Analyzer2D.ControlWindow: 1 -Analyzer2D.DataWindow: 1 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Time -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: True -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: True -Analyzer2D.Minimum: 3.000000000000 -Analyzer2D.Maximum: 10.000000000000 -Analyzer2D.Delta: 1.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 52.305000000000 -Analyzer2D.MaximumGradient: 76291.281000000003 -Analyzer2D.DrawModeObjects: draw_maximum -Analyzer2D.DrawModeColumns: draw_maximum -Analyzer2D.PixelSize: 1 -Analyzer2D.ColorMode: window_in_gradient_mode -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/cfgs/burst_duration.cfg b/cfgs/burst_duration.cfg deleted file mode 100644 index 6958790..0000000 --- a/cfgs/burst_duration.cfg +++ /dev/null @@ -1,77 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 1 - - -################################################################################ -< NEW DISPLAYING WINDOW burst_duration.c1 > -################################################################################ -window_name burst_duration.c1 -window_type single -window_id 1 -window_position_x 596 -window_position_y 238 -window_width 600 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 39435569.853000000119 -window_minimum_y 1.377000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Int. Between Evt}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_filter_module evt_type 1 54000006 -window_filter_module evt_type_label 1 "Elapsed time in MPI" - -< NEW ANALYZER2D > -Analyzer2D.Name: burst_duration -Analyzer2D.X: 2297 -Analyzer2D.Y: 30 -Analyzer2D.Width: 391 -Analyzer2D.Height: 472 -Analyzer2D.ControlWindow: 1 -Analyzer2D.DataWindow: 1 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Sum bursts -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: False -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: False -Analyzer2D.Minimum: 18.918000000000 -Analyzer2D.Maximum: 191006.618399999978 -Analyzer2D.Delta: 1000000000000000.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 952444.935999999987 -Analyzer2D.MaximumGradient: 959765.031999999890 -Analyzer2D.DrawModeObjects: draw_maximum -Analyzer2D.DrawModeColumns: draw_maximum -Analyzer2D.PixelSize: 1 -Analyzer2D.ColorMode: window_in_gradient_mode -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/cfgs/burst_useful.cfg b/cfgs/burst_useful.cfg deleted file mode 100644 index ac54618..0000000 --- a/cfgs/burst_useful.cfg +++ /dev/null @@ -1,150 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 3 - - -################################################################################ -< NEW DISPLAYING WINDOW burst_duration > -################################################################################ -window_name burst_duration -window_type single -window_id 1 -window_position_x 567 -window_position_y 209 -window_width 600 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 57174.218999999997 -window_minimum_y 24.956000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Int. Between Evt}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_filter_module evt_type 1 54000006 -window_filter_module evt_type_label 1 "Elapsed time in MPI" - -################################################################################ -< NEW DISPLAYING WINDOW burst_MPI_ellapsed_time > -################################################################################ -window_name burst_MPI_ellapsed_time -window_type single -window_id 2 -window_position_x 567 -window_position_y 209 -window_width 600 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 12582.013999999999 -window_minimum_y 21.416000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Div}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_semantic_module compose_thread Div { 1, { 1 1000.000000000000 } } -window_filter_module evt_type 1 54000006 -window_filter_module evt_type_label 1 "Elapsed time in MPI" - -################################################################################ -< NEW DISPLAYING WINDOW useful_burst > -################################################################################ -window_name useful_burst -window_type composed -window_id 3 -window_factors 1.000000000000 1.000000000000 -window_operation substract -window_identifiers 1 2 -window_position_x 1960 -window_position_y 174 -window_width 600 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_units Microseconds -window_maximum_y 57174.218999999997 -window_minimum_y -4162.623000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_random -window_drawmode_rows draw_random -window_pixel_size 1 -window_labels_to_draw 0 -window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } - -< NEW ANALYZER2D > -Analyzer2D.Name: useful_burst -Analyzer2D.X: 2327 -Analyzer2D.Y: 31 -Analyzer2D.Width: 841 -Analyzer2D.Height: 365 -Analyzer2D.ControlWindow: 3 -Analyzer2D.DataWindow: 3 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Sum bursts -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: False -Analyzer2D.HorizVert: Vertical -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: True -Analyzer2D.Minimum: -5122.957000000000 -Analyzer2D.Maximum: 142551.129350000003 -Analyzer2D.Delta: 738.370431750000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: -5122.957000000000 -Analyzer2D.MaximumGradient: 16572593.654999990016 -Analyzer2D.DrawModeObjects: draw_maximum -Analyzer2D.DrawModeColumns: draw_maximum -Analyzer2D.PixelSize: 1 -Analyzer2D.ColorMode: window_in_gradient_mode -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/cfgs/cycles.cfg b/cfgs/cycles.cfg deleted file mode 100644 index 7d2091e..0000000 --- a/cfgs/cycles.cfg +++ /dev/null @@ -1,141 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 3 - - -################################################################################ -< NEW DISPLAYING WINDOW cycles.c1.c1 > -################################################################################ -window_name cycles.c1.c1 -window_type single -window_id 1 -window_position_x 300 -window_position_y 23 -window_width 600 -window_height 114 -window_comm_lines_enabled true -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 194620048057.000000000000 -window_minimum_y 0.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode 1 -window_drawmode_rows 1 -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_filter_module evt_type 1 42000059 - -################################################################################ -< NEW DISPLAYING WINDOW Useful.c1.c1 > -################################################################################ -window_name Useful.c1.c1 -window_type single -window_id 2 -window_position_x 323 -window_position_y 46 -window_width 600 -window_height 134 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered true -window_physical_filtered true -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 1.000000000000 -window_minimum_y 0.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode 1 -window_drawmode_rows 1 -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Useful}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, =}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, All}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } - -################################################################################ -< NEW DISPLAYING WINDOW Useful Cycles Timeline > -################################################################################ -window_name Useful Cycles Timeline -window_type composed -window_id 3 -window_factors 1.000000000000 1.000000000000 -window_operation product -window_identifiers 1 2 -window_position_x 391 -window_position_y 146 -window_width 767 -window_height 314 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_units Microseconds -window_maximum_y 11047386945.000000000000 -window_minimum_y 5433.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode 1 -window_drawmode_rows 1 -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } - -< NEW ANALYZER2D > -Analyzer2D.Name: cycles -Analyzer2D.X: 414 -Analyzer2D.Y: 54 -Analyzer2D.Width: 556 -Analyzer2D.Height: 764 -Analyzer2D.ControlWindow: 3 -Analyzer2D.DataWindow: 3 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Sum bursts -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: False -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: False -Analyzer2D.Minimum: 0.000000000000 -Analyzer2D.Maximum: 1000000000000000000.000000000000 -Analyzer2D.Delta: 1000000000000000000.000000000000 -Analyzer2D.ComputeGradient: False -Analyzer2D.MinimumGradient: 0 -Analyzer2D.MaximumGradient: 999999999999999999 -Analyzer2D.PixelSize: 1 -Analyzer2D.CodeColor: False - diff --git a/cfgs/dimemas.collectives b/cfgs/dimemas.collectives deleted file mode 100644 index 3b86752..0000000 --- a/cfgs/dimemas.collectives +++ /dev/null @@ -1,14 +0,0 @@ -Machine globalop: 0 0 LOG MAX 0 MAX -Machine globalop: 0 1 LOG MAX 0 MAX -Machine globalop: 0 2 LOG MAX 0 MAX -Machine globalop: 0 3 LOG MAX 0 MAX -Machine globalop: 0 4 LOG MAX 0 MAX -Machine globalop: 0 5 LOG MAX 0 MAX -Machine globalop: 0 6 LOG MAX 0 MAX -Machine globalop: 0 7 LOG MAX 0 MAX -Machine globalop: 0 8 LIN 2MAX LIN 2MAX -Machine globalop: 0 9 LOG MAX 0 MAX -Machine globalop: 0 10 LOG 2MAX 0 MAX -Machine globalop: 0 11 LOG MAX 0 MAX -Machine globalop: 0 12 LOG MAX 0 MAX -Machine globalop: 0 13 LOG MAX 0 MAX diff --git a/cfgs/dimemas_ideal.cfg b/cfgs/dimemas_ideal.cfg deleted file mode 100644 index d0f46f0..0000000 --- a/cfgs/dimemas_ideal.cfg +++ /dev/null @@ -1,159 +0,0 @@ -#DIMEMAS_CONFIGURATION - -/******************************************************************************* - * RECORDS DEFINITION - NOTES TO MANUAL EDIT OF THIS FILE - ******************************************************************************* - * - * The records here described follow the SDDF file structure. Each field must be - * sepparated by commas in the actual records. - * - * 'int' and 'double' values are expresed as is. 'char[]' values must be always - * wrapped by quotes. 'double' and 'int' arrays must be in the following format: - * - * [n] { value_1, value_2, ..., value_n } - * - * Where 'n' is the number of elements in the array - * - ******************************************************************************* - - "wide area network information" { - 1: char[] "name of the wide area network simulated" - 2: int "number of machines in wan" - 3: int "number of dedicated connections between machines in the simulated" - "system" - 4: int "function that models influence of traffic in the non dedicated" - "network." - "options: 1 EXPONENTIAl, 2 LOGARITHMIC, 3 LINEAR, 4 CONSTANT" - 5: double "maximal value of traffic in the network" - 6: double "external net bandwidth (MBps)" - 7: int "external network collective communications model" - "options: 1 CONSTANT, 2 LINEAR, 3 LOGARITHMIC" -};; - -"environment information" { - 1: char[] "machine name" - 2: int "machine ID"; - 3: char[] "architecture used to instrument" - 4: int "number of nodes on virtual machine" - 5: double "data tranfer rate between nodes (MBps)" - "0 means instantaneous communication" - 6: int "maximun number of messages on network" - "0 means no limit" - "1 means bus contention" - 7: int "internal network collective communications model" - "options: 1 CONSTANT, 2 LINEAR, 3 LOGARITHMIC" -};; - - -"node information" { - 1: int "machine ID" - 2: char[] "architecture node name" - 3: int "number of processors within node" - 4: double "relative processor speed (divisive factor)" - 5: double "latency time (s) of intra-node communications model" - 6: double "bandwidth (MBps) of intra-node communications model" - "0 means instantaneous communication" - 7: int "maximum number of concurrent messages of intra-node" - "communications model:" - "0 means no limit" - "1 means bus contention" - 8: int "input links of intra-node communications model" - 9: int "output links of intra-node communications model" - 10: double "latency time (s) of inter-node communications model" - 11: int "input links of inter-node communications model" - 12: int "input links of intra-node communications model" - 13: double "latency time (s) of inter-machines (WAN) communications model" -};; - -"multi node information" { - 1: int "machine ID" - 2: int "number of nodes with same configuration" - 3: char[] "architecture node name" - 4: int "number of processors within node" - 5: double "processor speed ratio wrt. original execution (divisive factor)" - "0 means instantaneous | negative value means fixed duration in s." - 6: double "latency time (s) of intra-node communications model" - 7: double "bandwidth (MBps) of intra-node communications model" - "0 means instantaneous communication" - 8: int "maximum number of concurrent messages of intra-node" - "communications model: " - "0 means no limit" - "1 means bus contention" - 9: int "input links of intra-node communications model" - 10: int "output links of intra-node communications model" - 11: double "latency time (s) of inter-node communications model" - 12: int "input links of inter-node communications model" - 13: int "input links of intra-node communications model" - 14: double "latency time (s) of inter-machines (WAN) communications model" -};; - -"mapping information" { - 1: char[] "application tracefile name" - 2: int "number of tasks in application" - 3: int[] "list of nodes where each application tasks is mapped" -};; - -"predefined mapping information" { - 1: char[] "application tracefile name" (OPTIONAL) - 2: char[] "predefined map identifier" - "options: FILL_NODES | TASKS_PER_NODE | INTERLEAVED" -} - -"configuration files" { - 1: char[] "scheduler definition filename" - 2: char[] "file system definition filename" - 3: char[] "communications fine tuning configuration filename" - 4: char[] "sensitivity configuration filename" -};; - -"modules information" { - 1: int "module type" - 2: int "module value" - 3: double "module speed ration wrt. original execution (divisive factor)" - "0 means instantaneous | negative value means fixed duration in s." -};; - -"file system parameters" { - 1: double "disk latency" - 2: double "disk bandwidth"; - 3: double "block size"; - 4: int "concurrent requests"; - 5: double "hit ratio"; -};; - -"dedicated connection information" { - 1: int "connection ID" - 2: int "source machine ID" - 3: int "destination machine ID" - 4: double "bandwidth of the connection (MBps)" - 5: int[] "list of tags that will use the connection" - 6: int "size of first message (bytes) to apply the comparision to use the" - "connection" - 7: char[] "size condition that should meet messages to use the connection" - "it can be <, =, > and (referent to first message size)" - 8: char[] "operation. options: & AND, | OR" - 9: int "size of second condition that should meet messages to use the" - "connection" - 10: char[] "size condition that should meet messages to use the connection" - "it can be <, =, > and its is referent to second message size" - 11: int[] "list of communicators of coll. Operations that can use the" - "connection" - 12: double "latency of dedicated connection (s)" - 13: double "latency due to distance (s)" -};; - -*******************************************************************************/ - - -"wide area network information" {"", 1, 0, 4, 0.0, 0.0, 1};; - -"environment information" {"", 0, "", REPLACE_BY_NTASKS, 0.0, 0, 1};; - -"multi node information" {0, REPLACE_BY_NTASKS, "", REPLACE_BY_CPUS_PER_NODE, 1.0, 0.0, 0.0, 0, 1, 0, 0.0, 1, 1, 0.0};; - -"predefined mapping information" {"1 TASKS_PER_NODE"};; - -"configuration files" {"", "", "REPLACE_BY_COLLECTIVES_PATH", ""};; - -"file system parameters" {0.0, 0.0, 8.0, 0, 1.0};; - diff --git a/cfgs/efficiency_table-global.gp b/cfgs/efficiency_table-global.gp deleted file mode 100644 index c7a5ef6..0000000 --- a/cfgs/efficiency_table-global.gp +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/gnuplot -set output "./efficiency_table-global.png" - -#Prepare the plot size - -#REPLACE_BY_SIZE - -set datafile separator "," - -set title "" -unset key - -set format cb "%.0f%%" -set cbrange[0:100] -set tmargin 1 - - -set cbtics ('<50%%' 50, '60%%' 60, '70%%' 70, '80%%' 80, "90%%" 90, "100%%" 100) - -set palette defined (0.00 "#000000", 0.01 "#FF0000", 0.60 "#E67C73", 0.75 "#F9950A", 0.85 "#AADC32", 0.90 "#5CC863", 1.00 "#57BB8A") - - -set xtics offset 0,15 -set ytics offset -27,0 left -set yrange [] reverse - -plot \ - 'efficiency_table-global.csv' \ - matrix rowheaders columnheaders using 1:2:3 with image, \ - 'efficiency_table-global.csv' \ - matrix rowheaders columnheaders using 1:2:(sprintf("%.2f",$3) ) with labels diff --git a/cfgs/efficiency_table-hybrid.gp b/cfgs/efficiency_table-hybrid.gp deleted file mode 100644 index 0e726cb..0000000 --- a/cfgs/efficiency_table-hybrid.gp +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/gnuplot -set output "./efficiency_table-hybrid.png" - -#Prepare the plot size - -#REPLACE_BY_SIZE - -set datafile separator "," - -set title "" -unset key - -set format cb "%.0f%%" -set cbrange[0:100] -set tmargin 1 - - -set cbtics ('<50%%' 50, '60%%' 60, '70%%' 70, '80%%' 80, "90%%" 90, "100%%" 100) - -set palette defined (0.00 "#000000", 0.01 "#FF0000", 0.60 "#E67C73", 0.75 "#F9950A", 0.85 "#AADC32", 0.90 "#5CC863", 1.00 "#57BB8A") - -set xtics offset 0,15 -set ytics offset -37,0 left -set yrange [] reverse - -plot \ - 'efficiency_table-hybrid.csv' \ - matrix rowheaders columnheaders using 1:2:3 with image, \ - 'efficiency_table-hybrid.csv' \ - matrix rowheaders columnheaders using 1:2:(sprintf("%.2f",$3) ) with labels diff --git a/cfgs/efficiency_table.gp b/cfgs/efficiency_table.gp deleted file mode 100644 index cda44e1..0000000 --- a/cfgs/efficiency_table.gp +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/gnuplot -set output "./efficiency_table.png" - -#Prepare the plot size - -#REPLACE_BY_SIZE - -set datafile separator "," - -set title "" -unset key - -set format cb "%.0f%%" -set cbrange[0:100] -set tmargin 1 - - -set cbtics ('<50%%' 50, '60%%' 60, '70%%' 70, '80%%' 80, "90%%" 90, "100%%" 100) - -set palette defined (0.00 "#000000", 0.01 "#FF0000", 0.60 "#E67C73", 0.75 "#F9950A", 0.85 "#AADC32", 0.90 "#5CC863", 1.00 "#57BB8A") - - -set xtics offset 0,15 -set ytics offset -27,0 left -set yrange [] reverse - -plot \ - 'efficiency_table.csv' \ - matrix rowheaders columnheaders using 1:2:3 with image, \ - 'efficiency_table.csv' \ - matrix rowheaders columnheaders using 1:2:(sprintf("%.2f",$3) ) with labels diff --git a/cfgs/flushing-cycles.cfg b/cfgs/flushing-cycles.cfg deleted file mode 100644 index dcd59fe..0000000 --- a/cfgs/flushing-cycles.cfg +++ /dev/null @@ -1,148 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 3 - - -################################################################################ -< NEW DISPLAYING WINDOW Flushing.c1.c1 > -################################################################################ -window_name Flushing.c1.c1 -window_type single -window_id 1 -window_position_x 442 -window_position_y 116 -window_width 600 -window_height 115 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 34.000000000000 -window_minimum_y 0.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } -window_filter_module evt_type 1 40000003 -window_filter_module evt_type_label 1 "Flushing Traces" - -################################################################################ -< NEW DISPLAYING WINDOW cycles.c1 > -################################################################################ -window_name cycles.c1 -window_type single -window_id 2 -window_position_x 417 -window_position_y 404 -window_width 600 -window_height 114 -window_comm_lines_enabled true -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 194620048057.000000000000 -window_minimum_y 0.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_random -window_drawmode_rows draw_random -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_filter_module evt_type 1 42000059 -window_filter_module evt_type_label 1 "PAPI_TOT_CYC [Total cycles]" - -################################################################################ -< NEW DISPLAYING WINDOW flushing_cycles > -################################################################################ -window_name flushing_cycles -window_type composed -window_id 3 -window_factors 1.000000000000 1.000000000000 -window_operation product -window_identifiers 1 2 -window_position_x 2357 -window_position_y 217 -window_width 600 -window_height 115 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_units Microseconds -window_maximum_y 79325.000000000000 -window_minimum_y 48287.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_random -window_drawmode_rows draw_random -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } - -< NEW ANALYZER2D > -Analyzer2D.Name: flushing_cycles_hist -Analyzer2D.X: 2597 -Analyzer2D.Y: 55 -Analyzer2D.Width: 436 -Analyzer2D.Height: 587 -Analyzer2D.ControlWindow: 3 -Analyzer2D.DataWindow: 3 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Sum bursts -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: False -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: False -Analyzer2D.Minimum: 0.000000000000 -Analyzer2D.Maximum: 1000000000000000000.000000000000 -Analyzer2D.Delta: 1000000000000000000.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 48287.000000000000 -Analyzer2D.MaximumGradient: 79325.000000000000 -Analyzer2D.DrawModeObjects: draw_maximum -Analyzer2D.DrawModeColumns: draw_maximum -Analyzer2D.PixelSize: 1 -Analyzer2D.ColorMode: window_in_gradient_mode -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/cfgs/flushing-inst.cfg b/cfgs/flushing-inst.cfg deleted file mode 100644 index 767e690..0000000 --- a/cfgs/flushing-inst.cfg +++ /dev/null @@ -1,148 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 3 - - -################################################################################ -< NEW DISPLAYING WINDOW Flushing.c1 > -################################################################################ -window_name Flushing.c1 -window_type single -window_id 1 -window_position_x 355 -window_position_y 29 -window_width 600 -window_height 115 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 34.000000000000 -window_minimum_y 0.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } -window_filter_module evt_type 1 40000003 -window_filter_module evt_type_label 1 "Flushing Traces" - -################################################################################ -< NEW DISPLAYING WINDOW Instructions.c1.c2.c1 > -################################################################################ -window_name Instructions.c1.c2.c1 -window_type single -window_id 2 -window_position_x 384 -window_position_y 58 -window_width 600 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 206968292.000000000000 -window_minimum_y 3314.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_filter_module evt_type 1 42000050 -window_filter_module evt_type_label 1 "PAPI_TOT_INS [Instr completed]" - -################################################################################ -< NEW DISPLAYING WINDOW flushing_inst > -################################################################################ -window_name flushing_inst -window_type composed -window_id 3 -window_factors 1.000000000000 1.000000000000 -window_operation product -window_identifiers 1 2 -window_position_x 2364 -window_position_y 128 -window_width 600 -window_height 115 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_units Microseconds -window_maximum_y 17421.000000000000 -window_minimum_y 17121.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_random -window_drawmode_rows draw_random -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } - -< NEW ANALYZER2D > -Analyzer2D.Name: flushing_inst_hist -Analyzer2D.X: 2597 -Analyzer2D.Y: 55 -Analyzer2D.Width: 436 -Analyzer2D.Height: 587 -Analyzer2D.ControlWindow: 3 -Analyzer2D.DataWindow: 3 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Sum bursts -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: False -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: False -Analyzer2D.Minimum: 0.000000000000 -Analyzer2D.Maximum: 1000000000000000000.000000000000 -Analyzer2D.Delta: 1000000000000000000.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 17121.000000000000 -Analyzer2D.MaximumGradient: 17421.000000000000 -Analyzer2D.DrawModeObjects: draw_maximum -Analyzer2D.DrawModeColumns: draw_maximum -Analyzer2D.PixelSize: 1 -Analyzer2D.ColorMode: window_in_gradient_mode -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/cfgs/flushing.cfg b/cfgs/flushing.cfg deleted file mode 100644 index 6c7e258..0000000 --- a/cfgs/flushing.cfg +++ /dev/null @@ -1,76 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 1 - - -################################################################################ -< NEW DISPLAYING WINDOW Flushing > -################################################################################ -window_name Flushing -window_type single -window_id 1 -window_position_x 2529 -window_position_y 349 -window_width 600 -window_height 115 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 34.000000000000 -window_minimum_y 0.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 72, { All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_filter_module evt_type 1 40000003 -window_filter_module evt_type_label 1 "Flushing Traces" - -< NEW ANALYZER2D > -Analyzer2D.Name: flushing -Analyzer2D.X: 2717 -Analyzer2D.Y: 204 -Analyzer2D.Width: 617 -Analyzer2D.Height: 606 -Analyzer2D.ControlWindow: 1 -Analyzer2D.DataWindow: 1 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Time -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: False -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: True -Analyzer2D.Minimum: 1.000000000000 -Analyzer2D.Maximum: 1.000000000000 -Analyzer2D.Delta: 1.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 18.053000000000 -Analyzer2D.MaximumGradient: 35.420000000000 -Analyzer2D.DrawModeObjects: draw_maximum -Analyzer2D.DrawModeColumns: draw_maximum -Analyzer2D.PixelSize: 1 -Analyzer2D.ColorMode: window_in_gradient_mode -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/cfgs/instructions.cfg b/cfgs/instructions.cfg deleted file mode 100644 index 777df0c..0000000 --- a/cfgs/instructions.cfg +++ /dev/null @@ -1,141 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 3 - - -################################################################################ -< NEW DISPLAYING WINDOW Instructions > -################################################################################ -window_name Instructions -window_type single -window_id 1 -window_position_x 346 -window_position_y 41 -window_width 600 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 18640295641.000000000000 -window_minimum_y 5257.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode 1 -window_drawmode_rows 1 -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_filter_module evt_type 1 42000050 - -################################################################################ -< NEW DISPLAYING WINDOW Useful > -################################################################################ -window_name Useful -window_type single -window_id 2 -window_position_x 369 -window_position_y 92 -window_width 600 -window_height 134 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered true -window_physical_filtered true -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 1.000000000000 -window_minimum_y 0.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode 1 -window_drawmode_rows 1 -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Useful}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, =}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, All}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } - -################################################################################ -< NEW DISPLAYING WINDOW Useful Instructions Timeline > -################################################################################ -window_name Useful Instructions Timeline -window_type composed -window_id 3 -window_factors 1.000000000000 1.000000000000 -window_operation product -window_identifiers 1 2 -window_position_x 406 -window_position_y 340 -window_width 767 -window_height 314 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_units Microseconds -window_maximum_y 18640295641.000000000000 -window_minimum_y 5257.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode 1 -window_drawmode_rows 1 -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } - -< NEW ANALYZER2D > -Analyzer2D.Name: instructions -Analyzer2D.X: 366 -Analyzer2D.Y: 24 -Analyzer2D.Width: 556 -Analyzer2D.Height: 764 -Analyzer2D.ControlWindow: 3 -Analyzer2D.DataWindow: 3 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Sum bursts -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: False -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: False -Analyzer2D.Minimum: 0.000000000000 -Analyzer2D.Maximum: 1000000000000000000.000000000000 -Analyzer2D.Delta: 1000000000000000000.000000000000 -Analyzer2D.ComputeGradient: False -Analyzer2D.MinimumGradient: 0 -Analyzer2D.MaximumGradient: 999999999999999999 -Analyzer2D.PixelSize: 1 -Analyzer2D.CodeColor: False - diff --git a/cfgs/io-call-cycles.cfg b/cfgs/io-call-cycles.cfg deleted file mode 100644 index 6140f79..0000000 --- a/cfgs/io-call-cycles.cfg +++ /dev/null @@ -1,148 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 3 - - -################################################################################ -< NEW DISPLAYING WINDOW I/O call.c1.c1 > -################################################################################ -window_name I/O call.c1.c1 -window_type single -window_id 1 -window_position_x 616 -window_position_y 290 -window_width 729 -window_height 154 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 13.000000000000 -window_minimum_y 1.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_random -window_drawmode_rows draw_random -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } -window_filter_module evt_type 1 40000004 -window_filter_module evt_type_label 1 "I/O calls" - -################################################################################ -< NEW DISPLAYING WINDOW cycles.c1.c1.c1 > -################################################################################ -window_name cycles.c1.c1.c1 -window_type single -window_id 2 -window_position_x 645 -window_position_y 319 -window_width 600 -window_height 114 -window_comm_lines_enabled true -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 194620048057.000000000000 -window_minimum_y 0.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_random -window_drawmode_rows draw_random -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_filter_module evt_type 1 42000059 -window_filter_module evt_type_label 1 "PAPI_TOT_CYC [Total cycles]" - -################################################################################ -< NEW DISPLAYING WINDOW posixio_cycles > -################################################################################ -window_name posixio_cycles -window_type composed -window_id 3 -window_factors 1.000000000000 1.000000000000 -window_operation product -window_identifiers 1 2 -window_position_x 2298 -window_position_y 141 -window_width 729 -window_height 154 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_units Microseconds -window_maximum_y 72043816.000000000000 -window_minimum_y 3111.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_random -window_drawmode_rows draw_random -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } - -< NEW ANALYZER2D > -Analyzer2D.Name: posixio_cycles_hist -Analyzer2D.X: 2597 -Analyzer2D.Y: 55 -Analyzer2D.Width: 436 -Analyzer2D.Height: 587 -Analyzer2D.ControlWindow: 3 -Analyzer2D.DataWindow: 3 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Sum bursts -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: False -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: False -Analyzer2D.Minimum: 0.000000000000 -Analyzer2D.Maximum: 1000000000000000000.000000000000 -Analyzer2D.Delta: 1000000000000000000.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 6685313.000000000000 -Analyzer2D.MaximumGradient: 8988531.000000000000 -Analyzer2D.DrawModeObjects: draw_maximum -Analyzer2D.DrawModeColumns: draw_maximum -Analyzer2D.PixelSize: 1 -Analyzer2D.ColorMode: window_in_gradient_mode -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/cfgs/io-call-instructions.cfg b/cfgs/io-call-instructions.cfg deleted file mode 100644 index af8bc3b..0000000 --- a/cfgs/io-call-instructions.cfg +++ /dev/null @@ -1,148 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 3 - - -################################################################################ -< NEW DISPLAYING WINDOW I/O call.c1 > -################################################################################ -window_name I/O call.c1 -window_type single -window_id 1 -window_position_x 355 -window_position_y 29 -window_width 729 -window_height 154 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 13.000000000000 -window_minimum_y 1.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } -window_filter_module evt_type 1 40000004 -window_filter_module evt_type_label 1 "I/O calls" - -################################################################################ -< NEW DISPLAYING WINDOW Instructions.c1.c2 > -################################################################################ -window_name Instructions.c1.c2 -window_type single -window_id 2 -window_position_x 384 -window_position_y 58 -window_width 600 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 206968292.000000000000 -window_minimum_y 3314.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_filter_module evt_type 1 42000050 -window_filter_module evt_type_label 1 "PAPI_TOT_INS [Instr completed]" - -################################################################################ -< NEW DISPLAYING WINDOW posixio_inst > -################################################################################ -window_name posixio_inst -window_type composed -window_id 3 -window_factors 1.000000000000 1.000000000000 -window_operation product -window_identifiers 1 2 -window_position_x 413 -window_position_y 87 -window_width 729 -window_height 154 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_units Microseconds -window_maximum_y 3625596.000000000000 -window_minimum_y 3676.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_random -window_drawmode_rows draw_random -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } - -< NEW ANALYZER2D > -Analyzer2D.Name: posixio_inst_hist -Analyzer2D.X: 2597 -Analyzer2D.Y: 55 -Analyzer2D.Width: 436 -Analyzer2D.Height: 587 -Analyzer2D.ControlWindow: 3 -Analyzer2D.DataWindow: 3 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Sum bursts -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: False -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: False -Analyzer2D.Minimum: 0.000000000000 -Analyzer2D.Maximum: 1000000000000000000.000000000000 -Analyzer2D.Delta: 1000000000000000000.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 2247759.000000000000 -Analyzer2D.MaximumGradient: 2503143.000000000000 -Analyzer2D.DrawModeObjects: draw_maximum -Analyzer2D.DrawModeColumns: draw_maximum -Analyzer2D.PixelSize: 1 -Analyzer2D.ColorMode: window_in_gradient_mode -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/cfgs/io-call-reverse.cfg b/cfgs/io-call-reverse.cfg deleted file mode 100644 index 196ec98..0000000 --- a/cfgs/io-call-reverse.cfg +++ /dev/null @@ -1,76 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 1 - - -################################################################################ -< NEW DISPLAYING WINDOW I/O call > -################################################################################ -window_name I/O call -window_type single -window_id 1 -window_position_x 2614 -window_position_y 238 -window_width 729 -window_height 154 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 18.000000000000 -window_minimum_y 0.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_filter_module evt_type 1 40000004 -window_filter_module evt_type_label 1 "I/O calls" - -< NEW ANALYZER2D > -Analyzer2D.Name: io-call -Analyzer2D.X: 2971 -Analyzer2D.Y: 50 -Analyzer2D.Width: 646 -Analyzer2D.Height: 565 -Analyzer2D.ControlWindow: 1 -Analyzer2D.DataWindow: 1 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Time -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: False -Analyzer2D.HorizVert: Vertical -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: True -Analyzer2D.Minimum: 1.000000000000 -Analyzer2D.Maximum: 13.000000000000 -Analyzer2D.Delta: 1.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 453.192000000000 -Analyzer2D.MaximumGradient: 17291.650000000001 -Analyzer2D.DrawModeObjects: draw_maximum -Analyzer2D.DrawModeColumns: draw_maximum -Analyzer2D.PixelSize: 1 -Analyzer2D.ColorMode: window_in_gradient_mode -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/cfgs/modelfactors-all.gp b/cfgs/modelfactors-all.gp deleted file mode 100644 index e05e716..0000000 --- a/cfgs/modelfactors-all.gp +++ /dev/null @@ -1,36 +0,0 @@ -#Gnuplot template for the projection functions - -#Prepare the axes -#REPLACE_BY_XRANGE -set xlabel "Number of Processes" -set logscale x -#REPLACE_BY_YRANGE -set ylabel "Efficiency" -set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) -set grid ytics - -set style line 1 lt 7 dt 2 lw 1.5 lc rgb "#0E3D59" -set style line 2 lt 7 dt 2 lw 1.5 lc rgb "#88A61B" -set style line 3 lt 7 dt 2 lw 1.5 lc rgb "#F29F05" -set style line 4 lt 7 dt 2 lw 1.5 lc rgb "#F25C05" -set style line 5 lt 7 dt 2 lw 1.5 lc rgb "#D92525" - -set key left bottom Left reverse - -#REPLACE_BY_PARA_FUNCTION -#REPLACE_BY_LOAD_FUNCTION -#REPLACE_BY_COMM_FUNCTION -#REPLACE_BY_COMP_FUNCTION -#REPLACE_BY_GLOB_FUNCTION - -plot para(x) title "Parallel Efficiency" ls 1,\ - load(x) title "Load Balance" ls 2,\ - comm(x) title "Communication Efficiency" ls 3,\ - comp(x) title "Computation Scalability" ls 4,\ - glob(x) title "Global Efficiency" ls 5,\ - '-' with points notitle ls 1,\ - '-' with points notitle ls 2,\ - '-' with points notitle ls 3,\ - '-' with points notitle ls 4,\ - '-' with points notitle ls 5 - diff --git a/cfgs/modelfactors-comm.gp b/cfgs/modelfactors-comm.gp deleted file mode 100644 index 6984035..0000000 --- a/cfgs/modelfactors-comm.gp +++ /dev/null @@ -1,26 +0,0 @@ - -#Gnuplot template for the projection functions - -#REPLACE_BY_TRACE_NAMES -#REPLACE_BY_XRANGE -set xlabel "Number of Processes" -#REPLACE_BY_XTICS_LABEL -## set logscale x -#REPLACE_BY_YRANGE -set ylabel "Efficiency (%)" -# set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) -# set grid ytics - -set style line 1 lt 7 dt 2 lw 1.5 lc rgb "red" -set style line 2 lt 7 dt 2 lw 1.5 lc rgb "green" -set style line 3 lt 7 dt 2 lw 1.5 lc rgb "orange" -set style line 4 lt 7 dt 2 lw 1.5 lc rgb "blue" -set style line 5 lt 7 dt 2 lw 1.5 lc rgb "magenta" - -set key left bottom Left reverse - - -plot '-' with linespoints title "Communication Efficiency" ls 1,\ - '-' with linespoints title "Serialization Efficiency" ls 2,\ - '-' with linespoints title "Transfer Efficiency" ls 3 - diff --git a/cfgs/modelfactors-hybrid.gp b/cfgs/modelfactors-hybrid.gp deleted file mode 100644 index 606ccd2..0000000 --- a/cfgs/modelfactors-hybrid.gp +++ /dev/null @@ -1,37 +0,0 @@ - -#Gnuplot template for the projection functions - -#REPLACE_BY_TRACE_NAMES -#REPLACE_BY_XRANGE -set xlabel "Number of Processes" -#REPLACE_BY_XTICS_LABEL -## set logscale x -#REPLACE_BY_YRANGE -set ylabel "Efficiency (%)" -# set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) -# set grid ytics - - -set style line 1 lt 8 dt 2 lw 2.0 lc rgb '#4B0082' # indigo - -set style line 2 lt 7 dt 2 lw 1.5 lc rgb "forest-green" -set style line 3 lt 6 dt 2 lw 1.5 lc rgb '#00FF00' # lime -set style line 4 lt 6 dt 2 lw 1.5 lc rgb '#008B8B' # darkcyan - -set style line 5 lt 5 dt 2 lw 1.5 lc rgb "red" -set style line 6 lt 4 dt 2 lw 1.5 lc rgb "orange" -set style line 7 lt 4 dt 2 lw 1.5 lc rgb "salmon" - -set key left bottom Left reverse - - - -plot '-' with linespoints title "Hybrid Efficiency" ls 1,\ - '-' with linespoints title "MPI Parallel efficiency" ls 2,\ - '-' with linespoints title "MPI Load balance" ls 3,\ - '-' with linespoints title "MPI Communication efficiency" ls 4,\ -#REPLACE_BY_OMP_PAR_EFF -#REPLACE_BY_OMP_LB -#REPLACE_BY_OMP_COMM - - diff --git a/cfgs/modelfactors-mpi-hybrid.gp b/cfgs/modelfactors-mpi-hybrid.gp deleted file mode 100644 index 3e30589..0000000 --- a/cfgs/modelfactors-mpi-hybrid.gp +++ /dev/null @@ -1,32 +0,0 @@ - -#Gnuplot template for the projection functions - -#REPLACE_BY_TRACE_NAMES -#REPLACE_BY_XRANGE -set xlabel "Number of Processes" -#REPLACE_BY_XTICS_LABEL -## set logscale x -#REPLACE_BY_YRANGE -set ylabel "Efficiency (%)" -# set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) -# set grid ytics - -set style line 1 lt 5 dt 2 lw 1.5 lc rgb '#1c1044' # dark blue - -set style line 2 lt 8 dt 2 lw 1.5 lc rgb "forest-green" -set style line 3 lt 6 dt 2 lw 1.5 lc rgb '#00FF00' # lime -set style line 4 lt 5 dt 2 lw 1.5 lc rgb '#008B8B' # darkcyan - - -set style line 8 lt 4 dt 2 lw 1.5 lc rgb '#7CFC00' # lawngreen -set style line 9 lt 4 dt 2 lw 1.5 lc rgb '#00FFFF' # cyan - - -set key left bottom Left reverse - - -plot '-' with linespoints title "MPI Parallel efficiency" ls 2,\ - '-' with linespoints title "MPI Load balance" ls 3,\ - '-' with linespoints title "MPI Communication efficiency" ls 4,\ - '-' with linespoints title "Serialization Efficiency" ls 8,\ - '-' with linespoints title "Transfer Efficiency" ls 9 diff --git a/cfgs/modelfactors-onlydata.gp b/cfgs/modelfactors-onlydata.gp deleted file mode 100644 index 4d0ff06..0000000 --- a/cfgs/modelfactors-onlydata.gp +++ /dev/null @@ -1,31 +0,0 @@ - -#Gnuplot template for the projection functions - -#REPLACE_BY_TRACE_NAMES -#REPLACE_BY_XRANGE -set xlabel "Number of Processes" -#REPLACE_BY_XTICS_LABEL -# set logscale x -#REPLACE_BY_YRANGE -## set yrange [0:] -set ylabel "Efficiency (%)" - -# set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) -# set grid ytics - -set style line 1 lt 8 dt 2 lw 2.0 lc rgb '#4B0082' # indigo -set style line 2 lt 7 dt 2 lw 1.5 lc rgb "red" -set style line 3 lt 7 dt 2 lw 1.5 lc rgb "green" - -set style line 4 lt 5 dt 5 lw 1.5 lc rgb "blue" -set style line 5 lt 3 dt 1 lw 1.8 lc rgb "dark-grey" - -set key left bottom Left reverse - - -plot '-' with linespoints title "Parallel Efficiency" ls 1,\ - '-' with linespoints title "Load Balance" ls 2,\ - '-' with linespoints title "Communication Efficiency" ls 3,\ - '-' with linespoints title "Computation Scalability" ls 4,\ - '-' with linespoints title "Global Efficiency" ls 5 - diff --git a/cfgs/modelfactors-scale.gp b/cfgs/modelfactors-scale.gp deleted file mode 100644 index febb708..0000000 --- a/cfgs/modelfactors-scale.gp +++ /dev/null @@ -1,27 +0,0 @@ - -#Gnuplot template for the projection functions - -#REPLACE_BY_TRACE_NAMES -#REPLACE_BY_XRANGE -set xlabel "Number of Processes" -#REPLACE_BY_XTICS_LABEL -# set logscale x -#REPLACE_BY_YRANGE -set ylabel "Efficiency (%)" -# set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) -# set grid ytics - -set style line 1 lt 5 dt 2 lw 2.0 lc rgb "blue" -set style line 2 lt 7 dt 2 lw 1.5 lc rgb "cyan" -# set style line 2 lt 7 dt 2 lw 1.5 lc rgb '#6A5ACD' # slateblue -set style line 3 lt 7 dt 2 lw 1.5 lc rgb '#4682B4' # steelblue -set style line 4 lt 7 dt 2 lw 1.5 lc rgb '#8A2BE2' # blueviolet - -set key left bottom Left reverse - - -plot '-' with linespoints title "Computation Scalability" ls 1,\ - '-' with linespoints title "IPC Scalability" ls 2,\ - '-' with linespoints title "Instruction Scalability" ls 3,\ - '-' with linespoints title "Frequency Scalability" ls 4 - diff --git a/cfgs/mpi-call-outside.cfg b/cfgs/mpi-call-outside.cfg deleted file mode 100644 index d4e7613..0000000 --- a/cfgs/mpi-call-outside.cfg +++ /dev/null @@ -1,77 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 1 - - -################################################################################ -< NEW DISPLAYING WINDOW MPI call > -################################################################################ -window_name MPI call -window_type single -window_id 1 -window_position_x 2753 -window_position_y 151 -window_width 856 -window_height 303 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 124.000000000000 -window_minimum_y 3.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, [x,y]}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_semantic_module task Thread i { 1, { 1 0.000000000000 } } -window_filter_module evt_type 2 50000001 50000005 -window_filter_module evt_type_label 2 "MPI Point-to-point" "Unknown" - -< NEW ANALYZER2D > -Analyzer2D.Name: MPI call profile -Analyzer2D.X: 2262 -Analyzer2D.Y: 247 -Analyzer2D.Width: 1381 -Analyzer2D.Height: 658 -Analyzer2D.ControlWindow: 1 -Analyzer2D.DataWindow: 1 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Time -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: True -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: True -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: False -Analyzer2D.Minimum: 0.000000000000 -Analyzer2D.Maximum: 200.000000000000 -Analyzer2D.Delta: 1.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 5.988000000000 -Analyzer2D.MaximumGradient: 60105183.622000001371 -Analyzer2D.DrawModeObjects: draw_maximum -Analyzer2D.DrawModeColumns: draw_maximum -Analyzer2D.PixelSize: 1 -Analyzer2D.ColorMode: window_in_gradient_mode -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/cfgs/mpi-io-cycles.cfg b/cfgs/mpi-io-cycles.cfg deleted file mode 100644 index ed0a0a6..0000000 --- a/cfgs/mpi-io-cycles.cfg +++ /dev/null @@ -1,149 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 3 - - -################################################################################ -< NEW DISPLAYING WINDOW cycles.c1.c1 > -################################################################################ -window_name cycles.c1.c1 -window_type single -window_id 1 -window_position_x 529 -window_position_y 203 -window_width 600 -window_height 114 -window_comm_lines_enabled true -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 194620048057.000000000000 -window_minimum_y 0.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_random -window_drawmode_rows draw_random -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_filter_module evt_type 1 42000059 -window_filter_module evt_type_label 1 "PAPI_TOT_CYC [Total cycles]" - -################################################################################ -< NEW DISPLAYING WINDOW MPI call.c1.c1 > -################################################################################ -window_name MPI call.c1.c1 -window_type single -window_id 2 -window_position_x 2303 -window_position_y 308 -window_width 642 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 1.000000000000 -window_minimum_y 1.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_random -window_drawmode_rows draw_random -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } -window_semantic_module task Thread i { 1, { 1 0.000000000000 } } -window_filter_module evt_type 1 50000005 -window_filter_module evt_type_label 1 "MPI I/O" - -################################################################################ -< NEW DISPLAYING WINDOW mpiio_cycles > -################################################################################ -window_name mpiio_cycles -window_type composed -window_id 3 -window_factors 1.000000000000 1.000000000000 -window_operation product -window_identifiers 1 2 -window_position_x 587 -window_position_y 261 -window_width 600 -window_height 114 -window_comm_lines_enabled true -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_units Microseconds -window_maximum_y 194620048057.000000000000 -window_minimum_y 0.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_random -window_drawmode_rows draw_random -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } - -< NEW ANALYZER2D > -Analyzer2D.Name: mpiio_cycles_hist -Analyzer2D.X: 2468 -Analyzer2D.Y: 122 -Analyzer2D.Width: 600 -Analyzer2D.Height: 300 -Analyzer2D.ControlWindow: 3 -Analyzer2D.DataWindow: 3 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Sum bursts -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: False -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: False -Analyzer2D.Minimum: 0.000000000000 -Analyzer2D.Maximum: 1000000000000000000.000000000000 -Analyzer2D.Delta: 1000000000000000000.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 28106010.000000000000 -Analyzer2D.MaximumGradient: 168707925.000000000000 -Analyzer2D.DrawModeObjects: draw_maximum -Analyzer2D.DrawModeColumns: draw_maximum -Analyzer2D.PixelSize: 1 -Analyzer2D.ColorMode: window_in_gradient_mode -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/cfgs/mpi-io-instructions.cfg b/cfgs/mpi-io-instructions.cfg deleted file mode 100644 index e683b74..0000000 --- a/cfgs/mpi-io-instructions.cfg +++ /dev/null @@ -1,149 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 3 - - -################################################################################ -< NEW DISPLAYING WINDOW Instructions.c1 > -################################################################################ -window_name Instructions.c1 -window_type single -window_id 1 -window_position_x 413 -window_position_y 87 -window_width 600 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 206968292.000000000000 -window_minimum_y 3314.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_filter_module evt_type 1 42000050 -window_filter_module evt_type_label 1 "PAPI_TOT_INS [Instr completed]" - -################################################################################ -< NEW DISPLAYING WINDOW MPI-io-sign.c1 > -################################################################################ -window_name MPI-io-sign.c1 -window_type single -window_id 2 -window_position_x 442 -window_position_y 116 -window_width 642 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 1.000000000000 -window_minimum_y 1.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } -window_semantic_module task Thread i { 1, { 1 0.000000000000 } } -window_filter_module evt_type 1 50000005 -window_filter_module evt_type_label 1 "MPI I/O" - -################################################################################ -< NEW DISPLAYING WINDOW mpiio_inst > -################################################################################ -window_name mpiio_inst -window_type composed -window_id 3 -window_factors 1.000000000000 1.000000000000 -window_operation product -window_identifiers 1 2 -window_position_x 2334 -window_position_y 232 -window_width 600 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_units Microseconds -window_maximum_y 206968292.000000000000 -window_minimum_y 3314.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } - -< NEW ANALYZER2D > -Analyzer2D.Name: mpiio_inst_hist_2 -Analyzer2D.X: 2427 -Analyzer2D.Y: 124 -Analyzer2D.Width: 600 -Analyzer2D.Height: 300 -Analyzer2D.ControlWindow: 3 -Analyzer2D.DataWindow: 3 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Sum bursts -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: False -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: False -Analyzer2D.Minimum: 0.000000000000 -Analyzer2D.Maximum: 1000000000000000000.000000000000 -Analyzer2D.Delta: 1000000000000000000.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 9583794.000000000000 -Analyzer2D.MaximumGradient: 108820758.000000000000 -Analyzer2D.DrawModeObjects: draw_maximum -Analyzer2D.DrawModeColumns: draw_maximum -Analyzer2D.PixelSize: 1 -Analyzer2D.ColorMode: window_in_gradient_mode -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/cfgs/mpi-io-reverse.cfg b/cfgs/mpi-io-reverse.cfg deleted file mode 100644 index 143be0d..0000000 --- a/cfgs/mpi-io-reverse.cfg +++ /dev/null @@ -1,77 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 1 - - -################################################################################ -< NEW DISPLAYING WINDOW MPI call > -################################################################################ -window_name MPI call -window_type single -window_id 1 -window_position_x 2360 -window_position_y 274 -window_width 642 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 150.000000000000 -window_minimum_y 3.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_semantic_module task Thread i { 1, { 1 0.000000000000 } } -window_filter_module evt_type 1 50000005 -window_filter_module evt_type_label 1 "MPI I/O" - -< NEW ANALYZER2D > -Analyzer2D.Name: MPI-IO -Analyzer2D.X: 2550 -Analyzer2D.Y: 40 -Analyzer2D.Width: 979 -Analyzer2D.Height: 501 -Analyzer2D.ControlWindow: 1 -Analyzer2D.DataWindow: 1 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Time -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: True -Analyzer2D.HorizVert: Vertical -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: True -Analyzer2D.Minimum: 142.000000000000 -Analyzer2D.Maximum: 150.000000000000 -Analyzer2D.Delta: 1.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 514.500000000000 -Analyzer2D.MaximumGradient: 6606164.996000000276 -Analyzer2D.DrawModeObjects: draw_maximum -Analyzer2D.DrawModeColumns: draw_maximum -Analyzer2D.PixelSize: 1 -Analyzer2D.ColorMode: window_in_gradient_mode -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/cfgs/mpi-io.cfg b/cfgs/mpi-io.cfg deleted file mode 100644 index 1f683ac..0000000 --- a/cfgs/mpi-io.cfg +++ /dev/null @@ -1,77 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 1 - - -################################################################################ -< NEW DISPLAYING WINDOW MPI call > -################################################################################ -window_name MPI call -window_type single -window_id 1 -window_position_x 2360 -window_position_y 274 -window_width 642 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 150.000000000000 -window_minimum_y 3.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_semantic_module task Thread i { 1, { 1 0.000000000000 } } -window_filter_module evt_type 1 50000005 -window_filter_module evt_type_label 1 "MPI I/O" - -< NEW ANALYZER2D > -Analyzer2D.Name: MPI-IO -Analyzer2D.X: 2862 -Analyzer2D.Y: 40 -Analyzer2D.Width: 667 -Analyzer2D.Height: 483 -Analyzer2D.ControlWindow: 1 -Analyzer2D.DataWindow: 1 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Time -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: True -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: True -Analyzer2D.Minimum: 141.000000000000 -Analyzer2D.Maximum: 150.000000000000 -Analyzer2D.Delta: 1.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 225.865000000000 -Analyzer2D.MaximumGradient: 698578.567999999970 -Analyzer2D.DrawModeObjects: draw_maximum -Analyzer2D.DrawModeColumns: draw_maximum -Analyzer2D.PixelSize: 1 -Analyzer2D.ColorMode: window_in_gradient_mode -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/cfgs/mpi-master-thread.cfg b/cfgs/mpi-master-thread.cfg deleted file mode 100644 index a130f31..0000000 --- a/cfgs/mpi-master-thread.cfg +++ /dev/null @@ -1,79 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 1 - - -################################################################################ -< NEW DISPLAYING WINDOW MPI call > -################################################################################ -window_name MPI call -window_type single -window_id 1 -window_position_x 2214 -window_position_y 148 -window_width 756 -window_height 390 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 70.000000000000 -window_minimum_y 0.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -#REPLACE_BY_APP_FILTER -#REPLACE_BY_TASKS_FILTER -#REPLACE_BY_THREADS_FILTER -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, [x,y]}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_semantic_module task Thread i { 1, { 1 0.000000000000 } } -window_filter_module evt_type 2 50000001 50000005 -window_filter_module evt_type_label 2 "MPI Point-to-point" "Unknown" - -< NEW ANALYZER2D > -Analyzer2D.Name: MPI call profile -Analyzer2D.X: 807 -Analyzer2D.Y: 207 -Analyzer2D.Width: 881 -Analyzer2D.Height: 582 -Analyzer2D.ControlWindow: 1 -Analyzer2D.DataWindow: 1 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Time -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: True -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: True -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: False -Analyzer2D.Minimum: 0.000000000000 -Analyzer2D.Maximum: 200.000000000000 -Analyzer2D.Delta: 1.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 11.456000000000 -Analyzer2D.MaximumGradient: 11868747.607000000775 -Analyzer2D.DrawModeObjects: draw_maximum -Analyzer2D.DrawModeColumns: draw_maximum -Analyzer2D.PixelSize: 1 -Analyzer2D.ColorMode: window_in_gradient_mode -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/cfgs/runtime.cfg b/cfgs/runtime.cfg deleted file mode 100644 index 5438cda..0000000 --- a/cfgs/runtime.cfg +++ /dev/null @@ -1,72 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 1 - - -################################################################################ -< NEW DISPLAYING WINDOW MPI call > -################################################################################ -window_name MPI call -window_type single -window_id 1 -window_position_x 377 -window_position_y 288 -window_width 600 -window_height 114 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 70.000000000000 -window_minimum_y 0.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode 1 -window_drawmode_rows 1 -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, [x,y]}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_semantic_module task Thread i { 1, { 1 0.000000000000 } } -window_filter_module evt_type 2 50000001 50000005 - -< NEW ANALYZER2D > -Analyzer2D.Name: runtime -Analyzer2D.X: 487 -Analyzer2D.Y: 156 -Analyzer2D.Width: 1271 -Analyzer2D.Height: 616 -Analyzer2D.ControlWindow: 1 -Analyzer2D.DataWindow: 1 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Time -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: True -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: True -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: False -Analyzer2D.Minimum: 0.000000000000 -Analyzer2D.Maximum: 200.000000000000 -Analyzer2D.Delta: 200.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 55413948.358000002801 -Analyzer2D.MaximumGradient: 55413948.358000002801 -Analyzer2D.PixelSize: 1 -Analyzer2D.CodeColor: False - diff --git a/cfgs/runtime_app.cfg b/cfgs/runtime_app.cfg deleted file mode 100644 index 791859f..0000000 --- a/cfgs/runtime_app.cfg +++ /dev/null @@ -1,74 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 1 - - -################################################################################ -< NEW DISPLAYING WINDOW runtimeapp > -################################################################################ -window_name runtimeapp -window_type single -window_id 1 -window_position_x 2457 -window_position_y 507 -window_width 581 -window_height 118 -window_comm_lines_enabled true -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered false -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 0.000000000000 -window_minimum_y 0.000000000000 -window_compute_y_max true -window_level appl -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { 1 } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, None}, {evt_value, None} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } - -< NEW ANALYZER2D > -Analyzer2D.Name: runtimeapp -Analyzer2D.X: 2513 -Analyzer2D.Y: 133 -Analyzer2D.Width: 257 -Analyzer2D.Height: 251 -Analyzer2D.ControlWindow: 1 -Analyzer2D.DataWindow: 1 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Time -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: False -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: True -Analyzer2D.Minimum: 0.000000000000 -Analyzer2D.Maximum: 0.000000000000 -Analyzer2D.Delta: 1.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 5232000.069000000134 -Analyzer2D.MaximumGradient: 5232000.069000000134 -Analyzer2D.DrawModeObjects: draw_maximum -Analyzer2D.DrawModeColumns: draw_maximum -Analyzer2D.PixelSize: 1 -Analyzer2D.ColorMode: window_in_gradient_mode -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/cfgs/time_computing.cfg b/cfgs/time_computing.cfg deleted file mode 100644 index 4bb17ad..0000000 --- a/cfgs/time_computing.cfg +++ /dev/null @@ -1,114 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 3 - - -################################################################################ -< NEW DISPLAYING WINDOW interval from states.c3 > -################################################################################ -window_name interval from states.c3 -window_type single -window_id 1 -window_position_x 377 -window_position_y 29 -window_width 600 -window_height 134 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_logical_filtered true -window_physical_filtered true -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Nanoseconds -window_maximum_y 325782.880999999994 -window_minimum_y 28.247000000000 -window_compute_y_max false -window_level thread -window_scale_relative 0.276616926766 -window_end_time_relative 0.276616926766 -window_object appl { 1, { All } } -window_begin_time_relative 0.261615159840 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, State Record Dur.}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, None}, {to_obj, None}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, None}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_semantic_module thread State Record Dur. { 1, { 2 0.000000000000 1.000000000000 } } - -################################################################################ -< NEW DISPLAYING WINDOW MPI elapsed time (ns).c1 > -################################################################################ -window_name MPI elapsed time (ns).c1 -window_type single -window_id 2 -window_position_x 406 -window_position_y 58 -window_width 600 -window_height 134 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_logical_filtered true -window_physical_filtered true -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Nanoseconds -window_maximum_y 34405654.000000000000 -window_minimum_y 28247.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 0.276616926766 -window_end_time_relative 0.276616926766 -window_object appl { 1, { All } } -window_begin_time_relative 0.261615159840 -window_open false -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, None}, {to_obj, None}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_filter_module evt_type 1 54000006 -window_filter_module evt_type_label 1 "Elapsed time in MPI" - -################################################################################ -< NEW DISPLAYING WINDOW time outside MPI > -################################################################################ -window_name time outside MPI -window_type composed -window_id 3 -window_factors 1.000000000000 1.000000000000 -window_operation substract -window_identifiers 1 2 -window_position_x 367 -window_position_y 385 -window_width 600 -window_height 134 -window_comm_lines_enabled false -window_flags_enabled false -window_noncolor_mode true -window_color_mode window_in_null_gradient_mode -window_units Microseconds -window_maximum_y 472609042.000000000000 -window_minimum_y 2905.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open true -window_drawmode draw_maximum -window_drawmode_rows draw_maximum -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } - diff --git a/cfgs/timings.cfg b/cfgs/timings.cfg deleted file mode 100644 index d2061ef..0000000 --- a/cfgs/timings.cfg +++ /dev/null @@ -1,72 +0,0 @@ -#ParaverCFG -ConfigFile.Version: 3.4 -ConfigFile.NumWindows: 1 - - -################################################################################ -< NEW DISPLAYING WINDOW States > -################################################################################ -window_name States -window_type single -window_id 1 -window_position_x 393 -window_position_y 88 -window_width 767 -window_height 314 -window_comm_lines_enabled true -window_flags_enabled false -window_noncolor_mode true -window_logical_filtered true -window_physical_filtered false -window_comm_fromto true -window_comm_tagsize true -window_comm_typeval true -window_units Microseconds -window_maximum_y 18.000000000000 -window_minimum_y 0.000000000000 -window_compute_y_max false -window_level thread -window_scale_relative 1.000000000000 -window_end_time_relative 1.000000000000 -window_object appl { 1, { All } } -window_begin_time_relative 0.000000000000 -window_open false -window_drawmode 1 -window_drawmode_rows 1 -window_pixel_size 1 -window_labels_to_draw 1 -window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, State As Is}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, All}, {evt_value, All} } } -window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } - -< NEW ANALYZER2D > -Analyzer2D.Name: States Histogram -Analyzer2D.X: 396 -Analyzer2D.Y: 433 -Analyzer2D.Width: 600 -Analyzer2D.Height: 300 -Analyzer2D.ControlWindow: 1 -Analyzer2D.DataWindow: 1 -Analyzer2D.Accumulator: Semantic -Analyzer2D.Statistic: Time -Analyzer2D.CalculateAll: True -Analyzer2D.HideCols: True -Analyzer2D.HorizVert: Horizontal -Analyzer2D.Color: True -Analyzer2D.SemanticColor: False -Analyzer2D.Zoom: Disabled -Analyzer2D.SortCols: False -Analyzer2D.SortCriteria: Average -Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -Analyzer2D.AnalysisLimits: Alltrace -Analyzer2D.ComputeYScale: True -Analyzer2D.Minimum: 1.000000000000 -Analyzer2D.Maximum: 18.000000000000 -Analyzer2D.Delta: 1.000000000000 -Analyzer2D.ComputeGradient: True -Analyzer2D.MinimumGradient: 273554.000000000000 -Analyzer2D.MaximumGradient: 9395816375.000000000000 -Analyzer2D.PixelSize: 1 -Analyzer2D.CodeColor: False -Analyzer2D.ShowOnlyTotals: False -Analyzer2D.ShortHeaderLabels: True - diff --git a/hybridmetrics.py b/hybridmetrics.py deleted file mode 100644 index 4cc6abb..0000000 --- a/hybridmetrics.py +++ /dev/null @@ -1,1496 +0,0 @@ -#!/usr/bin/env python3 - -"""Functions to compute the model metrics.""" - -from __future__ import print_function, division -import sys - - -from rawdata import * -from collections import OrderedDict - -# error import variables -error_import_pandas = False -error_import_seaborn = False -error_import_matplotlib = False -error_import_numpy = False - -try: - import numpy as np -except ImportError: - error_import_numpy = True - - -try: - import pandas as pd -except ImportError: - error_import_pandas = True - -try: - import seaborn as sns -except ImportError: - error_import_seaborn = True - - -try: - import matplotlib.pyplot as plt -except ImportError: - error_import_matplotlib = True - - - -# Contains all model factor entries with a printable name. -# This is used to generate and print all model factors, so, if an entry is added, -# it should be added here, too. - -other_metrics_doc = OrderedDict([('elapsed_time', 'Elapsed time (sec)'), - ('efficiency', 'Efficiency'), - ('speedup', 'Speedup'), - ('ipc', 'Average IPC'), - ('freq', 'Average frequency (GHz)'), - ('flushing', 'Flushing (%)'), - ('io_mpiio', 'MPI I/O (%)'), - ('io_posix', 'POSIX I/O (%)'), - ('io_eff', 'I/O Efficiency (%)')]) - -mod_factors_doc = OrderedDict([('global_eff', 'Global efficiency'), - ('parallel_eff', '-- Parallel efficiency'), - ('load_balance', ' -- Load balance'), - ('comm_eff', ' -- Communication efficiency'), - ('comp_scale', '-- Computation scalability'), - ('ipc_scale', ' -- IPC scalability'), - ('inst_scale', ' -- Instruction scalability'), - ('freq_scale', ' -- Frequency scalability')]) - -mod_factors_scale_plus_io_doc = OrderedDict([('comp_scale', '-- Computation scalability + I/O'), - ('ipc_scale', ' -- IPC scalability'), - ('inst_scale', ' -- Instruction scalability'), - ('freq_scale', ' -- Frequency scalability')]) - -mod_hybrid_factors_doc = OrderedDict([ - ('hybrid_eff', '-- Hybrid Parallel efficiency'), - ('mpi_parallel_eff', ' -- MPI Parallel efficiency'), - ('mpi_load_balance', ' -- MPI Load balance'), - ('mpi_comm_eff', ' -- MPI Communication efficiency'), - ('serial_eff', ' -- Serialization efficiency'), - ('transfer_eff', ' -- Transfer efficiency'), - ('omp_parallel_eff', ' -- OMP Parallel efficiency'), - ('omp_load_balance', ' -- OMP Load balance'), - ('omp_comm_eff', ' -- OMP Communication efficiency')]) - - -def create_mod_factors(trace_list): - """Creates 2D dictionary of the model factors and initializes with an empty - string. The mod_factors dictionary has the format: [mod factor key][trace]. - """ - global mod_factors_doc - mod_factors = {} - for key in mod_factors_doc: - trace_dict = {} - for trace_name in trace_list: - trace_dict[trace_name] = 0.0 - mod_factors[key] = trace_dict - - return mod_factors - - -def create_mod_factors_scale_io(trace_list): - """Creates 2D dictionary of the model factors and initializes with an empty - string. The mod_factors dictionary has the format: [mod factor key][trace]. - """ - global mod_factors_scale_plus_io_doc - mod_factors_scale_plus_io = {} - for key in mod_factors_scale_plus_io_doc: - trace_dict = {} - for trace_name in trace_list: - trace_dict[trace_name] = 0.0 - mod_factors_scale_plus_io[key] = trace_dict - - return mod_factors_scale_plus_io - - -def create_hybrid_mod_factors(trace_list): - """Creates 2D dictionary of the hybrid model factors and initializes with an empty - string. The hybrid_factors dictionary has the format: [mod factor key][trace]. - """ - global mod_hybrid_factors_doc - hybrid_factors = {} - for key in mod_hybrid_factors_doc: - trace_dict = {} - for trace_name in trace_list: - trace_dict[trace_name] = 0.0 - hybrid_factors[key] = trace_dict - - return hybrid_factors - - -def create_other_metrics(trace_list): - """Creates 2D dictionary of the other metrics and initializes with an empty - string. The other_metrics dictionary has the format: [mod factor key][trace]. - """ - global other_metrics_doc - other_metrics = {} - for key in other_metrics_doc: - trace_dict = {} - for trace_name in trace_list: - trace_dict[trace_name] = 0.0 - other_metrics[key] = trace_dict - - return other_metrics - - -def get_scaling_type(raw_data, trace_list, trace_processes, cmdl_args): - """Guess the scaling type (weak/strong) based on the useful instructions. - Computes the normalized instruction ratio for all measurements, whereas the - normalized instruction ratio is (instructions ratio / process ratio) with - the smallest run as reference. For exact weak scaling the normalized ratio - should be exactly 1 and for exact strong scaling it should be close to zero - with an upper bound of 0.5. The eps value defines the threshold to be - considered weak scaling and should give enough buffer to safely handle - non-ideal scaling. - """ - eps = 0.9 - normalized_inst_ratio = 0 - - # Check if there is only one trace. - if len(trace_list) == 1: - return 'strong' - - for trace in trace_list: - try: # except NaN - inst_ratio = float(raw_data['useful_ins'][trace]) / float(raw_data['useful_ins'][trace_list[0]]) - except: - inst_ratio = 0.0 - try: # except NaN - proc_ratio = float(trace_processes[trace]) / float(trace_processes[trace_list[0]]) - except: - proc_ratio = 'NaN' - - normalized_inst_ratio += inst_ratio / proc_ratio - - # Get the average inst increase. Ignore ratio of first trace 1.0) - normalized_inst_ratio = (normalized_inst_ratio - 1) / (len(trace_list) - 1) - - scaling_computed = '' - - if normalized_inst_ratio > eps: - scaling_computed = 'weak' - else: - scaling_computed = 'strong' - - if cmdl_args.scaling == 'auto': - if cmdl_args.debug: - print('==DEBUG== Detected ' + scaling_computed + ' scaling.') - print('') - return scaling_computed - - if cmdl_args.scaling == 'weak': - if scaling_computed == 'strong': - print('==Warning== Scaling set to weak scaling but detected strong scaling.') - print('') - return 'weak' - - if cmdl_args.scaling == 'strong': - if scaling_computed == 'weak': - print('==Warning== Scaling set to strong scaling but detected weak scaling.') - print('') - return 'strong' - - print('==Error== reached undefined control flow state.') - sys.exit(1) - - -def compute_model_factors(raw_data, trace_list, trace_processes, trace_mode, list_mpi_procs_count, cmdl_args): - """Computes the model factors from the gathered raw data and returns the - according dictionary of model factors.""" - mod_factors = create_mod_factors(trace_list) - hybrid_factors = create_hybrid_mod_factors(trace_list) - other_metrics = create_other_metrics(trace_list) - mod_factors_scale_plus_io = create_mod_factors_scale_io(trace_list) - - # Guess the weak or strong scaling - scaling = get_scaling_type(raw_data, trace_list, trace_processes, cmdl_args) - - # Loop over all traces - for trace in trace_list: - - proc_ratio = float(trace_processes[trace]) / float(trace_processes[trace_list[0]]) - total_procs = trace_processes[trace] - - # Flushing measurements - try: # except NaN - other_metrics['flushing'][trace] = float(raw_data['flushing_tot'][trace]) \ - / (raw_data['runtime'][trace] * total_procs) * 100.0 - except: - other_metrics['flushing'][trace] = 0.0 - - # I/O measurements - try: # except NaN - other_metrics['io_mpiio'][trace] = float(raw_data['mpiio_tot'][trace]) \ - / (raw_data['runtime'][trace] * total_procs) * 100.0 - except: - other_metrics['io_mpiio'][trace] = 0.0 - - try: # except NaN - other_metrics['io_posix'][trace] = float(raw_data['io_tot'][trace]) \ - / (raw_data['runtime'][trace] * total_procs) * 100.0 - except: - other_metrics['io_posix'][trace] = 0.0 - try: # except NaN - io_total = float(raw_data['mpiio_tot'][trace] + raw_data['flushing_tot'][trace] - + raw_data['io_tot'][trace]) - other_metrics['io_eff'][trace] = float(raw_data['useful_tot'][trace]) \ - / (raw_data['useful_tot'][trace] + io_total) * 100.0 - except: - other_metrics['io_eff'][trace] = 0.0 - - # Basic efficiency factors - try: # except NaN - if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: - print("Hello, I pass for this conditional", trace) - mod_factors['load_balance'][trace] = float(raw_data['useful_plus_io_avg'][trace]) \ - / float(raw_data['useful_plus_io_max'][trace]) * 100.0 - else: - mod_factors['load_balance'][trace] = float(raw_data['useful_avg'][trace]) \ - / float(raw_data['useful_max'][trace]) * 100.0 - except: - mod_factors['load_balance'][trace] = 'NaN' - - try: # except NaN - if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: - mod_factors['comm_eff'][trace] = float(raw_data['useful_plus_io_max'][trace] \ - / raw_data['runtime'][trace] * 100.0) - else: - mod_factors['comm_eff'][trace] = float(raw_data['useful_max'][trace] \ - / raw_data['runtime'][trace] * 100.0) - except: - mod_factors['comm_eff'][trace] = 'NaN' - - try: # except NaN - if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: - mod_factors['parallel_eff'][trace] = float(raw_data['useful_plus_io_avg'][trace]) \ - / float(raw_data['runtime'][trace]) * 100.0 - else: - mod_factors['parallel_eff'][trace] = float(mod_factors['load_balance'][trace]) \ - * float(mod_factors['comm_eff'][trace]) / 100.0 - except: - mod_factors['parallel_eff'][trace] = 'NaN' - - try: # except NaN - if len(trace_list) > 1: - if scaling == 'strong': - mod_factors['comp_scale'][trace] = float(raw_data['useful_tot'][trace_list[0]]) \ - / float(raw_data['useful_tot'][trace]) * 100.0 - else: - mod_factors['comp_scale'][trace] = float(raw_data['useful_tot'][trace_list[0]]) \ - / float(raw_data['useful_tot'][trace]) * proc_ratio * 100.0 - else: - mod_factors['comp_scale'][trace] = 'Non-Avail' - except: - mod_factors['comp_scale'][trace] = 'NaN' - - # Computation Scale + Serial I/O - try: # except NaN - if len(trace_list) > 1: - if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: - io_serial_0 = float(raw_data['io_tot'][trace_list[0]] + raw_data['flushing_tot'][trace_list[0]]) - io_serial_n = float(raw_data['io_tot'][trace] + raw_data['flushing_tot'][trace]) - if scaling == 'strong': - mod_factors_scale_plus_io['comp_scale'][trace] = float((raw_data['useful_tot'][trace_list[0]] - + io_serial_0) / (raw_data['useful_tot'][trace] - + io_serial_n) * 100.0) - else: - mod_factors_scale_plus_io['comp_scale'][trace] = (float(raw_data['useful_tot'][trace_list[0]]) - + io_serial_0) \ - / (float(raw_data['useful_tot'][trace]) - + io_serial_n) * proc_ratio * 100.0 - else: - mod_factors_scale_plus_io['comp_scale'][trace] = float(mod_factors['comp_scale'][trace]) - else: - mod_factors_scale_plus_io['comp_scale'][trace] = 'Non-Avail' - except: - mod_factors_scale_plus_io['comp_scale'][trace] = 'NaN' - - - try: # except NaN - if len(trace_list) > 1: - mod_factors['global_eff'][trace] = float(mod_factors['parallel_eff'][trace]) \ - * float(mod_factors['comp_scale'][trace]) / 100.0 - else: - mod_factors['global_eff'][trace] = float(mod_factors['parallel_eff'][trace]) * 100.0 / 100.0 - except: - mod_factors['global_eff'][trace] = 'NaN' - - # Hybrid metrics calculation - # -------> MPI metrics - try: # except NaN - hybrid_factors['mpi_load_balance'][trace] = float(raw_data['outsidempi_avg'][trace]) \ - / float(raw_data['outsidempi_max'][trace]) * 100.0 - except: - hybrid_factors['mpi_load_balance'][trace] = 'NaN' - - try: # except NaN - hybrid_factors['mpi_comm_eff'][trace] = float(raw_data['outsidempi_max'][trace]) \ - / float(raw_data['runtime'][trace]) * 100.0 - except: - hybrid_factors['mpi_comm_eff'][trace] = 'NaN' - - # ------------> BEGIN MPI communication sub-metrics - try: # except NaN - hybrid_factors['serial_eff'][trace] = float(raw_data['outsidempi_dim'][trace]) \ - / float(raw_data['runtime_dim'][trace]) * 100.0 - if hybrid_factors['serial_eff'][trace] > 100.0: - hybrid_factors['serial_eff'][trace] = 'Warning!' - except: - if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': - hybrid_factors['serial_eff'][trace] = 'NaN' - else: - hybrid_factors['serial_eff'][trace] = 'Non-Avail' - - try: # except NaN - if hybrid_factors['serial_eff'][trace] != 'Warning!': - hybrid_factors['transfer_eff'][trace] = float(hybrid_factors['mpi_comm_eff'][trace]) \ - / float(hybrid_factors['serial_eff'][trace]) * 100.0 - else: - hybrid_factors['transfer_eff'][trace] = float(raw_data['runtime_dim'][trace]) \ - / float(raw_data['runtime'][trace]) * 100.0 - if hybrid_factors['transfer_eff'][trace] > 100.0: - hybrid_factors['transfer_eff'][trace] = 'Warning!' - except: - if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': - hybrid_factors['transfer_eff'][trace] = 'NaN' - else: - hybrid_factors['transfer_eff'][trace] = 'Non-Avail' - # --------------> END MPI communication sub-metrics - try: # except NaN - if raw_data['outsidempi_tot'][trace] == raw_data['outsidempi_tot_diff'][trace]: - hybrid_factors['mpi_parallel_eff'][trace] = float(raw_data['outsidempi_tot'][trace] / - (raw_data['runtime'][trace] - * list_mpi_procs_count[trace]) * 100.0) - else: - hybrid_factors['mpi_parallel_eff'][trace] = float(raw_data['outsidempi_tot_diff'][trace] - / (raw_data['runtime'][trace] - * trace_processes[trace]) * 100.0) - except: - hybrid_factors['mpi_parallel_eff'][trace] = 'NaN' - - # -------> Metrics for the second parallel paradigm - try: # except NaN - hybrid_factors['omp_comm_eff'][trace] = float(mod_factors['comm_eff'][trace]) \ - / float(hybrid_factors['mpi_comm_eff'][trace]) * 100.0 - except: - hybrid_factors['omp_comm_eff'][trace] = 'NaN' - - try: # except NaN - hybrid_factors['omp_load_balance'][trace] = float(mod_factors['load_balance'][trace]) \ - / float(hybrid_factors['mpi_load_balance'][trace]) * 100.0 - except: - hybrid_factors['omp_load_balance'][trace] = 'NaN' - - try: # except NaN - hybrid_factors['omp_parallel_eff'][trace] = float(mod_factors['parallel_eff'][trace]) \ - / float(hybrid_factors['mpi_parallel_eff'][trace]) * 100.0 - except: - hybrid_factors['omp_parallel_eff'][trace] = 'NaN' - - # -------> Global Hybrid Metric - try: # except NaN - hybrid_factors['hybrid_eff'][trace] = float(hybrid_factors['mpi_parallel_eff'][trace]) \ - * float(hybrid_factors['omp_parallel_eff'][trace]) / 100.0 - except: - hybrid_factors['hybrid_eff'][trace] = 'NaN' - - # Basic scalability factors - try: # except NaN - other_metrics['ipc'][trace] = float(raw_data['useful_ins'][trace]) \ - / float(raw_data['useful_cyc'][trace]) - except: - other_metrics['ipc'][trace] = 'NaN' - try: # except NaN - if len(trace_list) > 1: - mod_factors['ipc_scale'][trace] = float(other_metrics['ipc'][trace]) \ - / float(other_metrics['ipc'][trace_list[0]]) * 100.0 - else: - mod_factors['ipc_scale'][trace] = 'Non-Avail' - except: - mod_factors['ipc_scale'][trace] = 'NaN' - - # IPC scale + Serial I/O - try: # except NaN - if len(trace_list) > 1: - if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: - ipc_serial_io_0 = (raw_data['useful_ins'][trace_list[0]] + raw_data['io_ins'][trace_list[0]] - + raw_data['flushing_ins'][trace_list[0]]) \ - / (raw_data['useful_cyc'][trace_list[0]] + raw_data['io_cyc'][trace_list[0]] - + raw_data['flushing_cyc'][trace_list[0]]) - - ipc_serial_io_n = (raw_data['useful_ins'][trace] + raw_data['io_ins'][trace] - + raw_data['flushing_ins'][trace])\ - / (raw_data['useful_cyc'][trace] + raw_data['io_cyc'][trace] - + raw_data['flushing_cyc'][trace]) - mod_factors_scale_plus_io['ipc_scale'][trace] = float(ipc_serial_io_n / ipc_serial_io_0 * 100.0) - else: - mod_factors_scale_plus_io['ipc_scale'][trace] = float(mod_factors['ipc_scale'][trace]) - else: - mod_factors_scale_plus_io['ipc_scale'][trace] = 'Non-Avail' - except: - mod_factors_scale_plus_io['ipc_scale'][trace] = 'NaN' - - try: # except NaN - other_metrics['freq'][trace] = float(raw_data['useful_cyc'][trace]) \ - / float(raw_data['useful_tot'][trace]) / 1000 - except: - other_metrics['freq'][trace] = 'NaN' - try: # except NaN - if len(trace_list) > 1: - mod_factors['freq_scale'][trace] = float(other_metrics['freq'][trace]) \ - / float(other_metrics['freq'][trace_list[0]]) * 100.0 - else: - mod_factors['freq_scale'][trace] = 'Non-Avail' - except: - mod_factors['freq_scale'][trace] = 'NaN' - - # freq scale + Serial I/O - try: # except NaN - if len(trace_list) > 1: - if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: - freq_serial_io_0 = (float(raw_data['useful_cyc'][trace_list[0]]) - + float(raw_data['io_cyc'][trace_list[0]]) - + float(raw_data['flushing_cyc'][trace_list[0]]))\ - / (raw_data['useful_tot'][trace_list[0]] - + float(raw_data['io_tot'][trace_list[0]]) - + float(raw_data['flushing_tot'][trace_list[0]])) / 1000 - - freq_serial_io_n = (float(raw_data['useful_cyc'][trace]) - + float(raw_data['io_cyc'][trace]) - + float(raw_data['flushing_cyc'][trace]))\ - / (float(raw_data['useful_tot'][trace]) - + float(raw_data['io_tot'][trace]) - + float(raw_data['flushing_tot'][trace])) / 1000 - mod_factors_scale_plus_io['freq_scale'][trace] = float(freq_serial_io_n / freq_serial_io_0 * 100.0) - else: - mod_factors_scale_plus_io['freq_scale'][trace] = float(mod_factors['freq_scale'][trace]) - else: - mod_factors_scale_plus_io['freq_scale'][trace] = 'Non-Avail' - except: - mod_factors_scale_plus_io['freq_scale'][trace] = 'NaN' - - try: # except NaN - if len(trace_list) > 1: - if scaling == 'strong': - mod_factors['inst_scale'][trace] = float(raw_data['useful_ins'][trace_list[0]])\ - / float(raw_data['useful_ins'][trace]) * 100.0 - else: - mod_factors['inst_scale'][trace] = float(raw_data['useful_ins'][trace_list[0]]) \ - / float(raw_data['useful_ins'][trace]) * proc_ratio * 100.0 - else: - mod_factors['inst_scale'][trace] = 'Non-Avail' - except: - mod_factors['inst_scale'][trace] = 'NaN' - - # ins scale + Serial I/O - try: # except NaN - if len(trace_list) > 1: - if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: - useful_ins_plus_io_0 = float(raw_data['useful_ins'][trace_list[0]]) \ - + float(raw_data['io_ins'][trace_list[0]]) \ - + float(raw_data['flushing_ins'][trace_list[0]]) - useful_ins_plus_io_n = float(raw_data['useful_ins'][trace]) \ - + float(raw_data['io_ins'][trace]) \ - + float(raw_data['flushing_ins'][trace]) - if scaling == 'strong': - mod_factors_scale_plus_io['inst_scale'][trace] = float(useful_ins_plus_io_0) \ - / float(useful_ins_plus_io_n) * 100.0 - else: - mod_factors_scale_plus_io['inst_scale'][trace] = float(useful_ins_plus_io_0) \ - / float(useful_ins_plus_io_n)\ - * proc_ratio * 100.0 - else: - mod_factors_scale_plus_io['inst_scale'][trace] = float(mod_factors['inst_scale'][trace]) - else: - mod_factors_scale_plus_io['inst_scale'][trace] = 'Non-Avail' - except: - mod_factors_scale_plus_io['inst_scale'][trace] = 'NaN' - - try: # except NaN - if len(trace_list) > 1: - if scaling == 'strong': - other_metrics['speedup'][trace] = float(raw_data['runtime'][trace_list[0]] - / raw_data['runtime'][trace]) - else: - other_metrics['speedup'][trace] = float(raw_data['runtime'][trace_list[0]] - / raw_data['runtime'][trace] * proc_ratio) - else: - other_metrics['speedup'][trace] = 'Non-Avail' - except: - other_metrics['speedup'][trace] = 'NaN' - - try: # except NaN - other_metrics['elapsed_time'][trace] = float(raw_data['runtime'][trace] * 0.000001) - except: - other_metrics['elapsed_time'][trace] = 'NaN' - - try: # except NaN - if len(trace_list) > 1: - #other_metrics['efficiency'][trace] = other_metrics['speedup'][trace] / proc_ratio - if scaling == 'strong': - other_metrics['efficiency'][trace] = raw_data['runtime'][trace_list[0]] \ - / (raw_data['runtime'][trace] * proc_ratio) - else: - other_metrics['efficiency'][trace] = raw_data['runtime'][trace_list[0]] \ - / raw_data['runtime'][trace] - else: - other_metrics['efficiency'][trace] = 'Non-Avail' - except: - other_metrics['efficiency'][trace] = 'NaN' - - return mod_factors, mod_factors_scale_plus_io, hybrid_factors, other_metrics - - -def print_mod_factors_table(mod_factors, other_metrics, mod_factors_scale_plus_io, hybrid_factors, trace_list, - trace_processes, trace_tasks, trace_threads,trace_mode): - """Prints the model factors table in human readable form on stdout.""" - global mod_factors_doc, mod_hybrid_factors_doc - - warning_io = [] - warning_flush = [] - warning_flush_wrong = [] - warning_simulation = [] - for trace in trace_list: - if 10.0 <= other_metrics['flushing'][trace] < 15.0: - warning_flush.append(1) - elif other_metrics['flushing'][trace] >= 15.0: - warning_flush_wrong.append(1) - if other_metrics['io_posix'][trace] >= 5.0: - warning_io.append(1) - if hybrid_factors['serial_eff'][trace] == "Warning!" \ - or hybrid_factors['serial_eff'][trace] == "Warning!": - warning_simulation.append(1) - - if len(warning_flush_wrong) > 0: - print("WARNING! Flushing in a trace is too high. Disabling standard output metrics...") - print(" Flushing is an overhead due to the tracer, please review your trace.") - print('') - return - - # Update the hybrid parallelism mode - trace_mode_doc = trace_mode[trace_list[0]] - if trace_mode_doc[0:len("Detailed+MPI+")] == "Detailed+MPI+": - mod_hybrid_factors_doc['omp_parallel_eff'] = " -- " + \ - trace_mode_doc[len("Detailed+MPI+"):] + " Parallel efficiency" - mod_hybrid_factors_doc['omp_load_balance'] = " -- " + \ - trace_mode_doc[len("Detailed+MPI+"):] + " Load Balance" - mod_hybrid_factors_doc['omp_comm_eff'] = " -- " + \ - trace_mode_doc[len("Detailed+MPI+"):] + " Communication efficiency" - - # print('') - print('\n Overview of the Efficiency metrics:') - - longest_name = len(sorted(mod_hybrid_factors_doc.values(), key=len)[-1]) - - line = ''.rjust(longest_name) - if len(trace_list) == 1: - limit_min = trace_processes[trace_list[0]] - limit_max = trace_processes[trace_list[0]] - else: - limit_min = trace_processes[trace_list[0]] - limit_max = trace_processes[trace_list[len(trace_list)-1]] - - # To control same number of processes for the header on plots and table - same_procs = True - procs_trace_prev = trace_processes[trace_list[0]] - tasks_trace_prev = trace_tasks[trace_list[0]] - threads_trace_prev = trace_threads[trace_list[0]] - - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ - and threads_trace_prev == threads: - same_procs *= True - else: - same_procs *= False - - # BEGIN To adjust header to big number of processes - procs_header = [] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if limit_min == limit_max and same_procs and len(trace_list) > 1: - s_xtics = (str(trace_processes[trace]) + '[' + str(index+1) + ']') - else: - s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') - if s_xtics in procs_header: - s_xtics += '[' + str(index + 1) + ']' - procs_header.append(s_xtics) - - max_len_header = 0 - for proc_h in procs_header: - if max_len_header < len(proc_h): - max_len_header = len(proc_h) - - value_to_adjust = 10 - if max_len_header > value_to_adjust: - value_to_adjust = max_len_header + 1 - # END To adjust header to big number of processes - - label_xtics = [] - for index, trace in enumerate(trace_list): - line += ' | ' - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if limit_min == limit_max and same_procs and len(trace_list) > 1: - s_xtics = (str(trace_processes[trace]) + '[' + str(index+1) + ']') - else: - s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') - if s_xtics in label_xtics: - s_xtics += '[' + str(index + 1) + ']' - label_xtics.append(s_xtics) - line += s_xtics.rjust(value_to_adjust) - - print(''.ljust(len(line), '=')) - print(line) - line_procs_factors = line - - print(''.ljust(len(line), '=')) - - for mod_key in mod_factors_doc: - line = mod_factors_doc[mod_key].ljust(longest_name) - for trace in trace_list: - line += ' | ' - try: # except NaN - line += ('{0:.2f}%'.format(mod_factors[mod_key][trace])).rjust(value_to_adjust) - except ValueError: - line += ('{}'.format(mod_factors[mod_key][trace])).rjust(value_to_adjust) - print(line) - - io_metrics = [] - io_metrics.append(0) - for mod_key in mod_factors_scale_plus_io_doc: - for trace in trace_list: - if mod_factors_scale_plus_io[mod_key][trace] != 0: - io_metrics.append(1) - - if (len(warning_flush) >= 1 or len(warning_io) >= 1) and len(trace_list) > 1: - print(''.ljust(len(line_procs_factors), '-')) - for mod_key in mod_factors_scale_plus_io_doc: - line = mod_factors_scale_plus_io_doc[mod_key].ljust(longest_name) - for trace in trace_list: - line += ' | ' - try: # except NaN - line += ('{0:.2f}%'.format(mod_factors_scale_plus_io[mod_key][trace])).rjust(value_to_adjust) - except ValueError: - line += ('{}'.format(mod_factors_scale_plus_io[mod_key][trace])).rjust(value_to_adjust) - print(line) - print(''.ljust(len(line_procs_factors), '=')) - else: - print(''.ljust(len(line_procs_factors), '-')) - - for mod_key in mod_hybrid_factors_doc: - line = mod_hybrid_factors_doc[mod_key].ljust(longest_name) - for trace in trace_list: - line += ' | ' - try: # except NaN - if str(hybrid_factors[mod_key][trace]) != 'nan': - line += ('{0:.2f}%'.format(hybrid_factors[mod_key][trace])).rjust(value_to_adjust) - elif str(hybrid_factors[mod_key][trace]) == 'nan': - line += ('{}'.format('NaN')).rjust(value_to_adjust) - else: - line += ('{}'.format(hybrid_factors[mod_key][trace])).rjust(value_to_adjust) - except ValueError: - line += ('{}'.format(hybrid_factors[mod_key][trace])).rjust(value_to_adjust) - print(line) - - print(''.ljust(len(line_procs_factors), '=')) - if len(warning_simulation) > 0: - print("===> Warning! Metrics obtained from simulated traces exceed 100%. " - "Please review original and simulated traces.") - print('') - - -def print_other_metrics_table(other_metrics, trace_list, trace_processes, trace_tasks, trace_threads): - """Prints the other metrics table in human readable form on stdout.""" - global other_metrics_doc - - print('Overview of the Efficiency, Speedup, IPC and Frequency:') - - longest_name = len(sorted(other_metrics_doc.values(), key=len)[-1]) - - line = ''.rjust(longest_name) - if len(trace_list) == 1: - limit_min = trace_processes[trace_list[0]] - limit_max = trace_processes[trace_list[0]] - else: - limit_min = trace_processes[trace_list[0]] - limit_max = trace_processes[trace_list[len(trace_list)-1]] - - # To control same number of processes for the header on plots and table - same_procs = True - procs_trace_prev = trace_processes[trace_list[0]] - tasks_trace_prev = trace_tasks[trace_list[0]] - threads_trace_prev = trace_threads[trace_list[0]] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ - and threads_trace_prev == threads: - same_procs *= True - else: - same_procs *= False - - # BEGIN To adjust header to big number of processes - procs_header = [] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if limit_min == limit_max and same_procs and len(trace_list) > 1: - s_xtics = (str(trace_processes[trace]) + '[' + str(index+1) + ']') - else: - s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') - if s_xtics in procs_header: - s_xtics += '[' + str(index + 1) + ']' - procs_header.append(s_xtics) - - max_len_header = 0 - for proc_h in procs_header: - if max_len_header < len(proc_h): - max_len_header = len(proc_h) - - value_to_adjust = 10 - if max_len_header > value_to_adjust: - value_to_adjust = max_len_header + 1 - # END To adjust header to big number of processes - - label_xtics = [] - for index, trace in enumerate(trace_list): - line += ' | ' - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if limit_min == limit_max and same_procs and len(trace_list) > 1: - s_xtics = (str(trace_processes[trace]) + '[' + str(index+1) + ']') - else: - s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') - if s_xtics in label_xtics: - s_xtics += '[' + str(index + 1) + ']' - label_xtics.append(s_xtics) - line += s_xtics.rjust(value_to_adjust) - - print(''.ljust(len(line), '-')) - print(line) - line_head = line - print(''.ljust(len(line), '-')) - - for mod_key in other_metrics_doc: - line = other_metrics_doc[mod_key].ljust(longest_name) - if len(trace_list) > 1: - if mod_key in ['speedup', 'ipc', 'freq', 'elapsed_time', 'efficiency']: - for trace in trace_list: - line += ' | ' - try: # except NaN - line += ('{0:.2f}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - except ValueError: - line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - print(line) - else: - if mod_key in ['ipc', 'freq', 'elapsed_time']: - for trace in trace_list: - line += ' | ' - try: # except NaN - line += ('{0:.2f}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - except ValueError: - line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - print(line) - print(''.ljust(len(line_head), '-')) - # print('') - - warning_io = [] - warning_flush = [] - for trace in trace_list: - if other_metrics['flushing'][trace] >= 10.0: - warning_flush.append(1) - if other_metrics['io_mpiio'][trace] >= 5.0 or other_metrics['io_posix'][trace] >= 5.0: - warning_io.append(1) - - for mod_key in other_metrics_doc: - line = other_metrics_doc[mod_key].ljust(longest_name) - # Print empty line to separate values - if mod_key in ['freq'] and len(warning_flush) > 0: - print('') - print("Overview of tracer\'s flushing weight:") - print(''.ljust(len(line_head), '-')) - - if mod_key not in ['speedup', 'ipc', 'freq', 'elapsed_time', 'efficiency']: - if mod_key in ['flushing']: - for trace in trace_list: - line += ' | ' - try: # except NaN - line += ('{0:.2f}%'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - except ValueError: - line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - if len(warning_flush) > 0: - print(line) - print(''.ljust(len(line_head), '-')) - elif mod_key in ['io_mpiio','io_posix','io_eff']: - for trace in trace_list: - line += ' | ' - try: # except NaN - line += ('{0:.2f}%'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - except ValueError: - line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - if len(warning_io) > 0: - print(line) - - # Print headers I/O - if mod_key in ['flushing'] and len(warning_io) > 0: - print(''.ljust(len(line), ' ')) - print('Overview of File I/O weight:') - print(''.ljust(len(line), '-')) - if mod_key in ['io_eff'] and len(warning_io) > 0: - print(''.ljust(len(line), '-')) - # print('') - - if len(warning_flush) > 0: - message_warning_flush = "WARNING! %Flushing is high and affects computation of efficiency metrics." - else: - message_warning_flush = "" - if len(warning_io) > 0: - message_warning_io = "WARNING! % File I/O is high and affects computation of efficiency metrics." - else: - message_warning_io = "" - print(message_warning_flush+message_warning_io) - # print('') - - -def print_efficiency_table(mod_factors, hybrid_factors, trace_list, trace_processes, trace_tasks, trace_threads): - """Prints the model factors table in human readable form on stdout.""" - global mod_factors_doc, mod_hybrid_factors_doc - - delimiter = ',' - # To control same number of processes for the header on plots and table - same_procs = True - procs_trace_prev = trace_processes[trace_list[0]] - tasks_trace_prev = trace_tasks[trace_list[0]] - threads_trace_prev = trace_threads[trace_list[0]] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ - and threads_trace_prev == threads: - same_procs *= True - else: - same_procs *= False - - if len(trace_list) == 1: - limit_min = trace_processes[trace_list[0]] - limit_max = trace_processes[trace_list[0]] - else: - limit_min = trace_processes[trace_list[0]] - limit_max = trace_processes[trace_list[len(trace_list) - 1]] - - file_path = os.path.join(os.getcwd(), 'efficiency_table-global.csv') - with open(file_path, 'w') as output: - line = '\"Number of processes\"' - - label_xtics = [] - for index, trace in enumerate(trace_list): - line += delimiter - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if limit_min == limit_max and same_procs and len(trace_list) > 1: - s_xtics = (str(trace_processes[trace]) + '[' + str(index + 1) + ']') - else: - s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') - if s_xtics in label_xtics: - s_xtics += '[' + str(index + 1) + ']' - label_xtics.append(s_xtics) - line += s_xtics - - output.write(line + '\n') - - for mod_key in mod_factors_doc: - if mod_key not in ['speedup', 'ipc', 'freq', 'elapsed_time', 'efficiency', 'flushing', 'io_mpiio', 'io_posix']: - if mod_key in ['parallel_eff', 'comp_scale']: - line = "\"" + mod_factors_doc[mod_key].replace(' ', '', 2) + "\"" - elif mod_key in ['load_balance', 'comm_eff','ipc_scale', 'inst_scale','freq_scale']: - line = "\"" + mod_factors_doc[mod_key].replace(' ', '', 2) + "\"" - else: - line = "\"" + mod_factors_doc[mod_key] + "\"" - for trace in trace_list: - line += delimiter - try: # except NaN - if mod_factors[mod_key][trace] == "Non-Avail" or mod_factors[mod_key][trace] == "Warning!": - line += '0.00' - else: - line += '{0:.2f}'.format(mod_factors[mod_key][trace]) - except ValueError: - line += '{}'.format(mod_factors[mod_key][trace]) - output.write(line + '\n') - - # Create Gnuplot file for efficiency plot - gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'efficiency_table-global.gp') - content = [] - - with open(gp_template) as f: - content = f.readlines() - - limit_procs = 500 + len(trace_list) * 65 - ## print(limit_procs) - - # Replace xrange - content = [line.replace('#REPLACE_BY_SIZE', ''.join(['set terminal pngcairo enhanced dashed crop size ', - str(limit_procs), ',460 font "Latin Modern Roman,14"'])) - for line in content] - - file_path = os.path.join(os.getcwd(), 'efficiency_table_global.gp') - with open(file_path, 'w') as f: - f.writelines(content) - # print('======== Plot (gnuplot File): EFFICIENCY Table ========') - if len(trace_list) > 1: - print('Global Efficiency Table written to ' + file_path[:len(file_path) - 3] + '.gp') - # print('') - - delimiter = ',' - file_path = os.path.join(os.getcwd(), 'efficiency_table-hybrid.csv') - with open(file_path, 'w') as output: - line = '\"Number of processes\"' - label_xtics = [] - for index, trace in enumerate(trace_list): - line += delimiter - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if limit_min == limit_max and same_procs and len(trace_list) > 1: - s_xtics = (str(trace_processes[trace]) + '[' + str(index + 1) + ']') - else: - s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') - if s_xtics in label_xtics: - s_xtics += '[' + str(index + 1) + ']' - label_xtics.append(s_xtics) - line += s_xtics - output.write(line + '\n') - - for mod_key in mod_hybrid_factors_doc: - if mod_key in ['mpi_parallel_eff', 'omp_parallel_eff']: - line = "\"" + mod_hybrid_factors_doc[mod_key].replace(' ', '', 2) + "\"" - elif mod_key in ['mpi_load_balance', 'mpi_comm_eff','omp_load_balance', 'omp_comm_eff']: - line = "\"" + mod_hybrid_factors_doc[mod_key].replace(' ', '', 2) + "\"" - elif mod_key in ['serial_eff', 'transfer_eff']: - line = "\"" + mod_hybrid_factors_doc[mod_key].replace(' ', ' ', 2) + "\"" - else: - line = "\"" + mod_hybrid_factors_doc[mod_key] + "\"" - for trace in trace_list: - line += delimiter - try: # except NaN - if hybrid_factors[mod_key][trace] == "Non-Avail" or hybrid_factors[mod_key][trace] == "Warning!": - line += '0.00' - else: - line += '{0:.2f}'.format(hybrid_factors[mod_key][trace]) - except ValueError: - line += '{}'.format(hybrid_factors[mod_key][trace]) - output.write(line + '\n') - - # Create Gnuplot file for efficiency plot - gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'efficiency_table-hybrid.gp') - content = [] - - with open(gp_template) as f: - content = f.readlines() - - limit_procs = 510 + len(trace_list) * 80 - - # Replace xrange - content = [line.replace('#REPLACE_BY_SIZE', ''.join(['set terminal pngcairo enhanced dashed crop size ', - str(limit_procs), ',460 font "Latin Modern Roman,14"'])) - for line in content] - - file_path = os.path.join(os.getcwd(), 'efficiency_table-hybrid.gp') - with open(file_path, 'w') as f: - f.writelines(content) - # print('======== Plot (gnuplot File): EFFICIENCY Table ========') - if len(trace_list) > 1: - print('Hybrid Efficiency Table written to ' + file_path[:len(file_path) - 3] + '.gp') - #print('') - - -def print_mod_factors_csv(mod_factors, hybrid_factors, trace_list, trace_processes): - """Prints the model factors table in a csv file.""" - global mod_factors_doc, mod_hybrid_factors_doc - - delimiter = ';' - # File is stored in the trace directory - # file_path = os.path.join(os.path.dirname(os.path.realpath(trace_list[0])), 'modelfactors.csv') - # File is stored in the execution directory - file_path = os.path.join(os.getcwd(), 'modelfactors.csv') - - with open(file_path, 'w') as output: - line = "\"Number of processes\"" - for trace in trace_list: - line += delimiter - line += str(trace_processes[trace]) - output.write(line + '\n') - - for mod_key in mod_factors_doc: - line = "\"" + mod_factors_doc[mod_key].replace(' ', '', 2) + "\"" - for trace in trace_list: - line += delimiter - try: # except NaN - line += '{0:.6f}'.format(mod_factors[mod_key][trace]) - except ValueError: - line += '{}'.format(mod_factors[mod_key][trace]) - output.write(line + '\n') - - for mod_key in mod_hybrid_factors_doc: - line = "\"" + mod_hybrid_factors_doc[mod_key].replace(' ', '', 2) + "\"" - for trace in trace_list: - line += delimiter - try: # except NaN - line += '{0:.6f}'.format(hybrid_factors[mod_key][trace]) - except ValueError: - line += '{}'.format(hybrid_factors[mod_key][trace]) - output.write(line + '\n') - - output.write('#\n') - - print('======== Output Files: Metrics and Plots ========') - print('Model factors written to ' + file_path) - - -def print_other_metrics_csv(other_metrics, trace_list, trace_processes): - """Prints the model factors table in a csv file.""" - global other_metrics_doc - - delimiter = ';' - # File is stored in the trace directory - # file_path = os.path.join(os.path.dirname(os.path.realpath(trace_list[0])), 'modelfactors.csv') - # File is stored in the execution directory - file_path = os.path.join(os.getcwd(), 'other_metrics.csv') - - with open(file_path, 'w') as output: - line = 'Number of processes' - for trace in trace_list: - line += delimiter - line += str(trace_processes[trace]) - output.write(line + '\n') - - for mod_key in other_metrics_doc: - line = other_metrics_doc[mod_key].replace(' ', '', 2) - for trace in trace_list: - line += delimiter - try: # except NaN - line += '{0:.6f}'.format(other_metrics[mod_key][trace]) - except ValueError: - line += '{}'.format(other_metrics[mod_key][trace]) - output.write(line + '\n') - - output.write('#\n') - - print('') - print('======== Output File: Other Metrics ========') - print('Speedup, IPC, Frequency, I/O and Flushing written to ' + file_path) - - -def plots_efficiency_table_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args): - # Plotting using python - # For plotting using python, read the csv file - - file_path = os.path.join(os.getcwd(), 'efficiency_table-hybrid.csv') - df = pd.read_csv(file_path) - metrics = df['Number of processes'].tolist() - - traces_procs = list(df.keys())[1:] - - # To control same number of processes for the header on plots and table - same_procs = True - procs_trace_prev = trace_processes[trace_list[0]] - tasks_trace_prev = trace_tasks[trace_list[0]] - threads_trace_prev = trace_threads[trace_list[0]] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ - and threads_trace_prev == threads: - same_procs *= True - else: - same_procs *= False - - # Set limit for projection - if cmdl_args.limit: - limit = cmdl_args.limit - else: - limit = str(trace_processes[trace_list[len(trace_list)-1]]) - - limit_min = trace_processes[trace_list[0]] - - # To xticks label - label_xtics = [] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if int(limit) == int(limit_min) and same_procs: - s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' - else: - s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' - if s_xtics in label_xtics: - s_xtics += '[' + str(index + 1) + ']' - label_xtics.append(s_xtics) - ##### End xticks - - # BEGIN To adjust header to big number of processes - max_len_header = 7 - for labelx in label_xtics: - if len(labelx) > max_len_header: - max_len_header = len(labelx) - - # END To adjust header to big number of processes - - list_data = [] - for index, rows in df.iterrows(): - list_temp = [] - for value in list(rows)[1:]: - if float(value) == 0.0: - list_temp.append(np.nan) - else: - list_temp.append(float(value)) - # print(list_temp) - list_data.append(list_temp) - - list_np = np.array(list_data) - - idx = metrics - cols = label_xtics - # cols = traces_procs - df = pd.DataFrame(list_np, index=idx, columns=cols) - - # min for 1 trace is x=3 for the (x,y) in figsize - size_figure_y = len(idx) * 0.40 - size_figure_x = len(cols) * 0.14 * max_len_header - plt.figure(figsize=(size_figure_x, size_figure_y)) - - ax = sns.heatmap(df, cmap='RdYlGn', linewidths=0.05, annot=True, vmin=0, vmax=100, center=75, \ - fmt='.2f', annot_kws={"size": 10}, cbar_kws={'label': 'Percentage(%)'}) - ## to align ylabels to left - plt.yticks(rotation=0, ha='left') - - ax.xaxis.tick_top() - # to adjust metrics - len_pad = 0 - for metric in metrics: - if len(metric) > len_pad: - len_pad = len(metric) - - ax.yaxis.set_tick_params(pad=len_pad + 164) - - plt.savefig('efficiency_table-hybrid-matplot.png', bbox_inches='tight') - - # General Metrics plot - - file_path = os.path.join(os.getcwd(), 'efficiency_table-global.csv') - df = pd.read_csv(file_path) - metrics = df['Number of processes'].tolist() - - traces_procs = list(df.keys())[1:] - - list_data = [] - for index, rows in df.iterrows(): - list_temp = [] - for value in list(rows)[1:]: - if float(value) == 0.0: - list_temp.append(np.nan) - else: - list_temp.append(float(value)) - # print(list_temp) - list_data.append(list_temp) - - list_np = np.array(list_data) - - idx = metrics - cols = label_xtics - # cols = traces_procs - df = pd.DataFrame(list_np, index=idx, columns=cols) - - # min for 1 traces is x=3 for the (x,y) in figsize - size_figure_y = len(idx) * 0.40 - size_figure_x = len(cols) * 0.14 * max_len_header - plt.figure(figsize=(size_figure_x, size_figure_y)) - - ax = sns.heatmap(df, cmap='RdYlGn', linewidths=0.05, annot=True, vmin=0, vmax=100, center=75, \ - fmt='.2f', annot_kws={"size": 10}, cbar_kws={'label': 'Percentage(%)'}) - ## to align ylabels to left - plt.yticks(rotation=0, ha='left') - ax.xaxis.tick_top() - # to adjust metrics - len_pad = 0 - for metric in metrics: - if len(metric) > len_pad: - len_pad = len(metric) - - ax.yaxis.set_tick_params(pad=len_pad + 140) - - plt.savefig('efficiency_table-global-matplot.png', bbox_inches='tight') - - -def plots_modelfactors_matplot(trace_list, trace_processes,trace_tasks, trace_threads, cmdl_args): - # Plotting using python - # For plotting using python, read the csv file - file_path = os.path.join(os.getcwd(), 'modelfactors.csv') - df = pd.read_csv(file_path, sep=';') - - traces_procs = list(df.keys())[1:] - - list_data = [] - for index, rows in df.iterrows(): - list_temp = [] - for value in list(rows)[1:]: - if value != 'Non-Avail' and value != 'Warning!': - list_temp.append(float(value)) - elif value == 'Non-Avail' or value == 'Warning!': - list_temp.append('NaN') - # print(list_temp) - list_data.append(list_temp) - - # To control same number of processes for the header on plots and table - same_procs = True - procs_trace_prev = trace_processes[trace_list[0]] - tasks_trace_prev = trace_tasks[trace_list[0]] - threads_trace_prev = trace_threads[trace_list[0]] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ - and threads_trace_prev == threads: - same_procs *= True - else: - same_procs *= False - - # Set limit for projection - if cmdl_args.limit: - limit = cmdl_args.limit - else: - limit = str(trace_processes[trace_list[len(trace_list)-1]]) - - limit_min = trace_processes[trace_list[0]] - - # To xticks label - label_xtics = [] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if int(limit) == int(limit_min) and same_procs: - s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' - else: - s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' - if s_xtics in label_xtics: - s_xtics += '[' + str(index + 1) + ']' - label_xtics.append(s_xtics) - - ## Global Metrics - plt.figure() - max_global = max([max(list_data[0]), max(list_data[1]), max(list_data[2]), - max(list_data[3]), max(list_data[4])]) - plt.plot(traces_procs, list_data[0], 'o-', color='black', label='Global Efficiency') - plt.plot(traces_procs, list_data[1], 's--', color='magenta', label='Parallel Efficiency') - plt.plot(traces_procs, list_data[2], '*:', color='red', label='Load Balance') - plt.plot(traces_procs, list_data[3], 'x-.', color='green', label='Communication efficiency') - plt.plot(traces_procs, list_data[4], 'v--', color='blue', label='Computation scalability') - plt.xlabel("Number of Processes") - plt.ylabel("Efficiency (%)") - plt.xticks(tuple(traces_procs), tuple(label_xtics)) - # print(max_global) - if float(max_global) < 100: - max_global = 100 - - plt.ylim(0, max_global+5) - plt.legend() - plt.savefig('modelfactors-global-matplot.png', bbox_inches='tight') - - ## Scale Metrics - plt.figure() - max_scale = max([max(list_data[4]), max(list_data[5]), max(list_data[6]), max(list_data[7])]) - plt.plot(traces_procs, list_data[4], 'v-', color='blue', markerfacecolor='blue', label='Computation scalability') - plt.plot(traces_procs, list_data[5], 'v--', color='skyblue', label='IPC scalability') - plt.plot(traces_procs, list_data[6], 'v:', color='gray', label='Instruction scalability') - plt.plot(traces_procs, list_data[7], 'v-.', color='darkviolet', label='Frequency scalability') - plt.xlabel("Number of Processes") - plt.ylabel("Efficiency (%)") - plt.xticks(tuple(traces_procs), tuple(label_xtics)) - if float(max_scale) < 100: - max_scale = 100 - - plt.ylim(0, max_scale+5) - plt.legend() - plt.savefig('modelfactors-scale-matplot.png', bbox_inches='tight') - - ## Hybrid Metrics - - plt.figure() - max_hybrid = max([max(list_data[8]), max(list_data[9]), max(list_data[10]), max(list_data[11]), - max(list_data[14]), max(list_data[15]), max(list_data[16])]) - plt.plot(traces_procs, list_data[8], 's-', color='purple', label='Hybrid Parallel efficiency') - plt.plot(traces_procs, list_data[9], 'o--', color='green', label='MPI Parallel efficiency') - plt.plot(traces_procs, list_data[10], 'x-.', color='lime', label='MPI Load balance') - plt.plot(traces_procs, list_data[11], '*:', color='lightseagreen', label='MPI Communication efficiency') - plt.plot(traces_procs, list_data[14], 'h--', color='red', label='OpenMP Parallel efficiency') - plt.plot(traces_procs, list_data[15], 'v-.', color='orange', label='OpenMP Load Balance') - plt.plot(traces_procs, list_data[16], 'X:', color='salmon', label='OpenMP Communication efficiency') - plt.xlabel("Number of Processes") - plt.ylabel("Efficiency (%)") - plt.xticks(tuple(traces_procs), tuple(label_xtics)) - if float(max_hybrid) < 100: - max_hybrid = 100 - - plt.ylim(0, max_hybrid+5) - plt.legend() - plt.savefig('modelfactors-hybrid-matplot.png', bbox_inches='tight') - - ## MPI Metrics - plt.figure() - if max(list_data[12]) != 'NaN' and max(list_data[13]) != 'NaN': - max_mpi = max([max(list_data[9]), max(list_data[10]), max(list_data[11]), - max(list_data[12]), max(list_data[13])]) - plt.plot(traces_procs, list_data[9], 's-', color='green', label='MPI Parallel efficiency') - plt.plot(traces_procs, list_data[10], 'v:', color='lime', label='MPI Load balance') - plt.plot(traces_procs, list_data[11], 'o-.', color='lightseagreen', label='MPI Communication efficiency') - plt.plot(traces_procs, list_data[12], linestyle=(0, (3, 10, 1, 10)), marker='s', color='gold', - label='Serialization efficiency') - plt.plot(traces_procs, list_data[13], linestyle=(0, (3, 5, 1, 5)), marker='x', color='tomato', label='Transfer efficiency') - elif max(list_data[12]) == 'NaN' and max(list_data[13]) == 'NaN': - max_mpi = max([max(list_data[9]), max(list_data[10]), max(list_data[11])]) - plt.plot(traces_procs, list_data[9], 's-', color='green', label='MPI Parallel efficiency') - plt.plot(traces_procs, list_data[10], 'v:', color='lime', label='MPI Load balance') - plt.plot(traces_procs, list_data[11], 'o-.', color='lightseagreen', label='MPI Communication efficiency') - elif max(list_data[12]) != 'NaN' and max(list_data[13]) == 'NaN': - max_mpi = max([max(list_data[9]), max(list_data[10]), max(list_data[11]), - max(list_data[12])]) - plt.plot(traces_procs, list_data[9], 's-', color='green', label='MPI Parallel efficiency') - plt.plot(traces_procs, list_data[10], 'v:', color='lime', label='MPI Load balance') - plt.plot(traces_procs, list_data[11], 'o-.', color='lightseagreen', label='MPI Communication efficiency') - plt.plot(traces_procs, list_data[12], linestyle=(0, (3, 10, 1, 10)), marker='s', color='gold', label='Serialization efficiency') - else: - max_mpi = max([max(list_data[9]), max(list_data[10]), max(list_data[11]),max(list_data[13])]) - plt.plot(traces_procs, list_data[9], 's-', color='green', label='MPI Parallel efficiency') - plt.plot(traces_procs, list_data[10], 'v:', color='lime', label='MPI Load balance') - plt.plot(traces_procs, list_data[11], 'o-.', color='lightseagreen', label='MPI Communication efficiency') - plt.plot(traces_procs, list_data[13], linestyle=(0, (3, 5, 1, 5)), marker='x', color='tomato', label='Transfer efficiency') - - plt.xlabel("Number of Processes") - plt.ylabel("Efficiency (%)") - plt.xticks(tuple(traces_procs), tuple(label_xtics)) - if float(max_mpi) < 100: - max_mpi = 100 - - plt.ylim(0, max_mpi+5) - plt.legend() - plt.savefig('modelfactors-mpi-matplot.png', bbox_inches='tight') - - -def plots_speedup_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args): - # Plotting using python - # For plotting using python, read the csv file - file_path = os.path.join(os.getcwd(), 'other_metrics.csv') - df = pd.read_csv(file_path, sep=';') - - traces_procs = list(df.keys())[1:] - - list_data = [] - for index, rows in df.iterrows(): - list_temp = [] - for value in list(rows)[1:]: - if value != 'Non-Avail': - list_temp.append(float(value)) - elif value == 'Non-Avail': - list_temp.append('NaN') - # print(list_temp) - list_data.append(list_temp) - - # To control same number of processes for the header on plots and table - same_procs = True - procs_trace_prev = trace_processes[trace_list[0]] - tasks_trace_prev = trace_tasks[trace_list[0]] - threads_trace_prev = trace_threads[trace_list[0]] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ - and threads_trace_prev == threads: - same_procs *= True - else: - same_procs *= False - - # Set limit for projection - if cmdl_args.limit: - limit = cmdl_args.limit - else: - limit = str(trace_processes[trace_list[len(trace_list) - 1]]) - - limit_min = trace_processes[trace_list[0]] - - # proc_ratio to ideal speedup - proc_ratio = [] - for index, trace in enumerate(trace_list): - proc_ratio.append(trace_processes[trace]/trace_processes[trace_list[0]]) - # print(proc_ratio) - - # To xticks label - label_xtics = [] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if int(limit) == int(limit_min) and same_procs: - s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' - elif int(limit) == int(limit_min) and not same_procs: - s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' \ - + str(threads) + ')' - else: - s_xtics = str(trace_processes[trace]) - label_xtics.append(s_xtics) - - int_traces_procs = [] - prev_procs = int(float(traces_procs[0])) - int_traces_procs.append(int(float(traces_procs[0]))) - count_rep = 0 - for procs in traces_procs[1:]: - if prev_procs == int(float(procs)): - count_rep += 1 - int_traces_procs.append(int(float(procs))+ (2*count_rep)) - prev_procs = int(float(procs)) - else: - int_traces_procs.append(int(float(procs))) - prev_procs = int(float(procs)) - count_rep = 0 - - ### Plot: Global Metrics - plt.figure() - for x, y in zip(int_traces_procs, list_data[2]): - label = "{:.2f}".format(y) - plt.annotate(label, (x, y), textcoords="offset points", xytext=(0, 10), ha='center') - - plt.plot(int_traces_procs, list_data[2], 'o-', color='blue', label='measured') - plt.plot(int_traces_procs, proc_ratio, 'o-', color='black', label='ideal') - plt.xlabel("Number of Processes") - plt.ylabel("SpeedUp") - plt.xticks(tuple(int_traces_procs), tuple(label_xtics)) - #plt.yscale('log') - plt.legend() - #plt.xlim(0, ) - plt.ylim(0, ) - plt.savefig('speedup-matplot.png', bbox_inches='tight') - - ### Plot: Efficiency - # print(list_data) - plt.figure() - for x, y in zip(int_traces_procs, list_data[1]): - label = "{:.2f}".format(y) - plt.annotate(label, (x, y), textcoords="offset points", xytext=(0, 10), ha='center') - - plt.plot(int_traces_procs, list_data[1], 'o-', color='blue', label='measured') - plt.axhline(y=1, color='black', linestyle='-', label='ideal') - plt.xlabel("Number of Processes") - plt.ylabel("Efficiency") - plt.xticks(tuple(int_traces_procs), tuple(label_xtics)) - # plt.yscale('log') - max_y = max(list_data[1]) - if max_y < 1.1: - max_y = 1.0 - #plt.xlim(0, ) - plt.ylim(0,max_y+0.1) - plt.legend() - plt.savefig('efficiency-matplot.png', bbox_inches='tight') \ No newline at end of file diff --git a/modelfactors.py b/modelfactors.py deleted file mode 100755 index 4eb5c6b..0000000 --- a/modelfactors.py +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env python3 - -"""modelfactors.py Generates performance metrics from a set of Paraver traces.""" - -from __future__ import print_function, division -import subprocess -import plots -from utils import parse_arguments, check_installation -from tracemetadata import get_traces_from_args -from rawdata import gather_raw_data, print_raw_data_table, print_raw_data_csv -from simplemetrics import compute_model_factors, print_mod_factors_csv, print_efficiency_table, \ - print_mod_factors_table, read_mod_factors_csv, print_other_metrics_table, \ - print_other_metrics_csv, plots_efficiency_table_matplot, plots_modelfactors_matplot, plots_speedup_matplot - -import hybridmetrics - -# error import variables -error_import_pandas = False -error_import_seaborn = False -error_import_matplotlib = False -error_import_scipy = False -error_import_numpy = False - -try: - import pandas as pd -except ImportError: - error_import_pandas = True - -try: - import seaborn as sns -except ImportError: - error_import_seaborn = True - -try: - import matplotlib - matplotlib.use('pdf') - import matplotlib.pyplot as plt00 -except ImportError: - error_import_matplotlib = True - -try: - import scipy.optimize -except ImportError: - error_import_scipy = True - - -try: - import numpy -except ImportError: - error_import_numpy = True - - -__author__ = "Sandra Mendez" -__copyright__ = "Copyright 2019, Barcelona Supercomputing Center (BSC)" -__version_major__ = 0 -__version_minor__ = 3 -__version_micro__ = 7 -__version__ = str(__version_major__) + "." + str(__version_minor__) + "." + str(__version_micro__) - - -if __name__ == "__main__": - """Main control flow. - Currently the script only accepts one parameter, which is a list of traces - that are processed. This can be a regex with wild cards and only valid trace - files are kept at the end. - """ - # Parse command line arguments - cmdl_args = parse_arguments() - - # Check if paramedir and Dimemas are in the path - check_installation(cmdl_args) - - # Process the trace list - trace_list, trace_processes, trace_tasks, trace_threads, trace_task_per_node, trace_mode = \ - get_traces_from_args(cmdl_args) - - # To validate the metric type - trace_metrics = 0 - # print(trace_mode) - for trace in trace_list: - if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace][:15] == 'Detailed+OpenMP' \ - or trace_mode[trace][:17] == 'Detailed+Pthreads' or trace_mode[trace][:13] == 'Detailed+CUDA' \ - or trace_mode[trace][:14] == 'Detailed+OmpSs' or trace_mode[trace][:15] == 'Detailed+OpenCL' \ - or trace_mode[trace] == 'Detailed' or trace_mode[trace][:5] == 'Burst': - trace_metrics += 1 - - # Analyze the traces and gather the raw input data - raw_data, list_mpi_procs_count = gather_raw_data(trace_list, trace_processes, trace_task_per_node, - trace_mode, cmdl_args) - print_raw_data_csv(raw_data, trace_list, trace_processes) - - # Compute the model factors and print them - - if cmdl_args.metrics == 'hybrid' and trace_metrics == 0: - mod_factors, mod_factors_scale_plus_io, hybrid_factors, other_metrics = \ - hybridmetrics.compute_model_factors(raw_data, trace_list, trace_processes, - trace_mode, list_mpi_procs_count, cmdl_args) - hybridmetrics.print_other_metrics_table(other_metrics, trace_list, trace_processes, trace_tasks, trace_threads) - hybridmetrics.print_other_metrics_csv(other_metrics, trace_list, trace_processes) - hybridmetrics.print_mod_factors_table(mod_factors, other_metrics, mod_factors_scale_plus_io, hybrid_factors, - trace_list, trace_processes, trace_tasks, trace_threads, trace_mode) - hybridmetrics.print_mod_factors_csv(mod_factors, hybrid_factors, trace_list, trace_processes) - hybridmetrics.print_efficiency_table(mod_factors, hybrid_factors, trace_list, trace_processes, trace_tasks, - trace_threads) - # Plotting efficiency table with matplotlib - error_plot_table = False - if error_import_numpy or error_import_pandas or error_import_matplotlib or error_import_seaborn: - print('Numpy/Pandas/Matplotlib/Seaborn modules not available. ' - 'Skipping efficiency table plotting with python.') - if len(trace_list) > 1: - out_ver_gnuplot = subprocess.check_output(["gnuplot", "--version"]) - if 'gnuplot 5.' not in str(out_ver_gnuplot): - print('It requires gnuplot version 5.0 or higher. ' - 'Skipping efficiency table and lineal plotting with gnuplot.') - else: - try: - output_gnuplot_g = subprocess.check_output(["gnuplot", "efficiency_table_global.gp"]) - output_gnuplot_h = subprocess.check_output(["gnuplot", "efficiency_table-hybrid.gp"]) - except: - print(output_gnuplot_g) - print(output_gnuplot_h) - error_plot_table = True - - if not error_plot_table: - hybridmetrics.plots_efficiency_table_matplot(trace_list, trace_processes, trace_tasks, - trace_threads, cmdl_args) - if len(trace_list) > 1: - hybridmetrics.plots_modelfactors_matplot(trace_list, trace_processes, trace_tasks, - trace_threads, cmdl_args) - hybridmetrics.plots_speedup_matplot(trace_list, trace_processes, trace_tasks, - trace_threads, cmdl_args) - - # Plotting if SciPy and NumPy are installed. - error_plot_lineal = False - if error_import_numpy or error_import_scipy: - print('Scipy/NumPy module not available. Skipping lineal plotting.') - error_plot_lineal = True - - if not error_plot_lineal: - if len(trace_list) > 1: - plots.plot_hybrid_metrics(mod_factors, hybrid_factors, trace_list, trace_processes, - trace_tasks, trace_threads, trace_mode, cmdl_args) - - if len(trace_list) == 1: - subprocess.check_output(["rm", "efficiency_table_global.gp"]) - subprocess.check_output(["rm", "efficiency_table-hybrid.gp"]) - - elif cmdl_args.metrics == 'simple' or trace_metrics > 0: - mod_factors, mod_factors_scale_plus_io, other_metrics = compute_model_factors(raw_data, trace_list, - trace_processes, trace_mode, - list_mpi_procs_count, cmdl_args) - print_other_metrics_table(other_metrics, trace_list, trace_processes) - print_other_metrics_csv(other_metrics, trace_list, trace_processes) - print_mod_factors_table(mod_factors, other_metrics, mod_factors_scale_plus_io, trace_list, trace_processes) - print_mod_factors_csv(mod_factors, trace_list, trace_processes) - print_efficiency_table(mod_factors, trace_list, trace_processes) - - # Plotting efficiency table with matplotlib - error_plot_table = False - if error_import_numpy or error_import_pandas or error_import_matplotlib or error_import_seaborn: - print('Numpy/Pandas/Matplotlib/Seaborn modules not available. ' - 'Skipping efficiency table plotting with python.') - if len(trace_list) > 1: - out_ver_gnuplot = subprocess.check_output(["gnuplot", "--version"]) - if 'gnuplot 5.' not in str(out_ver_gnuplot): - print('It requires gnuplot version 5.0 or higher. ' - 'Skipping efficiency table and lineal plotting with gnuplot.') - else: - try: - output_gnuplot = subprocess.check_output(["gnuplot", "efficiency_table.gp"]) - except: - print(output_gnuplot) - error_plot_table = True - - if not error_plot_table: - plots_efficiency_table_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args) - if len(trace_list) > 1: - plots_modelfactors_matplot(trace_list, trace_mode, trace_processes, trace_tasks, trace_threads, - cmdl_args) - - # Plotting if SciPy and NumPy are installed. - error_plot_lineal = False - if error_import_numpy or error_import_scipy: - print('Scipy/NumPy module not available. Skipping lineal plotting.') - error_plot_lineal = True - # sys.exit(1) - if not error_plot_lineal: - if len(trace_list) > 1: - plots.plot_simple_metrics(mod_factors, trace_list, trace_processes, trace_mode, cmdl_args) - plots_speedup_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args) - if len(trace_list) == 1: - subprocess.check_output(["rm", "efficiency_table.gp"]) diff --git a/plots.py b/plots.py deleted file mode 100644 index 0e28cf1..0000000 --- a/plots.py +++ /dev/null @@ -1,738 +0,0 @@ -#!/usr/bin/env python3 - -"""Functions to plots efficiencies metrics""" - -from __future__ import print_function, division -import os -from collections import OrderedDict - -# error import variables -error_import_scipy = False -error_import_numpy = False - -try: - import scipy.optimize -except ImportError: - error_import_scipy = True - # print('==ERROR== Could not import SciPy. Please make sure to install a current version.') - -try: - import numpy -except ImportError: - error_import_numpy = True - # print('==ERROR== Could not import NumPy. Please make sure to install a current version.') - - -mod_hybrid_factors_doc = OrderedDict([ - ('hybrid_eff', '-- Hybrid Parallel efficiency'), - ('mpi_parallel_eff', ' -- MPI Parallel efficiency'), - ('mpi_load_balance', ' -- MPI Load balance'), - ('mpi_comm_eff', ' -- MPI Communication efficiency'), - ('serial_eff', ' -- Serialization efficiency'), - ('transfer_eff', ' -- Transfer efficiency'), - ('omp_parallel_eff', ' -- OMP Parallel efficiency'), - ('omp_load_balance', ' -- OMP Load balance'), - ('omp_comm_eff', ' -- OMP Communication efficiency')]) - - -def plot_hybrid_metrics(mod_factors, hybrid_factors, trace_list, trace_processes, trace_tasks, trace_threads, trace_mode, cmdl_args): - """Computes the projection from the gathered model factors and returns the - according dictionary of fitted prediction functions.""" - - global mod_hybrid_factors_doc - - # Update the hybrid parallelism mode - trace_mode_doc = trace_mode[trace_list[0]] - if trace_mode_doc[0:len("Detailed+MPI+")] == "Detailed+MPI+": - mod_hybrid_factors_doc['omp_parallel_eff'] = " -- " + \ - trace_mode_doc[len("Detailed+MPI+"):] + " Parallel efficiency" - mod_hybrid_factors_doc['omp_load_balance'] = " -- " + \ - trace_mode_doc[len("Detailed+MPI+"):] + " Load Balance" - mod_hybrid_factors_doc['omp_comm_eff'] = " -- " + \ - trace_mode_doc[len("Detailed+MPI+"):] + " Communication efficiency" - - if cmdl_args.debug: - print('==DEBUG== Computing projection of model factors.') - - number_traces = len(trace_list) - x_proc = numpy.zeros(number_traces) - y_para = numpy.zeros(number_traces) - y_load = numpy.zeros(number_traces) - y_comm = numpy.zeros(number_traces) - y_comp = numpy.zeros(number_traces) - y_glob = numpy.zeros(number_traces) - y_ipc_scale = numpy.zeros(number_traces) - y_inst_scale = numpy.zeros(number_traces) - y_freq_scale = numpy.zeros(number_traces) - y_hybrid_par = numpy.zeros(number_traces) - y_mpi_par = numpy.zeros(number_traces) - y_mpi_load = numpy.zeros(number_traces) - y_mpi_comm = numpy.zeros(number_traces) - y_comm_serial = numpy.zeros(number_traces) - y_comm_transfer = numpy.zeros(number_traces) - y_omp_par = numpy.zeros(number_traces) - y_omp_load = numpy.zeros(number_traces) - y_omp_comm = numpy.zeros(number_traces) - - #Convert dictionaries to NumPy arrays - for index, trace in enumerate(trace_list): - x_proc[index] = trace_processes[trace] - y_para[index] = mod_factors['parallel_eff'][trace] - y_load[index] = mod_factors['load_balance'][trace] - y_comm[index] = mod_factors['comm_eff'][trace] - y_comp[index] = mod_factors['comp_scale'][trace] - y_glob[index] = mod_factors['global_eff'][trace] - y_ipc_scale[index] = mod_factors['ipc_scale'][trace] - y_inst_scale[index] = mod_factors['inst_scale'][trace] - y_freq_scale[index] = mod_factors['freq_scale'][trace] - y_hybrid_par[index] = hybrid_factors['hybrid_eff'][trace] - y_mpi_par[index] = hybrid_factors['mpi_parallel_eff'][trace] - y_mpi_load[index] = hybrid_factors['mpi_load_balance'][trace] - y_mpi_comm[index] = hybrid_factors['mpi_comm_eff'][trace] - if trace_mode[trace] == 'Detailed+MPI' \ - or trace_mode[trace] == 'Detailed+MPI+OpenMP': - if hybrid_factors['serial_eff'][trace] != 'Non-Avail' and hybrid_factors['serial_eff'][trace] != 'Warning!': - y_comm_serial[index] = hybrid_factors['serial_eff'][trace] - else: - y_comm_serial[index] = 0.0 - if hybrid_factors['transfer_eff'][trace] != 'Non-Avail' \ - and hybrid_factors['transfer_eff'][trace] != 'Warning!': - y_comm_transfer[index] = hybrid_factors['transfer_eff'][trace] - else: - y_comm_transfer[index] = 0.0 - else: - y_comm_serial[index] = 0.0 - y_comm_transfer[index] = 0.0 - y_omp_par[index] = hybrid_factors['omp_parallel_eff'][trace] - y_omp_load[index] = hybrid_factors['omp_load_balance'][trace] - y_omp_comm[index] = hybrid_factors['omp_comm_eff'][trace] - - # Set limit for projection - if cmdl_args.limit: - limit = cmdl_args.limit - else: - limit = str(trace_processes[trace_list[len(trace_list)-1]]) - - limit_min = str(int(x_proc[0])) - # limit_min = str(0) - # To extract the trace name to show in the plots - title_string = "" - for index, trace in enumerate(trace_list): - folder_trace_name = trace.split('/') - trace_name_to_show = folder_trace_name[len(folder_trace_name) - 1] - title_string += '(' + str(index+1) + ') ' + trace_name_to_show + "\\" + 'n' - - title_string += '"' + " noenhanced" - - # To control same number of processes for the header on plots and table - same_procs = True - procs_trace_prev = trace_processes[trace_list[0]] - tasks_trace_prev = trace_tasks[trace_list[0]] - threads_trace_prev = trace_threads[trace_list[0]] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ - and threads_trace_prev == threads: - same_procs *= True - else: - same_procs *= False - - # Create Gnuplot file for main plot - gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-onlydata.gp') - content = [] - with open(gp_template) as f: - content = f.readlines() - - # Replace xrange - if int(limit) == int(limit_min): - limit_plot = int(limit_min) + 5 * (len(trace_list) - 1) - else: - limit_plot = int(limit) - - # To xticks label - label_xtics = 'set xtics (' - glabel_xtics = [] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_threads[trace] - s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' - if int(limit) == int(limit_min) and same_procs: - proc_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' \ - + '[' + str(index + 1) + ']' - real_procs = str(trace_processes[trace] + index * 5) - elif int(limit) == int(limit_min) and not same_procs: - proc_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' - real_procs = str(trace_processes[trace] + index * 5) - if s_xtics in glabel_xtics: - proc_xtics += '[' + str(index + 1) + ']' - glabel_xtics.append(s_xtics) - else: - proc_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' - real_procs = str(trace_processes[trace]) - if s_xtics in glabel_xtics: - proc_xtics += '[' + str(index + 1) + ']' - glabel_xtics.append(s_xtics) - - label_xtics += '"' + proc_xtics + '" ' + real_procs + ', ' - - content = [line.replace('#REPLACE_BY_XRANGE', '' - .join(['set xrange [', str(limit_min), ':', str(limit_plot), ']'])) for line in content] - content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2] + ') '])) for - line in content] - content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in - content] - - max_global = max([max(y_para), max(y_load), max(y_comm), max(y_comp), max(y_glob)]) - content = [line.replace('#REPLACE_BY_YRANGE', '' - .join(['set yrange [0:', str(max_global+5), ']'])) for line in content] - - file_path = os.path.join(os.getcwd(), 'modelfactors.gp') - with open(file_path, 'w') as f: - f.writelines(content) - - # Add data points to gnuplot file - with open(file_path, 'a') as f: - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index] + index * 5), str(y_para[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_para[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index] + index * 5), str(y_load[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_load[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index] + index * 5), str(y_comm[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_comm[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index] + index * 5), str(y_comp[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_comp[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index] + index * 5), str(y_glob[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_glob[index]), '\n']) - f.write(line) - f.write('e\n') - - f.write('\n') - f.write('pause -1\n') - - # print('========= Plot (gnuplot File): EFFICIENCY METRICS ==========') - print('Efficiency metrics plot written to ' + file_path) - # print('') - - # Create Gnuplot file for scalability plot - gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-scale.gp') - content = [] - with open(gp_template) as f: - content = f.readlines() - - # Replace xrange - content = [line.replace('#REPLACE_BY_XRANGE', ''.join(['set xrange [',limit_min,':',str(limit_plot),']']) ) for line in content] - content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2]+') '])) for - line in content] - content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in - content] - - max_comp = max([max(y_comp), max(y_ipc_scale), max(y_inst_scale), max(y_freq_scale)]) - content = [line.replace('#REPLACE_BY_YRANGE', '' - .join(['set yrange [0:', str(max_comp+5), ']'])) for line in content] - - file_path = os.path.join(os.getcwd(), 'modelfactors-scale.gp') - with open(file_path, 'w') as f: - f.writelines(content) - - # Add data points to gnuplot file - with open(file_path, 'a') as f: - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_comp[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_comp[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_ipc_scale[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_ipc_scale[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_inst_scale[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_inst_scale[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_freq_scale[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_freq_scale[index]), '\n']) - f.write(line) - f.write('e\n') - - f.write('\n') - f.write('pause -1\n') - - # print('======== Plot (gnuplot File): SCALABILITY METRICS ========') - print('Scalability metrics plot written to ' + file_path) - - # Create Gnuplot file for hybrid metrics plot - gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-hybrid.gp') - content = [] - with open(gp_template) as f: - content = f.readlines() - - # Replace xrange - content = [line.replace('#REPLACE_BY_XRANGE', ''.join(['set xrange [',limit_min,':',str(limit_plot),']']) ) for line in content] - content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2]+') '])) for - line in content] - content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in - content] - - string_omp =" '-' with linespoints title " + '"' + (mod_hybrid_factors_doc['omp_parallel_eff'])[5:] + '"' + " ls 5,\\" - content = [line.replace('#REPLACE_BY_OMP_PAR_EFF', ''.join([string_omp])) for line in content] - - string_omp = " '-' with linespoints title " + '"' + (mod_hybrid_factors_doc['omp_load_balance'])[9:] + '"' + " ls 6,\\" - content = [line.replace('#REPLACE_BY_OMP_LB', ''.join([string_omp])) for line in content] - - string_omp = " '-' with linespoints title " + '"' + (mod_hybrid_factors_doc['omp_comm_eff'])[9:] + '"' + " ls 7" - content = [line.replace('#REPLACE_BY_OMP_COMM', ''.join([string_omp])) for line in content] - - max_hybrid = max([max(y_hybrid_par), max(y_mpi_par), max(y_mpi_comm), max(y_mpi_load), - max(y_omp_par), max(y_omp_comm), max(y_omp_load)]) - content = [line.replace('#REPLACE_BY_YRANGE', '' - .join(['set yrange [0:', str(max_hybrid+5), ']'])) for line in content] - - file_path = os.path.join(os.getcwd(), 'modelfactors-hybrid.gp') - with open(file_path, 'w') as f: - f.writelines(content) - - # Add data points to gnuplot file - with open(file_path, 'a') as f: - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_hybrid_par[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_hybrid_par[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_par[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_mpi_par[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_load[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_mpi_load[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_comm[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_mpi_comm[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_omp_par[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_omp_par[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_omp_load[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_omp_load[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_omp_comm[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_omp_comm[index]), '\n']) - f.write(line) - f.write('e\n') - - f.write('\n') - f.write('pause -1\n') - - print('Hybrid metrics plot written to ' + file_path) - - # Create Gnuplot file for MPI hybrid metrics plot - gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-mpi-hybrid.gp') - content = [] - with open(gp_template) as f: - content = f.readlines() - - # Replace xrange - content = [line.replace('#REPLACE_BY_XRANGE', ''.join(['set xrange [',limit_min,':',str(limit_plot),']']) ) for line in content] - content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2]+') '])) for - line in content] - content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in - content] - - max_mpi = max([max(y_mpi_par), max(y_mpi_comm), max(y_mpi_load), - max(y_comm_serial), max(y_comm_transfer)]) - content = [line.replace('#REPLACE_BY_YRANGE', '' - .join(['set yrange [0:', str(max_mpi+5), ']'])) for line in content] - - file_path = os.path.join(os.getcwd(), 'modelfactors-mpi-hybrid.gp') - with open(file_path, 'w') as f: - f.writelines(content) - - # Add data points to gnuplot file - with open(file_path, 'a') as f: - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_par[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_mpi_par[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_load[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_mpi_load[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_comm[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_mpi_comm[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if str(y_comm_serial[index]) != 0.0: - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_comm_serial[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_comm_serial[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if str(y_comm_transfer[index]) != 0.0: - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_comm_transfer[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_comm_transfer[index]), '\n']) - f.write(line) - f.write('e\n') - - - f.write('\n') - f.write('pause -1\n') - - print('MPI hybrid metrics plot written to ' + file_path) - - -def plot_simple_metrics(mod_factors, trace_list, trace_processes, trace_mode, cmdl_args): - """Computes the projection from the gathered model factors and returns the - according dictionary of fitted prediction functions.""" - - if cmdl_args.debug: - print('==DEBUG== Computing projection of model factors.') - - number_traces = len(trace_list) - x_proc = numpy.zeros(number_traces) - y_para = numpy.zeros(number_traces) - y_load = numpy.zeros(number_traces) - y_comm = numpy.zeros(number_traces) - y_comp = numpy.zeros(number_traces) - y_glob = numpy.zeros(number_traces) - y_comm_serial = numpy.zeros(number_traces) - y_comm_transfer = numpy.zeros(number_traces) - y_ipc_scale = numpy.zeros(number_traces) - y_inst_scale = numpy.zeros(number_traces) - y_freq_scale = numpy.zeros(number_traces) - - #Convert dictionaries to NumPy arrays - for index, trace in enumerate(trace_list): - x_proc[index] = trace_processes[trace] - y_para[index] = mod_factors['parallel_eff'][trace] - y_load[index] = mod_factors['load_balance'][trace] - y_comm[index] = mod_factors['comm_eff'][trace] - y_comp[index] = mod_factors['comp_scale'][trace] - y_glob[index] = mod_factors['global_eff'][trace] - if trace_mode[trace][:5] != 'Burst': - y_ipc_scale[index] = mod_factors['ipc_scale'][trace] - y_inst_scale[index] = mod_factors['inst_scale'][trace] - y_freq_scale[index] = mod_factors['freq_scale'][trace] - else: - y_ipc_scale[index] = 0.0 - y_inst_scale[index] = 0.0 - y_freq_scale[index] = 0.0 - if trace_mode[trace] == 'Detailed+MPI': - if mod_factors['serial_eff'][trace] != 'Warning!': - y_comm_serial[index] = mod_factors['serial_eff'][trace] - else: - y_comm_serial[index] = 0.0 - if mod_factors['transfer_eff'][trace] != 'Warning!': - y_comm_transfer[index] = mod_factors['transfer_eff'][trace] - else: - y_comm_transfer[index] = 0.0 - else: - y_comm_serial[index] = 0.0 - y_comm_transfer[index] = 0.0 - - #Set limit for projection - if cmdl_args.limit: - limit = cmdl_args.limit - else: - limit = str(trace_processes[trace_list[len(trace_list) - 1]]) - - limit_min = str(int(x_proc[0])) - # limit_min = str(0) - # To extract the trace name to show in the plots - title_string = "" - for index, trace in enumerate(trace_list): - folder_trace_name = trace.split('/') - trace_name_to_show = folder_trace_name[len(folder_trace_name) - 1] - title_string += '(' + str(index+1) + ') ' + trace_name_to_show + "\\" + 'n' - title_string += '"' + " noenhanced" - - #Create Gnuplot file for main plot - gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-onlydata.gp') - content = [] - with open(gp_template) as f: - content = f.readlines() - - #Replace xrange - if int(limit) == int(limit_min): - limit_plot = int(limit_min) + 5 * (len(trace_list) - 1 ) - else: - limit_plot = int(limit) - - # To xticks label - label_xtics = 'set xtics (' - for index, trace in enumerate(trace_list): - - if int(limit) == int(limit_min): - label_xtics += '"' + str(trace_processes[trace]) + '[' + str(index + 1) + ']' + '" ' \ - + str(trace_processes[trace] + index * 5) + ', ' - else: - label_xtics += '"' + str(trace_processes[trace]) + '" ' + str(trace_processes[trace]) + ', ' - - content = [line.replace('#REPLACE_BY_XRANGE', '' - .join(['set xrange [',limit_min,':', str(limit_plot),']'])) for line in content] - content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2] + ') '])) for - line in content] - - content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in - content] - - max_global = max([max(y_para), max(y_load), max(y_comm), max(y_comp), max(y_glob)]) - content = [line.replace('#REPLACE_BY_YRANGE', '' - .join(['set yrange [0:', str(max_global+5), ']'])) for line in content] - - file_path = os.path.join(os.getcwd(), 'modelfactors.gp') - with open(file_path, 'w') as f: - f.writelines(content) - - #Add data points to gnuplot file - with open(file_path, 'a') as f: - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_para[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_para[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_load[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_load[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_comm[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_comm[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_comp[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_comp[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_glob[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_glob[index]), '\n']) - f.write(line) - f.write('e\n') - - f.write('\n') - f.write('pause -1\n') - - # print('========= Plot (gnuplot File): EFFICIENCY METRICS ==========') - print('Efficiency metrics plot written to ' + file_path) - # print('') - - # Create Gnuplot file for communication plot - gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-comm.gp') - content = [] - with open(gp_template) as f: - content = f.readlines() - - # Replace xrange - content = [line.replace('#REPLACE_BY_XRANGE', ''.join(['set xrange [',limit_min,':',str(limit_plot),']']) ) for line in content] - content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2]+') '])) for - line in content] - - content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in - content] - - max_comm = max([max(y_comm), max(y_comm_serial), max(y_comm_transfer)]) - content = [line.replace('#REPLACE_BY_YRANGE', '' - .join(['set yrange [0:', str(max_comm+5), ']'])) for line in content] - - file_path = os.path.join(os.getcwd(), 'modelfactors-comm.gp') - with open(file_path, 'w') as f: - f.writelines(content) - - # Add data points to gnuplot file - with open(file_path, 'a') as f: - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_comm[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_comm[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if str(y_comm_serial[index]) != 0.0: - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_comm_serial[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_comm_serial[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if str(y_comm_transfer[index]) != 0.0: - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_comm_transfer[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_comm_transfer[index]), '\n']) - f.write(line) - f.write('e\n') - - f.write('\n') - f.write('pause -1\n') - - # print('======= Plot (gnuplot File): COMMUNICATION METRICS ========') - print('Communication Efficiency plot written to ' + file_path) - # print('') - - # Create Gnuplot file for scalability plot - gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-scale.gp') - content = [] - with open(gp_template) as f: - content = f.readlines() - - # Replace xrange - content = [line.replace('#REPLACE_BY_XRANGE', ''.join(['set xrange [',limit_min,':',str(limit_plot),']']) ) for line in content] - content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2]+') '])) for - line in content] - content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in - content] - max_comp = max([max(y_comp), max(y_ipc_scale), max(y_inst_scale), max(y_freq_scale)]) - content = [line.replace('#REPLACE_BY_YRANGE', '' - .join(['set yrange [0:', str(max_comp+5), ']'])) for line in content] - - file_path = os.path.join(os.getcwd(), 'modelfactors-scale.gp') - with open(file_path, 'w') as f: - f.writelines(content) - - # Add data points to gnuplot file - with open(file_path, 'a') as f: - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_comp[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_comp[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_ipc_scale[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_ipc_scale[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_inst_scale[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_inst_scale[index]), '\n']) - f.write(line) - f.write('e\n') - - for index in range(0, number_traces): - if int(limit) == int(limit_min): - line = ' '.join([str(x_proc[index]+index*5), str(y_freq_scale[index]), '\n']) - else: - line = ' '.join([str(x_proc[index]), str(y_freq_scale[index]), '\n']) - f.write(line) - f.write('e\n') - - f.write('\n') - f.write('pause -1\n') - - # print('======== Plot (gnuplot File): SCALABILITY METRICS ========') - print('Scalability metrics plot written to ' + file_path) \ No newline at end of file diff --git a/rawdata.py b/rawdata.py deleted file mode 100644 index edbdf4d..0000000 --- a/rawdata.py +++ /dev/null @@ -1,857 +0,0 @@ -#!/usr/bin/env python3 - -"""Functions to extract rawdata from each trace.""" - -from __future__ import print_function, division -import os -import time -import math -import gzip -import shutil -from collections import OrderedDict -from tracemetadata import human_readable -from utils import run_command, move_files,remove_files, create_temp_folder - - -# Contains all raw data entries with a printable name. -# This is used to generate and print all raw data, so, if an entry is added, it -# should be added here, too. -raw_data_doc = OrderedDict([('runtime', 'Runtime (us)'), - ('runtime_dim', 'Runtime (ideal)'), - ('useful_avg', 'Useful duration (average)'), - ('useful_max', 'Useful duration (maximum)'), - ('useful_tot', 'Useful duration (total)'), - ('useful_dim', 'Useful duration (ideal, max)'), - ('useful_ins', 'Useful instructions (total)'), - ('useful_cyc', 'Useful cycles (total)'), - ('outsidempi_avg', 'Outside MPI duration (average)'), - ('outsidempi_max', 'Outside MPI duration (maximum)'), - ('outsidempi_dim', 'Outside MPI duration (ideal,maximum)'), - ('outsidempi_tot', 'Outside MPI duration (total)'), - ('mpicomm_tot', 'Communication MPI duration (total)'), - ('outsidempi_tot_diff', 'Outside MPI duration rescaled (total*threads)'), - ('flushing_avg', 'Flushing duration (average)'), - ('flushing_max', 'Flushing duration (maximum)'), - ('flushing_tot', 'Flushing duration (total)'), - ('flushing_cyc', 'Flushing cycles (total)'), - ('flushing_ins', 'Flushing instructions (total)'), - ('io_tot', 'Posix I/O duration (total)'), - ('io_max', 'Posix I/O duration (maximum)'), - ('io_avg', 'Posix I/O duration (avg)'), - ('io_std', 'Posix I/O duration (std)'), - ('io_cyc', 'Posix I/O cycles (total)'), - ('io_ins', 'Posix I/O instructions (total)'), - ('useful_plus_io_avg', 'Serial I/O plus useful duration (avg)'), - ('useful_plus_io_max', 'Serial I/O plus useful duration (max)'), - ('io_state_tot', 'state I/O duration (total)'), - ('io_state_avg', 'state I/O duration (avg)'), - ('io_state_max', 'state I/O duration (maximum)'), - ('mpiio_tot', 'MPI I/O duration (total)'), - ('mpiio_max', 'MPI I/O duration (maximum)'), - ('mpiio_avg', 'MPI I/O duration (avg)'), - ('mpiio_std', 'MPI I/O duration (std)'), - ('mpiio_cyc', 'MPI I/O cycles (total)'), - ('mpiio_ins', 'MPI I/O instructions (total)'), - ('burst_useful_tot', 'Burst Useful (total)'), - ('burst_useful_max', 'Burst Useful (max)'), - ('burst_useful_avg', 'Burst Useful (avg)')]) - - -def create_raw_data(trace_list): - """Creates 2D dictionary of the raw input data and initializes with zero. - The raw_data dictionary has the format: [raw data key][trace]. - """ - global raw_data_doc - raw_data = {} - for key in raw_data_doc: - trace_dict = {} - for trace_name in trace_list: - trace_dict[trace_name] = 0.0 - - raw_data[key] = trace_dict - - return raw_data - - -def gather_raw_data(trace_list, trace_processes, trace_task_per_node, trace_mode, cmdl_args): - """Gathers all raw data needed to generate the model factors. Return raw - data in a 2D dictionary """ - raw_data = create_raw_data(trace_list) - global list_mpi_procs_count - list_mpi_procs_count = dict() - - cfgs = {} - cfgs['root_dir'] = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs') - cfgs['timings'] = os.path.join(cfgs['root_dir'], 'timings.cfg') - cfgs['runtime'] = os.path.join(cfgs['root_dir'], 'runtime_app.cfg') - cfgs['cycles'] = os.path.join(cfgs['root_dir'], 'cycles.cfg') - cfgs['instructions'] = os.path.join(cfgs['root_dir'], 'instructions.cfg') - cfgs['flushing'] = os.path.join(cfgs['root_dir'], 'flushing.cfg') - cfgs['mpi_io'] = os.path.join(cfgs['root_dir'], 'mpi-io-reverse.cfg') - cfgs['outside_mpi'] = os.path.join(cfgs['root_dir'], 'mpi-call-outside.cfg') - cfgs['io_call'] = os.path.join(cfgs['root_dir'], 'io-call-reverse.cfg') - cfgs['io_cycles'] = os.path.join(cfgs['root_dir'], 'io-call-cycles.cfg') - cfgs['io_inst'] = os.path.join(cfgs['root_dir'], 'io-call-instructions.cfg') - cfgs['mpiio_cycles'] = os.path.join(cfgs['root_dir'], 'mpi-io-cycles.cfg') - cfgs['mpiio_inst'] = os.path.join(cfgs['root_dir'], 'mpi-io-instructions.cfg') - cfgs['flushing_cycles'] = os.path.join(cfgs['root_dir'], 'flushing-cycles.cfg') - cfgs['flushing_inst'] = os.path.join(cfgs['root_dir'], 'flushing-inst.cfg') - cfgs['burst_useful'] = os.path.join(cfgs['root_dir'], 'burst_useful.cfg') - - # Main loop over all traces - # This can be parallelized: the loop iterations have no dependencies - path_dest = create_temp_folder('scratch_out_basicanalysis', cmdl_args) - - for trace in trace_list: - time_tot = time.time() - if trace[-7:] == ".prv.gz": - trace_name_control = trace[:-7] - trace_name = trace[:-7] + '_' + str(trace_processes[trace]) + 'P' - elif trace[-4:] == ".prv": - trace_name_control = trace[:-4] - trace_name = trace[:-4] + '_' + str(trace_processes[trace]) + 'P' - - line = 'Analyzing ' + os.path.basename(trace) - line += ' (' + str(trace_processes[trace]) + ' processes' - line += ', ' + str(trace_task_per_node[trace]) + ' tasks per node' - line += ', ' + str(trace_mode[trace]) + ' mode' - line += ', ' + human_readable(os.path.getsize(trace)) + ')' - print(line) - - # Create simulated ideal trace with Dimemas - if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': - time_dim = time.time() - trace_sim = create_ideal_trace(trace, trace_processes[trace], trace_task_per_node[trace], cmdl_args) - trace_name_sim = trace_sim[:-4] - # print(trace_sim) - time_dim = time.time() - time_dim - if not trace_sim == '': - print('Successfully created simulated trace with Dimemas in {0:.1f} seconds.'.format(time_dim)) - else: - print('Failed to create simulated trace with Dimemas.') - - # Run paramedir for the original and simulated trace - time_pmd = time.time() - cmd_normal = ['paramedir', trace] - - cmd_normal.extend([cfgs['timings'], trace_name + '.timings.stats']) - cmd_normal.extend([cfgs['runtime'], trace_name + '.runtime.stats']) - cmd_normal.extend([cfgs['cycles'], trace_name + '.cycles.stats']) - cmd_normal.extend([cfgs['instructions'], trace_name + '.instructions.stats']) - cmd_normal.extend([cfgs['flushing'], trace_name + '.flushing.stats']) - cmd_normal.extend([cfgs['io_call'], trace_name + '.posixio_call.stats']) - cmd_normal.extend([cfgs['io_cycles'], trace_name + '.posixio-cycles.stats']) - cmd_normal.extend([cfgs['io_inst'], trace_name + '.posixio-inst.stats']) - cmd_normal.extend([cfgs['flushing_cycles'], trace_name + '.flushing-cycles.stats']) - cmd_normal.extend([cfgs['flushing_inst'], trace_name + '.flushing-inst.stats']) - - if trace_mode[trace][:12] == 'Detailed+MPI': - cmd_normal.extend([cfgs['mpi_io'], trace_name + '.mpi_io.stats']) - cmd_normal.extend([cfgs['outside_mpi'], trace_name + '.outside_mpi.stats']) - cmd_normal.extend([cfgs['mpiio_cycles'], trace_name + '.mpiio-cycles.stats']) - cmd_normal.extend([cfgs['mpiio_inst'], trace_name + '.mpiio-inst.stats']) - - if trace_mode[trace][:9] == 'Burst+MPI': - cmd_normal.extend([cfgs['burst_useful'], trace_name + '.burst_useful.stats']) - - run_command(cmd_normal, cmdl_args) - - if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': - cmd_ideal = ['paramedir', trace_sim] - cmd_ideal.extend([cfgs['timings'], trace_name_sim + '.timings.stats']) - cmd_ideal.extend([cfgs['runtime'], trace_name_sim + '.runtime.stats']) - cmd_ideal.extend([cfgs['outside_mpi'], trace_name_sim + '.outside_mpi.stats']) - - if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': - if not trace_sim == '': - # print(cmd_ideal) - run_command(cmd_ideal, cmdl_args) - - time_pmd = time.time() - time_pmd - - error_timing = 0 - error_counters = 0 - error_ideal = 0 - - # Check if all files are created - if not os.path.exists(trace_name + '.timings.stats') or \ - not os.path.exists(trace_name + '.runtime.stats'): - print('==ERROR== Failed to compute timing information with paramedir.') - error_timing = 1 - - if not os.path.exists(trace_name + '.outside_mpi.stats') and trace_mode[trace][:5] != 'Burst' \ - and 'MPI' in trace_mode[trace]: - print('==ERROR== Failed to compute outside MPI timing information with paramedir.') - error_timing = 1 - - if not os.path.exists(trace_name + '.cycles.stats') or \ - not os.path.exists(trace_name + '.instructions.stats'): - print('==ERROR== Failed to compute counter information with paramedir.') - error_counters = 1 - - if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': - if not os.path.exists(trace_name_sim + '.timings.stats') or \ - not os.path.exists(trace_name_sim + '.runtime.stats') or \ - not os.path.exists(trace_name_sim + '.outside_mpi.stats'): - print('==ERROR== Failed to compute simulated timing information with paramedir.') - error_ideal = 1 - trace_sim = '' - - if error_timing or error_counters or error_ideal: - print('Failed to analyze trace with paramedir') - else: - print('Successfully analyzed trace with paramedir in {0:.1f} seconds.'.format(time_pmd)) - - # Parse the paramedir output files - time_prs = time.time() - - # Get total, average, and maximum useful duration - if os.path.exists(trace_name + '.timings.stats'): - content = [] - with open(trace_name + '.timings.stats') as f: - content = f.readlines() - - for line in content: - if line.split(): - if line.split()[0] == 'Total': - raw_data['useful_tot'][trace] = float(line.split()[1]) - if line.split()[0] == 'Average': - raw_data['useful_avg'][trace] = float(line.split()[1]) - if line.split()[0] == 'Maximum': - raw_data['useful_max'][trace] = float(line.split()[1]) - else: - raw_data['useful_tot'][trace] = 'NaN' - raw_data['useful_avg'][trace] = 'NaN' - raw_data['useful_max'][trace] = 'NaN' - f.close() - - # Get total File IO, average IO, and maximum IO duration - dict_trace_posixio = {} - if os.path.exists(trace_name + '.posixio_call.stats'): - content = [] - with open(trace_name + '.posixio_call.stats') as f: - content = f.readlines() - - for line in content: - - for field in line.split("\n"): - line_list = field.split("\t") - if "Total" in field.split("\t"): - count_procs = len(line_list[1:]) - list_io_tot = [float(iotime) for iotime in line_list[1:count_procs]] - dict_trace_posixio[trace] = list_io_tot - raw_data['io_tot'][trace] = sum(list_io_tot) - elif "Average" in field.split("\t"): - if count_procs != 0: - raw_data['io_avg'][trace] = float(sum(list_io_tot)/count_procs) - raw_data['io_std'][trace] = math.sqrt(sum([(number - raw_data['io_avg'][trace]) ** 2 - for number in list_io_tot]) - / (len(list_io_tot) - 1)) - else: - raw_data['io_avg'][trace] = 0.0 - raw_data['io_std'][trace] = 0.0 - elif "Maximum" in field.split("\t"): - raw_data['io_max'][trace] = max(list_io_tot) - else: - raw_data['io_tot'][trace] = 0.0 - raw_data['io_avg'][trace] = 0.0 - raw_data['io_max'][trace] = 0.0 - raw_data['io_std'][trace] = 0.0 - f.close() - - # Get total MPI IO, average IO, and maximum IO duration for MPI-IO - dict_trace_mpiio={} - if os.path.exists(trace_name + '.mpi_io.stats') and trace_mode[trace][:12] == 'Detailed+MPI': - content = [] - with open(trace_name + '.mpi_io.stats') as f: - content = f.readlines() - - for line in content: - for field in line.split("\n"): - line_list = field.split("\t") - if "Total" in field.split("\t"): - count_procs = len(line_list[1:]) - list_mpiio_tot = [float(iotime) for iotime in line_list[1:count_procs]] - dict_trace_mpiio[trace] = list_mpiio_tot - # print(list_mpiio_tot) - raw_data['mpiio_tot'][trace] = sum(list_mpiio_tot) - elif "Average" in field.split("\t"): - if count_procs != 0: - raw_data['mpiio_avg'][trace] = sum(list_mpiio_tot)/count_procs - raw_data['mpiio_std'][trace] = math.sqrt(sum([(number - raw_data['mpiio_avg'][trace]) ** 2 - for number in list_mpiio_tot]) - /(len(list_mpiio_tot) - 1)) - else: - raw_data['mpiio_avg'][trace] = 0.0 - raw_data['mpiio_std'][trace] = 0.0 - elif "Maximum" in field.split("\t"): - raw_data['mpiio_max'][trace] = max(list_mpiio_tot) - else: - raw_data['mpiio_tot'][trace] = 0.0 - raw_data['mpiio_avg'][trace] = 0.0 - raw_data['mpiio_max'][trace] = 0.0 - raw_data['mpiio_std'][trace] = 0.0 - f.close() - - # Get total State IO, average IO, and maximum IO duration - #print(dict_trace_mpiio) - if os.path.exists(trace_name + '.timings.stats'): - content = [] - # io_index = " " - with open(trace_name + '.timings.stats') as f: - content = f.readlines() - useful_plus_io = [] - useful_comp = [] - io_time = [] - count_line = 1 - for line in content: - for field in line.split("\n"): - line_list = field.split("\t") - # print(line_list) - if count_line == 1: - try: - if os.path.exists(trace_name_control + '.pcf'): - io_index = line_list.index("I/O") - else: - io_index = line_list.index('Unknown state 12') - except: - io_index = " " - elif io_index != " ": - if "Total" in field.split("\t"): - raw_data['io_state_tot'][trace] = float(line_list[io_index]) - elif "Average" in field.split("\t"): - raw_data['io_state_avg'][trace] = float(line_list[io_index]) - elif "Maximum" in field.split("\t"): - raw_data['io_state_max'][trace] = float(line_list[io_index]) - elif "Minimum" not in line_list and "StDev" not in line_list \ - and "Avg/Max" not in line_list and len(line_list) > 1: - posixio_index = len(useful_plus_io) - # print(dict_trace_posixio[trace][posixio_index]) - if len(dict_trace_posixio) != 0: - sum_aux = float(line_list[1]) + float(dict_trace_posixio[trace][posixio_index]) - else: - sum_aux = float(line_list[1]) - - useful_plus_io.append(float(sum_aux)) - useful_comp.append(float(line_list[1])) - io_time.append(float(line_list[io_index])) - count_line += 1 - if io_index != " ": - if len(useful_plus_io) != 0: - useful_io_avg = float(sum(useful_plus_io) / len(useful_plus_io)) - else: - useful_io_avg = 0.0 - # print(len(useful_plus_io)) - # mpiio is not included in useful + IO - raw_data['useful_plus_io_avg'][trace] = float(useful_io_avg) - raw_data['useful_plus_io_max'][trace] = float(max(useful_plus_io)) - else: - raw_data['useful_plus_io_avg'][trace] = 0.0 - raw_data['useful_plus_io_max'][trace] = 0.0 - raw_data['io_state_tot'][trace] = 0.0 - raw_data['io_state_avg'][trace] = 0.0 - raw_data['io_state_max'][trace] = 0.0 - else: - raw_data['useful_plus_io_avg'][trace] = 'NaN' - raw_data['useful_plus_io_max'][trace] = 'NaN' - raw_data['io_state_tot'][trace] = 'NaN' - raw_data['io_state_avg'][trace] = 'NaN' - raw_data['io_state_max'][trace] = 'NaN' - f.close() - - - # Get runtime - if os.path.exists(trace_name + '.runtime.stats'): - content = [] - with open(trace_name + '.runtime.stats') as f: - content = f.readlines() - - for line in content: - if line.split(): - if line.split()[0] == 'Average': - raw_data['runtime'][trace] = float(line.split()[1]) - else: - raw_data['runtime'][trace] = 'NaN' - - # Get total, average, and maximum outside MPI - # list_mpi_procs_count = [] - if os.path.exists(trace_name + '.outside_mpi.stats') and trace_mode[trace][:12] == 'Detailed+MPI': - content = [] - with open(trace_name + '.outside_mpi.stats') as f: - content = f.readlines() - list_outside_mpi = [] - - list_thread_outside_mpi = [] - init_count_thread = False - count_threads = 1 - for line1 in content[1:(len(content) - 8)]: - line = line1.split("\t") - # print(line) - if line: - if line[0] != 'Total' and line[0] != 'Average' \ - and line[0] != 'Maximum' and line[0] != 'StDev' \ - and line[0] != 'Avg/Max': - # To extract the count of MPI tasks - if float(line[1]) != raw_data['runtime'][trace]: - list_outside_mpi.append(float(line[1])) - # To extract the count of threads per MPI task - if len(list_outside_mpi) > 1: - list_thread_outside_mpi.append(count_threads) - count_threads = 1 - else: - if len(list_outside_mpi) == 1 and not init_count_thread: - count_threads = 2 - init_count_thread = True - else: - count_threads += 1 - - list_thread_outside_mpi.append(count_threads) - - count = 0 - equal_threads = True - # Evaluate if the count of threads per MPI task are equal - while count < (len(list_thread_outside_mpi) - 1) and equal_threads: - if list_thread_outside_mpi[count] != list_thread_outside_mpi[count+1]: - equal_threads = False - count += 1 - rescaled_outside_mpi = [] - # This is need to calculate the MPI_Par_Eff with the right #MPI_tasks and #threads - if not equal_threads: - # print("\n===== Different number of threads per mpi task \n") - rescaled_outside_mpi.append(list_outside_mpi[0] * (list_thread_outside_mpi[0])) - # This is the sum of the outsidempi by the threads - sum_outside_mpi_threads = list_thread_outside_mpi[0] - - for i in range(1,len(list_outside_mpi)): - rescaled_outside_mpi.append(list_outside_mpi[i] * (list_thread_outside_mpi[i])) - sum_outside_mpi_threads += list_thread_outside_mpi[i] - raw_data['outsidempi_tot_diff'][trace] = sum(rescaled_outside_mpi) - raw_data['outsidempi_tot'][trace] = sum(list_outside_mpi) - # Only the average is updated with the rescaled outsidempi - if sum(list_thread_outside_mpi) != 0: - raw_data['outsidempi_avg'][trace] = sum(rescaled_outside_mpi) / sum(list_thread_outside_mpi) - else: - raw_data['outsidempi_avg'][trace] = 'NaN' - # Maximum outsidempi is the same, although the count of threads is different - # because it waits that the MPI task has the max outsidempi. - raw_data['outsidempi_max'][trace] = max(list_outside_mpi) - else: - # print("\n===== Equal number of threads per mpi task \n") - raw_data['outsidempi_tot_diff'][trace] = sum(list_outside_mpi) - list_mpi_procs_count[trace] = len(list_outside_mpi) - raw_data['outsidempi_tot'][trace] = sum(list_outside_mpi) - if len(list_outside_mpi) != 0: - raw_data['outsidempi_avg'][trace] = sum(list_outside_mpi) / len(list_outside_mpi) - else: - raw_data['outsidempi_avg'][trace] = 'NaN' - raw_data['outsidempi_max'][trace] = max(list_outside_mpi) - for line2 in content[(len(content) - 7):]: - line_aux = line2.split("\t") - - if line_aux[0] == 'Total': - #print(line_aux) - count_mpiop = len(line_aux[1:]) - list_mpi_tot = [float(mpitime) for mpitime in line_aux[2:count_mpiop]] - #print(list_mpi_tot) - raw_data['mpicomm_tot'][trace] = sum(list_mpi_tot) - else: - raw_data['outsidempi_tot'][trace] = 'NaN' - raw_data['outsidempi_avg'][trace] = 'NaN' - raw_data['outsidempi_max'][trace] = 'NaN' - raw_data['mpicomm_tot'][trace] = 'NaN' - f.close() - # Get total, average, and maximum flushing duration - if os.path.exists(trace_name + '.flushing.stats'): - content = [] - with open(trace_name + '.flushing.stats') as f: - content = f.readlines() - flushing_exist = ('\tBegin\t\n' in content) or ('\tvalue 1\t\n' in content) - - if flushing_exist: - for line in content: - if line.split(): - if line.split()[0] == 'Total': - raw_data['flushing_tot'][trace] = float(line.split()[1]) - if line.split()[0] == 'Average': - raw_data['flushing_avg'][trace] = float(line.split()[1]) - if line.split()[0] == 'Maximum': - raw_data['flushing_max'][trace] = float(line.split()[1]) - else: - raw_data['flushing_tot'][trace] = 0.0 - raw_data['flushing_avg'][trace] = 0.0 - raw_data['flushing_max'][trace] = 0.0 - else: - raw_data['flushing_tot'][trace] = 0.0 - raw_data['flushing_avg'][trace] = 0.0 - raw_data['flushing_max'][trace] = 0.0 - - # Get total flushing cycles - if os.path.exists(trace_name + '.flushing-cycles.stats'): - content = [] - with open(trace_name + '.flushing-cycles.stats') as f: - content = f.readlines() - - for line in content: - if line.split(): - if line.split()[0] == 'Total': - raw_data['flushing_cyc'][trace] = int(float(line.split()[1])) - else: - raw_data['flushing_cyc'][trace] = 0.0 - - # Get total flushing instructions - if os.path.exists(trace_name + '.flushing-inst.stats'): - content = [] - with open(trace_name + '.flushing-inst.stats') as f: - content = f.readlines() - - for line in content: - if line.split(): - if line.split()[0] == 'Total': - raw_data['flushing_ins'][trace] = int(float(line.split()[1])) - else: - raw_data['flushing_ins'][trace] = 0.0 - - # Get total posixio cycles - if os.path.exists(trace_name + '.posixio-cycles.stats'): - content = [] - with open(trace_name + '.posixio-cycles.stats') as f: - content = f.readlines() - - for line in content: - if line.split(): - if line.split()[0] == 'Total': - raw_data['io_cyc'][trace] = int(float(line.split()[1])) - else: - raw_data['io_cyc'][trace] = 0.0 - - # Get total posixio instructions - if os.path.exists(trace_name + '.posixio-inst.stats'): - content = [] - with open(trace_name + '.posixio-inst.stats') as f: - content = f.readlines() - - for line in content: - if line.split(): - if line.split()[0] == 'Total': - raw_data['io_ins'][trace] = int(float(line.split()[1])) - else: - raw_data['io_ins'][trace] = 0.0 - - # Get total mpiio instructions - if os.path.exists(trace_name + '.mpiio-cycles.stats') \ - and trace_mode[trace][:12] == 'Detailed+MPI': - content = [] - with open(trace_name + '.mpiio-cycles.stats') as f: - content = f.readlines() - - for line in content: - if line.split(): - if line.split()[0] == 'Total': - raw_data['mpiio_cyc'][trace] = int(float(line.split()[1])) - else: - raw_data['mpiio_cyc'][trace] = 0.0 - - # Get total mpiio instructions - if os.path.exists(trace_name + '.mpiio-inst.stats') \ - and trace_mode[trace][:12] == 'Detailed+MPI': - content = [] - with open(trace_name + '.mpiio-inst.stats') as f: - content = f.readlines() - - for line in content: - if line.split(): - if line.split()[0] == 'Total': - raw_data['mpiio_ins'][trace] = int(float(line.split()[1])) - else: - raw_data['mpiio_ins'][trace] = 0.0 - - # Get useful cycles - if os.path.exists(trace_name + '.cycles.stats'): - content = [] - with open(trace_name + '.cycles.stats') as f: - content = f.readlines() - - for line in content: - if line.split(): - if line.split()[0] == 'Total': - raw_data['useful_cyc'][trace] = int(float(line.split()[1])) - else: - raw_data['useful_cyc'][trace] = 'NaN' - - # Get useful instructions - if os.path.exists(trace_name + '.instructions.stats'): - content = [] - with open(trace_name + '.instructions.stats') as f: - content = f.readlines() - - for line in content: - if line.split(): - if line.split()[0] == 'Total': - raw_data['useful_ins'][trace] = int(float(line.split()[1])) - else: - raw_data['useful_ins'][trace] = 'NaN' - - # Get Efficiencies for BurstMode - if trace_mode[trace] == 'Burst+MPI': - # Get total, maximum and avg from burst useful - if os.path.exists(trace_name + '.burst_useful.stats'): - content = [] - list_burst_tot = [] - with open(trace_name + '.burst_useful.stats') as f: - content = f.readlines() - for line in content: - for field in line.split("\n"): - line_list = field.split("\t") - if "Total" in field.split("\t"): - count_procs = len(line_list[1:]) - list_burst_tot = [float(burst_time) for burst_time in line_list[1:count_procs]] - - raw_data['burst_useful_tot'][trace] = sum(list_burst_tot) - raw_data['burst_useful_avg'][trace] = sum(list_burst_tot)/len(list_burst_tot) - raw_data['burst_useful_max'][trace] = max(list_burst_tot) - - else: - raw_data['burst_useful_avg'][trace] = 'NaN' - raw_data['burst_useful_max'][trace] = 'NaN' - raw_data['burst_useful_tot'][trace] = 'NaN' - else: - raw_data['burst_useful_avg'][trace] = 0.0 - raw_data['burst_useful_max'][trace] = 0.0 - raw_data['burst_useful_tot'][trace] = 0.0 - - # Get timing for SIMULATED traces - if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': - # Get maximum useful duration for simulated trace - if os.path.exists(trace_name_sim + '.timings.stats'): - content = [] - with open(trace_name_sim + '.timings.stats') as f: - content = f.readlines() - - for line in content: - if line.split(): - if line.split()[0] == 'Maximum': - raw_data['useful_dim'][trace] = float(line.split()[1]) - else: - raw_data['useful_dim'][trace] = 'NaN' - - # Get runtime for simulated trace - if os.path.exists(trace_name_sim + '.runtime.stats'): - content = [] - with open(trace_name_sim + '.runtime.stats') as f: - content = f.readlines() - - for line in content: - if line.split(): - if line.split()[0] == 'Average': - raw_data['runtime_dim'][trace] = float(line.split()[1]) - else: - raw_data['runtime_dim'][trace] = 'NaN' - - # Get outsideMPI max for simulated trace - if os.path.exists(trace_name_sim + '.outside_mpi.stats'): - with open(trace_name_sim + '.outside_mpi.stats') as f: - content = f.readlines() - list_outside_mpi = [] - list_thread_outside_mpi = [] - init_count_thread = False - count_threads = 1 - for line1 in content[1:(len(content) - 8)]: - line = line1.split("\t") - # print(line) - if line: - if line[0] != 'Total' and line[0] != 'Average' \ - and line[0] != 'Maximum' and line[0] != 'StDev' \ - and line[0] != 'Avg/Max': - # To extract the count of MPI tasks - if float(line[1]) != raw_data['runtime_dim'][trace]: - list_outside_mpi.append(float(line[1])) - # To extract the count of threads per MPI task - if len(list_outside_mpi) > 1: - list_thread_outside_mpi.append(count_threads) - count_threads = 1 - else: - if len(list_outside_mpi) == 1 and not init_count_thread: - count_threads = 2 - init_count_thread = True - else: - count_threads += 1 - - if line[0] == "THREAD 1.1.1": - max_time_outside_mpi = float(line[1]) - - list_thread_outside_mpi.append(count_threads) - if len(list_outside_mpi) != 0: - raw_data['outsidempi_dim'][trace] = max(list_outside_mpi) - else: - raw_data['outsidempi_dim'][trace] = max_time_outside_mpi - else: - raw_data['outsidempi_dim'][trace] = 0.0 - else: - raw_data['useful_dim'][trace] = 'Non-Avail' - raw_data['runtime_dim'][trace] = 'Non-Avail' - raw_data['outsidempi_dim'][trace] = 'Non-Avail' - - # Remove paramedir output files - move_files(trace_name + '.timings.stats', path_dest, cmdl_args) - move_files(trace_name + '.runtime.stats', path_dest, cmdl_args) - move_files(trace_name + '.cycles.stats', path_dest, cmdl_args) - move_files(trace_name + '.instructions.stats', path_dest, cmdl_args) - move_files(trace_name + '.flushing.stats', path_dest, cmdl_args) - move_files(trace_name + '.posixio-cycles.stats', path_dest, cmdl_args) - move_files(trace_name + '.posixio-inst.stats', path_dest, cmdl_args) - move_files(trace_name + '.flushing-cycles.stats', path_dest, cmdl_args) - move_files(trace_name + '.flushing-inst.stats', path_dest, cmdl_args) - - if trace_mode[trace][:12] == 'Detailed+MPI': - move_files(trace_name + '.mpi_io.stats', path_dest, cmdl_args) - move_files(trace_name + '.posixio_call.stats', path_dest, cmdl_args) - move_files(trace_name + '.outside_mpi.stats', path_dest, cmdl_args) - move_files(trace_name + '.mpiio-cycles.stats', path_dest, cmdl_args) - move_files(trace_name + '.mpiio-inst.stats', path_dest, cmdl_args) - - if trace_mode[trace][:9] == 'Burst+MPI': - move_files(trace_name + '.2dh_BurstEff.stats', path_dest, cmdl_args) - move_files(trace_name + '.burst_useful.stats', path_dest, cmdl_args) - - if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': - move_files(trace_name_sim + '.timings.stats', path_dest, cmdl_args) - move_files(trace_name_sim + '.runtime.stats', path_dest, cmdl_args) - move_files(trace_name_sim + '.outside_mpi.stats', path_dest, cmdl_args) - move_files(trace_sim, path_dest, cmdl_args) - move_files(trace_sim[:-4] + '.pcf', path_dest, cmdl_args) - move_files(trace_sim[:-4] + '.row', path_dest, cmdl_args) - # To move simulation trace and ideal cfg - move_files(trace_sim[:-8] + '.dim', path_dest, cmdl_args) - remove_files(trace_sim[:-8] + '.row', cmdl_args) - remove_files(trace_sim[:-8] + '.pcf', cmdl_args) - move_files(trace_sim[:-8] + '.dimemas_ideal.cfg', path_dest, cmdl_args) - if trace[-7:] == ".prv.gz": - remove_files(trace_name_control + '.prv', cmdl_args) - # move_files(trace_name + '.prv', path_dest, cmdl_args) - - time_prs = time.time() - time_prs - - time_tot = time.time() - time_tot - print('Finished successfully in {0:.1f} seconds.'.format(time_tot)) - print('') - - return raw_data, list_mpi_procs_count - - -def create_ideal_trace(trace, processes, task_per_node, cmdl_args): - """Runs prv2dim and dimemas with ideal configuration for given trace.""" - if trace[-4:] == ".prv": - trace_dim = trace[:-4] + '_' + str(processes) + 'P' + '.dim' - trace_sim = trace[:-4] + '_' + str(processes) + 'P' + '.sim.prv' - trace_name = trace[:-4] - cmd = ['prv2dim', trace, trace_dim] - elif trace[-7:] == ".prv.gz": - with gzip.open(trace, 'rb') as f_in: - with open(trace[:-7] + '.prv', 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - trace_dim = trace[:-7] + '_' + str(processes) + 'P' + '.dim' - trace_sim = trace[:-7] + '_' + str(processes) + 'P' + '.sim.prv' - trace_name = trace[:-7] - trace_unzip = trace[:-3] - cmd = ['prv2dim', trace_unzip, trace_dim] - - run_command(cmd, cmdl_args) - - if os.path.isfile(trace_dim): - if cmdl_args.debug: - print('==DEBUG== Created file ' + trace_dim) - else: - print('==Error== ' + trace_dim + 'could not be creaeted.') - return - - # Create Dimemas configuration - cfg_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs') - - content = [] - with open(os.path.join(cfg_dir, 'dimemas_ideal.cfg')) as f: - content = f.readlines() - - content = [line.replace('REPLACE_BY_CPUS_PER_NODE', str(task_per_node)) for line in content] - content = [line.replace('REPLACE_BY_NTASKS', str(processes)) for line in content] - content = [line.replace('REPLACE_BY_COLLECTIVES_PATH', os.path.join(cfg_dir, 'dimemas.collectives')) for line in - content] - - with open(trace_name + '_' + str(processes) + 'P' + '.dimemas_ideal.cfg', 'w') as f: - f.writelines(content) - - cmd = ['Dimemas', '-S', '32k', '--dim', trace_dim, '-p', trace_sim, trace_name + '_' + str(processes) - + 'P' + '.dimemas_ideal.cfg'] - run_command(cmd, cmdl_args) - - if os.path.isfile(trace_sim): - if cmdl_args.debug: - print('==DEBUG== Created file ' + trace_sim) - return trace_sim - else: - print('==Error== ' + trace_sim + ' could not be created.') - return '' - - -def print_raw_data_table(raw_data, trace_list, trace_processes): - """Prints the raw data table in human readable form on stdout.""" - global raw_data_doc - - print('Overview of the collected raw data:') - - longest_name = len(sorted(raw_data_doc.values(), key=len)[-1]) - - line = ''.rjust(longest_name) - for trace in trace_list: - line += ' | ' - line += str(trace_processes[trace]).rjust(15) - print(''.ljust(len(line), '-')) - print(line) - - print(''.ljust(len(line), '-')) - final_line_raw_data = ''.ljust(len(line), '-') - - for data_key in raw_data_doc: - line = raw_data_doc[data_key].ljust(longest_name) - for trace in trace_list: - line += ' | ' - if raw_data[data_key][trace] != "Non-Avail" and raw_data[data_key][trace] != 'NaN': - line += str(round((raw_data[data_key][trace]),2)).rjust(15) - else: - line += str(raw_data[data_key][trace]).rjust(15) - print(line) - print(final_line_raw_data) - print('') - - -def print_raw_data_csv(raw_data, trace_list, trace_processes): - """Prints the model factors table in a csv file.""" - global raw_data_doc - - delimiter = ';' - # File is stored in the trace directory - # file_path = os.path.join(os.path.dirname(os.path.realpath(trace_list[0])), 'modelfactors.csv') - # File is stored in the execution directory - file_path = os.path.join(os.getcwd(), 'rawdata.csv') - - with open(file_path, 'w') as output: - line = 'Number of processes' - for trace in trace_list: - line += delimiter - line += str(trace_processes[trace]) - output.write(line + '\n') - - for raw_key in raw_data_doc: - line = '#' + raw_data_doc[raw_key] - for trace in trace_list: - line += delimiter - try: # except NaN - line += '{0:.2f}'.format(raw_data[raw_key][trace]) - except ValueError: - line += '{}'.format(raw_data[raw_key][trace]) - output.write(line + '\n') - - print('======== Output Files: Traces raw data and intermediate data ========') - print('Raw data written to ' + file_path) - file_path_intermediate = os.path.join(os.getcwd(), 'scratch_out_basicanalysis') - print('Intermediate file written to ' + file_path_intermediate) - print('') \ No newline at end of file diff --git a/simplemetrics.py b/simplemetrics.py deleted file mode 100644 index fcf0bf8..0000000 --- a/simplemetrics.py +++ /dev/null @@ -1,1249 +0,0 @@ -#!/usr/bin/env python3 - -"""Functions to compute the model metrics.""" - -from __future__ import print_function, division -import sys - -from rawdata import * -from collections import OrderedDict - -# error import variables -error_import_pandas = False -error_import_seaborn = False -error_import_matplotlib = False -error_import_numpy = False - -try: - import numpy as np -except ImportError: - error_import_numpy = True - - -try: - import pandas as pd -except ImportError: - error_import_pandas = True - -try: - import seaborn as sns -except ImportError: - error_import_seaborn = True - - -try: - import matplotlib.pyplot as plt -except ImportError: - error_import_matplotlib = True - - - -# Contains all model factor entries with a printable name. -# This is used to generate and print all model factors, so, if an entry is added, -# it should be added here, too. - -other_metrics_doc = OrderedDict([('elapsed_time', 'Elapsed time (sec)'), - ('efficiency', 'Efficiency'), - ('speedup', 'Speedup'), - ('ipc', 'Average IPC'), - ('freq', 'Average frequency (GHz)'), - ('flushing', 'Flushing (%)'), - ('io_mpiio', 'MPI I/O (%)'), - ('io_posix', 'POSIX I/O (%)'), - ('io_eff', 'I/O Efficiency (%)')]) - -mod_factors_doc = OrderedDict([('global_eff', 'Global efficiency'), - ('parallel_eff', '-- Parallel efficiency'), - ('load_balance', ' -- Load balance'), - ('comm_eff', ' -- Communication efficiency'), - ('serial_eff', ' -- Serialization efficiency'), - ('transfer_eff', ' -- Transfer efficiency'), - ('comp_scale', '-- Computation scalability'), - ('ipc_scale', ' -- IPC scalability'), - ('inst_scale', ' -- Instruction scalability'), - ('freq_scale', ' -- Frequency scalability')]) - -mod_factors_scale_plus_io_doc = OrderedDict([('comp_scale', '-- Computation scalability + I/O'), - ('ipc_scale', ' -- IPC scalability'), - ('inst_scale', ' -- Instruction scalability'), - ('freq_scale', ' -- Frequency scalability')]) - - -def create_mod_factors(trace_list): - """Creates 2D dictionary of the model factors and initializes with an empty - string. The mod_factors dictionary has the format: [mod factor key][trace]. - """ - global mod_factors_doc - mod_factors = {} - for key in mod_factors_doc: - trace_dict = {} - for trace_name in trace_list: - trace_dict[trace_name] = 0.0 - mod_factors[key] = trace_dict - - return mod_factors - - -def create_mod_factors_scale_io(trace_list): - """Creates 2D dictionary of the model factors and initializes with an empty - string. The mod_factors dictionary has the format: [mod factor key][trace]. - """ - global mod_factors_scale_plus_io_doc - mod_factors_scale_plus_io = {} - for key in mod_factors_scale_plus_io_doc: - trace_dict = {} - for trace_name in trace_list: - trace_dict[trace_name] = 0.0 - mod_factors_scale_plus_io[key] = trace_dict - - return mod_factors_scale_plus_io - - -def create_other_metrics(trace_list): - """Creates 2D dictionary of the model factors and initializes with an empty - string. The mod_factors dictionary has the format: [mod factor key][trace]. - """ - global other_metrics_doc - other_metrics = {} - for key in other_metrics_doc: - trace_dict = {} - for trace_name in trace_list: - trace_dict[trace_name] = 0.0 - other_metrics[key] = trace_dict - - return other_metrics - - -def get_scaling_type(raw_data, trace_list, trace_processes, cmdl_args): - """Guess the scaling type (weak/strong) based on the useful instructions. - Computes the normalized instruction ratio for all measurements, whereas the - normalized instruction ratio is (instructions ratio / process ratio) with - the smallest run as reference. For exact weak scaling the normalized ratio - should be exactly 1 and for exact strong scaling it should be close to zero - with an upper bound of 0.5. The eps value defines the threshold to be - considered weak scaling and should give enough buffer to safely handle - non-ideal scaling. - """ - eps = 0.9 - normalized_inst_ratio = 0 - - # Check if there is only one trace. - if len(trace_list) == 1: - return 'strong' - - for trace in trace_list: - try: # except NaN - inst_ratio = float(raw_data['useful_ins'][trace]) / float(raw_data['useful_ins'][trace_list[0]]) - except: - inst_ratio = 0.0 - try: # except NaN - proc_ratio = float(trace_processes[trace]) / float(trace_processes[trace_list[0]]) - except: - proc_ratio = 'NaN' - - normalized_inst_ratio += inst_ratio / proc_ratio - - # Get the average inst increase. Ignore ratio of first trace 1.0) - normalized_inst_ratio = (normalized_inst_ratio - 1) / (len(trace_list) - 1) - - scaling_computed = '' - - if normalized_inst_ratio > eps: - scaling_computed = 'weak' - else: - scaling_computed = 'strong' - - if cmdl_args.scaling == 'auto': - if cmdl_args.debug: - print('==DEBUG== Detected ' + scaling_computed + ' scaling.') - print('') - return scaling_computed - - if cmdl_args.scaling == 'weak': - if scaling_computed == 'strong': - print('==Warning== Scaling set to weak scaling but detected strong scaling.') - print('') - return 'weak' - - if cmdl_args.scaling == 'strong': - if scaling_computed == 'weak': - print('==Warning== Scaling set to strong scaling but detected weak scaling.') - print('') - return 'strong' - - print('==Error== reached undefined control flow state.') - sys.exit(1) - - -def compute_model_factors(raw_data, trace_list, trace_processes, trace_mode, list_mpi_procs_count, cmdl_args): - """Computes the model factors from the gathered raw data and returns the - according dictionary of model factors.""" - mod_factors = create_mod_factors(trace_list) - other_metrics = create_other_metrics(trace_list) - mod_factors_scale_plus_io = create_mod_factors_scale_io(trace_list) - - # Guess the weak or strong scaling - scaling = get_scaling_type(raw_data, trace_list, trace_processes, cmdl_args) - - # Loop over all traces - for trace in trace_list: - - proc_ratio = float(trace_processes[trace]) / float(trace_processes[trace_list[0]]) - - # Flushing measurements - try: # except NaN - other_metrics['flushing'][trace] = raw_data['flushing_tot'][trace] \ - / (raw_data['runtime'][trace] * trace_processes[trace]) * 100.0 - except: - other_metrics['flushing'][trace] = 0.0 - - # I/O measurements - try: # except NaN - other_metrics['io_mpiio'][trace] = raw_data['mpiio_tot'][trace] \ - / (raw_data['runtime'][trace] * trace_processes[trace]) * 100.0 - except: - other_metrics['io_mpiio'][trace] = 0.0 - - try: # except NaN - other_metrics['io_posix'][trace] = raw_data['io_tot'][trace] \ - / (raw_data['runtime'][trace] * trace_processes[trace]) * 100.0 - except: - other_metrics['io_posix'][trace] = 0.0 - try: # except NaN - io_total = raw_data['mpiio_tot'][trace] + raw_data['flushing_tot'][trace] + raw_data['io_tot'][trace] - other_metrics['io_eff'][trace] = raw_data['useful_tot'][trace] \ - / (raw_data['useful_tot'][trace] + io_total) * 100.0 - except: - other_metrics['io_eff'][trace] = 0.0 - - # Basic efficiency factors - try: # except NaN - if trace_mode[trace] == 'Burst+MPI': - mod_factors['load_balance'][trace] = float(raw_data['burst_useful_avg'][trace]) \ - / float(raw_data['burst_useful_max'][trace]) * 100.0 - else: - if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: - mod_factors['load_balance'][trace] = raw_data['useful_plus_io_avg'][trace] \ - / raw_data['useful_plus_io_max'][trace] * 100.0 - else: - mod_factors['load_balance'][trace] = raw_data['useful_avg'][trace] \ - / raw_data['useful_max'][trace] * 100.0 - except: - mod_factors['load_balance'][trace] = 'NaN' - - try: # except NaN - if trace_mode[trace] == 'Burst+MPI': - mod_factors['comm_eff'][trace] = float(raw_data['burst_useful_max'][trace]) \ - / raw_data['runtime'][trace] * 100.0 - else: - if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: - mod_factors['comm_eff'][trace] = raw_data['useful_plus_io_max'][trace] \ - / raw_data['runtime'][trace] * 100.0 - else: - mod_factors['comm_eff'][trace] = raw_data['useful_max'][trace] \ - / raw_data['runtime'][trace] * 100.0 - except: - mod_factors['comm_eff'][trace] = 'NaN' - - try: # except NaN - mod_factors['serial_eff'][trace] = float(raw_data['outsidempi_dim'][trace]) \ - / float(raw_data['runtime_dim'][trace]) * 100.0 - if mod_factors['serial_eff'][trace] > 100.0: - mod_factors['serial_eff'][trace] = 'Warning!' - except: - if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': - mod_factors['serial_eff'][trace] = 'NaN' - else: - mod_factors['serial_eff'][trace] = 'Non-Avail' - - try: # except NaN - if mod_factors['serial_eff'][trace] != 'Warning!': - mod_factors['transfer_eff'][trace] = mod_factors['comm_eff'][trace] \ - / mod_factors['serial_eff'][trace] * 100.0 - else: - mod_factors['transfer_eff'][trace] = float(raw_data['runtime_dim'][trace]) \ - / float(raw_data['runtime'][trace]) * 100.0 - - if mod_factors['transfer_eff'][trace] > 100.0: - mod_factors['transfer_eff'][trace] = 'Warning!' - except: - if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': - mod_factors['transfer_eff'][trace] = 'NaN' - else: - mod_factors['transfer_eff'][trace] = 'Non-Avail' - - # Parallel Efficiency - try: # except NaN - if trace_mode[trace] == 'Burst+MPI': - mod_factors['parallel_eff'][trace] = float(raw_data['burst_useful_avg'][trace]) \ - / float(raw_data['runtime'][trace]) * 100.0 - else: - if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: - mod_factors['parallel_eff'][trace] = float(raw_data['useful_plus_io_avg'][trace]) \ - / float(raw_data['runtime'][trace]) * 100.0 - else: - mod_factors['parallel_eff'][trace] = mod_factors['load_balance'][trace] \ - * mod_factors['comm_eff'][trace] / 100.0 - except: - mod_factors['parallel_eff'][trace] = 'NaN' - - # Computation Scale only useful computation - try: # except NaN - if len(trace_list) > 1: - if trace_mode[trace] == 'Burst+MPI': - if scaling == 'strong': - mod_factors['comp_scale'][trace] = raw_data['burst_useful_tot'][trace_list[0]] \ - / raw_data['burst_useful_tot'][trace] * 100.0 - else: - mod_factors['comp_scale'][trace] = raw_data['burst_useful_tot'][trace_list[0]] \ - / raw_data['burst_useful_tot'][trace] \ - * proc_ratio * 100.0 - else: - if scaling == 'strong': - mod_factors['comp_scale'][trace] = raw_data['useful_tot'][trace_list[0]] \ - / raw_data['useful_tot'][trace] * 100.0 - else: - mod_factors['comp_scale'][trace] = raw_data['useful_tot'][trace_list[0]] \ - / raw_data['useful_tot'][trace] \ - * proc_ratio * 100.0 - else: - mod_factors['comp_scale'][trace] = 'Non-Avail' - except: - mod_factors['comp_scale'][trace] = 'NaN' - - # Computation Scale + Serial I/O - try: # except NaN - if len(trace_list) > 1: - if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: - io_serial_0 = raw_data['io_tot'][trace_list[0]] + raw_data['flushing_tot'][trace_list[0]] - io_serial_n = raw_data['io_tot'][trace] + raw_data['flushing_tot'][trace] - if scaling == 'strong': - mod_factors_scale_plus_io['comp_scale'][trace] = (raw_data['useful_tot'][trace_list[0]] - + io_serial_0) / (raw_data['useful_tot'][trace] - + io_serial_n) * 100.0 - else: - mod_factors_scale_plus_io['comp_scale'][trace] = (raw_data['useful_tot'][trace_list[0]] - + io_serial_0) \ - / (raw_data['useful_tot'][trace] + io_serial_n) \ - * proc_ratio * 100.0 - else: - mod_factors_scale_plus_io['comp_scale'][trace] = mod_factors['comp_scale'][trace] - else: - mod_factors_scale_plus_io['comp_scale'][trace] = 'Non-Avail' - except: - mod_factors_scale_plus_io['comp_scale'][trace] = 'NaN' - - try: # except NaN - if len(trace_list) > 1: - mod_factors['global_eff'][trace] = mod_factors['parallel_eff'][trace] \ - * mod_factors['comp_scale'][trace] / 100.0 - else: - mod_factors['global_eff'][trace] = mod_factors['parallel_eff'][trace] \ - * 100.0 / 100.0 - except: - mod_factors['global_eff'][trace] = 'NaN' - - # Basic scalability factors - try: # except NaN - other_metrics['ipc'][trace] = float(raw_data['useful_ins'][trace]) \ - / float(raw_data['useful_cyc'][trace]) - except: - other_metrics['ipc'][trace] = 'NaN' - try: # except NaN - if len(trace_list) > 1: - if trace_mode[trace][:5] != 'Burst': - mod_factors['ipc_scale'][trace] = other_metrics['ipc'][trace] \ - / other_metrics['ipc'][trace_list[0]] * 100.0 - else: - mod_factors['ipc_scale'][trace] = 'Non-Avail' - else: - mod_factors['ipc_scale'][trace] = 'Non-Avail' - except: - mod_factors['ipc_scale'][trace] = 'NaN' - - # IPC scale + Serial I/O - try: # except NaN - if len(trace_list) > 1: - if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: - ipc_serial_io_0 = (raw_data['useful_ins'][trace_list[0]] + raw_data['io_ins'][trace_list[0]] - + raw_data['flushing_ins'][trace_list[0]]) \ - / (raw_data['useful_cyc'][trace_list[0]] + raw_data['io_cyc'][trace_list[0]] - + raw_data['flushing_cyc'][trace_list[0]]) - - ipc_serial_io_n = (raw_data['useful_ins'][trace] + raw_data['io_ins'][trace] - + raw_data['flushing_ins'][trace]) \ - / (raw_data['useful_cyc'][trace] + raw_data['io_cyc'][trace] - + raw_data['flushing_cyc'][trace]) - mod_factors_scale_plus_io['ipc_scale'][trace] = ipc_serial_io_n / ipc_serial_io_0 * 100.0 - else: - mod_factors_scale_plus_io['ipc_scale'][trace] = 0.0 - else: - mod_factors_scale_plus_io['ipc_scale'][trace] = 'Non-Avail' - except: - mod_factors_scale_plus_io['ipc_scale'][trace] = mod_factors['ipc_scale'][trace] - - try: # except NaN - other_metrics['freq'][trace] = float(raw_data['useful_cyc'][trace]) \ - / float(raw_data['useful_tot'][trace]) / 1000 - except: - other_metrics['freq'][trace] = 'NaN' - - try: # except NaN - if len(trace_list) > 1: - if trace_mode[trace][:5] != 'Burst': - mod_factors['freq_scale'][trace] = other_metrics['freq'][trace] \ - / other_metrics['freq'][trace_list[0]] * 100.0 - else: - mod_factors['freq_scale'][trace] = 'Non-Avail' - else: - mod_factors['freq_scale'][trace] = 'Non-Avail' - except: - mod_factors['freq_scale'][trace] = 'NaN' - - # freq scale + Serial I/O - try: # except NaN - if len(trace_list) > 1: - if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: - freq_serial_io_0 = (float(raw_data['useful_cyc'][trace_list[0]]) - + float(raw_data['io_cyc'][trace_list[0]]) - + float(raw_data['flushing_cyc'][trace_list[0]])) \ - / (float(raw_data['useful_tot'][trace_list[0]]) - + float(raw_data['io_tot'][trace_list[0]]) - + float(raw_data['flushing_tot'][trace_list[0]])) / 1000 - - freq_serial_io_n = (float(raw_data['useful_cyc'][trace]) + float(raw_data['io_cyc'][trace]) - + float(raw_data['flushing_cyc'][trace])) \ - / (float(raw_data['useful_tot'][trace]) - + float(raw_data['io_tot'][trace]) - + float(raw_data['flushing_tot'][trace])) / 1000 - mod_factors_scale_plus_io['freq_scale'][trace] = freq_serial_io_n / freq_serial_io_0 * 100.0 - else: - mod_factors_scale_plus_io['freq_scale'][trace] = mod_factors['freq_scale'][trace] - else: - mod_factors_scale_plus_io['freq_scale'][trace] = 'Non-Avail' - except: - mod_factors_scale_plus_io['freq_scale'][trace] = 'NaN' - - try: # except NaN - if len(trace_list) > 1: - if trace_mode[trace][:5] != 'Burst': - if scaling == 'strong': - mod_factors['inst_scale'][trace] = float(raw_data['useful_ins'][trace_list[0]]) \ - / float(raw_data['useful_ins'][trace]) * 100.0 - else: - mod_factors['inst_scale'][trace] = float(raw_data['useful_ins'][trace_list[0]]) \ - / float(raw_data['useful_ins'][trace]) \ - * proc_ratio * 100.0 - else: - mod_factors['inst_scale'][trace] = 'Non-Avail' - else: - mod_factors['inst_scale'][trace] = 'Non-Avail' - except: - mod_factors['inst_scale'][trace] = 'NaN' - - # ins scale + Serial I/O - try: # except NaN - if len(trace_list) > 1: - if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: - useful_ins_plus_io_0 = float(raw_data['useful_ins'][trace_list[0]]) \ - + float(raw_data['io_ins'][trace_list[0]]) \ - + float(raw_data['flushing_ins'][trace_list[0]]) - useful_ins_plus_io_n = float(raw_data['useful_ins'][trace]) \ - + float(raw_data['io_ins'][trace]) \ - + float(raw_data['flushing_ins'][trace]) - if scaling == 'strong': - mod_factors_scale_plus_io['inst_scale'][trace] = useful_ins_plus_io_0 \ - / useful_ins_plus_io_n * 100.0 - else: - mod_factors_scale_plus_io['inst_scale'][trace] = useful_ins_plus_io_0 / useful_ins_plus_io_n \ - * proc_ratio * 100.0 - else: - mod_factors_scale_plus_io['inst_scale'][trace] = mod_factors['inst_scale'][trace] - else: - mod_factors_scale_plus_io['inst_scale'][trace] = 'Non-Avail' - except: - mod_factors_scale_plus_io['inst_scale'][trace] = 'NaN' - - try: # except NaN - if len(trace_list) > 1: - other_metrics['speedup'][trace] = raw_data['runtime'][trace_list[0]] \ - / raw_data['runtime'][trace] - #if scaling == 'strong': - # other_metrics['speedup'][trace] = raw_data['runtime'][trace_list[0]] \ - # / raw_data['runtime'][trace] - #else: - # other_metrics['speedup'][trace] = raw_data['runtime'][trace_list[0]] \ - # / raw_data['runtime'][trace] * proc_ratio - else: - other_metrics['speedup'][trace] = 'Non-Avail' - except: - other_metrics['speedup'][trace] = 'NaN' - - try: # except NaN - other_metrics['elapsed_time'][trace] = raw_data['runtime'][trace] * 0.000001 - except: - other_metrics['elapsed_time'][trace] = 'NaN' - - try: # except NaN - if len(trace_list) > 1: - #other_metrics['efficiency'][trace] = other_metrics['speedup'][trace] / proc_ratio - if scaling == 'strong': - other_metrics['efficiency'][trace] = raw_data['runtime'][trace_list[0]] \ - / (raw_data['runtime'][trace] * proc_ratio) - else: - other_metrics['efficiency'][trace] = raw_data['runtime'][trace_list[0]] \ - / raw_data['runtime'][trace] - else: - other_metrics['efficiency'][trace] = 'Non-Avail' - except: - other_metrics['efficiency'][trace] = 'NaN' - - return mod_factors, mod_factors_scale_plus_io, other_metrics - - -def read_mod_factors_csv(cmdl_args): - """Reads the model factors table from a csv file.""" - global mod_factors_doc - - delimiter = ';' - file_path = cmdl_args.project - - # Read csv to list of lines - if os.path.isfile(file_path) and file_path[-4:] == '.csv': - with open(file_path, 'r') as f: - lines = f.readlines() - lines = [line.rstrip('\n') for line in lines] - else: - print('==ERROR==', file_path, 'is not a valid csv file.') - sys.exit(1) - - # Get the number of processes of the traces - processes = lines[0].split(delimiter) - processes.pop(0) - - # Create artificial trace_list and trace_processes - trace_list = [] - trace_processes = dict() - for process in processes: - trace_list.append(process) - trace_processes[process] = int(process) - - # Create empty mod_factors handle - mod_factors = create_mod_factors(trace_list) - - # Get mod_factor_doc keys - mod_factors_keys = list(mod_factors_doc.items()) - - # Iterate over the data lines - for index, line in enumerate(lines[1:len(mod_factors_keys) + 1]): - key = mod_factors_keys[index][0] - line = line.split(delimiter) - for index, trace in enumerate(trace_list): - mod_factors[key][trace] = float(line[index + 1]) - - if cmdl_args.debug: - print_mod_factors_table(mod_factors, trace_list, trace_processes) - - return mod_factors, trace_list, trace_processes - - -def print_mod_factors_table(mod_factors, other_metrics, mod_factors_scale_plus_io, trace_list, trace_processes): - """Prints the model factors table in human readable form on stdout.""" - global mod_factors_doc - global mod_factors_scale_plus_io_doc - - warning_io = [] - warning_flush = [] - warning_flush_wrong = [] - warning_simulation = [] - for trace in trace_list: - if 10.0 <= other_metrics['flushing'][trace] < 15.0: - warning_flush.append(1) - elif other_metrics['flushing'][trace] >= 15.0: - warning_flush_wrong.append(1) - if other_metrics['io_posix'][trace] >= 5.0: - warning_io.append(1) - if mod_factors['serial_eff'][trace] == "Warning!" \ - or mod_factors['transfer_eff'][trace] == "Warning!": - warning_simulation.append(1) - if len(warning_flush_wrong) > 0: - print("WARNING! Flushing in a trace is too high. Disabling standard output metrics...") - print(" Flushing is an overhead due to the tracer, please review your trace.") - print('') - return - - print('Overview of the Efficiency metrics:') - - longest_name = len(sorted(mod_factors_doc.values(), key=len)[-1]) - - line = ''.rjust(longest_name) - - if len(trace_list) == 1: - limit_min = trace_processes[trace_list[0]] - limit_max = trace_processes[trace_list[0]] - else: - limit_min = trace_processes[trace_list[0]] - limit_max = trace_processes[trace_list[len(trace_list)-1]] - - # BEGIN To adjust header to big number of processes - procs_header = [] - for index, trace in enumerate(trace_list): - if limit_min == limit_max and len(trace_list) > 1: - procs_header.append(str(trace_processes[trace]) + '[' + str(index+1) + ']') - else: - procs_header.append(str(trace_processes[trace])) - - max_len_header = 0 - for proc_h in procs_header: - if max_len_header < len(proc_h): - max_len_header = len(proc_h) - - value_to_adjust = 10 - if max_len_header > value_to_adjust: - value_to_adjust = max_len_header + 1 - # END To adjust header to big number of processes - - for index, trace in enumerate(trace_list): - line += ' | ' - if limit_min == limit_max and len(trace_list) > 1: - line += (str(trace_processes[trace]) + '[' + str(index+1) + ']').rjust(value_to_adjust) - else: - line += (str(trace_processes[trace])).rjust(value_to_adjust) - - print(''.ljust(len(line), '=')) - print(line) - line_procs_factors = line - - print(''.ljust(len(line), '=')) - - for mod_key in mod_factors_doc: - line = mod_factors_doc[mod_key].ljust(longest_name) - for trace in trace_list: - line += ' | ' - try: # except NaN - line += ('{0:.2f}%'.format(mod_factors[mod_key][trace])).rjust(value_to_adjust) - except ValueError: - line += ('{}'.format(mod_factors[mod_key][trace])).rjust(value_to_adjust) - print(line) - - # print('') - - io_metrics = [] - io_metrics.append(0) - for mod_key in mod_factors_scale_plus_io_doc: - for trace in trace_list: - if mod_factors_scale_plus_io[mod_key][trace] != 0: - io_metrics.append(1) - - if (len(warning_flush) >= 1 or len(warning_io) >= 1) and len(trace_list) > 1: - print(''.ljust(len(line_procs_factors), '-')) - for mod_key in mod_factors_scale_plus_io_doc: - line = mod_factors_scale_plus_io_doc[mod_key].ljust(longest_name) - for trace in trace_list: - line += ' | ' - try: # except NaN - line += ('{0:.2f}%'.format(mod_factors_scale_plus_io[mod_key][trace])).rjust(value_to_adjust) - except ValueError: - line += ('{}'.format(mod_factors_scale_plus_io[mod_key][trace])).rjust(value_to_adjust) - print(line) - - print(''.ljust(len(line_procs_factors), '=')) - - if len(warning_simulation) > 0: - print("===> Warning! Metrics obtained from simulated traces exceed 100%. " - "Please review original and simulated traces.") - print('') - - -def print_other_metrics_table(other_metrics, trace_list, trace_processes): - """Prints the model factors table in human readable form on stdout.""" - global other_metrics_doc - - print('Overview of the Speedup, IPC and Frequency:') - - longest_name = len(sorted(other_metrics_doc.values(), key=len)[-1]) - - line = ''.rjust(longest_name) - if len(trace_list) == 1: - limit_min = trace_processes[trace_list[0]] - limit_max = trace_processes[trace_list[0]] - else: - limit_min = trace_processes[trace_list[0]] - limit_max = trace_processes[trace_list[len(trace_list)-1]] - - # BEGIN To adjust header to big number of processes - procs_header = [] - for index, trace in enumerate(trace_list): - if limit_min == limit_max and len(trace_list) > 1: - procs_header.append(str(trace_processes[trace]) + '[' + str(index+1) + ']') - else: - procs_header.append(str(trace_processes[trace])) - - max_len_header = 0 - for proc_h in procs_header: - if max_len_header < len(proc_h): - max_len_header = len(proc_h) - - value_to_adjust = 10 - if max_len_header > value_to_adjust: - value_to_adjust = max_len_header + 1 - # END To adjust header to big number of processes - - for index, trace in enumerate(trace_list): - line += ' | ' - if limit_min == limit_max and len(trace_list) > 1: - line += (str(trace_processes[trace]) + '[' + str(index+1) + ']').rjust(value_to_adjust) - else: - line += (str(trace_processes[trace])).rjust(value_to_adjust) - - print(''.ljust(len(line), '-')) - print(line) - line_head = line - print(''.ljust(len(line), '-')) - - for mod_key in other_metrics_doc: - line = other_metrics_doc[mod_key].ljust(longest_name) - if len(trace_list) > 1: - if mod_key in ['speedup', 'ipc', 'freq', 'elapsed_time', 'efficiency']: - for trace in trace_list: - line += ' | ' - try: # except NaN - line += ('{0:.2f}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - except ValueError: - line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - print(line) - else: - if mod_key in ['ipc', 'freq', 'elapsed_time']: - for trace in trace_list: - line += ' | ' - try: # except NaN - line += ('{0:.2f}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - except ValueError: - line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - print(line) - print(''.ljust(len(line_head), '-')) - # print('') - - warning_io = [] - warning_flush = [] - for trace in trace_list: - if other_metrics['flushing'][trace] >= 10.0: - warning_flush.append(1) - if other_metrics['io_mpiio'][trace] >= 5.0 or other_metrics['io_posix'][trace] >= 5.0: - warning_io.append(1) - - for mod_key in other_metrics_doc: - line = other_metrics_doc[mod_key].ljust(longest_name) - # Print empty line to separate values - if mod_key in ['freq'] and len(warning_flush) > 0: - print("Overview of tracer\'s flushing weight:") - print(''.ljust(len(line_head), '-')) - - if mod_key not in ['speedup', 'ipc', 'freq', 'elapsed_time','efficiency']: - if mod_key in ['flushing']: - for trace in trace_list: - line += ' | ' - try: # except NaN - line += ('{0:.2f}%'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - except ValueError: - line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - if len(warning_flush) > 0: - print(line) - print(''.ljust(len(line_head), '-')) - elif mod_key in ['io_mpiio','io_posix']: - for trace in trace_list: - line += ' | ' - try: # except NaN - line += ('{0:.2f}%'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - except ValueError: - line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - if len(warning_io) > 0: - print(line) - elif mod_key in ['io_eff']: - for trace in trace_list: - line += ' | ' - try: # except NaN - line += ('{0:.2f}%'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - except ValueError: - line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) - if len(warning_io) > 0 or len(warning_flush) > 0: - print(line) - - # Print headers I/O - if mod_key in ['flushing'] and len(warning_io) > 0: - print(''.ljust(len(line), ' ')) - print('Overview of File I/O weight:') - print(''.ljust(len(line), '-')) - if mod_key in ['io_eff'] and len(warning_io) > 0: - print(''.ljust(len(line), '-')) - # print('') - - if len(warning_flush) > 0: - message_warning_flush = "WARNING! %Flushing is high and affects computation of efficiency metrics." - else: - message_warning_flush = "" - if len(warning_io) > 0: - message_warning_io = "WARNING! % File I/O is high and affects computation of efficiency metrics." - else: - message_warning_io = "" - print(message_warning_flush + message_warning_io) - print('') - - -def print_efficiency_table(mod_factors, trace_list, trace_processes): - """Prints the model factors table in human readable form on stdout.""" - global mod_factors_doc - - longest_name = len(sorted(mod_factors_doc.values(), key=len)[-1]) - delimiter = ',' - file_path = os.path.join(os.getcwd(), 'efficiency_table.csv') - with open(file_path, 'w') as output: - line = '\"Number of processes\"' - if len(trace_list) == 1: - limit_min = trace_processes[trace_list[0]] - limit_max = trace_processes[trace_list[0]] - else: - limit_min = trace_processes[trace_list[0]] - limit_max = trace_processes[trace_list[len(trace_list)-1]] - - for index, trace in enumerate(trace_list): - line += delimiter - if limit_min == limit_max and len(trace_list) > 1: - line += str(trace_processes[trace]) + '[' + str(index+1) + ']' - else: - line += str(trace_processes[trace]) - output.write(line + '\n') - - for mod_key in mod_factors_doc: - if mod_key not in ['speedup', 'ipc', 'freq', 'elapsed_time', 'efficiency', 'flushing', 'io_mpiio', 'io_posix']: - if mod_key in ['parallel_eff', 'comp_scale']: - line = "\"" + mod_factors_doc[mod_key].replace(' ', '', 2) + "\"" - elif mod_key in ['load_balance', 'comm_eff','ipc_scale', 'inst_scale','freq_scale']: - line = "\"" + mod_factors_doc[mod_key].replace(' ', ' ', 2) + "\"" - elif mod_key in ['serial_eff', 'transfer_eff']: - line = "\" " + mod_factors_doc[mod_key].replace(' ', ' ', 4) + "\"" - else: - line = "\"" + mod_factors_doc[mod_key] + "\"" - for trace in trace_list: - line += delimiter - try: # except NaN - if mod_factors[mod_key][trace] == "Non-Avail" or mod_factors[mod_key][trace] == "Warning!": - line += '0.00' - else: - line += '{0:.2f}'.format(mod_factors[mod_key][trace]) - except ValueError: - line += '{}'.format(mod_factors[mod_key][trace]) - output.write(line + '\n') - # print('') - - # Create Gnuplot file for efficiency plot - gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'efficiency_table.gp') - content = [] - - with open(gp_template) as f: - content = f.readlines() - - limit_procs = 500 + len(trace_list) * 60 - - # Replace xrange - content = [line.replace('#REPLACE_BY_SIZE', ''.join(['set terminal pngcairo enhanced dashed crop size ', - str(limit_procs), ',460 font "Latin Modern Roman,14"'])) - for line in content] - - file_path = os.path.join(os.getcwd(), 'efficiency_table.gp') - with open(file_path, 'w') as f: - f.writelines(content) - - # print('======== Plot (gnuplot File): EFFICIENCY Table ========') - if len(trace_list) > 1: - print('Efficiency Table written to ' + file_path[:len(file_path) - 3] + '.gp') - # print('') - - -def print_mod_factors_csv(mod_factors, trace_list, trace_processes): - """Prints the model factors table in a csv file.""" - global mod_factors_doc - - delimiter = ';' - # File is stored in the trace directory - # file_path = os.path.join(os.path.dirname(os.path.realpath(trace_list[0])), 'modelfactors.csv') - # File is stored in the execution directory - file_path = os.path.join(os.getcwd(), 'modelfactors.csv') - - with open(file_path, 'w') as output: - line = "\"Number of processes\"" - for trace in trace_list: - line += delimiter - line += str(trace_processes[trace]) - output.write(line + '\n') - - for mod_key in mod_factors_doc: - line = "\"" + mod_factors_doc[mod_key].replace(' ', '', 2) + "\"" - for trace in trace_list: - line += delimiter - try: # except NaN - line += '{0:.6f}'.format(mod_factors[mod_key][trace]) - except ValueError: - line += '{}'.format(mod_factors[mod_key][trace]) - output.write(line + '\n') - - output.write('#\n') - - print('======== Output Files: Metrics and Plots ========') - print('Model factors written to ' + file_path) - - -def print_other_metrics_csv(other_metrics, trace_list, trace_processes): - """Prints the model factors table in a csv file.""" - global other_metrics_doc - - delimiter = ';' - # File is stored in the trace directory - # file_path = os.path.join(os.path.dirname(os.path.realpath(trace_list[0])), 'modelfactors.csv') - # File is stored in the execution directory - file_path = os.path.join(os.getcwd(), 'other_metrics.csv') - - with open(file_path, 'w') as output: - line = "\"Number of processes\"" - for trace in trace_list: - line += delimiter - line += str(trace_processes[trace]) - output.write(line + '\n') - - for mod_key in other_metrics_doc: - line = "\"" + other_metrics_doc[mod_key].replace(' ', '', 2) +"\"" - for trace in trace_list: - line += delimiter - try: # except NaN - line += '{0:.6f}'.format(other_metrics[mod_key][trace]) - except ValueError: - line += '{}'.format(other_metrics[mod_key][trace]) - output.write(line + '\n') - - output.write('#\n') - - print('======== Output File: Other Metrics ========') - print('Speedup, IPC, Frequency, I/O and Flushing written to ' + file_path) - print('') - - -def plots_efficiency_table_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args): - # Plotting using python - # For plotting using python, read the csv file - - file_path = os.path.join(os.getcwd(), 'efficiency_table.csv') - df = pd.read_csv(file_path) - metrics = df['Number of processes'].tolist() - - traces_procs = list(df.keys())[1:] - - # To control same number of processes for the header on plots and table - same_procs = True - procs_trace_prev = trace_processes[trace_list[0]] - tasks_trace_prev = trace_tasks[trace_list[0]] - threads_trace_prev = trace_threads[trace_list[0]] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ - and threads_trace_prev == threads: - same_procs *= True - else: - same_procs *= False - - # Set limit for projection - if cmdl_args.limit: - limit = cmdl_args.limit - else: - limit = str(trace_processes[trace_list[len(trace_list)-1]]) - - limit_min = trace_processes[trace_list[0]] - - # To xticks label - label_xtics = [] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if int(limit) == int(limit_min) and same_procs: - s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' - elif int(limit) == int(limit_min) and not same_procs: - s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' - else: - s_xtics = str(trace_processes[trace]) - if s_xtics in label_xtics: - s_xtics += '[' + str(index + 1) + ']' - label_xtics.append(s_xtics) - ##### End xticks - - # BEGIN To adjust header to big number of processes - max_len_header = 7 - for labelx in label_xtics: - if len(labelx) > max_len_header: - max_len_header = len(labelx) - - # END To adjust header to big number of processes - list_data = [] - for index, rows in df.iterrows(): - list_temp = [] - for value in list(rows)[1:]: - if float(value) == 0.0: - list_temp.append(np.nan) - else: - list_temp.append(float(value)) - # print(list_temp) - list_data.append(list_temp) - - list_np = np.array(list_data) - - idx = metrics - cols = label_xtics - #cols = traces_procs - df = pd.DataFrame(list_np, index=idx, columns=cols) - - # min for 1 traces is x=3 for the (x,y) in figsize - size_figure_y = len(idx) * 0.40 - size_figure_x = len(cols) * 0.16 * max_len_header - plt.figure(figsize=(size_figure_x, size_figure_y)) - - ax = sns.heatmap(df, cmap='RdYlGn', linewidths=0.05, annot=True, vmin=0, vmax=100, center=75, \ - fmt='.2f', annot_kws={"size": 10}, cbar_kws={'label': 'Percentage(%)'}) - ## to align ylabels to left - plt.yticks(rotation=0, ha='left') - ax.xaxis.tick_top() - # to adjust metrics - len_pad = 0 - for metric in metrics: - if len(metric) > len_pad: - len_pad = len(metric) - - ax.yaxis.set_tick_params(pad=len_pad + 120) - plt.savefig('efficiency_table-matplot.png', bbox_inches='tight') - - -def plots_modelfactors_matplot(trace_list, trace_mode, trace_processes, trace_tasks, trace_threads, cmdl_args): - # Plotting using python - # For plotting using python, read the csv file - file_path = os.path.join(os.getcwd(), 'modelfactors.csv') - df = pd.read_csv(file_path, sep=';') - - # metrics = df['Number of processes'].tolist() - - traces_procs = list(df.keys())[1:] - - list_data = [] - for index, rows in df.iterrows(): - list_temp = [] - for value in list(rows)[1:]: - if value != 'Non-Avail' and value != "Warning!": - list_temp.append(float(value)) - elif value == 'Non-Avail' or value == "Warning!": - list_temp.append(float('nan')) - #print(list_temp) - list_data.append(list_temp) - - # To control same number of processes for the header on plots and table - same_procs = True - procs_trace_prev = trace_processes[trace_list[0]] - tasks_trace_prev = trace_tasks[trace_list[0]] - threads_trace_prev = trace_threads[trace_list[0]] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_tasks[trace] - if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ - and threads_trace_prev == threads: - same_procs *= True - else: - same_procs *= False - - # Set limit for projection - if cmdl_args.limit: - limit = cmdl_args.limit - else: - limit = str(trace_processes[trace_list[len(trace_list)-1]]) - - limit_min = trace_processes[trace_list[0]] - - # To xticks label - label_xtics = [] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_tasks[trace] - if int(limit) == int(limit_min) and same_procs: - s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' - elif int(limit) == int(limit_min) and not same_procs: - s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' \ - + str(threads) + ')' - else: - s_xtics = str(trace_processes[trace]) - label_xtics.append(s_xtics) - - ### Plot: Global Metrics - # print(list_data) - plt.figure() - max_global = max([max(list_data[0]), max(list_data[1]), max(list_data[2]), max(list_data[3]), max(list_data[6])]) - plt.plot(traces_procs, list_data[0], 'o-', color='black', label='Global Efficiency') - plt.plot(traces_procs, list_data[1], 's--', color='magenta', label='Parallel Efficiency') - plt.plot(traces_procs, list_data[2], 'X:', color='red', linewidth=2, label='Load Balance') - plt.plot(traces_procs, list_data[3], 'x-.', color='green', label='Communication efficiency') - plt.plot(traces_procs, list_data[6], 'v--', color='blue', label='Computation scalability') - plt.xlabel("Number of Processes") - plt.ylabel("Efficiency (%)") - plt.xticks(tuple(traces_procs), tuple(label_xtics)) - # print(max_global) - if float(max_global) < 100: - max_global = 100 - - plt.ylim(0, float(max_global)+5) - plt.legend() - plt.savefig('modelfactors-matplot.png', bbox_inches='tight') - - # Plot: Comm Metrics - if trace_mode[trace] == 'Detailed+MPI': - plt.figure() - max_comm = max([max(list_data[3]), max(list_data[4]), max(list_data[5])]) - plt.plot(traces_procs, list_data[3], 's-', color='green', label='Communication efficiency') - plt.plot(traces_procs, list_data[4], 'h--', color='gold', label='Serialization efficiency') - plt.plot(traces_procs, list_data[5], 'x:', color='tomato', label='Transfer efficiency') - plt.xlabel("Number of Processes") - plt.ylabel("Efficiency (%)") - plt.xticks(tuple(traces_procs), tuple(label_xtics)) - if float(max_comm) < 100: - max_comm = 100 - - plt.ylim(0, float(max_comm)+5) - plt.legend() - plt.savefig('modelfactors-comm-matplot.png', bbox_inches='tight') - - ### Plot: Scale Metrics - if trace_mode[trace][:5] != 'Burst': - plt.figure() - max_scale = max([max(list_data[6]), max(list_data[7]), max(list_data[8]), max(list_data[9])]) - plt.plot(traces_procs, list_data[6], 'v-', color='blue', label='Computation scalability') - plt.plot(traces_procs, list_data[7], 'v--', color='skyblue', label='IPC scalability') - plt.plot(traces_procs, list_data[8], 'v:', color='gray', label='Instruction scalability') - plt.plot(traces_procs, list_data[9], 'v-.', color='darkviolet', label='Frequency scalability') - plt.xlabel("Number of Processes") - plt.ylabel("Efficiency (%)") - plt.xticks(tuple(traces_procs), tuple(label_xtics)) - if float(max_scale) < 100: - max_scale = 100 - - plt.ylim(0, float(max_scale)+5) - plt.legend() - plt.savefig('modelfactors-scale-matplot.png', bbox_inches='tight') - - -def plots_speedup_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args): - # Plotting using python - # For plotting using python, read the csv file - file_path = os.path.join(os.getcwd(), 'other_metrics.csv') - df = pd.read_csv(file_path, sep=';') - - traces_procs = list(df.keys())[1:] - - list_data = [] - for index, rows in df.iterrows(): - list_temp = [] - for value in list(rows)[1:]: - if value != 'Non-Avail': - list_temp.append(float(value)) - elif value == 'Non-Avail': - list_temp.append(float('nan')) - # print(list_temp) - list_data.append(list_temp) - - # To control same number of processes for the header on plots and table - same_procs = True - procs_trace_prev = trace_processes[trace_list[0]] - tasks_trace_prev = trace_tasks[trace_list[0]] - threads_trace_prev = trace_threads[trace_list[0]] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ - and threads_trace_prev == threads: - same_procs *= True - else: - same_procs *= False - - # Set limit for projection - if cmdl_args.limit: - limit = cmdl_args.limit - else: - limit = str(trace_processes[trace_list[len(trace_list) - 1]]) - - limit_min = trace_processes[trace_list[0]] - - # proc_ratio to ideal speedup - proc_ratio = [] - for index, trace in enumerate(trace_list): - proc_ratio.append(trace_processes[trace]/trace_processes[trace_list[0]]) - # print(proc_ratio) - - # To xticks label - label_xtics = [] - for index, trace in enumerate(trace_list): - tasks = trace_tasks[trace] - threads = trace_threads[trace] - if int(limit) == int(limit_min) and same_procs: - s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' - elif int(limit) == int(limit_min) and not same_procs: - s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' \ - + str(threads) + ')' - else: - s_xtics = str(trace_processes[trace]) - label_xtics.append(s_xtics) - - ### Plot: SpeedUp - # print(list_data) - - int_traces_procs = [] - prev_procs = int(float(traces_procs[0])) - int_traces_procs.append(int(float(traces_procs[0]))) - count_rep = 0 - for procs in traces_procs[1:]: - if prev_procs == int(float(procs)): - count_rep += 1 - int_traces_procs.append(int(float(procs)) + (2 * count_rep)) - prev_procs = int(float(procs)) - else: - int_traces_procs.append(int(float(procs))) - prev_procs = int(float(procs)) - count_rep = 0 - - - plt.figure() - for x, y in zip(int_traces_procs, list_data[2]): - label = "{:.2f}".format(y) - plt.annotate(label, (x, y), textcoords="offset points", xytext=(0, 10), ha='center') - plt.plot(int_traces_procs, list_data[2], 'o-', color='blue', label='measured') - plt.plot(int_traces_procs, proc_ratio, 'o-', color='black', label='ideal') - plt.xlabel("Number of Processes") - plt.ylabel("SpeedUp") - plt.xticks(tuple(int_traces_procs), tuple(label_xtics)) - #plt.yscale('log') - plt.legend() - # plt.xlim(0, ) - plt.ylim(0, ) - plt.savefig('speedup-matplot.png', bbox_inches='tight') - - ### Plot: Efficiency - # print(list_data) - plt.figure() - for x, y in zip(int_traces_procs, list_data[1]): - label = "{:.2f}".format(y) - plt.annotate(label, (x, y), textcoords="offset points", xytext=(0, 10), ha='center') - - plt.plot(int_traces_procs, list_data[1], 'o-', color='blue', label='measured') - plt.axhline(y=1, color='black', linestyle='-', label='ideal') - plt.xlabel("Number of Processes") - plt.ylabel("Efficiency") - plt.xticks(tuple(int_traces_procs), tuple(label_xtics)) - # plt.yscale('log') - max_y = max(list_data[1]) - if max_y < 1.1: - max_y = 1.1 - plt.ylim(0,max_y+0.1) - # plt.xlim(0, ) - plt.legend() - plt.savefig('efficiency-matplot.png', bbox_inches='tight') \ No newline at end of file diff --git a/tracemetadata.py b/tracemetadata.py deleted file mode 100644 index de85bd4..0000000 --- a/tracemetadata.py +++ /dev/null @@ -1,376 +0,0 @@ -#!/usr/bin/env python3 - -"""Functions to extract metadata information from each trace.""" - -from __future__ import print_function, division -import os -import time -import sys -import math -import re -import fnmatch -import mmap -import gzip -import multiprocessing -import threading -from multiprocessing import Process - - -def get_traces_from_args(cmdl_args): - """Filters the given list to extract traces, i.e. matching *.prv and sorts - the traces in ascending order based on the number of processes in the trace. - Excludes all files other than *.prv and ignores also simulated traces from - this script, i.e. *.sim.prv - Returns list of trace paths and dictionary with the number of processes. - """ - - def get_processes(prv_file): - return trace_processes[prv_file], trace_tasks[prv_file], trace_threads[prv_file] - - trace_list = [x for x in cmdl_args.trace_list if (fnmatch.fnmatch(x, '*.prv') or fnmatch.fnmatch(x, '*.prv.gz')) - if not fnmatch.fnmatch(x, '*.sim.prv')] - if not trace_list: - print('==Error== could not find any traces matching "', ' '.join(cmdl_args.trace_list)) - sys.exit(1) - - trace_processes = dict() - trace_tasks = dict() - trace_threads = dict() - trace_mode = dict() - trace_task_per_node = dict() - - trace_list_temp = [] - trace_list_removed = [] - for trace in trace_list: - if float(os.path.getsize(trace)/1024/1024) < float(cmdl_args.max_trace_size): - trace_list_temp.append(trace) - else: - trace_list_removed.append(trace) - - if len(trace_list_temp) < 1: - print('==Error== All traces exceed the maximum size (', cmdl_args.max_trace_size, 'MiB)') - for trace_upper in trace_list_removed: - print(trace_upper) - sys.exit(1) - - print("Running modelfactors.py for the following traces list:") - trace_list = trace_list_temp - for trace in trace_list: - print(trace) - - if len(trace_list_removed) > 0: - print("\nFollowing traces were excluded to be analyzed (size >", cmdl_args.max_trace_size, "MiB): ") - for trace in trace_list_removed: - print(trace) - - print('\nExtracting metadata from the traces list.') - # This part could be parallelized - # t1 = time.perf_counter() - for trace in trace_list: - trace_processes[trace], trace_tasks[trace], trace_threads[trace] = get_num_processes(trace) - trace_list = sorted(trace_list, key=get_processes) - - t1 = time.perf_counter() - jobs = [] - manager = multiprocessing.Manager() - trace_mode = manager.dict() - for trace in trace_list: - p_act = Process(target=get_trace_mode, args=(trace, cmdl_args, trace_mode)) - jobs.append(p_act) - p_act.start() - - for p in jobs: - p.join() - - t2 = time.perf_counter() - - print('Successfully Metadata Extraction in {0:.1f} seconds.\n'.format(t2 - t1)) - - for trace in trace_list: - trace_task_per_node[trace] = get_task_per_node(trace) - - print("Starting Analysis for the following sorted traces list:") - print_overview(trace_list, trace_processes, trace_tasks, trace_threads, trace_mode, trace_task_per_node) - return trace_list, trace_processes, trace_tasks, trace_threads, trace_task_per_node, trace_mode - - -def get_num_processes(prv_file): - """Gets the number of processes in a trace from the according .row file. - Please note: return value needs to be integer because this function is also - used as sorting key. - """ - if prv_file[-4:] == ".prv": - tracefile = open(prv_file) - for line in tracefile: - header_trace = line.split('_') - break - tracefile.close() - - if prv_file[-7:] == ".prv.gz": - with gzip.open(prv_file, 'rt') as f: - for line in f: - if "#Paraver" in line: - header_trace = line.split('_') - break - f.close() - - - #print(header_trace) - header_to_print = header_trace[1].split(':')[3].split('(') - tasks = header_to_print[0] - threads = header_to_print[1] - list_procspernode = header_trace[1].split('(')[2].split(')')[0].split(',') - #print(list_procspernode) - total_procs = 0 - for proc_node in list_procspernode: - total_procs += int(proc_node.split(':')[0]) - - return int(total_procs), int(tasks), int(threads) - - -def get_tasks_threads(prv_file): - """Gets the tasks and threads from the .prv file. - """ - if prv_file[-4:] == ".prv": - tracefile = open(prv_file) - for line in tracefile: - header_trace = line.split('_') - break - tracefile.close() - - if prv_file[-7:] == ".prv.gz": - with gzip.open(prv_file, 'rt') as f: - for line in f: - if "#Paraver" in line: - header_trace = line.split('_') - break - f.close() - header_to_print = header_trace[1].split(':')[3].split('(') - tasks = header_to_print[0] - threads = header_to_print[1] - return int(tasks), int(threads) - - -def get_task_per_node(prv_file): - """Gets the number of processes and nodes in a trace from the according .row file. - """ - row_file = True - - if prv_file[-4:] == ".prv": - if os.path.exists(prv_file[:-4] + '.row'): - tracefile = open(prv_file[:-4] + '.row') - else: - tracefile = prv_file[:-4] - row_file = False - elif prv_file[-7:] == ".prv.gz": - if os.path.exists(prv_file[:-7] + '.row'): - tracefile = open(prv_file[:-7] + '.row') - else: - tracefile = prv_file[:-7] - row_file = False - - tasks = 0 - nodes = 1 - if row_file: - for line in tracefile: - if "LEVEL CPU SIZE" in line: - tasks = int(line[15:]) - if "LEVEL NODE SIZE" in line: - nodes = int(line[15:]) - tracefile.close() - task_nodes = math.ceil(int(tasks) / int(nodes)) - else: - if prv_file[-4:] == ".prv": - tracefile = open(prv_file) - for line in tracefile: - header_trace = line.split('_') - break - tracefile.close() - - if prv_file[-7:] == ".prv.gz": - with gzip.open(prv_file, 'rt') as f: - for line in f: - if "#Paraver" in line: - header_trace = line.split('_') - break - f.close() - task_nodes = int(header_trace[1].split(':')[1].split('(')[1].replace(')','').split(',')[0]) - - return int(task_nodes) - - -def get_trace_mode(prv_file, cmdl_args, trace_mode): - """Gets the trace mode by detecting the event 40000018:2 in .prv file - to detect the Burst mode trace in another case is Detailed mode. - 50000001 for MPI, 60000001 for OpenMP, 61000000 for pthreads, 63000001 for CUDA - """ - mode_trace = '' - burst = 0 - pcf_file = True - if prv_file[-4:] == ".prv": - file_pcf = prv_file[:-4] + '.pcf' - tracefile = open(prv_file) - for line in tracefile: - if "40000018:2" in line: - burst = 1 - break - tracefile.close() - if prv_file[-7:] == ".prv.gz": - file_pcf = prv_file[:-7] + '.pcf' - with gzip.open(prv_file, 'rt') as f: - for line in f: - if "40000018:2" in line: - burst = 1 - break - f.close() - - if burst == 1: - mode_trace = 'Burst' - else: - mode_trace = 'Detailed' - - if os.path.exists(file_pcf) and cmdl_args.trace_mode_detection == 'pcf': - with open(file_pcf, 'rb', 0) as file, \ - mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) as s: - if s.find(b' 500000') != -1: - mode_trace += '+MPI' - if s.find(b' 610000') != -1: - mode_trace += '+Pthreads' - elif s.find(b' 60000') != -1: - if not s.find(b' 60000019') == s.find(b' 60000'): - mode_trace += '+OpenMP' - if s.find(b' 630000') != -1 or s.find(b' 631000') != -1 or s.find(b' 632000') != -1: - mode_trace += '+CUDA' - if s.find(b' 9200001') != -1: - mode_trace += '+OmpSs' - if s.find(b' 642000') != -1 or s.find(b' 6400001') != -1 or s.find(b' 641000') != -1: - mode_trace += '+OpenCL' - else: - if s.find(b' 610000') != -1: - mode_trace += '+Pthreads' - elif s.find(b' 60000') != -1: - if not s.find(b' 60000019') == s.find(b' 60000'): - mode_trace += '+OpenMP' - if s.find(b' 630000') != -1 or s.find(b' 631000') != -1 or s.find(b' 632000') != -1: - mode_trace += '+CUDA' - if s.find(b' 9200001') != -1: - mode_trace += '+OmpSs' - if s.find(b' 642000') != -1 or s.find(b' 6400001') != -1 or s.find(b' 641000') != -1: - mode_trace += '+OpenCL' - file.close() - else: - count_mpi = 0 - count_omp = 0 - count_pthreads = 0 - count_cuda = 0 - count_ompss = 0 - count_opencl = 0 - if prv_file[-4:] == ".prv": - with open(prv_file, 'rb', 0) as file, \ - mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_COPY) as s: - # 2:cpu_id:appl_id:task_id:thread_id:time:event_type:event_value - mpi = re.compile(rb'\n2:\w+:\w+:[1-4]:1:\w+:50000\w\w\w:') - omp = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:60000018:') - cuda = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:63\w\w\w\w\w\w:') - pthreads = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:610000\w\w:') - ompss = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:9200001:') - opencl = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:64\w\w\w\w\w\w:') - if mpi.search(s): - count_mpi = 1 - mpi_trace = '+MPI' - if omp.search(s): - count_omp = 1 - omp_trace = '+OpenMP' - elif cuda.search(s): - count_cuda = 1 - cuda_trace = '+CUDA' - elif pthreads.search(s): - count_pthreads = 1 - pthreads_trace = '+Pthreads' - elif ompss.search(s): - count_ompss = 1 - ompss_trace = '+OmpSs' - elif opencl.search(s): - count_opencl = 1 - opencl_trace = '+OpenCL' - file.close() - elif prv_file[-7:] == ".prv.gz": - handle = open(prv_file, "rb") - mapped = mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) - gzfile = gzip.GzipFile(mode="r", fileobj=mapped) - - # 2:cpu_id:appl_id:task_id:thread_id:time:event_type:event_value - mpi = re.compile(rb'\n2:\w+:\w+:[1-4]:1:\w+:50000\w\w\w:') - omp = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:60000018:') - cuda = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:63\w\w\w\w\w\w:') - pthreads = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:610000\w\w:') - ompss = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:9200001:') - opencl = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:64\w\w\w\w\w\w:') - s = gzfile.read() - if mpi.search(s): - count_mpi = 1 - mpi_trace = '+MPI' - if omp.search(s): - count_omp = 1 - omp_trace = '+OpenMP' - elif cuda.search(s): - count_cuda = 1 - cuda_trace = '+CUDA' - elif pthreads.search(s): - count_pthreads = 1 - pthreads_trace = '+Pthreads' - elif ompss.search(s): - count_ompss = 1 - ompss_trace = '+OmpSs' - elif opencl.search(s): - count_opencl = 1 - opencl_trace = '+OpenCL' - - handle.close() - if count_mpi > 0: - mode_trace += mpi_trace - if count_omp > 0: - mode_trace += omp_trace - if count_pthreads > 0: - mode_trace += pthreads_trace - if count_ompss > 0: - mode_trace += ompss_trace - if count_cuda > 0: - mode_trace += cuda_trace - if count_opencl > 0: - mode_trace += opencl_trace - - trace_mode[prv_file] = mode_trace - #return mode_trace - - -def human_readable(size, precision=1): - """Converts a given size in bytes to the value in human readable form.""" - suffixes = ['B', 'KB', 'MB', 'GB', 'TB'] - suffixIndex = 0 - while size > 1024 and suffixIndex < 4: - suffixIndex += 1 - size = size / 1024.0 - return "%.*f%s" % (precision, size, suffixes[suffixIndex]) - - -def print_overview(trace_list, trace_processes, trace_tasks, trace_threads, trace_mode, trace_task_per_node): - """Prints an overview of the traces that will be processed.""" - #print('Running', os.path.basename(__file__), 'for the following traces:') - - file_path = os.path.join(os.getcwd(), 'traces_metadata.txt') - with open(file_path, 'w') as output: - for index, trace in enumerate(trace_list): - line = '[' + str(index+1) + '] ' + trace - - line += ', ' + str(trace_processes[trace]) \ - + '(' + str(trace_tasks[trace]) + 'x' + str(trace_threads[trace]) + ')' + ' processes' - line += ', ' + str(trace_task_per_node[trace]) + ' tasks per node' - line += ', ' + human_readable(os.path.getsize(trace)) - line += ', ' + str(trace_mode[trace]) + ' mode' - print(line) - output.write(line + '\n') - - print('======== Output Files: Traces metadata ========') - print('Traces metadata written to ' + file_path) - print('') \ No newline at end of file diff --git a/utils.py b/utils.py deleted file mode 100644 index e457c32..0000000 --- a/utils.py +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env python3 - -"""modelfactors.py utils.""" - -from __future__ import print_function, division -import os -import sys -import subprocess -import tempfile -import argparse -import shutil - - -try: - import scipy.optimize -except ImportError: - print('==Error== Could not import SciPy. Please make sure you have installed SciPy.') - -try: - import numpy -except ImportError: - print('==Error== Could not import NumPy. Please make sure you have installed NumPy.') - -try: - import pandas as pd -except ImportError: - print('==Error== Could not import pandas. Please make sure you have installed pandas.') -try: - import seaborn as sns -except ImportError: - print('==Error== Could not import seaborn. Please make sure you have installed seaborn.') - -try: - import matplotlib.pyplot as plt -except ImportError: - print('==Error== Could not import matplotlib. Please make sure you have installed matplotlib.') - - -__author__ = "Sandra Mendez" -__copyright__ = "Copyright 2019, Barcelona Supercomputing Center (BSC)" -__version_major__ = 0 -__version_minor__ = 3 -__version_micro__ = 7 -__version__ = str(__version_major__) + "." + str(__version_minor__) + "." + str(__version_micro__) - - -def parse_arguments(): - """Parses the command line arguments. - Currently the script only accepts one parameter list, which is the list of - traces that are processed. This can be a regex and only valid trace files - are kept at the end. - """ - parser = argparse.ArgumentParser(description='Generates performance metrics from a set of Paraver traces.') - parser.add_argument('trace_list', nargs='*', - help='list of traces to process. Accepts wild cards and automatically filters for ' - 'valid traces'), - parser.add_argument("-m", "--metrics", choices=['simple', 'hybrid'], default='hybrid', - help='select the kind of efficiency metrics (single parallelism or hybrid, default: hybrid)') - parser.add_argument("-v", "--version", action='version', version='%(prog)s {version}'.format(version=__version__)) - parser.add_argument("-d", "--debug", help="increase output verbosity to debug level", action="store_true") - parser.add_argument("-s", "--scaling", - help="define whether the measurements are weak or strong scaling (default: auto)", - choices=['weak', 'strong', 'auto'], default='auto') - parser.add_argument('--limit', help='limit number of cores for the plots ' - '(default: max processes of the trace list )') - parser.add_argument("-ms", "--max_trace_size", help='set the maximum trace size in MiB allowed.' - ' (default: 1024 MiB )', default=1024.0) - parser.add_argument("-tmd", "--trace_mode_detection", choices=['pcf', 'prv'], default='pcf', - help='select .prv or .pcf file for trace mode detection. ' - 'For customized traces, i.e. cut, filtered and so on, you must select' - ' the .prv file. (default: pcf)') - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - - cmdl_args = parser.parse_args() - - if cmdl_args.debug: - print('==DEBUG== Running in debug mode.') - - return cmdl_args - - -def which(cmd): - """Returns path to cmd in path or None if not available.""" - for path in os.environ["PATH"].split(os.pathsep): - path = path.strip('"') - cmd_path = os.path.join(path, cmd) - if os.path.isfile(cmd_path) and os.access(cmd_path, os.X_OK): - return cmd_path - - return None - - -def check_installation(cmdl_args): - """Check if Dimemas and paramedir are in the path.""" - - if not which('Dimemas'): - print('Could not find Dimemas. Please make sure Dimemas is correctly installed and in the path.') - sys.exit(1) - if not which('paramedir'): - print('Could not find paramedir. Please make sure Paraver is correctly installed and in the path.') - sys.exit(1) - - if not which('python3'): - print('==> WARNING!!! It requires python version 3 or higher for a full functionality.') - option_user = input("Do you want to proceed with the analysis? (Yes/No)[Yes]: ").upper() - if option_user == 'NO': - sys.exit(1) - - if cmdl_args.debug: - print('==DEBUG== Using', __file__, __version__) - print('==DEBUG== Using', sys.executable, ".".join(map(str, sys.version_info[:3]))) - - try: - print('==DEBUG== Using', 'SciPy', scipy.__version__) - except NameError: - print('==DEBUG== SciPy not installed.') - - try: - print('==DEBUG== Using', 'NumPy', numpy.__version__) - except NameError: - print('==DEBUG== NumPy not installed.') - - print('==DEBUG== Using', which('Dimemas')) - print('==DEBUG== Using', which('paramedir')) - print('') - - return - - -def run_command(cmd, cmdl_args): - """Runs a command and forwards the return value.""" - if cmdl_args.debug: - print('==DEBUG== Executing:', ' '.join(cmd)) - - # In debug mode, keep the output. Otherwise, redirect it to devnull. - if cmdl_args.debug: - out = tempfile.NamedTemporaryFile(suffix='.out', prefix=cmd[0] + '_', dir='./', delete=False) - err = tempfile.NamedTemporaryFile(suffix='.err', prefix=cmd[0] + '_', dir='./', delete=False) - else: - out = open(os.devnull, 'w') - err = open(os.devnull, 'w') - - return_value = subprocess.call(cmd, stdout=out, stderr=err) - - out.close - err.close - - if return_value == 0: - if cmdl_args.debug: - os.remove(out.name) - os.remove(err.name) - else: - print('==ERROR== ' + ' '.join(cmd) + ' failed with return value ' + str(return_value) + '!') - print('See ' + out.name + ' and ' + err.name + ' for more details.') - - return return_value - - -def create_temp_folder(folder_name, cmdl_args): - path_output_aux = os.getcwd() + '/' + folder_name - - if os.path.exists(path_output_aux): - shutil.rmtree(path_output_aux) - os.makedirs(path_output_aux) - return (path_output_aux) - - -def move_files(path_source, path_dest, cmdl_args): - """Wraps os.remove with a try clause.""" - try: - shutil.move(path_source, path_dest) - except: - if cmdl_args.debug: - print('==DEBUG== Failed to move ' + path_source + '!') - - -def remove_files(path,cmdl_args): - """Wraps os.remove with a try clause.""" - try: - os.remove(path) - except: - if cmdl_args.debug: - print('==DEBUG== Failed to remove ' + path + '!') \ No newline at end of file -- GitLab From 17d17369e270618e6878cc64b1e05e1ca0f20e8c Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 6 Sep 2022 09:55:40 +0200 Subject: [PATCH 08/40] Fixed merge error --- script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script.sh b/script.sh index b28e5b8..8f6d240 100755 --- a/script.sh +++ b/script.sh @@ -365,7 +365,7 @@ Create_metrics() echo "Creating metrics and storing theme in Metrics folder" echo - python3 Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w ".././modelfactors.py *" + python3 Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "modelfactors.py *" mv analysis."$Jobs_scheduler" Metrics cd Metrics||(echo "Error Metrics folder doesn't exists"; exit 1) state5=$("$job" --wait analysis."$Jobs_scheduler") -- GitLab From b7b8b6319a9ad92fcd4479839b4c3a263357b00a Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 6 Sep 2022 10:06:00 +0200 Subject: [PATCH 09/40] Name changes and minor comprovations --- README.md | 6 +++--- script.sh => perf_metrics.bash | 27 +++++++++++++++++++++------ config.bash => perf_metrics.config | 0 3 files changed, 24 insertions(+), 9 deletions(-) rename script.sh => perf_metrics.bash (94%) mode change 100755 => 100644 rename config.bash => perf_metrics.config (100%) diff --git a/README.md b/README.md index 388dea6..2652f01 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,9 @@ Also the different modules needed to compile and execute NEMO should be loaded b # Usage * Copy all the content of this folder into the folder with the input data for NEMO -* Edit the file config.bash adding the information required. -* Execute script.sh +* Edit the file perf_metrics.config and replace the parameters values with the suited information. +* Execute perf_metrics.bash ``` -./script.sh +./perf_metrics.bash ``` * If the script executed without problems the data will be ready at the Metrics folder. diff --git a/script.sh b/perf_metrics.bash old mode 100755 new mode 100644 similarity index 94% rename from script.sh rename to perf_metrics.bash index 8f6d240..d899bcc --- a/script.sh +++ b/perf_metrics.bash @@ -5,7 +5,7 @@ main() if [ $# -gt 0 ]; then - echo "This script does not accept arguments, parameters need to be added to config.bash Aborting" + echo "This script does not accept arguments, parameters need to be added to perf_metrics.config Aborting" exit 1 fi @@ -16,8 +16,8 @@ dir=$(pwd) echo echo "Using the following configuration:" echo -source "$dir"/config.bash -grep -o '^[^#]*' config.bash +source "$dir"/perf_metrics.config +grep -o '^[^#]*' perf_metrics.config echo Init @@ -67,7 +67,7 @@ Init() Nemo_cores="${Nemo_cores:-( 48 )}" Jobs_n_cores="${Jobs_n_cores:-48}" Jobs_scheduler="${Jobs_scheduler:-"slurm"}" - time="${Jobs_time:-"0"}" + time="${Jobs_time:-0}" queue="${Jobs_queue:-""}" compile="${Compilation_compile:-"false"}" cfg="${Compilation_ref:-"ORCA2_ICE_PISCES"}" @@ -94,6 +94,21 @@ Test_arguments() echo exit 1 fi + + + #Nemo_cores contains ints? + + re='^[0-9]+$' + for core in "${Nemo_cores[@]}" + do + + if ! [[ $core =~ $re ]] ; then + echo "Error Nemo_cores has to contain integer values" + echo + exit 1 + fi + done + # cfg exists? if ! test -d "${Nemo_path}/cfgs/${cfg}"; then echo "configuration: ${cfg} doesn't exists in ${Nemo_path}/cfgs dir" @@ -293,7 +308,7 @@ Create_metrics() Job_completed "$state2" if [ $Completed == false ]; then echo "Nemo execution failed look at run.err and ocean.output for more info" - echo "Remember that the namelist files are now the default, change theme in order to fit with the input files in the dir " + echo "Remember that the namelist files copied are the default ones, change theme in order to fit with the input files in the dir " echo exit 1 else @@ -336,7 +351,7 @@ Create_metrics() state4=$("$job" --wait run_extrae."$Jobs_scheduler") Job_completed "$state4" if [ $Completed == false ]; then - echo "Nemo execution failed no traces files generated more info inside run_extrae.err" + echo "Nemo execution failed, no traces files generated more info inside run_extrae.err" echo exit 1 fi diff --git a/config.bash b/perf_metrics.config similarity index 100% rename from config.bash rename to perf_metrics.config -- GitLab From fcb49b1648105584c3db24f64324ef7103b04c0a Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 6 Sep 2022 10:43:47 +0200 Subject: [PATCH 10/40] Added bash format explanation --- perf_metrics.config | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/perf_metrics.config b/perf_metrics.config index aa4ad29..5bcf07a 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -1,3 +1,7 @@ +# Respect the bash format, no spaces between variables, the = symbol and the respective value. +# Arrays need a space after the open and before the closing parenthesism, the elements are separated by spaces. + +################################################################################# # Nemo_path: Relative path to nemo installation folder containing the cfgs and arch dirs # Nemo_cores: List of nºcores used for executing Nemo, ( 4 48 ) makes the script execute and @@ -6,10 +10,10 @@ Nemo_path="../NEMO" Nemo_cores=( 4 24 48 ) -# Jobs_n_cores: nºcores used for executing other scripts. More than 4 is not optimal -# Jobs_scheduler: Available (slurm/lsf) -# Jobs_time: Max duration of the job in min -# Jobs_queue: Queue used +# Jobs_n_cores: nºcores used for executing other scripts. +# Jobs_scheduler: Available (slurm/lsf). +# Jobs_time: Max duration of the job in min. +# Jobs_queue: Queue used. Jobs_n_cores=4 Jobs_scheduler="slurm" @@ -17,10 +21,10 @@ Jobs_time="60" Jobs_queue=debug # Compilation_compile: When false only compiles NEMO if arch file lacks the needed flags, when true always compiles NEMO. -# Compilation_ref: Reference configuration -# Compilation_arch: Architecture used (without the -arch sufix and the .fcm) -# Compilation_name: Name of the new configutation (Important to not be an existing one) -# Compilation_sub: Add or remove subcomponents +# Compilation_ref: Reference configuration. +# Compilation_arch: Architecture used (without the -arch sufix and the .fcm). +# Compilation_name: Name of the new configutation (Important to not be an existing one). +# Compilation_sub: Add or remove subcomponents. Compilation_compile="false" @@ -29,7 +33,7 @@ Compilation_arch="X64_MN4" Compilation_name="ORCA2_EXTRAE" Compilation_sub="OCE del_key 'key_si3 key_top'" -# List of modules loaded +# List of modules loaded. # Required: # - Perl interpreter # - Fortran compiler (ifort, gfortran, pgfortran, ftn, …) @@ -39,7 +43,5 @@ Compilation_sub="OCE del_key 'key_si3 key_top'" # - Paraver # - Dimemas 4.2 -devel # - Python3 -# - gnuplot -# EXTRAE BASICANALYSIS gcc/7.2.0 intel/2017.4 impi/2018.4 netcdf/4.4.1.1 hdf5/1.8.19 DIMEMAS/5.4.2-devel Modules="EXTRAE BASICANALYSIS gcc intel/2018.3 impi/2018.4 netcdf/4.4.1.1 hdf5/1.8.19 DIMEMAS/5.4.2-devel perl" -- GitLab From 6fc01d024aa722cbf722b9395f6e83a3dccb9725 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Wed, 7 Sep 2022 08:31:46 +0200 Subject: [PATCH 11/40] New variable for Nemo iterations (12) --- perf_metrics.bash | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/perf_metrics.bash b/perf_metrics.bash index d899bcc..e758f18 100644 --- a/perf_metrics.bash +++ b/perf_metrics.bash @@ -75,6 +75,7 @@ Init() name_cfg="${Compilation_name:-"ORCA2_EXTRAE"}" comp_cfg="${Compilation_sub:-""}" Modules="${Modules:-""}" + Nemo_iterations=12 } @@ -294,7 +295,7 @@ Create_metrics() #Changing iterations, big traces generate problems. - sed -i 's|nn_itend * =.*|nn_itend = 12 ! last time step (std 5475)|g' namelist_cfg + sed -i "s|nn_itend * =.*|nn_itend = $Nemo_iterations ! last time step (std 5475)|g" namelist_cfg #Generating function list in case of missing @@ -360,7 +361,7 @@ Create_metrics() mv nemo.row nemo_"$core".row echo "Cutting best iteration" echo - magiccut/./magicCut nemo_"${core}".prv 12 > cut_"$core".out 2>&1 + magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > cut_"$core".out 2>&1 if ! ls nemo_"$core".best_cut.prv; then echo "Cut failed, aborting." echo -- GitLab From 39853ec830b36af8782aa2f531bb0c3f82d23089 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Fri, 9 Sep 2022 12:55:04 +0200 Subject: [PATCH 12/40] 1+2 --- README.md | 5 +- cfgs/.directory | 4 + cfgs/2dh_BurstEfficiency.cfg | 221 +++++ cfgs/barrier-syncr-time.cfg | 80 ++ cfgs/burst_duration.cfg | 77 ++ cfgs/burst_useful.cfg | 150 ++++ cfgs/cycles.cfg | 141 +++ cfgs/dimemas.collectives | 14 + cfgs/dimemas_ideal.cfg | 159 ++++ cfgs/efficiency_table-global.gp | 31 + cfgs/efficiency_table-hybrid.gp | 30 + cfgs/efficiency_table.gp | 31 + cfgs/flushing-cycles.cfg | 148 +++ cfgs/flushing-inst.cfg | 148 +++ cfgs/flushing.cfg | 76 ++ cfgs/instructions.cfg | 141 +++ cfgs/io-call-cycles.cfg | 148 +++ cfgs/io-call-instructions.cfg | 148 +++ cfgs/io-call-reverse.cfg | 76 ++ cfgs/modelfactors-all.gp | 36 + cfgs/modelfactors-comm.gp | 26 + cfgs/modelfactors-hybrid.gp | 37 + cfgs/modelfactors-mpi-hybrid.gp | 32 + cfgs/modelfactors-onlydata.gp | 31 + cfgs/modelfactors-scale.gp | 27 + cfgs/mpi-call-outside.cfg | 77 ++ cfgs/mpi-io-cycles.cfg | 149 +++ cfgs/mpi-io-instructions.cfg | 149 +++ cfgs/mpi-io-reverse.cfg | 77 ++ cfgs/mpi-io.cfg | 77 ++ cfgs/mpi-master-thread.cfg | 79 ++ cfgs/runtime.cfg | 72 ++ cfgs/runtime_app.cfg | 74 ++ cfgs/time_computing.cfg | 114 +++ cfgs/timings.cfg | 72 ++ hybridmetrics.py | 1496 +++++++++++++++++++++++++++++++ modelfactors.py | 192 ++++ perf_metrics.bash | 449 ++++++---- perf_metrics.config | 12 +- plots.py | 738 +++++++++++++++ rawdata.py | 857 ++++++++++++++++++ simplemetrics.py | 1249 ++++++++++++++++++++++++++ trace.sh | 1 + tracemetadata.py | 376 ++++++++ utils.py | 186 ++++ 45 files changed, 8339 insertions(+), 174 deletions(-) create mode 100644 cfgs/.directory create mode 100644 cfgs/2dh_BurstEfficiency.cfg create mode 100644 cfgs/barrier-syncr-time.cfg create mode 100644 cfgs/burst_duration.cfg create mode 100644 cfgs/burst_useful.cfg create mode 100644 cfgs/cycles.cfg create mode 100644 cfgs/dimemas.collectives create mode 100644 cfgs/dimemas_ideal.cfg create mode 100644 cfgs/efficiency_table-global.gp create mode 100644 cfgs/efficiency_table-hybrid.gp create mode 100644 cfgs/efficiency_table.gp create mode 100644 cfgs/flushing-cycles.cfg create mode 100644 cfgs/flushing-inst.cfg create mode 100644 cfgs/flushing.cfg create mode 100644 cfgs/instructions.cfg create mode 100644 cfgs/io-call-cycles.cfg create mode 100644 cfgs/io-call-instructions.cfg create mode 100644 cfgs/io-call-reverse.cfg create mode 100644 cfgs/modelfactors-all.gp create mode 100644 cfgs/modelfactors-comm.gp create mode 100644 cfgs/modelfactors-hybrid.gp create mode 100644 cfgs/modelfactors-mpi-hybrid.gp create mode 100644 cfgs/modelfactors-onlydata.gp create mode 100644 cfgs/modelfactors-scale.gp create mode 100644 cfgs/mpi-call-outside.cfg create mode 100644 cfgs/mpi-io-cycles.cfg create mode 100644 cfgs/mpi-io-instructions.cfg create mode 100644 cfgs/mpi-io-reverse.cfg create mode 100644 cfgs/mpi-io.cfg create mode 100644 cfgs/mpi-master-thread.cfg create mode 100644 cfgs/runtime.cfg create mode 100644 cfgs/runtime_app.cfg create mode 100644 cfgs/time_computing.cfg create mode 100644 cfgs/timings.cfg create mode 100644 hybridmetrics.py create mode 100755 modelfactors.py mode change 100644 => 100755 perf_metrics.bash create mode 100644 plots.py create mode 100644 rawdata.py create mode 100644 simplemetrics.py create mode 100644 tracemetadata.py create mode 100644 utils.py diff --git a/README.md b/README.md index 2652f01..beb7052 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # Nemo modelfactors -Script used to get important metrics from NEMO +Script used to get important metrics from NEMO. + +The script obtains data from a single nemo timestep, so keep in mind that the results of this basic analysis +will only be accurate on longer simulations where the inicialitzation and finalitzation are close to 0% of the time. # Installation and requeriments diff --git a/cfgs/.directory b/cfgs/.directory new file mode 100644 index 0000000..5ff3922 --- /dev/null +++ b/cfgs/.directory @@ -0,0 +1,4 @@ +[Dolphin] +Timestamp=2019,6,19,17,11,21 +Version=4 +ViewMode=1 diff --git a/cfgs/2dh_BurstEfficiency.cfg b/cfgs/2dh_BurstEfficiency.cfg new file mode 100644 index 0000000..9429c78 --- /dev/null +++ b/cfgs/2dh_BurstEfficiency.cfg @@ -0,0 +1,221 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 5 +ConfigFile.BeginDescription +ConfigFile.EndDescription + +################################################################################ +< NEW DISPLAYING WINDOW Burst Region duration.c1.c1 > +################################################################################ +window_name Burst Region duration.c1.c1 +window_type single +window_id 1 +window_position_x 484 +window_position_y 126 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 39435569.853000000119 +window_minimum_y 1.377000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Int. Between Evt}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 54000006 +window_filter_module evt_type_label 1 "Elapsed time in MPI" + +################################################################################ +< NEW DISPLAYING WINDOW MPI ellapsed time In burst Region.c3.c1 > +################################################################################ +window_name MPI ellapsed time In burst Region.c3.c1 +window_type single +window_id 2 +window_position_x 502 +window_position_y 144 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 39435569.853000000119 +window_minimum_y 1.377000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Div}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module compose_thread Div { 1, { 1 1000.000000000000 } } +window_filter_module evt_type 1 54000006 +window_filter_module evt_type_label 1 "Elapsed time in MPI" + +################################################################################ +< NEW DISPLAYING WINDOW Computing time in Region.c1 > +################################################################################ +window_name Computing time in Region.c1 +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation substract +window_identifiers 1 2 +window_position_x 538 +window_position_y 180 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 39435569.853000000119 +window_minimum_y 1.377000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 0 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +################################################################################ +< NEW DISPLAYING WINDOW Burst Region duration.c2 > +################################################################################ +window_name Burst Region duration.c2 +window_type single +window_id 4 +window_position_x 556 +window_position_y 198 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 39435569.853000000119 +window_minimum_y 1.377000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Int. Between Evt}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 54000006 +window_filter_module evt_type_label 1 "Elapsed time in MPI" + +################################################################################ +< NEW DISPLAYING WINDOW Burst efficiency > +################################################################################ +window_name Burst efficiency +window_type composed +window_id 5 +window_factors 1.000000000000 1.000000000000 +window_operation divide +window_identifiers 3 4 +window_position_x 475 +window_position_y 145 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 1.100000000000 +window_minimum_y 0.000000004381 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 3 +window_drawmode_rows 3 +window_pixel_size 1 +window_labels_to_draw 0 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: 2DH - efficiency +Analyzer2D.X: 130 +Analyzer2D.Y: 233 +Analyzer2D.Width: 365 +Analyzer2D.Height: 218 +Analyzer2D.ControlWindow: 5 +Analyzer2D.DataWindow: 5 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Average value +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Enabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1.000000000000 +Analyzer2D.Delta: 2.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 0.142980245071 +Analyzer2D.MaximumGradient: 0.184636349192 +Analyzer2D.PixelSize: 1 +Analyzer2D.CodeColor: False +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/barrier-syncr-time.cfg b/cfgs/barrier-syncr-time.cfg new file mode 100644 index 0000000..ee7b63d --- /dev/null +++ b/cfgs/barrier-syncr-time.cfg @@ -0,0 +1,80 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW MPI call.SerAndTransfer > +################################################################################ +window_name MPI call.SerAndTransfer +window_type single +window_id 1 +window_position_x 3024 +window_position_y 486 +window_width 729 +window_height 320 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 70.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max true +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module task Thread i { 1, { 1 0.000000000000 } } +window_filter_module evt_type 2 50000001 50000002 +window_filter_module evt_type_label 2 "MPI Point-to-point" "MPI Collective Comm" +window_filter_module evt_value 3 5.000000000000 6.000000000000 8.000000000000 +window_filter_module evt_value_label 3 "MPI_Wait" "MPI_Waitall" "MPI_Barrier" +window_synchronize 1 + +< NEW ANALYZER2D > +Analyzer2D.Name: MPI-Ser-Transfer +Analyzer2D.X: 2810 +Analyzer2D.Y: 48 +Analyzer2D.Width: 868 +Analyzer2D.Height: 491 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: True +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: True +Analyzer2D.Minimum: 3.000000000000 +Analyzer2D.Maximum: 10.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 52.305000000000 +Analyzer2D.MaximumGradient: 76291.281000000003 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/burst_duration.cfg b/cfgs/burst_duration.cfg new file mode 100644 index 0000000..6958790 --- /dev/null +++ b/cfgs/burst_duration.cfg @@ -0,0 +1,77 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW burst_duration.c1 > +################################################################################ +window_name burst_duration.c1 +window_type single +window_id 1 +window_position_x 596 +window_position_y 238 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 39435569.853000000119 +window_minimum_y 1.377000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Int. Between Evt}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 54000006 +window_filter_module evt_type_label 1 "Elapsed time in MPI" + +< NEW ANALYZER2D > +Analyzer2D.Name: burst_duration +Analyzer2D.X: 2297 +Analyzer2D.Y: 30 +Analyzer2D.Width: 391 +Analyzer2D.Height: 472 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 18.918000000000 +Analyzer2D.Maximum: 191006.618399999978 +Analyzer2D.Delta: 1000000000000000.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 952444.935999999987 +Analyzer2D.MaximumGradient: 959765.031999999890 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/burst_useful.cfg b/cfgs/burst_useful.cfg new file mode 100644 index 0000000..ac54618 --- /dev/null +++ b/cfgs/burst_useful.cfg @@ -0,0 +1,150 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW burst_duration > +################################################################################ +window_name burst_duration +window_type single +window_id 1 +window_position_x 567 +window_position_y 209 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 57174.218999999997 +window_minimum_y 24.956000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Int. Between Evt}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 54000006 +window_filter_module evt_type_label 1 "Elapsed time in MPI" + +################################################################################ +< NEW DISPLAYING WINDOW burst_MPI_ellapsed_time > +################################################################################ +window_name burst_MPI_ellapsed_time +window_type single +window_id 2 +window_position_x 567 +window_position_y 209 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 12582.013999999999 +window_minimum_y 21.416000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Div}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module compose_thread Div { 1, { 1 1000.000000000000 } } +window_filter_module evt_type 1 54000006 +window_filter_module evt_type_label 1 "Elapsed time in MPI" + +################################################################################ +< NEW DISPLAYING WINDOW useful_burst > +################################################################################ +window_name useful_burst +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation substract +window_identifiers 1 2 +window_position_x 1960 +window_position_y 174 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 57174.218999999997 +window_minimum_y -4162.623000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 0 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: useful_burst +Analyzer2D.X: 2327 +Analyzer2D.Y: 31 +Analyzer2D.Width: 841 +Analyzer2D.Height: 365 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Vertical +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: True +Analyzer2D.Minimum: -5122.957000000000 +Analyzer2D.Maximum: 142551.129350000003 +Analyzer2D.Delta: 738.370431750000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: -5122.957000000000 +Analyzer2D.MaximumGradient: 16572593.654999990016 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/cycles.cfg b/cfgs/cycles.cfg new file mode 100644 index 0000000..7d2091e --- /dev/null +++ b/cfgs/cycles.cfg @@ -0,0 +1,141 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW cycles.c1.c1 > +################################################################################ +window_name cycles.c1.c1 +window_type single +window_id 1 +window_position_x 300 +window_position_y 23 +window_width 600 +window_height 114 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 194620048057.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 42000059 + +################################################################################ +< NEW DISPLAYING WINDOW Useful.c1.c1 > +################################################################################ +window_name Useful.c1.c1 +window_type single +window_id 2 +window_position_x 323 +window_position_y 46 +window_width 600 +window_height 134 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered true +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 1.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Useful}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, =}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, All}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +################################################################################ +< NEW DISPLAYING WINDOW Useful Cycles Timeline > +################################################################################ +window_name Useful Cycles Timeline +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation product +window_identifiers 1 2 +window_position_x 391 +window_position_y 146 +window_width 767 +window_height 314 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 11047386945.000000000000 +window_minimum_y 5433.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: cycles +Analyzer2D.X: 414 +Analyzer2D.Y: 54 +Analyzer2D.Width: 556 +Analyzer2D.Height: 764 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1000000000000000000.000000000000 +Analyzer2D.Delta: 1000000000000000000.000000000000 +Analyzer2D.ComputeGradient: False +Analyzer2D.MinimumGradient: 0 +Analyzer2D.MaximumGradient: 999999999999999999 +Analyzer2D.PixelSize: 1 +Analyzer2D.CodeColor: False + diff --git a/cfgs/dimemas.collectives b/cfgs/dimemas.collectives new file mode 100644 index 0000000..3b86752 --- /dev/null +++ b/cfgs/dimemas.collectives @@ -0,0 +1,14 @@ +Machine globalop: 0 0 LOG MAX 0 MAX +Machine globalop: 0 1 LOG MAX 0 MAX +Machine globalop: 0 2 LOG MAX 0 MAX +Machine globalop: 0 3 LOG MAX 0 MAX +Machine globalop: 0 4 LOG MAX 0 MAX +Machine globalop: 0 5 LOG MAX 0 MAX +Machine globalop: 0 6 LOG MAX 0 MAX +Machine globalop: 0 7 LOG MAX 0 MAX +Machine globalop: 0 8 LIN 2MAX LIN 2MAX +Machine globalop: 0 9 LOG MAX 0 MAX +Machine globalop: 0 10 LOG 2MAX 0 MAX +Machine globalop: 0 11 LOG MAX 0 MAX +Machine globalop: 0 12 LOG MAX 0 MAX +Machine globalop: 0 13 LOG MAX 0 MAX diff --git a/cfgs/dimemas_ideal.cfg b/cfgs/dimemas_ideal.cfg new file mode 100644 index 0000000..d0f46f0 --- /dev/null +++ b/cfgs/dimemas_ideal.cfg @@ -0,0 +1,159 @@ +#DIMEMAS_CONFIGURATION + +/******************************************************************************* + * RECORDS DEFINITION - NOTES TO MANUAL EDIT OF THIS FILE + ******************************************************************************* + * + * The records here described follow the SDDF file structure. Each field must be + * sepparated by commas in the actual records. + * + * 'int' and 'double' values are expresed as is. 'char[]' values must be always + * wrapped by quotes. 'double' and 'int' arrays must be in the following format: + * + * [n] { value_1, value_2, ..., value_n } + * + * Where 'n' is the number of elements in the array + * + ******************************************************************************* + + "wide area network information" { + 1: char[] "name of the wide area network simulated" + 2: int "number of machines in wan" + 3: int "number of dedicated connections between machines in the simulated" + "system" + 4: int "function that models influence of traffic in the non dedicated" + "network." + "options: 1 EXPONENTIAl, 2 LOGARITHMIC, 3 LINEAR, 4 CONSTANT" + 5: double "maximal value of traffic in the network" + 6: double "external net bandwidth (MBps)" + 7: int "external network collective communications model" + "options: 1 CONSTANT, 2 LINEAR, 3 LOGARITHMIC" +};; + +"environment information" { + 1: char[] "machine name" + 2: int "machine ID"; + 3: char[] "architecture used to instrument" + 4: int "number of nodes on virtual machine" + 5: double "data tranfer rate between nodes (MBps)" + "0 means instantaneous communication" + 6: int "maximun number of messages on network" + "0 means no limit" + "1 means bus contention" + 7: int "internal network collective communications model" + "options: 1 CONSTANT, 2 LINEAR, 3 LOGARITHMIC" +};; + + +"node information" { + 1: int "machine ID" + 2: char[] "architecture node name" + 3: int "number of processors within node" + 4: double "relative processor speed (divisive factor)" + 5: double "latency time (s) of intra-node communications model" + 6: double "bandwidth (MBps) of intra-node communications model" + "0 means instantaneous communication" + 7: int "maximum number of concurrent messages of intra-node" + "communications model:" + "0 means no limit" + "1 means bus contention" + 8: int "input links of intra-node communications model" + 9: int "output links of intra-node communications model" + 10: double "latency time (s) of inter-node communications model" + 11: int "input links of inter-node communications model" + 12: int "input links of intra-node communications model" + 13: double "latency time (s) of inter-machines (WAN) communications model" +};; + +"multi node information" { + 1: int "machine ID" + 2: int "number of nodes with same configuration" + 3: char[] "architecture node name" + 4: int "number of processors within node" + 5: double "processor speed ratio wrt. original execution (divisive factor)" + "0 means instantaneous | negative value means fixed duration in s." + 6: double "latency time (s) of intra-node communications model" + 7: double "bandwidth (MBps) of intra-node communications model" + "0 means instantaneous communication" + 8: int "maximum number of concurrent messages of intra-node" + "communications model: " + "0 means no limit" + "1 means bus contention" + 9: int "input links of intra-node communications model" + 10: int "output links of intra-node communications model" + 11: double "latency time (s) of inter-node communications model" + 12: int "input links of inter-node communications model" + 13: int "input links of intra-node communications model" + 14: double "latency time (s) of inter-machines (WAN) communications model" +};; + +"mapping information" { + 1: char[] "application tracefile name" + 2: int "number of tasks in application" + 3: int[] "list of nodes where each application tasks is mapped" +};; + +"predefined mapping information" { + 1: char[] "application tracefile name" (OPTIONAL) + 2: char[] "predefined map identifier" + "options: FILL_NODES | TASKS_PER_NODE | INTERLEAVED" +} + +"configuration files" { + 1: char[] "scheduler definition filename" + 2: char[] "file system definition filename" + 3: char[] "communications fine tuning configuration filename" + 4: char[] "sensitivity configuration filename" +};; + +"modules information" { + 1: int "module type" + 2: int "module value" + 3: double "module speed ration wrt. original execution (divisive factor)" + "0 means instantaneous | negative value means fixed duration in s." +};; + +"file system parameters" { + 1: double "disk latency" + 2: double "disk bandwidth"; + 3: double "block size"; + 4: int "concurrent requests"; + 5: double "hit ratio"; +};; + +"dedicated connection information" { + 1: int "connection ID" + 2: int "source machine ID" + 3: int "destination machine ID" + 4: double "bandwidth of the connection (MBps)" + 5: int[] "list of tags that will use the connection" + 6: int "size of first message (bytes) to apply the comparision to use the" + "connection" + 7: char[] "size condition that should meet messages to use the connection" + "it can be <, =, > and (referent to first message size)" + 8: char[] "operation. options: & AND, | OR" + 9: int "size of second condition that should meet messages to use the" + "connection" + 10: char[] "size condition that should meet messages to use the connection" + "it can be <, =, > and its is referent to second message size" + 11: int[] "list of communicators of coll. Operations that can use the" + "connection" + 12: double "latency of dedicated connection (s)" + 13: double "latency due to distance (s)" +};; + +*******************************************************************************/ + + +"wide area network information" {"", 1, 0, 4, 0.0, 0.0, 1};; + +"environment information" {"", 0, "", REPLACE_BY_NTASKS, 0.0, 0, 1};; + +"multi node information" {0, REPLACE_BY_NTASKS, "", REPLACE_BY_CPUS_PER_NODE, 1.0, 0.0, 0.0, 0, 1, 0, 0.0, 1, 1, 0.0};; + +"predefined mapping information" {"1 TASKS_PER_NODE"};; + +"configuration files" {"", "", "REPLACE_BY_COLLECTIVES_PATH", ""};; + +"file system parameters" {0.0, 0.0, 8.0, 0, 1.0};; + diff --git a/cfgs/efficiency_table-global.gp b/cfgs/efficiency_table-global.gp new file mode 100644 index 0000000..c7a5ef6 --- /dev/null +++ b/cfgs/efficiency_table-global.gp @@ -0,0 +1,31 @@ +#!/bin/gnuplot +set output "./efficiency_table-global.png" + +#Prepare the plot size + +#REPLACE_BY_SIZE + +set datafile separator "," + +set title "" +unset key + +set format cb "%.0f%%" +set cbrange[0:100] +set tmargin 1 + + +set cbtics ('<50%%' 50, '60%%' 60, '70%%' 70, '80%%' 80, "90%%" 90, "100%%" 100) + +set palette defined (0.00 "#000000", 0.01 "#FF0000", 0.60 "#E67C73", 0.75 "#F9950A", 0.85 "#AADC32", 0.90 "#5CC863", 1.00 "#57BB8A") + + +set xtics offset 0,15 +set ytics offset -27,0 left +set yrange [] reverse + +plot \ + 'efficiency_table-global.csv' \ + matrix rowheaders columnheaders using 1:2:3 with image, \ + 'efficiency_table-global.csv' \ + matrix rowheaders columnheaders using 1:2:(sprintf("%.2f",$3) ) with labels diff --git a/cfgs/efficiency_table-hybrid.gp b/cfgs/efficiency_table-hybrid.gp new file mode 100644 index 0000000..0e726cb --- /dev/null +++ b/cfgs/efficiency_table-hybrid.gp @@ -0,0 +1,30 @@ +#!/bin/gnuplot +set output "./efficiency_table-hybrid.png" + +#Prepare the plot size + +#REPLACE_BY_SIZE + +set datafile separator "," + +set title "" +unset key + +set format cb "%.0f%%" +set cbrange[0:100] +set tmargin 1 + + +set cbtics ('<50%%' 50, '60%%' 60, '70%%' 70, '80%%' 80, "90%%" 90, "100%%" 100) + +set palette defined (0.00 "#000000", 0.01 "#FF0000", 0.60 "#E67C73", 0.75 "#F9950A", 0.85 "#AADC32", 0.90 "#5CC863", 1.00 "#57BB8A") + +set xtics offset 0,15 +set ytics offset -37,0 left +set yrange [] reverse + +plot \ + 'efficiency_table-hybrid.csv' \ + matrix rowheaders columnheaders using 1:2:3 with image, \ + 'efficiency_table-hybrid.csv' \ + matrix rowheaders columnheaders using 1:2:(sprintf("%.2f",$3) ) with labels diff --git a/cfgs/efficiency_table.gp b/cfgs/efficiency_table.gp new file mode 100644 index 0000000..cda44e1 --- /dev/null +++ b/cfgs/efficiency_table.gp @@ -0,0 +1,31 @@ +#!/bin/gnuplot +set output "./efficiency_table.png" + +#Prepare the plot size + +#REPLACE_BY_SIZE + +set datafile separator "," + +set title "" +unset key + +set format cb "%.0f%%" +set cbrange[0:100] +set tmargin 1 + + +set cbtics ('<50%%' 50, '60%%' 60, '70%%' 70, '80%%' 80, "90%%" 90, "100%%" 100) + +set palette defined (0.00 "#000000", 0.01 "#FF0000", 0.60 "#E67C73", 0.75 "#F9950A", 0.85 "#AADC32", 0.90 "#5CC863", 1.00 "#57BB8A") + + +set xtics offset 0,15 +set ytics offset -27,0 left +set yrange [] reverse + +plot \ + 'efficiency_table.csv' \ + matrix rowheaders columnheaders using 1:2:3 with image, \ + 'efficiency_table.csv' \ + matrix rowheaders columnheaders using 1:2:(sprintf("%.2f",$3) ) with labels diff --git a/cfgs/flushing-cycles.cfg b/cfgs/flushing-cycles.cfg new file mode 100644 index 0000000..dcd59fe --- /dev/null +++ b/cfgs/flushing-cycles.cfg @@ -0,0 +1,148 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW Flushing.c1.c1 > +################################################################################ +window_name Flushing.c1.c1 +window_type single +window_id 1 +window_position_x 442 +window_position_y 116 +window_width 600 +window_height 115 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 34.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } +window_filter_module evt_type 1 40000003 +window_filter_module evt_type_label 1 "Flushing Traces" + +################################################################################ +< NEW DISPLAYING WINDOW cycles.c1 > +################################################################################ +window_name cycles.c1 +window_type single +window_id 2 +window_position_x 417 +window_position_y 404 +window_width 600 +window_height 114 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 194620048057.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 42000059 +window_filter_module evt_type_label 1 "PAPI_TOT_CYC [Total cycles]" + +################################################################################ +< NEW DISPLAYING WINDOW flushing_cycles > +################################################################################ +window_name flushing_cycles +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation product +window_identifiers 1 2 +window_position_x 2357 +window_position_y 217 +window_width 600 +window_height 115 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 79325.000000000000 +window_minimum_y 48287.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: flushing_cycles_hist +Analyzer2D.X: 2597 +Analyzer2D.Y: 55 +Analyzer2D.Width: 436 +Analyzer2D.Height: 587 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1000000000000000000.000000000000 +Analyzer2D.Delta: 1000000000000000000.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 48287.000000000000 +Analyzer2D.MaximumGradient: 79325.000000000000 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/flushing-inst.cfg b/cfgs/flushing-inst.cfg new file mode 100644 index 0000000..767e690 --- /dev/null +++ b/cfgs/flushing-inst.cfg @@ -0,0 +1,148 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW Flushing.c1 > +################################################################################ +window_name Flushing.c1 +window_type single +window_id 1 +window_position_x 355 +window_position_y 29 +window_width 600 +window_height 115 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 34.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } +window_filter_module evt_type 1 40000003 +window_filter_module evt_type_label 1 "Flushing Traces" + +################################################################################ +< NEW DISPLAYING WINDOW Instructions.c1.c2.c1 > +################################################################################ +window_name Instructions.c1.c2.c1 +window_type single +window_id 2 +window_position_x 384 +window_position_y 58 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 206968292.000000000000 +window_minimum_y 3314.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 42000050 +window_filter_module evt_type_label 1 "PAPI_TOT_INS [Instr completed]" + +################################################################################ +< NEW DISPLAYING WINDOW flushing_inst > +################################################################################ +window_name flushing_inst +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation product +window_identifiers 1 2 +window_position_x 2364 +window_position_y 128 +window_width 600 +window_height 115 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 17421.000000000000 +window_minimum_y 17121.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: flushing_inst_hist +Analyzer2D.X: 2597 +Analyzer2D.Y: 55 +Analyzer2D.Width: 436 +Analyzer2D.Height: 587 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1000000000000000000.000000000000 +Analyzer2D.Delta: 1000000000000000000.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 17121.000000000000 +Analyzer2D.MaximumGradient: 17421.000000000000 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/flushing.cfg b/cfgs/flushing.cfg new file mode 100644 index 0000000..6c7e258 --- /dev/null +++ b/cfgs/flushing.cfg @@ -0,0 +1,76 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW Flushing > +################################################################################ +window_name Flushing +window_type single +window_id 1 +window_position_x 2529 +window_position_y 349 +window_width 600 +window_height 115 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 34.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 72, { All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 40000003 +window_filter_module evt_type_label 1 "Flushing Traces" + +< NEW ANALYZER2D > +Analyzer2D.Name: flushing +Analyzer2D.X: 2717 +Analyzer2D.Y: 204 +Analyzer2D.Width: 617 +Analyzer2D.Height: 606 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: True +Analyzer2D.Minimum: 1.000000000000 +Analyzer2D.Maximum: 1.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 18.053000000000 +Analyzer2D.MaximumGradient: 35.420000000000 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/instructions.cfg b/cfgs/instructions.cfg new file mode 100644 index 0000000..777df0c --- /dev/null +++ b/cfgs/instructions.cfg @@ -0,0 +1,141 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW Instructions > +################################################################################ +window_name Instructions +window_type single +window_id 1 +window_position_x 346 +window_position_y 41 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 18640295641.000000000000 +window_minimum_y 5257.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 42000050 + +################################################################################ +< NEW DISPLAYING WINDOW Useful > +################################################################################ +window_name Useful +window_type single +window_id 2 +window_position_x 369 +window_position_y 92 +window_width 600 +window_height 134 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered true +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 1.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Useful}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, =}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, All}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +################################################################################ +< NEW DISPLAYING WINDOW Useful Instructions Timeline > +################################################################################ +window_name Useful Instructions Timeline +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation product +window_identifiers 1 2 +window_position_x 406 +window_position_y 340 +window_width 767 +window_height 314 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 18640295641.000000000000 +window_minimum_y 5257.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: instructions +Analyzer2D.X: 366 +Analyzer2D.Y: 24 +Analyzer2D.Width: 556 +Analyzer2D.Height: 764 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1000000000000000000.000000000000 +Analyzer2D.Delta: 1000000000000000000.000000000000 +Analyzer2D.ComputeGradient: False +Analyzer2D.MinimumGradient: 0 +Analyzer2D.MaximumGradient: 999999999999999999 +Analyzer2D.PixelSize: 1 +Analyzer2D.CodeColor: False + diff --git a/cfgs/io-call-cycles.cfg b/cfgs/io-call-cycles.cfg new file mode 100644 index 0000000..6140f79 --- /dev/null +++ b/cfgs/io-call-cycles.cfg @@ -0,0 +1,148 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW I/O call.c1.c1 > +################################################################################ +window_name I/O call.c1.c1 +window_type single +window_id 1 +window_position_x 616 +window_position_y 290 +window_width 729 +window_height 154 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 13.000000000000 +window_minimum_y 1.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } +window_filter_module evt_type 1 40000004 +window_filter_module evt_type_label 1 "I/O calls" + +################################################################################ +< NEW DISPLAYING WINDOW cycles.c1.c1.c1 > +################################################################################ +window_name cycles.c1.c1.c1 +window_type single +window_id 2 +window_position_x 645 +window_position_y 319 +window_width 600 +window_height 114 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 194620048057.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 42000059 +window_filter_module evt_type_label 1 "PAPI_TOT_CYC [Total cycles]" + +################################################################################ +< NEW DISPLAYING WINDOW posixio_cycles > +################################################################################ +window_name posixio_cycles +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation product +window_identifiers 1 2 +window_position_x 2298 +window_position_y 141 +window_width 729 +window_height 154 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 72043816.000000000000 +window_minimum_y 3111.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: posixio_cycles_hist +Analyzer2D.X: 2597 +Analyzer2D.Y: 55 +Analyzer2D.Width: 436 +Analyzer2D.Height: 587 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1000000000000000000.000000000000 +Analyzer2D.Delta: 1000000000000000000.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 6685313.000000000000 +Analyzer2D.MaximumGradient: 8988531.000000000000 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/io-call-instructions.cfg b/cfgs/io-call-instructions.cfg new file mode 100644 index 0000000..af8bc3b --- /dev/null +++ b/cfgs/io-call-instructions.cfg @@ -0,0 +1,148 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW I/O call.c1 > +################################################################################ +window_name I/O call.c1 +window_type single +window_id 1 +window_position_x 355 +window_position_y 29 +window_width 729 +window_height 154 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 13.000000000000 +window_minimum_y 1.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } +window_filter_module evt_type 1 40000004 +window_filter_module evt_type_label 1 "I/O calls" + +################################################################################ +< NEW DISPLAYING WINDOW Instructions.c1.c2 > +################################################################################ +window_name Instructions.c1.c2 +window_type single +window_id 2 +window_position_x 384 +window_position_y 58 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 206968292.000000000000 +window_minimum_y 3314.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 42000050 +window_filter_module evt_type_label 1 "PAPI_TOT_INS [Instr completed]" + +################################################################################ +< NEW DISPLAYING WINDOW posixio_inst > +################################################################################ +window_name posixio_inst +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation product +window_identifiers 1 2 +window_position_x 413 +window_position_y 87 +window_width 729 +window_height 154 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 3625596.000000000000 +window_minimum_y 3676.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: posixio_inst_hist +Analyzer2D.X: 2597 +Analyzer2D.Y: 55 +Analyzer2D.Width: 436 +Analyzer2D.Height: 587 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1000000000000000000.000000000000 +Analyzer2D.Delta: 1000000000000000000.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 2247759.000000000000 +Analyzer2D.MaximumGradient: 2503143.000000000000 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/io-call-reverse.cfg b/cfgs/io-call-reverse.cfg new file mode 100644 index 0000000..196ec98 --- /dev/null +++ b/cfgs/io-call-reverse.cfg @@ -0,0 +1,76 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW I/O call > +################################################################################ +window_name I/O call +window_type single +window_id 1 +window_position_x 2614 +window_position_y 238 +window_width 729 +window_height 154 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 18.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 40000004 +window_filter_module evt_type_label 1 "I/O calls" + +< NEW ANALYZER2D > +Analyzer2D.Name: io-call +Analyzer2D.X: 2971 +Analyzer2D.Y: 50 +Analyzer2D.Width: 646 +Analyzer2D.Height: 565 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Vertical +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: True +Analyzer2D.Minimum: 1.000000000000 +Analyzer2D.Maximum: 13.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 453.192000000000 +Analyzer2D.MaximumGradient: 17291.650000000001 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/modelfactors-all.gp b/cfgs/modelfactors-all.gp new file mode 100644 index 0000000..e05e716 --- /dev/null +++ b/cfgs/modelfactors-all.gp @@ -0,0 +1,36 @@ +#Gnuplot template for the projection functions + +#Prepare the axes +#REPLACE_BY_XRANGE +set xlabel "Number of Processes" +set logscale x +#REPLACE_BY_YRANGE +set ylabel "Efficiency" +set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) +set grid ytics + +set style line 1 lt 7 dt 2 lw 1.5 lc rgb "#0E3D59" +set style line 2 lt 7 dt 2 lw 1.5 lc rgb "#88A61B" +set style line 3 lt 7 dt 2 lw 1.5 lc rgb "#F29F05" +set style line 4 lt 7 dt 2 lw 1.5 lc rgb "#F25C05" +set style line 5 lt 7 dt 2 lw 1.5 lc rgb "#D92525" + +set key left bottom Left reverse + +#REPLACE_BY_PARA_FUNCTION +#REPLACE_BY_LOAD_FUNCTION +#REPLACE_BY_COMM_FUNCTION +#REPLACE_BY_COMP_FUNCTION +#REPLACE_BY_GLOB_FUNCTION + +plot para(x) title "Parallel Efficiency" ls 1,\ + load(x) title "Load Balance" ls 2,\ + comm(x) title "Communication Efficiency" ls 3,\ + comp(x) title "Computation Scalability" ls 4,\ + glob(x) title "Global Efficiency" ls 5,\ + '-' with points notitle ls 1,\ + '-' with points notitle ls 2,\ + '-' with points notitle ls 3,\ + '-' with points notitle ls 4,\ + '-' with points notitle ls 5 + diff --git a/cfgs/modelfactors-comm.gp b/cfgs/modelfactors-comm.gp new file mode 100644 index 0000000..6984035 --- /dev/null +++ b/cfgs/modelfactors-comm.gp @@ -0,0 +1,26 @@ + +#Gnuplot template for the projection functions + +#REPLACE_BY_TRACE_NAMES +#REPLACE_BY_XRANGE +set xlabel "Number of Processes" +#REPLACE_BY_XTICS_LABEL +## set logscale x +#REPLACE_BY_YRANGE +set ylabel "Efficiency (%)" +# set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) +# set grid ytics + +set style line 1 lt 7 dt 2 lw 1.5 lc rgb "red" +set style line 2 lt 7 dt 2 lw 1.5 lc rgb "green" +set style line 3 lt 7 dt 2 lw 1.5 lc rgb "orange" +set style line 4 lt 7 dt 2 lw 1.5 lc rgb "blue" +set style line 5 lt 7 dt 2 lw 1.5 lc rgb "magenta" + +set key left bottom Left reverse + + +plot '-' with linespoints title "Communication Efficiency" ls 1,\ + '-' with linespoints title "Serialization Efficiency" ls 2,\ + '-' with linespoints title "Transfer Efficiency" ls 3 + diff --git a/cfgs/modelfactors-hybrid.gp b/cfgs/modelfactors-hybrid.gp new file mode 100644 index 0000000..606ccd2 --- /dev/null +++ b/cfgs/modelfactors-hybrid.gp @@ -0,0 +1,37 @@ + +#Gnuplot template for the projection functions + +#REPLACE_BY_TRACE_NAMES +#REPLACE_BY_XRANGE +set xlabel "Number of Processes" +#REPLACE_BY_XTICS_LABEL +## set logscale x +#REPLACE_BY_YRANGE +set ylabel "Efficiency (%)" +# set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) +# set grid ytics + + +set style line 1 lt 8 dt 2 lw 2.0 lc rgb '#4B0082' # indigo + +set style line 2 lt 7 dt 2 lw 1.5 lc rgb "forest-green" +set style line 3 lt 6 dt 2 lw 1.5 lc rgb '#00FF00' # lime +set style line 4 lt 6 dt 2 lw 1.5 lc rgb '#008B8B' # darkcyan + +set style line 5 lt 5 dt 2 lw 1.5 lc rgb "red" +set style line 6 lt 4 dt 2 lw 1.5 lc rgb "orange" +set style line 7 lt 4 dt 2 lw 1.5 lc rgb "salmon" + +set key left bottom Left reverse + + + +plot '-' with linespoints title "Hybrid Efficiency" ls 1,\ + '-' with linespoints title "MPI Parallel efficiency" ls 2,\ + '-' with linespoints title "MPI Load balance" ls 3,\ + '-' with linespoints title "MPI Communication efficiency" ls 4,\ +#REPLACE_BY_OMP_PAR_EFF +#REPLACE_BY_OMP_LB +#REPLACE_BY_OMP_COMM + + diff --git a/cfgs/modelfactors-mpi-hybrid.gp b/cfgs/modelfactors-mpi-hybrid.gp new file mode 100644 index 0000000..3e30589 --- /dev/null +++ b/cfgs/modelfactors-mpi-hybrid.gp @@ -0,0 +1,32 @@ + +#Gnuplot template for the projection functions + +#REPLACE_BY_TRACE_NAMES +#REPLACE_BY_XRANGE +set xlabel "Number of Processes" +#REPLACE_BY_XTICS_LABEL +## set logscale x +#REPLACE_BY_YRANGE +set ylabel "Efficiency (%)" +# set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) +# set grid ytics + +set style line 1 lt 5 dt 2 lw 1.5 lc rgb '#1c1044' # dark blue + +set style line 2 lt 8 dt 2 lw 1.5 lc rgb "forest-green" +set style line 3 lt 6 dt 2 lw 1.5 lc rgb '#00FF00' # lime +set style line 4 lt 5 dt 2 lw 1.5 lc rgb '#008B8B' # darkcyan + + +set style line 8 lt 4 dt 2 lw 1.5 lc rgb '#7CFC00' # lawngreen +set style line 9 lt 4 dt 2 lw 1.5 lc rgb '#00FFFF' # cyan + + +set key left bottom Left reverse + + +plot '-' with linespoints title "MPI Parallel efficiency" ls 2,\ + '-' with linespoints title "MPI Load balance" ls 3,\ + '-' with linespoints title "MPI Communication efficiency" ls 4,\ + '-' with linespoints title "Serialization Efficiency" ls 8,\ + '-' with linespoints title "Transfer Efficiency" ls 9 diff --git a/cfgs/modelfactors-onlydata.gp b/cfgs/modelfactors-onlydata.gp new file mode 100644 index 0000000..4d0ff06 --- /dev/null +++ b/cfgs/modelfactors-onlydata.gp @@ -0,0 +1,31 @@ + +#Gnuplot template for the projection functions + +#REPLACE_BY_TRACE_NAMES +#REPLACE_BY_XRANGE +set xlabel "Number of Processes" +#REPLACE_BY_XTICS_LABEL +# set logscale x +#REPLACE_BY_YRANGE +## set yrange [0:] +set ylabel "Efficiency (%)" + +# set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) +# set grid ytics + +set style line 1 lt 8 dt 2 lw 2.0 lc rgb '#4B0082' # indigo +set style line 2 lt 7 dt 2 lw 1.5 lc rgb "red" +set style line 3 lt 7 dt 2 lw 1.5 lc rgb "green" + +set style line 4 lt 5 dt 5 lw 1.5 lc rgb "blue" +set style line 5 lt 3 dt 1 lw 1.8 lc rgb "dark-grey" + +set key left bottom Left reverse + + +plot '-' with linespoints title "Parallel Efficiency" ls 1,\ + '-' with linespoints title "Load Balance" ls 2,\ + '-' with linespoints title "Communication Efficiency" ls 3,\ + '-' with linespoints title "Computation Scalability" ls 4,\ + '-' with linespoints title "Global Efficiency" ls 5 + diff --git a/cfgs/modelfactors-scale.gp b/cfgs/modelfactors-scale.gp new file mode 100644 index 0000000..febb708 --- /dev/null +++ b/cfgs/modelfactors-scale.gp @@ -0,0 +1,27 @@ + +#Gnuplot template for the projection functions + +#REPLACE_BY_TRACE_NAMES +#REPLACE_BY_XRANGE +set xlabel "Number of Processes" +#REPLACE_BY_XTICS_LABEL +# set logscale x +#REPLACE_BY_YRANGE +set ylabel "Efficiency (%)" +# set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) +# set grid ytics + +set style line 1 lt 5 dt 2 lw 2.0 lc rgb "blue" +set style line 2 lt 7 dt 2 lw 1.5 lc rgb "cyan" +# set style line 2 lt 7 dt 2 lw 1.5 lc rgb '#6A5ACD' # slateblue +set style line 3 lt 7 dt 2 lw 1.5 lc rgb '#4682B4' # steelblue +set style line 4 lt 7 dt 2 lw 1.5 lc rgb '#8A2BE2' # blueviolet + +set key left bottom Left reverse + + +plot '-' with linespoints title "Computation Scalability" ls 1,\ + '-' with linespoints title "IPC Scalability" ls 2,\ + '-' with linespoints title "Instruction Scalability" ls 3,\ + '-' with linespoints title "Frequency Scalability" ls 4 + diff --git a/cfgs/mpi-call-outside.cfg b/cfgs/mpi-call-outside.cfg new file mode 100644 index 0000000..d4e7613 --- /dev/null +++ b/cfgs/mpi-call-outside.cfg @@ -0,0 +1,77 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW MPI call > +################################################################################ +window_name MPI call +window_type single +window_id 1 +window_position_x 2753 +window_position_y 151 +window_width 856 +window_height 303 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 124.000000000000 +window_minimum_y 3.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, [x,y]}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module task Thread i { 1, { 1 0.000000000000 } } +window_filter_module evt_type 2 50000001 50000005 +window_filter_module evt_type_label 2 "MPI Point-to-point" "Unknown" + +< NEW ANALYZER2D > +Analyzer2D.Name: MPI call profile +Analyzer2D.X: 2262 +Analyzer2D.Y: 247 +Analyzer2D.Width: 1381 +Analyzer2D.Height: 658 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: True +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: True +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 200.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 5.988000000000 +Analyzer2D.MaximumGradient: 60105183.622000001371 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/mpi-io-cycles.cfg b/cfgs/mpi-io-cycles.cfg new file mode 100644 index 0000000..ed0a0a6 --- /dev/null +++ b/cfgs/mpi-io-cycles.cfg @@ -0,0 +1,149 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW cycles.c1.c1 > +################################################################################ +window_name cycles.c1.c1 +window_type single +window_id 1 +window_position_x 529 +window_position_y 203 +window_width 600 +window_height 114 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 194620048057.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 42000059 +window_filter_module evt_type_label 1 "PAPI_TOT_CYC [Total cycles]" + +################################################################################ +< NEW DISPLAYING WINDOW MPI call.c1.c1 > +################################################################################ +window_name MPI call.c1.c1 +window_type single +window_id 2 +window_position_x 2303 +window_position_y 308 +window_width 642 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 1.000000000000 +window_minimum_y 1.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } +window_semantic_module task Thread i { 1, { 1 0.000000000000 } } +window_filter_module evt_type 1 50000005 +window_filter_module evt_type_label 1 "MPI I/O" + +################################################################################ +< NEW DISPLAYING WINDOW mpiio_cycles > +################################################################################ +window_name mpiio_cycles +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation product +window_identifiers 1 2 +window_position_x 587 +window_position_y 261 +window_width 600 +window_height 114 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 194620048057.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: mpiio_cycles_hist +Analyzer2D.X: 2468 +Analyzer2D.Y: 122 +Analyzer2D.Width: 600 +Analyzer2D.Height: 300 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1000000000000000000.000000000000 +Analyzer2D.Delta: 1000000000000000000.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 28106010.000000000000 +Analyzer2D.MaximumGradient: 168707925.000000000000 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/mpi-io-instructions.cfg b/cfgs/mpi-io-instructions.cfg new file mode 100644 index 0000000..e683b74 --- /dev/null +++ b/cfgs/mpi-io-instructions.cfg @@ -0,0 +1,149 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW Instructions.c1 > +################################################################################ +window_name Instructions.c1 +window_type single +window_id 1 +window_position_x 413 +window_position_y 87 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 206968292.000000000000 +window_minimum_y 3314.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 42000050 +window_filter_module evt_type_label 1 "PAPI_TOT_INS [Instr completed]" + +################################################################################ +< NEW DISPLAYING WINDOW MPI-io-sign.c1 > +################################################################################ +window_name MPI-io-sign.c1 +window_type single +window_id 2 +window_position_x 442 +window_position_y 116 +window_width 642 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 1.000000000000 +window_minimum_y 1.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } +window_semantic_module task Thread i { 1, { 1 0.000000000000 } } +window_filter_module evt_type 1 50000005 +window_filter_module evt_type_label 1 "MPI I/O" + +################################################################################ +< NEW DISPLAYING WINDOW mpiio_inst > +################################################################################ +window_name mpiio_inst +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation product +window_identifiers 1 2 +window_position_x 2334 +window_position_y 232 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 206968292.000000000000 +window_minimum_y 3314.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: mpiio_inst_hist_2 +Analyzer2D.X: 2427 +Analyzer2D.Y: 124 +Analyzer2D.Width: 600 +Analyzer2D.Height: 300 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1000000000000000000.000000000000 +Analyzer2D.Delta: 1000000000000000000.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 9583794.000000000000 +Analyzer2D.MaximumGradient: 108820758.000000000000 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/mpi-io-reverse.cfg b/cfgs/mpi-io-reverse.cfg new file mode 100644 index 0000000..143be0d --- /dev/null +++ b/cfgs/mpi-io-reverse.cfg @@ -0,0 +1,77 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW MPI call > +################################################################################ +window_name MPI call +window_type single +window_id 1 +window_position_x 2360 +window_position_y 274 +window_width 642 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 150.000000000000 +window_minimum_y 3.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module task Thread i { 1, { 1 0.000000000000 } } +window_filter_module evt_type 1 50000005 +window_filter_module evt_type_label 1 "MPI I/O" + +< NEW ANALYZER2D > +Analyzer2D.Name: MPI-IO +Analyzer2D.X: 2550 +Analyzer2D.Y: 40 +Analyzer2D.Width: 979 +Analyzer2D.Height: 501 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: True +Analyzer2D.HorizVert: Vertical +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: True +Analyzer2D.Minimum: 142.000000000000 +Analyzer2D.Maximum: 150.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 514.500000000000 +Analyzer2D.MaximumGradient: 6606164.996000000276 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/mpi-io.cfg b/cfgs/mpi-io.cfg new file mode 100644 index 0000000..1f683ac --- /dev/null +++ b/cfgs/mpi-io.cfg @@ -0,0 +1,77 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW MPI call > +################################################################################ +window_name MPI call +window_type single +window_id 1 +window_position_x 2360 +window_position_y 274 +window_width 642 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 150.000000000000 +window_minimum_y 3.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module task Thread i { 1, { 1 0.000000000000 } } +window_filter_module evt_type 1 50000005 +window_filter_module evt_type_label 1 "MPI I/O" + +< NEW ANALYZER2D > +Analyzer2D.Name: MPI-IO +Analyzer2D.X: 2862 +Analyzer2D.Y: 40 +Analyzer2D.Width: 667 +Analyzer2D.Height: 483 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: True +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: True +Analyzer2D.Minimum: 141.000000000000 +Analyzer2D.Maximum: 150.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 225.865000000000 +Analyzer2D.MaximumGradient: 698578.567999999970 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/mpi-master-thread.cfg b/cfgs/mpi-master-thread.cfg new file mode 100644 index 0000000..a130f31 --- /dev/null +++ b/cfgs/mpi-master-thread.cfg @@ -0,0 +1,79 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW MPI call > +################################################################################ +window_name MPI call +window_type single +window_id 1 +window_position_x 2214 +window_position_y 148 +window_width 756 +window_height 390 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 70.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +#REPLACE_BY_APP_FILTER +#REPLACE_BY_TASKS_FILTER +#REPLACE_BY_THREADS_FILTER +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, [x,y]}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module task Thread i { 1, { 1 0.000000000000 } } +window_filter_module evt_type 2 50000001 50000005 +window_filter_module evt_type_label 2 "MPI Point-to-point" "Unknown" + +< NEW ANALYZER2D > +Analyzer2D.Name: MPI call profile +Analyzer2D.X: 807 +Analyzer2D.Y: 207 +Analyzer2D.Width: 881 +Analyzer2D.Height: 582 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: True +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: True +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 200.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 11.456000000000 +Analyzer2D.MaximumGradient: 11868747.607000000775 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/runtime.cfg b/cfgs/runtime.cfg new file mode 100644 index 0000000..5438cda --- /dev/null +++ b/cfgs/runtime.cfg @@ -0,0 +1,72 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW MPI call > +################################################################################ +window_name MPI call +window_type single +window_id 1 +window_position_x 377 +window_position_y 288 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 70.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, [x,y]}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module task Thread i { 1, { 1 0.000000000000 } } +window_filter_module evt_type 2 50000001 50000005 + +< NEW ANALYZER2D > +Analyzer2D.Name: runtime +Analyzer2D.X: 487 +Analyzer2D.Y: 156 +Analyzer2D.Width: 1271 +Analyzer2D.Height: 616 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: True +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: True +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 200.000000000000 +Analyzer2D.Delta: 200.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 55413948.358000002801 +Analyzer2D.MaximumGradient: 55413948.358000002801 +Analyzer2D.PixelSize: 1 +Analyzer2D.CodeColor: False + diff --git a/cfgs/runtime_app.cfg b/cfgs/runtime_app.cfg new file mode 100644 index 0000000..791859f --- /dev/null +++ b/cfgs/runtime_app.cfg @@ -0,0 +1,74 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW runtimeapp > +################################################################################ +window_name runtimeapp +window_type single +window_id 1 +window_position_x 2457 +window_position_y 507 +window_width 581 +window_height 118 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered false +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 0.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max true +window_level appl +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { 1 } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, None}, {evt_value, None} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: runtimeapp +Analyzer2D.X: 2513 +Analyzer2D.Y: 133 +Analyzer2D.Width: 257 +Analyzer2D.Height: 251 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: True +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 0.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 5232000.069000000134 +Analyzer2D.MaximumGradient: 5232000.069000000134 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/time_computing.cfg b/cfgs/time_computing.cfg new file mode 100644 index 0000000..4bb17ad --- /dev/null +++ b/cfgs/time_computing.cfg @@ -0,0 +1,114 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW interval from states.c3 > +################################################################################ +window_name interval from states.c3 +window_type single +window_id 1 +window_position_x 377 +window_position_y 29 +window_width 600 +window_height 134 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered true +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Nanoseconds +window_maximum_y 325782.880999999994 +window_minimum_y 28.247000000000 +window_compute_y_max false +window_level thread +window_scale_relative 0.276616926766 +window_end_time_relative 0.276616926766 +window_object appl { 1, { All } } +window_begin_time_relative 0.261615159840 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, State Record Dur.}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, None}, {to_obj, None}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, None}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module thread State Record Dur. { 1, { 2 0.000000000000 1.000000000000 } } + +################################################################################ +< NEW DISPLAYING WINDOW MPI elapsed time (ns).c1 > +################################################################################ +window_name MPI elapsed time (ns).c1 +window_type single +window_id 2 +window_position_x 406 +window_position_y 58 +window_width 600 +window_height 134 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered true +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Nanoseconds +window_maximum_y 34405654.000000000000 +window_minimum_y 28247.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 0.276616926766 +window_end_time_relative 0.276616926766 +window_object appl { 1, { All } } +window_begin_time_relative 0.261615159840 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, None}, {to_obj, None}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 54000006 +window_filter_module evt_type_label 1 "Elapsed time in MPI" + +################################################################################ +< NEW DISPLAYING WINDOW time outside MPI > +################################################################################ +window_name time outside MPI +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation substract +window_identifiers 1 2 +window_position_x 367 +window_position_y 385 +window_width 600 +window_height 134 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 472609042.000000000000 +window_minimum_y 2905.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open true +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + diff --git a/cfgs/timings.cfg b/cfgs/timings.cfg new file mode 100644 index 0000000..d2061ef --- /dev/null +++ b/cfgs/timings.cfg @@ -0,0 +1,72 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW States > +################################################################################ +window_name States +window_type single +window_id 1 +window_position_x 393 +window_position_y 88 +window_width 767 +window_height 314 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 18.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, State As Is}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, All}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: States Histogram +Analyzer2D.X: 396 +Analyzer2D.Y: 433 +Analyzer2D.Width: 600 +Analyzer2D.Height: 300 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: True +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: True +Analyzer2D.Minimum: 1.000000000000 +Analyzer2D.Maximum: 18.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 273554.000000000000 +Analyzer2D.MaximumGradient: 9395816375.000000000000 +Analyzer2D.PixelSize: 1 +Analyzer2D.CodeColor: False +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/hybridmetrics.py b/hybridmetrics.py new file mode 100644 index 0000000..4cc6abb --- /dev/null +++ b/hybridmetrics.py @@ -0,0 +1,1496 @@ +#!/usr/bin/env python3 + +"""Functions to compute the model metrics.""" + +from __future__ import print_function, division +import sys + + +from rawdata import * +from collections import OrderedDict + +# error import variables +error_import_pandas = False +error_import_seaborn = False +error_import_matplotlib = False +error_import_numpy = False + +try: + import numpy as np +except ImportError: + error_import_numpy = True + + +try: + import pandas as pd +except ImportError: + error_import_pandas = True + +try: + import seaborn as sns +except ImportError: + error_import_seaborn = True + + +try: + import matplotlib.pyplot as plt +except ImportError: + error_import_matplotlib = True + + + +# Contains all model factor entries with a printable name. +# This is used to generate and print all model factors, so, if an entry is added, +# it should be added here, too. + +other_metrics_doc = OrderedDict([('elapsed_time', 'Elapsed time (sec)'), + ('efficiency', 'Efficiency'), + ('speedup', 'Speedup'), + ('ipc', 'Average IPC'), + ('freq', 'Average frequency (GHz)'), + ('flushing', 'Flushing (%)'), + ('io_mpiio', 'MPI I/O (%)'), + ('io_posix', 'POSIX I/O (%)'), + ('io_eff', 'I/O Efficiency (%)')]) + +mod_factors_doc = OrderedDict([('global_eff', 'Global efficiency'), + ('parallel_eff', '-- Parallel efficiency'), + ('load_balance', ' -- Load balance'), + ('comm_eff', ' -- Communication efficiency'), + ('comp_scale', '-- Computation scalability'), + ('ipc_scale', ' -- IPC scalability'), + ('inst_scale', ' -- Instruction scalability'), + ('freq_scale', ' -- Frequency scalability')]) + +mod_factors_scale_plus_io_doc = OrderedDict([('comp_scale', '-- Computation scalability + I/O'), + ('ipc_scale', ' -- IPC scalability'), + ('inst_scale', ' -- Instruction scalability'), + ('freq_scale', ' -- Frequency scalability')]) + +mod_hybrid_factors_doc = OrderedDict([ + ('hybrid_eff', '-- Hybrid Parallel efficiency'), + ('mpi_parallel_eff', ' -- MPI Parallel efficiency'), + ('mpi_load_balance', ' -- MPI Load balance'), + ('mpi_comm_eff', ' -- MPI Communication efficiency'), + ('serial_eff', ' -- Serialization efficiency'), + ('transfer_eff', ' -- Transfer efficiency'), + ('omp_parallel_eff', ' -- OMP Parallel efficiency'), + ('omp_load_balance', ' -- OMP Load balance'), + ('omp_comm_eff', ' -- OMP Communication efficiency')]) + + +def create_mod_factors(trace_list): + """Creates 2D dictionary of the model factors and initializes with an empty + string. The mod_factors dictionary has the format: [mod factor key][trace]. + """ + global mod_factors_doc + mod_factors = {} + for key in mod_factors_doc: + trace_dict = {} + for trace_name in trace_list: + trace_dict[trace_name] = 0.0 + mod_factors[key] = trace_dict + + return mod_factors + + +def create_mod_factors_scale_io(trace_list): + """Creates 2D dictionary of the model factors and initializes with an empty + string. The mod_factors dictionary has the format: [mod factor key][trace]. + """ + global mod_factors_scale_plus_io_doc + mod_factors_scale_plus_io = {} + for key in mod_factors_scale_plus_io_doc: + trace_dict = {} + for trace_name in trace_list: + trace_dict[trace_name] = 0.0 + mod_factors_scale_plus_io[key] = trace_dict + + return mod_factors_scale_plus_io + + +def create_hybrid_mod_factors(trace_list): + """Creates 2D dictionary of the hybrid model factors and initializes with an empty + string. The hybrid_factors dictionary has the format: [mod factor key][trace]. + """ + global mod_hybrid_factors_doc + hybrid_factors = {} + for key in mod_hybrid_factors_doc: + trace_dict = {} + for trace_name in trace_list: + trace_dict[trace_name] = 0.0 + hybrid_factors[key] = trace_dict + + return hybrid_factors + + +def create_other_metrics(trace_list): + """Creates 2D dictionary of the other metrics and initializes with an empty + string. The other_metrics dictionary has the format: [mod factor key][trace]. + """ + global other_metrics_doc + other_metrics = {} + for key in other_metrics_doc: + trace_dict = {} + for trace_name in trace_list: + trace_dict[trace_name] = 0.0 + other_metrics[key] = trace_dict + + return other_metrics + + +def get_scaling_type(raw_data, trace_list, trace_processes, cmdl_args): + """Guess the scaling type (weak/strong) based on the useful instructions. + Computes the normalized instruction ratio for all measurements, whereas the + normalized instruction ratio is (instructions ratio / process ratio) with + the smallest run as reference. For exact weak scaling the normalized ratio + should be exactly 1 and for exact strong scaling it should be close to zero + with an upper bound of 0.5. The eps value defines the threshold to be + considered weak scaling and should give enough buffer to safely handle + non-ideal scaling. + """ + eps = 0.9 + normalized_inst_ratio = 0 + + # Check if there is only one trace. + if len(trace_list) == 1: + return 'strong' + + for trace in trace_list: + try: # except NaN + inst_ratio = float(raw_data['useful_ins'][trace]) / float(raw_data['useful_ins'][trace_list[0]]) + except: + inst_ratio = 0.0 + try: # except NaN + proc_ratio = float(trace_processes[trace]) / float(trace_processes[trace_list[0]]) + except: + proc_ratio = 'NaN' + + normalized_inst_ratio += inst_ratio / proc_ratio + + # Get the average inst increase. Ignore ratio of first trace 1.0) + normalized_inst_ratio = (normalized_inst_ratio - 1) / (len(trace_list) - 1) + + scaling_computed = '' + + if normalized_inst_ratio > eps: + scaling_computed = 'weak' + else: + scaling_computed = 'strong' + + if cmdl_args.scaling == 'auto': + if cmdl_args.debug: + print('==DEBUG== Detected ' + scaling_computed + ' scaling.') + print('') + return scaling_computed + + if cmdl_args.scaling == 'weak': + if scaling_computed == 'strong': + print('==Warning== Scaling set to weak scaling but detected strong scaling.') + print('') + return 'weak' + + if cmdl_args.scaling == 'strong': + if scaling_computed == 'weak': + print('==Warning== Scaling set to strong scaling but detected weak scaling.') + print('') + return 'strong' + + print('==Error== reached undefined control flow state.') + sys.exit(1) + + +def compute_model_factors(raw_data, trace_list, trace_processes, trace_mode, list_mpi_procs_count, cmdl_args): + """Computes the model factors from the gathered raw data and returns the + according dictionary of model factors.""" + mod_factors = create_mod_factors(trace_list) + hybrid_factors = create_hybrid_mod_factors(trace_list) + other_metrics = create_other_metrics(trace_list) + mod_factors_scale_plus_io = create_mod_factors_scale_io(trace_list) + + # Guess the weak or strong scaling + scaling = get_scaling_type(raw_data, trace_list, trace_processes, cmdl_args) + + # Loop over all traces + for trace in trace_list: + + proc_ratio = float(trace_processes[trace]) / float(trace_processes[trace_list[0]]) + total_procs = trace_processes[trace] + + # Flushing measurements + try: # except NaN + other_metrics['flushing'][trace] = float(raw_data['flushing_tot'][trace]) \ + / (raw_data['runtime'][trace] * total_procs) * 100.0 + except: + other_metrics['flushing'][trace] = 0.0 + + # I/O measurements + try: # except NaN + other_metrics['io_mpiio'][trace] = float(raw_data['mpiio_tot'][trace]) \ + / (raw_data['runtime'][trace] * total_procs) * 100.0 + except: + other_metrics['io_mpiio'][trace] = 0.0 + + try: # except NaN + other_metrics['io_posix'][trace] = float(raw_data['io_tot'][trace]) \ + / (raw_data['runtime'][trace] * total_procs) * 100.0 + except: + other_metrics['io_posix'][trace] = 0.0 + try: # except NaN + io_total = float(raw_data['mpiio_tot'][trace] + raw_data['flushing_tot'][trace] + + raw_data['io_tot'][trace]) + other_metrics['io_eff'][trace] = float(raw_data['useful_tot'][trace]) \ + / (raw_data['useful_tot'][trace] + io_total) * 100.0 + except: + other_metrics['io_eff'][trace] = 0.0 + + # Basic efficiency factors + try: # except NaN + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: + print("Hello, I pass for this conditional", trace) + mod_factors['load_balance'][trace] = float(raw_data['useful_plus_io_avg'][trace]) \ + / float(raw_data['useful_plus_io_max'][trace]) * 100.0 + else: + mod_factors['load_balance'][trace] = float(raw_data['useful_avg'][trace]) \ + / float(raw_data['useful_max'][trace]) * 100.0 + except: + mod_factors['load_balance'][trace] = 'NaN' + + try: # except NaN + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: + mod_factors['comm_eff'][trace] = float(raw_data['useful_plus_io_max'][trace] \ + / raw_data['runtime'][trace] * 100.0) + else: + mod_factors['comm_eff'][trace] = float(raw_data['useful_max'][trace] \ + / raw_data['runtime'][trace] * 100.0) + except: + mod_factors['comm_eff'][trace] = 'NaN' + + try: # except NaN + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: + mod_factors['parallel_eff'][trace] = float(raw_data['useful_plus_io_avg'][trace]) \ + / float(raw_data['runtime'][trace]) * 100.0 + else: + mod_factors['parallel_eff'][trace] = float(mod_factors['load_balance'][trace]) \ + * float(mod_factors['comm_eff'][trace]) / 100.0 + except: + mod_factors['parallel_eff'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + if scaling == 'strong': + mod_factors['comp_scale'][trace] = float(raw_data['useful_tot'][trace_list[0]]) \ + / float(raw_data['useful_tot'][trace]) * 100.0 + else: + mod_factors['comp_scale'][trace] = float(raw_data['useful_tot'][trace_list[0]]) \ + / float(raw_data['useful_tot'][trace]) * proc_ratio * 100.0 + else: + mod_factors['comp_scale'][trace] = 'Non-Avail' + except: + mod_factors['comp_scale'][trace] = 'NaN' + + # Computation Scale + Serial I/O + try: # except NaN + if len(trace_list) > 1: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: + io_serial_0 = float(raw_data['io_tot'][trace_list[0]] + raw_data['flushing_tot'][trace_list[0]]) + io_serial_n = float(raw_data['io_tot'][trace] + raw_data['flushing_tot'][trace]) + if scaling == 'strong': + mod_factors_scale_plus_io['comp_scale'][trace] = float((raw_data['useful_tot'][trace_list[0]] + + io_serial_0) / (raw_data['useful_tot'][trace] + + io_serial_n) * 100.0) + else: + mod_factors_scale_plus_io['comp_scale'][trace] = (float(raw_data['useful_tot'][trace_list[0]]) + + io_serial_0) \ + / (float(raw_data['useful_tot'][trace]) + + io_serial_n) * proc_ratio * 100.0 + else: + mod_factors_scale_plus_io['comp_scale'][trace] = float(mod_factors['comp_scale'][trace]) + else: + mod_factors_scale_plus_io['comp_scale'][trace] = 'Non-Avail' + except: + mod_factors_scale_plus_io['comp_scale'][trace] = 'NaN' + + + try: # except NaN + if len(trace_list) > 1: + mod_factors['global_eff'][trace] = float(mod_factors['parallel_eff'][trace]) \ + * float(mod_factors['comp_scale'][trace]) / 100.0 + else: + mod_factors['global_eff'][trace] = float(mod_factors['parallel_eff'][trace]) * 100.0 / 100.0 + except: + mod_factors['global_eff'][trace] = 'NaN' + + # Hybrid metrics calculation + # -------> MPI metrics + try: # except NaN + hybrid_factors['mpi_load_balance'][trace] = float(raw_data['outsidempi_avg'][trace]) \ + / float(raw_data['outsidempi_max'][trace]) * 100.0 + except: + hybrid_factors['mpi_load_balance'][trace] = 'NaN' + + try: # except NaN + hybrid_factors['mpi_comm_eff'][trace] = float(raw_data['outsidempi_max'][trace]) \ + / float(raw_data['runtime'][trace]) * 100.0 + except: + hybrid_factors['mpi_comm_eff'][trace] = 'NaN' + + # ------------> BEGIN MPI communication sub-metrics + try: # except NaN + hybrid_factors['serial_eff'][trace] = float(raw_data['outsidempi_dim'][trace]) \ + / float(raw_data['runtime_dim'][trace]) * 100.0 + if hybrid_factors['serial_eff'][trace] > 100.0: + hybrid_factors['serial_eff'][trace] = 'Warning!' + except: + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + hybrid_factors['serial_eff'][trace] = 'NaN' + else: + hybrid_factors['serial_eff'][trace] = 'Non-Avail' + + try: # except NaN + if hybrid_factors['serial_eff'][trace] != 'Warning!': + hybrid_factors['transfer_eff'][trace] = float(hybrid_factors['mpi_comm_eff'][trace]) \ + / float(hybrid_factors['serial_eff'][trace]) * 100.0 + else: + hybrid_factors['transfer_eff'][trace] = float(raw_data['runtime_dim'][trace]) \ + / float(raw_data['runtime'][trace]) * 100.0 + if hybrid_factors['transfer_eff'][trace] > 100.0: + hybrid_factors['transfer_eff'][trace] = 'Warning!' + except: + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + hybrid_factors['transfer_eff'][trace] = 'NaN' + else: + hybrid_factors['transfer_eff'][trace] = 'Non-Avail' + # --------------> END MPI communication sub-metrics + try: # except NaN + if raw_data['outsidempi_tot'][trace] == raw_data['outsidempi_tot_diff'][trace]: + hybrid_factors['mpi_parallel_eff'][trace] = float(raw_data['outsidempi_tot'][trace] / + (raw_data['runtime'][trace] + * list_mpi_procs_count[trace]) * 100.0) + else: + hybrid_factors['mpi_parallel_eff'][trace] = float(raw_data['outsidempi_tot_diff'][trace] + / (raw_data['runtime'][trace] + * trace_processes[trace]) * 100.0) + except: + hybrid_factors['mpi_parallel_eff'][trace] = 'NaN' + + # -------> Metrics for the second parallel paradigm + try: # except NaN + hybrid_factors['omp_comm_eff'][trace] = float(mod_factors['comm_eff'][trace]) \ + / float(hybrid_factors['mpi_comm_eff'][trace]) * 100.0 + except: + hybrid_factors['omp_comm_eff'][trace] = 'NaN' + + try: # except NaN + hybrid_factors['omp_load_balance'][trace] = float(mod_factors['load_balance'][trace]) \ + / float(hybrid_factors['mpi_load_balance'][trace]) * 100.0 + except: + hybrid_factors['omp_load_balance'][trace] = 'NaN' + + try: # except NaN + hybrid_factors['omp_parallel_eff'][trace] = float(mod_factors['parallel_eff'][trace]) \ + / float(hybrid_factors['mpi_parallel_eff'][trace]) * 100.0 + except: + hybrid_factors['omp_parallel_eff'][trace] = 'NaN' + + # -------> Global Hybrid Metric + try: # except NaN + hybrid_factors['hybrid_eff'][trace] = float(hybrid_factors['mpi_parallel_eff'][trace]) \ + * float(hybrid_factors['omp_parallel_eff'][trace]) / 100.0 + except: + hybrid_factors['hybrid_eff'][trace] = 'NaN' + + # Basic scalability factors + try: # except NaN + other_metrics['ipc'][trace] = float(raw_data['useful_ins'][trace]) \ + / float(raw_data['useful_cyc'][trace]) + except: + other_metrics['ipc'][trace] = 'NaN' + try: # except NaN + if len(trace_list) > 1: + mod_factors['ipc_scale'][trace] = float(other_metrics['ipc'][trace]) \ + / float(other_metrics['ipc'][trace_list[0]]) * 100.0 + else: + mod_factors['ipc_scale'][trace] = 'Non-Avail' + except: + mod_factors['ipc_scale'][trace] = 'NaN' + + # IPC scale + Serial I/O + try: # except NaN + if len(trace_list) > 1: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: + ipc_serial_io_0 = (raw_data['useful_ins'][trace_list[0]] + raw_data['io_ins'][trace_list[0]] + + raw_data['flushing_ins'][trace_list[0]]) \ + / (raw_data['useful_cyc'][trace_list[0]] + raw_data['io_cyc'][trace_list[0]] + + raw_data['flushing_cyc'][trace_list[0]]) + + ipc_serial_io_n = (raw_data['useful_ins'][trace] + raw_data['io_ins'][trace] + + raw_data['flushing_ins'][trace])\ + / (raw_data['useful_cyc'][trace] + raw_data['io_cyc'][trace] + + raw_data['flushing_cyc'][trace]) + mod_factors_scale_plus_io['ipc_scale'][trace] = float(ipc_serial_io_n / ipc_serial_io_0 * 100.0) + else: + mod_factors_scale_plus_io['ipc_scale'][trace] = float(mod_factors['ipc_scale'][trace]) + else: + mod_factors_scale_plus_io['ipc_scale'][trace] = 'Non-Avail' + except: + mod_factors_scale_plus_io['ipc_scale'][trace] = 'NaN' + + try: # except NaN + other_metrics['freq'][trace] = float(raw_data['useful_cyc'][trace]) \ + / float(raw_data['useful_tot'][trace]) / 1000 + except: + other_metrics['freq'][trace] = 'NaN' + try: # except NaN + if len(trace_list) > 1: + mod_factors['freq_scale'][trace] = float(other_metrics['freq'][trace]) \ + / float(other_metrics['freq'][trace_list[0]]) * 100.0 + else: + mod_factors['freq_scale'][trace] = 'Non-Avail' + except: + mod_factors['freq_scale'][trace] = 'NaN' + + # freq scale + Serial I/O + try: # except NaN + if len(trace_list) > 1: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: + freq_serial_io_0 = (float(raw_data['useful_cyc'][trace_list[0]]) + + float(raw_data['io_cyc'][trace_list[0]]) + + float(raw_data['flushing_cyc'][trace_list[0]]))\ + / (raw_data['useful_tot'][trace_list[0]] + + float(raw_data['io_tot'][trace_list[0]]) + + float(raw_data['flushing_tot'][trace_list[0]])) / 1000 + + freq_serial_io_n = (float(raw_data['useful_cyc'][trace]) + + float(raw_data['io_cyc'][trace]) + + float(raw_data['flushing_cyc'][trace]))\ + / (float(raw_data['useful_tot'][trace]) + + float(raw_data['io_tot'][trace]) + + float(raw_data['flushing_tot'][trace])) / 1000 + mod_factors_scale_plus_io['freq_scale'][trace] = float(freq_serial_io_n / freq_serial_io_0 * 100.0) + else: + mod_factors_scale_plus_io['freq_scale'][trace] = float(mod_factors['freq_scale'][trace]) + else: + mod_factors_scale_plus_io['freq_scale'][trace] = 'Non-Avail' + except: + mod_factors_scale_plus_io['freq_scale'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + if scaling == 'strong': + mod_factors['inst_scale'][trace] = float(raw_data['useful_ins'][trace_list[0]])\ + / float(raw_data['useful_ins'][trace]) * 100.0 + else: + mod_factors['inst_scale'][trace] = float(raw_data['useful_ins'][trace_list[0]]) \ + / float(raw_data['useful_ins'][trace]) * proc_ratio * 100.0 + else: + mod_factors['inst_scale'][trace] = 'Non-Avail' + except: + mod_factors['inst_scale'][trace] = 'NaN' + + # ins scale + Serial I/O + try: # except NaN + if len(trace_list) > 1: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: + useful_ins_plus_io_0 = float(raw_data['useful_ins'][trace_list[0]]) \ + + float(raw_data['io_ins'][trace_list[0]]) \ + + float(raw_data['flushing_ins'][trace_list[0]]) + useful_ins_plus_io_n = float(raw_data['useful_ins'][trace]) \ + + float(raw_data['io_ins'][trace]) \ + + float(raw_data['flushing_ins'][trace]) + if scaling == 'strong': + mod_factors_scale_plus_io['inst_scale'][trace] = float(useful_ins_plus_io_0) \ + / float(useful_ins_plus_io_n) * 100.0 + else: + mod_factors_scale_plus_io['inst_scale'][trace] = float(useful_ins_plus_io_0) \ + / float(useful_ins_plus_io_n)\ + * proc_ratio * 100.0 + else: + mod_factors_scale_plus_io['inst_scale'][trace] = float(mod_factors['inst_scale'][trace]) + else: + mod_factors_scale_plus_io['inst_scale'][trace] = 'Non-Avail' + except: + mod_factors_scale_plus_io['inst_scale'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + if scaling == 'strong': + other_metrics['speedup'][trace] = float(raw_data['runtime'][trace_list[0]] + / raw_data['runtime'][trace]) + else: + other_metrics['speedup'][trace] = float(raw_data['runtime'][trace_list[0]] + / raw_data['runtime'][trace] * proc_ratio) + else: + other_metrics['speedup'][trace] = 'Non-Avail' + except: + other_metrics['speedup'][trace] = 'NaN' + + try: # except NaN + other_metrics['elapsed_time'][trace] = float(raw_data['runtime'][trace] * 0.000001) + except: + other_metrics['elapsed_time'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + #other_metrics['efficiency'][trace] = other_metrics['speedup'][trace] / proc_ratio + if scaling == 'strong': + other_metrics['efficiency'][trace] = raw_data['runtime'][trace_list[0]] \ + / (raw_data['runtime'][trace] * proc_ratio) + else: + other_metrics['efficiency'][trace] = raw_data['runtime'][trace_list[0]] \ + / raw_data['runtime'][trace] + else: + other_metrics['efficiency'][trace] = 'Non-Avail' + except: + other_metrics['efficiency'][trace] = 'NaN' + + return mod_factors, mod_factors_scale_plus_io, hybrid_factors, other_metrics + + +def print_mod_factors_table(mod_factors, other_metrics, mod_factors_scale_plus_io, hybrid_factors, trace_list, + trace_processes, trace_tasks, trace_threads,trace_mode): + """Prints the model factors table in human readable form on stdout.""" + global mod_factors_doc, mod_hybrid_factors_doc + + warning_io = [] + warning_flush = [] + warning_flush_wrong = [] + warning_simulation = [] + for trace in trace_list: + if 10.0 <= other_metrics['flushing'][trace] < 15.0: + warning_flush.append(1) + elif other_metrics['flushing'][trace] >= 15.0: + warning_flush_wrong.append(1) + if other_metrics['io_posix'][trace] >= 5.0: + warning_io.append(1) + if hybrid_factors['serial_eff'][trace] == "Warning!" \ + or hybrid_factors['serial_eff'][trace] == "Warning!": + warning_simulation.append(1) + + if len(warning_flush_wrong) > 0: + print("WARNING! Flushing in a trace is too high. Disabling standard output metrics...") + print(" Flushing is an overhead due to the tracer, please review your trace.") + print('') + return + + # Update the hybrid parallelism mode + trace_mode_doc = trace_mode[trace_list[0]] + if trace_mode_doc[0:len("Detailed+MPI+")] == "Detailed+MPI+": + mod_hybrid_factors_doc['omp_parallel_eff'] = " -- " + \ + trace_mode_doc[len("Detailed+MPI+"):] + " Parallel efficiency" + mod_hybrid_factors_doc['omp_load_balance'] = " -- " + \ + trace_mode_doc[len("Detailed+MPI+"):] + " Load Balance" + mod_hybrid_factors_doc['omp_comm_eff'] = " -- " + \ + trace_mode_doc[len("Detailed+MPI+"):] + " Communication efficiency" + + # print('') + print('\n Overview of the Efficiency metrics:') + + longest_name = len(sorted(mod_hybrid_factors_doc.values(), key=len)[-1]) + + line = ''.rjust(longest_name) + if len(trace_list) == 1: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[0]] + else: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[len(trace_list)-1]] + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # BEGIN To adjust header to big number of processes + procs_header = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if limit_min == limit_max and same_procs and len(trace_list) > 1: + s_xtics = (str(trace_processes[trace]) + '[' + str(index+1) + ']') + else: + s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') + if s_xtics in procs_header: + s_xtics += '[' + str(index + 1) + ']' + procs_header.append(s_xtics) + + max_len_header = 0 + for proc_h in procs_header: + if max_len_header < len(proc_h): + max_len_header = len(proc_h) + + value_to_adjust = 10 + if max_len_header > value_to_adjust: + value_to_adjust = max_len_header + 1 + # END To adjust header to big number of processes + + label_xtics = [] + for index, trace in enumerate(trace_list): + line += ' | ' + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if limit_min == limit_max and same_procs and len(trace_list) > 1: + s_xtics = (str(trace_processes[trace]) + '[' + str(index+1) + ']') + else: + s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') + if s_xtics in label_xtics: + s_xtics += '[' + str(index + 1) + ']' + label_xtics.append(s_xtics) + line += s_xtics.rjust(value_to_adjust) + + print(''.ljust(len(line), '=')) + print(line) + line_procs_factors = line + + print(''.ljust(len(line), '=')) + + for mod_key in mod_factors_doc: + line = mod_factors_doc[mod_key].ljust(longest_name) + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(mod_factors[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(mod_factors[mod_key][trace])).rjust(value_to_adjust) + print(line) + + io_metrics = [] + io_metrics.append(0) + for mod_key in mod_factors_scale_plus_io_doc: + for trace in trace_list: + if mod_factors_scale_plus_io[mod_key][trace] != 0: + io_metrics.append(1) + + if (len(warning_flush) >= 1 or len(warning_io) >= 1) and len(trace_list) > 1: + print(''.ljust(len(line_procs_factors), '-')) + for mod_key in mod_factors_scale_plus_io_doc: + line = mod_factors_scale_plus_io_doc[mod_key].ljust(longest_name) + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(mod_factors_scale_plus_io[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(mod_factors_scale_plus_io[mod_key][trace])).rjust(value_to_adjust) + print(line) + print(''.ljust(len(line_procs_factors), '=')) + else: + print(''.ljust(len(line_procs_factors), '-')) + + for mod_key in mod_hybrid_factors_doc: + line = mod_hybrid_factors_doc[mod_key].ljust(longest_name) + for trace in trace_list: + line += ' | ' + try: # except NaN + if str(hybrid_factors[mod_key][trace]) != 'nan': + line += ('{0:.2f}%'.format(hybrid_factors[mod_key][trace])).rjust(value_to_adjust) + elif str(hybrid_factors[mod_key][trace]) == 'nan': + line += ('{}'.format('NaN')).rjust(value_to_adjust) + else: + line += ('{}'.format(hybrid_factors[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(hybrid_factors[mod_key][trace])).rjust(value_to_adjust) + print(line) + + print(''.ljust(len(line_procs_factors), '=')) + if len(warning_simulation) > 0: + print("===> Warning! Metrics obtained from simulated traces exceed 100%. " + "Please review original and simulated traces.") + print('') + + +def print_other_metrics_table(other_metrics, trace_list, trace_processes, trace_tasks, trace_threads): + """Prints the other metrics table in human readable form on stdout.""" + global other_metrics_doc + + print('Overview of the Efficiency, Speedup, IPC and Frequency:') + + longest_name = len(sorted(other_metrics_doc.values(), key=len)[-1]) + + line = ''.rjust(longest_name) + if len(trace_list) == 1: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[0]] + else: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[len(trace_list)-1]] + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # BEGIN To adjust header to big number of processes + procs_header = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if limit_min == limit_max and same_procs and len(trace_list) > 1: + s_xtics = (str(trace_processes[trace]) + '[' + str(index+1) + ']') + else: + s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') + if s_xtics in procs_header: + s_xtics += '[' + str(index + 1) + ']' + procs_header.append(s_xtics) + + max_len_header = 0 + for proc_h in procs_header: + if max_len_header < len(proc_h): + max_len_header = len(proc_h) + + value_to_adjust = 10 + if max_len_header > value_to_adjust: + value_to_adjust = max_len_header + 1 + # END To adjust header to big number of processes + + label_xtics = [] + for index, trace in enumerate(trace_list): + line += ' | ' + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if limit_min == limit_max and same_procs and len(trace_list) > 1: + s_xtics = (str(trace_processes[trace]) + '[' + str(index+1) + ']') + else: + s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') + if s_xtics in label_xtics: + s_xtics += '[' + str(index + 1) + ']' + label_xtics.append(s_xtics) + line += s_xtics.rjust(value_to_adjust) + + print(''.ljust(len(line), '-')) + print(line) + line_head = line + print(''.ljust(len(line), '-')) + + for mod_key in other_metrics_doc: + line = other_metrics_doc[mod_key].ljust(longest_name) + if len(trace_list) > 1: + if mod_key in ['speedup', 'ipc', 'freq', 'elapsed_time', 'efficiency']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + print(line) + else: + if mod_key in ['ipc', 'freq', 'elapsed_time']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + print(line) + print(''.ljust(len(line_head), '-')) + # print('') + + warning_io = [] + warning_flush = [] + for trace in trace_list: + if other_metrics['flushing'][trace] >= 10.0: + warning_flush.append(1) + if other_metrics['io_mpiio'][trace] >= 5.0 or other_metrics['io_posix'][trace] >= 5.0: + warning_io.append(1) + + for mod_key in other_metrics_doc: + line = other_metrics_doc[mod_key].ljust(longest_name) + # Print empty line to separate values + if mod_key in ['freq'] and len(warning_flush) > 0: + print('') + print("Overview of tracer\'s flushing weight:") + print(''.ljust(len(line_head), '-')) + + if mod_key not in ['speedup', 'ipc', 'freq', 'elapsed_time', 'efficiency']: + if mod_key in ['flushing']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + if len(warning_flush) > 0: + print(line) + print(''.ljust(len(line_head), '-')) + elif mod_key in ['io_mpiio','io_posix','io_eff']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + if len(warning_io) > 0: + print(line) + + # Print headers I/O + if mod_key in ['flushing'] and len(warning_io) > 0: + print(''.ljust(len(line), ' ')) + print('Overview of File I/O weight:') + print(''.ljust(len(line), '-')) + if mod_key in ['io_eff'] and len(warning_io) > 0: + print(''.ljust(len(line), '-')) + # print('') + + if len(warning_flush) > 0: + message_warning_flush = "WARNING! %Flushing is high and affects computation of efficiency metrics." + else: + message_warning_flush = "" + if len(warning_io) > 0: + message_warning_io = "WARNING! % File I/O is high and affects computation of efficiency metrics." + else: + message_warning_io = "" + print(message_warning_flush+message_warning_io) + # print('') + + +def print_efficiency_table(mod_factors, hybrid_factors, trace_list, trace_processes, trace_tasks, trace_threads): + """Prints the model factors table in human readable form on stdout.""" + global mod_factors_doc, mod_hybrid_factors_doc + + delimiter = ',' + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + if len(trace_list) == 1: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[0]] + else: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[len(trace_list) - 1]] + + file_path = os.path.join(os.getcwd(), 'efficiency_table-global.csv') + with open(file_path, 'w') as output: + line = '\"Number of processes\"' + + label_xtics = [] + for index, trace in enumerate(trace_list): + line += delimiter + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if limit_min == limit_max and same_procs and len(trace_list) > 1: + s_xtics = (str(trace_processes[trace]) + '[' + str(index + 1) + ']') + else: + s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') + if s_xtics in label_xtics: + s_xtics += '[' + str(index + 1) + ']' + label_xtics.append(s_xtics) + line += s_xtics + + output.write(line + '\n') + + for mod_key in mod_factors_doc: + if mod_key not in ['speedup', 'ipc', 'freq', 'elapsed_time', 'efficiency', 'flushing', 'io_mpiio', 'io_posix']: + if mod_key in ['parallel_eff', 'comp_scale']: + line = "\"" + mod_factors_doc[mod_key].replace(' ', '', 2) + "\"" + elif mod_key in ['load_balance', 'comm_eff','ipc_scale', 'inst_scale','freq_scale']: + line = "\"" + mod_factors_doc[mod_key].replace(' ', '', 2) + "\"" + else: + line = "\"" + mod_factors_doc[mod_key] + "\"" + for trace in trace_list: + line += delimiter + try: # except NaN + if mod_factors[mod_key][trace] == "Non-Avail" or mod_factors[mod_key][trace] == "Warning!": + line += '0.00' + else: + line += '{0:.2f}'.format(mod_factors[mod_key][trace]) + except ValueError: + line += '{}'.format(mod_factors[mod_key][trace]) + output.write(line + '\n') + + # Create Gnuplot file for efficiency plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'efficiency_table-global.gp') + content = [] + + with open(gp_template) as f: + content = f.readlines() + + limit_procs = 500 + len(trace_list) * 65 + ## print(limit_procs) + + # Replace xrange + content = [line.replace('#REPLACE_BY_SIZE', ''.join(['set terminal pngcairo enhanced dashed crop size ', + str(limit_procs), ',460 font "Latin Modern Roman,14"'])) + for line in content] + + file_path = os.path.join(os.getcwd(), 'efficiency_table_global.gp') + with open(file_path, 'w') as f: + f.writelines(content) + # print('======== Plot (gnuplot File): EFFICIENCY Table ========') + if len(trace_list) > 1: + print('Global Efficiency Table written to ' + file_path[:len(file_path) - 3] + '.gp') + # print('') + + delimiter = ',' + file_path = os.path.join(os.getcwd(), 'efficiency_table-hybrid.csv') + with open(file_path, 'w') as output: + line = '\"Number of processes\"' + label_xtics = [] + for index, trace in enumerate(trace_list): + line += delimiter + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if limit_min == limit_max and same_procs and len(trace_list) > 1: + s_xtics = (str(trace_processes[trace]) + '[' + str(index + 1) + ']') + else: + s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') + if s_xtics in label_xtics: + s_xtics += '[' + str(index + 1) + ']' + label_xtics.append(s_xtics) + line += s_xtics + output.write(line + '\n') + + for mod_key in mod_hybrid_factors_doc: + if mod_key in ['mpi_parallel_eff', 'omp_parallel_eff']: + line = "\"" + mod_hybrid_factors_doc[mod_key].replace(' ', '', 2) + "\"" + elif mod_key in ['mpi_load_balance', 'mpi_comm_eff','omp_load_balance', 'omp_comm_eff']: + line = "\"" + mod_hybrid_factors_doc[mod_key].replace(' ', '', 2) + "\"" + elif mod_key in ['serial_eff', 'transfer_eff']: + line = "\"" + mod_hybrid_factors_doc[mod_key].replace(' ', ' ', 2) + "\"" + else: + line = "\"" + mod_hybrid_factors_doc[mod_key] + "\"" + for trace in trace_list: + line += delimiter + try: # except NaN + if hybrid_factors[mod_key][trace] == "Non-Avail" or hybrid_factors[mod_key][trace] == "Warning!": + line += '0.00' + else: + line += '{0:.2f}'.format(hybrid_factors[mod_key][trace]) + except ValueError: + line += '{}'.format(hybrid_factors[mod_key][trace]) + output.write(line + '\n') + + # Create Gnuplot file for efficiency plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'efficiency_table-hybrid.gp') + content = [] + + with open(gp_template) as f: + content = f.readlines() + + limit_procs = 510 + len(trace_list) * 80 + + # Replace xrange + content = [line.replace('#REPLACE_BY_SIZE', ''.join(['set terminal pngcairo enhanced dashed crop size ', + str(limit_procs), ',460 font "Latin Modern Roman,14"'])) + for line in content] + + file_path = os.path.join(os.getcwd(), 'efficiency_table-hybrid.gp') + with open(file_path, 'w') as f: + f.writelines(content) + # print('======== Plot (gnuplot File): EFFICIENCY Table ========') + if len(trace_list) > 1: + print('Hybrid Efficiency Table written to ' + file_path[:len(file_path) - 3] + '.gp') + #print('') + + +def print_mod_factors_csv(mod_factors, hybrid_factors, trace_list, trace_processes): + """Prints the model factors table in a csv file.""" + global mod_factors_doc, mod_hybrid_factors_doc + + delimiter = ';' + # File is stored in the trace directory + # file_path = os.path.join(os.path.dirname(os.path.realpath(trace_list[0])), 'modelfactors.csv') + # File is stored in the execution directory + file_path = os.path.join(os.getcwd(), 'modelfactors.csv') + + with open(file_path, 'w') as output: + line = "\"Number of processes\"" + for trace in trace_list: + line += delimiter + line += str(trace_processes[trace]) + output.write(line + '\n') + + for mod_key in mod_factors_doc: + line = "\"" + mod_factors_doc[mod_key].replace(' ', '', 2) + "\"" + for trace in trace_list: + line += delimiter + try: # except NaN + line += '{0:.6f}'.format(mod_factors[mod_key][trace]) + except ValueError: + line += '{}'.format(mod_factors[mod_key][trace]) + output.write(line + '\n') + + for mod_key in mod_hybrid_factors_doc: + line = "\"" + mod_hybrid_factors_doc[mod_key].replace(' ', '', 2) + "\"" + for trace in trace_list: + line += delimiter + try: # except NaN + line += '{0:.6f}'.format(hybrid_factors[mod_key][trace]) + except ValueError: + line += '{}'.format(hybrid_factors[mod_key][trace]) + output.write(line + '\n') + + output.write('#\n') + + print('======== Output Files: Metrics and Plots ========') + print('Model factors written to ' + file_path) + + +def print_other_metrics_csv(other_metrics, trace_list, trace_processes): + """Prints the model factors table in a csv file.""" + global other_metrics_doc + + delimiter = ';' + # File is stored in the trace directory + # file_path = os.path.join(os.path.dirname(os.path.realpath(trace_list[0])), 'modelfactors.csv') + # File is stored in the execution directory + file_path = os.path.join(os.getcwd(), 'other_metrics.csv') + + with open(file_path, 'w') as output: + line = 'Number of processes' + for trace in trace_list: + line += delimiter + line += str(trace_processes[trace]) + output.write(line + '\n') + + for mod_key in other_metrics_doc: + line = other_metrics_doc[mod_key].replace(' ', '', 2) + for trace in trace_list: + line += delimiter + try: # except NaN + line += '{0:.6f}'.format(other_metrics[mod_key][trace]) + except ValueError: + line += '{}'.format(other_metrics[mod_key][trace]) + output.write(line + '\n') + + output.write('#\n') + + print('') + print('======== Output File: Other Metrics ========') + print('Speedup, IPC, Frequency, I/O and Flushing written to ' + file_path) + + +def plots_efficiency_table_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args): + # Plotting using python + # For plotting using python, read the csv file + + file_path = os.path.join(os.getcwd(), 'efficiency_table-hybrid.csv') + df = pd.read_csv(file_path) + metrics = df['Number of processes'].tolist() + + traces_procs = list(df.keys())[1:] + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # Set limit for projection + if cmdl_args.limit: + limit = cmdl_args.limit + else: + limit = str(trace_processes[trace_list[len(trace_list)-1]]) + + limit_min = trace_processes[trace_list[0]] + + # To xticks label + label_xtics = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if int(limit) == int(limit_min) and same_procs: + s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' + else: + s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' + if s_xtics in label_xtics: + s_xtics += '[' + str(index + 1) + ']' + label_xtics.append(s_xtics) + ##### End xticks + + # BEGIN To adjust header to big number of processes + max_len_header = 7 + for labelx in label_xtics: + if len(labelx) > max_len_header: + max_len_header = len(labelx) + + # END To adjust header to big number of processes + + list_data = [] + for index, rows in df.iterrows(): + list_temp = [] + for value in list(rows)[1:]: + if float(value) == 0.0: + list_temp.append(np.nan) + else: + list_temp.append(float(value)) + # print(list_temp) + list_data.append(list_temp) + + list_np = np.array(list_data) + + idx = metrics + cols = label_xtics + # cols = traces_procs + df = pd.DataFrame(list_np, index=idx, columns=cols) + + # min for 1 trace is x=3 for the (x,y) in figsize + size_figure_y = len(idx) * 0.40 + size_figure_x = len(cols) * 0.14 * max_len_header + plt.figure(figsize=(size_figure_x, size_figure_y)) + + ax = sns.heatmap(df, cmap='RdYlGn', linewidths=0.05, annot=True, vmin=0, vmax=100, center=75, \ + fmt='.2f', annot_kws={"size": 10}, cbar_kws={'label': 'Percentage(%)'}) + ## to align ylabels to left + plt.yticks(rotation=0, ha='left') + + ax.xaxis.tick_top() + # to adjust metrics + len_pad = 0 + for metric in metrics: + if len(metric) > len_pad: + len_pad = len(metric) + + ax.yaxis.set_tick_params(pad=len_pad + 164) + + plt.savefig('efficiency_table-hybrid-matplot.png', bbox_inches='tight') + + # General Metrics plot + + file_path = os.path.join(os.getcwd(), 'efficiency_table-global.csv') + df = pd.read_csv(file_path) + metrics = df['Number of processes'].tolist() + + traces_procs = list(df.keys())[1:] + + list_data = [] + for index, rows in df.iterrows(): + list_temp = [] + for value in list(rows)[1:]: + if float(value) == 0.0: + list_temp.append(np.nan) + else: + list_temp.append(float(value)) + # print(list_temp) + list_data.append(list_temp) + + list_np = np.array(list_data) + + idx = metrics + cols = label_xtics + # cols = traces_procs + df = pd.DataFrame(list_np, index=idx, columns=cols) + + # min for 1 traces is x=3 for the (x,y) in figsize + size_figure_y = len(idx) * 0.40 + size_figure_x = len(cols) * 0.14 * max_len_header + plt.figure(figsize=(size_figure_x, size_figure_y)) + + ax = sns.heatmap(df, cmap='RdYlGn', linewidths=0.05, annot=True, vmin=0, vmax=100, center=75, \ + fmt='.2f', annot_kws={"size": 10}, cbar_kws={'label': 'Percentage(%)'}) + ## to align ylabels to left + plt.yticks(rotation=0, ha='left') + ax.xaxis.tick_top() + # to adjust metrics + len_pad = 0 + for metric in metrics: + if len(metric) > len_pad: + len_pad = len(metric) + + ax.yaxis.set_tick_params(pad=len_pad + 140) + + plt.savefig('efficiency_table-global-matplot.png', bbox_inches='tight') + + +def plots_modelfactors_matplot(trace_list, trace_processes,trace_tasks, trace_threads, cmdl_args): + # Plotting using python + # For plotting using python, read the csv file + file_path = os.path.join(os.getcwd(), 'modelfactors.csv') + df = pd.read_csv(file_path, sep=';') + + traces_procs = list(df.keys())[1:] + + list_data = [] + for index, rows in df.iterrows(): + list_temp = [] + for value in list(rows)[1:]: + if value != 'Non-Avail' and value != 'Warning!': + list_temp.append(float(value)) + elif value == 'Non-Avail' or value == 'Warning!': + list_temp.append('NaN') + # print(list_temp) + list_data.append(list_temp) + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # Set limit for projection + if cmdl_args.limit: + limit = cmdl_args.limit + else: + limit = str(trace_processes[trace_list[len(trace_list)-1]]) + + limit_min = trace_processes[trace_list[0]] + + # To xticks label + label_xtics = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if int(limit) == int(limit_min) and same_procs: + s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' + else: + s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' + if s_xtics in label_xtics: + s_xtics += '[' + str(index + 1) + ']' + label_xtics.append(s_xtics) + + ## Global Metrics + plt.figure() + max_global = max([max(list_data[0]), max(list_data[1]), max(list_data[2]), + max(list_data[3]), max(list_data[4])]) + plt.plot(traces_procs, list_data[0], 'o-', color='black', label='Global Efficiency') + plt.plot(traces_procs, list_data[1], 's--', color='magenta', label='Parallel Efficiency') + plt.plot(traces_procs, list_data[2], '*:', color='red', label='Load Balance') + plt.plot(traces_procs, list_data[3], 'x-.', color='green', label='Communication efficiency') + plt.plot(traces_procs, list_data[4], 'v--', color='blue', label='Computation scalability') + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency (%)") + plt.xticks(tuple(traces_procs), tuple(label_xtics)) + # print(max_global) + if float(max_global) < 100: + max_global = 100 + + plt.ylim(0, max_global+5) + plt.legend() + plt.savefig('modelfactors-global-matplot.png', bbox_inches='tight') + + ## Scale Metrics + plt.figure() + max_scale = max([max(list_data[4]), max(list_data[5]), max(list_data[6]), max(list_data[7])]) + plt.plot(traces_procs, list_data[4], 'v-', color='blue', markerfacecolor='blue', label='Computation scalability') + plt.plot(traces_procs, list_data[5], 'v--', color='skyblue', label='IPC scalability') + plt.plot(traces_procs, list_data[6], 'v:', color='gray', label='Instruction scalability') + plt.plot(traces_procs, list_data[7], 'v-.', color='darkviolet', label='Frequency scalability') + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency (%)") + plt.xticks(tuple(traces_procs), tuple(label_xtics)) + if float(max_scale) < 100: + max_scale = 100 + + plt.ylim(0, max_scale+5) + plt.legend() + plt.savefig('modelfactors-scale-matplot.png', bbox_inches='tight') + + ## Hybrid Metrics + + plt.figure() + max_hybrid = max([max(list_data[8]), max(list_data[9]), max(list_data[10]), max(list_data[11]), + max(list_data[14]), max(list_data[15]), max(list_data[16])]) + plt.plot(traces_procs, list_data[8], 's-', color='purple', label='Hybrid Parallel efficiency') + plt.plot(traces_procs, list_data[9], 'o--', color='green', label='MPI Parallel efficiency') + plt.plot(traces_procs, list_data[10], 'x-.', color='lime', label='MPI Load balance') + plt.plot(traces_procs, list_data[11], '*:', color='lightseagreen', label='MPI Communication efficiency') + plt.plot(traces_procs, list_data[14], 'h--', color='red', label='OpenMP Parallel efficiency') + plt.plot(traces_procs, list_data[15], 'v-.', color='orange', label='OpenMP Load Balance') + plt.plot(traces_procs, list_data[16], 'X:', color='salmon', label='OpenMP Communication efficiency') + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency (%)") + plt.xticks(tuple(traces_procs), tuple(label_xtics)) + if float(max_hybrid) < 100: + max_hybrid = 100 + + plt.ylim(0, max_hybrid+5) + plt.legend() + plt.savefig('modelfactors-hybrid-matplot.png', bbox_inches='tight') + + ## MPI Metrics + plt.figure() + if max(list_data[12]) != 'NaN' and max(list_data[13]) != 'NaN': + max_mpi = max([max(list_data[9]), max(list_data[10]), max(list_data[11]), + max(list_data[12]), max(list_data[13])]) + plt.plot(traces_procs, list_data[9], 's-', color='green', label='MPI Parallel efficiency') + plt.plot(traces_procs, list_data[10], 'v:', color='lime', label='MPI Load balance') + plt.plot(traces_procs, list_data[11], 'o-.', color='lightseagreen', label='MPI Communication efficiency') + plt.plot(traces_procs, list_data[12], linestyle=(0, (3, 10, 1, 10)), marker='s', color='gold', + label='Serialization efficiency') + plt.plot(traces_procs, list_data[13], linestyle=(0, (3, 5, 1, 5)), marker='x', color='tomato', label='Transfer efficiency') + elif max(list_data[12]) == 'NaN' and max(list_data[13]) == 'NaN': + max_mpi = max([max(list_data[9]), max(list_data[10]), max(list_data[11])]) + plt.plot(traces_procs, list_data[9], 's-', color='green', label='MPI Parallel efficiency') + plt.plot(traces_procs, list_data[10], 'v:', color='lime', label='MPI Load balance') + plt.plot(traces_procs, list_data[11], 'o-.', color='lightseagreen', label='MPI Communication efficiency') + elif max(list_data[12]) != 'NaN' and max(list_data[13]) == 'NaN': + max_mpi = max([max(list_data[9]), max(list_data[10]), max(list_data[11]), + max(list_data[12])]) + plt.plot(traces_procs, list_data[9], 's-', color='green', label='MPI Parallel efficiency') + plt.plot(traces_procs, list_data[10], 'v:', color='lime', label='MPI Load balance') + plt.plot(traces_procs, list_data[11], 'o-.', color='lightseagreen', label='MPI Communication efficiency') + plt.plot(traces_procs, list_data[12], linestyle=(0, (3, 10, 1, 10)), marker='s', color='gold', label='Serialization efficiency') + else: + max_mpi = max([max(list_data[9]), max(list_data[10]), max(list_data[11]),max(list_data[13])]) + plt.plot(traces_procs, list_data[9], 's-', color='green', label='MPI Parallel efficiency') + plt.plot(traces_procs, list_data[10], 'v:', color='lime', label='MPI Load balance') + plt.plot(traces_procs, list_data[11], 'o-.', color='lightseagreen', label='MPI Communication efficiency') + plt.plot(traces_procs, list_data[13], linestyle=(0, (3, 5, 1, 5)), marker='x', color='tomato', label='Transfer efficiency') + + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency (%)") + plt.xticks(tuple(traces_procs), tuple(label_xtics)) + if float(max_mpi) < 100: + max_mpi = 100 + + plt.ylim(0, max_mpi+5) + plt.legend() + plt.savefig('modelfactors-mpi-matplot.png', bbox_inches='tight') + + +def plots_speedup_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args): + # Plotting using python + # For plotting using python, read the csv file + file_path = os.path.join(os.getcwd(), 'other_metrics.csv') + df = pd.read_csv(file_path, sep=';') + + traces_procs = list(df.keys())[1:] + + list_data = [] + for index, rows in df.iterrows(): + list_temp = [] + for value in list(rows)[1:]: + if value != 'Non-Avail': + list_temp.append(float(value)) + elif value == 'Non-Avail': + list_temp.append('NaN') + # print(list_temp) + list_data.append(list_temp) + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # Set limit for projection + if cmdl_args.limit: + limit = cmdl_args.limit + else: + limit = str(trace_processes[trace_list[len(trace_list) - 1]]) + + limit_min = trace_processes[trace_list[0]] + + # proc_ratio to ideal speedup + proc_ratio = [] + for index, trace in enumerate(trace_list): + proc_ratio.append(trace_processes[trace]/trace_processes[trace_list[0]]) + # print(proc_ratio) + + # To xticks label + label_xtics = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if int(limit) == int(limit_min) and same_procs: + s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' + elif int(limit) == int(limit_min) and not same_procs: + s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' \ + + str(threads) + ')' + else: + s_xtics = str(trace_processes[trace]) + label_xtics.append(s_xtics) + + int_traces_procs = [] + prev_procs = int(float(traces_procs[0])) + int_traces_procs.append(int(float(traces_procs[0]))) + count_rep = 0 + for procs in traces_procs[1:]: + if prev_procs == int(float(procs)): + count_rep += 1 + int_traces_procs.append(int(float(procs))+ (2*count_rep)) + prev_procs = int(float(procs)) + else: + int_traces_procs.append(int(float(procs))) + prev_procs = int(float(procs)) + count_rep = 0 + + ### Plot: Global Metrics + plt.figure() + for x, y in zip(int_traces_procs, list_data[2]): + label = "{:.2f}".format(y) + plt.annotate(label, (x, y), textcoords="offset points", xytext=(0, 10), ha='center') + + plt.plot(int_traces_procs, list_data[2], 'o-', color='blue', label='measured') + plt.plot(int_traces_procs, proc_ratio, 'o-', color='black', label='ideal') + plt.xlabel("Number of Processes") + plt.ylabel("SpeedUp") + plt.xticks(tuple(int_traces_procs), tuple(label_xtics)) + #plt.yscale('log') + plt.legend() + #plt.xlim(0, ) + plt.ylim(0, ) + plt.savefig('speedup-matplot.png', bbox_inches='tight') + + ### Plot: Efficiency + # print(list_data) + plt.figure() + for x, y in zip(int_traces_procs, list_data[1]): + label = "{:.2f}".format(y) + plt.annotate(label, (x, y), textcoords="offset points", xytext=(0, 10), ha='center') + + plt.plot(int_traces_procs, list_data[1], 'o-', color='blue', label='measured') + plt.axhline(y=1, color='black', linestyle='-', label='ideal') + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency") + plt.xticks(tuple(int_traces_procs), tuple(label_xtics)) + # plt.yscale('log') + max_y = max(list_data[1]) + if max_y < 1.1: + max_y = 1.0 + #plt.xlim(0, ) + plt.ylim(0,max_y+0.1) + plt.legend() + plt.savefig('efficiency-matplot.png', bbox_inches='tight') \ No newline at end of file diff --git a/modelfactors.py b/modelfactors.py new file mode 100755 index 0000000..4eb5c6b --- /dev/null +++ b/modelfactors.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 + +"""modelfactors.py Generates performance metrics from a set of Paraver traces.""" + +from __future__ import print_function, division +import subprocess +import plots +from utils import parse_arguments, check_installation +from tracemetadata import get_traces_from_args +from rawdata import gather_raw_data, print_raw_data_table, print_raw_data_csv +from simplemetrics import compute_model_factors, print_mod_factors_csv, print_efficiency_table, \ + print_mod_factors_table, read_mod_factors_csv, print_other_metrics_table, \ + print_other_metrics_csv, plots_efficiency_table_matplot, plots_modelfactors_matplot, plots_speedup_matplot + +import hybridmetrics + +# error import variables +error_import_pandas = False +error_import_seaborn = False +error_import_matplotlib = False +error_import_scipy = False +error_import_numpy = False + +try: + import pandas as pd +except ImportError: + error_import_pandas = True + +try: + import seaborn as sns +except ImportError: + error_import_seaborn = True + +try: + import matplotlib + matplotlib.use('pdf') + import matplotlib.pyplot as plt00 +except ImportError: + error_import_matplotlib = True + +try: + import scipy.optimize +except ImportError: + error_import_scipy = True + + +try: + import numpy +except ImportError: + error_import_numpy = True + + +__author__ = "Sandra Mendez" +__copyright__ = "Copyright 2019, Barcelona Supercomputing Center (BSC)" +__version_major__ = 0 +__version_minor__ = 3 +__version_micro__ = 7 +__version__ = str(__version_major__) + "." + str(__version_minor__) + "." + str(__version_micro__) + + +if __name__ == "__main__": + """Main control flow. + Currently the script only accepts one parameter, which is a list of traces + that are processed. This can be a regex with wild cards and only valid trace + files are kept at the end. + """ + # Parse command line arguments + cmdl_args = parse_arguments() + + # Check if paramedir and Dimemas are in the path + check_installation(cmdl_args) + + # Process the trace list + trace_list, trace_processes, trace_tasks, trace_threads, trace_task_per_node, trace_mode = \ + get_traces_from_args(cmdl_args) + + # To validate the metric type + trace_metrics = 0 + # print(trace_mode) + for trace in trace_list: + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace][:15] == 'Detailed+OpenMP' \ + or trace_mode[trace][:17] == 'Detailed+Pthreads' or trace_mode[trace][:13] == 'Detailed+CUDA' \ + or trace_mode[trace][:14] == 'Detailed+OmpSs' or trace_mode[trace][:15] == 'Detailed+OpenCL' \ + or trace_mode[trace] == 'Detailed' or trace_mode[trace][:5] == 'Burst': + trace_metrics += 1 + + # Analyze the traces and gather the raw input data + raw_data, list_mpi_procs_count = gather_raw_data(trace_list, trace_processes, trace_task_per_node, + trace_mode, cmdl_args) + print_raw_data_csv(raw_data, trace_list, trace_processes) + + # Compute the model factors and print them + + if cmdl_args.metrics == 'hybrid' and trace_metrics == 0: + mod_factors, mod_factors_scale_plus_io, hybrid_factors, other_metrics = \ + hybridmetrics.compute_model_factors(raw_data, trace_list, trace_processes, + trace_mode, list_mpi_procs_count, cmdl_args) + hybridmetrics.print_other_metrics_table(other_metrics, trace_list, trace_processes, trace_tasks, trace_threads) + hybridmetrics.print_other_metrics_csv(other_metrics, trace_list, trace_processes) + hybridmetrics.print_mod_factors_table(mod_factors, other_metrics, mod_factors_scale_plus_io, hybrid_factors, + trace_list, trace_processes, trace_tasks, trace_threads, trace_mode) + hybridmetrics.print_mod_factors_csv(mod_factors, hybrid_factors, trace_list, trace_processes) + hybridmetrics.print_efficiency_table(mod_factors, hybrid_factors, trace_list, trace_processes, trace_tasks, + trace_threads) + # Plotting efficiency table with matplotlib + error_plot_table = False + if error_import_numpy or error_import_pandas or error_import_matplotlib or error_import_seaborn: + print('Numpy/Pandas/Matplotlib/Seaborn modules not available. ' + 'Skipping efficiency table plotting with python.') + if len(trace_list) > 1: + out_ver_gnuplot = subprocess.check_output(["gnuplot", "--version"]) + if 'gnuplot 5.' not in str(out_ver_gnuplot): + print('It requires gnuplot version 5.0 or higher. ' + 'Skipping efficiency table and lineal plotting with gnuplot.') + else: + try: + output_gnuplot_g = subprocess.check_output(["gnuplot", "efficiency_table_global.gp"]) + output_gnuplot_h = subprocess.check_output(["gnuplot", "efficiency_table-hybrid.gp"]) + except: + print(output_gnuplot_g) + print(output_gnuplot_h) + error_plot_table = True + + if not error_plot_table: + hybridmetrics.plots_efficiency_table_matplot(trace_list, trace_processes, trace_tasks, + trace_threads, cmdl_args) + if len(trace_list) > 1: + hybridmetrics.plots_modelfactors_matplot(trace_list, trace_processes, trace_tasks, + trace_threads, cmdl_args) + hybridmetrics.plots_speedup_matplot(trace_list, trace_processes, trace_tasks, + trace_threads, cmdl_args) + + # Plotting if SciPy and NumPy are installed. + error_plot_lineal = False + if error_import_numpy or error_import_scipy: + print('Scipy/NumPy module not available. Skipping lineal plotting.') + error_plot_lineal = True + + if not error_plot_lineal: + if len(trace_list) > 1: + plots.plot_hybrid_metrics(mod_factors, hybrid_factors, trace_list, trace_processes, + trace_tasks, trace_threads, trace_mode, cmdl_args) + + if len(trace_list) == 1: + subprocess.check_output(["rm", "efficiency_table_global.gp"]) + subprocess.check_output(["rm", "efficiency_table-hybrid.gp"]) + + elif cmdl_args.metrics == 'simple' or trace_metrics > 0: + mod_factors, mod_factors_scale_plus_io, other_metrics = compute_model_factors(raw_data, trace_list, + trace_processes, trace_mode, + list_mpi_procs_count, cmdl_args) + print_other_metrics_table(other_metrics, trace_list, trace_processes) + print_other_metrics_csv(other_metrics, trace_list, trace_processes) + print_mod_factors_table(mod_factors, other_metrics, mod_factors_scale_plus_io, trace_list, trace_processes) + print_mod_factors_csv(mod_factors, trace_list, trace_processes) + print_efficiency_table(mod_factors, trace_list, trace_processes) + + # Plotting efficiency table with matplotlib + error_plot_table = False + if error_import_numpy or error_import_pandas or error_import_matplotlib or error_import_seaborn: + print('Numpy/Pandas/Matplotlib/Seaborn modules not available. ' + 'Skipping efficiency table plotting with python.') + if len(trace_list) > 1: + out_ver_gnuplot = subprocess.check_output(["gnuplot", "--version"]) + if 'gnuplot 5.' not in str(out_ver_gnuplot): + print('It requires gnuplot version 5.0 or higher. ' + 'Skipping efficiency table and lineal plotting with gnuplot.') + else: + try: + output_gnuplot = subprocess.check_output(["gnuplot", "efficiency_table.gp"]) + except: + print(output_gnuplot) + error_plot_table = True + + if not error_plot_table: + plots_efficiency_table_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args) + if len(trace_list) > 1: + plots_modelfactors_matplot(trace_list, trace_mode, trace_processes, trace_tasks, trace_threads, + cmdl_args) + + # Plotting if SciPy and NumPy are installed. + error_plot_lineal = False + if error_import_numpy or error_import_scipy: + print('Scipy/NumPy module not available. Skipping lineal plotting.') + error_plot_lineal = True + # sys.exit(1) + if not error_plot_lineal: + if len(trace_list) > 1: + plots.plot_simple_metrics(mod_factors, trace_list, trace_processes, trace_mode, cmdl_args) + plots_speedup_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args) + if len(trace_list) == 1: + subprocess.check_output(["rm", "efficiency_table.gp"]) diff --git a/perf_metrics.bash b/perf_metrics.bash old mode 100644 new mode 100755 index e758f18..bd7685c --- a/perf_metrics.bash +++ b/perf_metrics.bash @@ -1,56 +1,34 @@ #!/bin/bash -main() -{ - -if [ $# -gt 0 ]; then - - echo "This script does not accept arguments, parameters need to be added to perf_metrics.config Aborting" - exit 1 - -fi - -#Get script directory -dir=$(pwd) - -echo -echo "Using the following configuration:" -echo -source "$dir"/perf_metrics.config -grep -o '^[^#]*' perf_metrics.config -echo - -Init -Test_arguments -Test_Comp -Create_metrics - -} - +# Functions +#Checks if the job submission ended correctly. Job_completed() { if [ "$Jobs_scheduler" == "slurm" ]; then - local id1=${1##* } + local id1 + id1=${1##* } sleep 5 - if ! scontrol show job $id1 | grep -q 'JobState=COMPLETED'; then + if ! scontrol show job "$id1" | grep -q 'JobState=COMPLETED'; then Completed=false else Completed=true fi elif [ "$Jobs_scheduler" == "lsf" ]; then - local id1=$(head -n1 1 | cut -d'<' -f2 | cut -d'>' -f1) + local id2 + id2=$(head -n1 "$1" | cut -d'<' -f2 | cut -d'>' -f1) sleep 5 - if ! bjobs -l $id | grep -q 'Status '; then + if ! bjobs -l "$id2" | grep -q 'Status '; then Completed=false else Completed=true fi elif [ "$Jobs_scheduler" == "torque" ]; then - local id1=$(head -n1 $1 | awk '{ print $3 }') + local id3 + id3=$(head -n1 "$1" | awk '{ print $3 }') sleep 5 - if ! qstat f $id | grep -q 'exit_status = 0'; then + if ! qstat f "$id3" | grep -q 'exit_status = 0'; then Completed=false else Completed=true @@ -60,9 +38,215 @@ Job_completed() } +# Check if nemo is compiled for using extrae + +Compile_extrae() +{ + #Get flag lines + + line=$(sed -n '/^%FCFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) + line2=$(sed -n '/^%FPPFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) + line3=$(sed -n '/^%LDFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) + + + # If -rdynamic is not there recompilation is requiered and -rdynamic added + + if ! echo "${line}"|grep -q "\-rdynamic\b"; then + echo "-rdynamic flag not found in FCFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " + sed -i '/^%FCFLAGS/ s/$/ -rdynamic/' "${Nemo_path}"/arch/arch-"${arch}".fcm + compile_ext=true + fi + + if ! echo "${line3}"|grep -q "\-export-dynamic\b"; then + echo "-export-dynamic flag not found in LDFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " + sed -i '/^%LDFLAGS/ s/$/ -export-dynamic/' "${Nemo_path}"/arch/arch-"${arch}".fcm + compile_ext=true + fi + + # If finstrument-functions is not there recompilation is requiered and -finstrument-functions added + if ! echo "${line}"|grep -q "\-finstrument-functions\b"; then + echo "-finstrument-functions flag not found in arch-${arch}.fcm: editing arch-${arch}.fcm " + sed -i '/^%FCFLAGS/ s/$/ -finstrument-functions/' "${Nemo_path}"/arch/arch-"${arch}".fcm + compile_ext=true + fi + + # If -g is not there, recompilation is requiered and -g added + if ! echo "${line}"|grep -q "\-g\b"; then + echo "-g flag not found in arch-${arch}.fcm: editing arch-${arch}.fcm " + sed -i '/^%FCFLAGS/ s/$/ -g /' "${Nemo_path}"/arch/arch-"${arch}".fcm + compile_ext=true + fi + + + if [ "$compile" == true ]; then + echo 'compile parameter is inicialized true' + fi + + # If -pg is there recompilation is requiered and -pg removed + + sed -i 's/-pg//g' "${Nemo_path}"/arch/arch-"${arch}".fcm + + if echo "${line}"|grep -q "\-pg\b"; then + echo "-pg flag found in FCFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " + compile_ext=true + fi + + if echo "${line2}"|grep -q "\-pg\b"; then + echo "-pg flag found in FPPFLAGS arch-${arch}.fcm : editing arch-${arch}.fcm " + compile_ext=true + fi + + if echo "${line3}"|grep -q "\-pg\b"; then + echo "-pg flag found in LDFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " + compile_ext=true + fi + + # If nemo executable is not on the run file compile + + if ! test -f "${Nemo_path}/cfgs/${name_cfg}/EXP00/nemo"; then + echo "nemo executable not found in ${name_cfg}" + compile_ext=true + fi + + + if [ "$compile" == true ] || [ "$compile_ext" == true ]; then + + echo "Compiling Nemo for EXTRAE" + echo "Output of the compilation in compile.err and compile.out" + + printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" + python3 Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + + state1=$("$job" --wait compile."$Jobs_scheduler") + echo + Job_completed "$state1" + if [ $Completed == false ]; then + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in compile.err" + echo + exit 1 + else + echo "Nemo compilation successful" + echo + fi + + else + echo "Compilation not needed" + echo + fi + + #Copy all the EXP00 data but don't overwrite namelist just the executable + cd "$dir" || echo "Error original dir doesn't exist" exit + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . + cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo . + +} + +#Check if Nemo is compiled for using Gprof + +Compile_gprof() +{ + + if [ "$compile" == true ]; then + echo 'compile parameter is inicialized true' + fi + + # Checking if Gprof_arch file is present + if ! test -f "${Nemo_path}/arch/arch-${arch}_GPROF.fcm"; then + cp "${Nemo_path}"/arch/arch-"${arch}".fcm "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm + fi + + # Get text lines corresponding to the flags + line=$(sed -n '/^%FCFLAGS /p' "$Nemo_path"/arch/arch-"${arch}"_GPROF.fcm) + line2=$(sed -n '/^%FPPFLAGS /p' "$Nemo_path"/arch/arch-"${arch}"_GPROF.fcm) + line3=$(sed -n '/^%LDFLAGS /p' "$Nemo_path"/arch/arch-"${arch}"_GPROF.fcm) + + # If -g is not there, recompilation is requiered and -g added + if ! echo "${line}"|grep -q "\-g\b"; then + echo "-g flag not found in arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " + sed -i '/^%FCFLAGS/ s/$/ -g /' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm + compile_gprof=true + fi + + # If -pg is not there recompilation is requiered and -pg added + + if ! echo "${line}"|grep -q "\-pg\b"; then + echo "-pg flag not found in FCFLAGS arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " + sed -i '/^%FCFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm + compile_gprof=true + fi + + if ! echo "${line2}"|grep -q "\-pg\b"; then + echo "-pg flag not found in FPPFLAGS arch-${arch}_GPROF.fcm : editing arch-${arch}_GPROF.fcm " + sed -i '/^%FPPFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm + compile_gprof=true + fi + + if ! echo "${line3}"|grep -q "\-pg\b"; then + echo "-pg flag not found in LDFLAGS arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " + sed -i '/^%LDFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm + compile_gprof=true + fi + + # If nemo executable is not on the run file compile + + if ! test -f "${Nemo_path}/cfgs/${name_cfg}_GPROF/EXP00/nemo"; then + echo "nemo executable not found in ${name_cfg}_GPROF" + compile_gprof=true + fi + + if [ "$compile" == true ] || [ "$compile_gprof" == true ]; then + echo "Compiling Nemo for GPROF" + echo "Output of the compilation in compile.err and compile.out" + + printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg" + python3 Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + + state1=$("$job" --wait compile."$Jobs_scheduler") + echo + Job_completed "$state1" + if [ "$Completed" == false ]; then + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in compile.err" + echo + exit 1 + else + echo "Nemo compilation successful" + echo + fi + + else + echo "Compilation not needed" + echo + fi + + #Copy all the EXP00 data but don't overwrite namelist just the executable + cd "$dir" || echo "Error original dir doesn't exist" exit + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/* . + cp "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/nemo . + + + if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then + sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) + fi + + #Solving NEMO input file common errors + + if test -f "weights_core_orca2_bicubic_noc.nc"; then + mv weights_core_orca2_bicubic_noc.nc weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES + fi + + if test -f "weights_core_orca2_bilinear_noc.nc"; then + mv weights_core_orca2_bilinear_noc.nc weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES + fi + +} + + + + +#Init variables with default values in case of missing + Init() { - #Init variables with default values in case of missing Nemo_path="${Nemo_path:-"."}" Nemo_cores="${Nemo_cores:-( 48 )}" Jobs_n_cores="${Jobs_n_cores:-48}" @@ -76,6 +260,8 @@ Init() comp_cfg="${Compilation_sub:-""}" Modules="${Modules:-""}" Nemo_iterations=12 + compile_ext=false + compile_gprof=false } @@ -168,138 +354,21 @@ Test_arguments() } +# Generates a list of Nemo functions -Test_Comp() +Gprof_functions() { - # Checking if compilation is needed - - if [ "$compile" == true ]; then - echo 'compile parameter is inicialized true' - fi - # Get text lines corresponding to the flags - line=$(sed -n '/^%FCFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) - line2=$(sed -n '/^%FPPFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) - line3=$(sed -n '/^%LDFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) - - # If -g is not there, recompilation is requiered and -g added - if ! echo "${line}"|grep -q "\-g\b"; then - echo "-g flag not found in arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%FCFLAGS/ s/$/ -g /' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile=true - fi - - # If finstrument-functions is not there recompilation is requiered and -finstrument-functions added - if ! echo "${line}"|grep -q "\-finstrument-functions\b"; then - echo "-finstrument-functions flag not found in arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%FCFLAGS/ s/$/ -finstrument-functions/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile=true - fi - - # If nemo executable is not on the run file compile - - if ! test -f "${Nemo_path}/cfgs/${name_cfg}/EXP00/nemo"; then - echo "nemo executable not found in cfg" - compile=true - fi - - # If -pg is not there recompilation is requiered and -pg added - - if ! echo "${line}"|grep -q "\-pg\b"; then - echo "-pg flag not found in FCFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%FCFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile=true - fi - - if ! echo "${line2}"|grep -q "\-pg\b"; then - echo "-pg flag not found in FPPFLAGS arch-${arch}.fcm : editing arch-${arch}.fcm " - sed -i '/^%FPPFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile=true - fi - - if ! echo "${line3}"|grep -q "\-pg\b"; then - echo "-pg flag not found in LDFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%LDFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile=true - fi - - - # If -rdynamic is not there recompilation is requiered and -rdynamic added - - if ! echo "${line}"|grep -q "\-rdynamic\b"; then - echo "-rdynamic flag not found in FCFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%FCFLAGS/ s/$/ -rdynamic/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile=true - fi - - if ! echo "${line3}"|grep -q "\-export-dynamic\b"; then - echo "-export-dynamic flag not found in LDFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%LDFLAGS/ s/$/ -export-dynamic/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile=true - fi - - - - #Compile the program if needed - - if [ $compile == true ]; then - echo "Compiling Nemo, expected duration 35m" - echo "Output of the compilation in compile.err and compile.out" + #Generating function list in case of missing + if ! test -f "extrae_functions.txt"; then - printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" - python3 Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + Compile_gprof - state1=$("$job" --wait compile."$Jobs_scheduler") - echo - Job_completed "$state1" - if [ $Completed == false ]; then - echo "Nemo compilation failed, remember to load all the needed modules. Check the details in compile.err" - echo - exit 1 - else - echo "Nemo compilation successful" - echo - fi - else - echo "Compilation not needed" - echo - fi - - - #Copy all the EXP00 data but don't overwrite namelist just the executable - cd "$dir" || echo "Error original dir doesn't exist" exit - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo . - - if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then - sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) - fi - - #Solving NEMO input file common errors - - if test -f "weights_core_orca2_bicubic_noc.nc"; then - mv weights_core_orca2_bicubic_noc.nc weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES - fi - - if test -f "weights_core_orca2_bilinear_noc.nc"; then - mv weights_core_orca2_bilinear_noc.nc weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES - fi - - -} - -Create_metrics() -{ - - - #Changing iterations, big traces generate problems. - sed -i "s|nn_itend * =.*|nn_itend = $Nemo_iterations ! last time step (std 5475)|g" namelist_cfg - - #Generating function list in case of missing + #Changing iterations, big traces generate problems. + sed -i "s|nn_itend * =.*|nn_itend = $Nemo_iterations ! last time step (std 5475)|g" namelist_cfg - if ! test -f "extrae_functions_for_xml.txt"; then rm gmon* 2> /dev/null echo "Runing Nemo with 2 cores to obtain function data..." echo @@ -330,17 +399,28 @@ Create_metrics() echo fi - ./extraf.sh nemo extrae_functions.txt + + else - echo "Functions already listed, file extrae_functions_for_xml.txt does exist" + echo "Functions already listed, file extrae_functions.txt does exist" echo + fi - - sed -i "s|list=.*|list=\"${dir}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" extrae.xml - # Run nemo with extrae +} + + +#Gets a trace from Nemo and cuts it to obtain a single timestep + +Get_trace() +{ + +Compile_extrae +# Run nemo with extrae + ./extraf.sh nemo extrae_functions.txt + sed -i "s|list=.*|list=\"${dir}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" extrae.xml for core in "${Nemo_cores[@]}" do @@ -363,7 +443,7 @@ Create_metrics() echo magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > cut_"$core".out 2>&1 if ! ls nemo_"$core".best_cut.prv; then - echo "Cut failed, aborting." + echo "Cut failed, look at cut_$core.out for more info." echo exit 1 fi @@ -374,14 +454,20 @@ Create_metrics() fi cp nemo_"$core".best_cut.* Metrics + done - - # Create performance metrics +} +Create_metrics() +{ + + + + # Create performance metrics echo "Creating metrics and storing theme in Metrics folder" echo - python3 Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "modelfactors.py *" + python3 Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w ".././modelfactors.py -ms 100000 *" mv analysis."$Jobs_scheduler" Metrics cd Metrics||(echo "Error Metrics folder doesn't exists"; exit 1) state5=$("$job" --wait analysis."$Jobs_scheduler") @@ -400,4 +486,33 @@ Create_metrics() echo "------------------------------------------------------------------------------" echo } + +main() +{ + +if [ $# -gt 0 ]; then + + echo "This script does not accept arguments, parameters need to be added to perf_metrics.config Aborting" + exit 1 + +fi + +#Get script directory +dir=$(pwd) + +echo +echo "Using the following configuration:" +echo +source "$dir"/perf_metrics.config +grep -o '^[^#]*' perf_metrics.config +echo + +Init +Test_arguments +Gprof_functions +Get_trace +Create_metrics + +} + main "$@"; exit diff --git a/perf_metrics.config b/perf_metrics.config index 5bcf07a..ebe66fd 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -8,16 +8,16 @@ # get Nemo traces with 4 and 48 cores. 2 different nºcores are needed to obtain scalability data. Nemo_path="../NEMO" -Nemo_cores=( 4 24 48 ) +Nemo_cores=( 4 24 48 96 192) -# Jobs_n_cores: nºcores used for executing other scripts. +# Jobs_n_cores: nºcores used for other jobs like compiling nemo. # Jobs_scheduler: Available (slurm/lsf). # Jobs_time: Max duration of the job in min. # Jobs_queue: Queue used. -Jobs_n_cores=4 +Jobs_n_cores=96 Jobs_scheduler="slurm" -Jobs_time="60" +Jobs_time=60 Jobs_queue=debug # Compilation_compile: When false only compiles NEMO if arch file lacks the needed flags, when true always compiles NEMO. @@ -29,8 +29,8 @@ Jobs_queue=debug Compilation_compile="false" Compilation_ref="ORCA2_ICE_PISCES" -Compilation_arch="X64_MN4" -Compilation_name="ORCA2_EXTRAE" +Compilation_arch="X64_MN4_UNCOUPLED" +Compilation_name="ORCA2_UNCOUPLED" Compilation_sub="OCE del_key 'key_si3 key_top'" # List of modules loaded. diff --git a/plots.py b/plots.py new file mode 100644 index 0000000..0e28cf1 --- /dev/null +++ b/plots.py @@ -0,0 +1,738 @@ +#!/usr/bin/env python3 + +"""Functions to plots efficiencies metrics""" + +from __future__ import print_function, division +import os +from collections import OrderedDict + +# error import variables +error_import_scipy = False +error_import_numpy = False + +try: + import scipy.optimize +except ImportError: + error_import_scipy = True + # print('==ERROR== Could not import SciPy. Please make sure to install a current version.') + +try: + import numpy +except ImportError: + error_import_numpy = True + # print('==ERROR== Could not import NumPy. Please make sure to install a current version.') + + +mod_hybrid_factors_doc = OrderedDict([ + ('hybrid_eff', '-- Hybrid Parallel efficiency'), + ('mpi_parallel_eff', ' -- MPI Parallel efficiency'), + ('mpi_load_balance', ' -- MPI Load balance'), + ('mpi_comm_eff', ' -- MPI Communication efficiency'), + ('serial_eff', ' -- Serialization efficiency'), + ('transfer_eff', ' -- Transfer efficiency'), + ('omp_parallel_eff', ' -- OMP Parallel efficiency'), + ('omp_load_balance', ' -- OMP Load balance'), + ('omp_comm_eff', ' -- OMP Communication efficiency')]) + + +def plot_hybrid_metrics(mod_factors, hybrid_factors, trace_list, trace_processes, trace_tasks, trace_threads, trace_mode, cmdl_args): + """Computes the projection from the gathered model factors and returns the + according dictionary of fitted prediction functions.""" + + global mod_hybrid_factors_doc + + # Update the hybrid parallelism mode + trace_mode_doc = trace_mode[trace_list[0]] + if trace_mode_doc[0:len("Detailed+MPI+")] == "Detailed+MPI+": + mod_hybrid_factors_doc['omp_parallel_eff'] = " -- " + \ + trace_mode_doc[len("Detailed+MPI+"):] + " Parallel efficiency" + mod_hybrid_factors_doc['omp_load_balance'] = " -- " + \ + trace_mode_doc[len("Detailed+MPI+"):] + " Load Balance" + mod_hybrid_factors_doc['omp_comm_eff'] = " -- " + \ + trace_mode_doc[len("Detailed+MPI+"):] + " Communication efficiency" + + if cmdl_args.debug: + print('==DEBUG== Computing projection of model factors.') + + number_traces = len(trace_list) + x_proc = numpy.zeros(number_traces) + y_para = numpy.zeros(number_traces) + y_load = numpy.zeros(number_traces) + y_comm = numpy.zeros(number_traces) + y_comp = numpy.zeros(number_traces) + y_glob = numpy.zeros(number_traces) + y_ipc_scale = numpy.zeros(number_traces) + y_inst_scale = numpy.zeros(number_traces) + y_freq_scale = numpy.zeros(number_traces) + y_hybrid_par = numpy.zeros(number_traces) + y_mpi_par = numpy.zeros(number_traces) + y_mpi_load = numpy.zeros(number_traces) + y_mpi_comm = numpy.zeros(number_traces) + y_comm_serial = numpy.zeros(number_traces) + y_comm_transfer = numpy.zeros(number_traces) + y_omp_par = numpy.zeros(number_traces) + y_omp_load = numpy.zeros(number_traces) + y_omp_comm = numpy.zeros(number_traces) + + #Convert dictionaries to NumPy arrays + for index, trace in enumerate(trace_list): + x_proc[index] = trace_processes[trace] + y_para[index] = mod_factors['parallel_eff'][trace] + y_load[index] = mod_factors['load_balance'][trace] + y_comm[index] = mod_factors['comm_eff'][trace] + y_comp[index] = mod_factors['comp_scale'][trace] + y_glob[index] = mod_factors['global_eff'][trace] + y_ipc_scale[index] = mod_factors['ipc_scale'][trace] + y_inst_scale[index] = mod_factors['inst_scale'][trace] + y_freq_scale[index] = mod_factors['freq_scale'][trace] + y_hybrid_par[index] = hybrid_factors['hybrid_eff'][trace] + y_mpi_par[index] = hybrid_factors['mpi_parallel_eff'][trace] + y_mpi_load[index] = hybrid_factors['mpi_load_balance'][trace] + y_mpi_comm[index] = hybrid_factors['mpi_comm_eff'][trace] + if trace_mode[trace] == 'Detailed+MPI' \ + or trace_mode[trace] == 'Detailed+MPI+OpenMP': + if hybrid_factors['serial_eff'][trace] != 'Non-Avail' and hybrid_factors['serial_eff'][trace] != 'Warning!': + y_comm_serial[index] = hybrid_factors['serial_eff'][trace] + else: + y_comm_serial[index] = 0.0 + if hybrid_factors['transfer_eff'][trace] != 'Non-Avail' \ + and hybrid_factors['transfer_eff'][trace] != 'Warning!': + y_comm_transfer[index] = hybrid_factors['transfer_eff'][trace] + else: + y_comm_transfer[index] = 0.0 + else: + y_comm_serial[index] = 0.0 + y_comm_transfer[index] = 0.0 + y_omp_par[index] = hybrid_factors['omp_parallel_eff'][trace] + y_omp_load[index] = hybrid_factors['omp_load_balance'][trace] + y_omp_comm[index] = hybrid_factors['omp_comm_eff'][trace] + + # Set limit for projection + if cmdl_args.limit: + limit = cmdl_args.limit + else: + limit = str(trace_processes[trace_list[len(trace_list)-1]]) + + limit_min = str(int(x_proc[0])) + # limit_min = str(0) + # To extract the trace name to show in the plots + title_string = "" + for index, trace in enumerate(trace_list): + folder_trace_name = trace.split('/') + trace_name_to_show = folder_trace_name[len(folder_trace_name) - 1] + title_string += '(' + str(index+1) + ') ' + trace_name_to_show + "\\" + 'n' + + title_string += '"' + " noenhanced" + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # Create Gnuplot file for main plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-onlydata.gp') + content = [] + with open(gp_template) as f: + content = f.readlines() + + # Replace xrange + if int(limit) == int(limit_min): + limit_plot = int(limit_min) + 5 * (len(trace_list) - 1) + else: + limit_plot = int(limit) + + # To xticks label + label_xtics = 'set xtics (' + glabel_xtics = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' + if int(limit) == int(limit_min) and same_procs: + proc_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' \ + + '[' + str(index + 1) + ']' + real_procs = str(trace_processes[trace] + index * 5) + elif int(limit) == int(limit_min) and not same_procs: + proc_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' + real_procs = str(trace_processes[trace] + index * 5) + if s_xtics in glabel_xtics: + proc_xtics += '[' + str(index + 1) + ']' + glabel_xtics.append(s_xtics) + else: + proc_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' + real_procs = str(trace_processes[trace]) + if s_xtics in glabel_xtics: + proc_xtics += '[' + str(index + 1) + ']' + glabel_xtics.append(s_xtics) + + label_xtics += '"' + proc_xtics + '" ' + real_procs + ', ' + + content = [line.replace('#REPLACE_BY_XRANGE', '' + .join(['set xrange [', str(limit_min), ':', str(limit_plot), ']'])) for line in content] + content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2] + ') '])) for + line in content] + content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in + content] + + max_global = max([max(y_para), max(y_load), max(y_comm), max(y_comp), max(y_glob)]) + content = [line.replace('#REPLACE_BY_YRANGE', '' + .join(['set yrange [0:', str(max_global+5), ']'])) for line in content] + + file_path = os.path.join(os.getcwd(), 'modelfactors.gp') + with open(file_path, 'w') as f: + f.writelines(content) + + # Add data points to gnuplot file + with open(file_path, 'a') as f: + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index] + index * 5), str(y_para[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_para[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index] + index * 5), str(y_load[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_load[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index] + index * 5), str(y_comm[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comm[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index] + index * 5), str(y_comp[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comp[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index] + index * 5), str(y_glob[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_glob[index]), '\n']) + f.write(line) + f.write('e\n') + + f.write('\n') + f.write('pause -1\n') + + # print('========= Plot (gnuplot File): EFFICIENCY METRICS ==========') + print('Efficiency metrics plot written to ' + file_path) + # print('') + + # Create Gnuplot file for scalability plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-scale.gp') + content = [] + with open(gp_template) as f: + content = f.readlines() + + # Replace xrange + content = [line.replace('#REPLACE_BY_XRANGE', ''.join(['set xrange [',limit_min,':',str(limit_plot),']']) ) for line in content] + content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2]+') '])) for + line in content] + content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in + content] + + max_comp = max([max(y_comp), max(y_ipc_scale), max(y_inst_scale), max(y_freq_scale)]) + content = [line.replace('#REPLACE_BY_YRANGE', '' + .join(['set yrange [0:', str(max_comp+5), ']'])) for line in content] + + file_path = os.path.join(os.getcwd(), 'modelfactors-scale.gp') + with open(file_path, 'w') as f: + f.writelines(content) + + # Add data points to gnuplot file + with open(file_path, 'a') as f: + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comp[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comp[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_ipc_scale[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_ipc_scale[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_inst_scale[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_inst_scale[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_freq_scale[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_freq_scale[index]), '\n']) + f.write(line) + f.write('e\n') + + f.write('\n') + f.write('pause -1\n') + + # print('======== Plot (gnuplot File): SCALABILITY METRICS ========') + print('Scalability metrics plot written to ' + file_path) + + # Create Gnuplot file for hybrid metrics plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-hybrid.gp') + content = [] + with open(gp_template) as f: + content = f.readlines() + + # Replace xrange + content = [line.replace('#REPLACE_BY_XRANGE', ''.join(['set xrange [',limit_min,':',str(limit_plot),']']) ) for line in content] + content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2]+') '])) for + line in content] + content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in + content] + + string_omp =" '-' with linespoints title " + '"' + (mod_hybrid_factors_doc['omp_parallel_eff'])[5:] + '"' + " ls 5,\\" + content = [line.replace('#REPLACE_BY_OMP_PAR_EFF', ''.join([string_omp])) for line in content] + + string_omp = " '-' with linespoints title " + '"' + (mod_hybrid_factors_doc['omp_load_balance'])[9:] + '"' + " ls 6,\\" + content = [line.replace('#REPLACE_BY_OMP_LB', ''.join([string_omp])) for line in content] + + string_omp = " '-' with linespoints title " + '"' + (mod_hybrid_factors_doc['omp_comm_eff'])[9:] + '"' + " ls 7" + content = [line.replace('#REPLACE_BY_OMP_COMM', ''.join([string_omp])) for line in content] + + max_hybrid = max([max(y_hybrid_par), max(y_mpi_par), max(y_mpi_comm), max(y_mpi_load), + max(y_omp_par), max(y_omp_comm), max(y_omp_load)]) + content = [line.replace('#REPLACE_BY_YRANGE', '' + .join(['set yrange [0:', str(max_hybrid+5), ']'])) for line in content] + + file_path = os.path.join(os.getcwd(), 'modelfactors-hybrid.gp') + with open(file_path, 'w') as f: + f.writelines(content) + + # Add data points to gnuplot file + with open(file_path, 'a') as f: + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_hybrid_par[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_hybrid_par[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_par[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_mpi_par[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_load[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_mpi_load[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_comm[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_mpi_comm[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_omp_par[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_omp_par[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_omp_load[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_omp_load[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_omp_comm[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_omp_comm[index]), '\n']) + f.write(line) + f.write('e\n') + + f.write('\n') + f.write('pause -1\n') + + print('Hybrid metrics plot written to ' + file_path) + + # Create Gnuplot file for MPI hybrid metrics plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-mpi-hybrid.gp') + content = [] + with open(gp_template) as f: + content = f.readlines() + + # Replace xrange + content = [line.replace('#REPLACE_BY_XRANGE', ''.join(['set xrange [',limit_min,':',str(limit_plot),']']) ) for line in content] + content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2]+') '])) for + line in content] + content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in + content] + + max_mpi = max([max(y_mpi_par), max(y_mpi_comm), max(y_mpi_load), + max(y_comm_serial), max(y_comm_transfer)]) + content = [line.replace('#REPLACE_BY_YRANGE', '' + .join(['set yrange [0:', str(max_mpi+5), ']'])) for line in content] + + file_path = os.path.join(os.getcwd(), 'modelfactors-mpi-hybrid.gp') + with open(file_path, 'w') as f: + f.writelines(content) + + # Add data points to gnuplot file + with open(file_path, 'a') as f: + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_par[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_mpi_par[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_load[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_mpi_load[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_comm[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_mpi_comm[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if str(y_comm_serial[index]) != 0.0: + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comm_serial[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comm_serial[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if str(y_comm_transfer[index]) != 0.0: + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comm_transfer[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comm_transfer[index]), '\n']) + f.write(line) + f.write('e\n') + + + f.write('\n') + f.write('pause -1\n') + + print('MPI hybrid metrics plot written to ' + file_path) + + +def plot_simple_metrics(mod_factors, trace_list, trace_processes, trace_mode, cmdl_args): + """Computes the projection from the gathered model factors and returns the + according dictionary of fitted prediction functions.""" + + if cmdl_args.debug: + print('==DEBUG== Computing projection of model factors.') + + number_traces = len(trace_list) + x_proc = numpy.zeros(number_traces) + y_para = numpy.zeros(number_traces) + y_load = numpy.zeros(number_traces) + y_comm = numpy.zeros(number_traces) + y_comp = numpy.zeros(number_traces) + y_glob = numpy.zeros(number_traces) + y_comm_serial = numpy.zeros(number_traces) + y_comm_transfer = numpy.zeros(number_traces) + y_ipc_scale = numpy.zeros(number_traces) + y_inst_scale = numpy.zeros(number_traces) + y_freq_scale = numpy.zeros(number_traces) + + #Convert dictionaries to NumPy arrays + for index, trace in enumerate(trace_list): + x_proc[index] = trace_processes[trace] + y_para[index] = mod_factors['parallel_eff'][trace] + y_load[index] = mod_factors['load_balance'][trace] + y_comm[index] = mod_factors['comm_eff'][trace] + y_comp[index] = mod_factors['comp_scale'][trace] + y_glob[index] = mod_factors['global_eff'][trace] + if trace_mode[trace][:5] != 'Burst': + y_ipc_scale[index] = mod_factors['ipc_scale'][trace] + y_inst_scale[index] = mod_factors['inst_scale'][trace] + y_freq_scale[index] = mod_factors['freq_scale'][trace] + else: + y_ipc_scale[index] = 0.0 + y_inst_scale[index] = 0.0 + y_freq_scale[index] = 0.0 + if trace_mode[trace] == 'Detailed+MPI': + if mod_factors['serial_eff'][trace] != 'Warning!': + y_comm_serial[index] = mod_factors['serial_eff'][trace] + else: + y_comm_serial[index] = 0.0 + if mod_factors['transfer_eff'][trace] != 'Warning!': + y_comm_transfer[index] = mod_factors['transfer_eff'][trace] + else: + y_comm_transfer[index] = 0.0 + else: + y_comm_serial[index] = 0.0 + y_comm_transfer[index] = 0.0 + + #Set limit for projection + if cmdl_args.limit: + limit = cmdl_args.limit + else: + limit = str(trace_processes[trace_list[len(trace_list) - 1]]) + + limit_min = str(int(x_proc[0])) + # limit_min = str(0) + # To extract the trace name to show in the plots + title_string = "" + for index, trace in enumerate(trace_list): + folder_trace_name = trace.split('/') + trace_name_to_show = folder_trace_name[len(folder_trace_name) - 1] + title_string += '(' + str(index+1) + ') ' + trace_name_to_show + "\\" + 'n' + title_string += '"' + " noenhanced" + + #Create Gnuplot file for main plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-onlydata.gp') + content = [] + with open(gp_template) as f: + content = f.readlines() + + #Replace xrange + if int(limit) == int(limit_min): + limit_plot = int(limit_min) + 5 * (len(trace_list) - 1 ) + else: + limit_plot = int(limit) + + # To xticks label + label_xtics = 'set xtics (' + for index, trace in enumerate(trace_list): + + if int(limit) == int(limit_min): + label_xtics += '"' + str(trace_processes[trace]) + '[' + str(index + 1) + ']' + '" ' \ + + str(trace_processes[trace] + index * 5) + ', ' + else: + label_xtics += '"' + str(trace_processes[trace]) + '" ' + str(trace_processes[trace]) + ', ' + + content = [line.replace('#REPLACE_BY_XRANGE', '' + .join(['set xrange [',limit_min,':', str(limit_plot),']'])) for line in content] + content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2] + ') '])) for + line in content] + + content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in + content] + + max_global = max([max(y_para), max(y_load), max(y_comm), max(y_comp), max(y_glob)]) + content = [line.replace('#REPLACE_BY_YRANGE', '' + .join(['set yrange [0:', str(max_global+5), ']'])) for line in content] + + file_path = os.path.join(os.getcwd(), 'modelfactors.gp') + with open(file_path, 'w') as f: + f.writelines(content) + + #Add data points to gnuplot file + with open(file_path, 'a') as f: + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_para[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_para[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_load[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_load[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comm[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comm[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comp[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comp[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_glob[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_glob[index]), '\n']) + f.write(line) + f.write('e\n') + + f.write('\n') + f.write('pause -1\n') + + # print('========= Plot (gnuplot File): EFFICIENCY METRICS ==========') + print('Efficiency metrics plot written to ' + file_path) + # print('') + + # Create Gnuplot file for communication plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-comm.gp') + content = [] + with open(gp_template) as f: + content = f.readlines() + + # Replace xrange + content = [line.replace('#REPLACE_BY_XRANGE', ''.join(['set xrange [',limit_min,':',str(limit_plot),']']) ) for line in content] + content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2]+') '])) for + line in content] + + content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in + content] + + max_comm = max([max(y_comm), max(y_comm_serial), max(y_comm_transfer)]) + content = [line.replace('#REPLACE_BY_YRANGE', '' + .join(['set yrange [0:', str(max_comm+5), ']'])) for line in content] + + file_path = os.path.join(os.getcwd(), 'modelfactors-comm.gp') + with open(file_path, 'w') as f: + f.writelines(content) + + # Add data points to gnuplot file + with open(file_path, 'a') as f: + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comm[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comm[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if str(y_comm_serial[index]) != 0.0: + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comm_serial[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comm_serial[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if str(y_comm_transfer[index]) != 0.0: + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comm_transfer[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comm_transfer[index]), '\n']) + f.write(line) + f.write('e\n') + + f.write('\n') + f.write('pause -1\n') + + # print('======= Plot (gnuplot File): COMMUNICATION METRICS ========') + print('Communication Efficiency plot written to ' + file_path) + # print('') + + # Create Gnuplot file for scalability plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-scale.gp') + content = [] + with open(gp_template) as f: + content = f.readlines() + + # Replace xrange + content = [line.replace('#REPLACE_BY_XRANGE', ''.join(['set xrange [',limit_min,':',str(limit_plot),']']) ) for line in content] + content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2]+') '])) for + line in content] + content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in + content] + max_comp = max([max(y_comp), max(y_ipc_scale), max(y_inst_scale), max(y_freq_scale)]) + content = [line.replace('#REPLACE_BY_YRANGE', '' + .join(['set yrange [0:', str(max_comp+5), ']'])) for line in content] + + file_path = os.path.join(os.getcwd(), 'modelfactors-scale.gp') + with open(file_path, 'w') as f: + f.writelines(content) + + # Add data points to gnuplot file + with open(file_path, 'a') as f: + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comp[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comp[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_ipc_scale[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_ipc_scale[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_inst_scale[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_inst_scale[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_freq_scale[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_freq_scale[index]), '\n']) + f.write(line) + f.write('e\n') + + f.write('\n') + f.write('pause -1\n') + + # print('======== Plot (gnuplot File): SCALABILITY METRICS ========') + print('Scalability metrics plot written to ' + file_path) \ No newline at end of file diff --git a/rawdata.py b/rawdata.py new file mode 100644 index 0000000..edbdf4d --- /dev/null +++ b/rawdata.py @@ -0,0 +1,857 @@ +#!/usr/bin/env python3 + +"""Functions to extract rawdata from each trace.""" + +from __future__ import print_function, division +import os +import time +import math +import gzip +import shutil +from collections import OrderedDict +from tracemetadata import human_readable +from utils import run_command, move_files,remove_files, create_temp_folder + + +# Contains all raw data entries with a printable name. +# This is used to generate and print all raw data, so, if an entry is added, it +# should be added here, too. +raw_data_doc = OrderedDict([('runtime', 'Runtime (us)'), + ('runtime_dim', 'Runtime (ideal)'), + ('useful_avg', 'Useful duration (average)'), + ('useful_max', 'Useful duration (maximum)'), + ('useful_tot', 'Useful duration (total)'), + ('useful_dim', 'Useful duration (ideal, max)'), + ('useful_ins', 'Useful instructions (total)'), + ('useful_cyc', 'Useful cycles (total)'), + ('outsidempi_avg', 'Outside MPI duration (average)'), + ('outsidempi_max', 'Outside MPI duration (maximum)'), + ('outsidempi_dim', 'Outside MPI duration (ideal,maximum)'), + ('outsidempi_tot', 'Outside MPI duration (total)'), + ('mpicomm_tot', 'Communication MPI duration (total)'), + ('outsidempi_tot_diff', 'Outside MPI duration rescaled (total*threads)'), + ('flushing_avg', 'Flushing duration (average)'), + ('flushing_max', 'Flushing duration (maximum)'), + ('flushing_tot', 'Flushing duration (total)'), + ('flushing_cyc', 'Flushing cycles (total)'), + ('flushing_ins', 'Flushing instructions (total)'), + ('io_tot', 'Posix I/O duration (total)'), + ('io_max', 'Posix I/O duration (maximum)'), + ('io_avg', 'Posix I/O duration (avg)'), + ('io_std', 'Posix I/O duration (std)'), + ('io_cyc', 'Posix I/O cycles (total)'), + ('io_ins', 'Posix I/O instructions (total)'), + ('useful_plus_io_avg', 'Serial I/O plus useful duration (avg)'), + ('useful_plus_io_max', 'Serial I/O plus useful duration (max)'), + ('io_state_tot', 'state I/O duration (total)'), + ('io_state_avg', 'state I/O duration (avg)'), + ('io_state_max', 'state I/O duration (maximum)'), + ('mpiio_tot', 'MPI I/O duration (total)'), + ('mpiio_max', 'MPI I/O duration (maximum)'), + ('mpiio_avg', 'MPI I/O duration (avg)'), + ('mpiio_std', 'MPI I/O duration (std)'), + ('mpiio_cyc', 'MPI I/O cycles (total)'), + ('mpiio_ins', 'MPI I/O instructions (total)'), + ('burst_useful_tot', 'Burst Useful (total)'), + ('burst_useful_max', 'Burst Useful (max)'), + ('burst_useful_avg', 'Burst Useful (avg)')]) + + +def create_raw_data(trace_list): + """Creates 2D dictionary of the raw input data and initializes with zero. + The raw_data dictionary has the format: [raw data key][trace]. + """ + global raw_data_doc + raw_data = {} + for key in raw_data_doc: + trace_dict = {} + for trace_name in trace_list: + trace_dict[trace_name] = 0.0 + + raw_data[key] = trace_dict + + return raw_data + + +def gather_raw_data(trace_list, trace_processes, trace_task_per_node, trace_mode, cmdl_args): + """Gathers all raw data needed to generate the model factors. Return raw + data in a 2D dictionary """ + raw_data = create_raw_data(trace_list) + global list_mpi_procs_count + list_mpi_procs_count = dict() + + cfgs = {} + cfgs['root_dir'] = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs') + cfgs['timings'] = os.path.join(cfgs['root_dir'], 'timings.cfg') + cfgs['runtime'] = os.path.join(cfgs['root_dir'], 'runtime_app.cfg') + cfgs['cycles'] = os.path.join(cfgs['root_dir'], 'cycles.cfg') + cfgs['instructions'] = os.path.join(cfgs['root_dir'], 'instructions.cfg') + cfgs['flushing'] = os.path.join(cfgs['root_dir'], 'flushing.cfg') + cfgs['mpi_io'] = os.path.join(cfgs['root_dir'], 'mpi-io-reverse.cfg') + cfgs['outside_mpi'] = os.path.join(cfgs['root_dir'], 'mpi-call-outside.cfg') + cfgs['io_call'] = os.path.join(cfgs['root_dir'], 'io-call-reverse.cfg') + cfgs['io_cycles'] = os.path.join(cfgs['root_dir'], 'io-call-cycles.cfg') + cfgs['io_inst'] = os.path.join(cfgs['root_dir'], 'io-call-instructions.cfg') + cfgs['mpiio_cycles'] = os.path.join(cfgs['root_dir'], 'mpi-io-cycles.cfg') + cfgs['mpiio_inst'] = os.path.join(cfgs['root_dir'], 'mpi-io-instructions.cfg') + cfgs['flushing_cycles'] = os.path.join(cfgs['root_dir'], 'flushing-cycles.cfg') + cfgs['flushing_inst'] = os.path.join(cfgs['root_dir'], 'flushing-inst.cfg') + cfgs['burst_useful'] = os.path.join(cfgs['root_dir'], 'burst_useful.cfg') + + # Main loop over all traces + # This can be parallelized: the loop iterations have no dependencies + path_dest = create_temp_folder('scratch_out_basicanalysis', cmdl_args) + + for trace in trace_list: + time_tot = time.time() + if trace[-7:] == ".prv.gz": + trace_name_control = trace[:-7] + trace_name = trace[:-7] + '_' + str(trace_processes[trace]) + 'P' + elif trace[-4:] == ".prv": + trace_name_control = trace[:-4] + trace_name = trace[:-4] + '_' + str(trace_processes[trace]) + 'P' + + line = 'Analyzing ' + os.path.basename(trace) + line += ' (' + str(trace_processes[trace]) + ' processes' + line += ', ' + str(trace_task_per_node[trace]) + ' tasks per node' + line += ', ' + str(trace_mode[trace]) + ' mode' + line += ', ' + human_readable(os.path.getsize(trace)) + ')' + print(line) + + # Create simulated ideal trace with Dimemas + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + time_dim = time.time() + trace_sim = create_ideal_trace(trace, trace_processes[trace], trace_task_per_node[trace], cmdl_args) + trace_name_sim = trace_sim[:-4] + # print(trace_sim) + time_dim = time.time() - time_dim + if not trace_sim == '': + print('Successfully created simulated trace with Dimemas in {0:.1f} seconds.'.format(time_dim)) + else: + print('Failed to create simulated trace with Dimemas.') + + # Run paramedir for the original and simulated trace + time_pmd = time.time() + cmd_normal = ['paramedir', trace] + + cmd_normal.extend([cfgs['timings'], trace_name + '.timings.stats']) + cmd_normal.extend([cfgs['runtime'], trace_name + '.runtime.stats']) + cmd_normal.extend([cfgs['cycles'], trace_name + '.cycles.stats']) + cmd_normal.extend([cfgs['instructions'], trace_name + '.instructions.stats']) + cmd_normal.extend([cfgs['flushing'], trace_name + '.flushing.stats']) + cmd_normal.extend([cfgs['io_call'], trace_name + '.posixio_call.stats']) + cmd_normal.extend([cfgs['io_cycles'], trace_name + '.posixio-cycles.stats']) + cmd_normal.extend([cfgs['io_inst'], trace_name + '.posixio-inst.stats']) + cmd_normal.extend([cfgs['flushing_cycles'], trace_name + '.flushing-cycles.stats']) + cmd_normal.extend([cfgs['flushing_inst'], trace_name + '.flushing-inst.stats']) + + if trace_mode[trace][:12] == 'Detailed+MPI': + cmd_normal.extend([cfgs['mpi_io'], trace_name + '.mpi_io.stats']) + cmd_normal.extend([cfgs['outside_mpi'], trace_name + '.outside_mpi.stats']) + cmd_normal.extend([cfgs['mpiio_cycles'], trace_name + '.mpiio-cycles.stats']) + cmd_normal.extend([cfgs['mpiio_inst'], trace_name + '.mpiio-inst.stats']) + + if trace_mode[trace][:9] == 'Burst+MPI': + cmd_normal.extend([cfgs['burst_useful'], trace_name + '.burst_useful.stats']) + + run_command(cmd_normal, cmdl_args) + + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + cmd_ideal = ['paramedir', trace_sim] + cmd_ideal.extend([cfgs['timings'], trace_name_sim + '.timings.stats']) + cmd_ideal.extend([cfgs['runtime'], trace_name_sim + '.runtime.stats']) + cmd_ideal.extend([cfgs['outside_mpi'], trace_name_sim + '.outside_mpi.stats']) + + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + if not trace_sim == '': + # print(cmd_ideal) + run_command(cmd_ideal, cmdl_args) + + time_pmd = time.time() - time_pmd + + error_timing = 0 + error_counters = 0 + error_ideal = 0 + + # Check if all files are created + if not os.path.exists(trace_name + '.timings.stats') or \ + not os.path.exists(trace_name + '.runtime.stats'): + print('==ERROR== Failed to compute timing information with paramedir.') + error_timing = 1 + + if not os.path.exists(trace_name + '.outside_mpi.stats') and trace_mode[trace][:5] != 'Burst' \ + and 'MPI' in trace_mode[trace]: + print('==ERROR== Failed to compute outside MPI timing information with paramedir.') + error_timing = 1 + + if not os.path.exists(trace_name + '.cycles.stats') or \ + not os.path.exists(trace_name + '.instructions.stats'): + print('==ERROR== Failed to compute counter information with paramedir.') + error_counters = 1 + + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + if not os.path.exists(trace_name_sim + '.timings.stats') or \ + not os.path.exists(trace_name_sim + '.runtime.stats') or \ + not os.path.exists(trace_name_sim + '.outside_mpi.stats'): + print('==ERROR== Failed to compute simulated timing information with paramedir.') + error_ideal = 1 + trace_sim = '' + + if error_timing or error_counters or error_ideal: + print('Failed to analyze trace with paramedir') + else: + print('Successfully analyzed trace with paramedir in {0:.1f} seconds.'.format(time_pmd)) + + # Parse the paramedir output files + time_prs = time.time() + + # Get total, average, and maximum useful duration + if os.path.exists(trace_name + '.timings.stats'): + content = [] + with open(trace_name + '.timings.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['useful_tot'][trace] = float(line.split()[1]) + if line.split()[0] == 'Average': + raw_data['useful_avg'][trace] = float(line.split()[1]) + if line.split()[0] == 'Maximum': + raw_data['useful_max'][trace] = float(line.split()[1]) + else: + raw_data['useful_tot'][trace] = 'NaN' + raw_data['useful_avg'][trace] = 'NaN' + raw_data['useful_max'][trace] = 'NaN' + f.close() + + # Get total File IO, average IO, and maximum IO duration + dict_trace_posixio = {} + if os.path.exists(trace_name + '.posixio_call.stats'): + content = [] + with open(trace_name + '.posixio_call.stats') as f: + content = f.readlines() + + for line in content: + + for field in line.split("\n"): + line_list = field.split("\t") + if "Total" in field.split("\t"): + count_procs = len(line_list[1:]) + list_io_tot = [float(iotime) for iotime in line_list[1:count_procs]] + dict_trace_posixio[trace] = list_io_tot + raw_data['io_tot'][trace] = sum(list_io_tot) + elif "Average" in field.split("\t"): + if count_procs != 0: + raw_data['io_avg'][trace] = float(sum(list_io_tot)/count_procs) + raw_data['io_std'][trace] = math.sqrt(sum([(number - raw_data['io_avg'][trace]) ** 2 + for number in list_io_tot]) + / (len(list_io_tot) - 1)) + else: + raw_data['io_avg'][trace] = 0.0 + raw_data['io_std'][trace] = 0.0 + elif "Maximum" in field.split("\t"): + raw_data['io_max'][trace] = max(list_io_tot) + else: + raw_data['io_tot'][trace] = 0.0 + raw_data['io_avg'][trace] = 0.0 + raw_data['io_max'][trace] = 0.0 + raw_data['io_std'][trace] = 0.0 + f.close() + + # Get total MPI IO, average IO, and maximum IO duration for MPI-IO + dict_trace_mpiio={} + if os.path.exists(trace_name + '.mpi_io.stats') and trace_mode[trace][:12] == 'Detailed+MPI': + content = [] + with open(trace_name + '.mpi_io.stats') as f: + content = f.readlines() + + for line in content: + for field in line.split("\n"): + line_list = field.split("\t") + if "Total" in field.split("\t"): + count_procs = len(line_list[1:]) + list_mpiio_tot = [float(iotime) for iotime in line_list[1:count_procs]] + dict_trace_mpiio[trace] = list_mpiio_tot + # print(list_mpiio_tot) + raw_data['mpiio_tot'][trace] = sum(list_mpiio_tot) + elif "Average" in field.split("\t"): + if count_procs != 0: + raw_data['mpiio_avg'][trace] = sum(list_mpiio_tot)/count_procs + raw_data['mpiio_std'][trace] = math.sqrt(sum([(number - raw_data['mpiio_avg'][trace]) ** 2 + for number in list_mpiio_tot]) + /(len(list_mpiio_tot) - 1)) + else: + raw_data['mpiio_avg'][trace] = 0.0 + raw_data['mpiio_std'][trace] = 0.0 + elif "Maximum" in field.split("\t"): + raw_data['mpiio_max'][trace] = max(list_mpiio_tot) + else: + raw_data['mpiio_tot'][trace] = 0.0 + raw_data['mpiio_avg'][trace] = 0.0 + raw_data['mpiio_max'][trace] = 0.0 + raw_data['mpiio_std'][trace] = 0.0 + f.close() + + # Get total State IO, average IO, and maximum IO duration + #print(dict_trace_mpiio) + if os.path.exists(trace_name + '.timings.stats'): + content = [] + # io_index = " " + with open(trace_name + '.timings.stats') as f: + content = f.readlines() + useful_plus_io = [] + useful_comp = [] + io_time = [] + count_line = 1 + for line in content: + for field in line.split("\n"): + line_list = field.split("\t") + # print(line_list) + if count_line == 1: + try: + if os.path.exists(trace_name_control + '.pcf'): + io_index = line_list.index("I/O") + else: + io_index = line_list.index('Unknown state 12') + except: + io_index = " " + elif io_index != " ": + if "Total" in field.split("\t"): + raw_data['io_state_tot'][trace] = float(line_list[io_index]) + elif "Average" in field.split("\t"): + raw_data['io_state_avg'][trace] = float(line_list[io_index]) + elif "Maximum" in field.split("\t"): + raw_data['io_state_max'][trace] = float(line_list[io_index]) + elif "Minimum" not in line_list and "StDev" not in line_list \ + and "Avg/Max" not in line_list and len(line_list) > 1: + posixio_index = len(useful_plus_io) + # print(dict_trace_posixio[trace][posixio_index]) + if len(dict_trace_posixio) != 0: + sum_aux = float(line_list[1]) + float(dict_trace_posixio[trace][posixio_index]) + else: + sum_aux = float(line_list[1]) + + useful_plus_io.append(float(sum_aux)) + useful_comp.append(float(line_list[1])) + io_time.append(float(line_list[io_index])) + count_line += 1 + if io_index != " ": + if len(useful_plus_io) != 0: + useful_io_avg = float(sum(useful_plus_io) / len(useful_plus_io)) + else: + useful_io_avg = 0.0 + # print(len(useful_plus_io)) + # mpiio is not included in useful + IO + raw_data['useful_plus_io_avg'][trace] = float(useful_io_avg) + raw_data['useful_plus_io_max'][trace] = float(max(useful_plus_io)) + else: + raw_data['useful_plus_io_avg'][trace] = 0.0 + raw_data['useful_plus_io_max'][trace] = 0.0 + raw_data['io_state_tot'][trace] = 0.0 + raw_data['io_state_avg'][trace] = 0.0 + raw_data['io_state_max'][trace] = 0.0 + else: + raw_data['useful_plus_io_avg'][trace] = 'NaN' + raw_data['useful_plus_io_max'][trace] = 'NaN' + raw_data['io_state_tot'][trace] = 'NaN' + raw_data['io_state_avg'][trace] = 'NaN' + raw_data['io_state_max'][trace] = 'NaN' + f.close() + + + # Get runtime + if os.path.exists(trace_name + '.runtime.stats'): + content = [] + with open(trace_name + '.runtime.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Average': + raw_data['runtime'][trace] = float(line.split()[1]) + else: + raw_data['runtime'][trace] = 'NaN' + + # Get total, average, and maximum outside MPI + # list_mpi_procs_count = [] + if os.path.exists(trace_name + '.outside_mpi.stats') and trace_mode[trace][:12] == 'Detailed+MPI': + content = [] + with open(trace_name + '.outside_mpi.stats') as f: + content = f.readlines() + list_outside_mpi = [] + + list_thread_outside_mpi = [] + init_count_thread = False + count_threads = 1 + for line1 in content[1:(len(content) - 8)]: + line = line1.split("\t") + # print(line) + if line: + if line[0] != 'Total' and line[0] != 'Average' \ + and line[0] != 'Maximum' and line[0] != 'StDev' \ + and line[0] != 'Avg/Max': + # To extract the count of MPI tasks + if float(line[1]) != raw_data['runtime'][trace]: + list_outside_mpi.append(float(line[1])) + # To extract the count of threads per MPI task + if len(list_outside_mpi) > 1: + list_thread_outside_mpi.append(count_threads) + count_threads = 1 + else: + if len(list_outside_mpi) == 1 and not init_count_thread: + count_threads = 2 + init_count_thread = True + else: + count_threads += 1 + + list_thread_outside_mpi.append(count_threads) + + count = 0 + equal_threads = True + # Evaluate if the count of threads per MPI task are equal + while count < (len(list_thread_outside_mpi) - 1) and equal_threads: + if list_thread_outside_mpi[count] != list_thread_outside_mpi[count+1]: + equal_threads = False + count += 1 + rescaled_outside_mpi = [] + # This is need to calculate the MPI_Par_Eff with the right #MPI_tasks and #threads + if not equal_threads: + # print("\n===== Different number of threads per mpi task \n") + rescaled_outside_mpi.append(list_outside_mpi[0] * (list_thread_outside_mpi[0])) + # This is the sum of the outsidempi by the threads + sum_outside_mpi_threads = list_thread_outside_mpi[0] + + for i in range(1,len(list_outside_mpi)): + rescaled_outside_mpi.append(list_outside_mpi[i] * (list_thread_outside_mpi[i])) + sum_outside_mpi_threads += list_thread_outside_mpi[i] + raw_data['outsidempi_tot_diff'][trace] = sum(rescaled_outside_mpi) + raw_data['outsidempi_tot'][trace] = sum(list_outside_mpi) + # Only the average is updated with the rescaled outsidempi + if sum(list_thread_outside_mpi) != 0: + raw_data['outsidempi_avg'][trace] = sum(rescaled_outside_mpi) / sum(list_thread_outside_mpi) + else: + raw_data['outsidempi_avg'][trace] = 'NaN' + # Maximum outsidempi is the same, although the count of threads is different + # because it waits that the MPI task has the max outsidempi. + raw_data['outsidempi_max'][trace] = max(list_outside_mpi) + else: + # print("\n===== Equal number of threads per mpi task \n") + raw_data['outsidempi_tot_diff'][trace] = sum(list_outside_mpi) + list_mpi_procs_count[trace] = len(list_outside_mpi) + raw_data['outsidempi_tot'][trace] = sum(list_outside_mpi) + if len(list_outside_mpi) != 0: + raw_data['outsidempi_avg'][trace] = sum(list_outside_mpi) / len(list_outside_mpi) + else: + raw_data['outsidempi_avg'][trace] = 'NaN' + raw_data['outsidempi_max'][trace] = max(list_outside_mpi) + for line2 in content[(len(content) - 7):]: + line_aux = line2.split("\t") + + if line_aux[0] == 'Total': + #print(line_aux) + count_mpiop = len(line_aux[1:]) + list_mpi_tot = [float(mpitime) for mpitime in line_aux[2:count_mpiop]] + #print(list_mpi_tot) + raw_data['mpicomm_tot'][trace] = sum(list_mpi_tot) + else: + raw_data['outsidempi_tot'][trace] = 'NaN' + raw_data['outsidempi_avg'][trace] = 'NaN' + raw_data['outsidempi_max'][trace] = 'NaN' + raw_data['mpicomm_tot'][trace] = 'NaN' + f.close() + # Get total, average, and maximum flushing duration + if os.path.exists(trace_name + '.flushing.stats'): + content = [] + with open(trace_name + '.flushing.stats') as f: + content = f.readlines() + flushing_exist = ('\tBegin\t\n' in content) or ('\tvalue 1\t\n' in content) + + if flushing_exist: + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['flushing_tot'][trace] = float(line.split()[1]) + if line.split()[0] == 'Average': + raw_data['flushing_avg'][trace] = float(line.split()[1]) + if line.split()[0] == 'Maximum': + raw_data['flushing_max'][trace] = float(line.split()[1]) + else: + raw_data['flushing_tot'][trace] = 0.0 + raw_data['flushing_avg'][trace] = 0.0 + raw_data['flushing_max'][trace] = 0.0 + else: + raw_data['flushing_tot'][trace] = 0.0 + raw_data['flushing_avg'][trace] = 0.0 + raw_data['flushing_max'][trace] = 0.0 + + # Get total flushing cycles + if os.path.exists(trace_name + '.flushing-cycles.stats'): + content = [] + with open(trace_name + '.flushing-cycles.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['flushing_cyc'][trace] = int(float(line.split()[1])) + else: + raw_data['flushing_cyc'][trace] = 0.0 + + # Get total flushing instructions + if os.path.exists(trace_name + '.flushing-inst.stats'): + content = [] + with open(trace_name + '.flushing-inst.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['flushing_ins'][trace] = int(float(line.split()[1])) + else: + raw_data['flushing_ins'][trace] = 0.0 + + # Get total posixio cycles + if os.path.exists(trace_name + '.posixio-cycles.stats'): + content = [] + with open(trace_name + '.posixio-cycles.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['io_cyc'][trace] = int(float(line.split()[1])) + else: + raw_data['io_cyc'][trace] = 0.0 + + # Get total posixio instructions + if os.path.exists(trace_name + '.posixio-inst.stats'): + content = [] + with open(trace_name + '.posixio-inst.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['io_ins'][trace] = int(float(line.split()[1])) + else: + raw_data['io_ins'][trace] = 0.0 + + # Get total mpiio instructions + if os.path.exists(trace_name + '.mpiio-cycles.stats') \ + and trace_mode[trace][:12] == 'Detailed+MPI': + content = [] + with open(trace_name + '.mpiio-cycles.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['mpiio_cyc'][trace] = int(float(line.split()[1])) + else: + raw_data['mpiio_cyc'][trace] = 0.0 + + # Get total mpiio instructions + if os.path.exists(trace_name + '.mpiio-inst.stats') \ + and trace_mode[trace][:12] == 'Detailed+MPI': + content = [] + with open(trace_name + '.mpiio-inst.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['mpiio_ins'][trace] = int(float(line.split()[1])) + else: + raw_data['mpiio_ins'][trace] = 0.0 + + # Get useful cycles + if os.path.exists(trace_name + '.cycles.stats'): + content = [] + with open(trace_name + '.cycles.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['useful_cyc'][trace] = int(float(line.split()[1])) + else: + raw_data['useful_cyc'][trace] = 'NaN' + + # Get useful instructions + if os.path.exists(trace_name + '.instructions.stats'): + content = [] + with open(trace_name + '.instructions.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['useful_ins'][trace] = int(float(line.split()[1])) + else: + raw_data['useful_ins'][trace] = 'NaN' + + # Get Efficiencies for BurstMode + if trace_mode[trace] == 'Burst+MPI': + # Get total, maximum and avg from burst useful + if os.path.exists(trace_name + '.burst_useful.stats'): + content = [] + list_burst_tot = [] + with open(trace_name + '.burst_useful.stats') as f: + content = f.readlines() + for line in content: + for field in line.split("\n"): + line_list = field.split("\t") + if "Total" in field.split("\t"): + count_procs = len(line_list[1:]) + list_burst_tot = [float(burst_time) for burst_time in line_list[1:count_procs]] + + raw_data['burst_useful_tot'][trace] = sum(list_burst_tot) + raw_data['burst_useful_avg'][trace] = sum(list_burst_tot)/len(list_burst_tot) + raw_data['burst_useful_max'][trace] = max(list_burst_tot) + + else: + raw_data['burst_useful_avg'][trace] = 'NaN' + raw_data['burst_useful_max'][trace] = 'NaN' + raw_data['burst_useful_tot'][trace] = 'NaN' + else: + raw_data['burst_useful_avg'][trace] = 0.0 + raw_data['burst_useful_max'][trace] = 0.0 + raw_data['burst_useful_tot'][trace] = 0.0 + + # Get timing for SIMULATED traces + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + # Get maximum useful duration for simulated trace + if os.path.exists(trace_name_sim + '.timings.stats'): + content = [] + with open(trace_name_sim + '.timings.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Maximum': + raw_data['useful_dim'][trace] = float(line.split()[1]) + else: + raw_data['useful_dim'][trace] = 'NaN' + + # Get runtime for simulated trace + if os.path.exists(trace_name_sim + '.runtime.stats'): + content = [] + with open(trace_name_sim + '.runtime.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Average': + raw_data['runtime_dim'][trace] = float(line.split()[1]) + else: + raw_data['runtime_dim'][trace] = 'NaN' + + # Get outsideMPI max for simulated trace + if os.path.exists(trace_name_sim + '.outside_mpi.stats'): + with open(trace_name_sim + '.outside_mpi.stats') as f: + content = f.readlines() + list_outside_mpi = [] + list_thread_outside_mpi = [] + init_count_thread = False + count_threads = 1 + for line1 in content[1:(len(content) - 8)]: + line = line1.split("\t") + # print(line) + if line: + if line[0] != 'Total' and line[0] != 'Average' \ + and line[0] != 'Maximum' and line[0] != 'StDev' \ + and line[0] != 'Avg/Max': + # To extract the count of MPI tasks + if float(line[1]) != raw_data['runtime_dim'][trace]: + list_outside_mpi.append(float(line[1])) + # To extract the count of threads per MPI task + if len(list_outside_mpi) > 1: + list_thread_outside_mpi.append(count_threads) + count_threads = 1 + else: + if len(list_outside_mpi) == 1 and not init_count_thread: + count_threads = 2 + init_count_thread = True + else: + count_threads += 1 + + if line[0] == "THREAD 1.1.1": + max_time_outside_mpi = float(line[1]) + + list_thread_outside_mpi.append(count_threads) + if len(list_outside_mpi) != 0: + raw_data['outsidempi_dim'][trace] = max(list_outside_mpi) + else: + raw_data['outsidempi_dim'][trace] = max_time_outside_mpi + else: + raw_data['outsidempi_dim'][trace] = 0.0 + else: + raw_data['useful_dim'][trace] = 'Non-Avail' + raw_data['runtime_dim'][trace] = 'Non-Avail' + raw_data['outsidempi_dim'][trace] = 'Non-Avail' + + # Remove paramedir output files + move_files(trace_name + '.timings.stats', path_dest, cmdl_args) + move_files(trace_name + '.runtime.stats', path_dest, cmdl_args) + move_files(trace_name + '.cycles.stats', path_dest, cmdl_args) + move_files(trace_name + '.instructions.stats', path_dest, cmdl_args) + move_files(trace_name + '.flushing.stats', path_dest, cmdl_args) + move_files(trace_name + '.posixio-cycles.stats', path_dest, cmdl_args) + move_files(trace_name + '.posixio-inst.stats', path_dest, cmdl_args) + move_files(trace_name + '.flushing-cycles.stats', path_dest, cmdl_args) + move_files(trace_name + '.flushing-inst.stats', path_dest, cmdl_args) + + if trace_mode[trace][:12] == 'Detailed+MPI': + move_files(trace_name + '.mpi_io.stats', path_dest, cmdl_args) + move_files(trace_name + '.posixio_call.stats', path_dest, cmdl_args) + move_files(trace_name + '.outside_mpi.stats', path_dest, cmdl_args) + move_files(trace_name + '.mpiio-cycles.stats', path_dest, cmdl_args) + move_files(trace_name + '.mpiio-inst.stats', path_dest, cmdl_args) + + if trace_mode[trace][:9] == 'Burst+MPI': + move_files(trace_name + '.2dh_BurstEff.stats', path_dest, cmdl_args) + move_files(trace_name + '.burst_useful.stats', path_dest, cmdl_args) + + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + move_files(trace_name_sim + '.timings.stats', path_dest, cmdl_args) + move_files(trace_name_sim + '.runtime.stats', path_dest, cmdl_args) + move_files(trace_name_sim + '.outside_mpi.stats', path_dest, cmdl_args) + move_files(trace_sim, path_dest, cmdl_args) + move_files(trace_sim[:-4] + '.pcf', path_dest, cmdl_args) + move_files(trace_sim[:-4] + '.row', path_dest, cmdl_args) + # To move simulation trace and ideal cfg + move_files(trace_sim[:-8] + '.dim', path_dest, cmdl_args) + remove_files(trace_sim[:-8] + '.row', cmdl_args) + remove_files(trace_sim[:-8] + '.pcf', cmdl_args) + move_files(trace_sim[:-8] + '.dimemas_ideal.cfg', path_dest, cmdl_args) + if trace[-7:] == ".prv.gz": + remove_files(trace_name_control + '.prv', cmdl_args) + # move_files(trace_name + '.prv', path_dest, cmdl_args) + + time_prs = time.time() - time_prs + + time_tot = time.time() - time_tot + print('Finished successfully in {0:.1f} seconds.'.format(time_tot)) + print('') + + return raw_data, list_mpi_procs_count + + +def create_ideal_trace(trace, processes, task_per_node, cmdl_args): + """Runs prv2dim and dimemas with ideal configuration for given trace.""" + if trace[-4:] == ".prv": + trace_dim = trace[:-4] + '_' + str(processes) + 'P' + '.dim' + trace_sim = trace[:-4] + '_' + str(processes) + 'P' + '.sim.prv' + trace_name = trace[:-4] + cmd = ['prv2dim', trace, trace_dim] + elif trace[-7:] == ".prv.gz": + with gzip.open(trace, 'rb') as f_in: + with open(trace[:-7] + '.prv', 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + trace_dim = trace[:-7] + '_' + str(processes) + 'P' + '.dim' + trace_sim = trace[:-7] + '_' + str(processes) + 'P' + '.sim.prv' + trace_name = trace[:-7] + trace_unzip = trace[:-3] + cmd = ['prv2dim', trace_unzip, trace_dim] + + run_command(cmd, cmdl_args) + + if os.path.isfile(trace_dim): + if cmdl_args.debug: + print('==DEBUG== Created file ' + trace_dim) + else: + print('==Error== ' + trace_dim + 'could not be creaeted.') + return + + # Create Dimemas configuration + cfg_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs') + + content = [] + with open(os.path.join(cfg_dir, 'dimemas_ideal.cfg')) as f: + content = f.readlines() + + content = [line.replace('REPLACE_BY_CPUS_PER_NODE', str(task_per_node)) for line in content] + content = [line.replace('REPLACE_BY_NTASKS', str(processes)) for line in content] + content = [line.replace('REPLACE_BY_COLLECTIVES_PATH', os.path.join(cfg_dir, 'dimemas.collectives')) for line in + content] + + with open(trace_name + '_' + str(processes) + 'P' + '.dimemas_ideal.cfg', 'w') as f: + f.writelines(content) + + cmd = ['Dimemas', '-S', '32k', '--dim', trace_dim, '-p', trace_sim, trace_name + '_' + str(processes) + + 'P' + '.dimemas_ideal.cfg'] + run_command(cmd, cmdl_args) + + if os.path.isfile(trace_sim): + if cmdl_args.debug: + print('==DEBUG== Created file ' + trace_sim) + return trace_sim + else: + print('==Error== ' + trace_sim + ' could not be created.') + return '' + + +def print_raw_data_table(raw_data, trace_list, trace_processes): + """Prints the raw data table in human readable form on stdout.""" + global raw_data_doc + + print('Overview of the collected raw data:') + + longest_name = len(sorted(raw_data_doc.values(), key=len)[-1]) + + line = ''.rjust(longest_name) + for trace in trace_list: + line += ' | ' + line += str(trace_processes[trace]).rjust(15) + print(''.ljust(len(line), '-')) + print(line) + + print(''.ljust(len(line), '-')) + final_line_raw_data = ''.ljust(len(line), '-') + + for data_key in raw_data_doc: + line = raw_data_doc[data_key].ljust(longest_name) + for trace in trace_list: + line += ' | ' + if raw_data[data_key][trace] != "Non-Avail" and raw_data[data_key][trace] != 'NaN': + line += str(round((raw_data[data_key][trace]),2)).rjust(15) + else: + line += str(raw_data[data_key][trace]).rjust(15) + print(line) + print(final_line_raw_data) + print('') + + +def print_raw_data_csv(raw_data, trace_list, trace_processes): + """Prints the model factors table in a csv file.""" + global raw_data_doc + + delimiter = ';' + # File is stored in the trace directory + # file_path = os.path.join(os.path.dirname(os.path.realpath(trace_list[0])), 'modelfactors.csv') + # File is stored in the execution directory + file_path = os.path.join(os.getcwd(), 'rawdata.csv') + + with open(file_path, 'w') as output: + line = 'Number of processes' + for trace in trace_list: + line += delimiter + line += str(trace_processes[trace]) + output.write(line + '\n') + + for raw_key in raw_data_doc: + line = '#' + raw_data_doc[raw_key] + for trace in trace_list: + line += delimiter + try: # except NaN + line += '{0:.2f}'.format(raw_data[raw_key][trace]) + except ValueError: + line += '{}'.format(raw_data[raw_key][trace]) + output.write(line + '\n') + + print('======== Output Files: Traces raw data and intermediate data ========') + print('Raw data written to ' + file_path) + file_path_intermediate = os.path.join(os.getcwd(), 'scratch_out_basicanalysis') + print('Intermediate file written to ' + file_path_intermediate) + print('') \ No newline at end of file diff --git a/simplemetrics.py b/simplemetrics.py new file mode 100644 index 0000000..fcf0bf8 --- /dev/null +++ b/simplemetrics.py @@ -0,0 +1,1249 @@ +#!/usr/bin/env python3 + +"""Functions to compute the model metrics.""" + +from __future__ import print_function, division +import sys + +from rawdata import * +from collections import OrderedDict + +# error import variables +error_import_pandas = False +error_import_seaborn = False +error_import_matplotlib = False +error_import_numpy = False + +try: + import numpy as np +except ImportError: + error_import_numpy = True + + +try: + import pandas as pd +except ImportError: + error_import_pandas = True + +try: + import seaborn as sns +except ImportError: + error_import_seaborn = True + + +try: + import matplotlib.pyplot as plt +except ImportError: + error_import_matplotlib = True + + + +# Contains all model factor entries with a printable name. +# This is used to generate and print all model factors, so, if an entry is added, +# it should be added here, too. + +other_metrics_doc = OrderedDict([('elapsed_time', 'Elapsed time (sec)'), + ('efficiency', 'Efficiency'), + ('speedup', 'Speedup'), + ('ipc', 'Average IPC'), + ('freq', 'Average frequency (GHz)'), + ('flushing', 'Flushing (%)'), + ('io_mpiio', 'MPI I/O (%)'), + ('io_posix', 'POSIX I/O (%)'), + ('io_eff', 'I/O Efficiency (%)')]) + +mod_factors_doc = OrderedDict([('global_eff', 'Global efficiency'), + ('parallel_eff', '-- Parallel efficiency'), + ('load_balance', ' -- Load balance'), + ('comm_eff', ' -- Communication efficiency'), + ('serial_eff', ' -- Serialization efficiency'), + ('transfer_eff', ' -- Transfer efficiency'), + ('comp_scale', '-- Computation scalability'), + ('ipc_scale', ' -- IPC scalability'), + ('inst_scale', ' -- Instruction scalability'), + ('freq_scale', ' -- Frequency scalability')]) + +mod_factors_scale_plus_io_doc = OrderedDict([('comp_scale', '-- Computation scalability + I/O'), + ('ipc_scale', ' -- IPC scalability'), + ('inst_scale', ' -- Instruction scalability'), + ('freq_scale', ' -- Frequency scalability')]) + + +def create_mod_factors(trace_list): + """Creates 2D dictionary of the model factors and initializes with an empty + string. The mod_factors dictionary has the format: [mod factor key][trace]. + """ + global mod_factors_doc + mod_factors = {} + for key in mod_factors_doc: + trace_dict = {} + for trace_name in trace_list: + trace_dict[trace_name] = 0.0 + mod_factors[key] = trace_dict + + return mod_factors + + +def create_mod_factors_scale_io(trace_list): + """Creates 2D dictionary of the model factors and initializes with an empty + string. The mod_factors dictionary has the format: [mod factor key][trace]. + """ + global mod_factors_scale_plus_io_doc + mod_factors_scale_plus_io = {} + for key in mod_factors_scale_plus_io_doc: + trace_dict = {} + for trace_name in trace_list: + trace_dict[trace_name] = 0.0 + mod_factors_scale_plus_io[key] = trace_dict + + return mod_factors_scale_plus_io + + +def create_other_metrics(trace_list): + """Creates 2D dictionary of the model factors and initializes with an empty + string. The mod_factors dictionary has the format: [mod factor key][trace]. + """ + global other_metrics_doc + other_metrics = {} + for key in other_metrics_doc: + trace_dict = {} + for trace_name in trace_list: + trace_dict[trace_name] = 0.0 + other_metrics[key] = trace_dict + + return other_metrics + + +def get_scaling_type(raw_data, trace_list, trace_processes, cmdl_args): + """Guess the scaling type (weak/strong) based on the useful instructions. + Computes the normalized instruction ratio for all measurements, whereas the + normalized instruction ratio is (instructions ratio / process ratio) with + the smallest run as reference. For exact weak scaling the normalized ratio + should be exactly 1 and for exact strong scaling it should be close to zero + with an upper bound of 0.5. The eps value defines the threshold to be + considered weak scaling and should give enough buffer to safely handle + non-ideal scaling. + """ + eps = 0.9 + normalized_inst_ratio = 0 + + # Check if there is only one trace. + if len(trace_list) == 1: + return 'strong' + + for trace in trace_list: + try: # except NaN + inst_ratio = float(raw_data['useful_ins'][trace]) / float(raw_data['useful_ins'][trace_list[0]]) + except: + inst_ratio = 0.0 + try: # except NaN + proc_ratio = float(trace_processes[trace]) / float(trace_processes[trace_list[0]]) + except: + proc_ratio = 'NaN' + + normalized_inst_ratio += inst_ratio / proc_ratio + + # Get the average inst increase. Ignore ratio of first trace 1.0) + normalized_inst_ratio = (normalized_inst_ratio - 1) / (len(trace_list) - 1) + + scaling_computed = '' + + if normalized_inst_ratio > eps: + scaling_computed = 'weak' + else: + scaling_computed = 'strong' + + if cmdl_args.scaling == 'auto': + if cmdl_args.debug: + print('==DEBUG== Detected ' + scaling_computed + ' scaling.') + print('') + return scaling_computed + + if cmdl_args.scaling == 'weak': + if scaling_computed == 'strong': + print('==Warning== Scaling set to weak scaling but detected strong scaling.') + print('') + return 'weak' + + if cmdl_args.scaling == 'strong': + if scaling_computed == 'weak': + print('==Warning== Scaling set to strong scaling but detected weak scaling.') + print('') + return 'strong' + + print('==Error== reached undefined control flow state.') + sys.exit(1) + + +def compute_model_factors(raw_data, trace_list, trace_processes, trace_mode, list_mpi_procs_count, cmdl_args): + """Computes the model factors from the gathered raw data and returns the + according dictionary of model factors.""" + mod_factors = create_mod_factors(trace_list) + other_metrics = create_other_metrics(trace_list) + mod_factors_scale_plus_io = create_mod_factors_scale_io(trace_list) + + # Guess the weak or strong scaling + scaling = get_scaling_type(raw_data, trace_list, trace_processes, cmdl_args) + + # Loop over all traces + for trace in trace_list: + + proc_ratio = float(trace_processes[trace]) / float(trace_processes[trace_list[0]]) + + # Flushing measurements + try: # except NaN + other_metrics['flushing'][trace] = raw_data['flushing_tot'][trace] \ + / (raw_data['runtime'][trace] * trace_processes[trace]) * 100.0 + except: + other_metrics['flushing'][trace] = 0.0 + + # I/O measurements + try: # except NaN + other_metrics['io_mpiio'][trace] = raw_data['mpiio_tot'][trace] \ + / (raw_data['runtime'][trace] * trace_processes[trace]) * 100.0 + except: + other_metrics['io_mpiio'][trace] = 0.0 + + try: # except NaN + other_metrics['io_posix'][trace] = raw_data['io_tot'][trace] \ + / (raw_data['runtime'][trace] * trace_processes[trace]) * 100.0 + except: + other_metrics['io_posix'][trace] = 0.0 + try: # except NaN + io_total = raw_data['mpiio_tot'][trace] + raw_data['flushing_tot'][trace] + raw_data['io_tot'][trace] + other_metrics['io_eff'][trace] = raw_data['useful_tot'][trace] \ + / (raw_data['useful_tot'][trace] + io_total) * 100.0 + except: + other_metrics['io_eff'][trace] = 0.0 + + # Basic efficiency factors + try: # except NaN + if trace_mode[trace] == 'Burst+MPI': + mod_factors['load_balance'][trace] = float(raw_data['burst_useful_avg'][trace]) \ + / float(raw_data['burst_useful_max'][trace]) * 100.0 + else: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: + mod_factors['load_balance'][trace] = raw_data['useful_plus_io_avg'][trace] \ + / raw_data['useful_plus_io_max'][trace] * 100.0 + else: + mod_factors['load_balance'][trace] = raw_data['useful_avg'][trace] \ + / raw_data['useful_max'][trace] * 100.0 + except: + mod_factors['load_balance'][trace] = 'NaN' + + try: # except NaN + if trace_mode[trace] == 'Burst+MPI': + mod_factors['comm_eff'][trace] = float(raw_data['burst_useful_max'][trace]) \ + / raw_data['runtime'][trace] * 100.0 + else: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: + mod_factors['comm_eff'][trace] = raw_data['useful_plus_io_max'][trace] \ + / raw_data['runtime'][trace] * 100.0 + else: + mod_factors['comm_eff'][trace] = raw_data['useful_max'][trace] \ + / raw_data['runtime'][trace] * 100.0 + except: + mod_factors['comm_eff'][trace] = 'NaN' + + try: # except NaN + mod_factors['serial_eff'][trace] = float(raw_data['outsidempi_dim'][trace]) \ + / float(raw_data['runtime_dim'][trace]) * 100.0 + if mod_factors['serial_eff'][trace] > 100.0: + mod_factors['serial_eff'][trace] = 'Warning!' + except: + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + mod_factors['serial_eff'][trace] = 'NaN' + else: + mod_factors['serial_eff'][trace] = 'Non-Avail' + + try: # except NaN + if mod_factors['serial_eff'][trace] != 'Warning!': + mod_factors['transfer_eff'][trace] = mod_factors['comm_eff'][trace] \ + / mod_factors['serial_eff'][trace] * 100.0 + else: + mod_factors['transfer_eff'][trace] = float(raw_data['runtime_dim'][trace]) \ + / float(raw_data['runtime'][trace]) * 100.0 + + if mod_factors['transfer_eff'][trace] > 100.0: + mod_factors['transfer_eff'][trace] = 'Warning!' + except: + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + mod_factors['transfer_eff'][trace] = 'NaN' + else: + mod_factors['transfer_eff'][trace] = 'Non-Avail' + + # Parallel Efficiency + try: # except NaN + if trace_mode[trace] == 'Burst+MPI': + mod_factors['parallel_eff'][trace] = float(raw_data['burst_useful_avg'][trace]) \ + / float(raw_data['runtime'][trace]) * 100.0 + else: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: + mod_factors['parallel_eff'][trace] = float(raw_data['useful_plus_io_avg'][trace]) \ + / float(raw_data['runtime'][trace]) * 100.0 + else: + mod_factors['parallel_eff'][trace] = mod_factors['load_balance'][trace] \ + * mod_factors['comm_eff'][trace] / 100.0 + except: + mod_factors['parallel_eff'][trace] = 'NaN' + + # Computation Scale only useful computation + try: # except NaN + if len(trace_list) > 1: + if trace_mode[trace] == 'Burst+MPI': + if scaling == 'strong': + mod_factors['comp_scale'][trace] = raw_data['burst_useful_tot'][trace_list[0]] \ + / raw_data['burst_useful_tot'][trace] * 100.0 + else: + mod_factors['comp_scale'][trace] = raw_data['burst_useful_tot'][trace_list[0]] \ + / raw_data['burst_useful_tot'][trace] \ + * proc_ratio * 100.0 + else: + if scaling == 'strong': + mod_factors['comp_scale'][trace] = raw_data['useful_tot'][trace_list[0]] \ + / raw_data['useful_tot'][trace] * 100.0 + else: + mod_factors['comp_scale'][trace] = raw_data['useful_tot'][trace_list[0]] \ + / raw_data['useful_tot'][trace] \ + * proc_ratio * 100.0 + else: + mod_factors['comp_scale'][trace] = 'Non-Avail' + except: + mod_factors['comp_scale'][trace] = 'NaN' + + # Computation Scale + Serial I/O + try: # except NaN + if len(trace_list) > 1: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: + io_serial_0 = raw_data['io_tot'][trace_list[0]] + raw_data['flushing_tot'][trace_list[0]] + io_serial_n = raw_data['io_tot'][trace] + raw_data['flushing_tot'][trace] + if scaling == 'strong': + mod_factors_scale_plus_io['comp_scale'][trace] = (raw_data['useful_tot'][trace_list[0]] + + io_serial_0) / (raw_data['useful_tot'][trace] + + io_serial_n) * 100.0 + else: + mod_factors_scale_plus_io['comp_scale'][trace] = (raw_data['useful_tot'][trace_list[0]] + + io_serial_0) \ + / (raw_data['useful_tot'][trace] + io_serial_n) \ + * proc_ratio * 100.0 + else: + mod_factors_scale_plus_io['comp_scale'][trace] = mod_factors['comp_scale'][trace] + else: + mod_factors_scale_plus_io['comp_scale'][trace] = 'Non-Avail' + except: + mod_factors_scale_plus_io['comp_scale'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + mod_factors['global_eff'][trace] = mod_factors['parallel_eff'][trace] \ + * mod_factors['comp_scale'][trace] / 100.0 + else: + mod_factors['global_eff'][trace] = mod_factors['parallel_eff'][trace] \ + * 100.0 / 100.0 + except: + mod_factors['global_eff'][trace] = 'NaN' + + # Basic scalability factors + try: # except NaN + other_metrics['ipc'][trace] = float(raw_data['useful_ins'][trace]) \ + / float(raw_data['useful_cyc'][trace]) + except: + other_metrics['ipc'][trace] = 'NaN' + try: # except NaN + if len(trace_list) > 1: + if trace_mode[trace][:5] != 'Burst': + mod_factors['ipc_scale'][trace] = other_metrics['ipc'][trace] \ + / other_metrics['ipc'][trace_list[0]] * 100.0 + else: + mod_factors['ipc_scale'][trace] = 'Non-Avail' + else: + mod_factors['ipc_scale'][trace] = 'Non-Avail' + except: + mod_factors['ipc_scale'][trace] = 'NaN' + + # IPC scale + Serial I/O + try: # except NaN + if len(trace_list) > 1: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: + ipc_serial_io_0 = (raw_data['useful_ins'][trace_list[0]] + raw_data['io_ins'][trace_list[0]] + + raw_data['flushing_ins'][trace_list[0]]) \ + / (raw_data['useful_cyc'][trace_list[0]] + raw_data['io_cyc'][trace_list[0]] + + raw_data['flushing_cyc'][trace_list[0]]) + + ipc_serial_io_n = (raw_data['useful_ins'][trace] + raw_data['io_ins'][trace] + + raw_data['flushing_ins'][trace]) \ + / (raw_data['useful_cyc'][trace] + raw_data['io_cyc'][trace] + + raw_data['flushing_cyc'][trace]) + mod_factors_scale_plus_io['ipc_scale'][trace] = ipc_serial_io_n / ipc_serial_io_0 * 100.0 + else: + mod_factors_scale_plus_io['ipc_scale'][trace] = 0.0 + else: + mod_factors_scale_plus_io['ipc_scale'][trace] = 'Non-Avail' + except: + mod_factors_scale_plus_io['ipc_scale'][trace] = mod_factors['ipc_scale'][trace] + + try: # except NaN + other_metrics['freq'][trace] = float(raw_data['useful_cyc'][trace]) \ + / float(raw_data['useful_tot'][trace]) / 1000 + except: + other_metrics['freq'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + if trace_mode[trace][:5] != 'Burst': + mod_factors['freq_scale'][trace] = other_metrics['freq'][trace] \ + / other_metrics['freq'][trace_list[0]] * 100.0 + else: + mod_factors['freq_scale'][trace] = 'Non-Avail' + else: + mod_factors['freq_scale'][trace] = 'Non-Avail' + except: + mod_factors['freq_scale'][trace] = 'NaN' + + # freq scale + Serial I/O + try: # except NaN + if len(trace_list) > 1: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: + freq_serial_io_0 = (float(raw_data['useful_cyc'][trace_list[0]]) + + float(raw_data['io_cyc'][trace_list[0]]) + + float(raw_data['flushing_cyc'][trace_list[0]])) \ + / (float(raw_data['useful_tot'][trace_list[0]]) + + float(raw_data['io_tot'][trace_list[0]]) + + float(raw_data['flushing_tot'][trace_list[0]])) / 1000 + + freq_serial_io_n = (float(raw_data['useful_cyc'][trace]) + float(raw_data['io_cyc'][trace]) + + float(raw_data['flushing_cyc'][trace])) \ + / (float(raw_data['useful_tot'][trace]) + + float(raw_data['io_tot'][trace]) + + float(raw_data['flushing_tot'][trace])) / 1000 + mod_factors_scale_plus_io['freq_scale'][trace] = freq_serial_io_n / freq_serial_io_0 * 100.0 + else: + mod_factors_scale_plus_io['freq_scale'][trace] = mod_factors['freq_scale'][trace] + else: + mod_factors_scale_plus_io['freq_scale'][trace] = 'Non-Avail' + except: + mod_factors_scale_plus_io['freq_scale'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + if trace_mode[trace][:5] != 'Burst': + if scaling == 'strong': + mod_factors['inst_scale'][trace] = float(raw_data['useful_ins'][trace_list[0]]) \ + / float(raw_data['useful_ins'][trace]) * 100.0 + else: + mod_factors['inst_scale'][trace] = float(raw_data['useful_ins'][trace_list[0]]) \ + / float(raw_data['useful_ins'][trace]) \ + * proc_ratio * 100.0 + else: + mod_factors['inst_scale'][trace] = 'Non-Avail' + else: + mod_factors['inst_scale'][trace] = 'Non-Avail' + except: + mod_factors['inst_scale'][trace] = 'NaN' + + # ins scale + Serial I/O + try: # except NaN + if len(trace_list) > 1: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: + useful_ins_plus_io_0 = float(raw_data['useful_ins'][trace_list[0]]) \ + + float(raw_data['io_ins'][trace_list[0]]) \ + + float(raw_data['flushing_ins'][trace_list[0]]) + useful_ins_plus_io_n = float(raw_data['useful_ins'][trace]) \ + + float(raw_data['io_ins'][trace]) \ + + float(raw_data['flushing_ins'][trace]) + if scaling == 'strong': + mod_factors_scale_plus_io['inst_scale'][trace] = useful_ins_plus_io_0 \ + / useful_ins_plus_io_n * 100.0 + else: + mod_factors_scale_plus_io['inst_scale'][trace] = useful_ins_plus_io_0 / useful_ins_plus_io_n \ + * proc_ratio * 100.0 + else: + mod_factors_scale_plus_io['inst_scale'][trace] = mod_factors['inst_scale'][trace] + else: + mod_factors_scale_plus_io['inst_scale'][trace] = 'Non-Avail' + except: + mod_factors_scale_plus_io['inst_scale'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + other_metrics['speedup'][trace] = raw_data['runtime'][trace_list[0]] \ + / raw_data['runtime'][trace] + #if scaling == 'strong': + # other_metrics['speedup'][trace] = raw_data['runtime'][trace_list[0]] \ + # / raw_data['runtime'][trace] + #else: + # other_metrics['speedup'][trace] = raw_data['runtime'][trace_list[0]] \ + # / raw_data['runtime'][trace] * proc_ratio + else: + other_metrics['speedup'][trace] = 'Non-Avail' + except: + other_metrics['speedup'][trace] = 'NaN' + + try: # except NaN + other_metrics['elapsed_time'][trace] = raw_data['runtime'][trace] * 0.000001 + except: + other_metrics['elapsed_time'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + #other_metrics['efficiency'][trace] = other_metrics['speedup'][trace] / proc_ratio + if scaling == 'strong': + other_metrics['efficiency'][trace] = raw_data['runtime'][trace_list[0]] \ + / (raw_data['runtime'][trace] * proc_ratio) + else: + other_metrics['efficiency'][trace] = raw_data['runtime'][trace_list[0]] \ + / raw_data['runtime'][trace] + else: + other_metrics['efficiency'][trace] = 'Non-Avail' + except: + other_metrics['efficiency'][trace] = 'NaN' + + return mod_factors, mod_factors_scale_plus_io, other_metrics + + +def read_mod_factors_csv(cmdl_args): + """Reads the model factors table from a csv file.""" + global mod_factors_doc + + delimiter = ';' + file_path = cmdl_args.project + + # Read csv to list of lines + if os.path.isfile(file_path) and file_path[-4:] == '.csv': + with open(file_path, 'r') as f: + lines = f.readlines() + lines = [line.rstrip('\n') for line in lines] + else: + print('==ERROR==', file_path, 'is not a valid csv file.') + sys.exit(1) + + # Get the number of processes of the traces + processes = lines[0].split(delimiter) + processes.pop(0) + + # Create artificial trace_list and trace_processes + trace_list = [] + trace_processes = dict() + for process in processes: + trace_list.append(process) + trace_processes[process] = int(process) + + # Create empty mod_factors handle + mod_factors = create_mod_factors(trace_list) + + # Get mod_factor_doc keys + mod_factors_keys = list(mod_factors_doc.items()) + + # Iterate over the data lines + for index, line in enumerate(lines[1:len(mod_factors_keys) + 1]): + key = mod_factors_keys[index][0] + line = line.split(delimiter) + for index, trace in enumerate(trace_list): + mod_factors[key][trace] = float(line[index + 1]) + + if cmdl_args.debug: + print_mod_factors_table(mod_factors, trace_list, trace_processes) + + return mod_factors, trace_list, trace_processes + + +def print_mod_factors_table(mod_factors, other_metrics, mod_factors_scale_plus_io, trace_list, trace_processes): + """Prints the model factors table in human readable form on stdout.""" + global mod_factors_doc + global mod_factors_scale_plus_io_doc + + warning_io = [] + warning_flush = [] + warning_flush_wrong = [] + warning_simulation = [] + for trace in trace_list: + if 10.0 <= other_metrics['flushing'][trace] < 15.0: + warning_flush.append(1) + elif other_metrics['flushing'][trace] >= 15.0: + warning_flush_wrong.append(1) + if other_metrics['io_posix'][trace] >= 5.0: + warning_io.append(1) + if mod_factors['serial_eff'][trace] == "Warning!" \ + or mod_factors['transfer_eff'][trace] == "Warning!": + warning_simulation.append(1) + if len(warning_flush_wrong) > 0: + print("WARNING! Flushing in a trace is too high. Disabling standard output metrics...") + print(" Flushing is an overhead due to the tracer, please review your trace.") + print('') + return + + print('Overview of the Efficiency metrics:') + + longest_name = len(sorted(mod_factors_doc.values(), key=len)[-1]) + + line = ''.rjust(longest_name) + + if len(trace_list) == 1: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[0]] + else: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[len(trace_list)-1]] + + # BEGIN To adjust header to big number of processes + procs_header = [] + for index, trace in enumerate(trace_list): + if limit_min == limit_max and len(trace_list) > 1: + procs_header.append(str(trace_processes[trace]) + '[' + str(index+1) + ']') + else: + procs_header.append(str(trace_processes[trace])) + + max_len_header = 0 + for proc_h in procs_header: + if max_len_header < len(proc_h): + max_len_header = len(proc_h) + + value_to_adjust = 10 + if max_len_header > value_to_adjust: + value_to_adjust = max_len_header + 1 + # END To adjust header to big number of processes + + for index, trace in enumerate(trace_list): + line += ' | ' + if limit_min == limit_max and len(trace_list) > 1: + line += (str(trace_processes[trace]) + '[' + str(index+1) + ']').rjust(value_to_adjust) + else: + line += (str(trace_processes[trace])).rjust(value_to_adjust) + + print(''.ljust(len(line), '=')) + print(line) + line_procs_factors = line + + print(''.ljust(len(line), '=')) + + for mod_key in mod_factors_doc: + line = mod_factors_doc[mod_key].ljust(longest_name) + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(mod_factors[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(mod_factors[mod_key][trace])).rjust(value_to_adjust) + print(line) + + # print('') + + io_metrics = [] + io_metrics.append(0) + for mod_key in mod_factors_scale_plus_io_doc: + for trace in trace_list: + if mod_factors_scale_plus_io[mod_key][trace] != 0: + io_metrics.append(1) + + if (len(warning_flush) >= 1 or len(warning_io) >= 1) and len(trace_list) > 1: + print(''.ljust(len(line_procs_factors), '-')) + for mod_key in mod_factors_scale_plus_io_doc: + line = mod_factors_scale_plus_io_doc[mod_key].ljust(longest_name) + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(mod_factors_scale_plus_io[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(mod_factors_scale_plus_io[mod_key][trace])).rjust(value_to_adjust) + print(line) + + print(''.ljust(len(line_procs_factors), '=')) + + if len(warning_simulation) > 0: + print("===> Warning! Metrics obtained from simulated traces exceed 100%. " + "Please review original and simulated traces.") + print('') + + +def print_other_metrics_table(other_metrics, trace_list, trace_processes): + """Prints the model factors table in human readable form on stdout.""" + global other_metrics_doc + + print('Overview of the Speedup, IPC and Frequency:') + + longest_name = len(sorted(other_metrics_doc.values(), key=len)[-1]) + + line = ''.rjust(longest_name) + if len(trace_list) == 1: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[0]] + else: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[len(trace_list)-1]] + + # BEGIN To adjust header to big number of processes + procs_header = [] + for index, trace in enumerate(trace_list): + if limit_min == limit_max and len(trace_list) > 1: + procs_header.append(str(trace_processes[trace]) + '[' + str(index+1) + ']') + else: + procs_header.append(str(trace_processes[trace])) + + max_len_header = 0 + for proc_h in procs_header: + if max_len_header < len(proc_h): + max_len_header = len(proc_h) + + value_to_adjust = 10 + if max_len_header > value_to_adjust: + value_to_adjust = max_len_header + 1 + # END To adjust header to big number of processes + + for index, trace in enumerate(trace_list): + line += ' | ' + if limit_min == limit_max and len(trace_list) > 1: + line += (str(trace_processes[trace]) + '[' + str(index+1) + ']').rjust(value_to_adjust) + else: + line += (str(trace_processes[trace])).rjust(value_to_adjust) + + print(''.ljust(len(line), '-')) + print(line) + line_head = line + print(''.ljust(len(line), '-')) + + for mod_key in other_metrics_doc: + line = other_metrics_doc[mod_key].ljust(longest_name) + if len(trace_list) > 1: + if mod_key in ['speedup', 'ipc', 'freq', 'elapsed_time', 'efficiency']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + print(line) + else: + if mod_key in ['ipc', 'freq', 'elapsed_time']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + print(line) + print(''.ljust(len(line_head), '-')) + # print('') + + warning_io = [] + warning_flush = [] + for trace in trace_list: + if other_metrics['flushing'][trace] >= 10.0: + warning_flush.append(1) + if other_metrics['io_mpiio'][trace] >= 5.0 or other_metrics['io_posix'][trace] >= 5.0: + warning_io.append(1) + + for mod_key in other_metrics_doc: + line = other_metrics_doc[mod_key].ljust(longest_name) + # Print empty line to separate values + if mod_key in ['freq'] and len(warning_flush) > 0: + print("Overview of tracer\'s flushing weight:") + print(''.ljust(len(line_head), '-')) + + if mod_key not in ['speedup', 'ipc', 'freq', 'elapsed_time','efficiency']: + if mod_key in ['flushing']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + if len(warning_flush) > 0: + print(line) + print(''.ljust(len(line_head), '-')) + elif mod_key in ['io_mpiio','io_posix']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + if len(warning_io) > 0: + print(line) + elif mod_key in ['io_eff']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + if len(warning_io) > 0 or len(warning_flush) > 0: + print(line) + + # Print headers I/O + if mod_key in ['flushing'] and len(warning_io) > 0: + print(''.ljust(len(line), ' ')) + print('Overview of File I/O weight:') + print(''.ljust(len(line), '-')) + if mod_key in ['io_eff'] and len(warning_io) > 0: + print(''.ljust(len(line), '-')) + # print('') + + if len(warning_flush) > 0: + message_warning_flush = "WARNING! %Flushing is high and affects computation of efficiency metrics." + else: + message_warning_flush = "" + if len(warning_io) > 0: + message_warning_io = "WARNING! % File I/O is high and affects computation of efficiency metrics." + else: + message_warning_io = "" + print(message_warning_flush + message_warning_io) + print('') + + +def print_efficiency_table(mod_factors, trace_list, trace_processes): + """Prints the model factors table in human readable form on stdout.""" + global mod_factors_doc + + longest_name = len(sorted(mod_factors_doc.values(), key=len)[-1]) + delimiter = ',' + file_path = os.path.join(os.getcwd(), 'efficiency_table.csv') + with open(file_path, 'w') as output: + line = '\"Number of processes\"' + if len(trace_list) == 1: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[0]] + else: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[len(trace_list)-1]] + + for index, trace in enumerate(trace_list): + line += delimiter + if limit_min == limit_max and len(trace_list) > 1: + line += str(trace_processes[trace]) + '[' + str(index+1) + ']' + else: + line += str(trace_processes[trace]) + output.write(line + '\n') + + for mod_key in mod_factors_doc: + if mod_key not in ['speedup', 'ipc', 'freq', 'elapsed_time', 'efficiency', 'flushing', 'io_mpiio', 'io_posix']: + if mod_key in ['parallel_eff', 'comp_scale']: + line = "\"" + mod_factors_doc[mod_key].replace(' ', '', 2) + "\"" + elif mod_key in ['load_balance', 'comm_eff','ipc_scale', 'inst_scale','freq_scale']: + line = "\"" + mod_factors_doc[mod_key].replace(' ', ' ', 2) + "\"" + elif mod_key in ['serial_eff', 'transfer_eff']: + line = "\" " + mod_factors_doc[mod_key].replace(' ', ' ', 4) + "\"" + else: + line = "\"" + mod_factors_doc[mod_key] + "\"" + for trace in trace_list: + line += delimiter + try: # except NaN + if mod_factors[mod_key][trace] == "Non-Avail" or mod_factors[mod_key][trace] == "Warning!": + line += '0.00' + else: + line += '{0:.2f}'.format(mod_factors[mod_key][trace]) + except ValueError: + line += '{}'.format(mod_factors[mod_key][trace]) + output.write(line + '\n') + # print('') + + # Create Gnuplot file for efficiency plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'efficiency_table.gp') + content = [] + + with open(gp_template) as f: + content = f.readlines() + + limit_procs = 500 + len(trace_list) * 60 + + # Replace xrange + content = [line.replace('#REPLACE_BY_SIZE', ''.join(['set terminal pngcairo enhanced dashed crop size ', + str(limit_procs), ',460 font "Latin Modern Roman,14"'])) + for line in content] + + file_path = os.path.join(os.getcwd(), 'efficiency_table.gp') + with open(file_path, 'w') as f: + f.writelines(content) + + # print('======== Plot (gnuplot File): EFFICIENCY Table ========') + if len(trace_list) > 1: + print('Efficiency Table written to ' + file_path[:len(file_path) - 3] + '.gp') + # print('') + + +def print_mod_factors_csv(mod_factors, trace_list, trace_processes): + """Prints the model factors table in a csv file.""" + global mod_factors_doc + + delimiter = ';' + # File is stored in the trace directory + # file_path = os.path.join(os.path.dirname(os.path.realpath(trace_list[0])), 'modelfactors.csv') + # File is stored in the execution directory + file_path = os.path.join(os.getcwd(), 'modelfactors.csv') + + with open(file_path, 'w') as output: + line = "\"Number of processes\"" + for trace in trace_list: + line += delimiter + line += str(trace_processes[trace]) + output.write(line + '\n') + + for mod_key in mod_factors_doc: + line = "\"" + mod_factors_doc[mod_key].replace(' ', '', 2) + "\"" + for trace in trace_list: + line += delimiter + try: # except NaN + line += '{0:.6f}'.format(mod_factors[mod_key][trace]) + except ValueError: + line += '{}'.format(mod_factors[mod_key][trace]) + output.write(line + '\n') + + output.write('#\n') + + print('======== Output Files: Metrics and Plots ========') + print('Model factors written to ' + file_path) + + +def print_other_metrics_csv(other_metrics, trace_list, trace_processes): + """Prints the model factors table in a csv file.""" + global other_metrics_doc + + delimiter = ';' + # File is stored in the trace directory + # file_path = os.path.join(os.path.dirname(os.path.realpath(trace_list[0])), 'modelfactors.csv') + # File is stored in the execution directory + file_path = os.path.join(os.getcwd(), 'other_metrics.csv') + + with open(file_path, 'w') as output: + line = "\"Number of processes\"" + for trace in trace_list: + line += delimiter + line += str(trace_processes[trace]) + output.write(line + '\n') + + for mod_key in other_metrics_doc: + line = "\"" + other_metrics_doc[mod_key].replace(' ', '', 2) +"\"" + for trace in trace_list: + line += delimiter + try: # except NaN + line += '{0:.6f}'.format(other_metrics[mod_key][trace]) + except ValueError: + line += '{}'.format(other_metrics[mod_key][trace]) + output.write(line + '\n') + + output.write('#\n') + + print('======== Output File: Other Metrics ========') + print('Speedup, IPC, Frequency, I/O and Flushing written to ' + file_path) + print('') + + +def plots_efficiency_table_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args): + # Plotting using python + # For plotting using python, read the csv file + + file_path = os.path.join(os.getcwd(), 'efficiency_table.csv') + df = pd.read_csv(file_path) + metrics = df['Number of processes'].tolist() + + traces_procs = list(df.keys())[1:] + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # Set limit for projection + if cmdl_args.limit: + limit = cmdl_args.limit + else: + limit = str(trace_processes[trace_list[len(trace_list)-1]]) + + limit_min = trace_processes[trace_list[0]] + + # To xticks label + label_xtics = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if int(limit) == int(limit_min) and same_procs: + s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' + elif int(limit) == int(limit_min) and not same_procs: + s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' + else: + s_xtics = str(trace_processes[trace]) + if s_xtics in label_xtics: + s_xtics += '[' + str(index + 1) + ']' + label_xtics.append(s_xtics) + ##### End xticks + + # BEGIN To adjust header to big number of processes + max_len_header = 7 + for labelx in label_xtics: + if len(labelx) > max_len_header: + max_len_header = len(labelx) + + # END To adjust header to big number of processes + list_data = [] + for index, rows in df.iterrows(): + list_temp = [] + for value in list(rows)[1:]: + if float(value) == 0.0: + list_temp.append(np.nan) + else: + list_temp.append(float(value)) + # print(list_temp) + list_data.append(list_temp) + + list_np = np.array(list_data) + + idx = metrics + cols = label_xtics + #cols = traces_procs + df = pd.DataFrame(list_np, index=idx, columns=cols) + + # min for 1 traces is x=3 for the (x,y) in figsize + size_figure_y = len(idx) * 0.40 + size_figure_x = len(cols) * 0.16 * max_len_header + plt.figure(figsize=(size_figure_x, size_figure_y)) + + ax = sns.heatmap(df, cmap='RdYlGn', linewidths=0.05, annot=True, vmin=0, vmax=100, center=75, \ + fmt='.2f', annot_kws={"size": 10}, cbar_kws={'label': 'Percentage(%)'}) + ## to align ylabels to left + plt.yticks(rotation=0, ha='left') + ax.xaxis.tick_top() + # to adjust metrics + len_pad = 0 + for metric in metrics: + if len(metric) > len_pad: + len_pad = len(metric) + + ax.yaxis.set_tick_params(pad=len_pad + 120) + plt.savefig('efficiency_table-matplot.png', bbox_inches='tight') + + +def plots_modelfactors_matplot(trace_list, trace_mode, trace_processes, trace_tasks, trace_threads, cmdl_args): + # Plotting using python + # For plotting using python, read the csv file + file_path = os.path.join(os.getcwd(), 'modelfactors.csv') + df = pd.read_csv(file_path, sep=';') + + # metrics = df['Number of processes'].tolist() + + traces_procs = list(df.keys())[1:] + + list_data = [] + for index, rows in df.iterrows(): + list_temp = [] + for value in list(rows)[1:]: + if value != 'Non-Avail' and value != "Warning!": + list_temp.append(float(value)) + elif value == 'Non-Avail' or value == "Warning!": + list_temp.append(float('nan')) + #print(list_temp) + list_data.append(list_temp) + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_tasks[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # Set limit for projection + if cmdl_args.limit: + limit = cmdl_args.limit + else: + limit = str(trace_processes[trace_list[len(trace_list)-1]]) + + limit_min = trace_processes[trace_list[0]] + + # To xticks label + label_xtics = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_tasks[trace] + if int(limit) == int(limit_min) and same_procs: + s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' + elif int(limit) == int(limit_min) and not same_procs: + s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' \ + + str(threads) + ')' + else: + s_xtics = str(trace_processes[trace]) + label_xtics.append(s_xtics) + + ### Plot: Global Metrics + # print(list_data) + plt.figure() + max_global = max([max(list_data[0]), max(list_data[1]), max(list_data[2]), max(list_data[3]), max(list_data[6])]) + plt.plot(traces_procs, list_data[0], 'o-', color='black', label='Global Efficiency') + plt.plot(traces_procs, list_data[1], 's--', color='magenta', label='Parallel Efficiency') + plt.plot(traces_procs, list_data[2], 'X:', color='red', linewidth=2, label='Load Balance') + plt.plot(traces_procs, list_data[3], 'x-.', color='green', label='Communication efficiency') + plt.plot(traces_procs, list_data[6], 'v--', color='blue', label='Computation scalability') + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency (%)") + plt.xticks(tuple(traces_procs), tuple(label_xtics)) + # print(max_global) + if float(max_global) < 100: + max_global = 100 + + plt.ylim(0, float(max_global)+5) + plt.legend() + plt.savefig('modelfactors-matplot.png', bbox_inches='tight') + + # Plot: Comm Metrics + if trace_mode[trace] == 'Detailed+MPI': + plt.figure() + max_comm = max([max(list_data[3]), max(list_data[4]), max(list_data[5])]) + plt.plot(traces_procs, list_data[3], 's-', color='green', label='Communication efficiency') + plt.plot(traces_procs, list_data[4], 'h--', color='gold', label='Serialization efficiency') + plt.plot(traces_procs, list_data[5], 'x:', color='tomato', label='Transfer efficiency') + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency (%)") + plt.xticks(tuple(traces_procs), tuple(label_xtics)) + if float(max_comm) < 100: + max_comm = 100 + + plt.ylim(0, float(max_comm)+5) + plt.legend() + plt.savefig('modelfactors-comm-matplot.png', bbox_inches='tight') + + ### Plot: Scale Metrics + if trace_mode[trace][:5] != 'Burst': + plt.figure() + max_scale = max([max(list_data[6]), max(list_data[7]), max(list_data[8]), max(list_data[9])]) + plt.plot(traces_procs, list_data[6], 'v-', color='blue', label='Computation scalability') + plt.plot(traces_procs, list_data[7], 'v--', color='skyblue', label='IPC scalability') + plt.plot(traces_procs, list_data[8], 'v:', color='gray', label='Instruction scalability') + plt.plot(traces_procs, list_data[9], 'v-.', color='darkviolet', label='Frequency scalability') + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency (%)") + plt.xticks(tuple(traces_procs), tuple(label_xtics)) + if float(max_scale) < 100: + max_scale = 100 + + plt.ylim(0, float(max_scale)+5) + plt.legend() + plt.savefig('modelfactors-scale-matplot.png', bbox_inches='tight') + + +def plots_speedup_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args): + # Plotting using python + # For plotting using python, read the csv file + file_path = os.path.join(os.getcwd(), 'other_metrics.csv') + df = pd.read_csv(file_path, sep=';') + + traces_procs = list(df.keys())[1:] + + list_data = [] + for index, rows in df.iterrows(): + list_temp = [] + for value in list(rows)[1:]: + if value != 'Non-Avail': + list_temp.append(float(value)) + elif value == 'Non-Avail': + list_temp.append(float('nan')) + # print(list_temp) + list_data.append(list_temp) + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # Set limit for projection + if cmdl_args.limit: + limit = cmdl_args.limit + else: + limit = str(trace_processes[trace_list[len(trace_list) - 1]]) + + limit_min = trace_processes[trace_list[0]] + + # proc_ratio to ideal speedup + proc_ratio = [] + for index, trace in enumerate(trace_list): + proc_ratio.append(trace_processes[trace]/trace_processes[trace_list[0]]) + # print(proc_ratio) + + # To xticks label + label_xtics = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if int(limit) == int(limit_min) and same_procs: + s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' + elif int(limit) == int(limit_min) and not same_procs: + s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' \ + + str(threads) + ')' + else: + s_xtics = str(trace_processes[trace]) + label_xtics.append(s_xtics) + + ### Plot: SpeedUp + # print(list_data) + + int_traces_procs = [] + prev_procs = int(float(traces_procs[0])) + int_traces_procs.append(int(float(traces_procs[0]))) + count_rep = 0 + for procs in traces_procs[1:]: + if prev_procs == int(float(procs)): + count_rep += 1 + int_traces_procs.append(int(float(procs)) + (2 * count_rep)) + prev_procs = int(float(procs)) + else: + int_traces_procs.append(int(float(procs))) + prev_procs = int(float(procs)) + count_rep = 0 + + + plt.figure() + for x, y in zip(int_traces_procs, list_data[2]): + label = "{:.2f}".format(y) + plt.annotate(label, (x, y), textcoords="offset points", xytext=(0, 10), ha='center') + plt.plot(int_traces_procs, list_data[2], 'o-', color='blue', label='measured') + plt.plot(int_traces_procs, proc_ratio, 'o-', color='black', label='ideal') + plt.xlabel("Number of Processes") + plt.ylabel("SpeedUp") + plt.xticks(tuple(int_traces_procs), tuple(label_xtics)) + #plt.yscale('log') + plt.legend() + # plt.xlim(0, ) + plt.ylim(0, ) + plt.savefig('speedup-matplot.png', bbox_inches='tight') + + ### Plot: Efficiency + # print(list_data) + plt.figure() + for x, y in zip(int_traces_procs, list_data[1]): + label = "{:.2f}".format(y) + plt.annotate(label, (x, y), textcoords="offset points", xytext=(0, 10), ha='center') + + plt.plot(int_traces_procs, list_data[1], 'o-', color='blue', label='measured') + plt.axhline(y=1, color='black', linestyle='-', label='ideal') + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency") + plt.xticks(tuple(int_traces_procs), tuple(label_xtics)) + # plt.yscale('log') + max_y = max(list_data[1]) + if max_y < 1.1: + max_y = 1.1 + plt.ylim(0,max_y+0.1) + # plt.xlim(0, ) + plt.legend() + plt.savefig('efficiency-matplot.png', bbox_inches='tight') \ No newline at end of file diff --git a/trace.sh b/trace.sh index fa295ae..7c38951 100755 --- a/trace.sh +++ b/trace.sh @@ -4,6 +4,7 @@ export EXTRAE_CONFIG_FILE=./extrae.xml # Load the tracing library (choose C/Fortran) #export LD_PRELOAD=${EXTRAE_HOME}/lib/libmpitrace.so +export EXTRAE_SKIP_AUTO_LIBRARY_INITIALIZE=1 export LD_PRELOAD=${EXTRAE_HOME}/lib/libmpitracecf.so # Run the program $* diff --git a/tracemetadata.py b/tracemetadata.py new file mode 100644 index 0000000..de85bd4 --- /dev/null +++ b/tracemetadata.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python3 + +"""Functions to extract metadata information from each trace.""" + +from __future__ import print_function, division +import os +import time +import sys +import math +import re +import fnmatch +import mmap +import gzip +import multiprocessing +import threading +from multiprocessing import Process + + +def get_traces_from_args(cmdl_args): + """Filters the given list to extract traces, i.e. matching *.prv and sorts + the traces in ascending order based on the number of processes in the trace. + Excludes all files other than *.prv and ignores also simulated traces from + this script, i.e. *.sim.prv + Returns list of trace paths and dictionary with the number of processes. + """ + + def get_processes(prv_file): + return trace_processes[prv_file], trace_tasks[prv_file], trace_threads[prv_file] + + trace_list = [x for x in cmdl_args.trace_list if (fnmatch.fnmatch(x, '*.prv') or fnmatch.fnmatch(x, '*.prv.gz')) + if not fnmatch.fnmatch(x, '*.sim.prv')] + if not trace_list: + print('==Error== could not find any traces matching "', ' '.join(cmdl_args.trace_list)) + sys.exit(1) + + trace_processes = dict() + trace_tasks = dict() + trace_threads = dict() + trace_mode = dict() + trace_task_per_node = dict() + + trace_list_temp = [] + trace_list_removed = [] + for trace in trace_list: + if float(os.path.getsize(trace)/1024/1024) < float(cmdl_args.max_trace_size): + trace_list_temp.append(trace) + else: + trace_list_removed.append(trace) + + if len(trace_list_temp) < 1: + print('==Error== All traces exceed the maximum size (', cmdl_args.max_trace_size, 'MiB)') + for trace_upper in trace_list_removed: + print(trace_upper) + sys.exit(1) + + print("Running modelfactors.py for the following traces list:") + trace_list = trace_list_temp + for trace in trace_list: + print(trace) + + if len(trace_list_removed) > 0: + print("\nFollowing traces were excluded to be analyzed (size >", cmdl_args.max_trace_size, "MiB): ") + for trace in trace_list_removed: + print(trace) + + print('\nExtracting metadata from the traces list.') + # This part could be parallelized + # t1 = time.perf_counter() + for trace in trace_list: + trace_processes[trace], trace_tasks[trace], trace_threads[trace] = get_num_processes(trace) + trace_list = sorted(trace_list, key=get_processes) + + t1 = time.perf_counter() + jobs = [] + manager = multiprocessing.Manager() + trace_mode = manager.dict() + for trace in trace_list: + p_act = Process(target=get_trace_mode, args=(trace, cmdl_args, trace_mode)) + jobs.append(p_act) + p_act.start() + + for p in jobs: + p.join() + + t2 = time.perf_counter() + + print('Successfully Metadata Extraction in {0:.1f} seconds.\n'.format(t2 - t1)) + + for trace in trace_list: + trace_task_per_node[trace] = get_task_per_node(trace) + + print("Starting Analysis for the following sorted traces list:") + print_overview(trace_list, trace_processes, trace_tasks, trace_threads, trace_mode, trace_task_per_node) + return trace_list, trace_processes, trace_tasks, trace_threads, trace_task_per_node, trace_mode + + +def get_num_processes(prv_file): + """Gets the number of processes in a trace from the according .row file. + Please note: return value needs to be integer because this function is also + used as sorting key. + """ + if prv_file[-4:] == ".prv": + tracefile = open(prv_file) + for line in tracefile: + header_trace = line.split('_') + break + tracefile.close() + + if prv_file[-7:] == ".prv.gz": + with gzip.open(prv_file, 'rt') as f: + for line in f: + if "#Paraver" in line: + header_trace = line.split('_') + break + f.close() + + + #print(header_trace) + header_to_print = header_trace[1].split(':')[3].split('(') + tasks = header_to_print[0] + threads = header_to_print[1] + list_procspernode = header_trace[1].split('(')[2].split(')')[0].split(',') + #print(list_procspernode) + total_procs = 0 + for proc_node in list_procspernode: + total_procs += int(proc_node.split(':')[0]) + + return int(total_procs), int(tasks), int(threads) + + +def get_tasks_threads(prv_file): + """Gets the tasks and threads from the .prv file. + """ + if prv_file[-4:] == ".prv": + tracefile = open(prv_file) + for line in tracefile: + header_trace = line.split('_') + break + tracefile.close() + + if prv_file[-7:] == ".prv.gz": + with gzip.open(prv_file, 'rt') as f: + for line in f: + if "#Paraver" in line: + header_trace = line.split('_') + break + f.close() + header_to_print = header_trace[1].split(':')[3].split('(') + tasks = header_to_print[0] + threads = header_to_print[1] + return int(tasks), int(threads) + + +def get_task_per_node(prv_file): + """Gets the number of processes and nodes in a trace from the according .row file. + """ + row_file = True + + if prv_file[-4:] == ".prv": + if os.path.exists(prv_file[:-4] + '.row'): + tracefile = open(prv_file[:-4] + '.row') + else: + tracefile = prv_file[:-4] + row_file = False + elif prv_file[-7:] == ".prv.gz": + if os.path.exists(prv_file[:-7] + '.row'): + tracefile = open(prv_file[:-7] + '.row') + else: + tracefile = prv_file[:-7] + row_file = False + + tasks = 0 + nodes = 1 + if row_file: + for line in tracefile: + if "LEVEL CPU SIZE" in line: + tasks = int(line[15:]) + if "LEVEL NODE SIZE" in line: + nodes = int(line[15:]) + tracefile.close() + task_nodes = math.ceil(int(tasks) / int(nodes)) + else: + if prv_file[-4:] == ".prv": + tracefile = open(prv_file) + for line in tracefile: + header_trace = line.split('_') + break + tracefile.close() + + if prv_file[-7:] == ".prv.gz": + with gzip.open(prv_file, 'rt') as f: + for line in f: + if "#Paraver" in line: + header_trace = line.split('_') + break + f.close() + task_nodes = int(header_trace[1].split(':')[1].split('(')[1].replace(')','').split(',')[0]) + + return int(task_nodes) + + +def get_trace_mode(prv_file, cmdl_args, trace_mode): + """Gets the trace mode by detecting the event 40000018:2 in .prv file + to detect the Burst mode trace in another case is Detailed mode. + 50000001 for MPI, 60000001 for OpenMP, 61000000 for pthreads, 63000001 for CUDA + """ + mode_trace = '' + burst = 0 + pcf_file = True + if prv_file[-4:] == ".prv": + file_pcf = prv_file[:-4] + '.pcf' + tracefile = open(prv_file) + for line in tracefile: + if "40000018:2" in line: + burst = 1 + break + tracefile.close() + if prv_file[-7:] == ".prv.gz": + file_pcf = prv_file[:-7] + '.pcf' + with gzip.open(prv_file, 'rt') as f: + for line in f: + if "40000018:2" in line: + burst = 1 + break + f.close() + + if burst == 1: + mode_trace = 'Burst' + else: + mode_trace = 'Detailed' + + if os.path.exists(file_pcf) and cmdl_args.trace_mode_detection == 'pcf': + with open(file_pcf, 'rb', 0) as file, \ + mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) as s: + if s.find(b' 500000') != -1: + mode_trace += '+MPI' + if s.find(b' 610000') != -1: + mode_trace += '+Pthreads' + elif s.find(b' 60000') != -1: + if not s.find(b' 60000019') == s.find(b' 60000'): + mode_trace += '+OpenMP' + if s.find(b' 630000') != -1 or s.find(b' 631000') != -1 or s.find(b' 632000') != -1: + mode_trace += '+CUDA' + if s.find(b' 9200001') != -1: + mode_trace += '+OmpSs' + if s.find(b' 642000') != -1 or s.find(b' 6400001') != -1 or s.find(b' 641000') != -1: + mode_trace += '+OpenCL' + else: + if s.find(b' 610000') != -1: + mode_trace += '+Pthreads' + elif s.find(b' 60000') != -1: + if not s.find(b' 60000019') == s.find(b' 60000'): + mode_trace += '+OpenMP' + if s.find(b' 630000') != -1 or s.find(b' 631000') != -1 or s.find(b' 632000') != -1: + mode_trace += '+CUDA' + if s.find(b' 9200001') != -1: + mode_trace += '+OmpSs' + if s.find(b' 642000') != -1 or s.find(b' 6400001') != -1 or s.find(b' 641000') != -1: + mode_trace += '+OpenCL' + file.close() + else: + count_mpi = 0 + count_omp = 0 + count_pthreads = 0 + count_cuda = 0 + count_ompss = 0 + count_opencl = 0 + if prv_file[-4:] == ".prv": + with open(prv_file, 'rb', 0) as file, \ + mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_COPY) as s: + # 2:cpu_id:appl_id:task_id:thread_id:time:event_type:event_value + mpi = re.compile(rb'\n2:\w+:\w+:[1-4]:1:\w+:50000\w\w\w:') + omp = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:60000018:') + cuda = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:63\w\w\w\w\w\w:') + pthreads = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:610000\w\w:') + ompss = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:9200001:') + opencl = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:64\w\w\w\w\w\w:') + if mpi.search(s): + count_mpi = 1 + mpi_trace = '+MPI' + if omp.search(s): + count_omp = 1 + omp_trace = '+OpenMP' + elif cuda.search(s): + count_cuda = 1 + cuda_trace = '+CUDA' + elif pthreads.search(s): + count_pthreads = 1 + pthreads_trace = '+Pthreads' + elif ompss.search(s): + count_ompss = 1 + ompss_trace = '+OmpSs' + elif opencl.search(s): + count_opencl = 1 + opencl_trace = '+OpenCL' + file.close() + elif prv_file[-7:] == ".prv.gz": + handle = open(prv_file, "rb") + mapped = mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) + gzfile = gzip.GzipFile(mode="r", fileobj=mapped) + + # 2:cpu_id:appl_id:task_id:thread_id:time:event_type:event_value + mpi = re.compile(rb'\n2:\w+:\w+:[1-4]:1:\w+:50000\w\w\w:') + omp = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:60000018:') + cuda = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:63\w\w\w\w\w\w:') + pthreads = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:610000\w\w:') + ompss = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:9200001:') + opencl = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:64\w\w\w\w\w\w:') + s = gzfile.read() + if mpi.search(s): + count_mpi = 1 + mpi_trace = '+MPI' + if omp.search(s): + count_omp = 1 + omp_trace = '+OpenMP' + elif cuda.search(s): + count_cuda = 1 + cuda_trace = '+CUDA' + elif pthreads.search(s): + count_pthreads = 1 + pthreads_trace = '+Pthreads' + elif ompss.search(s): + count_ompss = 1 + ompss_trace = '+OmpSs' + elif opencl.search(s): + count_opencl = 1 + opencl_trace = '+OpenCL' + + handle.close() + if count_mpi > 0: + mode_trace += mpi_trace + if count_omp > 0: + mode_trace += omp_trace + if count_pthreads > 0: + mode_trace += pthreads_trace + if count_ompss > 0: + mode_trace += ompss_trace + if count_cuda > 0: + mode_trace += cuda_trace + if count_opencl > 0: + mode_trace += opencl_trace + + trace_mode[prv_file] = mode_trace + #return mode_trace + + +def human_readable(size, precision=1): + """Converts a given size in bytes to the value in human readable form.""" + suffixes = ['B', 'KB', 'MB', 'GB', 'TB'] + suffixIndex = 0 + while size > 1024 and suffixIndex < 4: + suffixIndex += 1 + size = size / 1024.0 + return "%.*f%s" % (precision, size, suffixes[suffixIndex]) + + +def print_overview(trace_list, trace_processes, trace_tasks, trace_threads, trace_mode, trace_task_per_node): + """Prints an overview of the traces that will be processed.""" + #print('Running', os.path.basename(__file__), 'for the following traces:') + + file_path = os.path.join(os.getcwd(), 'traces_metadata.txt') + with open(file_path, 'w') as output: + for index, trace in enumerate(trace_list): + line = '[' + str(index+1) + '] ' + trace + + line += ', ' + str(trace_processes[trace]) \ + + '(' + str(trace_tasks[trace]) + 'x' + str(trace_threads[trace]) + ')' + ' processes' + line += ', ' + str(trace_task_per_node[trace]) + ' tasks per node' + line += ', ' + human_readable(os.path.getsize(trace)) + line += ', ' + str(trace_mode[trace]) + ' mode' + print(line) + output.write(line + '\n') + + print('======== Output Files: Traces metadata ========') + print('Traces metadata written to ' + file_path) + print('') \ No newline at end of file diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..e457c32 --- /dev/null +++ b/utils.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 + +"""modelfactors.py utils.""" + +from __future__ import print_function, division +import os +import sys +import subprocess +import tempfile +import argparse +import shutil + + +try: + import scipy.optimize +except ImportError: + print('==Error== Could not import SciPy. Please make sure you have installed SciPy.') + +try: + import numpy +except ImportError: + print('==Error== Could not import NumPy. Please make sure you have installed NumPy.') + +try: + import pandas as pd +except ImportError: + print('==Error== Could not import pandas. Please make sure you have installed pandas.') +try: + import seaborn as sns +except ImportError: + print('==Error== Could not import seaborn. Please make sure you have installed seaborn.') + +try: + import matplotlib.pyplot as plt +except ImportError: + print('==Error== Could not import matplotlib. Please make sure you have installed matplotlib.') + + +__author__ = "Sandra Mendez" +__copyright__ = "Copyright 2019, Barcelona Supercomputing Center (BSC)" +__version_major__ = 0 +__version_minor__ = 3 +__version_micro__ = 7 +__version__ = str(__version_major__) + "." + str(__version_minor__) + "." + str(__version_micro__) + + +def parse_arguments(): + """Parses the command line arguments. + Currently the script only accepts one parameter list, which is the list of + traces that are processed. This can be a regex and only valid trace files + are kept at the end. + """ + parser = argparse.ArgumentParser(description='Generates performance metrics from a set of Paraver traces.') + parser.add_argument('trace_list', nargs='*', + help='list of traces to process. Accepts wild cards and automatically filters for ' + 'valid traces'), + parser.add_argument("-m", "--metrics", choices=['simple', 'hybrid'], default='hybrid', + help='select the kind of efficiency metrics (single parallelism or hybrid, default: hybrid)') + parser.add_argument("-v", "--version", action='version', version='%(prog)s {version}'.format(version=__version__)) + parser.add_argument("-d", "--debug", help="increase output verbosity to debug level", action="store_true") + parser.add_argument("-s", "--scaling", + help="define whether the measurements are weak or strong scaling (default: auto)", + choices=['weak', 'strong', 'auto'], default='auto') + parser.add_argument('--limit', help='limit number of cores for the plots ' + '(default: max processes of the trace list )') + parser.add_argument("-ms", "--max_trace_size", help='set the maximum trace size in MiB allowed.' + ' (default: 1024 MiB )', default=1024.0) + parser.add_argument("-tmd", "--trace_mode_detection", choices=['pcf', 'prv'], default='pcf', + help='select .prv or .pcf file for trace mode detection. ' + 'For customized traces, i.e. cut, filtered and so on, you must select' + ' the .prv file. (default: pcf)') + + if len(sys.argv) == 1: + parser.print_help() + sys.exit(1) + + cmdl_args = parser.parse_args() + + if cmdl_args.debug: + print('==DEBUG== Running in debug mode.') + + return cmdl_args + + +def which(cmd): + """Returns path to cmd in path or None if not available.""" + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + cmd_path = os.path.join(path, cmd) + if os.path.isfile(cmd_path) and os.access(cmd_path, os.X_OK): + return cmd_path + + return None + + +def check_installation(cmdl_args): + """Check if Dimemas and paramedir are in the path.""" + + if not which('Dimemas'): + print('Could not find Dimemas. Please make sure Dimemas is correctly installed and in the path.') + sys.exit(1) + if not which('paramedir'): + print('Could not find paramedir. Please make sure Paraver is correctly installed and in the path.') + sys.exit(1) + + if not which('python3'): + print('==> WARNING!!! It requires python version 3 or higher for a full functionality.') + option_user = input("Do you want to proceed with the analysis? (Yes/No)[Yes]: ").upper() + if option_user == 'NO': + sys.exit(1) + + if cmdl_args.debug: + print('==DEBUG== Using', __file__, __version__) + print('==DEBUG== Using', sys.executable, ".".join(map(str, sys.version_info[:3]))) + + try: + print('==DEBUG== Using', 'SciPy', scipy.__version__) + except NameError: + print('==DEBUG== SciPy not installed.') + + try: + print('==DEBUG== Using', 'NumPy', numpy.__version__) + except NameError: + print('==DEBUG== NumPy not installed.') + + print('==DEBUG== Using', which('Dimemas')) + print('==DEBUG== Using', which('paramedir')) + print('') + + return + + +def run_command(cmd, cmdl_args): + """Runs a command and forwards the return value.""" + if cmdl_args.debug: + print('==DEBUG== Executing:', ' '.join(cmd)) + + # In debug mode, keep the output. Otherwise, redirect it to devnull. + if cmdl_args.debug: + out = tempfile.NamedTemporaryFile(suffix='.out', prefix=cmd[0] + '_', dir='./', delete=False) + err = tempfile.NamedTemporaryFile(suffix='.err', prefix=cmd[0] + '_', dir='./', delete=False) + else: + out = open(os.devnull, 'w') + err = open(os.devnull, 'w') + + return_value = subprocess.call(cmd, stdout=out, stderr=err) + + out.close + err.close + + if return_value == 0: + if cmdl_args.debug: + os.remove(out.name) + os.remove(err.name) + else: + print('==ERROR== ' + ' '.join(cmd) + ' failed with return value ' + str(return_value) + '!') + print('See ' + out.name + ' and ' + err.name + ' for more details.') + + return return_value + + +def create_temp_folder(folder_name, cmdl_args): + path_output_aux = os.getcwd() + '/' + folder_name + + if os.path.exists(path_output_aux): + shutil.rmtree(path_output_aux) + os.makedirs(path_output_aux) + return (path_output_aux) + + +def move_files(path_source, path_dest, cmdl_args): + """Wraps os.remove with a try clause.""" + try: + shutil.move(path_source, path_dest) + except: + if cmdl_args.debug: + print('==DEBUG== Failed to move ' + path_source + '!') + + +def remove_files(path,cmdl_args): + """Wraps os.remove with a try clause.""" + try: + os.remove(path) + except: + if cmdl_args.debug: + print('==DEBUG== Failed to remove ' + path + '!') \ No newline at end of file -- GitLab From f2321fba162eda3a2d71480f9bf365407bb08d86 Mon Sep 17 00:00:00 2001 From: sparonuz Date: Fri, 9 Sep 2022 15:31:18 +0200 Subject: [PATCH 13/40] Update README.md --- README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index beb7052..cd009b6 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,21 @@ # Nemo modelfactors -Script used to get important metrics from NEMO. +This project is intended to compute important performance metrics for a NEMO run. -The script obtains data from a single nemo timestep, so keep in mind that the results of this basic analysis -will only be accurate on longer simulations where the inicialitzation and finalitzation are close to 0% of the time. +The statistic produced is focusing on the timestep loop, so these numbers are not related to the initialization and finalization parts. # Installation and requeriments -This script requires the following BSCTOOLS to be installed, loaded and available through the PATH environment variable. +This script relies on the following tools: * *Extrae (4.0.0 or above)* * *Paraver* * *Dimemas (latest has a bug use 5.4.2-devel instead)* * *Basicanalysis* -*Tools available at https://tools.bsc.es/downloads* +*They can be downloaded at https://tools.bsc.es/downloads* and need to be installed, loaded and available through the PATH environment variable. -Also the different modules needed to compile and execute NEMO should be loaded before the script execution. +Here the list of the modules that need to be loaded before the script execution. * Perl interpreter * Fortran compiler (ifort, gfortran, pgfortran, ftn, …), @@ -32,4 +31,4 @@ Also the different modules needed to compile and execute NEMO should be loaded b ``` ./perf_metrics.bash ``` -* If the script executed without problems the data will be ready at the Metrics folder. +* If the script executes without problems the data will be ready in the Metrics folder. -- GitLab From f01cca5671ecfcffe37758a6d175d2ba58fe0c5c Mon Sep 17 00:00:00 2001 From: cpenadep Date: Mon, 12 Sep 2022 09:39:28 +0200 Subject: [PATCH 14/40] Secondary files moved to subfolder src --- perf_metrics.bash | 20 +++++++++--------- Job_Creator.py => src/Job_Creator.py | 0 src/__pycache__/hybridmetrics.cpython-34.pyc | Bin 0 -> 35256 bytes src/__pycache__/plots.cpython-34.pyc | Bin 0 -> 22771 bytes src/__pycache__/rawdata.cpython-34.pyc | Bin 0 -> 21439 bytes src/__pycache__/simplemetrics.cpython-34.pyc | Bin 0 -> 28847 bytes src/__pycache__/tracemetadata.cpython-34.pyc | Bin 0 -> 10662 bytes src/__pycache__/utils.cpython-34.pyc | Bin 0 -> 6502 bytes {cfgs => src/cfgs}/.directory | 0 {cfgs => src/cfgs}/2dh_BurstEfficiency.cfg | 0 {cfgs => src/cfgs}/barrier-syncr-time.cfg | 0 {cfgs => src/cfgs}/burst_duration.cfg | 0 {cfgs => src/cfgs}/burst_useful.cfg | 0 {cfgs => src/cfgs}/cycles.cfg | 0 {cfgs => src/cfgs}/dimemas.collectives | 0 {cfgs => src/cfgs}/dimemas_ideal.cfg | 0 {cfgs => src/cfgs}/efficiency_table-global.gp | 0 {cfgs => src/cfgs}/efficiency_table-hybrid.gp | 0 {cfgs => src/cfgs}/efficiency_table.gp | 0 {cfgs => src/cfgs}/flushing-cycles.cfg | 0 {cfgs => src/cfgs}/flushing-inst.cfg | 0 {cfgs => src/cfgs}/flushing.cfg | 0 {cfgs => src/cfgs}/instructions.cfg | 0 {cfgs => src/cfgs}/io-call-cycles.cfg | 0 {cfgs => src/cfgs}/io-call-instructions.cfg | 0 {cfgs => src/cfgs}/io-call-reverse.cfg | 0 {cfgs => src/cfgs}/modelfactors-all.gp | 0 {cfgs => src/cfgs}/modelfactors-comm.gp | 0 {cfgs => src/cfgs}/modelfactors-hybrid.gp | 0 {cfgs => src/cfgs}/modelfactors-mpi-hybrid.gp | 0 {cfgs => src/cfgs}/modelfactors-onlydata.gp | 0 {cfgs => src/cfgs}/modelfactors-scale.gp | 0 {cfgs => src/cfgs}/mpi-call-outside.cfg | 0 {cfgs => src/cfgs}/mpi-io-cycles.cfg | 0 {cfgs => src/cfgs}/mpi-io-instructions.cfg | 0 {cfgs => src/cfgs}/mpi-io-reverse.cfg | 0 {cfgs => src/cfgs}/mpi-io.cfg | 0 {cfgs => src/cfgs}/mpi-master-thread.cfg | 0 {cfgs => src/cfgs}/runtime.cfg | 0 {cfgs => src/cfgs}/runtime_app.cfg | 0 {cfgs => src/cfgs}/time_computing.cfg | 0 {cfgs => src/cfgs}/timings.cfg | 0 extrae.xml => src/extrae.xml | 0 extraf.sh => src/extraf.sh | 0 gthrottling.sh => src/gthrottling.sh | 0 hybridmetrics.py => src/hybridmetrics.py | 0 {magiccut => src/magiccut}/README.md | 0 {magiccut => src/magiccut}/bin/TraceCutter.py | 0 {magiccut => src/magiccut}/magicCut | 0 .../magiccut}/templates/cutter_template.xml | 0 modelfactors.py => src/modelfactors.py | 0 plots.py => src/plots.py | 0 rawdata.py => src/rawdata.py | 0 simplemetrics.py => src/simplemetrics.py | 0 trace.sh => src/trace.sh | 2 +- tracemetadata.py => src/tracemetadata.py | 0 utils.py => src/utils.py | 0 57 files changed, 11 insertions(+), 11 deletions(-) rename Job_Creator.py => src/Job_Creator.py (100%) create mode 100644 src/__pycache__/hybridmetrics.cpython-34.pyc create mode 100644 src/__pycache__/plots.cpython-34.pyc create mode 100644 src/__pycache__/rawdata.cpython-34.pyc create mode 100644 src/__pycache__/simplemetrics.cpython-34.pyc create mode 100644 src/__pycache__/tracemetadata.cpython-34.pyc create mode 100644 src/__pycache__/utils.cpython-34.pyc rename {cfgs => src/cfgs}/.directory (100%) rename {cfgs => src/cfgs}/2dh_BurstEfficiency.cfg (100%) rename {cfgs => src/cfgs}/barrier-syncr-time.cfg (100%) rename {cfgs => src/cfgs}/burst_duration.cfg (100%) rename {cfgs => src/cfgs}/burst_useful.cfg (100%) rename {cfgs => src/cfgs}/cycles.cfg (100%) rename {cfgs => src/cfgs}/dimemas.collectives (100%) rename {cfgs => src/cfgs}/dimemas_ideal.cfg (100%) rename {cfgs => src/cfgs}/efficiency_table-global.gp (100%) rename {cfgs => src/cfgs}/efficiency_table-hybrid.gp (100%) rename {cfgs => src/cfgs}/efficiency_table.gp (100%) rename {cfgs => src/cfgs}/flushing-cycles.cfg (100%) rename {cfgs => src/cfgs}/flushing-inst.cfg (100%) rename {cfgs => src/cfgs}/flushing.cfg (100%) rename {cfgs => src/cfgs}/instructions.cfg (100%) rename {cfgs => src/cfgs}/io-call-cycles.cfg (100%) rename {cfgs => src/cfgs}/io-call-instructions.cfg (100%) rename {cfgs => src/cfgs}/io-call-reverse.cfg (100%) rename {cfgs => src/cfgs}/modelfactors-all.gp (100%) rename {cfgs => src/cfgs}/modelfactors-comm.gp (100%) rename {cfgs => src/cfgs}/modelfactors-hybrid.gp (100%) rename {cfgs => src/cfgs}/modelfactors-mpi-hybrid.gp (100%) rename {cfgs => src/cfgs}/modelfactors-onlydata.gp (100%) rename {cfgs => src/cfgs}/modelfactors-scale.gp (100%) rename {cfgs => src/cfgs}/mpi-call-outside.cfg (100%) rename {cfgs => src/cfgs}/mpi-io-cycles.cfg (100%) rename {cfgs => src/cfgs}/mpi-io-instructions.cfg (100%) rename {cfgs => src/cfgs}/mpi-io-reverse.cfg (100%) rename {cfgs => src/cfgs}/mpi-io.cfg (100%) rename {cfgs => src/cfgs}/mpi-master-thread.cfg (100%) rename {cfgs => src/cfgs}/runtime.cfg (100%) rename {cfgs => src/cfgs}/runtime_app.cfg (100%) rename {cfgs => src/cfgs}/time_computing.cfg (100%) rename {cfgs => src/cfgs}/timings.cfg (100%) rename extrae.xml => src/extrae.xml (100%) rename extraf.sh => src/extraf.sh (100%) rename gthrottling.sh => src/gthrottling.sh (100%) rename hybridmetrics.py => src/hybridmetrics.py (100%) rename {magiccut => src/magiccut}/README.md (100%) rename {magiccut => src/magiccut}/bin/TraceCutter.py (100%) rename {magiccut => src/magiccut}/magicCut (100%) rename {magiccut => src/magiccut}/templates/cutter_template.xml (100%) rename modelfactors.py => src/modelfactors.py (100%) rename plots.py => src/plots.py (100%) rename rawdata.py => src/rawdata.py (100%) rename simplemetrics.py => src/simplemetrics.py (100%) rename trace.sh => src/trace.sh (85%) rename tracemetadata.py => src/tracemetadata.py (100%) rename utils.py => src/utils.py (100%) diff --git a/perf_metrics.bash b/perf_metrics.bash index bd7685c..4fb3326 100755 --- a/perf_metrics.bash +++ b/perf_metrics.bash @@ -115,7 +115,7 @@ Compile_extrae() echo "Output of the compilation in compile.err and compile.out" printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" - python3 Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" state1=$("$job" --wait compile."$Jobs_scheduler") echo @@ -199,7 +199,7 @@ Compile_gprof() echo "Output of the compilation in compile.err and compile.out" printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg" - python3 Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" state1=$("$job" --wait compile."$Jobs_scheduler") echo @@ -333,7 +333,7 @@ Test_arguments() echo exit 1 else - sed -i 's|home=.*|home="'"$EXTRAE_HOME"'"|g' extrae.xml + sed -i 's|home=.*|home="'"$EXTRAE_HOME"'"|g' src/extrae.xml fi # Adding -d to variable if not empty @@ -372,7 +372,7 @@ Gprof_functions() rm gmon* 2> /dev/null echo "Runing Nemo with 2 cores to obtain function data..." echo - python3 Job_Creator.py -f "run" -j "run" --set-core 4 -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np 2 ./nemo" + python3 ./src/Job_Creator.py -f "run" -j "run" --set-core 4 -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np 2 ./nemo" state2=$("$job" --wait run."$Jobs_scheduler") Job_completed "$state2" @@ -387,7 +387,7 @@ Gprof_functions() fi echo "Gthrottling functions ..." echo - python3 Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./gthrottling.sh nemo" + python3 ./src/Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./src/gthrottling.sh nemo" state3=$("$job" --wait gthrottling."$Jobs_scheduler") Job_completed "$state3" if [ $Completed == false ]; then @@ -419,15 +419,15 @@ Get_trace() Compile_extrae # Run nemo with extrae - ./extraf.sh nemo extrae_functions.txt - sed -i "s|list=.*|list=\"${dir}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" extrae.xml + ./src/extraf.sh nemo extrae_functions.txt + sed -i "s|list=.*|list=\"${dir}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" src/extrae.xml for core in "${Nemo_cores[@]}" do echo "Creating trace with $core cores..." echo - python3 Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core ./trace.sh ./nemo" + python3 ./src/Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core ./src/trace.sh ./nemo" state4=$("$job" --wait run_extrae."$Jobs_scheduler") Job_completed "$state4" @@ -441,7 +441,7 @@ Compile_extrae mv nemo.row nemo_"$core".row echo "Cutting best iteration" echo - magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > cut_"$core".out 2>&1 + ./src/magiccut/magicCut nemo_"${core}".prv "$Nemo_iterations" > cut_"$core".out 2>&1 if ! ls nemo_"$core".best_cut.prv; then echo "Cut failed, look at cut_$core.out for more info." echo @@ -467,7 +467,7 @@ Create_metrics() # Create performance metrics echo "Creating metrics and storing theme in Metrics folder" echo - python3 Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w ".././modelfactors.py -ms 100000 *" + python3 ./src/Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./../src/modelfactors.py -ms 100000 *" mv analysis."$Jobs_scheduler" Metrics cd Metrics||(echo "Error Metrics folder doesn't exists"; exit 1) state5=$("$job" --wait analysis."$Jobs_scheduler") diff --git a/Job_Creator.py b/src/Job_Creator.py similarity index 100% rename from Job_Creator.py rename to src/Job_Creator.py diff --git a/src/__pycache__/hybridmetrics.cpython-34.pyc b/src/__pycache__/hybridmetrics.cpython-34.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f76b6cfc79bc957778ae9e53f58a837b70f5df2f GIT binary patch literal 35256 zcmeHw32djEPRg(L$W~HGnETjTZ?QJf ze&65y-n=(6ASv1LR&52qH2U@X`|s|*um9?vzwYWz{m0+gIvYz`>W@_DXCLCn@u~cS ztyEU2O9(BMwN=4V3%0sstA&`l6jK&cW2z8Wm*N5uSA~RnfvE{qNUDW4b*T+dNp-1R zG3<~qC1Iyp>61)A(4{VQtC`|P70>mk>nY{FOIencMfOg0v1q9abnZ7Pdqbf`I7WzmLCiFB!Kw?ukW zwpSu(Rlh_ARCZ7zLn=Egkqs(4B9VhkDsknGK4_H>vEHyxpv_TO_hoWw%Mh zQQ7Sh*`cyKC9+FpcT40xmAzjgdsKFuk@sDtm+x)4)-cJ*u*gt9M~Ap_#{2 z_PE5yRQ3s#eNy6R^$C@omiQKxJ*l#%Bn~2+R@tW{zD;G%sO+014w^izvd>6-yUISR zvS%fZCO@aL=On&UW#6K*&r2K>JFl`YNPM@-zEx#klsE`Dqp}wyem}?a`SFXj`%f(u zXRG;AvEo!q&TMI6ajBYfs#kN)LMfXoI19O|o1d*r-r}?wx4oW4H(#u#=S(iIJ)6Ig zuOR=~am%yzBItb1&F0)(b~-;>J@taul`CWxE4ggCnqSD(22ScHhfmI)N^W-COXcR~ z^0WC|adyQ^ovg>(DvP;Xc4^UTJ8vSf{Nik_^NAZdH#47e&YU~p#pm2yxi;b_@o9;O zoIR(Xs*QW?bA_eK)qHWj)^*CnoISgdo-d^r7W4U1t@GJ)XPh$!o(qx}OO^a`t^3?_ z=g(XSUL*mJTGj>#S+AmGO3fEaS26{8-+x--PN+p)iy1dlDC9K2hKUL1oJoj46-t?G z8fY@bSy0Jw8~{FpG{>ZQ?dY2YDP*gDdjii(#r$lBy%)xnVlSpEvzbD!wh3qeh)&gD zOwC-$7xL8=uLIOHV8a?EDC&eEQu$(~YCyIa5Tv+EGGd~jB)tJ0)1aq<&IsYGv0>}f z$k|7Nd#S4{SKNG7+PVeMPsw9Tvh|@8(-8ujYO+^b(OEJ?Xg!L`>iRNIgydbs4q|Gu z?~VYX3F}b5l5_KBIPB6{H2U+Bp073aS69`|6f1K%S6aL+oaY7eCfvP!?A=yve(u?G zt=r5D>$I79qHQK{i#8K*y*AHUTpaL|FrO3nUpa*!uhg2QmLFABrKT@Gpw?_jimBy| zYL!WSYE5?3)tFk1t0QGc%`j@IRU1#(D-k_EZr46^!p&u>xr%de+R0*J!I00mD^6)H zv{cPyW~(K);$(_hCtu80(F^$+vfRv9uOiXOEi6`7QY0^iFxG@g=LNmYrTv^C%vUp5 z5b=7h0nXdc9GG96s~o7zx|!5vkxBB;h{$#J+SDO zuH|N{4;(1w7D^McWShnusJOES^i1;C@WmBxU{>gu4%Om$lv3$o^%F^}+v2Ck>apFu zNcoqZC0+P~p4}26nqKMBi`Z1s*c+su-JO8*M%RUMcjLW#AA|cDuzPhdj+i&r+%=|@ zd?^EfR(&*q^lw5Ree#a8PY*J~FZfgG^-V&v;`^Du5%X{Z`yq-q+)X{vS9j@~QPVdj zQ>cp<*Xfz5Z%)tr_#JzugcZ#7S-97^E=u0CSvtaN?i*ei`y(jBozWe6bCNHZQ-k! zx@J?RV*~eDe2if+vZ^3Lr&{e$BiSxBf(A&6h*wq-bm>s%P1X*{nz4`sltgn`V7p1V zXfBlP=hBcrD7iF-4+v~ei(DW&tJAArCDb)kJ|+*@KClA)nTH^&pI*vU^!`lN5_`E* zU0H-YaWj{>egM0BsW?9l&KY8d1KybX_);Y|w^RUM5U{Tmy?2ADp~$1L6}j7!Yaj;^ zKswlZ^ChwZ4mf$tr%YwZ%`N1L)yjV7CZ-q?ScvcfcSMn|ID0~fD+coc2mAOen0Oc ztIn0BIS_&QE19_*+Hn!tMYNTY88 zUe1f1&7Ad=7bj`GcxI_ustq1JI(_oVx1K(F)S1pz!FXYy)H;H4YSTxL&Scyo`T!8; z&Csmms;p?FhpsGDK@x)@O7qD(Vu`|n4k0|@5+-X0jvhVfx+NDi!2q4TimIY}q#Lpj z^oy7!1!t~Mx(Rj|to7vC@is43$Q8W=5;Ik=%}gyXk=T9uXr(x1#h@MDoN{VT85^VkbIsNH+52q zNX+X#QxnQ1oU7SJ)~Rnkr^QqdKFyWjedyfEQwUTlX(eL=R^0BkV%8)+o%r0x|Moy^ zqn)(7tPX^|R);kJ&Nq%&mz8RWr7ZVp;F?VmBYsBk`8j-KBl%fuDZx^VU4lD;Y$RCs ztFpdhX~be)6}g7-4uusC+X%oQ=3u>k8LM+jEngmxz2>b0Yn`f!RUGd+^(Lbquv1w6 z2GixgUh7sX4+%K-CG0;Y3wNoQep>BTLt|L2hp>mp&ef@o%oI16-6@o)cc}nb+g*{h z3q%7llh}Z)&_q6sl3Axlh|w>{dPOwwG;ehv+s`t2d~DN$3j_bRCeV zU)P~{(69DD0An%@1__Os{JtC_W8q`c75}al75|kM75_UsTD8#Vk;vP1&-0#=Y zRO-&)(fS!2x_%#MQNJs(X2SkpY&}|F!#573%Sg!Ui3Un;LS2)hB8wQBUQdmZQ3sL0TjtN%NR=289M<6+{xtpjhI0C|jko=(_2`I^- z5GupawGJPmYdulZwbV2$rjx8kFs4G|PB$Sk?#86CM~b_Ic#%w8vi;#{z(Vx^s7ciI zc*LjvW=%e{$dF@8ke~-2EHGPjsUdXr#0GTr+y-6aj6=?Ct8o9QC)eo6ypdX)(9AJMISk8Do&Yw~lh?$J>SbsZh1zMIz1sG z==QZjXA&FG>3t%EPB$WgZhtFuCNYdIAZG}j24jL=5CAjL3Y|&R=mJ(^Qb2ul=;#AM z!WdWksU}qQM2+fl(;BzGlLI7x^O2i``u-PecMs@7>N}Q62=x}a*(4fzH0@aL=xW}v zUhQn&vEJ6jk;it2wyT5nN7-f`Yqx@GE7JXDlA|K5 z3{a23&Mjl&kR%11JyuV2tw~&OOdRhvV?x7)#sm{ujEOhtLh56}Bt*xANz|pjvuR9x zwWoPZ{G0CPG4X{S)7nr!qs1mM)Z0k(8HrGzGnJi4TA8MptmJm;jq|gHm?LHt!mbVa z#noqEsC?Z8J_0jBLba)PQ>f%+%YQ0F!RXQn{^&BV>!Zsg zggOSJ%OvWKY37c9sJG7Uz0_-}9Ga78StwE0I#@0q)kCBFiF?f$FnMT<$y^^-CLvV) zI+U*d6HVjl@B5m^)j#$&kE`G93$>y?4oqTb(FyfrXz5|<+Hf-^LiwXz%)BOLZ9|Bo z{q1;WKvuXH%kC%pj=a2vxP7sH>`Z@OtUl#{z6ebOoZB zO`^{HolWgr8K}4O+<@Pz=0>v8vgjT)t^BS#0EYqVZ8JG%%Fl>aM=#eV@gfp0;4_2n z#MS|R4HTV&ntlB7+G0fe4YL+JetaX`oLgED)%kpeW+bQ&q3m<0>=Tu-o2xFl^cpgH z@67CM$<0#x9J1a)7aT!4DY{p$%XiC3S4&VdLa$ACAftJ`c--xVZmq6|?GR5Lg1R(Q zdvwO(LoXkE%`Fr9>e+^l*LFOWD-=rmoiom22AXxMxT{y8fQRZc3+Dhh5EQ%tqxbdg zxS1RCO=%06Wp^-K6oLF=`SK86xc(K$5!%zF3Tg(-o zJ?Cz1#7Wu@hH8Unpq`s^H?a!a88{5KF&Jg=2!f#Lvn#W1CsQ{tm7M~OZm4Vd#8kkA zQt_1kvlw!9G3!PK)TX)}2J`>b4;mZx=rVm*!NL#ypoMZ_&oO zM;JTC;4uW_Bkq%o(+KZUG3z^1Z~%HkbiSa=SDJd|3N*Uq)MBGXeAuYU>F5F1Gc%`u3yrD#TUtJ1T~Kr5H#QvWws%;mSj>J1O7TJezipn8 z7PH3vch(-O+kQCq2qEnlo?_O1{GafjL+=KxF8k5gQG0;zVlgP!_u$t-nKAoKiIeuW z`WyRUo0taUG239)|52BN){r%fcaGI>v&12k(3rLv7#~7?AJ!#BaUflA0+;!3^;_Es ziO+!jXuTHJ9{Yg(U~H#-zqQS;ou9iIHuX%!(LaaXe--tFOrS8hJ8;QWG(2TLKz5{Q0^L~ z{t%23~Gp>^&y5e#0EtL6)QcWD-F&pPBYLoSbRY3jV%6U zxL!kBfanc(;9Nb!LN==OAv=k`lxyo6)9KbVUzzb!|}XHE&oxt)UVv7RJ3+LFuxcB zX7|NdxY-;%NWs|9H6umG&__rZA#JIh9?5P7BR-~zQz{C$MFVbRKa4BSn zI;K`Psq0DQelCVMP&Z>fMAVown>fw^1Y}nEmOX>6T-&Uc5AeO%COCm{xr@?l7)Y3j?C2cV3$9sswwo)kE^xqjIBbm3#`FZU0xxbr!SXfUIgS_ z*~22B#=Qpj8gxAM!I&B-JyO$|@~eq>wM$NfzyTBLY-HMXmrNIt=>k4%qiILEoRHbG zL#^(R1yxR6Sj#6=wHuXtjFn$LsB5}cYKp9;P8f;QcR!XiS@m8}ek_T()?- z^)p>8|4H%!K4GGMQJ(6G6T{;f8j zyg$n$ME8>EA~IdTCrsYY$kV#y<&csMo&yvMd8~uf-Xk;EY_IkGe|&H4_&FX>5xF!f zdpgz3l|l}FUCz~|1-OI5DJCl^kPH{-CRV9tAq`Dp?c~p{h!{F~T+4NoISs(j@dhDG zWZT-=nJ1nZCKBub=`Hnc;y`%w}9TPc2p9 z;o`e&PEJnNo(;|e)bY{B-6`Q1%vH3($-=z~{zY^ZlS176&SC*BX*o3UMm~4bSt%_+ z_NBl2r28f`;B2isA|Te1C<6yAYD2Bg4~YB+CnxzKRJT4c@FY%>zzI!c#MOb?6a=L@H9TQA+VF5kBHc-Hc5Yy4IUb*{ynRp+*KwATKP zsl$^8=hk-D?02l)wkEt-F;f&7yjDDV^yuS`FB3b@>Z40e=?X;SBJ>xc(WvAXmJ0B9 zg7X6R^LgkAMB&kxaq`w{_!;l(=fpJ6EO>E@=A742$S>p}ZRd+YG_&lb zDu6IDbf1>OHLgVrQ?F0zo(`o9&`l5x2&X0T@L*epznC<%%|Liv8Xadt&V}Pzdbx@- zYhDlZDQS4LrOk^Z!O~Z~Ug;6&kkXl~^hqbqL_tc{xdzw2@^o1P_7Tt zZH?HQfhTEihHu;;;8PGoA>c!t?~(l7h!0sCAvkYFtN{YTZ#ds3>waq!()|3JQF^!S z{wP{K%Tq!8p~@-{Uq8_f(Fj72_|sTea!RK&Gtv`dRd@NJG$Yytz1?JDeK zNAVO}T0z|5z}>+!dlZg$G&lQC?IIe568zWPTBLj+p=bnsFub+F)rDZe>R}J`QGttt z`P+YjpbgOoETdJ@u5 z2bg^^_OskSibfH@lm--Wbp)0%$tE1N_>m(EgfiGQDH?U#hy!&q=0ijc-nd^~1HUpL8+-Yr zHG{5%bV$*tAM6uEBe7z!@{up9uNQuzs8%)d%Ld;9CyAg9kh?tb| z24wb#b%<>%zhcjD_F+wW_-R51q+1P0TAoO!V)FCL4Vep`ZKq1oaPC zE*G(ID|Af_`S(d>;S<-80V&d-k=0~dXf=Vp%J1v-)*MsBektNt6OBb6rMZY;!7Bf1 z0)NtwaCa=UL(fIk8>j&vduZkwD*nE9=!2kEGXN0fG0vdRgEElyh34k`_(Vjelw)VU zfOfAkW$n=V($9X!TyFqVCC=aTz)Ns)UQ0w;^rYk_BIXOH+y%x#Fy)pQ@DQR~X5bBQU?nAK`+Mv9^k6*jj*>g_?sx`<1Gx3)V{dx}l&{jjL|H!9rBL7c4~ zu8U@3iM?$(wawv-r(mCy8x?%`Gl8hQJ1pB$I2JNwaK>jAqP(_c>2vcC9_yDY_}wsB zdvhI?6O;pC`wiea>)t?{#}l{x`zLPm+KJn8)5Pt_Z5FpfH&=|u+#e&wf1JUGiNKQh zc*6YR5LO$KWWzJ&o#Nse#PcFsU}bl=HW-^Sp(7`(#Z-3-2)!Fw3Im%;ZicproJGdRuQ zdl|gS;QJVSfWh}O_yGnVWblIsC0XTi-4KLu_8?N(6`x2*?N(UN~ z^QA_CYg7wdk#nG0>^Al%WEHW=FqaV!NPZ>zNv2b*ws3FR_$KZ*7RN0ey8r|KYv zPvvn4TRiKERS>ot@TcI?phSa{4TT;`DEyC*&dSKga?ug>ffB#y^PE z71C6w=6xd9g%bSNT;zQul*1BI1q06rmw^*d;egL(w(?Gc?(o-iT zSZ$AjbVZ3p3tecX0W@Nk9Jm!Ri+IGoIReRtS!i+_l3jF|<~N`wNEGuSEervULYMH< zmKK@Vr1A-Ch9Uz-1I%Ya1(6l# zjSXVafgc9vo*3l^KTXaDxMMUDLIh!3i`eb)aG$z{L0yMLi}`k@{JVA>Mhf25H5Vfz z57_%#!(IgJ1$>%GSANRIAC=zqQHbFP>#|P4$r%xwIS}@cu?4{(bzv7KYYp*DnBn&i zlG7(E-mkE9;?)f9yn4%p>^@;@^yu2JhDNY>V@wdmTpvV-3biC(VR1PISpf+lQ`awND;5~#{0Kr>N$kQ^ZdXt=lP?nfU z*lJR_G6|m&_+t(5hg(cSJ!#Jfe5nCm?mKHaX>}d$VbWsM_!Cr5+W(M*QXojvllDI+ z*O|0+$Y9dG0+CKn+A$G^f=Sy1d$IfqB)M0T&EzeARZOe*G-Hpw&KY|=G-IEL%-AP+ zZsBe-7R5F1UH3X;A8O}}#m0NjGj=b8pf^W^An_^-JVo!(uy)(p|AGiGr$nHsjnx@> z&9+VGo75&}D>pprW^H)K*`My{Rs3hu&iYTNfe^vD!EXp2>6$1{+_iq5mJ|F}ix z`4TLt`yY5{$9N2Gyt%~!L_;rrt&}f{{DJ!d?a3*TD0nfTHVcWZM)UIHXyNVD5&C9Q z#(VrHz^ztDgATj4S9AA`^qV|VDB)g(%HxRm*_x@GHX~_%v9{A($#*iiSkQSv%)U3> ze6^ax?I0!R7P;hFY!cwjgHqr5!$GMXW5pdd@X{6Yr?zpnX%ndJ zG(>ZKCrlypi}`=zpg4#6{Qod0+@C~4+@E4Vd34-%KgT#lOvtpKXM7I>4g~k75wIux z5wP#hvc2wqz@?~2X09ah=`QF-m@ z^c-|N>9p5D%PmFz3M|h=A@RC#e0)>L?RqCW{{(|~GWa$GUI)CEX++Ms_pE>jR2E)` z)IzG1)D9I|sXFRoB>V?JbwAXlI^u(u(;3%1`WNMJ`*Y%J1LC_tcA z3VRW5m0|>25YoX_gFC^ES`dt3xJ4o)>z}cxeSi+OO)c-&(%UZHiUMXF0`ws&C2(B| zR0)uDaf1l|k@zu+nDSuYQA#_sg>w?rrAsz?*PkSy>i0Wtp<*Bx?)Sa;%MSOut3Oj#Rcu5UMMhh8^^G5Z@+ z25sDD^y|ny3q11EfiRfcGhjeCw`1$iZA^R2ZFq?5x&0bvJLh&%<~Gf_nA@0@Vni2P zI%cWp6)}(W+=dOBb6f6r16(k-KdR^UkI0QlK!G{Qi46#u+fxE! zID??)H1dYZn<;-HnA0>Bi>)}A|54bBfW3fEa|Jo4b*-@;=xNf$X##B*RERWZB4%#G zLkvn$?O$$TR-GU3eVr4={WbJVEqZZcea;NFkdIu>n7dAv{r(gPPnN^Zb3Eu5%orc0 z_HfuhwZCqig24*cVT-{kFa_0_YM3eRVG4=qqG1Zj?CEGUQ~VMdcFz;XIAcX^1DHD7 zaTjD|FEF*5I-?C!hZ6-;R5B6AY|Gtx$mP6|79F~TXobmxWwQ*j)Jit4re$YfzBd#iO$*~Ro+1=!fT(M z^#?IFLQ()){(~6)W{707x`t}(Asp2ABhm;o4fWcqpr6NgP2ivxU0oWP<%zFWtmkgs#@t*7}An_H+t<92Y)>k#nt^_vy(Uh(2mPW$YYjLSrP6-UlxPWTA$t2+|n ztBspSaU`3k_&xh-27c$cxvCdi%w%tU9N23+8`XwJ!8QT!`b7u$woUeH8e1bFlqL)~agkI=}@og_3kU9|uL-<+LRK>?|io4IIuSW}&P^<%I z!ncOp_O}Ku`TcR@wvYd#5bCx-Wi)7yKsnS4#Zl58i}Ap~1~~cev^Ut@NYhUKBle(Q zrYM;1k`s>k^_*@y#JzyK-HhsY;&TF@%4>+V*s+?pTQId?Kd`@X&xgndmIb1a=Kru^ z8=kgJxR!Rc{BeO0u6O$K2ZgD@Az1$a`*Kxq_y-M5?&-^K6H`A_5bz7Z8Mt2{qYA0e z#6>kjCjn7i2p{u`K!_Jf<@bDO@QX+nXd|wbUx70;_}O+58nsAQPQz9WClIiinrDT+ zMf_Vd11?XRp9TXiPkrJj1I|%e?K9wL7T8{7>Id5m76%MCW8kE~D*-be$icqH0{aPu z*2J;Fg5A{V;9uLpfOo6%BNmz2l>Z_hvfx3%%7Qa*1VPz!egO!$Xn=rIZUjM%$ms$EUk?Tlrx^tG1OrSEnEe5=GXO!- zE>cx02Ctyq!2@IxP*}7VjEHao0hZoA+Vz7@@mP>Y;)uY71axNO&>oN);r(D?2#z#} z3}B=g!!h9 zbS&BdKyT;*77KKl3g{AaBMODr3UM725cs1sb^xk^Tl@&}uz)-gfFS7&P*?k0Yw9i~ zNZrvWbun7jqwb-*P#3qe)d`AwyXp_POsoFzSoi@KTGij+`l$NDsgG5W0_3@9A z_(gXh!EvF=69H8M+M;}X`5i&@$UV^vf?AouJuO&HHiMw9W{R8=kkic|XaT-Kj|J&j zbfy^u?J}MCX2J4wGYDFX@7p7mX9eVJ0D`19(9c>B*Zq9Cjs3hS+RvCR>-96f;;;2A z;!<`_x&>Pq_rb5Uo4NFsrd0Fd9*2R(8bM}OJ~+^0B1gJa_xv5ZRnMwBcI)#(?RC2q zHOih>*|+g)4f%zJ=!O@iXJ#Ni$D>mnttafi5?I^-1K0}!dr@GEAGb}*>cqn?_Ti35 zdr4A1^&W|*9WZ#t;G0FF;#awvHjRUir&={dQtz=T5v=oi<^3kMD?&ii^>1IN>x*CE zPX7AZrRT^0<93;0dS#tqN*cVe#YQfdF~{Yg%%sI-si_> z&o&$EAWr5M#WJD&xav3D)jgu*K>poy5Ms)6e+II~EeKDrT9hWd1f;bRPQ_n=E75h= zW3vCHnsq;rTC6fL0Z9`o0q6y4gTXgFhCuAShrJkXEv$7t!>`daw-wZ4%M+7cV&2W= zinWcn{jlYiIM!k}fIof(hwy770BZI1nPJt-Io~_2)>E$mzZ}&4T{eyfpljR0c2DE! z7S+RIB^Q@mTt-pbWPEvop2`JcbS+kzn5d1c^NpRCpdP{Z#`ZOSIp?MB z0u?ruTt@fhW&*6=pI1@DCVdfNt2-_(B}@3G&G}qy(7;5-jD6worDP>jSSS^1+YG?E zMMe8r`*lSq*F^&H96TF7sZYjKsw;)urKIj@&w_8xJiZ>OwsGC>42ALtdIx+V(F@Jr z5L(+D={tB7h0Fg`TxOhfn?6{^Gh^_~f*t@CZG=8VKjIpKF&|=Q2!c7AaqJ2~w8pWU z_{Li4$lO07%0FiCCk+0S!Jjj@$$(lf?ZqT|BFMP<@<;a;V}H%yZxDDfC>`P71ci;4 zT-FN1nw;B8&2JdI^j|~2a z!D|fO!+`3-(4nc*eEePpa$c&1qRUJ1!V0ZU^g0-qd8d>0NaCWm%zG>wwNsn4b}HQS zzGmvnSX?WAi-J{b%o?^g2RfpS)&@8Y4B7p!UsDxJS$nOWp3nH2khe}wja(8s6|kc zdl2I+&~0r80aEygZfo!!Ew`^zxAoqzZtGi`bX#w0t=pQ8>b8E$XsFiJXVrCB=Jfh{ zIxN`fw5|nuDmXZVbXf8f=&(XIWLOLwO105FPP+Rc=3+S-6%0q|$;OnqH`1*BN zQsF>{bttUE8uzIV)tlA^X-%8aelaI+mA1-Xv3xV@ABXe*ekgwn)m>{H*sAEjRvUC+ zv?hc8AsrYk%21$OQLWMZHqmsszukZbnzN9K%qYx^iVO-fV#EPe|Lhd_iOa9G8NW+zXoU zxY!Bl+AX{}5hiJhv2s@+&CtSa(*}+E7sU(>8#S!b)a$_@2Xz^=Vz{BT{I|LgoUjW} z(=H9h+P6siN+S&dRg-=Vbp(X`x>64>q;Qd40fNKwMmGoTqMY)=Gb5tQ^z~~{BpCe~ zn0WpDoUA&KJ2ZSl=@(VDMaV!^=Nl)({xBK71~V_p4ah+KW@|FQDh@L|Mh!@WYVHyf zyrzWGf_oz+?z=lBaPuX)RjY2-90i*yBpRwH|62lDN2C}Pr4Wg4X_7Z8$~p;A9T?{W zs)@WaakiDWRR{KwmIBYci#fIAJLK9j+)ABW#%TxKl@8NpX>N{(F&1%PqKezHmsj{w zvu6JVTb28J2q+f8(FRX_3@8$L?dCg3v+z#EV?w^}vZX}gwTbiATNi7Cl||fhwzL># z2J1qHXRWwfi60gp<&wKMR^Y!oW~{`2d(5Q#{IS}GI{O|*=kthRj#A<+?V;Z&Uayv4 zl;iDQyI7Lp&5f_LMI-hw7-z7T!9E5P4E8f1tLL?4mao!;?cRr^@$KJa#_tSC9hf=2 zJ`4hV*_kn#diWB|+zoSOT8CU&p2i^?eiJ9|FxR&}yoqg*F_9o=xYEcM4r{*9^u5m# z<{^?+ehkGovV>h=3A^r*CEPA%O1j9wgo(Sq9<=-9=CKVhTqdm%u$C$Ko(|)-G8|rRePq(q`_a^9zf(kq;(bT(wy7h9ZfOBd?U) zqBjyrHKyG{{)#scd4VhZ7FWE4{B!?;Wpthq>?c{0m(*p*0EhKl<95k{d;N8slZz`_ zLec-V_#*z0;#a&6;M$vbFJlu7rWicP;AsX020zN+lMKGd;MW=a1_N4lg#{Keh6ctS zz8y#4wI6}NuvExBPC8a7uq15^0CZL#WCHocuGfc->BJrG@;~&{aO}uV{EV~fS8OLr6uS(~dnHq*Na|iX zmRPb~Anke$Y`a}7x@fT}ilV?Swm`9|w~Mx0Z2PCbivCy>K^MKi7HIpUaEkUve-&D^ z>33!>FYky~B1LC6b}EU(nRCy~oO9;PnYlCP=s#oKk-z!-ZP~xAG4`*_^)rI}EBKiG z3S$~$Ge{MtsVuLs1(nUHY{AE7d`uyzkLCSr#?LYQEFWMCK{gY_vjCe3F-p7GaxZ@j z)G(Wgu<61O^XpM|C&G*ZQxru*l@NQops=?Zs4+GZlTd%Rj@l(q^&r#GLl>J>nHFYR zglD2mi}6f1)8ahS!?a$W*~GLyp6O@W0MBe@+91y)m^MV2xiEWQVHwm-GHsas8kmnW zZG>rCcpgpp*tOBEmA&Wg7P94Bv0$dk#Z)O@ESo8Pb~cyI=>>eu)Pi0%a#`~ciFH)9 z;w2+jD5q!bT2_~qdoO3A{-sgHie5G}-O#l&xor8|bt`gv`IeE>(rB-;b!;qko)=SB zGDaq!*Yk~jmaX1}QZ8MxABYO6R0?%2UU_Zp3i)D2OW(@mGli^PNuold&ILS5*^jIk zwVo|5Ebx~1NQkEazFWv;Gb98zNfA>wa+y4@vRhOU&0pj2T&1Z-tZZZo=B#e;I@{fK zuG>%C9rP9#N}fDlzIer(XR5FU&s0a|nUFnrCd9RP&UVo#@Ys2#`T64n=1-!GSr%kd&~;CYE`bqV z8zUkxqHL-Fl0s}AJ${#!eKb=*hXN=%2B9d_!mR9P^8r@nGocodi&424lZ)MQF)kN- zE9FU8f<>H`ROvuF{xtNrT!*X$iE&WU>^HGFw&fNSD<%Y^?TUc!? zt762qvSc;Fs$1Ap6{FgX5<#L4ByK}u4y4D1bUTn<8xnUQn`}sr1L?CNy$+<`hHP>m z12&{jK%g|6ZAiZZ8MGk-4kTeiHan0Z8!{*$;2k`{970dce^E)JsXWZ}*7Q};&xoi2 z8+pgajO`L^Ycs5x;4K#-Oxwon0yVQ;YG#%_DP6WJI+5V`Z9VONbf&;1ikJHdffi%9zR+e6=xav=kdkCTaTcr{yq% zTF|ry2!9XI{|;8$&X#oMQl@FvP^PVH9yNCIDVSo_6mL64Mc#I82di#p+AiMr7J?=4 zD533UiRun~@4@F8mN9oogJej;9!cC#wxdcCxVEiJey zOuHyVB@BG+8D4>Mds!82iCI8KY#>lYYtL}SKzKZwpdc!dT z9&)i21VSR}cftHZW1b;$6550)?G>d-mZ%+Ubn1{t)P6e%Q6b$>YuX{vzDq|AqYrQ^ zlom9>T!H%mQlQ#6D-0_R&rw!_WZJyj2~>H&5q$cz_5)iDN2sb!ha zBcPNPsRL{tRttkp%hQsJ_>Gbvm~3?Y{S9^f6cbRU`h8Z$CRyS80u?LaZ_+a8tvM56?Kq#(7+$_Qx>b(GoG*}kZk%9qKI*qN z$qEPA>S}nfCOXY|82w33^`g}^Edbif9Iz2`|EU$ZeD!XhTu*eF+y~*#$^DJUYB9Mc zG>MCEE!rp76I~|v#YpGmPDNXg%lAF)lk15tllyOvLGHh|BA0K<+9%f&T_*RNQCYg; zsdj=ZvR!zE7wpN2jr&QOpJc%mX{uu-U26e7wXp@XVcSkOSr^;3ooRZYon6beIR=cc z>15kB5l584!VQz=r}Tl3-S=^mcCM*yiBDnDz*2`M?WgJnO`012g=_I2cgt$A5tH^O zt;pps676fTC%UY~H{zX>`)*GQ>GHRn_Q~}`m&uLxbWUzhZwqqyD_HyFdZNqZ{%{q! zwpm!$M!dRV8*#o<8}VAxbhT~77n&YuUu?1w+6CHW+2`k514?wkxEb$=3?uWQt zxYXQUqcUFTw99hkEAGm#uU8rGeyb~AbyvRDR2jyL>{ox?kPXx`gY^u|l(2EhdS;}a z8K%r!h|&iEc0a(myWZS~`}IgzJq%KZR_|l~2Cd$I_q9-TezMTMqI;stivEXvozKhf z^tT`vqqRl6leM+)^hB4*z1rV7xo-}%AeW!Xv@cyxbeUXbpmTBqn_G~}4~*I;*ArbP z_ru2^_fK1q%MXIuC)X2QCO5sgbLnOWTS%9mQngR6C%Q~-e6VwJ`w}h43Zj+?-B_vT_IfnSkA&vlo?0%51 z95YRwcZW-6r<=c9yWEAIheyN?BHHr%yHkZK@TurAIiW;t-XEWJy(J)h) zf|fG$@?C@Ozi;aiy0gt>vqeM873NYJ?y+$zoH3SD#o5$su3Sa~CA1>osroIrIc~tg zfK|HB=;cfTQT=7yA^X^CtF?h`(i7CTq$j4GkM*L%;k%? zNvBI~E1WB3>)>3$EZ6h1hQ28BMgVjfK}tfD(3Nr}<`zt)-(5Ozxuh2^UNORi7rs<1 zjGcTBGOTn>XN&^4*)fL){Ktsk>k$dsVk^qTin6bwj;2(NlSJ`3^x*MqU?YesQ%7v4s&!$hkoqp@;$xE-EH4^Bc663VNabQkQfu>`| zPC!R}R>;sx`Ak-~{PV?J!Sb191CAN$_@w2}=L$OV;e2i(S57bBX+&hnvE#dJ<;=uf zY1W)Dvqq+zy*+Wu%szWqq(_b(ov`)h&_qFBD2}-WH(?st3GN&+URqv>OyzTCnKb0Y z_emTio3K*ZO9*M(=fidz8K24*p+x2hWa@P!nUZ>NnNA^N%k;2^Oz8px7g+(}Y~W{! zIWAI{B!iWNB+2WqoxYa7aPrjI3tXI+9ub+OF^~TDlTSVwRgK*!8+(u-znAj+C=V}| zeCC#(Pjjs>4$y;xl#C(y0X{an5*H!BuhFw{BsLejJ-7h00OX|!zqlD`&}^X@ zX-puqmKKjH#$+?~KXa&8q|x~7X5x``c<`Eu|Ko7*bj`#rnu%omSLu_2rLsN`cVC0i z&)mx8bLDj=*Pt}H#KQ4;nO#R5X~?7dbj+?79qONy*>!Y-v+J0{0jul~d+fbqdsFvv z<=ZLRC7C7co#5*#*3n8Rl{Zt*AG{&i657P_HL=C24zu30s;lyD1YVcvD<`j}&z?KS zSM{}xofW};4*TA8^9(!D;OP9Nn3 z1^#VE7*C@G{-#6ylVX97Z_on&j>Ca0|0jKVg=l|+>9A^U{hq^)QrSgO31YUa{pt|g z={k=XO3`h{np(%E5M^GtU%%JLTT9wRCJ?gT45bU9dGQq z72t34{Qb!CVg0fKW(j*NO9h(0gd;BgK5K=voKeUu=vJ4ZXYvHMy3*-coJ^(DmcNJt z7b}b>8ea&lV5S6!X8FLl6`a${*?XE5p|>DiAO!gPnib%n(L+|0n6Rn0;zABiIf5XF zte&Q~eXB?0ajuZnO;b0mr~s5R<{i`OwhOn36w|URl2x+8*##}1&KPs%Ok7|Qm`kb~ z!nlPH3l27|;8L2Jw}Q(&l!CtLNQ=JdFw6cf(Ko$4TG8cn;|ME;LgNq%rN%MV3`#a< zI53+DFOy187c&u&rC#cJ-cz^eF)hMItRA}vI(bY8H9==;b`zAXH$i#cL?Bnt^d&36 zc{Avg$%^p}N?Pm*tWC4UJR)oa^JzZBeL_BzPnU~n^LFvxOvI!T$;F)MWmC|K>0Vhw;!%((Zz$i0NlqB8@tL?;`e zxUxmviQheHSm{<{N>A%wUs8!HeQW&oD!s}+WwRPl2JpLC?M5!baR+^2C8lmu2Yo>$ ziCzYjgK7l%IR8xoE23;sVxS`Kn^c0p=<^LJJJc9(s0KZsRJN(-lo4e+aznt~uJo#t z=#g^WXuA(MJ@|x`hInMo_w^}*z(1lMR5k%`qxo*M5ck(<+T!#u`B**hRvk_o zt?pKmYW=fU+5K33w8JUra7nyD$2Q1$;}Wd*IKbuXD*31%10hE&OQ*^7Lnm~!Qu4^D zKkCIHF@lhwya?sNVHrQOQrcUHHgVUjk2)CL!h^t%^*o zpNH^3%3POl%OFMvp`msN4`R)rKM*?NOA%%TZNndh@W5MsfQRrjMpT6Gh!O2$^N2#_ zK3_Fl03#?3ZSx{;yM|j2f+y~fxv=6d=KH1q!69~w)kyznsBEO!UHF> z2%G0_V0H#7C?HTi9uI(K5PPL<<*o|}I!vu0$PL56eFi8*2F@!Cu{%9K7fDjpQ zD16ttTp=>w@&Rt&yI|V5rQ?Li)GJbm%r**@%-c=bmF83_U% zV3@few!;Nb77e1D5I4o40uSSkcwZj6l0dMOh{X7z&$BtHBJZG4!4a0FfJ=TG4sIE7ikQsYkB>54q$`x+(UX*k)FC{)!w%hWJpT4Jbs~*6 z#s`u@q!Y{rXh7H?P(!h{`S^Bxt{1Ux%&~R)G4{+y)DS)ubAzV>xnLt z`;V*0wP|11M!e+M2<*78jUNB({U|r?&}hVQ+G6xjg3}geY@(zO$tCVwWCc#$HE?=L z_qfIYC7Y4p%~Jfxo{7Q<(eXUw4u?3oK_wjiGk(Qr3{ftLggdSbQ;Gh~6nD*%To8?| zgtVsrq2zjK?4TxgQiXLL4_^TWXvew2&8NY|XdK6}b(av~9cboof~=>XJnz8A@kN?! zu$d)Ap&xz_jqlIKEATz^yq1fBxYv?>!Jc(Ec&hH|#0`PW$`OYZrLx)`(0CSdF`lF3 zc}iYD(ve%>09{Ar>t4llck_AiTYAqCpR5U85Eb`@nL@@kK|dcq-N%`U_Z;e|ZM%(z1dr#er$1xgB( zlqgxG#Gu5aWFI9Y(9z&(_k$&R#0AcM5L`|N@&@)N*zTE2wCD4$<7%+VxnP*P4v;$m z{q}N~0&*=7js@{%mjWf?T&dTc2a@WL;!`fDhwJVEaCP`Oy9e+d_O5e%?z$L7e!l@V7}fg6@UjlRY!o~1^_)Om$EBYyx+ zmusi(%v-B#3VX-9dHn)4xQH@mJ%m0hkaS8Y5$QmL%}r1np0{us6Xlh{f0CvmEr zwJkd}`OfLS7x#jf+Lf~e(CqGW`t&)cPj{d0+kO9MvOn=3|MOgN{5wMYSK;}~ApRM_XT#plvid@u*$3!mX#Ct`q zm*WvpiHnVd$R#AL9#QEN8%dE%%5+pz`o+e8$PI{%L6I928$%*DBsPXcZdhoggfKv} zSFD7D5f?^+BYnb1a-?4v0~{F?#t=t_g)zdBlrTm)GA4|1j--V#!I4Q}OmSpd7&9E1 z6~-J#_6S4g$h(<}j4Vfv3FB#w92dq3 zjyxlbXF2kmFrMegNnyOekr##W5=UMZ#wm`R7RDKlydsQOIdWDQ=Qwg+7>gXaAdHI~ z`IInTDj0>^l~N|s?lZTl`C@GY*+x6DQM+5puaqk#tDQ7Up43PY zWg4Y?qqMP^U#V3ijyMtP%TC$GzherU_eFh$DF8|4jq&;^x^QlXM*C$_B8%2p*` zxVvgkzRhvn*fI+wwtlE^w`3MpOC7Y0!gc@}a?2ZA8y&PptzoBkP(!Y_O5$f`2NLsj zR0qipxpLLArxhmR4RcF1%b_lk#rs8jOd%o;z3C3hKy9mGf%wK|nT4IZD%1L#*DeJ~ z?8PUkcfof>_u`Xe`-ACA@d8Vc(q@87DKY!NJlRH`Q1_lkY^z!>%C_>0p$eotLL&<% zu(Dz=1lD1etYV>3GL*s&H`d51VL!>0%9gcOuCB7$rZ1{gkj^@&q_H4WSMUxh$ub#0 zRf*6+q{dCVh+WOwK_g?CbDX0|G^$?0At`2pa9lA6i@MH4 z5EcNLHepx|BNzh!i9xb?XmkljG)SzuPA!3Qjcm!zyool-E6xrqmCSOX!a`EME^#~p zGn?H-N%Wsy6j(42mXJpyH06}lKr~pV9Zc3)5GFIz!Q&PP!gE_7jy?ef$ciSY4+J)* zLLzEYJCV9+5p&}^(U>(wiCQ!Oh1*l1P?NeST@9*Gs7+^_iUM17==PRrHS(%mNyyQ2 zl%uOb&KceOcqHeD9}iiXwjYbc?7|`uiyBR2`0G)>^Z0LFL{Ju@rHSnqMMH?iTgOE! z#93jnJt3NuH6&W>RW-w+84;)Ix>%;DCYm9n`jOHwv4-rgpXb@j(oZbv24+IcH3jp& zUR%){YbBas^>TG{tD*Bei-{1)asx(OwlSdJD>v43yJXg~2_1hoWQjcEdWfwRFlz(0 zQZqLSjhFOyi7X@CDc!&IF3;JwB#BkX#($WRlR#ryd81a$^t2OuQeCC7)8K@!J#C`Zf9uga$(93&<%_o6+A% z{O;oy#?QKOKv0vRiQpQ2h`1)+7os3ruSYb(Vm%_(-AL5QiHSxp6~Py7^ZOyugrAI- zbPLGzB-|whM5FHj_Bg<305IkNdjo)R2bc%|_Bp_00I=Ty4g>%P9pF#^aM%Hk1OQVG za5Mlo<^ab7fN2Lf5dfTYfKvg$X$LqH0GxGza{<6T4p0XWi;a2elxV0{6n%Ifci7|m zKGZ}bA=VRQ5v`c`GMf6E!q_Li5E7GEDclJO^KOW8v2gfZ&TaKF7SjJhVjU3aMj!QZ zpiRP7(pXreTX9jn%80+`k~_rYe&~?<{VsCf`YGhFq{}#_nIzIwS1Tc!SX3PrT`U|C z>5~d}lwqL`dkfvLa0iz4!TO#022_14-Z6>O7^JF^qhXLE*|A1~zN}W*Ngro{|4+E1 z;XctAB3z{{jEhu|+9e`dNzqKQqW`7T3~G0prr9SZT2M0JC%BUT5k~dKGh%&M00t(A zk2>0V9~Ei6XbR@zunW( z(|{~=wo}N}(~tz63jj?>(D?w+lmsohAT*a_Help}WaJ_n(2qTg=(60WyyY4xYEjv_ zyKFHYb%;t^EJtAnup+X9A<-Q2+5vVSJa*uM$PNJV*ntZo(E;j+?h!)vg%7*x|6a6X z!rzM$flv1Ma30hFi;Ip0Z4@Xna|n4>6Y{|%}A%i>O0nE9CTrqeWcqLe0@R~|zX{eqeL zTFh4!`UF??Ehcc)B``uvIkii*{d<=i*=+N@9ZIs9xPl&kwvyWF%63Eq5iowutrzA&-W4o!@_!|w>iRLN|>)Q_FIfyh1OB@I*&F3l!dR>8C^#GI)xPhZmz}QShdDP zW6aZ34^pBvE}G+FeVnJ9F|mxP<*l3B|4TH-;6na8AxG_X%$P!$-|fW^E8}SWNn?pS z#9#L|;BmTu*Lm9cGUkUHViJSgDH6wpQn9zW*i-RcihX*QV#{2tuv@X4vRJ(kuYb1} z-{tCI+FtT7qEd>!N~vO`1I+wFXyjL?{VK#rNGe?J^QlngKKMZEQsH)0W96-AP~P?} zO$v`~F3?!Kby;gcTN9*4Y+z%CBES%!AW^tOgIDziiL}P#%<_v zMY5l4g3FOjP$R7DY7ncALCks$f(v^MVog=P+^?EKQB)}h4BmDc#Bh=f!j+@!%26Lq zF4zB|Z-+rBNds$ArTz7G45BmAOUH~%;Xe)e8e9&r5B@sTH4YHcoS<+IhdT9$*1Xv6 z4e|IOLNq@7ZI>bZw*W(E&QnqH{(pJk{>vWo#8Z%UX`U+s7!cO+gy9a4DSJf~K8Vbz zsGQ~cPX;wjGd4tPuXqQEo7IZe+$)elo&P6;0mIOr4bUJxDS$?c32OSPQaA8-jwV0Z ziC^pB?`+D0gFEPNcJS*|^_>{=W_sdQCoa1KSR(~R2$OQ=Smb>?7GdfFR1R>zuc`ww zi9>bscsbeJCl^bt{h~U=`8@Oz60$&VUFY2WL=0c8xyQ+J5jk7$WGkIfHk3`XAUESl zm_U7MV7VR{Sng=QIpp%Kc)c*B24fb+h0=)HIv}ceFm{_FB*Ot>iQ1+ibw>r!L%5)V zY)MG&V)Lr$g?xm`M(^r6f@ z8TNHJnY*j+Qj)&QWxwxI91e0q{&1M|=Xxg{-vxd)&7<`{9IhAXyIh6$D_K?`-^d`n zI`iYP-6n~T$9y74{0Bb_2aFj9d9=WTz`LS#h{ueK*xo~v2h5i20DBv-=zC&wA7XAJY;WD6_ZptooXI* zY#-H1O@dF2I6lZ6%N}|u)qIK&n;Gs%z-(ni{ZguaIpsseX!lC0c?j!Mxx-v9r&_8NHvdg_!NOBQ5^pO zOtUoqHbvS$%4-46J_&mDT56eEaY3M{^+%)41(xF=D;Wex*P|q|inp?2dr>sF*Sn%& z%V+V{f@mERZt>Yt>RY5rYsI=QVnIr3Pq)DZ7=@}!hGUp(y5@Li{OhRchvT~pjGtFVf$7X0 zHmR6g)cxab#ophgm|WC7pWdam4`i|W^XYnX9N*>oucy^Sq!>{tMPH>>JOqRYlr=Zo;R1nY-)6bBFZ4czgzSE0B8c6;cs#pbRP6r6MExuD^{}fzK6bHvNbLUF zq))pt_X*j3X)=i2uTDNzyI)f5es!|`5`8`FDv*y|Y-tj^=cat@%G@Vp_hBHrFHb#| z-7hP4U!JNzoW$3|t^)bk#Rey_+c&+NFVZ*d6GY-ap&)-e6{Ppb^kW72Pze&5uK)2A zz8*mo$R`N4V@Z&YcNOGgzaSF-2?cp?I!KToJzkJsR)YL!y8a%0J%T8ZPY}5UX}&+> z)4R-lLUs?&1hLzkd92>QqS$TD)DP2_*_G3k0{PgLTchTQSs%MH_X*kk^FVfw%s!Ug zuPSzr%+~*W244@m3glR}+>!s)IUoDjjVH4y%}Mqte9qbSl(1ce_`Dp!Z5qM1vHIk} z+1a`KtoWIB?!G@u4o1Mv-ParkgJIZ7VfP)Zg1!3pXYpOG|K2P+8P7RR#@D4P$ZO&} zBFz!@njR=9oHn<_KUd28Xim8q3Q46DSfv#C&OGTL%c;gemMMko=w9xP|9H+x%f0cr zy`8pj_r}e8dpf4%=I+`G!3((kd;FF!J%`ObyLb-2qV%x2r@o)Q9wSm9AERoesp~#Q z|4A#&A9XP*J)=o|7e;?oF*>Q&f3yc*52Fg?V^pm)`}fjDK6I&8lHc#%ZX`JZSCZ#; zUr8R^WhMCy9jbSHXsq=Km1HL^SCXByTuFX3FLePQ+}TxOYvTe>7kJhK{d)u}OVW)+HBRKbiNfqJCYrFZK_b&+{gY%8(0(`fGcYi|XLP zXZ;PvQ)a4C$`({gAvg9p4|AGPesYI1)hZ9ByCV-X&W=0`>pSwW%Gi+y?`%gNT%a9! z@R4`q!4d7`Ipan5HhOEH;v5_39c-0{XDvJOVEH@p;LGgDgImzaqiwj(6K3Co-T&Y) z@C|PP#Q-LbKq7J!{4e_)qH+|BAE07#r0Wz@c{^H0_F_LCP9OMMR31h6SMpIP_I!h& z|5`$SGZ^}93H>b}6nmSJ@^?CAV2&R^Tewdxo1=KZj3<|varyL1Gln#dJmEp-@9=|8oJ~>>I+1mP5?XyM zw2Z|P_8)N6d@1t>_E$euz#%i!(hHReA6(-TX4O*3z)4yh8ec6{ae^A>A?YX@pYqbx zF|VwCz0}w;t5k{)vT@0BRWEd%iaW)C#{JFGnNxHOjgHCPEmXE}6b@(GJSWi4v?Ilp zRSU<|YJAk!wEJ-G8wYo-Eb?YYS!x=f|y*q)7*_%QbtHU^r=9Sx{$_8QP~3c|h$HiZX_` zd<=&?DbpTtpdN{iO1Ssg_w_iCrfhUJCw-?N$;W0_M|#Nn1_n;w+j{yd{+yV`&QX{7(>Hdv18dy z^KN$4Zfp6rcI{z=S~EgxIj?3ugi0ahHl|5 zKm|6B4nP+p*<$PSlN_dUFnJHXIMIvq@n6|^ zS+6}wVkRB(wa=VBec}4`tJhDT);+rZgdj95H&puyw`TDlLyM{ix_XCD%ghjiq$ z-NV)8!elJ3KaQ#ri*H^0^cw6}9!INK!#x+6?qgWAUwa$@kEQ-pVeNw*6>$f&p`wm` zyIQf&_&S9`_4Nf8H>fw{dT!7)8FK(?&Be~j3pgg(NG|rKyk}$flgh(a3Rl`%;*l0F zIw!Vq_f1guU&7CtK>(K==R9wX(7A0o0nGE+uY^QBDwg<^GEO}AtW;`+1}-jzvQN`( z1++?uVKShT#E%j@^Wf^q=9&LZw5g)Fu=lOwlz~n997Cca}NJ~tUaj)%{M>+l@l-Kn-tLb@2>hs&_+sa<{<(f zM&PUVpQaHaV_tRYbGW#M45be9EeB{)Q?fE9U1l`%2xVpv(40|RvBS7PVGq4sy;H5- ztLijDmGt8$9%;IXQ{=*=<}#l`0BgjleBAoV^(-|OOLuM9>gq9AsJV|&_9M9Xu$Q95 z-3>EM(J=}}D4@3YiU0qj^;`^tV1V7Suz%(XE6j1!YWK#^l~&8uc*0J^*^$zZ#}noh zaWz4~Bn8tHP!}@iDcDQFQ3@6)7^Ywd!Ou^GgdC)IlOhkRckCxiwLeY4F$xAL=tE#9 zZ@hN>!r4XrcorXXl(OPiYSo3acQL`*sq6`3Ezbk>g_RXGsYJ47?H-z*uB?qvNV9aF zZ>f%lu9N>!#?Fe71 z7^Pu^e8N0AE*36EV#IHUYF0b4iJ_+*S+A9=?HD`(x_JT57k3&N(3|ILllir+DdMO3_>2d9H zY0i_meSF!^ViATz{#)Rj4RbD>v%g=kyUx z6JMTM9Q3wUmgG+4dRdvoyV50?scq(xj=U$*xsYWm;dlaV4>hf0R%-J+a z%UOvgr1oH!6G79c)ea*JCqwa2R6DNCKJF744E1YCZCo4DGBON}Ycu@WuO-4dV6)m# zXozq^{o$mR3{7i0KA|ah5!{aEE(nb`A^WcwuM?wP{eG=g)QiPF2jpN#&Hiq;p=%x`%YICSRq3r?o zFw%!qDOv7mryNSS<-iSG%wV=Cm|v4E0Q+Ts$-i zZ2&8@2a?3W5z2%nLy6FgHi?=?wNYTthX%rFa4>{4r|_R>4ndFmLmj>nNVA5~l5|&& zEXCgr!;Rnx8}OE|SMbf3%t*|BAZ2bqpbvEQU@u4fzsUn77A2;9R^lV_xP)aOq6Y z3FG%QTsZ5&FN(`x5eIAUZdfs3^*C7anp+D88+$1W`-H1ux&>GLE5_LG@{nL2${KD? zuHzac_~Yw`4^9i*hQn?k<$$*X4##P{&;!S=9;5~C!O4dy*o$oPg+^@|sBhhrp(=QA zH+FqOJ+&K_PpCtT1)&t}j7;OX1~|a;2Uke!V?q-#+MNckS|cjfV`3dh*e=D>%2i|CDZe;Kd}!r^lRH{ZF;(IPIGneCa5SSCw!+#9C5BaLr9pOF9wUWpgf$7<|X_ zLtca7P8-zcmZFv42yugY=wTh-h+D^nO*~ra!_gvA+?TNViVa`fUK!_VNVm>f};)-xTnPT>{Rm^c%+J34E)L!D8|5ncPMi= zVynGmR=cPh7i6LCF}gKK>K>-+f~0O3HK`kW)`%`CeUqinDSht|OQiG6Vf!3j>w#@N z^X>`wX*%7QR-U}vvSGhUhcmTOUUlB>L3_um<-4V-dVPuc3cBAFxp8QoM%+AuK&=dV zvvBfgOEDKym6)dp>TW%T%-ikIz1t6|58LU}rx!1rd;3$^R+1OJ4V~U$qT2~<#*iBb zfISYkrD9;)v7z5C@xEUPTZUQoQ|;j$3y~sLar{TCe{#TS0cp`nT5TT8sm+6Rm=G+R zcM+gI*pjn-pyej6U$}Pp?D-4%bMNHOUweBgf9=Bc{FSSV7qI&gGfSHlY*(6}0ZnHe zy-jhg-S6|TyH@s5$gEao_Ca-5+p~HI+7tP?2s7*v=E{w;OK;$2z4>`C93(;9-T3X| zKGxyh#}zF;SjEQ?Q7V3_Tg4a)*YRiz8fQg&bREh~d)PaF_44Hl=Wks4^aZH=?2Xs# zlxp8>u~w>9EEW>jXE4}nMMxWxDhyTY zkT(e@Zcs~v4r!^-n6?LwGT{ts5&UP`MKmV=(b|{n@H~Fjw=n=<#wSa|posoW^9JAK zgvb~M>7@p^YJy9~fVaXls^gUgG`Arc5!=tv&LW?KMlJ^I9-QWeyAYwt2iLGM#8N>N zfg^zk0qf_)GF1sm7`(AH$rB@9;3X$04KSHSv0bqO0hpe*KO+7YZ+)Mq2D~OCCMHo& zhvTnHj=w0#Vd-#;N`XoIVnU~L@b74t>D|1zlgEKF5G95r;v>`=wu zuqu?e^^awWHKl|{2@PJ+Ql{U2`5L_eOTK;sTf_3L8x4BF6>WI)E3b6gFX0^8ThZ}s z0k2MhIcDvBSFz=Nw_LjCybeSS5!iupXuV`U3yUx>P(VG@4(m<*A#H;_(iS{BUgm3r zLEDd49->=*l8MVsg(-r1WX(h$H)(upN96-mY`?5;VUviafOb!X;>g1C+Q{_s-ZJmQ zm=?fT5ZAC%gWW0KOQMN_dtolMQpQ6!-XC!H#dE#%3^$LLhRAV?_lJ4$J1Som=4_zn zSxeLC06evcNFua1GziZ>fi=qvI?aqW0iQg@Ift}`&;eRSYUXdDdbisokQl{}yUiN9 z4f+f^1Z;Gj-B*gBTX3&|Il#~x+z|}+10Vc%vNsqlndnhzg8Rut>G^*c{R02+h7gz} zIt>hw`$|;QU&Y&9dc-&4+pl20Yy;6O3F zUO{=d0jgquo}76g_LC>i0#b_IS`GA;GK2Pl$Mk)V`|+ez*?&PVT6ftY%0ALww)@cb zWF1-LnqNe-*{Li1ID_oJGizT0OC~)vr5$}fQ{WV~N3T1*lx7v|xY46n^JQXY_L8?4yK!C|>VBEN z&LqZjb~}nq)#5#ar#{u0c)H@wW)@MVo%l2zVPJ2NJ7p%(-NE8`B@T8A!E`%L_2zkF zH;O?5f|{KU7o*NY@8cPFu`s=LWbN*9BVAn%dVh&J9PRFlNHj8yd5AU&64-ngMrRw~ zc$j7?ZND}hN+4$%Kl59_&rGyqd3se)KHpAY&v^?+I!f4eCNI?lRb1hw;1&fH3b5?K zok|4c3P|4v?-w*_Eeto4nr#CQ-l=)y{&Wwy4VH8jXfEUpVcyi1%6x(HXgSSOCl3V7 zm`Cc)Xw!QNHn~+PogvOFEG$G6(>Hd+)Mp6O_}*v|)A7&iGZIb?B|j0Lh;JsJPwx4D D^)FlB literal 0 HcmV?d00001 diff --git a/src/__pycache__/simplemetrics.cpython-34.pyc b/src/__pycache__/simplemetrics.cpython-34.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c3167444d4830e6ef0aa4b4d65305549e2f8301a GIT binary patch literal 28847 zcmeHwdvsjKdEeZ-4*-kBgLn`CK~k4*E=7P3y(vZWbX zv{iqq>o7g?^4A{xE)P zKW!_OS1OIrQh8gIEVX2-w5^sRDjiW4QzNPrRq3cuL{%xKo?&WCmEvltOQpL&6<6tQ z#jr=hgoM3nbx<;)fTT+IsoC*{>XRQ_&>PpbSqDu1uU(d_$F{(gz?Quzl|{*=Vgiw9Nyw8ZaF z`7#L>rdDnBFfeJcN$%Ac1wy8O7xzfR%@RQ?H-f4#)f zqlmhc`l05b^DE_Cy;v#NoO;E{RhE`l>IJ8MwcspO@`aMKRH(bfTy6Rm=X%Qa`j_2e zxt>`txxDUt@kX(R{1;M|XB|P%`;?n6xP|;oF;_qTjF&8wvdgtXK2t9)6`Dhj=_iLz z&f!`im+}&Yg@t0SSSaUKy~Ja!cvo$?P{^+=r@Joch!-g?=bF8b+$gx&#e(z1#dBVC z!7WsqV}25!R#=d8`0>{@Q(pH%X{C0xSYB)<&zqQY_yE$2mCVv|u~=#LK6&v8=ZRxa z1n?s~*HBmAp7E6`+Y)Rg4dtBm9s7J}=teY*B3YuX1 z)Rc44B(zbLD%pGn#bnDlaLI8T5WWFvj!E;nF*Zw5$!`7j9G+Lo#ax!77cMJNE4W3| zpgo|~w;X8v2&zR)4qB%h_wOqDTXzoA@(FV*HT_jVp^TkrJ zzUuX0pbXilt{#Ksgeelma;nE5@2e{1Iv6qQ=HRO10vS88VNgj%;HDWV$VYK=*QYF*arwTN1as?$|R%`$4K zH5*S@i4nb&vYS6~&Mjo?g_?76#>wLgz}RHnRj0BL`lc4Lxq8K|IoWdFDVB?M3|6s; zEH{hwt4MSTOUw1u1P1~P!Q`EGp3&c9#?Kil`DzwlE?zHG+@);&l=CdUTSwD5*9)t2 z&tiCUg}G@dZxLOUAAF|+AoL5&tQA2Xp=ybiV0`q}(g-0HLV+i&&T?p~z)U}VV`{$ONxg_x$lo6Is+ zzYG>J8E1DAbl$|4RPHXkcXuy%BQQaJFh<)d z_-NgXnepLq>1%uQ3S%*1#-c6H0xH>T5Dp_f7u8Kr3H&>RUs(cw4hth7D3>@}t;e_u zVm$-Q0!p}UD|}-4sJhhnAX8`N-lNvzYIR!GvHBrjOiirC)dZ+R3CL=z2_{aicL5{z z3yzE{JZN-!P^fJc%8G|@w(cNXms&SSbuFgWy3|^?n!FFqtfsK;#@NX5aEr$$@ea6i zy+@Hgs;)FWv zFFop#$=V~l$y!K4kt}9TX#0egepGVl5*_OQ<(#GQ}g__<2xEW%$RiKP0VQ|FuUwG zFIz;h<{SYR`*nO)Q6Xyyea73)qd$dfBoY5>X=r&(|- zQl7tkuPiSYTxY(rg6$pq{_*L1^afs7t~ocdrImt{FDw+xhRgMZW zUK9}B-W37{(}Km~4Ya#lSy{a5%&#nf3Cv&1E)>v@s~{}-BT$|~QQ4B;3j+jx`z!qU z)3Yui(JcmETzA(?=;@Kc*aUdNi(JTF@RS!NYrSZ8rCwLioH;X_b;}q7P+T%ovsS3Hp=~2HzfuQFObP8gpMKd=+G)XuP&wfmrklsk zoO#T3D=u1s37Wf#reb(x81j&M%J@o3&O)hj6Oaq=ZTdp0%Zro>WiN)rY~AZJpOzQP z7v@(My_kqWUZl2K^P+`Dv7SnJ-EQ_~CZDZm_5M1b!IYWIg__rsTgsO*S$DCPj$kgN zM_XS9d0o%Q)bwJKFRcNxPD-#4{W?8pT9P3Yhpd+G+@1#)3mRC1@fofJD;V4w!IMhF zt$1X}irRfv#G1x$FMbd5UwbGrZpZDU)q`-r>am7^uA+z~twcvGVY#PKRxZvG@iU6w z_u*GIl7Bd;TAK%U3GN88kzn1g%leLG6pMKs*je@%z-O${&&!J3m>ZP+Ji$ z<)XTUq@yU47%r;km5XAdH?+zm>Y|>J4u)D~`Z~tWni|H|lS0+os?d*O`wCc)+^jN_ zs4M#w?ug_hlr!AR{x5eWR@ARc#=x0`fgnK_k46nIdYFw!C%-JH-L+wBAtesgrkzSPmNuh|>=!4{aw6SQDV zDmuMsm%g(}m-a?N{dlnAA?U~s-4XIdM<<9@ zFbKaGf%2@Cd>ZIxw){9h5q~D^i2GvYt*l9}-rk-TQJDH-4{zdKxzZsgw%B~75F z>ws5}M_Hfl3sIxrqM)&N7_Oi6d)Vrb88tIb=yGQ!mZ$UkJ^Vz}^w2MMiyr=3w7rMF z8#O)j^94OL3A%@0>6lq3Vq1MOn>5-nJ$-(YPv%N&<0tbzzi{)_w8s2p3ddXnQ}ov< z{5Gr~!4#O}E7OYhxn>e|C%dgWK(v+y_Q<^XjkwP(y;97@iGcUY+&CKNjNXU7nZ??F z#F`L{*M8mg>I*MNywehGE3ivnvfCA+JbnKnp#uZRNc-X zlc*Uqxq(4{)wOXIJSu6FmAJ&p%IN-47hA=JJk$lQ@#b-Cy-$|E&}QQA0PFpfL=4kE zBjtp9*QaEUFzi_C_jdv)<9%NK<*UgQuqKqTr@J*I6A;Dx5)IE<2_c@f5;f1}ROh71 zN~wD0$xPMloCs#>0b$+gvR-KnAt#7Gl?_3tR?HlesA~-tm4kX1TA$FNAYqb2cUadI zDyo&Ji(1>zfj9Q}UqwpgkEkERNWrGSMepIBR+$ri)C_SjQ;+H@tC=3NmYRgtOf(6h zMxlQuQ8#MuhMD-@#KxKU3(}nSnfP3SGm&fw&BUqJqs*r1j%PA0M1HLGDCEaO&y(u| zGLsGCSs6>mrx29v$IVcXgeCgPERFHW^+C0UzXN9Z$IS9s_pt=ND^AFE*0mC5F?qPV z1$>j~+s49s{e7o3Bg~p^z9rdrPGIrGj5sN63cw&X#a7~0)1=Y6)ihb`ZDp5f!s0up z$z8h2>N|SPG%*RSX<`!Er-@0_jcRU~CcS+dr^$bm=Cn_fZzl~iLPOvBh(Z$piDQN| zGz~(jp~b3IKxiclk%ksTrcT1DHo3lS%W*Q3CRBi#I-!)-Seizf6*fd-7TQ*gLuw6w zx3#97KNJ&voKX5nCX{*6S`SS^Xh<+8O`;x>uWeXP{zu zgsongoU_%3MZ(uBu}Qpw#LM`d#Q;H&NG@Z%Dn82 z#g3GvY{R7iz*IHzus39AePa>xbfzI!P%Gd2%0iY#9~gI-?dq?r)TzNl+cWv%lGkfG z$p?3Y4ed6n!9C3+l!w`3sgP%@o5SI!JD*xEl%KrV?72`WPd#!2R4%nm?luM~21gKd z^ozXM&Ua8nL9tH)zmd7soJ+gMg!9k6yo%ezBbHfFoefbTgnpFdntxEP6M z_lroSiBN2Q*WfN;Q@INZ8mwke)MDRstBk;ChW$vSA8qNX%9e{4Q1jaA{ljV<8DMO~ zBT9*?S&+Z+GLSob*Wfk9bga*fT9{;KpKSj82b&9nYnrO7Fbey6}7Vog&l5! zZKXV)RIL^#RRaPCN|LH(<_`P0`c)l}19^K{#Td)PUb@zY4YD5%{9$Xhdc>N=mU?Ye zxu3J@*zoWg`&*arN!uy)vut*K&FQBL*}QC6?VFNQ&(4?NKEnMj>*Q)ToCPe#)3>b$ zZ_!kIp&6YC7low~Mk{1*EeW?1r@7*BU3e&hYHF;iM>P+_W|I%Qf@IL7>C>eOj7qhK z5b-lLdtm2z`st^hK69oy4*O!cQb(s?bSuKj<9AK(9PG`l|NYWSFQvxPU07b}o#NRQ zm~)n~<74Hokq=(9f-T7FVFTFUntL}}d=G;c*Oq%EQ1oo!Tp2GgoqAI&adN5o#9D8bBtO#3J!+jPF(EbLplQ?{o zU=L#6$Yz7^mtbR!n~#ddt)9qXtXxCZc6-q3iwxrzX|NvkStF5TWGFHPf1Lqq!rE{5 zV3l)g$d%(p%pdO9#5d>gTl*QL^6QOB7?-e1!fw_0GFOk8xi9g{#DYZNOIZyPTr_gs zQk5C5Ki8Ei+sd}wEApvhMv8x?Ro)-YK*E@E*ZJ9S)*2cyOo{M;LaC|)%}>hO(nf=a zE?qI!DC)_7Gf>tspQOQo1}jRd?CKo~e`jeH!GvzYZ5Nj%0jw#U0%pFot4nRZ1DzRi+ zoD1&#(gJw|Y4v^^e`jHITF2T8Wzvw0!rN@bSv69zf;K)J?$|p+9mAqLqV5A@xkBqM zerH1$VIu;sTX2xOx-%umFgcvFXK1N)L)9uVuo+hk=9HC zBLozgCF+yX?FpG>K~_#Vu1%@Q=EE217``>YD3+b8D@*Wh)6FO8geyO-^60ngRN#Yt6&-Q%aOZ(P z3tf|>68ESBM^-pnp`SO3g`3W5Wd*J|^ue8Wr_ja=&3#WHv%E82!6Aoo;}3XBw&B~> zv$w6eTM70o;;cEhtuxK;=Z>G6KDn@dplLt1e%qRAmd~6y^N{09)6SFHU*DV({J^wxv8}-sx42j=!(m9gjXE+;-vVNKDb?rF z$53Mz0wVYhRb2X_2*`%*S^YL;?4Ai62BaGABa-h zJH^_NSbS5v62Xbp>+|EHNJ$U3J!Wpw;VnJnCz`Iq4_oA8FImECIFM!NyDX5$>k&ki zStXAjdu1Y<^MH*&4l zMc4(n1ZkBZv>dF^7MmcEa9k z4?uwHhgjQhjRSN|Si7y`_M}Z0=^;R^I0Wgw$ONc&Tld;~?45|k?VWG~0!TwV0%5k# z0z3nVBU}Suv&JEe?nJDeVjFV7HOQ8h<|1q*eunWIfVV&E5*HvqGue=URUj!s-oxSx z#3WK9u?h_7cn?_+3oUSr#t)GCxZFxsf-|_Gl*A1P>oJtU6F`Iz1_}A(DK1zgl;FP( z7ecJ!Lr7v3Y`0Rrk5vLcQ2;&E_fmTcfK^@+cnJR`?B%G-V3oMWDv(Aa2CH-#tnyQv z(UTX|pMtO(6ZsKXrAuQKfC$Kt{!UJ;0vA;O5m2sr$im-Qz)2%YCIzchEo5WWNHtic z)iI4#bjKj0_5fO7dk1rL7hjYL0w$x2FA@;GivDq+1e>zJ>W^&5)%kw5E{NG+71@5V zo%mQqgncpwl><(`9Bt!=EzF^rxyu#|-*g^>Qxe5!LA9N;1|6D=fFkdx>v83NAwncW zpl3fOUDH5V(?D3Qc3rm7uJ)l4v7Ga55@bPk^ni`l2HcPaNz;*9T?BHtE! z`2iY#g*h-={c3>L&I($4RTJl{Kx<#s*fiXo7vrrVW5>4YPW8ojRrlh6^x|mzwf5q} zF@O4FM*@(4RlWFd>~i(PF+tcOwPN9;;1}*itEC5}rGN68TY5I`xAgO`zNKg5m#b&v zoh_~XUYyJDRwJO>Zl=y5Y5s*SW(ao{DdB)(#h`Umect_+({1*@c~;;>Y4Q6_n~ z3Z!hTBFJ357{y=dTDT*h?aJ?@lx)sIz?R<8@ZajfXw>$AKBr>0{WG|?dGz+SoVLB)cB1!o==5$X;{G_9{XPcoX91SPQ!)3wOlp7W{!O8{ z9h}j{>B-i)T*l-mZpylCPT!)m59mTa$bXVr6-z6c+hlN%!E+3-yC@Mmw3xEa#~)$v zqYU1PAT_LwXMup>0-P$X0fiAlxj)9>9Sn&4wQ230jJ=D&yAh-zD*)tE6mZ|e#J|qq zZG1Dj1Z69ZC}(^jL<=fIoykR(c_0BBzed5fK>zpp!Ufd26EqTO z$wHoRa13hEP=fzDTnI&t9@ThW)d5Y7kg+0qLue=g^r0q=0*i~436P26hQr0b)e7v8 zjs?0p5Gg8y$q0P}_8(&c0KaWy4V+G6In`fgp>4z_na&Q^q)MPT(V;df)DF(fEm{;L zV5PfD0MA8>@w;Vv`Je?&B8ZSO0rMCT8sJkNx-G~Vcr%6?%dX_TN9e!2ksevv=^qvP zdxien79e?h`Fn`p!sX)zh@cIArHzkk$oaS0xwtClBiL9niU1Bu`13qj5TpUffytM| zUH^cvz6_Ulpwolun)ufGlwdPCH(Tum*4Gzme8MK!0qvp9X|I6xGJZF*x_Z&V-`Tb) z3CkqRh$8D=gK}C_(Hz^uyn+G@2=wWkt%6W#Duvf1P%+fQ0O0+Q9*@vedS__n9#tb_ zSdFkWEugR|UQ~fq3m8_6sF0O(jp5G%)ouJNM*O;9&D19mb!DBi^GlIBuI7OFiVy0D zv_or4kE?OoKF8pkC$kN(#kBSa5W;=09U*ALR0_+&ZR==rq_qL+Wzd#OKk{{|D@%!HOROUMZ(dVKo$@is)_MRN;_wdU}KjxH`=DEYPx!{&73m#UZ=&3Aw=_z>ofw;5%zdS1gcffB}6*vL5lB5F}2HOvX$UK?!=?tA*N zi*I=3++&$XuVgMgk$z12+&nI)_X;jFWzY{Umlt!{qFcc+L>xHGL7kY-)_Afm=T?@T z8V-&+%{@o&zV|o|iGAnRdZ^RyUHcKbQzLwNMxu2)1^`s}?vJ*MJ>Y26LP|TxT0u&Gjj;@azl*?&{iHd`dIA1Du=Yj%8`XKsoNGQGS4ID2H%aIEUZboABM@V+4A^0*g-)el(0|hyzm|7HEMql(mrv zznLzJhuR^G#Z}{|-VXN@#L&qjg1rm>V3F4fP#_k3+Dxm`?NQE+<1G-7Mgp)=QTzBQ zdcpt-(R_;|jP$I*Zii5uZN~6sjI}N5+A{66t-`X6A`{}xg0+TcAY1j$R)1DSflFxp zOJW-ppZng;X|I6xGJZEUwE9**5ne^;ku?j$=Nm7>stBVKpBcjzsjbBpUS3KKHm6?M z5|s^kgS-jj(WFmAp=4C`+)LaKGkJ^MM7EEQkia(Ef%{QDi9~tt!a60&roAt5p?Lz< z8ToOZ(#9nUX0=m+nT9r&i=_$#JfE;R!4(pRkV-m=RG^#%&8Zk`M=!BEucSPq#_d=tTU`+fT+_=RXMx7YA>v|m>8ef&I*`m;X# zd7N^dqRVygjrcUpd%`-QqfVd@f{qB+a`96NR4Cu|S0l=U&A*yMn6WKqDj1~j$ldDI zd^N_Mt6iSQck@<$H%Cy&tNm^uCw~Uifxpx~JP?!0ZZZ#dyiN8INT3?%iwuBm086Ic zC+IN+QTcgD6%Zd~6T#|_M^IqDf%_i8Ja-3dI3mjeOJe(ge5(ah{|OQ%j4}Qb+!>cd z_LYKW<}xDt_EB`9Gzo64<)9&(w!L+o5rwe3|JJ|O|vcQ zE#TXh&8EAkh;NLD=MJ0vk9naBLQ^?9MP9-t$b>#}m3!|PWqEk4l;_pv#YAirlXN3? zdpVHznIM|qF-r7&5|!>4+{UCt2$j0ux@sO-!Cx3teKMjVYHMSENBV`H#=<2FYO z(unM;)z@RAP15CtK2^g&3h=4gF5KKs0(jFpSRby9OOO-e}LNKeyK=*f@yH6oj$sSAW${ z@<~nV>>3=5&<*XSQvH}c%S@1H#R#-6lK=Wnc4-~H#h5W*5N=~c4&1|tnYjza+xSA* zubt9_K{c|D3593AquFl+y&qY>L)GDp1Z4;O#{h(O^LEJRarC5)2H?gx03+O!DR)>J zB?@pH?uG<{!>y2VX(cr9(6K{*4oi3uB|9_=-V0?2ANJ7%b?@_5&pPxas0f)3NbZBw zMm!J4#x+M>+o7((exRU*hf#Tt43;q}@0BJ)v4X+UXSREy?1ZF%wAvHJ-&q#2Q(Z$d z4wDXwh%xB|YDBw^@J>verG&E0!isPyHFaTgxw|>MiOenf`DaA%KHpN#-?k2WR^zsn zy48in3o@$y(e4$1@}-p}+&c5Izr&w@e|GV?k)M7151TKXO?NF67b#Juqq(K*aytHW z{z~cbaymiT;$|^lzv^fD%-M7-TZZ3nI(h?QD#L8!7Mc&?A_p;{;NJQnJftHFOLYi% zCl}H^LY}#PvnG}UFDmtP=jO966LA$}!DUY9b9r@S43``hO6hJF$18ZdpJ!jq!o#<) zP)|pev-w-JIW%{6il9@tn{OF7ZyG|g*E>I7Y2dU8uAzaGOr3^<)E%2aJI`Lui*%JI z*7D-@ig*%VWpASN+||A{aQ%fxu!w3}$Rl4x^^J=z(POuQzVXP3XP1|~NI6>uk6bsq zn(mpwfu{2=g}BIaiJm#dMcn2DZ=G^2-38TlUAVnkFXpb-(y>OiQLLrA`2=T|<#cRS z5(c0a!1bO;k+gywd*yn&phDrBY36jBP}8U+W7@| z%4qKxj}k1rm9!&`m#CqgCO)X0b95=oaJBLVrQl2`Wr$ZRq4ZE%OPLS8OGCO0k^3*m zv$vB;y)wd?271J<=!pAHyh_LARfBFq@F3!8FfU5(ngj}^UQC>U(p^GAei!o%OxsGS zmhR)!f`m*SXI9%E8)-Y+KrVO}VH&05ng<$yJ8YYv@ZFq5HQv-|ZcNqMuZ)>qpNarh za0aUABwVcrZL0EdNh1DGq)k8x57oTgAL+pnk=^*0zCujjZKa@`zYC7rN&LG_{D3IL z@_pReMS<2H!$Fe)sO;nRWQ5*G+wJ4lK6^XL^Zks*Ecf4`?%-;mop?y#_Z)s}d+^0? z2~xCvQan_EP4W{1ZYSl##E1S!^d}^qezu{A3+ef-jjr|p*hPwJL zh^{>%@M_Vunt?(TE;$+s0Wra4!iPit6Sgz>>G)6E1WjSbZbMT5IoNZ6q2P7_2VXyi zoeUcoKo*h-M?PQy9&q>9aloDtKMwzfZUCr$RXt6wYM=@8AOd(&xj%1FWgq~6XbOx% zxfnb&U8AX}GH41;x$;VPX1i?f*!{mHK*Kcqcl|NhyxBNhgB=xIV?@9No1CVf3_?>m zIt2sdgVS4^e5gc^C{oYzsf=8WuG?pOB+Q7}svU zybdJse8jd`b3lSx2rU3MfD(X6fSo!?P&YgcNTNXpyxc*!8-$Vevd}KT5fTE9fMn0c zvd~(96p{d>fMkC_f}{g7{n@-xnppTP@B!Fy%y<(sB4`YviN#M$uo^%OJ>QN*5m}!g zAZ0f63JyB~&0(MuX2Kg}0_l-L8@Q)ww&EVHA{U@X?okjh48$UISfAp9=?<&nuCC2ksi&^t9!`U|Sc$XOt%nCR zU&a3gD4Zn)9N}o4+H^hzBq-QSIoyd3rkxI$7LcGiUXl(bJ|rY(H!RHuIOX>auf?II}O}jGX#ehsIaLt~F1?IZ@PnhiAV&1ZlG$XaCsdU#8Fff2{ zb2zxdJVat&K9!E(puF~r5K!Js$`$61X7#n^k;c??I=1K*3gzbbxh<|QZ$@rRO?lCI zc!oE}K(yKY;$eZ~qCVPIs5Sdrjp4=Y?&sOrH?TwFVeM`hio5@eH2<75ZO72HrOvKR zH6vHi@#rG%=WLE|b%lD2v}I8?N6dR!W6>b2Lzx z$sL_O-R`!wv4ldnajD78Bop_GZ1R^FP#OV%Lhwrr78uC+_G^p@tX^kK%uhgN%KeuN zzKp<&KsdvJOJG_r-Vki<#aGS0#NUt>CVzhrz#M@LLSN!r-?Vh#LG~Gxk*mfpFl` z1IPUu0x!YC0Q&BKubXiWYr2OqVPYqA-=U#AK%z?*QB%27pgkcgdT5c!PI<`1;-U2q_*m z?>gVQx6sN!3rU$I0QeJFQ?>;7@WS!6kS%}@)GubM%>aII-t&W^N-_Zd?E)rok#j<@ zCJw1Vk%ltlP*_Vmn3;RMxXZ(bMFadVP`OC=U!V!WVA3X@gGDj3LyV+QHuh|L&v9qytH@|AP?~& z)R|FX?snC^AYgVDLa9V3%s#Jmog22XNzQJOW?l(Hhf%w3JMXfjnePf4Mz!J3}5LCTE5SY|D zPvCaA_h5F31(nzbe*wm*yTr8H_>P>SJUFQuKPkCF(&_sZwUkhW>TIHl1U>oj_aoF; z(u)k*%y8L05%rjpa0+7i9r@9m>SK{jb8Swo+lM?MS*LMX(DDTFJnEVNB%hf(CsPA1 z!%!P)a0lT6{fC+qkCRU5jYrWi6M9gkZdFhuvZ(*m<5GH z651n9Y5>Sg(&9uL8z|^~?`(jQwZ(@218;3q60m{No$jZvIQIJqtt})ltye}f_lt^H zb3#R>rJbB;31nMx0(U=&(T5Xw-BD}kycRCLCBlW{BV1t;--fVli-l+ogtCBZ0&I_7 zXl`Og{5@`OvampBrDZr2)oW=SEZ|E~@VY4R_c%m4#m7K+x=D2ZEKY|y}YUg3jME^ zHV(2wI~eR^u$RFB28S7(X7CJy8Uv`Jlz63!1DANLi1A&N+X8@vgP}R$FbBb~*L@lQ zf2CA-h%BrD9;!H00~MS&gA*zp5)%-T(bKA019}gwg3PC literal 0 HcmV?d00001 diff --git a/src/__pycache__/tracemetadata.cpython-34.pyc b/src/__pycache__/tracemetadata.cpython-34.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7fb3916000088f2ebeae698e667ba613cb113840 GIT binary patch literal 10662 zcmd5?O>7)TcCPB4e-24e5=qh2PfNDe8vTq&{o1ttNy-scKaA;(?6!!(aHfYGYB)2j zZi-ZglK|Fk);T%81-n>aPXU6Qf@BXlBnKlw00YS(mjHd(1o25g;@iTo5JdUjtL~m5 z^|O1(0wLMO>Uyv0y?Ryer(X3p`TpGB|I5M3FD?l2wdnaVQNM=AeU73kgpE>)sv#Ou ztQx{L#HuN5Q%GtxMI#~X1Y@9{6m~MIr-Yq~>Sb~dW#gq@4(eZuaG z>Um-3qk6xv`?;PFjl5VL5cUB00Lhg2AvhfrRglSWeV3?W=p5IFL>0s3xxQOeF~k7Z z_lPRG0CPzEPzrmmr~gIa0;?-MC*YaAHv+1dF#j{o&uUsvAWvkw- zwbW{v5UiSNty)gGvTPBOQ+x^mih>agv{k+7m1^NYL8e-NRCm$iZb1g=In}B-u3M3W zjh{h089cNOZW`r}@h8QP4N)_B?uPgyAv!GlYJ!;E5@JP&nI}?o46$VhcR_dnvshNp z!eBt37(#lcSiy3o1>R*-2rns~2=T{pMMf<~Sm~HiB_WnTH_2s6Y}!O>X5j;DsZpJWPZgG_Dnw5Ibm)4?E2=Yc6c!>XmlrO%l3 zrO&WPcw74t`5=2}pq?U<#l;NeKx7 zbX(CAe5as`!{v{mBRA^}&rwjuWye~oKXRH@qwadWy2;Atp0w&kr)WVnE6erfl6Ab; zR*$T5vue35<+(WvKQxBMh8|mGx8gLbgxgY8M_G$y*Qr|2YeH+TuP&mkRkPY!UmchF z#3K}Wz}d}8W4#JcxzVs{^#*D!peW##n>{SlmzpibXl~1L>#OUHvIklrFJwnT{_3vf zR#d(1h0{4+1UAit;U75Ox`JV134k^3Z?((bGC06Yt97|>$7v1*3L*Uy1Yvx)L?SMg)skDh)M!=84fjfMTfbNs z)R(xN$K&2c0UHP@EZiUq?6GBqrov6axVjTPEo^QT>|%@qJpvAlp%h;jVvV-6=_h$# z`WgOLT=M;Fyu{=jYE9n2a3V%=B9NObFm%b*y|_T z?dBIRU!GN}r7mB#Dy{WK)oQjp>|Jc1a`SPtC1W)@@=RJ!X@}HlkMwrFjxDdH3olu5 zXgg!Ot(G^jy;rfxeaAZHC+^g56#NSxtT&saXuZpc#k1f`xmT^#XtXv6BOH-tIi-lM ze<62sCv?M64iP2Y1KF0f=z#fhm@BUyDmZ3jr|&@6OO8^s`+|XZ=ap!AgQQC{4^lc#(!|IZlrU(iNf=># z)OzHoM|EeTFc@SjtJMbUr|QF)YM2HN<-mjP(K6XKjvbmF5>&}^R@?UOuH;Gz=Hro5 z4U*huhgNAFVo-tMP1+FYqHCvD^;GO+y96r}$hK{GXISA1rb-hp87i_=kOl^O%m{R))WLPsx6H8f`V z<|%UNGYdy)U_^r$`(a9`PhTYRsaX3|+`o@efA=GF>7j<0H!H@7B@Q4F#_4>%gp{222EW2pFP1jD~2R`*2J~*fbhKDohY3k$wuSHcwFaIJ5Ak=;SDz@ep<( zM2e}R6VNXLbL}L8G(fY|pDG+IQiWqXc8h!bzj_~v_bPwO!A9t;W4lwXR1kKMfhelh z1_uw?FV8g`1R`XWoheJP$G0Ap8|!F-k>yhf%pDFG4pb19uTowIy0=UgCOmOisKdv# zIA{PArX6twj17^4iPMChlKS5D6BJ$qvh`Aim`$&*k2`}y23R_R7htCb&-m9sG!ABmr#VeQC+5nD^y&i;u;m# zQ4}(~57iB7pe0l@RLoLwlZyAL_#G-J9_+?oU61CL!YMrgf_nsoFkiWqq~1#E7O>(i zL~bSCLZ3pPkYj;?CX+it2?;jg2{yTkBTF{S1cu2Q)!?KUg=JD7lSvLC`+6ZT#cRKe zAY%+6>zpD&%?1PBh-qb*)-PY6HQqa-`iCA``+FhVXdT!=3tVT(#lWz`SHQ=p4Y~P| zK0eM-q5-xZMx8vy5lK4>Eyx6EDPW4|3U>ti9U^_p=6EMkLacpDfA`6;@kv=CA+dsE zJ4Q4kBhK!Qk`-_yO({uk2P0DOh2%5;k#Z*h5E3#7-E%HGA>=;bW(bys&-}UUAs710 zL-BZ8sOJqYD^~Ox!82)}nPHzvo(Mh@Q$Zd;OW-?+%61~($+hn!e5Of06R8(HIm$n< zGlE#hsKXF(6}=#B5WXPSh718t;t&h4aQ+7X{*$JFQwxS*$2MQE{10R0ec36P4w__Z z`8-_yjosRZy~+)z1E+U$hkCQ_lBe+pZqNSS>}_j$?g!TVEqnGU<;eU&-FWxj%q+KR z3$eo{usL8$FoiyWQ19U%8-yJN+lL)RkKyQ3`>Epr8YG2~+x1*+O2~GwDG3rdBrOMt zic@cB19Od#)5>W#kPR$k*}m8rOq9>c+R~6!v6C!6wmNo>V_MzNwLsfVjw+znPw=>8 zkA#_mm$KvmV;3UKxW-W?`Kdf&#JoHJzjXw76i2?qv&m_LN3D=Yr=P=kM)9yW`!mD> zWQw4y1}Q7{9K~K}JV|**HcL7Z+~QX3+*7(WEkeXv2dJWRjLIfK|QsuPKbx5P=8@iP`d{aC=3Qt zegJb4tvac}lEA|ZZcaxSBdpVB7~>lmb{nO}u+9juZ7!hbzmNN`eZvyhM8)j{L_e<#f|T5Z#>M6 z@A6n*nZ*1N&HPtp^H4NZEukq!`X?s%atXc17`x2ZNeIxcgm#&jaouCc&uItfpfjj!`;rO99FYm zn55kBuqbgwf|7&R4u_l_-o~9FR=Vj}6}8fLl3KyLd;O|>-Cjp39(#RQVy`3Jz5-%z z(t37wN3U=3>z?TKD8KHFUcbk$!_n*8{5lf79^+SJo_C8Cq^3FTul_kfu?FV0M|Ad4 z*sz7%bqxJahJuZKqBAC%n__Ek4>XsgzK{Ye)*GUUdynBB=rM*uQp2J%E}ECb)<_TZ zCPN{+gQBxvG!Zn6_CTu)#o_#j=o}!fUa>Mi;(%l(mbx~8Mmm-{EM4bdWLplt+_rQM z2@l2*^C*bwaZ$yI7y5?V04tK*Z(|Xx=<)()t#?66sn;z?hoP@pke@R2RSWVr41Lvt z{3}CWwIIJ@XwQOdL6}`)z8pn7Pi3 zsqZ5$;gj9YAhKPl`MK_#I8Cu5sHFHXNb+ogOvG?tq9;-ui%5)ecGsno7;cC(`+njC zSdZE?uYUH#T*#+OCx0OG%v?&oMK2SR@A~NzdXjN>o>TXYX5M`}gEv2Ug68VdNf0Bw zOKlUAe(J=%)%Ls_!c=2<;vH1gZO4xhJx=HeynZsQ^p$#1~Kxou1! zMyG_cRJd8eMr|#wbWZ`@*#9Z=R?pQ(;PC$YU9g-5{O1sFklbp)=0nYoPMvYSIc&vt^>_=?R@wG>riRfCJkfK00~CT!~-NX zdWtfPK+FO$;9&23>*aISCusG2rv%IeP*}UAUJ+quW&S6JJ1pG zC6B?fV4GW^R51M{@X*9)2@Wpt#RvII&f?=} zhz+UhS$24I63mJ5ju2}H5iTLB;Pe=IKvU4A^vHZ6?@v#+nn+V41z)!GM*@5a2$!|^ z*h8iQSA^sPbs8P5%j>xJL2@16^ei?Uazd-cK;E#8@8h|H=Y2dM-e8}@7oVKG;HfTQ zQuz6kYkulz@p$d1zSJ%xf&^c@A?NH=_+y_SMFf;$wdn0(Ac41*9c%IjbS z=xqEgC95|7i@xuHSP_8FEc}~z0K5zxnz2UoJVrB+%vyyZv|O=&FG}r?8^? zN%{zcb`RTT?4{+hhtDf;`;BXu@K)5R7JYq1>6YS;6TD5_CvH5GpYXo%jRng$PFgxm zJ%II7l%eu-@ffZ@Xz6S@ZZkN=<|jF_3S{oCfBJIxW8GW#+UuT0*4&-4KBQ02-0lZz z5t=`~6SAS|o`>&epbQ*Q;U=c*ekk=5q?R17vQcFx6=WS`{~Uay;o#!gQ@jTieRQBk z2scT?IT(ZPYIok0l1ovi2ra8!2W84qEoMAO!ev1ZiiAUZc&CLujy8wXfVdNhak(1_ zt^I~6ugm>Nd+moxjN>8{YC)KVksw_vRdLZ%3UVcUbcau5oKi`VMN%deJjpkywvUQC zR6L-9)C_rZ6xXPrPzcvje7S_EnQyoGu9dHSIB~Bi5vR=_sU$Pd!#-I)eM_9V1b@2T gaIO#sF8K_~+0m`;&fi^T{%-zAel$Ok|6(xrzmPy_LI3~& literal 0 HcmV?d00001 diff --git a/src/__pycache__/utils.cpython-34.pyc b/src/__pycache__/utils.cpython-34.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fb7791e446442b12d07adc3d6c5081f6688e4a99 GIT binary patch literal 6502 zcmb_gOLN@D5guS)(_{dG6~ z-I%QZ{U4{D#&O2}#YTSb;`vit>PLeymoW>i!CaI11`AAPnJg$VtHcZ%Eiu2$tg@!S zbA?%z;^K9n` zvyMJD_&R?g{<; z1|8;l>tu2)h+N@!Z6}VTYWD`bAA7!PKP3}sndwwddSTq@_CqK3qA(qEyPx`~QJ-(yyMlY6if!K) zZch4AZfpJERk>xP$=XX~>)D}et5*{)r%k?k2|p$5%}9o?#*i1($uD2RGO**GAH}}6 z`D$E8=#$BH>`&VKOA)#vNj_YSdIRZgZO8oLdlxR9=T~j%2tNvKzTWQ%=|n*fM;>nR zRT0KQ^2Mv`tF5Oc9RJfAn$&1JWDE2&agF0z#ibqsFJi`aV7&M@*%M|wX6zA0OtLK( zC;@N)CXoM_UHEhU4sb9|9sx&y7i@tV9^cTUQ|yUZD(I$jQzs+WcD@6yb1VipFyo17 z7IfO~X(1T>EZP0YJ8k>)4u+2vPcRqGbc$_(6Ba!+x*WU-bgx5Lqu>ljg8yI?o&)kX zMmfL9+_yB(bIg6a*aeYHPw3 znXJH=SJQ1<<9L2jb3b3LQBqfJlfGjdwOSRLwUbI7-NGl<< zqf+CYw7f0+UUEO*T29R4sMt`OU&+<8=K<3kTe?c`#_q=v{LFIzTLan(lm-zZ$w#)R z5_2g?#+R0I5!0Hp9eGexT2+8XU!>LT!KU=wwC0Mg-S^|5!_>TH9AcBKzGg4j*gNm>oXt*zSg`d_HVu$F&vM<0w z(Xt9!g1pIRkc(_{Z(HD)aW**X089}Jw5e)S+7tEz+KY;11RToe^2OnvNcAnNOmtQ? zqq530NYeUV;Kj*pedA$2*u*kWp#zb!Et#etg~TUyGvQyA{J@I-x1R;lN7(a7E8kGXxc(IG9h@kV=+w!KVdnv5gKC%5?x z&^dh^O8~iY?^EZw*A{Jz!_71E#Vhht6~R@{>)0G%68vdska|&a+tJ-XfYwu>KTG(^ z%C+lP@7+WSaJL_Z6h)wf%r;1-c55;ngXaNdlt`zpUGsJkj9Rk?d477yniuke|qWCG8|(E=Wt;;6hX8!njkGJ=`3d%Qg4t> z57*9scNwhJc{Hp#XUrLO^Ndk57LAj}v{5yV8S~~*^f6XGYSfJxvtf=KlcszF^bV<$ zekELGT&uX$9-95$L6luUXr+)_qL6UyyL&~fK&VChHK|NXQ?M>K8TE27T>t8%G&aSoUsTrxXCc<40j-6Ii&&Pq)Xuc9X^nv0yM(U^KoyY@9 zd5VmU(29afTE5@8{dudDmaw#>O2rz(Q-@~gbIh8vc#&3g#{1T+f`j~;-VS?@8 z-2_9HKs>T6YLb0#vIQI{M5P&nrWORr&lHQN*iObrS6%V6uDa%_Y$3+edaqg4HJmlW z;+ex{-703y9?Z(6(-kBwZw>`6lvfPR=}Fir62JAB(TwVW=we^7fM(zwO^L-OW=GbJk#awCUwqk}qxkZ^PfIR_)L#)mQ_ z)(a~Q-yE{mojezL#kH(jZxC-s;rq$$m6ebA#+AEkx7Kc+Idg{Jijhz~?0Xc+oI{qWGN5G>r#OB?;@1w~|H3Z;z&`vI0i!`$D*&zZ~p5_4f_dIfHL_IsS{ekk- zCw%c6p_bR8*0;EScbQw}+U;ce0O9XZ1|qA_%9<7I;%ak1Rtcj>cNm8e;uoA_I5qz0* z$U-~0y&w;a!e4w|h)-y+$7Y`Sz-mfl z60i^X6Z(9BD4cckxK7!Uer@2k-ZIyo8OyC&rjqKGh@HJG`ADnCI8{uw+&1NaR#gVE z6e1m?4_r8G)hbyN-N?t0cC>;x#Ilej#@iie1~AsK`};3yV~)zK81)m5rIXG66YZ&v zh8ahoj2YmE+(6HvP1BF=L%g_x&frMCe{3AiMR9f`JV#$Vu_t&9&I-N{&5&Z?NZK2K zB0AawJ1F5hyCLl!sv*_RPC1JaHcFfUcYGVD*iIG>T;WD$rHCnH5?tPZ7;CMvq#VpD z)7XI_l47(PRMf}kCkm_32}=D~8_tU=sncwY@#-(&mZ<^&dc-Kf9^`$D4efwzV5o|Z z(HYDhsDdz|&;#%idie1FqlXiR=^jJ4iOYn_f8IjD~6h=YCP=oAS_4U)`4+ieyCd*;>2q!d93Rmz>L>bCnwmX)ugE`KBCduiogMk&*c(uF4Pt#q(dwfr>%h{H z(MU)+*@3U;-yqSQJ*e*jdq-1@4te2!4>>Zz4hd~^z}FR1mMc&Y)v%!5LmP%9EsYqI zj^k%(J7XWAl Date: Mon, 12 Sep 2022 10:09:32 +0200 Subject: [PATCH 15/40] Functions moved to another file to improve main workflow readability --- perf_metrics.bash | 503 ++------------------------------------------ perf_metrics.config | 4 +- src/functions.bash | 486 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 504 insertions(+), 489 deletions(-) create mode 100644 src/functions.bash diff --git a/perf_metrics.bash b/perf_metrics.bash index 4fb3326..6f28d98 100755 --- a/perf_metrics.bash +++ b/perf_metrics.bash @@ -1,492 +1,5 @@ #!/bin/bash -# Functions - -#Checks if the job submission ended correctly. -Job_completed() -{ - if [ "$Jobs_scheduler" == "slurm" ]; then - local id1 - id1=${1##* } - sleep 5 - if ! scontrol show job "$id1" | grep -q 'JobState=COMPLETED'; then - Completed=false - else - Completed=true - fi - - elif [ "$Jobs_scheduler" == "lsf" ]; then - local id2 - id2=$(head -n1 "$1" | cut -d'<' -f2 | cut -d'>' -f1) - sleep 5 - if ! bjobs -l "$id2" | grep -q 'Status '; then - Completed=false - else - Completed=true - fi - elif [ "$Jobs_scheduler" == "torque" ]; then - local id3 - id3=$(head -n1 "$1" | awk '{ print $3 }') - sleep 5 - if ! qstat f "$id3" | grep -q 'exit_status = 0'; then - Completed=false - else - Completed=true - fi - fi - - -} - -# Check if nemo is compiled for using extrae - -Compile_extrae() -{ - #Get flag lines - - line=$(sed -n '/^%FCFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) - line2=$(sed -n '/^%FPPFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) - line3=$(sed -n '/^%LDFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) - - - # If -rdynamic is not there recompilation is requiered and -rdynamic added - - if ! echo "${line}"|grep -q "\-rdynamic\b"; then - echo "-rdynamic flag not found in FCFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%FCFLAGS/ s/$/ -rdynamic/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile_ext=true - fi - - if ! echo "${line3}"|grep -q "\-export-dynamic\b"; then - echo "-export-dynamic flag not found in LDFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%LDFLAGS/ s/$/ -export-dynamic/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile_ext=true - fi - - # If finstrument-functions is not there recompilation is requiered and -finstrument-functions added - if ! echo "${line}"|grep -q "\-finstrument-functions\b"; then - echo "-finstrument-functions flag not found in arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%FCFLAGS/ s/$/ -finstrument-functions/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile_ext=true - fi - - # If -g is not there, recompilation is requiered and -g added - if ! echo "${line}"|grep -q "\-g\b"; then - echo "-g flag not found in arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%FCFLAGS/ s/$/ -g /' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile_ext=true - fi - - - if [ "$compile" == true ]; then - echo 'compile parameter is inicialized true' - fi - - # If -pg is there recompilation is requiered and -pg removed - - sed -i 's/-pg//g' "${Nemo_path}"/arch/arch-"${arch}".fcm - - if echo "${line}"|grep -q "\-pg\b"; then - echo "-pg flag found in FCFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " - compile_ext=true - fi - - if echo "${line2}"|grep -q "\-pg\b"; then - echo "-pg flag found in FPPFLAGS arch-${arch}.fcm : editing arch-${arch}.fcm " - compile_ext=true - fi - - if echo "${line3}"|grep -q "\-pg\b"; then - echo "-pg flag found in LDFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " - compile_ext=true - fi - - # If nemo executable is not on the run file compile - - if ! test -f "${Nemo_path}/cfgs/${name_cfg}/EXP00/nemo"; then - echo "nemo executable not found in ${name_cfg}" - compile_ext=true - fi - - - if [ "$compile" == true ] || [ "$compile_ext" == true ]; then - - echo "Compiling Nemo for EXTRAE" - echo "Output of the compilation in compile.err and compile.out" - - printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" - python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" - - state1=$("$job" --wait compile."$Jobs_scheduler") - echo - Job_completed "$state1" - if [ $Completed == false ]; then - echo "Nemo compilation failed, remember to load all the needed modules. Check the details in compile.err" - echo - exit 1 - else - echo "Nemo compilation successful" - echo - fi - - else - echo "Compilation not needed" - echo - fi - - #Copy all the EXP00 data but don't overwrite namelist just the executable - cd "$dir" || echo "Error original dir doesn't exist" exit - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo . - -} - -#Check if Nemo is compiled for using Gprof - -Compile_gprof() -{ - - if [ "$compile" == true ]; then - echo 'compile parameter is inicialized true' - fi - - # Checking if Gprof_arch file is present - if ! test -f "${Nemo_path}/arch/arch-${arch}_GPROF.fcm"; then - cp "${Nemo_path}"/arch/arch-"${arch}".fcm "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm - fi - - # Get text lines corresponding to the flags - line=$(sed -n '/^%FCFLAGS /p' "$Nemo_path"/arch/arch-"${arch}"_GPROF.fcm) - line2=$(sed -n '/^%FPPFLAGS /p' "$Nemo_path"/arch/arch-"${arch}"_GPROF.fcm) - line3=$(sed -n '/^%LDFLAGS /p' "$Nemo_path"/arch/arch-"${arch}"_GPROF.fcm) - - # If -g is not there, recompilation is requiered and -g added - if ! echo "${line}"|grep -q "\-g\b"; then - echo "-g flag not found in arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " - sed -i '/^%FCFLAGS/ s/$/ -g /' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm - compile_gprof=true - fi - - # If -pg is not there recompilation is requiered and -pg added - - if ! echo "${line}"|grep -q "\-pg\b"; then - echo "-pg flag not found in FCFLAGS arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " - sed -i '/^%FCFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm - compile_gprof=true - fi - - if ! echo "${line2}"|grep -q "\-pg\b"; then - echo "-pg flag not found in FPPFLAGS arch-${arch}_GPROF.fcm : editing arch-${arch}_GPROF.fcm " - sed -i '/^%FPPFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm - compile_gprof=true - fi - - if ! echo "${line3}"|grep -q "\-pg\b"; then - echo "-pg flag not found in LDFLAGS arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " - sed -i '/^%LDFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm - compile_gprof=true - fi - - # If nemo executable is not on the run file compile - - if ! test -f "${Nemo_path}/cfgs/${name_cfg}_GPROF/EXP00/nemo"; then - echo "nemo executable not found in ${name_cfg}_GPROF" - compile_gprof=true - fi - - if [ "$compile" == true ] || [ "$compile_gprof" == true ]; then - echo "Compiling Nemo for GPROF" - echo "Output of the compilation in compile.err and compile.out" - - printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg" - python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" - - state1=$("$job" --wait compile."$Jobs_scheduler") - echo - Job_completed "$state1" - if [ "$Completed" == false ]; then - echo "Nemo compilation failed, remember to load all the needed modules. Check the details in compile.err" - echo - exit 1 - else - echo "Nemo compilation successful" - echo - fi - - else - echo "Compilation not needed" - echo - fi - - #Copy all the EXP00 data but don't overwrite namelist just the executable - cd "$dir" || echo "Error original dir doesn't exist" exit - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/* . - cp "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/nemo . - - - if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then - sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) - fi - - #Solving NEMO input file common errors - - if test -f "weights_core_orca2_bicubic_noc.nc"; then - mv weights_core_orca2_bicubic_noc.nc weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES - fi - - if test -f "weights_core_orca2_bilinear_noc.nc"; then - mv weights_core_orca2_bilinear_noc.nc weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES - fi - -} - - - - -#Init variables with default values in case of missing - -Init() -{ - Nemo_path="${Nemo_path:-"."}" - Nemo_cores="${Nemo_cores:-( 48 )}" - Jobs_n_cores="${Jobs_n_cores:-48}" - Jobs_scheduler="${Jobs_scheduler:-"slurm"}" - time="${Jobs_time:-0}" - queue="${Jobs_queue:-""}" - compile="${Compilation_compile:-"false"}" - cfg="${Compilation_ref:-"ORCA2_ICE_PISCES"}" - arch="${Compilation_arch:-"X64_MN4"}" - name_cfg="${Compilation_name:-"ORCA2_EXTRAE"}" - comp_cfg="${Compilation_sub:-""}" - Modules="${Modules:-""}" - Nemo_iterations=12 - compile_ext=false - compile_gprof=false -} - - -# Checks if the paths given are correct. - -Test_arguments() -{ - # Nemo path correct? - if ! test -d "${Nemo_path}"; then - echo "Nemo relative path: ${Nemo_path} is not found" - echo - exit 1 - fi - #Nemo_cores is array? - if [[ ! "$(declare -p Nemo_cores)" =~ "declare -a" ]]; then - echo "Error Nemo_cores has to be a bash array like ( 4 24 48 )" - echo - exit 1 - fi - - - #Nemo_cores contains ints? - - re='^[0-9]+$' - for core in "${Nemo_cores[@]}" - do - - if ! [[ $core =~ $re ]] ; then - echo "Error Nemo_cores has to contain integer values" - echo - exit 1 - fi - done - - # cfg exists? - if ! test -d "${Nemo_path}/cfgs/${cfg}"; then - echo "configuration: ${cfg} doesn't exists in ${Nemo_path}/cfgs dir" - echo - exit 1 - fi - # arch exists? - if ! test -f "${Nemo_path}/arch/arch-${arch}.fcm"; then - echo "architecture: arch-${arch}.fcm doesn't exists in ${Nemo_path}/arch dir" - echo - exit 1 - fi - # scheduler correct? - if [ "$Jobs_scheduler" != "slurm" ] && [ "$Jobs_scheduler" != "lsf" ] && [ "$Jobs_scheduler" != "torque" ]; then - echo "$Jobs_scheduler is not a valid scheduler" - echo - exit 1 - fi - #Nemo_cores is array? - if [[ ! "$(declare -p Nemo_cores)" =~ "declare -a" ]]; then - echo "Error, variable Nemo_cores has to be an array" - echo - exit 1 - fi - #Modules available - if ! module load $Modules;then - echo "Error loading modules aborting" - echo - exit 1 - fi - echo - #$EXTRAE_HOME loaded ? - if ! test -d "${EXTRAE_HOME}"; then - echo "Extrae relative path: ${EXTRAE_HOME} is not found" - echo - exit 1 - else - sed -i 's|home=.*|home="'"$EXTRAE_HOME"'"|g' src/extrae.xml - fi - - # Adding -d to variable if not empty - if [ -n "$comp_cfg" ]; then - comp_cfg="-d $comp_cfg" - fi - - # Creating auxiliar vars for submiting jobs - if [ "$Jobs_scheduler" == "slurm" ]; then - job="sbatch" - elif [ "$Jobs_scheduler" == "lsf" ]; then - job="bsub" - elif [ "$Jobs_scheduler" == "torque" ]; then - job="qsub" - fi - - -} - - -# Generates a list of Nemo functions - -Gprof_functions() -{ - - - #Generating function list in case of missing - if ! test -f "extrae_functions.txt"; then - - Compile_gprof - - - #Changing iterations, big traces generate problems. - sed -i "s|nn_itend * =.*|nn_itend = $Nemo_iterations ! last time step (std 5475)|g" namelist_cfg - - rm gmon* 2> /dev/null - echo "Runing Nemo with 2 cores to obtain function data..." - echo - python3 ./src/Job_Creator.py -f "run" -j "run" --set-core 4 -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np 2 ./nemo" - - state2=$("$job" --wait run."$Jobs_scheduler") - Job_completed "$state2" - if [ $Completed == false ]; then - echo "Nemo execution failed look at run.err and ocean.output for more info" - echo "Remember that the namelist files copied are the default ones, change theme in order to fit with the input files in the dir " - echo - exit 1 - else - echo "Gprof files generated " - echo - fi - echo "Gthrottling functions ..." - echo - python3 ./src/Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./src/gthrottling.sh nemo" - state3=$("$job" --wait gthrottling."$Jobs_scheduler") - Job_completed "$state3" - if [ $Completed == false ]; then - echo "Error listing functions, look at gthrottling.err for more info" - echo - exit 1 - else - echo "Functions listed correctly" - echo - fi - - - - - else - echo "Functions already listed, file extrae_functions.txt does exist" - echo - - fi - -} - - -#Gets a trace from Nemo and cuts it to obtain a single timestep - -Get_trace() -{ - -Compile_extrae - -# Run nemo with extrae - ./src/extraf.sh nemo extrae_functions.txt - sed -i "s|list=.*|list=\"${dir}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" src/extrae.xml - for core in "${Nemo_cores[@]}" - - do - - echo "Creating trace with $core cores..." - echo - python3 ./src/Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core ./src/trace.sh ./nemo" - - state4=$("$job" --wait run_extrae."$Jobs_scheduler") - Job_completed "$state4" - if [ $Completed == false ]; then - echo "Nemo execution failed, no traces files generated more info inside run_extrae.err" - echo - exit 1 - fi - mv nemo.prv nemo_"$core".prv - mv nemo.pcf nemo_"$core".pcf - mv nemo.row nemo_"$core".row - echo "Cutting best iteration" - echo - ./src/magiccut/magicCut nemo_"${core}".prv "$Nemo_iterations" > cut_"$core".out 2>&1 - if ! ls nemo_"$core".best_cut.prv; then - echo "Cut failed, look at cut_$core.out for more info." - echo - exit 1 - fi - echo - # Creating folder - if ! test -d "Metrics"; then - mkdir Metrics - fi - - cp nemo_"$core".best_cut.* Metrics - - done - -} - -Create_metrics() -{ - - - - # Create performance metrics - echo "Creating metrics and storing theme in Metrics folder" - echo - python3 ./src/Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./../src/modelfactors.py -ms 100000 *" - mv analysis."$Jobs_scheduler" Metrics - cd Metrics||(echo "Error Metrics folder doesn't exists"; exit 1) - state5=$("$job" --wait analysis."$Jobs_scheduler") - Job_completed "$state5" - if [ $Completed == false ]; then - echo "Error, metrics have not generated check Metrics/analysis.err to get more details" - echo - exit 1 - - fi - - echo "------------------------------------------------------------------------------" - echo "------------------------- Script Completed -----------------------------------" - echo "----------------------- Data in Metrics folder -------------------------------" - echo "------------------------------------------------------------------------------" - echo "------------------------------------------------------------------------------" - echo -} - main() { @@ -503,14 +16,30 @@ dir=$(pwd) echo echo "Using the following configuration:" echo + +#Load functions from file +source "$dir"/src/functions.bash + +#Load parameters from file source "$dir"/perf_metrics.config + +# print parameters grep -o '^[^#]*' perf_metrics.config echo +#Init variables Init + +#Test if parameters are valid Test_arguments + +#Create the list of important functions from NEMO Gprof_functions + +#Get the traces of the executions and cut 1 timestep Get_trace + +#Generate the performance metrics Create_metrics } diff --git a/perf_metrics.config b/perf_metrics.config index ebe66fd..96a65f7 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -29,8 +29,8 @@ Jobs_queue=debug Compilation_compile="false" Compilation_ref="ORCA2_ICE_PISCES" -Compilation_arch="X64_MN4_UNCOUPLED" -Compilation_name="ORCA2_UNCOUPLED" +Compilation_arch="Your-arch-file" +Compilation_name="ORCA2_EXTRAE" Compilation_sub="OCE del_key 'key_si3 key_top'" # List of modules loaded. diff --git a/src/functions.bash b/src/functions.bash new file mode 100644 index 0000000..144c860 --- /dev/null +++ b/src/functions.bash @@ -0,0 +1,486 @@ +# Functions + +#Checks if the job submission ended correctly. +Job_completed() +{ + if [ "$Jobs_scheduler" == "slurm" ]; then + local id1 + id1=${1##* } + sleep 5 + if ! scontrol show job "$id1" | grep -q 'JobState=COMPLETED'; then + Completed=false + else + Completed=true + fi + + elif [ "$Jobs_scheduler" == "lsf" ]; then + local id2 + id2=$(head -n1 "$1" | cut -d'<' -f2 | cut -d'>' -f1) + sleep 5 + if ! bjobs -l "$id2" | grep -q 'Status '; then + Completed=false + else + Completed=true + fi + elif [ "$Jobs_scheduler" == "torque" ]; then + local id3 + id3=$(head -n1 "$1" | awk '{ print $3 }') + sleep 5 + if ! qstat f "$id3" | grep -q 'exit_status = 0'; then + Completed=false + else + Completed=true + fi + fi + + +} + +# Check if nemo is compiled for using extrae + +Compile_extrae() +{ + #Get flag lines + + line=$(sed -n '/^%FCFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) + line2=$(sed -n '/^%FPPFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) + line3=$(sed -n '/^%LDFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) + + + # If -rdynamic is not there recompilation is requiered and -rdynamic added + + if ! echo "${line}"|grep -q "\-rdynamic\b"; then + echo "-rdynamic flag not found in FCFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " + sed -i '/^%FCFLAGS/ s/$/ -rdynamic/' "${Nemo_path}"/arch/arch-"${arch}".fcm + compile_ext=true + fi + + if ! echo "${line3}"|grep -q "\-export-dynamic\b"; then + echo "-export-dynamic flag not found in LDFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " + sed -i '/^%LDFLAGS/ s/$/ -export-dynamic/' "${Nemo_path}"/arch/arch-"${arch}".fcm + compile_ext=true + fi + + # If finstrument-functions is not there recompilation is requiered and -finstrument-functions added + if ! echo "${line}"|grep -q "\-finstrument-functions\b"; then + echo "-finstrument-functions flag not found in arch-${arch}.fcm: editing arch-${arch}.fcm " + sed -i '/^%FCFLAGS/ s/$/ -finstrument-functions/' "${Nemo_path}"/arch/arch-"${arch}".fcm + compile_ext=true + fi + + # If -g is not there, recompilation is requiered and -g added + if ! echo "${line}"|grep -q "\-g\b"; then + echo "-g flag not found in arch-${arch}.fcm: editing arch-${arch}.fcm " + sed -i '/^%FCFLAGS/ s/$/ -g /' "${Nemo_path}"/arch/arch-"${arch}".fcm + compile_ext=true + fi + + + if [ "$compile" == true ]; then + echo 'compile parameter is inicialized true' + fi + + # If -pg is there recompilation is requiered and -pg removed + + sed -i 's/-pg//g' "${Nemo_path}"/arch/arch-"${arch}".fcm + + if echo "${line}"|grep -q "\-pg\b"; then + echo "-pg flag found in FCFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " + compile_ext=true + fi + + if echo "${line2}"|grep -q "\-pg\b"; then + echo "-pg flag found in FPPFLAGS arch-${arch}.fcm : editing arch-${arch}.fcm " + compile_ext=true + fi + + if echo "${line3}"|grep -q "\-pg\b"; then + echo "-pg flag found in LDFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " + compile_ext=true + fi + + # If nemo executable is not on the run file compile + + if ! test -f "${Nemo_path}/cfgs/${name_cfg}/EXP00/nemo"; then + echo "nemo executable not found in ${name_cfg}" + compile_ext=true + fi + + + if [ "$compile" == true ] || [ "$compile_ext" == true ]; then + + echo "Compiling Nemo for EXTRAE" + echo "Output of the compilation in compile.err and compile.out" + + printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" + python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + + state1=$("$job" --wait compile."$Jobs_scheduler") + echo + Job_completed "$state1" + if [ $Completed == false ]; then + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in compile.err" + echo + exit 1 + else + echo "Nemo compilation successful" + echo + fi + + else + echo "Compilation not needed" + echo + fi + + #Copy all the EXP00 data but don't overwrite namelist just the executable + cd "$dir" || echo "Error original dir doesn't exist" exit + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . + cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo . + +} + +#Check if Nemo is compiled for using Gprof + +Compile_gprof() +{ + + if [ "$compile" == true ]; then + echo 'compile parameter is inicialized true' + fi + + # Checking if Gprof_arch file is present + if ! test -f "${Nemo_path}/arch/arch-${arch}_GPROF.fcm"; then + cp "${Nemo_path}"/arch/arch-"${arch}".fcm "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm + fi + + # Get text lines corresponding to the flags + line=$(sed -n '/^%FCFLAGS /p' "$Nemo_path"/arch/arch-"${arch}"_GPROF.fcm) + line2=$(sed -n '/^%FPPFLAGS /p' "$Nemo_path"/arch/arch-"${arch}"_GPROF.fcm) + line3=$(sed -n '/^%LDFLAGS /p' "$Nemo_path"/arch/arch-"${arch}"_GPROF.fcm) + + # If -g is not there, recompilation is requiered and -g added + if ! echo "${line}"|grep -q "\-g\b"; then + echo "-g flag not found in arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " + sed -i '/^%FCFLAGS/ s/$/ -g /' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm + compile_gprof=true + fi + + # If -pg is not there recompilation is requiered and -pg added + + if ! echo "${line}"|grep -q "\-pg\b"; then + echo "-pg flag not found in FCFLAGS arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " + sed -i '/^%FCFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm + compile_gprof=true + fi + + if ! echo "${line2}"|grep -q "\-pg\b"; then + echo "-pg flag not found in FPPFLAGS arch-${arch}_GPROF.fcm : editing arch-${arch}_GPROF.fcm " + sed -i '/^%FPPFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm + compile_gprof=true + fi + + if ! echo "${line3}"|grep -q "\-pg\b"; then + echo "-pg flag not found in LDFLAGS arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " + sed -i '/^%LDFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm + compile_gprof=true + fi + + # If nemo executable is not on the run file compile + + if ! test -f "${Nemo_path}/cfgs/${name_cfg}_GPROF/EXP00/nemo"; then + echo "nemo executable not found in ${name_cfg}_GPROF" + compile_gprof=true + fi + + if [ "$compile" == true ] || [ "$compile_gprof" == true ]; then + echo "Compiling Nemo for GPROF" + echo "Output of the compilation in compile.err and compile.out" + + printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg" + python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + + state1=$("$job" --wait compile."$Jobs_scheduler") + echo + Job_completed "$state1" + if [ "$Completed" == false ]; then + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in compile.err" + echo + exit 1 + else + echo "Nemo compilation successful" + echo + fi + + else + echo "Compilation not needed" + echo + fi + + #Copy all the EXP00 data but don't overwrite namelist just the executable + cd "$dir" || echo "Error original dir doesn't exist" exit + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/* . + cp "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/nemo . + + + if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then + sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) + fi + + #Solving NEMO input file common errors + + if test -f "weights_core_orca2_bicubic_noc.nc"; then + mv weights_core_orca2_bicubic_noc.nc weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES + fi + + if test -f "weights_core_orca2_bilinear_noc.nc"; then + mv weights_core_orca2_bilinear_noc.nc weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES + fi + +} + + + + +#Init variables with default values in case of missing + +Init() +{ + Nemo_path="${Nemo_path:-"."}" + Nemo_cores="${Nemo_cores:-( 48 )}" + Jobs_n_cores="${Jobs_n_cores:-48}" + Jobs_scheduler="${Jobs_scheduler:-"slurm"}" + time="${Jobs_time:-0}" + queue="${Jobs_queue:-""}" + compile="${Compilation_compile:-"false"}" + cfg="${Compilation_ref:-"ORCA2_ICE_PISCES"}" + arch="${Compilation_arch:-"X64_MN4"}" + name_cfg="${Compilation_name:-"ORCA2_EXTRAE"}" + comp_cfg="${Compilation_sub:-""}" + Modules="${Modules:-""}" + Nemo_iterations=12 + compile_ext=false + compile_gprof=false +} + + +# Checks if the paths given are correct. + +Test_arguments() +{ + # Nemo path correct? + if ! test -d "${Nemo_path}"; then + echo "Nemo relative path: ${Nemo_path} is not found" + echo + exit 1 + fi + #Nemo_cores is array? + if [[ ! "$(declare -p Nemo_cores)" =~ "declare -a" ]]; then + echo "Error Nemo_cores has to be a bash array like ( 4 24 48 )" + echo + exit 1 + fi + + + #Nemo_cores contains ints? + + re='^[0-9]+$' + for core in "${Nemo_cores[@]}" + do + + if ! [[ $core =~ $re ]] ; then + echo "Error Nemo_cores has to contain integer values" + echo + exit 1 + fi + done + + # cfg exists? + if ! test -d "${Nemo_path}/cfgs/${cfg}"; then + echo "configuration: ${cfg} doesn't exists in ${Nemo_path}/cfgs dir" + echo + exit 1 + fi + # arch exists? + if ! test -f "${Nemo_path}/arch/arch-${arch}.fcm"; then + echo "architecture: arch-${arch}.fcm doesn't exists in ${Nemo_path}/arch dir" + echo + exit 1 + fi + # scheduler correct? + if [ "$Jobs_scheduler" != "slurm" ] && [ "$Jobs_scheduler" != "lsf" ] && [ "$Jobs_scheduler" != "torque" ]; then + echo "$Jobs_scheduler is not a valid scheduler" + echo + exit 1 + fi + #Nemo_cores is array? + if [[ ! "$(declare -p Nemo_cores)" =~ "declare -a" ]]; then + echo "Error, variable Nemo_cores has to be an array" + echo + exit 1 + fi + #Modules available + if ! module load $Modules;then + echo "Error loading modules aborting" + echo + exit 1 + fi + echo + #$EXTRAE_HOME loaded ? + if ! test -d "${EXTRAE_HOME}"; then + echo "Extrae relative path: ${EXTRAE_HOME} is not found" + echo + exit 1 + else + sed -i 's|home=.*|home="'"$EXTRAE_HOME"'"|g' src/extrae.xml + fi + + # Adding -d to variable if not empty + if [ -n "$comp_cfg" ]; then + comp_cfg="-d $comp_cfg" + fi + + # Creating auxiliar vars for submiting jobs + if [ "$Jobs_scheduler" == "slurm" ]; then + job="sbatch" + elif [ "$Jobs_scheduler" == "lsf" ]; then + job="bsub" + elif [ "$Jobs_scheduler" == "torque" ]; then + job="qsub" + fi + + +} + + +# Generates a list of Nemo functions + +Gprof_functions() +{ + + + #Generating function list in case of missing + if ! test -f "extrae_functions.txt"; then + + Compile_gprof + + + #Changing iterations, big traces generate problems. + sed -i "s|nn_itend * =.*|nn_itend = $Nemo_iterations ! last time step (std 5475)|g" namelist_cfg + + rm gmon* 2> /dev/null + echo "Runing Nemo with 2 cores to obtain function data..." + echo + python3 ./src/Job_Creator.py -f "run" -j "run" --set-core 4 -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np 2 ./nemo" + + state2=$("$job" --wait run."$Jobs_scheduler") + Job_completed "$state2" + if [ $Completed == false ]; then + echo "Nemo execution failed look at run.err and ocean.output for more info" + echo "Remember that the namelist files copied are the default ones, change theme in order to fit with the input files in the dir " + echo + exit 1 + else + echo "Gprof files generated " + echo + fi + echo "Gthrottling functions ..." + echo + python3 ./src/Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./src/gthrottling.sh nemo" + state3=$("$job" --wait gthrottling."$Jobs_scheduler") + Job_completed "$state3" + if [ $Completed == false ]; then + echo "Error listing functions, look at gthrottling.err for more info" + echo + exit 1 + else + echo "Functions listed correctly" + echo + fi + + + + + else + echo "Functions already listed, file extrae_functions.txt does exist" + echo + + fi + +} + + +#Gets a trace from Nemo and cuts it to obtain a single timestep + +Get_trace() +{ + +Compile_extrae + +# Run nemo with extrae + ./src/extraf.sh nemo extrae_functions.txt + sed -i "s|list=.*|list=\"${dir}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" src/extrae.xml + for core in "${Nemo_cores[@]}" + + do + + echo "Creating trace with $core cores..." + echo + python3 ./src/Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core ./src/trace.sh ./nemo" + + state4=$("$job" --wait run_extrae."$Jobs_scheduler") + Job_completed "$state4" + if [ $Completed == false ]; then + echo "Nemo execution failed, no traces files generated more info inside run_extrae.err" + echo + exit 1 + fi + mv nemo.prv nemo_"$core".prv + mv nemo.pcf nemo_"$core".pcf + mv nemo.row nemo_"$core".row + echo "Cutting best iteration" + echo + ./src/magiccut/magicCut nemo_"${core}".prv "$Nemo_iterations" > cut_"$core".out 2>&1 + if ! ls nemo_"$core".best_cut.prv; then + echo "Cut failed, look at cut_$core.out for more info." + echo + exit 1 + fi + echo + # Creating folder + if ! test -d "Metrics"; then + mkdir Metrics + fi + + cp nemo_"$core".best_cut.* Metrics + + done + +} + +Create_metrics() +{ + + + + # Create performance metrics + echo "Creating metrics and storing theme in Metrics folder" + echo + python3 ./src/Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./../src/modelfactors.py -ms 100000 *" + mv analysis."$Jobs_scheduler" Metrics + cd Metrics||(echo "Error Metrics folder doesn't exists"; exit 1) + state5=$("$job" --wait analysis."$Jobs_scheduler") + Job_completed "$state5" + if [ $Completed == false ]; then + echo "Error, metrics have not generated check Metrics/analysis.err to get more details" + echo + exit 1 + + fi + + echo "------------------------------------------------------------------------------" + echo "------------------------- Script Completed -----------------------------------" + echo "----------------------- Data in Metrics folder -------------------------------" + echo "------------------------------------------------------------------------------" + echo "------------------------------------------------------------------------------" + echo +} \ No newline at end of file -- GitLab From 3e4104b7ccdcf16f6f8fd8148814ffb56a35358d Mon Sep 17 00:00:00 2001 From: cpenadep Date: Mon, 12 Sep 2022 10:26:07 +0200 Subject: [PATCH 16/40] Highligthed the changes needed in cfg file --- README.md | 13 +++++++++---- perf_metrics.config | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cd009b6..4f45dcf 100644 --- a/README.md +++ b/README.md @@ -17,16 +17,21 @@ This script relies on the following tools: Here the list of the modules that need to be loaded before the script execution. -* Perl interpreter +* Perl interpreter. * Fortran compiler (ifort, gfortran, pgfortran, ftn, …), * Message Passing Interface (MPI) implementation (e.g. OpenMPI or MPICH). -* Network Common Data Form (NetCDF) library with its underlying Hierarchical Data Form (HDF) +* Network Common Data Form (NetCDF) library with its underlying Hierarchical Data Form (HDF). # Usage -* Copy all the content of this folder into the folder with the input data for NEMO -* Edit the file perf_metrics.config and replace the parameters values with the suited information. +* Copy all the content of this folder into the folder with the input data for NEMO. +* Edit the file perf_metrics.config and replace the parameters values with the suited information. +* MINIMUM CHANGES: + * Change the Nemo_path value to the path were NEMO is installed in your machine. + * Change the Compilation_arch value to the name of the arch file that you use to compile NEMO. + * Delete or modify the Modules value to suit the name of the modules you need to load. + * Change the Jobs_scheduler value with the name of the scheduler installed in your machine (currently supports slurm,lsf and torque) * Execute perf_metrics.bash ``` ./perf_metrics.bash diff --git a/perf_metrics.config b/perf_metrics.config index 96a65f7..37374c5 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -11,7 +11,7 @@ Nemo_path="../NEMO" Nemo_cores=( 4 24 48 96 192) # Jobs_n_cores: nºcores used for other jobs like compiling nemo. -# Jobs_scheduler: Available (slurm/lsf). +# Jobs_scheduler: Available (slurm/lsf/torque). # Jobs_time: Max duration of the job in min. # Jobs_queue: Queue used. -- GitLab From cb7e1c13605ead4ea42cbd93b26a66aa9b8097c9 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Mon, 12 Sep 2022 10:31:21 +0200 Subject: [PATCH 17/40] Fixed typos --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4f45dcf..1438ef3 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,11 @@ Here the list of the modules that need to be loaded before the script execution. * Copy all the content of this folder into the folder with the input data for NEMO. * Edit the file perf_metrics.config and replace the parameters values with the suited information. -* MINIMUM CHANGES: - * Change the Nemo_path value to the path were NEMO is installed in your machine. - * Change the Compilation_arch value to the name of the arch file that you use to compile NEMO. - * Delete or modify the Modules value to suit the name of the modules you need to load. - * Change the Jobs_scheduler value with the name of the scheduler installed in your machine (currently supports slurm,lsf and torque) +* MINIMUM CHANGES perf_metrics.config : + * Nemo_path, change the value to the path were NEMO is installed in your machine. + * Compilation_arch, replace the value with the name of the arch file that you use to compile NEMO. + * Modules, change the value to suit the name of the modules you need to load. + * Jobs_scheduler, replace the value with the name of the scheduler installed in your machine (currently supports slurm, lsf and torque) * Execute perf_metrics.bash ``` ./perf_metrics.bash -- GitLab From 70c25a37a7a02813c2db59db45d913f1b9df715b Mon Sep 17 00:00:00 2001 From: cpenadep Date: Mon, 12 Sep 2022 15:03:26 +0200 Subject: [PATCH 18/40] Now magiccut is added as a submodule --- .gitmodules | 4 + src/magiccut | 1 + src/magiccut/README.md | 4 - src/magiccut/bin/TraceCutter.py | 266 --------------------- src/magiccut/magicCut | 64 ----- src/magiccut/templates/cutter_template.xml | 23 -- 6 files changed, 5 insertions(+), 357 deletions(-) create mode 100644 .gitmodules create mode 160000 src/magiccut delete mode 100644 src/magiccut/README.md delete mode 100755 src/magiccut/bin/TraceCutter.py delete mode 100755 src/magiccut/magicCut delete mode 100755 src/magiccut/templates/cutter_template.xml diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..ff1e283 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "src/magiccut"] + path = src/magiccut + url = https://earth.bsc.es/gitlab/ces/magiccut.git + branch = numpy_version diff --git a/src/magiccut b/src/magiccut new file mode 160000 index 0000000..13b714f --- /dev/null +++ b/src/magiccut @@ -0,0 +1 @@ +Subproject commit 13b714f08a265150dc3b5cc05c5303fa0a650448 diff --git a/src/magiccut/README.md b/src/magiccut/README.md deleted file mode 100644 index 3015d29..0000000 --- a/src/magiccut/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Magic Cut - -A tool created to automatically cut a single iteration from a **extrae trace** using function events (60000019) as a reference. -Created using NEMO traces, not tested with other codes. \ No newline at end of file diff --git a/src/magiccut/bin/TraceCutter.py b/src/magiccut/bin/TraceCutter.py deleted file mode 100755 index 4b26cd1..0000000 --- a/src/magiccut/bin/TraceCutter.py +++ /dev/null @@ -1,266 +0,0 @@ -import numpy as np -import xml.etree.ElementTree as ET -import time as clock_time -from os.path import exists, realpath - - -def get_command_line_arguments(): - """ - Returns a list of files that have been provided as a command line argument - :return: list of files - """ - - # Parse and assert command line options - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("-i", "--input-data", required=True, type=PathType(exists=True, type='file'), - help="File with filtered info on cores ID, function ID, and timings") - parser.add_argument("--ts-number", default=None, type=int, - help="Number of time-steps present in the trace") - parser.add_argument("-t", "--tool-path", default=None, type=PathType(exists=True, type='dir'), - help="Path to Magicut templates") - parser.add_argument("--function-file", required=True, type=PathType(exists=True, type='file'), - help="File with names corresponding to function IDs") - - args = parser.parse_args() - - if args.input_data is None: - parser.error("Input data are needed") - else: - if not exists(args.input_data): - parser.error("The specified input does not exists") - if args.function_file is not None and not exists(args.function_file): - parser.error("The specified file does not exists") - if args.ts_number is None: - print("Warning: magicCut will try to guess the number of time-steps") - if args.tool_path is None: - args.tool_path = "/".join(realpath(__file__).split("/")[:-2]) + "/templates/" - if not exists(args.tool_path): - parser.error("The specified path to tools does not exist: " + args.tool_path) - - return args - - -class PathType(object): - def __init__(self, exists=True, type='file', dash_ok=True): - '''exists: - True: a path that does exist - False: a path that does not exist, in a valid parent directory - None: don't care - type: file, dir, symlink, None, or a function returning True for valid paths - None: don't care - dash_ok: whether to allow "-" as stdin/stdout''' - - assert exists in (True, False, None) - assert type in ('file', 'dir', 'symlink', None) or hasattr(type, '__call__') - - self._exists = exists - self._type = type - self._dash_ok = dash_ok - - def __call__(self, string): - from argparse import ArgumentTypeError as err - import os - if string == '-': - # the special argument "-" means sys.std{in,out} - if self._type == 'dir': - raise err('standard input/output (-) not allowed as directory path') - elif self._type == 'symlink': - raise err('standard input/output (-) not allowed as symlink path') - elif not self._dash_ok: - raise err('standard input/output (-) not allowed') - else: - e = os.path.exists(string) - if self._exists == True: - if not e: - raise err("path does not exist: '%s'" % string) - - if self._type is None: - pass - elif self._type == 'file': - if not os.path.isfile(string): - raise err("path is not a file: '%s'" % string) - elif self._type == 'symlink': - if not os.path.symlink(string): - raise err("path is not a symlink: '%s'" % string) - elif self._type == 'dir': - if not os.path.isdir(string): - raise err("path is not a directory: '%s'" % string) - elif not self._type(string): - raise err("path not valid: '%s'" % string) - else: - if self._exists == False and e: - raise err("path exists: '%s'" % string) - - p = os.path.dirname(os.path.normpath(string)) or '.' - if not os.path.isdir(p): - raise err("parent path is not a directory: '%s'" % p) - elif not os.path.exists(p): - raise err("parent directory does not exist: '%s'" % p) - - return string - - -def get_function_name(functions_name_file, function_ids): - # Opens file with format #ID FUNCTION_NAME - with open(functions_name_file) as f: - ids_name_list = f.read() - id = [l.split()[0] for l in ids_name_list.splitlines()] - names = [l.split()[1] for l in ids_name_list.splitlines()] - function_names = [] - for ID in function_ids: - index = id.index(str(ID)) - function_names.append(names[index]) - return function_names - - -def save_function_names(function_ids, functions_name_file): - # Opens file with format #ID FUNCTION_NAME - with open(functions_name_file) as f: - ids_name_list = f.read() - id = [l.split()[0] for l in ids_name_list.splitlines()] - names = [l.split()[1] for l in ids_name_list.splitlines()] - with open('functions.txt', 'w') as f: - for ID in function_ids: - index = id.index(str(ID)) - f.write(id[index] + " " + names[index] + "\n") - print("Functions that are called once per time-step saved to functions.txt") - - -def cut_time(tool_path, input_data, function_name_file, time_steps=None): - # Path to the cutter template - template_path = tool_path + "cutter_template.xml" - print("Starting the magic") - start = clock_time.time() - - # Read trace and load function calls - cpu_ids, time, function_ids = np.genfromtxt(input_data, dtype='int', unpack='True') - - print("for opening the input_data: ") - end = clock_time.time() - print(end - start) - - # Find set of different routines, and number of cores - unique_function_ids, call_counts = np.unique(function_ids, return_counts=True) - cpu_number = len(np.unique(cpu_ids)) - - function_names = get_function_name(function_name_file, unique_function_ids) - - if time_steps is None: - # Remember that everything is multiplied by N cores - # Creates an histogram with the number of how many functions are called n times - counts = np.bincount(call_counts) - # Suppose that the most common value is the number of ts - time_steps = np.argmax(counts) - # Find functions that are called once per time-step - functions_called_once_per_step = unique_function_ids[call_counts == time_steps] - else: - # We just search for that functions that appears once per core per time-step - functions_called_once_per_step = unique_function_ids[call_counts / cpu_number == time_steps] - - if not len(functions_called_once_per_step): - raise Exception("No function has been found which is called once per timestep") - - # Find which routine is called in first place - magic_sentinel = [n for n in function_names if n.count("magiccut")] - if magic_sentinel: - # An artificial first routine was introduces - index = function_names.index(magic_sentinel[0]) - first_routine = unique_function_ids[index] - else: - # see which functions is first - first_routine = functions_called_once_per_step[0] - - print("First called routine: ", function_names[unique_function_ids.tolist().index(first_routine)]) - # Find number of nemo_proc_number - nemo_proc = cpu_ids[function_ids == first_routine] - nemo_proc = np.unique(nemo_proc) - nemo_proc_number = nemo_proc.shape[0] - - nemo_proc_min = min(nemo_proc) - nemo_proc_max = max(nemo_proc) - - print("Actual n_proc", nemo_proc_number, "with ", time_steps, "time steps") - - # Find the index of the fastest step - time_step = np.ones([nemo_proc_number, time_steps]) - - ts_min_index = np.zeros(nemo_proc_number, dtype='int') - ts_max_index = np.zeros(nemo_proc_number, dtype='int') - - ts_time = np.zeros([nemo_proc_number, time_steps]) - start = clock_time.time() - # For each processor - for index, proc in enumerate(nemo_proc): - - # Get the starting time of all time-step - ts_time[index] = time[(cpu_ids == proc) & (function_ids == first_routine)] - - # Compute the duration of each ts - ts_duration = np.diff(ts_time[index]) - - # Index of the shortest ts - ts_min_index[index] = np.argmin(ts_duration) - - # Index of the largest ts - ts_max_index[index] = np.argmax(ts_duration[1:-1]) + 1 - - # Evaluate the most common index for best ts - counts = np.bincount(ts_min_index) - best_ts_index = np.argmax(counts) - - # Evaluate the most common index for worst ts - counts = np.bincount(ts_max_index) - worst_ts_index = np.argmax(counts) - - print("for finding the index of the slowest / fastest step: ", worst_ts_index, "/", best_ts_index) - end = clock_time.time() - print(end - start) - - # Find the start and the end of the best step - best_ts_start = min(ts_time[:, best_ts_index]) - best_ts_end = max(ts_time[:, best_ts_index + 1]) - - # Find the start and the end of the best step - worst_ts_start = min(ts_time[:, worst_ts_index]) - worst_ts_end = max(ts_time[:, worst_ts_index + 1]) - - print("Worst / best time step's duration: ", worst_ts_end - worst_ts_start, best_ts_end - best_ts_start) - - # Open the xml template - tree = ET.parse(template_path) - - cutter = tree.find('cutter') - for tasks in tree.iter('tasks'): - tasks.text = str(nemo_proc_min) + "-" + str(nemo_proc_max) - for minimum_time in tree.iter('minimum_time'): - minimum_time.text = str(best_ts_start - 1000) - for maximum_time in tree.iter('maximum_time'): - maximum_time.text = str(best_ts_end + 1000) - - # Create paramedir cutter file - tree.write('best_time_cutter.xml') - for minimum_time in tree.iter('minimum_time'): - minimum_time.text = str(worst_ts_start - 1000) - for maximum_time in tree.iter('maximum_time'): - maximum_time.text = str(worst_ts_end + 1000) - - # Create paramedir cutter file - tree.write('worst_time_cutter.xml') - - return unique_function_ids.tolist() - - -if __name__ == "__main__": - # Get files from command line - args = get_command_line_arguments() - - tool_path = args.tool_path - input_data = args.input_data - function_file = args.function_file - ts_number = args.ts_number - - unique_functions = cut_time(tool_path, input_data, function_file, ts_number) - # unique_functions = [6, 33] - save_function_names(unique_functions, function_file) diff --git a/src/magiccut/magicCut b/src/magiccut/magicCut deleted file mode 100755 index 9d0b6aa..0000000 --- a/src/magiccut/magicCut +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -# Get inputs arg -if [[ $# -lt 2 ]] -then - echo $0 "expects trace and number of timesteps as input" - echo "abort" - exit 1 -fi - -# trace file to cut -trace_file=$1 -# Number of timesteps -time_steps=$2 - -if [[ ! -e ${trace_file} ]] -then - echo "The specified trace does not exist: "${trace_file} - echo "Abort" - exit 1 -fi -trace_file=`readlink -f ${trace_file}` -pcf_trace_file=${trace_file/.prv/.pcf} -trace_folder=${trace_file%/*} - -tool_path=`readlink -f $0` -tool_path=${tool_path//'/magicCut'} - - -trace_base_name=${trace_file//.exe.prv.gz} -trace_base_name=${trace_base_name//.exe.prv} -trace_base_name=${trace_base_name//.prv.gz} -trace_base_name=${trace_base_name//.prv} - -# grep through prv file if functions are there -func_check=`grep -m 1 60000019 ${pcf_trace_file} | wc -l` -# Stores the list of function and extrae ids -begin_line_number=`awk '/60000019 User function/ {print FNR}' ${pcf_trace_file}` -begin_line_number=$((begin_line_number+2)) -end_line_number=`tail -n +${begin_line_number} ${pcf_trace_file} | grep -nm 1 EVENT_TYPE | awk -F: '{print $1}'` -end_line_number=$((begin_line_number+end_line_number-2)) - -# removes also the blank lines at the end -sed -n "${begin_line_number},${end_line_number}p" ${pcf_trace_file}| awk '{print $1, $2}' | awk NF > FUNCTION_ID_NAMES.txt - -if [[ ${func_check} -gt 0 ]] -then - CPU_T_ID=${trace_folder}/CPU_T_ID.txt - cat /dev/null > ${CPU_T_ID} - - # Retrieve function's ID - echo "Retrieve function's ID" - grep ':60000019:' ${trace_file} |\ - grep -v ':0:' |\ - awk -F : '{print $2, $6, $8}' > ${CPU_T_ID} - # Finds best time step - python ${tool_path}/bin/TraceCutter.py --input-data ${CPU_T_ID} --ts-number ${time_steps} --function-file FUNCTION_ID_NAMES.txt - #rm ${CPU_T_ID} - echo "start paramedir cutter" - time paramedir -c best_time_cutter.xml ${trace_file} -o ${trace_base_name}.best_cut.prv -else - echo -e "Functions must be present in the trace for the script to work.\nAborting ... " - exit 1 -fi diff --git a/src/magiccut/templates/cutter_template.xml b/src/magiccut/templates/cutter_template.xml deleted file mode 100755 index 3394112..0000000 --- a/src/magiccut/templates/cutter_template.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - 1-256 - 0 - 1 - %START% - %END% - 0 - 100 - 0 - 1 - 0 - 0 - - -- GitLab From 1236b30013b55a33cbb72ac9abbc493ec9cf272c Mon Sep 17 00:00:00 2001 From: cpenadep Date: Mon, 12 Sep 2022 16:08:46 +0200 Subject: [PATCH 19/40] Now the run of Nemo is done in Run_NEMO and input data path added to config file --- README.md | 4 ++- perf_metrics.config | 6 ++-- src/functions.bash | 88 ++++++++++++++++++++++++++++----------------- 3 files changed, 62 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 1438ef3..58941d3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # Nemo modelfactors +The project was developed tested using NEMO 4.2 version. This project is intended to compute important performance metrics for a NEMO run. @@ -25,10 +26,11 @@ Here the list of the modules that need to be loaded before the script execution. # Usage -* Copy all the content of this folder into the folder with the input data for NEMO. +* Clone this repository wherever you please. * Edit the file perf_metrics.config and replace the parameters values with the suited information. * MINIMUM CHANGES perf_metrics.config : * Nemo_path, change the value to the path were NEMO is installed in your machine. + * Nemo_input_data, change the value to the path were the input data for the configuration is downloaded. * Compilation_arch, replace the value with the name of the arch file that you use to compile NEMO. * Modules, change the value to suit the name of the modules you need to load. * Jobs_scheduler, replace the value with the name of the scheduler installed in your machine (currently supports slurm, lsf and torque) diff --git a/perf_metrics.config b/perf_metrics.config index 37374c5..575f178 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -3,11 +3,13 @@ ################################################################################# -# Nemo_path: Relative path to nemo installation folder containing the cfgs and arch dirs +# Nemo_path: Path to nemo installation folder containing the cfgs and arch dirs. +# Nemo_input_data: Path to the input data needed to run the nemo cfg. # Nemo_cores: List of nºcores used for executing Nemo, ( 4 48 ) makes the script execute and # get Nemo traces with 4 and 48 cores. 2 different nºcores are needed to obtain scalability data. -Nemo_path="../NEMO" +Nemo_path="../NEMO/" +Nemo_input_data="../DATA/ORCA2_ICE_v4.2.0/" Nemo_cores=( 4 24 48 96 192) # Jobs_n_cores: nºcores used for other jobs like compiling nemo. diff --git a/src/functions.bash b/src/functions.bash index 144c860..eb5c5bd 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -40,6 +40,8 @@ Job_completed() Compile_extrae() { + cd "$dir" || (echo "Error original dir doesn't exist"; exit 1) + #Get flag lines line=$(sed -n '/^%FCFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) @@ -133,10 +135,10 @@ Compile_extrae() fi #Copy all the EXP00 data but don't overwrite namelist just the executable - cd "$dir" || echo "Error original dir doesn't exist" exit - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo . - + cd "$dir" ||( echo "Error original dir doesn't exist"; exit 1 ) + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* Run_NEMO + cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo Run_NEMO + cd Run_NEMO||( echo "Error Run_NEMO folder doesn't exists"; exit 1 ) } #Check if Nemo is compiled for using Gprof @@ -144,6 +146,8 @@ Compile_extrae() Compile_gprof() { + cd "$dir" || (echo "Error original dir doesn't exist"; exit 1) + if [ "$compile" == true ]; then echo 'compile parameter is inicialized true' fi @@ -216,24 +220,27 @@ Compile_gprof() echo fi - #Copy all the EXP00 data but don't overwrite namelist just the executable - cd "$dir" || echo "Error original dir doesn't exist" exit - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/* . - cp "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/nemo . + + + + #Copy all the EXP00 data in RUN_NEMO folder but don't overwrite the namelist, just the executable. + cd "$dir" || (echo "Error original dir doesn't exist"; exit 1) + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/* Run_NEMO + cp "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/nemo Run_NEMO if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then - sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) + sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' Run_NEMO/context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) fi #Solving NEMO input file common errors - if test -f "weights_core_orca2_bicubic_noc.nc"; then - mv weights_core_orca2_bicubic_noc.nc weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES + if test -f "Run_NEMO/weights_core_orca2_bicubic_noc.nc"; then + mv Run_NEMO/weights_core_orca2_bicubic_noc.nc Run_NEMO/weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES fi if test -f "weights_core_orca2_bilinear_noc.nc"; then - mv weights_core_orca2_bilinear_noc.nc weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES + mv Run_NEMO/weights_core_orca2_bilinear_noc.nc Run_NEMO/weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES fi } @@ -245,7 +252,8 @@ Compile_gprof() Init() { - Nemo_path="${Nemo_path:-"."}" + Nemo_path="${Nemo_path:-"./"}" + Nemo_input_data="${Nemo_input_data=:-"./"}" Nemo_cores="${Nemo_cores:-( 48 )}" Jobs_n_cores="${Jobs_n_cores:-48}" Jobs_scheduler="${Jobs_scheduler:-"slurm"}" @@ -273,6 +281,21 @@ Test_arguments() echo exit 1 fi + + + # Is Nemo data path correct? + if ! test -d "${Nemo_input_data}"; then + echo "Nemo input data path: ${Nemo_input_data} is not found" + echo + exit 1 + + else + if ! test -d "Run_NEMO"; then + mkdir Run_NEMO + fi + cp -P "${Nemo_input_data}"* Run_NEMO + fi + #Nemo_cores is array? if [[ ! "$(declare -p Nemo_cores)" =~ "declare -a" ]]; then echo "Error Nemo_cores has to be a bash array like ( 4 24 48 )" @@ -357,25 +380,22 @@ Test_arguments() Gprof_functions() { - + cd Run_NEMO||(echo "Error Run_NEMO folder doesn't exists"; exit 1) #Generating function list in case of missing if ! test -f "extrae_functions.txt"; then Compile_gprof - - #Changing iterations, big traces generate problems. sed -i "s|nn_itend * =.*|nn_itend = $Nemo_iterations ! last time step (std 5475)|g" namelist_cfg - rm gmon* 2> /dev/null echo "Runing Nemo with 2 cores to obtain function data..." echo - python3 ./src/Job_Creator.py -f "run" -j "run" --set-core 4 -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np 2 ./nemo" + python3 ./../src/Job_Creator.py -f "run" -j "run" --set-core 4 -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np 2 ./nemo" state2=$("$job" --wait run."$Jobs_scheduler") Job_completed "$state2" if [ $Completed == false ]; then - echo "Nemo execution failed look at run.err and ocean.output for more info" + echo "Nemo execution failed look at Run_NEMO/run.err and Run_NEMO/ocean.output for more info" echo "Remember that the namelist files copied are the default ones, change theme in order to fit with the input files in the dir " echo exit 1 @@ -385,11 +405,11 @@ Gprof_functions() fi echo "Gthrottling functions ..." echo - python3 ./src/Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./src/gthrottling.sh nemo" + python3 ./../src/Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./../src/gthrottling.sh nemo" state3=$("$job" --wait gthrottling."$Jobs_scheduler") Job_completed "$state3" if [ $Completed == false ]; then - echo "Error listing functions, look at gthrottling.err for more info" + echo "Error listing functions, look at Run_NEMO/gthrottling.err for more info" echo exit 1 else @@ -401,7 +421,7 @@ Gprof_functions() else - echo "Functions already listed, file extrae_functions.txt does exist" + echo "Functions already listed, file Run_NEMO/extrae_functions.txt does exist" echo fi @@ -417,20 +437,20 @@ Get_trace() Compile_extrae # Run nemo with extrae - ./src/extraf.sh nemo extrae_functions.txt - sed -i "s|list=.*|list=\"${dir}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" src/extrae.xml + ./../src/extraf.sh nemo extrae_functions.txt + sed -i "s|list=.*|list=\"${dir}/Run_NEMO/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" ../src/extrae.xml for core in "${Nemo_cores[@]}" do echo "Creating trace with $core cores..." echo - python3 ./src/Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core ./src/trace.sh ./nemo" + python3 ./../src/Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core ./../src/trace.sh ./nemo" state4=$("$job" --wait run_extrae."$Jobs_scheduler") Job_completed "$state4" if [ $Completed == false ]; then - echo "Nemo execution failed, no traces files generated more info inside run_extrae.err" + echo "Nemo execution failed, no traces files generated more info inside Run_NEMO/run_extrae.err" echo exit 1 fi @@ -439,21 +459,23 @@ Compile_extrae mv nemo.row nemo_"$core".row echo "Cutting best iteration" echo - ./src/magiccut/magicCut nemo_"${core}".prv "$Nemo_iterations" > cut_"$core".out 2>&1 + ./../src/magiccut/magicCut nemo_"${core}".prv "$Nemo_iterations" > cut_"$core".out 2>&1 if ! ls nemo_"$core".best_cut.prv; then - echo "Cut failed, look at cut_$core.out for more info." + echo "Cut failed, look at Run_NEMO/cut_$core.out for more info." echo exit 1 fi echo # Creating folder - if ! test -d "Metrics"; then - mkdir Metrics + + if ! test -d "../Metrics"; then + mkdir ../Metrics fi + cp nemo_"$core".best_cut.* ../Metrics - cp nemo_"$core".best_cut.* Metrics - done + + done } @@ -461,7 +483,7 @@ Create_metrics() { - + cd "$dir" || (echo "Error original dir doesn't exist"; exit 1) # Create performance metrics echo "Creating metrics and storing theme in Metrics folder" echo -- GitLab From 75df6af1b99ae70022d188cf876ac3961d1ea413 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Mon, 12 Sep 2022 16:25:44 +0200 Subject: [PATCH 20/40] Fixed mistake --- src/functions.bash | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/functions.bash b/src/functions.bash index eb5c5bd..7d7cb1d 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -136,6 +136,7 @@ Compile_extrae() #Copy all the EXP00 data but don't overwrite namelist just the executable cd "$dir" ||( echo "Error original dir doesn't exist"; exit 1 ) + echow "" cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* Run_NEMO cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo Run_NEMO cd Run_NEMO||( echo "Error Run_NEMO folder doesn't exists"; exit 1 ) @@ -221,10 +222,8 @@ Compile_gprof() fi - - #Copy all the EXP00 data in RUN_NEMO folder but don't overwrite the namelist, just the executable. - cd "$dir" || (echo "Error original dir doesn't exist"; exit 1) + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/* Run_NEMO cp "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/nemo Run_NEMO @@ -243,6 +242,8 @@ Compile_gprof() mv Run_NEMO/weights_core_orca2_bilinear_noc.nc Run_NEMO/weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES fi + cd Run_NEMO||(echo "Error Run_NEMO folder doesn't exists"; exit 1) + } -- GitLab From 660ad49185683dc8bf4809d7f3e1d51551edb9eb Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 13 Sep 2022 09:15:17 +0200 Subject: [PATCH 21/40] Added cores_per_node to config --- perf_metrics.config | 4 +++- src/Job_Creator.py | 14 +++++++++----- src/functions.bash | 13 +++++++------ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/perf_metrics.config b/perf_metrics.config index 575f178..39125fa 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -13,11 +13,13 @@ Nemo_input_data="../DATA/ORCA2_ICE_v4.2.0/" Nemo_cores=( 4 24 48 96 192) # Jobs_n_cores: nºcores used for other jobs like compiling nemo. +# Jobs_cores_per_node: define the number of cores per node. # Jobs_scheduler: Available (slurm/lsf/torque). # Jobs_time: Max duration of the job in min. # Jobs_queue: Queue used. -Jobs_n_cores=96 +Jobs_n_cores=96 +Jobs_cores_per_node=0 Jobs_scheduler="slurm" Jobs_time=60 Jobs_queue=debug diff --git a/src/Job_Creator.py b/src/Job_Creator.py index dc70f90..97cc727 100644 --- a/src/Job_Creator.py +++ b/src/Job_Creator.py @@ -22,7 +22,7 @@ def get_command_line_arguments(): help="Set slurm time, in minutes") parser.add_argument("--set-core", default=1, type=int, help="Set number of cores to be used for the job") - parser.add_argument("--set-core-per-node", default=48, type=int, + parser.add_argument("--set-core-per-node", default=0, type=int, help="Set number of cores to be used for the job") parser.add_argument("-j", "--job-name", default=None, help="Name of the job you want to create or modify") @@ -65,7 +65,7 @@ def create_job_slurm(args): if cores is not None: file.append("#SBATCH --ntasks " + str(cores) + "\n") - if cores_per_node is not None: + if cores_per_node is not None and not 0: file.append("#SBATCH --ntasks-per-node " + str(cores_per_node) + "\n") if time is not None and not 0: file.append("#SBATCH --time " + str(time) + "\n") @@ -125,9 +125,13 @@ def create_job_torque(args): cores_per_node = args.set_core_per_node queue = args.set_queue workload = args.set_workload - nodes = (cores//cores_per_node)+1 - hours = time // 60 - minutes = time % 60 + if cores_per_node is not None: + nodes = (cores//cores_per_node)+1 + + if time is not None: + hours = time // 60 + minutes = time % 60 + file = ["#!/bin/bash \n", "############################################################################### \n", "#PBS -o "+str(name)+".out \n#PBS -e "+str(name)+".err \n" diff --git a/src/functions.bash b/src/functions.bash index 7d7cb1d..e3832aa 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -115,7 +115,7 @@ Compile_extrae() echo "Output of the compilation in compile.err and compile.out" printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" - python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" state1=$("$job" --wait compile."$Jobs_scheduler") echo @@ -202,7 +202,7 @@ Compile_gprof() echo "Output of the compilation in compile.err and compile.out" printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg" - python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" state1=$("$job" --wait compile."$Jobs_scheduler") echo @@ -257,6 +257,7 @@ Init() Nemo_input_data="${Nemo_input_data=:-"./"}" Nemo_cores="${Nemo_cores:-( 48 )}" Jobs_n_cores="${Jobs_n_cores:-48}" + Jobs_cores_per_node="${Jobs_cores_per_node:-0}" Jobs_scheduler="${Jobs_scheduler:-"slurm"}" time="${Jobs_time:-0}" queue="${Jobs_queue:-""}" @@ -391,7 +392,7 @@ Gprof_functions() rm gmon* 2> /dev/null echo "Runing Nemo with 2 cores to obtain function data..." echo - python3 ./../src/Job_Creator.py -f "run" -j "run" --set-core 4 -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np 2 ./nemo" + python3 ./../src/Job_Creator.py -f "run" -j "run" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np 2 ./nemo" state2=$("$job" --wait run."$Jobs_scheduler") Job_completed "$state2" @@ -406,7 +407,7 @@ Gprof_functions() fi echo "Gthrottling functions ..." echo - python3 ./../src/Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./../src/gthrottling.sh nemo" + python3 ./../src/Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./../src/gthrottling.sh nemo" state3=$("$job" --wait gthrottling."$Jobs_scheduler") Job_completed "$state3" if [ $Completed == false ]; then @@ -446,7 +447,7 @@ Compile_extrae echo "Creating trace with $core cores..." echo - python3 ./../src/Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core ./../src/trace.sh ./nemo" + python3 ./../src/Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core ./../src/trace.sh ./nemo" state4=$("$job" --wait run_extrae."$Jobs_scheduler") Job_completed "$state4" @@ -488,7 +489,7 @@ Create_metrics() # Create performance metrics echo "Creating metrics and storing theme in Metrics folder" echo - python3 ./src/Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./../src/modelfactors.py -ms 100000 *" + python3 ./src/Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./../src/modelfactors.py -ms 100000 *" mv analysis."$Jobs_scheduler" Metrics cd Metrics||(echo "Error Metrics folder doesn't exists"; exit 1) state5=$("$job" --wait analysis."$Jobs_scheduler") -- GitLab From 1b93fa7175e234ebe686f78072af0ae823f457ba Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 13 Sep 2022 09:30:05 +0200 Subject: [PATCH 22/40] Fixed mistakes --- src/functions.bash | 1 - src/trace.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/functions.bash b/src/functions.bash index e3832aa..8564d5e 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -136,7 +136,6 @@ Compile_extrae() #Copy all the EXP00 data but don't overwrite namelist just the executable cd "$dir" ||( echo "Error original dir doesn't exist"; exit 1 ) - echow "" cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* Run_NEMO cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo Run_NEMO cd Run_NEMO||( echo "Error Run_NEMO folder doesn't exists"; exit 1 ) diff --git a/src/trace.sh b/src/trace.sh index 12a53dc..f81f6d5 100755 --- a/src/trace.sh +++ b/src/trace.sh @@ -1,7 +1,7 @@ #!/bin/sh # Configure Extrae -export EXTRAE_CONFIG_FILE=./src/extrae.xml +export EXTRAE_CONFIG_FILE=./../src/extrae.xml # Load the tracing library (choose C/Fortran) #export LD_PRELOAD=${EXTRAE_HOME}/lib/libmpitrace.so export EXTRAE_SKIP_AUTO_LIBRARY_INITIALIZE=1 -- GitLab From 52ba0e2bb4e5a5c86039860ba1112ee51dcbd33e Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 13 Sep 2022 12:11:11 +0200 Subject: [PATCH 23/40] Fixed some errors related with absolute path --- perf_metrics.config | 40 +++++++++++++++++----------------- src/Job_Creator.py | 8 +++---- src/functions.bash | 52 +++++++++++++++++++++++++-------------------- 3 files changed, 54 insertions(+), 46 deletions(-) diff --git a/perf_metrics.config b/perf_metrics.config index 39125fa..0d14865 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -3,41 +3,43 @@ ################################################################################# -# Nemo_path: Path to nemo installation folder containing the cfgs and arch dirs. -# Nemo_input_data: Path to the input data needed to run the nemo cfg. -# Nemo_cores: List of nºcores used for executing Nemo, ( 4 48 ) makes the script execute and +# Nemo_path (string) : Path to nemo installation folder containing the cfgs and arch dirs. +# Nemo_input_data (string): Path to the input data needed to run the nemo cfg. +# Nemo_run (string): Path where the folder Run_NEMO will be created. +# Nemo_cores (array): List of nºcores used for executing Nemo, ( 4 48 ) makes the script execute and # get Nemo traces with 4 and 48 cores. 2 different nºcores are needed to obtain scalability data. -Nemo_path="../NEMO/" -Nemo_input_data="../DATA/ORCA2_ICE_v4.2.0/" +Nemo_path="Nemo_installation_path" +Nemo_input_data="Nemo_input_data_path" +Nemo_run="." Nemo_cores=( 4 24 48 96 192) -# Jobs_n_cores: nºcores used for other jobs like compiling nemo. -# Jobs_cores_per_node: define the number of cores per node. -# Jobs_scheduler: Available (slurm/lsf/torque). -# Jobs_time: Max duration of the job in min. -# Jobs_queue: Queue used. +# Jobs_n_cores (integer): nºcores used for other jobs like compiling nemo. +# Jobs_cores_per_node (integer): define the number of cores per node. +# Jobs_scheduler (string): Available (slurm/lsf/torque). +# Jobs_time (integer): Max duration of the job in min. +# Jobs_queue (string): Queue used. -Jobs_n_cores=96 +Jobs_n_cores= 4 Jobs_cores_per_node=0 Jobs_scheduler="slurm" Jobs_time=60 -Jobs_queue=debug +Jobs_queue="debug" -# Compilation_compile: When false only compiles NEMO if arch file lacks the needed flags, when true always compiles NEMO. -# Compilation_ref: Reference configuration. -# Compilation_arch: Architecture used (without the -arch sufix and the .fcm). -# Compilation_name: Name of the new configutation (Important to not be an existing one). -# Compilation_sub: Add or remove subcomponents. +# Compilation_compile (string): When false only compiles NEMO if arch file lacks the needed flags, when true always compiles NEMO. +# Compilation_ref (string): Reference configuration. +# Compilation_arch (string): Architecture used (without the -arch sufix and the .fcm). +# Compilation_name (string): Name of the new configutation (Important to not be an existing one). +# Compilation_sub (string): Add or remove subcomponents. Compilation_compile="false" Compilation_ref="ORCA2_ICE_PISCES" -Compilation_arch="Your-arch-file" +Compilation_arch="Your-arch-file" Compilation_name="ORCA2_EXTRAE" Compilation_sub="OCE del_key 'key_si3 key_top'" -# List of modules loaded. +# Modules (string): List of modules loaded. # Required: # - Perl interpreter # - Fortran compiler (ifort, gfortran, pgfortran, ftn, …) diff --git a/src/Job_Creator.py b/src/Job_Creator.py index 97cc727..94180fd 100644 --- a/src/Job_Creator.py +++ b/src/Job_Creator.py @@ -65,9 +65,9 @@ def create_job_slurm(args): if cores is not None: file.append("#SBATCH --ntasks " + str(cores) + "\n") - if cores_per_node is not None and not 0: + if cores_per_node is not None and cores_per_node != 0: file.append("#SBATCH --ntasks-per-node " + str(cores_per_node) + "\n") - if time is not None and not 0: + if time is not None and time!= 0: file.append("#SBATCH --time " + str(time) + "\n") if name is not None: file.append("#SBATCH -J " + name + "\n") @@ -100,7 +100,7 @@ def create_job_lsf(args): if cores is not None: file.append("#BSUB -n " + str(cores) + "\n") - if time is not None and not 0: + if time is not None and time!= 0: file.append("#BSUB -W " + str(time) + "\n") if name is not None: file.append("#BSUB-J " + name + "\n") @@ -139,7 +139,7 @@ def create_job_torque(args): if cores is not None: file.append("#PBS -l nodes" + str(nodes) + ":ppn=" + str(cores) + "\n") - if time is not None and not 0: + if time is not None and time != 0: file.append("#PBS -l cput=" + str(hours) + ":"+str(minutes)+ ":00\n") if name is not None: file.append("#PBS -N " + name + "\n") diff --git a/src/functions.bash b/src/functions.bash index 8564d5e..3afa808 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -115,7 +115,7 @@ Compile_extrae() echo "Output of the compilation in compile.err and compile.out" printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" - python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + python3 src/./Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" state1=$("$job" --wait compile."$Jobs_scheduler") echo @@ -135,9 +135,10 @@ Compile_extrae() fi #Copy all the EXP00 data but don't overwrite namelist just the executable - cd "$dir" ||( echo "Error original dir doesn't exist"; exit 1 ) + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* Run_NEMO cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo Run_NEMO + cd Run_NEMO||( echo "Error Run_NEMO folder doesn't exists"; exit 1 ) } @@ -201,7 +202,7 @@ Compile_gprof() echo "Output of the compilation in compile.err and compile.out" printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg" - python3 ./src/Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + python3 src/./Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" state1=$("$job" --wait compile."$Jobs_scheduler") echo @@ -278,7 +279,7 @@ Test_arguments() { # Nemo path correct? if ! test -d "${Nemo_path}"; then - echo "Nemo relative path: ${Nemo_path} is not found" + echo "Nemo relative path: ${Nemo_path} is not found, modify it in perf_metrics.config with the correct path" echo exit 1 fi @@ -286,7 +287,7 @@ Test_arguments() # Is Nemo data path correct? if ! test -d "${Nemo_input_data}"; then - echo "Nemo input data path: ${Nemo_input_data} is not found" + echo "Nemo input data path: ${Nemo_input_data} is not found, modify it in perf_metrics.config with the correct path" echo exit 1 @@ -294,12 +295,12 @@ Test_arguments() if ! test -d "Run_NEMO"; then mkdir Run_NEMO fi - cp -P "${Nemo_input_data}"* Run_NEMO + cp -P "${Nemo_input_data}"/* Run_NEMO fi #Nemo_cores is array? if [[ ! "$(declare -p Nemo_cores)" =~ "declare -a" ]]; then - echo "Error Nemo_cores has to be a bash array like ( 4 24 48 )" + echo "Error Nemo_cores has to be a bash array like ( 4 24 48 ), modify it in perf_metrics.config with a valid value" echo exit 1 fi @@ -312,7 +313,7 @@ Test_arguments() do if ! [[ $core =~ $re ]] ; then - echo "Error Nemo_cores has to contain integer values" + echo "Error Nemo_cores has to contain integer values, modify it in perf_metrics.config with a valid value" echo exit 1 fi @@ -320,31 +321,31 @@ Test_arguments() # cfg exists? if ! test -d "${Nemo_path}/cfgs/${cfg}"; then - echo "configuration: ${cfg} doesn't exists in ${Nemo_path}/cfgs dir" + echo "configuration: ${cfg} doesn't exists in ${Nemo_path}/cfgs dir, modify it in perf_metrics.config with a valid cfg" echo exit 1 fi # arch exists? if ! test -f "${Nemo_path}/arch/arch-${arch}.fcm"; then - echo "architecture: arch-${arch}.fcm doesn't exists in ${Nemo_path}/arch dir" + echo "architecture: arch-${arch}.fcm doesn't exists in ${Nemo_path}/arch dir, modify it in perf_metrics.config with a valid arch" echo exit 1 fi # scheduler correct? if [ "$Jobs_scheduler" != "slurm" ] && [ "$Jobs_scheduler" != "lsf" ] && [ "$Jobs_scheduler" != "torque" ]; then - echo "$Jobs_scheduler is not a valid scheduler" + echo "$Jobs_scheduler is not a valid scheduler, modify it in perf_metrics.config with a valid scheduler (slurm/lsf/torque)" echo exit 1 fi #Nemo_cores is array? if [[ ! "$(declare -p Nemo_cores)" =~ "declare -a" ]]; then - echo "Error, variable Nemo_cores has to be an array" + echo "Error, variable Nemo_cores has to be an array, modify it in perf_metrics.config with a proper value" echo exit 1 fi #Modules available if ! module load $Modules;then - echo "Error loading modules aborting" + echo "Error loading modules aborting, modify the modules loaded in perf_metrics.config" echo exit 1 fi @@ -357,7 +358,7 @@ Test_arguments() else sed -i 's|home=.*|home="'"$EXTRAE_HOME"'"|g' src/extrae.xml fi - + # Adding -d to variable if not empty if [ -n "$comp_cfg" ]; then comp_cfg="-d $comp_cfg" @@ -372,6 +373,10 @@ Test_arguments() job="qsub" fi + # Changing trace.sh reference to extrae.xml + + sed -i "s|export EXTRAE_CONFIG_FILE=.*|export EXTRAE_CONFIG_FILE=$dir/src/./extrae.xml|g" src/trace.sh + } @@ -382,6 +387,7 @@ Gprof_functions() { cd Run_NEMO||(echo "Error Run_NEMO folder doesn't exists"; exit 1) + #Generating function list in case of missing if ! test -f "extrae_functions.txt"; then @@ -391,7 +397,7 @@ Gprof_functions() rm gmon* 2> /dev/null echo "Runing Nemo with 2 cores to obtain function data..." echo - python3 ./../src/Job_Creator.py -f "run" -j "run" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np 2 ./nemo" + python3 "$dir"/src/./Job_Creator.py -f "run" -j "run" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np 2 ./nemo" state2=$("$job" --wait run."$Jobs_scheduler") Job_completed "$state2" @@ -406,7 +412,7 @@ Gprof_functions() fi echo "Gthrottling functions ..." echo - python3 ./../src/Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./../src/gthrottling.sh nemo" + python3 "$dir"/src/./Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./gthrottling.sh nemo" state3=$("$job" --wait gthrottling."$Jobs_scheduler") Job_completed "$state3" if [ $Completed == false ]; then @@ -438,7 +444,7 @@ Get_trace() Compile_extrae # Run nemo with extrae - ./../src/extraf.sh nemo extrae_functions.txt + "$dir"/src/./extraf.sh nemo extrae_functions.txt sed -i "s|list=.*|list=\"${dir}/Run_NEMO/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" ../src/extrae.xml for core in "${Nemo_cores[@]}" @@ -446,7 +452,7 @@ Compile_extrae echo "Creating trace with $core cores..." echo - python3 ./../src/Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core ./../src/trace.sh ./nemo" + python3 "$dir"/src/./Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core "$dir"/src/./trace.sh ./nemo" state4=$("$job" --wait run_extrae."$Jobs_scheduler") Job_completed "$state4" @@ -469,10 +475,10 @@ Compile_extrae echo # Creating folder - if ! test -d "../Metrics"; then - mkdir ../Metrics + if ! test -d "$dir/Metrics"; then + mkdir "$dir"/Metrics fi - cp nemo_"$core".best_cut.* ../Metrics + cp nemo_"$core".best_cut.* "$dir"/Metrics @@ -488,7 +494,7 @@ Create_metrics() # Create performance metrics echo "Creating metrics and storing theme in Metrics folder" echo - python3 ./src/Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "./../src/modelfactors.py -ms 100000 *" + python3 src/./Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./modelfactors.py -ms 100000 *" mv analysis."$Jobs_scheduler" Metrics cd Metrics||(echo "Error Metrics folder doesn't exists"; exit 1) state5=$("$job" --wait analysis."$Jobs_scheduler") @@ -506,4 +512,4 @@ Create_metrics() echo "------------------------------------------------------------------------------" echo "------------------------------------------------------------------------------" echo -} \ No newline at end of file +} -- GitLab From 53249f7c805335985efc1833c2147919c4aa0fc0 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 13 Sep 2022 12:32:03 +0200 Subject: [PATCH 24/40] Now exits if sbatch fails submitting a job --- src/functions.bash | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/functions.bash b/src/functions.bash index 3afa808..6362da4 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -117,7 +117,11 @@ Compile_extrae() printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" python3 src/./Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" - state1=$("$job" --wait compile."$Jobs_scheduler") + + if ! state1=$("$job" --wait compile."$Jobs_scheduler"); then + exit 1: + fi + echo Job_completed "$state1" if [ $Completed == false ]; then @@ -204,7 +208,10 @@ Compile_gprof() printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg" python3 src/./Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" - state1=$("$job" --wait compile."$Jobs_scheduler") + if ! state1=$("$job" --wait compile."$Jobs_scheduler"); then + exit 1 + fi + echo Job_completed "$state1" if [ "$Completed" == false ]; then @@ -399,7 +406,9 @@ Gprof_functions() echo python3 "$dir"/src/./Job_Creator.py -f "run" -j "run" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np 2 ./nemo" - state2=$("$job" --wait run."$Jobs_scheduler") + if ! state2=$("$job" --wait run."$Jobs_scheduler"); then + exit 1 + fi Job_completed "$state2" if [ $Completed == false ]; then echo "Nemo execution failed look at Run_NEMO/run.err and Run_NEMO/ocean.output for more info" @@ -413,7 +422,9 @@ Gprof_functions() echo "Gthrottling functions ..." echo python3 "$dir"/src/./Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./gthrottling.sh nemo" - state3=$("$job" --wait gthrottling."$Jobs_scheduler") + if ! state3=$("$job" --wait gthrottling."$Jobs_scheduler"); then + exit 1 + fi Job_completed "$state3" if [ $Completed == false ]; then echo "Error listing functions, look at Run_NEMO/gthrottling.err for more info" @@ -454,7 +465,9 @@ Compile_extrae echo python3 "$dir"/src/./Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core "$dir"/src/./trace.sh ./nemo" - state4=$("$job" --wait run_extrae."$Jobs_scheduler") + if ! state4=$("$job" --wait run_extrae."$Jobs_scheduler") ; then + exit 1 + fi Job_completed "$state4" if [ $Completed == false ]; then echo "Nemo execution failed, no traces files generated more info inside Run_NEMO/run_extrae.err" @@ -497,7 +510,9 @@ Create_metrics() python3 src/./Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./modelfactors.py -ms 100000 *" mv analysis."$Jobs_scheduler" Metrics cd Metrics||(echo "Error Metrics folder doesn't exists"; exit 1) - state5=$("$job" --wait analysis."$Jobs_scheduler") + if ! state5=$("$job" --wait analysis."$Jobs_scheduler"); then + exit 1 + fi Job_completed "$state5" if [ $Completed == false ]; then echo "Error, metrics have not generated check Metrics/analysis.err to get more details" -- GitLab From 490b654fd7e90877083886f9d35a39b531e6212d Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 13 Sep 2022 12:54:28 +0200 Subject: [PATCH 25/40] Logs are now stored in logs dir --- src/functions.bash | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/functions.bash b/src/functions.bash index 6362da4..d6f64dd 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -125,12 +125,13 @@ Compile_extrae() echo Job_completed "$state1" if [ $Completed == false ]; then - echo "Nemo compilation failed, remember to load all the needed modules. Check the details in compile.err" + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in logs/compile.err" echo exit 1 else echo "Nemo compilation successful" echo + mv compile.* "$dir"/logs fi else @@ -215,12 +216,13 @@ Compile_gprof() echo Job_completed "$state1" if [ "$Completed" == false ]; then - echo "Nemo compilation failed, remember to load all the needed modules. Check the details in compile.err" + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in logs/compile.err" echo exit 1 else echo "Nemo compilation successful" echo + mv compile.* "$dir"/logs fi else @@ -384,6 +386,10 @@ Test_arguments() sed -i "s|export EXTRAE_CONFIG_FILE=.*|export EXTRAE_CONFIG_FILE=$dir/src/./extrae.xml|g" src/trace.sh + # Creating logs dir + if ! test -d "logs"; then + mkdir logs + fi } @@ -411,13 +417,14 @@ Gprof_functions() fi Job_completed "$state2" if [ $Completed == false ]; then - echo "Nemo execution failed look at Run_NEMO/run.err and Run_NEMO/ocean.output for more info" + echo "Nemo execution failed look at logs/run.err and Run_NEMO/ocean.output for more info" echo "Remember that the namelist files copied are the default ones, change theme in order to fit with the input files in the dir " echo exit 1 else echo "Gprof files generated " echo + mv run.* "$dir"/logs fi echo "Gthrottling functions ..." echo @@ -427,12 +434,13 @@ Gprof_functions() fi Job_completed "$state3" if [ $Completed == false ]; then - echo "Error listing functions, look at Run_NEMO/gthrottling.err for more info" + echo "Error listing functions, look at logs/gthrottling.err for more info" echo exit 1 else echo "Functions listed correctly" echo + mv gthrottling.* "$dir"/logs fi @@ -470,10 +478,13 @@ Compile_extrae fi Job_completed "$state4" if [ $Completed == false ]; then - echo "Nemo execution failed, no traces files generated more info inside Run_NEMO/run_extrae.err" + echo "Nemo execution failed, no traces files generated more info inside logs/run_extrae.err" echo exit 1 fi + + mv run_extrae.* "$dir"/logs + mv nemo.prv nemo_"$core".prv mv nemo.pcf nemo_"$core".pcf mv nemo.row nemo_"$core".row @@ -513,14 +524,16 @@ Create_metrics() if ! state5=$("$job" --wait analysis."$Jobs_scheduler"); then exit 1 fi + Job_completed "$state5" if [ $Completed == false ]; then - echo "Error, metrics have not generated check Metrics/analysis.err to get more details" + echo "Error, metrics have not generated check logs/analysis.err to get more details" echo exit 1 fi - + cp analysis.out overview.txt + mv analysis.* "$dir"/logs echo "------------------------------------------------------------------------------" echo "------------------------- Script Completed -----------------------------------" echo "----------------------- Data in Metrics folder -------------------------------" -- GitLab From c5e99533b58f912a678ace1fd5159890e9f0149b Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 13 Sep 2022 15:11:18 +0200 Subject: [PATCH 26/40] Now NEMO run dir can be defined in config --- README.md | 2 +- perf_metrics.config | 2 +- src/functions.bash | 65 ++++++++++++++++++++++++++------------------- 3 files changed, 39 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 58941d3..abf8159 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Nemo modelfactors -The project was developed tested using NEMO 4.2 version. +The project was developed and tested using NEMO 4.2 version. This project is intended to compute important performance metrics for a NEMO run. diff --git a/perf_metrics.config b/perf_metrics.config index 0d14865..86c051c 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -21,7 +21,7 @@ Nemo_cores=( 4 24 48 96 192) # Jobs_queue (string): Queue used. Jobs_n_cores= 4 -Jobs_cores_per_node=0 +Jobs_cores_per_node= Jobs_scheduler="slurm" Jobs_time=60 Jobs_queue="debug" diff --git a/src/functions.bash b/src/functions.bash index d6f64dd..ebf0faa 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -141,10 +141,10 @@ Compile_extrae() #Copy all the EXP00 data but don't overwrite namelist just the executable - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* Run_NEMO - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo Run_NEMO + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* "${Nemo_run}"/Run_NEMO + cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Nemo_run}"/Run_NEMO - cd Run_NEMO||( echo "Error Run_NEMO folder doesn't exists"; exit 1 ) + cd "${Nemo_run}"/Run_NEMO||( echo "Error ${Nemo_run}/Run_NEMO folder doesn't exists"; exit 1 ) } #Check if Nemo is compiled for using Gprof @@ -233,25 +233,25 @@ Compile_gprof() #Copy all the EXP00 data in RUN_NEMO folder but don't overwrite the namelist, just the executable. - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/* Run_NEMO - cp "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/nemo Run_NEMO + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/* "${Nemo_run}"/Run_NEMO + cp "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/nemo "${Nemo_run}"/Run_NEMO if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then - sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' Run_NEMO/context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) + sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' "${Nemo_run}"/Run_NEMO/context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) fi #Solving NEMO input file common errors - if test -f "Run_NEMO/weights_core_orca2_bicubic_noc.nc"; then - mv Run_NEMO/weights_core_orca2_bicubic_noc.nc Run_NEMO/weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES + if test -f "${Nemo_run}/Run_NEMO/weights_core_orca2_bicubic_noc.nc"; then + mv "${Nemo_run}"/Run_NEMO/weights_core_orca2_bicubic_noc.nc "${Nemo_run}"/Run_NEMO/weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES fi - if test -f "weights_core_orca2_bilinear_noc.nc"; then - mv Run_NEMO/weights_core_orca2_bilinear_noc.nc Run_NEMO/weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES + if test -f "${Nemo_run}/Run_NEMO/weights_core_orca2_bilinear_noc.nc"; then + mv "${Nemo_run}"/Run_NEMO/weights_core_orca2_bilinear_noc.nc "${Nemo_run}"/Run_NEMO/weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES fi - cd Run_NEMO||(echo "Error Run_NEMO folder doesn't exists"; exit 1) + cd "${Nemo_run}"/Run_NEMO||(echo "Error ${Nemo_run}/Run_NEMO folder doesn't exists"; exit 1) } @@ -264,6 +264,7 @@ Init() { Nemo_path="${Nemo_path:-"./"}" Nemo_input_data="${Nemo_input_data=:-"./"}" + Nemo_run="${Nemo_run=:-"./"}" Nemo_cores="${Nemo_cores:-( 48 )}" Jobs_n_cores="${Jobs_n_cores:-48}" Jobs_cores_per_node="${Jobs_cores_per_node:-0}" @@ -299,14 +300,21 @@ Test_arguments() echo "Nemo input data path: ${Nemo_input_data} is not found, modify it in perf_metrics.config with the correct path" echo exit 1 - - else - if ! test -d "Run_NEMO"; then - mkdir Run_NEMO - fi - cp -P "${Nemo_input_data}"/* Run_NEMO + fi + # Is Nemo Run path correct? + if ! test -d "${Nemo_run}"; then + echo "Nemo run path: ${Nemo_run} is not found, modify it in perf_metrics.config with the correct path" + echo + exit 1 fi + # Create Run_Nemo dir after comprovations + if ! test -d "${Nemo_run}/Run_NEMO"; then + mkdir "${Nemo_run}"/Run_NEMO + fi + cp -P "${Nemo_input_data}"/* "${Nemo_run}"/Run_NEMO + + #Nemo_cores is array? if [[ ! "$(declare -p Nemo_cores)" =~ "declare -a" ]]; then echo "Error Nemo_cores has to be a bash array like ( 4 24 48 ), modify it in perf_metrics.config with a valid value" @@ -387,9 +395,10 @@ Test_arguments() sed -i "s|export EXTRAE_CONFIG_FILE=.*|export EXTRAE_CONFIG_FILE=$dir/src/./extrae.xml|g" src/trace.sh # Creating logs dir - if ! test -d "logs"; then - mkdir logs - fi + if ! test -d "logs"; then + mkdir logs + + fi } @@ -399,7 +408,7 @@ Test_arguments() Gprof_functions() { - cd Run_NEMO||(echo "Error Run_NEMO folder doesn't exists"; exit 1) + cd "${Nemo_run}"/Run_NEMO||(echo "Error ${Nemo_run}/Run_NEMO folder doesn't exists"; exit 1) #Generating function list in case of missing if ! test -f "extrae_functions.txt"; then @@ -417,8 +426,8 @@ Gprof_functions() fi Job_completed "$state2" if [ $Completed == false ]; then - echo "Nemo execution failed look at logs/run.err and Run_NEMO/ocean.output for more info" - echo "Remember that the namelist files copied are the default ones, change theme in order to fit with the input files in the dir " + echo "Nemo execution failed look at logs/run.err and ${Nemo_run}/Run_NEMO/ocean.output for more info" + echo "Remember that the namelist files copied are the default ones, change them in order to fit with the input files in the dir " echo exit 1 else @@ -447,7 +456,7 @@ Gprof_functions() else - echo "Functions already listed, file Run_NEMO/extrae_functions.txt does exist" + echo "Functions already listed, file ${Nemo_run}/Run_NEMO/extrae_functions.txt does exist" echo fi @@ -464,7 +473,7 @@ Compile_extrae # Run nemo with extrae "$dir"/src/./extraf.sh nemo extrae_functions.txt - sed -i "s|list=.*|list=\"${dir}/Run_NEMO/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" ../src/extrae.xml + sed -i "s|list=.*|list=\"${Nemo_run}/Run_NEMO/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" "$dir"/src/extrae.xml for core in "${Nemo_cores[@]}" do @@ -490,9 +499,9 @@ Compile_extrae mv nemo.row nemo_"$core".row echo "Cutting best iteration" echo - ./../src/magiccut/magicCut nemo_"${core}".prv "$Nemo_iterations" > cut_"$core".out 2>&1 + "$dir"/src/magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > "$dir"/logs/cut_"$core".out 2>&1 if ! ls nemo_"$core".best_cut.prv; then - echo "Cut failed, look at Run_NEMO/cut_$core.out for more info." + echo "Cut failed, look at ${dir}/logs/cut_$core.out for more info." echo exit 1 fi @@ -516,7 +525,7 @@ Create_metrics() cd "$dir" || (echo "Error original dir doesn't exist"; exit 1) # Create performance metrics - echo "Creating metrics and storing theme in Metrics folder" + echo "Creating metrics and storing them in Metrics folder" echo python3 src/./Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./modelfactors.py -ms 100000 *" mv analysis."$Jobs_scheduler" Metrics -- GitLab From fd9d7ae3e9223a437d602b77035c9fb4d4624f36 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Wed, 14 Sep 2022 09:17:03 +0200 Subject: [PATCH 27/40] Now each execution to obtain a trace has logs --- src/functions.bash | 55 +++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/src/functions.bash b/src/functions.bash index ebf0faa..37090cb 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -112,28 +112,29 @@ Compile_extrae() if [ "$compile" == true ] || [ "$compile_ext" == true ]; then echo "Compiling Nemo for EXTRAE" - echo "Output of the compilation in compile.err and compile.out" + printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" - python3 src/./Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + python3 src/./Job_Creator.py -f "compile_extrae" -j "compile_extrae" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" - if ! state1=$("$job" --wait compile."$Jobs_scheduler"); then + if ! state1=$("$job" --wait compile_extrae."$Jobs_scheduler"); then exit 1: fi echo Job_completed "$state1" if [ $Completed == false ]; then - echo "Nemo compilation failed, remember to load all the needed modules. Check the details in logs/compile.err" + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in logs/compile_extrae.err" echo exit 1 else echo "Nemo compilation successful" echo - mv compile.* "$dir"/logs + fi - + mv compile_extrae.* "$dir"/logs + else echo "Compilation not needed" echo @@ -172,7 +173,7 @@ Compile_gprof() if ! echo "${line}"|grep -q "\-g\b"; then echo "-g flag not found in arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " sed -i '/^%FCFLAGS/ s/$/ -g /' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm - compile_gprof=true + comp_gprof=true fi # If -pg is not there recompilation is requiered and -pg added @@ -180,51 +181,50 @@ Compile_gprof() if ! echo "${line}"|grep -q "\-pg\b"; then echo "-pg flag not found in FCFLAGS arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " sed -i '/^%FCFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm - compile_gprof=true + comp_gprof=true fi if ! echo "${line2}"|grep -q "\-pg\b"; then echo "-pg flag not found in FPPFLAGS arch-${arch}_GPROF.fcm : editing arch-${arch}_GPROF.fcm " sed -i '/^%FPPFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm - compile_gprof=true + comp_gprof=true fi if ! echo "${line3}"|grep -q "\-pg\b"; then echo "-pg flag not found in LDFLAGS arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " sed -i '/^%LDFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm - compile_gprof=true + comp_gprof=true fi # If nemo executable is not on the run file compile if ! test -f "${Nemo_path}/cfgs/${name_cfg}_GPROF/EXP00/nemo"; then echo "nemo executable not found in ${name_cfg}_GPROF" - compile_gprof=true + comp_gprof=true fi - if [ "$compile" == true ] || [ "$compile_gprof" == true ]; then + if [ "$compile" == true ] || [ "$comp_gprof" == true ]; then echo "Compiling Nemo for GPROF" - echo "Output of the compilation in compile.err and compile.out" - printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg" - python3 src/./Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + python3 src/./Job_Creator.py -f "compile_gprof" -j "compile_gprof" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" - if ! state1=$("$job" --wait compile."$Jobs_scheduler"); then + if ! state1=$("$job" --wait compile_gprof."$Jobs_scheduler"); then exit 1 fi echo Job_completed "$state1" if [ "$Completed" == false ]; then - echo "Nemo compilation failed, remember to load all the needed modules. Check the details in logs/compile.err" + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in logs/compile_gprof.err" echo exit 1 else echo "Nemo compilation successful" echo - mv compile.* "$dir"/logs + fi - + mv compile_gprof.* "$dir"/logs + else echo "Compilation not needed" echo @@ -433,8 +433,9 @@ Gprof_functions() else echo "Gprof files generated " echo - mv run.* "$dir"/logs fi + mv run.* "$dir"/logs + echo "Gthrottling functions ..." echo python3 "$dir"/src/./Job_Creator.py -f "gthrottling" -j "gthrottling" --set-core 4 --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./gthrottling.sh nemo" @@ -449,12 +450,10 @@ Gprof_functions() else echo "Functions listed correctly" echo - mv gthrottling.* "$dir"/logs + fi - + mv gthrottling.* "$dir"/logs - - else echo "Functions already listed, file ${Nemo_run}/Run_NEMO/extrae_functions.txt does exist" echo @@ -480,19 +479,19 @@ Compile_extrae echo "Creating trace with $core cores..." echo - python3 "$dir"/src/./Job_Creator.py -f "run_extrae" -j "run_extrae" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core "$dir"/src/./trace.sh ./nemo" + python3 "$dir"/src/./Job_Creator.py -f "run_extrae_$core" -j "run_extrae_$core" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core "$dir"/src/./trace.sh ./nemo" - if ! state4=$("$job" --wait run_extrae."$Jobs_scheduler") ; then + if ! state4=$("$job" --wait run_extrae_$core."$Jobs_scheduler") ; then exit 1 fi Job_completed "$state4" if [ $Completed == false ]; then - echo "Nemo execution failed, no traces files generated more info inside logs/run_extrae.err" + echo "Nemo execution failed, no traces files generated more info inside logs/run_extrae_$core.err" echo exit 1 fi - mv run_extrae.* "$dir"/logs + mv run_extrae_"$core".* "$dir"/logs mv nemo.prv nemo_"$core".prv mv nemo.pcf nemo_"$core".pcf -- GitLab From 585b91f42fbaf18e3115d653e9a446b2058b51aa Mon Sep 17 00:00:00 2001 From: cpenadep Date: Wed, 14 Sep 2022 09:44:54 +0200 Subject: [PATCH 28/40] Fixed spelling errors --- perf_metrics.config | 6 +++--- src/functions.bash | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/perf_metrics.config b/perf_metrics.config index 86c051c..cccfd7a 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -28,9 +28,9 @@ Jobs_queue="debug" # Compilation_compile (string): When false only compiles NEMO if arch file lacks the needed flags, when true always compiles NEMO. # Compilation_ref (string): Reference configuration. -# Compilation_arch (string): Architecture used (without the -arch sufix and the .fcm). -# Compilation_name (string): Name of the new configutation (Important to not be an existing one). -# Compilation_sub (string): Add or remove subcomponents. +# Compilation_arch (string): Architecture used (without the -arch suffix and the .fcm). +# Compilation_name (string): Name of the new configuration (Important to not be an existing one). +# Compilation_sub (string): Add or remove sub-components. Compilation_compile="false" diff --git a/src/functions.bash b/src/functions.bash index 37090cb..34ded5d 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -496,7 +496,7 @@ Compile_extrae mv nemo.prv nemo_"$core".prv mv nemo.pcf nemo_"$core".pcf mv nemo.row nemo_"$core".row - echo "Cutting best iteration" + echo "Cutting up best iteration" echo "$dir"/src/magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > "$dir"/logs/cut_"$core".out 2>&1 if ! ls nemo_"$core".best_cut.prv; then -- GitLab From a8f9412a9180812b2857239f4d57802126452451 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Wed, 14 Sep 2022 11:50:42 +0200 Subject: [PATCH 29/40] Now all the output is located in Output dir --- perf_metrics.config | 13 +++-- src/functions.bash | 113 +++++++++++++++++++++++++------------------- 2 files changed, 73 insertions(+), 53 deletions(-) diff --git a/perf_metrics.config b/perf_metrics.config index cccfd7a..16fa533 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -3,15 +3,18 @@ ################################################################################# +# Output (string): Path where the Output dir, containing all the output files, will be created. + +Output=".." + # Nemo_path (string) : Path to nemo installation folder containing the cfgs and arch dirs. # Nemo_input_data (string): Path to the input data needed to run the nemo cfg. # Nemo_run (string): Path where the folder Run_NEMO will be created. # Nemo_cores (array): List of nºcores used for executing Nemo, ( 4 48 ) makes the script execute and # get Nemo traces with 4 and 48 cores. 2 different nºcores are needed to obtain scalability data. -Nemo_path="Nemo_installation_path" -Nemo_input_data="Nemo_input_data_path" -Nemo_run="." +Nemo_path="NEMO_INSTALLATION_PATH" +Nemo_input_data="NEMO_INPUT_DATA_PATH" Nemo_cores=( 4 24 48 96 192) # Jobs_n_cores (integer): nºcores used for other jobs like compiling nemo. @@ -20,7 +23,7 @@ Nemo_cores=( 4 24 48 96 192) # Jobs_time (integer): Max duration of the job in min. # Jobs_queue (string): Queue used. -Jobs_n_cores= 4 +Jobs_n_cores=4 Jobs_cores_per_node= Jobs_scheduler="slurm" Jobs_time=60 @@ -35,7 +38,7 @@ Jobs_queue="debug" Compilation_compile="false" Compilation_ref="ORCA2_ICE_PISCES" -Compilation_arch="Your-arch-file" +Compilation_arch="YOUR_ARCH_FILE" Compilation_name="ORCA2_EXTRAE" Compilation_sub="OCE del_key 'key_si3 key_top'" diff --git a/src/functions.bash b/src/functions.bash index 34ded5d..a73d802 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -125,7 +125,7 @@ Compile_extrae() echo Job_completed "$state1" if [ $Completed == false ]; then - echo "Nemo compilation failed, remember to load all the needed modules. Check the details in logs/compile_extrae.err" + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in ${logs_path}/compile_extrae.err" echo exit 1 else @@ -133,7 +133,7 @@ Compile_extrae() echo fi - mv compile_extrae.* "$dir"/logs + mv compile_extrae.* "${logs_path}" else echo "Compilation not needed" @@ -142,10 +142,10 @@ Compile_extrae() #Copy all the EXP00 data but don't overwrite namelist just the executable - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* "${Nemo_run}"/Run_NEMO - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Nemo_run}"/Run_NEMO + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* "${Run_path}" + cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Run_path}" - cd "${Nemo_run}"/Run_NEMO||( echo "Error ${Nemo_run}/Run_NEMO folder doesn't exists"; exit 1 ) + cd "${Run_path}"||( echo "Error ${Run_path} folder doesn't exists"; exit 1 ) } #Check if Nemo is compiled for using Gprof @@ -215,7 +215,7 @@ Compile_gprof() echo Job_completed "$state1" if [ "$Completed" == false ]; then - echo "Nemo compilation failed, remember to load all the needed modules. Check the details in logs/compile_gprof.err" + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in ${logs_path}/compile_gprof.err" echo exit 1 else @@ -223,7 +223,7 @@ Compile_gprof() echo fi - mv compile_gprof.* "$dir"/logs + mv compile_gprof.* "${logs_path}" else echo "Compilation not needed" @@ -233,25 +233,25 @@ Compile_gprof() #Copy all the EXP00 data in RUN_NEMO folder but don't overwrite the namelist, just the executable. - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/* "${Nemo_run}"/Run_NEMO - cp "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/nemo "${Nemo_run}"/Run_NEMO + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/* "${Run_path}" + cp "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/nemo "${Run_path}" if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then - sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' "${Nemo_run}"/Run_NEMO/context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) + sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' "${Run_path}"/context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) fi #Solving NEMO input file common errors - if test -f "${Nemo_run}/Run_NEMO/weights_core_orca2_bicubic_noc.nc"; then - mv "${Nemo_run}"/Run_NEMO/weights_core_orca2_bicubic_noc.nc "${Nemo_run}"/Run_NEMO/weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES + if test -f "${Run_path}/weights_core_orca2_bicubic_noc.nc"; then + mv "${Run_path}"/weights_core_orca2_bicubic_noc.nc "${Run_path}"/weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES fi - if test -f "${Nemo_run}/Run_NEMO/weights_core_orca2_bilinear_noc.nc"; then - mv "${Nemo_run}"/Run_NEMO/weights_core_orca2_bilinear_noc.nc "${Nemo_run}"/Run_NEMO/weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES + if test -f "${Run_path}/weights_core_orca2_bilinear_noc.nc"; then + mv "${Run_path}"/weights_core_orca2_bilinear_noc.nc "${Run_path}"/weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES fi - cd "${Nemo_run}"/Run_NEMO||(echo "Error ${Nemo_run}/Run_NEMO folder doesn't exists"; exit 1) + cd "${Run_path}"||(echo "Error ${Run_path} folder doesn't exists"; exit 1) } @@ -262,9 +262,9 @@ Compile_gprof() Init() { + Output="${Output=:-"../"}" Nemo_path="${Nemo_path:-"./"}" Nemo_input_data="${Nemo_input_data=:-"./"}" - Nemo_run="${Nemo_run=:-"./"}" Nemo_cores="${Nemo_cores:-( 48 )}" Jobs_n_cores="${Jobs_n_cores:-48}" Jobs_cores_per_node="${Jobs_cores_per_node:-0}" @@ -287,6 +287,23 @@ Init() Test_arguments() { + + # Does Output path exist? + if ! test -d "${Output}"; then + echo "Output path: ${Output} is not found, modify it in perf_metrics.config with the correct path" + echo + exit 1 + fi + + Output_dir="${Output}/Output" + + if ! test -d "${Output_dir}"; then + + mkdir "${Output_dir}" + + fi + + # Nemo path correct? if ! test -d "${Nemo_path}"; then echo "Nemo relative path: ${Nemo_path} is not found, modify it in perf_metrics.config with the correct path" @@ -301,18 +318,14 @@ Test_arguments() echo exit 1 fi - # Is Nemo Run path correct? - if ! test -d "${Nemo_run}"; then - echo "Nemo run path: ${Nemo_run} is not found, modify it in perf_metrics.config with the correct path" - echo - exit 1 - fi - + + Run_path="${Output_dir}/Run_NEMO" # Create Run_Nemo dir after comprovations - if ! test -d "${Nemo_run}/Run_NEMO"; then - mkdir "${Nemo_run}"/Run_NEMO + if ! test -d "${Run_path}"; then + + mkdir "${Run_path}" fi - cp -P "${Nemo_input_data}"/* "${Nemo_run}"/Run_NEMO + cp -P "${Nemo_input_data}"/* "${Run_path}" #Nemo_cores is array? @@ -394,9 +407,11 @@ Test_arguments() sed -i "s|export EXTRAE_CONFIG_FILE=.*|export EXTRAE_CONFIG_FILE=$dir/src/./extrae.xml|g" src/trace.sh + logs_path="${Output_dir}/logs" # Creating logs dir - if ! test -d "logs"; then - mkdir logs + if ! test -d "${logs_path}"; then + + mkdir "$logs_path" fi @@ -408,7 +423,7 @@ Test_arguments() Gprof_functions() { - cd "${Nemo_run}"/Run_NEMO||(echo "Error ${Nemo_run}/Run_NEMO folder doesn't exists"; exit 1) + cd "${Run_path}"||(echo "Error ${Run_path} folder doesn't exists"; exit 1) #Generating function list in case of missing if ! test -f "extrae_functions.txt"; then @@ -426,7 +441,7 @@ Gprof_functions() fi Job_completed "$state2" if [ $Completed == false ]; then - echo "Nemo execution failed look at logs/run.err and ${Nemo_run}/Run_NEMO/ocean.output for more info" + echo "Nemo execution failed look at ${logs_path}/run.err and ${Run_path}/ocean.output for more info" echo "Remember that the namelist files copied are the default ones, change them in order to fit with the input files in the dir " echo exit 1 @@ -434,7 +449,7 @@ Gprof_functions() echo "Gprof files generated " echo fi - mv run.* "$dir"/logs + mv run.* "${logs_path}" echo "Gthrottling functions ..." echo @@ -444,7 +459,7 @@ Gprof_functions() fi Job_completed "$state3" if [ $Completed == false ]; then - echo "Error listing functions, look at logs/gthrottling.err for more info" + echo "Error listing functions, look at ${logs_path}/gthrottling.err for more info" echo exit 1 else @@ -452,10 +467,10 @@ Gprof_functions() echo fi - mv gthrottling.* "$dir"/logs + mv gthrottling.* "${logs_path}" else - echo "Functions already listed, file ${Nemo_run}/Run_NEMO/extrae_functions.txt does exist" + echo "Functions already listed, file ${Run_path}/extrae_functions.txt does exist" echo fi @@ -472,7 +487,7 @@ Compile_extrae # Run nemo with extrae "$dir"/src/./extraf.sh nemo extrae_functions.txt - sed -i "s|list=.*|list=\"${Nemo_run}/Run_NEMO/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" "$dir"/src/extrae.xml + sed -i "s|list=.*|list=\"${Run_path}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" "$dir"/src/extrae.xml for core in "${Nemo_cores[@]}" do @@ -486,31 +501,33 @@ Compile_extrae fi Job_completed "$state4" if [ $Completed == false ]; then - echo "Nemo execution failed, no traces files generated more info inside logs/run_extrae_$core.err" + echo "Nemo execution failed, no traces files generated more info inside ${logs_path}/run_extrae_$core.err" echo exit 1 fi - mv run_extrae_"$core".* "$dir"/logs + mv run_extrae_"$core".* "${logs_path}" mv nemo.prv nemo_"$core".prv mv nemo.pcf nemo_"$core".pcf mv nemo.row nemo_"$core".row echo "Cutting up best iteration" echo - "$dir"/src/magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > "$dir"/logs/cut_"$core".out 2>&1 + "$dir"/src/magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > "${logs_path}"/cut_"$core".out 2>&1 if ! ls nemo_"$core".best_cut.prv; then - echo "Cut failed, look at ${dir}/logs/cut_$core.out for more info." + echo "Cut failed, look at "${logs_path}"/cut_$core.out for more info." echo exit 1 fi echo # Creating folder - if ! test -d "$dir/Metrics"; then - mkdir "$dir"/Metrics + Metrics_path="${Output_dir}/Metrics" + + if ! test -d "${Metrics_path}"; then + mkdir "${Metrics_path}" fi - cp nemo_"$core".best_cut.* "$dir"/Metrics + cp nemo_"$core".best_cut.* "${Metrics_path}" @@ -524,27 +541,27 @@ Create_metrics() cd "$dir" || (echo "Error original dir doesn't exist"; exit 1) # Create performance metrics - echo "Creating metrics and storing them in Metrics folder" + echo "Creating metrics and storing them in ${Metrics_path} folder" echo python3 src/./Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./modelfactors.py -ms 100000 *" - mv analysis."$Jobs_scheduler" Metrics - cd Metrics||(echo "Error Metrics folder doesn't exists"; exit 1) + mv analysis."$Jobs_scheduler" "${Metrics_path}" + cd "${Metrics_path}"||(echo "Error ${Metrics_path} folder doesn't exists"; exit 1) if ! state5=$("$job" --wait analysis."$Jobs_scheduler"); then exit 1 fi Job_completed "$state5" if [ $Completed == false ]; then - echo "Error, metrics have not generated check logs/analysis.err to get more details" + echo "Error, metrics have not generated check ${logs_path}/analysis.err to get more details" echo exit 1 fi cp analysis.out overview.txt - mv analysis.* "$dir"/logs + mv analysis.* "${logs_path}" echo "------------------------------------------------------------------------------" echo "------------------------- Script Completed -----------------------------------" - echo "----------------------- Data in Metrics folder -------------------------------" + echo "----------------------- Data in ${Metrics_path} folder -------------------------------" echo "------------------------------------------------------------------------------" echo "------------------------------------------------------------------------------" echo -- GitLab From 0d1bf7997d40ebfd316e86999d5e442e35781807 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Thu, 15 Sep 2022 09:39:15 +0200 Subject: [PATCH 30/40] Code cleaned, relative path now work, fixed bugs --- perf_metrics.bash | 3 ++ src/functions.bash | 105 ++++++++++++++++++++++++--------------------- 2 files changed, 60 insertions(+), 48 deletions(-) diff --git a/perf_metrics.bash b/perf_metrics.bash index 6f28d98..b427da1 100755 --- a/perf_metrics.bash +++ b/perf_metrics.bash @@ -33,12 +33,15 @@ Init #Test if parameters are valid Test_arguments +cd "${Run_path}"||(echo "Error ${Run_path} folder doesn't exists"; exit 1) + #Create the list of important functions from NEMO Gprof_functions #Get the traces of the executions and cut 1 timestep Get_trace +cd "${Metrics_path}"||(echo "Error ${Metrics_path} folder doesn't exists"; exit 1) #Generate the performance metrics Create_metrics diff --git a/src/functions.bash b/src/functions.bash index a73d802..a4a442a 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -40,8 +40,7 @@ Job_completed() Compile_extrae() { - cd "$dir" || (echo "Error original dir doesn't exist"; exit 1) - + #Get flag lines line=$(sed -n '/^%FCFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) @@ -114,8 +113,8 @@ Compile_extrae() echo "Compiling Nemo for EXTRAE" - printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" - python3 src/./Job_Creator.py -f "compile_extrae" -j "compile_extrae" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + printf -v workload1 "cd ${Nemo_path}; ./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg; cd ${Run_path}" + python3 "$dir"/src/./Job_Creator.py -f "compile_extrae" -j "compile_extrae" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" if ! state1=$("$job" --wait compile_extrae."$Jobs_scheduler"); then @@ -145,7 +144,7 @@ Compile_extrae() cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* "${Run_path}" cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Run_path}" - cd "${Run_path}"||( echo "Error ${Run_path} folder doesn't exists"; exit 1 ) + } #Check if Nemo is compiled for using Gprof @@ -153,8 +152,6 @@ Compile_extrae() Compile_gprof() { - cd "$dir" || (echo "Error original dir doesn't exist"; exit 1) - if [ "$compile" == true ]; then echo 'compile parameter is inicialized true' fi @@ -205,8 +202,8 @@ Compile_gprof() if [ "$compile" == true ] || [ "$comp_gprof" == true ]; then echo "Compiling Nemo for GPROF" - printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg" - python3 src/./Job_Creator.py -f "compile_gprof" -j "compile_gprof" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + printf -v workload1 "cd ${Nemo_path}; ./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg; cd ${Run_path};" + python3 "$dir"/src/./Job_Creator.py -f "compile_gprof" -j "compile_gprof" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" if ! state1=$("$job" --wait compile_gprof."$Jobs_scheduler"); then exit 1 @@ -251,7 +248,7 @@ Compile_gprof() mv "${Run_path}"/weights_core_orca2_bilinear_noc.nc "${Run_path}"/weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES fi - cd "${Run_path}"||(echo "Error ${Run_path} folder doesn't exists"; exit 1) + } @@ -288,44 +285,43 @@ Init() Test_arguments() { + # Does Output path exist? if ! test -d "${Output}"; then echo "Output path: ${Output} is not found, modify it in perf_metrics.config with the correct path" echo exit 1 + # Transform to absolute path + else + cd "${Output}" + Output=$(pwd) + cd "$dir" fi - Output_dir="${Output}/Output" - - if ! test -d "${Output_dir}"; then - - mkdir "${Output_dir}" - - fi - - # Nemo path correct? if ! test -d "${Nemo_path}"; then echo "Nemo relative path: ${Nemo_path} is not found, modify it in perf_metrics.config with the correct path" echo exit 1 + # Transform to absolute path + else + cd "${Nemo_path}" + Nemo_path=$(pwd) + cd "$dir" fi - - # Is Nemo data path correct? if ! test -d "${Nemo_input_data}"; then echo "Nemo input data path: ${Nemo_input_data} is not found, modify it in perf_metrics.config with the correct path" echo exit 1 + # Transform to absolute path + else + cd "${Nemo_input_data}" + Nemo_input_data=$(pwd) + cd "$dir" fi - Run_path="${Output_dir}/Run_NEMO" - # Create Run_Nemo dir after comprovations - if ! test -d "${Run_path}"; then - - mkdir "${Run_path}" - fi - cp -P "${Nemo_input_data}"/* "${Run_path}" + #Nemo_cores is array? @@ -407,24 +403,43 @@ Test_arguments() sed -i "s|export EXTRAE_CONFIG_FILE=.*|export EXTRAE_CONFIG_FILE=$dir/src/./extrae.xml|g" src/trace.sh + + #Creating output folder after comprovations + + Output_dir="${Output}/Output" + mkdir -p "${Output_dir}" + + + # Creating logs folder + logs_path="${Output_dir}/logs" - # Creating logs dir - if ! test -d "${logs_path}"; then - - mkdir "$logs_path" + mkdir -p "$logs_path" - fi + + + + # Create Run_Nemo folder + + Run_path="${Output_dir}/Run_NEMO" + mkdir -p "${Run_path}" + cp -P "${Nemo_input_data}"/* "${Run_path}" + + + # Creating Metricsfolder + + Metrics_path="${Output_dir}/Metrics" + mkdir -p "${Metrics_path}" } + + # Generates a list of Nemo functions Gprof_functions() { - cd "${Run_path}"||(echo "Error ${Run_path} folder doesn't exists"; exit 1) - #Generating function list in case of missing if ! test -f "extrae_functions.txt"; then @@ -486,6 +501,7 @@ Get_trace() Compile_extrae # Run nemo with extrae + "$dir"/src/./extraf.sh nemo extrae_functions.txt sed -i "s|list=.*|list=\"${Run_path}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" "$dir"/src/extrae.xml for core in "${Nemo_cores[@]}" @@ -515,18 +531,13 @@ Compile_extrae echo "$dir"/src/magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > "${logs_path}"/cut_"$core".out 2>&1 if ! ls nemo_"$core".best_cut.prv; then - echo "Cut failed, look at "${logs_path}"/cut_$core.out for more info." + echo "Cut failed, look at ${logs_path}/cut_$core.out for more info." echo exit 1 fi echo - # Creating folder - Metrics_path="${Output_dir}/Metrics" - - if ! test -d "${Metrics_path}"; then - mkdir "${Metrics_path}" - fi + cp nemo_"$core".best_cut.* "${Metrics_path}" @@ -538,14 +549,12 @@ Compile_extrae Create_metrics() { - - cd "$dir" || (echo "Error original dir doesn't exist"; exit 1) # Create performance metrics echo "Creating metrics and storing them in ${Metrics_path} folder" echo - python3 src/./Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./modelfactors.py -ms 100000 *" - mv analysis."$Jobs_scheduler" "${Metrics_path}" - cd "${Metrics_path}"||(echo "Error ${Metrics_path} folder doesn't exists"; exit 1) + python3 "$dir"/src/./Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./modelfactors.py -ms 100000 *" + + if ! state5=$("$job" --wait analysis."$Jobs_scheduler"); then exit 1 fi @@ -561,7 +570,7 @@ Create_metrics() mv analysis.* "${logs_path}" echo "------------------------------------------------------------------------------" echo "------------------------- Script Completed -----------------------------------" - echo "----------------------- Data in ${Metrics_path} folder -------------------------------" + echo "--- Data in ${Metrics_path} folder ---" echo "------------------------------------------------------------------------------" echo "------------------------------------------------------------------------------" echo -- GitLab From 2cd586d4ca0dde58e0a97773e7a70db445e2606f Mon Sep 17 00:00:00 2001 From: cpenadep Date: Thu, 15 Sep 2022 09:50:17 +0200 Subject: [PATCH 31/40] Now gthrottling forks the for to go quicker. --- src/gthrottling.sh | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/src/gthrottling.sh b/src/gthrottling.sh index 27c4f50..e91d785 100755 --- a/src/gthrottling.sh +++ b/src/gthrottling.sh @@ -3,27 +3,26 @@ # Usage: ./extract_gprof.sh path/to/executable/executable # Output file: extrae_functions.txt -rm gprof_functions -rm suspected_functions_names_only -rm suspected_functions -rm extrae_functions.txt # nm tool lists the symbols from objects and we select the ones with type T|t which the type is in the text section nm $1 | grep -i " T " | awk '{print $3}' > function_names.txt echo "See the function names from the binary in the file function_names.txt" dir=$(dirname $1) -for i in `ls $dir/gmon*`; -do +analyze_gmon() + + { + local n=$(echo "$2" | sed 's/[^0-9]*//g') + local i=$2 echo -e "Analyzing "$i"\n" - gprof $1 $i >gprof_temp + gprof $1 $i >gprof_temp_"$n" #We extract from each gprof file only the part about the functions, number of calls and durations, the call-paths ar enot needed - cat gprof_temp | grep -v ":" | awk 'BEGIN{k=0}{if($1=="%") {k=k+1};if(k>0 && k<2 && $1==$1+0) print $0}' > temp + cat gprof_temp_"$n" | grep -v ":" | awk 'BEGIN{k=0}{if($1=="%") {k=k+1};if(k>0 && k<2 && $1==$1+0) print $0}' > temp_"$n" #We save the name of the functions - cat temp | awk '{if($7~/^$/) print $4;else print $7}' > gprof_functions + cat temp_"$n" | awk '{if($7~/^$/) print $4;else print $7}' > gprof_functions_"$n" #From the initial list we save only the ones that gprof files include - cat function_names.txt | grep -w -f gprof_functions > extrae_new_list + cat function_names.txt | grep -w -f gprof_functions_"$n" > extrae_new_list_"$n" #We apply the throttling rule: # 1) If there is no information about each call of a function is suspected if the total duration is less than 0.1% of the total execution time @@ -31,24 +30,34 @@ do # 2.1) If its duration is less or equal to 5% ($1<=5, you can change them according to your application) of the total execution time # 2.2) If the duration of each call is less than 0.001s, then exclude it # 3) If the total execution time of this function is 0.0%, then remove it - cat temp | awk '{if($7~/^$/ && $1<0.1) print $4" "$1; else if(NF==7 && $4>10000 && (($1<=5 || $5<=0.001)) || $1==0.0) print $7" "$4}' >> suspected_functions - awk '{print $1}' suspected_functions >> suspected_functions_names_only + cat temp_"$n" | awk '{if($7~/^$/ && $1<0.1) print $4" "$1; else if(NF==7 && $4>10000 && (($1<=5 || $5<=0.001)) || $1==0.0) print $7" "$4}' >> suspected_functions_"$n" + awk '{print $1}' suspected_functions_"$n" >> suspected_functions_names_only_"$n" # Sort and remove any double functions from the list with suspected functions - cat suspected_functions_names_only | sort | uniq > temp_file - mv temp_file suspected_functions_names_only + cat suspected_functions_names_only_"$n" | sort | uniq > temp_file_"$n" + mv temp_file_"$n" suspected_functions_names_only_"$n" # Create a new fucntion list with the non suspected functions - cat extrae_new_list | grep -w -v -f suspected_functions_names_only >> extrae_functions.txt -done + cat extrae_new_list_"$n" | grep -w -v -f suspected_functions_names_only_"$n" >> extrae_functions.txt + + rm extrae_new_list_"$n" + rm suspected_functions_names_only_"$n" + rm suspected_functions_"$n" + rm gprof_temp_"$n" + rm temp_"$n" + rm gprof_functions_"$n" + +} +for i in `ls $dir/gmon*`; do analyze_gmon "$1" "$i" & done + +wait #Sort and uniw the useful functions because are called from many processors and can by include twice cat extrae_functions.txt | sort | uniq > temp2 mv temp2 extrae_functions.txt -rm temp echo -e "Input function list: "function_names.txt" "`wc -l function_names.txt | awk '{print $1}'`" functions" echo -e "New function list: extrae_functions.txt "`wc -l extrae_functions.txt | awk '{print $1}'`" functions" - +rm function_names.txt exit -- GitLab From ea4a59a7bfeb8451c25f1671c212cde2fadf1000 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Thu, 15 Sep 2022 10:55:32 +0200 Subject: [PATCH 32/40] Extra clarification --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index abf8159..f98650c 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,9 @@ Here the list of the modules that need to be loaded before the script execution. # Usage * Clone this repository wherever you please. -* Edit the file perf_metrics.config and replace the parameters values with the suited information. -* MINIMUM CHANGES perf_metrics.config : +* Inside the cloned repository use `git submodule update --init` to load the submodules. +* ***Edit the file perf_metrics.config and replace the parameters values with the suited information.*** +* ***MINIMUM CHANGES perf_metrics.config:*** * Nemo_path, change the value to the path were NEMO is installed in your machine. * Nemo_input_data, change the value to the path were the input data for the configuration is downloaded. * Compilation_arch, replace the value with the name of the arch file that you use to compile NEMO. @@ -38,4 +39,4 @@ Here the list of the modules that need to be loaded before the script execution. ``` ./perf_metrics.bash ``` -* If the script executes without problems the data will be ready in the Metrics folder. +* If the script executes without problems the data will be by default ready inside ../Output/Metrics folder. The Output dir path can be changed at perf_metrics.config. -- GitLab From e182f5aa477fdf0c65842bdd291e63ff1f9c0e9a Mon Sep 17 00:00:00 2001 From: cpenadep Date: Fri, 16 Sep 2022 10:35:11 +0200 Subject: [PATCH 33/40] Made the for to obtain traces parallel. Now submodules load at the beginning of the script --- README.md | 3 +-- perf_metrics.bash | 3 +++ src/functions.bash | 7 +++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f98650c..006aed1 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,6 @@ Here the list of the modules that need to be loaded before the script execution. # Usage * Clone this repository wherever you please. -* Inside the cloned repository use `git submodule update --init` to load the submodules. * ***Edit the file perf_metrics.config and replace the parameters values with the suited information.*** * ***MINIMUM CHANGES perf_metrics.config:*** * Nemo_path, change the value to the path were NEMO is installed in your machine. @@ -35,7 +34,7 @@ Here the list of the modules that need to be loaded before the script execution. * Compilation_arch, replace the value with the name of the arch file that you use to compile NEMO. * Modules, change the value to suit the name of the modules you need to load. * Jobs_scheduler, replace the value with the name of the scheduler installed in your machine (currently supports slurm, lsf and torque) -* Execute perf_metrics.bash +* Execute perf_metrics.bash inside the git repository ``` ./perf_metrics.bash ``` diff --git a/perf_metrics.bash b/perf_metrics.bash index b427da1..3520439 100755 --- a/perf_metrics.bash +++ b/perf_metrics.bash @@ -10,6 +10,9 @@ if [ $# -gt 0 ]; then fi + +#Load submodules +git submodule update --init #Get script directory dir=$(pwd) diff --git a/src/functions.bash b/src/functions.bash index a4a442a..03d17d5 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -507,7 +507,7 @@ Compile_extrae for core in "${Nemo_cores[@]}" do - + ( echo "Creating trace with $core cores..." echo python3 "$dir"/src/./Job_Creator.py -f "run_extrae_$core" -j "run_extrae_$core" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core "$dir"/src/./trace.sh ./nemo" @@ -536,14 +536,13 @@ Compile_extrae exit 1 fi echo - - cp nemo_"$core".best_cut.* "${Metrics_path}" - + )& done + wait } Create_metrics() -- GitLab From d711dab43daa2f17ad276cf87afe7a932ad84ed9 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Fri, 16 Sep 2022 10:47:32 +0200 Subject: [PATCH 34/40] Expanded explanation --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 006aed1..ade9efd 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Here the list of the modules that need to be loaded before the script execution. # Usage * Clone this repository wherever you please. +* Don't move the content of the repository outside , the sub-modules won't load and the script will fail. If you want you can init the sub-modules manually with `git submodule update --init` and then move the content. * ***Edit the file perf_metrics.config and replace the parameters values with the suited information.*** * ***MINIMUM CHANGES perf_metrics.config:*** * Nemo_path, change the value to the path were NEMO is installed in your machine. @@ -34,7 +35,7 @@ Here the list of the modules that need to be loaded before the script execution. * Compilation_arch, replace the value with the name of the arch file that you use to compile NEMO. * Modules, change the value to suit the name of the modules you need to load. * Jobs_scheduler, replace the value with the name of the scheduler installed in your machine (currently supports slurm, lsf and torque) -* Execute perf_metrics.bash inside the git repository +* Execute perf_metrics.bash ``` ./perf_metrics.bash ``` -- GitLab From 36743e5351ac938d8440433ef01084a53916e8f1 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Mon, 19 Sep 2022 12:01:43 +0200 Subject: [PATCH 35/40] link instead of copy, dir for gprof, deleted outputs --- perf_metrics.bash | 5 ++- src/functions.bash | 105 ++++++++++++++++++++++++++------------------- 2 files changed, 65 insertions(+), 45 deletions(-) diff --git a/perf_metrics.bash b/perf_metrics.bash index 3520439..f9dd8cb 100755 --- a/perf_metrics.bash +++ b/perf_metrics.bash @@ -36,11 +36,12 @@ Init #Test if parameters are valid Test_arguments -cd "${Run_path}"||(echo "Error ${Run_path} folder doesn't exists"; exit 1) +cd "${Gprof_path}"||(echo "Error ${Gprof_path} folder doesn't exists"; exit 1) #Create the list of important functions from NEMO -Gprof_functions +Gprof_functions & +cd "${Run_path}"||(echo "Error ${Run_path} folder doesn't exists"; exit 1) #Get the traces of the executions and cut 1 timestep Get_trace diff --git a/src/functions.bash b/src/functions.bash index 03d17d5..d5b8259 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -6,7 +6,10 @@ Job_completed() if [ "$Jobs_scheduler" == "slurm" ]; then local id1 id1=${1##* } - sleep 5 + until ! scontrol show job "$id1" | grep -q 'JobState=COMPLETING' + do + sleep 1 + done if ! scontrol show job "$id1" | grep -q 'JobState=COMPLETED'; then Completed=false else @@ -16,7 +19,6 @@ Job_completed() elif [ "$Jobs_scheduler" == "lsf" ]; then local id2 id2=$(head -n1 "$1" | cut -d'<' -f2 | cut -d'>' -f1) - sleep 5 if ! bjobs -l "$id2" | grep -q 'Status '; then Completed=false else @@ -25,7 +27,6 @@ Job_completed() elif [ "$Jobs_scheduler" == "torque" ]; then local id3 id3=$(head -n1 "$1" | awk '{ print $3 }') - sleep 5 if ! qstat f "$id3" | grep -q 'exit_status = 0'; then Completed=false else @@ -117,22 +118,23 @@ Compile_extrae() python3 "$dir"/src/./Job_Creator.py -f "compile_extrae" -j "compile_extrae" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" - if ! state1=$("$job" --wait compile_extrae."$Jobs_scheduler"); then + if ! state1=$("$job" "$wait" compile_extrae."$Jobs_scheduler"); then exit 1: fi echo Job_completed "$state1" + mv compile_extrae.* "${logs_path}" if [ $Completed == false ]; then echo "Nemo compilation failed, remember to load all the needed modules. Check the details in ${logs_path}/compile_extrae.err" echo exit 1 else - echo "Nemo compilation successful" + echo "Nemo Extrae compilation successful" echo fi - mv compile_extrae.* "${logs_path}" + else echo "Compilation not needed" @@ -142,9 +144,17 @@ Compile_extrae() #Copy all the EXP00 data but don't overwrite namelist just the executable cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* "${Run_path}" - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Run_path}" + cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Run_path}" + + + + if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then + sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' "${Run_path}"/context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) + fi + + sed -i '/|g" "$dir"/src/extrae.xml + for core in "${Nemo_cores[@]}" do @@ -512,24 +527,27 @@ Compile_extrae echo python3 "$dir"/src/./Job_Creator.py -f "run_extrae_$core" -j "run_extrae_$core" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core "$dir"/src/./trace.sh ./nemo" - if ! state4=$("$job" --wait run_extrae_$core."$Jobs_scheduler") ; then + if ! state4=$("$job" "$wait" run_extrae_$core."$Jobs_scheduler") ; then exit 1 fi Job_completed "$state4" + mv run_extrae_"$core".* "${logs_path}" + if [ $Completed == false ]; then echo "Nemo execution failed, no traces files generated more info inside ${logs_path}/run_extrae_$core.err" echo exit 1 fi - mv run_extrae_"$core".* "${logs_path}" - + mv nemo.prv nemo_"$core".prv mv nemo.pcf nemo_"$core".pcf mv nemo.row nemo_"$core".row echo "Cutting up best iteration" echo "$dir"/src/magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > "${logs_path}"/cut_"$core".out 2>&1 + + if ! ls nemo_"$core".best_cut.prv; then echo "Cut failed, look at ${logs_path}/cut_$core.out for more info." echo @@ -554,19 +572,20 @@ Create_metrics() python3 "$dir"/src/./Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "$dir/src/./modelfactors.py -ms 100000 *" - if ! state5=$("$job" --wait analysis."$Jobs_scheduler"); then + if ! state5=$("$job" "$wait" analysis."$Jobs_scheduler"); then exit 1 fi Job_completed "$state5" + cp analysis.out overview.txt + mv analysis.* "${logs_path}" if [ $Completed == false ]; then echo "Error, metrics have not generated check ${logs_path}/analysis.err to get more details" echo exit 1 fi - cp analysis.out overview.txt - mv analysis.* "${logs_path}" + echo "------------------------------------------------------------------------------" echo "------------------------- Script Completed -----------------------------------" echo "--- Data in ${Metrics_path} folder ---" -- GitLab From fe9888e1327305bbb5f2e95b1d40c54880009d37 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 20 Sep 2022 12:12:49 +0200 Subject: [PATCH 36/40] Now dir for every execution are created, run files are deleted after finishing --- perf_metrics.config | 12 ++++-- src/functions.bash | 90 ++++++++++++++++++++++----------------------- 2 files changed, 52 insertions(+), 50 deletions(-) diff --git a/perf_metrics.config b/perf_metrics.config index 16fa533..59ee9cc 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -27,21 +27,25 @@ Jobs_n_cores=4 Jobs_cores_per_node= Jobs_scheduler="slurm" Jobs_time=60 -Jobs_queue="debug" +Jobs_queue= -# Compilation_compile (string): When false only compiles NEMO if arch file lacks the needed flags, when true always compiles NEMO. +# Compilation_compile (boolean): When false only compiles NEMO if arch file lacks the needed flags, when true always compiles NEMO. # Compilation_ref (string): Reference configuration. # Compilation_arch (string): Architecture used (without the -arch suffix and the .fcm). # Compilation_name (string): Name of the new configuration (Important to not be an existing one). # Compilation_sub (string): Add or remove sub-components. -Compilation_compile="false" +Compilation_compile=false Compilation_ref="ORCA2_ICE_PISCES" Compilation_arch="YOUR_ARCH_FILE" Compilation_name="ORCA2_EXTRAE" Compilation_sub="OCE del_key 'key_si3 key_top'" - + +# Clean (boolean): If true, at the end of the script, all residual files from NEMO executions (data, outputs, executable, folders) are deleted. + +Clean=true + # Modules (string): List of modules loaded. # Required: # - Perl interpreter diff --git a/src/functions.bash b/src/functions.bash index d5b8259..e7a1c47 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -141,19 +141,7 @@ Compile_extrae() echo fi - #Copy all the EXP00 data but don't overwrite namelist just the executable - - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* "${Run_path}" - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Run_path}" - - - - if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then - sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' "${Run_path}"/context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) - fi - - sed -i '/ /dev/null sed -i "s|list=.*|list=\"${Run_path}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" "$dir"/src/extrae.xml + # Change iterations + sed -i "s|nn_itend * =.*|nn_itend = $Nemo_iterations ! last time step (std 5475)|g" "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/namelist_cfg + if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then + sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) + fi + sed -i '/ "${logs_path}"/cut_"$core".out 2>&1 - if ! ls nemo_"$core".best_cut.prv; then echo "Cut failed, look at ${logs_path}/cut_$core.out for more info." echo @@ -555,9 +545,9 @@ Get_trace() fi echo cp nemo_"$core".best_cut.* "${Metrics_path}" + cd "$Run_path" )& - - + done wait @@ -586,6 +576,14 @@ Create_metrics() fi + # Removing run folders + if [ $Clean == true ]; then + rm -r -f "$Run_path" + mv "${Gprof_path}"/extrae_functions* "$dir" + rm -r -f "$Gprof_path"/* + mv "${dir}"/extrae_functions* "${Gprof_path}" + fi + echo "------------------------------------------------------------------------------" echo "------------------------- Script Completed -----------------------------------" echo "--- Data in ${Metrics_path} folder ---" -- GitLab From 5e96722bb6a86e9e803ad378db064f8a55c21844 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 20 Sep 2022 12:33:39 +0200 Subject: [PATCH 37/40] Fixed naming mistake --- src/functions.bash | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/functions.bash b/src/functions.bash index e7a1c47..f21ba9e 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -427,7 +427,7 @@ Gprof_functions() { #Generating function list in case of missing - if ! test -f "extrae_functions_${cfg}.txt"; then + if ! test -f "extrae_functions_${name_cfg}.txt"; then Compile_gprof #Changing iterations, big traces generate problems. @@ -466,14 +466,14 @@ Gprof_functions() echo exit 1 else - mv extrae_functions.txt extrae_functions_"${cfg}".txt + mv extrae_functions.txt extrae_functions_"${name_cfg}".txt echo "Functions listed correctly" echo fi else - echo "Functions already listed, file ${Gprof_path}/extrae_functions_${cfg}.txt does exist." + echo "Functions already listed, file ${Gprof_path}/extrae_functions_${name_cfg}.txt does exist." echo fi @@ -488,7 +488,7 @@ Get_trace() Compile_extrae wait - "$dir"/src/./extraf.sh "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Gprof_path}"/extrae_functions_"${cfg}".txt > /dev/null + "$dir"/src/./extraf.sh "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo "${Gprof_path}"/extrae_functions_"${name_cfg}".txt > /dev/null sed -i "s|list=.*|list=\"${Run_path}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" "$dir"/src/extrae.xml # Change iterations -- GitLab From 44cb3e215ddd61c403810f70da90144d1abad834 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 20 Sep 2022 14:41:28 +0200 Subject: [PATCH 38/40] Now exit terminates all the processes --- perf_metrics.bash | 2 ++ src/functions.bash | 14 ++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/perf_metrics.bash b/perf_metrics.bash index f9dd8cb..9470d4a 100755 --- a/perf_metrics.bash +++ b/perf_metrics.bash @@ -52,3 +52,5 @@ Create_metrics } main "$@"; exit + + diff --git a/src/functions.bash b/src/functions.bash index f21ba9e..95792cc 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -1,3 +1,5 @@ + + # Functions #Checks if the job submission ended correctly. @@ -41,7 +43,8 @@ Job_completed() Compile_extrae() { - + trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT + #Get flag lines line=$(sed -n '/^%FCFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) @@ -149,6 +152,7 @@ Compile_extrae() Compile_gprof() { + trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT if [ "$compile" == true ]; then echo 'compile parameter is inicialized true' @@ -425,7 +429,7 @@ Test_arguments() Gprof_functions() { - + trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT #Generating function list in case of missing if ! test -f "extrae_functions_${name_cfg}.txt"; then @@ -484,7 +488,7 @@ Gprof_functions() Get_trace() { - + trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT Compile_extrae wait @@ -504,6 +508,7 @@ Get_trace() do ( + trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT # Create folder for n cores mkdir -p "$core"_cores @@ -528,19 +533,16 @@ Get_trace() if [ $Completed == false ]; then echo "Nemo execution failed, no traces files generated more info inside ${logs_path}/run_extrae_$core.err" - echo exit 1 fi mv nemo.prv nemo_"$core".prv mv nemo.pcf nemo_"$core".pcf mv nemo.row nemo_"$core".row echo "Cutting up best iteration" - echo "$dir"/src/magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > "${logs_path}"/cut_"$core".out 2>&1 if ! ls nemo_"$core".best_cut.prv; then echo "Cut failed, look at ${logs_path}/cut_$core.out for more info." - echo exit 1 fi echo -- GitLab From bdf053d16c451b5cfdb1656291dd797eadc1a45c Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 20 Sep 2022 15:34:24 +0200 Subject: [PATCH 39/40] Fixed mistake where exits loops instead script --- src/functions.bash | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/functions.bash b/src/functions.bash index 95792cc..996ec0d 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -508,8 +508,7 @@ Get_trace() do ( - trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT - + # Create folder for n cores mkdir -p "$core"_cores cd "$core"_cores @@ -526,6 +525,7 @@ Get_trace() python3 "$dir"/src/./Job_Creator.py -f "run_extrae_$core" -j "run_extrae_$core" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core "$dir"/src/./trace.sh ./nemo" if ! state4=$("$job" "$wait" run_extrae_$core."$Jobs_scheduler") ; then + kill -- -$$ exit 1 fi Job_completed "$state4" @@ -533,6 +533,7 @@ Get_trace() if [ $Completed == false ]; then echo "Nemo execution failed, no traces files generated more info inside ${logs_path}/run_extrae_$core.err" + kill -- -$$ exit 1 fi mv nemo.prv nemo_"$core".prv @@ -543,6 +544,7 @@ Get_trace() if ! ls nemo_"$core".best_cut.prv; then echo "Cut failed, look at ${logs_path}/cut_$core.out for more info." + kill -- -$$ exit 1 fi echo -- GitLab From c2b6653f1107df9beee45198b154f595472262ae Mon Sep 17 00:00:00 2001 From: cpenadep Date: Tue, 20 Sep 2022 15:35:21 +0200 Subject: [PATCH 40/40] Fixed mistake exiting script --- src/functions.bash | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/functions.bash b/src/functions.bash index 996ec0d..fd3de97 100644 --- a/src/functions.bash +++ b/src/functions.bash @@ -525,7 +525,7 @@ Get_trace() python3 "$dir"/src/./Job_Creator.py -f "run_extrae_$core" -j "run_extrae_$core" --set-core "$core" --set-core-per-node "$Jobs_cores_per_node" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "mpirun -np $core "$dir"/src/./trace.sh ./nemo" if ! state4=$("$job" "$wait" run_extrae_$core."$Jobs_scheduler") ; then - kill -- -$$ + kill 0 exit 1 fi Job_completed "$state4" @@ -533,7 +533,7 @@ Get_trace() if [ $Completed == false ]; then echo "Nemo execution failed, no traces files generated more info inside ${logs_path}/run_extrae_$core.err" - kill -- -$$ + kill 0 exit 1 fi mv nemo.prv nemo_"$core".prv @@ -544,7 +544,7 @@ Get_trace() if ! ls nemo_"$core".best_cut.prv; then echo "Cut failed, look at ${logs_path}/cut_$core.out for more info." - kill -- -$$ + kill 0 exit 1 fi echo -- GitLab