From fbda91c3a47c8b58da7775c09dc73417b4c8e0fb Mon Sep 17 00:00:00 2001 From: cpenadep Date: Wed, 7 Sep 2022 10:31:22 +0200 Subject: [PATCH 1/8] Two compilation version --- perf_metrics.bash | 229 +++++++++++++++++++++++++++++++------------- perf_metrics.config | 10 +- 2 files changed, 166 insertions(+), 73 deletions(-) diff --git a/perf_metrics.bash b/perf_metrics.bash index e758f18..eed3e80 100644 --- a/perf_metrics.bash +++ b/perf_metrics.bash @@ -22,7 +22,8 @@ echo Init Test_arguments -Test_Comp +Gprof_functions +Get_trace Create_metrics } @@ -168,88 +169,65 @@ Test_arguments() } - -Test_Comp() +Compile_Gprof() { # Checking if compilation is needed - if [ "$compile" == true ]; then - echo 'compile parameter is inicialized true' - fi - - # Get text lines corresponding to the flags - line=$(sed -n '/^%FCFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) - line2=$(sed -n '/^%FPPFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) - line3=$(sed -n '/^%LDFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) - - # If -g is not there, recompilation is requiered and -g added - if ! echo "${line}"|grep -q "\-g\b"; then - echo "-g flag not found in arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%FCFLAGS/ s/$/ -g /' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile=true + echo 'compile parameter is inicialized true' fi - # If finstrument-functions is not there recompilation is requiered and -finstrument-functions added - if ! echo "${line}"|grep -q "\-finstrument-functions\b"; then - echo "-finstrument-functions flag not found in arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%FCFLAGS/ s/$/ -finstrument-functions/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile=true + # Checking if Gprof_arch file is present + if ! test -f "${Nemo_path}/arch/arch-${arch}_GPROF.fcm"; then + cp "${Nemo_path}"/arch/arch-"${arch}".fcm "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm fi - # If nemo executable is not on the run file compile + # Get text lines corresponding to the flags + line=$(sed -n '/^%FCFLAGS /p' "$Nemo_path"/arch/arch-"${arch}"_GPROF.fcm) + line2=$(sed -n '/^%FPPFLAGS /p' "$Nemo_path"/arch/arch-"${arch}"_GPROF.fcm) + line3=$(sed -n '/^%LDFLAGS /p' "$Nemo_path"/arch/arch-"${arch}"_GPROF.fcm) - if ! test -f "${Nemo_path}/cfgs/${name_cfg}/EXP00/nemo"; then - echo "nemo executable not found in cfg" - compile=true + # If -g is not there, recompilation is requiered and -g added + if ! echo "${line}"|grep -q "\-g\b"; then + echo "-g flag not found in arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " + sed -i '/^%FCFLAGS/ s/$/ -g /' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm + compile_gprof=true fi - + # If -pg is not there recompilation is requiered and -pg added if ! echo "${line}"|grep -q "\-pg\b"; then - echo "-pg flag not found in FCFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%FCFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile=true + echo "-pg flag not found in FCFLAGS arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " + sed -i '/^%FCFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm + compile_gprof=true fi if ! echo "${line2}"|grep -q "\-pg\b"; then - echo "-pg flag not found in FPPFLAGS arch-${arch}.fcm : editing arch-${arch}.fcm " - sed -i '/^%FPPFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile=true + echo "-pg flag not found in FPPFLAGS arch-${arch}_GPROF.fcm : editing arch-${arch}_GPROF.fcm " + sed -i '/^%FPPFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm + compile_gprof=true fi if ! echo "${line3}"|grep -q "\-pg\b"; then - echo "-pg flag not found in LDFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%LDFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile=true + echo "-pg flag not found in LDFLAGS arch-${arch}_GPROF.fcm: editing arch-${arch}_GPROF.fcm " + sed -i '/^%LDFLAGS/ s/$/ -pg/' "${Nemo_path}"/arch/arch-"${arch}"_GPROF.fcm + compile_gprof=true fi + # If nemo executable is not on the run file compile - # If -rdynamic is not there recompilation is requiered and -rdynamic added - - if ! echo "${line}"|grep -q "\-rdynamic\b"; then - echo "-rdynamic flag not found in FCFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%FCFLAGS/ s/$/ -rdynamic/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile=true + if ! test -f "${Nemo_path}/cfgs/${name_cfg}_GPROF/EXP00/nemo"; then + echo "nemo executable not found in ${name_cfg}_GPROF" + compile_gprof=true fi - if ! echo "${line3}"|grep -q "\-export-dynamic\b"; then - echo "-export-dynamic flag not found in LDFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%LDFLAGS/ s/$/ -export-dynamic/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile=true - fi - - - - #Compile the program if needed - - if [ $compile == true ]; then - echo "Compiling Nemo, expected duration 35m" + if [ $compile == true ] || [ $compile_gprof == true ]; then + echo "Compiling Nemo for GPROF, expected duration 35m" echo "Output of the compilation in compile.err and compile.out" - printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" + printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg" python3 Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" - + state1=$("$job" --wait compile."$Jobs_scheduler") echo Job_completed "$state1" @@ -267,11 +245,10 @@ Test_Comp() echo fi - #Copy all the EXP00 data but don't overwrite namelist just the executable cd "$dir" || echo "Error original dir doesn't exist" exit - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo . + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/* . + cp "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/nemo . if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) @@ -286,20 +263,22 @@ Test_Comp() if test -f "weights_core_orca2_bilinear_noc.nc"; then mv weights_core_orca2_bilinear_noc.nc weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES fi - } - -Create_metrics() +Gprof_functions() { - - #Changing iterations, big traces generate problems. - sed -i "s|nn_itend * =.*|nn_itend = $Nemo_iterations ! last time step (std 5475)|g" namelist_cfg - #Generating function list in case of missing + if ! test -f "extrae_functions_for_xml.txt"; then + + Compile_gprof + + + #Changing iterations, big traces generate problems. + sed -i "s|nn_itend * =.*|nn_itend = $Nemo_iterations ! last time step (std 5475)|g" namelist_cfg + rm gmon* 2> /dev/null echo "Runing Nemo with 2 cores to obtain function data..." echo @@ -331,15 +310,123 @@ Create_metrics() fi ./extraf.sh nemo extrae_functions.txt + sed -i "s|list=.*|list=\"${dir}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" extrae.xml else echo "Functions already listed, file extrae_functions_for_xml.txt does exist" echo + + fi + +} + +Compile_extrae() +{ + + # Check if compilation is needed for coupling nemo with extrae + + line=$(sed -n '/^%FCFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) + line2=$(sed -n '/^%FPPFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) + line3=$(sed -n '/^%LDFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) + + + # If -rdynamic is not there recompilation is requiered and -rdynamic added + + if ! echo "${line}"|grep -q "\-rdynamic\b"; then + echo "-rdynamic flag not found in FCFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " + sed -i '/^%FCFLAGS/ s/$/ -rdynamic/' "${Nemo_path}"/arch/arch-"${arch}".fcm + compile=true + fi + + if ! echo "${line3}"|grep -q "\-export-dynamic\b"; then + echo "-export-dynamic flag not found in LDFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " + sed -i '/^%LDFLAGS/ s/$/ -export-dynamic/' "${Nemo_path}"/arch/arch-"${arch}".fcm + compile=true + fi + + # If finstrument-functions is not there recompilation is requiered and -finstrument-functions added + if ! echo "${line}"|grep -q "\-finstrument-functions\b"; then + echo "-finstrument-functions flag not found in arch-${arch}.fcm: editing arch-${arch}.fcm " + sed -i '/^%FCFLAGS/ s/$/ -finstrument-functions/' "${Nemo_path}"/arch/arch-"${arch}".fcm + compile=true fi - sed -i "s|list=.*|list=\"${dir}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" extrae.xml + # If -g is not there, recompilation is requiered and -g added + if ! echo "${line}"|grep -q "\-g\b"; then + echo "-g flag not found in arch-${arch}.fcm: editing arch-${arch}.fcm " + sed -i '/^%FCFLAGS/ s/$/ -g /' "${Nemo_path}"/arch/arch-"${arch}".fcm + compile_gprof=true + fi + - # Run nemo with extrae + if [ "$compile" == true ]; then + echo 'compile parameter is inicialized true' + fi + + # If -pg is there recompilation is requiered and -pg removed + + sed -i 's/-pg//g' "${Nemo_path}"/arch/arch-"${arch}".fcm + + if echo "${line}"|grep -q "\-pg\b"; then + echo "-pg flag found in FCFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " + compile=true + fi + + if echo "${line2}"|grep -q "\-pg\b"; then + echo "-pg flag found in FPPFLAGS arch-${arch}.fcm : editing arch-${arch}.fcm " + compile=true + fi + + if echo "${line3}"|grep -q "\-pg\b"; then + echo "-pg flag found in LDFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " + compile=true + fi + + # If nemo executable is not on the run file compile + + if ! test -f "${Nemo_path}/cfgs/${name_cfg}/EXP00/nemo"; then + echo "nemo executable not found in ${name_cfg}" + compile=true + fi + + + if [ $compile == true ]; then + + echo "Compiling Nemo for EXTRAE, expected duration 35m" + echo "Output of the compilation in compile.err and compile.out" + + printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" + python3 Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + + state1=$("$job" --wait compile."$Jobs_scheduler") + echo + Job_completed "$state1" + if [ $Completed == false ]; then + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in compile.err" + echo + exit 1 + else + echo "Nemo compilation successful" + echo + fi + + else + echo "Compilation not needed" + echo + fi + + #Copy all the EXP00 data but don't overwrite namelist just the executable + cd "$dir" || echo "Error original dir doesn't exist" exit + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . + cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo . + +} +Get_trace() +{ + +Compile_extrae + +# Run nemo with extrae for core in "${Nemo_cores[@]}" @@ -375,6 +462,12 @@ Create_metrics() cp nemo_"$core".best_cut.* Metrics done + +} + +Create_metrics() +{ + # Create performance metrics diff --git a/perf_metrics.config b/perf_metrics.config index 5bcf07a..d24f869 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -8,16 +8,16 @@ # get Nemo traces with 4 and 48 cores. 2 different nºcores are needed to obtain scalability data. Nemo_path="../NEMO" -Nemo_cores=( 4 24 48 ) +Nemo_cores=( 4 24 48 96) -# Jobs_n_cores: nºcores used for executing other scripts. +# Jobs_n_cores: nºcores used for other jobs like compiling nemo. # Jobs_scheduler: Available (slurm/lsf). # Jobs_time: Max duration of the job in min. # Jobs_queue: Queue used. -Jobs_n_cores=4 +Jobs_n_cores=96 Jobs_scheduler="slurm" -Jobs_time="60" +Jobs_time=60 Jobs_queue=debug # Compilation_compile: When false only compiles NEMO if arch file lacks the needed flags, when true always compiles NEMO. @@ -30,7 +30,7 @@ Jobs_queue=debug Compilation_compile="false" Compilation_ref="ORCA2_ICE_PISCES" Compilation_arch="X64_MN4" -Compilation_name="ORCA2_EXTRAE" +Compilation_name="ORCA2" Compilation_sub="OCE del_key 'key_si3 key_top'" # List of modules loaded. -- GitLab From bc5201a4671fd612b85966d2eabad25980e68cdb Mon Sep 17 00:00:00 2001 From: cpenadep Date: Thu, 8 Sep 2022 12:43:09 +0200 Subject: [PATCH 2/8] Changed to EXTRAE 3.7.1 --- extraf.sh | 2 +- perf_metrics.bash | 469 +++++++++++++++++++++++--------------------- perf_metrics.config | 8 +- 3 files changed, 249 insertions(+), 230 deletions(-) diff --git a/extraf.sh b/extraf.sh index d77f02a..6508c17 100755 --- a/extraf.sh +++ b/extraf.sh @@ -7,7 +7,7 @@ # List all the objects from the binary, select only the ones from the functions list and create a function file as Extrae demands -nm $1 | grep -i " T " | grep -w -f $2 | awk '{print $3" # "$1}' > extrae_functions_for_xml.txt +nm $1 | grep -i " T " | grep -w -f $2 | awk '{print $1" # "$3}' > extrae_functions_for_xml.txt echo "See the function names in the file extrae_functions_for_xml.txt" diff --git a/perf_metrics.bash b/perf_metrics.bash index eed3e80..f90289c 100644 --- a/perf_metrics.bash +++ b/perf_metrics.bash @@ -1,34 +1,8 @@ #!/bin/bash -main() -{ - -if [ $# -gt 0 ]; then - - echo "This script does not accept arguments, parameters need to be added to perf_metrics.config Aborting" - exit 1 - -fi - -#Get script directory -dir=$(pwd) - -echo -echo "Using the following configuration:" -echo -source "$dir"/perf_metrics.config -grep -o '^[^#]*' perf_metrics.config -echo - -Init -Test_arguments -Gprof_functions -Get_trace -Create_metrics - -} - +# Functions +#Checks if the job submission ended correctly. Job_completed() { if [ "$Jobs_scheduler" == "slurm" ]; then @@ -61,118 +35,114 @@ Job_completed() } -Init() +# Check if nemo is compiled for using extrae + +Compile_extrae() { - #Init variables with default values in case of missing - Nemo_path="${Nemo_path:-"."}" - Nemo_cores="${Nemo_cores:-( 48 )}" - Jobs_n_cores="${Jobs_n_cores:-48}" - Jobs_scheduler="${Jobs_scheduler:-"slurm"}" - time="${Jobs_time:-0}" - queue="${Jobs_queue:-""}" - compile="${Compilation_compile:-"false"}" - cfg="${Compilation_ref:-"ORCA2_ICE_PISCES"}" - arch="${Compilation_arch:-"X64_MN4"}" - name_cfg="${Compilation_name:-"ORCA2_EXTRAE"}" - comp_cfg="${Compilation_sub:-""}" - Modules="${Modules:-""}" - Nemo_iterations=12 -} + #Get flag lines + line=$(sed -n '/^%FCFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) + line2=$(sed -n '/^%FPPFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) + line3=$(sed -n '/^%LDFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) -# Checks if the paths given are correct. -Test_arguments() -{ - # Nemo path correct? - if ! test -d "${Nemo_path}"; then - echo "Nemo relative path: ${Nemo_path} is not found" - echo - exit 1 + # If -rdynamic is not there recompilation is requiered and -rdynamic added + + if ! echo "${line}"|grep -q "\-rdynamic\b"; then + echo "-rdynamic flag not found in FCFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " + sed -i '/^%FCFLAGS/ s/$/ -rdynamic/' "${Nemo_path}"/arch/arch-"${arch}".fcm + compile_ext=true fi - #Nemo_cores is array? - if [[ ! "$(declare -p Nemo_cores)" =~ "declare -a" ]]; then - echo "Error Nemo_cores has to be a bash array like ( 4 24 48 )" - echo - exit 1 + + if ! echo "${line3}"|grep -q "\-export-dynamic\b"; then + echo "-export-dynamic flag not found in LDFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " + sed -i '/^%LDFLAGS/ s/$/ -export-dynamic/' "${Nemo_path}"/arch/arch-"${arch}".fcm + compile_ext=true fi - - #Nemo_cores contains ints? - - re='^[0-9]+$' - for core in "${Nemo_cores[@]}" - do + # If finstrument-functions is not there recompilation is requiered and -finstrument-functions added + if ! echo "${line}"|grep -q "\-finstrument-functions\b"; then + echo "-finstrument-functions flag not found in arch-${arch}.fcm: editing arch-${arch}.fcm " + sed -i '/^%FCFLAGS/ s/$/ -finstrument-functions/' "${Nemo_path}"/arch/arch-"${arch}".fcm + compile_ext=true + fi - if ! [[ $core =~ $re ]] ; then - echo "Error Nemo_cores has to contain integer values" - echo - exit 1 + # If -g is not there, recompilation is requiered and -g added + if ! echo "${line}"|grep -q "\-g\b"; then + echo "-g flag not found in arch-${arch}.fcm: editing arch-${arch}.fcm " + sed -i '/^%FCFLAGS/ s/$/ -g /' "${Nemo_path}"/arch/arch-"${arch}".fcm + compile_ext=true fi - done - # cfg exists? - if ! test -d "${Nemo_path}/cfgs/${cfg}"; then - echo "configuration: ${cfg} doesn't exists in ${Nemo_path}/cfgs dir" - echo - exit 1 + + if [ "$compile" == true ]; then + echo 'compile parameter is inicialized true' fi - # arch exists? - if ! test -f "${Nemo_path}/arch/arch-${arch}.fcm"; then - echo "architecture: arch-${arch}.fcm doesn't exists in ${Nemo_path}/arch dir" - echo - exit 1 + + # If -pg is there recompilation is requiered and -pg removed + + sed -i 's/-pg//g' "${Nemo_path}"/arch/arch-"${arch}".fcm + + if echo "${line}"|grep -q "\-pg\b"; then + echo "-pg flag found in FCFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " + compile_ext=true fi - # scheduler correct? - if [ "$Jobs_scheduler" != "slurm" ] && [ "$Jobs_scheduler" != "lsf" ] && [ "$Jobs_scheduler" != "torque" ]; then - echo "$Jobs_scheduler is not a valid scheduler" - echo - exit 1 + + if echo "${line2}"|grep -q "\-pg\b"; then + echo "-pg flag found in FPPFLAGS arch-${arch}.fcm : editing arch-${arch}.fcm " + compile_ext=true fi - #Nemo_cores is array? - if [[ ! "$(declare -p Nemo_cores)" =~ "declare -a" ]]; then - echo "Error, variable Nemo_cores has to be an array" - echo - exit 1 + + if echo "${line3}"|grep -q "\-pg\b"; then + echo "-pg flag found in LDFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " + compile_ext=true fi - #Modules available - if ! module load $Modules;then - echo "Error loading modules aborting" - echo - exit 1 + + # If nemo executable is not on the run file compile + + if ! test -f "${Nemo_path}/cfgs/${name_cfg}/EXP00/nemo"; then + echo "nemo executable not found in ${name_cfg}" + compile_ext=true fi - echo - #$EXTRAE_HOME loaded ? - if ! test -d "${EXTRAE_HOME}"; then - echo "Extrae relative path: ${EXTRAE_HOME} is not found" + + + if [ $compile == true ] || [ $compile_ext == true ]; then + + echo "Compiling Nemo for EXTRAE" + echo "Output of the compilation in compile.err and compile.out" + + printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" + python3 Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" + + state1=$("$job" --wait compile."$Jobs_scheduler") echo - exit 1 + Job_completed "$state1" + if [ $Completed == false ]; then + echo "Nemo compilation failed, remember to load all the needed modules. Check the details in compile.err" + echo + exit 1 + else + echo "Nemo compilation successful" + echo + fi + else - sed -i 's|home=.*|home="'"$EXTRAE_HOME"'"|g' extrae.xml + echo "Compilation not needed" + echo fi - # Adding -d to variable if not empty - if [ -n "$comp_cfg" ]; then - comp_cfg="-d $comp_cfg" - fi - - # Creating auxiliar vars for submiting jobs - if [ "$Jobs_scheduler" == "slurm" ]; then - job="sbatch" - elif [ "$Jobs_scheduler" == "lsf" ]; then - job="bsub" - elif [ "$Jobs_scheduler" == "torque" ]; then - job="qsub" - fi + #Copy all the EXP00 data but don't overwrite namelist just the executable + cd "$dir" || echo "Error original dir doesn't exist" exit + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . + cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo . - } +#Check if Nemo is compiled for using Gprof -Compile_Gprof() +Compile_gprof() { - # Checking if compilation is needed if [ "$compile" == true ]; then echo 'compile parameter is inicialized true' fi @@ -222,7 +192,7 @@ Compile_Gprof() fi if [ $compile == true ] || [ $compile_gprof == true ]; then - echo "Compiling Nemo for GPROF, expected duration 35m" + echo "Compiling Nemo for GPROF" echo "Output of the compilation in compile.err and compile.out" printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg}_GPROF -m ${arch}_GPROF -j$Jobs_n_cores $comp_cfg" @@ -249,6 +219,7 @@ Compile_Gprof() cd "$dir" || echo "Error original dir doesn't exist" exit cp -n "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/* . cp "${Nemo_path}"/cfgs/"${name_cfg}"_GPROF/EXP00/nemo . + if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) @@ -265,13 +236,129 @@ Compile_Gprof() fi } -Gprof_functions() + + + + +#Init variables with default values in case of missing + +Init() { + Nemo_path="${Nemo_path:-"."}" + Nemo_cores="${Nemo_cores:-( 48 )}" + Jobs_n_cores="${Jobs_n_cores:-48}" + Jobs_scheduler="${Jobs_scheduler:-"slurm"}" + time="${Jobs_time:-0}" + queue="${Jobs_queue:-""}" + compile="${Compilation_compile:-"false"}" + cfg="${Compilation_ref:-"ORCA2_ICE_PISCES"}" + arch="${Compilation_arch:-"X64_MN4"}" + name_cfg="${Compilation_name:-"ORCA2_EXTRAE"}" + comp_cfg="${Compilation_sub:-""}" + Modules="${Modules:-""}" + Nemo_iterations=12 + compile_ext=false + compile_gprof=false +} - #Generating function list in case of missing + +# Checks if the paths given are correct. + +Test_arguments() +{ + # Nemo path correct? + if ! test -d "${Nemo_path}"; then + echo "Nemo relative path: ${Nemo_path} is not found" + echo + exit 1 + fi + #Nemo_cores is array? + if [[ ! "$(declare -p Nemo_cores)" =~ "declare -a" ]]; then + echo "Error Nemo_cores has to be a bash array like ( 4 24 48 )" + echo + exit 1 + fi + + #Nemo_cores contains ints? + + re='^[0-9]+$' + for core in "${Nemo_cores[@]}" + do - if ! test -f "extrae_functions_for_xml.txt"; then + if ! [[ $core =~ $re ]] ; then + echo "Error Nemo_cores has to contain integer values" + echo + exit 1 + fi + done + + # cfg exists? + if ! test -d "${Nemo_path}/cfgs/${cfg}"; then + echo "configuration: ${cfg} doesn't exists in ${Nemo_path}/cfgs dir" + echo + exit 1 + fi + # arch exists? + if ! test -f "${Nemo_path}/arch/arch-${arch}.fcm"; then + echo "architecture: arch-${arch}.fcm doesn't exists in ${Nemo_path}/arch dir" + echo + exit 1 + fi + # scheduler correct? + if [ "$Jobs_scheduler" != "slurm" ] && [ "$Jobs_scheduler" != "lsf" ] && [ "$Jobs_scheduler" != "torque" ]; then + echo "$Jobs_scheduler is not a valid scheduler" + echo + exit 1 + fi + #Nemo_cores is array? + if [[ ! "$(declare -p Nemo_cores)" =~ "declare -a" ]]; then + echo "Error, variable Nemo_cores has to be an array" + echo + exit 1 + fi + #Modules available + if ! module load $Modules;then + echo "Error loading modules aborting" + echo + exit 1 + fi + echo + #$EXTRAE_HOME loaded ? + if ! test -d "${EXTRAE_HOME}"; then + echo "Extrae relative path: ${EXTRAE_HOME} is not found" + echo + exit 1 + else + sed -i 's|home=.*|home="'"$EXTRAE_HOME"'"|g' extrae.xml + fi + + # Adding -d to variable if not empty + if [ -n "$comp_cfg" ]; then + comp_cfg="-d $comp_cfg" + fi + + # Creating auxiliar vars for submiting jobs + if [ "$Jobs_scheduler" == "slurm" ]; then + job="sbatch" + elif [ "$Jobs_scheduler" == "lsf" ]; then + job="bsub" + elif [ "$Jobs_scheduler" == "torque" ]; then + job="qsub" + fi + + +} + + +# Generates a list of Nemo functions + +Gprof_functions() +{ + + + #Generating function list in case of missing + if ! test -f "extrae_functions.txt"; then Compile_gprof @@ -309,125 +396,28 @@ Gprof_functions() echo fi - ./extraf.sh nemo extrae_functions.txt - sed -i "s|list=.*|list=\"${dir}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" extrae.xml + + else - echo "Functions already listed, file extrae_functions_for_xml.txt does exist" + echo "Functions already listed, file extrae_functions.txt does exist" echo fi } -Compile_extrae() -{ - - # Check if compilation is needed for coupling nemo with extrae - - line=$(sed -n '/^%FCFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) - line2=$(sed -n '/^%FPPFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) - line3=$(sed -n '/^%LDFLAGS /p' "$Nemo_path"/arch/arch-"${arch}".fcm) - - - # If -rdynamic is not there recompilation is requiered and -rdynamic added - - if ! echo "${line}"|grep -q "\-rdynamic\b"; then - echo "-rdynamic flag not found in FCFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%FCFLAGS/ s/$/ -rdynamic/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile=true - fi - - if ! echo "${line3}"|grep -q "\-export-dynamic\b"; then - echo "-export-dynamic flag not found in LDFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%LDFLAGS/ s/$/ -export-dynamic/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile=true - fi - - # If finstrument-functions is not there recompilation is requiered and -finstrument-functions added - if ! echo "${line}"|grep -q "\-finstrument-functions\b"; then - echo "-finstrument-functions flag not found in arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%FCFLAGS/ s/$/ -finstrument-functions/' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile=true - fi - # If -g is not there, recompilation is requiered and -g added - if ! echo "${line}"|grep -q "\-g\b"; then - echo "-g flag not found in arch-${arch}.fcm: editing arch-${arch}.fcm " - sed -i '/^%FCFLAGS/ s/$/ -g /' "${Nemo_path}"/arch/arch-"${arch}".fcm - compile_gprof=true - fi - - - if [ "$compile" == true ]; then - echo 'compile parameter is inicialized true' - fi - - # If -pg is there recompilation is requiered and -pg removed - - sed -i 's/-pg//g' "${Nemo_path}"/arch/arch-"${arch}".fcm - - if echo "${line}"|grep -q "\-pg\b"; then - echo "-pg flag found in FCFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " - compile=true - fi +#Gets a trace from Nemo and cuts it to obtain a single timestep - if echo "${line2}"|grep -q "\-pg\b"; then - echo "-pg flag found in FPPFLAGS arch-${arch}.fcm : editing arch-${arch}.fcm " - compile=true - fi - - if echo "${line3}"|grep -q "\-pg\b"; then - echo "-pg flag found in LDFLAGS arch-${arch}.fcm: editing arch-${arch}.fcm " - compile=true - fi - - # If nemo executable is not on the run file compile - - if ! test -f "${Nemo_path}/cfgs/${name_cfg}/EXP00/nemo"; then - echo "nemo executable not found in ${name_cfg}" - compile=true - fi - - - if [ $compile == true ]; then - - echo "Compiling Nemo for EXTRAE, expected duration 35m" - echo "Output of the compilation in compile.err and compile.out" - - printf -v workload1 "cd ${Nemo_path}\n./makenemo -r ${cfg} -n ${name_cfg} -m ${arch} -j$Jobs_n_cores $comp_cfg" - python3 Job_Creator.py -f "compile" -j "compile" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "${workload1}" - - state1=$("$job" --wait compile."$Jobs_scheduler") - echo - Job_completed "$state1" - if [ $Completed == false ]; then - echo "Nemo compilation failed, remember to load all the needed modules. Check the details in compile.err" - echo - exit 1 - else - echo "Nemo compilation successful" - echo - fi - - else - echo "Compilation not needed" - echo - fi - - #Copy all the EXP00 data but don't overwrite namelist just the executable - cd "$dir" || echo "Error original dir doesn't exist" exit - cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo . - -} Get_trace() { Compile_extrae # Run nemo with extrae - + ./extraf.sh nemo extrae_functions.txt + sed -i "s|list=.*|list=\"${dir}/extrae_functions_for_xml.txt\" exclude-automatic-functions=\"yes\">|g" extrae.xml for core in "${Nemo_cores[@]}" do @@ -461,6 +451,7 @@ Compile_extrae fi cp nemo_"$core".best_cut.* Metrics + done } @@ -469,12 +460,11 @@ Create_metrics() { + # Create performance metrics - - echo "Creating metrics and storing theme in Metrics folder" echo - python3 Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "modelfactors.py *" + python3 Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "modelfactors.py -ms 100000 *" mv analysis."$Jobs_scheduler" Metrics cd Metrics||(echo "Error Metrics folder doesn't exists"; exit 1) state5=$("$job" --wait analysis."$Jobs_scheduler") @@ -493,4 +483,33 @@ Create_metrics() echo "------------------------------------------------------------------------------" echo } + +main() +{ + +if [ $# -gt 0 ]; then + + echo "This script does not accept arguments, parameters need to be added to perf_metrics.config Aborting" + exit 1 + +fi + +#Get script directory +dir=$(pwd) + +echo +echo "Using the following configuration:" +echo +source "$dir"/perf_metrics.config +grep -o '^[^#]*' perf_metrics.config +echo + +Init +Test_arguments +Gprof_functions +Get_trace +Create_metrics + +} + main "$@"; exit diff --git a/perf_metrics.config b/perf_metrics.config index d24f869..768de74 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -8,7 +8,7 @@ # get Nemo traces with 4 and 48 cores. 2 different nºcores are needed to obtain scalability data. Nemo_path="../NEMO" -Nemo_cores=( 4 24 48 96) +Nemo_cores=( 4 24 48 96 192) # Jobs_n_cores: nºcores used for other jobs like compiling nemo. # Jobs_scheduler: Available (slurm/lsf). @@ -29,8 +29,8 @@ Jobs_queue=debug Compilation_compile="false" Compilation_ref="ORCA2_ICE_PISCES" -Compilation_arch="X64_MN4" -Compilation_name="ORCA2" +Compilation_arch="X64_MN4_UNCOUPLED" +Compilation_name="ORCA2_UNCOUPLED" Compilation_sub="OCE del_key 'key_si3 key_top'" # List of modules loaded. @@ -44,4 +44,4 @@ Compilation_sub="OCE del_key 'key_si3 key_top'" # - Dimemas 4.2 -devel # - Python3 -Modules="EXTRAE BASICANALYSIS gcc intel/2018.3 impi/2018.4 netcdf/4.4.1.1 hdf5/1.8.19 DIMEMAS/5.4.2-devel perl" +Modules="EXTRAE/3.7.1 BASICANALYSIS gcc intel/2018.3 impi/2018.4 netcdf/4.4.1.1 hdf5/1.8.19 DIMEMAS/5.4.2-devel perl" -- GitLab From 3f91b7961d249ee44beabeb97f66d3c66c3ad310 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Thu, 8 Sep 2022 12:46:49 +0200 Subject: [PATCH 3/8] Changed Extrae version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2652f01..1a3e106 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Script used to get important metrics from NEMO This script requires the following BSCTOOLS to be installed, loaded and available through the PATH environment variable. -* *Extrae (4.0.0 or above)* +* *Extrae (3.7.1)* * *Paraver* * *Dimemas (latest has a bug use 5.4.2-devel instead)* * *Basicanalysis* -- GitLab From 18f06ce47f9778251f95d0216e5c9093585b5a43 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Thu, 8 Sep 2022 12:56:47 +0200 Subject: [PATCH 4/8] Merged solutions for issue 1 and 2 --- cfgs/.directory | 4 + cfgs/2dh_BurstEfficiency.cfg | 221 +++++ cfgs/barrier-syncr-time.cfg | 80 ++ cfgs/burst_duration.cfg | 77 ++ cfgs/burst_useful.cfg | 150 ++++ cfgs/cycles.cfg | 141 +++ cfgs/dimemas.collectives | 14 + cfgs/dimemas_ideal.cfg | 159 ++++ cfgs/efficiency_table-global.gp | 31 + cfgs/efficiency_table-hybrid.gp | 30 + cfgs/efficiency_table.gp | 31 + cfgs/flushing-cycles.cfg | 148 +++ cfgs/flushing-inst.cfg | 148 +++ cfgs/flushing.cfg | 76 ++ cfgs/instructions.cfg | 141 +++ cfgs/io-call-cycles.cfg | 148 +++ cfgs/io-call-instructions.cfg | 148 +++ cfgs/io-call-reverse.cfg | 76 ++ cfgs/modelfactors-all.gp | 36 + cfgs/modelfactors-comm.gp | 26 + cfgs/modelfactors-hybrid.gp | 37 + cfgs/modelfactors-mpi-hybrid.gp | 32 + cfgs/modelfactors-onlydata.gp | 31 + cfgs/modelfactors-scale.gp | 27 + cfgs/mpi-call-outside.cfg | 77 ++ cfgs/mpi-io-cycles.cfg | 149 +++ cfgs/mpi-io-instructions.cfg | 149 +++ cfgs/mpi-io-reverse.cfg | 77 ++ cfgs/mpi-io.cfg | 77 ++ cfgs/mpi-master-thread.cfg | 79 ++ cfgs/runtime.cfg | 72 ++ cfgs/runtime_app.cfg | 74 ++ cfgs/time_computing.cfg | 114 +++ cfgs/timings.cfg | 72 ++ hybridmetrics.py | 1496 +++++++++++++++++++++++++++++++ modelfactors.py | 192 ++++ perf_metrics.bash | 2 +- plots.py | 738 +++++++++++++++ rawdata.py | 857 ++++++++++++++++++ simplemetrics.py | 1249 ++++++++++++++++++++++++++ tracemetadata.py | 376 ++++++++ utils.py | 186 ++++ 42 files changed, 8047 insertions(+), 1 deletion(-) create mode 100644 cfgs/.directory create mode 100644 cfgs/2dh_BurstEfficiency.cfg create mode 100644 cfgs/barrier-syncr-time.cfg create mode 100644 cfgs/burst_duration.cfg create mode 100644 cfgs/burst_useful.cfg create mode 100644 cfgs/cycles.cfg create mode 100644 cfgs/dimemas.collectives create mode 100644 cfgs/dimemas_ideal.cfg create mode 100644 cfgs/efficiency_table-global.gp create mode 100644 cfgs/efficiency_table-hybrid.gp create mode 100644 cfgs/efficiency_table.gp create mode 100644 cfgs/flushing-cycles.cfg create mode 100644 cfgs/flushing-inst.cfg create mode 100644 cfgs/flushing.cfg create mode 100644 cfgs/instructions.cfg create mode 100644 cfgs/io-call-cycles.cfg create mode 100644 cfgs/io-call-instructions.cfg create mode 100644 cfgs/io-call-reverse.cfg create mode 100644 cfgs/modelfactors-all.gp create mode 100644 cfgs/modelfactors-comm.gp create mode 100644 cfgs/modelfactors-hybrid.gp create mode 100644 cfgs/modelfactors-mpi-hybrid.gp create mode 100644 cfgs/modelfactors-onlydata.gp create mode 100644 cfgs/modelfactors-scale.gp create mode 100644 cfgs/mpi-call-outside.cfg create mode 100644 cfgs/mpi-io-cycles.cfg create mode 100644 cfgs/mpi-io-instructions.cfg create mode 100644 cfgs/mpi-io-reverse.cfg create mode 100644 cfgs/mpi-io.cfg create mode 100644 cfgs/mpi-master-thread.cfg create mode 100644 cfgs/runtime.cfg create mode 100644 cfgs/runtime_app.cfg create mode 100644 cfgs/time_computing.cfg create mode 100644 cfgs/timings.cfg create mode 100644 hybridmetrics.py create mode 100755 modelfactors.py create mode 100644 plots.py create mode 100644 rawdata.py create mode 100644 simplemetrics.py create mode 100644 tracemetadata.py create mode 100644 utils.py diff --git a/cfgs/.directory b/cfgs/.directory new file mode 100644 index 0000000..5ff3922 --- /dev/null +++ b/cfgs/.directory @@ -0,0 +1,4 @@ +[Dolphin] +Timestamp=2019,6,19,17,11,21 +Version=4 +ViewMode=1 diff --git a/cfgs/2dh_BurstEfficiency.cfg b/cfgs/2dh_BurstEfficiency.cfg new file mode 100644 index 0000000..9429c78 --- /dev/null +++ b/cfgs/2dh_BurstEfficiency.cfg @@ -0,0 +1,221 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 5 +ConfigFile.BeginDescription +ConfigFile.EndDescription + +################################################################################ +< NEW DISPLAYING WINDOW Burst Region duration.c1.c1 > +################################################################################ +window_name Burst Region duration.c1.c1 +window_type single +window_id 1 +window_position_x 484 +window_position_y 126 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 39435569.853000000119 +window_minimum_y 1.377000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Int. Between Evt}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 54000006 +window_filter_module evt_type_label 1 "Elapsed time in MPI" + +################################################################################ +< NEW DISPLAYING WINDOW MPI ellapsed time In burst Region.c3.c1 > +################################################################################ +window_name MPI ellapsed time In burst Region.c3.c1 +window_type single +window_id 2 +window_position_x 502 +window_position_y 144 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 39435569.853000000119 +window_minimum_y 1.377000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Div}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module compose_thread Div { 1, { 1 1000.000000000000 } } +window_filter_module evt_type 1 54000006 +window_filter_module evt_type_label 1 "Elapsed time in MPI" + +################################################################################ +< NEW DISPLAYING WINDOW Computing time in Region.c1 > +################################################################################ +window_name Computing time in Region.c1 +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation substract +window_identifiers 1 2 +window_position_x 538 +window_position_y 180 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 39435569.853000000119 +window_minimum_y 1.377000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 0 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +################################################################################ +< NEW DISPLAYING WINDOW Burst Region duration.c2 > +################################################################################ +window_name Burst Region duration.c2 +window_type single +window_id 4 +window_position_x 556 +window_position_y 198 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 39435569.853000000119 +window_minimum_y 1.377000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Int. Between Evt}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 54000006 +window_filter_module evt_type_label 1 "Elapsed time in MPI" + +################################################################################ +< NEW DISPLAYING WINDOW Burst efficiency > +################################################################################ +window_name Burst efficiency +window_type composed +window_id 5 +window_factors 1.000000000000 1.000000000000 +window_operation divide +window_identifiers 3 4 +window_position_x 475 +window_position_y 145 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 1.100000000000 +window_minimum_y 0.000000004381 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 3 +window_drawmode_rows 3 +window_pixel_size 1 +window_labels_to_draw 0 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: 2DH - efficiency +Analyzer2D.X: 130 +Analyzer2D.Y: 233 +Analyzer2D.Width: 365 +Analyzer2D.Height: 218 +Analyzer2D.ControlWindow: 5 +Analyzer2D.DataWindow: 5 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Average value +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Enabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1.000000000000 +Analyzer2D.Delta: 2.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 0.142980245071 +Analyzer2D.MaximumGradient: 0.184636349192 +Analyzer2D.PixelSize: 1 +Analyzer2D.CodeColor: False +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/barrier-syncr-time.cfg b/cfgs/barrier-syncr-time.cfg new file mode 100644 index 0000000..ee7b63d --- /dev/null +++ b/cfgs/barrier-syncr-time.cfg @@ -0,0 +1,80 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW MPI call.SerAndTransfer > +################################################################################ +window_name MPI call.SerAndTransfer +window_type single +window_id 1 +window_position_x 3024 +window_position_y 486 +window_width 729 +window_height 320 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 70.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max true +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module task Thread i { 1, { 1 0.000000000000 } } +window_filter_module evt_type 2 50000001 50000002 +window_filter_module evt_type_label 2 "MPI Point-to-point" "MPI Collective Comm" +window_filter_module evt_value 3 5.000000000000 6.000000000000 8.000000000000 +window_filter_module evt_value_label 3 "MPI_Wait" "MPI_Waitall" "MPI_Barrier" +window_synchronize 1 + +< NEW ANALYZER2D > +Analyzer2D.Name: MPI-Ser-Transfer +Analyzer2D.X: 2810 +Analyzer2D.Y: 48 +Analyzer2D.Width: 868 +Analyzer2D.Height: 491 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: True +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: True +Analyzer2D.Minimum: 3.000000000000 +Analyzer2D.Maximum: 10.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 52.305000000000 +Analyzer2D.MaximumGradient: 76291.281000000003 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/burst_duration.cfg b/cfgs/burst_duration.cfg new file mode 100644 index 0000000..6958790 --- /dev/null +++ b/cfgs/burst_duration.cfg @@ -0,0 +1,77 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW burst_duration.c1 > +################################################################################ +window_name burst_duration.c1 +window_type single +window_id 1 +window_position_x 596 +window_position_y 238 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 39435569.853000000119 +window_minimum_y 1.377000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Int. Between Evt}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 54000006 +window_filter_module evt_type_label 1 "Elapsed time in MPI" + +< NEW ANALYZER2D > +Analyzer2D.Name: burst_duration +Analyzer2D.X: 2297 +Analyzer2D.Y: 30 +Analyzer2D.Width: 391 +Analyzer2D.Height: 472 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 18.918000000000 +Analyzer2D.Maximum: 191006.618399999978 +Analyzer2D.Delta: 1000000000000000.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 952444.935999999987 +Analyzer2D.MaximumGradient: 959765.031999999890 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/burst_useful.cfg b/cfgs/burst_useful.cfg new file mode 100644 index 0000000..ac54618 --- /dev/null +++ b/cfgs/burst_useful.cfg @@ -0,0 +1,150 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW burst_duration > +################################################################################ +window_name burst_duration +window_type single +window_id 1 +window_position_x 567 +window_position_y 209 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 57174.218999999997 +window_minimum_y 24.956000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Int. Between Evt}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 54000006 +window_filter_module evt_type_label 1 "Elapsed time in MPI" + +################################################################################ +< NEW DISPLAYING WINDOW burst_MPI_ellapsed_time > +################################################################################ +window_name burst_MPI_ellapsed_time +window_type single +window_id 2 +window_position_x 567 +window_position_y 209 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 12582.013999999999 +window_minimum_y 21.416000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Div}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module compose_thread Div { 1, { 1 1000.000000000000 } } +window_filter_module evt_type 1 54000006 +window_filter_module evt_type_label 1 "Elapsed time in MPI" + +################################################################################ +< NEW DISPLAYING WINDOW useful_burst > +################################################################################ +window_name useful_burst +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation substract +window_identifiers 1 2 +window_position_x 1960 +window_position_y 174 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 57174.218999999997 +window_minimum_y -4162.623000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 0 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: useful_burst +Analyzer2D.X: 2327 +Analyzer2D.Y: 31 +Analyzer2D.Width: 841 +Analyzer2D.Height: 365 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Vertical +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: True +Analyzer2D.Minimum: -5122.957000000000 +Analyzer2D.Maximum: 142551.129350000003 +Analyzer2D.Delta: 738.370431750000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: -5122.957000000000 +Analyzer2D.MaximumGradient: 16572593.654999990016 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/cycles.cfg b/cfgs/cycles.cfg new file mode 100644 index 0000000..7d2091e --- /dev/null +++ b/cfgs/cycles.cfg @@ -0,0 +1,141 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW cycles.c1.c1 > +################################################################################ +window_name cycles.c1.c1 +window_type single +window_id 1 +window_position_x 300 +window_position_y 23 +window_width 600 +window_height 114 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 194620048057.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 42000059 + +################################################################################ +< NEW DISPLAYING WINDOW Useful.c1.c1 > +################################################################################ +window_name Useful.c1.c1 +window_type single +window_id 2 +window_position_x 323 +window_position_y 46 +window_width 600 +window_height 134 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered true +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 1.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Useful}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, =}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, All}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +################################################################################ +< NEW DISPLAYING WINDOW Useful Cycles Timeline > +################################################################################ +window_name Useful Cycles Timeline +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation product +window_identifiers 1 2 +window_position_x 391 +window_position_y 146 +window_width 767 +window_height 314 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 11047386945.000000000000 +window_minimum_y 5433.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: cycles +Analyzer2D.X: 414 +Analyzer2D.Y: 54 +Analyzer2D.Width: 556 +Analyzer2D.Height: 764 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1000000000000000000.000000000000 +Analyzer2D.Delta: 1000000000000000000.000000000000 +Analyzer2D.ComputeGradient: False +Analyzer2D.MinimumGradient: 0 +Analyzer2D.MaximumGradient: 999999999999999999 +Analyzer2D.PixelSize: 1 +Analyzer2D.CodeColor: False + diff --git a/cfgs/dimemas.collectives b/cfgs/dimemas.collectives new file mode 100644 index 0000000..3b86752 --- /dev/null +++ b/cfgs/dimemas.collectives @@ -0,0 +1,14 @@ +Machine globalop: 0 0 LOG MAX 0 MAX +Machine globalop: 0 1 LOG MAX 0 MAX +Machine globalop: 0 2 LOG MAX 0 MAX +Machine globalop: 0 3 LOG MAX 0 MAX +Machine globalop: 0 4 LOG MAX 0 MAX +Machine globalop: 0 5 LOG MAX 0 MAX +Machine globalop: 0 6 LOG MAX 0 MAX +Machine globalop: 0 7 LOG MAX 0 MAX +Machine globalop: 0 8 LIN 2MAX LIN 2MAX +Machine globalop: 0 9 LOG MAX 0 MAX +Machine globalop: 0 10 LOG 2MAX 0 MAX +Machine globalop: 0 11 LOG MAX 0 MAX +Machine globalop: 0 12 LOG MAX 0 MAX +Machine globalop: 0 13 LOG MAX 0 MAX diff --git a/cfgs/dimemas_ideal.cfg b/cfgs/dimemas_ideal.cfg new file mode 100644 index 0000000..d0f46f0 --- /dev/null +++ b/cfgs/dimemas_ideal.cfg @@ -0,0 +1,159 @@ +#DIMEMAS_CONFIGURATION + +/******************************************************************************* + * RECORDS DEFINITION - NOTES TO MANUAL EDIT OF THIS FILE + ******************************************************************************* + * + * The records here described follow the SDDF file structure. Each field must be + * sepparated by commas in the actual records. + * + * 'int' and 'double' values are expresed as is. 'char[]' values must be always + * wrapped by quotes. 'double' and 'int' arrays must be in the following format: + * + * [n] { value_1, value_2, ..., value_n } + * + * Where 'n' is the number of elements in the array + * + ******************************************************************************* + + "wide area network information" { + 1: char[] "name of the wide area network simulated" + 2: int "number of machines in wan" + 3: int "number of dedicated connections between machines in the simulated" + "system" + 4: int "function that models influence of traffic in the non dedicated" + "network." + "options: 1 EXPONENTIAl, 2 LOGARITHMIC, 3 LINEAR, 4 CONSTANT" + 5: double "maximal value of traffic in the network" + 6: double "external net bandwidth (MBps)" + 7: int "external network collective communications model" + "options: 1 CONSTANT, 2 LINEAR, 3 LOGARITHMIC" +};; + +"environment information" { + 1: char[] "machine name" + 2: int "machine ID"; + 3: char[] "architecture used to instrument" + 4: int "number of nodes on virtual machine" + 5: double "data tranfer rate between nodes (MBps)" + "0 means instantaneous communication" + 6: int "maximun number of messages on network" + "0 means no limit" + "1 means bus contention" + 7: int "internal network collective communications model" + "options: 1 CONSTANT, 2 LINEAR, 3 LOGARITHMIC" +};; + + +"node information" { + 1: int "machine ID" + 2: char[] "architecture node name" + 3: int "number of processors within node" + 4: double "relative processor speed (divisive factor)" + 5: double "latency time (s) of intra-node communications model" + 6: double "bandwidth (MBps) of intra-node communications model" + "0 means instantaneous communication" + 7: int "maximum number of concurrent messages of intra-node" + "communications model:" + "0 means no limit" + "1 means bus contention" + 8: int "input links of intra-node communications model" + 9: int "output links of intra-node communications model" + 10: double "latency time (s) of inter-node communications model" + 11: int "input links of inter-node communications model" + 12: int "input links of intra-node communications model" + 13: double "latency time (s) of inter-machines (WAN) communications model" +};; + +"multi node information" { + 1: int "machine ID" + 2: int "number of nodes with same configuration" + 3: char[] "architecture node name" + 4: int "number of processors within node" + 5: double "processor speed ratio wrt. original execution (divisive factor)" + "0 means instantaneous | negative value means fixed duration in s." + 6: double "latency time (s) of intra-node communications model" + 7: double "bandwidth (MBps) of intra-node communications model" + "0 means instantaneous communication" + 8: int "maximum number of concurrent messages of intra-node" + "communications model: " + "0 means no limit" + "1 means bus contention" + 9: int "input links of intra-node communications model" + 10: int "output links of intra-node communications model" + 11: double "latency time (s) of inter-node communications model" + 12: int "input links of inter-node communications model" + 13: int "input links of intra-node communications model" + 14: double "latency time (s) of inter-machines (WAN) communications model" +};; + +"mapping information" { + 1: char[] "application tracefile name" + 2: int "number of tasks in application" + 3: int[] "list of nodes where each application tasks is mapped" +};; + +"predefined mapping information" { + 1: char[] "application tracefile name" (OPTIONAL) + 2: char[] "predefined map identifier" + "options: FILL_NODES | TASKS_PER_NODE | INTERLEAVED" +} + +"configuration files" { + 1: char[] "scheduler definition filename" + 2: char[] "file system definition filename" + 3: char[] "communications fine tuning configuration filename" + 4: char[] "sensitivity configuration filename" +};; + +"modules information" { + 1: int "module type" + 2: int "module value" + 3: double "module speed ration wrt. original execution (divisive factor)" + "0 means instantaneous | negative value means fixed duration in s." +};; + +"file system parameters" { + 1: double "disk latency" + 2: double "disk bandwidth"; + 3: double "block size"; + 4: int "concurrent requests"; + 5: double "hit ratio"; +};; + +"dedicated connection information" { + 1: int "connection ID" + 2: int "source machine ID" + 3: int "destination machine ID" + 4: double "bandwidth of the connection (MBps)" + 5: int[] "list of tags that will use the connection" + 6: int "size of first message (bytes) to apply the comparision to use the" + "connection" + 7: char[] "size condition that should meet messages to use the connection" + "it can be <, =, > and (referent to first message size)" + 8: char[] "operation. options: & AND, | OR" + 9: int "size of second condition that should meet messages to use the" + "connection" + 10: char[] "size condition that should meet messages to use the connection" + "it can be <, =, > and its is referent to second message size" + 11: int[] "list of communicators of coll. Operations that can use the" + "connection" + 12: double "latency of dedicated connection (s)" + 13: double "latency due to distance (s)" +};; + +*******************************************************************************/ + + +"wide area network information" {"", 1, 0, 4, 0.0, 0.0, 1};; + +"environment information" {"", 0, "", REPLACE_BY_NTASKS, 0.0, 0, 1};; + +"multi node information" {0, REPLACE_BY_NTASKS, "", REPLACE_BY_CPUS_PER_NODE, 1.0, 0.0, 0.0, 0, 1, 0, 0.0, 1, 1, 0.0};; + +"predefined mapping information" {"1 TASKS_PER_NODE"};; + +"configuration files" {"", "", "REPLACE_BY_COLLECTIVES_PATH", ""};; + +"file system parameters" {0.0, 0.0, 8.0, 0, 1.0};; + diff --git a/cfgs/efficiency_table-global.gp b/cfgs/efficiency_table-global.gp new file mode 100644 index 0000000..c7a5ef6 --- /dev/null +++ b/cfgs/efficiency_table-global.gp @@ -0,0 +1,31 @@ +#!/bin/gnuplot +set output "./efficiency_table-global.png" + +#Prepare the plot size + +#REPLACE_BY_SIZE + +set datafile separator "," + +set title "" +unset key + +set format cb "%.0f%%" +set cbrange[0:100] +set tmargin 1 + + +set cbtics ('<50%%' 50, '60%%' 60, '70%%' 70, '80%%' 80, "90%%" 90, "100%%" 100) + +set palette defined (0.00 "#000000", 0.01 "#FF0000", 0.60 "#E67C73", 0.75 "#F9950A", 0.85 "#AADC32", 0.90 "#5CC863", 1.00 "#57BB8A") + + +set xtics offset 0,15 +set ytics offset -27,0 left +set yrange [] reverse + +plot \ + 'efficiency_table-global.csv' \ + matrix rowheaders columnheaders using 1:2:3 with image, \ + 'efficiency_table-global.csv' \ + matrix rowheaders columnheaders using 1:2:(sprintf("%.2f",$3) ) with labels diff --git a/cfgs/efficiency_table-hybrid.gp b/cfgs/efficiency_table-hybrid.gp new file mode 100644 index 0000000..0e726cb --- /dev/null +++ b/cfgs/efficiency_table-hybrid.gp @@ -0,0 +1,30 @@ +#!/bin/gnuplot +set output "./efficiency_table-hybrid.png" + +#Prepare the plot size + +#REPLACE_BY_SIZE + +set datafile separator "," + +set title "" +unset key + +set format cb "%.0f%%" +set cbrange[0:100] +set tmargin 1 + + +set cbtics ('<50%%' 50, '60%%' 60, '70%%' 70, '80%%' 80, "90%%" 90, "100%%" 100) + +set palette defined (0.00 "#000000", 0.01 "#FF0000", 0.60 "#E67C73", 0.75 "#F9950A", 0.85 "#AADC32", 0.90 "#5CC863", 1.00 "#57BB8A") + +set xtics offset 0,15 +set ytics offset -37,0 left +set yrange [] reverse + +plot \ + 'efficiency_table-hybrid.csv' \ + matrix rowheaders columnheaders using 1:2:3 with image, \ + 'efficiency_table-hybrid.csv' \ + matrix rowheaders columnheaders using 1:2:(sprintf("%.2f",$3) ) with labels diff --git a/cfgs/efficiency_table.gp b/cfgs/efficiency_table.gp new file mode 100644 index 0000000..cda44e1 --- /dev/null +++ b/cfgs/efficiency_table.gp @@ -0,0 +1,31 @@ +#!/bin/gnuplot +set output "./efficiency_table.png" + +#Prepare the plot size + +#REPLACE_BY_SIZE + +set datafile separator "," + +set title "" +unset key + +set format cb "%.0f%%" +set cbrange[0:100] +set tmargin 1 + + +set cbtics ('<50%%' 50, '60%%' 60, '70%%' 70, '80%%' 80, "90%%" 90, "100%%" 100) + +set palette defined (0.00 "#000000", 0.01 "#FF0000", 0.60 "#E67C73", 0.75 "#F9950A", 0.85 "#AADC32", 0.90 "#5CC863", 1.00 "#57BB8A") + + +set xtics offset 0,15 +set ytics offset -27,0 left +set yrange [] reverse + +plot \ + 'efficiency_table.csv' \ + matrix rowheaders columnheaders using 1:2:3 with image, \ + 'efficiency_table.csv' \ + matrix rowheaders columnheaders using 1:2:(sprintf("%.2f",$3) ) with labels diff --git a/cfgs/flushing-cycles.cfg b/cfgs/flushing-cycles.cfg new file mode 100644 index 0000000..dcd59fe --- /dev/null +++ b/cfgs/flushing-cycles.cfg @@ -0,0 +1,148 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW Flushing.c1.c1 > +################################################################################ +window_name Flushing.c1.c1 +window_type single +window_id 1 +window_position_x 442 +window_position_y 116 +window_width 600 +window_height 115 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 34.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } +window_filter_module evt_type 1 40000003 +window_filter_module evt_type_label 1 "Flushing Traces" + +################################################################################ +< NEW DISPLAYING WINDOW cycles.c1 > +################################################################################ +window_name cycles.c1 +window_type single +window_id 2 +window_position_x 417 +window_position_y 404 +window_width 600 +window_height 114 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 194620048057.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 42000059 +window_filter_module evt_type_label 1 "PAPI_TOT_CYC [Total cycles]" + +################################################################################ +< NEW DISPLAYING WINDOW flushing_cycles > +################################################################################ +window_name flushing_cycles +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation product +window_identifiers 1 2 +window_position_x 2357 +window_position_y 217 +window_width 600 +window_height 115 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 79325.000000000000 +window_minimum_y 48287.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: flushing_cycles_hist +Analyzer2D.X: 2597 +Analyzer2D.Y: 55 +Analyzer2D.Width: 436 +Analyzer2D.Height: 587 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1000000000000000000.000000000000 +Analyzer2D.Delta: 1000000000000000000.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 48287.000000000000 +Analyzer2D.MaximumGradient: 79325.000000000000 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/flushing-inst.cfg b/cfgs/flushing-inst.cfg new file mode 100644 index 0000000..767e690 --- /dev/null +++ b/cfgs/flushing-inst.cfg @@ -0,0 +1,148 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW Flushing.c1 > +################################################################################ +window_name Flushing.c1 +window_type single +window_id 1 +window_position_x 355 +window_position_y 29 +window_width 600 +window_height 115 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 34.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } +window_filter_module evt_type 1 40000003 +window_filter_module evt_type_label 1 "Flushing Traces" + +################################################################################ +< NEW DISPLAYING WINDOW Instructions.c1.c2.c1 > +################################################################################ +window_name Instructions.c1.c2.c1 +window_type single +window_id 2 +window_position_x 384 +window_position_y 58 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 206968292.000000000000 +window_minimum_y 3314.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 42000050 +window_filter_module evt_type_label 1 "PAPI_TOT_INS [Instr completed]" + +################################################################################ +< NEW DISPLAYING WINDOW flushing_inst > +################################################################################ +window_name flushing_inst +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation product +window_identifiers 1 2 +window_position_x 2364 +window_position_y 128 +window_width 600 +window_height 115 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 17421.000000000000 +window_minimum_y 17121.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: flushing_inst_hist +Analyzer2D.X: 2597 +Analyzer2D.Y: 55 +Analyzer2D.Width: 436 +Analyzer2D.Height: 587 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1000000000000000000.000000000000 +Analyzer2D.Delta: 1000000000000000000.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 17121.000000000000 +Analyzer2D.MaximumGradient: 17421.000000000000 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/flushing.cfg b/cfgs/flushing.cfg new file mode 100644 index 0000000..6c7e258 --- /dev/null +++ b/cfgs/flushing.cfg @@ -0,0 +1,76 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW Flushing > +################################################################################ +window_name Flushing +window_type single +window_id 1 +window_position_x 2529 +window_position_y 349 +window_width 600 +window_height 115 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 34.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 72, { All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All, All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 40000003 +window_filter_module evt_type_label 1 "Flushing Traces" + +< NEW ANALYZER2D > +Analyzer2D.Name: flushing +Analyzer2D.X: 2717 +Analyzer2D.Y: 204 +Analyzer2D.Width: 617 +Analyzer2D.Height: 606 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: True +Analyzer2D.Minimum: 1.000000000000 +Analyzer2D.Maximum: 1.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 18.053000000000 +Analyzer2D.MaximumGradient: 35.420000000000 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/instructions.cfg b/cfgs/instructions.cfg new file mode 100644 index 0000000..777df0c --- /dev/null +++ b/cfgs/instructions.cfg @@ -0,0 +1,141 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW Instructions > +################################################################################ +window_name Instructions +window_type single +window_id 1 +window_position_x 346 +window_position_y 41 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 18640295641.000000000000 +window_minimum_y 5257.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 42000050 + +################################################################################ +< NEW DISPLAYING WINDOW Useful > +################################################################################ +window_name Useful +window_type single +window_id 2 +window_position_x 369 +window_position_y 92 +window_width 600 +window_height 134 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered true +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 1.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Useful}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, =}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, All}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +################################################################################ +< NEW DISPLAYING WINDOW Useful Instructions Timeline > +################################################################################ +window_name Useful Instructions Timeline +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation product +window_identifiers 1 2 +window_position_x 406 +window_position_y 340 +window_width 767 +window_height 314 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 18640295641.000000000000 +window_minimum_y 5257.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: instructions +Analyzer2D.X: 366 +Analyzer2D.Y: 24 +Analyzer2D.Width: 556 +Analyzer2D.Height: 764 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1000000000000000000.000000000000 +Analyzer2D.Delta: 1000000000000000000.000000000000 +Analyzer2D.ComputeGradient: False +Analyzer2D.MinimumGradient: 0 +Analyzer2D.MaximumGradient: 999999999999999999 +Analyzer2D.PixelSize: 1 +Analyzer2D.CodeColor: False + diff --git a/cfgs/io-call-cycles.cfg b/cfgs/io-call-cycles.cfg new file mode 100644 index 0000000..6140f79 --- /dev/null +++ b/cfgs/io-call-cycles.cfg @@ -0,0 +1,148 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW I/O call.c1.c1 > +################################################################################ +window_name I/O call.c1.c1 +window_type single +window_id 1 +window_position_x 616 +window_position_y 290 +window_width 729 +window_height 154 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 13.000000000000 +window_minimum_y 1.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } +window_filter_module evt_type 1 40000004 +window_filter_module evt_type_label 1 "I/O calls" + +################################################################################ +< NEW DISPLAYING WINDOW cycles.c1.c1.c1 > +################################################################################ +window_name cycles.c1.c1.c1 +window_type single +window_id 2 +window_position_x 645 +window_position_y 319 +window_width 600 +window_height 114 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 194620048057.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 42000059 +window_filter_module evt_type_label 1 "PAPI_TOT_CYC [Total cycles]" + +################################################################################ +< NEW DISPLAYING WINDOW posixio_cycles > +################################################################################ +window_name posixio_cycles +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation product +window_identifiers 1 2 +window_position_x 2298 +window_position_y 141 +window_width 729 +window_height 154 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 72043816.000000000000 +window_minimum_y 3111.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: posixio_cycles_hist +Analyzer2D.X: 2597 +Analyzer2D.Y: 55 +Analyzer2D.Width: 436 +Analyzer2D.Height: 587 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1000000000000000000.000000000000 +Analyzer2D.Delta: 1000000000000000000.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 6685313.000000000000 +Analyzer2D.MaximumGradient: 8988531.000000000000 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/io-call-instructions.cfg b/cfgs/io-call-instructions.cfg new file mode 100644 index 0000000..af8bc3b --- /dev/null +++ b/cfgs/io-call-instructions.cfg @@ -0,0 +1,148 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW I/O call.c1 > +################################################################################ +window_name I/O call.c1 +window_type single +window_id 1 +window_position_x 355 +window_position_y 29 +window_width 729 +window_height 154 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 13.000000000000 +window_minimum_y 1.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } +window_filter_module evt_type 1 40000004 +window_filter_module evt_type_label 1 "I/O calls" + +################################################################################ +< NEW DISPLAYING WINDOW Instructions.c1.c2 > +################################################################################ +window_name Instructions.c1.c2 +window_type single +window_id 2 +window_position_x 384 +window_position_y 58 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 206968292.000000000000 +window_minimum_y 3314.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 42000050 +window_filter_module evt_type_label 1 "PAPI_TOT_INS [Instr completed]" + +################################################################################ +< NEW DISPLAYING WINDOW posixio_inst > +################################################################################ +window_name posixio_inst +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation product +window_identifiers 1 2 +window_position_x 413 +window_position_y 87 +window_width 729 +window_height 154 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 3625596.000000000000 +window_minimum_y 3676.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: posixio_inst_hist +Analyzer2D.X: 2597 +Analyzer2D.Y: 55 +Analyzer2D.Width: 436 +Analyzer2D.Height: 587 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1000000000000000000.000000000000 +Analyzer2D.Delta: 1000000000000000000.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 2247759.000000000000 +Analyzer2D.MaximumGradient: 2503143.000000000000 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/io-call-reverse.cfg b/cfgs/io-call-reverse.cfg new file mode 100644 index 0000000..196ec98 --- /dev/null +++ b/cfgs/io-call-reverse.cfg @@ -0,0 +1,76 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW I/O call > +################################################################################ +window_name I/O call +window_type single +window_id 1 +window_position_x 2614 +window_position_y 238 +window_width 729 +window_height 154 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 18.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 40000004 +window_filter_module evt_type_label 1 "I/O calls" + +< NEW ANALYZER2D > +Analyzer2D.Name: io-call +Analyzer2D.X: 2971 +Analyzer2D.Y: 50 +Analyzer2D.Width: 646 +Analyzer2D.Height: 565 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Vertical +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: True +Analyzer2D.Minimum: 1.000000000000 +Analyzer2D.Maximum: 13.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 453.192000000000 +Analyzer2D.MaximumGradient: 17291.650000000001 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/modelfactors-all.gp b/cfgs/modelfactors-all.gp new file mode 100644 index 0000000..e05e716 --- /dev/null +++ b/cfgs/modelfactors-all.gp @@ -0,0 +1,36 @@ +#Gnuplot template for the projection functions + +#Prepare the axes +#REPLACE_BY_XRANGE +set xlabel "Number of Processes" +set logscale x +#REPLACE_BY_YRANGE +set ylabel "Efficiency" +set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) +set grid ytics + +set style line 1 lt 7 dt 2 lw 1.5 lc rgb "#0E3D59" +set style line 2 lt 7 dt 2 lw 1.5 lc rgb "#88A61B" +set style line 3 lt 7 dt 2 lw 1.5 lc rgb "#F29F05" +set style line 4 lt 7 dt 2 lw 1.5 lc rgb "#F25C05" +set style line 5 lt 7 dt 2 lw 1.5 lc rgb "#D92525" + +set key left bottom Left reverse + +#REPLACE_BY_PARA_FUNCTION +#REPLACE_BY_LOAD_FUNCTION +#REPLACE_BY_COMM_FUNCTION +#REPLACE_BY_COMP_FUNCTION +#REPLACE_BY_GLOB_FUNCTION + +plot para(x) title "Parallel Efficiency" ls 1,\ + load(x) title "Load Balance" ls 2,\ + comm(x) title "Communication Efficiency" ls 3,\ + comp(x) title "Computation Scalability" ls 4,\ + glob(x) title "Global Efficiency" ls 5,\ + '-' with points notitle ls 1,\ + '-' with points notitle ls 2,\ + '-' with points notitle ls 3,\ + '-' with points notitle ls 4,\ + '-' with points notitle ls 5 + diff --git a/cfgs/modelfactors-comm.gp b/cfgs/modelfactors-comm.gp new file mode 100644 index 0000000..6984035 --- /dev/null +++ b/cfgs/modelfactors-comm.gp @@ -0,0 +1,26 @@ + +#Gnuplot template for the projection functions + +#REPLACE_BY_TRACE_NAMES +#REPLACE_BY_XRANGE +set xlabel "Number of Processes" +#REPLACE_BY_XTICS_LABEL +## set logscale x +#REPLACE_BY_YRANGE +set ylabel "Efficiency (%)" +# set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) +# set grid ytics + +set style line 1 lt 7 dt 2 lw 1.5 lc rgb "red" +set style line 2 lt 7 dt 2 lw 1.5 lc rgb "green" +set style line 3 lt 7 dt 2 lw 1.5 lc rgb "orange" +set style line 4 lt 7 dt 2 lw 1.5 lc rgb "blue" +set style line 5 lt 7 dt 2 lw 1.5 lc rgb "magenta" + +set key left bottom Left reverse + + +plot '-' with linespoints title "Communication Efficiency" ls 1,\ + '-' with linespoints title "Serialization Efficiency" ls 2,\ + '-' with linespoints title "Transfer Efficiency" ls 3 + diff --git a/cfgs/modelfactors-hybrid.gp b/cfgs/modelfactors-hybrid.gp new file mode 100644 index 0000000..606ccd2 --- /dev/null +++ b/cfgs/modelfactors-hybrid.gp @@ -0,0 +1,37 @@ + +#Gnuplot template for the projection functions + +#REPLACE_BY_TRACE_NAMES +#REPLACE_BY_XRANGE +set xlabel "Number of Processes" +#REPLACE_BY_XTICS_LABEL +## set logscale x +#REPLACE_BY_YRANGE +set ylabel "Efficiency (%)" +# set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) +# set grid ytics + + +set style line 1 lt 8 dt 2 lw 2.0 lc rgb '#4B0082' # indigo + +set style line 2 lt 7 dt 2 lw 1.5 lc rgb "forest-green" +set style line 3 lt 6 dt 2 lw 1.5 lc rgb '#00FF00' # lime +set style line 4 lt 6 dt 2 lw 1.5 lc rgb '#008B8B' # darkcyan + +set style line 5 lt 5 dt 2 lw 1.5 lc rgb "red" +set style line 6 lt 4 dt 2 lw 1.5 lc rgb "orange" +set style line 7 lt 4 dt 2 lw 1.5 lc rgb "salmon" + +set key left bottom Left reverse + + + +plot '-' with linespoints title "Hybrid Efficiency" ls 1,\ + '-' with linespoints title "MPI Parallel efficiency" ls 2,\ + '-' with linespoints title "MPI Load balance" ls 3,\ + '-' with linespoints title "MPI Communication efficiency" ls 4,\ +#REPLACE_BY_OMP_PAR_EFF +#REPLACE_BY_OMP_LB +#REPLACE_BY_OMP_COMM + + diff --git a/cfgs/modelfactors-mpi-hybrid.gp b/cfgs/modelfactors-mpi-hybrid.gp new file mode 100644 index 0000000..3e30589 --- /dev/null +++ b/cfgs/modelfactors-mpi-hybrid.gp @@ -0,0 +1,32 @@ + +#Gnuplot template for the projection functions + +#REPLACE_BY_TRACE_NAMES +#REPLACE_BY_XRANGE +set xlabel "Number of Processes" +#REPLACE_BY_XTICS_LABEL +## set logscale x +#REPLACE_BY_YRANGE +set ylabel "Efficiency (%)" +# set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) +# set grid ytics + +set style line 1 lt 5 dt 2 lw 1.5 lc rgb '#1c1044' # dark blue + +set style line 2 lt 8 dt 2 lw 1.5 lc rgb "forest-green" +set style line 3 lt 6 dt 2 lw 1.5 lc rgb '#00FF00' # lime +set style line 4 lt 5 dt 2 lw 1.5 lc rgb '#008B8B' # darkcyan + + +set style line 8 lt 4 dt 2 lw 1.5 lc rgb '#7CFC00' # lawngreen +set style line 9 lt 4 dt 2 lw 1.5 lc rgb '#00FFFF' # cyan + + +set key left bottom Left reverse + + +plot '-' with linespoints title "MPI Parallel efficiency" ls 2,\ + '-' with linespoints title "MPI Load balance" ls 3,\ + '-' with linespoints title "MPI Communication efficiency" ls 4,\ + '-' with linespoints title "Serialization Efficiency" ls 8,\ + '-' with linespoints title "Transfer Efficiency" ls 9 diff --git a/cfgs/modelfactors-onlydata.gp b/cfgs/modelfactors-onlydata.gp new file mode 100644 index 0000000..4d0ff06 --- /dev/null +++ b/cfgs/modelfactors-onlydata.gp @@ -0,0 +1,31 @@ + +#Gnuplot template for the projection functions + +#REPLACE_BY_TRACE_NAMES +#REPLACE_BY_XRANGE +set xlabel "Number of Processes" +#REPLACE_BY_XTICS_LABEL +# set logscale x +#REPLACE_BY_YRANGE +## set yrange [0:] +set ylabel "Efficiency (%)" + +# set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) +# set grid ytics + +set style line 1 lt 8 dt 2 lw 2.0 lc rgb '#4B0082' # indigo +set style line 2 lt 7 dt 2 lw 1.5 lc rgb "red" +set style line 3 lt 7 dt 2 lw 1.5 lc rgb "green" + +set style line 4 lt 5 dt 5 lw 1.5 lc rgb "blue" +set style line 5 lt 3 dt 1 lw 1.8 lc rgb "dark-grey" + +set key left bottom Left reverse + + +plot '-' with linespoints title "Parallel Efficiency" ls 1,\ + '-' with linespoints title "Load Balance" ls 2,\ + '-' with linespoints title "Communication Efficiency" ls 3,\ + '-' with linespoints title "Computation Scalability" ls 4,\ + '-' with linespoints title "Global Efficiency" ls 5 + diff --git a/cfgs/modelfactors-scale.gp b/cfgs/modelfactors-scale.gp new file mode 100644 index 0000000..febb708 --- /dev/null +++ b/cfgs/modelfactors-scale.gp @@ -0,0 +1,27 @@ + +#Gnuplot template for the projection functions + +#REPLACE_BY_TRACE_NAMES +#REPLACE_BY_XRANGE +set xlabel "Number of Processes" +#REPLACE_BY_XTICS_LABEL +# set logscale x +#REPLACE_BY_YRANGE +set ylabel "Efficiency (%)" +# set ytics ( 0, "10%%" 10, "20%%" 20, "30%%" 30, "40%%" 40, "50%%" 50, "60%%" 60, "70%%" 70, "80%%" 80, "90%%" 90, "100%%" 100 ) +# set grid ytics + +set style line 1 lt 5 dt 2 lw 2.0 lc rgb "blue" +set style line 2 lt 7 dt 2 lw 1.5 lc rgb "cyan" +# set style line 2 lt 7 dt 2 lw 1.5 lc rgb '#6A5ACD' # slateblue +set style line 3 lt 7 dt 2 lw 1.5 lc rgb '#4682B4' # steelblue +set style line 4 lt 7 dt 2 lw 1.5 lc rgb '#8A2BE2' # blueviolet + +set key left bottom Left reverse + + +plot '-' with linespoints title "Computation Scalability" ls 1,\ + '-' with linespoints title "IPC Scalability" ls 2,\ + '-' with linespoints title "Instruction Scalability" ls 3,\ + '-' with linespoints title "Frequency Scalability" ls 4 + diff --git a/cfgs/mpi-call-outside.cfg b/cfgs/mpi-call-outside.cfg new file mode 100644 index 0000000..d4e7613 --- /dev/null +++ b/cfgs/mpi-call-outside.cfg @@ -0,0 +1,77 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW MPI call > +################################################################################ +window_name MPI call +window_type single +window_id 1 +window_position_x 2753 +window_position_y 151 +window_width 856 +window_height 303 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 124.000000000000 +window_minimum_y 3.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, [x,y]}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module task Thread i { 1, { 1 0.000000000000 } } +window_filter_module evt_type 2 50000001 50000005 +window_filter_module evt_type_label 2 "MPI Point-to-point" "Unknown" + +< NEW ANALYZER2D > +Analyzer2D.Name: MPI call profile +Analyzer2D.X: 2262 +Analyzer2D.Y: 247 +Analyzer2D.Width: 1381 +Analyzer2D.Height: 658 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: True +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: True +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 200.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 5.988000000000 +Analyzer2D.MaximumGradient: 60105183.622000001371 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/mpi-io-cycles.cfg b/cfgs/mpi-io-cycles.cfg new file mode 100644 index 0000000..ed0a0a6 --- /dev/null +++ b/cfgs/mpi-io-cycles.cfg @@ -0,0 +1,149 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW cycles.c1.c1 > +################################################################################ +window_name cycles.c1.c1 +window_type single +window_id 1 +window_position_x 529 +window_position_y 203 +window_width 600 +window_height 114 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 194620048057.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 42000059 +window_filter_module evt_type_label 1 "PAPI_TOT_CYC [Total cycles]" + +################################################################################ +< NEW DISPLAYING WINDOW MPI call.c1.c1 > +################################################################################ +window_name MPI call.c1.c1 +window_type single +window_id 2 +window_position_x 2303 +window_position_y 308 +window_width 642 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 1.000000000000 +window_minimum_y 1.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } +window_semantic_module task Thread i { 1, { 1 0.000000000000 } } +window_filter_module evt_type 1 50000005 +window_filter_module evt_type_label 1 "MPI I/O" + +################################################################################ +< NEW DISPLAYING WINDOW mpiio_cycles > +################################################################################ +window_name mpiio_cycles +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation product +window_identifiers 1 2 +window_position_x 587 +window_position_y 261 +window_width 600 +window_height 114 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 194620048057.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_random +window_drawmode_rows draw_random +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: mpiio_cycles_hist +Analyzer2D.X: 2468 +Analyzer2D.Y: 122 +Analyzer2D.Width: 600 +Analyzer2D.Height: 300 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1000000000000000000.000000000000 +Analyzer2D.Delta: 1000000000000000000.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 28106010.000000000000 +Analyzer2D.MaximumGradient: 168707925.000000000000 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/mpi-io-instructions.cfg b/cfgs/mpi-io-instructions.cfg new file mode 100644 index 0000000..e683b74 --- /dev/null +++ b/cfgs/mpi-io-instructions.cfg @@ -0,0 +1,149 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW Instructions.c1 > +################################################################################ +window_name Instructions.c1 +window_type single +window_id 1 +window_position_x 413 +window_position_y 87 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 206968292.000000000000 +window_minimum_y 3314.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 42000050 +window_filter_module evt_type_label 1 "PAPI_TOT_INS [Instr completed]" + +################################################################################ +< NEW DISPLAYING WINDOW MPI-io-sign.c1 > +################################################################################ +window_name MPI-io-sign.c1 +window_type single +window_id 2 +window_position_x 442 +window_position_y 116 +window_width 642 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 1.000000000000 +window_minimum_y 1.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Sign}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, Sign}, {topcompose2, Sign} } } +window_semantic_module task Thread i { 1, { 1 0.000000000000 } } +window_filter_module evt_type 1 50000005 +window_filter_module evt_type_label 1 "MPI I/O" + +################################################################################ +< NEW DISPLAYING WINDOW mpiio_inst > +################################################################################ +window_name mpiio_inst +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation product +window_identifiers 1 2 +window_position_x 2334 +window_position_y 232 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 206968292.000000000000 +window_minimum_y 3314.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: mpiio_inst_hist_2 +Analyzer2D.X: 2427 +Analyzer2D.Y: 124 +Analyzer2D.Width: 600 +Analyzer2D.Height: 300 +Analyzer2D.ControlWindow: 3 +Analyzer2D.DataWindow: 3 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Sum bursts +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 1000000000000000000.000000000000 +Analyzer2D.Delta: 1000000000000000000.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 9583794.000000000000 +Analyzer2D.MaximumGradient: 108820758.000000000000 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/mpi-io-reverse.cfg b/cfgs/mpi-io-reverse.cfg new file mode 100644 index 0000000..143be0d --- /dev/null +++ b/cfgs/mpi-io-reverse.cfg @@ -0,0 +1,77 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW MPI call > +################################################################################ +window_name MPI call +window_type single +window_id 1 +window_position_x 2360 +window_position_y 274 +window_width 642 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 150.000000000000 +window_minimum_y 3.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module task Thread i { 1, { 1 0.000000000000 } } +window_filter_module evt_type 1 50000005 +window_filter_module evt_type_label 1 "MPI I/O" + +< NEW ANALYZER2D > +Analyzer2D.Name: MPI-IO +Analyzer2D.X: 2550 +Analyzer2D.Y: 40 +Analyzer2D.Width: 979 +Analyzer2D.Height: 501 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: True +Analyzer2D.HorizVert: Vertical +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: True +Analyzer2D.Minimum: 142.000000000000 +Analyzer2D.Maximum: 150.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 514.500000000000 +Analyzer2D.MaximumGradient: 6606164.996000000276 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/mpi-io.cfg b/cfgs/mpi-io.cfg new file mode 100644 index 0000000..1f683ac --- /dev/null +++ b/cfgs/mpi-io.cfg @@ -0,0 +1,77 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW MPI call > +################################################################################ +window_name MPI call +window_type single +window_id 1 +window_position_x 2360 +window_position_y 274 +window_width 642 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 150.000000000000 +window_minimum_y 3.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module task Thread i { 1, { 1 0.000000000000 } } +window_filter_module evt_type 1 50000005 +window_filter_module evt_type_label 1 "MPI I/O" + +< NEW ANALYZER2D > +Analyzer2D.Name: MPI-IO +Analyzer2D.X: 2862 +Analyzer2D.Y: 40 +Analyzer2D.Width: 667 +Analyzer2D.Height: 483 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: True +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: True +Analyzer2D.Minimum: 141.000000000000 +Analyzer2D.Maximum: 150.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 225.865000000000 +Analyzer2D.MaximumGradient: 698578.567999999970 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/mpi-master-thread.cfg b/cfgs/mpi-master-thread.cfg new file mode 100644 index 0000000..a130f31 --- /dev/null +++ b/cfgs/mpi-master-thread.cfg @@ -0,0 +1,79 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW MPI call > +################################################################################ +window_name MPI call +window_type single +window_id 1 +window_position_x 2214 +window_position_y 148 +window_width 756 +window_height 390 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 70.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +#REPLACE_BY_APP_FILTER +#REPLACE_BY_TASKS_FILTER +#REPLACE_BY_THREADS_FILTER +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, [x,y]}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module task Thread i { 1, { 1 0.000000000000 } } +window_filter_module evt_type 2 50000001 50000005 +window_filter_module evt_type_label 2 "MPI Point-to-point" "Unknown" + +< NEW ANALYZER2D > +Analyzer2D.Name: MPI call profile +Analyzer2D.X: 807 +Analyzer2D.Y: 207 +Analyzer2D.Width: 881 +Analyzer2D.Height: 582 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: True +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: True +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 200.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 11.456000000000 +Analyzer2D.MaximumGradient: 11868747.607000000775 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/runtime.cfg b/cfgs/runtime.cfg new file mode 100644 index 0000000..5438cda --- /dev/null +++ b/cfgs/runtime.cfg @@ -0,0 +1,72 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW MPI call > +################################################################################ +window_name MPI call +window_type single +window_id 1 +window_position_x 377 +window_position_y 288 +window_width 600 +window_height 114 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 70.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Thread i}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, [x,y]}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module task Thread i { 1, { 1 0.000000000000 } } +window_filter_module evt_type 2 50000001 50000005 + +< NEW ANALYZER2D > +Analyzer2D.Name: runtime +Analyzer2D.X: 487 +Analyzer2D.Y: 156 +Analyzer2D.Width: 1271 +Analyzer2D.Height: 616 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: True +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: True +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: False +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 200.000000000000 +Analyzer2D.Delta: 200.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 55413948.358000002801 +Analyzer2D.MaximumGradient: 55413948.358000002801 +Analyzer2D.PixelSize: 1 +Analyzer2D.CodeColor: False + diff --git a/cfgs/runtime_app.cfg b/cfgs/runtime_app.cfg new file mode 100644 index 0000000..791859f --- /dev/null +++ b/cfgs/runtime_app.cfg @@ -0,0 +1,74 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW runtimeapp > +################################################################################ +window_name runtimeapp +window_type single +window_id 1 +window_position_x 2457 +window_position_y 507 +window_width 581 +window_height 118 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered false +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 0.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max true +window_level appl +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { 1 } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, None}, {evt_value, None} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: runtimeapp +Analyzer2D.X: 2513 +Analyzer2D.Y: 133 +Analyzer2D.Width: 257 +Analyzer2D.Height: 251 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: False +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: True +Analyzer2D.Minimum: 0.000000000000 +Analyzer2D.Maximum: 0.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 5232000.069000000134 +Analyzer2D.MaximumGradient: 5232000.069000000134 +Analyzer2D.DrawModeObjects: draw_maximum +Analyzer2D.DrawModeColumns: draw_maximum +Analyzer2D.PixelSize: 1 +Analyzer2D.ColorMode: window_in_gradient_mode +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/cfgs/time_computing.cfg b/cfgs/time_computing.cfg new file mode 100644 index 0000000..4bb17ad --- /dev/null +++ b/cfgs/time_computing.cfg @@ -0,0 +1,114 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 3 + + +################################################################################ +< NEW DISPLAYING WINDOW interval from states.c3 > +################################################################################ +window_name interval from states.c3 +window_type single +window_id 1 +window_position_x 377 +window_position_y 29 +window_width 600 +window_height 134 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered true +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Nanoseconds +window_maximum_y 325782.880999999994 +window_minimum_y 28.247000000000 +window_compute_y_max false +window_level thread +window_scale_relative 0.276616926766 +window_end_time_relative 0.276616926766 +window_object appl { 1, { All } } +window_begin_time_relative 0.261615159840 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, State Record Dur.}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, None}, {to_obj, None}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, None}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_semantic_module thread State Record Dur. { 1, { 2 0.000000000000 1.000000000000 } } + +################################################################################ +< NEW DISPLAYING WINDOW MPI elapsed time (ns).c1 > +################################################################################ +window_name MPI elapsed time (ns).c1 +window_type single +window_id 2 +window_position_x 406 +window_position_y 58 +window_width 600 +window_height 134 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_logical_filtered true +window_physical_filtered true +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Nanoseconds +window_maximum_y 34405654.000000000000 +window_minimum_y 28247.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 0.276616926766 +window_end_time_relative 0.276616926766 +window_object appl { 1, { All } } +window_begin_time_relative 0.261615159840 +window_open false +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Next Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, None}, {to_obj, None}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 54000006 +window_filter_module evt_type_label 1 "Elapsed time in MPI" + +################################################################################ +< NEW DISPLAYING WINDOW time outside MPI > +################################################################################ +window_name time outside MPI +window_type composed +window_id 3 +window_factors 1.000000000000 1.000000000000 +window_operation substract +window_identifiers 1 2 +window_position_x 367 +window_position_y 385 +window_width 600 +window_height 134 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_color_mode window_in_null_gradient_mode +window_units Microseconds +window_maximum_y 472609042.000000000000 +window_minimum_y 2905.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open true +window_drawmode draw_maximum +window_drawmode_rows draw_maximum +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 5, { {appl, Adding}, {task, Adding}, {node, Adding}, {system, Adding}, {workload, Adding}, } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + diff --git a/cfgs/timings.cfg b/cfgs/timings.cfg new file mode 100644 index 0000000..d2061ef --- /dev/null +++ b/cfgs/timings.cfg @@ -0,0 +1,72 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 + + +################################################################################ +< NEW DISPLAYING WINDOW States > +################################################################################ +window_name States +window_type single +window_id 1 +window_position_x 393 +window_position_y 88 +window_width 767 +window_height 314 +window_comm_lines_enabled true +window_flags_enabled false +window_noncolor_mode true +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Microseconds +window_maximum_y 18.000000000000 +window_minimum_y 0.000000000000 +window_compute_y_max false +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open false +window_drawmode 1 +window_drawmode_rows 1 +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, State As Is}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, All}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, As Is}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } + +< NEW ANALYZER2D > +Analyzer2D.Name: States Histogram +Analyzer2D.X: 396 +Analyzer2D.Y: 433 +Analyzer2D.Width: 600 +Analyzer2D.Height: 300 +Analyzer2D.ControlWindow: 1 +Analyzer2D.DataWindow: 1 +Analyzer2D.Accumulator: Semantic +Analyzer2D.Statistic: Time +Analyzer2D.CalculateAll: True +Analyzer2D.HideCols: True +Analyzer2D.HorizVert: Horizontal +Analyzer2D.Color: True +Analyzer2D.SemanticColor: False +Analyzer2D.Zoom: Disabled +Analyzer2D.SortCols: False +Analyzer2D.SortCriteria: Average +Analyzer2D.Parameters: 4 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 -179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000000000 +Analyzer2D.AnalysisLimits: Alltrace +Analyzer2D.ComputeYScale: True +Analyzer2D.Minimum: 1.000000000000 +Analyzer2D.Maximum: 18.000000000000 +Analyzer2D.Delta: 1.000000000000 +Analyzer2D.ComputeGradient: True +Analyzer2D.MinimumGradient: 273554.000000000000 +Analyzer2D.MaximumGradient: 9395816375.000000000000 +Analyzer2D.PixelSize: 1 +Analyzer2D.CodeColor: False +Analyzer2D.ShowOnlyTotals: False +Analyzer2D.ShortHeaderLabels: True + diff --git a/hybridmetrics.py b/hybridmetrics.py new file mode 100644 index 0000000..4cc6abb --- /dev/null +++ b/hybridmetrics.py @@ -0,0 +1,1496 @@ +#!/usr/bin/env python3 + +"""Functions to compute the model metrics.""" + +from __future__ import print_function, division +import sys + + +from rawdata import * +from collections import OrderedDict + +# error import variables +error_import_pandas = False +error_import_seaborn = False +error_import_matplotlib = False +error_import_numpy = False + +try: + import numpy as np +except ImportError: + error_import_numpy = True + + +try: + import pandas as pd +except ImportError: + error_import_pandas = True + +try: + import seaborn as sns +except ImportError: + error_import_seaborn = True + + +try: + import matplotlib.pyplot as plt +except ImportError: + error_import_matplotlib = True + + + +# Contains all model factor entries with a printable name. +# This is used to generate and print all model factors, so, if an entry is added, +# it should be added here, too. + +other_metrics_doc = OrderedDict([('elapsed_time', 'Elapsed time (sec)'), + ('efficiency', 'Efficiency'), + ('speedup', 'Speedup'), + ('ipc', 'Average IPC'), + ('freq', 'Average frequency (GHz)'), + ('flushing', 'Flushing (%)'), + ('io_mpiio', 'MPI I/O (%)'), + ('io_posix', 'POSIX I/O (%)'), + ('io_eff', 'I/O Efficiency (%)')]) + +mod_factors_doc = OrderedDict([('global_eff', 'Global efficiency'), + ('parallel_eff', '-- Parallel efficiency'), + ('load_balance', ' -- Load balance'), + ('comm_eff', ' -- Communication efficiency'), + ('comp_scale', '-- Computation scalability'), + ('ipc_scale', ' -- IPC scalability'), + ('inst_scale', ' -- Instruction scalability'), + ('freq_scale', ' -- Frequency scalability')]) + +mod_factors_scale_plus_io_doc = OrderedDict([('comp_scale', '-- Computation scalability + I/O'), + ('ipc_scale', ' -- IPC scalability'), + ('inst_scale', ' -- Instruction scalability'), + ('freq_scale', ' -- Frequency scalability')]) + +mod_hybrid_factors_doc = OrderedDict([ + ('hybrid_eff', '-- Hybrid Parallel efficiency'), + ('mpi_parallel_eff', ' -- MPI Parallel efficiency'), + ('mpi_load_balance', ' -- MPI Load balance'), + ('mpi_comm_eff', ' -- MPI Communication efficiency'), + ('serial_eff', ' -- Serialization efficiency'), + ('transfer_eff', ' -- Transfer efficiency'), + ('omp_parallel_eff', ' -- OMP Parallel efficiency'), + ('omp_load_balance', ' -- OMP Load balance'), + ('omp_comm_eff', ' -- OMP Communication efficiency')]) + + +def create_mod_factors(trace_list): + """Creates 2D dictionary of the model factors and initializes with an empty + string. The mod_factors dictionary has the format: [mod factor key][trace]. + """ + global mod_factors_doc + mod_factors = {} + for key in mod_factors_doc: + trace_dict = {} + for trace_name in trace_list: + trace_dict[trace_name] = 0.0 + mod_factors[key] = trace_dict + + return mod_factors + + +def create_mod_factors_scale_io(trace_list): + """Creates 2D dictionary of the model factors and initializes with an empty + string. The mod_factors dictionary has the format: [mod factor key][trace]. + """ + global mod_factors_scale_plus_io_doc + mod_factors_scale_plus_io = {} + for key in mod_factors_scale_plus_io_doc: + trace_dict = {} + for trace_name in trace_list: + trace_dict[trace_name] = 0.0 + mod_factors_scale_plus_io[key] = trace_dict + + return mod_factors_scale_plus_io + + +def create_hybrid_mod_factors(trace_list): + """Creates 2D dictionary of the hybrid model factors and initializes with an empty + string. The hybrid_factors dictionary has the format: [mod factor key][trace]. + """ + global mod_hybrid_factors_doc + hybrid_factors = {} + for key in mod_hybrid_factors_doc: + trace_dict = {} + for trace_name in trace_list: + trace_dict[trace_name] = 0.0 + hybrid_factors[key] = trace_dict + + return hybrid_factors + + +def create_other_metrics(trace_list): + """Creates 2D dictionary of the other metrics and initializes with an empty + string. The other_metrics dictionary has the format: [mod factor key][trace]. + """ + global other_metrics_doc + other_metrics = {} + for key in other_metrics_doc: + trace_dict = {} + for trace_name in trace_list: + trace_dict[trace_name] = 0.0 + other_metrics[key] = trace_dict + + return other_metrics + + +def get_scaling_type(raw_data, trace_list, trace_processes, cmdl_args): + """Guess the scaling type (weak/strong) based on the useful instructions. + Computes the normalized instruction ratio for all measurements, whereas the + normalized instruction ratio is (instructions ratio / process ratio) with + the smallest run as reference. For exact weak scaling the normalized ratio + should be exactly 1 and for exact strong scaling it should be close to zero + with an upper bound of 0.5. The eps value defines the threshold to be + considered weak scaling and should give enough buffer to safely handle + non-ideal scaling. + """ + eps = 0.9 + normalized_inst_ratio = 0 + + # Check if there is only one trace. + if len(trace_list) == 1: + return 'strong' + + for trace in trace_list: + try: # except NaN + inst_ratio = float(raw_data['useful_ins'][trace]) / float(raw_data['useful_ins'][trace_list[0]]) + except: + inst_ratio = 0.0 + try: # except NaN + proc_ratio = float(trace_processes[trace]) / float(trace_processes[trace_list[0]]) + except: + proc_ratio = 'NaN' + + normalized_inst_ratio += inst_ratio / proc_ratio + + # Get the average inst increase. Ignore ratio of first trace 1.0) + normalized_inst_ratio = (normalized_inst_ratio - 1) / (len(trace_list) - 1) + + scaling_computed = '' + + if normalized_inst_ratio > eps: + scaling_computed = 'weak' + else: + scaling_computed = 'strong' + + if cmdl_args.scaling == 'auto': + if cmdl_args.debug: + print('==DEBUG== Detected ' + scaling_computed + ' scaling.') + print('') + return scaling_computed + + if cmdl_args.scaling == 'weak': + if scaling_computed == 'strong': + print('==Warning== Scaling set to weak scaling but detected strong scaling.') + print('') + return 'weak' + + if cmdl_args.scaling == 'strong': + if scaling_computed == 'weak': + print('==Warning== Scaling set to strong scaling but detected weak scaling.') + print('') + return 'strong' + + print('==Error== reached undefined control flow state.') + sys.exit(1) + + +def compute_model_factors(raw_data, trace_list, trace_processes, trace_mode, list_mpi_procs_count, cmdl_args): + """Computes the model factors from the gathered raw data and returns the + according dictionary of model factors.""" + mod_factors = create_mod_factors(trace_list) + hybrid_factors = create_hybrid_mod_factors(trace_list) + other_metrics = create_other_metrics(trace_list) + mod_factors_scale_plus_io = create_mod_factors_scale_io(trace_list) + + # Guess the weak or strong scaling + scaling = get_scaling_type(raw_data, trace_list, trace_processes, cmdl_args) + + # Loop over all traces + for trace in trace_list: + + proc_ratio = float(trace_processes[trace]) / float(trace_processes[trace_list[0]]) + total_procs = trace_processes[trace] + + # Flushing measurements + try: # except NaN + other_metrics['flushing'][trace] = float(raw_data['flushing_tot'][trace]) \ + / (raw_data['runtime'][trace] * total_procs) * 100.0 + except: + other_metrics['flushing'][trace] = 0.0 + + # I/O measurements + try: # except NaN + other_metrics['io_mpiio'][trace] = float(raw_data['mpiio_tot'][trace]) \ + / (raw_data['runtime'][trace] * total_procs) * 100.0 + except: + other_metrics['io_mpiio'][trace] = 0.0 + + try: # except NaN + other_metrics['io_posix'][trace] = float(raw_data['io_tot'][trace]) \ + / (raw_data['runtime'][trace] * total_procs) * 100.0 + except: + other_metrics['io_posix'][trace] = 0.0 + try: # except NaN + io_total = float(raw_data['mpiio_tot'][trace] + raw_data['flushing_tot'][trace] + + raw_data['io_tot'][trace]) + other_metrics['io_eff'][trace] = float(raw_data['useful_tot'][trace]) \ + / (raw_data['useful_tot'][trace] + io_total) * 100.0 + except: + other_metrics['io_eff'][trace] = 0.0 + + # Basic efficiency factors + try: # except NaN + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: + print("Hello, I pass for this conditional", trace) + mod_factors['load_balance'][trace] = float(raw_data['useful_plus_io_avg'][trace]) \ + / float(raw_data['useful_plus_io_max'][trace]) * 100.0 + else: + mod_factors['load_balance'][trace] = float(raw_data['useful_avg'][trace]) \ + / float(raw_data['useful_max'][trace]) * 100.0 + except: + mod_factors['load_balance'][trace] = 'NaN' + + try: # except NaN + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: + mod_factors['comm_eff'][trace] = float(raw_data['useful_plus_io_max'][trace] \ + / raw_data['runtime'][trace] * 100.0) + else: + mod_factors['comm_eff'][trace] = float(raw_data['useful_max'][trace] \ + / raw_data['runtime'][trace] * 100.0) + except: + mod_factors['comm_eff'][trace] = 'NaN' + + try: # except NaN + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: + mod_factors['parallel_eff'][trace] = float(raw_data['useful_plus_io_avg'][trace]) \ + / float(raw_data['runtime'][trace]) * 100.0 + else: + mod_factors['parallel_eff'][trace] = float(mod_factors['load_balance'][trace]) \ + * float(mod_factors['comm_eff'][trace]) / 100.0 + except: + mod_factors['parallel_eff'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + if scaling == 'strong': + mod_factors['comp_scale'][trace] = float(raw_data['useful_tot'][trace_list[0]]) \ + / float(raw_data['useful_tot'][trace]) * 100.0 + else: + mod_factors['comp_scale'][trace] = float(raw_data['useful_tot'][trace_list[0]]) \ + / float(raw_data['useful_tot'][trace]) * proc_ratio * 100.0 + else: + mod_factors['comp_scale'][trace] = 'Non-Avail' + except: + mod_factors['comp_scale'][trace] = 'NaN' + + # Computation Scale + Serial I/O + try: # except NaN + if len(trace_list) > 1: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: + io_serial_0 = float(raw_data['io_tot'][trace_list[0]] + raw_data['flushing_tot'][trace_list[0]]) + io_serial_n = float(raw_data['io_tot'][trace] + raw_data['flushing_tot'][trace]) + if scaling == 'strong': + mod_factors_scale_plus_io['comp_scale'][trace] = float((raw_data['useful_tot'][trace_list[0]] + + io_serial_0) / (raw_data['useful_tot'][trace] + + io_serial_n) * 100.0) + else: + mod_factors_scale_plus_io['comp_scale'][trace] = (float(raw_data['useful_tot'][trace_list[0]]) + + io_serial_0) \ + / (float(raw_data['useful_tot'][trace]) + + io_serial_n) * proc_ratio * 100.0 + else: + mod_factors_scale_plus_io['comp_scale'][trace] = float(mod_factors['comp_scale'][trace]) + else: + mod_factors_scale_plus_io['comp_scale'][trace] = 'Non-Avail' + except: + mod_factors_scale_plus_io['comp_scale'][trace] = 'NaN' + + + try: # except NaN + if len(trace_list) > 1: + mod_factors['global_eff'][trace] = float(mod_factors['parallel_eff'][trace]) \ + * float(mod_factors['comp_scale'][trace]) / 100.0 + else: + mod_factors['global_eff'][trace] = float(mod_factors['parallel_eff'][trace]) * 100.0 / 100.0 + except: + mod_factors['global_eff'][trace] = 'NaN' + + # Hybrid metrics calculation + # -------> MPI metrics + try: # except NaN + hybrid_factors['mpi_load_balance'][trace] = float(raw_data['outsidempi_avg'][trace]) \ + / float(raw_data['outsidempi_max'][trace]) * 100.0 + except: + hybrid_factors['mpi_load_balance'][trace] = 'NaN' + + try: # except NaN + hybrid_factors['mpi_comm_eff'][trace] = float(raw_data['outsidempi_max'][trace]) \ + / float(raw_data['runtime'][trace]) * 100.0 + except: + hybrid_factors['mpi_comm_eff'][trace] = 'NaN' + + # ------------> BEGIN MPI communication sub-metrics + try: # except NaN + hybrid_factors['serial_eff'][trace] = float(raw_data['outsidempi_dim'][trace]) \ + / float(raw_data['runtime_dim'][trace]) * 100.0 + if hybrid_factors['serial_eff'][trace] > 100.0: + hybrid_factors['serial_eff'][trace] = 'Warning!' + except: + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + hybrid_factors['serial_eff'][trace] = 'NaN' + else: + hybrid_factors['serial_eff'][trace] = 'Non-Avail' + + try: # except NaN + if hybrid_factors['serial_eff'][trace] != 'Warning!': + hybrid_factors['transfer_eff'][trace] = float(hybrid_factors['mpi_comm_eff'][trace]) \ + / float(hybrid_factors['serial_eff'][trace]) * 100.0 + else: + hybrid_factors['transfer_eff'][trace] = float(raw_data['runtime_dim'][trace]) \ + / float(raw_data['runtime'][trace]) * 100.0 + if hybrid_factors['transfer_eff'][trace] > 100.0: + hybrid_factors['transfer_eff'][trace] = 'Warning!' + except: + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + hybrid_factors['transfer_eff'][trace] = 'NaN' + else: + hybrid_factors['transfer_eff'][trace] = 'Non-Avail' + # --------------> END MPI communication sub-metrics + try: # except NaN + if raw_data['outsidempi_tot'][trace] == raw_data['outsidempi_tot_diff'][trace]: + hybrid_factors['mpi_parallel_eff'][trace] = float(raw_data['outsidempi_tot'][trace] / + (raw_data['runtime'][trace] + * list_mpi_procs_count[trace]) * 100.0) + else: + hybrid_factors['mpi_parallel_eff'][trace] = float(raw_data['outsidempi_tot_diff'][trace] + / (raw_data['runtime'][trace] + * trace_processes[trace]) * 100.0) + except: + hybrid_factors['mpi_parallel_eff'][trace] = 'NaN' + + # -------> Metrics for the second parallel paradigm + try: # except NaN + hybrid_factors['omp_comm_eff'][trace] = float(mod_factors['comm_eff'][trace]) \ + / float(hybrid_factors['mpi_comm_eff'][trace]) * 100.0 + except: + hybrid_factors['omp_comm_eff'][trace] = 'NaN' + + try: # except NaN + hybrid_factors['omp_load_balance'][trace] = float(mod_factors['load_balance'][trace]) \ + / float(hybrid_factors['mpi_load_balance'][trace]) * 100.0 + except: + hybrid_factors['omp_load_balance'][trace] = 'NaN' + + try: # except NaN + hybrid_factors['omp_parallel_eff'][trace] = float(mod_factors['parallel_eff'][trace]) \ + / float(hybrid_factors['mpi_parallel_eff'][trace]) * 100.0 + except: + hybrid_factors['omp_parallel_eff'][trace] = 'NaN' + + # -------> Global Hybrid Metric + try: # except NaN + hybrid_factors['hybrid_eff'][trace] = float(hybrid_factors['mpi_parallel_eff'][trace]) \ + * float(hybrid_factors['omp_parallel_eff'][trace]) / 100.0 + except: + hybrid_factors['hybrid_eff'][trace] = 'NaN' + + # Basic scalability factors + try: # except NaN + other_metrics['ipc'][trace] = float(raw_data['useful_ins'][trace]) \ + / float(raw_data['useful_cyc'][trace]) + except: + other_metrics['ipc'][trace] = 'NaN' + try: # except NaN + if len(trace_list) > 1: + mod_factors['ipc_scale'][trace] = float(other_metrics['ipc'][trace]) \ + / float(other_metrics['ipc'][trace_list[0]]) * 100.0 + else: + mod_factors['ipc_scale'][trace] = 'Non-Avail' + except: + mod_factors['ipc_scale'][trace] = 'NaN' + + # IPC scale + Serial I/O + try: # except NaN + if len(trace_list) > 1: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: + ipc_serial_io_0 = (raw_data['useful_ins'][trace_list[0]] + raw_data['io_ins'][trace_list[0]] + + raw_data['flushing_ins'][trace_list[0]]) \ + / (raw_data['useful_cyc'][trace_list[0]] + raw_data['io_cyc'][trace_list[0]] + + raw_data['flushing_cyc'][trace_list[0]]) + + ipc_serial_io_n = (raw_data['useful_ins'][trace] + raw_data['io_ins'][trace] + + raw_data['flushing_ins'][trace])\ + / (raw_data['useful_cyc'][trace] + raw_data['io_cyc'][trace] + + raw_data['flushing_cyc'][trace]) + mod_factors_scale_plus_io['ipc_scale'][trace] = float(ipc_serial_io_n / ipc_serial_io_0 * 100.0) + else: + mod_factors_scale_plus_io['ipc_scale'][trace] = float(mod_factors['ipc_scale'][trace]) + else: + mod_factors_scale_plus_io['ipc_scale'][trace] = 'Non-Avail' + except: + mod_factors_scale_plus_io['ipc_scale'][trace] = 'NaN' + + try: # except NaN + other_metrics['freq'][trace] = float(raw_data['useful_cyc'][trace]) \ + / float(raw_data['useful_tot'][trace]) / 1000 + except: + other_metrics['freq'][trace] = 'NaN' + try: # except NaN + if len(trace_list) > 1: + mod_factors['freq_scale'][trace] = float(other_metrics['freq'][trace]) \ + / float(other_metrics['freq'][trace_list[0]]) * 100.0 + else: + mod_factors['freq_scale'][trace] = 'Non-Avail' + except: + mod_factors['freq_scale'][trace] = 'NaN' + + # freq scale + Serial I/O + try: # except NaN + if len(trace_list) > 1: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: + freq_serial_io_0 = (float(raw_data['useful_cyc'][trace_list[0]]) + + float(raw_data['io_cyc'][trace_list[0]]) + + float(raw_data['flushing_cyc'][trace_list[0]]))\ + / (raw_data['useful_tot'][trace_list[0]] + + float(raw_data['io_tot'][trace_list[0]]) + + float(raw_data['flushing_tot'][trace_list[0]])) / 1000 + + freq_serial_io_n = (float(raw_data['useful_cyc'][trace]) + + float(raw_data['io_cyc'][trace]) + + float(raw_data['flushing_cyc'][trace]))\ + / (float(raw_data['useful_tot'][trace]) + + float(raw_data['io_tot'][trace]) + + float(raw_data['flushing_tot'][trace])) / 1000 + mod_factors_scale_plus_io['freq_scale'][trace] = float(freq_serial_io_n / freq_serial_io_0 * 100.0) + else: + mod_factors_scale_plus_io['freq_scale'][trace] = float(mod_factors['freq_scale'][trace]) + else: + mod_factors_scale_plus_io['freq_scale'][trace] = 'Non-Avail' + except: + mod_factors_scale_plus_io['freq_scale'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + if scaling == 'strong': + mod_factors['inst_scale'][trace] = float(raw_data['useful_ins'][trace_list[0]])\ + / float(raw_data['useful_ins'][trace]) * 100.0 + else: + mod_factors['inst_scale'][trace] = float(raw_data['useful_ins'][trace_list[0]]) \ + / float(raw_data['useful_ins'][trace]) * proc_ratio * 100.0 + else: + mod_factors['inst_scale'][trace] = 'Non-Avail' + except: + mod_factors['inst_scale'][trace] = 'NaN' + + # ins scale + Serial I/O + try: # except NaN + if len(trace_list) > 1: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: + useful_ins_plus_io_0 = float(raw_data['useful_ins'][trace_list[0]]) \ + + float(raw_data['io_ins'][trace_list[0]]) \ + + float(raw_data['flushing_ins'][trace_list[0]]) + useful_ins_plus_io_n = float(raw_data['useful_ins'][trace]) \ + + float(raw_data['io_ins'][trace]) \ + + float(raw_data['flushing_ins'][trace]) + if scaling == 'strong': + mod_factors_scale_plus_io['inst_scale'][trace] = float(useful_ins_plus_io_0) \ + / float(useful_ins_plus_io_n) * 100.0 + else: + mod_factors_scale_plus_io['inst_scale'][trace] = float(useful_ins_plus_io_0) \ + / float(useful_ins_plus_io_n)\ + * proc_ratio * 100.0 + else: + mod_factors_scale_plus_io['inst_scale'][trace] = float(mod_factors['inst_scale'][trace]) + else: + mod_factors_scale_plus_io['inst_scale'][trace] = 'Non-Avail' + except: + mod_factors_scale_plus_io['inst_scale'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + if scaling == 'strong': + other_metrics['speedup'][trace] = float(raw_data['runtime'][trace_list[0]] + / raw_data['runtime'][trace]) + else: + other_metrics['speedup'][trace] = float(raw_data['runtime'][trace_list[0]] + / raw_data['runtime'][trace] * proc_ratio) + else: + other_metrics['speedup'][trace] = 'Non-Avail' + except: + other_metrics['speedup'][trace] = 'NaN' + + try: # except NaN + other_metrics['elapsed_time'][trace] = float(raw_data['runtime'][trace] * 0.000001) + except: + other_metrics['elapsed_time'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + #other_metrics['efficiency'][trace] = other_metrics['speedup'][trace] / proc_ratio + if scaling == 'strong': + other_metrics['efficiency'][trace] = raw_data['runtime'][trace_list[0]] \ + / (raw_data['runtime'][trace] * proc_ratio) + else: + other_metrics['efficiency'][trace] = raw_data['runtime'][trace_list[0]] \ + / raw_data['runtime'][trace] + else: + other_metrics['efficiency'][trace] = 'Non-Avail' + except: + other_metrics['efficiency'][trace] = 'NaN' + + return mod_factors, mod_factors_scale_plus_io, hybrid_factors, other_metrics + + +def print_mod_factors_table(mod_factors, other_metrics, mod_factors_scale_plus_io, hybrid_factors, trace_list, + trace_processes, trace_tasks, trace_threads,trace_mode): + """Prints the model factors table in human readable form on stdout.""" + global mod_factors_doc, mod_hybrid_factors_doc + + warning_io = [] + warning_flush = [] + warning_flush_wrong = [] + warning_simulation = [] + for trace in trace_list: + if 10.0 <= other_metrics['flushing'][trace] < 15.0: + warning_flush.append(1) + elif other_metrics['flushing'][trace] >= 15.0: + warning_flush_wrong.append(1) + if other_metrics['io_posix'][trace] >= 5.0: + warning_io.append(1) + if hybrid_factors['serial_eff'][trace] == "Warning!" \ + or hybrid_factors['serial_eff'][trace] == "Warning!": + warning_simulation.append(1) + + if len(warning_flush_wrong) > 0: + print("WARNING! Flushing in a trace is too high. Disabling standard output metrics...") + print(" Flushing is an overhead due to the tracer, please review your trace.") + print('') + return + + # Update the hybrid parallelism mode + trace_mode_doc = trace_mode[trace_list[0]] + if trace_mode_doc[0:len("Detailed+MPI+")] == "Detailed+MPI+": + mod_hybrid_factors_doc['omp_parallel_eff'] = " -- " + \ + trace_mode_doc[len("Detailed+MPI+"):] + " Parallel efficiency" + mod_hybrid_factors_doc['omp_load_balance'] = " -- " + \ + trace_mode_doc[len("Detailed+MPI+"):] + " Load Balance" + mod_hybrid_factors_doc['omp_comm_eff'] = " -- " + \ + trace_mode_doc[len("Detailed+MPI+"):] + " Communication efficiency" + + # print('') + print('\n Overview of the Efficiency metrics:') + + longest_name = len(sorted(mod_hybrid_factors_doc.values(), key=len)[-1]) + + line = ''.rjust(longest_name) + if len(trace_list) == 1: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[0]] + else: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[len(trace_list)-1]] + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # BEGIN To adjust header to big number of processes + procs_header = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if limit_min == limit_max and same_procs and len(trace_list) > 1: + s_xtics = (str(trace_processes[trace]) + '[' + str(index+1) + ']') + else: + s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') + if s_xtics in procs_header: + s_xtics += '[' + str(index + 1) + ']' + procs_header.append(s_xtics) + + max_len_header = 0 + for proc_h in procs_header: + if max_len_header < len(proc_h): + max_len_header = len(proc_h) + + value_to_adjust = 10 + if max_len_header > value_to_adjust: + value_to_adjust = max_len_header + 1 + # END To adjust header to big number of processes + + label_xtics = [] + for index, trace in enumerate(trace_list): + line += ' | ' + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if limit_min == limit_max and same_procs and len(trace_list) > 1: + s_xtics = (str(trace_processes[trace]) + '[' + str(index+1) + ']') + else: + s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') + if s_xtics in label_xtics: + s_xtics += '[' + str(index + 1) + ']' + label_xtics.append(s_xtics) + line += s_xtics.rjust(value_to_adjust) + + print(''.ljust(len(line), '=')) + print(line) + line_procs_factors = line + + print(''.ljust(len(line), '=')) + + for mod_key in mod_factors_doc: + line = mod_factors_doc[mod_key].ljust(longest_name) + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(mod_factors[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(mod_factors[mod_key][trace])).rjust(value_to_adjust) + print(line) + + io_metrics = [] + io_metrics.append(0) + for mod_key in mod_factors_scale_plus_io_doc: + for trace in trace_list: + if mod_factors_scale_plus_io[mod_key][trace] != 0: + io_metrics.append(1) + + if (len(warning_flush) >= 1 or len(warning_io) >= 1) and len(trace_list) > 1: + print(''.ljust(len(line_procs_factors), '-')) + for mod_key in mod_factors_scale_plus_io_doc: + line = mod_factors_scale_plus_io_doc[mod_key].ljust(longest_name) + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(mod_factors_scale_plus_io[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(mod_factors_scale_plus_io[mod_key][trace])).rjust(value_to_adjust) + print(line) + print(''.ljust(len(line_procs_factors), '=')) + else: + print(''.ljust(len(line_procs_factors), '-')) + + for mod_key in mod_hybrid_factors_doc: + line = mod_hybrid_factors_doc[mod_key].ljust(longest_name) + for trace in trace_list: + line += ' | ' + try: # except NaN + if str(hybrid_factors[mod_key][trace]) != 'nan': + line += ('{0:.2f}%'.format(hybrid_factors[mod_key][trace])).rjust(value_to_adjust) + elif str(hybrid_factors[mod_key][trace]) == 'nan': + line += ('{}'.format('NaN')).rjust(value_to_adjust) + else: + line += ('{}'.format(hybrid_factors[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(hybrid_factors[mod_key][trace])).rjust(value_to_adjust) + print(line) + + print(''.ljust(len(line_procs_factors), '=')) + if len(warning_simulation) > 0: + print("===> Warning! Metrics obtained from simulated traces exceed 100%. " + "Please review original and simulated traces.") + print('') + + +def print_other_metrics_table(other_metrics, trace_list, trace_processes, trace_tasks, trace_threads): + """Prints the other metrics table in human readable form on stdout.""" + global other_metrics_doc + + print('Overview of the Efficiency, Speedup, IPC and Frequency:') + + longest_name = len(sorted(other_metrics_doc.values(), key=len)[-1]) + + line = ''.rjust(longest_name) + if len(trace_list) == 1: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[0]] + else: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[len(trace_list)-1]] + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # BEGIN To adjust header to big number of processes + procs_header = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if limit_min == limit_max and same_procs and len(trace_list) > 1: + s_xtics = (str(trace_processes[trace]) + '[' + str(index+1) + ']') + else: + s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') + if s_xtics in procs_header: + s_xtics += '[' + str(index + 1) + ']' + procs_header.append(s_xtics) + + max_len_header = 0 + for proc_h in procs_header: + if max_len_header < len(proc_h): + max_len_header = len(proc_h) + + value_to_adjust = 10 + if max_len_header > value_to_adjust: + value_to_adjust = max_len_header + 1 + # END To adjust header to big number of processes + + label_xtics = [] + for index, trace in enumerate(trace_list): + line += ' | ' + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if limit_min == limit_max and same_procs and len(trace_list) > 1: + s_xtics = (str(trace_processes[trace]) + '[' + str(index+1) + ']') + else: + s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') + if s_xtics in label_xtics: + s_xtics += '[' + str(index + 1) + ']' + label_xtics.append(s_xtics) + line += s_xtics.rjust(value_to_adjust) + + print(''.ljust(len(line), '-')) + print(line) + line_head = line + print(''.ljust(len(line), '-')) + + for mod_key in other_metrics_doc: + line = other_metrics_doc[mod_key].ljust(longest_name) + if len(trace_list) > 1: + if mod_key in ['speedup', 'ipc', 'freq', 'elapsed_time', 'efficiency']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + print(line) + else: + if mod_key in ['ipc', 'freq', 'elapsed_time']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + print(line) + print(''.ljust(len(line_head), '-')) + # print('') + + warning_io = [] + warning_flush = [] + for trace in trace_list: + if other_metrics['flushing'][trace] >= 10.0: + warning_flush.append(1) + if other_metrics['io_mpiio'][trace] >= 5.0 or other_metrics['io_posix'][trace] >= 5.0: + warning_io.append(1) + + for mod_key in other_metrics_doc: + line = other_metrics_doc[mod_key].ljust(longest_name) + # Print empty line to separate values + if mod_key in ['freq'] and len(warning_flush) > 0: + print('') + print("Overview of tracer\'s flushing weight:") + print(''.ljust(len(line_head), '-')) + + if mod_key not in ['speedup', 'ipc', 'freq', 'elapsed_time', 'efficiency']: + if mod_key in ['flushing']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + if len(warning_flush) > 0: + print(line) + print(''.ljust(len(line_head), '-')) + elif mod_key in ['io_mpiio','io_posix','io_eff']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + if len(warning_io) > 0: + print(line) + + # Print headers I/O + if mod_key in ['flushing'] and len(warning_io) > 0: + print(''.ljust(len(line), ' ')) + print('Overview of File I/O weight:') + print(''.ljust(len(line), '-')) + if mod_key in ['io_eff'] and len(warning_io) > 0: + print(''.ljust(len(line), '-')) + # print('') + + if len(warning_flush) > 0: + message_warning_flush = "WARNING! %Flushing is high and affects computation of efficiency metrics." + else: + message_warning_flush = "" + if len(warning_io) > 0: + message_warning_io = "WARNING! % File I/O is high and affects computation of efficiency metrics." + else: + message_warning_io = "" + print(message_warning_flush+message_warning_io) + # print('') + + +def print_efficiency_table(mod_factors, hybrid_factors, trace_list, trace_processes, trace_tasks, trace_threads): + """Prints the model factors table in human readable form on stdout.""" + global mod_factors_doc, mod_hybrid_factors_doc + + delimiter = ',' + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + if len(trace_list) == 1: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[0]] + else: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[len(trace_list) - 1]] + + file_path = os.path.join(os.getcwd(), 'efficiency_table-global.csv') + with open(file_path, 'w') as output: + line = '\"Number of processes\"' + + label_xtics = [] + for index, trace in enumerate(trace_list): + line += delimiter + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if limit_min == limit_max and same_procs and len(trace_list) > 1: + s_xtics = (str(trace_processes[trace]) + '[' + str(index + 1) + ']') + else: + s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') + if s_xtics in label_xtics: + s_xtics += '[' + str(index + 1) + ']' + label_xtics.append(s_xtics) + line += s_xtics + + output.write(line + '\n') + + for mod_key in mod_factors_doc: + if mod_key not in ['speedup', 'ipc', 'freq', 'elapsed_time', 'efficiency', 'flushing', 'io_mpiio', 'io_posix']: + if mod_key in ['parallel_eff', 'comp_scale']: + line = "\"" + mod_factors_doc[mod_key].replace(' ', '', 2) + "\"" + elif mod_key in ['load_balance', 'comm_eff','ipc_scale', 'inst_scale','freq_scale']: + line = "\"" + mod_factors_doc[mod_key].replace(' ', '', 2) + "\"" + else: + line = "\"" + mod_factors_doc[mod_key] + "\"" + for trace in trace_list: + line += delimiter + try: # except NaN + if mod_factors[mod_key][trace] == "Non-Avail" or mod_factors[mod_key][trace] == "Warning!": + line += '0.00' + else: + line += '{0:.2f}'.format(mod_factors[mod_key][trace]) + except ValueError: + line += '{}'.format(mod_factors[mod_key][trace]) + output.write(line + '\n') + + # Create Gnuplot file for efficiency plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'efficiency_table-global.gp') + content = [] + + with open(gp_template) as f: + content = f.readlines() + + limit_procs = 500 + len(trace_list) * 65 + ## print(limit_procs) + + # Replace xrange + content = [line.replace('#REPLACE_BY_SIZE', ''.join(['set terminal pngcairo enhanced dashed crop size ', + str(limit_procs), ',460 font "Latin Modern Roman,14"'])) + for line in content] + + file_path = os.path.join(os.getcwd(), 'efficiency_table_global.gp') + with open(file_path, 'w') as f: + f.writelines(content) + # print('======== Plot (gnuplot File): EFFICIENCY Table ========') + if len(trace_list) > 1: + print('Global Efficiency Table written to ' + file_path[:len(file_path) - 3] + '.gp') + # print('') + + delimiter = ',' + file_path = os.path.join(os.getcwd(), 'efficiency_table-hybrid.csv') + with open(file_path, 'w') as output: + line = '\"Number of processes\"' + label_xtics = [] + for index, trace in enumerate(trace_list): + line += delimiter + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if limit_min == limit_max and same_procs and len(trace_list) > 1: + s_xtics = (str(trace_processes[trace]) + '[' + str(index + 1) + ']') + else: + s_xtics = (str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')') + if s_xtics in label_xtics: + s_xtics += '[' + str(index + 1) + ']' + label_xtics.append(s_xtics) + line += s_xtics + output.write(line + '\n') + + for mod_key in mod_hybrid_factors_doc: + if mod_key in ['mpi_parallel_eff', 'omp_parallel_eff']: + line = "\"" + mod_hybrid_factors_doc[mod_key].replace(' ', '', 2) + "\"" + elif mod_key in ['mpi_load_balance', 'mpi_comm_eff','omp_load_balance', 'omp_comm_eff']: + line = "\"" + mod_hybrid_factors_doc[mod_key].replace(' ', '', 2) + "\"" + elif mod_key in ['serial_eff', 'transfer_eff']: + line = "\"" + mod_hybrid_factors_doc[mod_key].replace(' ', ' ', 2) + "\"" + else: + line = "\"" + mod_hybrid_factors_doc[mod_key] + "\"" + for trace in trace_list: + line += delimiter + try: # except NaN + if hybrid_factors[mod_key][trace] == "Non-Avail" or hybrid_factors[mod_key][trace] == "Warning!": + line += '0.00' + else: + line += '{0:.2f}'.format(hybrid_factors[mod_key][trace]) + except ValueError: + line += '{}'.format(hybrid_factors[mod_key][trace]) + output.write(line + '\n') + + # Create Gnuplot file for efficiency plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'efficiency_table-hybrid.gp') + content = [] + + with open(gp_template) as f: + content = f.readlines() + + limit_procs = 510 + len(trace_list) * 80 + + # Replace xrange + content = [line.replace('#REPLACE_BY_SIZE', ''.join(['set terminal pngcairo enhanced dashed crop size ', + str(limit_procs), ',460 font "Latin Modern Roman,14"'])) + for line in content] + + file_path = os.path.join(os.getcwd(), 'efficiency_table-hybrid.gp') + with open(file_path, 'w') as f: + f.writelines(content) + # print('======== Plot (gnuplot File): EFFICIENCY Table ========') + if len(trace_list) > 1: + print('Hybrid Efficiency Table written to ' + file_path[:len(file_path) - 3] + '.gp') + #print('') + + +def print_mod_factors_csv(mod_factors, hybrid_factors, trace_list, trace_processes): + """Prints the model factors table in a csv file.""" + global mod_factors_doc, mod_hybrid_factors_doc + + delimiter = ';' + # File is stored in the trace directory + # file_path = os.path.join(os.path.dirname(os.path.realpath(trace_list[0])), 'modelfactors.csv') + # File is stored in the execution directory + file_path = os.path.join(os.getcwd(), 'modelfactors.csv') + + with open(file_path, 'w') as output: + line = "\"Number of processes\"" + for trace in trace_list: + line += delimiter + line += str(trace_processes[trace]) + output.write(line + '\n') + + for mod_key in mod_factors_doc: + line = "\"" + mod_factors_doc[mod_key].replace(' ', '', 2) + "\"" + for trace in trace_list: + line += delimiter + try: # except NaN + line += '{0:.6f}'.format(mod_factors[mod_key][trace]) + except ValueError: + line += '{}'.format(mod_factors[mod_key][trace]) + output.write(line + '\n') + + for mod_key in mod_hybrid_factors_doc: + line = "\"" + mod_hybrid_factors_doc[mod_key].replace(' ', '', 2) + "\"" + for trace in trace_list: + line += delimiter + try: # except NaN + line += '{0:.6f}'.format(hybrid_factors[mod_key][trace]) + except ValueError: + line += '{}'.format(hybrid_factors[mod_key][trace]) + output.write(line + '\n') + + output.write('#\n') + + print('======== Output Files: Metrics and Plots ========') + print('Model factors written to ' + file_path) + + +def print_other_metrics_csv(other_metrics, trace_list, trace_processes): + """Prints the model factors table in a csv file.""" + global other_metrics_doc + + delimiter = ';' + # File is stored in the trace directory + # file_path = os.path.join(os.path.dirname(os.path.realpath(trace_list[0])), 'modelfactors.csv') + # File is stored in the execution directory + file_path = os.path.join(os.getcwd(), 'other_metrics.csv') + + with open(file_path, 'w') as output: + line = 'Number of processes' + for trace in trace_list: + line += delimiter + line += str(trace_processes[trace]) + output.write(line + '\n') + + for mod_key in other_metrics_doc: + line = other_metrics_doc[mod_key].replace(' ', '', 2) + for trace in trace_list: + line += delimiter + try: # except NaN + line += '{0:.6f}'.format(other_metrics[mod_key][trace]) + except ValueError: + line += '{}'.format(other_metrics[mod_key][trace]) + output.write(line + '\n') + + output.write('#\n') + + print('') + print('======== Output File: Other Metrics ========') + print('Speedup, IPC, Frequency, I/O and Flushing written to ' + file_path) + + +def plots_efficiency_table_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args): + # Plotting using python + # For plotting using python, read the csv file + + file_path = os.path.join(os.getcwd(), 'efficiency_table-hybrid.csv') + df = pd.read_csv(file_path) + metrics = df['Number of processes'].tolist() + + traces_procs = list(df.keys())[1:] + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # Set limit for projection + if cmdl_args.limit: + limit = cmdl_args.limit + else: + limit = str(trace_processes[trace_list[len(trace_list)-1]]) + + limit_min = trace_processes[trace_list[0]] + + # To xticks label + label_xtics = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if int(limit) == int(limit_min) and same_procs: + s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' + else: + s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' + if s_xtics in label_xtics: + s_xtics += '[' + str(index + 1) + ']' + label_xtics.append(s_xtics) + ##### End xticks + + # BEGIN To adjust header to big number of processes + max_len_header = 7 + for labelx in label_xtics: + if len(labelx) > max_len_header: + max_len_header = len(labelx) + + # END To adjust header to big number of processes + + list_data = [] + for index, rows in df.iterrows(): + list_temp = [] + for value in list(rows)[1:]: + if float(value) == 0.0: + list_temp.append(np.nan) + else: + list_temp.append(float(value)) + # print(list_temp) + list_data.append(list_temp) + + list_np = np.array(list_data) + + idx = metrics + cols = label_xtics + # cols = traces_procs + df = pd.DataFrame(list_np, index=idx, columns=cols) + + # min for 1 trace is x=3 for the (x,y) in figsize + size_figure_y = len(idx) * 0.40 + size_figure_x = len(cols) * 0.14 * max_len_header + plt.figure(figsize=(size_figure_x, size_figure_y)) + + ax = sns.heatmap(df, cmap='RdYlGn', linewidths=0.05, annot=True, vmin=0, vmax=100, center=75, \ + fmt='.2f', annot_kws={"size": 10}, cbar_kws={'label': 'Percentage(%)'}) + ## to align ylabels to left + plt.yticks(rotation=0, ha='left') + + ax.xaxis.tick_top() + # to adjust metrics + len_pad = 0 + for metric in metrics: + if len(metric) > len_pad: + len_pad = len(metric) + + ax.yaxis.set_tick_params(pad=len_pad + 164) + + plt.savefig('efficiency_table-hybrid-matplot.png', bbox_inches='tight') + + # General Metrics plot + + file_path = os.path.join(os.getcwd(), 'efficiency_table-global.csv') + df = pd.read_csv(file_path) + metrics = df['Number of processes'].tolist() + + traces_procs = list(df.keys())[1:] + + list_data = [] + for index, rows in df.iterrows(): + list_temp = [] + for value in list(rows)[1:]: + if float(value) == 0.0: + list_temp.append(np.nan) + else: + list_temp.append(float(value)) + # print(list_temp) + list_data.append(list_temp) + + list_np = np.array(list_data) + + idx = metrics + cols = label_xtics + # cols = traces_procs + df = pd.DataFrame(list_np, index=idx, columns=cols) + + # min for 1 traces is x=3 for the (x,y) in figsize + size_figure_y = len(idx) * 0.40 + size_figure_x = len(cols) * 0.14 * max_len_header + plt.figure(figsize=(size_figure_x, size_figure_y)) + + ax = sns.heatmap(df, cmap='RdYlGn', linewidths=0.05, annot=True, vmin=0, vmax=100, center=75, \ + fmt='.2f', annot_kws={"size": 10}, cbar_kws={'label': 'Percentage(%)'}) + ## to align ylabels to left + plt.yticks(rotation=0, ha='left') + ax.xaxis.tick_top() + # to adjust metrics + len_pad = 0 + for metric in metrics: + if len(metric) > len_pad: + len_pad = len(metric) + + ax.yaxis.set_tick_params(pad=len_pad + 140) + + plt.savefig('efficiency_table-global-matplot.png', bbox_inches='tight') + + +def plots_modelfactors_matplot(trace_list, trace_processes,trace_tasks, trace_threads, cmdl_args): + # Plotting using python + # For plotting using python, read the csv file + file_path = os.path.join(os.getcwd(), 'modelfactors.csv') + df = pd.read_csv(file_path, sep=';') + + traces_procs = list(df.keys())[1:] + + list_data = [] + for index, rows in df.iterrows(): + list_temp = [] + for value in list(rows)[1:]: + if value != 'Non-Avail' and value != 'Warning!': + list_temp.append(float(value)) + elif value == 'Non-Avail' or value == 'Warning!': + list_temp.append('NaN') + # print(list_temp) + list_data.append(list_temp) + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # Set limit for projection + if cmdl_args.limit: + limit = cmdl_args.limit + else: + limit = str(trace_processes[trace_list[len(trace_list)-1]]) + + limit_min = trace_processes[trace_list[0]] + + # To xticks label + label_xtics = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if int(limit) == int(limit_min) and same_procs: + s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' + else: + s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' + if s_xtics in label_xtics: + s_xtics += '[' + str(index + 1) + ']' + label_xtics.append(s_xtics) + + ## Global Metrics + plt.figure() + max_global = max([max(list_data[0]), max(list_data[1]), max(list_data[2]), + max(list_data[3]), max(list_data[4])]) + plt.plot(traces_procs, list_data[0], 'o-', color='black', label='Global Efficiency') + plt.plot(traces_procs, list_data[1], 's--', color='magenta', label='Parallel Efficiency') + plt.plot(traces_procs, list_data[2], '*:', color='red', label='Load Balance') + plt.plot(traces_procs, list_data[3], 'x-.', color='green', label='Communication efficiency') + plt.plot(traces_procs, list_data[4], 'v--', color='blue', label='Computation scalability') + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency (%)") + plt.xticks(tuple(traces_procs), tuple(label_xtics)) + # print(max_global) + if float(max_global) < 100: + max_global = 100 + + plt.ylim(0, max_global+5) + plt.legend() + plt.savefig('modelfactors-global-matplot.png', bbox_inches='tight') + + ## Scale Metrics + plt.figure() + max_scale = max([max(list_data[4]), max(list_data[5]), max(list_data[6]), max(list_data[7])]) + plt.plot(traces_procs, list_data[4], 'v-', color='blue', markerfacecolor='blue', label='Computation scalability') + plt.plot(traces_procs, list_data[5], 'v--', color='skyblue', label='IPC scalability') + plt.plot(traces_procs, list_data[6], 'v:', color='gray', label='Instruction scalability') + plt.plot(traces_procs, list_data[7], 'v-.', color='darkviolet', label='Frequency scalability') + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency (%)") + plt.xticks(tuple(traces_procs), tuple(label_xtics)) + if float(max_scale) < 100: + max_scale = 100 + + plt.ylim(0, max_scale+5) + plt.legend() + plt.savefig('modelfactors-scale-matplot.png', bbox_inches='tight') + + ## Hybrid Metrics + + plt.figure() + max_hybrid = max([max(list_data[8]), max(list_data[9]), max(list_data[10]), max(list_data[11]), + max(list_data[14]), max(list_data[15]), max(list_data[16])]) + plt.plot(traces_procs, list_data[8], 's-', color='purple', label='Hybrid Parallel efficiency') + plt.plot(traces_procs, list_data[9], 'o--', color='green', label='MPI Parallel efficiency') + plt.plot(traces_procs, list_data[10], 'x-.', color='lime', label='MPI Load balance') + plt.plot(traces_procs, list_data[11], '*:', color='lightseagreen', label='MPI Communication efficiency') + plt.plot(traces_procs, list_data[14], 'h--', color='red', label='OpenMP Parallel efficiency') + plt.plot(traces_procs, list_data[15], 'v-.', color='orange', label='OpenMP Load Balance') + plt.plot(traces_procs, list_data[16], 'X:', color='salmon', label='OpenMP Communication efficiency') + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency (%)") + plt.xticks(tuple(traces_procs), tuple(label_xtics)) + if float(max_hybrid) < 100: + max_hybrid = 100 + + plt.ylim(0, max_hybrid+5) + plt.legend() + plt.savefig('modelfactors-hybrid-matplot.png', bbox_inches='tight') + + ## MPI Metrics + plt.figure() + if max(list_data[12]) != 'NaN' and max(list_data[13]) != 'NaN': + max_mpi = max([max(list_data[9]), max(list_data[10]), max(list_data[11]), + max(list_data[12]), max(list_data[13])]) + plt.plot(traces_procs, list_data[9], 's-', color='green', label='MPI Parallel efficiency') + plt.plot(traces_procs, list_data[10], 'v:', color='lime', label='MPI Load balance') + plt.plot(traces_procs, list_data[11], 'o-.', color='lightseagreen', label='MPI Communication efficiency') + plt.plot(traces_procs, list_data[12], linestyle=(0, (3, 10, 1, 10)), marker='s', color='gold', + label='Serialization efficiency') + plt.plot(traces_procs, list_data[13], linestyle=(0, (3, 5, 1, 5)), marker='x', color='tomato', label='Transfer efficiency') + elif max(list_data[12]) == 'NaN' and max(list_data[13]) == 'NaN': + max_mpi = max([max(list_data[9]), max(list_data[10]), max(list_data[11])]) + plt.plot(traces_procs, list_data[9], 's-', color='green', label='MPI Parallel efficiency') + plt.plot(traces_procs, list_data[10], 'v:', color='lime', label='MPI Load balance') + plt.plot(traces_procs, list_data[11], 'o-.', color='lightseagreen', label='MPI Communication efficiency') + elif max(list_data[12]) != 'NaN' and max(list_data[13]) == 'NaN': + max_mpi = max([max(list_data[9]), max(list_data[10]), max(list_data[11]), + max(list_data[12])]) + plt.plot(traces_procs, list_data[9], 's-', color='green', label='MPI Parallel efficiency') + plt.plot(traces_procs, list_data[10], 'v:', color='lime', label='MPI Load balance') + plt.plot(traces_procs, list_data[11], 'o-.', color='lightseagreen', label='MPI Communication efficiency') + plt.plot(traces_procs, list_data[12], linestyle=(0, (3, 10, 1, 10)), marker='s', color='gold', label='Serialization efficiency') + else: + max_mpi = max([max(list_data[9]), max(list_data[10]), max(list_data[11]),max(list_data[13])]) + plt.plot(traces_procs, list_data[9], 's-', color='green', label='MPI Parallel efficiency') + plt.plot(traces_procs, list_data[10], 'v:', color='lime', label='MPI Load balance') + plt.plot(traces_procs, list_data[11], 'o-.', color='lightseagreen', label='MPI Communication efficiency') + plt.plot(traces_procs, list_data[13], linestyle=(0, (3, 5, 1, 5)), marker='x', color='tomato', label='Transfer efficiency') + + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency (%)") + plt.xticks(tuple(traces_procs), tuple(label_xtics)) + if float(max_mpi) < 100: + max_mpi = 100 + + plt.ylim(0, max_mpi+5) + plt.legend() + plt.savefig('modelfactors-mpi-matplot.png', bbox_inches='tight') + + +def plots_speedup_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args): + # Plotting using python + # For plotting using python, read the csv file + file_path = os.path.join(os.getcwd(), 'other_metrics.csv') + df = pd.read_csv(file_path, sep=';') + + traces_procs = list(df.keys())[1:] + + list_data = [] + for index, rows in df.iterrows(): + list_temp = [] + for value in list(rows)[1:]: + if value != 'Non-Avail': + list_temp.append(float(value)) + elif value == 'Non-Avail': + list_temp.append('NaN') + # print(list_temp) + list_data.append(list_temp) + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # Set limit for projection + if cmdl_args.limit: + limit = cmdl_args.limit + else: + limit = str(trace_processes[trace_list[len(trace_list) - 1]]) + + limit_min = trace_processes[trace_list[0]] + + # proc_ratio to ideal speedup + proc_ratio = [] + for index, trace in enumerate(trace_list): + proc_ratio.append(trace_processes[trace]/trace_processes[trace_list[0]]) + # print(proc_ratio) + + # To xticks label + label_xtics = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if int(limit) == int(limit_min) and same_procs: + s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' + elif int(limit) == int(limit_min) and not same_procs: + s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' \ + + str(threads) + ')' + else: + s_xtics = str(trace_processes[trace]) + label_xtics.append(s_xtics) + + int_traces_procs = [] + prev_procs = int(float(traces_procs[0])) + int_traces_procs.append(int(float(traces_procs[0]))) + count_rep = 0 + for procs in traces_procs[1:]: + if prev_procs == int(float(procs)): + count_rep += 1 + int_traces_procs.append(int(float(procs))+ (2*count_rep)) + prev_procs = int(float(procs)) + else: + int_traces_procs.append(int(float(procs))) + prev_procs = int(float(procs)) + count_rep = 0 + + ### Plot: Global Metrics + plt.figure() + for x, y in zip(int_traces_procs, list_data[2]): + label = "{:.2f}".format(y) + plt.annotate(label, (x, y), textcoords="offset points", xytext=(0, 10), ha='center') + + plt.plot(int_traces_procs, list_data[2], 'o-', color='blue', label='measured') + plt.plot(int_traces_procs, proc_ratio, 'o-', color='black', label='ideal') + plt.xlabel("Number of Processes") + plt.ylabel("SpeedUp") + plt.xticks(tuple(int_traces_procs), tuple(label_xtics)) + #plt.yscale('log') + plt.legend() + #plt.xlim(0, ) + plt.ylim(0, ) + plt.savefig('speedup-matplot.png', bbox_inches='tight') + + ### Plot: Efficiency + # print(list_data) + plt.figure() + for x, y in zip(int_traces_procs, list_data[1]): + label = "{:.2f}".format(y) + plt.annotate(label, (x, y), textcoords="offset points", xytext=(0, 10), ha='center') + + plt.plot(int_traces_procs, list_data[1], 'o-', color='blue', label='measured') + plt.axhline(y=1, color='black', linestyle='-', label='ideal') + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency") + plt.xticks(tuple(int_traces_procs), tuple(label_xtics)) + # plt.yscale('log') + max_y = max(list_data[1]) + if max_y < 1.1: + max_y = 1.0 + #plt.xlim(0, ) + plt.ylim(0,max_y+0.1) + plt.legend() + plt.savefig('efficiency-matplot.png', bbox_inches='tight') \ No newline at end of file diff --git a/modelfactors.py b/modelfactors.py new file mode 100755 index 0000000..4eb5c6b --- /dev/null +++ b/modelfactors.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 + +"""modelfactors.py Generates performance metrics from a set of Paraver traces.""" + +from __future__ import print_function, division +import subprocess +import plots +from utils import parse_arguments, check_installation +from tracemetadata import get_traces_from_args +from rawdata import gather_raw_data, print_raw_data_table, print_raw_data_csv +from simplemetrics import compute_model_factors, print_mod_factors_csv, print_efficiency_table, \ + print_mod_factors_table, read_mod_factors_csv, print_other_metrics_table, \ + print_other_metrics_csv, plots_efficiency_table_matplot, plots_modelfactors_matplot, plots_speedup_matplot + +import hybridmetrics + +# error import variables +error_import_pandas = False +error_import_seaborn = False +error_import_matplotlib = False +error_import_scipy = False +error_import_numpy = False + +try: + import pandas as pd +except ImportError: + error_import_pandas = True + +try: + import seaborn as sns +except ImportError: + error_import_seaborn = True + +try: + import matplotlib + matplotlib.use('pdf') + import matplotlib.pyplot as plt00 +except ImportError: + error_import_matplotlib = True + +try: + import scipy.optimize +except ImportError: + error_import_scipy = True + + +try: + import numpy +except ImportError: + error_import_numpy = True + + +__author__ = "Sandra Mendez" +__copyright__ = "Copyright 2019, Barcelona Supercomputing Center (BSC)" +__version_major__ = 0 +__version_minor__ = 3 +__version_micro__ = 7 +__version__ = str(__version_major__) + "." + str(__version_minor__) + "." + str(__version_micro__) + + +if __name__ == "__main__": + """Main control flow. + Currently the script only accepts one parameter, which is a list of traces + that are processed. This can be a regex with wild cards and only valid trace + files are kept at the end. + """ + # Parse command line arguments + cmdl_args = parse_arguments() + + # Check if paramedir and Dimemas are in the path + check_installation(cmdl_args) + + # Process the trace list + trace_list, trace_processes, trace_tasks, trace_threads, trace_task_per_node, trace_mode = \ + get_traces_from_args(cmdl_args) + + # To validate the metric type + trace_metrics = 0 + # print(trace_mode) + for trace in trace_list: + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace][:15] == 'Detailed+OpenMP' \ + or trace_mode[trace][:17] == 'Detailed+Pthreads' or trace_mode[trace][:13] == 'Detailed+CUDA' \ + or trace_mode[trace][:14] == 'Detailed+OmpSs' or trace_mode[trace][:15] == 'Detailed+OpenCL' \ + or trace_mode[trace] == 'Detailed' or trace_mode[trace][:5] == 'Burst': + trace_metrics += 1 + + # Analyze the traces and gather the raw input data + raw_data, list_mpi_procs_count = gather_raw_data(trace_list, trace_processes, trace_task_per_node, + trace_mode, cmdl_args) + print_raw_data_csv(raw_data, trace_list, trace_processes) + + # Compute the model factors and print them + + if cmdl_args.metrics == 'hybrid' and trace_metrics == 0: + mod_factors, mod_factors_scale_plus_io, hybrid_factors, other_metrics = \ + hybridmetrics.compute_model_factors(raw_data, trace_list, trace_processes, + trace_mode, list_mpi_procs_count, cmdl_args) + hybridmetrics.print_other_metrics_table(other_metrics, trace_list, trace_processes, trace_tasks, trace_threads) + hybridmetrics.print_other_metrics_csv(other_metrics, trace_list, trace_processes) + hybridmetrics.print_mod_factors_table(mod_factors, other_metrics, mod_factors_scale_plus_io, hybrid_factors, + trace_list, trace_processes, trace_tasks, trace_threads, trace_mode) + hybridmetrics.print_mod_factors_csv(mod_factors, hybrid_factors, trace_list, trace_processes) + hybridmetrics.print_efficiency_table(mod_factors, hybrid_factors, trace_list, trace_processes, trace_tasks, + trace_threads) + # Plotting efficiency table with matplotlib + error_plot_table = False + if error_import_numpy or error_import_pandas or error_import_matplotlib or error_import_seaborn: + print('Numpy/Pandas/Matplotlib/Seaborn modules not available. ' + 'Skipping efficiency table plotting with python.') + if len(trace_list) > 1: + out_ver_gnuplot = subprocess.check_output(["gnuplot", "--version"]) + if 'gnuplot 5.' not in str(out_ver_gnuplot): + print('It requires gnuplot version 5.0 or higher. ' + 'Skipping efficiency table and lineal plotting with gnuplot.') + else: + try: + output_gnuplot_g = subprocess.check_output(["gnuplot", "efficiency_table_global.gp"]) + output_gnuplot_h = subprocess.check_output(["gnuplot", "efficiency_table-hybrid.gp"]) + except: + print(output_gnuplot_g) + print(output_gnuplot_h) + error_plot_table = True + + if not error_plot_table: + hybridmetrics.plots_efficiency_table_matplot(trace_list, trace_processes, trace_tasks, + trace_threads, cmdl_args) + if len(trace_list) > 1: + hybridmetrics.plots_modelfactors_matplot(trace_list, trace_processes, trace_tasks, + trace_threads, cmdl_args) + hybridmetrics.plots_speedup_matplot(trace_list, trace_processes, trace_tasks, + trace_threads, cmdl_args) + + # Plotting if SciPy and NumPy are installed. + error_plot_lineal = False + if error_import_numpy or error_import_scipy: + print('Scipy/NumPy module not available. Skipping lineal plotting.') + error_plot_lineal = True + + if not error_plot_lineal: + if len(trace_list) > 1: + plots.plot_hybrid_metrics(mod_factors, hybrid_factors, trace_list, trace_processes, + trace_tasks, trace_threads, trace_mode, cmdl_args) + + if len(trace_list) == 1: + subprocess.check_output(["rm", "efficiency_table_global.gp"]) + subprocess.check_output(["rm", "efficiency_table-hybrid.gp"]) + + elif cmdl_args.metrics == 'simple' or trace_metrics > 0: + mod_factors, mod_factors_scale_plus_io, other_metrics = compute_model_factors(raw_data, trace_list, + trace_processes, trace_mode, + list_mpi_procs_count, cmdl_args) + print_other_metrics_table(other_metrics, trace_list, trace_processes) + print_other_metrics_csv(other_metrics, trace_list, trace_processes) + print_mod_factors_table(mod_factors, other_metrics, mod_factors_scale_plus_io, trace_list, trace_processes) + print_mod_factors_csv(mod_factors, trace_list, trace_processes) + print_efficiency_table(mod_factors, trace_list, trace_processes) + + # Plotting efficiency table with matplotlib + error_plot_table = False + if error_import_numpy or error_import_pandas or error_import_matplotlib or error_import_seaborn: + print('Numpy/Pandas/Matplotlib/Seaborn modules not available. ' + 'Skipping efficiency table plotting with python.') + if len(trace_list) > 1: + out_ver_gnuplot = subprocess.check_output(["gnuplot", "--version"]) + if 'gnuplot 5.' not in str(out_ver_gnuplot): + print('It requires gnuplot version 5.0 or higher. ' + 'Skipping efficiency table and lineal plotting with gnuplot.') + else: + try: + output_gnuplot = subprocess.check_output(["gnuplot", "efficiency_table.gp"]) + except: + print(output_gnuplot) + error_plot_table = True + + if not error_plot_table: + plots_efficiency_table_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args) + if len(trace_list) > 1: + plots_modelfactors_matplot(trace_list, trace_mode, trace_processes, trace_tasks, trace_threads, + cmdl_args) + + # Plotting if SciPy and NumPy are installed. + error_plot_lineal = False + if error_import_numpy or error_import_scipy: + print('Scipy/NumPy module not available. Skipping lineal plotting.') + error_plot_lineal = True + # sys.exit(1) + if not error_plot_lineal: + if len(trace_list) > 1: + plots.plot_simple_metrics(mod_factors, trace_list, trace_processes, trace_mode, cmdl_args) + plots_speedup_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args) + if len(trace_list) == 1: + subprocess.check_output(["rm", "efficiency_table.gp"]) diff --git a/perf_metrics.bash b/perf_metrics.bash index f90289c..215e7fd 100644 --- a/perf_metrics.bash +++ b/perf_metrics.bash @@ -464,7 +464,7 @@ Create_metrics() # Create performance metrics echo "Creating metrics and storing theme in Metrics folder" echo - python3 Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w "modelfactors.py -ms 100000 *" + python3 Job_Creator.py -f "analysis" -j "analysis" --set-core "${Jobs_n_cores}" -s "$Jobs_scheduler" --set-time "$time" --set-queue "$queue" -w ".././modelfactors.py -ms 100000 *" mv analysis."$Jobs_scheduler" Metrics cd Metrics||(echo "Error Metrics folder doesn't exists"; exit 1) state5=$("$job" --wait analysis."$Jobs_scheduler") diff --git a/plots.py b/plots.py new file mode 100644 index 0000000..0e28cf1 --- /dev/null +++ b/plots.py @@ -0,0 +1,738 @@ +#!/usr/bin/env python3 + +"""Functions to plots efficiencies metrics""" + +from __future__ import print_function, division +import os +from collections import OrderedDict + +# error import variables +error_import_scipy = False +error_import_numpy = False + +try: + import scipy.optimize +except ImportError: + error_import_scipy = True + # print('==ERROR== Could not import SciPy. Please make sure to install a current version.') + +try: + import numpy +except ImportError: + error_import_numpy = True + # print('==ERROR== Could not import NumPy. Please make sure to install a current version.') + + +mod_hybrid_factors_doc = OrderedDict([ + ('hybrid_eff', '-- Hybrid Parallel efficiency'), + ('mpi_parallel_eff', ' -- MPI Parallel efficiency'), + ('mpi_load_balance', ' -- MPI Load balance'), + ('mpi_comm_eff', ' -- MPI Communication efficiency'), + ('serial_eff', ' -- Serialization efficiency'), + ('transfer_eff', ' -- Transfer efficiency'), + ('omp_parallel_eff', ' -- OMP Parallel efficiency'), + ('omp_load_balance', ' -- OMP Load balance'), + ('omp_comm_eff', ' -- OMP Communication efficiency')]) + + +def plot_hybrid_metrics(mod_factors, hybrid_factors, trace_list, trace_processes, trace_tasks, trace_threads, trace_mode, cmdl_args): + """Computes the projection from the gathered model factors and returns the + according dictionary of fitted prediction functions.""" + + global mod_hybrid_factors_doc + + # Update the hybrid parallelism mode + trace_mode_doc = trace_mode[trace_list[0]] + if trace_mode_doc[0:len("Detailed+MPI+")] == "Detailed+MPI+": + mod_hybrid_factors_doc['omp_parallel_eff'] = " -- " + \ + trace_mode_doc[len("Detailed+MPI+"):] + " Parallel efficiency" + mod_hybrid_factors_doc['omp_load_balance'] = " -- " + \ + trace_mode_doc[len("Detailed+MPI+"):] + " Load Balance" + mod_hybrid_factors_doc['omp_comm_eff'] = " -- " + \ + trace_mode_doc[len("Detailed+MPI+"):] + " Communication efficiency" + + if cmdl_args.debug: + print('==DEBUG== Computing projection of model factors.') + + number_traces = len(trace_list) + x_proc = numpy.zeros(number_traces) + y_para = numpy.zeros(number_traces) + y_load = numpy.zeros(number_traces) + y_comm = numpy.zeros(number_traces) + y_comp = numpy.zeros(number_traces) + y_glob = numpy.zeros(number_traces) + y_ipc_scale = numpy.zeros(number_traces) + y_inst_scale = numpy.zeros(number_traces) + y_freq_scale = numpy.zeros(number_traces) + y_hybrid_par = numpy.zeros(number_traces) + y_mpi_par = numpy.zeros(number_traces) + y_mpi_load = numpy.zeros(number_traces) + y_mpi_comm = numpy.zeros(number_traces) + y_comm_serial = numpy.zeros(number_traces) + y_comm_transfer = numpy.zeros(number_traces) + y_omp_par = numpy.zeros(number_traces) + y_omp_load = numpy.zeros(number_traces) + y_omp_comm = numpy.zeros(number_traces) + + #Convert dictionaries to NumPy arrays + for index, trace in enumerate(trace_list): + x_proc[index] = trace_processes[trace] + y_para[index] = mod_factors['parallel_eff'][trace] + y_load[index] = mod_factors['load_balance'][trace] + y_comm[index] = mod_factors['comm_eff'][trace] + y_comp[index] = mod_factors['comp_scale'][trace] + y_glob[index] = mod_factors['global_eff'][trace] + y_ipc_scale[index] = mod_factors['ipc_scale'][trace] + y_inst_scale[index] = mod_factors['inst_scale'][trace] + y_freq_scale[index] = mod_factors['freq_scale'][trace] + y_hybrid_par[index] = hybrid_factors['hybrid_eff'][trace] + y_mpi_par[index] = hybrid_factors['mpi_parallel_eff'][trace] + y_mpi_load[index] = hybrid_factors['mpi_load_balance'][trace] + y_mpi_comm[index] = hybrid_factors['mpi_comm_eff'][trace] + if trace_mode[trace] == 'Detailed+MPI' \ + or trace_mode[trace] == 'Detailed+MPI+OpenMP': + if hybrid_factors['serial_eff'][trace] != 'Non-Avail' and hybrid_factors['serial_eff'][trace] != 'Warning!': + y_comm_serial[index] = hybrid_factors['serial_eff'][trace] + else: + y_comm_serial[index] = 0.0 + if hybrid_factors['transfer_eff'][trace] != 'Non-Avail' \ + and hybrid_factors['transfer_eff'][trace] != 'Warning!': + y_comm_transfer[index] = hybrid_factors['transfer_eff'][trace] + else: + y_comm_transfer[index] = 0.0 + else: + y_comm_serial[index] = 0.0 + y_comm_transfer[index] = 0.0 + y_omp_par[index] = hybrid_factors['omp_parallel_eff'][trace] + y_omp_load[index] = hybrid_factors['omp_load_balance'][trace] + y_omp_comm[index] = hybrid_factors['omp_comm_eff'][trace] + + # Set limit for projection + if cmdl_args.limit: + limit = cmdl_args.limit + else: + limit = str(trace_processes[trace_list[len(trace_list)-1]]) + + limit_min = str(int(x_proc[0])) + # limit_min = str(0) + # To extract the trace name to show in the plots + title_string = "" + for index, trace in enumerate(trace_list): + folder_trace_name = trace.split('/') + trace_name_to_show = folder_trace_name[len(folder_trace_name) - 1] + title_string += '(' + str(index+1) + ') ' + trace_name_to_show + "\\" + 'n' + + title_string += '"' + " noenhanced" + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # Create Gnuplot file for main plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-onlydata.gp') + content = [] + with open(gp_template) as f: + content = f.readlines() + + # Replace xrange + if int(limit) == int(limit_min): + limit_plot = int(limit_min) + 5 * (len(trace_list) - 1) + else: + limit_plot = int(limit) + + # To xticks label + label_xtics = 'set xtics (' + glabel_xtics = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' + if int(limit) == int(limit_min) and same_procs: + proc_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' \ + + '[' + str(index + 1) + ']' + real_procs = str(trace_processes[trace] + index * 5) + elif int(limit) == int(limit_min) and not same_procs: + proc_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' + real_procs = str(trace_processes[trace] + index * 5) + if s_xtics in glabel_xtics: + proc_xtics += '[' + str(index + 1) + ']' + glabel_xtics.append(s_xtics) + else: + proc_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' + real_procs = str(trace_processes[trace]) + if s_xtics in glabel_xtics: + proc_xtics += '[' + str(index + 1) + ']' + glabel_xtics.append(s_xtics) + + label_xtics += '"' + proc_xtics + '" ' + real_procs + ', ' + + content = [line.replace('#REPLACE_BY_XRANGE', '' + .join(['set xrange [', str(limit_min), ':', str(limit_plot), ']'])) for line in content] + content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2] + ') '])) for + line in content] + content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in + content] + + max_global = max([max(y_para), max(y_load), max(y_comm), max(y_comp), max(y_glob)]) + content = [line.replace('#REPLACE_BY_YRANGE', '' + .join(['set yrange [0:', str(max_global+5), ']'])) for line in content] + + file_path = os.path.join(os.getcwd(), 'modelfactors.gp') + with open(file_path, 'w') as f: + f.writelines(content) + + # Add data points to gnuplot file + with open(file_path, 'a') as f: + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index] + index * 5), str(y_para[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_para[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index] + index * 5), str(y_load[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_load[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index] + index * 5), str(y_comm[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comm[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index] + index * 5), str(y_comp[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comp[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index] + index * 5), str(y_glob[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_glob[index]), '\n']) + f.write(line) + f.write('e\n') + + f.write('\n') + f.write('pause -1\n') + + # print('========= Plot (gnuplot File): EFFICIENCY METRICS ==========') + print('Efficiency metrics plot written to ' + file_path) + # print('') + + # Create Gnuplot file for scalability plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-scale.gp') + content = [] + with open(gp_template) as f: + content = f.readlines() + + # Replace xrange + content = [line.replace('#REPLACE_BY_XRANGE', ''.join(['set xrange [',limit_min,':',str(limit_plot),']']) ) for line in content] + content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2]+') '])) for + line in content] + content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in + content] + + max_comp = max([max(y_comp), max(y_ipc_scale), max(y_inst_scale), max(y_freq_scale)]) + content = [line.replace('#REPLACE_BY_YRANGE', '' + .join(['set yrange [0:', str(max_comp+5), ']'])) for line in content] + + file_path = os.path.join(os.getcwd(), 'modelfactors-scale.gp') + with open(file_path, 'w') as f: + f.writelines(content) + + # Add data points to gnuplot file + with open(file_path, 'a') as f: + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comp[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comp[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_ipc_scale[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_ipc_scale[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_inst_scale[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_inst_scale[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_freq_scale[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_freq_scale[index]), '\n']) + f.write(line) + f.write('e\n') + + f.write('\n') + f.write('pause -1\n') + + # print('======== Plot (gnuplot File): SCALABILITY METRICS ========') + print('Scalability metrics plot written to ' + file_path) + + # Create Gnuplot file for hybrid metrics plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-hybrid.gp') + content = [] + with open(gp_template) as f: + content = f.readlines() + + # Replace xrange + content = [line.replace('#REPLACE_BY_XRANGE', ''.join(['set xrange [',limit_min,':',str(limit_plot),']']) ) for line in content] + content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2]+') '])) for + line in content] + content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in + content] + + string_omp =" '-' with linespoints title " + '"' + (mod_hybrid_factors_doc['omp_parallel_eff'])[5:] + '"' + " ls 5,\\" + content = [line.replace('#REPLACE_BY_OMP_PAR_EFF', ''.join([string_omp])) for line in content] + + string_omp = " '-' with linespoints title " + '"' + (mod_hybrid_factors_doc['omp_load_balance'])[9:] + '"' + " ls 6,\\" + content = [line.replace('#REPLACE_BY_OMP_LB', ''.join([string_omp])) for line in content] + + string_omp = " '-' with linespoints title " + '"' + (mod_hybrid_factors_doc['omp_comm_eff'])[9:] + '"' + " ls 7" + content = [line.replace('#REPLACE_BY_OMP_COMM', ''.join([string_omp])) for line in content] + + max_hybrid = max([max(y_hybrid_par), max(y_mpi_par), max(y_mpi_comm), max(y_mpi_load), + max(y_omp_par), max(y_omp_comm), max(y_omp_load)]) + content = [line.replace('#REPLACE_BY_YRANGE', '' + .join(['set yrange [0:', str(max_hybrid+5), ']'])) for line in content] + + file_path = os.path.join(os.getcwd(), 'modelfactors-hybrid.gp') + with open(file_path, 'w') as f: + f.writelines(content) + + # Add data points to gnuplot file + with open(file_path, 'a') as f: + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_hybrid_par[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_hybrid_par[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_par[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_mpi_par[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_load[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_mpi_load[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_comm[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_mpi_comm[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_omp_par[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_omp_par[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_omp_load[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_omp_load[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_omp_comm[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_omp_comm[index]), '\n']) + f.write(line) + f.write('e\n') + + f.write('\n') + f.write('pause -1\n') + + print('Hybrid metrics plot written to ' + file_path) + + # Create Gnuplot file for MPI hybrid metrics plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-mpi-hybrid.gp') + content = [] + with open(gp_template) as f: + content = f.readlines() + + # Replace xrange + content = [line.replace('#REPLACE_BY_XRANGE', ''.join(['set xrange [',limit_min,':',str(limit_plot),']']) ) for line in content] + content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2]+') '])) for + line in content] + content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in + content] + + max_mpi = max([max(y_mpi_par), max(y_mpi_comm), max(y_mpi_load), + max(y_comm_serial), max(y_comm_transfer)]) + content = [line.replace('#REPLACE_BY_YRANGE', '' + .join(['set yrange [0:', str(max_mpi+5), ']'])) for line in content] + + file_path = os.path.join(os.getcwd(), 'modelfactors-mpi-hybrid.gp') + with open(file_path, 'w') as f: + f.writelines(content) + + # Add data points to gnuplot file + with open(file_path, 'a') as f: + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_par[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_mpi_par[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_load[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_mpi_load[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_mpi_comm[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_mpi_comm[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if str(y_comm_serial[index]) != 0.0: + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comm_serial[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comm_serial[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if str(y_comm_transfer[index]) != 0.0: + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comm_transfer[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comm_transfer[index]), '\n']) + f.write(line) + f.write('e\n') + + + f.write('\n') + f.write('pause -1\n') + + print('MPI hybrid metrics plot written to ' + file_path) + + +def plot_simple_metrics(mod_factors, trace_list, trace_processes, trace_mode, cmdl_args): + """Computes the projection from the gathered model factors and returns the + according dictionary of fitted prediction functions.""" + + if cmdl_args.debug: + print('==DEBUG== Computing projection of model factors.') + + number_traces = len(trace_list) + x_proc = numpy.zeros(number_traces) + y_para = numpy.zeros(number_traces) + y_load = numpy.zeros(number_traces) + y_comm = numpy.zeros(number_traces) + y_comp = numpy.zeros(number_traces) + y_glob = numpy.zeros(number_traces) + y_comm_serial = numpy.zeros(number_traces) + y_comm_transfer = numpy.zeros(number_traces) + y_ipc_scale = numpy.zeros(number_traces) + y_inst_scale = numpy.zeros(number_traces) + y_freq_scale = numpy.zeros(number_traces) + + #Convert dictionaries to NumPy arrays + for index, trace in enumerate(trace_list): + x_proc[index] = trace_processes[trace] + y_para[index] = mod_factors['parallel_eff'][trace] + y_load[index] = mod_factors['load_balance'][trace] + y_comm[index] = mod_factors['comm_eff'][trace] + y_comp[index] = mod_factors['comp_scale'][trace] + y_glob[index] = mod_factors['global_eff'][trace] + if trace_mode[trace][:5] != 'Burst': + y_ipc_scale[index] = mod_factors['ipc_scale'][trace] + y_inst_scale[index] = mod_factors['inst_scale'][trace] + y_freq_scale[index] = mod_factors['freq_scale'][trace] + else: + y_ipc_scale[index] = 0.0 + y_inst_scale[index] = 0.0 + y_freq_scale[index] = 0.0 + if trace_mode[trace] == 'Detailed+MPI': + if mod_factors['serial_eff'][trace] != 'Warning!': + y_comm_serial[index] = mod_factors['serial_eff'][trace] + else: + y_comm_serial[index] = 0.0 + if mod_factors['transfer_eff'][trace] != 'Warning!': + y_comm_transfer[index] = mod_factors['transfer_eff'][trace] + else: + y_comm_transfer[index] = 0.0 + else: + y_comm_serial[index] = 0.0 + y_comm_transfer[index] = 0.0 + + #Set limit for projection + if cmdl_args.limit: + limit = cmdl_args.limit + else: + limit = str(trace_processes[trace_list[len(trace_list) - 1]]) + + limit_min = str(int(x_proc[0])) + # limit_min = str(0) + # To extract the trace name to show in the plots + title_string = "" + for index, trace in enumerate(trace_list): + folder_trace_name = trace.split('/') + trace_name_to_show = folder_trace_name[len(folder_trace_name) - 1] + title_string += '(' + str(index+1) + ') ' + trace_name_to_show + "\\" + 'n' + title_string += '"' + " noenhanced" + + #Create Gnuplot file for main plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-onlydata.gp') + content = [] + with open(gp_template) as f: + content = f.readlines() + + #Replace xrange + if int(limit) == int(limit_min): + limit_plot = int(limit_min) + 5 * (len(trace_list) - 1 ) + else: + limit_plot = int(limit) + + # To xticks label + label_xtics = 'set xtics (' + for index, trace in enumerate(trace_list): + + if int(limit) == int(limit_min): + label_xtics += '"' + str(trace_processes[trace]) + '[' + str(index + 1) + ']' + '" ' \ + + str(trace_processes[trace] + index * 5) + ', ' + else: + label_xtics += '"' + str(trace_processes[trace]) + '" ' + str(trace_processes[trace]) + ', ' + + content = [line.replace('#REPLACE_BY_XRANGE', '' + .join(['set xrange [',limit_min,':', str(limit_plot),']'])) for line in content] + content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2] + ') '])) for + line in content] + + content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in + content] + + max_global = max([max(y_para), max(y_load), max(y_comm), max(y_comp), max(y_glob)]) + content = [line.replace('#REPLACE_BY_YRANGE', '' + .join(['set yrange [0:', str(max_global+5), ']'])) for line in content] + + file_path = os.path.join(os.getcwd(), 'modelfactors.gp') + with open(file_path, 'w') as f: + f.writelines(content) + + #Add data points to gnuplot file + with open(file_path, 'a') as f: + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_para[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_para[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_load[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_load[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comm[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comm[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comp[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comp[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_glob[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_glob[index]), '\n']) + f.write(line) + f.write('e\n') + + f.write('\n') + f.write('pause -1\n') + + # print('========= Plot (gnuplot File): EFFICIENCY METRICS ==========') + print('Efficiency metrics plot written to ' + file_path) + # print('') + + # Create Gnuplot file for communication plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-comm.gp') + content = [] + with open(gp_template) as f: + content = f.readlines() + + # Replace xrange + content = [line.replace('#REPLACE_BY_XRANGE', ''.join(['set xrange [',limit_min,':',str(limit_plot),']']) ) for line in content] + content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2]+') '])) for + line in content] + + content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in + content] + + max_comm = max([max(y_comm), max(y_comm_serial), max(y_comm_transfer)]) + content = [line.replace('#REPLACE_BY_YRANGE', '' + .join(['set yrange [0:', str(max_comm+5), ']'])) for line in content] + + file_path = os.path.join(os.getcwd(), 'modelfactors-comm.gp') + with open(file_path, 'w') as f: + f.writelines(content) + + # Add data points to gnuplot file + with open(file_path, 'a') as f: + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comm[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comm[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if str(y_comm_serial[index]) != 0.0: + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comm_serial[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comm_serial[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if str(y_comm_transfer[index]) != 0.0: + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comm_transfer[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comm_transfer[index]), '\n']) + f.write(line) + f.write('e\n') + + f.write('\n') + f.write('pause -1\n') + + # print('======= Plot (gnuplot File): COMMUNICATION METRICS ========') + print('Communication Efficiency plot written to ' + file_path) + # print('') + + # Create Gnuplot file for scalability plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'modelfactors-scale.gp') + content = [] + with open(gp_template) as f: + content = f.readlines() + + # Replace xrange + content = [line.replace('#REPLACE_BY_XRANGE', ''.join(['set xrange [',limit_min,':',str(limit_plot),']']) ) for line in content] + content = [line.replace('#REPLACE_BY_XTICS_LABEL', ''.join([label_xtics[:-2]+') '])) for + line in content] + content = [line.replace('#REPLACE_BY_TRACE_NAMES', ''.join(["set title " + '"' + ''])) for line in + content] + max_comp = max([max(y_comp), max(y_ipc_scale), max(y_inst_scale), max(y_freq_scale)]) + content = [line.replace('#REPLACE_BY_YRANGE', '' + .join(['set yrange [0:', str(max_comp+5), ']'])) for line in content] + + file_path = os.path.join(os.getcwd(), 'modelfactors-scale.gp') + with open(file_path, 'w') as f: + f.writelines(content) + + # Add data points to gnuplot file + with open(file_path, 'a') as f: + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_comp[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_comp[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_ipc_scale[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_ipc_scale[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_inst_scale[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_inst_scale[index]), '\n']) + f.write(line) + f.write('e\n') + + for index in range(0, number_traces): + if int(limit) == int(limit_min): + line = ' '.join([str(x_proc[index]+index*5), str(y_freq_scale[index]), '\n']) + else: + line = ' '.join([str(x_proc[index]), str(y_freq_scale[index]), '\n']) + f.write(line) + f.write('e\n') + + f.write('\n') + f.write('pause -1\n') + + # print('======== Plot (gnuplot File): SCALABILITY METRICS ========') + print('Scalability metrics plot written to ' + file_path) \ No newline at end of file diff --git a/rawdata.py b/rawdata.py new file mode 100644 index 0000000..edbdf4d --- /dev/null +++ b/rawdata.py @@ -0,0 +1,857 @@ +#!/usr/bin/env python3 + +"""Functions to extract rawdata from each trace.""" + +from __future__ import print_function, division +import os +import time +import math +import gzip +import shutil +from collections import OrderedDict +from tracemetadata import human_readable +from utils import run_command, move_files,remove_files, create_temp_folder + + +# Contains all raw data entries with a printable name. +# This is used to generate and print all raw data, so, if an entry is added, it +# should be added here, too. +raw_data_doc = OrderedDict([('runtime', 'Runtime (us)'), + ('runtime_dim', 'Runtime (ideal)'), + ('useful_avg', 'Useful duration (average)'), + ('useful_max', 'Useful duration (maximum)'), + ('useful_tot', 'Useful duration (total)'), + ('useful_dim', 'Useful duration (ideal, max)'), + ('useful_ins', 'Useful instructions (total)'), + ('useful_cyc', 'Useful cycles (total)'), + ('outsidempi_avg', 'Outside MPI duration (average)'), + ('outsidempi_max', 'Outside MPI duration (maximum)'), + ('outsidempi_dim', 'Outside MPI duration (ideal,maximum)'), + ('outsidempi_tot', 'Outside MPI duration (total)'), + ('mpicomm_tot', 'Communication MPI duration (total)'), + ('outsidempi_tot_diff', 'Outside MPI duration rescaled (total*threads)'), + ('flushing_avg', 'Flushing duration (average)'), + ('flushing_max', 'Flushing duration (maximum)'), + ('flushing_tot', 'Flushing duration (total)'), + ('flushing_cyc', 'Flushing cycles (total)'), + ('flushing_ins', 'Flushing instructions (total)'), + ('io_tot', 'Posix I/O duration (total)'), + ('io_max', 'Posix I/O duration (maximum)'), + ('io_avg', 'Posix I/O duration (avg)'), + ('io_std', 'Posix I/O duration (std)'), + ('io_cyc', 'Posix I/O cycles (total)'), + ('io_ins', 'Posix I/O instructions (total)'), + ('useful_plus_io_avg', 'Serial I/O plus useful duration (avg)'), + ('useful_plus_io_max', 'Serial I/O plus useful duration (max)'), + ('io_state_tot', 'state I/O duration (total)'), + ('io_state_avg', 'state I/O duration (avg)'), + ('io_state_max', 'state I/O duration (maximum)'), + ('mpiio_tot', 'MPI I/O duration (total)'), + ('mpiio_max', 'MPI I/O duration (maximum)'), + ('mpiio_avg', 'MPI I/O duration (avg)'), + ('mpiio_std', 'MPI I/O duration (std)'), + ('mpiio_cyc', 'MPI I/O cycles (total)'), + ('mpiio_ins', 'MPI I/O instructions (total)'), + ('burst_useful_tot', 'Burst Useful (total)'), + ('burst_useful_max', 'Burst Useful (max)'), + ('burst_useful_avg', 'Burst Useful (avg)')]) + + +def create_raw_data(trace_list): + """Creates 2D dictionary of the raw input data and initializes with zero. + The raw_data dictionary has the format: [raw data key][trace]. + """ + global raw_data_doc + raw_data = {} + for key in raw_data_doc: + trace_dict = {} + for trace_name in trace_list: + trace_dict[trace_name] = 0.0 + + raw_data[key] = trace_dict + + return raw_data + + +def gather_raw_data(trace_list, trace_processes, trace_task_per_node, trace_mode, cmdl_args): + """Gathers all raw data needed to generate the model factors. Return raw + data in a 2D dictionary """ + raw_data = create_raw_data(trace_list) + global list_mpi_procs_count + list_mpi_procs_count = dict() + + cfgs = {} + cfgs['root_dir'] = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs') + cfgs['timings'] = os.path.join(cfgs['root_dir'], 'timings.cfg') + cfgs['runtime'] = os.path.join(cfgs['root_dir'], 'runtime_app.cfg') + cfgs['cycles'] = os.path.join(cfgs['root_dir'], 'cycles.cfg') + cfgs['instructions'] = os.path.join(cfgs['root_dir'], 'instructions.cfg') + cfgs['flushing'] = os.path.join(cfgs['root_dir'], 'flushing.cfg') + cfgs['mpi_io'] = os.path.join(cfgs['root_dir'], 'mpi-io-reverse.cfg') + cfgs['outside_mpi'] = os.path.join(cfgs['root_dir'], 'mpi-call-outside.cfg') + cfgs['io_call'] = os.path.join(cfgs['root_dir'], 'io-call-reverse.cfg') + cfgs['io_cycles'] = os.path.join(cfgs['root_dir'], 'io-call-cycles.cfg') + cfgs['io_inst'] = os.path.join(cfgs['root_dir'], 'io-call-instructions.cfg') + cfgs['mpiio_cycles'] = os.path.join(cfgs['root_dir'], 'mpi-io-cycles.cfg') + cfgs['mpiio_inst'] = os.path.join(cfgs['root_dir'], 'mpi-io-instructions.cfg') + cfgs['flushing_cycles'] = os.path.join(cfgs['root_dir'], 'flushing-cycles.cfg') + cfgs['flushing_inst'] = os.path.join(cfgs['root_dir'], 'flushing-inst.cfg') + cfgs['burst_useful'] = os.path.join(cfgs['root_dir'], 'burst_useful.cfg') + + # Main loop over all traces + # This can be parallelized: the loop iterations have no dependencies + path_dest = create_temp_folder('scratch_out_basicanalysis', cmdl_args) + + for trace in trace_list: + time_tot = time.time() + if trace[-7:] == ".prv.gz": + trace_name_control = trace[:-7] + trace_name = trace[:-7] + '_' + str(trace_processes[trace]) + 'P' + elif trace[-4:] == ".prv": + trace_name_control = trace[:-4] + trace_name = trace[:-4] + '_' + str(trace_processes[trace]) + 'P' + + line = 'Analyzing ' + os.path.basename(trace) + line += ' (' + str(trace_processes[trace]) + ' processes' + line += ', ' + str(trace_task_per_node[trace]) + ' tasks per node' + line += ', ' + str(trace_mode[trace]) + ' mode' + line += ', ' + human_readable(os.path.getsize(trace)) + ')' + print(line) + + # Create simulated ideal trace with Dimemas + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + time_dim = time.time() + trace_sim = create_ideal_trace(trace, trace_processes[trace], trace_task_per_node[trace], cmdl_args) + trace_name_sim = trace_sim[:-4] + # print(trace_sim) + time_dim = time.time() - time_dim + if not trace_sim == '': + print('Successfully created simulated trace with Dimemas in {0:.1f} seconds.'.format(time_dim)) + else: + print('Failed to create simulated trace with Dimemas.') + + # Run paramedir for the original and simulated trace + time_pmd = time.time() + cmd_normal = ['paramedir', trace] + + cmd_normal.extend([cfgs['timings'], trace_name + '.timings.stats']) + cmd_normal.extend([cfgs['runtime'], trace_name + '.runtime.stats']) + cmd_normal.extend([cfgs['cycles'], trace_name + '.cycles.stats']) + cmd_normal.extend([cfgs['instructions'], trace_name + '.instructions.stats']) + cmd_normal.extend([cfgs['flushing'], trace_name + '.flushing.stats']) + cmd_normal.extend([cfgs['io_call'], trace_name + '.posixio_call.stats']) + cmd_normal.extend([cfgs['io_cycles'], trace_name + '.posixio-cycles.stats']) + cmd_normal.extend([cfgs['io_inst'], trace_name + '.posixio-inst.stats']) + cmd_normal.extend([cfgs['flushing_cycles'], trace_name + '.flushing-cycles.stats']) + cmd_normal.extend([cfgs['flushing_inst'], trace_name + '.flushing-inst.stats']) + + if trace_mode[trace][:12] == 'Detailed+MPI': + cmd_normal.extend([cfgs['mpi_io'], trace_name + '.mpi_io.stats']) + cmd_normal.extend([cfgs['outside_mpi'], trace_name + '.outside_mpi.stats']) + cmd_normal.extend([cfgs['mpiio_cycles'], trace_name + '.mpiio-cycles.stats']) + cmd_normal.extend([cfgs['mpiio_inst'], trace_name + '.mpiio-inst.stats']) + + if trace_mode[trace][:9] == 'Burst+MPI': + cmd_normal.extend([cfgs['burst_useful'], trace_name + '.burst_useful.stats']) + + run_command(cmd_normal, cmdl_args) + + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + cmd_ideal = ['paramedir', trace_sim] + cmd_ideal.extend([cfgs['timings'], trace_name_sim + '.timings.stats']) + cmd_ideal.extend([cfgs['runtime'], trace_name_sim + '.runtime.stats']) + cmd_ideal.extend([cfgs['outside_mpi'], trace_name_sim + '.outside_mpi.stats']) + + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + if not trace_sim == '': + # print(cmd_ideal) + run_command(cmd_ideal, cmdl_args) + + time_pmd = time.time() - time_pmd + + error_timing = 0 + error_counters = 0 + error_ideal = 0 + + # Check if all files are created + if not os.path.exists(trace_name + '.timings.stats') or \ + not os.path.exists(trace_name + '.runtime.stats'): + print('==ERROR== Failed to compute timing information with paramedir.') + error_timing = 1 + + if not os.path.exists(trace_name + '.outside_mpi.stats') and trace_mode[trace][:5] != 'Burst' \ + and 'MPI' in trace_mode[trace]: + print('==ERROR== Failed to compute outside MPI timing information with paramedir.') + error_timing = 1 + + if not os.path.exists(trace_name + '.cycles.stats') or \ + not os.path.exists(trace_name + '.instructions.stats'): + print('==ERROR== Failed to compute counter information with paramedir.') + error_counters = 1 + + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + if not os.path.exists(trace_name_sim + '.timings.stats') or \ + not os.path.exists(trace_name_sim + '.runtime.stats') or \ + not os.path.exists(trace_name_sim + '.outside_mpi.stats'): + print('==ERROR== Failed to compute simulated timing information with paramedir.') + error_ideal = 1 + trace_sim = '' + + if error_timing or error_counters or error_ideal: + print('Failed to analyze trace with paramedir') + else: + print('Successfully analyzed trace with paramedir in {0:.1f} seconds.'.format(time_pmd)) + + # Parse the paramedir output files + time_prs = time.time() + + # Get total, average, and maximum useful duration + if os.path.exists(trace_name + '.timings.stats'): + content = [] + with open(trace_name + '.timings.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['useful_tot'][trace] = float(line.split()[1]) + if line.split()[0] == 'Average': + raw_data['useful_avg'][trace] = float(line.split()[1]) + if line.split()[0] == 'Maximum': + raw_data['useful_max'][trace] = float(line.split()[1]) + else: + raw_data['useful_tot'][trace] = 'NaN' + raw_data['useful_avg'][trace] = 'NaN' + raw_data['useful_max'][trace] = 'NaN' + f.close() + + # Get total File IO, average IO, and maximum IO duration + dict_trace_posixio = {} + if os.path.exists(trace_name + '.posixio_call.stats'): + content = [] + with open(trace_name + '.posixio_call.stats') as f: + content = f.readlines() + + for line in content: + + for field in line.split("\n"): + line_list = field.split("\t") + if "Total" in field.split("\t"): + count_procs = len(line_list[1:]) + list_io_tot = [float(iotime) for iotime in line_list[1:count_procs]] + dict_trace_posixio[trace] = list_io_tot + raw_data['io_tot'][trace] = sum(list_io_tot) + elif "Average" in field.split("\t"): + if count_procs != 0: + raw_data['io_avg'][trace] = float(sum(list_io_tot)/count_procs) + raw_data['io_std'][trace] = math.sqrt(sum([(number - raw_data['io_avg'][trace]) ** 2 + for number in list_io_tot]) + / (len(list_io_tot) - 1)) + else: + raw_data['io_avg'][trace] = 0.0 + raw_data['io_std'][trace] = 0.0 + elif "Maximum" in field.split("\t"): + raw_data['io_max'][trace] = max(list_io_tot) + else: + raw_data['io_tot'][trace] = 0.0 + raw_data['io_avg'][trace] = 0.0 + raw_data['io_max'][trace] = 0.0 + raw_data['io_std'][trace] = 0.0 + f.close() + + # Get total MPI IO, average IO, and maximum IO duration for MPI-IO + dict_trace_mpiio={} + if os.path.exists(trace_name + '.mpi_io.stats') and trace_mode[trace][:12] == 'Detailed+MPI': + content = [] + with open(trace_name + '.mpi_io.stats') as f: + content = f.readlines() + + for line in content: + for field in line.split("\n"): + line_list = field.split("\t") + if "Total" in field.split("\t"): + count_procs = len(line_list[1:]) + list_mpiio_tot = [float(iotime) for iotime in line_list[1:count_procs]] + dict_trace_mpiio[trace] = list_mpiio_tot + # print(list_mpiio_tot) + raw_data['mpiio_tot'][trace] = sum(list_mpiio_tot) + elif "Average" in field.split("\t"): + if count_procs != 0: + raw_data['mpiio_avg'][trace] = sum(list_mpiio_tot)/count_procs + raw_data['mpiio_std'][trace] = math.sqrt(sum([(number - raw_data['mpiio_avg'][trace]) ** 2 + for number in list_mpiio_tot]) + /(len(list_mpiio_tot) - 1)) + else: + raw_data['mpiio_avg'][trace] = 0.0 + raw_data['mpiio_std'][trace] = 0.0 + elif "Maximum" in field.split("\t"): + raw_data['mpiio_max'][trace] = max(list_mpiio_tot) + else: + raw_data['mpiio_tot'][trace] = 0.0 + raw_data['mpiio_avg'][trace] = 0.0 + raw_data['mpiio_max'][trace] = 0.0 + raw_data['mpiio_std'][trace] = 0.0 + f.close() + + # Get total State IO, average IO, and maximum IO duration + #print(dict_trace_mpiio) + if os.path.exists(trace_name + '.timings.stats'): + content = [] + # io_index = " " + with open(trace_name + '.timings.stats') as f: + content = f.readlines() + useful_plus_io = [] + useful_comp = [] + io_time = [] + count_line = 1 + for line in content: + for field in line.split("\n"): + line_list = field.split("\t") + # print(line_list) + if count_line == 1: + try: + if os.path.exists(trace_name_control + '.pcf'): + io_index = line_list.index("I/O") + else: + io_index = line_list.index('Unknown state 12') + except: + io_index = " " + elif io_index != " ": + if "Total" in field.split("\t"): + raw_data['io_state_tot'][trace] = float(line_list[io_index]) + elif "Average" in field.split("\t"): + raw_data['io_state_avg'][trace] = float(line_list[io_index]) + elif "Maximum" in field.split("\t"): + raw_data['io_state_max'][trace] = float(line_list[io_index]) + elif "Minimum" not in line_list and "StDev" not in line_list \ + and "Avg/Max" not in line_list and len(line_list) > 1: + posixio_index = len(useful_plus_io) + # print(dict_trace_posixio[trace][posixio_index]) + if len(dict_trace_posixio) != 0: + sum_aux = float(line_list[1]) + float(dict_trace_posixio[trace][posixio_index]) + else: + sum_aux = float(line_list[1]) + + useful_plus_io.append(float(sum_aux)) + useful_comp.append(float(line_list[1])) + io_time.append(float(line_list[io_index])) + count_line += 1 + if io_index != " ": + if len(useful_plus_io) != 0: + useful_io_avg = float(sum(useful_plus_io) / len(useful_plus_io)) + else: + useful_io_avg = 0.0 + # print(len(useful_plus_io)) + # mpiio is not included in useful + IO + raw_data['useful_plus_io_avg'][trace] = float(useful_io_avg) + raw_data['useful_plus_io_max'][trace] = float(max(useful_plus_io)) + else: + raw_data['useful_plus_io_avg'][trace] = 0.0 + raw_data['useful_plus_io_max'][trace] = 0.0 + raw_data['io_state_tot'][trace] = 0.0 + raw_data['io_state_avg'][trace] = 0.0 + raw_data['io_state_max'][trace] = 0.0 + else: + raw_data['useful_plus_io_avg'][trace] = 'NaN' + raw_data['useful_plus_io_max'][trace] = 'NaN' + raw_data['io_state_tot'][trace] = 'NaN' + raw_data['io_state_avg'][trace] = 'NaN' + raw_data['io_state_max'][trace] = 'NaN' + f.close() + + + # Get runtime + if os.path.exists(trace_name + '.runtime.stats'): + content = [] + with open(trace_name + '.runtime.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Average': + raw_data['runtime'][trace] = float(line.split()[1]) + else: + raw_data['runtime'][trace] = 'NaN' + + # Get total, average, and maximum outside MPI + # list_mpi_procs_count = [] + if os.path.exists(trace_name + '.outside_mpi.stats') and trace_mode[trace][:12] == 'Detailed+MPI': + content = [] + with open(trace_name + '.outside_mpi.stats') as f: + content = f.readlines() + list_outside_mpi = [] + + list_thread_outside_mpi = [] + init_count_thread = False + count_threads = 1 + for line1 in content[1:(len(content) - 8)]: + line = line1.split("\t") + # print(line) + if line: + if line[0] != 'Total' and line[0] != 'Average' \ + and line[0] != 'Maximum' and line[0] != 'StDev' \ + and line[0] != 'Avg/Max': + # To extract the count of MPI tasks + if float(line[1]) != raw_data['runtime'][trace]: + list_outside_mpi.append(float(line[1])) + # To extract the count of threads per MPI task + if len(list_outside_mpi) > 1: + list_thread_outside_mpi.append(count_threads) + count_threads = 1 + else: + if len(list_outside_mpi) == 1 and not init_count_thread: + count_threads = 2 + init_count_thread = True + else: + count_threads += 1 + + list_thread_outside_mpi.append(count_threads) + + count = 0 + equal_threads = True + # Evaluate if the count of threads per MPI task are equal + while count < (len(list_thread_outside_mpi) - 1) and equal_threads: + if list_thread_outside_mpi[count] != list_thread_outside_mpi[count+1]: + equal_threads = False + count += 1 + rescaled_outside_mpi = [] + # This is need to calculate the MPI_Par_Eff with the right #MPI_tasks and #threads + if not equal_threads: + # print("\n===== Different number of threads per mpi task \n") + rescaled_outside_mpi.append(list_outside_mpi[0] * (list_thread_outside_mpi[0])) + # This is the sum of the outsidempi by the threads + sum_outside_mpi_threads = list_thread_outside_mpi[0] + + for i in range(1,len(list_outside_mpi)): + rescaled_outside_mpi.append(list_outside_mpi[i] * (list_thread_outside_mpi[i])) + sum_outside_mpi_threads += list_thread_outside_mpi[i] + raw_data['outsidempi_tot_diff'][trace] = sum(rescaled_outside_mpi) + raw_data['outsidempi_tot'][trace] = sum(list_outside_mpi) + # Only the average is updated with the rescaled outsidempi + if sum(list_thread_outside_mpi) != 0: + raw_data['outsidempi_avg'][trace] = sum(rescaled_outside_mpi) / sum(list_thread_outside_mpi) + else: + raw_data['outsidempi_avg'][trace] = 'NaN' + # Maximum outsidempi is the same, although the count of threads is different + # because it waits that the MPI task has the max outsidempi. + raw_data['outsidempi_max'][trace] = max(list_outside_mpi) + else: + # print("\n===== Equal number of threads per mpi task \n") + raw_data['outsidempi_tot_diff'][trace] = sum(list_outside_mpi) + list_mpi_procs_count[trace] = len(list_outside_mpi) + raw_data['outsidempi_tot'][trace] = sum(list_outside_mpi) + if len(list_outside_mpi) != 0: + raw_data['outsidempi_avg'][trace] = sum(list_outside_mpi) / len(list_outside_mpi) + else: + raw_data['outsidempi_avg'][trace] = 'NaN' + raw_data['outsidempi_max'][trace] = max(list_outside_mpi) + for line2 in content[(len(content) - 7):]: + line_aux = line2.split("\t") + + if line_aux[0] == 'Total': + #print(line_aux) + count_mpiop = len(line_aux[1:]) + list_mpi_tot = [float(mpitime) for mpitime in line_aux[2:count_mpiop]] + #print(list_mpi_tot) + raw_data['mpicomm_tot'][trace] = sum(list_mpi_tot) + else: + raw_data['outsidempi_tot'][trace] = 'NaN' + raw_data['outsidempi_avg'][trace] = 'NaN' + raw_data['outsidempi_max'][trace] = 'NaN' + raw_data['mpicomm_tot'][trace] = 'NaN' + f.close() + # Get total, average, and maximum flushing duration + if os.path.exists(trace_name + '.flushing.stats'): + content = [] + with open(trace_name + '.flushing.stats') as f: + content = f.readlines() + flushing_exist = ('\tBegin\t\n' in content) or ('\tvalue 1\t\n' in content) + + if flushing_exist: + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['flushing_tot'][trace] = float(line.split()[1]) + if line.split()[0] == 'Average': + raw_data['flushing_avg'][trace] = float(line.split()[1]) + if line.split()[0] == 'Maximum': + raw_data['flushing_max'][trace] = float(line.split()[1]) + else: + raw_data['flushing_tot'][trace] = 0.0 + raw_data['flushing_avg'][trace] = 0.0 + raw_data['flushing_max'][trace] = 0.0 + else: + raw_data['flushing_tot'][trace] = 0.0 + raw_data['flushing_avg'][trace] = 0.0 + raw_data['flushing_max'][trace] = 0.0 + + # Get total flushing cycles + if os.path.exists(trace_name + '.flushing-cycles.stats'): + content = [] + with open(trace_name + '.flushing-cycles.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['flushing_cyc'][trace] = int(float(line.split()[1])) + else: + raw_data['flushing_cyc'][trace] = 0.0 + + # Get total flushing instructions + if os.path.exists(trace_name + '.flushing-inst.stats'): + content = [] + with open(trace_name + '.flushing-inst.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['flushing_ins'][trace] = int(float(line.split()[1])) + else: + raw_data['flushing_ins'][trace] = 0.0 + + # Get total posixio cycles + if os.path.exists(trace_name + '.posixio-cycles.stats'): + content = [] + with open(trace_name + '.posixio-cycles.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['io_cyc'][trace] = int(float(line.split()[1])) + else: + raw_data['io_cyc'][trace] = 0.0 + + # Get total posixio instructions + if os.path.exists(trace_name + '.posixio-inst.stats'): + content = [] + with open(trace_name + '.posixio-inst.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['io_ins'][trace] = int(float(line.split()[1])) + else: + raw_data['io_ins'][trace] = 0.0 + + # Get total mpiio instructions + if os.path.exists(trace_name + '.mpiio-cycles.stats') \ + and trace_mode[trace][:12] == 'Detailed+MPI': + content = [] + with open(trace_name + '.mpiio-cycles.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['mpiio_cyc'][trace] = int(float(line.split()[1])) + else: + raw_data['mpiio_cyc'][trace] = 0.0 + + # Get total mpiio instructions + if os.path.exists(trace_name + '.mpiio-inst.stats') \ + and trace_mode[trace][:12] == 'Detailed+MPI': + content = [] + with open(trace_name + '.mpiio-inst.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['mpiio_ins'][trace] = int(float(line.split()[1])) + else: + raw_data['mpiio_ins'][trace] = 0.0 + + # Get useful cycles + if os.path.exists(trace_name + '.cycles.stats'): + content = [] + with open(trace_name + '.cycles.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['useful_cyc'][trace] = int(float(line.split()[1])) + else: + raw_data['useful_cyc'][trace] = 'NaN' + + # Get useful instructions + if os.path.exists(trace_name + '.instructions.stats'): + content = [] + with open(trace_name + '.instructions.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Total': + raw_data['useful_ins'][trace] = int(float(line.split()[1])) + else: + raw_data['useful_ins'][trace] = 'NaN' + + # Get Efficiencies for BurstMode + if trace_mode[trace] == 'Burst+MPI': + # Get total, maximum and avg from burst useful + if os.path.exists(trace_name + '.burst_useful.stats'): + content = [] + list_burst_tot = [] + with open(trace_name + '.burst_useful.stats') as f: + content = f.readlines() + for line in content: + for field in line.split("\n"): + line_list = field.split("\t") + if "Total" in field.split("\t"): + count_procs = len(line_list[1:]) + list_burst_tot = [float(burst_time) for burst_time in line_list[1:count_procs]] + + raw_data['burst_useful_tot'][trace] = sum(list_burst_tot) + raw_data['burst_useful_avg'][trace] = sum(list_burst_tot)/len(list_burst_tot) + raw_data['burst_useful_max'][trace] = max(list_burst_tot) + + else: + raw_data['burst_useful_avg'][trace] = 'NaN' + raw_data['burst_useful_max'][trace] = 'NaN' + raw_data['burst_useful_tot'][trace] = 'NaN' + else: + raw_data['burst_useful_avg'][trace] = 0.0 + raw_data['burst_useful_max'][trace] = 0.0 + raw_data['burst_useful_tot'][trace] = 0.0 + + # Get timing for SIMULATED traces + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + # Get maximum useful duration for simulated trace + if os.path.exists(trace_name_sim + '.timings.stats'): + content = [] + with open(trace_name_sim + '.timings.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Maximum': + raw_data['useful_dim'][trace] = float(line.split()[1]) + else: + raw_data['useful_dim'][trace] = 'NaN' + + # Get runtime for simulated trace + if os.path.exists(trace_name_sim + '.runtime.stats'): + content = [] + with open(trace_name_sim + '.runtime.stats') as f: + content = f.readlines() + + for line in content: + if line.split(): + if line.split()[0] == 'Average': + raw_data['runtime_dim'][trace] = float(line.split()[1]) + else: + raw_data['runtime_dim'][trace] = 'NaN' + + # Get outsideMPI max for simulated trace + if os.path.exists(trace_name_sim + '.outside_mpi.stats'): + with open(trace_name_sim + '.outside_mpi.stats') as f: + content = f.readlines() + list_outside_mpi = [] + list_thread_outside_mpi = [] + init_count_thread = False + count_threads = 1 + for line1 in content[1:(len(content) - 8)]: + line = line1.split("\t") + # print(line) + if line: + if line[0] != 'Total' and line[0] != 'Average' \ + and line[0] != 'Maximum' and line[0] != 'StDev' \ + and line[0] != 'Avg/Max': + # To extract the count of MPI tasks + if float(line[1]) != raw_data['runtime_dim'][trace]: + list_outside_mpi.append(float(line[1])) + # To extract the count of threads per MPI task + if len(list_outside_mpi) > 1: + list_thread_outside_mpi.append(count_threads) + count_threads = 1 + else: + if len(list_outside_mpi) == 1 and not init_count_thread: + count_threads = 2 + init_count_thread = True + else: + count_threads += 1 + + if line[0] == "THREAD 1.1.1": + max_time_outside_mpi = float(line[1]) + + list_thread_outside_mpi.append(count_threads) + if len(list_outside_mpi) != 0: + raw_data['outsidempi_dim'][trace] = max(list_outside_mpi) + else: + raw_data['outsidempi_dim'][trace] = max_time_outside_mpi + else: + raw_data['outsidempi_dim'][trace] = 0.0 + else: + raw_data['useful_dim'][trace] = 'Non-Avail' + raw_data['runtime_dim'][trace] = 'Non-Avail' + raw_data['outsidempi_dim'][trace] = 'Non-Avail' + + # Remove paramedir output files + move_files(trace_name + '.timings.stats', path_dest, cmdl_args) + move_files(trace_name + '.runtime.stats', path_dest, cmdl_args) + move_files(trace_name + '.cycles.stats', path_dest, cmdl_args) + move_files(trace_name + '.instructions.stats', path_dest, cmdl_args) + move_files(trace_name + '.flushing.stats', path_dest, cmdl_args) + move_files(trace_name + '.posixio-cycles.stats', path_dest, cmdl_args) + move_files(trace_name + '.posixio-inst.stats', path_dest, cmdl_args) + move_files(trace_name + '.flushing-cycles.stats', path_dest, cmdl_args) + move_files(trace_name + '.flushing-inst.stats', path_dest, cmdl_args) + + if trace_mode[trace][:12] == 'Detailed+MPI': + move_files(trace_name + '.mpi_io.stats', path_dest, cmdl_args) + move_files(trace_name + '.posixio_call.stats', path_dest, cmdl_args) + move_files(trace_name + '.outside_mpi.stats', path_dest, cmdl_args) + move_files(trace_name + '.mpiio-cycles.stats', path_dest, cmdl_args) + move_files(trace_name + '.mpiio-inst.stats', path_dest, cmdl_args) + + if trace_mode[trace][:9] == 'Burst+MPI': + move_files(trace_name + '.2dh_BurstEff.stats', path_dest, cmdl_args) + move_files(trace_name + '.burst_useful.stats', path_dest, cmdl_args) + + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + move_files(trace_name_sim + '.timings.stats', path_dest, cmdl_args) + move_files(trace_name_sim + '.runtime.stats', path_dest, cmdl_args) + move_files(trace_name_sim + '.outside_mpi.stats', path_dest, cmdl_args) + move_files(trace_sim, path_dest, cmdl_args) + move_files(trace_sim[:-4] + '.pcf', path_dest, cmdl_args) + move_files(trace_sim[:-4] + '.row', path_dest, cmdl_args) + # To move simulation trace and ideal cfg + move_files(trace_sim[:-8] + '.dim', path_dest, cmdl_args) + remove_files(trace_sim[:-8] + '.row', cmdl_args) + remove_files(trace_sim[:-8] + '.pcf', cmdl_args) + move_files(trace_sim[:-8] + '.dimemas_ideal.cfg', path_dest, cmdl_args) + if trace[-7:] == ".prv.gz": + remove_files(trace_name_control + '.prv', cmdl_args) + # move_files(trace_name + '.prv', path_dest, cmdl_args) + + time_prs = time.time() - time_prs + + time_tot = time.time() - time_tot + print('Finished successfully in {0:.1f} seconds.'.format(time_tot)) + print('') + + return raw_data, list_mpi_procs_count + + +def create_ideal_trace(trace, processes, task_per_node, cmdl_args): + """Runs prv2dim and dimemas with ideal configuration for given trace.""" + if trace[-4:] == ".prv": + trace_dim = trace[:-4] + '_' + str(processes) + 'P' + '.dim' + trace_sim = trace[:-4] + '_' + str(processes) + 'P' + '.sim.prv' + trace_name = trace[:-4] + cmd = ['prv2dim', trace, trace_dim] + elif trace[-7:] == ".prv.gz": + with gzip.open(trace, 'rb') as f_in: + with open(trace[:-7] + '.prv', 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + trace_dim = trace[:-7] + '_' + str(processes) + 'P' + '.dim' + trace_sim = trace[:-7] + '_' + str(processes) + 'P' + '.sim.prv' + trace_name = trace[:-7] + trace_unzip = trace[:-3] + cmd = ['prv2dim', trace_unzip, trace_dim] + + run_command(cmd, cmdl_args) + + if os.path.isfile(trace_dim): + if cmdl_args.debug: + print('==DEBUG== Created file ' + trace_dim) + else: + print('==Error== ' + trace_dim + 'could not be creaeted.') + return + + # Create Dimemas configuration + cfg_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs') + + content = [] + with open(os.path.join(cfg_dir, 'dimemas_ideal.cfg')) as f: + content = f.readlines() + + content = [line.replace('REPLACE_BY_CPUS_PER_NODE', str(task_per_node)) for line in content] + content = [line.replace('REPLACE_BY_NTASKS', str(processes)) for line in content] + content = [line.replace('REPLACE_BY_COLLECTIVES_PATH', os.path.join(cfg_dir, 'dimemas.collectives')) for line in + content] + + with open(trace_name + '_' + str(processes) + 'P' + '.dimemas_ideal.cfg', 'w') as f: + f.writelines(content) + + cmd = ['Dimemas', '-S', '32k', '--dim', trace_dim, '-p', trace_sim, trace_name + '_' + str(processes) + + 'P' + '.dimemas_ideal.cfg'] + run_command(cmd, cmdl_args) + + if os.path.isfile(trace_sim): + if cmdl_args.debug: + print('==DEBUG== Created file ' + trace_sim) + return trace_sim + else: + print('==Error== ' + trace_sim + ' could not be created.') + return '' + + +def print_raw_data_table(raw_data, trace_list, trace_processes): + """Prints the raw data table in human readable form on stdout.""" + global raw_data_doc + + print('Overview of the collected raw data:') + + longest_name = len(sorted(raw_data_doc.values(), key=len)[-1]) + + line = ''.rjust(longest_name) + for trace in trace_list: + line += ' | ' + line += str(trace_processes[trace]).rjust(15) + print(''.ljust(len(line), '-')) + print(line) + + print(''.ljust(len(line), '-')) + final_line_raw_data = ''.ljust(len(line), '-') + + for data_key in raw_data_doc: + line = raw_data_doc[data_key].ljust(longest_name) + for trace in trace_list: + line += ' | ' + if raw_data[data_key][trace] != "Non-Avail" and raw_data[data_key][trace] != 'NaN': + line += str(round((raw_data[data_key][trace]),2)).rjust(15) + else: + line += str(raw_data[data_key][trace]).rjust(15) + print(line) + print(final_line_raw_data) + print('') + + +def print_raw_data_csv(raw_data, trace_list, trace_processes): + """Prints the model factors table in a csv file.""" + global raw_data_doc + + delimiter = ';' + # File is stored in the trace directory + # file_path = os.path.join(os.path.dirname(os.path.realpath(trace_list[0])), 'modelfactors.csv') + # File is stored in the execution directory + file_path = os.path.join(os.getcwd(), 'rawdata.csv') + + with open(file_path, 'w') as output: + line = 'Number of processes' + for trace in trace_list: + line += delimiter + line += str(trace_processes[trace]) + output.write(line + '\n') + + for raw_key in raw_data_doc: + line = '#' + raw_data_doc[raw_key] + for trace in trace_list: + line += delimiter + try: # except NaN + line += '{0:.2f}'.format(raw_data[raw_key][trace]) + except ValueError: + line += '{}'.format(raw_data[raw_key][trace]) + output.write(line + '\n') + + print('======== Output Files: Traces raw data and intermediate data ========') + print('Raw data written to ' + file_path) + file_path_intermediate = os.path.join(os.getcwd(), 'scratch_out_basicanalysis') + print('Intermediate file written to ' + file_path_intermediate) + print('') \ No newline at end of file diff --git a/simplemetrics.py b/simplemetrics.py new file mode 100644 index 0000000..fcf0bf8 --- /dev/null +++ b/simplemetrics.py @@ -0,0 +1,1249 @@ +#!/usr/bin/env python3 + +"""Functions to compute the model metrics.""" + +from __future__ import print_function, division +import sys + +from rawdata import * +from collections import OrderedDict + +# error import variables +error_import_pandas = False +error_import_seaborn = False +error_import_matplotlib = False +error_import_numpy = False + +try: + import numpy as np +except ImportError: + error_import_numpy = True + + +try: + import pandas as pd +except ImportError: + error_import_pandas = True + +try: + import seaborn as sns +except ImportError: + error_import_seaborn = True + + +try: + import matplotlib.pyplot as plt +except ImportError: + error_import_matplotlib = True + + + +# Contains all model factor entries with a printable name. +# This is used to generate and print all model factors, so, if an entry is added, +# it should be added here, too. + +other_metrics_doc = OrderedDict([('elapsed_time', 'Elapsed time (sec)'), + ('efficiency', 'Efficiency'), + ('speedup', 'Speedup'), + ('ipc', 'Average IPC'), + ('freq', 'Average frequency (GHz)'), + ('flushing', 'Flushing (%)'), + ('io_mpiio', 'MPI I/O (%)'), + ('io_posix', 'POSIX I/O (%)'), + ('io_eff', 'I/O Efficiency (%)')]) + +mod_factors_doc = OrderedDict([('global_eff', 'Global efficiency'), + ('parallel_eff', '-- Parallel efficiency'), + ('load_balance', ' -- Load balance'), + ('comm_eff', ' -- Communication efficiency'), + ('serial_eff', ' -- Serialization efficiency'), + ('transfer_eff', ' -- Transfer efficiency'), + ('comp_scale', '-- Computation scalability'), + ('ipc_scale', ' -- IPC scalability'), + ('inst_scale', ' -- Instruction scalability'), + ('freq_scale', ' -- Frequency scalability')]) + +mod_factors_scale_plus_io_doc = OrderedDict([('comp_scale', '-- Computation scalability + I/O'), + ('ipc_scale', ' -- IPC scalability'), + ('inst_scale', ' -- Instruction scalability'), + ('freq_scale', ' -- Frequency scalability')]) + + +def create_mod_factors(trace_list): + """Creates 2D dictionary of the model factors and initializes with an empty + string. The mod_factors dictionary has the format: [mod factor key][trace]. + """ + global mod_factors_doc + mod_factors = {} + for key in mod_factors_doc: + trace_dict = {} + for trace_name in trace_list: + trace_dict[trace_name] = 0.0 + mod_factors[key] = trace_dict + + return mod_factors + + +def create_mod_factors_scale_io(trace_list): + """Creates 2D dictionary of the model factors and initializes with an empty + string. The mod_factors dictionary has the format: [mod factor key][trace]. + """ + global mod_factors_scale_plus_io_doc + mod_factors_scale_plus_io = {} + for key in mod_factors_scale_plus_io_doc: + trace_dict = {} + for trace_name in trace_list: + trace_dict[trace_name] = 0.0 + mod_factors_scale_plus_io[key] = trace_dict + + return mod_factors_scale_plus_io + + +def create_other_metrics(trace_list): + """Creates 2D dictionary of the model factors and initializes with an empty + string. The mod_factors dictionary has the format: [mod factor key][trace]. + """ + global other_metrics_doc + other_metrics = {} + for key in other_metrics_doc: + trace_dict = {} + for trace_name in trace_list: + trace_dict[trace_name] = 0.0 + other_metrics[key] = trace_dict + + return other_metrics + + +def get_scaling_type(raw_data, trace_list, trace_processes, cmdl_args): + """Guess the scaling type (weak/strong) based on the useful instructions. + Computes the normalized instruction ratio for all measurements, whereas the + normalized instruction ratio is (instructions ratio / process ratio) with + the smallest run as reference. For exact weak scaling the normalized ratio + should be exactly 1 and for exact strong scaling it should be close to zero + with an upper bound of 0.5. The eps value defines the threshold to be + considered weak scaling and should give enough buffer to safely handle + non-ideal scaling. + """ + eps = 0.9 + normalized_inst_ratio = 0 + + # Check if there is only one trace. + if len(trace_list) == 1: + return 'strong' + + for trace in trace_list: + try: # except NaN + inst_ratio = float(raw_data['useful_ins'][trace]) / float(raw_data['useful_ins'][trace_list[0]]) + except: + inst_ratio = 0.0 + try: # except NaN + proc_ratio = float(trace_processes[trace]) / float(trace_processes[trace_list[0]]) + except: + proc_ratio = 'NaN' + + normalized_inst_ratio += inst_ratio / proc_ratio + + # Get the average inst increase. Ignore ratio of first trace 1.0) + normalized_inst_ratio = (normalized_inst_ratio - 1) / (len(trace_list) - 1) + + scaling_computed = '' + + if normalized_inst_ratio > eps: + scaling_computed = 'weak' + else: + scaling_computed = 'strong' + + if cmdl_args.scaling == 'auto': + if cmdl_args.debug: + print('==DEBUG== Detected ' + scaling_computed + ' scaling.') + print('') + return scaling_computed + + if cmdl_args.scaling == 'weak': + if scaling_computed == 'strong': + print('==Warning== Scaling set to weak scaling but detected strong scaling.') + print('') + return 'weak' + + if cmdl_args.scaling == 'strong': + if scaling_computed == 'weak': + print('==Warning== Scaling set to strong scaling but detected weak scaling.') + print('') + return 'strong' + + print('==Error== reached undefined control flow state.') + sys.exit(1) + + +def compute_model_factors(raw_data, trace_list, trace_processes, trace_mode, list_mpi_procs_count, cmdl_args): + """Computes the model factors from the gathered raw data and returns the + according dictionary of model factors.""" + mod_factors = create_mod_factors(trace_list) + other_metrics = create_other_metrics(trace_list) + mod_factors_scale_plus_io = create_mod_factors_scale_io(trace_list) + + # Guess the weak or strong scaling + scaling = get_scaling_type(raw_data, trace_list, trace_processes, cmdl_args) + + # Loop over all traces + for trace in trace_list: + + proc_ratio = float(trace_processes[trace]) / float(trace_processes[trace_list[0]]) + + # Flushing measurements + try: # except NaN + other_metrics['flushing'][trace] = raw_data['flushing_tot'][trace] \ + / (raw_data['runtime'][trace] * trace_processes[trace]) * 100.0 + except: + other_metrics['flushing'][trace] = 0.0 + + # I/O measurements + try: # except NaN + other_metrics['io_mpiio'][trace] = raw_data['mpiio_tot'][trace] \ + / (raw_data['runtime'][trace] * trace_processes[trace]) * 100.0 + except: + other_metrics['io_mpiio'][trace] = 0.0 + + try: # except NaN + other_metrics['io_posix'][trace] = raw_data['io_tot'][trace] \ + / (raw_data['runtime'][trace] * trace_processes[trace]) * 100.0 + except: + other_metrics['io_posix'][trace] = 0.0 + try: # except NaN + io_total = raw_data['mpiio_tot'][trace] + raw_data['flushing_tot'][trace] + raw_data['io_tot'][trace] + other_metrics['io_eff'][trace] = raw_data['useful_tot'][trace] \ + / (raw_data['useful_tot'][trace] + io_total) * 100.0 + except: + other_metrics['io_eff'][trace] = 0.0 + + # Basic efficiency factors + try: # except NaN + if trace_mode[trace] == 'Burst+MPI': + mod_factors['load_balance'][trace] = float(raw_data['burst_useful_avg'][trace]) \ + / float(raw_data['burst_useful_max'][trace]) * 100.0 + else: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: + mod_factors['load_balance'][trace] = raw_data['useful_plus_io_avg'][trace] \ + / raw_data['useful_plus_io_max'][trace] * 100.0 + else: + mod_factors['load_balance'][trace] = raw_data['useful_avg'][trace] \ + / raw_data['useful_max'][trace] * 100.0 + except: + mod_factors['load_balance'][trace] = 'NaN' + + try: # except NaN + if trace_mode[trace] == 'Burst+MPI': + mod_factors['comm_eff'][trace] = float(raw_data['burst_useful_max'][trace]) \ + / raw_data['runtime'][trace] * 100.0 + else: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: + mod_factors['comm_eff'][trace] = raw_data['useful_plus_io_max'][trace] \ + / raw_data['runtime'][trace] * 100.0 + else: + mod_factors['comm_eff'][trace] = raw_data['useful_max'][trace] \ + / raw_data['runtime'][trace] * 100.0 + except: + mod_factors['comm_eff'][trace] = 'NaN' + + try: # except NaN + mod_factors['serial_eff'][trace] = float(raw_data['outsidempi_dim'][trace]) \ + / float(raw_data['runtime_dim'][trace]) * 100.0 + if mod_factors['serial_eff'][trace] > 100.0: + mod_factors['serial_eff'][trace] = 'Warning!' + except: + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + mod_factors['serial_eff'][trace] = 'NaN' + else: + mod_factors['serial_eff'][trace] = 'Non-Avail' + + try: # except NaN + if mod_factors['serial_eff'][trace] != 'Warning!': + mod_factors['transfer_eff'][trace] = mod_factors['comm_eff'][trace] \ + / mod_factors['serial_eff'][trace] * 100.0 + else: + mod_factors['transfer_eff'][trace] = float(raw_data['runtime_dim'][trace]) \ + / float(raw_data['runtime'][trace]) * 100.0 + + if mod_factors['transfer_eff'][trace] > 100.0: + mod_factors['transfer_eff'][trace] = 'Warning!' + except: + if trace_mode[trace] == 'Detailed+MPI' or trace_mode[trace] == 'Detailed+MPI+OpenMP': + mod_factors['transfer_eff'][trace] = 'NaN' + else: + mod_factors['transfer_eff'][trace] = 'Non-Avail' + + # Parallel Efficiency + try: # except NaN + if trace_mode[trace] == 'Burst+MPI': + mod_factors['parallel_eff'][trace] = float(raw_data['burst_useful_avg'][trace]) \ + / float(raw_data['runtime'][trace]) * 100.0 + else: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 5.0: + mod_factors['parallel_eff'][trace] = float(raw_data['useful_plus_io_avg'][trace]) \ + / float(raw_data['runtime'][trace]) * 100.0 + else: + mod_factors['parallel_eff'][trace] = mod_factors['load_balance'][trace] \ + * mod_factors['comm_eff'][trace] / 100.0 + except: + mod_factors['parallel_eff'][trace] = 'NaN' + + # Computation Scale only useful computation + try: # except NaN + if len(trace_list) > 1: + if trace_mode[trace] == 'Burst+MPI': + if scaling == 'strong': + mod_factors['comp_scale'][trace] = raw_data['burst_useful_tot'][trace_list[0]] \ + / raw_data['burst_useful_tot'][trace] * 100.0 + else: + mod_factors['comp_scale'][trace] = raw_data['burst_useful_tot'][trace_list[0]] \ + / raw_data['burst_useful_tot'][trace] \ + * proc_ratio * 100.0 + else: + if scaling == 'strong': + mod_factors['comp_scale'][trace] = raw_data['useful_tot'][trace_list[0]] \ + / raw_data['useful_tot'][trace] * 100.0 + else: + mod_factors['comp_scale'][trace] = raw_data['useful_tot'][trace_list[0]] \ + / raw_data['useful_tot'][trace] \ + * proc_ratio * 100.0 + else: + mod_factors['comp_scale'][trace] = 'Non-Avail' + except: + mod_factors['comp_scale'][trace] = 'NaN' + + # Computation Scale + Serial I/O + try: # except NaN + if len(trace_list) > 1: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: + io_serial_0 = raw_data['io_tot'][trace_list[0]] + raw_data['flushing_tot'][trace_list[0]] + io_serial_n = raw_data['io_tot'][trace] + raw_data['flushing_tot'][trace] + if scaling == 'strong': + mod_factors_scale_plus_io['comp_scale'][trace] = (raw_data['useful_tot'][trace_list[0]] + + io_serial_0) / (raw_data['useful_tot'][trace] + + io_serial_n) * 100.0 + else: + mod_factors_scale_plus_io['comp_scale'][trace] = (raw_data['useful_tot'][trace_list[0]] + + io_serial_0) \ + / (raw_data['useful_tot'][trace] + io_serial_n) \ + * proc_ratio * 100.0 + else: + mod_factors_scale_plus_io['comp_scale'][trace] = mod_factors['comp_scale'][trace] + else: + mod_factors_scale_plus_io['comp_scale'][trace] = 'Non-Avail' + except: + mod_factors_scale_plus_io['comp_scale'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + mod_factors['global_eff'][trace] = mod_factors['parallel_eff'][trace] \ + * mod_factors['comp_scale'][trace] / 100.0 + else: + mod_factors['global_eff'][trace] = mod_factors['parallel_eff'][trace] \ + * 100.0 / 100.0 + except: + mod_factors['global_eff'][trace] = 'NaN' + + # Basic scalability factors + try: # except NaN + other_metrics['ipc'][trace] = float(raw_data['useful_ins'][trace]) \ + / float(raw_data['useful_cyc'][trace]) + except: + other_metrics['ipc'][trace] = 'NaN' + try: # except NaN + if len(trace_list) > 1: + if trace_mode[trace][:5] != 'Burst': + mod_factors['ipc_scale'][trace] = other_metrics['ipc'][trace] \ + / other_metrics['ipc'][trace_list[0]] * 100.0 + else: + mod_factors['ipc_scale'][trace] = 'Non-Avail' + else: + mod_factors['ipc_scale'][trace] = 'Non-Avail' + except: + mod_factors['ipc_scale'][trace] = 'NaN' + + # IPC scale + Serial I/O + try: # except NaN + if len(trace_list) > 1: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: + ipc_serial_io_0 = (raw_data['useful_ins'][trace_list[0]] + raw_data['io_ins'][trace_list[0]] + + raw_data['flushing_ins'][trace_list[0]]) \ + / (raw_data['useful_cyc'][trace_list[0]] + raw_data['io_cyc'][trace_list[0]] + + raw_data['flushing_cyc'][trace_list[0]]) + + ipc_serial_io_n = (raw_data['useful_ins'][trace] + raw_data['io_ins'][trace] + + raw_data['flushing_ins'][trace]) \ + / (raw_data['useful_cyc'][trace] + raw_data['io_cyc'][trace] + + raw_data['flushing_cyc'][trace]) + mod_factors_scale_plus_io['ipc_scale'][trace] = ipc_serial_io_n / ipc_serial_io_0 * 100.0 + else: + mod_factors_scale_plus_io['ipc_scale'][trace] = 0.0 + else: + mod_factors_scale_plus_io['ipc_scale'][trace] = 'Non-Avail' + except: + mod_factors_scale_plus_io['ipc_scale'][trace] = mod_factors['ipc_scale'][trace] + + try: # except NaN + other_metrics['freq'][trace] = float(raw_data['useful_cyc'][trace]) \ + / float(raw_data['useful_tot'][trace]) / 1000 + except: + other_metrics['freq'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + if trace_mode[trace][:5] != 'Burst': + mod_factors['freq_scale'][trace] = other_metrics['freq'][trace] \ + / other_metrics['freq'][trace_list[0]] * 100.0 + else: + mod_factors['freq_scale'][trace] = 'Non-Avail' + else: + mod_factors['freq_scale'][trace] = 'Non-Avail' + except: + mod_factors['freq_scale'][trace] = 'NaN' + + # freq scale + Serial I/O + try: # except NaN + if len(trace_list) > 1: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: + freq_serial_io_0 = (float(raw_data['useful_cyc'][trace_list[0]]) + + float(raw_data['io_cyc'][trace_list[0]]) + + float(raw_data['flushing_cyc'][trace_list[0]])) \ + / (float(raw_data['useful_tot'][trace_list[0]]) + + float(raw_data['io_tot'][trace_list[0]]) + + float(raw_data['flushing_tot'][trace_list[0]])) / 1000 + + freq_serial_io_n = (float(raw_data['useful_cyc'][trace]) + float(raw_data['io_cyc'][trace]) + + float(raw_data['flushing_cyc'][trace])) \ + / (float(raw_data['useful_tot'][trace]) + + float(raw_data['io_tot'][trace]) + + float(raw_data['flushing_tot'][trace])) / 1000 + mod_factors_scale_plus_io['freq_scale'][trace] = freq_serial_io_n / freq_serial_io_0 * 100.0 + else: + mod_factors_scale_plus_io['freq_scale'][trace] = mod_factors['freq_scale'][trace] + else: + mod_factors_scale_plus_io['freq_scale'][trace] = 'Non-Avail' + except: + mod_factors_scale_plus_io['freq_scale'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + if trace_mode[trace][:5] != 'Burst': + if scaling == 'strong': + mod_factors['inst_scale'][trace] = float(raw_data['useful_ins'][trace_list[0]]) \ + / float(raw_data['useful_ins'][trace]) * 100.0 + else: + mod_factors['inst_scale'][trace] = float(raw_data['useful_ins'][trace_list[0]]) \ + / float(raw_data['useful_ins'][trace]) \ + * proc_ratio * 100.0 + else: + mod_factors['inst_scale'][trace] = 'Non-Avail' + else: + mod_factors['inst_scale'][trace] = 'Non-Avail' + except: + mod_factors['inst_scale'][trace] = 'NaN' + + # ins scale + Serial I/O + try: # except NaN + if len(trace_list) > 1: + if other_metrics['io_posix'][trace] > 0.0 or other_metrics['flushing'][trace] > 0.0: + useful_ins_plus_io_0 = float(raw_data['useful_ins'][trace_list[0]]) \ + + float(raw_data['io_ins'][trace_list[0]]) \ + + float(raw_data['flushing_ins'][trace_list[0]]) + useful_ins_plus_io_n = float(raw_data['useful_ins'][trace]) \ + + float(raw_data['io_ins'][trace]) \ + + float(raw_data['flushing_ins'][trace]) + if scaling == 'strong': + mod_factors_scale_plus_io['inst_scale'][trace] = useful_ins_plus_io_0 \ + / useful_ins_plus_io_n * 100.0 + else: + mod_factors_scale_plus_io['inst_scale'][trace] = useful_ins_plus_io_0 / useful_ins_plus_io_n \ + * proc_ratio * 100.0 + else: + mod_factors_scale_plus_io['inst_scale'][trace] = mod_factors['inst_scale'][trace] + else: + mod_factors_scale_plus_io['inst_scale'][trace] = 'Non-Avail' + except: + mod_factors_scale_plus_io['inst_scale'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + other_metrics['speedup'][trace] = raw_data['runtime'][trace_list[0]] \ + / raw_data['runtime'][trace] + #if scaling == 'strong': + # other_metrics['speedup'][trace] = raw_data['runtime'][trace_list[0]] \ + # / raw_data['runtime'][trace] + #else: + # other_metrics['speedup'][trace] = raw_data['runtime'][trace_list[0]] \ + # / raw_data['runtime'][trace] * proc_ratio + else: + other_metrics['speedup'][trace] = 'Non-Avail' + except: + other_metrics['speedup'][trace] = 'NaN' + + try: # except NaN + other_metrics['elapsed_time'][trace] = raw_data['runtime'][trace] * 0.000001 + except: + other_metrics['elapsed_time'][trace] = 'NaN' + + try: # except NaN + if len(trace_list) > 1: + #other_metrics['efficiency'][trace] = other_metrics['speedup'][trace] / proc_ratio + if scaling == 'strong': + other_metrics['efficiency'][trace] = raw_data['runtime'][trace_list[0]] \ + / (raw_data['runtime'][trace] * proc_ratio) + else: + other_metrics['efficiency'][trace] = raw_data['runtime'][trace_list[0]] \ + / raw_data['runtime'][trace] + else: + other_metrics['efficiency'][trace] = 'Non-Avail' + except: + other_metrics['efficiency'][trace] = 'NaN' + + return mod_factors, mod_factors_scale_plus_io, other_metrics + + +def read_mod_factors_csv(cmdl_args): + """Reads the model factors table from a csv file.""" + global mod_factors_doc + + delimiter = ';' + file_path = cmdl_args.project + + # Read csv to list of lines + if os.path.isfile(file_path) and file_path[-4:] == '.csv': + with open(file_path, 'r') as f: + lines = f.readlines() + lines = [line.rstrip('\n') for line in lines] + else: + print('==ERROR==', file_path, 'is not a valid csv file.') + sys.exit(1) + + # Get the number of processes of the traces + processes = lines[0].split(delimiter) + processes.pop(0) + + # Create artificial trace_list and trace_processes + trace_list = [] + trace_processes = dict() + for process in processes: + trace_list.append(process) + trace_processes[process] = int(process) + + # Create empty mod_factors handle + mod_factors = create_mod_factors(trace_list) + + # Get mod_factor_doc keys + mod_factors_keys = list(mod_factors_doc.items()) + + # Iterate over the data lines + for index, line in enumerate(lines[1:len(mod_factors_keys) + 1]): + key = mod_factors_keys[index][0] + line = line.split(delimiter) + for index, trace in enumerate(trace_list): + mod_factors[key][trace] = float(line[index + 1]) + + if cmdl_args.debug: + print_mod_factors_table(mod_factors, trace_list, trace_processes) + + return mod_factors, trace_list, trace_processes + + +def print_mod_factors_table(mod_factors, other_metrics, mod_factors_scale_plus_io, trace_list, trace_processes): + """Prints the model factors table in human readable form on stdout.""" + global mod_factors_doc + global mod_factors_scale_plus_io_doc + + warning_io = [] + warning_flush = [] + warning_flush_wrong = [] + warning_simulation = [] + for trace in trace_list: + if 10.0 <= other_metrics['flushing'][trace] < 15.0: + warning_flush.append(1) + elif other_metrics['flushing'][trace] >= 15.0: + warning_flush_wrong.append(1) + if other_metrics['io_posix'][trace] >= 5.0: + warning_io.append(1) + if mod_factors['serial_eff'][trace] == "Warning!" \ + or mod_factors['transfer_eff'][trace] == "Warning!": + warning_simulation.append(1) + if len(warning_flush_wrong) > 0: + print("WARNING! Flushing in a trace is too high. Disabling standard output metrics...") + print(" Flushing is an overhead due to the tracer, please review your trace.") + print('') + return + + print('Overview of the Efficiency metrics:') + + longest_name = len(sorted(mod_factors_doc.values(), key=len)[-1]) + + line = ''.rjust(longest_name) + + if len(trace_list) == 1: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[0]] + else: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[len(trace_list)-1]] + + # BEGIN To adjust header to big number of processes + procs_header = [] + for index, trace in enumerate(trace_list): + if limit_min == limit_max and len(trace_list) > 1: + procs_header.append(str(trace_processes[trace]) + '[' + str(index+1) + ']') + else: + procs_header.append(str(trace_processes[trace])) + + max_len_header = 0 + for proc_h in procs_header: + if max_len_header < len(proc_h): + max_len_header = len(proc_h) + + value_to_adjust = 10 + if max_len_header > value_to_adjust: + value_to_adjust = max_len_header + 1 + # END To adjust header to big number of processes + + for index, trace in enumerate(trace_list): + line += ' | ' + if limit_min == limit_max and len(trace_list) > 1: + line += (str(trace_processes[trace]) + '[' + str(index+1) + ']').rjust(value_to_adjust) + else: + line += (str(trace_processes[trace])).rjust(value_to_adjust) + + print(''.ljust(len(line), '=')) + print(line) + line_procs_factors = line + + print(''.ljust(len(line), '=')) + + for mod_key in mod_factors_doc: + line = mod_factors_doc[mod_key].ljust(longest_name) + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(mod_factors[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(mod_factors[mod_key][trace])).rjust(value_to_adjust) + print(line) + + # print('') + + io_metrics = [] + io_metrics.append(0) + for mod_key in mod_factors_scale_plus_io_doc: + for trace in trace_list: + if mod_factors_scale_plus_io[mod_key][trace] != 0: + io_metrics.append(1) + + if (len(warning_flush) >= 1 or len(warning_io) >= 1) and len(trace_list) > 1: + print(''.ljust(len(line_procs_factors), '-')) + for mod_key in mod_factors_scale_plus_io_doc: + line = mod_factors_scale_plus_io_doc[mod_key].ljust(longest_name) + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(mod_factors_scale_plus_io[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(mod_factors_scale_plus_io[mod_key][trace])).rjust(value_to_adjust) + print(line) + + print(''.ljust(len(line_procs_factors), '=')) + + if len(warning_simulation) > 0: + print("===> Warning! Metrics obtained from simulated traces exceed 100%. " + "Please review original and simulated traces.") + print('') + + +def print_other_metrics_table(other_metrics, trace_list, trace_processes): + """Prints the model factors table in human readable form on stdout.""" + global other_metrics_doc + + print('Overview of the Speedup, IPC and Frequency:') + + longest_name = len(sorted(other_metrics_doc.values(), key=len)[-1]) + + line = ''.rjust(longest_name) + if len(trace_list) == 1: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[0]] + else: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[len(trace_list)-1]] + + # BEGIN To adjust header to big number of processes + procs_header = [] + for index, trace in enumerate(trace_list): + if limit_min == limit_max and len(trace_list) > 1: + procs_header.append(str(trace_processes[trace]) + '[' + str(index+1) + ']') + else: + procs_header.append(str(trace_processes[trace])) + + max_len_header = 0 + for proc_h in procs_header: + if max_len_header < len(proc_h): + max_len_header = len(proc_h) + + value_to_adjust = 10 + if max_len_header > value_to_adjust: + value_to_adjust = max_len_header + 1 + # END To adjust header to big number of processes + + for index, trace in enumerate(trace_list): + line += ' | ' + if limit_min == limit_max and len(trace_list) > 1: + line += (str(trace_processes[trace]) + '[' + str(index+1) + ']').rjust(value_to_adjust) + else: + line += (str(trace_processes[trace])).rjust(value_to_adjust) + + print(''.ljust(len(line), '-')) + print(line) + line_head = line + print(''.ljust(len(line), '-')) + + for mod_key in other_metrics_doc: + line = other_metrics_doc[mod_key].ljust(longest_name) + if len(trace_list) > 1: + if mod_key in ['speedup', 'ipc', 'freq', 'elapsed_time', 'efficiency']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + print(line) + else: + if mod_key in ['ipc', 'freq', 'elapsed_time']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + print(line) + print(''.ljust(len(line_head), '-')) + # print('') + + warning_io = [] + warning_flush = [] + for trace in trace_list: + if other_metrics['flushing'][trace] >= 10.0: + warning_flush.append(1) + if other_metrics['io_mpiio'][trace] >= 5.0 or other_metrics['io_posix'][trace] >= 5.0: + warning_io.append(1) + + for mod_key in other_metrics_doc: + line = other_metrics_doc[mod_key].ljust(longest_name) + # Print empty line to separate values + if mod_key in ['freq'] and len(warning_flush) > 0: + print("Overview of tracer\'s flushing weight:") + print(''.ljust(len(line_head), '-')) + + if mod_key not in ['speedup', 'ipc', 'freq', 'elapsed_time','efficiency']: + if mod_key in ['flushing']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + if len(warning_flush) > 0: + print(line) + print(''.ljust(len(line_head), '-')) + elif mod_key in ['io_mpiio','io_posix']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + if len(warning_io) > 0: + print(line) + elif mod_key in ['io_eff']: + for trace in trace_list: + line += ' | ' + try: # except NaN + line += ('{0:.2f}%'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + except ValueError: + line += ('{}'.format(other_metrics[mod_key][trace])).rjust(value_to_adjust) + if len(warning_io) > 0 or len(warning_flush) > 0: + print(line) + + # Print headers I/O + if mod_key in ['flushing'] and len(warning_io) > 0: + print(''.ljust(len(line), ' ')) + print('Overview of File I/O weight:') + print(''.ljust(len(line), '-')) + if mod_key in ['io_eff'] and len(warning_io) > 0: + print(''.ljust(len(line), '-')) + # print('') + + if len(warning_flush) > 0: + message_warning_flush = "WARNING! %Flushing is high and affects computation of efficiency metrics." + else: + message_warning_flush = "" + if len(warning_io) > 0: + message_warning_io = "WARNING! % File I/O is high and affects computation of efficiency metrics." + else: + message_warning_io = "" + print(message_warning_flush + message_warning_io) + print('') + + +def print_efficiency_table(mod_factors, trace_list, trace_processes): + """Prints the model factors table in human readable form on stdout.""" + global mod_factors_doc + + longest_name = len(sorted(mod_factors_doc.values(), key=len)[-1]) + delimiter = ',' + file_path = os.path.join(os.getcwd(), 'efficiency_table.csv') + with open(file_path, 'w') as output: + line = '\"Number of processes\"' + if len(trace_list) == 1: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[0]] + else: + limit_min = trace_processes[trace_list[0]] + limit_max = trace_processes[trace_list[len(trace_list)-1]] + + for index, trace in enumerate(trace_list): + line += delimiter + if limit_min == limit_max and len(trace_list) > 1: + line += str(trace_processes[trace]) + '[' + str(index+1) + ']' + else: + line += str(trace_processes[trace]) + output.write(line + '\n') + + for mod_key in mod_factors_doc: + if mod_key not in ['speedup', 'ipc', 'freq', 'elapsed_time', 'efficiency', 'flushing', 'io_mpiio', 'io_posix']: + if mod_key in ['parallel_eff', 'comp_scale']: + line = "\"" + mod_factors_doc[mod_key].replace(' ', '', 2) + "\"" + elif mod_key in ['load_balance', 'comm_eff','ipc_scale', 'inst_scale','freq_scale']: + line = "\"" + mod_factors_doc[mod_key].replace(' ', ' ', 2) + "\"" + elif mod_key in ['serial_eff', 'transfer_eff']: + line = "\" " + mod_factors_doc[mod_key].replace(' ', ' ', 4) + "\"" + else: + line = "\"" + mod_factors_doc[mod_key] + "\"" + for trace in trace_list: + line += delimiter + try: # except NaN + if mod_factors[mod_key][trace] == "Non-Avail" or mod_factors[mod_key][trace] == "Warning!": + line += '0.00' + else: + line += '{0:.2f}'.format(mod_factors[mod_key][trace]) + except ValueError: + line += '{}'.format(mod_factors[mod_key][trace]) + output.write(line + '\n') + # print('') + + # Create Gnuplot file for efficiency plot + gp_template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cfgs', 'efficiency_table.gp') + content = [] + + with open(gp_template) as f: + content = f.readlines() + + limit_procs = 500 + len(trace_list) * 60 + + # Replace xrange + content = [line.replace('#REPLACE_BY_SIZE', ''.join(['set terminal pngcairo enhanced dashed crop size ', + str(limit_procs), ',460 font "Latin Modern Roman,14"'])) + for line in content] + + file_path = os.path.join(os.getcwd(), 'efficiency_table.gp') + with open(file_path, 'w') as f: + f.writelines(content) + + # print('======== Plot (gnuplot File): EFFICIENCY Table ========') + if len(trace_list) > 1: + print('Efficiency Table written to ' + file_path[:len(file_path) - 3] + '.gp') + # print('') + + +def print_mod_factors_csv(mod_factors, trace_list, trace_processes): + """Prints the model factors table in a csv file.""" + global mod_factors_doc + + delimiter = ';' + # File is stored in the trace directory + # file_path = os.path.join(os.path.dirname(os.path.realpath(trace_list[0])), 'modelfactors.csv') + # File is stored in the execution directory + file_path = os.path.join(os.getcwd(), 'modelfactors.csv') + + with open(file_path, 'w') as output: + line = "\"Number of processes\"" + for trace in trace_list: + line += delimiter + line += str(trace_processes[trace]) + output.write(line + '\n') + + for mod_key in mod_factors_doc: + line = "\"" + mod_factors_doc[mod_key].replace(' ', '', 2) + "\"" + for trace in trace_list: + line += delimiter + try: # except NaN + line += '{0:.6f}'.format(mod_factors[mod_key][trace]) + except ValueError: + line += '{}'.format(mod_factors[mod_key][trace]) + output.write(line + '\n') + + output.write('#\n') + + print('======== Output Files: Metrics and Plots ========') + print('Model factors written to ' + file_path) + + +def print_other_metrics_csv(other_metrics, trace_list, trace_processes): + """Prints the model factors table in a csv file.""" + global other_metrics_doc + + delimiter = ';' + # File is stored in the trace directory + # file_path = os.path.join(os.path.dirname(os.path.realpath(trace_list[0])), 'modelfactors.csv') + # File is stored in the execution directory + file_path = os.path.join(os.getcwd(), 'other_metrics.csv') + + with open(file_path, 'w') as output: + line = "\"Number of processes\"" + for trace in trace_list: + line += delimiter + line += str(trace_processes[trace]) + output.write(line + '\n') + + for mod_key in other_metrics_doc: + line = "\"" + other_metrics_doc[mod_key].replace(' ', '', 2) +"\"" + for trace in trace_list: + line += delimiter + try: # except NaN + line += '{0:.6f}'.format(other_metrics[mod_key][trace]) + except ValueError: + line += '{}'.format(other_metrics[mod_key][trace]) + output.write(line + '\n') + + output.write('#\n') + + print('======== Output File: Other Metrics ========') + print('Speedup, IPC, Frequency, I/O and Flushing written to ' + file_path) + print('') + + +def plots_efficiency_table_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args): + # Plotting using python + # For plotting using python, read the csv file + + file_path = os.path.join(os.getcwd(), 'efficiency_table.csv') + df = pd.read_csv(file_path) + metrics = df['Number of processes'].tolist() + + traces_procs = list(df.keys())[1:] + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # Set limit for projection + if cmdl_args.limit: + limit = cmdl_args.limit + else: + limit = str(trace_processes[trace_list[len(trace_list)-1]]) + + limit_min = trace_processes[trace_list[0]] + + # To xticks label + label_xtics = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if int(limit) == int(limit_min) and same_procs: + s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' + elif int(limit) == int(limit_min) and not same_procs: + s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' + str(threads) + ')' + else: + s_xtics = str(trace_processes[trace]) + if s_xtics in label_xtics: + s_xtics += '[' + str(index + 1) + ']' + label_xtics.append(s_xtics) + ##### End xticks + + # BEGIN To adjust header to big number of processes + max_len_header = 7 + for labelx in label_xtics: + if len(labelx) > max_len_header: + max_len_header = len(labelx) + + # END To adjust header to big number of processes + list_data = [] + for index, rows in df.iterrows(): + list_temp = [] + for value in list(rows)[1:]: + if float(value) == 0.0: + list_temp.append(np.nan) + else: + list_temp.append(float(value)) + # print(list_temp) + list_data.append(list_temp) + + list_np = np.array(list_data) + + idx = metrics + cols = label_xtics + #cols = traces_procs + df = pd.DataFrame(list_np, index=idx, columns=cols) + + # min for 1 traces is x=3 for the (x,y) in figsize + size_figure_y = len(idx) * 0.40 + size_figure_x = len(cols) * 0.16 * max_len_header + plt.figure(figsize=(size_figure_x, size_figure_y)) + + ax = sns.heatmap(df, cmap='RdYlGn', linewidths=0.05, annot=True, vmin=0, vmax=100, center=75, \ + fmt='.2f', annot_kws={"size": 10}, cbar_kws={'label': 'Percentage(%)'}) + ## to align ylabels to left + plt.yticks(rotation=0, ha='left') + ax.xaxis.tick_top() + # to adjust metrics + len_pad = 0 + for metric in metrics: + if len(metric) > len_pad: + len_pad = len(metric) + + ax.yaxis.set_tick_params(pad=len_pad + 120) + plt.savefig('efficiency_table-matplot.png', bbox_inches='tight') + + +def plots_modelfactors_matplot(trace_list, trace_mode, trace_processes, trace_tasks, trace_threads, cmdl_args): + # Plotting using python + # For plotting using python, read the csv file + file_path = os.path.join(os.getcwd(), 'modelfactors.csv') + df = pd.read_csv(file_path, sep=';') + + # metrics = df['Number of processes'].tolist() + + traces_procs = list(df.keys())[1:] + + list_data = [] + for index, rows in df.iterrows(): + list_temp = [] + for value in list(rows)[1:]: + if value != 'Non-Avail' and value != "Warning!": + list_temp.append(float(value)) + elif value == 'Non-Avail' or value == "Warning!": + list_temp.append(float('nan')) + #print(list_temp) + list_data.append(list_temp) + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_tasks[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # Set limit for projection + if cmdl_args.limit: + limit = cmdl_args.limit + else: + limit = str(trace_processes[trace_list[len(trace_list)-1]]) + + limit_min = trace_processes[trace_list[0]] + + # To xticks label + label_xtics = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_tasks[trace] + if int(limit) == int(limit_min) and same_procs: + s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' + elif int(limit) == int(limit_min) and not same_procs: + s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' \ + + str(threads) + ')' + else: + s_xtics = str(trace_processes[trace]) + label_xtics.append(s_xtics) + + ### Plot: Global Metrics + # print(list_data) + plt.figure() + max_global = max([max(list_data[0]), max(list_data[1]), max(list_data[2]), max(list_data[3]), max(list_data[6])]) + plt.plot(traces_procs, list_data[0], 'o-', color='black', label='Global Efficiency') + plt.plot(traces_procs, list_data[1], 's--', color='magenta', label='Parallel Efficiency') + plt.plot(traces_procs, list_data[2], 'X:', color='red', linewidth=2, label='Load Balance') + plt.plot(traces_procs, list_data[3], 'x-.', color='green', label='Communication efficiency') + plt.plot(traces_procs, list_data[6], 'v--', color='blue', label='Computation scalability') + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency (%)") + plt.xticks(tuple(traces_procs), tuple(label_xtics)) + # print(max_global) + if float(max_global) < 100: + max_global = 100 + + plt.ylim(0, float(max_global)+5) + plt.legend() + plt.savefig('modelfactors-matplot.png', bbox_inches='tight') + + # Plot: Comm Metrics + if trace_mode[trace] == 'Detailed+MPI': + plt.figure() + max_comm = max([max(list_data[3]), max(list_data[4]), max(list_data[5])]) + plt.plot(traces_procs, list_data[3], 's-', color='green', label='Communication efficiency') + plt.plot(traces_procs, list_data[4], 'h--', color='gold', label='Serialization efficiency') + plt.plot(traces_procs, list_data[5], 'x:', color='tomato', label='Transfer efficiency') + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency (%)") + plt.xticks(tuple(traces_procs), tuple(label_xtics)) + if float(max_comm) < 100: + max_comm = 100 + + plt.ylim(0, float(max_comm)+5) + plt.legend() + plt.savefig('modelfactors-comm-matplot.png', bbox_inches='tight') + + ### Plot: Scale Metrics + if trace_mode[trace][:5] != 'Burst': + plt.figure() + max_scale = max([max(list_data[6]), max(list_data[7]), max(list_data[8]), max(list_data[9])]) + plt.plot(traces_procs, list_data[6], 'v-', color='blue', label='Computation scalability') + plt.plot(traces_procs, list_data[7], 'v--', color='skyblue', label='IPC scalability') + plt.plot(traces_procs, list_data[8], 'v:', color='gray', label='Instruction scalability') + plt.plot(traces_procs, list_data[9], 'v-.', color='darkviolet', label='Frequency scalability') + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency (%)") + plt.xticks(tuple(traces_procs), tuple(label_xtics)) + if float(max_scale) < 100: + max_scale = 100 + + plt.ylim(0, float(max_scale)+5) + plt.legend() + plt.savefig('modelfactors-scale-matplot.png', bbox_inches='tight') + + +def plots_speedup_matplot(trace_list, trace_processes, trace_tasks, trace_threads, cmdl_args): + # Plotting using python + # For plotting using python, read the csv file + file_path = os.path.join(os.getcwd(), 'other_metrics.csv') + df = pd.read_csv(file_path, sep=';') + + traces_procs = list(df.keys())[1:] + + list_data = [] + for index, rows in df.iterrows(): + list_temp = [] + for value in list(rows)[1:]: + if value != 'Non-Avail': + list_temp.append(float(value)) + elif value == 'Non-Avail': + list_temp.append(float('nan')) + # print(list_temp) + list_data.append(list_temp) + + # To control same number of processes for the header on plots and table + same_procs = True + procs_trace_prev = trace_processes[trace_list[0]] + tasks_trace_prev = trace_tasks[trace_list[0]] + threads_trace_prev = trace_threads[trace_list[0]] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if procs_trace_prev == trace_processes[trace] and tasks_trace_prev == tasks \ + and threads_trace_prev == threads: + same_procs *= True + else: + same_procs *= False + + # Set limit for projection + if cmdl_args.limit: + limit = cmdl_args.limit + else: + limit = str(trace_processes[trace_list[len(trace_list) - 1]]) + + limit_min = trace_processes[trace_list[0]] + + # proc_ratio to ideal speedup + proc_ratio = [] + for index, trace in enumerate(trace_list): + proc_ratio.append(trace_processes[trace]/trace_processes[trace_list[0]]) + # print(proc_ratio) + + # To xticks label + label_xtics = [] + for index, trace in enumerate(trace_list): + tasks = trace_tasks[trace] + threads = trace_threads[trace] + if int(limit) == int(limit_min) and same_procs: + s_xtics = str(trace_processes[trace]) + '[' + str(index + 1) + ']' + elif int(limit) == int(limit_min) and not same_procs: + s_xtics = str(trace_processes[trace]) + '(' + str(tasks) + 'x' \ + + str(threads) + ')' + else: + s_xtics = str(trace_processes[trace]) + label_xtics.append(s_xtics) + + ### Plot: SpeedUp + # print(list_data) + + int_traces_procs = [] + prev_procs = int(float(traces_procs[0])) + int_traces_procs.append(int(float(traces_procs[0]))) + count_rep = 0 + for procs in traces_procs[1:]: + if prev_procs == int(float(procs)): + count_rep += 1 + int_traces_procs.append(int(float(procs)) + (2 * count_rep)) + prev_procs = int(float(procs)) + else: + int_traces_procs.append(int(float(procs))) + prev_procs = int(float(procs)) + count_rep = 0 + + + plt.figure() + for x, y in zip(int_traces_procs, list_data[2]): + label = "{:.2f}".format(y) + plt.annotate(label, (x, y), textcoords="offset points", xytext=(0, 10), ha='center') + plt.plot(int_traces_procs, list_data[2], 'o-', color='blue', label='measured') + plt.plot(int_traces_procs, proc_ratio, 'o-', color='black', label='ideal') + plt.xlabel("Number of Processes") + plt.ylabel("SpeedUp") + plt.xticks(tuple(int_traces_procs), tuple(label_xtics)) + #plt.yscale('log') + plt.legend() + # plt.xlim(0, ) + plt.ylim(0, ) + plt.savefig('speedup-matplot.png', bbox_inches='tight') + + ### Plot: Efficiency + # print(list_data) + plt.figure() + for x, y in zip(int_traces_procs, list_data[1]): + label = "{:.2f}".format(y) + plt.annotate(label, (x, y), textcoords="offset points", xytext=(0, 10), ha='center') + + plt.plot(int_traces_procs, list_data[1], 'o-', color='blue', label='measured') + plt.axhline(y=1, color='black', linestyle='-', label='ideal') + plt.xlabel("Number of Processes") + plt.ylabel("Efficiency") + plt.xticks(tuple(int_traces_procs), tuple(label_xtics)) + # plt.yscale('log') + max_y = max(list_data[1]) + if max_y < 1.1: + max_y = 1.1 + plt.ylim(0,max_y+0.1) + # plt.xlim(0, ) + plt.legend() + plt.savefig('efficiency-matplot.png', bbox_inches='tight') \ No newline at end of file diff --git a/tracemetadata.py b/tracemetadata.py new file mode 100644 index 0000000..de85bd4 --- /dev/null +++ b/tracemetadata.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python3 + +"""Functions to extract metadata information from each trace.""" + +from __future__ import print_function, division +import os +import time +import sys +import math +import re +import fnmatch +import mmap +import gzip +import multiprocessing +import threading +from multiprocessing import Process + + +def get_traces_from_args(cmdl_args): + """Filters the given list to extract traces, i.e. matching *.prv and sorts + the traces in ascending order based on the number of processes in the trace. + Excludes all files other than *.prv and ignores also simulated traces from + this script, i.e. *.sim.prv + Returns list of trace paths and dictionary with the number of processes. + """ + + def get_processes(prv_file): + return trace_processes[prv_file], trace_tasks[prv_file], trace_threads[prv_file] + + trace_list = [x for x in cmdl_args.trace_list if (fnmatch.fnmatch(x, '*.prv') or fnmatch.fnmatch(x, '*.prv.gz')) + if not fnmatch.fnmatch(x, '*.sim.prv')] + if not trace_list: + print('==Error== could not find any traces matching "', ' '.join(cmdl_args.trace_list)) + sys.exit(1) + + trace_processes = dict() + trace_tasks = dict() + trace_threads = dict() + trace_mode = dict() + trace_task_per_node = dict() + + trace_list_temp = [] + trace_list_removed = [] + for trace in trace_list: + if float(os.path.getsize(trace)/1024/1024) < float(cmdl_args.max_trace_size): + trace_list_temp.append(trace) + else: + trace_list_removed.append(trace) + + if len(trace_list_temp) < 1: + print('==Error== All traces exceed the maximum size (', cmdl_args.max_trace_size, 'MiB)') + for trace_upper in trace_list_removed: + print(trace_upper) + sys.exit(1) + + print("Running modelfactors.py for the following traces list:") + trace_list = trace_list_temp + for trace in trace_list: + print(trace) + + if len(trace_list_removed) > 0: + print("\nFollowing traces were excluded to be analyzed (size >", cmdl_args.max_trace_size, "MiB): ") + for trace in trace_list_removed: + print(trace) + + print('\nExtracting metadata from the traces list.') + # This part could be parallelized + # t1 = time.perf_counter() + for trace in trace_list: + trace_processes[trace], trace_tasks[trace], trace_threads[trace] = get_num_processes(trace) + trace_list = sorted(trace_list, key=get_processes) + + t1 = time.perf_counter() + jobs = [] + manager = multiprocessing.Manager() + trace_mode = manager.dict() + for trace in trace_list: + p_act = Process(target=get_trace_mode, args=(trace, cmdl_args, trace_mode)) + jobs.append(p_act) + p_act.start() + + for p in jobs: + p.join() + + t2 = time.perf_counter() + + print('Successfully Metadata Extraction in {0:.1f} seconds.\n'.format(t2 - t1)) + + for trace in trace_list: + trace_task_per_node[trace] = get_task_per_node(trace) + + print("Starting Analysis for the following sorted traces list:") + print_overview(trace_list, trace_processes, trace_tasks, trace_threads, trace_mode, trace_task_per_node) + return trace_list, trace_processes, trace_tasks, trace_threads, trace_task_per_node, trace_mode + + +def get_num_processes(prv_file): + """Gets the number of processes in a trace from the according .row file. + Please note: return value needs to be integer because this function is also + used as sorting key. + """ + if prv_file[-4:] == ".prv": + tracefile = open(prv_file) + for line in tracefile: + header_trace = line.split('_') + break + tracefile.close() + + if prv_file[-7:] == ".prv.gz": + with gzip.open(prv_file, 'rt') as f: + for line in f: + if "#Paraver" in line: + header_trace = line.split('_') + break + f.close() + + + #print(header_trace) + header_to_print = header_trace[1].split(':')[3].split('(') + tasks = header_to_print[0] + threads = header_to_print[1] + list_procspernode = header_trace[1].split('(')[2].split(')')[0].split(',') + #print(list_procspernode) + total_procs = 0 + for proc_node in list_procspernode: + total_procs += int(proc_node.split(':')[0]) + + return int(total_procs), int(tasks), int(threads) + + +def get_tasks_threads(prv_file): + """Gets the tasks and threads from the .prv file. + """ + if prv_file[-4:] == ".prv": + tracefile = open(prv_file) + for line in tracefile: + header_trace = line.split('_') + break + tracefile.close() + + if prv_file[-7:] == ".prv.gz": + with gzip.open(prv_file, 'rt') as f: + for line in f: + if "#Paraver" in line: + header_trace = line.split('_') + break + f.close() + header_to_print = header_trace[1].split(':')[3].split('(') + tasks = header_to_print[0] + threads = header_to_print[1] + return int(tasks), int(threads) + + +def get_task_per_node(prv_file): + """Gets the number of processes and nodes in a trace from the according .row file. + """ + row_file = True + + if prv_file[-4:] == ".prv": + if os.path.exists(prv_file[:-4] + '.row'): + tracefile = open(prv_file[:-4] + '.row') + else: + tracefile = prv_file[:-4] + row_file = False + elif prv_file[-7:] == ".prv.gz": + if os.path.exists(prv_file[:-7] + '.row'): + tracefile = open(prv_file[:-7] + '.row') + else: + tracefile = prv_file[:-7] + row_file = False + + tasks = 0 + nodes = 1 + if row_file: + for line in tracefile: + if "LEVEL CPU SIZE" in line: + tasks = int(line[15:]) + if "LEVEL NODE SIZE" in line: + nodes = int(line[15:]) + tracefile.close() + task_nodes = math.ceil(int(tasks) / int(nodes)) + else: + if prv_file[-4:] == ".prv": + tracefile = open(prv_file) + for line in tracefile: + header_trace = line.split('_') + break + tracefile.close() + + if prv_file[-7:] == ".prv.gz": + with gzip.open(prv_file, 'rt') as f: + for line in f: + if "#Paraver" in line: + header_trace = line.split('_') + break + f.close() + task_nodes = int(header_trace[1].split(':')[1].split('(')[1].replace(')','').split(',')[0]) + + return int(task_nodes) + + +def get_trace_mode(prv_file, cmdl_args, trace_mode): + """Gets the trace mode by detecting the event 40000018:2 in .prv file + to detect the Burst mode trace in another case is Detailed mode. + 50000001 for MPI, 60000001 for OpenMP, 61000000 for pthreads, 63000001 for CUDA + """ + mode_trace = '' + burst = 0 + pcf_file = True + if prv_file[-4:] == ".prv": + file_pcf = prv_file[:-4] + '.pcf' + tracefile = open(prv_file) + for line in tracefile: + if "40000018:2" in line: + burst = 1 + break + tracefile.close() + if prv_file[-7:] == ".prv.gz": + file_pcf = prv_file[:-7] + '.pcf' + with gzip.open(prv_file, 'rt') as f: + for line in f: + if "40000018:2" in line: + burst = 1 + break + f.close() + + if burst == 1: + mode_trace = 'Burst' + else: + mode_trace = 'Detailed' + + if os.path.exists(file_pcf) and cmdl_args.trace_mode_detection == 'pcf': + with open(file_pcf, 'rb', 0) as file, \ + mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) as s: + if s.find(b' 500000') != -1: + mode_trace += '+MPI' + if s.find(b' 610000') != -1: + mode_trace += '+Pthreads' + elif s.find(b' 60000') != -1: + if not s.find(b' 60000019') == s.find(b' 60000'): + mode_trace += '+OpenMP' + if s.find(b' 630000') != -1 or s.find(b' 631000') != -1 or s.find(b' 632000') != -1: + mode_trace += '+CUDA' + if s.find(b' 9200001') != -1: + mode_trace += '+OmpSs' + if s.find(b' 642000') != -1 or s.find(b' 6400001') != -1 or s.find(b' 641000') != -1: + mode_trace += '+OpenCL' + else: + if s.find(b' 610000') != -1: + mode_trace += '+Pthreads' + elif s.find(b' 60000') != -1: + if not s.find(b' 60000019') == s.find(b' 60000'): + mode_trace += '+OpenMP' + if s.find(b' 630000') != -1 or s.find(b' 631000') != -1 or s.find(b' 632000') != -1: + mode_trace += '+CUDA' + if s.find(b' 9200001') != -1: + mode_trace += '+OmpSs' + if s.find(b' 642000') != -1 or s.find(b' 6400001') != -1 or s.find(b' 641000') != -1: + mode_trace += '+OpenCL' + file.close() + else: + count_mpi = 0 + count_omp = 0 + count_pthreads = 0 + count_cuda = 0 + count_ompss = 0 + count_opencl = 0 + if prv_file[-4:] == ".prv": + with open(prv_file, 'rb', 0) as file, \ + mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_COPY) as s: + # 2:cpu_id:appl_id:task_id:thread_id:time:event_type:event_value + mpi = re.compile(rb'\n2:\w+:\w+:[1-4]:1:\w+:50000\w\w\w:') + omp = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:60000018:') + cuda = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:63\w\w\w\w\w\w:') + pthreads = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:610000\w\w:') + ompss = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:9200001:') + opencl = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:64\w\w\w\w\w\w:') + if mpi.search(s): + count_mpi = 1 + mpi_trace = '+MPI' + if omp.search(s): + count_omp = 1 + omp_trace = '+OpenMP' + elif cuda.search(s): + count_cuda = 1 + cuda_trace = '+CUDA' + elif pthreads.search(s): + count_pthreads = 1 + pthreads_trace = '+Pthreads' + elif ompss.search(s): + count_ompss = 1 + ompss_trace = '+OmpSs' + elif opencl.search(s): + count_opencl = 1 + opencl_trace = '+OpenCL' + file.close() + elif prv_file[-7:] == ".prv.gz": + handle = open(prv_file, "rb") + mapped = mmap.mmap(handle.fileno(), 0, access=mmap.ACCESS_READ) + gzfile = gzip.GzipFile(mode="r", fileobj=mapped) + + # 2:cpu_id:appl_id:task_id:thread_id:time:event_type:event_value + mpi = re.compile(rb'\n2:\w+:\w+:[1-4]:1:\w+:50000\w\w\w:') + omp = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:60000018:') + cuda = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:63\w\w\w\w\w\w:') + pthreads = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:610000\w\w:') + ompss = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:9200001:') + opencl = re.compile(rb'\n2:\w+:\w+:[1-3]:[1-3]:\w+:64\w\w\w\w\w\w:') + s = gzfile.read() + if mpi.search(s): + count_mpi = 1 + mpi_trace = '+MPI' + if omp.search(s): + count_omp = 1 + omp_trace = '+OpenMP' + elif cuda.search(s): + count_cuda = 1 + cuda_trace = '+CUDA' + elif pthreads.search(s): + count_pthreads = 1 + pthreads_trace = '+Pthreads' + elif ompss.search(s): + count_ompss = 1 + ompss_trace = '+OmpSs' + elif opencl.search(s): + count_opencl = 1 + opencl_trace = '+OpenCL' + + handle.close() + if count_mpi > 0: + mode_trace += mpi_trace + if count_omp > 0: + mode_trace += omp_trace + if count_pthreads > 0: + mode_trace += pthreads_trace + if count_ompss > 0: + mode_trace += ompss_trace + if count_cuda > 0: + mode_trace += cuda_trace + if count_opencl > 0: + mode_trace += opencl_trace + + trace_mode[prv_file] = mode_trace + #return mode_trace + + +def human_readable(size, precision=1): + """Converts a given size in bytes to the value in human readable form.""" + suffixes = ['B', 'KB', 'MB', 'GB', 'TB'] + suffixIndex = 0 + while size > 1024 and suffixIndex < 4: + suffixIndex += 1 + size = size / 1024.0 + return "%.*f%s" % (precision, size, suffixes[suffixIndex]) + + +def print_overview(trace_list, trace_processes, trace_tasks, trace_threads, trace_mode, trace_task_per_node): + """Prints an overview of the traces that will be processed.""" + #print('Running', os.path.basename(__file__), 'for the following traces:') + + file_path = os.path.join(os.getcwd(), 'traces_metadata.txt') + with open(file_path, 'w') as output: + for index, trace in enumerate(trace_list): + line = '[' + str(index+1) + '] ' + trace + + line += ', ' + str(trace_processes[trace]) \ + + '(' + str(trace_tasks[trace]) + 'x' + str(trace_threads[trace]) + ')' + ' processes' + line += ', ' + str(trace_task_per_node[trace]) + ' tasks per node' + line += ', ' + human_readable(os.path.getsize(trace)) + line += ', ' + str(trace_mode[trace]) + ' mode' + print(line) + output.write(line + '\n') + + print('======== Output Files: Traces metadata ========') + print('Traces metadata written to ' + file_path) + print('') \ No newline at end of file diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..e457c32 --- /dev/null +++ b/utils.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 + +"""modelfactors.py utils.""" + +from __future__ import print_function, division +import os +import sys +import subprocess +import tempfile +import argparse +import shutil + + +try: + import scipy.optimize +except ImportError: + print('==Error== Could not import SciPy. Please make sure you have installed SciPy.') + +try: + import numpy +except ImportError: + print('==Error== Could not import NumPy. Please make sure you have installed NumPy.') + +try: + import pandas as pd +except ImportError: + print('==Error== Could not import pandas. Please make sure you have installed pandas.') +try: + import seaborn as sns +except ImportError: + print('==Error== Could not import seaborn. Please make sure you have installed seaborn.') + +try: + import matplotlib.pyplot as plt +except ImportError: + print('==Error== Could not import matplotlib. Please make sure you have installed matplotlib.') + + +__author__ = "Sandra Mendez" +__copyright__ = "Copyright 2019, Barcelona Supercomputing Center (BSC)" +__version_major__ = 0 +__version_minor__ = 3 +__version_micro__ = 7 +__version__ = str(__version_major__) + "." + str(__version_minor__) + "." + str(__version_micro__) + + +def parse_arguments(): + """Parses the command line arguments. + Currently the script only accepts one parameter list, which is the list of + traces that are processed. This can be a regex and only valid trace files + are kept at the end. + """ + parser = argparse.ArgumentParser(description='Generates performance metrics from a set of Paraver traces.') + parser.add_argument('trace_list', nargs='*', + help='list of traces to process. Accepts wild cards and automatically filters for ' + 'valid traces'), + parser.add_argument("-m", "--metrics", choices=['simple', 'hybrid'], default='hybrid', + help='select the kind of efficiency metrics (single parallelism or hybrid, default: hybrid)') + parser.add_argument("-v", "--version", action='version', version='%(prog)s {version}'.format(version=__version__)) + parser.add_argument("-d", "--debug", help="increase output verbosity to debug level", action="store_true") + parser.add_argument("-s", "--scaling", + help="define whether the measurements are weak or strong scaling (default: auto)", + choices=['weak', 'strong', 'auto'], default='auto') + parser.add_argument('--limit', help='limit number of cores for the plots ' + '(default: max processes of the trace list )') + parser.add_argument("-ms", "--max_trace_size", help='set the maximum trace size in MiB allowed.' + ' (default: 1024 MiB )', default=1024.0) + parser.add_argument("-tmd", "--trace_mode_detection", choices=['pcf', 'prv'], default='pcf', + help='select .prv or .pcf file for trace mode detection. ' + 'For customized traces, i.e. cut, filtered and so on, you must select' + ' the .prv file. (default: pcf)') + + if len(sys.argv) == 1: + parser.print_help() + sys.exit(1) + + cmdl_args = parser.parse_args() + + if cmdl_args.debug: + print('==DEBUG== Running in debug mode.') + + return cmdl_args + + +def which(cmd): + """Returns path to cmd in path or None if not available.""" + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + cmd_path = os.path.join(path, cmd) + if os.path.isfile(cmd_path) and os.access(cmd_path, os.X_OK): + return cmd_path + + return None + + +def check_installation(cmdl_args): + """Check if Dimemas and paramedir are in the path.""" + + if not which('Dimemas'): + print('Could not find Dimemas. Please make sure Dimemas is correctly installed and in the path.') + sys.exit(1) + if not which('paramedir'): + print('Could not find paramedir. Please make sure Paraver is correctly installed and in the path.') + sys.exit(1) + + if not which('python3'): + print('==> WARNING!!! It requires python version 3 or higher for a full functionality.') + option_user = input("Do you want to proceed with the analysis? (Yes/No)[Yes]: ").upper() + if option_user == 'NO': + sys.exit(1) + + if cmdl_args.debug: + print('==DEBUG== Using', __file__, __version__) + print('==DEBUG== Using', sys.executable, ".".join(map(str, sys.version_info[:3]))) + + try: + print('==DEBUG== Using', 'SciPy', scipy.__version__) + except NameError: + print('==DEBUG== SciPy not installed.') + + try: + print('==DEBUG== Using', 'NumPy', numpy.__version__) + except NameError: + print('==DEBUG== NumPy not installed.') + + print('==DEBUG== Using', which('Dimemas')) + print('==DEBUG== Using', which('paramedir')) + print('') + + return + + +def run_command(cmd, cmdl_args): + """Runs a command and forwards the return value.""" + if cmdl_args.debug: + print('==DEBUG== Executing:', ' '.join(cmd)) + + # In debug mode, keep the output. Otherwise, redirect it to devnull. + if cmdl_args.debug: + out = tempfile.NamedTemporaryFile(suffix='.out', prefix=cmd[0] + '_', dir='./', delete=False) + err = tempfile.NamedTemporaryFile(suffix='.err', prefix=cmd[0] + '_', dir='./', delete=False) + else: + out = open(os.devnull, 'w') + err = open(os.devnull, 'w') + + return_value = subprocess.call(cmd, stdout=out, stderr=err) + + out.close + err.close + + if return_value == 0: + if cmdl_args.debug: + os.remove(out.name) + os.remove(err.name) + else: + print('==ERROR== ' + ' '.join(cmd) + ' failed with return value ' + str(return_value) + '!') + print('See ' + out.name + ' and ' + err.name + ' for more details.') + + return return_value + + +def create_temp_folder(folder_name, cmdl_args): + path_output_aux = os.getcwd() + '/' + folder_name + + if os.path.exists(path_output_aux): + shutil.rmtree(path_output_aux) + os.makedirs(path_output_aux) + return (path_output_aux) + + +def move_files(path_source, path_dest, cmdl_args): + """Wraps os.remove with a try clause.""" + try: + shutil.move(path_source, path_dest) + except: + if cmdl_args.debug: + print('==DEBUG== Failed to move ' + path_source + '!') + + +def remove_files(path,cmdl_args): + """Wraps os.remove with a try clause.""" + try: + os.remove(path) + except: + if cmdl_args.debug: + print('==DEBUG== Failed to remove ' + path + '!') \ No newline at end of file -- GitLab From 72091764228f20f0b07db323aa7fd53f8d695b43 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Thu, 8 Sep 2022 15:06:08 +0200 Subject: [PATCH 5/8] Added information --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1a3e106..2384fc5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # Nemo modelfactors -Script used to get important metrics from NEMO +Script used to get important metrics from NEMO. + +The script obtains data from a single nemo timestep, so keep in mind that the results of this basic analysis +will only be accurate on longer simulations where the inicialitzation and finalitzation are close to 0% of the time. # Installation and requeriments -- GitLab From 50869e42585d351c6cf4f949bad3ea3679c94116 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Thu, 8 Sep 2022 16:01:45 +0200 Subject: [PATCH 6/8] Fixed EXTRAE 4.0.1 error --- README.md | 2 +- extraf.sh | 2 +- perf_metrics.config | 2 +- trace.sh | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2384fc5..beb7052 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ will only be accurate on longer simulations where the inicialitzation and finali This script requires the following BSCTOOLS to be installed, loaded and available through the PATH environment variable. -* *Extrae (3.7.1)* +* *Extrae (4.0.0 or above)* * *Paraver* * *Dimemas (latest has a bug use 5.4.2-devel instead)* * *Basicanalysis* diff --git a/extraf.sh b/extraf.sh index 6508c17..d77f02a 100755 --- a/extraf.sh +++ b/extraf.sh @@ -7,7 +7,7 @@ # List all the objects from the binary, select only the ones from the functions list and create a function file as Extrae demands -nm $1 | grep -i " T " | grep -w -f $2 | awk '{print $1" # "$3}' > extrae_functions_for_xml.txt +nm $1 | grep -i " T " | grep -w -f $2 | awk '{print $3" # "$1}' > extrae_functions_for_xml.txt echo "See the function names in the file extrae_functions_for_xml.txt" diff --git a/perf_metrics.config b/perf_metrics.config index 768de74..ebe66fd 100644 --- a/perf_metrics.config +++ b/perf_metrics.config @@ -44,4 +44,4 @@ Compilation_sub="OCE del_key 'key_si3 key_top'" # - Dimemas 4.2 -devel # - Python3 -Modules="EXTRAE/3.7.1 BASICANALYSIS gcc intel/2018.3 impi/2018.4 netcdf/4.4.1.1 hdf5/1.8.19 DIMEMAS/5.4.2-devel perl" +Modules="EXTRAE BASICANALYSIS gcc intel/2018.3 impi/2018.4 netcdf/4.4.1.1 hdf5/1.8.19 DIMEMAS/5.4.2-devel perl" diff --git a/trace.sh b/trace.sh index fa295ae..7c38951 100755 --- a/trace.sh +++ b/trace.sh @@ -4,6 +4,7 @@ export EXTRAE_CONFIG_FILE=./extrae.xml # Load the tracing library (choose C/Fortran) #export LD_PRELOAD=${EXTRAE_HOME}/lib/libmpitrace.so +export EXTRAE_SKIP_AUTO_LIBRARY_INITIALIZE=1 export LD_PRELOAD=${EXTRAE_HOME}/lib/libmpitracecf.so # Run the program $* -- GitLab From 73ee4454fd1b2ac28256a8f7d1df168616820a01 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Fri, 9 Sep 2022 08:43:34 +0200 Subject: [PATCH 7/8] Minor chage --- perf_metrics.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 perf_metrics.bash diff --git a/perf_metrics.bash b/perf_metrics.bash old mode 100644 new mode 100755 index 215e7fd..4ae1856 --- a/perf_metrics.bash +++ b/perf_metrics.bash @@ -440,7 +440,7 @@ Compile_extrae echo magiccut/./magicCut nemo_"${core}".prv "$Nemo_iterations" > cut_"$core".out 2>&1 if ! ls nemo_"$core".best_cut.prv; then - echo "Cut failed, aborting." + echo "Cut failed, look at cut_$core.out for more info." echo exit 1 fi -- GitLab From 1c45c7032038b10f642f053fa3deef2bd9c1b08a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Pe=C3=B1a=20de=20Pedro?= Date: Fri, 9 Sep 2022 08:59:17 +0200 Subject: [PATCH 8/8] Shellcheck changes --- perf_metrics.bash | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/perf_metrics.bash b/perf_metrics.bash index 4ae1856..bd7685c 100755 --- a/perf_metrics.bash +++ b/perf_metrics.bash @@ -6,26 +6,29 @@ Job_completed() { if [ "$Jobs_scheduler" == "slurm" ]; then - local id1=${1##* } + local id1 + id1=${1##* } sleep 5 - if ! scontrol show job $id1 | grep -q 'JobState=COMPLETED'; then + if ! scontrol show job "$id1" | grep -q 'JobState=COMPLETED'; then Completed=false else Completed=true fi elif [ "$Jobs_scheduler" == "lsf" ]; then - local id1=$(head -n1 1 | cut -d'<' -f2 | cut -d'>' -f1) + local id2 + id2=$(head -n1 "$1" | cut -d'<' -f2 | cut -d'>' -f1) sleep 5 - if ! bjobs -l $id | grep -q 'Status '; then + if ! bjobs -l "$id2" | grep -q 'Status '; then Completed=false else Completed=true fi elif [ "$Jobs_scheduler" == "torque" ]; then - local id1=$(head -n1 $1 | awk '{ print $3 }') + local id3 + id3=$(head -n1 "$1" | awk '{ print $3 }') sleep 5 - if ! qstat f $id | grep -q 'exit_status = 0'; then + if ! qstat f "$id3" | grep -q 'exit_status = 0'; then Completed=false else Completed=true @@ -106,7 +109,7 @@ Compile_extrae() fi - if [ $compile == true ] || [ $compile_ext == true ]; then + if [ "$compile" == true ] || [ "$compile_ext" == true ]; then echo "Compiling Nemo for EXTRAE" echo "Output of the compilation in compile.err and compile.out" @@ -191,7 +194,7 @@ Compile_gprof() compile_gprof=true fi - if [ $compile == true ] || [ $compile_gprof == true ]; then + if [ "$compile" == true ] || [ "$compile_gprof" == true ]; then echo "Compiling Nemo for GPROF" echo "Output of the compilation in compile.err and compile.out" @@ -201,7 +204,7 @@ Compile_gprof() state1=$("$job" --wait compile."$Jobs_scheduler") echo Job_completed "$state1" - if [ $Completed == false ]; then + if [ "$Completed" == false ]; then echo "Nemo compilation failed, remember to load all the needed modules. Check the details in compile.err" echo exit 1 -- GitLab