From 87c7e5975e3c1661a9959a5d7391ee4aa6fa03e3 Mon Sep 17 00:00:00 2001 From: cpenadep Date: Mon, 5 Sep 2022 15:44:54 +0200 Subject: [PATCH] Revert "Merge branch 'revert-6eb0eafd' into 'main'" This reverts merge request !5 --- Job_Creator.py | 40 +++++++++++++++++++++++++++++++++++++++- config.bash | 2 +- script.sh | 44 +++++++++++++++++++++++++------------------- 3 files changed, 65 insertions(+), 21 deletions(-) diff --git a/Job_Creator.py b/Job_Creator.py index 15330c8..dc70f90 100644 --- a/Job_Creator.py +++ b/Job_Creator.py @@ -117,6 +117,41 @@ def create_job_lsf(args): lsf_job.write(line) +def create_job_torque(args): + file_name = args.file_name + time = args.set_time + cores = args.set_core + name = args.job_name + cores_per_node = args.set_core_per_node + queue = args.set_queue + workload = args.set_workload + nodes = (cores//cores_per_node)+1 + hours = time // 60 + minutes = time % 60 + file = ["#!/bin/bash \n", + "############################################################################### \n", + "#PBS -o "+str(name)+".out \n#PBS -e "+str(name)+".err \n" + "#PBS --constraint=perfparanoid \n"] + + if cores is not None: + file.append("#PBS -l nodes" + str(nodes) + ":ppn=" + str(cores) + "\n") + if time is not None and not 0: + file.append("#PBS -l cput=" + str(hours) + ":"+str(minutes)+ ":00\n") + if name is not None: + file.append("#PBS -N " + name + "\n") + if queue is not None and not len(queue) == 0: + file.append("#PBS -q " + queue + "\n") + + if workload is not None: + file.append("\n") + for work in workload: + file.append(str(work) + "") + + with open(file_name, "w") as torque_job: + for line in file: + torque_job.write(line) + + def modify_job(args): file_name = args.file_name time = args.set_time @@ -148,6 +183,8 @@ if __name__ == "__main__": args.file_name = args.file_name if args.file_name.count(".slurm") else args.file_name + ".slurm" elif args.scheduler == "lsf": args.file_name = args.file_name if args.file_name.count(".lsf") else args.file_name + ".lsf" + elif args.scheduler == "torque": + args.file_name = args.file_name if args.file_name.count(".torque") else args.file_name + ".torque" if os.path.exists(str(args.file_name)): os.remove(str(args.file_name)) @@ -156,4 +193,5 @@ if __name__ == "__main__": create_job_slurm(args) elif args.scheduler == "lsf": create_job_lsf(args) - + elif args.scheduler == "torque": + create_job_torque(args) diff --git a/config.bash b/config.bash index 8d0f4c8..aa4ad29 100644 --- a/config.bash +++ b/config.bash @@ -19,7 +19,7 @@ Jobs_queue=debug # Compilation_compile: When false only compiles NEMO if arch file lacks the needed flags, when true always compiles NEMO. # Compilation_ref: Reference configuration # Compilation_arch: Architecture used (without the -arch sufix and the .fcm) -# Compilation_name: Name of the new configutation +# Compilation_name: Name of the new configutation (Important to not be an existing one) # Compilation_sub: Add or remove subcomponents diff --git a/script.sh b/script.sh index 4a97f5c..b28e5b8 100755 --- a/script.sh +++ b/script.sh @@ -39,15 +39,24 @@ Job_completed() Completed=true fi - else - local id1=${head -n1 1 | cut -d'<' -f2 | cut -d'>' -f1} + elif [ "$Jobs_scheduler" == "lsf" ]; then + local id1=$(head -n1 1 | cut -d'<' -f2 | cut -d'>' -f1) sleep 5 if ! bjobs -l $id | grep -q 'Status '; then Completed=false else Completed=true fi + elif [ "$Jobs_scheduler" == "torque" ]; then + local id1=$(head -n1 $1 | awk '{ print $3 }') + sleep 5 + if ! qstat f $id | grep -q 'exit_status = 0'; then + Completed=false + else + Completed=true + fi fi + } @@ -98,7 +107,7 @@ Test_arguments() exit 1 fi # scheduler correct? - if [ "$Jobs_scheduler" != "slurm" ] && [ "$Jobs_scheduler" != "lsf" ]; then + if [ "$Jobs_scheduler" != "slurm" ] && [ "$Jobs_scheduler" != "lsf" ] && [ "$Jobs_scheduler" != "torque" ]; then echo "$Jobs_scheduler is not a valid scheduler" echo exit 1 @@ -133,8 +142,10 @@ Test_arguments() # Creating auxiliar vars for submiting jobs if [ "$Jobs_scheduler" == "slurm" ]; then job="sbatch" - else + elif [ "$Jobs_scheduler" == "lsf" ]; then job="bsub" + elif [ "$Jobs_scheduler" == "torque" ]; then + job="qsub" fi @@ -241,30 +252,25 @@ Test_Comp() fi - #Rename the namelist_cfg if exists in order to not overwrite it - if test -f "namelist_cfg"; then - mv namelist_cfg namelist_cfg_old - cd "$dir" || echo "Error original dir doesn't exist" exit - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . - rm namelist_cfg - mv namelist_cfg_old namelist_cfg - else - cd "$dir" || echo "Error original dir doesn't exist" exit - cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . - fi + #Copy all the EXP00 data but don't overwrite namelist just the executable + cd "$dir" || echo "Error original dir doesn't exist" exit + cp -n "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/* . + cp "${Nemo_path}"/cfgs/"${name_cfg}"/EXP00/nemo . if [[ $comp_cfg == "-d OCE del_key 'key_si3 key_top'" ]]; then sed -i '/_def_nemo-ice.xml\|def_nemo-pisces.xml/d' context_nemo.xml #DELETE ICE AND PISCES CONTEXT (NOT USED) fi - #Solving NEMO 4.2 Errors - sed -i 's|ln_zdfiwm * = .true.|ln_zdfiwm = .false.|g' namelist_cfg #CHANGE DUE TO NON EXISTING FILES + #Solving NEMO input file common errors + if test -f "weights_core_orca2_bicubic_noc.nc"; then mv weights_core_orca2_bicubic_noc.nc weights_core2_orca2_bicub.nc #RENAME WRONG NAMED FILES fi + if test -f "weights_core_orca2_bilinear_noc.nc"; then mv weights_core_orca2_bilinear_noc.nc weights_core2_orca2_bilin.nc #RENAME WRONG NAMED FILES fi + } @@ -278,7 +284,6 @@ Create_metrics() #Generating function list in case of missing if ! test -f "extrae_functions_for_xml.txt"; then - rm gmon* 2> /dev/null echo "Runing Nemo with 2 cores to obtain function data..." echo @@ -287,7 +292,8 @@ Create_metrics() state2=$("$job" --wait run."$Jobs_scheduler") Job_completed "$state2" if [ $Completed == false ]; then - echo "Nemo execution failed no gprof files generated look at run.err for more info" + echo "Nemo execution failed look at run.err and ocean.output for more info" + echo "Remember that the namelist files are now the default, change theme in order to fit with the input files in the dir " echo exit 1 else -- GitLab