From 2d07a51ca3de991e112e8951406f46bcb5e2925a Mon Sep 17 00:00:00 2001 From: Miguel Andres-Martinez Date: Fri, 9 Dec 2022 16:18:47 +0100 Subject: [PATCH] support SCRATCH_PROJECT_DIR for slurm platforms --- autosubmit/job/job.py | 1 + autosubmit/platforms/ecplatform.py | 8 ++++---- autosubmit/platforms/headers/slurm_header.py | 8 ++++---- autosubmit/platforms/lsfplatform.py | 4 ++-- autosubmit/platforms/paramiko_platform.py | 2 +- autosubmit/platforms/paramiko_submitter.py | 1 + autosubmit/platforms/pbsplatform.py | 4 ++-- autosubmit/platforms/platform.py | 3 ++- autosubmit/platforms/psplatform.py | 8 ++++---- autosubmit/platforms/sgeplatform.py | 4 ++-- autosubmit/platforms/slurmplatform.py | 2 +- 11 files changed, 24 insertions(+), 21 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 9cde353c4..df62a17b9 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1145,6 +1145,7 @@ class Job(object): parameters['CURRENT_HYPERTHREADING'] = job_platform.hyperthreading parameters['CURRENT_TYPE'] = job_platform.type parameters['CURRENT_SCRATCH_DIR'] = job_platform.scratch + parameters['CURRENT_PROJ_DIR'] = job_platform.project_dir parameters['CURRENT_ROOTDIR'] = job_platform.root_dir parameters['CURRENT_LOGDIR'] = job_platform.get_files_path() parameters['ROOTDIR'] = os.path.join( diff --git a/autosubmit/platforms/ecplatform.py b/autosubmit/platforms/ecplatform.py index 710973342..0533e1755 100644 --- a/autosubmit/platforms/ecplatform.py +++ b/autosubmit/platforms/ecplatform.py @@ -88,7 +88,7 @@ class EcPlatform(ParamikoPlatform): """ Updates commands for platforms """ - self.root_dir = os.path.join(self.scratch, self.project, self.user, self.expid) + self.root_dir = os.path.join(self.scratch, self.project_dir, self.user, self.expid) self.remote_log_dir = os.path.join(self.root_dir, "LOG_" + self.expid) self.cancel_cmd = "eceaccess-job-delete" self._checkjob_cmd = "ecaccess-job-list " @@ -99,11 +99,11 @@ class EcPlatform(ParamikoPlatform): self.put_cmd = "ecaccess-file-put" self.get_cmd = "ecaccess-file-get" self.del_cmd = "ecaccess-file-delete" - self.mkdir_cmd = ("ecaccess-file-mkdir " + self.host + ":" + self.scratch + "/" + self.project + "/" + + self.mkdir_cmd = ("ecaccess-file-mkdir " + self.host + ":" + self.scratch + "/" + self.project_dir + "/" + self.user + "/" + self.expid + "; " + "ecaccess-file-mkdir " + self.host + ":" + self.remote_log_dir) - self.check_remote_permissions_cmd = "ecaccess-file-mkdir " + os.path.join(self.scratch,self.project,self.user,"_permission_checker_azxbyc") - self.check_remote_permissions_remove_cmd = "ecaccess-file-rmdir " + os.path.join(self.scratch,self.project,self.user,"_permission_checker_azxbyc") + self.check_remote_permissions_cmd = "ecaccess-file-mkdir " + os.path.join(self.scratch,self.project_dir,self.user,"_permission_checker_azxbyc") + self.check_remote_permissions_remove_cmd = "ecaccess-file-rmdir " + os.path.join(self.scratch,self.project_dir,self.user,"_permission_checker_azxbyc") def get_checkhost_cmd(self): return self._checkhost_cmd diff --git a/autosubmit/platforms/headers/slurm_header.py b/autosubmit/platforms/headers/slurm_header.py index 45fe9e58d..4a8830519 100644 --- a/autosubmit/platforms/headers/slurm_header.py +++ b/autosubmit/platforms/headers/slurm_header.py @@ -149,8 +149,8 @@ class SlurmHeader(object): #SBATCH -n %NUMPROC% #SBATCH -t %WALLCLOCK%:00 #SBATCH -J %JOBNAME% -#SBATCH --output=%CURRENT_SCRATCH_DIR%/%CURRENT_PROJ%/%CURRENT_USER%/%DEFAULT.EXPID%/LOG_%DEFAULT.EXPID%/%OUT_LOG_DIRECTIVE% -#SBATCH --error=%CURRENT_SCRATCH_DIR%/%CURRENT_PROJ%/%CURRENT_USER%/%DEFAULT.EXPID%/LOG_%DEFAULT.EXPID%/%ERR_LOG_DIRECTIVE% +#SBATCH --output=%CURRENT_SCRATCH_DIR%/%CURRENT_PROJ_DIR%/%CURRENT_USER%/%DEFAULT.EXPID%/LOG_%DEFAULT.EXPID%/%OUT_LOG_DIRECTIVE% +#SBATCH --error=%CURRENT_SCRATCH_DIR%/%CURRENT_PROJ_DIR%/%CURRENT_USER%/%DEFAULT.EXPID%/LOG_%DEFAULT.EXPID%/%ERR_LOG_DIRECTIVE% %CUSTOM_DIRECTIVES% #%X11% # @@ -172,8 +172,8 @@ class SlurmHeader(object): #%TASKS_PER_NODE_DIRECTIVE% #SBATCH -t %WALLCLOCK%:00 #SBATCH -J %JOBNAME% -#SBATCH --output=%CURRENT_SCRATCH_DIR%/%CURRENT_PROJ%/%CURRENT_USER%/%DEFAULT.EXPID%/LOG_%DEFAULT.EXPID%/%OUT_LOG_DIRECTIVE% -#SBATCH --error=%CURRENT_SCRATCH_DIR%/%CURRENT_PROJ%/%CURRENT_USER%/%DEFAULT.EXPID%/LOG_%DEFAULT.EXPID%/%ERR_LOG_DIRECTIVE% +#SBATCH --output=%CURRENT_SCRATCH_DIR%/%CURRENT_PROJ_DIR%/%CURRENT_USER%/%DEFAULT.EXPID%/LOG_%DEFAULT.EXPID%/%OUT_LOG_DIRECTIVE% +#SBATCH --error=%CURRENT_SCRATCH_DIR%/%CURRENT_PROJ_DIR%/%CURRENT_USER%/%DEFAULT.EXPID%/LOG_%DEFAULT.EXPID%/%ERR_LOG_DIRECTIVE% %CUSTOM_DIRECTIVES% #%X11% # diff --git a/autosubmit/platforms/lsfplatform.py b/autosubmit/platforms/lsfplatform.py index 0cc011157..634ff620c 100644 --- a/autosubmit/platforms/lsfplatform.py +++ b/autosubmit/platforms/lsfplatform.py @@ -69,7 +69,7 @@ class LsfPlatform(ParamikoPlatform): """ Updates commands for platforms """ - self.root_dir = os.path.join(self.scratch, self.project, self.user, self.expid) + self.root_dir = os.path.join(self.scratch, self.project_dir, self.user, self.expid) self.remote_log_dir = os.path.join(self.root_dir, "LOG_" + self.expid) self.cancel_cmd = "bkill" self._checkjob_cmd = "bjobs " @@ -162,4 +162,4 @@ class LsfPlatform(ParamikoPlatform): # :return: True # :rtype: bool # """ - # self.connected = True \ No newline at end of file + # self.connected = True diff --git a/autosubmit/platforms/paramiko_platform.py b/autosubmit/platforms/paramiko_platform.py index c7d3e78b6..df5414469 100644 --- a/autosubmit/platforms/paramiko_platform.py +++ b/autosubmit/platforms/paramiko_platform.py @@ -1225,7 +1225,7 @@ class ParamikoPlatform(Platform): def check_remote_permissions(self): try: - path = os.path.join(self.scratch, self.project, self.user, "permission_checker_azxbyc") + path = os.path.join(self.scratch, self.project_dir, self.user, "permission_checker_azxbyc") try: self._ftpChannel.mkdir(path) self._ftpChannel.rmdir(path) diff --git a/autosubmit/platforms/paramiko_submitter.py b/autosubmit/platforms/paramiko_submitter.py index da2745eed..14df07128 100644 --- a/autosubmit/platforms/paramiko_submitter.py +++ b/autosubmit/platforms/paramiko_submitter.py @@ -176,6 +176,7 @@ class ParamikoSubmitter(Submitter): remote_platform.exclusivity = platform_data[section].get('EXCLUSIVITY', "") remote_platform.user = platform_data[section].get('USER', "") remote_platform.scratch = platform_data[section].get('SCRATCH_DIR', "") + remote_platform.project_dir = platform_data[section].get('SCRATCH_PROJECT_DIR', remote_platform.project) remote_platform.temp_dir = platform_data[section].get('TEMP_DIR', "") remote_platform._default_queue = platform_data[section].get('QUEUE', "") remote_platform._serial_queue = platform_data[section].get('SERIAL_QUEUE', "") diff --git a/autosubmit/platforms/pbsplatform.py b/autosubmit/platforms/pbsplatform.py index 2bf7f86d7..132b8715c 100644 --- a/autosubmit/platforms/pbsplatform.py +++ b/autosubmit/platforms/pbsplatform.py @@ -82,7 +82,7 @@ class PBSPlatform(ParamikoPlatform): """ Updates commands for platforms """ - self.root_dir = os.path.join(self.scratch, self.project, self.user, self.expid) + self.root_dir = os.path.join(self.scratch, self.project_dir, self.user, self.expid) self.remote_log_dir = os.path.join(self.root_dir, "LOG_" + self.expid) self.cancel_cmd = "ssh " + self.host + " qdel" self._checkhost_cmd = "ssh " + self.host + " echo 1" @@ -152,4 +152,4 @@ class PBSPlatform(ParamikoPlatform): # :return: True # :rtype: bool # """ - # self.connected = True \ No newline at end of file + # self.connected = True diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index f8999d219..6b463947e 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -39,6 +39,7 @@ class Platform(object): self.exclusivity = '' self.type = '' self.scratch = '' + self.project_dir = '' self.temp_dir = '' self.root_dir = '' self.service = None @@ -606,4 +607,4 @@ class Platform(object): """ Sends a Submit file Script, execute it in the platform and retrieves the Jobs_ID of all jobs at once. """ - raise NotImplementedError \ No newline at end of file + raise NotImplementedError diff --git a/autosubmit/platforms/psplatform.py b/autosubmit/platforms/psplatform.py index 3bd5c5d72..38f153a76 100644 --- a/autosubmit/platforms/psplatform.py +++ b/autosubmit/platforms/psplatform.py @@ -65,15 +65,15 @@ class PsPlatform(ParamikoPlatform): """ Updates commands for platforms """ - self.root_dir = os.path.join(self.scratch, self.project, self.user, self.expid) + self.root_dir = os.path.join(self.scratch, self.project_dir, self.user, self.expid) self.remote_log_dir = os.path.join(self.root_dir, "LOG_" + self.expid) self.cancel_cmd = "kill -SIGINT" self._checkhost_cmd = "echo 1" self.put_cmd = "scp" self.get_cmd = "scp" self.mkdir_cmd = "mkdir -p " + self.remote_log_dir - self.remove_checker = "rm -rf " + os.path.join(self.scratch, self.project,self.user,"ps_permission_checker_azxbyc") - self.mkdir_checker = "mkdir -p " + os.path.join(self.scratch, self.project,self.user,"ps_permission_checker_azxbyc") + self.remove_checker = "rm -rf " + os.path.join(self.scratch, self.project_dir,self.user,"ps_permission_checker_azxbyc") + self.mkdir_checker = "mkdir -p " + os.path.join(self.scratch, self.project_dir,self.user,"ps_permission_checker_azxbyc") def get_checkhost_cmd(self): return self._checkhost_cmd @@ -121,4 +121,4 @@ class PsPlatform(ParamikoPlatform): self.send_command(self.remove_checker) return True except Exception as e: - return False \ No newline at end of file + return False diff --git a/autosubmit/platforms/sgeplatform.py b/autosubmit/platforms/sgeplatform.py index b1a70a583..58671cd98 100644 --- a/autosubmit/platforms/sgeplatform.py +++ b/autosubmit/platforms/sgeplatform.py @@ -69,7 +69,7 @@ class SgePlatform(ParamikoPlatform): """ Updates commands for platforms """ - self.root_dir = os.path.join(self.scratch, self.project, self.user, self.expid) + self.root_dir = os.path.join(self.scratch, self.project_dir, self.user, self.expid) self.remote_log_dir = os.path.join(self.root_dir, "LOG_" + self.expid) self.cancel_cmd = "qdel" self._checkhost_cmd = "echo 1" @@ -137,4 +137,4 @@ class SgePlatform(ParamikoPlatform): :return: True :rtype: bool """ - self.connected = True \ No newline at end of file + self.connected = True diff --git a/autosubmit/platforms/slurmplatform.py b/autosubmit/platforms/slurmplatform.py index 598bced4f..082f09dd6 100644 --- a/autosubmit/platforms/slurmplatform.py +++ b/autosubmit/platforms/slurmplatform.py @@ -280,7 +280,7 @@ class SlurmPlatform(ParamikoPlatform): Updates commands for platforms """ self.root_dir = os.path.join( - self.scratch, self.project, self.user, self.expid) + self.scratch, self.project_dir, self.user, self.expid) self.remote_log_dir = os.path.join(self.root_dir, "LOG_" + self.expid) self.cancel_cmd = "scancel" self._checkhost_cmd = "echo 1" -- GitLab