From 6495bb50ab5a27194ceadaa5cd9c7aea50530e3a Mon Sep 17 00:00:00 2001 From: Albert Puiggros Date: Wed, 2 Oct 2024 10:24:21 +0200 Subject: [PATCH 1/8] Started with the modifications of the rocrate generation --- a000/conf/autosubmit_a000.yml | 19 +++ a000/conf/expdef_a000.yml | 43 +++++++ a000/conf/jobs_a000.yml | 36 ++++++ a000/conf/metadata/experiment_data.yml | 157 +++++++++++++++++++++++++ a000/conf/platforms_a000.yml | 66 +++++++++++ a000/pkl/job_list_a000.pkl | Bin 0 -> 602 bytes a000/pkl/job_packages_a000.db | Bin 0 -> 16384 bytes a000/plot/a000_20241001_1720.pdf | 0 autosubmit.db | Bin 0 -> 16384 bytes autosubmit/autosubmit.py | 78 +++++++----- autosubmit/provenance/rocrate.py | 12 +- metadata/data/job_data_a000.db | Bin 0 -> 24576 bytes metadata/data/job_data_a000.sql | 0 metadata/structures/structure_a000.db | Bin 0 -> 12288 bytes 14 files changed, 381 insertions(+), 30 deletions(-) create mode 100644 a000/conf/autosubmit_a000.yml create mode 100644 a000/conf/expdef_a000.yml create mode 100644 a000/conf/jobs_a000.yml create mode 100644 a000/conf/metadata/experiment_data.yml create mode 100644 a000/conf/platforms_a000.yml create mode 100644 a000/pkl/job_list_a000.pkl create mode 100644 a000/pkl/job_packages_a000.db create mode 100644 a000/plot/a000_20241001_1720.pdf create mode 100644 autosubmit.db create mode 100755 metadata/data/job_data_a000.db create mode 100644 metadata/data/job_data_a000.sql create mode 100644 metadata/structures/structure_a000.db diff --git a/a000/conf/autosubmit_a000.yml b/a000/conf/autosubmit_a000.yml new file mode 100644 index 000000000..3baf17f14 --- /dev/null +++ b/a000/conf/autosubmit_a000.yml @@ -0,0 +1,19 @@ +CONFIG: + # Current version of Autosubmit. + AUTOSUBMIT_VERSION: "4.0.98" + # Maximum number of jobs permitted in the waiting status. + MAXWAITINGJOBS: 20 + # Total number of jobs in the workflow. + TOTALJOBS: 20 + SAFETYSLEEPTIME: 10 + RETRIALS: 0 +#wrappers: +# wrapper_sim: +# TYPE: "vertical" +# JOBS_IN_WRAPPER: "SIM" +MAIL: + NOTIFICATIONS: False + TO: +STORAGE: + TYPE: pkl + COPY_REMOTE_LOGS: true diff --git a/a000/conf/expdef_a000.yml b/a000/conf/expdef_a000.yml new file mode 100644 index 000000000..741759205 --- /dev/null +++ b/a000/conf/expdef_a000.yml @@ -0,0 +1,43 @@ +DEFAULT: + # Job experiment ID. + EXPID: "a000" + # Default HPC platform name. + HPCARCH: "local" +EXPERIMENT: + # List of start dates + DATELIST: '20000101' + # List of members. + MEMBERS: fc0 + # Unit of the chunk size. Can be hour, day, month, or year. + CHUNKSIZEUNIT: month + # Size of each chunk. + CHUNKSIZE: '4' + # Number of chunks of the experiment. + NUMCHUNKS: '2' + CHUNKINI: '' + # Calendar used for the experiment. Can be standard or noleap. + CALENDAR: standard +PROJECT: + # Type of the project. + PROJECT_TYPE: none + # Folder to hold the project sources. + PROJECT_DESTINATION: '' +GIT: + PROJECT_ORIGIN: '' + PROJECT_BRANCH: '' + PROJECT_COMMIT: '' + PROJECT_SUBMODULES: '' + FETCH_SINGLE_BRANCH: true +SVN: + PROJECT_URL: '' + PROJECT_REVISION: '' +LOCAL: + PROJECT_PATH: '' +PROJECT_FILES: + FILE_PROJECT_CONF: '' + FILE_JOBS_CONF: '' + JOB_SCRIPTS_TYPE: '' +RERUN: + RERUN: false + RERUN_JOBLIST: '' + diff --git a/a000/conf/jobs_a000.yml b/a000/conf/jobs_a000.yml new file mode 100644 index 000000000..974499931 --- /dev/null +++ b/a000/conf/jobs_a000.yml @@ -0,0 +1,36 @@ +JOBS: + LOCAL_SETUP: + FILE: LOCAL_SETUP.sh + PLATFORM: LOCAL + RUNNING: once + REMOTE_SETUP: + FILE: REMOTE_SETUP.sh + DEPENDENCIES: LOCAL_SETUP + WALLCLOCK: 00:05 + RUNNING: once + INI: + FILE: INI.sh + DEPENDENCIES: REMOTE_SETUP + RUNNING: member + WALLCLOCK: 00:05 + SIM: + FILE: SIM.sh + DEPENDENCIES: INI SIM-1 + RUNNING: chunk + WALLCLOCK: 00:05 + POST: + FILE: POST.sh + DEPENDENCIES: SIM + RUNNING: once + WALLCLOCK: 00:05 + CLEAN: + FILE: CLEAN.sh + DEPENDENCIES: POST + RUNNING: once + WALLCLOCK: 00:05 + TRANSFER: + FILE: TRANSFER.sh + PLATFORM: LOCAL + DEPENDENCIES: CLEAN + RUNNING: member + diff --git a/a000/conf/metadata/experiment_data.yml b/a000/conf/metadata/experiment_data.yml new file mode 100644 index 000000000..203b1f761 --- /dev/null +++ b/a000/conf/metadata/experiment_data.yml @@ -0,0 +1,157 @@ +CONFIG: + AUTOSUBMIT_VERSION: 4.0.98 + MAXWAITINGJOBS: 20 + RETRIALS: 0 + SAFETYSLEEPTIME: 10 + TOTALJOBS: 20 +DEFAULT: + EXPID: a000 + HPCARCH: LOCAL +EXPERIMENT: + CALENDAR: standard + CHUNKINI: '' + CHUNKSIZE: 4 + CHUNKSIZEUNIT: month + DATELIST: '20000101' + MEMBERS: fc0 + NUMCHUNKS: 2 +GIT: + FETCH_SINGLE_BRANCH: true + PROJECT_BRANCH: '' + PROJECT_COMMIT: '' + PROJECT_ORIGIN: '' + PROJECT_SUBMODULES: '' +JOBS: + CLEAN: + ADDITIONAL_FILES: [] + DEPENDENCIES: + POST: {} + FILE: CLEAN.sh + RUNNING: once + WALLCLOCK: 00:05 + INI: + ADDITIONAL_FILES: [] + DEPENDENCIES: + REMOTE_SETUP: {} + FILE: INI.sh + RUNNING: member + WALLCLOCK: 00:05 + LOCAL_SETUP: + ADDITIONAL_FILES: [] + DEPENDENCIES: {} + FILE: LOCAL_SETUP.sh + PLATFORM: LOCAL + RUNNING: once + POST: + ADDITIONAL_FILES: [] + DEPENDENCIES: + SIM: {} + FILE: POST.sh + RUNNING: once + WALLCLOCK: 00:05 + REMOTE_SETUP: + ADDITIONAL_FILES: [] + DEPENDENCIES: + LOCAL_SETUP: {} + FILE: REMOTE_SETUP.sh + RUNNING: once + WALLCLOCK: 00:05 + SIM: + ADDITIONAL_FILES: [] + DEPENDENCIES: + INI: {} + SIM-1: {} + FILE: SIM.sh + RUNNING: chunk + WALLCLOCK: 00:05 + TRANSFER: + ADDITIONAL_FILES: [] + DEPENDENCIES: + CLEAN: {} + FILE: TRANSFER.sh + PLATFORM: LOCAL + RUNNING: member +LOCAL: + PROJECT_PATH: '' +MAIL: + NOTIFICATIONS: false + TO: null +PLATFORMS: + BSCEARTH000: + ADD_PROJECT_TO_HOST: false + HOST: bscearth000 + PROJECT: Earth + QUEUE: serial + SCRATCH_DIR: /esarchive/scratch + TYPE: ps + USER: null + ECMWF-XC40: + ADD_PROJECT_TO_HOST: false + HOST: cca + MAX_WALLCLOCK: 48:00 + PROJECT: spesiccf + QUEUE: np + SCRATCH_DIR: /scratch/ms + SERIAL_QUEUE: ns + TYPE: ecaccess + USER: null + VERSION: pbs + MARENOSTRUM4: + ADD_PROJECT_TO_HOST: false + HOST: mn1.bsc.es + MAX_WALLCLOCK: 48:00 + PROJECT: bsc32 + QUEUE: debug + SCRATCH_DIR: /gpfs/scratch + TEMP_DIR: '' + TYPE: slurm + USER: null + MARENOSTRUM_ARCHIVE: + ADD_PROJECT_TO_HOST: false + HOST: dt02.bsc.es + PROJECT: bsc32 + SCRATCH_DIR: /gpfs/scratch + TEST_SUITE: false + TYPE: ps + USER: null + NORD3: + HOST: nord1.bsc.es + MAX_WALLCLOCK: 48:00 + PROJECT: bsc32 + QUEUE: debug + SCRATCH_DIR: /gpfs/scratch + TYPE: SLURM + USER: null + TRANSFER_NODE: + ADD_PROJECT_TO_HOST: false + HOST: dt01.bsc.es + PROJECT: bsc32 + SCRATCH_DIR: /gpfs/scratch + TYPE: ps + USER: null + TRANSFER_NODE_BSCEARTH000: + ADD_PROJECT_TO_HOST: false + HOST: bscearth000 + PROJECT: Earth + QUEUE: serial + SCRATCH_DIR: /esarchive/scratch + TYPE: ps + USER: null +PROJDIR: /home/albertpuiggros/autosubmit/a000/proj/ +PROJECT: + PROJECT_DESTINATION: '' + PROJECT_TYPE: none +PROJECT_FILES: + FILE_JOBS_CONF: '' + FILE_PROJECT_CONF: '' + JOB_SCRIPTS_TYPE: '' +RERUN: + RERUN: false + RERUN_JOBLIST: '' +ROOTDIR: /home/albertpuiggros/autosubmit/a000 +STORAGE: + COPY_REMOTE_LOGS: true + TYPE: pkl +SVN: + PROJECT_REVISION: '' + PROJECT_URL: '' diff --git a/a000/conf/platforms_a000.yml b/a000/conf/platforms_a000.yml new file mode 100644 index 000000000..5ae520b5f --- /dev/null +++ b/a000/conf/platforms_a000.yml @@ -0,0 +1,66 @@ +PLATFORMS: + MARENOSTRUM4: + TYPE: slurm + HOST: mn1.bsc.es + PROJECT: bsc32 + USER: + QUEUE: debug + SCRATCH_DIR: /gpfs/scratch + ADD_PROJECT_TO_HOST: false + MAX_WALLCLOCK: 48:00 + TEMP_DIR: '' + MARENOSTRUM_ARCHIVE: + TYPE: ps + HOST: dt02.bsc.es + PROJECT: bsc32 + USER: + SCRATCH_DIR: /gpfs/scratch + ADD_PROJECT_TO_HOST: false + TEST_SUITE: false + + TRANSFER_NODE: + TYPE: ps + HOST: dt01.bsc.es + PROJECT: bsc32 + USER: + ADD_PROJECT_TO_HOST: false + SCRATCH_DIR: /gpfs/scratch + + TRANSFER_NODE_BSCEARTH000: + TYPE: ps + HOST: bscearth000 + USER: + PROJECT: Earth + ADD_PROJECT_TO_HOST: false + QUEUE: serial + SCRATCH_DIR: /esarchive/scratch + + BSCEARTH000: + TYPE: ps + HOST: bscearth000 + USER: + PROJECT: Earth + ADD_PROJECT_TO_HOST: false + QUEUE: serial + SCRATCH_DIR: /esarchive/scratch + NORD3: + TYPE: SLURM + HOST: nord1.bsc.es + PROJECT: bsc32 + USER: + QUEUE: debug + SCRATCH_DIR: /gpfs/scratch + MAX_WALLCLOCK: 48:00 + + ECMWF-XC40: + TYPE: ecaccess + VERSION: pbs + HOST: cca + USER: + PROJECT: spesiccf + ADD_PROJECT_TO_HOST: false + SCRATCH_DIR: /scratch/ms + QUEUE: np + SERIAL_QUEUE: ns + MAX_WALLCLOCK: 48:00 + diff --git a/a000/pkl/job_list_a000.pkl b/a000/pkl/job_list_a000.pkl new file mode 100644 index 0000000000000000000000000000000000000000..306971d884f14c6cade353041f6805d629bd3d29 GIT binary patch literal 602 zcmZ{g%T9w(5Qc%?Y1Lk=)q1JAIx7QxgQmfRw51JTG8;o0ilK?pOymLF=-PMk#T*7{ zpotI;XZ|_=cV^~8c}7VHXiJi`)-=tVFQ=2a=NRtO3gwwJqYs*t%cp#!NU*31R9~ z^3DqZjqL8#(s6OfXUjZ!7A_BlcqKs19MicmvGAHL6{wJ$DmMq7k$_?5DRk)C6U%ut WY`kSg73y?sFWx1c7Dja+Jp2Of&52O} literal 0 HcmV?d00001 diff --git a/a000/pkl/job_packages_a000.db b/a000/pkl/job_packages_a000.db new file mode 100644 index 0000000000000000000000000000000000000000..4d586736781e54df2a34e37f46295b728a6b1d0a GIT binary patch literal 16384 zcmeI%&rZTH9Ki9Gp@bNn7lVnH5-yMs#qa{6;YLgpZXT8*)u6zZZK5~ht9bQ=^aXql zI~-voL@!<>-zHuE?$_C$gb+oW^ES^@!B(1Rwwb2tWV=5P$##AOL~2Kr0kUrPKV)x4Hw- z?{;2<>)9hGsD;ahrkWa=>TOk{AfJhmjCFUa=%#k38Kl=u(wo(4wxL8)xg2jV$Mq%y zx%KUMVlov@Pwd==w%)MQm~pJ3$jc|Q?>GE-G1U)kIi`3W3)dbB#}8`C)<{7<6N}pY zKXaspQK_j$n;x__sZFz9(dF|sO*iwg=;F21QjPMx8mvm@`@-wF_Slz3`QGkG8&009U<00Izz00bZa0SG_<0uVSvfiyp2r{`x@u~;;P=ac;Q zK(<4JQO_C_E(^uN_0?+sKV!++AubSeh5!U0009U<00Izz00bZa0SG`~uRxrK($U~2 ogw_6k&XV)Jy^(|f1Rwwb2tWV=5P$##AOHafK;Xay6s|;n0b1dZ=l}o! literal 0 HcmV?d00001 diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 6bc39f862..06b266957 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -626,6 +626,11 @@ class Autosubmit: subparser.add_argument('expid', help='experiment identifier') subparser.add_argument('-v', '--update_version', action='store_true', default=False, help='Update experiment version') + # Provenance + subparser = subparsers.add_parser( + 'provenance', description = 'Produce provenance for autosubmit') + subparser.add_argument('--rocrate', action='store_true', + help='Produce an RO-Crate file') # Archive subparser = subparsers.add_parser( 'archive', description='archives an experiment') @@ -636,8 +641,8 @@ class Autosubmit: help='Only does a container without compress') subparser.add_argument('-v', '--update_version', action='store_true', default=False, help='Update experiment version') - subparser.add_argument('--rocrate', action='store_true', default=False, - help='Produce an RO-Crate file') + #subparser.add_argument('--rocrate', action='store_true', default=False, + # help='Produce an RO-Crate file') # Unarchive subparser = subparsers.add_parser( 'unarchive', description='unarchives an experiment') @@ -648,8 +653,8 @@ class Autosubmit: help='Untar an uncompressed tar') subparser.add_argument('-v', '--update_version', action='store_true', default=False, help='Update experiment version') - subparser.add_argument('--rocrate', action='store_true', default=False, - help='Unarchive an RO-Crate file') + #subparser.add_argument('--rocrate', action='store_true', default=False, + # help='Unarchive an RO-Crate file') # update proj files subparser = subparsers.add_parser('upgrade', description='Updates autosubmit 3 proj files to autosubmit 4') subparser.add_argument('expid', help='experiment identifier') @@ -767,10 +772,12 @@ class Autosubmit: return Autosubmit.update_version(args.expid) elif args.command == 'upgrade': return Autosubmit.upgrade_scripts(args.expid,files=args.files) + elif args.command == 'provenance': + return Autosubmit.provenance(args.expid, rocrate=args.rocrate) #need to create the provenance function elif args.command == 'archive': - return Autosubmit.archive(args.expid, noclean=args.noclean, uncompress=args.uncompress, rocrate=args.rocrate) + return Autosubmit.archive(args.expid, noclean=args.noclean, uncompress=args.uncompress) #, rocrate=args.rocrate) elif args.command == 'unarchive': - return Autosubmit.unarchive(args.expid, uncompressed=args.uncompressed, rocrate=args.rocrate) + return Autosubmit.unarchive(args.expid, uncompressed=args.uncompressed) #, rocrate=args.rocrate) elif args.command == 'readme': if os.path.isfile(Autosubmit.readme_path): @@ -4219,9 +4226,30 @@ class Autosubmit: from autosubmit.provenance.rocrate import create_rocrate_archive return create_rocrate_archive(as_conf, rocrate_json, jobs, start_time, end_time, path) + + @staticmethod + def provenance(expid, rocrate = False): + """"" + :param expid: experiment identifier + :type expid: str + :param rocrate: flag to enable RO-Crate + :type rocrate: bool + """"" + exp_folder = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) + tmp_folder = os.path.join(exp_folder, BasicConfig.LOCAL_TMP_DIR) #Join experiment folder dir with tmp dir + + aslogs_folder = os.path.joinj(tmp_folder, BasicConfig.LOCAL_ASLOG_DIR) + + if rocrate: + Autosubmit.rocrate(expid, Path(aslogs_folder)) + Log.info('RO-Crate ZIP file created!') + else : + raise AutosubmitCritical( + "Can not create RO-Crate ZIP file", 7012) + @staticmethod - def archive(expid, noclean=True, uncompress=True, rocrate=False): + def archive(expid, noclean=True, uncompress=True): """ Archives an experiment: call clean (if experiment is of version 3 or later), compress folder to tar.gz and moves to year's folder @@ -4271,27 +4299,23 @@ class Autosubmit: raise AutosubmitCritical(f"Failed to create year-directory {str(year)} for experiment {expid}", 7012, str(e)) Log.info(f"Archiving in year {str(year)}") - if rocrate: - Autosubmit.rocrate(expid, Path(year_path)) - Log.info('RO-Crate ZIP file created!') - else: - # Creating tar file - Log.info("Creating tar file ... ") - try: - if not uncompress: - compress_type = "w:gz" - output_filepath = '{0}.tar.gz'.format(expid) - else: - compress_type = "w" - output_filepath = '{0}.tar'.format(expid) - with tarfile.open(os.path.join(year_path, output_filepath), compress_type) as tar: - tar.add(exp_folder, arcname='') - tar.close() - os.chmod(os.path.join(year_path, output_filepath), 0o775) - except Exception as e: - raise AutosubmitCritical("Can not write tar file", 7012, str(e)) + # Creating tar file + Log.info("Creating tar file ... ") + try: + if not uncompress: + compress_type = "w:gz" + output_filepath = '{0}.tar.gz'.format(expid) + else: + compress_type = "w" + output_filepath = '{0}.tar'.format(expid) + with tarfile.open(os.path.join(year_path, output_filepath), compress_type) as tar: + tar.add(exp_folder, arcname='') + tar.close() + os.chmod(os.path.join(year_path, output_filepath), 0o775) + except Exception as e: + raise AutosubmitCritical("Can not write tar file", 7012, str(e)) - Log.info("Tar file created!") + Log.info("Tar file created!") try: shutil.rmtree(exp_folder) diff --git a/autosubmit/provenance/rocrate.py b/autosubmit/provenance/rocrate.py index de77b3e5b..24e702a46 100644 --- a/autosubmit/provenance/rocrate.py +++ b/autosubmit/provenance/rocrate.py @@ -23,6 +23,7 @@ import datetime import json import mimetypes import os +import time import subprocess from pathlib import Path from textwrap import dedent @@ -545,7 +546,7 @@ def create_rocrate_archive( encoding_format=None, exampleOfWork={'@id': formal_parameter['@id']}) create_action.append_to('result', {'@id': file_entity['@id']}) - + # Merge with user provided values. # NOTE: It is important that this call happens after the JSON-LD has # been constructed by ro-crate-py, as methods like ``add`` will @@ -555,8 +556,13 @@ def create_rocrate_archive( patch = json.loads(rocrate_json['PATCH']) for jsonld_node in patch['@graph']: crate.add_or_update_jsonld(jsonld_node) - + # Write RO-Crate ZIP. - crate.write_zip(Path(path, f"{expid}.zip")) + #What date/time we want to use to define the zip file? I guess it should be the LAST modification time, right? + #Should I re-use the code in archive() using the full date instead of only the year? The problem is that the + #zip is defined in this function so all the (modified) code should be moved here. Still, I'm exploring other posibilites + #to query the las modified time within this function (Not very used to the code still...) + date = time.strftime("%Y%m%d", time.localtime(os.path.getmtime(path))) + crate.write_zip(Path(path, f"{expid}-{date}.zip")) Log.info(f'RO-Crate archive written to {experiment_path}') return crate diff --git a/metadata/data/job_data_a000.db b/metadata/data/job_data_a000.db new file mode 100755 index 0000000000000000000000000000000000000000..9cc1b8bb696d5aee03148680e462b830fca827c3 GIT binary patch literal 24576 zcmeI2PjBPa5x}MRPn-ZoyA2SZ7A4fn8d+O^>>wK-+S1gs%`8PKB(-Zd2nd>bvYC}g zElDMt8hh&@$6j;IN9b4Rp`WJTpoh*p@kx{{dgE*^Sqx#re1B%%%x~rm9|=2O8eztj zL7a^IOgX6hzEZ7L{;DXIN@W9nH{n-aYjCqJ8}PRZzkevSSKV$@_LHBsx5LWD)_*G7 z|K5K2!N=QA;Fc~TKm>>Y5g-CYfC#)TfzHO}&YvDu!!YXeS9;s~!aDX$wZp!NFTK8> z`J0-}RF^5b*=FZTVS-|rik$QaZz6xhcYpZf+WR{XA6EbQAoDMWTuzXG*K=*IdTg+n zxbo;jMOvlY{F6|-j;O4rsqs(qn6WfzLtb1mHj9UW%6k8`0QoZb`~< zX`AA*bI_UVm7_tz{fzfl502t~I0(x|)5+y1%u3fyGe0RG9fVPsUf(60P14m?CGmIJ z_hViv8wFz!T?Q!~{>~o`gJB%JSgrhJT(T;Ucyjf9sanVtzFK3-gDi}r)q^8Gx`f!R z4$bu>dQrOI&`-cF(X!_nk2+uWr;76f310pobV{%OU<=~GP5-J2n*F?klg*pQT6?u-QDUB zb)l2L8uKI^@hI~ka*M6^7Bnu7&C1teyywAva|wHkB6N=`BSRG6#_}c+6tD7)=cVD_ z%WF&!kH$leg>yND@q0>9==&tvS*^X|R&|N#+ zZ|r~mr1n^;b=33cs_yFMDV%W~7;814*R@>LD0VpN33D$TgR!owcNolT;yjzVwyqi| z*LW?o=?2{2)Zjp=pXi$E!h$$D@P{cEilGrrhC`g~xR$M+qOzNsd(p+4+W5r~&C;yy zg$E~E%VnNnouY~?nQ)ZK4-`U*op<#%iusL31Le-Tnrdrj*kxFnYSd5yW-&M~!|7NE zwc+G$=ngh&Es$$88%=?Nb&dfMb`JufO*`wEPaXYl3?AQNx*#mt> zb2MA;y3UNI&}=iiH#Jw@$;=dOe9@DsA?BlI;hnJ9bC}!fiZ-r`HWd2z(`!N3P~8*D z?#xE96IL~WANIv?GQ+Mm>q1H}Q_}_M8;qvfZo^)*Ox2kA{?*qwkZ*lX)d1C|vR zZA9B&qr&H^VQ66dQ#7g3IBI-aVrHpH)9jvx29-d&%z&biGYbkxrtdpwz-qHfZE$qN zJhuVbWj5sIqVF(kuer(&aRmGYW^^r4P3CrBN0}kH%_0f~1~M(~NkB80I!$Pqk)o$; zacSwVw7Ls*#W`VE*5=O3&fJFXs{L>)M`wO4TQ<*))meZ6l}k*wDts7WCIrClbqMDfZ^&1w?AldC zpK$uKrRSz_K|0pY8m)VI_I7@_@L=+sp6;Se!a0nJ%mdG~+HWxSeqp#9>Yc9KvdcSm zG^X0_86F@oi)FHm7l1-SaQJX_ogXt6)1r_#3@2e?+->75ky3cK_Qi>3*=(Nr34CVw zme0}&ZU1dqYg%^uV1cS*^x(+|j>t@n;-r7S=+D^w>`hp*&5$S4I?qq`&b7nE;Pb!_ z;QNV4fx@wQe7TSmLEzu65rt%ccj+_?g29{fVLAAD1*wn3dNYrsF{*-+1ZQq<-YX`R zIf-7{mgUL{bsdj*-5>Y5g-CYfCvx)B0vO)01+SpMBrT@@SwW3`@!OWec<~){L?>OM1Tko0U|&I zhyW2F0z`la5CI}U1c<;pOyEQGjo$y?;Rus!M1Tko0U|&IhyW2F0z`la5CI}U1S%DJ g|EDY<0z`la5CI}U1c(3;AOb{y2oM1x@a_}%KQ6j)?EnA( literal 0 HcmV?d00001 diff --git a/metadata/data/job_data_a000.sql b/metadata/data/job_data_a000.sql new file mode 100644 index 000000000..e69de29bb diff --git a/metadata/structures/structure_a000.db b/metadata/structures/structure_a000.db new file mode 100644 index 0000000000000000000000000000000000000000..f1366887705445729211c545b2b99ae565a8f95e GIT binary patch literal 12288 zcmeI0PjAyO7{;B(Yhlt7PK;Y!F`yM2O;=St#JDzyECie0m%lcw-TFB|3sDH!L(I5)rFRDd`LfB*=900@8p2!H?xfB*=900{gm zflsfs(q^lreQt{3;c*nr-bcxJ5}gQsDw64um?pWtLZ|OKq02(&h3B%oDpp-*OBV4_ z5>J?jW`g;>kog18+x{y<#H->1zk4untMf+N5f^doN`38Bp}6_BrIGQ;yJ&X$;aF;S zKAehJt$E%Vc_Yu8>iRn=&kZ?B#!q7YFu$1}P2a2 Date: Fri, 11 Oct 2024 09:19:05 +0200 Subject: [PATCH 2/8] Added provenance functionality to generate the rocrate without the need of archiving (Now testing) --- a000/conf/autosubmit_a000.yml | 19 --- a000/conf/expdef_a000.yml | 43 ------- a000/conf/jobs_a000.yml | 36 ------ a000/conf/metadata/experiment_data.yml | 157 ------------------------- a000/conf/platforms_a000.yml | 66 ----------- a000/pkl/job_list_a000.pkl | Bin 602 -> 0 bytes a000/pkl/job_packages_a000.db | Bin 16384 -> 0 bytes a000/plot/a000_20241001_1720.pdf | 0 autosubmit.db | Bin 16384 -> 16384 bytes autosubmit/autosubmit.py | 61 ++++++---- metadata/data/job_data_a000.db | Bin 24576 -> 0 bytes metadata/structures/structure_a000.db | Bin 12288 -> 0 bytes 12 files changed, 36 insertions(+), 346 deletions(-) delete mode 100644 a000/conf/autosubmit_a000.yml delete mode 100644 a000/conf/expdef_a000.yml delete mode 100644 a000/conf/jobs_a000.yml delete mode 100644 a000/conf/metadata/experiment_data.yml delete mode 100644 a000/conf/platforms_a000.yml delete mode 100644 a000/pkl/job_list_a000.pkl delete mode 100644 a000/pkl/job_packages_a000.db delete mode 100644 a000/plot/a000_20241001_1720.pdf delete mode 100755 metadata/data/job_data_a000.db delete mode 100644 metadata/structures/structure_a000.db diff --git a/a000/conf/autosubmit_a000.yml b/a000/conf/autosubmit_a000.yml deleted file mode 100644 index 3baf17f14..000000000 --- a/a000/conf/autosubmit_a000.yml +++ /dev/null @@ -1,19 +0,0 @@ -CONFIG: - # Current version of Autosubmit. - AUTOSUBMIT_VERSION: "4.0.98" - # Maximum number of jobs permitted in the waiting status. - MAXWAITINGJOBS: 20 - # Total number of jobs in the workflow. - TOTALJOBS: 20 - SAFETYSLEEPTIME: 10 - RETRIALS: 0 -#wrappers: -# wrapper_sim: -# TYPE: "vertical" -# JOBS_IN_WRAPPER: "SIM" -MAIL: - NOTIFICATIONS: False - TO: -STORAGE: - TYPE: pkl - COPY_REMOTE_LOGS: true diff --git a/a000/conf/expdef_a000.yml b/a000/conf/expdef_a000.yml deleted file mode 100644 index 741759205..000000000 --- a/a000/conf/expdef_a000.yml +++ /dev/null @@ -1,43 +0,0 @@ -DEFAULT: - # Job experiment ID. - EXPID: "a000" - # Default HPC platform name. - HPCARCH: "local" -EXPERIMENT: - # List of start dates - DATELIST: '20000101' - # List of members. - MEMBERS: fc0 - # Unit of the chunk size. Can be hour, day, month, or year. - CHUNKSIZEUNIT: month - # Size of each chunk. - CHUNKSIZE: '4' - # Number of chunks of the experiment. - NUMCHUNKS: '2' - CHUNKINI: '' - # Calendar used for the experiment. Can be standard or noleap. - CALENDAR: standard -PROJECT: - # Type of the project. - PROJECT_TYPE: none - # Folder to hold the project sources. - PROJECT_DESTINATION: '' -GIT: - PROJECT_ORIGIN: '' - PROJECT_BRANCH: '' - PROJECT_COMMIT: '' - PROJECT_SUBMODULES: '' - FETCH_SINGLE_BRANCH: true -SVN: - PROJECT_URL: '' - PROJECT_REVISION: '' -LOCAL: - PROJECT_PATH: '' -PROJECT_FILES: - FILE_PROJECT_CONF: '' - FILE_JOBS_CONF: '' - JOB_SCRIPTS_TYPE: '' -RERUN: - RERUN: false - RERUN_JOBLIST: '' - diff --git a/a000/conf/jobs_a000.yml b/a000/conf/jobs_a000.yml deleted file mode 100644 index 974499931..000000000 --- a/a000/conf/jobs_a000.yml +++ /dev/null @@ -1,36 +0,0 @@ -JOBS: - LOCAL_SETUP: - FILE: LOCAL_SETUP.sh - PLATFORM: LOCAL - RUNNING: once - REMOTE_SETUP: - FILE: REMOTE_SETUP.sh - DEPENDENCIES: LOCAL_SETUP - WALLCLOCK: 00:05 - RUNNING: once - INI: - FILE: INI.sh - DEPENDENCIES: REMOTE_SETUP - RUNNING: member - WALLCLOCK: 00:05 - SIM: - FILE: SIM.sh - DEPENDENCIES: INI SIM-1 - RUNNING: chunk - WALLCLOCK: 00:05 - POST: - FILE: POST.sh - DEPENDENCIES: SIM - RUNNING: once - WALLCLOCK: 00:05 - CLEAN: - FILE: CLEAN.sh - DEPENDENCIES: POST - RUNNING: once - WALLCLOCK: 00:05 - TRANSFER: - FILE: TRANSFER.sh - PLATFORM: LOCAL - DEPENDENCIES: CLEAN - RUNNING: member - diff --git a/a000/conf/metadata/experiment_data.yml b/a000/conf/metadata/experiment_data.yml deleted file mode 100644 index 203b1f761..000000000 --- a/a000/conf/metadata/experiment_data.yml +++ /dev/null @@ -1,157 +0,0 @@ -CONFIG: - AUTOSUBMIT_VERSION: 4.0.98 - MAXWAITINGJOBS: 20 - RETRIALS: 0 - SAFETYSLEEPTIME: 10 - TOTALJOBS: 20 -DEFAULT: - EXPID: a000 - HPCARCH: LOCAL -EXPERIMENT: - CALENDAR: standard - CHUNKINI: '' - CHUNKSIZE: 4 - CHUNKSIZEUNIT: month - DATELIST: '20000101' - MEMBERS: fc0 - NUMCHUNKS: 2 -GIT: - FETCH_SINGLE_BRANCH: true - PROJECT_BRANCH: '' - PROJECT_COMMIT: '' - PROJECT_ORIGIN: '' - PROJECT_SUBMODULES: '' -JOBS: - CLEAN: - ADDITIONAL_FILES: [] - DEPENDENCIES: - POST: {} - FILE: CLEAN.sh - RUNNING: once - WALLCLOCK: 00:05 - INI: - ADDITIONAL_FILES: [] - DEPENDENCIES: - REMOTE_SETUP: {} - FILE: INI.sh - RUNNING: member - WALLCLOCK: 00:05 - LOCAL_SETUP: - ADDITIONAL_FILES: [] - DEPENDENCIES: {} - FILE: LOCAL_SETUP.sh - PLATFORM: LOCAL - RUNNING: once - POST: - ADDITIONAL_FILES: [] - DEPENDENCIES: - SIM: {} - FILE: POST.sh - RUNNING: once - WALLCLOCK: 00:05 - REMOTE_SETUP: - ADDITIONAL_FILES: [] - DEPENDENCIES: - LOCAL_SETUP: {} - FILE: REMOTE_SETUP.sh - RUNNING: once - WALLCLOCK: 00:05 - SIM: - ADDITIONAL_FILES: [] - DEPENDENCIES: - INI: {} - SIM-1: {} - FILE: SIM.sh - RUNNING: chunk - WALLCLOCK: 00:05 - TRANSFER: - ADDITIONAL_FILES: [] - DEPENDENCIES: - CLEAN: {} - FILE: TRANSFER.sh - PLATFORM: LOCAL - RUNNING: member -LOCAL: - PROJECT_PATH: '' -MAIL: - NOTIFICATIONS: false - TO: null -PLATFORMS: - BSCEARTH000: - ADD_PROJECT_TO_HOST: false - HOST: bscearth000 - PROJECT: Earth - QUEUE: serial - SCRATCH_DIR: /esarchive/scratch - TYPE: ps - USER: null - ECMWF-XC40: - ADD_PROJECT_TO_HOST: false - HOST: cca - MAX_WALLCLOCK: 48:00 - PROJECT: spesiccf - QUEUE: np - SCRATCH_DIR: /scratch/ms - SERIAL_QUEUE: ns - TYPE: ecaccess - USER: null - VERSION: pbs - MARENOSTRUM4: - ADD_PROJECT_TO_HOST: false - HOST: mn1.bsc.es - MAX_WALLCLOCK: 48:00 - PROJECT: bsc32 - QUEUE: debug - SCRATCH_DIR: /gpfs/scratch - TEMP_DIR: '' - TYPE: slurm - USER: null - MARENOSTRUM_ARCHIVE: - ADD_PROJECT_TO_HOST: false - HOST: dt02.bsc.es - PROJECT: bsc32 - SCRATCH_DIR: /gpfs/scratch - TEST_SUITE: false - TYPE: ps - USER: null - NORD3: - HOST: nord1.bsc.es - MAX_WALLCLOCK: 48:00 - PROJECT: bsc32 - QUEUE: debug - SCRATCH_DIR: /gpfs/scratch - TYPE: SLURM - USER: null - TRANSFER_NODE: - ADD_PROJECT_TO_HOST: false - HOST: dt01.bsc.es - PROJECT: bsc32 - SCRATCH_DIR: /gpfs/scratch - TYPE: ps - USER: null - TRANSFER_NODE_BSCEARTH000: - ADD_PROJECT_TO_HOST: false - HOST: bscearth000 - PROJECT: Earth - QUEUE: serial - SCRATCH_DIR: /esarchive/scratch - TYPE: ps - USER: null -PROJDIR: /home/albertpuiggros/autosubmit/a000/proj/ -PROJECT: - PROJECT_DESTINATION: '' - PROJECT_TYPE: none -PROJECT_FILES: - FILE_JOBS_CONF: '' - FILE_PROJECT_CONF: '' - JOB_SCRIPTS_TYPE: '' -RERUN: - RERUN: false - RERUN_JOBLIST: '' -ROOTDIR: /home/albertpuiggros/autosubmit/a000 -STORAGE: - COPY_REMOTE_LOGS: true - TYPE: pkl -SVN: - PROJECT_REVISION: '' - PROJECT_URL: '' diff --git a/a000/conf/platforms_a000.yml b/a000/conf/platforms_a000.yml deleted file mode 100644 index 5ae520b5f..000000000 --- a/a000/conf/platforms_a000.yml +++ /dev/null @@ -1,66 +0,0 @@ -PLATFORMS: - MARENOSTRUM4: - TYPE: slurm - HOST: mn1.bsc.es - PROJECT: bsc32 - USER: - QUEUE: debug - SCRATCH_DIR: /gpfs/scratch - ADD_PROJECT_TO_HOST: false - MAX_WALLCLOCK: 48:00 - TEMP_DIR: '' - MARENOSTRUM_ARCHIVE: - TYPE: ps - HOST: dt02.bsc.es - PROJECT: bsc32 - USER: - SCRATCH_DIR: /gpfs/scratch - ADD_PROJECT_TO_HOST: false - TEST_SUITE: false - - TRANSFER_NODE: - TYPE: ps - HOST: dt01.bsc.es - PROJECT: bsc32 - USER: - ADD_PROJECT_TO_HOST: false - SCRATCH_DIR: /gpfs/scratch - - TRANSFER_NODE_BSCEARTH000: - TYPE: ps - HOST: bscearth000 - USER: - PROJECT: Earth - ADD_PROJECT_TO_HOST: false - QUEUE: serial - SCRATCH_DIR: /esarchive/scratch - - BSCEARTH000: - TYPE: ps - HOST: bscearth000 - USER: - PROJECT: Earth - ADD_PROJECT_TO_HOST: false - QUEUE: serial - SCRATCH_DIR: /esarchive/scratch - NORD3: - TYPE: SLURM - HOST: nord1.bsc.es - PROJECT: bsc32 - USER: - QUEUE: debug - SCRATCH_DIR: /gpfs/scratch - MAX_WALLCLOCK: 48:00 - - ECMWF-XC40: - TYPE: ecaccess - VERSION: pbs - HOST: cca - USER: - PROJECT: spesiccf - ADD_PROJECT_TO_HOST: false - SCRATCH_DIR: /scratch/ms - QUEUE: np - SERIAL_QUEUE: ns - MAX_WALLCLOCK: 48:00 - diff --git a/a000/pkl/job_list_a000.pkl b/a000/pkl/job_list_a000.pkl deleted file mode 100644 index 306971d884f14c6cade353041f6805d629bd3d29..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 602 zcmZ{g%T9w(5Qc%?Y1Lk=)q1JAIx7QxgQmfRw51JTG8;o0ilK?pOymLF=-PMk#T*7{ zpotI;XZ|_=cV^~8c}7VHXiJi`)-=tVFQ=2a=NRtO3gwwJqYs*t%cp#!NU*31R9~ z^3DqZjqL8#(s6OfXUjZ!7A_BlcqKs19MicmvGAHL6{wJ$DmMq7k$_?5DRk)C6U%ut WY`kSg73y?sFWx1c7Dja+Jp2Of&52O} diff --git a/a000/pkl/job_packages_a000.db b/a000/pkl/job_packages_a000.db deleted file mode 100644 index 4d586736781e54df2a34e37f46295b728a6b1d0a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI%&rZTH9Ki9Gp@bNn7lVnH5-yMs#qa{6;YLgpZXT8*)u6zZZK5~ht9bQ=^aXql zI~-voL@!<>-zHuE?$_C$gb+oW^ES^@!B(>wK-+S1gs%`8PKB(-Zd2nd>bvYC}g zElDMt8hh&@$6j;IN9b4Rp`WJTpoh*p@kx{{dgE*^Sqx#re1B%%%x~rm9|=2O8eztj zL7a^IOgX6hzEZ7L{;DXIN@W9nH{n-aYjCqJ8}PRZzkevSSKV$@_LHBsx5LWD)_*G7 z|K5K2!N=QA;Fc~TKm>>Y5g-CYfC#)TfzHO}&YvDu!!YXeS9;s~!aDX$wZp!NFTK8> z`J0-}RF^5b*=FZTVS-|rik$QaZz6xhcYpZf+WR{XA6EbQAoDMWTuzXG*K=*IdTg+n zxbo;jMOvlY{F6|-j;O4rsqs(qn6WfzLtb1mHj9UW%6k8`0QoZb`~< zX`AA*bI_UVm7_tz{fzfl502t~I0(x|)5+y1%u3fyGe0RG9fVPsUf(60P14m?CGmIJ z_hViv8wFz!T?Q!~{>~o`gJB%JSgrhJT(T;Ucyjf9sanVtzFK3-gDi}r)q^8Gx`f!R z4$bu>dQrOI&`-cF(X!_nk2+uWr;76f310pobV{%OU<=~GP5-J2n*F?klg*pQT6?u-QDUB zb)l2L8uKI^@hI~ka*M6^7Bnu7&C1teyywAva|wHkB6N=`BSRG6#_}c+6tD7)=cVD_ z%WF&!kH$leg>yND@q0>9==&tvS*^X|R&|N#+ zZ|r~mr1n^;b=33cs_yFMDV%W~7;814*R@>LD0VpN33D$TgR!owcNolT;yjzVwyqi| z*LW?o=?2{2)Zjp=pXi$E!h$$D@P{cEilGrrhC`g~xR$M+qOzNsd(p+4+W5r~&C;yy zg$E~E%VnNnouY~?nQ)ZK4-`U*op<#%iusL31Le-Tnrdrj*kxFnYSd5yW-&M~!|7NE zwc+G$=ngh&Es$$88%=?Nb&dfMb`JufO*`wEPaXYl3?AQNx*#mt> zb2MA;y3UNI&}=iiH#Jw@$;=dOe9@DsA?BlI;hnJ9bC}!fiZ-r`HWd2z(`!N3P~8*D z?#xE96IL~WANIv?GQ+Mm>q1H}Q_}_M8;qvfZo^)*Ox2kA{?*qwkZ*lX)d1C|vR zZA9B&qr&H^VQ66dQ#7g3IBI-aVrHpH)9jvx29-d&%z&biGYbkxrtdpwz-qHfZE$qN zJhuVbWj5sIqVF(kuer(&aRmGYW^^r4P3CrBN0}kH%_0f~1~M(~NkB80I!$Pqk)o$; zacSwVw7Ls*#W`VE*5=O3&fJFXs{L>)M`wO4TQ<*))meZ6l}k*wDts7WCIrClbqMDfZ^&1w?AldC zpK$uKrRSz_K|0pY8m)VI_I7@_@L=+sp6;Se!a0nJ%mdG~+HWxSeqp#9>Yc9KvdcSm zG^X0_86F@oi)FHm7l1-SaQJX_ogXt6)1r_#3@2e?+->75ky3cK_Qi>3*=(Nr34CVw zme0}&ZU1dqYg%^uV1cS*^x(+|j>t@n;-r7S=+D^w>`hp*&5$S4I?qq`&b7nE;Pb!_ z;QNV4fx@wQe7TSmLEzu65rt%ccj+_?g29{fVLAAD1*wn3dNYrsF{*-+1ZQq<-YX`R zIf-7{mgUL{bsdj*-5>Y5g-CYfCvx)B0vO)01+SpMBrT@@SwW3`@!OWec<~){L?>OM1Tko0U|&I zhyW2F0z`la5CI}U1c<;pOyEQGjo$y?;Rus!M1Tko0U|&IhyW2F0z`la5CI}U1S%DJ g|EDY<0z`la5CI}U1c(3;AOb{y2oM1x@a_}%KQ6j)?EnA( diff --git a/metadata/structures/structure_a000.db b/metadata/structures/structure_a000.db deleted file mode 100644 index f1366887705445729211c545b2b99ae565a8f95e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI0PjAyO7{;B(Yhlt7PK;Y!F`yM2O;=St#JDzyECie0m%lcw-TFB|3sDH!L(I5)rFRDd`LfB*=900@8p2!H?xfB*=900{gm zflsfs(q^lreQt{3;c*nr-bcxJ5}gQsDw64um?pWtLZ|OKq02(&h3B%oDpp-*OBV4_ z5>J?jW`g;>kog18+x{y<#H->1zk4untMf+N5f^doN`38Bp}6_BrIGQ;yJ&X$;aF;S zKAehJt$E%Vc_Yu8>iRn=&kZ?B#!q7YFu$1}P2a2 Date: Fri, 11 Oct 2024 12:17:57 +0200 Subject: [PATCH 3/8] Unit test draft for provenance function --- test/unit/test_provenance.py | 57 ++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 test/unit/test_provenance.py diff --git a/test/unit/test_provenance.py b/test/unit/test_provenance.py new file mode 100644 index 000000000..0c69a8f35 --- /dev/null +++ b/test/unit/test_provenance.py @@ -0,0 +1,57 @@ +import pytest +from pathlib import Path +from autosubmit.autosubmit import Autosubmit +from log.log import AutosubmitCritical +import os +from autosubmitconfigparser.config.basicconfig import BasicConfig + +@pytest.fixture +def test_provenance_rocrate_success(mocker, tmp_path): + """ + Test the provenance function when rocrate=True and the process is successful. + """ + mock_rocrate = mocker.patch('autosubmit.autosubmit.Autosubmit.rocrate') + mock_log_info = mocker.patch('Log.info') + mock_basic_config = mocker.patch('BasicConfig') + + expid = "expid123" + + path = tmp_path / "expid123/tmp/ASLOGS" + + exp_folder = os.path.join(tmp_path, expid) + + tmp_folder = os.path.join(exp_folder, mock_basic_config.LOCAL_TMP_DIR) + + aslogs_folder = os.path.join(tmp_folder, mock_basic_config.LOCAL_ASLOG_DIR) + + expected_aslogs_path = os.path.join(tmp_folder, aslogs_folder) + + Autosubmit.provenance("expid123", rocrate=True) + + mock_rocrate.assert_called_once_with("expid123", Path(expected_aslogs_path)) + mock_log_info.assert_called_once_with('RO-Crate ZIP file created!') + + assert expected_aslogs_path == path + +def test_provenance_rocrate_failure(mocker): + + mock_rocrate = mocker.patch('autosubmit.autosubmit.Autosubmit.rocrate', side_effect=Exception("Mocked exception")) + + with pytest.raises(Exception) as excinfo: + Autosubmit.provenance("expid123", rocrate=True) + + assert "Error creating RO-Crate ZIP file" in str(excinfo.value) + + mock_rocrate.assert_called_once() + + +def test_provenance_no_rocrate(mocker): + + mock_rocrate = mocker.patch('autosubmit.autosubmit.Autosubmit.rocrate') + + with pytest.raises(AutosubmitCritical) as excinfo: + Autosubmit.provenance("expid123", rocrate=False) + + assert "Can not create RO-Crate ZIP file" in str(excinfo.value) + + mock_rocrate.assert_not_called() -- GitLab From b2180f192471094dd75ae087f043627c7a8c2725 Mon Sep 17 00:00:00 2001 From: Albert Puiggros Date: Mon, 14 Oct 2024 11:07:57 +0200 Subject: [PATCH 4/8] Unit test finished, now the code need to be reviewed --- autosubmit/autosubmit.py | 6 ++- test/unit/test_provenance.py | 83 +++++++++++++++++++----------------- 2 files changed, 48 insertions(+), 41 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 1ec68f453..1cb8107b4 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -4246,8 +4246,10 @@ class Autosubmit: try: Autosubmit.rocrate(expid, Path(aslogs_folder)) Log.info('RO-Crate ZIP file created!') - except Exception as e: - raise(f"Error creating RO-Crate ZIP file: {str(e)}", 7012) + except Exception as e: + raise AutosubmitCritical( + f"Error creating RO-Crate ZIP file: {str(e)}", 7012) + else : raise AutosubmitCritical( diff --git a/test/unit/test_provenance.py b/test/unit/test_provenance.py index 0c69a8f35..dd38cbf3d 100644 --- a/test/unit/test_provenance.py +++ b/test/unit/test_provenance.py @@ -1,57 +1,62 @@ import pytest from pathlib import Path from autosubmit.autosubmit import Autosubmit -from log.log import AutosubmitCritical +from log.log import AutosubmitCritical, Log import os -from autosubmitconfigparser.config.basicconfig import BasicConfig +from unittest.mock import patch @pytest.fixture -def test_provenance_rocrate_success(mocker, tmp_path): +def mock_paths(tmp_path): """ - Test the provenance function when rocrate=True and the process is successful. + Fixture to set temporary paths for BasicConfig values. """ - mock_rocrate = mocker.patch('autosubmit.autosubmit.Autosubmit.rocrate') - mock_log_info = mocker.patch('Log.info') - mock_basic_config = mocker.patch('BasicConfig') - - expid = "expid123" - - path = tmp_path / "expid123/tmp/ASLOGS" + with patch('autosubmitconfigparser.config.basicconfig.BasicConfig.LOCAL_ROOT_DIR', str(tmp_path)), \ + patch('autosubmitconfigparser.config.basicconfig.BasicConfig.LOCAL_TMP_DIR', 'tmp'), \ + patch('autosubmitconfigparser.config.basicconfig.BasicConfig.LOCAL_ASLOG_DIR', 'ASLOGS'): + yield tmp_path # Provide the temporary path to the test - exp_folder = os.path.join(tmp_path, expid) - - tmp_folder = os.path.join(exp_folder, mock_basic_config.LOCAL_TMP_DIR) +def test_provenance_rocrate_success(mock_paths): + """ + Test the provenance function when rocrate=True and the process is successful. + """ + with patch('autosubmit.autosubmit.Autosubmit.rocrate') as mock_rocrate, \ + patch('log.log.Log.info') as mock_log_info: - aslogs_folder = os.path.join(tmp_folder, mock_basic_config.LOCAL_ASLOG_DIR) - - expected_aslogs_path = os.path.join(tmp_folder, aslogs_folder) - - Autosubmit.provenance("expid123", rocrate=True) + expid = "expid123" - mock_rocrate.assert_called_once_with("expid123", Path(expected_aslogs_path)) - mock_log_info.assert_called_once_with('RO-Crate ZIP file created!') + exp_folder = os.path.join(str(mock_paths), expid) + tmp_folder = os.path.join(exp_folder, 'tmp') # Using the mocked value directly + aslogs_folder = os.path.join(tmp_folder, 'ASLOGS') # Using the mocked value directly + expected_aslogs_path = aslogs_folder # Adjust the path based on your expectations - assert expected_aslogs_path == path - -def test_provenance_rocrate_failure(mocker): - - mock_rocrate = mocker.patch('autosubmit.autosubmit.Autosubmit.rocrate', side_effect=Exception("Mocked exception")) - - with pytest.raises(Exception) as excinfo: - Autosubmit.provenance("expid123", rocrate=True) - - assert "Error creating RO-Crate ZIP file" in str(excinfo.value) - - mock_rocrate.assert_called_once() + # Call the provenance function + Autosubmit.provenance(expid, rocrate=True) + # Assertions + mock_rocrate.assert_called_once_with(expid, Path(expected_aslogs_path)) + mock_log_info.assert_called_once_with('RO-Crate ZIP file created!') + + assert Path(expected_aslogs_path) == mock_paths / "expid123/tmp/ASLOGS" -def test_provenance_no_rocrate(mocker): +def test_provenance_rocrate_failure(): + with patch('autosubmit.autosubmit.Autosubmit.rocrate', side_effect=Exception("Mocked exception")) as mock_rocrate: + + # Testing when Autosubmit.rocrate fails + with pytest.raises(AutosubmitCritical) as excinfo: + Autosubmit.provenance("expid123", rocrate=True) - mock_rocrate = mocker.patch('autosubmit.autosubmit.Autosubmit.rocrate') + # Assert that the correct error message is raised + assert "Error creating RO-Crate ZIP file: Mocked exception" in str(excinfo) + + # Make sure that the rocrate method was called once + mock_rocrate.assert_called_once() - with pytest.raises(AutosubmitCritical) as excinfo: - Autosubmit.provenance("expid123", rocrate=False) - assert "Can not create RO-Crate ZIP file" in str(excinfo.value) +def test_provenance_no_rocrate(): + with patch('autosubmit.autosubmit.Autosubmit.rocrate') as mock_rocrate: + with pytest.raises(AutosubmitCritical) as excinfo: + Autosubmit.provenance("expid123", rocrate=False) - mock_rocrate.assert_not_called() + # Assert that the correct error message is raised + assert "Can not create RO-Crate ZIP file" in str(excinfo) + mock_rocrate.assert_not_called() # Ensure rocrate was never called -- GitLab From 7f9845a3b5e0139bdfb7166ef9774d9a35f7fcba Mon Sep 17 00:00:00 2001 From: Albert Puiggros Date: Fri, 18 Oct 2024 10:03:43 +0200 Subject: [PATCH 5/8] Fixed archive/unarchive definitions and unit test for provenance --- autosubmit/autosubmit.py | 14 ++++++-------- test/unit/test_provenance.py | 31 ++++++++++++++----------------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 1cb8107b4..0dd696617 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -776,9 +776,9 @@ class Autosubmit: elif args.command == 'provenance': return Autosubmit.provenance(args.expid, rocrate=args.rocrate) #need to create the provenance function elif args.command == 'archive': - return Autosubmit.archive(args.expid, noclean=args.noclean, uncompress=args.uncompress) #, rocrate=args.rocrate) + return Autosubmit.archive(args.expid, noclean=args.noclean, uncompress=args.uncompress, rocrate=args.rocrate) elif args.command == 'unarchive': - return Autosubmit.unarchive(args.expid, uncompressed=args.uncompressed) #, rocrate=args.rocrate) + return Autosubmit.unarchive(args.expid, uncompressed=args.uncompressed, rocrate=args.rocrate) elif args.command == 'readme': if os.path.isfile(Autosubmit.readme_path): @@ -4238,9 +4238,9 @@ class Autosubmit: """"" exp_folder = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) - tmp_folder = os.path.join(exp_folder, BasicConfig.LOCAL_TMP_DIR) #Join experiment folder dir with tmp dir - - aslogs_folder = os.path.join(tmp_folder, BasicConfig.LOCAL_ASLOG_DIR) #Join tmp directory with aslogs + tmp_folder = os.path.join(exp_folder, BasicConfig.LOCAL_TMP_DIR) + + aslogs_folder = os.path.join(tmp_folder, BasicConfig.LOCAL_ASLOG_DIR) if rocrate: try: @@ -4249,11 +4249,9 @@ class Autosubmit: except Exception as e: raise AutosubmitCritical( f"Error creating RO-Crate ZIP file: {str(e)}", 7012) - - else : raise AutosubmitCritical( - "Can not create RO-Crate ZIP file", 7012) + "Can not create RO-Crate ZIP file. Argument '--rocrate' required", 7012) @staticmethod def archive(expid, noclean=True, uncompress=True, rocrate=False): diff --git a/test/unit/test_provenance.py b/test/unit/test_provenance.py index dd38cbf3d..43d29272d 100644 --- a/test/unit/test_provenance.py +++ b/test/unit/test_provenance.py @@ -1,7 +1,7 @@ import pytest from pathlib import Path from autosubmit.autosubmit import Autosubmit -from log.log import AutosubmitCritical, Log +from log.log import AutosubmitCritical import os from unittest.mock import patch @@ -13,7 +13,7 @@ def mock_paths(tmp_path): with patch('autosubmitconfigparser.config.basicconfig.BasicConfig.LOCAL_ROOT_DIR', str(tmp_path)), \ patch('autosubmitconfigparser.config.basicconfig.BasicConfig.LOCAL_TMP_DIR', 'tmp'), \ patch('autosubmitconfigparser.config.basicconfig.BasicConfig.LOCAL_ASLOG_DIR', 'ASLOGS'): - yield tmp_path # Provide the temporary path to the test + yield tmp_path def test_provenance_rocrate_success(mock_paths): """ @@ -23,40 +23,37 @@ def test_provenance_rocrate_success(mock_paths): patch('log.log.Log.info') as mock_log_info: expid = "expid123" - exp_folder = os.path.join(str(mock_paths), expid) - tmp_folder = os.path.join(exp_folder, 'tmp') # Using the mocked value directly - aslogs_folder = os.path.join(tmp_folder, 'ASLOGS') # Using the mocked value directly - expected_aslogs_path = aslogs_folder # Adjust the path based on your expectations + tmp_folder = os.path.join(exp_folder, 'tmp') + aslogs_folder = os.path.join(tmp_folder, 'ASLOGS') + expected_aslogs_path = aslogs_folder - # Call the provenance function Autosubmit.provenance(expid, rocrate=True) - # Assertions mock_rocrate.assert_called_once_with(expid, Path(expected_aslogs_path)) mock_log_info.assert_called_once_with('RO-Crate ZIP file created!') - - assert Path(expected_aslogs_path) == mock_paths / "expid123/tmp/ASLOGS" def test_provenance_rocrate_failure(): + """ + Test the provenance function when Autosubmit.rocrate fails + """ with patch('autosubmit.autosubmit.Autosubmit.rocrate', side_effect=Exception("Mocked exception")) as mock_rocrate: - # Testing when Autosubmit.rocrate fails with pytest.raises(AutosubmitCritical) as excinfo: Autosubmit.provenance("expid123", rocrate=True) - # Assert that the correct error message is raised assert "Error creating RO-Crate ZIP file: Mocked exception" in str(excinfo) - - # Make sure that the rocrate method was called once + mock_rocrate.assert_called_once() def test_provenance_no_rocrate(): + """ + Test the provenance function when rocrate=False + """ with patch('autosubmit.autosubmit.Autosubmit.rocrate') as mock_rocrate: with pytest.raises(AutosubmitCritical) as excinfo: Autosubmit.provenance("expid123", rocrate=False) - # Assert that the correct error message is raised - assert "Can not create RO-Crate ZIP file" in str(excinfo) - mock_rocrate.assert_not_called() # Ensure rocrate was never called + assert "Can not create RO-Crate ZIP file. Argument '--rocrate' required" in str(excinfo) + mock_rocrate.assert_not_called() -- GitLab From d337f2774ae599cceb31b50114031515f0dfb2fb Mon Sep 17 00:00:00 2001 From: Albert Puiggros Date: Wed, 30 Oct 2024 13:33:25 +0100 Subject: [PATCH 6/8] Fixed some aspects of the code --- autosubmit.db | Bin 16384 -> 0 bytes autosubmit/autosubmit.py | 19 ++++++++++--------- autosubmit/provenance/rocrate.py | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) delete mode 100644 autosubmit.db diff --git a/autosubmit.db b/autosubmit.db deleted file mode 100644 index 4ba475450f42f659f1755d4a76fb85e46f9d78dd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI&O=`kG7zW^(L{cJXy6bk_DM~k87ffYnr7_wx3a%vT6apq{5=*;m577hkUcEx0 zdl#KpL)4ZYK;9u?{{J4b`5^aqmY)drFpmN+VHf0-P)eGN5kfTi7UY{HMHY7JH0$^$ z*T{MFUXlaU@*6Rlz{}fB*y_009U<00Izz00g!I164b#ROoZ!jb~yy9zBXE z_UB<N5cKJ1TnB8@m-M6e-PtnfmbvoRv8!u-v^c}@w zxf0<-98}f49S%CHp%;jToLPIhq$lZ7U-@QD@w5<;ABZr?OUm9#d3&{}`_o@%WIdFwPa=PjNTa{Hy;U-gSL47>GRbUI zEsf1(&`*pHe0&gq00bZa0SG_<0uX=z1Rwwb2pp?G8vj${i^vBF0uX=z1Rwwb2tWV= z5P$##AOL}55hzi$qUS#${KWrjVyutlg183+AOHafKmY;|fB*y_009U<00O%L3RU!P Ds!fLq diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 0dd696617..a2910cf65 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -774,7 +774,7 @@ class Autosubmit: elif args.command == 'upgrade': return Autosubmit.upgrade_scripts(args.expid,files=args.files) elif args.command == 'provenance': - return Autosubmit.provenance(args.expid, rocrate=args.rocrate) #need to create the provenance function + return Autosubmit.provenance(args.expid, rocrate=args.rocrate) elif args.command == 'archive': return Autosubmit.archive(args.expid, noclean=args.noclean, uncompress=args.uncompress, rocrate=args.rocrate) elif args.command == 'unarchive': @@ -4229,19 +4229,20 @@ class Autosubmit: return create_rocrate_archive(as_conf, rocrate_json, jobs, start_time, end_time, path) @staticmethod - def provenance(expid, rocrate = False): - """"" + def provenance(expid, rocrate=False): + """" :param expid: experiment identifier :type expid: str :param rocrate: flag to enable RO-Crate :type rocrate: bool """"" - exp_folder = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) + aslogs_folder = Path( + BasicConfig.LOCAL_ROOT_DIR, + expid, + BasicConfig.LOCAL_TMP_DIR, + BasicConfig.LOCAL_ASLOG_DIR + ) - tmp_folder = os.path.join(exp_folder, BasicConfig.LOCAL_TMP_DIR) - - aslogs_folder = os.path.join(tmp_folder, BasicConfig.LOCAL_ASLOG_DIR) - if rocrate: try: Autosubmit.rocrate(expid, Path(aslogs_folder)) @@ -4249,7 +4250,7 @@ class Autosubmit: except Exception as e: raise AutosubmitCritical( f"Error creating RO-Crate ZIP file: {str(e)}", 7012) - else : + else: raise AutosubmitCritical( "Can not create RO-Crate ZIP file. Argument '--rocrate' required", 7012) diff --git a/autosubmit/provenance/rocrate.py b/autosubmit/provenance/rocrate.py index 24e702a46..027fc2b9f 100644 --- a/autosubmit/provenance/rocrate.py +++ b/autosubmit/provenance/rocrate.py @@ -562,7 +562,7 @@ def create_rocrate_archive( #Should I re-use the code in archive() using the full date instead of only the year? The problem is that the #zip is defined in this function so all the (modified) code should be moved here. Still, I'm exploring other posibilites #to query the las modified time within this function (Not very used to the code still...) - date = time.strftime("%Y%m%d", time.localtime(os.path.getmtime(path))) + date = time.strftime("%Y%m%d%H%M%S", time.localtime(os.path.getmtime(path))) crate.write_zip(Path(path, f"{expid}-{date}.zip")) Log.info(f'RO-Crate archive written to {experiment_path}') return crate -- GitLab From ff94e7690da5092d6130b52853635e615ec3253b Mon Sep 17 00:00:00 2001 From: Albert Puiggros Date: Tue, 3 Dec 2024 10:05:26 +0100 Subject: [PATCH 7/8] Run provenance feature --- autosubmit/autosubmit.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index a2910cf65..eb1fd89cd 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2389,6 +2389,9 @@ class Autosubmit: exp_history.finish_current_experiment_run() except: Log.warning("Database is locked") + ### Create rocrate object if requested + if "rocrate.yml" in os.listdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "conf")): + Autosubmit.provenance(expid, rocrate = TRUE) except (portalocker.AlreadyLocked, portalocker.LockException) as e: message = "We have detected that there is another Autosubmit instance using the experiment\n. Stop other Autosubmit instances that are using the experiment or delete autosubmit.lock file located on tmp folder" terminate_child_process(expid) -- GitLab From 120888cd4e45eb7a6e57a82470fae2625dd574fc Mon Sep 17 00:00:00 2001 From: Albert Puiggros Date: Tue, 3 Dec 2024 12:39:52 +0100 Subject: [PATCH 8/8] Rocrate object automatic generation when running experiment with rocrate.yml file in config --- autosubmit/autosubmit.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index eb1fd89cd..a7fb1573b 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2390,8 +2390,11 @@ class Autosubmit: except: Log.warning("Database is locked") ### Create rocrate object if requested - if "rocrate.yml" in os.listdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "conf")): - Autosubmit.provenance(expid, rocrate = TRUE) + rocrate_data = as_conf.experiment_data.get("ROCRATE") + if rocrate_data: + Autosubmit.provenance(expid, rocrate=True) + else: + print("rocrate.yml not found in CONFIG. Can't create rocrate object.") except (portalocker.AlreadyLocked, portalocker.LockException) as e: message = "We have detected that there is another Autosubmit instance using the experiment\n. Stop other Autosubmit instances that are using the experiment or delete autosubmit.lock file located on tmp folder" terminate_child_process(expid) -- GitLab