From 72bf8ecc9c46c0d231c2ccb70f127b12d80fdea2 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Fri, 6 Sep 2024 14:33:48 +0200 Subject: [PATCH] Add notebook for auto-schism2 --- AutoSchism2.ipynb | 380 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 380 insertions(+) create mode 100644 AutoSchism2.ipynb diff --git a/AutoSchism2.ipynb b/AutoSchism2.ipynb new file mode 100644 index 0000000..275a7c2 --- /dev/null +++ b/AutoSchism2.ipynb @@ -0,0 +1,380 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "45f97884-7de7-44f1-8c4d-8b19945612ca", + "metadata": {}, + "outputs": [], + "source": [ + "#### \n", + "#### Settings to use a different user or repository.\n", + "####\n", + "bsc_user = \"bsc032????\"\n", + "edito_user = \"abcd???\"\n", + "# Where the workflow and configuration files are stored\n", + "repo = \"https://earth.bsc.es/gitlab/bdepaula/auto-schism2\"\n", + "# Branch \n", + "branch = \"main\" \n", + "# Where are the configuration files relative to git repo path\n", + "conf_path = \"conf\"\n", + "# Main platform of the experiment\n", + "hpcarch = \"marenostrum5\"\n", + "# A description of this experiment\n", + "exp_description = \"Auto Schism2 demo\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0f32698f-d482-4fee-935e-45f9295642fa", + "metadata": {}, + "outputs": [], + "source": [ + "#BSC connection checker\n", + "host = \"glogin1.bsc.es\"\n", + "scratch_dir = \"/gpfs/scratch\"\n", + "project = \"bsc32\"\n", + "cmd = f\"{bsc_user}@{host} echo Connected\"\n", + "!ssh -o StrictHostKeyChecking=accept-new {cmd} " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d2995f3-bc84-43ed-87af-cc80baf34e0d", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture output --no-stderr\n", + "!autosubmit expid -min -repo {repo} -H {hpcarch} -b {branch} -conf {conf_path} -d \"{exp_description}\"\n", + "# -min: \n", + " # Generates a minimal configuration file that replaces the one shown in previous examples\n", + " # Mainly used to load versioned configuration files, and the to-go configuration in AS 4+\n", + "# -b: Specifies the git branch\n", + "# -conf : Specifies the path to the versioned configuration files. The root folder is the git repository root." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f726e15f-153d-4ec3-af3f-d994564ec29f", + "metadata": {}, + "outputs": [], + "source": [ + "# Show expid output\n", + "output.show()\n", + "#Get Expid directly from this output for the other commands\n", + "expid_command_output = output.stdout\n", + "expid = expid_command_output.split(\"\\r\\n\")[-2].split(\" \")[1]\n", + "#If you want to use another expid, comment the previous lines and uncomment this one\n", + "#expid = \"a000\"\n", + "%store expid" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a9050449-4eb0-493e-8155-f61abee4c56d", + "metadata": {}, + "outputs": [], + "source": [ + "# FileStructure\n", + "!ls /home/onyxia/autosubmit/{expid}\n", + "# Mount it in the file explorer\n", + "!ln -s /home/onyxia/autosubmit/{expid} . 2> /dev/null # in case folder is already mounted\n", + "# The exp_root in this example is the expid folder.\n", + "# We have the expid in our current_directory as a symbolic link\n", + "exp_root = expid\n", + "%store exp_root" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "567dc42c-0622-40ac-87a1-f325e7b81075", + "metadata": {}, + "outputs": [], + "source": [ + "# This command downloads the git_repo and, by default, puts it inside the folder {exp_root}/proj/git_project\n", + "!autosubmit refresh {expid} -v" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bfe5d70b-b6bf-44d8-872a-11efd6bb21bf", + "metadata": {}, + "outputs": [], + "source": [ + "# Autosubmit refresh uses the content inside this file, to know where are the rest of the config files\n", + "!cat {expid}/conf/minimal.yml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e1b4600-7372-4592-85a0-b503bc291897", + "metadata": {}, + "outputs": [], + "source": [ + "!ls /home/onyxia/autosubmit/a000" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "adcc0ad2-c95d-483a-a169-eb0fea85062d", + "metadata": {}, + "outputs": [], + "source": [ + "# We can find the rest of the config in the DEFAULT.CUSTOM_CONFIG path\n", + "# %PROJDIR% is an AS_placeholder that points to {exp_root}/%PROJECT.PROJECT_DESTINATION%\n", + "!ls {exp_root}/proj/git_project/{conf_path}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6eef4a5e-ad4c-4b78-8a43-33fdf683ca61", + "metadata": {}, + "outputs": [], + "source": [ + "#In this repo, we can find the templates under %PROJDIR%/templates\n", + "!ls {exp_root}/proj/git_project/templates" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "49efa78f-7ecd-42a7-962f-488e4493d471", + "metadata": {}, + "outputs": [], + "source": [ + "# This step is slightly different than in the other notebooks.\n", + "# In this expid, the platform.yml is already there with the definition of the platforms. \n", + "#!cat {exp_root}/proj/git_project/as_conf/platforms.yml\n", + "# We will overwrite only what we want to change from that file.\n", + "# This means that instead of filling the whole platforms.yml we will fill only the necessary.\n", + "platforms_file = f\"\"\"PLATFORMS:\n", + " MARENOSTRUM5:\n", + " TYPE: slurm\n", + " HOST: glogin1.bsc.es\n", + " PROJECT: bsc32\n", + " USER: {bsc_user}\n", + " QUEUE: gp_debug\n", + " SCRATCH_DIR: /gpfs/scratch\n", + " ADD_PROJECT_TO_HOST: False\n", + " MAX_WALLCLOCK: '48:00'\n", + "\"\"\"\n", + "# And store it in our home directory.\n", + "import os \n", + "with open(f\"{os.environ['HOME']}/platforms.yml\", \"w\") as fd:\n", + " fd.write(platforms_file)\n", + "\n", + "# For Autosubmit to acknowledge this file, we must modify the {exp_root}/conf/minimal.yml %DEFAULT.CUSTOM_CONFIG% key and add a POST key. \n", + "# You can also use the file_explorer\n", + "minimal=f\"\"\"CONFIG:\n", + " # Current version of Autosubmit.\n", + " AUTOSUBMIT_VERSION: \"4.1.10\"\n", + " # Total number of jobs in the workflow.\n", + " TOTALJOBS: 20\n", + " # Maximum number of jobs permitted in the waiting status.\n", + " MAXWAITINGJOBS: 20\n", + "DEFAULT:\n", + " # Job experiment ID.\n", + " EXPID: \"{expid}\"\n", + " # Default HPC platform name.\n", + " HPCARCH: {hpcarch}\n", + " #hint: use %PROJDIR% to point to the project folder (where the project is cloned)\n", + " # Custom configuration location.\n", + " CUSTOM_CONFIG: \n", + " PRE: \"%PROJDIR%/{conf_path}\"\n", + " POST: \"~/platforms.yml\"\n", + "PROJECT:\n", + " # Type of the project.\n", + " PROJECT_TYPE: git\n", + " # Folder to hold the project sources.\n", + " PROJECT_DESTINATION: git_project\n", + "EXPERIMENT:\n", + " DATELIST: \"20000101\"\n", + " MEMBERS: \"fc0\"\n", + " CHUNKSIZEUNIT: month\n", + " CHUNKSIZE: \"4\"\n", + " NUMCHUNKS: \"2\"\n", + " CHUNKINI: ''\n", + " CALENDAR: standard\n", + "GIT:\n", + " PROJECT_ORIGIN: \"{repo}\"\n", + " PROJECT_BRANCH: \"{branch}\"\n", + " PROJECT_COMMIT: ''\n", + " PROJECT_SUBMODULES: ''\n", + " FETCH_SINGLE_BRANCH: true\n", + "\"\"\"\n", + "# And store it in the conf path\n", + "with open(f\"{expid}/conf/minimal.yml\", \"w\") as fd:\n", + " fd.write(minimal)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5cbde5e-88cd-4aff-86f0-93754e7cd5d3", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture output --no-stderr\n", + "!autosubmit create {expid} -v" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5213a410-1c1e-403f-9966-4657c574e3b7", + "metadata": {}, + "outputs": [], + "source": [ + "# Capture the file_path to the plot\n", + "create_command_output = output.stdout\n", + "parsed_by_lines = create_command_output.split(\"\\r\\n\")\n", + "for line in parsed_by_lines:\n", + " if \"Plot created\".casefold() in line.casefold():\n", + " filepath = line.split(\" \")[-1].split(\".\")[0] +\".pdf\"\n", + "print(filepath) \n", + "# Then plot it using Jupyter Python.\n", + "from IPython.display import IFrame\n", + "from pathlib import Path\n", + "IFrame(f\"{expid}/plot/{Path(filepath).name}\", width=600, height=300)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8bbf4ae9-d46e-4cd7-8e3a-8ba5fbfdb0e0", + "metadata": {}, + "outputs": [], + "source": [ + "!autosubmit run {expid} " + ] + }, + { + "cell_type": "markdown", + "id": "5296c737-0091-4365-a36a-3882fa457bfe", + "metadata": {}, + "source": [ + "> NOTE: In the next sessions you will learn to use the Autosubmit GUI to monitor your experiments. Here we will use the Autosubmit command-line interface for that." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "03f56d72-a0b8-4cc3-b60e-0bcb690eb155", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture monitor_output --no-stderr\n", + "!autosubmit monitor {expid}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "09e4ed93-4044-4881-9e9d-944b73687586", + "metadata": {}, + "outputs": [], + "source": [ + "# Capture the file_path to the plot\n", + "monitor_command_output = monitor_output.stdout\n", + "parsed_by_lines = monitor_command_output.split(\"\\r\\n\")\n", + "for line in parsed_by_lines:\n", + " if \"Plot created\".casefold() in line.casefold():\n", + " filepath = line.split(\" \")[-1].split(\".\")[0] +\".pdf\"\n", + "print(filepath) \n", + "# Then plot it using Jupyter Python.\n", + "from IPython.display import IFrame\n", + "from pathlib import Path\n", + "IFrame(f\"{expid}/plot/{Path(filepath).name}\", width=600, height=300)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fb9d4cc2-6c92-49d4-af59-bff8230d3227", + "metadata": {}, + "outputs": [], + "source": [ + "#Aslogs are located inside\n", + "!ls -hart {exp_root}/tmp/ASLOGS" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2830d792-925f-44f0-b7ab-b5d6046fde49", + "metadata": {}, + "outputs": [], + "source": [ + "#template logs are downloaded from the remote platform and stored inside\n", + "!ls -hart {exp_root}/tmp/LOG_{expid}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6886bf0a-fa0b-47c9-b8f9-fd00bd963f18", + "metadata": {}, + "outputs": [], + "source": [ + "# Workflow ran inside\n", + "cmd = f\"{bsc_user}@{host} ls -l {scratch_dir}/{project}/{bsc_user}/{expid}\"\n", + "!ssh -o StrictHostKeyChecking=accept-new {cmd} " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8fb7d4c7-b8e6-41f6-98a7-e84ae0bfc2de", + "metadata": {}, + "outputs": [], + "source": [ + "# TODO: we need to export this for now due to an issue in the service configuration\n", + "import os\n", + "%env MC_HOST_s3=https://{os.environ[\"AWS_ACCESS_KEY_ID\"]}:{os.environ[\"AWS_SECRET_ACCESS_KEY\"]}:{os.environ[\"AWS_SESSION_TOKEN\"]}@{os.environ[\"AWS_S3_ENDPOINT\"]}\n", + "# Persist the work\n", + "# If you are launching from your own project, use this\n", + "!mc cp --recursive /home/onyxia/autosubmit s3/oidc-{edito_user}\n", + "# If you are launching from project-modellab project, use this\n", + "# !mc cp --recursive /home/onyxia/autosubmit s3/project-modellab/{edito_user}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20c07797-e7c2-45ae-a157-b0b2ac3b03e7", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} -- GitLab