From fdb422fd2ad779058a836b8508bb365a2351c7b1 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Tue, 5 Dec 2023 15:19:25 +0100 Subject: [PATCH 1/5] initial docker image add docker image entrypoint #52 quickfix documentation #52 --- docker/Dockerfile | 20 +++++++++++++++ docker/README.md | 60 +++++++++++++++++++++++++++++++++++++++++++++ docker/autosubmitrc | 9 +++++++ docker/build.sh | 3 +++ docker/run.sh | 7 ++++++ docker/stop.sh | 3 +++ 6 files changed, 102 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100644 docker/autosubmitrc create mode 100755 docker/build.sh create mode 100755 docker/run.sh create mode 100755 docker/stop.sh diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..4aca610f --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.7.17-slim-bullseye + +ARG AUTOSUBMIT_ROOT_DIR=/app/autosubmit/ + +ARG API_VERSION=4.0.0b2 + +WORKDIR "${AUTOSUBMIT_ROOT_DIR}" + +COPY autosubmitrc /etc/ + +RUN pip3 install autosubmit-api==${API_VERSION} + +# Setup API env vars +ENV PROTECTION_LEVEL=ALL +ENV JWT_SECRET="M87;Z$,o5?MSC(/@#-LbzgE3PH-5ki.ZvS}N.s09v>I#v8I'00THrA-:ykh3HX?" +ENV CAS_LOGIN_URL="" +ENV CAS_VERIFY_URL="" + +ENTRYPOINT ["autosubmit_api"] +CMD ["start", "-b=0.0.0.0:8000", "--log-file=api.log"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..1882c227 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,60 @@ +## Prerequisites + +This image needs access to the files generated by autosubmit + +## Build + +To build the image you can execute `./build.sh`, or: + +```bash +docker build -t autosubmit-api . +``` + +## Run + +### Default run + +If you followed the normal installation path of autosubmit (creating the autosubmit directory `~/autosubmit`). You can run `./run.sh`. + +Else, you can set which are the directories that holds your DDBB files and experiment directories like this: + +```bash +docker run --name autosubmit-api-container \ + --rm -d -p 8099:8000 \ + -v ${YOUR_DB_PATH}:/app/autosubmit/database \ + -v ${YOUR_EXPERIMENTS_PATH}:/app/autosubmit/experiments \ + autosubmit-api +``` + +This will expose a default API instance on port `8099`. + +### Custom run (Recommended) + +Alternatively, you can run the command `autosubmit_api` with custom parameters by adding additional arguments like this: + +```bash +# Run with 6 workers, log level DEBUG, and map from port 9000 +docker run --name autosubmit-api-container \ + --rm -p 8099:9000 \ + -v ${YOUR_DB_PATH}:/app/autosubmit/database \ + -v ${YOUR_EXPERIMENTS_PATH}:/app/autosubmit/experiments \ + autosubmit-api start -w=6 --log-level=DEBUG -b=0.0.0.0:9000 +``` + +> **IMPORTANT** Always use the `-b` flag with `0.0.0.0` as the host to properly map the container API port + + +## Env Variables Configuration + +The `autosubmit_api` use env variables to configure some features like auth. + +Here is an example: + +```bash +docker run --name autosubmit-api-container \ + --rm -p 8099:9000 \ + -v ${YOUR_DB_PATH}:/app/autosubmit/database \ + -v ${YOUR_EXPERIMENTS_PATH}:/app/autosubmit/experiments \ + -e "PROTECTION_LEVEL=NONE" \ # This will disable protection + autosubmit-api start -w=6 --log-level=DEBUG -b=0.0.0.0:9000 +``` \ No newline at end of file diff --git a/docker/autosubmitrc b/docker/autosubmitrc new file mode 100644 index 00000000..9d518477 --- /dev/null +++ b/docker/autosubmitrc @@ -0,0 +1,9 @@ +[database] +path = /app/autosubmit/database +filename = autosubmit.db + +[local] +path = /app/autosubmit/experiments + +[globallogs] +path = /app/autosubmit/logs diff --git a/docker/build.sh b/docker/build.sh new file mode 100755 index 00000000..2cc723e9 --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker build -t autosubmit-api --progress=plain . &> build.log diff --git a/docker/run.sh b/docker/run.sh new file mode 100755 index 00000000..44da60e4 --- /dev/null +++ b/docker/run.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +docker run --name autosubmit-api-container \ + --rm -d -p 8099:8000 \ + -v ~/autosubmit:/app/autosubmit/database \ + -v ~/autosubmit:/app/autosubmit/experiments \ + autosubmit-api \ No newline at end of file diff --git a/docker/stop.sh b/docker/stop.sh new file mode 100755 index 00000000..7936c91e --- /dev/null +++ b/docker/stop.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker stop autosubmit-api-container \ No newline at end of file -- GitLab From 69dafe29326b40a498c882d5703993f1f8013a80 Mon Sep 17 00:00:00 2001 From: ltenorio Date: Fri, 31 May 2024 15:25:37 +0200 Subject: [PATCH 2/5] update Dockerfile --- docker/Dockerfile | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4aca610f..ad995273 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,20 +1,34 @@ -FROM python:3.7.17-slim-bullseye +FROM python:3.8-slim-bullseye -ARG AUTOSUBMIT_ROOT_DIR=/app/autosubmit/ +# Default API version +ARG API_VERSION=4.0.0b8 -ARG API_VERSION=4.0.0b2 +# Set the environment +ARG AUTOSUBMIT_ROOT_DIR=/app/autosubmit/ WORKDIR "${AUTOSUBMIT_ROOT_DIR}" COPY autosubmitrc /etc/ +RUN apt-get update && apt-get install -y git graphviz + RUN pip3 install autosubmit-api==${API_VERSION} # Setup API env vars ENV PROTECTION_LEVEL=ALL -ENV JWT_SECRET="M87;Z$,o5?MSC(/@#-LbzgE3PH-5ki.ZvS}N.s09v>I#v8I'00THrA-:ykh3HX?" +ENV JWT_SECRET="k87;Zg,o5?MSC(/@#-LbzgE3PH-5ki.ZvS}N.s09v>I#v8I'00THrA-:ykh3HX?" + ENV CAS_LOGIN_URL="" ENV CAS_VERIFY_URL="" +ENV RUN_BACKGROUND_TASKS_ON_START="False" +ENV DISABLE_BACKGROUND_TASKS="False" + +ENV GITHUB_OAUTH_CLIENT_ID="" +ENV GITHUB_OAUTH_CLIENT_SECRET="" +ENV GITHUB_OAUTH_WHITELIST_ORGANIZATION="" +ENV GITHUB_OAUTH_WHITELIST_TEAM="" + +# Entrypoint ENTRYPOINT ["autosubmit_api"] CMD ["start", "-b=0.0.0.0:8000", "--log-file=api.log"] -- GitLab From a7aec6ae3476228c15781dfb997233ac4be2d227 Mon Sep 17 00:00:00 2001 From: ltenorio Date: Mon, 3 Jun 2024 13:35:57 +0200 Subject: [PATCH 3/5] allow autosubmitrc configuration --- docker/Dockerfile | 9 ++++----- docker/README.md | 27 +++++++++++++++++++++++++-- docker/autosubmitrc | 1 + 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ad995273..32f400fe 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,12 +8,12 @@ ARG AUTOSUBMIT_ROOT_DIR=/app/autosubmit/ WORKDIR "${AUTOSUBMIT_ROOT_DIR}" -COPY autosubmitrc /etc/ - RUN apt-get update && apt-get install -y git graphviz RUN pip3 install autosubmit-api==${API_VERSION} +COPY autosubmitrc /app/configs/autosubmitrc + # Setup API env vars ENV PROTECTION_LEVEL=ALL ENV JWT_SECRET="k87;Zg,o5?MSC(/@#-LbzgE3PH-5ki.ZvS}N.s09v>I#v8I'00THrA-:ykh3HX?" @@ -21,14 +21,13 @@ ENV JWT_SECRET="k87;Zg,o5?MSC(/@#-LbzgE3PH-5ki.ZvS}N.s09v>I#v8I'00THrA-:ykh3HX?" ENV CAS_LOGIN_URL="" ENV CAS_VERIFY_URL="" -ENV RUN_BACKGROUND_TASKS_ON_START="False" -ENV DISABLE_BACKGROUND_TASKS="False" - ENV GITHUB_OAUTH_CLIENT_ID="" ENV GITHUB_OAUTH_CLIENT_SECRET="" ENV GITHUB_OAUTH_WHITELIST_ORGANIZATION="" ENV GITHUB_OAUTH_WHITELIST_TEAM="" +ENV AUTOSUBMIT_CONFIGURATION="/app/configs/autosubmitrc" + # Entrypoint ENTRYPOINT ["autosubmit_api"] CMD ["start", "-b=0.0.0.0:8000", "--log-file=api.log"] diff --git a/docker/README.md b/docker/README.md index 1882c227..ff5576d9 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,6 +1,6 @@ ## Prerequisites -This image needs access to the files generated by autosubmit +* This image needs access to the files generated by autosubmit ## Build @@ -10,6 +10,12 @@ To build the image you can execute `./build.sh`, or: docker build -t autosubmit-api . ``` +You can build with a different version of the API by running: + +```bash +docker build --build-arg API_VERSION=4.0.0b5 -t autosubmit-api . +``` + ## Run ### Default run @@ -44,7 +50,7 @@ docker run --name autosubmit-api-container \ > **IMPORTANT** Always use the `-b` flag with `0.0.0.0` as the host to properly map the container API port -## Env Variables Configuration +## Environment Variables Configuration The `autosubmit_api` use env variables to configure some features like auth. @@ -57,4 +63,21 @@ docker run --name autosubmit-api-container \ -v ${YOUR_EXPERIMENTS_PATH}:/app/autosubmit/experiments \ -e "PROTECTION_LEVEL=NONE" \ # This will disable protection autosubmit-api start -w=6 --log-level=DEBUG -b=0.0.0.0:9000 +``` + + +## Configuring the `.autosubmitrc` + +The default `.autosubmitrc` of this image is configured to use SQLite and uses some predefined paths like `/app/autosubmit/database` for the main DB files and `/app/autosubmit/experiments` for the experiments root directory. + +To modify it you have to make a new `.autosubmitrc` available in the container and mount its directory into the `/app/configs/` directory. + +```bash +docker run --name autosubmit-api-container \ + --rm -d -p 8099:8000 \ + -v ${YOUR_DB_PATH}:/app/autosubmit/database \ + -v ${YOUR_EXPERIMENTS_PATH}:/app/autosubmit/experiments \ + -e "PROTECTION_LEVEL=NONE" \ + -v ${YOUR_AUTOSUBMIT_CONFIGURATION_PATH}:/app/configs \ # Mount the directory with your .autosubmitrc file in the configs directory + autosubmit-api ``` \ No newline at end of file diff --git a/docker/autosubmitrc b/docker/autosubmitrc index 9d518477..aaa413e2 100644 --- a/docker/autosubmitrc +++ b/docker/autosubmitrc @@ -1,5 +1,6 @@ [database] path = /app/autosubmit/database +backend = sqlite filename = autosubmit.db [local] -- GitLab From 7c4270c1130a8753287bdf9c6da2da2196a513ce Mon Sep 17 00:00:00 2001 From: ltenorio Date: Fri, 7 Jun 2024 13:31:30 +0200 Subject: [PATCH 4/5] add pip install options --- docker/Dockerfile | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 32f400fe..fc8a5c83 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,8 +1,5 @@ FROM python:3.8-slim-bullseye -# Default API version -ARG API_VERSION=4.0.0b8 - # Set the environment ARG AUTOSUBMIT_ROOT_DIR=/app/autosubmit/ @@ -10,7 +7,19 @@ WORKDIR "${AUTOSUBMIT_ROOT_DIR}" RUN apt-get update && apt-get install -y git graphviz -RUN pip3 install autosubmit-api==${API_VERSION} +# Install Autosubmit API +# If GIT_REF is set, it will install from the specified branch/tag/commit. +# Otherwise, if AUTOSUBMIT_VERSION is set, it will install the specified version from pypi. +# Otherwise, it will install the latest version from pypi. +ARG GIT_REF +ARG AUTOSUBMIT_VERSION +RUN if [ -n "${GIT_REF}" ] ; then \ + pip3 install git+https://earth.bsc.es/gitlab/es/autosubmit-api.git@${GIT_REF}; \ + elif [ -n "${AUTOSUBMIT_VERSION}" ] ; then \ + pip3 install autosubmit-api==${AUTOSUBMIT_VERSION}; \ + else \ + pip3 install autosubmit-api; \ + fi COPY autosubmitrc /app/configs/autosubmitrc -- GitLab From fc972f016d22a4103f5550d85cdbe5fd13cae9ea Mon Sep 17 00:00:00 2001 From: ltenorio Date: Thu, 11 Jul 2024 11:17:56 +0200 Subject: [PATCH 5/5] clear apt cache --- docker/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index fc8a5c83..2b94298a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,11 +1,13 @@ -FROM python:3.8-slim-bullseye +FROM python:3.9-slim-bullseye # Set the environment ARG AUTOSUBMIT_ROOT_DIR=/app/autosubmit/ WORKDIR "${AUTOSUBMIT_ROOT_DIR}" -RUN apt-get update && apt-get install -y git graphviz +RUN apt-get update && apt-get install -y git graphviz gcc lsof && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # Install Autosubmit API # If GIT_REF is set, it will install from the specified branch/tag/commit. -- GitLab