From 3377a74884b70106f0c15441c05fe96098fafbbb Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Wed, 5 Jun 2024 12:52:39 +0200 Subject: [PATCH 01/11] pip install from git --- dockerfiles/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 3de4a04a7..baf1920df 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -126,7 +126,8 @@ RUN micromamba install --yes --name base --channel conda-forge \ /usr/local/bin/_activate_current_env.sh # Install Autosubmit. -RUN pip install autosubmit==4.0.84 +ARG GIT_REF=master +RUN pip install git+https://earth.bsc.es/gitlab/es/autosubmit.git@${GIT_REF} # Configure Autosubmit. RUN autosubmit configure \ -- GitLab From a65286541a4afe29790bb83fd1004b67775806c6 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Wed, 5 Jun 2024 17:00:21 +0200 Subject: [PATCH 02/11] update apt deps --- dockerfiles/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index baf1920df..c85e7a990 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -95,16 +95,17 @@ RUN mkdir -pv "${AUTOSUBMIT_ROOT_DIR}/logs" && \ # TODO: add something like xpdf=3.04+git20210103-3 if GUI/X is needed. RUN apt update && \ apt install -y \ + gcc \ bash=5.1-2+deb11u1 \ ca-certificates=20210119 \ - curl=7.74.0-1.3+deb11u7 \ + curl \ desktop-file-utils=0.26-1 \ dialog=1.3-20201126-1 \ graphviz=2.42.2-5 \ iputils-ping=3:20210202-1 \ less=551-2 \ net-tools=1.60+git20181103.0eebece-1 \ - openssh-server=1:8.4p1-5+deb11u1 \ + openssh-server \ python3-tk=3.9.2-1 \ sqlite3=3.34.1-3 \ sudo=1.9.5p2-3+deb11u1 \ -- GitLab From 33b0d73edfe4a604c9f2b5f356fdf050b9ce15a8 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Wed, 5 Jun 2024 17:15:47 +0200 Subject: [PATCH 03/11] allow autosubmitrc change --- dockerfiles/Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index c85e7a990..074cbc943 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -137,6 +137,14 @@ RUN autosubmit configure \ -lr "${AUTOSUBMIT_ROOT_DIR}/experiments/" && \ autosubmit install +# Copy file /home/${MAMBA_USER}/.autosubmitrc to ${CONFIGS_DIR}/autosubmitrc +ARG CONFIGS_DIR=/app/configs/ +RUN mkdir -pv ${CONFIGS_DIR} && \ + cp "/home/${MAMBA_USER}/.autosubmitrc" "${CONFIGS_DIR}/autosubmitrc" && \ + chown -R "${MAMBA_USER}:${MAMBA_USER}" "${CONFIGS_DIR}" + +ENV AUTOSUBMIT_CONFIGURATION="${CONFIGS_DIR}/autosubmitrc" + # SSH (for Autosubmit local platform.) USER root -- GitLab From 0b6eac6db78024a3c3cdc8d49111aacd9156fc17 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Thu, 6 Jun 2024 12:01:55 +0200 Subject: [PATCH 04/11] pip install options & bookworm update --- dockerfiles/Dockerfile | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 074cbc943..39267f7dd 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -9,7 +9,7 @@ FROM mambaorg/micromamba:1.4.9-bullseye-slim AS micromamba # image autosubmit-base, for example, with the users, permissions, # dependencies, and SSH. Which would tremendously reduce this. -FROM debian:bullseye-slim +FROM debian:bookworm-slim ARG AUTOSUBMIT_ROOT_DIR=/app/autosubmit/ @@ -96,20 +96,20 @@ RUN mkdir -pv "${AUTOSUBMIT_ROOT_DIR}/logs" && \ RUN apt update && \ apt install -y \ gcc \ - bash=5.1-2+deb11u1 \ - ca-certificates=20210119 \ + bash \ + ca-certificates \ curl \ desktop-file-utils=0.26-1 \ - dialog=1.3-20201126-1 \ - graphviz=2.42.2-5 \ - iputils-ping=3:20210202-1 \ - less=551-2 \ - net-tools=1.60+git20181103.0eebece-1 \ + dialog \ + graphviz \ + iputils-ping \ + less \ + net-tools \ openssh-server \ - python3-tk=3.9.2-1 \ - sqlite3=3.34.1-3 \ - sudo=1.9.5p2-3+deb11u1 \ - vim=2:8.2.2434-3+deb11u1 \ + python3-tk \ + sqlite3 \ + sudo \ + vim \ xdg-utils=1.1.3-4.1 && \ update-ca-certificates && \ apt-get clean && \ @@ -126,9 +126,19 @@ RUN micromamba install --yes --name base --channel conda-forge \ micromamba clean --all --yes && \ /usr/local/bin/_activate_current_env.sh -# Install Autosubmit. -ARG GIT_REF=master -RUN pip install git+https://earth.bsc.es/gitlab/es/autosubmit.git@${GIT_REF} +# Install Autosubmit. +# If GIT_REF is set, it will install from the specified branch/tag/commit. +# Otherwise, if AUTOSUBMIT_VERSION is set, it will install the specified version from pypi. +# Otherwise, it will install the latest version from pypi. +ARG GIT_REF +ARG AUTOSUBMIT_VERSION +RUN if [ -n "${GIT_REF}" ] ; then \ + pip install git+https://earth.bsc.es/gitlab/es/autosubmit.git@${GIT_REF}; \ + elif [ -n "${AUTOSUBMIT_VERSION}" ] ; then \ + pip install autosubmit==${AUTOSUBMIT_VERSION}; \ + else \ + pip install autosubmit; \ + fi # Configure Autosubmit. RUN autosubmit configure \ -- GitLab From 6a16f0468e03b8967546c9e905362419dec4c6a9 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Thu, 6 Jun 2024 15:27:21 +0200 Subject: [PATCH 05/11] update config path --- {dockerfiles => docker}/Dockerfile | 14 ++++++-------- {dockerfiles => docker}/README.md | 0 {dockerfiles => docker}/docker-compose.yml | 0 {dockerfiles => docker}/test.sh | 0 4 files changed, 6 insertions(+), 8 deletions(-) rename {dockerfiles => docker}/Dockerfile (93%) rename {dockerfiles => docker}/README.md (100%) rename {dockerfiles => docker}/docker-compose.yml (100%) rename {dockerfiles => docker}/test.sh (100%) diff --git a/dockerfiles/Dockerfile b/docker/Dockerfile similarity index 93% rename from dockerfiles/Dockerfile rename to docker/Dockerfile index 39267f7dd..318027411 100644 --- a/dockerfiles/Dockerfile +++ b/docker/Dockerfile @@ -144,16 +144,14 @@ RUN if [ -n "${GIT_REF}" ] ; then \ RUN autosubmit configure \ -db "${AUTOSUBMIT_ROOT_DIR}/database/" \ -dbf autosubmit.db \ - -lr "${AUTOSUBMIT_ROOT_DIR}/experiments/" && \ - autosubmit install + -lr "${AUTOSUBMIT_ROOT_DIR}/experiments/" -# Copy file /home/${MAMBA_USER}/.autosubmitrc to ${CONFIGS_DIR}/autosubmitrc -ARG CONFIGS_DIR=/app/configs/ -RUN mkdir -pv ${CONFIGS_DIR} && \ - cp "/home/${MAMBA_USER}/.autosubmitrc" "${CONFIGS_DIR}/autosubmitrc" && \ - chown -R "${MAMBA_USER}:${MAMBA_USER}" "${CONFIGS_DIR}" +# Copy the default config file /home/${MAMBA_USER}/.autosubmitrc to ${AUTOSUBMIT_ROOT_DIR}/autosubmitrc +RUN cp "/home/${MAMBA_USER}/.autosubmitrc" "${AUTOSUBMIT_ROOT_DIR}/autosubmitrc" && \ + chown -R "${MAMBA_USER}:${MAMBA_USER}" "${AUTOSUBMIT_ROOT_DIR}" -ENV AUTOSUBMIT_CONFIGURATION="${CONFIGS_DIR}/autosubmitrc" +# Set AUTOSUBMIT_CONFIGURATION to the path of the Autosubmit configuration file. +ENV AUTOSUBMIT_CONFIGURATION="${AUTOSUBMIT_ROOT_DIR}/autosubmitrc" # SSH (for Autosubmit local platform.) USER root diff --git a/dockerfiles/README.md b/docker/README.md similarity index 100% rename from dockerfiles/README.md rename to docker/README.md diff --git a/dockerfiles/docker-compose.yml b/docker/docker-compose.yml similarity index 100% rename from dockerfiles/docker-compose.yml rename to docker/docker-compose.yml diff --git a/dockerfiles/test.sh b/docker/test.sh similarity index 100% rename from dockerfiles/test.sh rename to docker/test.sh -- GitLab From 1f50d7790b6d87ec1e9698e0589b693c88e4e0b5 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Ku Date: Thu, 13 Jun 2024 14:39:23 +0200 Subject: [PATCH 06/11] EDITO Helm/Docker Demo --- docker/demo/Dockerfile | 30 ++++++++++++ docker/demo/README.md | 55 ++++++++++++++++++++++ docker/demo/entrypoint.sh | 7 +++ docker/demo/helm/.helmignore | 23 +++++++++ docker/demo/helm/Chart.yaml | 24 ++++++++++ docker/demo/helm/templates/deployment.yaml | 32 +++++++++++++ docker/demo/helm/templates/ingress.yaml | 48 +++++++++++++++++++ docker/demo/helm/templates/service.yaml | 23 +++++++++ docker/demo/helm/values.yaml | 10 ++++ 9 files changed, 252 insertions(+) create mode 100644 docker/demo/Dockerfile create mode 100644 docker/demo/README.md create mode 100644 docker/demo/entrypoint.sh create mode 100644 docker/demo/helm/.helmignore create mode 100644 docker/demo/helm/Chart.yaml create mode 100644 docker/demo/helm/templates/deployment.yaml create mode 100644 docker/demo/helm/templates/ingress.yaml create mode 100644 docker/demo/helm/templates/service.yaml create mode 100644 docker/demo/helm/values.yaml diff --git a/docker/demo/Dockerfile b/docker/demo/Dockerfile new file mode 100644 index 000000000..ac3be4ba6 --- /dev/null +++ b/docker/demo/Dockerfile @@ -0,0 +1,30 @@ +FROM python:3.8-slim-bookworm + +# Install apt dependencies +RUN apt-get update && \ + apt-get install -y git graphviz sqlite3 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Install Autosubmit +ARG AUTOSUBMIT_VERSION=4.0.105 +ARG AUTOSUBMIT_API_VERSION=4.0.0b8 + +RUN pip3 install \ + autosubmit==${AUTOSUBMIT_VERSION} \ + autosubmit-api==${AUTOSUBMIT_API_VERSION} \ + jupyterlab + +# Set the environment variables +ENV PROTECTION_LEVEL=NONE + +# Install Autosubmit +RUN autosubmit configure &&\ + autosubmit install + +# Copy entrypoint script +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +# Set the entrypoint +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/demo/README.md b/docker/demo/README.md new file mode 100644 index 000000000..b86253b06 --- /dev/null +++ b/docker/demo/README.md @@ -0,0 +1,55 @@ + +## Prepare Minikube + +### Build the docker images inside minikube + +Build the images inside Minikube + +```bash +eval $(minikube docker-env) + +docker build -t as-demo:latest ~/projects/autosubmit/docker/demo +docker build --build-arg "AUTOSUBMIT_API_SOURCE=/api" --build-arg="PUBLIC_URL=/gui"\ + -t as-gui-demo:latest ~/projects/autosubmitreact-update/docker +``` + +The script above use the default names of the images. You can change and set them in the `values.yaml` as you wish. + +### Install Nginx Ingress Controller + +Install nginx Ingress Controller https://kubernetes.github.io/ingress-nginx/deploy/ + +```bash +helm upgrade --install ingress-nginx ingress-nginx \ + --repo https://kubernetes.github.io/ingress-nginx \ + --namespace ingress-nginx --create-namespace +``` + +Enable Ingress for testing + +```bash +minikube addons enable ingress +``` + +## Install using helm + +Install helm + +```bash +helm install test-demo . +``` + + +Clean up helm + +```bash +helm uninstall test-demo +``` + +## Test locally + +Forward nginx ingress controller port `80` to `localhost:8080`: + +```bash +kubectl port-forward --namespace=ingress-nginx service/ingress-nginx-controller 8080:80 +``` \ No newline at end of file diff --git a/docker/demo/entrypoint.sh b/docker/demo/entrypoint.sh new file mode 100644 index 000000000..b26154217 --- /dev/null +++ b/docker/demo/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Run jupyter lab as daemon +jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --allow-root --NotebookApp.base_url=/jupyterlab & + +# Run the command passed by docker run +autosubmit_api start -b 0.0.0.0:8000 diff --git a/docker/demo/helm/.helmignore b/docker/demo/helm/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/docker/demo/helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/docker/demo/helm/Chart.yaml b/docker/demo/helm/Chart.yaml new file mode 100644 index 000000000..7875694c6 --- /dev/null +++ b/docker/demo/helm/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: autosubmit-demo +description: A Helm chart for trying Autosubmit tools + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/docker/demo/helm/templates/deployment.yaml b/docker/demo/helm/templates/deployment.yaml new file mode 100644 index 000000000..2482ccd9f --- /dev/null +++ b/docker/demo/helm/templates/deployment.yaml @@ -0,0 +1,32 @@ +# Add deployment and service +apiVersion: apps/v1 +kind: Deployment +metadata: + name: autosubmit-deployment + labels: + app: autosubmit-demo +spec: + replicas: 1 + selector: + matchLabels: + app: autosubmit-demo + template: + metadata: + labels: + app: autosubmit-demo + spec: + containers: + # Main Autosubmit docker image with exposed ports 8888 and 8000 + - name: autosubmit-container + image: {{ .Values.images.autosubmitDemo.image }} + imagePullPolicy: {{ .Values.images.autosubmitDemo.imagePullPolicy }} + ports: + - containerPort: 8888 + - containerPort: 8000 + # GUI docker image with exposed port 3000 + - name: autosubmit-gui-container + image: {{ .Values.images.autosubmitDemoGUI.image }} + imagePullPolicy: {{ .Values.images.autosubmitDemoGUI.imagePullPolicy }} + ports: + - containerPort: 8080 + diff --git a/docker/demo/helm/templates/ingress.yaml b/docker/demo/helm/templates/ingress.yaml new file mode 100644 index 000000000..733096deb --- /dev/null +++ b/docker/demo/helm/templates/ingress.yaml @@ -0,0 +1,48 @@ +# Rewrite ingress +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: autosubmit-ingress + annotations: + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/rewrite-target: /$2 +spec: + ingressClassName: nginx + rules: + - host: {{ .Values.ingress.host }} + http: + paths: + - pathType: ImplementationSpecific + path: "/api(/|$)(.*)" + backend: + service: + name: autosubmit-demo-service + port: + number: 8000 +--- +# Prefix ingress +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: autosubmit-ingress-gui +spec: + ingressClassName: nginx + rules: + - host: {{ .Values.ingress.host }} + http: + paths: + - pathType: Prefix + path: "/jupyterlab" + backend: + service: + name: autosubmit-demo-service + port: + number: 8888 + - pathType: Prefix + path: "/gui" + backend: + service: + name: autosubmit-demo-service + port: + number: 8080 + \ No newline at end of file diff --git a/docker/demo/helm/templates/service.yaml b/docker/demo/helm/templates/service.yaml new file mode 100644 index 000000000..1432a0d32 --- /dev/null +++ b/docker/demo/helm/templates/service.yaml @@ -0,0 +1,23 @@ +# Add service +apiVersion: v1 +kind: Service +metadata: + name: autosubmit-demo-service + labels: + app: autosubmit-demo +spec: + selector: + app: autosubmit-demo + ports: + - name: jupyterlab-port + protocol: TCP + port: 8888 + targetPort: 8888 + - name: api-port + protocol: TCP + port: 8000 + targetPort: 8000 + - name: gui-port + protocol: TCP + port: 8080 + targetPort: 8080 diff --git a/docker/demo/helm/values.yaml b/docker/demo/helm/values.yaml new file mode 100644 index 000000000..5322d4e3a --- /dev/null +++ b/docker/demo/helm/values.yaml @@ -0,0 +1,10 @@ +images: + autosubmitDemo: + image: as-demo + imagePullPolicy: Never + autosubmitDemoGUI: + image: as-gui-demo + imagePullPolicy: Never + +ingress: + host: localhost \ No newline at end of file -- GitLab From 2353da5481c31e90814e3854b7235b7ee555ffd8 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Fri, 14 Jun 2024 10:34:16 +0200 Subject: [PATCH 07/11] update helm image values --- docker/demo/helm/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/demo/helm/values.yaml b/docker/demo/helm/values.yaml index 5322d4e3a..97ca8b105 100644 --- a/docker/demo/helm/values.yaml +++ b/docker/demo/helm/values.yaml @@ -1,10 +1,10 @@ images: autosubmitDemo: - image: as-demo - imagePullPolicy: Never + image: autosubmit/edito-demo:as-4.0.105-api-4.0.0b8 + imagePullPolicy: IfNotPresent autosubmitDemoGUI: - image: as-gui-demo - imagePullPolicy: Never + image: autosubmit/gui:edito-demo + imagePullPolicy: IfNotPresent ingress: host: localhost \ No newline at end of file -- GitLab From 31680f3b06d3a6fa45a5ef361b7cdcec5c6536ee Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Wed, 19 Jun 2024 11:30:42 +0200 Subject: [PATCH 08/11] add jupyter lab token in demo --- docker/demo/Dockerfile | 4 ++++ docker/demo/entrypoint.sh | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docker/demo/Dockerfile b/docker/demo/Dockerfile index ac3be4ba6..6999be801 100644 --- a/docker/demo/Dockerfile +++ b/docker/demo/Dockerfile @@ -17,6 +17,7 @@ RUN pip3 install \ # Set the environment variables ENV PROTECTION_LEVEL=NONE +ENV JUPYTER_TOKEN="" # Install Autosubmit RUN autosubmit configure &&\ @@ -26,5 +27,8 @@ RUN autosubmit configure &&\ COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh +EXPOSE 8888 +EXPOSE 8000 + # Set the entrypoint ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/demo/entrypoint.sh b/docker/demo/entrypoint.sh index b26154217..56ba7d23a 100644 --- a/docker/demo/entrypoint.sh +++ b/docker/demo/entrypoint.sh @@ -1,7 +1,11 @@ #!/bin/bash -# Run jupyter lab as daemon -jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --allow-root --NotebookApp.base_url=/jupyterlab & +# Run jupyter lab as daemon and assign token if env variable exists +if [ -n "$JUPYTER_TOKEN" ]; then + jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --allow-root --NotebookApp.base_url=/jupyterlab --NotebookApp.token=$JUPYTER_TOKEN & +else + jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --allow-root --NotebookApp.base_url=/jupyterlab & +fi # Run the command passed by docker run autosubmit_api start -b 0.0.0.0:8000 -- GitLab From 86fe55ac48f2788387f1784fa7f43fa5ee0c8357 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Wed, 19 Jun 2024 16:56:05 +0200 Subject: [PATCH 09/11] add Vault --- docker/demo/Dockerfile | 20 ++++++++++++++++- docker/demo/entrypoint.sh | 3 +++ docker/demo/load_ssh_private_key.sh | 34 +++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 docker/demo/load_ssh_private_key.sh diff --git a/docker/demo/Dockerfile b/docker/demo/Dockerfile index 6999be801..1feee9c0f 100644 --- a/docker/demo/Dockerfile +++ b/docker/demo/Dockerfile @@ -2,10 +2,25 @@ FROM python:3.8-slim-bookworm # Install apt dependencies RUN apt-get update && \ - apt-get install -y git graphviz sqlite3 && \ + apt-get install -y git graphviz sqlite3 wget gpg lsb-release && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +# Install Vault +RUN wget -O- https://apt.releases.hashicorp.com/gpg | apt-key add - && \ + echo "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" > /etc/apt/sources.list.d/vault.list && \ + apt-get update && \ + apt-get install -y vault && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Prepare Vault ENVs https://github.com/InseeFrLab/helm-charts-interactive-services/blob/main/charts/library-chart/templates/_secret.tpl +ENV VAULT_ADDR="" +ENV VAULT_TOKEN="" +ENV VAULT_RELATIVE_PATH="" +ENV VAULT_TOP_DIR="" +ENV VAULT_MOUNT="" + # Install Autosubmit ARG AUTOSUBMIT_VERSION=4.0.105 ARG AUTOSUBMIT_API_VERSION=4.0.0b8 @@ -27,6 +42,9 @@ RUN autosubmit configure &&\ COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh +COPY load_ssh_private_key.sh /load_ssh_private_key.sh +RUN chmod +x /load_ssh_private_key.sh + EXPOSE 8888 EXPOSE 8000 diff --git a/docker/demo/entrypoint.sh b/docker/demo/entrypoint.sh index 56ba7d23a..9585a20b8 100644 --- a/docker/demo/entrypoint.sh +++ b/docker/demo/entrypoint.sh @@ -1,5 +1,8 @@ #!/bin/bash +# Execute /load_ssh_private_key.sh +/load_ssh_private_key.sh + # Run jupyter lab as daemon and assign token if env variable exists if [ -n "$JUPYTER_TOKEN" ]; then jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --allow-root --NotebookApp.base_url=/jupyterlab --NotebookApp.token=$JUPYTER_TOKEN & diff --git a/docker/demo/load_ssh_private_key.sh b/docker/demo/load_ssh_private_key.sh new file mode 100644 index 000000000..bbe62c36c --- /dev/null +++ b/docker/demo/load_ssh_private_key.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +SSH_PATH=$HOME/.ssh +mkdir -p $SSH_PATH + +# If VAULT_ADDR is not set, then we are not running in a container +if [ -z "${VAULT_ADDR}" ] +then + # VAULT_ADDR is not set + ssh-keygen -b 4096 -t rsa -f "${SSH_PATH}/id_rsa" -q -N "" + SSH_PRIVATE_KEY=$(base64 "${SSH_PATH}/id_rsa" -w 0) + SSH_PUBLIC_KEY=$(base64 "${SSH_PATH}/id_rsa.pub" -w 0) + +else + # Original script, assumes VAULT_ADDR is set + echo "begin script" + whoami + set -euo pipefail + + SSH_PRIVATE_KEY=$(vault kv get -field=SSH_PRIVATE_KEY "${VAULT_MOUNT}/${VAULT_TOP_DIR}/autosubmit" || echo "") + if [ -z "$SSH_PRIVATE_KEY" ] + then + ssh-keygen -b 4096 -t rsa -f "${SSH_PATH}/id_rsa" -q -N "" + SSH_PRIVATE_KEY=$(base64 "${SSH_PATH}/id_rsa" -w 0) + SSH_PUBLIC_KEY=$(base64 "${SSH_PATH}/id_rsa.pub" -w 0) + vault kv put "${VAULT_MOUNT}/${VAULT_TOP_DIR}/autosubmit" SSH_PRIVATE_KEY="${SSH_PRIVATE_KEY}" SSH_PUBLIC_KEY="${SSH_PUBLIC_KEY}" + else + SSH_PUBLIC_KEY=$(vault kv get -field=SSH_PUBLIC_KEY "${VAULT_MOUNT}/${VAULT_TOP_DIR}/autosubmit" || echo "") + echo $SSH_PRIVATE_KEY | base64 -d > "${SSH_PATH}/id_rsa" + echo $SSH_PUBLIC_KEY | base64 -d > "${SSH_PATH}/id_rsa.pub" + fi +fi + +chmod 600 "${SSH_PATH}/id_rsa" \ No newline at end of file -- GitLab From cce1a1bc425f1d611cd2bf42aeb0c36f559ef482 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Wed, 19 Jun 2024 18:21:40 +0200 Subject: [PATCH 10/11] Using EDITO team's base image (tested and working). --- docker/demo/Dockerfile | 35 +++++++++++------------------------ docker/demo/entrypoint.sh | 4 ++-- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/docker/demo/Dockerfile b/docker/demo/Dockerfile index 1feee9c0f..777e1ff29 100644 --- a/docker/demo/Dockerfile +++ b/docker/demo/Dockerfile @@ -1,25 +1,12 @@ -FROM python:3.8-slim-bookworm - -# Install apt dependencies -RUN apt-get update && \ - apt-get install -y git graphviz sqlite3 wget gpg lsb-release && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Install Vault -RUN wget -O- https://apt.releases.hashicorp.com/gpg | apt-key add - && \ - echo "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" > /etc/apt/sources.list.d/vault.list && \ - apt-get update && \ - apt-get install -y vault && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Prepare Vault ENVs https://github.com/InseeFrLab/helm-charts-interactive-services/blob/main/charts/library-chart/templates/_secret.tpl -ENV VAULT_ADDR="" -ENV VAULT_TOKEN="" -ENV VAULT_RELATIVE_PATH="" -ENV VAULT_TOP_DIR="" -ENV VAULT_MOUNT="" +FROM inseefrlab/onyxia-jupyter-python:py3.10.12 + +# Avoid interactive stuff +ENV DEBIAN_FRONTEND=noninteractive +RUN sudo ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime &&\ + sudo apt update -y -q &&\ + sudo apt install wget curl python3 python3-tk python3-dev graphviz git sqlite3 gpg lsb-release -y -q &&\ + sudo apt install build-essential libssl-dev libffi-dev -y -q &&\ + sudo rm -rf /var/lib/apt/lists/* # Install Autosubmit ARG AUTOSUBMIT_VERSION=4.0.105 @@ -40,10 +27,10 @@ RUN autosubmit configure &&\ # Copy entrypoint script COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh +RUN sudo chmod +x /entrypoint.sh COPY load_ssh_private_key.sh /load_ssh_private_key.sh -RUN chmod +x /load_ssh_private_key.sh +RUN sudo chmod +x /load_ssh_private_key.sh EXPOSE 8888 EXPOSE 8000 diff --git a/docker/demo/entrypoint.sh b/docker/demo/entrypoint.sh index 9585a20b8..1f1858451 100644 --- a/docker/demo/entrypoint.sh +++ b/docker/demo/entrypoint.sh @@ -5,9 +5,9 @@ # Run jupyter lab as daemon and assign token if env variable exists if [ -n "$JUPYTER_TOKEN" ]; then - jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --allow-root --NotebookApp.base_url=/jupyterlab --NotebookApp.token=$JUPYTER_TOKEN & + jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --NotebookApp.base_url=/jupyterlab --NotebookApp.token=$JUPYTER_TOKEN & else - jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --allow-root --NotebookApp.base_url=/jupyterlab & + jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --NotebookApp.base_url=/jupyterlab & fi # Run the command passed by docker run -- GitLab From 55bee0d9b5348337fc2d73e39164b0c01a4b3b6c Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Thu, 20 Jun 2024 15:02:53 +0200 Subject: [PATCH 11/11] Add Subversion to the image (cc @agayayav) --- docker/demo/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/demo/Dockerfile b/docker/demo/Dockerfile index 777e1ff29..fd998946d 100644 --- a/docker/demo/Dockerfile +++ b/docker/demo/Dockerfile @@ -4,7 +4,7 @@ FROM inseefrlab/onyxia-jupyter-python:py3.10.12 ENV DEBIAN_FRONTEND=noninteractive RUN sudo ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime &&\ sudo apt update -y -q &&\ - sudo apt install wget curl python3 python3-tk python3-dev graphviz git sqlite3 gpg lsb-release -y -q &&\ + sudo apt install wget curl python3 python3-tk python3-dev graphviz git subversion sqlite3 gpg lsb-release -y -q &&\ sudo apt install build-essential libssl-dev libffi-dev -y -q &&\ sudo rm -rf /var/lib/apt/lists/* -- GitLab