diff --git a/dockerfiles/Dockerfile b/docker/Dockerfile similarity index 82% rename from dockerfiles/Dockerfile rename to docker/Dockerfile index 3de4a04a7f6a9558c7e67f1f9ef1bb820d0dab1b..31802741187f9a39e808ee5e3b74384daf0b1277 100644 --- a/dockerfiles/Dockerfile +++ b/docker/Dockerfile @@ -9,7 +9,7 @@ FROM mambaorg/micromamba:1.4.9-bullseye-slim AS micromamba # image autosubmit-base, for example, with the users, permissions, # dependencies, and SSH. Which would tremendously reduce this. -FROM debian:bullseye-slim +FROM debian:bookworm-slim ARG AUTOSUBMIT_ROOT_DIR=/app/autosubmit/ @@ -95,20 +95,21 @@ RUN mkdir -pv "${AUTOSUBMIT_ROOT_DIR}/logs" && \ # TODO: add something like xpdf=3.04+git20210103-3 if GUI/X is needed. RUN apt update && \ apt install -y \ - bash=5.1-2+deb11u1 \ - ca-certificates=20210119 \ - curl=7.74.0-1.3+deb11u7 \ + gcc \ + bash \ + ca-certificates \ + curl \ desktop-file-utils=0.26-1 \ - dialog=1.3-20201126-1 \ - graphviz=2.42.2-5 \ - iputils-ping=3:20210202-1 \ - less=551-2 \ - net-tools=1.60+git20181103.0eebece-1 \ - openssh-server=1:8.4p1-5+deb11u1 \ - python3-tk=3.9.2-1 \ - sqlite3=3.34.1-3 \ - sudo=1.9.5p2-3+deb11u1 \ - vim=2:8.2.2434-3+deb11u1 \ + dialog \ + graphviz \ + iputils-ping \ + less \ + net-tools \ + openssh-server \ + python3-tk \ + sqlite3 \ + sudo \ + vim \ xdg-utils=1.1.3-4.1 && \ update-ca-certificates && \ apt-get clean && \ @@ -125,15 +126,32 @@ RUN micromamba install --yes --name base --channel conda-forge \ micromamba clean --all --yes && \ /usr/local/bin/_activate_current_env.sh -# Install Autosubmit. -RUN pip install autosubmit==4.0.84 +# Install Autosubmit. +# If GIT_REF is set, it will install from the specified branch/tag/commit. +# Otherwise, if AUTOSUBMIT_VERSION is set, it will install the specified version from pypi. +# Otherwise, it will install the latest version from pypi. +ARG GIT_REF +ARG AUTOSUBMIT_VERSION +RUN if [ -n "${GIT_REF}" ] ; then \ + pip install git+https://earth.bsc.es/gitlab/es/autosubmit.git@${GIT_REF}; \ + elif [ -n "${AUTOSUBMIT_VERSION}" ] ; then \ + pip install autosubmit==${AUTOSUBMIT_VERSION}; \ + else \ + pip install autosubmit; \ + fi # Configure Autosubmit. RUN autosubmit configure \ -db "${AUTOSUBMIT_ROOT_DIR}/database/" \ -dbf autosubmit.db \ - -lr "${AUTOSUBMIT_ROOT_DIR}/experiments/" && \ - autosubmit install + -lr "${AUTOSUBMIT_ROOT_DIR}/experiments/" + +# Copy the default config file /home/${MAMBA_USER}/.autosubmitrc to ${AUTOSUBMIT_ROOT_DIR}/autosubmitrc +RUN cp "/home/${MAMBA_USER}/.autosubmitrc" "${AUTOSUBMIT_ROOT_DIR}/autosubmitrc" && \ + chown -R "${MAMBA_USER}:${MAMBA_USER}" "${AUTOSUBMIT_ROOT_DIR}" + +# Set AUTOSUBMIT_CONFIGURATION to the path of the Autosubmit configuration file. +ENV AUTOSUBMIT_CONFIGURATION="${AUTOSUBMIT_ROOT_DIR}/autosubmitrc" # SSH (for Autosubmit local platform.) USER root diff --git a/dockerfiles/README.md b/docker/README.md similarity index 100% rename from dockerfiles/README.md rename to docker/README.md diff --git a/docker/demo/Dockerfile b/docker/demo/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..fd998946db7593ae7324f821529bc2f43e366e9f --- /dev/null +++ b/docker/demo/Dockerfile @@ -0,0 +1,39 @@ +FROM inseefrlab/onyxia-jupyter-python:py3.10.12 + +# Avoid interactive stuff +ENV DEBIAN_FRONTEND=noninteractive +RUN sudo ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime &&\ + sudo apt update -y -q &&\ + sudo apt install wget curl python3 python3-tk python3-dev graphviz git subversion sqlite3 gpg lsb-release -y -q &&\ + sudo apt install build-essential libssl-dev libffi-dev -y -q &&\ + sudo rm -rf /var/lib/apt/lists/* + +# Install Autosubmit +ARG AUTOSUBMIT_VERSION=4.0.105 +ARG AUTOSUBMIT_API_VERSION=4.0.0b8 + +RUN pip3 install \ + autosubmit==${AUTOSUBMIT_VERSION} \ + autosubmit-api==${AUTOSUBMIT_API_VERSION} \ + jupyterlab + +# Set the environment variables +ENV PROTECTION_LEVEL=NONE +ENV JUPYTER_TOKEN="" + +# Install Autosubmit +RUN autosubmit configure &&\ + autosubmit install + +# Copy entrypoint script +COPY entrypoint.sh /entrypoint.sh +RUN sudo chmod +x /entrypoint.sh + +COPY load_ssh_private_key.sh /load_ssh_private_key.sh +RUN sudo chmod +x /load_ssh_private_key.sh + +EXPOSE 8888 +EXPOSE 8000 + +# Set the entrypoint +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/demo/README.md b/docker/demo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b86253b06818a450defa074baa2786d3adb6133c --- /dev/null +++ b/docker/demo/README.md @@ -0,0 +1,55 @@ + +## Prepare Minikube + +### Build the docker images inside minikube + +Build the images inside Minikube + +```bash +eval $(minikube docker-env) + +docker build -t as-demo:latest ~/projects/autosubmit/docker/demo +docker build --build-arg "AUTOSUBMIT_API_SOURCE=/api" --build-arg="PUBLIC_URL=/gui"\ + -t as-gui-demo:latest ~/projects/autosubmitreact-update/docker +``` + +The script above use the default names of the images. You can change and set them in the `values.yaml` as you wish. + +### Install Nginx Ingress Controller + +Install nginx Ingress Controller https://kubernetes.github.io/ingress-nginx/deploy/ + +```bash +helm upgrade --install ingress-nginx ingress-nginx \ + --repo https://kubernetes.github.io/ingress-nginx \ + --namespace ingress-nginx --create-namespace +``` + +Enable Ingress for testing + +```bash +minikube addons enable ingress +``` + +## Install using helm + +Install helm + +```bash +helm install test-demo . +``` + + +Clean up helm + +```bash +helm uninstall test-demo +``` + +## Test locally + +Forward nginx ingress controller port `80` to `localhost:8080`: + +```bash +kubectl port-forward --namespace=ingress-nginx service/ingress-nginx-controller 8080:80 +``` \ No newline at end of file diff --git a/docker/demo/entrypoint.sh b/docker/demo/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..1f185845110ccf895d9c5b04e8688423ec7895ee --- /dev/null +++ b/docker/demo/entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Execute /load_ssh_private_key.sh +/load_ssh_private_key.sh + +# Run jupyter lab as daemon and assign token if env variable exists +if [ -n "$JUPYTER_TOKEN" ]; then + jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --NotebookApp.base_url=/jupyterlab --NotebookApp.token=$JUPYTER_TOKEN & +else + jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --NotebookApp.base_url=/jupyterlab & +fi + +# Run the command passed by docker run +autosubmit_api start -b 0.0.0.0:8000 diff --git a/docker/demo/helm/.helmignore b/docker/demo/helm/.helmignore new file mode 100644 index 0000000000000000000000000000000000000000..0e8a0eb36f4ca2c939201c0d54b5d82a1ea34778 --- /dev/null +++ b/docker/demo/helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/docker/demo/helm/Chart.yaml b/docker/demo/helm/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7875694c6f4f944a463c5fb4f3ef5ba29d3580c2 --- /dev/null +++ b/docker/demo/helm/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: autosubmit-demo +description: A Helm chart for trying Autosubmit tools + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/docker/demo/helm/templates/deployment.yaml b/docker/demo/helm/templates/deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2482ccd9f89d0f08d74239022bc94b4be56f789b --- /dev/null +++ b/docker/demo/helm/templates/deployment.yaml @@ -0,0 +1,32 @@ +# Add deployment and service +apiVersion: apps/v1 +kind: Deployment +metadata: + name: autosubmit-deployment + labels: + app: autosubmit-demo +spec: + replicas: 1 + selector: + matchLabels: + app: autosubmit-demo + template: + metadata: + labels: + app: autosubmit-demo + spec: + containers: + # Main Autosubmit docker image with exposed ports 8888 and 8000 + - name: autosubmit-container + image: {{ .Values.images.autosubmitDemo.image }} + imagePullPolicy: {{ .Values.images.autosubmitDemo.imagePullPolicy }} + ports: + - containerPort: 8888 + - containerPort: 8000 + # GUI docker image with exposed port 3000 + - name: autosubmit-gui-container + image: {{ .Values.images.autosubmitDemoGUI.image }} + imagePullPolicy: {{ .Values.images.autosubmitDemoGUI.imagePullPolicy }} + ports: + - containerPort: 8080 + diff --git a/docker/demo/helm/templates/ingress.yaml b/docker/demo/helm/templates/ingress.yaml new file mode 100644 index 0000000000000000000000000000000000000000..733096debb94ce7b6691a2217dfa2b715ce8afa9 --- /dev/null +++ b/docker/demo/helm/templates/ingress.yaml @@ -0,0 +1,48 @@ +# Rewrite ingress +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: autosubmit-ingress + annotations: + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/rewrite-target: /$2 +spec: + ingressClassName: nginx + rules: + - host: {{ .Values.ingress.host }} + http: + paths: + - pathType: ImplementationSpecific + path: "/api(/|$)(.*)" + backend: + service: + name: autosubmit-demo-service + port: + number: 8000 +--- +# Prefix ingress +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: autosubmit-ingress-gui +spec: + ingressClassName: nginx + rules: + - host: {{ .Values.ingress.host }} + http: + paths: + - pathType: Prefix + path: "/jupyterlab" + backend: + service: + name: autosubmit-demo-service + port: + number: 8888 + - pathType: Prefix + path: "/gui" + backend: + service: + name: autosubmit-demo-service + port: + number: 8080 + \ No newline at end of file diff --git a/docker/demo/helm/templates/service.yaml b/docker/demo/helm/templates/service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1432a0d3278de5d4ec4491e034cf7988988504e9 --- /dev/null +++ b/docker/demo/helm/templates/service.yaml @@ -0,0 +1,23 @@ +# Add service +apiVersion: v1 +kind: Service +metadata: + name: autosubmit-demo-service + labels: + app: autosubmit-demo +spec: + selector: + app: autosubmit-demo + ports: + - name: jupyterlab-port + protocol: TCP + port: 8888 + targetPort: 8888 + - name: api-port + protocol: TCP + port: 8000 + targetPort: 8000 + - name: gui-port + protocol: TCP + port: 8080 + targetPort: 8080 diff --git a/docker/demo/helm/values.yaml b/docker/demo/helm/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97ca8b1059b6e0b9e4794df4449fcffd9fa2dced --- /dev/null +++ b/docker/demo/helm/values.yaml @@ -0,0 +1,10 @@ +images: + autosubmitDemo: + image: autosubmit/edito-demo:as-4.0.105-api-4.0.0b8 + imagePullPolicy: IfNotPresent + autosubmitDemoGUI: + image: autosubmit/gui:edito-demo + imagePullPolicy: IfNotPresent + +ingress: + host: localhost \ No newline at end of file diff --git a/docker/demo/load_ssh_private_key.sh b/docker/demo/load_ssh_private_key.sh new file mode 100644 index 0000000000000000000000000000000000000000..bbe62c36c5a2ff2a037a6d3726e4c01fa6a5b2ea --- /dev/null +++ b/docker/demo/load_ssh_private_key.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +SSH_PATH=$HOME/.ssh +mkdir -p $SSH_PATH + +# If VAULT_ADDR is not set, then we are not running in a container +if [ -z "${VAULT_ADDR}" ] +then + # VAULT_ADDR is not set + ssh-keygen -b 4096 -t rsa -f "${SSH_PATH}/id_rsa" -q -N "" + SSH_PRIVATE_KEY=$(base64 "${SSH_PATH}/id_rsa" -w 0) + SSH_PUBLIC_KEY=$(base64 "${SSH_PATH}/id_rsa.pub" -w 0) + +else + # Original script, assumes VAULT_ADDR is set + echo "begin script" + whoami + set -euo pipefail + + SSH_PRIVATE_KEY=$(vault kv get -field=SSH_PRIVATE_KEY "${VAULT_MOUNT}/${VAULT_TOP_DIR}/autosubmit" || echo "") + if [ -z "$SSH_PRIVATE_KEY" ] + then + ssh-keygen -b 4096 -t rsa -f "${SSH_PATH}/id_rsa" -q -N "" + SSH_PRIVATE_KEY=$(base64 "${SSH_PATH}/id_rsa" -w 0) + SSH_PUBLIC_KEY=$(base64 "${SSH_PATH}/id_rsa.pub" -w 0) + vault kv put "${VAULT_MOUNT}/${VAULT_TOP_DIR}/autosubmit" SSH_PRIVATE_KEY="${SSH_PRIVATE_KEY}" SSH_PUBLIC_KEY="${SSH_PUBLIC_KEY}" + else + SSH_PUBLIC_KEY=$(vault kv get -field=SSH_PUBLIC_KEY "${VAULT_MOUNT}/${VAULT_TOP_DIR}/autosubmit" || echo "") + echo $SSH_PRIVATE_KEY | base64 -d > "${SSH_PATH}/id_rsa" + echo $SSH_PUBLIC_KEY | base64 -d > "${SSH_PATH}/id_rsa.pub" + fi +fi + +chmod 600 "${SSH_PATH}/id_rsa" \ No newline at end of file diff --git a/dockerfiles/docker-compose.yml b/docker/docker-compose.yml similarity index 100% rename from dockerfiles/docker-compose.yml rename to docker/docker-compose.yml diff --git a/dockerfiles/test.sh b/docker/test.sh similarity index 100% rename from dockerfiles/test.sh rename to docker/test.sh