Dev 221 djlabhub chain breaker

chain-breaker/.gitignore

@@ -0,0 +1 @@
+.env

chain-breaker/README.md

@@ -0,0 +1,15 @@
+TODO - will make this a proper README.md later
+
+Pro
+- Break the chain of dependencies of self-managed base images
+  - Reduce image maintenance hell
+  - Use jupyter/minimal-notebook as base image instead
+- Compatible with previous supported feature as many as possible
+- Provide simpler way for mainteners to manage system dependencies and for users to install packages
+- Provide a way to config jupyter through environment variables
+- This can be used as jupyter lab, so we can archive djlab-docker and only focus on this repo
+- Add jupyterhub docker for single user image testing at local
+
+Con
+- Username if jovyan by default, consider this is a minor issue, since it's only visible in the terminal
+- Larger image size, this can be optimized when it becomes a problem

chain-breaker/hub/Dockerfile

@@ -0,0 +1,5 @@
+ARG JUPYTERHUB_VERSION
+FROM jupyterhub/jupyterhub:${JUPYTERHUB_VERSION}
+
+COPY ./config/jupyterhub_config.py /etc/jupyterhub/jupyterhub_config.py
+RUN pip install dockerspawner oauthenticator

chain-breaker/hub/config/jupyterhub_config.py

@@ -0,0 +1,94 @@
+import os
+import pwd
+from traitlets.config import Config
+
+c = Config() if "c" not in locals() else c
+
+# get the current user
+user = [u for u in pwd.getpwall() if u.pw_uid == os.getuid()][0]
+
+## Class for authenticating users.
+#
+#          This should be a subclass of :class:`jupyterhub.auth.Authenticator`
+#
+#          with an :meth:`authenticate` method that:
+#
+#          - is a coroutine (asyncio or tornado)
+#          - returns username on success, None on failure
+#          - takes two arguments: (handler, data),
+#            where `handler` is the calling web.RequestHandler,
+#            and `data` is the POST form data from the login page.
+#
+#          .. versionchanged:: 1.0
+#              authenticators may be registered via entry points,
+#              e.g. `c.JupyterHub.authenticator_class = 'pam'`
+#
+#  Currently installed:
+#    - default: jupyterhub.auth.PAMAuthenticator
+#    - dummy: jupyterhub.auth.DummyAuthenticator
+#    - null: jupyterhub.auth.NullAuthenticator
+#    - pam: jupyterhub.auth.PAMAuthenticator
+#  Default: 'jupyterhub.auth.PAMAuthenticator'
+c.JupyterHub.authenticator_class = "jupyterhub.auth.DummyAuthenticator"
+
+# ## TODO - callback_url needs to enable ssl
+# c.JupyterHub.authenticator_class = "oauthenticator.generic.GenericOAuthenticator"
+# c.GenericOAuthenticator.client_id = os.getenv("OAUTH2_CLIENT_ID")
+# c.GenericOAuthenticator.client_secret = os.getenv("OAUTH2_CLIENT_SECRET")
+# c.GenericOAuthenticator.oauth_callback_url = "https://127.0.0.1:8000/hub/oauth_callback"
+# c.GenericOAuthenticator.authorize_url = "https://keycloak-qa.datajoint.io/realms/datajoint/protocol/openid-connect/auth"
+# c.GenericOAuthenticator.token_url = "https://keycloak-qa.datajoint.io/realms/datajoint/protocol/openid-connect/token"
+# c.GenericOAuthenticator.userdata_url = "https://keycloak-qa.datajoint.io/realms/datajoint/protocol/openid-connect/userinfo"
+# c.GenericOAuthenticator.login_service = "Datajoint"
+# c.GenericOAuthenticator.username_claim = "preferred_username"
+# c.GenericOAuthenticator.enable_auth_state = True
+# c.GenericOAuthenticator.scope = ["openid"]
+# c.GenericOAuthenticator.claim_groups_key = "groups"
+# c.GenericOAuthenticator.admin_groups = ["datajoint"]
+
+## The class to use for spawning single-user servers.
+#
+#          Should be a subclass of :class:`jupyterhub.spawner.Spawner`.
+#
+#          .. versionchanged:: 1.0
+#              spawners may be registered via entry points,
+#              e.g. `c.JupyterHub.spawner_class = 'localprocess'`
+#
+#  Currently installed:
+#    - default: jupyterhub.spawner.LocalProcessSpawner
+#    - localprocess: jupyterhub.spawner.LocalProcessSpawner
+#    - simple: jupyterhub.spawner.SimpleLocalProcessSpawner
+#  Default: 'jupyterhub.spawner.LocalProcessSpawner'
+c.JupyterHub.spawner_class = "dockerspawner.DockerSpawner"
+
+## The ip address for the Hub process to *bind* to.
+#
+#          By default, the hub listens on localhost only. This address must be accessible from
+#          the proxy and user servers. You may need to set this to a public ip or '' for all
+#          interfaces if the proxy or user servers are in containers or on a different host.
+#
+#          See `hub_connect_ip` for cases where the bind and connect address should differ,
+#          or `hub_bind_url` for setting the full bind URL.
+#  Default: '127.0.0.1'
+c.JupyterHub.hub_ip = ""
+
+c.DockerSpawner.network_name = os.getenv("DOCKER_NETWORK_NAME", "jupyterhub_network")
+
+c.DockerSpawner.container_image = "datajoint/djlabhub:singleuser-4.0.2-py3.10-qa"
+
+c.DockerSpawner.environment = {
+    ## Jupyter Official Environment Variables
+    "DOCKER_STACKS_JUPYTER_CMD": "lab",
+    ## Extended by Datajoint
+    ## Before Start Hook
+    "DJLABHUB_REPO": "https://github.com/datajoint/datajoint-tutorials.git",
+    "DJLABHUB_REPO_BRANCH": "main",
+    "DJLABHUB_REPO_INSTALL": "TRUE",
+    ## Jupyter Config
+    # "JUPYTER_SERVER_APP_IP": "0.0.0.0",
+    # "JUPYTER_SERVER_APP_PASSWORD": "",
+    # "JUPYTER_SERVER_APP_PORT": "8889",
+    # "JUPYTER_SERVER_APP_ROOT_DIR": "/home/jovyan",
+    "JUPYTER_FILE_CONTENTS_MANAGER_ROOT_DIR": "/home/jovyan",
+    "JUPYTER_YDOCEXTENSION_DISABLE_RTC": "TRUE",
+}

chain-breaker/hub/docker-compose.yaml

@@ -0,0 +1,29 @@
+version: '3.7'
+services:
+  hub:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        - JUPYTERHUB_VERSION
+    image: datajoint/djlabhub:hub-${JUPYTERHUB_VERSION}
+    container_name: djlabhub-hub
+    user: root
+    env_file: .env
+    # network to connect the hub and singleusers
+    networks:
+      - jupyterhub_network
+    environment:
+      - DOCKER_NETWORK_NAME=jupyterhub_network
+    command: jupyterhub -f /etc/jupyterhub/jupyterhub_config.py
+    ports:
+      - 8000:8000
+    volumes:
+      - ./config/jupyterhub_config.py:/etc/jupyterhub/jupyterhub_config.py
+      - /var/run/docker.sock:/var/run/docker.sock
+
+networks:
+  jupyterhub_network:
+    name: jupyterhub_network
+    driver: bridge
+

chain-breaker/hub/example.env

@@ -0,0 +1,2 @@
+# build
+JUPYTERHUB_VERSION=4.0.2

chain-breaker/singleuser/Dockerfile

@@ -0,0 +1,25 @@
+ARG JUPYTERHUB_VERSION
+FROM quay.io/jupyter/minimal-notebook:hub-${JUPYTERHUB_VERSION}
+
+COPY ./config /tmp/config
+
+USER root
+RUN \
+    # Install dependencies: apt
+    bash /tmp/config/apt_install.sh \
+    # Add startup hook
+    && cp /tmp/config/before_start_hook.sh /usr/local/bin/before-notebook.d/ \
+    && chmod +x /usr/local/bin/before-notebook.d/before_start_hook.sh \
+    # Add jupyter*config*.py
+    && cp /tmp/config/jupyter*config*.py /etc/jupyter/
+
+USER ${NB_UID}
+ARG PYTHON_VERSION
+RUN \
+    # remove default work directory
+    [ -d "/home/jovyan/work" ] && rm -r /home/jovyan/work \
+    # Install dependencies: pip and conda
+    && conda install -y -n base python=${PYTHON_VERSION} \
+    && pip install -r /tmp/config/pip_requirements.txt \
+    && conda install --yes --file /tmp/config/conda_requirements.txt \
+    && conda clean --all -f -y 

chain-breaker/singleuser/config/apt_install.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# Add any other apt repos here
+sudo apt update
+
+# Install
+sudo apt-get install mysql-client -y --no-install-recommends
+sudo apt-get clean
+rm -rf /var/lib/apt/lists/*
+
+# Other installation
+wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq
+chmod +x /usr/bin/yq

chain-breaker/singleuser/config/before_start_hook.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+echo "INFO::Datajoint Startup Hook"
+
+# Changing Markdown Preview to preview as default
+echo "INFO::Changing Markdown Preview to preview as default"
+yq '.properties.defaultViewers.default = {"markdown":"Markdown Preview"}' \
+  /opt/conda/share/jupyter/lab/schemas/@jupyterlab/docmanager-extension/plugin.json -o json -i
+
+# clone and install DJLABHUB_REPO or DJLABHUB_REPO_SUBPATH
+# for private repo, include PAT(Personal Access Token) in the https url
+if [[ ! -z "${DJLABHUB_REPO}" ]]; then
+  REPO_NAME=$(basename $DJLABHUB_REPO | sed 's/.git//')
+
+  echo "INFO::Cloning repo $DJLABHUB_REPO"
+  git clone $DJLABHUB_REPO $HOME/$REPO_NAME
+  if [[ ! -z "${DJLABHUB_REPO_BRANCH}" ]]; then
+    echo "INFO::Switch to branch $DJLABHUB_REPO_BRANCH"
+    git -C $HOME/$REPO_NAME switch $DJLABHUB_REPO_BRANCH
+  fi
+
+  if [[ $DJLABHUB_REPO_INSTALL == "TRUE" ]]; then
+    echo "INFO::Installing repo"
+    pip install -e $HOME/$REPO_NAME
+  fi
+fi

chain-breaker/singleuser/config/conda_requirements.txt

@@ -0,0 +1,2 @@
+gh
+jupyter-collaboration

chain-breaker/singleuser/config/jupyter_jupyterlab_server_config.py

@@ -0,0 +1,25 @@
+import os
+import pwd
+
+# Configuration file for lab. https://jupyterlab-server.readthedocs.io/en/latest/api/app-config.html
+
+# c = get_config()  # noqa
+from traitlets.config import Config
+
+c = Config() if "c" not in locals() else c
+
+# get the current user
+user = [u for u in pwd.getpwall() if u.pw_uid == os.getuid()][0]
+
+## The default URL to redirect to from `/`
+#  Default: '/lab'
+jupyter_lab_default_url = os.getenv("JUPYTER_LAB_APP_DEFAULT_URL")
+c.LabApp.default_url = (
+    "/lab/tree{}".format(
+        jupyter_lab_default_url.replace(
+            os.getenv("JUPYTER_FILE_CONTENTS_MANAGER_ROOT_DIR", "/home/jovyan"), ""
+        )
+    )
+    if jupyter_lab_default_url
+    else "/lab"
+)

chain-breaker/singleuser/config/jupyter_jupyternotebook_server_config.py

@@ -0,0 +1,25 @@
+import os
+import pwd
+
+# Configuration file for notebook. https://jupyter-notebook.readthedocs.io/en/5.7.4/config.html
+
+# c = get_config()  # noqa
+from traitlets.config import Config
+
+c = Config() if "c" not in locals() else c
+
+# get the current user
+user = [u for u in pwd.getpwall() if u.pw_uid == os.getuid()][0]
+
+## The default URL to redirect to from `/`
+#  Default: '/lab'
+jupyter_lab_default_url = os.getenv("JUPYTER_LAB_APP_DEFAULT_URL")
+c.LabApp.default_url = (
+    "/lab/tree{}".format(
+        jupyter_lab_default_url.replace(
+            os.getenv("JUPYTER_FILE_CONTENTS_MANAGER_ROOT_DIR", "/home/jovyan"), ""
+        )
+    )
+    if jupyter_lab_default_url
+    else "/lab"
+)

chain-breaker/singleuser/config/jupyter_server_config.py

@@ -0,0 +1,94 @@
+import os
+import pwd
+import hashlib
+import random
+import json
+
+# Configuration file for jupyter-server. https://jupyter-server.readthedocs.io/en/latest/other/full-config.html#other-full-config
+
+# c = get_config()  # noqa
+from traitlets.config import Config
+
+c = Config() if "c" not in locals() else c
+
+# get the current user
+user = [u for u in pwd.getpwall() if u.pw_uid == os.getuid()][0]
+
+## Whether to allow the user to run the server as root.
+#  Default: False
+c.ServerApp.allow_root = False
+
+## The IP address the Jupyter server will listen on.
+#  Default: 'localhost'
+c.ServerApp.ip = os.getenv("JUPYTER_SERVER_APP_IP", "0.0.0.0")
+
+
+## DEPRECATED in 2.0. Use PasswordIdentityProvider.hashed_password
+#  Default: ''
+# c.ServerApp.password = ''
+def passwd(passphrase: str):
+    salt_len = 12
+    h = hashlib.new("sha256")
+    salt = ("%0" + str(salt_len) + "x") % random.getrandbits(4 * salt_len)
+    h.update(passphrase.encode("utf-8") + salt.encode("ascii"))
+    return ":".join(("sha256", salt, h.hexdigest()))
+
+
+jupyter_server_password = os.getenv("JUPYTER_SERVER_APP_PASSWORD", "datajoint")
+c.PasswordIdentityProvider.hashed_password = (
+    passwd(jupyter_server_password) if jupyter_server_password else ""
+)
+
+## The port the server will listen on (env: JUPYTER_PORT).
+#  Default: 0
+c.ServerApp.port = int(os.getenv("JUPYTER_SERVER_APP_PORT", 8888))
+
+## The directory to use for notebooks and kernels.
+#  Default: ''
+c.ServerApp.root_dir = os.getenv("JUPYTER_SERVER_APP_ROOT_DIR", user.pw_dir)
+
+## Supply overrides for terminado. Currently only supports "shell_command".
+#  Default: {}
+c.ServerApp.terminado_settings = json.loads(
+    os.getenv(
+        "JUPYTER_SERVER_APP_TERMINADO_SETTINGS",
+        f'{{"shell_command": ["{user.pw_shell}"]}}',
+    )
+)
+
+
+## Python callable or importstring thereof
+#  See also: ContentsManager.pre_save_hook
+# c.FileContentsManager.pre_save_hook = None
+def scrub_output_pre_save(model, **kwargs):
+    """scrub output before saving notebooks"""
+    if not os.getenv("JUPYTER_FILE_CONTENTS_MANAGER_SAVE_OUTPUT", "FALSE") == "TRUE":
+        # only run on notebooks
+        if model["type"] != "notebook":
+            return
+        # only run on nbformat v4
+        if model["content"]["nbformat"] != 4:
+            return
+
+        model["content"]["metadata"].pop("signature", None)
+        for cell in model["content"]["cells"]:
+            if cell["cell_type"] != "code":
+                continue
+            cell["outputs"] = []
+            cell["execution_count"] = None
+    else:
+        return
+
+
+c.FileContentsManager.pre_save_hook = scrub_output_pre_save
+
+
+#  Default: ''
+c.FileContentsManager.root_dir = os.getenv(
+    "JUPYTER_FILE_CONTENTS_MANAGER_ROOT_DIR", "/home/jovyan"
+)
+
+## Jupyter collaboration extension
+c.YDocExtension.disable_rtc = (
+    os.getenv("JUPYTER_YDOCEXTENSION_DISABLE_RTC", "FALSE").upper() == "TRUE"
+)

chain-breaker/singleuser/docker-compose.yaml

@@ -0,0 +1,21 @@
+version: '3.7'
+services:
+  singleuser:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        - PYTHON_VERSION
+        - JUPYTERHUB_VERSION
+    image: datajoint/djlabhub:singleuser-${JUPYTERHUB_VERSION}-py${PYTHON_VERSION}
+    container_name: djlabhub-singleuser
+    env_file: .env
+    ports:
+      - $JUPYTER_SERVER_APP_PORT:$JUPYTER_SERVER_APP_PORT
+    volumes:
+      - ./config/before_start_hook.sh:/usr/local/bin/before-notebook.d/before_start_hook.sh
+      - ./config/jupyter_server_config.py:/etc/jupyter/jupyter_server_config.py
+      - ./config/jupyter_jupyterlab_server_config.py:/etc/jupyter/jupyter_jupyterlab_server_config.py
+  djlab:
+    extends: singleuser
+    image: datajoint/djlab:py${PYTHON_VERSION}