From e8fa194d4d2eb4d23f817cad82b46764c40bc7cd Mon Sep 17 00:00:00 2001 From: andersdberg Date: Mon, 8 Aug 2022 08:24:47 -0700 Subject: [PATCH 01/57] Add datascience notebook and github actions --- .github/workflows/main.yaml | 115 +++++++++++++++++ kernels/datascience-notebook/.pythonrc | 3 + kernels/datascience-notebook/Dockerfile | 116 ++++++++++++++++++ kernels/datascience-notebook/README.md | 20 +++ kernels/datascience-notebook/environment.txt | 7 ++ .../datascience-notebook/ipython_config.py | 12 ++ kernels/datascience-notebook/requirements.txt | 12 ++ kernels/datascience-notebook/run.sh | 50 ++++++++ .../datascience-notebook/secrets_helper.py | 34 +++++ 9 files changed, 369 insertions(+) create mode 100644 .github/workflows/main.yaml create mode 100644 kernels/datascience-notebook/.pythonrc create mode 100644 kernels/datascience-notebook/Dockerfile create mode 100644 kernels/datascience-notebook/README.md create mode 100644 kernels/datascience-notebook/environment.txt create mode 100644 kernels/datascience-notebook/ipython_config.py create mode 100644 kernels/datascience-notebook/requirements.txt create mode 100755 kernels/datascience-notebook/run.sh create mode 100644 kernels/datascience-notebook/secrets_helper.py diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 00000000..31d893ed --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,115 @@ +name: Build kernel images + +on: + push: + branches: + - main + +jobs: + + build-python-images: + runs-on: ubuntu-latest + strategy: + matrix: + # version: ["3.8.8", "3.9.13", "3.10.5"] 3.10.5 fails with dependency conflicts + # We may need to have separate requirements.txt for each version, or replace + # dependencies on the fly + version: ["3.8.8", "3.9.13"] + directory: ["datascience-notebook"] + # The datascience-notebook base image does not support ARM + # We would need to build and maintain our own base image + # architecture: ["arm", "amd"] + steps: + - name: Checkout code + uses: actions/checkout@v2 + + # TODO: Log into DockerHub to prevent rate limiting + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Create context + run: | + docker context create github-action + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + endpoint: github-action + version: v0.8.2 + env: + DOCKER_CONTEXT: github-action + + - name: Cache Docker layers + uses: actions/cache@v2 + id: docker-cache + with: + path: "/tmp/.buildx-cache" + key: "${{ runner.os }}-buildx-${{ matrix.directory }}-${{ matrix.version }}" + restore-keys: "${{ runner.os }}-buildx-" + + - name: Build arguments + id: build-args + run: | + # Image Name + container_registry=ghcr.io/${{ github.repository_owner }} + image_name=kernel-${{ matrix.directory }} + full_image_name="${container_registry}/${image_name}" + + # Image Tags + image_sha_tag="${GITHUB_SHA:0:12}" # first 12 numbers of the SHA + image_version_tag="python-$(version=${{ matrix.version }} && echo ${version%.*} )" # removes patch version + + full_image_name_tagged='' + + if [ "${GITHUB_EVENT_NAME}" = 'push' ]; then + full_image_name_tagged="${full_image_name}:${image_version_tag}" + elif [ "${GITHUB_EVENT_NAME}" = 'pull_request' ]; then + full_image_name_tagged="${full_image_name}:${image_version_tag}-${image_sha_tag}" + fi + + echo "::set-output name=FULL_IMAGE_NAME::${full_image_name}" + echo "::set-output name=FULL_IMAGE_NAME_TAGGED::${full_image_name_tagged}" + + echo "::set-output name=BUILD_URL::https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + echo "::set-output name=BUILD_TIMESTAMP::$(date --utc --iso-8601=seconds)" + + echo "full_image_name: $full_image_name" + echo "image_version_tag: $image_version_tag" + echo "image_sha_tag: $image_sha_tag" + echo "full_image_name_tagged: $full_image_name_tagged" + + - name: Build image + env: + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTEXT: github-action + run: | + ( + cd ${GITHUB_WORKSPACE}/kernels/${{ matrix.directory }} + + docker buildx build \ + --pull \ + --output 'type=docker' \ + --platform=linux/arm64 \ + --progress plain \ + --cache-from 'type=local,src=/tmp/.buildx-cache' \ + --cache-to 'type=local,dest=/tmp/.buildx-cache' \ + --tag '${{ steps.build-args.outputs.FULL_IMAGE_NAME_TAGGED }}' \ + --build-arg PYTHON_VERSION=${{ matrix.version }} \ + --build-arg 'NBL_ARG_BUILD_TIMESTAMP=${{ steps.build-args.outputs.BUILD_TIMESTAMP }}' \ + --build-arg 'NBL_ARG_BUILD_URL=${{ steps.build-args.outputs.BUILD_URL }}' \ + --build-arg 'NBL_ARG_REVISION=${{ github.sha }}' \ + --build-arg 'NBL_ARG_VERSION=${{ github.ref }}' \ + . + ) + + - name: Publish image + run: | + docker push --all-tags ${{ steps.build-args.outputs.FULL_IMAGE_NAME }} diff --git a/kernels/datascience-notebook/.pythonrc b/kernels/datascience-notebook/.pythonrc new file mode 100644 index 00000000..0bcc2d9f --- /dev/null +++ b/kernels/datascience-notebook/.pythonrc @@ -0,0 +1,3 @@ +import pandas as pd + +import dx diff --git a/kernels/datascience-notebook/Dockerfile b/kernels/datascience-notebook/Dockerfile new file mode 100644 index 00000000..a1e60fa7 --- /dev/null +++ b/kernels/datascience-notebook/Dockerfile @@ -0,0 +1,116 @@ +# syntax = docker/dockerfile:1.4.1 +ARG BASE_IMAGE=jupyter/datascience-notebook +ARG PYTHON_VERSION=3.9.6 +# hadolint ignore=DL3006 +FROM ${BASE_IMAGE}:python-${PYTHON_VERSION} + +USER root + +# Set up log file for magics +RUN touch /var/log/noteable_magics.log && \ + chown 4004:4004 /var/log/noteable_magics.log + +# When image is run, run the code with the environment +# activated: +SHELL ["/bin/bash", "-c"] + +WORKDIR /tmp + +# hadolint ignore=DL3008,DL3015 +RUN apt-get update && \ + apt-get install -y jq procps git unixodbc-dev g++ \ + && rm -rf /var/lib/apt/lists/* + +ENV TINI_VERSION=v0.19.0 +RUN TINI_BINARY=$(if [ "$(uname -m)" = "aarch64" ]; then echo "tini-arm64"; else echo "tini"; fi); echo "${TINI_BINARY}" \ + && wget -q -O /usr/local/bin/tini "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/${TINI_BINARY}" \ + && chmod +x /usr/local/bin/tini + +ENV NB_USER="noteable" \ + NB_UID=4004 \ + NB_GID=4004 + +# Create the default unprivileged user +RUN groupadd --gid 4004 noteable && \ + useradd --uid 4004 --shell /bin/false --create-home --no-log-init --gid noteable noteable && \ + chown --recursive noteable:noteable /home/noteable + +RUN mkdir /etc/ipython && chown noteable:noteable /etc/ipython +RUN mkdir -p /etc/noteable && chown noteable:noteable /etc/noteable + +RUN chown noteable:noteable "${JULIA_PKGDIR}" && \ + chown noteable:noteable "${CONDA_DIR}" && \ + fix-permissions "${JULIA_PKGDIR}" && \ + fix-permissions "${CONDA_DIR}" + +# Run non-privileged user +USER noteable + +ENV PATH="/home/noteable/.local/bin:${PATH}" \ + HOME="/home/noteable" \ + XDG_CACHE_HOME="/home/noteable/.cache/" \ + GOOGLE_APPLICATION_CREDENTIALS="/vault/secrets/gcp-credentials" + +# hadolint ignore=DL3045 +COPY environment.txt ./ + +# hadolint ignore=DL3045 +COPY requirements.txt ./ + +# hadolint ignore=SC2034 +RUN conda install --file environment.txt + +# hadolint ignore=DL3045 +COPY requirements.txt ./ + +# hadolint ignore=SC1008,SC2155,DL3042,SC2102 +RUN pip install -I --quiet --no-cache-dir "git+https://github.com/noteable-io/noteable-notebook-magics.git@main" && \ + pip install -I --quiet --no-cache-dir -r requirements.txt + +# Copy over any python commands that need to run on startup +# that aren't covered by IPython extensions +COPY .pythonrc /home/noteable/.pythonrc + +# Enable the widgets nbextension +# hadolint ignore=SC1008 +RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension + +# Smoke test to ensure packages were installed properly +# hadolint ignore=SC1008 +RUN python -c "import noteable_magics" + +RUN git config --global user.name "Noteable Kernel" && \ + git config --global user.email "engineering@noteable.io" + +# https://ipython.readthedocs.io/en/stable/config/intro.html#systemwide-configuration +COPY ipython_config.py /etc/ipython + +# Set standard working directory for noteable project +WORKDIR /etc/noteable/project + +# Add the entrypoint script to the $PATH +COPY run.sh /usr/local/bin +COPY secrets_helper.py /tmp/secrets_helper.py + +EXPOSE 50001-50005 + +# Use tini to manage passing signals to the child kernel process +# -g will ensure signals are passed to the entire child process *group*, +# not just the immediate child process (bash) +# https://github.com/krallin/tini#process-group-killing +ENTRYPOINT ["tini", "-g", "--"] +CMD ["run.sh"] + +# Labels +ARG NBL_ARG_BUILD_TIMESTAMP="undefined" +ARG NBL_ARG_REVISION="undefined" +ARG NBL_ARG_PYTHON_VERSION="3.9.6" +ARG NBL_ARG_BUILD_URL="undefined" +ARG NBL_ARG_VERSION="undefined" +LABEL org.opencontainers.image.created="${NBL_ARG_BUILD_TIMESTAMP}" \ + org.opencontainers.image.revision="${NBL_ARG_REVISION}" \ + org.opencontainers.image.source="https://github.com/noteable-io/polymorph" \ + org.opencontainers.image.title="noteable-python-${NBL_ARG_PYTHON_VERSION}" \ + org.opencontainers.image.url="${NBL_ARG_BUILD_URL}" \ + org.opencontainers.image.vendor="Noteable" \ + org.opencontainers.image.version="${NBL_ARG_VERSION}" diff --git a/kernels/datascience-notebook/README.md b/kernels/datascience-notebook/README.md new file mode 100644 index 00000000..8ecf387b --- /dev/null +++ b/kernels/datascience-notebook/README.md @@ -0,0 +1,20 @@ +# Multitenant Python Image + +Entrypoint is used to implement signal-based interrupts, since `ipykernel` does not support message-based interupts. + +## Building Locally +You'll need to provide a git credential string located at `${HOME}/.git-credentials`: + +```shell +echo "${GITHUB_USER_NAME}:${GITHUB_PERSONAL_ACCESS_TOKEN}" > ${HOME}/.git-credentials +``` + +The [personal access token](https://github.com/settings/tokens) needs to have +the `read:packages, repo` scope (and make sure to enable SSO on it). + +```shell +# Optional step to help you auto-load your built docker container into minikube for use with Gate +eval $(minikube docker-env) + +DOCKER_BUILDKIT=1 docker build --secret "id=git-credentials,src=${HOME}/.git-credentials" -t local/noteable-python:latest . +``` diff --git a/kernels/datascience-notebook/environment.txt b/kernels/datascience-notebook/environment.txt new file mode 100644 index 00000000..b370b004 --- /dev/null +++ b/kernels/datascience-notebook/environment.txt @@ -0,0 +1,7 @@ +ipykernel=5.5.* +ipython=8.0.* +vdom=0.6 +papermill=2.2.* +ipywidgets=7.6.* +plotly=4.14.3 +geopandas=0.11.0 \ No newline at end of file diff --git a/kernels/datascience-notebook/ipython_config.py b/kernels/datascience-notebook/ipython_config.py new file mode 100644 index 00000000..91146934 --- /dev/null +++ b/kernels/datascience-notebook/ipython_config.py @@ -0,0 +1,12 @@ +c.InteractiveShellApp.extensions = [ + "sql", + "noteable_magics", +] + +c.SqlMagic.feedback = False +c.SqlMagic.autopandas = True +c.NTBLMagic.project_dir = "/etc/noteable/project" +c.NoteableDataLoaderMagic.return_head = False +c.IPythonKernel._execute_sleep=0.15 +# 10 minutes to support large files +c.NTBLMagic.planar_ally_default_timeout_seconds = 600 diff --git a/kernels/datascience-notebook/requirements.txt b/kernels/datascience-notebook/requirements.txt new file mode 100644 index 00000000..af44cd02 --- /dev/null +++ b/kernels/datascience-notebook/requirements.txt @@ -0,0 +1,12 @@ +dx==1.1.2 +# Datasources-related packages here on down, alphabetized please for easy cut/paste across files and repos. +google-cloud-bigquery-storage==2.6.3 +psycopg2-binary==2.9.3 +pyodbc==4.0.32 +redshift_connector==2.0.907 +snowflake_sqlalchemy==1.3.4 +sqlalchemy-bigquery==1.3.0 +sqlalchemy-databricks==0.2.0 +sqlalchemy-redshift==0.8.9 +trino[sqlalchemy]==0.313.0 +astroid==2.12.2 diff --git a/kernels/datascience-notebook/run.sh b/kernels/datascience-notebook/run.sh new file mode 100755 index 00000000..f5fe306b --- /dev/null +++ b/kernels/datascience-notebook/run.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -o pipefail +set -o nounset +set -o errexit + +echo "Local time: $(date)" + +set -x + +connection_file=/tmp/connection_file.json + +cp /etc/noteable/connections/connection_file.json ${connection_file} + +kernel_name=$(jq -r .kernel_name /tmp/connection_file.json) + +# Inject Secrets into environment (see script docstring for more info) +# set +x to avoid echoing the Secrets in plaintext to logs +set +x +echo "Injecting Secrets into environment, echoing is turned off" +eval "$(python /tmp/secrets_helper.py)" +echo "Done injecting Secrets, turning echoing back on" +set -x + +case $kernel_name in + + python | python3) + echo "Starting Python kernel" + # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONSTARTUP + export PYTHONSTARTUP=~/.pythonrc + python -m ipykernel_launcher -f ${connection_file} --debug + ;; + + ir) + echo "Starting R kernel" + R --slave -e "IRkernel::main()" --args ${connection_file} + ;; + + julia | julia-1.6) + echo "Starting Julia kernel" + # project path necessary to keep julia form using its defaults + julia -i --color=yes --project=/etc/noteable/project /opt/julia/packages/IJulia/e8kqU/src/kernel.jl ${connection_file} + ;; + + *) + echo "Unrecognized '$kernel_name' kernel, falling back to Python" + # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONSTARTUP + export PYTHONSTARTUP=~/.pythonrc + python -m ipykernel_launcher -f ${connection_file} --debug + ;; +esac diff --git a/kernels/datascience-notebook/secrets_helper.py b/kernels/datascience-notebook/secrets_helper.py new file mode 100644 index 00000000..e731babd --- /dev/null +++ b/kernels/datascience-notebook/secrets_helper.py @@ -0,0 +1,34 @@ +""" +This script helps inject Secrets into the Kernel environment. + +The Vault Agent will volume mount files into the Kernel container +at /vault/secrets. Noteable Secrets will be in .env suffix files. + +We want to parse all those files and export them as environment variables +in the bash script that kicks off the Kernel (ipykernel_launcher etc). + +Doing that scripting in bash is a pain, so we do it in Python here and +bash just does an `eval` on the output. + +Some defensive programming to highlight: + - Env vars in the output are all uppercased + - If an env var is already set, we don't overwrite it + - We use shlex to quote the output so bash eval does not cause nasty side effects +""" +import os +import pathlib +import shlex + +output = [] + +secrets_directory = os.environ.get("VAULT_SECRETS_PATH", "/vault/secrets") + +directory = pathlib.Path(secrets_directory) +if directory.exists(): + for file in directory.glob("*.env"): + name = file.stem.upper() + if name not in os.environ: + content = file.read_text() + output.append(f"export {name}={shlex.quote(content)}") + +print("\n".join(output)) From 4d1b83848196bcd6906a7b55345678a7b3b21b40 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Mon, 8 Aug 2022 08:26:07 -0700 Subject: [PATCH 02/57] add action on PR --- .github/workflows/main.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 31d893ed..bd026b18 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -5,6 +5,10 @@ on: branches: - main + pull: + branches: + - main + jobs: build-python-images: From f4796aa1dc41aaca79fd198f8cdf530a3e1c3b95 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Mon, 8 Aug 2022 08:26:37 -0700 Subject: [PATCH 03/57] add action on PR --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index bd026b18..9aa008fb 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -5,7 +5,7 @@ on: branches: - main - pull: + pull_request: branches: - main From 38cc6b76633054980d8bc72dfea826dfce08530a Mon Sep 17 00:00:00 2001 From: andersdberg <79153744+andersdberg@users.noreply.github.com> Date: Wed, 19 Oct 2022 10:20:55 -0700 Subject: [PATCH 04/57] Update .github/workflows/main.yaml Co-authored-by: Diego Rodriguez --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 9aa008fb..6fffe749 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -12,7 +12,7 @@ on: jobs: build-python-images: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: # version: ["3.8.8", "3.9.13", "3.10.5"] 3.10.5 fails with dependency conflicts From 1d8427b30ff0c7238e5e413414affbee31b8d9ed Mon Sep 17 00:00:00 2001 From: andersdberg <79153744+andersdberg@users.noreply.github.com> Date: Wed, 19 Oct 2022 10:21:02 -0700 Subject: [PATCH 05/57] Update .github/workflows/main.yaml Co-authored-by: Diego Rodriguez --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 6fffe749..9416f41a 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -44,7 +44,7 @@ jobs: docker context create github-action - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 with: endpoint: github-action version: v0.8.2 From 4486d829b2cb90105be4a28bf13f922bef69989e Mon Sep 17 00:00:00 2001 From: andersdberg <79153744+andersdberg@users.noreply.github.com> Date: Wed, 19 Oct 2022 10:21:51 -0700 Subject: [PATCH 06/57] Update .github/workflows/main.yaml Co-authored-by: Diego Rodriguez --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 9416f41a..14e4371c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -47,7 +47,7 @@ jobs: uses: docker/setup-buildx-action@v2 with: endpoint: github-action - version: v0.8.2 + version: v0.9.1 env: DOCKER_CONTEXT: github-action From ba915e72d20e2641ef30ffd783f5078e9e2ee380 Mon Sep 17 00:00:00 2001 From: andersdberg <79153744+andersdberg@users.noreply.github.com> Date: Wed, 19 Oct 2022 10:22:38 -0700 Subject: [PATCH 07/57] Update .github/workflows/main.yaml Co-authored-by: Diego Rodriguez --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 14e4371c..d23fb322 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -56,7 +56,7 @@ jobs: id: docker-cache with: path: "/tmp/.buildx-cache" - key: "${{ runner.os }}-buildx-${{ matrix.directory }}-${{ matrix.version }}" + key: "${{ runner.os }}-${{env.RUNNER_ARCH}}-buildx-${{ matrix.directory }}-${{ matrix.version }}" restore-keys: "${{ runner.os }}-buildx-" - name: Build arguments From db1e20dfb7e7a87f40b6b5be79d93e3886be79b1 Mon Sep 17 00:00:00 2001 From: andersdberg <79153744+andersdberg@users.noreply.github.com> Date: Wed, 19 Oct 2022 10:22:47 -0700 Subject: [PATCH 08/57] Update .github/workflows/main.yaml Co-authored-by: Diego Rodriguez --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index d23fb322..cf01e472 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -52,7 +52,7 @@ jobs: DOCKER_CONTEXT: github-action - name: Cache Docker layers - uses: actions/cache@v2 + uses: actions/cache@v3 id: docker-cache with: path: "/tmp/.buildx-cache" From 1ed41a94b485dfe7473fff6ead72be20ea36b180 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Wed, 19 Oct 2022 10:25:36 -0700 Subject: [PATCH 09/57] address comments (and test without qemu) --- .github/workflows/main.yaml | 6 +++--- kernels/datascience-notebook/Dockerfile | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index cf01e472..00712b97 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -18,7 +18,7 @@ jobs: # version: ["3.8.8", "3.9.13", "3.10.5"] 3.10.5 fails with dependency conflicts # We may need to have separate requirements.txt for each version, or replace # dependencies on the fly - version: ["3.8.8", "3.9.13"] + version: ["3.8.13", "3.9.13"] directory: ["datascience-notebook"] # The datascience-notebook base image does not support ARM # We would need to build and maintain our own base image @@ -36,8 +36,8 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + # - name: Set up QEMU + # uses: docker/setup-qemu-action@v2 - name: Create context run: | diff --git a/kernels/datascience-notebook/Dockerfile b/kernels/datascience-notebook/Dockerfile index a1e60fa7..49935645 100644 --- a/kernels/datascience-notebook/Dockerfile +++ b/kernels/datascience-notebook/Dockerfile @@ -1,6 +1,6 @@ # syntax = docker/dockerfile:1.4.1 ARG BASE_IMAGE=jupyter/datascience-notebook -ARG PYTHON_VERSION=3.9.6 +ARG PYTHON_VERSION=3.9.13 # hadolint ignore=DL3006 FROM ${BASE_IMAGE}:python-${PYTHON_VERSION} From b0dc6d9c3815b9e3a91bf6df7c008df3963a20f1 Mon Sep 17 00:00:00 2001 From: andersdberg <79153744+andersdberg@users.noreply.github.com> Date: Wed, 19 Oct 2022 10:26:57 -0700 Subject: [PATCH 10/57] Update .github/workflows/main.yaml Co-authored-by: Diego Rodriguez --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 00712b97..a738c982 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -57,7 +57,7 @@ jobs: with: path: "/tmp/.buildx-cache" key: "${{ runner.os }}-${{env.RUNNER_ARCH}}-buildx-${{ matrix.directory }}-${{ matrix.version }}" - restore-keys: "${{ runner.os }}-buildx-" + restore-keys: "${{ runner.os }}-${{env.RUNNER_ARCH}}-buildx-${{ matrix.directory }}-" - name: Build arguments id: build-args From 05d61149d37542de6e164bd98f48f1233d13ba21 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Wed, 2 Nov 2022 15:47:38 -0700 Subject: [PATCH 11/57] address comment suggestion --- .github/workflows/main.yaml | 192 ++++++++++++++++++++++-------------- 1 file changed, 116 insertions(+), 76 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index a738c982..dc6ba208 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -9,6 +9,10 @@ on: branches: - main +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + jobs: build-python-images: @@ -36,84 +40,120 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v2 - - - name: Create context - run: | - docker context create github-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - endpoint: github-action - version: v0.9.1 - env: - DOCKER_CONTEXT: github-action - - - name: Cache Docker layers - uses: actions/cache@v3 - id: docker-cache + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v3 with: - path: "/tmp/.buildx-cache" - key: "${{ runner.os }}-${{env.RUNNER_ARCH}}-buildx-${{ matrix.directory }}-${{ matrix.version }}" - restore-keys: "${{ runner.os }}-${{env.RUNNER_ARCH}}-buildx-${{ matrix.directory }}-" + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=latest,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }} + type=schedule + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + labels: | + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + + - name: Get current time + uses: josStorer/get-current-time@84e5c63cf4cc28dc797be7bb0bfc0171b8c468ce + id: current-time - - name: Build arguments - id: build-args - run: | - # Image Name - container_registry=ghcr.io/${{ github.repository_owner }} - image_name=kernel-${{ matrix.directory }} - full_image_name="${container_registry}/${image_name}" - - # Image Tags - image_sha_tag="${GITHUB_SHA:0:12}" # first 12 numbers of the SHA - image_version_tag="python-$(version=${{ matrix.version }} && echo ${version%.*} )" # removes patch version + # - name: Create context + # run: | + # docker context create github-action + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 + # with: + # endpoint: github-action + # version: v0.9.1 + # env: + # DOCKER_CONTEXT: github-action + + # - name: Cache Docker layers + # uses: actions/cache@v3 + # id: docker-cache + # with: + # path: "/tmp/.buildx-cache" + # key: "${{ runner.os }}-${{env.RUNNER_ARCH}}-buildx-${{ matrix.directory }}-${{ matrix.version }}" + # restore-keys: "${{ runner.os }}-${{env.RUNNER_ARCH}}-buildx-${{ matrix.directory }}-" + + # - name: Build arguments + # id: build-args + # run: | + # # Image Name + # container_registry=ghcr.io/${{ github.repository_owner }} + # image_name=kernel-${{ matrix.directory }} + # full_image_name="${container_registry}/${image_name}" + + # # Image Tags + # image_sha_tag="${GITHUB_SHA:0:12}" # first 12 numbers of the SHA + # image_version_tag="python-$(version=${{ matrix.version }} && echo ${version%.*} )" # removes patch version - full_image_name_tagged='' - - if [ "${GITHUB_EVENT_NAME}" = 'push' ]; then - full_image_name_tagged="${full_image_name}:${image_version_tag}" - elif [ "${GITHUB_EVENT_NAME}" = 'pull_request' ]; then - full_image_name_tagged="${full_image_name}:${image_version_tag}-${image_sha_tag}" - fi - - echo "::set-output name=FULL_IMAGE_NAME::${full_image_name}" - echo "::set-output name=FULL_IMAGE_NAME_TAGGED::${full_image_name_tagged}" - - echo "::set-output name=BUILD_URL::https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" - echo "::set-output name=BUILD_TIMESTAMP::$(date --utc --iso-8601=seconds)" - - echo "full_image_name: $full_image_name" - echo "image_version_tag: $image_version_tag" - echo "image_sha_tag: $image_sha_tag" - echo "full_image_name_tagged: $full_image_name_tagged" - - - name: Build image - env: - DOCKER_CONTENT_TRUST: 1 - DOCKER_CONTEXT: github-action - run: | - ( - cd ${GITHUB_WORKSPACE}/kernels/${{ matrix.directory }} + # full_image_name_tagged='' + + # if [ "${GITHUB_EVENT_NAME}" = 'push' ]; then + # full_image_name_tagged="${full_image_name}:${image_version_tag}" + # elif [ "${GITHUB_EVENT_NAME}" = 'pull_request' ]; then + # full_image_name_tagged="${full_image_name}:${image_version_tag}-${image_sha_tag}" + # fi + + # echo "::set-output name=FULL_IMAGE_NAME::${full_image_name}" + # echo "::set-output name=FULL_IMAGE_NAME_TAGGED::${full_image_name_tagged}" + + # echo "::set-output name=BUILD_URL::https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + # echo "::set-output name=BUILD_TIMESTAMP::$(date --utc --iso-8601=seconds)" + + # echo "full_image_name: $full_image_name" + # echo "image_version_tag: $image_version_tag" + # echo "image_sha_tag: $image_sha_tag" + # echo "full_image_name_tagged: $full_image_name_tagged" + + # - name: Build image + # env: + # DOCKER_CONTENT_TRUST: 1 + # DOCKER_CONTEXT: github-action + # run: | + # ( + # cd ${GITHUB_WORKSPACE}/kernels/${{ matrix.directory }} - docker buildx build \ - --pull \ - --output 'type=docker' \ - --platform=linux/arm64 \ - --progress plain \ - --cache-from 'type=local,src=/tmp/.buildx-cache' \ - --cache-to 'type=local,dest=/tmp/.buildx-cache' \ - --tag '${{ steps.build-args.outputs.FULL_IMAGE_NAME_TAGGED }}' \ - --build-arg PYTHON_VERSION=${{ matrix.version }} \ - --build-arg 'NBL_ARG_BUILD_TIMESTAMP=${{ steps.build-args.outputs.BUILD_TIMESTAMP }}' \ - --build-arg 'NBL_ARG_BUILD_URL=${{ steps.build-args.outputs.BUILD_URL }}' \ - --build-arg 'NBL_ARG_REVISION=${{ github.sha }}' \ - --build-arg 'NBL_ARG_VERSION=${{ github.ref }}' \ - . - ) - - - name: Publish image - run: | - docker push --all-tags ${{ steps.build-args.outputs.FULL_IMAGE_NAME }} + # docker buildx build \ + # --pull \ + # --output 'type=docker' \ + # --platform=linux/arm64 \ + # --progress plain \ + # --cache-from 'type=local,src=/tmp/.buildx-cache' \ + # --cache-to 'type=local,dest=/tmp/.buildx-cache' \ + # --tag '${{ steps.build-args.outputs.FULL_IMAGE_NAME_TAGGED }}' \ + # --build-arg PYTHON_VERSION=${{ matrix.version }} \ + # --build-arg 'NBL_ARG_BUILD_TIMESTAMP=${{ steps.build-args.outputs.BUILD_TIMESTAMP }}' \ + # --build-arg 'NBL_ARG_BUILD_URL=${{ steps.build-args.outputs.BUILD_URL }}' \ + # --build-arg 'NBL_ARG_REVISION=${{ github.sha }}' \ + # --build-arg 'NBL_ARG_VERSION=${{ github.ref }}' \ + # . + # ) + + # - name: Publish image + # run: | + # docker push --all-tags ${{ steps.build-args.outputs.FULL_IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + secrets: | + "expel_artifactory_connection_url_file=${{ secrets.EXPEL_ARTIFACTORY_CONNECTION_URL }}" + "git-credentials=${{ secrets.GIT_CREDENTIALS }}" + build-args: | + "NBL_ARG_BUILD_TIMESTAMP=${{ steps.current-time.outputs.formattedTime }}" + "NBL_ARG_REVISION=${{ github.sha }}" + "NBL_ARG_BUILD_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + "NBL_ARG_VERSION=${{ github.ref }}" \ No newline at end of file From 2dc89411f8c3379e19214f2c1caf53d916235292 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Wed, 2 Nov 2022 15:53:10 -0700 Subject: [PATCH 12/57] debug --- .github/workflows/main.yaml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index dc6ba208..e0eb544d 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -31,7 +31,11 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - # TODO: Log into DockerHub to prevent rate limiting + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + endpoint: github-action + version: v0.9.1 - name: Login to GitHub Container Registry uses: docker/login-action@v2 @@ -40,9 +44,6 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v3 @@ -65,14 +66,6 @@ jobs: # run: | # docker context create github-action - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v2 - # with: - # endpoint: github-action - # version: v0.9.1 - # env: - # DOCKER_CONTEXT: github-action - # - name: Cache Docker layers # uses: actions/cache@v3 # id: docker-cache From 5586d3d5396b984923c33f9db378679c6beea09a Mon Sep 17 00:00:00 2001 From: andersdberg Date: Wed, 2 Nov 2022 19:24:44 -0700 Subject: [PATCH 13/57] remove pinned version from buildx for testing --- .github/workflows/main.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index e0eb544d..301e8730 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -35,7 +35,6 @@ jobs: uses: docker/setup-buildx-action@v2 with: endpoint: github-action - version: v0.9.1 - name: Login to GitHub Container Registry uses: docker/login-action@v2 From 5e7c36113cd0823ab8504cf5f7c20e2d8794f2b2 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Wed, 2 Nov 2022 19:32:52 -0700 Subject: [PATCH 14/57] self-hosted runner --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 301e8730..0391c47d 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -16,7 +16,7 @@ env: jobs: build-python-images: - runs-on: ubuntu-22.04 + runs-on: kubernetes-organization-runner strategy: matrix: # version: ["3.8.8", "3.9.13", "3.10.5"] 3.10.5 fails with dependency conflicts From 334d6536195609a16f395aba68e9a30ea5d1142a Mon Sep 17 00:00:00 2001 From: andersdberg <79153744+andersdberg@users.noreply.github.com> Date: Wed, 2 Nov 2022 20:27:27 -0700 Subject: [PATCH 15/57] Update .github/workflows/main.yaml Co-authored-by: Diego Rodriguez --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 0391c47d..d0f41bb8 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -22,7 +22,7 @@ jobs: # version: ["3.8.8", "3.9.13", "3.10.5"] 3.10.5 fails with dependency conflicts # We may need to have separate requirements.txt for each version, or replace # dependencies on the fly - version: ["3.8.13", "3.9.13"] + version: ["3.8.15", "3.9.15"] directory: ["datascience-notebook"] # The datascience-notebook base image does not support ARM # We would need to build and maintain our own base image From 8a6fc03fbf7ef626d5be59721a71d4f2a1f1bf42 Mon Sep 17 00:00:00 2001 From: andersdberg <79153744+andersdberg@users.noreply.github.com> Date: Wed, 2 Nov 2022 20:27:36 -0700 Subject: [PATCH 16/57] Update .github/workflows/main.yaml Co-authored-by: Diego Rodriguez --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index d0f41bb8..f9f00968 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: endpoint: github-action From d1a0b617db690760f32be5d0a36f6eb536e45321 Mon Sep 17 00:00:00 2001 From: andersdberg <79153744+andersdberg@users.noreply.github.com> Date: Wed, 2 Nov 2022 20:27:43 -0700 Subject: [PATCH 17/57] Update .github/workflows/main.yaml Co-authored-by: Diego Rodriguez --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index f9f00968..75442cc8 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -45,7 +45,7 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v4 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | From 5e8fcf26429e185668e60e99d1d03f9799ec07cd Mon Sep 17 00:00:00 2001 From: andersdberg <79153744+andersdberg@users.noreply.github.com> Date: Wed, 2 Nov 2022 20:28:04 -0700 Subject: [PATCH 18/57] Update kernels/datascience-notebook/Dockerfile Co-authored-by: Diego Rodriguez --- kernels/datascience-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernels/datascience-notebook/Dockerfile b/kernels/datascience-notebook/Dockerfile index 49935645..73775ee9 100644 --- a/kernels/datascience-notebook/Dockerfile +++ b/kernels/datascience-notebook/Dockerfile @@ -1,4 +1,4 @@ -# syntax = docker/dockerfile:1.4.1 +# syntax = docker/dockerfile:1.4.3 ARG BASE_IMAGE=jupyter/datascience-notebook ARG PYTHON_VERSION=3.9.13 # hadolint ignore=DL3006 From 16e87f3bd4a3fcd2f6e726cfaab753b512179118 Mon Sep 17 00:00:00 2001 From: andersdberg <79153744+andersdberg@users.noreply.github.com> Date: Wed, 2 Nov 2022 20:28:21 -0700 Subject: [PATCH 19/57] Update kernels/datascience-notebook/Dockerfile Co-authored-by: Diego Rodriguez --- kernels/datascience-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernels/datascience-notebook/Dockerfile b/kernels/datascience-notebook/Dockerfile index 73775ee9..8cfb00e4 100644 --- a/kernels/datascience-notebook/Dockerfile +++ b/kernels/datascience-notebook/Dockerfile @@ -1,6 +1,6 @@ # syntax = docker/dockerfile:1.4.3 ARG BASE_IMAGE=jupyter/datascience-notebook -ARG PYTHON_VERSION=3.9.13 +ARG PYTHON_VERSION=3.9.15 # hadolint ignore=DL3006 FROM ${BASE_IMAGE}:python-${PYTHON_VERSION} From 5ee2de1dc7be9958e3c1da6cb58c94200b97cd11 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Fri, 4 Nov 2022 09:04:31 -0700 Subject: [PATCH 20/57] Empty-Commit From 7b636c270efef70069665f1603060d6396e5270a Mon Sep 17 00:00:00 2001 From: andersdberg Date: Fri, 4 Nov 2022 09:07:44 -0700 Subject: [PATCH 21/57] update --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 75442cc8..2b530669 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -16,7 +16,7 @@ env: jobs: build-python-images: - runs-on: kubernetes-organization-runner + runs-on: ubuntu-22.04 strategy: matrix: # version: ["3.8.8", "3.9.13", "3.10.5"] 3.10.5 fails with dependency conflicts From 37b0044ba81d5d2537db93336a11b6fdbaf694d0 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Fri, 4 Nov 2022 09:09:31 -0700 Subject: [PATCH 22/57] revert to v2, there is no v3 --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 2b530669..fef1047c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v2 with: endpoint: github-action From 0806f95b3b12a6a71bf07a965d8e9bd5082fbabf Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 12 Jan 2023 13:44:37 -0700 Subject: [PATCH 23/57] triggering build for refresh --- .github/workflows/main.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index fef1047c..858aa3d9 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -16,7 +16,7 @@ env: jobs: build-python-images: - runs-on: ubuntu-22.04 + runs-on: kubernetes-organization-runner strategy: matrix: # version: ["3.8.8", "3.9.13", "3.10.5"] 3.10.5 fails with dependency conflicts @@ -24,9 +24,6 @@ jobs: # dependencies on the fly version: ["3.8.15", "3.9.15"] directory: ["datascience-notebook"] - # The datascience-notebook base image does not support ARM - # We would need to build and maintain our own base image - # architecture: ["arm", "amd"] steps: - name: Checkout code uses: actions/checkout@v2 From cc011de2d03232bc56a5cbda90d0e213fff68ca5 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Fri, 27 Jan 2023 12:07:04 -0700 Subject: [PATCH 24/57] Revert "add action on PR" This reverts commit f4796aa1dc41aaca79fd198f8cdf530a3e1c3b95. --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 858aa3d9..debceed5 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -5,7 +5,7 @@ on: branches: - main - pull_request: + pull: branches: - main From e83ae07cb3bf74fd027d55ec8982f2773564fef8 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Fri, 27 Jan 2023 12:09:48 -0700 Subject: [PATCH 25/57] revert to working --- .github/workflows/main.yaml | 197 +++++++++++++++--------------------- 1 file changed, 84 insertions(+), 113 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index debceed5..9b26cc60 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -5,33 +5,29 @@ on: branches: - main - pull: + pull_request: branches: - main -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - jobs: build-python-images: - runs-on: kubernetes-organization-runner + runs-on: ubuntu-latest strategy: matrix: # version: ["3.8.8", "3.9.13", "3.10.5"] 3.10.5 fails with dependency conflicts # We may need to have separate requirements.txt for each version, or replace # dependencies on the fly - version: ["3.8.15", "3.9.15"] + version: ["3.8.8", "3.9.13"] directory: ["datascience-notebook"] + # The datascience-notebook base image does not support ARM + # We would need to build and maintain our own base image + # architecture: ["arm", "amd"] steps: - name: Checkout code uses: actions/checkout@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - endpoint: github-action + # TODO: Log into DockerHub to prevent rate limiting - name: Login to GitHub Container Registry uses: docker/login-action@v2 @@ -40,109 +36,84 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=raw,value=latest,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }} - type=schedule - type=ref,event=branch - type=ref,event=tag - type=ref,event=pr - labels: | - org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} - - - name: Get current time - uses: josStorer/get-current-time@84e5c63cf4cc28dc797be7bb0bfc0171b8c468ce - id: current-time + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 - # - name: Create context - # run: | - # docker context create github-action - - # - name: Cache Docker layers - # uses: actions/cache@v3 - # id: docker-cache - # with: - # path: "/tmp/.buildx-cache" - # key: "${{ runner.os }}-${{env.RUNNER_ARCH}}-buildx-${{ matrix.directory }}-${{ matrix.version }}" - # restore-keys: "${{ runner.os }}-${{env.RUNNER_ARCH}}-buildx-${{ matrix.directory }}-" + - name: Create context + run: | + docker context create github-action + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + endpoint: github-action + version: v0.8.2 + env: + DOCKER_CONTEXT: github-action + + - name: Cache Docker layers + uses: actions/cache@v2 + id: docker-cache + with: + path: "/tmp/.buildx-cache" + key: "${{ runner.os }}-buildx-${{ matrix.directory }}-${{ matrix.version }}" + restore-keys: "${{ runner.os }}-buildx-" - # - name: Build arguments - # id: build-args - # run: | - # # Image Name - # container_registry=ghcr.io/${{ github.repository_owner }} - # image_name=kernel-${{ matrix.directory }} - # full_image_name="${container_registry}/${image_name}" - - # # Image Tags - # image_sha_tag="${GITHUB_SHA:0:12}" # first 12 numbers of the SHA - # image_version_tag="python-$(version=${{ matrix.version }} && echo ${version%.*} )" # removes patch version + - name: Build arguments + id: build-args + run: | + # Image Name + container_registry=ghcr.io/${{ github.repository_owner }} + image_name=kernel-${{ matrix.directory }} + full_image_name="${container_registry}/${image_name}" + + # Image Tags + image_sha_tag="${GITHUB_SHA:0:12}" # first 12 numbers of the SHA + image_version_tag="python-$(version=${{ matrix.version }} && echo ${version%.*} )" # removes patch version - # full_image_name_tagged='' - - # if [ "${GITHUB_EVENT_NAME}" = 'push' ]; then - # full_image_name_tagged="${full_image_name}:${image_version_tag}" - # elif [ "${GITHUB_EVENT_NAME}" = 'pull_request' ]; then - # full_image_name_tagged="${full_image_name}:${image_version_tag}-${image_sha_tag}" - # fi - - # echo "::set-output name=FULL_IMAGE_NAME::${full_image_name}" - # echo "::set-output name=FULL_IMAGE_NAME_TAGGED::${full_image_name_tagged}" - - # echo "::set-output name=BUILD_URL::https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" - # echo "::set-output name=BUILD_TIMESTAMP::$(date --utc --iso-8601=seconds)" - - # echo "full_image_name: $full_image_name" - # echo "image_version_tag: $image_version_tag" - # echo "image_sha_tag: $image_sha_tag" - # echo "full_image_name_tagged: $full_image_name_tagged" - - # - name: Build image - # env: - # DOCKER_CONTENT_TRUST: 1 - # DOCKER_CONTEXT: github-action - # run: | - # ( - # cd ${GITHUB_WORKSPACE}/kernels/${{ matrix.directory }} + full_image_name_tagged='' + + if [ "${GITHUB_EVENT_NAME}" = 'push' ]; then + full_image_name_tagged="${full_image_name}:${image_version_tag}" + elif [ "${GITHUB_EVENT_NAME}" = 'pull_request' ]; then + full_image_name_tagged="${full_image_name}:${image_version_tag}-${image_sha_tag}" + fi + + echo "::set-output name=FULL_IMAGE_NAME::${full_image_name}" + echo "::set-output name=FULL_IMAGE_NAME_TAGGED::${full_image_name_tagged}" + + echo "::set-output name=BUILD_URL::https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + echo "::set-output name=BUILD_TIMESTAMP::$(date --utc --iso-8601=seconds)" + + echo "full_image_name: $full_image_name" + echo "image_version_tag: $image_version_tag" + echo "image_sha_tag: $image_sha_tag" + echo "full_image_name_tagged: $full_image_name_tagged" + + - name: Build image + env: + DOCKER_CONTENT_TRUST: 1 + DOCKER_CONTEXT: github-action + run: | + ( + cd ${GITHUB_WORKSPACE}/kernels/${{ matrix.directory }} - # docker buildx build \ - # --pull \ - # --output 'type=docker' \ - # --platform=linux/arm64 \ - # --progress plain \ - # --cache-from 'type=local,src=/tmp/.buildx-cache' \ - # --cache-to 'type=local,dest=/tmp/.buildx-cache' \ - # --tag '${{ steps.build-args.outputs.FULL_IMAGE_NAME_TAGGED }}' \ - # --build-arg PYTHON_VERSION=${{ matrix.version }} \ - # --build-arg 'NBL_ARG_BUILD_TIMESTAMP=${{ steps.build-args.outputs.BUILD_TIMESTAMP }}' \ - # --build-arg 'NBL_ARG_BUILD_URL=${{ steps.build-args.outputs.BUILD_URL }}' \ - # --build-arg 'NBL_ARG_REVISION=${{ github.sha }}' \ - # --build-arg 'NBL_ARG_VERSION=${{ github.ref }}' \ - # . - # ) - - # - name: Publish image - # run: | - # docker push --all-tags ${{ steps.build-args.outputs.FULL_IMAGE_NAME }} - - - name: Build and push Docker image - uses: docker/build-push-action@v2 - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - secrets: | - "expel_artifactory_connection_url_file=${{ secrets.EXPEL_ARTIFACTORY_CONNECTION_URL }}" - "git-credentials=${{ secrets.GIT_CREDENTIALS }}" - build-args: | - "NBL_ARG_BUILD_TIMESTAMP=${{ steps.current-time.outputs.formattedTime }}" - "NBL_ARG_REVISION=${{ github.sha }}" - "NBL_ARG_BUILD_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - "NBL_ARG_VERSION=${{ github.ref }}" \ No newline at end of file + docker buildx build \ + --pull \ + --output 'type=docker' \ + --platform=linux/arm64 \ + --progress plain \ + --cache-from 'type=local,src=/tmp/.buildx-cache' \ + --cache-to 'type=local,dest=/tmp/.buildx-cache' \ + --tag '${{ steps.build-args.outputs.FULL_IMAGE_NAME_TAGGED }}' \ + --build-arg PYTHON_VERSION=${{ matrix.version }} \ + --build-arg 'NBL_ARG_BUILD_TIMESTAMP=${{ steps.build-args.outputs.BUILD_TIMESTAMP }}' \ + --build-arg 'NBL_ARG_BUILD_URL=${{ steps.build-args.outputs.BUILD_URL }}' \ + --build-arg 'NBL_ARG_REVISION=${{ github.sha }}' \ + --build-arg 'NBL_ARG_VERSION=${{ github.ref }}' \ + . + ) + + - name: Publish image + run: | + docker push --all-tags ${{ steps.build-args.outputs.FULL_IMAGE_NAME }} \ No newline at end of file From 4978b8c5df46849b58e33b221c50df6e1ce3318b Mon Sep 17 00:00:00 2001 From: andersdberg Date: Fri, 27 Jan 2023 12:30:52 -0700 Subject: [PATCH 26/57] change cache --- .github/workflows/main.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 9b26cc60..5c8a4dcd 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -47,7 +47,7 @@ jobs: uses: docker/setup-buildx-action@v1 with: endpoint: github-action - version: v0.8.2 + version: v0.10.0 env: DOCKER_CONTEXT: github-action @@ -55,7 +55,7 @@ jobs: uses: actions/cache@v2 id: docker-cache with: - path: "/tmp/.buildx-cache" + path: "/tmp/buildx-cache" key: "${{ runner.os }}-buildx-${{ matrix.directory }}-${{ matrix.version }}" restore-keys: "${{ runner.os }}-buildx-" @@ -103,8 +103,8 @@ jobs: --output 'type=docker' \ --platform=linux/arm64 \ --progress plain \ - --cache-from 'type=local,src=/tmp/.buildx-cache' \ - --cache-to 'type=local,dest=/tmp/.buildx-cache' \ + --cache-from 'type=local,src=/tmp/buildx-cache' \ + --cache-to 'type=local,dest=/tmp/buildx-cache' \ --tag '${{ steps.build-args.outputs.FULL_IMAGE_NAME_TAGGED }}' \ --build-arg PYTHON_VERSION=${{ matrix.version }} \ --build-arg 'NBL_ARG_BUILD_TIMESTAMP=${{ steps.build-args.outputs.BUILD_TIMESTAMP }}' \ @@ -115,5 +115,7 @@ jobs: ) - name: Publish image + env: + DOCKER_CONTEXT: github-action run: | - docker push --all-tags ${{ steps.build-args.outputs.FULL_IMAGE_NAME }} \ No newline at end of file + docker push --all-tags ${{ steps.build-args.outputs.FULL_IMAGE_NAME }} From 054c258ce5868da7f93c05f5af0a135f291a78f1 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Fri, 27 Jan 2023 13:16:01 -0700 Subject: [PATCH 27/57] update versions --- .github/workflows/main.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 5c8a4dcd..cdef3c38 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -18,14 +18,14 @@ jobs: # version: ["3.8.8", "3.9.13", "3.10.5"] 3.10.5 fails with dependency conflicts # We may need to have separate requirements.txt for each version, or replace # dependencies on the fly - version: ["3.8.8", "3.9.13"] + version: ["3.9.13", "3.8.13"] directory: ["datascience-notebook"] # The datascience-notebook base image does not support ARM # We would need to build and maintain our own base image # architecture: ["arm", "amd"] steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 # TODO: Log into DockerHub to prevent rate limiting @@ -52,7 +52,7 @@ jobs: DOCKER_CONTEXT: github-action - name: Cache Docker layers - uses: actions/cache@v2 + uses: actions/cache@v3 id: docker-cache with: path: "/tmp/buildx-cache" From 832224297927fc16f35f1e342209922fe10ce126 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Fri, 27 Jan 2023 13:35:27 -0700 Subject: [PATCH 28/57] Update to latest version of notebook image with new features --- kernels/datascience-notebook/.pythonrc | 5 ++ kernels/datascience-notebook/Dockerfile | 41 +++++++---- kernels/datascience-notebook/environment.txt | 2 +- .../git_credential_helper.py | 71 +++++++++++++++++++ kernels/datascience-notebook/git_wrapper.sh | 16 +++++ .../datascience-notebook/ipython_config.py | 5 +- kernels/datascience-notebook/requirements.txt | 24 +++---- kernels/datascience-notebook/run.sh | 8 +-- 8 files changed, 139 insertions(+), 33 deletions(-) create mode 100644 kernels/datascience-notebook/git_credential_helper.py create mode 100644 kernels/datascience-notebook/git_wrapper.sh diff --git a/kernels/datascience-notebook/.pythonrc b/kernels/datascience-notebook/.pythonrc index 0bcc2d9f..6a1eac85 100644 --- a/kernels/datascience-notebook/.pythonrc +++ b/kernels/datascience-notebook/.pythonrc @@ -1,3 +1,8 @@ import pandas as pd import dx + +dx.set_option("DISPLAY_MAX_ROWS", 50_000) +dx.set_option("DISPLAY_MAX_COLUMNS", 100) +dx.set_option("ENABLE_DATALINK", True) +dx.set_option("ENABLE_ASSIGNMENT", False) \ No newline at end of file diff --git a/kernels/datascience-notebook/Dockerfile b/kernels/datascience-notebook/Dockerfile index 8cfb00e4..b3ea6120 100644 --- a/kernels/datascience-notebook/Dockerfile +++ b/kernels/datascience-notebook/Dockerfile @@ -1,11 +1,17 @@ # syntax = docker/dockerfile:1.4.3 ARG BASE_IMAGE=jupyter/datascience-notebook -ARG PYTHON_VERSION=3.9.15 +ARG PYTHON_VERSION=3.9.13 # hadolint ignore=DL3006 FROM ${BASE_IMAGE}:python-${PYTHON_VERSION} USER root +# datascience-notebook:python-3.9.13 includes psutil 5.9.2 with cooked C lib, but +# later pip installs end up installing 5.9.4, but for some +# reason 'import psutil' will end up getting the python 5.9.4 but the +# C lib from 5.9.2, and, unlike Smeagol, it hateses the precious. +RUN pip uninstall -y psutil + # Set up log file for magics RUN touch /var/log/noteable_magics.log && \ chown 4004:4004 /var/log/noteable_magics.log @@ -21,11 +27,6 @@ RUN apt-get update && \ apt-get install -y jq procps git unixodbc-dev g++ \ && rm -rf /var/lib/apt/lists/* -ENV TINI_VERSION=v0.19.0 -RUN TINI_BINARY=$(if [ "$(uname -m)" = "aarch64" ]; then echo "tini-arm64"; else echo "tini"; fi); echo "${TINI_BINARY}" \ - && wget -q -O /usr/local/bin/tini "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/${TINI_BINARY}" \ - && chmod +x /usr/local/bin/tini - ENV NB_USER="noteable" \ NB_UID=4004 \ NB_GID=4004 @@ -43,6 +44,19 @@ RUN chown noteable:noteable "${JULIA_PKGDIR}" && \ fix-permissions "${JULIA_PKGDIR}" && \ fix-permissions "${CONDA_DIR}" +ENV TINI_VERSION=v0.19.0 +RUN TINI_BINARY=$(if [ "$(uname -m)" = "aarch64" ]; then echo "tini-arm64"; else echo "tini"; fi); echo "${TINI_BINARY}" \ + && wget -q -O /usr/local/bin/tini "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/${TINI_BINARY}" \ + && chmod +x /usr/local/bin/tini + + + + + + + + + # Run non-privileged user USER noteable @@ -54,18 +68,14 @@ ENV PATH="/home/noteable/.local/bin:${PATH}" \ # hadolint ignore=DL3045 COPY environment.txt ./ -# hadolint ignore=DL3045 -COPY requirements.txt ./ - # hadolint ignore=SC2034 -RUN conda install --file environment.txt +RUN mamba install --file environment.txt # hadolint ignore=DL3045 COPY requirements.txt ./ # hadolint ignore=SC1008,SC2155,DL3042,SC2102 -RUN pip install -I --quiet --no-cache-dir "git+https://github.com/noteable-io/noteable-notebook-magics.git@main" && \ - pip install -I --quiet --no-cache-dir -r requirements.txt +RUN pip install -I --no-cache-dir -r requirements.txt # Copy over any python commands that need to run on startup # that aren't covered by IPython extensions @@ -80,7 +90,10 @@ RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension RUN python -c "import noteable_magics" RUN git config --global user.name "Noteable Kernel" && \ - git config --global user.email "engineering@noteable.io" + git config --global user.email "engineering@noteable.io" && \ + git config --global safe.directory /etc/noteable/project && \ + git config --global credential.helper /git_credential_helper.py && \ + git config --global credential.useHttpPath true # https://ipython.readthedocs.io/en/stable/config/intro.html#systemwide-configuration COPY ipython_config.py /etc/ipython @@ -91,6 +104,8 @@ WORKDIR /etc/noteable/project # Add the entrypoint script to the $PATH COPY run.sh /usr/local/bin COPY secrets_helper.py /tmp/secrets_helper.py +COPY git_credential_helper.py /git_credential_helper.py +COPY git-wrapper.sh /usr/local/bin/git EXPOSE 50001-50005 diff --git a/kernels/datascience-notebook/environment.txt b/kernels/datascience-notebook/environment.txt index b370b004..20d65f2f 100644 --- a/kernels/datascience-notebook/environment.txt +++ b/kernels/datascience-notebook/environment.txt @@ -1,4 +1,4 @@ -ipykernel=5.5.* +jupyter_client=7.3.* ipython=8.0.* vdom=0.6 papermill=2.2.* diff --git a/kernels/datascience-notebook/git_credential_helper.py b/kernels/datascience-notebook/git_credential_helper.py new file mode 100644 index 00000000..755efe42 --- /dev/null +++ b/kernels/datascience-notebook/git_credential_helper.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +""" +This script is used as a Git credential helper https://git-scm.com/docs/git-credential. +We iterate through all the git credential secrets on the file system and return the first one that matches the requested URL. +If no match is found, we return an empty response. +An empty response will cause Git to use the next credential helper in the list, or prompt the user for credentials. + +To test this script: + +$ cat > /tmp/demo.git-cred < dict: + """Parse the input from Git into a dictionary.""" + return dict(line.split("=", 1) for line in input_.splitlines()) + + +def format_output(data: dict) -> str: + """Format the output to Git.""" + return "\n".join(f"{key}={value}" for key, value in data.items()) + + +def find_secret(input_data: dict) -> Optional[dict]: + """Find the secret that matches the input data.""" + secrets_dir = Path(os.environ.get("NTBL_SECRETS_DIR", "/vault/secrets")) + if not secrets_dir.exists(): + return None + + keys_to_match = ["host", "protocol", "path"] + for secret_path in secrets_dir.glob("*.git-cred"): + secret_data = json.loads(secret_path.read_text()) + meta = secret_data["meta"] + if all(meta[key] == input_data.get(key) for key in keys_to_match): + return secret_data["data"] + + return None + + +def main(stdin=sys.stdin, stdout=sys.stdout): + """Main entrypoint.""" + parsed_input = parse_input(stdin.read()) + if (secret := find_secret(parsed_input)) is not None: + print(format_output(secret), file=stdout) + + +if __name__ == "__main__": + main() diff --git a/kernels/datascience-notebook/git_wrapper.sh b/kernels/datascience-notebook/git_wrapper.sh new file mode 100644 index 00000000..dff1bff6 --- /dev/null +++ b/kernels/datascience-notebook/git_wrapper.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# This script wraps git to only allow certain commands to be run. +# We mainly want to prevent users from getting into unknown states by checking out other branches, etc. + +# Allowed command list +allowed_commands=( "commit" "pull" "push" "status" "diff" "add" "fetch" "log" ) + +# Check if the command is allowed +# shellcheck disable=SC2076 +if [[ ! " ${allowed_commands[*]} " =~ " ${1} " ]]; then + echo "That git command is not allowed, contact support@noteable.io if you think this is a mistake." + exit 1 +fi + +# Otherwise pass through to git at /usr/bin/git +exec /usr/bin/git "$@" diff --git a/kernels/datascience-notebook/ipython_config.py b/kernels/datascience-notebook/ipython_config.py index 91146934..fb66a186 100644 --- a/kernels/datascience-notebook/ipython_config.py +++ b/kernels/datascience-notebook/ipython_config.py @@ -1,5 +1,4 @@ c.InteractiveShellApp.extensions = [ - "sql", "noteable_magics", ] @@ -7,6 +6,6 @@ c.SqlMagic.autopandas = True c.NTBLMagic.project_dir = "/etc/noteable/project" c.NoteableDataLoaderMagic.return_head = False -c.IPythonKernel._execute_sleep=0.15 +c.IPythonKernel._execute_sleep = 0.15 # 10 minutes to support large files -c.NTBLMagic.planar_ally_default_timeout_seconds = 600 +c.NTBLMagic.planar_ally_default_timeout_seconds = 600 \ No newline at end of file diff --git a/kernels/datascience-notebook/requirements.txt b/kernels/datascience-notebook/requirements.txt index af44cd02..3177ffd7 100644 --- a/kernels/datascience-notebook/requirements.txt +++ b/kernels/datascience-notebook/requirements.txt @@ -1,12 +1,12 @@ -dx==1.1.2 -# Datasources-related packages here on down, alphabetized please for easy cut/paste across files and repos. -google-cloud-bigquery-storage==2.6.3 -psycopg2-binary==2.9.3 -pyodbc==4.0.32 -redshift_connector==2.0.907 -snowflake_sqlalchemy==1.3.4 -sqlalchemy-bigquery==1.3.0 -sqlalchemy-databricks==0.2.0 -sqlalchemy-redshift==0.8.9 -trino[sqlalchemy]==0.313.0 -astroid==2.12.2 +# Scheduler/ochestration packages +dagstermill==0.16.15 +papermill-origami==0.0.9 +cloudpickle==2.2.0 +flytekitplugins-papermill==1.2.4 + +# https://github.com/noteable-io/ packages +git+https://www.github.com/noteable-io/dx.git@4be0c105aea40248d066a1a8beb74ff00d0b5bd3 +git+https://www.github.com/noteable-io/noteable-notebook-magics.git@5d54f4cd94c46f617459db5e5a0110d081393936 +git+https://www.github.com/noteable-io/sidecar_comms.git@eed16c3ab900a8abe19b654fa775646bc38dd519 + +# (All of the datasources modules are now explicit requirements within noteable-notebook-magics.) \ No newline at end of file diff --git a/kernels/datascience-notebook/run.sh b/kernels/datascience-notebook/run.sh index f5fe306b..a6325856 100755 --- a/kernels/datascience-notebook/run.sh +++ b/kernels/datascience-notebook/run.sh @@ -27,24 +27,24 @@ case $kernel_name in echo "Starting Python kernel" # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONSTARTUP export PYTHONSTARTUP=~/.pythonrc - python -m ipykernel_launcher -f ${connection_file} --debug + exec python -m ipykernel_launcher -f ${connection_file} --debug ;; ir) echo "Starting R kernel" - R --slave -e "IRkernel::main()" --args ${connection_file} + exec R --slave -e "IRkernel::main()" --args ${connection_file} ;; julia | julia-1.6) echo "Starting Julia kernel" # project path necessary to keep julia form using its defaults - julia -i --color=yes --project=/etc/noteable/project /opt/julia/packages/IJulia/e8kqU/src/kernel.jl ${connection_file} + exec julia -i --color=yes --project=/etc/noteable/project /opt/julia/packages/IJulia/e8kqU/src/kernel.jl ${connection_file} ;; *) echo "Unrecognized '$kernel_name' kernel, falling back to Python" # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONSTARTUP export PYTHONSTARTUP=~/.pythonrc - python -m ipykernel_launcher -f ${connection_file} --debug + exec python -m ipykernel_launcher -f ${connection_file} --debug ;; esac From 929c8e63218db037f058d43de6feee08a5539051 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Fri, 27 Jan 2023 13:40:26 -0700 Subject: [PATCH 29/57] rename file --- kernels/datascience-notebook/{git_wrapper.sh => git-wrapper.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename kernels/datascience-notebook/{git_wrapper.sh => git-wrapper.sh} (100%) diff --git a/kernels/datascience-notebook/git_wrapper.sh b/kernels/datascience-notebook/git-wrapper.sh similarity index 100% rename from kernels/datascience-notebook/git_wrapper.sh rename to kernels/datascience-notebook/git-wrapper.sh From edef0f3a0cbb32ea0e7689d1bce2b45d94c85437 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Fri, 27 Jan 2023 14:29:31 -0700 Subject: [PATCH 30/57] adding missing envs --- kernels/datascience-notebook/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernels/datascience-notebook/Dockerfile b/kernels/datascience-notebook/Dockerfile index b3ea6120..5f0735c3 100644 --- a/kernels/datascience-notebook/Dockerfile +++ b/kernels/datascience-notebook/Dockerfile @@ -39,6 +39,9 @@ RUN groupadd --gid 4004 noteable && \ RUN mkdir /etc/ipython && chown noteable:noteable /etc/ipython RUN mkdir -p /etc/noteable && chown noteable:noteable /etc/noteable +ENV JULIA_PKGDIR=/opt/julia \ + CONDA_DIR=/opt/conda + RUN chown noteable:noteable "${JULIA_PKGDIR}" && \ chown noteable:noteable "${CONDA_DIR}" && \ fix-permissions "${JULIA_PKGDIR}" && \ From 4b6ed7962b8e67dd48f4809570c59569d97cab94 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Fri, 27 Jan 2023 14:51:04 -0700 Subject: [PATCH 31/57] debug --- kernels/datascience-notebook/Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/kernels/datascience-notebook/Dockerfile b/kernels/datascience-notebook/Dockerfile index 5f0735c3..89d90e87 100644 --- a/kernels/datascience-notebook/Dockerfile +++ b/kernels/datascience-notebook/Dockerfile @@ -39,10 +39,8 @@ RUN groupadd --gid 4004 noteable && \ RUN mkdir /etc/ipython && chown noteable:noteable /etc/ipython RUN mkdir -p /etc/noteable && chown noteable:noteable /etc/noteable -ENV JULIA_PKGDIR=/opt/julia \ - CONDA_DIR=/opt/conda - -RUN chown noteable:noteable "${JULIA_PKGDIR}" && \ +RUN echo "${JULIA_PKGDIR} -- ${CONDA_DIR}" && \ + chown noteable:noteable "${JULIA_PKGDIR}" && \ chown noteable:noteable "${CONDA_DIR}" && \ fix-permissions "${JULIA_PKGDIR}" && \ fix-permissions "${CONDA_DIR}" From 958ba3d754786937a090af0ffe3a5f2dccb69fe4 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Fri, 27 Jan 2023 15:00:50 -0700 Subject: [PATCH 32/57] revert --- kernels/datascience-notebook/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernels/datascience-notebook/Dockerfile b/kernels/datascience-notebook/Dockerfile index 89d90e87..b3ea6120 100644 --- a/kernels/datascience-notebook/Dockerfile +++ b/kernels/datascience-notebook/Dockerfile @@ -39,8 +39,7 @@ RUN groupadd --gid 4004 noteable && \ RUN mkdir /etc/ipython && chown noteable:noteable /etc/ipython RUN mkdir -p /etc/noteable && chown noteable:noteable /etc/noteable -RUN echo "${JULIA_PKGDIR} -- ${CONDA_DIR}" && \ - chown noteable:noteable "${JULIA_PKGDIR}" && \ +RUN chown noteable:noteable "${JULIA_PKGDIR}" && \ chown noteable:noteable "${CONDA_DIR}" && \ fix-permissions "${JULIA_PKGDIR}" && \ fix-permissions "${CONDA_DIR}" From 953c2b0f1987095867270f2c3062b9741c51aa1e Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 11:46:55 -0700 Subject: [PATCH 33/57] add 3.7 --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index cdef3c38..9b47d474 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -18,7 +18,7 @@ jobs: # version: ["3.8.8", "3.9.13", "3.10.5"] 3.10.5 fails with dependency conflicts # We may need to have separate requirements.txt for each version, or replace # dependencies on the fly - version: ["3.9.13", "3.8.13"] + version: ["3.7.12", "3.8.13", "3.9.13"] directory: ["datascience-notebook"] # The datascience-notebook base image does not support ARM # We would need to build and maintain our own base image From c185dab5543df476a7ae3db194aadad5c034a3fe Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 11:56:06 -0700 Subject: [PATCH 34/57] change to amd... --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 9b47d474..37266788 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -101,7 +101,7 @@ jobs: docker buildx build \ --pull \ --output 'type=docker' \ - --platform=linux/arm64 \ + --platform=linux/amd64 \ --progress plain \ --cache-from 'type=local,src=/tmp/buildx-cache' \ --cache-to 'type=local,dest=/tmp/buildx-cache' \ From 0ce6239f237918effca422d280f3c49d68ed42d4 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 12:06:15 -0700 Subject: [PATCH 35/57] update packages --- kernels/datascience-notebook/environment.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernels/datascience-notebook/environment.txt b/kernels/datascience-notebook/environment.txt index 20d65f2f..674d224f 100644 --- a/kernels/datascience-notebook/environment.txt +++ b/kernels/datascience-notebook/environment.txt @@ -1,7 +1,7 @@ jupyter_client=7.3.* -ipython=8.0.* +ipython vdom=0.6 papermill=2.2.* ipywidgets=7.6.* plotly=4.14.3 -geopandas=0.11.0 \ No newline at end of file +geopandas \ No newline at end of file From ff22605a783b196b05463e8d45c8b0e7a8ca3414 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 13:39:54 -0700 Subject: [PATCH 36/57] Add environment specific requirements --- kernels/datascience-notebook/Dockerfile | 4 ++-- .../datascience-notebook/environment-3.7.12.txt | 8 ++++++++ .../{environment.txt => environment-3.8.13.txt} | 0 .../datascience-notebook/environment-3.9.13.txt | 7 +++++++ requirements-3.7.12.txt | 16 ++++++++++++++++ requirements.txt => requirements-3.8.13.txt | 0 requirements-3.9.13.txt | 12 ++++++++++++ 7 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 kernels/datascience-notebook/environment-3.7.12.txt rename kernels/datascience-notebook/{environment.txt => environment-3.8.13.txt} (100%) create mode 100644 kernels/datascience-notebook/environment-3.9.13.txt create mode 100644 requirements-3.7.12.txt rename requirements.txt => requirements-3.8.13.txt (100%) create mode 100644 requirements-3.9.13.txt diff --git a/kernels/datascience-notebook/Dockerfile b/kernels/datascience-notebook/Dockerfile index b3ea6120..a9debff6 100644 --- a/kernels/datascience-notebook/Dockerfile +++ b/kernels/datascience-notebook/Dockerfile @@ -66,13 +66,13 @@ ENV PATH="/home/noteable/.local/bin:${PATH}" \ GOOGLE_APPLICATION_CREDENTIALS="/vault/secrets/gcp-credentials" # hadolint ignore=DL3045 -COPY environment.txt ./ +COPY environment-${PYTHON_VERSION}.txt ./environment.txt # hadolint ignore=SC2034 RUN mamba install --file environment.txt # hadolint ignore=DL3045 -COPY requirements.txt ./ +COPY requirements-${PYTHON_VERSION}.txt ./requirements.txt # hadolint ignore=SC1008,SC2155,DL3042,SC2102 RUN pip install -I --no-cache-dir -r requirements.txt diff --git a/kernels/datascience-notebook/environment-3.7.12.txt b/kernels/datascience-notebook/environment-3.7.12.txt new file mode 100644 index 00000000..c09adcca --- /dev/null +++ b/kernels/datascience-notebook/environment-3.7.12.txt @@ -0,0 +1,8 @@ +jupyter_client=7.3.* +ipython +vdom=0.6 +papermill=2.2.* +ipywidgets=7.6.* +geopandas +pyspark==3.2.1 +openjdk==8.0.332 diff --git a/kernels/datascience-notebook/environment.txt b/kernels/datascience-notebook/environment-3.8.13.txt similarity index 100% rename from kernels/datascience-notebook/environment.txt rename to kernels/datascience-notebook/environment-3.8.13.txt diff --git a/kernels/datascience-notebook/environment-3.9.13.txt b/kernels/datascience-notebook/environment-3.9.13.txt new file mode 100644 index 00000000..20d65f2f --- /dev/null +++ b/kernels/datascience-notebook/environment-3.9.13.txt @@ -0,0 +1,7 @@ +jupyter_client=7.3.* +ipython=8.0.* +vdom=0.6 +papermill=2.2.* +ipywidgets=7.6.* +plotly=4.14.3 +geopandas=0.11.0 \ No newline at end of file diff --git a/requirements-3.7.12.txt b/requirements-3.7.12.txt new file mode 100644 index 00000000..f2b02995 --- /dev/null +++ b/requirements-3.7.12.txt @@ -0,0 +1,16 @@ +# Scheduler/ochestration packages +dagstermill==0.16.15 +papermill-origami==0.0.9 +cloudpickle==2.2.0 +flytekitplugins-papermill==1.2.4 +# Added for ease of use with integration partners +pyiceberg==0.3.0 +fugue==0.8.0 +fugue-jupyter==0.2.2 + +# https://github.com/noteable-io/ packages +git+https://www.github.com/noteable-io/dx.git@a7df2821182293546d7d7a9ede3cdcc0c946d570 +git+https://www.github.com/noteable-io/noteable-notebook-magics.git@a3b00faedcba9f38c7b75b4f45f4eba7e1ce313e +git+https://www.github.com/noteable-io/sidecar_comms.git@35b7cf8ad6b15daf020954c9029aaae5779f324a + +# (All of the datasources modules are now explicit requirements within noteable-notebook-magics.) diff --git a/requirements.txt b/requirements-3.8.13.txt similarity index 100% rename from requirements.txt rename to requirements-3.8.13.txt diff --git a/requirements-3.9.13.txt b/requirements-3.9.13.txt new file mode 100644 index 00000000..110ef814 --- /dev/null +++ b/requirements-3.9.13.txt @@ -0,0 +1,12 @@ +# Scheduler/ochestration packages +dagstermill==0.16.15 +papermill-origami==0.0.9 +cloudpickle==2.2.0 +flytekitplugins-papermill==1.2.4 + +# https://github.com/noteable-io/ packages +git+https://www.github.com/noteable-io/dx.git@a7df2821182293546d7d7a9ede3cdcc0c946d570 +git+https://www.github.com/noteable-io/noteable-notebook-magics.git@a6a6801da8ea7ccb72b9f354d4780699c3d99f73 +git+https://www.github.com/noteable-io/sidecar_comms.git@6ee04efe60b855c465727f120f8f50a7bfa60097 + +# (All of the datasources modules are now explicit requirements within noteable-notebook-magics.) \ No newline at end of file From 8e651a203157eb620c751a5184e47d0a856853f1 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 13:40:46 -0700 Subject: [PATCH 37/57] missing change --- kernels/datascience-notebook/environment-3.8.13.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernels/datascience-notebook/environment-3.8.13.txt b/kernels/datascience-notebook/environment-3.8.13.txt index 674d224f..20d65f2f 100644 --- a/kernels/datascience-notebook/environment-3.8.13.txt +++ b/kernels/datascience-notebook/environment-3.8.13.txt @@ -1,7 +1,7 @@ jupyter_client=7.3.* -ipython +ipython=8.0.* vdom=0.6 papermill=2.2.* ipywidgets=7.6.* plotly=4.14.3 -geopandas \ No newline at end of file +geopandas=0.11.0 \ No newline at end of file From acd10bc83501c5d5c5ffdd01fdbe4553b09e0b54 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 13:47:27 -0700 Subject: [PATCH 38/57] remove reusable workflow for now --- .github/workflows/reusable-docker-build.yml | 142 -------------------- 1 file changed, 142 deletions(-) delete mode 100644 .github/workflows/reusable-docker-build.yml diff --git a/.github/workflows/reusable-docker-build.yml b/.github/workflows/reusable-docker-build.yml deleted file mode 100644 index 5958de5b..00000000 --- a/.github/workflows/reusable-docker-build.yml +++ /dev/null @@ -1,142 +0,0 @@ -name: docker - -on: - workflow_call: - inputs: - dockerfile: - description: "Path to the Dockerfile to build" - type: string - default: Dockerfile - context: - description: "The context for Docker build" - type: string - default: "." - platforms: - description: "Comma separate list of platforms to build on" - type: string - required: false - default: "linux/amd64,linux/arm64" - images: - description: "The image names that we want to build" - type: string - required: true - tags: - description: "The various tags to be attached to the built image" - type: string - required: false - default: "" - labels: - description: "The various labels to attach to the built image" - type: string - required: false - default: | - org.opencontainers.image.url=https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - org.opencontainers.image.vendor=Noteable - org.opencontainers.image.version=${{ github.ref }} - target: - description: "Sets the target stage to build" - type: string - required: false - build_args: - description: "Additional build args to pass to the Docker build" - type: string - required: false - default: "" - secrets: - # We login to Dockerhub to prevent rate limiting issues when pulling images - # https://docs.docker.com/docker-hub/download-rate-limit/ - DOCKERHUB_USER: - required: true - DOCKERHUB_PASSWORD: - required: true - -jobs: - build: - permissions: - id-token: write - contents: read - packages: write - - if: | - github.event_name == 'push' || - (github.event_name == 'pull_request' && github.event.pull_request.state == 'open') - runs-on: ubuntu-22.04 - steps: - - name: Checkout the code - uses: actions/checkout@v3 - - - name: Copy common files - run: make copy-common-files - - name: Log in to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Log in to the Container registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - version: v0.10.1 - - # Note: The outputs in github action will show duplicate labels being generated for the meta outputs. - # When the Docker engine builds, it will only take the later values, and our custom labels get added - # at the end. https://github.com/docker/metadata-action/issues/125 - - name: Docker metadata for labels and tags - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ inputs.images }} - tags: ${{ inputs.tags }} - labels: ${{ inputs.labels }} - - - name: Build and push - uses: docker/build-push-action@v3 - with: - platforms: ${{ inputs.platforms }} - context: ${{ inputs.context }} - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - target: ${{ inputs.target }} - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: ${{ inputs.build_args }} - - clear_cache: - permissions: - contents: read - actions: write - # If the PR is closed (or merged), we want to clear the cache - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.state == 'closed' }} - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v3 - - - name: Cleanup - run: | - gh extension install actions/gh-actions-cache - - REPO=${{ github.repository }} - BRANCH=${{ github.ref }} - - echo "Fetching list of cache key" - cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH | cut -f 1 ) - - ## Setting this to not fail the workflow while deleting cache keys. - set +e - echo "Deleting caches..." - for cacheKey in $cacheKeysForPR - do - gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm - done - echo "Done" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 47860e4d05747261aadca4e46e32a7f323365710 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 13:47:44 -0700 Subject: [PATCH 39/57] dockerfile updates --- kernels/datascience-notebook/Dockerfile | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/kernels/datascience-notebook/Dockerfile b/kernels/datascience-notebook/Dockerfile index a9debff6..0b5ceca0 100644 --- a/kernels/datascience-notebook/Dockerfile +++ b/kernels/datascience-notebook/Dockerfile @@ -39,23 +39,15 @@ RUN groupadd --gid 4004 noteable && \ RUN mkdir /etc/ipython && chown noteable:noteable /etc/ipython RUN mkdir -p /etc/noteable && chown noteable:noteable /etc/noteable -RUN chown noteable:noteable "${JULIA_PKGDIR}" && \ - chown noteable:noteable "${CONDA_DIR}" && \ - fix-permissions "${JULIA_PKGDIR}" && \ +RUN chown noteable:noteable "${CONDA_DIR}" && \ fix-permissions "${CONDA_DIR}" -ENV TINI_VERSION=v0.19.0 -RUN TINI_BINARY=$(if [ "$(uname -m)" = "aarch64" ]; then echo "tini-arm64"; else echo "tini"; fi); echo "${TINI_BINARY}" \ - && wget -q -O /usr/local/bin/tini "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/${TINI_BINARY}" \ - && chmod +x /usr/local/bin/tini - - - - - - - - +# `rust` is required for install of some packages under arm (and future x86 packages) +# hadolint ignore=SC1091 +RUN wget -O rustup.sh -q https://sh.rustup.rs && \ + sh rustup.sh -y && \ + rm rustup.sh && \ + source "$HOME/.cargo/env" # Run non-privileged user USER noteable From 2230b3946fd1e04eb88b4f49a1a48352661bb544 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 14:05:25 -0700 Subject: [PATCH 40/57] build arg not propagating --- kernels/datascience-notebook/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernels/datascience-notebook/Dockerfile b/kernels/datascience-notebook/Dockerfile index 0b5ceca0..d7055ce9 100644 --- a/kernels/datascience-notebook/Dockerfile +++ b/kernels/datascience-notebook/Dockerfile @@ -4,6 +4,8 @@ ARG PYTHON_VERSION=3.9.13 # hadolint ignore=DL3006 FROM ${BASE_IMAGE}:python-${PYTHON_VERSION} +ENV PYTHON_VERSION=${PYTHON_VERSION} + USER root # datascience-notebook:python-3.9.13 includes psutil 5.9.2 with cooked C lib, but From 6f1b36fb8414c426e25980f4a4a0725b1f51853c Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 14:11:20 -0700 Subject: [PATCH 41/57] bump versions --- .github/workflows/main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 37266788..8ab4f6a3 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -44,10 +44,10 @@ jobs: docker context create github-action - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 with: endpoint: github-action - version: v0.10.0 + version: v0.10.4 env: DOCKER_CONTEXT: github-action From dcd48edc0c6a0a11c25f4929787bcc96e43200b2 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 14:13:36 -0700 Subject: [PATCH 42/57] debug --- .github/workflows/main.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 8ab4f6a3..e312de23 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -103,8 +103,7 @@ jobs: --output 'type=docker' \ --platform=linux/amd64 \ --progress plain \ - --cache-from 'type=local,src=/tmp/buildx-cache' \ - --cache-to 'type=local,dest=/tmp/buildx-cache' \ + --no-cache \ --tag '${{ steps.build-args.outputs.FULL_IMAGE_NAME_TAGGED }}' \ --build-arg PYTHON_VERSION=${{ matrix.version }} \ --build-arg 'NBL_ARG_BUILD_TIMESTAMP=${{ steps.build-args.outputs.BUILD_TIMESTAMP }}' \ From c1d5129130f2d694ba613fac2ed8ae458b3bcef2 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 14:14:36 -0700 Subject: [PATCH 43/57] debug --- kernels/datascience-notebook/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernels/datascience-notebook/Dockerfile b/kernels/datascience-notebook/Dockerfile index d7055ce9..180d910d 100644 --- a/kernels/datascience-notebook/Dockerfile +++ b/kernels/datascience-notebook/Dockerfile @@ -4,8 +4,6 @@ ARG PYTHON_VERSION=3.9.13 # hadolint ignore=DL3006 FROM ${BASE_IMAGE}:python-${PYTHON_VERSION} -ENV PYTHON_VERSION=${PYTHON_VERSION} - USER root # datascience-notebook:python-3.9.13 includes psutil 5.9.2 with cooked C lib, but @@ -59,6 +57,8 @@ ENV PATH="/home/noteable/.local/bin:${PATH}" \ XDG_CACHE_HOME="/home/noteable/.cache/" \ GOOGLE_APPLICATION_CREDENTIALS="/vault/secrets/gcp-credentials" +ARG PYTHON_VERSION + # hadolint ignore=DL3045 COPY environment-${PYTHON_VERSION}.txt ./environment.txt From 577ee69a1c85cca1fff08cdf36825d7343de3e4b Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 14:18:24 -0700 Subject: [PATCH 44/57] debug --- .../datascience-notebook/requirements-3.7.12.txt | 0 .../{requirements.txt => requirements-3.8.13.txt} | 0 .../datascience-notebook/requirements-3.9.13.txt | 6 +++--- requirements-3.8.13.txt => requirements.txt | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename requirements-3.7.12.txt => kernels/datascience-notebook/requirements-3.7.12.txt (100%) rename kernels/datascience-notebook/{requirements.txt => requirements-3.8.13.txt} (100%) rename requirements-3.9.13.txt => kernels/datascience-notebook/requirements-3.9.13.txt (59%) rename requirements-3.8.13.txt => requirements.txt (100%) diff --git a/requirements-3.7.12.txt b/kernels/datascience-notebook/requirements-3.7.12.txt similarity index 100% rename from requirements-3.7.12.txt rename to kernels/datascience-notebook/requirements-3.7.12.txt diff --git a/kernels/datascience-notebook/requirements.txt b/kernels/datascience-notebook/requirements-3.8.13.txt similarity index 100% rename from kernels/datascience-notebook/requirements.txt rename to kernels/datascience-notebook/requirements-3.8.13.txt diff --git a/requirements-3.9.13.txt b/kernels/datascience-notebook/requirements-3.9.13.txt similarity index 59% rename from requirements-3.9.13.txt rename to kernels/datascience-notebook/requirements-3.9.13.txt index 110ef814..3177ffd7 100644 --- a/requirements-3.9.13.txt +++ b/kernels/datascience-notebook/requirements-3.9.13.txt @@ -5,8 +5,8 @@ cloudpickle==2.2.0 flytekitplugins-papermill==1.2.4 # https://github.com/noteable-io/ packages -git+https://www.github.com/noteable-io/dx.git@a7df2821182293546d7d7a9ede3cdcc0c946d570 -git+https://www.github.com/noteable-io/noteable-notebook-magics.git@a6a6801da8ea7ccb72b9f354d4780699c3d99f73 -git+https://www.github.com/noteable-io/sidecar_comms.git@6ee04efe60b855c465727f120f8f50a7bfa60097 +git+https://www.github.com/noteable-io/dx.git@4be0c105aea40248d066a1a8beb74ff00d0b5bd3 +git+https://www.github.com/noteable-io/noteable-notebook-magics.git@5d54f4cd94c46f617459db5e5a0110d081393936 +git+https://www.github.com/noteable-io/sidecar_comms.git@eed16c3ab900a8abe19b654fa775646bc38dd519 # (All of the datasources modules are now explicit requirements within noteable-notebook-magics.) \ No newline at end of file diff --git a/requirements-3.8.13.txt b/requirements.txt similarity index 100% rename from requirements-3.8.13.txt rename to requirements.txt From af437354b7fb0a9d324a0bed16dba47c1cca8975 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 15:26:43 -0700 Subject: [PATCH 45/57] debug --- kernels/datascience-notebook/requirements-3.7.12.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/kernels/datascience-notebook/requirements-3.7.12.txt b/kernels/datascience-notebook/requirements-3.7.12.txt index f2b02995..aa1378c7 100644 --- a/kernels/datascience-notebook/requirements-3.7.12.txt +++ b/kernels/datascience-notebook/requirements-3.7.12.txt @@ -1,6 +1,5 @@ # Scheduler/ochestration packages dagstermill==0.16.15 -papermill-origami==0.0.9 cloudpickle==2.2.0 flytekitplugins-papermill==1.2.4 # Added for ease of use with integration partners From 930f5a6ca84bc96f886e7bcd2ea8132d386eba1e Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 15:37:22 -0700 Subject: [PATCH 46/57] debug --- kernels/datascience-notebook/requirements-3.7.12.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernels/datascience-notebook/requirements-3.7.12.txt b/kernels/datascience-notebook/requirements-3.7.12.txt index aa1378c7..07ba2a23 100644 --- a/kernels/datascience-notebook/requirements-3.7.12.txt +++ b/kernels/datascience-notebook/requirements-3.7.12.txt @@ -3,7 +3,7 @@ dagstermill==0.16.15 cloudpickle==2.2.0 flytekitplugins-papermill==1.2.4 # Added for ease of use with integration partners -pyiceberg==0.3.0 +pyiceberg fugue==0.8.0 fugue-jupyter==0.2.2 From 2744114e49479d42b54d6a7a1a17ce0dc82f49f0 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 15:44:06 -0700 Subject: [PATCH 47/57] debug --- kernels/datascience-notebook/requirements-3.7.12.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernels/datascience-notebook/requirements-3.7.12.txt b/kernels/datascience-notebook/requirements-3.7.12.txt index 07ba2a23..cfd0b44c 100644 --- a/kernels/datascience-notebook/requirements-3.7.12.txt +++ b/kernels/datascience-notebook/requirements-3.7.12.txt @@ -8,7 +8,7 @@ fugue==0.8.0 fugue-jupyter==0.2.2 # https://github.com/noteable-io/ packages -git+https://www.github.com/noteable-io/dx.git@a7df2821182293546d7d7a9ede3cdcc0c946d570 +# git+https://www.github.com/noteable-io/dx.git@a7df2821182293546d7d7a9ede3cdcc0c946d570 git+https://www.github.com/noteable-io/noteable-notebook-magics.git@a3b00faedcba9f38c7b75b4f45f4eba7e1ce313e git+https://www.github.com/noteable-io/sidecar_comms.git@35b7cf8ad6b15daf020954c9029aaae5779f324a From f4bce285f9a50f1e24f42eaedb47f06f780e01c5 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 16:07:54 -0700 Subject: [PATCH 48/57] debug --- kernels/datascience-notebook/requirements-3.7.12.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernels/datascience-notebook/requirements-3.7.12.txt b/kernels/datascience-notebook/requirements-3.7.12.txt index cfd0b44c..a42fee4b 100644 --- a/kernels/datascience-notebook/requirements-3.7.12.txt +++ b/kernels/datascience-notebook/requirements-3.7.12.txt @@ -9,7 +9,7 @@ fugue-jupyter==0.2.2 # https://github.com/noteable-io/ packages # git+https://www.github.com/noteable-io/dx.git@a7df2821182293546d7d7a9ede3cdcc0c946d570 -git+https://www.github.com/noteable-io/noteable-notebook-magics.git@a3b00faedcba9f38c7b75b4f45f4eba7e1ce313e +# git+https://www.github.com/noteable-io/noteable-notebook-magics.git@a3b00faedcba9f38c7b75b4f45f4eba7e1ce313e git+https://www.github.com/noteable-io/sidecar_comms.git@35b7cf8ad6b15daf020954c9029aaae5779f324a # (All of the datasources modules are now explicit requirements within noteable-notebook-magics.) From 0035088149e3603b6b4b471c25bcdebd1ddd495f Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 16:20:36 -0700 Subject: [PATCH 49/57] debug --- kernels/datascience-notebook/requirements-3.7.12.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernels/datascience-notebook/requirements-3.7.12.txt b/kernels/datascience-notebook/requirements-3.7.12.txt index a42fee4b..dee4ec43 100644 --- a/kernels/datascience-notebook/requirements-3.7.12.txt +++ b/kernels/datascience-notebook/requirements-3.7.12.txt @@ -1,5 +1,5 @@ # Scheduler/ochestration packages -dagstermill==0.16.15 +dagstermill cloudpickle==2.2.0 flytekitplugins-papermill==1.2.4 # Added for ease of use with integration partners From 1684f0c13ffb96d1b8e617f1db793b6369693134 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 16:31:13 -0700 Subject: [PATCH 50/57] debug --- kernels/datascience-notebook/requirements-3.7.12.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernels/datascience-notebook/requirements-3.7.12.txt b/kernels/datascience-notebook/requirements-3.7.12.txt index dee4ec43..9e82d796 100644 --- a/kernels/datascience-notebook/requirements-3.7.12.txt +++ b/kernels/datascience-notebook/requirements-3.7.12.txt @@ -1,7 +1,7 @@ # Scheduler/ochestration packages dagstermill cloudpickle==2.2.0 -flytekitplugins-papermill==1.2.4 +flytekitplugins-papermill # Added for ease of use with integration partners pyiceberg fugue==0.8.0 From f69da7545881952682acec1e85c922ac811dbea6 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 22:16:29 -0700 Subject: [PATCH 51/57] debug --- kernels/datascience-notebook/requirements-3.7.12.txt | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/kernels/datascience-notebook/requirements-3.7.12.txt b/kernels/datascience-notebook/requirements-3.7.12.txt index 9e82d796..19deef2a 100644 --- a/kernels/datascience-notebook/requirements-3.7.12.txt +++ b/kernels/datascience-notebook/requirements-3.7.12.txt @@ -1,7 +1,7 @@ # Scheduler/ochestration packages -dagstermill +# dagstermill cloudpickle==2.2.0 -flytekitplugins-papermill +# flytekitplugins-papermill # Added for ease of use with integration partners pyiceberg fugue==0.8.0 @@ -10,6 +10,11 @@ fugue-jupyter==0.2.2 # https://github.com/noteable-io/ packages # git+https://www.github.com/noteable-io/dx.git@a7df2821182293546d7d7a9ede3cdcc0c946d570 # git+https://www.github.com/noteable-io/noteable-notebook-magics.git@a3b00faedcba9f38c7b75b4f45f4eba7e1ce313e -git+https://www.github.com/noteable-io/sidecar_comms.git@35b7cf8ad6b15daf020954c9029aaae5779f324a +# git+https://www.github.com/noteable-io/sidecar_comms.git@35b7cf8ad6b15daf020954c9029aaae5779f324a + +# Conflict +# dagstermill 0.3.0 depends on ipykernel>=4.9.0 +# flytekitplugins-papermill 0.1.0 depends on ipykernel>=5.0.0 +# sidecar-comms 0.1.0 depends on ipykernel<7.0.0 and >=6.20.2 # (All of the datasources modules are now explicit requirements within noteable-notebook-magics.) From 6a653edb7d96e7f003d690f8c55c70e195c2f73a Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 22:24:12 -0700 Subject: [PATCH 52/57] debug --- kernels/datascience-notebook/requirements-3.7.12.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernels/datascience-notebook/requirements-3.7.12.txt b/kernels/datascience-notebook/requirements-3.7.12.txt index 19deef2a..aa46a76e 100644 --- a/kernels/datascience-notebook/requirements-3.7.12.txt +++ b/kernels/datascience-notebook/requirements-3.7.12.txt @@ -3,7 +3,7 @@ cloudpickle==2.2.0 # flytekitplugins-papermill # Added for ease of use with integration partners -pyiceberg +# pyiceberg fugue==0.8.0 fugue-jupyter==0.2.2 From 8117176d74fade438f91af823a3afe9e7ec11752 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 22:35:03 -0700 Subject: [PATCH 53/57] debug --- kernels/datascience-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernels/datascience-notebook/Dockerfile b/kernels/datascience-notebook/Dockerfile index 180d910d..bc132097 100644 --- a/kernels/datascience-notebook/Dockerfile +++ b/kernels/datascience-notebook/Dockerfile @@ -81,7 +81,7 @@ RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension # Smoke test to ensure packages were installed properly # hadolint ignore=SC1008 -RUN python -c "import noteable_magics" +RUN if [ "$PYTHON_VERSION" != "3.7.12" ]; then python -c "import noteable_magics"; fi RUN git config --global user.name "Noteable Kernel" && \ git config --global user.email "engineering@noteable.io" && \ From 4cff6736e5a80a60702a942e5a81a4ef1f183f53 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 22:58:10 -0700 Subject: [PATCH 54/57] debug --- .github/workflows/main.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index e312de23..391e236c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -92,7 +92,6 @@ jobs: - name: Build image env: - DOCKER_CONTENT_TRUST: 1 DOCKER_CONTEXT: github-action run: | ( @@ -117,4 +116,6 @@ jobs: env: DOCKER_CONTEXT: github-action run: | + docker images + docker push --all-tags ${{ steps.build-args.outputs.FULL_IMAGE_NAME }} From 22d87d3e70b4b99c26acf1a1e2c1c426b09af53c Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 23:14:37 -0700 Subject: [PATCH 55/57] debug --- .github/workflows/main.yaml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 391e236c..3ca1c137 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -99,10 +99,12 @@ jobs: docker buildx build \ --pull \ + --push \ --output 'type=docker' \ --platform=linux/amd64 \ --progress plain \ - --no-cache \ + --cache-from 'type=local,src=/tmp/buildx-cache' \ + --cache-to 'type=local,dest=/tmp/buildx-cache' \ --tag '${{ steps.build-args.outputs.FULL_IMAGE_NAME_TAGGED }}' \ --build-arg PYTHON_VERSION=${{ matrix.version }} \ --build-arg 'NBL_ARG_BUILD_TIMESTAMP=${{ steps.build-args.outputs.BUILD_TIMESTAMP }}' \ @@ -111,11 +113,3 @@ jobs: --build-arg 'NBL_ARG_VERSION=${{ github.ref }}' \ . ) - - - name: Publish image - env: - DOCKER_CONTEXT: github-action - run: | - docker images - - docker push --all-tags ${{ steps.build-args.outputs.FULL_IMAGE_NAME }} From b565344d98703076503677d302f166f6dee67ce0 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Thu, 9 Mar 2023 23:18:08 -0700 Subject: [PATCH 56/57] debug --- .github/workflows/main.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 3ca1c137..8aec5589 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -100,7 +100,6 @@ jobs: docker buildx build \ --pull \ --push \ - --output 'type=docker' \ --platform=linux/amd64 \ --progress plain \ --cache-from 'type=local,src=/tmp/buildx-cache' \ From caaa4e2da164214f5c8d52856ca8906288877f05 Mon Sep 17 00:00:00 2001 From: andersdberg Date: Fri, 10 Mar 2023 10:39:45 -0700 Subject: [PATCH 57/57] add 3.10.9 --- .github/workflows/main.yaml | 2 +- kernels/datascience-notebook/environment-3.10.9.txt | 7 +++++++ kernels/datascience-notebook/requirements-3.10.9.txt | 12 ++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 kernels/datascience-notebook/environment-3.10.9.txt create mode 100644 kernels/datascience-notebook/requirements-3.10.9.txt diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 8aec5589..96fa9bc3 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -18,7 +18,7 @@ jobs: # version: ["3.8.8", "3.9.13", "3.10.5"] 3.10.5 fails with dependency conflicts # We may need to have separate requirements.txt for each version, or replace # dependencies on the fly - version: ["3.7.12", "3.8.13", "3.9.13"] + version: ["3.7.12", "3.8.13", "3.9.13", "3.10.9"] directory: ["datascience-notebook"] # The datascience-notebook base image does not support ARM # We would need to build and maintain our own base image diff --git a/kernels/datascience-notebook/environment-3.10.9.txt b/kernels/datascience-notebook/environment-3.10.9.txt new file mode 100644 index 00000000..20d65f2f --- /dev/null +++ b/kernels/datascience-notebook/environment-3.10.9.txt @@ -0,0 +1,7 @@ +jupyter_client=7.3.* +ipython=8.0.* +vdom=0.6 +papermill=2.2.* +ipywidgets=7.6.* +plotly=4.14.3 +geopandas=0.11.0 \ No newline at end of file diff --git a/kernels/datascience-notebook/requirements-3.10.9.txt b/kernels/datascience-notebook/requirements-3.10.9.txt new file mode 100644 index 00000000..3177ffd7 --- /dev/null +++ b/kernels/datascience-notebook/requirements-3.10.9.txt @@ -0,0 +1,12 @@ +# Scheduler/ochestration packages +dagstermill==0.16.15 +papermill-origami==0.0.9 +cloudpickle==2.2.0 +flytekitplugins-papermill==1.2.4 + +# https://github.com/noteable-io/ packages +git+https://www.github.com/noteable-io/dx.git@4be0c105aea40248d066a1a8beb74ff00d0b5bd3 +git+https://www.github.com/noteable-io/noteable-notebook-magics.git@5d54f4cd94c46f617459db5e5a0110d081393936 +git+https://www.github.com/noteable-io/sidecar_comms.git@eed16c3ab900a8abe19b654fa775646bc38dd519 + +# (All of the datasources modules are now explicit requirements within noteable-notebook-magics.) \ No newline at end of file