From 4cb8188794a0a25926cfe5a91e18a03ad2268ee9 Mon Sep 17 00:00:00 2001 From: Dimitris Moraitis <1026101+d-mo@users.noreply.github.com> Date: Thu, 18 Apr 2024 13:00:51 +0300 Subject: [PATCH 1/2] Add initial GHCR build workflow --- .github/workflows/docker-image.yml | 52 ++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/docker-image.yml diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 0000000..66ccaba --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,52 @@ +name: Docker Image CI + +on: + push: + branches: [ "main" ] + tags: [ 'v*.*.*' ] + pull_request: + branches: [ "main" ] + +jobs: + build_operator_image: + runs-on: self-hosted + steps: + - name: 'Checkout GitHub Action' + uses: actions/checkout@main + - name: 'Login to GitHub Container Registry' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{github.actor}} + password: ${{secrets.GITHUB_TOKEN}} + - uses: actions/checkout@v3 + - name: Build the Docker image + run: | + if [[ $GITHUB_REF == 'refs/heads/main' ]]; then + TAG=latest + else + TAG=$(echo ${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}| tr / -) + fi + docker build . --file Dockerfile --tag ghcr.io/dell/remote-development-operator:$TAG + docker push ghcr.io/dell/remote-development-operator:$TAG + build_devenv_image: + runs-on: self-hosted + steps: + - name: 'Checkout GitHub Action' + uses: actions/checkout@main + - name: 'Login to GitHub Container Registry' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{github.actor}} + password: ${{secrets.GITHUB_TOKEN}} + - uses: actions/checkout@v3 + - name: Build the Docker image + run: | + if [[ $GITHUB_REF == 'refs/heads/main' ]]; then + TAG=latest + else + TAG=$(echo ${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}| tr / -) + fi + docker build . --file Dockerfile --tag ghcr.io/dell/remote-development-operator/devenv:$TAG + docker push ghcr.io/dell/remote-development-operator/devenv:$TAG From 2ba4a2e463432b0664eed48f418e35355a2eea37 Mon Sep 17 00:00:00 2001 From: Dimitris Moraitis Date: Mon, 22 Apr 2024 12:12:05 +0800 Subject: [PATCH 2/2] Add pre-commit workflow, remove gitlab-ci.yml --- .github/workflows/docker-image.yml | 12 +- .gitlab-ci.yml | 227 ----------------------------- operator/chart/values.yaml | 2 +- 3 files changed, 11 insertions(+), 230 deletions(-) delete mode 100644 .gitlab-ci.yml diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 66ccaba..bc38abf 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -8,6 +8,14 @@ on: branches: [ "main" ] jobs: + pre-commit: + runs-on: self-hosted + steps: + - name: Run pre-commit + run: | + pip install pre-commit + pre-commit install-hooks + pre-commit run -a --color=always --show-diff-on-failure build_operator_image: runs-on: self-hosted steps: @@ -25,7 +33,7 @@ jobs: if [[ $GITHUB_REF == 'refs/heads/main' ]]; then TAG=latest else - TAG=$(echo ${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}| tr / -) + TAG=$(echo ${GITHUB_HEAD_REF:-${GITHUB_REF#refs/tags/}}| tr / -) fi docker build . --file Dockerfile --tag ghcr.io/dell/remote-development-operator:$TAG docker push ghcr.io/dell/remote-development-operator:$TAG @@ -46,7 +54,7 @@ jobs: if [[ $GITHUB_REF == 'refs/heads/main' ]]; then TAG=latest else - TAG=$(echo ${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}| tr / -) + TAG=$(echo ${GITHUB_HEAD_REF:-${GITHUB_REF#refs/tags/}}| tr / -) fi docker build . --file Dockerfile --tag ghcr.io/dell/remote-development-operator/devenv:$TAG docker push ghcr.io/dell/remote-development-operator/devenv:$TAG diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index a5e8999..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,227 +0,0 @@ -workflow: - rules: - # Disable MR pipelines. - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - when: never - # Fallback rule allows all other pipelines. - - when: always - -include: -- project: frontier/mist/gitlab-ci-templates - file: pre-commit.yml - ref: main -- project: frontier/mist/gitlab-ci-templates - file: docker-build.yml - ref: main -# Defines .base job template. -- project: frontier/cloud-hosted-beta/eoaas-deployment-pipeline - file: gitlab-ci-templates/base.gitlab-ci.yml - ref: main -# Defines .crane and .crane-copy-multi-image templates. -- project: frontier/cloud-hosted-beta/eoaas-deployment-pipeline - file: gitlab-ci-templates/crane.gitlab-ci.yml - ref: main - -variables: - # Variables with descriptions so that a nice form is generated in the Gitlab UI - # when a pipeline gets triggered. - DEVENV_NAME: - description: Name of the DevEnv that will be created. - value: "" - APP_ENV: - description: Name of the app deployment that will be targeted. - value: "" - NAMESPACE: - description: Namespace where to deploy the DevEnv and look for the target services. - value: default - SELECTORS: - description: Labels of target EO services to develop for. - value: app=api-v2 - MOUNT_PATH: - description: The path in the target pod were the code will be mounted at. - value: "/mist" - MODE: - description: Choose if target pod should be modified or cloned. - value: "modify" - options: ["modify", "clone"] - PORT: - description: The port of the target pod that should be exposed when using the clone mode. - value: "8080" - RELOAD_SIGNAL: - description: The Unix signal to send to the target pod to reload the code. - value: "HUP" - options: ["TERM", "HUP"] - RELOAD_CMD: - description: The command to run to target pods when reloading the code. - value: "" - POST_MOUNT_POD_CMD: - description: The command to run after the target pod mounts the code. - EXCLUDED: - description: Excluded paths. - value: '[".git", ".vscode", "__pycache__", ".pyc", "/landing", "/portal", "/jenkins", "/settings", "/ui", "/tests", "/docker/nginx/static"]' - - # Remaining variables that will not be rendered in the Gitlab manual pipeline form. - OPERATOR_IMAGE_NAME: remote-development-operator - DEVENV_IMAGE_NAME: remote-development-environment - BASE_DOMAIN: dev.nativeedge.dell.com - - -# The stages are not really being used because all jobs define explicit dependencies -# with `needs` (DAG pipeline). -stages: -- test -- build -- publish -- deploy operator -- deploy devenv -- undeploy - - -# Build docker images. -build operator image: - extends: .docker_build # from included template - stage: build - variables: - EXT_REGISTRY: $ARTIFACTORY_LOCAL_REGISTRY - EXT_REGISTRY_USER: $ARTIFACTORY_DOCKER_PUSH_USER # Taken from project settings - EXT_REGISTRY_PASSWORD: $ARTIFACTORY_DOCKER_PUSH_PASSWORD # Taken from project settings - EXT_REGISTRY_IMAGE: $ARTIFACTORY_LOCAL_REGISTRY/$OPERATOR_IMAGE_NAME - script: docker_simple_build_and_push - needs: [] -build devenv image: - extends: build operator image - variables: - EXT_REGISTRY_IMAGE: $ARTIFACTORY_LOCAL_REGISTRY/$DEVENV_IMAGE_NAME - script: docker_simple_build_and_push devenv - -# Publish docker images to ECR. -publish operator image to ECR: - extends: .crane - stage: publish - script: copy_image $OPERATOR_IMAGE_NAME:$CI_COMMIT_SHA - needs: [build operator image] - -publish devenv image to ECR: - extends: .crane - stage: publish - script: copy_image $DEVENV_IMAGE_NAME:$CI_COMMIT_SHA - needs: [build devenv image] - -# Deploy operator. -deploy operator: - extends: .base - variables: - GIT_STRATEGY: fetch - stage: deploy operator - rules: - - if: $APP_ENV == "" - when: manual - - when: always - script: - - |- - if [ -z "$APP_ENV" ]; then - echo "Deploying operator on host cluster" - else - echo "Deploying operator inside $APP_ENV vcluster" - vcluster connect $APP_ENV - sleep 5 - fi - kubectl cluster-info - kubectl apply -f https://github.com/nolar/kopf/raw/main/peering.yaml || kubectl apply -f https://github.com/nolar/kopf/raw/main/peering.yaml # Retry needed sometimes - kubectl wait --for condition=established --timeout=60s CustomResourceDefinition/kopfpeerings.kopf.dev - kubectl apply -f operator/crd.yaml - helm upgrade --install --namespace $NAMESPACE \ - --set image.tag="$CI_COMMIT_SHA" \ - --wait --timeout=2m \ - remote-development-operator operator/chart - needs: - - publish operator image to ECR - - publish devenv image to ECR - -# Deploy devenv. -deploy devenv: - extends: .base - variables: - GIT_STRATEGY: fetch - stage: deploy devenv - needs: - - deploy operator - rules: - - if: $APP_ENV == "" - when: manual - - if: $DEVENV_NAME == "" - when: manual - - when: always - script: - - |- - if [ -z "$APP_ENV" ]; then - echo "Deploying DevEnv on host cluster" - else - echo "Deploying DevEnv" - vcluster connect $APP_ENV - echo "Connected to vcluster" - kubectl cluster-info - fi - echo "Excluded:" $EXCLUDED - echo "Mount path:" $MOUNT_PATH - echo "Selectors:" $SELECTORS - echo "Mode:" $MODE - echo "Port:" $PORT - echo "Reload signal:" $RELOAD_SIGNAL - echo "Reload command:" $RELOAD_CMD - echo "Post mount command:" $POST_MOUNT_POD_CMD - curl https://gitlab.dell.com/$GITLAB_USER_LOGIN.keys > user.keys - export GITHUB_USER_LOGIN=$(echo $GITLAB_USER_LOGIN|tr '_' '-') - curl -L -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GITHUB_DELL_TOKEN}" https://eos2git.cec.lab.emc.com/api/v3/users/$GITHUB_USER_LOGIN/keys | jq -r '.[].key' >> user.keys - export SSH_PUB_KEYS=$(sed 's/^/ - /g' user.keys|sed '1i\\') - export BASE_IMAGE=702886132326.dkr.ecr.eu-west-1.amazonaws.com/eo-images-gitlab/remote-development-environment - export MOUNTS="" - export KIND="deployment" - export GROUP_NAME="eoaas-development" - export ENTRYPOINTS="{}" - for SELECTOR in $SELECTORS - do - export LABELS=$(echo $SELECTOR |sed -r 's/=/:\ /g'|sed -r 's/,/\n/g'|sed -r 's/^/ /g'|sed '1i\\') - echo "The label is ${LABELS}" - export MOUNT="$(cat examples/mounts-template.yaml | envsubst)" - export MOUNTS="$MOUNTS - $MOUNT" - echo "$MOUNTS" - done - export IMAGE=$BASE_IMAGE:$CI_COMMIT_SHA - if [ -z "$DEVENV_NAME" ]; then export DEVENV_NAME=$(echo "$GITHUB_USER_LOGIN-dev-env" | tr '[:upper:]' '[:lower:]'); fi - cat examples/devenv-template.yaml | envsubst > devenv.yaml - cat devenv.yaml - kubectl apply -f devenv.yaml - sleep 1 - echo - echo - echo "You can configure your local IDE to synchronize your local code over SSH. If using vscode/vscodium/code-server, please install the \`emeraldwalk.runonsave\` extension and put the following contents inside \`.vscode/settings.json\` within your project" - cat << EOF - { - "emeraldwalk.runonsave": { - "commands": [ - { - "match": ".*", - "isAsync": true, - "cmd": $(kubectl get devenv -n $NAMESPACE $DEVENV_NAME -o json | jq ".status.create_update_dev_env.cmd") - } - ] - } - } - EOF - echo - echo "Waiting for DNS to propagate and for SSH to be up..." - export SSH_HOST="$DEVENV_NAME.$BASE_DOMAIN" - while ! $(nc -z "$SSH_HOST" 22 2> /dev/null); do - sleep 1 # wait for 1 second before check again - done - echo "SSH server is ready. Connect using \`ssh docker@$DEVENV_NAME.$BASE_DOMAIN\`" - -undeploy: - extends: .base - when: manual - stage: undeploy - script: - - kubectl delete devenv -n $NAMESPACE $DEVENV_NAME - - helm delete -n $NAMESPACE remote-development-operator diff --git a/operator/chart/values.yaml b/operator/chart/values.yaml index a1c13e4..64934dc 100644 --- a/operator/chart/values.yaml +++ b/operator/chart/values.yaml @@ -1,3 +1,3 @@ image: - repository: 702886132326.dkr.ecr.eu-west-1.amazonaws.com/eo-images-gitlab/remote-development-operator + repository: ghcr.io/dell/remote-development-operator tag: main