From 247eec5f4e812429a2bf575faf1cc0b52dcbe14c Mon Sep 17 00:00:00 2001 From: Piotr Fus Date: Thu, 12 Oct 2023 12:24:39 +0200 Subject: [PATCH] SNOW-937199 Provide images for various OpenJDK docker images --- ci/_init.sh | 28 ++++-- ci/build.sh | 4 +- ci/image/.gitignore | 1 + ci/image/Dockerfile.jdbc-centos7-openjdk-test | 89 +++++++++++++++++++ ci/image/build.sh | 20 ++--- ci/image/update.sh | 15 +--- 6 files changed, 123 insertions(+), 34 deletions(-) create mode 100644 ci/image/Dockerfile.jdbc-centos7-openjdk-test diff --git a/ci/_init.sh b/ci/_init.sh index df932feb1..d1e585b85 100755 --- a/ci/_init.sh +++ b/ci/_init.sh @@ -1,4 +1,5 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e export PLATFORM=$(echo $(uname) | tr '[:upper:]' '[:lower:]') export INTERNAL_REPO=nexus.int.snowflakecomputing.com:8086 @@ -15,18 +16,27 @@ mkdir -p $WORKSPACE export DRIVER_NAME=jdbc -# Build images -BUILD_IMAGE_VERSION=1 - # Test Images TEST_IMAGE_VERSION=1 -declare -A BUILD_IMAGE_NAMES=( -) -export BUILD_IMAGE_NAMES - declare -A TEST_IMAGE_NAMES=( - [$DRIVER_NAME-centos6-default]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos6-default-test:$BUILD_IMAGE_VERSION + [$DRIVER_NAME-centos6-default]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos6-default-test:$TEST_IMAGE_VERSION + [$DRIVER_NAME-centos7-openjdk8]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos7-openjdk8-test:$TEST_IMAGE_VERSION + [$DRIVER_NAME-centos7-openjdk11]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos7-openjdk11-test:$TEST_IMAGE_VERSION + [$DRIVER_NAME-centos7-openjdk17]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos7-openjdk17-test:$TEST_IMAGE_VERSION ) export TEST_IMAGE_NAMES +declare -A TEST_IMAGE_DOCKERFILES=( + [$DRIVER_NAME-centos6-default]=jdbc-centos6-default-test + [$DRIVER_NAME-centos7-openjdk8]=jdbc-centos7-openjdk-test + [$DRIVER_NAME-centos7-openjdk11]=jdbc-centos7-openjdk-test + [$DRIVER_NAME-centos7-openjdk17]=jdbc-centos7-openjdk-test +) + +declare -A TEST_IMAGE_BUILD_ARGS=( + [$DRIVER_NAME-centos7-openjdk8]="--target jdbc-centos7-openjdk-yum --build-arg=JDK_PACKAGE=java-1.8.0-openjdk-devel" + [$DRIVER_NAME-centos7-openjdk11]="--target jdbc-centos7-openjdk-yum --build-arg=JDK_PACKAGE=java-11-openjdk-devel" # pragma: allowlist secret + [$DRIVER_NAME-centos7-openjdk17]="--target jdbc-centos7-openjdk17" +) + diff --git a/ci/build.sh b/ci/build.sh index 902523c3d..e933479c5 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -1,4 +1,6 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e + # # Build JDBC driver # diff --git a/ci/image/.gitignore b/ci/image/.gitignore index 523ff310c..3527545e4 100644 --- a/ci/image/.gitignore +++ b/ci/image/.gitignore @@ -1,2 +1,3 @@ pom.xml dependencies/ +*.jar \ No newline at end of file diff --git a/ci/image/Dockerfile.jdbc-centos7-openjdk-test b/ci/image/Dockerfile.jdbc-centos7-openjdk-test new file mode 100644 index 000000000..4c4dd17b6 --- /dev/null +++ b/ci/image/Dockerfile.jdbc-centos7-openjdk-test @@ -0,0 +1,89 @@ +FROM centos:7 AS jdbc-centos7-openjdk-base +ARG JDK_PACKAGE + +# update OS +RUN yum -y update && \ + yum -y install epel-release && \ + yum -y install centos-release-scl + +# install Development tools +RUN yum -y groupinstall "Development Tools" && \ + yum -y install zlib-devel + +# git +RUN curl -o - https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.26.0.tar.gz | tar xfz - && \ + cd git-2.26.0 && \ + ./configure --prefix=/opt/git && make && make install && \ + ln -s /opt/git/bin/git /usr/local/bin/git + +# python +RUN yum -y install rh-python36 +COPY scripts/python3.6.sh /usr/local/bin/python3.6 +COPY scripts/python3.6.sh /usr/local/bin/python3 +RUN chmod a+x /usr/local/bin/python3.6 /usr/local/bin/python3 +COPY scripts/pip.sh /usr/local/bin/pip +RUN chmod a+x /usr/local/bin/pip +RUN pip install -U pip +RUN pip install -U snowflake-connector-python + +# aws +RUN pip install -U awscli +COPY scripts/aws.sh /usr/local/bin/aws +RUN chmod a+x /usr/local/bin/aws + +# zstd +RUN yum -y install zstd + +# jq +RUN yum -y install jq + +# gosu +RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/1.11/gosu-amd64" +RUN chmod +x /usr/local/bin/gosu +COPY scripts/entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh + +# Maven +RUN curl -o - https://dlcdn.apache.org/maven/maven-3/3.9.5/binaries/apache-maven-3.9.5-bin.tar.gz | tar xfz - -C /opt && \ + ln -s /opt/apache-maven-3.9.5/bin/mvn /usr/local/bin/mvn + +# workspace +RUN mkdir -p /home/user && \ + chmod 777 /home/user +WORKDIR /home/user + +COPY pom.xml /root +COPY dependencies /root/dependencies + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] + +###### OpenJDK from yum +FROM jdbc-centos7-openjdk-base AS jdbc-centos7-openjdk-yum + +# Java +RUN yum -y install $JDK_PACKAGE + +RUN cd /root && \ + mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ + -Dnot-self-contained-jar \ + --batch-mode --fail-never compile && \ + mv $HOME/.m2 /home/user && \ + chmod -R 777 /home/user/.m2 + +###### OpenJDK 17 from archive (not available in yum) +FROM jdbc-centos7-openjdk-base AS jdbc-centos7-openjdk17 + +# Java +RUN curl -o - https://download.java.net/java/GA/jdk17.0.2/dfd4a8d0985749f896bed50d7138ee7f/8/GPL/openjdk-17.0.2_linux-x64_bin.tar.gz | tar xfz - -C /opt && \ + ln -s /opt/jdk-17.0.2 /opt/jdk-17 + +RUN echo export JAVA_HOME=/opt/jdk-17 >> ~/.bashrc +RUN echo export PATH=\$JAVA_HOME/bin:\$PATH >> ~/.bashrc + +RUN export JAVA_HOME=/opt/jdk-17 && \ + cd /root && \ + mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ + -Dnot-self-contained-jar \ + --batch-mode --fail-never compile && \ + mv $HOME/.m2 /home/user && \ + chmod -R 777 /home/user/.m2 \ No newline at end of file diff --git a/ci/image/build.sh b/ci/image/build.sh index cce0000b6..81c474b4a 100755 --- a/ci/image/build.sh +++ b/ci/image/build.sh @@ -1,4 +1,5 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e # # Build Docker images # @@ -7,22 +8,17 @@ THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source $THIS_DIR/../_init.sh cp -p $THIS_DIR/../../pom.xml $THIS_DIR -cp -rp $THIS_DIR/../../dependencies/ $THIS_DIR - -for name in "${!BUILD_IMAGE_NAMES[@]}"; do - docker build \ - --pull \ - --file $THIS_DIR/Dockerfile.$name-build \ - --label snowflake \ - --label $DRIVER_NAME \ - --tag ${BUILD_IMAGE_NAMES[$name]} . -done +mkdir -p dependencies && cp -rp $THIS_DIR/../../dependencies/ $THIS_DIR/dependencies for name in "${!TEST_IMAGE_NAMES[@]}"; do + echo "Building $name" docker build \ + --progress=plain \ + --platform=linux/x86_64 \ --pull \ - --file $THIS_DIR/Dockerfile.$name-test \ + --file $THIS_DIR/Dockerfile.$(echo ${TEST_IMAGE_DOCKERFILES[$name]}) \ --label snowflake \ --label $DRIVER_NAME \ + $(echo ${TEST_IMAGE_BUILD_ARGS[$name]}) \ --tag ${TEST_IMAGE_NAMES[$name]} . done diff --git a/ci/image/update.sh b/ci/image/update.sh index f1642d899..07ffd9fd2 100755 --- a/ci/image/update.sh +++ b/ci/image/update.sh @@ -1,4 +1,6 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e + # # Build Docker images # @@ -12,17 +14,6 @@ source $THIS_DIR/../scripts/login_docker.sh for image in $(docker images --format "{{.ID}},{{.Repository}}:{{.Tag}}" | grep "nexus.int.snowflakecomputing.com" | grep "client-$DRIVER_NAME"); do target_id=$(echo $image | awk -F, '{print $1}') target_name=$(echo $image | awk -F, '{print $2}') - for name in "${!BUILD_IMAGE_NAMES[@]}"; do - if [[ "$target_name" == "${BUILD_IMAGE_NAMES[$name]}" ]]; then - echo $name - docker_hub_image_name=$(echo ${BUILD_IMAGE_NAMES[$name]/$DOCKER_REGISTRY_NAME/snowflakedb}) - set -x - docker tag $target_id $docker_hub_image_name - set +x - docker push "${BUILD_IMAGE_NAMES[$name]}" - docker push "$docker_hub_image_name" - fi - done for name in "${!TEST_IMAGE_NAMES[@]}"; do if [[ "$target_name" == "${TEST_IMAGE_NAMES[$name]}" ]]; then echo $name