diff --git a/build.py b/build.py index f97743b..74f50ae 100755 --- a/build.py +++ b/build.py @@ -1,16 +1,71 @@ #!/usr/bin/env python3 -from subprocess import run +import subprocess import fire import json +import os import sys import tempfile +SUPPORT_PLATFORMS = ["linux/amd64", "linux/arm64"] + + +def get_current_platform(): + import platform + + system_os = platform.system().lower() + machine = platform.machine() + + if system_os != "linux": + raise ValueError(f"unsupported platform: {system_os}") + + cur_platform = None + if machine in ["x86_64", "amd64"]: + cur_platform = f"{system_os}/amd64" + elif platform.machine() in ["aarch64", "arm64"]: + cur_platform = f"{system_os}/arm64" + + assert cur_platform in SUPPORT_PLATFORMS + return cur_platform + def log(s): print(f"\033[96m{s}\033[0m") +def run(cmd, shell=True, check=True, interactive=False, **kw): + log(f"=> run: {cmd}") + if interactive: + choice = input("Contine? Enter for yes, other for Exit. ") + if choice.strip(): + sys.exit(0) + return subprocess.run(cmd, shell=shell, check=check, **kw) + + +def get_git_tag(cwd): + return ( + run( + "git describe --tag --abbrev=0", + stdout=subprocess.PIPE, + cwd=cwd, + ) + .stdout.decode() + .strip() + ) + + +def get_git_commit(cwd): + return ( + run( + "git rev-parse HEAD", + stdout=subprocess.PIPE, + cwd=cwd, + ) + .stdout.decode() + .strip() + ) + + def _buildx( dockerfile, image_name, @@ -21,9 +76,14 @@ def _buildx( write_to_docker=False, cwd=".", ): + if len(tags) < 1: + raise ValueError("tags must not be empty") + current_platform = get_current_platform() + if target: target = f"--target {target}" + # Build with tempfile.NamedTemporaryFile() as fp: fp.close() metadata_file = fp.name @@ -41,12 +101,11 @@ def _buildx( --output {output} \ --metadata-file {metadata_file} . """.strip() - log(f"-> run: {cmd}") run(cmd, shell=True, cwd=cwd, check=True) digest = json.load(open(metadata_file))["containerimage.digest"] + log(f"-> [build] {image_name} digest: {digest}") - log(f"-> {image_name} digest: {digest}") - + # FIXME: merge into the Build step # [Optional] writes to local image store, so it will appear in `docker images` if write_to_docker: for tag in tags: @@ -55,35 +114,65 @@ def _buildx( --file {dockerfile} {target} {build_args} \ --output type=docker,name={image_name}:{tag} . """.strip() - log(f"-> run: {cmd}") run(cmd, shell=True, cwd=cwd, check=True) + # Push + info_format = '--format "{{json .Manifest}}"' + fetch_info_cmd = ( + f"docker buildx imagetools inspect {info_format} {image_name}:{tags[0]}" + ) + image_info = run( + fetch_info_cmd, + shell=True, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + if image_info.returncode != 0: + if "not found" in image_info.stdout.decode(): + exist_manifests = [] + else: + raise RuntimeError(image_info.stdout.decode()) + else: + exist_manifests = json.loads(image_info.stdout).get("manifests") + if not exist_manifests: + # FIXME: get architecture from exist image(no manifests) + # + # for now we assume the exist image is for linux/amd64 + exist_manifests = [ + { + "platform": {"os": "linux", "architecture": "amd64"}, + "digest": exist_manifests["digest"], + } + ] + digests_to_push = set([digest]) + for m in exist_manifests: + platform = m["platform"]["os"] + "/" + m["platform"]["architecture"] + if platform in SUPPORT_PLATFORMS and platform != current_platform: + digests_to_push.add(m["digest"]) + + tag_param = " ".join(f"--tag {image_name}:{tag}" for tag in tags) + digest_param = " ".join(f"{image_name}@{d}" for d in digests_to_push) + log(f"-> [push] tags: {tag_param}, digests: {digest_param}") if push: - # Push tag from digests - # - # TODO: update digest for the architecture - # - # manifests=$(docker buildx imagetools inspect --raw $image_name) - # parse_manifests="import json; d = $manifests; \ - # print(d['manifests']) \ - # for m in d['manifests'] - # if m['platform']['architecture'] == ... - # " - # x=$(python3 -c "$parse_manifests") - tag_param = " ".join(f"--tag {image_name}:{tag}" for tag in tags) - cmd = ( - f"docker buildx imagetools create {tag_param} {image_name}@{digest}" - ) - log(f"-> run: {cmd}") + cmd = f"docker buildx imagetools create {tag_param} {digest_param}" run(cmd, shell=True, cwd=cwd) -# TODO -def setup(): - # git clone --recurse-submodules https://github.com/hstreamdb/LogDevice.git - # git clone --recurse-submodules https://github.com/hstreamdb/docker-haskell.git - # git clone --recurse-submodules https://github.com/hstreamdb/hstream.git - # cd $LD_DIR && git checkout -b stable origin/stable +def setup(work_dir=".", use_container_builder=True): + ld_dir = os.path.join(work_dir, "LogDevice") + hs_dir = os.path.join(work_dir, "docker-haskell") + cmd = f""" + git clone --recurse-submodules https://github.com/hstreamdb/LogDevice.git {ld_dir} + git clone --recurse-submodules https://github.com/hstreamdb/docker-haskell.git {hs_dir} + """ + run(cmd, shell=True, check=True) + run( + "git checkout -b stable origin/stable", + shell=True, + cwd=ld_dir, + check=True, + ) # XXX: Required for push-by-digest feature # @@ -92,16 +181,19 @@ def setup(): # # - https://github.com/docker/buildx/issues/1453 # - https://github.com/moby/buildkit/issues/2343 - # - # cmd = "docker buildx create --use --name build --node build --driver-opt network=host" - print("TODO") + if use_container_builder: + cmd = "docker buildx create --use --name build --node build --driver-opt network=host" + run(cmd, shell=True, check=True) def logdevice_builder( ld_dir="./LogDevice", no_push=False, no_write_to_docker=False ): - # FIXME: also update submodules? - run("git checkout stable", shell=True, cwd=ld_dir) + run( + "git checkout stable && git submodule update --init --recursive", + shell=True, + cwd=ld_dir, + ) _buildx( "docker/Dockerfile.builder", "hstreamdb/logdevice-builder", @@ -113,8 +205,11 @@ def logdevice_builder( def logdevice(ld_dir="./LogDevice", no_push=False, no_write_to_docker=False): - # FIXME: also update submodules? - run("git checkout stable", shell=True, cwd=ld_dir) + run( + "git checkout stable && git submodule update --init --recursive", + shell=True, + cwd=ld_dir, + ) _buildx( "docker/Dockerfile", "hstreamdb/logdevice", @@ -137,8 +232,11 @@ def logdevice(ld_dir="./LogDevice", no_push=False, no_write_to_docker=False): def logdevice_builder_rqlite( ld_dir="./LogDevice", no_push=False, no_write_to_docker=False ): - # FIXME: also update submodules? - run("git checkout main", shell=True, cwd=ld_dir) + run( + "git checkout main && git submodule update --init --recursive", + shell=True, + cwd=ld_dir, + ) _buildx( "docker/Dockerfile.builder", "hstreamdb/logdevice-builder", @@ -152,8 +250,11 @@ def logdevice_builder_rqlite( def logdevice_rqlite( ld_dir="./LogDevice", no_push=False, no_write_to_docker=False ): - # FIXME: also update submodules? - run("git checkout main", shell=True, cwd=ld_dir) + run( + "git checkout main && git submodule update --init --recursive", + shell=True, + cwd=ld_dir, + ) _buildx( "docker/Dockerfile", "hstreamdb/logdevice", @@ -173,9 +274,57 @@ def logdevice_rqlite( ) -# TODO -# grpc -# ghc +def grpc( + hs_dir="./docker-haskell", + no_push=False, + no_write_to_docker=False, + version="1.54.2", +): + _buildx( + "dockerfiles/grpc", + "ghcr.io/hstreamdb/grpc", + [version], + build_args=f"--build-arg GRPC=v{version}", + push=not no_push, + write_to_docker=not no_write_to_docker, + cwd=hs_dir, + ) + + +def _ghc(ghcs, hs_dir, no_push, no_write_to_docker): + _buildx( + "dockerfiles/ghc_from_haskell", + "ghcr.io/hstreamdb/ghc", + ghcs, + build_args=f"--build-arg GHC={ghcs[0]}", + push=not no_push, + write_to_docker=not no_write_to_docker, + cwd=hs_dir, + ) + + +def ghc810(hs_dir="./docker-haskell", no_push=False, no_write_to_docker=False): + _ghc(["8.10.7", "8.10"], hs_dir, no_push, no_write_to_docker) + + +def ghc904(hs_dir="./docker-haskell", no_push=False, no_write_to_docker=False): + _ghc(["9.4.8", "9.4"], hs_dir, no_push, no_write_to_docker) + + +def ghc906(hs_dir="./docker-haskell", no_push=False, no_write_to_docker=False): + _ghc(["9.6.5", "9.6"], hs_dir, no_push, no_write_to_docker) + + +def ghc(hs_dir="./docker-haskell", no_push=False, no_write_to_docker=False): + ghc810( + hs_dir=hs_dir, no_push=no_push, no_write_to_docker=no_write_to_docker + ) + ghc904( + hs_dir=hs_dir, no_push=no_push, no_write_to_docker=no_write_to_docker + ) + ghc906( + hs_dir=hs_dir, no_push=no_push, no_write_to_docker=no_write_to_docker + ) def hsthrift( @@ -204,59 +353,93 @@ def hadmin_store( ) -def haskell810( - hs_dir="./docker-haskell", no_push=False, no_write_to_docker=False -): +def _haskell(tags, ghc, ld_image, hs_dir, no_push, no_write_to_docker): _buildx( "Dockerfile", "hstreamdb/haskell", - ["8.10.7", "8.10"], - build_args="--build-arg GHC=8.10.7 --build-arg LD_CLIENT_IMAGE=hstreamdb/logdevice-client", + tags, + build_args=f"--build-arg GHC={ghc} --build-arg LD_CLIENT_IMAGE={ld_image}", push=not no_push, write_to_docker=not no_write_to_docker, cwd=hs_dir, ) +def haskell810( + hs_dir="./docker-haskell", no_push=False, no_write_to_docker=False +): + _haskell( + ["8.10.7", "8.10"], + "8.10.7", + "hstreamdb/logdevice-client", + hs_dir, + no_push, + no_write_to_docker, + ) + + def haskell810_rqlite( hs_dir="./docker-haskell", no_push=False, no_write_to_docker=False ): - _buildx( - "Dockerfile", - "hstreamdb/haskell", + _haskell( ["rqlite_8.10.7", "rqlite_8.10"], - build_args="--build-arg GHC=8.10.7 --build-arg LD_CLIENT_IMAGE=hstreamdb/logdevice-client:rqlite", - push=not no_push, - write_to_docker=not no_write_to_docker, - cwd=hs_dir, + "8.10.7", + "hstreamdb/logdevice-client:rqlite", + hs_dir, + no_push, + no_write_to_docker, ) def haskell904( hs_dir="./docker-haskell", no_push=False, no_write_to_docker=False ): - _buildx( - "Dockerfile", - "hstreamdb/haskell", + _haskell( ["9.4.8", "9.4", "latest"], - build_args="--build-arg GHC=9.4.8 --build-arg LD_CLIENT_IMAGE=hstreamdb/logdevice-client", - push=not no_push, - write_to_docker=not no_write_to_docker, - cwd=hs_dir, + "9.4.8", + "hstreamdb/logdevice-client", + hs_dir, + no_push, + no_write_to_docker, ) def haskell904_rqlite( hs_dir="./docker-haskell", no_push=False, no_write_to_docker=False ): - _buildx( - "Dockerfile", - "hstreamdb/haskell", + _haskell( ["rqlite_9.4.8", "rqlite_9.4", "rqlite"], - build_args="--build-arg GHC=9.4.8 --build-arg LD_CLIENT_IMAGE=hstreamdb/logdevice-client:rqlite", - push=not no_push, - write_to_docker=not no_write_to_docker, - cwd=hs_dir, + "9.4.8", + "hstreamdb/logdevice-client:rqlite", + hs_dir, + no_push, + no_write_to_docker, + ) + + +def haskell906( + hs_dir="./docker-haskell", no_push=False, no_write_to_docker=False +): + _haskell( + ["9.6.5", "9.6"], + "9.6.5", + "hstreamdb/logdevice-client", + hs_dir, + no_push, + no_write_to_docker, + ) + + +def haskell906_rqlite( + hs_dir="./docker-haskell", no_push=False, no_write_to_docker=False +): + _haskell( + ["rqlite_9.6.5", "rqlite_9.6"], + "9.6.5", + "hstreamdb/logdevice-client:rqlite", + hs_dir, + no_push, + no_write_to_docker, ) @@ -273,10 +456,77 @@ def haskell(hs_dir="./docker-haskell", no_push=False, no_write_to_docker=False): haskell904_rqlite( hs_dir=hs_dir, no_push=no_push, no_write_to_docker=no_write_to_docker ) + haskell906( + hs_dir=hs_dir, no_push=no_push, no_write_to_docker=no_write_to_docker + ) + haskell906_rqlite( + hs_dir=hs_dir, no_push=no_push, no_write_to_docker=no_write_to_docker + ) + +# TODO: hstream_builder -# TODO -# hstream + +def hstream( + hstream_dir="./hstream", + tag="v0.19.5", + build_cache="no_cache", # "no_cache" or "cache", currently arm builds have to use "no_cache" + no_push=False, + no_write_to_docker=False, +): + if not os.path.exists(hstream_dir): + run( + f"git clone --recurse-submodules -b {tag} https://github.com/hstreamdb/hstream.git {hstream_dir}" + ) + git_tag = get_git_tag(hstream_dir) + git_commit = get_git_commit(hstream_dir) + build_args = f""" + --build-arg HS_IMAGE=hstreamdb/haskell:9.4 \ + --build-arg LD_IMAGE=hstreamdb/logdevice:latest \ + --build-arg BUILD_CACHE={build_cache} \ + --build-arg HSTREAM_VERSION={git_tag} \ + --build-arg HSTREAM_VERSION_COMMIT={git_commit} + """.strip() + _buildx( + "docker/Dockerfile", + "hstreamdb/hstream", + [tag], + build_args=build_args, + push=not no_push, + write_to_docker=not no_write_to_docker, + cwd=hstream_dir, + ) + + +def hstream_rqlite( + hstream_dir="./hstream", + tag="v0.19.5", + build_cache="no_cache", # "no_cache" or "cache", currently arm builds have to use "no_cache" + no_push=False, + no_write_to_docker=False, +): + if not os.path.exists(hstream_dir): + run( + f"git clone --recurse-submodules -b {tag} https://github.com/hstreamdb/hstream.git {hstream_dir}" + ) + git_tag = get_git_tag(hstream_dir) + git_commit = get_git_commit(hstream_dir) + build_args = f""" + --build-arg HS_IMAGE=hstreamdb/haskell:rqlite_9.4 \ + --build-arg LD_IMAGE=hstreamdb/logdevice:rqlite \ + --build-arg BUILD_CACHE={build_cache} \ + --build-arg HSTREAM_VERSION={git_tag} \ + --build-arg HSTREAM_VERSION_COMMIT={git_commit} + """.strip() + _buildx( + "docker/Dockerfile", + "hstreamdb/hstream", + ["rqlite_" + tag], + build_args=build_args, + push=not no_push, + write_to_docker=not no_write_to_docker, + cwd=hstream_dir, + ) if __name__ == "__main__": @@ -289,14 +539,21 @@ def haskell(hs_dir="./docker-haskell", no_push=False, no_write_to_docker=False): logdevice, logdevice_builder_rqlite, logdevice_rqlite, - # TODO: grpc, ghc + grpc, + ghc810, + ghc904, + ghc906, + ghc, hsthrift, hadmin_store, haskell810, haskell810_rqlite, haskell904, haskell904_rqlite, + haskell906, + haskell906_rqlite, haskell, - # TODO: hstream + hstream, + hstream_rqlite, ] fire.Fire({fn.__name__: fn for fn in fns}) diff --git a/dockerfiles/ghc_from_haskell b/dockerfiles/ghc_from_haskell index e76a70b..b939eaa 100644 --- a/dockerfiles/ghc_from_haskell +++ b/dockerfiles/ghc_from_haskell @@ -32,8 +32,13 @@ RUN apt-get update && \ update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-11 20 && \ rm -rf /var/lib/apt/lists/* -ARG CABAL_INSTALL=3.8.1.0 -ARG CABAL_INSTALL_RELEASE_KEY=E9EC5616017C3EE26B33468CCE1ED8AE0B011D8C +# ------------------------------------------------------------------------------ +# Cabal +# +# https://downloads.haskell.org/~cabal + +ARG CABAL_INSTALL=3.10.3.0 +ARG CABAL_INSTALL_RELEASE_KEY=EAF2A9A722C0C96F2B431CA511AAD8CEDEE0CAEF # sha256 from https://downloads.haskell.org/~cabal/cabal-install-$CABAL_INSTALL/SHA256SUMS RUN \ @@ -45,10 +50,10 @@ RUN \ CABAL_INSTALL_SHA256SUMS_URL="https://downloads.haskell.org/~cabal/cabal-install-$CABAL_INSTALL/SHA256SUMS"; \ case "$ARCH" in \ 'aarch64') \ - CABAL_INSTALL_SHA256='c7fa9029f2f829432dd9dcf764e58605fbb7431db79234feb3e46684a9b37214'; \ + CABAL_INSTALL_SHA256='92d341620c60294535f03098bff796ef6de2701de0c4fcba249cde18a2923013'; \ ;; \ 'x86_64') \ - CABAL_INSTALL_SHA256='c71a1a46fd42d235bb86be968660815c24950e5da2d1ff4640da025ab520424b'; \ + CABAL_INSTALL_SHA256='1d7a7131402295b01f25be5373fde095a404c45f9b5a5508fb7474bb0d3d057a'; \ ;; \ *) echo >&2 "error: unsupported architecture '$ARCH'"; exit 1 ;; \ esac; \ @@ -68,8 +73,12 @@ RUN \ \ cabal --version -ARG GHC=9.2.8 -# Note that this key is not for ghc9.4+ +# ------------------------------------------------------------------------------ +# GHC +# +# https://downloads.haskell.org/~ghc + +ARG GHC=9.6.5 ARG GHC_RELEASE_KEY=88B57FCF7DB53B4DB3BFA4B1588764FBE22D19C4 # GHC 8.10 requires LLVM version 9 - 12 on aarch64 @@ -85,23 +94,28 @@ RUN \ set -eux; \ cd /tmp; \ ARCH="$(uname -m)"; \ - GHC_EXTRACT_NAME="ghc-$GHC"; \ + GHC_EXTRACT_NAME="ghc-$GHC-$ARCH-unknown-linux"; \ case "$ARCH" in \ 'aarch64') \ case "$GHC" in \ '8.10.7') \ GHC_URL="https://downloads.haskell.org/~ghc/$GHC/ghc-$GHC-$ARCH-deb10-linux.tar.xz"; \ - GHC_SHA256='fad2417f9b295233bf8ade79c0e6140896359e87be46cb61cd1d35863d9d0e55'; \ + GHC_SHA256="fad2417f9b295233bf8ade79c0e6140896359e87be46cb61cd1d35863d9d0e55"; \ + GHC_EXTRACT_NAME="ghc-$GHC"; \ ;; \ '9.2.8') \ GHC_URL="https://downloads.haskell.org/~ghc/$GHC/ghc-$GHC-$ARCH-deb10-linux.tar.xz"; \ GHC_SHA256='645433359d8ad9e7b286f85ef5111db1b787ee3712c24c5dfde7c62769aa59a4'; \ + GHC_EXTRACT_NAME="ghc-$GHC"; \ ;; \ '9.4.8') \ GHC_URL="https://downloads.haskell.org/~ghc/$GHC/ghc-$GHC-$ARCH-deb10-linux.tar.xz"; \ GHC_SHA256='278e287e1ee624712b9c6d7803d1cf915ca1cce56e013b0a16215eb8dfeb1531'; \ GHC_RELEASE_KEY='88b57fcf7db53b4db3bfa4b1588764fbe22d19c4'; \ - GHC_EXTRACT_NAME="ghc-$GHC-$ARCH-unknown-linux"; \ + ;; \ + '9.6.5') \ + GHC_URL="https://downloads.haskell.org/~ghc/$GHC/ghc-$GHC-$ARCH-deb10-linux.tar.xz"; \ + GHC_SHA256='fc4e2df394b9d6f48bdb0abaabad1c97825264cbe4e75e7463fc794cb3f87047'; \ ;; \ *) echo >&2 "error: unsupported ghc '$GHC'" ; exit 1 ;; \ esac; \ @@ -111,10 +125,12 @@ RUN \ '8.10.7') \ GHC_URL="https://downloads.haskell.org/~ghc/$GHC/ghc-$GHC-$ARCH-fedora27-linux.tar.xz"; \ GHC_SHA256='b6ed67049a23054a8042e65c9976d5e196e5ee4e83b29b2ee35c8a22ab1e5b73'; \ + GHC_EXTRACT_NAME="ghc-$GHC"; \ ;; \ '9.2.8') \ GHC_URL="https://downloads.haskell.org/~ghc/$GHC/ghc-$GHC-$ARCH-ubuntu20.04-linux.tar.xz"; \ GHC_SHA256='6e4adc184a53ca9d9dd8c11c6611d0643fdc3b76550ae769e378d9edb2bda745'; \ + GHC_EXTRACT_NAME="ghc-$GHC"; \ ;; \ '9.4.8') \ GHC_URL="https://downloads.haskell.org/~ghc/$GHC/ghc-$GHC-$ARCH-deb11-linux.tar.xz"; \ @@ -122,6 +138,10 @@ RUN \ GHC_RELEASE_KEY='88b57fcf7db53b4db3bfa4b1588764fbe22d19c4'; \ GHC_EXTRACT_NAME="ghc-$GHC-$ARCH-unknown-linux"; \ ;; \ + '9.6.5') \ + GHC_URL="https://downloads.haskell.org/~ghc/$GHC/ghc-$GHC-$ARCH-deb11-linux.tar.xz"; \ + GHC_SHA256='8cd8dc9c870540dab142f2dc4eda174a98be7312975696169c363704d8fcf094'; \ + ;; \ *) echo >&2 "error: unsupported ghc '$GHC'" ; exit 1 ;; \ esac; \ ;; \ @@ -145,6 +165,8 @@ RUN \ \ "/opt/ghc/$GHC/bin/ghc" --version +# ------------------------------------------------------------------------------ + ENV PATH /root/.cabal/bin:/root/.local/bin:/opt/ghc/${GHC}/bin:$PATH CMD ["ghci"]