deps: bumping yq to v4.40.7

Since yq frequently updates, let's upgrade to a version from February to bypass potential issues with versions 4.41-4.43 for now. We can always upgrade to the newest version if necessary. Fixes kata-containers#9354 Depends-on:github.com/kata-containers/tests#5818 Signed-off-by: Beraldo Leal <[email protected]>
DataDog · Jun 17, 2024 · c5bc286 · c5bc286
1 parent d229e89
commit c5bc286
Show file tree

Hide file tree

Showing 47 changed files with 376 additions and 476 deletions.
diff --git a/ci/install_libseccomp.sh b/ci/install_libseccomp.sh
@@ -23,11 +23,11 @@ workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
 # Variables for libseccomp
 libseccomp_version="${LIBSECCOMP_VERSION:-""}"
 if [ -z "${libseccomp_version}" ]; then
-    libseccomp_version=$(get_from_kata_deps "externals.libseccomp.version")
+    libseccomp_version=$(get_from_kata_deps ".externals.libseccomp.version")
 fi
 libseccomp_url="${LIBSECCOMP_URL:-""}"
 if [ -z "${libseccomp_url}" ]; then
-    libseccomp_url=$(get_from_kata_deps "externals.libseccomp.url")
+    libseccomp_url=$(get_from_kata_deps ".externals.libseccomp.url")
 fi
 libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz"
 libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}"
@@ -36,11 +36,11 @@ cflags="-O2"
 # Variables for gperf
 gperf_version="${GPERF_VERSION:-""}"
 if [ -z "${gperf_version}" ]; then
-    gperf_version=$(get_from_kata_deps "externals.gperf.version")
+    gperf_version=$(get_from_kata_deps ".externals.gperf.version")
 fi
 gperf_url="${GPERF_URL:-""}"
 if [ -z "${gperf_url}" ]; then
-    gperf_url=$(get_from_kata_deps "externals.gperf.url")
+    gperf_url=$(get_from_kata_deps ".externals.gperf.url")
 fi
 gperf_tarball="gperf-${gperf_version}.tar.gz"
 gperf_tarball_url="${gperf_url}/${gperf_tarball}"

diff --git a/ci/install_yq.sh b/ci/install_yq.sh
@@ -16,7 +16,7 @@ die() {
 # Install via binary download, as we may not have golang installed at this point
 function install_yq() {
 	local yq_pkg="github.com/mikefarah/yq"
-	local yq_version=3.4.1
+	local yq_version=v4.40.7
 	local precmd=""
 	INSTALL_IN_GOPATH=${INSTALL_IN_GOPATH:-true}
 

diff --git a/docs/Developer-Guide.md b/docs/Developer-Guide.md
@@ -461,7 +461,7 @@ and repository utilized can be found by looking at the [versions file](../versio
 Find the correct version of QEMU from the versions file:
 ```bash
 $ source kata-containers/tools/packaging/scripts/lib.sh
-$ qemu_version="$(get_from_kata_deps "assets.hypervisor.qemu.version")"
+$ qemu_version="$(get_from_kata_deps ".assets.hypervisor.qemu.version")"
 $ echo "${qemu_version}"
 ```
 Get source from the matching branch of QEMU:

diff --git a/docs/how-to/how-to-pull-images-in-guest-with-kata.md b/docs/how-to/how-to-pull-images-in-guest-with-kata.md
@@ -35,27 +35,23 @@ $ git clone -b "${nydus_snapshotter_version}" "${nydus_snapshotter_url}" "${nydu
 2. Configure DaemonSet file
 ```bash
 $ pushd "$nydus_snapshotter_install_dir"
-$ yq write -i \
->	 misc/snapshotter/base/nydus-snapshotter.yaml \
->	 'data.FS_DRIVER' \
->	 "proxy" --style=double
+$ yq -i \
+>	 '.data.FS_DRIVER = "proxy"' -P \
+>	 misc/snapshotter/base/nydus-snapshotter.yaml
 # Disable to read snapshotter config from configmap
-$ yq write -i \
->	 misc/snapshotter/base/nydus-snapshotter.yaml \
->	 'data.ENABLE_CONFIG_FROM_VOLUME' \
->	 "false" --style=double
+$ yq -i \
+>	 'data.ENABLE_CONFIG_FROM_VOLUME = "false"' -P \
+>	 misc/snapshotter/base/nydus-snapshotter.yaml
 # Enable to run snapshotter as a systemd service 
 # (skip if you want to run nydus snapshotter as a standalone process)
-$ yq write -i \
->	 misc/snapshotter/base/nydus-snapshotter.yaml \
->	 'data.ENABLE_SYSTEMD_SERVICE' \
->	 "true" --style=double
+$ yq -i \
+>	 'data.ENABLE_SYSTEMD_SERVICE = "true"' -P \
+>	 misc/snapshotter/base/nydus-snapshotter.yaml
 # Enable "runtime specific snapshotter" feature in containerd when configuring containerd for snapshotter
 # (skip if you want to configure nydus snapshotter as a global snapshotter in containerd)
-$ yq write -i \
->	 misc/snapshotter/base/nydus-snapshotter.yaml \
->	 'data.ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER' \
->	 "true" --style=double
+$ yq -i \
+>	 'data.ENABLE_RUNTIME_SPECIFIC_SNAPSHOTTER = "true"' -P \
+>	 misc/snapshotter/base/nydus-snapshotter.yaml
 ```
 
 3. Install `nydus snapshotter` as a DaemonSet

diff --git a/docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md b/docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md
@@ -44,8 +44,8 @@ $ popd
 - Build a custom QEMU
 ```bash
 $ source kata-containers/tools/packaging/scripts/lib.sh
-$ qemu_url="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.url")"
-$ qemu_tag="$(get_from_kata_deps "assets.hypervisor.qemu-snp-experimental.tag")"
+$ qemu_url="$(get_from_kata_deps ".assets.hypervisor.qemu-snp-experimental.url")"
+$ qemu_tag="$(get_from_kata_deps ".assets.hypervisor.qemu-snp-experimental.tag")"
 $ git clone "${qemu_url}"
 $ pushd qemu
 $ git checkout "${qemu_tag}"

diff --git a/src/runtime/golang.mk b/src/runtime/golang.mk
@@ -27,7 +27,17 @@ ifeq (,$(not_check_version))
     ifneq (,$(install_yq))
         $(error "ERROR: install yq failed")
     endif
-    golang_version_min=$(shell $(GOPATH)/bin/yq r ../../versions.yaml languages.golang.version)
+
+    YQ_VERSION=$(shell $(GOPATH)/bin/yq --version | grep -oE "version v?[0-9]+" | grep -oE "[0-9]+")
+    QUERY="languages.golang.version"
+
+    ifneq (,$(findstring 4,$(YQ_VERSION)))
+        YQ_CMD=$(GOPATH)/bin/yq eval .$(QUERY) ../../versions.yaml
+    else
+        YQ_CMD=$(GOPATH)/bin/yq r ../../versions.yaml $(QUERY)
+    endif
+
+    golang_version_min=$(shell $(YQ_CMD))
 
     ifeq (,$(golang_version_min))
         $(error "ERROR: cannot determine minimum golang version")

diff --git a/src/runtime/virtcontainers/pkg/cloud-hypervisor/Makefile b/src/runtime/virtcontainers/pkg/cloud-hypervisor/Makefile
@@ -25,7 +25,7 @@ update-yaml:
 ifndef YQ
 	$(MK_DIR)/../../../../../ci//install_yq.sh
 endif
-	clh_version=$(shell yq r $(VERSIONS_FILE) assets.hypervisor.cloud_hypervisor.version); \
+	clh_version=$(shell yq .assets.hypervisor.cloud_hypervisor.version $(VERSIONS_FILE)); \
 	curl -OL https://raw.githubusercontent.com/cloud-hypervisor/cloud-hypervisor/$$clh_version/vmm/src/api/openapi/cloud-hypervisor.yaml
 
 clean-generated-code:

diff --git a/tests/cmd/github-labels/github-labels.sh b/tests/cmd/github-labels/github-labels.sh
@@ -48,7 +48,7 @@ merge_yaml()
 	[ -n "$out" ] || die "need output file"
 
 	need_yq
-	yq merge "$file1" --append "$file2" > "$out"
+  yq eval-all '. as $item ireduce ({}; . *+ $item)' "$file1" "$file2" > "$out"
 }
 
 check_yaml()
@@ -58,7 +58,7 @@ check_yaml()
 	[ -n "$file" ] || die "need file to check"
 
 	need_yq
-	yq read "$file" >/dev/null
+	yq "$file" >/dev/null
 
 	[ -z "$(command -v yamllint)" ] && die "need yamllint installed"
 

diff --git a/tests/common.bash b/tests/common.bash
@@ -497,11 +497,19 @@ function ensure_yq() {
 
 # dependency: What we want to get the version from the versions.yaml file
 function get_from_kata_deps() {
-        local dependency="$1"
         versions_file="${repo_root_dir}/versions.yaml"
 
         command -v yq &>/dev/null || die 'yq command is not in your $PATH'
-        result=$("yq" read -X "$versions_file" "$dependency")
+
+        yq_version=$(yq --version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | cut -d. -f1)
+        if [ "$yq_version" -eq 3 ]; then
+          dependency=$(echo "$1" | sed "s/^\.//g")
+          result=$("yq" read "$versions_file" "$dependency")
+        else
+          dependency=$1
+          result=$("yq" "$dependency | explode (.)" "$versions_file")
+        fi
+
         [ "$result" = "null" ] && result=""
         echo "$result"
 }
@@ -743,7 +751,7 @@ function get_dep_from_yaml_db(){
 
         "${repo_root_dir}/ci/install_yq.sh" >&2
 
-        result=$("${GOPATH}/bin/yq" r -X "$versions_file" "$dependency")
+        result=$("${GOPATH}/bin/yq" "$dependency" "$versions_file")
         [ "$result" = "null" ] && result=""
         echo "$result"
 }
@@ -759,7 +767,7 @@ function get_test_version(){
 
         db="${cidir}/../versions.yaml"
 
-        get_dep_from_yaml_db "${db}" "${dependency}"
+        get_dep_from_yaml_db "${db}" ".${dependency}"
 }
 
 # Load vhost, vhost_net, vhost_vsock modules.

diff --git a/tests/functional/kata-deploy/kata-deploy.bats b/tests/functional/kata-deploy/kata-deploy.bats
@@ -29,40 +29,30 @@ setup() {
 	sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
 
 	# Enable debug for Kata Containers
-	yq write -i \
-	  "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
-	  'spec.template.spec.containers[0].env[1].value' \
-	  --tag '!!str' "true"
+	yq -i \
+	  '.spec.template.spec.containers[0].env[1].value = "true"' \
+	  "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
 	# Create the runtime class only for the shim that's being tested
-	yq write -i \
-	  "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
-	  'spec.template.spec.containers[0].env[2].value' \
-	  "${KATA_HYPERVISOR}"
+	yq -i \
+	  ".spec.template.spec.containers[0].env[2].value = \"${KATA_HYPERVISOR}\"" \
+	  "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
 	# Set the tested hypervisor as the default `kata` shim
-	yq write -i \
-	  "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
-	  'spec.template.spec.containers[0].env[3].value' \
-	  "${KATA_HYPERVISOR}"
+	yq -i \
+	  ".spec.template.spec.containers[0].env[3].value = \"${KATA_HYPERVISOR}\"" \
+	  "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
 	# Let the `kata-deploy` script take care of the runtime class creation / removal
-	yq write -i \
-	  "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
-	  'spec.template.spec.containers[0].env[4].value' \
-	  --tag '!!str' "true"
+	yq -i \
+	  '.spec.template.spec.containers[0].env[4].value = "true"' \
+	  "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
 	# Let the `kata-deploy` create the default `kata` runtime class
-	yq write -i \
-	  "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
-	  'spec.template.spec.containers[0].env[5].value' \
-	  --tag '!!str' "true"
+	yq -i \
+	  '.spec.template.spec.containers[0].env[5].value = "true"' \
+	  "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
 
 	if [ "${KATA_HOST_OS}" = "cbl-mariner" ]; then
-		yq write -i \
-		  "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
-		  'spec.template.spec.containers[0].env[+].name' \
-		  "HOST_OS"
-		yq write -i \
-		  "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
-		  'spec.template.spec.containers[0].env[-1].value' \
-		  "${KATA_HOST_OS}"
+		yq -i \
+		  ".spec.template.spec.containers[0].env += [{\"name\": \"HOST_OS\", \"value\": \"${KATA_HOST_OS}\"}]" \
+		  "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
 	fi
 
 	echo "::group::Final kata-deploy.yaml that is used in the test"
@@ -144,25 +134,21 @@ teardown() {
 	kubectl -n kube-system wait --timeout=10m --for=delete -l name=kata-deploy pod
 
 	# Let the `kata-deploy` script take care of the runtime class creation / removal
-	yq write -i \
-	  "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
-	  'spec.template.spec.containers[0].env[4].value' \
-	  --tag '!!str' "true"
+	yq -i \
+	  '.spec.template.spec.containers[0].env[4].value = "true"' \
+	  "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
 	# Create the runtime class only for the shim that's being tested
-	yq write -i \
-	  "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
-	  'spec.template.spec.containers[0].env[2].value' \
-	  "${KATA_HYPERVISOR}"
+	yq -i \
+	  ".spec.template.spec.containers[0].env[2].value = \"${KATA_HYPERVISOR}\"" \
+	  "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
 	# Set the tested hypervisor as the default `kata` shim
-	yq write -i \
-	  "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml" \
-	  'spec.template.spec.containers[0].env[3].value' \
-	  "${KATA_HYPERVISOR}"
+	yq -i \
+	  ".spec.template.spec.containers[0].env[3].value = \"${KATA_HYPERVISOR}\"" \
+	  "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
 	# Let the `kata-deploy` create the default `kata` runtime class
-	yq write -i \
-	  "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml" \
-	  'spec.template.spec.containers[0].env[5].value' \
-	  --tag '!!str' "true"
+	yq -i \
+	  '.spec.template.spec.containers[0].env[5].value = "true"' \
+	  "tools/packaging/kata-deploy/kata-deploy/base/kata-deploy.yaml"
 
 	sed -i -e "s|quay.io/kata-containers/kata-deploy:latest|${DOCKER_REGISTRY}/${DOCKER_REPO}:${DOCKER_TAG}|g" "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"
 	cat "tools/packaging/kata-deploy/kata-cleanup/base/kata-cleanup.yaml"

diff --git a/tests/functional/kata-monitor/gha-run.sh b/tests/functional/kata-monitor/gha-run.sh
@@ -36,15 +36,15 @@ function install_dependencies() {
 	# - cri-tools
 	# - containerd
 	#   - cri-container-cni release tarball already includes CNI plugins
-	cri_tools_version=$(get_from_kata_deps "externals.critools.latest")
+	cri_tools_version=$(get_from_kata_deps ".externals.critools.latest")
 	declare -a github_deps
 	github_deps[0]="cri_tools:${cri_tools_version}"
 	case "${CONTAINER_ENGINE}" in
 		containerd)
-			github_deps[1]="cri_containerd:$(get_from_kata_deps "externals.containerd.${CONTAINERD_VERSION}")"
+			github_deps[1]="cri_containerd:$(get_from_kata_deps ".externals.containerd.${CONTAINERD_VERSION}")"
 			;;
 		crio)
-			github_deps[1]="cni_plugins:$(get_from_kata_deps "externals.cni-plugins.version")"
+			github_deps[1]="cni_plugins:$(get_from_kata_deps ".externals.cni-plugins.version")"
 			;;
 	esac
 

diff --git a/tests/functional/vfio/vfio_fedora_vm_wrapper.sh b/tests/functional/vfio/vfio_fedora_vm_wrapper.sh
@@ -174,8 +174,8 @@ ${environment}
     pushd /workspace
     source tests/common.bash
     ensure_yq
-    cri_containerd=\$(get_from_kata_deps "externals.containerd.lts")
-    cri_tools=\$(get_from_kata_deps "externals.critools.latest")
+    cri_containerd=\$(get_from_kata_deps ".externals.containerd.lts")
+    cri_tools=\$(get_from_kata_deps ".externals.critools.latest")
     install_cri_containerd \${cri_containerd}
     install_cri_tools \${cri_tools}
 

diff --git a/tests/gha-run-k8s-common.sh b/tests/gha-run-k8s-common.sh
@@ -127,9 +127,9 @@ function install_kustomize() {
 	fi
 
 	ensure_yq
-	version=$(get_from_kata_deps "externals.kustomize.version")
+	version=$(get_from_kata_deps ".externals.kustomize.version")
 	arch=$(arch_to_golang)
-	checksum=$(get_from_kata_deps "externals.kustomize.checksum.${arch}")
+	checksum=$(get_from_kata_deps ".externals.kustomize.checksum.${arch}")
 
 	local tarball="kustomize_${version}_linux_${arch}.tar.gz"
 	curl -Lf -o "$tarball" "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/${version}/${tarball}"

diff --git a/tests/install_go.sh b/tests/install_go.sh
@@ -67,7 +67,7 @@ shift $(( $OPTIND - 1 ))
 go_version="${1:-""}"
 
 if [ -z "$go_version" ] && [ "${USE_VERSIONS_FILE}"  = "true" ] ;then
-	go_version=$(get_from_kata_deps "languages.golang.meta.newest-version")
+	go_version=$(get_from_kata_deps ".languages.golang.meta.newest-version")
 fi
 
 if [ -z "$go_version" ];then

diff --git a/tests/install_rust.sh b/tests/install_rust.sh
@@ -17,7 +17,7 @@ rustarch=$(arch_to_rust)
 
 version="${1:-""}"
 if [ -z "${version}" ]; then
-	version=$(get_from_kata_deps "languages.rust.meta.newest-version")
+	version=$(get_from_kata_deps ".languages.rust.meta.newest-version")
 fi
 
 echo "Install rust ${version}"

diff --git a/tests/integration/cri-containerd/gha-run.sh b/tests/integration/cri-containerd/gha-run.sh
@@ -43,8 +43,8 @@ function install_dependencies() {
 	#   - cri-container-cni release tarball already includes CNI plugins
 	# - cri-tools
 	declare -a github_deps
-	github_deps[0]="cri_containerd:$(get_from_kata_deps "externals.containerd.${CONTAINERD_VERSION}")"
-	github_deps[1]="cri_tools:$(get_from_kata_deps "externals.critools.latest")"
+	github_deps[0]="cri_containerd:$(get_from_kata_deps ".externals.containerd.${CONTAINERD_VERSION}")"
+	github_deps[1]="cri_tools:$(get_from_kata_deps ".externals.critools.latest")"
 
 	for github_dep in "${github_deps[@]}"; do
 		IFS=":" read -r -a dep <<< "${github_dep}"
@@ -53,7 +53,7 @@ function install_dependencies() {
 
 	# Clone containerd as we'll need to build it in order to run the tests
 	# base_version: The version to be intalled in the ${major}.${minor} format
-	clone_cri_containerd $(get_from_kata_deps "externals.containerd.${CONTAINERD_VERSION}")
+	clone_cri_containerd $(get_from_kata_deps ".externals.containerd.${CONTAINERD_VERSION}")
 }
 
 function run() {

diff --git a/tests/integration/kubernetes/confidential_kbs.sh b/tests/integration/kubernetes/confidential_kbs.sh
@@ -149,7 +149,7 @@ kbs_install_cli() {
 	# Mininum required version to build the client (read from versions.yaml)
 	local rust_version
 	ensure_yq
-	rust_version=$(get_from_kata_deps "externals.coco-trustee.toolchain")
+	rust_version=$(get_from_kata_deps ".externals.coco-trustee.toolchain")
 	# Currently kata version from version.yaml is 1.72.0
 	# which doesn't match the requirement, so let's pass
 	# the required version.
@@ -205,10 +205,10 @@ function kbs_k8s_deploy() {
 	ensure_yq
 
 	# Read from versions.yaml
-	repo=$(get_from_kata_deps "externals.coco-trustee.url")
-	version=$(get_from_kata_deps "externals.coco-trustee.version")
-	image=$(get_from_kata_deps "externals.coco-trustee.image")
-	image_tag=$(get_from_kata_deps "externals.coco-trustee.image_tag")
+	repo=$(get_from_kata_deps ".externals.coco-trustee.url")
+	version=$(get_from_kata_deps ".externals.coco-trustee.version")
+	image=$(get_from_kata_deps ".externals.coco-trustee.image")
+	image_tag=$(get_from_kata_deps ".externals.coco-trustee.image_tag")
 
 	# The ingress handler for AKS relies on the cluster's name which in turn
 	# contain the HEAD commit of the kata-containers repository (supposedly the

diff --git a/tests/integration/kubernetes/filter_k8s_test.sh b/tests/integration/kubernetes/filter_k8s_test.sh
@@ -26,7 +26,7 @@ main()
 		${repo_root_dir}/ci/install_yq.sh > /dev/null
 	fi
 
-        local K8S_SKIP_UNION=$("${GOPATH_LOCAL}/bin/yq" read "${K8S_CONFIG_FILE}" "${K8S_FILTER_FLAG}")
+        local K8S_SKIP_UNION=$("${GOPATH_LOCAL}/bin/yq" ".${K8S_FILTER_FLAG}" "${K8S_CONFIG_FILE}")
         [ "${K8S_SKIP_UNION}" == "null" ] && return
         mapfile -t _K8S_SKIP_UNION <<< "${K8S_SKIP_UNION}"