Merge branch 'main' into fix/fix_linting

.github/CODEOWNERS

@@ -1,5 +1,6 @@
 * @teutonet/k8s
 /charts/base-cluster/ @cwrau @marvinWolff @tasches  @teutonet-bot
+/charts/common/ @cwrau @marvinWolff @tasches  @teutonet-bot
 /charts/stellio-context-broker/ @cwrau @ocaner-biz  @teutonet-bot
 /charts/t8s-cluster/ @cwrau @marvinWolff @tasches  @teutonet-bot
 /charts/teuto-portal-k8s-worker/ @cwrau @marvinWolff @tasches  @teutonet-bot

.github/artifacthub-repo.yaml

@@ -0,0 +1,12 @@
+# Artifact Hub repository metadata file
+#
+# Some settings like the verified publisher flag or the ignored packages won't
+# be applied until the next time the repository is processed. Please keep in
+# mind that the repository won't be processed if it has not changed since the
+# last time it was processed. Depending on the repository kind, this is checked
+# in a different way. For Helm http based repositories, we consider it has
+# changed if the `index.yaml` file changes. For git based repositories, it does
+# when the hash of the last commit in the branch you set up changes. This does
+# NOT apply to ownership claim operations, which are processed immediately.
+#
+repositoryID: bc3ec3a0-5f14-46f9-941f-c5f4f0264b2a

.github/release-please/config.json

@@ -32,6 +32,9 @@
     },
     "charts/teuto-portal-k8s-worker": {
       "package-name": "teuto-portal-k8s-worker"
+    },
+    "charts/common": {
+      "package-name": "common"
     }
   }
 }

.github/release-please/manifest.json

@@ -1 +1 @@
-{"charts/base-cluster":"4.12.0","charts/teuto-portal-k8s-worker":"1.0.1","charts/t8s-cluster":"4.1.0","charts/stellio-context-broker":"0.1.0","charts/chirpstack":"0.1.0"}
+{"charts/base-cluster":"6.0.0","charts/teuto-portal-k8s-worker":"1.0.2","charts/t8s-cluster":"6.0.0","charts/stellio-context-broker":"0.1.0","charts/chirpstack":"0.1.0"}

.github/scripts/create-values-diff.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 [[ "$RUNNER_DEBUG" == 1 ]] && set -x
+[[ $- == *x* ]] && export RUNNER_DEBUG=1
 
 set -eu
 set -o pipefail
@@ -21,6 +22,11 @@ else
   dryRun=false
 fi
 
+if yq -e '.type == "library"' "$chart/Chart.yaml" >/dev/null; then
+  echo "Skipping library chart '$chart'" >/dev/stderr
+  exit 0
+fi
+
 GITHUB_API_URL="${GITHUB_API_URL:-https://api.github.com}"
 
 if command -v gh &>/dev/null; then
@@ -38,48 +44,14 @@ GITHUB_WORKSPACE="${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel)}"
 GITHUB_DEFAULT_BRANCH="${GITHUB_DEFAULT_BRANCH:-main}"
 GITHUB_SERVER_URL="${GITHUB_SERVER_URL:-https://github.com}"
 GITHUB_REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}"
-SCRIPTS="$GITHUB_WORKSPACE/.github/scripts/"
-TMP_DIR="$(mktemp -d)"
-trap 'rm -rf "$TMP_DIR"' EXIT
 
-cd "$GITHUB_WORKSPACE"
+[[ ! -v TMP_DIR ]] && trap 'rm -rf "$TMP_DIR"' EXIT
+TMP_DIR="${TMP_DIR:-$(mktemp -d)}"
 
-function splitYamlIntoDir() {
-  local yaml="${1?}"
-  local dir="${2?}"
-  local IFS=$'\n'
-  local selector
-
-  for selector in $(yq -c -s '.[] | select(.kind and .metadata.name) | {kind: .kind, namespace: .metadata.namespace, name: .metadata.name}' <"$yaml"); do
-    local resourceName
-    local kind
-    local namespace
-    local name
-    kind="$(jq --argjson selector "$selector" -n -r '$selector.kind')"
-    namespace="$(jq --argjson selector "$selector" -n -r '$selector.namespace')"
-    name="$(jq --argjson selector "$selector" -n -r '$selector.name')"
-
-    resourceName="$dir/$namespace/$kind/$name.yaml"
-    if [[ -f "$resourceName" ]]; then
-      echo "'$resourceName' shouldn't already exist" >/dev/stderr
-      return 1
-    fi
-    mkdir -p "$(dirname "$resourceName")"
-    # shellcheck disable=SC2016
-    yq -y -s --argjson selector "$selector" '.[] | select((.kind == $selector.kind) and (.metadata.namespace == $selector.namespace) and (.metadata.name == $selector.name)) | del(.metadata.labels.chart) | del(.metadata.labels["helm.sh/chart"])' <"$yaml" >"$resourceName"
-    if [[ "$kind" == "HelmRelease" ]]; then
-      (
-        "$SCRIPTS/templateHelmRelease" -1 <<<"$(sed -s '$a---' <(yq -s -y '.[] | select(.apiVersion | contains("source.toolkit.fluxcd.io"))' <"$yaml") "$resourceName")" >"${resourceName}_templated"
-        splitYamlIntoDir "${resourceName}_templated" "$(dirname "$resourceName")/$(basename -s .yaml "$resourceName")"
-        rm "${resourceName}_templated"
-      ) &
-    fi
-  done
-  wait
-}
+cd "$GITHUB_WORKSPACE"
 
 function generateComment() {
-  local chart="charts/${1?}"
+  local chart="${1?}"
   local -A diffs
   local newResourcesDir
   local originalResourcesDir
@@ -93,13 +65,13 @@ function generateComment() {
       mkdir "$originalResourcesDir" "$newResourcesDir"
 
       (
-        "$SCRIPTS/templateGitHelmChart" -1 "$GITHUB_REPO_URL" "$chart" "${GITHUB_DEFAULT_BRANCH}" "$values" | yq -y -S >"$originalResourcesDir.yaml"
-        splitYamlIntoDir "$originalResourcesDir.yaml" "$originalResourcesDir"
+        "$(dirname "$0")/templateGitHelmChart" -1 "$GITHUB_REPO_URL" "$chart" "${GITHUB_DEFAULT_BRANCH}" "$values" | yq -y -S >"$originalResourcesDir.yaml"
+        "$(dirname "$0")/splitYamlIntoDir" "$originalResourcesDir.yaml" "$originalResourcesDir"
       ) &
 
       (
-        "$SCRIPTS/templateLocalHelmChart" -1 "$chart" "$values" | yq -y -S >"$newResourcesDir.yaml"
-        splitYamlIntoDir "$newResourcesDir.yaml" "$newResourcesDir"
+        "$(dirname "$0")/templateLocalHelmChart" -1 "$chart" "$values" | yq -y -S >"$newResourcesDir.yaml"
+        "$(dirname "$0")/splitYamlIntoDir" "$newResourcesDir.yaml" "$newResourcesDir"
       ) &
 
       wait

.github/scripts/enforce-trusted-registries.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 [[ "$RUNNER_DEBUG" == 1 ]] && set -x
+[[ $- == *x* ]] && export RUNNER_DEBUG=1
 
 set -eu
 set -o pipefail
@@ -11,20 +12,28 @@ function getUntrustedImages() {
 
   trustedImagesRegex="$(yq -r -f .github/scripts/trusted_images_regex.jq <.github/trusted_registries.yaml)"
 
-  yq -r '.annotations["artifacthub.io/images"]' <"$chart/Chart.yaml" |
+  yq -r '.annotations["artifacthub.io/images"]' "$chart/Chart.yaml" |
     yq -r '.[] | .image' |
-    grep -v -E "$trustedImagesRegex"
+    grep -v -E "$trustedImagesRegex" |
+    sort -u
 }
 
 function enforceTrustedImages() {
   local chart="${1?}"
   local untrustedImages=()
+  if yq -e '.type == "library"' "$chart/Chart.yaml" >/dev/null; then
+    echo "Skipping library chart '$chart'" >/dev/stderr
+    return 0
+  fi
 
   mapfile -t untrustedImages < <(getUntrustedImages "$chart")
-  if ((${#untrustedImages[@]} > 0)); then
+  if [[ "${#untrustedImages[@]}" -gt 0 ]]; then
     echo "found ${#untrustedImages[@]} untrusted images in '$chart', please fix;" >/dev/stderr
     for untrustedImage in "${untrustedImages[@]}"; do
-      echo "    $untrustedImage" >/dev/stderr
+      echo "  > $untrustedImage, found in the following resources:" >/dev/stderr
+      # shellcheck disable=SC2016
+      yq --arg image "$untrustedImage" -r '.annotations["artifacthub.io/images"] | split("\n")[] | select(contains($image))' "$chart/Chart.yaml" |
+        awk '{print "      - " $NF}' >/dev/stderr
     done
     return 1
   fi

.github/scripts/extract-artifacthub-images.sh

@@ -1,20 +1,13 @@
 #!/usr/bin/env bash
 
 [[ "$RUNNER_DEBUG" == 1 ]] && set -x
+[[ $- == *x* ]] && export RUNNER_DEBUG=1
 
 set -eu
 set -o pipefail
 
-TMP_DIR=$(mktemp -d)
-trap 'rm -rf "$TMP_DIR"' EXIT
-
-function templateRemoteHelmChart() {
-  "$(dirname "$0")/templateRemoteHelmChart"
-}
-
-function templateLocalHelmChart() {
-  "$(dirname "$0")/templateLocalHelmChart" "$@"
-}
+[[ ! -v TMP_DIR ]] && trap 'rm -rf "$TMP_DIR"' EXIT
+TMP_DIR="${TMP_DIR:-$(mktemp -d)}"
 
 function templateHelmChart() {
   local chart="$1"
@@ -28,20 +21,33 @@ function templateHelmChart() {
   if [[ "$numberOfHelmReleases" -gt 0 ]]; then
     for index in $(seq 0 $((numberOfHelmReleases - 1))); do
       echo ---
-      yq <<<"$yaml" -erys '([.[] | select(.kind == "HelmRelease")]['"$index"']),(.[] | select(.kind | IN(["GitRepository", "HelmRepository"][])))' | templateRemoteHelmChart
+      yq <<<"$yaml" -erys '([.[] | select(.kind == "HelmRelease")]['"$index"']),(.[] | select(.kind | IN(["GitRepository", "HelmRepository"][])))' | "$(dirname "$0")/templateRemoteHelmChart"
     done
   fi
 }
 
 function getImages() {
   local chart="$1"
-  templateLocalHelmChart "$chart" |
-    grep -E '\s+image: \S+' |
-    grep -v 'artifacthub-ignore' |
-    awk '{print $NF}' |
-    tr -d '"' |
-    sort -u |
-    jq -Rn '[[inputs][] | {image: .}]' | yq -y
+  local tmpDir
+  tmpDir="$(mktemp -d -p "$TMP_DIR")"
+  "$(dirname "$0")/templateLocalHelmChart" -1 "$chart" >"$tmpDir/helmRelease.yaml"
+  "$(dirname "$0")/splitYamlIntoDir" "$tmpDir/helmRelease.yaml" "$tmpDir/helmRelease"
+
+  (
+    cd "$tmpDir/helmRelease"
+    rm -f -- */HelmRelease/*.yaml
+    grep -Er '\s+image: \S+$' |
+      grep -v 'artifacthub-ignore' |
+      awk '{print $3 " # " $1}' |
+      tr -d '"' |
+      sed 's#:$##' |
+      sort -k1 -k2 |
+      uniq |
+      column -t |
+      jq -Rn '[[inputs][] | {image: .}]' |
+      yq -y |
+      tr -d "'"
+  )
 }
 
 function updateChartYaml() {
@@ -53,13 +59,13 @@ function updateChartYaml() {
   (
     echo "artifacthub.io/images: |"
     getImages "$chart" | awk '{print "  " $0}'
-  ) | tee "$tmpDir/images.yaml"
+  ) | tee "$tmpDir/images.yaml" >/dev/stderr
 
   if yq -e .annotations "$chart/Chart.yaml" >/dev/null; then
     echo "Existing annotations:" >/dev/stderr
-    yq -y '.annotations | del(.["artifacthub.io/images"])' "$chart/Chart.yaml" | tee "$tmpDir/annotations.yaml"
+    yq -y '.annotations | del(.["artifacthub.io/images"])' "$chart/Chart.yaml" | tee "$tmpDir/annotations.yaml" >/dev/stderr
     echo "Cleaned Chart.yaml:" >/dev/stderr
-    yq -y '. | del(.annotations)' "$chart/Chart.yaml" | tee >(sponge "$chart/Chart.yaml")
+    yq -y '. | del(.annotations)' "$chart/Chart.yaml" | tee >(sponge "$chart/Chart.yaml") >/dev/stderr
   else
     touch "$tmpDir/annotations.yaml"
   fi
@@ -76,6 +82,10 @@ function updateChartYaml() {
 }
 
 if [[ "$#" == 1 ]] && [[ -d "$1" ]]; then
+  if yq -e '.type == "library"' "$1/Chart.yaml" >/dev/null; then
+    echo "Skipping library chart '$1'" >/dev/stderr
+    exit 0
+  fi
   if ! [[ -f "$1/ci/artifacthub-values.yaml" ]]; then
     echo "There is no 'artifacthub-values.yaml' in 'charts/$1/ci', exiting" >/dev/stderr
     exit 1
@@ -86,6 +96,10 @@ else
     [[ "$chart" == "charts/*" ]] && continue
     [[ -f "$chart/ci/artifacthub-values.yaml" ]] || continue
 
+    if yq -e '.type == "library"' "$chart/Chart.yaml" >/dev/null; then
+      echo "Skipping library chart '$chart'" >/dev/stderr
+      exit 0
+    fi
     updateChartYaml "$chart"
   done
 fi

.github/scripts/prepare-values.sh

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 
+[[ "$RUNNER_DEBUG" == 1 ]] && set -x
+[[ $- == *x* ]] && export RUNNER_DEBUG=1
+
 function mergeYaml() {
   local valuesFile="${1?}"
   local overrideJson="${2?}"

.github/scripts/splitYamlIntoDir

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+[[ "$RUNNER_DEBUG" == 1 ]] && set -x
+[[ $- == *x* ]] && export RUNNER_DEBUG=1
+
+set -eu
+set -o pipefail
+
+function splitYamlIntoDir() {
+  local yaml="${1?}"
+  local dir="${2?}"
+  local IFS=$'\n'
+  local selector
+
+  for selector in $(yq -c -s '.[] | select(.kind and .metadata.name) | {kind: .kind, namespace: .metadata.namespace, name: .metadata.name}' <"$yaml"); do
+    local resourceName
+    local kind
+    local namespace
+    local name
+    kind="$(jq --argjson selector "$selector" -n -r '$selector.kind')"
+    namespace="$(jq --argjson selector "$selector" -n -r '$selector.namespace')"
+    name="$(jq --argjson selector "$selector" -n -r '$selector.name')"
+
+    resourceName="$dir/$namespace/$kind/$name.yaml"
+    if [[ -f "$resourceName" ]]; then
+      echo "'$resourceName' shouldn't already exist" >/dev/stderr
+      return 1
+    fi
+    mkdir -p "$(dirname "$resourceName")"
+    # shellcheck disable=SC2016
+    yq -y -s --argjson selector "$selector" '.[] | select((.kind == $selector.kind) and (.metadata.namespace == $selector.namespace) and (.metadata.name == $selector.name)) | del(.metadata.labels.chart) | del(.metadata.labels["helm.sh/chart"])' <"$yaml" >"$resourceName"
+    if [[ "$kind" == "HelmRelease" ]]; then
+      (
+        "$(dirname "$0")/templateHelmRelease" -1 <<<"$(sed -s '$a---' <(yq -s -y '.[] | select(.apiVersion | contains("source.toolkit.fluxcd.io"))' <"$yaml") "$resourceName")" >"${resourceName}_templated"
+        splitYamlIntoDir "${resourceName}_templated" "$(dirname "$resourceName")/$(basename -s .yaml "$resourceName")"
+        rm "${resourceName}_templated"
+      ) &
+    fi
+  done
+  wait
+}
+
+splitYamlIntoDir "$@"

.github/scripts/sync-codeowners.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 [[ "$RUNNER_DEBUG" == 1 ]] && set -x
+[[ $- == *x* ]] && export RUNNER_DEBUG=1
 
 echo "* @teutonet/k8s"
 

.github/scripts/templateHelmChart.sh

@@ -1,12 +1,13 @@
 #!/usr/bin/env bash
 
 [[ "$RUNNER_DEBUG" == 1 ]] && set -x
+[[ $- == *x* ]] && export RUNNER_DEBUG=1
 
 set -eu
 set -o pipefail
 
-TMP_DIR="$(mktemp -d)"
-trap 'rm -rf "$TMP_DIR"' EXIT
+[[ ! -v TMP_DIR ]] && trap 'rm -rf "$TMP_DIR"' EXIT
+TMP_DIR="${TMP_DIR:-$(mktemp -d)}"
 
 function templateGitHelmRelease() {
   local gitUrl="$1"
@@ -115,11 +116,9 @@ function templateSubHelmCharts() {
   yaml=$(cat -)
   numberOfHelmReleases=$(yq <<<"$yaml" -ers '[.[] | select(.kind == "HelmRelease")] | length')
   echo "$yaml"
-  if [[ "$numberOfHelmReleases" -gt 0 ]]; then
-    for index in $(seq 0 $((numberOfHelmReleases - 1))); do
-      yq <<<"$yaml" -erys '([.[] | select(.kind == "HelmRelease")]['"$index"']),(.[] | select(.kind | IN(["GitRepository", "HelmRepository"][])))' | templateHelmRelease >"$tmpDir/$index.yaml" &
-    done
-  fi
+  for index in $(seq 0 $((numberOfHelmReleases - 1))); do
+    yq <<<"$yaml" -erys '([.[] | select(.kind == "HelmRelease")]['"$index"']),(.[] | select(.kind | IN(["GitRepository", "HelmRepository"][])))' | templateHelmRelease >"$tmpDir/$index.yaml" &
+  done
   wait
   for index in $(seq 0 $((numberOfHelmReleases - 1))); do
     echo ---

.github/scripts/trusted_images_regex.jq

@@ -8,6 +8,6 @@
       else
         "\(.[0:-1] | join("/")):\($type)"
       end |
-      "^\(.)$"
+      "^\(.)(\\s|$)"
 ] |
   join("|")

.github/scripts/validate-pullrequest.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 [[ "$RUNNER_DEBUG" == 1 ]] && set -x
+[[ $- == *x* ]] && export RUNNER_DEBUG=1
 
 set -eu
 set -o pipefail

.github/trusted_registries.yaml

@@ -36,3 +36,4 @@ registries:
     provider-os: ALL_IMAGES
   k8s.gcr.io:
     sig-storage: ALL_IMAGES
+  registry-gitlab.teuto.net: ALL_IMAGES

.github/workflows/auto-assign-pr.yaml

@@ -9,4 +9,4 @@ jobs:
   auto-assign:
     runs-on: ubuntu-latest
     steps:
-      - uses: kentaro-m/auto-assign-action@v1.2.5
+      - uses: kentaro-m/auto-assign-action@v2.0.0