Introduce the NativeLink Kubernetes operator

A single `kubectl apply -k` now deploys NativeLink in a
self-configuring, self-healing and self-updating fashion.

To achieve this we implement a two-stage depoyment to asynchronously
reconciliate various parts of NativeLink Kustomizations.

First, we deploy Flux Alerts that trigger Tekton Pipelines on
GitRepository updates to bring required images into the cluster.

Second, and technically at the same time, we start a Flux Kustomization
to deploy a NativeLink Kustomization.

This is similar to the previous 01_operations and 02_applicaion scripts,
but now happens fully automated in the cluster and no longer requires a
local Nix installation as all tag evaluations have become implementation
details of the Tekton Pipelines.

This commit also changes the K8s resource layout to a "best-practice"
Kustomize directory layout. This further reduces code duplication and
gives third parties greater flexibility and more useful reference points
to build custom NativeLink setups.

Includes an overhaul of the Kubernetes documentation.

.github/styles/config/vocabularies/TraceMachina/accept.txt

@@ -5,6 +5,7 @@ Cloudflare
 ELB
 GPUs
 Goma
+Kustomization
 LLD
 LLVM
 Machina

.github/workflows/lre.yaml

@@ -78,19 +78,38 @@ jobs:
         uses: >- # v4
           DeterminateSystems/magic-nix-cache-action@fc6aaceb40b9845a02b91e059ec147e78d1b4e41
 
-      - name: Start Kubernetes cluster (Infra)
+      - name: Start Kubernetes cluster
         run: >
           nix run .#native up
 
-      - name: Start Kubernetes cluster (Operations)
+      - name: Start NativeLink operator
         run: >
           nix develop --impure --command
-          bash -c "./deployment-examples/kubernetes/01_operations.sh"
+          bash -c "kubectl apply -k deployment-examples/kubernetes"
 
-      - name: Start Kubernetes cluster (Application)
+      - name: Wait for Tekton pipelines
         run: >
           nix develop --impure --command
-          bash -c "./deployment-examples/kubernetes/02_application.sh"
+          bash -c "kubectl wait \
+            --for=condition=Succeeded \
+            --timeout=45m \
+            pipelinerun \
+                -l tekton.dev/pipeline=rebuild-nativelink"
+
+      - name: Wait for CAS
+        run: >
+          nix develop --impure --command
+          bash -c "kubectl rollout status deploy/nativelink-cas"
+
+      - name: Wait for scheduler
+        run: >
+          nix develop --impure --command
+          bash -c "kubectl rollout status deploy/nativelink-scheduler"
+
+      - name: Wait for worker
+        run: >
+          nix develop --impure --command
+          bash -c "kubectl rollout status deploy/nativelink-worker-lre-cc"
 
       - name: Get gateway IPs
         id: gateway-ips

deploy/chromium-example/kustomization.yaml

@@ -0,0 +1,35 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+components:
+- ../../kubernetes/components/operator
+
+patches:
+- patch: |-
+    - op: replace
+      path: /spec/path
+      value: ./kubernetes/overlays/chromium
+  target:
+    kind: Kustomization
+    name: nativelink
+- patch: |-
+    - op: replace
+      path: /spec/url
+      value: https://github.com/aaronmondal/nativelink
+  target:
+    kind: GitRepository
+    name: nativelink
+- patch: |-
+    - op: replace
+      path: /spec/ref/branch
+      value: flux
+  target:
+    kind: GitRepository
+    name: nativelink
+- patch: |-
+    - op: replace
+      path: /spec/eventMetadata/flakeOutput
+      value: github:TraceMachina/nativelink#nativelink-worker-siso-chromium
+  target:
+    kind: Alert
+    name: nativelink-worker-alert

deploy/dev/kustomization.yaml

@@ -0,0 +1,69 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+components:
+- ../../kubernetes/components/operator
+
+# Change this value to deploy custom overlays.
+patches:
+- patch: |-
+    - op: replace
+      path: /spec/path
+      value: ./kubernetes/overlays/lre
+  target:
+    kind: Kustomization
+    name: nativelink
+
+# Modify this value to change the URL of the repository with deployment files.
+#
+# This is usually only necessary if you change deployment YAML files or
+# NativeLink config files. If you only intend to change the Rust sources you can
+# leave this as is and need to ensure that the Alerts below are patched to build
+# your local sources.
+- patch: |-
+    - op: replace
+      path: /spec/url
+      value: https://github.com/TraceMachina/nativelink
+  target:
+    kind: GitRepository
+    name: nativelink
+
+# Patch this to change to track a custom branch.
+- patch: |-
+    - op: replace
+      path: /spec/ref/branch
+      value: main
+  target:
+    kind: GitRepository
+    name: nativelink
+
+# Setting the flake outputs to `./src_root#xxx` causes the Tekton pipelines to
+# build nativelink from your local sources.
+#
+# During development, the following formats might be useful as well:
+#
+# `github:user/repo#outname` to build an image from an arbitrary flake output.
+#
+# `github:TraceMachina/nativelink?ref=pull/<PR_NUMBER>/head#<OUT>` to deploy a
+# outputs from a Pull request.
+- patch: |-
+    - op: replace
+      path: /spec/eventMetadata/flakeOutput
+      value: ./src_root#image
+  target:
+    kind: Alert
+    name: nativelink-image-alert
+- patch: |-
+    - op: replace
+      path: /spec/eventMetadata/flakeOutput
+      value: ./src_root#nativelink-worker-init
+  target:
+    kind: Alert
+    name: nativelink-worker-init-alert
+- patch: |-
+    - op: replace
+      path: /spec/eventMetadata/flakeOutput
+      value: ./src_root#nativelink-worker-lre-cc
+  target:
+    kind: Alert
+    name: nativelink-worker-alert

deploy/kubernetes-example/kustomization.yaml

@@ -0,0 +1,35 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+components:
+- ../../kubernetes/components/operator
+
+patches:
+- patch: |-
+    - op: replace
+      path: /spec/path
+      value: ./kubernetes/overlays/lre
+  target:
+    kind: Kustomization
+    name: nativelink
+- patch: |-
+    - op: replace
+      path: /spec/url
+      value: https://github.com/aaronmondal/nativelink
+  target:
+    kind: GitRepository
+    name: nativelink
+- patch: |-
+    - op: replace
+      path: /spec/ref/branch
+      value: flux
+  target:
+    kind: GitRepository
+    name: nativelink
+- patch: |-
+    - op: replace
+      path: /spec/eventMetadata/flakeOutput
+      value: github:TraceMachina/nativelink#nativelink-worker-lre-cc
+  target:
+    kind: Alert
+    name: nativelink-worker-alert