Add example for Reclient.

deployment-examples/docker-compose-reclient/README.md

@@ -0,0 +1,90 @@
+# Reclient deployment
+
+This example shows how to set up a Nativ Link to work with Reclient for
+Chromium builds.  This configuration is an example that runs all of the
+services on a single machine which is obviously pointless, however it is a
+good starting point to allow you to move the CAS and Worker roles to other
+machines.
+
+## Roles
+
+There are a number of roles required to perform a Reclient set up:
+
+ - Native Link Scheduler - The Native Link instance that handles all of the
+   communications from Reproxy.  This includes scheduling jobs on the workers
+   and proxying the CAS requests.
+
+ - Native Link CAS - The storage instance that stores all required build files
+   and the build outputs.  This also acts as the AC (action cache) which stores
+   the results of previous actions.
+
+ - Native Link Worker - There can be many of these and they perform the build
+   work.  They are rate limited to perform one build per core using the
+   cpu_count property.  The worker_precondition_script.sh ensures that they have
+   sufficient RAM to perform a build before it starts.
+
+## Running
+
+To start Native Link simply execute `docker-compose up` from this directory.
+A new Native Link instance with all of the backing will be built and started on
+port 50052.
+
+Create the file buildtools/reclient_cfgs/reproxy.cfg with the following:
+```
+instance=main
+service=127.0.0.1:50052
+# Disables both TLS and authentication
+service_no_security=true
+# Required to stop autoninja from complaining about authentication despite being
+# implied by service_no_security
+service_no_auth=true
+# Try not to use local execution, can't set to 0 otherwise it appears to ignore
+local_resource_fraction=0.00001
+log_format=reducedtext
+automatic_auth=false
+gcert_refresh_timeout=20
+fail_early_min_action_count=4000
+fail_early_min_fallback_ratio=0.5
+deps_cache_max_mb=256
+# TODO(b/276727504) Re-enable once noop build shutdown time bug is fixed
+# enable_deps_cache=true
+async_reproxy_termination=true
+use_unified_uploads=true
+fast_log_collection=true
+depsscanner_address=exec://%s/buildtools/reclient/scandeps_server
+
+# Improve upload/download concurrency
+max_concurrent_streams_per_conn=50
+max_concurrent_requests_per_conn=50
+min_grpc_connections=50
+cas_concurrency=1000
+
+# Native Link doesn't currently support compressed blob upload
+compression_threshold=-1
+use_batches=false
+
+# Metric metadata
+metrics_namespace=main
+```
+
+Now that the environment is set up you simply need to enable it for your
+Chromium build by modifying your `args.gn` file to contain:
+
+```
+use_remoteexec=true
+rbe_cfg_dir="../../buildtools/reclient_cfgs/linux"
+```
+
+Alternately run:
+
+```sh
+gn gen --args="use_remoteexec=true rbe_cfg_dir=\"../../buildtools/reclient_cfgs/linux\"" out/Default
+```
+
+Now run your build using autoninja which will automatically start reproxy and
+do all of your execution for you.
+
+You should set a crontab to clean up stale docker images nightly on workers:
+```sh
+docker image prune -a --filter "until=24h" -f
+```

deployment-examples/docker-compose-reclient/docker-compose.yml

@@ -0,0 +1,72 @@
+# Copyright 2023 The Native Link Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '3.4'
+
+services:
+  native_link_local_cas:
+    image: tracemachina/native-link:latest
+    build:
+      context: ../..
+      dockerfile: ./deployment-examples/docker-compose/Dockerfile
+      network: host
+    volumes:
+      - ${NATIVE_LINK_DIR:-~/.cache/native-link}:/root/.cache/native-link
+      - type: bind
+        source: .
+        target: /root
+    environment:
+      RUST_LOG: ${RUST_LOG:-warn}
+    ports: [ "50051:50051/tcp" ]
+    command: |
+      turbo-cache /root/local-storage-cas.json
+
+  native_link_scheduler:
+    image: tracemachina/native-link:latest
+    volumes:
+      - type: bind
+        source: .
+        target: /root
+    environment:
+      RUST_LOG: ${RUST_LOG:-warn}
+      CAS_ENDPOINT: native_link_local_cas
+    ports: [ "50052:50052/tcp", "50061:50061/tcp" ]
+    command: |
+      turbo-cache /root/scheduler.json
+
+  native_link_executor:
+    image: tracemachina/native-link-worker:latest
+    build:
+      context: ../..
+      dockerfile: ./deployment-examples/docker-compose/Dockerfile
+      network: host
+      args:
+        ADDITIONAL_SETUP_WORKER_CMD: >-
+          DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y docker.io &&
+          rm -rf /var/lib/apt/lists/*
+    volumes:
+      - ${NATIVE_LINK_DIR:-~/.cache/native-link}:/root/.cache/native-link
+      - type: bind
+        source: .
+        target: /root
+      - type: bind
+        source: /var/run/docker.sock
+        target: /var/run/docker.sock
+    environment:
+      RUST_LOG: ${RUST_LOG:-warn}
+      CAS_ENDPOINT: native_link_local_cas
+      SCHEDULER_ENDPOINT: native_link_scheduler
+      NATIVE_LINK_DIR: ${HOME}/.cache/native-link
+    command: |
+      turbo-cache /root/worker.json

deployment-examples/docker-compose-reclient/local-storage-cas.json

@@ -0,0 +1,77 @@
+// This configuration will place objects in various folders in
+// `~/.cache/native-link`. It will store all data on disk and
+// allows for restarts of the underlying service. It is optimized
+// to use some in-memory optimizations for hot paths.
+{
+  "stores": {
+    "CAS_MAIN_STORE": {
+      "fast_slow": {
+        "fast": {
+          "memory": {
+            "eviction_policy": {
+              // 1gb
+              "max_bytes": 1000000000
+            }
+          }
+        },
+        "slow": {
+          "filesystem": {
+            "content_path": "/root/.cache/native-link/content_path-cas",
+            "temp_path": "/root/.cache/native-link/tmp_path-cas",
+            "eviction_policy": {
+              // 5gb.
+              "max_bytes": 5000000000,
+              // 200mb
+              "evict_bytes": 200000000
+            }
+          }
+        }
+      }
+    },
+    "AC_MAIN_STORE": {
+      "fast_slow": {
+        "fast": {
+          "memory": {
+            "eviction_policy": {
+              // 200mb
+              "max_bytes": 200000000
+            }
+          }
+        },
+        "slow": {
+           "filesystem": {
+             "content_path": "/root/.cache/native-link/content_path-cas_ac",
+             "temp_path": "/root/.cache/native-link/tmp_path-cas_ac",
+             "eviction_policy": {
+                // 1gb.
+                "max_bytes": 1000000000,
+             }
+          }
+        }
+      }
+    }
+  },
+  "servers": [{
+    "listen_address": "0.0.0.0:50051",
+    "services": {
+      "cas": {
+        "main": {
+          "cas_store": "CAS_MAIN_STORE"
+        }
+      },
+      "ac": {
+        "main": {
+          "ac_store": "AC_MAIN_STORE"
+        }
+      },
+      "capabilities": {},
+      "bytestream": {
+        "cas_stores": {
+          "main": "CAS_MAIN_STORE",
+        },
+        // According to https://github.com/grpc/grpc.github.io/issues/371 16KiB - 64KiB is optimal.
+        "max_bytes_per_stream": 64000, // 64kb.
+      }
+    }
+  }]
+}

deployment-examples/docker-compose-reclient/run_in_container.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+# This script is run instead of the provided entrypoint in the action provided
+# to the worker.  It is passed the full entrypoint.  This allows it to be
+# executed in a container rather than on the host machine.  In this case, we are
+# already running in a container, so we execute Docker-in-Docker which causes
+# a few headaches about volumes to mount.
+
+# If a container is specified and starts with docker:// then run the command in Docker
+if [ "$(echo "$CONTAINER" | cut -c-9)" = "docker://" ]; then
+    # Top level work directory is /root/.cache/native-link/work/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+    # However, we are doing Docker-in-Docker and therefore the work directory is
+    # on the host and we need to account for that.  This strips off the
+    # /root/.cache/native-link and that is populated with $HOST_ROOT from the
+    # worker.json which is then populated by ${NATIVE_LINK_DIR} from the
+    # docker-compose.yml.
+    WORK_DIRECTORY=$(pwd | cut -c25-94)
+    WORKING_DIRECTORY=$(pwd | cut -c95-)
+    CONTAINER_NAME=$(echo "$CONTAINER" | cut -c10-)
+    docker pull -q $CONTAINER_NAME > /dev/null
+    exec docker run --rm --network none -w /work${WORKING_DIRECTORY} -v ${HOST_ROOT}${WORK_DIRECTORY}:/work $CONTAINER_NAME /bin/env "$@"
+fi
+
+# Default to simply running the command outside of a container
+exec "$@"

deployment-examples/docker-compose-reclient/scheduler.json

@@ -0,0 +1,86 @@
+{
+  "stores": {
+    "GRPC_LOCAL_STORE": {
+      "grpc": {
+        "instance_name": "main",
+        "endpoints": ["grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"],
+        "store_type": "CAS"
+      }
+    },
+    "GRPC_LOCAL_AC_STORE": {
+      "grpc": {
+        "instance_name": "main",
+        "endpoints": ["grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"],
+        "store_type": "AC"
+      }
+    }
+  },
+  "schedulers": {
+    "MAIN_SCHEDULER": {
+      "property_modifier": {
+        "modifications": [
+          {"Add": {"name": "cpu_count", "value": "1"}},
+          {"Remove": "label:action_default"},
+        ],
+        "scheduler": {
+          "simple": {
+            "supported_platform_properties": {
+              "cpu_count": "Minimum",
+              "container-image": "Priority"
+            }
+          }
+        }
+      }
+    }
+  },
+  "servers": [{
+    "listen_address": "0.0.0.0:50052",
+    "services": {
+      "ac": {
+        // Chromium default is:
+        // projects/rbe-chrome-untrusted/instances/default_instance
+        "main": {
+          "ac_store": "GRPC_LOCAL_AC_STORE"
+        }
+      },
+      "cas": {
+        "main": {
+         "cas_store": "GRPC_LOCAL_STORE"
+        }
+      },
+      "bytestream": {
+        "cas_stores": {
+         "main": "GRPC_LOCAL_STORE"
+        },
+        "max_bytes_per_stream": 64000,
+      },
+      "execution": {
+        "main": {
+          "cas_store": "GRPC_LOCAL_STORE",
+          "scheduler": "MAIN_SCHEDULER",
+        }
+      },
+      "capabilities": {
+        "main": {
+          "remote_execution": {
+            "scheduler": "MAIN_SCHEDULER",
+          }
+        }
+      }
+    }
+  }, {
+    "listen_address": "0.0.0.0:50061",
+    "services": {
+      // Note: This should be served on a different port, because it has
+      // a different permission set than the other services.
+      // In other words, this service is a backend api. The ones above
+      // are a frontend api.
+      "worker_api": {
+        "scheduler": "MAIN_SCHEDULER",
+      },
+      "prometheus": {
+        "path": "/metrics"
+      },
+    }
+  }]
+}