-
Notifications
You must be signed in to change notification settings - Fork 45
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
5d94684
commit f5da2d3
Showing
30 changed files
with
1,433 additions
and
49 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| name: Build remote build docker image | ||
|
|
||
| on: | ||
| push: | ||
| branches: | ||
| - "feature/*" | ||
| workflow_dispatch: | ||
|
|
||
| env: | ||
| REGISTRY: ${{ secrets.REGISTRY }} | ||
| IMAGE_NAME: ${{ github.repository }} | ||
|
|
||
| jobs: | ||
| build-and-push-image: | ||
| runs-on: ubuntu-20.04 | ||
| if: "${{ !startsWith(github.event.head_commit.message, 'GitBook: [#') }}" | ||
| permissions: | ||
| contents: read | ||
| id-token: write | ||
| packages: write | ||
|
|
||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v3 | ||
|
|
||
| - id: "auth" | ||
| name: "Authenticate to Google Cloud" | ||
| uses: "google-github-actions/auth@v1" | ||
| with: | ||
| token_format: "access_token" | ||
| workload_identity_provider: ${{ secrets.GCR_WORKLOAD_IDENTITY_PROVIDER }} | ||
| service_account: ${{ secrets.GCR_SERVICE_ACCOUNT }} | ||
|
|
||
| - name: Login to GCR | ||
| uses: docker/login-action@v1 | ||
| with: | ||
| registry: gcr.io | ||
| username: oauth2accesstoken | ||
| password: ${{ steps.auth.outputs.access_token }} | ||
|
|
||
| - name: Generate docker tags/labels from github build context | ||
| id: meta | ||
| uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 | ||
| with: | ||
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-remotebuild | ||
| tags: | | ||
| type=ref,event=tag | ||
| type=sha,prefix= | ||
| type=raw,enable=${{ github.ref == 'refs/heads/develop' }},value=develop | ||
| type=raw,enable=${{ startsWith(github.ref, 'refs/heads/release/v') }},value=${{ steps.extract_branch.outputs.release_train }} | ||
| flavor: | | ||
| latest=${{ startsWith(github.ref, 'refs/tags/') }} | ||
| - name: Build and push Docker image | ||
| uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc | ||
| with: | ||
| secrets: | | ||
| "github_token=${{ secrets.GHUB_ACCESS_TOKEN }}" | ||
| context: . | ||
| file: ./Dockerfile.remotebuild | ||
| push: true | ||
| tags: ${{ steps.meta.outputs.tags }} | ||
| # org.opencontainers.image.version will match the tag name | ||
| labels: ${{ steps.meta.outputs.labels }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| FROM rust:1.78-alpine as rust-builder | ||
|
|
||
| FROM golang:1.22-alpine3.19 | ||
|
|
||
| ENV RUSTUP_HOME=/usr/local/rustup \ | ||
| CARGO_HOME=/usr/local/cargo \ | ||
| PATH=/usr/local/cargo/bin:$PATH \ | ||
| RUST_VERSION=1.78.0 | ||
|
|
||
| COPY --from=rust-builder /usr/local/cargo /usr/local/cargo/ | ||
| COPY --from=rust-builder /usr/local/rustup /usr/local/rustup/ | ||
|
|
||
| # Install needed rustup components | ||
| RUN rustup component add rustfmt && cargo install rustfmt || true | ||
| RUN rustup default stable | ||
| RUN rustup target add wasm32-unknown-unknown | ||
|
|
||
| # Install build essentials and curl | ||
| RUN apk update && apk add curl build-base sccache | ||
|
|
||
| # Set the Shared Compilation Cache for Cargo to use sccache | ||
| # This greatly improves build times when running concurrent builds | ||
| ENV RUSTC_WRAPPER=/usr/bin/sccache | ||
|
|
||
| # Install buf | ||
| RUN GO111MODULE=on GOBIN=/usr/local/bin go install github.com/bufbuild/buf/cmd/[email protected] | ||
|
|
||
| # Build substreams cli | ||
| # RUN curl -s https://api.github.com/repos/streamingfast/substreams/releases/latest | awk "/download.url.*linux_$(uname -m)/ {print \$2}" | sed 's/"//g' | curl -L $1 | tar zxf - | ||
|
|
||
| WORKDIR /app/data | ||
|
|
||
| # Add the entire cli source code | ||
| COPY . ./substreams-cli | ||
|
|
||
| WORKDIR /app/data/substreams-cli | ||
|
|
||
| # Install the substreams cli | ||
| RUN go install ./cmd/substreams | ||
|
|
||
| WORKDIR /app/data | ||
|
|
||
| # Remove all the source code to have a smaller image | ||
| RUN rm -r substreams-cli | ||
|
|
||
| # Add only the remotebuild source code | ||
| COPY remotebuild remotebuild | ||
|
|
||
| EXPOSE 9000 | ||
|
|
||
| WORKDIR /app/data/remotebuild | ||
|
|
||
| RUN go install . | ||
|
|
||
| ENTRYPOINT [ "remotebuild" ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,138 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "os" | ||
| "strings" | ||
|
|
||
| "github.com/spf13/cobra" | ||
| "github.com/streamingfast/cli" | ||
| "github.com/streamingfast/cli/sflags" | ||
| "github.com/streamingfast/dgrpc" | ||
| "go.uber.org/zap" | ||
|
|
||
| pbbuild "github.com/streamingfast/substreams/remotebuild/pb/sf/remotebuild/v1" | ||
| ) | ||
|
|
||
| func init() { | ||
| remoteBuildCmd.Flags().StringArray("env", nil, "Environment variables to pass to the build server") | ||
| remoteBuildCmd.Flags().StringP("output-name", "o", "substreams.spkg", "Spkg name to collect") | ||
|
|
||
| rootCmd.AddCommand(remoteBuildCmd) | ||
| } | ||
|
|
||
| var remoteBuildCmd = &cobra.Command{ | ||
| Use: "remote-build <remote-build-url> <zipped_source_code>", | ||
| Short: "Send request to remote build server to build and package a substreams", | ||
| Long: cli.Dedent(` | ||
| Call the remote build server to build and package a substream. You can also add in environment variables to pass to the build server. | ||
| Which will be injected in your substreams build. Make sure you align the make package output command with the collect pattern. By default | ||
| it will collect the substreams.spkg file, but if you gave it a different name with the -o or --output-file flag, you should pass it here too. | ||
| Example of valide make package commands: substreams pack: | ||
| .PHONE: package | ||
| pack: build | ||
| substreams pack -o my-substreams.spkg substreams.yaml | ||
| Then you will need to pass in the --output-name="my-substreams.spkg" flag to the remote-build command. | ||
| Examples: substreams remote-build localhost:9000 my-substream.zip --env="ENV0=test0,ENV1=test1" --env "ENV2=test2" --env "ENV3=test3" --output-name="my-substream.spkg" | ||
| `), | ||
| RunE: remoteBuildE, | ||
| Args: cobra.ExactArgs(2), | ||
| SilenceUsage: true, | ||
| } | ||
|
|
||
| func remoteBuildE(cmd *cobra.Command, args []string) error { | ||
| cloudRunServiceURL := args[0] | ||
| filepath := args[1] | ||
| envs := sflags.MustGetStringArray(cmd, "env") | ||
| spkgName := sflags.MustGetString(cmd, "output-name") | ||
|
|
||
| plaintext := false | ||
| if strings.HasPrefix(cloudRunServiceURL, "localhost") { | ||
| plaintext = true | ||
| } | ||
|
|
||
| cloudRunServiceURL = strings.Replace(cloudRunServiceURL, "https://", "", 1) | ||
| if len(strings.Split(cloudRunServiceURL, ":")) == 1 { | ||
| if plaintext { | ||
| cloudRunServiceURL = fmt.Sprintf("%s:9000", cloudRunServiceURL) | ||
| } else { | ||
| // add the port if it is missing | ||
| cloudRunServiceURL = fmt.Sprintf("%s:443", cloudRunServiceURL) | ||
| } | ||
| } | ||
|
|
||
| credsOption, err := dgrpc.WithAutoTransportCredentials(false, plaintext, false) | ||
| cli.NoError(err, "unable to create auto transport credentials option") | ||
|
|
||
| fmt.Printf("Connecting to remote build server at %s ...\n", cloudRunServiceURL) | ||
| conn, err := dgrpc.NewClientConn(cloudRunServiceURL, credsOption) | ||
| cli.NoError(err, "unable to create external client") | ||
| defer func() { | ||
| if err := conn.Close(); err != nil { | ||
| zlog.Error("unable to close connection gracefully", zap.Error(err)) | ||
| } | ||
| }() | ||
|
|
||
| fmt.Printf("Reading file: %s ...\n", filepath) | ||
| b, err := os.ReadFile(filepath) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to read file: %w", err) | ||
| } | ||
|
|
||
| fmt.Println("Sending build request ...") | ||
| client := pbbuild.NewBuildServiceClient(conn) | ||
| buildResponse, err := client.Build(context.Background(), &pbbuild.BuildRequest{ | ||
| SourceCode: b, | ||
| Env: envs, | ||
| CollectPattern: spkgName, | ||
| }) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to build: %w", err) | ||
| } | ||
|
|
||
| for { | ||
| resp, err := buildResponse.Recv() | ||
| if err != nil { | ||
| return fmt.Errorf("failed to build: %w", err) | ||
| } | ||
|
|
||
| if resp == nil { | ||
| break | ||
| } | ||
|
|
||
| if resp.Error != "" { | ||
| if resp.Logs != "" { | ||
| fmt.Print(resp.Logs) | ||
| } | ||
|
|
||
| return fmt.Errorf("failed to build: %s", resp.Error) | ||
| } | ||
|
|
||
| if len(resp.Artifacts) != 0 { | ||
| // printout the rest of the logs, if there are any | ||
| if resp.Logs != "" { | ||
| fmt.Print(resp.Logs) | ||
| } | ||
| for _, artifact := range resp.Artifacts { | ||
| err = os.WriteFile(artifact.Filename, artifact.Content, 0644) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to write file: %w", err) | ||
| } | ||
| } | ||
| break | ||
| } | ||
|
|
||
| // print out the logs as we get them, if there are any | ||
| if resp.Logs != "" { | ||
| fmt.Print(resp.Logs) | ||
| } | ||
| } | ||
|
|
||
| fmt.Println("All done!") | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.