Merge pull request #3 from sagaxyz/feature/ssc-scripts-mainnet-statesync

.github/workflows/shfmt.yml

@@ -0,0 +1,22 @@
+name: shfmt
+
+on: [push]
+
+permissions:
+ contents: read
+jobs:
+  shfmt:
+    name: Run shfmt
+    runs-on: ubuntu-latest
+    env:
+      GOPRIVATE: github.com/sagaxyz/*
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '>=1.22.2'
+      - uses: actions/checkout@v4
+      - name: Install shfmt
+        run: go install mvdan.cc/sh/v3/cmd/shfmt@latest
+      - name: Check Shell Script Formatting
+        run: find . -type f -name "*.sh" -exec shfmt -i 2 -ci -d {} +

README.md

@@ -1,8 +1,7 @@
 # helper-tools
-
 This repository contains some helper tools, scripts etc.
 
 - gcp-ssc-scripts - bash scripts to simplify SSC full nodes deployment
 
 
-More tools will be added in the future.
+For detailed instructions check Readme files in subdirectories. More tools will be added in the future.

gcp-ssc-scripts/README.md

@@ -0,0 +1,18 @@
+# SSC startup scripts
+These scripts simplify a spin-up process of SSC nodes. Initially designed for GCP, they can be easily reused for basically any Linux machine.
+
+## Requirements
+Ubuntu 20.04/22.04. Other Linux distributions are not yet tested though will likely work too.
+
+## What do they do
+- Install Go and other helper tools
+- Create sscdserviceuser
+- Build sscd (SSC daemon binary)
+- Create systemd service for sscd
+- Init sscd and adjust configuration files
+- Modify configuration files depending of the node type - RPC, gRPC, archival and so on
+- Start sscd service
+
+## How to use
+
+For GCP these scripts can be used as [startup scritps](https://cloud.google.com/compute/docs/instances/startup-scripts/linux). For other cases you can just run one of these scripts manually.

gcp-ssc-scripts/mainnet/ssc-archive.sh

@@ -4,8 +4,8 @@ set -e pipefail
 
 # if service exists, do nothing
 if systemctl list-units --full -all | grep -q 'sscd.service'; then
-   echo "Service exists."
-   exit 0
+  echo "Service exists."
+  exit 0
 fi
 
 # install dependencies
@@ -14,8 +14,11 @@ apt upgrade -y
 apt install build-essential make wget git daemon jq python3-pip -y
 pip install yq
 
-wget https://golang.org/dl/go1.22.2.linux-amd64.tar.gz
-tar -C /usr/local -xzf go1.22.2.linux-amd64.tar.gz
+GO_VERSION="${GO_VERSION:-1.22.2}"
+SSC_VERSION="${SSC_VERSION:-v0.1.5}"
+
+wget https://golang.org/dl/go$GO_VERSION.linux-amd64.tar.gz
+tar -C /usr/local -xzf go$GO_VERSION.linux-amd64.tar.gz
 # shellcheck disable=SC2016
 # shellcheck disable=SC1091
 echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile
@@ -32,9 +35,8 @@ chown -R sscdserviceuser:sscdserviceuser /opt/sscd
 mkdir -p /var/log/sscd
 chown -R sscdserviceuser:sscdserviceuser /var/log/sscd
 
-
 # install ssc
-sudo -u sscdserviceuser git clone -b v0.1.5 https://github.com/sagaxyz/ssc /tmp/ssc
+sudo -u sscdserviceuser git clone -b $SSC_VERSION https://github.com/sagaxyz/ssc /tmp/ssc
 sudo -u sscdserviceuser PATH=$PATH:/usr/local/go/bin make install -C /tmp/ssc/
 cp /opt/sscd/go/bin/sscd /usr/local/bin/
 sudo -u sscdserviceuser sscd version
@@ -62,18 +64,24 @@ EOF
 # init ssc
 sudo -u sscdserviceuser HOME=/opt/sscd/ sscd init saganode --chain-id ssc-1
 
-# set persistent peers
-sudo -u sscdserviceuser sed -i 's/persistent_peers = ""/persistent_peers = "[email protected]:26014,[email protected]:17300,[email protected]:11156,[email protected]:11156,[email protected]:11156"/g' /opt/sscd/.ssc/config/config.toml
+# set seeds
+sudo -u sscdserviceuser tomlq -t '.p2p.seeds = "[email protected]:26656,[email protected]:26656,[email protected]:26656"' /opt/sscd/.ssc/config/config.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/config.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/config.toml.out /opt/sscd/.ssc/config/config.toml
 
 # get genesis file
 sudo -u sscdserviceuser wget -O /opt/sscd/.ssc/config/genesis.json 'https://raw.githubusercontent.com/sagaxyz/mainnet/main/genesis/genesis.json'
 
 # adjust config.toml
-sudo -u sscdserviceuser tomlq -t '.rpc.laddr = "tcp://0.0.0.0:26657"' /opt/sscd/.ssc/config/config.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/config.toml.out > /dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/config.toml.out /opt/sscd/.ssc/config/config.toml
+sudo -u sscdserviceuser tomlq -t '.rpc.laddr = "tcp://0.0.0.0:26657"' /opt/sscd/.ssc/config/config.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/config.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/config.toml.out /opt/sscd/.ssc/config/config.toml
 
 # adjust app.toml
-sudo -u sscdserviceuser tomlq -t '.pruning = "nothing"' /opt/sscd/.ssc/config/app.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/app.toml.out > /dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/app.toml.out /opt/sscd/.ssc/config/app.toml
+sudo -u sscdserviceuser tomlq -t '.pruning = "nothing"' /opt/sscd/.ssc/config/app.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/app.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/app.toml.out /opt/sscd/.ssc/config/app.toml
+
+# for archival node we need to get a snapshot
+sudo -u sscdserviceuser wget -O /opt/sscd/latest.tar https://saga-snapshots-mainnet.s3.amazonaws.com/ssc/archive/latest.tar
+sudo -u sscdserviceuser tar -xvf /opt/sscd/latest.tar -C /opt/sscd/.ssc/
+sudo -u sscdserviceuser mv /opt/sscd/.ssc/root/ /opt/sscd/.ssc/
+sudo -u sscdserviceuser rm -f /opt/sscd/latest.tar
 
 # start
 systemctl enable sscd.service
-systemctl start sscd
+systemctl start sscd

gcp-ssc-scripts/mainnet/ssc-grpc.sh

@@ -4,8 +4,8 @@ set -e pipefail
 
 # if service exists, do nothing
 if systemctl list-units --full -all | grep -q 'sscd.service'; then
-   echo "Service exists."
-   exit 0
+  echo "Service exists."
+  exit 0
 fi
 
 # install dependencies
@@ -14,8 +14,11 @@ apt upgrade -y
 apt install build-essential make wget git daemon jq python3-pip -y
 pip install yq
 
-wget https://golang.org/dl/go1.22.2.linux-amd64.tar.gz
-tar -C /usr/local -xzf go1.22.2.linux-amd64.tar.gz
+GO_VERSION="${GO_VERSION:-1.22.2}"
+SSC_VERSION="${SSC_VERSION:-v0.1.5}"
+
+wget https://golang.org/dl/go$GO_VERSION.linux-amd64.tar.gz
+tar -C /usr/local -xzf go$GO_VERSION.linux-amd64.tar.gz
 # shellcheck disable=SC2016
 # shellcheck disable=SC1091
 echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile
@@ -32,9 +35,8 @@ chown -R sscdserviceuser:sscdserviceuser /opt/sscd
 mkdir -p /var/log/sscd
 chown -R sscdserviceuser:sscdserviceuser /var/log/sscd
 
-
 # install ssc
-sudo -u sscdserviceuser git clone -b v0.1.5 https://github.com/sagaxyz/ssc /tmp/ssc
+sudo -u sscdserviceuser git clone -b $SSC_VERSION https://github.com/sagaxyz/ssc /tmp/ssc
 sudo -u sscdserviceuser PATH=$PATH:/usr/local/go/bin make install -C /tmp/ssc/
 cp /opt/sscd/go/bin/sscd /usr/local/bin/
 sudo -u sscdserviceuser sscd version
@@ -62,15 +64,30 @@ EOF
 # init ssc
 sudo -u sscdserviceuser HOME=/opt/sscd/ sscd init saganode --chain-id ssc-1
 
-# set persistent peers
-sudo -u sscdserviceuser sed -i 's/persistent_peers = ""/persistent_peers = "[email protected]:26014,[email protected]:17300,[email protected]:11156,[email protected]:11156,[email protected]:11156"/g' /opt/sscd/.ssc/config/config.toml
+# set seeds
+sudo -u sscdserviceuser tomlq -t '.p2p.seeds = "[email protected]:26656,[email protected]:26656,[email protected]:26656"' /opt/sscd/.ssc/config/config.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/config.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/config.toml.out /opt/sscd/.ssc/config/config.toml
 
 # get genesis file
 sudo -u sscdserviceuser wget -O /opt/sscd/.ssc/config/genesis.json 'https://raw.githubusercontent.com/sagaxyz/mainnet/main/genesis/genesis.json'
 
 # adjust app.toml
-sudo -u sscdserviceuser tomlq -t '.grpc.address = "0.0.0.0:9090"' /opt/sscd/.ssc/config/app.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/app.toml.out > /dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/app.toml.out /opt/sscd/.ssc/config/app.toml
+sudo -u sscdserviceuser tomlq -t '.grpc.address = "0.0.0.0:9090"' /opt/sscd/.ssc/config/app.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/app.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/app.toml.out /opt/sscd/.ssc/config/app.toml
+
+# obtain state sync actual data
+RPC_URL=https://ssc-rpc.sagarpc.io
+CURRENT_BLOCK=$(curl -s $RPC_URL/status | jq '.result.sync_info.latest_block_height' | awk 'gsub("\"","",$0)')
+TRUST_HEIGHT=$((CURRENT_BLOCK - 1000))
+TRUST_BLOCK=$(curl -s $RPC_URL/block\?height=$TRUST_HEIGHT)
+TRUST_HASH=$(echo $TRUST_BLOCK | jq -r '.result.block_id.hash')
+
+# configure state sync
+sudo -u sscdserviceuser tomlq -t '.statesync.enable = true' /opt/sscd/.ssc/config/config.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/config.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/config.toml.out /opt/sscd/.ssc/config/config.toml
+sudo -u sscdserviceuser tomlq -t '.statesync.trust_height = '$TRUST_HEIGHT'' /opt/sscd/.ssc/config/config.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/config.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/config.toml.out /opt/sscd/.ssc/config/config.toml
+sudo -u sscdserviceuser tomlq -t '.statesync.trust_hash = '\"$TRUST_HASH\"'' /opt/sscd/.ssc/config/config.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/config.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/config.toml.out /opt/sscd/.ssc/config/config.toml
+sudo -u sscdserviceuser tomlq -t '.statesync.rpc_servers = "tcp://ssc-statesync-eu.sagarpc.io:80,tcp://ssc-statesync-us.sagarpc.io:80,tcp://ssc-statesync-kr.sagarpc.io:80"' /opt/sscd/.ssc/config/config.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/config.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/config.toml.out /opt/sscd/.ssc/config/config.toml
+sudo -u sscdserviceuser tomlq -t '.statesync.discovery_time = "30s"' /opt/sscd/.ssc/config/config.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/config.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/config.toml.out /opt/sscd/.ssc/config/config.toml
+sudo -u sscdserviceuser tomlq -t '.statesync.chunk_request_timeout = "60s"' /opt/sscd/.ssc/config/config.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/config.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/config.toml.out /opt/sscd/.ssc/config/config.toml
 
 # start
 systemctl enable sscd.service
-systemctl start sscd
+systemctl start sscd

gcp-ssc-scripts/mainnet/ssc-lcd.sh

@@ -4,8 +4,8 @@ set -e pipefail
 
 # if service exists, do nothing
 if systemctl list-units --full -all | grep -q 'sscd.service'; then
-   echo "Service exists."
-   exit 0
+  echo "Service exists."
+  exit 0
 fi
 
 # install dependencies
@@ -14,8 +14,11 @@ apt upgrade -y
 apt install build-essential make wget git daemon jq python3-pip -y
 pip install yq
 
-wget https://golang.org/dl/go1.22.2.linux-amd64.tar.gz
-tar -C /usr/local -xzf go1.22.2.linux-amd64.tar.gz
+GO_VERSION="${GO_VERSION:-1.22.2}"
+SSC_VERSION="${SSC_VERSION:-v0.1.5}"
+
+wget https://golang.org/dl/go$GO_VERSION.linux-amd64.tar.gz
+tar -C /usr/local -xzf go$GO_VERSION.linux-amd64.tar.gz
 # shellcheck disable=SC2016
 # shellcheck disable=SC1091
 echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile
@@ -32,9 +35,8 @@ chown -R sscdserviceuser:sscdserviceuser /opt/sscd
 mkdir -p /var/log/sscd
 chown -R sscdserviceuser:sscdserviceuser /var/log/sscd
 
-
 # install ssc
-sudo -u sscdserviceuser git clone -b v0.1.5 https://github.com/sagaxyz/ssc /tmp/ssc
+sudo -u sscdserviceuser git clone -b $SSC_VERSION https://github.com/sagaxyz/ssc /tmp/ssc
 sudo -u sscdserviceuser PATH=$PATH:/usr/local/go/bin make install -C /tmp/ssc/
 cp /opt/sscd/go/bin/sscd /usr/local/bin/
 sudo -u sscdserviceuser sscd version
@@ -63,17 +65,32 @@ EOF
 # init ssc
 sudo -u sscdserviceuser HOME=/opt/sscd/ sscd init saganode --chain-id ssc-1
 
-# set persistent peers
-sudo -u sscdserviceuser sed -i 's/persistent_peers = ""/persistent_peers = "[email protected]:26014,[email protected]:17300,[email protected]:11156,[email protected]:11156,[email protected]:11156"/g' /opt/sscd/.ssc/config/config.toml
+# set seeds
+sudo -u sscdserviceuser tomlq -t '.p2p.seeds = "[email protected]:26656,[email protected]:26656,[email protected]:26656"' /opt/sscd/.ssc/config/config.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/config.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/config.toml.out /opt/sscd/.ssc/config/config.toml
 
 # get genesis file
 sudo -u sscdserviceuser wget -O /opt/sscd/.ssc/config/genesis.json 'https://raw.githubusercontent.com/sagaxyz/mainnet/main/genesis/genesis.json'
 
 # adjust app.toml
-sudo -u sscdserviceuser tomlq -t '.api.enable = true' /opt/sscd/.ssc/config/app.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/app.toml.out > /dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/app.toml.out /opt/sscd/.ssc/config/app.toml
-sudo -u sscdserviceuser tomlq -t '.api.address = "tcp://0.0.0.0:80"' /opt/sscd/.ssc/config/app.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/app.toml.out > /dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/app.toml.out /opt/sscd/.ssc/config/app.toml
-sudo -u sscdserviceuser tomlq -t '.api."enabled-unsafe-cors" = true' /opt/sscd/.ssc/config/app.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/app.toml.out > /dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/app.toml.out /opt/sscd/.ssc/config/app.toml
+sudo -u sscdserviceuser tomlq -t '.api.enable = true' /opt/sscd/.ssc/config/app.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/app.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/app.toml.out /opt/sscd/.ssc/config/app.toml
+sudo -u sscdserviceuser tomlq -t '.api.address = "tcp://0.0.0.0:80"' /opt/sscd/.ssc/config/app.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/app.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/app.toml.out /opt/sscd/.ssc/config/app.toml
+sudo -u sscdserviceuser tomlq -t '.api."enabled-unsafe-cors" = true' /opt/sscd/.ssc/config/app.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/app.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/app.toml.out /opt/sscd/.ssc/config/app.toml
+
+# obtain state sync actual data
+RPC_URL=https://ssc-rpc.sagarpc.io
+CURRENT_BLOCK=$(curl -s $RPC_URL/status | jq '.result.sync_info.latest_block_height' | awk 'gsub("\"","",$0)')
+TRUST_HEIGHT=$((CURRENT_BLOCK - 1000))
+TRUST_BLOCK=$(curl -s $RPC_URL/block\?height=$TRUST_HEIGHT)
+TRUST_HASH=$(echo $TRUST_BLOCK | jq -r '.result.block_id.hash')
+
+# configure state sync
+sudo -u sscdserviceuser tomlq -t '.statesync.enable = true' /opt/sscd/.ssc/config/config.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/config.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/config.toml.out /opt/sscd/.ssc/config/config.toml
+sudo -u sscdserviceuser tomlq -t '.statesync.trust_height = '$TRUST_HEIGHT'' /opt/sscd/.ssc/config/config.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/config.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/config.toml.out /opt/sscd/.ssc/config/config.toml
+sudo -u sscdserviceuser tomlq -t '.statesync.trust_hash = '\"$TRUST_HASH\"'' /opt/sscd/.ssc/config/config.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/config.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/config.toml.out /opt/sscd/.ssc/config/config.toml
+sudo -u sscdserviceuser tomlq -t '.statesync.rpc_servers = "tcp://ssc-statesync-eu.sagarpc.io:80,tcp://ssc-statesync-us.sagarpc.io:80,tcp://ssc-statesync-kr.sagarpc.io:80"' /opt/sscd/.ssc/config/config.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/config.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/config.toml.out /opt/sscd/.ssc/config/config.toml
+sudo -u sscdserviceuser tomlq -t '.statesync.discovery_time = "30s"' /opt/sscd/.ssc/config/config.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/config.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/config.toml.out /opt/sscd/.ssc/config/config.toml
+sudo -u sscdserviceuser tomlq -t '.statesync.chunk_request_timeout = "60s"' /opt/sscd/.ssc/config/config.toml | sudo -u sscdserviceuser tee /opt/sscd/.ssc/config/config.toml.out >/dev/null && sudo -u sscdserviceuser mv /opt/sscd/.ssc/config/config.toml.out /opt/sscd/.ssc/config/config.toml
 
 # start
 systemctl enable sscd.service
-systemctl start sscd
+systemctl start sscd