Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(cannon): Add backfill config #398

Merged
merged 13 commits into from
Oct 15, 2024
Merged

feat(cannon): Add backfill config #398

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
c1fbdfa
feat(cannon): Add backfill config
samcm Oct 14, 2024
714391d
chore: linting
samcm Oct 14, 2024
80daa8f
chore: linting
samcm Oct 14, 2024
17da307
chore: Add docker logs for failed assertion, disable beaconBlobSidecar
samcm Oct 15, 2024
c70e88c
refactor: Update log field with additional parameters
samcm Oct 15, 2024
2bf3eb2
fix: Update docker logs command to tail 150 lines
samcm Oct 15, 2024
b32be66
fix: Update docker logs command to include stderr output
samcm Oct 15, 2024
392eddb
fix: Increase the tail length for docker logs to 1000
samcm Oct 15, 2024
afd3390
feat: Add logging for derived finalized epoch
samcm Oct 15, 2024
932cade
fix: Update docker logs command to show recent logs
samcm Oct 15, 2024
9e6ae49
chore: Update debug logs in checkpoint iterator
samcm Oct 15, 2024
30db0f7
fix: Update block_total_bytes and block_total_bytes_compressed
samcm Oct 15, 2024
b427255
chore: comment out unused tmate session setup block
samcm Oct 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/cannon/assert_clickhouse.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ yq e '.networks[].types[] | [.name, .assert.query, .assert.expected] | @tsv' "$S
break
else
echo "Assertion failed for $name. Expected: $expected, Got: $result"
docker logs xatu-cannon 2>&1 | grep -i "$name " | tail -n 10 || docker logs xatu-cannon --tail 10
sleep 2
continue
fi
Expand Down
4 changes: 2 additions & 2 deletions .github/cannon/seeding.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,8 @@ networks:
AND epoch_start_date_time = '2024-08-09 02:24:00'
AND block_root = '0xff8e97fdf4033e499681ff0eb056cf5f2877a8f9b16f14e2db3956b75190b9e7'
AND block_version = 'deneb'
AND block_total_bytes = 204103
AND block_total_bytes_compressed = 95454
AND block_total_bytes = 94749
AND block_total_bytes_compressed = 55636
AND parent_root = '0x54264e13e97a137bed2cc1a52cf76a94bb5e385507d34ff3902d18672f4ef79e'
AND state_root = '0x3598ed522160cb6dd0aab216f03b26d61b32d87854c3d38754897017a66004f5'
AND proposer_index = 416934
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/cannon-smoke-test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ jobs:
- name: Create Xatu Cannon config
run: |
cat <<EOF > /tmp/cannon_config.yaml
logging: debug
name: cannon-smoke-test
labels:
ethpandaops: rocks
Expand All @@ -97,7 +98,7 @@ jobs:
beaconBlock:
enabled: true
beaconBlobSidecar:
enabled: true
enabled: false
proposerDuty:
enabled: true
beaconCommittee:
Expand Down
17 changes: 16 additions & 1 deletion pkg/cannon/cannon.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,7 @@ func (c *Cannon) startBeaconBlockProcessor(ctx context.Context) error {
c.beacon,
finalizedCheckpoint,
3,
&c.Config.Derivers.AttesterSlashingConfig.Iterator,
),
c.beacon,
clientMeta,
Expand All @@ -382,6 +383,7 @@ func (c *Cannon) startBeaconBlockProcessor(ctx context.Context) error {
c.beacon,
finalizedCheckpoint,
3,
&c.Config.Derivers.ProposerSlashingConfig.Iterator,
),
c.beacon,
clientMeta,
Expand All @@ -400,6 +402,7 @@ func (c *Cannon) startBeaconBlockProcessor(ctx context.Context) error {
c.beacon,
finalizedCheckpoint,
3,
&c.Config.Derivers.VoluntaryExitConfig.Iterator,
),
c.beacon,
clientMeta,
Expand All @@ -418,6 +421,7 @@ func (c *Cannon) startBeaconBlockProcessor(ctx context.Context) error {
c.beacon,
finalizedCheckpoint,
3,
&c.Config.Derivers.DepositConfig.Iterator,
),
c.beacon,
clientMeta,
Expand All @@ -436,6 +440,7 @@ func (c *Cannon) startBeaconBlockProcessor(ctx context.Context) error {
c.beacon,
finalizedCheckpoint,
3,
&c.Config.Derivers.BLSToExecutionConfig.Iterator,
),
c.beacon,
clientMeta,
Expand All @@ -454,6 +459,7 @@ func (c *Cannon) startBeaconBlockProcessor(ctx context.Context) error {
c.beacon,
finalizedCheckpoint,
3,
&c.Config.Derivers.ExecutionTransactionConfig.Iterator,
),
c.beacon,
clientMeta,
Expand All @@ -472,6 +478,7 @@ func (c *Cannon) startBeaconBlockProcessor(ctx context.Context) error {
c.beacon,
finalizedCheckpoint,
3,
&c.Config.Derivers.WithdrawalConfig.Iterator,
),
c.beacon,
clientMeta,
Expand All @@ -490,6 +497,7 @@ func (c *Cannon) startBeaconBlockProcessor(ctx context.Context) error {
c.beacon,
finalizedCheckpoint,
3,
&c.Config.Derivers.BeaconBlockConfig.Iterator,
),
c.beacon,
clientMeta,
Expand Down Expand Up @@ -524,6 +532,7 @@ func (c *Cannon) startBeaconBlockProcessor(ctx context.Context) error {
c.beacon,
finalizedCheckpoint,
3,
&c.Config.Derivers.BeaconBlobSidecarConfig.Iterator,
),
c.beacon,
clientMeta,
Expand All @@ -542,6 +551,7 @@ func (c *Cannon) startBeaconBlockProcessor(ctx context.Context) error {
c.beacon,
finalizedCheckpoint,
3,
&c.Config.Derivers.ProposerDutyConfig.Iterator,
),
c.beacon,
clientMeta,
Expand All @@ -560,6 +570,7 @@ func (c *Cannon) startBeaconBlockProcessor(ctx context.Context) error {
c.beacon,
finalizedCheckpoint,
3,
&c.Config.Derivers.ElaboratedAttestationConfig.Iterator,
),
c.beacon,
clientMeta,
Expand All @@ -578,6 +589,7 @@ func (c *Cannon) startBeaconBlockProcessor(ctx context.Context) error {
c.beacon,
finalizedCheckpoint,
2,
&c.Config.Derivers.BeaconValidatorsConfig.Iterator,
),
c.beacon,
clientMeta,
Expand All @@ -596,6 +608,7 @@ func (c *Cannon) startBeaconBlockProcessor(ctx context.Context) error {
c.beacon,
finalizedCheckpoint,
2,
&c.Config.Derivers.BeaconCommitteeConfig.Iterator,
),
c.beacon,
clientMeta,
Expand All @@ -621,7 +634,9 @@ func (c *Cannon) startBeaconBlockProcessor(ctx context.Context) error {

go func() {
if err := c.startDeriverWhenReady(ctx, d); err != nil {
c.log.WithError(err).Error("Failed to start deriver")
c.log.
WithField("deriver", d.Name()).
WithError(err).Fatal("Failed to start deriver")
}
}()
}
Expand Down
8 changes: 6 additions & 2 deletions pkg/cannon/deriver/beacon/eth/v1/beacon_blob.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ const (
)

type BeaconBlobDeriverConfig struct {
Enabled bool `yaml:"enabled" default:"true"`
Enabled bool `yaml:"enabled" default:"true"`
Iterator iterator.BackfillingCheckpointConfig `yaml:"iterator"`
}

type BeaconBlobDeriver struct {
Expand All @@ -45,7 +46,10 @@ type BeaconBlobDeriver struct {

func NewBeaconBlobDeriver(log logrus.FieldLogger, config *BeaconBlobDeriverConfig, iter *iterator.BackfillingCheckpoint, beacon *ethereum.BeaconNode, clientMeta *xatu.ClientMeta) *BeaconBlobDeriver {
return &BeaconBlobDeriver{
log: log.WithField("module", "cannon/event/beacon/eth/v1/beacon_blob"),
log: log.WithFields(logrus.Fields{
"module": "cannon/event/beacon/eth/v1/beacon_blob",
"type": BeaconBlobDeriverName.String(),
}),
cfg: config,
iterator: iter,
beacon: beacon,
Expand Down
8 changes: 6 additions & 2 deletions pkg/cannon/deriver/beacon/eth/v1/beacon_committee.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ const (
)

type BeaconCommitteeDeriverConfig struct {
Enabled bool `yaml:"enabled" default:"true"`
Enabled bool `yaml:"enabled" default:"true"`
Iterator iterator.BackfillingCheckpointConfig `yaml:"iterator"`
}

type BeaconCommitteeDeriver struct {
Expand All @@ -43,7 +44,10 @@ type BeaconCommitteeDeriver struct {

func NewBeaconCommitteeDeriver(log logrus.FieldLogger, config *BeaconCommitteeDeriverConfig, iter *iterator.BackfillingCheckpoint, beacon *ethereum.BeaconNode, clientMeta *xatu.ClientMeta) *BeaconCommitteeDeriver {
return &BeaconCommitteeDeriver{
log: log.WithField("module", "cannon/event/beacon/eth/v1/beacon_committee"),
log: log.WithFields(logrus.Fields{
"module": "cannon/event/beacon/eth/v1/beacon_committee",
"type": BeaconCommitteeDeriverName.String(),
}),
cfg: config,
iterator: iter,
beacon: beacon,
Expand Down
10 changes: 7 additions & 3 deletions pkg/cannon/deriver/beacon/eth/v1/beacon_validators.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,9 @@ const (
)

type BeaconValidatorsDeriverConfig struct {
Enabled bool `yaml:"enabled" default:"true"`
ChunkSize int `yaml:"chunkSize" default:"100"`
Enabled bool `yaml:"enabled" default:"true"`
ChunkSize int `yaml:"chunkSize" default:"100"`
Iterator iterator.BackfillingCheckpointConfig `yaml:"iterator"`
}

type BeaconValidatorsDeriver struct {
Expand All @@ -44,7 +45,10 @@ type BeaconValidatorsDeriver struct {

func NewBeaconValidatorsDeriver(log logrus.FieldLogger, config *BeaconValidatorsDeriverConfig, iter *iterator.BackfillingCheckpoint, beacon *ethereum.BeaconNode, clientMeta *xatu.ClientMeta) *BeaconValidatorsDeriver {
return &BeaconValidatorsDeriver{
log: log.WithField("module", "cannon/event/beacon/eth/v1/validators"),
log: log.WithFields(logrus.Fields{
"module": "cannon/event/beacon/eth/v1/validators",
"type": BeaconValidatorsDeriverName.String(),
}),
cfg: config,
iterator: iter,
beacon: beacon,
Expand Down
8 changes: 6 additions & 2 deletions pkg/cannon/deriver/beacon/eth/v1/proposer_duty.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ const (
)

type ProposerDutyDeriverConfig struct {
Enabled bool `yaml:"enabled" default:"true"`
Enabled bool `yaml:"enabled" default:"true"`
Iterator iterator.BackfillingCheckpointConfig `yaml:"iterator"`
}

type ProposerDutyDeriver struct {
Expand All @@ -44,7 +45,10 @@ type ProposerDutyDeriver struct {

func NewProposerDutyDeriver(log logrus.FieldLogger, config *ProposerDutyDeriverConfig, iter *iterator.BackfillingCheckpoint, beacon *ethereum.BeaconNode, clientMeta *xatu.ClientMeta) *ProposerDutyDeriver {
return &ProposerDutyDeriver{
log: log.WithField("module", "cannon/event/beacon/eth/v1/proposer_duty"),
log: log.WithFields(logrus.Fields{
"module": "cannon/event/beacon/eth/v1/proposer_duty",
"type": ProposerDutyDeriverName.String(),
}),
cfg: config,
iterator: iter,
beacon: beacon,
Expand Down
8 changes: 6 additions & 2 deletions pkg/cannon/deriver/beacon/eth/v2/attester_slashing.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ const (
)

type AttesterSlashingDeriverConfig struct {
Enabled bool `yaml:"enabled" default:"true"`
Enabled bool `yaml:"enabled" default:"true"`
Iterator iterator.BackfillingCheckpointConfig `yaml:"iterator"`
}

type AttesterSlashingDeriver struct {
Expand All @@ -42,7 +43,10 @@ type AttesterSlashingDeriver struct {

func NewAttesterSlashingDeriver(log logrus.FieldLogger, config *AttesterSlashingDeriverConfig, iter *iterator.BackfillingCheckpoint, beacon *ethereum.BeaconNode, clientMeta *xatu.ClientMeta) *AttesterSlashingDeriver {
return &AttesterSlashingDeriver{
log: log.WithField("module", "cannon/event/beacon/eth/v2/attester_slashing"),
log: log.WithFields(logrus.Fields{
"module": "cannon/event/beacon/eth/v2/attester_slashing",
"type": AttesterSlashingDeriverName.String(),
}),
cfg: config,
iterator: iter,
beacon: beacon,
Expand Down
8 changes: 6 additions & 2 deletions pkg/cannon/deriver/beacon/eth/v2/beacon_block.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ const (
)

type BeaconBlockDeriverConfig struct {
Enabled bool `yaml:"enabled" default:"true"`
Enabled bool `yaml:"enabled" default:"true"`
Iterator iterator.BackfillingCheckpointConfig `yaml:"iterator"`
}

type BeaconBlockDeriver struct {
Expand All @@ -46,7 +47,10 @@ type BeaconBlockDeriver struct {

func NewBeaconBlockDeriver(log logrus.FieldLogger, config *BeaconBlockDeriverConfig, iter *iterator.BackfillingCheckpoint, beacon *ethereum.BeaconNode, clientMeta *xatu.ClientMeta) *BeaconBlockDeriver {
return &BeaconBlockDeriver{
log: log.WithField("module", "cannon/event/beacon/eth/v2/beacon_block"),
log: log.WithFields(logrus.Fields{
"module": "cannon/event/beacon/eth/v2/beacon_block",
"type": BeaconBlockDeriverName.String(),
}),
cfg: config,
iterator: iter,
beacon: beacon,
Expand Down
8 changes: 6 additions & 2 deletions pkg/cannon/deriver/beacon/eth/v2/bls_to_execution_change.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ const (
)

type BLSToExecutionChangeDeriverConfig struct {
Enabled bool `yaml:"enabled" default:"true"`
Enabled bool `yaml:"enabled" default:"true"`
Iterator iterator.BackfillingCheckpointConfig `yaml:"iterator"`
}

type BLSToExecutionChangeDeriver struct {
Expand All @@ -44,7 +45,10 @@ type BLSToExecutionChangeDeriver struct {

func NewBLSToExecutionChangeDeriver(log logrus.FieldLogger, config *BLSToExecutionChangeDeriverConfig, iter *iterator.BackfillingCheckpoint, beacon *ethereum.BeaconNode, clientMeta *xatu.ClientMeta) *BLSToExecutionChangeDeriver {
return &BLSToExecutionChangeDeriver{
log: log.WithField("module", "cannon/event/beacon/eth/v2/bls_to_execution_change"),
log: log.WithFields(logrus.Fields{
"module": "cannon/event/beacon/eth/v2/bls_to_execution_change",
"type": BLSToExecutionChangeDeriverName.String(),
}),
cfg: config,
iterator: iter,
beacon: beacon,
Expand Down
8 changes: 6 additions & 2 deletions pkg/cannon/deriver/beacon/eth/v2/deposit.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ const (
)

type DepositDeriverConfig struct {
Enabled bool `yaml:"enabled" default:"true"`
Enabled bool `yaml:"enabled" default:"true"`
Iterator iterator.BackfillingCheckpointConfig `yaml:"iterator"`
}

type DepositDeriver struct {
Expand All @@ -42,7 +43,10 @@ type DepositDeriver struct {

func NewDepositDeriver(log logrus.FieldLogger, config *DepositDeriverConfig, iter *iterator.BackfillingCheckpoint, beacon *ethereum.BeaconNode, clientMeta *xatu.ClientMeta) *DepositDeriver {
return &DepositDeriver{
log: log.WithField("module", "cannon/event/beacon/eth/v2/deposit"),
log: log.WithFields(logrus.Fields{
"module": "cannon/event/beacon/eth/v2/deposit",
"type": DepositDeriverName.String(),
}),
cfg: config,
iterator: iter,
beacon: beacon,
Expand Down
8 changes: 6 additions & 2 deletions pkg/cannon/deriver/beacon/eth/v2/elaborated_attestation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ const (
)

type ElaboratedAttestationDeriverConfig struct {
Enabled bool `yaml:"enabled" default:"true"`
Enabled bool `yaml:"enabled" default:"true"`
Iterator iterator.BackfillingCheckpointConfig `yaml:"iterator"`
}

type ElaboratedAttestationDeriver struct {
Expand All @@ -43,7 +44,10 @@ type ElaboratedAttestationDeriver struct {

func NewElaboratedAttestationDeriver(log logrus.FieldLogger, config *ElaboratedAttestationDeriverConfig, iter *iterator.BackfillingCheckpoint, beacon *ethereum.BeaconNode, clientMeta *xatu.ClientMeta) *ElaboratedAttestationDeriver {
return &ElaboratedAttestationDeriver{
log: log.WithField("module", "cannon/event/beacon/eth/v2/elaborated_attestation"),
log: log.WithFields(logrus.Fields{
"module": "cannon/event/beacon/eth/v2/elaborated_attestation",
"type": ElaboratedAttestationDeriverName.String(),
}),
cfg: config,
iterator: iter,
beacon: beacon,
Expand Down
8 changes: 6 additions & 2 deletions pkg/cannon/deriver/beacon/eth/v2/execution_transaction.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ type ExecutionTransactionDeriver struct {
}

type ExecutionTransactionDeriverConfig struct {
Enabled bool `yaml:"enabled" default:"true"`
Enabled bool `yaml:"enabled" default:"true"`
Iterator iterator.BackfillingCheckpointConfig `yaml:"iterator"`
}

const (
Expand All @@ -47,7 +48,10 @@ const (

func NewExecutionTransactionDeriver(log logrus.FieldLogger, config *ExecutionTransactionDeriverConfig, iter *iterator.BackfillingCheckpoint, beacon *ethereum.BeaconNode, clientMeta *xatu.ClientMeta) *ExecutionTransactionDeriver {
return &ExecutionTransactionDeriver{
log: log.WithField("module", "cannon/event/beacon/eth/v2/execution_transaction"),
log: log.WithFields(logrus.Fields{
"module": "cannon/event/beacon/eth/v2/execution_transaction",
"type": ExecutionTransactionDeriverName.String(),
}),
cfg: config,
iterator: iter,
beacon: beacon,
Expand Down
8 changes: 6 additions & 2 deletions pkg/cannon/deriver/beacon/eth/v2/proposer_slashing.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ const (
)

type ProposerSlashingDeriverConfig struct {
Enabled bool `yaml:"enabled" default:"true"`
Enabled bool `yaml:"enabled" default:"true"`
Iterator iterator.BackfillingCheckpointConfig `yaml:"iterator"`
}

type ProposerSlashingDeriver struct {
Expand All @@ -42,7 +43,10 @@ type ProposerSlashingDeriver struct {

func NewProposerSlashingDeriver(log logrus.FieldLogger, config *ProposerSlashingDeriverConfig, iter *iterator.BackfillingCheckpoint, beacon *ethereum.BeaconNode, clientMeta *xatu.ClientMeta) *ProposerSlashingDeriver {
return &ProposerSlashingDeriver{
log: log.WithField("module", "cannon/event/beacon/eth/v2/proposer_slashing"),
log: log.WithFields(logrus.Fields{
"module": "cannon/event/beacon/eth/v2/proposer_slashing",
"type": ProposerSlashingDeriverName.String(),
}),
cfg: config,
iterator: iter,
beacon: beacon,
Expand Down
Loading
Loading