feat(L1 follower): adjust to recent CodecV7 and contract changes (#1120)

* port changes from #1013

* port changes from #1068

* go.mod tidy

* fix compile error

* fix goimports

* fix log

* address review comments

* upgrade golang.org/x/net to 0.23.0

* port changes from #1018

* fix tests and linter errors

* address review comments

* refactor rollup sync service / verifier to use CalldataBlobSource to retrieve data from L1

* add configuration and initialize blob clients

* fix unit tests

* remove unused code

* address review comments

* address more review comments

* implement first version of new da-codec and to handle multiple batches submitted in one transaction

* add CommitBatchDAV7 and handle multiple commit events submitted in a single transactions

* fix bug due to previous batch being empty when processing the first batch within a set of batches

* Allow using MPT

* update to latest da-codec

* add field to CommittedBatchMeta to store LastL1MessageQueueHash for CodecV7 batches

* adjust rollup verifier to support CodecV7 batches

* address review comments

* fix issues after merge

* go mod tidy

* fix unit tests

* update da-codec

* add test TestValidateBatchCodecV7

* go mod tidy

* do not log error on shutdown

* add sanity check for version to deserialization of committedBatchMetaV7

* port changes from #1073

* chore: auto version bump [bot]

* address review comments

* add more logs

* disable ENRUpdater if DA sync mode is enabled

* exit pipeline if context is cancelled

* correctly handle override by setting the head of the chain to the parent's height so that created blocks will always become part of canonical chain

* fix error with genesis event being nil

* chore: auto version bump [bot]

* chore: auto version bump [bot]

* adjust to renaming in CodecV7

* implement carrying forward of L1 MessageQueue index

* fix issue after upgrading from old storage to new format where batchIndex was 0 and all batches would be skipped

* add new RevertBatch event

* add commitBatches to be able to read calldata of CodecV7/EuclidV2 committed batches

* implement finding of L1 message queue height for initial batch in recovery mode

* add sanity checks for computed batches from events and batch hashes given via calldata from commit transaction

* update ScrollChain ABI

* chore: auto version bump [bot]

* remove initial batch form DAQueue

* go mod tidy

* fix underflow bug when l1DeploymentBlock==0

* fix bug with wrong parentBatchHash of first batch of batches submitted in a single tx

* update to latest da-codec

* address review comments

* address review comments

* fix bug when l1MessageV2StartIndex==0 serialized to [] (empty slice) which would always be decoded to non-existing instead of 0

* chore: auto version bump [bot]

* cache go dependencies in Dockerfile.mockccc

* chore: auto version bump [bot]

* add INFO log when reverting rollup events in L1 follower mode

* change LastProcessedMessageQueueIndex of finalize event to  totalL1MessagesPoppedOverall

* cleanup

* chore: auto version bump [bot]

---------

Co-authored-by: Ömer Faruk Irmak <[email protected]>
Co-authored-by: Thegaram <[email protected]>
Co-authored-by: jonastheis <[email protected]>

Author: 3 people

.dockerignore

@@ -3,3 +3,5 @@
 build/_workspace
 build/_bin
 tests/testdata
+
+tmp/

Dockerfile.mockccc

@@ -7,14 +7,18 @@ ARG SCROLL_LIB_PATH=/scroll/lib
 # Build Geth in a stock Go builder container
 FROM scrolltech/go-rust-builder:go-1.21-rust-nightly-2023-12-03 as builder
 
-ADD . /go-ethereum
-RUN cd /go-ethereum && env GO111MODULE=on go run build/ci.go install ./cmd/geth
+WORKDIR /go-ethereum
+COPY go.mod go.sum ./
+RUN go mod download -x
+
+ADD . ./
+RUN env GO111MODULE=on go run build/ci.go install ./cmd/geth
 
 # Pull Geth into a second stage deploy alpine container
 FROM ubuntu:20.04
 
-RUN apt-get -qq update \
-    && apt-get -qq install -y --no-install-recommends ca-certificates
+#RUN apt-get -qq update \
+#    && apt-get -qq install -y --no-install-recommends ca-certificates
 
 ENV CGO_LDFLAGS="-ldl"
 

core/rawdb/accessors_da_syncer.go

@@ -1,23 +1,33 @@
 package rawdb
 
 import (
+	"bytes"
 	"math/big"
 
 	"github.com/scroll-tech/go-ethereum/ethdb"
 	"github.com/scroll-tech/go-ethereum/log"
+	"github.com/scroll-tech/go-ethereum/rlp"
 )
 
-// WriteDASyncedL1BlockNumber writes the highest synced L1 block number to the database.
-func WriteDASyncedL1BlockNumber(db ethdb.KeyValueWriter, L1BlockNumber uint64) {
-	value := big.NewInt(0).SetUint64(L1BlockNumber).Bytes()
+type DAProcessedBatchMeta struct {
+	BatchIndex            uint64
+	L1BlockNumber         uint64
+	TotalL1MessagesPopped uint64
+}
 
+// WriteDAProcessedBatchMeta writes the batch metadata of the latest processed DA batch.
+func WriteDAProcessedBatchMeta(db ethdb.KeyValueWriter, daProcessedBatchMeta *DAProcessedBatchMeta) {
+	value, err := rlp.EncodeToBytes(daProcessedBatchMeta)
+	if err != nil {
+		log.Crit("failed to RLP encode committed batch metadata", "batch index", daProcessedBatchMeta.BatchIndex, "committed batch meta", daProcessedBatchMeta, "err", err)
+	}
 	if err := db.Put(daSyncedL1BlockNumberKey, value); err != nil {
-		log.Crit("Failed to update DA synced L1 block number", "err", err)
+		log.Crit("Failed to update DAProcessedBatchMeta", "err", err)
 	}
 }
 
-// ReadDASyncedL1BlockNumber retrieves the highest synced L1 block number.
-func ReadDASyncedL1BlockNumber(db ethdb.Reader) *uint64 {
+// ReadDAProcessedBatchMeta retrieves the batch metadata of the latest processed DA batch.
+func ReadDAProcessedBatchMeta(db ethdb.Reader) *DAProcessedBatchMeta {
 	data, err := db.Get(daSyncedL1BlockNumberKey)
 	if err != nil && isNotFoundErr(err) {
 		return nil
@@ -29,11 +39,25 @@ func ReadDASyncedL1BlockNumber(db ethdb.Reader) *uint64 {
 		return nil
 	}
 
-	number := new(big.Int).SetBytes(data)
-	if !number.IsUint64() {
-		log.Crit("Unexpected DA synced L1 block number in database", "number", number)
+	// Try decoding from the newest format for future proofness, then the older one for old data.
+	daProcessedBatchMeta := new(DAProcessedBatchMeta)
+	if err = rlp.Decode(bytes.NewReader(data), daProcessedBatchMeta); err == nil {
+		return daProcessedBatchMeta
 	}
 
-	value := number.Uint64()
-	return &value
+	// Before storing DAProcessedBatchMeta we used to store a single uint64 value for the L1 block number.
+	l1BlockNumber := new(big.Int).SetBytes(data)
+	if !l1BlockNumber.IsUint64() {
+		log.Crit("Unexpected DA synced L1 block number in database", "number", l1BlockNumber)
+	}
+
+	// We can simply set only the L1BlockNumber because carrying forward the totalL1MessagesPopped is not required before EuclidV2 (CodecV7)
+	// (the parentTotalL1MessagePopped is given via the parentBatchHeader).
+	// Nodes need to update to the new version to be able to continue syncing after EuclidV2 (CodecV7). Therefore,
+	// the only nodes that might read a uint64 value are nodes that were running L1 follower before the EuclidV2.
+	return &DAProcessedBatchMeta{
+		BatchIndex:            0,
+		L1BlockNumber:         l1BlockNumber.Uint64(),
+		TotalL1MessagesPopped: 0,
+	}
 }

core/rawdb/accessors_l1_message.go

@@ -383,9 +383,13 @@ func ReadFirstQueueIndexNotInL2Block(db ethdb.Reader, l2BlockHash common.Hash) *
 
 // WriteL1MessageV2StartIndex writes the start index of L1 messages that are from L1MessageQueueV2.
 func WriteL1MessageV2StartIndex(db ethdb.KeyValueWriter, queueIndex uint64) {
-	value := big.NewInt(0).SetUint64(queueIndex).Bytes()
+	// Write with binary.BigEndian.PutUint64 to ensure that 0 values are written as 8 bytes.
+	// big.NewInt(0).SetUint64(l1BlockNumber).Bytes() would write 0 as empty slice which leads to problems when reading
+	// the value as non-existent and 0 are not distinguishable.
+	var buf [8]byte
+	binary.BigEndian.PutUint64(buf[:], queueIndex)
 
-	if err := db.Put(l1MessageV2StartIndexKey, value); err != nil {
+	if err := db.Put(l1MessageV2StartIndexKey, buf[:]); err != nil {
 		log.Crit("Failed to update L1MessageV2 start index", "err", err)
 	}
 }
@@ -402,13 +406,11 @@ func ReadL1MessageV2StartIndex(db ethdb.Reader) *uint64 {
 	if len(data) == 0 {
 		return nil
 	}
-
-	number := new(big.Int).SetBytes(data)
-	if !number.IsUint64() {
-		log.Crit("Unexpected number for L1MessageV2 start index", "number", number)
+	if len(data) != 8 {
+		return nil
 	}
+	res := binary.BigEndian.Uint64(data)
 
-	res := number.Uint64()
 	return &res
 }
 

core/rawdb/accessors_rollup_event.go

@@ -25,7 +25,7 @@ type CommittedBatchMeta struct {
 	ChunkBlockRanges []*ChunkBlockRange
 
 	// introduced with CodecV7
-	LastL1MessageQueueHash common.Hash
+	PostL1MessageQueueHash common.Hash
 }
 
 type committedBatchMetaV0 struct {
@@ -170,7 +170,7 @@ func WriteCommittedBatchMeta(db ethdb.KeyValueWriter, batchIndex uint64, committ
 		committedBatchMetaToStore = &committedBatchMetaV7{
 			Version:                committedBatchMeta.Version,
 			ChunkBlockRanges:       committedBatchMeta.ChunkBlockRanges,
-			LastL1MessageQueueHash: committedBatchMeta.LastL1MessageQueueHash,
+			LastL1MessageQueueHash: committedBatchMeta.PostL1MessageQueueHash,
 		}
 	}
 
@@ -202,7 +202,7 @@ func ReadCommittedBatchMeta(db ethdb.Reader, batchIndex uint64) (*CommittedBatch
 		return &CommittedBatchMeta{
 			Version:                cbm7.Version,
 			ChunkBlockRanges:       cbm7.ChunkBlockRanges,
-			LastL1MessageQueueHash: cbm7.LastL1MessageQueueHash,
+			PostL1MessageQueueHash: cbm7.LastL1MessageQueueHash,
 		}, nil
 	}
 
@@ -214,7 +214,7 @@ func ReadCommittedBatchMeta(db ethdb.Reader, batchIndex uint64) (*CommittedBatch
 	return &CommittedBatchMeta{
 		Version:                cbm0.Version,
 		ChunkBlockRanges:       cbm0.ChunkBlockRanges,
-		LastL1MessageQueueHash: common.Hash{},
+		PostL1MessageQueueHash: common.Hash{},
 	}, nil
 }
 

core/rawdb/accessors_rollup_event_test.go

@@ -182,15 +182,15 @@ func TestWriteReadDeleteCommittedBatchMeta(t *testing.T) {
 			meta: &CommittedBatchMeta{
 				Version:                7,
 				ChunkBlockRanges:       []*ChunkBlockRange{{StartBlockNumber: 0, EndBlockNumber: 10}},
-				LastL1MessageQueueHash: common.Hash{1, 2, 3, 4, 5, 6, 7},
+				PostL1MessageQueueHash: common.Hash{1, 2, 3, 4, 5, 6, 7},
 			},
 		},
 		{
 			batchIndex: 255,
 			meta: &CommittedBatchMeta{
 				Version:                255,
 				ChunkBlockRanges:       []*ChunkBlockRange{{StartBlockNumber: 0, EndBlockNumber: 10}, {StartBlockNumber: 11, EndBlockNumber: 20}},
-				LastL1MessageQueueHash: common.Hash{255},
+				PostL1MessageQueueHash: common.Hash{255},
 			},
 		},
 	}
@@ -237,7 +237,7 @@ func TestOverwriteCommittedBatchMeta(t *testing.T) {
 	newMeta := &CommittedBatchMeta{
 		Version:                255,
 		ChunkBlockRanges:       []*ChunkBlockRange{{StartBlockNumber: 0, EndBlockNumber: 20}, {StartBlockNumber: 21, EndBlockNumber: 30}},
-		LastL1MessageQueueHash: common.Hash{255},
+		PostL1MessageQueueHash: common.Hash{255},
 	}
 
 	// write initial meta
@@ -282,5 +282,5 @@ func compareCommittedBatchMeta(a, b *CommittedBatchMeta) bool {
 		}
 	}
 
-	return a.LastL1MessageQueueHash == b.LastL1MessageQueueHash
+	return a.PostL1MessageQueueHash == b.PostL1MessageQueueHash
 }

go.mod

@@ -51,7 +51,7 @@ require (
 	github.com/prometheus/tsdb v0.7.1
 	github.com/rjeczalik/notify v0.9.1
 	github.com/rs/cors v1.7.0
-	github.com/scroll-tech/da-codec v0.1.3-0.20250210041951-d028c537b995
+	github.com/scroll-tech/da-codec v0.1.3-0.20250226072559-f8a8d3898f54
 	github.com/scroll-tech/zktrie v0.8.4
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/sourcegraph/conc v0.3.0

go.sum

@@ -396,8 +396,8 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/scroll-tech/da-codec v0.1.3-0.20250210041951-d028c537b995 h1:Zo1p42CUS9pADSKoDD0ZoDxf4dQ3gttqWZlV+RSeImk=
-github.com/scroll-tech/da-codec v0.1.3-0.20250210041951-d028c537b995/go.mod h1:UZhhjzqYsyEhcvY0Y+SP+oMdeOUqFn/UXpbAYuPGzg0=
+github.com/scroll-tech/da-codec v0.1.3-0.20250226072559-f8a8d3898f54 h1:qVpsVu1J91opTn6HYeuzWcBRVhQmPR8g05i+PlOjlI4=
+github.com/scroll-tech/da-codec v0.1.3-0.20250226072559-f8a8d3898f54/go.mod h1:xECEHZLVzbdUn+tNbRJhRIjLGTOTmnFQuTgUTeVLX58=
 github.com/scroll-tech/zktrie v0.8.4 h1:UagmnZ4Z3ITCk+aUq9NQZJNAwnWl4gSxsLb2Nl7IgRE=
 github.com/scroll-tech/zktrie v0.8.4/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk=
 github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=

params/version.go

@@ -24,7 +24,7 @@ import (
 const (
 	VersionMajor = 5         // Major version component of the current release
 	VersionMinor = 8         // Minor version component of the current release
-	VersionPatch = 17        // Patch version component of the current release
+	VersionPatch = 18        // Patch version component of the current release
 	VersionMeta  = "mainnet" // Version metadata to append to the version string
 )