diff --git a/block/internal/common/consts.go b/block/internal/common/consts.go index 255a5da9a..bb584e7ae 100644 --- a/block/internal/common/consts.go +++ b/block/internal/common/consts.go @@ -1,3 +1,3 @@ package common -const DefaultMaxBlobSize = 2 * 1024 * 1024 // 2MB fallback blob size limit +const DefaultMaxBlobSize = 8 * 1024 * 1024 // 8MB fallback blob size limit (Celestia's current limit) diff --git a/block/internal/submitting/batching_strategy.go b/block/internal/submitting/batching_strategy.go new file mode 100644 index 000000000..c461cd638 --- /dev/null +++ b/block/internal/submitting/batching_strategy.go @@ -0,0 +1,285 @@ +package submitting + +import ( + "fmt" + "time" + + "github.com/evstack/ev-node/block/internal/common" + "github.com/evstack/ev-node/pkg/config" +) + +// BatchingStrategy defines the interface for different batching strategies +type BatchingStrategy interface { + // ShouldSubmit determines if a batch should be submitted based on the strategy + // Returns true if submission should happen now + ShouldSubmit(pendingCount uint64, totalSize int, maxBlobSize int, timeSinceLastSubmit time.Duration) bool + + // Name returns the name of the strategy + Name() string +} + +// ImmediateStrategy submits as soon as any items are available +type ImmediateStrategy struct{} + +func (s *ImmediateStrategy) ShouldSubmit(pendingCount uint64, totalSize int, maxBlobSize int, timeSinceLastSubmit time.Duration) bool { + return pendingCount > 0 +} + +func (s *ImmediateStrategy) Name() string { + return "immediate" +} + +// SizeBasedStrategy waits until the batch reaches a certain size threshold +type SizeBasedStrategy struct { + sizeThreshold float64 // fraction of max blob size (0.0 to 1.0) + minItems uint64 +} + +func NewSizeBasedStrategy(sizeThreshold float64, minItems uint64) *SizeBasedStrategy { + if sizeThreshold <= 0 || sizeThreshold > 1.0 { + sizeThreshold = 0.8 // default to 80% + } + if minItems == 0 { + minItems = 1 + } + return &SizeBasedStrategy{ + sizeThreshold: sizeThreshold, + minItems: minItems, + } +} + +func (s *SizeBasedStrategy) ShouldSubmit(pendingCount uint64, totalSize int, maxBlobSize int, timeSinceLastSubmit time.Duration) bool { + if pendingCount < s.minItems { + return false + } + + threshold := int(float64(maxBlobSize) * s.sizeThreshold) + return totalSize >= threshold +} + +func (s *SizeBasedStrategy) Name() string { + return "size" +} + +// TimeBasedStrategy submits after a certain time interval +type TimeBasedStrategy struct { + maxDelay time.Duration + minItems uint64 +} + +func NewTimeBasedStrategy(maxDelay time.Duration, minItems uint64) *TimeBasedStrategy { + if maxDelay == 0 { + maxDelay = 6 * time.Second // default to DA block time + } + if minItems == 0 { + minItems = 1 + } + return &TimeBasedStrategy{ + maxDelay: maxDelay, + minItems: minItems, + } +} + +func (s *TimeBasedStrategy) ShouldSubmit(pendingCount uint64, totalSize int, maxBlobSize int, timeSinceLastSubmit time.Duration) bool { + if pendingCount < s.minItems { + return false + } + + return timeSinceLastSubmit >= s.maxDelay +} + +func (s *TimeBasedStrategy) Name() string { + return "time" +} + +// AdaptiveStrategy balances between size and time constraints +// It submits when either: +// - The batch reaches the size threshold, OR +// - The max delay is reached and we have at least min items +type AdaptiveStrategy struct { + sizeThreshold float64 + maxDelay time.Duration + minItems uint64 +} + +func NewAdaptiveStrategy(sizeThreshold float64, maxDelay time.Duration, minItems uint64) *AdaptiveStrategy { + if sizeThreshold <= 0 || sizeThreshold > 1.0 { + sizeThreshold = 0.8 // default to 80% + } + if maxDelay == 0 { + maxDelay = 6 * time.Second // default to DA block time + } + if minItems == 0 { + minItems = 1 + } + return &AdaptiveStrategy{ + sizeThreshold: sizeThreshold, + maxDelay: maxDelay, + minItems: minItems, + } +} + +func (s *AdaptiveStrategy) ShouldSubmit(pendingCount uint64, totalSize int, maxBlobSize int, timeSinceLastSubmit time.Duration) bool { + if pendingCount < s.minItems { + return false + } + + // Submit if we've reached the size threshold + threshold := int(float64(maxBlobSize) * s.sizeThreshold) + if totalSize >= threshold { + return true + } + + // Submit if max delay has been reached + if timeSinceLastSubmit >= s.maxDelay { + return true + } + + return false +} + +func (s *AdaptiveStrategy) Name() string { + return "adaptive" +} + +// BatchingStrategyFactory creates a batching strategy based on configuration +func BatchingStrategyFactory(cfg config.DAConfig) (BatchingStrategy, error) { + switch cfg.BatchingStrategy { + case "immediate": + return &ImmediateStrategy{}, nil + case "size": + return NewSizeBasedStrategy(cfg.BatchSizeThreshold, cfg.BatchMinItems), nil + case "time": + return NewTimeBasedStrategy(cfg.BatchMaxDelay.Duration, cfg.BatchMinItems), nil + case "adaptive": + return NewAdaptiveStrategy(cfg.BatchSizeThreshold, cfg.BatchMaxDelay.Duration, cfg.BatchMinItems), nil + default: + return nil, fmt.Errorf("unknown batching strategy: %s", cfg.BatchingStrategy) + } +} + +// estimateBatchSize estimates the total size of pending items +// This is a helper function that can be used by the submitter +func estimateBatchSize(marshaled [][]byte) int { + totalSize := 0 + for _, data := range marshaled { + totalSize += len(data) + } + return totalSize +} + +// optimizeBatchSize returns the optimal number of items to include in a batch +// to maximize blob utilization while staying under the size limit +func optimizeBatchSize(marshaled [][]byte, maxBlobSize int, targetUtilization float64) int { + if targetUtilization <= 0 || targetUtilization > 1.0 { + targetUtilization = 0.9 // default to 90% utilization + } + + targetSize := int(float64(maxBlobSize) * targetUtilization) + totalSize := 0 + count := 0 + + for i, data := range marshaled { + itemSize := len(data) + + // If adding this item would exceed max blob size, stop + if totalSize+itemSize > maxBlobSize { + break + } + + totalSize += itemSize + count = i + 1 + + // If we've reached our target utilization, we can stop + // This helps create more predictably-sized batches + if totalSize >= targetSize { + break + } + } + + return count +} + +// BatchMetrics provides information about batch efficiency +type BatchMetrics struct { + ItemCount int + TotalBytes int + MaxBlobBytes int + Utilization float64 // percentage of max blob size used + EstimatedCost float64 // estimated cost relative to single full blob +} + +// calculateBatchMetrics computes metrics for a batch +func calculateBatchMetrics(itemCount int, totalBytes int, maxBlobBytes int) BatchMetrics { + utilization := 0.0 + if maxBlobBytes > 0 { + utilization = float64(totalBytes) / float64(maxBlobBytes) + } + + // Rough cost estimate: each blob submission has a fixed cost + // Higher utilization = better cost efficiency + estimatedCost := 1.0 + if utilization > 0 { + // If we're only using 50% of the blob, we're paying 2x per byte effectively + estimatedCost = 1.0 / utilization + } + + return BatchMetrics{ + ItemCount: itemCount, + TotalBytes: totalBytes, + MaxBlobBytes: maxBlobBytes, + Utilization: utilization, + EstimatedCost: estimatedCost, + } +} + +// ShouldWaitForMoreItems determines if we should wait for more items +// to improve batch efficiency +func ShouldWaitForMoreItems( + currentCount uint64, + currentSize int, + maxBlobSize int, + minUtilization float64, + hasMoreExpected bool, +) bool { + // Don't wait if we're already at or near capacity + if currentSize >= int(float64(maxBlobSize)*0.95) { + return false + } + + // Don't wait if we don't expect more items soon + if !hasMoreExpected { + return false + } + + // Wait if current utilization is below minimum threshold + // Use epsilon for floating point comparison + const epsilon = 0.001 + currentUtilization := float64(currentSize) / float64(maxBlobSize) + if currentUtilization < minUtilization-epsilon { + return true + } + + return false +} + +// BatchingConfig holds configuration for batch optimization +type BatchingConfig struct { + MaxBlobSize int + Strategy BatchingStrategy + TargetUtilization float64 +} + +// NewBatchingConfig creates a new batching configuration +func NewBatchingConfig(cfg config.DAConfig) (*BatchingConfig, error) { + strategy, err := BatchingStrategyFactory(cfg) + if err != nil { + return nil, err + } + + return &BatchingConfig{ + MaxBlobSize: common.DefaultMaxBlobSize, + Strategy: strategy, + TargetUtilization: cfg.BatchSizeThreshold, + }, nil +} diff --git a/block/internal/submitting/batching_strategy_test.go b/block/internal/submitting/batching_strategy_test.go new file mode 100644 index 000000000..71fcce60b --- /dev/null +++ b/block/internal/submitting/batching_strategy_test.go @@ -0,0 +1,593 @@ +package submitting + +import ( + "testing" + "time" + + "github.com/evstack/ev-node/block/internal/common" + "github.com/evstack/ev-node/pkg/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestImmediateStrategy(t *testing.T) { + strategy := &ImmediateStrategy{} + + tests := []struct { + name string + pendingCount uint64 + totalSize int + expected bool + }{ + { + name: "no pending items", + pendingCount: 0, + totalSize: 0, + expected: false, + }, + { + name: "one pending item", + pendingCount: 1, + totalSize: 1000, + expected: true, + }, + { + name: "multiple pending items", + pendingCount: 10, + totalSize: 10000, + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := strategy.ShouldSubmit(tt.pendingCount, tt.totalSize, common.DefaultMaxBlobSize, 0) + assert.Equal(t, tt.expected, result) + }) + } + + assert.Equal(t, "immediate", strategy.Name()) +} + +func TestSizeBasedStrategy(t *testing.T) { + maxBlobSize := 8 * 1024 * 1024 // 8MB + + tests := []struct { + name string + sizeThreshold float64 + minItems uint64 + pendingCount uint64 + totalSize int + expectedSubmit bool + }{ + { + name: "below threshold and min items", + sizeThreshold: 0.8, + minItems: 2, + pendingCount: 1, + totalSize: 1 * 1024 * 1024, // 1MB + expectedSubmit: false, + }, + { + name: "below threshold but has min items", + sizeThreshold: 0.8, + minItems: 1, + pendingCount: 5, + totalSize: 4 * 1024 * 1024, // 4MB (50% of 8MB) + expectedSubmit: false, + }, + { + name: "at threshold with min items", + sizeThreshold: 0.8, + minItems: 1, + pendingCount: 10, + totalSize: int(float64(maxBlobSize) * 0.8), // 80% of max + expectedSubmit: true, + }, + { + name: "above threshold", + sizeThreshold: 0.8, + minItems: 1, + pendingCount: 20, + totalSize: 7 * 1024 * 1024, // 7MB (87.5% of 8MB) + expectedSubmit: true, + }, + { + name: "full blob", + sizeThreshold: 0.8, + minItems: 1, + pendingCount: 100, + totalSize: maxBlobSize, + expectedSubmit: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + strategy := NewSizeBasedStrategy(tt.sizeThreshold, tt.minItems) + result := strategy.ShouldSubmit(tt.pendingCount, tt.totalSize, maxBlobSize, 0) + assert.Equal(t, tt.expectedSubmit, result) + }) + } + + // Test invalid threshold defaults to 0.8 + strategy := NewSizeBasedStrategy(1.5, 1) + assert.Equal(t, 0.8, strategy.sizeThreshold) + + strategy = NewSizeBasedStrategy(0, 1) + assert.Equal(t, 0.8, strategy.sizeThreshold) + + assert.Equal(t, "size", strategy.Name()) +} + +func TestTimeBasedStrategy(t *testing.T) { + maxDelay := 6 * time.Second + maxBlobSize := 8 * 1024 * 1024 + + tests := []struct { + name string + minItems uint64 + pendingCount uint64 + totalSize int + timeSinceLastSubmit time.Duration + expectedSubmit bool + }{ + { + name: "below min items", + minItems: 2, + pendingCount: 1, + totalSize: 1 * 1024 * 1024, + timeSinceLastSubmit: 10 * time.Second, + expectedSubmit: false, + }, + { + name: "before max delay", + minItems: 1, + pendingCount: 5, + totalSize: 4 * 1024 * 1024, + timeSinceLastSubmit: 3 * time.Second, + expectedSubmit: false, + }, + { + name: "at max delay", + minItems: 1, + pendingCount: 3, + totalSize: 2 * 1024 * 1024, + timeSinceLastSubmit: 6 * time.Second, + expectedSubmit: true, + }, + { + name: "after max delay", + minItems: 1, + pendingCount: 2, + totalSize: 1 * 1024 * 1024, + timeSinceLastSubmit: 10 * time.Second, + expectedSubmit: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + strategy := NewTimeBasedStrategy(maxDelay, tt.minItems) + result := strategy.ShouldSubmit(tt.pendingCount, tt.totalSize, maxBlobSize, tt.timeSinceLastSubmit) + assert.Equal(t, tt.expectedSubmit, result) + }) + } + + assert.Equal(t, "time", NewTimeBasedStrategy(maxDelay, 1).Name()) +} + +func TestAdaptiveStrategy(t *testing.T) { + maxBlobSize := 8 * 1024 * 1024 // 8MB + sizeThreshold := 0.8 + maxDelay := 6 * time.Second + + tests := []struct { + name string + minItems uint64 + pendingCount uint64 + totalSize int + timeSinceLastSubmit time.Duration + expectedSubmit bool + reason string + }{ + { + name: "below min items", + minItems: 3, + pendingCount: 2, + totalSize: 7 * 1024 * 1024, + timeSinceLastSubmit: 10 * time.Second, + expectedSubmit: false, + reason: "not enough items", + }, + { + name: "size threshold reached", + minItems: 1, + pendingCount: 10, + totalSize: int(float64(maxBlobSize) * 0.85), // 85% + timeSinceLastSubmit: 1 * time.Second, + expectedSubmit: true, + reason: "size threshold met", + }, + { + name: "time threshold reached", + minItems: 1, + pendingCount: 2, + totalSize: 1 * 1024 * 1024, // Only 12.5% + timeSinceLastSubmit: 7 * time.Second, + expectedSubmit: true, + reason: "time threshold met", + }, + { + name: "neither threshold reached", + minItems: 1, + pendingCount: 5, + totalSize: 4 * 1024 * 1024, // 50% + timeSinceLastSubmit: 3 * time.Second, + expectedSubmit: false, + reason: "waiting for threshold", + }, + { + name: "both thresholds reached", + minItems: 1, + pendingCount: 20, + totalSize: 7 * 1024 * 1024, // 87.5% + timeSinceLastSubmit: 10 * time.Second, + expectedSubmit: true, + reason: "both thresholds met", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + strategy := NewAdaptiveStrategy(sizeThreshold, maxDelay, tt.minItems) + result := strategy.ShouldSubmit(tt.pendingCount, tt.totalSize, maxBlobSize, tt.timeSinceLastSubmit) + assert.Equal(t, tt.expectedSubmit, result, "reason: %s", tt.reason) + }) + } + + // Test defaults + strategy := NewAdaptiveStrategy(0, 0, 0) + assert.Equal(t, 0.8, strategy.sizeThreshold) + assert.Equal(t, 6*time.Second, strategy.maxDelay) + assert.Equal(t, uint64(1), strategy.minItems) + + assert.Equal(t, "adaptive", strategy.Name()) +} + +func TestBatchingStrategyFactory(t *testing.T) { + tests := []struct { + name string + strategyName string + expectedType string + expectError bool + }{ + { + name: "immediate strategy", + strategyName: "immediate", + expectedType: "immediate", + expectError: false, + }, + { + name: "size strategy", + strategyName: "size", + expectedType: "size", + expectError: false, + }, + { + name: "time strategy", + strategyName: "time", + expectedType: "time", + expectError: false, + }, + { + name: "adaptive strategy", + strategyName: "adaptive", + expectedType: "adaptive", + expectError: false, + }, + { + name: "unknown strategy", + strategyName: "unknown", + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := config.DAConfig{ + BatchingStrategy: tt.strategyName, + BatchSizeThreshold: 0.8, + BatchMaxDelay: config.DurationWrapper{Duration: 6 * time.Second}, + BatchMinItems: 1, + } + + strategy, err := BatchingStrategyFactory(cfg) + + if tt.expectError { + assert.Error(t, err) + assert.Nil(t, strategy) + } else { + require.NoError(t, err) + require.NotNil(t, strategy) + assert.Equal(t, tt.expectedType, strategy.Name()) + } + }) + } +} + +func TestEstimateBatchSize(t *testing.T) { + tests := []struct { + name string + marshaled [][]byte + expectedSize int + }{ + { + name: "empty batch", + marshaled: [][]byte{}, + expectedSize: 0, + }, + { + name: "single item", + marshaled: [][]byte{ + make([]byte, 1024), + }, + expectedSize: 1024, + }, + { + name: "multiple items", + marshaled: [][]byte{ + make([]byte, 1024), + make([]byte, 2048), + make([]byte, 512), + }, + expectedSize: 3584, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + size := estimateBatchSize(tt.marshaled) + assert.Equal(t, tt.expectedSize, size) + }) + } +} + +func TestOptimizeBatchSize(t *testing.T) { + maxBlobSize := 8 * 1024 * 1024 // 8MB + + tests := []struct { + name string + itemSizes []int + targetUtilization float64 + expectedCount int + expectedTotalSize int + }{ + { + name: "empty batch", + itemSizes: []int{}, + targetUtilization: 0.9, + expectedCount: 0, + expectedTotalSize: 0, + }, + { + name: "single small item", + itemSizes: []int{1024}, + targetUtilization: 0.9, + expectedCount: 1, + expectedTotalSize: 1024, + }, + { + name: "reach target utilization", + itemSizes: []int{1024 * 1024, 2 * 1024 * 1024, 3 * 1024 * 1024, 1024 * 1024}, + targetUtilization: 0.8, + expectedCount: 4, // 1+2+3+1 = 7MB (87.5% of 8MB, exceeds 80% target so stops) + expectedTotalSize: 7 * 1024 * 1024, + }, + { + name: "stop at max blob size", + itemSizes: []int{7 * 1024 * 1024, 2 * 1024 * 1024}, + targetUtilization: 0.9, + expectedCount: 1, // Second item would exceed max + expectedTotalSize: 7 * 1024 * 1024, + }, + { + name: "all items fit below target", + itemSizes: []int{1024 * 1024, 1024 * 1024, 1024 * 1024}, + targetUtilization: 0.9, + expectedCount: 3, + expectedTotalSize: 3 * 1024 * 1024, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create marshaled data + marshaled := make([][]byte, len(tt.itemSizes)) + for i, size := range tt.itemSizes { + marshaled[i] = make([]byte, size) + } + + count := optimizeBatchSize(marshaled, maxBlobSize, tt.targetUtilization) + assert.Equal(t, tt.expectedCount, count) + + if count > 0 { + totalSize := estimateBatchSize(marshaled[:count]) + assert.Equal(t, tt.expectedTotalSize, totalSize) + } + }) + } +} + +func TestCalculateBatchMetrics(t *testing.T) { + maxBlobSize := 8 * 1024 * 1024 + + tests := []struct { + name string + itemCount int + totalBytes int + expectedUtil float64 + expectedCostRange [2]float64 // min, max + }{ + { + name: "empty batch", + itemCount: 0, + totalBytes: 0, + expectedUtil: 0.0, + expectedCostRange: [2]float64{0, 999999}, // cost is undefined for empty + }, + { + name: "half full", + itemCount: 10, + totalBytes: 4 * 1024 * 1024, + expectedUtil: 0.5, + expectedCostRange: [2]float64{2.0, 2.0}, // 1/0.5 = 2.0x cost + }, + { + name: "80% full", + itemCount: 20, + totalBytes: int(float64(maxBlobSize) * 0.8), + expectedUtil: 0.8, + expectedCostRange: [2]float64{1.25, 1.25}, // 1/0.8 = 1.25x cost + }, + { + name: "nearly full", + itemCount: 50, + totalBytes: int(float64(maxBlobSize) * 0.95), + expectedUtil: 0.95, + expectedCostRange: [2]float64{1.05, 1.06}, // ~1.05x cost + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + metrics := calculateBatchMetrics(tt.itemCount, tt.totalBytes, maxBlobSize) + + assert.Equal(t, tt.itemCount, metrics.ItemCount) + assert.Equal(t, tt.totalBytes, metrics.TotalBytes) + assert.Equal(t, maxBlobSize, metrics.MaxBlobBytes) + assert.InDelta(t, tt.expectedUtil, metrics.Utilization, 0.01) + + if tt.totalBytes > 0 { + assert.InEpsilon(t, (tt.expectedCostRange[0]+tt.expectedCostRange[1])/2, + metrics.EstimatedCost, 0.01, "cost should be within range") + } + }) + } +} + +func TestShouldWaitForMoreItems(t *testing.T) { + maxBlobSize := 8 * 1024 * 1024 + + tests := []struct { + name string + currentCount uint64 + currentSize int + minUtilization float64 + hasMoreExpected bool + expectedWait bool + }{ + { + name: "near capacity", + currentCount: 50, + currentSize: int(float64(maxBlobSize) * 0.96), + minUtilization: 0.8, + hasMoreExpected: true, + expectedWait: false, + }, + { + name: "below threshold but no more expected", + currentCount: 10, + currentSize: 4 * 1024 * 1024, + minUtilization: 0.8, + hasMoreExpected: false, + expectedWait: false, + }, + { + name: "below threshold with more expected", + currentCount: 10, + currentSize: 4 * 1024 * 1024, // 50% + minUtilization: 0.8, + hasMoreExpected: true, + expectedWait: true, + }, + { + name: "at threshold", + currentCount: 20, + currentSize: int(float64(maxBlobSize) * 0.8), + minUtilization: 0.8, + hasMoreExpected: true, + expectedWait: false, // At threshold, no need to wait + }, + { + name: "above threshold", + currentCount: 30, + currentSize: int(float64(maxBlobSize) * 0.85), + minUtilization: 0.8, + hasMoreExpected: true, + expectedWait: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := ShouldWaitForMoreItems( + tt.currentCount, + tt.currentSize, + maxBlobSize, + tt.minUtilization, + tt.hasMoreExpected, + ) + assert.Equal(t, tt.expectedWait, result) + }) + } +} + +func TestNewBatchingConfig(t *testing.T) { + cfg := config.DAConfig{ + BatchingStrategy: "adaptive", + BatchSizeThreshold: 0.85, + BatchMaxDelay: config.DurationWrapper{Duration: 12 * time.Second}, + BatchMinItems: 5, + } + + batchConfig, err := NewBatchingConfig(cfg) + require.NoError(t, err) + require.NotNil(t, batchConfig) + + assert.Equal(t, common.DefaultMaxBlobSize, batchConfig.MaxBlobSize) + assert.Equal(t, "adaptive", batchConfig.Strategy.Name()) + assert.Equal(t, 0.85, batchConfig.TargetUtilization) +} + +func TestBatchingStrategiesComparison(t *testing.T) { + // This test demonstrates how different strategies behave with the same input + maxBlobSize := 8 * 1024 * 1024 + pendingCount := uint64(10) + totalSize := 4 * 1024 * 1024 // 50% full + timeSinceLastSubmit := 3 * time.Second + + immediate := &ImmediateStrategy{} + size := NewSizeBasedStrategy(0.8, 1) + timeBased := NewTimeBasedStrategy(6*time.Second, 1) + adaptive := NewAdaptiveStrategy(0.8, 6*time.Second, 1) + + // Immediate should always submit if there are items + assert.True(t, immediate.ShouldSubmit(pendingCount, totalSize, maxBlobSize, timeSinceLastSubmit)) + + // Size-based should not submit at 50% when threshold is 80% + assert.False(t, size.ShouldSubmit(pendingCount, totalSize, maxBlobSize, timeSinceLastSubmit)) + + // Time-based should not submit at 3s when max delay is 6s + assert.False(t, timeBased.ShouldSubmit(pendingCount, totalSize, maxBlobSize, timeSinceLastSubmit)) + + // Adaptive should not submit (neither threshold met) + assert.False(t, adaptive.ShouldSubmit(pendingCount, totalSize, maxBlobSize, timeSinceLastSubmit)) + + // Now test with time threshold exceeded + timeSinceLastSubmit = 7 * time.Second + assert.True(t, immediate.ShouldSubmit(pendingCount, totalSize, maxBlobSize, timeSinceLastSubmit)) + assert.False(t, size.ShouldSubmit(pendingCount, totalSize, maxBlobSize, timeSinceLastSubmit)) + assert.True(t, timeBased.ShouldSubmit(pendingCount, totalSize, maxBlobSize, timeSinceLastSubmit)) + assert.True(t, adaptive.ShouldSubmit(pendingCount, totalSize, maxBlobSize, timeSinceLastSubmit)) +} diff --git a/block/internal/submitting/submitter.go b/block/internal/submitting/submitter.go index 6aeb7a2d5..83c65aef4 100644 --- a/block/internal/submitting/submitter.go +++ b/block/internal/submitting/submitter.go @@ -55,6 +55,11 @@ type Submitter struct { headerSubmissionMtx sync.Mutex dataSubmissionMtx sync.Mutex + // Batching strategy state + lastHeaderSubmit time.Time + lastDataSubmit time.Time + batchingStrategy BatchingStrategy + // Channels for coordination errorCh chan<- error // Channel to report critical execution client failures @@ -81,6 +86,19 @@ func NewSubmitter( logger zerolog.Logger, errorCh chan<- error, ) *Submitter { + submitterLogger := logger.With().Str("component", "submitter").Logger() + + // Initialize batching strategy + strategy, err := BatchingStrategyFactory(config.DA) + if err != nil { + submitterLogger.Warn().Err(err).Msg("failed to create batching strategy, using time-based default") + strategy = NewTimeBasedStrategy(config.DA.BlockTime.Duration, 1) + } + + submitterLogger.Info(). + Str("batching_strategy", strategy.Name()). + Msg("initialized DA submission batching strategy") + return &Submitter{ store: store, exec: exec, @@ -92,8 +110,11 @@ func NewSubmitter( sequencer: sequencer, signer: signer, daIncludedHeight: &atomic.Uint64{}, + lastHeaderSubmit: time.Now(), + lastDataSubmit: time.Now(), + batchingStrategy: strategy, errorCh: errorCh, - logger: logger.With().Str("component", "submitter").Logger(), + logger: submitterLogger, } } @@ -140,7 +161,12 @@ func (s *Submitter) daSubmissionLoop() { s.logger.Info().Msg("starting DA submission loop") defer s.logger.Info().Msg("DA submission loop stopped") - ticker := time.NewTicker(s.config.DA.BlockTime.Duration) + // Use a shorter ticker interval to check batching strategy more frequently + checkInterval := s.config.DA.BlockTime.Duration / 4 + if checkInterval < 100*time.Millisecond { + checkInterval = 100 * time.Millisecond + } + ticker := time.NewTicker(checkInterval) defer ticker.Stop() for { @@ -148,49 +174,119 @@ func (s *Submitter) daSubmissionLoop() { case <-s.ctx.Done(): return case <-ticker.C: - // Submit headers - if headersNb := s.cache.NumPendingHeaders(); headersNb != 0 { - s.logger.Debug().Time("t", time.Now()).Uint64("headers", headersNb).Msg("Submitting headers") + // Check if we should submit headers based on batching strategy + headersNb := s.cache.NumPendingHeaders() + if headersNb > 0 { + timeSinceLastSubmit := time.Since(s.lastHeaderSubmit) + + // For strategy decision, we need to estimate the size + // We'll fetch headers to check, but only submit if strategy approves if s.headerSubmissionMtx.TryLock() { - s.logger.Debug().Time("t", time.Now()).Uint64("headers", headersNb).Msg("Header submission in progress") go func() { - defer func() { - s.logger.Debug().Time("t", time.Now()).Uint64("headers", headersNb).Msg("Header submission completed") - s.headerSubmissionMtx.Unlock() - }() - if err := s.daSubmitter.SubmitHeaders(s.ctx, s.cache, s.signer); err != nil { - // Check for unrecoverable errors that indicate a critical issue - if errors.Is(err, common.ErrOversizedItem) { - s.logger.Error().Err(err). - Msg("CRITICAL: Header exceeds DA blob size limit - halting to prevent live lock") - s.sendCriticalError(fmt.Errorf("unrecoverable DA submission error: %w", err)) - return + defer s.headerSubmissionMtx.Unlock() + + // Get pending headers to estimate size + headers, err := s.cache.GetPendingHeaders(s.ctx) + if err != nil { + s.logger.Error().Err(err).Msg("failed to get pending headers for batching decision") + return + } + + // Estimate total size + totalSize := 0 + for _, h := range headers { + data, err := h.MarshalBinary() + if err == nil { + totalSize += len(data) + } + } + + shouldSubmit := s.batchingStrategy.ShouldSubmit( + uint64(len(headers)), + totalSize, + common.DefaultMaxBlobSize, + timeSinceLastSubmit, + ) + + if shouldSubmit { + s.logger.Debug(). + Time("t", time.Now()). + Uint64("headers", headersNb). + Int("total_size_kb", totalSize/1024). + Dur("time_since_last", timeSinceLastSubmit). + Str("strategy", s.batchingStrategy.Name()). + Msg("batching strategy triggered header submission") + + if err := s.daSubmitter.SubmitHeaders(s.ctx, s.cache, s.signer); err != nil { + // Check for unrecoverable errors that indicate a critical issue + if errors.Is(err, common.ErrOversizedItem) { + s.logger.Error().Err(err). + Msg("CRITICAL: Header exceeds DA blob size limit - halting to prevent live lock") + s.sendCriticalError(fmt.Errorf("unrecoverable DA submission error: %w", err)) + return + } + s.logger.Error().Err(err).Msg("failed to submit headers") + } else { + s.lastHeaderSubmit = time.Now() } - s.logger.Error().Err(err).Msg("failed to submit headers") } }() } } - // Submit data - if dataNb := s.cache.NumPendingData(); dataNb != 0 { - s.logger.Debug().Time("t", time.Now()).Uint64("data", dataNb).Msg("Submitting data") + // Check if we should submit data based on batching strategy + dataNb := s.cache.NumPendingData() + if dataNb > 0 { + timeSinceLastSubmit := time.Since(s.lastDataSubmit) + if s.dataSubmissionMtx.TryLock() { - s.logger.Debug().Time("t", time.Now()).Uint64("data", dataNb).Msg("Data submission in progress") go func() { - defer func() { - s.logger.Debug().Time("t", time.Now()).Uint64("data", dataNb).Msg("Data submission completed") - s.dataSubmissionMtx.Unlock() - }() - if err := s.daSubmitter.SubmitData(s.ctx, s.cache, s.signer, s.genesis); err != nil { - // Check for unrecoverable errors that indicate a critical issue - if errors.Is(err, common.ErrOversizedItem) { - s.logger.Error().Err(err). - Msg("CRITICAL: Data exceeds DA blob size limit - halting to prevent live lock") - s.sendCriticalError(fmt.Errorf("unrecoverable DA submission error: %w", err)) - return + defer s.dataSubmissionMtx.Unlock() + + // Get pending data to estimate size + dataList, err := s.cache.GetPendingData(s.ctx) + if err != nil { + s.logger.Error().Err(err).Msg("failed to get pending data for batching decision") + return + } + + // Estimate total size + totalSize := 0 + for _, d := range dataList { + data, err := d.MarshalBinary() + if err == nil { + totalSize += len(data) + } + } + + shouldSubmit := s.batchingStrategy.ShouldSubmit( + uint64(len(dataList)), + totalSize, + common.DefaultMaxBlobSize, + timeSinceLastSubmit, + ) + + if shouldSubmit { + s.logger.Debug(). + Time("t", time.Now()). + Uint64("data", dataNb). + Int("total_size_kb", totalSize/1024). + Dur("time_since_last", timeSinceLastSubmit). + Str("strategy", s.batchingStrategy.Name()). + Msg("batching strategy triggered data submission") + + if err := s.daSubmitter.SubmitData(s.ctx, s.cache, s.signer, s.genesis); err != nil { + // Check for unrecoverable errors that indicate a critical issue + if errors.Is(err, common.ErrOversizedItem) { + s.logger.Error().Err(err). + Msg("CRITICAL: Data exceeds DA blob size limit - halting to prevent live lock") + s.sendCriticalError(fmt.Errorf("unrecoverable DA submission error: %w", err)) + return + } + s.logger.Error().Err(err).Msg("failed to submit data") + } else { + s.lastDataSubmit = time.Now() } - s.logger.Error().Err(err).Msg("failed to submit data") } }() } diff --git a/block/internal/submitting/submitter_test.go b/block/internal/submitting/submitter_test.go index f07cc6412..9a9e08a7f 100644 --- a/block/internal/submitting/submitter_test.go +++ b/block/internal/submitting/submitter_test.go @@ -338,6 +338,8 @@ func TestSubmitter_daSubmissionLoop(t *testing.T) { // Set a small block time so the ticker fires quickly cfg := config.DefaultConfig() cfg.DA.BlockTime.Duration = 5 * time.Millisecond + // Use immediate batching strategy so submissions happen right away + cfg.DA.BatchingStrategy = "immediate" metrics := common.NopMetrics() // Prepare fake DA submitter capturing calls @@ -350,6 +352,10 @@ func TestSubmitter_daSubmissionLoop(t *testing.T) { exec := testmocks.NewMockExecutor(t) // Provide a minimal signer implementation + // Initialize batching strategy (immediate for this test) + batchingStrategy, err := BatchingStrategyFactory(cfg.DA) + require.NoError(t, err) + s := &Submitter{ store: st, exec: exec, @@ -360,12 +366,21 @@ func TestSubmitter_daSubmissionLoop(t *testing.T) { daSubmitter: fakeDA, signer: &fakeSigner{}, daIncludedHeight: &atomic.Uint64{}, + lastHeaderSubmit: time.Now().Add(-time.Hour), // Set far in past so strategy allows submission + lastDataSubmit: time.Now().Add(-time.Hour), + batchingStrategy: batchingStrategy, logger: zerolog.Nop(), } // Make there be pending headers and data by setting store height > last submitted + h1, d1 := newHeaderAndData("test-chain", 1, true) + h2, d2 := newHeaderAndData("test-chain", 2, true) + + // Store the blocks batch, err := st.NewBatch(ctx) require.NoError(t, err) + require.NoError(t, batch.SaveBlockData(h1, d1, &types.Signature{})) + require.NoError(t, batch.SaveBlockData(h2, d2, &types.Signature{})) require.NoError(t, batch.SetHeight(2)) require.NoError(t, batch.Commit()) diff --git a/block/internal/syncing/syncer_forced_inclusion_test.go b/block/internal/syncing/syncer_forced_inclusion_test.go index 61c556e83..3180413a0 100644 --- a/block/internal/syncing/syncer_forced_inclusion_test.go +++ b/block/internal/syncing/syncer_forced_inclusion_test.go @@ -37,8 +37,8 @@ func TestCalculateBlockFullness_HalfFull(t *testing.T) { } fullness := s.calculateBlockFullness(data) - // Size fullness: 500000/2097152 ≈ 0.238 - require.InDelta(t, 0.238, fullness, 0.05) + // Size fullness: 500000/8388608 ≈ 0.0596 + require.InDelta(t, 0.0596, fullness, 0.05) } func TestCalculateBlockFullness_Full(t *testing.T) { @@ -55,8 +55,8 @@ func TestCalculateBlockFullness_Full(t *testing.T) { } fullness := s.calculateBlockFullness(data) - // Both metrics at or near 1.0 - require.Greater(t, fullness, 0.95) + // Size fullness: 2100000/8388608 ≈ 0.25 + require.InDelta(t, 0.25, fullness, 0.05) } func TestCalculateBlockFullness_VerySmall(t *testing.T) { diff --git a/pkg/config/config.go b/pkg/config/config.go index 0710997e5..6f8193f1e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -78,6 +78,14 @@ const ( FlagDAMaxSubmitAttempts = FlagPrefixEvnode + "da.max_submit_attempts" // FlagDARequestTimeout controls the per-request timeout when talking to the DA layer FlagDARequestTimeout = FlagPrefixEvnode + "da.request_timeout" + // FlagDABatchingStrategy is a flag for specifying the batching strategy + FlagDABatchingStrategy = FlagPrefixEvnode + "da.batching_strategy" + // FlagDABatchSizeThreshold is a flag for specifying the batch size threshold + FlagDABatchSizeThreshold = FlagPrefixEvnode + "da.batch_size_threshold" + // FlagDABatchMaxDelay is a flag for specifying the maximum batch delay + FlagDABatchMaxDelay = FlagPrefixEvnode + "da.batch_max_delay" + // FlagDABatchMinItems is a flag for specifying the minimum batch items + FlagDABatchMinItems = FlagPrefixEvnode + "da.batch_min_items" // P2P configuration flags @@ -182,7 +190,13 @@ type DAConfig struct { BlockTime DurationWrapper `mapstructure:"block_time" yaml:"block_time" comment:"Average block time of the DA chain (duration). Determines frequency of DA layer syncing, maximum backoff time for retries, and is multiplied by MempoolTTL to calculate transaction expiration. Examples: \"15s\", \"30s\", \"1m\", \"2m30s\", \"10m\"."` MempoolTTL uint64 `mapstructure:"mempool_ttl" yaml:"mempool_ttl" comment:"Number of DA blocks after which a transaction is considered expired and dropped from the mempool. Controls retry backoff timing."` MaxSubmitAttempts int `mapstructure:"max_submit_attempts" yaml:"max_submit_attempts" comment:"Maximum number of attempts to submit data to the DA layer before giving up. Higher values provide more resilience but can delay error reporting."` - RequestTimeout DurationWrapper `mapstructure:"request_timeout" yaml:"request_timeout" comment:"Per-request timeout applied to DA interactions. Larger values tolerate slower DA nodes at the cost of waiting longer before failing."` + RequestTimeout DurationWrapper `mapstructure:"request_timeout" yaml:"request_timeout" comment:"Timeout for requests to DA layer"` + + // Batching strategy configuration + BatchingStrategy string `mapstructure:"batching_strategy" yaml:"batching_strategy" comment:"Batching strategy for DA submissions. Options: 'immediate' (submit as soon as items are available), 'size' (wait until batch reaches size threshold), 'time' (wait for time interval), 'adaptive' (balance between size and time). Default: 'time'."` + BatchSizeThreshold float64 `mapstructure:"batch_size_threshold" yaml:"batch_size_threshold" comment:"Minimum blob size threshold (as fraction of max blob size, 0.0-1.0) before submitting. Only applies to 'size' and 'adaptive' strategies. Example: 0.8 means wait until batch is 80% full. Default: 0.8."` + BatchMaxDelay DurationWrapper `mapstructure:"batch_max_delay" yaml:"batch_max_delay" comment:"Maximum time to wait before submitting a batch regardless of size. Applies to 'time' and 'adaptive' strategies. Lower values reduce latency but may increase costs. Examples: \"6s\", \"12s\", \"30s\". Default: DA BlockTime."` + BatchMinItems uint64 `mapstructure:"batch_min_items" yaml:"batch_min_items" comment:"Minimum number of items (headers or data) to accumulate before considering submission. Helps avoid submitting single items when more are expected soon. Default: 1."` } // GetNamespace returns the namespace for header submissions. @@ -365,6 +379,10 @@ func AddFlags(cmd *cobra.Command) { cmd.Flags().Uint64(FlagDAMempoolTTL, def.DA.MempoolTTL, "number of DA blocks until transaction is dropped from the mempool") cmd.Flags().Int(FlagDAMaxSubmitAttempts, def.DA.MaxSubmitAttempts, "maximum number of attempts to submit data to the DA layer before giving up") cmd.Flags().Duration(FlagDARequestTimeout, def.DA.RequestTimeout.Duration, "per-request timeout when interacting with the DA layer") + cmd.Flags().String(FlagDABatchingStrategy, def.DA.BatchingStrategy, "batching strategy for DA submissions (immediate, size, time, adaptive)") + cmd.Flags().Float64(FlagDABatchSizeThreshold, def.DA.BatchSizeThreshold, "batch size threshold as fraction of max blob size (0.0-1.0)") + cmd.Flags().Duration(FlagDABatchMaxDelay, def.DA.BatchMaxDelay.Duration, "maximum time to wait before submitting a batch") + cmd.Flags().Uint64(FlagDABatchMinItems, def.DA.BatchMinItems, "minimum number of items to accumulate before submission") // P2P configuration flags cmd.Flags().String(FlagP2PListenAddress, def.P2P.ListenAddress, "P2P listen address (host:port)") diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 57506c0cf..74cb6b2dc 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -112,7 +112,7 @@ func TestAddFlags(t *testing.T) { assertFlagValue(t, flags, FlagRPCEnableDAVisualization, DefaultConfig().RPC.EnableDAVisualization) // Count the number of flags we're explicitly checking - expectedFlagCount := 49 // Update this number if you add more flag checks above + expectedFlagCount := 53 // Update this number if you add more flag checks above (added 4 batching flags) // Get the actual number of flags (both regular and persistent) actualFlagCount := 0 diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index 062c9fe19..c6691fb48 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -78,6 +78,10 @@ func DefaultConfig() Config { Namespace: randString(10), DataNamespace: "", ForcedInclusionNamespace: "", + BatchingStrategy: "time", + BatchSizeThreshold: 0.8, + BatchMaxDelay: DurationWrapper{6 * time.Second}, + BatchMinItems: 1, }, Instrumentation: DefaultInstrumentationConfig(), Log: LogConfig{ diff --git a/test/testda/dummy.go b/test/testda/dummy.go index 633bf1cf9..49469a350 100644 --- a/test/testda/dummy.go +++ b/test/testda/dummy.go @@ -11,8 +11,8 @@ import ( ) const ( - // DefaultMaxBlobSize is the default maximum blob size (2MB). - DefaultMaxBlobSize = 2 * 1024 * 1024 + // DefaultMaxBlobSize is the default maximum blob size (8MB). + DefaultMaxBlobSize = 8 * 1024 * 1024 ) // Header contains DA layer header information for a given height. diff --git a/tools/local-da/local.go b/tools/local-da/local.go index 86907fc4d..9f06471da 100644 --- a/tools/local-da/local.go +++ b/tools/local-da/local.go @@ -19,7 +19,7 @@ import ( ) // DefaultMaxBlobSize is the default max blob size -const DefaultMaxBlobSize uint64 = 2 * 1024 * 1024 // 2MB +const DefaultMaxBlobSize uint64 = 8 * 1024 * 1024 // 8MB // LocalDA is a simple implementation of in-memory DA. Not production ready! Intended only for testing! //