feat(aggregation-mode): Bump fee when proof verification times out

aggregation_mode/proof_aggregator/src/backend/config.rs

@@ -21,6 +21,11 @@ pub struct Config {
     pub sp1_chunk_aggregator_vk_hash: String,
     pub monthly_budget_eth: f64,
     pub db_connection_urls: Vec<String>,
+    pub max_bump_retries: u16,
+    pub bump_retry_interval_seconds: u64,
+    pub base_bump_percentage: u64,
+    pub max_fee_bump_percentage: u64,
+    pub priority_fee_wei: u128,
 }
 
 impl Config {

aggregation_mode/proof_aggregator/src/backend/mod.rs

@@ -22,10 +22,10 @@ use alloy::{
     consensus::{BlobTransactionSidecar, EnvKzgSettings, EthereumTxEnvelope, TxEip4844WithSidecar},
     eips::{eip4844::BYTES_PER_BLOB, eip7594::BlobTransactionSidecarEip7594, Encodable2718},
     hex,
-    network::EthereumWallet,
+    network::{EthereumWallet, TransactionBuilder},
     primitives::{utils::parse_ether, Address, U256},
-    providers::{PendingTransactionError, Provider, ProviderBuilder},
-    rpc::types::TransactionReceipt,
+    providers::{PendingTransactionError, Provider, ProviderBuilder, WalletProvider},
+    rpc::types::{TransactionReceipt, TransactionRequest},
     signers::local::LocalSigner,
 };
 use config::Config;
@@ -334,7 +334,91 @@ impl ProofAggregator {
 
         info!("Sending proof to ProofAggregationService contract...");
 
-        let tx_req = match aggregated_proof {
+        let max_retries = self.config.max_bump_retries;
+
+        let mut last_error: Option<AggregatedProofSubmissionError> = None;
+
+        // Get the nonce once at the beginning and reuse it for all retries
+        let nonce = self
+            .proof_aggregation_service
+            .provider()
+            .get_transaction_count(
+                self.proof_aggregation_service
+                    .provider()
+                    .default_signer_address(),
+            )
+            .await
+            .map_err(|e| {
+                RetryError::Transient(
+                    AggregatedProofSubmissionError::SendVerifyAggregatedProofTransaction(format!(
+                        "Failed to get nonce: {e}"
+                    )),
+                )
+            })?;
+
+        info!("Using nonce {} for all retry attempts", nonce);
+
+        for attempt in 0..max_retries {
+            info!("Transaction attempt {} of {}", attempt + 1, max_retries);
+
+            // Wrap the entire transaction submission in a result to catch all errors, passing
+            // the same nonce to all attempts
+            let attempt_result = self
+                .try_submit_transaction(
+                    &blob,
+                    blob_versioned_hash,
+                    aggregated_proof,
+                    attempt as u64,
+                    nonce,
+                )
+                .await;
+
+            match attempt_result {
+                Ok(receipt) => {
+                    info!(
+                        "Transaction confirmed successfully on attempt {}",
+                        attempt + 1
+                    );
+                    return Ok(receipt);
+                }
+                Err(err) => {
+                    warn!("Attempt {} failed: {:?}", attempt + 1, err);
+                    last_error = Some(err);
+
+                    if attempt < max_retries - 1 {
+                        info!("Retrying with bumped gas fees and same nonce {}...", nonce);
+
+                        tokio::time::sleep(Duration::from_millis(500)).await;
+                    } else {
+                        warn!("Max retries ({}) exceeded", max_retries);
+                    }
+                }
+            }
+        }
+
+        // If we exhausted all retries, return the last error
+        Err(RetryError::Transient(last_error.unwrap_or_else(|| {
+            AggregatedProofSubmissionError::SendVerifyAggregatedProofTransaction(
+                "Max retries exceeded with no error details".to_string(),
+            )
+        })))
+    }
+
+    async fn try_submit_transaction(
+        &self,
+        blob: &BlobTransactionSidecar,
+        blob_versioned_hash: [u8; 32],
+        aggregated_proof: &AlignedProof,
+        attempt: u64,
+        nonce: u64,
+    ) -> Result<TransactionReceipt, AggregatedProofSubmissionError> {
+        let retry_interval = Duration::from_secs(self.config.bump_retry_interval_seconds);
+        let base_bump_percentage = self.config.base_bump_percentage;
+        let max_fee_bump_percentage = self.config.max_fee_bump_percentage;
+        let priority_fee_wei = self.config.priority_fee_wei;
+
+        // Build the transaction request
+        let mut tx_req = match aggregated_proof {
             AlignedProof::SP1(proof) => self
                 .proof_aggregation_service
                 .verifyAggregationSP1(
@@ -343,81 +427,144 @@ impl ProofAggregator {
                     proof.proof_with_pub_values.bytes().into(),
                     self.sp1_chunk_aggregator_vk_hash_bytes.into(),
                 )
-                .sidecar(blob)
+                .sidecar(blob.clone())
                 .into_transaction_request(),
             AlignedProof::Risc0(proof) => {
-                let encoded_seal = encode_seal(&proof.receipt)
-                    .map_err(|e| AggregatedProofSubmissionError::Risc0EncodingSeal(e.to_string()))
-                    .map_err(RetryError::Permanent)?;
+                let encoded_seal = encode_seal(&proof.receipt).map_err(|e| {
+                    AggregatedProofSubmissionError::Risc0EncodingSeal(e.to_string())
+                })?;
                 self.proof_aggregation_service
                     .verifyAggregationRisc0(
                         blob_versioned_hash.into(),
                         encoded_seal.into(),
                         proof.receipt.journal.bytes.clone().into(),
                         self.risc0_chunk_aggregator_image_id_bytes.into(),
                     )
-                    .sidecar(blob)
+                    .sidecar(blob.clone())
                     .into_transaction_request()
             }
         };
 
+        // Set the nonce explicitly
+        tx_req = tx_req.with_nonce(nonce);
+
+        // Apply gas fee bump for retries
+        if attempt > 0 {
+            tx_req = self
+                .apply_gas_fee_bump(
+                    base_bump_percentage,
+                    max_fee_bump_percentage,
+                    priority_fee_wei,
+                    tx_req,
+                )
+                .await?;
+        }
+
         let provider = self.proof_aggregation_service.provider();
+
+        // Fill the transaction
         let envelope = provider
             .fill(tx_req)
             .await
             .map_err(|err| {
-                AggregatedProofSubmissionError::SendVerifyAggregatedProofTransaction(
-                    err.to_string(),
-                )
-            })
-            .map_err(RetryError::Transient)?
+                AggregatedProofSubmissionError::SendVerifyAggregatedProofTransaction(format!(
+                    "Failed to fill transaction: {err}"
+                ))
+            })?
             .try_into_envelope()
             .map_err(|err| {
-                AggregatedProofSubmissionError::SendVerifyAggregatedProofTransaction(
-                    err.to_string(),
-                )
-            })
-            .map_err(RetryError::Transient)?;
+                AggregatedProofSubmissionError::SendVerifyAggregatedProofTransaction(format!(
+                    "Failed to convert to envelope: {err}"
+                ))
+            })?;
+
+        // Convert to EIP-4844 transaction
         let tx: EthereumTxEnvelope<TxEip4844WithSidecar<BlobTransactionSidecarEip7594>> = envelope
             .try_into_pooled()
             .map_err(|err| {
-                AggregatedProofSubmissionError::SendVerifyAggregatedProofTransaction(
-                    err.to_string(),
-                )
-            })
-            .map_err(RetryError::Transient)?
+                AggregatedProofSubmissionError::SendVerifyAggregatedProofTransaction(format!(
+                    "Failed to pool transaction: {err}"
+                ))
+            })?
             .try_map_eip4844(|tx| {
                 tx.try_map_sidecar(|sidecar| sidecar.try_into_7594(EnvKzgSettings::Default.get()))
             })
             .map_err(|err| {
-                AggregatedProofSubmissionError::SendVerifyAggregatedProofTransaction(
-                    err.to_string(),
-                )
-            })
-            .map_err(RetryError::Transient)?;
+                AggregatedProofSubmissionError::SendVerifyAggregatedProofTransaction(format!(
+                    "Failed to convert to EIP-7594: {err}"
+                ))
+            })?;
 
+        // Send the transaction
         let encoded_tx = tx.encoded_2718();
         let pending_tx = provider
             .send_raw_transaction(&encoded_tx)
             .await
             .map_err(|err| {
-                AggregatedProofSubmissionError::SendVerifyAggregatedProofTransaction(
-                    err.to_string(),
-                )
-            })
-            .map_err(RetryError::Transient)?;
+                AggregatedProofSubmissionError::SendVerifyAggregatedProofTransaction(format!(
+                    "Failed to send raw transaction: {err}"
+                ))
+            })?;
 
-        let receipt = pending_tx
-            .get_receipt()
-            .await
-            .map_err(|err| {
-                AggregatedProofSubmissionError::SendVerifyAggregatedProofTransaction(
-                    err.to_string(),
-                )
-            })
-            .map_err(RetryError::Transient)?;
+        info!(
+            "Transaction sent with nonce {}, waiting for confirmation...",
+            nonce
+        );
+
+        // Wait for the receipt with timeout
+        let receipt_result = tokio::time::timeout(retry_interval, pending_tx.get_receipt()).await;
+
+        match receipt_result {
+            Ok(Ok(receipt)) => Ok(receipt),
+            Ok(Err(err)) => Err(
+                AggregatedProofSubmissionError::SendVerifyAggregatedProofTransaction(format!(
+                    "Error getting receipt: {err}"
+                )),
+            ),
+            Err(_) => Err(
+                AggregatedProofSubmissionError::SendVerifyAggregatedProofTransaction(format!(
+                    "Transaction timeout after {} seconds",
+                    retry_interval.as_secs()
+                )),
+            ),
+        }
+    }
+
+    // Updates the gas fees of a `TransactionRequest` using a fixed bump strategy.
+    // Intended for retrying an on-chain submission after a timeout.
+    //
+    // Strategy:
+    // - Fetch the current network gas price.
+    // - Apply `base_bump_percentage` to compute a bumped base fee.
+    // - Apply `max_fee_bump_percentage` on top of the bumped base fee to set `max_fee_per_gas`.
+    // - Set `max_priority_fee_per_gas` to a fixed value derived from `priority_fee_wei`.
+    //
+    // Fees are recomputed on each retry using the latest gas price (no incremental per-attempt bump).
+
+    async fn apply_gas_fee_bump(
+        &self,
+        base_bump_percentage: u64,
+        max_fee_bump_percentage: u64,
+        priority_fee_wei: u128,
+        tx_req: TransactionRequest,
+    ) -> Result<TransactionRequest, AggregatedProofSubmissionError> {
+        let provider = self.proof_aggregation_service.provider();
 
-        Ok(receipt)
+        let current_gas_price = provider
+            .get_gas_price()
+            .await
+            .map_err(|e| AggregatedProofSubmissionError::GasPriceError(e.to_string()))?;
+
+        let new_base_fee = current_gas_price as f64 * (1.0 + base_bump_percentage as f64 / 100.0);
+        let new_max_fee = new_base_fee * (1.0 + max_fee_bump_percentage as f64 / 100.0);
+        let new_priority_fee = priority_fee_wei;
+
+        Ok(tx_req
+            // In TransactionRequest docs the gas_price field is defined as
+            // "The max base fee per gas the sender is willing to pay."
+            .with_gas_price(new_base_fee as u128)
+            .with_max_fee_per_gas(new_max_fee as u128)
+            .with_max_priority_fee_per_gas(new_priority_fee))
     }
 
     async fn wait_until_can_submit_aggregated_proof(

config-files/config-proof-aggregator-ethereum-package.yaml

@@ -27,6 +27,13 @@ monthly_budget_eth: 15.0
 sp1_chunk_aggregator_vk_hash: "00d6e32a34f68ea643362b96615591c94ee0bf99ee871740ab2337966a4f77af"
 risc0_chunk_aggregator_image_id: "8908f01022827e80a5de71908c16ee44f4a467236df20f62e7c994491629d74c"
 
+# These values modify the bumping behavior after the aggregated proof on-chain submission times out.
+max_bump_retries: 5
+bump_retry_interval_seconds: 120
+base_bump_percentage: 10
+max_fee_bump_percentage: 100
+priority_fee_wei: 2000000000
+
 ecdsa:
   private_key_store_path: "config-files/anvil.proof-aggregator.ecdsa.key.json"
   private_key_store_password: ""

config-files/config-proof-aggregator.yaml

@@ -24,8 +24,15 @@ monthly_budget_eth: 15.0
 # These program ids are the ones from the chunk aggregator programs
 # Can be found in the Proof Aggregation Service deployment config
 # (remember to trim the 0x prefix)
-sp1_chunk_aggregator_vk_hash: "00ba19eed0aaeb0151f07b8d3ee7c659bcd29f3021e48fb42766882f55b84509"
-risc0_chunk_aggregator_image_id: "d8cfdd5410c70395c0a1af1842a0148428cc46e353355faccfba694dd4862dbf"
+sp1_chunk_aggregator_vk_hash: "00d6e32a34f68ea643362b96615591c94ee0bf99ee871740ab2337966a4f77af"
+risc0_chunk_aggregator_image_id: "8908f01022827e80a5de71908c16ee44f4a467236df20f62e7c994491629d74c"
+
+# These values modify the bumping behavior after the aggregated proof on-chain submission times out.
+max_bump_retries: 5
+bump_retry_interval_seconds: 120
+base_bump_percentage: 10
+max_fee_bump_percentage: 100
+priority_fee_wei: 2000000000
 
 ecdsa:
   private_key_store_path: "config-files/anvil.proof-aggregator.ecdsa.key.json"