feat: binary tree mulitproofs (#322)

## Overview As part of a Blobstream integration for a zkEVM rollup, we're validating merkle proofs of _full blobs_ into data roots in the EVM. While it is possible to use a series of binary tree inclusion proofs, this amounts to a lot of unnecessary sha256 hashing, which imposes a high proving cost on the zkEVM. This pull request will add mulitproofs to the binary tree library.  ## Summary by CodeRabbit ## Summary by CodeRabbit - **New Features** - Added a submodule for the `forge-std` library to enhance dependency management for smart contract development. - Introduced functions for calculating subtree sizes and counting trailing zero bits in the `Utils` file. - Added support for multi-proof verification in the Binary Merkle Tree library with new verification functions. - Developed the `BinaryMerkleMultiproof` structure for efficient Merkle tree proof verification. - Introduced a structured JSON document for proof-related data to facilitate storage and retrieval. - **Tests** - Enhanced testing capabilities for multi-proof scenarios in the `BinaryMerkleProofTest` contract.  --------- Co-authored-by: rach-id <[email protected]>
celestiaorg · Jul 25, 2024 · 57c8c01 · 57c8c01
1 parent 7c0c5e4
commit 57c8c01
Show file tree

Hide file tree

Showing 15 changed files with 199 additions and 33 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -7,3 +7,6 @@
 [submodule "lib/openzeppelin-contracts-upgradeable"]
 	path = lib/openzeppelin-contracts-upgradeable
 	url = https://github.com/OpenZeppelin/openzeppelin-contracts-upgradeable
+[submodule "lib/forge-std"]
+	path = lib/forge-std
+	url = https://github.com/foundry-rs/forge-std
diff --git a/lib/forge-std b/lib/forge-std
diff --git a/remappings.txt b/remappings.txt
@@ -1,6 +1,6 @@
 @openzeppelin/contracts/=lib/openzeppelin-contracts-upgradeable/contracts/
 ds-test/=lib/ds-test/src/
 erc4626-tests/=lib/openzeppelin-contracts-upgradeable/lib/erc4626-tests/
-forge-std/=lib/openzeppelin-contracts-upgradeable/lib/forge-std/src/
+forge-std/=lib/forge-std/src/
 openzeppelin-contracts-upgradeable/=lib/openzeppelin-contracts-upgradeable/
 openzeppelin-contracts/=lib/openzeppelin-contracts/
diff --git a/src/lib/tree/Utils.sol b/src/lib/tree/Utils.sol
@@ -76,3 +76,31 @@ function _getSplitPoint(uint256 x) pure returns (uint256) {
     }
     return k;
 }
+
+/// @notice Returns the size of the subtree adjacent to `begin` that does
+/// not overlap `end`.
+/// @param begin Begin index, inclusive.
+/// @param end End index, exclusive.
+function _nextSubtreeSize(uint256 begin, uint256 end) pure returns (uint256) {
+    uint256 ideal = _bitsTrailingZeroes(begin);
+    uint256 max = _bitsLen(end - begin) - 1;
+    if (ideal > max) {
+        return 1 << max;
+    }
+    return 1 << ideal;
+}
+
+/// @notice Returns the number of trailing zero bits in `x`; the result is
+/// 256 for `x` == 0.
+/// @param x Number.
+function _bitsTrailingZeroes(uint256 x) pure returns (uint256) {
+    uint256 mask = 1;
+    uint256 count = 0;
+
+    while (x != 0 && mask & x == 0) {
+        count++;
+        x >>= 1;
+    }
+
+    return count;
+}
diff --git a/src/lib/tree/binary/BinaryMerkleMultiproof.sol b/src/lib/tree/binary/BinaryMerkleMultiproof.sol
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: Apache-2.0
+pragma solidity ^0.8.22;
+
+/// @notice Merkle Tree Proof structure.
+struct BinaryMerkleMultiproof {
+    // List of side nodes to verify and calculate tree.
+    bytes32[] sideNodes;
+    // The (included) beginning key of the leaves to verify.
+    uint256 beginKey;
+    // The (excluded) ending key of the leaves to verify.
+    uint256 endKey;
+}
diff --git a/src/lib/tree/binary/BinaryMerkleTree.sol b/src/lib/tree/binary/BinaryMerkleTree.sol
@@ -5,6 +5,7 @@ import "../Constants.sol";
 import "../Utils.sol";
 import "./TreeHasher.sol";
 import "./BinaryMerkleProof.sol";
+import "./BinaryMerkleMultiproof.sol";
 
 /// @title Binary Merkle Tree.
 library BinaryMerkleTree {
@@ -77,6 +78,122 @@ library BinaryMerkleTree {
         return (computedHash == root, ErrorCodes.NoError);
     }
 
+    function verifyMulti(bytes32 root, BinaryMerkleMultiproof memory proof, bytes[] memory data)
+        internal
+        pure
+        returns (bool)
+    {
+        bytes32[] memory nodes = new bytes32[](data.length);
+        for (uint256 i = 0; i < data.length; i++) {
+            nodes[i] = leafDigest(data[i]);
+        }
+
+        return verifyMultiHashes(root, proof, nodes);
+    }
+
+    function verifyMultiHashes(bytes32 root, BinaryMerkleMultiproof memory proof, bytes32[] memory leafNodes)
+        internal
+        pure
+        returns (bool)
+    {
+        uint256 leafIndex = 0;
+        bytes32[] memory leftSubtrees = new bytes32[](proof.sideNodes.length);
+
+        for (uint256 i = 0; leafIndex != proof.beginKey && i < proof.sideNodes.length; ++i) {
+            uint256 subtreeSize = _nextSubtreeSize(leafIndex, proof.beginKey);
+            leftSubtrees[i] = proof.sideNodes[i];
+            leafIndex += subtreeSize;
+        }
+
+        uint256 proofRangeSubtreeEstimate = _getSplitPoint(proof.endKey) * 2;
+        if (proofRangeSubtreeEstimate < 1) {
+            proofRangeSubtreeEstimate = 1;
+        }
+
+        (bytes32 rootHash, uint256 proofHead,,) =
+            _computeRootMulti(proof, leafNodes, 0, proofRangeSubtreeEstimate, 0, 0);
+        for (uint256 i = proofHead; i < proof.sideNodes.length; ++i) {
+            rootHash = nodeDigest(rootHash, proof.sideNodes[i]);
+        }
+
+        return (rootHash == root);
+    }
+
+    function _computeRootMulti(
+        BinaryMerkleMultiproof memory proof,
+        bytes32[] memory leafNodes,
+        uint256 begin,
+        uint256 end,
+        uint256 headProof,
+        uint256 headLeaves
+    ) private pure returns (bytes32, uint256, uint256, bool) {
+        // reached a leaf
+        if (end - begin == 1) {
+            // if current range overlaps with proof range, pop and return a leaf
+            if (proof.beginKey <= begin && begin < proof.endKey) {
+                // Note: second return value is guaranteed to be `false` by
+                // construction.
+                return _popLeavesIfNonEmpty(leafNodes, headLeaves, leafNodes.length, headProof);
+            }
+
+            // if current range does not overlap with proof range,
+            // pop and return a proof node (leaf) if present,
+            // else return nil because leaf doesn't exist
+            return _popProofIfNonEmpty(proof.sideNodes, headProof, end, headLeaves);
+        }
+
+        // if current range does not overlap with proof range,
+        // pop and return a proof node if present,
+        // else return nil because subtree doesn't exist
+        if (end <= proof.beginKey || begin >= proof.endKey) {
+            return _popProofIfNonEmpty(proof.sideNodes, headProof, end, headLeaves);
+        }
+
+        // Recursively get left and right subtree
+        uint256 k = _getSplitPoint(end - begin);
+        (bytes32 left, uint256 newHeadProofLeft, uint256 newHeadLeavesLeft,) =
+            _computeRootMulti(proof, leafNodes, begin, begin + k, headProof, headLeaves);
+        (bytes32 right, uint256 newHeadProof, uint256 newHeadLeaves, bool rightIsNil) =
+            _computeRootMulti(proof, leafNodes, begin + k, end, newHeadProofLeft, newHeadLeavesLeft);
+
+        // only right leaf/subtree can be non-existent
+        if (rightIsNil == true) {
+            return (left, newHeadProof, newHeadLeaves, false);
+        }
+        bytes32 hash = nodeDigest(left, right);
+        return (hash, newHeadProof, newHeadLeaves, false);
+    }
+
+    function _popProofIfNonEmpty(bytes32[] memory nodes, uint256 headProof, uint256 end, uint256 headLeaves)
+        private
+        pure
+        returns (bytes32, uint256, uint256, bool)
+    {
+        (bytes32 node, uint256 newHead, bool isNil) = _popIfNonEmpty(nodes, headProof, end);
+        return (node, newHead, headLeaves, isNil);
+    }
+
+    function _popLeavesIfNonEmpty(bytes32[] memory nodes, uint256 headLeaves, uint256 end, uint256 headProof)
+        private
+        pure
+        returns (bytes32, uint256, uint256, bool)
+    {
+        (bytes32 node, uint256 newHead, bool isNil) = _popIfNonEmpty(nodes, headLeaves, end);
+        return (node, headProof, newHead, isNil);
+    }
+
+    function _popIfNonEmpty(bytes32[] memory nodes, uint256 head, uint256 end)
+        private
+        pure
+        returns (bytes32, uint256, bool)
+    {
+        if (nodes.length == 0 || head >= nodes.length || head >= end) {
+            bytes32 node;
+            return (node, head, true);
+        }
+        return (nodes[head], head + 1, false);
+    }
+
     /// @notice Use the leafHash and innerHashes to get the root merkle hash.
     /// If the length of the innerHashes slice isn't exactly correct, the result is nil.
     /// Recursive impl.

diff --git a/src/lib/tree/binary/test/BinaryMerkleTree.t.sol b/src/lib/tree/binary/test/BinaryMerkleTree.t.sol
@@ -6,6 +6,7 @@ import "forge-std/Vm.sol";
 
 import "../BinaryMerkleProof.sol";
 import "../BinaryMerkleTree.sol";
+import "../BinaryMerkleMultiproof.sol";
 
 /**
  * TEST VECTORS
@@ -333,4 +334,18 @@ contract BinaryMerkleProofTest is DSTest {
         vm.expectRevert("Invalid range: _begin or _end are out of bounds");
         BinaryMerkleTree.slice(data, 2, 5);
     }
+
+    // header.dat, blob.dat, and proofs.json test vectors included in ../../test/ and serialized to hex bytes using Rust
+    // The hard-coded serialized proofs and data were generated in Rust, with this code
+    // https://github.com/S1nus/hyperchain-da/blob/main/src/clients/celestia/evm_types.rs#L132
+    function testMultiproof() public {
+        bytes memory proofData =
+            hex"00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000090000000000000000000000000000000000000000000000000000000000000006ce29bcde696f84e35c5626904542a549b080e92603243b34794242473940706917519bf954f5b30495af5c8cdb9983e6319104badc1ea811ed2c421018a3ad7821ea268d3540deab8f9b2024464618610c9a7083620badcf505bda647cc8e9f82bfc87d990d8344f6efd44fcb09b46b87f9a92230d41329452efee8656c6760a9ad9f3a95af971e89e2a80b255bb56d5aae15de69803b52aa5079b33374b16e16178fc62a2f2ce6bf21909c0a0edea9525486e0ece65bff23499342cca38dd62";
+        BinaryMerkleMultiproof memory multiproof = abi.decode(proofData, (BinaryMerkleMultiproof));
+        bytes32 dataroot = hex"ef8920d86519bd5f8ce3c802b84fc9b9512483e4d4a5c9608b44af4d6639f7d1";
+        bytes memory leafData =
+            hex"00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000009000000000000000000000000000000000000000000000000000000000000012000000000000000000000000000000000000000000000000000000000000001a0000000000000000000000000000000000000000000000000000000000000022000000000000000000000000000000000000000000000000000000000000002a0000000000000000000000000000000000000000000000000000000000000032000000000000000000000000000000000000000000000000000000000000003a0000000000000000000000000000000000000000000000000000000000000042000000000000000000000000000000000000000000000000000000000000004a00000000000000000000000000000000000000000000000000000000000000520000000000000000000000000000000000000000000000000000000000000005a00000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000102030405746e218305fe3dbbef65feceed939fe8dd93c88b06c95473fbe344fb864060f3000000000000000000000000000000000000000000000000000000000000000000000000005a0000000000000000000000000000000000000000000000000102030405000000000000000000000000000000000000000000000000010203040555cd7fb524ae792c9d4bc8946d07209728c533a3e14d4e7c0c95c0b150d0c284000000000000000000000000000000000000000000000000000000000000000000000000005a00000000000000000000000000000000000000000000000001020304050000000000000000000000000000000000000000000000000102030405505c1e7c897461a152e152f1ff3ecc358fefdf1f69448ab1165b6ca76836933b000000000000000000000000000000000000000000000000000000000000000000000000005a00000000000000000000000000000000000000000000000001020304050000000000000000000000000000000000000000000000000102030405100a0548893d8eab0322f34f45ac84785cdf50dfab5102a12d958e6031bacebe000000000000000000000000000000000000000000000000000000000000000000000000005a0000000000000000000000000000000000000000000000000102030405000000000000000000000000000000000000000000000000010203040566e5eb1da67430f204a3c5615591f71316695c7ec1f1f713cde7e936d4a43ec1000000000000000000000000000000000000000000000000000000000000000000000000005a00000000000000000000000000000000000000000000000001020304050000000000000000000000000000000000000000000000000102030405d2a5de6299e28c2fec359a2718599f5ac22c2948a71d26a438295e531b6f4cb5000000000000000000000000000000000000000000000000000000000000000000000000005a00000000000000000000000000000000000000000000000001020304050000000000000000000000000000000000000000000000000102030405688c5238e50c0a8a556bfabff31bef1fa9cdd812c9fd4dcee5c2a0836f687fbf000000000000000000000000000000000000000000000000000000000000000000000000005a00000000000000000000000000000000000000000000000001020304050000000000000000000000000000000000000000000000000102030405b55a5b1efc2a22cdbfa21d050bd67147ff2b936c68354eb1a83bcdf14eb57e38000000000000000000000000000000000000000000000000000000000000000000000000005a000000000000000000000000000000000000000000000000010203040500000000000000000000000000000000000000000067480c4a88c4d129947e11c33fa811daa791771e591dd933498d1212d46b8cde9c34c28831b0b532000000000000";
+        bytes[] memory leaves = abi.decode(leafData, (bytes[]));
+        assertTrue(BinaryMerkleTree.verifyMulti(dataroot, multiproof, leaves));
+    }
 }
diff --git a/src/lib/tree/namespace/NamespaceMerkleMultiproof.sol b/src/lib/tree/namespace/NamespaceMerkleMultiproof.sol
@@ -5,9 +5,9 @@ import "./NamespaceNode.sol";
 
 /// @notice Namespace Merkle Tree Multiproof structure. Proves multiple leaves.
 struct NamespaceMerkleMultiproof {
-    // The beginning key of the leaves to verify.
+    // The (included) beginning key of the leaves to verify.
     uint256 beginKey;
-    // The ending key of the leaves to verify.
+    // The (excluded) ending key of the leaves to verify.
     uint256 endKey;
     // List of side nodes to verify and calculate tree.
     NamespaceNode[] sideNodes;

diff --git a/src/lib/tree/namespace/NamespaceMerkleTree.sol b/src/lib/tree/namespace/NamespaceMerkleTree.sol
@@ -193,34 +193,6 @@ library NamespaceMerkleTree {
         return namespaceNodeEquals(rootHash, root);
     }
 
-    /// @notice Returns the size of the subtree adjacent to `begin` that does
-    /// not overlap `end`.
-    /// @param begin Begin index, inclusive.
-    /// @param end End index, exclusive.
-    function _nextSubtreeSize(uint256 begin, uint256 end) private pure returns (uint256) {
-        uint256 ideal = _bitsTrailingZeroes(begin);
-        uint256 max = _bitsLen(end - begin) - 1;
-        if (ideal > max) {
-            return 1 << max;
-        }
-        return 1 << ideal;
-    }
-
-    /// @notice Returns the number of trailing zero bits in `x`; the result is
-    /// 256 for `x` == 0.
-    /// @param x Number.
-    function _bitsTrailingZeroes(uint256 x) private pure returns (uint256) {
-        uint256 mask = 1;
-        uint256 count = 0;
-
-        while (x != 0 && mask & x == 0) {
-            count++;
-            x >>= 1;
-        }
-
-        return count;
-    }
-
     /// @notice Computes the NMT root recursively.
     /// @param proof Namespace Merkle multiproof for the leaves.
     /// @param leafNodes Leaf nodes for which inclusion is proven.

diff --git a/src/lib/tree/namespace/test/NamespaceMerkleMultiproof.t.sol b/src/lib/tree/namespace/test/NamespaceMerkleMultiproof.t.sol
diff --git a/src/lib/tree/test/blob.dat b/src/lib/tree/test/blob.dat
diff --git a/src/lib/tree/test/header.dat b/src/lib/tree/test/header.dat
diff --git a/src/lib/tree/test/proofs.json b/src/lib/tree/test/proofs.json
@@ -0,0 +1 @@
+[{"start":8,"end":32,"nodes":["AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABOLCUcGcDNOGgcYmOnu7snv+cn+3G+vkto91wnXa3kVQ","/////////////////////////////////////////////////////////////////////////////zmvU+iSdf6GDmfvDMVa0YqTan9iPIiX5UHyC8zhZkkf"],"leaf_hash":"","is_max_namespace_ignored":true},{"start":0,"end":32,"nodes":["/////////////////////////////////////////////////////////////////////////////xZ6gFJq4RO/FIE75WZbKQOZmS3FCVTEVM/dKR/kzDZz"],"leaf_hash":"","is_max_namespace_ignored":true},{"start":0,"end":32,"nodes":["/////////////////////////////////////////////////////////////////////////////5MEmpTXPlH6UVRm8X2csA+EaccobRIwYWyOF5ls5abx"],"leaf_hash":"","is_max_namespace_ignored":true},{"start":0,"end":32,"nodes":["/////////////////////////////////////////////////////////////////////////////3wn2BGIhHap4sg/oUMt6THYs/c8kj+mFXPoFL3NxC9I"],"leaf_hash":"","is_max_namespace_ignored":true},{"start":0,"end":32,"nodes":["//////////////////////////////////////////////////////////////////////////////vTJmXZEYdLpfIuqDC7XrkhCLaw6GE1Iz1EzwDecR57"],"leaf_hash":"","is_max_namespace_ignored":true},{"start":0,"end":32,"nodes":["/////////////////////////////////////////////////////////////////////////////+HJEAbaxVdhII24yEvjUz8rAJmA8T9ratZdZHUASsLe"],"leaf_hash":"","is_max_namespace_ignored":true},{"start":0,"end":32,"nodes":["/////////////////////////////////////////////////////////////////////////////4K3KpNnwd8qcf40yM88DYQ087APK4Kjc98+WAchu59l"],"leaf_hash":"","is_max_namespace_ignored":true},{"start":0,"end":32,"nodes":["/////////////////////////////////////////////////////////////////////////////5wXTQR9kfP9JmlezblQvri0MM7UccRF1qUZ/2ELgyMf"],"leaf_hash":"","is_max_namespace_ignored":true},{"start":0,"end":24,"nodes":["AAAAAAAAAAAAAAAAAAAAAAAAAAAABpbeXXkKu9gAAAAAAAAAAAAAAAAAAAAAAAAAAABnSAxKiMTRKTLbUSuhrp5YtSI7UOw4sUkTDR1mrRutu+xxGAq64vXA","/////////////////////////////////////////////////////////////////////////////9+uMDUChYyrcrRy2fQ9h15MuGF69AmtPLH3FGuTFCUX"],"leaf_hash":"","is_max_namespace_ignored":true}]