From 5709f03dd2a8abed0c86675e46e0c8b0c4a0da60 Mon Sep 17 00:00:00 2001 From: h5law <53987565+h5law@users.noreply.github.com> Date: Tue, 9 Jan 2024 17:33:08 +0000 Subject: [PATCH 1/3] feat: add MerkleRoot type and Sum method --- root_test.go | 39 +++++++++++++++++++++++++++++++++++++++ smst.go | 6 ++---- smt.go | 2 +- types.go | 19 +++++++++++++++++-- 4 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 root_test.go diff --git a/root_test.go b/root_test.go new file mode 100644 index 0000000..cd6b34d --- /dev/null +++ b/root_test.go @@ -0,0 +1,39 @@ +package smt_test + +import ( + "crypto/sha256" + "encoding/binary" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/pokt-network/smt" + "github.com/pokt-network/smt/kvstore/simplemap" +) + +func TestMerkleRoot_SumTrie(t *testing.T) { + nodeStore := simplemap.NewSimpleMap() + trie := smt.NewSparseMerkleSumTrie(nodeStore, sha256.New()) + for i := uint64(0); i < 10; i++ { + require.NoError(t, trie.Update([]byte(fmt.Sprintf("key%d", i)), []byte(fmt.Sprintf("value%d", i)), i)) + } + root := trie.Root() + require.Equal(t, root.Sum(true), getSumBzHelper(t, root)) +} + +func TestMerkleRoot_Trie(t *testing.T) { + nodeStore := simplemap.NewSimpleMap() + trie := smt.NewSparseMerkleTrie(nodeStore, sha256.New()) + for i := 0; i < 10; i++ { + require.NoError(t, trie.Update([]byte(fmt.Sprintf("key%d", i)), []byte(fmt.Sprintf("value%d", i)))) + } + root := trie.Root() + require.Equal(t, root.Sum(false), uint64(0)) +} + +func getSumBzHelper(t *testing.T, r []byte) uint64 { + var sumbz [8]byte // Using sha256 + copy(sumbz[:], []byte(r)[len([]byte(r))-8:]) // Using sha256 so - 8 bytes + return binary.BigEndian.Uint64(sumbz[:]) +} diff --git a/smst.go b/smst.go index d1c4368..2a6a4b6 100644 --- a/smst.go +++ b/smst.go @@ -112,14 +112,12 @@ func (smst *SMST) Commit() error { } // Root returns the root hash of the trie with the total sum bytes appended -func (smst *SMST) Root() []byte { +func (smst *SMST) Root() MerkleRoot { return smst.SMT.Root() // [digest]+[binary sum] } // Sum returns the uint64 sum of the entire trie func (smst *SMST) Sum() uint64 { - var sumBz [sumSize]byte digest := smst.Root() - copy(sumBz[:], digest[len(digest)-sumSize:]) - return binary.BigEndian.Uint64(sumBz[:]) + return digest.Sum(true) } diff --git a/smt.go b/smt.go index f4ceacd..558db63 100644 --- a/smt.go +++ b/smt.go @@ -675,7 +675,7 @@ func (smt *SMT) commit(node trieNode) error { } // Root returns the root hash of the trie -func (smt *SMT) Root() []byte { +func (smt *SMT) Root() MerkleRoot { return hashNode(smt.Spec(), smt.trie) } diff --git a/types.go b/types.go index ace6cad..4439764 100644 --- a/types.go +++ b/types.go @@ -1,6 +1,7 @@ package smt import ( + "encoding/binary" "hash" ) @@ -14,6 +15,20 @@ var ( defaultSum [sumSize]byte ) +// MerkleRoot is a type alias for a byte slice returned from the Root method +type MerkleRoot []byte + +// Sum returns the uint64 sum of the merkle root, if sumTrie is true +// otherwise it returns 0, as it would result in undefined behaviour. +func (r MerkleRoot) Sum(sumTrie bool) uint64 { + if sumTrie { + var sumbz [sumSize]byte + copy(sumbz[:], []byte(r)[len([]byte(r))-sumSize:]) + return binary.BigEndian.Uint64(sumbz[:]) + } + return 0 +} + // SparseMerkleTrie represents a Sparse Merkle Trie. type SparseMerkleTrie interface { // Update inserts a value into the SMT. @@ -23,7 +38,7 @@ type SparseMerkleTrie interface { // Get descends the trie to access a value. Returns nil if key is not present. Get(key []byte) ([]byte, error) // Root computes the Merkle root digest. - Root() []byte + Root() MerkleRoot // Prove computes a Merkle proof of inclusion or exclusion of a key. Prove(key []byte) (*SparseMerkleProof, error) // ProveClosest computes a Merkle proof of inclusion for a key in the trie @@ -46,7 +61,7 @@ type SparseMerkleSumTrie interface { // Get descends the trie to access a value. Returns nil if key is not present. Get(key []byte) ([]byte, uint64, error) // Root computes the Merkle root digest. - Root() []byte + Root() MerkleRoot // Sum computes the total sum of the Merkle trie Sum() uint64 // Prove computes a Merkle proof of inclusion or exclusion of a key. From ad3911caa5fb4d65ac132bdb45ea392af9b0bd11 Mon Sep 17 00:00:00 2001 From: h5law <53987565+h5law@users.noreply.github.com> Date: Tue, 9 Jan 2024 17:41:18 +0000 Subject: [PATCH 2/3] chore: update documentation with root info --- docs/merkle-sum-trie.md | 10 ++++++++++ docs/smt.md | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/docs/merkle-sum-trie.md b/docs/merkle-sum-trie.md index 757536e..226adab 100644 --- a/docs/merkle-sum-trie.md +++ b/docs/merkle-sum-trie.md @@ -10,6 +10,7 @@ + [General Trie Structure](#general-trie-structure) + [Binary Sum Digests](#binary-sum-digests) - [Sum](#sum) +- [Roots](#roots) - [Nil Values](#nil-values) @@ -248,6 +249,15 @@ graph TB The `Sum()` function adds functionality to easily retrieve the trie's current sum as a `uint64`. +## Roots + +Roots are `[]byte` types aliases by the `MerkleRoot` type. This type has one +method `Sum(sumTrie bool) uint64`. For the SMST this method is used by its own +`Sum()` method to return the total sum of the trie. + +The `MerkleRoot` type being an alias means it can be used in place of the +`[]byte` type. Specifically for proofs. + ## Nil Values A `nil` value and `0` weight is the same as the placeholder value and default diff --git a/docs/smt.md b/docs/smt.md index 31a0509..598569d 100644 --- a/docs/smt.md +++ b/docs/smt.md @@ -17,6 +17,7 @@ - [Values](#values) * [Nil values](#nil-values) - [Hashers & Digests](#hashers--digests) +- [Roots](#roots) - [Proofs](#proofs) * [Verification](#verification) * [Closest Proof](#closest-proof) @@ -342,6 +343,14 @@ graph TD VH --ValueHash-->L ``` +## Roots + +Roots are `[]byte` types aliases by the `MerkleRoot` type. This type has one +method `Sum(sumTrie bool) uint64`. For the SMT this method **always** returns 0. + +The `MerkleRoot` type being an alias means it can be used in place of the +`[]byte` type. Specifically for proofs. + ## Proofs The `SparseMerkleProof` type contains the information required for inclusion From b8ada3a37683a1ac0dc6b161b7e7e645fccbccea Mon Sep 17 00:00:00 2001 From: h5law <53987565+h5law@users.noreply.github.com> Date: Tue, 9 Jan 2024 20:23:38 +0000 Subject: [PATCH 3/3] feat: panic on Sum() call and generalise for all hasher sizes --- docs/merkle-sum-trie.md | 10 +++--- docs/smt.md | 44 ++++------------------- root_test.go | 77 ++++++++++++++++++++++++++++++++--------- smst.go | 2 +- types.go | 17 ++++----- 5 files changed, 82 insertions(+), 68 deletions(-) diff --git a/docs/merkle-sum-trie.md b/docs/merkle-sum-trie.md index 226adab..1c3678d 100644 --- a/docs/merkle-sum-trie.md +++ b/docs/merkle-sum-trie.md @@ -251,12 +251,12 @@ sum as a `uint64`. ## Roots -Roots are `[]byte` types aliases by the `MerkleRoot` type. This type has one -method `Sum(sumTrie bool) uint64`. For the SMST this method is used by its own -`Sum()` method to return the total sum of the trie. +The root of the tree is a slice of bytes. `MerkleRoot` is an alias for `[]byte`. +This design enables easily passing around the data (e.g. on-chain) +while maintaining primitive usage in different use cases (e.g. proofs). -The `MerkleRoot` type being an alias means it can be used in place of the -`[]byte` type. Specifically for proofs. +`MerkleRoot` provides helpers, such as retrieving the `Sum(sumTrie bool)uint64` +to interface with data it captures. ## Nil Values diff --git a/docs/smt.md b/docs/smt.md index 598569d..14cc4d0 100644 --- a/docs/smt.md +++ b/docs/smt.md @@ -27,9 +27,6 @@ - [Database](#database) * [Database Submodules](#database-submodules) + [SimpleMap](#simplemap) - + [Badger](#badger) - * [Data Loss](#data-loss) -- [Sparse Merkle Sum Trie](#sparse-merkle-sum-trie) @@ -345,11 +342,13 @@ graph TD ## Roots -Roots are `[]byte` types aliases by the `MerkleRoot` type. This type has one -method `Sum(sumTrie bool) uint64`. For the SMT this method **always** returns 0. +The root of the tree is a slice of bytes. `MerkleRoot` is an alias for `[]byte`. +This design enables easily passing around the data (e.g. on-chain) +while maintaining primitive usage in different use cases (e.g. proofs). -The `MerkleRoot` type being an alias means it can be used in place of the -`[]byte` type. Specifically for proofs. +`MerkleRoot` provides helpers, such as retrieving the `Sum(sumTrie bool)uint64` +to interface with data it captures. However, for the SMT it **always** panics, +as there is no sum. ## Proofs @@ -478,33 +477,4 @@ the [`kvstore`](../kvstore/) directory. #### SimpleMap -This library defines the `SimpleMap` interface which is implemented as an -extremely simple in-memory key-value store. - -Although it is a submodule, it is ideal for simple, testing or non-production -use cases. It is used in the tests throughout the library. - -See [simplemap.go](../kvstore/simplemap/simplemap.go) for the implementation -details. - -#### Badger - -This library defines the `BadgerStore` interface which is implemented as a -wrapper around the [BadgerDB](https://github.com/dgraph-io/badger) v4 key-value -database. It's interface exposes numerous extra methods not used by the trie, -However it can still be used as a node-store with both in-memory and persistent -options. - -See [badger-store.md](./badger-store.md.md) for the details of the implementation. - -### Data Loss - -In the event of a system crash or unexpected failure of the program utilising -the SMT, if the `Commit()` function has not been called, any changes to the trie -will be lost. This is due to the underlying database not being changed **until** -the `Commit()` function is called and changes are persisted. - -## Sparse Merkle Sum Trie - -This library also implements a Sparse Merkle Sum Trie (SMST), the documentation -for which can be found [here](./merkle-sum-trie.md). +This l diff --git a/root_test.go b/root_test.go index cd6b34d..da6293c 100644 --- a/root_test.go +++ b/root_test.go @@ -2,8 +2,10 @@ package smt_test import ( "crypto/sha256" + "crypto/sha512" "encoding/binary" "fmt" + "hash" "testing" "github.com/stretchr/testify/require" @@ -12,28 +14,69 @@ import ( "github.com/pokt-network/smt/kvstore/simplemap" ) -func TestMerkleRoot_SumTrie(t *testing.T) { - nodeStore := simplemap.NewSimpleMap() - trie := smt.NewSparseMerkleSumTrie(nodeStore, sha256.New()) - for i := uint64(0); i < 10; i++ { - require.NoError(t, trie.Update([]byte(fmt.Sprintf("key%d", i)), []byte(fmt.Sprintf("value%d", i)), i)) +func TestMerkleRoot_TrieTypes(t *testing.T) { + tests := []struct { + desc string + sumTree bool + hasher hash.Hash + expectedPanic string + }{ + { + desc: "successfully: gets sum of sha256 hasher SMST", + sumTree: true, + hasher: sha256.New(), + expectedPanic: "", + }, + { + desc: "successfully: gets sum of sha512 hasher SMST", + sumTree: true, + hasher: sha512.New(), + expectedPanic: "", + }, + { + desc: "failure: panics for sha256 hasher SMT", + sumTree: false, + hasher: sha256.New(), + expectedPanic: "roo#sum: not a merkle sum trie", + }, + { + desc: "failure: panics for sha512 hasher SMT", + sumTree: false, + hasher: sha512.New(), + expectedPanic: "roo#sum: not a merkle sum trie", + }, } - root := trie.Root() - require.Equal(t, root.Sum(true), getSumBzHelper(t, root)) -} -func TestMerkleRoot_Trie(t *testing.T) { nodeStore := simplemap.NewSimpleMap() - trie := smt.NewSparseMerkleTrie(nodeStore, sha256.New()) - for i := 0; i < 10; i++ { - require.NoError(t, trie.Update([]byte(fmt.Sprintf("key%d", i)), []byte(fmt.Sprintf("value%d", i)))) + for _, tt := range tests { + tt := tt + t.Run(tt.desc, func(t *testing.T) { + t.Cleanup(func() { + require.NoError(t, nodeStore.ClearAll()) + }) + if tt.sumTree { + trie := smt.NewSparseMerkleSumTrie(nodeStore, tt.hasher) + for i := uint64(0); i < 10; i++ { + require.NoError(t, trie.Update([]byte(fmt.Sprintf("key%d", i)), []byte(fmt.Sprintf("value%d", i)), i)) + } + root := trie.Root() + require.Equal(t, root.Sum(), getSumBzHelper(t, root)) + return + } + trie := smt.NewSparseMerkleTrie(nodeStore, tt.hasher) + for i := 0; i < 10; i++ { + require.NoError(t, trie.Update([]byte(fmt.Sprintf("key%d", i)), []byte(fmt.Sprintf("value%d", i)))) + } + if panicStr := recover(); panicStr != nil { + require.Equal(t, tt.expectedPanic, panicStr) + } + }) } - root := trie.Root() - require.Equal(t, root.Sum(false), uint64(0)) } func getSumBzHelper(t *testing.T, r []byte) uint64 { - var sumbz [8]byte // Using sha256 - copy(sumbz[:], []byte(r)[len([]byte(r))-8:]) // Using sha256 so - 8 bytes - return binary.BigEndian.Uint64(sumbz[:]) + sumSize := len(r) % 32 + sumBz := make([]byte, sumSize) + copy(sumBz[:], []byte(r)[len([]byte(r))-sumSize:]) + return binary.BigEndian.Uint64(sumBz[:]) } diff --git a/smst.go b/smst.go index 2a6a4b6..1de9b14 100644 --- a/smst.go +++ b/smst.go @@ -119,5 +119,5 @@ func (smst *SMST) Root() MerkleRoot { // Sum returns the uint64 sum of the entire trie func (smst *SMST) Sum() uint64 { digest := smst.Root() - return digest.Sum(true) + return digest.Sum() } diff --git a/types.go b/types.go index 4439764..150d41f 100644 --- a/types.go +++ b/types.go @@ -18,15 +18,16 @@ var ( // MerkleRoot is a type alias for a byte slice returned from the Root method type MerkleRoot []byte -// Sum returns the uint64 sum of the merkle root, if sumTrie is true -// otherwise it returns 0, as it would result in undefined behaviour. -func (r MerkleRoot) Sum(sumTrie bool) uint64 { - if sumTrie { - var sumbz [sumSize]byte - copy(sumbz[:], []byte(r)[len([]byte(r))-sumSize:]) - return binary.BigEndian.Uint64(sumbz[:]) +// Sum returns the uint64 sum of the merkle root, it checks the length of the +// merkle root and if it is no the same as the size of the SMST's expected +// root hash it will panic. +func (r MerkleRoot) Sum() uint64 { + if len(r)%32 == 0 { + panic("roo#sum: not a merkle sum trie") } - return 0 + var sumbz [sumSize]byte + copy(sumbz[:], []byte(r)[len([]byte(r))-sumSize:]) + return binary.BigEndian.Uint64(sumbz[:]) } // SparseMerkleTrie represents a Sparse Merkle Trie.