Implement get-compact-range using RFC 6962 methods

This is a proof of concept change demonstrating that it is possible to
obtain arbitrary compact ranges from a Merkle tree log that restricts
itself only to endpoints represented in RFC 6962, in constant time
interaction complexity.

Specifically, it is possible to obtain comact range [begin, end) by
calling "get consistency proof" endpoints <= 2 times for carefully
crafted tree sizes. In a few cases where it is impossible to get certain
hashes, this approach falls back to calling the "get entries" endpoint 1
time to obtain between 1-3 entries and reconstruct the compact range.

Overall, the interaction with the log is limited by 2 calls, and each
call is limited in size.

exp/README.md

@@ -0,0 +1,7 @@
+Experimental
+------------
+
+This directory contains a Go module with experimental features not included into
+the main Go module of this repository. These must be used with caution.
+
+The idea of this module is similar to Go's https://pkg.go.dev/golang.org/x/exp.

exp/get_compact_range.go

@@ -0,0 +1,139 @@
+package merkle
+
+import (
+	"fmt"
+
+	"github.com/transparency-dev/merkle/compact"
+	"github.com/transparency-dev/merkle/proof"
+)
+
+type HashGetter interface {
+	GetConsistencyProof(first, second uint64) ([][]byte, error)
+	GetLeafHashes(begin, end uint64) ([][]byte, error)
+}
+
+func GetCompactRange(rf *compact.RangeFactory, begin, end, size uint64, hg HashGetter) (*compact.Range, error) {
+	if begin > size || end > size {
+		return nil, fmt.Errorf("[%d, %d) out of range in %d", begin, end, size)
+	}
+	if begin >= end {
+		return rf.NewEmptyRange(begin), nil
+	}
+
+	if size <= 3 || end == 1 {
+		hashes, err := hg.GetLeafHashes(begin, end)
+		if err != nil {
+			return nil, fmt.Errorf("GetLeafHashes(%d, %d): %v", begin, end, err)
+		}
+		if got, want := uint64(len(hashes)), end-begin; got != want {
+			return nil, fmt.Errorf("GetLeafHashes(%d, %d): %d hashes, want %d", begin, end, got, want)
+		}
+		r := rf.NewEmptyRange(begin)
+		for _, h := range hashes {
+			if err := r.Append(h, nil); err != nil {
+				return nil, fmt.Errorf("Append: %v", err)
+			}
+		}
+		return r, nil
+	}
+	// size >= 4 && end >= 2
+
+	known := make(map[compact.NodeID][]byte)
+
+	store := func(nodes proof.Nodes, hashes [][]byte) error {
+		_, b, e := nodes.Ephem()
+		wantSize := len(nodes.IDs) - (e - b)
+		if b != e {
+			wantSize++
+		}
+		if got := len(hashes); got != wantSize {
+			return fmt.Errorf("proof size mismatch: got %d, want %d", got, wantSize)
+		}
+
+		idx := 0
+		for _, hash := range hashes {
+			if idx == b && b+1 < e {
+				idx = e - 1
+				continue
+			}
+			known[nodes.IDs[idx]] = hash
+			idx++
+		}
+		return nil
+	}
+
+	newRange := func(begin, end uint64) (*compact.Range, error) {
+		size := compact.RangeSize(begin, end)
+		ids := compact.RangeNodes(begin, end, make([]compact.NodeID, 0, size))
+		hashes := make([][]byte, 0, len(ids))
+		for _, id := range ids {
+			if hash, ok := known[id]; ok {
+				hashes = append(hashes, hash)
+			} else {
+				return nil, fmt.Errorf("hash not known: %+v", id)
+			}
+		}
+		return rf.NewRange(begin, end, hashes)
+	}
+
+	fetch := func(first, second uint64) error {
+		nodes, err := proof.Consistency(first, second)
+		if err != nil {
+			return fmt.Errorf("proof.Consistency: %v", err)
+		}
+		hashes, err := hg.GetConsistencyProof(first, second)
+		if err != nil {
+			return fmt.Errorf("GetConsistencyProof(%d, %d): %v", first, second, err)
+		}
+		store(nodes, hashes)
+		return nil
+	}
+
+	mid, _ := compact.Decompose(begin, end)
+	mid += begin
+	if err := fetch(begin, mid); err != nil {
+		return nil, err
+	}
+
+	if begin == 0 && end == 2 || end == 3 {
+		if err := fetch(3, 4); err != nil {
+			return nil, err
+		}
+	}
+	if end <= 3 {
+		return newRange(begin, end)
+	}
+	// end >= 4
+
+	if (end-1)&(end-2) != 0 { // end-1 is not a power of 2.
+		if err := fetch(end-1, end); err != nil {
+			return nil, err
+		}
+		r, err := newRange(begin, end-1)
+		if err != nil {
+			return nil, err
+		}
+		if err := r.Append(known[compact.NewNodeID(0, end-1)], nil); err != nil {
+			return nil, fmt.Errorf("Append: %v", err)
+		}
+		return r, nil
+	}
+
+	// At this point: end >= 4, end-1 is a power of 2; thus, end-2 is not a power of 2.
+	if err := fetch(end-2, end); err != nil {
+		return nil, err
+	}
+	r := rf.NewEmptyRange(begin)
+	if end-2 > begin {
+		var err error
+		if r, err = newRange(begin, end-2); err != nil {
+			return nil, err
+		}
+	}
+	for index := r.End(); index < end; index++ {
+		if err := r.Append(known[compact.NewNodeID(0, index)], nil); err != nil {
+			return nil, fmt.Errorf("Append: %v", err)
+		}
+	}
+	return r, nil
+}

exp/get_compact_range_test.go

@@ -0,0 +1,116 @@
+package merkle_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/transparency-dev/merkle"
+	"github.com/transparency-dev/merkle/compact"
+	"github.com/transparency-dev/merkle/proof"
+)
+
+func TestGetCompactRange(t *testing.T) {
+	rf := compact.RangeFactory{Hash: func(left, right []byte) []byte {
+		return append(append(make([]byte, 0, len(left)+len(right)), left...), right...)
+	}}
+	tr := newTree(t, 256, &rf)
+
+	test := func(begin, end, size uint64) {
+		t.Run(fmt.Sprintf("%d:%d_%d", size, begin, end), func(t *testing.T) {
+			got, err := merkle.GetCompactRange(&rf, begin, end, size, tr)
+			if err != nil {
+				t.Fatalf("GetCompactRange: %v", err)
+			}
+			want, err := tr.getCompactRange(begin, end)
+			if err != nil {
+				t.Fatalf("GetCompactRange: %v", err)
+			}
+			if diff := cmp.Diff(got, want); diff != "" {
+				t.Fatalf("Diff: %s", diff)
+			}
+		})
+	}
+
+	for begin := uint64(0); begin <= tr.size; begin++ {
+		for end := begin; end <= tr.size; end++ {
+			for size := end; size < end+5 && size < tr.size; size++ {
+				test(begin, end, size)
+			}
+			test(begin, end, tr.size)
+		}
+	}
+}
+
+type tree struct {
+	rf    *compact.RangeFactory
+	size  uint64
+	nodes map[compact.NodeID][]byte
+}
+
+func newTree(t *testing.T, size uint64, rf *compact.RangeFactory) *tree {
+	hash := func(leaf uint64) []byte {
+		if leaf >= 256 {
+			t.Fatalf("leaf %d not supported in this test", leaf)
+		}
+		return []byte{byte(leaf)}
+	}
+
+	nodes := make(map[compact.NodeID][]byte, size*2-1)
+	r := rf.NewEmptyRange(0)
+	for i := uint64(0); i < size; i++ {
+		nodes[compact.NewNodeID(0, i)] = hash(i)
+		if err := r.Append(hash(i), func(id compact.NodeID, hash []byte) {
+			nodes[id] = hash
+		}); err != nil {
+			t.Fatalf("Append: %v", err)
+		}
+	}
+	return &tree{rf: rf, size: size, nodes: nodes}
+}
+
+func (t *tree) GetConsistencyProof(first, second uint64) ([][]byte, error) {
+	if first > t.size || second > t.size {
+		return nil, fmt.Errorf("%d or %d is beyond %d", first, second, t.size)
+	}
+	nodes, err := proof.Consistency(first, second)
+	if err != nil {
+		return nil, err
+	}
+	hashes, err := t.getNodes(nodes.IDs)
+	if err != nil {
+		return nil, err
+	}
+	return nodes.Rehash(hashes, t.rf.Hash)
+}
+
+func (t *tree) GetLeafHashes(begin, end uint64) ([][]byte, error) {
+	if begin >= end {
+		return nil, nil
+	}
+	ids := make([]compact.NodeID, 0, end-begin)
+	for i := begin; i < end; i++ {
+		ids = append(ids, compact.NewNodeID(0, i))
+	}
+	return t.getNodes(ids)
+}
+
+func (t *tree) getCompactRange(begin, end uint64) (*compact.Range, error) {
+	hashes, err := t.getNodes(compact.RangeNodes(begin, end))
+	if err != nil {
+		return nil, err
+	}
+	return t.rf.NewRange(begin, end, hashes)
+}
+
+func (t *tree) getNodes(ids []compact.NodeID) ([][]byte, error) {
+	hashes := make([][]byte, len(ids))
+	for i, id := range ids {
+		if hash, ok := t.nodes[id]; ok {
+			hashes[i] = hash
+		} else {
+			return nil, fmt.Errorf("node %+v not found", id)
+		}
+	}
+	return hashes, nil
+}

exp/go.mod

@@ -0,0 +1,5 @@
+module github.com/transparency-dev/merkle/exp
+
+go 1.16
+
+require github.com/transparency-dev/merkle v0.0.0-20220425113829-c120179f55ad // indirect

exp/go.sum

@@ -0,0 +1,4 @@
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/transparency-dev/merkle v0.0.0-20220425113829-c120179f55ad h1:82yvTO+VijfWulMsMQvqQSZ0zNEAgmEUeBG+ArrO9Js=
+github.com/transparency-dev/merkle v0.0.0-20220425113829-c120179f55ad/go.mod h1:B8FIw5LTq6DaULoHsVFRzYIUDkl8yuSwCdZnOZGKL/A=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=