diff --git a/api/docgen/exampledata/blobProof.json b/api/docgen/exampledata/blobProof.json index 1ee87af9e3..d23f4b6aa0 100644 --- a/api/docgen/exampledata/blobProof.json +++ b/api/docgen/exampledata/blobProof.json @@ -1,31 +1,53 @@ -[ - { - "end": 8, - "nodes": [ - "/////////////////////////////////////////////////////////////////////////////wuxStDHcZ7+b5byNQMVLJbzBT3wmObsThoQ0sCTjTCP" +{ + "ShareToRowRootProof": [ + { + "start": 3, + "end": 4, + "nodes": [ + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ72eTVOUxB9THxFjAEwtTePJQA1b0xcz2f6TJc400Uw", + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBD0CYbGYoGN4q9VfSmeGZeg/h1NDBA/jtXjZrrKRHE6", + "/////////////////////////////////////////////////////////////////////////////8KDE4JDf0N2lZB7DW1Fpasdk/wz4jHOxuBPAk5Vf5ZI" + ] + }, + { + "end": 1, + "nodes": [ + "//////////////////////////////////////7//////////////////////////////////////plEqgR/c4IAVkNdYRWOYOAESD4whneKR54Dz5Dfe4p2", + "//////////////////////////////////////7//////////////////////////////////////lrD0qJ9dspxSO1Yl8NDioZfgOm8Yj63Y+BGDRHlKCRj", + "/////////////////////////////////////////////////////////////////////////////xQyI+g89aM6rhy9rl2eKr0Uc2NPauf3fkLY3Z+gBtuM" + ] + } + ], + "RowProof": { + "row_roots": [ + "00000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000808080808080808BC517066A5A8C81E2A4353DB500EBB3410047A93D2EE8ADF0B6797B9A5519557", + "0000000000000000000000000000000000000000000808080808080808FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE015AB6AAC6FAF0ABF26F9453AF390FDA3B39EB384F0B71D0170D84CF69CBA2BC" ], - "is_max_namespace_ignored": true - }, - { - "end": 8, - "nodes": [ - "//////////////////////////////////////////////////////////////////////////////n1NeJxPU2bZUAccKZZ+LAu2Wj5ajbVYURV9ojhSKwp" + "proofs": [ + { + "total": 16, + "index": 1, + "leaf_hash": "lJek/BHnKH6PyRB8jlk69F6EY9Tfx2LRanaF74JVciU=", + "aunts": [ + "bLjvftajE6jVsgQQBkV4RUPESRc+v4bhP0Ljf36858Q=", + "QaF9mNskaURxk98S3BExB1PzRAjOqVydrDLvUu0B5/M=", + "K2xW8JJ3Ff4FvtbfZi5ZD/ygnswaNCNIKXsSzbO2Jrc=", + "uySRG/gINLAgGgywJCTiXMlFkfQivF1O1zLg5+RRUP8=" + ] + }, + { + "total": 16, + "index": 2, + "leaf_hash": "h+4ND52kT4qkc9nWW22dIMAK/4YjkC6fBoD01WF0+Uo=", + "aunts": [ + "2x8OISRBMLYJRV8NfTNtVvZUg2F7MtCK5xCZuE9fQwQ=", + "Xvr5IalE2y3pxHjxh5kcHFSRaz4g5MxdOj4NIGwRXY0=", + "K2xW8JJ3Ff4FvtbfZi5ZD/ygnswaNCNIKXsSzbO2Jrc=", + "uySRG/gINLAgGgywJCTiXMlFkfQivF1O1zLg5+RRUP8=" + ] + } ], - "is_max_namespace_ignored": true - }, - { - "end": 8, - "nodes": [ - "/////////////////////////////////////////////////////////////////////////////0xK8BKnzDmwK0HR4ZJvyB4kh3jPPXGxaGPFoga8vPxF" - ], - "is_max_namespace_ignored": true - }, - { - "end": 7, - "nodes": [ - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAMJ/xGlNMdEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwn/EaU0x0UTO9HUGKjyjcv5U2gHeSjJ8S1rftqv6k8kxlVWW8e/7", - "/////////////////////////////////////////////////////////////////////////////wexh4khLQ9HQ2X6nh9wU5B+m6r+LWwPTEDTa5/CosDF" - ], - "is_max_namespace_ignored": true + "start_row": 1, + "end_row": 3 } -] +} diff --git a/blob/blob.go b/blob/blob.go index 7fca83859d..a21c53c2f1 100644 --- a/blob/blob.go +++ b/blob/blob.go @@ -6,7 +6,10 @@ import ( "errors" "fmt" + "github.com/tendermint/tendermint/crypto/merkle" + "github.com/tendermint/tendermint/pkg/consts" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + coretypes "github.com/tendermint/tendermint/types" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/shares" @@ -16,42 +19,73 @@ import ( "github.com/celestiaorg/celestia-node/share" ) +//nolint:unused var errEmptyShares = errors.New("empty shares") -// The Proof is a set of nmt proofs that can be verified only through -// the included method (due to limitation of the nmt https://github.com/celestiaorg/nmt/issues/218). -// Proof proves the WHOLE namespaced data to the row roots. -// TODO (@vgonkivs): rework `Proof` in order to prove a particular blob. -// https://github.com/celestiaorg/celestia-node/issues/2303 -type Proof []*nmt.Proof - -func (p Proof) Len() int { return len(p) } +// Proof constructs the proof of a blob to the data root. +type Proof struct { + // ShareToRowRootProof the proofs of the shares to the row roots they belong to. + // If the blob spans across multiple rows, then this will contain multiple proofs. + ShareToRowRootProof []*tmproto.NMTProof + // RowToDataRootProof the proofs of the row roots containing the blob shares + // to the data root. + RowToDataRootProof coretypes.RowProof +} -// equal is a temporary method that compares two proofs. -// should be removed in BlobService V1. -func (p Proof) equal(input Proof) error { - if p.Len() != input.Len() { - return ErrInvalidProof +// Verify takes a blob and a data root and verifies if the +// provided blob was committed to the given data root. +func (p *Proof) Verify(blob *Blob, dataRoot []byte) (bool, error) { + blobCommitment, err := types.CreateCommitment(ToAppBlobs(blob)[0]) + if err != nil { + return false, err + } + if !blob.Commitment.Equal(blobCommitment) { + return false, fmt.Errorf( + "%w: generated commitment does not match the provided blob commitment", + ErrMismatchCommitment, + ) } + rawShares, err := BlobsToShares(blob) + if err != nil { + return false, err + } + return p.VerifyShares(rawShares, blob.namespace, dataRoot) +} - for i, proof := range p { - pNodes := proof.Nodes() - inputNodes := input[i].Nodes() - for i, node := range pNodes { - if !bytes.Equal(node, inputNodes[i]) { - return ErrInvalidProof - } - } +// VerifyShares takes a set of shares, a namespace and a data root, and verifies if the +// provided shares are committed to by the data root. +func (p *Proof) VerifyShares(rawShares [][]byte, namespace share.Namespace, dataRoot []byte) (bool, error) { + // verify the row proof + if err := p.RowToDataRootProof.Validate(dataRoot); err != nil { + return false, fmt.Errorf("%w: invalid row root to data root proof", err) + } - if proof.Start() != input[i].Start() || proof.End() != input[i].End() { - return ErrInvalidProof + // verify the share proof + ns := append([]byte{namespace.Version()}, namespace.ID()...) + cursor := int32(0) + for i, proof := range p.ShareToRowRootProof { + sharesUsed := proof.End - proof.Start + if len(rawShares) < int(sharesUsed+cursor) { + return false, fmt.Errorf("%w: invalid number of shares", ErrInvalidProof) } - - if !bytes.Equal(proof.LeafHash(), input[i].LeafHash()) { - return ErrInvalidProof + nmtProof := nmt.NewInclusionProof( + int(proof.Start), + int(proof.End), + proof.Nodes, + true, + ) + valid := nmtProof.VerifyInclusion( + consts.NewBaseHashFunc(), + ns, + rawShares[cursor:sharesUsed+cursor], + p.RowToDataRootProof.RowRoots[i], + ) + if !valid { + return false, ErrInvalidProof } + cursor += sharesUsed } - return nil + return true, nil } // Blob represents any application-specific binary data that anyone can submit to Celestia. @@ -172,3 +206,10 @@ func (b *Blob) UnmarshalJSON(data []byte) error { b.index = blob.Index return nil } + +// proveRowRootsToDataRoot creates a set of binary merkle proofs for all the +// roots defined by the range [start, end). +func proveRowRootsToDataRoot(roots [][]byte, start, end int) []*merkle.Proof { + _, proofs := merkle.ProofsFromByteSlices(roots) + return proofs[start:end] +} diff --git a/blob/helper.go b/blob/helper.go index c1de773c24..615d5fbc08 100644 --- a/blob/helper.go +++ b/blob/helper.go @@ -39,7 +39,7 @@ func BlobsToShares(blobs ...*Blob) ([]share.Share, error) { // ToAppBlobs converts node's blob type to the blob type from celestia-app. func ToAppBlobs(blobs ...*Blob) []*apptypes.Blob { - appBlobs := make([]*apptypes.Blob, 0, len(blobs)) + appBlobs := make([]*apptypes.Blob, len(blobs)) for i := range blobs { appBlobs[i] = &blobs[i].Blob } diff --git a/blob/parser.go b/blob/parser.go index 1b5ddc4a7a..c0aec5a5aa 100644 --- a/blob/parser.go +++ b/blob/parser.go @@ -21,6 +21,8 @@ type parser struct { // set tries to find the first blob's share by skipping padding shares and // sets the metadata of the blob(index and length) +// +//nolint:unused func (p *parser) set(index int, shrs []shares.Share) ([]shares.Share, error) { if len(shrs) == 0 { return nil, errEmptyShares @@ -113,6 +115,8 @@ func (p *parser) parse() (*Blob, error) { // skipPadding iterates through the shares until non-padding share will be found. It guarantees that // the returned set of shares will start with non-padding share(or empty set of shares). +// +//nolint:unused func (p *parser) skipPadding(shares []shares.Share) ([]shares.Share, error) { if len(shares) == 0 { return nil, errEmptyShares @@ -144,6 +148,7 @@ func (p *parser) verify(blob *Blob) bool { return p.verifyFn(blob) } +//nolint:unused func (p *parser) isEmpty() bool { return p.index == 0 && p.length == 0 && len(p.shares) == 0 } diff --git a/blob/service.go b/blob/service.go index b1c394f075..3ed4400a4a 100644 --- a/blob/service.go +++ b/blob/service.go @@ -11,6 +11,8 @@ import ( "github.com/cosmos/cosmos-sdk/types" logging "github.com/ipfs/go-log/v2" + "github.com/tendermint/tendermint/libs/bytes" + core "github.com/tendermint/tendermint/types" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" @@ -30,8 +32,9 @@ import ( ) var ( - ErrBlobNotFound = errors.New("blob: not found") - ErrInvalidProof = errors.New("blob: invalid proof") + ErrBlobNotFound = errors.New("blob: not found") + ErrInvalidProof = errors.New("blob: invalid proof") + ErrMismatchCommitment = errors.New("blob: mismatched commitment") log = logging.Logger("blob") tracer = otel.Tracer("blob/service") @@ -315,7 +318,7 @@ func (s *Service) Included( sharesParser := &parser{verifyFn: func(blob *Blob) bool { return blob.compareCommitments(commitment) }} - _, resProof, err := s.retrieve(ctx, height, namespace, sharesParser) + blob, _, err := s.retrieve(ctx, height, namespace, sharesParser) switch { case err == nil: case errors.Is(err, ErrBlobNotFound): @@ -323,7 +326,11 @@ func (s *Service) Included( default: return false, err } - return true, resProof.equal(*proof) + header, err := s.headerGetter(ctx, height) + if err != nil { + return false, err + } + return proof.Verify(blob, header.DataHash) } // retrieve retrieves blobs and their proofs by requesting the whole namespace and @@ -347,127 +354,194 @@ func (s *Service) retrieve( return nil, nil, err } + eds, err := s.shareGetter.GetEDS(ctx, header) + if err != nil { + return nil, nil, err + } headerGetterSpan.SetStatus(codes.Ok, "") headerGetterSpan.AddEvent("received eds", trace.WithAttributes( attribute.Int64("eds-size", int64(len(header.DAH.RowRoots))))) - rowIndex := -1 + // find the index of the row where the blob could start + inclusiveNamespaceStartRowIndex := -1 for i, row := range header.DAH.RowRoots { if !namespace.IsOutsideRange(row, row) { - rowIndex = i + inclusiveNamespaceStartRowIndex = i break } } + if inclusiveNamespaceStartRowIndex == -1 { + return nil, nil, ErrBlobNotFound + } - getCtx, getSharesSpan := tracer.Start(ctx, "get-shares-by-namespace") + // end exclusive index of the row root containing the namespace + exclusiveNamespaceEndRowIndex := inclusiveNamespaceStartRowIndex + for _, row := range header.DAH.RowRoots[inclusiveNamespaceStartRowIndex:] { + if namespace.IsOutsideRange(row, row) { + break + } + exclusiveNamespaceEndRowIndex++ + } + if exclusiveNamespaceEndRowIndex == inclusiveNamespaceStartRowIndex { + return nil, nil, fmt.Errorf("couldn't find the row index of the namespace end") + } - // collect shares for the requested namespace - namespacedShares, err := s.shareGetter.GetSharesByNamespace(getCtx, header, namespace) - if err != nil { - if errors.Is(err, share.ErrNotFound) { - err = ErrBlobNotFound + // calculate the square size + squareSize := len(header.DAH.RowRoots) / 2 + + // get all the shares of the rows containing the namespace + _, getSharesSpan := tracer.Start(ctx, "get-all-shares-in-namespace") + // store the ODS shares of the rows containing the blob + odsShares := make([]share.Share, 0, (exclusiveNamespaceEndRowIndex-inclusiveNamespaceStartRowIndex)*squareSize) + // store the EDS shares of the rows containing the blob + edsShares := make([][]shares.Share, exclusiveNamespaceEndRowIndex-inclusiveNamespaceStartRowIndex) + + for rowIndex := inclusiveNamespaceStartRowIndex; rowIndex < exclusiveNamespaceEndRowIndex; rowIndex++ { + rowShares := eds.Row(uint(rowIndex)) + odsShares = append(odsShares, rowShares[:squareSize]...) + rowAppShares, err := toAppShares(rowShares...) + if err != nil { + return nil, nil, err } - getSharesSpan.SetStatus(codes.Error, err.Error()) - return nil, nil, err + edsShares[rowIndex-inclusiveNamespaceStartRowIndex] = rowAppShares } getSharesSpan.SetStatus(codes.Ok, "") getSharesSpan.AddEvent("received shares", trace.WithAttributes( - attribute.Int64("eds-size", int64(len(header.DAH.RowRoots))))) - - var ( - appShares = make([]shares.Share, 0) - proofs = make(Proof, 0) - ) + attribute.Int64("eds-size", int64(squareSize*2)))) - for _, row := range namespacedShares { - if len(row.Shares) == 0 { - // the above condition means that we've faced with an Absence Proof. - // This Proof proves that the namespace was not found in the DAH, so - // we can return `ErrBlobNotFound`. - return nil, nil, ErrBlobNotFound + // go over the shares until finding the requested blobs + for currentShareIndex := 0; currentShareIndex < len(odsShares); { + currentShareApp, err := shares.NewShare(odsShares[currentShareIndex]) + if err != nil { + return nil, nil, err } - appShares, err = toAppShares(row.Shares...) + // skip if it's a padding share + isPadding, err := currentShareApp.IsPadding() if err != nil { return nil, nil, err } + if isPadding { + currentShareIndex++ + continue + } + isCompactShare, err := currentShareApp.IsCompactShare() + if err != nil { + return nil, nil, err + } + if isCompactShare { + currentShareIndex++ + continue + } + isSequenceStart, err := currentShareApp.IsSequenceStart() + if err != nil { + return nil, nil, err + } + if isSequenceStart { + // calculate the blob length + sequenceLength, err := currentShareApp.SequenceLen() + if err != nil { + return nil, nil, err + } + blobLen := shares.SparseSharesNeeded(sequenceLength) - proofs = append(proofs, row.Proof) - index := row.Proof.Start() - - for { - var ( - isComplete bool - shrs []shares.Share - wasEmpty = sharesParser.isEmpty() - ) - - if wasEmpty { - // create a parser if it is empty - shrs, err = sharesParser.set(rowIndex*len(header.DAH.RowRoots)+index, appShares) - if err != nil { - if errors.Is(err, errEmptyShares) { - // reset parser as `skipPadding` can update next blob's index - sharesParser.reset() - appShares = nil - break - } - return nil, nil, err - } - - // update index and shares if padding shares were detected. - if len(appShares) != len(shrs) { - index += len(appShares) - len(shrs) - appShares = shrs - } + exclusiveEndShareIndex := currentShareIndex + blobLen + if exclusiveEndShareIndex > len(odsShares) { + // this blob spans to the next row which has a namespace > requested namespace. + // this means that we can stop. + return nil, nil, ErrBlobNotFound + } + // convert the blob shares to app shares + blobShares := odsShares[currentShareIndex:exclusiveEndShareIndex] + appBlobShares, err := toAppShares(blobShares...) + if err != nil { + return nil, nil, err } - shrs, isComplete = sharesParser.addShares(appShares) - // move to the next row if the blob is incomplete + // parse the blob + sharesParser.length = blobLen + _, isComplete := sharesParser.addShares(appBlobShares) if !isComplete { - appShares = nil - break + return nil, nil, fmt.Errorf("expected the shares to construct a full blob") } - // otherwise construct blob blob, err := sharesParser.parse() if err != nil { return nil, nil, err } - if sharesParser.verify(blob) { - return blob, &proofs, nil - } - - index += len(appShares) - len(shrs) - appShares = shrs - sharesParser.reset() + // number of shares per EDS row + numberOfSharesPerEDSRow := squareSize * 2 + // number of shares from square start to namespace start + sharesFromSquareStartToNsStart := inclusiveNamespaceStartRowIndex * numberOfSharesPerEDSRow + // number of rows from namespace start row to current row + rowsFromNsStartToCurrentRow := currentShareIndex / squareSize + // number of shares from namespace row start to current row + sharesFromNsStartToCurrentRow := rowsFromNsStartToCurrentRow * numberOfSharesPerEDSRow + // number of shares from the beginning of current row to current share + sharesFromCurrentRowStart := currentShareIndex % squareSize + // setting the index manually since we didn't use the parser.set() method + blob.index = sharesFromSquareStartToNsStart + + sharesFromNsStartToCurrentRow + + sharesFromCurrentRowStart + + if blob.Namespace().Equals(namespace) && sharesParser.verify(blob) { + // now that we found the requested blob, we will create + // its inclusion proof. + inclusiveBlobStartRowIndex := blob.index / (squareSize * 2) + exclusiveBlobEndRowIndex := inclusiveNamespaceStartRowIndex + exclusiveEndShareIndex/squareSize + if (currentShareIndex+blobLen)%squareSize != 0 { + // if the row is not complete with the blob shares, + // then we increment the exclusive blob end row index + // so that it's exclusive. + exclusiveBlobEndRowIndex++ + } - if !wasEmpty { - // remove proofs for prev rows if verified blob spans multiple rows - proofs = proofs[len(proofs)-1:] - } - } + // create the row roots to data root inclusion proof + rowProofs := proveRowRootsToDataRoot( + append(header.DAH.RowRoots, header.DAH.ColumnRoots...), + inclusiveBlobStartRowIndex, + exclusiveBlobEndRowIndex, + ) + rowRoots := make([]bytes.HexBytes, exclusiveBlobEndRowIndex-inclusiveBlobStartRowIndex) + for index, rowRoot := range header.DAH.RowRoots[inclusiveBlobStartRowIndex:exclusiveBlobEndRowIndex] { + rowRoots[index] = rowRoot + } - rowIndex++ - if sharesParser.isEmpty() { - proofs = nil - } - } + edsShareStart := inclusiveBlobStartRowIndex - inclusiveNamespaceStartRowIndex + edsShareEnd := exclusiveBlobEndRowIndex - inclusiveNamespaceStartRowIndex + // create the share to row root proofs + shareToRowRootProofs, _, err := pkgproof.CreateShareToRowRootProofs( + squareSize, + edsShares[edsShareStart:edsShareEnd], + header.DAH.RowRoots[inclusiveBlobStartRowIndex:exclusiveBlobEndRowIndex], + currentShareIndex%squareSize, + (exclusiveEndShareIndex-1)%squareSize, + ) + if err != nil { + return nil, nil, err + } - err = ErrBlobNotFound - for _, sh := range appShares { - ok, err := sh.IsPadding() - if err != nil { - return nil, nil, err - } - if !ok { - err = fmt.Errorf("incomplete blob with the "+ - "namespace: %s detected at %d: %w", namespace.String(), height, err) - log.Error(err) + proof := Proof{ + ShareToRowRootProof: shareToRowRootProofs, + RowToDataRootProof: core.RowProof{ + RowRoots: rowRoots, + Proofs: rowProofs, + StartRow: uint32(inclusiveBlobStartRowIndex), + EndRow: uint32(exclusiveBlobEndRowIndex) - 1, + }, + } + return blob, &proof, nil + } + sharesParser.reset() + currentShareIndex += blobLen + } else { + // this is a continuation of a previous blob + // we can skip + currentShareIndex++ } } - return nil, nil, err + return nil, nil, ErrBlobNotFound } // getBlobs retrieves the DAH and fetches all shares from the requested Namespace and converts diff --git a/blob/service_test.go b/blob/service_test.go index 101c755358..a8de4817bc 100644 --- a/blob/service_test.go +++ b/blob/service_test.go @@ -17,11 +17,14 @@ import ( "github.com/stretchr/testify/require" "github.com/tendermint/tendermint/crypto/merkle" tmrand "github.com/tendermint/tendermint/libs/rand" + coretypes "github.com/tendermint/tendermint/types" "github.com/celestiaorg/celestia-app/pkg/appconsts" appns "github.com/celestiaorg/celestia-app/pkg/namespace" pkgproof "github.com/celestiaorg/celestia-app/pkg/proof" "github.com/celestiaorg/celestia-app/pkg/shares" + "github.com/celestiaorg/celestia-app/pkg/square" + "github.com/celestiaorg/celestia-app/test/util/testfactory" blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/go-header/store" "github.com/celestiaorg/nmt" @@ -227,24 +230,13 @@ func TestBlobService_Get(t *testing.T) { proof, ok := res.(*Proof) assert.True(t, ok) - verifyFn := func(t *testing.T, rawShares [][]byte, proof *Proof, namespace share.Namespace) { - for _, row := range header.DAH.RowRoots { - to := 0 - for _, p := range *proof { - from := to - to = p.End() - p.Start() + from - eq := p.VerifyInclusion(share.NewSHA256Hasher(), namespace.ToNMT(), rawShares[from:to], row) - if eq == true { - return - } - } - } - t.Fatal("could not prove the shares") + verifyFn := func(t *testing.T, blob *Blob, proof *Proof) { + valid, err := proof.Verify(blob, header.DataHash) + require.NoError(t, err) + require.True(t, valid) } - rawShares, err := BlobsToShares(blobsWithDiffNamespaces[1]) - require.NoError(t, err) - verifyFn(t, rawShares, proof, blobsWithDiffNamespaces[1].Namespace()) + verifyFn(t, blobsWithDiffNamespaces[1], proof) }, }, { @@ -287,7 +279,7 @@ func TestBlobService_Get(t *testing.T) { require.ErrorIs(t, err, ErrInvalidProof) included, ok := res.(bool) require.True(t, ok) - require.True(t, included) + require.False(t, included) }, }, { @@ -342,7 +334,7 @@ func TestBlobService_Get(t *testing.T) { originalDataWidth := len(h.DAH.RowRoots) / 2 sizes := []int{blobSize0, blobSize1} for i, proof := range proofs { - require.True(t, sizes[i]/originalDataWidth+1 == proof.Len()) + require.True(t, sizes[i]/originalDataWidth+1 == len(proof.ShareToRowRootProof)) } }, }, @@ -377,30 +369,23 @@ func TestBlobService_Get(t *testing.T) { var proof Proof require.NoError(t, json.Unmarshal(jsonData, &proof)) - newProof, err := service.GetProof(ctx, 1, - blobsWithDiffNamespaces[1].Namespace(), - blobsWithDiffNamespaces[1].Commitment, - ) + header, err := service.headerGetter(ctx, 1) + require.NoError(t, err) + valid, err := proof.Verify(blobsWithDiffNamespaces[1], header.DataHash) require.NoError(t, err) - require.NoError(t, proof.equal(*newProof)) + require.True(t, valid) }, }, { name: "internal error", doFn: func() (interface{}, error) { ctrl := gomock.NewController(t) - shareService := service.shareGetter shareGetterMock := shareMock.NewMockModule(ctrl) shareGetterMock.EXPECT(). - GetSharesByNamespace(gomock.Any(), gomock.Any(), gomock.Any()). + GetEDS(gomock.Any(), gomock.Any()). DoAndReturn( - func( - ctx context.Context, h *header.ExtendedHeader, ns share.Namespace, - ) (share.NamespacedShares, error) { - if ns.Equals(blobsWithDiffNamespaces[0].Namespace()) { - return nil, errors.New("internal error") - } - return shareService.GetSharesByNamespace(ctx, h, ns) + func(context.Context, *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { + return nil, errors.New("internal error") }).AnyTimes() service.shareGetter = shareGetterMock @@ -412,13 +397,8 @@ func TestBlobService_Get(t *testing.T) { ) }, expectedResult: func(res interface{}, err error) { - blobs, ok := res.([]*Blob) - assert.True(t, ok) assert.Error(t, err) assert.Contains(t, err.Error(), "internal error") - assert.Equal(t, blobs[0].Namespace(), blobsWithSameNamespace[0].Namespace()) - assert.NotEmpty(t, blobs) - assert.Len(t, blobs, len(blobsWithSameNamespace)) }, }, } @@ -861,19 +841,38 @@ func BenchmarkGetByCommitment(b *testing.B) { } } -func createServiceWithSub(ctx context.Context, t testing.TB, blobs []*Blob) *Service { +func createServiceWithSub(ctx context.Context, t *testing.T, blobs []*Blob) *Service { + acc := "test" + kr := testfactory.GenerateKeyring(acc) + signer := blobtypes.NewKeyringSigner(kr, acc, "test") + addr, err := signer.GetSignerInfo().GetAddress() + require.NoError(t, err) + bs := ipld.NewMemBlockservice() batching := ds_sync.MutexWrap(ds.NewMapDatastore()) headerStore, err := store.NewStore[*header.ExtendedHeader](batching) require.NoError(t, err) edsses := make([]*rsmt2d.ExtendedDataSquare, len(blobs)) + for i, blob := range blobs { - rawShares, err := BlobsToShares(blob) + msg, err := blobtypes.NewMsgPayForBlobs( + addr.String(), + &blob.Blob, + ) require.NoError(t, err) - eds, err := ipld.AddShares(ctx, rawShares, bs) + coreTx := edstest.BuildCoreTx(t, signer, msg, &blob.Blob) + dataSquare, err := square.Construct( + coretypes.Txs{coreTx}.ToSliceOfBytes(), + appconsts.LatestVersion, + appconsts.SquareSizeUpperBound(appconsts.LatestVersion), + ) + require.NoError(t, err) + + eds, err := ipld.AddShares(ctx, shares.ToBytes(dataSquare), bs) require.NoError(t, err) edsses[i] = eds } + headers := headertest.ExtendedHeadersFromEdsses(t, edsses) err = headerStore.Init(ctx, headers[0]) @@ -884,7 +883,6 @@ func createServiceWithSub(ctx context.Context, t testing.TB, blobs []*Blob) *Ser fn := func(ctx context.Context, height uint64) (*header.ExtendedHeader, error) { return headers[height-1], nil - // return headerStore.GetByHeight(ctx, height) } fn2 := func(ctx context.Context) (<-chan *header.ExtendedHeader, error) { headerChan := make(chan *header.ExtendedHeader, len(headers)) @@ -1063,3 +1061,125 @@ func generateCommitmentProofFromBlock( return commitmentProof } + +func TestBlobVerify(t *testing.T) { + _, blobs, nss, eds, _, _, dataRoot := edstest.GenerateTestBlock(t, 200, 10) + + // create the blob from the data + blob, err := NewBlob( + uint8(blobs[5].ShareVersion), + nss[5].Bytes(), + blobs[5].Data, + ) + require.NoError(t, err) + + // convert the blob to a number of shares + blobShares, err := BlobsToShares(blob) + require.NoError(t, err) + + // find the first share of the blob in the ODS + startShareIndex := -1 + for i, sh := range eds.FlattenedODS() { + if bytes.Equal(sh, blobShares[0]) { + startShareIndex = i + break + } + } + require.Greater(t, startShareIndex, 0) + + // create an inclusion proof of the blob using the share range instead of the commitment + sharesProof, err := pkgproof.NewShareInclusionProofFromEDS( + eds, + nss[5], + shares.NewRange(startShareIndex, startShareIndex+len(blobShares)), + ) + require.NoError(t, err) + require.NoError(t, sharesProof.Validate(dataRoot)) + + blobProof := Proof{ + ShareToRowRootProof: sharesProof.ShareProofs, + RowToDataRootProof: sharesProof.RowProof, + } + tests := []struct { + name string + blob Blob + proof Proof + dataRoot []byte + expectErr bool + }{ + { + name: "invalid blob commitment", + dataRoot: dataRoot, + proof: blobProof, + blob: func() Blob { + b := *blob + b.Commitment = []byte{0x1} + return b + }(), + expectErr: true, + }, + { + name: "invalid row proof", + dataRoot: dataRoot, + proof: func() Proof { + p := blobProof + p.RowToDataRootProof.StartRow = 10 + p.RowToDataRootProof.EndRow = 15 + return p + }(), + blob: *blob, + expectErr: true, + }, + { + name: "malformed blob and proof", + dataRoot: dataRoot, + proof: func() Proof { + p := blobProof + p.ShareToRowRootProof = p.ShareToRowRootProof[1:] + return p + }(), + blob: func() Blob { + b := *blob + b.Commitment = []byte{0x1} + return b + }(), + expectErr: true, + }, + { + name: "mismatched number of share proofs and row proofs", + dataRoot: dataRoot, + proof: func() Proof { + p := blobProof + p.ShareToRowRootProof = p.ShareToRowRootProof[1:] + return p + }(), + blob: *blob, + expectErr: true, + }, + { + name: "invalid data root", + dataRoot: []byte{0x1, 0x2}, + proof: blobProof, + blob: *blob, + expectErr: true, + }, + { + name: "valid proof", + dataRoot: dataRoot, + blob: *blob, + proof: blobProof, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + valid, err := test.proof.Verify(&test.blob, test.dataRoot) + if test.expectErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.True(t, valid) + } + }) + } +} diff --git a/go.mod b/go.mod index 1ac1249148..c8ef17a79e 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/BurntSushi/toml v1.4.0 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.13.0 + github.com/celestiaorg/celestia-app v1.14.0 github.com/celestiaorg/go-fraud v0.2.1 github.com/celestiaorg/go-header v0.6.2 github.com/celestiaorg/go-libp2p-messenger v0.2.0 diff --git a/go.sum b/go.sum index e159da5846..53949a8bc4 100644 --- a/go.sum +++ b/go.sum @@ -353,8 +353,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.13.0 h1:7MWEox6lim6WDyiP84Y2/ERfWUJxWPfZlKxzO6OFcig= -github.com/celestiaorg/celestia-app v1.13.0/go.mod h1:CF9VZwWAlTU0Is/BOsmxqkbkYnnmrgl0YRlSBIzr0m0= +github.com/celestiaorg/celestia-app v1.14.0 h1:Av1Q8de41WRABQ+mnwHwj4a6Rd5sqEUZqrTwuA1LbBM= +github.com/celestiaorg/celestia-app v1.14.0/go.mod h1:OPkbzIvBUGwTvfunQ/uh7qEekjkix59kB0CsK8+i6uM= github.com/celestiaorg/celestia-core v1.38.0-tm-v0.34.29 h1:HwbA4OegRvXX0aNchBA7Cmu+oIxnH7xRcOhISuDP0ak= github.com/celestiaorg/celestia-core v1.38.0-tm-v0.34.29/go.mod h1:MyElURdWAOJkOp84WZnfEUJ+OLvTwOOHG2lbK9E8XRI= github.com/celestiaorg/cosmos-sdk v1.23.0-sdk-v0.46.16 h1:N2uETI13szEKnGAdKhtTR0EsrpcW0AwRKYER74WLnuw= diff --git a/share/eds/edstest/testing.go b/share/eds/edstest/testing.go index 5f6dcfa6f7..3b122b8d35 100644 --- a/share/eds/edstest/testing.go +++ b/share/eds/edstest/testing.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/require" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" coretypes "github.com/tendermint/tendermint/types" "github.com/celestiaorg/celestia-app/app" @@ -138,7 +139,19 @@ func createTestBlobTransaction( ns := namespace.RandomBlobNamespace() msg, blob := blobfactory.RandMsgPayForBlobsWithNamespaceAndSigner(addr.String(), ns, size) require.NoError(t, err) + cTx := BuildCoreTx(t, signer, msg, blob) + return ns, msg, blob, cTx +} +// BuildCoreTx takes a signer, a message and a blob and creates a core transaction. +// The core transaction is the final form of a transaction that gets pushed +// into the square builder. +func BuildCoreTx( + t *testing.T, + signer *types.KeyringSigner, + msg *types.MsgPayForBlobs, + blob *tmproto.Blob, +) coretypes.Tx { builder := signer.NewTxBuilder() stx, err := signer.BuildSignedTx(builder, msg) require.NoError(t, err) @@ -146,5 +159,5 @@ func createTestBlobTransaction( require.NoError(t, err) cTx, err := coretypes.MarshalBlobTx(rawTx, blob) require.NoError(t, err) - return ns, msg, blob, cTx + return cTx }