From 2f18e6cbe026b006202dab6078504046c467e470 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Wed, 17 May 2023 12:14:05 +0100 Subject: [PATCH] feat: createExtendedProviderAd helper to guide extended provider usage (#8) the expected way to use extended providers is with a special advert that doesn't include a context or entries, so a helper is provider to help guide usage. The Advertisement constructor lets you do anything that's legal but requires you to pass null for the special case of a ad with no context id, entries or previous cid. While the helper lets you skip those nulls as you've clearly opt'd in to the extendedProvider world. ```js /** * Advertise that **all** past and future entries in this chain are now * available from a new, additional provider by specifying the root provider * and the additional providers along with no context id and no entries cid. * * To advertise that subset of entries are available from additional providers * specify the relevant context id to identify that group. * * Note: it is not yet possible to unannounce an extended provider once announced. * see: https://github.com/ipni/storetheindex/issues/1745 * * @param {object} config * @param {Provider[]} config.providers * @param {Link | null} config.previous * @param {Bytes | null} [config.context] */ export function createExtendedProviderAd ({ previous, providers, context = null }) { if (!providers || !Array.isArray(providers) || providers.length < 2) { throw new Error('at least 2 providers are required, the root provider and the new extended provider') } return new Advertisement({ previous, providers, entries: null, context }) } ``` Adds and reworks some missing validation checks and typos. License: MIT --------- Signed-off-by: Oli Evans --- README.md | 44 ++++++++-------- advertisement.js | 77 +++++++++++++++++++++------- examples/extended-providers.js | 11 ++-- index.js | 2 +- package-lock.json | 4 +- test/advertisement.test.js | 91 +++++++++++++++++++++++++++++++++- 6 files changed, 180 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 2257aff..0af4880 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ const block = await Block.encode({ value, codec: dagJson, hasher: sha256 }) fs.writeFileSync(block.cid.toString(), block.bytes) ``` -An `dag-json` encoded Advertisement (re-formated for readability): +A `dag-json` encoded Advertisement (formatted for readability): ```json { @@ -96,7 +96,13 @@ An `dag-json` encoded Advertisement (re-formated for readability): ## Extended Providers -Encode a signed advertisement with an Extended Providers section where the entries are available from multiple providers or different protocols. +Encode a signed advertisement with an Extended Providers section and no context id or entries cid to announce that **all** previous and future entries are available from multiple providers or different protocols. + +You only need to announce the additional providers once. Subsequent ExtendedProvider advertisements are additive. The indexer will record that your entries are available from the union of all the ExtendedProvider records. + +Note: it is not currently possible to remove a Provider once announced ([issue](https://github.com/ipni/storetheindex/issues/1745)) + +You may announce a set of ExtendedProviders with a context to inform the indexer that only the subset of entries with the same context id are available from these extended providers. The first provider passed to the Advertisement constructor is used as the top level provider for older indexers that don't yet support the `ExtendedProvider` property. @@ -108,11 +114,9 @@ import { sha256 } from 'multiformats/hashes/sha2' import * as dagJson from '@ipld/dag-json' import { createEd25519PeerId } from '@libp2p/peer-id-factory' -import { Provider, Advertisement } from '../index.js' +import { Provider, createExtendedProviderAd } from '../index.js' -const previous = null // CID for previous batch. Pass `null` for the first advertisement in your chain -const entries = CID.parse('baguqeera4vd5tybgxaub4elwag6v7yhswhflfyopogr7r32b7dpt5mqfmmoq') // entry batch to provide -const context = new Uint8Array([99]) // custom id for a set of multihashes +const previous = null // CID for previous advertisement. Pass `null` for the first advertisement in your chain // create a provider for each peer + protocol that will provider your entries const bits = new Provider({ protocol: 'bitswap', addresses: ['/ip4/12.34.56.1/tcp/999/ws'], peerId: await createEd25519PeerId() }) @@ -128,8 +132,9 @@ const graf = new Provider({ } }) -// an advertisement with a single multiple providers -const advert = new Advertisement({ providers: [http, bits, graf], entries, context, previous }) +// create an ad with the extra provider info and no context or entries +// to denote that they apply to all previous and future advertisements +const advert = createExtendedProviderAd({ providers: [http, bits, graf], previous }) // sign and export to IPLD form per schema const value = await advert.encodeAndSign() @@ -141,8 +146,7 @@ const block = await Block.encode({ value, codec: dagJson, hasher: sha256 }) fs.writeFileSync(block.cid.toString(), block.bytes) ``` -
- dag-json output +A `dag-json` encoded Advertisement (formatted for readability): ```json { @@ -151,11 +155,11 @@ fs.writeFileSync(block.cid.toString(), block.bytes) ], "ContextID": { "/": { - "bytes": "Yw" + "bytes": "" } }, "Entries": { - "/": "baguqeera4vd5tybgxaub4elwag6v7yhswhflfyopogr7r32b7dpt5mqfmmoq" + "/": "bafkreehdwdcefgh4dqkjv67uzcmw7oje" }, "ExtendedProvider": { "Override": false, @@ -164,7 +168,7 @@ fs.writeFileSync(block.cid.toString(), block.bytes) "Addresses": [ "/dns4/dag.house/tcp/443/https" ], - "ID": "12D3KooWPPwQ99nqqBJhAYZnvicHDfx7o855fUzBVBVgBQ4PotMU", + "ID": "12D3KooWQWY5d9xp8on1cizKBdbscfKo1qcyovk7KwV9kKKXWpwK", "Metadata": { "/": { "bytes": "gID0AQ" @@ -172,7 +176,7 @@ fs.writeFileSync(block.cid.toString(), block.bytes) }, "Signature": { "/": { - "bytes": "CiQIARIgycGrz1Pkp8va7HhAM0+MHumsG5MxgcpUJOeSBeyH1f8SKS9pbmRleGVyL2luZ2VzdC9leHRlbmRlZFByb3ZpZGVyU2lnbmF0dXJlGiISIBNXY+VA96CrdsbGe54bA7TGHgfB9z05ZzVApWVzdMOWKkD1sZAZVMkYAkugiqlDpiU1o1KkYCcmyA+ozWNOMgfvk7g3eDyIGP1oIHBUQuOcIYd0RB0VdV5/Kl1uV8KQKysJ" + "bytes": "CiQIARIg2k4OefnZgOzUQo0VQE5Yg9KubOpw3gTWHeQprfuidZISKS9pbmRleGVyL2luZ2VzdC9leHRlbmRlZFByb3ZpZGVyU2lnbmF0dXJlGiISIOrC3ZauKlzBVU7HWLR3VjlW79cf9D7xMKMcbqBXA1bQKkB1rdZLHfzTDpfZZ2IH6HJHsGSkaKbmRD+QSIIb0z73sKoSMutXeuJiK2cJ54PL6m2hPCWJyV9fBcuYMKDAXVwA" } } }, @@ -180,7 +184,7 @@ fs.writeFileSync(block.cid.toString(), block.bytes) "Addresses": [ "/ip4/12.34.56.1/tcp/999/ws" ], - "ID": "12D3KooWLcR73mkaEfNy9i9nDq3NBqFZwBvnvZqVo1MUV6BAvfMB", + "ID": "12D3KooWJn37snQzNk3BTBgzGFJpxg7er8CLofq2789PqaPzPF1g", "Metadata": { "/": { "bytes": "gBI" @@ -188,7 +192,7 @@ fs.writeFileSync(block.cid.toString(), block.bytes) }, "Signature": { "/": { - "bytes": "CiQIARIgoGDoEx0useLQEWElUSa7imb/59IygDSjK5qRKlRcSEISKS9pbmRleGVyL2luZ2VzdC9leHRlbmRlZFByb3ZpZGVyU2lnbmF0dXJlGiISIPP+xTvUwEfL4LrCnL+Pj79ARZ0bl6hBYpzZ4aFN48wFKkDI0GIqaUqQxaEgcE7WBVH+wdc6Ppp4WgKekTAV9RpniR7zprFobQdkHuFmnaepeSbOIBwyrL1ENGRbCxC94ykF" + "bytes": "CiQIARIghSB7P4RGuK3xYFMW/Z5fKNvzMqDb424fhxkTRfde5B8SKS9pbmRleGVyL2luZ2VzdC9leHRlbmRlZFByb3ZpZGVyU2lnbmF0dXJlGiISIBV1UXktJsOIfiXLGueJmvbpMYOXwdk8tMzRWOBSb4VIKkCzt7tvBMp/mjM4P2A3qU5XfvWF0/7M2cBoNLCM24jVu1roj5yyj1NA/xLA+ap97YY79EPx7eQWEnMxF15wr2EL" } } }, @@ -196,7 +200,7 @@ fs.writeFileSync(block.cid.toString(), block.bytes) "Addresses": [ "/ip4/120.0.0.1/tcp/1234" ], - "ID": "12D3KooWShFBk7jQLFYAPrzeHmdL5nYgrrEfiyJFUJfhguCPUJq3", + "ID": "12D3KooWG8RfSPYd5RgUFsAdJL1HnAvGMsce7CLJ1hcQTQa7cVAQ", "Metadata": { "/": { "bytes": "kBKjaFBpZWNlQ0lE2CpYJQABcBIgWZSEOQZfKWGe9BKAy7kyvlLFbZnFlmtl4BESOfCYu+9sVmVyaWZpZWREZWFs9W1GYXN0UmV0cmlldmFs9Q" @@ -204,7 +208,7 @@ fs.writeFileSync(block.cid.toString(), block.bytes) }, "Signature": { "/": { - "bytes": "CiQIARIg+sO3dlJXJxqd/6oCcmOR3ZvhHQIfoqtoxnDx6n/amD4SKS9pbmRleGVyL2luZ2VzdC9leHRlbmRlZFByb3ZpZGVyU2lnbmF0dXJlGiISIAuLbadWtTs8Bhx1s/w1/BHsrAfwNMy1Y88O1LrrxtehKkA6OjXq4rgD07uZoHzZw4Sd4cGbgdIXBO1vB1Pag5FqcuhP4R3Hi0O9QpoPdzxlXDKHYYVS+vrNUzLGiT8/STgL" + "bytes": "CiQIARIgXcaHEiXQHTgt2OE9I4oWwNv7gtbqWMCI03gSEh1O00kSKS9pbmRleGVyL2luZ2VzdC9leHRlbmRlZFByb3ZpZGVyU2lnbmF0dXJlGiISIGyVi9n2pbJhuoXyE4k+SzPKpL0eb2nENXNUWaN0i/eLKkBluzmx84WCkzLkFo+XtYzpuqR5t8aJXf8Y55XoNhSPT79UAvwSMWLbKy2C9GXORQb5hCHye1cOaT11zisssKMA" } } } @@ -216,10 +220,10 @@ fs.writeFileSync(block.cid.toString(), block.bytes) "bytes": "gID0AQ" } }, - "Provider": "12D3KooWPPwQ99nqqBJhAYZnvicHDfx7o855fUzBVBVgBQ4PotMU", + "Provider": "12D3KooWQWY5d9xp8on1cizKBdbscfKo1qcyovk7KwV9kKKXWpwK", "Signature": { "/": { - "bytes": "CiQIARIgycGrz1Pkp8va7HhAM0+MHumsG5MxgcpUJOeSBeyH1f8SGy9pbmRleGVyL2luZ2VzdC9hZFNpZ25hdHVyZRoiEiDoF5DFOpj4mxco1sWVnC6KEsjfd3yz9i47SS4NJAhSNCpAsn5C2HUI1K5/FtXZ8+Xcr6V4AGxstCMIudf6B3H3bGw3OcCfDOS01MgNyArtp9dW2XobykWhan7r2g/3VRYQDw" + "bytes": "CiQIARIg2k4OefnZgOzUQo0VQE5Yg9KubOpw3gTWHeQprfuidZISGy9pbmRleGVyL2luZ2VzdC9hZFNpZ25hdHVyZRoiEiARPrSzHMsp4L/L9zSmNQz2ooRAEznsM76n+BfkIewNlipA5Q3UW14STPAyTotfP7pHGseL1Yi8Bh5hf+X0yuYAAsIsRpnYQJKrAcWxQS+oGwQLa4pJ+NXCiro6M98Ey2SlBQ" } } } diff --git a/advertisement.js b/advertisement.js index e706b92..f82aa96 100644 --- a/advertisement.js +++ b/advertisement.js @@ -18,10 +18,16 @@ export const AD_SIG_CODEC = new TextEncoder().encode('/indexer/ingest/adSignatur export const EP_SIG_CODEC = new TextEncoder().encode('/indexer/ingest/extendedProviderSignature') export const SIG_DOMAIN = 'indexer' -// instead of making Entires optional there is a magic CID, a stubby 16 byte sha256 of empty bytes +// instead of making Entries optional there is a magic CID, a stubby 16 byte sha256 of empty bytes // https://github.com/ipni/go-libipni/blob/81286e4b32baed09e6151ce4f8e763f449b81331/ingest/schema/schema.go#L64-L69 export const NO_ENTRIES = CID.parse('bafkreehdwdcefgh4dqkjv67uzcmw7oje') +// an empty byte array signifies no context should be applied +export const NO_CONTEXT = new Uint8Array() + +// maximum number of bytes accepted as Advertisement.ContextID. +export const MAX_CONTEXT_ID_LENGTH = 64 + /** * Sign the serialized form of an Advertisement or a Provider * @param {PeerId} peerId @@ -56,26 +62,42 @@ export async function hashSignableBytes (bytes) { export class Advertisement { /** * @param {object} config - * @param {Provider[]|Provider} config.providers - * @param {Link} [config.entries] - * @param {Bytes} config.context - * @param {Link | null} config.previous - * @param {boolean} [config.remove] - * @param {boolean} [config.override] + * @param {Link | null} config.previous - CID of previous Advertisement + * @param {Provider[]|Provider} config.providers - Array of Provider info where entries are available + * @param {Link | null} config.entries - CID for an EntryBatch, an array of content multihashes you're providing + * @param {Bytes | null} config.context - A custom id used to group subsets of advertisements + * @param {boolean} [config.remove] - true if this represents entries that are no longer retrievable. + * @param {boolean} [config.override] - true if the extended providers specified should be used instead of any previously announced without a context. */ - constructor ({ previous, providers, context, entries = NO_ENTRIES, remove = false, override = false }) { - if (!providers || !context) { - throw new Error('providers and context are required') + constructor ({ previous, providers, context, entries, remove = false, override = false }) { + if (!providers) { + throw new Error('providers are required') + } + if (entries === undefined) { + throw new Error('entries must be set. To specify no entries pass null') + } + if (context === undefined) { + throw new Error('context must be set. To specify no context pass null') } if (previous === undefined) { throw new Error('previous must be set. If this is your first advertisement pass null') } - this.previous = previous + if (context !== null && context.byteLength > MAX_CONTEXT_ID_LENGTH) { + throw new Error(`context must be less than ${MAX_CONTEXT_ID_LENGTH} bytes`) + } this.providers = Array.isArray(providers) ? providers : [providers] - this.entries = entries - this.context = context + this.previous = previous + this.entries = entries ?? NO_ENTRIES + this.context = context ?? NO_CONTEXT this.remove = remove this.override = override + if (this.remove && this.providers.length > 1) { + // see: https://github.com/ipni/go-libipni/blob/afe2d8ea45b86c2a22f756ee521741c8f99675e5/ingest/schema/envelope.go#L126-L127 + throw new Error('remove may only be true when there is a single provider. IsRm is not supported for ExtendedProvider advertisements') + } + if (this.override && (this.context.byteLength === 0 || this.providers.length < 2)) { + throw new Error('override may only be true when a context is set and more than 1 provider') + } } /** @@ -85,11 +107,6 @@ export class Advertisement { const ad = this const provider = ad.providers[0] - if (ad.remove && ad.providers.length > 1) { - // see: https://github.com/ipni/go-libipni/blob/afe2d8ea45b86c2a22f756ee521741c8f99675e5/ingest/schema/envelope.go#L126-L127 - throw new Error('rm ads are not supported for extended provider signatures') - } - /** @type {import('./schema').AdvertisementOutput} AdvertisementOutput */ const value = { Provider: provider.peerId.toString(), @@ -142,3 +159,27 @@ export class Advertisement { ]) } } + +/** + * Advertise that **all** past and future entries in this chain are now + * available from a new, additional provider by specifying the root provider + * and the additional providers along with no context id and no entries cid. + * + * To advertise that subset of entries are available from additional providers + * specify the relevant context id to identify that group. + * + * Note: it is not yet possible to unannounce an extended provider once announced. + * see: https://github.com/ipni/storetheindex/issues/1745 + * + * @param {object} config + * @param {Link | null} config.previous - CID of previous Advertisement + * @param {Provider[]} config.providers - Two or more Provider objects where entries are available + * @param {Bytes | null} [config.context] - A custom id used to group subsets of advertisements + * @param {boolean} [config.override] - true if the providers should be used instead of any previously announced without a context. + */ +export function createExtendedProviderAd ({ previous, providers, context = null, override = false }) { + if (!providers || !Array.isArray(providers) || providers.length < 2) { + throw new Error('at least 2 providers are required, the root provider and the new extended provider') + } + return new Advertisement({ previous, providers, entries: null, context, override }) +} diff --git a/examples/extended-providers.js b/examples/extended-providers.js index 90c7c5e..55ab908 100644 --- a/examples/extended-providers.js +++ b/examples/extended-providers.js @@ -5,11 +5,9 @@ import { sha256 } from 'multiformats/hashes/sha2' import * as dagJson from '@ipld/dag-json' import { createEd25519PeerId } from '@libp2p/peer-id-factory' -import { Provider, Advertisement } from '../index.js' +import { Provider, createExtendedProviderAd } from '../index.js' -const previous = null // CID for previous batch. Pass `null` for the first advertisement in your chain -const entries = CID.parse('baguqeera4vd5tybgxaub4elwag6v7yhswhflfyopogr7r32b7dpt5mqfmmoq') // entry batch to provide -const context = new Uint8Array([99]) // custom id for a set of multihashes +const previous = null // CID for previous advertisement. Pass `null` for the first advertisement in your chain // create a provider for each peer + protocol that will provider your entries const bits = new Provider({ protocol: 'bitswap', addresses: ['/ip4/12.34.56.1/tcp/999/ws'], peerId: await createEd25519PeerId() }) @@ -25,8 +23,9 @@ const graf = new Provider({ } }) -// an advertisement with a single multiple providers -const advert = new Advertisement({ providers: [http, bits, graf], entries, context, previous }) +// create an ad with the extra provider info and no context or entries +// to denote that they apply to all previous and future advertisements +const advert = createExtendedProviderAd({ providers: [http, bits, graf], previous }) // sign and export to IPLD form per schema const value = await advert.encodeAndSign() diff --git a/index.js b/index.js index 8df3f54..3f69cb7 100644 --- a/index.js +++ b/index.js @@ -1,2 +1,2 @@ export { Provider } from './provider.js' -export { Advertisement } from './advertisement.js' +export { Advertisement, createExtendedProviderAd } from './advertisement.js' diff --git a/package-lock.json b/package-lock.json index 396f80a..cdea906 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,11 +1,11 @@ { - "name": "ipni", + "name": "@web3-storage/ipni", "version": "1.0.1", "lockfileVersion": 2, "requires": true, "packages": { "": { - "name": "ipni", + "name": "@web3-storage/ipni", "version": "1.0.1", "license": "MIT", "dependencies": { diff --git a/test/advertisement.test.js b/test/advertisement.test.js index 7dc6593..f31f861 100644 --- a/test/advertisement.test.js +++ b/test/advertisement.test.js @@ -5,7 +5,7 @@ import { parse as parseSchema } from 'ipld-schema' import { create as createValidator } from 'ipld-schema-validator' import { readFile } from 'fs/promises' import { Provider, HTTP_PREFIX, BITSWAP_PREFIX, GRAPHSYNC_PREFIX } from '../provider.js' -import { Advertisement, hashSignableBytes } from '../advertisement.js' +import { Advertisement, hashSignableBytes, createExtendedProviderAd } from '../advertisement.js' import { encode, decode } from '@ipld/dag-json' const schema = await readFile('schema.ipldsch', { encoding: 'utf8' }) @@ -126,7 +126,7 @@ test('extended provider interop', async t => { peerId: await loadPeerId('test/fixtures/ad-2/peerId.json') }) const context = Buffer.from(expected.ContextID['/'].bytes, 'base64') - const ad = new Advertisement({ providers: [p1, p2], context, previous: null }) + const ad = new Advertisement({ providers: [p1, p2], context, entries: null, previous: null }) const value = await ad.encodeAndSign() t.deepEqual(decode(encode(value)), decode(encode(expected))) }) @@ -167,6 +167,93 @@ test('parity with publisher-lambda with previous', async t => { // t.is(adCid.toString(), 'baguqeeracy3dyhdtqxo2wqcvekr6zbdsztjw54uqezjvrnadyrfnpvwzcxta') }) +test('createExtendedProviderAd helper', async t => { + const message = 'at least 2 providers are required, the root provider and the new extended provider' + const provider = new Provider({ + protocol: 'http', + addresses: '/dns4/example.org/tcp/443/https', + peerId: await createEd25519PeerId() + }) + // @ts-expect-error + t.throws(() => createExtendedProviderAd({ previous: null }), { message }) + // @ts-expect-error + t.throws(() => createExtendedProviderAd({ providers: provider, previous: null }), { message }) + + t.throws(() => createExtendedProviderAd({ providers: [provider], previous: null }), { message }) + + const ad = createExtendedProviderAd({ providers: [provider, provider], previous: null }) + const value = await ad.encodeAndSign() + t.is(value.ExtendedProvider?.Providers.length, 2) +}) + +test('max context id length', async t => { + // see: https://github.com/ipni/go-libipni/blob/b6e9a9def00b2db19aa4a3bc5fc33b3b1575530e/ingest/schema/schema.go#L18-L19 + const providers = new Provider({ + protocol: 'http', + addresses: '/dns4/example.org/tcp/443/https', + peerId: await createEd25519PeerId() + }) + t.throws( + () => new Advertisement({ previous: null, context: new Uint8Array(65).fill(1), providers, entries: null }), + { message: 'context must be less than 64 bytes' } + ) + const ad = new Advertisement({ previous: null, context: new Uint8Array(64).fill(1), providers, entries: null }) + const value = await ad.encodeAndSign() + t.deepEqual(value.ContextID, new Uint8Array(64).fill(1)) +}) + +test('override', async t => { + // see: https://github.com/ipni/go-libipni/blob/b6e9a9def00b2db19aa4a3bc5fc33b3b1575530e/ingest/schema/schema.go#L18-L19 + const provider = new Provider({ + protocol: 'http', + addresses: '/dns4/example.org/tcp/443/https', + peerId: await createEd25519PeerId() + }) + const providers = [provider, provider] + const entries = null + const previous = null + const override = true + t.throws( + () => new Advertisement({ previous, context: null, override, providers, entries }), + { message: 'override may only be true when a context is set and more than 1 provider' } + ) + + t.throws( + () => new Advertisement({ previous, context: new Uint8Array(), override, providers, entries }), + { message: 'override may only be true when a context is set and more than 1 provider' } + ) + + t.throws( + () => new Advertisement({ previous, context: new Uint8Array(1).fill(1), override, providers: provider, entries }), + { message: 'override may only be true when a context is set and more than 1 provider' } + ) + + const ad = new Advertisement({ previous, context: new Uint8Array(1).fill(1), override, providers, entries }) + const value = await ad.encodeAndSign() + t.true(value.ExtendedProvider?.Override) +}) + +test('remove', async t => { + // see: https://github.com/ipni/go-libipni/blob/b6e9a9def00b2db19aa4a3bc5fc33b3b1575530e/ingest/schema/schema.go#L18-L19 + const provider = new Provider({ + protocol: 'http', + addresses: '/dns4/example.org/tcp/443/https', + peerId: await createEd25519PeerId() + }) + const previous = null + const context = null + const entries = null + const remove = true + t.throws( + () => new Advertisement({ remove, previous, context, providers: [provider, provider], entries }), + { message: 'remove may only be true when there is a single provider. IsRm is not supported for ExtendedProvider advertisements' } + ) + + const ad = new Advertisement({ remove, previous, context, providers: provider, entries }) + const value = await ad.encodeAndSign() + t.true(value.IsRm) +}) + /** * @param {string} path */