Support Ipni-Cid-Schema-Type HTTP header

README.md

@@ -42,36 +42,16 @@ After a configured amount of time without any updates from a provider (`PollInte
 
 The configuration values that control this are documented [here](https://pkg.go.dev/github.com/ipni/storetheindex/config#Discovery), and their default values are specified [here](https://github.com/ipni/storetheindex/blob/main/doc/config.md#discovery). A custom polling configuration may be applied for specific providers using the `PollOverrides` configuration value to specify per-provider [Polling configuration](https://pkg.go.dev/github.com/ipni/storetheindex/config#Polling).
 
-## Indexer CLI Commands
-There are a number of client commands included with storetheindex. Their purpose is to perform simple indexing and lookup actions against a running daemon.  These can be helpful to test that an indexer is working. These include the following commands:
+## Indexer Administration CLI Commands
+There are a number of administrative commands supported by storetheindex. These commands allow you to perform operations on a running indexer daemon. For a list of admin commands, see:
 
-Informational:
-
-- `find` Find value by CID or multihash in indexer
-- `providers` Show information about providers known to the indexer
-  - `get` Get information about a specified provider
-  - `list` List the known providers
-
-Administrative:
-
-- `admin` Perform admin activities with an indexer
-  - `allow` Allow advertisements and content from peer
-  - `block` Block advertisements and content from peer
-  - `import-providers` Import provider information from another indexer
-  - `reload-config` Reload various settings from the configuration file
-  - `sync` Sync indexer with provider
-- `init` Initialize or upgrade indexer node config file
-
-Testing:
-
-- `import` Imports data to indexer from different sources
-- `register` Register provider information with an indexer
-- `synthetic` Generate synthetic load to import in indexer
+```
+./storetheindex admin -help
+```
 
 ## Help
 To see a list of available commands, see `storetheindex --help`. For help with command usage, see `storetheindex <command> --help`.
 
-
 ## Configuration
 The storetheindex config file [documentation](https://github.com/ipni/storetheindex/blob/main/doc/config.md#the-storetheindex-config-file)
 

assigner/core/assigner_test.go

@@ -127,7 +127,6 @@ func TestAssignerAll(t *testing.T) {
 
 	var assignNum int
 	var assigns []int
-	timeout := time.NewTimer(3 * time.Second)
 	open := true
 	for open {
 		select {
@@ -137,11 +136,10 @@ func TestAssignerAll(t *testing.T) {
 			}
 			t.Log("Publisher", peer2IDStr, "assigned to indexer", assignNum)
 			assigns = append(assigns, assignNum)
-		case <-timeout.C:
+		case <-time.After(3 * time.Second):
 			t.Fatal("timed out waiting for assignment")
 		}
 	}
-	timeout.Stop()
 	require.Equal(t, 1, len(assigns))
 	require.Equal(t, 0, assigns[0])
 
@@ -162,7 +160,6 @@ func TestAssignerAll(t *testing.T) {
 	require.NoError(t, err)
 
 	assigns = assigns[:0]
-	timeout.Reset(3 * time.Second)
 	open = true
 	for open {
 		select {
@@ -172,11 +169,10 @@ func TestAssignerAll(t *testing.T) {
 			}
 			assigns = append(assigns, assignNum)
 			t.Log("Publisher", peer3IDStr, "assigned to indexer", assignNum)
-		case <-timeout.C:
+		case <-time.After(3 * time.Second):
 			t.Fatal("timed out waiting for assignment")
 		}
 	}
-	timeout.Stop()
 	sort.Ints(assigns)
 	require.Equal(t, []int{0, 1}, assigns)
 
@@ -253,7 +249,6 @@ func TestAssignerOne(t *testing.T) {
 
 	var assignNum int
 	var assigns []int
-	timeout := time.NewTimer(3 * time.Second)
 	open := true
 	for open {
 		select {
@@ -263,11 +258,10 @@ func TestAssignerOne(t *testing.T) {
 			}
 			t.Log("Publisher", peer2IDStr, "assigned to indexer", assignNum)
 			assigns = append(assigns, assignNum)
-		case <-timeout.C:
+		case <-time.After(3 * time.Second):
 			t.Fatal("timed out waiting for assignment")
 		}
 	}
-	timeout.Stop()
 	require.Equal(t, 1, len(assigns))
 	require.Equal(t, 0, assigns[0])
 
@@ -286,7 +280,6 @@ func TestAssignerOne(t *testing.T) {
 	require.NoError(t, err)
 
 	assigns = assigns[:0]
-	timeout.Reset(3 * time.Second)
 	open = true
 	for open {
 		select {
@@ -296,11 +289,10 @@ func TestAssignerOne(t *testing.T) {
 			}
 			assigns = append(assigns, assignNum)
 			t.Log("Publisher", peer3IDStr, "assigned to indexer", assignNum)
-		case <-timeout.C:
+		case <-time.After(3 * time.Second):
 			t.Fatal("timed out waiting for assignment")
 		}
 	}
-	timeout.Stop()
 	require.Equal(t, 1, len(assigns))
 	require.Equal(t, 1, assigns[0])
 
@@ -409,7 +401,6 @@ func TestAssignerPreferred(t *testing.T) {
 
 	var assignNum int
 	var assigns []int
-	timeout := time.NewTimer(3 * time.Second)
 	open := true
 	for open {
 		select {
@@ -419,11 +410,10 @@ func TestAssignerPreferred(t *testing.T) {
 			}
 			t.Log("Publisher", peer2IDStr, "assigned to indexer", assignNum)
 			assigns = append(assigns, assignNum)
-		case <-timeout.C:
+		case <-time.After(3 * time.Second):
 			t.Fatal("timed out waiting for assignment")
 		}
 	}
-	timeout.Stop()
 	require.Equal(t, 1, len(assigns))
 	require.Equal(t, 1, assigns[0], "expected assignment to indexer 1")
 
@@ -442,7 +432,6 @@ func TestAssignerPreferred(t *testing.T) {
 	require.NoError(t, err)
 
 	assigns = assigns[:0]
-	timeout.Reset(3 * time.Second)
 	open = true
 	for open {
 		select {
@@ -452,11 +441,10 @@ func TestAssignerPreferred(t *testing.T) {
 			}
 			assigns = append(assigns, assignNum)
 			t.Log("Publisher", peer3IDStr, "assigned to indexer", assignNum)
-		case <-timeout.C:
+		case <-time.After(3 * time.Second):
 			t.Fatal("timed out waiting for assignment")
 		}
 	}
-	timeout.Stop()
 	require.Equal(t, 1, len(assigns))
 	require.Equal(t, 1, assigns[0], "expected assignment to indexer 1")
 
@@ -551,11 +539,10 @@ func TestPoolIndexerOffline(t *testing.T) {
 	err = assigner.Announce(ctx, adCid, addrInfo)
 	require.NoError(t, err)
 
-	timeout := time.NewTimer(2 * time.Second)
 	select {
 	case <-asmtChan:
 		t.Fatal("shouold not see assignment with offline indexer")
-	case <-timeout.C:
+	case <-time.After(2 * time.Second):
 	}
 	require.False(t, assigner.InitDone())
 
@@ -566,13 +553,11 @@ func TestPoolIndexerOffline(t *testing.T) {
 	err = assigner.Announce(ctx, adCid, addrInfo)
 	require.NoError(t, err)
 
-	timeout.Reset(2 * time.Second)
 	select {
 	case <-asmtChan:
-	case <-timeout.C:
+	case <-time.After(2 * time.Second):
 		t.Fatal("timed out waiting for assignment")
 	}
-	timeout.Stop()
 
 	require.True(t, assigner.InitDone())
 
@@ -717,7 +702,6 @@ func TestFreezeHandoff(t *testing.T) {
 
 	var assignNum int
 	var assigns []int
-	timeout := time.NewTimer(3 * time.Second)
 	open := true
 	for open {
 		select {
@@ -727,11 +711,10 @@ func TestFreezeHandoff(t *testing.T) {
 			}
 			t.Log("Publisher", peer3IDStr, "assigned to indexer", assignNum)
 			assigns = append(assigns, assignNum)
-		case <-timeout.C:
+		case <-time.After(3 * time.Second):
 			t.Fatal("timed out waiting for assignment")
 		}
 	}
-	timeout.Stop()
 	require.Equal(t, 1, len(assigns))
 	require.Equal(t, 0, assigns[0])
 
@@ -744,7 +727,6 @@ func TestFreezeHandoff(t *testing.T) {
 	assigner.PollNow()
 	var handoffCount int
 
-	timeout.Reset(5 * time.Second)
 	for handoffCount < 2 {
 		select {
 		case handoff := <-handoffChan:
@@ -753,16 +735,15 @@ func TestFreezeHandoff(t *testing.T) {
 			pubID := <-handoffPubs
 			require.True(t, peer1IDStr == pubID || peer3IDStr == pubID)
 			handoffCount++
-		case <-timeout.C:
+		case <-time.After(5 * time.Second):
 			t.Fatal("timed out waiting for handoff")
 		}
 	}
 
-	timeout.Reset(time.Second)
 	select {
 	case <-handoffChan:
 		t.Fatal("should not have another")
-	case <-timeout.C:
+	case <-time.After(time.Second):
 	}
 
 	require.NoError(t, assigner.Close())
@@ -803,12 +784,10 @@ func TestAssignerNoIndexers(t *testing.T) {
 	require.NoError(t, err)
 
 	// Check that channel is closed without receiving anything.
-	timeout := time.NewTimer(3 * time.Second)
-	defer timeout.Stop()
 	select {
 	case _, open := <-asmtChan:
 		require.False(t, open)
-	case <-timeout.C:
+	case <-time.After(3 * time.Second):
 		t.Fatal("timed out waiting for assignment")
 	}
 }

assigner/server/server_test.go

@@ -80,9 +80,12 @@ func TestAssignOnAnnounce(t *testing.T) {
 		"--listen-ingest=/ip4/127.0.0.1/tcp/3601",
 		"--listen-p2p=/ip4/127.0.0.1/tcp/3603",
 	)
-	stiCfg, err := sticfg.Load(filepath.Join(rnr.Dir, ".storetheindex", "config"))
+	stiCfgPath := filepath.Join(rnr.Dir, ".storetheindex", "config")
+	stiCfg, err := sticfg.Load(stiCfgPath)
 	require.NoError(t, err)
 	indexerID := stiCfg.Identity.PeerID
+	stiCfg.Indexer.FreezeAtPercent = 99.0
+	stiCfg.Save(stiCfgPath)
 	t.Log("Initialized indexer", indexerID)
 
 	indexerReady := testcmd.NewStdoutWatcher(indexerReadyMatch)

command/flags.go

@@ -22,13 +22,6 @@ var cacheSizeFlag = &cli.Int64Flag{
 	Required: false,
 }
 
-var fileFlag = &cli.StringFlag{
-	Name:     "file",
-	Usage:    "Source file for import",
-	Aliases:  []string{"f"},
-	Required: true,
-}
-
 var listenAdminFlag = &cli.StringFlag{
 	Name:     "listen-admin",
 	Usage:    "Admin HTTP API listen address or 'none' to disable, overrides config",

command/loadtest.go

@@ -10,6 +10,5 @@ var LoadtestCmd = &cli.Command{
 	Subcommands: []*cli.Command{
 		loadGenCmd,
 		loadGenVerifyCmd,
-		syntheticCmd,
 	},
 }