Enable compactor and alertmanager in target all

Signed-off-by: SungJin1212 <[email protected]>
cortexproject · Sep 11, 2024 · fd5ee03 · fd5ee03
1 parent d0f253c
commit fd5ee03
Show file tree

Hide file tree

Showing 13 changed files with 176 additions and 61 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 ## master / unreleased
 
+* [CHANGE] Enable Compactor and Alertmanager in target all. #6204
 * [FEATURE] Ruler: Experimental: Add `ruler.frontend-address` to allow query to query frontends instead of ingesters. #6151
 * [FEATURE] Ruler: Minimize chances of missed rule group evaluations that can occur due to OOM kills, bad underlying nodes, or due to an unhealthy ruler that appears in the ring as healthy. This feature is enabled via `-ruler.enable-ha-evaluation` flag. #6129
 * [ENHANCEMENT] Query Frontend: Add peakSample in query stats response. #6188

diff --git a/integration/api_endpoints_test.go b/integration/api_endpoints_test.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -22,10 +23,19 @@ func TestIndexAPIEndpoint(t *testing.T) {
 	require.NoError(t, err)
 	defer s.Close()
 
+	configOverrides := map[string]string{
+		// alert manager
+		"-alertmanager.web.external-url":   "http://localhost/alertmanager",
+		"-alertmanager-storage.backend":    "local",
+		"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
+	}
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
+
 	// Start Cortex in single binary mode, reading the config from file.
 	require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile))
 
-	cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, nil, "", 9009, 9095)
+	cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, configOverrides, "", 9009, 9095)
 	require.NoError(t, s.StartAndWaitReady(cortex1))
 
 	// GET / should succeed
@@ -44,10 +54,19 @@ func TestConfigAPIEndpoint(t *testing.T) {
 	require.NoError(t, err)
 	defer s.Close()
 
+	configOverrides := map[string]string{
+		// alert manager
+		"-alertmanager.web.external-url":   "http://localhost/alertmanager",
+		"-alertmanager-storage.backend":    "local",
+		"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
+	}
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
+
 	// Start Cortex in single binary mode, reading the config from file.
 	require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile))
 
-	cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, nil, "", 9009, 9095)
+	cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, configOverrides, "", 9009, 9095)
 	require.NoError(t, s.StartAndWaitReady(cortex1))
 
 	// Get config from /config API endpoint.
@@ -62,6 +81,7 @@ func TestConfigAPIEndpoint(t *testing.T) {
 	// Start again Cortex in single binary with the exported config
 	// and ensure it starts (pass the readiness probe).
 	require.NoError(t, writeFileToSharedDir(s, cortexConfigFile, body))
-	cortex2 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-2", cortexConfigFile, nil, "", 9009, 9095)
+	configOverrides["-alertmanager.cluster.peers"] = cortex1.HTTPEndpoint()
+	cortex2 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-2", cortexConfigFile, configOverrides, "", 9009, 9095)
 	require.NoError(t, s.StartAndWaitReady(cortex2))
 }
diff --git a/integration/getting_started_single_process_config_local_test.go b/integration/getting_started_single_process_config_local_test.go
@@ -4,6 +4,7 @@
 package integration
 
 import (
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -24,7 +25,14 @@ func TestGettingStartedSingleProcessConfigWithFilesystem(t *testing.T) {
 	// Start Cortex components.
 	require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile))
 
-	flags := map[string]string{}
+	flags := map[string]string{
+		// alert manager
+		"-alertmanager.web.external-url":   "http://localhost/alertmanager",
+		"-alertmanager-storage.backend":    "local",
+		"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
+	}
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
 
 	cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, flags, "", 9009, 9095)
 	require.NoError(t, s.StartAndWaitReady(cortex))

diff --git a/integration/getting_started_single_process_config_test.go b/integration/getting_started_single_process_config_test.go
@@ -5,6 +5,7 @@ package integration
 
 import (
 	"fmt"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -38,7 +39,13 @@ func TestGettingStartedSingleProcessConfigWithBlocksStorage(t *testing.T) {
 		"-blocks-storage.s3.bucket-name":       bucketName,
 		"-blocks-storage.s3.endpoint":          fmt.Sprintf("%s-minio-9000:9000", networkName),
 		"-blocks-storage.s3.insecure":          "true",
+		// alert manager
+		"-alertmanager.web.external-url":   "http://localhost/alertmanager",
+		"-alertmanager-storage.backend":    "local",
+		"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
 	}
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
 
 	cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, flags, "", 9009, 9095)
 	require.NoError(t, s.StartAndWaitReady(cortex))

diff --git a/integration/getting_started_with_gossiped_ring_test.go b/integration/getting_started_with_gossiped_ring_test.go
@@ -5,6 +5,7 @@ package integration
 
 import (
 	"fmt"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -45,8 +46,15 @@ func TestGettingStartedWithGossipedRing(t *testing.T) {
 		"-blocks-storage.s3.insecure":                              "true",
 		"-store-gateway.sharding-ring.wait-stability-min-duration": "0", // start quickly
 		"-store-gateway.sharding-ring.wait-stability-max-duration": "0", // start quickly
+		// alert manager
+		"-alertmanager.web.external-url":   "http://localhost/alertmanager",
+		"-alertmanager-storage.backend":    "local",
+		"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
 	}
 
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
+
 	// This cortex will fail to join the cluster configured in yaml file. That's fine.
 	cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", "config1.yaml", e2e.MergeFlags(flags, map[string]string{
 		"-ingester.lifecycler.addr": networkName + "-cortex-1", // Ingester's hostname in docker setup

diff --git a/integration/integration_memberlist_single_binary_test.go b/integration/integration_memberlist_single_binary_test.go
@@ -43,6 +43,9 @@ func testSingleBinaryEnv(t *testing.T, tlsEnabled bool, flags map[string]string)
 	require.NoError(t, err)
 	defer s.Close()
 
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
+
 	// Start dependencies
 	minio := e2edb.NewMinio(9000, bucketName)
 	// Look ma, no Consul!
@@ -116,16 +119,21 @@ func testSingleBinaryEnv(t *testing.T, tlsEnabled bool, flags map[string]string)
 }
 
 func newSingleBinary(name string, servername string, join string, testFlags map[string]string) *e2ecortex.CortexService {
-	flags := map[string]string{
-		"-ingester.final-sleep":              "0s",
-		"-ingester.join-after":               "0s", // join quickly
-		"-ingester.min-ready-duration":       "0s",
-		"-ingester.num-tokens":               "512",
-		"-ingester.observe-period":           "5s", // to avoid conflicts in tokens
-		"-ring.store":                        "memberlist",
-		"-memberlist.bind-port":              "8000",
-		"-memberlist.left-ingesters-timeout": "600s", // effectively disable
-	}
+	flags := mergeFlags(
+		AlertmanagerLocalFlags(),
+		map[string]string{
+			"-ingester.final-sleep":              "0s",
+			"-ingester.join-after":               "0s", // join quickly
+			"-ingester.min-ready-duration":       "0s",
+			"-ingester.num-tokens":               "512",
+			"-ingester.observe-period":           "5s", // to avoid conflicts in tokens
+			"-ring.store":                        "memberlist",
+			"-memberlist.bind-port":              "8000",
+			"-memberlist.left-ingesters-timeout": "600s", // effectively disable
+			// alert manager
+			"-alertmanager.web.external-url": "http://localhost/alertmanager",
+		},
+	)
 
 	if join != "" {
 		flags["-memberlist.join"] = join
@@ -158,6 +166,9 @@ func TestSingleBinaryWithMemberlistScaling(t *testing.T) {
 	require.NoError(t, err)
 	defer s.Close()
 
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
+
 	minio := e2edb.NewMinio(9000, bucketName)
 	require.NoError(t, s.StartAndWaitReady(minio))
 

diff --git a/integration/otlp_test.go b/integration/otlp_test.go
@@ -6,6 +6,7 @@ package integration
 import (
 	"fmt"
 	"math/rand"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -41,7 +42,13 @@ func TestOTLP(t *testing.T) {
 		"-blocks-storage.s3.endpoint":                   fmt.Sprintf("%s-minio-9000:9000", networkName),
 		"-blocks-storage.s3.insecure":                   "true",
 		"-blocks-storage.tsdb.enable-native-histograms": "true",
+		// alert manager
+		"-alertmanager.web.external-url":   "http://localhost/alertmanager",
+		"-alertmanager-storage.backend":    "local",
+		"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
 	}
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
 
 	cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, flags, "", 9009, 9095)
 	require.NoError(t, s.StartAndWaitReady(cortex))

diff --git a/integration/querier_test.go b/integration/querier_test.go
@@ -365,26 +365,33 @@ func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) {
 
 				// Configure the blocks storage to frequently compact TSDB head
 				// and ship blocks to the storage.
-				flags := mergeFlags(BlocksStorageFlags(), map[string]string{
-					"-blocks-storage.tsdb.block-ranges-period":          blockRangePeriod.String(),
-					"-blocks-storage.tsdb.ship-interval":                "1s",
-					"-blocks-storage.bucket-store.sync-interval":        "1s",
-					"-blocks-storage.tsdb.retention-period":             ((blockRangePeriod * 2) - 1).String(),
-					"-blocks-storage.bucket-store.index-cache.backend":  testCfg.indexCacheBackend,
-					"-blocks-storage.bucket-store.bucket-index.enabled": strconv.FormatBool(testCfg.bucketIndexEnabled),
-					"-querier.query-store-for-labels-enabled":           "true",
-					"-querier.thanos-engine":                            strconv.FormatBool(thanosEngine),
-					// Ingester.
-					"-ring.store":      "consul",
-					"-consul.hostname": consul.NetworkHTTPEndpoint(),
-					// Distributor.
-					"-distributor.replication-factor": strconv.FormatInt(seriesReplicationFactor, 10),
-					// Store-gateway.
-					"-store-gateway.sharding-enabled":                 strconv.FormatBool(testCfg.blocksShardingEnabled),
-					"-store-gateway.sharding-ring.store":              "consul",
-					"-store-gateway.sharding-ring.consul.hostname":    consul.NetworkHTTPEndpoint(),
-					"-store-gateway.sharding-ring.replication-factor": "1",
-				})
+				flags := mergeFlags(
+					BlocksStorageFlags(),
+					AlertmanagerLocalFlags(),
+					map[string]string{
+						"-blocks-storage.tsdb.block-ranges-period":          blockRangePeriod.String(),
+						"-blocks-storage.tsdb.ship-interval":                "1s",
+						"-blocks-storage.bucket-store.sync-interval":        "1s",
+						"-blocks-storage.tsdb.retention-period":             ((blockRangePeriod * 2) - 1).String(),
+						"-blocks-storage.bucket-store.index-cache.backend":  testCfg.indexCacheBackend,
+						"-blocks-storage.bucket-store.bucket-index.enabled": strconv.FormatBool(testCfg.bucketIndexEnabled),
+						"-querier.query-store-for-labels-enabled":           "true",
+						"-querier.thanos-engine":                            strconv.FormatBool(thanosEngine),
+						// Ingester.
+						"-ring.store":      "consul",
+						"-consul.hostname": consul.NetworkHTTPEndpoint(),
+						// Distributor.
+						"-distributor.replication-factor": strconv.FormatInt(seriesReplicationFactor, 10),
+						// Store-gateway.
+						"-store-gateway.sharding-enabled":                 strconv.FormatBool(testCfg.blocksShardingEnabled),
+						"-store-gateway.sharding-ring.store":              "consul",
+						"-store-gateway.sharding-ring.consul.hostname":    consul.NetworkHTTPEndpoint(),
+						"-store-gateway.sharding-ring.replication-factor": "1",
+						// alert manager
+						"-alertmanager.web.external-url": "http://localhost/alertmanager",
+					},
+				)
+				require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs/user-1.yaml", []byte(cortexAlertmanagerUserConfigYaml)))
 
 				// Add the cache address to the flags.
 				if testCfg.indexCacheBackend == tsdb.IndexCacheBackendMemcached {

diff --git a/integration/query_fuzz_test.go b/integration/query_fuzz_test.go
@@ -45,25 +45,32 @@ func TestVerticalShardingFuzz(t *testing.T) {
 	consul2 := e2edb.NewConsulWithName("consul2")
 	require.NoError(t, s.StartAndWaitReady(consul1, consul2))
 
-	flags := map[string]string{
-		"-store.engine":                                     blocksStorageEngine,
-		"-blocks-storage.backend":                           "filesystem",
-		"-blocks-storage.tsdb.head-compaction-interval":     "4m",
-		"-blocks-storage.tsdb.block-ranges-period":          "2h",
-		"-blocks-storage.tsdb.ship-interval":                "1h",
-		"-blocks-storage.bucket-store.sync-interval":        "15m",
-		"-blocks-storage.tsdb.retention-period":             "2h",
-		"-blocks-storage.bucket-store.index-cache.backend":  tsdb.IndexCacheBackendInMemory,
-		"-blocks-storage.bucket-store.bucket-index.enabled": "true",
-		"-querier.query-store-for-labels-enabled":           "true",
-		// Ingester.
-		"-ring.store":      "consul",
-		"-consul.hostname": consul1.NetworkHTTPEndpoint(),
-		// Distributor.
-		"-distributor.replication-factor": "1",
-		// Store-gateway.
-		"-store-gateway.sharding-enabled": "false",
-	}
+	flags := mergeFlags(
+		AlertmanagerLocalFlags(),
+		map[string]string{
+			"-store.engine":                                     blocksStorageEngine,
+			"-blocks-storage.backend":                           "filesystem",
+			"-blocks-storage.tsdb.head-compaction-interval":     "4m",
+			"-blocks-storage.tsdb.block-ranges-period":          "2h",
+			"-blocks-storage.tsdb.ship-interval":                "1h",
+			"-blocks-storage.bucket-store.sync-interval":        "15m",
+			"-blocks-storage.tsdb.retention-period":             "2h",
+			"-blocks-storage.bucket-store.index-cache.backend":  tsdb.IndexCacheBackendInMemory,
+			"-blocks-storage.bucket-store.bucket-index.enabled": "true",
+			"-querier.query-store-for-labels-enabled":           "true",
+			// Ingester.
+			"-ring.store":      "consul",
+			"-consul.hostname": consul1.NetworkHTTPEndpoint(),
+			// Distributor.
+			"-distributor.replication-factor": "1",
+			// Store-gateway.
+			"-store-gateway.sharding-enabled": "false",
+			// alert manager
+			"-alertmanager.web.external-url": "http://localhost/alertmanager",
+		},
+	)
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
 
 	path1 := path.Join(s.SharedDir(), "cortex-1")
 	path2 := path.Join(s.SharedDir(), "cortex-2")

diff --git a/integration/ruler_test.go b/integration/ruler_test.go
@@ -154,7 +154,13 @@ func TestRulerAPISingleBinary(t *testing.T) {
 		"-ruler-storage.local.directory": filepath.Join(e2e.ContainerSharedDir, "ruler_configs"),
 		"-ruler.poll-interval":           "2s",
 		"-ruler.rule-path":               filepath.Join(e2e.ContainerSharedDir, "rule_tmp/"),
+		// alert manager
+		"-alertmanager.web.external-url":   "http://localhost/alertmanager",
+		"-alertmanager-storage.backend":    "local",
+		"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
 	}
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
 
 	// Start Cortex components.
 	require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile))

diff --git a/integration/runtime_config_test.go b/integration/runtime_config_test.go
@@ -40,16 +40,26 @@ func TestLoadRuntimeConfigFromStorageBackend(t *testing.T) {
 			name: "no storage backend provided",
 			flags: map[string]string{
 				"-runtime-config.file": filePath,
+				// alert manager
+				"-alertmanager.web.external-url":   "http://localhost/alertmanager",
+				"-alertmanager-storage.backend":    "local",
+				"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
 			},
 		},
 		{
 			name: "filesystem as storage backend",
 			flags: map[string]string{
 				"-runtime-config.file":    filePath,
 				"-runtime-config.backend": "filesystem",
+				// alert manager
+				"-alertmanager.web.external-url":   "http://localhost/alertmanager",
+				"-alertmanager-storage.backend":    "local",
+				"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
 			},
 		},
 	}
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
 
 	for i, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -79,7 +89,14 @@ func TestLoadRuntimeConfigFromCloudStorage(t *testing.T) {
 		"-runtime-config.s3.insecure":          "true",
 		"-runtime-config.file":                 configFileName,
 		"-runtime-config.reload-period":        "2s",
+		// alert manager
+		"-alertmanager.web.external-url":   "http://localhost/alertmanager",
+		"-alertmanager-storage.backend":    "local",
+		"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
 	}
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
+
 	// create s3 storage backend
 	minio := e2edb.NewMinio(9000, bucketName)
 	require.NoError(t, s.StartAndWaitReady(minio))