diff --git a/CHANGELOG.md b/CHANGELOG.md index c8c3f95425..6232653467 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,11 +38,13 @@ * [ENHANCEMENT] StoreGateway: Add new `cortex_bucket_store_chunk_pool_inuse_bytes` metric to track the usage in chunk pool. #6310 * [BUGFIX] Runtime-config: Handle absolute file paths when working directory is not / #6224 * [BUGFIX] Ruler: Allow rule evaluation to complete during shutdown. #6326 +* [BUGFIX] Ring: update ring with new ip address when instance is lost, rejoins, but heartbeat is disabled #6271 ## 1.18.1 2024-10-14 * [BUGFIX] Backporting upgrade to go 1.22.7 to patch CVE-2024-34155, CVE-2024-34156, CVE-2024-34158 #6217 #6264 + ## 1.18.0 2024-09-03 * [CHANGE] Ingester: Remove `-querier.query-store-for-labels-enabled` flag. Querying long-term store for labels is always enabled. #5984 diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index 8ad3b494da..d8d1543f41 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -738,6 +738,9 @@ func (i *Lifecycler) initRing(ctx context.Context) error { level.Info(i.logger).Log("msg", "existing entry found in ring", "state", i.GetState(), "tokens", len(tokens), "ring", i.RingName) + // Update the address if it has changed + instanceDesc.Addr = i.Addr + // Update the ring if the instance has been changed and the heartbeat is disabled. // We dont need to update KV here when heartbeat is enabled as this info will eventually be update on KV // on the next heartbeat diff --git a/pkg/ring/lifecycler_test.go b/pkg/ring/lifecycler_test.go index cdd684dd96..2822695b2e 100644 --- a/pkg/ring/lifecycler_test.go +++ b/pkg/ring/lifecycler_test.go @@ -40,6 +40,14 @@ func testLifecyclerConfig(ringConfig Config, id string) LifecyclerConfig { return lifecyclerConfig } +// testLifecyclerConfigWithAddr creates a LifecyclerConfig with the given address. +// This is useful for testing when we want to set the address to a specific value. +func testLifecyclerConfigWithAddr(ringConfig Config, id string, addr string) LifecyclerConfig { + l := testLifecyclerConfig(ringConfig, id) + l.Addr = addr + return l +} + func checkNormalised(d interface{}, id string) bool { desc, ok := d.(*Desc) return ok && @@ -644,8 +652,8 @@ func TestRestartIngester_DisabledHeartbeat_unregister_on_shutdown_false(t *testi } // Starts Ingester and wait it to became active - startIngesterAndWaitActive := func(ingId string) *Lifecycler { - lifecyclerConfig := testLifecyclerConfig(ringConfig, ingId) + startIngesterAndWaitActive := func(ingId string, addr string) *Lifecycler { + lifecyclerConfig := testLifecyclerConfigWithAddr(ringConfig, ingId, addr) // Disabling heartBeat and unregister_on_shutdown lifecyclerConfig.UnregisterOnShutdown = false lifecyclerConfig.HeartbeatPeriod = 0 @@ -662,10 +670,10 @@ func TestRestartIngester_DisabledHeartbeat_unregister_on_shutdown_false(t *testi // test if the ingester 2 became active after: // * Clean Shutdown (LEAVING after shutdown) // * Crashes while in the PENDING or JOINING state - l1 := startIngesterAndWaitActive("ing1") + l1 := startIngesterAndWaitActive("ing1", "0.0.0.0") defer services.StopAndAwaitTerminated(context.Background(), l1) //nolint:errcheck - l2 := startIngesterAndWaitActive("ing2") + l2 := startIngesterAndWaitActive("ing2", "0.0.0.0") ingesters := poll(func(desc *Desc) bool { return len(desc.Ingesters) == 2 && desc.Ingesters["ing1"].State == ACTIVE && desc.Ingesters["ing2"].State == ACTIVE @@ -684,7 +692,7 @@ func TestRestartIngester_DisabledHeartbeat_unregister_on_shutdown_false(t *testi assert.Equal(t, LEAVING, ingesters["ing2"].State) // Start Ingester2 again - Should flip back to ACTIVE in the ring - l2 = startIngesterAndWaitActive("ing2") + l2 = startIngesterAndWaitActive("ing2", "0.0.0.0") require.NoError(t, services.StopAndAwaitTerminated(context.Background(), l2)) // Simulate ingester2 crash on startup and left the ring with JOINING state @@ -698,7 +706,7 @@ func TestRestartIngester_DisabledHeartbeat_unregister_on_shutdown_false(t *testi }) require.NoError(t, err) - l2 = startIngesterAndWaitActive("ing2") + l2 = startIngesterAndWaitActive("ing2", "0.0.0.0") require.NoError(t, services.StopAndAwaitTerminated(context.Background(), l2)) // Simulate ingester2 crash on startup and left the ring with PENDING state @@ -712,7 +720,26 @@ func TestRestartIngester_DisabledHeartbeat_unregister_on_shutdown_false(t *testi }) require.NoError(t, err) - l2 = startIngesterAndWaitActive("ing2") + l2 = startIngesterAndWaitActive("ing2", "0.0.0.0") + require.NoError(t, services.StopAndAwaitTerminated(context.Background(), l2)) + + // Simulate ingester2 crashing and left the ring with ACTIVE state, but when it comes up + // it has a different ip address + startIngesterAndWaitActive("ing2", "0.0.0.0") + ingesters = poll(func(desc *Desc) bool { + return desc.Ingesters["ing2"].State == ACTIVE && desc.Ingesters["ing2"].Addr == "0.0.0.0:1" + }) + assert.Equal(t, ACTIVE, ingesters["ing2"].State) + assert.Equal(t, "0.0.0.0:1", ingesters["ing2"].Addr) + + l2 = startIngesterAndWaitActive("ing2", "1.1.1.1") + + // The ring should have the new ip address + ingesters = poll(func(desc *Desc) bool { + return desc.Ingesters["ing2"].State == ACTIVE && desc.Ingesters["ing2"].Addr == "1.1.1.1:1" + }) + assert.Equal(t, ACTIVE, ingesters["ing2"].State) + assert.Equal(t, "1.1.1.1:1", ingesters["ing2"].Addr) require.NoError(t, services.StopAndAwaitTerminated(context.Background(), l2)) }