diff --git a/CHANGELOG.md b/CHANGELOG.md index bdb62a2330d..9afce650ec7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ * [CHANGE] General: enabled `-log.buffered` by default. The `-log.buffered` has been deprecated and will be removed in Mimir 2.13. #6131 * [CHANGE] Ingester: changed default `-blocks-storage.tsdb.series-hash-cache-max-size-bytes` setting from `1GB` to `350MB`. The new default cache size is enough to store the hashes for all series in a ingester, assuming up to 2M in-memory series per ingester and using the default 13h retention period for local TSDB blocks in the ingesters. #6129 * [CHANGE] Query-frontend: removed `cortex_query_frontend_workers_enqueued_requests_total`. Use `cortex_query_frontend_enqueue_duration_seconds_count` instead. #6121 +* [CHANGE] Ingester / querier: enable ingester to querier chunks streaming by default and mark it as stable. #6174 +* [CHANGE] Ingester / querier: enable ingester query request minimisation by default and mark it as stable. #6174 * [CHANGE] Ingester: changed the default value for the experimental configuration parameter `-blocks-storage.tsdb.early-head-compaction-min-estimated-series-reduction-percentage` from 10 to 15. #6186 * [CHANGE] Update Go version to 1.21.2. #6244 * [FEATURE] Query-frontend: add experimental support for query blocking. Queries are blocked on a per-tenant basis and is configured via the limit `blocked_queries`. #5609 diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index f7e64e58b85..1abf4a29ec7 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -1722,7 +1722,7 @@ "required": false, "desc": "Request ingesters stream chunks. Ingesters will only respond with a stream of chunks if the target ingester supports this, and this preference will be ignored by ingesters that do not support this.", "fieldValue": null, - "fieldDefaultValue": false, + "fieldDefaultValue": true, "fieldFlag": "querier.prefer-streaming-chunks-from-ingesters", "fieldType": "boolean", "fieldCategory": "experimental" @@ -1747,7 +1747,7 @@ "fieldDefaultValue": 256, "fieldFlag": "querier.streaming-chunks-per-ingester-buffer-size", "fieldType": "int", - "fieldCategory": "experimental" + "fieldCategory": "advanced" }, { "kind": "field", @@ -1766,7 +1766,7 @@ "required": false, "desc": "If true, when querying ingesters, only the minimum required ingesters required to reach quorum will be queried initially, with other ingesters queried only if needed due to failures from the initial set of ingesters. Enabling this option reduces resource consumption for the happy path at the cost of increased latency for the unhappy path.", "fieldValue": null, - "fieldDefaultValue": false, + "fieldDefaultValue": true, "fieldFlag": "querier.minimize-ingester-requests", "fieldType": "boolean", "fieldCategory": "experimental" @@ -1780,7 +1780,7 @@ "fieldDefaultValue": 3000000000, "fieldFlag": "querier.minimize-ingester-requests-hedging-delay", "fieldType": "duration", - "fieldCategory": "experimental" + "fieldCategory": "advanced" }, { "kind": "field", diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index f1836968c69..fca3d36f79d 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -1660,11 +1660,11 @@ Usage of ./cmd/mimir/mimir: -querier.max-samples int Maximum number of samples a single query can load into memory. This config option should be set on query-frontend too when query sharding is enabled. (default 50000000) -querier.minimize-ingester-requests - [experimental] If true, when querying ingesters, only the minimum required ingesters required to reach quorum will be queried initially, with other ingesters queried only if needed due to failures from the initial set of ingesters. Enabling this option reduces resource consumption for the happy path at the cost of increased latency for the unhappy path. + [experimental] If true, when querying ingesters, only the minimum required ingesters required to reach quorum will be queried initially, with other ingesters queried only if needed due to failures from the initial set of ingesters. Enabling this option reduces resource consumption for the happy path at the cost of increased latency for the unhappy path. (default true) -querier.minimize-ingester-requests-hedging-delay duration - [experimental] Delay before initiating requests to further ingesters when request minimization is enabled and the initially selected set of ingesters have not all responded. Ignored if -querier.minimize-ingester-requests is not enabled. (default 3s) + Delay before initiating requests to further ingesters when request minimization is enabled and the initially selected set of ingesters have not all responded. Ignored if -querier.minimize-ingester-requests is not enabled. (default 3s) -querier.prefer-streaming-chunks-from-ingesters - [experimental] Request ingesters stream chunks. Ingesters will only respond with a stream of chunks if the target ingester supports this, and this preference will be ignored by ingesters that do not support this. + [experimental] Request ingesters stream chunks. Ingesters will only respond with a stream of chunks if the target ingester supports this, and this preference will be ignored by ingesters that do not support this. (default true) -querier.prefer-streaming-chunks-from-store-gateways [experimental] Request store-gateways stream chunks. Store-gateways will only respond with a stream of chunks if the target store-gateway supports this, and this preference will be ignored by store-gateways that do not support this. -querier.query-ingesters-within duration @@ -1692,7 +1692,7 @@ Usage of ./cmd/mimir/mimir: -querier.store-gateway-client.tls-server-name string Override the expected name on the server certificate. -querier.streaming-chunks-per-ingester-buffer-size uint - [experimental] Number of series to buffer per ingester when streaming chunks from ingesters. (default 256) + Number of series to buffer per ingester when streaming chunks from ingesters. (default 256) -querier.streaming-chunks-per-store-gateway-buffer-size uint [experimental] Number of series to buffer per store-gateway when streaming chunks from store-gateways. (default 256) -querier.timeout duration diff --git a/docs/sources/mimir/configure/about-versioning.md b/docs/sources/mimir/configure/about-versioning.md index b7898bfe4ee..7613c0a0020 100644 --- a/docs/sources/mimir/configure/about-versioning.md +++ b/docs/sources/mimir/configure/about-versioning.md @@ -112,9 +112,9 @@ The following features are currently experimental: - `-ingester.client.circuit-breaker.cooldown-period` - Querier - Use of Redis cache backend (`-blocks-storage.bucket-store.metadata-cache.backend=redis`) - - Streaming chunks from ingester to querier (`-querier.prefer-streaming-chunks-from-ingesters`, `-querier.streaming-chunks-per-ingester-buffer-size`) + - Streaming chunks from ingester to querier (`-querier.prefer-streaming-chunks-from-ingesters`) - Streaming chunks from store-gateway to querier (`-querier.prefer-streaming-chunks-from-store-gateways`, `-querier.streaming-chunks-per-store-gateway-buffer-size`) - - Ingester query request minimisation (`-querier.minimize-ingester-requests`, `-querier.minimize-ingester-requests-hedging-delay`) + - Ingester query request minimisation (`-querier.minimize-ingester-requests`) - Limiting queries based on the estimated number of chunks that will be used (`-querier.max-estimated-fetched-chunks-per-query-multiplier`) - Max concurrency for tenant federated queries (`-tenant-federation.max-concurrent`) - Query-frontend diff --git a/docs/sources/mimir/references/configuration-parameters/index.md b/docs/sources/mimir/references/configuration-parameters/index.md index ad609046af1..b871dedc1bb 100644 --- a/docs/sources/mimir/references/configuration-parameters/index.md +++ b/docs/sources/mimir/references/configuration-parameters/index.md @@ -1217,7 +1217,7 @@ store_gateway_client: # with a stream of chunks if the target ingester supports this, and this # preference will be ignored by ingesters that do not support this. # CLI flag: -querier.prefer-streaming-chunks-from-ingesters -[prefer_streaming_chunks_from_ingesters: | default = false] +[prefer_streaming_chunks_from_ingesters: | default = true] # (experimental) Request store-gateways stream chunks. Store-gateways will only # respond with a stream of chunks if the target store-gateway supports this, and @@ -1225,8 +1225,8 @@ store_gateway_client: # CLI flag: -querier.prefer-streaming-chunks-from-store-gateways [prefer_streaming_chunks_from_store_gateways: | default = false] -# (experimental) Number of series to buffer per ingester when streaming chunks -# from ingesters. +# (advanced) Number of series to buffer per ingester when streaming chunks from +# ingesters. # CLI flag: -querier.streaming-chunks-per-ingester-buffer-size [streaming_chunks_per_ingester_series_buffer_size: | default = 256] @@ -1241,12 +1241,11 @@ store_gateway_client: # ingesters. Enabling this option reduces resource consumption for the happy # path at the cost of increased latency for the unhappy path. # CLI flag: -querier.minimize-ingester-requests -[minimize_ingester_requests: | default = false] +[minimize_ingester_requests: | default = true] -# (experimental) Delay before initiating requests to further ingesters when -# request minimization is enabled and the initially selected set of ingesters -# have not all responded. Ignored if -querier.minimize-ingester-requests is not -# enabled. +# (advanced) Delay before initiating requests to further ingesters when request +# minimization is enabled and the initially selected set of ingesters have not +# all responded. Ignored if -querier.minimize-ingester-requests is not enabled. # CLI flag: -querier.minimize-ingester-requests-hedging-delay [minimize_ingester_requests_hedging_delay: | default = 3s] diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index bba619fdbff..4fd3f40d691 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -52,12 +52,12 @@ type Config struct { ShuffleShardingIngestersEnabled bool `yaml:"shuffle_sharding_ingesters_enabled" category:"advanced"` - PreferStreamingChunksFromIngesters bool `yaml:"prefer_streaming_chunks_from_ingesters" category:"experimental"` + PreferStreamingChunksFromIngesters bool `yaml:"prefer_streaming_chunks_from_ingesters" category:"experimental"` // Enabled by default as of Mimir 2.11, remove altogether in 2.12. PreferStreamingChunksFromStoreGateways bool `yaml:"prefer_streaming_chunks_from_store_gateways" category:"experimental"` - StreamingChunksPerIngesterSeriesBufferSize uint64 `yaml:"streaming_chunks_per_ingester_series_buffer_size" category:"experimental"` + StreamingChunksPerIngesterSeriesBufferSize uint64 `yaml:"streaming_chunks_per_ingester_series_buffer_size" category:"advanced"` StreamingChunksPerStoreGatewaySeriesBufferSize uint64 `yaml:"streaming_chunks_per_store_gateway_series_buffer_size" category:"experimental"` - MinimizeIngesterRequests bool `yaml:"minimize_ingester_requests" category:"experimental"` - MinimiseIngesterRequestsHedgingDelay time.Duration `yaml:"minimize_ingester_requests_hedging_delay" category:"experimental"` + MinimizeIngesterRequests bool `yaml:"minimize_ingester_requests" category:"experimental"` // Enabled by default as of Mimir 2.11, remove altogether in 2.12. + MinimiseIngesterRequestsHedgingDelay time.Duration `yaml:"minimize_ingester_requests_hedging_delay" category:"advanced"` // PromQL engine config. EngineConfig engine.Config `yaml:",inline"` @@ -86,11 +86,11 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.MaxQueryIntoFuture, "querier.max-query-into-future", 10*time.Minute, "Maximum duration into the future you can query. 0 to disable.") f.DurationVar(&cfg.QueryStoreAfter, queryStoreAfterFlag, 12*time.Hour, "The time after which a metric should be queried from storage and not just ingesters. 0 means all queries are sent to store. If this option is enabled, the time range of the query sent to the store-gateway will be manipulated to ensure the query end is not more recent than 'now - query-store-after'.") f.BoolVar(&cfg.ShuffleShardingIngestersEnabled, "querier.shuffle-sharding-ingesters-enabled", true, fmt.Sprintf("Fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since -%s. If this setting is false or -%s is '0', queriers always query all ingesters (ingesters shuffle sharding on read path is disabled).", validation.QueryIngestersWithinFlag, validation.QueryIngestersWithinFlag)) - f.BoolVar(&cfg.PreferStreamingChunksFromIngesters, "querier.prefer-streaming-chunks-from-ingesters", false, "Request ingesters stream chunks. Ingesters will only respond with a stream of chunks if the target ingester supports this, and this preference will be ignored by ingesters that do not support this.") + f.BoolVar(&cfg.PreferStreamingChunksFromIngesters, "querier.prefer-streaming-chunks-from-ingesters", true, "Request ingesters stream chunks. Ingesters will only respond with a stream of chunks if the target ingester supports this, and this preference will be ignored by ingesters that do not support this.") f.BoolVar(&cfg.PreferStreamingChunksFromStoreGateways, "querier.prefer-streaming-chunks-from-store-gateways", false, "Request store-gateways stream chunks. Store-gateways will only respond with a stream of chunks if the target store-gateway supports this, and this preference will be ignored by store-gateways that do not support this.") const minimiseIngesterRequestsFlagName = "querier.minimize-ingester-requests" - f.BoolVar(&cfg.MinimizeIngesterRequests, minimiseIngesterRequestsFlagName, false, "If true, when querying ingesters, only the minimum required ingesters required to reach quorum will be queried initially, with other ingesters queried only if needed due to failures from the initial set of ingesters. Enabling this option reduces resource consumption for the happy path at the cost of increased latency for the unhappy path.") + f.BoolVar(&cfg.MinimizeIngesterRequests, minimiseIngesterRequestsFlagName, true, "If true, when querying ingesters, only the minimum required ingesters required to reach quorum will be queried initially, with other ingesters queried only if needed due to failures from the initial set of ingesters. Enabling this option reduces resource consumption for the happy path at the cost of increased latency for the unhappy path.") f.DurationVar(&cfg.MinimiseIngesterRequestsHedgingDelay, minimiseIngesterRequestsFlagName+"-hedging-delay", 3*time.Second, "Delay before initiating requests to further ingesters when request minimization is enabled and the initially selected set of ingesters have not all responded. Ignored if -"+minimiseIngesterRequestsFlagName+" is not enabled.") // Why 256 series / ingester/store-gateway?