From c53b02149f27a97f38a8c3e314078719c2130cb5 Mon Sep 17 00:00:00 2001 From: liyun95 Date: Tue, 5 Nov 2024 17:57:17 +0800 Subject: [PATCH 01/10] update release notes for 2.4.15 Signed-off-by: liyun95 --- site/en/Variables.json | 6 +++--- site/en/release_notes.md | 30 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/site/en/Variables.json b/site/en/Variables.json index 9332439c0..01e9c84b6 100644 --- a/site/en/Variables.json +++ b/site/en/Variables.json @@ -1,6 +1,6 @@ { - "milvus_release_version": "2.4.14", - "milvus_release_tag": "2.4.14", + "milvus_release_version": "2.4.15", + "milvus_release_tag": "2.4.15", "milvus_deb_name": "milvus_2.2.0-1_amd64", "milvus_rpm_name": "milvus-2.2.0-1.el7.x86_64", "milvus_python_sdk_version": "2.4.x", @@ -10,7 +10,7 @@ "milvus_go_sdk_version": "2.3.x", "milvus_go_sdk_real_version": "2.4.0", "milvus_java_sdk_version": "2.4.x", - "milvus_java_sdk_real_version": "2.4.7", + "milvus_java_sdk_real_version": "2.4.8", "milvus_csharp_sdk_version": "2.2.x", "milvus_csharp_sdk_real_version": "2.2.14", "milvus_restful_sdk_version": "2.4.x", diff --git a/site/en/release_notes.md b/site/en/release_notes.md index 9b6e67370..bc6ee7e74 100644 --- a/site/en/release_notes.md +++ b/site/en/release_notes.md @@ -7,6 +7,36 @@ title: Release Notes Find out what’s new in Milvus! This page summarizes new features, improvements, known issues, and bug fixes in each release. You can find the release notes for each released version after v2.4.0 in this section. We suggest that you regularly visit this page to learn about updates. +## v2.4.15 + +Release Date: November 5, 2024 + +| Milvus version | Python SDK version | Java SDK version | Node.js SDK version | +|----------------|--------------------|------------------|---------------------| +| 2.4.15 | 2.4.9 | 2.4.8 | 2.4.9 | + +Milvus 2.4.15 was a critical bug-fix release focused on enhancing system stability, performance, and compatibility. This version addressed a major deadlock issue that could occur during QueryNode crashes and introduced compatibility updates for the backup tool with the database feature. Additionally, Milvus 2.4.15 improved delete performance and stability through significant optimizations in L0 handling. **Upgrading to v2.4.15 was strongly recommended** to benefit from these critical enhancements. + +### Critical bug fixes + +- Resolved a deadlock issue if the QueryNode crashed during shard client initialization ([#37354](https://github.com/milvus-io/milvus/pull/37354)). +- Reverted the enhancement to support databases for bulk insert ([#37421](https://github.com/milvus-io/milvus/pull/37421)). + +### Bug fixes + +- Fixed a bug where certain expressions did not correctly parse values ([#37342](https://github.com/milvus-io/milvus/pull/37342)). +- Enhanced the Proxy to retry getting the shard leader on unloaded collections ([#37326](https://github.com/milvus-io/milvus/pull/37326)). +- Corrected an issue where the L0 row count metrics value was always empty ([#37307](https://github.com/milvus-io/milvus/pull/37307)). +- Skipped marking compaction timeout for mixed and L0 compaction scenarios ([#37194](https://github.com/milvus-io/milvus/pull/37194)). +- Rectified the containment logic of OffsetOrderedArray ([#37309](https://github.com/milvus-io/milvus/pull/37309)). +- Added a check for resources when loading delta logs ([#37263](https://github.com/milvus-io/milvus/pull/37263)). + +### Improvements + +- Moved L0 logic outside of the delta lock for better performance ([#37340](https://github.com/milvus-io/milvus/pull/37340)). +- Released compacted growing segments if present in the dropped list ([#37266](https://github.com/milvus-io/milvus/pull/37266)). +- Introduced middleware to monitor RESTful V2 input/output RPC stats ([#37224](https://github.com/milvus-io/milvus/pull/37224), [#37440](https://github.com/milvus-io/milvus/pull/37440)). + ## v2.4.14 Release Date: October 31, 2024 From 50cb52eef6d076bceb108295b4b0f6cc4a75ddbd Mon Sep 17 00:00:00 2001 From: ChengZi Date: Thu, 7 Nov 2024 17:21:01 +0800 Subject: [PATCH 02/10] Update RAG_with_pii_and_milvus.md optimize description in PII masker --- site/en/integrations/RAG_with_pii_and_milvus.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/site/en/integrations/RAG_with_pii_and_milvus.md b/site/en/integrations/RAG_with_pii_and_milvus.md index dd782b5e1..3a53e50ab 100644 --- a/site/en/integrations/RAG_with_pii_and_milvus.md +++ b/site/en/integrations/RAG_with_pii_and_milvus.md @@ -10,9 +10,8 @@ PII (Personally Identifiable Information) is a type of sensitive data that can b [PII Masker](https://github.com/HydroXai/pii-masker-v1/tree/main), developed by [HydroX AI](https://www.hydrox.ai/), is an advanced open-source tool designed to protect your sensitive data by leveraging cutting-edge AI models. Whether you're handling customer data, performing data analysis, or ensuring compliance with privacy regulations, PII Masker provides a robust, scalable solution to keep your information secure. -In this tutorial, we will show you how to build a RAG(Retrieval-Augmented Generation) pipeline with Milvus and PII Masker. +In this tutorial, we will show how to use PII Masker with Milvus to protect private data in RAG(Retrieval-Augmented Generation) applications. By combining the strengths of PII Masker's data masking capabilities with Milvus's efficient data retrieval, you can create secure, privacy-compliant pipelines for handling sensitive information with confidence. This approach ensures your applications are equipped to meet privacy standards and protect user data effectively. -This effectively protects PII data. ## Preparation From 331d16abb6cf9a5cf96b4772414dbb05d39c0741 Mon Sep 17 00:00:00 2001 From: tinswzy Date: Thu, 7 Nov 2024 21:12:01 +0800 Subject: [PATCH 03/10] update manage-partitions.md --- site/en/userGuide/manage-partitions.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/site/en/userGuide/manage-partitions.md b/site/en/userGuide/manage-partitions.md index 5ff3c5055..4ee2a2d2a 100644 --- a/site/en/userGuide/manage-partitions.md +++ b/site/en/userGuide/manage-partitions.md @@ -719,7 +719,7 @@ client.load_partitions( partition_names=["partitionA", "partitionB"] ) -res = client.get_load_status( +res = client.get_load_state( collection_name="quick_setup", partition_name="partitionA" ) @@ -730,7 +730,7 @@ res = client.get_load_status( # "state": "" # } -res = client.get_load_status( +res = client.get_load_state( collection_name="quick_setup", partition_name="partitionB" ) @@ -921,7 +921,7 @@ client.release_partitions( partition_names=["_default", "partitionA", "partitionB"] ) -res = client.get_load_status( +res = client.get_load_state( collection_name="quick_setup", ) From 79455054400ffce7844af69f388a67ba93350312 Mon Sep 17 00:00:00 2001 From: tinswzy Date: Thu, 7 Nov 2024 21:21:53 +0800 Subject: [PATCH 04/10] update index-scalar-fields document --- site/en/userGuide/manage-indexes/index-scalar-fields.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/site/en/userGuide/manage-indexes/index-scalar-fields.md b/site/en/userGuide/manage-indexes/index-scalar-fields.md index 3fec56853..4985dd081 100644 --- a/site/en/userGuide/manage-indexes/index-scalar-fields.md +++ b/site/en/userGuide/manage-indexes/index-scalar-fields.md @@ -49,7 +49,7 @@ client = MilvusClient( uri="http://localhost:19530" ) -index_params = client.create_index_params() # Prepare an empty IndexParams object, without having to specify any index parameters +index_params = MilvusClient.prepare_index_params() # Prepare an empty IndexParams object, without having to specify any index parameters index_params.add_index( field_name="scalar_1", # Name of the scalar field to be indexed @@ -122,7 +122,7 @@ The example below creates an inverted index for the scalar field `scalar_2`. ```python -index_params = client.create_index_params() # Prepare an IndexParams object +index_params = MilvusClient.prepare_index_params() # Prepare an IndexParams object index_params.add_index( field_name="scalar_2", # Name of the scalar field to be indexed @@ -170,7 +170,7 @@ await client.createIndex({ __Methods and Parameters__ -- __create_index_params()__ +- __prepare_index_params()__ Prepares an __IndexParams__ object. From 3cc38e75f6e7bf1660b9bfc6f4fcea237e36004c Mon Sep 17 00:00:00 2001 From: tinswzy Date: Thu, 7 Nov 2024 21:28:55 +0800 Subject: [PATCH 05/10] update search-query-get docs --- .../search-query-get/multi-vector-search.md | 2 +- .../search-query-get/single-vector-search.md | 18 ++++++++++-------- .../search-query-get/with-iterators.md | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/site/en/userGuide/search-query-get/multi-vector-search.md b/site/en/userGuide/search-query-get/multi-vector-search.md index 25e9f0ca0..bbaa82f84 100644 --- a/site/en/userGuide/search-query-get/multi-vector-search.md +++ b/site/en/userGuide/search-query-get/multi-vector-search.md @@ -37,7 +37,7 @@ import random # Connect to Milvus connections.connect( - host="10.102.7.3", # Replace with your Milvus server IP + host="127.0.0.1", # Replace with your Milvus server IP port="19530" ) diff --git a/site/en/userGuide/search-query-get/single-vector-search.md b/site/en/userGuide/search-query-get/single-vector-search.md index fb9b199b5..62f26c292 100644 --- a/site/en/userGuide/search-query-get/single-vector-search.md +++ b/site/en/userGuide/search-query-get/single-vector-search.md @@ -39,10 +39,12 @@ The code snippet below repurposes the existing code to establish a connection to ```python +from pymilvus import MilvusClient +import random + # 1. Set up a Milvus client client = MilvusClient( - uri=CLUSTER_ENDPOINT, - token=TOKEN + uri="http://localhost:19530" ) # 2. Create a collection @@ -424,7 +426,7 @@ Here is an example of searching for the top 5 entities that are most similar to ```python # Single vector search res = client.search( - collection_name="test_collection", # Replace with the actual name of your collection + collection_name="quick_setup", # Replace with the actual name of your collection # Replace with your query vector data=[[0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]], limit=5, # Max. number of search results to return @@ -648,7 +650,7 @@ Here is an example of searching for two distinct sets of the most similar entiti ```python # Bulk-vector search res = client.search( - collection_name="test_collection", # Replace with the actual name of your collection + collection_name="quick_setup", # Replace with the actual name of your collection data=[ [0.19886812562848388, 0.06023560599112088, 0.6976963061752597, 0.2614474506242501, 0.838729485096104], [0.3172005263489739, 0.9719044792798428, -0.36981146090600725, -0.4860894583077995, 0.95791889146345] @@ -1309,7 +1311,7 @@ Here is an example of returning results with `color` attribute values: ```python # Search with output fields res = client.search( - collection_name="test_collection", # Replace with the actual name of your collection + collection_name="quick_setup", # Replace with the actual name of your collection data=[[0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]], limit=5, # Max. number of search results to return search_params={"metric_type": "IP", "params": {}}, # Search parameters @@ -1474,7 +1476,7 @@ Filter results whose __color__ is prefixed with __red__: ```python # Search with filter res = client.search( - collection_name="test_collection", # Replace with the actual name of your collection + collection_name="quick_setup", # Replace with the actual name of your collection data=[[0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]], limit=5, # Max. number of search results to return search_params={"metric_type": "IP", "params": {}}, # Search parameters @@ -1603,7 +1605,7 @@ Filter results whose __color__ contains the letters __ll__ anywhere within the s ```python # Infix match on color field res = client.search( - collection_name="test_collection", # Replace with the actual name of your collection + collection_name="quick_setup", # Replace with the actual name of your collection data=[[0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]], limit=5, # Max. number of search results to return search_params={"metric_type": "IP", "params": {}}, # Search parameters @@ -1714,7 +1716,7 @@ search_params = { } res = client.search( - collection_name="test_collection", # Replace with the actual name of your collection + collection_name="quick_setup", # Replace with the actual name of your collection data=[[0.3580376395471989, -0.6023495712049978, 0.18414012509913835, -0.26286205330961354, 0.9029438446296592]], limit=3, # Max. number of search results to return search_params=search_params, # Search parameters diff --git a/site/en/userGuide/search-query-get/with-iterators.md b/site/en/userGuide/search-query-get/with-iterators.md index 64ab6b3b4..1b0c974cd 100644 --- a/site/en/userGuide/search-query-get/with-iterators.md +++ b/site/en/userGuide/search-query-get/with-iterators.md @@ -249,7 +249,7 @@ To search with an iterator, call the [searchIterator()](https://milvus.io/api-re ```python -from pymilvus import Collection +from pymilvus import Collection,connections # 4. Search with iterator connections.connect(host="127.0.0.1", port=19530) From 5ce48a95247046070a4c3e4c1ce0ca8c63df177d Mon Sep 17 00:00:00 2001 From: AnthonyTsu1984 <115786031+AnthonyTsu1984@users.noreply.github.com> Date: Fri, 8 Nov 2024 09:31:45 +0800 Subject: [PATCH 06/10] update errors Signed-off-by: AnthonyTsu1984 <115786031+AnthonyTsu1984@users.noreply.github.com> --- site/en/menuStructure/en.json | 12 ++++++------ .../en/tutorials/movie_recommendation_with_milvus.md | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/site/en/menuStructure/en.json b/site/en/menuStructure/en.json index 6d817f5de..1f8ec635c 100644 --- a/site/en/menuStructure/en.json +++ b/site/en/menuStructure/en.json @@ -1368,12 +1368,6 @@ "id": "milvus_hybrid_search_retriever.md", "order": 3, "children": [] - }, - { - "label": "PrivateGPT", - "id": "use_milvus_in_private_gpt.md", - "order": 4, - "children": [] } ] }, @@ -1425,6 +1419,12 @@ "id": "rag_with_langflow.md", "order": 7, "children": [] + }, + { + "label": "PrivateGPT", + "id": "use_milvus_in_private_gpt.md", + "order": 8, + "children": [] } ] }, diff --git a/site/en/tutorials/movie_recommendation_with_milvus.md b/site/en/tutorials/movie_recommendation_with_milvus.md index 64cc0df4a..6ba08ce5d 100644 --- a/site/en/tutorials/movie_recommendation_with_milvus.md +++ b/site/en/tutorials/movie_recommendation_with_milvus.md @@ -32,7 +32,7 @@ If you are using Google Colab, to enable dependencies just installed, you may ne We will use OpenAI as the LLM in this example. You should prepare the [api key](https://platform.openai.com/docs/quickstart) `OPENAI_API_KEY` as an environment variable. - + ```python From 9cfb19ca782667f9b34868051bbcc443a1574665 Mon Sep 17 00:00:00 2001 From: AnthonyTsu1984 <115786031+AnthonyTsu1984@users.noreply.github.com> Date: Mon, 11 Nov 2024 13:48:38 +0800 Subject: [PATCH 07/10] Update disk_index.md --- site/en/reference/disk_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/en/reference/disk_index.md b/site/en/reference/disk_index.md index 1a5f52be3..75d681d63 100644 --- a/site/en/reference/disk_index.md +++ b/site/en/reference/disk_index.md @@ -18,7 +18,7 @@ Currently, a vector field only supports one index type. Milvus automatically del ## Prerequisites To use DiskANN, note that -- DiskANN is enabled by default. If you prefer in-memory index over on-disk index, you are advised to disable this feature for a better performance. +- DiskANN is disabled by default. If you prefer in-memory index over on-disk index, you are advised to disable this feature for a better performance. - To disable it, you can change `queryNode.enableDisk` to `false` in your milvus configuration file. - To enable it again, you can set `queryNode.enableDisk` to `true`. - The Milvus instance runs on Ubuntu 18.04.6 or a later release. From 178cb665f3a468e07241849446c4d7e4ad90ecf8 Mon Sep 17 00:00:00 2001 From: codingjaguar Date: Mon, 11 Nov 2024 14:48:39 +0800 Subject: [PATCH 08/10] Update product_faq.md to correct the description about load and data freshness. --- site/en/faq/product_faq.md | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/site/en/faq/product_faq.md b/site/en/faq/product_faq.md index 9e3951ca1..60cfbd14e 100644 --- a/site/en/faq/product_faq.md +++ b/site/en/faq/product_faq.md @@ -21,14 +21,6 @@ Milvus cannot be installed or run on non-x86 platforms. Your CPU must support one of the following instruction sets to run Milvus: SSE4.2, AVX, AVX2, AVX512. These are all x86-dedicated SIMD instruction sets. -#### What is the maximum dataset size Milvus can handle? - - -Theoretically, the maximum dataset size Milvus can handle is determined by the hardware it is run on, specifically system memory and storage: - -- Milvus loads all specified collections and partitions into memory before running queries. Therefore, memory size determines the maximum amount of data Milvus can query. -- When new entities and and collection-related schema (currently only MinIO is supported for data persistence) are added to Milvus, system storage determines the maximum allowable size of inserted data. - #### Where does Milvus store data? Milvus deals with two types of data, inserted data and metadata. @@ -67,13 +59,13 @@ An insert operation must not exceed 1,024 MB in size. This is a limit imposed by No. If partitions for a search are specified, Milvus searches the specified partitions only. -#### Does Milvus load the entire collection when partitions are specified for a search? +#### Does Milvus need to load the entire collection when partitions are specified for a search? -No. Milvus has varied behavior. Data must be loaded to memory before searching. +It depends on what data is needed for search. All partitions potentially show up in search result must be loaded before searching. -- If you know which partitions your data are located in, call `load_partition()` to load the intended partition(s) *then* specify partition(s) in the `search()` method call. -- If you do not know the exact partitions, call `load_collection()` before calling `search()`. -- If you fail to load collections or partitions before searching, Milvus returns an error. +- For example, if you only want to search specific parition(s), you don't need to load all. Call `load_partition()` to load the intended partition(s) *then* specify partition(s) in the `search()` method call. +- If you want to search all partitions, call `load_collection()` to load the whole collection including all partitions. +- If you fail to load the collection or specific partition(s) before searching, Milvus will return an error. #### Can indexes be created after inserting vectors? @@ -89,7 +81,7 @@ See [Vector Index](index.md) for more information. #### How does Milvus flush data? -Milvus returns success when inserted data are loaded to the message queue. However, the data are not yet flushed to the disk. Then Milvus' data node writes the data in the message queue to persistent storage as incremental logs. If `flush()` is called, the data node is forced to write all data in the message queue to persistent storage immediately. +Milvus returns success when inserted data are ingested to the message queue. However, the data are not yet flushed to the disk. Then Milvus' data node writes the data in the message queue to persistent storage as incremental logs. If `flush()` is called, the data node is forced to write all data in the message queue to persistent storage immediately. #### What is normalization? Why is normalization needed? @@ -137,9 +129,9 @@ In current release, Milvus supports both INT64 and string. Yes. You can deploy Milvus cluster with multiple nodes via Helm Chart on Kubernetes. Refer to [Scale Guide](scaleout.md) for more instruction. -#### Does the query perform in memory? What are incremental data and historical data? +#### What are growing segment and sealed segment? -Yes. When a query request comes, Milvus searches both incremental data and historical data by loading them into memory. Incremental data are in the growing segments, which are buffered in memory before they reach the threshold to be persisted in storage engine, while historical data are from the sealed segments that are stored in the object storage. Incremental data and historical data together constitute the whole dataset to search. +When a search request comes, Milvus searches both incremental data and historical data. Incremental data are recent updates, they are stored in the growing segments, which are buffered in memory before they reach the threshold to be persisted in object storage and a more efficient index is built for them, while historical data are updates a while ago. They are in the sealed segments which have been persisted in the object storage. Incremental data and historical data together constitute the whole dataset for search. This design makes any data ingested to Milvus instantly searchable. For Milvus Distributed, there are more complex factors that decide when a record just ingested can show up in search result. Learn more nuance about that at [consistency levels](https://milvus.io/docs/consistency.md). #### Is Milvus available for concurrent search? From ea71ebed1bf18dfe4ab40032341b6f1c807c007e Mon Sep 17 00:00:00 2001 From: AnthonyTsu1984 <115786031+AnthonyTsu1984@users.noreply.github.com> Date: Tue, 12 Nov 2024 18:37:30 +0800 Subject: [PATCH 09/10] update docs Signed-off-by: AnthonyTsu1984 <115786031+AnthonyTsu1984@users.noreply.github.com> --- site/en/menuStructure/en.json | 12 ++++++------ site/en/reference/users_and_roles.md | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/site/en/menuStructure/en.json b/site/en/menuStructure/en.json index 6d817f5de..2056bb87c 100644 --- a/site/en/menuStructure/en.json +++ b/site/en/menuStructure/en.json @@ -304,12 +304,6 @@ "order": 9, "children": [] }, - { - "label": "Users and Roles", - "id": "users_and_roles.md", - "order": 10, - "children": [] - }, { "label": "In-memory Replica", "id": "replica.md", @@ -1057,6 +1051,12 @@ "order": 0, "children": [] }, + { + "label": "Users, Privileges, and Roles", + "id": "users_and_roles.md", + "order": 10, + "children": [] + }, { "label": "Enable RBAC", "id": "rbac.md", diff --git a/site/en/reference/users_and_roles.md b/site/en/reference/users_and_roles.md index 3614fb56d..c9f8f01b4 100644 --- a/site/en/reference/users_and_roles.md +++ b/site/en/reference/users_and_roles.md @@ -5,7 +5,7 @@ summary: Learn about the definition of users, roles, objects, and privileges in title: Users and Roles --- -# Users and Roles +# Users, Privileges, and Roles This topic provides an overview of Role-Based Access Control (RBAC) in Milvus, detailing the definitions and relationships between users, roles, objects, and privileges. From 0ad59543b99a4c071b72a32b7dea031d63cfd643 Mon Sep 17 00:00:00 2001 From: AnthonyTsu1984 <115786031+AnthonyTsu1984@users.noreply.github.com> Date: Tue, 12 Nov 2024 18:41:18 +0800 Subject: [PATCH 10/10] update docs Signed-off-by: AnthonyTsu1984 <115786031+AnthonyTsu1984@users.noreply.github.com> --- site/en/reference/users_and_roles.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/en/reference/users_and_roles.md b/site/en/reference/users_and_roles.md index c9f8f01b4..65b994561 100644 --- a/site/en/reference/users_and_roles.md +++ b/site/en/reference/users_and_roles.md @@ -2,7 +2,7 @@ id: users_and_roles.md related_key: users, roles summary: Learn about the definition of users, roles, objects, and privileges in role-based access control (RBAC). -title: Users and Roles +title: Users, Privileges, and Roles --- # Users, Privileges, and Roles