Remove filter out serverless cluster and add support to extract index…

… name Allow extract index name for both serverless and non-serverless clusters Allow different key formats: - datasource-id::TIMESERIES::<index-name>:0 - datasource-id::<index-name>:0 - <index-name> (non-serverless case) Signed-off-by: Anan Zhuang <[email protected]>
opensearch-project · Nov 15, 2024 · 55dc623 · 55dc623
1 parent 413697d
commit 55dc623
Show file tree

Hide file tree

Showing 4 changed files with 87 additions and 86 deletions.
diff --git a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.test.ts b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.test.ts
@@ -10,11 +10,18 @@ import { SavedObjectsClientContract } from 'opensearch-dashboards/public';
 import { DATA_STRUCTURE_META_TYPES, DataStructure, Dataset } from '../../../../../common';
 import * as services from '../../../../services';
 import { IDataPluginServices } from 'src/plugins/data/public';
+import { of } from 'rxjs';
 
 jest.mock('../../../../services', () => {
+  const mockSearchFunction = jest.fn();
+
   return {
-    getSearchService: jest.fn(),
     getIndexPatterns: jest.fn(),
+    getSearchService: jest.fn(() => ({
+      getDefaultSearchInterceptor: () => ({
+        search: mockSearchFunction,
+      }),
+    })),
     getQueryService: () => ({
       queryString: {
         getLanguageService: () => ({
@@ -90,9 +97,7 @@ describe('indexTypeConfig', () => {
 
   test('should fetch data sources for unknown type', async () => {
     mockSavedObjectsClient.find = jest.fn().mockResolvedValue({
-      savedObjects: [
-        { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '3.0' } },
-      ],
+      savedObjects: [{ id: 'ds1', attributes: { title: 'DataSource 1' } }],
     });
 
     const result = await indexTypeConfig.fetch(mockServices as IDataPluginServices, [
@@ -104,30 +109,54 @@ describe('indexTypeConfig', () => {
     expect(result.hasNext).toBe(true);
   });
 
-  test('should filter out data sources with versions lower than 1.0.0', async () => {
-    mockSavedObjectsClient.find = jest.fn().mockResolvedValue({
-      savedObjects: [
-        { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '1.0' } },
-        {
-          id: 'ds2',
-          attributes: { title: 'DataSource 2', dataSourceVersion: '' },
+  describe('fetchIndices', () => {
+    test('should extract index names correctly from different formats', async () => {
+      const mockResponse = {
+        rawResponse: {
+          aggregations: {
+            indices: {
+              buckets: [
+                // Serverless format with TIMESERIES
+                { key: '123::TIMESERIES::sample-index-1:0' },
+                // Serverless format without TIMESERIES
+                { key: '123::sample-index-2:0' },
+                // Non-serverless format
+                { key: 'simple-index' },
+              ],
+            },
+          },
         },
-        { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceVersion: '2.17.0' } },
-        {
-          id: 'ds4',
-          attributes: { title: 'DataSource 4', dataSourceVersion: '.0' },
-        },
-      ],
+      };
+
+      const searchService = services.getSearchService();
+      const interceptor = searchService.getDefaultSearchInterceptor();
+      (interceptor.search as jest.Mock).mockReturnValue(of(mockResponse));
+
+      const result = await indexTypeConfig.fetch(mockServices as IDataPluginServices, [
+        { id: 'datasource1', title: 'DataSource 1', type: 'DATA_SOURCE' },
+      ]);
+
+      expect(result.children).toEqual([
+        { id: 'datasource1::sample-index-1', title: 'sample-index-1', type: 'INDEX' },
+        { id: 'datasource1::sample-index-2', title: 'sample-index-2', type: 'INDEX' },
+        { id: 'datasource1::simple-index', title: 'simple-index', type: 'INDEX' },
+      ]);
     });
 
-    const result = await indexTypeConfig.fetch(mockServices as IDataPluginServices, [
-      { id: 'unknown', title: 'Unknown', type: 'UNKNOWN' },
-    ]);
+    test('should handle response without aggregations', async () => {
+      const mockResponse = {
+        rawResponse: {},
+      };
 
-    expect(result.children).toHaveLength(2);
-    expect(result.children?.[0].title).toBe('DataSource 1');
-    expect(result.children?.[1].title).toBe('DataSource 3');
-    expect(result.children?.some((child) => child.title === 'DataSource 2')).toBe(false);
-    expect(result.hasNext).toBe(true);
+      const searchService = services.getSearchService();
+      const interceptor = searchService.getDefaultSearchInterceptor();
+      (interceptor.search as jest.Mock).mockReturnValue(of(mockResponse));
+
+      const result = await indexTypeConfig.fetch(mockServices as IDataPluginServices, [
+        { id: 'datasource1', title: 'DataSource 1', type: 'DATA_SOURCE' },
+      ]);
+
+      expect(result.children).toEqual([]);
+    });
   });
 });
diff --git a/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts b/src/plugins/data/public/query/query_string/dataset_service/lib/index_type.ts
@@ -6,7 +6,6 @@
 import { SavedObjectsClientContract } from 'opensearch-dashboards/public';
 import { map } from 'rxjs/operators';
 import { i18n } from '@osd/i18n';
-import semver from 'semver';
 import {
   DEFAULT_DATA,
   DataStructure,
@@ -120,16 +119,11 @@ const fetchDataSources = async (client: SavedObjectsClientContract) => {
     type: 'data-source',
     perPage: 10000,
   });
-  const dataSources: DataStructure[] = response.savedObjects
-    .filter((savedObject) => {
-      const coercedVersion = semver.coerce(savedObject.attributes.dataSourceVersion);
-      return coercedVersion ? semver.satisfies(coercedVersion, '>=1.0.0') : false;
-    })
-    .map((savedObject) => ({
-      id: savedObject.id,
-      title: savedObject.attributes.title,
-      type: 'DATA_SOURCE',
-    }));
+  const dataSources: DataStructure[] = response.savedObjects.map((savedObject) => ({
+    id: savedObject.id,
+    title: savedObject.attributes.title,
+    type: 'DATA_SOURCE',
+  }));
 
   return injectMetaToDataStructures(dataSources);
 };
@@ -158,9 +152,22 @@ const fetchIndices = async (dataStructure: DataStructure): Promise<string[]> =>
 
   const searchResponseToArray = (response: any) => {
     const { rawResponse } = response;
-    return rawResponse.aggregations
-      ? rawResponse.aggregations.indices.buckets.map((bucket: { key: any }) => bucket.key)
-      : [];
+    if (!rawResponse.aggregations) {
+      return [];
+    }
+
+    return rawResponse.aggregations.indices.buckets.map((bucket: { key: string }) => {
+      const key = bucket.key;
+      // Handle the case of serverless cluster where key format is either:
+      // - datasource-id::TIMESERIES::<index-name>:0
+      // - datasource-id::<index-name>:0
+      // Note: Index names cannot contain ':' or '::' in OpenSearch, so these delimiters
+      // are guaranteed to be part of the serverless format, not the index name
+      const parts = key.split('::');
+      const lastPart = parts[parts.length - 1] || '';
+      // extract index name or return original key if pattern doesn't match
+      return lastPart.split(':')[0] || key;
+    });
   };
 
   return search

diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.test.ts b/src/plugins/query_enhancements/public/datasets/s3_type.test.ts
@@ -141,9 +141,7 @@ describe('s3TypeConfig', () => {
 
     it('should fetch data sources for unknown type', async () => {
       mockSavedObjectsClient.find = jest.fn().mockResolvedValue({
-        savedObjects: [
-          { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '3.0' } },
-        ],
+        savedObjects: [{ id: 'ds1', attributes: { title: 'DataSource 1' } }],
       });
 
       const result = await s3TypeConfig.fetch(mockServices as IDataPluginServices, [
@@ -154,33 +152,6 @@ describe('s3TypeConfig', () => {
       expect(result.children?.[0].title).toBe('DataSource 1');
       expect(result.hasNext).toBe(true);
     });
-
-    it('should filter out data sources with versions lower than 1.0.0', async () => {
-      mockSavedObjectsClient.find = jest.fn().mockResolvedValue({
-        savedObjects: [
-          { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '1.0' } },
-          {
-            id: 'ds2',
-            attributes: { title: 'DataSource 2', dataSourceVersion: '' },
-          },
-          { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceVersion: '2.17.0' } },
-          {
-            id: 'ds4',
-            attributes: { title: 'DataSource 4', dataSourceVersion: '.0' },
-          },
-        ],
-      });
-
-      const result = await s3TypeConfig.fetch(mockServices as IDataPluginServices, [
-        { id: 'unknown', title: 'Unknown', type: 'UNKNOWN' },
-      ]);
-
-      expect(result.children).toHaveLength(2);
-      expect(result.children?.[0].title).toBe('DataSource 1');
-      expect(result.children?.[1].title).toBe('DataSource 3');
-      expect(result.children?.some((child) => child.title === 'DataSource 2')).toBe(false);
-      expect(result.hasNext).toBe(true);
-    });
   });
 
   test('fetchFields returns table fields', async () => {

diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.ts b/src/plugins/query_enhancements/public/datasets/s3_type.ts
@@ -6,7 +6,6 @@
 import { i18n } from '@osd/i18n';
 import { trimEnd } from 'lodash';
 import { HttpSetup, SavedObjectsClientContract } from 'opensearch-dashboards/public';
-import semver from 'semver';
 import {
   DATA_STRUCTURE_META_TYPES,
   DEFAULT_DATA,
@@ -198,22 +197,17 @@ const fetchDataSources = async (client: SavedObjectsClientContract): Promise<Dat
     type: 'data-source',
     perPage: 10000,
   });
-  const dataSources: DataStructure[] = resp.savedObjects
-    .filter((savedObject) => {
-      const coercedVersion = semver.coerce(savedObject.attributes.dataSourceVersion);
-      return coercedVersion ? semver.satisfies(coercedVersion, '>=1.0.0') : false;
-    })
-    .map((savedObject) => ({
-      id: savedObject.id,
-      title: savedObject.attributes.title,
-      type: 'DATA_SOURCE',
-      meta: {
-        query: {
-          id: savedObject.id,
-        },
-        type: DATA_STRUCTURE_META_TYPES.CUSTOM,
-      } as DataStructureCustomMeta,
-    }));
+  const dataSources: DataStructure[] = resp.savedObjects.map((savedObject) => ({
+    id: savedObject.id,
+    title: savedObject.attributes.title,
+    type: 'DATA_SOURCE',
+    meta: {
+      query: {
+        id: savedObject.id,
+      },
+      type: DATA_STRUCTURE_META_TYPES.CUSTOM,
+    } as DataStructureCustomMeta,
+  }));
   return dataSources;
 };