diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb97c6862..626ed4c60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: needs: Get-CI-Image-Tag strategy: matrix: - java: [11, 17] + java: [21] os: [ ubuntu-latest ] name: Build and Test security-analytics with JDK ${{ matrix.java }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -55,14 +55,14 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} - name: Upload failed logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: logs-ubuntu path: build/testclusters/integTest-*/logs/* - name: Upload Artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: security-analytics-plugin-${{ matrix.os }} path: security-analytics-artifacts @@ -73,7 +73,7 @@ jobs: WORKING_DIR: ${{ matrix.working_directory }}. strategy: matrix: - java: [11, 17] + java: [21] os: [ windows-latest, macos-latest ] include: - os: windows-latest @@ -113,21 +113,21 @@ jobs: cp ./build/distributions/*.zip security-analytics-artifacts - name: Upload failed logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: ${{ failure() && matrix.os == 'macos-latest' }} with: name: logs-mac path: build/testclusters/integTest-*/logs/* - name: Upload failed logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: ${{ failure() && matrix.os == 'windows-latest' }} with: name: logs-windows path: build\testclusters\integTest-*\logs\* - name: Upload Artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: security-analytics-plugin-${{ matrix.os }} path: security-analytics-artifacts diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml index 36cb4d216..c0760c0b4 100644 --- a/.github/workflows/multi-node-test-workflow.yml +++ b/.github/workflows/multi-node-test-workflow.yml @@ -19,7 +19,7 @@ jobs: needs: Get-CI-Image-Tag strategy: matrix: - java: [ 11, 17, 21 ] + java: [ 21 ] # Job name name: Build and test Security Analytics on linux # This job runs on Linux @@ -45,7 +45,7 @@ jobs: chown -R 1000:1000 `pwd` su `id -un 1000` -c "./gradlew integTest -PnumNodes=3" - name: Upload failed logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: logs diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml index ffd91de13..f7b6fd09f 100644 --- a/.github/workflows/security-test-workflow.yml +++ b/.github/workflows/security-test-workflow.yml @@ -15,7 +15,7 @@ jobs: build: strategy: matrix: - java: [ 11, 17, 21 ] + java: [ 21 ] # Job name name: Build and test SecurityAnalytics # This job runs on Linux diff --git a/build.gradle b/build.gradle index edddc3bf3..88ec6f985 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ import org.opensearch.gradle.test.RestIntegTestTask buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "2.17.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.18.1-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') diff --git a/release-notes/opensearch-security-analytics.release-notes-2.17.0.0.md b/release-notes/opensearch-security-analytics.release-notes-2.17.0.0.md new file mode 100644 index 000000000..287ccda70 --- /dev/null +++ b/release-notes/opensearch-security-analytics.release-notes-2.17.0.0.md @@ -0,0 +1,27 @@ +## Version 2.17.0.0 2024-09-05 + +Compatible with OpenSearch 2.17.0 + +### Maintenance +* update build.gradle to use alerting-spi snapshot version ([#1217](https://github.com/opensearch-project/security-analytics/pull/1217)) + +### Enhancement +* added triggers in getDetectors API response ([#1226](https://github.com/opensearch-project/security-analytics/pull/1226)) +* secure rest tests for threat intel monitor apis ([#1212](https://github.com/opensearch-project/security-analytics/pull/1212)) + +### Bug Fixes +* Adds user validation for threat intel transport layer classes and stashes the thread context for all system index interactions ([#1207](https://github.com/opensearch-project/security-analytics/pull/1207)) +* fix mappings integ tests ([#1213](https://github.com/opensearch-project/security-analytics/pull/1213)) +* Bug fixes for threat intel ([#1223](https://github.com/opensearch-project/security-analytics/pull/1223)) +* make threat intel run with standard detectors ([#1234](https://github.com/opensearch-project/security-analytics/pull/1234)) +* Fixed searchString bug. Removed nested IOC mapping structure. ([#1239](https://github.com/opensearch-project/security-analytics/pull/1239)) +* adds toggling refresh disable/enable for deactivate/activate operation while updating URL_DOWNLOAD type configs ([#1240](https://github.com/opensearch-project/security-analytics/pull/1240)) +* Make threat intel source config release lock event driven ([#1254](https://github.com/opensearch-project/security-analytics/pull/1254)) +* Fix S3 validation errors not caught by action listener ([#1257](https://github.com/opensearch-project/security-analytics/pull/1257)) +* Clean up empty IOC indices created by failed source configs ([#1267](https://github.com/opensearch-project/security-analytics/pull/1267)) +* Fix threat intel multinode tests ([#1274](https://github.com/opensearch-project/security-analytics/pull/1274)) +* Update threat intel job mapping to new version ([#1272](https://github.com/opensearch-project/security-analytics/pull/1272)) +* Stash context for List IOCs Api ([#1278](https://github.com/opensearch-project/security-analytics/pull/1278)) + +### Documentation +* Added 2.17.0 release notes. ([#1290](https://github.com/opensearch-project/security-analytics/pull/1290)) \ No newline at end of file diff --git a/release-notes/opensearch-security-analytics.release-notes-2.17.1.0.md b/release-notes/opensearch-security-analytics.release-notes-2.17.1.0.md new file mode 100644 index 000000000..820c79255 --- /dev/null +++ b/release-notes/opensearch-security-analytics.release-notes-2.17.1.0.md @@ -0,0 +1,15 @@ +## Version 2.17.1.0 2024-09-27 + +Compatible with OpenSearch 2.17.1 + +### Maintenance +* upgrade upload artifacts ([#1305](https://github.com/opensearch-project/security-analytics/pull/1305)) +* Incremented version to 2.17.1 ([#1304](https://github.com/opensearch-project/security-analytics/pull/1304)) + +### Bug Fixes +* [Alerts in Correlations] Stash context for system index ([#1297](https://github.com/opensearch-project/security-analytics/pull/1297)) +* threat intel monitor bug fixes ([#1317](https://github.com/opensearch-project/security-analytics/pull/1317)) + + +### Documentation +* Added 2.17.1 release notes. ([#1331](https://github.com/opensearch-project/security-analytics/pull/1331)) diff --git a/release-notes/opensearch-security-analytics.release-notes-2.18.0.0.md b/release-notes/opensearch-security-analytics.release-notes-2.18.0.0.md new file mode 100644 index 000000000..a389a2128 --- /dev/null +++ b/release-notes/opensearch-security-analytics.release-notes-2.18.0.0.md @@ -0,0 +1,24 @@ +## Version 2.18.0.0 2024-10-28 + +Compatible with OpenSearch 2.18.0 + +### Maintenance +* Incremented version to 2.18.0 ([#1314](https://github.com/opensearch-project/security-analytics/pull/1314)) +* update to lucene 9.12 ([#1349](https://github.com/opensearch-project/security-analytics/pull/1349)) + +### Refactoring +* separate doc-level monitor query indices created by detectors ([#1324](https://github.com/opensearch-project/security-analytics/pull/1324)) +* update number of replicas of system indices to 1-20 and number of primary shards for system indices to 1 ([#1358](https://github.com/opensearch-project/security-analytics/pull/1358)) +* update min number of replicas to 0 ([#1364](https://github.com/opensearch-project/security-analytics/pull/1364)) +* updated dedicated query index settings to true ([#1365](https://github.com/opensearch-project/security-analytics/pull/1365)) +* set the refresh policy to IMMEDIATE when updating correlation alerts ([#1382](https://github.com/opensearch-project/security-analytics/pull/1382)) + +### Bug Fixes +* remove redundant logic to fix OS launch exception and updates actions/upload-artifac2 to @V3 ([#1303](https://github.com/opensearch-project/security-analytics/pull/1303)) +* Add null check while adding fetched iocs into per-indicator-type map ([#1335](https://github.com/opensearch-project/security-analytics/pull/1335)) +* Fix notifications listener leak in threat intel monitor ([#1361](https://github.com/opensearch-project/security-analytics/pull/1361)) +* [Bug] Fixed ListIOCs number of findings cap. ([#1373](https://github.com/opensearch-project/security-analytics/pull/1373)) +* [Bug] Add exists check for IOCs index. ([#1392](https://github.com/opensearch-project/security-analytics/pull/1392)) + +### Documentation +* Added 2.18.0 release notes. ([#1399](https://github.com/opensearch-project/security-analytics/pull/1399)) \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 95ca88315..2762bd5f8 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -70,7 +70,7 @@ import org.opensearch.securityanalytics.action.IndexDetectorAction; import org.opensearch.securityanalytics.action.IndexRuleAction; import org.opensearch.securityanalytics.action.ListCorrelationsAction; -import org.opensearch.securityanalytics.action.ListIOCsAction; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsAction; import org.opensearch.securityanalytics.action.SearchCorrelationRuleAction; import org.opensearch.securityanalytics.action.SearchCustomLogTypeAction; import org.opensearch.securityanalytics.action.SearchDetectorAction; @@ -113,7 +113,7 @@ import org.opensearch.securityanalytics.resthandler.RestIndexDetectorAction; import org.opensearch.securityanalytics.resthandler.RestIndexRuleAction; import org.opensearch.securityanalytics.resthandler.RestListCorrelationAction; -import org.opensearch.securityanalytics.resthandler.RestListIOCsAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestListIOCsAction; import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationAction; import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationRuleAction; import org.opensearch.securityanalytics.resthandler.RestSearchCustomLogTypeAction; @@ -197,7 +197,7 @@ import org.opensearch.securityanalytics.transport.TransportIndexDetectorAction; import org.opensearch.securityanalytics.transport.TransportIndexRuleAction; import org.opensearch.securityanalytics.transport.TransportListCorrelationAction; -import org.opensearch.securityanalytics.transport.TransportListIOCsAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportListIOCsAction; import org.opensearch.securityanalytics.transport.TransportSearchCorrelationAction; import org.opensearch.securityanalytics.transport.TransportSearchCorrelationRuleAction; import org.opensearch.securityanalytics.transport.TransportSearchCustomLogTypeAction; @@ -226,6 +226,7 @@ import static org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE; import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.SOURCE_CONFIG_FIELD; import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; +import static org.opensearch.securityanalytics.util.CorrelationIndices.CORRELATION_ALERT_INDEX; public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension, RemoteMonitorRunnerExtension { @@ -284,7 +285,11 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map @Override public Collection getSystemIndexDescriptors(Settings settings) { - return Collections.singletonList(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data")); + List descriptors = List.of( + new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data"), + new SystemIndexDescriptor(CORRELATION_ALERT_INDEX, "System index used for Correlation Alerts") + ); + return descriptors; } @@ -327,7 +332,7 @@ public Collection createComponents(Client client, TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService); IocFindingService iocFindingService = new IocFindingService(client, clusterService, xContentRegistry); ThreatIntelAlertService threatIntelAlertService = new ThreatIntelAlertService(client, clusterService, xContentRegistry); - SaIoCScanService ioCScanService = new SaIoCScanService(client, xContentRegistry, iocFindingService, threatIntelAlertService, notificationService); + SaIoCScanService ioCScanService = new SaIoCScanService(client, clusterService, xContentRegistry, iocFindingService, threatIntelAlertService, notificationService); DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService = new DefaultTifSourceConfigLoaderService(builtInTIFMetadataLoader, client, saTifSourceConfigManagementService); return List.of( detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, threatIntelAlertService, @@ -502,7 +507,9 @@ public List> getSettings() { SecurityAnalyticsSettings.BATCH_SIZE, SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT, SecurityAnalyticsSettings.IOC_INDEX_RETENTION_PERIOD, - SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_INDEX_PATTERN + SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_INDEX_PATTERN, + SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT, + SecurityAnalyticsSettings.ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES ); } diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java index 459f523b7..02f6595ec 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java @@ -5,6 +5,8 @@ package org.opensearch.securityanalytics.config.monitors; import java.util.List; +import java.util.Random; +import java.util.UUID; import java.util.stream.Collectors; import org.opensearch.common.inject.Inject; import org.opensearch.securityanalytics.logtype.LogTypeService; @@ -25,6 +27,10 @@ public static String getRuleIndex(String logType) { return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", logType); } + public static String getRuleIndexOptimized(String logType) { + return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-optimized-%s", logType, UUID.randomUUID()); + } + public static String getAlertsIndex(String logType) { return String.format(Locale.getDefault(), ".opensearch-sap-%s-alerts", logType); } diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertService.java b/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertService.java index e5c43698b..56f446cd4 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertService.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertService.java @@ -13,6 +13,7 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.WriteRequest; import org.opensearch.action.update.UpdateRequest; import org.opensearch.client.Client; import org.opensearch.common.lucene.uid.Versions; @@ -212,9 +213,10 @@ public void acknowledgeAlerts(List alertIds, ActionListener() { @Override public void onResponse(SearchResponse searchResponse) { + // Set the refresh policy on the BulkRequest + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); // Iterate through the search hits for (SearchHit hit : searchResponse.getHits().getHits()) { - // Construct a script to update the document with the new state and acknowledgedTime // Construct a script to update the document with the new state and acknowledgedTime Script script = new Script(ScriptType.INLINE, "painless", "ctx._source.state = params.state; ctx._source.acknowledged_time = params.time", diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java b/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java index 7ed1bb0ae..d081fd00f 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java @@ -81,6 +81,7 @@ public void sendNotification(String configId, String severity, String subject, S sendNotificationResponse -> { if (sendNotificationResponse.getStatus() == RestStatus.OK) { logger.info("Successfully sent a notification, Notification Event: " + sendNotificationResponse.getNotificationEvent()); + listener.onResponse(null); } else { listener.onFailure(new Exception("Error while sending a notification, Notification Event: " + sendNotificationResponse.getNotificationEvent())); } diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java index c6ffd8551..7f4c84eb9 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java @@ -4,11 +4,13 @@ */ package org.opensearch.securityanalytics.correlation.index.codec; +import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.backward_codecs.lucene95.Lucene95Codec; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.opensearch.index.mapper.MapperService; +import org.opensearch.securityanalytics.correlation.index.codec.correlation9120.CorrelationCodec9120; import org.opensearch.securityanalytics.correlation.index.codec.correlation950.CorrelationCodec950; import org.opensearch.securityanalytics.correlation.index.codec.correlation990.CorrelationCodec990; import org.opensearch.securityanalytics.correlation.index.codec.correlation990.PerFieldCorrelationVectorsFormat990; @@ -32,9 +34,16 @@ public enum CorrelationCodecVersion { new PerFieldCorrelationVectorsFormat990(Optional.empty()), (userCodec, mapperService) -> new CorrelationCodec990(userCodec, new PerFieldCorrelationVectorsFormat990(Optional.of(mapperService))), CorrelationCodec990::new + ), + V_9_12_0( + "CorrelationCodec9120", + new Lucene912Codec(), + new PerFieldCorrelationVectorsFormat990(Optional.empty()), + (userCodec, mapperService) -> new CorrelationCodec9120(userCodec, new PerFieldCorrelationVectorsFormat990(Optional.of(mapperService))), + CorrelationCodec9120::new ); - private static final CorrelationCodecVersion CURRENT = V_9_9_0; + private static final CorrelationCodecVersion CURRENT = V_9_12_0; private final String codecName; private final Codec defaultCodecDelegate; private final PerFieldKnnVectorsFormat perFieldKnnVectorsFormat; diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation9120/CorrelationCodec9120.java b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation9120/CorrelationCodec9120.java new file mode 100644 index 000000000..1aa6e6824 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation9120/CorrelationCodec9120.java @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.correlation.index.codec.correlation9120; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecVersion; + +public class CorrelationCodec9120 extends FilterCodec { + private static final CorrelationCodecVersion VERSION = CorrelationCodecVersion.V_9_12_0; + private final PerFieldKnnVectorsFormat perFieldCorrelationVectorsFormat; + + public CorrelationCodec9120() { + this(VERSION.getDefaultCodecDelegate(), VERSION.getPerFieldCorrelationVectorsFormat()); + } + + public CorrelationCodec9120(Codec delegate, PerFieldKnnVectorsFormat perFieldCorrelationVectorsFormat) { + super(VERSION.getCodecName(), delegate); + this.perFieldCorrelationVectorsFormat = perFieldCorrelationVectorsFormat; + } + + @Override + public KnnVectorsFormat knnVectorsFormat() { + return perFieldCorrelationVectorsFormat; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java index d6cae5304..22b415230 100644 --- a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -558,8 +558,17 @@ private void rolloverIndex( request.getCreateIndexRequest().index(pattern) .mapping(map) .settings(isCorrelation? - Settings.builder().put("index.hidden", true).put("index.correlation", true).build(): - Settings.builder().put("index.hidden", true).build() + Settings.builder() + .put("index.hidden", true) + .put("index.correlation", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(): + Settings.builder() + .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build() ); request.addMaxIndexDocsCondition(docsCondition); request.addMaxIndexAgeCondition(ageCondition); diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java index 80d0ae50d..2e77b0831 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java @@ -70,8 +70,6 @@ public void ensureLogTypesLoaded() { private List loadBuiltinLogTypes() throws URISyntaxException, IOException { List logTypes = new ArrayList<>(); - String pathurl = Paths.get(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH).toURI()).toString(); - final String url = Objects.requireNonNull(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH)).toURI().toString(); Path dirPath = null; if (url.contains("!")) { diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java index bc4452650..9c0d5ef72 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java @@ -63,10 +63,10 @@ import org.opensearch.securityanalytics.model.LogType; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; -import static org.opensearch.action.support.ActiveShardCount.ALL; import static org.opensearch.securityanalytics.model.FieldMappingDoc.LOG_TYPES; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.DEFAULT_MAPPING_SCHEMA; - +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; /** * @@ -459,7 +459,8 @@ public void ensureConfigIndexIsInitialized(ActionListener listener) { isConfigIndexInitialized = false; Settings indexSettings = Settings.builder() .put("index.hidden", true) - .put("index.auto_expand_replicas", "0-all") + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) .build(); CreateIndexRequest createIndexRequest = new CreateIndexRequest(); diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java index ec5ed713a..695a9d65a 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java @@ -9,7 +9,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; +import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.StepListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; import org.opensearch.action.bulk.BulkRequest; @@ -18,6 +20,7 @@ import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.Streams; @@ -33,6 +36,7 @@ import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.transport.RemoteTransportException; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -48,6 +52,9 @@ import java.util.Map; import java.util.UUID; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; + public class STIX2IOCFeedStore implements FeedStore { public static final String IOC_INDEX_NAME_BASE = ".opensearch-sap-iocs"; public static final String IOC_ALL_INDEX_PATTERN = IOC_INDEX_NAME_BASE + "-*"; @@ -80,7 +87,6 @@ public STIX2IOCFeedStore( this.baseListener = listener; batchSize = clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE); newActiveIndex = getNewActiveIndex(saTifSourceConfig.getId()); - initSourceConfigIndexes(); } @Override @@ -113,7 +119,15 @@ public void storeIOCs(Map actionToIOCs) { } public void indexIocs(List iocs) throws IOException { - bulkIndexIocs(iocs, newActiveIndex); + StepListener initSourceConfigIndexesListener = new StepListener<>(); + initSourceConfigIndexes(initSourceConfigIndexesListener); + initSourceConfigIndexesListener.whenComplete(r -> { + bulkIndexIocs(iocs, newActiveIndex); + }, e -> { + log.error("Failed to init source config indexes"); + baseListener.onFailure(e); + }); + } private void bulkIndexIocs(List iocs, String activeIndex) throws IOException { @@ -197,7 +211,7 @@ public SATIFSourceConfig getSaTifSourceConfig() { return saTifSourceConfig; } - private void initSourceConfigIndexes() { + private void initSourceConfigIndexes(StepListener stepListener) { String iocIndexPattern = getAllIocIndexPatternById(saTifSourceConfig.getId()); initFeedIndex(newActiveIndex, ActionListener.wrap( r -> { @@ -214,10 +228,10 @@ private void initSourceConfigIndexes() { ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails().add(iocToIndexDetails); } }); - + stepListener.onResponse(null); }, e-> { log.error("Failed to initialize the IOC index and save the IOCs", e); - baseListener.onFailure(e); + stepListener.onFailure(e); } )); } @@ -226,17 +240,29 @@ private void initFeedIndex(String feedIndexName, ActionListener { log.info("Created system index {}", feedIndexName); listener.onResponse(r); }, e -> { + if (e instanceof ResourceAlreadyExistsException || (e instanceof RemoteTransportException && e.getCause() instanceof ResourceAlreadyExistsException)) { + log.debug("index {} already exist", feedIndexName); + listener.onResponse(null); + return; + } log.error("Failed to create system index {}", feedIndexName); listener.onFailure(e); } )); + } else { + listener.onResponse(null); } } } diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index b0f0fed74..cc11fe36d 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -12,6 +12,8 @@ public class SecurityAnalyticsSettings { public static final String CORRELATION_INDEX = "index.correlation"; + public static final int minSystemIndexReplicas = 0; + public static final int maxSystemIndexReplicas = 20; public static Setting INDEX_TIMEOUT = Setting.positiveTimeSetting("plugins.security_analytics.index_timeout", TimeValue.timeValueSeconds(60), @@ -237,4 +239,20 @@ public static final List> settings() { Setting.Property.NodeScope, Setting.Property.Dynamic ); + /** + * Maximum terms in Terms query search query submitted during ioc scan + */ + public static final Setting IOC_SCAN_MAX_TERMS_COUNT = Setting.intSetting( + "plugins.security_analytics.ioc.scan_max_terms_count", + 65536, + 1, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES = Setting.boolSetting( + "plugins.security_analytics.enable_detectors_with_dedicated_query_indices", + true, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/action/ListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsAction.java similarity index 88% rename from src/main/java/org/opensearch/securityanalytics/action/ListIOCsAction.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsAction.java index ae4912bbc..f9e5bde66 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/ListIOCsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsAction.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.action; +package org.opensearch.securityanalytics.threatIntel.action; import org.opensearch.action.ActionType; diff --git a/src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionRequest.java similarity index 97% rename from src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionRequest.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionRequest.java index dead1cd3f..cb57213b9 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionRequest.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionRequest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.action; +package org.opensearch.securityanalytics.threatIntel.action; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; @@ -14,7 +14,6 @@ import org.opensearch.securityanalytics.commons.model.IOCType; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.stream.Collectors; diff --git a/src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionResponse.java similarity index 96% rename from src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionResponse.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionResponse.java index 741f3cf36..0f142fbf0 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionResponse.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionResponse.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.action; +package org.opensearch.securityanalytics.threatIntel.action; import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java index 8001f37ea..4a5ab1446 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java @@ -5,7 +5,6 @@ package org.opensearch.securityanalytics.threatIntel.common; -import org.opensearch.securityanalytics.commons.model.IOC; import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; import org.opensearch.securityanalytics.threatIntel.model.S3Source; @@ -13,9 +12,8 @@ import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; +import java.util.regex.Pattern; /** * Source config dto validator @@ -24,7 +22,34 @@ public class SourceConfigDtoValidator { public List validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto) { List errorMsgs = new ArrayList<>(); - if (sourceConfigDto.getIocTypes().isEmpty()) { + String nameRegex = "^[a-zA-Z0-9 _-]{1,128}$"; + Pattern namePattern = Pattern.compile(nameRegex); + + int MAX_RULE_DESCRIPTION_LENGTH = 65535; + String descriptionRegex = "^.{0," + MAX_RULE_DESCRIPTION_LENGTH + "}$"; + Pattern descriptionPattern = Pattern.compile(descriptionRegex); + + if (sourceConfigDto.getName() == null || sourceConfigDto.getName().isEmpty()) { + errorMsgs.add("Name must not be empty"); + } else if (sourceConfigDto.getName() != null && namePattern.matcher(sourceConfigDto.getName()).matches() == false) { + errorMsgs.add("Name must be less than 128 characters and only consist of upper and lowercase letters, numbers 0-9, hyphens, spaces, and underscores"); + } + + if (sourceConfigDto.getFormat() == null || sourceConfigDto.getFormat().isEmpty()) { + errorMsgs.add("Format must not be empty"); + } else if (sourceConfigDto.getFormat() != null && sourceConfigDto.getFormat().length() > 50) { + errorMsgs.add("Format must be 50 characters or less"); + } + + if (sourceConfigDto.getDescription() != null && descriptionPattern.matcher(sourceConfigDto.getDescription()).matches() == false) { + errorMsgs.add("Description must be " + MAX_RULE_DESCRIPTION_LENGTH + " characters or less"); + } + + if (sourceConfigDto.getSource() == null) { + errorMsgs.add("Source must not be empty"); + } + + if (sourceConfigDto.getIocTypes() == null || sourceConfigDto.getIocTypes().isEmpty()) { errorMsgs.add("Must specify at least one IOC type"); } else { for (String s: sourceConfigDto.getIocTypes()) { @@ -34,34 +59,41 @@ public List validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto } } - switch (sourceConfigDto.getType()) { - case IOC_UPLOAD: - if (sourceConfigDto.isEnabled()) { - errorMsgs.add("Job Scheduler cannot be enabled for IOC_UPLOAD type"); - } - if (sourceConfigDto.getSchedule() != null) { - errorMsgs.add("Cannot pass in schedule for IOC_UPLOAD type"); - } - if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof IocUploadSource == false) { - errorMsgs.add("Source must be IOC_UPLOAD type"); - } - break; - case S3_CUSTOM: - if (sourceConfigDto.getSchedule() == null) { - errorMsgs.add("Must pass in schedule for S3_CUSTOM type"); - } - if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof S3Source == false) { - errorMsgs.add("Source must be S3_CUSTOM type"); - } - break; - case URL_DOWNLOAD: - if (sourceConfigDto.getSchedule() == null) { - errorMsgs.add("Must pass in schedule for URL_DOWNLOAD source type"); - } - if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof UrlDownloadSource == false) { - errorMsgs.add("Source must be URL_DOWNLOAD source type"); - } - break; + if (sourceConfigDto.getType() == null) { + errorMsgs.add("Type must not be empty"); + } else { + switch (sourceConfigDto.getType()) { + case IOC_UPLOAD: + if (sourceConfigDto.isEnabled()) { + errorMsgs.add("Job Scheduler cannot be enabled for IOC_UPLOAD type"); + } + if (sourceConfigDto.getSchedule() != null) { + errorMsgs.add("Cannot pass in schedule for IOC_UPLOAD type"); + } + if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof IocUploadSource == false) { + errorMsgs.add("Source must be IOC_UPLOAD type"); + } + if (sourceConfigDto.getSource() instanceof IocUploadSource && ((IocUploadSource) sourceConfigDto.getSource()).getIocs() == null) { + errorMsgs.add("Ioc list must include at least one ioc"); + } + break; + case S3_CUSTOM: + if (sourceConfigDto.getSchedule() == null) { + errorMsgs.add("Must pass in schedule for S3_CUSTOM type"); + } + if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof S3Source == false) { + errorMsgs.add("Source must be S3_CUSTOM type"); + } + break; + case URL_DOWNLOAD: + if (sourceConfigDto.getSchedule() == null) { + errorMsgs.add("Must pass in schedule for URL_DOWNLOAD source type"); + } + if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof UrlDownloadSource == false) { + errorMsgs.add("Source must be URL_DOWNLOAD source type"); + } + break; + } } return errorMsgs; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java index f88aebd9b..eb467c55e 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java @@ -73,24 +73,13 @@ public void onFailure(final Exception e) { * * @param lockModel the lock model */ - public void releaseLock(final LockModel lockModel) { - log.debug("Releasing lock with id [{}]", lockModel.getLockId()); + public void releaseLock(final LockModel lockModel, final ActionListener listener) { lockService.release( lockModel, - ActionListener.wrap(released -> {}, exception -> log.error("Failed to release the lock", exception)) - ); - } - - /** - * Wrapper method of LockService#release - * - * @param lockModel the lock model - */ - public void releaseLockEventDriven(final LockModel lockModel, final ActionListener listener) { - log.debug("Releasing lock with id [{}]", lockModel.getLockId()); - lockService.release( - lockModel, - ActionListener.wrap(listener::onResponse, exception -> log.error("Failed to release the lock", exception)) + ActionListener.wrap(listener::onResponse, exception -> { + log.error("Failed to release the lock", exception); + listener.onFailure(exception); + }) ); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java index 62eee1a57..f435db5ce 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java @@ -14,6 +14,7 @@ import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; @@ -31,6 +32,8 @@ import java.util.ArrayList; import java.util.List; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; import static org.opensearch.securityanalytics.util.DetectorUtils.getEmptySearchResponse; /** @@ -112,14 +115,19 @@ public void bulkIndexEntities(List newEntityList, List updatedEn } } actionListener.onResponse(null); - }, actionListener::onFailure), bulkRequestList.size()); + }, e1 -> { + log.error("Failed to bulk index " + getEntityName(), e1); + actionListener.onFailure(e1); + }), bulkRequestList.size()); + for (BulkRequest req : bulkRequestList) { try { - client.bulk(req, groupedListener); //todo why stash context here? + client.bulk(req, groupedListener); } catch (Exception e) { log.error( () -> new ParameterizedMessage("Failed to bulk save {} {}.", req.batchSize(), getEntityName()), e); + groupedListener.onFailure(e); } } }, e -> { @@ -247,7 +255,9 @@ public void createIndexIfNotExists(final ActionListener listener) { public abstract String getEntityName(); protected Settings.Builder getIndexSettings() { - return Settings.builder().put("index.hidden", true); + return Settings.builder().put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas); } public abstract String getEntityAliasName(); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java index f3238460a..f60af7afd 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java @@ -40,6 +40,13 @@ public void scanIoCs(IocScanContext iocScanContext, long startTime = System.currentTimeMillis(); IocLookupDtos iocLookupDtos = extractIocsPerType(data, iocScanContext); + if (iocLookupDtos.getIocsPerIocTypeMap().isEmpty()) { + log.error("Threat intel monitor {}: Unexpected scenario that non-zero number of docs are fetched from indices containing iocs but iocs-per-type map constructed is empty", + iocScanContext.getMonitor().getId() + ); + scanCallback.accept(Collections.emptyList(), null); + return; + } BiConsumer, Exception> iocScanResultConsumer = (List maliciousIocs, Exception e) -> { long scanEndTime = System.currentTimeMillis(); long timeTaken = scanEndTime - startTime; @@ -49,10 +56,10 @@ public void scanIoCs(IocScanContext iocScanContext, (iocFindings, e1) -> { if (e1 != null) { log.error( - () -> new ParameterizedMessage("Threat intel monitor {}: Failed to create ioc findings/ ", + () -> new ParameterizedMessage("Threat intel monitor {}: Failed to create ioc findings", iocScanContext.getMonitor().getId(), data.size()), e1); - scanCallback.accept(null, e1); + scanCallback.accept(data, e1); } else { BiConsumer, Exception> triggerResultConsumer = (alerts, e2) -> { if (e2 != null) { @@ -60,8 +67,8 @@ public void scanIoCs(IocScanContext iocScanContext, () -> new ParameterizedMessage("Threat intel monitor {}: Failed to execute threat intel triggers/ ", iocScanContext.getMonitor().getId(), data.size()), e2); - scanCallback.accept(null, e2); - return; + // if findings are generated successfully but alerts/notifications fail we mark execution as succeeded, so that duplicate findings are not created + scanCallback.accept(data, null); } else { scanCallback.accept(data, null); } @@ -121,28 +128,28 @@ abstract void matchAgainstThreatIntelAndReturnMaliciousIocs( for (PerIocTypeScanInput iocTypeToIndexFieldMapping : context.getThreatIntelInput().getPerIocTypeScanInputList()) { String iocType = iocTypeToIndexFieldMapping.getIocType().toLowerCase(); String concreteIndex = getIndexName(datum); - if (context.getConcreteIndexToMonitorInputIndicesMap().containsKey(concreteIndex) - && false == context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).isEmpty() - ) { + if (context.getConcreteIndexToMonitorInputIndicesMap().containsKey(concreteIndex)) { // if concrete index resolves to multiple monitor input indices, it's undesirable. We just pick any one of the monitor input indices to get fields for each ioc. String index = context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).get(0); - List fields = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index); - for (String field : fields) { - List vals = getValuesAsStringList(datum, field); - String id = getId(datum); - String docId = id + ":" + index; - Set iocs = docIdToIocsMap.getOrDefault(docId, new HashSet<>()); - iocs.addAll(vals); - docIdToIocsMap.put(docId, iocs); - for (String ioc : vals) { - Set docIds = iocValueToDocIdMap.getOrDefault(ioc, new HashSet<>()); - docIds.add(docId); - iocValueToDocIdMap.put(ioc, docIds); - } - if (false == vals.isEmpty()) { - iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>()); + List fieldsConfiguredInMonitorForCurrentIndex = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index); + if(fieldsConfiguredInMonitorForCurrentIndex != null && false == fieldsConfiguredInMonitorForCurrentIndex.isEmpty()) { + for (String field : fieldsConfiguredInMonitorForCurrentIndex) { + List vals = getValuesAsStringList(datum, field); + String id = getId(datum); + String docId = id + ":" + index; + Set iocs = docIdToIocsMap.getOrDefault(docId, new HashSet<>()); iocs.addAll(vals); - iocsPerIocTypeMap.put(iocType, iocs); + docIdToIocsMap.put(docId, iocs); + for (String ioc : vals) { + Set docIds = iocValueToDocIdMap.getOrDefault(ioc, new HashSet<>()); + docIds.add(docId); + iocValueToDocIdMap.put(ioc, docIds); + } + if (false == vals.isEmpty()) { + iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>()); + iocs.addAll(vals); + iocsPerIocTypeMap.put(iocType, iocs); + } } } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java index 7181dda2e..8a3c4a206 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java @@ -7,6 +7,7 @@ import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.action.support.GroupedActionListener; import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.document.DocumentField; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentType; @@ -28,6 +29,7 @@ import org.opensearch.securityanalytics.model.STIX2IOC; import org.opensearch.securityanalytics.model.threatintel.IocFinding; import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; import org.opensearch.securityanalytics.threatIntel.iocscan.dao.ThreatIntelAlertService; import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext; @@ -54,16 +56,17 @@ public class SaIoCScanService extends IoCScanService { private static final Logger log = LogManager.getLogger(SaIoCScanService.class); - public static final int MAX_TERMS = 65536; //TODO make ioc index setting based. use same setting value to create index private final Client client; + private final ClusterService clusterService; private final NamedXContentRegistry xContentRegistry; private final IocFindingService iocFindingService; private final ThreatIntelAlertService threatIntelAlertService; private final NotificationService notificationService; - public SaIoCScanService(Client client, NamedXContentRegistry xContentRegistry, IocFindingService iocFindingService, + public SaIoCScanService(Client client, ClusterService clusterService, NamedXContentRegistry xContentRegistry, IocFindingService iocFindingService, ThreatIntelAlertService threatIntelAlertService, NotificationService notificationService) { this.client = client; + this.clusterService = clusterService; this.xContentRegistry = xContentRegistry; this.iocFindingService = iocFindingService; this.threatIntelAlertService = threatIntelAlertService; @@ -110,7 +113,7 @@ private void executeTrigger(List iocFindings, } else { fetchExistingAlertsForTrigger(monitor, triggerMatchedFindings, trigger, ActionListener.wrap( existingAlerts -> { - executeActionsAndSaveAlerts(iocFindings, trigger, monitor, existingAlerts, triggerMatchedFindings, threatIntelTrigger, listener); + saveAlertsAndExecuteActions(iocFindings, trigger, monitor, existingAlerts, triggerMatchedFindings, threatIntelTrigger, listener); }, e -> { log.error(() -> new ParameterizedMessage( @@ -129,7 +132,7 @@ private void executeTrigger(List iocFindings, } } - private void executeActionsAndSaveAlerts(List iocFindings, + private void saveAlertsAndExecuteActions(List iocFindings, Trigger trigger, Monitor monitor, List existingAlerts, @@ -144,36 +147,38 @@ private void executeActionsAndSaveAlerts(List iocFindings, newAlerts, existingAlerts); if (false == trigger.getActions().isEmpty()) { - GroupedActionListener notifsListener = new GroupedActionListener<>(ActionListener.wrap( - r -> { - saveAlerts(new ArrayList<>(iocToUpdatedAlertsMap.values()), - newAlerts, - monitor, - (threatIntelAlerts, e) -> { - if (e != null) { - log.error(String.format("Threat intel monitor %s: Failed to save alerts for trigger {}", monitor.getId(), trigger.getId()), e); - listener.onFailure(e); - } else { + saveAlerts(new ArrayList<>(iocToUpdatedAlertsMap.values()), + newAlerts, + monitor, + (threatIntelAlerts, e) -> { + if (e != null) { + log.error(String.format("Threat intel monitor %s: Failed to save alerts for trigger %s", monitor.getId(), trigger.getId()), e); + listener.onFailure(e); + } else { + GroupedActionListener notifsListener = new GroupedActionListener<>(ActionListener.wrap( + r -> { listener.onResponse(threatIntelAlerts); + }, ex -> { + log.error(String.format("Threat intel monitor {}: Failed to send notification for trigger {}", monitor.getId(), trigger.getId()), ex); + listener.onFailure(new SecurityAnalyticsException("Failed to send notification", RestStatus.INTERNAL_SERVER_ERROR, ex)); } - }); - }, e -> { - log.error(String.format("Threat intel monitor %s: Failed to send notification for trigger {}", monitor.getId(), trigger.getId()), e); - listener.onFailure(new SecurityAnalyticsException("Failed to send notification", RestStatus.INTERNAL_SERVER_ERROR, e)); - } - ), trigger.getActions().size()); - for (Action action : trigger.getActions()) { - try { - String transformedSubject = NotificationService.compileTemplate(ctx, action.getSubjectTemplate()); - String transformedMessage = NotificationService.compileTemplate(ctx, action.getMessageTemplate()); - String configId = action.getDestinationId(); - notificationService.sendNotification(configId, trigger.getSeverity(), transformedSubject, transformedMessage, notifsListener); - } catch (Exception e) { - log.error(String.format("Threat intel monitor %s: Failed to send notification to %s for trigger %s", monitor.getId(), action.getDestinationId(), trigger.getId()), e); - notifsListener.onFailure(new SecurityAnalyticsException("Failed to send notification", RestStatus.INTERNAL_SERVER_ERROR, e)); - } + ), trigger.getActions().size()); + + for (Action action : trigger.getActions()) { + try { + String transformedSubject = NotificationService.compileTemplate(ctx, action.getSubjectTemplate()); + String transformedMessage = NotificationService.compileTemplate(ctx, action.getMessageTemplate()); + String configId = action.getDestinationId(); + notificationService.sendNotification(configId, trigger.getSeverity(), transformedSubject, transformedMessage, notifsListener); + } catch (Exception ex) { + log.error(String.format("Threat intel monitor %s: Failed to send notification to %s for trigger %s", monitor.getId(), action.getDestinationId(), trigger.getId()), ex); + notifsListener.onFailure(new SecurityAnalyticsException("Failed to send notification", RestStatus.INTERNAL_SERVER_ERROR, ex)); + } + + } + } + }); - } } else { saveAlerts(new ArrayList<>(iocToUpdatedAlertsMap.values()), newAlerts, @@ -232,7 +237,7 @@ private GroupedActionListener> getGroupedListenerForAllTr r -> { List list = new ArrayList<>(); r.forEach(list::addAll); - triggerResultConsumer.accept(list, null); //todo change emptylist to actual response + triggerResultConsumer.accept(list, null); }, e -> { log.error(() -> new ParameterizedMessage( "Threat intel monitor {} Failed to execute triggers {}", monitor.getId()), @@ -329,12 +334,13 @@ private void performScanForMaliciousIocsPerIocType( GroupedActionListener listener) { // TODO change ioc indices max terms count to 100k and experiment // TODO add fuzzy postings on ioc value field to enable bloomfilter on iocs as an index data structure and benchmark performance - GroupedActionListener perIocTypeListener = getGroupedListenerForIocScanPerIocType(iocs, monitor, iocType, listener); + int maxTerms = clusterService.getClusterSettings().get(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT); + GroupedActionListener perIocTypeListener = getGroupedListenerForIocScanPerIocType(iocs, monitor, iocType, listener, maxTerms); List iocList = new ArrayList<>(iocs); int totalIocs = iocList.size(); - for (int start = 0; start < totalIocs; start += MAX_TERMS) { - int end = Math.min(start + MAX_TERMS, totalIocs); + for (int start = 0; start < totalIocs; start += maxTerms) { + int end = Math.min(start + maxTerms, totalIocs); List iocsSublist = iocList.subList(start, end); SearchRequest searchRequest = getSearchRequestForIocType(indices, iocType, iocsSublist); client.search(searchRequest, ActionListener.wrap( @@ -356,7 +362,7 @@ private void performScanForMaliciousIocsPerIocType( ); } } - listener.onResponse(new SearchHitsOrException( + perIocTypeListener.onResponse(new SearchHitsOrException( searchResponse.getHits() == null || searchResponse.getHits().getHits() == null ? emptyList() : Arrays.asList(searchResponse.getHits().getHits()), null)); }, @@ -366,7 +372,7 @@ private void performScanForMaliciousIocsPerIocType( iocsSublist.size(), iocType), e ); - listener.onResponse(new SearchHitsOrException(emptyList(), e)); + perIocTypeListener.onResponse(new SearchHitsOrException(emptyList(), e)); } )); } @@ -387,7 +393,7 @@ private static SearchRequest getSearchRequestForIocType(List indices, St * grouped listener for a given ioc type to listen and collate malicious iocs in search hits from batched search calls. * batching done for every 65536 or MAX_TERMS setting number of iocs in a list. */ - private GroupedActionListener getGroupedListenerForIocScanPerIocType(Set iocs, Monitor monitor, String iocType, GroupedActionListener groupedListenerForAllIocTypes) { + private GroupedActionListener getGroupedListenerForIocScanPerIocType(Set iocs, Monitor monitor, String iocType, GroupedActionListener groupedListenerForAllIocTypes, int maxTerms) { return new GroupedActionListener<>( ActionListener.wrap( (Collection searchHitsOrExceptions) -> { @@ -419,8 +425,7 @@ private GroupedActionListener getGroupedListenerForIocSca groupedListenerForAllIocTypes.onResponse(new SearchHitsOrException(emptyList(), e)); } ), - //TODO fix groupsize - getGroupSizeForIocs(iocs) // batch into #MAX_TERMS setting + getGroupSizeForIocs(iocs, maxTerms) ); } @@ -436,8 +441,8 @@ private Exception buildException(Collection searchHitsOrE return e; } - private static int getGroupSizeForIocs(Set iocs) { - return iocs.size() / MAX_TERMS + (iocs.size() % MAX_TERMS == 0 ? 0 : 1); + private static int getGroupSizeForIocs(Set iocs, int maxTerms) { + return iocs.size() / maxTerms + (iocs.size() % maxTerms == 0 ? 0 : 1); } @Override diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java index 9e9022294..89dc729b4 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java @@ -115,7 +115,7 @@ protected Runnable updateJobRunner(final ScheduledJobParameter jobParameter) { ActionListener.wrap(lock -> { updateJobParameter(jobParameter, lockService.getRenewLockRunnable(new AtomicReference<>(lock)), ActionListener.wrap( - r -> lockService.releaseLockEventDriven(lock, ActionListener.wrap( + r -> lockService.releaseLock(lock, ActionListener.wrap( response -> { log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); }, @@ -125,7 +125,7 @@ protected Runnable updateJobRunner(final ScheduledJobParameter jobParameter) { )), e -> { log.error("Failed to update job parameter " + jobParameter.getName(), e); - lockService.releaseLockEventDriven(lock, ActionListener.wrap( + lockService.releaseLock(lock, ActionListener.wrap( response -> { log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); }, diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFSourceConfigRunner.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFSourceConfigRunner.java index fc01bbad7..1c098cfc9 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFSourceConfigRunner.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFSourceConfigRunner.java @@ -108,7 +108,7 @@ protected Runnable retrieveLockAndUpdateConfig(final SATIFSourceConfig saTifSour updateSourceConfigAndIOCs(saTifSourceConfig, lockService.getRenewLockRunnable(new AtomicReference<>(lock)), ActionListener.wrap( r -> { - lockService.releaseLockEventDriven(lock, ActionListener.wrap( + lockService.releaseLock(lock, ActionListener.wrap( response -> { log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); }, @@ -119,7 +119,7 @@ protected Runnable retrieveLockAndUpdateConfig(final SATIFSourceConfig saTifSour }, e -> { log.error("Failed to update threat intel source config " + saTifSourceConfig.getName(), e); - lockService.releaseLockEventDriven(lock, ActionListener.wrap( + lockService.releaseLock(lock, ActionListener.wrap( response -> { log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); }, diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java index 51b909334..2c634ce70 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java @@ -92,7 +92,7 @@ public class SATIFSourceConfig implements TIFSourceConfig, Writeable, ScheduledJ public SATIFSourceConfig(String id, Long version, String name, String format, SourceConfigType type, String description, User createdByUser, Instant createdAt, Source source, Instant enabledTime, Instant lastUpdateTime, Schedule schedule, TIFJobState state, RefreshType refreshType, Instant lastRefreshedTime, User lastRefreshedUser, - Boolean isEnabled, IocStoreConfig iocStoreConfig, List iocTypes, boolean enabledForScan) { + boolean isEnabled, IocStoreConfig iocStoreConfig, List iocTypes, boolean enabledForScan) { this.id = id == null ? UUIDs.base64UUID() : id; this.version = version != null ? version : NO_VERSION; this.name = name; @@ -289,7 +289,7 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio RefreshType refreshType = null; Instant lastRefreshedTime = null; User lastRefreshedUser = null; - Boolean isEnabled = null; + boolean isEnabled = true; boolean enabledForScan = true; IocStoreConfig iocStoreConfig = null; List iocTypes = new ArrayList<>(); @@ -303,16 +303,28 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio case SOURCE_CONFIG_FIELD: break; case NAME_FIELD: - name = xcp.text(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + name = null; + } else { + name = xcp.text(); + } break; case VERSION_FIELD: version = xcp.longValue(); break; case FORMAT_FIELD: - format = xcp.text(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + format = null; + } else { + format = xcp.text(); + } break; case TYPE_FIELD: - sourceConfigType = toSourceConfigType(xcp.text()); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + sourceConfigType = null; + } else { + sourceConfigType = toSourceConfigType(xcp.text()); + } break; case DESCRIPTION_FIELD: if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java index 3ba64d47a..222a345ed 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java @@ -123,7 +123,7 @@ private List convertToIocDtos(List stix2IocList) { public SATIFSourceConfigDto(String id, Long version, String name, String format, SourceConfigType type, String description, User createdByUser, Instant createdAt, Source source, Instant enabledTime, Instant lastUpdateTime, Schedule schedule, TIFJobState state, RefreshType refreshType, Instant lastRefreshedTime, User lastRefreshedUser, - Boolean isEnabled, List iocTypes, boolean enabledForScan) { + boolean isEnabled, List iocTypes, boolean enabledForScan) { this.id = id == null ? UUIDs.base64UUID() : id; this.version = version != null ? version : NO_VERSION; this.name = name; @@ -314,7 +314,7 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver RefreshType refreshType = null; Instant lastRefreshedTime = null; User lastRefreshedUser = null; - Boolean isEnabled = null; + boolean isEnabled = true; List iocTypes = new ArrayList<>(); boolean enabledForScan = true; @@ -326,13 +326,25 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver case SOURCE_CONFIG_FIELD: break; case NAME_FIELD: - name = xcp.text(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + name = null; + } else { + name = xcp.text(); + } break; case FORMAT_FIELD: - format = xcp.text(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + format = null; + } else { + format = xcp.text(); + } break; case TYPE_FIELD: - sourceConfigType = toSourceConfigType(xcp.text()); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + sourceConfigType = null; + } else { + sourceConfigType = toSourceConfigType(xcp.text()); + } break; case DESCRIPTION_FIELD: if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { @@ -426,7 +438,6 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver case ENABLED_FIELD: isEnabled = xcp.booleanValue(); break; - case ENABLED_FOR_SCAN_FIELD: enabledForScan = xcp.booleanValue(); break; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java index 6864f7a98..2421e5e5c 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java @@ -106,7 +106,7 @@ protected void doExecute(Task task, DocLevelMonitorFanOutRequest request, Action iocTypeToIndicesMap -> { onGetIocTypeToIndices(iocTypeToIndicesMap, request, actionListener); }, e -> { - log.error(() -> new ParameterizedMessage("Unexpected Failure in threat intel monitor {} fan out action", request.getMonitor().getId()), e); + log.error(() -> new ParameterizedMessage("Unexpected Failure in threat intel monitor {} fan out action while fetching threat intel ioc indices", request.getMonitor().getId()), e); actionListener.onResponse( new DocLevelMonitorFanOutResponse( clusterService.localNode().getId(), @@ -162,6 +162,20 @@ private void onGetIocTypeToIndices(Map> iocTypeToIndicesMap }; ActionListener> searchHitsListener = ActionListener.wrap( (List hits) -> { + if (hits.isEmpty()) { + actionListener.onResponse( + new DocLevelMonitorFanOutResponse( + clusterService.localNode().getId(), + request.getExecutionId(), + request.getMonitor().getId(), + updatedLastRunContext, + new InputRunResults(Collections.emptyList(), null, null), + Collections.emptyMap(), + null + ) + ); + return; + } BiConsumer resultConsumer = (r, e) -> { if (e == null) { actionListener.onResponse( @@ -195,7 +209,7 @@ private void onGetIocTypeToIndices(Map> iocTypeToIndicesMap ), resultConsumer); }, e -> { - log.error("unexpected error while", e); + log.error("unexpected error while trying to query shards and fetch docs before scanning for malicious IoC's", e); actionListener.onFailure(e); } ); @@ -290,6 +304,11 @@ private void fetchLatestDocsFromShard( // recursive call to fetch docs with updated seq no. fetchLatestDocsFromShard(shardId, fromSeqNo, updatedToSeqNo, searchHitsSoFar, monitor, shardLastSeenMapForIndex, updateLastRunContext, fieldsToFetch, listener); }, e -> { + if(e.getMessage().contains("all shards failed") && e.getCause().getMessage().contains("No mapping found for [_seq_no] in order to sort on")) { + // this implies that the index being queried doesn't have any docs and hence doesn't understand the in-built _seq_no field mapping + listener.onResponse(new SearchHitsOrException(Collections.emptyList(), null)); + return; + } log.error(() -> new ParameterizedMessage("Threat intel Monitor {}: Failed to search shard {} in index {}", monitor.getId(), shard, shardId.getIndexName()), e); listener.onResponse(new SearchHitsOrException(searchHitsSoFar, e)); } diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestListIOCsAction.java similarity index 90% rename from src/main/java/org/opensearch/securityanalytics/resthandler/RestListIOCsAction.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestListIOCsAction.java index f068be77c..e40aa6f71 100644 --- a/src/main/java/org/opensearch/securityanalytics/resthandler/RestListIOCsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestListIOCsAction.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.resthandler; +package org.opensearch.securityanalytics.threatIntel.resthandler; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -18,9 +18,9 @@ import org.opensearch.rest.RestResponse; import org.opensearch.rest.action.RestResponseListener; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; -import org.opensearch.securityanalytics.action.ListIOCsAction; -import org.opensearch.securityanalytics.action.ListIOCsActionRequest; -import org.opensearch.securityanalytics.action.ListIOCsActionResponse; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsAction; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionRequest; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionResponse; import java.io.IOException; import java.util.List; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java index c8e9bd12b..dee9ae013 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -50,10 +50,10 @@ import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.threatIntel.action.monitor.SearchThreatIntelMonitorAction; import org.opensearch.securityanalytics.threatIntel.action.monitor.request.SearchThreatIntelMonitorRequest; -import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.util.IndexUtils; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.threadpool.ThreadPool; @@ -80,6 +80,7 @@ import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.SOURCE_CONFIG_FIELD; import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.STATE_FIELD; import static org.opensearch.securityanalytics.transport.TransportIndexDetectorAction.PLUGIN_OWNER_FIELD; +import static org.opensearch.securityanalytics.util.IndexUtils.shouldUpdateIndex; /** * CRUD for threat intel feeds source config object @@ -93,7 +94,6 @@ public class SATIFSourceConfigService { private final NamedXContentRegistry xContentRegistry; private final TIFLockService lockService; - public SATIFSourceConfigService(final Client client, final ClusterService clusterService, ThreadPool threadPool, @@ -139,7 +139,7 @@ public void indexTIFSourceConfig(SATIFSourceConfig saTifSourceConfig, } }, exception -> { log.error("Failed to create threat intel source config index", exception); - lockService.releaseLockEventDriven(lock, ActionListener.wrap( + lockService.releaseLock(lock, ActionListener.wrap( r -> { log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); actionListener.onFailure(exception); @@ -197,33 +197,75 @@ private String getIndexMapping() { /** * Index name: .opensearch-sap--job * Mapping: /mappings/threat_intel_job_mapping.json + * Updates the job index mapping if currently on a previous version * * @param stepListener setup listener */ public void createJobIndexIfNotExists(final StepListener stepListener) { // check if job index exists if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == true) { - stepListener.onResponse(null); - return; - } - final CreateIndexRequest createIndexRequest = new CreateIndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME).mapping(getIndexMapping()) - .settings(SecurityAnalyticsPlugin.TIF_JOB_INDEX_SETTING); - StashedThreadContext.run(client, () -> client.admin().indices().create(createIndexRequest, ActionListener.wrap( - r -> { - log.debug("[{}] index created", SecurityAnalyticsPlugin.JOB_INDEX_NAME); - stepListener.onResponse(null); - }, e -> { - if (e instanceof ResourceAlreadyExistsException) { - log.info("Index [{}] already exists", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + checkAndUpdateJobIndexMapping(stepListener); + } else { + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME).mapping(getIndexMapping()) + .settings(SecurityAnalyticsPlugin.TIF_JOB_INDEX_SETTING); + client.admin().indices().create(createIndexRequest, ActionListener.wrap( + r -> { + log.debug("[{}] index created", SecurityAnalyticsPlugin.JOB_INDEX_NAME); stepListener.onResponse(null); - return; + }, e -> { + if (e instanceof ResourceAlreadyExistsException) { + log.info("Index [{}] already exists", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + stepListener.onResponse(null); + return; + } + log.error("Failed to create [{}] index", SecurityAnalyticsPlugin.JOB_INDEX_NAME, e); + stepListener.onFailure(e); } - log.error("Failed to create [{}] index", SecurityAnalyticsPlugin.JOB_INDEX_NAME, e); - stepListener.onFailure(e); - } - ))); + )); + } } + private void checkAndUpdateJobIndexMapping(StepListener stepListener) { + try { + // Check if job index contains old mapping, if so update index mapping (current version = 2) + if (shouldUpdateIndex(clusterService.state().metadata().index(SecurityAnalyticsPlugin.JOB_INDEX_NAME), getIndexMapping())) { + log.info("Old schema version found for [{}] index, updating the index mapping", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + IndexUtils.updateIndexMapping( + SecurityAnalyticsPlugin.JOB_INDEX_NAME, + getIndexMapping(), clusterService.state(), client.admin().indices(), + ActionListener.wrap( + r -> { + log.info("Successfully updated index mapping for [{}] index", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + stepListener.onResponse(null); + }, e -> { + // Check if version is updated despite failure + try { + // Check if job index still contains older mapping + if (shouldUpdateIndex(clusterService.state().metadata().index(SecurityAnalyticsPlugin.JOB_INDEX_NAME), getIndexMapping())) { + log.error("Job index still contains older mapping, failed to update job index mapping", e); + stepListener.onFailure(e); + } else { + // If job index contains newest mapping, then return success + log.info("Successfully updated index mapping for [{}] index", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + stepListener.onResponse(null); + } + } catch (IOException exception) { + log.error("Failed to check if job index contains older mapping. Failed to update job index mapping", e); + stepListener.onFailure(e); + } + } + ), + false + ); + } else { + // If job index contains newest mapping, then do nothing + stepListener.onResponse(null); + } + } catch (IOException e) { + log.error("Failed to check and update job index mapping", e); + stepListener.onFailure(e); + } + } // Get TIF source config public void getTIFSourceConfig( @@ -234,7 +276,7 @@ public void getTIFSourceConfig( client.get(getRequest, ActionListener.wrap( getResponse -> { if (!getResponse.isExists()) { - actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"Threat intel source config [%s] not found.", tifSourceConfigId), RestStatus.NOT_FOUND))); + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "Threat intel source config [%s] not found.", tifSourceConfigId), RestStatus.NOT_FOUND))); return; } SATIFSourceConfig saTifSourceConfig = null; @@ -246,7 +288,7 @@ public void getTIFSourceConfig( saTifSourceConfig = SATIFSourceConfig.docParse(xcp, getResponse.getId(), getResponse.getVersion()); } if (saTifSourceConfig == null) { - actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"No threat intel source config exists [%s]", tifSourceConfigId), RestStatus.BAD_REQUEST))); + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "No threat intel source config exists [%s]", tifSourceConfigId), RestStatus.BAD_REQUEST))); } else { log.debug("Threat intel source config with id [{}] fetched", getResponse.getId()); actionListener.onResponse(saTifSourceConfig); @@ -366,9 +408,9 @@ public void deleteTIFSourceConfig( log.info("Deleted threat intel source config [{}] successfully", saTifSourceConfig.getId()); actionListener.onResponse(deleteResponse); } else if (deleteResponse.status().equals(RestStatus.NOT_FOUND)) { - actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Threat intel source config with id [{%s}] not found", saTifSourceConfig.getId()), RestStatus.NOT_FOUND))); + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "Threat intel source config with id [{%s}] not found", saTifSourceConfig.getId()), RestStatus.NOT_FOUND))); } else { - actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Failed to delete threat intel source config [{%s}]", saTifSourceConfig.getId()), deleteResponse.status()))); + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "Failed to delete threat intel source config [{%s}]", saTifSourceConfig.getId()), deleteResponse.status()))); } }, e -> { log.error("Failed to delete threat intel source config with id [{}]", saTifSourceConfig.getId()); @@ -407,7 +449,7 @@ public void deleteJobSchedulerLockIfJobDisabled( log.info("Threat intel job scheduler lock with id [{}] not found", id); actionListener.onResponse(deleteResponse); } else { - actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Failed to delete threat intel job scheduler lock with id [{%s}]", id), deleteResponse.status()))); + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "Failed to delete threat intel job scheduler lock with id [{%s}]", id), deleteResponse.status()))); } }, e -> { log.error("Failed to delete threat intel job scheduler lock with id [{}]", id); @@ -453,7 +495,7 @@ private void deleteIocIndex(Set indicesToDelete, Boolean backgroundJob, if (!response.isAcknowledged()) { log.error("Could not delete one or more IOC indices: " + index); if (backgroundJob == false) { - listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Could not delete one or more IOC indices: " + index), RestStatus.BAD_REQUEST))); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "Could not delete one or more IOC indices: " + index), RestStatus.BAD_REQUEST))); } } else { log.debug("Successfully deleted one or more IOC indices:" + index); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java index ecdeaa683..afd22f799 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java @@ -20,10 +20,8 @@ import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigResponse; -import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; -import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; import org.opensearch.securityanalytics.transport.SecureTransportAction; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; @@ -106,7 +104,7 @@ private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest reques user, ActionListener.wrap( saTifSourceConfigDtoResponse -> { - lockService.releaseLockEventDriven(lock, ActionListener.wrap( + lockService.releaseLock(lock, ActionListener.wrap( r -> { log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); listener.onResponse(new SAIndexTIFSourceConfigResponse( @@ -129,7 +127,7 @@ private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest reques }, e -> { String action = RestRequest.Method.PUT.equals(request.getMethod()) ? "update" : "create"; log.error(String.format("Failed to %s IOCs and threat intel source config", action), e); - lockService.releaseLockEventDriven(lock, ActionListener.wrap( + lockService.releaseLock(lock, ActionListener.wrap( r -> { log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); listener.onFailure(e); @@ -146,7 +144,7 @@ private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest reques } catch (Exception e) { String action = RestRequest.Method.PUT.equals(request.getMethod()) ? "update" : "create"; log.error(String.format("Failed to %s IOCs and threat intel source config", action), e); - lockService.releaseLockEventDriven(lock, ActionListener.wrap( + lockService.releaseLock(lock, ActionListener.wrap( r -> { log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); listener.onFailure(e); diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportListIOCsAction.java similarity index 69% rename from src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportListIOCsAction.java index 77c117784..80a6b538c 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportListIOCsAction.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.transport; +package org.opensearch.securityanalytics.threatIntel.transport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -19,40 +19,40 @@ import org.opensearch.cluster.routing.Preference; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.commons.alerting.model.Table; +import org.opensearch.commons.authuser.User; import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.BoolQueryBuilder; -import org.opensearch.index.query.Operator; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.Aggregations; +import org.opensearch.search.aggregations.bucket.terms.Terms; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.sort.FieldSortBuilder; import org.opensearch.search.sort.SortBuilder; import org.opensearch.search.sort.SortBuilders; import org.opensearch.search.sort.SortOrder; -import org.opensearch.securityanalytics.action.ListIOCsAction; -import org.opensearch.securityanalytics.action.ListIOCsActionRequest; -import org.opensearch.securityanalytics.action.ListIOCsActionResponse; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsAction; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionRequest; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionResponse; import org.opensearch.securityanalytics.model.DetailedSTIX2IOCDto; import org.opensearch.securityanalytics.model.STIX2IOC; import org.opensearch.securityanalytics.model.STIX2IOCDto; -import org.opensearch.securityanalytics.model.threatintel.IocFinding; -import org.opensearch.securityanalytics.model.threatintel.IocWithFeeds; -import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; -import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsRequest; -import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsResponse; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; -import org.opensearch.securityanalytics.threatIntel.transport.TransportSearchTIFSourceConfigsAction; +import org.opensearch.securityanalytics.transport.SecureTransportAction; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; @@ -83,6 +83,12 @@ public class TransportListIOCsAction extends HandledTransportAction defaultTifConfigsLoadedListener = null; try { defaultTifConfigsLoadedListener = new StepListener<>(); @@ -200,6 +219,7 @@ private void listIocs(List iocIndices) { .version(true) .seqNoAndPrimaryTerm(true) .fetchSource(true) + .trackTotalHits(true) .query(boolQueryBuilder) .sort(sortBuilder) .size(request.getTable().getSize()) @@ -217,80 +237,7 @@ public void onResponse(SearchResponse searchResponse) { onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT)); } - // Concurrently compiling a separate list of IOC IDs to create the subsequent GetIocFindingsRequest - Set iocIds = new HashSet<>(); - List iocs = new ArrayList<>(); - Arrays.stream(searchResponse.getHits().getHits()) - .forEach(hit -> { - try { - XContentParser xcp = XContentType.JSON.xContent().createParser( - xContentRegistry, - LoggingDeprecationHandler.INSTANCE, - hit.getSourceAsString()); - xcp.nextToken(); - - STIX2IOCDto ioc = STIX2IOCDto.parse(xcp, hit.getId(), hit.getVersion()); - - iocIds.add(ioc.getId()); - iocs.add(ioc); - } catch (Exception e) { - log.error( - () -> new ParameterizedMessage("Failed to parse IOC doc from hit {}", hit.getId()), e - ); - } - }); - - GetIocFindingsRequest getFindingsRequest = new GetIocFindingsRequest( - Collections.emptyList(), - new ArrayList<>(iocIds), - null, - null, - new Table( - "asc", - "timestamp", - request.getTable().getMissing(), - 10000, - 0, - "" - ) - ); - - // Calling GetIocFindings API to get number of findings for each returned IOC - client.execute(GetIocFindingsAction.INSTANCE, getFindingsRequest, new ActionListener<>() { - @Override - public void onResponse(GetIocFindingsResponse getFindingsResponse) { - // Iterate through the GetIocFindingsResponse to count occurrences of each IOC - Map iocIdToNumFindings = new HashMap<>(); - for (IocFinding iocFinding : getFindingsResponse.getIocFindings()) { - for (IocWithFeeds iocWithFeeds : iocFinding.getFeedIds()) { - // Set the count to 0 if it's not already - iocIdToNumFindings.putIfAbsent(iocWithFeeds.getIocId(), 0); - // Increment the count for the IOC - iocIdToNumFindings.merge(iocWithFeeds.getIocId(), 1, Integer::sum); - } - } - - // Iterate through each IOC returned by the SearchRequest to create the detailed model for response - List iocDetails = new ArrayList<>(); - iocs.forEach((ioc) -> { - Integer numFindings = iocIdToNumFindings.get(ioc.getId()); - if (numFindings == null) { - // Logging instances of 'null' separately from 0 instances for investigation purposes - log.debug("Null number of findings found for IOC {}", ioc.getId()); - numFindings = 0; - } - iocDetails.add(new DetailedSTIX2IOCDto(ioc, numFindings)); - }); - - onOperation(new ListIOCsActionResponse(searchResponse.getHits().getTotalHits().value, iocDetails)); - } - - @Override - public void onFailure(Exception e) { - log.error("Failed to get IOC findings count:", e); - listener.onFailure(SecurityAnalyticsException.wrap(e)); - } - }); + getFindingsCount(searchResponse); } @Override @@ -306,6 +253,79 @@ public void onFailure(Exception e) { }); } + private void getFindingsCount(SearchResponse iocSearchResponse) { + // Concurrently compiling a separate list of IOC IDs to create the subsequent findings count searchRequest + Set iocIds = new HashSet<>(); + List iocs = new ArrayList<>(); + Arrays.stream(iocSearchResponse.getHits().getHits()) + .forEach(hit -> { + try { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, + hit.getSourceAsString()); + xcp.nextToken(); + + STIX2IOCDto ioc = STIX2IOCDto.parse(xcp, hit.getId(), hit.getVersion()); + + iocIds.add(ioc.getId()); + iocs.add(ioc); + } catch (Exception e) { + log.error( + () -> new ParameterizedMessage("Failed to parse IOC doc from hit {}", hit.getId()), e + ); + } + }); + + // Create an aggregation query that will group by the IOC IDs in the findings + SearchSourceBuilder findingsCountSourceBuilder = new SearchSourceBuilder() + .fetchSource(false) + .trackTotalHits(true) + .query(QueryBuilders.termsQuery(IOC_ID_KEYWORD_FIELD, iocIds)) + .size(0) + .aggregation( + AggregationBuilders + .terms(IOC_COUNT_AGG_NAME) + .field(IOC_ID_KEYWORD_FIELD) + .size(iocIds.size()) + ); + + iocFindingService.search(findingsCountSourceBuilder, new ActionListener<>() { + @Override + public void onResponse(SearchResponse findingsSearchResponse) { + Map iocIdToNumFindings = new HashMap<>(); + + // Retrieve and store the counts from the aggregation response + Aggregations aggregations = findingsSearchResponse.getAggregations(); + if (aggregations != null) { + Terms iocIdCount = aggregations.get(IOC_COUNT_AGG_NAME); + if (iocIdCount != null) { + for (Terms.Bucket bucket : iocIdCount.getBuckets()) { + String iocId = bucket.getKeyAsString(); + long findingCount = bucket.getDocCount(); + iocIdToNumFindings.put(iocId, (int) findingCount); + } + } + } + + // Iterate through each IOC returned by the SearchRequest to create the detailed model for response + List iocDetails = new ArrayList<>(); + iocs.forEach((ioc) -> { + Integer numFindings = iocIdToNumFindings.getOrDefault(ioc.getId(), 0); + iocDetails.add(new DetailedSTIX2IOCDto(ioc, numFindings)); + }); + + // Return API response + onOperation(new ListIOCsActionResponse(iocSearchResponse.getHits().getTotalHits().value, iocDetails)); + } + + @Override + public void onFailure(Exception e) { + log.error("Failed to get IOC findings count:", e); + listener.onFailure(SecurityAnalyticsException.wrap(e)); + } + }); + } private void onOperation(ListIOCsActionResponse response) { this.response.set(response); if (counter.compareAndSet(false, true)) { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java index a72844e40..4c5bd5e4d 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java @@ -107,7 +107,7 @@ protected void doExecute(final Task task, final PutTIFJobRequest request, final internalDoExecute(request, lock, listener); } catch (Exception e) { log.error("Failed execution to put tif job action", e); - lockService.releaseLockEventDriven(lock, ActionListener.wrap( + lockService.releaseLock(lock, ActionListener.wrap( r -> { log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); listener.onFailure(e); @@ -147,7 +147,7 @@ protected void internalDoExecute( } }, exception -> { log.error("Failed to save tif job parameter", exception); - lockService.releaseLockEventDriven(lock, ActionListener.wrap( + lockService.releaseLock(lock, ActionListener.wrap( r -> { log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); listener.onFailure(exception); @@ -176,7 +176,7 @@ protected ActionListener postIndexingTifJobParameter( createThreatIntelFeedData(tifJobParameter, lockService.getRenewLockRunnable(lockReference), ActionListener.wrap( threatIntelIndicesResponse -> { if (threatIntelIndicesResponse.isAcknowledged()) { - lockService.releaseLockEventDriven(lockReference.get(), ActionListener.wrap( + lockService.releaseLock(lockReference.get(), ActionListener.wrap( r -> { log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); listener.onResponse(new AcknowledgedResponse(true)); @@ -200,7 +200,7 @@ protected ActionListener postIndexingTifJobParameter( log.error("Internal server error"); exception = e; } - lockService.releaseLockEventDriven(lock, ActionListener.wrap( + lockService.releaseLock(lock, ActionListener.wrap( r -> { log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); listener.onFailure(exception); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java index 3edb6ea94..c9e364da7 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java @@ -239,6 +239,7 @@ private Monitor buildThreatIntelMonitor(IndexThreatIntelMonitorRequest request) triggers, Collections.emptyMap(), new DataSources(), + false, PLUGIN_OWNER_FIELD ); } catch (Exception e) { diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportAckCorrelationAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportAckCorrelationAlertsAction.java index 917d0349c..7032819de 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportAckCorrelationAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportAckCorrelationAlertsAction.java @@ -67,6 +67,8 @@ protected void doExecute(Task task, AckCorrelationAlertsRequest request, ActionL return; } + this.threadPool.getThreadContext().stashContext(); + if (!request.getCorrelationAlertIds().isEmpty()) { correlationAlertService.acknowledgeAlerts( request.getCorrelationAlertIds(), diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetCorrelationAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetCorrelationAlertsAction.java index cdca86a23..a19817e5b 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetCorrelationAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetCorrelationAlertsAction.java @@ -64,6 +64,8 @@ protected void doExecute(Task task, GetCorrelationAlertsRequest request, ActionL return; } + this.threadPool.getThreadContext().stashContext(); + if (request.getCorrelationRuleId() != null) { correlationAlertService.getCorrelationAlerts( request.getCorrelationRuleId(), diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index 28ec5fcd8..f415d0f2a 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -155,6 +155,8 @@ public class TransportIndexDetectorAction extends HandledTransportAction detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), PLUGIN_OWNER_FIELD); + true), enableDetectorWithDedicatedQueryIndices, PLUGIN_OWNER_FIELD); return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } @@ -886,14 +890,14 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest( Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, monitorName, false, detector.getSchedule(), detector.getLastUpdateTime(), null, Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), - new DataSources(detector.getRuleIndex(), + new DataSources(enableDetectorWithDedicatedQueryIndices? detector.getRuleIndex() + "_chained_findings": detector.getRuleIndex(), detector.getFindingsIndex(), detector.getFindingsIndexPattern(), detector.getAlertsIndex(), detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), PLUGIN_OWNER_FIELD); + true), enableDetectorWithDedicatedQueryIndices, PLUGIN_OWNER_FIELD); return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } @@ -1063,7 +1067,7 @@ public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) { detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), PLUGIN_OWNER_FIELD); + true), false, PLUGIN_OWNER_FIELD); listener.onResponse(new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null)); } @@ -1247,7 +1251,13 @@ void createDetector() { request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic)); request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic)); request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic)); - request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); + + if (enableDetectorWithDedicatedQueryIndices) { + // disabling the setting after enabling it will mean delete & re-create the detector + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); + } else { + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); + } User originalContextUser = this.user; log.debug("user from original context is {}", originalContextUser); @@ -1364,7 +1374,16 @@ void onGetResponse(Detector currentDetector, User user) { request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic)); request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic)); request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic)); - request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); + if (currentDetector.getRuleIndex().contains("optimized")) { + request.getDetector().setRuleIndex(currentDetector.getRuleIndex()); + } else { + if (enableDetectorWithDedicatedQueryIndices) { + // disabling the setting after enabling it will mean delete & re-create the detector + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); + } else { + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); + } + } request.getDetector().setUser(user); if (!detector.getInputs().isEmpty()) { @@ -1800,4 +1819,8 @@ private void setFilterByEnabled(boolean filterByEnabled) { private void setEnabledWorkflowUsage(boolean enabledWorkflowUsage) { this.enabledWorkflowUsage = enabledWorkflowUsage; } + + private void setEnabledDetectorsWithDedicatedQueryIndices(boolean enabledDetectorsWithDedicatedQueryIndices) { + this.enableDetectorWithDedicatedQueryIndices = enabledDetectorsWithDedicatedQueryIndices; + } } diff --git a/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java b/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java index 375342d09..36fd5e37d 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java @@ -7,6 +7,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.alias.Alias; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.core.action.ActionListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; @@ -26,6 +27,9 @@ import java.nio.charset.Charset; import java.util.Objects; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; + public class CorrelationIndices { private static final Logger log = LogManager.getLogger(CorrelationIndices.class); @@ -55,9 +59,15 @@ public static String correlationMappings() throws IOException { public void initCorrelationIndex(ActionListener actionListener) throws IOException { if (!correlationIndexExists()) { + Settings indexSettings = Settings.builder() + .put("index.hidden", true) + .put("index.correlation", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(CORRELATION_HISTORY_INDEX_PATTERN) .mapping(correlationMappings()) - .settings(Settings.builder().put("index.hidden", true).put("index.correlation", true).build()); + .settings(indexSettings); indexRequest.alias(new Alias(CORRELATION_HISTORY_WRITE_INDEX)); client.admin().indices().create(indexRequest, actionListener); } else { @@ -67,9 +77,15 @@ public void initCorrelationIndex(ActionListener actionListe public void initCorrelationMetadataIndex(ActionListener actionListener) throws IOException { if (!correlationMetadataIndexExists()) { + Settings indexSettings = Settings.builder() + .put("index.hidden", true) + .put("index.correlation", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(CORRELATION_METADATA_INDEX) .mapping(correlationMappings()) - .settings(Settings.builder().put("index.hidden", true).put("index.correlation", true).build()); + .settings(indexSettings); client.admin().indices().create(indexRequest, actionListener); } else { actionListener.onResponse(new CreateIndexResponse(true, true, CORRELATION_METADATA_INDEX)); @@ -136,6 +152,8 @@ public static String correlationAlertIndexMappings() throws IOException { public void initCorrelationAlertIndex(ActionListener actionListener) throws IOException { Settings correlationAlertSettings = Settings.builder() .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(CORRELATION_ALERT_INDEX) .mapping(correlationAlertIndexMappings()) diff --git a/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java index d131e47b4..27f6475f8 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.core.action.ActionListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; @@ -23,6 +24,9 @@ import java.util.Objects; import org.opensearch.securityanalytics.model.CorrelationRule; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; + public class CorrelationRuleIndices { private static final Logger log = LogManager.getLogger(CorrelationRuleIndices.class); @@ -45,9 +49,14 @@ public static String correlationRuleIndexMappings() throws IOException { public void initCorrelationRuleIndex(ActionListener actionListener) throws IOException { if (!correlationRuleIndexExists()) { + Settings indexSettings = Settings.builder() + .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(CorrelationRule.CORRELATION_RULE_INDEX).mapping( correlationRuleIndexMappings() - ).settings(Settings.builder().put("index.hidden", true).build()); + ).settings(indexSettings); client.admin().indices().create(indexRequest, actionListener); } } diff --git a/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java b/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java index 21eb460d5..d2b5661fe 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.nio.charset.Charset; import java.util.Objects; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; public class CustomLogTypeIndices { @@ -42,9 +44,11 @@ public static String customLogTypeMappings() throws IOException { public void initCustomLogTypeIndex(ActionListener actionListener) throws IOException { if (!customLogTypeIndexExists()) { + // Security Analytics log types index is small. 1 primary shard is enough Settings indexSettings = Settings.builder() .put("index.hidden", true) - .put("index.auto_expand_replicas", "0-all") + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(LogTypeService.LOG_TYPE_INDEX) .mapping(customLogTypeMappings()) diff --git a/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java b/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java index d6a81e134..83eb058e0 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java @@ -23,6 +23,9 @@ import java.nio.charset.Charset; import java.util.Objects; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; + public class DetectorIndices { private static final Logger log = LogManager.getLogger(DetectorIndices.class); @@ -45,9 +48,14 @@ public static String detectorMappings() throws IOException { public void initDetectorIndex(ActionListener actionListener) throws IOException { if (!detectorIndexExists()) { + Settings indexSettings = Settings.builder() + .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(Detector.DETECTORS_INDEX) .mapping(detectorMappings()) - .settings(Settings.builder().put("index.hidden", true).build()); + .settings(indexSettings); client.indices().create(indexRequest, actionListener); } } diff --git a/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java b/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java index 9d5066308..c4dafed85 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java @@ -31,9 +31,9 @@ public class IndexUtils { - private static final String _META = "_meta"; - private static final Integer NO_SCHEMA_VERSION = 0; - private static final String SCHEMA_VERSION = "schema_version"; + public static final String _META = "_meta"; + public static final Integer NO_SCHEMA_VERSION = 0; + public static final String SCHEMA_VERSION = "schema_version"; public static Boolean detectorIndexUpdated = false; public static Boolean customRuleIndexUpdated = false; diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java index 94a98106b..b1ff516d5 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java @@ -65,6 +65,8 @@ import java.util.stream.Stream; import static org.opensearch.securityanalytics.model.Detector.NO_VERSION; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; public class RuleIndices { @@ -93,6 +95,8 @@ public void initRuleIndex(ActionListener actionListener, bo if (!ruleIndexExists(isPrepackaged)) { Settings indexSettings = Settings.builder() .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(getRuleIndex(isPrepackaged)) .mapping(ruleMappings()) diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java index 8f0f1cab5..7aa2def36 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java @@ -46,13 +46,9 @@ public static String ruleTopicIndexSettings() throws IOException { public void initRuleTopicIndexTemplate(ActionListener actionListener) throws IOException { getAllRuleIndices(ActionListener.wrap(allRuleIndices -> { // Compose list of all patterns to cover all query indices - List indexPatterns = new ArrayList<>(); - for(String ruleIndex : allRuleIndices) { - indexPatterns.add(ruleIndex + "*"); - } ComposableIndexTemplate template = new ComposableIndexTemplate( - indexPatterns, + allRuleIndices, new Template( Settings.builder().loadFromSource(ruleTopicIndexSettings(), XContentType.JSON).build(), null, @@ -87,7 +83,8 @@ private void getAllRuleIndices(ActionListener> listener) { listener.onResponse( logTypes .stream() - .map(logType -> DetectorMonitorConfig.getRuleIndex(logType)) + // use index pattern here to define rule topic index template for all query indices which match the pattern + .map(logType -> DetectorMonitorConfig.getRuleIndex(logType) + "*") .collect(Collectors.toList()) ); }, listener::onFailure)); diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index e4a14ac56..f196c4a5a 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -1,2 +1,3 @@ org.opensearch.securityanalytics.correlation.index.codec.correlation950.CorrelationCodec950 -org.opensearch.securityanalytics.correlation.index.codec.correlation990.CorrelationCodec990 \ No newline at end of file +org.opensearch.securityanalytics.correlation.index.codec.correlation990.CorrelationCodec990 +org.opensearch.securityanalytics.correlation.index.codec.correlation9120.CorrelationCodec9120 \ No newline at end of file diff --git a/src/main/resources/mappings/threat_intel_job_mapping.json b/src/main/resources/mappings/threat_intel_job_mapping.json index f437efe6f..fbc1c03dc 100644 --- a/src/main/resources/mappings/threat_intel_job_mapping.json +++ b/src/main/resources/mappings/threat_intel_job_mapping.json @@ -1,4 +1,5 @@ { + "dynamic": true, "_meta" : { "schema_version": 2 }, diff --git a/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java b/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java index 47c33e138..d6294ee76 100644 --- a/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java +++ b/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java @@ -75,7 +75,7 @@ public void testCreateDetectorWithThreatIntelEnabled_updateDetectorWithThreatInt " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(2, response.getHits().getTotalHits().value); @@ -275,7 +275,7 @@ public void testCreateDetectorWithThreatIntelDisabled_updateDetectorWithThreatIn " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -372,7 +372,7 @@ public void testCreateDetectorWithThreatIntelEnabledAndNoRules_triggerDetectionT " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -466,7 +466,7 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerDetectionTypeOnlyThr " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -561,7 +561,7 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerWithBothDetectionTyp " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -653,7 +653,7 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithThreatIntelDete " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -745,7 +745,7 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -802,4 +802,4 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT /** findings are present but alerts are NOT generated as detection type mentioned in trigger is threat_intel only but finding is from rules*/ Assert.assertEquals(3, getAlertsBody.get("total_alerts")); } -} +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index ab66e1d9f..26ffe4344 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -269,6 +269,7 @@ void setDebugLogLevel() throws IOException, InterruptedException { makeRequest(client(), "PUT", "_cluster/settings", Collections.emptyMap(), se, new BasicHeader("Content-Type", "application/json")); + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "true"); } protected final List clusterPermissions = List.of( diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 75ee7cd89..4a7fffd06 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -900,6 +900,16 @@ public static Action randomAction(String destinationId) { return new Action(name, destinationId, template, template, throttleEnabled, throttle, OpenSearchRestTestCase.randomAlphaOfLength(10), null); } + public static Action randomThreatInteMonitorAction(String destinationId) { + String name = OpenSearchRestTestCase.randomUnicodeOfLength(10); + Script template = randomTemplateScript("Threat intel Monitor {{ctx.monitor.name}} just entered alert status. Please investigate the issue.\n" + + " - Trigger: {{ctx.trigger.name}}\n" + + " - Severity: {{ctx.trigger.severity}}", null); + Boolean throttleEnabled = false; + Throttle throttle = randomThrottle(null, null); + return new Action(name, destinationId, template, template, throttleEnabled, throttle, OpenSearchRestTestCase.randomAlphaOfLength(10), null); + } + public static Script randomTemplateScript(String source, Map params) { if (params == null) { params = new HashMap<>(); @@ -1874,6 +1884,68 @@ public static String windowsIndexMappingOnlyNumericAndText() { " }"; } + public static String oldThreatIntelJobMapping() { + return " \"dynamic\": \"strict\",\n" + + " \"_meta\": {\n" + + " \"schema_version\": 1\n" + + " },\n" + + " \"properties\": {\n" + + " \"schema_version\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"enabled_time\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"indices\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"last_update_time\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"schedule\": {\n" + + " \"properties\": {\n" + + " \"interval\": {\n" + + " \"properties\": {\n" + + " \"period\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"start_time\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"unit\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"state\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"update_enabled\": {\n" + + " \"type\": \"boolean\"\n" + + " },\n" + + " \"update_stats\": {\n" + + " \"properties\": {\n" + + " \"last_failed_at_in_epoch_millis\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"last_processing_time_in_millis\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"last_skipped_at_in_epoch_millis\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"last_succeeded_at_in_epoch_millis\": {\n" + + " \"type\": \"long\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; + } public static String randomDoc(int severity, int version, String opCode) { String doc = "{\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigRequestTests.java b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigRequestTests.java index b572d2ca6..e40516e25 100644 --- a/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigRequestTests.java +++ b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigRequestTests.java @@ -5,6 +5,7 @@ package org.opensearch.securityanalytics.action; import org.junit.Assert; +import org.opensearch.action.ActionRequestValidationException; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.rest.RestRequest; @@ -33,4 +34,40 @@ public void testTIFSourceConfigPostRequest() throws IOException { Assert.assertEquals(RestRequest.Method.POST, newRequest.getMethod()); Assert.assertNotNull(newRequest.getTIFConfigDto()); } + + public void testValidateSourceConfigPostRequest() { + // Source config with invalid: name, format, source, ioc type, source config type + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + false, + null, + true + ); + String id = saTifSourceConfigDto.getId(); + SAIndexTIFSourceConfigRequest request = new SAIndexTIFSourceConfigRequest(id, RestRequest.Method.POST, saTifSourceConfigDto); + Assert.assertNotNull(request); + + ActionRequestValidationException exception = request.validate(); + assertEquals(5, exception.validationErrors().size()); + assertTrue(exception.validationErrors().contains("Name must not be empty")); + assertTrue(exception.validationErrors().contains("Format must not be empty")); + assertTrue(exception.validationErrors().contains("Source must not be empty")); + assertTrue(exception.validationErrors().contains("Must specify at least one IOC type")); + assertTrue(exception.validationErrors().contains("Type must not be empty")); + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java index 06e464d34..bbb3eb71a 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java @@ -95,6 +95,7 @@ public void testGetAlerts_success() { List.of(), Map.of(), new DataSources(), + true, TransportIndexDetectorAction.PLUGIN_OWNER_FIELD ), new DocumentLevelTrigger("trigger_id_1", "my_trigger", "severity_low", List.of(), new Script("")), @@ -129,6 +130,7 @@ public void testGetAlerts_success() { List.of(), Map.of(), new DataSources(), + true, TransportIndexDetectorAction.PLUGIN_OWNER_FIELD ), new DocumentLevelTrigger("trigger_id_1", "my_trigger", "severity_low", List.of(), new Script("")), diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index 77b54dbde..c755bf1d5 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -295,7 +295,7 @@ public void testGetAlertsByStartTimeAndEndTimeSuccess() throws IOException, Inte } public void testGetAlerts_noDetector_failure() throws IOException { - // Call GetAlerts API + // Call GetAlerts API Map params = new HashMap<>(); params.put("detector_id", "nonexistent_detector_id"); try { @@ -820,7 +820,7 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException Collections.emptyList()); Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), List.of(new DetectorTrigger("randomtrigegr", "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of())) - ); + ); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); @@ -831,7 +831,7 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); // 5 for rules, 1 for match_all query in chained findings monitor @@ -869,8 +869,8 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException } } - assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue()); - assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue()); + assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue()); + assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue()); Map params = new HashMap<>(); params.put("detector_id", detectorId); @@ -894,13 +894,13 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException List> queries = (List>) finding.get("queries"); Set findingRuleIds = queries.stream().map(it -> it.get("id").toString()).collect(Collectors.toSet()); - // In the case of bucket level monitors, queries will always contain one value - String aggRuleId = findingRuleIds.iterator().next(); - List findingDocs = (List) finding.get("related_doc_ids"); + // In the case of bucket level monitors, queries will always contain one value + String aggRuleId = findingRuleIds.iterator().next(); + List findingDocs = (List) finding.get("related_doc_ids"); - if (aggRuleId.equals(sumRuleId)) { - assertTrue(List.of("1", "2", "3", "4", "5", "6", "7").containsAll(findingDocs)); - } + if (aggRuleId.equals(sumRuleId)) { + assertTrue(List.of("1", "2", "3", "4", "5", "6", "7").containsAll(findingDocs)); + } } assertTrue(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8").containsAll(docLevelFinding)); diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index 90b008581..ab06b8d30 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -1223,7 +1223,7 @@ public void testCreateDetectorWithNotCondition_verifyFindingsAndNoFindings_succe " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index 67eb3a313..d389797c5 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -1656,8 +1656,8 @@ public void testTraverseAndCopy() { " \"type\":\"keyword\"," + " \"ignore_above\":256" + " }" + - " }" + - " }" + + " }" + + " }" + " }" + " }" + "}"; @@ -1723,7 +1723,7 @@ public void testAzureMappings() throws IOException { Detector detector = randomDetectorWithInputs(List.of(input), "azure"); createDetector(detector); - List hits = executeSearch(".opensearch-sap-azure-detectors-queries-000001", matchAllSearchBody); + List hits = executeSearch(".opensearch-sap-azure-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(127, hits.size()); } @@ -1748,7 +1748,7 @@ public void testADLDAPMappings() throws IOException { Detector detector = randomDetectorWithInputs(List.of(input), "ad_ldap"); createDetector(detector); - List hits = executeSearch(".opensearch-sap-ad_ldap-detectors-queries-000001", matchAllSearchBody); + List hits = executeSearch(".opensearch-sap-ad_ldap-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(11, hits.size()); } @@ -1773,7 +1773,7 @@ public void testCloudtrailMappings() throws IOException { Detector detector = randomDetectorWithInputs(List.of(input), "cloudtrail"); createDetector(detector); - List hits = executeSearch(".opensearch-sap-cloudtrail-detectors-queries-000001", matchAllSearchBody); + List hits = executeSearch(".opensearch-sap-cloudtrail-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(39, hits.size()); } @@ -1798,7 +1798,7 @@ public void testS3Mappings() throws IOException { Detector detector = randomDetectorWithInputs(List.of(input), "s3"); createDetector(detector); - List hits = executeSearch(".opensearch-sap-s3-detectors-queries-000001", matchAllSearchBody); + List hits = executeSearch(".opensearch-sap-s3-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(1, hits.size()); } @@ -1825,7 +1825,7 @@ public void testWAFMappings() throws IOException { Detector detector = randomDetectorWithInputs(List.of(input), "waf"); createDetector(detector); - List hits = executeSearch(".opensearch-sap-waf-detectors-queries-000001", matchAllSearchBody); + List hits = executeSearch(".opensearch-sap-waf-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(5, hits.size()); } diff --git a/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigDtoTests.java b/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigDtoTests.java index c9215af5a..e5b2ed7c5 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigDtoTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigDtoTests.java @@ -10,11 +10,15 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.model.S3Source; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; import static org.opensearch.securityanalytics.TestHelpers.randomSATIFSourceConfigDto; @@ -36,6 +40,34 @@ public void testParseFunction() throws IOException { assertEqualsSaTifSourceConfigDtos(saTifSourceConfigDto, newSaTifSourceConfigDto); } + public void testParseFunctionWithNullValues() throws IOException { + // Source config with invalid name and format + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + "randomId", + null, + null, + null, + SourceConfigType.S3_CUSTOM, + null, + null, + null, + new S3Source("bucket", "objectkey", "region", "rolearn"), + null, + null, + new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS), + null, + null, + null, + null, + true, + List.of("ip"), + true + ); + String json = toJsonString(saTifSourceConfigDto); + SATIFSourceConfigDto newSaTifSourceConfigDto = SATIFSourceConfigDto.parse(getParser(json), saTifSourceConfigDto.getId(), null); + assertEqualsSaTifSourceConfigDtos(saTifSourceConfigDto, newSaTifSourceConfigDto); + } + public XContentParser getParser(String xc) throws IOException { XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); parser.nextToken(); diff --git a/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java b/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java index 2687907d1..8fa8ec395 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java @@ -10,12 +10,17 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.S3Source; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; import static org.opensearch.securityanalytics.TestHelpers.randomSATIFSourceConfig; @@ -37,6 +42,35 @@ public void testParseFunction() throws IOException { assertEqualsSaTifSourceConfigs(saTifSourceConfig, newSaTifSourceConfig); } + public void testParseFunctionWithNullValues() throws IOException { + // Source config with invalid name and format + SATIFSourceConfig saTifSourceConfig = new SATIFSourceConfig( + null, + null, + null, + null, + SourceConfigType.S3_CUSTOM, + null, + null, + null, + new S3Source("bucket", "objectkey", "region", "rolearn"), + null, + null, + new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS), + null, + null, + null, + null, + true, + new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(IOCType.DOMAIN_NAME_TYPE), "indexPattern", "writeIndex"))), + List.of("ip"), + true + ); + String json = toJsonString(saTifSourceConfig); + SATIFSourceConfig newSaTifSourceConfig = SATIFSourceConfig.parse(getParser(json), saTifSourceConfig.getId(), null); + assertEqualsSaTifSourceConfigs(saTifSourceConfig, newSaTifSourceConfig); + } + public XContentParser getParser(String xc) throws IOException { XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); parser.nextToken(); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index a156344b6..f58c91831 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -1042,7 +1042,7 @@ public void testCreateDetector_verifyWorkflowCreation_success_WithoutGroupByRule " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(2, response.getHits().getTotalHits().value); @@ -1110,7 +1110,7 @@ public void testCreateDetector_verifyWorkflowCreation_success_WithGroupByRulesIn " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(2, response.getHits().getTotalHits().value); @@ -1379,7 +1379,7 @@ public void testCreateDetector_workflowWithDuplicateMonitor_failure() throws IOE " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(2, response.getHits().getTotalHits().value); @@ -1449,7 +1449,7 @@ public void testCreateDetector_verifyWorkflowExecutionBucketLevelDocLevelMonitor " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(2, response.getHits().getTotalHits().value); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index 75cbfd858..4aaf4f35b 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -161,7 +161,7 @@ private void validateDetectorDeletion(final String detectorId) throws IOExceptio Assert.assertEquals(0, hits.size()); } - @SuppressWarnings("unchecked") + @SuppressWarnings("unchecked") public void testCreatingADetector() throws IOException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); @@ -722,6 +722,12 @@ public void testCreatingADetectorWithAggregationRules() throws IOException { if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(secondMonitorType)){ bucketLevelMonitorId = secondMonitorId; } + String thirdMonitorId = monitorIds.get(2); + String thirdMonitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + thirdMonitorId))).get("monitor")).get("monitor_type"); + monitorTypes.add(thirdMonitorType); + if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(thirdMonitorType)){ + bucketLevelMonitorId = thirdMonitorId; + } Assert.assertTrue(Arrays.asList(MonitorType.BUCKET_LEVEL_MONITOR.getValue(), MonitorType.DOC_LEVEL_MONITOR.getValue()).containsAll(monitorTypes)); indexDoc(index, "1", randomProductDocument()); @@ -775,7 +781,13 @@ public void testAggRuleCount() throws IOException { Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); - String bucketLevelMonitorId = ((List) (detectorAsMap).get("monitor_id")).get(1); + String bucketLevelMonitorId = ""; + Map monitorOpts = ((Map) (detectorAsMap).get("bucket_monitor_id_rule_id")); + for (Map.Entry monitorOpt: monitorOpts.entrySet()) { + if (!(monitorOpt.getKey().equals("-1") || monitorOpt.getKey().equals("chained_findings_monitor"))) { + bucketLevelMonitorId = monitorOpt.getValue().toString(); + } + } // condition: sel | count(*) by name > 2 indexDoc(index, "1", randomProductDocument()); indexDoc(index, "2", randomProductDocument()); @@ -840,7 +852,7 @@ public void testUpdateADetector() throws IOException { " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); Assert.assertEquals(5, response.getHits().getTotalHits().value); String rule = randomRule(); @@ -869,7 +881,7 @@ public void testUpdateADetector() throws IOException { " }\n" + " }\n" + "}"; - response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); Assert.assertEquals(6, response.getHits().getTotalHits().value); } @@ -1056,10 +1068,10 @@ public void testDeletingADetector_single_Monitor() throws IOException { Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -1069,12 +1081,12 @@ public void testDeletingADetector_single_Monitor() throws IOException { String detectorId1 = createDetector(detector1); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -1096,12 +1108,12 @@ public void testDeletingADetector_single_Monitor() throws IOException { String detectorId2 = createDetector(detector2); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); hit = hits.get(0); @@ -1137,22 +1149,22 @@ public void testDeletingADetector_single_Monitor() throws IOException { Assert.assertFalse(doesIndexExist(String.format(Locale.ROOT, ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); } @@ -1166,10 +1178,10 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -1179,12 +1191,12 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc String detectorId1 = createDetector(detector1); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -1206,12 +1218,12 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc String detectorId2 = createDetector(detector2); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); hit = hits.get(0); @@ -1247,26 +1259,27 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc Assert.assertFalse(doesIndexExist(String.format(Locale.ROOT, ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); } + @SuppressWarnings("unchecked") public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() throws IOException { String index1 = "test_index_1"; createIndex(index1, Settings.EMPTY); @@ -1284,6 +1297,10 @@ public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() thro List.of(index1) ); String detectorId1 = createDetector(detector1); + Response response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; // Create detector #2 of type test_windows Detector detector2 = randomDetectorWithTriggers( @@ -1293,29 +1310,19 @@ public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() thro ); String detectorId2 = createDetector(detector2); - - Assert.assertTrue(doesIndexExist(".opensearch-sap-test_windows-detectors-queries-000001")); - Assert.assertTrue(doesIndexExist(".opensearch-sap-test_windows-detectors-queries-000002")); - - // Check if both query indices have proper settings applied from index template - Map settings = getIndexSettingsAsMap(".opensearch-sap-test_windows-detectors-queries-000001"); - assertTrue(settings.containsKey("index.analysis.char_filter.rule_ws_filter.pattern")); - assertTrue(settings.containsKey("index.hidden")); - settings = getIndexSettingsAsMap(".opensearch-sap-test_windows-detectors-queries-000002"); - assertTrue(settings.containsKey("index.analysis.char_filter.rule_ws_filter.pattern")); - assertTrue(settings.containsKey("index.hidden")); + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId2 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex2 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), null); Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); - // We deleted 1 detector, but 1 detector with same type exists, so we expect queryIndex to be present - Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); - Assert.assertTrue(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000002", "test_windows"))); deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId2, Collections.emptyMap(), null); Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); // We deleted all detectors of type windows, so we expect that queryIndex is deleted - Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); - Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000002", "test_windows"))); + Assert.assertFalse(doesIndexExist(ruleTopicIndex1)); + Assert.assertFalse(doesIndexExist(ruleTopicIndex2)); String request = "{\n" + " \"query\" : {\n" + @@ -1461,9 +1468,9 @@ public void testCreatingADetectorWithTimestampFieldAliasMapping_verifyTimeRangeI Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI); updateRequest.setJsonEntity(MediaTypeRegistry.JSON.contentBuilder().map(Map.of( - "index_name", index, - "field", "time", - "alias", "timestamp")) + "index_name", index, + "field", "time", + "alias", "timestamp")) .toString()); Response apiResponse = client().performRequest(updateRequest); assertEquals(HttpStatus.SC_OK, apiResponse.getStatusLine().getStatusCode()); @@ -1722,4 +1729,157 @@ public void testDetector_withAlias_endToEnd_success() throws IOException { List findings = (List) getFindingsBody.get("findings"); Assert.assertEquals(findings.size(), 1); } + + @SuppressWarnings("unchecked") + public void testCreatingDetectorWithDynamicQueryIndexDisabledAndThenEnabledToUpdate() throws IOException { + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "false"); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId1 = responseBody.get("_id").toString(); + + detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + String detectorId2 = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId1 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; + hits = executeSearch(Detector.DETECTORS_INDEX, request); + hit = hits.get(0); + + String monitorId2 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId1, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + executeResponse = executeAlertingMonitor(monitorId2, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + Assert.assertTrue(doesIndexExist(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "-000001")); + + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "true"); + + Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); + + indexDoc(index, "2", randomDoc()); + + executeResponse = executeAlertingMonitor(monitorId1, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + executeResponse = executeAlertingMonitor(monitorId2, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; + Assert.assertTrue(doesIndexExist(ruleTopicIndex1)); + } + + @SuppressWarnings("unchecked") + public void testCreatingDetectorWithDynamicQueryIndexEnabledAndThenDisabled() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId1 = responseBody.get("_id").toString(); + + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; + Assert.assertTrue(doesIndexExist(ruleTopicIndex1)); + Assert.assertFalse(doesIndexExist(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "-000001")); + + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "false"); + + Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); + + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; + Assert.assertTrue(doesIndexExist(ruleTopicIndex1)); + + detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + Assert.assertTrue(doesIndexExist(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "-000001")); + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java index 63703a201..bc86e11a1 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java @@ -8,26 +8,130 @@ import org.junit.Assert; import org.opensearch.client.Response; import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; import org.opensearch.securityanalytics.TestHelpers; -import org.opensearch.securityanalytics.action.ListIOCsActionResponse; +import org.opensearch.securityanalytics.model.DetailedSTIX2IOCDto; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.IocWithFeeds; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionResponse; import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.model.STIX2IOC; import org.opensearch.securityanalytics.model.STIX2IOCDto; import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.securityanalytics.util.STIX2IOCGenerator; import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; public class ListIOCsRestApiIT extends SecurityAnalyticsRestTestCase { + public void testListIOCsWithNoFindingsIndex() throws IOException { + // Delete findings system indexes if they exist + try { + makeRequest(client(), "DELETE", IocFindingService.IOC_FINDING_INDEX_PATTERN_REGEXP, Collections.emptyMap(), null); + } catch (IndexNotFoundException indexNotFoundException) { + logger.info("No threat intel findings indexes to delete."); + } catch (Exception e) { + logger.error(e.getMessage()); + } + + // Create IOCs + String searchString = "test-list-iocs-no-findings-index"; + Map iocs = new HashMap<>(); + for (int i = 0; i < 100; i++) { + String iocId = searchString + "-" + i; + iocs.put( + iocId, + new STIX2IOCDto( + iocId, + iocId + "-name", + new IOCType(IOCType.IPV4_TYPE), + "ipv4value" + i, + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L + ) + ); + } + + // Creating source config + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + "test_list_ioc_" + searchString, + "STIX", + SourceConfigType.IOC_UPLOAD, + null, + null, + null, + new IocUploadSource(null, new ArrayList<>(iocs.values())), + null, + null, + null, + null, + null, + null, + null, + false, + List.of(IOCType.IPV4_TYPE), + true + ); + + // Create the IOC system indexes using IOC_UPLOAD config + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + + // Call ListIOCs API + Map params = Map.of( + "searchString", searchString, + "size", "10000" + ); + Response iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), params, null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + Map respMap = asMap(iocResponse); + + // Evaluate response + int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(iocs.size(), totalHits); + + List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(iocs.size(), iocHits.size()); + + iocHits.forEach((hit) -> { + String iocId = (String) hit.get(STIX2IOC.ID_FIELD); + String iocName = (String) hit.get(STIX2IOC.NAME_FIELD); + String iocValue = (String) hit.get(STIX2IOC.VALUE_FIELD); + + STIX2IOCDto iocDto = iocs.get(iocId); + assertNotNull(iocDto); + + assertEquals(iocDto.getId(), iocId); + assertEquals(iocDto.getName(), iocName); + assertEquals(iocDto.getValue(), iocValue); + + int findingsNum = (int) hit.get(DetailedSTIX2IOCDto.NUM_FINDINGS_FIELD); + int expectedNumFindings = 0; + assertEquals(expectedNumFindings, findingsNum); + }); + } + public void testListIOCsBySearchString() throws IOException { String searchString = "test-search-string"; List iocs = List.of( @@ -126,4 +230,121 @@ public void testListIOCsBySearchString() throws IOException { } // TODO: Implement additional tests using various query param combinations + + public void testListIOCsNumFindings() throws Exception { + // Create IOCs + String searchString = "test-list-iocs-num-findings"; + List iocs = new ArrayList<>(); + Map> iocIdFindingsNum = new HashMap<>(); + for (int i = 0; i < 5; i++) { + String iocId = searchString + "-" + i; + iocs.add( + new STIX2IOCDto( + iocId, + iocId + "-name", + new IOCType(IOCType.IPV4_TYPE), + "ipv4value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L + ) + ); + + // Confirming the ListIOCs API can return a findings count greater than 10,000 by giving the first IOC 10,005 findings + int numFindings = i == 0 ? 10005 : randomInt(10); + List iocFindings = generateIOCMatches(numFindings, iocId); + + // Tracking the number of findings expected for each IOC + iocIdFindingsNum.put(iocId, iocFindings); + } + + // Creating source config + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + "test_list_ioc_" + searchString, + "STIX", + SourceConfigType.IOC_UPLOAD, + null, + null, + null, + new IocUploadSource(null, iocs), + null, + null, + null, + null, + null, + null, + null, + false, + List.of(IOCType.IPV4_TYPE), + true + ); + + // Create the IOC system indexes using IOC_UPLOAD config + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + + // Generate IOC matches + for (Map.Entry> entry : iocIdFindingsNum.entrySet()) { + ingestIOCMatches(entry.getValue()); + } + + // Call ListIOCs API + Response iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("searchString", searchString), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + Map respMap = asMap(iocResponse); + + // Evaluate response + int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(iocs.size(), totalHits); + + List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(iocs.size(), iocHits.size()); + + iocHits.forEach((hit) -> { + String iocId = (String) hit.get(STIX2IOC.ID_FIELD); + int findingsNum = (int) hit.get(DetailedSTIX2IOCDto.NUM_FINDINGS_FIELD); + int expectedNumFindings = iocIdFindingsNum.get(iocId).size(); + assertEquals(expectedNumFindings, findingsNum); + }); + } + + private List generateIOCMatches(int numMatches, String iocId) { + List iocFindings = new ArrayList<>(); + String monitorId = randomAlphaOfLength(10); + String monitorName = randomAlphaOfLength(10); + for (int i = 0; i < numMatches; i++) { + iocFindings.add(new IocFinding( + randomAlphaOfLength(10), + randomList(1, 10, () -> randomAlphaOfLength(10)),//docIds + randomList(1, 10, () -> new IocWithFeeds( + iocId, + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10)) + ), //feedIds + monitorId, + monitorName, + randomAlphaOfLength(10), + IOCType.IPV4_TYPE, + Instant.now(), + randomAlphaOfLength(10) + )); + } + return iocFindings; + } + + private void ingestIOCMatches(List iocFindings) throws IOException { + for (IocFinding iocFinding: iocFindings) { + makeRequest(client(), "POST", IocFindingService.IOC_FINDING_ALIAS_NAME + "/_doc?refresh", Map.of(), + toHttpEntity(iocFinding)); + } + } } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java index 6d7f4b550..9c258e0e8 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java @@ -110,14 +110,14 @@ public void testCreatingARule_custom_category() throws IOException { } catch (ResponseException e) { assertEquals(HttpStatus.SC_BAD_REQUEST, e.getResponse().getStatusLine().getStatusCode()); Assert.assertTrue( - e.getMessage().contains("Invalid rule category") + e.getMessage().contains("Invalid rule category") ); } } public void testCreatingAggregationRule() throws SigmaError, IOException { Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "windows"), - new StringEntity(countAggregationTestRule()), new BasicHeader("Content-Type", "application/json")); + new StringEntity(countAggregationTestRule()), new BasicHeader("Content-Type", "application/json")); Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); @@ -130,24 +130,24 @@ public void testCreatingAggregationRule() throws SigmaError, IOException { String index = Rule.CUSTOM_RULES_INDEX; String request = "{\n" + - " \"query\": {\n" + - " \"nested\": {\n" + - " \"path\": \"rule\",\n" + - " \"query\": {\n" + - " \"bool\": {\n" + - " \"must\": [\n" + - " { \"match\": {\"rule.category\": \"windows\"}}\n" + - " ]\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - "}"; + " \"query\": {\n" + + " \"nested\": {\n" + + " \"path\": \"rule\",\n" + + " \"query\": {\n" + + " \"bool\": {\n" + + " \"must\": [\n" + + " { \"match\": {\"rule.category\": \"windows\"}}\n" + + " ]\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(index, request); XContentParser xcp = XContentType.JSON.xContent() - .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, hits.get(0).getSourceAsString()); + .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, hits.get(0).getSourceAsString()); Rule result = Rule.docParse(xcp, null, null); Assert.assertEquals(1, result.getAggregationQueries().size()); @@ -728,7 +728,7 @@ public void testDeletingUsedRule() throws IOException { " }\n" + " }\n" + "}"; - List hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request); + List hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request); Assert.assertEquals(2, hits.size()); Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.RULE_BASE_URI + "/" + createdId, Collections.singletonMap("forced", "true"), null); @@ -741,7 +741,7 @@ public void testDeletingUsedRule() throws IOException { " }\n" + " }\n" + "}"; - hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request); + hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request); Assert.assertEquals(0, hits.size()); index = Rule.CUSTOM_RULES_INDEX; diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java index 489c60754..9b457f7df 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java @@ -15,13 +15,15 @@ import org.junit.Assert; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; +import org.opensearch.client.WarningFailureException; +import org.opensearch.common.settings.Settings; import org.opensearch.core.rest.RestStatus; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; -import org.opensearch.securityanalytics.action.ListIOCsActionRequest; -import org.opensearch.securityanalytics.action.ListIOCsActionResponse; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionRequest; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionResponse; import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.model.STIX2IOCDto; import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; @@ -42,6 +44,7 @@ import static org.opensearch.jobscheduler.spi.utils.LockService.LOCK_INDEX_NAME; import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.JOB_INDEX_NAME; +import static org.opensearch.securityanalytics.TestHelpers.oldThreatIntelJobMapping; import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.IOC_ALL_INDEX_PATTERN; import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getAllIocIndexPatternById; @@ -208,7 +211,7 @@ public void testCreateIocUploadSourceConfigIncorrectIocTypes() throws IOExceptio } } - public void testUpdateIocUploadSourceConfig() throws IOException, InterruptedException { + public void testUpdateIocUploadSourceConfig() throws IOException { // Create source config with IPV4 IOCs String feedName = "test_update"; String feedFormat = "STIX"; @@ -365,7 +368,7 @@ public void testUpdateIocUploadSourceConfig() throws IOException, InterruptedExc assertEquals(1, iocHits.size()); } - public void testActivateDeactivateIocUploadSourceConfig() throws IOException, InterruptedException { + public void testActivateDeactivateIocUploadSourceConfig() throws IOException { // Create source config with IPV4 IOCs String feedName = "test_update"; String feedFormat = "STIX"; @@ -536,7 +539,7 @@ public void testActivateDeactivateIocUploadSourceConfig() throws IOException, In assertTrue((Boolean) scr.get("enabled_for_scan")); } - public void testActivateDeactivateUrlDownloadSourceConfig() throws IOException, InterruptedException { + public void testActivateDeactivateUrlDownloadSourceConfig() throws IOException { // Search source configs when none are created String request = "{\n" + " \"query\" : {\n" + @@ -889,7 +892,7 @@ public void testSearchAndCreateDefaultSourceConfig() throws IOException { Assert.assertEquals(1, ((Map) ((Map) responseBody.get("hits")).get("total")).get("value")); } - public void testUpdateDefaultSourceConfigThrowsError() throws IOException, InterruptedException { + public void testUpdateDefaultSourceConfigThrowsError() throws IOException { // Search source configs when none are created String request = "{\n" + " \"query\" : {\n" + @@ -951,6 +954,106 @@ public void testUpdateDefaultSourceConfigThrowsError() throws IOException, Inter } } + public void testUpdateJobIndexMapping() throws IOException { + // Create job index with old threat intel mapping + // Try catch needed because of warning when creating a system index which is needed to replicate previous tif job mapping + try { + createIndex(JOB_INDEX_NAME, Settings.EMPTY, oldThreatIntelJobMapping()); + } catch (WarningFailureException e) { + // Ensure index was created with old mappings + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(0, hits.size()); + + Map props = getIndexMappingsAPIFlat(JOB_INDEX_NAME); + assertTrue(props.containsKey("enabled_time")); + assertTrue(props.containsKey("schedule.interval.start_time")); + assertFalse(props.containsKey("source_config.source.ioc_upload.file_name")); + assertFalse(props.containsKey("source_config.source.s3.object_key")); + } + + // Create new threat intel source config + SATIFSourceConfigDto saTifSourceConfigDto = getSatifSourceConfigDto(); + + Response makeResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(makeResponse)); + Map responseBody = asMap(makeResponse); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + + // Ensure source config document was indexed + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + + // Ensure index mappings were updated + Map props = getIndexMappingsAPIFlat(JOB_INDEX_NAME); + assertTrue(props.containsKey("source_config.source.ioc_upload.file_name")); + assertTrue(props.containsKey("source_config.source.s3.object_key")); + assertTrue(props.containsKey("source_config.description")); + assertTrue(props.containsKey("source_config.last_update_time")); + assertTrue(props.containsKey("source_config.refresh_type")); + } + + private static SATIFSourceConfigDto getSatifSourceConfigDto() { + String feedName = "test_ioc_upload"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.IOC_UPLOAD; + + List iocs = List.of(new STIX2IOCDto( + "id", + "name", + new IOCType(IOCType.IPV4_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); + Boolean enabled = false; + List iocTypes = List.of(IOCType.IPV4_TYPE); + return new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + } + @Override protected boolean preserveIndicesUponCompletion() { return false; diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java index 988fb48d0..77fafd157 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java @@ -3,36 +3,40 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.HttpStatus; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.junit.Assert; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; -import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.commons.alerting.model.IntervalSchedule; import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.action.Action; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; -import org.opensearch.securityanalytics.action.ListIOCsActionRequest; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionRequest; import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorTrigger; -import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.model.STIX2IOCDto; import org.opensearch.securityanalytics.threatIntel.common.RefreshType; import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.iocscan.dao.ThreatIntelAlertService; import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeScanInputDto; -import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; -import org.opensearch.securityanalytics.threatIntel.model.S3Source; +import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelTriggerDto; import java.io.IOException; +import java.nio.charset.Charset; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -42,134 +46,153 @@ import java.util.Map; import static java.util.Collections.emptyList; +import static org.opensearch.securityanalytics.TestHelpers.randomAction; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; import static org.opensearch.securityanalytics.TestHelpers.randomIndex; +import static org.opensearch.securityanalytics.TestHelpers.randomThreatInteMonitorAction; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; import static org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction.SEARCH_THREAT_INTEL_MONITOR_PATH; public class ThreatIntelMonitorRestApiIT extends SecurityAnalyticsRestTestCase { - private final String iocIndexMappings = "\"properties\": {\n" + - " \"name\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"type\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"value\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"severity\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"spec_version\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"created\": {\n" + - " \"type\": \"date\"\n" + - " },\n" + - " \"modified\": {\n" + - " \"type\": \"date\"\n" + - " },\n" + - " \"description\": {\n" + - " \"type\": \"text\"\n" + - " },\n" + - " \"labels\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"feed_id\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"feed_name\": {\n" + - " \"type\": \"keyword\"\n" + - " }\n" + - " }"; - - private List testIocs = new ArrayList<>(); - - public void indexSourceConfigsAndIocs(int num, List iocVals) throws IOException { - for (int i = 0; i < num; i++) { - String configId = "id" + i; - String iocActiveIndex = ".opensearch-sap-ioc-" + configId + Instant.now().toEpochMilli(); - String indexPattern = ".opensearch-sap-ioc-" + configId; - indexTifSourceConfig(num, configId, indexPattern, iocActiveIndex, i); - - // Create the index before ingesting docs to ensure the mappings are correct - createIndex(iocActiveIndex, Settings.EMPTY, iocIndexMappings); - - // Refresh testIocs list between tests - testIocs = new ArrayList<>(); - for (int i1 = 0; i1 < iocVals.size(); i1++) { - indexIocs(iocVals, iocActiveIndex, i1, configId); - } + private final Logger log = LogManager.getLogger(ThreatIntelMonitorRestApiIT.class); + + private List testIocDtos = new ArrayList<>(); + + public String indexSourceConfigsAndIocs(List iocVals) throws IOException { + testIocDtos = new ArrayList<>(); + for (int i1 = 0; i1 < iocVals.size(); i1++) { + // create IOCs + STIX2IOCDto stix2IOCDto = new STIX2IOCDto( + "id" + i1, + "random", + new IOCType(IOCType.IPV4_TYPE), + iocVals.get(i1), + "", + Instant.now(), + Instant.now(), + "", + emptyList(), + "spec", + "configId", + "", + 1L + ); + + testIocDtos.add(stix2IOCDto); } + return indexTifSourceConfig(testIocDtos); } - private void indexIocs(List iocVals, String iocIndexName, int i1, String configId) throws IOException { - String iocId = iocIndexName + i1; - STIX2IOC stix2IOC = new STIX2IOC( - iocId, - "random", - new IOCType(IOCType.IPV4_TYPE), - iocVals.get(i1), - "", - Instant.now(), - Instant.now(), - "", - emptyList(), - "spec", - configId, - "", - STIX2IOC.NO_VERSION - ); - - // Add IOC to testIocs List for future validation - testIocs.add(stix2IOC); - - indexDoc(iocIndexName, iocId, stix2IOC.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).toString()); - List searchHits = executeSearch(iocIndexName, getMatchAllSearchRequestString(iocVals.size())); - assertEquals(searchHits.size(), i1 + 1); + public String indexSourceConfigsAndIocs(List ipVals, List hashVals, List domainVals) throws IOException { + testIocDtos = new ArrayList<>(); + for (int i1 = 0; i1 < ipVals.size(); i1++) { + // create IOCs + STIX2IOCDto stix2IOCDto = new STIX2IOCDto( + "id" + randomAlphaOfLength(3), + "random", + new IOCType(IOCType.IPV4_TYPE), + ipVals.get(i1), + "", + Instant.now(), + Instant.now(), + "", + emptyList(), + "spec", + "configId", + "", + 1L + ); + + testIocDtos.add(stix2IOCDto); + } + for (int i1 = 0; i1 < hashVals.size(); i1++) { + // create IOCs + STIX2IOCDto stix2IOCDto = new STIX2IOCDto( + "id" + randomAlphaOfLength(3), + "random", + new IOCType(IOCType.HASHES_TYPE), + hashVals.get(i1), + "", + Instant.now(), + Instant.now(), + "", + emptyList(), + "spec", + "configId", + "", + 1L + ); + + testIocDtos.add(stix2IOCDto); + } + for (int i1 = 0; i1 < domainVals.size(); i1++) { + // create IOCs + STIX2IOCDto stix2IOCDto = new STIX2IOCDto( + "id" + randomAlphaOfLength(3), + "random", + new IOCType(IOCType.DOMAIN_NAME_TYPE), + domainVals.get(i1), + "", + Instant.now(), + Instant.now(), + "", + emptyList(), + "spec", + "configId", + "", + 1L + ); + + testIocDtos.add(stix2IOCDto); + } + return indexTifSourceConfig(testIocDtos); } - private void indexTifSourceConfig(int num, String configId, String indexPattern, String iocActiveIndex, int i) throws IOException { - SATIFSourceConfig config = new SATIFSourceConfig( - configId, + private String indexTifSourceConfig(List testIocDtos) throws IOException { + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + "configId", SATIFSourceConfig.NO_VERSION, "name1", "STIX2", - SourceConfigType.S3_CUSTOM, + SourceConfigType.IOC_UPLOAD, "description", null, Instant.now(), - new S3Source("bucketname", "key", "region", "roleArn"), + new IocUploadSource(null, testIocDtos), null, Instant.now(), - new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES), + null, TIFJobState.AVAILABLE, RefreshType.FULL, null, null, false, - new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(IOCType.IPV4_TYPE), indexPattern, iocActiveIndex))), - List.of(IOCType.IPV4_TYPE), + List.of(IOCType.IPV4_TYPE, IOCType.HASHES_TYPE, IOCType.DOMAIN_NAME_TYPE), true ); - String indexName = SecurityAnalyticsPlugin.JOB_INDEX_NAME; - Response response = indexDoc(indexName, configId, config.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).toString()); + + Response makeResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(makeResponse)); + + Assert.assertEquals(201, makeResponse.getStatusLine().getStatusCode()); + Map responseBody = asMap(makeResponse); + return responseBody.get("_id").toString(); } public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", Map.of(), null); Map responseAsMap = responseAsMap(iocFindingsResponse); Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); List vals = List.of("ip1", "ip2"); - indexSourceConfigsAndIocs(1, vals); + String createdId = indexSourceConfigsAndIocs(vals); + String index = "alias1"; Map> testAlias = createTestAlias(index, 1, true); String monitorName = "test_monitor_name"; - /**create monitor */ ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto(index); Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); @@ -186,6 +209,8 @@ public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { final String monitorId = responseBody.get("id").toString(); Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Assert.assertEquals(200, executeResponse.getStatusLine().getStatusCode()); Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); @@ -199,9 +224,8 @@ public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { } } - Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); - assertEquals(1, 1); String matchAllRequest = getMatchAllRequest(); Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); @@ -212,7 +236,6 @@ public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { assertEquals(totalHitsVal.intValue(), 1); makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); - iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", Map.of(), null); responseAsMap = responseAsMap(iocFindingsResponse); @@ -237,7 +260,7 @@ public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); // Use ListIOCs API to confirm expected number of findings are returned - String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, "id0"); + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, createdId); Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); Map listIocsResponseMap = responseAsMap(listIocsResponse); List> iocsMap = (List>) listIocsResponseMap.get("iocs"); @@ -245,7 +268,7 @@ public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { iocsMap.forEach((iocDetails) -> { String iocId = (String) iocDetails.get("id"); int numFindings = (Integer) iocDetails.get("num_findings"); - assertTrue(testIocs.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertTrue(testIocDtos.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); assertEquals(2, numFindings); }); @@ -296,13 +319,102 @@ public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { assertEquals(totalHitsVal.intValue(), 0); } + public void testCreateThreatIntelMonitor_configureMultipleIndicatorTypesInMonitor() throws IOException { + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List ipVals = List.of("ip1", "ip2"); + List hashVals = List.of("h1", "h2"); + List domainVals = List.of("d1", "d2"); + String createdId = indexSourceConfigsAndIocs(ipVals, hashVals, domainVals); + + String ipIndex = "ipAlias"; + createTestAlias(ipIndex, 1, true); + String hashIndex = "hashAlias"; + createTestAlias(hashIndex, 1, true); + String domainIndex = "domainAlias"; + createTestAlias(domainIndex, 1, true); + + + /**create monitor */ + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDtoWithMultipleIndicatorTypesToScan(ipIndex, hashIndex, domainIndex); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + fail(); + } catch (Exception e) { + /** creating a second threat intel monitor should fail*/ + assertTrue(e.getMessage().contains("already exists")); + } + + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Assert.assertEquals(200, executeResponse.getStatusLine().getStatusCode()); + + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : ipVals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(ipIndex, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + for (String val : hashVals) { + String doc = String.format("{\"hash\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(hashIndex, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + for (String val : domainVals) { + String doc = String.format("{\"domain\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(domainIndex, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + + executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + String matchAllRequest = getMatchAllRequest(); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + HashMap totalHits = (HashMap) hits.get("total"); + Integer totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 1); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(6, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(6, searchHits.size()); + } + public void testCreateThreatIntelMonitor() throws IOException { + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", Map.of(), null); Map responseAsMap = responseAsMap(iocFindingsResponse); Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); List vals = List.of("ip1", "ip2"); - indexSourceConfigsAndIocs(1, vals); + String createdId = indexSourceConfigsAndIocs(vals); String index = createTestIndex(randomIndex(), windowsIndexMapping()); String monitorName = "test_monitor_name"; @@ -331,6 +443,8 @@ public void testCreateThreatIntelMonitor() throws IOException { String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); try { indexDoc(index, "" + i++, doc); + indexDoc(index, "" + i++, String.format("{\"ip\":\"1.2.3.4\", \"ip1\":\"1.2.3.4\"}", val, val)); + indexDoc(index, "" + i++, String.format("{\"random\":\"%s\", \"random1\":\"%s\"}", val, val)); } catch (IOException e) { fail(); } @@ -338,7 +452,6 @@ public void testCreateThreatIntelMonitor() throws IOException { Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); - assertEquals(1, 1); String matchAllRequest = getMatchAllRequest(); Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); @@ -374,7 +487,7 @@ public void testCreateThreatIntelMonitor() throws IOException { Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); // Use ListIOCs API to confirm expected number of findings are returned - String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, "id0"); + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, createdId); Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); Map listIocsResponseMap = responseAsMap(listIocsResponse); List> iocsMap = (List>) listIocsResponseMap.get("iocs"); @@ -382,7 +495,7 @@ public void testCreateThreatIntelMonitor() throws IOException { iocsMap.forEach((iocDetails) -> { String iocId = (String) iocDetails.get("id"); int numFindings = (Integer) iocDetails.get("num_findings"); - assertTrue(testIocs.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertTrue(testIocDtos.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); assertEquals(2, numFindings); }); @@ -425,7 +538,280 @@ public void testCreateThreatIntelMonitor() throws IOException { assertEquals(totalHitsVal.intValue(), 0); } - public void testCreateThreatIntelMonitorWithExistingDetector() throws IOException { + // verify scenario where findings are generated but alert creation fails - monitor execution should be marked successful + public void testCreateThreatIntelMonitor_testExecution_findingSucceedsButAlertFails_ExecutionSucceeds() throws IOException { + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List vals = List.of("ip1", "ip2"); + String createdId = indexSourceConfigsAndIocs(vals); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + String monitorName = "test_monitor_name"; + + + /**create monitor */ + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto(index); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + fail(); + } catch (Exception e) { + /** creating a second threat intel monitor should fail*/ + assertTrue(e.getMessage().contains("already exists")); + } + + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + indexDoc(index, "" + i++, String.format("{\"ip\":\"1.2.3.4\", \"ip1\":\"1.2.3.4\"}", val, val)); + indexDoc(index, "" + i++, String.format("{\"random\":\"%s\", \"random1\":\"%s\"}", val, val)); + } catch (IOException e) { + fail(); + } + } + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + String matchAllRequest = getMatchAllRequest(); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + HashMap totalHits = (HashMap) hits.get("total"); + Integer totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 1); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(2, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + closeIndex(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); + openIndex(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME); + // Use ListIOCs API to confirm expected number of findings are returned + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, createdId); + Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); + Map listIocsResponseMap = responseAsMap(listIocsResponse); + List> iocsMap = (List>) listIocsResponseMap.get("iocs"); + assertEquals(2, iocsMap.size()); + iocsMap.forEach((iocDetails) -> { + String iocId = (String) iocDetails.get("id"); + int numFindings = (Integer) iocDetails.get("num_findings"); + assertTrue(testIocDtos.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertEquals(2, numFindings); + }); + + //alerts via system index search + searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + // alerts via API + Map params = new HashMap<>(); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + Assert.assertEquals(4, getAlertsBody.get("total_alerts")); + + + ThreatIntelMonitorDto updateMonitorDto = new ThreatIntelMonitorDto( + monitorId, + iocScanMonitor.getName() + "update", + iocScanMonitor.getPerIocTypeScanInputList(), + new IntervalSchedule(5, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(iocScanMonitor.getTriggers().get(0), iocScanMonitor.getTriggers().get(1)) + ); + //update monitor + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), toHttpEntity(updateMonitorDto)); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + responseBody = asMap(response); + assertEquals(responseBody.get("id").toString(), monitorId); + assertEquals(((HashMap) responseBody.get("monitor")).get("name").toString(), iocScanMonitor.getName() + "update"); + + //delete + Response delete = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), null); + Assert.assertEquals(200, delete.getStatusLine().getStatusCode()); + + searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + totalHits = (HashMap) hits.get("total"); + totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 0); + } + + // verify scenario where alerts and findings are generated but notification fails - monitor execution should be marked successful + public void testCreateThreatIntelMonitor_testNotifications_invaliConfigExecutionSucceeds() throws IOException { + + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List vals = List.of("ip1", "ip2"); + String createdId = indexSourceConfigsAndIocs(vals); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + String monitorName = "test_monitor_name"; + + Action triggerAction1 = randomThreatInteMonitorAction(randomAlphaOfLength(10)); + Action triggerAction2 = randomThreatInteMonitorAction(randomAlphaOfLength(10)); + /**create monitor with trigger*/ + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDtoWithTriggers(index, List.of(triggerAction1, triggerAction2)); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + fail(); + } catch (Exception e) { + /** creating a second threat intel monitor should fail*/ + assertTrue(e.getMessage().contains("already exists")); + } + + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + indexDoc(index, "" + i++, String.format("{\"ip\":\"1.2.3.4\", \"ip1\":\"1.2.3.4\"}", val, val)); + indexDoc(index, "" + i++, String.format("{\"random\":\"%s\", \"random1\":\"%s\"}", val, val)); + } catch (IOException e) { + fail(); + } + } + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + String matchAllRequest = getMatchAllRequest(); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + HashMap totalHits = (HashMap) hits.get("total"); + Integer totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 1); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(2, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(2, searchHits.size()); + + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + executeAlertingMonitor(monitorId, Collections.emptyMap()); + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); + + // Use ListIOCs API to confirm expected number of findings are returned + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, createdId); + Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); + Map listIocsResponseMap = responseAsMap(listIocsResponse); + List> iocsMap = (List>) listIocsResponseMap.get("iocs"); + assertEquals(2, iocsMap.size()); + iocsMap.forEach((iocDetails) -> { + String iocId = (String) iocDetails.get("id"); + int numFindings = (Integer) iocDetails.get("num_findings"); + assertTrue(testIocDtos.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertEquals(2, numFindings); + }); + + //alerts via system index search + searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(2, searchHits.size()); + + // alerts via API + Map params = new HashMap<>(); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + Assert.assertEquals(2, getAlertsBody.get("total_alerts")); + + + ThreatIntelMonitorDto updateMonitorDto = new ThreatIntelMonitorDto( + monitorId, + iocScanMonitor.getName() + "update", + iocScanMonitor.getPerIocTypeScanInputList(), + new IntervalSchedule(5, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(iocScanMonitor.getTriggers().get(0)) + ); + //update monitor + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), toHttpEntity(updateMonitorDto)); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + responseBody = asMap(response); + assertEquals(responseBody.get("id").toString(), monitorId); + assertEquals(((HashMap) responseBody.get("monitor")).get("name").toString(), iocScanMonitor.getName() + "update"); + + //delete + Response delete = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), null); + Assert.assertEquals(200, delete.getStatusLine().getStatusCode()); + + searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + totalHits = (HashMap) hits.get("total"); + totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 0); + } + + public void testCreateThreatIntelMonitorWithExistingDetector() throws IOException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); // Execute CreateMappingsAction to add alias mapping for index @@ -451,7 +837,7 @@ public void testCreateThreatIntelMonitorWithExistingDetector() throws IOExceptio Map responseAsMap = responseAsMap(iocFindingsResponse); Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); List vals = List.of("ip1", "ip2"); - indexSourceConfigsAndIocs(1, vals); + String createdId = indexSourceConfigsAndIocs(vals); String monitorName = "test_monitor_name"; @@ -486,7 +872,6 @@ public void testCreateThreatIntelMonitorWithExistingDetector() throws IOExceptio Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); - assertEquals(1, 1); String matchAllRequest = getMatchAllRequest(); Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); @@ -522,7 +907,7 @@ public void testCreateThreatIntelMonitorWithExistingDetector() throws IOExceptio Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); // Use ListIOCs API to confirm expected number of findings are returned - String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, "id0"); + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, createdId); Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); Map listIocsResponseMap = responseAsMap(listIocsResponse); List> iocsMap = (List>) listIocsResponseMap.get("iocs"); @@ -530,7 +915,7 @@ public void testCreateThreatIntelMonitorWithExistingDetector() throws IOExceptio iocsMap.forEach((iocDetails) -> { String iocId = (String) iocDetails.get("id"); int numFindings = (Integer) iocDetails.get("num_findings"); - assertTrue(testIocs.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertTrue(testIocDtos.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); assertEquals(2, numFindings); }); @@ -616,6 +1001,39 @@ public static ThreatIntelMonitorDto randomIocScanMonitorDto(String index) { List.of(t1, t2, t3, t4)); } + public static ThreatIntelMonitorDto randomIocScanMonitorDtoWithTriggers(String index, List actions) { + ThreatIntelTriggerDto t1 = new ThreatIntelTriggerDto(List.of(), List.of(IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE), actions, "match", null, "severity"); + + return new ThreatIntelMonitorDto( + Monitor.NO_ID, + randomAlphaOfLength(10), + List.of(new PerIocTypeScanInputDto(IOCType.IPV4_TYPE, Map.of(index, List.of("ip")))), + new IntervalSchedule(1, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(t1)); + } + + public static ThreatIntelMonitorDto randomIocScanMonitorDtoWithMultipleIndicatorTypesToScan(String ipIndex, String hashIndex, String domainIndex) { + ThreatIntelTriggerDto t1 = new ThreatIntelTriggerDto(List.of(ipIndex, "randomIndex"), List.of(IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE), emptyList(), "match", null, "severity"); + ThreatIntelTriggerDto t2 = new ThreatIntelTriggerDto(List.of("randomIndex"), List.of(IOCType.DOMAIN_NAME_TYPE), emptyList(), "nomatch", null, "severity"); + ThreatIntelTriggerDto t3 = new ThreatIntelTriggerDto(emptyList(), List.of(IOCType.DOMAIN_NAME_TYPE), emptyList(), "domainmatchsonomatch", null, "severity"); + ThreatIntelTriggerDto t4 = new ThreatIntelTriggerDto(List.of(ipIndex), emptyList(), emptyList(), "indexmatch", null, "severity"); + + return new ThreatIntelMonitorDto( + Monitor.NO_ID, + randomAlphaOfLength(10), + List.of( + new PerIocTypeScanInputDto(IOCType.IPV4_TYPE, Map.of(ipIndex, List.of("ip"))), + new PerIocTypeScanInputDto(IOCType.HASHES_TYPE, Map.of(hashIndex, List.of("hash"))), + new PerIocTypeScanInputDto(IOCType.DOMAIN_NAME_TYPE, Map.of(domainIndex, List.of("domain"))) + ), + new IntervalSchedule(1, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(t1, t2, t3, t4)); + } + @Override protected boolean preserveIndicesUponCompletion() { return false; diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java index ec6041322..2f5aa90a3 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java @@ -60,7 +60,7 @@ public void testReleaseLock_whenValidInput_thenSucceed() { LOCK_DURATION_IN_SECONDS, false ); - noOpsLockService.releaseLockEventDriven(lockModel, ActionListener.wrap( + noOpsLockService.releaseLock(lockModel, ActionListener.wrap( Assert::assertFalse, e -> fail() )); } diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java index def2b21e5..36de85ebf 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java @@ -57,6 +57,7 @@ public void testThreatInputSerde() throws IOException { emptyList(), emptyMap(), new DataSources(), + false, "security_analytics" ); BytesStreamOutput monitorOut = new BytesStreamOutput(); diff --git a/src/test/resources/ad_ldap-sample.json b/src/test/resources/ad_ldap-sample.json index 4e9b2caf3..f6beef93c 100644 --- a/src/test/resources/ad_ldap-sample.json +++ b/src/test/resources/ad_ldap-sample.json @@ -8,7 +8,7 @@ "SearchFilter": "1234", "azure.platformlogs.result_type": "1234", "azure.signinlogs.result_description": "1234", - "azure.signinlogs.properties.device_detail.is_compliant": "1234", + "azure.signinlogs.properties.device_detail.is_compliant": false, "resource_display_name": "1234", "azure.signinlogs.properties.authentication_requirement": "1234", "target_resources": "1234", diff --git a/src/test/resources/cloudtrail-sample.json b/src/test/resources/cloudtrail-sample.json index 0cf13e37a..e3c812f35 100644 --- a/src/test/resources/cloudtrail-sample.json +++ b/src/test/resources/cloudtrail-sample.json @@ -6,10 +6,10 @@ "source.as.organization.name": "213123", "source.ip": "213123", "userIdentity.arn": "213123", - "eventName": "213123", + "eventName": "DeleteIdentity", "eventType": "213123", "errorCode": "213123", - "eventSource": "213123", + "eventSource": "ses.amazonaws.com", "tlsDetails.tlsVersion": "213123", "user_agent.name": "213123", "threat.matched.providers": "213123", diff --git a/src/test/resources/waf-sample.json b/src/test/resources/waf-sample.json index cf925a70e..0cb24f5f0 100644 --- a/src/test/resources/waf-sample.json +++ b/src/test/resources/waf-sample.json @@ -54,5 +54,5 @@ "name": "awswaf:managed:aws:bot-control:signal:known_bot_data_center" } ], - "waf.request.headers.user_agent": "111" + "waf.request.headers.user_agent": "WPScan v" }