diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb97c6862..626ed4c60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: needs: Get-CI-Image-Tag strategy: matrix: - java: [11, 17] + java: [21] os: [ ubuntu-latest ] name: Build and Test security-analytics with JDK ${{ matrix.java }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -55,14 +55,14 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} - name: Upload failed logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: logs-ubuntu path: build/testclusters/integTest-*/logs/* - name: Upload Artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: security-analytics-plugin-${{ matrix.os }} path: security-analytics-artifacts @@ -73,7 +73,7 @@ jobs: WORKING_DIR: ${{ matrix.working_directory }}. strategy: matrix: - java: [11, 17] + java: [21] os: [ windows-latest, macos-latest ] include: - os: windows-latest @@ -113,21 +113,21 @@ jobs: cp ./build/distributions/*.zip security-analytics-artifacts - name: Upload failed logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: ${{ failure() && matrix.os == 'macos-latest' }} with: name: logs-mac path: build/testclusters/integTest-*/logs/* - name: Upload failed logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: ${{ failure() && matrix.os == 'windows-latest' }} with: name: logs-windows path: build\testclusters\integTest-*\logs\* - name: Upload Artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: security-analytics-plugin-${{ matrix.os }} path: security-analytics-artifacts diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml index 36cb4d216..c0760c0b4 100644 --- a/.github/workflows/multi-node-test-workflow.yml +++ b/.github/workflows/multi-node-test-workflow.yml @@ -19,7 +19,7 @@ jobs: needs: Get-CI-Image-Tag strategy: matrix: - java: [ 11, 17, 21 ] + java: [ 21 ] # Job name name: Build and test Security Analytics on linux # This job runs on Linux @@ -45,7 +45,7 @@ jobs: chown -R 1000:1000 `pwd` su `id -un 1000` -c "./gradlew integTest -PnumNodes=3" - name: Upload failed logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: logs diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml index ffd91de13..f7b6fd09f 100644 --- a/.github/workflows/security-test-workflow.yml +++ b/.github/workflows/security-test-workflow.yml @@ -15,7 +15,7 @@ jobs: build: strategy: matrix: - java: [ 11, 17, 21 ] + java: [ 21 ] # Job name name: Build and test SecurityAnalytics # This job runs on Linux diff --git a/build.gradle b/build.gradle index 9748df033..88ec6f985 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ import org.opensearch.gradle.test.RestIntegTestTask buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "2.15.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.18.1-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') @@ -18,6 +18,9 @@ buildscript { if (buildVersionQualifier) { opensearch_build += "-${buildVersionQualifier}" } + + alerting_spi_build = opensearch_build + alerting_spi_build += "-SNAPSHOT" if (isSnapshot) { opensearch_build += "-SNAPSHOT" @@ -171,7 +174,7 @@ dependencies { compileOnly "org.opensearch.client:opensearch-rest-client:${opensearch_version}" compileOnly "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}" - compileOnly "org.opensearch.alerting:alerting-spi:${opensearch_build}" + compileOnly "org.opensearch.alerting:alerting-spi:${alerting_spi_build}" implementation "org.apache.commons:commons-csv:1.10.0" compileOnly "com.google.guava:guava:32.1.3-jre" diff --git a/release-notes/opensearch-security-analytics.release-notes-2.16.0.0.md b/release-notes/opensearch-security-analytics.release-notes-2.16.0.0.md new file mode 100644 index 000000000..d67712959 --- /dev/null +++ b/release-notes/opensearch-security-analytics.release-notes-2.16.0.0.md @@ -0,0 +1,35 @@ +## Version 2.16.0.0 2024-07-23 + +Compatible with OpenSearch 2.16.0 + +### Features +* Threat Intel Analytics ([#1098](https://github.com/opensearch-project/security-analytics/pull/1098)) + +### Maintenance +* Incremented version to 2.16.0. ([#1197](https://github.com/opensearch-project/security-analytics/pull/1197)) +* Fix build CI error due to action runner env upgrade node 20 ([#1143](https://github.com/opensearch-project/security-analytics/pull/1143)) + +### Enhancement +* added correlationAlert integ tests ([#1099](https://github.com/opensearch-project/security-analytics/pull/1099)) +* add filter to list ioc api to fetch only from available and refreshing apis. null check for alias of ioc indices ([#1131](https://github.com/opensearch-project/security-analytics/pull/1131)) +* Changes threat intel default store config model ([#1133](https://github.com/opensearch-project/security-analytics/pull/1133)) +* adds new tif source config type - url download ([#1142](https://github.com/opensearch-project/security-analytics/pull/1142)) + +### Bug Fixes +* pass integ tests ([#1082](https://github.com/opensearch-project/security-analytics/pull/1082)) +* set blank response when indexNotFound exception ([#1125](https://github.com/opensearch-project/security-analytics/pull/1125)) +* throw error when no iocs are stored due to incompatible ioc types from S3 downloaded iocs file ([#1129](https://github.com/opensearch-project/security-analytics/pull/1129)) +* fix findingIds filter on ioc findings search api ([#1130](https://github.com/opensearch-project/security-analytics/pull/1130)) +* Adjusted IOCTypes usage ([#1156](https://github.com/opensearch-project/security-analytics/pull/1156)) +* Fix the job scheduler parser, action listeners, and multi-node test ([#1157](https://github.com/opensearch-project/security-analytics/pull/1157)) +* ListIOCs API to return number of findings per IOC ([#1163](https://github.com/opensearch-project/security-analytics/pull/1163)) +* Ioc upload integ tests and fix update ([#1162](https://github.com/opensearch-project/security-analytics/pull/1162)) +* [BUG] Resolve aliases in monitor input to concrete indices before computing ioc-containing fields from concrete index docs ([#1173](https://github.com/opensearch-project/security-analytics/pull/1173)) +* Enum fix ([#1178](https://github.com/opensearch-project/security-analytics/pull/1178)) +* fix bug: threat intel monitor finding doesnt contain all doc_ids containing malicious IOC ([#1184](https://github.com/opensearch-project/security-analytics/pull/1184)) +* Fixed bulk indexing for IOCs ([#1187](https://github.com/opensearch-project/security-analytics/pull/1187)) +* Fix ioc upload update behavior and change error response ([#1192](https://github.com/opensearch-project/security-analytics/pull/1192)) +* Catch and wrap exceptions. ([#1198](https://github.com/opensearch-project/security-analytics/pull/1198)) + +### Documentation +* Added 2.16.0 release notes. ([#1196](https://github.com/opensearch-project/security-analytics/pull/1196)) \ No newline at end of file diff --git a/release-notes/opensearch-security-analytics.release-notes-2.17.0.0.md b/release-notes/opensearch-security-analytics.release-notes-2.17.0.0.md new file mode 100644 index 000000000..287ccda70 --- /dev/null +++ b/release-notes/opensearch-security-analytics.release-notes-2.17.0.0.md @@ -0,0 +1,27 @@ +## Version 2.17.0.0 2024-09-05 + +Compatible with OpenSearch 2.17.0 + +### Maintenance +* update build.gradle to use alerting-spi snapshot version ([#1217](https://github.com/opensearch-project/security-analytics/pull/1217)) + +### Enhancement +* added triggers in getDetectors API response ([#1226](https://github.com/opensearch-project/security-analytics/pull/1226)) +* secure rest tests for threat intel monitor apis ([#1212](https://github.com/opensearch-project/security-analytics/pull/1212)) + +### Bug Fixes +* Adds user validation for threat intel transport layer classes and stashes the thread context for all system index interactions ([#1207](https://github.com/opensearch-project/security-analytics/pull/1207)) +* fix mappings integ tests ([#1213](https://github.com/opensearch-project/security-analytics/pull/1213)) +* Bug fixes for threat intel ([#1223](https://github.com/opensearch-project/security-analytics/pull/1223)) +* make threat intel run with standard detectors ([#1234](https://github.com/opensearch-project/security-analytics/pull/1234)) +* Fixed searchString bug. Removed nested IOC mapping structure. ([#1239](https://github.com/opensearch-project/security-analytics/pull/1239)) +* adds toggling refresh disable/enable for deactivate/activate operation while updating URL_DOWNLOAD type configs ([#1240](https://github.com/opensearch-project/security-analytics/pull/1240)) +* Make threat intel source config release lock event driven ([#1254](https://github.com/opensearch-project/security-analytics/pull/1254)) +* Fix S3 validation errors not caught by action listener ([#1257](https://github.com/opensearch-project/security-analytics/pull/1257)) +* Clean up empty IOC indices created by failed source configs ([#1267](https://github.com/opensearch-project/security-analytics/pull/1267)) +* Fix threat intel multinode tests ([#1274](https://github.com/opensearch-project/security-analytics/pull/1274)) +* Update threat intel job mapping to new version ([#1272](https://github.com/opensearch-project/security-analytics/pull/1272)) +* Stash context for List IOCs Api ([#1278](https://github.com/opensearch-project/security-analytics/pull/1278)) + +### Documentation +* Added 2.17.0 release notes. ([#1290](https://github.com/opensearch-project/security-analytics/pull/1290)) \ No newline at end of file diff --git a/release-notes/opensearch-security-analytics.release-notes-2.17.1.0.md b/release-notes/opensearch-security-analytics.release-notes-2.17.1.0.md new file mode 100644 index 000000000..820c79255 --- /dev/null +++ b/release-notes/opensearch-security-analytics.release-notes-2.17.1.0.md @@ -0,0 +1,15 @@ +## Version 2.17.1.0 2024-09-27 + +Compatible with OpenSearch 2.17.1 + +### Maintenance +* upgrade upload artifacts ([#1305](https://github.com/opensearch-project/security-analytics/pull/1305)) +* Incremented version to 2.17.1 ([#1304](https://github.com/opensearch-project/security-analytics/pull/1304)) + +### Bug Fixes +* [Alerts in Correlations] Stash context for system index ([#1297](https://github.com/opensearch-project/security-analytics/pull/1297)) +* threat intel monitor bug fixes ([#1317](https://github.com/opensearch-project/security-analytics/pull/1317)) + + +### Documentation +* Added 2.17.1 release notes. ([#1331](https://github.com/opensearch-project/security-analytics/pull/1331)) diff --git a/release-notes/opensearch-security-analytics.release-notes-2.18.0.0.md b/release-notes/opensearch-security-analytics.release-notes-2.18.0.0.md new file mode 100644 index 000000000..a389a2128 --- /dev/null +++ b/release-notes/opensearch-security-analytics.release-notes-2.18.0.0.md @@ -0,0 +1,24 @@ +## Version 2.18.0.0 2024-10-28 + +Compatible with OpenSearch 2.18.0 + +### Maintenance +* Incremented version to 2.18.0 ([#1314](https://github.com/opensearch-project/security-analytics/pull/1314)) +* update to lucene 9.12 ([#1349](https://github.com/opensearch-project/security-analytics/pull/1349)) + +### Refactoring +* separate doc-level monitor query indices created by detectors ([#1324](https://github.com/opensearch-project/security-analytics/pull/1324)) +* update number of replicas of system indices to 1-20 and number of primary shards for system indices to 1 ([#1358](https://github.com/opensearch-project/security-analytics/pull/1358)) +* update min number of replicas to 0 ([#1364](https://github.com/opensearch-project/security-analytics/pull/1364)) +* updated dedicated query index settings to true ([#1365](https://github.com/opensearch-project/security-analytics/pull/1365)) +* set the refresh policy to IMMEDIATE when updating correlation alerts ([#1382](https://github.com/opensearch-project/security-analytics/pull/1382)) + +### Bug Fixes +* remove redundant logic to fix OS launch exception and updates actions/upload-artifac2 to @V3 ([#1303](https://github.com/opensearch-project/security-analytics/pull/1303)) +* Add null check while adding fetched iocs into per-indicator-type map ([#1335](https://github.com/opensearch-project/security-analytics/pull/1335)) +* Fix notifications listener leak in threat intel monitor ([#1361](https://github.com/opensearch-project/security-analytics/pull/1361)) +* [Bug] Fixed ListIOCs number of findings cap. ([#1373](https://github.com/opensearch-project/security-analytics/pull/1373)) +* [Bug] Add exists check for IOCs index. ([#1392](https://github.com/opensearch-project/security-analytics/pull/1392)) + +### Documentation +* Added 2.18.0 release notes. ([#1399](https://github.com/opensearch-project/security-analytics/pull/1399)) \ No newline at end of file diff --git a/security-analytics-commons-1.0.0.jar b/security-analytics-commons-1.0.0.jar index 9ae59c9c2..8aa98470d 100644 Binary files a/security-analytics-commons-1.0.0.jar and b/security-analytics-commons-1.0.0.jar differ diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 95ca88315..2762bd5f8 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -70,7 +70,7 @@ import org.opensearch.securityanalytics.action.IndexDetectorAction; import org.opensearch.securityanalytics.action.IndexRuleAction; import org.opensearch.securityanalytics.action.ListCorrelationsAction; -import org.opensearch.securityanalytics.action.ListIOCsAction; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsAction; import org.opensearch.securityanalytics.action.SearchCorrelationRuleAction; import org.opensearch.securityanalytics.action.SearchCustomLogTypeAction; import org.opensearch.securityanalytics.action.SearchDetectorAction; @@ -113,7 +113,7 @@ import org.opensearch.securityanalytics.resthandler.RestIndexDetectorAction; import org.opensearch.securityanalytics.resthandler.RestIndexRuleAction; import org.opensearch.securityanalytics.resthandler.RestListCorrelationAction; -import org.opensearch.securityanalytics.resthandler.RestListIOCsAction; +import org.opensearch.securityanalytics.threatIntel.resthandler.RestListIOCsAction; import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationAction; import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationRuleAction; import org.opensearch.securityanalytics.resthandler.RestSearchCustomLogTypeAction; @@ -197,7 +197,7 @@ import org.opensearch.securityanalytics.transport.TransportIndexDetectorAction; import org.opensearch.securityanalytics.transport.TransportIndexRuleAction; import org.opensearch.securityanalytics.transport.TransportListCorrelationAction; -import org.opensearch.securityanalytics.transport.TransportListIOCsAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportListIOCsAction; import org.opensearch.securityanalytics.transport.TransportSearchCorrelationAction; import org.opensearch.securityanalytics.transport.TransportSearchCorrelationRuleAction; import org.opensearch.securityanalytics.transport.TransportSearchCustomLogTypeAction; @@ -226,6 +226,7 @@ import static org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE; import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.SOURCE_CONFIG_FIELD; import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; +import static org.opensearch.securityanalytics.util.CorrelationIndices.CORRELATION_ALERT_INDEX; public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension, RemoteMonitorRunnerExtension { @@ -284,7 +285,11 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map @Override public Collection getSystemIndexDescriptors(Settings settings) { - return Collections.singletonList(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data")); + List descriptors = List.of( + new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data"), + new SystemIndexDescriptor(CORRELATION_ALERT_INDEX, "System index used for Correlation Alerts") + ); + return descriptors; } @@ -327,7 +332,7 @@ public Collection createComponents(Client client, TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService); IocFindingService iocFindingService = new IocFindingService(client, clusterService, xContentRegistry); ThreatIntelAlertService threatIntelAlertService = new ThreatIntelAlertService(client, clusterService, xContentRegistry); - SaIoCScanService ioCScanService = new SaIoCScanService(client, xContentRegistry, iocFindingService, threatIntelAlertService, notificationService); + SaIoCScanService ioCScanService = new SaIoCScanService(client, clusterService, xContentRegistry, iocFindingService, threatIntelAlertService, notificationService); DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService = new DefaultTifSourceConfigLoaderService(builtInTIFMetadataLoader, client, saTifSourceConfigManagementService); return List.of( detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, threatIntelAlertService, @@ -502,7 +507,9 @@ public List> getSettings() { SecurityAnalyticsSettings.BATCH_SIZE, SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT, SecurityAnalyticsSettings.IOC_INDEX_RETENTION_PERIOD, - SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_INDEX_PATTERN + SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_INDEX_PATTERN, + SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT, + SecurityAnalyticsSettings.ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES ); } diff --git a/src/main/java/org/opensearch/securityanalytics/action/GetDetectorResponse.java b/src/main/java/org/opensearch/securityanalytics/action/GetDetectorResponse.java index 0d700b88c..955db1455 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/GetDetectorResponse.java +++ b/src/main/java/org/opensearch/securityanalytics/action/GetDetectorResponse.java @@ -69,6 +69,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws .field(Detector.LAST_UPDATE_TIME_FIELD, detector.getLastUpdateTime()) .field(Detector.ENABLED_TIME_FIELD, detector.getEnabledTime()) .field(Detector.THREAT_INTEL_ENABLED_FIELD, detector.getThreatIntelEnabled()) + .field(Detector.TRIGGERS_FIELD, detector.getTriggers()) .endObject(); return builder.endObject(); } diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java index 459f523b7..02f6595ec 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java @@ -5,6 +5,8 @@ package org.opensearch.securityanalytics.config.monitors; import java.util.List; +import java.util.Random; +import java.util.UUID; import java.util.stream.Collectors; import org.opensearch.common.inject.Inject; import org.opensearch.securityanalytics.logtype.LogTypeService; @@ -25,6 +27,10 @@ public static String getRuleIndex(String logType) { return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", logType); } + public static String getRuleIndexOptimized(String logType) { + return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-optimized-%s", logType, UUID.randomUUID()); + } + public static String getAlertsIndex(String logType) { return String.format(Locale.getDefault(), ".opensearch-sap-%s-alerts", logType); } diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertService.java b/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertService.java index e5c43698b..56f446cd4 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertService.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertService.java @@ -13,6 +13,7 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.support.WriteRequest; import org.opensearch.action.update.UpdateRequest; import org.opensearch.client.Client; import org.opensearch.common.lucene.uid.Versions; @@ -212,9 +213,10 @@ public void acknowledgeAlerts(List alertIds, ActionListener() { @Override public void onResponse(SearchResponse searchResponse) { + // Set the refresh policy on the BulkRequest + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); // Iterate through the search hits for (SearchHit hit : searchResponse.getHits().getHits()) { - // Construct a script to update the document with the new state and acknowledgedTime // Construct a script to update the document with the new state and acknowledgedTime Script script = new Script(ScriptType.INLINE, "painless", "ctx._source.state = params.state; ctx._source.acknowledged_time = params.time", diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java b/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java index 7ed1bb0ae..d081fd00f 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/alert/notifications/NotificationService.java @@ -81,6 +81,7 @@ public void sendNotification(String configId, String severity, String subject, S sendNotificationResponse -> { if (sendNotificationResponse.getStatus() == RestStatus.OK) { logger.info("Successfully sent a notification, Notification Event: " + sendNotificationResponse.getNotificationEvent()); + listener.onResponse(null); } else { listener.onFailure(new Exception("Error while sending a notification, Notification Event: " + sendNotificationResponse.getNotificationEvent())); } diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java index c6ffd8551..7f4c84eb9 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java @@ -4,11 +4,13 @@ */ package org.opensearch.securityanalytics.correlation.index.codec; +import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.backward_codecs.lucene95.Lucene95Codec; import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.opensearch.index.mapper.MapperService; +import org.opensearch.securityanalytics.correlation.index.codec.correlation9120.CorrelationCodec9120; import org.opensearch.securityanalytics.correlation.index.codec.correlation950.CorrelationCodec950; import org.opensearch.securityanalytics.correlation.index.codec.correlation990.CorrelationCodec990; import org.opensearch.securityanalytics.correlation.index.codec.correlation990.PerFieldCorrelationVectorsFormat990; @@ -32,9 +34,16 @@ public enum CorrelationCodecVersion { new PerFieldCorrelationVectorsFormat990(Optional.empty()), (userCodec, mapperService) -> new CorrelationCodec990(userCodec, new PerFieldCorrelationVectorsFormat990(Optional.of(mapperService))), CorrelationCodec990::new + ), + V_9_12_0( + "CorrelationCodec9120", + new Lucene912Codec(), + new PerFieldCorrelationVectorsFormat990(Optional.empty()), + (userCodec, mapperService) -> new CorrelationCodec9120(userCodec, new PerFieldCorrelationVectorsFormat990(Optional.of(mapperService))), + CorrelationCodec9120::new ); - private static final CorrelationCodecVersion CURRENT = V_9_9_0; + private static final CorrelationCodecVersion CURRENT = V_9_12_0; private final String codecName; private final Codec defaultCodecDelegate; private final PerFieldKnnVectorsFormat perFieldKnnVectorsFormat; diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation9120/CorrelationCodec9120.java b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation9120/CorrelationCodec9120.java new file mode 100644 index 000000000..1aa6e6824 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation9120/CorrelationCodec9120.java @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.correlation.index.codec.correlation9120; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecVersion; + +public class CorrelationCodec9120 extends FilterCodec { + private static final CorrelationCodecVersion VERSION = CorrelationCodecVersion.V_9_12_0; + private final PerFieldKnnVectorsFormat perFieldCorrelationVectorsFormat; + + public CorrelationCodec9120() { + this(VERSION.getDefaultCodecDelegate(), VERSION.getPerFieldCorrelationVectorsFormat()); + } + + public CorrelationCodec9120(Codec delegate, PerFieldKnnVectorsFormat perFieldCorrelationVectorsFormat) { + super(VERSION.getCodecName(), delegate); + this.perFieldCorrelationVectorsFormat = perFieldCorrelationVectorsFormat; + } + + @Override + public KnnVectorsFormat knnVectorsFormat() { + return perFieldCorrelationVectorsFormat; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java index d6cae5304..22b415230 100644 --- a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -558,8 +558,17 @@ private void rolloverIndex( request.getCreateIndexRequest().index(pattern) .mapping(map) .settings(isCorrelation? - Settings.builder().put("index.hidden", true).put("index.correlation", true).build(): - Settings.builder().put("index.hidden", true).build() + Settings.builder() + .put("index.hidden", true) + .put("index.correlation", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(): + Settings.builder() + .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build() ); request.addMaxIndexDocsCondition(docsCondition); request.addMaxIndexAgeCondition(ageCondition); diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java index 80d0ae50d..2e77b0831 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/BuiltinLogTypeLoader.java @@ -70,8 +70,6 @@ public void ensureLogTypesLoaded() { private List loadBuiltinLogTypes() throws URISyntaxException, IOException { List logTypes = new ArrayList<>(); - String pathurl = Paths.get(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH).toURI()).toString(); - final String url = Objects.requireNonNull(BuiltinLogTypeLoader.class.getClassLoader().getResource(BASE_PATH)).toURI().toString(); Path dirPath = null; if (url.contains("!")) { diff --git a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java index bc4452650..9c0d5ef72 100644 --- a/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java +++ b/src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java @@ -63,10 +63,10 @@ import org.opensearch.securityanalytics.model.LogType; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; -import static org.opensearch.action.support.ActiveShardCount.ALL; import static org.opensearch.securityanalytics.model.FieldMappingDoc.LOG_TYPES; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.DEFAULT_MAPPING_SCHEMA; - +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; /** * @@ -459,7 +459,8 @@ public void ensureConfigIndexIsInitialized(ActionListener listener) { isConfigIndexInitialized = false; Settings indexSettings = Settings.builder() .put("index.hidden", true) - .put("index.auto_expand_replicas", "0-all") + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) .build(); CreateIndexRequest createIndexRequest = new CreateIndexRequest(); diff --git a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java index 7769203b2..b7f15a094 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java +++ b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOC.java @@ -11,12 +11,14 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.commons.model.STIX2; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.securityanalytics.util.XContentUtils; import java.io.IOException; @@ -205,7 +207,18 @@ public static STIX2IOC parse(XContentParser xcp, String id, Long version) throws name = xcp.text(); break; case TYPE_FIELD: - type = new IOCType(xcp.text()); + String typeString = xcp.text(); + try { + type = new IOCType(typeString); + } catch (Exception e) { + String error = String.format( + "Couldn't parse IOC type '%s' while deserializing STIX2IOC with ID '%s': ", + typeString, + id + ); + logger.error(error, e); + throw new SecurityAnalyticsException(error, RestStatus.BAD_REQUEST, e); + } break; case VALUE_FIELD: value = xcp.text(); diff --git a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java index 2145c55cb..7a59b0ee0 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java +++ b/src/main/java/org/opensearch/securityanalytics/model/STIX2IOCDto.java @@ -5,26 +5,31 @@ package org.opensearch.securityanalytics.model; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParserUtils; import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.commons.model.STIX2; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import java.io.IOException; import java.time.Instant; import java.util.ArrayList; import java.util.List; -import java.util.Locale; /** * A data transfer object for the [STIX2IOC] data model. */ public class STIX2IOCDto implements Writeable, ToXContentObject { + private static final Logger logger = LogManager.getLogger(STIX2IOCDto.class); + private String id; private String name; private IOCType type; @@ -175,7 +180,18 @@ public static STIX2IOCDto parse(XContentParser xcp, String id, Long version) thr name = xcp.text(); break; case STIX2.TYPE_FIELD: - type = new IOCType(xcp.text()); + String typeString = xcp.text(); + try { + type = new IOCType(typeString); + } catch (Exception e) { + String error = String.format( + "Couldn't parse IOC type '%s' while deserializing STIX2IOCDto with ID '%s': ", + typeString, + id + ); + logger.error(error, e); + throw new SecurityAnalyticsException(error, RestStatus.BAD_REQUEST, e); + } break; case STIX2.VALUE_FIELD: value = xcp.text(); diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java index ec5ed713a..695a9d65a 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java @@ -9,7 +9,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchException; +import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.DocWriteRequest; +import org.opensearch.action.StepListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; import org.opensearch.action.bulk.BulkRequest; @@ -18,6 +20,7 @@ import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.Streams; @@ -33,6 +36,7 @@ import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.transport.RemoteTransportException; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -48,6 +52,9 @@ import java.util.Map; import java.util.UUID; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; + public class STIX2IOCFeedStore implements FeedStore { public static final String IOC_INDEX_NAME_BASE = ".opensearch-sap-iocs"; public static final String IOC_ALL_INDEX_PATTERN = IOC_INDEX_NAME_BASE + "-*"; @@ -80,7 +87,6 @@ public STIX2IOCFeedStore( this.baseListener = listener; batchSize = clusterService.getClusterSettings().get(SecurityAnalyticsSettings.BATCH_SIZE); newActiveIndex = getNewActiveIndex(saTifSourceConfig.getId()); - initSourceConfigIndexes(); } @Override @@ -113,7 +119,15 @@ public void storeIOCs(Map actionToIOCs) { } public void indexIocs(List iocs) throws IOException { - bulkIndexIocs(iocs, newActiveIndex); + StepListener initSourceConfigIndexesListener = new StepListener<>(); + initSourceConfigIndexes(initSourceConfigIndexesListener); + initSourceConfigIndexesListener.whenComplete(r -> { + bulkIndexIocs(iocs, newActiveIndex); + }, e -> { + log.error("Failed to init source config indexes"); + baseListener.onFailure(e); + }); + } private void bulkIndexIocs(List iocs, String activeIndex) throws IOException { @@ -197,7 +211,7 @@ public SATIFSourceConfig getSaTifSourceConfig() { return saTifSourceConfig; } - private void initSourceConfigIndexes() { + private void initSourceConfigIndexes(StepListener stepListener) { String iocIndexPattern = getAllIocIndexPatternById(saTifSourceConfig.getId()); initFeedIndex(newActiveIndex, ActionListener.wrap( r -> { @@ -214,10 +228,10 @@ private void initSourceConfigIndexes() { ((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails().add(iocToIndexDetails); } }); - + stepListener.onResponse(null); }, e-> { log.error("Failed to initialize the IOC index and save the IOCs", e); - baseListener.onFailure(e); + stepListener.onFailure(e); } )); } @@ -226,17 +240,29 @@ private void initFeedIndex(String feedIndexName, ActionListener { log.info("Created system index {}", feedIndexName); listener.onResponse(r); }, e -> { + if (e instanceof ResourceAlreadyExistsException || (e instanceof RemoteTransportException && e.getCause() instanceof ResourceAlreadyExistsException)) { + log.debug("index {} already exist", feedIndexName); + listener.onResponse(null); + return; + } log.error("Failed to create system index {}", feedIndexName); listener.onFailure(e); } )); + } else { + listener.onResponse(null); } } } diff --git a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java index 113a70499..31f4c6f2a 100644 --- a/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java +++ b/src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFetchService.java @@ -7,10 +7,12 @@ import com.amazonaws.AmazonServiceException; import com.amazonaws.SdkClientException; +import com.fasterxml.jackson.databind.RuntimeJsonMappingException; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchException; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; @@ -24,6 +26,7 @@ import org.opensearch.securityanalytics.action.TestS3ConnectionResponse; import org.opensearch.securityanalytics.commons.connector.Connector; import org.opensearch.securityanalytics.commons.connector.S3Connector; +import org.opensearch.securityanalytics.commons.connector.exceptions.ConnectorParsingException; import org.opensearch.securityanalytics.commons.connector.factory.InputCodecFactory; import org.opensearch.securityanalytics.commons.connector.factory.S3ClientFactory; import org.opensearch.securityanalytics.commons.connector.factory.StsAssumeRoleCredentialsProviderFactory; @@ -74,6 +77,9 @@ public class STIX2IOCFetchService { private final Logger log = LogManager.getLogger(STIX2IOCFetchService.class); private final String ENDPOINT_CONFIG_PATH = "/threatIntelFeed/internalAuthEndpoint.txt"; + public final String REGION_REGEX = "^.{1,20}$"; + public final String ROLE_ARN_REGEX = "^arn:aws:iam::\\d{12}:role/[\\w+=,.@-]{1,64}$"; + private Client client; private ClusterService clusterService; private STIX2IOCConnectorFactory connectorFactory; @@ -105,45 +111,109 @@ public void onlyIndexIocs(SATIFSourceConfig saTifSourceConfig, List stix2IOCList, ActionListener listener) { STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener); + Instant startTime = Instant.now(); + Instant endTime; + Exception exception = null; + RestStatus restStatus = null; try { + log.info("Started IOC index step at {}.", startTime); feedStore.indexIocs(stix2IOCList); + } catch (IllegalArgumentException e) { + exception = e; + restStatus = RestStatus.BAD_REQUEST; + } catch (OpenSearchException e) { + exception = e; + restStatus = e.status(); } catch (Exception e) { - log.error("Failed to index IOCs from source config", e); - listener.onFailure(e); + exception = e; + restStatus = RestStatus.INTERNAL_SERVER_ERROR; + } + endTime = Instant.now(); + long took = Duration.between(startTime, endTime).toMillis(); + + if (exception != null && restStatus != null) { + String errorText = getErrorText(saTifSourceConfig, "index", took); + log.error(errorText, exception); + listener.onFailure(new SecurityAnalyticsException(errorText, restStatus, exception)); + } else { + log.info("IOC index step took {} milliseconds.", took); } } public void downloadAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, ActionListener listener) { - S3ConnectorConfig s3ConnectorConfig = constructS3ConnectorConfig(saTifSourceConfig); + S3ConnectorConfig s3ConnectorConfig; + try { + s3ConnectorConfig = constructS3ConnectorConfig(saTifSourceConfig); + } catch (SecurityAnalyticsException e) { + listener.onFailure(e); + return; + } + Connector s3Connector = constructS3Connector(s3ConnectorConfig); STIX2IOCFeedStore feedStore = new STIX2IOCFeedStore(client, clusterService, saTifSourceConfig, listener); STIX2IOCConsumer consumer = new STIX2IOCConsumer(batchSize, feedStore, UpdateType.REPLACE); Instant startTime = Instant.now(); Instant endTime; + Exception exception = null; + RestStatus restStatus = null; try { log.info("Started IOC download step at {}.", startTime); s3Connector.load(consumer); + } catch (IllegalArgumentException | ConnectorParsingException | RuntimeJsonMappingException e) { + exception = e; + restStatus = RestStatus.BAD_REQUEST; + } catch (StsException | S3Exception e) { + exception = e; + restStatus = RestStatus.fromCode(e.statusCode()); + } catch (AmazonServiceException e) { + exception = e; + restStatus = RestStatus.fromCode(e.getStatusCode()); + } catch (SdkException | SdkClientException e) { + // SdkException is a RunTimeException that doesn't have a status code. + // Logging the full exception, and providing generic response as output. + exception = e; + restStatus = RestStatus.FORBIDDEN; } catch (Exception e) { - endTime = Instant.now(); - log.error("Failed to download IOCs after {} milliseconds.", Duration.between(startTime, endTime).toMillis(), e); - listener.onFailure(e); - return; + exception = e; + restStatus = RestStatus.INTERNAL_SERVER_ERROR; } endTime = Instant.now(); - log.info("IOC load step took {} milliseconds.", Duration.between(startTime, endTime).toMillis()); + long took = Duration.between(startTime, endTime).toMillis(); + + if (exception != null && restStatus != null) { + String errorText = getErrorText(saTifSourceConfig, "download", took); + log.error(errorText, exception); + listener.onFailure(new SecurityAnalyticsException(errorText, restStatus, exception)); + return; + } else { + log.info("IOC download step took {} milliseconds.", took); + } startTime = Instant.now(); try { log.info("Started IOC flush at {}.", startTime); consumer.flushIOCs(); + } catch (IllegalArgumentException e) { + exception = e; + restStatus = RestStatus.BAD_REQUEST; + } catch (OpenSearchException e) { + exception = e; + restStatus = e.status(); } catch (Exception e) { - endTime = Instant.now(); - log.error("Failed to flush IOCs queue after {} milliseconds.", Duration.between(startTime, endTime).toMillis(), e); - listener.onFailure(e); + exception = e; + restStatus = RestStatus.INTERNAL_SERVER_ERROR; } endTime = Instant.now(); - log.info("IOC flush step took {} milliseconds.", Duration.between(startTime, endTime).toMillis()); + took = Duration.between(startTime, endTime).toMillis(); + + if (exception != null && restStatus != null) { + String errorText = getErrorText(saTifSourceConfig, "index", took); + log.error(errorText, exception); + listener.onFailure(new SecurityAnalyticsException(errorText, restStatus, exception)); + } else { + log.info("IOC flush step took {} milliseconds.", took); + } } public void testS3Connection(S3ConnectorConfig s3ConnectorConfig, ActionListener listener) { @@ -160,24 +230,23 @@ private void testS3ClientConnection(S3ConnectorConfig s3ConnectorConfig, ActionL HeadObjectResponse response = connector.testS3Connection(s3ConnectorConfig); listener.onResponse(new TestS3ConnectionResponse(RestStatus.fromCode(response.sdkHttpResponse().statusCode()), "")); } catch (NoSuchKeyException noSuchKeyException) { - log.warn("S3Client connection test failed with NoSuchKeyException: ", noSuchKeyException); + log.error("S3Client connection test failed with NoSuchKeyException: ", noSuchKeyException); listener.onResponse(new TestS3ConnectionResponse(RestStatus.fromCode(noSuchKeyException.statusCode()), noSuchKeyException.awsErrorDetails().errorMessage())); } catch (S3Exception s3Exception) { - log.warn("S3Client connection test failed with S3Exception: ", s3Exception); + log.error("S3Client connection test failed with S3Exception: ", s3Exception); listener.onResponse(new TestS3ConnectionResponse(RestStatus.fromCode(s3Exception.statusCode()), "Resource not found.")); } catch (StsException stsException) { - log.warn("S3Client connection test failed with StsException: ", stsException); + log.error("S3Client connection test failed with StsException: ", stsException); listener.onResponse(new TestS3ConnectionResponse(RestStatus.fromCode(stsException.statusCode()), stsException.awsErrorDetails().errorMessage())); } catch (SdkException sdkException) { // SdkException is a RunTimeException that doesn't have a status code. // Logging the full exception, and providing generic response as output. - log.warn("S3Client connection test failed with SdkException: ", sdkException); + log.error("S3Client connection test failed with SdkException: ", sdkException); listener.onResponse(new TestS3ConnectionResponse(RestStatus.FORBIDDEN, "Resource not found.")); } catch (Exception e) { - log.warn("S3Client connection test failed with error: ", e); + log.error("S3Client connection test failed with error: ", e); listener.onFailure(SecurityAnalyticsException.wrap(e)); } - } private void testAmazonS3Connection(S3ConnectorConfig s3ConnectorConfig, ActionListener listener) { @@ -186,15 +255,15 @@ private void testAmazonS3Connection(S3ConnectorConfig s3ConnectorConfig, ActionL boolean response = connector.testAmazonS3Connection(s3ConnectorConfig); listener.onResponse(new TestS3ConnectionResponse(response ? RestStatus.OK : RestStatus.FORBIDDEN, "")); } catch (AmazonServiceException e) { - log.warn("AmazonS3 connection test failed with AmazonServiceException: ", e); + log.error("AmazonS3 connection test failed with AmazonServiceException: ", e); listener.onResponse(new TestS3ConnectionResponse(RestStatus.fromCode(e.getStatusCode()), e.getErrorMessage())); } catch (SdkClientException e) { // SdkException is a RunTimeException that doesn't have a status code. // Logging the full exception, and providing generic response as output. - log.warn("AmazonS3 connection test failed with SdkClientException: ", e); + log.error("AmazonS3 connection test failed with SdkClientException: ", e); listener.onResponse(new TestS3ConnectionResponse(RestStatus.FORBIDDEN, "Resource not found.")); } catch (Exception e) { - log.warn("AmazonS3 connection test failed with error: ", e); + log.error("AmazonS3 connection test failed with error: ", e); listener.onFailure(SecurityAnalyticsException.wrap(e)); } } @@ -229,12 +298,12 @@ private S3ConnectorConfig constructS3ConnectorConfig(SATIFSourceConfig saTifSour } private void validateS3ConnectorConfig(S3ConnectorConfig s3ConnectorConfig) { - if (s3ConnectorConfig.getRoleArn() == null || s3ConnectorConfig.getRoleArn().isEmpty()) { - throw new IllegalArgumentException("Role arn is required."); + if (s3ConnectorConfig.getRoleArn() == null || !s3ConnectorConfig.getRoleArn().matches(ROLE_ARN_REGEX)) { + throw new SecurityAnalyticsException("Role arn is empty or malformed.", RestStatus.BAD_REQUEST, new IllegalArgumentException()); } - if (s3ConnectorConfig.getRegion() == null || s3ConnectorConfig.getRegion().isEmpty()) { - throw new IllegalArgumentException("Region is required."); + if (s3ConnectorConfig.getRegion() == null || !s3ConnectorConfig.getRegion().matches(REGION_REGEX)) { + throw new SecurityAnalyticsException("Region is empty or malformed.", RestStatus.BAD_REQUEST, new IllegalArgumentException()); } } @@ -276,13 +345,13 @@ public void downloadFromUrlAndIndexIOCs(SATIFSourceConfig saTifSourceConfig, Act } } catch (Exception e) { log.error("Failed to download the IoCs in CSV format for source " + saTifSourceConfig.getId()); - listener.onFailure(e); + listener.onFailure(SecurityAnalyticsException.wrap(e)); return; } break; default: log.error("unsupported feed format for url download:" + source.getFeedFormat()); - listener.onFailure(new UnsupportedOperationException("unsupported feed format for url download:" + source.getFeedFormat())); + listener.onFailure(SecurityAnalyticsException.wrap(new UnsupportedOperationException("unsupported feed format for url download:" + source.getFeedFormat()))); } } @@ -322,6 +391,23 @@ private void parseAndSaveThreatIntelFeedDataCSV(Iterator iterator, SA feedStore.indexIocs(iocs); } + /** + * Helper function for generating error message text. + * @param saTifSourceConfig The config for which IOCs are being downloaded/indexed. + * @param action The action that was being taken when the error occurred; e.g., "download", or "index". + * @param duration The amount of time, in milliseconds, it took for the action to fail. + * @return The error message text. + */ + private String getErrorText(SATIFSourceConfig saTifSourceConfig, String action, long duration) { + return String.format( + "Failed to %s IOCs from source config '%s' with ID %s after %s milliseconds: ", + action, + saTifSourceConfig.getName(), + saTifSourceConfig.getId(), + duration + ); + } + public static class STIX2IOCFetchResponse extends ActionResponse implements ToXContentObject { public static String IOCS_FIELD = "iocs"; public static String TOTAL_FIELD = "total"; diff --git a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java index b0f0fed74..cc11fe36d 100644 --- a/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java +++ b/src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java @@ -12,6 +12,8 @@ public class SecurityAnalyticsSettings { public static final String CORRELATION_INDEX = "index.correlation"; + public static final int minSystemIndexReplicas = 0; + public static final int maxSystemIndexReplicas = 20; public static Setting INDEX_TIMEOUT = Setting.positiveTimeSetting("plugins.security_analytics.index_timeout", TimeValue.timeValueSeconds(60), @@ -237,4 +239,20 @@ public static final List> settings() { Setting.Property.NodeScope, Setting.Property.Dynamic ); + /** + * Maximum terms in Terms query search query submitted during ioc scan + */ + public static final Setting IOC_SCAN_MAX_TERMS_COUNT = Setting.intSetting( + "plugins.security_analytics.ioc.scan_max_terms_count", + 65536, + 1, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + + public static final Setting ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES = Setting.boolSetting( + "plugins.security_analytics.enable_detectors_with_dedicated_query_indices", + true, + Setting.Property.NodeScope, Setting.Property.Dynamic + ); + } \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/action/ListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsAction.java similarity index 88% rename from src/main/java/org/opensearch/securityanalytics/action/ListIOCsAction.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsAction.java index ae4912bbc..f9e5bde66 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/ListIOCsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsAction.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.action; +package org.opensearch.securityanalytics.threatIntel.action; import org.opensearch.action.ActionType; diff --git a/src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionRequest.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionRequest.java similarity index 97% rename from src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionRequest.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionRequest.java index dead1cd3f..cb57213b9 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionRequest.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionRequest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.action; +package org.opensearch.securityanalytics.threatIntel.action; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; @@ -14,7 +14,6 @@ import org.opensearch.securityanalytics.commons.model.IOCType; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.stream.Collectors; diff --git a/src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionResponse.java similarity index 96% rename from src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionResponse.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionResponse.java index 741f3cf36..0f142fbf0 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/ListIOCsActionResponse.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/ListIOCsActionResponse.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.action; +package org.opensearch.securityanalytics.threatIntel.action; import org.opensearch.core.action.ActionResponse; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java index 247bcd134..7bebd8fb1 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAGetTIFSourceConfigResponse.java @@ -63,25 +63,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject() .field(_ID, id) .field(_VERSION, version); - builder.startObject("source_config") - .field(SATIFSourceConfigDto.NAME_FIELD, saTifSourceConfigDto.getName()) - .field(SATIFSourceConfigDto.FORMAT_FIELD, saTifSourceConfigDto.getFormat()) - .field(SATIFSourceConfigDto.TYPE_FIELD, saTifSourceConfigDto.getType()) - .field(SATIFSourceConfigDto.IOC_TYPES_FIELD, saTifSourceConfigDto.getIocTypes()) - .field(SATIFSourceConfigDto.DESCRIPTION_FIELD, saTifSourceConfigDto.getDescription()) - .field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, saTifSourceConfigDto.getCreatedByUser()) - .field(SATIFSourceConfigDto.CREATED_AT_FIELD, saTifSourceConfigDto.getCreatedAt()) - .field(SATIFSourceConfigDto.SOURCE_FIELD, saTifSourceConfigDto.getSource()) - .field(SATIFSourceConfigDto.ENABLED_FIELD, saTifSourceConfigDto.isEnabled()) - .field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, saTifSourceConfigDto.getEnabledTime()) - .field(SATIFSourceConfigDto.LAST_UPDATE_TIME_FIELD, saTifSourceConfigDto.getLastUpdateTime()) - .field(SATIFSourceConfigDto.SCHEDULE_FIELD, saTifSourceConfigDto.getSchedule()) - .field(SATIFSourceConfigDto.STATE_FIELD, saTifSourceConfigDto.getState()) - .field(SATIFSourceConfigDto.REFRESH_TYPE_FIELD, saTifSourceConfigDto.getRefreshType()) - .field(SATIFSourceConfigDto.LAST_REFRESHED_USER_FIELD, saTifSourceConfigDto.getLastRefreshedUser()) - .field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, saTifSourceConfigDto.getLastRefreshedTime()); - - builder.endObject(); + saTifSourceConfigDto.innerXcontent(builder); return builder.endObject(); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java index 7a1881162..209563f7c 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/action/SAIndexTIFSourceConfigResponse.java @@ -17,6 +17,7 @@ import java.io.IOException; +import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto.SOURCE_CONFIG_FIELD; import static org.opensearch.securityanalytics.util.RestHandlerUtils._ID; import static org.opensearch.securityanalytics.util.RestHandlerUtils._VERSION; @@ -56,40 +57,25 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject() .field(_ID, id) .field(_VERSION, version); - - builder.startObject("source_config") - .field(SATIFSourceConfigDto.NAME_FIELD, saTifSourceConfigDto.getName()) - .field(SATIFSourceConfigDto.FORMAT_FIELD, saTifSourceConfigDto.getFormat()) - .field(SATIFSourceConfigDto.TYPE_FIELD, saTifSourceConfigDto.getType()) - .field(SATIFSourceConfigDto.IOC_TYPES_FIELD, saTifSourceConfigDto.getIocTypes()) - .field(SATIFSourceConfigDto.DESCRIPTION_FIELD, saTifSourceConfigDto.getDescription()) - .field(SATIFSourceConfigDto.CREATED_BY_USER_FIELD, saTifSourceConfigDto.getCreatedByUser()) - .field(SATIFSourceConfigDto.CREATED_AT_FIELD, saTifSourceConfigDto.getCreatedAt()) - .field(SATIFSourceConfigDto.SOURCE_FIELD, saTifSourceConfigDto.getSource()) - .field(SATIFSourceConfigDto.ENABLED_FIELD, saTifSourceConfigDto.isEnabled()) - .field(SATIFSourceConfigDto.ENABLED_TIME_FIELD, saTifSourceConfigDto.getEnabledTime()) - .field(SATIFSourceConfigDto.LAST_UPDATE_TIME_FIELD, saTifSourceConfigDto.getLastUpdateTime()) - .field(SATIFSourceConfigDto.SCHEDULE_FIELD, saTifSourceConfigDto.getSchedule()) - .field(SATIFSourceConfigDto.STATE_FIELD, saTifSourceConfigDto.getState()) - .field(SATIFSourceConfigDto.REFRESH_TYPE_FIELD, saTifSourceConfigDto.getRefreshType()) - .field(SATIFSourceConfigDto.LAST_REFRESHED_USER_FIELD, saTifSourceConfigDto.getLastRefreshedUser()) - .field(SATIFSourceConfigDto.LAST_REFRESHED_TIME_FIELD, saTifSourceConfigDto.getLastRefreshedTime()); - - builder.endObject(); + saTifSourceConfigDto.innerXcontent(builder); return builder.endObject(); } + @Override public String getTIFConfigId() { return id; } + @Override public Long getVersion() { return version; } + @Override public TIFSourceConfigDto getTIFConfigDto() { return saTifSourceConfigDto; } + public RestStatus getStatus() { return status; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java index 8001f37ea..4a5ab1446 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/SourceConfigDtoValidator.java @@ -5,7 +5,6 @@ package org.opensearch.securityanalytics.threatIntel.common; -import org.opensearch.securityanalytics.commons.model.IOC; import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; import org.opensearch.securityanalytics.threatIntel.model.S3Source; @@ -13,9 +12,8 @@ import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; +import java.util.regex.Pattern; /** * Source config dto validator @@ -24,7 +22,34 @@ public class SourceConfigDtoValidator { public List validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto) { List errorMsgs = new ArrayList<>(); - if (sourceConfigDto.getIocTypes().isEmpty()) { + String nameRegex = "^[a-zA-Z0-9 _-]{1,128}$"; + Pattern namePattern = Pattern.compile(nameRegex); + + int MAX_RULE_DESCRIPTION_LENGTH = 65535; + String descriptionRegex = "^.{0," + MAX_RULE_DESCRIPTION_LENGTH + "}$"; + Pattern descriptionPattern = Pattern.compile(descriptionRegex); + + if (sourceConfigDto.getName() == null || sourceConfigDto.getName().isEmpty()) { + errorMsgs.add("Name must not be empty"); + } else if (sourceConfigDto.getName() != null && namePattern.matcher(sourceConfigDto.getName()).matches() == false) { + errorMsgs.add("Name must be less than 128 characters and only consist of upper and lowercase letters, numbers 0-9, hyphens, spaces, and underscores"); + } + + if (sourceConfigDto.getFormat() == null || sourceConfigDto.getFormat().isEmpty()) { + errorMsgs.add("Format must not be empty"); + } else if (sourceConfigDto.getFormat() != null && sourceConfigDto.getFormat().length() > 50) { + errorMsgs.add("Format must be 50 characters or less"); + } + + if (sourceConfigDto.getDescription() != null && descriptionPattern.matcher(sourceConfigDto.getDescription()).matches() == false) { + errorMsgs.add("Description must be " + MAX_RULE_DESCRIPTION_LENGTH + " characters or less"); + } + + if (sourceConfigDto.getSource() == null) { + errorMsgs.add("Source must not be empty"); + } + + if (sourceConfigDto.getIocTypes() == null || sourceConfigDto.getIocTypes().isEmpty()) { errorMsgs.add("Must specify at least one IOC type"); } else { for (String s: sourceConfigDto.getIocTypes()) { @@ -34,34 +59,41 @@ public List validateSourceConfigDto(SATIFSourceConfigDto sourceConfigDto } } - switch (sourceConfigDto.getType()) { - case IOC_UPLOAD: - if (sourceConfigDto.isEnabled()) { - errorMsgs.add("Job Scheduler cannot be enabled for IOC_UPLOAD type"); - } - if (sourceConfigDto.getSchedule() != null) { - errorMsgs.add("Cannot pass in schedule for IOC_UPLOAD type"); - } - if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof IocUploadSource == false) { - errorMsgs.add("Source must be IOC_UPLOAD type"); - } - break; - case S3_CUSTOM: - if (sourceConfigDto.getSchedule() == null) { - errorMsgs.add("Must pass in schedule for S3_CUSTOM type"); - } - if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof S3Source == false) { - errorMsgs.add("Source must be S3_CUSTOM type"); - } - break; - case URL_DOWNLOAD: - if (sourceConfigDto.getSchedule() == null) { - errorMsgs.add("Must pass in schedule for URL_DOWNLOAD source type"); - } - if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof UrlDownloadSource == false) { - errorMsgs.add("Source must be URL_DOWNLOAD source type"); - } - break; + if (sourceConfigDto.getType() == null) { + errorMsgs.add("Type must not be empty"); + } else { + switch (sourceConfigDto.getType()) { + case IOC_UPLOAD: + if (sourceConfigDto.isEnabled()) { + errorMsgs.add("Job Scheduler cannot be enabled for IOC_UPLOAD type"); + } + if (sourceConfigDto.getSchedule() != null) { + errorMsgs.add("Cannot pass in schedule for IOC_UPLOAD type"); + } + if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof IocUploadSource == false) { + errorMsgs.add("Source must be IOC_UPLOAD type"); + } + if (sourceConfigDto.getSource() instanceof IocUploadSource && ((IocUploadSource) sourceConfigDto.getSource()).getIocs() == null) { + errorMsgs.add("Ioc list must include at least one ioc"); + } + break; + case S3_CUSTOM: + if (sourceConfigDto.getSchedule() == null) { + errorMsgs.add("Must pass in schedule for S3_CUSTOM type"); + } + if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof S3Source == false) { + errorMsgs.add("Source must be S3_CUSTOM type"); + } + break; + case URL_DOWNLOAD: + if (sourceConfigDto.getSchedule() == null) { + errorMsgs.add("Must pass in schedule for URL_DOWNLOAD source type"); + } + if (sourceConfigDto.getSource() != null && sourceConfigDto.getSource() instanceof UrlDownloadSource == false) { + errorMsgs.add("Source must be URL_DOWNLOAD source type"); + } + break; + } } return errorMsgs; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java index 98abf040a..eb467c55e 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFLockService.java @@ -46,7 +46,7 @@ public TIFLockService(final ClusterService clusterService, final Client client) } /** - * Synchronous method of #acquireLock + * Event-driven method of #acquireLock * * @param tifJobName tifJobName to acquire lock on * @param lockDurationSeconds the lock duration in seconds @@ -73,10 +73,13 @@ public void onFailure(final Exception e) { * * @param lockModel the lock model */ - public void releaseLock(final LockModel lockModel) { + public void releaseLock(final LockModel lockModel, final ActionListener listener) { lockService.release( lockModel, - ActionListener.wrap(released -> {}, exception -> log.error("Failed to release the lock", exception)) + ActionListener.wrap(listener::onResponse, exception -> { + log.error("Failed to release the lock", exception); + listener.onFailure(exception); + }) ); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java index 62eee1a57..f435db5ce 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dao/BaseEntityCrudService.java @@ -14,6 +14,7 @@ import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.WriteRequest; import org.opensearch.client.Client; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentFactory; @@ -31,6 +32,8 @@ import java.util.ArrayList; import java.util.List; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; import static org.opensearch.securityanalytics.util.DetectorUtils.getEmptySearchResponse; /** @@ -112,14 +115,19 @@ public void bulkIndexEntities(List newEntityList, List updatedEn } } actionListener.onResponse(null); - }, actionListener::onFailure), bulkRequestList.size()); + }, e1 -> { + log.error("Failed to bulk index " + getEntityName(), e1); + actionListener.onFailure(e1); + }), bulkRequestList.size()); + for (BulkRequest req : bulkRequestList) { try { - client.bulk(req, groupedListener); //todo why stash context here? + client.bulk(req, groupedListener); } catch (Exception e) { log.error( () -> new ParameterizedMessage("Failed to bulk save {} {}.", req.batchSize(), getEntityName()), e); + groupedListener.onFailure(e); } } }, e -> { @@ -247,7 +255,9 @@ public void createIndexIfNotExists(final ActionListener listener) { public abstract String getEntityName(); protected Settings.Builder getIndexSettings() { - return Settings.builder().put("index.hidden", true); + return Settings.builder().put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas); } public abstract String getEntityAliasName(); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeScanInputDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeScanInputDto.java index 36f34eebb..8d0756c18 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeScanInputDto.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/dto/PerIocTypeScanInputDto.java @@ -63,7 +63,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static PerIocTypeScanInputDto parse(XContentParser xcp) throws IOException { - String iocType = null; + String iocType = ""; Map> indexToFieldsMap = new HashMap<>(); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); @@ -73,7 +73,7 @@ public static PerIocTypeScanInputDto parse(XContentParser xcp) throws IOExceptio switch (fieldName) { case IOC_TYPE: - iocType = xcp.text(); + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) iocType = xcp.text(); break; case INDEX_TO_FIELDS_MAP: if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { @@ -83,7 +83,7 @@ public static PerIocTypeScanInputDto parse(XContentParser xcp) throws IOExceptio List fields = new ArrayList<>(); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { - fields.add(xcp.text()); + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) fields.add(xcp.text()); } return fields; }); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java index f3238460a..f60af7afd 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/IoCScanService.java @@ -40,6 +40,13 @@ public void scanIoCs(IocScanContext iocScanContext, long startTime = System.currentTimeMillis(); IocLookupDtos iocLookupDtos = extractIocsPerType(data, iocScanContext); + if (iocLookupDtos.getIocsPerIocTypeMap().isEmpty()) { + log.error("Threat intel monitor {}: Unexpected scenario that non-zero number of docs are fetched from indices containing iocs but iocs-per-type map constructed is empty", + iocScanContext.getMonitor().getId() + ); + scanCallback.accept(Collections.emptyList(), null); + return; + } BiConsumer, Exception> iocScanResultConsumer = (List maliciousIocs, Exception e) -> { long scanEndTime = System.currentTimeMillis(); long timeTaken = scanEndTime - startTime; @@ -49,10 +56,10 @@ public void scanIoCs(IocScanContext iocScanContext, (iocFindings, e1) -> { if (e1 != null) { log.error( - () -> new ParameterizedMessage("Threat intel monitor {}: Failed to create ioc findings/ ", + () -> new ParameterizedMessage("Threat intel monitor {}: Failed to create ioc findings", iocScanContext.getMonitor().getId(), data.size()), e1); - scanCallback.accept(null, e1); + scanCallback.accept(data, e1); } else { BiConsumer, Exception> triggerResultConsumer = (alerts, e2) -> { if (e2 != null) { @@ -60,8 +67,8 @@ public void scanIoCs(IocScanContext iocScanContext, () -> new ParameterizedMessage("Threat intel monitor {}: Failed to execute threat intel triggers/ ", iocScanContext.getMonitor().getId(), data.size()), e2); - scanCallback.accept(null, e2); - return; + // if findings are generated successfully but alerts/notifications fail we mark execution as succeeded, so that duplicate findings are not created + scanCallback.accept(data, null); } else { scanCallback.accept(data, null); } @@ -121,28 +128,28 @@ abstract void matchAgainstThreatIntelAndReturnMaliciousIocs( for (PerIocTypeScanInput iocTypeToIndexFieldMapping : context.getThreatIntelInput().getPerIocTypeScanInputList()) { String iocType = iocTypeToIndexFieldMapping.getIocType().toLowerCase(); String concreteIndex = getIndexName(datum); - if (context.getConcreteIndexToMonitorInputIndicesMap().containsKey(concreteIndex) - && false == context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).isEmpty() - ) { + if (context.getConcreteIndexToMonitorInputIndicesMap().containsKey(concreteIndex)) { // if concrete index resolves to multiple monitor input indices, it's undesirable. We just pick any one of the monitor input indices to get fields for each ioc. String index = context.getConcreteIndexToMonitorInputIndicesMap().get(concreteIndex).get(0); - List fields = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index); - for (String field : fields) { - List vals = getValuesAsStringList(datum, field); - String id = getId(datum); - String docId = id + ":" + index; - Set iocs = docIdToIocsMap.getOrDefault(docId, new HashSet<>()); - iocs.addAll(vals); - docIdToIocsMap.put(docId, iocs); - for (String ioc : vals) { - Set docIds = iocValueToDocIdMap.getOrDefault(ioc, new HashSet<>()); - docIds.add(docId); - iocValueToDocIdMap.put(ioc, docIds); - } - if (false == vals.isEmpty()) { - iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>()); + List fieldsConfiguredInMonitorForCurrentIndex = iocTypeToIndexFieldMapping.getIndexToFieldsMap().get(index); + if(fieldsConfiguredInMonitorForCurrentIndex != null && false == fieldsConfiguredInMonitorForCurrentIndex.isEmpty()) { + for (String field : fieldsConfiguredInMonitorForCurrentIndex) { + List vals = getValuesAsStringList(datum, field); + String id = getId(datum); + String docId = id + ":" + index; + Set iocs = docIdToIocsMap.getOrDefault(docId, new HashSet<>()); iocs.addAll(vals); - iocsPerIocTypeMap.put(iocType, iocs); + docIdToIocsMap.put(docId, iocs); + for (String ioc : vals) { + Set docIds = iocValueToDocIdMap.getOrDefault(ioc, new HashSet<>()); + docIds.add(docId); + iocValueToDocIdMap.put(ioc, docIds); + } + if (false == vals.isEmpty()) { + iocs = iocsPerIocTypeMap.getOrDefault(iocType, new HashSet<>()); + iocs.addAll(vals); + iocsPerIocTypeMap.put(iocType, iocs); + } } } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java index 47dfeed09..8a3c4a206 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/iocscan/service/SaIoCScanService.java @@ -7,6 +7,7 @@ import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.action.support.GroupedActionListener; import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.document.DocumentField; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentType; @@ -28,6 +29,7 @@ import org.opensearch.securityanalytics.model.STIX2IOC; import org.opensearch.securityanalytics.model.threatintel.IocFinding; import org.opensearch.securityanalytics.model.threatintel.ThreatIntelAlert; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; import org.opensearch.securityanalytics.threatIntel.iocscan.dao.ThreatIntelAlertService; import org.opensearch.securityanalytics.threatIntel.iocscan.dto.IocScanContext; @@ -54,16 +56,17 @@ public class SaIoCScanService extends IoCScanService { private static final Logger log = LogManager.getLogger(SaIoCScanService.class); - public static final int MAX_TERMS = 65536; //TODO make ioc index setting based. use same setting value to create index private final Client client; + private final ClusterService clusterService; private final NamedXContentRegistry xContentRegistry; private final IocFindingService iocFindingService; private final ThreatIntelAlertService threatIntelAlertService; private final NotificationService notificationService; - public SaIoCScanService(Client client, NamedXContentRegistry xContentRegistry, IocFindingService iocFindingService, + public SaIoCScanService(Client client, ClusterService clusterService, NamedXContentRegistry xContentRegistry, IocFindingService iocFindingService, ThreatIntelAlertService threatIntelAlertService, NotificationService notificationService) { this.client = client; + this.clusterService = clusterService; this.xContentRegistry = xContentRegistry; this.iocFindingService = iocFindingService; this.threatIntelAlertService = threatIntelAlertService; @@ -110,7 +113,7 @@ private void executeTrigger(List iocFindings, } else { fetchExistingAlertsForTrigger(monitor, triggerMatchedFindings, trigger, ActionListener.wrap( existingAlerts -> { - executeActionsAndSaveAlerts(iocFindings, trigger, monitor, existingAlerts, triggerMatchedFindings, threatIntelTrigger, listener); + saveAlertsAndExecuteActions(iocFindings, trigger, monitor, existingAlerts, triggerMatchedFindings, threatIntelTrigger, listener); }, e -> { log.error(() -> new ParameterizedMessage( @@ -129,7 +132,7 @@ private void executeTrigger(List iocFindings, } } - private void executeActionsAndSaveAlerts(List iocFindings, + private void saveAlertsAndExecuteActions(List iocFindings, Trigger trigger, Monitor monitor, List existingAlerts, @@ -144,36 +147,38 @@ private void executeActionsAndSaveAlerts(List iocFindings, newAlerts, existingAlerts); if (false == trigger.getActions().isEmpty()) { - GroupedActionListener notifsListener = new GroupedActionListener<>(ActionListener.wrap( - r -> { - saveAlerts(new ArrayList<>(iocToUpdatedAlertsMap.values()), - newAlerts, - monitor, - (threatIntelAlerts, e) -> { - if (e != null) { - log.error(String.format("Threat intel monitor %s: Failed to save alerts for trigger {}", monitor.getId(), trigger.getId()), e); - listener.onFailure(e); - } else { + saveAlerts(new ArrayList<>(iocToUpdatedAlertsMap.values()), + newAlerts, + monitor, + (threatIntelAlerts, e) -> { + if (e != null) { + log.error(String.format("Threat intel monitor %s: Failed to save alerts for trigger %s", monitor.getId(), trigger.getId()), e); + listener.onFailure(e); + } else { + GroupedActionListener notifsListener = new GroupedActionListener<>(ActionListener.wrap( + r -> { listener.onResponse(threatIntelAlerts); + }, ex -> { + log.error(String.format("Threat intel monitor {}: Failed to send notification for trigger {}", monitor.getId(), trigger.getId()), ex); + listener.onFailure(new SecurityAnalyticsException("Failed to send notification", RestStatus.INTERNAL_SERVER_ERROR, ex)); } - }); - }, e -> { - log.error(String.format("Threat intel monitor %s: Failed to send notification for trigger {}", monitor.getId(), trigger.getId()), e); - listener.onFailure(new SecurityAnalyticsException("Failed to send notification", RestStatus.INTERNAL_SERVER_ERROR, e)); - } - ), trigger.getActions().size()); - for (Action action : trigger.getActions()) { - try { - String transformedSubject = NotificationService.compileTemplate(ctx, action.getSubjectTemplate()); - String transformedMessage = NotificationService.compileTemplate(ctx, action.getMessageTemplate()); - String configId = action.getDestinationId(); - notificationService.sendNotification(configId, trigger.getSeverity(), transformedSubject, transformedMessage, notifsListener); - } catch (Exception e) { - log.error(String.format("Threat intel monitor %s: Failed to send notification to %s for trigger %s", monitor.getId(), action.getDestinationId(), trigger.getId()), e); - notifsListener.onFailure(new SecurityAnalyticsException("Failed to send notification", RestStatus.INTERNAL_SERVER_ERROR, e)); - } + ), trigger.getActions().size()); + + for (Action action : trigger.getActions()) { + try { + String transformedSubject = NotificationService.compileTemplate(ctx, action.getSubjectTemplate()); + String transformedMessage = NotificationService.compileTemplate(ctx, action.getMessageTemplate()); + String configId = action.getDestinationId(); + notificationService.sendNotification(configId, trigger.getSeverity(), transformedSubject, transformedMessage, notifsListener); + } catch (Exception ex) { + log.error(String.format("Threat intel monitor %s: Failed to send notification to %s for trigger %s", monitor.getId(), action.getDestinationId(), trigger.getId()), ex); + notifsListener.onFailure(new SecurityAnalyticsException("Failed to send notification", RestStatus.INTERNAL_SERVER_ERROR, ex)); + } + + } + } + }); - } } else { saveAlerts(new ArrayList<>(iocToUpdatedAlertsMap.values()), newAlerts, @@ -232,7 +237,7 @@ private GroupedActionListener> getGroupedListenerForAllTr r -> { List list = new ArrayList<>(); r.forEach(list::addAll); - triggerResultConsumer.accept(list, null); //todo change emptylist to actual response + triggerResultConsumer.accept(list, null); }, e -> { log.error(() -> new ParameterizedMessage( "Threat intel monitor {} Failed to execute triggers {}", monitor.getId()), @@ -329,12 +334,13 @@ private void performScanForMaliciousIocsPerIocType( GroupedActionListener listener) { // TODO change ioc indices max terms count to 100k and experiment // TODO add fuzzy postings on ioc value field to enable bloomfilter on iocs as an index data structure and benchmark performance - GroupedActionListener perIocTypeListener = getGroupedListenerForIocScanPerIocType(iocs, monitor, iocType, listener); + int maxTerms = clusterService.getClusterSettings().get(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT); + GroupedActionListener perIocTypeListener = getGroupedListenerForIocScanPerIocType(iocs, monitor, iocType, listener, maxTerms); List iocList = new ArrayList<>(iocs); int totalIocs = iocList.size(); - for (int start = 0; start < totalIocs; start += MAX_TERMS) { - int end = Math.min(start + MAX_TERMS, totalIocs); + for (int start = 0; start < totalIocs; start += maxTerms) { + int end = Math.min(start + maxTerms, totalIocs); List iocsSublist = iocList.subList(start, end); SearchRequest searchRequest = getSearchRequestForIocType(indices, iocType, iocsSublist); client.search(searchRequest, ActionListener.wrap( @@ -356,7 +362,7 @@ private void performScanForMaliciousIocsPerIocType( ); } } - listener.onResponse(new SearchHitsOrException( + perIocTypeListener.onResponse(new SearchHitsOrException( searchResponse.getHits() == null || searchResponse.getHits().getHits() == null ? emptyList() : Arrays.asList(searchResponse.getHits().getHits()), null)); }, @@ -366,7 +372,7 @@ private void performScanForMaliciousIocsPerIocType( iocsSublist.size(), iocType), e ); - listener.onResponse(new SearchHitsOrException(emptyList(), e)); + perIocTypeListener.onResponse(new SearchHitsOrException(emptyList(), e)); } )); } @@ -376,9 +382,9 @@ private static SearchRequest getSearchRequestForIocType(List indices, St SearchRequest searchRequest = new SearchRequest(indices.toArray(new String[0])); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); // add the iocs sublist - boolQueryBuilder.must(new TermsQueryBuilder(STIX2.VALUE_FIELD + ".keyword", iocsSublist)); + boolQueryBuilder.must(new TermsQueryBuilder(STIX2.VALUE_FIELD, iocsSublist)); // add ioc type filter - boolQueryBuilder.must(new TermsQueryBuilder(STIX2.TYPE_FIELD + ".keyword", iocType.toLowerCase(Locale.ROOT))); + boolQueryBuilder.must(new TermsQueryBuilder(STIX2.TYPE_FIELD, iocType.toLowerCase(Locale.ROOT))); searchRequest.source().query(boolQueryBuilder); return searchRequest; } @@ -387,7 +393,7 @@ private static SearchRequest getSearchRequestForIocType(List indices, St * grouped listener for a given ioc type to listen and collate malicious iocs in search hits from batched search calls. * batching done for every 65536 or MAX_TERMS setting number of iocs in a list. */ - private GroupedActionListener getGroupedListenerForIocScanPerIocType(Set iocs, Monitor monitor, String iocType, GroupedActionListener groupedListenerForAllIocTypes) { + private GroupedActionListener getGroupedListenerForIocScanPerIocType(Set iocs, Monitor monitor, String iocType, GroupedActionListener groupedListenerForAllIocTypes, int maxTerms) { return new GroupedActionListener<>( ActionListener.wrap( (Collection searchHitsOrExceptions) -> { @@ -419,8 +425,7 @@ private GroupedActionListener getGroupedListenerForIocSca groupedListenerForAllIocTypes.onResponse(new SearchHitsOrException(emptyList(), e)); } ), - //TODO fix groupsize - getGroupSizeForIocs(iocs) // batch into #MAX_TERMS setting + getGroupSizeForIocs(iocs, maxTerms) ); } @@ -436,8 +441,8 @@ private Exception buildException(Collection searchHitsOrE return e; } - private static int getGroupSizeForIocs(Set iocs) { - return iocs.size() / MAX_TERMS + (iocs.size() % MAX_TERMS == 0 ? 0 : 1); + private static int getGroupSizeForIocs(Set iocs, int maxTerms) { + return iocs.size() / maxTerms + (iocs.size() % maxTerms == 0 ? 0 : 1); } @Override diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java index 65d7e46e5..89dc729b4 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java @@ -115,10 +115,24 @@ protected Runnable updateJobRunner(final ScheduledJobParameter jobParameter) { ActionListener.wrap(lock -> { updateJobParameter(jobParameter, lockService.getRenewLockRunnable(new AtomicReference<>(lock)), ActionListener.wrap( - r -> lockService.releaseLock(lock), + r -> lockService.releaseLock(lock, ActionListener.wrap( + response -> { + log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif job parameter [%s].", lock.getLockId(), jobParameter.getName()), ex); + } + )), e -> { log.error("Failed to update job parameter " + jobParameter.getName(), e); - lockService.releaseLock(lock); + lockService.releaseLock(lock, ActionListener.wrap( + response -> { + log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif job parameter [%s].", lock.getLockId(), jobParameter.getName()), ex); + } + )); } )); }, e -> { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFSourceConfigRunner.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFSourceConfigRunner.java index 2b729f0f4..1c098cfc9 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFSourceConfigRunner.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFSourceConfigRunner.java @@ -107,10 +107,26 @@ protected Runnable retrieveLockAndUpdateConfig(final SATIFSourceConfig saTifSour ActionListener.wrap(lock -> { updateSourceConfigAndIOCs(saTifSourceConfig, lockService.getRenewLockRunnable(new AtomicReference<>(lock)), ActionListener.wrap( - r -> lockService.releaseLock(lock), + r -> { + lockService.releaseLock(lock, ActionListener.wrap( + response -> { + log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config [%s].", lock.getLockId(), saTifSourceConfig.getId()), ex); + } + )); + }, e -> { log.error("Failed to update threat intel source config " + saTifSourceConfig.getName(), e); - lockService.releaseLock(lock); + lockService.releaseLock(lock, ActionListener.wrap( + response -> { + log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config [%s].", lock.getLockId(), saTifSourceConfig.getId()), ex); + } + )); } )); }, e -> { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java index 51b909334..2c634ce70 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java @@ -92,7 +92,7 @@ public class SATIFSourceConfig implements TIFSourceConfig, Writeable, ScheduledJ public SATIFSourceConfig(String id, Long version, String name, String format, SourceConfigType type, String description, User createdByUser, Instant createdAt, Source source, Instant enabledTime, Instant lastUpdateTime, Schedule schedule, TIFJobState state, RefreshType refreshType, Instant lastRefreshedTime, User lastRefreshedUser, - Boolean isEnabled, IocStoreConfig iocStoreConfig, List iocTypes, boolean enabledForScan) { + boolean isEnabled, IocStoreConfig iocStoreConfig, List iocTypes, boolean enabledForScan) { this.id = id == null ? UUIDs.base64UUID() : id; this.version = version != null ? version : NO_VERSION; this.name = name; @@ -289,7 +289,7 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio RefreshType refreshType = null; Instant lastRefreshedTime = null; User lastRefreshedUser = null; - Boolean isEnabled = null; + boolean isEnabled = true; boolean enabledForScan = true; IocStoreConfig iocStoreConfig = null; List iocTypes = new ArrayList<>(); @@ -303,16 +303,28 @@ public static SATIFSourceConfig parse(XContentParser xcp, String id, Long versio case SOURCE_CONFIG_FIELD: break; case NAME_FIELD: - name = xcp.text(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + name = null; + } else { + name = xcp.text(); + } break; case VERSION_FIELD: version = xcp.longValue(); break; case FORMAT_FIELD: - format = xcp.text(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + format = null; + } else { + format = xcp.text(); + } break; case TYPE_FIELD: - sourceConfigType = toSourceConfigType(xcp.text()); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + sourceConfigType = null; + } else { + sourceConfigType = toSourceConfigType(xcp.text()); + } break; case DESCRIPTION_FIELD: if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java index ee1dc4f6f..222a345ed 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java @@ -10,11 +10,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchException; +import org.opensearch.OpenSearchStatusException; import org.opensearch.common.UUIDs; import org.opensearch.commons.authuser.User; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -28,6 +31,7 @@ import org.opensearch.securityanalytics.threatIntel.common.RefreshType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.sacommons.TIFSourceConfigDto; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import java.io.IOException; import java.time.Instant; @@ -119,7 +123,7 @@ private List convertToIocDtos(List stix2IocList) { public SATIFSourceConfigDto(String id, Long version, String name, String format, SourceConfigType type, String description, User createdByUser, Instant createdAt, Source source, Instant enabledTime, Instant lastUpdateTime, Schedule schedule, TIFJobState state, RefreshType refreshType, Instant lastRefreshedTime, User lastRefreshedUser, - Boolean isEnabled, List iocTypes, boolean enabledForScan) { + boolean isEnabled, List iocTypes, boolean enabledForScan) { this.id = id == null ? UUIDs.base64UUID() : id; this.version = version != null ? version : NO_VERSION; this.name = name; @@ -211,12 +215,18 @@ public void writeTo(final StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { - builder.startObject() - .startObject(SOURCE_CONFIG_FIELD) - .field(NAME_FIELD, name) - .field(FORMAT_FIELD, format) - .field(TYPE_FIELD, type.name()) - .field(DESCRIPTION_FIELD, description); + builder.startObject(); + innerXcontent(builder); + builder.endObject(); + return builder; + } + + public XContentBuilder innerXcontent(XContentBuilder builder) throws IOException { + builder.startObject(SOURCE_CONFIG_FIELD); + builder.field(NAME_FIELD, name) + .field(FORMAT_FIELD, format) + .field(TYPE_FIELD, type.name()) + .field(DESCRIPTION_FIELD, description); if (createdByUser == null) { builder.nullField(CREATED_BY_USER_FIELD); } else { @@ -270,7 +280,6 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(ENABLED_FOR_SCAN_FIELD, enabledForScan); builder.field(IOC_TYPES_FIELD, iocTypes); builder.endObject(); - builder.endObject(); return builder; } @@ -305,7 +314,7 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver RefreshType refreshType = null; Instant lastRefreshedTime = null; User lastRefreshedUser = null; - Boolean isEnabled = null; + boolean isEnabled = true; List iocTypes = new ArrayList<>(); boolean enabledForScan = true; @@ -317,13 +326,25 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver case SOURCE_CONFIG_FIELD: break; case NAME_FIELD: - name = xcp.text(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + name = null; + } else { + name = xcp.text(); + } break; case FORMAT_FIELD: - format = xcp.text(); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + format = null; + } else { + format = xcp.text(); + } break; case TYPE_FIELD: - sourceConfigType = toSourceConfigType(xcp.text()); + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + sourceConfigType = null; + } else { + sourceConfigType = toSourceConfigType(xcp.text()); + } break; case DESCRIPTION_FIELD: if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { @@ -417,7 +438,6 @@ public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long ver case ENABLED_FIELD: isEnabled = xcp.booleanValue(); break; - case ENABLED_FOR_SCAN_FIELD: enabledForScan = xcp.booleanValue(); break; @@ -467,7 +487,7 @@ public static TIFJobState toState(String stateName) { return TIFJobState.valueOf(stateName); } catch (IllegalArgumentException e) { log.error("Invalid state, cannot be parsed.", e); - return null; + throw SecurityAnalyticsException.wrap(new OpenSearchStatusException("Invalid state, cannot be parsed.", RestStatus.BAD_REQUEST)); } } @@ -476,7 +496,7 @@ public static SourceConfigType toSourceConfigType(String type) { return SourceConfigType.valueOf(type); } catch (IllegalArgumentException e) { log.error("Invalid source config type, cannot be parsed.", e); - return null; + throw SecurityAnalyticsException.wrap(new OpenSearchStatusException("Invalid source config type, cannot be parsed.", RestStatus.BAD_REQUEST)); } } @@ -485,7 +505,7 @@ public static RefreshType toRefreshType(String stateName) { return RefreshType.valueOf(stateName); } catch (IllegalArgumentException e) { log.error("Invalid refresh type, cannot be parsed.", e); - return null; + throw SecurityAnalyticsException.wrap(new OpenSearchStatusException("Invalid refresh type, cannot be parsed.", RestStatus.BAD_REQUEST)); } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/PerIocTypeScanInput.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/PerIocTypeScanInput.java index 91e006aae..902551fd6 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/PerIocTypeScanInput.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/PerIocTypeScanInput.java @@ -58,7 +58,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } public static PerIocTypeScanInput parse(XContentParser xcp) throws IOException { - String iocType = null; + String iocType = ""; Map> indexToFieldsMap = new HashMap<>(); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); @@ -68,7 +68,7 @@ public static PerIocTypeScanInput parse(XContentParser xcp) throws IOException { switch (fieldName) { case IOC_TYPE: - iocType = xcp.text(); + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) iocType = xcp.text(); break; case INDEX_TO_FIELDS_MAP: if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { @@ -78,7 +78,7 @@ public static PerIocTypeScanInput parse(XContentParser xcp) throws IOException { List fields = new ArrayList<>(); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { - fields.add(xcp.text()); + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) fields.add(xcp.text()); } return fields; }); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelTrigger.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelTrigger.java index f467250c0..a2c35f409 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelTrigger.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelTrigger.java @@ -11,9 +11,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; public class ThreatIntelTrigger implements Writeable, ToXContentObject { public static final String DATA_SOURCES = "data_sources"; @@ -64,7 +62,7 @@ public static ThreatIntelTrigger parse(XContentParser xcp) throws IOException { List vals = new ArrayList<>(); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { - vals.add(xcp.text()); + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) vals.add(xcp.text()); } iocTypes.addAll(vals); break; @@ -72,7 +70,7 @@ public static ThreatIntelTrigger parse(XContentParser xcp) throws IOException { List ds = new ArrayList<>(); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { - ds.add(xcp.text()); + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) ds.add(xcp.text()); } dataSources.addAll(ds); break; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java index 6864f7a98..2421e5e5c 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/monitor/TransportThreatIntelMonitorFanOutAction.java @@ -106,7 +106,7 @@ protected void doExecute(Task task, DocLevelMonitorFanOutRequest request, Action iocTypeToIndicesMap -> { onGetIocTypeToIndices(iocTypeToIndicesMap, request, actionListener); }, e -> { - log.error(() -> new ParameterizedMessage("Unexpected Failure in threat intel monitor {} fan out action", request.getMonitor().getId()), e); + log.error(() -> new ParameterizedMessage("Unexpected Failure in threat intel monitor {} fan out action while fetching threat intel ioc indices", request.getMonitor().getId()), e); actionListener.onResponse( new DocLevelMonitorFanOutResponse( clusterService.localNode().getId(), @@ -162,6 +162,20 @@ private void onGetIocTypeToIndices(Map> iocTypeToIndicesMap }; ActionListener> searchHitsListener = ActionListener.wrap( (List hits) -> { + if (hits.isEmpty()) { + actionListener.onResponse( + new DocLevelMonitorFanOutResponse( + clusterService.localNode().getId(), + request.getExecutionId(), + request.getMonitor().getId(), + updatedLastRunContext, + new InputRunResults(Collections.emptyList(), null, null), + Collections.emptyMap(), + null + ) + ); + return; + } BiConsumer resultConsumer = (r, e) -> { if (e == null) { actionListener.onResponse( @@ -195,7 +209,7 @@ private void onGetIocTypeToIndices(Map> iocTypeToIndicesMap ), resultConsumer); }, e -> { - log.error("unexpected error while", e); + log.error("unexpected error while trying to query shards and fetch docs before scanning for malicious IoC's", e); actionListener.onFailure(e); } ); @@ -290,6 +304,11 @@ private void fetchLatestDocsFromShard( // recursive call to fetch docs with updated seq no. fetchLatestDocsFromShard(shardId, fromSeqNo, updatedToSeqNo, searchHitsSoFar, monitor, shardLastSeenMapForIndex, updateLastRunContext, fieldsToFetch, listener); }, e -> { + if(e.getMessage().contains("all shards failed") && e.getCause().getMessage().contains("No mapping found for [_seq_no] in order to sort on")) { + // this implies that the index being queried doesn't have any docs and hence doesn't understand the in-built _seq_no field mapping + listener.onResponse(new SearchHitsOrException(Collections.emptyList(), null)); + return; + } log.error(() -> new ParameterizedMessage("Threat intel Monitor {}: Failed to search shard {} in index {}", monitor.getId(), shard, shardId.getIndexName()), e); listener.onResponse(new SearchHitsOrException(searchHitsSoFar, e)); } diff --git a/src/main/java/org/opensearch/securityanalytics/resthandler/RestListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestListIOCsAction.java similarity index 90% rename from src/main/java/org/opensearch/securityanalytics/resthandler/RestListIOCsAction.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestListIOCsAction.java index f068be77c..e40aa6f71 100644 --- a/src/main/java/org/opensearch/securityanalytics/resthandler/RestListIOCsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/RestListIOCsAction.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.resthandler; +package org.opensearch.securityanalytics.threatIntel.resthandler; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -18,9 +18,9 @@ import org.opensearch.rest.RestResponse; import org.opensearch.rest.action.RestResponseListener; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; -import org.opensearch.securityanalytics.action.ListIOCsAction; -import org.opensearch.securityanalytics.action.ListIOCsActionRequest; -import org.opensearch.securityanalytics.action.ListIOCsActionResponse; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsAction; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionRequest; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionResponse; import java.io.IOException; import java.util.List; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestIndexThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestIndexThreatIntelMonitorAction.java index 21a725952..71485931b 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestIndexThreatIntelMonitorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestIndexThreatIntelMonitorAction.java @@ -18,6 +18,7 @@ import org.opensearch.securityanalytics.threatIntel.action.monitor.request.IndexThreatIntelMonitorRequest; import org.opensearch.securityanalytics.threatIntel.action.monitor.response.IndexThreatIntelMonitorResponse; import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import java.io.IOException; import java.util.List; @@ -50,7 +51,13 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli XContentParser xcp = request.contentParser(); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.nextToken(), xcp); - ThreatIntelMonitorDto iocScanMonitor = ThreatIntelMonitorDto.parse(xcp, id, null); + ThreatIntelMonitorDto iocScanMonitor; + try { + iocScanMonitor = ThreatIntelMonitorDto.parse(xcp, id, null); + } catch (Exception e) { + log.error("Failed to parse threat intel monitor: ", e); + throw new SecurityAnalyticsException("Failed to parse threat intel monitor: ", RestStatus.BAD_REQUEST, e); + } IndexThreatIntelMonitorRequest indexThreatIntelMonitorRequest = new IndexThreatIntelMonitorRequest(id, request.method(), iocScanMonitor); return channel -> client.execute(IndexThreatIntelMonitorAction.INSTANCE, indexThreatIntelMonitorRequest, getListener(channel, request.method())); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestUpdateThreatIntelAlertsStatusAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestUpdateThreatIntelAlertsStatusAction.java index 1fc333c21..5300dd8f7 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestUpdateThreatIntelAlertsStatusAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/resthandler/monitor/RestUpdateThreatIntelAlertsStatusAction.java @@ -1,29 +1,21 @@ package org.opensearch.securityanalytics.threatIntel.resthandler.monitor; +import org.apache.commons.lang3.StringUtils; import org.opensearch.client.node.NodeClient; import org.opensearch.commons.alerting.model.Alert; import org.opensearch.core.common.Strings; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestToXContentListener; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; -import org.opensearch.securityanalytics.action.AckAlertsAction; -import org.opensearch.securityanalytics.action.AckAlertsRequest; -import org.opensearch.securityanalytics.action.ListIOCsActionRequest; -import org.opensearch.securityanalytics.threatIntel.action.monitor.GetThreatIntelAlertsAction; import org.opensearch.securityanalytics.threatIntel.action.monitor.UpdateThreatIntelAlertStatusAction; import org.opensearch.securityanalytics.threatIntel.action.monitor.request.UpdateThreatIntelAlertStatusRequest; -import org.opensearch.securityanalytics.util.DetectorUtils; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Locale; -import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; - /** * Update status of list of threat intel alerts * Supported state to udpate to : ACKNOWLEDGED, COMPLETED @@ -48,6 +40,10 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { String state = request.param("state"); + if (StringUtils.isBlank(state)) { + throw new IllegalArgumentException("State param is required."); + } + Alert.State alertState = Alert.State.valueOf(state.toUpperCase()); List alertIds = List.of( Strings.commaDelimitedListToStringArray( @@ -59,5 +55,4 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli new RestToXContentListener<>(channel) ); } - } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelTriggerDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelTriggerDto.java index d82381b3d..f43512d40 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelTriggerDto.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/monitor/ThreatIntelTriggerDto.java @@ -81,9 +81,9 @@ public static ThreatIntelTriggerDto parse(XContentParser xcp) throws IOException List iocTypes = new ArrayList<>(); List dataSources = new ArrayList<>(); List actions = new ArrayList<>(); - String name = null; + String name = ""; String id = null; - String severity = null; + String severity = ""; XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { String fieldName = xcp.currentName(); @@ -94,7 +94,7 @@ public static ThreatIntelTriggerDto parse(XContentParser xcp) throws IOException List vals = new ArrayList<>(); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { - vals.add(xcp.text()); + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) vals.add(xcp.text()); } iocTypes.addAll(vals); break; @@ -102,7 +102,7 @@ public static ThreatIntelTriggerDto parse(XContentParser xcp) throws IOException List ds = new ArrayList<>(); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { - ds.add(xcp.text()); + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) ds.add(xcp.text()); } dataSources.addAll(ds); break; @@ -124,10 +124,10 @@ public static ThreatIntelTriggerDto parse(XContentParser xcp) throws IOException id = xcp.text(); break; case NAME_FIELD: - name = xcp.text(); + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) name = xcp.text(); break; case SEVERITY_FIELD: - severity = xcp.text(); + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) severity = xcp.text(); break; default: xcp.skipChildren(); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java index ed160bbf1..c247109d6 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DefaultTifSourceConfigLoaderService.java @@ -6,6 +6,7 @@ import org.opensearch.action.support.GroupedActionListener; import org.opensearch.client.Client; import org.opensearch.core.action.ActionListener; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.MatchQueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -21,6 +22,7 @@ import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; +import org.opensearch.transport.RemoteTransportException; import java.net.URL; import java.time.Instant; @@ -31,8 +33,11 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.opensearch.securityanalytics.util.DetectorUtils.getEmptySearchResponse; + //todo handle refresh, update tif config // todo block creation of url based config in transport layer + public class DefaultTifSourceConfigLoaderService { private static final Logger log = LogManager.getLogger(DefaultTifSourceConfigLoaderService.class); private final BuiltInTIFMetadataLoader tifMetadataLoader; @@ -64,8 +69,12 @@ public void createDefaultTifConfigsIfNotExists(ActionListener listener) { ActionListener.wrap(searchResponse -> { createTifConfigsThatDontExist(searchResponse, tifMetadataList, listener); }, e -> { - log.error("Failed to search tif config index for default tif configs", e); - listener.onFailure(e); + if (e instanceof IndexNotFoundException || (e instanceof RemoteTransportException && e.getCause() instanceof IndexNotFoundException)) { + createTifConfigsThatDontExist(getEmptySearchResponse(), tifMetadataList, listener); + } else { + log.error("Failed to search tif config index for default tif configs", e); + listener.onFailure(e); + } })); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java index 9c8f81025..815729f40 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java @@ -27,11 +27,11 @@ import org.opensearch.rest.RestRequest; import org.opensearch.search.SearchHit; import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.model.STIX2IOC; import org.opensearch.securityanalytics.model.STIX2IOCDto; import org.opensearch.securityanalytics.services.STIX2IOCFetchService; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; @@ -39,6 +39,7 @@ import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import java.time.Instant; @@ -160,16 +161,17 @@ public void createIocAndTIFSourceConfig( )); }, e -> { - log.error("Failed to download and save IOCs for threat intel source config [{}]", indexSaTifSourceConfigResponse.getId()); - saTifSourceConfigService.deleteTIFSourceConfig(indexSaTifSourceConfigResponse, ActionListener.wrap( + log.error("Failed to download and save IOCs for threat intel source config [{}]", indexSaTifSourceConfigResponse.getId(), e); + // set isDeleted as true because we want to delete failed source configs regardless if threat intel monitor exists + deleteAllIocsAndSourceConfig(ActionListener.wrap( deleteResponse -> { log.debug("Successfully deleted threat intel source config [{}]", indexSaTifSourceConfigResponse.getId()); listener.onFailure(e); }, ex -> { - log.error("Failed to delete threat intel source config [{}]", indexSaTifSourceConfigResponse.getId()); + log.error("Failed to delete threat intel source config [{}]", indexSaTifSourceConfigResponse.getId(), ex); listener.onFailure(ex); } - )); + ), indexSaTifSourceConfigResponse, true); }) ); }, e -> { @@ -275,6 +277,52 @@ public void updateIocAndTIFSourceConfig( try { saTifSourceConfigService.getTIFSourceConfig(saTifSourceConfigDto.getId(), ActionListener.wrap( retrievedSaTifSourceConfig -> { + // Due to the lack of a different API to do activate/deactivate we will check if enabled_for_scan variable is changed between model and request. + // If yes, we will ONLY update enabled_for_scan field and ignore any updates to the rest of the fields to simulate a dedicated activate/deactivate API. + if (retrievedSaTifSourceConfig.isEnabledForScan() != saTifSourceConfigDto.isEnabledForScan()) { + // FIXME add a disable_refresh api independent of update api so that it can be supported for default configs also + boolean isEnabled = URL_DOWNLOAD.equals(retrievedSaTifSourceConfig.getType()) ? + saTifSourceConfigDto.isEnabledForScan() : + retrievedSaTifSourceConfig.isEnabled(); + SATIFSourceConfig config = new SATIFSourceConfig( + retrievedSaTifSourceConfig.getId(), + retrievedSaTifSourceConfig.getVersion(), + retrievedSaTifSourceConfig.getName(), + retrievedSaTifSourceConfig.getFormat(), + retrievedSaTifSourceConfig.getType(), + retrievedSaTifSourceConfig.getDescription(), + retrievedSaTifSourceConfig.getCreatedByUser(), + retrievedSaTifSourceConfig.getCreatedAt(), + retrievedSaTifSourceConfig.getSource(), + retrievedSaTifSourceConfig.getEnabledTime(), + retrievedSaTifSourceConfig.getLastUpdateTime(), + retrievedSaTifSourceConfig.getSchedule(), + retrievedSaTifSourceConfig.getState(), + retrievedSaTifSourceConfig.getRefreshType(), + Instant.now(), + updatedByUser, + isEnabled, + retrievedSaTifSourceConfig.getIocStoreConfig(), + retrievedSaTifSourceConfig.getIocTypes(), + saTifSourceConfigDto.isEnabledForScan() // update only enabled_for_scan + ); + internalUpdateTIFSourceConfig(config, ActionListener.wrap( + r -> { + listener.onResponse(new SATIFSourceConfigDto(r)); + }, e -> { + String action = saTifSourceConfigDto.isEnabledForScan() ? "activate" : "deactivate"; + log.error(String.format("Failed to %s tif source config %s", action, retrievedSaTifSourceConfig.getId()), e); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchException(String.format("Failed to %s tif source config %s", action, retrievedSaTifSourceConfig.getId()), e))); + return; + } + )); + return; + } else if (SourceConfigType.URL_DOWNLOAD.equals(saTifSourceConfigDto.getType()) || saTifSourceConfigDto.getSource() instanceof UrlDownloadSource) { // fail if enabled_for_scan isn't changed and type is url download + log.error("Unsupported Threat intel Source Config Type passed - " + saTifSourceConfigDto.getType()); + listener.onFailure(new UnsupportedOperationException("Unsupported Threat intel Source Config Type passed - " + saTifSourceConfigDto.getType())); + return; + } + if (TIFJobState.AVAILABLE.equals(retrievedSaTifSourceConfig.getState()) == false && TIFJobState.REFRESH_FAILED.equals(retrievedSaTifSourceConfig.getState()) == false) { log.error("Invalid threat intel source config state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState()); listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException( @@ -311,14 +359,10 @@ public void updateIocAndTIFSourceConfig( log.info("Set threat intel source config as REFRESHING for [{}]", updatedSaTifSourceConfig.getId()); switch (updatedSaTifSourceConfig.getType()) { case S3_CUSTOM: - downloadAndSaveIocsToRefresh(listener, updatedSaTifSourceConfig); + downloadAndSaveIocsToRefresh(listener, updatedSaTifSourceConfig, null); break; case IOC_UPLOAD: - storeAndDeleteIocIndices( - iocs, - listener, - updatedSaTifSourceConfig - ); + downloadAndSaveIocsToRefresh(listener, updatedSaTifSourceConfig, iocs); break; } }, e -> { @@ -337,78 +381,6 @@ public void updateIocAndTIFSourceConfig( } } - private void storeAndDeleteIocIndices(List stix2IOCList, ActionListener listener, SATIFSourceConfig updatedSaTifSourceConfig) { - // Index the new iocs - downloadAndSaveIOCs(updatedSaTifSourceConfig, stix2IOCList, ActionListener.wrap( - downloadAndSaveIocsResponse -> { - - Set iocIndexPatterns = new HashSet<>(); - if (updatedSaTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) { - // get all the index patterns - DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) updatedSaTifSourceConfig.getIocStoreConfig(); - defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> iocIndexPatterns.add(e.getIndexPattern())); - } - - saTifSourceConfigService.getClusterState(ActionListener.wrap( - clusterStateResponse -> { - IocStoreConfig iocStoreConfig = updatedSaTifSourceConfig.getIocStoreConfig(); - Set activeIndices = new HashSet<>(); - Set indicesToDelete = new HashSet<>(); - - if (iocStoreConfig instanceof DefaultIocStoreConfig) { - DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) iocStoreConfig; - Set concreteIndices = SATIFSourceConfigService.getConcreteIndices(clusterStateResponse); - - // remove ioc types not specified in list - defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> !IOCType.supportedType(iocToIndexDetails.getIocType().toString())); - - // get the active indices - defaultIocStoreConfig.getIocToIndexDetails().forEach(e -> activeIndices.add(e.getActiveIndex())); - - for (String index : concreteIndices) { - if (false == activeIndices.contains(index)) { - indicesToDelete.add(index); - } - } - } - - // delete the old indices - saTifSourceConfigService.deleteAllIocIndices(indicesToDelete, true, null); - markSourceConfigAsAction( - updatedSaTifSourceConfig, - TIFJobState.AVAILABLE, - ActionListener.wrap( - saTifSourceConfigResponse -> { - SATIFSourceConfigDto returnedSaTifSourceConfigDto = new SATIFSourceConfigDto(saTifSourceConfigResponse); - listener.onResponse(returnedSaTifSourceConfigDto); - }, e -> { - log.error("Failed to index threat intel source config with id [{}]", updatedSaTifSourceConfig.getId()); - listener.onFailure(e); - } - )); - }, e -> { - log.error("Failed to get the cluster metadata"); - listener.onFailure(e); - } - ), iocIndexPatterns.toArray(new String[0])); - }, - e -> { - log.error("Failed to download and save IOCs for threat intel source config [{}]", updatedSaTifSourceConfig.getId(), e); - markSourceConfigAsAction(updatedSaTifSourceConfig, TIFJobState.REFRESH_FAILED, ActionListener.wrap( - r -> { - log.info("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId()); - listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchException( - String.format(Locale.getDefault(), "Failed to download and save IOCs for threat intel source config [%s]. Set source config as REFRESH_FAILED", updatedSaTifSourceConfig.getId()), - e))); - }, ex -> { - log.error("Failed to set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId()); - listener.onFailure(ex); - } - )); - }) - ); - } - public void internalUpdateTIFSourceConfig( final SATIFSourceConfig saTifSourceConfig, final ActionListener listener @@ -455,7 +427,7 @@ public void refreshTIFSourceConfig( saTifSourceConfig.setLastRefreshedTime(Instant.now()); markSourceConfigAsAction(saTifSourceConfig, TIFJobState.REFRESHING, ActionListener.wrap( updatedSourceConfig -> { - downloadAndSaveIocsToRefresh(listener, updatedSourceConfig); + downloadAndSaveIocsToRefresh(listener, updatedSourceConfig, null); }, e -> { log.error("Failed to set threat intel source config as REFRESHING for [{}]", saTifSourceConfig.getId()); listener.onFailure(e); @@ -468,19 +440,12 @@ public void refreshTIFSourceConfig( )); } - private void downloadAndSaveIocsToRefresh(ActionListener listener, SATIFSourceConfig updatedSourceConfig) { - downloadAndSaveIOCs(updatedSourceConfig, null, ActionListener.wrap( + private void downloadAndSaveIocsToRefresh(ActionListener listener, SATIFSourceConfig updatedSourceConfig, List stix2IOCList) { + downloadAndSaveIOCs(updatedSourceConfig, stix2IOCList, ActionListener.wrap( response -> { // delete old IOCs and update the source config deleteOldIocIndices(updatedSourceConfig, ActionListener.wrap( newIocStoreConfig -> { - List iocTypes = updatedSourceConfig.getIocTypes(); - if (newIocStoreConfig instanceof DefaultIocStoreConfig) { - DefaultIocStoreConfig defaultIocStoreConfig = (DefaultIocStoreConfig) newIocStoreConfig; - // remove ioc types not specified in list - defaultIocStoreConfig.getIocToIndexDetails().removeIf(iocToIndexDetails -> !IOCType.supportedType(iocToIndexDetails.getIocType().toString())); - updatedSourceConfig.setIocStoreConfig(defaultIocStoreConfig); - } // Update source config as succeeded, change state back to available markSourceConfigAsAction(updatedSourceConfig, TIFJobState.AVAILABLE, ActionListener.wrap( r -> { @@ -532,7 +497,7 @@ public void deleteTIFSourceConfig( // Check if all threat intel monitors are deleted saTifSourceConfigService.checkAndEnsureThreatIntelMonitorsDeleted(ActionListener.wrap( isDeleted -> { - deleteAllIocsAndSourceConfig(saTifSourceConfigId, listener, saTifSourceConfig, isDeleted); + deleteAllIocsAndSourceConfig(listener, saTifSourceConfig, isDeleted); }, e -> { log.error("Failed to check if all threat intel monitors are deleted or if multiple threat intel source configs exist"); listener.onFailure(e); @@ -541,7 +506,7 @@ public void deleteTIFSourceConfig( }, e -> { log.error("Failed to get threat intel source config for [{}]", saTifSourceConfigId); if (e instanceof IndexNotFoundException) { - listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"Threat intel source config [%s] not found.", saTifSourceConfigId), RestStatus.NOT_FOUND))); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Threat intel source config [%s] not found.", saTifSourceConfigId), RestStatus.NOT_FOUND))); } else { listener.onFailure(e); } @@ -689,11 +654,11 @@ private Integer numOfIndicesToDelete(Integer totalNumIndices, Integer totalNumIn return 0; } - private void deleteAllIocsAndSourceConfig(String saTifSourceConfigId, ActionListener listener, SATIFSourceConfig saTifSourceConfig, Boolean isDeleted) { + private void deleteAllIocsAndSourceConfig(ActionListener listener, SATIFSourceConfig saTifSourceConfig, Boolean isDeleted) { if (isDeleted == false) { listener.onFailure(new IllegalArgumentException("All threat intel monitors need to be deleted before deleting last threat intel source config")); } else { - log.debug("All threat intel monitors are deleted or multiple threat intel source configs exist, can delete threat intel source config [{}]", saTifSourceConfigId); + log.debug("All threat intel monitors are deleted or multiple threat intel source configs exist, can delete threat intel source config [{}]", saTifSourceConfig.getId()); markSourceConfigAsAction( saTifSourceConfig, TIFJobState.DELETING, @@ -707,16 +672,28 @@ private void deleteAllIocsAndSourceConfig(String saTifSourceConfigId, ActionList } saTifSourceConfigService.getClusterState(ActionListener.wrap( clusterStateResponse -> { - Set concreteIndices = SATIFSourceConfigService.getConcreteIndices(clusterStateResponse); + Set concreteIndices; + if (false == iocIndexPatterns.isEmpty()) { + concreteIndices = SATIFSourceConfigService.getConcreteIndices(clusterStateResponse); + } else { + concreteIndices = new HashSet<>(); + } saTifSourceConfigService.deleteAllIocIndices(concreteIndices, false, ActionListener.wrap( r -> { log.debug("Successfully deleted all ioc indices"); - saTifSourceConfigService.deleteTIFSourceConfig(updateSaTifSourceConfigResponse, ActionListener.wrap( - deleteResponse -> { - log.debug("Successfully deleted threat intel source config [{}]", updateSaTifSourceConfigResponse.getId()); - listener.onResponse(deleteResponse); + saTifSourceConfigService.deleteJobSchedulerLockIfJobDisabled(updateSaTifSourceConfigResponse, ActionListener.wrap( + deleteLockResponse -> { + saTifSourceConfigService.deleteTIFSourceConfig(updateSaTifSourceConfigResponse, ActionListener.wrap( + deleteResponse -> { + log.debug("Successfully deleted threat intel source config [{}]", updateSaTifSourceConfigResponse.getId()); + listener.onResponse(deleteResponse); + }, e -> { + log.error("Failed to delete threat intel source config [{}]", saTifSourceConfig.getId()); + listener.onFailure(e); + } + )); }, e -> { - log.error("Failed to delete threat intel source config [{}]", saTifSourceConfigId); + log.error("Failed to delete threat intel job scheduler lock [{}]", saTifSourceConfig.getId()); listener.onFailure(e); } )); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java index 35571facd..dee9ae013 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -50,10 +50,10 @@ import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.threatIntel.action.monitor.SearchThreatIntelMonitorAction; import org.opensearch.securityanalytics.threatIntel.action.monitor.request.SearchThreatIntelMonitorRequest; -import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.util.IndexUtils; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.threadpool.ThreadPool; @@ -72,6 +72,7 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.opensearch.jobscheduler.spi.utils.LockService.LOCK_INDEX_NAME; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.INDEX_TIMEOUT; import static org.opensearch.securityanalytics.threatIntel.common.TIFJobState.AVAILABLE; import static org.opensearch.securityanalytics.threatIntel.common.TIFJobState.REFRESHING; @@ -79,6 +80,7 @@ import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.SOURCE_CONFIG_FIELD; import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.STATE_FIELD; import static org.opensearch.securityanalytics.transport.TransportIndexDetectorAction.PLUGIN_OWNER_FIELD; +import static org.opensearch.securityanalytics.util.IndexUtils.shouldUpdateIndex; /** * CRUD for threat intel feeds source config object @@ -92,7 +94,6 @@ public class SATIFSourceConfigService { private final NamedXContentRegistry xContentRegistry; private final TIFLockService lockService; - public SATIFSourceConfigService(final Client client, final ClusterService clusterService, ThreadPool threadPool, @@ -137,9 +138,17 @@ public void indexTIFSourceConfig(SATIFSourceConfig saTifSourceConfig, actionListener.onFailure(e); } }, exception -> { - lockService.releaseLock(lock); - log.error("Failed to release lock", exception); - actionListener.onFailure(exception); + log.error("Failed to create threat intel source config index", exception); + lockService.releaseLock(lock, ActionListener.wrap( + r -> { + log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); + actionListener.onFailure(exception); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for threat intel source config.", lock.getLockId()), ex); + actionListener.onFailure(exception); + } + )); }); createJobIndexIfNotExists(createIndexStepListener); } @@ -179,7 +188,7 @@ private String getIndexMapping() { } } catch (IOException e) { log.error("Failed to get the threat intel index mapping", e); - throw new SecurityAnalyticsException("Failed to get threat intel index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); + throw new SecurityAnalyticsException("Failed to get threat intel index mapping", RestStatus.BAD_REQUEST, e); } } @@ -188,33 +197,75 @@ private String getIndexMapping() { /** * Index name: .opensearch-sap--job * Mapping: /mappings/threat_intel_job_mapping.json + * Updates the job index mapping if currently on a previous version * * @param stepListener setup listener */ public void createJobIndexIfNotExists(final StepListener stepListener) { // check if job index exists if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == true) { - stepListener.onResponse(null); - return; - } - final CreateIndexRequest createIndexRequest = new CreateIndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME).mapping(getIndexMapping()) - .settings(SecurityAnalyticsPlugin.TIF_JOB_INDEX_SETTING); - StashedThreadContext.run(client, () -> client.admin().indices().create(createIndexRequest, ActionListener.wrap( - r -> { - log.debug("[{}] index created", SecurityAnalyticsPlugin.JOB_INDEX_NAME); - stepListener.onResponse(null); - }, e -> { - if (e instanceof ResourceAlreadyExistsException) { - log.info("Index [{}] already exists", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + checkAndUpdateJobIndexMapping(stepListener); + } else { + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME).mapping(getIndexMapping()) + .settings(SecurityAnalyticsPlugin.TIF_JOB_INDEX_SETTING); + client.admin().indices().create(createIndexRequest, ActionListener.wrap( + r -> { + log.debug("[{}] index created", SecurityAnalyticsPlugin.JOB_INDEX_NAME); stepListener.onResponse(null); - return; + }, e -> { + if (e instanceof ResourceAlreadyExistsException) { + log.info("Index [{}] already exists", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + stepListener.onResponse(null); + return; + } + log.error("Failed to create [{}] index", SecurityAnalyticsPlugin.JOB_INDEX_NAME, e); + stepListener.onFailure(e); } - log.error("Failed to create [{}] index", SecurityAnalyticsPlugin.JOB_INDEX_NAME, e); - stepListener.onFailure(e); - } - ))); + )); + } } + private void checkAndUpdateJobIndexMapping(StepListener stepListener) { + try { + // Check if job index contains old mapping, if so update index mapping (current version = 2) + if (shouldUpdateIndex(clusterService.state().metadata().index(SecurityAnalyticsPlugin.JOB_INDEX_NAME), getIndexMapping())) { + log.info("Old schema version found for [{}] index, updating the index mapping", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + IndexUtils.updateIndexMapping( + SecurityAnalyticsPlugin.JOB_INDEX_NAME, + getIndexMapping(), clusterService.state(), client.admin().indices(), + ActionListener.wrap( + r -> { + log.info("Successfully updated index mapping for [{}] index", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + stepListener.onResponse(null); + }, e -> { + // Check if version is updated despite failure + try { + // Check if job index still contains older mapping + if (shouldUpdateIndex(clusterService.state().metadata().index(SecurityAnalyticsPlugin.JOB_INDEX_NAME), getIndexMapping())) { + log.error("Job index still contains older mapping, failed to update job index mapping", e); + stepListener.onFailure(e); + } else { + // If job index contains newest mapping, then return success + log.info("Successfully updated index mapping for [{}] index", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + stepListener.onResponse(null); + } + } catch (IOException exception) { + log.error("Failed to check if job index contains older mapping. Failed to update job index mapping", e); + stepListener.onFailure(e); + } + } + ), + false + ); + } else { + // If job index contains newest mapping, then do nothing + stepListener.onResponse(null); + } + } catch (IOException e) { + log.error("Failed to check and update job index mapping", e); + stepListener.onFailure(e); + } + } // Get TIF source config public void getTIFSourceConfig( @@ -225,7 +276,7 @@ public void getTIFSourceConfig( client.get(getRequest, ActionListener.wrap( getResponse -> { if (!getResponse.isExists()) { - actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"Threat intel source config [%s] not found.", tifSourceConfigId), RestStatus.NOT_FOUND))); + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "Threat intel source config [%s] not found.", tifSourceConfigId), RestStatus.NOT_FOUND))); return; } SATIFSourceConfig saTifSourceConfig = null; @@ -237,7 +288,7 @@ public void getTIFSourceConfig( saTifSourceConfig = SATIFSourceConfig.docParse(xcp, getResponse.getId(), getResponse.getVersion()); } if (saTifSourceConfig == null) { - actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"No threat intel source config exists [%s]", tifSourceConfigId), RestStatus.BAD_REQUEST))); + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "No threat intel source config exists [%s]", tifSourceConfigId), RestStatus.BAD_REQUEST))); } else { log.debug("Threat intel source config with id [{}] fetched", getResponse.getId()); actionListener.onResponse(saTifSourceConfig); @@ -255,12 +306,6 @@ public void searchTIFSourceConfigs( ) { SearchRequest searchRequest = getSearchRequest(searchSourceBuilder); - // Check to make sure the job index exists - if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == false) { - actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config index does not exist", RestStatus.BAD_REQUEST))); - return; - } - client.search(searchRequest, ActionListener.wrap( searchResponse -> { if (searchResponse.isTimedOut()) { @@ -360,12 +405,12 @@ public void deleteTIFSourceConfig( client.delete(request, ActionListener.wrap( deleteResponse -> { if (deleteResponse.status().equals(RestStatus.OK)) { - log.debug("Deleted threat intel source config [{}] successfully", saTifSourceConfig.getId()); + log.info("Deleted threat intel source config [{}] successfully", saTifSourceConfig.getId()); actionListener.onResponse(deleteResponse); } else if (deleteResponse.status().equals(RestStatus.NOT_FOUND)) { - actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Threat intel source config with id [{%s}] not found", saTifSourceConfig.getId()), RestStatus.NOT_FOUND))); + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "Threat intel source config with id [{%s}] not found", saTifSourceConfig.getId()), RestStatus.NOT_FOUND))); } else { - actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Failed to delete threat intel source config [{%s}]", saTifSourceConfig.getId()), deleteResponse.status()))); + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "Failed to delete threat intel source config [{%s}]", saTifSourceConfig.getId()), deleteResponse.status()))); } }, e -> { log.error("Failed to delete threat intel source config with id [{}]", saTifSourceConfig.getId()); @@ -374,6 +419,45 @@ public void deleteTIFSourceConfig( )); } + // Manually delete threat intel job scheduler lock if job is disabled + public void deleteJobSchedulerLockIfJobDisabled( + SATIFSourceConfig saTifSourceConfig, + final ActionListener actionListener + ) { + if (saTifSourceConfig.isEnabled()) { + actionListener.onResponse(null); + return; + } + + // check to make sure the job scheduler lock index exists + if (clusterService.state().metadata().hasIndex(LOCK_INDEX_NAME) == false) { + actionListener.onResponse(null); + return; + } + + String id = SecurityAnalyticsPlugin.JOB_INDEX_NAME + "-" + saTifSourceConfig.getId(); + DeleteRequest request = new DeleteRequest(LOCK_INDEX_NAME, id) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .timeout(clusterSettings.get(INDEX_TIMEOUT)); + + client.delete(request, ActionListener.wrap( + deleteResponse -> { + if (deleteResponse.status().equals(RestStatus.OK)) { + log.info("Deleted threat intel job scheduler lock [{}] successfully", id); + actionListener.onResponse(deleteResponse); + } else if (deleteResponse.status().equals(RestStatus.NOT_FOUND)) { + log.info("Threat intel job scheduler lock with id [{}] not found", id); + actionListener.onResponse(deleteResponse); + } else { + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "Failed to delete threat intel job scheduler lock with id [{%s}]", id), deleteResponse.status()))); + } + }, e -> { + log.error("Failed to delete threat intel job scheduler lock with id [{}]", id); + actionListener.onFailure(e); + } + )); + } + public void deleteAllIocIndices(Set indicesToDelete, Boolean backgroundJob, ActionListener listener) { if (indicesToDelete.isEmpty() == false) { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indicesToDelete.toArray(new String[0])); @@ -396,6 +480,8 @@ public void deleteAllIocIndices(Set indicesToDelete, Boolean backgroundJ } ) ); + } else if (listener != null) { + listener.onResponse(new AcknowledgedResponse(true)); } } @@ -409,7 +495,7 @@ private void deleteIocIndex(Set indicesToDelete, Boolean backgroundJob, if (!response.isAcknowledged()) { log.error("Could not delete one or more IOC indices: " + index); if (backgroundJob == false) { - listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Could not delete one or more IOC indices: " + index), RestStatus.INTERNAL_SERVER_ERROR))); + listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.ROOT, "Could not delete one or more IOC indices: " + index), RestStatus.BAD_REQUEST))); } } else { log.debug("Successfully deleted one or more IOC indices:" + index); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportDeleteTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportDeleteTIFSourceConfigAction.java index 4234c6592..58b1e5bc1 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportDeleteTIFSourceConfigAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportDeleteTIFSourceConfigAction.java @@ -2,34 +2,57 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.authuser.User; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.action.SADeleteTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.action.SADeleteTIFSourceConfigRequest; import org.opensearch.securityanalytics.threatIntel.action.SADeleteTIFSourceConfigResponse; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; import org.opensearch.securityanalytics.transport.SecureTransportAction; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.tasks.Task; +import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; public class TransportDeleteTIFSourceConfigAction extends HandledTransportAction implements SecureTransportAction { private static final Logger log = LogManager.getLogger(TransportDeleteTIFSourceConfigAction.class); + private final Settings settings; + private final ThreadPool threadPool; + private volatile Boolean filterByEnabled; private final SATIFSourceConfigManagementService saTifConfigService; @Inject public TransportDeleteTIFSourceConfigAction(TransportService transportService, ActionFilters actionFilters, + Settings settings, + final ThreadPool threadPool, final SATIFSourceConfigManagementService saTifConfigService) { super(SADeleteTIFSourceConfigAction.NAME, transportService, actionFilters, SADeleteTIFSourceConfigRequest::new); + this.settings = settings; + this.threadPool = threadPool; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); this.saTifConfigService = saTifConfigService; } @Override protected void doExecute(Task task, SADeleteTIFSourceConfigRequest request, ActionListener actionListener) { + User user = readUserFromThreadContext(this.threadPool); + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(validateBackendRoleMessage, RestStatus.FORBIDDEN))); + return; + } + this.threadPool.getThreadContext().stashContext(); + saTifConfigService.deleteTIFSourceConfig(request.getId(), ActionListener.wrap( response -> actionListener.onResponse( new SADeleteTIFSourceConfigResponse( diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java index c6ab88435..4e6d2f349 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetIocFindingsAction.java @@ -90,6 +90,8 @@ protected void doExecute(Task task, GetIocFindingsRequest request, ActionListene actionListener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); return; } + this.threadPool.getThreadContext().stashContext(); + Table tableProp = request.getTable(); FieldSortBuilder sortBuilder = SortBuilders .fieldSort(tableProp.getSortString()) @@ -118,7 +120,8 @@ protected void doExecute(Task task, GetIocFindingsRequest request, ActionListene List iocIds = request.getIocIds(); if (iocIds != null && !iocIds.isEmpty()) { BoolQueryBuilder iocIdQueryBuilder = QueryBuilders.boolQuery(); - iocIds.forEach(it -> iocIdQueryBuilder.should(QueryBuilders.matchQuery("ioc_feed_ids.ioc_id", it))); + // can't use match query because it analyzes the value and considers `hyphens` as word separators + iocIds.forEach(it -> iocIdQueryBuilder.should(QueryBuilders.matchPhraseQuery("ioc_feed_ids.ioc_id", it))); queryBuilder.filter(iocIdQueryBuilder); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetTIFSourceConfigAction.java index 240748cd0..51c5a6ad2 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetTIFSourceConfigAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetTIFSourceConfigAction.java @@ -60,7 +60,6 @@ protected void doExecute(Task task, SAGetTIFSourceConfigRequest request, ActionL actionListener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); return; } - this.threadPool.getThreadContext().stashContext(); saTifConfigService.getTIFSourceConfig(request.getId(), ActionListener.wrap( diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java index 9b6378cf9..afd22f799 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportIndexTIFSourceConfigAction.java @@ -20,10 +20,8 @@ import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigRequest; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigResponse; -import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; -import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; import org.opensearch.securityanalytics.transport.SecureTransportAction; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; @@ -82,6 +80,8 @@ protected void doExecute(final Task task, final SAIndexTIFSourceConfigRequest re listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(validateBackendRoleMessage, RestStatus.FORBIDDEN))); return; } + this.threadPool.getThreadContext().stashContext(); + retrieveLockAndCreateTIFConfig(request, listener, user); } @@ -89,19 +89,14 @@ private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest reques try { lockService.acquireLock(request.getTIFConfigDto().getId(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { if (lock == null) { + log.error("another processor is a lock, BAD_REQUEST error", RestStatus.BAD_REQUEST); listener.onFailure( new ConcurrentModificationException("another processor is holding a lock on the resource. Try again later") ); - log.error("another processor is a lock, BAD_REQUEST error", RestStatus.BAD_REQUEST); return; } try { SATIFSourceConfigDto saTifSourceConfigDto = request.getTIFConfigDto(); - if (SourceConfigType.URL_DOWNLOAD.equals(saTifSourceConfigDto.getType()) || saTifSourceConfigDto.getSource() instanceof UrlDownloadSource - && request.getMethod().equals(RestRequest.Method.POST)) { - listener.onFailure(new UnsupportedOperationException("Unsupported Threat intel Source Config Type passed - " + saTifSourceConfigDto.getType())); - return; - } saTifSourceConfigManagementService.createOrUpdateTifSourceConfig( saTifSourceConfigDto, lock, @@ -109,29 +104,61 @@ private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest reques user, ActionListener.wrap( saTifSourceConfigDtoResponse -> { - lockService.releaseLock(lock); - listener.onResponse(new SAIndexTIFSourceConfigResponse( - saTifSourceConfigDtoResponse.getId(), - saTifSourceConfigDtoResponse.getVersion(), - RestStatus.OK, - saTifSourceConfigDtoResponse + lockService.releaseLock(lock, ActionListener.wrap( + r -> { + log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); + listener.onResponse(new SAIndexTIFSourceConfigResponse( + saTifSourceConfigDtoResponse.getId(), + saTifSourceConfigDtoResponse.getVersion(), + RestStatus.OK, + saTifSourceConfigDtoResponse + )); + }, + e -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config [%s].", lock.getLockId(), saTifSourceConfigDto.getId()), e); + listener.onResponse(new SAIndexTIFSourceConfigResponse( + saTifSourceConfigDtoResponse.getId(), + saTifSourceConfigDtoResponse.getVersion(), + RestStatus.OK, + saTifSourceConfigDtoResponse + )); + } )); }, e -> { - lockService.releaseLock(lock); - log.error("Failed to create IOCs and threat intel source config"); - listener.onFailure(e); + String action = RestRequest.Method.PUT.equals(request.getMethod()) ? "update" : "create"; + log.error(String.format("Failed to %s IOCs and threat intel source config", action), e); + lockService.releaseLock(lock, ActionListener.wrap( + r -> { + log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); + listener.onFailure(e); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config.", lock.getLockId()), ex); + listener.onFailure(e); + } + )); } ) ); } catch (Exception e) { - lockService.releaseLock(lock); - listener.onFailure(e); - log.error("listener failed when executing", e); + String action = RestRequest.Method.PUT.equals(request.getMethod()) ? "update" : "create"; + log.error(String.format("Failed to %s IOCs and threat intel source config", action), e); + lockService.releaseLock(lock, ActionListener.wrap( + r -> { + log.debug("Released threat intel source config lock with id [{}]", lock.getLockId()); + listener.onFailure(e); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif source config.", lock.getLockId()), ex); + listener.onFailure(e); + } + )); } }, exception -> { + String action = RestRequest.Method.PUT.equals(request.getMethod()) ? "update" : "create"; + log.error(String.format("Failed to acquire lock while trying to %s tif source config", action), exception); listener.onFailure(exception); - log.error("execution failed", exception); })); } catch (Exception e) { log.error("Failed to acquire lock for job", e); diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportListIOCsAction.java similarity index 66% rename from src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportListIOCsAction.java index 1e91ce1f3..80a6b538c 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportListIOCsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportListIOCsAction.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.transport; +package org.opensearch.securityanalytics.threatIntel.transport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -19,40 +19,40 @@ import org.opensearch.cluster.routing.Preference; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.commons.alerting.model.Table; +import org.opensearch.commons.authuser.User; import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.BoolQueryBuilder; -import org.opensearch.index.query.Operator; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHit; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.Aggregations; +import org.opensearch.search.aggregations.bucket.terms.Terms; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.sort.FieldSortBuilder; import org.opensearch.search.sort.SortBuilder; import org.opensearch.search.sort.SortBuilders; import org.opensearch.search.sort.SortOrder; -import org.opensearch.securityanalytics.action.ListIOCsAction; -import org.opensearch.securityanalytics.action.ListIOCsActionRequest; -import org.opensearch.securityanalytics.action.ListIOCsActionResponse; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsAction; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionRequest; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionResponse; import org.opensearch.securityanalytics.model.DetailedSTIX2IOCDto; import org.opensearch.securityanalytics.model.STIX2IOC; import org.opensearch.securityanalytics.model.STIX2IOCDto; -import org.opensearch.securityanalytics.model.threatintel.IocFinding; -import org.opensearch.securityanalytics.model.threatintel.IocWithFeeds; -import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsAction; -import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsRequest; -import org.opensearch.securityanalytics.threatIntel.action.GetIocFindingsResponse; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; import org.opensearch.securityanalytics.threatIntel.service.DefaultTifSourceConfigLoaderService; import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; -import org.opensearch.securityanalytics.threatIntel.transport.TransportSearchTIFSourceConfigsAction; +import org.opensearch.securityanalytics.transport.SecureTransportAction; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; @@ -76,8 +76,6 @@ public class TransportListIOCsAction extends HandledTransportAction implements SecureTransportAction { private static final Logger log = LogManager.getLogger(TransportListIOCsAction.class); - public static final String STIX2_IOC_NESTED_PATH = "stix2_ioc."; - private final ClusterService clusterService; private final TransportSearchTIFSourceConfigsAction transportSearchTIFSourceConfigsAction; private final DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService; @@ -85,6 +83,12 @@ public class TransportListIOCsAction extends HandledTransportAction defaultTifConfigsLoadedListener = null; try { defaultTifConfigsLoadedListener = new StepListener<>(); @@ -184,36 +201,25 @@ private void listIocs(List iocIndices) { // If any of the 'type' options are 'ALL', do not apply 'type' filter if (request.getTypes() != null && request.getTypes().stream().noneMatch(type -> ListIOCsActionRequest.ALL_TYPES_FILTER.equalsIgnoreCase(type))) { for (String type : request.getTypes()) { - boolQueryBuilder.should(QueryBuilders.matchQuery(STIX2_IOC_NESTED_PATH + STIX2IOC.TYPE_FIELD, type)); + boolQueryBuilder.should(QueryBuilders.matchQuery(STIX2IOC.TYPE_FIELD, type)); } boolQueryBuilder.must(typeQueryBuilder); } if (request.getTable().getSearchString() != null && !request.getTable().getSearchString().isEmpty()) { - boolQueryBuilder.must( - QueryBuilders.queryStringQuery(request.getTable().getSearchString()) - .defaultOperator(Operator.OR) -// .field(STIX2_IOC_NESTED_PATH + STIX2IOC.ID_FIELD) // Currently not a column in UX table - .field(STIX2_IOC_NESTED_PATH + STIX2IOC.NAME_FIELD) - .field(STIX2_IOC_NESTED_PATH + STIX2IOC.VALUE_FIELD) - .field(STIX2_IOC_NESTED_PATH + STIX2IOC.SEVERITY_FIELD) - .field(STIX2_IOC_NESTED_PATH + STIX2IOC.CREATED_FIELD) - .field(STIX2_IOC_NESTED_PATH + STIX2IOC.MODIFIED_FIELD) -// .field(STIX2_IOC_NESTED_PATH + STIX2IOC.DESCRIPTION_FIELD) // Currently not a column in UX table -// .field(STIX2_IOC_NESTED_PATH + STIX2IOC.LABELS_FIELD) // Currently not a column in UX table -// .field(STIX2_IOC_NESTED_PATH + STIX2IOC.SPEC_VERSION_FIELD) // Currently not a column in UX table - ); + boolQueryBuilder.must(QueryBuilders.queryStringQuery(request.getTable().getSearchString())); } SortBuilder sortBuilder = SortBuilders - .fieldSort(STIX2_IOC_NESTED_PATH + request.getTable().getSortString()) + .fieldSort(request.getTable().getSortString()) .order(SortOrder.fromString(request.getTable().getSortOrder())); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() .version(true) .seqNoAndPrimaryTerm(true) .fetchSource(true) + .trackTotalHits(true) .query(boolQueryBuilder) .sort(sortBuilder) .size(request.getTable().getSize()) @@ -231,80 +237,7 @@ public void onResponse(SearchResponse searchResponse) { onFailures(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT)); } - // Concurrently compiling a separate list of IOC IDs to create the subsequent GetIocFindingsRequest - Set iocIds = new HashSet<>(); - List iocs = new ArrayList<>(); - Arrays.stream(searchResponse.getHits().getHits()) - .forEach(hit -> { - try { - XContentParser xcp = XContentType.JSON.xContent().createParser( - xContentRegistry, - LoggingDeprecationHandler.INSTANCE, - hit.getSourceAsString()); - xcp.nextToken(); - - STIX2IOCDto ioc = STIX2IOCDto.parse(xcp, hit.getId(), hit.getVersion()); - - iocIds.add(ioc.getId()); - iocs.add(ioc); - } catch (Exception e) { - log.error( - () -> new ParameterizedMessage("Failed to parse IOC doc from hit {}", hit.getId()), e - ); - } - }); - - GetIocFindingsRequest getFindingsRequest = new GetIocFindingsRequest( - Collections.emptyList(), - new ArrayList<>(iocIds), - null, - null, - new Table( - "asc", - "timestamp", - request.getTable().getMissing(), - 10000, - 0, - "" - ) - ); - - // Calling GetIocFindings API to get number of findings for each returned IOC - client.execute(GetIocFindingsAction.INSTANCE, getFindingsRequest, new ActionListener<>() { - @Override - public void onResponse(GetIocFindingsResponse getFindingsResponse) { - // Iterate through the GetIocFindingsResponse to count occurrences of each IOC - Map iocIdToNumFindings = new HashMap<>(); - for (IocFinding iocFinding : getFindingsResponse.getIocFindings()) { - for (IocWithFeeds iocWithFeeds : iocFinding.getFeedIds()) { - // Set the count to 0 if it's not already - iocIdToNumFindings.putIfAbsent(iocWithFeeds.getIocId(), 0); - // Increment the count for the IOC - iocIdToNumFindings.merge(iocWithFeeds.getIocId(), 1, Integer::sum); - } - } - - // Iterate through each IOC returned by the SearchRequest to create the detailed model for response - List iocDetails = new ArrayList<>(); - iocs.forEach((ioc) -> { - Integer numFindings = iocIdToNumFindings.get(ioc.getId()); - if (numFindings == null) { - // Logging instances of 'null' separately from 0 instances for investigation purposes - log.debug("Null number of findings found for IOC {}", ioc.getId()); - numFindings = 0; - } - iocDetails.add(new DetailedSTIX2IOCDto(ioc, numFindings)); - }); - - onOperation(new ListIOCsActionResponse(searchResponse.getHits().getTotalHits().value, iocDetails)); - } - - @Override - public void onFailure(Exception e) { - log.error("Failed to get IOC findings count:", e); - listener.onFailure(SecurityAnalyticsException.wrap(e)); - } - }); + getFindingsCount(searchResponse); } @Override @@ -320,6 +253,79 @@ public void onFailure(Exception e) { }); } + private void getFindingsCount(SearchResponse iocSearchResponse) { + // Concurrently compiling a separate list of IOC IDs to create the subsequent findings count searchRequest + Set iocIds = new HashSet<>(); + List iocs = new ArrayList<>(); + Arrays.stream(iocSearchResponse.getHits().getHits()) + .forEach(hit -> { + try { + XContentParser xcp = XContentType.JSON.xContent().createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, + hit.getSourceAsString()); + xcp.nextToken(); + + STIX2IOCDto ioc = STIX2IOCDto.parse(xcp, hit.getId(), hit.getVersion()); + + iocIds.add(ioc.getId()); + iocs.add(ioc); + } catch (Exception e) { + log.error( + () -> new ParameterizedMessage("Failed to parse IOC doc from hit {}", hit.getId()), e + ); + } + }); + + // Create an aggregation query that will group by the IOC IDs in the findings + SearchSourceBuilder findingsCountSourceBuilder = new SearchSourceBuilder() + .fetchSource(false) + .trackTotalHits(true) + .query(QueryBuilders.termsQuery(IOC_ID_KEYWORD_FIELD, iocIds)) + .size(0) + .aggregation( + AggregationBuilders + .terms(IOC_COUNT_AGG_NAME) + .field(IOC_ID_KEYWORD_FIELD) + .size(iocIds.size()) + ); + + iocFindingService.search(findingsCountSourceBuilder, new ActionListener<>() { + @Override + public void onResponse(SearchResponse findingsSearchResponse) { + Map iocIdToNumFindings = new HashMap<>(); + + // Retrieve and store the counts from the aggregation response + Aggregations aggregations = findingsSearchResponse.getAggregations(); + if (aggregations != null) { + Terms iocIdCount = aggregations.get(IOC_COUNT_AGG_NAME); + if (iocIdCount != null) { + for (Terms.Bucket bucket : iocIdCount.getBuckets()) { + String iocId = bucket.getKeyAsString(); + long findingCount = bucket.getDocCount(); + iocIdToNumFindings.put(iocId, (int) findingCount); + } + } + } + + // Iterate through each IOC returned by the SearchRequest to create the detailed model for response + List iocDetails = new ArrayList<>(); + iocs.forEach((ioc) -> { + Integer numFindings = iocIdToNumFindings.getOrDefault(ioc.getId(), 0); + iocDetails.add(new DetailedSTIX2IOCDto(ioc, numFindings)); + }); + + // Return API response + onOperation(new ListIOCsActionResponse(iocSearchResponse.getHits().getTotalHits().value, iocDetails)); + } + + @Override + public void onFailure(Exception e) { + log.error("Failed to get IOC findings count:", e); + listener.onFailure(SecurityAnalyticsException.wrap(e)); + } + }); + } private void onOperation(ListIOCsActionResponse response) { this.response.set(response); if (counter.compareAndSet(false, true)) { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java index 2c756b3d3..4c5bd5e4d 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java @@ -15,10 +15,13 @@ import org.opensearch.action.support.HandledTransportAction; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.commons.authuser.User; import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest; import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; @@ -27,6 +30,7 @@ import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; +import org.opensearch.securityanalytics.transport.SecureTransportAction; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -40,13 +44,16 @@ /** * Transport action to create job to fetch threat intel feed data and save IoCs */ -public class TransportPutTIFJobAction extends HandledTransportAction { +public class TransportPutTIFJobAction extends HandledTransportAction implements SecureTransportAction { // TODO refactor this into a service class that creates feed updation job. This is not necessary to be a transport action private static final Logger log = LogManager.getLogger(TransportPutTIFJobAction.class); private final TIFJobParameterService tifJobParameterService; private final TIFJobUpdateService tifJobUpdateService; private final TIFLockService lockService; + private final Settings settings; + private final ThreadPool threadPool; + private volatile Boolean filterByEnabled; /** * Default constructor @@ -64,16 +71,29 @@ public TransportPutTIFJobAction( final ThreadPool threadPool, final TIFJobParameterService tifJobParameterService, final TIFJobUpdateService tifJobUpdateService, - final TIFLockService lockService + final TIFLockService lockService, + Settings settings ) { super(PutTIFJobAction.NAME, transportService, actionFilters, PutTIFJobRequest::new); this.tifJobParameterService = tifJobParameterService; this.tifJobUpdateService = tifJobUpdateService; this.lockService = lockService; + this.threadPool = threadPool; + this.settings = settings; + this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); } @Override protected void doExecute(final Task task, final PutTIFJobRequest request, final ActionListener listener) { + User user = readUserFromThreadContext(this.threadPool); + + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + listener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } + this.threadPool.getThreadContext().stashContext(); + try { lockService.acquireLock(request.getName(), LOCK_DURATION_IN_SECONDS, ActionListener.wrap(lock -> { if (lock == null) { @@ -86,9 +106,17 @@ protected void doExecute(final Task task, final PutTIFJobRequest request, final try { internalDoExecute(request, lock, listener); } catch (Exception e) { - lockService.releaseLock(lock); - listener.onFailure(e); - log.error("listener failed when executing", e); + log.error("Failed execution to put tif job action", e); + lockService.releaseLock(lock, ActionListener.wrap( + r -> { + log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); + listener.onFailure(e); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif job parameter [%s].", lock.getLockId(), request.getName()), ex); + listener.onFailure(e); + } + )); } }, exception -> { listener.onFailure(exception); @@ -118,9 +146,17 @@ protected void internalDoExecute( listener.onFailure(e); } }, exception -> { - lockService.releaseLock(lock); - log.error("failed to release lock", exception); - listener.onFailure(exception); + log.error("Failed to save tif job parameter", exception); + lockService.releaseLock(lock, ActionListener.wrap( + r -> { + log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); + listener.onFailure(exception); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif job parameter [%s].", lock.getLockId(), request.getName()), ex); + listener.onFailure(exception); + } + )); }); tifJobParameterService.createJobIndexIfNotExists(createIndexStepListener); } @@ -140,22 +176,40 @@ protected ActionListener postIndexingTifJobParameter( createThreatIntelFeedData(tifJobParameter, lockService.getRenewLockRunnable(lockReference), ActionListener.wrap( threatIntelIndicesResponse -> { if (threatIntelIndicesResponse.isAcknowledged()) { - lockService.releaseLock(lockReference.get()); - listener.onResponse(new AcknowledgedResponse(true)); + lockService.releaseLock(lockReference.get(), ActionListener.wrap( + r -> { + log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); + listener.onResponse(new AcknowledgedResponse(true)); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif job parameter [%s].", lock.getLockId(), tifJobParameter.getName()), ex); + listener.onFailure(ex); + } + )); } else { listener.onFailure(new OpenSearchStatusException("creation of threat intel feed data failed", RestStatus.INTERNAL_SERVER_ERROR)); } }, listener::onFailure )); }, e -> { - lockService.releaseLock(lock); + Exception exception; if (e instanceof VersionConflictEngineException) { log.error("tifJobParameter already exists"); - listener.onFailure(new ResourceAlreadyExistsException("tifJobParameter [{}] already exists", tifJobParameter.getName())); + exception = new ResourceAlreadyExistsException("tifJobParameter [{}] already exists", tifJobParameter.getName()); } else { log.error("Internal server error"); - listener.onFailure(e); + exception = e; } + lockService.releaseLock(lock, ActionListener.wrap( + r -> { + log.debug("Released tif job parameter lock with id [{}]", lock.getLockId()); + listener.onFailure(exception); + }, + ex -> { + log.error(String.format("Unexpected failure while trying to release lock [%s] for tif job parameter [%s].", lock.getLockId(), tifJobParameter.getName()), ex); + listener.onFailure(exception); + } + )); } ); } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportRefreshTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportRefreshTIFSourceConfigAction.java index de809be45..0c8af386f 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportRefreshTIFSourceConfigAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportRefreshTIFSourceConfigAction.java @@ -60,6 +60,7 @@ protected void doExecute(Task task, SARefreshTIFSourceConfigRequest request, Act actionListener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); return; } + this.threadPool.getThreadContext().stashContext(); saTifSourceConfigManagementService.refreshTIFSourceConfig(request.getId(), user, ActionListener.wrap( r -> actionListener.onResponse( diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java index 9eb47f0a3..877728e31 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportSearchTIFSourceConfigsAction.java @@ -66,8 +66,8 @@ protected void doExecute(Task task, SASearchTIFSourceConfigsRequest request, Act actionListener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); return; } - this.threadPool.getThreadContext().stashContext(); // stash context to make calls as admin client + StepListener defaultTifConfigsLoadedListener; try { defaultTifConfigsLoadedListener = new StepListener<>(); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportDeleteThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportDeleteThreatIntelMonitorAction.java index 041a8cd99..1ecebc2e3 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportDeleteThreatIntelMonitorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportDeleteThreatIntelMonitorAction.java @@ -61,6 +61,8 @@ protected void doExecute(Task task, DeleteThreatIntelMonitorRequest request, Act listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(validateBackendRoleMessage, RestStatus.FORBIDDEN))); return; } + this.threadPool.getThreadContext().stashContext(); + AlertingPluginInterface.INSTANCE.deleteMonitor((NodeClient) client, new DeleteMonitorRequest(request.getMonitorId(), WriteRequest.RefreshPolicy.IMMEDIATE), listener); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportGetThreatIntelAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportGetThreatIntelAlertsAction.java index 71fb4a71f..65feccc6f 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportGetThreatIntelAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportGetThreatIntelAlertsAction.java @@ -102,12 +102,14 @@ protected void doExecute(Task task, GetThreatIntelAlertsRequest request, ActionL listener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); return; } + this.threadPool.getThreadContext().stashContext(); + //fetch monitors and search SearchRequest threatIntelMonitorsSearchRequest = new SearchRequest(); threatIntelMonitorsSearchRequest.indices(".opendistro-alerting-config"); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.owner", PLUGIN_OWNER_FIELD))); - boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE))); + boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.owner", PLUGIN_OWNER_FIELD))); + boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE))); threatIntelMonitorsSearchRequest.source(new SearchSourceBuilder().query(boolQueryBuilder)); transportSearchThreatIntelMonitorAction.execute(new SearchThreatIntelMonitorRequest(threatIntelMonitorsSearchRequest), ActionListener.wrap( searchResponse -> { @@ -139,7 +141,7 @@ private void getAlerts(List monitorIds, BoolQueryBuilder monitorIdMatchQuery = QueryBuilders.boolQuery(); for (String monitorId : monitorIds) { monitorIdMatchQuery.should(QueryBuilders.boolQuery() - .must(QueryBuilders.matchQuery("monitor_id", monitorId))); + .must(QueryBuilders.matchPhraseQuery("monitor_id", monitorId))); } queryBuilder.filter(monitorIdMatchQuery); diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java index 2d6a5bf30..c9e364da7 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportIndexThreatIntelMonitorAction.java @@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.OpenSearchException; import org.opensearch.OpenSearchStatusException; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.search.SearchRequest; @@ -106,6 +107,8 @@ protected void doExecute(Task task, IndexThreatIntelMonitorRequest request, Acti listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(validateBackendRoleMessage, RestStatus.FORBIDDEN))); return; } + this.threadPool.getThreadContext().stashContext(); + if(request.getMethod().equals(RestRequest.Method.PUT)) { indexMonitor(request, listener, user); return; @@ -115,8 +118,8 @@ protected void doExecute(Task task, IndexThreatIntelMonitorRequest request, Acti SearchRequest threatIntelMonitorsSearchRequest = new SearchRequest(); threatIntelMonitorsSearchRequest.indices(".opendistro-alerting-config"); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.owner", PLUGIN_OWNER_FIELD))); - boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE))); + boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.owner", PLUGIN_OWNER_FIELD))); + boolQueryBuilder.must().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE))); threatIntelMonitorsSearchRequest.source(new SearchSourceBuilder().query(boolQueryBuilder)); transportSearchThreatIntelMonitorAction.execute(new SearchThreatIntelMonitorRequest(threatIntelMonitorsSearchRequest), ActionListener.wrap( searchResponse -> { @@ -144,6 +147,9 @@ protected void doExecute(Task task, IndexThreatIntelMonitorRequest request, Acti } )); + } catch (OpenSearchException e) { + log.error(() -> new ParameterizedMessage("Unexpected failure while indexing threat intel monitor {} named {}", request.getId(), request.getMonitor().getName())); + listener.onFailure(new SecurityAnalyticsException("Unexpected failure while indexing threat intel monitor", e.status(), e)); } catch (Exception e) { log.error(() -> new ParameterizedMessage("Unexpected failure while indexing threat intel monitor {} named {}", request.getId(), request.getMonitor().getName())); listener.onFailure(new SecurityAnalyticsException("Unexpected failure while indexing threat intel monitor", RestStatus.INTERNAL_SERVER_ERROR, e)); @@ -161,8 +167,13 @@ private void indexMonitor(IndexThreatIntelMonitorRequest request, ActionListener IndexThreatIntelMonitorResponse response = getIndexThreatIntelMonitorResponse(r, user); listener.onResponse(response); }, e -> { - log.error("failed to creat threat intel monitor", e); - listener.onFailure(new SecurityAnalyticsException("Failed to create threat intel monitor", RestStatus.INTERNAL_SERVER_ERROR, e)); + String errorText = "Failed to create threat intel monitor"; + SecurityAnalyticsException exception = new SecurityAnalyticsException(errorText, RestStatus.INTERNAL_SERVER_ERROR, e); + log.error(errorText, e); + if (e instanceof OpenSearchException) { + exception = new SecurityAnalyticsException(errorText, ((OpenSearchException) e).status(), e); + } + listener.onFailure(exception); } )); } @@ -210,23 +221,33 @@ private Monitor buildThreatIntelMonitor(IndexThreatIntelMonitorRequest request) throw new RuntimeException(e); } } - return new Monitor( - request.getMethod() == RestRequest.Method.POST ? Monitor.NO_ID : request.getId(), - Monitor.NO_VERSION, - StringUtils.isBlank(request.getMonitor().getName()) ? "threat_intel_monitor" : request.getMonitor().getName(), - request.getMonitor().isEnabled(), - request.getMonitor().getSchedule(), - Instant.now(), - request.getMonitor().isEnabled() ? Instant.now() : null, - THREAT_INTEL_MONITOR_TYPE, - request.getMonitor().getUser(), - 1, - List.of(remoteDocLevelMonitorInput), - triggers, - Collections.emptyMap(), - new DataSources(), - PLUGIN_OWNER_FIELD - ); + + Monitor monitor; + try { + monitor = new Monitor( + request.getMethod() == RestRequest.Method.POST ? Monitor.NO_ID : request.getId(), + Monitor.NO_VERSION, + StringUtils.isBlank(request.getMonitor().getName()) ? "threat_intel_monitor" : request.getMonitor().getName(), + request.getMonitor().isEnabled(), + request.getMonitor().getSchedule(), + Instant.now(), + request.getMonitor().isEnabled() ? Instant.now() : null, + THREAT_INTEL_MONITOR_TYPE, + request.getMonitor().getUser(), + 1, + List.of(remoteDocLevelMonitorInput), + triggers, + Collections.emptyMap(), + new DataSources(), + false, + PLUGIN_OWNER_FIELD + ); + } catch (Exception e) { + String error = "Error occurred while parsing monitor."; + log.error(error, e); + throw new SecurityAnalyticsException(error, RestStatus.BAD_REQUEST, e); + } + return monitor; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportSearchThreatIntelMonitorAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportSearchThreatIntelMonitorAction.java index b918e02ec..11a1e1beb 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportSearchThreatIntelMonitorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportSearchThreatIntelMonitorAction.java @@ -1,5 +1,6 @@ package org.opensearch.securityanalytics.threatIntel.transport.monitor; +import org.opensearch.OpenSearchStatusException; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; @@ -17,6 +18,7 @@ import org.opensearch.commons.authuser.User; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; @@ -70,7 +72,11 @@ protected void doExecute(Task task, SearchThreatIntelMonitorRequest request, Act // log.info("Filtering result by: {}", user.getBackendRoles()); // addFilter(user, request.searchRequest().source(), "detector.user.backend_roles.keyword"); // } // TODO - + String validateBackendRoleMessage = validateUserBackendRoles(user, this.filterByEnabled); + if (!"".equals(validateBackendRoleMessage)) { + listener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); + return; + } this.threadPool.getThreadContext().stashContext(); //TODO change search request to fetch threat intel monitors diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportUpdateThreatIntelAlertStatusAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportUpdateThreatIntelAlertStatusAction.java index cb8d1d8a4..7aa828d0c 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportUpdateThreatIntelAlertStatusAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/monitor/TransportUpdateThreatIntelAlertStatusAction.java @@ -96,12 +96,14 @@ protected void doExecute(Task task, UpdateThreatIntelAlertStatusRequest request, listener.onFailure(new OpenSearchStatusException("Do not have permissions to resource", RestStatus.FORBIDDEN)); return; } + this.threadPool.getThreadContext().stashContext(); + //fetch monitors and search SearchRequest threatIntelMonitorsSearchRequest = new SearchRequest(); threatIntelMonitorsSearchRequest.indices(".opendistro-alerting-config"); BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.owner", PLUGIN_OWNER_FIELD))); - boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE))); + boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.owner", PLUGIN_OWNER_FIELD))); + boolQueryBuilder.should().add(new BoolQueryBuilder().must(QueryBuilders.matchPhraseQuery("monitor.monitor_type", ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE))); threatIntelMonitorsSearchRequest.source(new SearchSourceBuilder().query(boolQueryBuilder)); transportSearchThreatIntelMonitorAction.execute(new SearchThreatIntelMonitorRequest(threatIntelMonitorsSearchRequest), ActionListener.wrap( searchResponse -> { @@ -172,22 +174,22 @@ private static SearchSourceBuilder getSearchSourceQueryingForAlertsToUpdate(List BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); BoolQueryBuilder monitorIdMatchQuery = QueryBuilders.boolQuery(); for (String monitorId : monitorIds) { - monitorIdMatchQuery.should(QueryBuilders.matchQuery(ThreatIntelAlert.MONITOR_ID_FIELD, monitorId)); + monitorIdMatchQuery.should(QueryBuilders.matchPhraseQuery(ThreatIntelAlert.MONITOR_ID_FIELD, monitorId)); } queryBuilder.filter(monitorIdMatchQuery); BoolQueryBuilder idMatchQuery = QueryBuilders.boolQuery(); for (String id : request.getAlertIds()) { - idMatchQuery.should(QueryBuilders.matchQuery("_id", id)); + idMatchQuery.should(QueryBuilders.matchPhraseQuery("_id", id)); } queryBuilder.filter(idMatchQuery); if (request.getState() == Alert.State.COMPLETED) { - queryBuilder.filter(QueryBuilders.matchQuery(ThreatIntelAlert.STATE_FIELD, Alert.State.ACKNOWLEDGED.toString())); + queryBuilder.filter(QueryBuilders.matchPhraseQuery(ThreatIntelAlert.STATE_FIELD, Alert.State.ACKNOWLEDGED.toString())); } else if (request.getState() == Alert.State.ACKNOWLEDGED) { - queryBuilder.filter(QueryBuilders.matchQuery(ThreatIntelAlert.STATE_FIELD, Alert.State.ACTIVE.toString())); + queryBuilder.filter(QueryBuilders.matchPhraseQuery(ThreatIntelAlert.STATE_FIELD, Alert.State.ACTIVE.toString())); } else { log.error("Threat intel monitor not found. No alerts to update"); listener.onFailure(new SecurityAnalyticsException("Threat intel monitor not found. No alerts to update", @@ -272,14 +274,14 @@ private static SearchSourceBuilder getSearchSourceQueryingForUpdatedAlerts(List< BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); BoolQueryBuilder monitorIdMatchQuery = QueryBuilders.boolQuery(); for (String monitorId : monitorIds) { - monitorIdMatchQuery.should(QueryBuilders.matchQuery(ThreatIntelAlert.MONITOR_ID_FIELD, monitorId)); + monitorIdMatchQuery.should(QueryBuilders.matchPhraseQuery(ThreatIntelAlert.MONITOR_ID_FIELD, monitorId)); } queryBuilder.filter(monitorIdMatchQuery); BoolQueryBuilder idMatchQuery = QueryBuilders.boolQuery(); for (String id : alertIds) { - idMatchQuery.should(QueryBuilders.matchQuery("_id", id)); + idMatchQuery.should(QueryBuilders.matchPhraseQuery("_id", id)); } queryBuilder.filter(idMatchQuery); diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportAckCorrelationAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportAckCorrelationAlertsAction.java index 917d0349c..7032819de 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportAckCorrelationAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportAckCorrelationAlertsAction.java @@ -67,6 +67,8 @@ protected void doExecute(Task task, AckCorrelationAlertsRequest request, ActionL return; } + this.threadPool.getThreadContext().stashContext(); + if (!request.getCorrelationAlertIds().isEmpty()) { correlationAlertService.acknowledgeAlerts( request.getCorrelationAlertIds(), diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetCorrelationAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetCorrelationAlertsAction.java index cdca86a23..a19817e5b 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetCorrelationAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetCorrelationAlertsAction.java @@ -64,6 +64,8 @@ protected void doExecute(Task task, GetCorrelationAlertsRequest request, ActionL return; } + this.threadPool.getThreadContext().stashContext(); + if (request.getCorrelationRuleId() != null) { correlationAlertService.getCorrelationAlerts( request.getCorrelationRuleId(), diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index 28ec5fcd8..f415d0f2a 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -155,6 +155,8 @@ public class TransportIndexDetectorAction extends HandledTransportAction detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), PLUGIN_OWNER_FIELD); + true), enableDetectorWithDedicatedQueryIndices, PLUGIN_OWNER_FIELD); return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } @@ -886,14 +890,14 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest( Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, monitorName, false, detector.getSchedule(), detector.getLastUpdateTime(), null, Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), - new DataSources(detector.getRuleIndex(), + new DataSources(enableDetectorWithDedicatedQueryIndices? detector.getRuleIndex() + "_chained_findings": detector.getRuleIndex(), detector.getFindingsIndex(), detector.getFindingsIndexPattern(), detector.getAlertsIndex(), detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), PLUGIN_OWNER_FIELD); + true), enableDetectorWithDedicatedQueryIndices, PLUGIN_OWNER_FIELD); return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); } @@ -1063,7 +1067,7 @@ public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) { detector.getAlertsHistoryIndex(), detector.getAlertsHistoryIndexPattern(), DetectorMonitorConfig.getRuleIndexMappingsByType(), - true), PLUGIN_OWNER_FIELD); + true), false, PLUGIN_OWNER_FIELD); listener.onResponse(new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null)); } @@ -1247,7 +1251,13 @@ void createDetector() { request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic)); request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic)); request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic)); - request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); + + if (enableDetectorWithDedicatedQueryIndices) { + // disabling the setting after enabling it will mean delete & re-create the detector + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); + } else { + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); + } User originalContextUser = this.user; log.debug("user from original context is {}", originalContextUser); @@ -1364,7 +1374,16 @@ void onGetResponse(Detector currentDetector, User user) { request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic)); request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic)); request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic)); - request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); + if (currentDetector.getRuleIndex().contains("optimized")) { + request.getDetector().setRuleIndex(currentDetector.getRuleIndex()); + } else { + if (enableDetectorWithDedicatedQueryIndices) { + // disabling the setting after enabling it will mean delete & re-create the detector + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); + } else { + request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); + } + } request.getDetector().setUser(user); if (!detector.getInputs().isEmpty()) { @@ -1800,4 +1819,8 @@ private void setFilterByEnabled(boolean filterByEnabled) { private void setEnabledWorkflowUsage(boolean enabledWorkflowUsage) { this.enabledWorkflowUsage = enabledWorkflowUsage; } + + private void setEnabledDetectorsWithDedicatedQueryIndices(boolean enabledDetectorsWithDedicatedQueryIndices) { + this.enableDetectorWithDedicatedQueryIndices = enabledDetectorsWithDedicatedQueryIndices; + } } diff --git a/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java b/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java index 375342d09..36fd5e37d 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/CorrelationIndices.java @@ -7,6 +7,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.alias.Alias; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.core.action.ActionListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; @@ -26,6 +27,9 @@ import java.nio.charset.Charset; import java.util.Objects; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; + public class CorrelationIndices { private static final Logger log = LogManager.getLogger(CorrelationIndices.class); @@ -55,9 +59,15 @@ public static String correlationMappings() throws IOException { public void initCorrelationIndex(ActionListener actionListener) throws IOException { if (!correlationIndexExists()) { + Settings indexSettings = Settings.builder() + .put("index.hidden", true) + .put("index.correlation", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(CORRELATION_HISTORY_INDEX_PATTERN) .mapping(correlationMappings()) - .settings(Settings.builder().put("index.hidden", true).put("index.correlation", true).build()); + .settings(indexSettings); indexRequest.alias(new Alias(CORRELATION_HISTORY_WRITE_INDEX)); client.admin().indices().create(indexRequest, actionListener); } else { @@ -67,9 +77,15 @@ public void initCorrelationIndex(ActionListener actionListe public void initCorrelationMetadataIndex(ActionListener actionListener) throws IOException { if (!correlationMetadataIndexExists()) { + Settings indexSettings = Settings.builder() + .put("index.hidden", true) + .put("index.correlation", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(CORRELATION_METADATA_INDEX) .mapping(correlationMappings()) - .settings(Settings.builder().put("index.hidden", true).put("index.correlation", true).build()); + .settings(indexSettings); client.admin().indices().create(indexRequest, actionListener); } else { actionListener.onResponse(new CreateIndexResponse(true, true, CORRELATION_METADATA_INDEX)); @@ -136,6 +152,8 @@ public static String correlationAlertIndexMappings() throws IOException { public void initCorrelationAlertIndex(ActionListener actionListener) throws IOException { Settings correlationAlertSettings = Settings.builder() .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(CORRELATION_ALERT_INDEX) .mapping(correlationAlertIndexMappings()) diff --git a/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java index d131e47b4..27f6475f8 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/CorrelationRuleIndices.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.core.action.ActionListener; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; @@ -23,6 +24,9 @@ import java.util.Objects; import org.opensearch.securityanalytics.model.CorrelationRule; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; + public class CorrelationRuleIndices { private static final Logger log = LogManager.getLogger(CorrelationRuleIndices.class); @@ -45,9 +49,14 @@ public static String correlationRuleIndexMappings() throws IOException { public void initCorrelationRuleIndex(ActionListener actionListener) throws IOException { if (!correlationRuleIndexExists()) { + Settings indexSettings = Settings.builder() + .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(CorrelationRule.CORRELATION_RULE_INDEX).mapping( correlationRuleIndexMappings() - ).settings(Settings.builder().put("index.hidden", true).build()); + ).settings(indexSettings); client.admin().indices().create(indexRequest, actionListener); } } diff --git a/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java b/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java index 21eb460d5..d2b5661fe 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/CustomLogTypeIndices.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.nio.charset.Charset; import java.util.Objects; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; public class CustomLogTypeIndices { @@ -42,9 +44,11 @@ public static String customLogTypeMappings() throws IOException { public void initCustomLogTypeIndex(ActionListener actionListener) throws IOException { if (!customLogTypeIndexExists()) { + // Security Analytics log types index is small. 1 primary shard is enough Settings indexSettings = Settings.builder() .put("index.hidden", true) - .put("index.auto_expand_replicas", "0-all") + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(LogTypeService.LOG_TYPE_INDEX) .mapping(customLogTypeMappings()) diff --git a/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java b/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java index d6a81e134..83eb058e0 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/DetectorIndices.java @@ -23,6 +23,9 @@ import java.nio.charset.Charset; import java.util.Objects; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; + public class DetectorIndices { private static final Logger log = LogManager.getLogger(DetectorIndices.class); @@ -45,9 +48,14 @@ public static String detectorMappings() throws IOException { public void initDetectorIndex(ActionListener actionListener) throws IOException { if (!detectorIndexExists()) { + Settings indexSettings = Settings.builder() + .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) + .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(Detector.DETECTORS_INDEX) .mapping(detectorMappings()) - .settings(Settings.builder().put("index.hidden", true).build()); + .settings(indexSettings); client.indices().create(indexRequest, actionListener); } } diff --git a/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java b/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java index 9d5066308..c4dafed85 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/util/IndexUtils.java @@ -31,9 +31,9 @@ public class IndexUtils { - private static final String _META = "_meta"; - private static final Integer NO_SCHEMA_VERSION = 0; - private static final String SCHEMA_VERSION = "schema_version"; + public static final String _META = "_meta"; + public static final Integer NO_SCHEMA_VERSION = 0; + public static final String SCHEMA_VERSION = "schema_version"; public static Boolean detectorIndexUpdated = false; public static Boolean customRuleIndexUpdated = false; diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java index 94a98106b..b1ff516d5 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java @@ -65,6 +65,8 @@ import java.util.stream.Stream; import static org.opensearch.securityanalytics.model.Detector.NO_VERSION; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.maxSystemIndexReplicas; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.minSystemIndexReplicas; public class RuleIndices { @@ -93,6 +95,8 @@ public void initRuleIndex(ActionListener actionListener, bo if (!ruleIndexExists(isPrepackaged)) { Settings indexSettings = Settings.builder() .put("index.hidden", true) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("index.auto_expand_replicas", minSystemIndexReplicas + "-" + maxSystemIndexReplicas) .build(); CreateIndexRequest indexRequest = new CreateIndexRequest(getRuleIndex(isPrepackaged)) .mapping(ruleMappings()) diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java index 8f0f1cab5..7aa2def36 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java @@ -46,13 +46,9 @@ public static String ruleTopicIndexSettings() throws IOException { public void initRuleTopicIndexTemplate(ActionListener actionListener) throws IOException { getAllRuleIndices(ActionListener.wrap(allRuleIndices -> { // Compose list of all patterns to cover all query indices - List indexPatterns = new ArrayList<>(); - for(String ruleIndex : allRuleIndices) { - indexPatterns.add(ruleIndex + "*"); - } ComposableIndexTemplate template = new ComposableIndexTemplate( - indexPatterns, + allRuleIndices, new Template( Settings.builder().loadFromSource(ruleTopicIndexSettings(), XContentType.JSON).build(), null, @@ -87,7 +83,8 @@ private void getAllRuleIndices(ActionListener> listener) { listener.onResponse( logTypes .stream() - .map(logType -> DetectorMonitorConfig.getRuleIndex(logType)) + // use index pattern here to define rule topic index template for all query indices which match the pattern + .map(logType -> DetectorMonitorConfig.getRuleIndex(logType) + "*") .collect(Collectors.toList()) ); }, listener::onFailure)); diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index e4a14ac56..f196c4a5a 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -1,2 +1,3 @@ org.opensearch.securityanalytics.correlation.index.codec.correlation950.CorrelationCodec950 -org.opensearch.securityanalytics.correlation.index.codec.correlation990.CorrelationCodec990 \ No newline at end of file +org.opensearch.securityanalytics.correlation.index.codec.correlation990.CorrelationCodec990 +org.opensearch.securityanalytics.correlation.index.codec.correlation9120.CorrelationCodec9120 \ No newline at end of file diff --git a/src/main/resources/mappings/stix2_ioc_mapping.json b/src/main/resources/mappings/stix2_ioc_mapping.json index 8fd505f9a..deaf7c4a1 100644 --- a/src/main/resources/mappings/stix2_ioc_mapping.json +++ b/src/main/resources/mappings/stix2_ioc_mapping.json @@ -3,42 +3,38 @@ "schema_version": 1 }, "properties": { - "stix2_ioc": { - "properties": { - "name": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "value": { - "type": "keyword" - }, - "severity": { - "type": "keyword" - }, - "spec_version": { - "type": "keyword" - }, - "created": { - "type": "date" - }, - "modified": { - "type": "date" - }, - "description": { - "type": "text" - }, - "labels": { - "type": "keyword" - }, - "feed_id": { - "type": "keyword" - }, - "feed_name": { - "type": "keyword" - } - } + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "value": { + "type": "keyword" + }, + "severity": { + "type": "keyword" + }, + "spec_version": { + "type": "keyword" + }, + "created": { + "type": "date" + }, + "modified": { + "type": "date" + }, + "description": { + "type": "text" + }, + "labels": { + "type": "keyword" + }, + "feed_id": { + "type": "keyword" + }, + "feed_name": { + "type": "keyword" } } } diff --git a/src/main/resources/mappings/threat_intel_job_mapping.json b/src/main/resources/mappings/threat_intel_job_mapping.json index f437efe6f..fbc1c03dc 100644 --- a/src/main/resources/mappings/threat_intel_job_mapping.json +++ b/src/main/resources/mappings/threat_intel_job_mapping.json @@ -1,4 +1,5 @@ { + "dynamic": true, "_meta" : { "schema_version": 2 }, diff --git a/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java b/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java index 47c33e138..d6294ee76 100644 --- a/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java +++ b/src/test/java/org/opensearch/securityanalytics/DetectorThreatIntelIT.java @@ -75,7 +75,7 @@ public void testCreateDetectorWithThreatIntelEnabled_updateDetectorWithThreatInt " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(2, response.getHits().getTotalHits().value); @@ -275,7 +275,7 @@ public void testCreateDetectorWithThreatIntelDisabled_updateDetectorWithThreatIn " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -372,7 +372,7 @@ public void testCreateDetectorWithThreatIntelEnabledAndNoRules_triggerDetectionT " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -466,7 +466,7 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerDetectionTypeOnlyThr " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -561,7 +561,7 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerWithBothDetectionTyp " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -653,7 +653,7 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithThreatIntelDete " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -745,7 +745,7 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); @@ -802,4 +802,4 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT /** findings are present but alerts are NOT generated as detection type mentioned in trigger is threat_intel only but finding is from rules*/ Assert.assertEquals(3, getAlertsBody.get("total_alerts")); } -} +} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 2847b600a..26ffe4344 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -114,6 +114,7 @@ import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule; import static org.opensearch.securityanalytics.TestHelpers.vpcFlowMappings; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.IOC_ALL_INDEX_PATTERN; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD; @@ -268,6 +269,7 @@ void setDebugLogLevel() throws IOException, InterruptedException { makeRequest(client(), "PUT", "_cluster/settings", Collections.emptyMap(), se, new BasicHeader("Content-Type", "application/json")); + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "true"); } protected final List clusterPermissions = List.of( @@ -1542,6 +1544,24 @@ public List getIocFindingIndices() throws IOException { return indices; } + public List getIocIndices() throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/indices/" + IOC_ALL_INDEX_PATTERN + "?format=json")); + XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); + List responseList = xcp.list(); + List indices = new ArrayList<>(); + for (Object o : responseList) { + if (o instanceof Map) { + ((Map) o).forEach((BiConsumer) + (o1, o2) -> { + if (o1.equals("index")) { + indices.add((String) o2); + } + }); + } + } + return indices; + } + public List getQueryIndices(String detectorType) throws IOException { Response response = client().performRequest(new Request("GET", "/_cat/indices/" + DetectorMonitorConfig.getRuleIndex(detectorType) + "*?format=json")); XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 75ee7cd89..4a7fffd06 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -900,6 +900,16 @@ public static Action randomAction(String destinationId) { return new Action(name, destinationId, template, template, throttleEnabled, throttle, OpenSearchRestTestCase.randomAlphaOfLength(10), null); } + public static Action randomThreatInteMonitorAction(String destinationId) { + String name = OpenSearchRestTestCase.randomUnicodeOfLength(10); + Script template = randomTemplateScript("Threat intel Monitor {{ctx.monitor.name}} just entered alert status. Please investigate the issue.\n" + + " - Trigger: {{ctx.trigger.name}}\n" + + " - Severity: {{ctx.trigger.severity}}", null); + Boolean throttleEnabled = false; + Throttle throttle = randomThrottle(null, null); + return new Action(name, destinationId, template, template, throttleEnabled, throttle, OpenSearchRestTestCase.randomAlphaOfLength(10), null); + } + public static Script randomTemplateScript(String source, Map params) { if (params == null) { params = new HashMap<>(); @@ -1874,6 +1884,68 @@ public static String windowsIndexMappingOnlyNumericAndText() { " }"; } + public static String oldThreatIntelJobMapping() { + return " \"dynamic\": \"strict\",\n" + + " \"_meta\": {\n" + + " \"schema_version\": 1\n" + + " },\n" + + " \"properties\": {\n" + + " \"schema_version\": {\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " \"enabled_time\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"indices\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"last_update_time\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"schedule\": {\n" + + " \"properties\": {\n" + + " \"interval\": {\n" + + " \"properties\": {\n" + + " \"period\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"start_time\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"unit\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"state\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"update_enabled\": {\n" + + " \"type\": \"boolean\"\n" + + " },\n" + + " \"update_stats\": {\n" + + " \"properties\": {\n" + + " \"last_failed_at_in_epoch_millis\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"last_processing_time_in_millis\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"last_skipped_at_in_epoch_millis\": {\n" + + " \"type\": \"long\"\n" + + " },\n" + + " \"last_succeeded_at_in_epoch_millis\": {\n" + + " \"type\": \"long\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }"; + } public static String randomDoc(int severity, int version, String opCode) { String doc = "{\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigRequestTests.java b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigRequestTests.java index b572d2ca6..e40516e25 100644 --- a/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigRequestTests.java +++ b/src/test/java/org/opensearch/securityanalytics/action/IndexTIFSourceConfigRequestTests.java @@ -5,6 +5,7 @@ package org.opensearch.securityanalytics.action; import org.junit.Assert; +import org.opensearch.action.ActionRequestValidationException; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.rest.RestRequest; @@ -33,4 +34,40 @@ public void testTIFSourceConfigPostRequest() throws IOException { Assert.assertEquals(RestRequest.Method.POST, newRequest.getMethod()); Assert.assertNotNull(newRequest.getTIFConfigDto()); } + + public void testValidateSourceConfigPostRequest() { + // Source config with invalid: name, format, source, ioc type, source config type + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + false, + null, + true + ); + String id = saTifSourceConfigDto.getId(); + SAIndexTIFSourceConfigRequest request = new SAIndexTIFSourceConfigRequest(id, RestRequest.Method.POST, saTifSourceConfigDto); + Assert.assertNotNull(request); + + ActionRequestValidationException exception = request.validate(); + assertEquals(5, exception.validationErrors().size()); + assertTrue(exception.validationErrors().contains("Name must not be empty")); + assertTrue(exception.validationErrors().contains("Format must not be empty")); + assertTrue(exception.validationErrors().contains("Source must not be empty")); + assertTrue(exception.validationErrors().contains("Must specify at least one IOC type")); + assertTrue(exception.validationErrors().contains("Type must not be empty")); + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java index 06e464d34..bbb3eb71a 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java @@ -95,6 +95,7 @@ public void testGetAlerts_success() { List.of(), Map.of(), new DataSources(), + true, TransportIndexDetectorAction.PLUGIN_OWNER_FIELD ), new DocumentLevelTrigger("trigger_id_1", "my_trigger", "severity_low", List.of(), new Script("")), @@ -129,6 +130,7 @@ public void testGetAlerts_success() { List.of(), Map.of(), new DataSources(), + true, TransportIndexDetectorAction.PLUGIN_OWNER_FIELD ), new DocumentLevelTrigger("trigger_id_1", "my_trigger", "severity_low", List.of(), new Script("")), diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index 77b54dbde..c755bf1d5 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -295,7 +295,7 @@ public void testGetAlertsByStartTimeAndEndTimeSuccess() throws IOException, Inte } public void testGetAlerts_noDetector_failure() throws IOException { - // Call GetAlerts API + // Call GetAlerts API Map params = new HashMap<>(); params.put("detector_id", "nonexistent_detector_id"); try { @@ -820,7 +820,7 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException Collections.emptyList()); Detector detector = randomDetectorWithInputsAndTriggers(List.of(input), List.of(new DetectorTrigger("randomtrigegr", "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of())) - ); + ); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); @@ -831,7 +831,7 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); // 5 for rules, 1 for match_all query in chained findings monitor @@ -869,8 +869,8 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException } } - assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue()); - assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue()); + assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue()); + assertEquals(1, numberOfMonitorTypes.get(Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue()); Map params = new HashMap<>(); params.put("detector_id", detectorId); @@ -894,13 +894,13 @@ public void testMultipleAggregationAndDocRules_alertSuccess() throws IOException List> queries = (List>) finding.get("queries"); Set findingRuleIds = queries.stream().map(it -> it.get("id").toString()).collect(Collectors.toSet()); - // In the case of bucket level monitors, queries will always contain one value - String aggRuleId = findingRuleIds.iterator().next(); - List findingDocs = (List) finding.get("related_doc_ids"); + // In the case of bucket level monitors, queries will always contain one value + String aggRuleId = findingRuleIds.iterator().next(); + List findingDocs = (List) finding.get("related_doc_ids"); - if (aggRuleId.equals(sumRuleId)) { - assertTrue(List.of("1", "2", "3", "4", "5", "6", "7").containsAll(findingDocs)); - } + if (aggRuleId.equals(sumRuleId)) { + assertTrue(List.of("1", "2", "3", "4", "5", "6", "7").containsAll(findingDocs)); + } } assertTrue(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8").containsAll(docLevelFinding)); diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index 90b008581..ab06b8d30 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -1223,7 +1223,7 @@ public void testCreateDetectorWithNotCondition_verifyFindingsAndNoFindings_succe " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(1, response.getHits().getTotalHits().value); diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index 67eb3a313..d389797c5 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -1656,8 +1656,8 @@ public void testTraverseAndCopy() { " \"type\":\"keyword\"," + " \"ignore_above\":256" + " }" + - " }" + - " }" + + " }" + + " }" + " }" + " }" + "}"; @@ -1723,7 +1723,7 @@ public void testAzureMappings() throws IOException { Detector detector = randomDetectorWithInputs(List.of(input), "azure"); createDetector(detector); - List hits = executeSearch(".opensearch-sap-azure-detectors-queries-000001", matchAllSearchBody); + List hits = executeSearch(".opensearch-sap-azure-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(127, hits.size()); } @@ -1748,7 +1748,7 @@ public void testADLDAPMappings() throws IOException { Detector detector = randomDetectorWithInputs(List.of(input), "ad_ldap"); createDetector(detector); - List hits = executeSearch(".opensearch-sap-ad_ldap-detectors-queries-000001", matchAllSearchBody); + List hits = executeSearch(".opensearch-sap-ad_ldap-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(11, hits.size()); } @@ -1773,7 +1773,7 @@ public void testCloudtrailMappings() throws IOException { Detector detector = randomDetectorWithInputs(List.of(input), "cloudtrail"); createDetector(detector); - List hits = executeSearch(".opensearch-sap-cloudtrail-detectors-queries-000001", matchAllSearchBody); + List hits = executeSearch(".opensearch-sap-cloudtrail-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(39, hits.size()); } @@ -1798,7 +1798,7 @@ public void testS3Mappings() throws IOException { Detector detector = randomDetectorWithInputs(List.of(input), "s3"); createDetector(detector); - List hits = executeSearch(".opensearch-sap-s3-detectors-queries-000001", matchAllSearchBody); + List hits = executeSearch(".opensearch-sap-s3-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(1, hits.size()); } @@ -1825,7 +1825,7 @@ public void testWAFMappings() throws IOException { Detector detector = randomDetectorWithInputs(List.of(input), "waf"); createDetector(detector); - List hits = executeSearch(".opensearch-sap-waf-detectors-queries-000001", matchAllSearchBody); + List hits = executeSearch(".opensearch-sap-waf-detectors-queries-*", matchAllSearchBody); Assert.assertEquals(5, hits.size()); } diff --git a/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigDtoTests.java b/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigDtoTests.java index c9215af5a..e5b2ed7c5 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigDtoTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigDtoTests.java @@ -10,11 +10,15 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.model.S3Source; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; import static org.opensearch.securityanalytics.TestHelpers.randomSATIFSourceConfigDto; @@ -36,6 +40,34 @@ public void testParseFunction() throws IOException { assertEqualsSaTifSourceConfigDtos(saTifSourceConfigDto, newSaTifSourceConfigDto); } + public void testParseFunctionWithNullValues() throws IOException { + // Source config with invalid name and format + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + "randomId", + null, + null, + null, + SourceConfigType.S3_CUSTOM, + null, + null, + null, + new S3Source("bucket", "objectkey", "region", "rolearn"), + null, + null, + new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS), + null, + null, + null, + null, + true, + List.of("ip"), + true + ); + String json = toJsonString(saTifSourceConfigDto); + SATIFSourceConfigDto newSaTifSourceConfigDto = SATIFSourceConfigDto.parse(getParser(json), saTifSourceConfigDto.getId(), null); + assertEqualsSaTifSourceConfigDtos(saTifSourceConfigDto, newSaTifSourceConfigDto); + } + public XContentParser getParser(String xc) throws IOException { XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); parser.nextToken(); diff --git a/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java b/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java index 2687907d1..8fa8ec395 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/SATIFSourceConfigTests.java @@ -10,12 +10,17 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; import org.opensearch.securityanalytics.threatIntel.model.S3Source; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; import static org.opensearch.securityanalytics.TestHelpers.randomSATIFSourceConfig; @@ -37,6 +42,35 @@ public void testParseFunction() throws IOException { assertEqualsSaTifSourceConfigs(saTifSourceConfig, newSaTifSourceConfig); } + public void testParseFunctionWithNullValues() throws IOException { + // Source config with invalid name and format + SATIFSourceConfig saTifSourceConfig = new SATIFSourceConfig( + null, + null, + null, + null, + SourceConfigType.S3_CUSTOM, + null, + null, + null, + new S3Source("bucket", "objectkey", "region", "rolearn"), + null, + null, + new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS), + null, + null, + null, + null, + true, + new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(IOCType.DOMAIN_NAME_TYPE), "indexPattern", "writeIndex"))), + List.of("ip"), + true + ); + String json = toJsonString(saTifSourceConfig); + SATIFSourceConfig newSaTifSourceConfig = SATIFSourceConfig.parse(getParser(json), saTifSourceConfig.getId(), null); + assertEqualsSaTifSourceConfigs(saTifSourceConfig, newSaTifSourceConfig); + } + public XContentParser getParser(String xc) throws IOException { XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); parser.nextToken(); diff --git a/src/test/java/org/opensearch/securityanalytics/model/STIX2IOCDtoTests.java b/src/test/java/org/opensearch/securityanalytics/model/STIX2IOCDtoTests.java index 08a2b2185..110d75d50 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/STIX2IOCDtoTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/STIX2IOCDtoTests.java @@ -7,6 +7,9 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -32,4 +35,26 @@ public void testParseFunction() throws IOException { STIX2IOCDto newIoc = STIX2IOCDto.parse(parser(json), ioc.getId(), ioc.getVersion()); assertEqualIocDtos(ioc, newIoc); } + + public void testParseFunction_invalidType() throws IOException { + // Execute test case for each IOCType + for (String type : IOCType.types) { + STIX2IOCDto ioc = randomIocDto(new IOCType(type)); + String json = toJsonString(ioc); + + // Replace the IOCType with a fake type + String fakeType = "fake" + type; + final String invalidJson = json.replace(type, fakeType); + + SecurityAnalyticsException exception = assertThrows(SecurityAnalyticsException.class, () -> STIX2IOCDto.parse(parser(invalidJson), ioc.getId(), ioc.getVersion())); + assertEquals(RestStatus.BAD_REQUEST, exception.status()); + + String expectedError = String.format( + "Couldn't parse IOC type '%s' while deserializing STIX2IOCDto with ID '%s': ", + fakeType, + ioc.getId() + ); + assertTrue(exception.getMessage().contains(expectedError)); + } + } } diff --git a/src/test/java/org/opensearch/securityanalytics/model/STIX2IOCTests.java b/src/test/java/org/opensearch/securityanalytics/model/STIX2IOCTests.java index 251f7f97b..4323a03fb 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/STIX2IOCTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/STIX2IOCTests.java @@ -7,6 +7,9 @@ import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -32,4 +35,26 @@ public void testParseFunction() throws IOException { STIX2IOC newIoc = STIX2IOC.parse(parser(json), ioc.getId(), ioc.getVersion()); assertEqualIOCs(ioc, newIoc); } + + public void testParseFunction_invalidType() throws IOException { + // Execute test case for each IOCType + for (String type : IOCType.types) { + STIX2IOC ioc = randomIOC(new IOCType(type)); + String json = toJsonString(ioc); + + // Replace the IOCType with a fake type + String fakeType = "fake" + type; + final String invalidJson = json.replace(type, fakeType); + + SecurityAnalyticsException exception = assertThrows(SecurityAnalyticsException.class, () -> STIX2IOC.parse(parser(invalidJson), ioc.getId(), ioc.getVersion())); + assertEquals(RestStatus.BAD_REQUEST, exception.status()); + + String expectedError = String.format( + "Couldn't parse IOC type '%s' while deserializing STIX2IOC with ID '%s': ", + fakeType, + ioc.getId() + ); + assertTrue(exception.getMessage().contains(expectedError)); + } + } } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index a156344b6..f58c91831 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -1042,7 +1042,7 @@ public void testCreateDetector_verifyWorkflowCreation_success_WithoutGroupByRule " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(2, response.getHits().getTotalHits().value); @@ -1110,7 +1110,7 @@ public void testCreateDetector_verifyWorkflowCreation_success_WithGroupByRulesIn " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(2, response.getHits().getTotalHits().value); @@ -1379,7 +1379,7 @@ public void testCreateDetector_workflowWithDuplicateMonitor_failure() throws IOE " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(2, response.getHits().getTotalHits().value); @@ -1449,7 +1449,7 @@ public void testCreateDetector_verifyWorkflowExecutionBucketLevelDocLevelMonitor " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); assertEquals(2, response.getHits().getTotalHits().value); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index 75cbfd858..4aaf4f35b 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -161,7 +161,7 @@ private void validateDetectorDeletion(final String detectorId) throws IOExceptio Assert.assertEquals(0, hits.size()); } - @SuppressWarnings("unchecked") + @SuppressWarnings("unchecked") public void testCreatingADetector() throws IOException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); @@ -722,6 +722,12 @@ public void testCreatingADetectorWithAggregationRules() throws IOException { if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(secondMonitorType)){ bucketLevelMonitorId = secondMonitorId; } + String thirdMonitorId = monitorIds.get(2); + String thirdMonitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + thirdMonitorId))).get("monitor")).get("monitor_type"); + monitorTypes.add(thirdMonitorType); + if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(thirdMonitorType)){ + bucketLevelMonitorId = thirdMonitorId; + } Assert.assertTrue(Arrays.asList(MonitorType.BUCKET_LEVEL_MONITOR.getValue(), MonitorType.DOC_LEVEL_MONITOR.getValue()).containsAll(monitorTypes)); indexDoc(index, "1", randomProductDocument()); @@ -775,7 +781,13 @@ public void testAggRuleCount() throws IOException { Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); - String bucketLevelMonitorId = ((List) (detectorAsMap).get("monitor_id")).get(1); + String bucketLevelMonitorId = ""; + Map monitorOpts = ((Map) (detectorAsMap).get("bucket_monitor_id_rule_id")); + for (Map.Entry monitorOpt: monitorOpts.entrySet()) { + if (!(monitorOpt.getKey().equals("-1") || monitorOpt.getKey().equals("chained_findings_monitor"))) { + bucketLevelMonitorId = monitorOpt.getValue().toString(); + } + } // condition: sel | count(*) by name > 2 indexDoc(index, "1", randomProductDocument()); indexDoc(index, "2", randomProductDocument()); @@ -840,7 +852,7 @@ public void testUpdateADetector() throws IOException { " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); Assert.assertEquals(5, response.getHits().getTotalHits().value); String rule = randomRule(); @@ -869,7 +881,7 @@ public void testUpdateADetector() throws IOException { " }\n" + " }\n" + "}"; - response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); Assert.assertEquals(6, response.getHits().getTotalHits().value); } @@ -1056,10 +1068,10 @@ public void testDeletingADetector_single_Monitor() throws IOException { Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -1069,12 +1081,12 @@ public void testDeletingADetector_single_Monitor() throws IOException { String detectorId1 = createDetector(detector1); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -1096,12 +1108,12 @@ public void testDeletingADetector_single_Monitor() throws IOException { String detectorId2 = createDetector(detector2); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); hit = hits.get(0); @@ -1137,22 +1149,22 @@ public void testDeletingADetector_single_Monitor() throws IOException { Assert.assertFalse(doesIndexExist(String.format(Locale.ROOT, ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); } @@ -1166,10 +1178,10 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -1179,12 +1191,12 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc String detectorId1 = createDetector(detector1); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -1206,12 +1218,12 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc String detectorId2 = createDetector(detector2); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); hit = hits.get(0); @@ -1247,26 +1259,27 @@ public void testDeletingADetector_single_Monitor_workflow_enabled() throws IOExc Assert.assertFalse(doesIndexExist(String.format(Locale.ROOT, ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId1 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId2 + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); } + @SuppressWarnings("unchecked") public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() throws IOException { String index1 = "test_index_1"; createIndex(index1, Settings.EMPTY); @@ -1284,6 +1297,10 @@ public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() thro List.of(index1) ); String detectorId1 = createDetector(detector1); + Response response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; // Create detector #2 of type test_windows Detector detector2 = randomDetectorWithTriggers( @@ -1293,29 +1310,19 @@ public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() thro ); String detectorId2 = createDetector(detector2); - - Assert.assertTrue(doesIndexExist(".opensearch-sap-test_windows-detectors-queries-000001")); - Assert.assertTrue(doesIndexExist(".opensearch-sap-test_windows-detectors-queries-000002")); - - // Check if both query indices have proper settings applied from index template - Map settings = getIndexSettingsAsMap(".opensearch-sap-test_windows-detectors-queries-000001"); - assertTrue(settings.containsKey("index.analysis.char_filter.rule_ws_filter.pattern")); - assertTrue(settings.containsKey("index.hidden")); - settings = getIndexSettingsAsMap(".opensearch-sap-test_windows-detectors-queries-000002"); - assertTrue(settings.containsKey("index.analysis.char_filter.rule_ws_filter.pattern")); - assertTrue(settings.containsKey("index.hidden")); + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId2 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex2 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), null); Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); - // We deleted 1 detector, but 1 detector with same type exists, so we expect queryIndex to be present - Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); - Assert.assertTrue(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000002", "test_windows"))); deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId2, Collections.emptyMap(), null); Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); // We deleted all detectors of type windows, so we expect that queryIndex is deleted - Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); - Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000002", "test_windows"))); + Assert.assertFalse(doesIndexExist(ruleTopicIndex1)); + Assert.assertFalse(doesIndexExist(ruleTopicIndex2)); String request = "{\n" + " \"query\" : {\n" + @@ -1461,9 +1468,9 @@ public void testCreatingADetectorWithTimestampFieldAliasMapping_verifyTimeRangeI Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI); updateRequest.setJsonEntity(MediaTypeRegistry.JSON.contentBuilder().map(Map.of( - "index_name", index, - "field", "time", - "alias", "timestamp")) + "index_name", index, + "field", "time", + "alias", "timestamp")) .toString()); Response apiResponse = client().performRequest(updateRequest); assertEquals(HttpStatus.SC_OK, apiResponse.getStatusLine().getStatusCode()); @@ -1722,4 +1729,157 @@ public void testDetector_withAlias_endToEnd_success() throws IOException { List findings = (List) getFindingsBody.get("findings"); Assert.assertEquals(findings.size(), 1); } + + @SuppressWarnings("unchecked") + public void testCreatingDetectorWithDynamicQueryIndexDisabledAndThenEnabledToUpdate() throws IOException { + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "false"); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId1 = responseBody.get("_id").toString(); + + detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + + String detectorId2 = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId1 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; + hits = executeSearch(Detector.DETECTORS_INDEX, request); + hit = hits.get(0); + + String monitorId2 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId1, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + executeResponse = executeAlertingMonitor(monitorId2, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + Assert.assertTrue(doesIndexExist(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "-000001")); + + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "true"); + + Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); + + indexDoc(index, "2", randomDoc()); + + executeResponse = executeAlertingMonitor(monitorId1, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + executeResponse = executeAlertingMonitor(monitorId2, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; + Assert.assertTrue(doesIndexExist(ruleTopicIndex1)); + } + + @SuppressWarnings("unchecked") + public void testCreatingDetectorWithDynamicQueryIndexEnabledAndThenDisabled() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId1 = responseBody.get("_id").toString(); + + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + String ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; + Assert.assertTrue(doesIndexExist(ruleTopicIndex1)); + Assert.assertFalse(doesIndexExist(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "-000001")); + + updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "false"); + + Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); + + response = makeRequest(client(), "POST", ".opensearch-sap-detectors-config/_search", Map.of(), + new StringEntity("{\"query\": {\"match\": {\"_id\": \"" + detectorId1 + "\"}}}"), new BasicHeader("Content-Type", "application/json")); + ruleTopicIndex1 = ((Map) ((Map) ((List>) ((Map) responseAsMap(response).get("hits")) + .get("hits")).get(0).get("_source")).get("detector")).get("rule_topic_index").toString() + "-000001"; + Assert.assertTrue(doesIndexExist(ruleTopicIndex1)); + + detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + Assert.assertTrue(doesIndexExist(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "-000001")); + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java index 6207d57b7..bc86e11a1 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ListIOCsRestApiIT.java @@ -1,165 +1,350 @@ -///* -// * Copyright OpenSearch Contributors -// * SPDX-License-Identifier: Apache-2.0 -// */ -// -//package org.opensearch.securityanalytics.resthandler; -// -//import org.junit.After; -//import org.junit.Assert; -//import org.opensearch.client.Response; -//import org.opensearch.client.WarningFailureException; -//import org.opensearch.common.settings.Settings; -//import org.opensearch.commons.alerting.model.Table; -//import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; -//import org.opensearch.securityanalytics.TestHelpers; -//import org.opensearch.securityanalytics.action.ListIOCsActionRequest; -//import org.opensearch.securityanalytics.action.ListIOCsActionResponse; -//import org.opensearch.securityanalytics.commons.model.IOCType; -//import org.opensearch.securityanalytics.commons.model.STIX2; -//import org.opensearch.securityanalytics.model.STIX2IOC; -//import org.opensearch.securityanalytics.services.STIX2IOCFeedStore; -//import org.opensearch.securityanalytics.util.STIX2IOCGenerator; -// -//import java.io.IOException; -//import java.time.Instant; -//import java.util.Arrays; -//import java.util.Collections; -//import java.util.Comparator; -//import java.util.HashMap; -//import java.util.List; -//import java.util.Map; -//import java.util.stream.Collectors; -//import java.util.stream.IntStream; -// -//public class ListIOCsRestApiIT extends SecurityAnalyticsRestTestCase { -// private final String indexMapping = "\"properties\": {\n" + -// " \"stix2_ioc\": {\n" + -// " \"dynamic\": \"false\",\n" + -// " \"properties\": {\n" + -// " \"name\": {\n" + -// " \"type\": \"keyword\"\n" + -// " },\n" + -// " \"type\": {\n" + -// " \"type\": \"keyword\"\n" + -// " },\n" + -// " \"value\": {\n" + -// " \"type\": \"keyword\"\n" + -// " },\n" + -// " \"severity\": {\n" + -// " \"type\": \"keyword\"\n" + -// " },\n" + -// " \"spec_version\": {\n" + -// " \"type\": \"keyword\"\n" + -// " },\n" + -// " \"created\": {\n" + -// " \"type\": \"date\"\n" + -// " },\n" + -// " \"modified\": {\n" + -// " \"type\": \"date\"\n" + -// " },\n" + -// " \"description\": {\n" + -// " \"type\": \"text\"\n" + -// " },\n" + -// " \"labels\": {\n" + -// " \"type\": \"keyword\"\n" + -// " },\n" + -// " \"feed_id\": {\n" + -// " \"type\": \"keyword\"\n" + -// " }\n" + -// " }\n" + -// " }\n" + -// " }"; -// -// private String testFeedSourceConfigId; -// private String indexName; -// ListIOCsActionRequest request; -// -// @After -// public void cleanUp() throws IOException { -//// deleteIndex(indexName); -// -// testFeedSourceConfigId = null; -// indexName = null; -// request = null; -// } -// -// public void test_retrievesIOCs() throws IOException { -// // Create index with mappings -// testFeedSourceConfigId = TestHelpers.randomLowerCaseString(); -// indexName = STIX2IOCFeedStore.getIocIndexAlias(testFeedSourceConfigId); -// -// try { -// createIndex(indexName, Settings.EMPTY, indexMapping); -// } catch (WarningFailureException warningFailureException) { -// // Warns that index names starting with "." will be deprecated, but still creates the index -// } catch (Exception e) { -// fail(String.format("Test index creation failed with error: %s", e)); -// } -// -// // Ingest IOCs -// List iocs = IntStream.range(0, 5) -// .mapToObj(i -> STIX2IOCGenerator.randomIOC()) -// .collect(Collectors.toList()); -// for (STIX2IOC ioc : iocs) { -// indexDoc(indexName, "", STIX2IOCGenerator.toJsonString(ioc)); -// } -// -// request = new ListIOCsActionRequest( -// Arrays.asList(ListIOCsActionRequest.ALL_TYPES_FILTER), -// Arrays.asList(""), new Table( -// "asc", -// "name", -// null, -// iocs.size() + 1, -// 0, -// null) -// ); -// Map params = new HashMap<>(); -// params.put("sortString", request.getTable().getSortString()); -// params.put("size", request.getTable().getSize() + ""); -// params.put("sortOrder", request.getTable().getSortOrder()); -// params.put("searchString", request.getTable().getSearchString() == null ? "" : request.getTable().getSearchString()); -// params.put(ListIOCsActionRequest.TYPE_FIELD, String.join(",", request.getTypes())); -// params.put(STIX2IOC.FEED_ID_FIELD, String.join(",", request.getFeedIds())); -// -// // Retrieve IOCs -// Response response = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(request), params, null); -// Assert.assertEquals(200, response.getStatusLine().getStatusCode()); -// Map respMap = asMap(response); -// -// // Evaluate response -// int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); -// assertEquals(iocs.size(), totalHits); -// -// List> hits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); -// assertEquals(iocs.size(), hits.size()); -// -// // Sort for easy comparison -// iocs.sort(Comparator.comparing(STIX2IOC::getName)); -// hits.sort(Comparator.comparing(hit -> (String) hit.get(STIX2IOC.NAME_FIELD))); -// -// for (int i = 0; i < iocs.size(); i++) { -// Map hit = hits.get(i); -// STIX2IOC newIoc = new STIX2IOC( -// (String) hit.get(STIX2IOC.ID_FIELD), -// (String) hit.get(STIX2IOC.NAME_FIELD), -// IOCType.valueOf((String) hit.get(STIX2IOC.TYPE_FIELD)), -// (String) hit.get(STIX2IOC.VALUE_FIELD), -// (String) hit.get(STIX2IOC.SEVERITY_FIELD), -// Instant.parse((String) hit.get(STIX2IOC.CREATED_FIELD)), -// Instant.parse((String) hit.get(STIX2IOC.MODIFIED_FIELD)), -// (String) hit.get(STIX2IOC.DESCRIPTION_FIELD), -// (List) hit.get(STIX2IOC.LABELS_FIELD), -// (String) hit.get(STIX2IOC.SPEC_VERSION_FIELD), -// (String) hit.get(STIX2IOC.FEED_ID_FIELD), -// (String) hit.get(STIX2IOC.FEED_NAME_FIELD), -// Long.parseLong(String.valueOf(hit.get(STIX2IOC.VERSION_FIELD))) -// // TODO implement DetailedSTIX2IOCDto.NUM_FINDINGS_FIELD check when GetFindings API is added -// ); -//// fixme STIX2IOCGenerator.assertEqualIOCs(iocs.get(i), newIoc); -// } -// } -// -// // TODO: Implement additional tests using various query param combinations -//} +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.resthandler; + +import org.junit.Assert; +import org.opensearch.client.Response; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import org.opensearch.securityanalytics.TestHelpers; +import org.opensearch.securityanalytics.model.DetailedSTIX2IOCDto; +import org.opensearch.securityanalytics.model.threatintel.IocFinding; +import org.opensearch.securityanalytics.model.threatintel.IocWithFeeds; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionResponse; +import org.opensearch.securityanalytics.commons.model.IOCType; +import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.model.STIX2IOCDto; +import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; +import org.opensearch.securityanalytics.threatIntel.iocscan.dao.IocFindingService; +import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.util.STIX2IOCGenerator; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class ListIOCsRestApiIT extends SecurityAnalyticsRestTestCase { + + public void testListIOCsWithNoFindingsIndex() throws IOException { + // Delete findings system indexes if they exist + try { + makeRequest(client(), "DELETE", IocFindingService.IOC_FINDING_INDEX_PATTERN_REGEXP, Collections.emptyMap(), null); + } catch (IndexNotFoundException indexNotFoundException) { + logger.info("No threat intel findings indexes to delete."); + } catch (Exception e) { + logger.error(e.getMessage()); + } + + // Create IOCs + String searchString = "test-list-iocs-no-findings-index"; + Map iocs = new HashMap<>(); + for (int i = 0; i < 100; i++) { + String iocId = searchString + "-" + i; + iocs.put( + iocId, + new STIX2IOCDto( + iocId, + iocId + "-name", + new IOCType(IOCType.IPV4_TYPE), + "ipv4value" + i, + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L + ) + ); + } + + // Creating source config + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + "test_list_ioc_" + searchString, + "STIX", + SourceConfigType.IOC_UPLOAD, + null, + null, + null, + new IocUploadSource(null, new ArrayList<>(iocs.values())), + null, + null, + null, + null, + null, + null, + null, + false, + List.of(IOCType.IPV4_TYPE), + true + ); + + // Create the IOC system indexes using IOC_UPLOAD config + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + + // Call ListIOCs API + Map params = Map.of( + "searchString", searchString, + "size", "10000" + ); + Response iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), params, null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + Map respMap = asMap(iocResponse); + + // Evaluate response + int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(iocs.size(), totalHits); + + List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(iocs.size(), iocHits.size()); + + iocHits.forEach((hit) -> { + String iocId = (String) hit.get(STIX2IOC.ID_FIELD); + String iocName = (String) hit.get(STIX2IOC.NAME_FIELD); + String iocValue = (String) hit.get(STIX2IOC.VALUE_FIELD); + + STIX2IOCDto iocDto = iocs.get(iocId); + assertNotNull(iocDto); + + assertEquals(iocDto.getId(), iocId); + assertEquals(iocDto.getName(), iocName); + assertEquals(iocDto.getValue(), iocValue); + + int findingsNum = (int) hit.get(DetailedSTIX2IOCDto.NUM_FINDINGS_FIELD); + int expectedNumFindings = 0; + assertEquals(expectedNumFindings, findingsNum); + }); + } + + public void testListIOCsBySearchString() throws IOException { + String searchString = "test-search-string"; + List iocs = List.of( + // The 'name' field matches the searchString + new STIX2IOCDto( + "id1", + searchString, + new IOCType(IOCType.IPV4_TYPE), + "ipv4value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L + ), + // The 'value' field matches the searchString + new STIX2IOCDto( + "id2", + TestHelpers.randomLowerCaseString(), + new IOCType(IOCType.IPV4_TYPE), + searchString, + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L + ), + // No fields match the searchString + new STIX2IOCDto( + "id3", + "name", + new IOCType(IOCType.IPV4_TYPE), + "ipv4value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L + ) + ); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + "test_list_ioc_searchstring", + "STIX", + SourceConfigType.IOC_UPLOAD, + null, + null, + null, + new IocUploadSource(null, iocs), + null, + null, + null, + null, + null, + null, + null, + false, + List.of(IOCType.IPV4_TYPE), + true + ); + + // Create the IOC system indexes using IOC_UPLOAD config + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + + // Retrieve IOCs matching searchString + Response iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("searchString", searchString), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + Map respMap = asMap(iocResponse); + + // Evaluate response + int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(2, totalHits); + + List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(2, iocHits.size()); + + int nameMatch = (int) iocHits.stream().filter((hit) -> Objects.equals(hit.get(STIX2IOC.NAME_FIELD), searchString)).count(); + int valueMatch = (int) iocHits.stream().filter((hit) -> Objects.equals(hit.get(STIX2IOC.VALUE_FIELD), searchString)).count(); + assertEquals(1, nameMatch); + assertEquals(1, valueMatch); + } + + // TODO: Implement additional tests using various query param combinations + + public void testListIOCsNumFindings() throws Exception { + // Create IOCs + String searchString = "test-list-iocs-num-findings"; + List iocs = new ArrayList<>(); + Map> iocIdFindingsNum = new HashMap<>(); + for (int i = 0; i < 5; i++) { + String iocId = searchString + "-" + i; + iocs.add( + new STIX2IOCDto( + iocId, + iocId + "-name", + new IOCType(IOCType.IPV4_TYPE), + "ipv4value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L + ) + ); + + // Confirming the ListIOCs API can return a findings count greater than 10,000 by giving the first IOC 10,005 findings + int numFindings = i == 0 ? 10005 : randomInt(10); + List iocFindings = generateIOCMatches(numFindings, iocId); + + // Tracking the number of findings expected for each IOC + iocIdFindingsNum.put(iocId, iocFindings); + } + + // Creating source config + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + "test_list_ioc_" + searchString, + "STIX", + SourceConfigType.IOC_UPLOAD, + null, + null, + null, + new IocUploadSource(null, iocs), + null, + null, + null, + null, + null, + null, + null, + false, + List.of(IOCType.IPV4_TYPE), + true + ); + + // Create the IOC system indexes using IOC_UPLOAD config + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + + // Generate IOC matches + for (Map.Entry> entry : iocIdFindingsNum.entrySet()) { + ingestIOCMatches(entry.getValue()); + } + + // Call ListIOCs API + Response iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("searchString", searchString), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + Map respMap = asMap(iocResponse); + + // Evaluate response + int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(iocs.size(), totalHits); + + List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(iocs.size(), iocHits.size()); + + iocHits.forEach((hit) -> { + String iocId = (String) hit.get(STIX2IOC.ID_FIELD); + int findingsNum = (int) hit.get(DetailedSTIX2IOCDto.NUM_FINDINGS_FIELD); + int expectedNumFindings = iocIdFindingsNum.get(iocId).size(); + assertEquals(expectedNumFindings, findingsNum); + }); + } + + private List generateIOCMatches(int numMatches, String iocId) { + List iocFindings = new ArrayList<>(); + String monitorId = randomAlphaOfLength(10); + String monitorName = randomAlphaOfLength(10); + for (int i = 0; i < numMatches; i++) { + iocFindings.add(new IocFinding( + randomAlphaOfLength(10), + randomList(1, 10, () -> randomAlphaOfLength(10)),//docIds + randomList(1, 10, () -> new IocWithFeeds( + iocId, + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10)) + ), //feedIds + monitorId, + monitorName, + randomAlphaOfLength(10), + IOCType.IPV4_TYPE, + Instant.now(), + randomAlphaOfLength(10) + )); + } + return iocFindings; + } + + private void ingestIOCMatches(List iocFindings) throws IOException { + for (IocFinding iocFinding: iocFindings) { + makeRequest(client(), "POST", IocFindingService.IOC_FINDING_ALIAS_NAME + "/_doc?refresh", Map.of(), + toHttpEntity(iocFinding)); + } + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java index 6d7f4b550..9c258e0e8 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java @@ -110,14 +110,14 @@ public void testCreatingARule_custom_category() throws IOException { } catch (ResponseException e) { assertEquals(HttpStatus.SC_BAD_REQUEST, e.getResponse().getStatusLine().getStatusCode()); Assert.assertTrue( - e.getMessage().contains("Invalid rule category") + e.getMessage().contains("Invalid rule category") ); } } public void testCreatingAggregationRule() throws SigmaError, IOException { Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", "windows"), - new StringEntity(countAggregationTestRule()), new BasicHeader("Content-Type", "application/json")); + new StringEntity(countAggregationTestRule()), new BasicHeader("Content-Type", "application/json")); Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); @@ -130,24 +130,24 @@ public void testCreatingAggregationRule() throws SigmaError, IOException { String index = Rule.CUSTOM_RULES_INDEX; String request = "{\n" + - " \"query\": {\n" + - " \"nested\": {\n" + - " \"path\": \"rule\",\n" + - " \"query\": {\n" + - " \"bool\": {\n" + - " \"must\": [\n" + - " { \"match\": {\"rule.category\": \"windows\"}}\n" + - " ]\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - "}"; + " \"query\": {\n" + + " \"nested\": {\n" + + " \"path\": \"rule\",\n" + + " \"query\": {\n" + + " \"bool\": {\n" + + " \"must\": [\n" + + " { \"match\": {\"rule.category\": \"windows\"}}\n" + + " ]\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(index, request); XContentParser xcp = XContentType.JSON.xContent() - .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, hits.get(0).getSourceAsString()); + .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, hits.get(0).getSourceAsString()); Rule result = Rule.docParse(xcp, null, null); Assert.assertEquals(1, result.getAggregationQueries().size()); @@ -728,7 +728,7 @@ public void testDeletingUsedRule() throws IOException { " }\n" + " }\n" + "}"; - List hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request); + List hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request); Assert.assertEquals(2, hits.size()); Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.RULE_BASE_URI + "/" + createdId, Collections.singletonMap("forced", "true"), null); @@ -741,7 +741,7 @@ public void testDeletingUsedRule() throws IOException { " }\n" + " }\n" + "}"; - hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request); + hits = executeSearch(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request); Assert.assertEquals(0, hits.size()); index = Rule.CUSTOM_RULES_INDEX; diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java index 2725d71ee..e3460e561 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SATIFSourceConfigRestApiIT.java @@ -14,6 +14,7 @@ import org.junit.Before; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; +import org.opensearch.core.rest.RestStatus; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; @@ -477,97 +478,103 @@ public void testRetrieveMultipleIOCTypesSuccessfully() throws IOException, Inter // Only run tests when required system params are provided if (!canRunTests) return; - // Generate test IOCs for each type, and upload them to S3 - int numOfIOCs = 5; - stix2IOCGenerator = new STIX2IOCGenerator(); - s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator); - List allIocs = stix2IOCGenerator.getIocs(); - assertEquals("Incorrect total number of test IOCs generated.", IOCType.types.size() * numOfIOCs, allIocs.size()); + List numOfIOCsList = List.of( + 5, + 1000001 // Over 1 million IOCs + ); - // Create test feed - String feedName = "download_test_feed_name"; - String feedFormat = "STIX2"; - SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; - IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); + for (int numOfIOCs : numOfIOCsList) { + // Generate test IOCs for each type, and upload them to S3 + stix2IOCGenerator = new STIX2IOCGenerator(); + s3ObjectGenerator.write(numOfIOCs, objectKey, stix2IOCGenerator); + List allIocs = stix2IOCGenerator.getIocs(); + assertEquals("Incorrect total number of test IOCs generated.", IOCType.types.size() * numOfIOCs, allIocs.size()); - SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( - null, - null, - feedName, - feedFormat, - sourceConfigType, - null, - null, - Instant.now(), - source, - null, - Instant.now(), - schedule, - null, - null, - Instant.now(), - null, - true, - IOCType.types, - true - ); + // Create test feed + String feedName = "download_test_feed_name"; + String feedFormat = "STIX2"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); - // Confirm test feed was created successfully - Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); - Assert.assertEquals(201, response.getStatusLine().getStatusCode()); - Map responseBody = asMap(response); + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + true, + IOCType.types, + true + ); - String createdId = responseBody.get("_id").toString(); - Assert.assertNotEquals("Response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + // Confirm test feed was created successfully + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("Response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); - // Wait for feed to execute - String firstUpdatedTime = (String) ((Map)responseBody.get("source_config")).get("last_refreshed_time"); - waitUntil(() -> { - try { - return verifyJobRan(createdId, firstUpdatedTime); - } catch (IOException e) { - throw new RuntimeException("failed to verify that job ran"); - } - }, 240, TimeUnit.SECONDS); - // Confirm IOCs were ingested to system index for the feed - String indexName = getAllIocIndexPatternById(createdId); + // Wait for feed to execute + String firstUpdatedTime = (String) ((Map)responseBody.get("source_config")).get("last_refreshed_time"); + waitUntil(() -> { + try { + return verifyJobRan(createdId, firstUpdatedTime); + } catch (IOException e) { + throw new RuntimeException("failed to verify that job ran"); + } + }, 240, TimeUnit.SECONDS); - String request = "{\n" + - " \"size\" : 10000,\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; - List hits = executeSearch(indexName, request); + // Confirm IOCs were ingested to system index for the feed + String indexName = getAllIocIndexPatternById(createdId); - // Confirm expected number of results are returned - assertEquals(allIocs.size(), hits.size()); - List> iocHits = hits.stream() - .map(SearchHit::getSourceAsMap) - .collect(Collectors.toList()); + String request = "{\n" + + " \"size\" : 10000,\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(indexName, request); - // Sort IOC lists for easy comparison - allIocs.sort(Comparator.comparing(STIX2IOC::getName)); - iocHits.sort(Comparator.comparing(ioc -> (String) ioc.get(STIX2IOC.NAME_FIELD))); + // Confirm expected number of results are returned + assertEquals(allIocs.size(), hits.size()); + List> iocHits = hits.stream() + .map(SearchHit::getSourceAsMap) + .collect(Collectors.toList()); - // Confirm expected IOCs have been ingested - for (int i = 0; i < allIocs.size(); i++) { - assertEquals(stix2IOCGenerator.getIocs().get(i).getName(), iocHits.get(i).get(STIX2IOC.NAME_FIELD)); - assertEquals(stix2IOCGenerator.getIocs().get(i).getType().toString(), IOCType.fromString((String) iocHits.get(i).get(STIX2IOC.TYPE_FIELD))); - assertEquals(stix2IOCGenerator.getIocs().get(i).getValue(), iocHits.get(i).get(STIX2IOC.VALUE_FIELD)); - assertEquals(stix2IOCGenerator.getIocs().get(i).getSeverity(), iocHits.get(i).get(STIX2IOC.SEVERITY_FIELD)); + // Sort IOC lists for easy comparison + allIocs.sort(Comparator.comparing(STIX2IOC::getName)); + iocHits.sort(Comparator.comparing(ioc -> (String) ioc.get(STIX2IOC.NAME_FIELD))); + + // Confirm expected IOCs have been ingested + for (int i = 0; i < allIocs.size(); i++) { + assertEquals(stix2IOCGenerator.getIocs().get(i).getName(), iocHits.get(i).get(STIX2IOC.NAME_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getType().toString(), IOCType.fromString((String) iocHits.get(i).get(STIX2IOC.TYPE_FIELD))); + assertEquals(stix2IOCGenerator.getIocs().get(i).getValue(), iocHits.get(i).get(STIX2IOC.VALUE_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getSeverity(), iocHits.get(i).get(STIX2IOC.SEVERITY_FIELD)); - // TODO troubleshoot instant assertions + // TODO troubleshoot instant assertions // assertEquals(stix2IOCGenerator.getIocs().get(i).getCreated().toString(), iocHits.get(i).get(STIX2IOC.CREATED_FIELD)); // assertEquals(stix2IOCGenerator.getIocs().get(i).getModified().toString(), iocHits.get(i).get(STIX2IOC.MODIFIED_FIELD)); - assertEquals(stix2IOCGenerator.getIocs().get(i).getDescription(), iocHits.get(i).get(STIX2IOC.DESCRIPTION_FIELD)); - assertEquals(stix2IOCGenerator.getIocs().get(i).getLabels(), iocHits.get(i).get(STIX2IOC.LABELS_FIELD)); - assertEquals(createdId, iocHits.get(i).get(STIX2IOC.FEED_ID_FIELD)); - assertEquals(stix2IOCGenerator.getIocs().get(i).getSpecVersion(), iocHits.get(i).get(STIX2IOC.SPEC_VERSION_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getDescription(), iocHits.get(i).get(STIX2IOC.DESCRIPTION_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getLabels(), iocHits.get(i).get(STIX2IOC.LABELS_FIELD)); + assertEquals(createdId, iocHits.get(i).get(STIX2IOC.FEED_ID_FIELD)); + assertEquals(stix2IOCGenerator.getIocs().get(i).getSpecVersion(), iocHits.get(i).get(STIX2IOC.SPEC_VERSION_FIELD)); + } } } @@ -728,7 +735,7 @@ public void testWithNoIOCsToDownload() { } } - public void testWhenBucketObjectDoesNotExist() { + public void testWhenBucketObjectDoesNotExist() throws IOException { // Only run tests when required system params are provided if (!canRunTests) return; @@ -773,13 +780,85 @@ public void testWhenBucketObjectDoesNotExist() { true ); - Exception exception = assertThrows(ResponseException.class, () -> - makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)) - ); - - String expectedError = "{\"error\":{\"root_cause\":[{\"type\":\"no_such_key_exception\",\"reason\":\"The specified key does not exist."; - assertTrue("Exception contains unexpected message: " + exception.getMessage(), exception.getMessage().contains(expectedError)); + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + } catch (ResponseException exception) { + assertEquals(RestStatus.NOT_FOUND, restStatus(exception.getResponse())); + String expectedError = "The specified key does not exist."; + assertTrue("Exception contains unexpected message: " + exception.getMessage(), exception.getMessage().contains(expectedError)); + } } + + // ensure that source config was deleted + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(0, hits.size()); + + // ensure that ioc indices were deleted + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(0, hits.size()); + } + + public void testWhenRoleArnIsEmpty() throws IOException { + // Try to create a source config with empty roleArn + source = new S3Source("bucketName", "objectKey", "region", ""); + + // Create test feed + String feedName = "download_test_feed_name"; + String feedFormat = "STIX2"; + SourceConfigType sourceConfigType = SourceConfigType.S3_CUSTOM; + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES); + List iocTypes = List.of(IOCType.IPV4_TYPE); + + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + Instant.now(), + source, + null, + Instant.now(), + schedule, + null, + null, + Instant.now(), + null, + true, + iocTypes, + true + ); + + Exception exception = assertThrows(ResponseException.class, () -> + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)) + ); + + String expectedError = "Role arn is empty or malformed"; + assertTrue("Exception contains unexpected message: " + exception.getMessage(), exception.getMessage().contains(expectedError)); + + // ensure that source config is not created + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(0, hits.size()); } /** diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java index e233d26a7..9b457f7df 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/SourceConfigWithoutS3RestApiIT.java @@ -15,26 +15,36 @@ import org.junit.Assert; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; +import org.opensearch.client.WarningFailureException; +import org.opensearch.common.settings.Settings; import org.opensearch.core.rest.RestStatus; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; -import org.opensearch.securityanalytics.action.ListIOCsActionRequest; -import org.opensearch.securityanalytics.action.ListIOCsActionResponse; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionRequest; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionResponse; import org.opensearch.securityanalytics.commons.model.IOCType; import org.opensearch.securityanalytics.model.STIX2IOCDto; import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.threatIntel.model.UrlDownloadSource; import org.opensearch.securityanalytics.util.STIX2IOCGenerator; import java.io.IOException; +import java.net.URL; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import static org.opensearch.jobscheduler.spi.utils.LockService.LOCK_INDEX_NAME; import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.JOB_INDEX_NAME; +import static org.opensearch.securityanalytics.TestHelpers.oldThreatIntelJobMapping; import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.IOC_ALL_INDEX_PATTERN; import static org.opensearch.securityanalytics.services.STIX2IOCFeedStore.getAllIocIndexPatternById; @@ -201,7 +211,7 @@ public void testCreateIocUploadSourceConfigIncorrectIocTypes() throws IOExceptio } } - public void testUpdateIocUploadSourceConfig() throws IOException, InterruptedException { + public void testUpdateIocUploadSourceConfig() throws IOException { // Create source config with IPV4 IOCs String feedName = "test_update"; String feedFormat = "STIX"; @@ -341,6 +351,10 @@ public void testUpdateIocUploadSourceConfig() throws IOException, InterruptedExc response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + createdId, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); Assert.assertEquals(RestStatus.OK, restStatus(response)); + // Ensure that old ioc indices are retained (2 created from ioc upload source config + 1 from default source config) + List findingIndices = getIocIndices(); + Assert.assertEquals(3, findingIndices.size()); + // Retrieve all IOCs by feed Ids iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null); Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); @@ -352,7 +366,281 @@ public void testUpdateIocUploadSourceConfig() throws IOException, InterruptedExc iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); assertEquals(1, iocHits.size()); - Thread.sleep(10000); + } + + public void testActivateDeactivateIocUploadSourceConfig() throws IOException { + // Create source config with IPV4 IOCs + String feedName = "test_update"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.IOC_UPLOAD; + + List iocs = List.of(new STIX2IOCDto( + "1", + "ioc", + new IOCType(IOCType.IPV4_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); + Boolean enabled = false; + List iocTypes = List.of("ipv4-addr"); + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + // create source config with ipv4 ioc type + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(response)); + Map responseBody = asMap(response); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + Assert.assertEquals("Incorrect Location header", String.format(Locale.getDefault(), "%s/%s", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, createdId), response.getHeader("Location")); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + + // ensure same number of iocs got indexed + String indexName = getAllIocIndexPatternById(createdId); + hits = executeSearch(indexName, request); + Assert.assertEquals(iocs.size(), hits.size()); + + // Retrieve all IOCs by feed Ids + Response iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + Map respMap = asMap(iocResponse); + + // Evaluate response + int totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(iocs.size(), totalHits); + + List> iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(iocs.size(), iocHits.size()); + + // update source config to contain only hashes as an ioc type + iocs = Collections.emptyList(); + + iocUploadSource = new IocUploadSource(null, iocs); + iocTypes = List.of("hashes"); + saTifSourceConfigDto = new SATIFSourceConfigDto( + saTifSourceConfigDto.getId(), + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, false + ); + + // update source config with hashes ioc type + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + createdId, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.OK, restStatus(response)); + Map updateResponseAsMap = asMap(response); + assertNotNull(updateResponseAsMap); + assertTrue(updateResponseAsMap.containsKey("source_config")); + HashMap scr = (HashMap) updateResponseAsMap.get("source_config"); + assertTrue(scr.containsKey("enabled")); + assertFalse((Boolean) scr.get("enabled")); + assertTrue(scr.containsKey("enabled_for_scan")); + assertFalse((Boolean) scr.get("enabled_for_scan")); + + // Ensure that old ioc indices are retained (2 created from ioc upload source config + 1 from default source config) + List findingIndices = getIocIndices(); + Assert.assertEquals(2, findingIndices.size()); + + // Retrieve all IOCs by feed Ids + iocResponse = makeRequest(client(), "GET", STIX2IOCGenerator.getListIOCsURI(), Map.of("feed_ids", createdId + ",random"), null); + Assert.assertEquals(RestStatus.OK, restStatus(iocResponse)); + respMap = asMap(iocResponse); + + // Evaluate response - there should only be 1 ioc indexed according to the ioc type + totalHits = (int) respMap.get(ListIOCsActionResponse.TOTAL_HITS_FIELD); + assertEquals(1, totalHits); + + iocHits = (List>) respMap.get(ListIOCsActionResponse.HITS_FIELD); + assertEquals(1, iocHits.size()); + + saTifSourceConfigDto = new SATIFSourceConfigDto( + saTifSourceConfigDto.getId(), + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + // update source config with hashes ioc type + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + createdId, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.OK, restStatus(response)); + updateResponseAsMap = asMap(response); + assertNotNull(updateResponseAsMap); + assertTrue(updateResponseAsMap.containsKey("source_config")); + scr = (HashMap) updateResponseAsMap.get("source_config"); + assertTrue(scr.containsKey("enabled")); + assertFalse((Boolean) scr.get("enabled")); // since its not url_download type, this flag should remain unaffected by the activate action in update source api + assertTrue(scr.containsKey("enabled_for_scan")); + assertTrue((Boolean) scr.get("enabled_for_scan")); + } + + public void testActivateDeactivateUrlDownloadSourceConfig() throws IOException { + // Search source configs when none are created + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + + // Search all source configs + Response sourceConfigResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/_search", Collections.emptyMap(), new StringEntity(request), new BasicHeader("Content-type", "application/json")); + Assert.assertEquals(RestStatus.OK, restStatus(sourceConfigResponse)); + Map responseBody = asMap(sourceConfigResponse); + + // Expected value is 1 - only default source config + Assert.assertEquals(1, ((Map) ((Map) responseBody.get("hits")).get("total")).get("value")); + + // Update default source config + String feedName = "test_update_default"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.URL_DOWNLOAD; + UrlDownloadSource urlDownloadSource = new UrlDownloadSource(new URL("https://reputation.alienvault.com/reputation.generic"), "csv", false,0); + Boolean enabled = false; + List iocTypes = List.of("ipv4-addr"); + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + String id = "alienvault_reputation_ip_database"; + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + id, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + urlDownloadSource, + null, + null, + schedule, + null, + null, + null, + null, + enabled, + iocTypes, false + ); + + // update default source config with enabled_for_scan updated + Response response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.OK, restStatus(response)); + Map updateResponseAsMap = asMap(response); + assertNotNull(updateResponseAsMap); + assertTrue(updateResponseAsMap.containsKey("source_config")); + HashMap scr = (HashMap) updateResponseAsMap.get("source_config"); + assertTrue(scr.containsKey("enabled")); + assertFalse((Boolean) scr.get("enabled")); + assertTrue(scr.containsKey("enabled_for_scan")); + assertFalse((Boolean) scr.get("enabled_for_scan")); + + // Ensure that only 1 ioc index is present from default source + List findingIndices = getIocIndices(); + Assert.assertEquals(1, findingIndices.size()); + + // try to update default source config again to ensure operation is not accepted when enabled_for_scan is unchanged + try { + makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("unsupported_operation_exception")); + } + // activate source + saTifSourceConfigDto = new SATIFSourceConfigDto( + id, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + urlDownloadSource, + null, + null, + schedule, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + // update default source config with enabled_for_scan updated + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.OK, restStatus(response)); + updateResponseAsMap = asMap(response); + assertNotNull(updateResponseAsMap); + assertTrue(updateResponseAsMap.containsKey("source_config")); + scr = (HashMap) updateResponseAsMap.get("source_config"); + assertTrue(scr.containsKey("enabled")); + assertTrue((Boolean) scr.get("enabled")); + assertTrue(scr.containsKey("enabled_for_scan")); + assertTrue((Boolean) scr.get("enabled_for_scan")); } public void testDeleteIocUploadSourceConfigAndAllIocs() throws IOException { @@ -434,6 +722,10 @@ public void testDeleteIocUploadSourceConfigAndAllIocs() throws IOException { // ensure all iocs are deleted hits = executeSearch(IOC_ALL_INDEX_PATTERN, request); Assert.assertEquals(0, hits.size()); + + // ensure that lock is deleted + hits = executeSearch(LOCK_INDEX_NAME,request); + Assert.assertEquals(0, hits.size()); } public void testRefreshIocUploadSourceConfigFailure() throws IOException { @@ -582,4 +874,188 @@ public void testSearchIocUploadSourceConfig() throws IOException { Assert.assertEquals(2, ((Map) ((Map) respMap.get("hits")).get("total")).get("value")); } + public void testSearchAndCreateDefaultSourceConfig() throws IOException { + // Search source configs when none are created + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + + // Search all source configs + Response sourceConfigResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/_search", Collections.emptyMap(), new StringEntity(request), new BasicHeader("Content-type", "application/json")); + Assert.assertEquals(RestStatus.OK, restStatus(sourceConfigResponse)); + Map responseBody = asMap(sourceConfigResponse); + + // Expected value is 1 - only default source config + Assert.assertEquals(1, ((Map) ((Map) responseBody.get("hits")).get("total")).get("value")); + } + + public void testUpdateDefaultSourceConfigThrowsError() throws IOException { + // Search source configs when none are created + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + + // Search all source configs + Response sourceConfigResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI + "/_search", Collections.emptyMap(), new StringEntity(request), new BasicHeader("Content-type", "application/json")); + Assert.assertEquals(RestStatus.OK, restStatus(sourceConfigResponse)); + Map responseBody = asMap(sourceConfigResponse); + + // Expected value is 1 - only default source config + Assert.assertEquals(1, ((Map) ((Map) responseBody.get("hits")).get("total")).get("value")); + + // Update default source config + String feedName = "test_update_default"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.URL_DOWNLOAD; + UrlDownloadSource urlDownloadSource = new UrlDownloadSource(new URL("https://reputation.alienvault.com/reputation.generic"), "csv", false,0); + Boolean enabled = false; + List iocTypes = List.of("ipv4-addr"); + IntervalSchedule schedule = new IntervalSchedule(Instant.now(), 1, ChronoUnit.DAYS); + String id = "alienvault_reputation_ip_database"; + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + id, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + urlDownloadSource, + null, + null, + schedule, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + + // update default source config + try { + makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("unsupported_operation_exception")); + } + + // update default source config again to ensure lock was released + try { + makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI +"/" + id, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("unsupported_operation_exception")); + } + } + + public void testUpdateJobIndexMapping() throws IOException { + // Create job index with old threat intel mapping + // Try catch needed because of warning when creating a system index which is needed to replicate previous tif job mapping + try { + createIndex(JOB_INDEX_NAME, Settings.EMPTY, oldThreatIntelJobMapping()); + } catch (WarningFailureException e) { + // Ensure index was created with old mappings + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(0, hits.size()); + + Map props = getIndexMappingsAPIFlat(JOB_INDEX_NAME); + assertTrue(props.containsKey("enabled_time")); + assertTrue(props.containsKey("schedule.interval.start_time")); + assertFalse(props.containsKey("source_config.source.ioc_upload.file_name")); + assertFalse(props.containsKey("source_config.source.s3.object_key")); + } + + // Create new threat intel source config + SATIFSourceConfigDto saTifSourceConfigDto = getSatifSourceConfigDto(); + + Response makeResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(makeResponse)); + Map responseBody = asMap(makeResponse); + + String createdId = responseBody.get("_id").toString(); + Assert.assertNotEquals("response is missing Id", SATIFSourceConfigDto.NO_ID, createdId); + + int createdVersion = Integer.parseInt(responseBody.get("_version").toString()); + Assert.assertTrue("incorrect version", createdVersion > 0); + + // Ensure source config document was indexed + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(JOB_INDEX_NAME, request); + Assert.assertEquals(1, hits.size()); + + // Ensure index mappings were updated + Map props = getIndexMappingsAPIFlat(JOB_INDEX_NAME); + assertTrue(props.containsKey("source_config.source.ioc_upload.file_name")); + assertTrue(props.containsKey("source_config.source.s3.object_key")); + assertTrue(props.containsKey("source_config.description")); + assertTrue(props.containsKey("source_config.last_update_time")); + assertTrue(props.containsKey("source_config.refresh_type")); + } + + private static SATIFSourceConfigDto getSatifSourceConfigDto() { + String feedName = "test_ioc_upload"; + String feedFormat = "STIX"; + SourceConfigType sourceConfigType = SourceConfigType.IOC_UPLOAD; + + List iocs = List.of(new STIX2IOCDto( + "id", + "name", + new IOCType(IOCType.IPV4_TYPE), + "value", + "severity", + null, + null, + "description", + List.of("labels"), + "specversion", + "feedId", + "feedName", + 1L)); + + IocUploadSource iocUploadSource = new IocUploadSource(null, iocs); + Boolean enabled = false; + List iocTypes = List.of(IOCType.IPV4_TYPE); + return new SATIFSourceConfigDto( + null, + null, + feedName, + feedFormat, + sourceConfigType, + null, + null, + null, + iocUploadSource, + null, + null, + null, + null, + null, + null, + null, + enabled, + iocTypes, true + ); + } + + @Override + protected boolean preserveIndicesUponCompletion() { + return false; + } } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java index f81b913e7..77fafd157 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/ThreatIntelMonitorRestApiIT.java @@ -2,31 +2,41 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; +import org.apache.http.HttpStatus; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.junit.Assert; +import org.opensearch.client.Request; import org.opensearch.client.Response; -import org.opensearch.common.settings.Settings; +import org.opensearch.client.ResponseException; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.commons.alerting.model.IntervalSchedule; import org.opensearch.commons.alerting.model.Monitor; +import org.opensearch.commons.alerting.model.action.Action; +import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; -import org.opensearch.securityanalytics.action.ListIOCsActionRequest; +import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; +import org.opensearch.securityanalytics.threatIntel.action.ListIOCsActionRequest; import org.opensearch.securityanalytics.commons.model.IOCType; -import org.opensearch.securityanalytics.model.STIX2IOC; +import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.securityanalytics.model.STIX2IOCDto; import org.opensearch.securityanalytics.threatIntel.common.RefreshType; import org.opensearch.securityanalytics.threatIntel.common.SourceConfigType; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.iocscan.dao.ThreatIntelAlertService; import org.opensearch.securityanalytics.threatIntel.iocscan.dto.PerIocTypeScanInputDto; -import org.opensearch.securityanalytics.threatIntel.model.DefaultIocStoreConfig; -import org.opensearch.securityanalytics.threatIntel.model.S3Source; +import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelMonitorDto; import org.opensearch.securityanalytics.threatIntel.sacommons.monitor.ThreatIntelTriggerDto; import java.io.IOException; +import java.nio.charset.Charset; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -36,135 +46,378 @@ import java.util.Map; import static java.util.Collections.emptyList; +import static org.opensearch.securityanalytics.TestHelpers.randomAction; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; import static org.opensearch.securityanalytics.TestHelpers.randomIndex; +import static org.opensearch.securityanalytics.TestHelpers.randomThreatInteMonitorAction; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; import static org.opensearch.securityanalytics.threatIntel.resthandler.monitor.RestSearchThreatIntelMonitorAction.SEARCH_THREAT_INTEL_MONITOR_PATH; public class ThreatIntelMonitorRestApiIT extends SecurityAnalyticsRestTestCase { - private final String iocIndexMappings = "\"properties\": {\n" + - " \"stix2_ioc\": {\n" + - " \"properties\": {\n" + - " \"name\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"type\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"value\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"severity\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"spec_version\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"created\": {\n" + - " \"type\": \"date\"\n" + - " },\n" + - " \"modified\": {\n" + - " \"type\": \"date\"\n" + - " },\n" + - " \"description\": {\n" + - " \"type\": \"text\"\n" + - " },\n" + - " \"labels\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"feed_id\": {\n" + - " \"type\": \"keyword\"\n" + - " },\n" + - " \"feed_name\": {\n" + - " \"type\": \"keyword\"\n" + - " }\n" + - " }\n" + - " }\n" + - " }"; - - private List testIocs = new ArrayList<>(); - - public void indexSourceConfigsAndIocs(int num, List iocVals) throws IOException { - for (int i = 0; i < num; i++) { - String configId = "id" + i; - String iocActiveIndex = ".opensearch-sap-ioc-" + configId + Instant.now().toEpochMilli(); - String indexPattern = ".opensearch-sap-ioc-" + configId; - indexTifSourceConfig(num, configId, indexPattern, iocActiveIndex, i); - - // Create the index before ingesting docs to ensure the mappings are correct - createIndex(iocActiveIndex, Settings.EMPTY, iocIndexMappings); - - // Refresh testIocs list between tests - testIocs = new ArrayList<>(); - for (int i1 = 0; i1 < iocVals.size(); i1++) { - indexIocs(iocVals, iocActiveIndex, i1, configId); - } + private final Logger log = LogManager.getLogger(ThreatIntelMonitorRestApiIT.class); + + private List testIocDtos = new ArrayList<>(); + + public String indexSourceConfigsAndIocs(List iocVals) throws IOException { + testIocDtos = new ArrayList<>(); + for (int i1 = 0; i1 < iocVals.size(); i1++) { + // create IOCs + STIX2IOCDto stix2IOCDto = new STIX2IOCDto( + "id" + i1, + "random", + new IOCType(IOCType.IPV4_TYPE), + iocVals.get(i1), + "", + Instant.now(), + Instant.now(), + "", + emptyList(), + "spec", + "configId", + "", + 1L + ); + + testIocDtos.add(stix2IOCDto); } + return indexTifSourceConfig(testIocDtos); } - private void indexIocs(List iocVals, String iocIndexName, int i1, String configId) throws IOException { - String iocId = iocIndexName + i1; - STIX2IOC stix2IOC = new STIX2IOC( - iocId, - "random", - new IOCType(IOCType.IPV4_TYPE), - iocVals.get(i1), - "", - Instant.now(), - Instant.now(), - "", - emptyList(), - "spec", - configId, - "", - STIX2IOC.NO_VERSION - ); + public String indexSourceConfigsAndIocs(List ipVals, List hashVals, List domainVals) throws IOException { + testIocDtos = new ArrayList<>(); + for (int i1 = 0; i1 < ipVals.size(); i1++) { + // create IOCs + STIX2IOCDto stix2IOCDto = new STIX2IOCDto( + "id" + randomAlphaOfLength(3), + "random", + new IOCType(IOCType.IPV4_TYPE), + ipVals.get(i1), + "", + Instant.now(), + Instant.now(), + "", + emptyList(), + "spec", + "configId", + "", + 1L + ); - // Add IOC to testIocs List for future validation - testIocs.add(stix2IOC); + testIocDtos.add(stix2IOCDto); + } + for (int i1 = 0; i1 < hashVals.size(); i1++) { + // create IOCs + STIX2IOCDto stix2IOCDto = new STIX2IOCDto( + "id" + randomAlphaOfLength(3), + "random", + new IOCType(IOCType.HASHES_TYPE), + hashVals.get(i1), + "", + Instant.now(), + Instant.now(), + "", + emptyList(), + "spec", + "configId", + "", + 1L + ); - indexDoc(iocIndexName, iocId, stix2IOC.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).toString()); - List searchHits = executeSearch(iocIndexName, getMatchAllSearchRequestString(iocVals.size())); - assertEquals(searchHits.size(), i1 + 1); + testIocDtos.add(stix2IOCDto); + } + for (int i1 = 0; i1 < domainVals.size(); i1++) { + // create IOCs + STIX2IOCDto stix2IOCDto = new STIX2IOCDto( + "id" + randomAlphaOfLength(3), + "random", + new IOCType(IOCType.DOMAIN_NAME_TYPE), + domainVals.get(i1), + "", + Instant.now(), + Instant.now(), + "", + emptyList(), + "spec", + "configId", + "", + 1L + ); + + testIocDtos.add(stix2IOCDto); + } + return indexTifSourceConfig(testIocDtos); } - private void indexTifSourceConfig(int num, String configId, String indexPattern, String iocActiveIndex, int i) throws IOException { - SATIFSourceConfig config = new SATIFSourceConfig( - configId, + private String indexTifSourceConfig(List testIocDtos) throws IOException { + SATIFSourceConfigDto saTifSourceConfigDto = new SATIFSourceConfigDto( + "configId", SATIFSourceConfig.NO_VERSION, "name1", "STIX2", - SourceConfigType.S3_CUSTOM, + SourceConfigType.IOC_UPLOAD, "description", null, Instant.now(), - new S3Source("bucketname", "key", "region", "roleArn"), + new IocUploadSource(null, testIocDtos), null, Instant.now(), - new org.opensearch.jobscheduler.spi.schedule.IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES), + null, TIFJobState.AVAILABLE, RefreshType.FULL, null, null, false, - new DefaultIocStoreConfig(List.of(new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(IOCType.IPV4_TYPE), indexPattern, iocActiveIndex))), - List.of(IOCType.IPV4_TYPE), + List.of(IOCType.IPV4_TYPE, IOCType.HASHES_TYPE, IOCType.DOMAIN_NAME_TYPE), true ); - String indexName = SecurityAnalyticsPlugin.JOB_INDEX_NAME; - Response response = indexDoc(indexName, configId, config.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).toString()); + + Response makeResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_SOURCE_URI, Collections.emptyMap(), toHttpEntity(saTifSourceConfigDto)); + Assert.assertEquals(RestStatus.CREATED, restStatus(makeResponse)); + + Assert.assertEquals(201, makeResponse.getStatusLine().getStatusCode()); + Map responseBody = asMap(makeResponse); + return responseBody.get("_id").toString(); } public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", Map.of(), null); Map responseAsMap = responseAsMap(iocFindingsResponse); Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); List vals = List.of("ip1", "ip2"); - indexSourceConfigsAndIocs(1, vals); + String createdId = indexSourceConfigsAndIocs(vals); + String index = "alias1"; Map> testAlias = createTestAlias(index, 1, true); String monitorName = "test_monitor_name"; + /**create monitor */ + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto(index); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + fail(); + } catch (Exception e) { + /** creating a second threat intel monitor should fail*/ + assertTrue(e.getMessage().contains("already exists")); + } + + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Assert.assertEquals(200, executeResponse.getStatusLine().getStatusCode()); + + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + + executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + String matchAllRequest = getMatchAllRequest(); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + HashMap totalHits = (HashMap) hits.get("total"); + Integer totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 1); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(2, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + executeAlertingMonitor(monitorId, Collections.emptyMap()); + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); + + // Use ListIOCs API to confirm expected number of findings are returned + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, createdId); + Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); + Map listIocsResponseMap = responseAsMap(listIocsResponse); + List> iocsMap = (List>) listIocsResponseMap.get("iocs"); + assertEquals(2, iocsMap.size()); + iocsMap.forEach((iocDetails) -> { + String iocId = (String) iocDetails.get("id"); + int numFindings = (Integer) iocDetails.get("num_findings"); + assertTrue(testIocDtos.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertEquals(2, numFindings); + }); + + // Use ListIOCs API with large size to ensure matchQuery related bug is not throwing too many bool clauses exception + listIocsUri = String.format("?%s=%s", "size", 1000); + listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI, Collections.emptyMap(), null); + assertEquals(200, listIocsResponse.getStatusLine().getStatusCode()); + listIocsResponseMap = responseAsMap(listIocsResponse); + iocsMap = (List>) listIocsResponseMap.get("iocs"); + assertTrue(2 < iocsMap.size()); // number should be greater than custom source iocs because of default config + + //alerts via system index search + searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + // alerts via API + Map params = new HashMap<>(); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + Assert.assertEquals(4, getAlertsBody.get("total_alerts")); + + + ThreatIntelMonitorDto updateMonitorDto = new ThreatIntelMonitorDto( + monitorId, + iocScanMonitor.getName() + "update", + iocScanMonitor.getPerIocTypeScanInputList(), + new IntervalSchedule(5, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(iocScanMonitor.getTriggers().get(0), iocScanMonitor.getTriggers().get(1)) + ); + //update monitor + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), toHttpEntity(updateMonitorDto)); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + responseBody = asMap(response); + assertEquals(responseBody.get("id").toString(), monitorId); + assertEquals(((HashMap) responseBody.get("monitor")).get("name").toString(), iocScanMonitor.getName() + "update"); + + //delete + Response delete = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), null); + Assert.assertEquals(200, delete.getStatusLine().getStatusCode()); + + searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + totalHits = (HashMap) hits.get("total"); + totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 0); + } + + public void testCreateThreatIntelMonitor_configureMultipleIndicatorTypesInMonitor() throws IOException { + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List ipVals = List.of("ip1", "ip2"); + List hashVals = List.of("h1", "h2"); + List domainVals = List.of("d1", "d2"); + String createdId = indexSourceConfigsAndIocs(ipVals, hashVals, domainVals); + + String ipIndex = "ipAlias"; + createTestAlias(ipIndex, 1, true); + String hashIndex = "hashAlias"; + createTestAlias(hashIndex, 1, true); + String domainIndex = "domainAlias"; + createTestAlias(domainIndex, 1, true); + + + /**create monitor */ + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDtoWithMultipleIndicatorTypesToScan(ipIndex, hashIndex, domainIndex); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + fail(); + } catch (Exception e) { + /** creating a second threat intel monitor should fail*/ + assertTrue(e.getMessage().contains("already exists")); + } + + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Assert.assertEquals(200, executeResponse.getStatusLine().getStatusCode()); + + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : ipVals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(ipIndex, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + for (String val : hashVals) { + String doc = String.format("{\"hash\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(hashIndex, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + for (String val : domainVals) { + String doc = String.format("{\"domain\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(domainIndex, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + + executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + String matchAllRequest = getMatchAllRequest(); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + HashMap totalHits = (HashMap) hits.get("total"); + Integer totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 1); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(6, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(6, searchHits.size()); + } + + public void testCreateThreatIntelMonitor() throws IOException { + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List vals = List.of("ip1", "ip2"); + String createdId = indexSourceConfigsAndIocs(vals); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + String monitorName = "test_monitor_name"; + /**create monitor */ ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto(index); @@ -190,6 +443,8 @@ public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); try { indexDoc(index, "" + i++, doc); + indexDoc(index, "" + i++, String.format("{\"ip\":\"1.2.3.4\", \"ip1\":\"1.2.3.4\"}", val, val)); + indexDoc(index, "" + i++, String.format("{\"random\":\"%s\", \"random1\":\"%s\"}", val, val)); } catch (IOException e) { fail(); } @@ -197,7 +452,6 @@ public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); - assertEquals(1, 1); String matchAllRequest = getMatchAllRequest(); Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); @@ -233,7 +487,7 @@ public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); // Use ListIOCs API to confirm expected number of findings are returned - String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, "id0"); + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, createdId); Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); Map listIocsResponseMap = responseAsMap(listIocsResponse); List> iocsMap = (List>) listIocsResponseMap.get("iocs"); @@ -241,7 +495,7 @@ public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { iocsMap.forEach((iocDetails) -> { String iocId = (String) iocDetails.get("id"); int numFindings = (Integer) iocDetails.get("num_findings"); - assertTrue(testIocs.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertTrue(testIocDtos.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); assertEquals(2, numFindings); }); @@ -284,13 +538,15 @@ public void testCreateThreatIntelMonitor_monitorAliases() throws IOException { assertEquals(totalHitsVal.intValue(), 0); } - public void testCreateThreatIntelMonitor() throws IOException { + // verify scenario where findings are generated but alert creation fails - monitor execution should be marked successful + public void testCreateThreatIntelMonitor_testExecution_findingSucceedsButAlertFails_ExecutionSucceeds() throws IOException { + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", Map.of(), null); Map responseAsMap = responseAsMap(iocFindingsResponse); Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); List vals = List.of("ip1", "ip2"); - indexSourceConfigsAndIocs(1, vals); + String createdId = indexSourceConfigsAndIocs(vals); String index = createTestIndex(randomIndex(), windowsIndexMapping()); String monitorName = "test_monitor_name"; @@ -312,6 +568,296 @@ public void testCreateThreatIntelMonitor() throws IOException { final String monitorId = responseBody.get("id").toString(); Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + indexDoc(index, "" + i++, String.format("{\"ip\":\"1.2.3.4\", \"ip1\":\"1.2.3.4\"}", val, val)); + indexDoc(index, "" + i++, String.format("{\"random\":\"%s\", \"random1\":\"%s\"}", val, val)); + } catch (IOException e) { + fail(); + } + } + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + String matchAllRequest = getMatchAllRequest(); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + HashMap totalHits = (HashMap) hits.get("total"); + Integer totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 1); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(2, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + closeIndex(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME); + executeAlertingMonitor(monitorId, Collections.emptyMap()); + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); + openIndex(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME); + // Use ListIOCs API to confirm expected number of findings are returned + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, createdId); + Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); + Map listIocsResponseMap = responseAsMap(listIocsResponse); + List> iocsMap = (List>) listIocsResponseMap.get("iocs"); + assertEquals(2, iocsMap.size()); + iocsMap.forEach((iocDetails) -> { + String iocId = (String) iocDetails.get("id"); + int numFindings = (Integer) iocDetails.get("num_findings"); + assertTrue(testIocDtos.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertEquals(2, numFindings); + }); + + //alerts via system index search + searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(4, searchHits.size()); + + // alerts via API + Map params = new HashMap<>(); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + Assert.assertEquals(4, getAlertsBody.get("total_alerts")); + + + ThreatIntelMonitorDto updateMonitorDto = new ThreatIntelMonitorDto( + monitorId, + iocScanMonitor.getName() + "update", + iocScanMonitor.getPerIocTypeScanInputList(), + new IntervalSchedule(5, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(iocScanMonitor.getTriggers().get(0), iocScanMonitor.getTriggers().get(1)) + ); + //update monitor + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), toHttpEntity(updateMonitorDto)); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + responseBody = asMap(response); + assertEquals(responseBody.get("id").toString(), monitorId); + assertEquals(((HashMap) responseBody.get("monitor")).get("name").toString(), iocScanMonitor.getName() + "update"); + + //delete + Response delete = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), null); + Assert.assertEquals(200, delete.getStatusLine().getStatusCode()); + + searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + totalHits = (HashMap) hits.get("total"); + totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 0); + } + + // verify scenario where alerts and findings are generated but notification fails - monitor execution should be marked successful + public void testCreateThreatIntelMonitor_testNotifications_invaliConfigExecutionSucceeds() throws IOException { + + updateClusterSetting(SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT.getKey(), "1"); + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List vals = List.of("ip1", "ip2"); + String createdId = indexSourceConfigsAndIocs(vals); + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + String monitorName = "test_monitor_name"; + + Action triggerAction1 = randomThreatInteMonitorAction(randomAlphaOfLength(10)); + Action triggerAction2 = randomThreatInteMonitorAction(randomAlphaOfLength(10)); + /**create monitor with trigger*/ + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDtoWithTriggers(index, List.of(triggerAction1, triggerAction2)); + Response response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + fail(); + } catch (Exception e) { + /** creating a second threat intel monitor should fail*/ + assertTrue(e.getMessage().contains("already exists")); + } + + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + int i = 1; + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + indexDoc(index, "" + i++, String.format("{\"ip\":\"1.2.3.4\", \"ip1\":\"1.2.3.4\"}", val, val)); + indexDoc(index, "" + i++, String.format("{\"random\":\"%s\", \"random1\":\"%s\"}", val, val)); + } catch (IOException e) { + fail(); + } + } + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + String matchAllRequest = getMatchAllRequest(); + Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + HashMap hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + HashMap totalHits = (HashMap) hits.get("total"); + Integer totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 1); + makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + + + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(2, ((List>) responseAsMap.get("ioc_findings")).size()); + + //alerts + List searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(2, searchHits.size()); + + for (String val : vals) { + String doc = String.format("{\"ip\":\"%s\", \"ip1\":\"%s\"}", val, val); + try { + indexDoc(index, "" + i++, doc); + } catch (IOException e) { + fail(); + } + } + executeAlertingMonitor(monitorId, Collections.emptyMap()); + iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); + + // Use ListIOCs API to confirm expected number of findings are returned + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, createdId); + Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); + Map listIocsResponseMap = responseAsMap(listIocsResponse); + List> iocsMap = (List>) listIocsResponseMap.get("iocs"); + assertEquals(2, iocsMap.size()); + iocsMap.forEach((iocDetails) -> { + String iocId = (String) iocDetails.get("id"); + int numFindings = (Integer) iocDetails.get("num_findings"); + assertTrue(testIocDtos.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertEquals(2, numFindings); + }); + + //alerts via system index search + searchHits = executeSearch(ThreatIntelAlertService.THREAT_INTEL_ALERT_ALIAS_NAME, matchAllRequest); + Assert.assertEquals(2, searchHits.size()); + + // alerts via API + Map params = new HashMap<>(); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_ALERTS_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + Assert.assertEquals(2, getAlertsBody.get("total_alerts")); + + + ThreatIntelMonitorDto updateMonitorDto = new ThreatIntelMonitorDto( + monitorId, + iocScanMonitor.getName() + "update", + iocScanMonitor.getPerIocTypeScanInputList(), + new IntervalSchedule(5, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(iocScanMonitor.getTriggers().get(0)) + ); + //update monitor + response = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), toHttpEntity(updateMonitorDto)); + Assert.assertEquals(200, response.getStatusLine().getStatusCode()); + responseBody = asMap(response); + assertEquals(responseBody.get("id").toString(), monitorId); + assertEquals(((HashMap) responseBody.get("monitor")).get("name").toString(), iocScanMonitor.getName() + "update"); + + //delete + Response delete = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI + "/" + monitorId, Collections.emptyMap(), null); + Assert.assertEquals(200, delete.getStatusLine().getStatusCode()); + + searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, + ContentType.APPLICATION_JSON.getMimeType(), ContentType.APPLICATION_JSON.getCharset().name())); + Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); + hits = (HashMap) asMap(searchMonitorResponse).get("hits"); + totalHits = (HashMap) hits.get("total"); + totalHitsVal = (Integer) totalHits.get("value"); + assertEquals(totalHitsVal.intValue(), 0); + } + + public void testCreateThreatIntelMonitorWithExistingDetector() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Response iocFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.THREAT_INTEL_BASE_URI + "/findings/_search", + Map.of(), null); + Map responseAsMap = responseAsMap(iocFindingsResponse); + Assert.assertEquals(0, ((List>) responseAsMap.get("ioc_findings")).size()); + List vals = List.of("ip1", "ip2"); + String createdId = indexSourceConfigsAndIocs(vals); + String monitorName = "test_monitor_name"; + + + /**create monitor */ + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto(index); + response = makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + Assert.assertEquals(201, response.getStatusLine().getStatusCode()); + Map responseBody = asMap(response); + + try { + makeRequest(client(), "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, Collections.emptyMap(), toHttpEntity(iocScanMonitor)); + fail(); + } catch (Exception e) { + /** creating a second threat intel monitor should fail*/ + assertTrue(e.getMessage().contains("already exists")); + } + + final String monitorId = responseBody.get("id").toString(); + Assert.assertNotEquals("response is missing Id", Monitor.NO_ID, monitorId); + Response alertingMonitorResponse = getAlertingMonitor(client(), monitorId); Assert.assertEquals(200, alertingMonitorResponse.getStatusLine().getStatusCode()); int i = 1; @@ -326,7 +872,6 @@ public void testCreateThreatIntelMonitor() throws IOException { Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); - assertEquals(1, 1); String matchAllRequest = getMatchAllRequest(); Response searchMonitorResponse = makeRequest(client(), "POST", SEARCH_THREAT_INTEL_MONITOR_PATH, Collections.emptyMap(), new StringEntity(matchAllRequest, ContentType.APPLICATION_JSON)); @@ -362,7 +907,7 @@ public void testCreateThreatIntelMonitor() throws IOException { Assert.assertEquals(4, ((List>) responseAsMap.get("ioc_findings")).size()); // Use ListIOCs API to confirm expected number of findings are returned - String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, "id0"); + String listIocsUri = String.format("?%s=%s", ListIOCsActionRequest.FEED_IDS_FIELD, createdId); Response listIocsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.LIST_IOCS_URI + listIocsUri, Collections.emptyMap(), null); Map listIocsResponseMap = responseAsMap(listIocsResponse); List> iocsMap = (List>) listIocsResponseMap.get("iocs"); @@ -370,7 +915,7 @@ public void testCreateThreatIntelMonitor() throws IOException { iocsMap.forEach((iocDetails) -> { String iocId = (String) iocDetails.get("id"); int numFindings = (Integer) iocDetails.get("num_findings"); - assertTrue(testIocs.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); + assertTrue(testIocDtos.stream().anyMatch(ioc -> iocId.equals(ioc.getId()))); assertEquals(2, numFindings); }); @@ -413,6 +958,24 @@ public void testCreateThreatIntelMonitor() throws IOException { assertEquals(totalHitsVal.intValue(), 0); } + public void testCreateThreatIntelMonitor_invalidMonitorJson() throws IOException { + ThreatIntelMonitorDto iocScanMonitor = randomIocScanMonitorDto("test-index"); + + String monitorJson = iocScanMonitor.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).toString(); + final String invalidMonitorJson = monitorJson.replace("\"interval\":1", "\"interval\":100000000000000000000000000000000000000"); + + ResponseException exception = Assert.assertThrows(ResponseException.class, + () -> makeRequest( + client(), + "POST", SecurityAnalyticsPlugin.THREAT_INTEL_MONITOR_URI, + Collections.emptyMap(), + new StringEntity(invalidMonitorJson, ContentType.APPLICATION_JSON) + ) + ); + Assert.assertTrue(exception.getMessage().contains("Failed to parse threat intel monitor: ")); + Assert.assertTrue(exception.getMessage().contains("\"status\":400")); + } + public static String getMatchAllRequest() { return "{\n" + " \"query\" : {\n" + @@ -438,6 +1001,39 @@ public static ThreatIntelMonitorDto randomIocScanMonitorDto(String index) { List.of(t1, t2, t3, t4)); } + public static ThreatIntelMonitorDto randomIocScanMonitorDtoWithTriggers(String index, List actions) { + ThreatIntelTriggerDto t1 = new ThreatIntelTriggerDto(List.of(), List.of(IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE), actions, "match", null, "severity"); + + return new ThreatIntelMonitorDto( + Monitor.NO_ID, + randomAlphaOfLength(10), + List.of(new PerIocTypeScanInputDto(IOCType.IPV4_TYPE, Map.of(index, List.of("ip")))), + new IntervalSchedule(1, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(t1)); + } + + public static ThreatIntelMonitorDto randomIocScanMonitorDtoWithMultipleIndicatorTypesToScan(String ipIndex, String hashIndex, String domainIndex) { + ThreatIntelTriggerDto t1 = new ThreatIntelTriggerDto(List.of(ipIndex, "randomIndex"), List.of(IOCType.IPV4_TYPE, IOCType.DOMAIN_NAME_TYPE), emptyList(), "match", null, "severity"); + ThreatIntelTriggerDto t2 = new ThreatIntelTriggerDto(List.of("randomIndex"), List.of(IOCType.DOMAIN_NAME_TYPE), emptyList(), "nomatch", null, "severity"); + ThreatIntelTriggerDto t3 = new ThreatIntelTriggerDto(emptyList(), List.of(IOCType.DOMAIN_NAME_TYPE), emptyList(), "domainmatchsonomatch", null, "severity"); + ThreatIntelTriggerDto t4 = new ThreatIntelTriggerDto(List.of(ipIndex), emptyList(), emptyList(), "indexmatch", null, "severity"); + + return new ThreatIntelMonitorDto( + Monitor.NO_ID, + randomAlphaOfLength(10), + List.of( + new PerIocTypeScanInputDto(IOCType.IPV4_TYPE, Map.of(ipIndex, List.of("ip"))), + new PerIocTypeScanInputDto(IOCType.HASHES_TYPE, Map.of(hashIndex, List.of("hash"))), + new PerIocTypeScanInputDto(IOCType.DOMAIN_NAME_TYPE, Map.of(domainIndex, List.of("domain"))) + ), + new IntervalSchedule(1, ChronoUnit.MINUTES, Instant.now()), + false, + null, + List.of(t1, t2, t3, t4)); + } + @Override protected boolean preserveIndicesUponCompletion() { return false; diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java index 7a95e746f..2f5aa90a3 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/common/ThreatIntelLockServiceTests.java @@ -12,6 +12,7 @@ import java.time.Instant; import java.util.concurrent.atomic.AtomicReference; +import org.junit.Assert; import org.junit.Before; import org.opensearch.action.DocWriteResponse; import org.opensearch.action.update.UpdateRequest; @@ -52,7 +53,16 @@ public void testAcquireLock_whenCalled_thenNotBlocked() { public void testReleaseLock_whenValidInput_thenSucceed() { // Cannot test because LockService is final class // Simply calling method to increase coverage - noOpsLockService.releaseLock(null); + LockModel lockModel = new LockModel( + TestHelpers.randomLowerCaseString(), + TestHelpers.randomLowerCaseString(), + Instant.now(), + LOCK_DURATION_IN_SECONDS, + false + ); + noOpsLockService.releaseLock(lockModel, ActionListener.wrap( + Assert::assertFalse, e -> fail() + )); } public void testRenewLock_whenCalled_thenNotBlocked() { diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java index def2b21e5..36de85ebf 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/model/monitor/ThreatIntelInputTests.java @@ -57,6 +57,7 @@ public void testThreatInputSerde() throws IOException { emptyList(), emptyMap(), new DataSources(), + false, "security_analytics" ); BytesStreamOutput monitorOut = new BytesStreamOutput(); diff --git a/src/test/java/org/opensearch/securityanalytics/util/STIX2IOCGenerator.java b/src/test/java/org/opensearch/securityanalytics/util/STIX2IOCGenerator.java index 59302beac..6c040d9a2 100644 --- a/src/test/java/org/opensearch/securityanalytics/util/STIX2IOCGenerator.java +++ b/src/test/java/org/opensearch/securityanalytics/util/STIX2IOCGenerator.java @@ -193,6 +193,10 @@ public static STIX2IOCDto randomIocDto() { return new STIX2IOCDto(randomIOC()); } + public static STIX2IOCDto randomIocDto(IOCType type) { + return new STIX2IOCDto(randomIOC(type)); + } + public static STIX2IOCDto randomIocDto( String id, String name, diff --git a/src/test/resources/ad_ldap-sample.json b/src/test/resources/ad_ldap-sample.json index 3b89ca581..f6beef93c 100644 --- a/src/test/resources/ad_ldap-sample.json +++ b/src/test/resources/ad_ldap-sample.json @@ -2,13 +2,13 @@ "azure.signinlogs.properties.user_id": "1234", "azure.activitylogs.category": "1234", "azure.platformlogs.operation_name": "1234", - "modified_properties.new_value": "1234", + "ModifiedProperties.NewValue": "1234", "azure.resource.provider": "1234", "azure.signinlogs.properties.conditional_access_status": "1234", "SearchFilter": "1234", "azure.platformlogs.result_type": "1234", "azure.signinlogs.result_description": "1234", - "azure.signinlogs.properties.device_detail.is_compliant": "1234", + "azure.signinlogs.properties.device_detail.is_compliant": false, "resource_display_name": "1234", "azure.signinlogs.properties.authentication_requirement": "1234", "target_resources": "1234", @@ -18,5 +18,6 @@ "EventID": 12345, "azure.signinlogs.properties.network_location_details": "1234", "azure.auditlogs.properties.activity_display_name": "1234", - "creationTime": "2022-12-27T20:29:31.734Z" + "creationTime": "2022-12-27T20:29:31.734Z", + "DeviceDetail.isCompliant": true } diff --git a/src/test/resources/azure-sample.json b/src/test/resources/azure-sample.json index bccf2cf40..7c0facebd 100644 --- a/src/test/resources/azure-sample.json +++ b/src/test/resources/azure-sample.json @@ -1,27 +1,44 @@ { - "azure.signinlogs.props.user_id": "111", + "azure.signinlogs.properties.user_id": "111", "azure.activitylogs.category": "111", "modified_properties.new_value": "111", + "azure.activitylogs.identity.claims_initiated_by_user.name": "111", "azure.resource.provider": "111", - "azure.signinlogs.props.conditional_access_status": "111", + "azure.signinlogs.properties.conditional_access_status": "111", "SearchFilter": "111", "azure.platformlogs.result_type": "111", - "azure.signinlogs.props.device_detail.is_compliant": true, + "azure.signinlogs.properties.device_detail.is_compliant": true, "ResourceDisplayName": 111, - "azure.signinlogs.props.authentication_requirement": "111", + "azure.signinlogs.properties.resource_tenant_id": "111", + "azure.signinlogs.properties.authentication_requirement": "111", "TargetResources": "111", "Workload": "111", - "azure.signinlogs.props.device_detail.device_id": "111", + "azure.signinlogs.properties.device_detail.device_id": "111", "azure.platformlogs.operation_name": "111", - "azure.signinlogs.props.resource_id": "111", + "azure.signinlogs.properties.resource_id": "111", "EventID": 1234, - "azure.signinlogs.props.network_location_details": "111", - "azure.auditlogs.props.activity_display_name": "111", - "azure.signinlogs.result-description": "111", + "failure_status_reason": "111", + "azure.signinlogs.properties.network_location_details": "111", + "azure.auditlogs.properties.activity_display_name": "111", + "azure.signinlogs.result_description": "111", "eventSource": "111", "eventName": "111", "azure.platformlogs.status": "111", "azure.auditlogs.props.logged_by_service": "111", - "properties_message": "111", - "creationTime": "2022-12-27T20:29:31.734Z" + "properties.message": "111", + "creationTime": "2022-12-27T20:29:31.734Z", + "Count": "111", + "azure.signinlogs.properties.app_id": "111", + "azure.signinlogs.properties.client_app_used": true, + "ActivityDetails": "111", + "Target": "111", + "azure.signinlogs.properties.device_detail.trust_type": "111", + "azure.signinlogs.properties.home_tenant_id": "111", + "ConsentContext.IsAdminConsent": true, + "InitiatedBy": "111", + "azure.activitylogs.operation_name": "111", + "user_agent.name": "111", + "azure.signinlogs.properties.risk_state": "111", + "riskEventType": "111", + "azure.auditlogs.properties.logged_by_service": "111" } diff --git a/src/test/resources/cloudtrail-sample.json b/src/test/resources/cloudtrail-sample.json index 02e9ee2fb..e3c812f35 100644 --- a/src/test/resources/cloudtrail-sample.json +++ b/src/test/resources/cloudtrail-sample.json @@ -6,26 +6,31 @@ "source.as.organization.name": "213123", "source.ip": "213123", "userIdentity.arn": "213123", - "eventName": "213123", + "eventName": "DeleteIdentity", "eventType": "213123", "errorCode": "213123", - "eventSource": "213123", + "eventSource": "ses.amazonaws.com", "tlsDetails.tlsVersion": "213123", "user_agent.name": "213123", "threat.matched.providers": "213123", "aws-cloudtrail-event_name": "123", "aws-cloudtrail-event_source": "123", "aws-cloudtrail-event_type": "123", - "aws-cloudtrail-error_message": "123", + "aws.cloudtrail.error_message": "123", "aws-cloudtrail-error_code": "123", - "aws-cloudtrail-response_elements-text": "123", - "aws-cloudtrail-response_elements-pending_modified_values-master_user_password": "123", + "aws.cloudtrail.response_elements.text": "123", + "aws.cloudtrail.response_elements.pending_modified_values.master_user_password": "123", "aws-cloudtrail-response_elements-publicly_accessible": "123", "aws-cloudtrail-request_parameters-arn": "123", - "aws-cloudtrail-request_parameters-attribute": "123", + "aws.cloudtrail.request_parameters.attribute": "123", "aws-cloudtrail-request_parameters-username": "123", - "aws-cloudtrail-request_parameters-container_definitions-command": "123", - "aws-cloudtrail-user_identity-session_context-session_issuer-type": "123", + "aws.cloudtrail.request_parameters.container_definitions.command": "123", + "aws.cloudtrail.user_identity.session_context.session_issuer.type": "123", "aws-cloudtrail-user_identity-arn": "123", + "aws.cloudtrail.user_identity.type": "123", + "aws.cloudtrail.response_elements.publicly_accessible": "123", + "aws.cloudtrail.user_agent": "123", + "requestParameters": "123", + "type": "123", "eventTime": "2022-12-27T20:29:31.734Z" } diff --git a/src/test/resources/s3-sample.json b/src/test/resources/s3-sample.json index 440dcf184..68f84b042 100644 --- a/src/test/resources/s3-sample.json +++ b/src/test/resources/s3-sample.json @@ -8,7 +8,7 @@ "Operation": "123", "RequestURI_key": "123", "aws.s3access.requester": "1234", - "aws-cloudtrail-event_source": "123", - "aws-cloudtrail-event_name": "123", + "aws.cloudtrail.event_source": "123", + "aws.cloudtrail.event_name": "123", "eventTime": "123" } diff --git a/src/test/resources/waf-sample.json b/src/test/resources/waf-sample.json index 51a09cb4b..0cb24f5f0 100644 --- a/src/test/resources/waf-sample.json +++ b/src/test/resources/waf-sample.json @@ -53,5 +53,6 @@ { "name": "awswaf:managed:aws:bot-control:signal:known_bot_data_center" } - ] + ], + "waf.request.headers.user_agent": "WPScan v" }