[AUTO] Incremented version to 2.16.1.

.github/workflows/ci.yml

@@ -19,7 +19,7 @@ jobs:
     needs: Get-CI-Image-Tag
     strategy:
       matrix:
-        java: [11, 17]
+        java: [21]
         os: [ ubuntu-latest ]
     name: Build and Test security-analytics with JDK ${{ matrix.java }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
@@ -55,14 +55,14 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Upload failed logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: failure()
         with:
           name: logs-ubuntu
           path: build/testclusters/integTest-*/logs/*
 
       - name: Upload Artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v3
         with:
           name: security-analytics-plugin-${{ matrix.os }}
           path: security-analytics-artifacts
@@ -73,7 +73,7 @@ jobs:
       WORKING_DIR: ${{ matrix.working_directory }}.
     strategy:
       matrix:
-        java: [11, 17]
+        java: [21]
         os: [ windows-latest, macos-latest ]
         include:
           - os: windows-latest
@@ -113,21 +113,21 @@ jobs:
           cp ./build/distributions/*.zip security-analytics-artifacts
 
       - name: Upload failed logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: ${{ failure() && matrix.os == 'macos-latest' }}
         with:
           name: logs-mac
           path: build/testclusters/integTest-*/logs/*
 
       - name: Upload failed logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: ${{ failure() && matrix.os == 'windows-latest' }}
         with:
           name: logs-windows
           path: build\testclusters\integTest-*\logs\*
 
       - name: Upload Artifacts
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v3
         with:
           name: security-analytics-plugin-${{ matrix.os }}
           path: security-analytics-artifacts

.github/workflows/multi-node-test-workflow.yml

@@ -19,7 +19,7 @@ jobs:
     needs: Get-CI-Image-Tag
     strategy:
       matrix:
-        java: [ 11, 17, 21 ]
+        java: [ 21 ]
     # Job name
     name: Build and test Security Analytics on linux
     # This job runs on Linux
@@ -45,7 +45,7 @@ jobs:
           chown -R 1000:1000 `pwd`
           su `id -un 1000` -c "./gradlew integTest -PnumNodes=3"
       - name: Upload failed logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: failure()
         with:
           name: logs

.github/workflows/security-test-workflow.yml

@@ -15,7 +15,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [ 11, 17, 21 ]
+        java: [ 21 ]
     # Job name
     name: Build and test SecurityAnalytics
     # This job runs on Linux

build.gradle

@@ -7,7 +7,7 @@ import org.opensearch.gradle.test.RestIntegTestTask
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "2.15.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "2.18.1-SNAPSHOT")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')
@@ -18,6 +18,9 @@ buildscript {
         if (buildVersionQualifier) {
             opensearch_build += "-${buildVersionQualifier}"
         }
+
+        alerting_spi_build = opensearch_build
+        alerting_spi_build += "-SNAPSHOT"
         if (isSnapshot) {
             opensearch_build += "-SNAPSHOT"
 
@@ -171,7 +174,7 @@ dependencies {
     compileOnly "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
     compileOnly "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
     compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}"
-    compileOnly "org.opensearch.alerting:alerting-spi:${opensearch_build}"
+    compileOnly "org.opensearch.alerting:alerting-spi:${alerting_spi_build}"
     implementation "org.apache.commons:commons-csv:1.10.0"
     compileOnly "com.google.guava:guava:32.1.3-jre"
 

release-notes/opensearch-security-analytics.release-notes-2.16.0.0.md

@@ -0,0 +1,35 @@
+## Version 2.16.0.0 2024-07-23
+
+Compatible with OpenSearch 2.16.0
+
+### Features
+* Threat Intel Analytics ([#1098](https://github.com/opensearch-project/security-analytics/pull/1098))
+
+### Maintenance
+* Incremented version to 2.16.0. ([#1197](https://github.com/opensearch-project/security-analytics/pull/1197))
+* Fix build CI error due to action runner env upgrade node 20 ([#1143](https://github.com/opensearch-project/security-analytics/pull/1143))
+
+### Enhancement
+* added correlationAlert integ tests ([#1099](https://github.com/opensearch-project/security-analytics/pull/1099))
+* add filter to list ioc api to fetch only from available and refreshing apis. null check for alias of ioc indices ([#1131](https://github.com/opensearch-project/security-analytics/pull/1131))
+* Changes threat intel default store config model ([#1133](https://github.com/opensearch-project/security-analytics/pull/1133))
+* adds new tif source config type - url download ([#1142](https://github.com/opensearch-project/security-analytics/pull/1142))
+
+### Bug Fixes
+* pass integ tests ([#1082](https://github.com/opensearch-project/security-analytics/pull/1082))
+* set blank response when indexNotFound exception ([#1125](https://github.com/opensearch-project/security-analytics/pull/1125))
+* throw error when no iocs are stored due to incompatible ioc types from S3 downloaded iocs file ([#1129](https://github.com/opensearch-project/security-analytics/pull/1129))
+* fix findingIds filter on ioc findings search api ([#1130](https://github.com/opensearch-project/security-analytics/pull/1130))
+* Adjusted IOCTypes usage ([#1156](https://github.com/opensearch-project/security-analytics/pull/1156))
+* Fix the job scheduler parser, action listeners, and multi-node test ([#1157](https://github.com/opensearch-project/security-analytics/pull/1157))
+* ListIOCs API to return number of findings per IOC ([#1163](https://github.com/opensearch-project/security-analytics/pull/1163))
+* Ioc upload integ tests and fix update ([#1162](https://github.com/opensearch-project/security-analytics/pull/1162))
+* [BUG] Resolve aliases in monitor input to concrete indices before computing ioc-containing fields from concrete index docs ([#1173](https://github.com/opensearch-project/security-analytics/pull/1173))
+* Enum fix ([#1178](https://github.com/opensearch-project/security-analytics/pull/1178))
+* fix bug: threat intel monitor finding doesnt contain all doc_ids containing malicious IOC ([#1184](https://github.com/opensearch-project/security-analytics/pull/1184))
+* Fixed bulk indexing for IOCs ([#1187](https://github.com/opensearch-project/security-analytics/pull/1187))
+* Fix ioc upload update behavior and change error response ([#1192](https://github.com/opensearch-project/security-analytics/pull/1192))
+* Catch and wrap exceptions. ([#1198](https://github.com/opensearch-project/security-analytics/pull/1198))
+
+### Documentation
+* Added 2.16.0 release notes. ([#1196](https://github.com/opensearch-project/security-analytics/pull/1196))

release-notes/opensearch-security-analytics.release-notes-2.17.0.0.md

@@ -0,0 +1,27 @@
+## Version 2.17.0.0 2024-09-05
+
+Compatible with OpenSearch 2.17.0
+
+### Maintenance
+* update build.gradle to use alerting-spi snapshot version ([#1217](https://github.com/opensearch-project/security-analytics/pull/1217))
+
+### Enhancement
+* added triggers in getDetectors API response ([#1226](https://github.com/opensearch-project/security-analytics/pull/1226))
+* secure rest tests for threat intel monitor apis ([#1212](https://github.com/opensearch-project/security-analytics/pull/1212))
+
+### Bug Fixes
+* Adds user validation for threat intel transport layer classes and stashes the thread context for all system index interactions ([#1207](https://github.com/opensearch-project/security-analytics/pull/1207))
+* fix mappings integ tests ([#1213](https://github.com/opensearch-project/security-analytics/pull/1213))
+* Bug fixes for threat intel ([#1223](https://github.com/opensearch-project/security-analytics/pull/1223))
+* make threat intel run with standard detectors ([#1234](https://github.com/opensearch-project/security-analytics/pull/1234))
+* Fixed searchString bug. Removed nested IOC mapping structure. ([#1239](https://github.com/opensearch-project/security-analytics/pull/1239))
+* adds toggling refresh disable/enable for deactivate/activate operation while updating URL_DOWNLOAD type configs ([#1240](https://github.com/opensearch-project/security-analytics/pull/1240))
+* Make threat intel source config release lock event driven ([#1254](https://github.com/opensearch-project/security-analytics/pull/1254))
+* Fix S3 validation errors not caught by action listener ([#1257](https://github.com/opensearch-project/security-analytics/pull/1257))
+* Clean up empty IOC indices created by failed source configs ([#1267](https://github.com/opensearch-project/security-analytics/pull/1267))
+* Fix threat intel multinode tests ([#1274](https://github.com/opensearch-project/security-analytics/pull/1274))
+* Update threat intel job mapping to new version ([#1272](https://github.com/opensearch-project/security-analytics/pull/1272))
+* Stash context for List IOCs Api ([#1278](https://github.com/opensearch-project/security-analytics/pull/1278))
+
+### Documentation
+* Added 2.17.0 release notes. ([#1290](https://github.com/opensearch-project/security-analytics/pull/1290))

release-notes/opensearch-security-analytics.release-notes-2.17.1.0.md

@@ -0,0 +1,15 @@
+## Version 2.17.1.0 2024-09-27
+
+Compatible with OpenSearch 2.17.1
+
+### Maintenance
+* upgrade upload artifacts ([#1305](https://github.com/opensearch-project/security-analytics/pull/1305))
+* Incremented version to 2.17.1 ([#1304](https://github.com/opensearch-project/security-analytics/pull/1304))
+
+### Bug Fixes
+* [Alerts in Correlations] Stash context for system index ([#1297](https://github.com/opensearch-project/security-analytics/pull/1297))
+* threat intel monitor bug fixes ([#1317](https://github.com/opensearch-project/security-analytics/pull/1317))
+
+
+### Documentation
+* Added 2.17.1 release notes. ([#1331](https://github.com/opensearch-project/security-analytics/pull/1331))

release-notes/opensearch-security-analytics.release-notes-2.18.0.0.md

@@ -0,0 +1,24 @@
+## Version 2.18.0.0 2024-10-28
+
+Compatible with OpenSearch 2.18.0
+
+### Maintenance
+* Incremented version to 2.18.0 ([#1314](https://github.com/opensearch-project/security-analytics/pull/1314))
+* update to lucene 9.12 ([#1349](https://github.com/opensearch-project/security-analytics/pull/1349))
+
+### Refactoring
+* separate doc-level monitor query indices created by detectors ([#1324](https://github.com/opensearch-project/security-analytics/pull/1324))
+* update number of replicas of system indices to 1-20 and number of primary shards for system indices to 1 ([#1358](https://github.com/opensearch-project/security-analytics/pull/1358))
+* update min number of replicas to 0 ([#1364](https://github.com/opensearch-project/security-analytics/pull/1364))
+* updated dedicated query index settings to true ([#1365](https://github.com/opensearch-project/security-analytics/pull/1365))
+* set the refresh policy to IMMEDIATE when updating correlation alerts ([#1382](https://github.com/opensearch-project/security-analytics/pull/1382))
+
+### Bug Fixes
+* remove redundant logic to fix OS launch exception and updates actions/upload-artifac2 to @V3 ([#1303](https://github.com/opensearch-project/security-analytics/pull/1303))
+* Add null check while adding fetched iocs into per-indicator-type map ([#1335](https://github.com/opensearch-project/security-analytics/pull/1335))
+* Fix notifications listener leak in threat intel monitor ([#1361](https://github.com/opensearch-project/security-analytics/pull/1361))
+* [Bug] Fixed ListIOCs number of findings cap. ([#1373](https://github.com/opensearch-project/security-analytics/pull/1373))
+* [Bug] Add exists check for IOCs index. ([#1392](https://github.com/opensearch-project/security-analytics/pull/1392))
+
+### Documentation
+* Added 2.18.0 release notes. ([#1399](https://github.com/opensearch-project/security-analytics/pull/1399))

src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java

@@ -70,7 +70,7 @@
 import org.opensearch.securityanalytics.action.IndexDetectorAction;
 import org.opensearch.securityanalytics.action.IndexRuleAction;
 import org.opensearch.securityanalytics.action.ListCorrelationsAction;
-import org.opensearch.securityanalytics.action.ListIOCsAction;
+import org.opensearch.securityanalytics.threatIntel.action.ListIOCsAction;
 import org.opensearch.securityanalytics.action.SearchCorrelationRuleAction;
 import org.opensearch.securityanalytics.action.SearchCustomLogTypeAction;
 import org.opensearch.securityanalytics.action.SearchDetectorAction;
@@ -113,7 +113,7 @@
 import org.opensearch.securityanalytics.resthandler.RestIndexDetectorAction;
 import org.opensearch.securityanalytics.resthandler.RestIndexRuleAction;
 import org.opensearch.securityanalytics.resthandler.RestListCorrelationAction;
-import org.opensearch.securityanalytics.resthandler.RestListIOCsAction;
+import org.opensearch.securityanalytics.threatIntel.resthandler.RestListIOCsAction;
 import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationAction;
 import org.opensearch.securityanalytics.resthandler.RestSearchCorrelationRuleAction;
 import org.opensearch.securityanalytics.resthandler.RestSearchCustomLogTypeAction;
@@ -197,7 +197,7 @@
 import org.opensearch.securityanalytics.transport.TransportIndexDetectorAction;
 import org.opensearch.securityanalytics.transport.TransportIndexRuleAction;
 import org.opensearch.securityanalytics.transport.TransportListCorrelationAction;
-import org.opensearch.securityanalytics.transport.TransportListIOCsAction;
+import org.opensearch.securityanalytics.threatIntel.transport.TransportListIOCsAction;
 import org.opensearch.securityanalytics.transport.TransportSearchCorrelationAction;
 import org.opensearch.securityanalytics.transport.TransportSearchCorrelationRuleAction;
 import org.opensearch.securityanalytics.transport.TransportSearchCustomLogTypeAction;
@@ -226,6 +226,7 @@
 import static org.opensearch.securityanalytics.threatIntel.iocscan.service.ThreatIntelMonitorRunner.THREAT_INTEL_MONITOR_TYPE;
 import static org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig.SOURCE_CONFIG_FIELD;
 import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX;
+import static org.opensearch.securityanalytics.util.CorrelationIndices.CORRELATION_ALERT_INDEX;
 
 public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension, RemoteMonitorRunnerExtension {
 
@@ -284,7 +285,11 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map
 
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
-        return Collections.singletonList(new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data"));
+        List<SystemIndexDescriptor> descriptors = List.of(
+                new SystemIndexDescriptor(THREAT_INTEL_DATA_INDEX_NAME_PREFIX, "System index used for threat intel data"),
+                new SystemIndexDescriptor(CORRELATION_ALERT_INDEX, "System index used for Correlation Alerts")
+        );
+        return descriptors;
     }
 
 
@@ -327,7 +332,7 @@ public Collection<Object> createComponents(Client client,
         TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService);
         IocFindingService iocFindingService = new IocFindingService(client, clusterService, xContentRegistry);
         ThreatIntelAlertService threatIntelAlertService = new ThreatIntelAlertService(client, clusterService, xContentRegistry);
-        SaIoCScanService ioCScanService = new SaIoCScanService(client, xContentRegistry, iocFindingService, threatIntelAlertService, notificationService);
+        SaIoCScanService ioCScanService = new SaIoCScanService(client, clusterService, xContentRegistry, iocFindingService, threatIntelAlertService, notificationService);
         DefaultTifSourceConfigLoaderService defaultTifSourceConfigLoaderService = new DefaultTifSourceConfigLoaderService(builtInTIFMetadataLoader, client, saTifSourceConfigManagementService);
         return List.of(
                 detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, threatIntelAlertService,
@@ -502,7 +507,9 @@ public List<Setting<?>> getSettings() {
                 SecurityAnalyticsSettings.BATCH_SIZE,
                 SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT,
                 SecurityAnalyticsSettings.IOC_INDEX_RETENTION_PERIOD,
-                SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_INDEX_PATTERN
+                SecurityAnalyticsSettings.IOC_MAX_INDICES_PER_INDEX_PATTERN,
+                SecurityAnalyticsSettings.IOC_SCAN_MAX_TERMS_COUNT,
+                SecurityAnalyticsSettings.ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES
         );
     }
 

src/main/java/org/opensearch/securityanalytics/action/GetDetectorResponse.java

@@ -69,6 +69,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 .field(Detector.LAST_UPDATE_TIME_FIELD, detector.getLastUpdateTime())
                 .field(Detector.ENABLED_TIME_FIELD, detector.getEnabledTime())
                 .field(Detector.THREAT_INTEL_ENABLED_FIELD, detector.getThreatIntelEnabled())
+                .field(Detector.TRIGGERS_FIELD, detector.getTriggers())
                 .endObject();
         return builder.endObject();
     }

src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java

@@ -5,6 +5,8 @@
 package org.opensearch.securityanalytics.config.monitors;
 
 import java.util.List;
+import java.util.Random;
+import java.util.UUID;
 import java.util.stream.Collectors;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.securityanalytics.logtype.LogTypeService;
@@ -25,6 +27,10 @@ public static String getRuleIndex(String logType) {
         return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", logType);
     }
 
+    public static String getRuleIndexOptimized(String logType) {
+        return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-optimized-%s", logType, UUID.randomUUID());
+    }
+
     public static String getAlertsIndex(String logType) {
         return String.format(Locale.getDefault(), ".opensearch-sap-%s-alerts", logType);
     }

src/main/java/org/opensearch/securityanalytics/correlation/alert/CorrelationAlertService.java

@@ -13,6 +13,7 @@
 import org.opensearch.action.index.IndexResponse;
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.search.SearchResponse;
+import org.opensearch.action.support.WriteRequest;
 import org.opensearch.action.update.UpdateRequest;
 import org.opensearch.client.Client;
 import org.opensearch.common.lucene.uid.Versions;
@@ -212,9 +213,10 @@ public void acknowledgeAlerts(List<String> alertIds, ActionListener<AckCorrelati
         client.search(searchRequest, new ActionListener<SearchResponse>() {
             @Override
             public void onResponse(SearchResponse searchResponse) {
+                // Set the refresh policy on the BulkRequest
+                bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
                 // Iterate through the search hits
                 for (SearchHit hit : searchResponse.getHits().getHits()) {
-                    // Construct a script to update the document with the new state and acknowledgedTime
                     // Construct a script to update the document with the new state and acknowledgedTime
                     Script script = new Script(ScriptType.INLINE, "painless",
                             "ctx._source.state = params.state; ctx._source.acknowledged_time = params.time",