build.gradle

/*
 * Copyright OpenSearch Contributors
 * SPDX-License-Identifier: Apache-2.0
 */

import java.util.concurrent.Callable
import org.opensearch.gradle.test.RestIntegTestTask
import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask

buildscript {
    ext {
        isSnapshot = "true" == System.getProperty("build.snapshot", "true")
        opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
        buildVersionQualifier = System.getProperty("build.version_qualifier", "")
        version_tokens = opensearch_version.tokenize('-')
        opensearch_build = version_tokens[0] + '.0'
        if (buildVersionQualifier) {
            opensearch_build += "-${buildVersionQualifier}"
        }
        if (isSnapshot) {
            opensearch_build += "-SNAPSHOT"
        }
        common_utils_version = System.getProperty("common_utils.version", opensearch_build)
        kotlin_version = System.getProperty("kotlin.version", "1.8.21")
    }

    repositories {
        mavenLocal()
        maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
        mavenCentral()
        maven { url "https://plugins.gradle.org/m2/" }
    }

    dependencies {
        classpath "org.opensearch.gradle:build-tools:${opensearch_version}"
        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${kotlin_version}"
        classpath "org.jetbrains.kotlin:kotlin-allopen:${kotlin_version}"
        classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.23.4"
        classpath "org.jacoco:org.jacoco.agent:0.8.11"
    }
}

plugins {
    id "com.netflix.nebula.ospackage-base" version "11.6.0"
    id "com.dorongold.task-tree" version "1.5"
    id 'java-library'
}

apply plugin: 'java'
apply plugin: 'jacoco'
apply plugin: 'idea'
apply plugin: 'opensearch.opensearchplugin'
apply plugin: 'opensearch.pluginzip'
apply plugin: 'opensearch.testclusters'
apply plugin: 'io.gitlab.arturbosch.detekt'
apply plugin: 'org.jetbrains.kotlin.jvm'
apply plugin: 'org.jetbrains.kotlin.plugin.allopen'

def usingRemoteCluster = System.properties.containsKey('tests.rest.cluster') || System.properties.containsKey('tests.cluster')
def usingMultiNode = project.properties.containsKey('numNodes')

check.dependsOn jacocoTestReport

opensearchplugin {
    name 'opensearch-observability'
    description 'OpenSearch Plugin for OpenSearch Dashboards Observability'
    classname "org.opensearch.observability.ObservabilityPlugin"
}

publishing {
    publications {
        pluginZip(MavenPublication) { publication ->
            pom {
                name = 'opensearch-observability'
                description = 'OpenSearch Observability plugin'
                groupId = "org.opensearch.plugin"
                licenses {
                    license {
                        name = 'The Apache License, Version 2.0'
                        url = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
                    }
                }
                developers {
                    developer {
                        name = 'OpenSearch'
                        url = 'https://github.com/opensearch-project/observability'
                    }
                }
            }
        }
    }
    repositories {
        maven {
            name = "Snapshots" //  optional target repository name
            url = "https://aws.oss.sonatype.org/content/repositories/snapshots"
            credentials {
                username "$System.env.SONATYPE_USERNAME"
                password "$System.env.SONATYPE_PASSWORD"
            }
        }
    }
}

allOpen {
    annotation("org.opensearch.observability.util.OpenForTesting")
}

configurations {
    ktlint
    zipArchive
}

detekt {
    config = files("detekt.yml")
    buildUponDefaultConfig = true
}

configurations.testImplementation {
    exclude module: "securemock"
}

configurations.all {
    if (it.state != Configuration.State.UNRESOLVED) return
    resolutionStrategy {
        force "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
        force "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${kotlin_version}"
        force "org.jetbrains.kotlin:kotlin-reflect:${kotlin_version}"
        force "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}"
        force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
        force "org.mockito:mockito-core:${versions.mockito}"
        force "org.yaml:snakeyaml:${versions.snakeyaml}"
        force "org.slf4j:slf4j-api:2.0.0"
        force "ch.qos.logback:logback-classic:1.3.14"
        force "ch.qos.logback:logback-core:1.3.14"
    }
}

repositories {
    mavenLocal()
    maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
    mavenCentral()
    maven { url "https://plugins.gradle.org/m2/" }
}

ext {
    projectSubstitutions = [:]
    licenseFile = rootProject.file('LICENSE.txt')
    noticeFile = rootProject.file('NOTICE.txt')
}

plugins.withId('java') {
    sourceCompatibility = targetCompatibility = "11"
}

plugins.withId('org.jetbrains.kotlin.jvm') {
    compileKotlin.kotlinOptions.jvmTarget = compileTestKotlin.kotlinOptions.jvmTarget = "11"
}

allprojects {
    group = "org.opensearch"

    version = "${opensearch_build}"
    plugins.withId('java') {
        sourceCompatibility = targetCompatibility = "11"
    }
}

dependencies {
    implementation "org.opensearch:opensearch:${opensearch_version}"
    implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
    implementation "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}"
    implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.9"
    implementation "${group}:common-utils:${common_utils_version}"
    implementation 'org.json:json:20231013'
    implementation group: 'com.github.wnameless.json', name: 'json-flattener', version: '0.15.1'
    // json-base, jackson-databind, jackson-annotations are only used by json-flattener.
    // see https://github.com/opensearch-project/OpenSearch/issues/5395.
    implementation group: 'com.github.wnameless.json', name: 'json-base', version: '2.2.1'
    implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
    implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
    testImplementation(
            'org.assertj:assertj-core:3.16.1',
            'org.junit.jupiter:junit-jupiter-api:5.6.2'
    )
    testRuntimeOnly('org.junit.jupiter:junit-jupiter-engine:5.6.2')
    testImplementation "org.opensearch.test:framework:${opensearch_version}"
    testImplementation "org.jetbrains.kotlin:kotlin-test:${kotlin_version}"
    testImplementation "com.nhaarman.mockitokotlin2:mockito-kotlin:2.2.0"
    testImplementation "org.mockito:mockito-core:${versions.mockito}"
    testImplementation "org.mockito:mockito-junit-jupiter:${versions.mockito}"
    testImplementation "com.google.code.gson:gson:2.8.9"

    ktlint "com.pinterest:ktlint:0.47.1"
}

javadoc.enabled = false // turn off javadoc as it barfs on Kotlin code
licenseHeaders.enabled = true
// no need to validate pom, as we do not upload to maven/sonatype
validateNebulaPom.enabled = false
dependencyLicenses.enabled = false
thirdPartyAudit.enabled = false
// Allow @Test to be used in test classes not inherited from LuceneTestCase.
// see https://github.com/elastic/elasticsearch/blob/26c3dd6857d047054b2108f85bc2b040a9c81fe4/build-tools-internal/src/main/resources/forbidden/es-test-signatures.txt
forbiddenApis.ignoreFailures = true
// Allow test cases to be named Tests without having to be inherited from LuceneTestCase.
// see https://github.com/elastic/elasticsearch/blob/323f312bbc829a63056a79ebe45adced5099f6e6/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java
testingConventions.enabled = false
loggerUsageCheck.enabled = false

test {
    systemProperty 'tests.security.manager', 'false'
    useJUnitPlatform()
}

File repo = file("$buildDir/testclusters/repo")
def _numNodes = findProperty('numNodes') as Integer ?: 1

def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
opensearch_tmp_dir.mkdirs()

// As of ES 7.7 the sample-extension-plugin is being added to the list of plugins for the testCluster during build before
// the job-scheduler plugin (not a dependency for opensearch-observability) is causing build failures.
// The job-scheduler zip is added explicitly above but the sample-extension-plugin is added implicitly at some time during evaluation.
// Will need to do a deep dive to find out exactly what task adds the sample-extension-plugin and add job-scheduler there but a temporary hack is to
// reorder the plugins list after evaluation but prior to task execution when the plugins are installed.
afterEvaluate {
    testClusters.integTest.nodes.each { node ->
        def plugins = node.plugins
        def firstPlugin = plugins.get(0)
        plugins.remove(0)
        plugins.add(firstPlugin)
    }
}

tasks.withType(licenseHeaders.class) {
    additionalLicense 'AL   ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")'
}

task integTest(type: RestIntegTestTask) {
    description = "Run tests against a cluster that has security enabled"
    testClassesDirs = sourceSets.test.output.classesDirs
    classpath = sourceSets.test.runtimeClasspath
}
tasks.named("check").configure { dependsOn(integTest) }

integTest {
    systemProperty 'tests.security.manager', 'false'
    systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath

    systemProperty "https", System.getProperty("https")
    systemProperty "user", System.getProperty("user")
    systemProperty "password", System.getProperty("password")
    // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
    // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
    doFirst {
        // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
        // use longer timeouts for requests.
        def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null
        systemProperty 'cluster.debug', isDebuggingCluster
        // Set number of nodes system property to be used in tests
        systemProperty 'cluster.number_of_nodes', "${_numNodes}"
        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
        // not being written, the waitForAllConditions ensures it's written
        getClusters().forEach { cluster ->
            cluster.waitForAllConditions()
        }
    }

    // The -Dcluster.debug option makes the cluster debuggable; this makes the tests debuggable
    if (System.getProperty("test.debug") != null) {
        jvmArgs '-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=8000'
    }

    // https://github.com/opensearch-project/index-management/pull/93
    // if (System.getProperty("tests.clustername") != null) {
    //     exclude 'org/opensearch/observability/ObservabilityPluginIT.class'
    // }
    // This fails for JDK 17, excluding for now since it's only testing plugin presence and not necessary
    exclude 'org/opensearch/observability/ObservabilityPluginIT.class'

    if (System.getProperty("tests.rest.bwcsuite") == null) {
        filter {
            excludeTestsMatching "org.opensearch.observability.bwc.*IT"
        }
    }
}


Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin");
integTest.dependsOn(bundle)
integTest.getClusters().forEach { c -> c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile())) }

testClusters.integTest {
    testDistribution = "INTEG_TEST"

    // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
    if (_numNodes > 1) numberOfNodes = _numNodes
    // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
    // i.e. we have to use a custom property to flag when we want to debug opensearch JVM
    // since we also support multi node integration tests we increase debugPort per node
    if (System.getProperty("cluster.debug") != null) {
        def debugPort = 5005
        nodes.forEach { node ->
            node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}")
            debugPort += 1
        }
    }
    setting 'path.repo', repo.absolutePath
}

String baseVersion = "2.16.0"
String bwcVersion = baseVersion + ".0"
String baseName = "obsBwcCluster"
String bwcFilePath = "src/test/resources/bwc/"
String bwcObservabilityPlugin = "opensearch-observability-" + bwcVersion + ".zip"
// TODO: remove bwc job scheduler after 2.9 release
String bwcJobSchedulerPlugin = "opensearch-job-scheduler-" + bwcVersion + ".zip"
String remoteFileURL = "https://aws.oss.sonatype.org/service/local/artifact/maven/redirect?r=snapshots&g=org.opensearch.plugin&a=opensearch-observability&v=$bwcVersion-SNAPSHOT&p=zip"
String bwcJobSchedulerURL = "https://aws.oss.sonatype.org/service/local/artifact/maven/redirect?r=snapshots&g=org.opensearch.plugin&a=opensearch-job-scheduler&v=$bwcVersion-SNAPSHOT&p=zip"
String bwcSnapshotVersion = baseVersion + "-SNAPSHOT"

2.times {i ->
    testClusters {
        "${baseName}$i" {
            testDistribution = "ARCHIVE"
            versions = [bwcSnapshotVersion,opensearch_version]
            numberOfNodes = 3
            plugin(provider(new Callable<RegularFile>(){
                @Override
                RegularFile call() throws Exception {
                    return new RegularFile() {
                        @Override
                        File getAsFile() {
                            File dir = new File(bwcFilePath + "/job-scheduler/" + bwcVersion)
                            if (!dir.exists()) {
                                dir.mkdirs()
                            }
                            File file = new File(dir, bwcJobSchedulerPlugin)
                            if (!file.exists()) {
                                new URL(bwcJobSchedulerURL).withInputStream{ ins -> file.withOutputStream{ it << ins }}
                            }
                            return fileTree(bwcFilePath + "/job-scheduler/" + bwcVersion).getSingleFile()
                        }
                    }
                }
            }))
            plugin(provider(new Callable<RegularFile>(){
                @Override
                RegularFile call() throws Exception {
                    return new RegularFile() {
                        @Override
                        File getAsFile() {
                            File dir = new File(bwcFilePath + "observability/" + bwcVersion)
                            if (!dir.exists()) {
                                dir.mkdirs()
                            }
                            File file = new File(dir, bwcObservabilityPlugin)
                            if (!file.exists()) {
                                new URL(remoteFileURL).withInputStream{ ins -> file.withOutputStream{ it << ins }}
                            }
                            return fileTree(bwcFilePath + "observability/" + bwcVersion).getSingleFile()
                        }
                    }
                }
            }))
            setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
            setting 'http.content_type.required', 'true'
        }
    }
}

List<Provider<RegularFile>> plugins = []

// Ensure the artifact for the current project version is available to be used for the bwc tests
task prepareBwcTests {
    dependsOn bundle
    doLast {
        plugins = [
                project.getObjects().fileProperty().value(bundle.getArchiveFile())
        ]
    }
}


// Creates 2 test clusters with 3 nodes of the old version.
2.times {i ->
    task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) {
        dependsOn 'prepareBwcTests'
        useCluster testClusters."${baseName}$i"
        filter {
            includeTestsMatching "org.opensearch.observability.bwc.*IT"
        }
        systemProperty 'tests.rest.bwcsuite', 'old_cluster'
        systemProperty 'tests.rest.bwcsuite_round', 'old'
        systemProperty 'tests.plugin_bwc_version', bwcVersion
        nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}")
        nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}")
    }
}

// Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version
// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
// This is also used as a one third upgraded cluster for a rolling upgrade.
task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
    useCluster testClusters."${baseName}0"
    dependsOn "${baseName}#oldVersionClusterTask0"
    doFirst {
        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
    }
    filter {
        includeTestsMatching "org.opensearch.observability.bwc.*IT"
    }
    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
    systemProperty 'tests.rest.bwcsuite_round', 'first'
    systemProperty 'tests.plugin_bwc_version', bwcVersion
    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
}

// Upgrades the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded.
// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
// This is used for rolling upgrade.
task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
    dependsOn "${baseName}#mixedClusterTask"
    useCluster testClusters."${baseName}0"
    doFirst {
        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
    }
    filter {
        includeTestsMatching "org.opensearch.observability.bwc.*IT"
    }
    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
    systemProperty 'tests.rest.bwcsuite_round', 'second'
    systemProperty 'tests.plugin_bwc_version', bwcVersion
    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
}

// Upgrades the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded.
// This results in a fully upgraded cluster.
// This is used for rolling upgrade.
task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
    dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
    useCluster testClusters."${baseName}0"
    doFirst {
        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
    }
    filter {
        includeTestsMatching "org.opensearch.observability.bwc.*IT"
    }
    mustRunAfter "${baseName}#mixedClusterTask"
    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
    systemProperty 'tests.rest.bwcsuite_round', 'third'
    systemProperty 'tests.plugin_bwc_version', bwcVersion
    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
}

// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version
// at the same time resulting in a fully upgraded cluster.
task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) {
    dependsOn "${baseName}#oldVersionClusterTask1"
    useCluster testClusters."${baseName}1"
    doFirst {
        testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
    }
    filter {
        includeTestsMatching "org.opensearch.observability.bwc.*IT"
    }
    systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster'
    systemProperty 'tests.plugin_bwc_version', bwcVersion
    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}")
    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}")
}

// A bwc test suite which runs all the bwc tasks combined.
task bwcTestSuite(type: StandaloneRestIntegTestTask) {
    exclude '**/*Test*'
    exclude '**/*IT*'
    dependsOn tasks.named("${baseName}#mixedClusterTask")
    dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask")
    dependsOn tasks.named("${baseName}#fullRestartClusterTask")
}

task integTestRemote(type: RestIntegTestTask) {
    testClassesDirs = sourceSets.test.output.classesDirs
    classpath = sourceSets.test.runtimeClasspath
    systemProperty 'tests.security.manager', 'false'
    systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath

    systemProperty "https", System.getProperty("https")
    systemProperty "user", System.getProperty("user")
    systemProperty "password", System.getProperty("password")

    // Only rest case can run with remote cluster
    if (System.getProperty("tests.rest.cluster") != null) {
        filter {
            includeTestsMatching "org.opensearch.observability.rest.*IT"
        }
    }
}

run {
    doFirst {
        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
        // not being written, the waitForAllConditions ensures it's written
        getClusters().forEach { cluster ->
            cluster.waitForAllConditions()
        }
    }
    useCluster testClusters.integTest
}

task ktlint(type: JavaExec, group: "verification") {
    description = "Check Kotlin code style."
    mainClass = "com.pinterest.ktlint.Main"
    classpath = configurations.ktlint
    args "src/**/*.kt"
    // to generate report in checkstyle format prepend following args:
    // "--reporter=plain", "--reporter=checkstyle,output=${buildDir}/ktlint.xml"
    // see https://github.com/pinterest/ktlint#usage for more
}

check.dependsOn ktlint

task ktlintFormat(type: JavaExec, group: "formatting") {
    description = "Fix Kotlin code style deviations."
    mainClass = "com.pinterest.ktlint.Main"
    classpath = configurations.ktlint
    args "-F", "src/**/*.kt"
    jvmArgs "--add-opens=java.base/java.lang=ALL-UNNAMED"
}

compileKotlin { kotlinOptions.freeCompilerArgs = ['-Xjsr305=strict'] }

// Only apply jacoco test coverage if we are running a local single node cluster
if (!usingRemoteCluster && !usingMultiNode) {
    apply from: 'build-tools/opensearchplugin-coverage.gradle'
}

apply from: 'build-tools/pkgbuild.gradle'

// updateVersion: Task to auto increment to the next development iteration
task updateVersion {
    onlyIf { System.getProperty('newVersion') }
    doLast {
        ext.newVersion = System.getProperty('newVersion')
        println "Setting version to ${newVersion}."
         // String tokenization to support -SNAPSHOT
        ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true)
        ant.replaceregexp(file:'../.github/workflows/dashboards-observability-test-and-build-workflow.yml', match:'OPENSEARCH_PLUGIN_VERSION: \\d+.\\d+.\\d+.\\d+', replace:'OPENSEARCH_PLUGIN_VERSION: ' + newVersion.tokenize('-')[0] + '.0', flags:'g', byline:true)
        // Match key version in JSON files.
        ant.replaceregexp(file:'../dashboards-observability/opensearch_dashboards.json', match:'"version": "\\d+.\\d+.\\d+.\\d+', replace:'"version": ' + '"' + newVersion.tokenize('-')[0] + '.0', flags:'g', byline:true)
        ant.replaceregexp(file:'../dashboards-observability/package.json', match:'"version": "\\d+.\\d+.\\d+.\\d+', replace:'"version": ' + '"' + newVersion.tokenize('-')[0] + '.0', flags:'g', byline:true)
        // Match key opensearchDashboardsVersion in JSON files.
        ant.replaceregexp(file:'../dashboards-observability/opensearch_dashboards.json', match:'"opensearchDashboardsVersion": "\\d+.\\d+.\\d+', replace:'"opensearchDashboardsVersion": ' + '"' + newVersion.tokenize('-')[0], flags:'g', byline:true)
    }
}