Skip to content

Commit

Permalink
Add retry for windows on min snapshot and distribution (#5132)
Browse files Browse the repository at this point in the history 
Signed-off-by: Peter Zhu <[email protected]>
  • Loading branch information
@peterzhuamazon
peterzhuamazon authored Oct 24, 2024
1 parent d625cff commit e0c1b11
Show file tree
Hide file tree
Showing 4 changed files with 93 additions and 86 deletions.
38 changes: 20 additions & 18 deletions jenkins/opensearch-dashboards/distribution-build.jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -819,26 +819,28 @@ pipeline {
}
steps {
script {
def buildManifestObj = buildAssembleUpload(
componentName: "${COMPONENT_NAME}",
inputManifest: "manifests/${INPUT_MANIFEST}",
platform: 'windows',
architecture: 'x64',
distribution: 'zip',
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID
)
String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
String buildManifestUrlOpenSearch = [buildManifestObj.getArtifactRootUrl(JOB_NAME_OPENSEARCH, 'latest'), 'builds', 'opensearch', 'manifest.yml'].join('/')
String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
env.ARTIFACT_URL_LINUX_WINDOWS_X64_ZIP = artifactUrl
env.INDEX_FILE_PATH_X64_ZIP = buildManifestObj.getIndexFileRoot("${JOB_NAME}")
retry(3) {
def buildManifestObj = buildAssembleUpload(
componentName: "${COMPONENT_NAME}",
inputManifest: "manifests/${INPUT_MANIFEST}",
platform: 'windows',
architecture: 'x64',
distribution: 'zip',
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID
)
String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
String buildManifestUrlOpenSearch = [buildManifestObj.getArtifactRootUrl(JOB_NAME_OPENSEARCH, 'latest'), 'builds', 'opensearch', 'manifest.yml'].join('/')
String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
env.ARTIFACT_URL_LINUX_WINDOWS_X64_ZIP = artifactUrl
env.INDEX_FILE_PATH_X64_ZIP = buildManifestObj.getIndexFileRoot("${JOB_NAME}")

echo "buildManifestUrl (windows, x64, zip): ${buildManifestUrl}"
echo "artifactUrl (windows, x64, zip): ${artifactUrl}"
echo "buildManifestUrl (windows, x64, zip): ${buildManifestUrl}"
echo "artifactUrl (windows, x64, zip): ${artifactUrl}"

triggerIntegrationTests(buildManifestUrl, buildManifestUrlOpenSearch, 'windows', 'zip')
triggerIntegrationTests(buildManifestUrl, buildManifestUrlOpenSearch, 'windows', 'zip')
}
}
}
post {
Expand Down
44 changes: 23 additions & 21 deletions jenkins/opensearch/distribution-build.jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -780,27 +780,29 @@ pipeline {
}
steps {
script {
echo("Switching to Java ${env.javaVersionNumber} on Windows Docker Container")
sh("scoop reset `scoop list jdk | cut -d ' ' -f1 | grep ${env.javaVersionNumber} | head -1`")
def buildManifestObj = buildAssembleUpload(
componentName: "${COMPONENT_NAME}",
inputManifest: "manifests/${INPUT_MANIFEST}",
platform: 'windows',
architecture: 'x64',
distribution: 'zip',
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID
)
String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
env.ARTIFACT_URL_LINUX_WINDOWS_X64_ZIP = artifactUrl
env.INDEX_FILE_PATH_X64_ZIP = buildManifestObj.getIndexFileRoot("${JOB_NAME}")

echo "buildManifestUrl (windows, x64, zip): ${buildManifestUrl}"
echo "artifactUrl (windows, x64, zip): ${artifactUrl}"

triggerIntegrationTests(buildManifestUrl, 'windows', 'zip')
retry(3) {
echo("Switching to Java ${env.javaVersionNumber} on Windows Docker Container")
sh("scoop reset `scoop list jdk | cut -d ' ' -f1 | grep ${env.javaVersionNumber} | head -1`")
def buildManifestObj = buildAssembleUpload(
componentName: "${COMPONENT_NAME}",
inputManifest: "manifests/${INPUT_MANIFEST}",
platform: 'windows',
architecture: 'x64',
distribution: 'zip',
continueOnError: params.CONTINUE_ON_ERROR,
incremental: params.INCREMENTAL,
previousBuildId: params.PREVIOUS_BUILD_ID
)
String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER)
String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER)
env.ARTIFACT_URL_LINUX_WINDOWS_X64_ZIP = artifactUrl
env.INDEX_FILE_PATH_X64_ZIP = buildManifestObj.getIndexFileRoot("${JOB_NAME}")

echo "buildManifestUrl (windows, x64, zip): ${buildManifestUrl}"
echo "artifactUrl (windows, x64, zip): ${artifactUrl}"

triggerIntegrationTests(buildManifestUrl, 'windows', 'zip')
}
}
}
post {
Expand Down
32 changes: 17 additions & 15 deletions jenkins/opensearch/publish-min-snapshots.jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,21 +193,23 @@ pipeline {
}
steps {
script {
echo("Switching to Java ${env.javaVersionNumber} on Windows Docker Container")
sh("scoop reset `scoop list jdk | cut -d ' ' -f1 | grep ${env.javaVersionNumber} | head -1`")
buildManifest(
componentName: "OpenSearch",
inputManifest: "manifests/${INPUT_MANIFEST}",
platform: 'windows',
architecture: 'x64',
distribution: 'zip',
snapshot: true
)
echo("Uploading windows x64 zip min snapshots to S3")
uploadMinSnapshotsToS3(
fileActions: [createSha512Checksums()],
distribution: 'zip'
)
retry(3) {
echo("Switching to Java ${env.javaVersionNumber} on Windows Docker Container")
sh("scoop reset `scoop list jdk | cut -d ' ' -f1 | grep ${env.javaVersionNumber} | head -1`")
buildManifest(
componentName: "OpenSearch",
inputManifest: "manifests/${INPUT_MANIFEST}",
platform: 'windows',
architecture: 'x64',
distribution: 'zip',
snapshot: true
)
echo("Uploading windows x64 zip min snapshots to S3")
uploadMinSnapshotsToS3(
fileActions: [createSha512Checksums()],
distribution: 'zip'
)
}
}
}
post {
Expand Down
65 changes: 33 additions & 32 deletions tests/jenkins/jenkinsjob-regression-files/opensearch/publish-min-snapshots.jenkinsfile.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,42 +183,43 @@
publish-min-snapshots.stage(windows-x64-zip, groovy.lang.Closure)
publish-min-snapshots.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-Windows2019-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-windows2019-opensearch-build-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]])
publish-min-snapshots.script(groovy.lang.Closure)
publish-min-snapshots.echo(Switching to Java 17 on Windows Docker Container)
publish-min-snapshots.sh(scoop reset `scoop list jdk | cut -d ' ' -f1 | grep 17 | head -1`)
publish-min-snapshots.buildManifest({componentName=OpenSearch, inputManifest=manifests/3.0.0/opensearch-3.0.0.yml, platform=windows, architecture=x64, distribution=zip, snapshot=true})
buildManifest.legacySCM(groovy.lang.Closure)
buildManifest.library({[email protected], retriever=null})
buildManifest.readYaml({file=manifests/3.0.0/opensearch-3.0.0.yml})
InputManifest.asBoolean()
buildManifest.sh(./build.sh manifests/3.0.0/opensearch-3.0.0.yml -d zip --component OpenSearch -p windows -a x64 --snapshot)
publish-min-snapshots.echo(Uploading windows x64 zip min snapshots to S3)
publish-min-snapshots.createSha512Checksums()
publish-min-snapshots.uploadMinSnapshotsToS3({fileActions=[groovy.lang.Closure], distribution=zip})
uploadMinSnapshotsToS3.legacySCM(groovy.lang.Closure)
uploadMinSnapshotsToS3.library({[email protected], retriever=null})
uploadMinSnapshotsToS3.readYaml({file=manifests/3.0.0/opensearch-3.0.0.yml})
InputManifest.asBoolean()
uploadMinSnapshotsToS3.echo(Retreving build manifest from: /tmp/workspace/zip/builds/opensearch/manifest.yml)
uploadMinSnapshotsToS3.readYaml({file=/tmp/workspace/zip/builds/opensearch/manifest.yml})
BuildManifest.asBoolean()
uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts)
createSha512Checksums.sh({script=find /tmp/workspace/zip/builds/opensearch/dist -type f, returnStdout=true})
createSha512Checksums.echo(Not generating sha for bbb in /tmp/workspace/zip/builds/opensearch/dist, doesn't match allowed types [.tar.gz, .zip, .rpm, .deb])
createSha512Checksums.echo(Not generating sha for ccc in /tmp/workspace/zip/builds/opensearch/dist, doesn't match allowed types [.tar.gz, .zip, .rpm, .deb])
uploadMinSnapshotsToS3.echo(Start copying files: version-3.0.0-SNAPSHOT architecture-x64 platform-windows buildid-8469 distribution-zip extension-zip)
uploadMinSnapshotsToS3.sh(
publish-min-snapshots.retry(3, groovy.lang.Closure)
publish-min-snapshots.echo(Switching to Java 17 on Windows Docker Container)
publish-min-snapshots.sh(scoop reset `scoop list jdk | cut -d ' ' -f1 | grep 17 | head -1`)
publish-min-snapshots.buildManifest({componentName=OpenSearch, inputManifest=manifests/3.0.0/opensearch-3.0.0.yml, platform=windows, architecture=x64, distribution=zip, snapshot=true})
buildManifest.legacySCM(groovy.lang.Closure)
buildManifest.library({[email protected], retriever=null})
buildManifest.readYaml({file=manifests/3.0.0/opensearch-3.0.0.yml})
InputManifest.asBoolean()
buildManifest.sh(./build.sh manifests/3.0.0/opensearch-3.0.0.yml -d zip --component OpenSearch -p windows -a x64 --snapshot)
publish-min-snapshots.echo(Uploading windows x64 zip min snapshots to S3)
publish-min-snapshots.createSha512Checksums()
publish-min-snapshots.uploadMinSnapshotsToS3({fileActions=[groovy.lang.Closure], distribution=zip})
uploadMinSnapshotsToS3.legacySCM(groovy.lang.Closure)
uploadMinSnapshotsToS3.library({[email protected], retriever=null})
uploadMinSnapshotsToS3.readYaml({file=manifests/3.0.0/opensearch-3.0.0.yml})
InputManifest.asBoolean()
uploadMinSnapshotsToS3.echo(Retreving build manifest from: /tmp/workspace/zip/builds/opensearch/manifest.yml)
uploadMinSnapshotsToS3.readYaml({file=/tmp/workspace/zip/builds/opensearch/manifest.yml})
BuildManifest.asBoolean()
uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts)
createSha512Checksums.sh({script=find /tmp/workspace/zip/builds/opensearch/dist -type f, returnStdout=true})
createSha512Checksums.echo(Not generating sha for bbb in /tmp/workspace/zip/builds/opensearch/dist, doesn't match allowed types [.tar.gz, .zip, .rpm, .deb])
createSha512Checksums.echo(Not generating sha for ccc in /tmp/workspace/zip/builds/opensearch/dist, doesn't match allowed types [.tar.gz, .zip, .rpm, .deb])
uploadMinSnapshotsToS3.echo(Start copying files: version-3.0.0-SNAPSHOT architecture-x64 platform-windows buildid-8469 distribution-zip extension-zip)
uploadMinSnapshotsToS3.sh(
cp -v /tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64.zip /tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip
cp -v /tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64.zip.sha512 /tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.sha512
cp -v /tmp/workspace/zip/builds/opensearch/dist/../manifest.yml /tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.build-manifest.yml
sed -i "s/.zip/-latest.zip/g" /tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.sha512
)
uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME})
uploadMinSnapshotsToS3.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT})
uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
uploadMinSnapshotsToS3.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
uploadMinSnapshotsToS3.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
uploadMinSnapshotsToS3.s3Upload({file=/tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip})
uploadMinSnapshotsToS3.s3Upload({file=/tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.sha512, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.sha512})
uploadMinSnapshotsToS3.s3Upload({file=/tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.build-manifest.yml, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.build-manifest.yml})
uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME})
uploadMinSnapshotsToS3.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT})
uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
uploadMinSnapshotsToS3.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
uploadMinSnapshotsToS3.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
uploadMinSnapshotsToS3.s3Upload({file=/tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip})
uploadMinSnapshotsToS3.s3Upload({file=/tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.sha512, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.sha512})
uploadMinSnapshotsToS3.s3Upload({file=/tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.build-manifest.yml, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.build-manifest.yml})
publish-min-snapshots.postCleanup()
postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})

0 comments on commit e0c1b11

Please sign in to comment.