diff --git a/private-cicd/Jenkinsfile b/private-cicd/Jenkinsfile
index 8d1dead65701..13ba47909786 100644
--- a/private-cicd/Jenkinsfile
+++ b/private-cicd/Jenkinsfile
@@ -146,6 +146,103 @@ boolean hasIntegrationStarted(build) {
return environment?.get('PRESUBMIT_INTEGRATION_STARTED') == 'true'
}
+@NonCPS
+String revisionHeadSha(revision) {
+ if (revision == null) {
+ return ''
+ }
+ try {
+ def hash = revision.pullHash?.toString() ?: ''
+ if (hash ==~ /^[0-9a-fA-F]{40}$/) {
+ return hash
+ }
+ } catch (Exception ignored) {
+ return ''
+ }
+ return ''
+}
+
+@NonCPS
+String indexedHeadSha(String jobName, int buildNumber) {
+ def job = jenkins.model.Jenkins.get().getItemByFullName(jobName)
+ if (job == null) {
+ return ''
+ }
+ // A manual build does not copy the indexed revision onto the build. The scan stores it on the job.
+ def hash = revisionHeadSha(job.getAction(jenkins.scm.api.SCMRevisionAction)?.revision)
+ if (hash) {
+ return hash
+ }
+ def build = job.getBuildByNumber(buildNumber)
+ return revisionHeadSha(build?.getAction(jenkins.scm.api.SCMRevisionAction)?.revision)
+}
+
+// A multibranch child only contains builds for one PR. Stop an older run of
+// that child before it holds the VM. A queued duplicate of the same indexed
+// SHA is cancelled; a queued item with no SHA is left alone.
+@NonCPS
+def supersedeMultibranchBuilds(String jobName, int buildNumber, String sha) {
+ def jenkins = jenkins.model.Jenkins.get()
+ def job = jenkins.getItemByFullName(jobName)
+ if (job == null) {
+ throw new IllegalStateException("Cannot find Jenkins job '${jobName}'.")
+ }
+
+ def results = []
+ def latest = revisionHeadSha(job.getAction(jenkins.scm.api.SCMRevisionAction)?.revision)
+ if (latest && sha && !latest.equalsIgnoreCase(sha)) {
+ def self = job.getBuildByNumber(buildNumber)
+ if (self != null && self.isBuilding() && !hasIntegrationStarted(self)) {
+ self.doStop()
+ results << "Stopping build #${buildNumber}: a newer head ${latest.take(12)} is indexed."
+ } else if (self != null && hasIntegrationStarted(self)) {
+ results << "Not stopping build #${buildNumber}: ONTAP integration has started."
+ }
+ return results
+ }
+ jenkins.queue.items.findAll { item ->
+ item.task?.fullName == job.fullName
+ }.each { item ->
+ def queued = revisionHeadSha(item.getAction(jenkins.scm.api.SCMRevisionAction)?.revision)
+ if (queued && sha && queued.equalsIgnoreCase(sha)) {
+ jenkins.queue.cancel(item)
+ results << "Cancelled a queued build for ${sha.take(12)}."
+ }
+ }
+ job.getBuilds().findAll { build ->
+ build.isBuilding() && build.number < buildNumber
+ }.each { build ->
+ if (hasIntegrationStarted(build)) {
+ results << "Not aborting build #${build.number}: ONTAP integration has started."
+ } else {
+ build.doStop()
+ results << "Aborted superseded build #${build.number}."
+ }
+ }
+ return results
+}
+
+@NonCPS
+int diffNumberForResolvedSha(String jobName, int buildNumber, String sha) {
+ def job = jenkins.model.Jenkins.get().getItemByFullName(jobName)
+ if (job == null || !sha) {
+ return 1
+ }
+ def previousShas = [] as Set
+ job.getBuilds().findAll {
+ it.number < buildNumber
+ }.each { build ->
+ def previousSha = build.getAction(
+ org.jenkinsci.plugins.workflow.cps.EnvActionImpl
+ )?.getOverriddenEnvironment()?.get('RESOLVED_SOURCE_SHA')
+ if (previousSha) {
+ previousShas << previousSha
+ }
+ }
+ previousShas.remove(sha)
+ return previousShas.size() + 1
+}
+
@NonCPS
def abortOlderBuildsForSource(String jobName, int buildNumber, String sourceKey) {
def job = jenkins.model.Jenkins.get().getItemByFullName(jobName)
@@ -198,6 +295,44 @@ boolean isValidBranchName(String branchName) {
}
}
+boolean isMultibranchPullRequest() {
+ return env.CHANGE_ID?.trim()
+}
+
+// Branch Source does not fill Declarative parameters. A multibranch run stores
+// one resolved pull-request context in the environment and reads it here.
+def sourceValue(String name) {
+ if (env.PRESUBMIT_MULTIBRANCH_READY == 'true') {
+ switch (name) {
+ case 'SOURCE_MODE':
+ return env.EFFECTIVE_SOURCE_MODE
+ case 'PR_ID':
+ return env.EFFECTIVE_PR_ID
+ case 'PR_ACTION':
+ return env.EFFECTIVE_PR_ACTION
+ case 'PR_DRAFT':
+ return env.EFFECTIVE_PR_DRAFT
+ case 'PR_REPOSITORY':
+ return env.EFFECTIVE_PR_REPOSITORY
+ case 'PR_BASE_BRANCH':
+ return env.EFFECTIVE_PR_BASE_BRANCH
+ case 'PR_HEAD_BRANCH':
+ return env.EFFECTIVE_PR_HEAD_BRANCH
+ case 'PR_HEAD_SHA':
+ return env.EFFECTIVE_PR_HEAD_SHA
+ case 'PR_AUTHOR_LOGIN':
+ return env.EFFECTIVE_PR_AUTHOR_LOGIN
+ case 'PR_AUTHOR_EMAIL':
+ return env.EFFECTIVE_PR_AUTHOR_EMAIL
+ case 'PR_URL':
+ return env.EFFECTIVE_PR_URL
+ case 'PR_TITLE':
+ return env.EFFECTIVE_PR_TITLE
+ }
+ }
+ return params.get(name)
+}
+
def validateInventory(Map inventory) {
if (containsPlaceholder(inventory)) {
error('VM inventory still contains PLACEHOLDER- values.')
@@ -321,13 +456,13 @@ void recordStageEvent(String stageName, String status) {
}
String presubmitSubjectBase() {
- def recipientName = params.PR_AUTHOR_LOGIN?.trim()
+ def recipientName = sourceValue('PR_AUTHOR_LOGIN')?.trim()
if (!recipientName && env.PRESUBMIT_MAIL_TO?.contains('@')) {
recipientName = env.PRESUBMIT_MAIL_TO.substring(0, env.PRESUBMIT_MAIL_TO.indexOf('@'))
}
- def sourceName = params.SOURCE_MODE == 'branch' ?
- "branch ${params.SOURCE_BRANCH}" : "PR-${params.PR_ID}"
- def title = params.PR_TITLE?.replaceAll(/[\r\n]+/, ' ')?.trim()
+ def sourceName = sourceValue('SOURCE_MODE') == 'branch' ?
+ "branch ${params.SOURCE_BRANCH}" : "PR-${sourceValue('PR_ID')}"
+ def title = sourceValue('PR_TITLE')?.replaceAll(/[\r\n]+/, ' ')?.trim()
if (title?.length() > 80) {
title = title.take(77) + '...'
}
@@ -365,15 +500,15 @@ boolean hasGithubAppCredential() {
List githubEnvironment() {
return [
- "GITHUB_REPOSITORY=${params.PR_REPOSITORY}",
- "GITHUB_HEAD_SHA=${params.PR_HEAD_SHA}",
- "PR_ID=${params.PR_ID}",
- "PR_TITLE=${params.PR_TITLE ?: ''}"
+ "GITHUB_REPOSITORY=${sourceValue('PR_REPOSITORY')}",
+ "GITHUB_HEAD_SHA=${sourceValue('PR_HEAD_SHA')}",
+ "PR_ID=${sourceValue('PR_ID')}",
+ "PR_TITLE=${sourceValue('PR_TITLE') ?: ''}"
]
}
void createGithubCheck() {
- if (params.SOURCE_MODE != 'pull_request' || !hasGithubAppCredential()) {
+ if (sourceValue('SOURCE_MODE') != 'pull_request' || !hasGithubAppCredential()) {
echo 'GitHub Check creation skipped: GitHub App credential is not configured.'
return
}
@@ -394,7 +529,7 @@ void createGithubCheck() {
throw new IllegalStateException("GitHub returned an invalid Check Run ID '${checkRunId}'.")
}
env.GITHUB_CHECK_RUN_ID = checkRunId
- echo "Created GitHub Check Run ${checkRunId} on ${params.PR_HEAD_SHA}."
+ echo "Created GitHub Check Run ${checkRunId} on ${sourceValue('PR_HEAD_SHA')}."
}
}
} catch (Exception e) {
@@ -435,10 +570,10 @@ void completeGithubCheck(String conclusion) {
}
void resolvePresubmitMailRecipient() {
- def webhookEmail = params.PR_AUTHOR_EMAIL?.trim()
+ def webhookEmail = sourceValue('PR_AUTHOR_EMAIL')?.trim()
env.PRESUBMIT_MAIL_TO = webhookEmail?.contains('@') &&
!webhookEmail.equalsIgnoreCase('null') ? webhookEmail : ''
- if (env.PRESUBMIT_MAIL_TO || params.SOURCE_MODE != 'pull_request' ||
+ if (env.PRESUBMIT_MAIL_TO || sourceValue('SOURCE_MODE') != 'pull_request' ||
!hasGithubAppCredential()) {
return
}
@@ -485,14 +620,14 @@ String presubmitInfoRow(String label, String valueHtml) {
}
String presubmitStartHtml() {
- def source = params.SOURCE_MODE == 'branch' ?
- "branch ${params.SOURCE_BRANCH}" : "PR-${params.PR_ID}"
- def sourceHtml = params.PR_URL ==~ /^https:\/\/github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/pull\/[0-9]+$/ ?
- "${htmlEscape(source)}" :
+ def source = sourceValue('SOURCE_MODE') == 'branch' ?
+ "branch ${params.SOURCE_BRANCH}" : "PR-${sourceValue('PR_ID')}"
+ def sourceHtml = sourceValue('PR_URL') ==~ /^https:\/\/github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/pull\/[0-9]+$/ ?
+ "${htmlEscape(source)}" :
htmlEscape(source)
def diffNumber = env.PRESUBMIT_DIFF_NUMBER?.trim()
def rows = presubmitInfoRow('Source', sourceHtml) +
- presubmitInfoRow('Title', htmlEscape(params.PR_TITLE ?: 'not provided'))
+ presubmitInfoRow('Title', htmlEscape(sourceValue('PR_TITLE') ?: 'not provided'))
if (diffNumber) {
rows += presubmitInfoRow('Diff', "#${htmlEscape(diffNumber)}")
}
@@ -524,12 +659,13 @@ results follows when this run finishes.
// GitHub App diagnostic skip them, as does a request that `Check eligibility`
// rejected by setting NOT_BUILT.
boolean isWorkerStageEnabled() {
- return params.SOURCE_MODE in ['pull_request', 'branch'] &&
- currentBuild.result != 'NOT_BUILT'
+ return sourceValue('SOURCE_MODE') in ['pull_request', 'branch'] &&
+ currentBuild.result != 'NOT_BUILT' &&
+ currentBuild.currentResult != 'NOT_BUILT'
}
boolean isDiagnoseRun() {
- return params.SOURCE_MODE == 'diagnose'
+ return sourceValue('SOURCE_MODE') == 'diagnose'
}
void discoverPullRequests() {
@@ -668,10 +804,10 @@ void validateSourceRequest() {
TimeZone.getTimeZone('UTC')
)
recordStageEvent('Validate source request', 'STARTED')
- if (!(params.SOURCE_MODE in ['pull_request', 'branch'])) {
- error("Unsupported SOURCE_MODE '${params.SOURCE_MODE}'.")
+ if (!(sourceValue('SOURCE_MODE') in ['pull_request', 'branch'])) {
+ error("Unsupported SOURCE_MODE '${sourceValue('SOURCE_MODE')}'.")
}
- if (params.SOURCE_MODE == 'branch') {
+ if (sourceValue('SOURCE_MODE') == 'branch') {
if (!isValidBranchName(params.SOURCE_BRANCH)) {
error('SOURCE_BRANCH is not a valid Git branch name.')
}
@@ -679,16 +815,16 @@ void validateSourceRequest() {
error('SOURCE_SHA must be a full 40-character Git SHA.')
}
} else {
- if (!(params.PR_ID ==~ /^[0-9]+$/)) {
+ if (!(sourceValue('PR_ID') ==~ /^[0-9]+$/)) {
error('PR_ID must be the numeric GitHub pull request number.')
}
- if (params.PR_REPOSITORY != params.EXPECTED_REPOSITORY) {
- error("Repository '${params.PR_REPOSITORY}' is not '${params.EXPECTED_REPOSITORY}'.")
+ if (sourceValue('PR_REPOSITORY') != params.EXPECTED_REPOSITORY) {
+ error("Repository '${sourceValue('PR_REPOSITORY')}' is not '${params.EXPECTED_REPOSITORY}'.")
}
- if (!(params.PR_HEAD_BRANCH ==~ /^[A-Za-z0-9._\/-]+$/)) {
+ if (!(sourceValue('PR_HEAD_BRANCH') ==~ /^[A-Za-z0-9._\/-]+$/)) {
error('PR_HEAD_BRANCH contains unsupported characters.')
}
- if (!(params.PR_HEAD_SHA ==~ /^[0-9a-fA-F]{40}$/) && params.PR_ACTION != 'closed') {
+ if (!(sourceValue('PR_HEAD_SHA') ==~ /^[0-9a-fA-F]{40}$/) && sourceValue('PR_ACTION') != 'closed') {
error('PR_HEAD_SHA must be a full 40-character Git SHA.')
}
}
@@ -718,15 +854,23 @@ void validateSourceRequest() {
}
}
try {
- def sourceKey = params.SOURCE_MODE == 'branch' ?
- "branch:${params.SOURCE_BRANCH}" :
- "pull_request:${params.PR_ID}"
- env.PRESUBMIT_DIFF_NUMBER = diffNumberForSource(
- env.JOB_NAME,
- env.BUILD_NUMBER.toInteger(),
- sourceKey,
- env.RESOLVED_SOURCE_SHA
- ).toString()
+ if (isMultibranchPullRequest()) {
+ env.PRESUBMIT_DIFF_NUMBER = diffNumberForResolvedSha(
+ env.JOB_NAME,
+ env.BUILD_NUMBER.toInteger(),
+ env.RESOLVED_SOURCE_SHA
+ ).toString()
+ } else {
+ def sourceKey = sourceValue('SOURCE_MODE') == 'branch' ?
+ "branch:${params.SOURCE_BRANCH}" :
+ "pull_request:${sourceValue('PR_ID')}"
+ env.PRESUBMIT_DIFF_NUMBER = diffNumberForSource(
+ env.JOB_NAME,
+ env.BUILD_NUMBER.toInteger(),
+ sourceKey,
+ env.RESOLVED_SOURCE_SHA
+ ).toString()
+ }
} catch (Exception e) {
env.PRESUBMIT_DIFF_NUMBER = ''
echo "Unable to determine the diff number: ${e}"
@@ -736,37 +880,51 @@ void validateSourceRequest() {
// five-minute discovery runs.
def diffLabel = env.PRESUBMIT_DIFF_NUMBER ?
" diff #${env.PRESUBMIT_DIFF_NUMBER}" : ''
- if (params.SOURCE_MODE == 'branch') {
+ if (sourceValue('SOURCE_MODE') == 'branch') {
currentBuild.displayName =
"#${env.BUILD_NUMBER} branch ${params.SOURCE_BRANCH}${diffLabel}"
currentBuild.description = "Branch ${params.SOURCE_BRANCH} @ ${params.SOURCE_SHA.take(12)}"
} else {
currentBuild.displayName =
- "#${env.BUILD_NUMBER} PR-${params.PR_ID}${diffLabel} ${params.PR_ACTION}"
+ "#${env.BUILD_NUMBER} PR-${sourceValue('PR_ID')}${diffLabel} ${sourceValue('PR_ACTION')}"
currentBuild.description =
- "${env.RESOLVED_SOURCE_SHA?.take(12)} ${params.PR_TITLE ?: params.PR_URL}"
+ "${env.RESOLVED_SOURCE_SHA?.take(12)} ${sourceValue('PR_TITLE') ?: sourceValue('PR_URL')}"
}
}
void checkPresubmitEligibility() {
recordStageEvent('Check eligibility', 'STARTED')
- if (params.SOURCE_MODE == 'pull_request') {
+ if (sourceValue('SOURCE_MODE') == 'pull_request') {
def eligibleActions = ['opened', 'synchronize', 'reopened', 'ready_for_review']
- if (params.PR_ACTION == 'closed' ||
- params.PR_DRAFT.toBoolean() ||
- params.PR_BASE_BRANCH != 'main' ||
- !(params.PR_ACTION in eligibleActions)) {
+ if (sourceValue('PR_ACTION') == 'closed' ||
+ sourceValue('PR_DRAFT').toBoolean() ||
+ sourceValue('PR_BASE_BRANCH') != 'main' ||
+ !(sourceValue('PR_ACTION') in eligibleActions)) {
currentBuild.result = 'NOT_BUILT'
- echo "Ignoring PR-${params.PR_ID}: action=${params.PR_ACTION}, draft=${params.PR_DRAFT}, base=${params.PR_BASE_BRANCH}"
+ echo "Ignoring PR-${sourceValue('PR_ID')}: action=${sourceValue('PR_ACTION')}, draft=${sourceValue('PR_DRAFT')}, base=${sourceValue('PR_BASE_BRANCH')}"
}
}
}
void abortSupersededRun() {
recordStageEvent('Abort superseded run', 'STARTED')
- def sourceKey = params.SOURCE_MODE == 'branch' ?
+ if (isMultibranchPullRequest()) {
+ try {
+ supersedeMultibranchBuilds(
+ env.JOB_NAME,
+ env.BUILD_NUMBER.toInteger(),
+ env.RESOLVED_SOURCE_SHA
+ ).each { result ->
+ echo result
+ }
+ } catch (Exception e) {
+ echo "Unable to inspect older builds; no build was aborted: ${e}"
+ }
+ return
+ }
+ def sourceKey = sourceValue('SOURCE_MODE') == 'branch' ?
"branch:${params.SOURCE_BRANCH}" :
- "pull_request:${params.PR_ID}"
+ "pull_request:${sourceValue('PR_ID')}"
try {
abortOlderBuildsForSource(
env.JOB_NAME,
@@ -782,8 +940,111 @@ void abortSupersededRun() {
void checkoutCiScripts() {
recordStageEvent('Checkout CI scripts', 'STARTED')
+ // Pilot for PR-119. Change this back to main before the folder includes every PR.
+ def ciBranch = 'feature/CSTACKEX-340'
+ if (env.PRESUBMIT_CI_SHA?.trim() && fileExists('private-cicd/scripts/resolve-pull-request.py')) {
+ echo "CI scripts already checked out from ${ciBranch} at ${env.PRESUBMIT_CI_SHA}."
+ return
+ }
deleteDir()
- checkout(scm)
+ def remote = [
+ url: params.CLOUDSTACK_GIT_URL,
+ refspec: "+refs/heads/${ciBranch}:refs/remotes/origin/${ciBranch}"
+ ]
+ def credentialId = params.GIT_CREDENTIALS_ID?.trim()
+ if (!credentialId) {
+ credentialId = params.GITHUB_APP_CREDENTIALS_ID?.trim()
+ }
+ if (credentialId && !credentialId.startsWith('PLACEHOLDER-')) {
+ remote.credentialsId = credentialId
+ }
+ def cloneOptions = [[
+ $class: 'CloneOption',
+ honorRefspec: true,
+ noTags: true,
+ shallow: true,
+ depth: 1
+ ]]
+ try {
+ checkout([
+ $class: 'GitSCM',
+ branches: [[name: "*/${ciBranch}"]],
+ doGenerateSubmoduleConfigurations: false,
+ extensions: cloneOptions + [[
+ $class: 'SparseCheckoutPaths',
+ sparseCheckoutPaths: [[path: 'private-cicd']]
+ ]],
+ userRemoteConfigs: [remote]
+ ])
+ } catch (Exception e) {
+ echo "Sparse CI checkout unavailable, using a full ${ciBranch} checkout: ${e}"
+ deleteDir()
+ checkout([
+ $class: 'GitSCM',
+ branches: [[name: "*/${ciBranch}"]],
+ doGenerateSubmoduleConfigurations: false,
+ extensions: cloneOptions,
+ userRemoteConfigs: [remote]
+ ])
+ }
+ env.PRESUBMIT_CI_SHA = sh(script: 'git rev-parse HEAD', returnStdout: true).trim()
+ echo "Checked out CI scripts from ${ciBranch} at ${env.PRESUBMIT_CI_SHA}."
+}
+
+void prepareMultibranchPullRequest() {
+ checkoutCiScripts()
+ def indexed = indexedHeadSha(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
+ if (!(indexed ==~ /^[0-9a-fA-F]{40}$/)) {
+ indexed = ''
+ echo "Jenkins did not expose an indexed head SHA for PR-${env.CHANGE_ID}. Using the GitHub pull request head."
+ }
+ def status = 1
+ withCredentials([
+ usernamePassword(
+ credentialsId: params.GITHUB_APP_CREDENTIALS_ID,
+ usernameVariable: 'GITHUB_APP',
+ passwordVariable: 'GITHUB_ACCESS_TOKEN'
+ )
+ ]) {
+ withEnv([
+ "PRESUBMIT_REPOSITORY=${params.EXPECTED_REPOSITORY}",
+ "PRESUBMIT_PR_ID=${env.CHANGE_ID}",
+ "PRESUBMIT_INDEXED_SHA=${indexed}",
+ "PRESUBMIT_CHANGE_TARGET=${env.CHANGE_TARGET ?: ''}"
+ ]) {
+ status = sh(
+ script: '''python3 private-cicd/scripts/resolve-pull-request.py \
+ --repository "$PRESUBMIT_REPOSITORY" \
+ --pr-id "$PRESUBMIT_PR_ID" \
+ --indexed-sha "$PRESUBMIT_INDEXED_SHA" \
+ --change-target "$PRESUBMIT_CHANGE_TARGET" \
+ --output multibranch-source.json''',
+ returnStatus: true
+ )
+ }
+ }
+ def decision = fileExists('multibranch-source.json') ? readJSON(file: 'multibranch-source.json') : [:]
+ if (status != 0 || !decision.eligible) {
+ currentBuild.result = 'NOT_BUILT'
+ echo "Not building PR-${env.CHANGE_ID}: ${decision.reason ?: 'pull request lookup failed.'}"
+ return
+ }
+ env.PRESUBMIT_MULTIBRANCH_READY = 'true'
+ env.EFFECTIVE_SOURCE_MODE = 'pull_request'
+ env.EFFECTIVE_PR_ID = decision.pr_id.toString()
+ env.EFFECTIVE_PR_ACTION = 'synchronize'
+ env.EFFECTIVE_PR_DRAFT = 'false'
+ env.EFFECTIVE_PR_REPOSITORY = params.EXPECTED_REPOSITORY
+ env.EFFECTIVE_PR_BASE_BRANCH = decision.pr_base_branch.toString()
+ env.EFFECTIVE_PR_HEAD_BRANCH = decision.pr_head_branch.toString()
+ env.EFFECTIVE_PR_HEAD_SHA = decision.pr_head_sha.toString()
+ env.EFFECTIVE_PR_AUTHOR_LOGIN = decision.pr_author_login?.toString() ?: ''
+ env.EFFECTIVE_PR_AUTHOR_EMAIL = decision.pr_author_email?.toString() ?: ''
+ env.EFFECTIVE_PR_URL = decision.pr_url?.toString() ?: ''
+ env.EFFECTIVE_PR_TITLE = decision.pr_title?.toString() ?: ''
+ env.RESOLVED_SOURCE_SHA = env.EFFECTIVE_PR_HEAD_SHA
+ env.RESOLVED_SOURCE_ID = "pr-${env.EFFECTIVE_PR_ID}"
+ echo "Accepted PR-${env.EFFECTIVE_PR_ID} at ${env.RESOLVED_SOURCE_SHA}."
}
// A full worker run takes over an hour, so reporting problems are diagnosed
@@ -793,11 +1054,11 @@ void diagnoseGithubApp() {
if (!hasGithubAppCredential()) {
error('GITHUB_APP_CREDENTIALS_ID is unset or still a placeholder.')
}
- def headSha = params.PR_HEAD_SHA?.trim() ?: params.SOURCE_SHA?.trim()
+ def headSha = sourceValue('PR_HEAD_SHA')?.trim() ?: params.SOURCE_SHA?.trim()
if (!(headSha ==~ /^[0-9a-f]{40}$/)) {
error('Set PR_HEAD_SHA to a full 40-character commit SHA to diagnose.')
}
- def repository = params.PR_REPOSITORY?.trim() ?: params.EXPECTED_REPOSITORY
+ def repository = sourceValue('PR_REPOSITORY')?.trim() ?: params.EXPECTED_REPOSITORY
withCredentials([
usernamePassword(
credentialsId: params.GITHUB_APP_CREDENTIALS_ID,
@@ -808,7 +1069,7 @@ void diagnoseGithubApp() {
withEnv([
"GITHUB_REPOSITORY=${repository}",
"GITHUB_HEAD_SHA=${headSha}",
- "PR_ID=${params.PR_ID ?: ''}"
+ "PR_ID=${sourceValue('PR_ID') ?: ''}"
]) {
sh 'private-cicd/scripts/github-check-run.sh diagnose'
}
@@ -863,18 +1124,18 @@ void checkoutSource() {
recordStageEvent('Checkout source', 'STARTED')
dir(env.SOURCE_DIR) {
deleteDir()
- def isBranch = params.SOURCE_MODE == 'branch'
- def sourceBranch = isBranch ? params.SOURCE_BRANCH : params.PR_HEAD_BRANCH
- def sourceSha = isBranch ? params.SOURCE_SHA : params.PR_HEAD_SHA
- def sourceLabel = isBranch ? "branch:${sourceBranch}" : "pull_request:${params.PR_ID}"
+ def isBranch = sourceValue('SOURCE_MODE') == 'branch'
+ def sourceBranch = isBranch ? params.SOURCE_BRANCH : sourceValue('PR_HEAD_BRANCH')
+ def sourceSha = isBranch ? params.SOURCE_SHA : sourceValue('PR_HEAD_SHA')
+ def sourceLabel = isBranch ? "branch:${sourceBranch}" : "pull_request:${sourceValue('PR_ID')}"
def remoteRef = isBranch ?
"refs/remotes/origin/source/${sourceBranch}" :
- "refs/remotes/origin/pr/${params.PR_ID}"
+ "refs/remotes/origin/pr/${sourceValue('PR_ID')}"
def remote = [
url: params.CLOUDSTACK_GIT_URL,
refspec: isBranch ?
"+refs/heads/${sourceBranch}:${remoteRef}" :
- "+refs/pull/${params.PR_ID}/head:${remoteRef}"
+ "+refs/pull/${sourceValue('PR_ID')}/head:${remoteRef}"
]
if (params.GIT_CREDENTIALS_ID?.trim()) {
remote.credentialsId = params.GIT_CREDENTIALS_ID
@@ -987,9 +1248,9 @@ void deployAndRunOntapIntegration() {
env.VM_NAME = vm.vcenter_vm as String
env.VM_SNAPSHOT = vm.snapshot as String
env.VM_BRIDGE = vm.bridge as String
- def sourceDescription = params.SOURCE_MODE == 'branch' ?
+ def sourceDescription = sourceValue('SOURCE_MODE') == 'branch' ?
"branch=${params.SOURCE_BRANCH}" :
- "PR-${params.PR_ID}"
+ "PR-${sourceValue('PR_ID')}"
currentBuild.description = "${sourceDescription} VM=${env.VM_ID}"
echo "Reserved compatible VM '${env.VM_ID}' via '${env.LOCKED_VM}'."
@@ -1063,7 +1324,11 @@ void deployAndRunOntapIntegration() {
}
void publishPresubmitResults() {
- if (params.SOURCE_MODE == 'discover') {
+ if (isMultibranchPullRequest() && currentBuild.currentResult == 'NOT_BUILT') {
+ echo "Multibranch PR-${env.CHANGE_ID} was not built."
+ return
+ }
+ if (sourceValue('SOURCE_MODE') == 'discover') {
echo 'Discovery build completed; worker reporting was skipped.'
return
}
@@ -1088,15 +1353,15 @@ void publishPresubmitResults() {
if (!env.PRESUBMIT_MAIL_TO?.trim()) {
resolvePresubmitMailRecipient()
}
- def sourceName = params.SOURCE_MODE == 'branch' ?
- "branch ${params.SOURCE_BRANCH}" : "PR-${params.PR_ID}"
+ def sourceName = sourceValue('SOURCE_MODE') == 'branch' ?
+ "branch ${params.SOURCE_BRANCH}" : "PR-${sourceValue('PR_ID')}"
try {
if (fileExists('private-cicd/scripts/generate-presubmit-report.py')) {
withEnv([
"PRESUBMIT_REPORT_RESULT=${result}",
"PRESUBMIT_REPORT_SOURCE=${sourceName}",
- "PRESUBMIT_REPORT_SOURCE_URL=${params.PR_URL ?: ''}",
- "PRESUBMIT_REPORT_TITLE=${params.PR_TITLE ?: ''}",
+ "PRESUBMIT_REPORT_SOURCE_URL=${sourceValue('PR_URL') ?: ''}",
+ "PRESUBMIT_REPORT_TITLE=${sourceValue('PR_TITLE') ?: ''}",
"PRESUBMIT_REPORT_SHA=${env.RESOLVED_SOURCE_SHA}",
"PRESUBMIT_REPORT_DIFF=${env.PRESUBMIT_DIFF_NUMBER ?: ''}",
"PRESUBMIT_REPORT_VM=${env.VM_ID ?: ''}"
@@ -1193,15 +1458,11 @@ spec:
timeout(time: 8, unit: 'HOURS')
}
- triggers {
- cron('H/5 * * * *')
- }
-
parameters {
choice(
name: 'SOURCE_MODE',
choices: ['discover', 'pull_request', 'branch', 'diagnose'],
- description: 'Timer builds use discover. Self-queued workers use pull_request. Use branch for a manual exact-SHA build, or diagnose to probe GitHub App Checks access using PR_HEAD_SHA.'
+ description: 'Legacy discover mode is not scheduled. Multibranch pull requests ignore this default. Use pull_request for a manual PR build, branch for an exact SHA, or diagnose to probe GitHub App Checks access.'
)
string(name: 'SOURCE_BRANCH', defaultValue: '', description: 'Manual branch mode: remote branch name, for example feature/CSTACKEX-223.')
string(name: 'SOURCE_SHA', defaultValue: '', description: 'Manual branch mode: exact full 40-character commit SHA.')
@@ -1260,7 +1521,7 @@ spec:
stage('Discover PR revisions') {
when {
expression {
- return params.SOURCE_MODE == 'discover'
+ return sourceValue('SOURCE_MODE') == 'discover' && !isMultibranchPullRequest()
}
}
steps {
@@ -1270,6 +1531,19 @@ spec:
}
}
+ stage('Prepare multibranch pull request') {
+ when {
+ expression {
+ return isMultibranchPullRequest()
+ }
+ }
+ steps {
+ script {
+ prepareMultibranchPullRequest()
+ }
+ }
+ }
+
stage('Validate source request') {
when {
expression {
diff --git a/private-cicd/README.md b/private-cicd/README.md
index ae48b35a9a75..df978fd69ddf 100644
--- a/private-cicd/README.md
+++ b/private-cicd/README.md
@@ -28,10 +28,11 @@ current implementation contract, see
## How it works
-The production Jenkins job loads [`Jenkinsfile`](Jenkinsfile) and helper scripts
-from the protected NetApp `main` branch. Every five minutes a short discovery
-build polls GitHub and self-queues a worker for each unseen PR head SHA. Each
-worker checks out the exact commit into a separate `cloudstack-src` directory,
+The production job is the multibranch folder `cloudstack-ontap-presubmit-mb`.
+Jenkins scans GitHub about once a minute. That scan is not a build. Each
+non-draft origin pull request into `main` gets its own child job. The child
+loads [`Jenkinsfile`](Jenkinsfile) and helper scripts from protected `main`,
+then checks the exact pull-request head SHA out into `cloudstack-src`. A child
then:
1. validates the request, builder, credentials, and VM inventory;
@@ -48,10 +49,10 @@ replacing the underlying build or test result.
## Entry points
-- **Pull request:** `SOURCE_MODE=discover` notices an open non-draft PR update
- to `main` and self-queues at most five `SOURCE_MODE=pull_request` workers in
- the same job. Extra SHAs wait for a later poll. Workers then wait on the
- `cloudstack-presubmit-vm` lock for a lab VM.
+- **Pull request:** a folder scan creates one child job per open non-draft
+ origin pull request into `main`. Drafts get no job. Compiles run together.
+ Integration waits on the `cloudstack-presubmit-vm` lock. There is no
+ five-worker cap. GitHub cannot call this Jenkins, so there is no webhook.
- **Manual branch:** a separate triggerless job uses `SOURCE_MODE=branch` with
a remote branch and exact 40-character commit SHA.
- **Local validation:** `scripts/validate-local.sh` checks the CI files without
diff --git a/private-cicd/docs/CREATE-PRESUBMIT-JOB.md b/private-cicd/docs/CREATE-PRESUBMIT-JOB.md
index c3ce8623a945..9e1b46871e85 100644
--- a/private-cicd/docs/CREATE-PRESUBMIT-JOB.md
+++ b/private-cicd/docs/CREATE-PRESUBMIT-JOB.md
@@ -19,12 +19,18 @@
# Create the auto-triggered ONTAP presubmit job
-This guide creates **one** Pipeline job, `cloudstack-ontap-presubmit`. Timer
-builds poll GitHub every five minutes and self-queue one worker build for each
-new eligible PR head SHA. Worker builds compile and package concurrently, then
-wait for an available `cloudstack-presubmit-vm` Lockable Resource before
-integration. Until `feature/CSTACKEX-223` is merged, the job loads trusted CI
-from that branch. After merge, change only **Branches to build** to `*/main`.
+The current job is the multibranch folder `cloudstack-ontap-presubmit-mb`.
+It scans GitHub about once a minute and creates one child job for every open
+non-draft origin pull request into `main`. Remote Jenkinsfile Provider loads
+`private-cicd/Jenkinsfile` from `main`. Match branches stays off. Draft pull
+requests get no job. The folder include filter can be narrowed to one `PR-`
+for the first run, then returned to `PR-*`. Do not add a webhook.
+
+The sections below describe the retired single Pipeline job and its five-minute
+`SOURCE_MODE=discover` timer. Do not create that second job. Keep
+`cloudstack-ontap-presubmit` disabled after the folder works. Do not delete the
+GitHub App, credentials, or the VM lock until a multibranch pull request has
+passed.
Do not create a GitHub repository webhook. OpenLab Jenkins is not reachable
from GitHub.com, so inbound `pull_request` deliveries fail with
@@ -36,8 +42,8 @@ item for this presubmit: that would run `Jenkinsfile` from the PR branch. This
design loads trusted CI from a fixed branch and checks the PR SHA out
separately.
-Do not create a separate poller, Multibranch Pipeline, or second production
-copy of this job.
+The multibranch folder above is the production job. Do not create another
+poller or a second production copy.
This guide starts before the Jenkins item exists and ends after polling and
required Check have been verified. Follow sections 1 through 21 in order. Do
diff --git a/private-cicd/docs/PRIVATE-CICD-GUIDE.md b/private-cicd/docs/PRIVATE-CICD-GUIDE.md
index a5617af9e47a..367408093e84 100644
--- a/private-cicd/docs/PRIVATE-CICD-GUIDE.md
+++ b/private-cicd/docs/PRIVATE-CICD-GUIDE.md
@@ -30,9 +30,9 @@ This is the consolidated operator and developer reference. The current [`Jenkins
The guide covers:
-- the trusted production Pipeline-from-SCM job with five-minute GitHub API
- discovery, including first-PR cutover from `feature/CSTACKEX-223` to `main`
- ([`CREATE-PRESUBMIT-JOB.md`](CREATE-PRESUBMIT-JOB.md));
+- the multibranch folder `cloudstack-ontap-presubmit-mb`, which scans about
+ once a minute and runs trusted CI from `main` for every non-draft origin
+ pull request into `main` ([`CREATE-PRESUBMIT-JOB.md`](CREATE-PRESUBMIT-JOB.md));
- the separate triggerless manual branch job;
- local CI-file validation and direct disposable-VM validation;
- Stage 1 build, unit-test, and Debian package handoff;
@@ -68,12 +68,10 @@ VM lock.
## 3. Architecture and data flow
```text
-Jenkins timer -> discover mode (every 5 minutes)
- -> GitHub API -> unseen PR head SHAs
- -> self-queue at most MAX_ACTIVE_WORKERS pull_request builds
- -> extra SHAs deferred with watermark held
- -> self-queued worker builds in the same job
- -> trusted Jenkinsfile/scripts from protected main
+Folder scan about once a minute (not a pipeline build)
+ -> one child job per open non-draft origin PR into main
+ -> trusted Jenkinsfile/scripts from protected main
+ -> exact indexed PR head SHA, confirmed with the pull request API
-> request, eligibility, credential, inventory, and builder checks
-> exact PR or branch SHA checkout into cloudstack-src
-> Maven clean install and unit tests
@@ -156,12 +154,10 @@ Lightweight checkout: off
PR code cannot replace trusted CI helpers before credentials are used. The source under test is checked out
separately.
-Until `feature/CSTACKEX-223` is merged, that Jenkinsfile is not on `main`. Create one job named
-`cloudstack-ontap-presubmit` that loads CI from `*/feature/CSTACKEX-223`, using
-[`CREATE-PRESUBMIT-JOB.md`](CREATE-PRESUBMIT-JOB.md). Its timer builds discover work and self-queue workers in the
-same item. After merge, change only **Branches to build** to `*/main`. Keep the same job, GitHub App, credentials,
-resources, and Check. Do not create a GitHub repository webhook: OpenLab Jenkins is not reachable from GitHub.com.
-Do not create a GitHub Organization folder for this presubmit.
+Production is the multibranch folder `cloudstack-ontap-presubmit-mb`. Remote Jenkinsfile
+Provider loads this Jenkinsfile from `main` with Match branches off. A folder scan,
+not a webhook, finds pull requests. Keep the old `cloudstack-ontap-presubmit` job
+disabled. Do not create a GitHub repository webhook or a GitHub Organization folder.
### Manual-job trust boundary
diff --git a/private-cicd/scripts/resolve-pull-request.py b/private-cicd/scripts/resolve-pull-request.py
new file mode 100644
index 000000000000..01f90de0a50a
--- /dev/null
+++ b/private-cicd/scripts/resolve-pull-request.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Decide whether a multibranch build may test one origin pull request."""
+
+import argparse
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+
+
+API_ROOT = "https://api.github.com"
+
+
+def parse_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--repository", required=True)
+ parser.add_argument("--pr-id", required=True)
+ parser.add_argument("--indexed-sha", required=True)
+ parser.add_argument("--change-target", required=True)
+ parser.add_argument("--output", required=True)
+ return parser.parse_args()
+
+
+def github_get(url, token):
+ request = urllib.request.Request(
+ url,
+ headers={
+ "Accept": "application/vnd.github+json",
+ "Authorization": f"Bearer {token}",
+ "X-GitHub-Api-Version": "2022-11-28",
+ "User-Agent": "cloudstack-ontap-presubmit",
+ },
+ )
+ try:
+ with urllib.request.urlopen(request, timeout=60) as response:
+ return json.loads(response.read().decode("utf-8"))
+ except urllib.error.HTTPError as error:
+ detail = error.read().decode("utf-8", errors="replace")
+ raise SystemExit(f"GitHub API {error.code} for {url}: {detail}") from error
+
+
+def is_main(ref):
+ return ref in ("main", "refs/heads/main")
+
+
+def assess(pull, pr_id, indexed_sha, change_target, expected_repository):
+ head = (pull or {}).get("head") or {}
+ base = (pull or {}).get("base") or {}
+ user = (pull or {}).get("user") or {}
+ repository = head.get("repo") or {}
+ decision = {
+ "eligible": False,
+ "reason": "",
+ "pr_id": str((pull or {}).get("number") or ""),
+ "pr_head_sha": head.get("sha") or "",
+ "pr_head_branch": head.get("ref") or "",
+ "pr_base_branch": base.get("ref") or "",
+ "pr_draft": bool((pull or {}).get("draft")),
+ "pr_title": (pull or {}).get("title") or "",
+ "pr_url": (pull or {}).get("html_url") or "",
+ "pr_author_login": user.get("login") or "",
+ "pr_author_email": user.get("email") or "",
+ "head_repository": repository.get("full_name") or "",
+ }
+ if not isinstance(pull, dict):
+ decision["reason"] = "GitHub pull request payload is missing."
+ elif decision["pr_id"] != str(pr_id):
+ decision["reason"] = "GitHub pull request number does not match the build."
+ elif decision["head_repository"] != expected_repository:
+ decision["reason"] = (
+ f"head repository '{decision['head_repository']}' is not "
+ f"'{expected_repository}'."
+ )
+ elif not is_main(decision["pr_base_branch"]) or not is_main(change_target):
+ decision["reason"] = (
+ f"base branch is '{decision['pr_base_branch']}' and change target "
+ f"is '{change_target}'."
+ )
+ elif decision["pr_draft"]:
+ decision["reason"] = "pull request is a draft."
+ elif not decision["pr_head_sha"] or len(decision["pr_head_sha"]) != 40:
+ decision["reason"] = "GitHub pull request head SHA is missing."
+ elif (indexed_sha or "").strip() and decision["pr_head_sha"].lower() != indexed_sha.strip().lower():
+ decision["reason"] = "indexed revision does not match the pull request head SHA."
+ else:
+ decision["eligible"] = True
+ return decision
+
+
+def main():
+ args = parse_args()
+ token = os.environ.get("GITHUB_ACCESS_TOKEN", "").strip()
+ if not token:
+ raise SystemExit("GITHUB_ACCESS_TOKEN is empty.")
+ if not args.pr_id.isdigit():
+ raise SystemExit("PR id must be numeric.")
+ pull = github_get(
+ f"{API_ROOT}/repos/{args.repository}/pulls/{args.pr_id}",
+ token,
+ )
+ decision = assess(
+ pull,
+ args.pr_id,
+ args.indexed_sha,
+ args.change_target,
+ args.repository,
+ )
+ with open(args.output, "w", encoding="utf-8") as handle:
+ json.dump(decision, handle)
+ handle.write("\n")
+ if not decision["eligible"]:
+ print(decision["reason"], file=sys.stderr)
+ raise SystemExit(2)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/private-cicd/scripts/test_resolve_pull_request.py b/private-cicd/scripts/test_resolve_pull_request.py
new file mode 100644
index 000000000000..739fe10a89e0
--- /dev/null
+++ b/private-cicd/scripts/test_resolve_pull_request.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Unit tests for resolve-pull-request.py."""
+
+import importlib.util
+import unittest
+from pathlib import Path
+
+
+SCRIPT = Path(__file__).with_name("resolve-pull-request.py")
+SPEC = importlib.util.spec_from_file_location("resolve_pull_request", SCRIPT)
+MODULE = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(MODULE)
+
+SHA = "a" * 40
+
+
+def pull(number="98", draft=False, base="main", repo="NetApp/cloudstack", sha=SHA):
+ return {
+ "number": int(number),
+ "title": "Test",
+ "html_url": f"https://github.com/NetApp/cloudstack/pull/{number}",
+ "draft": draft,
+ "base": {"ref": base},
+ "head": {
+ "ref": "feature/test",
+ "sha": sha,
+ "repo": {"full_name": repo} if repo is not None else None,
+ },
+ "user": {"login": "tester", "email": None},
+ }
+
+
+class AssessPullRequestTest(unittest.TestCase):
+ def test_accepts_matching_origin_pr(self):
+ decision = MODULE.assess(pull(), "98", SHA, "main", "NetApp/cloudstack")
+
+ self.assertTrue(decision["eligible"])
+ self.assertEqual("", decision["reason"])
+ self.assertEqual("tester", decision["pr_author_login"])
+ self.assertEqual("", decision["pr_author_email"])
+
+ def test_rejects_draft_fork_base_and_sha_mismatch(self):
+ cases = [
+ pull(draft=True),
+ pull(repo="other/cloudstack"),
+ pull(repo=None),
+ pull(base="release"),
+ pull(sha="b" * 40),
+ ]
+
+ for candidate in cases:
+ decision = MODULE.assess(
+ candidate, "98", SHA, "main", "NetApp/cloudstack")
+ self.assertFalse(decision["eligible"], candidate)
+
+ def test_accepts_github_head_when_jenkins_sha_is_missing(self):
+ decision = MODULE.assess(pull(), "98", "", "main", "NetApp/cloudstack")
+
+ self.assertTrue(decision["eligible"])
+ self.assertEqual(SHA, decision["pr_head_sha"])
+
+ def test_rejects_change_target_other_than_main(self):
+ decision = MODULE.assess(
+ pull(), "98", SHA, "feature/other", "NetApp/cloudstack")
+
+ self.assertFalse(decision["eligible"])
+ self.assertIn("change target", decision["reason"])
+
+
+if __name__ == "__main__":
+ unittest.main()