587 lines
23 KiB
Groovy
587 lines
23 KiB
Groovy
#!groovy
|
|
|
|
import com.freeleaps.devops.SourceFetcher
|
|
import com.freeleaps.devops.DependenciesResolver
|
|
import com.freeleaps.devops.CommitMessageLinter
|
|
import com.freeleaps.devops.ChangedComponentsDetector
|
|
import com.freeleaps.devops.CodeLintExecutor
|
|
import com.freeleaps.devops.SASTExecutor
|
|
import com.freeleaps.devops.ImageBuilder
|
|
import com.freeleaps.devops.SemanticReleasingExecutor
|
|
import com.freeleaps.devops.ArgoApplicationVersionUpdater
|
|
|
|
import com.freeleaps.devops.enums.DependenciesManager
|
|
import com.freeleaps.devops.enums.ServiceLanguage
|
|
import com.freeleaps.devops.enums.CodeLinterTypes
|
|
import com.freeleaps.devops.enums.ImageBuilderTypes
|
|
|
|
def generateComponentStages(component, configurations) {
|
|
def stages = []
|
|
stages.addAll([
|
|
// Build Agent Setup
|
|
{stage("${component.name} :: Build Agent Setup") {
|
|
script {
|
|
if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) {
|
|
def buildAgentImage = component.buildAgentImage
|
|
if (buildAgentImage == null || buildAgentImage.isEmpty()) {
|
|
log.warn("Pipeline", "Not set buildAgentImage for ${component.name}, using default build agent image")
|
|
|
|
def language = ServiceLanguage.parse(component.language)
|
|
switch(language) {
|
|
case ServiceLanguage.PYTHON:
|
|
buildAgentImage = "docker.io/python:3.10-slim-buster"
|
|
break
|
|
case ServiceLanguage.JS:
|
|
buildAgentImage = "docker.io/node:lts-alpine"
|
|
break
|
|
default:
|
|
error("Unknown service language")
|
|
}
|
|
}
|
|
log.info("Pipeline", "Using ${buildAgentImage} as build agent image for ${component.name}")
|
|
env."${component.name}_buildAgentImage" = buildAgentImage
|
|
}
|
|
}
|
|
}},
|
|
// Dependencies Resolving
|
|
{stage("${component.name} :: Dependencies Resolving") {
|
|
podTemplate(
|
|
label: "dep-resolver-${component.name}",
|
|
containers: [
|
|
containerTemplate(
|
|
name: 'dep-resolver',
|
|
image: env."${component.name}_buildAgentImage",
|
|
ttyEnabled: true,
|
|
command: 'sleep',
|
|
args: 'infinity'
|
|
)
|
|
]
|
|
) {
|
|
node("dep-resolver-${component.name}") {
|
|
container('dep-resolver') {
|
|
script {
|
|
if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) {
|
|
def buildAgentImage = env."${component.name}_buildAgentImage"
|
|
log.info("Pipeline", "Using ${buildAgentImage} as build agent image for dependencies resolving")
|
|
def sourceFetcher = new SourceFetcher(this)
|
|
sourceFetcher.fetch(configurations)
|
|
|
|
def language = ServiceLanguage.parse(component.language)
|
|
def depManager = DependenciesManager.parse(component.dependenciesManager)
|
|
|
|
def dependenciesResolver = new DependenciesResolver(this, language, env.workroot + "/" + component.root + "/")
|
|
dependenciesResolver.useManager(depManager)
|
|
|
|
if (component.buildCacheEnabled) {
|
|
dependenciesResolver.enableCachingSupport()
|
|
} else {
|
|
dependenciesResolver.disableCachingSupport()
|
|
}
|
|
|
|
dependenciesResolver.resolve(component)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}},
|
|
])
|
|
|
|
if (component.lintEnabled != null && component.lintEnabled) {
|
|
stages.addAll([
|
|
// Code Linter Environment Preparation
|
|
{stage("${component.name} :: Code Linter Preparation") {
|
|
script {
|
|
if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) {
|
|
if (component.lintEnabled != null && component.lintEnabled) {
|
|
log.info("Pipeline", "Code linting has enabled, preparing linter...")
|
|
|
|
if (component.linter == null || component.linter.isEmpty()) {
|
|
log.error("Pipeline", "Not set linter for ${component.name}, using default linter settings as fallback")
|
|
}
|
|
|
|
def linter = CodeLinterTypes.parse(component.linter)
|
|
|
|
if (linter == null) {
|
|
log.error("Pipeline", "Unknown linter for ${component.name}, skipping code linting")
|
|
}
|
|
|
|
if (linter.language != ServiceLanguage.parse(component.language)) {
|
|
log.error("Pipeline", "Linter ${linter.linter} is not supported for ${component.language}, skipping code linting")
|
|
}
|
|
|
|
log.info("Pipeline", "Using ${linter.linter} with image ${linter.containerImage} as linter for ${component.name}")
|
|
env."${component.name}_linterContainerImage" = linter.containerImage
|
|
} else {
|
|
log.info("Pipeline", "Code linting is not enabled for ${component.name}, skipping...")
|
|
}
|
|
}
|
|
}
|
|
}},
|
|
// Code Linting
|
|
{stage("${component.name} :: Code Linting") {
|
|
podTemplate(
|
|
label: "code-linter-${component.name}",
|
|
containers: [
|
|
containerTemplate(
|
|
name: 'code-linter',
|
|
image: env."${component.name}_linterContainerImage",
|
|
ttyEnabled: true,
|
|
command: 'sleep',
|
|
args: 'infinity'
|
|
)
|
|
]
|
|
) {
|
|
node("code-linter-${component.name}") {
|
|
container('code-linter') {
|
|
script {
|
|
if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) {
|
|
if (component.lintEnabled != null && component.lintEnabled) {
|
|
log.info("Pipeline", "Code linting has enabled, linting code...")
|
|
|
|
def sourceFetcher = new SourceFetcher(this)
|
|
sourceFetcher.fetch(configurations)
|
|
|
|
def linterType = CodeLinterTypes.parse(component.linter)
|
|
def language = ServiceLanguage.parse(component.language)
|
|
|
|
def depManager = DependenciesManager.parse(component.dependenciesManager)
|
|
|
|
// resolving deps from cache
|
|
def dependenciesResolver = new DependenciesResolver(this, language, env.workroot + "/" + component.root + "/")
|
|
dependenciesResolver.useManager(depManager)
|
|
|
|
if (component.buildCacheEnabled) {
|
|
dependenciesResolver.enableCachingSupport()
|
|
} else {
|
|
dependenciesResolver.disableCachingSupport()
|
|
}
|
|
|
|
dependenciesResolver.resolve(component)
|
|
|
|
// lint codes
|
|
def codeLintExecutor = new CodeLintExecutor(this, env.workroot + "/" + component.root + "/", component.linterConfig, linterType, component)
|
|
codeLintExecutor.execute()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}}
|
|
])
|
|
}
|
|
|
|
if (component.sastEnabled != null && component.sastEnabled) {
|
|
stages.addAll([
|
|
// SAST Scanner Environment Preparation
|
|
{stage("${component.name} :: SAST Scanner Preparation") {
|
|
script {
|
|
if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) {
|
|
if (component.sastEnabled != null && component.sastEnabled) {
|
|
log.info("Pipeline", "SAST scanning has enabled, preparing scanner...")
|
|
|
|
if (sastScanner == null || sastScanner.isEmpty()) {
|
|
log.error("Pipeline", "Not set sastScanner for ${component.name}")
|
|
}
|
|
|
|
def sastScannerType = SASTScannerTypes.parse(component.sastScanner)
|
|
if (sastScannerType == null) {
|
|
log.error("Pipeline", "Unknown SAST scanner for ${component.name}, skipping SAST scanning")
|
|
} else if (sastScannerType.language != ServiceLanguage.parse(component.language)) {
|
|
log.error("Pipeline", "SAST scanner ${sastScannerType.scanner} is not supported for ${component.language}, skipping SAST scanning")
|
|
} else {
|
|
log.info("Pipeline", "Using ${sastScanner} as SAST scanner for ${component.name}")
|
|
env."${component.name}_sastScannerContainerImage" = sastScannerType.containerImage
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}},
|
|
// SAST Scanning
|
|
{stage("${component.name} :: SAST Scanning") {
|
|
when {
|
|
expression {
|
|
return (env.executeMode == "fully" || env.changedComponents.contains(component.name)) && env.sastScannerContainerImage != null && !env.sastScannerContainerImage.isEmpty()
|
|
}
|
|
}
|
|
podTemplate(
|
|
label: "sast-scanner-${component.name}",
|
|
containers: [
|
|
containerTemplate(
|
|
name: 'sast-scanner',
|
|
image: env."${component.name}_sastScannerContainerImage",
|
|
ttyEnabled: true,
|
|
command: 'sleep',
|
|
args: 'infinity'
|
|
)
|
|
]
|
|
) {
|
|
node("sast-scanner-${component.name}") {
|
|
container('sast-scanner') {
|
|
script {
|
|
if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) {
|
|
if (component.sastEnabled != null && component.sastEnabled) {
|
|
log.info("Pipeline", "SAST scanning has enabled, scanning code...")
|
|
|
|
def sourceFetcher = new SourceFetcher(this)
|
|
sourceFetcher.fetch(configurations)
|
|
|
|
def sastScannerType = SASTScannerTypes.parse(component.sastScanner)
|
|
|
|
def sastScanner = new SASTExecutor(this, env.workroot + "/" + component.root + "/", sastScannerType)
|
|
sastScanner.scan()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}}
|
|
])
|
|
}
|
|
|
|
if (component.semanticReleaseEnabled != null && component.semanticReleaseEnabled) {
|
|
stages.addAll([
|
|
// Semantic Releasing
|
|
{stage("${component.name} :: Semantic Releasing") {
|
|
podTemplate(
|
|
label: "semantic-releasing-${component.name}",
|
|
containers: [
|
|
containerTemplate(
|
|
name: 'semantic-releasing',
|
|
image: 'node:18-bullseye-slim',
|
|
ttyEnabled: true,
|
|
command: 'sleep',
|
|
args: 'infinity'
|
|
)
|
|
]
|
|
) {
|
|
node("semantic-releasing-${component.name}") {
|
|
container('semantic-releasing') {
|
|
script {
|
|
if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) {
|
|
if (component.semanticReleaseEnabled != null && component.semanticReleaseEnabled) {
|
|
log.info("Pipeline", "Semantic releasing has enabled, releasing...")
|
|
|
|
if (env.SEMANTIC_RELEASED != null && !env.SEMANTIC_RELEASED.isEmpty() && env.SEMANTIC_RELEASED) {
|
|
log.info("Pipeline", "Semantic release has been executed, skipping...")
|
|
return
|
|
}
|
|
|
|
def sourceFetcher = new SourceFetcher(this)
|
|
sourceFetcher.fetch(configurations)
|
|
|
|
def semanticReleasingExecutor = new SemanticReleasingExecutor(this, env.workroot)
|
|
semanticReleasingExecutor.release(configurations.serviceGitCredentialsId)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}}
|
|
])
|
|
}
|
|
|
|
stages.addAll([
|
|
// Compilation & Packaging
|
|
{stage("${component.name} :: Compilation & Packaging") {
|
|
podTemplate(
|
|
label: "build-agent-${component.name}",
|
|
containers: [
|
|
containerTemplate(
|
|
name: 'build-agent',
|
|
image: env."${component.name}_buildAgentImage",
|
|
ttyEnabled: true,
|
|
command: 'sleep',
|
|
args: 'infinity',
|
|
resourceLimitCpu: "1",
|
|
resourceLimitMemory: "2Gi",
|
|
resourceRequestCpu: "0.5",
|
|
resourceRequestMemory: "1Gi"
|
|
)
|
|
]
|
|
) {
|
|
node("build-agent-${component.name}") {
|
|
container('build-agent') {
|
|
script {
|
|
if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) {
|
|
def buildAgentImage = env."${component.name}_buildAgentImage"
|
|
log.info("Pipeline", "Using ${buildAgentImage} as build agent image for compilation & packaging")
|
|
|
|
def sourceFetcher = new SourceFetcher(this)
|
|
sourceFetcher.fetch(configurations)
|
|
|
|
def language = ServiceLanguage.parse(component.language)
|
|
def depManager = DependenciesManager.parse(component.dependenciesManager)
|
|
|
|
def dependenciesResolver = new DependenciesResolver(this, language, env.workroot + "/" + component.root + "/")
|
|
dependenciesResolver.useManager(depManager)
|
|
if (component.buildCacheEnabled) {
|
|
dependenciesResolver.enableCachingSupport()
|
|
} else {
|
|
dependenciesResolver.disableCachingSupport()
|
|
}
|
|
|
|
dependenciesResolver.resolve(component)
|
|
|
|
dir(env.workroot + "/" + component.root) {
|
|
if (component.buildCommand != null && !component.buildCommand.isEmpty()) {
|
|
sh component.buildCommand
|
|
}
|
|
component.buildArtifacts.each { artifact ->
|
|
log.info("Pipeline", "Stashing artifact ${artifact} for ${component.name}...")
|
|
def artifactList = sh(script: "ls ${artifact} -al", returnStdout: true)
|
|
log.info("Pipeline", "Artifacts list: ${artifactList}")
|
|
def targetPathType = sh(
|
|
script: """
|
|
if [ -d "${artifact}" ]; then
|
|
echo "dir"
|
|
elif [ -f "${artifact}" ]; then
|
|
echo "file"
|
|
else
|
|
echo "unknown"
|
|
fi
|
|
""",
|
|
returnStdout: true
|
|
)
|
|
if (artifact == '.' || artifact == './') {
|
|
log.info("Pipeline", "Stashing root directory for ${component.name}...")
|
|
stash includes: "", name: "${component.name}-root"
|
|
} else if (targetPathType.trim() == "dir") {
|
|
stash includes: "${artifact}/**", name: "${component.name}-${artifact}"
|
|
} else {
|
|
stash includes: artifact, name: "${component.name}-${artifact}"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}},
|
|
// Image Builder Setup
|
|
{stage("${component.name} :: Image Builder Setup") {
|
|
script {
|
|
if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) {
|
|
log.info("Pipeline", "Ready to setup image builder for ${component.name}")
|
|
def imageBuilder
|
|
if (component.imageBuilder == null || component.imageBuilder.isEmpty()) {
|
|
log.info("Pipeline", "imageBuilder not set for ${component.name}, using kaniko as default image builder")
|
|
imageBuilder = ImageBuilderTypes.KANIKO
|
|
} else {
|
|
imageBuilder = ImageBuilderTypes.parse(component.imageBuilder)
|
|
if (imageBuilder == null) {
|
|
log.error("Pipeline", "Unknown image builder for ${component.name}, skipping image building")
|
|
}
|
|
}
|
|
|
|
env."${component.name}_imageBuilderImage" = imageBuilder.image
|
|
log.info("Pipeline", "Using ${imageBuilder.builder} (image: ${imageBuilder.image}) as image builder for ${component.name}")
|
|
}
|
|
}
|
|
}},
|
|
// Image Building & Publishing
|
|
{stage("${component.name} :: Image Building & Publishing") {
|
|
podTemplate(
|
|
label: "image-builder-${component.name}",
|
|
containers: [
|
|
containerTemplate(
|
|
name: 'image-builder',
|
|
image: env."${component.name}_imageBuilderImage",
|
|
privileged: true,
|
|
ttyEnabled: true,
|
|
command: 'sleep',
|
|
args: 'infinity'
|
|
)
|
|
],
|
|
volumes: [
|
|
hostPathVolume(hostPath: '/var/run/docker.sock', mountPath: '/var/run/docker.sock')
|
|
]
|
|
) {
|
|
node("image-builder-${component.name}") {
|
|
container('image-builder') {
|
|
script {
|
|
if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) {
|
|
def sourceFetcher = new SourceFetcher(this)
|
|
sourceFetcher.fetch(configurations)
|
|
|
|
dir(env.workroot + "/" + component.root) {
|
|
component.buildArtifacts.each { artifact ->
|
|
if (artifact == '.' || artifact == './') {
|
|
unstash "${component.name}-root"
|
|
} else {
|
|
unstash "${component.name}-${artifact}"
|
|
}
|
|
}
|
|
|
|
if (component.dockerfile != null && !component.dockerfile.isEmpty()) {
|
|
log.error("Pipeline", "Component ${component.name} dockerfile not set!")
|
|
}
|
|
|
|
def imageBuilderType = ImageBuilderTypes.parse(component.imageBuilder)
|
|
if (imageBuilderType == null) {
|
|
log.error("Pipeline", "Unknown image builder for ${component.name}, skipping image building")
|
|
}
|
|
|
|
def imageBuilder = new ImageBuilder(this,
|
|
env.workroot + "/" + component.root + "/",
|
|
component.imageBuildRoot,
|
|
component.dockerfilePath,
|
|
imageBuilderType
|
|
)
|
|
|
|
log.info("Pipeline", "Retrieve version of image from pervious stage...")
|
|
if (env.LATEST_VERSION == null || env.LATEST_VERSION.isEmpty()) {
|
|
log.warn("Pipeline", "LATEST_VERSION environment value not set, using 'snapshot-<BUILD_COMMIT_HASH>' as default version")
|
|
sh "git config --global --add safe.directory ${env.workroot}"
|
|
def commitHash = sh(script: 'git rev-parse HEAD', returnStdout: true).trim()
|
|
def shortCommitHash = commitHash.take(7)
|
|
env.LATEST_VERSION = "snapshot-${shortCommitHash}"
|
|
}
|
|
def version
|
|
|
|
imageBuilder.setManifestsOfImage(component.imageRegistry, component.imageRepository, component.imageName, env.LATEST_VERSION)
|
|
imageBuilder.useCredentials(component.registryCredentialsId)
|
|
imageBuilder.setArchitectures(component.imageReleaseArchitectures)
|
|
imageBuilder.build()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}},
|
|
{stage("${component.name} :: Argo Application Version Updating") {
|
|
script {
|
|
if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) {
|
|
def argoApplicationVersionUpdater = new ArgoApplicationVersionUpdater(this, configurations)
|
|
argoApplicationVersionUpdater.update(configurations.environmentSlug, component)
|
|
}
|
|
}
|
|
}}
|
|
])
|
|
return {
|
|
stages.each { stageClosure ->
|
|
stageClosure()
|
|
}
|
|
}
|
|
}
|
|
|
|
def call(Closure closure) {
|
|
def configurations = [:]
|
|
closure.resolveStrategy = Closure.DELEGATE_FIRST
|
|
closure.delegate = configurations
|
|
closure()
|
|
|
|
pipeline {
|
|
agent any
|
|
options {
|
|
buildDiscarder(logRotator(numToKeepStr: '25'))
|
|
timeout(time: 30, unit: 'MINUTES')
|
|
parallelsAlwaysFailFast()
|
|
}
|
|
|
|
stages {
|
|
stage("Pipeline :: Commit Linting If Enabled") {
|
|
when {
|
|
expression {
|
|
return configurations.commitMessageLintEnabled != null && configurations.commitMessageLintEnabled
|
|
}
|
|
}
|
|
agent {
|
|
kubernetes {
|
|
defaultContainer 'commit-message-linter'
|
|
yaml """
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
labels:
|
|
freeleaps-devops-system/milestone: commit-message-linting
|
|
spec:
|
|
containers:
|
|
- name: commit-message-linter
|
|
image: docker.io/commitlint/commitlint:master
|
|
command:
|
|
- cat
|
|
tty: true
|
|
volumeMounts:
|
|
- name: workspace
|
|
mountPath: /workspace
|
|
volumes:
|
|
- name: workspace
|
|
emptyDir: {}
|
|
"""
|
|
}
|
|
}
|
|
steps {
|
|
script {
|
|
log.info("Pipeline","Commit message linting is enabled")
|
|
def sourceFetcher = new SourceFetcher(this)
|
|
sourceFetcher.fetch(configurations)
|
|
|
|
def linter = new CommitMessageLinter(this)
|
|
linter.lint(configurations)
|
|
}
|
|
}
|
|
}
|
|
|
|
stage("Pipeline :: Execute Mode Detection") {
|
|
steps {
|
|
script {
|
|
def executeMode = configurations.executeMode
|
|
if (executeMode == null || executeMode.isEmpty()) {
|
|
log.warn("Pipeline","Not set executeMode, using fully as default execute mode")
|
|
env.executeMode = "fully"
|
|
} else if (executeMode == 'on-demand' && configurations.serviceGitRepoType != 'monorepo') {
|
|
log.warn("Pipeline","serviceGirRepoType is not monorepo, on-demand mode is not supported, using fully mode")
|
|
env.executeMode = "fully"
|
|
} else {
|
|
log.info("Pipeline","Using ${executeMode} as execute mode")
|
|
env.executeMode = executeMode
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage("Pipeline :: Code Changes Detection") {
|
|
when {
|
|
expression {
|
|
return env.executeMode == "on-demand"
|
|
}
|
|
}
|
|
|
|
steps {
|
|
script {
|
|
sourceFetcher.fetch(configurations)
|
|
|
|
def changedComponentsDetector = new ChangedComponentsDetector(this)
|
|
def changedComponents = changedComponentsDetector.detect(env.workroot, configurations.components)
|
|
|
|
log.info("Pipeline","Changed components: ${changedComponents}")
|
|
env.changedComponents = changedComponents.join(' ')
|
|
}
|
|
}
|
|
}
|
|
|
|
stage("Pipeline :: Components Build (Dynamic Generated Stages)") {
|
|
when {
|
|
expression {
|
|
return env.executeMode == "fully" || env.changedComponents.size() > 0
|
|
}
|
|
}
|
|
steps {
|
|
script {
|
|
configurations.components.each { component ->
|
|
log.info("Pipeline", "Executing generated stages for ${component.name}...")
|
|
generateComponentStages(component, configurations)()
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
}
|
|
} |