#!groovy import com.freeleaps.devops.SourceFetcher import com.freeleaps.devops.DependenciesResolver import com.freeleaps.devops.CommitMessageLinter import com.freeleaps.devops.ChangedComponentsDetector import com.freeleaps.devops.CodeLintExecutor import com.freeleaps.devops.SASTExecutor import com.freeleaps.devops.ImageBuilder import com.freeleaps.devops.SemanticReleasingExecutor import com.freeleaps.devops.enums.DependenciesManager import com.freeleaps.devops.enums.ServiceLanguage import com.freeleaps.devops.enums.CodeLinterTypes import com.freeleaps.devops.enums.ImageBuilderTypes def generateComponentStages(component, configurations) { def stages = [] stages.addAll([ // Build Agent Setup stage("${component.name} :: Build Agent Setup") { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { def buildAgentImage = component.buildAgentImage if (buildAgentImage == null || buildAgentImage.isEmpty()) { log.warn("Pipeline", "Not set buildAgentImage for ${component.name}, using default build agent image") def language = ServiceLanguage.parse(component.language) switch(language) { case ServiceLanguage.PYTHON: buildAgentImage = "docker.io/python:3.10-slim-buster" break case ServiceLanguage.JS: buildAgentImage = "docker.io/node:lts-alpine" break default: error("Unknown service language") } } log.info("Pipeline", "Using ${buildAgentImage} as build agent image for ${component.name}") env.buildAgentImage = buildAgentImage } } }, // Dependencies Resolving stage("${component.name} :: Dependencies Resolving") { podTemplate( label: "dep-resolver-${component.name}", containers: [ containerTemplate( name: 'dep-resolver', image: env.buildAgentImage, ttyEnabled: true, command: 'sleep', args: 'infinity' ) ] ) { node("dep-resolver-${component.name}") { container('dep-resolver') { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { log.info("Pipeline", "Using ${env.buildAgentImage} as build agent image for dependencies resolving") def sourceFetcher = new SourceFetcher(this) sourceFetcher.fetch(configurations) def language = ServiceLanguage.parse(component.language) def depManager = DependenciesManager.parse(component.dependenciesManager) def dependenciesResolver = new DependenciesResolver(this, language, env.workroot + "/" + component.root + "/") dependenciesResolver.useManager(depManager) if (component.buildCacheEnabled) { dependenciesResolver.enableCachingSupport() } else { dependenciesResolver.disableCachingSupport() } dependenciesResolver.resolve(component) } } } } } }, ]) if (component.lintEnabled != null && component.lintEnabled) { stages.addAll([ // Code Linter Environment Preparation stage("${component.name} :: Code Linter Preparation") { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { if (component.lintEnabled != null && component.lintEnabled) { log.info("Pipeline", "Code linting has enabled, preparing linter...") if (component.linter == null || component.linter.isEmpty()) { log.error("Pipeline", "Not set linter for ${component.name}, using default linter settings as fallback") } def linter = CodeLinterTypes.parse(component.linter) if (linter == null) { log.error("Pipeline", "Unknown linter for ${component.name}, skipping code linting") } if (linter.language != ServiceLanguage.parse(component.language)) { log.error("Pipeline", "Linter ${linter.linter} is not supported for ${component.language}, skipping code linting") } log.info("Pipeline", "Using ${linter.linter} with image ${linter.containerImage} as linter for ${component.name}") env.linterContainerImage = linter.containerImage } else { log.info("Pipeline", "Code linting is not enabled for ${component.name}, skipping...") } } } }, // Code Linting stage("${component.name} :: Code Linting") { podTemplate( label: "code-linter-${component.name}", containers: [ containerTemplate( name: 'code-linter', image: env.linterContainerImage, ttyEnabled: true, command: 'sleep', args: 'infinity' ) ] ) { node("code-linter-${component.name}") { container('code-linter') { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { if (component.lintEnabled != null && component.lintEnabled) { log.info("Pipeline", "Code linting has enabled, linting code...") def sourceFetcher = new SourceFetcher(this) sourceFetcher.fetch(configurations) def linterType = CodeLinterTypes.parse(component.linter) def language = ServiceLanguage.parse(component.language) def depManager = DependenciesManager.parse(component.dependenciesManager) // resolving deps from cache def dependenciesResolver = new DependenciesResolver(this, language, env.workroot + "/" + component.root + "/") dependenciesResolver.useManager(depManager) if (component.buildCacheEnabled) { dependenciesResolver.enableCachingSupport() } else { dependenciesResolver.disableCachingSupport() } dependenciesResolver.resolve(component) // lint codes def codeLintExecutor = new CodeLintExecutor(this, env.workroot + "/" + component.root + "/", component.linterConfig, linterType, component) codeLintExecutor.execute() } } } } } } } ]) } if (component.sastEnabled != null && component.sastEnabled) { stages.addAll([ // SAST Scanner Environment Preparation stage("${component.name} :: SAST Scanner Preparation") { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { if (component.sastEnabled != null && component.sastEnabled) { log.info("Pipeline", "SAST scanning has enabled, preparing scanner...") if (sastScanner == null || sastScanner.isEmpty()) { log.error("Pipeline", "Not set sastScanner for ${component.name}") } def sastScannerType = SASTScannerTypes.parse(component.sastScanner) if (sastScannerType == null) { log.error("Pipeline", "Unknown SAST scanner for ${component.name}, skipping SAST scanning") } else if (sastScannerType.language != ServiceLanguage.parse(component.language)) { log.error("Pipeline", "SAST scanner ${sastScannerType.scanner} is not supported for ${component.language}, skipping SAST scanning") } else { log.info("Pipeline", "Using ${sastScanner} as SAST scanner for ${component.name}") env.sastScannerContainerImage = sastScannerType.containerImage } } } } }, // SAST Scanning stage("${component.name} :: SAST Scanning") { when { expression { return (env.executeMode == "fully" || env.changedComponents.contains(component.name)) && env.sastScannerContainerImage != null && !env.sastScannerContainerImage.isEmpty() } } podTemplate( label: "sast-scanner-${component.name}", containers: [ containerTemplate( name: 'sast-scanner', image: env.sastScannerContainerImage, ttyEnabled: true, command: 'sleep', args: 'infinity' ) ] ) { node("sast-scanner-${component.name}") { container('sast-scanner') { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { if (component.sastEnabled != null && component.sastEnabled) { log.info("Pipeline", "SAST scanning has enabled, scanning code...") def sourceFetcher = new SourceFetcher(this) sourceFetcher.fetch(configurations) def sastScannerType = SASTScannerTypes.parse(component.sastScanner) def sastScanner = new SASTExecutor(this, env.workroot + "/" + component.root + "/", sastScannerType) sastScanner.scan() } } } } } } } ]) } if (component.semanticReleaseEnabled != null && component.semanticReleaseEnabled) { stages.addAll([ // Semantic Release Environment Preparation stage("${component.name} :: Semantic Release Preparation") { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { if (component.semanticReleaseEnabled != null && component.semanticReleaseEnabled) { log.info("Pipeline", "Semantic releasing has enabled, preparing semantic release environment...") env.semanticReleasingContainerImage = "docker.io/semantic-release/semantic-release:latest" } } } }, // Semantic Releasing stage("${component.name} :: Semantic Releasing") { podTemplate( label: "semantic-releasing-${component.name}", containers: [ containerTemplate( name: 'semantic-releasing', image: env.semanticReleasingContainerImage, ttyEnabled: true, command: 'sleep', args: 'infinity' ) ] ) { node("semantic-releasing-${component.name}") { container('semantic-releasing') { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { if (component.semanticReleaseEnabled != null && component.semanticReleaseEnabled) { log.info("Pipeline", "Semantic releasing has enabled, releasing...") def sourceFetcher = new SourceFetcher(this) sourceFetcher.fetch(configurations) def semanticReleasingExecutor = new SemanticReleasingExecutor(this, env.workroot + "/" + component.root + "/") semanticReleasingExecutor.release() } } } } } } } ]) } stages.addAll([ // Compilation & Packaging stage("${component.name} :: Compilation & Packaging") { podTemplate( label: "build-agent-${component.name}", containers: [ containerTemplate( name: 'build-agent', image: env.buildAgentImage, ttyEnabled: true, command: 'sleep', args: 'infinity' ) ] ) { node("build-agent-${component.name}") { container('build-agent') { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { log.info("Pipeline", "Using ${env.buildAgentImage} as build agent image for compilation & packaging") def sourceFetcher = new SourceFetcher(this) sourceFetcher.fetch(configurations) def language = ServiceLanguage.parse(component.language) def depManager = DependenciesManager.parse(component.dependenciesManager) def dependenciesResolver = new DependenciesResolver(this, language, env.workroot + "/" + component.root + "/") dependenciesResolver.useManager(depManager) if (component.buildCacheEnabled) { dependenciesResolver.enableCachingSupport() } else { dependenciesResolver.disableCachingSupport() } dependenciesResolver.resolve(component) dir(env.workroot + "/" + component.root) { if (component.buildCommand != null && !component.buildCommand.isEmpty()) { sh component.buildCommand } component.buildArtifacts.each { artifact -> stash includes: artifact, name: "${component.name}-${artifact}" } } } } } } } }, // Image Builder Setup stage("${component.name} :: Image Builder Setup") { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { log.info("Pipeline", "Ready to setup image builder for ${component.name}") def imageBuilder if (component.imageBuilder == null || component.imageBuilder.isEmpty()) { log.info("Pipeline", "imageBuilder not set for ${component.name}, using kaniko as default image builder") imageBuilder = ImageBuilderTypes.KANIKO } else { imageBuilder = ImageBuilderTypes.parse(component.imageBuilder) if (imageBuilder == null) { log.error("Pipeline", "Unknown image builder for ${component.name}, skipping image building") } } env.imageBuilderImage = imageBuilder.image log.info("Pipeline", "Using ${imageBuilder.builder} (image: ${imageBuilder.image}) as image builder for ${component.name}") } } }, // Image Building & Publishing stage("${component.name} :: Image Building & Publishing") { when { expression { return (env.executeMode == "fully" || env.changedComponents.contains(component.name)) && env.imageBuilderImage != null && !env.imageBuilderImage.isEmpty() } } podTemplate( label: "image-builder-${component.name}", containers: [ containerTemplate( name: 'image-builder', image: env.imageBuilderImage, ttyEnabled: true, command: 'sleep', args: 'infinity' ) ] ) { node("image-builder-${component.name}") { container('image-builder') { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { def sourceFetcher = new SourceFetcher(this) sourceFetcher.fetch(configurations) dir(env.workroot + "/" + component.root) { component.buildArtifacts.each { artifact -> unstash "${component.name}-${artifact}" } if (component.dockerfile != null && !component.dockerfile.isEmpty()) { log.error("Pipeline", "Component ${component.name} dockerfile not set!") } def imageBuilderType = ImageBuilderTypes.parse(component.imageBuilder) if (imageBuilderType == null) { log.error("Pipeline", "Unknown image builder for ${component.name}, skipping image building") } def imageBuilder = new ImageBuilder(this, env.workroot + "/" + component.root + "/", component.imageBuildRoot, component.dockerfilePath, imageBuilderType) log.info("Pipeline", "Retrieve version of image from pervious stage...") if (env.LATEST_VERSION == null || env.LATEST_VERSION.isEmpty()) { log.warn("Pipeline", "LATEST_VERSION environment value not set, using 'snapshot-' as default version") def commitHash = sh(script: 'git rev-parse HEAD', returnStdout: true).trim() def shortCommitHash = commitHash.take(7) env.LATEST_VERSION = "snapshot-${shortCommitHash}" } def version imageBuilder.build(component.imageName, component.imageRepository, component.imageRegistry, component.imageReleaseArchitectures, env.LATEST_VERSION) } } } } } } } ]) return stages } def call(Closure closure) { def configurations = [:] closure.resolveStrategy = Closure.DELEGATE_FIRST closure.delegate = configurations closure() pipeline { agent any options { buildDiscarder(logRotator(numToKeepStr: '25')) timeout(time: 30, unit: 'MINUTES') parallelsAlwaysFailFast() } stages { stage("Pipeline :: Commit Linting If Enabled") { when { expression { return configurations.commitMessageLintEnabled != null && configurations.commitMessageLintEnabled } } agent { kubernetes { defaultContainer 'commit-message-linter' yaml """ apiVersion: v1 kind: Pod metadata: labels: freeleaps-devops-system/milestone: commit-message-linting spec: containers: - name: commit-message-linter image: docker.io/commitlint/commitlint:master command: - cat tty: true volumeMounts: - name: workspace mountPath: /workspace volumes: - name: workspace emptyDir: {} """ } } steps { script { log.info("Pipeline","Commit message linting is enabled") def sourceFetcher = new SourceFetcher(this) sourceFetcher.fetch(configurations) def linter = new CommitMessageLinter(this) linter.lint(configurations) } } } stage("Pipeline :: Execute Mode Detection") { steps { script { def executeMode = configurations.executeMode if (executeMode == null || executeMode.isEmpty()) { log.warn("Pipeline","Not set executeMode, using fully as default execute mode") env.executeMode = "fully" } else if (executeMode == 'on-demand' && configurations.serviceGitRepoType != 'monorepo') { log.warn("Pipeline","serviceGirRepoType is not monorepo, on-demand mode is not supported, using fully mode") env.executeMode = "fully" } else { log.info("Pipeline","Using ${executeMode} as execute mode") env.executeMode = executeMode } } } } stage("Pipeline :: Code Changes Detection") { when { expression { return env.executeMode == "on-demand" } } steps { script { sourceFetcher.fetch(configurations) def changedComponentsDetector = new ChangedComponentsDetector(this) def changedComponents = changedComponentsDetector.detect(env.workroot, configurations.components) log.info("Pipeline","Changed components: ${changedComponents}") env.changedComponents = changedComponents.join(' ') } } } stage("Pipeline :: Components Build (Dynamic Generated Stages)") { when { expression { return env.executeMode == "fully" || env.changedComponents.size() > 0 } } steps { script { configurations.components.each { component -> generateComponentStages(component, configurations) } } } } } } }