From 60817c1be40f9afa48e5704a38e47deaf35dfefe Mon Sep 17 00:00:00 2001 From: zhenyus Date: Sun, 30 Mar 2025 23:37:19 +0800 Subject: [PATCH] feat: add gitea-webhook-ambassador service and migration script Signed-off-by: zhenyus --- .gitignore | 3 +- .../README.md | 0 .../freeleaps-repo-migrator | 0 apps/gitea-webhook-ambassador/.dockerignore | 22 + apps/gitea-webhook-ambassador/Dockerfile | 52 ++ apps/gitea-webhook-ambassador/Makefile | 68 ++ apps/gitea-webhook-ambassador/VERSION | 0 .../configs/configs.example.yaml | 49 ++ apps/gitea-webhook-ambassador/go.mod | 22 + apps/gitea-webhook-ambassador/go.sum | 38 + apps/gitea-webhook-ambassador/main.go | 758 ++++++++++++++++++ .../gitea-webhook-ambassador/configmap.yaml | 39 + .../gitea-webhook-ambassador/deployment.yaml | 100 +++ .../gitea-webhook-ambassador/service.yaml | 16 + freeleaps/helm-pkg/3rd/gitea/values.prod.yaml | 2 + 15 files changed, 1168 insertions(+), 1 deletion(-) rename apps/{20250325 => freeleaps-repo-migrator}/README.md (100%) rename apps/{20250325 => freeleaps-repo-migrator}/freeleaps-repo-migrator (100%) create mode 100644 apps/gitea-webhook-ambassador/.dockerignore create mode 100644 apps/gitea-webhook-ambassador/Dockerfile create mode 100644 apps/gitea-webhook-ambassador/Makefile create mode 100644 apps/gitea-webhook-ambassador/VERSION create mode 100644 apps/gitea-webhook-ambassador/configs/configs.example.yaml create mode 100644 apps/gitea-webhook-ambassador/go.mod create mode 100644 apps/gitea-webhook-ambassador/go.sum create mode 100644 apps/gitea-webhook-ambassador/main.go create mode 100644 cluster/manifests/freeleaps-devops-system/gitea-webhook-ambassador/configmap.yaml create mode 100644 cluster/manifests/freeleaps-devops-system/gitea-webhook-ambassador/deployment.yaml create mode 100644 cluster/manifests/freeleaps-devops-system/gitea-webhook-ambassador/service.yaml diff --git a/.gitignore b/.gitignore index 8b163d51..3ab72eb9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ cluster/ansible/venv cluster/ansible/manifests/inventory.ini .idea/* - +apps/gitea-webhook-ambassador/gitea-webhook-ambassador +apps/gitea-webhook-ambassador/config.yaml \ No newline at end of file diff --git a/apps/20250325/README.md b/apps/freeleaps-repo-migrator/README.md similarity index 100% rename from apps/20250325/README.md rename to apps/freeleaps-repo-migrator/README.md diff --git a/apps/20250325/freeleaps-repo-migrator b/apps/freeleaps-repo-migrator/freeleaps-repo-migrator similarity index 100% rename from apps/20250325/freeleaps-repo-migrator rename to apps/freeleaps-repo-migrator/freeleaps-repo-migrator diff --git a/apps/gitea-webhook-ambassador/.dockerignore b/apps/gitea-webhook-ambassador/.dockerignore new file mode 100644 index 00000000..41e92eb8 --- /dev/null +++ b/apps/gitea-webhook-ambassador/.dockerignore @@ -0,0 +1,22 @@ +# Git +.git +.gitignore + +# Build artifacts +gitea-webhook-ambassador +build/ + +# Development/test files +*_test.go +Makefile +README.md +LICENSE +docker-compose.yml + +# Editor configs +.idea/ +.vscode/ + +# Temporary files +*.log +*.tmp \ No newline at end of file diff --git a/apps/gitea-webhook-ambassador/Dockerfile b/apps/gitea-webhook-ambassador/Dockerfile new file mode 100644 index 00000000..c549ee29 --- /dev/null +++ b/apps/gitea-webhook-ambassador/Dockerfile @@ -0,0 +1,52 @@ +# Build stage +FROM golang:1.24-alpine AS builder + +# Set working directory +WORKDIR /app + +# Install build dependencies +RUN apk add --no-cache git make + +# Copy go.mod and go.sum (if present) +COPY go.mod . +COPY go.sum* . + +# Download dependencies +RUN go mod download + +# Copy source code +COPY . . + +# Build the application with version information +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o gitea-webhook-ambassador . + +# Runtime stage +FROM alpine:3.19 + +# Add non-root user +RUN addgroup -S appgroup && adduser -S appuser -G appgroup + +# Install runtime dependencies +RUN apk add --no-cache ca-certificates tzdata + +# Create necessary directories with appropriate permissions +RUN mkdir -p /app/config && \ + chown -R appuser:appgroup /app + +WORKDIR /app + +# Copy the binary from builder stage +COPY --from=builder /app/gitea-webhook-ambassador . + +# Copy default config (will be overridden by volume mount in production) +COPY config.yaml /app/config/ + +# Switch to non-root user +USER appuser + +# Expose the service port +EXPOSE 8080 + +# Default command (can be overridden at runtime) +ENTRYPOINT ["/app/gitea-webhook-ambassador"] +CMD ["-config=/app/config/config.yaml"] \ No newline at end of file diff --git a/apps/gitea-webhook-ambassador/Makefile b/apps/gitea-webhook-ambassador/Makefile new file mode 100644 index 00000000..48a73407 --- /dev/null +++ b/apps/gitea-webhook-ambassador/Makefile @@ -0,0 +1,68 @@ +.PHONY: build clean test lint docker-build docker-push run help + +# Variables +APP_NAME := gitea-webhook-ambassador +VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev") +LDFLAGS := -ldflags "-X main.version=$(VERSION) -s -w" +GO_FILES := $(shell find . -name "*.go" -type f) +IMAGE_NAME := freeleaps/$(APP_NAME) +IMAGE_TAG := $(VERSION) +CONFIG_FILE := config.yaml + +# Go commands +GO := go +GOFMT := gofmt +GOTEST := $(GO) test +GOBUILD := $(GO) build + +# Default target +.DEFAULT_GOAL := help + +# Build executable +build: $(GO_FILES) + @echo "Building $(APP_NAME)..." + $(GOBUILD) $(LDFLAGS) -o $(APP_NAME) . + +# Clean build artifacts +clean: + @echo "Cleaning up..." + @rm -f $(APP_NAME) + @rm -rf build/ + +# Run tests +test: + @echo "Running tests..." + $(GOTEST) -v ./... + +# Run linter +lint: + @which golangci-lint > /dev/null || (echo "Installing golangci-lint..." && go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest) + golangci-lint run + +# Build Docker image +docker-build: + @echo "Building Docker image $(IMAGE_NAME):$(IMAGE_TAG)..." + docker build -t $(IMAGE_NAME):$(IMAGE_TAG) . + docker tag $(IMAGE_NAME):$(IMAGE_TAG) $(IMAGE_NAME):latest + +# Push Docker image to registry +docker-push: docker-build + @echo "Pushing Docker image $(IMAGE_NAME):$(IMAGE_TAG)..." + docker push $(IMAGE_NAME):$(IMAGE_TAG) + docker push $(IMAGE_NAME):latest + +# Run locally +run: build + ./$(APP_NAME) -config=$(CONFIG_FILE) + +# Show help +help: + @echo "Gitea Webhook Ambassador - Makefile commands:" + @echo " build - Build the application" + @echo " clean - Remove build artifacts" + @echo " test - Run tests" + @echo " lint - Run linter" + @echo " docker-build - Build Docker image" + @echo " docker-push - Build and push Docker image to registry" + @echo " run - Build and run locally" + @echo " help - Show this help message" \ No newline at end of file diff --git a/apps/gitea-webhook-ambassador/VERSION b/apps/gitea-webhook-ambassador/VERSION new file mode 100644 index 00000000..e69de29b diff --git a/apps/gitea-webhook-ambassador/configs/configs.example.yaml b/apps/gitea-webhook-ambassador/configs/configs.example.yaml new file mode 100644 index 00000000..ee90fb58 --- /dev/null +++ b/apps/gitea-webhook-ambassador/configs/configs.example.yaml @@ -0,0 +1,49 @@ +server: + port: 8080 + webhookPath: "/webhook" + secretHeader: "X-Gitea-Signature" + +jenkins: + url: "http://jenkins.example.com" + username: "jenkins-user" + token: "jenkins-api-token" + timeout: 30 + +gitea: + secretToken: "your-gitea-webhook-secret" + projects: + # Simple configuration with different jobs for different branches + "owner/repo1": + defaultJob: "repo1-default-job" # Used when no specific branch match is found + branchJobs: + "main": "repo1-main-job" # Specific job for the main branch + "develop": "repo1-dev-job" # Specific job for the develop branch + "release": "repo1-release-job" # Specific job for the release branch + + # Advanced configuration with regex pattern matching + "owner/repo2": + defaultJob: "repo2-default-job" + branchJobs: + "main": "repo2-main-job" + branchPatterns: + - pattern: "^feature/.*$" # All feature branches + job: "repo2-feature-job" + - pattern: "^release/v[0-9]+\\.[0-9]+$" # Release branches like release/v1.0 + job: "repo2-release-job" + - pattern: "^hotfix/.*$" # All hotfix branches + job: "repo2-hotfix-job" + + # Simple configuration with just a default job + "owner/repo3": + defaultJob: "repo3-job" # This job is triggered for all branches + +logging: + level: "info" + format: "json" + file: "" + +worker: + poolSize: 10 + queueSize: 100 + maxRetries: 3 + retryBackoff: 1 \ No newline at end of file diff --git a/apps/gitea-webhook-ambassador/go.mod b/apps/gitea-webhook-ambassador/go.mod new file mode 100644 index 00000000..9de53e75 --- /dev/null +++ b/apps/gitea-webhook-ambassador/go.mod @@ -0,0 +1,22 @@ +module freeleaps.com/gitea-webhook-ambassador + +go 1.24.0 + +require ( + github.com/fsnotify/fsnotify v1.8.0 + github.com/go-playground/validator/v10 v10.26.0 + github.com/panjf2000/ants/v2 v2.11.2 + gopkg.in/yaml.v2 v2.4.0 +) + +require ( + github.com/gabriel-vasile/mimetype v1.4.8 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + golang.org/x/crypto v0.33.0 // indirect + golang.org/x/net v0.34.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.22.0 // indirect +) diff --git a/apps/gitea-webhook-ambassador/go.sum b/apps/gitea-webhook-ambassador/go.sum new file mode 100644 index 00000000..059117e6 --- /dev/null +++ b/apps/gitea-webhook-ambassador/go.sum @@ -0,0 +1,38 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= +github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= +github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/panjf2000/ants/v2 v2.11.2 h1:AVGpMSePxUNpcLaBO34xuIgM1ZdKOiGnpxLXixLi5Jo= +github.com/panjf2000/ants/v2 v2.11.2/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/apps/gitea-webhook-ambassador/main.go b/apps/gitea-webhook-ambassador/main.go new file mode 100644 index 00000000..131e4b07 --- /dev/null +++ b/apps/gitea-webhook-ambassador/main.go @@ -0,0 +1,758 @@ +package main + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/go-playground/validator/v10" + "github.com/panjf2000/ants/v2" + "gopkg.in/yaml.v2" +) + +// Configuration holds application configuration +type Configuration struct { + Server struct { + Port int `yaml:"port" validate:"required,gt=0"` + WebhookPath string `yaml:"webhookPath" validate:"required"` + SecretHeader string `yaml:"secretHeader" default:"X-Gitea-Signature"` + } `yaml:"server"` + + Jenkins struct { + URL string `yaml:"url" validate:"required,url"` + Username string `yaml:"username"` + Token string `yaml:"token"` + Timeout int `yaml:"timeout" default:"30"` + } `yaml:"jenkins"` + + Gitea struct { + SecretToken string `yaml:"secretToken"` + Projects map[string]ProjectConfig `yaml:"projects" validate:"required"` // repo name -> project config + } `yaml:"gitea"` + + Logging struct { + Level string `yaml:"level" default:"info" validate:"oneof=debug info warn error"` + Format string `yaml:"format" default:"text" validate:"oneof=text json"` + File string `yaml:"file"` + } `yaml:"logging"` + + Worker struct { + PoolSize int `yaml:"poolSize" default:"10" validate:"gt=0"` + QueueSize int `yaml:"queueSize" default:"100" validate:"gt=0"` + MaxRetries int `yaml:"maxRetries" default:"3" validate:"gte=0"` + RetryBackoff int `yaml:"retryBackoff" default:"1" validate:"gt=0"` // seconds + } `yaml:"worker"` +} + +// ProjectConfig represents the configuration for a specific repository +type ProjectConfig struct { + DefaultJob string `yaml:"defaultJob"` // Default Jenkins job to trigger + BranchJobs map[string]string `yaml:"branchJobs,omitempty"` // Branch-specific jobs + BranchPatterns []BranchPattern `yaml:"branchPatterns,omitempty"` +} + +// BranchPattern defines a pattern-based branch to job mapping +type BranchPattern struct { + Pattern string `yaml:"pattern"` // Regex pattern for branch name + Job string `yaml:"job"` // Jenkins job to trigger +} + +// GiteaWebhook represents the webhook payload from Gitea +type GiteaWebhook struct { + Secret string `json:"secret"` + Ref string `json:"ref"` + Before string `json:"before"` + After string `json:"after"` + CompareURL string `json:"compare_url"` + Commits []struct { + ID string `json:"id"` + Message string `json:"message"` + URL string `json:"url"` + Author struct { + Name string `json:"name"` + Email string `json:"email"` + Username string `json:"username"` + } `json:"author"` + } `json:"commits"` + Repository struct { + ID int `json:"id"` + Name string `json:"name"` + Owner struct { + ID int `json:"id"` + Login string `json:"login"` + FullName string `json:"full_name"` + } `json:"owner"` + FullName string `json:"full_name"` + Private bool `json:"private"` + CloneURL string `json:"clone_url"` + SSHURL string `json:"ssh_url"` + HTMLURL string `json:"html_url"` + DefaultBranch string `json:"default_branch"` + } `json:"repository"` + Pusher struct { + ID int `json:"id"` + Login string `json:"login"` + FullName string `json:"full_name"` + Email string `json:"email"` + } `json:"pusher"` +} + +type jobRequest struct { + jobName string + parameters map[string]string + eventID string + attempts int +} + +var ( + configFile = flag.String("config", "config.yaml", "Path to configuration file") + config Configuration + configMutex sync.RWMutex + validate = validator.New() + jobQueue chan jobRequest + httpClient *http.Client + logger *log.Logger + workerPool *ants.PoolWithFunc + + // For idempotency + processedEvents sync.Map + + // For config reloading + watcher *fsnotify.Watcher +) + +func main() { + flag.Parse() + + // Initialize basic logger temporarily + logger = log.New(os.Stdout, "", log.LstdFlags) + logger.Println("Starting Gitea Webhook Ambassador...") + + // Load initial configuration + if err := loadConfig(*configFile); err != nil { + logger.Fatalf("Failed to load configuration: %v", err) + } + + // Configure proper logger based on configuration + setupLogger() + + // Setup config file watcher for auto-reload + setupConfigWatcher(*configFile) + + // Configure HTTP client with timeout + configMutex.RLock() + httpClient = &http.Client{ + Timeout: time.Duration(config.Jenkins.Timeout) * time.Second, + } + + // Initialize job queue + jobQueue = make(chan jobRequest, config.Worker.QueueSize) + configMutex.RUnlock() + + // Initialize worker pool + initWorkerPool() + + // Configure webhook handler + http.HandleFunc(config.Server.WebhookPath, handleWebhook) + http.HandleFunc("/health", handleHealthCheck) + + // Start HTTP server + serverAddr := fmt.Sprintf(":%d", config.Server.Port) + logger.Printf("Server listening on %s", serverAddr) + if err := http.ListenAndServe(serverAddr, nil); err != nil { + logger.Fatalf("HTTP server error: %v", err) + } +} + +// setupLogger configures the logger based on application settings +func setupLogger() { + configMutex.RLock() + defer configMutex.RUnlock() + + // Determine log output + var logOutput io.Writer = os.Stdout + if config.Logging.File != "" { + file, err := os.OpenFile(config.Logging.File, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + logger.Printf("Failed to open log file %s: %v, using stdout instead", config.Logging.File, err) + } else { + logOutput = file + // Create a multiwriter to also log to stdout for important messages + logOutput = io.MultiWriter(file, os.Stdout) + } + } + + // Create new logger with proper format + var prefix string + var flags int + + // Set log format based on configuration + if config.Logging.Format == "json" { + // For JSON logging, we'll handle formatting in the custom writer + prefix = "" + flags = 0 + logOutput = &jsonLogWriter{out: logOutput} + } else { + // Text format with timestamp + prefix = "" + flags = log.LstdFlags | log.Lshortfile + } + + // Create the new logger + logger = log.New(logOutput, prefix, flags) + + // Log level will be checked in our custom log functions (not implemented here) + logger.Printf("Logger configured with level=%s, format=%s, output=%s", + config.Logging.Level, + config.Logging.Format, + func() string { + if config.Logging.File == "" { + return "stdout" + } + return config.Logging.File + }()) +} + +func setupConfigWatcher(configPath string) { + var err error + watcher, err = fsnotify.NewWatcher() + if err != nil { + logger.Fatalf("Failed to create file watcher: %v", err) + } + + // Extract directory containing the config file + configDir := filepath.Dir(configPath) + + go func() { + for { + select { + case event, ok := <-watcher.Events: + if !ok { + return + } + // Check if the config file was modified + if event.Op&fsnotify.Write == fsnotify.Write && + filepath.Base(event.Name) == filepath.Base(configPath) { + logger.Printf("Config file modified, reloading configuration") + if err := reloadConfig(configPath); err != nil { + logger.Printf("Error reloading config: %v", err) + } + } + case err, ok := <-watcher.Errors: + if !ok { + return + } + logger.Printf("Error watching config file: %v", err) + } + } + }() + + // Start watching the directory containing the config file + err = watcher.Add(configDir) + if err != nil { + logger.Fatalf("Failed to watch config directory: %v", err) + } + + logger.Printf("Watching config file for changes: %s", configPath) +} + +func loadConfig(file string) error { + f, err := os.Open(file) + if err != nil { + return fmt.Errorf("cannot open config file: %v", err) + } + defer f.Close() + + var newConfig Configuration + decoder := yaml.NewDecoder(f) + if err := decoder.Decode(&newConfig); err != nil { + return fmt.Errorf("cannot decode config: %v", err) + } + + // Set defaults + if newConfig.Server.SecretHeader == "" { + newConfig.Server.SecretHeader = "X-Gitea-Signature" + } + if newConfig.Jenkins.Timeout == 0 { + newConfig.Jenkins.Timeout = 30 + } + if newConfig.Worker.PoolSize == 0 { + newConfig.Worker.PoolSize = 10 + } + if newConfig.Worker.QueueSize == 0 { + newConfig.Worker.QueueSize = 100 + } + if newConfig.Worker.MaxRetries == 0 { + newConfig.Worker.MaxRetries = 3 + } + if newConfig.Worker.RetryBackoff == 0 { + newConfig.Worker.RetryBackoff = 1 + } + + // Handle legacy configuration format (where Projects is map[string]string) + // This is to maintain backward compatibility with existing configs + if len(newConfig.Gitea.Projects) == 0 { + // Check if we're dealing with a legacy config + var legacyConfig struct { + Gitea struct { + Projects map[string]string `yaml:"projects"` + } `yaml:"gitea"` + } + + // Reopen and reparse the file for legacy config + f.Seek(0, 0) + decoder = yaml.NewDecoder(f) + if err := decoder.Decode(&legacyConfig); err == nil && len(legacyConfig.Gitea.Projects) > 0 { + // Convert legacy config to new format + newConfig.Gitea.Projects = make(map[string]ProjectConfig) + for repo, jobName := range legacyConfig.Gitea.Projects { + newConfig.Gitea.Projects[repo] = ProjectConfig{ + DefaultJob: jobName, + } + } + logWarn("Using legacy configuration format. Consider updating to new format.") + } + } + + // Validate configuration + if err := validate.Struct(newConfig); err != nil { + return fmt.Errorf("invalid configuration: %v", err) + } + + configMutex.Lock() + config = newConfig + configMutex.Unlock() + + return nil +} + +func reloadConfig(file string) error { + if err := loadConfig(file); err != nil { + return err + } + + // Update logger configuration + setupLogger() + + configMutex.RLock() + defer configMutex.RUnlock() + + // Update HTTP client timeout + httpClient.Timeout = time.Duration(config.Jenkins.Timeout) * time.Second + + // If worker pool size has changed, reinitialize worker pool + poolSize := workerPool.Cap() + if poolSize != config.Worker.PoolSize { + logger.Printf("Worker pool size changed from %d to %d, reinitializing", + poolSize, config.Worker.PoolSize) + + // Must release the read lock before calling initWorkerPool which acquires a write lock + configMutex.RUnlock() + initWorkerPool() + configMutex.RLock() + } + + // If queue size has changed, create a new channel and copy items + if cap(jobQueue) != config.Worker.QueueSize { + logger.Printf("Job queue size changed from %d to %d, recreating", + cap(jobQueue), config.Worker.QueueSize) + + // Create new queue + newQueue := make(chan jobRequest, config.Worker.QueueSize) + + // Close the current queue channel to stop accepting new items + close(jobQueue) + + // Start a goroutine to drain the old queue and fill the new one + go func(oldQueue, newQueue chan jobRequest) { + for job := range oldQueue { + newQueue <- job + } + + configMutex.Lock() + jobQueue = newQueue + configMutex.Unlock() + }(jobQueue, newQueue) + } + + logger.Printf("Configuration reloaded successfully") + return nil +} + +func initWorkerPool() { + configMutex.Lock() + defer configMutex.Unlock() + + // Release existing pool if any + if workerPool != nil { + workerPool.Release() + } + + var err error + workerPool, err = ants.NewPoolWithFunc(config.Worker.PoolSize, func(i interface{}) { + job := i.(jobRequest) + success := triggerJenkinsJob(job) + + configMutex.RLock() + maxRetries := config.Worker.MaxRetries + retryBackoff := config.Worker.RetryBackoff + configMutex.RUnlock() + + // If job failed but we haven't reached max retries + if !success && job.attempts < maxRetries { + job.attempts++ + // Exponential backoff + backoff := time.Duration(retryBackoff<= 300 { + bodyBytes, _ := io.ReadAll(resp.Body) + logger.Printf("Jenkins returned error for job %s: status=%d, body=%s", + job.jobName, resp.StatusCode, string(bodyBytes)) + return false + } + + logger.Printf("Successfully triggered Jenkins job %s for event %s", + job.jobName, job.eventID) + return true +} + +// Custom JSON log writer +type jsonLogWriter struct { + out io.Writer +} + +func (w *jsonLogWriter) Write(p []byte) (n int, err error) { + // Parse the log message + message := string(p) + + // Create JSON structure + entry := map[string]interface{}{ + "timestamp": time.Now().Format(time.RFC3339), + "message": strings.TrimSpace(message), + "level": "info", // Default level, in a real implementation you'd parse this + } + + // Convert to JSON + jsonData, err := json.Marshal(entry) + if err != nil { + return 0, err + } + + // Write JSON with newline + return w.out.Write(append(jsonData, '\n')) +} + +// Add these utility functions for level-based logging +func logDebug(format string, v ...interface{}) { + configMutex.RLock() + level := config.Logging.Level + configMutex.RUnlock() + + if level == "debug" { + logger.Printf("[DEBUG] "+format, v...) + } +} + +func logInfo(format string, v ...interface{}) { + configMutex.RLock() + level := config.Logging.Level + configMutex.RUnlock() + + if level == "debug" || level == "info" { + logger.Printf("[INFO] "+format, v...) + } +} + +func logWarn(format string, v ...interface{}) { + configMutex.RLock() + level := config.Logging.Level + configMutex.RUnlock() + + if level == "debug" || level == "info" || level == "warn" { + logger.Printf("[WARN] "+format, v...) + } +} + +func logError(format string, v ...interface{}) { + // Error level logs are always shown + logger.Printf("[ERROR] "+format, v...) +} diff --git a/cluster/manifests/freeleaps-devops-system/gitea-webhook-ambassador/configmap.yaml b/cluster/manifests/freeleaps-devops-system/gitea-webhook-ambassador/configmap.yaml new file mode 100644 index 00000000..89daaf53 --- /dev/null +++ b/cluster/manifests/freeleaps-devops-system/gitea-webhook-ambassador/configmap.yaml @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: gitea-webhook-ambassador-config + namespace: freeleaps-devops-system + labels: + app: gitea-webhook-ambassador +data: + config.yaml: | + server: + port: 8080 + webhookPath: "/webhook" + secretHeader: "X-Gitea-Signature" + + jenkins: + url: "http://jenkins.freeleaps-devops-system.svc.cluster.local:8080" + username: "admin" + token: "115127e693f1bc6b7194f58ff6d6283bd0" + timeout: 30 + + gitea: + secretToken: "b510afe7b60acdb4261df0155117b7a2b5339cc9" + projects: + "freeleaps/freeleaps-service-hub": + defaultJob: "freeleaps/alpha/freeleaps-service-hub" + branchJobs: + "master": "freeleaps/prod/freeleaps-service-hub" + "dev": "freeleaps/alpha/freeleaps-service-hub" + + logging: + level: "info" + format: "json" + file: "" + + worker: + poolSize: 10 + queueSize: 100 + maxRetries: 3 + retryBackoff: 1 \ No newline at end of file diff --git a/cluster/manifests/freeleaps-devops-system/gitea-webhook-ambassador/deployment.yaml b/cluster/manifests/freeleaps-devops-system/gitea-webhook-ambassador/deployment.yaml new file mode 100644 index 00000000..744b0e83 --- /dev/null +++ b/cluster/manifests/freeleaps-devops-system/gitea-webhook-ambassador/deployment.yaml @@ -0,0 +1,100 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gitea-webhook-ambassador + namespace: freeleaps-devops-system + labels: + app: gitea-webhook-ambassador + component: ci-cd +spec: + replicas: 2 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + selector: + matchLabels: + app: gitea-webhook-ambassador + template: + metadata: + labels: + app: gitea-webhook-ambassador + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + prometheus.io/port: "8080" + spec: + containers: + - name: gitea-webhook-ambassador + image: freeleaps/gitea-webhook-ambassador:latest + imagePullPolicy: Always + ports: + - name: http + containerPort: 8080 + protocol: TCP + args: + - "-config=/app/config/config.yaml" + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 256Mi + livenessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: config + mountPath: /app/config + readOnly: true + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + capabilities: + drop: + - ALL + env: + - name: TZ + value: "UTC" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumes: + - name: config + configMap: + name: gitea-webhook-ambassador-config + securityContext: + fsGroup: 1000 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - gitea-webhook-ambassador + topologyKey: kubernetes.io/hostname + terminationGracePeriodSeconds: 30 \ No newline at end of file diff --git a/cluster/manifests/freeleaps-devops-system/gitea-webhook-ambassador/service.yaml b/cluster/manifests/freeleaps-devops-system/gitea-webhook-ambassador/service.yaml new file mode 100644 index 00000000..bb159130 --- /dev/null +++ b/cluster/manifests/freeleaps-devops-system/gitea-webhook-ambassador/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: gitea-webhook-ambassador + namespace: freeleaps-devops-system + labels: + app: gitea-webhook-ambassador +spec: + type: ClusterIP + ports: + - port: 8080 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: gitea-webhook-ambassador \ No newline at end of file diff --git a/freeleaps/helm-pkg/3rd/gitea/values.prod.yaml b/freeleaps/helm-pkg/3rd/gitea/values.prod.yaml index bef47bca..e18727e0 100644 --- a/freeleaps/helm-pkg/3rd/gitea/values.prod.yaml +++ b/freeleaps/helm-pkg/3rd/gitea/values.prod.yaml @@ -545,6 +545,8 @@ gitea: additionalConfigFromEnvs: - name: GITEA__SERVICE__DISABLE_REGISTRATION value: "true" + - name: GITEA__WEBHOOK__ALLOWED_HOST_LIST + value: "gitea-webhook-ambassador.freeleaps-devops-system.svc.freeleaps.cluster" ## @param gitea.podAnnotations Annotations for the Gitea pod podAnnotations: {}