Merged PR 66: merge into master

apps/gitea-webhook-ambassador/configs/configs.example.yaml

@@ -47,4 +47,8 @@ worker:
   poolSize: 10
   queueSize: 100
   maxRetries: 3
-  retryBackoff: 1
+  retryBackoff: 1
+
+eventCleanup:
+  interval: 3600
+  expireAfter: 7200

apps/gitea-webhook-ambassador/main.go

@@ -53,6 +53,11 @@ type Configuration struct {
 		MaxRetries   int `yaml:"maxRetries" default:"3" validate:"gte=0"`
 		RetryBackoff int `yaml:"retryBackoff" default:"1" validate:"gt=0"` // seconds
 	} `yaml:"worker"`
+
+	EventCleanup struct {
+		Interval    int `yaml:"interval" default:"3600"`    // seconds
+		ExpireAfter int `yaml:"expireAfter" default:"7200"` // seconds
+	} `yaml:"eventCleanup"`
 }
 
 // ProjectConfig represents the configuration for a specific repository
@@ -150,6 +155,9 @@ func main() {
 	// Setup config file watcher for auto-reload
 	setupConfigWatcher(*configFile)
 
+	// Start event cleanup goroutine
+	go cleanupEvents()
+
 	// Configure HTTP client with timeout
 	configMutex.RLock()
 	httpClient = &http.Client{
@@ -299,6 +307,12 @@ func loadConfig(file string) error {
 	if newConfig.Worker.RetryBackoff == 0 {
 		newConfig.Worker.RetryBackoff = 1
 	}
+	if newConfig.EventCleanup.Interval == 0 {
+		newConfig.EventCleanup.Interval = 3600
+	}
+	if newConfig.EventCleanup.ExpireAfter == 0 {
+		newConfig.EventCleanup.ExpireAfter = 7200
+	}
 
 	// Handle legacy configuration format (where Projects is map[string]string)
 	// This is to maintain backward compatibility with existing configs
@@ -525,11 +539,7 @@ func handleWebhook(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Store in processed events with a TTL (we'll use a goroutine to remove after 1 hour)
-	processedEvents.Store(eventID, true)
-	go func(key string) {
-		time.Sleep(1 * time.Hour)
-		processedEvents.Delete(key)
-	}(eventID)
+	processedEvents.Store(eventID, time.Now())
 
 	// Check if we have a Jenkins job mapping for this repository
 	configMutex.RLock()
@@ -742,3 +752,25 @@ func logError(format string, v ...interface{}) {
 	// Error level logs are always shown
 	logger.Printf("[ERROR] "+format, v...)
 }
+
+func cleanupEvents() {
+	for {
+		configMutex.RLock()
+		interval := time.Duration(config.EventCleanup.Interval) * time.Second
+		expireAfter := time.Duration(config.EventCleanup.ExpireAfter) * time.Second
+		configMutex.RUnlock()
+
+		time.Sleep(interval)
+
+		now := time.Now()
+		processedEvents.Range(func(key, value interface{}) bool {
+			if timestamp, ok := value.(time.Time); ok {
+				if now.Sub(timestamp) > expireAfter {
+					processedEvents.Delete(key)
+					logDebug("Cleaned up expired event: %v", key)
+				}
+			}
+			return true
+		})
+	}
+}

cluster/manifests/freeleaps-controls-system/rbac/rbac.yaml

@@ -0,0 +1,9 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: certificate-contributor
+  namespace: freeleaps-controls-system
+rules:
+  - apiGroups: ["cert-manager.io"]
+    resources: ["certificates"]
+    verbs: ["*"]

cluster/manifests/freeleaps-data-platform/doris/deploy.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# 创建命名空间
+kubectl create namespace freeleaps-data-platform
+
+# 安装Doris Operator CRD
+kubectl create -f https://raw.githubusercontent.com/apache/doris-operator/master/config/crd/bases/doris.apache.com_dorisclusters.yaml
+
+# 部署 Doris Operator
+kubectl apply -f https://raw.githubusercontent.com/apache/doris-operator/master/config/operator/operator.yaml
+
+# 创建 Doris 集群配置
+kubectl apply -f doris-cluster.yaml -n freeleaps-data-platform
+
+# 检查部署状态
+kubectl get pods -n freeleaps-data-platform -l app=doris

cluster/manifests/freeleaps-data-platform/doris/doris-cluster.yaml

@@ -0,0 +1,57 @@
+apiVersion: doris.apache.com/v1
+kind: DorisCluster
+metadata:
+  name: doris-cluster
+  namespace: freeleaps-data-platform
+spec:
+  clusterDomain: "freeleaps.cluster"
+
+  feSpec:
+    replicas: 1
+    image: apache/doris:2.0.2
+    resources:
+      requests:
+        cpu: "1"
+        memory: "2Gi"
+      limits:
+        cpu: "2"
+        memory: "4Gi"
+    service:
+      type: ClusterIP
+    configMap:
+      fe.conf: |
+        JAVA_OPTS="-Xmx2048m -XX:+UseG1GC"
+
+  beSpec:
+    replicas: 1
+    image: apache/doris:2.0.2
+    storage:
+      storageSize: "50Gi"
+      storageClassName: "azure-disk-std-ssd-lrs"
+    resources:
+      requests:
+        cpu: "2"
+        memory: "4Gi"
+      limits:
+        cpu: "4"
+        memory: "8Gi"
+    storage:
+      storageSize: "50Gi"
+      storageClassName: "standard"
+    configMap:
+      be.conf: |
+        JAVA_OPTS="-Xmx8192m -XX:+UseG1GC"
+        BE_ADDR=${POD_IP}:9060
+        BE_HTTP_PORT=8040
+        BE_PORT=9060
+        HEARTBEAT_SERVICE_PORT=9050
+        BRPC_PORT=8060
+
+  feAddress: doris-cluster-fe-service
+
+  monitoring:
+    enabled: true
+    prometheus:
+      serviceMonitor:
+        enabled: true
+        namespace: freeleaps-monitoring-system

cluster/manifests/freeleaps-data-platform/kafka/kafka.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+# 查看kafka的pod
+kubectl get pods -n freeleaps-data-platform
+# 查看kafka的服务
+kubectl get svc -n freeleaps-data-platform
+

cluster/manifests/freeleaps-data-platform/kafka/values.yaml

@@ -0,0 +1,75 @@
+global:
+  storageClass: "standard"
+
+nameOverride: "kafka"
+fullnameOverride: "kafka"
+namespaceOverride: "freeleaps-data-platform"
+
+kafka:
+  replicaCount: 1
+  heapOpts: "-Xmx1024m -Xms1024m"
+  
+  resources:
+    requests:
+      memory: "1Gi"
+      cpu: "2"
+    limits:
+      memory: "2Gi"
+      cpu: "1"
+
+  persistence:
+    enabled: true
+    size: 20Gi
+    mountPath: /bitnami/kafka
+
+  config:
+    num.partitions: 1
+    default.replication.factor: 1
+    min.insync.replicas: 1
+    auto.create.topics.enable: true
+    delete.topic.enable: true
+    log.retention.hours: 168
+    log.retention.bytes: 1073741824
+    
+  service:
+    type: ClusterIP
+    ports:
+      client: 9092
+      internal: 9093
+
+  metrics:
+    kafka:
+      enabled: true
+      serviceMonitor:
+        enabled: true
+        namespace: freeleaps-data-platform
+    jmx:
+      enabled: true
+
+zookeeper:
+  enabled: true
+  replicaCount: 1
+  
+  resources:
+    requests:
+      memory: "1Gi"
+      cpu: "1"
+    limits:
+      memory: "2Gi"
+      cpu: "1"
+
+  persistence:
+    enabled: true
+    size: 8Gi
+
+  service:
+    type: ClusterIP
+    port: 2181
+
+serviceAccount:
+  create: true
+  name: "kafka"
+
+networkPolicy:
+  enabled: true
+  allowExternal: true

cluster/manifests/freeleaps-data-platform/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: freeleaps-data-platform
+  labels:
+    name: freeleaps-data-platform

cluster/manifests/freeleaps-data-platform/pinot/certificate.yaml

@@ -0,0 +1,13 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: pinot-dot-mathmast-dot-com
+  namespace: freeleaps-data-platform
+spec:
+  commonName: pinot.mathmast.com
+  dnsNames:
+    - pinot.mathmast.com
+  issuerRef:
+    kind: ClusterIssuer
+    name: mathmast-dot-com
+  secretName: pinot-dot-mathmast-dot-com-tls

cluster/manifests/freeleaps-data-platform/pinot/values.yaml

@@ -0,0 +1,42 @@
+cluster:
+  name: pinot-cluster
+
+namespaceOverride: "freeleaps-data-platform"
+
+controller:
+  replicaCount: 1
+  persistence:
+    enabled: true
+    size: 20Gi
+  ingress:
+    enabled: true
+    annotations:
+      kubernetes.io/ingress.class: nginx
+      nginx.ingress.kubernetes.io/rewrite-target: /$2
+      cluster-issuer: mathmast-dot-com
+    hosts:
+      - host: pinot.freeleaps.com
+        paths:
+          - path: /pinot(/|$)(.*)
+            pathType: Prefix
+            port: 9000
+    tls:
+      - secretName: pinot-dot-mathmast-dot-com-tls
+        hosts:
+          - pinot.freeleaps.com
+
+broker:
+  replicaCount: 1
+
+server:
+  replicaCount: 1
+  persistence:
+    enabled: true
+    size: 50Gi
+
+zookeeper:
+  enabled: false
+  external:
+    enabled: true
+    host: "kafka-zookeeper"
+    port: 2181

cluster/manifests/freeleaps-data-platform/rbac/rbac.yaml

@@ -0,0 +1,9 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: data-platform-contributor
+  namespace: freeleaps-data-platform
+rules:
+  - apiGroups: ["*"]
+    resources: ["*"]
+    verbs: ["*"]

cluster/manifests/freeleaps-data-platform/star-rocks/values.yaml

@@ -0,0 +1,32 @@
+starrocks:
+    initPassword:
+        enabled: true
+        # 设置密码 secret，例如：
+        # kubectl create secret generic starrocks-root-pass --from-literal=password='g()()dpa$$word'
+        passwordSecret: starrocks-root-pass
+
+    starrocksFESpec:
+        replicas: 3
+        service:
+            type: LoadBalancer
+        resources:
+            requests:
+                cpu: 1
+                memory: 1Gi
+        storageSpec:
+            name: fe
+
+    starrocksBeSpec:
+        replicas: 3
+        resources:
+            requests:
+                cpu: 1
+                memory: 2Gi
+        storageSpec:
+            name: be
+            storageSize: 15Gi
+
+    starrocksFeProxySpec:
+        enabled: true
+        service:
+            type: LoadBalancer

cluster/manifests/freeleaps-monitoring-system/rbac/rbac.yaml

@@ -0,0 +1,19 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: prometheus-rules-contributor
+  namespace: freeleaps-monitoring-system
+rules:
+  - apiGroups: ["monitoring.coreos.com"]
+    resources: ["prometheusrules"]
+    verbs: ["*"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: service-monitor-contributor
+  namespace: freeleaps-monitoring-system
+rules:
+  - apiGroups: ["monitoring.coreos.com"]
+    resources: ["servicemonitors"]
+    verbs: ["*"]

cluster/manifests/helm-repos/REPO.list

@@ -12,3 +12,5 @@ descheduler,https://kubernetes-sigs.github.io/descheduler/,force-update
 kubernetes-dashboard,https://kubernetes.github.io/dashboard/,force-update
 grafana,https://grafana.github.io/helm-charts,force-update
 fluent,https://fluent.github.io/helm-charts,force-update
+pinot,https://raw.githubusercontent.com/apache/pinot/master/helm,force-update
+starrocks,https://starrocks.github.io/starrocks-kubernetes-operator,force-update