From b2b1fd274fcf5b28f970b528671c6f814bbb062a Mon Sep 17 00:00:00 2001 From: Joe <1029317664@qq.com> Date: Fri, 9 May 2025 10:07:00 +0800 Subject: [PATCH 1/7] feat: add freeleaps-data-platform --- .../pinot/kafka-monitoring-table.yaml | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 cluster/manifests/freeleaps-data-platform/pinot/kafka-monitoring-table.yaml diff --git a/cluster/manifests/freeleaps-data-platform/pinot/kafka-monitoring-table.yaml b/cluster/manifests/freeleaps-data-platform/pinot/kafka-monitoring-table.yaml new file mode 100644 index 00000000..605937b1 --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/pinot/kafka-monitoring-table.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kafka-monitoring-schema + namespace: freeleaps-data-platform +data: + schema.json: | + { + "schemaName": "kafka_metrics", + "dimensionFieldSpecs": [ + { + "name": "topic", + "dataType": "STRING" + }, + { + "name": "partition", + "dataType": "INT" + } + ], + "metricFieldSpecs": [ + { + "name": "messages_in_per_sec", + "dataType": "DOUBLE" + }, + { + "name": "bytes_in_per_sec", + "dataType": "DOUBLE" + } + ], + "timeFieldSpec": { + "incomingGranularitySpec": { + "timeType": "MILLISECONDS", + "timeFormat": "EPOCH", + "name": "timestamp", + "dataType": "LONG" + } + } + } + table.json: | + { + "tableName": "business_metrics", + "tableType": "REALTIME", + "segmentsConfig": { + "timeColumnName": "timestamp", + "timeType": "MILLISECONDS", + "retentionTimeUnit": "DAYS", + "retentionTimeValue": "90" + }, + "streamConfigs": { + "streamType": "kafka", + "stream.kafka.consumer.type": "lowlevel", + "stream.kafka.topic.name": "business-metrics", + "stream.kafka.broker.list": "kafka-headless.freeleaps-data-platform.svc.cluster.local:9092", + "stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory", + "stream.kafka.decoder.class.name": "org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder" + }, + "indexingConfig": { + "loadMode": "MMAP", + "aggregateMetrics": true + } + } + \ No newline at end of file From a7025081a1d55ccb3027fe595c363124a1f0e606 Mon Sep 17 00:00:00 2001 From: Joe <1029317664@qq.com> Date: Fri, 9 May 2025 10:14:16 +0800 Subject: [PATCH 2/7] feat: add freeleaps-data-platform --- .../freeleaps-data-platform/doris/deploy.sh | 16 ++++ .../doris/doris-cluster.yaml | 52 +++++++++++++ .../doris/storage.yaml | 8 ++ .../freeleaps-data-platform/kafka/kafka.sh | 6 ++ .../freeleaps-data-platform/kafka/values.yaml | 75 +++++++++++++++++++ .../freeleaps-data-platform/namespace.yaml | 6 ++ .../freeleaps-data-platform/pinot/values.yaml | 26 +++++++ cluster/manifests/helm-repos/REPO.list | 1 + 8 files changed, 190 insertions(+) create mode 100644 cluster/manifests/freeleaps-data-platform/doris/deploy.sh create mode 100644 cluster/manifests/freeleaps-data-platform/doris/doris-cluster.yaml create mode 100644 cluster/manifests/freeleaps-data-platform/doris/storage.yaml create mode 100644 cluster/manifests/freeleaps-data-platform/kafka/kafka.sh create mode 100644 cluster/manifests/freeleaps-data-platform/kafka/values.yaml create mode 100644 cluster/manifests/freeleaps-data-platform/namespace.yaml create mode 100644 cluster/manifests/freeleaps-data-platform/pinot/values.yaml diff --git a/cluster/manifests/freeleaps-data-platform/doris/deploy.sh b/cluster/manifests/freeleaps-data-platform/doris/deploy.sh new file mode 100644 index 00000000..06ce5406 --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/doris/deploy.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# 创建命名空间 +kubectl create namespace freeleaps-data-platform + +# 安装Doris Operator CRD +kubectl create -f https://raw.githubusercontent.com/apache/doris-operator/master/config/crd/bases/doris.apache.com_dorisclusters.yaml + +# 部署 Doris Operator +kubectl apply -f https://raw.githubusercontent.com/apache/doris-operator/master/config/operator/operator.yaml + +# 创建 Doris 集群配置 +kubectl apply -f doris-cluster.yaml -n freeleaps-data-platform + +# 检查部署状态 +kubectl get pods -n freeleaps-data-platform -l app=doris \ No newline at end of file diff --git a/cluster/manifests/freeleaps-data-platform/doris/doris-cluster.yaml b/cluster/manifests/freeleaps-data-platform/doris/doris-cluster.yaml new file mode 100644 index 00000000..bb326343 --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/doris/doris-cluster.yaml @@ -0,0 +1,52 @@ +apiVersion: doris.apache.com/v1 +kind: DorisCluster +metadata: + name: doris-cluster + namespace: freeleaps-data-platform +spec: + feSpec: + replicas: 1 + image: apache/doris:2.0.2 + resources: + requests: + cpu: "1" + memory: "2Gi" + limits: + cpu: "2" + memory: "4Gi" + service: + type: ClusterIP + configMap: + fe.conf: | + JAVA_OPTS="-Xmx2048m -XX:+UseG1GC" + + beSpec: + replicas: 1 + image: apache/doris:2.0.2 + resources: + requests: + cpu: "2" + memory: "4Gi" + limits: + cpu: "4" + memory: "8Gi" + storage: + storageSize: "50Gi" + storageClassName: "standard" + configMap: + be.conf: | + JAVA_OPTS="-Xmx8192m -XX:+UseG1GC" + BE_ADDR=${POD_IP}:9060 + BE_HTTP_PORT=8040 + BE_PORT=9060 + HEARTBEAT_SERVICE_PORT=9050 + BRPC_PORT=8060 + + feAddress: doris-cluster-fe-service + + monitoring: + enabled: true + prometheus: + serviceMonitor: + enabled: true + namespace: freeleaps-data-platform \ No newline at end of file diff --git a/cluster/manifests/freeleaps-data-platform/doris/storage.yaml b/cluster/manifests/freeleaps-data-platform/doris/storage.yaml new file mode 100644 index 00000000..f7012a0c --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/doris/storage.yaml @@ -0,0 +1,8 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: doris-storage +provisioner: kubernetes.io/azure-disk +parameters: + storageaccounttype: Standard_LRS + kind: Managed \ No newline at end of file diff --git a/cluster/manifests/freeleaps-data-platform/kafka/kafka.sh b/cluster/manifests/freeleaps-data-platform/kafka/kafka.sh new file mode 100644 index 00000000..924fdfee --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/kafka/kafka.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# 查看kafka的pod +kubectl get pods -n freeleaps-data-platform +# 查看kafka的服务 +kubectl get svc -n freeleaps-data-platform + diff --git a/cluster/manifests/freeleaps-data-platform/kafka/values.yaml b/cluster/manifests/freeleaps-data-platform/kafka/values.yaml new file mode 100644 index 00000000..77dde670 --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/kafka/values.yaml @@ -0,0 +1,75 @@ +global: + storageClass: "standard" + +nameOverride: "kafka" +fullnameOverride: "kafka" +namespaceOverride: "freeleaps-data-platform" + +kafka: + replicaCount: 1 + heapOpts: "-Xmx1024m -Xms1024m" + + resources: + requests: + memory: "1Gi" + cpu: "2" + limits: + memory: "2Gi" + cpu: "1" + + persistence: + enabled: true + size: 20Gi + mountPath: /bitnami/kafka + + config: + num.partitions: 1 + default.replication.factor: 1 + min.insync.replicas: 1 + auto.create.topics.enable: true + delete.topic.enable: true + log.retention.hours: 168 + log.retention.bytes: 1073741824 + + service: + type: ClusterIP + ports: + client: 9092 + internal: 9093 + + metrics: + kafka: + enabled: true + serviceMonitor: + enabled: true + namespace: freeleaps-data-platform + jmx: + enabled: true + +zookeeper: + enabled: true + replicaCount: 1 + + resources: + requests: + memory: "1Gi" + cpu: "1" + limits: + memory: "2Gi" + cpu: "1" + + persistence: + enabled: true + size: 8Gi + + service: + type: ClusterIP + port: 2181 + +serviceAccount: + create: true + name: "kafka" + +networkPolicy: + enabled: true + allowExternal: true \ No newline at end of file diff --git a/cluster/manifests/freeleaps-data-platform/namespace.yaml b/cluster/manifests/freeleaps-data-platform/namespace.yaml new file mode 100644 index 00000000..4329c2fe --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: freeleaps-devops-system + labels: + name: freeleaps-devops-system diff --git a/cluster/manifests/freeleaps-data-platform/pinot/values.yaml b/cluster/manifests/freeleaps-data-platform/pinot/values.yaml new file mode 100644 index 00000000..371da4de --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/pinot/values.yaml @@ -0,0 +1,26 @@ +cluster: + name: pinot-cluster + +namespaceOverride: "freeleaps-data-platform" + +controller: + replicaCount: 1 + persistence: + enabled: true + size: 20Gi + +broker: + replicaCount: 1 + +server: + replicaCount: 1 + persistence: + enabled: true + size: 50Gi + +zookeeper: + enabled: false + external: + enabled: true + host: "kafka-zookeeper" + port: 2181 \ No newline at end of file diff --git a/cluster/manifests/helm-repos/REPO.list b/cluster/manifests/helm-repos/REPO.list index dcea5527..99d8c1f3 100644 --- a/cluster/manifests/helm-repos/REPO.list +++ b/cluster/manifests/helm-repos/REPO.list @@ -12,3 +12,4 @@ descheduler,https://kubernetes-sigs.github.io/descheduler/,force-update kubernetes-dashboard,https://kubernetes.github.io/dashboard/,force-update grafana,https://grafana.github.io/helm-charts,force-update fluent,https://fluent.github.io/helm-charts,force-update +pinot,https://raw.githubusercontent.com/apache/pinot/master/helm,force-update \ No newline at end of file From 51cbfbef07d59959b78655ac5e68fc3f6b9b4668 Mon Sep 17 00:00:00 2001 From: Zheng Yang Date: Fri, 9 May 2025 02:16:08 +0000 Subject: [PATCH 3/7] Deleted kafka-monitoring-table.yaml --- .../pinot/kafka-monitoring-table.yaml | 62 ------------------- 1 file changed, 62 deletions(-) delete mode 100644 cluster/manifests/freeleaps-data-platform/pinot/kafka-monitoring-table.yaml diff --git a/cluster/manifests/freeleaps-data-platform/pinot/kafka-monitoring-table.yaml b/cluster/manifests/freeleaps-data-platform/pinot/kafka-monitoring-table.yaml deleted file mode 100644 index 605937b1..00000000 --- a/cluster/manifests/freeleaps-data-platform/pinot/kafka-monitoring-table.yaml +++ /dev/null @@ -1,62 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: kafka-monitoring-schema - namespace: freeleaps-data-platform -data: - schema.json: | - { - "schemaName": "kafka_metrics", - "dimensionFieldSpecs": [ - { - "name": "topic", - "dataType": "STRING" - }, - { - "name": "partition", - "dataType": "INT" - } - ], - "metricFieldSpecs": [ - { - "name": "messages_in_per_sec", - "dataType": "DOUBLE" - }, - { - "name": "bytes_in_per_sec", - "dataType": "DOUBLE" - } - ], - "timeFieldSpec": { - "incomingGranularitySpec": { - "timeType": "MILLISECONDS", - "timeFormat": "EPOCH", - "name": "timestamp", - "dataType": "LONG" - } - } - } - table.json: | - { - "tableName": "business_metrics", - "tableType": "REALTIME", - "segmentsConfig": { - "timeColumnName": "timestamp", - "timeType": "MILLISECONDS", - "retentionTimeUnit": "DAYS", - "retentionTimeValue": "90" - }, - "streamConfigs": { - "streamType": "kafka", - "stream.kafka.consumer.type": "lowlevel", - "stream.kafka.topic.name": "business-metrics", - "stream.kafka.broker.list": "kafka-headless.freeleaps-data-platform.svc.cluster.local:9092", - "stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory", - "stream.kafka.decoder.class.name": "org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder" - }, - "indexingConfig": { - "loadMode": "MMAP", - "aggregateMetrics": true - } - } - \ No newline at end of file From 2f7128a51c53f9641bffa48c0a95c407f2033957 Mon Sep 17 00:00:00 2001 From: zhenyus Date: Fri, 9 May 2025 13:10:13 +0800 Subject: [PATCH 4/7] feat: update namespaces and add RBAC roles for freeleaps data platform and monitoring systems Signed-off-by: zhenyus --- .../freeleaps-controls-system/rbac/rbac.yaml | 9 +++++++++ .../freeleaps-data-platform/namespace.yaml | 4 ++-- .../freeleaps-data-platform/rbac/rbac.yaml | 9 +++++++++ .../rbac/rbac.yaml | 19 +++++++++++++++++++ 4 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 cluster/manifests/freeleaps-controls-system/rbac/rbac.yaml create mode 100644 cluster/manifests/freeleaps-data-platform/rbac/rbac.yaml create mode 100644 cluster/manifests/freeleaps-monitoring-system/rbac/rbac.yaml diff --git a/cluster/manifests/freeleaps-controls-system/rbac/rbac.yaml b/cluster/manifests/freeleaps-controls-system/rbac/rbac.yaml new file mode 100644 index 00000000..90e42dd1 --- /dev/null +++ b/cluster/manifests/freeleaps-controls-system/rbac/rbac.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: certificate-contributor + namespace: freeleaps-controls-system +rules: + - apiGroups: ["cert-manager.io"] + resources: ["certificates"] + verbs: ["*"] \ No newline at end of file diff --git a/cluster/manifests/freeleaps-data-platform/namespace.yaml b/cluster/manifests/freeleaps-data-platform/namespace.yaml index 4329c2fe..e7df1b51 100644 --- a/cluster/manifests/freeleaps-data-platform/namespace.yaml +++ b/cluster/manifests/freeleaps-data-platform/namespace.yaml @@ -1,6 +1,6 @@ apiVersion: v1 kind: Namespace metadata: - name: freeleaps-devops-system + name: freeleaps-data-platform labels: - name: freeleaps-devops-system + name: freeleaps-data-platform diff --git a/cluster/manifests/freeleaps-data-platform/rbac/rbac.yaml b/cluster/manifests/freeleaps-data-platform/rbac/rbac.yaml new file mode 100644 index 00000000..eb98fe38 --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/rbac/rbac.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: data-platform-contributor + namespace: freeleaps-data-platform +rules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] \ No newline at end of file diff --git a/cluster/manifests/freeleaps-monitoring-system/rbac/rbac.yaml b/cluster/manifests/freeleaps-monitoring-system/rbac/rbac.yaml new file mode 100644 index 00000000..24039d4d --- /dev/null +++ b/cluster/manifests/freeleaps-monitoring-system/rbac/rbac.yaml @@ -0,0 +1,19 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: prometheus-rules-contributor + namespace: freeleaps-monitoring-system +rules: + - apiGroups: ["monitoring.coreos.com"] + resources: ["prometheusrules"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: service-monitor-contributor + namespace: freeleaps-monitoring-system +rules: + - apiGroups: ["monitoring.coreos.com"] + resources: ["servicemonitors"] + verbs: ["*"] \ No newline at end of file From f4df870452303c1e007aeb68ee3c3f54441adc8b Mon Sep 17 00:00:00 2001 From: Joe <1029317664@qq.com> Date: Fri, 9 May 2025 17:52:53 +0800 Subject: [PATCH 5/7] fix: update freeleaps-data-platform --- .../doris/doris-cluster.yaml | 7 +++- .../doris/storage.yaml | 8 ----- .../pinot/certificate.yaml | 13 ++++++++ .../freeleaps-data-platform/pinot/values.yaml | 16 ++++++++++ .../star-rocks/values.yaml | 32 +++++++++++++++++++ cluster/manifests/helm-repos/REPO.list | 3 +- 6 files changed, 69 insertions(+), 10 deletions(-) delete mode 100644 cluster/manifests/freeleaps-data-platform/doris/storage.yaml create mode 100644 cluster/manifests/freeleaps-data-platform/pinot/certificate.yaml create mode 100644 cluster/manifests/freeleaps-data-platform/star-rocks/values.yaml diff --git a/cluster/manifests/freeleaps-data-platform/doris/doris-cluster.yaml b/cluster/manifests/freeleaps-data-platform/doris/doris-cluster.yaml index bb326343..9ee2bb52 100644 --- a/cluster/manifests/freeleaps-data-platform/doris/doris-cluster.yaml +++ b/cluster/manifests/freeleaps-data-platform/doris/doris-cluster.yaml @@ -4,6 +4,8 @@ metadata: name: doris-cluster namespace: freeleaps-data-platform spec: + clusterDomain: "freeleaps.cluster" + feSpec: replicas: 1 image: apache/doris:2.0.2 @@ -23,6 +25,9 @@ spec: beSpec: replicas: 1 image: apache/doris:2.0.2 + storage: + storageSize: "50Gi" + storageClassName: "azure-disk-std-ssd-lrs" resources: requests: cpu: "2" @@ -49,4 +54,4 @@ spec: prometheus: serviceMonitor: enabled: true - namespace: freeleaps-data-platform \ No newline at end of file + namespace: freeleaps-monitoring-system \ No newline at end of file diff --git a/cluster/manifests/freeleaps-data-platform/doris/storage.yaml b/cluster/manifests/freeleaps-data-platform/doris/storage.yaml deleted file mode 100644 index f7012a0c..00000000 --- a/cluster/manifests/freeleaps-data-platform/doris/storage.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: doris-storage -provisioner: kubernetes.io/azure-disk -parameters: - storageaccounttype: Standard_LRS - kind: Managed \ No newline at end of file diff --git a/cluster/manifests/freeleaps-data-platform/pinot/certificate.yaml b/cluster/manifests/freeleaps-data-platform/pinot/certificate.yaml new file mode 100644 index 00000000..0942c51c --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/pinot/certificate.yaml @@ -0,0 +1,13 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: pinot-dot-mathmast-dot-com + namespace: freeleaps-data-platform +spec: + commonName: pinot.mathmast.com + dnsNames: + - pinot.mathmast.com + issuerRef: + kind: ClusterIssuer + name: mathmast-dot-com + secretName: pinot-dot-mathmast-dot-com-tls \ No newline at end of file diff --git a/cluster/manifests/freeleaps-data-platform/pinot/values.yaml b/cluster/manifests/freeleaps-data-platform/pinot/values.yaml index 371da4de..e48c0151 100644 --- a/cluster/manifests/freeleaps-data-platform/pinot/values.yaml +++ b/cluster/manifests/freeleaps-data-platform/pinot/values.yaml @@ -8,6 +8,22 @@ controller: persistence: enabled: true size: 20Gi + ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/rewrite-target: /$2 + cluster-issuer: mathmast-dot-com + hosts: + - host: pinot.freeleaps.com + paths: + - path: /pinot(/|$)(.*) + pathType: Prefix + port: 9000 + tls: + - secretName: pinot-dot-mathmast-dot-com-tls + hosts: + - pinot.freeleaps.com broker: replicaCount: 1 diff --git a/cluster/manifests/freeleaps-data-platform/star-rocks/values.yaml b/cluster/manifests/freeleaps-data-platform/star-rocks/values.yaml new file mode 100644 index 00000000..b46fef1c --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/star-rocks/values.yaml @@ -0,0 +1,32 @@ +starrocks: + initPassword: + enabled: true + # 设置密码 secret，例如： + # kubectl create secret generic starrocks-root-pass --from-literal=password='g()()dpa$$word' + passwordSecret: starrocks-root-pass + + starrocksFESpec: + replicas: 3 + service: + type: LoadBalancer + resources: + requests: + cpu: 1 + memory: 1Gi + storageSpec: + name: fe + + starrocksBeSpec: + replicas: 3 + resources: + requests: + cpu: 1 + memory: 2Gi + storageSpec: + name: be + storageSize: 15Gi + + starrocksFeProxySpec: + enabled: true + service: + type: LoadBalancer \ No newline at end of file diff --git a/cluster/manifests/helm-repos/REPO.list b/cluster/manifests/helm-repos/REPO.list index 99d8c1f3..04ad06c2 100644 --- a/cluster/manifests/helm-repos/REPO.list +++ b/cluster/manifests/helm-repos/REPO.list @@ -12,4 +12,5 @@ descheduler,https://kubernetes-sigs.github.io/descheduler/,force-update kubernetes-dashboard,https://kubernetes.github.io/dashboard/,force-update grafana,https://grafana.github.io/helm-charts,force-update fluent,https://fluent.github.io/helm-charts,force-update -pinot,https://raw.githubusercontent.com/apache/pinot/master/helm,force-update \ No newline at end of file +pinot,https://raw.githubusercontent.com/apache/pinot/master/helm,force-update +starrocks,https://starrocks.github.io/starrocks-kubernetes-operator \ No newline at end of file From 134caeaeb2d77f13645beb892f989ea4b3b35a70 Mon Sep 17 00:00:00 2001 From: Joe <1029317664@qq.com> Date: Sat, 10 May 2025 15:48:29 +0800 Subject: [PATCH 6/7] feat: add starrocks to repo list --- cluster/manifests/helm-repos/REPO.list | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/manifests/helm-repos/REPO.list b/cluster/manifests/helm-repos/REPO.list index 04ad06c2..9806fb42 100644 --- a/cluster/manifests/helm-repos/REPO.list +++ b/cluster/manifests/helm-repos/REPO.list @@ -13,4 +13,4 @@ kubernetes-dashboard,https://kubernetes.github.io/dashboard/,force-update grafana,https://grafana.github.io/helm-charts,force-update fluent,https://fluent.github.io/helm-charts,force-update pinot,https://raw.githubusercontent.com/apache/pinot/master/helm,force-update -starrocks,https://starrocks.github.io/starrocks-kubernetes-operator \ No newline at end of file +starrocks,https://starrocks.github.io/starrocks-kubernetes-operator,force-update \ No newline at end of file From 70c4951bdb441f8c7ae1c1927f505fc043b80db3 Mon Sep 17 00:00:00 2001 From: zhenyus Date: Sun, 11 May 2025 22:33:47 +0800 Subject: [PATCH 7/7] feat: add event cleanup configuration and functionality Signed-off-by: zhenyus --- .../configs/configs.example.yaml | 6 ++- apps/gitea-webhook-ambassador/main.go | 42 ++++++++++++++++--- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/apps/gitea-webhook-ambassador/configs/configs.example.yaml b/apps/gitea-webhook-ambassador/configs/configs.example.yaml index 55ff13e6..734cacc8 100644 --- a/apps/gitea-webhook-ambassador/configs/configs.example.yaml +++ b/apps/gitea-webhook-ambassador/configs/configs.example.yaml @@ -47,4 +47,8 @@ worker: poolSize: 10 queueSize: 100 maxRetries: 3 - retryBackoff: 1 \ No newline at end of file + retryBackoff: 1 + +eventCleanup: + interval: 3600 + expireAfter: 7200 \ No newline at end of file diff --git a/apps/gitea-webhook-ambassador/main.go b/apps/gitea-webhook-ambassador/main.go index 7124add9..1c14578c 100644 --- a/apps/gitea-webhook-ambassador/main.go +++ b/apps/gitea-webhook-ambassador/main.go @@ -53,6 +53,11 @@ type Configuration struct { MaxRetries int `yaml:"maxRetries" default:"3" validate:"gte=0"` RetryBackoff int `yaml:"retryBackoff" default:"1" validate:"gt=0"` // seconds } `yaml:"worker"` + + EventCleanup struct { + Interval int `yaml:"interval" default:"3600"` // seconds + ExpireAfter int `yaml:"expireAfter" default:"7200"` // seconds + } `yaml:"eventCleanup"` } // ProjectConfig represents the configuration for a specific repository @@ -150,6 +155,9 @@ func main() { // Setup config file watcher for auto-reload setupConfigWatcher(*configFile) + // Start event cleanup goroutine + go cleanupEvents() + // Configure HTTP client with timeout configMutex.RLock() httpClient = &http.Client{ @@ -299,6 +307,12 @@ func loadConfig(file string) error { if newConfig.Worker.RetryBackoff == 0 { newConfig.Worker.RetryBackoff = 1 } + if newConfig.EventCleanup.Interval == 0 { + newConfig.EventCleanup.Interval = 3600 + } + if newConfig.EventCleanup.ExpireAfter == 0 { + newConfig.EventCleanup.ExpireAfter = 7200 + } // Handle legacy configuration format (where Projects is map[string]string) // This is to maintain backward compatibility with existing configs @@ -525,11 +539,7 @@ func handleWebhook(w http.ResponseWriter, r *http.Request) { } // Store in processed events with a TTL (we'll use a goroutine to remove after 1 hour) - processedEvents.Store(eventID, true) - go func(key string) { - time.Sleep(1 * time.Hour) - processedEvents.Delete(key) - }(eventID) + processedEvents.Store(eventID, time.Now()) // Check if we have a Jenkins job mapping for this repository configMutex.RLock() @@ -742,3 +752,25 @@ func logError(format string, v ...interface{}) { // Error level logs are always shown logger.Printf("[ERROR] "+format, v...) } + +func cleanupEvents() { + for { + configMutex.RLock() + interval := time.Duration(config.EventCleanup.Interval) * time.Second + expireAfter := time.Duration(config.EventCleanup.ExpireAfter) * time.Second + configMutex.RUnlock() + + time.Sleep(interval) + + now := time.Now() + processedEvents.Range(func(key, value interface{}) bool { + if timestamp, ok := value.(time.Time); ok { + if now.Sub(timestamp) > expireAfter { + processedEvents.Delete(key) + logDebug("Cleaned up expired event: %v", key) + } + } + return true + }) + } +}