From 4cd3f695f477f217677a64981c779aefdf6b0b9b Mon Sep 17 00:00:00 2001 From: zhenyus Date: Mon, 21 Apr 2025 22:34:09 +0800 Subject: [PATCH] feat(opentelemetry): implement logging ingestion with OpenTelemetry collector and RBAC configuration Signed-off-by: zhenyus --- freeleaps/helm-pkg/authentication/values.yaml | 2 +- .../devsvc/templates/devsvc/deployment.yaml | 23 +++- .../templates/devsvc/opentelemetry-rbac.yaml | 46 +++++++ .../templates/devsvc/opentelemetry.yaml | 115 ++++++++++++++++++ freeleaps/helm-pkg/devsvc/values.alpha.yaml | 5 + freeleaps/helm-pkg/devsvc/values.yaml | 5 + 6 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 freeleaps/helm-pkg/devsvc/templates/devsvc/opentelemetry-rbac.yaml create mode 100644 freeleaps/helm-pkg/devsvc/templates/devsvc/opentelemetry.yaml diff --git a/freeleaps/helm-pkg/authentication/values.yaml b/freeleaps/helm-pkg/authentication/values.yaml index 41aa0d22..20f561ff 100644 --- a/freeleaps/helm-pkg/authentication/values.yaml +++ b/freeleaps/helm-pkg/authentication/values.yaml @@ -9,7 +9,7 @@ dashboard: metricsPrefix: freeleaps_authentication logIngest: enabled: false - lokiEndpoint: http://loki-gateway.freeleaps-logging-system/loki/api/v1/push + lokiEndpoint: http://loki-gateway.freeleaps-logging-system logPathPattern: /app/log/authentication/*.log logPath: /app/log/authentication fluentbit: diff --git a/freeleaps/helm-pkg/devsvc/templates/devsvc/deployment.yaml b/freeleaps/helm-pkg/devsvc/templates/devsvc/deployment.yaml index ca916d1b..8af898f7 100644 --- a/freeleaps/helm-pkg/devsvc/templates/devsvc/deployment.yaml +++ b/freeleaps/helm-pkg/devsvc/templates/devsvc/deployment.yaml @@ -6,6 +6,10 @@ metadata: app.kubernetes.io/name: "devsvc" app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Values.logIngest.enabled }} + annotations: + opentelemetry.io/config-checksum: {{ include (print $.Template.BasePath "/devsvc/opentelemetry.yaml") . | sha256sum }} +{{- end }} name: "devsvc" namespace: {{ .Release.Namespace | quote }} spec: @@ -25,7 +29,14 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} annotations: app.kubernetes.io/config-checksum: {{ include (print $.Template.BasePath "/devsvc/devsvc-config.yaml") . | sha256sum }} +{{- if .Values.logIngest.enabled }} + opentelemetry.io/config-checksum: {{ include (print $.Template.BasePath "/devsvc/opentelemetry.yaml") . | sha256sum }} + sidecar.opentelemetry.io/inject: "{{ .Release.Namespace}}/{{ .Release.Name }}-opentelemetry-collector" +{{- end }} spec: +{{- if .Values.logIngest.enabled }} + serviceAccountName: "{{ .Release.Name }}-otel-collector" +{{- end }} containers: - name: "devsvc" image: "{{ coalesce .Values.devsvc.image.registry .Values.global.registry "docker.io"}}/{{ coalesce .Values.devsvc.image.repository .Values.global.repository }}/{{ .Values.devsvc.image.name }}:{{ .Values.devsvc.image.tag | default "latest" }}" @@ -97,4 +108,14 @@ spec: secretKeyRef: name: devsvc-config key: {{ $key | snakecase | upper }} - {{- end }} \ No newline at end of file + {{- end }} +{{- if .Values.logIngest.enabled }} + volumeMounts: + - name: app-logs + mountPath: {{ .Values.logIngest.logPath }} +{{- end }} +{{- if .Values.logIngest.enabled }} + volumes: + - name: app-logs + emptyDir: {} +{{- end }} \ No newline at end of file diff --git a/freeleaps/helm-pkg/devsvc/templates/devsvc/opentelemetry-rbac.yaml b/freeleaps/helm-pkg/devsvc/templates/devsvc/opentelemetry-rbac.yaml new file mode 100644 index 00000000..8d25fa64 --- /dev/null +++ b/freeleaps/helm-pkg/devsvc/templates/devsvc/opentelemetry-rbac.yaml @@ -0,0 +1,46 @@ +{{- if .Values.logIngest.enabled }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-otel-collector + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-otel-collector +rules: + - apiGroups: [""] + resources: + - pods + - namespaces + - nodes + verbs: + - get + - watch + - list + - apiGroups: ["apps"] + resources: + - replicasets + - deployments + - statefulsets + - daemonsets + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-otel-collector +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Name }}-otel-collector +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-otel-collector + namespace: {{ .Release.Namespace }} +{{- end }} \ No newline at end of file diff --git a/freeleaps/helm-pkg/devsvc/templates/devsvc/opentelemetry.yaml b/freeleaps/helm-pkg/devsvc/templates/devsvc/opentelemetry.yaml new file mode 100644 index 00000000..ba1bcf9b --- /dev/null +++ b/freeleaps/helm-pkg/devsvc/templates/devsvc/opentelemetry.yaml @@ -0,0 +1,115 @@ +{{- if .Values.logIngest.enabled }} +apiVersion: opentelemetry.io/v1beta1 +kind: OpenTelemetryCollector +metadata: + name: {{ .Release.Name }}-opentelemetry-collector + namespace: {{ .Release.Namespace }} +spec: + mode: sidecar + image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:latest + serviceAccount: "{{ .Release.Name }}-otel-collector" + volumeMounts: + - name: app-logs + mountPath: {{ .Values.logIngest.logPath }} + securityContext: + allowPrivilegeEscalation: true + privileged: true + runAsUser: 0 + runAsGroup: 0 + env: + - name: KUBE_META_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KUBE_META_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBE_META_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: KUBE_META_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBE_META_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: KUBE_META_OBJECT_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['app.kubernetes.io/instance'] + config: + receivers: + filelog: + include: + - {{ .Values.logIngest.logPathPattern }} + start_at: end + include_file_path: false + include_file_name: false + operators: + - type: json_parser + parse_from: body + parse_to: attributes + processors: + resource: + attributes: + - action: insert + key: k8s.node.name + value: ${KUBE_META_NODE_NAME} + - action: insert + key: k8s.pod.name + value: ${KUBE_META_POD_NAME} + - action: insert + key: k8s.pod.ip + value: ${KUBE_META_POD_IP} + - action: insert + key: k8s.pod.uid + value: ${KUBE_META_POD_UID} + - action: insert + key: k8s.namespace.name + value: ${KUBE_META_NAMESPACE} + - action: insert + key: k8s.deployment.name + value: ${KUBE_META_OBJECT_NAME} + transform: + log_statements: + - context: log + statements: + - set(resource.attributes["application"], log.attributes["context"]["app"]) + - set(resource.attributes["environment"], log.attributes["context"]["env"]) + - set(resource.attributes["kubernetes_node_name"], resource.attributes["k8s.node.name"]) + - set(resource.attributes["kubernetes_pod_name"], resource.attributes["k8s.pod.name"]) + - set(resource.attributes["kubernetes_pod_ip"], resource.attributes["k8s.pod.ip"]) + - set(resource.attributes["kubernetes_deployment_name"], resource.attributes["k8s.deployment.name"]) + - set(resource.attributes["kubernetes_namespace"], resource.attributes["k8s.namespace.name"]) + - set(resource.attributes["body_json"], ParseJSON(log.body)) + - set(resource.attributes["body_json"]["kubernetes"]["pod"], resource.attributes["k8s.pod.name"]) + - set(resource.attributes["body_json"]["kubernetes"]["namespace"], resource.attributes["k8s.namespace.name"]) + - set(resource.attributes["body_json"]["kubernetes"]["pod_ip"], resource.attributes["k8s.pod.ip"]) + - set(resource.attributes["body_json"]["kubernetes"]["pod_uid"], resource.attributes["k8s.pod.uid"]) + - set(resource.attributes["body_json"]["kubernetes"]["deployment"], resource.attributes["k8s.deployment.name"]) + - set(resource.attributes["body_json"]["kubernetes"]["node"], resource.attributes["k8s.node.name"]) + - set(resource.attributes["body_json"]["kubernetes"]["namespace"], resource.attributes["k8s.namespace.name"]) + - set(log.body, resource.attributes["body_json"]) + - delete_key(resource.attributes, "body_json") + batch: + send_batch_size: 5 + timeout: 10s + exporters: + otlphttp/logs: + endpoint: {{ .Values.logIngest.lokiEndpoint }}/otlp + tls: + insecure: true + service: + telemetry: + logs: + level: info + pipelines: + logs: + receivers: [filelog] + processors: [resource, transform, batch] + exporters: [otlphttp/logs] +{{- end }} \ No newline at end of file diff --git a/freeleaps/helm-pkg/devsvc/values.alpha.yaml b/freeleaps/helm-pkg/devsvc/values.alpha.yaml index 8ee5252b..79aea663 100644 --- a/freeleaps/helm-pkg/devsvc/values.alpha.yaml +++ b/freeleaps/helm-pkg/devsvc/values.alpha.yaml @@ -4,6 +4,11 @@ global: nodeSelector: {} dashboard: enabled: false +logIngest: + enabled: true + lokiEndpoint: http://loki-gateway.freeleaps-logging-system + logPathPattern: /app/log/devsvc/*.log + logPath: /app/log/devsvc devsvc: replicas: 1 image: diff --git a/freeleaps/helm-pkg/devsvc/values.yaml b/freeleaps/helm-pkg/devsvc/values.yaml index f6b85451..a5a24e19 100644 --- a/freeleaps/helm-pkg/devsvc/values.yaml +++ b/freeleaps/helm-pkg/devsvc/values.yaml @@ -7,6 +7,11 @@ dashboard: name: freeleaps-prod-devsvc-dashboard title: Dev Service Dashboard metricsPrefix: freeleaps_devsvc +logIngest: + enabled: false + lokiEndpoint: http://loki-gateway.freeleaps-logging-system + logPathPattern: /app/log/devsvc/*.log + logPath: /app/log/devsvc devsvc: replicas: 1 image: