feat: add ServiceMonitor and metrics configuration for chat and freeleaps services

Signed-off-by: zhenyus <zhenyus@mathmast.com>
This commit is contained in:
zhenyus 2025-03-18 04:14:17 +08:00
parent 784db23bb5
commit c6016071c2
12 changed files with 1922 additions and 6 deletions

View File

@ -33,3 +33,5 @@ data:
CERT_PATH: {{ .Values.chat.configs.certPath | b64enc | quote }} CERT_PATH: {{ .Values.chat.configs.certPath | b64enc | quote }}
REDIS_URL: {{ .Values.chat.configs.redisUrl | b64enc | quote }} REDIS_URL: {{ .Values.chat.configs.redisUrl | b64enc | quote }}
REDIS_IS_CLUSTER: {{ .Values.chat.configs.redisIsCluster | b64enc | quote }} REDIS_IS_CLUSTER: {{ .Values.chat.configs.redisIsCluster | b64enc | quote }}
METRICS_ENABLED: {{ .Values.chat.configs.metricsEnabled | default false | toString | b64enc }}
PROBES_ENABLED: {{ .Values.chat.configs.probesEnabled | default false | toString | b64enc }}

View File

@ -0,0 +1,865 @@
{{- if .Values.dashboard.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ .Values.dashboard.name }}
namespace: {{ .Values.dashboard.namespace }}
labels:
grafana_dashboard: "1"
data:
{{ .Values.dashboard.name }}.json: |
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 36,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 9,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max",
"min"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.5.2",
"targets": [
{
"$$hashKey": "object:214",
"datasource": {
"type": "prometheus",
"uid": "e4584a9f-5364-4b3d-a851-7abbc5250820"
},
"editorMode": "code",
"expr": "increase({{ .Values.dashboard.metricsPrefix }}_http_requests_total[1m])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ `{{ method }} {{ handler }}` }}",
"range": true,
"refId": "A"
}
],
"title": "Total requests per minute",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "bars",
"fillOpacity": 100,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "4xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "HTTP 500"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#bf1b00",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 7,
"w": 10,
"x": 9,
"y": 0
},
"id": 13,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "11.5.2",
"targets": [
{
"$$hashKey": "object:140",
"datasource": {
"type": "prometheus",
"uid": "e4584a9f-5364-4b3d-a851-7abbc5250820"
},
"editorMode": "code",
"expr": "sum by (status) (rate({{ .Values.dashboard.metricsPrefix }}_http_requests_total[1m]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ `{{ status }}` }}",
"range": true,
"refId": "A"
}
],
"title": "Request per minute",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "errors"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#c15c17",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 7,
"w": 5,
"x": 19,
"y": 0
},
"id": 4,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "11.5.2",
"targets": [
{
"$$hashKey": "object:766",
"datasource": {
"type": "prometheus",
"uid": "e4584a9f-5364-4b3d-a851-7abbc5250820"
},
"editorMode": "code",
"expr": "sum(rate({{ .Values.dashboard.metricsPrefix }}_http_requests_total{status=\"5xx\"}[30s]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "errors",
"range": true,
"refId": "A"
}
],
"title": "Errors per second",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 9,
"x": 0,
"y": 7
},
"id": 6,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max",
"min"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.5.2",
"targets": [
{
"$$hashKey": "object:146",
"datasource": {
"type": "prometheus",
"uid": "e4584a9f-5364-4b3d-a851-7abbc5250820"
},
"editorMode": "code",
"expr": "{{ .Values.dashboard.metricsPrefix }}_http_request_duration_seconds_sum{handler!=\"none\"} / {{ .Values.dashboard.metricsPrefix }}_http_request_duration_seconds_count",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ `{{ handler }}` }}",
"range": true,
"refId": "A"
}
],
"title": "Average response time",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percentunit"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "none"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 10,
"x": 9,
"y": 7
},
"id": 11,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "11.5.2",
"targets": [
{
"$$hashKey": "object:1079",
"datasource": {
"type": "prometheus",
"uid": "e4584a9f-5364-4b3d-a851-7abbc5250820"
},
"editorMode": "code",
"expr": "increase({{ .Values.dashboard.metricsPrefix }}_http_request_duration_seconds_bucket{le=\"0.1\"}[1m]) \n/ ignoring (le) increase({{ .Values.dashboard.metricsPrefix }}_http_request_duration_seconds_count[1m])",
"format": "time_series",
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ `{{ handler }}` }}",
"refId": "A"
}
],
"title": "Requests under 100ms",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line+area"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "transparent",
"value": null
},
{
"color": "red",
"value": 0
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 9,
"x": 0,
"y": 15
},
"id": 16,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max",
"min"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "11.5.2",
"targets": [
{
"$$hashKey": "object:426",
"datasource": {
"type": "prometheus",
"uid": "e4584a9f-5364-4b3d-a851-7abbc5250820"
},
"editorMode": "code",
"expr": "histogram_quantile(0.9, rate({{ .Values.dashboard.metricsPrefix }}_http_request_duration_seconds_bucket{handler!=\"none\"}[30s]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ `{{ handler }}` }}",
"range": true,
"refId": "A"
}
],
"title": "Request duration [s] - p90",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 25,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 10,
"x": 9,
"y": 15
},
"id": 15,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max",
"min"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.5.2",
"targets": [
{
"$$hashKey": "object:426",
"datasource": {
"type": "prometheus",
"uid": "e4584a9f-5364-4b3d-a851-7abbc5250820"
},
"editorMode": "code",
"expr": "histogram_quantile(0.5, rate({{ .Values.dashboard.metricsPrefix }}_http_request_duration_seconds_bucket{handler!=\"none\"}[30s]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ `{{ handler }}` }}",
"range": true,
"refId": "A"
}
],
"title": "Request duration [s] - p50",
"type": "timeseries"
}
],
"preload": false,
"refresh": "5s",
"schemaVersion": 40,
"tags": ["freeleaps"],
"templating": {
"list": []
},
"time": {
"from": "now-5m",
"to": "now"
},
"timepicker": {
"refresh_intervals": []
},
"timezone": "",
"title": "{{ .Values.dashboard.title }}",
"uid": "",
"version": 11,
"weekStart": ""
}
{{- end }}

View File

@ -82,7 +82,7 @@ spec:
{{- end }} {{- end }}
{{- if .Values.chat.probes.readiness.config.failureThreshold }} {{- if .Values.chat.probes.readiness.config.failureThreshold }}
failureThreshold: {{ .Values.chat.probes.readiness.config.failureThreshold }} failureThreshold: {{ .Values.chat.probes.readiness.config.failureThreshold }}
{{- end }}:% {{- end }}
{{- if .Values.chat.probes.readiness.config.terminationGracePeriodSeconds }} {{- if .Values.chat.probes.readiness.config.terminationGracePeriodSeconds }}
terminationGracePeriodSeconds: {{ .Values.chat.probes.readiness.config.terminationGracePeriodSeconds }} terminationGracePeriodSeconds: {{ .Values.chat.probes.readiness.config.terminationGracePeriodSeconds }}
{{- end }} {{- end }}

View File

@ -0,0 +1,40 @@
{{ $namespace := .Release.Namespace }}
{{ $appVersion := .Chart.AppVersion | quote }}
{{ $releaseService := .Release.Service }}
{{ $releaseName := .Release.Name }}
{{- range $service := .Values.chat.services }}
{{- if $service.serviceMonitor.enabled }}
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ $service.name }}-monitor
namespace: {{ $service.serviceMonitor.namespace }}
labels:
app.kubernetes.io/version: {{ $appVersion }}
app.kubernetes.io/name: {{ $service.name }}-monitor
app.kubernetes.io/managed-by: {{ $releaseService }}
app.kubernetes.io/instance: {{ $releaseName }}
{{- if $service.serviceMonitor.labels }}
{{- toYaml $service.serviceMonitor.labels | nindent 4 }}
{{- end }}
spec:
endpoints:
- path: /api/_/metrics
targetPort: {{ $service.targetPort }}
{{- if $service.serviceMonitor.interval }}
interval: {{ $service.serviceMonitor.interval }}
{{- end }}
{{- if $service.serviceMonitor.scrapeTimeout }}
scrapeTimeout: {{ $service.serviceMonitor.scrapeTimeout }}
{{- end }}
namespaceSelector:
matchNames:
- {{ $namespace | quote }}
selector:
matchLabels:
app.kubernetes.io/name: {{ $service.name }}
app.kubernetes.io/instance: {{ $releaseName }}
{{- end }}
{{- end }}

View File

@ -2,6 +2,11 @@ global:
registry: docker.io registry: docker.io
repository: freeleaps repository: freeleaps
nodeSelector: {} nodeSelector: {}
dashboard:
enabled: true
name: freeleaps-prod-chat-dashboard
title: Chat Service Dashboard (PROD)
metricsPrefix: freeleaps_chat
chat: chat:
replicas: 1 replicas: 1
image: image:
@ -21,12 +26,40 @@ chat:
limits: limits:
cpu: 500m cpu: 500m
memory: 512Mi memory: 512Mi
probes: {} probes:
readiness:
type: httpGet
config:
path: /api/_/readyz
port: 8012
initialDelaySeconds: 5
periodSeconds: 30
timeoutSeconds: 3
successThreshold: 1
failureThreshold: 3
liveness:
type: httpGet
config:
path: /api/_/livez
port: 8012
initialDelaySeconds: 5
periodSeconds: 15
timeoutSeconds: 3
successThreshold: 1
failureThreshold: 3
terminationGracePeriodSeconds: 30
services: services:
- name: chat-service - name: chat-service
type: ClusterIP type: ClusterIP
port: 8012 port: 8012
targetPort: 8012 targetPort: 8012
serviceMonitor:
enabled: true
labels:
release: kube-prometheus-stack
namespace: freeleaps-monitoring-system
interval: 30s
scrapeTimeout: ""
ingresses: [] ingresses: []
configs: configs:
tz: UTC tz: UTC
@ -57,3 +90,5 @@ chat:
certPath: '' certPath: ''
redisUrl: "redis://:izrZtCmYk8@freeleaps-prod-redis-redis-cluster-headless.freeleaps-prod.svc.freeleaps.cluster:6379/0" redisUrl: "redis://:izrZtCmYk8@freeleaps-prod-redis-redis-cluster-headless.freeleaps-prod.svc.freeleaps.cluster:6379/0"
redisIsCluster: "true" redisIsCluster: "true"
metricsEnabled: "true"
probesEnabled: "true"

View File

@ -2,6 +2,11 @@ global:
registry: docker.io registry: docker.io
repository: freeleaps repository: freeleaps
nodeSelector: {} nodeSelector: {}
dashboard:
enabled: false
name: freeleaps-prod-chat-dashboard
title: Chat Service Dashboard
metricsPrefix: freeleaps_chat
chat: chat:
replicas: 1 replicas: 1
image: image:
@ -28,6 +33,13 @@ chat:
type: ClusterIP type: ClusterIP
port: 8007 port: 8007
targetPort: 8007 targetPort: 8007
serviceMonitor:
enabled: false
labels:
release: kube-prometheus-stack
namespace: freeleaps-monitoring-system
interval: 30s
scrapeTimeout: ""
# Defaults to {}, which means doesn't have any ingress # Defaults to {}, which means doesn't have any ingress
ingresses: {} ingresses: {}
configs: configs:
@ -87,3 +99,7 @@ chat:
redisUrl: "" redisUrl: ""
# REDIS_IS_CLUSTER # REDIS_IS_CLUSTER
redisIsCluster: "false" redisIsCluster: "false"
# METRICS_ENABLED
metricsEnabled: "false"
# PROBES_ENABLED
probesEnabled: "false"

View File

@ -0,0 +1,865 @@
{{- if .Values.dashboard.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ .Values.dashboard.name }}
namespace: {{ .Values.dashboard.namespace }}
labels:
grafana_dashboard: "1"
data:
{{ .Values.dashboard.name }}.json: |
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 36,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 9,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max",
"min"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.5.2",
"targets": [
{
"$$hashKey": "object:214",
"datasource": {
"type": "prometheus",
"uid": "e4584a9f-5364-4b3d-a851-7abbc5250820"
},
"editorMode": "code",
"expr": "increase({{ .Values.dashboard.metricsPrefix }}_http_requests_total[1m])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ `{{ method }} {{ handler }}` }}",
"range": true,
"refId": "A"
}
],
"title": "Total requests per minute",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "bars",
"fillOpacity": 100,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "4xx"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "HTTP 500"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#bf1b00",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 7,
"w": 10,
"x": 9,
"y": 0
},
"id": 13,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "11.5.2",
"targets": [
{
"$$hashKey": "object:140",
"datasource": {
"type": "prometheus",
"uid": "e4584a9f-5364-4b3d-a851-7abbc5250820"
},
"editorMode": "code",
"expr": "sum by (status) (rate({{ .Values.dashboard.metricsPrefix }}_http_requests_total[1m]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ `{{ status }}` }}",
"range": true,
"refId": "A"
}
],
"title": "Request per minute",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "errors"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#c15c17",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 7,
"w": 5,
"x": 19,
"y": 0
},
"id": 4,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "11.5.2",
"targets": [
{
"$$hashKey": "object:766",
"datasource": {
"type": "prometheus",
"uid": "e4584a9f-5364-4b3d-a851-7abbc5250820"
},
"editorMode": "code",
"expr": "sum(rate({{ .Values.dashboard.metricsPrefix }}_http_requests_total{status=\"5xx\"}[30s]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "errors",
"range": true,
"refId": "A"
}
],
"title": "Errors per second",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 9,
"x": 0,
"y": 7
},
"id": 6,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max",
"min"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.5.2",
"targets": [
{
"$$hashKey": "object:146",
"datasource": {
"type": "prometheus",
"uid": "e4584a9f-5364-4b3d-a851-7abbc5250820"
},
"editorMode": "code",
"expr": "{{ .Values.dashboard.metricsPrefix }}_http_request_duration_seconds_sum{handler!=\"none\"} / {{ .Values.dashboard.metricsPrefix }}_http_request_duration_seconds_count",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ `{{ handler }}` }}",
"range": true,
"refId": "A"
}
],
"title": "Average response time",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percentunit"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "none"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 10,
"x": 9,
"y": 7
},
"id": 11,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "11.5.2",
"targets": [
{
"$$hashKey": "object:1079",
"datasource": {
"type": "prometheus",
"uid": "e4584a9f-5364-4b3d-a851-7abbc5250820"
},
"editorMode": "code",
"expr": "increase({{ .Values.dashboard.metricsPrefix }}_http_request_duration_seconds_bucket{le=\"0.1\"}[1m]) \n/ ignoring (le) increase({{ .Values.dashboard.metricsPrefix }}_http_request_duration_seconds_count[1m])",
"format": "time_series",
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ `{{ handler }}` }}",
"refId": "A"
}
],
"title": "Requests under 100ms",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line+area"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "transparent",
"value": null
},
{
"color": "red",
"value": 0
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 9,
"x": 0,
"y": 15
},
"id": 16,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max",
"min"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "11.5.2",
"targets": [
{
"$$hashKey": "object:426",
"datasource": {
"type": "prometheus",
"uid": "e4584a9f-5364-4b3d-a851-7abbc5250820"
},
"editorMode": "code",
"expr": "histogram_quantile(0.9, rate({{ .Values.dashboard.metricsPrefix }}_http_request_duration_seconds_bucket{handler!=\"none\"}[30s]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ `{{ handler }}` }}",
"range": true,
"refId": "A"
}
],
"title": "Request duration [s] - p90",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 25,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 10,
"x": 9,
"y": 15
},
"id": 15,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max",
"min"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.5.2",
"targets": [
{
"$$hashKey": "object:426",
"datasource": {
"type": "prometheus",
"uid": "e4584a9f-5364-4b3d-a851-7abbc5250820"
},
"editorMode": "code",
"expr": "histogram_quantile(0.5, rate({{ .Values.dashboard.metricsPrefix }}_http_request_duration_seconds_bucket{handler!=\"none\"}[30s]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ `{{ handler }}` }}",
"range": true,
"refId": "A"
}
],
"title": "Request duration [s] - p50",
"type": "timeseries"
}
],
"preload": false,
"refresh": "5s",
"schemaVersion": 40,
"tags": ["freeleaps"],
"templating": {
"list": []
},
"time": {
"from": "now-5m",
"to": "now"
},
"timepicker": {
"refresh_intervals": []
},
"timezone": "",
"title": "{{ .Values.dashboard.title }}",
"uid": "",
"version": 11,
"weekStart": ""
}
{{- end }}

View File

@ -82,7 +82,7 @@ spec:
{{- end }} {{- end }}
{{- if .Values.freeleaps.probes.readiness.config.failureThreshold }} {{- if .Values.freeleaps.probes.readiness.config.failureThreshold }}
failureThreshold: {{ .Values.freeleaps.probes.readiness.config.failureThreshold }} failureThreshold: {{ .Values.freeleaps.probes.readiness.config.failureThreshold }}
{{- end }}:% {{- end }}
{{- if .Values.freeleaps.probes.readiness.config.terminationGracePeriodSeconds }} {{- if .Values.freeleaps.probes.readiness.config.terminationGracePeriodSeconds }}
terminationGracePeriodSeconds: {{ .Values.freeleaps.probes.readiness.config.terminationGracePeriodSeconds }} terminationGracePeriodSeconds: {{ .Values.freeleaps.probes.readiness.config.terminationGracePeriodSeconds }}
{{- end }} {{- end }}

View File

@ -34,3 +34,5 @@ data:
CERT_PATH: {{ .Values.freeleaps.configs.certPath | b64enc | quote }} CERT_PATH: {{ .Values.freeleaps.configs.certPath | b64enc | quote }}
REDIS_URL: {{ .Values.freeleaps.configs.redisUrl | b64enc | quote }} REDIS_URL: {{ .Values.freeleaps.configs.redisUrl | b64enc | quote }}
REDIS_IS_CLUSTER: {{ .Values.freeleaps.configs.redisIsCluster | b64enc | quote }} REDIS_IS_CLUSTER: {{ .Values.freeleaps.configs.redisIsCluster | b64enc | quote }}
METRICS_ENABLED: {{ .Values.freeleaps.configs.metricsEnabled | default false | toString | b64enc }}
PROBES_ENABLED: {{ .Values.freeleaps.configs.probesEnabled | default false | toString | b64enc }}

View File

@ -0,0 +1,40 @@
{{ $namespace := .Release.Namespace }}
{{ $appVersion := .Chart.AppVersion | quote }}
{{ $releaseService := .Release.Service }}
{{ $releaseName := .Release.Name }}
{{- range $service := .Values.freeleaps.services }}
{{- if $service.serviceMonitor.enabled }}
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ $service.name }}-monitor
namespace: {{ $service.serviceMonitor.namespace }}
labels:
app.kubernetes.io/version: {{ $appVersion }}
app.kubernetes.io/name: {{ $service.name }}-monitor
app.kubernetes.io/managed-by: {{ $releaseService }}
app.kubernetes.io/instance: {{ $releaseName }}
{{- if $service.serviceMonitor.labels }}
{{- toYaml $service.serviceMonitor.labels | nindent 4 }}
{{- end }}
spec:
endpoints:
- path: /api/_/metrics
targetPort: {{ $service.targetPort }}
{{- if $service.serviceMonitor.interval }}
interval: {{ $service.serviceMonitor.interval }}
{{- end }}
{{- if $service.serviceMonitor.scrapeTimeout }}
scrapeTimeout: {{ $service.serviceMonitor.scrapeTimeout }}
{{- end }}
namespaceSelector:
matchNames:
- {{ $namespace | quote }}
selector:
matchLabels:
app.kubernetes.io/name: {{ $service.name }}
app.kubernetes.io/instance: {{ $releaseName }}
{{- end }}
{{- end }}

View File

@ -2,6 +2,11 @@ global:
registry: docker.io registry: docker.io
repository: freeleaps repository: freeleaps
nodeSelector: {} nodeSelector: {}
dashboard:
enabled: true
name: freeleaps-prod-backend-dashboard
title: Backend Service Dashboard (PROD)
metricsPrefix: freeleaps_backend
freeleaps: freeleaps:
replicas: 1 replicas: 1
image: image:
@ -21,12 +26,40 @@ freeleaps:
limits: limits:
cpu: '500m' cpu: '500m'
memory: 512Mi memory: 512Mi
probes: {} probes:
readiness:
type: httpGet
config:
path: /api/_/readyz
port: 8001
initialDelaySeconds: 5
periodSeconds: 30
timeoutSeconds: 3
successThreshold: 1
failureThreshold: 3
liveness:
type: httpGet
config:
path: /api/_/livez
port: 8001
initialDelaySeconds: 5
periodSeconds: 15
timeoutSeconds: 3
successThreshold: 1
failureThreshold: 3
terminationGracePeriodSeconds: 30
services: services:
- name: freeleaps-service - name: freeleaps-service
type: ClusterIP type: ClusterIP
port: 8001 port: 8001
targetPort: 8001 targetPort: 8001
serviceMonitor:
enabled: true
labels:
release: kube-prometheus-stack
namespace: freeleaps-monitoring-system
interval: 30s
scrapeTimeout: ""
ingresses: [] ingresses: []
configs: configs:
tz: UTC tz: UTC
@ -58,3 +91,5 @@ freeleaps:
certPath: '' certPath: ''
redisUrl: "redis://r6Y@QTb*7BQN@hDGsN@freeleaps-prod-redis-redis-cluster-headless.freeleaps-prod.svc.freeleaps.cluster:6379/0" redisUrl: "redis://r6Y@QTb*7BQN@hDGsN@freeleaps-prod-redis-redis-cluster-headless.freeleaps-prod.svc.freeleaps.cluster:6379/0"
redisIsCluster: "true" redisIsCluster: "true"
metricsEnabled: "true"
probesEnabled: "true"

View File

@ -2,6 +2,11 @@ global:
registry: docker.io registry: docker.io
repository: freeleaps repository: freeleaps
nodeSelector: {} nodeSelector: {}
dashboard:
enabled: false
name: freeleaps-prod-backend-dashboard
title: Backend Service Dashboard
metricsPrefix: freeleaps_backend
freeleaps: freeleaps:
replicas: 1 replicas: 1
image: image:
@ -28,6 +33,13 @@ freeleaps:
type: ClusterIP type: ClusterIP
port: 8007 port: 8007
targetPort: 8007 targetPort: 8007
serviceMonitor:
enabled: false
labels:
release: kube-prometheus-stack
namespace: freeleaps-monitoring-system
interval: 30s
scrapeTimeout: ""
# Defaults to {}, which means doesn't have any ingress # Defaults to {}, which means doesn't have any ingress
ingresses: {} ingresses: {}
configs: configs:
@ -89,3 +101,7 @@ freeleaps:
redisUrl: "" redisUrl: ""
# REDIS_IS_CLUSTER # REDIS_IS_CLUSTER
redisIsCluster: "false" redisIsCluster: "false"
# METRICS_ENABLED
metricsEnabled: "false"
# PROBES_ENABLED
probesEnabled: "false"