Skip to content

Commit bb99737

Browse files
dtfranzdtfranz
committed
Prometheus Metrics
Adds prometheus to the test-e2e Makefile target, which stands up a barebones prometheus scraper to gather metrics from the operator-controller and catalogd pods during the e2e test run. When finished, the prometheus server is queried for a raw output of the metrics and stores it in metrics.out. These metrics will be analyzed in a later PR. Signed-off-by: Daniel Franz <[email protected]>
1 parent 44de6f2 commit bb99737

File tree

3 files changed

+235
-1
lines changed

3 files changed

+235
-1
lines changed

‎Makefile

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,17 @@ image-registry: ## Build the testdata catalog used for e2e tests and push it to
256256
test-e2e: KIND_CLUSTER_NAME := operator-controller-e2e
257257
test-e2e: KUSTOMIZE_BUILD_DIR := config/overlays/e2e
258258
test-e2e: GO_BUILD_EXTRA_FLAGS := -cover
259-
test-e2e: run image-registry e2e e2e-coverage kind-clean #HELP Run e2e test suite on local kind cluster
259+
test-e2e: run image-registry prometheus e2e e2e-coverage e2e-metrics #HELP Run e2e test suite on local kind cluster
260+
261+
.PHONY: prometheus
262+
prometheus: PROMETHEUS_NAMESPACE := olmv1-system
263+
prometheus: PROMETHEUS_VERSION := v0.83.0
264+
prometheus: #HELP Deploy Prometheus into specified namespace
265+
./hack/test/setup-monitoring.sh $(PROMETHEUS_NAMESPACE) $(PROMETHEUS_VERSION) $(KUSTOMIZE)
266+
267+
.PHONY: e2e-metrics
268+
e2e-metrics: #HELP Request metrics from prometheus; place in ARTIFACT_PATH if set
269+
curl 127.0.0.1:30900/metrics > $(if $(ARTIFACT_PATH),$(ARTIFACT_PATH),.)/metrics.out
260270

261271
.PHONY: extension-developer-e2e
262272
extension-developer-e2e: KUSTOMIZE_BUILD_DIR := config/overlays/cert-manager

‎hack/test/setup-monitoring.sh

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
help="setup-monitoring.sh is used to set up prometheus monitoring for e2e testing.
6+
7+
Usage:
8+
setup-monitoring.sh [PROMETHEUS_NAMESPACE] [PROMETHEUS_VERSION] [KUSTOMIZE]
9+
"
10+
11+
if [[ "$#" -ne 3 ]]; then
12+
echo "Illegal number of arguments passed"
13+
echo "${help}"
14+
exit 1
15+
fi
16+
17+
NAMESPACE=$1
18+
PROMETHEUS_VERSION=$2
19+
KUSTOMIZE=$3
20+
21+
TMPDIR=$(mktemp -d)
22+
trap 'echo "Cleaning up ${TMPDIR}"; rm -rf "${TMPDIR}"' EXIT
23+
curl -s "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/refs/tags/${PROMETHEUS_VERSION}/kustomization.yaml" > "${TMPDIR}/kustomization.yaml"
24+
curl -s "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/refs/tags/${PROMETHEUS_VERSION}/bundle.yaml" > "${TMPDIR}/bundle.yaml"
25+
(cd ${TMPDIR} && ${KUSTOMIZE} edit set namespace ${NAMESPACE}) && kubectl create -k "${TMPDIR}"
26+
kubectl wait --for=condition=Ready pods -n ${NAMESPACE} -l app.kubernetes.io/name=prometheus-operator
27+
28+
kubectl apply -f - << EOF
29+
apiVersion: v1
30+
kind: ServiceAccount
31+
metadata:
32+
name: prometheus
33+
namespace: ${NAMESPACE}
34+
---
35+
apiVersion: rbac.authorization.k8s.io/v1
36+
kind: ClusterRole
37+
metadata:
38+
name: prometheus
39+
rules:
40+
- apiGroups: [""]
41+
resources:
42+
- nodes
43+
- nodes/metrics
44+
- services
45+
- endpoints
46+
- pods
47+
verbs: ["get", "list", "watch"]
48+
- apiGroups: [""]
49+
resources:
50+
- configmaps
51+
verbs: ["get"]
52+
- apiGroups:
53+
- discovery.k8s.io
54+
resources:
55+
- endpointslices
56+
verbs: ["get", "list", "watch"]
57+
- apiGroups:
58+
- networking.k8s.io
59+
resources:
60+
- ingresses
61+
verbs: ["get", "list", "watch"]
62+
- nonResourceURLs: ["/metrics"]
63+
verbs: ["get"]
64+
---
65+
apiVersion: rbac.authorization.k8s.io/v1
66+
kind: ClusterRoleBinding
67+
metadata:
68+
name: prometheus
69+
roleRef:
70+
apiGroup: rbac.authorization.k8s.io
71+
kind: ClusterRole
72+
name: prometheus
73+
subjects:
74+
- kind: ServiceAccount
75+
name: prometheus
76+
namespace: ${NAMESPACE}
77+
EOF
78+
79+
kubectl apply -f - << EOF
80+
apiVersion: monitoring.coreos.com/v1
81+
kind: Prometheus
82+
metadata:
83+
name: prometheus
84+
namespace: ${NAMESPACE}
85+
spec:
86+
logLevel: debug
87+
serviceAccountName: prometheus
88+
scrapeTimeout: 30s
89+
scrapeInterval: 1m
90+
securityContext:
91+
runAsNonRoot: true
92+
runAsUser: 65534
93+
seccompProfile:
94+
type: RuntimeDefault
95+
serviceMonitorSelector: {}
96+
EOF
97+
98+
kubectl apply -f - << EOF
99+
apiVersion: networking.k8s.io/v1
100+
kind: NetworkPolicy
101+
metadata:
102+
name: prometheus
103+
namespace: ${NAMESPACE}
104+
spec:
105+
podSelector:
106+
matchLabels:
107+
app.kubernetes.io/name: prometheus
108+
policyTypes:
109+
- Egress
110+
egress:
111+
- {} # Allows all egress traffic for metrics requests
112+
EOF
113+
114+
# Give the operator time to create the pod
115+
kubectl wait --for=create pods -n ${NAMESPACE} prometheus-prometheus-0 --timeout=60s
116+
kubectl wait --for=condition=Ready pods -n ${NAMESPACE} prometheus-prometheus-0 --timeout=120s
117+
118+
# Authentication token for the scrape requests
119+
kubectl apply -f - <<EOF
120+
apiVersion: v1
121+
kind: Secret
122+
type: kubernetes.io/service-account-token
123+
metadata:
124+
name: prometheus-metrics-token
125+
namespace: ${NAMESPACE}
126+
annotations:
127+
kubernetes.io/service-account.name: prometheus
128+
EOF
129+
130+
# ServiceMonitors for operator-controller and catalogd
131+
kubectl apply -f - <<EOF
132+
apiVersion: monitoring.coreos.com/v1
133+
kind: ServiceMonitor
134+
metadata:
135+
name: operator-controller-controller-manager-metrics-monitor
136+
namespace: ${NAMESPACE}
137+
spec:
138+
endpoints:
139+
- path: /metrics
140+
port: https
141+
scheme: https
142+
authorization:
143+
credentials:
144+
name: prometheus-metrics-token
145+
key: token
146+
tlsConfig:
147+
insecureSkipVerify: false
148+
serverName: operator-controller-service.${NAMESPACE}.svc
149+
ca:
150+
secret:
151+
name: olmv1-cert
152+
key: ca.crt
153+
cert:
154+
secret:
155+
name: olmv1-cert
156+
key: tls.crt
157+
keySecret:
158+
name: olmv1-cert
159+
key: tls.key
160+
selector:
161+
matchLabels:
162+
control-plane: operator-controller-controller-manager
163+
EOF
164+
165+
CATD_SECRET=$(kubectl get secret -n ${NAMESPACE} -o jsonpath="{.items[*].metadata.name}" | tr ' ' '\n' | grep '^catalogd-service-cert')
166+
167+
kubectl apply -f - <<EOF
168+
apiVersion: monitoring.coreos.com/v1
169+
kind: ServiceMonitor
170+
metadata:
171+
name: catalogd-controller-manager-metrics-monitor
172+
namespace: ${NAMESPACE}
173+
spec:
174+
endpoints:
175+
- path: /metrics
176+
port: metrics
177+
scheme: https
178+
authorization:
179+
credentials:
180+
name: prometheus-metrics-token
181+
key: token
182+
tlsConfig:
183+
serverName: catalogd-service.${NAMESPACE}.svc
184+
insecureSkipVerify: false
185+
ca:
186+
secret:
187+
name: ${CATD_SECRET}
188+
key: ca.crt
189+
cert:
190+
secret:
191+
name: ${CATD_SECRET}
192+
key: tls.crt
193+
keySecret:
194+
name: ${CATD_SECRET}
195+
key: tls.key
196+
selector:
197+
matchLabels:
198+
app.kubernetes.io/name: catalogd
199+
EOF
200+
201+
# NodePort service to allow querying prometheus from outside the cluster
202+
# NOTE: This NodePort must also be configured in kind-config.yaml
203+
kubectl apply -f - <<EOF
204+
apiVersion: v1
205+
kind: Service
206+
metadata:
207+
name: prometheus-service
208+
namespace: ${NAMESPACE}
209+
spec:
210+
type: NodePort
211+
ports:
212+
- name: web
213+
nodePort: 30900
214+
port: 9090
215+
protocol: TCP
216+
targetPort: web
217+
selector:
218+
prometheus: prometheus
219+
EOF

‎kind-config.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@ nodes:
88
hostPort: 30000
99
listenAddress: "127.0.0.1"
1010
protocol: tcp
11+
# prometheus metrics service's NodePort
12+
- containerPort: 30900
13+
hostPort: 30900
14+
listenAddress: "127.0.0.1"
15+
protocol: tcp
1116
kubeadmConfigPatches:
1217
- |
1318
kind: ClusterConfiguration

0 commit comments

Comments
 (0)