michael_dileo/Keybard-Vagabond-Demo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

redaction (#1)

Browse Source 
Add the redacted source file for demo purposes

Reviewed-on: https://source.michaeldileo.org/michael_dileo/Keybard-Vagabond-Demo/pulls/1
Co-authored-by: Michael DiLeo <michael_dileo@proton.me>
Co-committed-by: Michael DiLeo <michael_dileo@proton.me>
This commit was merged in pull request #1.
This commit is contained in:
Michael DiLeo
2025-12-24 13:40:47 +00:00
committed by michael_dileo
parent 612235d52b
commit 7327d77dcd
333 changed files with 39286 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
manifests/infrastructure/authentik/authentik-server.yaml Normal file
View File

@@ -0,0 +1,95 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: authentik-server
namespace: authentik-system
labels:
app.kubernetes.io/name: authentik
app.kubernetes.io/component: server
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: authentik
app.kubernetes.io/component: server
template:
metadata:
labels:
app.kubernetes.io/name: authentik
app.kubernetes.io/component: server
spec:
serviceAccountName: authentik
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
containers:
- name: authentik
image: ghcr.io/goauthentik/server:2024.10.1
args: ["server"]
env: []
envFrom:
- secretRef:
name: authentik-database
- secretRef:
name: authentik-email
- secretRef:
name: authentik-secret-key
ports:
- name: http
containerPort: 9000
protocol: TCP
- name: metrics
containerPort: 9300
protocol: TCP
livenessProbe:
httpGet:
path: /-/health/live/
port: http
initialDelaySeconds: 30
periodSeconds: 30
readinessProbe:
httpGet:
path: /-/health/ready/
port: http
initialDelaySeconds: 30
periodSeconds: 30
volumeMounts:
- name: media
mountPath: /media
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
cpu: 1000m
memory: 1Gi
volumes:
- name: media
persistentVolumeClaim:
claimName: authentik-media
---
apiVersion: v1
kind: Service
metadata:
name: authentik-server
namespace: authentik-system
labels:
app.kubernetes.io/name: authentik
app.kubernetes.io/component: server
spec:
type: ClusterIP
ports:
- port: 80
targetPort: http
protocol: TCP
name: http
- port: 9300
targetPort: metrics
protocol: TCP
name: metrics
selector:
app.kubernetes.io/name: authentik
app.kubernetes.io/component: server

53
manifests/infrastructure/authentik/authentik-worker.yaml Normal file
View File

@@ -0,0 +1,53 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: authentik-worker
namespace: authentik-system
labels:
app.kubernetes.io/name: authentik
app.kubernetes.io/component: worker
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: authentik
app.kubernetes.io/component: worker
template:
metadata:
labels:
app.kubernetes.io/name: authentik
app.kubernetes.io/component: worker
spec:
serviceAccountName: authentik
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
containers:
- name: authentik
image: ghcr.io/goauthentik/server:2024.10.1
args: ["worker"]
env: []
envFrom:
- secretRef:
name: authentik-database
- secretRef:
name: authentik-email
- secretRef:
name: authentik-secret-key
volumeMounts:
- name: media
mountPath: /media
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
cpu: 500m
memory: 1Gi
volumes:
- name: media
persistentVolumeClaim:
claimName: authentik-media

26
manifests/infrastructure/authentik/ingress.yaml Normal file
View File

@@ -0,0 +1,26 @@
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: authentik
namespace: authentik-system
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
labels:
app.kubernetes.io/name: authentik
spec:
ingressClassName: nginx
tls: []
rules:
- host: auth.keyboardvagabond.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: authentik-server
port:
number: 80

19
manifests/infrastructure/authentik/kustomization.yaml Normal file
View File

@@ -0,0 +1,19 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: authentik-system
resources:
- namespace.yaml
- secret.yaml
- storage.yaml
- rbac.yaml
- authentik-server.yaml
- authentik-worker.yaml
- ingress.yaml
- monitoring.yaml
commonLabels:
app.kubernetes.io/name: authentik
app.kubernetes.io/managed-by: flux

17
manifests/infrastructure/authentik/monitoring.yaml Normal file
View File

@@ -0,0 +1,17 @@
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: authentik
namespace: authentik-system
labels:
app.kubernetes.io/name: authentik
spec:
selector:
matchLabels:
app.kubernetes.io/name: authentik
app.kubernetes.io/component: server
endpoints:
- port: metrics
interval: 30s
path: /metrics

7
manifests/infrastructure/authentik/namespace.yaml Normal file
View File

@@ -0,0 +1,7 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: authentik-system
labels:
name: authentik-system

37
manifests/infrastructure/authentik/rbac.yaml Normal file
View File

@@ -0,0 +1,37 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: authentik
namespace: authentik-system
labels:
app.kubernetes.io/name: authentik
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: authentik
labels:
app.kubernetes.io/name: authentik
rules:
- apiGroups: [""]
resources: ["secrets", "services", "configmaps"]
verbs: ["get", "create", "delete", "list", "patch"]
- apiGroups: ["extensions", "networking.k8s.io"]
resources: ["ingresses"]
verbs: ["get", "create", "delete", "list", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: authentik
labels:
app.kubernetes.io/name: authentik
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: authentik
subjects:
- kind: ServiceAccount
name: authentik
namespace: authentik-system

139
manifests/infrastructure/authentik/secret.yaml Normal file
View File

@@ -0,0 +1,139 @@
apiVersion: v1
kind: Secret
metadata:
name: authentik-database
namespace: authentik-system
type: Opaque
stringData:
AUTHENTIK_POSTGRESQL__HOST: ENC[AES256_GCM,data:9TdztE1I6SoZLb+4PwsLOALMz0iKjPwBvda+msKsDKkGirospK1eR7KU+xg4r3/f8ljxXxHfBfw=,iv:9LYyntD886h0eIyAUoqwy0X8CgL9J5eTPcElW7c8zrU=,tag:jcxBbbHBhn+TjjzqkCz8rQ==,type:str]
AUTHENTIK_POSTGRESQL__NAME: ENC[AES256_GCM,data:RkaWaRQgLs0F,iv:zdFK0E6P0MS+j05LMuq1jbyJOQ7Wsmy8PQJGFzB+HZw=,tag:rSrokvI5Z3Xloa/Y3xz7qg==,type:str]
AUTHENTIK_POSTGRESQL__USER: ENC[AES256_GCM,data:4z2ZTkz2MZwu,iv:tomRCn5oUafPCLCRrn39UHZUuFTngHN20/IP6qEO4r0=,tag:yX5Ey+jF5zJB35hONFnu5Q==,type:str]
AUTHENTIK_POSTGRESQL__PASSWORD: ENC[AES256_GCM,data:geaDqJ4GU0ycU64DbTQrt7KvrB6NnwCfoXWnmcmNnvQk79Uah4LCGRO3zEaRQ1QqoCk=,iv:btKTOY8UnSrcXpOEm3gayxlgHaiTnq3QMmhp564GTI4=,tag:P9N0opqZDVXcyu1PDmabIg==,type:str]
#ENC[AES256_GCM,data:XSuzEME0hIJ9CyNU3D/pml8O2/NKhHBMLOiPg6Whm20pscw3AY/sdBBo2vC+ghXkvDtEuzce381tW4Y2KvCvsi/p2mgzmsz68y0S,iv:uOM2z1+ujgeOiPoOJfEOCpuLMjcHu5kGjNJADyjY3p8=,tag:hfvU5Q/Db0VvjHDPhjjFmw==,type:comment]
#ENC[AES256_GCM,data:M/EmK6N+cEJwyZGVc2lTY2PJzOwTHv5KcngpE9zrs8sz1iBzWv9Esf9ZyPxaB96FowfHjVFxp6dkl2/KU1R0/e2ZKg6B5p4u7RAu46rD0x7Q35V2gGRYrgwWySkjG0i7Ycrfq/HvVw==,iv:gXd4PzY32YlaPusA4QHNfxwcu1BQuCuMemlrGHf2v78=,tag:mIVnAcc/Aq7naScEyK7Mbw==,type:comment]
#ENC[AES256_GCM,data:A4yF/J7uXPFq1tbrGqje+GMd0DXoUjHacH2mPWxFu1YvVy1azM+xTu1bG2E7R+6EBVdmdFUq3Vs=,iv:qUKbR45DE5/fEvtW+dA4mCWSD9qnyEllyowj1joz/1k=,tag:AsBhhZL4KJrnj3zAfAp2eQ==,type:comment]
#ENC[AES256_GCM,data:4YLA1zEo3+keEBW2qGW4Q599QVr87TjqEpSLLBGjpeDObBSO28yA+n7AxLyU+MInR4bBagAUVas=,iv:tx6qax/lPLNsk7l9h8B4ZFD/rDk+ule3CEfCghuCGTs=,tag:Ly9aFHJ8N3zUanFW//UxKw==,type:comment]
#ENC[AES256_GCM,data:Wr+HgQJoA/af8GWSB3GbCoOLDJ4qMdtOoofI1pICswdk2TEyX/HFyvHYwb5dSEgJV/p7WV5kW3KlDhgJ9T7g6yiK802jKrvzZLM6oIWVWbxOKigdxpfyYF0IM8svkVC4J6iod+w=,iv:rmVi7Mme1Pm3sJiqw8R7WdlQZUHR3I2eYOluG3yHDDw=,tag:+VO6VfCjpNN0puwi4Y4C7w==,type:comment]
#ENC[AES256_GCM,data:Q8BT3aHd8UZExGexxr4xFGtndGLWsIdPn+FOHGUwcMWWXwqMgH3IGN1aaTEXMydaFY9Ztvs=,iv:MAEAawMEdVEfVXStjuHVBWsaHGtGL2ZuEb/8kWENRcs=,tag:eYVMR5bbWBQW5FXlzI6z3g==,type:comment]
#ENC[AES256_GCM,data:knsJsY39Khpa+BnseltFMLI5mZl4pJDg5k1Fwms0/+Bb/bVjleQ43Tp/sNpwWyr+Jz5SAJWqYtOyRPjCLfbeJQZQBw4k+gZa2mLipjlvRjUV/cb02wwhbDTVZ2b/IYXhtY4sVaY+nQ==,iv:7okkXj1t2SdMx3593raRG2nUsPpf4rxizkq85CGbT1M=,tag:h/pHCguKcwPefi8OZdEBJg==,type:comment]
#ENC[AES256_GCM,data:vJ7suQfW7heDpdycfGwVoCPxC4gf3drB095qovQY+m8HTagyPjbg0Z55nj8iD4Hu1RmSvjXNhUk=,iv:AwFzxm7dQOqHKj1gFyPz3xpEg+vdqXLjpfbDG3KUTfs=,tag:OS7qad0oSNKKx84NEmvb0g==,type:comment]
#ENC[AES256_GCM,data:O4T1JWBdegex0cuVfwAeA0kXL6szR7v7ZLL5c5v5HsvJ6UrjM8jYDv6ab5J2XwpEmjp6s7hYqW8=,iv:klOmrg4h59Jsnc8PSA6kwhr4mGrD7p7BGKxFPOmKBXw=,tag:sNZ+bS8eYRLVOs7/oiG/Qw==,type:comment]
#ENC[AES256_GCM,data:tAhk//GOoD1DpOH9/MirfadQpWxYgMcuVUo5ilmpHjKVMwYmnjMdZyjVlmFzPr+L0w60I3W2GpQL8Of038ytm6PEl1VW9AP2Su/k6YkEPMSjm0VSfme3WPpUyP0kmD0MdQ9PrOw=,iv:Sf3Hjodop0wER5iA4t316A00X52dtLx7u9L8Hs1uZ/4=,tag:VNaNy37jxjaXU+X6vzomzw==,type:comment]
AUTHENTIK_REDIS__HOST: ENC[AES256_GCM,data:FVYkkGxa2qY6LOlkZKo0RyW0HPNU/UdgXyEsmxYhWMpzLxxXvUZqA0uIVSg0kNo=,iv:ngr4rd13tFqOip3chxPpOxIqdXUKq6TrAo0/ZLXRCDg=,tag:A15OavPIpkrLZndw34JeEQ==,type:str]
AUTHENTIK_REDIS__PORT: ENC[AES256_GCM,data:QJS1sw==,iv:gwC4DxKbKAlFbseLXi3EBS8KGpuJq7uJLcT5LXUSLYk=,tag:yonnh/bTUdO02x867m7ZwQ==,type:str]
AUTHENTIK_REDIS__PASSWORD: ENC[AES256_GCM,data:F4HLQQ6Ht+FmNXD4ptxXugqMKuRxuIb4rY/DDqObbSY=,iv:E3nM5QdonA8HBZoOXVD5yr2hWLf1qapAEV1RjQ2zh04=,tag:pHNYPS9HSXrjbNQ9PvjT9A==,type:str]
sops:
lastmodified: "2025-11-24T15:25:26Z"
mac: ENC[AES256_GCM,data:STkCUURHKnPRDuiS5aXfhj8/+at6A4qA4C3te2m+HMzwV7UfB57wK84JZIbF8649yzePxQ6naZQfoBhVOBsyXUvxcQdEEbyimHKfGhInXlXpCt/LTnG4nS51JvVBTLsgT/P/eeX6LKRG3hvoK9cV+jkxyrPfqa3I0Bhr2YBsF5k=,iv:QSebad42NkWP1jRYcu0YuuQbkAi2VTXfVCSyxlomOuo=,tag:aR6R4YJAdJnNFhYRdGaFPQ==,type:str]
pgp:
- created_at: "2025-07-10T13:58:33Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DZT3mpHTS/JgSAQdAnoDla7hPtWEhQmy3KFLW9RkB7qKOAlJVSqO5Sq/lgT4w
nV5zAaOimcbBnT66mJbN59xLUZ67k3RHngtPIjnnmP0iqa4p1VtSwdx1ypUAaIQT
1GgBCQIQ0mnWTxbUiUQvIlcJV3Hx4Ec5XuQNzNlYm5tXQD8Ttx/wLh3N+RdAefW5
mzNK3HbDVB/9IRcoNY8C+L0EiJrjHvQCDgnXKT2oH6wyTpG+m2bwpkRN+wT5d1Xl
gpRfYLm/N8Blcw==
=wn2E
-----END PGP MESSAGE-----
fp: B120595CA9A643B051731B32E67FF350227BA4E8
- created_at: "2025-07-10T13:58:33Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DSXzd60P2RKISAQdA+ARG+XplGtU+RvLQvJ6MFga8gSfrQA4Zks2JReyxnHUw
ui/BpxRdxJDL43Xa69R4VdcYXifDQlfVomDzEdlTBSuJHI9VhtHLnqUH3rXjBL0X
1GgBCQIQqfgaAeSCRb2AJINKueQe3dVAT8G3CYE588/UsFniV46u3FEO9h0+rG6e
J8xB8+pyiQz2v3Sz6qjeULT2dAJF+9qp4U0wyO2KTmbqwvGrX9od1/5WDkSu7J2I
o2IBbMiyDoMwbw==
=G5eE
-----END PGP MESSAGE-----
fp: 4A8AADB4EBAB9AF88EF7062373CECE06CC80D40C
encrypted_regex: ^(data|stringData)$
version: 3.10.2
---
apiVersion: v1
kind: Secret
metadata:
name: authentik-email
namespace: authentik-system
type: Opaque
stringData:
AUTHENTIK_EMAIL__HOST: ENC[AES256_GCM,data:nrv0Ut+QJWlbnMTvIgw6xl2bDA==,iv:tFEU0GoQRG/rzihtLNz6oKcwPbqgcRZEMwYtLOpIp+o=,tag:LAvsM/0IqRClNmsTnSLZPA==,type:str]
AUTHENTIK_EMAIL__PORT: ENC[AES256_GCM,data:vRH/,iv:H1IcwN0iOoBZ6p9YpQ1vkqSOL+Qtt/sttwks1cMl8OE=,tag:WtOihNSgF++daDYonCOJcw==,type:str]
AUTHENTIK_EMAIL__USERNAME: ENC[AES256_GCM,data:2Zo9Rkm7tqt1Fnh1tlv3RX9HJagoHJFwCYtroYM=,iv:Gez8R4YS31e/6F5qD4dbro1gqYEmr3Qbfvr1iPefgOg=,tag:3CU8XR37DOG4xhVl0IZ2eQ==,type:str]
AUTHENTIK_EMAIL__PASSWORD: ENC[AES256_GCM,data:5FEtUseuqSoMLcFExYO8UPeRbj9X1x8NcM88YR2OY6ngHKCmPg6zUrCnoPNp1TtbOlM=,iv:uiADagkl11OfVrxtmjzpl6PNZV+6hQSejoevigNfVNg=,tag:zFjzS//1KnOcpMS7zuKe8A==,type:str]
AUTHENTIK_EMAIL__USE_TLS: ENC[AES256_GCM,data:Rdsk5w==,iv:juupjOLf0d5GY9/mIEesiQO7e0i00vG7cydE7ob+tw8=,tag:bt0kzcPJFBjXvQ8CeFiMiw==,type:str]
AUTHENTIK_EMAIL__FROM: ENC[AES256_GCM,data:CwJOJfRSzLtG5QcCYb6WXrb+qSDJuMyGTJPj5sI=,iv:J3klbofKTWwpzBqyXMLEBBaUR5mAWP+m/xA7GCKNndo=,tag:EvVJSJ/CFBY68W18ABAIAg==,type:str]
sops:
lastmodified: "2025-11-24T15:25:26Z"
mac: ENC[AES256_GCM,data:STkCUURHKnPRDuiS5aXfhj8/+at6A4qA4C3te2m+HMzwV7UfB57wK84JZIbF8649yzePxQ6naZQfoBhVOBsyXUvxcQdEEbyimHKfGhInXlXpCt/LTnG4nS51JvVBTLsgT/P/eeX6LKRG3hvoK9cV+jkxyrPfqa3I0Bhr2YBsF5k=,iv:QSebad42NkWP1jRYcu0YuuQbkAi2VTXfVCSyxlomOuo=,tag:aR6R4YJAdJnNFhYRdGaFPQ==,type:str]
pgp:
- created_at: "2025-07-10T13:58:33Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DZT3mpHTS/JgSAQdAnoDla7hPtWEhQmy3KFLW9RkB7qKOAlJVSqO5Sq/lgT4w
nV5zAaOimcbBnT66mJbN59xLUZ67k3RHngtPIjnnmP0iqa4p1VtSwdx1ypUAaIQT
1GgBCQIQ0mnWTxbUiUQvIlcJV3Hx4Ec5XuQNzNlYm5tXQD8Ttx/wLh3N+RdAefW5
mzNK3HbDVB/9IRcoNY8C+L0EiJrjHvQCDgnXKT2oH6wyTpG+m2bwpkRN+wT5d1Xl
gpRfYLm/N8Blcw==
=wn2E
-----END PGP MESSAGE-----
fp: B120595CA9A643B051731B32E67FF350227BA4E8
- created_at: "2025-07-10T13:58:33Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DSXzd60P2RKISAQdA+ARG+XplGtU+RvLQvJ6MFga8gSfrQA4Zks2JReyxnHUw
ui/BpxRdxJDL43Xa69R4VdcYXifDQlfVomDzEdlTBSuJHI9VhtHLnqUH3rXjBL0X
1GgBCQIQqfgaAeSCRb2AJINKueQe3dVAT8G3CYE588/UsFniV46u3FEO9h0+rG6e
J8xB8+pyiQz2v3Sz6qjeULT2dAJF+9qp4U0wyO2KTmbqwvGrX9od1/5WDkSu7J2I
o2IBbMiyDoMwbw==
=G5eE
-----END PGP MESSAGE-----
fp: 4A8AADB4EBAB9AF88EF7062373CECE06CC80D40C
encrypted_regex: ^(data|stringData)$
version: 3.10.2
---
apiVersion: v1
kind: Secret
metadata:
name: authentik-secret-key
namespace: authentik-system
type: Opaque
stringData:
AUTHENTIK_SECRET_KEY: ENC[AES256_GCM,data:bZgis/HV+zhwFipQNQ95iDOhlU5GGGci0NsIQ/RZxT0Sn68X99R6EDs8mEIoAcegaSI=,iv:UETV43eddyhlFwvOoU/ElPWeTgnRx/azvNYD68lXbP8=,tag:dTzG9/QEmsvyMsfT5vM96A==,type:str]
AUTHENTIK_BOOTSTRAP_PASSWORD: ENC[AES256_GCM,data:U2j1UlFiriiZr7nhidk6hefsQw==,iv:nWT5yIDUDaLhxt7trkYngDL40tK1Muu3zmFX+rT6ubE=,tag:zkPMGT81TAdD40jxw09XfA==,type:str]
AUTHENTIK_BOOTSTRAP_TOKEN: ENC[AES256_GCM,data:Ju1ny+h227iw3213vKHJkPP62AsPnQ2ZSG99BVRHoQoPQr2PsysOJrkq4318RGvucXU=,iv:SIzXaYrfQeZSmmrx9hFOhgC7jkbnSgxatrmz4YZBu64=,tag:ue2ib/bwmlFTha9kdJU6LQ==,type:str]
sops:
lastmodified: "2025-11-24T15:25:26Z"
mac: ENC[AES256_GCM,data:STkCUURHKnPRDuiS5aXfhj8/+at6A4qA4C3te2m+HMzwV7UfB57wK84JZIbF8649yzePxQ6naZQfoBhVOBsyXUvxcQdEEbyimHKfGhInXlXpCt/LTnG4nS51JvVBTLsgT/P/eeX6LKRG3hvoK9cV+jkxyrPfqa3I0Bhr2YBsF5k=,iv:QSebad42NkWP1jRYcu0YuuQbkAi2VTXfVCSyxlomOuo=,tag:aR6R4YJAdJnNFhYRdGaFPQ==,type:str]
pgp:
- created_at: "2025-07-10T13:58:33Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DZT3mpHTS/JgSAQdAnoDla7hPtWEhQmy3KFLW9RkB7qKOAlJVSqO5Sq/lgT4w
nV5zAaOimcbBnT66mJbN59xLUZ67k3RHngtPIjnnmP0iqa4p1VtSwdx1ypUAaIQT
1GgBCQIQ0mnWTxbUiUQvIlcJV3Hx4Ec5XuQNzNlYm5tXQD8Ttx/wLh3N+RdAefW5
mzNK3HbDVB/9IRcoNY8C+L0EiJrjHvQCDgnXKT2oH6wyTpG+m2bwpkRN+wT5d1Xl
gpRfYLm/N8Blcw==
=wn2E
-----END PGP MESSAGE-----
fp: B120595CA9A643B051731B32E67FF350227BA4E8
- created_at: "2025-07-10T13:58:33Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DSXzd60P2RKISAQdA+ARG+XplGtU+RvLQvJ6MFga8gSfrQA4Zks2JReyxnHUw
ui/BpxRdxJDL43Xa69R4VdcYXifDQlfVomDzEdlTBSuJHI9VhtHLnqUH3rXjBL0X
1GgBCQIQqfgaAeSCRb2AJINKueQe3dVAT8G3CYE588/UsFniV46u3FEO9h0+rG6e
J8xB8+pyiQz2v3Sz6qjeULT2dAJF+9qp4U0wyO2KTmbqwvGrX9od1/5WDkSu7J2I
o2IBbMiyDoMwbw==
=G5eE
-----END PGP MESSAGE-----
fp: 4A8AADB4EBAB9AF88EF7062373CECE06CC80D40C
encrypted_regex: ^(data|stringData)$
version: 3.10.2

16
manifests/infrastructure/authentik/storage.yaml Normal file
View File

@@ -0,0 +1,16 @@
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: authentik-media
namespace: authentik-system
labels:
recurring-job.longhorn.io/source: enabled
recurring-job-group.longhorn.io/backup: enabled
spec:
accessModes:
- ReadWriteOnce
storageClassName: longhorn-retain
resources:
requests:
storage: 10Gi

298
manifests/infrastructure/celery-monitoring/DATABASE-CONFIG.md Normal file
View File

@@ -0,0 +1,298 @@
# Auto-Discovery Celery Metrics Exporter
The Celery metrics exporter now **automatically discovers** all Redis databases and their queues without requiring manual configuration. It scans all Redis databases (0-15) and identifies potential Celery queues based on patterns and naming conventions.
## How Auto-Discovery Works
### Automatic Database Scanning
- Scans Redis databases 0-15 by default
- Only monitors databases that contain keys
- Only includes databases that have identifiable queues
### Automatic Queue Discovery
The exporter supports two discovery modes:
#### Smart Filtering Mode (Default: `monitor_all_lists: false`)
Identifies queues using multiple strategies:
1. **Pattern Matching**: Matches known queue patterns from your applications:
- `celery`, `*_priority`, `default`, `mailers`, `push`, `scheduler`
- `streams`, `images`, `suggested_users`, `email`, `connectors`, `lists`, `inbox`, `imports`, `import_triggered`, `misc` (BookWyrm)
- `background`, `send` (PieFed)
- `high`, `mmo` (Pixelfed/Laravel)
2. **Heuristic Detection**: Identifies Redis lists containing queue-related keywords:
- Keys containing: `queue`, `celery`, `task`, `job`, `work`
3. **Type Checking**: Only considers Redis `list` type keys (Celery queues are Redis lists)
#### Monitor Everything Mode (`monitor_all_lists: true`)
- Monitors **ALL** Redis list-type keys in all databases
- No filtering or pattern matching
- Maximum visibility but potentially more noise
- Useful for debugging or comprehensive monitoring
### Which Mode Should You Use?
**Use Smart Filtering (default)** when:
- ✅ You want clean, relevant metrics
- ✅ You care about Prometheus cardinality limits
- ✅ Your applications use standard queue naming
- ✅ You want to avoid monitoring non-queue Redis lists
**Use Monitor Everything** when:
- ✅ You're debugging queue discovery issues
- ✅ You have non-standard queue names not covered by patterns
- ✅ You want absolute certainty you're not missing anything
- ✅ You have sufficient Prometheus storage/performance headroom
- ❌ You don't mind potential noise from non-queue lists
## Configuration (Optional)
While the exporter works completely automatically, you can customize its behavior via the `celery-exporter-config` ConfigMap:
```yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: celery-exporter-config
namespace: celery-monitoring
data:
config.yaml: |
# Auto-discovery settings
auto_discovery:
enabled: true
scan_databases: true # Scan all Redis databases 0-15
scan_queues: true # Auto-discover queues in each database
monitor_all_lists: false # If true, monitor ALL Redis lists, not just queue-like ones
# Queue patterns to look for (Redis list keys that are likely Celery queues)
queue_patterns:
- "celery"
- "*_priority"
- "default"
- "mailers"
- "push"
- "scheduler"
- "broadcast"
- "federation"
- "media"
- "user_dir"
# Optional: Database name mapping (if you want friendly names)
# If not specified, databases will be named "db_0", "db_1", etc.
database_names:
0: "piefed"
1: "mastodon"
2: "matrix"
3: "bookwyrm"
# Minimum queue length to report (avoid noise from empty queues)
min_queue_length: 0
# Maximum number of databases to scan (safety limit)
max_databases: 16
```
## Adding New Applications
**No configuration needed!** New applications are automatically discovered when they:
1. **Use a Redis database** (any database 0-15)
2. **Create queues** that match common patterns or contain queue-related keywords
3. **Use Redis lists** for their queues (standard Celery behavior)
### Custom Queue Patterns
If your application uses non-standard queue names, add them to the `queue_patterns` list:
```bash
kubectl edit configmap celery-exporter-config -n celery-monitoring
```
Add your pattern:
```yaml
queue_patterns:
- "celery"
- "*_priority"
- "my_custom_queue_*" # Add your pattern here
```
### Friendly Database Names
To give databases friendly names instead of `db_0`, `db_1`, etc.:
```yaml
database_names:
0: "piefed"
1: "mastodon"
2: "matrix"
3: "bookwyrm"
4: "my_new_app" # Add your app here
```
## Metrics Produced
The exporter produces these metrics for each discovered database:
### `celery_queue_length`
- **Labels**: `queue_name`, `database`, `db_number`
- **Description**: Number of pending tasks in each queue
- **Example**: `celery_queue_length{queue_name="celery", database="piefed", db_number="0"} 1234`
- **Special**: `queue_name="_total"` shows total tasks across all queues in a database
### `redis_connection_status`
- **Labels**: `database`, `db_number`
- **Description**: Connection status per database (1=connected, 0=disconnected)
- **Example**: `redis_connection_status{database="piefed", db_number="0"} 1`
### `celery_databases_discovered`
- **Description**: Total number of databases with queues discovered
- **Example**: `celery_databases_discovered 4`
### `celery_queues_discovered`
- **Labels**: `database`
- **Description**: Number of queues discovered per database
- **Example**: `celery_queues_discovered{database="bookwyrm"} 5`
### `celery_queue_info`
- **Description**: General information about all monitored queues
- **Includes**: Total lengths, Redis host, last update timestamp, auto-discovery status
## PromQL Query Examples
### Discovery Overview
```promql
# How many databases were discovered
celery_databases_discovered
# How many queues per database
celery_queues_discovered
# Auto-discovery status
celery_queue_info
```
### All Applications Overview
```promql
# All queue lengths grouped by database
sum by (database) (celery_queue_length{queue_name!="_total"})
# Total tasks across all databases
sum(celery_queue_length{queue_name="_total"})
# Individual queues (excluding totals)
celery_queue_length{queue_name!="_total"}
# Only active queues (> 0 tasks)
celery_queue_length{queue_name!="_total"} > 0
```
### Specific Applications
```promql
# PieFed queues only
celery_queue_length{database="piefed", queue_name!="_total"}
# BookWyrm high priority queue (if it exists)
celery_queue_length{database="bookwyrm", queue_name="high_priority"}
# All applications' main celery queue
celery_queue_length{queue_name="celery"}
# Database totals only
celery_queue_length{queue_name="_total"}
```
### Processing Rates
```promql
# Tasks processed per minute (negative = queue decreasing)
rate(celery_queue_length{queue_name!="_total"}[5m]) * -60
# Processing rate by database (using totals)
rate(celery_queue_length{queue_name="_total"}[5m]) * -60
# Overall processing rate across all databases
sum(rate(celery_queue_length{queue_name="_total"}[5m]) * -60)
```
### Health Monitoring
```promql
# Databases with connection issues
redis_connection_status == 0
# Queues growing too fast
increase(celery_queue_length{queue_name!="_total"}[5m]) > 1000
# Stalled processing (no change in 15 minutes)
changes(celery_queue_length{queue_name="_total"}[15m]) == 0 and celery_queue_length{queue_name="_total"} > 100
# Databases that stopped being discovered
changes(celery_databases_discovered[10m]) < 0
```
## Troubleshooting
### Check Auto-Discovery Status
```bash
# View current configuration
kubectl get configmap celery-exporter-config -n celery-monitoring -o yaml
# Check exporter logs for discovery results
kubectl logs -n celery-monitoring deployment/celery-metrics-exporter
# Look for discovery messages like:
# "Database 0 (piefed): 1 queues, 245 total keys"
# "Auto-discovery complete: Found 3 databases with queues"
```
### Test Redis Connectivity
```bash
# Test connection to specific database
kubectl exec -n redis-system redis-master-0 -- redis-cli -a PASSWORD -n DB_NUMBER ping
# Check what keys exist in a database
kubectl exec -n redis-system redis-master-0 -- redis-cli -a PASSWORD -n DB_NUMBER keys '*'
# Check if a key is a list (queue)
kubectl exec -n redis-system redis-master-0 -- redis-cli -a PASSWORD -n DB_NUMBER type QUEUE_NAME
# Check queue length manually
kubectl exec -n redis-system redis-master-0 -- redis-cli -a PASSWORD -n DB_NUMBER llen QUEUE_NAME
```
### Validate Metrics
```bash
# Port forward and check metrics endpoint
kubectl port-forward -n celery-monitoring svc/celery-metrics-exporter 8000:8000
# Check discovery metrics
curl http://localhost:8000/metrics | grep celery_databases_discovered
curl http://localhost:8000/metrics | grep celery_queues_discovered
# Check queue metrics
curl http://localhost:8000/metrics | grep celery_queue_length
```
### Debug Discovery Issues
If queues aren't being discovered:
1. **Check queue patterns** - Add your queue names to `queue_patterns`
2. **Verify queue type** - Ensure queues are Redis lists: `redis-cli type queue_name`
3. **Check database numbers** - Verify your app uses the expected Redis database
4. **Review logs** - Look for discovery debug messages in exporter logs
### Force Restart Discovery
```bash
# Restart the exporter to re-run discovery
kubectl rollout restart deployment/celery-metrics-exporter -n celery-monitoring
```
## Security Notes
- The exporter connects to Redis using the shared `redis-credentials` secret
- All database connections use the same Redis host and password
- Only queue length information is exposed, not queue contents
- The exporter scans all databases but only reports queue-like keys
- Metrics are scraped via ServiceMonitor for OpenTelemetry collection

203
manifests/infrastructure/celery-monitoring/README.md Normal file
View File

@@ -0,0 +1,203 @@
# Celery Monitoring (Flower)
This directory contains the infrastructure for monitoring Celery tasks across all applications in the cluster using Flower.
## Overview
- **Flower**: Web-based tool for monitoring and administrating Celery clusters
- **Multi-Application**: Monitors both PieFed and BookWyrm Celery tasks
- **Namespace**: `celery-monitoring`
- **URL**: `https://flower.keyboardvagabond.com`
## Components
- `namespace.yaml` - Dedicated namespace for monitoring
- `flower-deployment.yaml` - Flower application deployment
- `service.yaml` - Internal service for Flower
- `ingress.yaml` - External access with TLS and basic auth
- `kustomization.yaml` - Kustomize configuration
## Redis Database Monitoring
Flower monitors multiple Redis databases:
- **Database 0**: PieFed Celery broker
- **Database 3**: BookWyrm Celery broker
## Access & Security
- **Access Method**: kubectl port-forward (local access only)
- **Command**: `kubectl port-forward -n celery-monitoring svc/celery-flower 8080:5555`
- **URL**: http://localhost:8080
- **Security**: No authentication required (local access only)
- **Network Policies**: Cilium policies allow cluster and health check access only
### Port-Forward Setup
1. **Prerequisites**:
- Valid kubeconfig with access to the cluster
- kubectl installed and configured
- RBAC permissions to create port-forwards in celery-monitoring namespace
2. **Network Policies**: Cilium policies ensure:
- Port 5555 access from cluster and host (for port-forward)
- Redis access for monitoring (DB 0 & 3)
- Cluster-internal health checks
3. **No Authentication Required**:
- Port-forward provides secure local access
- No additional credentials needed
## **🔒 Simplified Security Architecture**
**Current Status**: ✅ **Local access via kubectl port-forward**
### **Security Model**
**1. Local Access Only**
- **Port-Forward**: `kubectl port-forward` provides secure tunnel to the service
- **No External Exposure**: Service is not accessible from outside the cluster
- **Authentication**: Kubernetes RBAC controls who can create port-forwards
- **Encryption**: Traffic encrypted via Kubernetes API tunnel
**2. Network Layer (Cilium Network Policies)**
- **`celery-flower-ingress`**: Allows cluster and host access for port-forward and health checks
- **`celery-flower-egress`**: Restricts outbound to Redis and DNS only
- **DNS Resolution**: Explicit DNS access for service discovery
- **Redis Connectivity**: Targeted access to Redis master (DB 0 & 3)
**3. Pod-Level Security**
- Resource limits (CPU: 500m, Memory: 256Mi)
- Health checks (liveness/readiness probes)
- Non-root container execution
- Read-only root filesystem (where possible)
### **How It Works**
1. **Access Layer**: kubectl port-forward creates secure tunnel via Kubernetes API
2. **Network Layer**: Cilium policies ensure only cluster traffic reaches pods
3. **Application Layer**: Flower connects only to authorized Redis databases
4. **Monitoring Layer**: Health checks ensure service availability
5. **Local Security**: Access requires valid kubeconfig and RBAC permissions
## Features
- **Flower Web UI**: Real-time task monitoring and worker status
- **Prometheus Metrics**: Custom Celery queue metrics exported to OpenObserve
- **Automated Alerts**: Queue size and connection status monitoring
- **Dashboard**: Visual monitoring of queue trends and processing rates
## Monitoring & Alerts
### Metrics Exported
**From Celery Metrics Exporter** (celery-monitoring namespace):
1. **`celery_queue_length`**: Number of pending tasks in each queue
- Labels: `queue_name`, `database` (piefed/bookwyrm)
2. **`redis_connection_status`**: Redis connectivity status (1=connected, 0=disconnected)
3. **`celery_queue_info`**: General information about queue status
**From Redis Exporter** (redis-system namespace):
4. **`redis_list_length`**: General Redis list lengths including Celery queues
5. **`redis_memory_used_bytes`**: Redis memory usage
6. **`redis_connected_clients`**: Number of connected Redis clients
7. **`redis_commands_total`**: Total Redis commands executed
### Alert Thresholds
- **PieFed Warning**: > 10,000 pending tasks
- **PieFed Critical**: > 50,000 pending tasks
- **BookWyrm Warning**: > 1,000 pending tasks
- **Redis Connection**: Connection lost alert
### OpenObserve Setup
1. **Deploy the monitoring infrastructure**:
```bash
kubectl apply -k manifests/infrastructure/celery-monitoring/
```
2. **Import alerts and dashboard**:
- Access OpenObserve dashboard
- Import alert configurations from the `openobserve-alert-configs` ConfigMap
- Import dashboard from the same ConfigMap
- Configure webhook URLs for notifications
3. **Verify metrics collection**:
```sql
SELECT * FROM metrics WHERE __name__ LIKE 'celery_%' ORDER BY _timestamp DESC LIMIT 10
```
### Useful Monitoring Queries
**Current queue sizes**:
```sql
SELECT queue_name, database, celery_queue_length
FROM metrics
WHERE _timestamp >= now() - interval '5 minutes'
GROUP BY queue_name, database
ORDER BY celery_queue_length DESC
```
**Queue processing rate**:
```sql
SELECT _timestamp,
celery_queue_length - LAG(celery_queue_length, 1) OVER (ORDER BY _timestamp) as processing_rate
FROM metrics
WHERE queue_name='celery' AND database='piefed'
AND _timestamp >= now() - interval '1 hour'
```
- Queue length monitoring
- Task history and details
- Performance metrics
- Multi-broker support
## Dependencies
- Redis (for Celery brokers)
- kubectl (for port-forward access)
- Valid kubeconfig with cluster access
## Testing & Validation
### Quick Access
```bash
# Start port-forward (runs in background)
kubectl port-forward -n celery-monitoring svc/celery-flower 8080:5555 &
# Access Flower UI
open http://localhost:8080
# or visit http://localhost:8080 in your browser
# Stop port-forward when done
pkill -f "kubectl port-forward.*celery-flower"
```
### Manual Testing Checklist
1. **Port-Forward Access**: ✅ Can access http://localhost:8080 after port-forward
2. **No External Access**: ❌ Service not accessible from outside cluster
3. **Redis Connectivity**: 📊 Shows tasks from both PieFed (DB 0) and BookWyrm (DB 3)
4. **Health Checks**: ✅ Pod shows Ready status
5. **Network Policies**: 🛡️ Egress restricted to DNS and Redis only
### Troubleshooting Commands
```bash
# Check Flower pod status
kubectl get pods -n celery-monitoring -l app.kubernetes.io/name=celery-flower
# View Flower logs
kubectl logs -n celery-monitoring -l app.kubernetes.io/name=celery-flower
# Test Redis connectivity
kubectl exec -n celery-monitoring -it deployment/celery-flower -- wget -qO- http://localhost:5555
# Check network policies
kubectl get cnp -n celery-monitoring
# Test port-forward connectivity
kubectl port-forward -n celery-monitoring svc/celery-flower 8080:5555 --dry-run=client
```
## Deployment
Deployed automatically via Flux GitOps from `manifests/cluster/flux-system/celery-monitoring.yaml`.

505
manifests/infrastructure/celery-monitoring/celery-metrics-exporter.yaml Normal file
View File

@@ -0,0 +1,505 @@
---
# Configuration for Celery Metrics Exporter
apiVersion: v1
kind: ConfigMap
metadata:
name: celery-exporter-config
namespace: celery-monitoring
labels:
app.kubernetes.io/name: celery-metrics-exporter
app.kubernetes.io/component: config
data:
config.yaml: |
# Auto-discovery settings
auto_discovery:
enabled: true
scan_databases: false # Only scan known databases, not all 0-15
scan_queues: true # Auto-discover queues in each database
monitor_all_lists: false # If true, monitor ALL Redis lists, not just queue-like ones
use_known_queues: true # Monitor known queues even if they don't exist as lists yet
# Queue patterns to look for (Redis list keys that are likely Celery queues)
queue_patterns:
- "celery"
- "*_priority" # high_priority, medium_priority, low_priority
- "default"
- "mailers"
- "push"
- "scheduler"
- "broadcast"
- "federation"
- "media"
- "user_dir"
# BookWyrm specific queues
- "streams"
- "images"
- "suggested_users"
- "email"
- "connectors"
- "lists"
- "inbox"
- "imports"
- "import_triggered"
- "misc"
# PieFed specific queues
- "background"
- "send"
# Pixelfed/Laravel specific queues
- "high"
- "mmo"
# Common queue patterns
- "*_queue"
- "queue_*"
# Known application configurations (monitored even when queues are empty)
known_applications:
- name: "piefed"
db: 0
queues: ["celery", "background", "send"]
- name: "bookwyrm"
db: 3
queues: ["high_priority", "medium_priority", "low_priority", "streams", "images", "suggested_users", "email", "connectors", "lists", "inbox", "imports", "import_triggered", "broadcast", "misc"]
- name: "mastodon"
db: 1
queues: ["default", "mailers", "push", "scheduler"]
# Optional: Database name mapping (if you want friendly names)
# If not specified, databases will be named "db_0", "db_1", etc.
database_names:
0: "piefed"
1: "mastodon"
2: "matrix"
3: "bookwyrm"
# Minimum queue length to report (avoid noise from empty queues)
min_queue_length: 0
# Maximum number of databases to scan (safety limit)
max_databases: 4
---
# Custom Celery Metrics Exporter Script
apiVersion: v1
kind: ConfigMap
metadata:
name: celery-metrics-script
namespace: celery-monitoring
data:
celery_metrics.py: |
#!/usr/bin/env python3
import redis
import time
import os
import yaml
import fnmatch
from prometheus_client import start_http_server, Gauge, Counter, Info
import logging
# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
# Prometheus metrics
celery_queue_length = Gauge('celery_queue_length', 'Length of Celery queue', ['queue_name', 'database', 'db_number'])
celery_queue_info = Info('celery_queue_info', 'Information about Celery queues')
redis_connection_status = Gauge('redis_connection_status', 'Redis connection status (1=connected, 0=disconnected)', ['database', 'db_number'])
databases_discovered = Gauge('celery_databases_discovered', 'Number of databases with queues discovered')
queues_discovered = Gauge('celery_queues_discovered', 'Total number of queues discovered', ['database'])
# Redis connection
REDIS_HOST = os.getenv('REDIS_HOST', 'redis-ha-haproxy.redis-system.svc.cluster.local')
REDIS_PORT = int(os.getenv('REDIS_PORT', '6379'))
REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', '')
def get_redis_client(db=0):
return redis.Redis(
host=REDIS_HOST,
port=REDIS_PORT,
password=REDIS_PASSWORD,
db=db,
decode_responses=True
)
def load_config():
"""Load configuration from YAML file"""
config_path = '/config/config.yaml'
default_config = {
'auto_discovery': {
'enabled': True,
'scan_databases': True,
'scan_queues': True
},
'queue_patterns': [
'celery',
'*_priority',
'default',
'mailers',
'push',
'scheduler',
'broadcast',
'federation',
'media',
'user_dir'
],
'database_names': {},
'min_queue_length': 0,
'max_databases': 16
}
try:
if os.path.exists(config_path):
with open(config_path, 'r') as f:
config = yaml.safe_load(f)
logger.info("Loaded configuration from file")
return {**default_config, **config}
else:
logger.info("No config file found, using defaults")
return default_config
except Exception as e:
logger.error(f"Error loading config: {e}, using defaults")
return default_config
def discover_queues_in_database(redis_client, db_number, queue_patterns, monitor_all_lists=False):
"""Discover all potential Celery queues in a Redis database"""
try:
# Get all keys in the database
all_keys = redis_client.keys('*')
discovered_queues = []
for key in all_keys:
# Check if key is a list (potential queue)
try:
key_type = redis_client.type(key)
if key_type == 'list':
if monitor_all_lists:
# Monitor ALL Redis lists
discovered_queues.append(key)
else:
# Smart filtering: Check if key matches any of our queue patterns
for pattern in queue_patterns:
if fnmatch.fnmatch(key, pattern):
discovered_queues.append(key)
break
else:
# Also include keys that look like queues (contain common queue words)
queue_indicators = ['queue', 'celery', 'task', 'job', 'work']
if any(indicator in key.lower() for indicator in queue_indicators):
discovered_queues.append(key)
except Exception as e:
logger.debug(f"Error checking key {key} in DB {db_number}: {e}")
continue
# Remove duplicates and sort
discovered_queues = sorted(list(set(discovered_queues)))
if discovered_queues:
mode = "all lists" if monitor_all_lists else "filtered queues"
logger.info(f"DB {db_number}: Discovered {len(discovered_queues)} {mode}: {discovered_queues}")
return discovered_queues
except Exception as e:
logger.error(f"Error discovering queues in DB {db_number}: {e}")
return []
def get_known_applications(config):
"""Get known application configurations"""
return config.get('known_applications', [])
def discover_databases_and_queues(config):
"""Hybrid approach: Use known applications + auto-discovery"""
max_databases = config.get('max_databases', 16)
queue_patterns = config.get('queue_patterns', ['celery', '*_priority'])
database_names = config.get('database_names', {})
monitor_all_lists = config.get('auto_discovery', {}).get('monitor_all_lists', False)
use_known_queues = config.get('auto_discovery', {}).get('use_known_queues', True)
discovered_databases = []
known_apps = get_known_applications(config) if use_known_queues else []
# Track which databases we've already processed from known apps
processed_dbs = set()
# First, add known applications (these are always monitored)
for app_config in known_apps:
db_number = app_config['db']
app_name = app_config['name']
known_queues = app_config['queues']
try:
redis_client = get_redis_client(db_number)
redis_client.ping() # Test connection
# For known apps, we monitor the queues even if they don't exist yet
discovered_databases.append({
'name': app_name,
'db_number': db_number,
'queues': known_queues,
'total_keys': redis_client.dbsize(),
'source': 'known_application'
})
processed_dbs.add(db_number)
logger.info(f"Known app {app_name} (DB {db_number}): {len(known_queues)} configured queues")
except Exception as e:
logger.error(f"Error connecting to known app {app_name} (DB {db_number}): {e}")
continue
# Then, do auto-discovery for remaining databases
for db_number in range(max_databases):
if db_number in processed_dbs:
continue # Skip databases we already processed
try:
redis_client = get_redis_client(db_number)
# Test connection and check if database has any keys
redis_client.ping()
db_size = redis_client.dbsize()
if db_size > 0:
# Discover queues in this database
queues = discover_queues_in_database(redis_client, db_number, queue_patterns, monitor_all_lists)
if queues: # Only include databases that have queues/lists
db_name = database_names.get(db_number, f"db_{db_number}")
discovered_databases.append({
'name': db_name,
'db_number': db_number,
'queues': queues,
'total_keys': db_size,
'source': 'auto_discovery'
})
mode = "lists" if monitor_all_lists else "queues"
logger.info(f"Auto-discovered DB {db_number} ({db_name}): {len(queues)} {mode}, {db_size} total keys")
except redis.ConnectionError:
logger.debug(f"Cannot connect to database {db_number}")
continue
except Exception as e:
logger.debug(f"Error checking database {db_number}: {e}")
continue
known_count = len([db for db in discovered_databases if db.get('source') == 'known_application'])
discovered_count = len([db for db in discovered_databases if db.get('source') == 'auto_discovery'])
logger.info(f"Hybrid discovery complete: {known_count} known applications, {discovered_count} auto-discovered databases")
return discovered_databases
def collect_metrics():
config = load_config()
if not config['auto_discovery']['enabled']:
logger.error("Auto-discovery is disabled in configuration")
return
# Discover databases and queues
databases = discover_databases_and_queues(config)
if not databases:
logger.warning("No databases with queues discovered")
databases_discovered.set(0)
return
databases_discovered.set(len(databases))
queue_info = {}
total_queues = 0
min_queue_length = config.get('min_queue_length', 0)
for db_config in databases:
db_name = db_config['name']
db_number = db_config['db_number']
queues = db_config['queues']
try:
redis_client = get_redis_client(db_number)
# Test connection
redis_client.ping()
redis_connection_status.labels(database=db_name, db_number=str(db_number)).set(1)
total_queue_length = 0
active_queues = 0
for queue_name in queues:
try:
queue_length = redis_client.llen(queue_name)
# Only report queues that meet minimum length threshold
if queue_length >= min_queue_length:
celery_queue_length.labels(
queue_name=queue_name,
database=db_name,
db_number=str(db_number)
).set(queue_length)
total_queue_length += queue_length
if queue_length > 0:
active_queues += 1
logger.info(f"{db_name} (DB {db_number}) {queue_name}: {queue_length} tasks")
except Exception as e:
logger.warning(f"Error checking {db_name} queue {queue_name}: {e}")
# Set total queue length for this database
celery_queue_length.labels(
queue_name='_total',
database=db_name,
db_number=str(db_number)
).set(total_queue_length)
# Track queues discovered per database
queues_discovered.labels(database=db_name).set(len(queues))
queue_info[f'{db_name}_total_length'] = str(total_queue_length)
queue_info[f'{db_name}_active_queues'] = str(active_queues)
queue_info[f'{db_name}_total_queues'] = str(len(queues))
queue_info[f'{db_name}_source'] = db_config.get('source', 'unknown')
total_queues += len(queues)
source_info = f" ({db_config.get('source', 'unknown')})" if 'source' in db_config else ""
if total_queue_length > 0:
logger.info(f"{db_name} (DB {db_number}){source_info}: {total_queue_length} total tasks in {active_queues}/{len(queues)} queues")
except Exception as e:
logger.error(f"Error collecting metrics for {db_name} (DB {db_number}): {e}")
redis_connection_status.labels(database=db_name, db_number=str(db_number)).set(0)
# Update global queue info
queue_info.update({
'redis_host': REDIS_HOST,
'last_update': str(int(time.time())),
'databases_monitored': str(len(databases)),
'total_queues_discovered': str(total_queues),
'auto_discovery_enabled': 'true'
})
celery_queue_info.info(queue_info)
if __name__ == '__main__':
# Start Prometheus metrics server
start_http_server(8000)
logger.info("Celery metrics exporter started on port 8000")
# Collect metrics every 60 seconds
while True:
collect_metrics()
time.sleep(60)
---
# Celery Metrics Exporter Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
name: celery-metrics-exporter
namespace: celery-monitoring
labels:
app.kubernetes.io/name: celery-metrics-exporter
app.kubernetes.io/component: metrics
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: celery-metrics-exporter
app.kubernetes.io/component: metrics
template:
metadata:
labels:
app.kubernetes.io/name: celery-metrics-exporter
app.kubernetes.io/component: metrics
spec:
containers:
- name: celery-metrics-exporter
image: python:3.11-slim
command:
- /bin/sh
- -c
- |
pip install redis prometheus_client pyyaml
python /scripts/celery_metrics.py
ports:
- containerPort: 8000
name: metrics
env:
- name: REDIS_HOST
value: "redis-ha-haproxy.redis-system.svc.cluster.local"
- name: REDIS_PORT
value: "6379"
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: redis-credentials
key: redis-password
volumeMounts:
- name: script
mountPath: /scripts
- name: config
mountPath: /config
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 200m
memory: 256Mi
livenessProbe:
httpGet:
path: /metrics
port: 8000
initialDelaySeconds: 60
periodSeconds: 30
readinessProbe:
httpGet:
path: /metrics
port: 8000
initialDelaySeconds: 30
periodSeconds: 10
volumes:
- name: script
configMap:
name: celery-metrics-script
defaultMode: 0755
- name: config
configMap:
name: celery-exporter-config
---
# Service for Celery Metrics Exporter
apiVersion: v1
kind: Service
metadata:
name: celery-metrics-exporter
namespace: celery-monitoring
labels:
app.kubernetes.io/name: celery-metrics-exporter
app.kubernetes.io/component: metrics
spec:
selector:
app.kubernetes.io/name: celery-metrics-exporter
app.kubernetes.io/component: metrics
ports:
- port: 8000
targetPort: 8000
name: metrics
---
# ServiceMonitor for OpenTelemetry Collection
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: celery-metrics-exporter
namespace: celery-monitoring
labels:
app.kubernetes.io/name: celery-metrics-exporter
app.kubernetes.io/component: metrics
spec:
selector:
matchLabels:
app.kubernetes.io/name: celery-metrics-exporter
app.kubernetes.io/component: metrics
endpoints:
- port: metrics
interval: 60s
path: /metrics

54
manifests/infrastructure/celery-monitoring/flower-deployment.yaml Normal file
View File

@@ -0,0 +1,54 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: celery-flower
namespace: celery-monitoring
labels:
app.kubernetes.io/name: celery-flower
app.kubernetes.io/component: monitoring
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: celery-flower
app.kubernetes.io/component: monitoring
template:
metadata:
labels:
app.kubernetes.io/name: celery-flower
app.kubernetes.io/component: monitoring
spec:
containers:
- name: flower
image: mher/flower:2.0.1
ports:
- containerPort: 5555
env:
- name: CELERY_BROKER_URL
value: "redis://:<REDIS_PASSWORD>@redis-ha-haproxy.redis-system.svc.cluster.local:6379/0"
- name: FLOWER_PORT
value: "5555"
# FLOWER_BASIC_AUTH removed - authentication handled by NGINX Ingress
# This allows Kubernetes health checks to work properly
- name: FLOWER_BROKER_API
value: "redis://:<REDIS_PASSWORD>@redis-ha-haproxy.redis-system.svc.cluster.local:6379/0,redis://:<REDIS_PASSWORD>@redis-ha-haproxy.redis-system.svc.cluster.local:6379/3"
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 500m
memory: 256Mi
livenessProbe:
httpGet:
path: /
port: 5555
initialDelaySeconds: 30
periodSeconds: 30
readinessProbe:
httpGet:
path: /
port: 5555
initialDelaySeconds: 10
periodSeconds: 10

11
manifests/infrastructure/celery-monitoring/kustomization.yaml Normal file
View File

@@ -0,0 +1,11 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- flower-deployment.yaml
- service.yaml
- network-policies.yaml
- redis-secret.yaml
- celery-metrics-exporter.yaml
# - openobserve-alerts.yaml

8
manifests/infrastructure/celery-monitoring/namespace.yaml Normal file
View File

@@ -0,0 +1,8 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: celery-monitoring
labels:
app.kubernetes.io/name: celery-monitoring
app.kubernetes.io/component: infrastructure

47
manifests/infrastructure/celery-monitoring/network-policies.yaml Normal file
View File

@@ -0,0 +1,47 @@
---
# Celery Monitoring Network Policies
# Port-forward and health check access to Flower with proper DNS/Redis connectivity
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: celery-flower-ingress
namespace: celery-monitoring
spec:
description: "Allow ingress to Flower from kubectl port-forward and health checks"
endpointSelector:
matchLabels:
app.kubernetes.io/name: celery-flower
app.kubernetes.io/component: monitoring
ingress:
# Allow kubectl port-forward access (from cluster nodes)
- fromEntities:
- cluster
- host
toPorts:
- ports:
- port: "5555"
protocol: TCP
---
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: celery-flower-egress
namespace: celery-monitoring
spec:
description: "Allow Flower to connect to Redis, DNS, and monitoring services"
endpointSelector:
matchLabels:
app.kubernetes.io/name: celery-flower
app.kubernetes.io/component: monitoring
egress:
# Allow all cluster-internal communication (like PieFed approach)
# This is more permissive but still secure within the cluster
- toEntities:
- cluster
- host
# Service access policy removed - using kubectl port-forward for local access
# Port-forward provides secure access without exposing the service externally

220
manifests/infrastructure/celery-monitoring/openobserve-alerts.yaml Normal file
View File

@@ -0,0 +1,220 @@
# Keeping for reference
# ---
# # OpenObserve Alert Configuration for Celery Queue Monitoring
# # This file contains the alert configurations that should be imported into OpenObserve
# apiVersion: v1
# kind: ConfigMap
# metadata:
# name: openobserve-alert-configs
# namespace: celery-monitoring
# labels:
# app.kubernetes.io/name: openobserve-alerts
# app.kubernetes.io/component: monitoring
# data:
# celery-queue-alerts.json: |
# {
# "alerts": [
# {
# "name": "PieFed Celery Queue High",
# "description": "PieFed Celery queue has more than 10,000 pending tasks",
# "query": "SELECT avg(celery_queue_length) as avg_queue_length FROM metrics WHERE queue_name='celery' AND database='piefed' AND _timestamp >= now() - interval '5 minutes'",
# "condition": "avg_queue_length > 10000",
# "frequency": "5m",
# "severity": "warning",
# "enabled": true,
# "actions": [
# {
# "type": "webhook",
# "webhook_url": "https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK",
# "message": "🚨 PieFed Celery queue is high: {{avg_queue_length}} tasks pending"
# }
# ]
# },
# {
# "name": "PieFed Celery Queue Critical",
# "description": "PieFed Celery queue has more than 50,000 pending tasks",
# "query": "SELECT avg(celery_queue_length) as avg_queue_length FROM metrics WHERE queue_name='celery' AND database='piefed' AND _timestamp >= now() - interval '5 minutes'",
# "condition": "avg_queue_length > 50000",
# "frequency": "2m",
# "severity": "critical",
# "enabled": true,
# "actions": [
# {
# "type": "webhook",
# "webhook_url": "https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK",
# "message": "🔥 CRITICAL: PieFed Celery queue is critically high: {{avg_queue_length}} tasks pending. Consider scaling workers!"
# }
# ]
# },
# {
# "name": "BookWyrm Celery Queue High",
# "description": "BookWyrm Celery queue has more than 1,000 pending tasks",
# "query": "SELECT avg(celery_queue_length) as avg_queue_length FROM metrics WHERE queue_name='total' AND database='bookwyrm' AND _timestamp >= now() - interval '5 minutes'",
# "condition": "avg_queue_length > 1000",
# "frequency": "5m",
# "severity": "warning",
# "enabled": true,
# "actions": [
# {
# "type": "webhook",
# "webhook_url": "https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK",
# "message": "📚 BookWyrm Celery queue is high: {{avg_queue_length}} tasks pending"
# }
# ]
# },
# {
# "name": "Redis Connection Lost",
# "description": "Redis connection is down for Celery monitoring",
# "query": "SELECT avg(redis_connection_status) as connection_status FROM metrics WHERE _timestamp >= now() - interval '2 minutes'",
# "condition": "connection_status < 1",
# "frequency": "1m",
# "severity": "critical",
# "enabled": true,
# "actions": [
# {
# "type": "webhook",
# "webhook_url": "https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK",
# "message": "💥 CRITICAL: Redis connection lost for Celery monitoring!"
# }
# ]
# },
# {
# "name": "Celery Queue Processing Stalled",
# "description": "Celery queue size hasn't decreased in 15 minutes",
# "query": "SELECT celery_queue_length FROM metrics WHERE queue_name='celery' AND database='piefed' AND _timestamp >= now() - interval '15 minutes' ORDER BY _timestamp DESC LIMIT 1",
# "condition": "celery_queue_length > (SELECT celery_queue_length FROM metrics WHERE queue_name='celery' AND database='piefed' AND _timestamp >= now() - interval '20 minutes' AND _timestamp < now() - interval '15 minutes' ORDER BY _timestamp DESC LIMIT 1)",
# "frequency": "10m",
# "severity": "warning",
# "enabled": true,
# "actions": [
# {
# "type": "webhook",
# "webhook_url": "https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK",
# "message": "⚠️ Celery queue processing appears stalled. Queue size hasn't decreased in 15 minutes."
# }
# ]
# }
# ]
# }
# dashboard-config.json: |
# {
# "dashboard": {
# "title": "Celery Queue Monitoring",
# "description": "Monitor Celery queue sizes and processing rates for PieFed and BookWyrm",
# "panels": [
# {
# "title": "PieFed Queue Length",
# "type": "line",
# "query": "SELECT _timestamp, celery_queue_length FROM metrics WHERE queue_name='celery' AND database='piefed' AND _timestamp >= now() - interval '24 hours'",
# "x_axis": "_timestamp",
# "y_axis": "celery_queue_length"
# },
# {
# "title": "BookWyrm Total Queue Length",
# "type": "line",
# "query": "SELECT _timestamp, celery_queue_length FROM metrics WHERE queue_name='total' AND database='bookwyrm' AND _timestamp >= now() - interval '24 hours'",
# "x_axis": "_timestamp",
# "y_axis": "celery_queue_length"
# },
# {
# "title": "Queue Processing Rate (PieFed)",
# "type": "line",
# "query": "SELECT _timestamp, celery_queue_length - LAG(celery_queue_length, 1) OVER (ORDER BY _timestamp) as processing_rate FROM metrics WHERE queue_name='celery' AND database='piefed' AND _timestamp >= now() - interval '6 hours'",
# "x_axis": "_timestamp",
# "y_axis": "processing_rate"
# },
# {
# "title": "Redis Connection Status",
# "type": "stat",
# "query": "SELECT redis_connection_status FROM metrics WHERE _timestamp >= now() - interval '5 minutes' ORDER BY _timestamp DESC LIMIT 1"
# },
# {
# "title": "Current Queue Sizes",
# "type": "table",
# "query": "SELECT queue_name, database, celery_queue_length FROM metrics WHERE _timestamp >= now() - interval '5 minutes' GROUP BY queue_name, database ORDER BY celery_queue_length DESC"
# }
# ]
# }
# }
# ---
# # Instructions ConfigMap
# apiVersion: v1
# kind: ConfigMap
# metadata:
# name: openobserve-setup-instructions
# namespace: celery-monitoring
# data:
# README.md: |
# # OpenObserve Celery Queue Monitoring Setup
# ## 1. Import Alerts
# 1. Access your OpenObserve dashboard
# 2. Go to Alerts → Import
# 3. Copy the contents of `celery-queue-alerts.json` from the `openobserve-alert-configs` ConfigMap
# 4. Paste and import the alert configurations
# ## 2. Configure Webhooks
# Update the webhook URLs in the alert configurations:
# - Replace `https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK` with your actual Slack webhook URL
# - Or configure other notification methods (email, Discord, etc.)
# ## 3. Import Dashboard
# 1. Go to Dashboards → Import
# 2. Copy the contents of `dashboard-config.json` from the `openobserve-alert-configs` ConfigMap
# 3. Paste and import the dashboard configuration
# ## 4. Verify Metrics
# Check that metrics are being collected:
# ```sql
# SELECT * FROM metrics WHERE __name__ LIKE 'celery_%' ORDER BY _timestamp DESC LIMIT 10
# ```
# ## 5. Alert Thresholds
# Current alert thresholds:
# - **PieFed Warning**: > 10,000 tasks
# - **PieFed Critical**: > 50,000 tasks
# - **BookWyrm Warning**: > 1,000 tasks
# - **Redis Connection**: Connection lost
# Adjust these thresholds based on your normal queue sizes and processing capacity.
# ## 6. Monitoring Queries
# Useful queries for monitoring:
# ### Current queue sizes:
# ```sql
# SELECT queue_name, database, celery_queue_length
# FROM metrics
# WHERE _timestamp >= now() - interval '5 minutes'
# GROUP BY queue_name, database
# ORDER BY celery_queue_length DESC
# ```
# ### Queue processing rate (tasks/minute):
# ```sql
# SELECT _timestamp,
# celery_queue_length - LAG(celery_queue_length, 1) OVER (ORDER BY _timestamp) as processing_rate
# FROM metrics
# WHERE queue_name='celery' AND database='piefed'
# AND _timestamp >= now() - interval '1 hour'
# ```
# ### Average queue size over time:
# ```sql
# SELECT DATE_TRUNC('hour', _timestamp) as hour,
# AVG(celery_queue_length) as avg_queue_length
# FROM metrics
# WHERE queue_name='celery' AND database='piefed'
# AND _timestamp >= now() - interval '24 hours'
# GROUP BY hour
# ORDER BY hour
# ```

42
manifests/infrastructure/celery-monitoring/redis-secret.yaml Normal file
View File

@@ -0,0 +1,42 @@
# Redis credentials for Celery monitoring
apiVersion: v1
kind: Secret
metadata:
name: redis-credentials
namespace: celery-monitoring
labels:
app.kubernetes.io/name: celery-monitoring
app.kubernetes.io/component: credentials
type: Opaque
stringData:
redis-password: ENC[AES256_GCM,data:F0QBEefly6IeZzyAU32dTLTV17bFl6TVq1gM3kDfHb4=,iv:Uj47EB6a20YBM4FVKEWBTZv0u9kLrzm2U1YWlwprDkI=,tag:T0ge1nLu1ogUyXCJ9G6m0w==,type:str]
sops:
lastmodified: "2025-08-25T14:29:57Z"
mac: ENC[AES256_GCM,data:S64r234afUX/Lk9TuE7OSCtIlgwD43WXQ78gFJEirGasKY8g27mn1UI16GN79qkS4+i0vg947dVpOkU2jruf897KXK8+672P9ycm4OJQ4uhHaDtKMG3YNPowo8RXFfwQ4v86JzwoUtcmDiK+xjGCTwtrtrU1hal/uN2LXcDZfj0=,iv:hPm8IdI/rBSRCxRNMNCEA/URebgFqQ/ecgcVLX5aQDo=,tag:Otbqwm24GkqNmhpy/drtlA==,type:str]
pgp:
- created_at: "2025-08-23T22:34:52Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DZT3mpHTS/JgSAQdAh9TpU95PiIZoVOgnXqbLZH37oLi2u63YBZUDE5QpBlww
5YNOarjb8tQ03/5jQ4b51USd15rGZBI04JM/V2PXSGRFpF2O7X0WyTw9kELUw2TF
1GgBCQIQ4Df+AQ48lRzu3PoLEwG5sF7p83G4LWXkdfZr9vFz7bpdQ/YzOOUg3TEJ
qoUq93Kbvo98dLIz9MS3qkzuh+E3S56wisziExm95vKinnzgztgIkZ7g6jkLevrK
xf/xvJVj5BVXtw==
=vqkj
-----END PGP MESSAGE-----
fp: B120595CA9A643B051731B32E67FF350227BA4E8
- created_at: "2025-08-23T22:34:52Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DSXzd60P2RKISAQdA2Eq3F3t1myCJVgwXufY3Z0K+Q3Tdzeu47/VoQCrY8kkw
mdtyPKmFwgtqFg8E9VRiZXwBRq3qscOki7yiGozFfGdhFmO0ZK9R/dJGOeLSStfy
1GgBCQIQbfMuXVRt14SVoTMZiHIDGcu5ZBq2iea6HmdeJoLqmweGLF/Vsbrx5pFI
hKyBVDwXE3gf1V03ts4QnbZESCrjNRyg1NsTxIsHPIu64DX6EnW13DNPI6TWZW9i
ni6ecXRfY+gpOw==
=RS4p
-----END PGP MESSAGE-----
fp: 4A8AADB4EBAB9AF88EF7062373CECE06CC80D40C
encrypted_regex: ^(data|stringData)$
version: 3.10.2

17
manifests/infrastructure/celery-monitoring/service.yaml Normal file
View File

@@ -0,0 +1,17 @@
---
apiVersion: v1
kind: Service
metadata:
name: celery-flower
namespace: celery-monitoring
labels:
app.kubernetes.io/name: celery-flower
app.kubernetes.io/component: monitoring
spec:
selector:
app.kubernetes.io/name: celery-flower
app.kubernetes.io/component: monitoring
ports:
- port: 5555
targetPort: 5555
name: http

28
manifests/infrastructure/cert-manager/cert-manager.yaml Normal file
View File

@@ -0,0 +1,28 @@
---
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: jetstack
namespace: cert-manager
spec:
interval: 5m0s
url: https://charts.jetstack.io
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: cert-manager
namespace: cert-manager
spec:
interval: 5m
chart:
spec:
chart: cert-manager
version: "<1.19.2"
sourceRef:
kind: HelmRepository
name: jetstack
namespace: cert-manager
interval: 1m
values:
installCRDs: true

5
manifests/infrastructure/cert-manager/kustomization.yaml Normal file
View File

@@ -0,0 +1,5 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- cert-manager.yaml

5
manifests/infrastructure/cert-manager/namespace.yaml Normal file
View File

@@ -0,0 +1,5 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: cert-manager

6
manifests/infrastructure/cilium/kustomization.yaml Normal file
View File

@@ -0,0 +1,6 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- repository.yaml
- release.yaml

63
manifests/infrastructure/cilium/release.yaml Normal file
View File

@@ -0,0 +1,63 @@
# manifests/infrastructure/cilium/release.yaml
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: cilium
namespace: kube-system
spec:
interval: 5m
chart:
spec:
chart: cilium
version: "1.18.3"
sourceRef:
kind: HelmRepository
name: cilium
namespace: kube-system
interval: 1m
values:
operator:
replicas: 2
ipam:
mode: kubernetes
# Explicitly use VLAN interface for inter-node communication
devices: "enp9s0"
nodePort:
enabled: true
hostFirewall:
enabled: true
hubble:
relay:
enabled: true
ui:
enabled: true
peerService:
clusterDomain: cluster.local
etcd:
clusterDomain: cluster.local
kubeProxyReplacement: true
securityContext:
capabilities:
ciliumAgent:
- CHOWN
- KILL
- NET_ADMIN
- NET_RAW
- IPC_LOCK
- SYS_ADMIN
- SYS_RESOURCE
- DAC_OVERRIDE
- FOWNER
- SETGID
- SETUID
cleanCiliumState:
- NET_ADMIN
- SYS_ADMIN
- SYS_RESOURCE
cgroup:
autoMount:
enabled: true
hostRoot: /sys/fs/cgroup
k8sServiceHost: api.keyboardvagabond.com
k8sServicePort: "6443"

9
manifests/infrastructure/cilium/repository.yaml Normal file
View File

@@ -0,0 +1,9 @@
---
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: cilium
namespace: kube-system
spec:
interval: 5m0s
url: https://helm.cilium.io/

6
manifests/infrastructure/cloudflared/kustomization.yaml Normal file
View File

@@ -0,0 +1,6 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- secret.yaml
- tunnel.yaml

9
manifests/infrastructure/cloudflared/namespace.yaml Normal file
View File

@@ -0,0 +1,9 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: cloudflared-system
labels:
name: cloudflared-system
pod-security.kubernetes.io/enforce: privileged
pod-security.kubernetes.io/enforce-version: latest

38
manifests/infrastructure/cloudflared/secret.yaml Normal file
View File

@@ -0,0 +1,38 @@
apiVersion: v1
kind: Secret
metadata:
name: cloudflared-credentials
namespace: cloudflared-system
type: Opaque
stringData:
tunnel-token: ENC[AES256_GCM,data:V5HpTcyJjVyQoS+BXdGYdUgBgQ+SLnEVBipNCQfX5AwyxsMdABhqikb0ShWw+QSOuGz23zCNSScoqyMnAFphRtzefK6psIQYYUSPeGJp81uldJ3Z+BtD13UjQefcvbKbkrZNYNbunlwsr8V52C3GUtIQaE+izhxnksVbGY1r0+G3y4DKw7vtvqgIYADklviMNe8XAl+MbWSmvI6t7TULgQc6F2bLWpvY1c8I/+hRmT+1cVsCHwZR4g==,iv:bcsFluzuyqHffmAwkVETH0RjzVjZY76+k7QNOrekyJg=,tag:PuE4/MkMiCEGpWjsYqGxqQ==,type:str]
sops:
lastmodified: "2025-11-24T15:25:52Z"
mac: ENC[AES256_GCM,data:oO97YDy+gs7WVndKrvc87yUX4l4Q5XzwooUQ2x2uHrLthbmd8mgAOvcZdpD3f/ne8VKRh6AkP1/AmgtEo9mPBQti+J/n+d+4nBnJQLBbQmsR1UBFgGHyQJgBh388RMbb75f8WTKxvQJeB9PVwVn+qFA6MXoZkFi80taA8bzTK1U=,iv:ZgcUMyd8gCNNc8UGBslx6MfZ+E0yYwd365En89MAHiQ=,tag:Jd08bmsFyQ5fINTXXt6dEw==,type:str]
pgp:
- created_at: "2025-11-24T15:25:52Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DZT3mpHTS/JgSAQdA6Q7ykZebfxuwWPlpg2PqyJfy9N/SN2Lit3bW4GwrCnww
oC2D08YgIbh49qkztTe7SAXrOgT2i9wseDjz9Pz2Qe6UtjvHLL7aXpHaBf2Mqmnj
1GYBCQIQaXHTJ3mbQEIppdw03rS8RPbbfbS6cvd7NMN6AQPxOVNRCUbMa0+Co0Df
UL+kwPCEO9Q4Vp7QJvIk7lNdCCT0s9rmN9UgYDlNFuT+SJfmyHFoOdAvKz/ruPyc
wzCqX1Q55vg=
=a3kv
-----END PGP MESSAGE-----
fp: B120595CA9A643B051731B32E67FF350227BA4E8
- created_at: "2025-11-24T15:25:52Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DSXzd60P2RKISAQdAp3ac25mat2oNFay7tSu81DG3klr3FaYBbryAX37Neykw
9Z5qBfgkyrqsOB71a6R6L3HcZ1JOxxZQddn4UyVp2tAwgPOnoFtIyz8jXht/vClF
1GYBCQIQGxM7v4toIcZw/dLKJOMfal3pvjbWq3p73Z7oTnkRjLuTDiXHWxYiz+eg
MSC7pnS0NTMvAeAPs6yNs5darIciaXsi7sIJxPxWiuME/1DnkTbdJFuWlbcU++tC
BjLgmmJ0zgo=
=+jRj
-----END PGP MESSAGE-----
fp: 4A8AADB4EBAB9AF88EF7062373CECE06CC80D40C
encrypted_regex: ^(data|stringData)$
version: 3.10.2

56
manifests/infrastructure/cloudflared/tunnel.yaml Normal file
View File

@@ -0,0 +1,56 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: cloudflared-deployment
namespace: cloudflared-system
spec:
replicas: 2
selector:
matchLabels:
pod: cloudflared
template:
metadata:
labels:
pod: cloudflared
spec:
securityContext:
sysctls:
# Allows ICMP traffic (ping, traceroute) to resources behind cloudflared.
- name: net.ipv4.ping_group_range
value: "65532 65532"
containers:
- image: cloudflare/cloudflared:latest
name: cloudflared
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 256Mi
env:
# Defines an environment variable for the tunnel token.
- name: TUNNEL_TOKEN
valueFrom:
secretKeyRef:
name: cloudflared-credentials
key: tunnel-token
command:
# Configures tunnel run parameters
- cloudflared
- tunnel
- --no-autoupdate
- --loglevel
- debug
- --metrics
- 0.0.0.0:2000
- run
livenessProbe:
httpGet:
# Cloudflared has a /ready endpoint which returns 200 if and only if
# it has an active connection to Cloudflare's network.
path: /ready
port: 2000
failureThreshold: 1
initialDelaySeconds: 10
periodSeconds: 10

31
manifests/infrastructure/cluster-issuers/cluster-issuers.yaml Normal file
View File

@@ -0,0 +1,31 @@
# manifests/infrastructure/cluster-issuers/cluster-issuers.yaml
---
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
name: letsencrypt-staging
spec:
acme:
server: https://acme-staging-v02.api.letsencrypt.org/directory
email: <EMAIL>
privateKeySecretRef:
name: letsencrypt-staging
solvers:
- http01:
ingress:
class: nginx
---
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
name: letsencrypt-production
spec:
acme:
server: https://acme-v02.api.letsencrypt.org/directory
email: <EMAIL>
privateKeySecretRef:
name: letsencrypt-production
solvers:
- http01:
ingress:
class: nginx

4
manifests/infrastructure/cluster-issuers/kustomization.yaml Normal file
View File

@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- cluster-issuers.yaml

59
manifests/infrastructure/cluster-policies/harbor-registry-firewall.yaml Normal file
View File

@@ -0,0 +1,59 @@
# Harbor Registry Firewall Rules for Direct Access
apiVersion: "cilium.io/v2"
kind: CiliumClusterwideNetworkPolicy
metadata:
name: "harbor-registry-host-firewall"
spec:
description: "Allow external access to ports 80/443 only for NGINX Ingress serving Harbor"
# Target NGINX Ingress Controller pods specifically (they use hostNetwork)
endpointSelector:
matchLabels:
app.kubernetes.io/name: "ingress-nginx"
app.kubernetes.io/component: "controller"
ingress:
# Allow external traffic to NGINX Ingress on HTTP/HTTPS ports
- fromEntities:
- world
- cluster
toPorts:
- ports:
- port: "80"
protocol: "TCP"
- port: "443"
protocol: "TCP"
# Allow cluster-internal traffic to NGINX Ingress
- fromEntities:
- cluster
toPorts:
- ports:
- port: "80"
protocol: "TCP"
- port: "443"
protocol: "TCP"
- port: "10254" # NGINX metrics port
protocol: "TCP"
---
# Allow NGINX Ingress to reach Harbor services
apiVersion: "cilium.io/v2"
kind: CiliumNetworkPolicy
metadata:
name: "harbor-services-access"
namespace: "harbor-registry"
spec:
description: "Allow NGINX Ingress Controller to reach Harbor services"
endpointSelector:
matchLabels:
app: "harbor"
ingress:
# Allow traffic from NGINX Ingress Controller
- fromEndpoints:
- matchLabels:
app.kubernetes.io/name: "ingress-nginx"
app.kubernetes.io/component: "controller"
# Allow traffic between Harbor components
- fromEndpoints:
- matchLabels:
app: "harbor"

262
manifests/infrastructure/cluster-policies/host-fw-control-plane.yaml Normal file
View File

@@ -0,0 +1,262 @@
# policies/host-fw-control-plane.yaml
apiVersion: "cilium.io/v2"
kind: CiliumClusterwideNetworkPolicy
metadata:
name: "host-fw-control-plane"
spec:
description: "control-plane specific access rules. Restricted to Tailscale network for security."
nodeSelector:
matchLabels:
node-role.kubernetes.io/control-plane: ""
ingress:
# Allow access to kube api from Tailscale network, VLAN, VIP, and external IPs
# VIP (<VIP_IP>) allows new nodes to bootstrap via VLAN without network changes
- fromCIDR:
- 100.64.0.0/10 # Tailscale CGNAT range
- 10.132.0.0/24 # VLAN subnet (includes VIP <VIP_IP> and node IPs)
- <VIP_IP>/32 # Explicit VIP for control plane (new node bootstrapping)
- <NODE_1_EXTERNAL_IP>/32 # n1 external IP
- <NODE_2_EXTERNAL_IP>/32 # n2 external IP
- <NODE_3_EXTERNAL_IP>/32 # n3 external IP
- fromEntities:
- cluster # Allow cluster-internal access
toPorts:
- ports:
- port: "6443"
protocol: "TCP"
# Allow access to talos from Tailscale network, VLAN, VIP, external IPs, and cluster
# Restricted access (not world) for security - authentication still required
# https://www.talos.dev/v1.4/learn-more/talos-network-connectivity/
- fromCIDR:
- 100.64.0.0/10 # Tailscale CGNAT range
- 10.132.0.0/24 # VLAN subnet for node bootstrapping
- <VIP_IP>/32 # VIP for control plane access
- <NODE_1_EXTERNAL_IP>/32 # n1 external IP
- <NODE_2_EXTERNAL_IP>/32 # n2 external IP
- <NODE_3_EXTERNAL_IP>/32 # n3 external IP
- fromEntities:
- cluster # Allow cluster-internal access
toPorts:
- ports:
- port: "50000"
protocol: "TCP"
- port: "50001"
protocol: "TCP"
# Allow worker nodes to access control plane Talos API
- fromEntities:
- remote-node
toPorts:
- ports:
- port: "50000"
protocol: "TCP"
- port: "50001"
protocol: "TCP"
# Allow kube-proxy-replacement from kube-apiserver
- fromEntities:
- kube-apiserver
toPorts:
- ports:
- port: "10250"
protocol: "TCP"
- port: "4244"
protocol: "TCP"
# Allow access from hubble-relay to hubble-peer (running on the node)
- fromEndpoints:
- matchLabels:
k8s-app: hubble-relay
toPorts:
- ports:
- port: "4244"
protocol: "TCP"
# Allow metrics-server to scrape
- fromEndpoints:
- matchLabels:
k8s-app: metrics-server
toPorts:
- ports:
- port: "10250"
protocol: "TCP"
# Allow ICMP Ping from/to anywhere.
- icmps:
- fields:
- type: 8
family: IPv4
- type: 128
family: IPv6
# Allow cilium tunnel/health checks from other nodes.
- fromEntities:
- remote-node
toPorts:
- ports:
- port: "8472"
protocol: "UDP"
- port: "4240"
protocol: "TCP"
# Allow etcd communication between control plane nodes
# Required for etcd cluster formation and peer communication
# Ports: 2379 (client API), 2380 (peer communication), 51871 (Talos etcd peer discovery)
- fromCIDR:
- 100.64.0.0/10 # Tailscale CGNAT range
- 10.132.0.0/24 # VLAN subnet (includes VIP <VIP_IP> and node IPs)
- <VIP_IP>/32 # Explicit VIP for control plane (new node bootstrapping)
- <NODE_1_EXTERNAL_IP>/32 # n1 external IP
- <NODE_2_EXTERNAL_IP>/32 # n2 external IP
- <NODE_3_EXTERNAL_IP>/32 # n3 external IP
- fromEntities:
- remote-node # Allow from other nodes (including bootstrapping control planes)
- cluster # Allow from cluster pods
toPorts:
- ports:
- port: "2379"
protocol: "TCP" # etcd client API
- port: "2380"
protocol: "TCP" # etcd peer communication
- port: "51871"
protocol: "UDP" # Talos etcd peer discovery
# HTTP and HTTPS access - allow external for Harbor direct access and Let's Encrypt challenges
# everything else is secured and I really hate this
- fromEntities:
- cluster
- world # Allow external access for Harbor and Let's Encrypt
- fromCIDR:
- 100.64.0.0/10 # Tailscale CGNAT range - allow Tailscale services (e.g., Kibana proxy)
toPorts:
- ports:
- port: "80"
protocol: "TCP"
- port: "443"
protocol: "TCP"
# Allow access from inside the cluster to the admission controller
- fromEntities:
- cluster
toPorts:
- ports:
- port: "8443"
protocol: "TCP"
# Allow PostgreSQL and Redis database connections from cluster
- fromEntities:
- cluster
toPorts:
- ports:
- port: "5432"
protocol: "TCP" # PostgreSQL
- port: "6379"
protocol: "TCP" # Redis
# Allow PostgreSQL monitoring/health checks and CloudNativePG coordination
- fromEntities:
- cluster
toPorts:
- ports:
- port: "9187"
protocol: "TCP" # PostgreSQL metrics port
- port: "8000"
protocol: "TCP" # CloudNativePG health endpoint
- port: "9443"
protocol: "TCP" # CloudNativePG operator webhook server
# Allow local kubelet health checks on control plane pods
# (kubelet on control plane needs to check health endpoints of local pods)
- fromEntities:
- host
toPorts:
- ports:
- port: "8000"
protocol: "TCP" # CloudNativePG health endpoint for kubelet probes
# OpenObserve and metrics collection ports
- fromEntities:
- cluster
toPorts:
- ports:
- port: "5080"
protocol: "TCP" # OpenObserve
- port: "10254"
protocol: "TCP" # NGINX Ingress metrics
egress:
# Allow all cluster communication (pods, services, nodes)
- toEntities:
- cluster
- remote-node
- host
# Allow etcd communication to other control plane nodes
# Required for etcd cluster formation and peer communication
- toCIDR:
- 10.132.0.0/24 # VLAN subnet (all control plane nodes)
- <VIP_IP>/32 # VIP
- toEntities:
- remote-node # Allow to other nodes
toPorts:
- ports:
- port: "2379"
protocol: "TCP" # etcd client API
- port: "2380"
protocol: "TCP" # etcd peer communication
- port: "51871"
protocol: "UDP" # Talos etcd peer discovery
# Allow control plane to reach CloudNativePG health endpoints on all nodes
- toEntities:
- cluster
- remote-node
- host
toPorts:
- ports:
- port: "8000"
protocol: "TCP" # CloudNativePG health endpoint
# Allow control plane to reach PostgreSQL databases on worker nodes
- toEntities:
- cluster
- remote-node
toPorts:
- ports:
- port: "5432"
protocol: "TCP" # PostgreSQL database
- port: "9187"
protocol: "TCP" # PostgreSQL metrics
- port: "8000"
protocol: "TCP" # CloudNativePG health endpoint (correct port)
- port: "8080"
protocol: "TCP" # Additional health/admin endpoints
- port: "9443"
protocol: "TCP" # CloudNativePG operator webhook server
# Allow DNS resolution
- toEntities:
- cluster
- remote-node
toPorts:
- ports:
- port: "53"
protocol: "TCP"
- port: "53"
protocol: "UDP"
# Allow outbound internet access for backup operations, image pulls, etc.
- toEntities:
- world
toPorts:
- ports:
- port: "443"
protocol: "TCP" # HTTPS
- port: "80"
protocol: "TCP" # HTTP
- port: "53"
protocol: "UDP" # DNS
- port: "123"
protocol: "UDP" # NTP time synchronization

199
manifests/infrastructure/cluster-policies/host-fw-worker-nodes.yaml Normal file
View File

@@ -0,0 +1,199 @@
# policies/host-fw-worker-nodes.yaml
apiVersion: "cilium.io/v2"
kind: CiliumClusterwideNetworkPolicy
metadata:
name: "host-fw-worker-nodes"
spec:
description: "Worker node firewall rules - more permissive for database workloads"
nodeSelector:
matchExpressions:
- key: node-role.kubernetes.io/control-plane
operator: DoesNotExist
ingress:
# Allow all cluster communication for database operations
- fromEntities:
- cluster
- remote-node
- host
# Allow PostgreSQL and Redis connections from anywhere in cluster
- fromEntities:
- cluster
toPorts:
- ports:
- port: "5432"
protocol: "TCP" # PostgreSQL
- port: "6379"
protocol: "TCP" # Redis
# Allow health check and monitoring ports
- fromEntities:
- cluster
toPorts:
- ports:
- port: "8000"
protocol: "TCP" # CloudNativePG health endpoint
- port: "8080"
protocol: "TCP"
- port: "9187"
protocol: "TCP" # PostgreSQL metrics
- port: "9443"
protocol: "TCP" # CloudNativePG operator webhook server
- port: "10250"
protocol: "TCP" # kubelet
# Allow kubelet access from VLAN for cluster operations
- fromCIDR:
- 10.132.0.0/24 # VLAN subnet
toPorts:
- ports:
- port: "10250"
protocol: "TCP" # kubelet API
# HTTP and HTTPS access - allow from cluster and Tailscale network
# Tailscale network needed for Tailscale operator proxy pods (e.g., Kibana via MagicDNS)
- fromEntities:
- cluster
- fromCIDR:
- 100.64.0.0/10 # Tailscale CGNAT range - allow Tailscale services
toPorts:
- ports:
- port: "80"
protocol: "TCP"
- port: "443"
protocol: "TCP"
# Allow access to Talos API from Tailscale network, VLAN, and external IPs
# Restricted access (not world) for security - authentication still required
- fromCIDR:
- 100.64.0.0/10 # Tailscale CGNAT range
- 10.132.0.0/24 # VLAN subnet for node bootstrapping
- <NODE_1_EXTERNAL_IP>/32 # n1 external IP
- <NODE_2_EXTERNAL_IP>/32 # n2 external IP
- <NODE_3_EXTERNAL_IP>/32 # n3 external IP
- fromEntities:
- cluster # Allow cluster-internal access
toPorts:
- ports:
- port: "50000"
protocol: "TCP"
- port: "50001"
protocol: "TCP"
# Allow ICMP Ping
- icmps:
- fields:
- type: 8
family: IPv4
- type: 128
family: IPv6
# Allow cilium tunnel/health checks
- fromEntities:
- remote-node
toPorts:
- ports:
- port: "8472"
protocol: "UDP"
- port: "4240"
protocol: "TCP"
# Allow hubble communication
- fromEndpoints:
- matchLabels:
k8s-app: hubble-relay
toPorts:
- ports:
- port: "4244"
protocol: "TCP"
# NGINX Ingress Controller metrics port
- fromEntities:
- cluster
toPorts:
- ports:
- port: "10254"
protocol: "TCP" # NGINX Ingress metrics
# OpenObserve metrics ingestion port
- fromEntities:
- cluster
toPorts:
- ports:
- port: "5080"
protocol: "TCP" # OpenObserve HTTP API
# Additional monitoring ports (removed unused Prometheus/Grafana ports)
# Note: OpenObserve is used instead of Prometheus/Grafana stack
egress:
# Allow all cluster communication (pods, services, nodes) - essential for CloudNativePG
- toEntities:
- cluster
- remote-node
- host
# Allow worker nodes to reach control plane services
- toEntities:
- cluster
- remote-node
toPorts:
- ports:
- port: "6443"
protocol: "TCP" # Kubernetes API server
- port: "8000"
protocol: "TCP" # CloudNativePG health endpoints
- port: "9443"
protocol: "TCP" # CloudNativePG operator webhook
- port: "5432"
protocol: "TCP" # PostgreSQL replication
- port: "9187"
protocol: "TCP" # PostgreSQL metrics
# Allow access to control plane via VLAN for node bootstrapping
# Explicit VIP access ensures new nodes can reach kubeapi without network changes
- toCIDR:
- 10.132.0.0/24 # VLAN subnet for cluster bootstrapping (includes VIP)
- <VIP_IP>/32 # Explicit VIP for control plane kubeapi
- <NODE_1_IP>/32 # n1 VLAN IP (fallback)
toPorts:
- ports:
- port: "6443"
protocol: "TCP" # Kubernetes API server
- port: "50000"
protocol: "TCP" # Talos API
- port: "50001"
protocol: "TCP" # Talos API trustd
# Allow DNS resolution
- toEndpoints:
- matchLabels:
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: "UDP"
- port: "53"
protocol: "TCP"
# Allow worker nodes to reach external services (OpenObserve, monitoring)
- toEntities:
- cluster
toPorts:
- ports:
- port: "5080"
protocol: "TCP" # OpenObserve
# Allow outbound internet access for NTP, image pulls, etc.
- toEntities:
- world
toPorts:
- ports:
- port: "443"
protocol: "TCP" # HTTPS
- port: "80"
protocol: "TCP" # HTTP
- port: "53"
protocol: "UDP" # DNS
- port: "123"
protocol: "UDP" # NTP time synchronization

68
manifests/infrastructure/cluster-policies/kubelet-rbac-fix.yaml Normal file
View File

@@ -0,0 +1,68 @@
---
# Fix for apiserver-kubelet-client RBAC permissions
# Required when adding new control plane nodes to Talos clusters
# This ensures the kubelet can access node/pods subresource for static pod management
#
# The system:kubelet-api-admin ClusterRole should already exist in Kubernetes,
# but we ensure the ClusterRoleBinding exists and has the correct permissions.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:apiserver-kubelet-client
annotations:
description: "Grants apiserver-kubelet-client permission to access nodes and pods for kubelet operations"
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:kubelet-api-admin
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: system:apiserver-kubelet-client
---
# Ensure the ClusterRole has nodes/pods subresource permission
# This may need to be created if it doesn't exist or updated if missing nodes/pods
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:kubelet-api-admin
labels:
kubernetes.io/bootstrapping: rbac-defaults
rules:
- apiGroups:
- ""
resources:
- nodes
- nodes/proxy
- nodes/stats
- nodes/log
- nodes/spec
- nodes/metrics
- nodes/pods # CRITICAL: Required for kubelet to get pod status on nodes
verbs:
- get
- list
- watch
- create
- patch
- update
- delete
- apiGroups:
- ""
resources:
- pods
- pods/status
- pods/log
- pods/exec
- pods/portforward
- pods/proxy
verbs:
- get
- list
- watch
- create
- patch
- update
- delete

7
manifests/infrastructure/cluster-policies/kustomization.yaml Normal file
View File

@@ -0,0 +1,7 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- host-fw-control-plane.yaml
- host-fw-worker-nodes.yaml
- harbor-registry-firewall.yaml

261
manifests/infrastructure/elasticsearch/README.md Normal file
View File

@@ -0,0 +1,261 @@
# Elasticsearch Infrastructure
This directory contains the Elasticsearch setup using ECK (Elastic Cloud on Kubernetes) operator for full-text search on the Kubernetes cluster.
## Architecture
- **ECK Operator**: Production-grade Elasticsearch deployment on Kubernetes
- **Single-node cluster**: Optimized for your 2-node cluster (can be scaled later)
- **Security enabled**: X-Pack security with custom role and user for Mastodon
- **Longhorn storage**: Distributed storage with 2-replica redundancy
- **Self-signed certificates**: Internal cluster communication with TLS
## Components
### **Core Components**
- `namespace.yaml`: Elasticsearch system namespace
- `repository.yaml`: Elastic Helm repository
- `operator.yaml`: ECK operator deployment
- Uses existing `longhorn-retain` storage class with backup labels on PVCs
- `cluster.yaml`: Elasticsearch and Kibana cluster configuration
### **Security Components**
- `secret.yaml`: SOPS-encrypted credentials for Elasticsearch admin and Mastodon user
- `security-setup.yaml`: Job to create Mastodon role and user after cluster deployment
### **Monitoring Components**
- `monitoring.yaml`: ServiceMonitor for OpenObserve integration + optional Kibana ingress
- Built-in metrics: Elasticsearch Prometheus exporter
## Services Created
ECK automatically creates these services:
- `elasticsearch-es-http`: HTTPS API access (port 9200)
- `elasticsearch-es-transport`: Internal cluster transport (port 9300)
- `kibana-kb-http`: Kibana web UI (port 5601) - optional management interface
## Connection Information
### For Applications (Mastodon)
Applications should connect using these connection parameters:
**Elasticsearch Connection:**
```yaml
host: elasticsearch-es-http.elasticsearch-system.svc.cluster.local
port: 9200
scheme: https # ECK uses HTTPS with self-signed certificates
user: mastodon
password: <password from elasticsearch-credentials secret>
```
### Getting Credentials
The Elasticsearch credentials are stored in SOPS-encrypted secrets:
```bash
# Get the admin password (auto-generated by ECK)
kubectl get secret elasticsearch-es-elastic-user -n elasticsearch-system -o jsonpath="{.data.elastic}" | base64 -d
# Get the Mastodon user password (set during security setup)
kubectl get secret elasticsearch-credentials -n elasticsearch-system -o jsonpath="{.data.password}" | base64 -d
```
## Deployment Steps
### 1. Encrypt Secrets
Before deploying, encrypt the secrets with SOPS:
```bash
# Edit and encrypt the Elasticsearch credentials
sops manifests/infrastructure/elasticsearch/secret.yaml
# Edit and encrypt the Mastodon Elasticsearch credentials
sops manifests/applications/mastodon/elasticsearch-secret.yaml
```
### 2. Deploy Infrastructure
The infrastructure will be deployed automatically by Flux when you commit:
```bash
git add manifests/infrastructure/elasticsearch/
git add manifests/cluster/flux-system/elasticsearch.yaml
git add manifests/cluster/flux-system/kustomization.yaml
git commit -m "Add Elasticsearch infrastructure for Mastodon search"
git push
```
### 3. Wait for Deployment
```bash
# Monitor ECK operator deployment
kubectl get pods -n elasticsearch-system -w
# Monitor Elasticsearch cluster startup
kubectl get elasticsearch -n elasticsearch-system -w
# Check cluster health
kubectl get elasticsearch elasticsearch -n elasticsearch-system -o yaml
```
### 4. Verify Security Setup
```bash
# Check if security setup job completed successfully
kubectl get jobs -n elasticsearch-system
# Verify Mastodon user was created
kubectl logs -n elasticsearch-system job/elasticsearch-security-setup
```
### 5. Update Mastodon
After Elasticsearch is running, deploy the updated Mastodon configuration:
```bash
git add manifests/applications/mastodon/
git commit -m "Enable Elasticsearch in Mastodon"
git push
```
### 6. Populate Search Indices
Once Mastodon is running with Elasticsearch enabled, populate the search indices:
```bash
# Get a Mastodon web pod
MASTODON_POD=$(kubectl get pods -n mastodon-application -l app.kubernetes.io/component=web -o jsonpath='{.items[0].metadata.name}')
# Run the search deployment command
kubectl exec -n mastodon-application $MASTODON_POD -- bin/tootctl search deploy
```
## Configuration Details
### Elasticsearch Configuration
- **Version**: 7.17.27 (latest 7.x compatible with Mastodon)
- **Preset**: `single_node_cluster` (optimized for single-node deployment)
- **Memory**: 2GB heap size (50% of 4GB container limit)
- **Storage**: 50GB persistent volume with existing `longhorn-retain` storage class
- **Security**: X-Pack security enabled with custom roles
### Security Configuration
Following the [Mastodon Elasticsearch documentation](https://docs.joinmastodon.org/admin/elasticsearch/), the setup includes:
- **Custom Role**: `mastodon_full_access` with minimal required permissions
- **Dedicated User**: `mastodon` with the custom role
- **TLS Encryption**: All connections use HTTPS with self-signed certificates
### Performance Configuration
- **JVM Settings**: Optimized for your cluster's resource constraints
- **Discovery**: Single-node discovery (can be changed for multi-node scaling)
- **Memory**: Conservative settings for 2-node cluster compatibility
- **Storage**: Optimized for SSD performance with proper disk watermarks
## Mastodon Integration
### Search Features Enabled
Once configured, Mastodon will provide full-text search for:
- Public statuses from accounts that opted into search results
- User's own statuses
- User's mentions, favourites, and bookmarks
- Account information (display names, usernames, bios)
### Search Index Deployment
The `tootctl search deploy` command will create these indices:
- `accounts_index`: User accounts and profiles
- `statuses_index`: User's own statuses, mentions, favourites, bookmarks
- `public_statuses_index`: Public searchable content
- `tags_index`: Hashtag search
## Monitoring Integration
### OpenObserve Metrics
Elasticsearch metrics are automatically collected and sent to OpenObserve:
- **Cluster Health**: Node status, cluster state, allocation
- **Performance**: Query latency, indexing rate, search performance
- **Storage**: Disk usage, index sizes, shard distribution
- **JVM**: Memory usage, garbage collection, heap statistics
### Kibana Management UI
Optional Kibana web interface available at `https://kibana.keyboardvagabond.com` for:
- Index management and monitoring
- Query development and testing
- Cluster configuration and troubleshooting
- Visual dashboards for Elasticsearch data
## Scaling Considerations
### Current Setup
- **Single-node cluster**: Optimized for current 2-node Kubernetes cluster
- **50GB storage**: Sufficient for small-to-medium Mastodon instances
- **2GB heap**: Conservative memory allocation
### Future Scaling
When adding more Kubernetes nodes:
1. Update `discovery.type` from `single-node` to `zen` in cluster configuration
2. Increase `nodeSets.count` to 2 or 3 for high availability
3. Change `ES_PRESET` to `small_cluster` in Mastodon configuration
4. Consider increasing storage and memory allocations
## Troubleshooting
### Common Issues
**Elasticsearch pods pending:**
- Check storage class and PVC creation
- Verify Longhorn is healthy and has available space
**Security setup job failing:**
- Check Elasticsearch cluster health
- Verify admin credentials are available
- Review job logs for API errors
**Mastodon search not working:**
- Verify Elasticsearch credentials in Mastodon secret
- Check network connectivity between namespaces
- Ensure search indices are created with `tootctl search deploy`
### Useful Commands
```bash
# Check Elasticsearch cluster status
kubectl get elasticsearch -n elasticsearch-system
# View Elasticsearch logs
kubectl logs -n elasticsearch-system -l elasticsearch.k8s.elastic.co/cluster-name=elasticsearch
# Check security setup
kubectl describe job elasticsearch-security-setup -n elasticsearch-system
# Test connectivity from Mastodon
kubectl exec -n mastodon-application deployment/mastodon-web -- curl -k https://elasticsearch-es-http.elasticsearch-system.svc.cluster.local:9200/_cluster/health
```
## Backup Integration
### S3 Backup Strategy
- **Longhorn Integration**: Elasticsearch volumes are automatically backed up to Backblaze B2
- **Volume Labels**: `backup.longhorn.io/enable: "true"` enables automatic S3 backup
- **Backup Frequency**: Follows existing Longhorn backup schedule
### Index Backup
For additional protection, consider periodic index snapshots:
```bash
# Create snapshot repository (one-time setup)
curl -k -u "mastodon:$ES_PASSWORD" -X PUT "https://elasticsearch-es-http.elasticsearch-system.svc.cluster.local:9200/_snapshot/s3_repository" -H 'Content-Type: application/json' -d'
{
"type": "s3",
"settings": {
"bucket": "longhorn-backup-bucket",
"region": "eu-central-003",
"endpoint": "<REPLACE_WITH_S3_ENDPOINT>"
}
}'
# Create manual snapshot
curl -k -u "mastodon:$ES_PASSWORD" -X PUT "https://elasticsearch-es-http.elasticsearch-system.svc.cluster.local:9200/_snapshot/s3_repository/snapshot_1"
```

149
manifests/infrastructure/elasticsearch/cluster.yaml Normal file
View File

@@ -0,0 +1,149 @@
---
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
name: elasticsearch
namespace: elasticsearch-system
labels:
app: elasticsearch
backup.longhorn.io/enable: "true" # Enable Longhorn S3 backup
spec:
version: 7.17.27 # Latest 7.x version compatible with Mastodon
# Single-node cluster (can be scaled later)
nodeSets:
- name: default
count: 1
config:
# Node configuration
node.store.allow_mmap: false # Required for containers
# Performance optimizations for 2-node cluster (similar to PostgreSQL)
cluster.routing.allocation.disk.threshold_enabled: true
cluster.routing.allocation.disk.watermark.low: "85%"
cluster.routing.allocation.disk.watermark.high: "90%"
cluster.routing.allocation.disk.watermark.flood_stage: "95%"
# Memory and performance settings
indices.memory.index_buffer_size: "20%"
indices.memory.min_index_buffer_size: "48mb"
indices.fielddata.cache.size: "30%"
indices.queries.cache.size: "20%"
# ECK manages discovery configuration automatically for single-node clusters
# Security settings - ECK manages TLS automatically
xpack.security.enabled: true
# Pod template for Elasticsearch nodes
podTemplate:
metadata:
labels:
app: elasticsearch
spec:
# Node selection and affinity - Prefer n2 but allow n1 if needed
nodeSelector: {}
tolerations: []
affinity:
nodeAffinity:
# PREFERRED: Prefer n2 for optimal distribution, but allow n1 if needed
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: kubernetes.io/hostname
operator: In
values: ["n2"]
# Resource configuration - Optimized for resource-constrained environment
containers:
- name: elasticsearch
resources:
requests:
cpu: 500m # 0.5 CPU core
memory: 2Gi # 2GB RAM (increased from 1Gi)
limits:
cpu: 1000m # Max 1 CPU core
memory: 4Gi # Max 4GB RAM (increased from 2Gi)
env:
# JVM heap size - should be 50% of container memory limit
- name: ES_JAVA_OPTS
value: "-Xms2g -Xmx2g"
# Security context - ECK manages this automatically
securityContext: {}
# Volume claim templates
volumeClaimTemplates:
- metadata:
name: elasticsearch-data
labels:
backup.longhorn.io/enable: "true" # Enable S3 backup
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 50Gi
storageClassName: longhorn-retain
# HTTP configuration
http:
service:
spec:
type: ClusterIP
selector:
elasticsearch.k8s.elastic.co/cluster-name: "elasticsearch"
tls:
selfSignedCertificate:
disabled: true # Disable TLS for internal Kubernetes communication
# Transport configuration
transport:
service:
spec:
type: ClusterIP
---
# Kibana deployment for optional web UI management
apiVersion: kibana.k8s.elastic.co/v1
kind: Kibana
metadata:
name: kibana
namespace: elasticsearch-system
spec:
version: 7.17.27
count: 1
elasticsearchRef:
name: elasticsearch
config:
server.publicBaseUrl: "https://kibana.keyboardvagabond.com"
podTemplate:
metadata:
labels:
app: kibana
spec:
containers:
- name: kibana
resources:
requests:
cpu: 50m # Reduced from 200m - actual usage ~26m
memory: 384Mi # Reduced from 1Gi - actual usage ~274MB
limits:
cpu: 400m # Reduced from 1000m but adequate for log analysis
memory: 768Mi # Reduced from 2Gi but adequate for dashboards
securityContext: {}
http:
service:
metadata:
annotations:
tailscale.com/hostname: kibana
spec:
type: LoadBalancer
loadBalancerClass: tailscale
tls:
selfSignedCertificate:
disabled: false

21
manifests/infrastructure/elasticsearch/kustomization.yaml Normal file
View File

@@ -0,0 +1,21 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: elasticsearch-system
resources:
- namespace.yaml
- repository.yaml
- operator.yaml
- cluster.yaml
- secret.yaml
- security-setup.yaml
- monitoring.yaml
# Apply resources in order
# 1. Namespace and repository first
# 2. Storage class and operator
# 3. Cluster configuration
# 4. Security setup (job runs after cluster is ready)
# 5. Monitoring and ingress

67
manifests/infrastructure/elasticsearch/monitoring.yaml Normal file
View File

@@ -0,0 +1,67 @@
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: elasticsearch-metrics
namespace: elasticsearch-system
labels:
app: elasticsearch
spec:
selector:
matchLabels:
elasticsearch.k8s.elastic.co/cluster-name: elasticsearch
endpoints:
- port: https
path: /_prometheus/metrics
scheme: https
tlsConfig:
insecureSkipVerify: true # Use self-signed certs
basicAuth:
username:
name: elasticsearch-es-elastic-user
key: elastic
password:
name: elasticsearch-es-elastic-user
key: elastic
interval: 30s
scrapeTimeout: 10s
namespaceSelector:
matchNames:
- elasticsearch-system
---
# Optional: Kibana ServiceMonitor if you want to monitor Kibana as well
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: kibana-metrics
namespace: elasticsearch-system
labels:
app: kibana
spec:
selector:
matchLabels:
kibana.k8s.elastic.co/name: kibana
endpoints:
- port: https
path: /api/status
scheme: https
tlsConfig:
insecureSkipVerify: true
basicAuth:
username:
name: elasticsearch-es-elastic-user
key: elastic
password:
name: elasticsearch-es-elastic-user
key: elastic
interval: 60s
scrapeTimeout: 30s
namespaceSelector:
matchNames:
- elasticsearch-system
---
# Note: Kibana is exposed via Tailscale LoadBalancer service (configured in cluster.yaml)
# No Ingress needed - the service type LoadBalancer with loadBalancerClass: tailscale
# automatically creates a Tailscale proxy pod and exposes the service via MagicDNS

8
manifests/infrastructure/elasticsearch/namespace.yaml Normal file
View File

@@ -0,0 +1,8 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: elasticsearch-system
labels:
name: elasticsearch-system
backup.longhorn.io/enable: "true" # Enable Longhorn S3 backup

55
manifests/infrastructure/elasticsearch/operator.yaml Normal file
View File

@@ -0,0 +1,55 @@
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: eck-operator
namespace: elasticsearch-system
spec:
interval: 5m
timeout: 10m
chart:
spec:
chart: eck-operator
version: "2.16.1" # Latest stable version
sourceRef:
kind: HelmRepository
name: elastic
namespace: elasticsearch-system
interval: 1m
values:
# ECK Operator Configuration
installCRDs: true
# Resource limits for operator - optimized based on actual usage
resources:
requests:
cpu: 25m # Reduced from 100m - actual usage ~4m
memory: 128Mi # Reduced from 150Mi - actual usage ~81MB
limits:
cpu: 200m # Reduced from 1000m but still adequate for operator tasks
memory: 256Mi # Reduced from 512Mi but still adequate
# Node selection for operator
nodeSelector: {}
tolerations: []
# Security configuration
podSecurityContext:
runAsNonRoot: true
# Webhook configuration
webhook:
enabled: true
# Metrics
metrics:
port: 0 # Disable metrics endpoint for now
# Logging
config:
logVerbosity: 0
metricsPort: 0
# Additional volumes/mounts if needed
extraVolumes: []
extraVolumeMounts: []

9
manifests/infrastructure/elasticsearch/repository.yaml Normal file
View File

@@ -0,0 +1,9 @@
---
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: elastic
namespace: elasticsearch-system
spec:
interval: 24h
url: https://helm.elastic.co

45
manifests/infrastructure/elasticsearch/secret.yaml Normal file
View File

@@ -0,0 +1,45 @@
apiVersion: v1
kind: Secret
metadata:
name: elasticsearch-credentials
namespace: elasticsearch-system
type: Opaque
stringData:
#ENC[AES256_GCM,data:xbndkZj3CeTZN5MphjUAxKiQbYIAAV0GuPmueWw7JwPk5fk6KpG/8FGrG00=,iv:0FV6SB6Ng+kaE66uVdDlx8Tv/3LAHCjuoWObi2mpUbU=,tag:1vLYGHl2WHvRVGz1bAqYFw==,type:comment]
#ENC[AES256_GCM,data:Jg3rWRjashFNg+0fEc7nELrCrCVTUOuCly2bYpMjiELrqxz7Xr5NzR4xiIByw/Ra9k6KC3AIliqprRq6zg==,iv:Iin+CpprebHEWq6JwmGYKdwraxuMIgJBODyLcL0/SGo=,tag:xzJgp/dyR7lfTlOHLySWHg==,type:comment]
username: ENC[AES256_GCM,data:PKlxhJfU4CY=,iv:9Bsw4V+yjWquFB4O9o3WxPMkAgOacsHrNf5DVNaU5hM=,tag:a9fyeD52Q/9amVeZ4U1Rzg==,type:str]
password: ENC[AES256_GCM,data:AsYI0SYTPCzxCxBfrk/aNSqKiBg+pXXxG0Ao0kshsO//WjKkCohBbSM54/oesjEylZk=,iv:skXOKX9ZshzJF3e+zJKGL67XT5rgTIfetUbobY/SSH0=,tag:08SrG9iAtGLzc/Ie9LK+/Q==,type:str]
#ENC[AES256_GCM,data:2r1sPMzdY0Pm00UNo+PD56tSm3p0SFzOclIfisaubHzG4xfDzffyO6fBGbqXJHvARkRzp+8ZWuaSWnQQae9O2EjyTlO0xt9U,iv:KXzBL1VFnj7cYXuhcPXSxS5LUYOGkUT301VLkyCPxsI=,tag:wv5XuHZMSV3FQqzMrTEQlg==,type:comment]
#ENC[AES256_GCM,data:V/09hOJMrROOeg9Jicj+PA1JowWmwabb5BsRvUcrJabcyJQ8Alm+QIyjK86zLVnz,iv:9qO//4Nf0Bb5a4VmFUZBx6QEP1dhCipHpv3GmKm7YkA=,tag:HYwPfqQwJTF8gGVoTUNi5Q==,type:comment]
admin-username: ENC[AES256_GCM,data:tLJw1egNQQ==,iv:7VvP+EdNIMB3dfIOa9xR+RYtUg+MJhJHrhux0Vy3BME=,tag:Av5j8jBG7vo4Si1oqphLAg==,type:str]
admin-password: ENC[AES256_GCM,data:2wOb7lAY+T92s/zYFr0ladWDFePyMZ/r,iv:CRK5FIbmG+SFtbPqvaUKi/W3HTAR+zn/C2DtU55J/7E=,tag:1TULM84wl8mkUU9FPg0Zkw==,type:str]
sops:
lastmodified: "2025-11-30T09:38:26Z"
mac: ENC[AES256_GCM,data:eY+5GdSvqXhbK+5HTmru9ItqZ3ivBls+6twWswhd3CnYtem3D++SyXxZlGuV9C8RPoiIUddl8XDNJBB6F+wC9MmbvokigYP3GsqCem2V1pvLpP5B0bMMO4y8JeyRVmXkTVIkA+syBDgPz3D05GSA0n9BNxh303Dmvv0EtCJ7pbI=,iv:H1pT3DnQmjqp7Pp6KHTHdj5etAx08IO1i+mjpvoQLcE=,tag:6thUf1j7bgQEfBzifni1nA==,type:str]
pgp:
- created_at: "2025-11-27T09:39:43Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DZT3mpHTS/JgSAQdAXiRkqvjErdtK7Mx1NbAHLYiybYUmto2yThAGLvCpzHcw
8b8b3RO6b9WQwYdtn6Ld3ghcXBhR/eUu8RX5TZwDL3uw4+sinRWzBYeMU2llFnwb
1GgBCQIQbKSPq4uVXVgUPEAmISfla/qePymV8eABHa3rRwYwnVsj5fez6bFoLfOz
wJfSDSrRDUmZT/rTLvHi3GXTfnaOYbg0aScf3SCbxaMf2K4zGTyPXwQUnRFUn9KI
yXvR8SRAC0SG3g==
=KCYR
-----END PGP MESSAGE-----
fp: B120595CA9A643B051731B32E67FF350227BA4E8
- created_at: "2025-11-27T09:39:43Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DSXzd60P2RKISAQdAZGa0E49mmUHnjAStIf6zY0n5lQJ7Zr+DRZkd7cIP5V0w
+fWI4RcQ3rfzZljfP9stegszFwL7MMuRes0PeDxT+zk3HAvOnJIocBoM96P48Ckm
1GgBCQIQA4kzGLnFD/pPsofvMjDXP2G+bGrvxBRgHG/vRpsTCI6tiOEd3VeSR9qe
DtaudhgKbbAfWSj9cKHULRkxrQoLHjoeIlN4V/4tRxYp3Mxj4t5myaZqxUY1+Kmc
IaU4qoz4LQAZ0Q==
=0MwX
-----END PGP MESSAGE-----
fp: 4A8AADB4EBAB9AF88EF7062373CECE06CC80D40C
encrypted_regex: ^(data|stringData)$
version: 3.10.2

88
manifests/infrastructure/elasticsearch/security-setup.yaml Normal file
View File

@@ -0,0 +1,88 @@
---
apiVersion: batch/v1
kind: Job
metadata:
name: elasticsearch-security-setup
namespace: elasticsearch-system
annotations:
# Run this job after Elasticsearch is ready
"helm.sh/hook": post-install,post-upgrade
"helm.sh/hook-weight": "10"
"helm.sh/hook-delete-policy": before-hook-creation
spec:
template:
metadata:
labels:
app: elasticsearch-security-setup
spec:
restartPolicy: Never
initContainers:
# Wait for Elasticsearch to be ready
- name: wait-for-elasticsearch
image: curlimages/curl:8.10.1
command:
- /bin/sh
- -c
- |
echo "Waiting for Elasticsearch to be ready..."
until curl -u "elastic:${ELASTIC_PASSWORD}" "http://elasticsearch-es-http:9200/_cluster/health?wait_for_status=yellow&timeout=300s"; do
echo "Elasticsearch not ready yet, sleeping..."
sleep 10
done
echo "Elasticsearch is ready!"
env:
- name: ELASTIC_PASSWORD
valueFrom:
secretKeyRef:
name: elasticsearch-es-elastic-user
key: elastic
containers:
- name: setup-security
image: curlimages/curl:8.10.1
command:
- /bin/sh
- -c
- |
echo "Setting up Elasticsearch security for Mastodon..."
# Create mastodon_full_access role
echo "Creating mastodon_full_access role..."
curl -X POST -u "elastic:${ELASTIC_PASSWORD}" \
"http://elasticsearch-es-http:9200/_security/role/mastodon_full_access" \
-H 'Content-Type: application/json' \
-d '{
"cluster": ["monitor"],
"indices": [{
"names": ["*"],
"privileges": ["read", "monitor", "write", "manage"]
}]
}'
echo "Role creation response: $?"
# Create mastodon user
echo "Creating mastodon user..."
curl -X POST -u "elastic:${ELASTIC_PASSWORD}" \
"http://elasticsearch-es-http:9200/_security/user/mastodon" \
-H 'Content-Type: application/json' \
-d '{
"password": "'"${MASTODON_PASSWORD}"'",
"roles": ["mastodon_full_access"]
}'
echo "User creation response: $?"
echo "Security setup completed!"
env:
- name: ELASTIC_PASSWORD
valueFrom:
secretKeyRef:
name: elasticsearch-es-elastic-user
key: elastic
- name: MASTODON_PASSWORD
valueFrom:
secretKeyRef:
name: elasticsearch-credentials
key: password
securityContext: {}
nodeSelector: {}
tolerations: []

147
manifests/infrastructure/harbor-registry/README.md Normal file
View File

@@ -0,0 +1,147 @@
# Harbor Registry with External PostgreSQL and Redis
This configuration sets up Harbor container registry to use your existing PostgreSQL and Redis infrastructure instead of embedded databases.
## Architecture
- **PostgreSQL**: Uses `harborRegistry` user and `harbor` database created during PostgreSQL cluster initialization
- **Redis**: Uses existing Redis primary-replica setup (database 0)
- **Storage**: Longhorn persistent volumes for Harbor registry data
- **Ingress**: NGINX ingress with Let's Encrypt certificates
## Database Integration
### PostgreSQL Setup
Harbor database and user are created declaratively during PostgreSQL cluster initialization using CloudNativePG's `postInitApplicationSQL` feature:
- **Database**: `harbor` (owned by `shared_user`)
- **User**: `harborRegistry` (with full permissions on harbor database)
- **Connection**: `postgresql-shared-rw.postgresql-system.svc.cluster.local:5432`
### Redis Setup
Harbor connects to your existing Redis infrastructure:
- **Primary**: `redis-ha-haproxy.redis-system.svc.cluster.local:6379`
- **Database**: `0` (default Redis database)
- **Authentication**: Uses password from `redis-credentials` secret
## Files Overview
- `harbor-database-credentials.yaml`: Harbor's database and Redis passwords (encrypt with SOPS before deployment)
- `harbor-registry.yaml`: Main Harbor Helm release with external database configuration
- `manual-ingress.yaml`: Ingress configuration for Harbor web UI
## Deployment Steps
### 1. Deploy PostgreSQL Changes
⚠️ **WARNING**: This will recreate the PostgreSQL cluster to add Harbor database creation.
```bash
kubectl apply -k manifests/infrastructure/postgresql/
```
### 2. Wait for PostgreSQL
```bash
kubectl get cluster -n postgresql-system -w
kubectl get pods -n postgresql-system -w
```
### 3. Deploy Harbor
```bash
kubectl apply -k manifests/infrastructure/harbor-registry/
```
### 4. Monitor Deployment
```bash
kubectl get pods,svc,ingress -n harbor-registry -w
```
## Verification
### Check Database
```bash
# Connect to PostgreSQL
kubectl exec -it postgresql-shared-1 -n postgresql-system -- psql -U postgres
# Check harbor database and user
\l harbor
\du "harborRegistry"
\c harbor
\dt
```
### Check Harbor
```bash
# Check Harbor pods
kubectl get pods -n harbor-registry
# Check Harbor logs
kubectl logs -f deployment/harbor-registry-core -n harbor-registry
# Access Harbor UI
open https://<YOUR_REGISTRY_URL>
```
## Configuration Details
### External Database Configuration
```yaml
postgresql:
enabled: false # Disable embedded PostgreSQL
externalDatabase:
host: "postgresql-shared-rw.postgresql-system.svc.cluster.local"
port: 5432
user: "harborRegistry"
database: "harbor"
existingSecret: "harbor-database-credentials"
existingSecretPasswordKey: "harbor-db-password"
sslmode: "disable" # Internal cluster communication
```
### External Redis Configuration
```yaml
redis:
enabled: false # Disable embedded Redis
externalRedis:
addr: "redis-ha-haproxy.redis-system.svc.cluster.local:6379"
db: "0"
existingSecret: "harbor-database-credentials"
existingSecretPasswordKey: "redis-password"
```
## Benefits
1. **Resource Efficiency**: No duplicate database instances
2. **Consistency**: Single source of truth for database configuration
3. **Backup Integration**: Harbor data included in existing PostgreSQL backup strategy
4. **Monitoring**: Harbor database metrics included in existing PostgreSQL monitoring
5. **Declarative Setup**: Database creation handled by PostgreSQL initialization
## Troubleshooting
### Database Connection Issues
```bash
# Test PostgreSQL connectivity
kubectl run test-pg --rm -it --image=postgres:16 -- psql -h postgresql-shared-rw.postgresql-system.svc.cluster.local -U harborRegistry -d harbor
# Check Harbor database credentials
kubectl get secret harbor-database-credentials -n harbor-registry -o yaml
```
### Redis Connection Issues
```bash
# Test Redis connectivity
kubectl run test-redis --rm -it --image=redis:7 -- redis-cli -h redis-ha-haproxy.redis-system.svc.cluster.local -a "$(kubectl get secret redis-credentials -n redis-system -o jsonpath='{.data.redis-password}' | base64 -d)"
```
### Harbor Logs
```bash
# Core service logs
kubectl logs -f deployment/harbor-registry-core -n harbor-registry
# Registry logs
kubectl logs -f deployment/harbor-registry-registry -n harbor-registry
# Job service logs
kubectl logs -f deployment/harbor-registry-jobservice -n harbor-registry
```

75
manifests/infrastructure/harbor-registry/coredns-harbor.yaml Normal file
View File

@@ -0,0 +1,75 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: coredns-harbor
namespace: kube-system
data:
Corefile: |
keyboardvagabond.com:53 {
hosts {
<NODE_1_IP> <YOUR_REGISTRY_URL>
<NODE_2_IP> <YOUR_REGISTRY_URL>
<NODE_3_IP> <YOUR_REGISTRY_URL>
fallthrough
}
log
errors
}
. {
forward . /etc/resolv.conf
cache 30
loadbalance
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: coredns-harbor
namespace: kube-system
spec:
replicas: 2
selector:
matchLabels:
k8s-app: coredns-harbor
template:
metadata:
labels:
k8s-app: coredns-harbor
spec:
containers:
- name: coredns
image: coredns/coredns:1.11.1
args: ["-conf", "/etc/coredns/Corefile"]
volumeMounts:
- name: config-volume
mountPath: /etc/coredns
ports:
- containerPort: 53
name: dns-udp
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
volumes:
- name: config-volume
configMap:
name: coredns-harbor
---
apiVersion: v1
kind: Service
metadata:
name: coredns-harbor
namespace: kube-system
spec:
selector:
k8s-app: coredns-harbor
clusterIP: 10.96.0.53
ports:
- name: dns-udp
port: 53
protocol: UDP
targetPort: 53
- name: dns-tcp
port: 53
protocol: TCP
targetPort: 53

156
manifests/infrastructure/harbor-registry/harbor-registry.yaml Normal file
View File

@@ -0,0 +1,156 @@
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: harbor-registry
namespace: harbor-registry
spec:
type: oci
interval: 5m0s
url: oci://registry-1.docker.io/bitnamicharts
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: harbor-registry
namespace: harbor-registry
spec:
interval: 5m
chart:
spec:
chart: harbor
version: "27.0.3"
sourceRef:
kind: HelmRepository
name: harbor-registry
namespace: harbor-registry
interval: 1m
values:
clusterDomain: cluster.local
externalURL: https://<YOUR_REGISTRY_URL>
adminPassword: Harbor12345
# Global ingress configuration
global:
ingressClassName: nginx
default:
storageClass: longhorn-single-delete
# Use current Bitnami registry (not legacy)
imageRegistry: "docker.io"
# Use embedded databases (PostgreSQL and Redis sub-charts)
# NOTE: Chart 27.0.3 uses Debian-based images - override PostgreSQL tag since default doesn't exist
postgresql:
enabled: true
# Override PostgreSQL image tag - default 17.5.0-debian-12-r20 doesn't exist
# Use bitnamilegacy repository where Debian images were moved
image:
repository: bitnamilegacy/postgresql
# Enable S3 backup for Harbor PostgreSQL database (daily + weekly)
persistence:
labels:
recurring-job.longhorn.io/source: "enabled"
recurring-job-group.longhorn.io/longhorn-s3-backup: "enabled"
recurring-job-group.longhorn.io/longhorn-s3-backup-weekly: "enabled"
redis:
enabled: true
image:
repository: bitnamilegacy/redis
# Disable external services globally
commonLabels:
app.kubernetes.io/managed-by: Helm
persistence:
persistentVolumeClaim:
registry:
size: 50Gi
storageClass: longhorn-single-delete
jobservice:
size: 10Gi
storageClass: longhorn-single-delete
# NOTE: Chart 27.0.3 still uses Debian-based images (legacy)
# Bitnami Secure Images use Photon Linux, but chart hasn't been updated yet
# Keeping Debian tags for now - these work but are in bitnamilegacy repository
# TODO: Update to Photon-based images when chart is updated
core:
image:
repository: bitnamilegacy/harbor-core
updateStrategy:
type: Recreate
# Keep Debian-based tag for now (chart default)
# Override only if needed - chart defaults to: 2.13.2-debian-12-r3
# image:
# registry: docker.io
# repository: bitnami/harbor-core
# tag: "2.13.2-debian-12-r3"
configMap:
EXTERNAL_URL: https://<YOUR_REGISTRY_URL>
WITH_CLAIR: "false"
WITH_TRIVY: "false"
WITH_NOTARY: "false"
# Optimize resources - Harbor usage is deployment-dependent, not user-dependent
resources:
requests:
cpu: 50m # Reduced from 500m - actual usage ~3m
memory: 128Mi # Reduced from 512Mi - actual usage ~76Mi
limits:
cpu: 200m # Conservative limit for occasional builds
memory: 256Mi # Conservative limit
portal:
# Use bitnamilegacy repository for Debian-based images
image:
repository: bitnamilegacy/harbor-portal
jobservice:
updateStrategy:
type: Recreate
# Use bitnamilegacy repository for Debian-based images
image:
repository: bitnamilegacy/harbor-jobservice
# Optimize resources - job service has minimal usage
resources:
requests:
cpu: 25m # Reduced from 500m - actual usage ~5m
memory: 64Mi # Reduced from 512Mi - actual usage ~29Mi
limits:
cpu: 100m # Conservative limit
memory: 128Mi # Conservative limit
registry:
updateStrategy:
type: Recreate
# Use bitnamilegacy repository for Debian-based images
server:
image:
repository: bitnamilegacy/harbor-registry
controller:
image:
repository: bitnamilegacy/harbor-registryctl
# Optimize resources - registry has minimal usage
resources:
requests:
cpu: 25m # Reduced from 500m - actual usage ~1m
memory: 64Mi # Reduced from 512Mi - actual usage ~46Mi
limits:
cpu: 100m # Conservative limit for image pushes/pulls
memory: 128Mi # Conservative limit
nginx:
# Bitnami-specific service override
service:
type: ClusterIP
# Use bitnamilegacy repository for Debian-based images
image:
repository: bitnamilegacy/nginx
notary:
server:
updateStrategy:
type: Recreate
signer:
updateStrategy:
type: Recreate
trivy:
image:
repository: bitnamilegacy/harbor-adapter-trivy
ingress:
enabled: false
service:
type: ClusterIP
ports:
http: 80
https: 443

6
manifests/infrastructure/harbor-registry/kustomization.yaml Normal file
View File

@@ -0,0 +1,6 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- harbor-registry.yaml
- manual-ingress.yaml

34
manifests/infrastructure/harbor-registry/manual-ingress.yaml Normal file
View File

@@ -0,0 +1,34 @@
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: harbor-registry-ingress
namespace: harbor-registry
annotations:
cert-manager.io/cluster-issuer: letsencrypt-production
# Harbor-specific settings
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
# SSL and redirect handling
nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
nginx.ingress.kubernetes.io/ssl-redirect: "false"
nginx.ingress.kubernetes.io/proxy-ssl-verify: "false"
spec:
ingressClassName: nginx
tls:
- hosts:
- <YOUR_REGISTRY_URL>
secretName: <YOUR_REGISTRY_URL>-tls
rules:
- host: <YOUR_REGISTRY_URL>
http:
paths:
# Harbor - route to HTTPS service to avoid internal redirects
- path: /
pathType: Prefix
backend:
service:
name: harbor-registry
port:
number: 443

5
manifests/infrastructure/harbor-registry/namespace.yaml Normal file
View File

@@ -0,0 +1,5 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: harbor-registry

73
manifests/infrastructure/ingress-nginx/ingress-nginx.yaml Normal file
View File

@@ -0,0 +1,73 @@
---
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: ingress-nginx
namespace: ingress-nginx
spec:
interval: 5m0s
url: https://kubernetes.github.io/ingress-nginx
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: ingress-nginx
namespace: ingress-nginx
spec:
interval: 5m
chart:
spec:
chart: ingress-nginx
version: ">=v4.12.0 <4.13.0"
sourceRef:
kind: HelmRepository
name: ingress-nginx
namespace: ingress-nginx
interval: 1m
values:
controller:
hostNetwork: true
hostPort:
enabled: true
kind: DaemonSet
service:
enabled: true
admissionWebhooks:
enabled: false
metrics:
enabled: true
serviceMonitor:
enabled: true
additionalLabels: {}
podAnnotations:
prometheus.io/scrape: "true"
prometheus.io/port: "10254"
ingressClassResource:
name: nginx
enabled: true
default: true
controllerValue: "k8s.io/ingress-nginx"
ingressClass: nginx
config:
use-forwarded-headers: "true"
compute-full-forwarded-for: "true"
use-proxy-protocol: "false"
ssl-redirect: "false"
force-ssl-redirect: "false"
# Cloudflare Real IP Configuration
# Trust CF-Connecting-IP header from Cloudflare IP ranges
proxy-real-ip-cidr: "103.21.244.0/22,103.22.200.0/22,103.31.4.0/22,104.16.0.0/12,108.162.192.0/18,131.0.72.0/22,141.101.64.0/18,162.158.0.0/15,172.64.0.0/13,173.245.48.0/20,188.114.96.0/20,190.93.240.0/20,197.234.240.0/22,198.41.128.0/17,199.27.128.0/21,2400:cb00::/32,2606:4700::/32,2803:f800::/32,2405:b500::/32,2405:8100::/32,2c0f:f248::/32,2a06:98c0::/29"
real-ip-header: "CF-Connecting-IP"
---
apiVersion: v1
kind: ConfigMap
metadata:
labels:
app: ingress-nginx
name: nginx-ingress-configuration
namespace: ingress-nginx
data:
ssl-redirect: "false"
hsts: "true"
server-tokens: "false"

5
manifests/infrastructure/ingress-nginx/kustomization.yaml Normal file
View File

@@ -0,0 +1,5 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- ingress-nginx.yaml

8
manifests/infrastructure/ingress-nginx/namespace.yaml Normal file
View File

@@ -0,0 +1,8 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: ingress-nginx
labels:
pod-security.kubernetes.io/enforce: privileged
pod-security.kubernetes.io/enforce-version: latest

277
manifests/infrastructure/longhorn/S3-API-OPTIMIZATION.md Normal file
View File

@@ -0,0 +1,277 @@
# Longhorn S3 API Call Optimization - Implementation Summary
## Problem Statement
Longhorn was making **145,000+ Class C API calls/day** to Backblaze B2, primarily `s3_list_objects` operations. This exceeded Backblaze's free tier (2,500 calls/day) and incurred significant costs.
### Root Cause
Even with `backupstore-poll-interval` set to `0`, Longhorn manager pods continuously poll the S3 backup target to check for new backups. With 3 manager pods (one per node) polling independently, this resulted in excessive API calls.
Reference: [Longhorn GitHub Issue #1547](https://github.com/longhorn/longhorn/issues/1547)
## Solution: NetworkPolicy-Based Access Control
Inspired by [this community solution](https://github.com/longhorn/longhorn/issues/1547#issuecomment-3395447100), we implemented **time-based network access control** using Kubernetes NetworkPolicies and CronJobs.
### Architecture
```
┌─────────────────────────────────────────────────┐
│ Normal State (21 hours/day) │
│ NetworkPolicy BLOCKS S3 access │
│ → Longhorn polls fail at network layer │
│ → S3 API calls: 0 │
└─────────────────────────────────────────────────┘
▼
┌─────────────────────────────────────────────────┐
│ Backup Window (3 hours/day: 1-4 AM) │
│ CronJob REMOVES NetworkPolicy at 12:55 AM │
│ → S3 access enabled │
│ → Recurring backups run automatically │
│ → CronJob RESTORES NetworkPolicy at 4:00 AM │
│ → S3 API calls: ~5,000-10,000/day │
└─────────────────────────────────────────────────┘
```
### Components
1. **NetworkPolicy** (`longhorn-block-s3-access`) - **Dynamically Managed**
- Targets: `app=longhorn-manager` pods
- Blocks: All egress except DNS and intra-cluster
- Effect: Prevents S3 API calls at network layer
- **Important**: NOT managed by Flux - only the CronJobs control it
- Flux manages the CronJobs/RBAC, but NOT the NetworkPolicy itself
2. **CronJob: Enable S3 Access** (`longhorn-enable-s3-access`)
- Schedule: `55 0 * * *` (12:55 AM daily)
- Action: Deletes NetworkPolicy
- Result: S3 access enabled 5 minutes before earliest backup
3. **CronJob: Disable S3 Access** (`longhorn-disable-s3-access`)
- Schedule: `0 4 * * *` (4:00 AM daily)
- Action: Re-creates NetworkPolicy
- Result: S3 access blocked after 3-hour backup window
4. **RBAC Resources**
- ServiceAccount: `longhorn-netpol-manager`
- Role: Permissions to manage NetworkPolicies
- RoleBinding: Binds role to service account
## Benefits
| Metric | Before | After | Improvement |
|--------|--------|-------|-------------|
| **Daily S3 API Calls** | 145,000+ | 5,000-10,000 | **93% reduction** |
| **Cost Impact** | Exceeds free tier | Within free tier | **$X/month savings** |
| **Automation** | Manual intervention | Fully automated | **Zero manual work** |
| **Backup Reliability** | Compromised | Maintained | **No impact** |
## Backup Schedule
| Type | Schedule | Retention | Window |
|------|----------|-----------|--------|
| **Daily** | 2:00 AM | 7 days | 12:55 AM - 4:00 AM |
| **Weekly** | 1:00 AM Sundays | 4 weeks | Same window |
## FluxCD Integration
**Critical Design Decision**: The NetworkPolicy is **dynamically managed by CronJobs**, NOT by Flux.
### Why This Matters
Flux continuously reconciles resources to match the Git repository state. If the NetworkPolicy were managed by Flux:
- CronJob deletes NetworkPolicy at 12:55 AM → Flux recreates it within minutes
- S3 remains blocked during backup window → Backups fail ❌
### How We Solved It
1. **NetworkPolicy is NOT in Git** - Only the CronJobs and RBAC are in `network-policy-s3-block.yaml`
2. **CronJobs are managed by Flux** - Flux ensures they exist and run on schedule
3. **NetworkPolicy is created by CronJob** - Without Flux labels/ownership
4. **Flux ignores the NetworkPolicy** - Not in Flux's inventory, so Flux won't touch it
### Verification
```bash
# Check Flux inventory (NetworkPolicy should NOT be listed)
kubectl get kustomization -n flux-system longhorn -o jsonpath='{.status.inventory.entries[*].id}' | grep -i network
# (Should return nothing)
# Check NetworkPolicy exists (managed by CronJobs)
kubectl get networkpolicy -n longhorn-system longhorn-block-s3-access
# (Should exist)
```
## Deployment
### Files Modified/Created
1. ✅ `network-policy-s3-block.yaml` - **NEW**: CronJobs and RBAC (NOT the NetworkPolicy itself)
2. ✅ `kustomization.yaml` - Added new file to resources
3. ✅ `BACKUP-GUIDE.md` - Updated with new solution documentation
4. ✅ `S3-API-OPTIMIZATION.md` - **NEW**: This implementation summary
5. ✅ `config-map.yaml` - Kept backup target configured (no changes needed)
6. ✅ `longhorn.yaml` - Reverted `backupstorePollInterval` (not needed)
### Deployment Steps
1. **Commit and push** changes to your k8s-fleet branch
2. **FluxCD will automatically apply** the new NetworkPolicy and CronJobs
3. **Monitor for one backup cycle**:
```bash
# Watch CronJobs
kubectl get cronjobs -n longhorn-system -w
# Check NetworkPolicy status
kubectl get networkpolicy -n longhorn-system
# Verify backups complete
kubectl get backups -n longhorn-system
```
### Verification Steps
#### Day 1: Initial Deployment
```bash
# 1. Verify NetworkPolicy is active (should exist immediately)
kubectl get networkpolicy -n longhorn-system longhorn-block-s3-access
# 2. Verify CronJobs are scheduled
kubectl get cronjobs -n longhorn-system | grep longhorn-.*-s3-access
# 3. Test: S3 access should be blocked
kubectl exec -n longhorn-system deploy/longhorn-ui -- curl -I https://<B2_ENDPOINT>
# Expected: Connection timeout or network error
```
#### Day 2: After First Backup Window
```bash
# 1. Check if CronJob ran successfully (should see completed job at 12:55 AM)
kubectl get jobs -n longhorn-system | grep enable-s3-access
# 2. Verify backups completed (check after 4:00 AM)
kubectl get backups -n longhorn-system
# Should see new backups with recent timestamps
# 3. Confirm NetworkPolicy was re-applied (after 4:00 AM)
kubectl get networkpolicy -n longhorn-system longhorn-block-s3-access
# Should exist again
# 4. Check CronJob logs
kubectl logs -n longhorn-system job/longhorn-enable-s3-access-<timestamp>
kubectl logs -n longhorn-system job/longhorn-disable-s3-access-<timestamp>
```
#### Week 1: Monitor S3 API Usage
```bash
# Monitor Backblaze B2 dashboard
# → Daily Class C transactions should drop from 145,000 to 5,000-10,000
# → Verify calls only occur during 1-4 AM window
```
## Manual Backup Outside Window
If you need to create a backup outside the scheduled window:
```bash
# 1. Temporarily remove NetworkPolicy
kubectl delete networkpolicy -n longhorn-system longhorn-block-s3-access
# 2. Create backup via Longhorn UI or:
kubectl create -f - <<EOF
apiVersion: longhorn.io/v1beta2
kind: Backup
metadata:
name: manual-backup-$(date +%s)
namespace: longhorn-system
spec:
snapshotName: <snapshot-name>
labels:
backup-type: manual
EOF
# 3. Wait for backup to complete
kubectl get backup -n longhorn-system manual-backup-* -w
# 4. Restore NetworkPolicy
kubectl apply -f manifests/infrastructure/longhorn/network-policy-s3-block.yaml
```
Or simply wait until the next automatic re-application at 4:00 AM.
## Troubleshooting
### NetworkPolicy Not Blocking S3
**Symptom**: S3 calls continue despite NetworkPolicy being active
**Check**:
```bash
# Verify NetworkPolicy is applied
kubectl describe networkpolicy -n longhorn-system longhorn-block-s3-access
# Check if CNI supports NetworkPolicies (Cilium does)
kubectl get pods -n kube-system | grep cilium
```
### Backups Failing
**Symptom**: Backups fail during scheduled window
**Check**:
```bash
# Verify NetworkPolicy was removed during backup window
kubectl get networkpolicy -n longhorn-system
# Should NOT exist between 12:55 AM - 4:00 AM
# Check enable-s3-access CronJob ran
kubectl get jobs -n longhorn-system | grep enable
# Check Longhorn manager logs
kubectl logs -n longhorn-system -l app=longhorn-manager --tail=100
```
### CronJobs Not Running
**Symptom**: CronJobs never execute
**Check**:
```bash
# Verify CronJobs exist and are scheduled
kubectl get cronjobs -n longhorn-system -o wide
# Check events
kubectl get events -n longhorn-system --sort-by='.lastTimestamp' | grep CronJob
# Manually trigger a job
kubectl create job -n longhorn-system test-enable --from=cronjob/longhorn-enable-s3-access
```
## Future Enhancements
1. **Adjust Window Size**: If backups consistently complete faster than 3 hours, reduce window to 2 hours (change disable CronJob to `0 3 * * *`)
2. **Alerting**: Add Prometheus alerts for:
- Backup failures during window
- CronJob execution failures
- NetworkPolicy re-creation failures
3. **Metrics**: Track actual S3 API call counts via Backblaze B2 API and alert if threshold exceeded
## References
- [Longhorn Issue #1547 - Excessive S3 Calls](https://github.com/longhorn/longhorn/issues/1547)
- [Community NetworkPolicy Solution](https://github.com/longhorn/longhorn/issues/1547#issuecomment-3395447100)
- [Longhorn Backup Target Documentation](https://longhorn.io/docs/1.9.0/snapshots-and-backups/backup-and-restore/set-backup-target/)
- [Kubernetes NetworkPolicy Documentation](https://kubernetes.io/docs/concepts/services-networking/network-policies/)
## Success Metrics
After 1 week of operation, you should observe:
- ✅ S3 API calls reduced by 85-93%
- ✅ Backblaze costs within free tier
- ✅ All scheduled backups completing successfully
- ✅ Zero manual intervention required
- ✅ Longhorn polls fail silently (network errors) outside backup window

200
manifests/infrastructure/longhorn/S3-API-SOLUTION-FINAL.md Normal file
View File

@@ -0,0 +1,200 @@
# Longhorn S3 API Call Reduction - Final Solution
## Problem Summary
Longhorn was making **145,000+ Class C API calls/day** to Backblaze B2, primarily `s3_list_objects` operations. This exceeded Backblaze's free tier (2,500 calls/day) by 58x, incurring significant costs.
## Root Cause
Longhorn's `backupstore-poll-interval` setting controls how frequently Longhorn managers poll the S3 backup target to check for new backups (primarily for Disaster Recovery volumes). With 3 manager pods and a low poll interval, this resulted in excessive API calls.
## Solution History
### Attempt 1: NetworkPolicy-Based Access Control ❌
**Approach**: Use NetworkPolicies dynamically managed by CronJobs to block S3 access outside backup windows (12:55 AM - 4:00 AM).
**Why It Failed**:
- NetworkPolicies that blocked external S3 also inadvertently blocked the Kubernetes API server
- Longhorn manager pods couldn't perform leader election or webhook operations
- Pods entered 1/2 Ready state with errors: `error retrieving resource lock longhorn-system/longhorn-manager-webhook-lock: dial tcp 10.96.0.1:443: i/o timeout`
- Even with CIDR-based rules (10.244.0.0/16 for pods, 10.96.0.0/12 for services), the NetworkPolicy was too aggressive
- Cilium/NetworkPolicy interaction complexity made it unreliable
**Files Created** (kept for reference):
- `network-policy-s3-block.yaml` - CronJobs and NetworkPolicy definitions
- Removed from `kustomization.yaml` but retained in repository
## Final Solution: Increased Poll Interval ✅
### Implementation
**Change**: Set `backupstore-poll-interval` to `86400` seconds (24 hours) instead of `0`.
**Location**: `manifests/infrastructure/longhorn/config-map.yaml`
```yaml
data:
default-resource.yaml: |-
"backup-target": "s3://<BUCKET_NAME>@<B2_ENDPOINT>/longhorn-backup"
"backup-target-credential-secret": "backblaze-credentials"
"backupstore-poll-interval": "86400" # 24 hours
"virtual-hosted-style": "true"
```
### Why This Works
1. **Dramatic Reduction**: Polling happens once per day instead of continuously
2. **No Breakage**: Kubernetes API, webhooks, and leader election work normally
3. **Simple**: No complex NetworkPolicies or CronJobs to manage
4. **Reliable**: Well-tested Longhorn configuration option
5. **Sufficient**: Backups don't require frequent polling since we use scheduled recurring jobs
### Expected Results
| Metric | Before | After | Improvement |
|--------|--------|-------|-------------|
| **Poll Frequency** | Every ~5 seconds | Every 24 hours | **99.99% reduction** |
| **Daily S3 API Calls** | 145,000+ | ~300-1,000 | **99% reduction** 📉 |
| **Backblaze Costs** | Exceeds free tier | Within free tier | ✅ |
| **System Stability** | Affected by NetworkPolicy | Stable | ✅ |
## Current Status
✅ **Applied**: ConfigMap updated with `backupstore-poll-interval: 86400`
✅ **Verified**: Longhorn manager pods are 2/2 Ready
✅ **Backups**: Continue working normally via recurring jobs
✅ **Monitoring**: Backblaze API usage should drop to <1,000 calls/day
## Monitoring
### Check Longhorn Manager Health
```bash
kubectl get pods -n longhorn-system -l app=longhorn-manager
# Should show: 2/2 Ready for all pods
```
### Check Poll Interval Setting
```bash
kubectl get configmap -n longhorn-system longhorn-default-resource -o jsonpath='{.data.default-resource\.yaml}' | grep backupstore-poll-interval
# Should show: "backupstore-poll-interval": "86400"
```
### Check Backups Continue Working
```bash
kubectl get backups -n longhorn-system --sort-by=.status.snapshotCreatedAt | tail -10
# Should see recent backups with "Completed" status
```
### Monitor Backblaze API Usage
1. Log into Backblaze B2 dashboard
2. Navigate to "Caps and Alerts"
3. Check "Class C Transactions" (includes `s3_list_objects`)
4. **Expected**: Should drop from 145,000/day to ~300-1,000/day within 24-48 hours
## Backup Schedule (Unchanged)
| Type | Schedule | Retention |
|------|----------|-----------|
| **Daily** | 2:00 AM | 7 days |
| **Weekly** | 1:00 AM Sundays | 4 weeks |
Backups are triggered by `RecurringJob` resources, not by polling.
## Why Polling Isn't Critical
**Longhorn's backupstore polling is primarily for**:
- Disaster Recovery (DR) volumes that need continuous sync
- Detecting backups created outside the cluster
**We don't use DR volumes**, and all backups are created by recurring jobs within the cluster, so:
- ✅ Once-daily polling is more than sufficient
- ✅ Backups work independently of polling frequency
- ✅ Manual backups via Longhorn UI still work immediately
## Troubleshooting
### If Pods Show 1/2 Ready
**Symptom**: Longhorn manager pods stuck at 1/2 Ready
**Cause**: NetworkPolicy may have been accidentally applied
**Solution**:
```bash
# Check for NetworkPolicy
kubectl get networkpolicy -n longhorn-system
# If found, delete it
kubectl delete networkpolicy -n longhorn-system longhorn-block-s3-access
# Wait 30 seconds
sleep 30
# Verify pods recover
kubectl get pods -n longhorn-system -l app=longhorn-manager
```
### If S3 API Calls Remain High
**Check poll interval is applied**:
```bash
kubectl get configmap -n longhorn-system longhorn-default-resource -o yaml
```
**Restart Longhorn managers to pick up changes**:
```bash
kubectl rollout restart daemonset -n longhorn-system longhorn-manager
```
### If Backups Fail
Backups should continue working normally since they're triggered by recurring jobs, not polling. If issues occur:
```bash
# Check recurring jobs
kubectl get recurringjobs -n longhorn-system
# Check recent backup jobs
kubectl get jobs -n longhorn-system | grep backup
# Check backup target connectivity (should work anytime)
MANAGER_POD=$(kubectl get pods -n longhorn-system -l app=longhorn-manager --no-headers | head -1 | awk '{print $1}')
kubectl exec -n longhorn-system "$MANAGER_POD" -c longhorn-manager -- curl -I https://<B2_ENDPOINT>
```
## References
- [Longhorn Issue #1547](https://github.com/longhorn/longhorn/issues/1547) - Original excessive S3 calls issue
- [Longhorn Backup Target Documentation](https://longhorn.io/docs/1.9.0/snapshots-and-backups/backup-and-restore/set-backup-target/)
- Longhorn version: v1.9.0
## Files Modified
1. ✅ `config-map.yaml` - Updated `backupstore-poll-interval` to 86400
2. ✅ `kustomization.yaml` - Removed network-policy-s3-block.yaml reference
3. ✅ `network-policy-s3-block.yaml` - Retained for reference (not applied)
4. ✅ `S3-API-SOLUTION-FINAL.md` - This document
## Lessons Learned
1. **NetworkPolicies are tricky**: Blocking external traffic can inadvertently block internal cluster communication
2. **Start simple**: Configuration-based solutions are often more reliable than complex automation
3. **Test thoroughly**: Always verify pods remain healthy after applying NetworkPolicies
4. **Understand the feature**: Longhorn's polling is for DR volumes, which we don't use
5. **24-hour polling is sufficient**: For non-DR use cases, frequent polling isn't necessary
## Success Metrics
Monitor these over the next week:
- ✅ Longhorn manager pods: 2/2 Ready
- ✅ Daily backups: Completing successfully
- ✅ S3 API calls: <1,000/day (down from 145,000)
- ✅ Backblaze costs: Within free tier
- ✅ No manual intervention required

41
manifests/infrastructure/longhorn/backblaze-secret.yaml Normal file
View File

@@ -0,0 +1,41 @@
apiVersion: v1
kind: Secret
metadata:
name: backblaze-credentials
namespace: longhorn-system
type: Opaque
stringData:
AWS_ACCESS_KEY_ID: ENC[AES256_GCM,data:OGCSNVoeABeigczChYkRTKjIsjEYDA+cNA==,iv:So6ipxl+te3LkPbtyOwixnvv4DPbzl0yCGT8cqPgPbY=,tag:ApaM+bBqi9BJU/EVraKWrQ==,type:str]
AWS_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:EMFNPCdt/V+2d4xnVARNTBBpY3UTqvpN3LezT/TZ7w==,iv:Q5pNnuKX+lUt/V4xpgF2Zg1q6e1znvG+laDNrLIrgBY=,tag:xGF/SvAJ9+tfuB7QdirAhw==,type:str]
AWS_ENDPOINTS: ENC[AES256_GCM,data:PSiRbt53KKK5XOOxIEiiycaFTriaJbuY0Z4Q9yC1xTwz9H/+hoOQ35w=,iv:pGwbR98F5C4N9Vca9btaJ9mKVS7XUkL8+Pva7TWTeTk=,tag:PxFllLIjj+wXDSXGuU/oLA==,type:str]
VIRTUAL_HOST_STYLE: ENC[AES256_GCM,data:a9RJ2Q==,iv:1VSTWiv1WFia0rgwkoZ9WftaLDdKtJabwiyY90AWvNY=,tag:tQZDFjqAABueZJ4bjD2PfA==,type:str]
sops:
lastmodified: "2025-06-30T18:44:50Z"
mac: ENC[AES256_GCM,data:5cdqJQiwoFwWfaNjtqNiaD5sY31979cdS4R6vBmNIKqd7ZaCMJLEKBm5lCLF7ow3+V17pxGhVu4EXX+rKVaNu6Qs6ivXtVM+kA0RutqPFnWDVfoZcnuW98IBjpyh4i9Y6Dra8zSda++Dt2R7Frouc/7lT74ANZYmSRN9WCYsTNg=,iv:s9c+YDDxAUdjWlzsx5jALux2UW5dtg56Pfi3FF4K0lU=,tag:U9bTTOZaqQ9lekpsIbUkWA==,type:str]
pgp:
- created_at: "2025-06-30T18:44:50Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DZT3mpHTS/JgSAQdAbJ88Og3rBkHDPJXf04xSp79A1rfXUDwsP2Wzz0rgI2ww
67XRMSSu2nUApEk08vf1ZF5ulewMQbnVjDDqvM8+BcgELllZVhnNW09NzMb5uPD+
1GgBCQIQXzEZTIi11OR5Z44vLkU64tF+yAPzA6j6y0lyemabOJLDB/XJiV/nq57h
+Udy8rg3sAmZt6FmBiTssKpxy6C6nFFSHVnTY7RhKg9p87AYKz36bSUI7TRhjZGb
f9U9EUo09Zh4JA==
=6fMP
-----END PGP MESSAGE-----
fp: B120595CA9A643B051731B32E67FF350227BA4E8
- created_at: "2025-06-30T18:44:50Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DSXzd60P2RKISAQdAPYpP5mUd4lVstNeGURyFoXbfPbaSH+IlSxgrh/wBfCEw
oI6DwAxkRAxLRwptJoQA9zU+N6LRN+o5kcHLMG/eNnUyNdAfNg17fs16UXf5N2Gi
1GgBCQIQRcLoTo+r7TyUUTxtPGIrQ7c5jy7WFRzm25XqLuvwTYipDTbQC5PyZu5R
4zFgx4ZfDayB3ldPMoAHZ8BeB2VTiQID+HRQGGbSSCM7U+HvzSXNuapNSGXpfWEA
qShkjhXz1sF7JQ==
=UqeC
-----END PGP MESSAGE-----
fp: 4A8AADB4EBAB9AF88EF7062373CECE06CC80D40C
encrypted_regex: ^(data|stringData)$
version: 3.10.2

78
manifests/infrastructure/longhorn/backup-examples.yaml Normal file
View File

@@ -0,0 +1,78 @@
# Examples of how to apply S3 backup recurring jobs to volumes
# These are examples - you would apply these patterns to your actual PVCs/StorageClasses
---
# Example 1: Apply backup labels to an existing PVC
# This requires the PVC to be labeled as a recurring job source first
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: example-app-data
namespace: default
labels:
# Enable this PVC as a source for recurring job labels
recurring-job.longhorn.io/source: "enabled"
# Apply daily backup job group
recurring-job-group.longhorn.io/longhorn-s3-backup: "enabled"
# OR apply weekly backup job group (choose one)
# recurring-job-group.longhorn.io/longhorn-s3-backup-weekly: "enabled"
# OR apply specific recurring job by name
# recurring-job.longhorn.io/s3-backup-daily: "enabled"
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
storageClassName: longhorn
---
# Example 2: StorageClass with automatic backup assignment
# Any PVC created with this StorageClass will automatically get backups
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: longhorn-backup-daily
provisioner: driver.longhorn.io
allowVolumeExpansion: true
reclaimPolicy: Retain
volumeBindingMode: Immediate
parameters:
numberOfReplicas: "2"
staleReplicaTimeout: "30"
fromBackup: ""
# Automatically assign backup jobs to volumes created with this StorageClass
recurringJobSelector: |
[
{
"name":"longhorn-s3-backup",
"isGroup":true
}
]
---
# Example 3: StorageClass for critical data with both daily and weekly backups
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: longhorn-backup-critical
provisioner: driver.longhorn.io
allowVolumeExpansion: true
reclaimPolicy: Retain
volumeBindingMode: Immediate
parameters:
numberOfReplicas: "2"
staleReplicaTimeout: "30"
fromBackup: ""
# Assign both daily and weekly backup groups
recurringJobSelector: |
[
{
"name":"longhorn-s3-backup",
"isGroup":true
},
{
"name":"longhorn-s3-backup-weekly",
"isGroup":true
}
]

37
manifests/infrastructure/longhorn/config-map.yaml Normal file
View File

@@ -0,0 +1,37 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: longhorn-default-resource
namespace: longhorn-system
data:
default-resource.yaml: ENC[AES256_GCM,data:vw2doEgVQYr1p9vHN9MLqoOSVM8LDBeowAvs2zOkwmGPue8QLxkxxpaFRy2zJH9igjXn30h1dsukmSZBfD9Y3cwrRcvuEZRMo3IsAJ6M1G/oeVpKc14Rll6/V48ZXPiB9qfn1upmUbJtl1EMyPc3vUetUD37fI81N3x4+bNK2OB6V8yGczuE3bJxIi4vV/Zay83Z3s0VyNRF4y18R3T0200Ib5KomANAZUMSCxKvjv4GOKHGYTVE5+C4LFxeOnPgmAtjV4x+lKcNCD1saNZ56yhVzsKVJClLdaRtIQ==,iv:s3OyHFQxd99NGwjXxHqa8rs9aYsl1vf+GCLNtvZ9nuc=,tag:2n8RLcHmp9ueKNm12MxjxQ==,type:str]
sops:
lastmodified: "2025-11-12T10:07:54Z"
mac: ENC[AES256_GCM,data:VBxywwWrVnKiyby+FzCdUlI89OkruNh1jyFE3cVXU/WR4FoCWclDSQ8v0FxT+/mS1/0eTX9XAXVIyqtzpAUU3YY3znq2CU8qsZa45B2PlPQP+7qGNBcyrpZZCsJxTYO/+jxr/9gV4pAJV27HFnyYfZDVZxArLUWQs32eJSdOfpc=,iv:7lbZjWhSEX7NisarWxCAAvw3+8v6wadq3/chrjWk2GQ=,tag:9AZyEuo7omdCbtRJ3YDarA==,type:str]
pgp:
- created_at: "2025-11-09T13:37:18Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DZT3mpHTS/JgSAQdAYMBTNc+JasEkeJpsS1d8OQ6iuhRTULXvFrGEia7gLXkw
+TRNuC4ZH+Lxmb5s3ImRX9dF1cMXoMGUCWJN/bScm5cLElNd2dHrtFoElVjn4/vI
1GgBCQIQ4jPpbQJym+xU5jS5rN3dtW6U60IYxX5rPvh0294bxgOzIIqI/oI/0qak
C4EYFsfH9plAOmvF56SnFX0PSczBjyUlngJ36NFHMN3any7qW/C0tYXFF3DDiOC3
kpa/moMr5CNTnQ==
=xVwB
-----END PGP MESSAGE-----
fp: B120595CA9A643B051731B32E67FF350227BA4E8
- created_at: "2025-11-09T13:37:18Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DSXzd60P2RKISAQdA9omTE+Cuy7BvMA8xfqsZv2o+Jh3QvOL+gZY/Z5CuVgIw
IBgwiVypHqwDf8loCVIdlo1/h5gctj/t11cxb2hKNRGQ0kFNLdpu5Mx+RbJZ/az/
1GgBCQIQB/gKeYbAqSxrJMKl/Q+6PfAXTAjH33K8IlDQKbF8q3QvoQDJJU3i0XwQ
ljhWRC/RZzO7hHXJqkR9z5sVIysHoEo+O9DZ0OzefjKb+GscdgSwJwGgsZzrVRXP
kSLdNO0eE5ubMQ==
=O/Lu
-----END PGP MESSAGE-----
fp: 4A8AADB4EBAB9AF88EF7062373CECE06CC80D40C
encrypted_regex: ^(data|stringData)$
version: 3.10.2

11
manifests/infrastructure/longhorn/kustomization.yaml Normal file
View File

@@ -0,0 +1,11 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- longhorn.yaml
- storageclass.yaml
- backblaze-secret.yaml
- config-map.yaml
- recurring-job-s3-backup.yaml
- network-policy-s3-block.yaml

64
manifests/infrastructure/longhorn/longhorn.yaml Normal file
View File

@@ -0,0 +1,64 @@
---
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: longhorn-repo
namespace: longhorn-system
spec:
interval: 5m0s
url: https://charts.longhorn.io
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: longhorn-release
namespace: longhorn-system
spec:
interval: 5m
chart:
spec:
chart: longhorn
version: v1.10.0
sourceRef:
kind: HelmRepository
name: longhorn-repo
namespace: longhorn-system
interval: 1m
values:
# Use hotfixed longhorn-manager image
image:
longhorn:
manager:
tag: v1.10.0-hotfix-1
defaultSettings:
defaultDataPath: /var/mnt/longhorn-storage
defaultReplicaCount: "2"
replicaNodeLevelSoftAntiAffinity: true
allowVolumeCreationWithDegradedAvailability: false
guaranteedInstanceManagerCpu: 5
createDefaultDiskLabeledNodes: true
# Multi-node optimized settings
storageMinimalAvailablePercentage: "20"
storageReservedPercentageForDefaultDisk: "15"
storageOverProvisioningPercentage: "200"
# Single replica for UI
service:
ui:
type: ClusterIP
# Longhorn UI replica count
longhornUI:
replicas: 1
# Enable metrics collection
metrics:
serviceMonitor:
enabled: true
longhornManager:
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Exists
longhornDriver:
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Exists

8
manifests/infrastructure/longhorn/namespace.yaml Normal file
View File

@@ -0,0 +1,8 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: longhorn-system
labels:
pod-security.kubernetes.io/enforce: privileged
pod-security.kubernetes.io/enforce-version: latest

211
manifests/infrastructure/longhorn/network-policy-s3-block.yaml Normal file
View File

@@ -0,0 +1,211 @@
---
# Longhorn S3 Access Control via NetworkPolicy
#
# NetworkPolicy that blocks external S3 access by default, with CronJobs to
# automatically remove it during backup windows (12:55 AM - 4:00 AM).
#
# Network Details:
# - Pod CIDR: 10.244.0.0/16 (within 10.0.0.0/8)
# - Service CIDR: 10.96.0.0/12 (within 10.0.0.0/8)
# - VLAN Network: 10.132.0.0/24 (within 10.0.0.0/8)
#
# How It Works:
# - NetworkPolicy is applied by default, blocking external S3 (Backblaze B2)
# - CronJob removes NetworkPolicy at 12:55 AM (5 min before earliest backup at 1 AM)
# - CronJob reapplies NetworkPolicy at 4:00 AM (after backup window closes)
# - Allows all internal cluster traffic (10.0.0.0/8) while blocking external S3
#
# Backup Schedule:
# - Daily backups: 2:00 AM
# - Weekly backups: 1:00 AM Sundays
# - Backup window: 12:55 AM - 4:00 AM (3 hours 5 minutes)
#
# See: BACKUP-GUIDE.md and S3-API-SOLUTION-FINAL.md for full documentation
---
# NetworkPolicy: Blocks S3 access by default
# This is applied initially, then managed by CronJobs below
# Using CiliumNetworkPolicy for better API server support via toEntities
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: longhorn-block-s3-access
namespace: longhorn-system
labels:
app: longhorn
purpose: s3-access-control
spec:
description: "Block external S3 access while allowing internal cluster communication"
endpointSelector:
matchLabels:
app: longhorn-manager
egress:
# Allow DNS to kube-system namespace
- toEndpoints:
- matchLabels:
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP
# Explicitly allow Kubernetes API server (critical for Longhorn)
# Cilium handles this specially - kube-apiserver entity is required
- toEntities:
- kube-apiserver
# Allow all internal cluster traffic (10.0.0.0/8)
# This includes:
# - Pod CIDR: 10.244.0.0/16
# - Service CIDR: 10.96.0.0/12 (API server already covered above)
# - VLAN Network: 10.132.0.0/24
# - All other internal 10.x.x.x addresses
- toCIDR:
- 10.0.0.0/8
# Allow pod-to-pod communication within cluster
# The 10.0.0.0/8 CIDR block above covers all pod-to-pod communication
# This explicit rule ensures instance-manager pods are reachable
- toEntities:
- cluster
# Block all other egress (including external S3 like Backblaze B2)
---
# RBAC for CronJobs that manage the NetworkPolicy
apiVersion: v1
kind: ServiceAccount
metadata:
name: longhorn-netpol-manager
namespace: longhorn-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: longhorn-netpol-manager
namespace: longhorn-system
rules:
- apiGroups: ["cilium.io"]
resources: ["ciliumnetworkpolicies"]
verbs: ["get", "create", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: longhorn-netpol-manager
namespace: longhorn-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: longhorn-netpol-manager
subjects:
- kind: ServiceAccount
name: longhorn-netpol-manager
namespace: longhorn-system
---
# CronJob: Remove NetworkPolicy before backups (12:55 AM daily)
# This allows S3 access during the backup window
apiVersion: batch/v1
kind: CronJob
metadata:
name: longhorn-enable-s3-access
namespace: longhorn-system
labels:
app: longhorn
purpose: s3-access-control
spec:
# Run at 12:55 AM daily (5 minutes before earliest backup at 1:00 AM Sunday weekly)
schedule: "55 0 * * *"
successfulJobsHistoryLimit: 2
failedJobsHistoryLimit: 2
concurrencyPolicy: Forbid
jobTemplate:
spec:
template:
metadata:
labels:
app: longhorn-netpol-manager
spec:
serviceAccountName: longhorn-netpol-manager
restartPolicy: OnFailure
containers:
- name: delete-netpol
image: bitnami/kubectl:latest
imagePullPolicy: IfNotPresent
command:
- /bin/sh
- -c
- |
echo "Removing CiliumNetworkPolicy to allow S3 access for backups..."
kubectl delete ciliumnetworkpolicy longhorn-block-s3-access -n longhorn-system --ignore-not-found=true
echo "S3 access enabled. Backups can proceed."
---
# CronJob: Re-apply NetworkPolicy after backups (4:00 AM daily)
# This blocks S3 access after the backup window closes
apiVersion: batch/v1
kind: CronJob
metadata:
name: longhorn-disable-s3-access
namespace: longhorn-system
labels:
app: longhorn
purpose: s3-access-control
spec:
# Run at 4:00 AM daily (gives 3 hours 5 minutes for backups to complete)
schedule: "0 4 * * *"
successfulJobsHistoryLimit: 2
failedJobsHistoryLimit: 2
concurrencyPolicy: Forbid
jobTemplate:
spec:
template:
metadata:
labels:
app: longhorn-netpol-manager
spec:
serviceAccountName: longhorn-netpol-manager
restartPolicy: OnFailure
containers:
- name: create-netpol
image: bitnami/kubectl:latest
imagePullPolicy: IfNotPresent
command:
- /bin/sh
- -c
- |
echo "Re-applying CiliumNetworkPolicy to block S3 access..."
kubectl apply -f - <<EOF
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: longhorn-block-s3-access
namespace: longhorn-system
labels:
app: longhorn
purpose: s3-access-control
spec:
description: "Block external S3 access while allowing internal cluster communication"
endpointSelector:
matchLabels:
app: longhorn-manager
egress:
# Allow DNS to kube-system namespace
- toEndpoints:
- matchLabels:
k8s-app: kube-dns
toPorts:
- ports:
- port: "53"
protocol: UDP
- port: "53"
protocol: TCP
# Explicitly allow Kubernetes API server (critical for Longhorn)
- toEntities:
- kube-apiserver
# Allow all internal cluster traffic (10.0.0.0/8)
- toCIDR:
- 10.0.0.0/8
# Allow pod-to-pod communication within cluster
# The 10.0.0.0/8 CIDR block above covers all pod-to-pod communication
- toEntities:
- cluster
# Block all other egress (including external S3)
EOF
echo "S3 access blocked. Polling stopped until next backup window."

34
manifests/infrastructure/longhorn/recurring-job-s3-backup.yaml Normal file
View File

@@ -0,0 +1,34 @@
---
apiVersion: longhorn.io/v1beta2
kind: RecurringJob
metadata:
name: s3-backup-daily
namespace: longhorn-system
spec:
cron: "0 2 * * *" # Daily at 2 AM
task: "backup"
groups:
- longhorn-s3-backup
retain: 7 # Keep 7 daily backups
concurrency: 2 # Max 2 concurrent backup jobs
labels:
recurring-job: "s3-backup-daily"
backup-type: "daily"
---
apiVersion: longhorn.io/v1beta2
kind: RecurringJob
metadata:
name: s3-backup-weekly
namespace: longhorn-system
spec:
cron: "0 1 * * 0" # Weekly on Sunday at 1 AM
task: "backup"
groups:
- longhorn-s3-backup-weekly
retain: 4 # Keep 4 weekly backups
concurrency: 1 # Only 1 concurrent weekly backup
labels:
recurring-job: "s3-backup-weekly"
backup-type: "weekly"
parameters:
full-backup-interval: "1" # Full backup every other week (alternating full/incremental)

81
manifests/infrastructure/longhorn/storageclass.yaml Normal file
View File

@@ -0,0 +1,81 @@
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: longhorn-retain
annotations:
storageclass.kubernetes.io/is-default-class: "false"
provisioner: driver.longhorn.io
allowVolumeExpansion: true
parameters:
numberOfReplicas: "2"
staleReplicaTimeout: "2880"
fromBackup: ""
fsType: "xfs"
dataLocality: "best-effort"
reclaimPolicy: Retain
volumeBindingMode: Immediate
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: longhorn-delete
annotations:
storageclass.kubernetes.io/is-default-class: "false"
provisioner: driver.longhorn.io
allowVolumeExpansion: true
parameters:
numberOfReplicas: "2"
staleReplicaTimeout: "2880"
fromBackup: ""
fsType: "xfs"
dataLocality: "best-effort"
reclaimPolicy: Delete
volumeBindingMode: Immediate
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: longhorn-single-delete
annotations:
storageclass.kubernetes.io/is-default-class: "false"
provisioner: driver.longhorn.io
allowVolumeExpansion: true
parameters:
numberOfReplicas: "1"
staleReplicaTimeout: "2880"
fromBackup: ""
fsType: "xfs"
dataLocality: "best-effort"
reclaimPolicy: Delete
volumeBindingMode: Immediate
---
# Redis-specific StorageClass
# Single replica as Redis handles replication at application level
# Note: volumeBindingMode is immutable after creation
# If this StorageClass already exists with matching configuration, Flux reconciliation
# may show an error but it's harmless - the existing StorageClass will continue to work.
# For new clusters, this will be created correctly.
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: longhorn-redis
annotations:
storageclass.kubernetes.io/is-default-class: "false"
provisioner: driver.longhorn.io
allowVolumeExpansion: true
parameters:
# Single replica as Redis handles replication at application level
numberOfReplicas: "1"
staleReplicaTimeout: "2880"
fsType: "xfs" # xfs to match existing Longhorn volumes
dataLocality: "strict-local" # Keep Redis data local to node
# Integrate with existing S3 backup infrastructure
recurringJobSelector: |
[
{
"name":"longhorn-s3-backup",
"isGroup":true
}
]
reclaimPolicy: Delete
volumeBindingMode: Immediate

86
manifests/infrastructure/metrics-server/README.md Normal file
View File

@@ -0,0 +1,86 @@
# Kubernetes Metrics Server
## Overview
This deploys the Kubernetes Metrics Server to provide resource metrics for nodes and pods. The metrics server enables `kubectl top` commands and provides metrics for Horizontal Pod Autoscaling (HPA) and Vertical Pod Autoscaling (VPA).
## Architecture
### Current Deployment (Simple)
- **Version**: v0.7.2 (latest stable)
- **Replicas**: 2 (HA across both cluster nodes)
- **TLS Mode**: Insecure TLS for initial deployment (`--kubelet-insecure-tls=true`)
- **Integration**: OpenObserve monitoring via ServiceMonitor
### Security Configuration
The current deployment uses `--kubelet-insecure-tls=true` for compatibility with Talos Linux. This is acceptable for internal cluster metrics as:
- Metrics traffic stays within the cluster network
- The VLAN provides network isolation
- No sensitive data is exposed via metrics
- Proper RBAC controls access to the metrics API
### Future Enhancements (Optional)
For production hardening, the repository includes:
- `certificate.yaml`: cert-manager certificates for proper TLS
- `metrics-server.yaml`: Full TLS-enabled deployment
- Switch to secure TLS by updating kustomization.yaml when needed
## Usage
### Basic Commands
```bash
# View node resource usage
kubectl top nodes
# View pod resource usage (all namespaces)
kubectl top pods --all-namespaces
# View pod resource usage (specific namespace)
kubectl top pods -n kube-system
# View pod resource usage with containers
kubectl top pods --containers
```
### Integration with Monitoring
The metrics server is automatically discovered by OpenObserve via ServiceMonitor for:
- Metrics server performance monitoring
- Resource usage dashboards
- Alerting on high resource consumption
## Troubleshooting
### Common Issues
1. **"Metrics API not available"**: Check pod status with `kubectl get pods -n metrics-server-system`
2. **TLS certificate errors**: Verify APIService with `kubectl get apiservice v1beta1.metrics.k8s.io`
3. **Resource limits**: Pods may be OOMKilled if cluster load is high
### Verification
```bash
# Check metrics server status
kubectl get pods -n metrics-server-system
# Verify API registration
kubectl get apiservice v1beta1.metrics.k8s.io
# Test metrics collection
kubectl top nodes
kubectl top pods -n metrics-server-system
```
## Configuration
### Resource Requests/Limits
- **CPU**: 100m request, 500m limit
- **Memory**: 200Mi request, 500Mi limit
- **Priority**: system-cluster-critical
### Node Scheduling
- Tolerates control plane taints
- Can schedule on both n1 (control plane) and n2 (worker)
- Uses node selector for Linux nodes only
## Monitoring Integration
- **ServiceMonitor**: Automatically scraped by OpenObserve
- **Metrics Path**: `/metrics` on HTTPS port
- **Scrape Interval**: 30 seconds
- **Dashboard**: Available in OpenObserve for resource analysis

50
manifests/infrastructure/metrics-server/certificate.yaml Normal file
View File

@@ -0,0 +1,50 @@
---
# Self-signed CA for metrics server (for internal cluster communication)
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
name: metrics-server-selfsigned-issuer
spec:
selfSigned: {}
---
# CA Certificate for metrics server
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: metrics-server-ca
namespace: metrics-server-system
spec:
secretName: metrics-server-ca-secret
commonName: "metrics-server-ca"
isCA: true
issuerRef:
name: metrics-server-selfsigned-issuer
kind: ClusterIssuer
---
# CA Issuer using the generated CA
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: metrics-server-ca-issuer
namespace: metrics-server-system
spec:
ca:
secretName: metrics-server-ca-secret
---
# TLS Certificate for metrics server
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: metrics-server-certs
namespace: metrics-server-system
spec:
secretName: metrics-server-certs
issuerRef:
name: metrics-server-ca-issuer
kind: Issuer
commonName: metrics-server
dnsNames:
- metrics-server
- metrics-server.metrics-server-system
- metrics-server.metrics-server-system.svc
- metrics-server.metrics-server-system.svc.cluster.local

16
manifests/infrastructure/metrics-server/kustomization.yaml Normal file
View File

@@ -0,0 +1,16 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
metadata:
name: metrics-server
namespace: metrics-server-system
resources:
- namespace.yaml
- metrics-server-simple.yaml # Use simple version for immediate deployment
- monitoring.yaml
commonLabels:
app.kubernetes.io/name: metrics-server
app.kubernetes.io/component: metrics-server

217
manifests/infrastructure/metrics-server/metrics-server-simple.yaml Normal file
View File

@@ -0,0 +1,217 @@
---
# Simplified metrics server deployment for immediate use (without cert-manager dependency)
# This version uses kubelet insecure TLS for initial setup
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: metrics-server-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
k8s-app: metrics-server
rbac.authorization.k8s.io/aggregate-to-admin: "true"
rbac.authorization.k8s.io/aggregate-to-edit: "true"
rbac.authorization.k8s.io/aggregate-to-view: "true"
name: system:aggregated-metrics-reader
rules:
- apiGroups:
- metrics.k8s.io
resources:
- pods
- nodes
verbs:
- get
- list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
k8s-app: metrics-server
name: system:metrics-server
rules:
- apiGroups:
- ""
resources:
- nodes/metrics
verbs:
- get
- apiGroups:
- ""
resources:
- pods
- nodes
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
k8s-app: metrics-server
name: metrics-server-auth-reader
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: extension-apiserver-authentication-reader
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: metrics-server-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
k8s-app: metrics-server
name: metrics-server:system:auth-delegator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:auth-delegator
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: metrics-server-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
k8s-app: metrics-server
name: system:metrics-server
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:metrics-server
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: metrics-server-system
---
apiVersion: v1
kind: Service
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: metrics-server-system
spec:
ports:
- name: https
port: 443
protocol: TCP
targetPort: https
selector:
k8s-app: metrics-server
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: metrics-server-system
spec:
replicas: 2 # HA setup for your 2-node cluster
selector:
matchLabels:
k8s-app: metrics-server
strategy:
rollingUpdate:
maxUnavailable: 0
template:
metadata:
labels:
k8s-app: metrics-server
spec:
containers:
- args:
- --cert-dir=/tmp
- --secure-port=10250
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --kubelet-use-node-status-port
- --metric-resolution=15s
# Talos-specific: Use insecure TLS for initial setup
- --kubelet-insecure-tls=true
image: registry.k8s.io/metrics-server/metrics-server:v0.7.2
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /livez
port: https
scheme: HTTPS
periodSeconds: 10
name: metrics-server
ports:
- containerPort: 10250
name: https
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /readyz
port: https
scheme: HTTPS
initialDelaySeconds: 20
periodSeconds: 10
resources:
requests:
cpu: 25m # Reduced from 100m - actual usage ~7-14m
memory: 64Mi # Reduced from 200Mi - actual usage ~48-52MB
limits:
cpu: 100m # Reduced from 500m but still adequate
memory: 128Mi # Reduced from 500Mi but still adequate
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
seccompProfile:
type: RuntimeDefault
volumeMounts:
- mountPath: /tmp
name: tmp-dir
nodeSelector:
kubernetes.io/os: linux
priorityClassName: system-cluster-critical
serviceAccountName: metrics-server
tolerations:
# Allow scheduling on control plane
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
volumes:
- emptyDir: {}
name: tmp-dir
---
apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
labels:
k8s-app: metrics-server
name: v1beta1.metrics.k8s.io
spec:
group: metrics.k8s.io
groupPriorityMinimum: 100
insecureSkipTLSVerify: true # For initial setup
service:
name: metrics-server
namespace: metrics-server-system
version: v1beta1
versionPriority: 100

228
manifests/infrastructure/metrics-server/metrics-server.yaml Normal file
View File

@@ -0,0 +1,228 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: metrics-server-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
k8s-app: metrics-server
rbac.authorization.k8s.io/aggregate-to-admin: "true"
rbac.authorization.k8s.io/aggregate-to-edit: "true"
rbac.authorization.k8s.io/aggregate-to-view: "true"
name: system:aggregated-metrics-reader
rules:
- apiGroups:
- metrics.k8s.io
resources:
- pods
- nodes
verbs:
- get
- list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
k8s-app: metrics-server
name: system:metrics-server
rules:
- apiGroups:
- ""
resources:
- nodes/metrics
verbs:
- get
- apiGroups:
- ""
resources:
- pods
- nodes
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
k8s-app: metrics-server
name: metrics-server-auth-reader
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: extension-apiserver-authentication-reader
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: metrics-server-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
k8s-app: metrics-server
name: metrics-server:system:auth-delegator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:auth-delegator
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: metrics-server-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
k8s-app: metrics-server
name: system:metrics-server
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:metrics-server
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: metrics-server-system
---
apiVersion: v1
kind: Service
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: metrics-server-system
spec:
ports:
- name: https
port: 443
protocol: TCP
targetPort: https
selector:
k8s-app: metrics-server
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: metrics-server-system
spec:
replicas: 2 # HA setup for your 2-node cluster
selector:
matchLabels:
k8s-app: metrics-server
strategy:
rollingUpdate:
maxUnavailable: 0
template:
metadata:
labels:
k8s-app: metrics-server
spec:
containers:
- args:
- --cert-dir=/tmp
- --secure-port=10250
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --kubelet-use-node-status-port
- --metric-resolution=30s
# Talos-specific configuration for proper TLS
- --kubelet-insecure-tls=false # Use proper TLS for production
- --tls-cert-file=/etc/certs/tls.crt
- --tls-private-key-file=/etc/certs/tls.key
- --requestheader-client-ca-file=/etc/certs/ca.crt
- --requestheader-allowed-names=aggregator
- --requestheader-extra-headers-prefix=X-Remote-Extra-
- --requestheader-group-headers=X-Remote-Group
- --requestheader-username-headers=X-Remote-User
image: registry.k8s.io/metrics-server/metrics-server:v0.7.2
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /livez
port: https
scheme: HTTPS
periodSeconds: 10
name: metrics-server
ports:
- containerPort: 10250
name: https
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /readyz
port: https
scheme: HTTPS
initialDelaySeconds: 20
periodSeconds: 10
resources:
requests:
cpu: 100m
memory: 200Mi
limits:
cpu: 500m
memory: 500Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
seccompProfile:
type: RuntimeDefault
volumeMounts:
- mountPath: /tmp
name: tmp-dir
- mountPath: /etc/certs
name: certs
readOnly: true
nodeSelector:
kubernetes.io/os: linux
priorityClassName: system-cluster-critical
serviceAccountName: metrics-server
tolerations:
# Allow scheduling on control plane
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
volumes:
- emptyDir: {}
name: tmp-dir
- name: certs
secret:
secretName: metrics-server-certs
---
apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
labels:
k8s-app: metrics-server
name: v1beta1.metrics.k8s.io
spec:
group: metrics.k8s.io
groupPriorityMinimum: 100
insecureSkipTLSVerify: false
service:
name: metrics-server
namespace: metrics-server-system
version: v1beta1
versionPriority: 100

26
manifests/infrastructure/metrics-server/monitoring.yaml Normal file
View File

@@ -0,0 +1,26 @@
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: metrics-server
namespace: metrics-server-system
labels:
k8s-app: metrics-server
spec:
selector:
matchLabels:
k8s-app: metrics-server
endpoints:
- port: https
interval: 30s
path: /metrics
scheme: https
tlsConfig:
# Use the cluster's CA to verify the metrics server certificate
caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
serverName: metrics-server.metrics-server-system.svc.cluster.local
insecureSkipVerify: false
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
namespaceSelector:
matchNames:
- metrics-server-system

10
manifests/infrastructure/metrics-server/namespace.yaml Normal file
View File

@@ -0,0 +1,10 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: metrics-server-system
labels:
name: metrics-server-system
pod-security.kubernetes.io/enforce: restricted
pod-security.kubernetes.io/audit: restricted
pod-security.kubernetes.io/warn: restricted

530
manifests/infrastructure/openobserve-collector/agent-collector.yaml Normal file
View File

@@ -0,0 +1,530 @@
apiVersion: opentelemetry.io/v1beta1
kind: OpenTelemetryCollector
metadata:
name: openobserve-collector-agent
namespace: openobserve-collector
spec:
managementState: managed
image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.127.0
config:
exporters:
otlphttp/openobserve:
endpoint: http://openobserve-openobserve-standalone.openobserve.svc.cluster.local:5080
headers:
Authorization: ${OPENOBSERVE_AUTH}
logs_endpoint: http://openobserve-openobserve-standalone.openobserve.svc.cluster.local:5080/api/default/v1/logs
metrics_endpoint: http://openobserve-openobserve-standalone.openobserve.svc.cluster.local:5080/api/default/v1/metrics
traces_endpoint: http://openobserve-openobserve-standalone.openobserve.svc.cluster.local:5080/api/default/v1/traces
# HTTP client configuration to match OpenObserve HTTP/1.1
compression: gzip
max_idle_conns: 50
max_idle_conns_per_host: 5
idle_conn_timeout: 120s
read_buffer_size: 8192
write_buffer_size: 8192
otlphttp/openobserve_k8s_events:
endpoint: http://openobserve-openobserve-standalone.openobserve.svc.cluster.local:5080
headers:
Authorization: ${OPENOBSERVE_AUTH}
stream-name: k8s_events
logs_endpoint: http://openobserve-openobserve-standalone.openobserve.svc.cluster.local:5080/api/default/v1/logs
# HTTP client configuration to match OpenObserve HTTP/1.1
compression: gzip
max_idle_conns: 50
max_idle_conns_per_host: 5
idle_conn_timeout: 120s
read_buffer_size: 8192
write_buffer_size: 8192
extensions:
zpages: {}
processors:
batch:
send_batch_size: 5000
timeout: 30s
send_batch_max_size: 6000
metadata_keys:
- k8s.namespace.name
- k8s.pod.name
k8sattributes:
auth_type: serviceAccount
extract:
labels:
- from: pod
key: app.kubernetes.io/name
tag_name: service.name
- from: pod
key: app.kubernetes.io/component
tag_name: k8s.app.component
metadata:
- k8s.pod.name
- k8s.namespace.name
- k8s.node.name
filter:
node_from_env_var: K8S_NODE_NAME
passthrough: false
pod_association:
- sources:
- from: resource_attribute
name: k8s.pod.uid
- sources:
- from: resource_attribute
name: k8s.pod.name
- from: resource_attribute
name: k8s.namespace.name
- from: resource_attribute
name: k8s.node.name
- sources:
- from: resource_attribute
name: k8s.pod.ip
- sources:
- from: resource_attribute
name: k8s.pod.name
- from: resource_attribute
name: k8s.namespace.name
- sources:
- from: connection
attributes:
actions:
- key: k8s_node_name
from_attribute: k8s.node.name
action: upsert
groupbyattrs/final:
keys:
- k8s_node_name
- direction
metricstransform:
transforms:
- include: system.network.io
match_type: strict
action: update
new_name: system_network_io
- include: system.cpu.time
match_type: strict
action: update
new_name: k8s_node_cpu_time
- include: system.cpu.utilization
match_type: strict
action: update
new_name: k8s_node_cpu_utilization
- include: k8s.node.cpu.utilization
match_type: strict
action: update
new_name: k8s_node_cpu_utilization
- include: system.memory.usage
match_type: strict
action: update
new_name: system_memory_usage
- include: system.memory.utilization
match_type: strict
action: update
new_name: k8s_node_memory_utilization
- include: system.filesystem.utilization
match_type: strict
action: update
new_name: k8s_node_filesystem_utilization
- include: container_fs_reads_total
match_type: strict
action: update
new_name: container_fs_reads_total
- include: container_fs_writes_total
match_type: strict
action: update
new_name: container_fs_writes_total
- include: k8s.pod.cpu_request_utilization
match_type: strict
action: update
new_name: k8s_pod_cpu_request_utilization
- include: k8s.pod.cpu_limit_utilization
match_type: strict
action: update
new_name: k8s_pod_cpu_limit_utilization
- include: k8s.pod.memory_request_utilization
match_type: strict
action: update
new_name: k8s_pod_memory_request_utilization
- include: k8s.pod.memory_limit_utilization
match_type: strict
action: update
new_name: k8s_pod_memory_limit_utilization
- include: k8s.container.cpu_request_utilization
match_type: strict
action: update
new_name: k8s_container_cpu_request_utilization
- include: k8s.container.cpu_limit_utilization
match_type: strict
action: update
new_name: k8s_container_cpu_limit_utilization
- include: k8s.container.memory_request_utilization
match_type: strict
action: update
new_name: k8s_container_memory_request_utilization
- include: k8s.container.memory_limit_utilization
match_type: strict
action: update
new_name: k8s_container_memory_limit_utilization
resourcedetection:
detectors:
- system
- env
- k8snode
override: true
system:
hostname_sources:
- os
- dns
# Filter out high-cardinality, low-value metrics
filter/drop_noisy_metrics:
metrics:
exclude:
match_type: regexp
metric_names:
- ".*_bucket$" # Drop histogram buckets for non-critical metrics
- "go_.*" # Drop Go runtime metrics
- "promhttp_.*" # Drop Prometheus HTTP metrics
- "process_.*" # Drop process metrics
- "container_spec_.*" # Drop container spec metrics
- "container_tasks_state" # Drop task state metrics
# Add intelligent trace sampling to reduce from 100% to ~15-20%
tail_sampling:
decision_wait: 10s
num_traces: 50000
expected_new_traces_per_sec: 10
policies:
# Always sample error traces (100%)
- name: errors
type: status_code
status_code:
status_codes: [ERROR]
# Always sample slow traces >1s (100%)
- name: slow-traces
type: latency
latency:
threshold_ms: 1000
# Always sample traces from critical namespaces (100%)
- name: critical-namespaces
type: string_attribute
string_attribute:
key: k8s.namespace.name
values: [kube-system, openobserve, cert-manager, ingress-nginx, longhorn-system]
# Sample 5% of normal traces (reduced from 10% for resource optimization)
- name: probabilistic
type: probabilistic
probabilistic:
sampling_percentage: 5
receivers:
filelog/std:
exclude:
- /var/log/pods/default_daemonset-collector*_*/opentelemetry-collector/*.log
include:
- /var/log/pods/*/*/*.log
include_file_name: false
include_file_path: true
operators:
- id: get-format
routes:
- expr: body matches "^\\{"
output: parser-docker
- expr: body matches "^[^ Z]+ "
output: parser-crio
- expr: body matches "^[^ Z]+Z"
output: parser-containerd
type: router
- id: parser-crio
output: extract_metadata_from_filepath
regex: ^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$
timestamp:
layout: 2006-01-02T15:04:05.999999999Z07:00
layout_type: gotime
parse_from: attributes.time
type: regex_parser
- id: parser-containerd
output: extract_metadata_from_filepath
regex: ^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$
timestamp:
layout: "%Y-%m-%dT%H:%M:%S.%LZ"
parse_from: attributes.time
type: regex_parser
- id: parser-docker
output: extract_metadata_from_filepath
timestamp:
layout: "%Y-%m-%dT%H:%M:%S.%LZ"
parse_from: attributes.time
type: json_parser
- cache:
size: 128
id: extract_metadata_from_filepath
parse_from: attributes["log.file.path"]
regex: ^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]{36})\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log$
type: regex_parser
- from: attributes.log
to: body
type: move
- from: attributes.stream
to: attributes["log.iostream"]
type: move
- from: attributes.container_name
to: resource["k8s.container.name"]
type: move
- from: attributes.namespace
to: resource["k8s.namespace.name"]
type: move
- from: attributes.pod_name
to: resource["k8s.pod.name"]
type: move
- from: attributes.restart_count
to: resource["k8s.container.restart_count"]
type: move
- from: attributes.uid
to: resource["k8s.pod.uid"]
type: move
start_at: end
hostmetrics:
collection_interval: 60s
root_path: /hostfs
scrapers:
cpu: {}
disk: {}
memory: {}
filesystem:
exclude_fs_types:
fs_types:
- autofs
- binfmt_misc
- bpf
- cgroup2
- configfs
- debugfs
- devpts
- devtmpfs
- fusectl
- hugetlbfs
- iso9660
- mqueue
- nsfs
- overlay
- proc
- procfs
- pstore
- rpc_pipefs
- securityfs
- selinuxfs
- squashfs
- sysfs
- tracefs
match_type: strict
exclude_mount_points:
match_type: regexp
mount_points:
- /dev/.*
- /proc/.*
- /sys/.*
- /run/k3s/containerd/.*
- /var/lib/docker/.*
- /var/lib/kubelet/.*
- /snap/.*
load: {}
network: {}
kubeletstats:
auth_type: serviceAccount
collection_interval: 60s
endpoint: https://${env:K8S_NODE_IP}:10250
extra_metadata_labels:
- container.id
- k8s.volume.type
insecure_skip_verify: true
metric_groups:
- node
- pod
- container
- volume
metrics:
k8s.pod.cpu_limit_utilization:
enabled: true
k8s.pod.cpu_request_utilization:
enabled: true
k8s.pod.memory_limit_utilization:
enabled: true
k8s.pod.memory_request_utilization:
enabled: true
k8s.container.cpu_limit_utilization:
enabled: true
k8s.container.cpu_request_utilization:
enabled: true
k8s.container.memory_limit_utilization:
enabled: true
k8s.container.memory_request_utilization:
enabled: true
otlp:
protocols:
grpc: {}
http: {}
prometheus:
config:
scrape_configs:
- job_name: otel-collector
scrape_interval: 30s
static_configs:
- targets:
- 0.0.0.0:8888
- job_name: postgresql-cnpg
scrape_interval: 60s
kubernetes_sd_configs:
- role: pod
namespaces:
names:
- postgresql-system
relabel_configs:
# Only scrape pods with the cnpg.io/cluster label
- source_labels: [__meta_kubernetes_pod_label_cnpg_io_cluster]
action: keep
regex: postgres-shared
# Use the metrics port (9187)
- source_labels: [__meta_kubernetes_pod_container_port_name]
action: keep
regex: metrics
# Set the metrics path
- target_label: __metrics_path__
replacement: /metrics
# Add useful labels
- source_labels: [__meta_kubernetes_pod_name]
target_label: instance
- source_labels: [__meta_kubernetes_pod_label_cnpg_io_cluster]
target_label: cnpg_cluster
- source_labels: [__meta_kubernetes_namespace]
target_label: kubernetes_namespace
# Celery and Redis metrics - direct scraping
- job_name: redis-exporter
scrape_interval: 30s
kubernetes_sd_configs:
- role: endpoints
namespaces:
names:
- redis-system
relabel_configs:
- source_labels: [__meta_kubernetes_service_name]
action: keep
regex: redis-exporter
- source_labels: [__meta_kubernetes_endpoint_port_name]
action: keep
regex: metrics
- source_labels: [__meta_kubernetes_namespace]
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_service_name]
target_label: kubernetes_service_name
- job_name: celery-metrics-exporter
scrape_interval: 60s
kubernetes_sd_configs:
- role: endpoints
namespaces:
names:
- celery-monitoring
relabel_configs:
- source_labels: [__meta_kubernetes_service_name]
action: keep
regex: celery-metrics-exporter
- source_labels: [__meta_kubernetes_endpoint_port_name]
action: keep
regex: metrics
- source_labels: [__meta_kubernetes_namespace]
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_service_name]
target_label: kubernetes_service_name
# Longhorn metrics still handled by target allocator via ServiceMonitor
service:
telemetry:
metrics:
address: 0.0.0.0:8888
pipelines:
logs:
exporters:
- otlphttp/openobserve
processors:
- batch
- k8sattributes
receivers:
- filelog/std
metrics:
exporters:
- otlphttp/openobserve
processors:
- batch
- k8sattributes
- attributes
- filter/drop_noisy_metrics
- metricstransform
receivers:
- kubeletstats
- hostmetrics
- prometheus
traces:
exporters:
- otlphttp/openobserve
processors:
- batch
- k8sattributes
- tail_sampling
receivers:
- otlp
env:
- name: K8S_NODE_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: K8S_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: OPENOBSERVE_AUTH
valueFrom:
secretKeyRef:
name: openobserve-collector-credentials
key: authorization
ingress:
route: {}
mode: daemonset
observability:
metrics:
enableMetrics: true
podDisruptionBudget:
maxUnavailable: 1
replicas: 1
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 300m
memory: 512Mi
securityContext:
runAsUser: 0
runAsGroup: 0
serviceAccount: openobserve-collector
hostNetwork: true
upgradeStrategy: automatic
volumeMounts:
- mountPath: /hostfs
name: hostfs
readOnly: true
- mountPath: /var/log/pods
name: varlogpods
readOnly: true
- mountPath: /hostfs/proc
name: proc
readOnly: true
- mountPath: /hostfs/sys
name: sys
readOnly: true
volumes:
- hostPath:
path: /
name: hostfs
- hostPath:
path: /var/log/pods
name: varlogpods
- hostPath:
path: /proc
name: proc
- hostPath:
path: /sys
name: sys

89
manifests/infrastructure/openobserve-collector/collector-sa.yaml Normal file
View File

@@ -0,0 +1,89 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: openobserve-collector
namespace: openobserve-collector
labels:
app: openobserve-collector
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: openobserve-collector
labels:
app: openobserve-collector
rules:
- nonResourceURLs: ["/metrics", "/metrics/cadvisor"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources:
- endpoints
- events
- namespaces
- namespaces/status
- nodes
- nodes/spec
- nodes/stats
- nodes/metrics
- nodes/proxy
- persistentvolumes
- persistentvolumeclaims
- pods
- pods/status
- replicationcontrollers
- replicationcontrollers/status
- resourcequotas
- services
- configmaps
verbs: ["get", "list", "watch"]
- apiGroups: ["monitoring.coreos.com"]
resources:
- servicemonitors
- podmonitors
- probes
- scrapeconfigs
verbs: ["*"]
- apiGroups: ["apps"]
resources:
- daemonsets
- deployments
- replicasets
- statefulsets
verbs: ["get", "list", "watch"]
- apiGroups: ["extensions"]
resources:
- ingresses
verbs: ["get", "list", "watch"]
- apiGroups: ["batch"]
resources:
- jobs
- cronjobs
verbs: ["get", "list", "watch"]
- apiGroups: ["autoscaling"]
resources:
- horizontalpodautoscalers
verbs: ["get", "list", "watch"]
- apiGroups: ["networking.k8s.io"]
resources:
- ingresses
verbs: ["get", "list", "watch"]
- apiGroups: ["discovery.k8s.io"]
resources:
- endpointslices
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: openobserve-collector
labels:
app: openobserve-collector
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: openobserve-collector
subjects:
- kind: ServiceAccount
name: openobserve-collector
namespace: openobserve-collector

115
manifests/infrastructure/openobserve-collector/collector-servicemonitors.yaml Normal file
View File

@@ -0,0 +1,115 @@
---
# ServiceMonitor for Agent Collector Self-Monitoring
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: openobserve-collector-agent-metrics
namespace: openobserve-collector
labels:
app.kubernetes.io/name: openobserve-collector-agent
app.kubernetes.io/component: metrics
spec:
selector:
matchLabels:
app.kubernetes.io/name: openobserve-collector-agent
app.kubernetes.io/component: opentelemetry-collector
endpoints:
- port: metrics
interval: 60s
path: /metrics
scheme: http
scrapeTimeout: 30s
honorLabels: true
relabelings:
- sourceLabels: [__meta_kubernetes_pod_name]
targetLabel: pod
- sourceLabels: [__meta_kubernetes_pod_node_name]
targetLabel: node
- sourceLabels: [__meta_kubernetes_namespace]
targetLabel: namespace
metricRelabelings:
- sourceLabels: [__name__]
regex: 'otelcol_.*'
action: keep
- sourceLabels: [__name__]
regex: 'up|scrape_.*'
action: keep
---
# ServiceMonitor for Gateway Collector Self-Monitoring
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: openobserve-collector-gateway-metrics
namespace: openobserve-collector
labels:
app.kubernetes.io/name: openobserve-collector-gateway
app.kubernetes.io/component: metrics
spec:
selector:
matchLabels:
app.kubernetes.io/name: openobserve-collector-gateway
app.kubernetes.io/component: opentelemetry-collector
endpoints:
- port: metrics
interval: 60s
path: /metrics
scheme: http
scrapeTimeout: 30s
honorLabels: true
relabelings:
- sourceLabels: [__meta_kubernetes_pod_name]
targetLabel: pod
- sourceLabels: [__meta_kubernetes_pod_node_name]
targetLabel: node
- sourceLabels: [__meta_kubernetes_namespace]
targetLabel: namespace
metricRelabelings:
- sourceLabels: [__name__]
regex: 'otelcol_.*'
action: keep
- sourceLabels: [__name__]
regex: 'up|scrape_.*'
action: keep
---
# Service for Agent Collector Metrics (if not auto-created)
apiVersion: v1
kind: Service
metadata:
name: openobserve-collector-agent-metrics
namespace: openobserve-collector
labels:
app.kubernetes.io/name: openobserve-collector-agent
app.kubernetes.io/component: opentelemetry-collector
spec:
type: ClusterIP
ports:
- name: metrics
port: 8888
protocol: TCP
targetPort: 8888
selector:
app.kubernetes.io/name: openobserve-collector-agent
app.kubernetes.io/component: opentelemetry-collector
---
# Service for Gateway Collector Metrics (if not auto-created)
apiVersion: v1
kind: Service
metadata:
name: openobserve-collector-gateway-metrics
namespace: openobserve-collector
labels:
app.kubernetes.io/name: openobserve-collector-gateway
app.kubernetes.io/component: opentelemetry-collector
spec:
type: ClusterIP
ports:
- name: metrics
port: 8888
protocol: TCP
targetPort: 8888
selector:
app.kubernetes.io/name: openobserve-collector-gateway
app.kubernetes.io/component: opentelemetry-collector

315
manifests/infrastructure/openobserve-collector/gateway-collector.yaml Normal file
View File

@@ -0,0 +1,315 @@
apiVersion: opentelemetry.io/v1beta1
kind: OpenTelemetryCollector
metadata:
name: openobserve-collector-gateway
namespace: openobserve-collector
spec:
config:
connectors:
servicegraph:
dimensions:
- http.method
latency_histogram_buckets:
- 1
- 2
- 3
- 4
- 5
store:
max_items: 10
ttl: 1s
spanmetrics:
aggregation_temporality: AGGREGATION_TEMPORALITY_CUMULATIVE
dimensions:
- default: GET
name: http.method
- name: http.status_code
dimensions_cache_size: 1000
exemplars:
enabled: true
histogram:
explicit:
buckets:
- 100us
- 1ms
- 2ms
- 6ms
- 10ms
- 100ms
- 250ms
- 500ms
- 1000ms
- 1400ms
- 2000ms
- 5s
- 10s
- 30s
- 60s
- 120s
- 300s
- 600s
metrics_flush_interval: 15s
exporters:
otlphttp/openobserve:
endpoint: http://openobserve-openobserve-standalone.openobserve.svc.cluster.local:5080/api/default/
headers:
Authorization: ${OPENOBSERVE_AUTH}
stream-name: default
# HTTP client configuration to match OpenObserve HTTP/1.1
compression: gzip
max_idle_conns: 50
max_idle_conns_per_host: 5
idle_conn_timeout: 120s
read_buffer_size: 8192
write_buffer_size: 8192
otlphttp/openobserve_k8s_events:
endpoint: http://openobserve-openobserve-standalone.openobserve.svc.cluster.local:5080/api/default/
headers:
Authorization: ${OPENOBSERVE_AUTH}
stream-name: k8s_events
# HTTP client configuration to match OpenObserve HTTP/1.1
compression: gzip
max_idle_conns: 50
max_idle_conns_per_host: 5
idle_conn_timeout: 120s
read_buffer_size: 8192
write_buffer_size: 8192
processors:
batch:
send_batch_size: 5000
timeout: 30s
send_batch_max_size: 6000
metadata_keys:
- k8s.namespace.name
- k8s.pod.name
k8sattributes:
auth_type: serviceAccount
extract:
labels:
- from: pod
key: app.kubernetes.io/name
tag_name: service.name
- from: pod
key: k8s-app
tag_name: service.name
- from: pod
key: app.kubernetes.io/instance
tag_name: k8s.app.instance
- from: pod
key: app.kubernetes.io/version
tag_name: service.version
- from: pod
key: app.kubernetes.io/component
tag_name: k8s.app.component
metadata:
- k8s.namespace.name
- k8s.pod.name
- k8s.node.name
- k8s.deployment.name
passthrough: false
pod_association:
- sources:
- from: resource_attribute
name: k8s.pod.uid
- sources:
- from: resource_attribute
name: k8s.pod.name
- from: resource_attribute
name: k8s.namespace.name
- from: resource_attribute
name: k8s.node.name
- sources:
- from: resource_attribute
name: k8s.pod.ip
- sources:
- from: resource_attribute
name: k8s.pod.name
- from: resource_attribute
name: k8s.namespace.name
- sources:
- from: connection
resourcedetection:
detectors:
- env
override: true
timeout: 2s
metricstransform:
transforms:
- include: k8s.node.allocatable_cpu
match_type: strict
action: update
new_name: machine_cpu_cores
- include: k8s.node.allocatable_memory
match_type: strict
action: update
new_name: machine_memory_bytes
- include: k8s.node.condition_ready
match_type: strict
action: update
new_name: k8s_node_condition_ready
- include: k8s.node.condition_memory_pressure
match_type: strict
action: update
new_name: k8s_node_condition_memory_pressure
- include: k8s.node.condition_disk_pressure
match_type: strict
action: update
new_name: k8s_node_condition_disk_pressure
- include: k8s.node.condition_pid_pressure
match_type: strict
action: update
new_name: k8s_node_condition_pid_pressure
receivers:
k8s_cluster:
allocatable_types_to_report:
- cpu
- memory
- storage
collection_interval: 60s
metrics:
k8s.container.cpu_limit:
enabled: false
k8s.container.cpu_request:
enabled: false
k8s.container.memory_limit:
enabled: false
k8s.container.memory_request:
enabled: false
node_conditions_to_report:
- Ready
- MemoryPressure
- DiskPressure
- PIDPressure
k8s_events:
auth_type: serviceAccount
k8sobjects:
auth_type: serviceAccount
objects:
- field_selector: status.phase=Running
interval: 15m
mode: pull
name: pods
- group: events.k8s.io
mode: watch
name: events
otlp:
protocols:
grpc: {}
http: {}
otlp/logs:
protocols:
http:
endpoint: 0.0.0.0:4418
prometheus:
config:
global:
scrape_interval: 30s
evaluation_interval: 30s
external_labels: {}
scrape_configs:
- job_name: 'nginx-ingress'
static_configs:
- targets: ['<NODE_1_EXTERNAL_IP>:10254', '<NODE_2_EXTERNAL_IP>:10254', '<NODE_3_EXTERNAL_IP>:10254']
metrics_path: /metrics
scrape_interval: 30s
metric_relabel_configs:
- source_labels: [__name__]
regex: 'nginx_ingress_controller_.*'
action: keep
target_allocator:
endpoint: http://openobserve-collector-gateway-targetallocator:80
interval: 30s
collector_id: "${POD_NAME}"
service:
telemetry:
metrics:
address: 0.0.0.0:8888
pipelines:
logs/fluentbit-forward:
exporters:
- otlphttp/openobserve
processors:
- batch
receivers:
- otlp/logs
logs/k8s_events:
exporters:
- otlphttp/openobserve_k8s_events
processors:
- batch
- k8sattributes
- resourcedetection
receivers:
- k8s_events
metrics:
exporters:
- otlphttp/openobserve
processors:
- batch
- k8sattributes
- resourcedetection
- metricstransform
receivers:
- k8s_cluster
- spanmetrics
- servicegraph
- prometheus # Re-enabled for ServiceMonitor scraping
traces:
exporters:
- otlphttp/openobserve
- spanmetrics
- servicegraph
processors:
- batch
- k8sattributes
- resourcedetection
receivers:
- otlp
daemonSetUpdateStrategy: {}
deploymentUpdateStrategy: {}
env:
- name: K8S_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: K8S_NODE_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: OPENOBSERVE_AUTH
valueFrom:
secretKeyRef:
name: openobserve-collector-credentials
key: authorization
image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.127.0
ingress:
route: {}
managementState: managed
mode: statefulset
observability:
metrics:
enableMetrics: true
podDisruptionBudget:
maxUnavailable: 1
replicas: 1
resources:
requests:
cpu: 200m
memory: 512Mi
limits:
cpu: 500m
memory: 1Gi
serviceAccount: openobserve-collector
targetAllocator:
enabled: true
serviceAccount: openobserve-collector
prometheusCR:
enabled: true
serviceMonitorSelector: {}
podMonitorSelector: {}
scrapeConfigSelector: {}
upgradeStrategy: automatic

10
manifests/infrastructure/openobserve-collector/kustomization.yaml Normal file
View File

@@ -0,0 +1,10 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- secret.yaml
- agent-collector.yaml
- collector-sa.yaml
- gateway-collector.yaml
- longhorn-servicemonitor.yaml
- collector-servicemonitors.yaml

18
manifests/infrastructure/openobserve-collector/longhorn-servicemonitor.yaml Normal file
View File

@@ -0,0 +1,18 @@
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: longhorn-prometheus-servicemonitor
namespace: openobserve-collector
labels:
name: longhorn-prometheus-servicemonitor
spec:
selector:
matchLabels:
app: longhorn-manager
namespaceSelector:
matchNames:
- longhorn-system
endpoints:
- port: manager
path: /metrics
interval: 30s

8
manifests/infrastructure/openobserve-collector/namespace.yaml Normal file
View File

@@ -0,0 +1,8 @@
apiVersion: v1
kind: Namespace
metadata:
name: openobserve-collector
labels:
name: openobserve-collector
pod-security.kubernetes.io/enforce: privileged
pod-security.kubernetes.io/enforce-version: latest

40
manifests/infrastructure/openobserve-collector/secret.yaml Normal file
View File

@@ -0,0 +1,40 @@
apiVersion: v1
kind: Secret
metadata:
name: openobserve-collector-credentials
namespace: openobserve-collector
type: Opaque
stringData:
#ENC[AES256_GCM,data:2yJkOijDONhJY+hZ7Tk/29jRrv93ztrisX4JELiErla/BPDcsLdZYoIuGPmtsIhYnazTXZoD,iv:4PLBGHbzsXscXJW4RANSlyEuIhkDpFEpT8CgEo8klLM=,tag:ymxpLqQr2MQFW+A3UV+SWg==,type:comment]
#ENC[AES256_GCM,data:AHTcPETrdrltvKOH1HLdAU57RuYA/G+dz9mhCUExN7SYmA==,iv:WEbMEVNPCVmqOkWtvVKxH/B1w+Kl5+agqZsHRirfCP8=,tag:KHRsIMS7Evx9WSDEThdHQA==,type:comment]
authorization: ENC[AES256_GCM,data:m3CSGlha/eLqLZOaLgg+ZFezabI0Ttwb77Fi7jLL1/u5riRe4hdDk0KaC9iIxob3ZUoSJBV70tGdy9U/QAAXy8zCfAPTekBTGIeUJnuDGYOjZoMzH6jtWtfA566T0WA7jLTZKrQT,iv:IXHN2Y8qYo2Gq8qO2lUz8Dr2OcO1Mh6xVcryzdhjtXo=,tag:S/RRNsQRbtPrXmCwoqoY4g==,type:str]
sops:
lastmodified: "2025-06-27T23:03:22Z"
mac: ENC[AES256_GCM,data:NjQww3sDDUCtmuCyNP1vbn+4x04dA12O+pE2GogwK4bfIyp6fSWEkKDu54a6rx/DyBJSoN9J/3Nb/nIqZ5dYCQRYYZpBFH+kdAQXgy1hnRHM6ck6gXkjGvLyPyS+UMrz1xJ7dIhse663SWD9s9JQCoPEECwYjPcjO6azK7dOvlY=,iv:YnCcpCWU2dTR7t/NbLNBNEj8vSpIYGaZ6zX79gaY4SY=,tag:TS0+mvJtcNTjU1rHmgcbdg==,type:str]
pgp:
- created_at: "2025-06-27T23:03:22Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DZT3mpHTS/JgSAQdAKEkftAs2xJfxjDSA3RfMtmtsnyC/OipUq3V24OqgCC8w
0TW/fUq769Ao8v0zIQ1BLPin4gHLCy49j9IKf68YXwZK/kXy/Qxq/g5OtvPyTKbn
1GYBCQIQGA7z3J4X7BwV83xHqieZPbPD7+YkLcpw+ceXuJlKE9ldoQR98vITs+S0
/NP71qmJ2SLBxl5sX5fRUceHY/DE7PapkWDit8mg7Mi2w+fBwLi4lymN2akoxTKX
aZcSZsj/vrw=
=Traa
-----END PGP MESSAGE-----
fp: B120595CA9A643B051731B32E67FF350227BA4E8
- created_at: "2025-06-27T23:03:22Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DSXzd60P2RKISAQdA6l2wYljh86fs8RTUJ/W1UY8NDxPo65TkZGSoRGFU0AQw
daYGSXKT0R60P9uxFrGvQXyfbIGw+fuW/rd85FFtpn47wtoBphr2Mb+9cnB6kuNO
1GYBCQIQ3JEH3kRETxoAuCKRBGn6heb+spMCjft9/fVTA31HjIoNFlYBYM0kSnc5
p9wcP6V9YDp47mEutzVLQACx/W2qBPb6GDZrdLTTBTuUvQeI/kttga0hHzqYLc6B
OYb4FxUXl5g=
=DoEk
-----END PGP MESSAGE-----
fp: 4A8AADB4EBAB9AF88EF7062373CECE06CC80D40C
encrypted_regex: ^(data|stringData)$
version: 3.10.2

7
manifests/infrastructure/openobserve/kustomization.yaml Normal file
View File

@@ -0,0 +1,7 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- secret.yaml
- openobserve.yaml
- manual-ingress.yaml

29
manifests/infrastructure/openobserve/manual-ingress.yaml Normal file
View File

@@ -0,0 +1,29 @@
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: openobserve-ingress
namespace: openobserve
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/enable-cors: "true"
# Fix HTTP/2 protocol errors by forcing HTTP/1.1 backend communication
nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
spec:
ingressClassName: nginx
tls: []
rules:
- host: obs.keyboardvagabond.com
http:
paths:
# OpenObserve - route to HTTP service
- path: /
pathType: Prefix
backend:
service:
name: openobserve-openobserve-standalone
port:
number: 5080

9
manifests/infrastructure/openobserve/namespace.yaml Normal file
View File

@@ -0,0 +1,9 @@
# manifests/infrastructure/openobserve/namespace.yaml
---
apiVersion: v1
kind: Namespace
metadata:
name: openobserve
labels:
pod-security.kubernetes.io/enforce: privileged
pod-security.kubernetes.io/enforce-version: latest

119
manifests/infrastructure/openobserve/openobserve.yaml Normal file
View File

@@ -0,0 +1,119 @@
# manifests/infrastructure/openobserve/openobserve.yaml
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: openobserve
namespace: openobserve
spec:
interval: 5m0s
url: https://charts.openobserve.ai
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: openobserve
namespace: openobserve
spec:
interval: 5m
chart:
spec:
chart: openobserve-standalone
version: ">=0.15.0"
sourceRef:
kind: HelmRepository
name: openobserve
namespace: openobserve
interval: 1m
values:
# Use SIMD-optimized image for ARM with NEON support
image:
repository: public.ecr.aws/zinclabs/openobserve
tag: v0.15.0-simd
# Basic configuration with memory optimization
config:
ZO_TELEMETRY: "false"
ZO_WEB_URL: "https://obs.keyboardvagabond.com"
# Aggressive data retention for resource-constrained environment
ZO_COMPACT_DATA_RETENTION_DAYS: "7" # Reduced from 14 to 7 days
ZO_COMPACT_RETENTION_LOGS: "7" # Explicit log retention
ZO_COMPACT_RETENTION_METRICS: "14" # Keep metrics longer than logs
ZO_COMPACT_RETENTION_TRACES: "3" # Traces are large, keep only 3 days
# Memory optimization settings - reduced for 5GB container limit
ZO_MEMORY_CACHE_MAX_SIZE: "1536" # Reduced to 1.5GB (was 2GB) - still good performance
ZO_MEMORY_CACHE_DATAFUSION_MAX_SIZE: "768" # Reduced to 768MB (was 1GB) - adequate for queries
ZO_MAX_FILE_SIZE_IN_MEMORY: "64" # Reduce memory table size to 64MB (default 256MB)
ZO_MEM_DUMP_THREAD_NUM: "2" # Use 2 threads for memory dumps (faster disk writes)
# Enable disk caching to reduce RAM usage
ZO_DISK_CACHE_ENABLED: "true"
ZO_DISK_CACHE_MAX_SIZE: "8192" # 8GB disk cache (in MB)
# Reduce field processing overhead
ZO_COLS_PER_RECORD_LIMIT: "500" # Limit fields per record (default 1000)
# Optimized compaction for memory efficiency
ZO_COMPACT_SYNC_TO_DB_INTERVAL: "10" # Reduced frequency (was 5s) to save memory
ZO_COMPACT_MAX_FILE_SIZE: "256" # Smaller files (256MB) to reduce memory buffers
ZO_COMPACT_INTERVAL: "120" # Less frequent compaction (2min vs 1min) to reduce memory spikes
ZO_COMPACT_STEP_SIZE: "500" # Fewer files per step to reduce memory usage
# Local storage for now - easy to migrate to S3 later
persistence:
size: 100Gi
storageClass: "longhorn"
# Resource limits optimized with memory configuration tunning
resources:
requests:
cpu: 512m
memory: 1.5Gi # Reasonable request for optimized caches
limits:
cpu: 2500m
memory: 5Gi # Keep at 5GB with optimized cache settings
ingress:
enabled: false
# Security context optimized for large volumes per Kubernetes docs
# https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#configure-volume-permission-and-ownership-change-policy-for-pods
securityContext:
fsGroup: 2000
runAsUser: 10000 # Match existing StatefulSet to avoid conflicts
runAsGroup: 3000 # Match existing StatefulSet to avoid conflicts
fsGroupChangePolicy: "OnRootMismatch" # Only change permissions if root ownership differs
runAsNonRoot: true
# Use secret for credentials (secure approach)
extraEnv:
- name: ZO_ROOT_USER_EMAIL
valueFrom:
secretKeyRef:
name: openobserve-credentials
key: ZO_ROOT_USER_EMAIL
- name: ZO_ROOT_USER_PASSWORD
valueFrom:
secretKeyRef:
name: openobserve-credentials
key: ZO_ROOT_USER_PASSWORD
# SMTP configuration for email alerts - all as environment variables
- name: ZO_SMTP_ENABLED
value: "true"
- name: ZO_SMTP_HOST
value: "<YOUR_SMTP_SERVER>"
- name: ZO_SMTP_PORT
value: "587"
- name: ZO_SMTP_USERNAME
value: "alerts@mail.keyboardvagabond.com"
- name: ZO_SMTP_FROM_EMAIL
value: "alerts@mail.keyboardvagabond.com"
- name: ZO_SMTP_REPLY_TO
value: "alerts@mail.keyboardvagabond.com"
- name: ZO_SMTP_ENCRYPTION
value: "starttls"
- name: ZO_SMTP_PASSWORD
valueFrom:
secretKeyRef:
name: openobserve-credentials
key: ZO_SMTP_PASSWORD

49
manifests/infrastructure/openobserve/secret.yaml Normal file
View File

@@ -0,0 +1,49 @@
apiVersion: v1
kind: Secret
metadata:
name: openobserve-credentials
namespace: openobserve
type: Opaque
stringData:
#ENC[AES256_GCM,data:ciQlpWxpLZm/OdqfpX3og3AIECXErnwAZsfgHqsVQ7tY7FKXJFLgIPInELDCMnbhxnpaqB3cpKKZfYo=,iv:TGGgEOflQ04BMxHYvPOMGM+E6inG4BhTPywKAkuIXwU=,tag:wAntPBIy8zw6OffBgCLL8A==,type:comment]
#ENC[AES256_GCM,data:5rTQeiBnHo372FnVAyhXcTstce0iVxt7DWSEkwuKa91JlJlgL1jw2a+Fc8NWjy4hbLSq4Qht,iv:NGVB8FOP+Dv3dRb3RS84FSFQgHj4UW3p/cr+8ozoGcI=,tag:1Sr3pJFMuDbl7+jfQEItmw==,type:comment]
ZO_ROOT_USER_PASSWORD: ENC[AES256_GCM,data:jW2zrcHb75ozVO+NzUaaEsdIOLlra1dHnKLgxvlhNY8AtqQ1BI+iB6379wpa,iv:e8XAFf2OCwnxzingUzba1HpkXWdbfA36U92N4ciSLKo=,tag:rZAQeEgJYapyHKMgnzUyfQ==,type:str]
ZO_ROOT_USER_EMAIL: ENC[AES256_GCM,data:uJql3q4n8MScoNDD1xow1UnRjIemw69Gwq8=,iv:WK/EDY9sG7yhUxQznPubbK5UlsqmfGqFWfZJMg69DRE=,tag:FG18/MIIM8aYMXZff2ljtg==,type:str]
#ENC[AES256_GCM,data:4R8+Sdiofs0W5FpzALUKOBehq6EsHCYf7ChJbEGLc8n9fzMbZbWkr2Syvjy/wXJ/,iv:caG3Up+sCQBYD1IQstR5PRfzgni49UKYVRR+jhqWWKM=,tag:LDCYOZHdAbuYIh6i09BbfA==,type:comment]
ZO_SMTP_ENABLED: ENC[AES256_GCM,data:fzbe1g==,iv:XQYUDCKVgvSSh/eEF+gzs4Wf8mH11hUw5RgWYJTuiRI=,tag:mHko4/V+/oX1jdQ/JManoQ==,type:str]
ZO_SMTP_HOST: ENC[AES256_GCM,data:28CFU8QH3/voR2Sdg2RwAOCGmg==,iv:f+Q0M1OPkIBpLIGc0Shh2Zba49w+7NLdjnWtJCpDGnM=,tag:w8LsbkFA4KXqc02ddJ/fuw==,type:str]
ZO_SMTP_PORT: ENC[AES256_GCM,data:o8f2,iv:U13muGbectPG41tMZgtmlDkzMdfQIWoP3pQwJRBH5SE=,tag:h5LwD5LIQhJqPwU+yXujkg==,type:str]
ZO_SMTP_USERNAME: ENC[AES256_GCM,data:gGt0Xp7HAPJMj28umdjCvGixdy9i65f+5i2sdjLa9ZY=,iv:z+KSvLdjyxr/0xYmk0Yb8140/7jieg41K1w2U3BT2Pk=,tag:NtIDdOPd9hA5TIDhz05b6A==,type:str]
ZO_SMTP_PASSWORD: ENC[AES256_GCM,data:v2BMTxQ9fgEsGGNYyiyzE/Xr46G732d/E9aitQbMqq46egDXrqjelyPn8J5dK0M+Oyo=,iv:CDlByQ/TZEr/8hZuTlcKeYdshib5z+wC39K/yfngiWQ=,tag:V4werptqvJoJr5mnYSh0hQ==,type:str]
ZO_SMTP_FROM_EMAIL: ENC[AES256_GCM,data:IdHjmM3ph8j2wR7U1Ayu9TcBvgIFeeQ6Q1p87RHGmB4=,iv:QxFXfcpoq7Z2Nkn7e6h8qTYn5Wt2LcveDHK3bvuFBP8=,tag:ZgyZtgOCTuZpJk3UDdG9xQ==,type:str]
ZO_SMTP_REPLY_TO: ENC[AES256_GCM,data:HtEazpWxxayEfuG2GBcMKam434BnmgYWFeLNCoWmQPg=,iv:fcgBJ+S+/X0L/vtKlP7PYbYaTPONy7VFyhW6r7BpumA=,tag:KEKtw1RwPpJYvWa6dHxQkQ==,type:str]
sops:
lastmodified: "2025-09-11T15:13:23Z"
mac: ENC[AES256_GCM,data:8aW1yhcqsgNTlHq45shvIaONm+4wd/5myj2e1CTbV+tSh2eA6u0Cj94DeifWxNPaX/wtlcb9atUrr3wuNAE6+k0UWoxVn6/2divipC7LtV7hLVQYwwB1xIm+aiAesILFg60BK0TKTlg6kgsPDJ74O0kKn09pm8pFKLBlO0pqj4E=,iv:4g75VE7di0FvzvCa8DCNSIILQroP1sK16tfTZRMBXKQ=,tag:lYykRQ21SdFC3TvYzXenOQ==,type:str]
pgp:
- created_at: "2025-09-11T15:04:12Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DZT3mpHTS/JgSAQdAx2g4TFggUbHlQySK6xGp6RvE03szSCAB3wKwneUrRi4w
uhj4z/S5sWG1wU46akQQdpdXfOp38uVPO+hNWl5pg4wyLAB3zTqi9CRPKJm6GflE
1GgBCQIQaxecQiWrs/IkjtHwilIGCFECizqpEg2DD3Y5zMVKgxDsnaFAXgeQmo0a
7BJaTABDnKh1sKQsAfED9dnSr63xmEUYPAdve6jn+No5IhF6fqkH06nppfKnxpAD
VUzF8FpItENOdg==
=s2tg
-----END PGP MESSAGE-----
fp: B120595CA9A643B051731B32E67FF350227BA4E8
- created_at: "2025-09-11T15:04:12Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hF4DSXzd60P2RKISAQdAcK2Bi/ozYs1mEHiqZ5oKzm6KAhqT6LYeK8xGjAmTzQAw
6bAfh7uN5TBza+cM4k7QQXfsgs2+39EGKRyFeitKW/WPORes5lMnsWsD/0zCLWWH
1GgBCQIQJZLult2JJmlrPTY1ILuuxfgzgV8Bh9yCDJDtyQJpsfKmPbqsUYC4Ner7
rMj6XA87dJEyRdxhxa2yx+/Wjd8RzcN9rgWQW+ruBsrPOvpAgUUvjDAMq/FIsdVI
pgurg1Z8+W0ldQ==
=p2GD
-----END PGP MESSAGE-----
fp: 4A8AADB4EBAB9AF88EF7062373CECE06CC80D40C
encrypted_regex: ^(data|stringData)$
version: 3.10.2

9
manifests/infrastructure/opentelemetry-operator/crds-kustomization.yaml Normal file
View File

@@ -0,0 +1,9 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
# Apply only the CRDs from OpenTelemetry operator
# This can be applied manually with: kubectl apply --server-side -k manifests/infrastructure/opentelemetry-operator/crds/
- https://raw.githubusercontent.com/open-telemetry/opentelemetry-operator/main/bundle/manifests/opentelemetry.io_instrumentations.yaml
- https://raw.githubusercontent.com/open-telemetry/opentelemetry-operator/main/bundle/manifests/opentelemetry.io_opentelemetrycollectors.yaml
- https://raw.githubusercontent.com/open-telemetry/opentelemetry-operator/main/bundle/manifests/opentelemetry.io_opampbridges.yaml
- https://raw.githubusercontent.com/open-telemetry/opentelemetry-operator/main/bundle/manifests/opentelemetry.io_targetallocators.yaml

11
manifests/infrastructure/opentelemetry-operator/kustomization.yaml Normal file
View File

@@ -0,0 +1,11 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
# Essential Prometheus operator CRDs for OpenTelemetry operator
- https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
- https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/main/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
- https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/main/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
- https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/main/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml
# OpenTelemetry operator with all required CRDs
- https://github.com/open-telemetry/opentelemetry-operator/releases/latest/download/opentelemetry-operator.yaml

6
manifests/infrastructure/opentelemetry-operator/namespace.yaml Normal file
View File

@@ -0,0 +1,6 @@
apiVersion: v1
kind: Namespace
metadata:
name: opentelemetry-system
labels:
name: opentelemetry-system

6
manifests/infrastructure/postgresql/Barman-cloud-plugin.md Normal file
View File

@@ -0,0 +1,6 @@
Aug 19, 2025
I tried to upgrade to the Barman Cloud plugin for backups instead of using longhorn,
but I couldn't get backups to work and ran into issues that I saw a lot of people online have.
I deleted the duplicate backups in postgres and went back to just longhorn backups. It's not as
ideal, but it actually works.

619
manifests/infrastructure/postgresql/POSTGRESQL-DISASTER-RECOVERY.md Normal file
View File

@@ -0,0 +1,619 @@
**This one was generated from the AI and I don't think it's quite right. I'll
go through it later.** I'm leaving it for reference.
# PostgreSQL CloudNativePG Disaster Recovery Guide
## 🚨 **CRITICAL: When to Use This Guide**
This guide is for **catastrophic failure scenarios** where:
- ✅ CloudNativePG cluster is completely broken/corrupted
- ✅ Longhorn volume backups are available (S3 or local snapshots)
- ✅ Normal CloudNativePG recovery methods have failed
- ✅ You need to restore from Longhorn backup volumes
**⚠️ WARNING**: This process involves temporary data exposure and should only be used when standard recovery fails.
---
## 📋 **Overview: Volume Adoption Strategy**
The key insight for CloudNativePG disaster recovery is using **Volume Adoption**:
1. **Restore Longhorn volumes** from backup
2. **Create fresh PVCs** with adoption annotations
3. **Deploy cluster with hibernation** to prevent initdb data erasure
4. **Retarget PVCs** to restored volumes
5. **Wake cluster** to adopt existing data
---
## 🛠️ **Step 1: Prepare for Recovery**
### 1.1 Clean Up Failed Cluster
```bash
# Remove broken cluster (DANGER: This deletes the cluster)
kubectl delete cluster postgres-shared -n postgresql-system
# Remove old PVCs if corrupted
kubectl delete pvc -n postgresql-system -l cnpg.io/cluster=postgres-shared
```
### 1.2 Identify Backup Volumes
```bash
# List available Longhorn backups
kubectl get volumebackup -n longhorn-system
# Note the backup names for data and WAL volumes:
# - postgres-shared-data-backup-20240809
# - postgres-shared-wal-backup-20240809
```
---
## 🔄 **Step 2: Restore Longhorn Volumes**
### 2.1 Create Volume Restore Jobs
```yaml
# longhorn-restore-data.yaml
apiVersion: longhorn.io/v1beta2
kind: Volume
metadata:
name: postgres-shared-data-recovered
namespace: longhorn-system
spec:
size: "400Gi"
numberOfReplicas: 2
fromBackup: "s3://your-bucket/@/longhorn?backup=backup-abcd1234&volume=postgres-shared-data"
# Replace with actual backup URL from Longhorn UI
---
# longhorn-restore-wal.yaml
apiVersion: longhorn.io/v1beta2
kind: Volume
metadata:
name: postgres-shared-wal-recovered
namespace: longhorn-system
spec:
size: "100Gi"
numberOfReplicas: 2
fromBackup: "s3://your-bucket/@/longhorn?backup=backup-efgh5678&volume=postgres-shared-wal"
# Replace with actual backup URL from Longhorn UI
```
Apply the restores:
```bash
kubectl apply -f longhorn-restore-data.yaml
kubectl apply -f longhorn-restore-wal.yaml
# Monitor restore progress
kubectl get volumes -n longhorn-system | grep recovered
```
### 2.2 Create PersistentVolumes for Restored Data
```yaml
# postgres-recovered-pvs.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: postgres-shared-data-recovered-pv
annotations:
pv.kubernetes.io/provisioned-by: driver.longhorn.io
spec:
capacity:
storage: 400Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Retain
storageClassName: longhorn-retain
csi:
driver: driver.longhorn.io
fsType: ext4
volumeAttributes:
numberOfReplicas: "2"
staleReplicaTimeout: "30"
volumeHandle: postgres-shared-data-recovered
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: postgres-shared-wal-recovered-pv
annotations:
pv.kubernetes.io/provisioned-by: driver.longhorn.io
spec:
capacity:
storage: 100Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Retain
storageClassName: longhorn-retain
csi:
driver: driver.longhorn.io
fsType: ext4
volumeAttributes:
numberOfReplicas: "2"
staleReplicaTimeout: "30"
volumeHandle: postgres-shared-wal-recovered
```
```bash
kubectl apply -f postgres-recovered-pvs.yaml
```
---
## 🎯 **Step 3: Create Fresh Cluster with Volume Adoption**
### 3.1 Create Adoption PVCs
```yaml
# postgres-adoption-pvcs.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: postgres-shared-1
namespace: postgresql-system
annotations:
# 🔑 CRITICAL: CloudNativePG adoption annotations
cnpg.io/cluster: postgres-shared
cnpg.io/instanceName: postgres-shared-1
cnpg.io/podRole: instance
# 🔑 CRITICAL: Prevent volume binding to wrong PV
volume.beta.kubernetes.io/storage-provisioner: driver.longhorn.io
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 400Gi
storageClassName: longhorn-retain
# 🔑 CRITICAL: This will be updated to point to recovered data later
volumeName: "" # Leave empty initially
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: postgres-shared-1-wal
namespace: postgresql-system
annotations:
# 🔑 CRITICAL: CloudNativePG adoption annotations
cnpg.io/cluster: postgres-shared
cnpg.io/instanceName: postgres-shared-1
cnpg.io/podRole: instance
cnpg.io/pvcRole: wal
volume.beta.kubernetes.io/storage-provisioner: driver.longhorn.io
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 100Gi
storageClassName: longhorn-retain
# 🔑 CRITICAL: This will be updated to point to recovered WAL later
volumeName: "" # Leave empty initially
```
```bash
kubectl apply -f postgres-adoption-pvcs.yaml
```
### 3.2 Deploy Cluster in Hibernation Mode
**🚨 CRITICAL**: The cluster MUST start in hibernation to prevent initdb from erasing your data!
```yaml
# postgres-shared-recovery.yaml
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: postgres-shared
namespace: postgresql-system
annotations:
# 🔑 CRITICAL: Hibernation prevents startup and data erasure
cnpg.io/hibernation: "on"
spec:
instances: 1
# 🔑 CRITICAL: Single instance prevents replication conflicts during recovery
minSyncReplicas: 0
maxSyncReplicas: 0
postgresql:
parameters:
# Performance and stability settings for recovery
max_connections: "200"
shared_buffers: "256MB"
effective_cache_size: "1GB"
maintenance_work_mem: "64MB"
checkpoint_completion_target: "0.9"
wal_buffers: "16MB"
default_statistics_target: "100"
random_page_cost: "1.1"
effective_io_concurrency: "200"
# 🔑 CRITICAL: Minimal logging during recovery
log_min_messages: "warning"
log_min_error_statement: "error"
log_statement: "none"
bootstrap:
# 🔑 CRITICAL: initdb bootstrap (NOT recovery mode)
# This will run even under hibernation
initdb:
database: postgres
owner: postgres
storage:
size: 400Gi
storageClass: longhorn-retain
walStorage:
size: 100Gi
storageClass: longhorn-retain
# 🔑 CRITICAL: Extended timeouts for recovery scenarios
startDelay: 3600 # 1 hour delay
stopDelay: 1800 # 30 minute stop delay
switchoverDelay: 1800 # 30 minute switchover delay
monitoring:
enabled: true
# Backup configuration (restore after recovery)
backup:
retentionPolicy: "7d"
barmanObjectStore:
destinationPath: "s3://your-backup-bucket/postgres-shared"
# Configure after cluster is stable
```
```bash
kubectl apply -f postgres-shared-recovery.yaml
# Verify cluster is hibernated (pods should NOT start)
kubectl get cluster postgres-shared -n postgresql-system
# Should show: STATUS = Hibernation
```
---
## 🔗 **Step 4: Retarget PVCs to Restored Data**
### 4.1 Generate Fresh PV UUIDs
```bash
# Generate new UUIDs for PV/PVC binding
DATA_PV_UUID=$(uuidgen | tr '[:upper:]' '[:lower:]')
WAL_PV_UUID=$(uuidgen | tr '[:upper:]' '[:lower:]')
echo "Data PV UUID: $DATA_PV_UUID"
echo "WAL PV UUID: $WAL_PV_UUID"
```
### 4.2 Patch PVs with Binding UUIDs
```bash
# Patch data PV
kubectl patch pv postgres-shared-data-recovered-pv -p "{
\"metadata\": {
\"uid\": \"$DATA_PV_UUID\"
},
\"spec\": {
\"claimRef\": {
\"name\": \"postgres-shared-1\",
\"namespace\": \"postgresql-system\",
\"uid\": \"$DATA_PV_UUID\"
}
}
}"
# Patch WAL PV
kubectl patch pv postgres-shared-wal-recovered-pv -p "{
\"metadata\": {
\"uid\": \"$WAL_PV_UUID\"
},
\"spec\": {
\"claimRef\": {
\"name\": \"postgres-shared-1-wal\",
\"namespace\": \"postgresql-system\",
\"uid\": \"$WAL_PV_UUID\"
}
}
}"
```
### 4.3 Patch PVCs with Matching UUIDs
```bash
# Patch data PVC
kubectl patch pvc postgres-shared-1 -n postgresql-system -p "{
\"metadata\": {
\"uid\": \"$DATA_PV_UUID\"
},
\"spec\": {
\"volumeName\": \"postgres-shared-data-recovered-pv\"
}
}"
# Patch WAL PVC
kubectl patch pvc postgres-shared-1-wal -n postgresql-system -p "{
\"metadata\": {
\"uid\": \"$WAL_PV_UUID\"
},
\"spec\": {
\"volumeName\": \"postgres-shared-wal-recovered-pv\"
}
}"
```
### 4.4 Verify PVC Binding
```bash
kubectl get pvc -n postgresql-system
# Both PVCs should show STATUS = Bound
```
---
## 🌅 **Step 5: Wake Cluster from Hibernation**
### 5.1 Remove Hibernation Annotation
```bash
# 🔑 CRITICAL: This starts the cluster with your restored data
kubectl annotate cluster postgres-shared -n postgresql-system cnpg.io/hibernation-
# Monitor cluster startup
kubectl get cluster postgres-shared -n postgresql-system -w
```
### 5.2 Monitor Pod Startup
```bash
# Watch pod creation and startup
kubectl get pods -n postgresql-system -l cnpg.io/cluster=postgres-shared -w
# Check logs for successful data adoption
kubectl logs postgres-shared-1 -n postgresql-system -f
```
**🔍 Expected Log Messages:**
```
INFO: PostgreSQL Database directory appears to contain a database
INFO: Looking at the contents of PostgreSQL database directory
INFO: Database found, skipping initialization
INFO: Starting PostgreSQL with recovered data
```
---
## 🔍 **Step 6: Verify Data Recovery**
### 6.1 Check Cluster Status
```bash
kubectl get cluster postgres-shared -n postgresql-system
# Should show: STATUS = Cluster in healthy state, PRIMARY = postgres-shared-1
```
### 6.2 Test Database Connectivity
```bash
# Test connection
kubectl exec postgres-shared-1 -n postgresql-system -- psql -c "\l"
# Verify all application databases exist
kubectl exec postgres-shared-1 -n postgresql-system -- psql -c "
SELECT datname, pg_size_pretty(pg_database_size(datname)) as size
FROM pg_database
WHERE datname NOT IN ('template0', 'template1', 'postgres')
ORDER BY pg_database_size(datname) DESC;
"
```
### 6.3 Verify Application Data
```bash
# Test specific application tables (example for Mastodon)
kubectl exec postgres-shared-1 -n postgresql-system -- psql mastodon_production -c "
SELECT COUNT(*) as total_accounts FROM accounts;
SELECT COUNT(*) as total_statuses FROM statuses;
"
```
---
## 📈 **Step 7: Scale to High Availability (Optional)**
### 7.1 Enable Replica Creation
```bash
# Scale cluster to 2 instances for HA
kubectl patch cluster postgres-shared -n postgresql-system -p '{
"spec": {
"instances": 2,
"minSyncReplicas": 0,
"maxSyncReplicas": 1
}
}'
```
### 7.2 Monitor Replica Join
```bash
# Watch replica creation and sync
kubectl get pods -n postgresql-system -l cnpg.io/cluster=postgres-shared -w
# Monitor replication lag
kubectl exec postgres-shared-1 -n postgresql-system -- psql -c "
SELECT client_addr, state, sent_lsn, write_lsn, flush_lsn, replay_lsn,
write_lag, flush_lag, replay_lag
FROM pg_stat_replication;
"
```
---
## 🔧 **Step 8: Application Connectivity (Service Aliases)**
### 8.1 Create Service Aliases for Application Compatibility
If your applications expect different service names (e.g., `postgresql-shared-*` vs `postgres-shared-*`):
```yaml
# postgresql-service-aliases.yaml
apiVersion: v1
kind: Service
metadata:
name: postgresql-shared-rw
namespace: postgresql-system
labels:
cnpg.io/cluster: postgres-shared
spec:
type: ClusterIP
ports:
- name: postgres
port: 5432
protocol: TCP
targetPort: 5432
selector:
cnpg.io/cluster: postgres-shared
cnpg.io/instanceRole: primary
---
apiVersion: v1
kind: Service
metadata:
name: postgresql-shared-ro
namespace: postgresql-system
labels:
cnpg.io/cluster: postgres-shared
spec:
type: ClusterIP
ports:
- name: postgres
port: 5432
protocol: TCP
targetPort: 5432
selector:
cnpg.io/cluster: postgres-shared
cnpg.io/instanceRole: replica
```
```bash
kubectl apply -f postgresql-service-aliases.yaml
```
### 8.2 Test Application Connectivity
```bash
# Test from application namespace
kubectl run test-connectivity --image=busybox --rm -it -- nc -zv postgresql-shared-rw.postgresql-system.svc.cluster.local 5432
```
---
## 🚨 **Troubleshooting Common Issues**
### Issue 1: Cluster Starts in initdb Mode (Data Loss Risk!)
**Symptoms**: Logs show "Initializing empty database"
**Solution**:
1. **IMMEDIATELY** scale cluster to 0 instances
2. Verify PVC adoption annotations are correct
3. Check that hibernation was properly used
```bash
kubectl patch cluster postgres-shared -n postgresql-system -p '{"spec":{"instances":0}}'
```
### Issue 2: PVC Binding Fails
**Symptoms**: PVCs stuck in "Pending" state
**Solution**:
1. Check PV/PVC UUID matching
2. Verify PV `claimRef` points to correct PVC
3. Ensure storage class exists
```bash
kubectl describe pvc postgres-shared-1 -n postgresql-system
kubectl describe pv postgres-shared-data-recovered-pv
```
### Issue 3: Pod Restart Loops
**Symptoms**: Pod continuously restarting with health check failures
**Solutions**:
1. Check Cilium network policies allow PostgreSQL traffic
2. Verify PostgreSQL data directory permissions
3. Check for TLS/SSL configuration issues
```bash
# Fix common permission issues
kubectl exec postgres-shared-1 -n postgresql-system -- chown -R postgres:postgres /var/lib/postgresql/data
```
### Issue 4: Replica Won't Join
**Symptoms**: Second instance fails to join with replication errors
**Solutions**:
1. Check primary is stable before adding replica
2. Verify network connectivity between pods
3. Monitor WAL streaming logs
```bash
# Check replication status
kubectl exec postgres-shared-1 -n postgresql-system -- psql -c "SELECT * FROM pg_stat_replication;"
```
---
## 📋 **Recovery Checklist**
**Pre-Recovery:**
- [ ] Backup current cluster state (if any)
- [ ] Identify Longhorn backup volume names
- [ ] Prepare fresh namespace if needed
- [ ] Verify Longhorn operator is functional
**Volume Restoration:**
- [ ] Restore data volume from Longhorn backup
- [ ] Restore WAL volume from Longhorn backup
- [ ] Create PersistentVolumes for restored data
- [ ] Verify volumes are healthy in Longhorn UI
**Cluster Recovery:**
- [ ] Create adoption PVCs with correct annotations
- [ ] Deploy cluster in hibernation mode
- [ ] Generate and assign PV/PVC UUIDs
- [ ] Patch PVs with claimRef binding
- [ ] Patch PVCs with volumeName binding
- [ ] Verify PVC binding before proceeding
**Startup:**
- [ ] Remove hibernation annotation
- [ ] Monitor pod startup logs for data adoption
- [ ] Verify cluster reaches healthy state
- [ ] Test database connectivity
**Validation:**
- [ ] Verify all application databases exist
- [ ] Test application table row counts
- [ ] Check database sizes match expectations
- [ ] Test application connectivity
**HA Setup (Optional):**
- [ ] Scale to 2+ instances
- [ ] Monitor replica join process
- [ ] Verify replication is working
- [ ] Test failover scenarios
**Cleanup:**
- [ ] Remove temporary PVs/PVCs
- [ ] Update backup configurations
- [ ] Document any configuration changes
- [ ] Test regular backup/restore procedures
---
## ⚠️ **CRITICAL SUCCESS FACTORS**
1. **🔑 Hibernation is MANDATORY**: Never start a cluster without hibernation when adopting existing data
2. **🔑 Single Instance First**: Always recover to single instance, then scale to HA
3. **🔑 UUID Matching**: PV and PVC UIDs must match exactly for binding
4. **🔑 Adoption Annotations**: CloudNativePG annotations must be present on PVCs
5. **🔑 Volume Naming**: PVC names must match CloudNativePG instance naming convention
6. **🔑 Network Policies**: Ensure Cilium policies allow PostgreSQL traffic
7. **🔑 Monitor Logs**: Watch startup logs carefully for data adoption confirmation
---
## 📚 **Additional Resources**
- [CloudNativePG Documentation](https://cloudnative-pg.io/documentation/)
- [Longhorn Backup & Restore](https://longhorn.io/docs/1.4.0/volumes-and-nodes/backup-and-restore/)
- [Kubernetes Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/)
- [PostgreSQL Recovery Documentation](https://www.postgresql.org/docs/current/backup-dump.html)
---
**🎉 This disaster recovery procedure has been tested and proven successful in production environments!**

508
manifests/infrastructure/postgresql/POSTGRESQL-PARTITIONING-STRATEGY.md Normal file
View File

@@ -0,0 +1,508 @@
Below is Claude's recommendation for a guide to partition tables in postgres for piefed and it seems similar to the
official docs, though I'd prefer that and use this as a reference. This guide sets up automatic backup functions,
which is nice. The reason I was looking in to this is that I've noticed about 500MB growth
in about a week and the largest tables are for votes, which wouldn't compress well. I think I would wait a bit longer to
do the partitioning migration than in the next few weeks (and also test it in a lower env), since even if 300GB is available
to the DB per node, that's still 600 weeks, so plenty of time. Piefed is talking about automatic backup of older posts to S3,
but that table was only about 80MB for me and it would probably do well to eventually compress it.
# PostgreSQL Partitioning Strategy for PieFed Database Growth
## 📊 **Current Status & Growth Analysis**
### **Database Size Assessment (August 2025)**
- **PieFed Database**: 975 MB (largest database in cluster)
- **Growth Rate**: 500 MB per week
- **Largest Tables**:
- `post_vote`: 280 MB (1,167,833 rows) - 20 days of data
- `post_reply_vote`: 271 MB (1,185,985 rows)
- `post_reply`: 201 MB
- `user`: 104 MB
### **Growth Projections**
- **Daily vote activity**: ~58,000 votes/day
- **Annual projection**: ~21M votes/year = ~5.1GB for `post_vote` alone
- **Total database projection**: 15-20GB annually across all tables
- **3-year projection**: 45-60GB total database size
## 🎯 **When to Begin Partitioning**
### **Trigger Points for Implementation**
#### **Phase 1: Immediate Planning (Current)**
- ✅ **Database size**: 975 MB (threshold: >500 MB)
- ✅ **Growth rate**: 500 MB/week (threshold: >100 MB/week)
- ✅ **Infrastructure capacity**: 400GB available per node
#### **Phase 2: Infrastructure Preparation (Next 1-2 months)**
**Trigger**: When database reaches 1.5-2GB
- Current trajectory: ~4-6 weeks from now
- **Action**: Add NetCup block storage volumes
- **Rationale**: Prepare infrastructure before partitioning implementation
#### **Phase 3: Partitioning Implementation (2-3 months)**
**Trigger**: When `post_vote` table reaches 500 MB or 2M rows
- Current trajectory: ~6-8 weeks from now
- **Action**: Implement time-based partitioning
- **Rationale**: Optimal size for initial partitioning without excessive complexity
#### **Phase 4: Archive Migration (3-4 months)**
**Trigger**: When historical data older than 3 months exists
- Current trajectory: ~12-16 weeks from now
- **Action**: Move old partitions to archive storage
- **Rationale**: Cost optimization for infrequently accessed data
## 🏗️ **Infrastructure Architecture**
### **Current Setup**
```yaml
# Current PostgreSQL Storage Configuration
storage:
size: 50Gi
storageClass: longhorn-postgresql
walStorage:
size: 10Gi
storageClass: longhorn-postgresql
```
### **Target Architecture**
```yaml
# Enhanced Multi-Volume Configuration
storage:
size: 50Gi # Recent data (2-3 months)
storageClass: longhorn-postgresql
walStorage:
size: 10Gi
storageClass: longhorn-postgresql
tablespaces:
- name: archive_data # Historical data (>3 months)
size: 500Gi
storageClass: netcup-block-storage
- name: temp_operations # Temporary operations
size: 100Gi
storageClass: netcup-block-storage
```
## 📋 **Implementation Plan**
### **Phase 1: Infrastructure Preparation**
#### **1.1 Add NetCup Block Storage**
```bash
# On each VPS (n1, n2, n3)
# 1. Attach 500GB block storage via NetCup control panel
# 2. Format and mount new volumes
sudo mkfs.ext4 /dev/sdb
sudo mkdir -p /mnt/postgres-archive
sudo mount /dev/sdb /mnt/postgres-archive
sudo chown 999:999 /mnt/postgres-archive
# Add to /etc/fstab for persistence
echo "/dev/sdb /mnt/postgres-archive ext4 defaults 0 2" >> /etc/fstab
```
#### **1.2 Create Storage Classes**
```yaml
# manifests/infrastructure/postgresql/netcup-block-storage.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: netcup-block-storage
provisioner: kubernetes.io/host-path
parameters:
type: Directory
path: /mnt/postgres-archive
volumeBindingMode: WaitForFirstConsumer
reclaimPolicy: Retain
```
#### **1.3 Update CloudNativePG Configuration**
```yaml
# manifests/infrastructure/postgresql/cluster-shared.yaml
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: postgres-shared
spec:
instances: 3
storage:
size: 50Gi
storageClass: longhorn-postgresql
walStorage:
size: 10Gi
storageClass: longhorn-postgresql
# Add tablespaces for multi-volume storage
tablespaces:
- name: archive_data
size: 500Gi
storageClass: netcup-block-storage
- name: temp_operations
size: 100Gi
storageClass: netcup-block-storage
# Enable partitioning extensions
bootstrap:
initdb:
database: shared_db
owner: shared_user
postInitSQL:
- "CREATE EXTENSION IF NOT EXISTS pg_partman"
- "CREATE EXTENSION IF NOT EXISTS pg_cron"
```
### **Phase 2: Partitioning Implementation**
#### **2.1 Install Required Extensions**
```sql
-- Connect to PieFed database
kubectl exec -n postgresql-system postgres-shared-2 -- psql -U postgres -d piefed
-- Install partitioning and scheduling extensions
CREATE EXTENSION IF NOT EXISTS pg_partman;
CREATE EXTENSION IF NOT EXISTS pg_cron;
-- Verify installation
SELECT name, default_version, installed_version
FROM pg_available_extensions
WHERE name IN ('pg_partman', 'pg_cron');
```
#### **2.2 Create Tablespaces**
```sql
-- Create tablespace for archive data
CREATE TABLESPACE archive_data LOCATION '/var/lib/postgresql/tablespaces/archive_data';
-- Create tablespace for temporary operations
CREATE TABLESPACE temp_operations LOCATION '/var/lib/postgresql/tablespaces/temp_operations';
-- Verify tablespaces
SELECT spcname, pg_tablespace_location(oid) FROM pg_tablespace;
```
#### **2.3 Partition the post_vote Table**
**Step 1: Backup Current Data**
```sql
-- Create backup of current table
CREATE TABLE post_vote_backup AS SELECT * FROM post_vote;
```
**Step 2: Create Partitioned Table Structure**
```sql
-- Rename existing table
ALTER TABLE post_vote RENAME TO post_vote_legacy;
-- Create new partitioned table
CREATE TABLE post_vote (
id INTEGER NOT NULL,
user_id INTEGER,
author_id INTEGER,
post_id INTEGER,
effect DOUBLE PRECISION,
created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL,
PRIMARY KEY (id, created_at) -- Include partition key in PK
) PARTITION BY RANGE (created_at);
-- Create indexes
CREATE INDEX idx_post_vote_created_at ON post_vote (created_at);
CREATE INDEX idx_post_vote_user_id ON post_vote (user_id);
CREATE INDEX idx_post_vote_post_id ON post_vote (post_id);
CREATE INDEX idx_post_vote_author_id ON post_vote (author_id);
```
**Step 3: Configure Automated Partitioning**
```sql
-- Set up pg_partman for monthly partitions
SELECT partman.create_parent(
p_parent_table => 'public.post_vote',
p_control => 'created_at',
p_type => 'range',
p_interval => 'monthly',
p_premake => 3, -- Pre-create 3 future partitions
p_start_partition => '2025-07-01' -- Start from July 2025
);
-- Configure retention and archive settings
UPDATE partman.part_config
SET retention = '12 months',
retention_keep_table = true,
infinite_time_partitions = true,
optimize_constraint = 30
WHERE parent_table = 'public.post_vote';
```
**Step 4: Create Initial Partitions**
```sql
-- Create July 2025 partition (historical data)
CREATE TABLE post_vote_p2025_07 PARTITION OF post_vote
FOR VALUES FROM ('2025-07-01') TO ('2025-08-01')
TABLESPACE archive_data; -- Place on archive storage
-- Create August 2025 partition (recent data)
CREATE TABLE post_vote_p2025_08 PARTITION OF post_vote
FOR VALUES FROM ('2025-08-01') TO ('2025-09-01'); -- Default tablespace
-- Create September 2025 partition (future data)
CREATE TABLE post_vote_p2025_09 PARTITION OF post_vote
FOR VALUES FROM ('2025-09-01') TO ('2025-10-01'); -- Default tablespace
```
**Step 5: Migrate Data**
```sql
-- Migrate data from legacy table
INSERT INTO post_vote
SELECT * FROM post_vote_legacy
ORDER BY created_at;
-- Verify data migration
SELECT
'Legacy' as source, COUNT(*) as row_count FROM post_vote_legacy
UNION ALL
SELECT
'Partitioned' as source, COUNT(*) as row_count FROM post_vote;
-- Check partition distribution
SELECT
schemaname,
tablename,
pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) as size,
(SELECT COUNT(*) FROM information_schema.table_constraints
WHERE table_name = pg_tables.tablename AND constraint_type = 'CHECK') as partition_count
FROM pg_tables
WHERE tablename LIKE 'post_vote_p%'
ORDER BY tablename;
```
#### **2.4 Set Up Automated Partition Management**
```sql
-- Create function to automatically move old partitions to archive storage
CREATE OR REPLACE FUNCTION move_old_partitions_to_archive()
RETURNS void AS $$
DECLARE
partition_name text;
archive_threshold date;
BEGIN
-- Move partitions older than 3 months to archive storage
archive_threshold := CURRENT_DATE - INTERVAL '3 months';
FOR partition_name IN
SELECT schemaname||'.'||tablename
FROM pg_tables
WHERE tablename LIKE 'post_vote_p%'
AND tablename < 'post_vote_p' || TO_CHAR(archive_threshold, 'YYYY_MM')
LOOP
-- Move partition to archive tablespace
EXECUTE format('ALTER TABLE %s SET TABLESPACE archive_data', partition_name);
RAISE NOTICE 'Moved partition % to archive storage', partition_name;
END LOOP;
END;
$$ LANGUAGE plpgsql;
-- Schedule monthly archive operations
SELECT cron.schedule(
'move-old-partitions',
'0 2 1 * *', -- 2 AM on the 1st of each month
'SELECT move_old_partitions_to_archive()'
);
-- Schedule partition maintenance
SELECT cron.schedule(
'partition-maintenance',
'0 1 * * 0', -- 1 AM every Sunday
'SELECT partman.run_maintenance_proc()'
);
```
### **Phase 3: Extend to Other Large Tables**
#### **3.1 Partition post_reply_vote Table**
```sql
-- Similar process for post_reply_vote (271 MB)
-- Follow same steps as post_vote table
```
#### **3.2 Partition post_reply Table**
```sql
-- Similar process for post_reply (201 MB)
-- Consider partitioning by created_at or parent post date
```
## 📊 **Monitoring and Maintenance**
### **Performance Monitoring Queries**
#### **Partition Size Monitoring**
```sql
-- Monitor partition sizes and locations
SELECT
schemaname,
tablename,
pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) as size,
pg_tablespace_name(reltablespace) as tablespace,
(SELECT COUNT(*) FROM information_schema.columns
WHERE table_name = pg_tables.tablename) as column_count
FROM pg_tables
WHERE tablename LIKE 'post_vote_p%'
ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC;
```
#### **Query Performance Analysis**
```sql
-- Analyze query performance across partitions
EXPLAIN (ANALYZE, BUFFERS)
SELECT COUNT(*)
FROM post_vote
WHERE created_at >= '2025-01-01'
AND created_at < '2025-12-31';
```
#### **Partition Pruning Verification**
```sql
-- Verify partition pruning is working
EXPLAIN (ANALYZE, BUFFERS)
SELECT *
FROM post_vote
WHERE created_at >= '2025-08-01'
AND created_at < '2025-09-01';
```
### **Storage Usage Monitoring**
```bash
# Monitor tablespace usage
kubectl exec -n postgresql-system postgres-shared-2 -- psql -U postgres -c "
SELECT
spcname as tablespace_name,
pg_tablespace_location(oid) as location,
pg_size_pretty(pg_tablespace_size(oid)) as size
FROM pg_tablespace
WHERE spcname NOT IN ('pg_default', 'pg_global');
"
# Monitor PVC usage
kubectl get pvc -n postgresql-system
kubectl describe pvc -n postgresql-system
```
### **Automated Maintenance Jobs**
```sql
-- View scheduled maintenance jobs
SELECT
jobname,
schedule,
command,
active,
jobid
FROM cron.job
ORDER BY jobname;
-- Check partition maintenance logs
SELECT * FROM partman.part_config_sub;
```
## 🚨 **Troubleshooting Guide**
### **Common Issues and Solutions**
#### **Issue: Partition Creation Fails**
```sql
-- Check partition configuration
SELECT * FROM partman.part_config WHERE parent_table = 'public.post_vote';
-- Manually create missing partition
SELECT partman.create_parent(
p_parent_table => 'public.post_vote',
p_control => 'created_at',
p_type => 'range',
p_interval => 'monthly'
);
```
#### **Issue: Query Not Using Partition Pruning**
```sql
-- Check if constraint exclusion is enabled
SHOW constraint_exclusion;
-- Enable if needed
SET constraint_exclusion = partition;
-- Update statistics
ANALYZE post_vote;
```
#### **Issue: Tablespace Out of Space**
```bash
# Check tablespace usage
df -h /mnt/postgres-archive
# Add additional block storage if needed
# Follow NetCup documentation for volume expansion
```
## 📖 **Documentation References**
### **CloudNativePG Documentation**
- [Tablespaces](https://cloudnative-pg.io/documentation/current/tablespaces/) - Official tablespace configuration guide
- [FAQ](https://cloudnative-pg.io/documentation/current/faq/) - Database management best practices
- [Controller](https://cloudnative-pg.io/documentation/current/controller/) - Storage management concepts
### **PostgreSQL Documentation**
- [Declarative Partitioning](https://www.postgresql.org/docs/16/ddl-partitioning.html) - Official partitioning guide
- [Tablespaces](https://www.postgresql.org/docs/16/manage-ag-tablespaces.html) - Tablespace management
- [pg_partman Extension](https://github.com/pgpartman/pg_partman) - Automated partition management
### **NetCup Documentation**
- [Block Storage](https://www.netcup.eu/bestellen/produkt.php?produkt=2594) - Block storage attachment guide
- [VPS Management](https://www.netcup.eu/vserver/) - VPS configuration documentation
## 🎯 **Success Metrics**
### **Performance Targets**
- **Recent data queries**: <250ms (50% improvement from current 506ms)
- **Historical data queries**: <800ms (acceptable for archive storage)
- **Storage cost reduction**: 70% for historical data
- **Backup time improvement**: 60% reduction for recent data backups
### **Capacity Planning**
- **Primary storage**: Maintain 50GB for 2-3 months of recent data
- **Archive storage**: Scale to 500GB initially, expand as needed
- **Growth accommodation**: Support 20GB/year growth for 25+ years
### **Operational Goals**
- **Zero downtime**: All operations performed online
- **Application transparency**: No code changes required
- **Automated management**: Minimal manual intervention
- **Disaster recovery**: Independent backup strategies per tier
## 📅 **Implementation Timeline**
| Phase | Duration | Key Deliverables |
|-------|----------|------------------|
| **Infrastructure Prep** | 2 weeks | NetCup block storage attached, storage classes configured |
| **Partitioning Setup** | 1 week | Extensions installed, tablespaces created |
| **post_vote Migration** | 1 week | Partitioned table structure, data migration |
| **Automation Setup** | 1 week | Automated partition management, monitoring |
| **Other Tables** | 2 weeks | post_reply_vote and post_reply partitioning |
| **Testing & Optimization** | 1 week | Performance testing, fine-tuning |
**Total Implementation Time**: 8 weeks
## ✅ **Pre-Implementation Checklist**
- [ ] NetCup block storage volumes attached to all nodes
- [ ] Storage classes created and tested
- [ ] CloudNativePG cluster configuration updated
- [ ] Backup of current database completed
- [ ] pg_partman and pg_cron extensions available
- [ ] Monitoring queries prepared
- [ ] Rollback plan documented
- [ ] Team training on partition management completed
---
**Last Updated**: August 2025
**Next Review**: September 2025
**Owner**: Database Administration Team

76
manifests/infrastructure/postgresql/PostgresqlVolumeRestore.md Normal file
View File

@@ -0,0 +1,76 @@
# Recovering a partition from Longhorn Backup volume
## Pull the volume in the longhorn ui
Under backups, choose which ones to restore (data and wal). Be sure that the replica count is 1,
the ReadWrite mode is ReadWriteOne. This should match what you had for the Pg volumes.
Get the volumes onto the same node. You may need to attach them, change the replica count,
then delete off of the undesired node.
## Swap the Volume under the PVC
Put CNPG into hibernate mode and wait for the database nodes to clear.
```yaml
cluster
metadata:
name: postgres-shared
namespace: postgresql-system
annotations:
# 🔑 CRITICAL: Hibernation prevents startup and data erasure
cnpg.io/hibernation: "on"
spec:
instances: 1 # it's way easier to start with one instance
# put the cluster into single node configuration
minSyncReplicas: 0
maxSyncReplicas: 0
```
If you haven't deleted the db cluster you should be able to use the same volume names as the preivous primary.
If you did, then you'll use postgresql-shared-1 or whatever your naming scheme is. But wait to make them
until AFTER the initdb runs the first time. If you are starting over, you'll have to set the
annotation for `lastGeneratedNode` to 0.
`kubectl patch clusters.postgresql.cnpg.io mydb --type=merge --subresource status --patch 'status: {latestGeneratedNode: 0}'` so that it'll create the first instance.
You'll also want to use a new PVC so that initdb clears out the data and then swap in your volume into that one.
Once you're past this stage, put it back into hibernation mode.
(why did I delete the files???)
Anyway, you need to swap the volume out from under the PVC that you're going to use.
You'll make a new pvc and set the (target?) uuid that identifies the volume to a new value.
I think this comes from longhorn. Make sure that the volume labels match the names of your recovery volumes.
Then you'll have to make sure that your PVCs are annotated with the same annotations on your previous PVCs
since CNPG puts it's own annotations on them. It'll look like the below from https://github.com/cloudnative-pg/cloudnative-pg/issues/5235. Make sure that versions and everything else matches. You need these otherwise the operator won't find a volume to use.
```yaml
annotations:
cnpg.io/nodeSerial: "1"
cnpg.io/operatorVersion: 1.24.0
cnpg.io/pvcStatus: ready
pv.kubernetes.io/bind-completed: "yes"
pv.kubernetes.io/bound-by-controller: "yes"
volume.beta.kubernetes.io/storage-provisioner: driver.longhorn.io
volume.kubernetes.io/storage-provisioner: driver.longhorn.io
finalizers:
- kubernetes.io/pvc-protection
labels:
cnpg.io/cluster: mydb
cnpg.io/instanceName: mydb-1
cnpg.io/instanceRole: primary
cnpg.io/pvcRole: PG_DATA
role: primary
name: mydb-1
namespace: mydb
ownerReferences:
- apiVersion: postgresql.cnpg.io/v1
controller: true
kind: Cluster
name: mydb
uid: f1111111-111a-111f-111d-11111111111f
```
### Go out of hibernation mode.
You should see your pod come up and be functional, without an initdb pod. Check it.
After a while, scale it back up.

341
manifests/infrastructure/postgresql/README.md Normal file
View File

@@ -0,0 +1,341 @@
# PostgreSQL Infrastructure
This directory contains the CloudNativePG setup for high-availability PostgreSQL on the Kubernetes cluster.
## Architecture
- **3 PostgreSQL instances**: 1 primary + 2 replicas for high availability
- **Synchronous replication**: Zero data loss (RPO=0) configuration
- **Node distribution**: Instances distributed across n1, n2, and n3 nodes
- **Current cluster**: `postgres-shared` with instances `postgres-shared-2` (primary), `postgres-shared-4`, `postgres-shared-5`
- **Longhorn storage**: Single replica (PostgreSQL handles replication)
- **Shared cluster**: One PostgreSQL cluster that applications can share
## Components
### **Core Components**
- `namespace.yaml`: PostgreSQL system namespace
- `repository.yaml`: CloudNativePG Helm repository
- `operator.yaml`: CloudNativePG operator deployment
- `postgresql-storageclass.yaml`: Optimized storage class for PostgreSQL
- `cluster-shared.yaml`: Shared PostgreSQL cluster configuration
### **Monitoring Components**
- `postgresql-dashboard-metrics.yaml`: Custom metrics ConfigMap for enhanced monitoring
- `postgresql-dashboard-rbac.yaml`: RBAC permissions for metrics collection
- Built-in ServiceMonitor: Automatically configured for OpenObserve integration
### **Backup Components**
- `backup-config.yaml`: CloudNativePG backup configuration
- Longhorn integration: S3 backup via label-based volume selection
## Services Created
CloudNativePG automatically creates these services:
- `postgresql-shared-rw`: Write operations (connects to primary)
- `postgresql-shared-ro`: Read-only operations (connects to replicas)
- `postgresql-shared-r`: Read operations (connects to any instance)
## Connection Information
### For Applications
Applications should connect using these connection parameters:
**Write Operations:**
```yaml
host: postgresql-shared-rw.postgresql-system.svc.cluster.local
port: 5432
database: shared_db
username: shared_user
```
**Read Operations:**
```yaml
host: postgresql-shared-ro.postgresql-system.svc.cluster.local
port: 5432
database: shared_db
username: shared_user
```
### Getting Credentials
The PostgreSQL password is auto-generated and stored in a secret:
```bash
# Get the password for the shared_user
kubectl get secret postgresql-shared-app -n postgresql-system -o jsonpath="{.data.password}" | base64 -d
# Get the superuser password
kubectl get secret postgresql-shared-superuser -n postgresql-system -o jsonpath="{.data.password}" | base64 -d
```
## Application Integration Example
Here's how an application deployment would connect:
```yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: example-app
spec:
template:
spec:
containers:
- name: app
image: example-app:latest
env:
- name: DB_HOST
value: "postgresql-shared-rw.postgresql-system.svc.cluster.local"
- name: DB_PORT
value: "5432"
- name: DB_NAME
value: "shared_db"
- name: DB_USER
value: "shared_user"
- name: DB_PASSWORD
valueFrom:
secretKeyRef:
name: postgresql-shared-app
key: password
```
## Monitoring
The PostgreSQL cluster includes comprehensive monitoring and observability:
### **Metrics & Monitoring** ✅ **OPERATIONAL**
- **Metrics Port**: 9187 - PostgreSQL metrics endpoint
- **ServiceMonitor**: Configured for OpenObserve integration
- **Built-in Metrics**: CloudNativePG provides extensive default metrics including:
- **Connection Metrics**: `cnpg_backends_total`, `cnpg_pg_settings_setting{name="max_connections"}`
- **Performance Metrics**: `cnpg_pg_stat_database_xact_commit`, `cnpg_pg_stat_database_xact_rollback`
- **Storage Metrics**: `cnpg_pg_database_size_bytes`, `cnpg_pg_stat_database_blks_hit`, `cnpg_pg_stat_database_blks_read`
- **Cluster Health**: `cnpg_collector_up`, `cnpg_collector_postgres_version`
- **Replication**: `cnpg_pg_stat_replication_*` metrics for streaming replication status
### **Custom Metrics System**
- **ConfigMap Support**: Custom queries can be defined via ConfigMaps
- **RBAC Configured**: PostgreSQL service account has permissions to read custom metrics ConfigMaps
- **Predefined Queries**: CloudNativePG includes `cnpg-default-monitoring` ConfigMap with standard queries
- **Monitoring Role**: Uses `pg_monitor` role for secure metrics collection
### **Dashboard Integration**
- **OpenObserve Ready**: All metrics automatically ingested into OpenObserve
- **Key Performance Indicators**:
- Connection utilization: `cnpg_backends_total / cnpg_pg_settings_setting{name="max_connections"} * 100`
- Buffer cache hit ratio: `cnpg_pg_stat_database_blks_hit / (cnpg_pg_stat_database_blks_hit + cnpg_pg_stat_database_blks_read) * 100`
- Transaction rate: `rate(cnpg_pg_stat_database_xact_commit[5m])`
- Rollback ratio: `cnpg_pg_stat_database_xact_rollback / (cnpg_pg_stat_database_xact_commit + cnpg_pg_stat_database_xact_rollback) * 100`
### **High Availability Monitoring**
- **Automatic Failover**: CloudNativePG handles primary/replica failover automatically
- **Health Checks**: Continuous health monitoring with automatic recovery
- **Streaming Replication**: Real-time replication status monitoring
## Backup Strategy
### **Longhorn Storage-Level Backups (Incremental)**
- **Daily backups**: 2 AM UTC, retain 14 days (2 weeks)
- **Weekly backups**: 1 AM Sunday, retain 8 weeks (2 months)
- **Snapshot cleanup**: 3 AM daily, keep 5 local snapshots
- **Target**: Backblaze B2 S3 storage via existing setup
- **Type**: Incremental (efficient change block detection)
### **CloudNativePG Application-Level Backups**
- **WAL archiving**: Continuous transaction log archiving
- **Point-in-time recovery**: Available via CloudNativePG
- **Retention**: 30-day backup retention policy
### **Backup Labels**
PostgreSQL volumes are automatically backed up based on labels:
```yaml
backup.longhorn.io/enable: "true"
app: postgresql-shared
```
## Scaling
To add more read replicas:
```yaml
# Edit cluster-shared.yaml
spec:
instances: 4 # Increase from 3 to 4 for additional read replica
```
## Troubleshooting
### **Cluster Status**
```bash
# Check cluster status
kubectl get cluster -n postgresql-system
kubectl describe cluster postgresql-shared -n postgresql-system
# Check pods
kubectl get pods -n postgresql-system
kubectl logs postgres-shared-2 -n postgresql-system # Current primary
```
### **Monitoring & Metrics**
```bash
# Check ServiceMonitor
kubectl get servicemonitor -n postgresql-system
kubectl describe servicemonitor postgresql-shared -n postgresql-system
# Check metrics endpoint directly
kubectl port-forward -n postgresql-system postgres-shared-2 9187:9187 # Primary instance
curl http://localhost:9187/metrics
# Check custom metrics ConfigMap
kubectl get configmap -n postgresql-system
kubectl describe configmap postgresql-dashboard-metrics -n postgresql-system
# Check RBAC permissions
kubectl get role,rolebinding -n postgresql-system
kubectl describe rolebinding postgresql-dashboard-metrics-reader -n postgresql-system
```
### **Port Forwarding**
Port forwarding allows you to connect to PostgreSQL from your local machine using standard database tools.
**⚠️ Important**: PostgreSQL requires SSL/TLS connections. When port forwarding, you must configure your client to handle SSL properly.
**Read-Only Replica (Load Balanced):**
```bash
# Forward to read-only service (load balances across all replicas)
kubectl port-forward -n postgresql-system svc/postgresql-shared-ro 5432:5432
# Get the password for shared_user
kubectl get secret postgres-shared-app -n postgresql-system -o jsonpath='{.data.password}' | base64 -d && echo
# Connect with SSL required (recommended):
# Connection string: postgresql://shared_user:<password>@localhost:5432/shared_db?sslmode=require
# Or configure your client:
# - host: localhost
# - port: 5432
# - database: shared_db
# - username: shared_user
# - password: <from secret above>
# - SSL mode: require (or disable for testing only)
```
**Specific Replica Pod:**
```bash
# List replica pods
kubectl get pods -n postgresql-system -l cnpg.io/instanceRole=replica
# Forward to specific replica pod (e.g., postgres-shared-4)
kubectl port-forward -n postgresql-system pod/postgres-shared-4 5432:5432
# Get the password for shared_user
kubectl get secret postgres-shared-app -n postgresql-system -o jsonpath='{.data.password}' | base64 -d && echo
# Connect with SSL required (recommended):
# Connection string: postgresql://shared_user:<password>@localhost:5432/shared_db?sslmode=require
# Or configure your client with SSL mode: require
```
**Primary (Read-Write) - For Maintenance Only:**
```bash
# Forward to read-write service (connects to primary)
kubectl port-forward -n postgresql-system svc/postgresql-shared-rw 5433:5432
# Note: Using port 5433 locally to avoid conflict if read-only is on 5432
# Get the password
kubectl get secret postgres-shared-app -n postgresql-system -o jsonpath='{.data.password}' | base64 -d && echo
# Connect using localhost:5433 with SSL mode: require
```
**SSL Configuration Notes:**
- **SSL is enabled** on PostgreSQL (ssl = on)
- For **port forwarding**, clients must explicitly configure SSL mode
- The server uses self-signed certificates, so clients will need to accept untrusted certificates
- For production clients connecting directly (not via port-forward), use proper SSL with CA verification
**Troubleshooting Port Forward "Broken Pipe" Errors:**
If you see `error: lost connection to pod` or `broken pipe` errors:
1. **Use direct pod port forwarding** instead of service port forwarding (more reliable):
```bash
# List available replica pods
kubectl get pods -n postgresql-system -l cnpg.io/instanceRole=replica
# Forward to specific replica pod (more stable)
kubectl port-forward -n postgresql-system pod/postgres-shared-4 5432:5432
```
2. **Configure your client with explicit SSL mode**:
- Use `sslmode=require` in your connection string (recommended)
- Or `sslmode=prefer` (allows fallback to non-SSL if SSL fails)
- Or `sslmode=disable` for testing only (not recommended)
3. **Connection string examples**:
```bash
# With SSL required (recommended)
postgresql://shared_user:<password>@localhost:5432/shared_db?sslmode=require
# With SSL preferred (allows fallback)
postgresql://shared_user:<password>@localhost:5432/shared_db?sslmode=prefer
# Without SSL (testing only)
postgresql://shared_user:<password>@localhost:5432/shared_db?sslmode=disable
```
**Getting the CA Certificate (for proper SSL verification):**
```bash
# Get the CA certificate from the cluster secret
kubectl get secret postgres-shared-ca -n postgresql-system -o jsonpath='{.data.ca\.crt}' | base64 -d > postgres-ca.crt
# Use with your client:
# Connection string: postgresql://shared_user:<password>@localhost:5432/shared_db?sslmode=verify-ca&sslrootcert=postgres-ca.crt
# Or configure your client to use the CA certificate file for SSL verification
```
### **Database Connection**
```bash
# Connect to PostgreSQL via exec
kubectl exec -it postgres-shared-2 -n postgresql-system -- psql -U shared_user -d shared_db
# Check replication status
kubectl exec -it postgres-shared-2 -n postgresql-system -- psql -U postgres -c "SELECT * FROM pg_stat_replication;"
# Check cluster health
kubectl exec -it postgres-shared-2 -n postgresql-system -- psql -U postgres -c "SELECT pg_is_in_recovery();"
```
### **Backup & Storage**
```bash
# Check PVC status
kubectl get pvc -n postgresql-system
kubectl describe pvc postgres-shared-2 -n postgresql-system # Primary instance PVC
# Check Longhorn volumes
kubectl get volumes -n longhorn-system
kubectl describe volume -n longhorn-system | grep postgresql
```
### **Long Running Queries**
When a long running query is happening, use this command on a node
```bash
kubectl exec -n postgresql-system postgres-shared-2 -- psql -U postgres -c "
SELECT
pid,
datname,
usename,
application_name,
now() - xact_start AS tx_duration,
now() - query_start AS query_duration,
state,
wait_event_type,
wait_event,
query
FROM pg_stat_activity
WHERE state != 'idle'
AND query NOT LIKE '%pg_stat_activity%'
AND (now() - xact_start > interval '10 seconds' OR now() - query_start > interval '10 seconds')
ORDER BY GREATEST(now() - xact_start, now() - query_start) DESC;
"
```

60
manifests/infrastructure/postgresql/backup-config.yaml Normal file
View File

@@ -0,0 +1,60 @@
---
# Longhorn Recurring Job for PostgreSQL Backup
apiVersion: longhorn.io/v1beta2
kind: RecurringJob
metadata:
name: postgresql-backup-daily
namespace: longhorn-system
spec:
# Incremental backup (snapshot-based)
task: backup
cron: "0 2 * * *" # Daily at 2 AM UTC
retain: 14 # Keep 14 daily backups (2 weeks)
concurrency: 2 # Max 2 concurrent backup operations
# Target PostgreSQL volumes using group-based selection
groups:
- postgresql-backup
# Labels for the recurring job itself
labels:
recurring-job: "postgresql-backup-daily"
backup-type: "daily"
---
# Weekly backup for longer retention
apiVersion: longhorn.io/v1beta2
kind: RecurringJob
metadata:
name: postgresql-backup-weekly
namespace: longhorn-system
spec:
task: backup
cron: "0 1 * * 0" # Weekly at 1 AM on Sunday
retain: 8 # Keep 8 weekly backups (2 months)
concurrency: 1
groups:
- postgresql-backup
labels:
recurring-job: "postgresql-backup-weekly"
backup-type: "weekly"
---
# Snapshot cleanup job for space management
apiVersion: longhorn.io/v1beta2
kind: RecurringJob
metadata:
name: postgresql-snapshot-cleanup
namespace: longhorn-system
spec:
task: snapshot-cleanup
cron: "0 3 * * *" # Daily at 3 AM UTC (after backup)
retain: 5 # Keep only 5 snapshots locally
concurrency: 2
groups:
- postgresql-backup
labels:
recurring-job: "postgresql-snapshot-cleanup"
backup-type: "cleanup"

69
manifests/infrastructure/postgresql/cert-manager-certificates.yaml Normal file
View File

@@ -0,0 +1,69 @@
---
# Self-signed issuer for PostgreSQL certificates
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: postgresql-selfsigned-issuer
namespace: postgresql-system
spec:
selfSigned: {}
---
# Server TLS certificate for PostgreSQL cluster
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: postgresql-shared-server-cert
namespace: postgresql-system
labels:
cnpg.io/reload: "" # Enable automatic reload by CloudNativePG
spec:
secretName: postgresql-shared-server-cert
commonName: postgresql-shared-rw
usages:
- server auth
dnsNames:
# Primary service (read-write)
- postgresql-shared-rw
- postgresql-shared-rw.postgresql-system
- postgresql-shared-rw.postgresql-system.svc
- postgresql-shared-rw.postgresql-system.svc.cluster.local
# Read service (read-only from any instance)
- postgresql-shared-r
- postgresql-shared-r.postgresql-system
- postgresql-shared-r.postgresql-system.svc
- postgresql-shared-r.postgresql-system.svc.cluster.local
# Read-only service (read-only replicas only)
- postgresql-shared-ro
- postgresql-shared-ro.postgresql-system
- postgresql-shared-ro.postgresql-system.svc
- postgresql-shared-ro.postgresql-system.svc.cluster.local
issuerRef:
name: postgresql-selfsigned-issuer
kind: Issuer
group: cert-manager.io
# Certificate duration (90 days to match CloudNativePG default)
duration: 2160h # 90 days
renewBefore: 168h # 7 days (matches CloudNativePG default)
---
# Client certificate for streaming replication
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: postgresql-shared-client-cert
namespace: postgresql-system
labels:
cnpg.io/reload: "" # Enable automatic reload by CloudNativePG
spec:
secretName: postgresql-shared-client-cert
commonName: streaming_replica
usages:
- client auth
issuerRef:
name: postgresql-selfsigned-issuer
kind: Issuer
group: cert-manager.io
# Certificate duration (90 days to match CloudNativePG default)
duration: 2160h # 90 days
renewBefore: 168h # 7 days (matches CloudNativePG default)

85
manifests/infrastructure/postgresql/cilium-cnpg-policies.yaml Normal file
View File

@@ -0,0 +1,85 @@
---
# Comprehensive CloudNativePG network policy for single-operator deployment
# This allows the Helm-deployed operator in postgresql-system to manage the cluster
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: cnpg-comprehensive-access
namespace: postgresql-system
spec:
description: "Allow CloudNativePG operator and cluster communication"
endpointSelector:
matchLabels:
cnpg.io/cluster: postgres-shared # Apply to postgres-shared cluster pods
ingress:
# Allow operator in same namespace to manage cluster
- fromEndpoints:
- matchLabels:
app.kubernetes.io/name: cloudnative-pg # Helm-deployed operator
toPorts:
- ports:
- port: "5432"
protocol: TCP # PostgreSQL database
- port: "8000"
protocol: TCP # CloudNativePG health endpoint
- port: "9187"
protocol: TCP # PostgreSQL metrics
# Allow cluster-wide access for applications and monitoring
- fromEntities:
- cluster
- host
- remote-node
- kube-apiserver # Explicitly allow API server (used for service port-forward)
toPorts:
- ports:
- port: "5432"
protocol: TCP # PostgreSQL database access
- port: "9187"
protocol: TCP # Metrics collection
# Allow pod-to-pod communication within cluster (replication)
- fromEndpoints:
- matchLabels:
cnpg.io/cluster: postgres-shared
toPorts:
- ports:
- port: "5432"
protocol: TCP # PostgreSQL replication
- port: "8000"
protocol: TCP # Health checks between replicas
---
# Allow CloudNativePG operator to reach webhook endpoints
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: cnpg-operator-webhook-access
namespace: postgresql-system
spec:
description: "Allow CloudNativePG operator webhook communication"
endpointSelector:
matchLabels:
app.kubernetes.io/name: cloudnative-pg # Helm-deployed operator
ingress:
# Allow Kubernetes API server to reach webhook
- fromEntities:
- host
- cluster
toPorts:
- ports:
- port: "9443"
protocol: TCP # CloudNativePG webhook port
egress:
# Allow operator to reach PostgreSQL pods for management
- toEndpoints:
- matchLabels:
cnpg.io/cluster: postgres-shared
toPorts:
- ports:
- port: "5432"
protocol: TCP
- port: "8000"
protocol: TCP
# Allow operator to reach Kubernetes API
- toEntities:
- cluster
- host
- remote-node

176
manifests/infrastructure/postgresql/cluster-shared.yaml Normal file
View File

@@ -0,0 +1,176 @@
---
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: postgres-shared
namespace: postgresql-system
labels:
app: postgresql-shared
backup.longhorn.io/enable: "true"
spec:
instances: 3
# Use CloudNativePG-compatible PostGIS image
# imageName: ghcr.io/cloudnative-pg/postgresql:16.6 # Standard image
imageName: <YOUR_REGISTRY_URL>/library/cnpg-postgis:16.6-3.4-v2
# Bootstrap with initial database and user
bootstrap:
initdb:
database: shared_db
owner: shared_user
encoding: UTF8
localeCollate: en_US.UTF-8
localeCType: en_US.UTF-8
# Install PostGIS extensions in template database (available to all databases)
postInitTemplateSQL:
- CREATE EXTENSION IF NOT EXISTS postgis;
- CREATE EXTENSION IF NOT EXISTS postgis_topology;
- CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
- CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder;
# PostgreSQL configuration for conservative scaling (3GB memory limit)
postgresql:
parameters:
# Performance optimizations for 3GB memory limit
max_connections: "300"
shared_buffers: "768MB" # 25% of 3GB memory limit
effective_cache_size: "2.25GB" # ~75% of 3GB memory limit
maintenance_work_mem: "192MB" # Scaled for 3GB memory limit
checkpoint_completion_target: "0.9"
wal_buffers: "24MB"
default_statistics_target: "100"
random_page_cost: "1.1" # Good for SSD storage
effective_io_concurrency: "200"
work_mem: "12MB" # Conservative: 300 connections = ~3.6GB total max
min_wal_size: "1GB"
max_wal_size: "6GB"
# Additional optimizations for your hardware (tuned for 2-core limit)
max_worker_processes: "8" # Scaled for 2 CPU cores
max_parallel_workers: "6" # Increased for better OLTP workload
max_parallel_workers_per_gather: "3" # Max 3 workers per query
max_parallel_maintenance_workers: "3" # For maintenance operations
# Network timeout adjustments for 100Mbps VLAN
wal_sender_timeout: "10s" # Increased from 5s for slower network
wal_receiver_timeout: "10s" # Increased from 5s for slower network
# Multi-instance HA configuration with asynchronous replication
synchronous_commit: "on" # favor data integrity
# Log long running queries
log_min_duration_statement: "5000" # Log queries > 5 seconds
log_line_prefix: "%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h "
log_statement: "none" # Only log slow queries, not all
# Query activity tracking - increase limit for complex queries
track_activity_query_size: "8192" # 8KB - allows full query text in pg_stat_activity
# Storage configuration using PostgreSQL-optimized storage class
storage:
size: 50Gi
storageClass: longhorn-postgresql
# Separate WAL storage for better I/O performance
walStorage:
size: 10Gi
storageClass: longhorn-postgresql
# Enable pod anti-affinity for HA cluster (distribute across nodes)
affinity:
enablePodAntiAffinity: true
topologyKey: kubernetes.io/hostname
resources:
requests:
cpu: 750m
memory: 1.5Gi
limits:
cpu: 2000m
memory: 3Gi
# Enable superuser access for maintenance
enableSuperuserAccess: true
# Certificate configuration using cert-manager
certificates:
serverTLSSecret: postgresql-shared-server-cert
serverCASecret: postgresql-shared-server-cert
clientCASecret: postgresql-shared-client-cert
replicationTLSSecret: postgresql-shared-client-cert
# Replication slot configuration - enabled for HA cluster
replicationSlots:
highAvailability:
enabled: true # Enable HA replication slots for multi-instance cluster
synchronizeReplicas:
enabled: true # Enable replica synchronization for HA
# Monitoring configuration for Prometheus metrics
monitoring:
enablePodMonitor: true
# Custom metrics for dashboard compatibility
customQueriesConfigMap:
- name: postgresql-dashboard-metrics
key: queries
- name: postgresql-connection-metrics
key: custom-queries
# Reasonable startup delay for stable 2-instance cluster
startDelay: 30
probes:
startup:
initialDelaySeconds: 60 # Allow PostgreSQL to start and begin recovery
periodSeconds: 10
timeoutSeconds: 10
failureThreshold: 90 # 15 minutes total for replica recovery with Longhorn storage
readiness:
initialDelaySeconds: 30 # Allow instance manager to initialize
periodSeconds: 10
timeoutSeconds: 10
failureThreshold: 3
liveness:
initialDelaySeconds: 120 # Allow full startup before liveness checks
periodSeconds: 30
timeoutSeconds: 10
failureThreshold: 3
primaryUpdateMethod: switchover # Use switchover instead of restart to prevent restart loops
primaryUpdateStrategy: unsupervised
# S3 backup configuration for CloudNativePG - TEMPORARILY DISABLED
# backup:
# # Backup retention policy
# retentionPolicy: "30d" # Keep backups for 30 days
#
# # S3 backup configuration for Backblaze B2
# barmanObjectStore:
# destinationPath: s3://postgresql-backups/cnpg
# s3Credentials:
# accessKeyId:
# name: postgresql-s3-backup-credentials
# key: AWS_ACCESS_KEY_ID
# secretAccessKey:
# name: postgresql-s3-backup-credentials
# key: AWS_SECRET_ACCESS_KEY
# endpointURL: <REPLACE_WITH_S3_ENDPOINT>
#
# # Backblaze B2 specific configuration
# data:
# compression: gzip
# encryption: AES256
# immediateCheckpoint: true
# jobs: 2 # Parallel backup jobs
#
# wal:
# compression: gzip
# encryption: AES256
# maxParallel: 2 # Parallel WAL archiving

18
manifests/infrastructure/postgresql/kustomization.yaml Normal file
View File

@@ -0,0 +1,18 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- repository.yaml
- operator.yaml
- postgresql-storageclass.yaml
- cert-manager-certificates.yaml
- cilium-cnpg-policies.yaml
- cluster-shared.yaml
- backup-config.yaml
- postgresql-s3-backup-secret.yaml
# - scheduled-backups.yaml # Removed - was using barmanObjectStore method
- postgresql-dashboard-metrics.yaml
- postgresql-dashboard-rbac.yaml
- postgresql-connection-metrics.yaml
- postgresql-service-alias.yaml

9
manifests/infrastructure/postgresql/namespace.yaml Normal file
View File

@@ -0,0 +1,9 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: postgresql-system
labels:
name: postgresql-system
pod-security.kubernetes.io/enforce: restricted
pod-security.kubernetes.io/enforce-version: latest

81
manifests/infrastructure/postgresql/network-policy.yaml.example Normal file
View File

@@ -0,0 +1,81 @@
# Example PostgreSQL Network Policies (not applied by default)
# Uncomment and customize these if you want to implement network security for PostgreSQL
# ---
# apiVersion: "cilium.io/v2"
# kind: CiliumNetworkPolicy
# metadata:
# name: "postgresql-ingress"
# namespace: postgresql-system
# spec:
# description: "Allow ingress traffic to PostgreSQL pods"
# endpointSelector:
# matchLabels:
# postgresql: postgresql-shared
# ingress:
# # Allow CloudNativePG operator status checks
# - fromEndpoints:
# - matchLabels:
# app.kubernetes.io/name: cloudnative-pg
# toPorts:
# - ports:
# - port: "8000" # Status port
# protocol: "TCP"
#
# # Allow PostgreSQL connections from applications
# - fromEntities:
# - cluster # Allow any pod in cluster to connect
# toPorts:
# - ports:
# - port: "5432" # PostgreSQL port
# protocol: "TCP"
#
# # Allow PostgreSQL replication between instances
# - fromEndpoints:
# - matchLabels:
# postgresql: postgresql-shared # Allow PostgreSQL pods to talk to each other
# toPorts:
# - ports:
# - port: "5432"
# protocol: "TCP"
#
# # Allow metrics scraping (for OpenObserve)
# - fromEndpoints:
# - matchLabels:
# app: openobserve-collector
# toPorts:
# - ports:
# - port: "9187" # Metrics port
# protocol: "TCP"
# ---
# apiVersion: "cilium.io/v2"
# kind: CiliumNetworkPolicy
# metadata:
# name: "postgresql-egress"
# namespace: postgresql-system
# spec:
# description: "Allow egress traffic from PostgreSQL pods"
# endpointSelector:
# matchLabels:
# postgresql: postgresql-shared
# egress:
# # Allow DNS resolution
# - toEndpoints:
# - matchLabels:
# k8s-app: kube-dns
# toPorts:
# - ports:
# - port: "53"
# protocol: "UDP"
# - port: "53"
# protocol: "TCP"
#
# # Allow PostgreSQL replication
# - toEndpoints:
# - matchLabels:
# postgresql: postgresql-shared
# toPorts:
# - ports:
# - port: "5432"
# protocol: "TCP"

Some files were not shown because too many files have changed in this diff Show More