redaction (#1)

Add the redacted source file for demo purposes Reviewed-on: https://source.michaeldileo.org/michael_dileo/Keybard-Vagabond-Demo/pulls/1 Co-authored-by: Michael DiLeo <michael_dileo@proton.me> Co-committed-by: Michael DiLeo <michael_dileo@proton.me>
2025-12-24 13:40:47 +00:00
parent 612235d52b
commit 7327d77dcd
333 changed files with 39286 additions and 1 deletions
--- a/manifests/infrastructure/openobserve/kustomization.yaml
+++ b/manifests/infrastructure/openobserve/kustomization.yaml
@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+ - namespace.yaml
+ - secret.yaml
+ - openobserve.yaml
+ - manual-ingress.yaml
--- a/manifests/infrastructure/openobserve/manual-ingress.yaml
+++ b/manifests/infrastructure/openobserve/manual-ingress.yaml
@@ -0,0 +1,29 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: openobserve-ingress
+  namespace: openobserve
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    nginx.ingress.kubernetes.io/enable-cors: "true"
+    # Fix HTTP/2 protocol errors by forcing HTTP/1.1 backend communication
+    nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
+    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+spec:
+  ingressClassName: nginx
+  tls: []
+  rules:
+  - host: obs.keyboardvagabond.com
+    http:
+      paths:
+      # OpenObserve - route to HTTP service
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: openobserve-openobserve-standalone
+            port:
+              number: 5080
--- a/manifests/infrastructure/openobserve/namespace.yaml
+++ b/manifests/infrastructure/openobserve/namespace.yaml
@@ -0,0 +1,9 @@
+# manifests/infrastructure/openobserve/namespace.yaml
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: openobserve
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/enforce-version: latest
--- a/manifests/infrastructure/openobserve/openobserve.yaml
+++ b/manifests/infrastructure/openobserve/openobserve.yaml
@@ -0,0 +1,119 @@
+# manifests/infrastructure/openobserve/openobserve.yaml
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: openobserve
+  namespace: openobserve
+spec:
+  interval: 5m0s
+  url: https://charts.openobserve.ai
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: openobserve
+  namespace: openobserve
+spec:
+  interval: 5m
+  chart:
+    spec:
+      chart: openobserve-standalone
+      version: ">=0.15.0"
+      sourceRef:
+        kind: HelmRepository
+        name: openobserve
+        namespace: openobserve
+      interval: 1m
+  values:
+    # Use SIMD-optimized image for ARM with NEON support
+    image:
+      repository: public.ecr.aws/zinclabs/openobserve
+      tag: v0.15.0-simd
+    
+    # Basic configuration with memory optimization
+    config:
+      ZO_TELEMETRY: "false"
+      ZO_WEB_URL: "https://obs.keyboardvagabond.com"
+      # Aggressive data retention for resource-constrained environment
+      ZO_COMPACT_DATA_RETENTION_DAYS: "7"  # Reduced from 14 to 7 days
+      ZO_COMPACT_RETENTION_LOGS: "7"       # Explicit log retention
+      ZO_COMPACT_RETENTION_METRICS: "14"   # Keep metrics longer than logs
+      ZO_COMPACT_RETENTION_TRACES: "3"     # Traces are large, keep only 3 days
+      
+      # Memory optimization settings - reduced for 5GB container limit
+      ZO_MEMORY_CACHE_MAX_SIZE: "1536"  # Reduced to 1.5GB (was 2GB) - still good performance
+      ZO_MEMORY_CACHE_DATAFUSION_MAX_SIZE: "768"   # Reduced to 768MB (was 1GB) - adequate for queries
+      ZO_MAX_FILE_SIZE_IN_MEMORY: "64"  # Reduce memory table size to 64MB (default 256MB)
+      ZO_MEM_DUMP_THREAD_NUM: "2"  # Use 2 threads for memory dumps (faster disk writes)
+      
+      # Enable disk caching to reduce RAM usage
+      ZO_DISK_CACHE_ENABLED: "true"
+      ZO_DISK_CACHE_MAX_SIZE: "8192"  # 8GB disk cache (in MB)
+      
+      # Reduce field processing overhead
+      ZO_COLS_PER_RECORD_LIMIT: "500"  # Limit fields per record (default 1000)
+      
+      # Optimized compaction for memory efficiency
+      ZO_COMPACT_SYNC_TO_DB_INTERVAL: "10"  # Reduced frequency (was 5s) to save memory
+      ZO_COMPACT_MAX_FILE_SIZE: "256"       # Smaller files (256MB) to reduce memory buffers
+      ZO_COMPACT_INTERVAL: "120"            # Less frequent compaction (2min vs 1min) to reduce memory spikes
+      ZO_COMPACT_STEP_SIZE: "500"           # Fewer files per step to reduce memory usage
+    
+    # Local storage for now - easy to migrate to S3 later
+    persistence:
+      size: 100Gi
+      storageClass: "longhorn"
+    
+    # Resource limits optimized with memory configuration tunning
+    resources:
+      requests:
+        cpu: 512m
+        memory: 1.5Gi    # Reasonable request for optimized caches
+      limits:
+        cpu: 2500m
+        memory: 5Gi      # Keep at 5GB with optimized cache settings
+    
+    ingress:
+      enabled: false
+    
+    # Security context optimized for large volumes per Kubernetes docs
+    # https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#configure-volume-permission-and-ownership-change-policy-for-pods
+    securityContext:
+      fsGroup: 2000
+      runAsUser: 10000  # Match existing StatefulSet to avoid conflicts
+      runAsGroup: 3000  # Match existing StatefulSet to avoid conflicts  
+      fsGroupChangePolicy: "OnRootMismatch"  # Only change permissions if root ownership differs
+      runAsNonRoot: true
+    
+    # Use secret for credentials (secure approach)
+    extraEnv:
+      - name: ZO_ROOT_USER_EMAIL
+        valueFrom:
+          secretKeyRef:
+            name: openobserve-credentials
+            key: ZO_ROOT_USER_EMAIL
+      - name: ZO_ROOT_USER_PASSWORD
+        valueFrom:
+          secretKeyRef:
+            name: openobserve-credentials
+            key: ZO_ROOT_USER_PASSWORD
+      # SMTP configuration for email alerts - all as environment variables
+      - name: ZO_SMTP_ENABLED
+        value: "true"
+      - name: ZO_SMTP_HOST
+        value: "<YOUR_SMTP_SERVER>"
+      - name: ZO_SMTP_PORT
+        value: "587"
+      - name: ZO_SMTP_USERNAME
+        value: "alerts@mail.keyboardvagabond.com"
+      - name: ZO_SMTP_FROM_EMAIL
+        value: "alerts@mail.keyboardvagabond.com"
+      - name: ZO_SMTP_REPLY_TO
+        value: "alerts@mail.keyboardvagabond.com"
+      - name: ZO_SMTP_ENCRYPTION
+        value: "starttls"
+      - name: ZO_SMTP_PASSWORD
+        valueFrom:
+          secretKeyRef:
+            name: openobserve-credentials
+            key: ZO_SMTP_PASSWORD
--- a/manifests/infrastructure/openobserve/secret.yaml
+++ b/manifests/infrastructure/openobserve/secret.yaml
@@ -0,0 +1,49 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: openobserve-credentials
+    namespace: openobserve
+type: Opaque
+stringData:
+    #ENC[AES256_GCM,data:ciQlpWxpLZm/OdqfpX3og3AIECXErnwAZsfgHqsVQ7tY7FKXJFLgIPInELDCMnbhxnpaqB3cpKKZfYo=,iv:TGGgEOflQ04BMxHYvPOMGM+E6inG4BhTPywKAkuIXwU=,tag:wAntPBIy8zw6OffBgCLL8A==,type:comment]
+    #ENC[AES256_GCM,data:5rTQeiBnHo372FnVAyhXcTstce0iVxt7DWSEkwuKa91JlJlgL1jw2a+Fc8NWjy4hbLSq4Qht,iv:NGVB8FOP+Dv3dRb3RS84FSFQgHj4UW3p/cr+8ozoGcI=,tag:1Sr3pJFMuDbl7+jfQEItmw==,type:comment]
+    ZO_ROOT_USER_PASSWORD: ENC[AES256_GCM,data:jW2zrcHb75ozVO+NzUaaEsdIOLlra1dHnKLgxvlhNY8AtqQ1BI+iB6379wpa,iv:e8XAFf2OCwnxzingUzba1HpkXWdbfA36U92N4ciSLKo=,tag:rZAQeEgJYapyHKMgnzUyfQ==,type:str]
+    ZO_ROOT_USER_EMAIL: ENC[AES256_GCM,data:uJql3q4n8MScoNDD1xow1UnRjIemw69Gwq8=,iv:WK/EDY9sG7yhUxQznPubbK5UlsqmfGqFWfZJMg69DRE=,tag:FG18/MIIM8aYMXZff2ljtg==,type:str]
+    #ENC[AES256_GCM,data:4R8+Sdiofs0W5FpzALUKOBehq6EsHCYf7ChJbEGLc8n9fzMbZbWkr2Syvjy/wXJ/,iv:caG3Up+sCQBYD1IQstR5PRfzgni49UKYVRR+jhqWWKM=,tag:LDCYOZHdAbuYIh6i09BbfA==,type:comment]
+    ZO_SMTP_ENABLED: ENC[AES256_GCM,data:fzbe1g==,iv:XQYUDCKVgvSSh/eEF+gzs4Wf8mH11hUw5RgWYJTuiRI=,tag:mHko4/V+/oX1jdQ/JManoQ==,type:str]
+    ZO_SMTP_HOST: ENC[AES256_GCM,data:28CFU8QH3/voR2Sdg2RwAOCGmg==,iv:f+Q0M1OPkIBpLIGc0Shh2Zba49w+7NLdjnWtJCpDGnM=,tag:w8LsbkFA4KXqc02ddJ/fuw==,type:str]
+    ZO_SMTP_PORT: ENC[AES256_GCM,data:o8f2,iv:U13muGbectPG41tMZgtmlDkzMdfQIWoP3pQwJRBH5SE=,tag:h5LwD5LIQhJqPwU+yXujkg==,type:str]
+    ZO_SMTP_USERNAME: ENC[AES256_GCM,data:gGt0Xp7HAPJMj28umdjCvGixdy9i65f+5i2sdjLa9ZY=,iv:z+KSvLdjyxr/0xYmk0Yb8140/7jieg41K1w2U3BT2Pk=,tag:NtIDdOPd9hA5TIDhz05b6A==,type:str]
+    ZO_SMTP_PASSWORD: ENC[AES256_GCM,data:v2BMTxQ9fgEsGGNYyiyzE/Xr46G732d/E9aitQbMqq46egDXrqjelyPn8J5dK0M+Oyo=,iv:CDlByQ/TZEr/8hZuTlcKeYdshib5z+wC39K/yfngiWQ=,tag:V4werptqvJoJr5mnYSh0hQ==,type:str]
+    ZO_SMTP_FROM_EMAIL: ENC[AES256_GCM,data:IdHjmM3ph8j2wR7U1Ayu9TcBvgIFeeQ6Q1p87RHGmB4=,iv:QxFXfcpoq7Z2Nkn7e6h8qTYn5Wt2LcveDHK3bvuFBP8=,tag:ZgyZtgOCTuZpJk3UDdG9xQ==,type:str]
+    ZO_SMTP_REPLY_TO: ENC[AES256_GCM,data:HtEazpWxxayEfuG2GBcMKam434BnmgYWFeLNCoWmQPg=,iv:fcgBJ+S+/X0L/vtKlP7PYbYaTPONy7VFyhW6r7BpumA=,tag:KEKtw1RwPpJYvWa6dHxQkQ==,type:str]
+sops:
+    lastmodified: "2025-09-11T15:13:23Z"
+    mac: ENC[AES256_GCM,data:8aW1yhcqsgNTlHq45shvIaONm+4wd/5myj2e1CTbV+tSh2eA6u0Cj94DeifWxNPaX/wtlcb9atUrr3wuNAE6+k0UWoxVn6/2divipC7LtV7hLVQYwwB1xIm+aiAesILFg60BK0TKTlg6kgsPDJ74O0kKn09pm8pFKLBlO0pqj4E=,iv:4g75VE7di0FvzvCa8DCNSIILQroP1sK16tfTZRMBXKQ=,tag:lYykRQ21SdFC3TvYzXenOQ==,type:str]
+    pgp:
+        - created_at: "2025-09-11T15:04:12Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hF4DZT3mpHTS/JgSAQdAx2g4TFggUbHlQySK6xGp6RvE03szSCAB3wKwneUrRi4w
+            uhj4z/S5sWG1wU46akQQdpdXfOp38uVPO+hNWl5pg4wyLAB3zTqi9CRPKJm6GflE
+            1GgBCQIQaxecQiWrs/IkjtHwilIGCFECizqpEg2DD3Y5zMVKgxDsnaFAXgeQmo0a
+            7BJaTABDnKh1sKQsAfED9dnSr63xmEUYPAdve6jn+No5IhF6fqkH06nppfKnxpAD
+            VUzF8FpItENOdg==
+            =s2tg
+            -----END PGP MESSAGE-----
+          fp: B120595CA9A643B051731B32E67FF350227BA4E8
+        - created_at: "2025-09-11T15:04:12Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hF4DSXzd60P2RKISAQdAcK2Bi/ozYs1mEHiqZ5oKzm6KAhqT6LYeK8xGjAmTzQAw
+            6bAfh7uN5TBza+cM4k7QQXfsgs2+39EGKRyFeitKW/WPORes5lMnsWsD/0zCLWWH
+            1GgBCQIQJZLult2JJmlrPTY1ILuuxfgzgV8Bh9yCDJDtyQJpsfKmPbqsUYC4Ner7
+            rMj6XA87dJEyRdxhxa2yx+/Wjd8RzcN9rgWQW+ruBsrPOvpAgUUvjDAMq/FIsdVI
+            pgurg1Z8+W0ldQ==
+            =p2GD
+            -----END PGP MESSAGE-----
+          fp: 4A8AADB4EBAB9AF88EF7062373CECE06CC80D40C
+    encrypted_regex: ^(data|stringData)$
+    version: 3.10.2