Keybard-Vagabond-Demo/manifests/infrastructure/elasticsearch/cluster.yaml

---
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
  name: elasticsearch
  namespace: elasticsearch-system
  labels:
    app: elasticsearch
    backup.longhorn.io/enable: "true"  # Enable Longhorn S3 backup
spec:
  version: 7.17.27  # Latest 7.x version compatible with Mastodon
  
  # Single-node cluster (can be scaled later)
  nodeSets:
  - name: default
    count: 1
    config:
      # Node configuration
      node.store.allow_mmap: false  # Required for containers
      
      # Performance optimizations for 2-node cluster (similar to PostgreSQL)
      cluster.routing.allocation.disk.threshold_enabled: true
      cluster.routing.allocation.disk.watermark.low: "85%"
      cluster.routing.allocation.disk.watermark.high: "90%"
      cluster.routing.allocation.disk.watermark.flood_stage: "95%"
      
      # Memory and performance settings
      indices.memory.index_buffer_size: "20%"
      indices.memory.min_index_buffer_size: "48mb"
      indices.fielddata.cache.size: "30%"
      indices.queries.cache.size: "20%"
      
      # ECK manages discovery configuration automatically for single-node clusters
      
      # Security settings - ECK manages TLS automatically
      xpack.security.enabled: true
    
    # Pod template for Elasticsearch nodes
    podTemplate:
      metadata:
        labels:
          app: elasticsearch
      spec:
        # Node selection and affinity - Prefer n2 but allow n1 if needed
        nodeSelector: {}
        tolerations: []
        affinity:
          nodeAffinity:
            # PREFERRED: Prefer n2 for optimal distribution, but allow n1 if needed
            preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              preference:
                matchExpressions:
                - key: kubernetes.io/hostname
                  operator: In
                  values: ["n2"]
        
        # Resource configuration - Optimized for resource-constrained environment
        containers:
        - name: elasticsearch
          resources:
            requests:
              cpu: 500m       # 0.5 CPU core 
              memory: 2Gi     # 2GB RAM (increased from 1Gi)
            limits:
              cpu: 1000m      # Max 1 CPU core
              memory: 4Gi     # Max 4GB RAM (increased from 2Gi)
          env:
          # JVM heap size - should be 50% of container memory limit  
          - name: ES_JAVA_OPTS
            value: "-Xms2g -Xmx2g"
          
          # Security context - ECK manages this automatically
          securityContext: {}
    
    # Volume claim templates
    volumeClaimTemplates:
    - metadata:
        name: elasticsearch-data
        labels:
          backup.longhorn.io/enable: "true"  # Enable S3 backup
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: 50Gi
        storageClassName: longhorn-retain

  # HTTP configuration
  http:
    service:
      spec:
        type: ClusterIP
        selector:
          elasticsearch.k8s.elastic.co/cluster-name: "elasticsearch"
    tls:
      selfSignedCertificate:
        disabled: true  # Disable TLS for internal Kubernetes communication

  # Transport configuration
  transport:
    service:
      spec:
        type: ClusterIP

---
# Kibana deployment for optional web UI management
apiVersion: kibana.k8s.elastic.co/v1
kind: Kibana
metadata:
  name: kibana
  namespace: elasticsearch-system
spec:
  version: 7.17.27
  count: 1
  elasticsearchRef:
    name: elasticsearch
  
  config:
    server.publicBaseUrl: "https://kibana.keyboardvagabond.com"
    
  podTemplate:
    metadata:
      labels:
        app: kibana
    spec:
      containers:
      - name: kibana
        resources:
          requests:
            cpu: 50m        # Reduced from 200m - actual usage ~26m
            memory: 384Mi   # Reduced from 1Gi - actual usage ~274MB
          limits:
            cpu: 400m       # Reduced from 1000m but adequate for log analysis
            memory: 768Mi   # Reduced from 2Gi but adequate for dashboards
        securityContext: {}

  http:
    service:
      metadata:
        annotations:
          tailscale.com/hostname: kibana
      spec:
        type: LoadBalancer
        loadBalancerClass: tailscale
    tls:
      selfSignedCertificate:
        disabled: false