From 9f7fa2493168418316d08e3b1d229933d0bb7d4c Mon Sep 17 00:00:00 2001 From: Michael DiLeo Date: Wed, 14 Jan 2026 22:33:45 +0200 Subject: [PATCH] update with latest build versions, includes custom build for postgres and migrating from v16 to v18 --- build/bookwyrm/README.md | 24 +- build/bookwyrm/bookwyrm-base/Dockerfile | 11 +- build/bookwyrm/bookwyrm-web/Dockerfile | 14 +- build/bookwyrm/bookwyrm-web/nginx.conf | 6 + build/bookwyrm/bookwyrm-worker/Dockerfile | 16 +- build/bookwyrm/build.sh | 18 +- build/piefed/README.md | 22 +- build/piefed/build-all.sh | 9 +- build/piefed/piefed-base/.dockerignore | 29 ++ build/piefed/piefed-base/Dockerfile | 72 ++--- build/piefed/piefed-base/entrypoint-init.sh | 66 +--- build/piefed/piefed-web/.dockerignore | 29 ++ build/piefed/piefed-web/Dockerfile | 15 +- build/piefed/piefed-worker/.dockerignore | 29 ++ build/piefed/piefed-worker/Dockerfile | 3 - build/pixelfed/README.md | 22 +- build/pixelfed/build-all.sh | 7 +- build/pixelfed/pixelfed-base/Dockerfile | 125 +++----- build/postgresql-postgis/Dockerfile | 289 ++++++++++++++++-- build/postgresql-postgis/Dockerfile.upgrade | 267 ++++++++++++++++ build/postgresql-postgis/Migration_Plan.md | 184 +++++++++++ .../postgresql-postgis/build-upgrade-image.sh | 140 +++++++++ build/postgresql-postgis/build.sh | 176 +++++++++-- .../docker-compose.test.yaml | 36 +++ build/postgresql-postgis/init-extensions.sql | 21 ++ build/postgresql-postgis/initdb-postgis.sh | 22 ++ build/postgresql-postgis/update-postgis.sh | 28 ++ 27 files changed, 1371 insertions(+), 309 deletions(-) create mode 100644 build/piefed/piefed-base/.dockerignore create mode 100644 build/piefed/piefed-web/.dockerignore create mode 100644 build/piefed/piefed-worker/.dockerignore create mode 100644 build/postgresql-postgis/Dockerfile.upgrade create mode 100644 build/postgresql-postgis/Migration_Plan.md create mode 100755 build/postgresql-postgis/build-upgrade-image.sh create mode 100644 build/postgresql-postgis/docker-compose.test.yaml create mode 100644 build/postgresql-postgis/init-extensions.sql create mode 100755 build/postgresql-postgis/initdb-postgis.sh create mode 100755 build/postgresql-postgis/update-postgis.sh diff --git a/build/bookwyrm/README.md b/build/bookwyrm/README.md index ad5d96a..8bd6880 100644 --- a/build/bookwyrm/README.md +++ b/build/bookwyrm/README.md @@ -43,7 +43,7 @@ build/bookwyrm/ ### **Prerequisites** - Docker with ARM64 support -- Access to Harbor registry (``) +- Access to Harbor registry (`registry.keyboardvagabond.com`) - Active Harbor login session ### **Build All Containers** @@ -76,12 +76,12 @@ cd .. # Build web container cd bookwyrm-web -docker build --platform linux/arm64 -t /library/bookwyrm-web:latest . +docker build --platform linux/arm64 -t registry.keyboardvagabond.com/library/bookwyrm-web:latest . cd .. # Build worker container cd bookwyrm-worker -docker build --platform linux/arm64 -t /library/bookwyrm-worker:latest . +docker build --platform linux/arm64 -t registry.keyboardvagabond.com/library/bookwyrm-worker:latest . ``` ## 🎯 **Container Specifications** @@ -139,32 +139,32 @@ DB_HOST=postgresql-shared-rw.postgresql-system.svc.cluster.local DB_PORT=5432 DB_NAME=bookwyrm DB_USER=bookwyrm_user -DB_PASSWORD= +DB_PASSWORD= # Redis Configuration -REDIS_BROKER_URL=redis://:@redis-ha-haproxy.redis-system.svc.cluster.local:6379/3 -REDIS_ACTIVITY_URL=redis://:@redis-ha-haproxy.redis-system.svc.cluster.local:6379/4 +REDIS_BROKER_URL=redis://:password@redis-ha-haproxy.redis-system.svc.cluster.local:6379/3 +REDIS_ACTIVITY_URL=redis://:password@redis-ha-haproxy.redis-system.svc.cluster.local:6379/4 # Application Settings -SECRET_KEY= +SECRET_KEY= DEBUG=false USE_HTTPS=true DOMAIN=bookwyrm.keyboardvagabond.com # S3 Storage USE_S3=true -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= AWS_STORAGE_BUCKET_NAME=bookwyrm-bucket AWS_S3_REGION_NAME=eu-central-003 -AWS_S3_ENDPOINT_URL= +AWS_S3_ENDPOINT_URL=https://s3.eu-central-003.backblazeb2.com AWS_S3_CUSTOM_DOMAIN=https://bm.keyboardvagabond.com # Email Configuration -EMAIL_HOST= +EMAIL_HOST=smtp.eu.mailgun.org EMAIL_PORT=587 EMAIL_HOST_USER=bookwyrm@mail.keyboardvagabond.com -EMAIL_HOST_PASSWORD= +EMAIL_HOST_PASSWORD= EMAIL_USE_TLS=true ``` diff --git a/build/bookwyrm/bookwyrm-base/Dockerfile b/build/bookwyrm/bookwyrm-base/Dockerfile index d693621..3882899 100644 --- a/build/bookwyrm/bookwyrm-base/Dockerfile +++ b/build/bookwyrm/bookwyrm-base/Dockerfile @@ -5,6 +5,11 @@ # Build stage - Install dependencies and prepare optimized source FROM python:3.11-slim AS builder +LABEL org.opencontainers.image.title="BookWyrm Base" \ + org.opencontainers.image.description="Shared base image for BookWyrm social reading platform" \ + org.opencontainers.image.source="https://github.com/bookwyrm-social/bookwyrm" \ + org.opencontainers.image.vendor="Keyboard Vagabond" + # Install build dependencies in a single layer RUN apt-get update && apt-get install -y --no-install-recommends \ git \ @@ -32,11 +37,11 @@ RUN python3 -m venv /opt/venv \ # Remove unnecessary files from source to reduce image size # Note: .dockerignore will exclude __pycache__, *.pyc, etc. automatically +# Note: Keep /app/locale for i18n support (translations) RUN rm -rf \ /app/.github \ /app/docker \ /app/nginx \ - /app/locale \ /app/bw-dev \ /app/bookwyrm/tests \ /app/bookwyrm/test* \ @@ -60,9 +65,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpq5 \ curl \ gettext \ - && rm -rf /var/lib/apt/lists/* \ + && apt-get autoremove -y \ && apt-get clean \ - && apt-get autoremove -y + && rm -rf /var/lib/apt/lists/* # Create bookwyrm user for security RUN useradd --create-home --shell /bin/bash --uid 1000 bookwyrm diff --git a/build/bookwyrm/bookwyrm-web/Dockerfile b/build/bookwyrm/bookwyrm-web/Dockerfile index 7f8896a..ed2d8e0 100644 --- a/build/bookwyrm/bookwyrm-web/Dockerfile +++ b/build/bookwyrm/bookwyrm-web/Dockerfile @@ -3,6 +3,9 @@ FROM bookwyrm-base AS bookwyrm-web +LABEL org.opencontainers.image.title="BookWyrm Web" \ + org.opencontainers.image.description="BookWyrm web server with Nginx and Gunicorn" + # Switch to root for system package installation USER root @@ -10,12 +13,12 @@ USER root RUN apt-get update && apt-get install -y --no-install-recommends \ nginx-light \ supervisor \ - && rm -rf /var/lib/apt/lists/* \ + && apt-get autoremove -y \ && apt-get clean \ - && apt-get autoremove -y + && rm -rf /var/lib/apt/lists/* -# Install Gunicorn in virtual environment -RUN /opt/venv/bin/pip install --no-cache-dir gunicorn +# Install Gunicorn in virtual environment (pinned for reproducible builds) +RUN /opt/venv/bin/pip install --no-cache-dir 'gunicorn>=23.0.0,<24.0.0' # Copy configuration files COPY nginx.conf /etc/nginx/nginx.conf @@ -43,8 +46,5 @@ EXPOSE 80 HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ CMD curl -f http://localhost:80/health/ || curl -f http://localhost:80/ || exit 1 -# Run as root to manage nginx and gunicorn via supervisor -USER root - ENTRYPOINT ["/entrypoint.sh"] CMD ["supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] \ No newline at end of file diff --git a/build/bookwyrm/bookwyrm-web/nginx.conf b/build/bookwyrm/bookwyrm-web/nginx.conf index cee2343..af1bfa6 100644 --- a/build/bookwyrm/bookwyrm-web/nginx.conf +++ b/build/bookwyrm/bookwyrm-web/nginx.conf @@ -77,6 +77,12 @@ http { add_header Content-Type text/plain; } + # Static files served via S3/CDN (bm.keyboardvagabond.com) + # No local static file serving needed when USE_S3=true + + # Images also served via S3/CDN + # No local image serving needed when USE_S3=true + # ActivityPub and federation endpoints location ~ ^/(inbox|user/.*/inbox|api|\.well-known) { proxy_pass http://127.0.0.1:8000; diff --git a/build/bookwyrm/bookwyrm-worker/Dockerfile b/build/bookwyrm/bookwyrm-worker/Dockerfile index bf7324d..b6320c0 100644 --- a/build/bookwyrm/bookwyrm-worker/Dockerfile +++ b/build/bookwyrm/bookwyrm-worker/Dockerfile @@ -3,18 +3,21 @@ FROM bookwyrm-base AS bookwyrm-worker +LABEL org.opencontainers.image.title="BookWyrm Worker" \ + org.opencontainers.image.description="BookWyrm Celery background task processor" + # Switch to root for system package installation USER root # Install only supervisor for worker management RUN apt-get update && apt-get install -y --no-install-recommends \ supervisor \ - && rm -rf /var/lib/apt/lists/* \ + && apt-get autoremove -y \ && apt-get clean \ - && apt-get autoremove -y + && rm -rf /var/lib/apt/lists/* -# Install Celery in virtual environment -RUN /opt/venv/bin/pip install --no-cache-dir celery[redis] +# Install Celery in virtual environment (pinned for reproducible builds) +RUN /opt/venv/bin/pip install --no-cache-dir 'celery[redis]>=5.6.0,<6.0.0' # Copy worker-specific configuration COPY supervisord-worker.conf /etc/supervisor/conf.d/supervisord.conf @@ -22,16 +25,11 @@ COPY entrypoint-worker.sh /entrypoint.sh # Set permissions efficiently RUN chmod +x /entrypoint.sh \ - && mkdir -p /var/log/supervisor /var/log/celery \ - && chown -R bookwyrm:bookwyrm /var/log/celery \ && chown -R bookwyrm:bookwyrm /app # Health check for worker HEALTHCHECK --interval=60s --timeout=10s --start-period=60s --retries=3 \ CMD /opt/venv/bin/celery -A celerywyrm inspect ping -d celery@$HOSTNAME || exit 1 -# Run as root to manage celery via supervisor -USER root - ENTRYPOINT ["/entrypoint.sh"] CMD ["supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] \ No newline at end of file diff --git a/build/bookwyrm/build.sh b/build/bookwyrm/build.sh index 03ee30f..3e72cb6 100755 --- a/build/bookwyrm/build.sh +++ b/build/bookwyrm/build.sh @@ -54,7 +54,7 @@ cd .. echo "" echo "Step 2/3: Building optimized web container..." cd bookwyrm-web -if docker build --platform linux/arm64 -t /library/bookwyrm-web:latest .; then +if docker build --platform linux/arm64 -t registry.keyboardvagabond.com/library/bookwyrm-web:latest .; then print_status "Web container built successfully!" else print_error "Failed to build web container" @@ -66,7 +66,7 @@ cd .. echo "" echo "Step 3/3: Building optimized worker container..." cd bookwyrm-worker -if docker build --platform linux/arm64 -t /library/bookwyrm-worker:latest .; then +if docker build --platform linux/arm64 -t registry.keyboardvagabond.com/library/bookwyrm-worker:latest .; then print_status "Worker container built successfully!" else print_error "Failed to build worker container" @@ -84,8 +84,8 @@ docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" | grep -E "( echo "" echo "Built containers:" -echo " • /library/bookwyrm-web:latest" -echo " • /library/bookwyrm-worker:latest" +echo " • registry.keyboardvagabond.com/library/bookwyrm-web:latest" +echo " • registry.keyboardvagabond.com/library/bookwyrm-worker:latest" # Ask if user wants to push echo "" @@ -96,13 +96,13 @@ if [[ $REPLY =~ ^[Yy]$ ]]; then echo "🚀 Pushing containers to registry..." # Login check - if ! docker info 2>/dev/null | grep -q ""; then + if ! docker info 2>/dev/null | grep -q "registry.keyboardvagabond.com"; then print_warning "You may need to login to Harbor registry first:" echo "" fi echo "Pushing web container..." - if docker push /library/bookwyrm-web:latest; then + if docker push registry.keyboardvagabond.com/library/bookwyrm-web:latest; then print_status "Web container pushed successfully!" else print_error "Failed to push web container" @@ -110,7 +110,7 @@ if [[ $REPLY =~ ^[Yy]$ ]]; then echo "" echo "Pushing worker container..." - if docker push /library/bookwyrm-worker:latest; then + if docker push registry.keyboardvagabond.com/library/bookwyrm-worker:latest; then print_status "Worker container pushed successfully!" else print_error "Failed to push worker container" @@ -120,6 +120,6 @@ if [[ $REPLY =~ ^[Yy]$ ]]; then print_status "All containers pushed to Harbor registry!" else echo "Skipping push. You can push later with:" - echo " docker push /library/bookwyrm-web:latest" - echo " docker push /library/bookwyrm-worker:latest" + echo " docker push registry.keyboardvagabond.com/library/bookwyrm-web:latest" + echo " docker push registry.keyboardvagabond.com/library/bookwyrm-worker:latest" fi \ No newline at end of file diff --git a/build/piefed/README.md b/build/piefed/README.md index 7064f36..484cb87 100644 --- a/build/piefed/README.md +++ b/build/piefed/README.md @@ -33,18 +33,18 @@ This will: 1. Build the base image with all PieFed dependencies 2. Build the web container with Nginx + Python/Flask (uWSGI) 3. Build the worker container with Celery workers -4. Push to your Harbor registry: `` +4. Push to your Harbor registry: `registry.keyboardvagabond.com` ### **Individual Container Builds** ```bash # Build just web container cd piefed-web && docker build --platform linux/arm64 \ - -t /library/piefed-web:latest . + -t registry.keyboardvagabond.com/library/piefed-web:latest . # Build just worker container cd piefed-worker && docker build --platform linux/arm64 \ - -t /library/piefed-worker:latest . + -t registry.keyboardvagabond.com/library/piefed-worker:latest . ``` ## 📦 **Container Details** @@ -85,14 +85,14 @@ PIEFED_DOMAIN=piefed.keyboardvagabond.com DB_HOST=postgresql-shared-rw.postgresql-system.svc.cluster.local DB_NAME=piefed DB_USER=piefed_user -DB_PASSWORD= +DB_PASSWORD=secure_password_here ``` #### **Redis Configuration** ```bash REDIS_HOST=redis-ha-haproxy.redis-system.svc.cluster.local REDIS_PORT=6379 -REDIS_PASSWORD= +REDIS_PASSWORD=redis_password_if_needed ``` #### **S3 Media Storage (Backblaze B2)** @@ -101,18 +101,18 @@ REDIS_PASSWORD= S3_ENABLED=true S3_BUCKET=piefed-bucket S3_REGION=eu-central-003 -S3_ENDPOINT= -S3_ACCESS_KEY= -S3_SECRET_KEY= +S3_ENDPOINT=https://s3.eu-central-003.backblazeb2.com +S3_ACCESS_KEY=your_b2_key_id +S3_SECRET_KEY=your_b2_secret_key S3_PUBLIC_URL=https://pfm.keyboardvagabond.com/ ``` -#### **Email (SMTP)** +#### **Email (Mailgun)** ```bash -MAIL_SERVER= +MAIL_SERVER=smtp.eu.mailgun.org MAIL_PORT=587 MAIL_USERNAME=piefed@mail.keyboardvagabond.com -MAIL_PASSWORD= +MAIL_PASSWORD= MAIL_USE_TLS=true MAIL_DEFAULT_SENDER=piefed@mail.keyboardvagabond.com ``` diff --git a/build/piefed/build-all.sh b/build/piefed/build-all.sh index ec2a03b..f503612 100755 --- a/build/piefed/build-all.sh +++ b/build/piefed/build-all.sh @@ -2,8 +2,8 @@ set -e # Configuration -REGISTRY="" -VERSION="v1.3.9" +REGISTRY="registry.keyboardvagabond.com" +VERSION="v1.5.1" PLATFORM="linux/arm64" # Colors for output @@ -65,6 +65,11 @@ echo -e "${BLUE}Built containers:${NC}" echo -e " • ${GREEN}$REGISTRY/library/piefed-web:$VERSION${NC}" echo -e " • ${GREEN}$REGISTRY/library/piefed-worker:$VERSION${NC}" +# Show image sizes +echo +echo -e "${BLUE}📊 Built image sizes:${NC}" +docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" | grep -E "(piefed-base|piefed-web|piefed-worker)" | head -10 + # Ask about pushing to registry echo read -p "Push all containers to Harbor registry? (y/N): " -n 1 -r diff --git a/build/piefed/piefed-base/.dockerignore b/build/piefed/piefed-base/.dockerignore new file mode 100644 index 0000000..e0c1c7c --- /dev/null +++ b/build/piefed/piefed-base/.dockerignore @@ -0,0 +1,29 @@ +# Git +.git +.gitignore + +# Documentation +*.md +README* + +# Python cache +__pycache__ +*.pyc +*.pyo +*.pyd +.pytest_cache +.coverage +htmlcov/ + +# Environment files +.env* +*.env + +# IDE +.vscode/ +.idea/ +*.swp +*.swo + +# Build artifacts +*.log diff --git a/build/piefed/piefed-base/Dockerfile b/build/piefed/piefed-base/Dockerfile index 00f08fc..d6a1d61 100644 --- a/build/piefed/piefed-base/Dockerfile +++ b/build/piefed/piefed-base/Dockerfile @@ -1,11 +1,8 @@ # Multi-stage build for smaller final image -FROM python:3.11-alpine AS builder +FROM python:3.11-alpine3.21 AS builder -# Use HTTP repositories to avoid SSL issues, then install dependencies -RUN echo "http://dl-cdn.alpinelinux.org/alpine/v3.22/main" > /etc/apk/repositories \ - && echo "http://dl-cdn.alpinelinux.org/alpine/v3.22/community" >> /etc/apk/repositories \ - && apk update \ - && apk add --no-cache \ +# Install build dependencies +RUN apk add --no-cache \ pkgconfig \ gcc \ python3-dev \ @@ -19,21 +16,24 @@ RUN echo "http://dl-cdn.alpinelinux.org/alpine/v3.22/main" > /etc/apk/repositori # Set working directory WORKDIR /app -# v1.3.x -ARG PIEFED_VERSION=main +# Clone PieFed source +ARG PIEFED_VERSION=v1.5.1 RUN git clone https://codeberg.org/rimu/pyfedi.git /app \ && cd /app \ && git checkout ${PIEFED_VERSION} \ && rm -rf .git -# Install Python dependencies to /app/venv +# Install Python dependencies to /app/venv and clean up cache/bytecode RUN python -m venv /app/venv \ && source /app/venv/bin/activate \ && pip install --no-cache-dir -r requirements.txt \ - && pip install --no-cache-dir uwsgi + && pip install --no-cache-dir uwsgi \ + && find /app/venv -name "*.pyc" -delete \ + && find /app/venv -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true \ + && find /app -name "*.pyo" -delete 2>/dev/null || true # Runtime stage - much smaller -FROM python:3.11-alpine AS runtime +FROM python:3.11-alpine3.21 AS runtime # Set environment variables ENV TZ=UTC @@ -41,55 +41,43 @@ ENV PYTHONUNBUFFERED=1 ENV PYTHONDONTWRITEBYTECODE=1 ENV PATH="/app/venv/bin:$PATH" -# Install only runtime dependencies -RUN echo "http://dl-cdn.alpinelinux.org/alpine/v3.22/main" > /etc/apk/repositories \ - && echo "http://dl-cdn.alpinelinux.org/alpine/v3.22/community" >> /etc/apk/repositories \ - && apk update \ - && apk add --no-cache \ +# Install only runtime dependencies (no redis server, nginx, dcron, or tesseract - not needed) +# - redis: using external Redis cluster, only Python client needed +# - nginx: only needed in web container, installed there +# - dcron: using Kubernetes CronJobs for scheduling +# - tesseract: OCR not used by PieFed +RUN apk add --no-cache \ ca-certificates \ curl \ su-exec \ - dcron \ libpq \ jpeg \ freetype \ lcms2 \ openjpeg \ tiff \ - nginx \ supervisor \ - redis \ - bash \ - tesseract-ocr \ - tesseract-ocr-data-eng + bash -# Create piefed user +# Create piefed user and set up directories in a single layer RUN addgroup -g 1000 piefed \ - && adduser -u 1000 -G piefed -s /bin/sh -D piefed + && adduser -u 1000 -G piefed -s /bin/sh -D piefed \ + && mkdir -p /app/logs /app/app/static/tmp /app/app/static/media \ + /var/log/piefed /var/run/piefed \ + && chown -R piefed:piefed /var/log/piefed /var/run/piefed # Set working directory WORKDIR /app -# Copy application and virtual environment from builder -COPY --from=builder /app /app -COPY --from=builder /app/venv /app/venv +# Copy application and virtual environment from builder (venv is inside /app) +COPY --from=builder --chown=piefed:piefed /app /app -# Compile translations (matching official Dockerfile) -RUN source /app/venv/bin/activate && \ - (pybabel compile -d app/translations || true) - -# Set proper permissions - ensure logs directory is writable for dual logging -RUN chown -R piefed:piefed /app \ - && mkdir -p /app/logs /app/app/static/tmp /app/app/static/media \ - && chown -R piefed:piefed /app/logs /app/app/static/tmp /app/app/static/media \ - && chmod -R 755 /app/logs /app/app/static/tmp /app/app/static/media \ - && chmod 777 /app/logs +# Compile translations and set permissions in a single layer +RUN source /app/venv/bin/activate \ + && (pybabel compile -d app/translations || true) \ + && chmod 755 /app/logs /app/app/static/tmp /app/app/static/media # Copy shared entrypoint utilities COPY entrypoint-common.sh /usr/local/bin/entrypoint-common.sh COPY entrypoint-init.sh /usr/local/bin/entrypoint-init.sh -RUN chmod +x /usr/local/bin/entrypoint-common.sh /usr/local/bin/entrypoint-init.sh - -# Create directories for logs and runtime -RUN mkdir -p /var/log/piefed /var/run/piefed \ - && chown -R piefed:piefed /var/log/piefed /var/run/piefed \ No newline at end of file +RUN chmod +x /usr/local/bin/entrypoint-common.sh /usr/local/bin/entrypoint-init.sh \ No newline at end of file diff --git a/build/piefed/piefed-base/entrypoint-init.sh b/build/piefed/piefed-base/entrypoint-init.sh index 897a1ca..fd27097 100644 --- a/build/piefed/piefed-base/entrypoint-init.sh +++ b/build/piefed/piefed-base/entrypoint-init.sh @@ -4,73 +4,11 @@ set -e # Database initialization entrypoint for PieFed # This script runs as a Kubernetes Job before web/worker pods start -log() { - echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" -} +# Source common functions (wait_for_db, wait_for_redis, log) +. /usr/local/bin/entrypoint-common.sh log "Starting PieFed database initialization..." -# Wait for database to be available -wait_for_db() { - log "Waiting for database connection..." - until python -c " -import psycopg2 -import os -from urllib.parse import urlparse - -try: - # Parse DATABASE_URL - database_url = os.environ.get('DATABASE_URL', '') - if not database_url: - raise Exception('DATABASE_URL not set') - - # Parse the URL to extract connection details - parsed = urlparse(database_url) - conn = psycopg2.connect( - host=parsed.hostname, - port=parsed.port or 5432, - database=parsed.path[1:], # Remove leading slash - user=parsed.username, - password=parsed.password - ) - conn.close() - print('Database connection successful') -except Exception as e: - print(f'Database connection failed: {e}') - exit(1) -" 2>/dev/null; do - log "Database not ready, waiting 2 seconds..." - sleep 2 - done - log "Database connection established" -} - -# Wait for Redis to be available -wait_for_redis() { - log "Waiting for Redis connection..." - until python -c " -import redis -import os - -try: - cache_redis_url = os.environ.get('CACHE_REDIS_URL', '') - if cache_redis_url: - r = redis.from_url(cache_redis_url) - else: - # Fallback to separate host/port for backwards compatibility - r = redis.Redis(host='redis', port=6379, password=os.environ.get('REDIS_PASSWORD', '')) - r.ping() - print('Redis connection successful') -except Exception as e: - print(f'Redis connection failed: {e}') - exit(1) -" 2>/dev/null; do - log "Redis not ready, waiting 2 seconds..." - sleep 2 - done - log "Redis connection established" -} - # Main initialization sequence main() { # Change to application directory diff --git a/build/piefed/piefed-web/.dockerignore b/build/piefed/piefed-web/.dockerignore new file mode 100644 index 0000000..e0c1c7c --- /dev/null +++ b/build/piefed/piefed-web/.dockerignore @@ -0,0 +1,29 @@ +# Git +.git +.gitignore + +# Documentation +*.md +README* + +# Python cache +__pycache__ +*.pyc +*.pyo +*.pyd +.pytest_cache +.coverage +htmlcov/ + +# Environment files +.env* +*.env + +# IDE +.vscode/ +.idea/ +*.swp +*.swo + +# Build artifacts +*.log diff --git a/build/piefed/piefed-web/Dockerfile b/build/piefed/piefed-web/Dockerfile index 303df5d..48e369e 100644 --- a/build/piefed/piefed-web/Dockerfile +++ b/build/piefed/piefed-web/Dockerfile @@ -1,6 +1,7 @@ FROM piefed-base AS piefed-web -# No additional Alpine packages needed - uWSGI installed via pip in base image +# Install nginx (only needed for web container) +RUN apk add --no-cache nginx # Web-specific Python configuration for Flask RUN echo 'import os' > /app/uwsgi_config.py && \ @@ -13,14 +14,10 @@ COPY supervisord-web.conf /etc/supervisor/conf.d/supervisord.conf COPY entrypoint-web.sh /entrypoint.sh RUN chmod +x /entrypoint.sh -# Create nginx directories and set permissions -RUN mkdir -p /var/log/nginx /var/log/supervisor /var/log/uwsgi \ - && chown -R nginx:nginx /var/log/nginx \ - && chown -R piefed:piefed /var/log/uwsgi \ - && mkdir -p /var/cache/nginx \ - && chown -R nginx:nginx /var/cache/nginx \ - && chown -R piefed:piefed /app/logs \ - && chmod -R 755 /app/logs +# Create nginx and log directories with proper permissions in a single layer +RUN mkdir -p /var/log/nginx /var/log/supervisor /var/log/uwsgi /var/cache/nginx \ + && chown -R nginx:nginx /var/log/nginx /var/cache/nginx \ + && chown -R piefed:piefed /var/log/uwsgi /app/logs # Health check optimized for web container HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ diff --git a/build/piefed/piefed-worker/.dockerignore b/build/piefed/piefed-worker/.dockerignore new file mode 100644 index 0000000..e0c1c7c --- /dev/null +++ b/build/piefed/piefed-worker/.dockerignore @@ -0,0 +1,29 @@ +# Git +.git +.gitignore + +# Documentation +*.md +README* + +# Python cache +__pycache__ +*.pyc +*.pyo +*.pyd +.pytest_cache +.coverage +htmlcov/ + +# Environment files +.env* +*.env + +# IDE +.vscode/ +.idea/ +*.swp +*.swo + +# Build artifacts +*.log diff --git a/build/piefed/piefed-worker/Dockerfile b/build/piefed/piefed-worker/Dockerfile index 8605f56..734a3b0 100644 --- a/build/piefed/piefed-worker/Dockerfile +++ b/build/piefed/piefed-worker/Dockerfile @@ -1,8 +1,5 @@ FROM piefed-base AS piefed-worker -# Install additional packages needed for worker container -RUN apk add --no-cache redis - # Worker-specific Python configuration for background processing RUN echo "import sys" > /app/worker_config.py && \ echo "sys.path.append('/app')" >> /app/worker_config.py diff --git a/build/pixelfed/README.md b/build/pixelfed/README.md index 8d13f17..02e0a7f 100644 --- a/build/pixelfed/README.md +++ b/build/pixelfed/README.md @@ -32,18 +32,18 @@ This will: 1. Build the base image with all Pixelfed dependencies 2. Build the web container with Nginx + PHP-FPM 3. Build the worker container with Horizon + Scheduler -4. Push to your Harbor registry: `` +4. Push to your Harbor registry: `registry.keyboardvagabond.com` ### **Individual Container Builds** ```bash # Build just web container cd pixelfed-web && docker build --platform linux/arm64 \ - -t /pixelfed/web:v6 . + -t registry.keyboardvagabond.com/pixelfed/web:v6 . # Build just worker container cd pixelfed-worker && docker build --platform linux/arm64 \ - -t /pixelfed/worker:v0.12.6 . + -t registry.keyboardvagabond.com/pixelfed/worker:v0.12.6 . ``` ## 📦 **Container Details** @@ -84,14 +84,14 @@ APP_DOMAIN=pixelfed.keyboardvagabond.com DB_HOST=postgresql-shared-rw.postgresql-system.svc.cluster.local DB_DATABASE=pixelfed DB_USERNAME=pixelfed -DB_PASSWORD= +DB_PASSWORD=secure_password_here ``` #### **Redis Configuration** ```bash REDIS_HOST=redis-ha-haproxy.redis-system.svc.cluster.local REDIS_PORT=6379 -REDIS_PASSWORD= +REDIS_PASSWORD=redis_password_if_needed ``` #### **S3 Media Storage (Backblaze B2)** @@ -104,12 +104,12 @@ FILESYSTEM_CLOUD=s3 FILESYSTEM_DISK=s3 # Backblaze B2 S3-compatible configuration -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= +AWS_ACCESS_KEY_ID=your_b2_key_id +AWS_SECRET_ACCESS_KEY=your_b2_secret_key AWS_DEFAULT_REGION=eu-central-003 AWS_BUCKET=pixelfed-bucket AWS_URL=https://pm.keyboardvagabond.com/ -AWS_ENDPOINT= +AWS_ENDPOINT=https://s3.eu-central-003.backblazeb2.com AWS_USE_PATH_STYLE_ENDPOINT=false AWS_ROOT= AWS_VISIBILITY=public @@ -118,13 +118,13 @@ AWS_VISIBILITY=public CDN_DOMAIN=pm.keyboardvagabond.com ``` -#### **Email (SMTP)** +#### **Email (Mailgun)** ```bash MAIL_MAILER=smtp -MAIL_HOST= +MAIL_HOST=smtp.eu.mailgun.org MAIL_PORT=587 MAIL_USERNAME=pixelfed@mail.keyboardvagabond.com -MAIL_PASSWORD= +MAIL_PASSWORD= MAIL_ENCRYPTION=tls MAIL_FROM_ADDRESS=pixelfed@mail.keyboardvagabond.com MAIL_FROM_NAME="Pixelfed at Keyboard Vagabond" diff --git a/build/pixelfed/build-all.sh b/build/pixelfed/build-all.sh index 99e3052..0c8fcb6 100755 --- a/build/pixelfed/build-all.sh +++ b/build/pixelfed/build-all.sh @@ -2,7 +2,7 @@ set -e # Configuration -REGISTRY="" +REGISTRY="registry.keyboardvagabond.com" VERSION="v0.12.6" PLATFORM="linux/arm64" @@ -64,6 +64,11 @@ echo -e "${BLUE}Built containers:${NC}" echo -e " • ${GREEN}$REGISTRY/library/pixelfed-web:$VERSION${NC}" echo -e " • ${GREEN}$REGISTRY/library/pixelfed-worker:$VERSION${NC}" +# Show image sizes +echo +echo -e "${BLUE}📊 Built image sizes:${NC}" +docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" | grep -E "(pixelfed-base|pixelfed-web|pixelfed-worker)" | head -10 + # Ask about pushing to registry echo read -p "Push all containers to Harbor registry? (y/N): " -n 1 -r diff --git a/build/pixelfed/pixelfed-base/Dockerfile b/build/pixelfed/pixelfed-base/Dockerfile index 6323574..1bfb3f3 100644 --- a/build/pixelfed/pixelfed-base/Dockerfile +++ b/build/pixelfed/pixelfed-base/Dockerfile @@ -1,17 +1,19 @@ # Multi-stage build for Pixelfed - optimized base image FROM php:8.3-fpm-alpine AS builder +LABEL org.opencontainers.image.title="Pixelfed Base" \ + org.opencontainers.image.description="Shared base image for Pixelfed photo sharing platform" \ + org.opencontainers.image.source="https://github.com/pixelfed/pixelfed" \ + org.opencontainers.image.vendor="Keyboard Vagabond" + # Set environment variables ENV PIXELFED_VERSION=v0.12.6 ENV TZ=UTC ENV APP_ENV=production ENV APP_DEBUG=false -# Use HTTP repositories and install build dependencies -RUN echo "http://dl-cdn.alpinelinux.org/alpine/v3.22/main" > /etc/apk/repositories \ - && echo "http://dl-cdn.alpinelinux.org/alpine/v3.22/community" >> /etc/apk/repositories \ - && apk update \ - && apk add --no-cache \ +# Install build dependencies in a single layer +RUN apk add --no-cache \ ca-certificates \ git \ curl \ @@ -28,19 +30,16 @@ RUN echo "http://dl-cdn.alpinelinux.org/alpine/v3.22/main" > /etc/apk/repositori icu-dev \ gettext-dev \ imagemagick-dev \ - # Node.js and build tools for asset compilation + # Node.js for asset compilation nodejs \ npm \ - # Compilation tools for native modules + # Build tools build-base \ - python3 \ - make \ - # Additional build tools for PECL extensions autoconf \ pkgconfig \ $PHPIZE_DEPS -# Install PHP extensions +# Install PHP extensions (done ONCE - will be copied to runtime stage) RUN docker-php-ext-configure gd --with-freetype --with-jpeg \ && docker-php-ext-install -j$(nproc) \ pdo_pgsql \ @@ -52,9 +51,15 @@ RUN docker-php-ext-configure gd --with-freetype --with-jpeg \ exif \ pcntl \ opcache \ - # Install ImageMagick PHP extension via PECL - && pecl install imagick \ - && docker-php-ext-enable imagick + # Build imagick from source for PHP 8.3 compatibility + && git clone https://github.com/Imagick/imagick.git --depth 1 -b master /tmp/imagick \ + && cd /tmp/imagick \ + && phpize \ + && ./configure \ + && make \ + && make install \ + && docker-php-ext-enable imagick \ + && rm -rf /tmp/imagick # Install Composer COPY --from=composer:2 /usr/bin/composer /usr/bin/composer @@ -82,10 +87,7 @@ RUN composer install --no-dev --optimize-autoloader --no-interaction \ && rm -f bootstrap/cache/packages.php bootstrap/cache/services.php || true \ && php artisan package:discover --ansi || true -# Install Node.js and build assets (skip post-install scripts to avoid node-datachannel compilation) -USER root -RUN apk add --no-cache nodejs npm -USER pixelfed +# Build frontend assets (skip post-install scripts to avoid node-datachannel compilation) RUN echo "ignore-scripts=true" > .npmrc \ && npm ci \ && npm run production \ @@ -99,21 +101,23 @@ USER root # ================================ FROM php:8.3-fpm-alpine AS pixelfed-base +LABEL org.opencontainers.image.title="Pixelfed Base" \ + org.opencontainers.image.description="Shared base image for Pixelfed photo sharing platform" \ + org.opencontainers.image.source="https://github.com/pixelfed/pixelfed" \ + org.opencontainers.image.vendor="Keyboard Vagabond" + # Set environment variables ENV TZ=UTC ENV APP_ENV=production ENV APP_DEBUG=false # Install only runtime dependencies (no -dev packages, no build tools) -RUN echo "http://dl-cdn.alpinelinux.org/alpine/v3.22/main" > /etc/apk/repositories \ - && echo "http://dl-cdn.alpinelinux.org/alpine/v3.22/community" >> /etc/apk/repositories \ - && apk update \ - && apk add --no-cache \ +RUN apk add --no-cache \ ca-certificates \ curl \ su-exec \ dcron \ - # Runtime libraries for PHP extensions (no -dev versions) + # Runtime libraries for PHP extensions libpng \ oniguruma \ libxml2 \ @@ -121,58 +125,23 @@ RUN echo "http://dl-cdn.alpinelinux.org/alpine/v3.22/main" > /etc/apk/repositori libjpeg-turbo \ libzip \ libpq \ - icu \ + icu-libs \ gettext \ - # Image optimization tools (runtime only) + # ImageMagick runtime libraries + imagemagick \ + imagemagick-libs \ + # Image optimization tools (required by Pixelfed) jpegoptim \ optipng \ pngquant \ gifsicle \ - imagemagick \ + # FFmpeg for video thumbnails (required by Pixelfed) ffmpeg \ && rm -rf /var/cache/apk/* -# Re-install PHP extensions in runtime stage (this ensures compatibility) -RUN apk add --no-cache --virtual .build-deps \ - libpng-dev \ - oniguruma-dev \ - libxml2-dev \ - freetype-dev \ - libjpeg-turbo-dev \ - libzip-dev \ - postgresql-dev \ - icu-dev \ - gettext-dev \ - imagemagick-dev \ - # Additional build tools for PECL extensions - autoconf \ - pkgconfig \ - git \ - $PHPIZE_DEPS \ - && docker-php-ext-configure gd --with-freetype --with-jpeg \ - && docker-php-ext-install -j$(nproc) \ - pdo_pgsql \ - pgsql \ - gd \ - zip \ - intl \ - bcmath \ - exif \ - pcntl \ - opcache \ - # Install ImageMagick PHP extension from source (PHP 8.3 compatibility) - && git clone https://github.com/Imagick/imagick.git --depth 1 /tmp/imagick \ - && cd /tmp/imagick \ - && git fetch origin master \ - && git switch master \ - && phpize \ - && ./configure \ - && make \ - && make install \ - && docker-php-ext-enable imagick \ - && rm -rf /tmp/imagick \ - && apk del .build-deps \ - && rm -rf /var/cache/apk/* +# Copy PHP extensions from builder (KEY OPTIMIZATION - no recompilation!) +COPY --from=builder /usr/local/lib/php/extensions/ /usr/local/lib/php/extensions/ +COPY --from=builder /usr/local/etc/php/conf.d/ /usr/local/etc/php/conf.d/ # Create pixelfed user RUN addgroup -g 1000 pixelfed \ @@ -184,7 +153,7 @@ WORKDIR /var/www/pixelfed # Copy application from builder (source + compiled assets + vendor dependencies) COPY --from=builder --chown=pixelfed:pixelfed /var/www/pixelfed /var/www/pixelfed -# Copy custom assets (logo, banners, etc.) to override defaults. Doesn't override the png versions. +# Copy custom assets (logo, banners, etc.) to override defaults COPY --chown=pixelfed:pixelfed custom-assets/img/*.svg /var/www/pixelfed/public/img/ # Clear any cached configuration files and set proper permissions @@ -193,15 +162,17 @@ RUN rm -rf /var/www/pixelfed/bootstrap/cache/*.php || true \ && chmod -R 755 /var/www/pixelfed/bootstrap/cache \ && chown -R pixelfed:pixelfed /var/www/pixelfed/bootstrap/cache -# Configure PHP for better performance -RUN echo "opcache.enable=1" >> /usr/local/etc/php/conf.d/docker-php-ext-opcache.ini \ - && echo "opcache.revalidate_freq=0" >> /usr/local/etc/php/conf.d/docker-php-ext-opcache.ini \ - && echo "opcache.validate_timestamps=0" >> /usr/local/etc/php/conf.d/docker-php-ext-opcache.ini \ - && echo "opcache.max_accelerated_files=10000" >> /usr/local/etc/php/conf.d/docker-php-ext-opcache.ini \ - && echo "opcache.memory_consumption=192" >> /usr/local/etc/php/conf.d/docker-php-ext-opcache.ini \ - && echo "opcache.max_wasted_percentage=10" >> /usr/local/etc/php/conf.d/docker-php-ext-opcache.ini \ - && echo "opcache.interned_strings_buffer=16" >> /usr/local/etc/php/conf.d/docker-php-ext-opcache.ini \ - && echo "opcache.fast_shutdown=1" >> /usr/local/etc/php/conf.d/docker-php-ext-opcache.ini +# Configure PHP OPcache for production performance +RUN { \ + echo "opcache.enable=1"; \ + echo "opcache.revalidate_freq=0"; \ + echo "opcache.validate_timestamps=0"; \ + echo "opcache.max_accelerated_files=10000"; \ + echo "opcache.memory_consumption=192"; \ + echo "opcache.max_wasted_percentage=10"; \ + echo "opcache.interned_strings_buffer=16"; \ + echo "opcache.fast_shutdown=1"; \ + } >> /usr/local/etc/php/conf.d/docker-php-ext-opcache.ini # Copy shared entrypoint utilities COPY entrypoint-common.sh /usr/local/bin/entrypoint-common.sh diff --git a/build/postgresql-postgis/Dockerfile b/build/postgresql-postgis/Dockerfile index 530ac90..56b9498 100644 --- a/build/postgresql-postgis/Dockerfile +++ b/build/postgresql-postgis/Dockerfile @@ -1,35 +1,270 @@ -# CloudNativePG-compatible PostGIS image -# Uses imresamu/postgis as base which has ARM64 support -FROM imresamu/postgis:16-3.4 +# ============================================================================= +# PostgreSQL 18 + PostGIS 3.6 for CloudNativePG (ARM64 build from source) +# ============================================================================= +# This Dockerfile builds PostGIS from source for ARM64 architecture since +# the official postgis/postgis images don't have ARM64 support for PG18 yet. +# +# Build: docker build --platform linux/arm64 -t cnpg-postgis:18-3.6 . +# Test: docker run --rm -e POSTGRES_PASSWORD=test cnpg-postgis:18-3.6 postgres --version +# ============================================================================= -# Get additional tools from CloudNativePG image -FROM ghcr.io/cloudnative-pg/postgresql:16.6 as cnpg-tools +# ----------------------------------------------------------------------------- +# Build arguments - Pin versions for reproducible builds +# ----------------------------------------------------------------------------- +ARG PG_MAJOR=18 +ARG POSTGIS_VERSION=3.6.1 +ARG GEOS_VERSION=3.13.0 +# PROJ 9.4.1 is more stable for building; 9.5.x has additional deps +ARG PROJ_VERSION=9.4.1 +ARG GDAL_VERSION=3.10.1 +ARG SFCGAL_VERSION=2.0.0 -# Final stage: PostGIS with CloudNativePG tools -FROM imresamu/postgis:16-3.4 +# ============================================================================= +# Stage 1: Build PostGIS and dependencies from source +# ============================================================================= +FROM postgres:${PG_MAJOR}-bookworm AS builder -USER root +ARG PG_MAJOR +ARG POSTGIS_VERSION +ARG GEOS_VERSION +ARG PROJ_VERSION +ARG GDAL_VERSION -# Fix user ID compatibility with CloudNativePG (user ID 26) -# CloudNativePG expects postgres user to have ID 26, but imresamu/postgis uses 999 -# The tape group (ID 26) already exists, so we'll change postgres user to use it -RUN usermod -u 26 -g 26 postgres && \ - delgroup postgres && \ - chown -R postgres:tape /var/lib/postgresql && \ - chown -R postgres:tape /var/run/postgresql - -# Copy barman and other tools from CloudNativePG image -COPY --from=cnpg-tools /usr/local/bin/barman* /usr/local/bin/ - -# Install any additional packages that CloudNativePG might need -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - curl \ - jq \ +# Install build dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + # Build tools + build-essential \ + cmake \ + ninja-build \ + pkg-config \ + git \ + wget \ + ca-certificates \ + # PostgreSQL development + postgresql-server-dev-${PG_MAJOR} \ + # Required libraries + libxml2-dev \ + libjson-c-dev \ + libprotobuf-c-dev \ + protobuf-c-compiler \ + libsqlite3-dev \ + sqlite3 \ + libtiff-dev \ + libcurl4-openssl-dev \ + libssl-dev \ + zlib1g-dev \ + liblzma-dev \ + libzstd-dev \ + libpng-dev \ + libjpeg-dev \ + libwebp-dev \ + # Additional dependencies + libpcre2-dev \ + autoconf \ + automake \ + libtool \ + # PROJ additional requirements + nlohmann-json3-dev \ + libgeotiff-dev \ && rm -rf /var/lib/apt/lists/* -# Switch back to postgres user (now with correct ID 26) +WORKDIR /build + +# ----------------------------------------------------------------------------- +# Build GEOS (Geometry Engine) +# ----------------------------------------------------------------------------- +RUN wget -q https://download.osgeo.org/geos/geos-${GEOS_VERSION}.tar.bz2 \ + && tar xjf geos-${GEOS_VERSION}.tar.bz2 \ + && cd geos-${GEOS_VERSION} \ + && mkdir build && cd build \ + && cmake .. \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_TESTING=OFF \ + && ninja \ + && ninja install \ + && cd /build && rm -rf geos-* + +# ----------------------------------------------------------------------------- +# Build PROJ (Cartographic Projections) +# ----------------------------------------------------------------------------- +RUN wget -q https://download.osgeo.org/proj/proj-${PROJ_VERSION}.tar.gz \ + && tar xzf proj-${PROJ_VERSION}.tar.gz \ + && cd proj-${PROJ_VERSION} \ + && mkdir build && cd build \ + && cmake .. \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_TESTING=OFF \ + -DENABLE_CURL=ON \ + -DENABLE_TIFF=ON \ + && ninja \ + && ninja install \ + && cd /build && rm -rf proj-* + +# Update library cache after PROJ install +RUN ldconfig + +# ----------------------------------------------------------------------------- +# Build GDAL (Geospatial Data Abstraction Library) +# ----------------------------------------------------------------------------- +RUN wget -q https://github.com/OSGeo/gdal/releases/download/v${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz \ + && tar xzf gdal-${GDAL_VERSION}.tar.gz \ + && cd gdal-${GDAL_VERSION} \ + && mkdir build && cd build \ + && cmake .. \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_TESTING=OFF \ + -DBUILD_APPS=OFF \ + -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF \ + -DOGR_BUILD_OPTIONAL_DRIVERS=OFF \ + -DGDAL_USE_GEOS=ON \ + -DGDAL_USE_PROJ=ON \ + -DGDAL_USE_TIFF=ON \ + -DGDAL_USE_GEOTIFF=ON \ + -DGDAL_USE_PNG=ON \ + -DGDAL_USE_JPEG=ON \ + -DGDAL_USE_WEBP=ON \ + -DGDAL_USE_CURL=ON \ + -DGDAL_USE_SQLITE3=ON \ + -DGDAL_USE_POSTGRESQL=ON \ + && ninja \ + && ninja install \ + && cd /build && rm -rf gdal-* + +# Update library cache after GDAL install +RUN ldconfig + +# ----------------------------------------------------------------------------- +# Build PostGIS +# ----------------------------------------------------------------------------- +# Set library paths so configure can find GDAL, GEOS, PROJ +ENV LD_LIBRARY_PATH=/usr/local/lib +ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig + +RUN wget -q https://download.osgeo.org/postgis/source/postgis-${POSTGIS_VERSION}.tar.gz \ + && tar xzf postgis-${POSTGIS_VERSION}.tar.gz \ + && cd postgis-${POSTGIS_VERSION} \ + && LDFLAGS="-L/usr/local/lib" \ + CPPFLAGS="-I/usr/local/include" \ + ./configure \ + --with-pgconfig=/usr/lib/postgresql/${PG_MAJOR}/bin/pg_config \ + --with-geosconfig=/usr/local/bin/geos-config \ + --with-projdir=/usr/local \ + --with-gdalconfig=/usr/local/bin/gdal-config \ + --with-protobufdir=/usr \ + --without-sfcgal \ + && make -j$(nproc) \ + && make install DESTDIR=/postgis-install \ + && cd /build && rm -rf postgis-* + +# ============================================================================= +# Stage 2: Get CNPG tools (barman-cloud for backup/restore) +# ============================================================================= +FROM ghcr.io/cloudnative-pg/postgresql:${PG_MAJOR} AS cnpg-tools + +# ============================================================================= +# Stage 3: Final runtime image +# ============================================================================= +FROM postgres:${PG_MAJOR}-bookworm + +ARG PG_MAJOR +ARG POSTGIS_VERSION + +LABEL maintainer="Keyboard Vagabond " +LABEL description="PostgreSQL ${PG_MAJOR} with PostGIS ${POSTGIS_VERSION} for CloudNativePG (ARM64)" +LABEL org.opencontainers.image.source="https://keyboardvagabond.com" + +ENV POSTGIS_MAJOR=3 +ENV POSTGIS_VERSION=${POSTGIS_VERSION} + +# Install runtime dependencies only (no build tools) +RUN apt-get update && apt-get install -y --no-install-recommends \ + # Runtime libraries for GEOS/PROJ/GDAL/PostGIS + libxml2 \ + libjson-c5 \ + libprotobuf-c1 \ + libsqlite3-0 \ + libtiff6 \ + libcurl4 \ + libssl3 \ + zlib1g \ + liblzma5 \ + libzstd1 \ + libpng16-16 \ + libjpeg62-turbo \ + libwebp7 \ + libpcre2-8-0 \ + # Additional utilities + ca-certificates \ + curl \ + jq \ + # Python for barman-cloud + python3 \ + python3-boto3 \ + python3-botocore \ + && rm -rf /var/lib/apt/lists/* + +# Copy compiled libraries from builder +COPY --from=builder /usr/local/lib/ /usr/local/lib/ +COPY --from=builder /usr/local/share/proj/ /usr/local/share/proj/ +COPY --from=builder /usr/local/share/gdal/ /usr/local/share/gdal/ +COPY --from=builder /usr/local/bin/geos-config /usr/local/bin/ +COPY --from=builder /usr/local/bin/gdal-config /usr/local/bin/ +COPY --from=builder /usr/local/bin/proj /usr/local/bin/ +COPY --from=builder /usr/local/bin/projinfo /usr/local/bin/ + +# Copy PostGIS installation (modern PostGIS uses extension dir, not contrib) +COPY --from=builder /postgis-install/usr/lib/postgresql/${PG_MAJOR}/lib/ /usr/lib/postgresql/${PG_MAJOR}/lib/ +COPY --from=builder /postgis-install/usr/share/postgresql/${PG_MAJOR}/extension/ /usr/share/postgresql/${PG_MAJOR}/extension/ + +# Update library cache +RUN ldconfig + +# Copy barman-cloud tools from CNPG image (they're in /usr/local/bin/) +COPY --from=cnpg-tools /usr/local/bin/barman* /usr/local/bin/ + +# ----------------------------------------------------------------------------- +# Fix user ID for CloudNativePG compatibility (requires UID 26) +# ----------------------------------------------------------------------------- +RUN set -eux; \ + CURRENT_UID=$(id -u postgres); \ + if [ "$CURRENT_UID" != "26" ]; then \ + # Check if UID 26 is already in use + if getent passwd 26 >/dev/null 2>&1; then \ + EXISTING_USER=$(getent passwd 26 | cut -d: -f1); \ + usermod -u 9999 "$EXISTING_USER" 2>/dev/null || true; \ + fi; \ + # Change postgres user to UID 26 + usermod -u 26 postgres; \ + # Fix ownership of postgres directories + find /var/lib/postgresql -user $CURRENT_UID -exec chown -h 26 {} \; 2>/dev/null || true; \ + find /var/run/postgresql -user $CURRENT_UID -exec chown -h 26 {} \; 2>/dev/null || true; \ + chown -R postgres:postgres /var/lib/postgresql /var/run/postgresql 2>/dev/null || true; \ + fi + +# Copy initialization and update scripts +RUN mkdir -p /docker-entrypoint-initdb.d +COPY ./initdb-postgis.sh /docker-entrypoint-initdb.d/10_postgis.sh +COPY ./update-postgis.sh /usr/local/bin/ +RUN chmod +x /docker-entrypoint-initdb.d/10_postgis.sh /usr/local/bin/update-postgis.sh + +# ----------------------------------------------------------------------------- +# Verify installation +# ----------------------------------------------------------------------------- +RUN set -eux; \ + postgres --version; \ + echo "GEOS: $(geos-config --version)"; \ + echo "PROJ: $(projinfo 2>&1 | head -1 || echo 'installed')"; \ + echo "GDAL: $(gdal-config --version)"; \ + id postgres; \ + ls -la /usr/lib/postgresql/${PG_MAJOR}/lib/postgis*.so || true + +# Switch to postgres user USER postgres -# Keep the standard PostgreSQL entrypoint -# CloudNativePG operator will manage the container lifecycle +EXPOSE 5432 diff --git a/build/postgresql-postgis/Dockerfile.upgrade b/build/postgresql-postgis/Dockerfile.upgrade new file mode 100644 index 0000000..acec277 --- /dev/null +++ b/build/postgresql-postgis/Dockerfile.upgrade @@ -0,0 +1,267 @@ +# ============================================================================= +# PostgreSQL 16→18 Upgrade Image for CloudNativePG pg_upgrade +# ============================================================================= +# This special image contains BOTH PG16 and PG18 binaries + PostGIS, required +# for CloudNativePG's declarative pg_upgrade feature. +# +# Use this image ONLY for the upgrade process. After upgrade completes, +# switch to the regular cnpg-postgis:18-3.6 image. +# +# Build: docker build --platform linux/arm64 -f Dockerfile.upgrade -t cnpg-postgis:upgrade-16-to-18 . +# ============================================================================= + +# ----------------------------------------------------------------------------- +# Build arguments - Pin versions for reproducible builds +# ----------------------------------------------------------------------------- +ARG PG_OLD=16 +ARG PG_NEW=18 +ARG POSTGIS_OLD=3.4.3 +ARG POSTGIS_NEW=3.6.1 +ARG GEOS_VERSION=3.13.0 +ARG PROJ_VERSION=9.4.1 +ARG GDAL_VERSION=3.10.1 + +# ============================================================================= +# Stage 1: Build PostGIS for PG16 (old version) +# ============================================================================= +FROM postgres:16-bookworm AS builder-pg16 + +ARG POSTGIS_OLD +ARG GEOS_VERSION +ARG PROJ_VERSION +ARG GDAL_VERSION + +# Install build dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential cmake ninja-build pkg-config git wget ca-certificates \ + postgresql-server-dev-16 \ + libxml2-dev libjson-c-dev libprotobuf-c-dev protobuf-c-compiler \ + libsqlite3-dev sqlite3 libtiff-dev libcurl4-openssl-dev libssl-dev \ + zlib1g-dev liblzma-dev libzstd-dev libpng-dev libjpeg-dev libwebp-dev \ + libpcre2-dev autoconf automake libtool nlohmann-json3-dev libgeotiff-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /build + +# Build GEOS +RUN wget -q https://download.osgeo.org/geos/geos-${GEOS_VERSION}.tar.bz2 \ + && tar xjf geos-${GEOS_VERSION}.tar.bz2 \ + && cd geos-${GEOS_VERSION} && mkdir build && cd build \ + && cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_TESTING=OFF \ + && ninja && ninja install \ + && cd /build && rm -rf geos-* + +# Build PROJ +RUN wget -q https://download.osgeo.org/proj/proj-${PROJ_VERSION}.tar.gz \ + && tar xzf proj-${PROJ_VERSION}.tar.gz \ + && cd proj-${PROJ_VERSION} && mkdir build && cd build \ + && cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_TESTING=OFF -DENABLE_CURL=ON -DENABLE_TIFF=ON \ + && ninja && ninja install \ + && cd /build && rm -rf proj-* && ldconfig + +# Build GDAL +RUN wget -q https://github.com/OSGeo/gdal/releases/download/v${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz \ + && tar xzf gdal-${GDAL_VERSION}.tar.gz \ + && cd gdal-${GDAL_VERSION} && mkdir build && cd build \ + && cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_TESTING=OFF -DBUILD_APPS=OFF \ + -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF -DOGR_BUILD_OPTIONAL_DRIVERS=OFF \ + -DGDAL_USE_GEOS=ON -DGDAL_USE_PROJ=ON -DGDAL_USE_TIFF=ON \ + -DGDAL_USE_GEOTIFF=ON -DGDAL_USE_PNG=ON -DGDAL_USE_JPEG=ON \ + -DGDAL_USE_WEBP=ON -DGDAL_USE_CURL=ON -DGDAL_USE_SQLITE3=ON \ + -DGDAL_USE_POSTGRESQL=ON \ + && ninja && ninja install \ + && cd /build && rm -rf gdal-* && ldconfig + +# Build PostGIS for PG16 +ENV LD_LIBRARY_PATH=/usr/local/lib +ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig + +RUN wget -q https://download.osgeo.org/postgis/source/postgis-${POSTGIS_OLD}.tar.gz \ + && tar xzf postgis-${POSTGIS_OLD}.tar.gz \ + && cd postgis-${POSTGIS_OLD} \ + && LDFLAGS="-L/usr/local/lib" CPPFLAGS="-I/usr/local/include" \ + ./configure \ + --with-pgconfig=/usr/lib/postgresql/16/bin/pg_config \ + --with-geosconfig=/usr/local/bin/geos-config \ + --with-projdir=/usr/local \ + --with-gdalconfig=/usr/local/bin/gdal-config \ + --with-protobufdir=/usr \ + --without-sfcgal \ + && make -j$(nproc) \ + && make install DESTDIR=/postgis-install-pg16 \ + && cd /build && rm -rf postgis-* + +# ============================================================================= +# Stage 2: Build PostGIS for PG18 (new version) +# ============================================================================= +FROM postgres:18-bookworm AS builder-pg18 + +ARG POSTGIS_NEW +ARG GEOS_VERSION +ARG PROJ_VERSION +ARG GDAL_VERSION + +# Install build dependencies (same as PG16) +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential cmake ninja-build pkg-config git wget ca-certificates \ + postgresql-server-dev-18 \ + libxml2-dev libjson-c-dev libprotobuf-c-dev protobuf-c-compiler \ + libsqlite3-dev sqlite3 libtiff-dev libcurl4-openssl-dev libssl-dev \ + zlib1g-dev liblzma-dev libzstd-dev libpng-dev libjpeg-dev libwebp-dev \ + libpcre2-dev autoconf automake libtool nlohmann-json3-dev libgeotiff-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /build + +# Build GEOS +RUN wget -q https://download.osgeo.org/geos/geos-${GEOS_VERSION}.tar.bz2 \ + && tar xjf geos-${GEOS_VERSION}.tar.bz2 \ + && cd geos-${GEOS_VERSION} && mkdir build && cd build \ + && cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_TESTING=OFF \ + && ninja && ninja install \ + && cd /build && rm -rf geos-* + +# Build PROJ +RUN wget -q https://download.osgeo.org/proj/proj-${PROJ_VERSION}.tar.gz \ + && tar xzf proj-${PROJ_VERSION}.tar.gz \ + && cd proj-${PROJ_VERSION} && mkdir build && cd build \ + && cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_TESTING=OFF -DENABLE_CURL=ON -DENABLE_TIFF=ON \ + && ninja && ninja install \ + && cd /build && rm -rf proj-* && ldconfig + +# Build GDAL +RUN wget -q https://github.com/OSGeo/gdal/releases/download/v${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz \ + && tar xzf gdal-${GDAL_VERSION}.tar.gz \ + && cd gdal-${GDAL_VERSION} && mkdir build && cd build \ + && cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_TESTING=OFF -DBUILD_APPS=OFF \ + -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF -DOGR_BUILD_OPTIONAL_DRIVERS=OFF \ + -DGDAL_USE_GEOS=ON -DGDAL_USE_PROJ=ON -DGDAL_USE_TIFF=ON \ + -DGDAL_USE_GEOTIFF=ON -DGDAL_USE_PNG=ON -DGDAL_USE_JPEG=ON \ + -DGDAL_USE_WEBP=ON -DGDAL_USE_CURL=ON -DGDAL_USE_SQLITE3=ON \ + -DGDAL_USE_POSTGRESQL=ON \ + && ninja && ninja install \ + && cd /build && rm -rf gdal-* && ldconfig + +# Build PostGIS for PG18 +ENV LD_LIBRARY_PATH=/usr/local/lib +ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig + +RUN wget -q https://download.osgeo.org/postgis/source/postgis-${POSTGIS_NEW}.tar.gz \ + && tar xzf postgis-${POSTGIS_NEW}.tar.gz \ + && cd postgis-${POSTGIS_NEW} \ + && LDFLAGS="-L/usr/local/lib" CPPFLAGS="-I/usr/local/include" \ + ./configure \ + --with-pgconfig=/usr/lib/postgresql/18/bin/pg_config \ + --with-geosconfig=/usr/local/bin/geos-config \ + --with-projdir=/usr/local \ + --with-gdalconfig=/usr/local/bin/gdal-config \ + --with-protobufdir=/usr \ + --without-sfcgal \ + && make -j$(nproc) \ + && make install DESTDIR=/postgis-install-pg18 \ + && cd /build && rm -rf postgis-* + +# ============================================================================= +# Stage 3: Get CNPG tools +# ============================================================================= +FROM ghcr.io/cloudnative-pg/postgresql:18 AS cnpg-tools + +# ============================================================================= +# Stage 4: Final multi-version runtime image +# ============================================================================= +FROM postgres:18-bookworm + +ARG PG_OLD=16 +ARG PG_NEW=18 +ARG POSTGIS_NEW + +LABEL maintainer="Keyboard Vagabond " +LABEL description="PostgreSQL 16→18 upgrade image with PostGIS for CloudNativePG pg_upgrade (ARM64)" +LABEL org.opencontainers.image.source="https://keyboardvagabond.com" +LABEL pg.upgrade.from="16" +LABEL pg.upgrade.to="18" + +ENV POSTGIS_MAJOR=3 +ENV POSTGIS_VERSION=${POSTGIS_NEW} + +# Install runtime dependencies + PostgreSQL 16 binaries +RUN apt-get update && apt-get install -y --no-install-recommends \ + # Runtime libraries for GEOS/PROJ/GDAL/PostGIS + libxml2 libjson-c5 libprotobuf-c1 libsqlite3-0 libtiff6 libcurl4 \ + libssl3 zlib1g liblzma5 libzstd1 libpng16-16 libjpeg62-turbo libwebp7 \ + libpcre2-8-0 ca-certificates curl jq \ + # Python for barman-cloud + python3 python3-boto3 python3-botocore \ + # PostgreSQL 16 binaries (required for pg_upgrade) + postgresql-16 \ + && rm -rf /var/lib/apt/lists/* + +# Copy compiled libraries from PG18 builder (shared between both versions) +COPY --from=builder-pg18 /usr/local/lib/ /usr/local/lib/ +COPY --from=builder-pg18 /usr/local/share/proj/ /usr/local/share/proj/ +COPY --from=builder-pg18 /usr/local/share/gdal/ /usr/local/share/gdal/ +COPY --from=builder-pg18 /usr/local/bin/geos-config /usr/local/bin/ +COPY --from=builder-pg18 /usr/local/bin/gdal-config /usr/local/bin/ +COPY --from=builder-pg18 /usr/local/bin/proj /usr/local/bin/ +COPY --from=builder-pg18 /usr/local/bin/projinfo /usr/local/bin/ + +# Copy PostGIS for PG16 (old version) +COPY --from=builder-pg16 /postgis-install-pg16/usr/lib/postgresql/16/lib/ /usr/lib/postgresql/16/lib/ +COPY --from=builder-pg16 /postgis-install-pg16/usr/share/postgresql/16/extension/ /usr/share/postgresql/16/extension/ + +# Copy PostGIS for PG18 (new version) +COPY --from=builder-pg18 /postgis-install-pg18/usr/lib/postgresql/18/lib/ /usr/lib/postgresql/18/lib/ +COPY --from=builder-pg18 /postgis-install-pg18/usr/share/postgresql/18/extension/ /usr/share/postgresql/18/extension/ + +# Update library cache +RUN ldconfig + +# Copy barman-cloud tools from CNPG image +COPY --from=cnpg-tools /usr/local/bin/barman* /usr/local/bin/ + +# ----------------------------------------------------------------------------- +# Fix user ID for CloudNativePG compatibility (requires UID 26) +# ----------------------------------------------------------------------------- +RUN set -eux; \ + CURRENT_UID=$(id -u postgres); \ + if [ "$CURRENT_UID" != "26" ]; then \ + if getent passwd 26 >/dev/null 2>&1; then \ + EXISTING_USER=$(getent passwd 26 | cut -d: -f1); \ + usermod -u 9999 "$EXISTING_USER" 2>/dev/null || true; \ + fi; \ + usermod -u 26 postgres; \ + find /var/lib/postgresql -user $CURRENT_UID -exec chown -h 26 {} \; 2>/dev/null || true; \ + find /var/run/postgresql -user $CURRENT_UID -exec chown -h 26 {} \; 2>/dev/null || true; \ + chown -R postgres:postgres /var/lib/postgresql /var/run/postgresql 2>/dev/null || true; \ + fi + +# Copy initialization scripts +RUN mkdir -p /docker-entrypoint-initdb.d +COPY ./initdb-postgis.sh /docker-entrypoint-initdb.d/10_postgis.sh +COPY ./update-postgis.sh /usr/local/bin/ +RUN chmod +x /docker-entrypoint-initdb.d/10_postgis.sh /usr/local/bin/update-postgis.sh + +# ----------------------------------------------------------------------------- +# Verify installation - both PG versions and PostGIS +# ----------------------------------------------------------------------------- +RUN set -eux; \ + echo "=== PostgreSQL Versions ==="; \ + /usr/lib/postgresql/16/bin/postgres --version; \ + /usr/lib/postgresql/18/bin/postgres --version; \ + echo "=== PostGIS Libraries ==="; \ + ls -la /usr/lib/postgresql/16/lib/postgis*.so; \ + ls -la /usr/lib/postgresql/18/lib/postgis*.so; \ + echo "=== pg_upgrade Available ==="; \ + /usr/lib/postgresql/18/bin/pg_upgrade --version; \ + echo "=== Shared Libraries ==="; \ + echo "GEOS: $(geos-config --version)"; \ + echo "GDAL: $(gdal-config --version)"; \ + id postgres + +USER postgres + +EXPOSE 5432 diff --git a/build/postgresql-postgis/Migration_Plan.md b/build/postgresql-postgis/Migration_Plan.md new file mode 100644 index 0000000..2e08b4b --- /dev/null +++ b/build/postgresql-postgis/Migration_Plan.md @@ -0,0 +1,184 @@ +# PostgreSQL 18 + PostGIS 3.6 for CloudNativePG (ARM64 Source Build) + +## Overview + +Upgrade from PostgreSQL 16 to PostgreSQL 18 with PostGIS 3.6 for ARM64 CloudNativePG deployment. + +**Why build from source?** +- The official `postgis/postgis:18-3.6` image only supports AMD64, not ARM64 +- `imresamu/postgis` hasn't released PG18 ARM64 images yet +- Building from source ensures ARM64 compatibility for your cluster + +**Current Setup:** +- Image: `registry.keyboardvagabond.com/library/cnpg-postgis:16.6-3.4-v2` +- Base: `imresamu/postgis:16-3.4` +- PostgreSQL: 16.6 +- PostGIS: 3.4 + +**Target Setup:** +- Image: `registry.keyboardvagabond.com/library/cnpg-postgis:18-3.6` +- Base: `postgres:18-bookworm` (official, ARM64 supported) +- PostgreSQL: 18.1 +- PostGIS: 3.6.1 (compiled from source) +- GEOS: 3.13.0 +- PROJ: 9.4.1 +- GDAL: 3.10.1 + +## Extensions Included + +| Extension | Description | Status | +|-----------|-------------|--------| +| postgis | Core GIS functionality | ✓ Compiled | +| postgis_topology | Topology support | ✓ Compiled | +| postgis_raster | Raster support | ✓ Compiled | +| fuzzystrmatch | Fuzzy string matching | ✓ Compiled | +| postgis_tiger_geocoder | US Census TIGER geocoder | ✓ Compiled | + +## Build Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Stage 1: Builder │ +│ - Base: postgres:18-bookworm (ARM64) │ +│ - Compile GEOS 3.13.0 │ +│ - Compile PROJ 9.5.1 │ +│ - Compile GDAL 3.10.1 │ +│ - Compile PostGIS 3.6.1 │ +├─────────────────────────────────────────────────────────────────┤ +│ Stage 2: CNPG Tools │ +│ - ghcr.io/cloudnative-pg/postgresql:18 │ +│ - Extract barman-cloud tools for backup/restore │ +├─────────────────────────────────────────────────────────────────┤ +│ Stage 3: Final Image (minimal) │ +│ - Base: postgres:18-bookworm (ARM64) │ +│ - Copy compiled libs from Stage 1 │ +│ - Copy barman tools from Stage 2 │ +│ - Fix postgres UID to 26 (CNPG requirement) │ +│ - Runtime dependencies only │ +└─────────────────────────────────────────────────────────────────┘ +``` + +## ⚠️ Important: PG18 Data Directory Change + +PostgreSQL 18 changed the default data directory path: + +| Version | Data Directory | +|---------|----------------| +| PG 13-17 | `/var/lib/postgresql/data` | +| PG 18+ | `/var/lib/postgresql` | + +This affects volume mounts in docker-compose and may require CNPG configuration changes. + +## Migration Steps + +### Phase 1: Build and Test Locally + +1. **Build the image (takes 15-30 minutes):** + ```bash + cd build/postgresql-postgis + chmod +x build.sh initdb-postgis.sh update-postgis.sh + ./build.sh + ``` + +2. **Test with docker-compose:** + ```bash + docker-compose -f docker-compose.test.yaml up -d + docker-compose -f docker-compose.test.yaml exec postgres psql -U postgres + + # In psql, verify: + SELECT postgis_full_version(); + SELECT ST_AsText(ST_Point(0, 0)); + \dx -- list extensions + + # Cleanup + docker-compose -f docker-compose.test.yaml down -v + ``` + +3. **Interactive testing:** + ```bash + docker run -it --rm -e POSTGRES_PASSWORD=test cnpg-postgis:18-3.6 bash + ``` + +### Phase 2: Push to Registry + +```bash +docker push registry.keyboardvagabond.com/library/cnpg-postgis:18-3.6 +``` + +### Phase 3: CNPG Upgrade + +**Option A: In-place upgrade (for testing)** + +1. Update `manifests/infrastructure/postgresql/cluster-shared.yaml`: + ```yaml + spec: + imageName: registry.keyboardvagabond.com/library/cnpg-postgis:18-3.6 + ``` + +2. CNPG will handle the rolling upgrade automatically. + +**Option B: Create new cluster and migrate (safer for production)** + +1. Create a new cluster with PG18 image +2. Use pg_dump/pg_restore or CNPG backup/restore +3. Switch applications to new cluster +4. Decommission old cluster + +## CNPG Operator Compatibility + +- Current operator: `>=0.20.0` (Helm chart) +- PostgreSQL 18 support: Requires CNPG operator 1.24+ +- Check current version: + ```bash + kubectl get deployment -n postgresql-system -l app.kubernetes.io/name=cloudnative-pg \ + -o jsonpath='{.items[0].spec.template.spec.containers[0].image}' + ``` + +If upgrade needed, update `manifests/infrastructure/postgresql/operator.yaml`: +```yaml +spec: + chart: + spec: + version: ">=0.23.0" # or specific version with PG18 support +``` + +## Rollback Plan + +If issues occur: +1. Change imageName back to `registry.keyboardvagabond.com/library/cnpg-postgis:16.6-3.4-v2` +2. CNPG will roll back to previous version +3. Restore from backup if data issues + +## Verification Checklist + +- [ ] Image builds successfully on M1 Mac (~15-30 min) +- [ ] postgres user has UID 26 +- [ ] GEOS, PROJ, GDAL compiled correctly +- [ ] PostGIS extensions install correctly +- [ ] barman-cloud tools are present +- [ ] Local docker-compose test passes +- [ ] Spatial queries work (`ST_Point`, `ST_AsText`, etc.) +- [ ] Image pushed to Harbor registry +- [ ] CNPG operator compatible with PG18 +- [ ] Test cluster upgrade in staging (if available) +- [ ] Production cluster upgrade successful +- [ ] All fediverse apps functioning correctly + +## Build Dependencies (compiled from source) + +| Library | Version | Purpose | +|---------|---------|---------| +| GEOS | 3.13.0 | Geometry operations | +| PROJ | 9.4.1 | Coordinate transformations | +| GDAL | 3.10.1 | Raster/vector data access | +| PostGIS | 3.6.1 | PostgreSQL spatial extension | + +## References + +- [PostgreSQL 18 Release Notes](https://www.postgresql.org/docs/18/release-18.html) +- [PostGIS 3.6 Release Notes](https://postgis.net/documentation/getting_started/) +- [docker-postgis GitHub](https://github.com/postgis/docker-postgis) +- [CloudNativePG Documentation](https://cloudnative-pg.io/documentation/) +- [GEOS Downloads](https://download.osgeo.org/geos/) +- [PROJ Downloads](https://download.osgeo.org/proj/) +- [GDAL Downloads](https://github.com/OSGeo/gdal/releases) diff --git a/build/postgresql-postgis/build-upgrade-image.sh b/build/postgresql-postgis/build-upgrade-image.sh new file mode 100755 index 0000000..23b9c6b --- /dev/null +++ b/build/postgresql-postgis/build-upgrade-image.sh @@ -0,0 +1,140 @@ +#!/bin/bash +set -euo pipefail + +# ============================================================================= +# Build script for PostgreSQL 16→18 Upgrade Image with PostGIS +# This image is used ONLY for the pg_upgrade process via CloudNativePG +# ============================================================================= + +# Configuration +REGISTRY="registry.keyboardvagabond.com/library" +IMAGE_NAME="cnpg-postgis" +TAG="upgrade-16-to-18" +FULL_IMAGE="${REGISTRY}/${IMAGE_NAME}:${TAG}" +LOCAL_IMAGE="${IMAGE_NAME}:${TAG}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } +log_step() { echo -e "${BLUE}[STEP]${NC} $1"; } + +cd "$(dirname "$0")" + +echo "" +echo "========================================================" +echo " PostgreSQL 16→18 Upgrade Image Build (ARM64)" +echo "========================================================" +echo "" +log_info "This builds a special image with BOTH PG16 and PG18 binaries" +log_info "Required for CloudNativePG declarative pg_upgrade" +log_warn "Build time: ~30-45 minutes (builds PostGIS twice)" +log_info "Target: ${FULL_IMAGE}" +echo "" + +# ============================================================================= +# Build the upgrade image +# ============================================================================= +log_step "Starting Docker build with Dockerfile.upgrade..." + +BUILD_START=$(date +%s) + +docker build \ + --platform linux/arm64 \ + --progress=plain \ + -f Dockerfile.upgrade \ + -t "${FULL_IMAGE}" \ + -t "${LOCAL_IMAGE}" \ + . + +BUILD_END=$(date +%s) +BUILD_TIME=$((BUILD_END - BUILD_START)) +BUILD_MINS=$((BUILD_TIME / 60)) +BUILD_SECS=$((BUILD_TIME % 60)) + +log_info "Build completed in ${BUILD_MINS}m ${BUILD_SECS}s" + +# ============================================================================= +# Test the upgrade image +# ============================================================================= +echo "" +log_step "Running verification tests..." + +# Test 1: Both PostgreSQL versions present +log_info "Test 1: Checking PostgreSQL versions..." +docker run --rm --platform linux/arm64 "${LOCAL_IMAGE}" bash -c ' + echo " PG16: $(/usr/lib/postgresql/16/bin/postgres --version)" + echo " PG18: $(/usr/lib/postgresql/18/bin/postgres --version)" +' + +# Test 2: pg_upgrade available +log_info "Test 2: Checking pg_upgrade..." +docker run --rm --platform linux/arm64 "${LOCAL_IMAGE}" bash -c ' + /usr/lib/postgresql/18/bin/pg_upgrade --version && echo " ✓ pg_upgrade available" +' + +# Test 3: User ID check +log_info "Test 3: Checking postgres user ID (should be 26)..." +POSTGRES_UID=$(docker run --rm --platform linux/arm64 "${LOCAL_IMAGE}" id -u postgres) +if [ "$POSTGRES_UID" = "26" ]; then + echo " ✓ postgres UID is 26 (CNPG compatible)" +else + log_error "postgres UID is ${POSTGRES_UID}, expected 26" + exit 1 +fi + +# Test 4: PostGIS for both versions +log_info "Test 4: Checking PostGIS libraries..." +docker run --rm --platform linux/arm64 "${LOCAL_IMAGE}" bash -c ' + echo " PG16 PostGIS:" + ls /usr/lib/postgresql/16/lib/postgis*.so 2>/dev/null | xargs -I{} basename {} | sed "s/^/ /" + echo " PG18 PostGIS:" + ls /usr/lib/postgresql/18/lib/postgis*.so 2>/dev/null | xargs -I{} basename {} | sed "s/^/ /" +' + +# Test 5: Shared libraries +log_info "Test 5: Checking shared libraries..." +docker run --rm --platform linux/arm64 "${LOCAL_IMAGE}" bash -c ' + echo " GEOS: $(geos-config --version 2>/dev/null || echo "not found")" + echo " GDAL: $(gdal-config --version 2>/dev/null || echo "not found")" + echo " PROJ: $(projinfo 2>&1 | head -1 || echo "installed")" +' + +# Test 6: Barman tools +log_info "Test 6: Checking barman-cloud tools..." +docker run --rm --platform linux/arm64 "${LOCAL_IMAGE}" \ + bash -c 'ls /usr/local/bin/barman* >/dev/null 2>&1 && echo " ✓ barman-cloud tools available" || echo " ✗ barman-cloud tools not found"' + +# ============================================================================= +# Summary +# ============================================================================= +echo "" +echo "========================================================" +log_info "Upgrade image build completed!" +echo "========================================================" +echo "" +echo "Images built:" +echo " Local: ${LOCAL_IMAGE}" +echo " Harbor: ${FULL_IMAGE}" +echo "" +echo "Build time: ${BUILD_MINS}m ${BUILD_SECS}s" +echo "" +echo "To push to Harbor registry:" +echo " docker push ${FULL_IMAGE}" +echo "" +echo "IMPORTANT: This image is for pg_upgrade ONLY!" +echo "After upgrade completes, switch to: ${REGISTRY}/${IMAGE_NAME}:18-3.6" +echo "" +log_warn "Next steps:" +echo " 1. Push image: docker push ${FULL_IMAGE}" +echo " 2. Take Longhorn snapshot of postgres-shared volumes" +echo " 3. Update cluster-shared.yaml imageName to: ${FULL_IMAGE}" +echo " 4. Apply and monitor the upgrade" +echo " 5. After success, switch to regular 18-3.6 image" +echo "" diff --git a/build/postgresql-postgis/build.sh b/build/postgresql-postgis/build.sh index 654238b..3c8a8c1 100755 --- a/build/postgresql-postgis/build.sh +++ b/build/postgresql-postgis/build.sh @@ -1,41 +1,173 @@ #!/bin/bash -set -e +set -euo pipefail -# Build script for ARM64 PostGIS image compatible with CloudNativePG +# ============================================================================= +# Build script for ARM64 PostgreSQL 18 + PostGIS 3.6 image for CloudNativePG +# This builds PostGIS from source since ARM64 packages aren't available yet +# ============================================================================= -REGISTRY="/library" +# Configuration +REGISTRY="registry.keyboardvagabond.com/library" IMAGE_NAME="cnpg-postgis" -TAG="16.6-3.4-v2" +PG_VERSION="18" +POSTGIS_VERSION="3.6" +TAG="${PG_VERSION}-${POSTGIS_VERSION}" FULL_IMAGE="${REGISTRY}/${IMAGE_NAME}:${TAG}" LOCAL_IMAGE="${IMAGE_NAME}:${TAG}" -echo "Building ARM64 PostGIS image: ${FULL_IMAGE}" +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +log_step() { + echo -e "${BLUE}[STEP]${NC} $1" +} + +# Change to script directory +cd "$(dirname "$0")" + +echo "" +echo "==============================================" +echo " PostgreSQL ${PG_VERSION} + PostGIS ${POSTGIS_VERSION} ARM64 Build" +echo "==============================================" +echo "" +log_info "Building from source (this will take 15-30 minutes)" +log_info "Target: ${FULL_IMAGE}" +echo "" + +# ============================================================================= # Build the image +# ============================================================================= +log_step "Starting Docker build..." + +BUILD_START=$(date +%s) + docker build \ --platform linux/arm64 \ + --progress=plain \ -t "${FULL_IMAGE}" \ + -t "${LOCAL_IMAGE}" \ . -echo "Image built successfully: ${FULL_IMAGE}" +BUILD_END=$(date +%s) +BUILD_TIME=$((BUILD_END - BUILD_START)) +BUILD_MINS=$((BUILD_TIME / 60)) +BUILD_SECS=$((BUILD_TIME % 60)) -# Test the image by running a container and checking PostGIS availability -echo "Testing PostGIS installation..." -docker run --rm --platform linux/arm64 "${FULL_IMAGE}" \ - postgres --version - -echo "Tagging image for local testing..." -docker tag "${FULL_IMAGE}" "${LOCAL_IMAGE}" - -echo "Image built and tagged as:" -echo " Harbor registry: ${FULL_IMAGE}" -echo " Local testing: ${LOCAL_IMAGE}" +log_info "Build completed in ${BUILD_MINS}m ${BUILD_SECS}s" +# ============================================================================= +# Test the image +# ============================================================================= echo "" -echo "To push to Harbor registry (when ready for deployment):" +log_step "Running tests..." + +# Test 1: PostgreSQL version +log_info "Test 1: Checking PostgreSQL version..." +PG_VER=$(docker run --rm --platform linux/arm64 "${LOCAL_IMAGE}" postgres --version) +echo " ${PG_VER}" + +# Test 2: User ID check (CNPG requires UID 26) +log_info "Test 2: Checking postgres user ID (should be 26)..." +POSTGRES_UID=$(docker run --rm --platform linux/arm64 "${LOCAL_IMAGE}" id -u postgres) +if [ "$POSTGRES_UID" = "26" ]; then + echo " ✓ postgres UID is 26 (CNPG compatible)" +else + log_error "postgres UID is ${POSTGRES_UID}, expected 26" + exit 1 +fi + +# Test 3: Library check +log_info "Test 3: Checking compiled libraries..." +docker run --rm --platform linux/arm64 "${LOCAL_IMAGE}" bash -c ' + echo " GEOS: $(geos-config --version 2>/dev/null || echo "not found")" + echo " GDAL: $(gdal-config --version 2>/dev/null || echo "not found")" + echo " PROJ: $(projinfo 2>&1 | head -1 || echo "installed")" +' + +# Test 4: PostGIS extension files +log_info "Test 4: Checking PostGIS extension files..." +docker run --rm --platform linux/arm64 "${LOCAL_IMAGE}" bash -c ' + ls /usr/lib/postgresql/18/lib/postgis*.so 2>/dev/null && echo " ✓ PostGIS shared libraries present" || echo " ✗ PostGIS libraries missing" + ls /usr/share/postgresql/18/extension/postgis*.control 2>/dev/null && echo " ✓ PostGIS extension control files present" || echo " ✗ Extension control files missing" +' + +# Test 5: Full PostGIS functionality test +log_info "Test 5: Testing PostGIS functionality..." +docker run --rm --platform linux/arm64 \ + -e POSTGRES_PASSWORD=testpassword \ + "${LOCAL_IMAGE}" \ + bash -c ' + set -e + # Initialize database + initdb -D /tmp/pgdata -U postgres >/dev/null 2>&1 + + # Start PostgreSQL + pg_ctl -D /tmp/pgdata -o "-c listen_addresses='\'\''" start -w >/dev/null 2>&1 + + # Create extensions + psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS postgis;" >/dev/null 2>&1 + psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS postgis_topology;" >/dev/null 2>&1 + psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;" >/dev/null 2>&1 + psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder;" >/dev/null 2>&1 + + # Get version + POSTGIS_VER=$(psql -U postgres -t -c "SELECT postgis_full_version();" 2>/dev/null | head -1 | xargs) + echo " PostGIS: ${POSTGIS_VER:0:80}..." + + # Test spatial query + psql -U postgres -c "SELECT ST_AsText(ST_Point(0,0));" >/dev/null 2>&1 + echo " ✓ Spatial queries working" + + # Stop PostgreSQL + pg_ctl -D /tmp/pgdata stop >/dev/null 2>&1 + ' && echo " ✓ All PostGIS extensions functional" || log_warn "PostGIS test had issues (check manually)" + +# Test 6: Barman tools +log_info "Test 6: Checking barman-cloud tools..." +docker run --rm --platform linux/arm64 "${LOCAL_IMAGE}" \ + bash -c 'ls /usr/local/bin/barman* >/dev/null 2>&1 && echo " ✓ barman-cloud tools available" || echo " ✗ barman-cloud tools not found"' + +# ============================================================================= +# Summary +# ============================================================================= +echo "" +echo "==============================================" +log_info "Build and tests completed!" +echo "==============================================" +echo "" +echo "Images built:" +echo " Local: ${LOCAL_IMAGE}" +echo " Harbor: ${FULL_IMAGE}" +echo "" +echo "Build time: ${BUILD_MINS}m ${BUILD_SECS}s" +echo "" +echo "To test interactively:" +echo " docker run -it --rm -e POSTGRES_PASSWORD=test ${LOCAL_IMAGE} bash" +echo "" +echo "To test with docker-compose:" +echo " docker-compose -f docker-compose.test.yaml up -d" +echo " docker-compose -f docker-compose.test.yaml exec postgres psql -U postgres" +echo "" +echo "To push to Harbor registry:" echo " docker push ${FULL_IMAGE}" - echo "" -echo "Build completed successfully!" -echo "Local testing image: ${LOCAL_IMAGE}" -echo "Harbor registry image: ${FULL_IMAGE}" +echo "To update CNPG cluster, change imageName in cluster-shared.yaml to:" +echo " imageName: ${FULL_IMAGE}" +echo "" +log_warn "NOTE: PG18 uses /var/lib/postgresql as data dir (not /var/lib/postgresql/data)" \ No newline at end of file diff --git a/build/postgresql-postgis/docker-compose.test.yaml b/build/postgresql-postgis/docker-compose.test.yaml new file mode 100644 index 0000000..8b45c27 --- /dev/null +++ b/build/postgresql-postgis/docker-compose.test.yaml @@ -0,0 +1,36 @@ +# Docker Compose for local testing of PostgreSQL 18 + PostGIS image +# +# Usage: +# docker-compose -f docker-compose.test.yaml up -d +# docker-compose -f docker-compose.test.yaml exec postgres psql -U postgres +# docker-compose -f docker-compose.test.yaml down -v +# +# NOTE: PostgreSQL 18 changed the data directory path! +# PG 13-17: /var/lib/postgresql/data +# PG 18+: /var/lib/postgresql +# +version: '3.8' + +services: + postgres: + image: cnpg-postgis:18-3.6 + platform: linux/arm64 + container_name: postgis-test + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: testpassword + POSTGRES_DB: testdb + ports: + - "5432:5432" + volumes: + # NOTE: PG18 uses /var/lib/postgresql (not /var/lib/postgresql/data) + - postgres_data:/var/lib/postgresql + - ./init-extensions.sql:/docker-entrypoint-initdb.d/20-extensions.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + +volumes: + postgres_data: diff --git a/build/postgresql-postgis/init-extensions.sql b/build/postgresql-postgis/init-extensions.sql new file mode 100644 index 0000000..5ed249c --- /dev/null +++ b/build/postgresql-postgis/init-extensions.sql @@ -0,0 +1,21 @@ +-- Initialize PostGIS extensions for testing +-- This mirrors what CNPG does in postInitTemplateSQL + +-- Core PostGIS +CREATE EXTENSION IF NOT EXISTS postgis; + +-- Topology support +CREATE EXTENSION IF NOT EXISTS postgis_topology; + +-- Fuzzy string matching (required for tiger geocoder) +CREATE EXTENSION IF NOT EXISTS fuzzystrmatch; + +-- US Census TIGER geocoder +CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder; + +-- Verify installations +SELECT 'PostgreSQL version: ' || version(); +SELECT 'PostGIS version: ' || postgis_full_version(); + +-- List all installed extensions +SELECT extname, extversion FROM pg_extension ORDER BY extname; diff --git a/build/postgresql-postgis/initdb-postgis.sh b/build/postgresql-postgis/initdb-postgis.sh new file mode 100755 index 0000000..1bb10f0 --- /dev/null +++ b/build/postgresql-postgis/initdb-postgis.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +# Perform all actions as $POSTGRES_USER +export PGUSER="$POSTGRES_USER" + +# Create the 'template_postgis' template db +psql --dbname="$POSTGRES_DB" <<- 'EOSQL' +CREATE DATABASE template_postgis IS_TEMPLATE true; +EOSQL + +# Load PostGIS into both template_database and $POSTGRES_DB +for DB in template_postgis "$POSTGRES_DB"; do + echo "Loading PostGIS extensions into $DB" + psql --dbname="$DB" <<-'EOSQL' + CREATE EXTENSION IF NOT EXISTS postgis; + CREATE EXTENSION IF NOT EXISTS postgis_topology; + CREATE EXTENSION IF NOT EXISTS fuzzystrmatch; + CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder; +EOSQL +done diff --git a/build/postgresql-postgis/update-postgis.sh b/build/postgresql-postgis/update-postgis.sh new file mode 100755 index 0000000..3e60e39 --- /dev/null +++ b/build/postgresql-postgis/update-postgis.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +set -e + +# Perform all actions as $POSTGRES_USER +export PGUSER="$POSTGRES_USER" + +POSTGIS_VERSION="${POSTGIS_VERSION%%+*}" + +# Load PostGIS into both template_database and $POSTGRES_DB +for DB in template_postgis "$POSTGRES_DB" "${@}"; do + echo "Updating PostGIS extensions '$DB' to $POSTGIS_VERSION" + psql --dbname="$DB" -c " + -- Upgrade PostGIS (includes raster) + CREATE EXTENSION IF NOT EXISTS postgis VERSION '$POSTGIS_VERSION'; + ALTER EXTENSION postgis UPDATE TO '$POSTGIS_VERSION'; + + -- Upgrade Topology + CREATE EXTENSION IF NOT EXISTS postgis_topology VERSION '$POSTGIS_VERSION'; + ALTER EXTENSION postgis_topology UPDATE TO '$POSTGIS_VERSION'; + + -- Install Tiger dependencies in case not already installed + CREATE EXTENSION IF NOT EXISTS fuzzystrmatch; + -- Upgrade US Tiger Geocoder + CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder VERSION '$POSTGIS_VERSION'; + ALTER EXTENSION postgis_tiger_geocoder UPDATE TO '$POSTGIS_VERSION'; + " +done