Merge branch 'prod-database-archiving' into 'database-operators'

add database archiving in prod

See merge request !9

specific-databases/overlays/jupyter-prod/postgresql.yaml

@@ -12,6 +12,8 @@ spec:
     parameters:
       max_wal_size: "2GB"
       wal_keep_size: "512MB"
+      archive_mode: "on"
+      archive_command: "cp %p /home/postgres/wal_archive/wal_files/%f > /tmp/archive.log 2>&1"s
       checkpoint_timeout: "5min"
       checkpoint_completion_target: "0.7"
   numberOfInstances: 3
@@ -21,6 +23,25 @@ spec:
     selector:
       matchLabels:
         postgres-volume: "true"
+  additionalVolumes:
+    - name: nfs-wal-archive
+      mountPath: /home/postgres/wal_archive
+      volumeSource:
+        nfs:
+          server: 10.0.2.191
+          path: /nfs/db_archives/production
+  initContainers:
+  - name: fix-permissions
+    image: busybox
+    command:
+      - sh
+      - -c
+      - >
+        stat -c '%u:%g' /home/postgres/wal_archive | grep -q '^101:103$' ||
+        chown -R 101:103 /home/postgres/wal_archive
+    volumeMounts:
+        - name: nfs-wal-archive
+          mountPath: /home/postgres/wal_archive
   users:
     cattle-monitoring-system.grafana: []
     portalgauss.portalgauss: []
@@ -74,6 +95,9 @@ spec:
       - host all all ::1/128 md5
       - local replication standby trust
       - hostssl replication standby all md5
+      - host replication postgres ::1/128 trust
+      - host replication postgres 127.0.0.1/32 trust
+      - hostnossl replication postgres 10.42.0.0/16 md5
       - hostnossl all all 10.42.0.0/16 md5
       - hostnossl all all all reject
       - hostssl all +zalandos all pam

specific-jobs/fleet.yaml

@@ -17,3 +17,9 @@ targetCustomizations:
       name: jupyter-stag
   kustomize:
     dir: overlays/jupyter-stag
+- name: jupyter-prod
+  clusterSelector:
+    matchLabels:
+      name: jupyter-prod
+  kustomize:
+    dir: overlays/jupyter-prod
\ No newline at end of file

specific-jobs/overlays/jupyter-prod/job.yaml

+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: pg-basebackup
+  namespace: database
+spec:
+  schedule: "0 2 * * 0"  # Every sunday at 2 AM
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: pg-backup
+            image: postgres:16
+            command: ["/bin/bash", "-c"]
+            args:
+              - |
+                set -e
+                TODAY=$(date +%F)
+                BACKUP_DIR="/home/postgres/wal_archive/backups"
+                ARCHIVE_DIR="/home/postgres/wal_archive/wal_files"
+
+                echo "[$(date)] Starting pg_basebackup..."
+
+                pg_basebackup -h $PGHOST -U $PGUSER -p $PGPORT -D "$BACKUP_DIR/$TODAY" -X fetch -Fp -R -v --write-recovery-conf
+
+                echo "[$(date)] Backup complete at $BACKUP_DIR/$TODAY"
+
+                # 2. Identify the starting WAL file
+                BACKUP_LABEL="$BACKUP_DIR/$TODAY/backup_label"
+                if [ ! -f "$BACKUP_LABEL" ]; then
+                  echo "backup_label not found!"
+                  exit 1
+                fi
+
+                START_WAL=$(grep "^START WAL LOCATION" "$BACKUP_LABEL" | awk '{print $4}')
+                START_SEG=$(psql -U $PGUSER  -Atc "SELECT pg_walfile_name('$START_WAL');")
+
+                echo "[$(date)] Starting WAL file for cleanup: $START_SEG"
+
+                # 3. Clean up old WAL files
+                echo "[$(date)] Running pg_archivecleanup..."
+                pg_archivecleanup -d "$ARCHIVE_DIR" "$START_SEG"
+
+                echo "[$(date)] Cleanup complete."
+
+                # 4. Delete old backups
+                find $BACKUP_DIR -maxdepth 1 -type d -mtime +14 | xargs rm -rf --
+
+            env:
+              - name: PGHOST
+                value: postgresql  
+              - name: PGPORT
+                value: "5432"
+              - name: PGUSER
+                valueFrom:
+                  secretKeyRef:
+                    name: postgres.postgresql.credentials.postgresql.acid.zalan.do
+                    key: username
+              - name: PGPASSWORD
+                valueFrom:
+                  secretKeyRef:
+                    name: postgres.postgresql.credentials.postgresql.acid.zalan.do
+                    key: password             
+            volumeMounts:
+              - name: wal-archive
+                mountPath: /home/postgres/wal_archive
+          restartPolicy: OnFailure
+          volumes:
+            - name: wal-archive
+              nfs:
+                server: 10.0.2.191
+                path: /nfs/db_archives/production
+
+
+

specific-jobs/overlays/jupyter-prod/kustomization.yaml

+resources:
+- job.yaml
\ No newline at end of file