diff --git a/Dockerfile b/Dockerfile index 2b6f261..07ebd53 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,10 @@ FROM alpine:3.22 -RUN apk add --no-cache restic docker-cli docker-cli-compose curl +RUN apk add --no-cache restic docker-cli docker-cli-compose curl zstd bash COPY backup.sh /backup.sh +COPY backup_local.sh /backup_local.sh +COPY backup_restic.sh /backup_restic.sh RUN chmod +x /backup.sh ENTRYPOINT [ "/backup.sh" ] diff --git a/README.md b/README.md index 23e7403..3c135b7 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,55 @@ # Database Backup Container -A lightweight Alpine-based Docker container for backing up MariaDB and PostgreSQL databases using Restic. +A lightweight Alpine-based Docker container for backing up MariaDB and PostgreSQL databases with support for both local snapshots and Restic backups. ## Overview -This container automatically detects the database type (MariaDB or PostgreSQL) in a target container and creates backups using Restic. It supports backing up databases from other Docker containers by executing dump commands inside them. +This container automatically detects the database type (MariaDB or PostgreSQL) in a target container and creates backups using either local file storage with compression or Restic repositories. It supports backing up databases from other Docker containers by executing dump commands inside them. ## Features - **Multi-database support**: Automatically detects and backs up MariaDB or PostgreSQL databases +- **Dual backup methods**: Choose between local compressed files or Restic repositories +- **Local snapshots**: Create compressed (zstd) local backup files with timestamps - **Restic integration**: Uses Restic for efficient, encrypted, and deduplicated backups - **Docker-in-Docker**: Can access and backup databases from other containers +- **Notification support**: Optional webhook notifications when backups complete - **Lightweight**: Based on Alpine Linux for minimal footprint ## Prerequisites - Docker with socket access (`/var/run/docker.sock`) - Target container with either MariaDB or PostgreSQL client tools -- Restic repository (local, S3, B2, etc.) +- For Restic backups: Restic repository (local, S3, B2, etc.) +- For local backups: Mounted volume for backup storage ## Environment Variables -### Required +### Required (All Methods) | Variable | Description | |----------|-------------| | `CONTAINER` | Name of the Docker container where the database is running | + +### For Restic Backups + +| Variable | Description | +|----------|-------------| | `RESTIC_PASSWORD` | Password for the Restic repository | | `RESTIC_REPOSITORY` | Restic repository URL (e.g., `s3:s3.amazonaws.com/bucket`, `/data/backups`) | +### For Local Backups + +| Variable | Description | +|----------|-------------| +| `TARGET_DIR` | Directory where backup files will be stored | + +### Optional + +| Variable | Description | +|----------|-------------| +| `NOTIFY_URL` | Optional webhook URL to call when backup completes | + ### Database-specific #### For MariaDB containers: @@ -45,7 +66,14 @@ This container automatically detects the database type (MariaDB or PostgreSQL) i ## Usage -### Basic Usage +### Backup Methods + +The container supports two backup methods: + +1. **`restic`** - Backup to Restic repositories (cloud storage, remote servers) +2. **`local`** - Create compressed local backup files + +### Restic Backups ```bash docker run --rm \ @@ -54,11 +82,25 @@ docker run --rm \ -e RESTIC_PASSWORD=my-secret-password \ -e RESTIC_REPOSITORY=s3:s3.amazonaws.com/my-backup-bucket \ -e MARIADB_ROOT_PASSWORD=db-password \ - gitea.ceperka.net/rosti/db-backup:latest + gitea.ceperka.net/rosti/db-backup:latest restic +``` + +### Local Backups + +```bash +docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /host/backup/path:/backups \ + -e CONTAINER=my-mariadb-container \ + -e TARGET_DIR=/backups \ + -e MARIADB_ROOT_PASSWORD=db-password \ + gitea.ceperka.net/rosti/db-backup:latest local ``` ### With Docker Compose +#### Restic Backup Setup + ```yaml version: '3.8' @@ -71,18 +113,51 @@ services: volumes: - db_data:/var/lib/mysql - backup: + backup-restic: image: gitea.ceperka.net/rosti/db-backup:latest depends_on: - database environment: CONTAINER: database RESTIC_PASSWORD: my-backup-password - RESTIC_REPOSITORY: /backups + RESTIC_REPOSITORY: s3:s3.amazonaws.com/my-backup-bucket MARIADB_ROOT_PASSWORD: secretpassword + NOTIFY_URL: https://hc-ping.com/your-healthcheck-uuid + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: ["restic"] + +volumes: + db_data: +``` + +#### Local Backup Setup + +```yaml +version: '3.8' + +services: + database: + image: mariadb:latest + environment: + MARIADB_ROOT_PASSWORD: secretpassword + MARIADB_DATABASE: myapp + volumes: + - db_data:/var/lib/mysql + + backup-local: + image: gitea.ceperka.net/rosti/db-backup:latest + depends_on: + - database + environment: + CONTAINER: database + TARGET_DIR: /backups + MARIADB_ROOT_PASSWORD: secretpassword + NOTIFY_URL: https://hc-ping.com/your-healthcheck-uuid volumes: - /var/run/docker.sock:/var/run/docker.sock - ./backups:/backups + command: ["local"] volumes: db_data: @@ -92,18 +167,27 @@ volumes: To run backups on a schedule, you can use cron or a container orchestrator: +#### Restic Backups ```bash -# Add to crontab for daily backups at 2 AM -0 2 * * * docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -e CONTAINER=my-db -e RESTIC_PASSWORD=pass -e RESTIC_REPOSITORY=/backups -e MARIADB_ROOT_PASSWORD=dbpass gitea.ceperka.net/rosti/db-backup:latest +# Add to crontab for daily Restic backups at 2 AM +0 2 * * * docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -e CONTAINER=my-db -e RESTIC_PASSWORD=pass -e RESTIC_REPOSITORY=s3:s3.amazonaws.com/bucket -e MARIADB_ROOT_PASSWORD=dbpass gitea.ceperka.net/rosti/db-backup:latest restic +``` + +#### Local Backups +```bash +# Add to crontab for daily local backups at 3 AM +0 3 * * * docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v /host/backups:/backups -e CONTAINER=my-db -e TARGET_DIR=/backups -e MARIADB_ROOT_PASSWORD=dbpass gitea.ceperka.net/rosti/db-backup:latest local ``` ### Kubernetes CronJob +#### Restic Backup CronJob + ```yaml apiVersion: batch/v1 kind: CronJob metadata: - name: database-backup + name: database-backup-restic spec: schedule: "0 2 * * *" # Daily at 2 AM jobTemplate: @@ -113,6 +197,7 @@ spec: containers: - name: backup image: gitea.ceperka.net/rosti/db-backup:latest + args: ["restic"] env: - name: CONTAINER value: "my-database-pod" @@ -138,12 +223,87 @@ spec: restartPolicy: OnFailure ``` +#### Local Backup CronJob + +```yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: database-backup-local +spec: + schedule: "0 3 * * *" # Daily at 3 AM + jobTemplate: + spec: + template: + spec: + containers: + - name: backup + image: gitea.ceperka.net/rosti/db-backup:latest + args: ["local"] + env: + - name: CONTAINER + value: "my-database-pod" + - name: TARGET_DIR + value: "/backups" + - name: MARIADB_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: db-secrets + key: root-password + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + - name: backup-storage + mountPath: /backups + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock + - name: backup-storage + persistentVolumeClaim: + claimName: backup-pvc + restartPolicy: OnFailure +``` + ## Backup File Naming +### Restic Backups Backups are stored with the following naming convention: - MariaDB: `mariadb_[CONTAINER]_[DB_NAME].sql` - PostgreSQL: `pgsql_[CONTAINER]_[DB_NAME].sql` +### Local Backups +Local backup files include timestamps and are compressed: +- MariaDB: `YYYYMMDD_HHMMSS_mariadb_[CONTAINER]_[DB_NAME].sql.zst` +- PostgreSQL: `YYYYMMDD_HHMMSS_pgsql_[CONTAINER]_[DB_NAME].sql.zst` + +Local backups use zstd compression for efficient storage and include atomic file operations (temporary files are renamed when complete). + +## MariaDB Backup Features + +The container includes comprehensive MariaDB backup options: + +- **`--add-drop-trigger`** - Add DROP TRIGGER statements +- **`--add-drop-table`** - Add DROP TABLE statements +- **`--add-drop-database`** - Add DROP DATABASE statements +- **`--hex-blob`** - Use hexadecimal notation for binary data +- **`--compress`** - Compress data in backup +- **`--events`** - Include events in backup +- **`--routines`** - Include stored procedures and functions +- **`--single-transaction`** - Consistent backup for InnoDB tables +- **`--triggers`** - Include triggers in backup + +## Notification Support + +Both backup methods support optional webhook notifications: + +```bash +# Set NOTIFY_URL to receive notifications when backups complete +-e NOTIFY_URL=https://hc-ping.com/your-healthcheck-uuid +``` + +The container will make a GET request to the URL after successful backup completion. + ## Supported Restic Repositories This container supports all Restic repository types: @@ -191,14 +351,31 @@ docker build -t gitea.ceperka.net/rosti/db-backup:dev . - Verify database credentials are correct - Ensure environment variables are properly set +4. **"TARGET_DIR does not exist"** (Local backups) + - Ensure the target directory is mounted as a volume + - Check directory permissions + +5. **"Unknown backup method"** + - Ensure you specify either `local` or `restic` as the command argument + ### Debug Mode To debug issues, you can run the container interactively: +#### Restic Debug ```bash docker run -it --rm \ -v /var/run/docker.sock:/var/run/docker.sock \ - --entrypoint /bin/sh \ + --entrypoint /bin/bash \ + gitea.ceperka.net/rosti/db-backup:latest +``` + +#### Local Debug +```bash +docker run -it --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /host/backup/path:/backups \ + --entrypoint /bin/bash \ gitea.ceperka.net/rosti/db-backup:latest ``` @@ -209,6 +386,3 @@ docker run -it --rm \ - Regularly rotate Restic repository passwords - Consider using encrypted storage for backup repositories -## License - -This project is licensed under the MIT License. diff --git a/Taskfile.yml b/Taskfile.yml index 0bd3bc1..c50331e 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -19,3 +19,6 @@ tasks: testdb: cmds: - docker run -d --name testmariadb --env MARIADB_USER=maria --env MARIADB_PASSWORD=maria --env MARIADB_DATABASE=maria --env MARIADB_ROOT_PASSWORD=maria mariadb:latest + test: + cmds: + - docker run --name testmariadb-snapshot --rm -e CONTAINER=testmariadb -e TARGET_DIR=/backup -v ./tmp:/backup {{ .IMAGE }}:{{ .TAG }} local diff --git a/backup.sh b/backup.sh index daeebee..5eacf7b 100644 --- a/backup.sh +++ b/backup.sh @@ -1,61 +1,9 @@ #!/bin/bash -# Environment variables -# -# CONTAINER - name of the container where the database is running -# RESTIC_PASSWORD -# RESTIC_REPOSITORY -# NOTIFY_URL - optional, URL to send notification to - -if [ -z "$CONTAINER" ]; then - echo "CONTAINER environment variable is not set." - exit 1 -fi - -if [ -z "$RESTIC_PASSWORD" ]; then - echo "RESTIC_PASSWORD environment variable is not set." - exit 1 -fi - -if [ -z "$RESTIC_REPOSITORY" ]; then - echo "RESTIC_REPOSITORY environment variable is not set." - exit 1 -fi - -if [ ! `docker info` ]; then - echo "Docker is not available." - exit 1 -fi - -echo "Starting backup for container: $CONTAINER" -if [ `docker exec -i $CONTAINER test -e /usr/bin/mariadb-dump && echo "yes" || echo "no"` = "yes" ]; then - DBTYPE="mariadb" - docker exec -i $CONTAINER mariadb-dump \ - --user=root \ - --password=$MARIADB_ROOT_PASSWORD \ - --add-drop-trigger \ - --add-drop-table \ - --add-drop-database \ - --hex-blob \ - --compress \ - --events \ - --routines \ - --single-transaction \ - --triggers | restic backup --stdin --stdin-filename=$DBTYPE_$CONTAINER_$DB_NAME.sql -elif [ `docker exec -i $CONTAINER test -e /usr/bin/pg_dump && echo "yes" || echo "no"` = "yes" ]; then - DBTYPE="pgsql" - docker exec -i $CONTAINER pg_dump --username=$DB_USER --password=$PGPASSWORD $DB_NAME | restic backup --stdin --stdin-filename=$DBTYPE_$CONTAINER_$DB_NAME.sql +if [ "$1" = "local" ]; then + source /backup_local.sh +elif [ "$1" = "restic" ]; then + source /backup_restic.sh else - echo "Unsupported database type or database client not found in the container." - exit 1 -fi - -echo "Backup completed successfully." - -echo "Running restic forget and prune..." - -restic forget --prune --keep-daily 7 --keep-weekly 4 - -if [ -n "$NOTIFY_URL" ]; then - curl $NOTIFY_URL + echo "Unknown backup method. Use 'local' or 'restic'." fi diff --git a/backup_local.sh b/backup_local.sh new file mode 100644 index 0000000..eaca647 --- /dev/null +++ b/backup_local.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# Environment variables +# +# CONTAINER - name of the container where the database is running +# NOTIFY_URL - optional, URL to send notification to +# TARGET_DIR - directory where to store the backup + +if [ -z "$CONTAINER" ]; then + echo "CONTAINER environment variable is not set." + exit 1 +fi + +if [ -z "$TARGET_DIR" ]; then + echo "TARGET_DIR environment variable is not set." + exit 1 +fi + +if [ ! `docker info` ]; then + echo "Docker is not available." + exit 1 +fi + +if [ ! -d "$TARGET_DIR" ]; then + echo "TARGET_DIR does not exist. Creating it..." + exit 2 +fi + + +echo "Starting backup for container: $CONTAINER" +if [ `docker exec -i $CONTAINER test -e /usr/bin/mariadb-dump && echo "yes" || echo "no"` = "yes" ]; then + DBTYPE="mariadb" + FILENAME=`date +"%Y%m%d_%H%M%S"`_$DBTYPE_$CONTAINER_$DB_NAME.sql.zst.tmp + docker exec -i $CONTAINER mariadb-dump \ + --user=root \ + --password=$MARIADB_ROOT_PASSWORD \ + --add-drop-trigger \ + --add-drop-table \ + --add-drop-database \ + --hex-blob \ + --compress \ + --events \ + --routines \ + --single-transaction \ + --triggers | zstd -1 > $TARGET_DIR/ + mv $TARGET_DIR/$DBTYPE_$CONTAINER_$DB_NAME.sql.zst.tmp $TARGET_DIR/$DBTYPE_$CONTAINER_$DB_NAME.sql.zst +elif [ `docker exec -i $CONTAINER test -e /usr/bin/pg_dump && echo "yes" || echo "no"` = "yes" ]; then + DBTYPE="pgsql" + FILENAME=`date +"%Y%m%d_%H%M%S"`_$DBTYPE_$CONTAINER_$DB_NAME.sql.zst + docker exec -i $CONTAINER pg_dump --username=$DB_USER --password=$PGPASSWORD $DB_NAME | zstd -1 > $TARGET_DIR/$FILENAME.tmp + mv $TARGET_DIR/$FILENAME.tmp $TARGET_DIR/$FILENAME +else + echo "Unsupported database type or database client not found in the container." + exit 1 +fi + +echo "Backup completed successfully." + +if [ -n "$NOTIFY_URL" ]; then + curl $NOTIFY_URL +fi diff --git a/backup_restic.sh b/backup_restic.sh new file mode 100644 index 0000000..daeebee --- /dev/null +++ b/backup_restic.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# Environment variables +# +# CONTAINER - name of the container where the database is running +# RESTIC_PASSWORD +# RESTIC_REPOSITORY +# NOTIFY_URL - optional, URL to send notification to + +if [ -z "$CONTAINER" ]; then + echo "CONTAINER environment variable is not set." + exit 1 +fi + +if [ -z "$RESTIC_PASSWORD" ]; then + echo "RESTIC_PASSWORD environment variable is not set." + exit 1 +fi + +if [ -z "$RESTIC_REPOSITORY" ]; then + echo "RESTIC_REPOSITORY environment variable is not set." + exit 1 +fi + +if [ ! `docker info` ]; then + echo "Docker is not available." + exit 1 +fi + +echo "Starting backup for container: $CONTAINER" +if [ `docker exec -i $CONTAINER test -e /usr/bin/mariadb-dump && echo "yes" || echo "no"` = "yes" ]; then + DBTYPE="mariadb" + docker exec -i $CONTAINER mariadb-dump \ + --user=root \ + --password=$MARIADB_ROOT_PASSWORD \ + --add-drop-trigger \ + --add-drop-table \ + --add-drop-database \ + --hex-blob \ + --compress \ + --events \ + --routines \ + --single-transaction \ + --triggers | restic backup --stdin --stdin-filename=$DBTYPE_$CONTAINER_$DB_NAME.sql +elif [ `docker exec -i $CONTAINER test -e /usr/bin/pg_dump && echo "yes" || echo "no"` = "yes" ]; then + DBTYPE="pgsql" + docker exec -i $CONTAINER pg_dump --username=$DB_USER --password=$PGPASSWORD $DB_NAME | restic backup --stdin --stdin-filename=$DBTYPE_$CONTAINER_$DB_NAME.sql +else + echo "Unsupported database type or database client not found in the container." + exit 1 +fi + +echo "Backup completed successfully." + +echo "Running restic forget and prune..." + +restic forget --prune --keep-daily 7 --keep-weekly 4 + +if [ -n "$NOTIFY_URL" ]; then + curl $NOTIFY_URL +fi