From 37c191f90e828ebd9e165d2bc24dfd43d036a8b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20=C5=A0trauch?= Date: Mon, 6 Jan 2025 14:56:01 +0100 Subject: [PATCH] Fix volume backup/sync scheduling --- .vscode/settings.json | 2 +- README.md | 4 ++++ Taskfile.yml | 27 +++++++++++++++++++---- incus/main.go | 26 +++++++++++++++++++--- main.go | 24 +++++++++++--------- scheduler/main.go | 51 ++++++++++++++++++++++++++++++++++++------- 6 files changed, 108 insertions(+), 26 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 3f1b037..121731f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,4 +2,4 @@ "cSpell.words": [ "Restic" ] -} \ No newline at end of file +} diff --git a/README.md b/README.md index 001d766..48e2918 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ This project is covering backup and sync scenarios for Incus instances. * Backups instances into Restic repository * Syncs instances to another incus instance +It handles automatic backups + ## KV Functions of Sentinel depends on user KV values configured on each instance. Here is what you can configure: @@ -61,3 +63,5 @@ Sentinel uses Incus's CLI interface, not its API. Currently it can work only on Synced instances have sync and backup flags disabled so if the remote system runs sentinel too it won't interfere with configuration of the main location. Volumes can be backed up in two ways. The first one is a snapshot and backup of directory where the snapshot is located. The second way is Incus's native export where a binary blob or an archive is exported and stored in Restic repo. In this case it can be imported back with incus import feature. + +Volumes are synced including snapshots and refresh is used in case the destination volume exists. diff --git a/Taskfile.yml b/Taskfile.yml index e9dcb3b..d780d8c 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -4,11 +4,13 @@ version: '3' vars: VERSION: v0 + DEPLOY_HOST: racker + GOARCH: amd64 tasks: build-bin: cmds: - - GOARCH={{ .GOARCH }} go build -o bin/incus-sentinel.{{ .VERSION }}.linux.{{ .GOARCH }} main.go + - CGO_ENABLED=0 GOARCH={{ .GOARCH }} go build -o bin/incus-sentinel.{{ .VERSION }}.linux.{{ .GOARCH }} main.go silent: false build: cmds: @@ -19,9 +21,26 @@ tasks: - task: build-bin vars: GOARCH: amd64 + deploy: + cmds: + - scp bin/incus-sentinel.{{ .VERSION }}.linux.{{ .GOARCH }} {{ .DEPLOY_HOST }}:/usr/local/bin/incus-sentinel.tmp + - ssh {{ .DEPLOY_HOST }} mv /usr/local/bin/incus-sentinel.tmp /usr/local/bin/incus-sentinel + - ssh {{ .DEPLOY_HOST }} systemctl restart incus-sentinel deploy-racker: cmds: - task: build - - scp bin/incus-sentinel.{{ .VERSION }}.linux.arm64 racker:/usr/local/bin/incus-sentinel.tmp - - ssh racker mv /usr/local/bin/incus-sentinel.tmp /usr/local/bin/incus-sentinel - - ssh racker systemctl restart incus-sentinel + - task: deploy + vars: + DEPLOY_HOST: racker + GOARCH: arm64 + deploy-rosti: + cmds: + - task: build + - task: deploy + vars: + DEPLOY_HOST: rosti-mars + GOARCH: amd64 + - task: deploy + vars: + DEPLOY_HOST: rosti-merkur + GOARCH: amd64 diff --git a/incus/main.go b/incus/main.go index aee6359..0965c66 100644 --- a/incus/main.go +++ b/incus/main.go @@ -40,6 +40,7 @@ func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []s // Start the restic backup command if err := resticCmd.Start(); err != nil { + log.Println("DEBUG", resticCmd.String()) return fmt.Errorf("failed to start restic backup: %w", err) } @@ -50,6 +51,7 @@ func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []s // Wait for the restic backup command to finish if err := resticCmd.Wait(); err != nil { + log.Println("DEBUG", resticCmd.String()) return fmt.Errorf("restic backup command failed: %w", err) } @@ -68,6 +70,7 @@ func (d *IncusDriver) GetInstances(target string) ([]Instance, error) { output, err := cmd.Output() if err != nil { + log.Println("DEBUG", cmd.String()) return nil, fmt.Errorf("failed to execute incus list: %w", err) } @@ -91,6 +94,7 @@ func (d *IncusDriver) GetPools(target string) ([]Pool, error) { output, err := cmd.Output() if err != nil { + log.Println("DEBUG", cmd.String()) return nil, fmt.Errorf("failed to execute incus list: %w", err) } @@ -121,6 +125,7 @@ func (d *IncusDriver) GetVolumes(target string) ([]Volume, error) { output, err := cmd.Output() if err != nil { + log.Println("DEBUG", cmd.String()) return nil, fmt.Errorf("failed to execute incus list: %w", err) } poolVolumes := []Volume{} @@ -166,6 +171,7 @@ func (d *IncusDriver) Sync(project string, sourceInstance string, targetInstance } out, err := cmd.CombinedOutput() if err != nil { + log.Println("DEBUG", cmd.String()) return fmt.Errorf("failed to execute incus copy: %w (%s)", err, string(out)) } @@ -182,6 +188,7 @@ func (d *IncusDriver) Backup(project string, instance string, tags []string) err // Create the restic backup command err := d.pipeToRestic(incusCmd, fmt.Sprintf("%s-%s.btrfs.instance", project, instance), tags) if err != nil { + log.Println("DEBUG", incusCmd.String()) return err } @@ -207,16 +214,25 @@ func (d *IncusDriver) SyncVolume(project string, sourcePool string, sourceVolume var cmd *exec.Cmd if found { - cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/custom/"+sourceVolume, targetHost+":"+targetPool+"/custom/"+targetVolume, "--mode", "push", "--refresh", "--project", project) + cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/"+sourceVolume, targetHost+":"+targetPool+"/"+targetVolume, "--mode", "push", "--refresh", "--project", project) } else { - cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/custom/"+sourceVolume, targetHost+":"+targetPool+"/custom/"+targetVolume, "--mode", "push", "--project", project) + cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/"+sourceVolume, targetHost+":"+targetPool+"/"+targetVolume, "--mode", "push", "--project", project) } out, err := cmd.CombinedOutput() if err != nil { + log.Println("DEBUG", cmd.String()) return fmt.Errorf("failed to execute incus storage volume copy: %w (%s)", err, string(out)) } + // Disable sync and backup on the remote side + cmd = exec.Command("incus", "storage", "volume", "set", targetHost+":"+targetPool, targetVolume, "user.backup=false", "user.sync=false", "--project", project) + out, err = cmd.CombinedOutput() + if err != nil { + log.Println("DEBUG", cmd.String()) + return fmt.Errorf("failed to execute incus storage volume set: %w (%s)", err, string(out)) + } + return nil } @@ -233,13 +249,15 @@ func (d *IncusDriver) BackupVolumeDir(project string, pool string, volume string cmd = exec.Command("incus", "storage", "volume", "snapshot", "create", pool, volume, backupSnapshot, "--project", project) out, err := cmd.CombinedOutput() if err != nil { - return fmt.Errorf("failed to execute incus storage volume copy: %w (%s)", err, string(out)) + log.Println("DEBUG", cmd.String()) + return fmt.Errorf("failed to execute incus storage volume create: %w (%s)", err, string(out)) } defer func() { cmd = exec.Command("incus", "storage", "volume", "snapshot", "delete", pool, volume, backupSnapshot, "--project", project) out, err := cmd.CombinedOutput() if err != nil { + log.Println("DEBUG", cmd.String()) log.Printf("failed to delete snapshot: %s (%s)", err.Error(), string(out)) } }() @@ -249,6 +267,7 @@ func (d *IncusDriver) BackupVolumeDir(project string, pool string, volume string cmd.Dir = fmt.Sprintf("/var/lib/incus/storage-pools/%s/custom-snapshots/%s_%s/%s", pool, project, volume, backupSnapshot) out, err = cmd.CombinedOutput() if err != nil { + log.Println("DEBUG", cmd.String()) return fmt.Errorf("failed to execute restic backup: %w (%s)", err, string(out)) } @@ -262,6 +281,7 @@ func (d *IncusDriver) BackupVolumeNative(project string, pool string, volume str err := d.pipeToRestic(incusCmd, fmt.Sprintf("%s-%s.btrfs.volume", pool, volume), tags) if err != nil { + log.Println("DEBUG", incusCmd.String()) return err } diff --git a/main.go b/main.go index 8acbac0..91ae50b 100644 --- a/main.go +++ b/main.go @@ -30,11 +30,13 @@ func main() { for _, inst := range insts { s := inst.Sentinel() - fmt.Printf("%s\n", inst.Name) - fmt.Printf(" Backup: %t (%s)\n", s.Backup, s.BackupSchedule) - fmt.Printf(" Sync: %t (%s)\n", s.Sync, s.SyncSchedule) - if s.Sync { - fmt.Printf(" Sync Target: %s (pool: %s, suffix: %s)\n", s.SyncTargetRemote, s.SyncTargetPool, s.SyncTargetInstanceSuffix) + if s.Backup || s.Sync { + fmt.Printf("%s\n", inst.Name) + fmt.Printf(" Backup: %t (%s)\n", s.Backup, s.BackupSchedule) + fmt.Printf(" Sync: %t (%s)\n", s.Sync, s.SyncSchedule) + if s.Sync { + fmt.Printf(" Sync Target: %s (pool: %s, suffix: %s)\n", s.SyncTargetRemote, s.SyncTargetPool, s.SyncTargetInstanceSuffix) + } } } @@ -54,11 +56,13 @@ func main() { for _, vol := range vols { s := vol.Sentinel() - fmt.Printf("%s/%s/%s\n", vol.Project, vol.Pool, vol.Name) - fmt.Printf(" Backup: %t (%s, %s)\n", s.Backup, s.BackupMode, s.BackupSchedule) - fmt.Printf(" Sync: %t (%s)\n", s.Sync, s.SyncSchedule) - if s.Sync { - fmt.Printf(" Sync Target: %s (pool: %s, suffix: %s)\n", s.SyncTargetRemote, s.SyncTargetPool, s.SyncTargetVolumeSuffix) + if s.Backup || s.Sync { + fmt.Printf("%s/%s/%s\n", vol.Project, vol.Pool, vol.Name) + fmt.Printf(" Backup: %t (%s, %s)\n", s.Backup, s.BackupMode, s.BackupSchedule) + fmt.Printf(" Sync: %t (%s)\n", s.Sync, s.SyncSchedule) + if s.Sync { + fmt.Printf(" Sync Target: %s (pool: %s, suffix: %s)\n", s.SyncTargetRemote, s.SyncTargetPool, s.SyncTargetVolumeSuffix) + } } } diff --git a/scheduler/main.go b/scheduler/main.go index 6949d37..86e6c6a 100644 --- a/scheduler/main.go +++ b/scheduler/main.go @@ -221,11 +221,16 @@ func (s *Scheduler) Run() error { } } -func (s *Scheduler) footprint(is []incus.Instance) string { +func (s *Scheduler) footprint(is []incus.Instance, vols []incus.Volume) string { footprint := "" for _, inst := range is { sen := inst.Sentinel() - footprint += inst.Name + sen.BackupSchedule + sen.SyncSchedule + footprint += inst.Project + inst.Name + sen.BackupSchedule + sen.SyncSchedule + } + + for _, vol := range vols { + sen := vol.Sentinel() + footprint += vol.Project + vol.Name + vol.Pool + sen.BackupSchedule + sen.SyncSchedule } return footprint @@ -238,7 +243,12 @@ func (s *Scheduler) refresh() error { return err } - err = s.setupInstanceSchedules(instances) + vols, err := s.driver.GetVolumes("") + if err != nil { + return err + } + + err = s.setupInstanceSchedules(instances, vols) if err != nil { return err } @@ -247,8 +257,8 @@ func (s *Scheduler) refresh() error { } // Refresh cron like schedulers for all instances -func (s *Scheduler) setupInstanceSchedules(is []incus.Instance) error { - if s.scheduledFootprint == s.footprint(is) { +func (s *Scheduler) setupInstanceSchedules(is []incus.Instance, vols []incus.Volume) error { + if s.scheduledFootprint == s.footprint(is, vols) { return nil } @@ -260,10 +270,11 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance) error { s.cron = cron.New() + // Instances for _, inst := range is { sen := inst.Sentinel() if sen.Backup { - log.Println(".. adding backup schedule for", inst.Name, sen.BackupSchedule) + log.Println(".. adding backup schedule for", inst.Project, inst.Name, sen.BackupSchedule) _, err := s.cron.AddFunc(sen.BackupSchedule, func() { s.planner <- schedulerPlan{Instance: inst, Reason: planReasonBackup} }) @@ -273,7 +284,7 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance) error { } if sen.Sync { - log.Println(".. adding sync schedule for", inst.Name, sen.SyncSchedule) + log.Println(".. adding sync schedule for", inst.Project, inst.Name, sen.SyncSchedule) _, err := s.cron.AddFunc(sen.SyncSchedule, func() { s.planner <- schedulerPlan{Instance: inst, Reason: planReasonSync} }) @@ -283,7 +294,31 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance) error { } } - s.scheduledFootprint = s.footprint(is) + // Volumes + for _, vol := range vols { + sen := vol.Sentinel() + if sen.Backup { + log.Println(".. adding backup schedule for", vol.Project, vol.Pool, vol.Name, sen.BackupSchedule) + _, err := s.cron.AddFunc(sen.BackupSchedule, func() { + s.planner <- schedulerPlan{Volume: vol, Reason: planReasonBackupVolume} + }) + if err != nil { + return err + } + } + + if sen.Sync { + log.Println(".. adding sync schedule for", vol.Project, vol.Pool, vol.Name, sen.SyncSchedule) + _, err := s.cron.AddFunc(sen.SyncSchedule, func() { + s.planner <- schedulerPlan{Volume: vol, Reason: planReasonSyncVolume} + }) + if err != nil { + return err + } + } + } + + s.scheduledFootprint = s.footprint(is, vols) s.cron.Start() return nil