Rosti/incus-sentinel
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix volume backup/sync scheduling

Browse source
This commit is contained in:
Adam Štrauch 2025-01-06 14:56:01 +01:00
parent ecff804038
commit 37c191f90e
Signed by: cx
GPG key ID: 7262DAFE292BCE20
6 changed files with 108 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.vscode/settings.json vendored
View file

@ -2,4 +2,4 @@
"cSpell.words": [
"Restic"
]
}
}

4
README.md
View file

@ -5,6 +5,8 @@ This project is covering backup and sync scenarios for Incus instances.
* Backups instances into Restic repository
* Syncs instances to another incus instance
It handles automatic backups
## KV
Functions of Sentinel depends on user KV values configured on each instance. Here is what you can configure:
@ -61,3 +63,5 @@ Sentinel uses Incus's CLI interface, not its API. Currently it can work only on
Synced instances have sync and backup flags disabled so if the remote system runs sentinel too it won't interfere with configuration of the main location.
Volumes can be backed up in two ways. The first one is a snapshot and backup of directory where the snapshot is located. The second way is Incus's native export where a binary blob or an archive is exported and stored in Restic repo. In this case it can be imported back with incus import feature.
Volumes are synced including snapshots and refresh is used in case the destination volume exists.

27
Taskfile.yml
View file

@ -4,11 +4,13 @@ version: '3'
vars:
VERSION: v0
DEPLOY_HOST: racker
GOARCH: amd64
tasks:
build-bin:
cmds:
- GOARCH={{ .GOARCH }} go build -o bin/incus-sentinel.{{ .VERSION }}.linux.{{ .GOARCH }} main.go
- CGO_ENABLED=0 GOARCH={{ .GOARCH }} go build -o bin/incus-sentinel.{{ .VERSION }}.linux.{{ .GOARCH }} main.go
silent: false
build:
cmds:
@ -19,9 +21,26 @@ tasks:
- task: build-bin
vars:
GOARCH: amd64
deploy:
cmds:
- scp bin/incus-sentinel.{{ .VERSION }}.linux.{{ .GOARCH }} {{ .DEPLOY_HOST }}:/usr/local/bin/incus-sentinel.tmp
- ssh {{ .DEPLOY_HOST }} mv /usr/local/bin/incus-sentinel.tmp /usr/local/bin/incus-sentinel
- ssh {{ .DEPLOY_HOST }} systemctl restart incus-sentinel
deploy-racker:
cmds:
- task: build
- scp bin/incus-sentinel.{{ .VERSION }}.linux.arm64 racker:/usr/local/bin/incus-sentinel.tmp
- ssh racker mv /usr/local/bin/incus-sentinel.tmp /usr/local/bin/incus-sentinel
- ssh racker systemctl restart incus-sentinel
- task: deploy
vars:
DEPLOY_HOST: racker
GOARCH: arm64
deploy-rosti:
cmds:
- task: build
- task: deploy
vars:
DEPLOY_HOST: rosti-mars
GOARCH: amd64
- task: deploy
vars:
DEPLOY_HOST: rosti-merkur
GOARCH: amd64

26
incus/main.go
View file

@ -40,6 +40,7 @@ func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []s
// Start the restic backup command
if err := resticCmd.Start(); err != nil {
log.Println("DEBUG", resticCmd.String())
return fmt.Errorf("failed to start restic backup: %w", err)
}
@ -50,6 +51,7 @@ func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []s
// Wait for the restic backup command to finish
if err := resticCmd.Wait(); err != nil {
log.Println("DEBUG", resticCmd.String())
return fmt.Errorf("restic backup command failed: %w", err)
}
@ -68,6 +70,7 @@ func (d *IncusDriver) GetInstances(target string) ([]Instance, error) {
output, err := cmd.Output()
if err != nil {
log.Println("DEBUG", cmd.String())
return nil, fmt.Errorf("failed to execute incus list: %w", err)
}
@ -91,6 +94,7 @@ func (d *IncusDriver) GetPools(target string) ([]Pool, error) {
output, err := cmd.Output()
if err != nil {
log.Println("DEBUG", cmd.String())
return nil, fmt.Errorf("failed to execute incus list: %w", err)
}
@ -121,6 +125,7 @@ func (d *IncusDriver) GetVolumes(target string) ([]Volume, error) {
output, err := cmd.Output()
if err != nil {
log.Println("DEBUG", cmd.String())
return nil, fmt.Errorf("failed to execute incus list: %w", err)
}
poolVolumes := []Volume{}
@ -166,6 +171,7 @@ func (d *IncusDriver) Sync(project string, sourceInstance string, targetInstance
}
out, err := cmd.CombinedOutput()
if err != nil {
log.Println("DEBUG", cmd.String())
return fmt.Errorf("failed to execute incus copy: %w (%s)", err, string(out))
}
@ -182,6 +188,7 @@ func (d *IncusDriver) Backup(project string, instance string, tags []string) err
// Create the restic backup command
err := d.pipeToRestic(incusCmd, fmt.Sprintf("%s-%s.btrfs.instance", project, instance), tags)
if err != nil {
log.Println("DEBUG", incusCmd.String())
return err
}
@ -207,16 +214,25 @@ func (d *IncusDriver) SyncVolume(project string, sourcePool string, sourceVolume
var cmd *exec.Cmd
if found {
cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/custom/"+sourceVolume, targetHost+":"+targetPool+"/custom/"+targetVolume, "--mode", "push", "--refresh", "--project", project)
cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/"+sourceVolume, targetHost+":"+targetPool+"/"+targetVolume, "--mode", "push", "--refresh", "--project", project)
} else {
cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/custom/"+sourceVolume, targetHost+":"+targetPool+"/custom/"+targetVolume, "--mode", "push", "--project", project)
cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/"+sourceVolume, targetHost+":"+targetPool+"/"+targetVolume, "--mode", "push", "--project", project)
}
out, err := cmd.CombinedOutput()
if err != nil {
log.Println("DEBUG", cmd.String())
return fmt.Errorf("failed to execute incus storage volume copy: %w (%s)", err, string(out))
}
// Disable sync and backup on the remote side
cmd = exec.Command("incus", "storage", "volume", "set", targetHost+":"+targetPool, targetVolume, "user.backup=false", "user.sync=false", "--project", project)
out, err = cmd.CombinedOutput()
if err != nil {
log.Println("DEBUG", cmd.String())
return fmt.Errorf("failed to execute incus storage volume set: %w (%s)", err, string(out))
}
return nil
}
@ -233,13 +249,15 @@ func (d *IncusDriver) BackupVolumeDir(project string, pool string, volume string
cmd = exec.Command("incus", "storage", "volume", "snapshot", "create", pool, volume, backupSnapshot, "--project", project)
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("failed to execute incus storage volume copy: %w (%s)", err, string(out))
log.Println("DEBUG", cmd.String())
return fmt.Errorf("failed to execute incus storage volume create: %w (%s)", err, string(out))
}
defer func() {
cmd = exec.Command("incus", "storage", "volume", "snapshot", "delete", pool, volume, backupSnapshot, "--project", project)
out, err := cmd.CombinedOutput()
if err != nil {
log.Println("DEBUG", cmd.String())
log.Printf("failed to delete snapshot: %s (%s)", err.Error(), string(out))
}
}()
@ -249,6 +267,7 @@ func (d *IncusDriver) BackupVolumeDir(project string, pool string, volume string
cmd.Dir = fmt.Sprintf("/var/lib/incus/storage-pools/%s/custom-snapshots/%s_%s/%s", pool, project, volume, backupSnapshot)
out, err = cmd.CombinedOutput()
if err != nil {
log.Println("DEBUG", cmd.String())
return fmt.Errorf("failed to execute restic backup: %w (%s)", err, string(out))
}
@ -262,6 +281,7 @@ func (d *IncusDriver) BackupVolumeNative(project string, pool string, volume str
err := d.pipeToRestic(incusCmd, fmt.Sprintf("%s-%s.btrfs.volume", pool, volume), tags)
if err != nil {
log.Println("DEBUG", incusCmd.String())
return err
}

24
main.go
View file

@ -30,11 +30,13 @@ func main() {
for _, inst := range insts {
s := inst.Sentinel()
fmt.Printf("%s\n", inst.Name)
fmt.Printf(" Backup: %t (%s)\n", s.Backup, s.BackupSchedule)
fmt.Printf(" Sync: %t (%s)\n", s.Sync, s.SyncSchedule)
if s.Sync {
fmt.Printf(" Sync Target: %s (pool: %s, suffix: %s)\n", s.SyncTargetRemote, s.SyncTargetPool, s.SyncTargetInstanceSuffix)
if s.Backup || s.Sync {
fmt.Printf("%s\n", inst.Name)
fmt.Printf(" Backup: %t (%s)\n", s.Backup, s.BackupSchedule)
fmt.Printf(" Sync: %t (%s)\n", s.Sync, s.SyncSchedule)
if s.Sync {
fmt.Printf(" Sync Target: %s (pool: %s, suffix: %s)\n", s.SyncTargetRemote, s.SyncTargetPool, s.SyncTargetInstanceSuffix)
}
}
}
@ -54,11 +56,13 @@ func main() {
for _, vol := range vols {
s := vol.Sentinel()
fmt.Printf("%s/%s/%s\n", vol.Project, vol.Pool, vol.Name)
fmt.Printf(" Backup: %t (%s, %s)\n", s.Backup, s.BackupMode, s.BackupSchedule)
fmt.Printf(" Sync: %t (%s)\n", s.Sync, s.SyncSchedule)
if s.Sync {
fmt.Printf(" Sync Target: %s (pool: %s, suffix: %s)\n", s.SyncTargetRemote, s.SyncTargetPool, s.SyncTargetVolumeSuffix)
if s.Backup || s.Sync {
fmt.Printf("%s/%s/%s\n", vol.Project, vol.Pool, vol.Name)
fmt.Printf(" Backup: %t (%s, %s)\n", s.Backup, s.BackupMode, s.BackupSchedule)
fmt.Printf(" Sync: %t (%s)\n", s.Sync, s.SyncSchedule)
if s.Sync {
fmt.Printf(" Sync Target: %s (pool: %s, suffix: %s)\n", s.SyncTargetRemote, s.SyncTargetPool, s.SyncTargetVolumeSuffix)
}
}
}

51
scheduler/main.go
View file

@ -221,11 +221,16 @@ func (s *Scheduler) Run() error {
}
}
func (s *Scheduler) footprint(is []incus.Instance) string {
func (s *Scheduler) footprint(is []incus.Instance, vols []incus.Volume) string {
footprint := ""
for _, inst := range is {
sen := inst.Sentinel()
footprint += inst.Name + sen.BackupSchedule + sen.SyncSchedule
footprint += inst.Project + inst.Name + sen.BackupSchedule + sen.SyncSchedule
}
for _, vol := range vols {
sen := vol.Sentinel()
footprint += vol.Project + vol.Name + vol.Pool + sen.BackupSchedule + sen.SyncSchedule
}
return footprint
@ -238,7 +243,12 @@ func (s *Scheduler) refresh() error {
return err
}
err = s.setupInstanceSchedules(instances)
vols, err := s.driver.GetVolumes("")
if err != nil {
return err
}
err = s.setupInstanceSchedules(instances, vols)
if err != nil {
return err
}
@ -247,8 +257,8 @@ func (s *Scheduler) refresh() error {
}
// Refresh cron like schedulers for all instances
func (s *Scheduler) setupInstanceSchedules(is []incus.Instance) error {
if s.scheduledFootprint == s.footprint(is) {
func (s *Scheduler) setupInstanceSchedules(is []incus.Instance, vols []incus.Volume) error {
if s.scheduledFootprint == s.footprint(is, vols) {
return nil
}
@ -260,10 +270,11 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance) error {
s.cron = cron.New()
// Instances
for _, inst := range is {
sen := inst.Sentinel()
if sen.Backup {
log.Println(".. adding backup schedule for", inst.Name, sen.BackupSchedule)
log.Println(".. adding backup schedule for", inst.Project, inst.Name, sen.BackupSchedule)
_, err := s.cron.AddFunc(sen.BackupSchedule, func() {
s.planner <- schedulerPlan{Instance: inst, Reason: planReasonBackup}
})
@ -273,7 +284,7 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance) error {
}
if sen.Sync {
log.Println(".. adding sync schedule for", inst.Name, sen.SyncSchedule)
log.Println(".. adding sync schedule for", inst.Project, inst.Name, sen.SyncSchedule)
_, err := s.cron.AddFunc(sen.SyncSchedule, func() {
s.planner <- schedulerPlan{Instance: inst, Reason: planReasonSync}
})
@ -283,7 +294,31 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance) error {
}
}
s.scheduledFootprint = s.footprint(is)
// Volumes
for _, vol := range vols {
sen := vol.Sentinel()
if sen.Backup {
log.Println(".. adding backup schedule for", vol.Project, vol.Pool, vol.Name, sen.BackupSchedule)
_, err := s.cron.AddFunc(sen.BackupSchedule, func() {
s.planner <- schedulerPlan{Volume: vol, Reason: planReasonBackupVolume}
})
if err != nil {
return err
}
}
if sen.Sync {
log.Println(".. adding sync schedule for", vol.Project, vol.Pool, vol.Name, sen.SyncSchedule)
_, err := s.cron.AddFunc(sen.SyncSchedule, func() {
s.planner <- schedulerPlan{Volume: vol, Reason: planReasonSyncVolume}
})
if err != nil {
return err
}
}
}
s.scheduledFootprint = s.footprint(is, vols)
s.cron.Start()
return nil