diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..3f1b037 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "cSpell.words": [ + "Restic" + ] +} \ No newline at end of file diff --git a/README.md b/README.md index 3144e7c..001d766 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,21 @@ Functions of Sentinel depends on user KV values configured on each instance. Her | user.sync-target-pool | pool0 | Target's storage pool | | user.sync-target-instance-suffix | -cold | Instance name suffix at the target side | +It can also backup and sync volumes, here is list of KV fields for them: + +| Key | Default | Purpose | +| ------------------------------ | --------- | ----------------------------------------------------------------------------- | +| user.backup | false | true/false, if true, regular backup job into Restic is performed | +| user.backup-mode | dir | dir or native, dir backs up directory of the volume, native uses incus export | +| user.sync | false | true/false, if true, regular sync job into Restic is performed | +| user.backup-notify-url | "" | Call this URL when backup is done | +| user.sync-notify-url | "" | Call this URL when sync is done | +| user.backup-schedule | 0 6 * * * | Cron-like line for backup scheduling | +| user.sync-schedule | 0 6 * * * | Cron-like line for sync scheduling | +| user.sync-target-remote | "" | Sync's target host (needs to be configured in Incus) | +| user.sync-target-pool | pool0 | Target's storage pool | +| user.sync-target-volume-suffix | -cold | Volume name suffix at the target side | + ### Examples Enabled sync and backup on existing instance: @@ -44,3 +59,5 @@ Also its binary needs to be available on the system where sentinel is running. Sentinel uses Incus's CLI interface, not its API. Currently it can work only on the same machine where Incus is running. Synced instances have sync and backup flags disabled so if the remote system runs sentinel too it won't interfere with configuration of the main location. + +Volumes can be backed up in two ways. The first one is a snapshot and backup of directory where the snapshot is located. The second way is Incus's native export where a binary blob or an archive is exported and stored in Restic repo. In this case it can be imported back with incus import feature. diff --git a/incus/main.go b/incus/main.go index e6657a2..aee6359 100644 --- a/incus/main.go +++ b/incus/main.go @@ -3,71 +3,28 @@ package incus import ( "encoding/json" "fmt" + "log" "os/exec" "strings" ) +// Name of the snapshot used for backup +const backupSnapshot = "backup-snapshot" + type IncusDriver struct{} func NewIncusDriver() *IncusDriver { return &IncusDriver{} } -func (d *IncusDriver) GetInstances(target string) ([]Instance, error) { - // Command: incus list -f json - - var cmd *exec.Cmd - if target == "" { - cmd = exec.Command("incus", "list", "--format", "json") - } else { - cmd = exec.Command("incus", "list", target+":", "--format", "json") - } - - output, err := cmd.Output() - if err != nil { - return nil, fmt.Errorf("failed to execute incus list: %w", err) - } - - var instances []Instance - err = json.Unmarshal(output, &instances) - if err != nil { - return nil, err - } - - return instances, nil -} - -func (d *IncusDriver) Sync(sourceInstance string, targetInstance string, targetHost string, targetPool string) error { - // incus copy edge0 racker1:edge0-cold -s pool0 --mode push -p default -p net_edge0 --stateless --refresh - // incus copy edge0 racker1:edge0-cold -s pool0 --mode push -p default -p net_edge0 --stateless - - instances, err := d.GetInstances(targetHost) - if err != nil { - return err - } - - var cmd *exec.Cmd - if len(instances) == 0 { - cmd = exec.Command("incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false") - } else { - cmd = exec.Command("incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false", "--refresh") - } - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to execute incus copy: %w (%s)", err, string(out)) - } - - return nil -} - -func (d *IncusDriver) Backup(instance string, tags []string) error { - // incus export ups - -q --compression=zstd --instance-only --optimized-storage | restic backup --stdin --stdin-filename ups.btrfs.zstd --tag instance - - // Create the incus export command - incusCmd := exec.Command("incus", "export", instance, "-", "-q", "--compression=zstd", "--instance-only", "--optimized-storage") - +func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []string) error { // Create the restic backup command - resticCmd := exec.Command("restic", "backup", "--host", instance, "--stdin", "--stdin-filename", fmt.Sprintf("%s.btrfs.zstd", instance), "--tag", strings.Join(tags, ",")) + var resticCmd *exec.Cmd + if len(tags) == 0 { + resticCmd = exec.Command("restic", "backup", "--stdin", "--stdin-filename", filename) + } else { + resticCmd = exec.Command("restic", "backup", "--stdin", "--stdin-filename", filename, "--tag", strings.Join(tags, ",")) + } // Connect the output of incusCmd to the input of resticCmd pipe, err := incusCmd.StdoutPipe() @@ -98,3 +55,215 @@ func (d *IncusDriver) Backup(instance string, tags []string) error { return nil } + +func (d *IncusDriver) GetInstances(target string) ([]Instance, error) { + // Command: incus list -f json + + var cmd *exec.Cmd + if target == "" { + cmd = exec.Command("incus", "list", "--format", "json", "--all-projects") + } else { + cmd = exec.Command("incus", "list", target+":", "--format", "json", "--all-projects") + } + + output, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("failed to execute incus list: %w", err) + } + + var instances []Instance + err = json.Unmarshal(output, &instances) + if err != nil { + return nil, err + } + + return instances, nil +} + +func (d *IncusDriver) GetPools(target string) ([]Pool, error) { + // Command: incus storage list -f json + var cmd *exec.Cmd + if target == "" { + cmd = exec.Command("incus", "storage", "list", "--format", "json") + } else { + cmd = exec.Command("incus", "storage", "list", target+":", "--format", "json") + } + + output, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("failed to execute incus list: %w", err) + } + + var pools []Pool + err = json.Unmarshal(output, &pools) + if err != nil { + return nil, err + } + + return pools, nil +} + +func (d *IncusDriver) GetVolumes(target string) ([]Volume, error) { + volumes := []Volume{} + var cmd *exec.Cmd + + pools, err := d.GetPools(target) + if err != nil { + return nil, err + } + + for _, pool := range pools { + if target == "" { + cmd = exec.Command("incus", "storage", "volume", "list", pool.Name, "--format", "json", "--all-projects") + } else { + cmd = exec.Command("incus", "storage", "volume", "list", target+":"+pool.Name, "--format", "json", "--all-projects") + } + + output, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("failed to execute incus list: %w", err) + } + poolVolumes := []Volume{} + err = json.Unmarshal(output, &poolVolumes) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal volumes: %w", err) + } + + for _, p := range poolVolumes { + // Skip volumes with "/" in their name because they are snapshots + if strings.Contains(p.Name, "/") { + continue + } + + // We skip everything except custom volumes + if p.Type != "custom" { + continue + } + + p.Pool = pool.Name + volumes = append(volumes, p) + } + + } + + return volumes, nil +} + +func (d *IncusDriver) Sync(project string, sourceInstance string, targetInstance string, targetHost string, targetPool string) error { + // incus copy edge0 racker1:edge0-cold -s pool0 --mode push -p default -p net_edge0 --stateless --refresh + // incus copy edge0 racker1:edge0-cold -s pool0 --mode push -p default -p net_edge0 --stateless + + instances, err := d.GetInstances(targetHost) + if err != nil { + return err + } + + var cmd *exec.Cmd + if len(instances) == 0 { + cmd = exec.Command("incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false", "--project", project) + } else { + cmd = exec.Command("incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false", "--project", project, "--refresh") + } + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to execute incus copy: %w (%s)", err, string(out)) + } + + return nil +} + +func (d *IncusDriver) Backup(project string, instance string, tags []string) error { + // incus export ups - -q --compression=zstd --instance-only --optimized-storage | restic backup --stdin --stdin-filename ups.btrfs.zstd --tag instance + + // Create the incus export command + // ? --compression=zstd parameter is not good for this because restic uses compression on its own + incusCmd := exec.Command("incus", "export", instance, "-", "-q", "--instance-only", "--optimized-storage", "--project", project) + + // Create the restic backup command + err := d.pipeToRestic(incusCmd, fmt.Sprintf("%s-%s.btrfs.instance", project, instance), tags) + if err != nil { + return err + } + + return nil +} + +func (d *IncusDriver) SyncVolume(project string, sourcePool string, sourceVolume string, targetHost string, targetPool string, targetVolume string) error { + // incus storage volume copy pool0/custom/node-27-apps merkur:pool0/custom/node-27-apps-old --mode push --refresh + + vols, err := d.GetVolumes(targetHost) + if err != nil { + return err + } + + found := false + for _, vol := range vols { + if vol.Project == project && vol.Pool == targetPool && vol.Name == targetVolume { + found = true + break + } + } + + var cmd *exec.Cmd + + if found { + cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/custom/"+sourceVolume, targetHost+":"+targetPool+"/custom/"+targetVolume, "--mode", "push", "--refresh", "--project", project) + } else { + cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/custom/"+sourceVolume, targetHost+":"+targetPool+"/custom/"+targetVolume, "--mode", "push", "--project", project) + } + + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to execute incus storage volume copy: %w (%s)", err, string(out)) + } + + return nil +} + +// Runs backup of volume's directory +func (d *IncusDriver) BackupVolumeDir(project string, pool string, volume string, tags []string) error { + // /var/lib/incus/storage-pools/{POOL}/custom/{PROJECT}_{VOLUME} + // /var/lib/incus/storage-pools/pool0/custom-snapshots/default_node-27-apps/backup-snapshot/ + + // TODO: check if volume is block device or filesystem and return error when it is a block device + + var cmd *exec.Cmd + + // Create snapshot + cmd = exec.Command("incus", "storage", "volume", "snapshot", "create", pool, volume, backupSnapshot, "--project", project) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to execute incus storage volume copy: %w (%s)", err, string(out)) + } + + defer func() { + cmd = exec.Command("incus", "storage", "volume", "snapshot", "delete", pool, volume, backupSnapshot, "--project", project) + out, err := cmd.CombinedOutput() + if err != nil { + log.Printf("failed to delete snapshot: %s (%s)", err.Error(), string(out)) + } + }() + + // Run restic backup + cmd = exec.Command("restic", "backup", "--tag", strings.Join(tags, ","), "./") + cmd.Dir = fmt.Sprintf("/var/lib/incus/storage-pools/%s/custom-snapshots/%s_%s/%s", pool, project, volume, backupSnapshot) + out, err = cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to execute restic backup: %w (%s)", err, string(out)) + } + + return nil +} + +// Backup volume with Incus's native volume export feature +func (d *IncusDriver) BackupVolumeNative(project string, pool string, volume string, tags []string) error { + // Create the incus export command + incusCmd := exec.Command("incus", "storage", "volume", "export", pool, volume, "--optimized-storage", "--volume-only", "--project", project) + + err := d.pipeToRestic(incusCmd, fmt.Sprintf("%s-%s.btrfs.volume", pool, volume), tags) + if err != nil { + return err + } + + return nil +} diff --git a/incus/types.go b/incus/types.go index ee6b0a3..a3a8add 100644 --- a/incus/types.go +++ b/incus/types.go @@ -85,3 +85,98 @@ func (i *Instance) Sentinel() InstanceSentinel { return s } + +type Pool struct { + Config map[string]string `json:"config"` + Description string `json:"description"` + Name string `json:"name"` + Driver string `json:"driver"` + UsedBy []string `json:"used_by"` + Status string `json:"status"` + Locations []string `json:"locations"` +} + +type VolumeSentinel struct { + Backup bool + BackupMode string + Sync bool + BackupSchedule string + BackupNotifyURL string + SyncNotifyURL string + SyncSchedule string + SyncTargetRemote string + SyncTargetPool string + SyncTargetVolumeSuffix string +} + +type Volume struct { + Config map[string]string `json:"config"` + Description string `json:"description"` + Name string `json:"name"` + Pool string `json:"pool"` + Type string `json:"type"` + UsedBy []string `json:"used_by"` + Location string `json:"location"` + ContentType string `json:"content_type"` + Project string `json:"project"` + CreatedAt time.Time `json:"created_at"` +} + +func (v *Volume) Sentinel() VolumeSentinel { + s := VolumeSentinel{ + Backup: false, + BackupMode: "dir", // dir or native + Sync: false, + BackupNotifyURL: "", + SyncNotifyURL: "", + BackupSchedule: "0 6 * * *", + SyncSchedule: "0 6 * * *", + SyncTargetRemote: "", + SyncTargetPool: "pool0", + SyncTargetVolumeSuffix: "-cold", + } + + if val, ok := v.Config["user.backup"]; ok { + s.Backup = val == "true" + } + + if val, ok := v.Config["user.backup-mode"]; ok { + if val == "native" || val == "dir" { + s.BackupMode = val + } + } + + if val, ok := v.Config["user.sync"]; ok { + s.Sync = val == "true" + } + + if val, ok := v.Config["user.backup-notify-url"]; ok { + s.BackupNotifyURL = val + } + + if val, ok := v.Config["user.sync-notify-url"]; ok { + s.SyncNotifyURL = val + } + + if val, ok := v.Config["user.backup-schedule"]; ok { + s.BackupSchedule = val + } + + if val, ok := v.Config["user.sync-schedule"]; ok { + s.SyncSchedule = val + } + + if val, ok := v.Config["user.sync-target-remote"]; ok { + s.SyncTargetRemote = val + } + + if val, ok := v.Config["user.sync-target-pool"]; ok { + s.SyncTargetPool = val + } + + if val, ok := v.Config["user.sync-target-instance-suffix"]; ok { + s.SyncTargetVolumeSuffix = val + } + + return s +} diff --git a/main.go b/main.go index d413b31..8acbac0 100644 --- a/main.go +++ b/main.go @@ -41,6 +41,30 @@ func main() { return nil }, }, + { + Name: "list-volumes", + Usage: "List all volumes with their sync and backup settings", + Flags: []cli.Flag{}, + Action: func(c context.Context, cmd *cli.Command) error { + i := incus.NewIncusDriver() + vols, err := i.GetVolumes("") + if err != nil { + return err + } + + for _, vol := range vols { + s := vol.Sentinel() + fmt.Printf("%s/%s/%s\n", vol.Project, vol.Pool, vol.Name) + fmt.Printf(" Backup: %t (%s, %s)\n", s.Backup, s.BackupMode, s.BackupSchedule) + fmt.Printf(" Sync: %t (%s)\n", s.Sync, s.SyncSchedule) + if s.Sync { + fmt.Printf(" Sync Target: %s (pool: %s, suffix: %s)\n", s.SyncTargetRemote, s.SyncTargetPool, s.SyncTargetVolumeSuffix) + } + } + + return nil + }, + }, { Name: "sync", Usage: "Syncs all instances where sync is enabled", @@ -56,7 +80,38 @@ func main() { s := inst.Sentinel() if s.Sync { fmt.Println(".. syncing", inst.Name) - err := i.Sync(inst.Name, fmt.Sprintf("%s%s", inst.Name, s.SyncTargetInstanceSuffix), s.SyncTargetRemote, s.SyncTargetPool) + err := i.Sync(inst.Project, inst.Name, fmt.Sprintf("%s%s", inst.Name, s.SyncTargetInstanceSuffix), s.SyncTargetRemote, s.SyncTargetPool) + if err != nil { + return err + } + } + } + + return nil + }, + }, + { + Name: "sync-volumes", + Usage: "Syncs all volumes where sync is enabled", + Flags: []cli.Flag{}, + Action: func(c context.Context, cmd *cli.Command) error { + i := incus.NewIncusDriver() + vols, err := i.GetVolumes("") + if err != nil { + return err + } + + for _, vol := range vols { + s := vol.Sentinel() + if s.Sync { + fmt.Printf(".. syncing %s/%s/%s\n", vol.Project, vol.Pool, vol.Name) + + targetPool := vol.Pool + if s.SyncTargetPool != "" { + targetPool = s.SyncTargetPool + } + + err := i.SyncVolume(vol.Project, vol.Pool, vol.Name, s.SyncTargetRemote, targetPool, fmt.Sprintf("%s%s", vol.Name, s.SyncTargetVolumeSuffix)) if err != nil { return err } @@ -81,7 +136,7 @@ func main() { s := inst.Sentinel() if s.Backup { fmt.Println(".. backing up", inst.Name) - err := i.Backup(inst.Name, []string{}) + err := i.Backup(inst.Project, inst.Name, []string{}) if err != nil { return err } @@ -91,6 +146,42 @@ func main() { return nil }, }, + { + Name: "backup-volumes", + Usage: "Backs up all volumes where backup is enabled", + Flags: []cli.Flag{}, + Action: func(c context.Context, cmd *cli.Command) error { + i := incus.NewIncusDriver() + vols, err := i.GetVolumes("") + if err != nil { + return err + } + + for _, vol := range vols { + s := vol.Sentinel() + if s.Backup { + fmt.Printf(".. backing up %s/%s/%s\n", vol.Project, vol.Pool, vol.Name) + + if s.BackupMode == "dir" { + err := i.BackupVolumeDir(vol.Project, vol.Pool, vol.Name, []string{}) + if err != nil { + return err + } + } else if s.BackupMode == "native" { + err := i.BackupVolumeNative(vol.Project, vol.Pool, vol.Name, []string{}) + if err != nil { + return err + } + } else { + return fmt.Errorf("invalid backup mode: %s", s.BackupMode) + } + + } + } + + return nil + }, + }, { Name: "run", Usage: "Runs the sentinel that syncs and backs up instances based on their configuration", diff --git a/scheduler/main.go b/scheduler/main.go index 7c298b4..6949d37 100644 --- a/scheduler/main.go +++ b/scheduler/main.go @@ -11,10 +11,15 @@ import ( const queueLength = 100 const planReasonBackup = "backup" +const planReasonBackupVolume = "backup-volume" const planReasonSync = "sync" +const planReasonSyncVolume = "sync-volume" + +var defaultTags = []string{"sentinel"} type schedulerPlan struct { Instance incus.Instance + Volume incus.Volume Reason string // backup or sync } @@ -59,40 +64,76 @@ func (s *Scheduler) do(plan schedulerPlan, done chan schedulerPlan) { // Do the actual work sen := plan.Instance.Sentinel() var err error - if plan.Reason == planReasonBackup { - start := time.Now() - err = s.driver.Backup(plan.Instance.Name, []string{"sentinel"}) + start := time.Now() + + notifyURL := "" + + switch plan.Reason { + case planReasonBackup: + err = s.driver.Backup(plan.Instance.Project, plan.Instance.Name, defaultTags) if err != nil { log.Printf("Failed to backup %s: %s", plan.Instance.Name, err.Error()) + return } log.Printf("Backup of %s took %s", plan.Instance.Name, time.Since(start).String()) - if sen.BackupNotifyURL != "" && s.notifier != nil { - err = s.notifier.Notify(sen.BackupNotifyURL) - if err != nil { - log.Printf("Failed to notify %s: %s", sen.BackupNotifyURL, err.Error()) - } - } - } - - if plan.Reason == planReasonSync { - start := time.Now() - - err = s.driver.Sync(plan.Instance.Name, plan.Instance.Name+sen.SyncTargetInstanceSuffix, sen.SyncTargetRemote, sen.SyncTargetPool) + notifyURL = sen.BackupNotifyURL + case planReasonSync: + err = s.driver.Sync(plan.Instance.Project, plan.Instance.Name, plan.Instance.Name+sen.SyncTargetInstanceSuffix, sen.SyncTargetRemote, sen.SyncTargetPool) if err != nil { log.Printf("Failed to sync %s: %s", plan.Instance.Name, err.Error()) + return } log.Printf("Sync of %s took %s", plan.Instance.Name, time.Since(start).String()) - if sen.SyncNotifyURL != "" && s.notifier != nil { - err = s.notifier.Notify(sen.SyncNotifyURL) - if err != nil { - log.Printf("Failed to notify %s: %s", sen.SyncNotifyURL, err.Error()) - } + notifyURL = sen.SyncNotifyURL + case planReasonBackupVolume: + sen := plan.Volume.Sentinel() + if sen.BackupMode == "dir" { + err = s.driver.BackupVolumeDir(plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, defaultTags) + } else if sen.BackupMode == "native" { + err = s.driver.BackupVolumeNative(plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, defaultTags) + } else { + log.Printf("Invalid backup mode for volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, sen.BackupMode) + return } + + if err != nil { + log.Printf("Failed to backup volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, err.Error()) + return + } + + log.Printf("Backup of volume %s/%s/%s took %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, time.Since(start).String()) + + notifyURL = sen.BackupNotifyURL + case planReasonSyncVolume: + sen := plan.Volume.Sentinel() + targetPool := plan.Volume.Pool + if sen.SyncTargetPool != "" { + targetPool = sen.SyncTargetPool + } + + err = s.driver.SyncVolume(plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, sen.SyncTargetRemote, targetPool, plan.Volume.Name+sen.SyncTargetVolumeSuffix) + if err != nil { + log.Printf("Failed to sync volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, err.Error()) + return + } + + log.Printf("Sync of volume %s/%s/%s took %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, time.Since(start).String()) + + notifyURL = sen.SyncNotifyURL + } + + if notifyURL != "" && s.notifier != nil { + err = s.notifier.Notify(notifyURL) + if err != nil { + log.Printf("Failed to notify %s: %s", notifyURL, err.Error()) + } + } else if notifyURL != "" && s.notifier == nil { + log.Println("Warning: No notifier configured, skipping notification") } } diff --git a/scheduler/types.go b/scheduler/types.go index 85c9d4b..9d05733 100644 --- a/scheduler/types.go +++ b/scheduler/types.go @@ -3,14 +3,14 @@ package scheduler import "gitea.ceperka.net/rosti/incus-sentinel/incus" type Driver interface { - // GetInstances retrieves a list of instances from the target GetInstances(target string) ([]incus.Instance, error) - - // Sync synchronizes instances between source and target - Sync(sourceInstance string, targetInstance string, targetHost string, targetPool string) error - - // Backup creates a backup of the specified instance with given tags - Backup(instance string, tags []string) error + GetPools(target string) ([]incus.Pool, error) + GetVolumes(target string) ([]incus.Volume, error) + Sync(project string, sourceInstance string, targetInstance string, targetHost string, targetPool string) error + Backup(project string, instance string, tags []string) error + SyncVolume(project string, sourcePool string, sourceVolume string, targetHost string, targetPool string, targetVolume string) error + BackupVolumeDir(project string, pool string, volume string, tags []string) error + BackupVolumeNative(project string, pool string, volume string, tags []string) error } type Notifier interface {