package incus import ( "encoding/json" "fmt" "log" "os/exec" "strings" ) // Name of the snapshot used for backup const backupSnapshot = "backup-snapshot" type IncusDriver struct{} func NewIncusDriver() *IncusDriver { return &IncusDriver{} } func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []string) error { // Create the restic backup command var resticCmd *exec.Cmd if len(tags) == 0 { resticCmd = exec.Command("restic", "backup", "--stdin", "--stdin-filename", filename) } else { resticCmd = exec.Command("restic", "backup", "--stdin", "--stdin-filename", filename, "--tag", strings.Join(tags, ",")) } // Connect the output of incusCmd to the input of resticCmd pipe, err := incusCmd.StdoutPipe() if err != nil { return fmt.Errorf("failed to create pipe: %w", err) } resticCmd.Stdin = pipe // Start the incus export command if err := incusCmd.Start(); err != nil { return fmt.Errorf("failed to start incus export: %w", err) } // Start the restic backup command if err := resticCmd.Start(); err != nil { return fmt.Errorf("failed to start restic backup: %w", err) } // Wait for the incus export command to finish if err := incusCmd.Wait(); err != nil { return fmt.Errorf("incus export command failed: %w", err) } // Wait for the restic backup command to finish if err := resticCmd.Wait(); err != nil { return fmt.Errorf("restic backup command failed: %w", err) } return nil } func (d *IncusDriver) GetInstances(target string) ([]Instance, error) { // Command: incus list -f json var cmd *exec.Cmd if target == "" { cmd = exec.Command("incus", "list", "--format", "json", "--all-projects") } else { cmd = exec.Command("incus", "list", target+":", "--format", "json", "--all-projects") } output, err := cmd.Output() if err != nil { return nil, fmt.Errorf("failed to execute incus list: %w", err) } var instances []Instance err = json.Unmarshal(output, &instances) if err != nil { return nil, err } return instances, nil } func (d *IncusDriver) GetPools(target string) ([]Pool, error) { // Command: incus storage list -f json var cmd *exec.Cmd if target == "" { cmd = exec.Command("incus", "storage", "list", "--format", "json") } else { cmd = exec.Command("incus", "storage", "list", target+":", "--format", "json") } output, err := cmd.Output() if err != nil { return nil, fmt.Errorf("failed to execute incus list: %w", err) } var pools []Pool err = json.Unmarshal(output, &pools) if err != nil { return nil, err } return pools, nil } func (d *IncusDriver) GetVolumes(target string) ([]Volume, error) { volumes := []Volume{} var cmd *exec.Cmd pools, err := d.GetPools(target) if err != nil { return nil, err } for _, pool := range pools { if target == "" { cmd = exec.Command("incus", "storage", "volume", "list", pool.Name, "--format", "json", "--all-projects") } else { cmd = exec.Command("incus", "storage", "volume", "list", target+":"+pool.Name, "--format", "json", "--all-projects") } output, err := cmd.Output() if err != nil { return nil, fmt.Errorf("failed to execute incus list: %w", err) } poolVolumes := []Volume{} err = json.Unmarshal(output, &poolVolumes) if err != nil { return nil, fmt.Errorf("failed to unmarshal volumes: %w", err) } for _, p := range poolVolumes { // Skip volumes with "/" in their name because they are snapshots if strings.Contains(p.Name, "/") { continue } // We skip everything except custom volumes if p.Type != "custom" { continue } p.Pool = pool.Name volumes = append(volumes, p) } } return volumes, nil } func (d *IncusDriver) Sync(project string, sourceInstance string, targetInstance string, targetHost string, targetPool string) error { // incus copy edge0 racker1:edge0-cold -s pool0 --mode push -p default -p net_edge0 --stateless --refresh // incus copy edge0 racker1:edge0-cold -s pool0 --mode push -p default -p net_edge0 --stateless instances, err := d.GetInstances(targetHost) if err != nil { return err } var cmd *exec.Cmd if len(instances) == 0 { cmd = exec.Command("incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false", "--project", project) } else { cmd = exec.Command("incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false", "--project", project, "--refresh") } out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("failed to execute incus copy: %w (%s)", err, string(out)) } return nil } func (d *IncusDriver) Backup(project string, instance string, tags []string) error { // incus export ups - -q --compression=zstd --instance-only --optimized-storage | restic backup --stdin --stdin-filename ups.btrfs.zstd --tag instance // Create the incus export command // ? --compression=zstd parameter is not good for this because restic uses compression on its own incusCmd := exec.Command("incus", "export", instance, "-", "-q", "--instance-only", "--optimized-storage", "--project", project) // Create the restic backup command err := d.pipeToRestic(incusCmd, fmt.Sprintf("%s-%s.btrfs.instance", project, instance), tags) if err != nil { return err } return nil } func (d *IncusDriver) SyncVolume(project string, sourcePool string, sourceVolume string, targetHost string, targetPool string, targetVolume string) error { // incus storage volume copy pool0/custom/node-27-apps merkur:pool0/custom/node-27-apps-old --mode push --refresh vols, err := d.GetVolumes(targetHost) if err != nil { return err } found := false for _, vol := range vols { if vol.Project == project && vol.Pool == targetPool && vol.Name == targetVolume { found = true break } } var cmd *exec.Cmd if found { cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/custom/"+sourceVolume, targetHost+":"+targetPool+"/custom/"+targetVolume, "--mode", "push", "--refresh", "--project", project) } else { cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/custom/"+sourceVolume, targetHost+":"+targetPool+"/custom/"+targetVolume, "--mode", "push", "--project", project) } out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("failed to execute incus storage volume copy: %w (%s)", err, string(out)) } return nil } // Runs backup of volume's directory func (d *IncusDriver) BackupVolumeDir(project string, pool string, volume string, tags []string) error { // /var/lib/incus/storage-pools/{POOL}/custom/{PROJECT}_{VOLUME} // /var/lib/incus/storage-pools/pool0/custom-snapshots/default_node-27-apps/backup-snapshot/ // TODO: check if volume is block device or filesystem and return error when it is a block device var cmd *exec.Cmd // Create snapshot cmd = exec.Command("incus", "storage", "volume", "snapshot", "create", pool, volume, backupSnapshot, "--project", project) out, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("failed to execute incus storage volume copy: %w (%s)", err, string(out)) } defer func() { cmd = exec.Command("incus", "storage", "volume", "snapshot", "delete", pool, volume, backupSnapshot, "--project", project) out, err := cmd.CombinedOutput() if err != nil { log.Printf("failed to delete snapshot: %s (%s)", err.Error(), string(out)) } }() // Run restic backup cmd = exec.Command("restic", "backup", "--tag", strings.Join(tags, ","), "./") cmd.Dir = fmt.Sprintf("/var/lib/incus/storage-pools/%s/custom-snapshots/%s_%s/%s", pool, project, volume, backupSnapshot) out, err = cmd.CombinedOutput() if err != nil { return fmt.Errorf("failed to execute restic backup: %w (%s)", err, string(out)) } return nil } // Backup volume with Incus's native volume export feature func (d *IncusDriver) BackupVolumeNative(project string, pool string, volume string, tags []string) error { // Create the incus export command incusCmd := exec.Command("incus", "storage", "volume", "export", pool, volume, "--optimized-storage", "--volume-only", "--project", project) err := d.pipeToRestic(incusCmd, fmt.Sprintf("%s-%s.btrfs.volume", pool, volume), tags) if err != nil { return err } return nil }