300 lines
9.2 KiB
Go
300 lines
9.2 KiB
Go
package incus
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"os/exec"
|
|
"strings"
|
|
)
|
|
|
|
// Name of the snapshot used for backup
|
|
const backupSnapshot = "backup-snapshot"
|
|
|
|
type IncusDriver struct{}
|
|
|
|
func NewIncusDriver() *IncusDriver {
|
|
return &IncusDriver{}
|
|
}
|
|
|
|
func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []string) error {
|
|
// Create the restic backup command
|
|
var resticCmd *exec.Cmd
|
|
if len(tags) == 0 {
|
|
resticCmd = exec.Command("restic", "backup", "--stdin", "--stdin-filename", filename)
|
|
} else {
|
|
resticCmd = exec.Command("restic", "backup", "--stdin", "--stdin-filename", filename, "--tag", strings.Join(tags, ","))
|
|
}
|
|
|
|
// Create a pipe to connect the two commands
|
|
r, w := io.Pipe()
|
|
|
|
// Connect the output of incusCmd to the input of resticCmd
|
|
incusCmd.Stdout = w
|
|
incusCmd.Stderr = os.Stderr
|
|
incusCmd.Env = os.Environ()
|
|
resticCmd.Stdin = r
|
|
resticCmd.Stdout = os.Stdout
|
|
resticCmd.Stderr = os.Stderr
|
|
resticCmd.Env = os.Environ()
|
|
|
|
// Start the incus export command
|
|
if err := incusCmd.Start(); err != nil {
|
|
return fmt.Errorf("failed to start incus export: %w", err)
|
|
}
|
|
|
|
// Start the restic backup command
|
|
if err := resticCmd.Start(); err != nil {
|
|
log.Println("DEBUG", resticCmd.String())
|
|
return fmt.Errorf("failed to start restic backup: %w", err)
|
|
}
|
|
|
|
// Wait for the incus export command to finish
|
|
if err := incusCmd.Wait(); err != nil {
|
|
log.Printf("incus export command failed: %v", err)
|
|
}
|
|
|
|
if err := w.Close(); err != nil {
|
|
log.Printf("failed to close pipe: %v", err)
|
|
}
|
|
|
|
// Wait for the restic backup command to finish
|
|
if err := resticCmd.Wait(); err != nil {
|
|
log.Println("DEBUG", resticCmd.String())
|
|
return fmt.Errorf("restic backup command failed: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (d *IncusDriver) GetInstances(target string) ([]Instance, error) {
|
|
// Command: incus list -f json
|
|
|
|
var cmd *exec.Cmd
|
|
if target == "" {
|
|
cmd = exec.Command("incus", "list", "--format", "json", "--all-projects")
|
|
} else {
|
|
cmd = exec.Command("incus", "list", target+":", "--format", "json", "--all-projects")
|
|
}
|
|
|
|
output, err := cmd.Output()
|
|
if err != nil {
|
|
log.Println("DEBUG", cmd.String())
|
|
return nil, fmt.Errorf("failed to execute incus list: %w", err)
|
|
}
|
|
|
|
var instances []Instance
|
|
err = json.Unmarshal(output, &instances)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return instances, nil
|
|
}
|
|
|
|
func (d *IncusDriver) GetPools(target string) ([]Pool, error) {
|
|
// Command: incus storage list -f json
|
|
var cmd *exec.Cmd
|
|
if target == "" {
|
|
cmd = exec.Command("incus", "storage", "list", "--format", "json")
|
|
} else {
|
|
cmd = exec.Command("incus", "storage", "list", target+":", "--format", "json")
|
|
}
|
|
|
|
output, err := cmd.Output()
|
|
if err != nil {
|
|
log.Println("DEBUG", cmd.String())
|
|
return nil, fmt.Errorf("failed to execute incus list: %w", err)
|
|
}
|
|
|
|
var pools []Pool
|
|
err = json.Unmarshal(output, &pools)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return pools, nil
|
|
}
|
|
|
|
func (d *IncusDriver) GetVolumes(target string) ([]Volume, error) {
|
|
volumes := []Volume{}
|
|
var cmd *exec.Cmd
|
|
|
|
pools, err := d.GetPools(target)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
for _, pool := range pools {
|
|
if target == "" {
|
|
cmd = exec.Command("incus", "storage", "volume", "list", pool.Name, "--format", "json", "--all-projects")
|
|
} else {
|
|
cmd = exec.Command("incus", "storage", "volume", "list", target+":"+pool.Name, "--format", "json", "--all-projects")
|
|
}
|
|
|
|
output, err := cmd.Output()
|
|
if err != nil {
|
|
log.Println("DEBUG", cmd.String())
|
|
return nil, fmt.Errorf("failed to execute incus list: %w", err)
|
|
}
|
|
poolVolumes := []Volume{}
|
|
err = json.Unmarshal(output, &poolVolumes)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to unmarshal volumes: %w", err)
|
|
}
|
|
|
|
for _, p := range poolVolumes {
|
|
// Skip volumes with "/" in their name because they are snapshots
|
|
if strings.Contains(p.Name, "/") {
|
|
continue
|
|
}
|
|
|
|
// We skip everything except custom volumes
|
|
if p.Type != "custom" {
|
|
continue
|
|
}
|
|
|
|
p.Pool = pool.Name
|
|
volumes = append(volumes, p)
|
|
}
|
|
|
|
}
|
|
|
|
return volumes, nil
|
|
}
|
|
|
|
func (d *IncusDriver) Sync(project string, sourceInstance string, targetInstance string, targetHost string, targetPool string) error {
|
|
// incus copy edge0 racker1:edge0-cold -s pool0 --mode push -p default -p net_edge0 --stateless --refresh
|
|
// incus copy edge0 racker1:edge0-cold -s pool0 --mode push -p default -p net_edge0 --stateless
|
|
|
|
instances, err := d.GetInstances(targetHost)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
var cmd *exec.Cmd
|
|
if len(instances) == 0 {
|
|
cmd = exec.Command("incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false", "--project", project)
|
|
} else {
|
|
cmd = exec.Command("incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false", "--project", project, "--refresh")
|
|
}
|
|
out, err := cmd.CombinedOutput()
|
|
if err != nil {
|
|
log.Println("DEBUG", cmd.String())
|
|
return fmt.Errorf("failed to execute incus copy: %w (%s)", err, string(out))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (d *IncusDriver) Backup(project string, instance string, tags []string) error {
|
|
// incus export ups - -q --compression=zstd --instance-only --optimized-storage | restic backup --stdin --stdin-filename ups.btrfs.zstd --tag instance
|
|
|
|
// Create the incus export command
|
|
// ? --compression=zstd parameter is not good for this because restic uses compression on its own
|
|
incusCmd := exec.Command("incus", "export", instance, "-", "-q", "--instance-only", "--optimized-storage", "--project", project)
|
|
|
|
// Create the restic backup command
|
|
err := d.pipeToRestic(incusCmd, fmt.Sprintf("%s-%s.btrfs.instance", project, instance), tags)
|
|
if err != nil {
|
|
log.Println("DEBUG", incusCmd.String())
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (d *IncusDriver) SyncVolume(project string, sourcePool string, sourceVolume string, targetHost string, targetPool string, targetVolume string) error {
|
|
// incus storage volume copy pool0/custom/node-27-apps merkur:pool0/custom/node-27-apps-old --mode push --refresh
|
|
|
|
vols, err := d.GetVolumes(targetHost)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
found := false
|
|
for _, vol := range vols {
|
|
if vol.Project == project && vol.Pool == targetPool && vol.Name == targetVolume {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
|
|
var cmd *exec.Cmd
|
|
|
|
if found {
|
|
cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/"+sourceVolume, targetHost+":"+targetPool+"/"+targetVolume, "--mode", "push", "--refresh", "--project", project)
|
|
} else {
|
|
cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/"+sourceVolume, targetHost+":"+targetPool+"/"+targetVolume, "--mode", "push", "--project", project)
|
|
}
|
|
|
|
out, err := cmd.CombinedOutput()
|
|
if err != nil {
|
|
log.Println("DEBUG", cmd.String())
|
|
return fmt.Errorf("failed to execute incus storage volume copy: %w (%s)", err, string(out))
|
|
}
|
|
|
|
// Disable sync and backup on the remote side
|
|
cmd = exec.Command("incus", "storage", "volume", "set", targetHost+":"+targetPool, targetVolume, "user.backup=false", "user.sync=false", "--project", project)
|
|
out, err = cmd.CombinedOutput()
|
|
if err != nil {
|
|
log.Println("DEBUG", cmd.String())
|
|
return fmt.Errorf("failed to execute incus storage volume set: %w (%s)", err, string(out))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Runs backup of volume's directory
|
|
func (d *IncusDriver) BackupVolumeDir(project string, pool string, volume string, tags []string) error {
|
|
// /var/lib/incus/storage-pools/{POOL}/custom/{PROJECT}_{VOLUME}
|
|
// /var/lib/incus/storage-pools/pool0/custom-snapshots/default_node-27-apps/backup-snapshot/
|
|
|
|
// TODO: check if volume is block device or filesystem and return error when it is a block device
|
|
|
|
var cmd *exec.Cmd
|
|
|
|
// Create snapshot
|
|
cmd = exec.Command("incus", "storage", "volume", "snapshot", "create", pool, volume, backupSnapshot, "--project", project)
|
|
out, err := cmd.CombinedOutput()
|
|
if err != nil {
|
|
log.Println("DEBUG", cmd.String())
|
|
return fmt.Errorf("failed to execute incus storage volume create: %w (%s)", err, string(out))
|
|
}
|
|
|
|
defer func() {
|
|
cmd = exec.Command("incus", "storage", "volume", "snapshot", "delete", pool, volume, backupSnapshot, "--project", project)
|
|
out, err := cmd.CombinedOutput()
|
|
if err != nil {
|
|
log.Println("DEBUG", cmd.String())
|
|
log.Printf("failed to delete snapshot: %s (%s)", err.Error(), string(out))
|
|
}
|
|
}()
|
|
|
|
// Run restic backup
|
|
cmd = exec.Command("restic", "backup", "--tag", strings.Join(tags, ","), "./")
|
|
cmd.Dir = fmt.Sprintf("/var/lib/incus/storage-pools/%s/custom-snapshots/%s_%s/%s", pool, project, volume, backupSnapshot)
|
|
out, err = cmd.CombinedOutput()
|
|
if err != nil {
|
|
log.Println("DEBUG", cmd.String())
|
|
return fmt.Errorf("failed to execute restic backup: %w (%s)", err, string(out))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Backup volume with Incus's native volume export feature
|
|
func (d *IncusDriver) BackupVolumeNative(project string, pool string, volume string, tags []string) error {
|
|
// Create the incus export command
|
|
incusCmd := exec.Command("incus", "storage", "volume", "export", pool, volume, "--optimized-storage", "--volume-only", "--project", project)
|
|
|
|
err := d.pipeToRestic(incusCmd, fmt.Sprintf("%s-%s.btrfs.volume", pool, volume), tags)
|
|
if err != nil {
|
|
log.Println("DEBUG", incusCmd.String())
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|