incus-sentinel/incus/main.go

package incus

import (
	"encoding/json"
	"fmt"
	"log"
	"os/exec"
	"strings"
)

// Name of the snapshot used for backup
const backupSnapshot = "backup-snapshot"

type IncusDriver struct{}

func NewIncusDriver() *IncusDriver {
	return &IncusDriver{}
}

func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []string) error {
	// Create the restic backup command
	var resticCmd *exec.Cmd
	if len(tags) == 0 {
		resticCmd = exec.Command("restic", "backup", "--stdin", "--stdin-filename", filename)
	} else {
		resticCmd = exec.Command("restic", "backup", "--stdin", "--stdin-filename", filename, "--tag", strings.Join(tags, ","))
	}

	// Connect the output of incusCmd to the input of resticCmd
	pipe, err := incusCmd.StdoutPipe()
	if err != nil {
		return fmt.Errorf("failed to create pipe: %w", err)
	}
	resticCmd.Stdin = pipe

	// Start the incus export command
	if err := incusCmd.Start(); err != nil {
		return fmt.Errorf("failed to start incus export: %w", err)
	}

	// Start the restic backup command
	if err := resticCmd.Start(); err != nil {
		log.Println("DEBUG", resticCmd.String())
		return fmt.Errorf("failed to start restic backup: %w", err)
	}

	// Wait for the incus export command to finish
	if err := incusCmd.Wait(); err != nil {
		return fmt.Errorf("incus export command failed: %w", err)
	}

	// Wait for the restic backup command to finish
	if err := resticCmd.Wait(); err != nil {
		log.Println("DEBUG", resticCmd.String())
		return fmt.Errorf("restic backup command failed: %w", err)
	}

	return nil
}

func (d *IncusDriver) GetInstances(target string) ([]Instance, error) {
	// Command: incus list -f json

	var cmd *exec.Cmd
	if target == "" {
		cmd = exec.Command("incus", "list", "--format", "json", "--all-projects")
	} else {
		cmd = exec.Command("incus", "list", target+":", "--format", "json", "--all-projects")
	}

	output, err := cmd.Output()
	if err != nil {
		log.Println("DEBUG", cmd.String())
		return nil, fmt.Errorf("failed to execute incus list: %w", err)
	}

	var instances []Instance
	err = json.Unmarshal(output, &instances)
	if err != nil {
		return nil, err
	}

	return instances, nil
}

func (d *IncusDriver) GetPools(target string) ([]Pool, error) {
	// Command: incus storage list -f json
	var cmd *exec.Cmd
	if target == "" {
		cmd = exec.Command("incus", "storage", "list", "--format", "json")
	} else {
		cmd = exec.Command("incus", "storage", "list", target+":", "--format", "json")
	}

	output, err := cmd.Output()
	if err != nil {
		log.Println("DEBUG", cmd.String())
		return nil, fmt.Errorf("failed to execute incus list: %w", err)
	}

	var pools []Pool
	err = json.Unmarshal(output, &pools)
	if err != nil {
		return nil, err
	}

	return pools, nil
}

func (d *IncusDriver) GetVolumes(target string) ([]Volume, error) {
	volumes := []Volume{}
	var cmd *exec.Cmd

	pools, err := d.GetPools(target)
	if err != nil {
		return nil, err
	}

	for _, pool := range pools {
		if target == "" {
			cmd = exec.Command("incus", "storage", "volume", "list", pool.Name, "--format", "json", "--all-projects")
		} else {
			cmd = exec.Command("incus", "storage", "volume", "list", target+":"+pool.Name, "--format", "json", "--all-projects")
		}

		output, err := cmd.Output()
		if err != nil {
			log.Println("DEBUG", cmd.String())
			return nil, fmt.Errorf("failed to execute incus list: %w", err)
		}
		poolVolumes := []Volume{}
		err = json.Unmarshal(output, &poolVolumes)
		if err != nil {
			return nil, fmt.Errorf("failed to unmarshal volumes: %w", err)
		}

		for _, p := range poolVolumes {
			// Skip volumes with "/" in their name because they are snapshots
			if strings.Contains(p.Name, "/") {
				continue
			}

			// We skip everything except custom volumes
			if p.Type != "custom" {
				continue
			}

			p.Pool = pool.Name
			volumes = append(volumes, p)
		}

	}

	return volumes, nil
}

func (d *IncusDriver) Sync(project string, sourceInstance string, targetInstance string, targetHost string, targetPool string) error {
	// incus copy edge0 racker1:edge0-cold -s pool0 --mode push -p default -p net_edge0 --stateless --refresh
	// incus copy edge0 racker1:edge0-cold -s pool0 --mode push -p default -p net_edge0 --stateless

	instances, err := d.GetInstances(targetHost)
	if err != nil {
		return err
	}

	var cmd *exec.Cmd
	if len(instances) == 0 {
		cmd = exec.Command("incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false", "--project", project)
	} else {
		cmd = exec.Command("incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false", "--project", project, "--refresh")
	}
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Println("DEBUG", cmd.String())
		return fmt.Errorf("failed to execute incus copy: %w (%s)", err, string(out))
	}

	return nil
}

func (d *IncusDriver) Backup(project string, instance string, tags []string) error {
	// incus export ups - -q --compression=zstd --instance-only --optimized-storage | restic backup --stdin --stdin-filename ups.btrfs.zstd --tag instance

	// Create the incus export command
	// ? --compression=zstd parameter is not good for this because restic uses compression on its own
	incusCmd := exec.Command("incus", "export", instance, "-", "-q", "--instance-only", "--optimized-storage", "--project", project)

	// Create the restic backup command
	err := d.pipeToRestic(incusCmd, fmt.Sprintf("%s-%s.btrfs.instance", project, instance), tags)
	if err != nil {
		log.Println("DEBUG", incusCmd.String())
		return err
	}

	return nil
}

func (d *IncusDriver) SyncVolume(project string, sourcePool string, sourceVolume string, targetHost string, targetPool string, targetVolume string) error {
	// incus storage volume copy pool0/custom/node-27-apps merkur:pool0/custom/node-27-apps-old --mode push --refresh

	vols, err := d.GetVolumes(targetHost)
	if err != nil {
		return err
	}

	found := false
	for _, vol := range vols {
		if vol.Project == project && vol.Pool == targetPool && vol.Name == targetVolume {
			found = true
			break
		}
	}

	var cmd *exec.Cmd

	if found {
		cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/"+sourceVolume, targetHost+":"+targetPool+"/"+targetVolume, "--mode", "push", "--refresh", "--project", project)
	} else {
		cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/"+sourceVolume, targetHost+":"+targetPool+"/"+targetVolume, "--mode", "push", "--project", project)
	}

	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Println("DEBUG", cmd.String())
		return fmt.Errorf("failed to execute incus storage volume copy: %w (%s)", err, string(out))
	}

	// Disable sync and backup on the remote side
	cmd = exec.Command("incus", "storage", "volume", "set", targetHost+":"+targetPool, targetVolume, "user.backup=false", "user.sync=false", "--project", project)
	out, err = cmd.CombinedOutput()
	if err != nil {
		log.Println("DEBUG", cmd.String())
		return fmt.Errorf("failed to execute incus storage volume set: %w (%s)", err, string(out))
	}

	return nil
}

// Runs backup of volume's directory
func (d *IncusDriver) BackupVolumeDir(project string, pool string, volume string, tags []string) error {
	// /var/lib/incus/storage-pools/{POOL}/custom/{PROJECT}_{VOLUME}
	// /var/lib/incus/storage-pools/pool0/custom-snapshots/default_node-27-apps/backup-snapshot/

	// TODO: check if volume is block device or filesystem and return error when it is a block device

	var cmd *exec.Cmd

	// Create snapshot
	cmd = exec.Command("incus", "storage", "volume", "snapshot", "create", pool, volume, backupSnapshot, "--project", project)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Println("DEBUG", cmd.String())
		return fmt.Errorf("failed to execute incus storage volume create: %w (%s)", err, string(out))
	}

	defer func() {
		cmd = exec.Command("incus", "storage", "volume", "snapshot", "delete", pool, volume, backupSnapshot, "--project", project)
		out, err := cmd.CombinedOutput()
		if err != nil {
			log.Println("DEBUG", cmd.String())
			log.Printf("failed to delete snapshot: %s (%s)", err.Error(), string(out))
		}
	}()

	// Run restic backup
	cmd = exec.Command("restic", "backup", "--tag", strings.Join(tags, ","), "./")
	cmd.Dir = fmt.Sprintf("/var/lib/incus/storage-pools/%s/custom-snapshots/%s_%s/%s", pool, project, volume, backupSnapshot)
	out, err = cmd.CombinedOutput()
	if err != nil {
		log.Println("DEBUG", cmd.String())
		return fmt.Errorf("failed to execute restic backup: %w (%s)", err, string(out))
	}

	return nil
}

// Backup volume with Incus's native volume export feature
func (d *IncusDriver) BackupVolumeNative(project string, pool string, volume string, tags []string) error {
	// Create the incus export command
	incusCmd := exec.Command("incus", "storage", "volume", "export", pool, volume, "--optimized-storage", "--volume-only", "--project", project)

	err := d.pipeToRestic(incusCmd, fmt.Sprintf("%s-%s.btrfs.volume", pool, volume), tags)
	if err != nil {
		log.Println("DEBUG", incusCmd.String())
		return err
	}

	return nil
}