From 517f0c848a3d03dc792e912bc7f0f7d62fc1de4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20=C5=A0trauch?= Date: Sat, 10 May 2025 02:33:25 +0200 Subject: [PATCH] Change timeout to 60 mins --- Taskfile.yml | 4 ++++ incus/main.go | 66 +++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 52 insertions(+), 18 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index cf34064..c51ca24 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -60,3 +60,7 @@ tasks: vars: DEPLOY_HOST: rosti-venus GOARCH: amd64 + - task: deploy + vars: + DEPLOY_HOST: root@stack-node-01.rosti.cz + GOARCH: amd64 diff --git a/incus/main.go b/incus/main.go index e6bbab3..97b020c 100644 --- a/incus/main.go +++ b/incus/main.go @@ -1,6 +1,7 @@ package incus import ( + "context" "encoding/json" "fmt" "io" @@ -8,11 +9,14 @@ import ( "os" "os/exec" "strings" + "time" ) // Name of the snapshot used for backup const backupSnapshot = "backup-snapshot" +const timeout = 60 * time.Minute + type IncusDriver struct{} func NewIncusDriver() *IncusDriver { @@ -20,12 +24,15 @@ func NewIncusDriver() *IncusDriver { } func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []string) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + // Create the restic backup command var resticCmd *exec.Cmd if len(tags) == 0 { - resticCmd = exec.Command("restic", "backup", "--stdin", "--stdin-filename", filename) + resticCmd = exec.CommandContext(ctx, "restic", "backup", "--stdin", "--stdin-filename", filename) } else { - resticCmd = exec.Command("restic", "backup", "--stdin", "--stdin-filename", filename, "--tag", strings.Join(tags, ",")) + resticCmd = exec.CommandContext(ctx, "restic", "backup", "--stdin", "--stdin-filename", filename, "--tag", strings.Join(tags, ",")) } // Create a pipe to connect the two commands @@ -72,11 +79,14 @@ func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []s func (d *IncusDriver) GetInstances(target string) ([]Instance, error) { // Command: incus list -f json + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + var cmd *exec.Cmd if target == "" { - cmd = exec.Command("incus", "list", "--format", "json", "--all-projects") + cmd = exec.CommandContext(ctx, "incus", "list", "--format", "json", "--all-projects") } else { - cmd = exec.Command("incus", "list", target+":", "--format", "json", "--all-projects") + cmd = exec.CommandContext(ctx, "incus", "list", target+":", "--format", "json", "--all-projects") } output, err := cmd.Output() @@ -95,12 +105,15 @@ func (d *IncusDriver) GetInstances(target string) ([]Instance, error) { } func (d *IncusDriver) GetPools(target string) ([]Pool, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + // Command: incus storage list -f json var cmd *exec.Cmd if target == "" { - cmd = exec.Command("incus", "storage", "list", "--format", "json") + cmd = exec.CommandContext(ctx, "incus", "storage", "list", "--format", "json") } else { - cmd = exec.Command("incus", "storage", "list", target+":", "--format", "json") + cmd = exec.CommandContext(ctx, "incus", "storage", "list", target+":", "--format", "json") } output, err := cmd.Output() @@ -122,6 +135,9 @@ func (d *IncusDriver) GetVolumes(target string) ([]Volume, error) { volumes := []Volume{} var cmd *exec.Cmd + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + pools, err := d.GetPools(target) if err != nil { return nil, err @@ -129,9 +145,9 @@ func (d *IncusDriver) GetVolumes(target string) ([]Volume, error) { for _, pool := range pools { if target == "" { - cmd = exec.Command("incus", "storage", "volume", "list", pool.Name, "--format", "json", "--all-projects") + cmd = exec.CommandContext(ctx, "incus", "storage", "volume", "list", pool.Name, "--format", "json", "--all-projects") } else { - cmd = exec.Command("incus", "storage", "volume", "list", target+":"+pool.Name, "--format", "json", "--all-projects") + cmd = exec.CommandContext(ctx, "incus", "storage", "volume", "list", target+":"+pool.Name, "--format", "json", "--all-projects") } output, err := cmd.Output() @@ -169,6 +185,9 @@ func (d *IncusDriver) Sync(project string, sourceInstance string, targetInstance // incus copy edge0 racker1:edge0-cold -s pool0 --mode push -p default -p net_edge0 --stateless --refresh // incus copy edge0 racker1:edge0-cold -s pool0 --mode push -p default -p net_edge0 --stateless + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + instances, err := d.GetInstances(targetHost) if err != nil { return err @@ -176,9 +195,9 @@ func (d *IncusDriver) Sync(project string, sourceInstance string, targetInstance var cmd *exec.Cmd if len(instances) == 0 { - cmd = exec.Command("incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false", "--project", project) + cmd = exec.CommandContext(ctx, "incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false", "--project", project, "--instance-only") } else { - cmd = exec.Command("incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false", "--project", project, "--refresh") + cmd = exec.CommandContext(ctx, "incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false", "--project", project, "--refresh", "--instance-only") } out, err := cmd.CombinedOutput() if err != nil { @@ -192,9 +211,12 @@ func (d *IncusDriver) Sync(project string, sourceInstance string, targetInstance func (d *IncusDriver) Backup(project string, instance string, tags []string) error { // incus export ups - -q --compression=zstd --instance-only --optimized-storage | restic backup --stdin --stdin-filename ups.btrfs.zstd --tag instance + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + // Create the incus export command // ? --compression=zstd parameter is not good for this because restic uses compression on its own - incusCmd := exec.Command("incus", "export", instance, "-", "-q", "--instance-only", "--optimized-storage", "--project", project) + incusCmd := exec.CommandContext(ctx, "incus", "export", instance, "-", "-q", "--instance-only", "--optimized-storage", "--project", project) // Create the restic backup command err := d.pipeToRestic(incusCmd, fmt.Sprintf("%s-%s.btrfs.instance", project, instance), tags) @@ -223,11 +245,13 @@ func (d *IncusDriver) SyncVolume(project string, sourcePool string, sourceVolume } var cmd *exec.Cmd + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() if found { - cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/"+sourceVolume, targetHost+":"+targetPool+"/"+targetVolume, "--mode", "push", "--refresh", "--project", project) + cmd = exec.CommandContext(ctx, "incus", "storage", "volume", "copy", sourcePool+"/"+sourceVolume, targetHost+":"+targetPool+"/"+targetVolume, "--mode", "push", "--refresh", "--project", project) } else { - cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/"+sourceVolume, targetHost+":"+targetPool+"/"+targetVolume, "--mode", "push", "--project", project) + cmd = exec.CommandContext(ctx, "incus", "storage", "volume", "copy", sourcePool+"/"+sourceVolume, targetHost+":"+targetPool+"/"+targetVolume, "--mode", "push", "--project", project) } out, err := cmd.CombinedOutput() @@ -237,7 +261,7 @@ func (d *IncusDriver) SyncVolume(project string, sourcePool string, sourceVolume } // Disable sync and backup on the remote side - cmd = exec.Command("incus", "storage", "volume", "set", targetHost+":"+targetPool, targetVolume, "user.backup=false", "user.sync=false", "--project", project) + cmd = exec.CommandContext(ctx, "incus", "storage", "volume", "set", targetHost+":"+targetPool, targetVolume, "user.backup=false", "user.sync=false", "--project", project) out, err = cmd.CombinedOutput() if err != nil { log.Println("DEBUG", cmd.String()) @@ -256,8 +280,11 @@ func (d *IncusDriver) BackupVolumeDir(project string, pool string, volume string var cmd *exec.Cmd + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + // Create snapshot - cmd = exec.Command("incus", "storage", "volume", "snapshot", "create", pool, volume, backupSnapshot, "--project", project) + cmd = exec.CommandContext(ctx, "incus", "storage", "volume", "snapshot", "create", pool, volume, backupSnapshot, "--project", project) out, err := cmd.CombinedOutput() if err != nil { log.Println("DEBUG", cmd.String()) @@ -265,7 +292,7 @@ func (d *IncusDriver) BackupVolumeDir(project string, pool string, volume string } defer func() { - cmd = exec.Command("incus", "storage", "volume", "snapshot", "delete", pool, volume, backupSnapshot, "--project", project) + cmd = exec.CommandContext(ctx, "incus", "storage", "volume", "snapshot", "delete", pool, volume, backupSnapshot, "--project", project) out, err := cmd.CombinedOutput() if err != nil { log.Println("DEBUG", cmd.String()) @@ -274,7 +301,7 @@ func (d *IncusDriver) BackupVolumeDir(project string, pool string, volume string }() // Run restic backup - cmd = exec.Command("restic", "backup", "--tag", strings.Join(tags, ","), "./") + cmd = exec.CommandContext(ctx, "restic", "backup", "--tag", strings.Join(tags, ","), "./") cmd.Dir = fmt.Sprintf("/var/lib/incus/storage-pools/%s/custom-snapshots/%s_%s/%s", pool, project, volume, backupSnapshot) out, err = cmd.CombinedOutput() if err != nil { @@ -287,8 +314,11 @@ func (d *IncusDriver) BackupVolumeDir(project string, pool string, volume string // Backup volume with Incus's native volume export feature func (d *IncusDriver) BackupVolumeNative(project string, pool string, volume string, tags []string) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + // Create the incus export command - incusCmd := exec.Command("incus", "storage", "volume", "export", pool, volume, "--optimized-storage", "--volume-only", "--project", project) + incusCmd := exec.CommandContext(ctx, "incus", "storage", "volume", "export", pool, volume, "--optimized-storage", "--volume-only", "--project", project) err := d.pipeToRestic(incusCmd, fmt.Sprintf("%s-%s.btrfs.volume", pool, volume), tags) if err != nil {