Change timeout to 60 mins
This commit is contained in:
parent
26e30f2f16
commit
517f0c848a
GPG key ID:
7262DAFE292BCE20
2 changed files with 52 additions and 18 deletions
|
|
@ -60,3 +60,7 @@ tasks:
|
|||
vars:
|
||||
DEPLOY_HOST: rosti-venus
|
||||
GOARCH: amd64
|
||||
- task: deploy
|
||||
vars:
|
||||
DEPLOY_HOST: root@stack-node-01.rosti.cz
|
||||
GOARCH: amd64
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package incus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
|
|
@ -8,11 +9,14 @@ import (
|
|||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Name of the snapshot used for backup
|
||||
const backupSnapshot = "backup-snapshot"
|
||||
|
||||
const timeout = 60 * time.Minute
|
||||
|
||||
type IncusDriver struct{}
|
||||
|
||||
func NewIncusDriver() *IncusDriver {
|
||||
|
|
@ -20,12 +24,15 @@ func NewIncusDriver() *IncusDriver {
|
|||
}
|
||||
|
||||
func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
// Create the restic backup command
|
||||
var resticCmd *exec.Cmd
|
||||
if len(tags) == 0 {
|
||||
resticCmd = exec.Command("restic", "backup", "--stdin", "--stdin-filename", filename)
|
||||
resticCmd = exec.CommandContext(ctx, "restic", "backup", "--stdin", "--stdin-filename", filename)
|
||||
} else {
|
||||
resticCmd = exec.Command("restic", "backup", "--stdin", "--stdin-filename", filename, "--tag", strings.Join(tags, ","))
|
||||
resticCmd = exec.CommandContext(ctx, "restic", "backup", "--stdin", "--stdin-filename", filename, "--tag", strings.Join(tags, ","))
|
||||
}
|
||||
|
||||
// Create a pipe to connect the two commands
|
||||
|
|
@ -72,11 +79,14 @@ func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []s
|
|||
func (d *IncusDriver) GetInstances(target string) ([]Instance, error) {
|
||||
// Command: incus list -f json
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
var cmd *exec.Cmd
|
||||
if target == "" {
|
||||
cmd = exec.Command("incus", "list", "--format", "json", "--all-projects")
|
||||
cmd = exec.CommandContext(ctx, "incus", "list", "--format", "json", "--all-projects")
|
||||
} else {
|
||||
cmd = exec.Command("incus", "list", target+":", "--format", "json", "--all-projects")
|
||||
cmd = exec.CommandContext(ctx, "incus", "list", target+":", "--format", "json", "--all-projects")
|
||||
}
|
||||
|
||||
output, err := cmd.Output()
|
||||
|
|
@ -95,12 +105,15 @@ func (d *IncusDriver) GetInstances(target string) ([]Instance, error) {
|
|||
}
|
||||
|
||||
func (d *IncusDriver) GetPools(target string) ([]Pool, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
// Command: incus storage list -f json
|
||||
var cmd *exec.Cmd
|
||||
if target == "" {
|
||||
cmd = exec.Command("incus", "storage", "list", "--format", "json")
|
||||
cmd = exec.CommandContext(ctx, "incus", "storage", "list", "--format", "json")
|
||||
} else {
|
||||
cmd = exec.Command("incus", "storage", "list", target+":", "--format", "json")
|
||||
cmd = exec.CommandContext(ctx, "incus", "storage", "list", target+":", "--format", "json")
|
||||
}
|
||||
|
||||
output, err := cmd.Output()
|
||||
|
|
@ -122,6 +135,9 @@ func (d *IncusDriver) GetVolumes(target string) ([]Volume, error) {
|
|||
volumes := []Volume{}
|
||||
var cmd *exec.Cmd
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
pools, err := d.GetPools(target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -129,9 +145,9 @@ func (d *IncusDriver) GetVolumes(target string) ([]Volume, error) {
|
|||
|
||||
for _, pool := range pools {
|
||||
if target == "" {
|
||||
cmd = exec.Command("incus", "storage", "volume", "list", pool.Name, "--format", "json", "--all-projects")
|
||||
cmd = exec.CommandContext(ctx, "incus", "storage", "volume", "list", pool.Name, "--format", "json", "--all-projects")
|
||||
} else {
|
||||
cmd = exec.Command("incus", "storage", "volume", "list", target+":"+pool.Name, "--format", "json", "--all-projects")
|
||||
cmd = exec.CommandContext(ctx, "incus", "storage", "volume", "list", target+":"+pool.Name, "--format", "json", "--all-projects")
|
||||
}
|
||||
|
||||
output, err := cmd.Output()
|
||||
|
|
@ -169,6 +185,9 @@ func (d *IncusDriver) Sync(project string, sourceInstance string, targetInstance
|
|||
// incus copy edge0 racker1:edge0-cold -s pool0 --mode push -p default -p net_edge0 --stateless --refresh
|
||||
// incus copy edge0 racker1:edge0-cold -s pool0 --mode push -p default -p net_edge0 --stateless
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
instances, err := d.GetInstances(targetHost)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -176,9 +195,9 @@ func (d *IncusDriver) Sync(project string, sourceInstance string, targetInstance
|
|||
|
||||
var cmd *exec.Cmd
|
||||
if len(instances) == 0 {
|
||||
cmd = exec.Command("incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false", "--project", project)
|
||||
cmd = exec.CommandContext(ctx, "incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false", "--project", project, "--instance-only")
|
||||
} else {
|
||||
cmd = exec.Command("incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false", "--project", project, "--refresh")
|
||||
cmd = exec.CommandContext(ctx, "incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "-c", "user.backup=false", "-c", "user.sync=false", "--project", project, "--refresh", "--instance-only")
|
||||
}
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
|
|
@ -192,9 +211,12 @@ func (d *IncusDriver) Sync(project string, sourceInstance string, targetInstance
|
|||
func (d *IncusDriver) Backup(project string, instance string, tags []string) error {
|
||||
// incus export ups - -q --compression=zstd --instance-only --optimized-storage | restic backup --stdin --stdin-filename ups.btrfs.zstd --tag instance
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
// Create the incus export command
|
||||
// ? --compression=zstd parameter is not good for this because restic uses compression on its own
|
||||
incusCmd := exec.Command("incus", "export", instance, "-", "-q", "--instance-only", "--optimized-storage", "--project", project)
|
||||
incusCmd := exec.CommandContext(ctx, "incus", "export", instance, "-", "-q", "--instance-only", "--optimized-storage", "--project", project)
|
||||
|
||||
// Create the restic backup command
|
||||
err := d.pipeToRestic(incusCmd, fmt.Sprintf("%s-%s.btrfs.instance", project, instance), tags)
|
||||
|
|
@ -223,11 +245,13 @@ func (d *IncusDriver) SyncVolume(project string, sourcePool string, sourceVolume
|
|||
}
|
||||
|
||||
var cmd *exec.Cmd
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
if found {
|
||||
cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/"+sourceVolume, targetHost+":"+targetPool+"/"+targetVolume, "--mode", "push", "--refresh", "--project", project)
|
||||
cmd = exec.CommandContext(ctx, "incus", "storage", "volume", "copy", sourcePool+"/"+sourceVolume, targetHost+":"+targetPool+"/"+targetVolume, "--mode", "push", "--refresh", "--project", project)
|
||||
} else {
|
||||
cmd = exec.Command("incus", "storage", "volume", "copy", sourcePool+"/"+sourceVolume, targetHost+":"+targetPool+"/"+targetVolume, "--mode", "push", "--project", project)
|
||||
cmd = exec.CommandContext(ctx, "incus", "storage", "volume", "copy", sourcePool+"/"+sourceVolume, targetHost+":"+targetPool+"/"+targetVolume, "--mode", "push", "--project", project)
|
||||
}
|
||||
|
||||
out, err := cmd.CombinedOutput()
|
||||
|
|
@ -237,7 +261,7 @@ func (d *IncusDriver) SyncVolume(project string, sourcePool string, sourceVolume
|
|||
}
|
||||
|
||||
// Disable sync and backup on the remote side
|
||||
cmd = exec.Command("incus", "storage", "volume", "set", targetHost+":"+targetPool, targetVolume, "user.backup=false", "user.sync=false", "--project", project)
|
||||
cmd = exec.CommandContext(ctx, "incus", "storage", "volume", "set", targetHost+":"+targetPool, targetVolume, "user.backup=false", "user.sync=false", "--project", project)
|
||||
out, err = cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
log.Println("DEBUG", cmd.String())
|
||||
|
|
@ -256,8 +280,11 @@ func (d *IncusDriver) BackupVolumeDir(project string, pool string, volume string
|
|||
|
||||
var cmd *exec.Cmd
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
// Create snapshot
|
||||
cmd = exec.Command("incus", "storage", "volume", "snapshot", "create", pool, volume, backupSnapshot, "--project", project)
|
||||
cmd = exec.CommandContext(ctx, "incus", "storage", "volume", "snapshot", "create", pool, volume, backupSnapshot, "--project", project)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
log.Println("DEBUG", cmd.String())
|
||||
|
|
@ -265,7 +292,7 @@ func (d *IncusDriver) BackupVolumeDir(project string, pool string, volume string
|
|||
}
|
||||
|
||||
defer func() {
|
||||
cmd = exec.Command("incus", "storage", "volume", "snapshot", "delete", pool, volume, backupSnapshot, "--project", project)
|
||||
cmd = exec.CommandContext(ctx, "incus", "storage", "volume", "snapshot", "delete", pool, volume, backupSnapshot, "--project", project)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
log.Println("DEBUG", cmd.String())
|
||||
|
|
@ -274,7 +301,7 @@ func (d *IncusDriver) BackupVolumeDir(project string, pool string, volume string
|
|||
}()
|
||||
|
||||
// Run restic backup
|
||||
cmd = exec.Command("restic", "backup", "--tag", strings.Join(tags, ","), "./")
|
||||
cmd = exec.CommandContext(ctx, "restic", "backup", "--tag", strings.Join(tags, ","), "./")
|
||||
cmd.Dir = fmt.Sprintf("/var/lib/incus/storage-pools/%s/custom-snapshots/%s_%s/%s", pool, project, volume, backupSnapshot)
|
||||
out, err = cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
|
|
@ -287,8 +314,11 @@ func (d *IncusDriver) BackupVolumeDir(project string, pool string, volume string
|
|||
|
||||
// Backup volume with Incus's native volume export feature
|
||||
func (d *IncusDriver) BackupVolumeNative(project string, pool string, volume string, tags []string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
|
||||
// Create the incus export command
|
||||
incusCmd := exec.Command("incus", "storage", "volume", "export", pool, volume, "--optimized-storage", "--volume-only", "--project", project)
|
||||
incusCmd := exec.CommandContext(ctx, "incus", "storage", "volume", "export", pool, volume, "--optimized-storage", "--volume-only", "--project", project)
|
||||
|
||||
err := d.pipeToRestic(incusCmd, fmt.Sprintf("%s-%s.btrfs.volume", pool, volume), tags)
|
||||
if err != nil {
|
||||
|
|
|
|||
Loading…
Reference in a new issue