From da09b8702034ef073b0904e2bb332d7e409be3b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20=C5=A0trauch?= Date: Fri, 10 Jan 2025 22:05:29 +0100 Subject: [PATCH] Better login, fix of backup scenarios where pipe is used when running under systemd --- Taskfile.yml | 10 +++++++++- http_notifier/main.go | 7 ++++++- incus/main.go | 23 +++++++++++++++++------ scheduler/main.go | 42 ++++++++++++++++++++++++++++++++---------- 4 files changed, 64 insertions(+), 18 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index d780d8c..f57334d 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -26,13 +26,21 @@ tasks: - scp bin/incus-sentinel.{{ .VERSION }}.linux.{{ .GOARCH }} {{ .DEPLOY_HOST }}:/usr/local/bin/incus-sentinel.tmp - ssh {{ .DEPLOY_HOST }} mv /usr/local/bin/incus-sentinel.tmp /usr/local/bin/incus-sentinel - ssh {{ .DEPLOY_HOST }} systemctl restart incus-sentinel - deploy-racker: + deploy-home: cmds: - task: build - task: deploy vars: DEPLOY_HOST: racker GOARCH: arm64 + - task: deploy + vars: + DEPLOY_HOST: racker1 + GOARCH: arm64 + - task: deploy + vars: + DEPLOY_HOST: deimos + GOARCH: amd64 deploy-rosti: cmds: - task: build diff --git a/http_notifier/main.go b/http_notifier/main.go index 3977028..de541cd 100644 --- a/http_notifier/main.go +++ b/http_notifier/main.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "slices" + "time" ) type HTTPNotifier struct{} @@ -13,7 +14,11 @@ func NewNotifier() *HTTPNotifier { } func (n *HTTPNotifier) Notify(url string) error { - resp, err := http.Get(url) + client := &http.Client{ + Timeout: 10 * time.Second, // Set timeout duration + } + + resp, err := client.Get(url) if err != nil { return fmt.Errorf("Failed to call URL %s: %s", url, err.Error()) } diff --git a/incus/main.go b/incus/main.go index 0965c66..e6bbab3 100644 --- a/incus/main.go +++ b/incus/main.go @@ -3,7 +3,9 @@ package incus import ( "encoding/json" "fmt" + "io" "log" + "os" "os/exec" "strings" ) @@ -26,12 +28,17 @@ func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []s resticCmd = exec.Command("restic", "backup", "--stdin", "--stdin-filename", filename, "--tag", strings.Join(tags, ",")) } + // Create a pipe to connect the two commands + r, w := io.Pipe() + // Connect the output of incusCmd to the input of resticCmd - pipe, err := incusCmd.StdoutPipe() - if err != nil { - return fmt.Errorf("failed to create pipe: %w", err) - } - resticCmd.Stdin = pipe + incusCmd.Stdout = w + incusCmd.Stderr = os.Stderr + incusCmd.Env = os.Environ() + resticCmd.Stdin = r + resticCmd.Stdout = os.Stdout + resticCmd.Stderr = os.Stderr + resticCmd.Env = os.Environ() // Start the incus export command if err := incusCmd.Start(); err != nil { @@ -46,7 +53,11 @@ func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []s // Wait for the incus export command to finish if err := incusCmd.Wait(); err != nil { - return fmt.Errorf("incus export command failed: %w", err) + log.Printf("incus export command failed: %v", err) + } + + if err := w.Close(); err != nil { + log.Printf("failed to close pipe: %v", err) } // Wait for the restic backup command to finish diff --git a/scheduler/main.go b/scheduler/main.go index 86e6c6a..c06c27a 100644 --- a/scheduler/main.go +++ b/scheduler/main.go @@ -71,45 +71,53 @@ func (s *Scheduler) do(plan schedulerPlan, done chan schedulerPlan) { switch plan.Reason { case planReasonBackup: + log.Printf(".. backup of %s/%s instance started", plan.Instance.Project, plan.Instance.Name) + err = s.driver.Backup(plan.Instance.Project, plan.Instance.Name, defaultTags) if err != nil { - log.Printf("Failed to backup %s: %s", plan.Instance.Name, err.Error()) + log.Printf(".. failed to backup %s: %s", plan.Instance.Name, err.Error()) return } - log.Printf("Backup of %s took %s", plan.Instance.Name, time.Since(start).String()) + log.Printf(".. backup of %s took %s", plan.Instance.Name, time.Since(start).String()) notifyURL = sen.BackupNotifyURL case planReasonSync: + log.Printf(".. syncing of %s/%s instance started", plan.Instance.Project, plan.Instance.Name) + err = s.driver.Sync(plan.Instance.Project, plan.Instance.Name, plan.Instance.Name+sen.SyncTargetInstanceSuffix, sen.SyncTargetRemote, sen.SyncTargetPool) if err != nil { - log.Printf("Failed to sync %s: %s", plan.Instance.Name, err.Error()) + log.Printf(".. failed to sync %s: %s", plan.Instance.Name, err.Error()) return } - log.Printf("Sync of %s took %s", plan.Instance.Name, time.Since(start).String()) + log.Printf(".. sync of %s took %s", plan.Instance.Name, time.Since(start).String()) notifyURL = sen.SyncNotifyURL case planReasonBackupVolume: + log.Printf(".. backup of %s/%s/%s volume started", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name) + sen := plan.Volume.Sentinel() if sen.BackupMode == "dir" { err = s.driver.BackupVolumeDir(plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, defaultTags) } else if sen.BackupMode == "native" { err = s.driver.BackupVolumeNative(plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, defaultTags) } else { - log.Printf("Invalid backup mode for volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, sen.BackupMode) + log.Printf(".. invalid backup mode for volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, sen.BackupMode) return } if err != nil { - log.Printf("Failed to backup volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, err.Error()) + log.Printf(".. failed to backup volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, err.Error()) return } - log.Printf("Backup of volume %s/%s/%s took %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, time.Since(start).String()) + log.Printf(".. backup of volume %s/%s/%s took %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, time.Since(start).String()) notifyURL = sen.BackupNotifyURL case planReasonSyncVolume: + log.Printf(".. syncing of %s/%s/%s volume started", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name) + sen := plan.Volume.Sentinel() targetPool := plan.Volume.Pool if sen.SyncTargetPool != "" { @@ -118,11 +126,11 @@ func (s *Scheduler) do(plan schedulerPlan, done chan schedulerPlan) { err = s.driver.SyncVolume(plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, sen.SyncTargetRemote, targetPool, plan.Volume.Name+sen.SyncTargetVolumeSuffix) if err != nil { - log.Printf("Failed to sync volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, err.Error()) + log.Printf(".. failed to sync volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, err.Error()) return } - log.Printf("Sync of volume %s/%s/%s took %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, time.Since(start).String()) + log.Printf(".. sync of volume %s/%s/%s took %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, time.Since(start).String()) notifyURL = sen.SyncNotifyURL } @@ -130,7 +138,7 @@ func (s *Scheduler) do(plan schedulerPlan, done chan schedulerPlan) { if notifyURL != "" && s.notifier != nil { err = s.notifier.Notify(notifyURL) if err != nil { - log.Printf("Failed to notify %s: %s", notifyURL, err.Error()) + log.Printf(".. failed to notify %s: %s", notifyURL, err.Error()) } } else if notifyURL != "" && s.notifier == nil { log.Println("Warning: No notifier configured, skipping notification") @@ -149,6 +157,8 @@ func (s *Scheduler) runMainLoop() error { ticker := time.NewTicker(2 * time.Second) defer ticker.Stop() + lastQueueLog := 0 + for { select { // New instance to plan @@ -186,6 +196,14 @@ func (s *Scheduler) runMainLoop() error { inProgress = true go s.do(<-queue, done) } + + if len(queue) > 0 && lastQueueLog != len(queue) { + log.Printf("Queue length: %d", len(queue)) + } else if len(queue) == 0 && lastQueueLog != 0 { + log.Println("Queue empty") + } + + lastQueueLog = len(queue) } } } @@ -276,6 +294,7 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance, vols []incus.Vol if sen.Backup { log.Println(".. adding backup schedule for", inst.Project, inst.Name, sen.BackupSchedule) _, err := s.cron.AddFunc(sen.BackupSchedule, func() { + log.Printf(".. placing backup of %s/%s into the queue", inst.Project, inst.Name) s.planner <- schedulerPlan{Instance: inst, Reason: planReasonBackup} }) if err != nil { @@ -286,6 +305,7 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance, vols []incus.Vol if sen.Sync { log.Println(".. adding sync schedule for", inst.Project, inst.Name, sen.SyncSchedule) _, err := s.cron.AddFunc(sen.SyncSchedule, func() { + log.Printf(".. placing sync of %s/%s into the queue", inst.Project, inst.Name) s.planner <- schedulerPlan{Instance: inst, Reason: planReasonSync} }) if err != nil { @@ -300,6 +320,7 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance, vols []incus.Vol if sen.Backup { log.Println(".. adding backup schedule for", vol.Project, vol.Pool, vol.Name, sen.BackupSchedule) _, err := s.cron.AddFunc(sen.BackupSchedule, func() { + log.Printf(".. placing volume backup of %s%s%s into the queue", vol.Project, vol.Pool, vol.Name) s.planner <- schedulerPlan{Volume: vol, Reason: planReasonBackupVolume} }) if err != nil { @@ -310,6 +331,7 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance, vols []incus.Vol if sen.Sync { log.Println(".. adding sync schedule for", vol.Project, vol.Pool, vol.Name, sen.SyncSchedule) _, err := s.cron.AddFunc(sen.SyncSchedule, func() { + log.Printf(".. placing volume sync of %s%s%s into the queue", vol.Project, vol.Pool, vol.Name) s.planner <- schedulerPlan{Volume: vol, Reason: planReasonSyncVolume} }) if err != nil {