Better login, fix of backup scenarios where pipe is used when running under systemd

Taskfile.yml

@@ -26,13 +26,21 @@ tasks:
     - scp bin/incus-sentinel.{{ .VERSION }}.linux.{{ .GOARCH }} {{ .DEPLOY_HOST }}:/usr/local/bin/incus-sentinel.tmp
     - ssh {{ .DEPLOY_HOST }} mv /usr/local/bin/incus-sentinel.tmp /usr/local/bin/incus-sentinel
     - ssh {{ .DEPLOY_HOST }} systemctl restart incus-sentinel
-  deploy-racker:
+  deploy-home:
     cmds:
     - task: build
     - task: deploy
       vars:
         DEPLOY_HOST: racker
         GOARCH: arm64
+    - task: deploy
+      vars:
+        DEPLOY_HOST: racker1
+        GOARCH: arm64
+    - task: deploy
+      vars:
+        DEPLOY_HOST: deimos
+        GOARCH: amd64
   deploy-rosti:
     cmds:
     - task: build

http_notifier/main.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/http"
 	"slices"
+	"time"
 )
 
 type HTTPNotifier struct{}
@@ -13,7 +14,11 @@ func NewNotifier() *HTTPNotifier {
 }
 
 func (n *HTTPNotifier) Notify(url string) error {
-	resp, err := http.Get(url)
+	client := &http.Client{
+		Timeout: 10 * time.Second, // Set timeout duration
+	}
+
+	resp, err := client.Get(url)
 	if err != nil {
 		return fmt.Errorf("Failed to call URL %s: %s", url, err.Error())
 	}

incus/main.go

@@ -3,7 +3,9 @@ package incus
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"log"
+	"os"
 	"os/exec"
 	"strings"
 )
@@ -26,12 +28,17 @@ func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []s
 		resticCmd = exec.Command("restic", "backup", "--stdin", "--stdin-filename", filename, "--tag", strings.Join(tags, ","))
 	}
 
+	// Create a pipe to connect the two commands
+	r, w := io.Pipe()
+
 	// Connect the output of incusCmd to the input of resticCmd
-	pipe, err := incusCmd.StdoutPipe()
+	incusCmd.Stdout = w
-	if err != nil {
+	incusCmd.Stderr = os.Stderr
-		return fmt.Errorf("failed to create pipe: %w", err)
+	incusCmd.Env = os.Environ()
-	}
+	resticCmd.Stdin = r
-	resticCmd.Stdin = pipe
+	resticCmd.Stdout = os.Stdout
+	resticCmd.Stderr = os.Stderr
+	resticCmd.Env = os.Environ()
 
 	// Start the incus export command
 	if err := incusCmd.Start(); err != nil {
@@ -46,7 +53,11 @@ func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []s
 
 	// Wait for the incus export command to finish
 	if err := incusCmd.Wait(); err != nil {
-		return fmt.Errorf("incus export command failed: %w", err)
+		log.Printf("incus export command failed: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		log.Printf("failed to close pipe: %v", err)
 	}
 
 	// Wait for the restic backup command to finish

scheduler/main.go

@@ -71,45 +71,53 @@ func (s *Scheduler) do(plan schedulerPlan, done chan schedulerPlan) {
 
 	switch plan.Reason {
 	case planReasonBackup:
+		log.Printf(".. backup of %s/%s instance started", plan.Instance.Project, plan.Instance.Name)
+
 		err = s.driver.Backup(plan.Instance.Project, plan.Instance.Name, defaultTags)
 		if err != nil {
-			log.Printf("Failed to backup %s: %s", plan.Instance.Name, err.Error())
+			log.Printf(".. failed to backup %s: %s", plan.Instance.Name, err.Error())
 			return
 		}
 
-		log.Printf("Backup of %s took %s", plan.Instance.Name, time.Since(start).String())
+		log.Printf(".. backup of %s took %s", plan.Instance.Name, time.Since(start).String())
 
 		notifyURL = sen.BackupNotifyURL
 	case planReasonSync:
+		log.Printf(".. syncing of %s/%s instance started", plan.Instance.Project, plan.Instance.Name)
+
 		err = s.driver.Sync(plan.Instance.Project, plan.Instance.Name, plan.Instance.Name+sen.SyncTargetInstanceSuffix, sen.SyncTargetRemote, sen.SyncTargetPool)
 		if err != nil {
-			log.Printf("Failed to sync %s: %s", plan.Instance.Name, err.Error())
+			log.Printf(".. failed to sync %s: %s", plan.Instance.Name, err.Error())
 			return
 		}
 
-		log.Printf("Sync of %s took %s", plan.Instance.Name, time.Since(start).String())
+		log.Printf(".. sync of %s took %s", plan.Instance.Name, time.Since(start).String())
 
 		notifyURL = sen.SyncNotifyURL
 	case planReasonBackupVolume:
+		log.Printf(".. backup of %s/%s/%s volume started", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name)
+
 		sen := plan.Volume.Sentinel()
 		if sen.BackupMode == "dir" {
 			err = s.driver.BackupVolumeDir(plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, defaultTags)
 		} else if sen.BackupMode == "native" {
 			err = s.driver.BackupVolumeNative(plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, defaultTags)
 		} else {
-			log.Printf("Invalid backup mode for volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, sen.BackupMode)
+			log.Printf(".. invalid backup mode for volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, sen.BackupMode)
 			return
 		}
 
 		if err != nil {
-			log.Printf("Failed to backup volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, err.Error())
+			log.Printf(".. failed to backup volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, err.Error())
 			return
 		}
 
-		log.Printf("Backup of volume %s/%s/%s took %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, time.Since(start).String())
+		log.Printf(".. backup of volume %s/%s/%s took %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, time.Since(start).String())
 
 		notifyURL = sen.BackupNotifyURL
 	case planReasonSyncVolume:
+		log.Printf(".. syncing of %s/%s/%s volume started", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name)
+
 		sen := plan.Volume.Sentinel()
 		targetPool := plan.Volume.Pool
 		if sen.SyncTargetPool != "" {
@@ -118,11 +126,11 @@ func (s *Scheduler) do(plan schedulerPlan, done chan schedulerPlan) {
 
 		err = s.driver.SyncVolume(plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, sen.SyncTargetRemote, targetPool, plan.Volume.Name+sen.SyncTargetVolumeSuffix)
 		if err != nil {
-			log.Printf("Failed to sync volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, err.Error())
+			log.Printf(".. failed to sync volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, err.Error())
 			return
 		}
 
-		log.Printf("Sync of volume %s/%s/%s took %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, time.Since(start).String())
+		log.Printf(".. sync of volume %s/%s/%s took %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, time.Since(start).String())
 
 		notifyURL = sen.SyncNotifyURL
 	}
@@ -130,7 +138,7 @@ func (s *Scheduler) do(plan schedulerPlan, done chan schedulerPlan) {
 	if notifyURL != "" && s.notifier != nil {
 		err = s.notifier.Notify(notifyURL)
 		if err != nil {
-			log.Printf("Failed to notify %s: %s", notifyURL, err.Error())
+			log.Printf(".. failed to notify %s: %s", notifyURL, err.Error())
 		}
 	} else if notifyURL != "" && s.notifier == nil {
 		log.Println("Warning: No notifier configured, skipping notification")
@@ -149,6 +157,8 @@ func (s *Scheduler) runMainLoop() error {
 	ticker := time.NewTicker(2 * time.Second)
 	defer ticker.Stop()
 
+	lastQueueLog := 0
+
 	for {
 		select {
 		// New instance to plan
@@ -186,6 +196,14 @@ func (s *Scheduler) runMainLoop() error {
 				inProgress = true
 				go s.do(<-queue, done)
 			}
+
+			if len(queue) > 0 && lastQueueLog != len(queue) {
+				log.Printf("Queue length: %d", len(queue))
+			} else if len(queue) == 0 && lastQueueLog != 0 {
+				log.Println("Queue empty")
+			}
+
+			lastQueueLog = len(queue)
 		}
 	}
 }
@@ -276,6 +294,7 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance, vols []incus.Vol
 		if sen.Backup {
 			log.Println(".. adding backup schedule for", inst.Project, inst.Name, sen.BackupSchedule)
 			_, err := s.cron.AddFunc(sen.BackupSchedule, func() {
+				log.Printf(".. placing backup of %s/%s into the queue", inst.Project, inst.Name)
 				s.planner <- schedulerPlan{Instance: inst, Reason: planReasonBackup}
 			})
 			if err != nil {
@@ -286,6 +305,7 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance, vols []incus.Vol
 		if sen.Sync {
 			log.Println(".. adding sync schedule for", inst.Project, inst.Name, sen.SyncSchedule)
 			_, err := s.cron.AddFunc(sen.SyncSchedule, func() {
+				log.Printf(".. placing sync of %s/%s into the queue", inst.Project, inst.Name)
 				s.planner <- schedulerPlan{Instance: inst, Reason: planReasonSync}
 			})
 			if err != nil {
@@ -300,6 +320,7 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance, vols []incus.Vol
 		if sen.Backup {
 			log.Println(".. adding backup schedule for", vol.Project, vol.Pool, vol.Name, sen.BackupSchedule)
 			_, err := s.cron.AddFunc(sen.BackupSchedule, func() {
+				log.Printf(".. placing volume backup of %s%s%s into the queue", vol.Project, vol.Pool, vol.Name)
 				s.planner <- schedulerPlan{Volume: vol, Reason: planReasonBackupVolume}
 			})
 			if err != nil {
@@ -310,6 +331,7 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance, vols []incus.Vol
 		if sen.Sync {
 			log.Println(".. adding sync schedule for", vol.Project, vol.Pool, vol.Name, sen.SyncSchedule)
 			_, err := s.cron.AddFunc(sen.SyncSchedule, func() {
+				log.Printf(".. placing volume sync of %s%s%s into the queue", vol.Project, vol.Pool, vol.Name)
 				s.planner <- schedulerPlan{Volume: vol, Reason: planReasonSyncVolume}
 			})
 			if err != nil {