Better login, fix of backup scenarios where pipe is used when running under systemd
This commit is contained in:
parent
37c191f90e
commit
da09b87020
4 changed files with 64 additions and 18 deletions
10
Taskfile.yml
10
Taskfile.yml
|
@ -26,13 +26,21 @@ tasks:
|
||||||
- scp bin/incus-sentinel.{{ .VERSION }}.linux.{{ .GOARCH }} {{ .DEPLOY_HOST }}:/usr/local/bin/incus-sentinel.tmp
|
- scp bin/incus-sentinel.{{ .VERSION }}.linux.{{ .GOARCH }} {{ .DEPLOY_HOST }}:/usr/local/bin/incus-sentinel.tmp
|
||||||
- ssh {{ .DEPLOY_HOST }} mv /usr/local/bin/incus-sentinel.tmp /usr/local/bin/incus-sentinel
|
- ssh {{ .DEPLOY_HOST }} mv /usr/local/bin/incus-sentinel.tmp /usr/local/bin/incus-sentinel
|
||||||
- ssh {{ .DEPLOY_HOST }} systemctl restart incus-sentinel
|
- ssh {{ .DEPLOY_HOST }} systemctl restart incus-sentinel
|
||||||
deploy-racker:
|
deploy-home:
|
||||||
cmds:
|
cmds:
|
||||||
- task: build
|
- task: build
|
||||||
- task: deploy
|
- task: deploy
|
||||||
vars:
|
vars:
|
||||||
DEPLOY_HOST: racker
|
DEPLOY_HOST: racker
|
||||||
GOARCH: arm64
|
GOARCH: arm64
|
||||||
|
- task: deploy
|
||||||
|
vars:
|
||||||
|
DEPLOY_HOST: racker1
|
||||||
|
GOARCH: arm64
|
||||||
|
- task: deploy
|
||||||
|
vars:
|
||||||
|
DEPLOY_HOST: deimos
|
||||||
|
GOARCH: amd64
|
||||||
deploy-rosti:
|
deploy-rosti:
|
||||||
cmds:
|
cmds:
|
||||||
- task: build
|
- task: build
|
||||||
|
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"slices"
|
"slices"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
type HTTPNotifier struct{}
|
type HTTPNotifier struct{}
|
||||||
|
@ -13,7 +14,11 @@ func NewNotifier() *HTTPNotifier {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *HTTPNotifier) Notify(url string) error {
|
func (n *HTTPNotifier) Notify(url string) error {
|
||||||
resp, err := http.Get(url)
|
client := &http.Client{
|
||||||
|
Timeout: 10 * time.Second, // Set timeout duration
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := client.Get(url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("Failed to call URL %s: %s", url, err.Error())
|
return fmt.Errorf("Failed to call URL %s: %s", url, err.Error())
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,7 +3,9 @@ package incus
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"log"
|
"log"
|
||||||
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
@ -26,12 +28,17 @@ func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []s
|
||||||
resticCmd = exec.Command("restic", "backup", "--stdin", "--stdin-filename", filename, "--tag", strings.Join(tags, ","))
|
resticCmd = exec.Command("restic", "backup", "--stdin", "--stdin-filename", filename, "--tag", strings.Join(tags, ","))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Create a pipe to connect the two commands
|
||||||
|
r, w := io.Pipe()
|
||||||
|
|
||||||
// Connect the output of incusCmd to the input of resticCmd
|
// Connect the output of incusCmd to the input of resticCmd
|
||||||
pipe, err := incusCmd.StdoutPipe()
|
incusCmd.Stdout = w
|
||||||
if err != nil {
|
incusCmd.Stderr = os.Stderr
|
||||||
return fmt.Errorf("failed to create pipe: %w", err)
|
incusCmd.Env = os.Environ()
|
||||||
}
|
resticCmd.Stdin = r
|
||||||
resticCmd.Stdin = pipe
|
resticCmd.Stdout = os.Stdout
|
||||||
|
resticCmd.Stderr = os.Stderr
|
||||||
|
resticCmd.Env = os.Environ()
|
||||||
|
|
||||||
// Start the incus export command
|
// Start the incus export command
|
||||||
if err := incusCmd.Start(); err != nil {
|
if err := incusCmd.Start(); err != nil {
|
||||||
|
@ -46,7 +53,11 @@ func (d *IncusDriver) pipeToRestic(incusCmd *exec.Cmd, filename string, tags []s
|
||||||
|
|
||||||
// Wait for the incus export command to finish
|
// Wait for the incus export command to finish
|
||||||
if err := incusCmd.Wait(); err != nil {
|
if err := incusCmd.Wait(); err != nil {
|
||||||
return fmt.Errorf("incus export command failed: %w", err)
|
log.Printf("incus export command failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := w.Close(); err != nil {
|
||||||
|
log.Printf("failed to close pipe: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wait for the restic backup command to finish
|
// Wait for the restic backup command to finish
|
||||||
|
|
|
@ -71,45 +71,53 @@ func (s *Scheduler) do(plan schedulerPlan, done chan schedulerPlan) {
|
||||||
|
|
||||||
switch plan.Reason {
|
switch plan.Reason {
|
||||||
case planReasonBackup:
|
case planReasonBackup:
|
||||||
|
log.Printf(".. backup of %s/%s instance started", plan.Instance.Project, plan.Instance.Name)
|
||||||
|
|
||||||
err = s.driver.Backup(plan.Instance.Project, plan.Instance.Name, defaultTags)
|
err = s.driver.Backup(plan.Instance.Project, plan.Instance.Name, defaultTags)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Failed to backup %s: %s", plan.Instance.Name, err.Error())
|
log.Printf(".. failed to backup %s: %s", plan.Instance.Name, err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Printf("Backup of %s took %s", plan.Instance.Name, time.Since(start).String())
|
log.Printf(".. backup of %s took %s", plan.Instance.Name, time.Since(start).String())
|
||||||
|
|
||||||
notifyURL = sen.BackupNotifyURL
|
notifyURL = sen.BackupNotifyURL
|
||||||
case planReasonSync:
|
case planReasonSync:
|
||||||
|
log.Printf(".. syncing of %s/%s instance started", plan.Instance.Project, plan.Instance.Name)
|
||||||
|
|
||||||
err = s.driver.Sync(plan.Instance.Project, plan.Instance.Name, plan.Instance.Name+sen.SyncTargetInstanceSuffix, sen.SyncTargetRemote, sen.SyncTargetPool)
|
err = s.driver.Sync(plan.Instance.Project, plan.Instance.Name, plan.Instance.Name+sen.SyncTargetInstanceSuffix, sen.SyncTargetRemote, sen.SyncTargetPool)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Failed to sync %s: %s", plan.Instance.Name, err.Error())
|
log.Printf(".. failed to sync %s: %s", plan.Instance.Name, err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Printf("Sync of %s took %s", plan.Instance.Name, time.Since(start).String())
|
log.Printf(".. sync of %s took %s", plan.Instance.Name, time.Since(start).String())
|
||||||
|
|
||||||
notifyURL = sen.SyncNotifyURL
|
notifyURL = sen.SyncNotifyURL
|
||||||
case planReasonBackupVolume:
|
case planReasonBackupVolume:
|
||||||
|
log.Printf(".. backup of %s/%s/%s volume started", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name)
|
||||||
|
|
||||||
sen := plan.Volume.Sentinel()
|
sen := plan.Volume.Sentinel()
|
||||||
if sen.BackupMode == "dir" {
|
if sen.BackupMode == "dir" {
|
||||||
err = s.driver.BackupVolumeDir(plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, defaultTags)
|
err = s.driver.BackupVolumeDir(plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, defaultTags)
|
||||||
} else if sen.BackupMode == "native" {
|
} else if sen.BackupMode == "native" {
|
||||||
err = s.driver.BackupVolumeNative(plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, defaultTags)
|
err = s.driver.BackupVolumeNative(plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, defaultTags)
|
||||||
} else {
|
} else {
|
||||||
log.Printf("Invalid backup mode for volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, sen.BackupMode)
|
log.Printf(".. invalid backup mode for volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, sen.BackupMode)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Failed to backup volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, err.Error())
|
log.Printf(".. failed to backup volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Printf("Backup of volume %s/%s/%s took %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, time.Since(start).String())
|
log.Printf(".. backup of volume %s/%s/%s took %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, time.Since(start).String())
|
||||||
|
|
||||||
notifyURL = sen.BackupNotifyURL
|
notifyURL = sen.BackupNotifyURL
|
||||||
case planReasonSyncVolume:
|
case planReasonSyncVolume:
|
||||||
|
log.Printf(".. syncing of %s/%s/%s volume started", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name)
|
||||||
|
|
||||||
sen := plan.Volume.Sentinel()
|
sen := plan.Volume.Sentinel()
|
||||||
targetPool := plan.Volume.Pool
|
targetPool := plan.Volume.Pool
|
||||||
if sen.SyncTargetPool != "" {
|
if sen.SyncTargetPool != "" {
|
||||||
|
@ -118,11 +126,11 @@ func (s *Scheduler) do(plan schedulerPlan, done chan schedulerPlan) {
|
||||||
|
|
||||||
err = s.driver.SyncVolume(plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, sen.SyncTargetRemote, targetPool, plan.Volume.Name+sen.SyncTargetVolumeSuffix)
|
err = s.driver.SyncVolume(plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, sen.SyncTargetRemote, targetPool, plan.Volume.Name+sen.SyncTargetVolumeSuffix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Failed to sync volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, err.Error())
|
log.Printf(".. failed to sync volume %s/%s/%s: %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Printf("Sync of volume %s/%s/%s took %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, time.Since(start).String())
|
log.Printf(".. sync of volume %s/%s/%s took %s", plan.Volume.Project, plan.Volume.Pool, plan.Volume.Name, time.Since(start).String())
|
||||||
|
|
||||||
notifyURL = sen.SyncNotifyURL
|
notifyURL = sen.SyncNotifyURL
|
||||||
}
|
}
|
||||||
|
@ -130,7 +138,7 @@ func (s *Scheduler) do(plan schedulerPlan, done chan schedulerPlan) {
|
||||||
if notifyURL != "" && s.notifier != nil {
|
if notifyURL != "" && s.notifier != nil {
|
||||||
err = s.notifier.Notify(notifyURL)
|
err = s.notifier.Notify(notifyURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Failed to notify %s: %s", notifyURL, err.Error())
|
log.Printf(".. failed to notify %s: %s", notifyURL, err.Error())
|
||||||
}
|
}
|
||||||
} else if notifyURL != "" && s.notifier == nil {
|
} else if notifyURL != "" && s.notifier == nil {
|
||||||
log.Println("Warning: No notifier configured, skipping notification")
|
log.Println("Warning: No notifier configured, skipping notification")
|
||||||
|
@ -149,6 +157,8 @@ func (s *Scheduler) runMainLoop() error {
|
||||||
ticker := time.NewTicker(2 * time.Second)
|
ticker := time.NewTicker(2 * time.Second)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
lastQueueLog := 0
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
// New instance to plan
|
// New instance to plan
|
||||||
|
@ -186,6 +196,14 @@ func (s *Scheduler) runMainLoop() error {
|
||||||
inProgress = true
|
inProgress = true
|
||||||
go s.do(<-queue, done)
|
go s.do(<-queue, done)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(queue) > 0 && lastQueueLog != len(queue) {
|
||||||
|
log.Printf("Queue length: %d", len(queue))
|
||||||
|
} else if len(queue) == 0 && lastQueueLog != 0 {
|
||||||
|
log.Println("Queue empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
lastQueueLog = len(queue)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -276,6 +294,7 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance, vols []incus.Vol
|
||||||
if sen.Backup {
|
if sen.Backup {
|
||||||
log.Println(".. adding backup schedule for", inst.Project, inst.Name, sen.BackupSchedule)
|
log.Println(".. adding backup schedule for", inst.Project, inst.Name, sen.BackupSchedule)
|
||||||
_, err := s.cron.AddFunc(sen.BackupSchedule, func() {
|
_, err := s.cron.AddFunc(sen.BackupSchedule, func() {
|
||||||
|
log.Printf(".. placing backup of %s/%s into the queue", inst.Project, inst.Name)
|
||||||
s.planner <- schedulerPlan{Instance: inst, Reason: planReasonBackup}
|
s.planner <- schedulerPlan{Instance: inst, Reason: planReasonBackup}
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -286,6 +305,7 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance, vols []incus.Vol
|
||||||
if sen.Sync {
|
if sen.Sync {
|
||||||
log.Println(".. adding sync schedule for", inst.Project, inst.Name, sen.SyncSchedule)
|
log.Println(".. adding sync schedule for", inst.Project, inst.Name, sen.SyncSchedule)
|
||||||
_, err := s.cron.AddFunc(sen.SyncSchedule, func() {
|
_, err := s.cron.AddFunc(sen.SyncSchedule, func() {
|
||||||
|
log.Printf(".. placing sync of %s/%s into the queue", inst.Project, inst.Name)
|
||||||
s.planner <- schedulerPlan{Instance: inst, Reason: planReasonSync}
|
s.planner <- schedulerPlan{Instance: inst, Reason: planReasonSync}
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -300,6 +320,7 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance, vols []incus.Vol
|
||||||
if sen.Backup {
|
if sen.Backup {
|
||||||
log.Println(".. adding backup schedule for", vol.Project, vol.Pool, vol.Name, sen.BackupSchedule)
|
log.Println(".. adding backup schedule for", vol.Project, vol.Pool, vol.Name, sen.BackupSchedule)
|
||||||
_, err := s.cron.AddFunc(sen.BackupSchedule, func() {
|
_, err := s.cron.AddFunc(sen.BackupSchedule, func() {
|
||||||
|
log.Printf(".. placing volume backup of %s%s%s into the queue", vol.Project, vol.Pool, vol.Name)
|
||||||
s.planner <- schedulerPlan{Volume: vol, Reason: planReasonBackupVolume}
|
s.planner <- schedulerPlan{Volume: vol, Reason: planReasonBackupVolume}
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -310,6 +331,7 @@ func (s *Scheduler) setupInstanceSchedules(is []incus.Instance, vols []incus.Vol
|
||||||
if sen.Sync {
|
if sen.Sync {
|
||||||
log.Println(".. adding sync schedule for", vol.Project, vol.Pool, vol.Name, sen.SyncSchedule)
|
log.Println(".. adding sync schedule for", vol.Project, vol.Pool, vol.Name, sen.SyncSchedule)
|
||||||
_, err := s.cron.AddFunc(sen.SyncSchedule, func() {
|
_, err := s.cron.AddFunc(sen.SyncSchedule, func() {
|
||||||
|
log.Printf(".. placing volume sync of %s%s%s into the queue", vol.Project, vol.Pool, vol.Name)
|
||||||
s.planner <- schedulerPlan{Volume: vol, Reason: planReasonSyncVolume}
|
s.planner <- schedulerPlan{Volume: vol, Reason: planReasonSyncVolume}
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
Loading…
Reference in a new issue