Initial commit

.gitea/workflows/main.yml

@@ -0,0 +1,40 @@
+name: Build and release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        type: string
+        description: 'Version'
+        required: true
+        default: 'v0'
+
+jobs:
+  test:
+    runs-on: [moon, amd64]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: '1.23'
+
+    # - name: Run tests
+    #   run: task test
+
+    - name: Build
+      run: |
+        task build VERSION=${{ github.event.inputs.version }}
+
+    - uses: actions/forgejo-release@v2
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        direction: upload
+        tag: ${{ github.event.inputs.version }}
+        title: ${{ github.event.inputs.version }}
+        url: https://gitea.ceperka.net
+        release-dir: bin/
+        release-notes-assistant: true

.gitignore

@@ -0,0 +1 @@
+bin/

README.md

@@ -0,0 +1,34 @@
+# Incus keeper
+
+This project is covering backup and sync scenarios for Incus instances.
+
+* Backups instances into Restic repository
+* Syncs instances to another incus instance
+* Backs up incus itself
+
+## KV
+
+Functions of Sentinel depends on user KV values configured on each instance. Here is what you can configure:
+
+| Key                              | Default   | Purpose                                                          |
+| -------------------------------- | --------- | ---------------------------------------------------------------- |
+| user.backup                      | false     | true/false, if true, regular backup job into Restic is performed |
+| user.sync                        | false     | true/false, if true, regular sync job into Restic is performed   |
+| user.backup-notify-url           | ""        | Call this URL when backup is done                                |
+| user.sync-notify-url             | ""        | Call this URL when sync is done                                  |
+| user.backup-schedule             | 0 6 * * * | Cron-like line for backup scheduling                             |
+| user.sync-schedule               | 0 6 * * * | Cron-like line for sync scheduling                               |
+| user.sync-target-remote          | ""        | Sync's target host (needs to be configured in Incus)             |
+| user.sync-target-pool            | pool0     | Target's storage pool                                            |
+| user.sync-target-instance-suffix | -cold     | Instance name suffix at the target side                          |
+
+## Important notes
+
+Only one backup or sync job can run the same time. There is internal queue of jobs that
+is picked up one by one. Cli commands sync and backup are independent from this queue.
+
+Restic needs two environment variables to be set:
+
+* RESTIC_PASSWORD
+* RESTIC_REPOSITORY
+

Taskfile.yml

@@ -0,0 +1,27 @@
+# https://taskfile.dev
+
+version: '3'
+
+vars:
+  VERSION: v0
+
+tasks:
+  build-bin:
+    cmds:
+      - GOARCH={{ .GOARCH }} go build -o bin/incus-sentinel.{{ .VERSION }}.linux.{{ .GOARCH }} main.go
+    silent: false
+  build:
+    cmds:
+    - mkdir -p bin
+    - task: build-bin
+      vars:
+        GOARCH: arm64
+    - task: build-bin
+      vars:
+        GOARCH: amd64
+  deploy-racker:
+    cmds:
+    - task: build
+    - scp bin/incus-sentinel.{{ .VERSION }}.linux.arm64 racker:/usr/local/bin/incus-sentinel.tmp
+    - ssh racker mv /usr/local/bin/incus-sentinel.tmp /usr/local/bin/incus-sentinel
+    - ssh racker systemctl restart incus-sentinel

dummy/main.go

@@ -0,0 +1,37 @@
+package dummy
+
+import (
+	"time"
+
+	"gitea.ceperka.net/rosti/incus-sentinel/incus"
+)
+
+type DummyDriver struct{}
+
+func NewDummyDriver() *DummyDriver {
+	return &DummyDriver{}
+}
+
+func (d *DummyDriver) GetInstances(target string) ([]incus.Instance, error) {
+	instances := []incus.Instance{
+		{
+			Name: "test0",
+			Config: map[string]string{
+				"sync":                        "true",
+				"sync-schedule":               "* * * * *",
+				"sync-target-remote":          "racker1",
+				"sync-target-pool":            "pool0",
+				"sync-target-instance-suffix": "-cold",
+			},
+		},
+	}
+	return instances, nil
+}
+func (d *DummyDriver) Sync(sourceInstance string, targetInstance string, targetHost string, targetPool string) error {
+	time.Sleep(5 * time.Second)
+	return nil
+}
+func (d *DummyDriver) Backup(instance string, tags []string) error {
+	time.Sleep(5 * time.Second)
+	return nil
+}

go.mod

@@ -0,0 +1,8 @@
+module gitea.ceperka.net/rosti/incus-sentinel
+
+go 1.23.4
+
+require (
+	github.com/robfig/cron/v3 v3.0.1
+	github.com/urfave/cli/v3 v3.0.0-beta1
+)

go.sum

@@ -0,0 +1,12 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/urfave/cli/v3 v3.0.0-beta1 h1:6DTaaUarcM0wX7qj5Hcvs+5Dm3dyUTBbEwIWAjcw9Zg=
+github.com/urfave/cli/v3 v3.0.0-beta1/go.mod h1:FnIeEMYu+ko8zP1F9Ypr3xkZMIDqW3DR92yUtY39q1Y=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

http_notifier/main.go

@@ -0,0 +1,27 @@
+package http_notifier
+
+import (
+	"fmt"
+	"net/http"
+	"slices"
+)
+
+type HTTPNotifier struct{}
+
+func NewNotifier() *HTTPNotifier {
+	return &HTTPNotifier{}
+}
+
+func (n *HTTPNotifier) Notify(url string) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return fmt.Errorf("Failed to call URL %s: %s", url, err.Error())
+	}
+	defer resp.Body.Close()
+
+	if !slices.Contains([]int{http.StatusOK, http.StatusCreated, http.StatusNoContent}, resp.StatusCode) {
+		return fmt.Errorf("Unexpected status code %d from URL %s", resp.StatusCode, url)
+	}
+
+	return nil
+}

incus/main.go

@@ -0,0 +1,100 @@
+package incus
+
+import (
+	"encoding/json"
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+type IncusDriver struct{}
+
+func NewIncusDriver() *IncusDriver {
+	return &IncusDriver{}
+}
+
+func (d *IncusDriver) GetInstances(target string) ([]Instance, error) {
+	// Command: incus list -f json
+
+	var cmd *exec.Cmd
+	if target == "" {
+		cmd = exec.Command("incus", "list", "--format", "json")
+	} else {
+		cmd = exec.Command("incus", "list", target+":", "--format", "json")
+	}
+
+	output, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute incus list: %w", err)
+	}
+
+	var instances []Instance
+	err = json.Unmarshal(output, &instances)
+	if err != nil {
+		return nil, err
+	}
+
+	return instances, nil
+}
+
+func (d *IncusDriver) Sync(sourceInstance string, targetInstance string, targetHost string, targetPool string) error {
+	// incus copy edge0 racker1:edge0-cold -s pool0 --mode push -p default -p net_edge0 --stateless --refresh
+	// incus copy edge0 racker1:edge0-cold -s pool0 --mode push -p default -p net_edge0 --stateless
+
+	instances, err := d.GetInstances(targetHost)
+	if err != nil {
+		return err
+	}
+
+	var cmd *exec.Cmd
+	if len(instances) == 0 {
+		cmd = exec.Command("incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless")
+	} else {
+		cmd = exec.Command("incus", "copy", sourceInstance, targetHost+":"+targetInstance, "-s", targetPool, "--mode", "push", "--stateless", "--refresh")
+	}
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to execute incus copy: %w (%s)", err, string(out))
+	}
+
+	return nil
+}
+
+func (d *IncusDriver) Backup(instance string, tags []string) error {
+	// incus export ups - -q --compression=zstd --instance-only --optimized-storage | restic backup --stdin --stdin-filename ups.btrfs.zstd --tag instance
+
+	// Create the incus export command
+	incusCmd := exec.Command("incus", "export", instance, "-", "-q", "--compression=zstd", "--instance-only", "--optimized-storage")
+
+	// Create the restic backup command
+	resticCmd := exec.Command("restic", "backup", "--host", instance, "--stdin", "--stdin-filename", fmt.Sprintf("%s.btrfs.zstd", instance), "--tag", strings.Join(tags, ","))
+
+	// Connect the output of incusCmd to the input of resticCmd
+	pipe, err := incusCmd.StdoutPipe()
+	if err != nil {
+		return fmt.Errorf("failed to create pipe: %w", err)
+	}
+	resticCmd.Stdin = pipe
+
+	// Start the incus export command
+	if err := incusCmd.Start(); err != nil {
+		return fmt.Errorf("failed to start incus export: %w", err)
+	}
+
+	// Start the restic backup command
+	if err := resticCmd.Start(); err != nil {
+		return fmt.Errorf("failed to start restic backup: %w", err)
+	}
+
+	// Wait for the incus export command to finish
+	if err := incusCmd.Wait(); err != nil {
+		return fmt.Errorf("incus export command failed: %w", err)
+	}
+
+	// Wait for the restic backup command to finish
+	if err := resticCmd.Wait(); err != nil {
+		return fmt.Errorf("restic backup command failed: %w", err)
+	}
+
+	return nil
+}

incus/types.go

@@ -0,0 +1,87 @@
+package incus
+
+import (
+	"time"
+)
+
+type InstanceSentinel struct {
+	Backup                   bool
+	Sync                     bool
+	BackupSchedule           string
+	BackupNotifyURL          string
+	SyncNotifyURL            string
+	SyncSchedule             string
+	SyncTargetRemote         string
+	SyncTargetPool           string
+	SyncTargetInstanceSuffix string
+}
+
+type Instance struct {
+	Architecture string                 `json:"architecture"`
+	Config       map[string]string      `json:"config"`
+	Devices      map[string]interface{} `json:"devices"`
+	Ephemeral    bool                   `json:"ephemeral"`
+	Profiles     []string               `json:"profiles"`
+	Stateful     bool                   `json:"stateful"`
+	Description  string                 `json:"description"`
+	CreatedAt    time.Time              `json:"created_at"`
+	Name         string                 `json:"name"`
+	Status       string                 `json:"status"`
+	StatusCode   int                    `json:"status_code"`
+	LastUsedAt   time.Time              `json:"last_used_at"`
+	Location     string                 `json:"location"`
+	Type         string                 `json:"type"`
+	Project      string                 `json:"project"`
+}
+
+func (i *Instance) Sentinel() InstanceSentinel {
+	s := InstanceSentinel{
+		Backup:                   false,
+		Sync:                     false,
+		BackupNotifyURL:          "",
+		SyncNotifyURL:            "",
+		BackupSchedule:           "0 6 * * *",
+		SyncSchedule:             "0 6 * * *",
+		SyncTargetRemote:         "",
+		SyncTargetPool:           "pool0",
+		SyncTargetInstanceSuffix: "-cold",
+	}
+
+	if val, ok := i.Config["user.backup"]; ok {
+		s.Backup = val == "true"
+	}
+
+	if val, ok := i.Config["user.sync"]; ok {
+		s.Sync = val == "true"
+	}
+
+	if val, ok := i.Config["user.backup-notify-url"]; ok {
+		s.BackupNotifyURL = val
+	}
+
+	if val, ok := i.Config["user.sync-notify-url"]; ok {
+		s.SyncNotifyURL = val
+	}
+
+	if val, ok := i.Config["user.backup-schedule"]; ok {
+		s.BackupSchedule = val
+	}
+
+	if val, ok := i.Config["user.sync-schedule"]; ok {
+		s.SyncSchedule = val
+	}
+
+	if val, ok := i.Config["user.sync-target-remote"]; ok {
+		s.SyncTargetRemote = val
+	}
+
+	if val, ok := i.Config["user.sync-target-pool"]; ok {
+		s.SyncTargetPool = val
+	}
+
+	if val, ok := i.Config["user.sync-target-instance-suffix"]; ok {
+		s.SyncTargetInstanceSuffix = val
+	}
+
+	return s
+}

main.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"gitea.ceperka.net/rosti/incus-sentinel/http_notifier"
+	"gitea.ceperka.net/rosti/incus-sentinel/incus"
+	"gitea.ceperka.net/rosti/incus-sentinel/scheduler"
+	"github.com/urfave/cli/v3"
+)
+
+func main() {
+	cmd := &cli.Command{
+		Name:  "incus-sentinel",
+		Usage: "Keeps an eye on sync and backups of Incus instances",
+		Commands: []*cli.Command{
+			{
+				Name:  "list",
+				Usage: "List all instances with their sync and backup settings",
+				Flags: []cli.Flag{},
+				Action: func(c context.Context, cmd *cli.Command) error {
+					i := incus.NewIncusDriver()
+					insts, err := i.GetInstances("")
+					if err != nil {
+						return err
+					}
+
+					for _, inst := range insts {
+						s := inst.Sentinel()
+						fmt.Printf("%s\n", inst.Name)
+						fmt.Printf("  Backup: %t (%s)\n", s.Backup, s.BackupSchedule)
+						fmt.Printf("  Sync: %t (%s)\n", s.Sync, s.SyncSchedule)
+						if s.Sync {
+							fmt.Printf("  Sync Target: %s (pool: %s, suffix: %s)\n", s.SyncTargetRemote, s.SyncTargetPool, s.SyncTargetInstanceSuffix)
+						}
+					}
+
+					return nil
+				},
+			},
+			{
+				Name:  "sync",
+				Usage: "Syncs all instances where sync is enabled",
+				Flags: []cli.Flag{},
+				Action: func(c context.Context, cmd *cli.Command) error {
+					i := incus.NewIncusDriver()
+					insts, err := i.GetInstances("")
+					if err != nil {
+						return err
+					}
+
+					for _, inst := range insts {
+						s := inst.Sentinel()
+						if s.Sync {
+							fmt.Println(".. syncing", inst.Name)
+							err := i.Sync(inst.Name, fmt.Sprintf("%s%s", inst.Name, s.SyncTargetInstanceSuffix), s.SyncTargetRemote, s.SyncTargetPool)
+							if err != nil {
+								return err
+							}
+						}
+					}
+
+					return nil
+				},
+			},
+			{
+				Name:  "backup",
+				Usage: "Backs up all instances where backup is enabled",
+				Flags: []cli.Flag{},
+				Action: func(c context.Context, cmd *cli.Command) error {
+					i := incus.NewIncusDriver()
+					insts, err := i.GetInstances("")
+					if err != nil {
+						return err
+					}
+
+					for _, inst := range insts {
+						s := inst.Sentinel()
+						if s.Backup {
+							fmt.Println(".. backing up", inst.Name)
+							err := i.Backup(inst.Name, []string{})
+							if err != nil {
+								return err
+							}
+						}
+					}
+
+					return nil
+				},
+			},
+			{
+				Name:  "run",
+				Usage: "Runs the sentinel that syncs and backs up instances based on their configuration",
+				Flags: []cli.Flag{},
+				Action: func(c context.Context, cmd *cli.Command) error {
+					i := incus.NewIncusDriver()
+					n := http_notifier.NewNotifier()
+
+					s := scheduler.NewScheduler(i, n)
+					err := s.Run()
+					if err != nil {
+						return err
+					}
+
+					return nil
+				},
+			},
+		},
+	}
+
+	if err := cmd.Run(context.Background(), os.Args); err != nil {
+		log.Fatal(err)
+	}
+}

scheduler/main.go

@@ -0,0 +1,249 @@
+package scheduler
+
+import (
+	"log"
+	"strings"
+	"time"
+
+	"gitea.ceperka.net/rosti/incus-sentinel/incus"
+	"github.com/robfig/cron/v3"
+)
+
+const queueLength = 100
+const planReasonBackup = "backup"
+const planReasonSync = "sync"
+
+type schedulerPlan struct {
+	Instance incus.Instance
+	Reason   string // backup or sync
+}
+
+// Scheduler is the main struct that handles the scheduling of backups and syncs.
+// It keeps internal queue of instances to be processed and it runs cron-like
+// schedules for each instance.
+type Scheduler struct {
+	driver   Driver
+	notifier Notifier
+
+	planner           chan schedulerPlan   // Receives instance to be backed up and/or synced
+	queueDumpRequest  chan bool            // Request to dump the queue
+	queueDumpResponse chan []schedulerPlan // Response with the queue
+	exit              chan bool            // Exit signal
+
+	scheduledFootprint string // Footprint of the scheduler, used to identify if the scheduler needs to be changed
+	cron               *cron.Cron
+}
+
+func NewScheduler(d Driver, n Notifier) *Scheduler {
+	cron.New(cron.WithSeconds())
+
+	s := &Scheduler{
+		driver:            d,
+		notifier:          n,
+		planner:           make(chan schedulerPlan, 10),
+		queueDumpRequest:  make(chan bool, 1),
+		queueDumpResponse: make(chan []schedulerPlan, 1),
+		exit:              make(chan bool),
+	}
+
+	go s.runMainLoop()
+
+	return s
+}
+
+func (s *Scheduler) do(plan schedulerPlan, done chan schedulerPlan) {
+	defer func() {
+		done <- plan
+	}()
+
+	// Do the actual work
+	sen := plan.Instance.Sentinel()
+	var err error
+	if plan.Reason == planReasonBackup {
+		start := time.Now()
+
+		err = s.driver.Backup(plan.Instance.Name, []string{"sentinel"})
+		if err != nil {
+			log.Printf("Failed to backup %s: %s", plan.Instance.Name, err.Error())
+		}
+
+		log.Printf("Backup of %s took %s", plan.Instance.Name, time.Since(start).String())
+
+		if sen.BackupNotifyURL != "" && s.notifier != nil {
+			err = s.notifier.Notify(sen.BackupNotifyURL)
+			if err != nil {
+				log.Printf("Failed to notify %s: %s", sen.BackupNotifyURL, err.Error())
+			}
+		}
+	}
+
+	if plan.Reason == planReasonSync {
+		start := time.Now()
+
+		err = s.driver.Sync(plan.Instance.Name, plan.Instance.Name+sen.SyncTargetInstanceSuffix, sen.SyncTargetRemote, sen.SyncTargetPool)
+		if err != nil {
+			log.Printf("Failed to sync %s: %s", plan.Instance.Name, err.Error())
+		}
+
+		log.Printf("Sync of %s took %s", plan.Instance.Name, time.Since(start).String())
+
+		if sen.SyncNotifyURL != "" && s.notifier != nil {
+			err = s.notifier.Notify(sen.SyncNotifyURL)
+			if err != nil {
+				log.Printf("Failed to notify %s: %s", sen.SyncNotifyURL, err.Error())
+			}
+		}
+	}
+}
+
+// Loop that does the heavy-lifting
+func (s *Scheduler) runMainLoop() error {
+	inProgress := false
+	done := make(chan schedulerPlan)
+	instancesToProcess := []schedulerPlan{}
+
+	queue := make(chan schedulerPlan, queueLength)
+	queuedInstances := make(map[string]schedulerPlan)
+
+	ticker := time.NewTicker(2 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		// New instance to plan
+		case newPlan := <-s.planner:
+			if len(queue) >= queueLength {
+				log.Printf("Queue full (%d)), dropping plan for %s of %s", len(queue), newPlan.Reason, newPlan.Instance.Name)
+				continue
+			}
+
+			// Check if instance is already planned
+			_, found := queuedInstances[newPlan.Reason+":"+newPlan.Instance.Name]
+
+			if !found {
+				queue <- newPlan
+				queuedInstances[newPlan.Reason+":"+newPlan.Instance.Name] = newPlan
+			}
+		// Instance is done, remove it from the tracker
+		case plan := <-done:
+			delete(queuedInstances, plan.Reason+":"+plan.Instance.Name)
+			inProgress = false
+		// Dump the queue
+		case <-s.queueDumpRequest:
+			response := []schedulerPlan{}
+			copy(response, instancesToProcess)
+			s.queueDumpResponse <- response
+		case <-s.exit:
+			if inProgress {
+				log.Println("Waiting for the current instance to finish ..")
+				<-done
+			}
+			return nil
+		// Check the queue and start processing
+		case <-ticker.C:
+			if !inProgress && len(queue) > 0 {
+				inProgress = true
+				go s.do(<-queue, done)
+			}
+		}
+	}
+}
+
+func (s *Scheduler) Close() {
+	s.exit <- true
+}
+
+func (s *Scheduler) Run() error {
+	lastQueueDump := ""
+
+	log.Println("Starting scheduler ..")
+	for {
+		s.refresh()
+
+		// Print the list of instances in the queue
+		s.queueDumpRequest <- true
+		plans := <-s.queueDumpResponse
+		if len(plans) > 0 {
+			output := []string{}
+			for _, plan := range plans {
+				output = append(output, plan.Reason+":"+plan.Instance.Name)
+			}
+
+			outputString := strings.Join(output, ", ")
+			if lastQueueDump != outputString {
+				log.Println("Queue:", outputString)
+				lastQueueDump = outputString
+			}
+		}
+
+		time.Sleep(15 * time.Second)
+	}
+}
+
+func (s *Scheduler) footprint(is []incus.Instance) string {
+	footprint := ""
+	for _, inst := range is {
+		sen := inst.Sentinel()
+		footprint += inst.Name + sen.BackupSchedule + sen.SyncSchedule
+	}
+
+	return footprint
+}
+
+// Checks if the scheduler needs to be refreshed and does it if needed
+func (s *Scheduler) refresh() error {
+	instances, err := s.driver.GetInstances("")
+	if err != nil {
+		return err
+	}
+
+	err = s.setupInstanceSchedules(instances)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Refresh cron like schedulers for all instances
+func (s *Scheduler) setupInstanceSchedules(is []incus.Instance) error {
+	if s.scheduledFootprint == s.footprint(is) {
+		return nil
+	}
+
+	log.Println("Refreshing scheduler ..")
+
+	if s.cron != nil {
+		s.cron.Stop()
+	}
+
+	s.cron = cron.New()
+
+	for _, inst := range is {
+		sen := inst.Sentinel()
+		if sen.Backup {
+			log.Println(".. adding backup schedule for", inst.Name, sen.BackupSchedule)
+			_, err := s.cron.AddFunc(sen.BackupSchedule, func() {
+				s.planner <- schedulerPlan{Instance: inst, Reason: planReasonBackup}
+			})
+			if err != nil {
+				return err
+			}
+		}
+
+		if sen.Sync {
+			log.Println(".. adding sync schedule for", inst.Name, sen.SyncSchedule)
+			_, err := s.cron.AddFunc(sen.SyncSchedule, func() {
+				s.planner <- schedulerPlan{Instance: inst, Reason: planReasonSync}
+			})
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	s.scheduledFootprint = s.footprint(is)
+	s.cron.Start()
+
+	return nil
+}

scheduler/types.go

@@ -0,0 +1,19 @@
+package scheduler
+
+import "gitea.ceperka.net/rosti/incus-sentinel/incus"
+
+type Driver interface {
+	// GetInstances retrieves a list of instances from the target
+	GetInstances(target string) ([]incus.Instance, error)
+
+	// Sync synchronizes instances between source and target
+	Sync(sourceInstance string, targetInstance string, targetHost string, targetPool string) error
+
+	// Backup creates a backup of the specified instance with given tags
+	Backup(instance string, tags []string) error
+}
+
+type Notifier interface {
+	// Notify sends a notification to the specified URL
+	Notify(url string) error
+}