Support for snapshots

* full implementation of snapshots
* tests of the snapshot backend
* Drone CI pipeline
* New snapshots handlers
* Filesystem driver
* S3 driver

.drone.yml

@@ -0,0 +1,25 @@
+kind: pipeline
+type: docker
+name: testing
+
+steps:
+- name: test
+  image: golang
+  environment:
+    SNAPSHOTS_S3_ENDPOINT: minio:9000
+    TEST_S3_ENDPOINT: minio:9000
+  commands:
+    - go mod tidy
+    - make test
+
+services:
+- name: minio
+  image: minio/minio:latest
+  environment:
+    MINIO_ROOT_USER: test
+    MINIO_ROOT_PASSWORD: testtest
+  command:
+  - server
+  - /data
+  - --console-address
+  - :9001

.gitignore

@@ -4,3 +4,4 @@ node-api
 api-node-17.http
 *-packr.go
 *.sqlite
+tmp/

Makefile

@@ -1,2 +1,21 @@
-production-build:
+.PHONY: test
-	podman run --rm --privileged -ti -v ${shell pwd}:/srv docker.io/library/golang:1.14-stretch /bin/sh -c "cd /srv && go build"
+test:
+	go test -v apps/*.go
+	go test -v apps/drivers/*.go
+
+build:
+	#podman run --rm --privileged -ti -v ${shell pwd}:/srv docker.io/library/golang:1.14-stretch /bin/sh -c "cd /srv && go build"
+	go build -a -ldflags "-linkmode external -extldflags '-static' -s -w"
+
+.PHONY: minio
+minio:
+	mkdir -p ./tmp/snapshots
+	-podman stop rosti-snapshots
+	-podman rm rosti-snapshots
+	podman run  -d --name rosti-snapshots \
+		-u 1000 \
+		-p 9000:9000 \
+		-p 9001:9001 \
+		-e MINIO_ROOT_USER=test \
+		-e MINIO_ROOT_PASSWORD=testtest \
+		minio/minio server /data --console-address ":9001"

README.md

@@ -1 +1,6 @@
+# Node API
 
+[![Build Status](https://droneci.ceperka.net/api/badges/Rosti/node-api/status.svg)](https://droneci.ceperka.net/Rosti/node-api)
+
+Node API is an microservice that runs on node servers. It provides interface between
+Docker and the admin site. 

apps/drivers/fs.go

@@ -0,0 +1,85 @@
+package drivers
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+)
+
+type FSDriver struct {
+	Path string // Where the objects are stored
+}
+
+// Create creates a new object in the storage with given body.
+// This just copying the given file into the right location.
+func (f FSDriver) Create(key string, filePath string) error {
+	err := os.MkdirAll(path.Dir(path.Join(f.Path, key)), 0755)
+	if err != nil {
+		return err
+	}
+
+	source, err := os.Open(filePath)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	destination, err := os.Create(path.Join(f.Path, key))
+	if err != nil {
+		return err
+	}
+	defer destination.Close()
+
+	_, err = io.Copy(destination, source)
+	return err
+}
+
+// Get copies file from the storage into a local file
+func (f FSDriver) Get(key string, filePath string) error {
+	source, err := os.Open(path.Join(f.Path, key))
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	destination, err := os.Create(filePath)
+	if err != nil {
+		return err
+	}
+	defer destination.Close()
+
+	_, err = io.Copy(destination, source)
+	return err
+}
+
+// Read returns content of given key
+func (f *FSDriver) Read(key string) ([]byte, error) {
+	body, err := ioutil.ReadFile(path.Join(f.Path, key))
+	return body, err
+}
+
+// Write writes content of body into key
+func (f FSDriver) Write(key string, body []byte) error {
+	return ioutil.WriteFile(path.Join(f.Path, key), body, 0644)
+}
+
+// List returns list of objects in fiven prefix
+func (f FSDriver) List(prefix string) ([]string, error) {
+	keys := []string{}
+
+	files, err := ioutil.ReadDir(path.Join(f.Path, prefix))
+	if err != nil {
+		return keys, err
+	}
+	for _, file := range files {
+		keys = append(keys, file.Name())
+	}
+
+	return keys, nil
+}
+
+// Delete deletes one object
+func (f FSDriver) Delete(key string) error {
+	return os.Remove(path.Join(f.Path, key))
+}

apps/drivers/fs_test.go

@@ -0,0 +1,123 @@
+package drivers
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const testDirectory = "./tmp"
+const testKey = "sourcefile"
+const testFile = testDirectory + "/" + testKey
+const testContent = "test content"
+
+func TestMain(m *testing.M) {
+	testS3Endpoint := "localhost:9000"
+	if os.Getenv("TEST_S3_ENDPOINT") != "" {
+		testS3Endpoint = os.Getenv("TEST_S3_ENDPOINT")
+	}
+
+	testS3Driver = S3Driver{
+		S3AccessKey: "test",
+		S3SecretKey: "testtest",
+		S3Endpoint:  testS3Endpoint,
+		S3SSL:       false,
+		Bucket:      "testsnapshots",
+	}
+
+	// Test directory
+	err := os.MkdirAll(testDirectory, 0755)
+	if err != nil {
+		fmt.Printf("Creating directory structure error: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Test source file
+	err = ioutil.WriteFile(testFile, []byte(testContent), 0755)
+	if err != nil {
+		fmt.Printf("Creating test file error: %v\n", err)
+		os.Exit(2)
+	}
+
+	exitVal := m.Run()
+
+	err = os.RemoveAll(testDirectory)
+	if err != nil {
+		fmt.Printf("Removing directory structure error: %v\n", err)
+		os.Exit(1)
+	}
+
+	os.Exit(exitVal)
+}
+
+func TestFSCreate(t *testing.T) {
+	driver := FSDriver{Path: testDirectory}
+
+	err := os.MkdirAll(testDirectory, 0755)
+	assert.Nil(t, err)
+
+	err = driver.Create("testkey.txt", testFile)
+	assert.Nil(t, err)
+
+	body, err := driver.Read("testkey.txt")
+	assert.Nil(t, err)
+
+	assert.Equal(t, []byte(testContent), body)
+}
+
+func TestFSWrite(t *testing.T) {
+	driver := FSDriver{Path: testDirectory}
+
+	err := driver.Write("testkey2", []byte("another test content"))
+	assert.Nil(t, err)
+
+	body, err := driver.Read("testkey2")
+	assert.Nil(t, err)
+
+	assert.Equal(t, []byte("another test content"), body)
+}
+
+func TestFSGet(t *testing.T) {
+	driver := FSDriver{Path: testDirectory}
+
+	err := driver.Get(testKey, testDirectory+"/testdstfile")
+	assert.Nil(t, err)
+
+	body, err := ioutil.ReadFile(testDirectory + "/testdstfile")
+	assert.Nil(t, err)
+
+	assert.Equal(t, []byte(testContent), body)
+}
+
+func TestFSRead(t *testing.T) {
+	driver := FSDriver{Path: testDirectory}
+	body, err := driver.Read(testKey)
+	assert.Nil(t, err)
+	assert.Equal(t, []byte(testContent), body)
+}
+
+func TestFSList(t *testing.T) {
+	driver := FSDriver{Path: testDirectory}
+	keys, err := driver.List("")
+	assert.Nil(t, err)
+	assert.Contains(t, keys, "testkey.txt")
+}
+
+func TestFSDelete(t *testing.T) {
+	driver := FSDriver{Path: testDirectory}
+	err := driver.Write("keytodelete", []byte("non-needed content"))
+	assert.Nil(t, err)
+
+	keys, err := driver.List("")
+	assert.Nil(t, err)
+	assert.Contains(t, keys, "keytodelete")
+
+	driver.Delete("keytodelete")
+
+	keys, err = driver.List("")
+	assert.Nil(t, err)
+	assert.NotContains(t, keys, "keytodelete")
+}

apps/drivers/s3.go

@@ -0,0 +1,157 @@
+package drivers
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+// S3Driver provides basic interface for S3 storage compatible with Driver interface
+type S3Driver struct {
+	// S3 storage related things
+	S3AccessKey string
+	S3SecretKey string
+	S3Endpoint  string
+	S3SSL       bool
+	Bucket      string
+}
+
+// Returns S3 client structure
+func (s *S3Driver) getMinioClient() (*minio.Client, error) {
+	client, err := minio.New(s.S3Endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(s.S3AccessKey, s.S3SecretKey, ""),
+		Secure: s.S3SSL,
+	})
+
+	if err == nil {
+		return client, s.prepareBucket(client)
+	}
+
+	return client, err
+}
+
+// Create configured bucket if it doesn't exist
+func (s *S3Driver) prepareBucket(client *minio.Client) error {
+	bucketInfos, err := client.ListBuckets(context.Background())
+	if err != nil {
+		return err
+	}
+
+	var found bool
+	for _, info := range bucketInfos {
+		if info.Name == s.Bucket {
+			found = true
+			break
+		}
+	}
+
+	if !found {
+		err = client.MakeBucket(context.Background(), s.Bucket, minio.MakeBucketOptions{})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Create creates a new object in the storage with given body
+func (s S3Driver) Create(key string, filePath string) error {
+	client, err := s.getMinioClient()
+	if err != nil {
+		return fmt.Errorf("getting minio client error: %v", err)
+	}
+
+	_, err = client.FPutObject(context.TODO(), s.Bucket, key, filePath, minio.PutObjectOptions{})
+	if err != nil {
+		return fmt.Errorf("copying object into S3 error: %v", err)
+	}
+
+	return nil
+}
+
+// Get copies file from S3 into a local file
+// filePath tells the method where the file should be stored
+func (s S3Driver) Get(key string, filePath string) error {
+	client, err := s.getMinioClient()
+	if err != nil {
+		return fmt.Errorf("getting minio client error: %v", err)
+	}
+
+	err = client.FGetObject(context.TODO(), s.Bucket, key, filePath, minio.GetObjectOptions{})
+	if err != nil {
+		return fmt.Errorf("getting the object from S3 error: %v", err)
+	}
+
+	return nil
+}
+
+// Write writes content of body into key
+func (s S3Driver) Write(key string, body []byte) error {
+	client, err := s.getMinioClient()
+	if err != nil {
+		return fmt.Errorf("getting minio client error: %v", err)
+	}
+
+	_, err = client.PutObject(context.TODO(), s.Bucket, key, bytes.NewReader(body), int64(len(body)), minio.PutObjectOptions{})
+	if err != nil {
+		return fmt.Errorf("getting the object from S3 error: %v", err)
+	}
+
+	return nil
+}
+
+// Read returns content of given key
+func (s S3Driver) Read(key string) ([]byte, error) {
+	client, err := s.getMinioClient()
+	if err != nil {
+		return []byte(""), fmt.Errorf("getting minio client error: %v", err)
+	}
+
+	object, err := client.GetObject(context.TODO(), s.Bucket, key, minio.GetObjectOptions{})
+	if err != nil {
+		return []byte(""), fmt.Errorf("getting the object from S3 error: %v", err)
+	}
+
+	body, err := ioutil.ReadAll(object)
+	if err != nil {
+		return []byte(""), fmt.Errorf("reading the object from S3 error: %v", err)
+	}
+
+	return body, nil
+}
+
+// List returns list of objects in fiven prefix
+func (s S3Driver) List(prefix string) ([]string, error) {
+	keys := []string{}
+
+	client, err := s.getMinioClient()
+	if err != nil {
+		return keys, fmt.Errorf("getting minio client error: %v", err)
+	}
+
+	for info := range client.ListObjects(context.TODO(), s.Bucket, minio.ListObjectsOptions{Prefix: ""}) {
+		keys = append(keys, info.Key)
+	}
+
+	return keys, nil
+}
+
+// Delete deletes one object
+func (s S3Driver) Delete(key string) error {
+	client, err := s.getMinioClient()
+	if err != nil {
+		return fmt.Errorf("getting minio client error: %v", err)
+	}
+
+	err = client.RemoveObject(context.Background(), s.Bucket, key, minio.RemoveObjectOptions{})
+	if err != nil {
+		return fmt.Errorf("removing object error: %v", err)
+	}
+
+	return nil
+}

apps/drivers/s3_test.go

@@ -0,0 +1,79 @@
+package drivers
+
+import (
+	"io/ioutil"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var testS3Driver S3Driver
+
+func TestS3Create(t *testing.T) {
+	err := testS3Driver.Create("testkey", testFile)
+	assert.Nil(t, err)
+
+	body, err := testS3Driver.Read("testkey")
+	assert.Nil(t, err)
+
+	assert.Equal(t, []byte(testContent), body)
+}
+
+func TestS3Write(t *testing.T) {
+	err := testS3Driver.Write("testkey", []byte(testContent))
+	assert.Nil(t, err)
+
+	body, err := testS3Driver.Read("testkey")
+	assert.Nil(t, err)
+
+	assert.Equal(t, []byte(testContent), body)
+}
+
+func TestS3Get(t *testing.T) {
+	err := testS3Driver.Write("testkey", []byte(testContent))
+	assert.Nil(t, err)
+
+	err = testS3Driver.Get("testkey", path.Join(testDirectory, "dsttestfile"))
+	assert.Nil(t, err)
+
+	body, err := ioutil.ReadFile(path.Join(testDirectory, "dsttestfile"))
+	assert.Nil(t, err)
+
+	assert.Equal(t, []byte(testContent), body)
+}
+
+func TestS3Read(t *testing.T) {
+	err := testS3Driver.Write("testkey", []byte(testContent))
+	assert.Nil(t, err)
+
+	body, err := testS3Driver.Read("testkey")
+	assert.Nil(t, err)
+
+	assert.Equal(t, []byte(testContent), body)
+}
+
+func TestS3List(t *testing.T) {
+	err := testS3Driver.Write("testkey", []byte(testContent))
+	assert.Nil(t, err)
+
+	keys, err := testS3Driver.List("")
+	assert.Nil(t, err)
+	assert.Contains(t, keys, "testkey")
+}
+
+func TestS3Delete(t *testing.T) {
+	err := testS3Driver.Write("testkey", []byte(testContent))
+	assert.Nil(t, err)
+
+	keys, err := testS3Driver.List("")
+	assert.Nil(t, err)
+	assert.Contains(t, keys, "testkey")
+
+	err = testS3Driver.Delete("testkey")
+	assert.Nil(t, err)
+
+	keys, err = testS3Driver.List("")
+	assert.Nil(t, err)
+	assert.NotContains(t, keys, "testkey")
+}

apps/drivers/types.go

@@ -0,0 +1,22 @@
+package drivers
+
+// DriverInterface defined methods every storage driver needs to implement
+type DriverInterface interface {
+	// Create creates a new object in the storage from given file name
+	Create(key string, filePath string) error
+
+	// Write writes content of body into key
+	Write(key string, body []byte) error
+
+	// Get returns content of given key
+	Get(key string, filePath string) error
+
+	// Read returns content of given key
+	Read(key string) ([]byte, error)
+
+	// List returns list of objects in fiven prefix
+	List(prefix string) ([]string, error)
+
+	// Delete deletes one object
+	Delete(key string) error
+}

apps/main.go

@@ -1,22 +1,25 @@
 package apps
 
 import (
-	"github.com/rosti-cz/node-api/common"
+	"github.com/jinzhu/gorm"
+	_ "github.com/jinzhu/gorm/dialects/sqlite"
 )
 
-func init() {
+// AppsProcessor encapsulates functions for apps manipulation
-	db := common.GetDBConnection()
+type AppsProcessor struct {
-	db.AutoMigrate(Label{})
+	DB *gorm.DB
-	db.AutoMigrate(App{})
+}
+
+func (a *AppsProcessor) Init() {
+	a.DB.AutoMigrate(Label{})
+	a.DB.AutoMigrate(App{})
 }
 
 // Get returns one app
-func Get(name string) (*App, error) {
+func (a *AppsProcessor) Get(name string) (*App, error) {
 	var app App
 
-	db := common.GetDBConnection()
+	err := a.DB.Preload("Labels").Where("name = ?", name).First(&app).Error
-
-	err := db.Preload("Labels").Where("name = ?", name).First(&app).Error
 	if err != nil {
 		return nil, err
 	}
@@ -25,12 +28,10 @@ func Get(name string) (*App, error) {
 }
 
 // List returns all apps located on this node
-func List() (*Apps, error) {
+func (a *AppsProcessor) List() (*Apps, error) {
 	var apps Apps
 
-	db := common.GetDBConnection()
+	err := a.DB.Preload("Labels").Find(&apps).Error
-
-	err := db.Preload("Labels").Find(&apps).Error
 	if err != nil {
 		return nil, err
 	}
@@ -39,7 +40,7 @@ func List() (*Apps, error) {
 }
 
 // New creates new record about application in the database
-func New(name string, SSHPort int, HTTPPort int, image string, CPU int, memory int) error {
+func (a *AppsProcessor) New(name string, SSHPort int, HTTPPort int, image string, CPU int, memory int) error {
 	app := App{
 		Name:     name,
 		SSHPort:  SSHPort,
@@ -49,8 +50,6 @@ func New(name string, SSHPort int, HTTPPort int, image string, CPU int, memory i
 		Memory:   memory,
 	}
 
-	db := common.GetDBConnection()
-
 	validationErrors := app.Validate()
 	if len(validationErrors) != 0 {
 		return ValidationError{
@@ -58,7 +57,7 @@ func New(name string, SSHPort int, HTTPPort int, image string, CPU int, memory i
 		}
 	}
 
-	if err := db.Create(&app).Error; err != nil {
+	if err := a.DB.Create(&app).Error; err != nil {
 		return err
 	}
 
@@ -66,12 +65,10 @@ func New(name string, SSHPort int, HTTPPort int, image string, CPU int, memory i
 }
 
 // Update changes value about app in the database
-func Update(name string, SSHPort int, HTTPPort int, image string, CPU int, memory int) (*App, error) {
+func (a *AppsProcessor) Update(name string, SSHPort int, HTTPPort int, image string, CPU int, memory int) (*App, error) {
 	var app App
 
-	db := common.GetDBConnection()
+	err := a.DB.Where("name = ?", name).First(&app).Error
-
-	err := db.Where("name = ?", name).First(&app).Error
 	if err != nil {
 		return &app, err
 	}
@@ -107,15 +104,13 @@ func Update(name string, SSHPort int, HTTPPort int, image string, CPU int, memor
 	}
 
 	// Apply the changes
-	err = db.Save(&app).Error
+	err = a.DB.Save(&app).Error
 	return &app, err
 }
 
 // UpdateResources updates various metrics saved in the database
-func UpdateResources(name string, state string, CPUUsage float64, memory int, diskUsageBytes int, diskUsageInodes int) error {
+func (a *AppsProcessor) UpdateResources(name string, state string, CPUUsage float64, memory int, diskUsageBytes int, diskUsageInodes int) error {
-	db := common.GetDBConnection()
+	err := a.DB.Model(&App{}).Where("name = ?", name).Updates(App{
-
-	err := db.Model(&App{}).Where("name = ?", name).Updates(App{
 		State:           state,
 		CPUUsage:        CPUUsage,
 		MemoryUsage:     memory,
@@ -126,48 +121,42 @@ func UpdateResources(name string, state string, CPUUsage float64, memory int, di
 }
 
 // UpdateState sets container's state
-func UpdateState(name string, state string) error {
+func (a *AppsProcessor) UpdateState(name string, state string) error {
-	db := common.GetDBConnection()
+	err := a.DB.Model(&App{}).Where("name = ?", name).Updates(App{
-
-	err := db.Model(&App{}).Where("name = ?", name).Updates(App{
 		State: state,
 	}).Error
 	return err
 }
 
 // Delete removes records about one app from the database
-func Delete(name string) error {
+func (a *AppsProcessor) Delete(name string) error {
-	db := common.GetDBConnection()
+	app, err := a.Get(name)
-
-	app, err := Get(name)
 	if err != nil {
 		return err
 	}
 
-	err = db.Where("app_id = ?", app.ID).Delete(&Label{}).Error
+	err = a.DB.Where("app_id = ?", app.ID).Delete(&Label{}).Error
 	if err != nil {
 		return err
 	}
 
-	err = db.Where("id = ?", app.ID).Delete(App{}).Error
+	err = a.DB.Where("id = ?", app.ID).Delete(App{}).Error
 
 	return err
 }
 
 // AddLabel adds label to existing application
-func AddLabel(appName string, label string) error {
+func (a *AppsProcessor) AddLabel(appName string, label string) error {
 	var app App
 
-	db := common.GetDBConnection()
+	err := a.DB.Where("name = ?", appName).First(&app).Error
-
-	err := db.Where("name = ?", appName).First(&app).Error
 	if err != nil {
 		return err
 	}
 
 	// Check if there is such label already
 	var count int
-	err = db.Model(&Label{}).Where("value = ? AND app_id = ?", label, app.ID).Count(&count).Error
+	err = a.DB.Model(&Label{}).Where("value = ? AND app_id = ?", label, app.ID).Count(&count).Error
 	if err != nil {
 		return err
 	}
@@ -177,19 +166,17 @@ func AddLabel(appName string, label string) error {
 	}
 
 	// And create the label
-	return db.Create(&Label{AppID: app.ID, Value: label}).Error
+	return a.DB.Create(&Label{AppID: app.ID, Value: label}).Error
 }
 
 // RemoveLabel removes label from existing application
-func RemoveLabel(appName string, label string) error {
+func (a *AppsProcessor) RemoveLabel(appName string, label string) error {
 	var app App
 
-	db := common.GetDBConnection()
+	err := a.DB.Where("name = ?", appName).First(&app).Error
-
-	err := db.Where("name = ?", appName).First(&app).Error
 	if err != nil {
 		return err
 	}
 
-	return db.Where("label = ? AND app_id = ?", label, app.ID).Delete(&Label{}).Error
+	return a.DB.Where("label = ? AND app_id = ?", label, app.ID).Delete(&Label{}).Error
 }

apps/snapshots.go

@@ -0,0 +1,271 @@
+package apps
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/mholt/archiver/v3"
+	"github.com/rosti-cz/node-api/apps/drivers"
+)
+
+const bucketName = "snapshots"
+const dateFormat = "20060102_150405"
+const keySplitCharacter = ":"
+
+// Snapshot contains metadata about a single snapshot
+type Snapshot struct {
+	AppName   string
+	TimeStamp int64
+	Labels    []string
+}
+
+// SnapshotIndexLine is struct holding information about a single snapshot
+func (s *Snapshot) ToString() string {
+	// Ignoring this error is intentional. There shouldn't be any problem with this, because all data types are ready to be marshaled.
+	body, _ := json.Marshal(s)
+	return string(body)
+}
+
+// KeyName returns keyname used to store the snapshot in the storage
+func (s *Snapshot) KeyName() string {
+	metadata := base64.StdEncoding.EncodeToString([]byte(s.ToString()))
+	// TODO: this can't be bigger than 1kB
+	return fmt.Sprintf("%s%s%d%s%s", s.AppName, keySplitCharacter, s.TimeStamp, keySplitCharacter, metadata)
+}
+
+// DecodeKeyName returns snapshot structure containing all metadata about a single snapshot
+func DecodeKeyName(key string) (Snapshot, error) {
+	parts := strings.Split(key, keySplitCharacter)
+	if len(parts) != 3 {
+		return Snapshot{}, errors.New("key name in incorrect format")
+	}
+
+	_metadata, err := base64.StdEncoding.DecodeString(parts[2])
+	if len(parts) != 3 {
+		return Snapshot{}, fmt.Errorf("base64 decoding error: %v", err)
+	}
+
+	snapshot := Snapshot{}
+	err = json.Unmarshal(_metadata, &snapshot)
+	if err != nil {
+		return snapshot, fmt.Errorf("metadata unmarshal error: %v", err)
+	}
+
+	return snapshot, nil
+}
+
+type Snapshots []Snapshot
+
+// SnapshotProcessor encapsulates everything realted to snapshots. Snapshot is an archive of app's
+// directory content. It's stored in S3.
+// The biggest problem in the implementation is speed of looking for snapshots by labels.
+// This is distributed interface for the snapshot storage and any node can handle the request message
+// so we don't have any locking mechanism here and we cannot created index of snapshots without
+// significant time spend on it. Let's deal with it later. I think we are fine for first 10k snapshots.
+type SnapshotProcessor struct {
+	AppsPath        string // Where apps are stored
+	TmpSnapshotPath string // where temporary location for snapshots is
+
+	Driver drivers.DriverInterface
+}
+
+// CreateSnapshot creates an archive of existing application and stores it in S3 storage
+// Returns key under which is the snapshot stored and/or error if there is any.
+// Metadata about the snapshot are encoded in the third part of the keyname.
+// The keyname cannot be bigger than 1 kB.
+func (s *SnapshotProcessor) CreateSnapshot(appName string, labels []string) (string, error) {
+	// Create an archive
+	archive := archiver.Zip{
+		CompressionLevel:       6,
+		MkdirAll:               true,
+		SelectiveCompression:   true,
+		ContinueOnError:        false,
+		OverwriteExisting:      false,
+		ImplicitTopLevelFolder: false,
+	}
+
+	snapshot := Snapshot{
+		AppName:   appName,
+		TimeStamp: time.Now().Unix(),
+		Labels:    labels,
+	}
+
+	tmpSnapshotArchivePath := path.Join(s.TmpSnapshotPath, snapshot.KeyName()+".zip")
+
+	err := os.Chdir(path.Join(s.AppsPath, appName))
+	if err != nil {
+		return snapshot.KeyName(), fmt.Errorf("change working directory error: %v", err)
+	}
+
+	err = archive.Archive([]string{"./"}, tmpSnapshotArchivePath)
+	if err != nil {
+		return snapshot.KeyName(), fmt.Errorf("compression error: %v", err)
+	}
+
+	// Clean after myself
+	defer func() {
+		err = os.Remove(tmpSnapshotArchivePath)
+		if err != nil {
+			log.Println("removing temporary snapshot file error:", err.Error())
+		}
+	}()
+
+	// Pipe it into the storage
+	err = s.Driver.Create(snapshot.KeyName(), tmpSnapshotArchivePath)
+	if err != nil {
+		return snapshot.KeyName(), fmt.Errorf("copying snapshot into S3 error: %v", err)
+	}
+
+	return snapshot.KeyName(), nil
+}
+
+// RestoreSnapshot restores snapshot into an existing application
+// If you need a new app from existing snapshot just create it.
+// This restored only content of the disk, doesn't create the container.
+func (s *SnapshotProcessor) RestoreSnapshot(key string, newAppName string) error {
+	tmpSnapshotArchivePath := path.Join(s.TmpSnapshotPath, key+".zip")
+
+	err := os.MkdirAll(path.Join(s.AppsPath, newAppName), 0755)
+	if err != nil {
+		return fmt.Errorf("creating destination path error: %v", err)
+	}
+
+	err = os.Chdir(path.Join(s.AppsPath, newAppName))
+	if err != nil {
+		return fmt.Errorf("creating destination path error: %v", err)
+	}
+
+	s.Driver.Get(key, tmpSnapshotArchivePath)
+	if err != nil {
+		return fmt.Errorf("getting the archive from S3 error: %v", err)
+	}
+
+	archive := archiver.Zip{
+		CompressionLevel:       6,
+		MkdirAll:               true,
+		SelectiveCompression:   true,
+		ContinueOnError:        false,
+		OverwriteExisting:      false,
+		ImplicitTopLevelFolder: false,
+	}
+
+	err = archive.Unarchive(tmpSnapshotArchivePath, "./")
+	if err != nil {
+		return fmt.Errorf("unarchiving error: %v", err)
+	}
+
+	err = os.Remove(tmpSnapshotArchivePath)
+	if err != nil {
+		return fmt.Errorf("removing the archive error: %v", err)
+	}
+
+	return nil
+}
+
+// ListAppSnapshots returns list of all snapshots related to a single application
+func (s *SnapshotProcessor) ListAppSnapshots(appName string) ([]Snapshot, error) {
+	snapshots := []Snapshot{}
+
+	keys, err := s.Driver.List("")
+	if err != nil {
+		return snapshots, err
+	}
+
+	for _, key := range keys {
+		if strings.HasPrefix(key, appName+keySplitCharacter) {
+			snapshot, err := DecodeKeyName(key)
+			if err != nil {
+				return snapshots, err
+			}
+
+			snapshots = append(snapshots, snapshot)
+		}
+	}
+
+	return snapshots, nil
+}
+
+// ListAppsSnapshots returns list of snapshots for all given apps
+func (s *SnapshotProcessor) ListAppsSnapshots(appNames []string) ([]Snapshot, error) {
+	snapshots := []Snapshot{}
+
+	keys, err := s.Driver.List("")
+	if err != nil {
+		return snapshots, err
+	}
+
+	for _, key := range keys {
+		for _, appName := range appNames {
+			if strings.HasPrefix(key, appName+keySplitCharacter) {
+				snapshot, err := DecodeKeyName(key)
+				if err != nil {
+					return snapshots, err
+				}
+
+				snapshots = append(snapshots, snapshot)
+			}
+		}
+	}
+
+	return snapshots, nil
+}
+
+// ListAppsSnapshotsByLabels returns list of snapshots with given label
+// TODO: this will be ok for now but probably little slow when users start using it more
+func (s *SnapshotProcessor) ListAppsSnapshotsByLabels(desiredLabel string) ([]Snapshot, error) {
+	snapshots := []Snapshot{}
+
+	keys, err := s.Driver.List("")
+	if err != nil {
+		return snapshots, err
+	}
+
+	for _, key := range keys {
+		snapshot, err := DecodeKeyName(key)
+		if err != nil {
+			return snapshots, err
+		}
+
+		for _, label := range snapshot.Labels {
+			if label == desiredLabel {
+				snapshots = append(snapshots, snapshot)
+			}
+		}
+	}
+
+	return snapshots, nil
+}
+
+// DeleteSnapshot delete's one snapshot
+func (s *SnapshotProcessor) DeleteSnapshot(key string) error {
+	err := s.Driver.Delete(key)
+	if err != nil {
+		return fmt.Errorf("removing snapshot error: %v", err)
+	}
+
+	return nil
+}
+
+// DeleteAppSnapshots deletes all snapshots related to a single application
+func (s *SnapshotProcessor) DeleteAppSnapshots(appName string) error {
+	snapshots, err := s.ListAppSnapshots(appName)
+	if err != nil {
+		return fmt.Errorf("removing snapshots error: %v", err)
+	}
+
+	for _, snapshot := range snapshots {
+		err = s.DeleteSnapshot(snapshot.KeyName())
+		if err != nil {
+			return fmt.Errorf("removing snapshots error: %v", err)
+		}
+	}
+
+	return nil
+}

apps/snapshots_test.go

@@ -0,0 +1,155 @@
+package apps
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/rosti-cz/node-api/apps/drivers"
+	"github.com/stretchr/testify/assert"
+)
+
+var snapshotProcessor *SnapshotProcessor
+
+var initialPwd string
+var testS3Endpoint string
+
+func TestMain(m *testing.M) {
+	testS3Endpoint = "localhost:9000"
+	if os.Getenv("TEST_S3_ENDPOINT") != "" {
+		testS3Endpoint = os.Getenv("TEST_S3_ENDPOINT")
+	}
+
+	pwd, err := os.Getwd()
+	if err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	}
+
+	toRemovePath := path.Join(pwd, "..", "tmp")
+	initialPwd = path.Join(pwd, "..")
+
+	// Removing temporary test files from previous runs
+	err = os.RemoveAll(toRemovePath)
+	if err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	}
+
+	minioHost := os.Getenv("MINIO_HOST")
+	if minioHost == "" {
+		minioHost = "localhost:9000"
+	}
+
+	snapshotProcessor = &SnapshotProcessor{
+		AppsPath:        path.Join(initialPwd, "tmp/apps"),
+		TmpSnapshotPath: path.Join(initialPwd, "tmp/snapshots"),
+
+		Driver: drivers.S3Driver{
+			S3AccessKey: "test",
+			S3SecretKey: "testtest",
+			S3Endpoint:  testS3Endpoint,
+			S3SSL:       false,
+			Bucket:      "testsnapshots",
+		},
+	}
+
+	err = os.MkdirAll(path.Join(initialPwd, "tmp/apps"), 0755)
+	if err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	}
+
+	err = os.MkdirAll(path.Join(initialPwd, "tmp/snapshots"), 0755)
+	if err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	}
+
+	code := m.Run()
+
+	os.Exit(code)
+}
+
+func TestSnapshot(t *testing.T) {
+	snapshot := Snapshot{
+		AppName:   "app_0102",
+		TimeStamp: 1634510035,
+		Labels:    []string{"userid:1"},
+	}
+
+	assert.Equal(t, "app_0102:1634510035:eyJBcHBOYW1lIjoiYXBwXzAxMDIiLCJUaW1lU3RhbXAiOjE2MzQ1MTAwMzUsIkxhYmVscyI6WyJ1c2VyaWQ6MSJdfQ==", snapshot.KeyName())
+
+	snapshot2, err := DecodeKeyName("app_0102:1634510035:eyJBcHBOYW1lIjoiYXBwXzAxMDIiLCJUaW1lU3RhbXAiOjE2MzQ1MTAwMzUsIkxhYmVscyI6WyJ1c2VyaWQ6MSJdfQ==")
+	assert.Nil(t, err)
+	assert.Equal(t, "app_0102", snapshot2.AppName)
+	assert.Equal(t, int64(1634510035), snapshot2.TimeStamp)
+	assert.Equal(t, "userid:1", snapshot2.Labels[0])
+}
+
+func TestCreateRestoreListSnapshot(t *testing.T) {
+	appName := "app_0101"
+
+	// Create an app structure
+	err := os.MkdirAll(path.Join(snapshotProcessor.AppsPath, appName), 0755)
+	if err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	}
+
+	_, err = exec.Command("bash", "-c", "echo content > "+path.Join(snapshotProcessor.AppsPath, appName)+"/a_file.txt").CombinedOutput()
+	if err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	}
+
+	// Create an snapshot
+	snapshotName, err := snapshotProcessor.CreateSnapshot(appName, []string{"app:test", "almost_no_content"})
+	assert.Nil(t, err)
+	assert.Equal(t, strings.HasPrefix(snapshotName, appName+":"), true)
+
+	// Remove app
+	err = os.RemoveAll(path.Join(snapshotProcessor.AppsPath, appName))
+	assert.Nil(t, err)
+
+	// Check if the file doesn't exist
+	_, err = os.Stat(path.Join(snapshotProcessor.AppsPath, appName, "a_file.txt"))
+	assert.Equal(t, os.IsNotExist(err), true)
+
+	// list snapshots
+	snapshots, err := snapshotProcessor.ListAppSnapshots(appName)
+	assert.Nil(t, err)
+
+	assert.True(t, len(snapshots) >= 1, true, "not snapshots found")
+	var found bool
+	for _, snapshot := range snapshots {
+		if snapshot.AppName == appName {
+			found = true
+		}
+	}
+	assert.True(t, found, "not the right snapshot found")
+
+	// List snapshots for list of apps
+	snapshots, err = snapshotProcessor.ListAppsSnapshots([]string{appName})
+	assert.Nil(t, err)
+
+	assert.True(t, len(snapshots) >= 1, true, "not snapshots found")
+	found = false
+	for _, snapshot := range snapshots {
+		if snapshot.AppName == appName {
+			found = true
+		}
+	}
+	assert.True(t, found, "not the right snapshot found")
+
+	// Restore snapshot
+	err = snapshotProcessor.RestoreSnapshot(snapshotName, appName)
+	assert.Nil(t, err)
+
+	_, err = os.Stat(path.Join(snapshotProcessor.AppsPath, appName, "a_file.txt"))
+	assert.Equal(t, os.IsNotExist(err), false)
+
+}

common/config.go

@@ -8,13 +8,19 @@ import (
 
 // Config keeps info about configuration of this daemon
 type Config struct {
-	Token          string `envconfig:"TOKEN" required:"true"`
+	Token                string `envconfig:"TOKEN" required:"true"`
-	AppsPath       string `envconfig:"APPS_PATH" default:"/srv"`            // Where applications are located
+	AppsPath             string `envconfig:"APPS_PATH" default:"/srv"`            // Where applications are located
-	AppsBindIPHTTP string `envconfig:"APPS_BIND_IP_HTTP" default:"0.0.0.0"` // On what IP apps' HTTP port gonna be bound
+	AppsBindIPHTTP       string `envconfig:"APPS_BIND_IP_HTTP" default:"0.0.0.0"` // On what IP apps' HTTP port gonna be bound
-	AppsBindIPSSH  string `envconfig:"APPS_BIND_IP_SSH" default:"0.0.0.0"`  // On what IP apps' SSH ports gonna be bound
+	AppsBindIPSSH        string `envconfig:"APPS_BIND_IP_SSH" default:"0.0.0.0"`  // On what IP apps' SSH ports gonna be bound
-	NATSURL        string `envconfig:"NATS_URL" required:"true"`
+	NATSURL              string `envconfig:"NATS_URL" required:"true"`
-	NATSAlias      string `envconfig:"NATS_ALIAS" required:"true"` // name/alias of the instance, ex. node-18
+	NATSAlias            string `envconfig:"NATS_ALIAS" required:"true"` // name/alias of the instance, ex. node-18
-	DatabasePath   string `envconfig:"DATABASE_PATH" default:"/var/lib/node-api/rosti.db"`
+	DatabasePath         string `envconfig:"DATABASE_PATH" default:"/var/lib/node-api/rosti.db"`
+	SnapshotsPath        string `envconfig:"SNAPSHOTS_PATH" default:"/mnt/apps/snapshots"` // snapshots path
+	SnapshotsS3AccessKey string `envconfig:"SNAPSHOTS_S3_ACCESS_KEY" required:"false"`
+	SnapshotsS3SecretKey string `envconfig:"SNAPSHOTS_S3_SECRET_KEY" required:"false"`
+	SnapshotsS3Endpoint  string `envconfig:"SNAPSHOTS_S3_ENDPOINT" required:"false"`
+	SnapshotsS3SSL       bool   `envconfig:"SNAPSHOTS_S3_SSL" required:"false" default:"true"`
+	SnapshotsS3Bucket    string `envconfig:"SNAPSHOTS_S3_BUCKET" required:"false" default:"snapshots"`
 }
 
 // GetConfig return configuration created based on environment variables

go.mod

@@ -11,14 +11,18 @@ require (
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/go-ole/go-ole v1.2.5 // indirect
 	github.com/gobuffalo/packr v1.30.1
+	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/jinzhu/gorm v1.9.14
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/labstack/echo v3.3.10+incompatible
 	github.com/labstack/gommon v0.3.0 // indirect
-	github.com/nats-io/nats.go v1.10.0
+	github.com/mholt/archiver/v3 v3.5.0
+	github.com/minio/minio-go/v7 v7.0.14
+	github.com/nats-io/nats-server/v2 v2.6.1 // indirect
+	github.com/nats-io/nats.go v1.12.3
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/pkg/errors v0.9.1
-	github.com/satori/go.uuid v1.2.0
 	github.com/shirou/gopsutil v2.20.6+incompatible
-	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
+	github.com/stretchr/testify v1.4.0
+	google.golang.org/protobuf v1.27.1 // indirect
 )

go.sum

@@ -1,21 +1,11 @@
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
-github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
-github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
-github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
 github.com/Microsoft/go-winio v0.4.18 h1:yjwCO1nhWEShaA5qsmPOBzAOjRCa2PRLsDNZ5yBWXpg=
 github.com/Microsoft/go-winio v0.4.18/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
-github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
-github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
-github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 h1:5sXbqlSomvdjlRbWyNqkPsJ3Fg+tQZCbgeX1VGljbQY=
 github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
-github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4=
+github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -23,9 +13,10 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM=
 github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
@@ -34,35 +25,17 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
+github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
+github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y=
 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w=
-github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
-github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
-github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
-github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y=
-github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
-github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
-github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
 github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY=
 github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
-github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
+github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
-github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
-github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
-github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
-github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
-github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
-github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
-github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o=
-github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
-github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
-github.com/go-openapi/spec v0.19.4 h1:ixzUSnHTd6hCemgtAJgluaTSGYpLNpJY4mA2DIkdOAo=
-github.com/go-openapi/spec v0.19.4/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
-github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
-github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
-github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
-github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/gobuffalo/envy v1.7.0 h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU=
 github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
@@ -72,88 +45,137 @@ github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b
 github.com/gobuffalo/packr v1.30.1 h1:hu1fuVR3fXEZR7rXNW3h8rqSML8EVAf6KNm0NKO/wKg=
 github.com/gobuffalo/packr v1.30.1/go.mod h1:ljMyFO2EcrnzsHsN99cvbq055Y9OhRrIaviy289eRuk=
 github.com/gobuffalo/packr/v2 v2.5.1/go.mod h1:8f9c96ITobJlPzI44jj+4tHnEKNt0xXWSVlXRN9X1Iw=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=
+github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jinzhu/gorm v1.9.14 h1:Kg3ShyTPcM6nzVo148fRrcMO6MNKuqtOUwnzqMgVniM=
 github.com/jinzhu/gorm v1.9.14/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M=
 github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
-github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/karrick/godirwalk v1.10.12/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
 github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
 github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s=
+github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
+github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
+github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A=
+github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/labstack/echo v1.4.4 h1:1bEiBNeGSUKxcPDGfZ/7IgdhJJZx8wV/pICJh4W2NJI=
 github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg=
 github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s=
-github.com/labstack/echo/v4 v4.0.0 h1:q1GH+caIXPP7H2StPIdzy/ez9CO0EepqYeUg6vi9SWM=
-github.com/labstack/echo/v4 v4.0.0/go.mod h1:tZv7nai5buKSg5h/8E6zz4LsD/Dqh9/91Mvs7Z5Zyno=
-github.com/labstack/gommon v0.2.8/go.mod h1:/tj9csK2iPSBvn+3NLM9e52usepMtrd5ilFYA+wQNJ4=
 github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
+github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
-github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
-github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA=
 github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
+github.com/mholt/archiver/v3 v3.5.0 h1:nE8gZIrw66cu4osS/U7UW7YDuGMHssxKutU8IfWxwWE=
+github.com/mholt/archiver/v3 v3.5.0/go.mod h1:qqTTPUK/HZPFgFQ/TJ3BzvTpF/dPtFVJXdQbCmeMxwc=
+github.com/minio/highwayhash v1.0.1 h1:dZ6IIu8Z14VlC0VpfKofAhCy74wu/Qb5gcn52yWoz/0=
+github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
+github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
+github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
+github.com/minio/minio-go/v7 v7.0.14 h1:T7cw8P586gVwEEd0y21kTYtloD576XZgP62N8pE130s=
+github.com/minio/minio-go/v7 v7.0.14/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs=
+github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
+github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI=
+github.com/nats-io/jwt v1.2.2 h1:w3GMTO969dFg+UOKTmmyuu7IGdusK+7Ytlt//OYH/uU=
-github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=
+github.com/nats-io/jwt v1.2.2/go.mod h1:/xX356yQA6LuXI9xWW7mZNpxgF2mBmGecH+Fj34sP5Q=
-github.com/nats-io/nats.go v1.10.0 h1:L8qnKaofSfNFbXg0C5F71LdjPRnmQwSsA4ukmkt1TvY=
+github.com/nats-io/jwt/v2 v2.0.3 h1:i/O6cmIsjpcQyWDYNcq2JyZ3/VTF8SJ4JWluI5OhpvI=
-github.com/nats-io/nats.go v1.10.0/go.mod h1:AjGArbfyR50+afOUotNX2Xs5SYHf+CoOa5HH1eEl2HE=
+github.com/nats-io/jwt/v2 v2.0.3/go.mod h1:VRP+deawSXyhNjXmxPCHskrR6Mq50BqpEI5SEcNiGlY=
-github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
+github.com/nats-io/nats-server/v2 v2.6.1 h1:cJy+ia7/4EaJL+ZYDmIy2rD1mDWTfckhtPBU0GYo8xM=
-github.com/nats-io/nkeys v0.1.4 h1:aEsHIssIk6ETN5m2/MD8Y4B2X7FfXrBAUdkyRvbVYzA=
+github.com/nats-io/nats-server/v2 v2.6.1/go.mod h1:Az91TbZiV7K4a6k/4v6YYdOKEoxCXj+iqhHVf/MlrKo=
-github.com/nats-io/nkeys v0.1.4/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s=
+github.com/nats-io/nats.go v1.12.3 h1:te0GLbRsjtejEkZKKiuk46tbfIn6FfCSv3WWSo1+51E=
+github.com/nats-io/nats.go v1.12.3/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w=
+github.com/nats-io/nkeys v0.2.0/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s=
+github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8=
+github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ=
+github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pierrec/lz4/v4 v4.0.3 h1:vNQKSVZNYUEAvRY9FaUXAF1XPbSOHJtDTiP41kzDz2E=
+github.com/pierrec/lz4/v4 v4.0.3/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
-github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/shirou/gopsutil v2.20.6+incompatible h1:P37G9YH8M4vqkKcwBosp+URN5O8Tay67D2MbR361ioY=
 github.com/shirou/gopsutil v2.20.6+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
-github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
@@ -162,96 +184,85 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/swaggo/echo-swagger v1.0.0 h1:ppQFt6Am3/MHIUmTpZOwi4gggMZ/W9zmKP4Z9ahTe5c=
-github.com/swaggo/echo-swagger v1.0.0/go.mod h1:Vnz3c2TGeFpoZPSV3CkWCrvyfU0016Gq/S0j4JspQnM=
-github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 h1:PyYN9JH5jY9j6av01SpfRMb+1DWg/i3MbGOKPxJ2wjM=
-github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14/go.mod h1:gxQT6pBGRuIGunNf/+tSOB5OHvguWi8Tbt82WOkf35E=
-github.com/swaggo/gin-swagger v1.2.0/go.mod h1:qlH2+W7zXGZkczuL+r2nEBR2JTT+/lX05Nn6vPhc7OI=
-github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+tv8Y=
-github.com/swaggo/swag v1.6.3 h1:N+uVPGP4H2hXoss2pt5dctoSUPKKRInr6qcTMOm0usI=
-github.com/swaggo/swag v1.6.3/go.mod h1:wcc83tB4Mb2aNiL/HP4MFeQdpHUrca+Rp/DRNgWAUio=
-github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
-github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0=
-github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
-github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI=
+github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
-github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4=
+github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4/go.mod h1:50wTf68f99/Zt14pr046Tgt3Lp2vLyFZKzbFXTOabXw=
 github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190130090550-b01c7a725664/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI=
+golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191204025024-5ee1b9f4859a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI=
+golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190611222205-d73e1c7e250b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190624180213-70d37148ca0c/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20191205060818-73c7173a9f7d h1:HjXQhd1u/svlhQb0V71w0I7RKZAI5Vd1lp/4FscZcJ4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
-golang.org/x/tools v0.0.0-20191205060818-73c7173a9f7d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
+gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
-gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
+gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
-gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

handlers.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/labstack/echo"
 	"github.com/rosti-cz/node-api/apps"
+	"github.com/rosti-cz/node-api/common"
 	"github.com/rosti-cz/node-api/docker"
 	"github.com/rosti-cz/node-api/node"
 )
@@ -25,7 +26,10 @@ func listAppsHandler(c echo.Context) error {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
 	}
 
-	applications, err := apps.List()
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	applications, err := processor.List()
 
 	if err != nil {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
@@ -43,7 +47,10 @@ func getAppHandler(c echo.Context) error {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
 	}
 
-	app, err := apps.Get(name)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(name)
 	if err != nil {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
 	}
@@ -62,7 +69,10 @@ func createAppHandler(c echo.Context) error {
 		return c.JSONPretty(http.StatusBadRequest, Message{Message: err.Error()}, JSONIndent)
 	}
 
-	err = apps.New(app.Name, app.SSHPort, app.HTTPPort, app.Image, app.CPU, app.Memory)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	err = processor.New(app.Name, app.SSHPort, app.HTTPPort, app.Image, app.CPU, app.Memory)
 	if err != nil {
 		if validationError, ok := err.(apps.ValidationError); ok {
 			return c.JSONPretty(http.StatusBadRequest, Message{Errors: validationError.Errors}, JSONIndent)
@@ -99,7 +109,10 @@ func updateAppHandler(c echo.Context) error {
 		return c.JSONPretty(http.StatusBadRequest, Message{Message: err.Error()}, JSONIndent)
 	}
 
-	appPointer, err := apps.Update(name, app.SSHPort, app.HTTPPort, app.Image, app.CPU, app.Memory)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	appPointer, err := processor.Update(name, app.SSHPort, app.HTTPPort, app.Image, app.CPU, app.Memory)
 	if err != nil {
 		if validationError, ok := err.(apps.ValidationError); ok {
 			return c.JSONPretty(http.StatusBadRequest, Message{Errors: validationError.Errors}, JSONIndent)
@@ -139,7 +152,10 @@ func updateAppHandler(c echo.Context) error {
 func stopAppHandler(c echo.Context) error {
 	name := c.Param("name")
 
-	app, err := apps.Get(name)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(name)
 	if err != nil {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
 	}
@@ -168,7 +184,10 @@ func stopAppHandler(c echo.Context) error {
 func startAppHandler(c echo.Context) error {
 	name := c.Param("name")
 
-	app, err := apps.Get(name)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(name)
 	if err != nil {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
 	}
@@ -200,7 +219,10 @@ func startAppHandler(c echo.Context) error {
 func restartAppHandler(c echo.Context) error {
 	name := c.Param("name")
 
-	app, err := apps.Get(name)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(name)
 	if err != nil {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
 	}
@@ -227,7 +249,10 @@ func setPasswordHandler(c echo.Context) error {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
 	}
 
-	app, err := apps.Get(name)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(name)
 	if err != nil {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
 	}
@@ -253,7 +278,10 @@ func setKeysHandler(c echo.Context) error {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
 	}
 
-	app, err := apps.Get(name)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(name)
 	if err != nil {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
 	}
@@ -279,7 +307,10 @@ func setServicesHandler(c echo.Context) error {
 		return c.JSONPretty(http.StatusBadRequest, Message{Message: err.Error()}, JSONIndent)
 	}
 
-	app, err := apps.Get(name)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(name)
 	if err != nil {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
 	}
@@ -344,7 +375,10 @@ func setServicesHandler(c echo.Context) error {
 func rebuildAppHandler(c echo.Context) error {
 	name := c.Param("name")
 
-	app, err := apps.Get(name)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(name)
 	if err != nil {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
 	}
@@ -385,7 +419,10 @@ func addLabelHandler(c echo.Context) error {
 		return c.JSONPretty(http.StatusBadRequest, Message{Message: err.Error()}, JSONIndent)
 	}
 
-	err = apps.AddLabel(name, label.Value)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	err = processor.AddLabel(name, label.Value)
 	if err != nil {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
 	}
@@ -403,7 +440,10 @@ func deleteLabelHandler(c echo.Context) error {
 		return c.JSONPretty(http.StatusBadRequest, Message{Message: err.Error()}, JSONIndent)
 	}
 
-	err = apps.RemoveLabel(name, label.Value)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	err = processor.RemoveLabel(name, label.Value)
 	if err != nil {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
 	}
@@ -416,7 +456,10 @@ func deleteLabelHandler(c echo.Context) error {
 func deleteAppHandler(c echo.Context) error {
 	name := c.Param("name")
 
-	app, err := apps.Get(name)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(name)
 	if err != nil {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
 	}
@@ -438,7 +481,7 @@ func deleteAppHandler(c echo.Context) error {
 			}
 		}
 
-		err = apps.Delete(app.Name)
+		err = processor.Delete(app.Name)
 		if err != nil {
 			log.Println("ERROR delete application problem: " + err.Error())
 		}
@@ -466,7 +509,10 @@ func getNodeInfoHandler(c echo.Context) error {
 func getAppProcessesHandler(c echo.Context) error {
 	name := c.Param("name")
 
-	app, err := apps.Get(name)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(name)
 	if err != nil {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
 	}
@@ -506,7 +552,10 @@ func metricsHandler(c echo.Context) error {
 	metrics += fmt.Sprintf("rosti_node_memory_index{hostname=\"%s\"} %f\n", hostname, node.MemoryIndex)
 	metrics += fmt.Sprintf("rosti_node_sold_memory{hostname=\"%s\"} %d\n", hostname, node.SoldMemory)
 
-	apps, err := apps.List()
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	apps, err := processor.List()
 	if err != nil {
 		return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
 	}

handlers_nats.go

@@ -1,13 +1,27 @@
 package main
 
+/*
+Message handling is pretty straightforward. When request is received it's
+usually asynchronic and the client is not waiting for an reply. In this case
+client writes down that something is happening here and when it's done publish()
+function is used to notify everybody that it's done. Client usually listens to
+this channel and it finish his part as it needs.
+
+In case there is a reply we use NATS's responde() method to return the data
+as soon as possible.
+*/
+
 import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"strings"
 
 	"github.com/nats-io/nats.go"
 	"github.com/pkg/errors"
 	"github.com/rosti-cz/node-api/apps"
+	"github.com/rosti-cz/node-api/apps/drivers"
+	"github.com/rosti-cz/node-api/common"
 	"github.com/rosti-cz/node-api/docker"
 	"github.com/rosti-cz/node-api/node"
 )
@@ -27,24 +41,30 @@ func _messageHandler(m *nats.Msg) error {
 	fmt.Printf("Received a message: %v\n", message)
 
 	eventHandlerMap := map[string](func(m *nats.Msg, message *RequestMessage) error){
-		"list":         listEventHandler,
+		"list":                  listEventHandler,
-		"get":          getEventHandler,
+		"get":                   getEventHandler,
-		"create":       createEventHandler,
+		"create":                createEventHandler,
-		"register":     registerEventHandler,
+		"register":              registerEventHandler,
-		"update":       updateEventHandler,
+		"update":                updateEventHandler,
-		"delete":       deleteEventHandler,
+		"delete":                deleteEventHandler,
-		"stop":         stopEventHandler,
+		"stop":                  stopEventHandler,
-		"start":        startEventHandler,
+		"start":                 startEventHandler,
-		"restart":      restartEventHandler,
+		"restart":               restartEventHandler,
-		"update_keys":  updateKeysEventHandler,
+		"update_keys":           updateKeysEventHandler,
-		"set_password": setPasswordEventHandler,
+		"set_password":          setPasswordEventHandler,
-		"processes":    processesEventHandler,
+		"processes":             processesEventHandler,
-		"enable_tech":  enableTechEventHandler,
+		"enable_tech":           enableTechEventHandler,
-		"rebuild":      rebuildEventHandler,
+		"rebuild":               rebuildEventHandler,
-		"add_label":    addLabelEventHandler,
+		"add_label":             addLabelEventHandler,
-		"remove_label": removeLabelEventHandler,
+		"remove_label":          removeLabelEventHandler,
-		"list_orphans": listOrphansEventHandler,
+		"list_orphans":          listOrphansEventHandler,
-		"node":         getNoteEventHandler,
+		"node":                  getNoteEventHandler,
+		"create_snapshot":       createSnapshotEventHandler,
+		"restore_from_snapshot": restoreFromSnapshotEventHandler,
+		"list_snapshots":        listSnapshotsEventHandler,
+		"list_apps_snapshots":   listAppsSnapshotsEventHandler,
+		"delete_snapshot":       deleteSnapshotEventHandler,
+		"delete_app_snapshots":  deleteAppSnapshotsEventHandler,
 	}
 
 	if eventHandler, ok := eventHandlerMap[message.Type]; ok {
@@ -59,8 +79,6 @@ func _messageHandler(m *nats.Msg) error {
 
 	// Enable one of the supported technologies or services (python, node, redis, ...)
 	// Rebuilds existing app, it keeps the data but creates the container again
-	// Adds new label
-	// Removes existing label
 
 	// Orphans returns directories in /srv that doesn't match any hosted application
 	// Return info about the node including performance index
@@ -77,7 +95,11 @@ func listEventHandler(m *nats.Msg, message *RequestMessage) error {
 		return errorReplyFormater(m, "backend error", err)
 	}
 
-	applications, err := apps.List()
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	processor.Init()
+	applications, err := processor.List()
 
 	if err != nil {
 		return errorReplyFormater(m, "backend error", err)
@@ -108,7 +130,10 @@ func getEventHandler(m *nats.Msg, message *RequestMessage) error {
 		return errorReplyFormater(m, "backend error", err)
 	}
 
-	app, err := apps.Get(message.AppName)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(message.AppName)
 	if err != nil {
 		return errorReplyFormater(m, "backend error", err)
 	}
@@ -143,7 +168,10 @@ func createEventHandler(m *nats.Msg, message *RequestMessage) error {
 		return err
 	}
 
-	err = apps.New(message.AppName, appTemplate.SSHPort, appTemplate.HTTPPort, appTemplate.Image, appTemplate.CPU, appTemplate.Memory)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	err = processor.New(message.AppName, appTemplate.SSHPort, appTemplate.HTTPPort, appTemplate.Image, appTemplate.CPU, appTemplate.Memory)
 	if err != nil {
 		if validationError, ok := err.(apps.ValidationError); ok {
 			log.Println("ERROR create application problem: " + validationError.Error())
@@ -190,7 +218,10 @@ func registerEventHandler(m *nats.Msg, message *RequestMessage) error {
 		return err
 	}
 
-	err = apps.New(message.AppName, appTemplate.SSHPort, appTemplate.HTTPPort, appTemplate.Image, appTemplate.CPU, appTemplate.Memory)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	err = processor.New(message.AppName, appTemplate.SSHPort, appTemplate.HTTPPort, appTemplate.Image, appTemplate.CPU, appTemplate.Memory)
 	if err != nil {
 		if validationError, ok := err.(apps.ValidationError); ok {
 			log.Println("ERROR create application problem: " + validationError.Error())
@@ -218,7 +249,10 @@ func updateEventHandler(m *nats.Msg, message *RequestMessage) error {
 		return err
 	}
 
-	app, err := apps.Update(message.AppName, appTemplate.SSHPort, appTemplate.HTTPPort, appTemplate.Image, appTemplate.CPU, appTemplate.Memory)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Update(message.AppName, appTemplate.SSHPort, appTemplate.HTTPPort, appTemplate.Image, appTemplate.CPU, appTemplate.Memory)
 	if err != nil {
 		if validationError, ok := err.(apps.ValidationError); ok {
 			log.Println("ERROR update application problem: " + validationError.Error())
@@ -265,7 +299,10 @@ func updateEventHandler(m *nats.Msg, message *RequestMessage) error {
 
 // Delete one app
 func deleteEventHandler(m *nats.Msg, message *RequestMessage) error {
-	app, err := apps.Get(message.AppName)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(message.AppName)
 	if err != nil {
 		log.Println("ERROR: delete app:", err.Error())
 	}
@@ -299,7 +336,7 @@ func deleteEventHandler(m *nats.Msg, message *RequestMessage) error {
 		}
 	}
 
-	err = apps.Delete(app.Name)
+	err = processor.Delete(app.Name)
 	if err != nil {
 		log.Println("ERROR delete application problem: " + err.Error())
 		publish(app.Name, "backend problem", true)
@@ -313,7 +350,10 @@ func deleteEventHandler(m *nats.Msg, message *RequestMessage) error {
 
 // Stop existing app
 func stopEventHandler(m *nats.Msg, message *RequestMessage) error {
-	app, err := apps.Get(message.AppName)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(message.AppName)
 	if err != nil {
 		log.Println("ERROR stop application problem: " + err.Error())
 		publish(app.Name, "backend problem", true)
@@ -348,7 +388,10 @@ func stopEventHandler(m *nats.Msg, message *RequestMessage) error {
 
 // Start existing app
 func startEventHandler(m *nats.Msg, message *RequestMessage) error {
-	app, err := apps.Get(message.AppName)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(message.AppName)
 	if err != nil {
 		log.Println("ERROR start application problem: " + err.Error())
 		publish(app.Name, "backend problem", true)
@@ -386,7 +429,10 @@ func startEventHandler(m *nats.Msg, message *RequestMessage) error {
 
 // Restart existing app
 func restartEventHandler(m *nats.Msg, message *RequestMessage) error {
-	app, err := apps.Get(message.AppName)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(message.AppName)
 	if err != nil {
 		log.Println("ERROR restart application problem: " + err.Error())
 		publish(app.Name, "backend problem", true)
@@ -420,7 +466,10 @@ func updateKeysEventHandler(m *nats.Msg, message *RequestMessage) error {
 
 	body := message.Payload
 
-	app, err := apps.Get(message.AppName)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(message.AppName)
 	if err != nil {
 		log.Println("ERROR keys update problem: " + err.Error())
 		publish(app.Name, "backend problem", true)
@@ -454,7 +503,10 @@ func setPasswordEventHandler(m *nats.Msg, message *RequestMessage) error {
 
 	password := message.Payload
 
-	app, err := apps.Get(message.AppName)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(message.AppName)
 	if err != nil {
 		log.Println("ERROR password update problem: " + err.Error())
 		publish(app.Name, "backend problem", true)
@@ -479,7 +531,10 @@ func setPasswordEventHandler(m *nats.Msg, message *RequestMessage) error {
 
 // Application processes
 func processesEventHandler(m *nats.Msg, message *RequestMessage) error {
-	app, err := apps.Get(message.AppName)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(message.AppName)
 	if err != nil {
 		log.Println("ERROR processes list problem: " + err.Error())
 		publish(app.Name, "backend problem", true)
@@ -527,7 +582,10 @@ func enableTechEventHandler(m *nats.Msg, message *RequestMessage) error {
 		return err
 	}
 
-	app, err := apps.Get(message.AppName)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(message.AppName)
 	if err != nil {
 		log.Println("ERROR enable tech problem: " + err.Error())
 		publish(app.Name, "backend problem", true)
@@ -552,7 +610,10 @@ func enableTechEventHandler(m *nats.Msg, message *RequestMessage) error {
 
 // Rebuilds existing app, it keeps the data but creates the container again
 func rebuildEventHandler(m *nats.Msg, message *RequestMessage) error {
-	app, err := apps.Get(message.AppName)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(message.AppName)
 	if err != nil {
 		log.Println("ERROR rebuild app problem: " + err.Error())
 		publish(app.Name, "backend problem", true)
@@ -597,7 +658,10 @@ func rebuildEventHandler(m *nats.Msg, message *RequestMessage) error {
 func addLabelEventHandler(m *nats.Msg, message *RequestMessage) error {
 	label := message.Payload
 
-	err := apps.AddLabel(message.AppName, label)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	err := processor.AddLabel(message.AppName, label)
 	if err != nil {
 		log.Println("ERROR add label problem: " + err.Error())
 		publish(message.AppName, "backend problem", true)
@@ -613,7 +677,10 @@ func addLabelEventHandler(m *nats.Msg, message *RequestMessage) error {
 func removeLabelEventHandler(m *nats.Msg, message *RequestMessage) error {
 	label := message.Payload
 
-	err := apps.RemoveLabel(message.AppName, label)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	err := processor.RemoveLabel(message.AppName, label)
 	if err != nil {
 		log.Println("ERROR remove label problem: " + err.Error())
 		publish(message.AppName, "backend problem", true)
@@ -647,7 +714,11 @@ func listOrphansEventHandler(m *nats.Msg, message *RequestMessage) error {
 	return nil
 }
 
-// Return info about the node including performance index
+/*
+getNoteEventHandler returns info about the node including performance index
+
+
+*/
 func getNoteEventHandler(m *nats.Msg, message *RequestMessage) error {
 	node, err := node.GetNodeInfo()
 	if err != nil {
@@ -675,3 +746,276 @@ func getNoteEventHandler(m *nats.Msg, message *RequestMessage) error {
 
 	return nil
 }
+
+/*
+createSnapshotEventHandler create snapshot of given application
+
+Uses appName from the message struct
+
+Payload: no payload needed
+Response: notification when it's done or error
+*/
+func createSnapshotEventHandler(m *nats.Msg, message *RequestMessage) error {
+	processor := apps.SnapshotProcessor{
+		AppsPath:        config.AppsPath,
+		TmpSnapshotPath: config.SnapshotsPath,
+
+		Driver: drivers.S3Driver{
+			S3AccessKey: config.SnapshotsS3AccessKey,
+			S3SecretKey: config.SnapshotsS3SecretKey,
+			S3Endpoint:  config.SnapshotsS3Endpoint,
+			S3SSL:       config.SnapshotsS3SSL,
+			Bucket:      config.SnapshotsS3Bucket,
+		},
+	}
+
+	_, err := processor.CreateSnapshot(message.AppName, strings.Split(message.Payload, ","))
+	if err != nil {
+		log.Println("ERROR create snapshot error: " + err.Error())
+		publish(message.AppName, "backend problem", true)
+		return err
+	}
+
+	publish(message.AppName, "snapshot created", false)
+
+	return nil
+}
+
+/*
+restoreFromSnapshotEventHandler restores app from a snapshot. The app has to exist
+before it can be restored. Any snapshot can be used to restore any application.
+Use labels to store information about what app should be created.
+
+Uses appName from the message struct as the destination app where the data should
+be restored
+
+Payload: string with the snapshot name
+Response: notification when it's done or error
+*/
+func restoreFromSnapshotEventHandler(m *nats.Msg, message *RequestMessage) error {
+	// Setup processors
+	snapshotProcessor := apps.SnapshotProcessor{
+		AppsPath:        config.AppsPath,
+		TmpSnapshotPath: config.SnapshotsPath,
+
+		Driver: drivers.S3Driver{
+			S3AccessKey: config.SnapshotsS3AccessKey,
+			S3SecretKey: config.SnapshotsS3SecretKey,
+			S3Endpoint:  config.SnapshotsS3Endpoint,
+			S3SSL:       config.SnapshotsS3SSL,
+			Bucket:      config.SnapshotsS3Bucket,
+		},
+	}
+
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(message.AppName)
+	if err != nil {
+		log.Println("ERROR restore snapshot problem: " + err.Error())
+		publish(app.Name, "backend problem", true)
+		return err
+	}
+
+	container := docker.Container{
+		App: app,
+	}
+
+	// Stop the container
+	status, err := container.Status()
+	if err != nil {
+		log.Println("ERROR restore snapshot problem: " + err.Error())
+		publish(app.Name, "backend problem", true)
+		return err
+	}
+
+	// Stop the container only when it exists
+	if status != "no-container" {
+		err = container.Stop()
+		if err != nil {
+			log.Println("ERROR restore snapshot problem: " + err.Error())
+			publish(app.Name, "backend problem", true)
+			return err
+		}
+	}
+
+	// Restore the data
+	err = snapshotProcessor.RestoreSnapshot(message.Payload, message.AppName)
+	if err != nil {
+		log.Println("ERROR restore snapshot error: " + err.Error())
+		publish(message.AppName, "backend problem", true)
+		return err
+	}
+
+	// Start the container
+	status, err = container.Status()
+	if err != nil {
+		return err
+	}
+	if status == "no-container" {
+		err = container.Create()
+		if err != nil {
+			log.Println("ERROR start application problem: " + err.Error())
+			publish(app.Name, "backend problem", true)
+			return err
+		}
+	}
+
+	err = container.Start()
+	if err != nil {
+		log.Println("ERROR start application problem: " + err.Error())
+		publish(app.Name, "backend problem", true)
+		return err
+	}
+
+	// notify clients that it's all done
+	publish(message.AppName, "snapshot restored", false)
+
+	return nil
+}
+
+/*
+listSnapshotsEventHandler returns list of snapshots related to a single application
+
+Uses appName from the message
+
+Payload: no payload needed
+Response: replies with list of snapshots or an error message
+*/
+func listSnapshotsEventHandler(m *nats.Msg, message *RequestMessage) error {
+	processor := apps.SnapshotProcessor{
+		AppsPath:        config.AppsPath,
+		TmpSnapshotPath: config.SnapshotsPath,
+
+		Driver: drivers.S3Driver{
+			S3AccessKey: config.SnapshotsS3AccessKey,
+			S3SecretKey: config.SnapshotsS3SecretKey,
+			S3Endpoint:  config.SnapshotsS3Endpoint,
+			S3SSL:       config.SnapshotsS3SSL,
+			Bucket:      config.SnapshotsS3Bucket,
+		},
+	}
+
+	snapshots, err := processor.ListAppSnapshots(message.AppName)
+	if err != nil {
+		return errorReplyFormater(m, "backend error", err)
+	}
+
+	reply := ReplyMessage{
+		Payload: snapshots,
+	}
+
+	data, err := json.Marshal(reply)
+	if err != nil {
+		return errorReplyFormater(m, "reply formatter error", err)
+	}
+
+	err = m.Respond(data)
+	if err != nil {
+		log.Println("ERROR: list of snapshots:", err.Error())
+	}
+	return nil
+}
+
+/*
+listAppsSnapshotsEventHandler returns list of snapshots related to list of application names.
+
+Payload: list of appNames separated by comma (no spaces)
+Response: replies with list of snapshots or an error message
+*/
+func listAppsSnapshotsEventHandler(m *nats.Msg, message *RequestMessage) error {
+	processor := apps.SnapshotProcessor{
+		AppsPath:        config.AppsPath,
+		TmpSnapshotPath: config.SnapshotsPath,
+
+		Driver: drivers.S3Driver{
+			S3AccessKey: config.SnapshotsS3AccessKey,
+			S3SecretKey: config.SnapshotsS3SecretKey,
+			S3Endpoint:  config.SnapshotsS3Endpoint,
+			S3SSL:       config.SnapshotsS3SSL,
+			Bucket:      config.SnapshotsS3Bucket,
+		},
+	}
+
+	snapshots, err := processor.ListAppsSnapshots(strings.Split(message.Payload, ","))
+	if err != nil {
+		return errorReplyFormater(m, "backend error", err)
+	}
+
+	reply := ReplyMessage{
+		Payload: snapshots,
+	}
+
+	data, err := json.Marshal(reply)
+	if err != nil {
+		return errorReplyFormater(m, "reply formatter error", err)
+	}
+
+	err = m.Respond(data)
+	if err != nil {
+		log.Println("ERROR: list of snapshots:", err.Error())
+	}
+	return nil
+}
+
+/*
+deleteSnapshotEventHandler delete a single snapshot. This is not bound to application name.
+
+Payload: string with a snapshot name
+Response: notification when it's done or error
+*/
+func deleteSnapshotEventHandler(m *nats.Msg, message *RequestMessage) error {
+	processor := apps.SnapshotProcessor{
+		AppsPath:        config.AppsPath,
+		TmpSnapshotPath: config.SnapshotsPath,
+
+		Driver: drivers.S3Driver{
+			S3AccessKey: config.SnapshotsS3AccessKey,
+			S3SecretKey: config.SnapshotsS3SecretKey,
+			S3Endpoint:  config.SnapshotsS3Endpoint,
+			S3SSL:       config.SnapshotsS3SSL,
+			Bucket:      config.SnapshotsS3Bucket,
+		},
+	}
+
+	err := processor.DeleteSnapshot(message.Payload)
+	if err != nil {
+		return errorReplyFormater(m, "backend error", err)
+	}
+
+	publish(message.AppName, "snapshot deleted", false)
+
+	return nil
+}
+
+/*
+deleteAppSnapshotsEventHandler deletes all snapshots related to a single application
+
+Uses appName from the message struct
+
+Payload: no payload needed
+Response: notification when it's done or error
+*/
+func deleteAppSnapshotsEventHandler(m *nats.Msg, message *RequestMessage) error {
+	processor := apps.SnapshotProcessor{
+		AppsPath:        config.AppsPath,
+		TmpSnapshotPath: config.SnapshotsPath,
+
+		Driver: drivers.S3Driver{
+			S3AccessKey: config.SnapshotsS3AccessKey,
+			S3SecretKey: config.SnapshotsS3SecretKey,
+			S3Endpoint:  config.SnapshotsS3Endpoint,
+			S3SSL:       config.SnapshotsS3SSL,
+			Bucket:      config.SnapshotsS3Bucket,
+		},
+	}
+
+	err := processor.DeleteAppSnapshots(message.AppName)
+	if err != nil {
+		return errorReplyFormater(m, "backend error", err)
+	}
+
+	publish(message.AppName, "snapshots deleted", false)
+
+	return nil
+}

main.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/labstack/echo"
 	"github.com/nats-io/nats.go"
+	"github.com/rosti-cz/node-api/apps"
 	"github.com/rosti-cz/node-api/common"
 	"github.com/rosti-cz/node-api/node"
 )
@@ -28,6 +29,12 @@ func _init() {
 	if err != nil {
 		log.Fatalln(err)
 	}
+
+	// Database migrations
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	processor.Init()
 }
 
 func main() {
@@ -138,5 +145,13 @@ func main() {
 	// Return info about the node including performance index
 	e.GET("/v1/node", getNodeInfoHandler)
 
-	e.Logger.Fatal(e.Start(":1323"))
+	err := e.Start(":1323")
+
+	// fmt.Println(err.Error())
+	// if strings.Contains(err.Error(), "SIGTERM") {
+	// 	e.Logger.Info(err)
+	// 	os.Exit(0)
+	// }
+
+	e.Logger.Fatal(err)
 }

stats.go

@@ -4,12 +4,17 @@ import (
 	"log"
 
 	"github.com/rosti-cz/node-api/apps"
+	"github.com/rosti-cz/node-api/common"
 	"github.com/rosti-cz/node-api/docker"
 )
 
 // updateUsage updates various resource usage of the container/app in the database
 func updateUsage(name string) error {
-	app, err := apps.Get(name)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+
+	app, err := processor.Get(name)
 	if err != nil {
 		return err
 	}
@@ -23,7 +28,7 @@ func updateUsage(name string) error {
 		return err
 	}
 
-	err = apps.UpdateResources(
+	err = processor.UpdateResources(
 		name,
 		state.State,
 		state.CPUUsage,
@@ -37,7 +42,10 @@ func updateUsage(name string) error {
 
 // Updates only container's state
 func updateState(name string) error {
-	app, err := apps.Get(name)
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	app, err := processor.Get(name)
 	if err != nil {
 		return err
 	}
@@ -50,7 +58,7 @@ func updateState(name string) error {
 		return err
 	}
 
-	err = apps.UpdateState(
+	err = processor.UpdateState(
 		app.Name,
 		state,
 	)
@@ -59,7 +67,10 @@ func updateState(name string) error {
 
 // gatherStats loops over all applications and calls updateUsage to write various metric into the database.
 func gatherStats() error {
-	appList, err := apps.List()
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	appList, err := processor.List()
 	if err != nil {
 		return err
 	}
@@ -76,7 +87,10 @@ func gatherStats() error {
 
 // gatherStates loops over all apps and updates their container state
 func gatherStates() error {
-	appList, err := apps.List()
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+	appList, err := processor.List()
 	if err != nil {
 		return err
 	}

tools.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/nats-io/nats.go"
 	"github.com/rosti-cz/node-api/apps"
+	"github.com/rosti-cz/node-api/common"
 	"github.com/rosti-cz/node-api/docker"
 )
 
@@ -58,6 +59,10 @@ func publish(appName string, state string, isErr bool) {
 // It's used in some async calls that need at least part of the
 // environment prepared.
 func waitForApp(appName string) error {
+	processor := apps.AppsProcessor{
+		DB: common.GetDBConnection(),
+	}
+
 	sleepFor := 5 * time.Second
 	loops := 6
 
@@ -68,7 +73,7 @@ func waitForApp(appName string) error {
 			continue
 		}
 
-		app, err := apps.Get(appName)
+		app, err := processor.Get(appName)
 		if err != nil {
 			return err
 		}