From bc50cb1105090c9625581be9f9a88158cfe4b8cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20=C5=A0trauch?= Date: Sat, 2 Oct 2021 20:00:35 +0200 Subject: [PATCH] Support for snapshots * full implementation of snapshots * tests of the snapshot backend * Drone CI pipeline * New snapshots handlers * Filesystem driver * S3 driver --- .drone.yml | 25 +++ .gitignore | 1 + Makefile | 23 ++- README.md | 5 + apps/drivers/fs.go | 85 ++++++++ apps/drivers/fs_test.go | 123 ++++++++++++ apps/drivers/s3.go | 157 +++++++++++++++ apps/drivers/s3_test.go | 79 ++++++++ apps/drivers/types.go | 22 +++ apps/main.go | 81 ++++---- apps/snapshots.go | 271 ++++++++++++++++++++++++++ apps/snapshots_test.go | 155 +++++++++++++++ common/config.go | 20 +- go.mod | 10 +- go.sum | 255 ++++++++++++------------ handlers.go | 83 ++++++-- handlers_nats.go | 420 ++++++++++++++++++++++++++++++++++++---- main.go | 17 +- stats.go | 26 ++- tools.go | 7 +- 20 files changed, 1621 insertions(+), 244 deletions(-) create mode 100644 .drone.yml create mode 100644 apps/drivers/fs.go create mode 100644 apps/drivers/fs_test.go create mode 100644 apps/drivers/s3.go create mode 100644 apps/drivers/s3_test.go create mode 100644 apps/drivers/types.go create mode 100644 apps/snapshots.go create mode 100644 apps/snapshots_test.go diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..5dceef2 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,25 @@ +kind: pipeline +type: docker +name: testing + +steps: +- name: test + image: golang + environment: + SNAPSHOTS_S3_ENDPOINT: minio:9000 + TEST_S3_ENDPOINT: minio:9000 + commands: + - go mod tidy + - make test + +services: +- name: minio + image: minio/minio:latest + environment: + MINIO_ROOT_USER: test + MINIO_ROOT_PASSWORD: testtest + command: + - server + - /data + - --console-address + - :9001 diff --git a/.gitignore b/.gitignore index 8202cae..f815c3a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ node-api api-node-17.http *-packr.go *.sqlite +tmp/ diff --git a/Makefile b/Makefile index 3e61608..21a2f80 100644 --- a/Makefile +++ b/Makefile @@ -1,2 +1,21 @@ -production-build: - podman run --rm --privileged -ti -v ${shell pwd}:/srv docker.io/library/golang:1.14-stretch /bin/sh -c "cd /srv && go build" +.PHONY: test +test: + go test -v apps/*.go + go test -v apps/drivers/*.go + +build: + #podman run --rm --privileged -ti -v ${shell pwd}:/srv docker.io/library/golang:1.14-stretch /bin/sh -c "cd /srv && go build" + go build -a -ldflags "-linkmode external -extldflags '-static' -s -w" + +.PHONY: minio +minio: + mkdir -p ./tmp/snapshots + -podman stop rosti-snapshots + -podman rm rosti-snapshots + podman run -d --name rosti-snapshots \ + -u 1000 \ + -p 9000:9000 \ + -p 9001:9001 \ + -e MINIO_ROOT_USER=test \ + -e MINIO_ROOT_PASSWORD=testtest \ + minio/minio server /data --console-address ":9001" diff --git a/README.md b/README.md index 8b13789..e171035 100644 --- a/README.md +++ b/README.md @@ -1 +1,6 @@ +# Node API +[![Build Status](https://droneci.ceperka.net/api/badges/Rosti/node-api/status.svg)](https://droneci.ceperka.net/Rosti/node-api) + +Node API is an microservice that runs on node servers. It provides interface between +Docker and the admin site. diff --git a/apps/drivers/fs.go b/apps/drivers/fs.go new file mode 100644 index 0000000..3f441b8 --- /dev/null +++ b/apps/drivers/fs.go @@ -0,0 +1,85 @@ +package drivers + +import ( + "io" + "io/ioutil" + "os" + "path" +) + +type FSDriver struct { + Path string // Where the objects are stored +} + +// Create creates a new object in the storage with given body. +// This just copying the given file into the right location. +func (f FSDriver) Create(key string, filePath string) error { + err := os.MkdirAll(path.Dir(path.Join(f.Path, key)), 0755) + if err != nil { + return err + } + + source, err := os.Open(filePath) + if err != nil { + return err + } + defer source.Close() + + destination, err := os.Create(path.Join(f.Path, key)) + if err != nil { + return err + } + defer destination.Close() + + _, err = io.Copy(destination, source) + return err +} + +// Get copies file from the storage into a local file +func (f FSDriver) Get(key string, filePath string) error { + source, err := os.Open(path.Join(f.Path, key)) + if err != nil { + return err + } + defer source.Close() + + destination, err := os.Create(filePath) + if err != nil { + return err + } + defer destination.Close() + + _, err = io.Copy(destination, source) + return err +} + +// Read returns content of given key +func (f *FSDriver) Read(key string) ([]byte, error) { + body, err := ioutil.ReadFile(path.Join(f.Path, key)) + return body, err +} + +// Write writes content of body into key +func (f FSDriver) Write(key string, body []byte) error { + return ioutil.WriteFile(path.Join(f.Path, key), body, 0644) +} + +// List returns list of objects in fiven prefix +func (f FSDriver) List(prefix string) ([]string, error) { + keys := []string{} + + files, err := ioutil.ReadDir(path.Join(f.Path, prefix)) + if err != nil { + return keys, err + } + for _, file := range files { + keys = append(keys, file.Name()) + } + + return keys, nil +} + +// Delete deletes one object +func (f FSDriver) Delete(key string) error { + return os.Remove(path.Join(f.Path, key)) +} diff --git a/apps/drivers/fs_test.go b/apps/drivers/fs_test.go new file mode 100644 index 0000000..51d1d01 --- /dev/null +++ b/apps/drivers/fs_test.go @@ -0,0 +1,123 @@ +package drivers + +import ( + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +const testDirectory = "./tmp" +const testKey = "sourcefile" +const testFile = testDirectory + "/" + testKey +const testContent = "test content" + +func TestMain(m *testing.M) { + testS3Endpoint := "localhost:9000" + if os.Getenv("TEST_S3_ENDPOINT") != "" { + testS3Endpoint = os.Getenv("TEST_S3_ENDPOINT") + } + + testS3Driver = S3Driver{ + S3AccessKey: "test", + S3SecretKey: "testtest", + S3Endpoint: testS3Endpoint, + S3SSL: false, + Bucket: "testsnapshots", + } + + // Test directory + err := os.MkdirAll(testDirectory, 0755) + if err != nil { + fmt.Printf("Creating directory structure error: %v\n", err) + os.Exit(1) + } + + // Test source file + err = ioutil.WriteFile(testFile, []byte(testContent), 0755) + if err != nil { + fmt.Printf("Creating test file error: %v\n", err) + os.Exit(2) + } + + exitVal := m.Run() + + err = os.RemoveAll(testDirectory) + if err != nil { + fmt.Printf("Removing directory structure error: %v\n", err) + os.Exit(1) + } + + os.Exit(exitVal) +} + +func TestFSCreate(t *testing.T) { + driver := FSDriver{Path: testDirectory} + + err := os.MkdirAll(testDirectory, 0755) + assert.Nil(t, err) + + err = driver.Create("testkey.txt", testFile) + assert.Nil(t, err) + + body, err := driver.Read("testkey.txt") + assert.Nil(t, err) + + assert.Equal(t, []byte(testContent), body) +} + +func TestFSWrite(t *testing.T) { + driver := FSDriver{Path: testDirectory} + + err := driver.Write("testkey2", []byte("another test content")) + assert.Nil(t, err) + + body, err := driver.Read("testkey2") + assert.Nil(t, err) + + assert.Equal(t, []byte("another test content"), body) +} + +func TestFSGet(t *testing.T) { + driver := FSDriver{Path: testDirectory} + + err := driver.Get(testKey, testDirectory+"/testdstfile") + assert.Nil(t, err) + + body, err := ioutil.ReadFile(testDirectory + "/testdstfile") + assert.Nil(t, err) + + assert.Equal(t, []byte(testContent), body) +} + +func TestFSRead(t *testing.T) { + driver := FSDriver{Path: testDirectory} + body, err := driver.Read(testKey) + assert.Nil(t, err) + assert.Equal(t, []byte(testContent), body) +} + +func TestFSList(t *testing.T) { + driver := FSDriver{Path: testDirectory} + keys, err := driver.List("") + assert.Nil(t, err) + assert.Contains(t, keys, "testkey.txt") +} + +func TestFSDelete(t *testing.T) { + driver := FSDriver{Path: testDirectory} + err := driver.Write("keytodelete", []byte("non-needed content")) + assert.Nil(t, err) + + keys, err := driver.List("") + assert.Nil(t, err) + assert.Contains(t, keys, "keytodelete") + + driver.Delete("keytodelete") + + keys, err = driver.List("") + assert.Nil(t, err) + assert.NotContains(t, keys, "keytodelete") +} diff --git a/apps/drivers/s3.go b/apps/drivers/s3.go new file mode 100644 index 0000000..316479f --- /dev/null +++ b/apps/drivers/s3.go @@ -0,0 +1,157 @@ +package drivers + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// S3Driver provides basic interface for S3 storage compatible with Driver interface +type S3Driver struct { + // S3 storage related things + S3AccessKey string + S3SecretKey string + S3Endpoint string + S3SSL bool + Bucket string +} + +// Returns S3 client structure +func (s *S3Driver) getMinioClient() (*minio.Client, error) { + client, err := minio.New(s.S3Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(s.S3AccessKey, s.S3SecretKey, ""), + Secure: s.S3SSL, + }) + + if err == nil { + return client, s.prepareBucket(client) + } + + return client, err +} + +// Create configured bucket if it doesn't exist +func (s *S3Driver) prepareBucket(client *minio.Client) error { + bucketInfos, err := client.ListBuckets(context.Background()) + if err != nil { + return err + } + + var found bool + for _, info := range bucketInfos { + if info.Name == s.Bucket { + found = true + break + } + } + + if !found { + err = client.MakeBucket(context.Background(), s.Bucket, minio.MakeBucketOptions{}) + if err != nil { + return err + } + } + + return nil +} + +// Create creates a new object in the storage with given body +func (s S3Driver) Create(key string, filePath string) error { + client, err := s.getMinioClient() + if err != nil { + return fmt.Errorf("getting minio client error: %v", err) + } + + _, err = client.FPutObject(context.TODO(), s.Bucket, key, filePath, minio.PutObjectOptions{}) + if err != nil { + return fmt.Errorf("copying object into S3 error: %v", err) + } + + return nil +} + +// Get copies file from S3 into a local file +// filePath tells the method where the file should be stored +func (s S3Driver) Get(key string, filePath string) error { + client, err := s.getMinioClient() + if err != nil { + return fmt.Errorf("getting minio client error: %v", err) + } + + err = client.FGetObject(context.TODO(), s.Bucket, key, filePath, minio.GetObjectOptions{}) + if err != nil { + return fmt.Errorf("getting the object from S3 error: %v", err) + } + + return nil +} + +// Write writes content of body into key +func (s S3Driver) Write(key string, body []byte) error { + client, err := s.getMinioClient() + if err != nil { + return fmt.Errorf("getting minio client error: %v", err) + } + + _, err = client.PutObject(context.TODO(), s.Bucket, key, bytes.NewReader(body), int64(len(body)), minio.PutObjectOptions{}) + if err != nil { + return fmt.Errorf("getting the object from S3 error: %v", err) + } + + return nil +} + +// Read returns content of given key +func (s S3Driver) Read(key string) ([]byte, error) { + client, err := s.getMinioClient() + if err != nil { + return []byte(""), fmt.Errorf("getting minio client error: %v", err) + } + + object, err := client.GetObject(context.TODO(), s.Bucket, key, minio.GetObjectOptions{}) + if err != nil { + return []byte(""), fmt.Errorf("getting the object from S3 error: %v", err) + } + + body, err := ioutil.ReadAll(object) + if err != nil { + return []byte(""), fmt.Errorf("reading the object from S3 error: %v", err) + } + + return body, nil +} + +// List returns list of objects in fiven prefix +func (s S3Driver) List(prefix string) ([]string, error) { + keys := []string{} + + client, err := s.getMinioClient() + if err != nil { + return keys, fmt.Errorf("getting minio client error: %v", err) + } + + for info := range client.ListObjects(context.TODO(), s.Bucket, minio.ListObjectsOptions{Prefix: ""}) { + keys = append(keys, info.Key) + } + + return keys, nil +} + +// Delete deletes one object +func (s S3Driver) Delete(key string) error { + client, err := s.getMinioClient() + if err != nil { + return fmt.Errorf("getting minio client error: %v", err) + } + + err = client.RemoveObject(context.Background(), s.Bucket, key, minio.RemoveObjectOptions{}) + if err != nil { + return fmt.Errorf("removing object error: %v", err) + } + + return nil +} diff --git a/apps/drivers/s3_test.go b/apps/drivers/s3_test.go new file mode 100644 index 0000000..46f5344 --- /dev/null +++ b/apps/drivers/s3_test.go @@ -0,0 +1,79 @@ +package drivers + +import ( + "io/ioutil" + "path" + "testing" + + "github.com/stretchr/testify/assert" +) + +var testS3Driver S3Driver + +func TestS3Create(t *testing.T) { + err := testS3Driver.Create("testkey", testFile) + assert.Nil(t, err) + + body, err := testS3Driver.Read("testkey") + assert.Nil(t, err) + + assert.Equal(t, []byte(testContent), body) +} + +func TestS3Write(t *testing.T) { + err := testS3Driver.Write("testkey", []byte(testContent)) + assert.Nil(t, err) + + body, err := testS3Driver.Read("testkey") + assert.Nil(t, err) + + assert.Equal(t, []byte(testContent), body) +} + +func TestS3Get(t *testing.T) { + err := testS3Driver.Write("testkey", []byte(testContent)) + assert.Nil(t, err) + + err = testS3Driver.Get("testkey", path.Join(testDirectory, "dsttestfile")) + assert.Nil(t, err) + + body, err := ioutil.ReadFile(path.Join(testDirectory, "dsttestfile")) + assert.Nil(t, err) + + assert.Equal(t, []byte(testContent), body) +} + +func TestS3Read(t *testing.T) { + err := testS3Driver.Write("testkey", []byte(testContent)) + assert.Nil(t, err) + + body, err := testS3Driver.Read("testkey") + assert.Nil(t, err) + + assert.Equal(t, []byte(testContent), body) +} + +func TestS3List(t *testing.T) { + err := testS3Driver.Write("testkey", []byte(testContent)) + assert.Nil(t, err) + + keys, err := testS3Driver.List("") + assert.Nil(t, err) + assert.Contains(t, keys, "testkey") +} + +func TestS3Delete(t *testing.T) { + err := testS3Driver.Write("testkey", []byte(testContent)) + assert.Nil(t, err) + + keys, err := testS3Driver.List("") + assert.Nil(t, err) + assert.Contains(t, keys, "testkey") + + err = testS3Driver.Delete("testkey") + assert.Nil(t, err) + + keys, err = testS3Driver.List("") + assert.Nil(t, err) + assert.NotContains(t, keys, "testkey") +} diff --git a/apps/drivers/types.go b/apps/drivers/types.go new file mode 100644 index 0000000..a29bea5 --- /dev/null +++ b/apps/drivers/types.go @@ -0,0 +1,22 @@ +package drivers + +// DriverInterface defined methods every storage driver needs to implement +type DriverInterface interface { + // Create creates a new object in the storage from given file name + Create(key string, filePath string) error + + // Write writes content of body into key + Write(key string, body []byte) error + + // Get returns content of given key + Get(key string, filePath string) error + + // Read returns content of given key + Read(key string) ([]byte, error) + + // List returns list of objects in fiven prefix + List(prefix string) ([]string, error) + + // Delete deletes one object + Delete(key string) error +} diff --git a/apps/main.go b/apps/main.go index addddb4..b71b48e 100644 --- a/apps/main.go +++ b/apps/main.go @@ -1,22 +1,25 @@ package apps import ( - "github.com/rosti-cz/node-api/common" + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/sqlite" ) -func init() { - db := common.GetDBConnection() - db.AutoMigrate(Label{}) - db.AutoMigrate(App{}) +// AppsProcessor encapsulates functions for apps manipulation +type AppsProcessor struct { + DB *gorm.DB +} + +func (a *AppsProcessor) Init() { + a.DB.AutoMigrate(Label{}) + a.DB.AutoMigrate(App{}) } // Get returns one app -func Get(name string) (*App, error) { +func (a *AppsProcessor) Get(name string) (*App, error) { var app App - db := common.GetDBConnection() - - err := db.Preload("Labels").Where("name = ?", name).First(&app).Error + err := a.DB.Preload("Labels").Where("name = ?", name).First(&app).Error if err != nil { return nil, err } @@ -25,12 +28,10 @@ func Get(name string) (*App, error) { } // List returns all apps located on this node -func List() (*Apps, error) { +func (a *AppsProcessor) List() (*Apps, error) { var apps Apps - db := common.GetDBConnection() - - err := db.Preload("Labels").Find(&apps).Error + err := a.DB.Preload("Labels").Find(&apps).Error if err != nil { return nil, err } @@ -39,7 +40,7 @@ func List() (*Apps, error) { } // New creates new record about application in the database -func New(name string, SSHPort int, HTTPPort int, image string, CPU int, memory int) error { +func (a *AppsProcessor) New(name string, SSHPort int, HTTPPort int, image string, CPU int, memory int) error { app := App{ Name: name, SSHPort: SSHPort, @@ -49,8 +50,6 @@ func New(name string, SSHPort int, HTTPPort int, image string, CPU int, memory i Memory: memory, } - db := common.GetDBConnection() - validationErrors := app.Validate() if len(validationErrors) != 0 { return ValidationError{ @@ -58,7 +57,7 @@ func New(name string, SSHPort int, HTTPPort int, image string, CPU int, memory i } } - if err := db.Create(&app).Error; err != nil { + if err := a.DB.Create(&app).Error; err != nil { return err } @@ -66,12 +65,10 @@ func New(name string, SSHPort int, HTTPPort int, image string, CPU int, memory i } // Update changes value about app in the database -func Update(name string, SSHPort int, HTTPPort int, image string, CPU int, memory int) (*App, error) { +func (a *AppsProcessor) Update(name string, SSHPort int, HTTPPort int, image string, CPU int, memory int) (*App, error) { var app App - db := common.GetDBConnection() - - err := db.Where("name = ?", name).First(&app).Error + err := a.DB.Where("name = ?", name).First(&app).Error if err != nil { return &app, err } @@ -107,15 +104,13 @@ func Update(name string, SSHPort int, HTTPPort int, image string, CPU int, memor } // Apply the changes - err = db.Save(&app).Error + err = a.DB.Save(&app).Error return &app, err } // UpdateResources updates various metrics saved in the database -func UpdateResources(name string, state string, CPUUsage float64, memory int, diskUsageBytes int, diskUsageInodes int) error { - db := common.GetDBConnection() - - err := db.Model(&App{}).Where("name = ?", name).Updates(App{ +func (a *AppsProcessor) UpdateResources(name string, state string, CPUUsage float64, memory int, diskUsageBytes int, diskUsageInodes int) error { + err := a.DB.Model(&App{}).Where("name = ?", name).Updates(App{ State: state, CPUUsage: CPUUsage, MemoryUsage: memory, @@ -126,48 +121,42 @@ func UpdateResources(name string, state string, CPUUsage float64, memory int, di } // UpdateState sets container's state -func UpdateState(name string, state string) error { - db := common.GetDBConnection() - - err := db.Model(&App{}).Where("name = ?", name).Updates(App{ +func (a *AppsProcessor) UpdateState(name string, state string) error { + err := a.DB.Model(&App{}).Where("name = ?", name).Updates(App{ State: state, }).Error return err } // Delete removes records about one app from the database -func Delete(name string) error { - db := common.GetDBConnection() - - app, err := Get(name) +func (a *AppsProcessor) Delete(name string) error { + app, err := a.Get(name) if err != nil { return err } - err = db.Where("app_id = ?", app.ID).Delete(&Label{}).Error + err = a.DB.Where("app_id = ?", app.ID).Delete(&Label{}).Error if err != nil { return err } - err = db.Where("id = ?", app.ID).Delete(App{}).Error + err = a.DB.Where("id = ?", app.ID).Delete(App{}).Error return err } // AddLabel adds label to existing application -func AddLabel(appName string, label string) error { +func (a *AppsProcessor) AddLabel(appName string, label string) error { var app App - db := common.GetDBConnection() - - err := db.Where("name = ?", appName).First(&app).Error + err := a.DB.Where("name = ?", appName).First(&app).Error if err != nil { return err } // Check if there is such label already var count int - err = db.Model(&Label{}).Where("value = ? AND app_id = ?", label, app.ID).Count(&count).Error + err = a.DB.Model(&Label{}).Where("value = ? AND app_id = ?", label, app.ID).Count(&count).Error if err != nil { return err } @@ -177,19 +166,17 @@ func AddLabel(appName string, label string) error { } // And create the label - return db.Create(&Label{AppID: app.ID, Value: label}).Error + return a.DB.Create(&Label{AppID: app.ID, Value: label}).Error } // RemoveLabel removes label from existing application -func RemoveLabel(appName string, label string) error { +func (a *AppsProcessor) RemoveLabel(appName string, label string) error { var app App - db := common.GetDBConnection() - - err := db.Where("name = ?", appName).First(&app).Error + err := a.DB.Where("name = ?", appName).First(&app).Error if err != nil { return err } - return db.Where("label = ? AND app_id = ?", label, app.ID).Delete(&Label{}).Error + return a.DB.Where("label = ? AND app_id = ?", label, app.ID).Delete(&Label{}).Error } diff --git a/apps/snapshots.go b/apps/snapshots.go new file mode 100644 index 0000000..46d222f --- /dev/null +++ b/apps/snapshots.go @@ -0,0 +1,271 @@ +package apps + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "log" + "os" + "path" + "strings" + "time" + + "github.com/mholt/archiver/v3" + "github.com/rosti-cz/node-api/apps/drivers" +) + +const bucketName = "snapshots" +const dateFormat = "20060102_150405" +const keySplitCharacter = ":" + +// Snapshot contains metadata about a single snapshot +type Snapshot struct { + AppName string + TimeStamp int64 + Labels []string +} + +// SnapshotIndexLine is struct holding information about a single snapshot +func (s *Snapshot) ToString() string { + // Ignoring this error is intentional. There shouldn't be any problem with this, because all data types are ready to be marshaled. + body, _ := json.Marshal(s) + return string(body) +} + +// KeyName returns keyname used to store the snapshot in the storage +func (s *Snapshot) KeyName() string { + metadata := base64.StdEncoding.EncodeToString([]byte(s.ToString())) + // TODO: this can't be bigger than 1kB + return fmt.Sprintf("%s%s%d%s%s", s.AppName, keySplitCharacter, s.TimeStamp, keySplitCharacter, metadata) +} + +// DecodeKeyName returns snapshot structure containing all metadata about a single snapshot +func DecodeKeyName(key string) (Snapshot, error) { + parts := strings.Split(key, keySplitCharacter) + if len(parts) != 3 { + return Snapshot{}, errors.New("key name in incorrect format") + } + + _metadata, err := base64.StdEncoding.DecodeString(parts[2]) + if len(parts) != 3 { + return Snapshot{}, fmt.Errorf("base64 decoding error: %v", err) + } + + snapshot := Snapshot{} + err = json.Unmarshal(_metadata, &snapshot) + if err != nil { + return snapshot, fmt.Errorf("metadata unmarshal error: %v", err) + } + + return snapshot, nil +} + +type Snapshots []Snapshot + +// SnapshotProcessor encapsulates everything realted to snapshots. Snapshot is an archive of app's +// directory content. It's stored in S3. +// The biggest problem in the implementation is speed of looking for snapshots by labels. +// This is distributed interface for the snapshot storage and any node can handle the request message +// so we don't have any locking mechanism here and we cannot created index of snapshots without +// significant time spend on it. Let's deal with it later. I think we are fine for first 10k snapshots. +type SnapshotProcessor struct { + AppsPath string // Where apps are stored + TmpSnapshotPath string // where temporary location for snapshots is + + Driver drivers.DriverInterface +} + +// CreateSnapshot creates an archive of existing application and stores it in S3 storage +// Returns key under which is the snapshot stored and/or error if there is any. +// Metadata about the snapshot are encoded in the third part of the keyname. +// The keyname cannot be bigger than 1 kB. +func (s *SnapshotProcessor) CreateSnapshot(appName string, labels []string) (string, error) { + // Create an archive + archive := archiver.Zip{ + CompressionLevel: 6, + MkdirAll: true, + SelectiveCompression: true, + ContinueOnError: false, + OverwriteExisting: false, + ImplicitTopLevelFolder: false, + } + + snapshot := Snapshot{ + AppName: appName, + TimeStamp: time.Now().Unix(), + Labels: labels, + } + + tmpSnapshotArchivePath := path.Join(s.TmpSnapshotPath, snapshot.KeyName()+".zip") + + err := os.Chdir(path.Join(s.AppsPath, appName)) + if err != nil { + return snapshot.KeyName(), fmt.Errorf("change working directory error: %v", err) + } + + err = archive.Archive([]string{"./"}, tmpSnapshotArchivePath) + if err != nil { + return snapshot.KeyName(), fmt.Errorf("compression error: %v", err) + } + + // Clean after myself + defer func() { + err = os.Remove(tmpSnapshotArchivePath) + if err != nil { + log.Println("removing temporary snapshot file error:", err.Error()) + } + }() + + // Pipe it into the storage + err = s.Driver.Create(snapshot.KeyName(), tmpSnapshotArchivePath) + if err != nil { + return snapshot.KeyName(), fmt.Errorf("copying snapshot into S3 error: %v", err) + } + + return snapshot.KeyName(), nil +} + +// RestoreSnapshot restores snapshot into an existing application +// If you need a new app from existing snapshot just create it. +// This restored only content of the disk, doesn't create the container. +func (s *SnapshotProcessor) RestoreSnapshot(key string, newAppName string) error { + tmpSnapshotArchivePath := path.Join(s.TmpSnapshotPath, key+".zip") + + err := os.MkdirAll(path.Join(s.AppsPath, newAppName), 0755) + if err != nil { + return fmt.Errorf("creating destination path error: %v", err) + } + + err = os.Chdir(path.Join(s.AppsPath, newAppName)) + if err != nil { + return fmt.Errorf("creating destination path error: %v", err) + } + + s.Driver.Get(key, tmpSnapshotArchivePath) + if err != nil { + return fmt.Errorf("getting the archive from S3 error: %v", err) + } + + archive := archiver.Zip{ + CompressionLevel: 6, + MkdirAll: true, + SelectiveCompression: true, + ContinueOnError: false, + OverwriteExisting: false, + ImplicitTopLevelFolder: false, + } + + err = archive.Unarchive(tmpSnapshotArchivePath, "./") + if err != nil { + return fmt.Errorf("unarchiving error: %v", err) + } + + err = os.Remove(tmpSnapshotArchivePath) + if err != nil { + return fmt.Errorf("removing the archive error: %v", err) + } + + return nil +} + +// ListAppSnapshots returns list of all snapshots related to a single application +func (s *SnapshotProcessor) ListAppSnapshots(appName string) ([]Snapshot, error) { + snapshots := []Snapshot{} + + keys, err := s.Driver.List("") + if err != nil { + return snapshots, err + } + + for _, key := range keys { + if strings.HasPrefix(key, appName+keySplitCharacter) { + snapshot, err := DecodeKeyName(key) + if err != nil { + return snapshots, err + } + + snapshots = append(snapshots, snapshot) + } + } + + return snapshots, nil +} + +// ListAppsSnapshots returns list of snapshots for all given apps +func (s *SnapshotProcessor) ListAppsSnapshots(appNames []string) ([]Snapshot, error) { + snapshots := []Snapshot{} + + keys, err := s.Driver.List("") + if err != nil { + return snapshots, err + } + + for _, key := range keys { + for _, appName := range appNames { + if strings.HasPrefix(key, appName+keySplitCharacter) { + snapshot, err := DecodeKeyName(key) + if err != nil { + return snapshots, err + } + + snapshots = append(snapshots, snapshot) + } + } + } + + return snapshots, nil +} + +// ListAppsSnapshotsByLabels returns list of snapshots with given label +// TODO: this will be ok for now but probably little slow when users start using it more +func (s *SnapshotProcessor) ListAppsSnapshotsByLabels(desiredLabel string) ([]Snapshot, error) { + snapshots := []Snapshot{} + + keys, err := s.Driver.List("") + if err != nil { + return snapshots, err + } + + for _, key := range keys { + snapshot, err := DecodeKeyName(key) + if err != nil { + return snapshots, err + } + + for _, label := range snapshot.Labels { + if label == desiredLabel { + snapshots = append(snapshots, snapshot) + } + } + } + + return snapshots, nil +} + +// DeleteSnapshot delete's one snapshot +func (s *SnapshotProcessor) DeleteSnapshot(key string) error { + err := s.Driver.Delete(key) + if err != nil { + return fmt.Errorf("removing snapshot error: %v", err) + } + + return nil +} + +// DeleteAppSnapshots deletes all snapshots related to a single application +func (s *SnapshotProcessor) DeleteAppSnapshots(appName string) error { + snapshots, err := s.ListAppSnapshots(appName) + if err != nil { + return fmt.Errorf("removing snapshots error: %v", err) + } + + for _, snapshot := range snapshots { + err = s.DeleteSnapshot(snapshot.KeyName()) + if err != nil { + return fmt.Errorf("removing snapshots error: %v", err) + } + } + + return nil +} diff --git a/apps/snapshots_test.go b/apps/snapshots_test.go new file mode 100644 index 0000000..ce1cd96 --- /dev/null +++ b/apps/snapshots_test.go @@ -0,0 +1,155 @@ +package apps + +import ( + "fmt" + "os" + "os/exec" + "path" + "strings" + "testing" + + "github.com/rosti-cz/node-api/apps/drivers" + "github.com/stretchr/testify/assert" +) + +var snapshotProcessor *SnapshotProcessor + +var initialPwd string +var testS3Endpoint string + +func TestMain(m *testing.M) { + testS3Endpoint = "localhost:9000" + if os.Getenv("TEST_S3_ENDPOINT") != "" { + testS3Endpoint = os.Getenv("TEST_S3_ENDPOINT") + } + + pwd, err := os.Getwd() + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + toRemovePath := path.Join(pwd, "..", "tmp") + initialPwd = path.Join(pwd, "..") + + // Removing temporary test files from previous runs + err = os.RemoveAll(toRemovePath) + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + minioHost := os.Getenv("MINIO_HOST") + if minioHost == "" { + minioHost = "localhost:9000" + } + + snapshotProcessor = &SnapshotProcessor{ + AppsPath: path.Join(initialPwd, "tmp/apps"), + TmpSnapshotPath: path.Join(initialPwd, "tmp/snapshots"), + + Driver: drivers.S3Driver{ + S3AccessKey: "test", + S3SecretKey: "testtest", + S3Endpoint: testS3Endpoint, + S3SSL: false, + Bucket: "testsnapshots", + }, + } + + err = os.MkdirAll(path.Join(initialPwd, "tmp/apps"), 0755) + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + err = os.MkdirAll(path.Join(initialPwd, "tmp/snapshots"), 0755) + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + code := m.Run() + + os.Exit(code) +} + +func TestSnapshot(t *testing.T) { + snapshot := Snapshot{ + AppName: "app_0102", + TimeStamp: 1634510035, + Labels: []string{"userid:1"}, + } + + assert.Equal(t, "app_0102:1634510035:eyJBcHBOYW1lIjoiYXBwXzAxMDIiLCJUaW1lU3RhbXAiOjE2MzQ1MTAwMzUsIkxhYmVscyI6WyJ1c2VyaWQ6MSJdfQ==", snapshot.KeyName()) + + snapshot2, err := DecodeKeyName("app_0102:1634510035:eyJBcHBOYW1lIjoiYXBwXzAxMDIiLCJUaW1lU3RhbXAiOjE2MzQ1MTAwMzUsIkxhYmVscyI6WyJ1c2VyaWQ6MSJdfQ==") + assert.Nil(t, err) + assert.Equal(t, "app_0102", snapshot2.AppName) + assert.Equal(t, int64(1634510035), snapshot2.TimeStamp) + assert.Equal(t, "userid:1", snapshot2.Labels[0]) +} + +func TestCreateRestoreListSnapshot(t *testing.T) { + appName := "app_0101" + + // Create an app structure + err := os.MkdirAll(path.Join(snapshotProcessor.AppsPath, appName), 0755) + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + _, err = exec.Command("bash", "-c", "echo content > "+path.Join(snapshotProcessor.AppsPath, appName)+"/a_file.txt").CombinedOutput() + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + // Create an snapshot + snapshotName, err := snapshotProcessor.CreateSnapshot(appName, []string{"app:test", "almost_no_content"}) + assert.Nil(t, err) + assert.Equal(t, strings.HasPrefix(snapshotName, appName+":"), true) + + // Remove app + err = os.RemoveAll(path.Join(snapshotProcessor.AppsPath, appName)) + assert.Nil(t, err) + + // Check if the file doesn't exist + _, err = os.Stat(path.Join(snapshotProcessor.AppsPath, appName, "a_file.txt")) + assert.Equal(t, os.IsNotExist(err), true) + + // list snapshots + snapshots, err := snapshotProcessor.ListAppSnapshots(appName) + assert.Nil(t, err) + + assert.True(t, len(snapshots) >= 1, true, "not snapshots found") + var found bool + for _, snapshot := range snapshots { + if snapshot.AppName == appName { + found = true + } + } + assert.True(t, found, "not the right snapshot found") + + // List snapshots for list of apps + snapshots, err = snapshotProcessor.ListAppsSnapshots([]string{appName}) + assert.Nil(t, err) + + assert.True(t, len(snapshots) >= 1, true, "not snapshots found") + found = false + for _, snapshot := range snapshots { + if snapshot.AppName == appName { + found = true + } + } + assert.True(t, found, "not the right snapshot found") + + // Restore snapshot + err = snapshotProcessor.RestoreSnapshot(snapshotName, appName) + assert.Nil(t, err) + + _, err = os.Stat(path.Join(snapshotProcessor.AppsPath, appName, "a_file.txt")) + assert.Equal(t, os.IsNotExist(err), false) + +} diff --git a/common/config.go b/common/config.go index f7eee62..4e8100c 100644 --- a/common/config.go +++ b/common/config.go @@ -8,13 +8,19 @@ import ( // Config keeps info about configuration of this daemon type Config struct { - Token string `envconfig:"TOKEN" required:"true"` - AppsPath string `envconfig:"APPS_PATH" default:"/srv"` // Where applications are located - AppsBindIPHTTP string `envconfig:"APPS_BIND_IP_HTTP" default:"0.0.0.0"` // On what IP apps' HTTP port gonna be bound - AppsBindIPSSH string `envconfig:"APPS_BIND_IP_SSH" default:"0.0.0.0"` // On what IP apps' SSH ports gonna be bound - NATSURL string `envconfig:"NATS_URL" required:"true"` - NATSAlias string `envconfig:"NATS_ALIAS" required:"true"` // name/alias of the instance, ex. node-18 - DatabasePath string `envconfig:"DATABASE_PATH" default:"/var/lib/node-api/rosti.db"` + Token string `envconfig:"TOKEN" required:"true"` + AppsPath string `envconfig:"APPS_PATH" default:"/srv"` // Where applications are located + AppsBindIPHTTP string `envconfig:"APPS_BIND_IP_HTTP" default:"0.0.0.0"` // On what IP apps' HTTP port gonna be bound + AppsBindIPSSH string `envconfig:"APPS_BIND_IP_SSH" default:"0.0.0.0"` // On what IP apps' SSH ports gonna be bound + NATSURL string `envconfig:"NATS_URL" required:"true"` + NATSAlias string `envconfig:"NATS_ALIAS" required:"true"` // name/alias of the instance, ex. node-18 + DatabasePath string `envconfig:"DATABASE_PATH" default:"/var/lib/node-api/rosti.db"` + SnapshotsPath string `envconfig:"SNAPSHOTS_PATH" default:"/mnt/apps/snapshots"` // snapshots path + SnapshotsS3AccessKey string `envconfig:"SNAPSHOTS_S3_ACCESS_KEY" required:"false"` + SnapshotsS3SecretKey string `envconfig:"SNAPSHOTS_S3_SECRET_KEY" required:"false"` + SnapshotsS3Endpoint string `envconfig:"SNAPSHOTS_S3_ENDPOINT" required:"false"` + SnapshotsS3SSL bool `envconfig:"SNAPSHOTS_S3_SSL" required:"false" default:"true"` + SnapshotsS3Bucket string `envconfig:"SNAPSHOTS_S3_BUCKET" required:"false" default:"snapshots"` } // GetConfig return configuration created based on environment variables diff --git a/go.mod b/go.mod index f52d9cc..cd6ff00 100644 --- a/go.mod +++ b/go.mod @@ -11,14 +11,18 @@ require ( github.com/docker/go-units v0.4.0 // indirect github.com/go-ole/go-ole v1.2.5 // indirect github.com/gobuffalo/packr v1.30.1 + github.com/golang/protobuf v1.5.2 // indirect github.com/jinzhu/gorm v1.9.14 github.com/kelseyhightower/envconfig v1.4.0 github.com/labstack/echo v3.3.10+incompatible github.com/labstack/gommon v0.3.0 // indirect - github.com/nats-io/nats.go v1.10.0 + github.com/mholt/archiver/v3 v3.5.0 + github.com/minio/minio-go/v7 v7.0.14 + github.com/nats-io/nats-server/v2 v2.6.1 // indirect + github.com/nats-io/nats.go v1.12.3 github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pkg/errors v0.9.1 - github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil v2.20.6+incompatible - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect + github.com/stretchr/testify v1.4.0 + google.golang.org/protobuf v1.27.1 // indirect ) diff --git a/go.sum b/go.sum index cb6e7b7..8bd5ca7 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,11 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= -github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.18 h1:yjwCO1nhWEShaA5qsmPOBzAOjRCa2PRLsDNZ5yBWXpg= github.com/Microsoft/go-winio v0.4.18/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= -github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 h1:5sXbqlSomvdjlRbWyNqkPsJ3Fg+tQZCbgeX1VGljbQY= github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= +github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -23,9 +13,10 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM= github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo= @@ -34,35 +25,17 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w= -github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= -github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= -github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= -github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= -github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.19.4 h1:ixzUSnHTd6hCemgtAJgluaTSGYpLNpJY4mA2DIkdOAo= -github.com/go-openapi/spec v0.19.4/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/gobuffalo/envy v1.7.0 h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= @@ -72,88 +45,137 @@ github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b github.com/gobuffalo/packr v1.30.1 h1:hu1fuVR3fXEZR7rXNW3h8rqSML8EVAf6KNm0NKO/wKg= github.com/gobuffalo/packr v1.30.1/go.mod h1:ljMyFO2EcrnzsHsN99cvbq055Y9OhRrIaviy289eRuk= github.com/gobuffalo/packr/v2 v2.5.1/go.mod h1:8f9c96ITobJlPzI44jj+4tHnEKNt0xXWSVlXRN9X1Iw= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= -github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jinzhu/gorm v1.9.14 h1:Kg3ShyTPcM6nzVo148fRrcMO6MNKuqtOUwnzqMgVniM= github.com/jinzhu/gorm v1.9.14/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/karrick/godirwalk v1.10.12/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s= +github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A= +github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/labstack/echo v1.4.4 h1:1bEiBNeGSUKxcPDGfZ/7IgdhJJZx8wV/pICJh4W2NJI= github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg= github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s= -github.com/labstack/echo/v4 v4.0.0 h1:q1GH+caIXPP7H2StPIdzy/ez9CO0EepqYeUg6vi9SWM= -github.com/labstack/echo/v4 v4.0.0/go.mod h1:tZv7nai5buKSg5h/8E6zz4LsD/Dqh9/91Mvs7Z5Zyno= -github.com/labstack/gommon v0.2.8/go.mod h1:/tj9csK2iPSBvn+3NLM9e52usepMtrd5ilFYA+wQNJ4= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/mholt/archiver/v3 v3.5.0 h1:nE8gZIrw66cu4osS/U7UW7YDuGMHssxKutU8IfWxwWE= +github.com/mholt/archiver/v3 v3.5.0/go.mod h1:qqTTPUK/HZPFgFQ/TJ3BzvTpF/dPtFVJXdQbCmeMxwc= +github.com/minio/highwayhash v1.0.1 h1:dZ6IIu8Z14VlC0VpfKofAhCy74wu/Qb5gcn52yWoz/0= +github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= +github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= +github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= +github.com/minio/minio-go/v7 v7.0.14 h1:T7cw8P586gVwEEd0y21kTYtloD576XZgP62N8pE130s= +github.com/minio/minio-go/v7 v7.0.14/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats.go v1.10.0 h1:L8qnKaofSfNFbXg0C5F71LdjPRnmQwSsA4ukmkt1TvY= -github.com/nats-io/nats.go v1.10.0/go.mod h1:AjGArbfyR50+afOUotNX2Xs5SYHf+CoOa5HH1eEl2HE= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.4 h1:aEsHIssIk6ETN5m2/MD8Y4B2X7FfXrBAUdkyRvbVYzA= -github.com/nats-io/nkeys v0.1.4/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s= +github.com/nats-io/jwt v1.2.2 h1:w3GMTO969dFg+UOKTmmyuu7IGdusK+7Ytlt//OYH/uU= +github.com/nats-io/jwt v1.2.2/go.mod h1:/xX356yQA6LuXI9xWW7mZNpxgF2mBmGecH+Fj34sP5Q= +github.com/nats-io/jwt/v2 v2.0.3 h1:i/O6cmIsjpcQyWDYNcq2JyZ3/VTF8SJ4JWluI5OhpvI= +github.com/nats-io/jwt/v2 v2.0.3/go.mod h1:VRP+deawSXyhNjXmxPCHskrR6Mq50BqpEI5SEcNiGlY= +github.com/nats-io/nats-server/v2 v2.6.1 h1:cJy+ia7/4EaJL+ZYDmIy2rD1mDWTfckhtPBU0GYo8xM= +github.com/nats-io/nats-server/v2 v2.6.1/go.mod h1:Az91TbZiV7K4a6k/4v6YYdOKEoxCXj+iqhHVf/MlrKo= +github.com/nats-io/nats.go v1.12.3 h1:te0GLbRsjtejEkZKKiuk46tbfIn6FfCSv3WWSo1+51E= +github.com/nats-io/nats.go v1.12.3/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= +github.com/nats-io/nkeys v0.2.0/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s= +github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= +github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pierrec/lz4/v4 v4.0.3 h1:vNQKSVZNYUEAvRY9FaUXAF1XPbSOHJtDTiP41kzDz2E= +github.com/pierrec/lz4/v4 v4.0.3/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shirou/gopsutil v2.20.6+incompatible h1:P37G9YH8M4vqkKcwBosp+URN5O8Tay67D2MbR361ioY= github.com/shirou/gopsutil v2.20.6+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= @@ -162,96 +184,85 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/swaggo/echo-swagger v1.0.0 h1:ppQFt6Am3/MHIUmTpZOwi4gggMZ/W9zmKP4Z9ahTe5c= -github.com/swaggo/echo-swagger v1.0.0/go.mod h1:Vnz3c2TGeFpoZPSV3CkWCrvyfU0016Gq/S0j4JspQnM= -github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 h1:PyYN9JH5jY9j6av01SpfRMb+1DWg/i3MbGOKPxJ2wjM= -github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14/go.mod h1:gxQT6pBGRuIGunNf/+tSOB5OHvguWi8Tbt82WOkf35E= -github.com/swaggo/gin-swagger v1.2.0/go.mod h1:qlH2+W7zXGZkczuL+r2nEBR2JTT+/lX05Nn6vPhc7OI= -github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+tv8Y= -github.com/swaggo/swag v1.6.3 h1:N+uVPGP4H2hXoss2pt5dctoSUPKKRInr6qcTMOm0usI= -github.com/swaggo/swag v1.6.3/go.mod h1:wcc83tB4Mb2aNiL/HP4MFeQdpHUrca+Rp/DRNgWAUio= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0= -github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4= +github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v0.0.0-20170224212429-dcecefd839c4/go.mod h1:50wTf68f99/Zt14pr046Tgt3Lp2vLyFZKzbFXTOabXw= github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190130090550-b01c7a725664/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191204025024-5ee1b9f4859a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190611222205-d73e1c7e250b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190624180213-70d37148ca0c/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20191205060818-73c7173a9f7d h1:HjXQhd1u/svlhQb0V71w0I7RKZAI5Vd1lp/4FscZcJ4= -golang.org/x/tools v0.0.0-20191205060818-73c7173a9f7d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= -gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/handlers.go b/handlers.go index 8231cf5..4e99349 100644 --- a/handlers.go +++ b/handlers.go @@ -9,6 +9,7 @@ import ( "github.com/labstack/echo" "github.com/rosti-cz/node-api/apps" + "github.com/rosti-cz/node-api/common" "github.com/rosti-cz/node-api/docker" "github.com/rosti-cz/node-api/node" ) @@ -25,7 +26,10 @@ func listAppsHandler(c echo.Context) error { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } - applications, err := apps.List() + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + applications, err := processor.List() if err != nil { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) @@ -43,7 +47,10 @@ func getAppHandler(c echo.Context) error { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } - app, err := apps.Get(name) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(name) if err != nil { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } @@ -62,7 +69,10 @@ func createAppHandler(c echo.Context) error { return c.JSONPretty(http.StatusBadRequest, Message{Message: err.Error()}, JSONIndent) } - err = apps.New(app.Name, app.SSHPort, app.HTTPPort, app.Image, app.CPU, app.Memory) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + err = processor.New(app.Name, app.SSHPort, app.HTTPPort, app.Image, app.CPU, app.Memory) if err != nil { if validationError, ok := err.(apps.ValidationError); ok { return c.JSONPretty(http.StatusBadRequest, Message{Errors: validationError.Errors}, JSONIndent) @@ -99,7 +109,10 @@ func updateAppHandler(c echo.Context) error { return c.JSONPretty(http.StatusBadRequest, Message{Message: err.Error()}, JSONIndent) } - appPointer, err := apps.Update(name, app.SSHPort, app.HTTPPort, app.Image, app.CPU, app.Memory) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + appPointer, err := processor.Update(name, app.SSHPort, app.HTTPPort, app.Image, app.CPU, app.Memory) if err != nil { if validationError, ok := err.(apps.ValidationError); ok { return c.JSONPretty(http.StatusBadRequest, Message{Errors: validationError.Errors}, JSONIndent) @@ -139,7 +152,10 @@ func updateAppHandler(c echo.Context) error { func stopAppHandler(c echo.Context) error { name := c.Param("name") - app, err := apps.Get(name) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(name) if err != nil { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } @@ -168,7 +184,10 @@ func stopAppHandler(c echo.Context) error { func startAppHandler(c echo.Context) error { name := c.Param("name") - app, err := apps.Get(name) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(name) if err != nil { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } @@ -200,7 +219,10 @@ func startAppHandler(c echo.Context) error { func restartAppHandler(c echo.Context) error { name := c.Param("name") - app, err := apps.Get(name) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(name) if err != nil { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } @@ -227,7 +249,10 @@ func setPasswordHandler(c echo.Context) error { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } - app, err := apps.Get(name) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(name) if err != nil { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } @@ -253,7 +278,10 @@ func setKeysHandler(c echo.Context) error { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } - app, err := apps.Get(name) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(name) if err != nil { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } @@ -279,7 +307,10 @@ func setServicesHandler(c echo.Context) error { return c.JSONPretty(http.StatusBadRequest, Message{Message: err.Error()}, JSONIndent) } - app, err := apps.Get(name) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(name) if err != nil { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } @@ -344,7 +375,10 @@ func setServicesHandler(c echo.Context) error { func rebuildAppHandler(c echo.Context) error { name := c.Param("name") - app, err := apps.Get(name) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(name) if err != nil { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } @@ -385,7 +419,10 @@ func addLabelHandler(c echo.Context) error { return c.JSONPretty(http.StatusBadRequest, Message{Message: err.Error()}, JSONIndent) } - err = apps.AddLabel(name, label.Value) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + err = processor.AddLabel(name, label.Value) if err != nil { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } @@ -403,7 +440,10 @@ func deleteLabelHandler(c echo.Context) error { return c.JSONPretty(http.StatusBadRequest, Message{Message: err.Error()}, JSONIndent) } - err = apps.RemoveLabel(name, label.Value) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + err = processor.RemoveLabel(name, label.Value) if err != nil { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } @@ -416,7 +456,10 @@ func deleteLabelHandler(c echo.Context) error { func deleteAppHandler(c echo.Context) error { name := c.Param("name") - app, err := apps.Get(name) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(name) if err != nil { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } @@ -438,7 +481,7 @@ func deleteAppHandler(c echo.Context) error { } } - err = apps.Delete(app.Name) + err = processor.Delete(app.Name) if err != nil { log.Println("ERROR delete application problem: " + err.Error()) } @@ -466,7 +509,10 @@ func getNodeInfoHandler(c echo.Context) error { func getAppProcessesHandler(c echo.Context) error { name := c.Param("name") - app, err := apps.Get(name) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(name) if err != nil { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } @@ -506,7 +552,10 @@ func metricsHandler(c echo.Context) error { metrics += fmt.Sprintf("rosti_node_memory_index{hostname=\"%s\"} %f\n", hostname, node.MemoryIndex) metrics += fmt.Sprintf("rosti_node_sold_memory{hostname=\"%s\"} %d\n", hostname, node.SoldMemory) - apps, err := apps.List() + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + apps, err := processor.List() if err != nil { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } diff --git a/handlers_nats.go b/handlers_nats.go index be3a8bd..5ff3433 100644 --- a/handlers_nats.go +++ b/handlers_nats.go @@ -1,13 +1,27 @@ package main +/* +Message handling is pretty straightforward. When request is received it's +usually asynchronic and the client is not waiting for an reply. In this case +client writes down that something is happening here and when it's done publish() +function is used to notify everybody that it's done. Client usually listens to +this channel and it finish his part as it needs. + +In case there is a reply we use NATS's responde() method to return the data +as soon as possible. +*/ + import ( "encoding/json" "fmt" "log" + "strings" "github.com/nats-io/nats.go" "github.com/pkg/errors" "github.com/rosti-cz/node-api/apps" + "github.com/rosti-cz/node-api/apps/drivers" + "github.com/rosti-cz/node-api/common" "github.com/rosti-cz/node-api/docker" "github.com/rosti-cz/node-api/node" ) @@ -27,24 +41,30 @@ func _messageHandler(m *nats.Msg) error { fmt.Printf("Received a message: %v\n", message) eventHandlerMap := map[string](func(m *nats.Msg, message *RequestMessage) error){ - "list": listEventHandler, - "get": getEventHandler, - "create": createEventHandler, - "register": registerEventHandler, - "update": updateEventHandler, - "delete": deleteEventHandler, - "stop": stopEventHandler, - "start": startEventHandler, - "restart": restartEventHandler, - "update_keys": updateKeysEventHandler, - "set_password": setPasswordEventHandler, - "processes": processesEventHandler, - "enable_tech": enableTechEventHandler, - "rebuild": rebuildEventHandler, - "add_label": addLabelEventHandler, - "remove_label": removeLabelEventHandler, - "list_orphans": listOrphansEventHandler, - "node": getNoteEventHandler, + "list": listEventHandler, + "get": getEventHandler, + "create": createEventHandler, + "register": registerEventHandler, + "update": updateEventHandler, + "delete": deleteEventHandler, + "stop": stopEventHandler, + "start": startEventHandler, + "restart": restartEventHandler, + "update_keys": updateKeysEventHandler, + "set_password": setPasswordEventHandler, + "processes": processesEventHandler, + "enable_tech": enableTechEventHandler, + "rebuild": rebuildEventHandler, + "add_label": addLabelEventHandler, + "remove_label": removeLabelEventHandler, + "list_orphans": listOrphansEventHandler, + "node": getNoteEventHandler, + "create_snapshot": createSnapshotEventHandler, + "restore_from_snapshot": restoreFromSnapshotEventHandler, + "list_snapshots": listSnapshotsEventHandler, + "list_apps_snapshots": listAppsSnapshotsEventHandler, + "delete_snapshot": deleteSnapshotEventHandler, + "delete_app_snapshots": deleteAppSnapshotsEventHandler, } if eventHandler, ok := eventHandlerMap[message.Type]; ok { @@ -59,8 +79,6 @@ func _messageHandler(m *nats.Msg) error { // Enable one of the supported technologies or services (python, node, redis, ...) // Rebuilds existing app, it keeps the data but creates the container again - // Adds new label - // Removes existing label // Orphans returns directories in /srv that doesn't match any hosted application // Return info about the node including performance index @@ -77,7 +95,11 @@ func listEventHandler(m *nats.Msg, message *RequestMessage) error { return errorReplyFormater(m, "backend error", err) } - applications, err := apps.List() + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + processor.Init() + applications, err := processor.List() if err != nil { return errorReplyFormater(m, "backend error", err) @@ -108,7 +130,10 @@ func getEventHandler(m *nats.Msg, message *RequestMessage) error { return errorReplyFormater(m, "backend error", err) } - app, err := apps.Get(message.AppName) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(message.AppName) if err != nil { return errorReplyFormater(m, "backend error", err) } @@ -143,7 +168,10 @@ func createEventHandler(m *nats.Msg, message *RequestMessage) error { return err } - err = apps.New(message.AppName, appTemplate.SSHPort, appTemplate.HTTPPort, appTemplate.Image, appTemplate.CPU, appTemplate.Memory) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + err = processor.New(message.AppName, appTemplate.SSHPort, appTemplate.HTTPPort, appTemplate.Image, appTemplate.CPU, appTemplate.Memory) if err != nil { if validationError, ok := err.(apps.ValidationError); ok { log.Println("ERROR create application problem: " + validationError.Error()) @@ -190,7 +218,10 @@ func registerEventHandler(m *nats.Msg, message *RequestMessage) error { return err } - err = apps.New(message.AppName, appTemplate.SSHPort, appTemplate.HTTPPort, appTemplate.Image, appTemplate.CPU, appTemplate.Memory) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + err = processor.New(message.AppName, appTemplate.SSHPort, appTemplate.HTTPPort, appTemplate.Image, appTemplate.CPU, appTemplate.Memory) if err != nil { if validationError, ok := err.(apps.ValidationError); ok { log.Println("ERROR create application problem: " + validationError.Error()) @@ -218,7 +249,10 @@ func updateEventHandler(m *nats.Msg, message *RequestMessage) error { return err } - app, err := apps.Update(message.AppName, appTemplate.SSHPort, appTemplate.HTTPPort, appTemplate.Image, appTemplate.CPU, appTemplate.Memory) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Update(message.AppName, appTemplate.SSHPort, appTemplate.HTTPPort, appTemplate.Image, appTemplate.CPU, appTemplate.Memory) if err != nil { if validationError, ok := err.(apps.ValidationError); ok { log.Println("ERROR update application problem: " + validationError.Error()) @@ -265,7 +299,10 @@ func updateEventHandler(m *nats.Msg, message *RequestMessage) error { // Delete one app func deleteEventHandler(m *nats.Msg, message *RequestMessage) error { - app, err := apps.Get(message.AppName) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(message.AppName) if err != nil { log.Println("ERROR: delete app:", err.Error()) } @@ -299,7 +336,7 @@ func deleteEventHandler(m *nats.Msg, message *RequestMessage) error { } } - err = apps.Delete(app.Name) + err = processor.Delete(app.Name) if err != nil { log.Println("ERROR delete application problem: " + err.Error()) publish(app.Name, "backend problem", true) @@ -313,7 +350,10 @@ func deleteEventHandler(m *nats.Msg, message *RequestMessage) error { // Stop existing app func stopEventHandler(m *nats.Msg, message *RequestMessage) error { - app, err := apps.Get(message.AppName) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(message.AppName) if err != nil { log.Println("ERROR stop application problem: " + err.Error()) publish(app.Name, "backend problem", true) @@ -348,7 +388,10 @@ func stopEventHandler(m *nats.Msg, message *RequestMessage) error { // Start existing app func startEventHandler(m *nats.Msg, message *RequestMessage) error { - app, err := apps.Get(message.AppName) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(message.AppName) if err != nil { log.Println("ERROR start application problem: " + err.Error()) publish(app.Name, "backend problem", true) @@ -386,7 +429,10 @@ func startEventHandler(m *nats.Msg, message *RequestMessage) error { // Restart existing app func restartEventHandler(m *nats.Msg, message *RequestMessage) error { - app, err := apps.Get(message.AppName) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(message.AppName) if err != nil { log.Println("ERROR restart application problem: " + err.Error()) publish(app.Name, "backend problem", true) @@ -420,7 +466,10 @@ func updateKeysEventHandler(m *nats.Msg, message *RequestMessage) error { body := message.Payload - app, err := apps.Get(message.AppName) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(message.AppName) if err != nil { log.Println("ERROR keys update problem: " + err.Error()) publish(app.Name, "backend problem", true) @@ -454,7 +503,10 @@ func setPasswordEventHandler(m *nats.Msg, message *RequestMessage) error { password := message.Payload - app, err := apps.Get(message.AppName) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(message.AppName) if err != nil { log.Println("ERROR password update problem: " + err.Error()) publish(app.Name, "backend problem", true) @@ -479,7 +531,10 @@ func setPasswordEventHandler(m *nats.Msg, message *RequestMessage) error { // Application processes func processesEventHandler(m *nats.Msg, message *RequestMessage) error { - app, err := apps.Get(message.AppName) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(message.AppName) if err != nil { log.Println("ERROR processes list problem: " + err.Error()) publish(app.Name, "backend problem", true) @@ -527,7 +582,10 @@ func enableTechEventHandler(m *nats.Msg, message *RequestMessage) error { return err } - app, err := apps.Get(message.AppName) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(message.AppName) if err != nil { log.Println("ERROR enable tech problem: " + err.Error()) publish(app.Name, "backend problem", true) @@ -552,7 +610,10 @@ func enableTechEventHandler(m *nats.Msg, message *RequestMessage) error { // Rebuilds existing app, it keeps the data but creates the container again func rebuildEventHandler(m *nats.Msg, message *RequestMessage) error { - app, err := apps.Get(message.AppName) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(message.AppName) if err != nil { log.Println("ERROR rebuild app problem: " + err.Error()) publish(app.Name, "backend problem", true) @@ -597,7 +658,10 @@ func rebuildEventHandler(m *nats.Msg, message *RequestMessage) error { func addLabelEventHandler(m *nats.Msg, message *RequestMessage) error { label := message.Payload - err := apps.AddLabel(message.AppName, label) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + err := processor.AddLabel(message.AppName, label) if err != nil { log.Println("ERROR add label problem: " + err.Error()) publish(message.AppName, "backend problem", true) @@ -613,7 +677,10 @@ func addLabelEventHandler(m *nats.Msg, message *RequestMessage) error { func removeLabelEventHandler(m *nats.Msg, message *RequestMessage) error { label := message.Payload - err := apps.RemoveLabel(message.AppName, label) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + err := processor.RemoveLabel(message.AppName, label) if err != nil { log.Println("ERROR remove label problem: " + err.Error()) publish(message.AppName, "backend problem", true) @@ -647,7 +714,11 @@ func listOrphansEventHandler(m *nats.Msg, message *RequestMessage) error { return nil } -// Return info about the node including performance index +/* +getNoteEventHandler returns info about the node including performance index + + +*/ func getNoteEventHandler(m *nats.Msg, message *RequestMessage) error { node, err := node.GetNodeInfo() if err != nil { @@ -675,3 +746,276 @@ func getNoteEventHandler(m *nats.Msg, message *RequestMessage) error { return nil } + +/* +createSnapshotEventHandler create snapshot of given application + +Uses appName from the message struct + +Payload: no payload needed +Response: notification when it's done or error +*/ +func createSnapshotEventHandler(m *nats.Msg, message *RequestMessage) error { + processor := apps.SnapshotProcessor{ + AppsPath: config.AppsPath, + TmpSnapshotPath: config.SnapshotsPath, + + Driver: drivers.S3Driver{ + S3AccessKey: config.SnapshotsS3AccessKey, + S3SecretKey: config.SnapshotsS3SecretKey, + S3Endpoint: config.SnapshotsS3Endpoint, + S3SSL: config.SnapshotsS3SSL, + Bucket: config.SnapshotsS3Bucket, + }, + } + + _, err := processor.CreateSnapshot(message.AppName, strings.Split(message.Payload, ",")) + if err != nil { + log.Println("ERROR create snapshot error: " + err.Error()) + publish(message.AppName, "backend problem", true) + return err + } + + publish(message.AppName, "snapshot created", false) + + return nil +} + +/* +restoreFromSnapshotEventHandler restores app from a snapshot. The app has to exist +before it can be restored. Any snapshot can be used to restore any application. +Use labels to store information about what app should be created. + +Uses appName from the message struct as the destination app where the data should +be restored + +Payload: string with the snapshot name +Response: notification when it's done or error +*/ +func restoreFromSnapshotEventHandler(m *nats.Msg, message *RequestMessage) error { + // Setup processors + snapshotProcessor := apps.SnapshotProcessor{ + AppsPath: config.AppsPath, + TmpSnapshotPath: config.SnapshotsPath, + + Driver: drivers.S3Driver{ + S3AccessKey: config.SnapshotsS3AccessKey, + S3SecretKey: config.SnapshotsS3SecretKey, + S3Endpoint: config.SnapshotsS3Endpoint, + S3SSL: config.SnapshotsS3SSL, + Bucket: config.SnapshotsS3Bucket, + }, + } + + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(message.AppName) + if err != nil { + log.Println("ERROR restore snapshot problem: " + err.Error()) + publish(app.Name, "backend problem", true) + return err + } + + container := docker.Container{ + App: app, + } + + // Stop the container + status, err := container.Status() + if err != nil { + log.Println("ERROR restore snapshot problem: " + err.Error()) + publish(app.Name, "backend problem", true) + return err + } + + // Stop the container only when it exists + if status != "no-container" { + err = container.Stop() + if err != nil { + log.Println("ERROR restore snapshot problem: " + err.Error()) + publish(app.Name, "backend problem", true) + return err + } + } + + // Restore the data + err = snapshotProcessor.RestoreSnapshot(message.Payload, message.AppName) + if err != nil { + log.Println("ERROR restore snapshot error: " + err.Error()) + publish(message.AppName, "backend problem", true) + return err + } + + // Start the container + status, err = container.Status() + if err != nil { + return err + } + if status == "no-container" { + err = container.Create() + if err != nil { + log.Println("ERROR start application problem: " + err.Error()) + publish(app.Name, "backend problem", true) + return err + } + } + + err = container.Start() + if err != nil { + log.Println("ERROR start application problem: " + err.Error()) + publish(app.Name, "backend problem", true) + return err + } + + // notify clients that it's all done + publish(message.AppName, "snapshot restored", false) + + return nil +} + +/* +listSnapshotsEventHandler returns list of snapshots related to a single application + +Uses appName from the message + +Payload: no payload needed +Response: replies with list of snapshots or an error message +*/ +func listSnapshotsEventHandler(m *nats.Msg, message *RequestMessage) error { + processor := apps.SnapshotProcessor{ + AppsPath: config.AppsPath, + TmpSnapshotPath: config.SnapshotsPath, + + Driver: drivers.S3Driver{ + S3AccessKey: config.SnapshotsS3AccessKey, + S3SecretKey: config.SnapshotsS3SecretKey, + S3Endpoint: config.SnapshotsS3Endpoint, + S3SSL: config.SnapshotsS3SSL, + Bucket: config.SnapshotsS3Bucket, + }, + } + + snapshots, err := processor.ListAppSnapshots(message.AppName) + if err != nil { + return errorReplyFormater(m, "backend error", err) + } + + reply := ReplyMessage{ + Payload: snapshots, + } + + data, err := json.Marshal(reply) + if err != nil { + return errorReplyFormater(m, "reply formatter error", err) + } + + err = m.Respond(data) + if err != nil { + log.Println("ERROR: list of snapshots:", err.Error()) + } + return nil +} + +/* +listAppsSnapshotsEventHandler returns list of snapshots related to list of application names. + +Payload: list of appNames separated by comma (no spaces) +Response: replies with list of snapshots or an error message +*/ +func listAppsSnapshotsEventHandler(m *nats.Msg, message *RequestMessage) error { + processor := apps.SnapshotProcessor{ + AppsPath: config.AppsPath, + TmpSnapshotPath: config.SnapshotsPath, + + Driver: drivers.S3Driver{ + S3AccessKey: config.SnapshotsS3AccessKey, + S3SecretKey: config.SnapshotsS3SecretKey, + S3Endpoint: config.SnapshotsS3Endpoint, + S3SSL: config.SnapshotsS3SSL, + Bucket: config.SnapshotsS3Bucket, + }, + } + + snapshots, err := processor.ListAppsSnapshots(strings.Split(message.Payload, ",")) + if err != nil { + return errorReplyFormater(m, "backend error", err) + } + + reply := ReplyMessage{ + Payload: snapshots, + } + + data, err := json.Marshal(reply) + if err != nil { + return errorReplyFormater(m, "reply formatter error", err) + } + + err = m.Respond(data) + if err != nil { + log.Println("ERROR: list of snapshots:", err.Error()) + } + return nil +} + +/* +deleteSnapshotEventHandler delete a single snapshot. This is not bound to application name. + +Payload: string with a snapshot name +Response: notification when it's done or error +*/ +func deleteSnapshotEventHandler(m *nats.Msg, message *RequestMessage) error { + processor := apps.SnapshotProcessor{ + AppsPath: config.AppsPath, + TmpSnapshotPath: config.SnapshotsPath, + + Driver: drivers.S3Driver{ + S3AccessKey: config.SnapshotsS3AccessKey, + S3SecretKey: config.SnapshotsS3SecretKey, + S3Endpoint: config.SnapshotsS3Endpoint, + S3SSL: config.SnapshotsS3SSL, + Bucket: config.SnapshotsS3Bucket, + }, + } + + err := processor.DeleteSnapshot(message.Payload) + if err != nil { + return errorReplyFormater(m, "backend error", err) + } + + publish(message.AppName, "snapshot deleted", false) + + return nil +} + +/* +deleteAppSnapshotsEventHandler deletes all snapshots related to a single application + +Uses appName from the message struct + +Payload: no payload needed +Response: notification when it's done or error +*/ +func deleteAppSnapshotsEventHandler(m *nats.Msg, message *RequestMessage) error { + processor := apps.SnapshotProcessor{ + AppsPath: config.AppsPath, + TmpSnapshotPath: config.SnapshotsPath, + + Driver: drivers.S3Driver{ + S3AccessKey: config.SnapshotsS3AccessKey, + S3SecretKey: config.SnapshotsS3SecretKey, + S3Endpoint: config.SnapshotsS3Endpoint, + S3SSL: config.SnapshotsS3SSL, + Bucket: config.SnapshotsS3Bucket, + }, + } + + err := processor.DeleteAppSnapshots(message.AppName) + if err != nil { + return errorReplyFormater(m, "backend error", err) + } + + publish(message.AppName, "snapshots deleted", false) + + return nil +} diff --git a/main.go b/main.go index a9f73af..6556043 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "github.com/labstack/echo" "github.com/nats-io/nats.go" + "github.com/rosti-cz/node-api/apps" "github.com/rosti-cz/node-api/common" "github.com/rosti-cz/node-api/node" ) @@ -28,6 +29,12 @@ func _init() { if err != nil { log.Fatalln(err) } + + // Database migrations + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + processor.Init() } func main() { @@ -138,5 +145,13 @@ func main() { // Return info about the node including performance index e.GET("/v1/node", getNodeInfoHandler) - e.Logger.Fatal(e.Start(":1323")) + err := e.Start(":1323") + + // fmt.Println(err.Error()) + // if strings.Contains(err.Error(), "SIGTERM") { + // e.Logger.Info(err) + // os.Exit(0) + // } + + e.Logger.Fatal(err) } diff --git a/stats.go b/stats.go index fd87488..227cebd 100644 --- a/stats.go +++ b/stats.go @@ -4,12 +4,17 @@ import ( "log" "github.com/rosti-cz/node-api/apps" + "github.com/rosti-cz/node-api/common" "github.com/rosti-cz/node-api/docker" ) // updateUsage updates various resource usage of the container/app in the database func updateUsage(name string) error { - app, err := apps.Get(name) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + + app, err := processor.Get(name) if err != nil { return err } @@ -23,7 +28,7 @@ func updateUsage(name string) error { return err } - err = apps.UpdateResources( + err = processor.UpdateResources( name, state.State, state.CPUUsage, @@ -37,7 +42,10 @@ func updateUsage(name string) error { // Updates only container's state func updateState(name string) error { - app, err := apps.Get(name) + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + app, err := processor.Get(name) if err != nil { return err } @@ -50,7 +58,7 @@ func updateState(name string) error { return err } - err = apps.UpdateState( + err = processor.UpdateState( app.Name, state, ) @@ -59,7 +67,10 @@ func updateState(name string) error { // gatherStats loops over all applications and calls updateUsage to write various metric into the database. func gatherStats() error { - appList, err := apps.List() + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + appList, err := processor.List() if err != nil { return err } @@ -76,7 +87,10 @@ func gatherStats() error { // gatherStates loops over all apps and updates their container state func gatherStates() error { - appList, err := apps.List() + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + appList, err := processor.List() if err != nil { return err } diff --git a/tools.go b/tools.go index eaffebd..893c134 100644 --- a/tools.go +++ b/tools.go @@ -9,6 +9,7 @@ import ( "github.com/nats-io/nats.go" "github.com/rosti-cz/node-api/apps" + "github.com/rosti-cz/node-api/common" "github.com/rosti-cz/node-api/docker" ) @@ -58,6 +59,10 @@ func publish(appName string, state string, isErr bool) { // It's used in some async calls that need at least part of the // environment prepared. func waitForApp(appName string) error { + processor := apps.AppsProcessor{ + DB: common.GetDBConnection(), + } + sleepFor := 5 * time.Second loops := 6 @@ -68,7 +73,7 @@ func waitForApp(appName string) error { continue } - app, err := apps.Get(appName) + app, err := processor.Get(appName) if err != nil { return err }