node-api/apps/snapshots.go
Adam Štrauch bc50cb1105
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/pr Build is passing
Support for snapshots
* full implementation of snapshots
* tests of the snapshot backend
* Drone CI pipeline
* New snapshots handlers
* Filesystem driver
* S3 driver
2021-10-26 18:57:52 +02:00

272 lines
7.4 KiB
Go

package apps
import (
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"log"
"os"
"path"
"strings"
"time"
"github.com/mholt/archiver/v3"
"github.com/rosti-cz/node-api/apps/drivers"
)
const bucketName = "snapshots"
const dateFormat = "20060102_150405"
const keySplitCharacter = ":"
// Snapshot contains metadata about a single snapshot
type Snapshot struct {
AppName string
TimeStamp int64
Labels []string
}
// SnapshotIndexLine is struct holding information about a single snapshot
func (s *Snapshot) ToString() string {
// Ignoring this error is intentional. There shouldn't be any problem with this, because all data types are ready to be marshaled.
body, _ := json.Marshal(s)
return string(body)
}
// KeyName returns keyname used to store the snapshot in the storage
func (s *Snapshot) KeyName() string {
metadata := base64.StdEncoding.EncodeToString([]byte(s.ToString()))
// TODO: this can't be bigger than 1kB
return fmt.Sprintf("%s%s%d%s%s", s.AppName, keySplitCharacter, s.TimeStamp, keySplitCharacter, metadata)
}
// DecodeKeyName returns snapshot structure containing all metadata about a single snapshot
func DecodeKeyName(key string) (Snapshot, error) {
parts := strings.Split(key, keySplitCharacter)
if len(parts) != 3 {
return Snapshot{}, errors.New("key name in incorrect format")
}
_metadata, err := base64.StdEncoding.DecodeString(parts[2])
if len(parts) != 3 {
return Snapshot{}, fmt.Errorf("base64 decoding error: %v", err)
}
snapshot := Snapshot{}
err = json.Unmarshal(_metadata, &snapshot)
if err != nil {
return snapshot, fmt.Errorf("metadata unmarshal error: %v", err)
}
return snapshot, nil
}
type Snapshots []Snapshot
// SnapshotProcessor encapsulates everything realted to snapshots. Snapshot is an archive of app's
// directory content. It's stored in S3.
// The biggest problem in the implementation is speed of looking for snapshots by labels.
// This is distributed interface for the snapshot storage and any node can handle the request message
// so we don't have any locking mechanism here and we cannot created index of snapshots without
// significant time spend on it. Let's deal with it later. I think we are fine for first 10k snapshots.
type SnapshotProcessor struct {
AppsPath string // Where apps are stored
TmpSnapshotPath string // where temporary location for snapshots is
Driver drivers.DriverInterface
}
// CreateSnapshot creates an archive of existing application and stores it in S3 storage
// Returns key under which is the snapshot stored and/or error if there is any.
// Metadata about the snapshot are encoded in the third part of the keyname.
// The keyname cannot be bigger than 1 kB.
func (s *SnapshotProcessor) CreateSnapshot(appName string, labels []string) (string, error) {
// Create an archive
archive := archiver.Zip{
CompressionLevel: 6,
MkdirAll: true,
SelectiveCompression: true,
ContinueOnError: false,
OverwriteExisting: false,
ImplicitTopLevelFolder: false,
}
snapshot := Snapshot{
AppName: appName,
TimeStamp: time.Now().Unix(),
Labels: labels,
}
tmpSnapshotArchivePath := path.Join(s.TmpSnapshotPath, snapshot.KeyName()+".zip")
err := os.Chdir(path.Join(s.AppsPath, appName))
if err != nil {
return snapshot.KeyName(), fmt.Errorf("change working directory error: %v", err)
}
err = archive.Archive([]string{"./"}, tmpSnapshotArchivePath)
if err != nil {
return snapshot.KeyName(), fmt.Errorf("compression error: %v", err)
}
// Clean after myself
defer func() {
err = os.Remove(tmpSnapshotArchivePath)
if err != nil {
log.Println("removing temporary snapshot file error:", err.Error())
}
}()
// Pipe it into the storage
err = s.Driver.Create(snapshot.KeyName(), tmpSnapshotArchivePath)
if err != nil {
return snapshot.KeyName(), fmt.Errorf("copying snapshot into S3 error: %v", err)
}
return snapshot.KeyName(), nil
}
// RestoreSnapshot restores snapshot into an existing application
// If you need a new app from existing snapshot just create it.
// This restored only content of the disk, doesn't create the container.
func (s *SnapshotProcessor) RestoreSnapshot(key string, newAppName string) error {
tmpSnapshotArchivePath := path.Join(s.TmpSnapshotPath, key+".zip")
err := os.MkdirAll(path.Join(s.AppsPath, newAppName), 0755)
if err != nil {
return fmt.Errorf("creating destination path error: %v", err)
}
err = os.Chdir(path.Join(s.AppsPath, newAppName))
if err != nil {
return fmt.Errorf("creating destination path error: %v", err)
}
s.Driver.Get(key, tmpSnapshotArchivePath)
if err != nil {
return fmt.Errorf("getting the archive from S3 error: %v", err)
}
archive := archiver.Zip{
CompressionLevel: 6,
MkdirAll: true,
SelectiveCompression: true,
ContinueOnError: false,
OverwriteExisting: false,
ImplicitTopLevelFolder: false,
}
err = archive.Unarchive(tmpSnapshotArchivePath, "./")
if err != nil {
return fmt.Errorf("unarchiving error: %v", err)
}
err = os.Remove(tmpSnapshotArchivePath)
if err != nil {
return fmt.Errorf("removing the archive error: %v", err)
}
return nil
}
// ListAppSnapshots returns list of all snapshots related to a single application
func (s *SnapshotProcessor) ListAppSnapshots(appName string) ([]Snapshot, error) {
snapshots := []Snapshot{}
keys, err := s.Driver.List("")
if err != nil {
return snapshots, err
}
for _, key := range keys {
if strings.HasPrefix(key, appName+keySplitCharacter) {
snapshot, err := DecodeKeyName(key)
if err != nil {
return snapshots, err
}
snapshots = append(snapshots, snapshot)
}
}
return snapshots, nil
}
// ListAppsSnapshots returns list of snapshots for all given apps
func (s *SnapshotProcessor) ListAppsSnapshots(appNames []string) ([]Snapshot, error) {
snapshots := []Snapshot{}
keys, err := s.Driver.List("")
if err != nil {
return snapshots, err
}
for _, key := range keys {
for _, appName := range appNames {
if strings.HasPrefix(key, appName+keySplitCharacter) {
snapshot, err := DecodeKeyName(key)
if err != nil {
return snapshots, err
}
snapshots = append(snapshots, snapshot)
}
}
}
return snapshots, nil
}
// ListAppsSnapshotsByLabels returns list of snapshots with given label
// TODO: this will be ok for now but probably little slow when users start using it more
func (s *SnapshotProcessor) ListAppsSnapshotsByLabels(desiredLabel string) ([]Snapshot, error) {
snapshots := []Snapshot{}
keys, err := s.Driver.List("")
if err != nil {
return snapshots, err
}
for _, key := range keys {
snapshot, err := DecodeKeyName(key)
if err != nil {
return snapshots, err
}
for _, label := range snapshot.Labels {
if label == desiredLabel {
snapshots = append(snapshots, snapshot)
}
}
}
return snapshots, nil
}
// DeleteSnapshot delete's one snapshot
func (s *SnapshotProcessor) DeleteSnapshot(key string) error {
err := s.Driver.Delete(key)
if err != nil {
return fmt.Errorf("removing snapshot error: %v", err)
}
return nil
}
// DeleteAppSnapshots deletes all snapshots related to a single application
func (s *SnapshotProcessor) DeleteAppSnapshots(appName string) error {
snapshots, err := s.ListAppSnapshots(appName)
if err != nil {
return fmt.Errorf("removing snapshots error: %v", err)
}
for _, snapshot := range snapshots {
err = s.DeleteSnapshot(snapshot.KeyName())
if err != nil {
return fmt.Errorf("removing snapshots error: %v", err)
}
}
return nil
}