node-api/apps/snapshots.go

package apps

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"os"
	"path"
	"strings"
	"time"

	"github.com/mholt/archiver/v3"
	"github.com/rosti-cz/node-api/apps/drivers"
)

const bucketName = "snapshots"
const dateFormat = "20060102_150405"
const keySplitCharacter = ":"

// Snapshot contains metadata about a single snapshot
type Snapshot struct {
	AppName   string   `json:"app_name"`
	TimeStamp int64    `json:"ts"`
	Labels    []string `json:"labels"`
}

// SnapshotIndexLine is struct holding information about a single snapshot
func (s *Snapshot) ToString() string {
	// Ignoring this error is intentional. There shouldn't be any problem with this, because all data types are ready to be marshaled.
	body, _ := json.Marshal(s)
	return string(body)
}

// KeyName returns keyname used to store the snapshot in the storage
func (s *Snapshot) KeyName() string {
	metadata := base64.StdEncoding.EncodeToString([]byte(s.ToString()))
	// TODO: this can't be bigger than 1kB
	return fmt.Sprintf("%s%s%d%s%s", s.AppName, keySplitCharacter, s.TimeStamp, keySplitCharacter, metadata)
}

// DecodeKeyName returns snapshot structure containing all metadata about a single snapshot
func DecodeKeyName(key string) (Snapshot, error) {
	parts := strings.Split(key, keySplitCharacter)
	if len(parts) != 3 {
		return Snapshot{}, errors.New("key name in incorrect format")
	}

	_metadata, err := base64.StdEncoding.DecodeString(parts[2])
	if len(parts) != 3 {
		return Snapshot{}, fmt.Errorf("base64 decoding error: %v", err)
	}

	snapshot := Snapshot{}
	err = json.Unmarshal(_metadata, &snapshot)
	if err != nil {
		return snapshot, fmt.Errorf("metadata unmarshal error: %v", err)
	}

	return snapshot, nil
}

type Snapshots []Snapshot

// SnapshotProcessor encapsulates everything realted to snapshots. Snapshot is an archive of app's
// directory content. It's stored in S3.
// The biggest problem in the implementation is speed of looking for snapshots by labels.
// This is distributed interface for the snapshot storage and any node can handle the request message
// so we don't have any locking mechanism here and we cannot created index of snapshots without
// significant time spend on it. Let's deal with it later. I think we are fine for first 10k snapshots.
type SnapshotProcessor struct {
	AppsPath        string // Where apps are stored
	TmpSnapshotPath string // where temporary location for snapshots is

	Driver drivers.DriverInterface
}

// CreateSnapshot creates an archive of existing application and stores it in S3 storage
// Returns key under which is the snapshot stored and/or error if there is any.
// Metadata about the snapshot are encoded in the third part of the keyname.
// The keyname cannot be bigger than 1 kB.
func (s *SnapshotProcessor) CreateSnapshot(appName string, labels []string) (string, error) {
	// Create an archive
	archive := archiver.Zip{
		CompressionLevel:       6,
		MkdirAll:               true,
		SelectiveCompression:   true,
		ContinueOnError:        true,
		OverwriteExisting:      false,
		ImplicitTopLevelFolder: false,
	}

	snapshot := Snapshot{
		AppName:   appName,
		TimeStamp: time.Now().Unix(),
		Labels:    labels,
	}

	tmpSnapshotArchivePath := path.Join(s.TmpSnapshotPath, snapshot.KeyName()+".zip")

	err := os.Chdir(path.Join(s.AppsPath, appName))
	if err != nil {
		return snapshot.KeyName(), fmt.Errorf("change working directory error: %v", err)
	}

	err = archive.Archive([]string{"./"}, tmpSnapshotArchivePath)
	if err != nil {
		return snapshot.KeyName(), fmt.Errorf("compression error: %v", err)
	}

	info, err := os.Stat(tmpSnapshotArchivePath)
	if err != nil {
		return snapshot.KeyName(), fmt.Errorf("temporary file stat error: %v", err)
	}
	snapshot.Labels = append(snapshot.Labels, fmt.Sprintf("size:%d", info.Size()))

	// Clean after myself
	defer func() {
		err = os.Remove(tmpSnapshotArchivePath)
		if err != nil {
			log.Println("removing temporary snapshot file error:", err.Error())
		}
	}()

	// Pipe it into the storage
	err = s.Driver.Create(snapshot.KeyName(), tmpSnapshotArchivePath)
	if err != nil {
		return snapshot.KeyName(), fmt.Errorf("copying snapshot into S3 error: %v", err)
	}

	return snapshot.KeyName(), nil
}

// RestoreSnapshot restores snapshot into an existing application
// If you need a new app from existing snapshot just create it.
// This restored only content of the disk, doesn't create the container.
func (s *SnapshotProcessor) RestoreSnapshot(key string, newAppName string) error {
	tmpSnapshotArchivePath := path.Join(s.TmpSnapshotPath, key+".zip")

	err := os.MkdirAll(path.Join(s.AppsPath, newAppName), 0755)
	if err != nil {
		return fmt.Errorf("creating destination path error: %v", err)
	}

	err = os.Chdir(path.Join(s.AppsPath, newAppName))
	if err != nil {
		return fmt.Errorf("creating destination path error: %v", err)
	}

	s.Driver.Get(key, tmpSnapshotArchivePath)
	if err != nil {
		return fmt.Errorf("getting the archive from S3 error: %v", err)
	}

	archive := archiver.Zip{
		CompressionLevel:       6,
		MkdirAll:               true,
		SelectiveCompression:   true,
		ContinueOnError:        false,
		OverwriteExisting:      false,
		ImplicitTopLevelFolder: false,
	}

	err = archive.Unarchive(tmpSnapshotArchivePath, "./")
	if err != nil {
		return fmt.Errorf("unarchiving error: %v", err)
	}

	err = os.Remove(tmpSnapshotArchivePath)
	if err != nil {
		return fmt.Errorf("removing the archive error: %v", err)
	}

	return nil
}

// ListAppSnapshots returns list of all snapshots related to a single application
func (s *SnapshotProcessor) ListAppSnapshots(appName string) ([]Snapshot, error) {
	snapshots := []Snapshot{}

	keys, err := s.Driver.List("")
	if err != nil {
		return snapshots, err
	}

	for _, key := range keys {
		if strings.HasPrefix(key, appName+keySplitCharacter) {
			snapshot, err := DecodeKeyName(key)
			if err != nil {
				return snapshots, err
			}

			snapshots = append(snapshots, snapshot)
		}
	}

	return snapshots, nil
}

// ListAppsSnapshots returns list of snapshots for all given apps
func (s *SnapshotProcessor) ListAppsSnapshots(appNames []string) ([]Snapshot, error) {
	snapshots := []Snapshot{}

	keys, err := s.Driver.List("")
	if err != nil {
		return snapshots, err
	}

	for _, key := range keys {
		for _, appName := range appNames {
			if strings.HasPrefix(key, appName+keySplitCharacter) {
				snapshot, err := DecodeKeyName(key)
				if err != nil {
					return snapshots, err
				}

				snapshots = append(snapshots, snapshot)
			}
		}
	}

	return snapshots, nil
}

// ListAppsSnapshotsByLabel returns list of snapshots with given label
// TODO: this will be ok for now but probably little slow when users start using it more
func (s *SnapshotProcessor) ListAppsSnapshotsByLabel(desiredLabel string) ([]Snapshot, error) {
	snapshots := []Snapshot{}

	keys, err := s.Driver.List("")
	if err != nil {
		return snapshots, err
	}

	for _, key := range keys {
		snapshot, err := DecodeKeyName(key)
		if err != nil {
			return snapshots, err
		}

		for _, label := range snapshot.Labels {
			if label == desiredLabel {
				snapshots = append(snapshots, snapshot)
			}
		}
	}

	return snapshots, nil
}

// DeleteSnapshot delete's one snapshot
func (s *SnapshotProcessor) DeleteSnapshot(key string) error {
	err := s.Driver.Delete(key)
	if err != nil {
		return fmt.Errorf("removing snapshot error: %v", err)
	}

	return nil
}

// DeleteAppSnapshots deletes all snapshots related to a single application
func (s *SnapshotProcessor) DeleteAppSnapshots(appName string) error {
	snapshots, err := s.ListAppSnapshots(appName)
	if err != nil {
		return fmt.Errorf("removing snapshots error: %v", err)
	}

	for _, snapshot := range snapshots {
		err = s.DeleteSnapshot(snapshot.KeyName())
		if err != nil {
			return fmt.Errorf("removing snapshots error: %v", err)
		}
	}

	return nil
}