package apps import ( "encoding/base64" "encoding/json" "errors" "fmt" "log" "os" "path" "strings" "time" "github.com/mholt/archiver/v3" "github.com/rosti-cz/node-api/apps/drivers" ) const bucketName = "snapshots" const dateFormat = "20060102_150405" const keySplitCharacter = ":" // Snapshot contains metadata about a single snapshot type Snapshot struct { AppName string `json:"app_name"` TimeStamp int64 `json:"ts"` Labels []string `json:"labels"` } // SnapshotIndexLine is struct holding information about a single snapshot func (s *Snapshot) ToString() string { // Ignoring this error is intentional. There shouldn't be any problem with this, because all data types are ready to be marshaled. body, _ := json.Marshal(s) return string(body) } // KeyName returns keyname used to store the snapshot in the storage func (s *Snapshot) KeyName() string { metadata := base64.StdEncoding.EncodeToString([]byte(s.ToString())) // TODO: this can't be bigger than 1kB return fmt.Sprintf("%s%s%d%s%s", s.AppName, keySplitCharacter, s.TimeStamp, keySplitCharacter, metadata) } // DecodeKeyName returns snapshot structure containing all metadata about a single snapshot func DecodeKeyName(key string) (Snapshot, error) { parts := strings.Split(key, keySplitCharacter) if len(parts) != 3 { return Snapshot{}, errors.New("key name in incorrect format") } _metadata, err := base64.StdEncoding.DecodeString(parts[2]) if len(parts) != 3 { return Snapshot{}, fmt.Errorf("base64 decoding error: %v", err) } snapshot := Snapshot{} err = json.Unmarshal(_metadata, &snapshot) if err != nil { return snapshot, fmt.Errorf("metadata unmarshal error: %v", err) } return snapshot, nil } type Snapshots []Snapshot // SnapshotProcessor encapsulates everything realted to snapshots. Snapshot is an archive of app's // directory content. It's stored in S3. // The biggest problem in the implementation is speed of looking for snapshots by labels. // This is distributed interface for the snapshot storage and any node can handle the request message // so we don't have any locking mechanism here and we cannot created index of snapshots without // significant time spend on it. Let's deal with it later. I think we are fine for first 10k snapshots. type SnapshotProcessor struct { AppsPath string // Where apps are stored TmpSnapshotPath string // where temporary location for snapshots is Driver drivers.DriverInterface } // CreateSnapshot creates an archive of existing application and stores it in S3 storage // Returns key under which is the snapshot stored and/or error if there is any. // Metadata about the snapshot are encoded in the third part of the keyname. // The keyname cannot be bigger than 1 kB. func (s *SnapshotProcessor) CreateSnapshot(appName string, labels []string) (string, error) { // Create an archive archive := archiver.Zip{ CompressionLevel: 6, MkdirAll: true, SelectiveCompression: true, ContinueOnError: true, OverwriteExisting: false, ImplicitTopLevelFolder: false, } snapshot := Snapshot{ AppName: appName, TimeStamp: time.Now().Unix(), Labels: labels, } tmpSnapshotArchivePath := path.Join(s.TmpSnapshotPath, snapshot.KeyName()+".zip") err := os.Chdir(path.Join(s.AppsPath, appName)) if err != nil { return snapshot.KeyName(), fmt.Errorf("change working directory error: %v", err) } err = archive.Archive([]string{"./"}, tmpSnapshotArchivePath) if err != nil { return snapshot.KeyName(), fmt.Errorf("compression error: %v", err) } info, err := os.Stat(tmpSnapshotArchivePath) if err != nil { return snapshot.KeyName(), fmt.Errorf("temporary file stat error: %v", err) } snapshot.Labels = append(snapshot.Labels, fmt.Sprintf("size:%d", info.Size())) // Clean after myself defer func() { err = os.Remove(tmpSnapshotArchivePath) if err != nil { log.Println("removing temporary snapshot file error:", err.Error()) } }() // Pipe it into the storage err = s.Driver.Create(snapshot.KeyName(), tmpSnapshotArchivePath) if err != nil { return snapshot.KeyName(), fmt.Errorf("copying snapshot into S3 error: %v", err) } return snapshot.KeyName(), nil } // RestoreSnapshot restores snapshot into an existing application // If you need a new app from existing snapshot just create it. // This restored only content of the disk, doesn't create the container. func (s *SnapshotProcessor) RestoreSnapshot(key string, newAppName string) error { tmpSnapshotArchivePath := path.Join(s.TmpSnapshotPath, key+".zip") err := os.MkdirAll(path.Join(s.AppsPath, newAppName), 0755) if err != nil { return fmt.Errorf("creating destination path error: %v", err) } err = os.Chdir(path.Join(s.AppsPath, newAppName)) if err != nil { return fmt.Errorf("creating destination path error: %v", err) } s.Driver.Get(key, tmpSnapshotArchivePath) if err != nil { return fmt.Errorf("getting the archive from S3 error: %v", err) } archive := archiver.Zip{ CompressionLevel: 6, MkdirAll: true, SelectiveCompression: true, ContinueOnError: false, OverwriteExisting: false, ImplicitTopLevelFolder: false, } err = archive.Unarchive(tmpSnapshotArchivePath, "./") if err != nil { return fmt.Errorf("unarchiving error: %v", err) } err = os.Remove(tmpSnapshotArchivePath) if err != nil { return fmt.Errorf("removing the archive error: %v", err) } return nil } // ListAppSnapshots returns list of all snapshots related to a single application func (s *SnapshotProcessor) ListAppSnapshots(appName string) ([]Snapshot, error) { snapshots := []Snapshot{} keys, err := s.Driver.List("") if err != nil { return snapshots, err } for _, key := range keys { if strings.HasPrefix(key, appName+keySplitCharacter) { snapshot, err := DecodeKeyName(key) if err != nil { return snapshots, err } snapshots = append(snapshots, snapshot) } } return snapshots, nil } // ListAppsSnapshots returns list of snapshots for all given apps func (s *SnapshotProcessor) ListAppsSnapshots(appNames []string) ([]Snapshot, error) { snapshots := []Snapshot{} keys, err := s.Driver.List("") if err != nil { return snapshots, err } for _, key := range keys { for _, appName := range appNames { if strings.HasPrefix(key, appName+keySplitCharacter) { snapshot, err := DecodeKeyName(key) if err != nil { return snapshots, err } snapshots = append(snapshots, snapshot) } } } return snapshots, nil } // ListAppsSnapshotsByLabel returns list of snapshots with given label // TODO: this will be ok for now but probably little slow when users start using it more func (s *SnapshotProcessor) ListAppsSnapshotsByLabel(desiredLabel string) ([]Snapshot, error) { snapshots := []Snapshot{} keys, err := s.Driver.List("") if err != nil { return snapshots, err } for _, key := range keys { snapshot, err := DecodeKeyName(key) if err != nil { return snapshots, err } for _, label := range snapshot.Labels { if label == desiredLabel { snapshots = append(snapshots, snapshot) } } } return snapshots, nil } // DeleteSnapshot delete's one snapshot func (s *SnapshotProcessor) DeleteSnapshot(key string) error { err := s.Driver.Delete(key) if err != nil { return fmt.Errorf("removing snapshot error: %v", err) } return nil } // DeleteAppSnapshots deletes all snapshots related to a single application func (s *SnapshotProcessor) DeleteAppSnapshots(appName string) error { snapshots, err := s.ListAppSnapshots(appName) if err != nil { return fmt.Errorf("removing snapshots error: %v", err) } for _, snapshot := range snapshots { err = s.DeleteSnapshot(snapshot.KeyName()) if err != nil { return fmt.Errorf("removing snapshots error: %v", err) } } return nil }