diff --git a/api.http b/api.http new file mode 100644 index 0000000..eaac4a0 --- /dev/null +++ b/api.http @@ -0,0 +1,27 @@ +POST http://localhost:1323/v1/apps +Content-type: application/json + +{ + "name": "test_1234", + "ssh_port": 10000, + "http_port": 10001, + "image": "docker.io/rosti/runtime:2020.04-1", + "cpu": 100, + "memory": 128000 +} + +### + +PUT http://localhost:1323/v1/apps/test_1234 +Content-type: application/json + +{ + "ssh_port": 36500, + "http_port": 36501 +} + + +### + +PUT http://localhost:1323/v1/apps/test_1234/start +Content-type: application/json diff --git a/apps/main.go b/apps/main.go index 8fb157f..e0c6a0d 100644 --- a/apps/main.go +++ b/apps/main.go @@ -1,6 +1,8 @@ package apps -import "github.com/rosti-cz/apps-api/common" +import ( + "github.com/rosti-cz/apps-api/common" +) func init() { db := common.GetDBConnection() @@ -56,7 +58,7 @@ func New(name string, SSHPort int, HTTPPort int, image string, CPU int, memory i } } - if err := db.Create(app).Error; err != nil { + if err := db.Create(&app).Error; err != nil { return err } @@ -64,32 +66,49 @@ func New(name string, SSHPort int, HTTPPort int, image string, CPU int, memory i } // Update changes value about app in the database -func Update(name string, SSHPort int, HTTPPort int, image string, CPU int, memory int) error { +func Update(name string, SSHPort int, HTTPPort int, image string, CPU int, memory int) (*App, error) { var app App db := common.GetDBConnection() err := db.First(&app).Where("name = ?", name).Error if err != nil { - return err + return &app, err } - app.SSHPort = SSHPort - app.HTTPPort = HTTPPort - app.Image = image - app.CPU = CPU - app.Memory = memory + // Update affected fields + if image != "" { + app.Image = image + } + + if CPU != 0 { + app.CPU = CPU + } + + if memory != 0 { + app.Memory = memory + } + + // SSH port and HTTP port cannot be turned off when they are once set + if SSHPort != 0 { + app.SSHPort = SSHPort + } + + // SSH port and HTTP port cannot be turned off when they are once set + if HTTPPort != 0 { + app.HTTPPort = HTTPPort + } validationErrors := app.Validate() if len(validationErrors) != 0 { - return ValidationError{ + return &app, ValidationError{ Errors: validationErrors, } } - err = db.Update(&app).Error - - return err + // Apply the changes + err = db.Save(&app).Error + return &app, err } // Delete removes records about one app from the database diff --git a/apps/types.go b/apps/types.go index 6ff1f00..a178b49 100644 --- a/apps/types.go +++ b/apps/types.go @@ -5,6 +5,8 @@ import ( "strings" "github.com/jinzhu/gorm" + // This is line from GORM documentation that imports database dialect + _ "github.com/jinzhu/gorm/dialects/sqlite" ) // ValidationError is error that holds multiple validation error messages @@ -25,18 +27,13 @@ type Label struct { type App struct { gorm.Model - Name string `json:"name" gorm:"primary_key"` - SSHPort int `json:"ssh_port"` - HTTPPort int `json:"http_port"` - Image string `json:"image"` - CPU int `json:"cpu"` // percentage, 200 means two CPU - Memory int `json:"memory"` // Limit in MB - Labels []Label `json:"labels"` // username:cx or user_id:1 - - Status string `json:"status"` // running, data-only (no container, data only), stopped - MemoryUsage int `json:"memory_usage"` // Usage in MB - DiskUsage int `json:"disk_usage"` // Usage in MB - + Name string `json:"name" gorm:"primary_key"` + SSHPort int `json:"ssh_port"` + HTTPPort int `json:"http_port"` + Image string `json:"image"` + CPU int `json:"cpu"` // percentage, 200 means two CPU + Memory int `json:"memory"` // Limit in MB + // Labels []Label `json:"labels"` // username:cx or user_id:1 } // Validate do basic checks of the struct values diff --git a/docker/docker.go b/docker/docker.go index 4de3086..db62df2 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -3,7 +3,10 @@ package docker import ( "context" "errors" + "io" + "io/ioutil" "log" + "os" "strconv" "strings" "time" @@ -63,6 +66,62 @@ func (d *Driver) nameToID(name string) (string, error) { return containerIDs[0], nil } +// Status return current status of container with given name +func (d *Driver) Status(name string) (string, error) { + status := "unknown" + + cli, err := d.getClient() + if err != nil { + return status, err + } + + containerID, err := d.nameToID(name) + if err != nil { + status = "no-container" + return status, err + } + + info, err := cli.ContainerInspect(context.TODO(), containerID) + if err != nil { + return status, err + } + + if info.State.Running { + status = "running" + } else { + status = "stopped" + } + + return status, nil + +} + +// Stats returns current CPU and memory usage +func (d *Driver) Stats(name string) (float64, int, error) { + cli, err := d.getClient() + if err != nil { + return 0.0, 0, err + } + + containerID, err := d.nameToID(name) + if err != nil { + return 0.0, 0, err + } + + stats, err := cli.ContainerStats(context.TODO(), containerID, false) + if err != nil { + return 0.0, 0, nil + } + + data, err := ioutil.ReadAll(stats.Body) + if err != nil { + return 0.0, 0, err + } + log.Println(data) + + return 0.0, 0, nil +} + // Remove removes container represented by containerID func (d *Driver) Remove(name string) error { log.Println("Removing container " + name) @@ -135,16 +194,16 @@ func (d *Driver) IsExist(name string) ([]string, error) { return containerIDs, err } - containers, err := cli.ContainerList(context.TODO(), types.ContainerListOptions{}) + containers, err := cli.ContainerList(context.TODO(), types.ContainerListOptions{All: true}) if err != nil { return containerIDs, err } // We go through the containers and pick the ones which match the task name for _, containerObject := range containers { - for _, name := range containerObject.Names { - name = strings.Trim(name, "/") - if strings.Split(name, ".")[0] == name { + for _, containerName := range containerObject.Names { + containerName = strings.TrimLeft(containerName, "/") + if containerName == name { containerIDs = append(containerIDs, containerObject.ID) } } @@ -153,6 +212,25 @@ func (d *Driver) IsExist(name string) ([]string, error) { return containerIDs, nil } +// pullImage pulls image into local docker instance +func (d *Driver) pullImage(image string) error { + log.Println("Pulling image " + image) + cli, err := d.getClient() + if err != nil { + return err + } + + stream, err := cli.ImagePull(context.TODO(), image, types.ImagePullOptions{}) + if err != nil { + return err + } + defer stream.Close() + + io.Copy(os.Stdout, stream) + + return nil +} + // Create creates the container // image - docker image // cmd - string slice of command and its arguments @@ -165,6 +243,11 @@ func (d *Driver) Create(name string, image string, volumePath string, HTTPPort i return "", err } + err = d.pullImage(image) + if err != nil { + return "", err + } + portmaps := make(nat.PortMap, 1) portbindingsHTTP := make([]nat.PortBinding, 1) @@ -191,15 +274,15 @@ func (d *Driver) Create(name string, image string, volumePath string, HTTPPort i }, &container.HostConfig{ Resources: container.Resources{ - CPUPeriod: 100, - CPUQuota: int64(CPU), + CPUPeriod: 100000, + CPUQuota: int64(CPU) * 1000, Memory: int64(memory*110/100)*1024 ^ 2, // Allow 10 % more memory so we have space for MemoryReservation MemoryReservation: int64(memory)*1024 ^ 2, // This should provide softer way how to limit the memory of our containers }, PortBindings: portmaps, AutoRemove: false, RestartPolicy: container.RestartPolicy{ - Name: "unless-stopped", + Name: "on-failure", MaximumRetryCount: 3, }, Binds: []string{ diff --git a/docker/du.go b/docker/tools.go similarity index 72% rename from docker/du.go rename to docker/tools.go index e4668ad..fe3fdff 100644 --- a/docker/du.go +++ b/docker/tools.go @@ -2,7 +2,9 @@ package docker import ( "bytes" + "os" "os/exec" + "path/filepath" "strconv" "strings" ) @@ -48,3 +50,23 @@ func du(path string) (int, int, error) { return space, inodes, nil } + +// Removes content of given directory +func removeDirectory(dir string) error { + d, err := os.Open(dir) + if err != nil { + return err + } + defer d.Close() + names, err := d.Readdirnames(-1) + if err != nil { + return err + } + for _, name := range names { + err = os.RemoveAll(filepath.Join(dir, name)) + if err != nil { + return err + } + } + return nil +} diff --git a/docker/types.go b/docker/types.go index a88c3e9..a3fbdd9 100644 --- a/docker/types.go +++ b/docker/types.go @@ -8,7 +8,12 @@ import ( // Container extends App struct from App type Container struct { - App apps.App + App apps.App `json:"app"` +} + +func (c *Container) getDriver() *Driver { + driver := &Driver{} + return driver } // volumeHostPath each container has one volume mounted into it, @@ -17,42 +22,105 @@ func (c *Container) volumeHostPath() string { } // Status returns state of the container -// Possible values: running, stopped, data-only +// Possible values: running, stopped, data-only, unknown func (c *Container) Status() (string, error) { - return "", nil + status := "unknown" + + // if _, err := os.Stat(path.Join("/srv", c.App.Name)); !os.IsNotExist(err) { + // status = "data-only" + // } + + driver := c.getDriver() + containerStatus, err := driver.Status(c.App.Name) + if err != nil { + return status, err + } + + if containerStatus == "no-container" { + status = "data-only" + } else { + status = containerStatus + } + + return status, nil } -// DiskSpace returns number of MB and inodes used by the container in it's mounted volume -func (c *Container) DiskSpace() (int, int, error) { +// DiskUsage returns number of MB and inodes used by the container in it's mounted volume +func (c *Container) DiskUsage() (int, int, error) { return du(c.volumeHostPath()) } +// ResourceUsage returns amount of memory in MB and CPU in % that the app occupies +func (c *Container) ResourceUsage() (float64, int, error) { + driver := c.getDriver() + + cpu, memory, err := driver.Stats(c.App.Name) + if err != nil { + return 0.0, 0, err + } + + return cpu, memory, nil +} + // Create creates the container func (c *Container) Create() error { - return nil + driver := c.getDriver() + _, err := driver.Create( + c.App.Name, + c.App.Image, + c.volumeHostPath(), + c.App.HTTPPort, + c.App.SSHPort, + c.App.CPU, + c.App.Memory, + []string{}, + ) + + return err } // Start starts the container func (c *Container) Start() error { - return nil + driver := c.getDriver() + + return driver.Start(c.App.Name) } // Stop stops the container func (c *Container) Stop() error { - return nil + driver := c.getDriver() + + return driver.Stop(c.App.Name) } -// Reset restarts the container -func (c *Container) Reset() error { - return nil +// Restart restarts the container +func (c *Container) Restart() error { + driver := c.getDriver() + + err := driver.Stop(c.App.Name) + if err != nil { + return err + } + + return driver.Start(c.App.Name) } // Destroy removes the container but keeps the data so it can be created again func (c *Container) Destroy() error { - return nil + driver := c.getDriver() + + return driver.Remove(c.App.Name) } // Delete removes both data and the container func (c *Container) Delete() error { - return nil + err := c.Destroy() + if err != nil { + return err + } + + volumePath := path.Join("/srv", c.App.Name) + err = removeDirectory(volumePath) + + return err } diff --git a/go.mod b/go.mod index 77d8260..2705283 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,14 @@ module github.com/rosti-cz/apps-api go 1.14 require ( + github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/docker v1.13.1 + github.com/docker/go-connections v0.4.0 + github.com/docker/go-units v0.4.0 // indirect github.com/jinzhu/gorm v1.9.14 github.com/labstack/echo v3.3.10+incompatible github.com/labstack/gommon v0.3.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/pkg/errors v0.9.1 // indirect golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect - github.com/docker/docker v1.13.1 - github.com/docker/go-connections v0.4.0 ) diff --git a/go.sum b/go.sum index c8221a9..931e8fa 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,14 @@ github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBK github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo= +github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= @@ -25,6 +33,10 @@ github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/main.go b/main.go index 40d71b0..7ca0ba6 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ import ( "github.com/labstack/echo" "github.com/rosti-cz/apps-api/apps" "github.com/rosti-cz/apps-api/common" + "github.com/rosti-cz/apps-api/docker" "github.com/rosti-cz/apps-api/nodes" ) @@ -15,6 +16,8 @@ const JSONIndent = " " func main() { // Close database at the end db := common.GetDBConnection() + // db.AutoMigrate(apps.Label{}) + // db.AutoMigrate(apps.App{}) defer db.Close() // API @@ -59,7 +62,21 @@ func main() { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } - return c.JSON(http.StatusOK, map[string]string{}) + container := docker.Container{ + App: app, + } + + err = container.Create() + if err != nil { + return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) + } + + err = container.Start() + if err != nil { + return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) + } + + return c.JSON(http.StatusOK, Message{Message: "ok"}) }) // Update existing app @@ -72,7 +89,7 @@ func main() { return c.JSONPretty(http.StatusBadRequest, Message{Message: err.Error()}, JSONIndent) } - err = apps.Update(name, app.SSHPort, app.HTTPPort, app.Image, app.CPU, app.Memory) + appPointer, err := apps.Update(name, app.SSHPort, app.HTTPPort, app.Image, app.CPU, app.Memory) if err != nil { if validationError, ok := err.(apps.ValidationError); ok { return c.JSONPretty(http.StatusBadRequest, Message{Errors: validationError.Errors}, JSONIndent) @@ -80,6 +97,27 @@ func main() { return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) } + app = *appPointer + + container := docker.Container{ + App: app, + } + + err = container.Destroy() + if err != nil { + return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) + } + + err = container.Create() + if err != nil { + return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) + } + + err = container.Start() + if err != nil { + return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) + } + return c.JSON(http.StatusOK, map[string]string{}) }) @@ -90,7 +128,23 @@ func main() { // Start existing app e.PUT("/v1/apps/:name/start", func(c echo.Context) error { - return c.JSON(http.StatusOK, map[string]string{}) + name := c.Param("name") + + app, err := apps.Get(name) + if err != nil { + return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) + } + + container := docker.Container{ + App: *app, + } + + err = container.Start() + if err != nil { + return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) + } + + return c.JSON(http.StatusOK, Message{Message: "ok"}) }) // Stop existing app