Rosti/node-api
3
0
Fork 0
Code Issues 2 Pull requests Projects Releases 12 Wiki Activity Actions

Docker support partly done

Browse source
This commit is contained in:
Adam Štrauch 2020-07-11 13:04:37 +02:00
parent 064d6e6487
commit 0181236e23
Signed by: cx
GPG key ID: 018304FFA8988F8D
9 changed files with 336 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
api.http Normal file
View file

@ -0,0 +1,27 @@
POST http://localhost:1323/v1/apps
Content-type: application/json
{
"name": "test_1234",
"ssh_port": 10000,
"http_port": 10001,
"image": "docker.io/rosti/runtime:2020.04-1",
"cpu": 100,
"memory": 128000
}
###
PUT http://localhost:1323/v1/apps/test_1234
Content-type: application/json
{
"ssh_port": 36500,
"http_port": 36501
}
###
PUT http://localhost:1323/v1/apps/test_1234/start
Content-type: application/json

45
apps/main.go
View file

@ -1,6 +1,8 @@
package apps
import "github.com/rosti-cz/apps-api/common"
import (
"github.com/rosti-cz/apps-api/common"
)
func init() {
db := common.GetDBConnection()
@ -56,7 +58,7 @@ func New(name string, SSHPort int, HTTPPort int, image string, CPU int, memory i
}
}
if err := db.Create(app).Error; err != nil {
if err := db.Create(&app).Error; err != nil {
return err
}
@ -64,32 +66,49 @@ func New(name string, SSHPort int, HTTPPort int, image string, CPU int, memory i
}
// Update changes value about app in the database
func Update(name string, SSHPort int, HTTPPort int, image string, CPU int, memory int) error {
func Update(name string, SSHPort int, HTTPPort int, image string, CPU int, memory int) (*App, error) {
var app App
db := common.GetDBConnection()
err := db.First(&app).Where("name = ?", name).Error
if err != nil {
return err
return &app, err
}
app.SSHPort = SSHPort
app.HTTPPort = HTTPPort
app.Image = image
app.CPU = CPU
app.Memory = memory
// Update affected fields
if image != "" {
app.Image = image
}
if CPU != 0 {
app.CPU = CPU
}
if memory != 0 {
app.Memory = memory
}
// SSH port and HTTP port cannot be turned off when they are once set
if SSHPort != 0 {
app.SSHPort = SSHPort
}
// SSH port and HTTP port cannot be turned off when they are once set
if HTTPPort != 0 {
app.HTTPPort = HTTPPort
}
validationErrors := app.Validate()
if len(validationErrors) != 0 {
return ValidationError{
return &app, ValidationError{
Errors: validationErrors,
}
}
err = db.Update(&app).Error
return err
// Apply the changes
err = db.Save(&app).Error
return &app, err
}
// Delete removes records about one app from the database

21
apps/types.go
View file

@ -5,6 +5,8 @@ import (
"strings"
"github.com/jinzhu/gorm"
// This is line from GORM documentation that imports database dialect
_ "github.com/jinzhu/gorm/dialects/sqlite"
)
// ValidationError is error that holds multiple validation error messages
@ -25,18 +27,13 @@ type Label struct {
type App struct {
gorm.Model
Name string `json:"name" gorm:"primary_key"`
SSHPort int `json:"ssh_port"`
HTTPPort int `json:"http_port"`
Image string `json:"image"`
CPU int `json:"cpu"` // percentage, 200 means two CPU
Memory int `json:"memory"` // Limit in MB
Labels []Label `json:"labels"` // username:cx or user_id:1
Status string `json:"status"` // running, data-only (no container, data only), stopped
MemoryUsage int `json:"memory_usage"` // Usage in MB
DiskUsage int `json:"disk_usage"` // Usage in MB
Name string `json:"name" gorm:"primary_key"`
SSHPort int `json:"ssh_port"`
HTTPPort int `json:"http_port"`
Image string `json:"image"`
CPU int `json:"cpu"` // percentage, 200 means two CPU
Memory int `json:"memory"` // Limit in MB
// Labels []Label `json:"labels"` // username:cx or user_id:1
}
// Validate do basic checks of the struct values

97
docker/docker.go
View file

@ -3,7 +3,10 @@ package docker
import (
"context"
"errors"
"io"
"io/ioutil"
"log"
"os"
"strconv"
"strings"
"time"
@ -63,6 +66,62 @@ func (d *Driver) nameToID(name string) (string, error) {
return containerIDs[0], nil
}
// Status return current status of container with given name
func (d *Driver) Status(name string) (string, error) {
status := "unknown"
cli, err := d.getClient()
if err != nil {
return status, err
}
containerID, err := d.nameToID(name)
if err != nil {
status = "no-container"
return status, err
}
info, err := cli.ContainerInspect(context.TODO(), containerID)
if err != nil {
return status, err
}
if info.State.Running {
status = "running"
} else {
status = "stopped"
}
return status, nil
}
// Stats returns current CPU and memory usage
func (d *Driver) Stats(name string) (float64, int, error) {
cli, err := d.getClient()
if err != nil {
return 0.0, 0, err
}
containerID, err := d.nameToID(name)
if err != nil {
return 0.0, 0, err
}
stats, err := cli.ContainerStats(context.TODO(), containerID, false)
if err != nil {
return 0.0, 0, nil
}
data, err := ioutil.ReadAll(stats.Body)
if err != nil {
return 0.0, 0, err
}
log.Println(data)
return 0.0, 0, nil
}
// Remove removes container represented by containerID
func (d *Driver) Remove(name string) error {
log.Println("Removing container " + name)
@ -135,16 +194,16 @@ func (d *Driver) IsExist(name string) ([]string, error) {
return containerIDs, err
}
containers, err := cli.ContainerList(context.TODO(), types.ContainerListOptions{})
containers, err := cli.ContainerList(context.TODO(), types.ContainerListOptions{All: true})
if err != nil {
return containerIDs, err
}
// We go through the containers and pick the ones which match the task name
for _, containerObject := range containers {
for _, name := range containerObject.Names {
name = strings.Trim(name, "/")
if strings.Split(name, ".")[0] == name {
for _, containerName := range containerObject.Names {
containerName = strings.TrimLeft(containerName, "/")
if containerName == name {
containerIDs = append(containerIDs, containerObject.ID)
}
}
@ -153,6 +212,25 @@ func (d *Driver) IsExist(name string) ([]string, error) {
return containerIDs, nil
}
// pullImage pulls image into local docker instance
func (d *Driver) pullImage(image string) error {
log.Println("Pulling image " + image)
cli, err := d.getClient()
if err != nil {
return err
}
stream, err := cli.ImagePull(context.TODO(), image, types.ImagePullOptions{})
if err != nil {
return err
}
defer stream.Close()
io.Copy(os.Stdout, stream)
return nil
}
// Create creates the container
// image - docker image
// cmd - string slice of command and its arguments
@ -165,6 +243,11 @@ func (d *Driver) Create(name string, image string, volumePath string, HTTPPort i
return "", err
}
err = d.pullImage(image)
if err != nil {
return "", err
}
portmaps := make(nat.PortMap, 1)
portbindingsHTTP := make([]nat.PortBinding, 1)
@ -191,15 +274,15 @@ func (d *Driver) Create(name string, image string, volumePath string, HTTPPort i
},
&container.HostConfig{
Resources: container.Resources{
CPUPeriod: 100,
CPUQuota: int64(CPU),
CPUPeriod: 100000,
CPUQuota: int64(CPU) * 1000,
Memory: int64(memory*110/100)*1024 ^ 2, // Allow 10 % more memory so we have space for MemoryReservation
MemoryReservation: int64(memory)*1024 ^ 2, // This should provide softer way how to limit the memory of our containers
},
PortBindings: portmaps,
AutoRemove: false,
RestartPolicy: container.RestartPolicy{
Name: "unless-stopped",
Name: "on-failure",
MaximumRetryCount: 3,
},
Binds: []string{

22
docker/du.go → docker/tools.go
View file

@ -2,7 +2,9 @@ package docker
import (
"bytes"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
)
@ -48,3 +50,23 @@ func du(path string) (int, int, error) {
return space, inodes, nil
}
// Removes content of given directory
func removeDirectory(dir string) error {
d, err := os.Open(dir)
if err != nil {
return err
}
defer d.Close()
names, err := d.Readdirnames(-1)
if err != nil {
return err
}
for _, name := range names {
err = os.RemoveAll(filepath.Join(dir, name))
if err != nil {
return err
}
}
return nil
}

94
docker/types.go
View file

@ -8,7 +8,12 @@ import (
// Container extends App struct from App
type Container struct {
App apps.App
App apps.App `json:"app"`
}
func (c *Container) getDriver() *Driver {
driver := &Driver{}
return driver
}
// volumeHostPath each container has one volume mounted into it,
@ -17,42 +22,105 @@ func (c *Container) volumeHostPath() string {
}
// Status returns state of the container
// Possible values: running, stopped, data-only
// Possible values: running, stopped, data-only, unknown
func (c *Container) Status() (string, error) {
return "", nil
status := "unknown"
// if _, err := os.Stat(path.Join("/srv", c.App.Name)); !os.IsNotExist(err) {
// status = "data-only"
// }
driver := c.getDriver()
containerStatus, err := driver.Status(c.App.Name)
if err != nil {
return status, err
}
if containerStatus == "no-container" {
status = "data-only"
} else {
status = containerStatus
}
return status, nil
}
// DiskSpace returns number of MB and inodes used by the container in it's mounted volume
func (c *Container) DiskSpace() (int, int, error) {
// DiskUsage returns number of MB and inodes used by the container in it's mounted volume
func (c *Container) DiskUsage() (int, int, error) {
return du(c.volumeHostPath())
}
// ResourceUsage returns amount of memory in MB and CPU in % that the app occupies
func (c *Container) ResourceUsage() (float64, int, error) {
driver := c.getDriver()
cpu, memory, err := driver.Stats(c.App.Name)
if err != nil {
return 0.0, 0, err
}
return cpu, memory, nil
}
// Create creates the container
func (c *Container) Create() error {
return nil
driver := c.getDriver()
_, err := driver.Create(
c.App.Name,
c.App.Image,
c.volumeHostPath(),
c.App.HTTPPort,
c.App.SSHPort,
c.App.CPU,
c.App.Memory,
[]string{},
)
return err
}
// Start starts the container
func (c *Container) Start() error {
return nil
driver := c.getDriver()
return driver.Start(c.App.Name)
}
// Stop stops the container
func (c *Container) Stop() error {
return nil
driver := c.getDriver()
return driver.Stop(c.App.Name)
}
// Reset restarts the container
func (c *Container) Reset() error {
return nil
// Restart restarts the container
func (c *Container) Restart() error {
driver := c.getDriver()
err := driver.Stop(c.App.Name)
if err != nil {
return err
}
return driver.Start(c.App.Name)
}
// Destroy removes the container but keeps the data so it can be created again
func (c *Container) Destroy() error {
return nil
driver := c.getDriver()
return driver.Remove(c.App.Name)
}
// Delete removes both data and the container
func (c *Container) Delete() error {
return nil
err := c.Destroy()
if err != nil {
return err
}
volumePath := path.Join("/srv", c.App.Name)
err = removeDirectory(volumePath)
return err
}

8
go.mod
View file

@ -3,10 +3,14 @@ module github.com/rosti-cz/apps-api
go 1.14
require (
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v1.13.1
github.com/docker/go-connections v0.4.0
github.com/docker/go-units v0.4.0 // indirect
github.com/jinzhu/gorm v1.9.14
github.com/labstack/echo v3.3.10+incompatible
github.com/labstack/gommon v0.3.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
github.com/docker/docker v1.13.1
github.com/docker/go-connections v0.4.0
)

12
go.sum
View file

@ -2,6 +2,14 @@ github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBK
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
@ -25,6 +33,10 @@ github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA=
github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=

60
main.go
View file

@ -6,6 +6,7 @@ import (
"github.com/labstack/echo"
"github.com/rosti-cz/apps-api/apps"
"github.com/rosti-cz/apps-api/common"
"github.com/rosti-cz/apps-api/docker"
"github.com/rosti-cz/apps-api/nodes"
)
@ -15,6 +16,8 @@ const JSONIndent = " "
func main() {
// Close database at the end
db := common.GetDBConnection()
// db.AutoMigrate(apps.Label{})
// db.AutoMigrate(apps.App{})
defer db.Close()
// API
@ -59,7 +62,21 @@ func main() {
return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
}
return c.JSON(http.StatusOK, map[string]string{})
container := docker.Container{
App: app,
}
err = container.Create()
if err != nil {
return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
}
err = container.Start()
if err != nil {
return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
}
return c.JSON(http.StatusOK, Message{Message: "ok"})
})
// Update existing app
@ -72,7 +89,7 @@ func main() {
return c.JSONPretty(http.StatusBadRequest, Message{Message: err.Error()}, JSONIndent)
}
err = apps.Update(name, app.SSHPort, app.HTTPPort, app.Image, app.CPU, app.Memory)
appPointer, err := apps.Update(name, app.SSHPort, app.HTTPPort, app.Image, app.CPU, app.Memory)
if err != nil {
if validationError, ok := err.(apps.ValidationError); ok {
return c.JSONPretty(http.StatusBadRequest, Message{Errors: validationError.Errors}, JSONIndent)
@ -80,6 +97,27 @@ func main() {
return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
}
app = *appPointer
container := docker.Container{
App: app,
}
err = container.Destroy()
if err != nil {
return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
}
err = container.Create()
if err != nil {
return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
}
err = container.Start()
if err != nil {
return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
}
return c.JSON(http.StatusOK, map[string]string{})
})
@ -90,7 +128,23 @@ func main() {
// Start existing app
e.PUT("/v1/apps/:name/start", func(c echo.Context) error {
return c.JSON(http.StatusOK, map[string]string{})
name := c.Param("name")
app, err := apps.Get(name)
if err != nil {
return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
}
container := docker.Container{
App: *app,
}
err = container.Start()
if err != nil {
return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent)
}
return c.JSON(http.StatusOK, Message{Message: "ok"})
})
// Stop existing app