Rosti/node-api
3
0
Fork 0
Code Issues 2 Pull Requests Actions Projects Releases 12 Wiki Activity

Upgade of docker client, ...
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source 
Docker client upgrade,
Miner detection
Flags for apps
This commit is contained in:
Adam Štrauch 2022-02-05 02:22:53 +01:00
parent 49f1a03b42
commit fdcdca237f
Signed by: cx
GPG Key ID: 018304FFA8988F8D
17 changed files with 955 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -12,5 +12,8 @@ On Fedora run podman API:
Root: sudo systemctl enable --now podman.socket
Rootless: podman system service -t 0 --log-level=debug
set -x DOCKER_SOCKET unix:/run/user/1000/podman/podman.sock
# or
export DOCKER_SOCKET=unix:/run/user/1000/podman/podman.sock

2
apps/main.go
View File

@ -117,7 +117,7 @@ func (a *AppsProcessor) UpdateResources(name string, state string, CPUUsage floa
MemoryUsage: memory,
DiskUsageBytes: diskUsageBytes,
DiskUsageInodes: diskUsageInodes,
Flags: flags,
Flags: flags.String(),
}).Error
return err
}

2
apps/types.go
View File

@ -86,7 +86,7 @@ type App struct {
// Disk usage in inodes
DiskUsageInodes int `json:"disk_usage_inodes"`
// Flags from detector of problems in the container
Flags detector.Flags `json:"flags" gorm:"type:text[]"`
Flags string `json:"flags"` // flags are separated by comma
// this is gathered in docker package and has to be assembled externally
Techs AppTechs `json:"techs,omitempty" gorm:"-"` // list of available technologies in the image

1
common/config.go
View File

@ -8,6 +8,7 @@ import (
// Config keeps info about configuration of this daemon
type Config struct {
DockerSocket string `envconfig:"DOCKER_SOCKET" default:"unix://var/run/docker.sock"`
Token string `envconfig:"TOKEN" required:"true"`
AppsPath string `envconfig:"APPS_PATH" default:"/srv"` // Where applications are located
AppsBindIPHTTP string `envconfig:"APPS_BIND_IP_HTTP" default:"0.0.0.0"` // On what IP apps' HTTP port gonna be bound

33
docker/docker.go → containers/docker.go
View File

@ -1,4 +1,4 @@
package docker
package containers
import (
"context"
@ -18,6 +18,7 @@ import (
"github.com/docker/docker/api/types/network"
dockerClient "github.com/docker/docker/client"
"github.com/docker/go-connections/nat"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/rosti-cz/node-api/detector"
)
@ -27,28 +28,18 @@ const statsDelay = 1
// Docker timeout
const dockerTimeout = 10
// DOCKER_SOCK tells where to connect to docker, it will be always local sock
const dockerSock = "/var/run/docker.sock"
const podmanSock = "/run/user/1000/podman/podman.sock"
// DOCKER_API_VERSION set API version of Docker, 1.40 belongs to Docker 19.03.11
const dockerAPIVersion = "1.38"
// DOCKER_API_VERSION set API version of Docker, 1.41 is needed for the platform struct
const dockerAPIVersion = "1.41"
// Driver keeps everything for connection to Docker
type Driver struct {
DockerSock string // Docker socket
BindIPHTTP string // IP to which containers are bound
BindIPSSH string // IP to which containers are bound
}
func (d *Driver) getClient() (*dockerClient.Client, error) {
var connectTo string
if _, err := os.Stat(podmanSock); !os.IsNotExist(err) {
connectTo = podmanSock
} else {
connectTo = dockerSock
}
cli, err := dockerClient.NewClient("unix://"+connectTo, dockerAPIVersion, nil, nil)
cli, err := dockerClient.NewClient(d.DockerSock, dockerAPIVersion, nil, nil)
if err != nil {
return cli, fmt.Errorf("get docker client error: %v", err)
}
@ -371,6 +362,10 @@ func (d *Driver) Create(name string, image string, volumePath string, HTTPPort i
},
},
&network.NetworkingConfig{},
&specs.Platform{
Architecture: "amd64",
OS: "linux",
},
name,
)
if err != nil {
@ -423,11 +418,9 @@ func (d *Driver) Exec(name string, cmd []string, stdin string, env []string, att
return &[]byte{}, err
}
respAttach, err := cli.ContainerExecAttach(ctx, resp.ID, types.ExecConfig{
AttachStdin: stdinEnabled,
AttachStdout: attachStdout,
AttachStderr: false,
Tty: attachStdout,
respAttach, err := cli.ContainerExecAttach(ctx, resp.ID, types.ExecStartCheck{
Detach: false,
Tty: true,
})
if err != nil {
return &[]byte{}, err

3
docker/docker_test.go → containers/docker_test.go
View File

@ -1,4 +1,4 @@
package docker
package containers
import (
"testing"
@ -8,6 +8,7 @@ import (
func TestGetProcesses(t *testing.T) {
driver := Driver{
DockerSock: "/run/user/1000/podman/podman.sock",
BindIPHTTP: "127.0.0.1",
BindIPSSH: "127.0.0.1",
}

2
docker/stats.go → containers/stats.go
View File

@ -1,4 +1,4 @@
package docker
package containers
// ContainerStats contains fields returned in docker stats function stream
type ContainerStats struct {

2
docker/tools.go → containers/tools.go
View File

@ -1,4 +1,4 @@
package docker
package containers
import (
"bytes"

23
docker/types.go → containers/types.go
View File

@ -1,4 +1,4 @@
package docker
package containers
import (
"errors"
@ -8,6 +8,7 @@ import (
"github.com/rosti-cz/node-api/apps"
"github.com/rosti-cz/node-api/common"
"github.com/rosti-cz/node-api/detector"
)
// username in the containers under which all containers run
@ -28,6 +29,7 @@ type Container struct {
func (c *Container) getDriver() *Driver {
config := common.GetConfig()
driver := &Driver{
DockerSock: config.DockerSocket,
BindIPHTTP: config.AppsBindIPHTTP,
BindIPSSH: config.AppsBindIPSSH,
}
@ -65,6 +67,16 @@ func (c *Container) GetState() (*apps.AppState, error) {
return nil, err
}
processes, err := c.GetSystemProcesses()
if err != nil {
return nil, err
}
flags, err := detector.Check(processes)
if err != nil {
return nil, err
}
state := apps.AppState{
State: status,
// CPUUsage: cpu,
@ -73,6 +85,7 @@ func (c *Container) GetState() (*apps.AppState, error) {
MemoryUsage: -1.0,
DiskUsageBytes: bytes,
DiskUsageInodes: inodes,
Flags: flags,
}
return &state, nil
@ -287,6 +300,14 @@ func (c *Container) GetProcessList() (*[]Process, error) {
return &processes, nil
}
// GetSystemProcesses return list of running system processes
func (c *Container) GetSystemProcesses() ([]string, error) {
driver := c.getDriver()
processes, err := driver.GetProcesses(c.App.Name)
return processes, err
}
// GetPrimaryTech returns primary tech configured in the container.
func (c *Container) GetPrimaryTech() (apps.AppTech, error) {
tech := apps.AppTech{}

3
contrib/test_messages.sh
View File

@ -3,4 +3,5 @@
nats pub --count 1 admin.apps.node-x.requests -w '{"type": "list"}'
nats pub --count 1 admin.apps.node-x.requests -w '{"type": "get", "app_name": "test_1234"}'
nats pub --count 1 admin.apps.node-x.requests -w '{"type": "create", "app_name": "natstest_0004", "payload": {"name": "natstest_0004", "image": "docker.io/rosti/runtime:2020.09-1", "cpu": 50, "memory": 256, "ssh_port": 20004, "http_port": 30004}}'
nats pub --count 1 admin.apps.node-local.requests -w '{"type": "create", "name": "testrunnats_5372", "payload": "{\"name\": \"testrunnats_5372\", \"image\": \"docker.io/rosti/runtime:2022.01-1\", \"cpu\": 50, \"memory\": 512, \"ssh_port\": 25372, \"http_port\": 35372}"}'
nats pub --count 1 admin.apps.node-local.requests -w '{"type": "create", "name": "testrunnats_5372", "payload": "{\"name\": \"testrunnats_5372\", \"image\": \"harbor.hq.rosti.cz/public/runtime:2022.01-1\", \"cpu\": 50, \"memory\": 512, \"ssh_port\": 25372, \"http_port\": 35372}"}'

7
detector/main.go
View File

@ -1,13 +1,17 @@
package detector
import (
"fmt"
"regexp"
"strings"
)
// Flags is list of strings describing problems found among the processes
type Flags []string
func (f *Flags) String() string {
return strings.Join(*f, ",")
}
// Check goes over patterns and tries to flag given list of processes with flags.
func Check(processes []string) (Flags, error) {
flags := Flags{}
@ -24,7 +28,6 @@ func Check(processes []string) (Flags, error) {
tmpFlags[flag] = true
break
}
fmt.Println(process, pattern, flag)
}
}
}

15
go.mod
View File

@ -5,25 +5,28 @@ go 1.14
require (
github.com/Microsoft/go-winio v0.4.18 // indirect
github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v1.13.1
github.com/containerd/containerd v1.5.9 // indirect
github.com/docker/docker v20.10.12+incompatible
github.com/docker/go-connections v0.4.0
github.com/docker/go-units v0.4.0 // indirect
github.com/go-ole/go-ole v1.2.5 // indirect
github.com/gobuffalo/packr v1.30.1
github.com/golang/protobuf v1.5.2 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/jinzhu/gorm v1.9.14
github.com/kelseyhightower/envconfig v1.4.0
github.com/labstack/echo v3.3.10+incompatible
github.com/labstack/gommon v0.3.0 // indirect
github.com/mholt/archiver/v3 v3.5.0
github.com/minio/minio-go/v7 v7.0.14
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/nats-io/nats-server/v2 v2.6.1 // indirect
github.com/nats-io/nats.go v1.12.3
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2
github.com/pkg/errors v0.9.1
github.com/satori/go.uuid v1.2.0
github.com/shirou/gopsutil v2.20.6+incompatible
github.com/stretchr/testify v1.4.0
google.golang.org/protobuf v1.27.1 // indirect
github.com/stretchr/testify v1.7.0
google.golang.org/grpc v1.44.0 // indirect
gotest.tools/v3 v3.1.0 // indirect
)

878
go.sum
View File

File diff suppressed because it is too large Load Diff

2
handlers.go
View File

@ -10,7 +10,7 @@ import (
"github.com/labstack/echo"
"github.com/rosti-cz/node-api/apps"
"github.com/rosti-cz/node-api/common"
"github.com/rosti-cz/node-api/docker"
docker "github.com/rosti-cz/node-api/containers"
"github.com/rosti-cz/node-api/node"
)

34
handlers_nats.go
View File

@ -22,7 +22,8 @@ import (
"github.com/pkg/errors"
"github.com/rosti-cz/node-api/apps"
"github.com/rosti-cz/node-api/common"
"github.com/rosti-cz/node-api/docker"
docker "github.com/rosti-cz/node-api/containers"
"github.com/rosti-cz/node-api/detector"
"github.com/rosti-cz/node-api/node"
)
@ -163,6 +164,19 @@ func getEventHandler(m *nats.Msg, message *RequestMessage) error {
log.Printf("backend error: %v\n", err)
return errorReplyFormater(m, "backend error", err)
}
processList, err := container.GetSystemProcesses()
if err != nil {
log.Printf("backend error: %v\n", err)
return errorReplyFormater(m, "backend error", err)
}
flags, err := detector.Check(processList)
if err != nil {
log.Printf("backend error: %v\n", err)
return errorReplyFormater(m, "backend error", err)
}
app.Flags = flags.String()
}
// Assembling reply message
@ -191,7 +205,7 @@ func createEventHandler(m *nats.Msg, message *RequestMessage) error {
body := []byte(message.Payload)
err := json.Unmarshal(body, &appTemplate)
if err != nil {
log.Println("ERROR create application problem: " + err.Error())
log.Println("ERROR create application problem (unmarshal): " + err.Error())
publish(message.AppName, "payload parsing problem", true)
return err
}
@ -202,11 +216,11 @@ func createEventHandler(m *nats.Msg, message *RequestMessage) error {
err = processor.New(message.AppName, appTemplate.SSHPort, appTemplate.HTTPPort, appTemplate.Image, appTemplate.CPU, appTemplate.Memory)
if err != nil {
if validationError, ok := err.(apps.ValidationError); ok {
log.Println("ERROR create application problem: " + validationError.Error())
log.Println("ERROR create application problem (validation problem): " + validationError.Error())
publish(message.AppName, "validation problem", true)
return err
}
log.Println("ERROR create application problem: " + err.Error())
log.Println("ERROR create application problem (processor.New): " + err.Error())
publish(message.AppName, "backend problem", true)
return err
}
@ -217,7 +231,7 @@ func createEventHandler(m *nats.Msg, message *RequestMessage) error {
err = container.Create()
if err != nil {
log.Println("ERROR create application problem: " + err.Error())
log.Println("ERROR create application problem (container.Create): " + err.Error())
publish(message.AppName, "backend problem", true)
return err
}
@ -237,7 +251,7 @@ func createEventHandler(m *nats.Msg, message *RequestMessage) error {
err = container.Start()
if err != nil {
log.Println("ERROR create application problem: " + err.Error())
log.Println("ERROR create application problem (container.Start): " + err.Error())
publish(message.AppName, "backend problem", true)
return err
}
@ -254,7 +268,7 @@ func registerEventHandler(m *nats.Msg, message *RequestMessage) error {
body := []byte(message.Payload)
err := json.Unmarshal(body, &appTemplate)
if err != nil {
log.Println("ERROR create application problem: " + err.Error())
log.Println("ERROR create application problem (Unmarshal): " + err.Error())
publish(message.AppName, "payload parsing problem", true)
return err
}
@ -265,11 +279,11 @@ func registerEventHandler(m *nats.Msg, message *RequestMessage) error {
err = processor.New(message.AppName, appTemplate.SSHPort, appTemplate.HTTPPort, appTemplate.Image, appTemplate.CPU, appTemplate.Memory)
if err != nil {
if validationError, ok := err.(apps.ValidationError); ok {
log.Println("ERROR create application problem: " + validationError.Error())
log.Println("ERROR create application problem (validation): " + validationError.Error())
publish(message.AppName, "validation problem", true)
return err
}
log.Println("ERROR create application problem: " + err.Error())
log.Println("ERROR create application problem (processor.New): " + err.Error())
publish(message.AppName, "backend problem", true)
return err
}
@ -285,7 +299,7 @@ func updateEventHandler(m *nats.Msg, message *RequestMessage) error {
body := []byte(message.Payload)
err := json.Unmarshal(body, &appTemplate)
if err != nil {
log.Println("ERROR update application problem: " + err.Error())
log.Println("ERROR update application problem (unmarshal): " + err.Error())
publish(message.AppName, "payload parsing problem", true)
return err
}

2
stats.go
View File

@ -5,7 +5,7 @@ import (
"github.com/rosti-cz/node-api/apps"
"github.com/rosti-cz/node-api/common"
"github.com/rosti-cz/node-api/docker"
docker "github.com/rosti-cz/node-api/containers"
)
// updateUsage updates various resource usage of the container/app in the database

2
tools.go
View File

@ -10,7 +10,7 @@ import (
"github.com/nats-io/nats.go"
"github.com/rosti-cz/node-api/apps"
"github.com/rosti-cz/node-api/common"
"github.com/rosti-cz/node-api/docker"
docker "github.com/rosti-cz/node-api/containers"
)
func errorReplyFormater(m *nats.Msg, message string, err error) error {