Upgade of docker client, ...

Docker client upgrade,
Miner detection
Flags for apps

README.md

@@ -12,5 +12,8 @@ On Fedora run podman API:
 
     Root: sudo systemctl enable --now podman.socket
     Rootless: podman system service -t 0 --log-level=debug
+              set -x DOCKER_SOCKET unix:/run/user/1000/podman/podman.sock
+              # or
+              export DOCKER_SOCKET=unix:/run/user/1000/podman/podman.sock
 
 

apps/main.go

@@ -117,7 +117,7 @@ func (a *AppsProcessor) UpdateResources(name string, state string, CPUUsage floa
 		MemoryUsage:     memory,
 		DiskUsageBytes:  diskUsageBytes,
 		DiskUsageInodes: diskUsageInodes,
-		Flags:           flags,
+		Flags:           flags.String(),
 	}).Error
 	return err
 }

apps/types.go

@@ -86,7 +86,7 @@ type App struct {
 	// Disk usage in inodes
 	DiskUsageInodes int `json:"disk_usage_inodes"`
 	// Flags from detector of problems in the container
-	Flags detector.Flags `json:"flags" gorm:"type:text[]"`
+	Flags string `json:"flags"` // flags are separated by comma
 
 	// this is gathered in docker package and has to be assembled externally
 	Techs       AppTechs `json:"techs,omitempty" gorm:"-"`        // list of available technologies in the image

common/config.go

@@ -8,6 +8,7 @@ import (
 
 // Config keeps info about configuration of this daemon
 type Config struct {
+	DockerSocket         string `envconfig:"DOCKER_SOCKET" default:"unix://var/run/docker.sock"`
 	Token                string `envconfig:"TOKEN" required:"true"`
 	AppsPath             string `envconfig:"APPS_PATH" default:"/srv"`            // Where applications are located
 	AppsBindIPHTTP       string `envconfig:"APPS_BIND_IP_HTTP" default:"0.0.0.0"` // On what IP apps' HTTP port gonna be bound

containers/docker.go

@@ -1,4 +1,4 @@
-package docker
+package containers
 
 import (
 	"context"
@@ -18,6 +18,7 @@ import (
 	"github.com/docker/docker/api/types/network"
 	dockerClient "github.com/docker/docker/client"
 	"github.com/docker/go-connections/nat"
+	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/rosti-cz/node-api/detector"
 )
 
@@ -27,28 +28,18 @@ const statsDelay = 1
 // Docker timeout
 const dockerTimeout = 10
 
-// DOCKER_SOCK tells where to connect to docker, it will be always local sock
+// DOCKER_API_VERSION set API version of Docker, 1.41 is needed for the platform struct
-const dockerSock = "/var/run/docker.sock"
+const dockerAPIVersion = "1.41"
-const podmanSock = "/run/user/1000/podman/podman.sock"
-
-// DOCKER_API_VERSION set API version of Docker, 1.40 belongs to Docker 19.03.11
-const dockerAPIVersion = "1.38"
 
 // Driver keeps everything for connection to Docker
 type Driver struct {
+	DockerSock string // Docker socket
 	BindIPHTTP string // IP to which containers are bound
 	BindIPSSH  string // IP to which containers are bound
 }
 
 func (d *Driver) getClient() (*dockerClient.Client, error) {
-	var connectTo string
+	cli, err := dockerClient.NewClient(d.DockerSock, dockerAPIVersion, nil, nil)
-	if _, err := os.Stat(podmanSock); !os.IsNotExist(err) {
-		connectTo = podmanSock
-	} else {
-		connectTo = dockerSock
-	}
-
-	cli, err := dockerClient.NewClient("unix://"+connectTo, dockerAPIVersion, nil, nil)
 	if err != nil {
 		return cli, fmt.Errorf("get docker client error: %v", err)
 	}
@@ -371,6 +362,10 @@ func (d *Driver) Create(name string, image string, volumePath string, HTTPPort i
 			},
 		},
 		&network.NetworkingConfig{},
+		&specs.Platform{
+			Architecture: "amd64",
+			OS:           "linux",
+		},
 		name,
 	)
 	if err != nil {
@@ -423,11 +418,9 @@ func (d *Driver) Exec(name string, cmd []string, stdin string, env []string, att
 		return &[]byte{}, err
 	}
 
-	respAttach, err := cli.ContainerExecAttach(ctx, resp.ID, types.ExecConfig{
+	respAttach, err := cli.ContainerExecAttach(ctx, resp.ID, types.ExecStartCheck{
-		AttachStdin:  stdinEnabled,
+		Detach: false,
-		AttachStdout: attachStdout,
+		Tty:    true,
-		AttachStderr: false,
-		Tty:          attachStdout,
 	})
 	if err != nil {
 		return &[]byte{}, err

containers/docker_test.go

@@ -1,4 +1,4 @@
-package docker
+package containers
 
 import (
 	"testing"
@@ -8,6 +8,7 @@ import (
 
 func TestGetProcesses(t *testing.T) {
 	driver := Driver{
+		DockerSock: "/run/user/1000/podman/podman.sock",
 		BindIPHTTP: "127.0.0.1",
 		BindIPSSH:  "127.0.0.1",
 	}

containers/stats.go

@@ -1,4 +1,4 @@
-package docker
+package containers
 
 // ContainerStats contains fields returned in docker stats function stream
 type ContainerStats struct {

containers/tools.go

@@ -1,4 +1,4 @@
-package docker
+package containers
 
 import (
 	"bytes"

containers/types.go

@@ -1,4 +1,4 @@
-package docker
+package containers
 
 import (
 	"errors"
@@ -8,6 +8,7 @@ import (
 
 	"github.com/rosti-cz/node-api/apps"
 	"github.com/rosti-cz/node-api/common"
+	"github.com/rosti-cz/node-api/detector"
 )
 
 // username in the containers under which all containers run
@@ -28,6 +29,7 @@ type Container struct {
 func (c *Container) getDriver() *Driver {
 	config := common.GetConfig()
 	driver := &Driver{
+		DockerSock: config.DockerSocket,
 		BindIPHTTP: config.AppsBindIPHTTP,
 		BindIPSSH:  config.AppsBindIPSSH,
 	}
@@ -65,6 +67,16 @@ func (c *Container) GetState() (*apps.AppState, error) {
 		return nil, err
 	}
 
+	processes, err := c.GetSystemProcesses()
+	if err != nil {
+		return nil, err
+	}
+
+	flags, err := detector.Check(processes)
+	if err != nil {
+		return nil, err
+	}
+
 	state := apps.AppState{
 		State: status,
 		// CPUUsage:        cpu,
@@ -73,6 +85,7 @@ func (c *Container) GetState() (*apps.AppState, error) {
 		MemoryUsage:     -1.0,
 		DiskUsageBytes:  bytes,
 		DiskUsageInodes: inodes,
+		Flags:           flags,
 	}
 
 	return &state, nil
@@ -287,6 +300,14 @@ func (c *Container) GetProcessList() (*[]Process, error) {
 	return &processes, nil
 }
 
+// GetSystemProcesses return list of running system processes
+func (c *Container) GetSystemProcesses() ([]string, error) {
+	driver := c.getDriver()
+
+	processes, err := driver.GetProcesses(c.App.Name)
+	return processes, err
+}
+
 // GetPrimaryTech returns primary tech configured in the container.
 func (c *Container) GetPrimaryTech() (apps.AppTech, error) {
 	tech := apps.AppTech{}

contrib/test_messages.sh

@@ -3,4 +3,5 @@
 nats pub --count 1 admin.apps.node-x.requests -w '{"type": "list"}'
 nats pub --count 1 admin.apps.node-x.requests -w '{"type": "get", "app_name": "test_1234"}'
 
-nats pub --count 1 admin.apps.node-x.requests -w '{"type": "create", "app_name": "natstest_0004", "payload": {"name": "natstest_0004", "image": "docker.io/rosti/runtime:2020.09-1", "cpu": 50, "memory": 256, "ssh_port": 20004, "http_port": 30004}}'
+nats pub --count 1 admin.apps.node-local.requests -w '{"type": "create", "name": "testrunnats_5372", "payload": "{\"name\": \"testrunnats_5372\", \"image\": \"docker.io/rosti/runtime:2022.01-1\", \"cpu\": 50, \"memory\": 512, \"ssh_port\": 25372, \"http_port\": 35372}"}'
+nats pub --count 1 admin.apps.node-local.requests -w '{"type": "create", "name": "testrunnats_5372", "payload": "{\"name\": \"testrunnats_5372\", \"image\": \"harbor.hq.rosti.cz/public/runtime:2022.01-1\", \"cpu\": 50, \"memory\": 512, \"ssh_port\": 25372, \"http_port\": 35372}"}'

detector/main.go

@@ -1,13 +1,17 @@
 package detector
 
 import (
-	"fmt"
 	"regexp"
+	"strings"
 )
 
 // Flags is list of strings describing problems found among the processes
 type Flags []string
 
+func (f *Flags) String() string {
+	return strings.Join(*f, ",")
+}
+
 // Check goes over patterns and tries to flag given list of processes with flags.
 func Check(processes []string) (Flags, error) {
 	flags := Flags{}
@@ -24,7 +28,6 @@ func Check(processes []string) (Flags, error) {
 					tmpFlags[flag] = true
 					break
 				}
-				fmt.Println(process, pattern, flag)
 			}
 		}
 	}

go.mod

@@ -5,25 +5,28 @@ go 1.14
 require (
 	github.com/Microsoft/go-winio v0.4.18 // indirect
 	github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect
-	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/containerd/containerd v1.5.9 // indirect
-	github.com/docker/docker v1.13.1
+	github.com/docker/docker v20.10.12+incompatible
 	github.com/docker/go-connections v0.4.0
-	github.com/docker/go-units v0.4.0 // indirect
 	github.com/go-ole/go-ole v1.2.5 // indirect
 	github.com/gobuffalo/packr v1.30.1
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/jinzhu/gorm v1.9.14
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/labstack/echo v3.3.10+incompatible
 	github.com/labstack/gommon v0.3.0 // indirect
 	github.com/mholt/archiver/v3 v3.5.0
 	github.com/minio/minio-go/v7 v7.0.14
+	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/nats-io/nats-server/v2 v2.6.1 // indirect
 	github.com/nats-io/nats.go v1.12.3
-	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.2
 	github.com/pkg/errors v0.9.1
 	github.com/satori/go.uuid v1.2.0
 	github.com/shirou/gopsutil v2.20.6+incompatible
-	github.com/stretchr/testify v1.4.0
+	github.com/stretchr/testify v1.7.0
-	google.golang.org/protobuf v1.27.1 // indirect
+	google.golang.org/grpc v1.44.0 // indirect
+	gotest.tools/v3 v3.1.0 // indirect
 )

handlers.go

@@ -10,7 +10,7 @@ import (
 	"github.com/labstack/echo"
 	"github.com/rosti-cz/node-api/apps"
 	"github.com/rosti-cz/node-api/common"
-	"github.com/rosti-cz/node-api/docker"
+	docker "github.com/rosti-cz/node-api/containers"
 	"github.com/rosti-cz/node-api/node"
 )
 

handlers_nats.go

@@ -22,7 +22,8 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rosti-cz/node-api/apps"
 	"github.com/rosti-cz/node-api/common"
-	"github.com/rosti-cz/node-api/docker"
+	docker "github.com/rosti-cz/node-api/containers"
+	"github.com/rosti-cz/node-api/detector"
 	"github.com/rosti-cz/node-api/node"
 )
 
@@ -163,6 +164,19 @@ func getEventHandler(m *nats.Msg, message *RequestMessage) error {
 			log.Printf("backend error: %v\n", err)
 			return errorReplyFormater(m, "backend error", err)
 		}
+
+		processList, err := container.GetSystemProcesses()
+		if err != nil {
+			log.Printf("backend error: %v\n", err)
+			return errorReplyFormater(m, "backend error", err)
+		}
+
+		flags, err := detector.Check(processList)
+		if err != nil {
+			log.Printf("backend error: %v\n", err)
+			return errorReplyFormater(m, "backend error", err)
+		}
+		app.Flags = flags.String()
 	}
 
 	// Assembling reply message
@@ -191,7 +205,7 @@ func createEventHandler(m *nats.Msg, message *RequestMessage) error {
 	body := []byte(message.Payload)
 	err := json.Unmarshal(body, &appTemplate)
 	if err != nil {
-		log.Println("ERROR create application problem: " + err.Error())
+		log.Println("ERROR create application problem (unmarshal): " + err.Error())
 		publish(message.AppName, "payload parsing problem", true)
 		return err
 	}
@@ -202,11 +216,11 @@ func createEventHandler(m *nats.Msg, message *RequestMessage) error {
 	err = processor.New(message.AppName, appTemplate.SSHPort, appTemplate.HTTPPort, appTemplate.Image, appTemplate.CPU, appTemplate.Memory)
 	if err != nil {
 		if validationError, ok := err.(apps.ValidationError); ok {
-			log.Println("ERROR create application problem: " + validationError.Error())
+			log.Println("ERROR create application problem (validation problem): " + validationError.Error())
 			publish(message.AppName, "validation problem", true)
 			return err
 		}
-		log.Println("ERROR create application problem: " + err.Error())
+		log.Println("ERROR create application problem (processor.New): " + err.Error())
 		publish(message.AppName, "backend problem", true)
 		return err
 	}
@@ -217,7 +231,7 @@ func createEventHandler(m *nats.Msg, message *RequestMessage) error {
 
 	err = container.Create()
 	if err != nil {
-		log.Println("ERROR create application problem: " + err.Error())
+		log.Println("ERROR create application problem (container.Create): " + err.Error())
 		publish(message.AppName, "backend problem", true)
 		return err
 	}
@@ -237,7 +251,7 @@ func createEventHandler(m *nats.Msg, message *RequestMessage) error {
 
 	err = container.Start()
 	if err != nil {
-		log.Println("ERROR create application problem: " + err.Error())
+		log.Println("ERROR create application problem (container.Start): " + err.Error())
 		publish(message.AppName, "backend problem", true)
 		return err
 	}
@@ -254,7 +268,7 @@ func registerEventHandler(m *nats.Msg, message *RequestMessage) error {
 	body := []byte(message.Payload)
 	err := json.Unmarshal(body, &appTemplate)
 	if err != nil {
-		log.Println("ERROR create application problem: " + err.Error())
+		log.Println("ERROR create application problem (Unmarshal): " + err.Error())
 		publish(message.AppName, "payload parsing problem", true)
 		return err
 	}
@@ -265,11 +279,11 @@ func registerEventHandler(m *nats.Msg, message *RequestMessage) error {
 	err = processor.New(message.AppName, appTemplate.SSHPort, appTemplate.HTTPPort, appTemplate.Image, appTemplate.CPU, appTemplate.Memory)
 	if err != nil {
 		if validationError, ok := err.(apps.ValidationError); ok {
-			log.Println("ERROR create application problem: " + validationError.Error())
+			log.Println("ERROR create application problem (validation): " + validationError.Error())
 			publish(message.AppName, "validation problem", true)
 			return err
 		}
-		log.Println("ERROR create application problem: " + err.Error())
+		log.Println("ERROR create application problem (processor.New): " + err.Error())
 		publish(message.AppName, "backend problem", true)
 		return err
 	}
@@ -285,7 +299,7 @@ func updateEventHandler(m *nats.Msg, message *RequestMessage) error {
 	body := []byte(message.Payload)
 	err := json.Unmarshal(body, &appTemplate)
 	if err != nil {
-		log.Println("ERROR update application problem: " + err.Error())
+		log.Println("ERROR update application problem (unmarshal): " + err.Error())
 		publish(message.AppName, "payload parsing problem", true)
 		return err
 	}

stats.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/rosti-cz/node-api/apps"
 	"github.com/rosti-cz/node-api/common"
-	"github.com/rosti-cz/node-api/docker"
+	docker "github.com/rosti-cz/node-api/containers"
 )
 
 // updateUsage updates various resource usage of the container/app in the database

tools.go

@@ -10,7 +10,7 @@ import (
 	"github.com/nats-io/nats.go"
 	"github.com/rosti-cz/node-api/apps"
 	"github.com/rosti-cz/node-api/common"
-	"github.com/rosti-cz/node-api/docker"
+	docker "github.com/rosti-cz/node-api/containers"
 )
 
 func errorReplyFormater(m *nats.Msg, message string, err error) error {