Refactoring

Discovery creation move into server package
Tests for server package

.gitignore

@@ -1,5 +1,6 @@
 # Files with secrets
 *secret*
+tmp/
 
 # Binaries
 lobby_*

daemon/config.go

@@ -4,23 +4,24 @@ import (
 	"log"
 
 	"github.com/kelseyhightower/envconfig"
+	"github.com/rosti-cz/server_lobby/server"
 )
 
 // Config keeps info about configuration of this daemon
 type Config struct {
-	Token                string   `envconfig:"TOKEN" required:"false"`                                            // Authentication token, if empty auth is disabled
+	Token                string        `envconfig:"TOKEN" required:"false"`                                            // Authentication token, if empty auth is disabled
-	Host                 string   `envconfig:"HOST" required:"false" default:"127.0.0.1"`                         // IP address used for the REST server to listen
+	Host                 string        `envconfig:"HOST" required:"false" default:"127.0.0.1"`                         // IP address used for the REST server to listen
-	Port                 uint16   `envconfig:"PORT" required:"false" default:"1313"`                              // Port related to the address above
+	Port                 uint16        `envconfig:"PORT" required:"false" default:"1313"`                              // Port related to the address above
-	NATSURL              string   `envconfig:"NATS_URL" required:"true"`                                          // NATS URL used to connect to the NATS server
+	NATSURL              string        `envconfig:"NATS_URL" required:"true"`                                          // NATS URL used to connect to the NATS server
-	NATSDiscoveryChannel string   `envconfig:"NATS_DISCOVERY_CHANNEL" required:"false" default:"lobby.discovery"` // Channel where the kepp alive packets are sent
+	NATSDiscoveryChannel string        `envconfig:"NATS_DISCOVERY_CHANNEL" required:"false" default:"lobby.discovery"` // Channel where the kepp alive packets are sent
-	Labels               []string `envconfig:"LABELS" required:"false" default:""`                                // List of labels
+	Labels               server.Labels `envconfig:"LABELS" required:"false" default:""`                                // List of labels
-	LabelsPath           string   `envconfig:"LABELS_PATH" required:"false" default:"/etc/lobby/labels"`          // Path where filesystem based labels are located
+	LabelsPath           string        `envconfig:"LABELS_PATH" required:"false" default:"/etc/lobby/labels"`          // Path where filesystem based labels are located
-	HostName             string   `envconfig:"HOSTNAME" required:"false"`                                         // Overrise local machine's hostname
+	HostName             string        `envconfig:"HOSTNAME" required:"false"`                                         // Overrise local machine's hostname
-	CleanEvery           uint     `envconfig:"CLEAN_EVERY" required:"false" default:"15"`                         // How often to clean the list of servers to get rid of the not alive ones
+	CleanEvery           uint          `envconfig:"CLEAN_EVERY" required:"false" default:"15"`                         // How often to clean the list of servers to get rid of the not alive ones
-	KeepAlive            uint     `envconfig:"KEEP_ALIVE" required:"false" default:"5"`                           // how often to send the keepalive message with all availabel information [secs]
+	KeepAlive            uint          `envconfig:"KEEP_ALIVE" required:"false" default:"5"`                           // how often to send the keepalive message with all availabel information [secs]
-	TTL                  uint     `envconfig:"TTL" required:"false" default:"30"`                                 // After how many secs is discovery record considered as invalid
+	TTL                  uint          `envconfig:"TTL" required:"false" default:"30"`                                 // After how many secs is discovery record considered as invalid
-	NodeExporterPort     uint     `envconfig:"NODE_EXPORTER_PORT" required:"false" default:"9100"`                // Default port where node_exporter listens on all registered servers
+	NodeExporterPort     uint          `envconfig:"NODE_EXPORTER_PORT" required:"false" default:"9100"`                // Default port where node_exporter listens on all registered servers
-	Register             bool     `envconfig:"REGISTER" required:"false" default:"true"`                          // If true (default) then local instance is registered with other instance (discovery packet is sent regularly)
+	Register             bool          `envconfig:"REGISTER" required:"false" default:"true"`                          // If true (default) then local instance is registered with other instance (discovery packet is sent regularly)
 }
 
 // GetConfig return configuration created based on environment variables

daemon/main.go

@@ -52,7 +52,7 @@ func cleanDiscoveryPool() {
 // sendGoodbyePacket is almost same as sendDiscoveryPacket but it's not running in loop
 // and it adds goodbye message so other nodes know this node is gonna die.
 func sendGoodbyePacket() {
-	discovery, err := getIdentification()
+	discovery, err := server.GetIdentification(config.HostName, config.Labels, config.LabelsPath)
 	if err != nil {
 		log.Printf("sending discovery identification error: %v\n", err)
 	}
@@ -66,7 +66,7 @@ func sendGoodbyePacket() {
 // sendDisoveryPacket sends discovery packet regularly so the network know we exist
 func sendDiscoveryPacket() {
 	for {
-		discovery, err := getIdentification()
+		discovery, err := server.GetIdentification(config.HostName, config.Labels, config.LabelsPath)
 		if err != nil {
 			log.Printf("sending discovery identification error: %v\n", err)
 		}

daemon/prometheus.go

@@ -39,8 +39,8 @@ func preparePrometheusOutput(name string, discoveries []server.Discovery) Promet
 
 		labels := map[string]string{}
 
-		for _, label := range discovery.FindLabels("prometheus:" + name) {
+		for _, label := range discovery.FindLabels("prometheus:" + name + ":") {
-			trimmed := strings.TrimPrefix(label, "prometheus:"+name+":")
+			trimmed := strings.TrimPrefix(label.String(), "prometheus:"+name+":")
 			parts := strings.SplitN(trimmed, ":", 2)
 			if len(parts) == 2 {
 				if parts[0] == "port" {
@@ -57,7 +57,7 @@ func preparePrometheusOutput(name string, discoveries []server.Discovery) Promet
 		// This has to be checked here again because FindLabels adds : at the end of the label name.
 		if !add {
 			for _, label := range discovery.Labels {
-				if label == "prometheus:"+name {
+				if label.String() == "prometheus:"+name {
 					add = true
 					break
 				}

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/nats-io/nats-server/v2 v2.4.0 // indirect
 	github.com/nats-io/nats.go v1.12.0
 	github.com/shirou/gopsutil/v3 v3.21.7
+	github.com/stretchr/testify v1.7.0
 	github.com/valyala/fasttemplate v1.2.1 // indirect
 	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect

go.sum

@@ -103,6 +103,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=

server/discovery.go

@@ -15,8 +15,8 @@ const TimeToLife = 60 // when server won't occur in the discovery channel longer
 
 // Discovery contains information about a single server and is used for server discovery
 type Discovery struct {
-	Hostname string   `json:"hostname"`
+	Hostname string `json:"hostname"`
-	Labels   []string `json:"labels"`
+	Labels   Labels `json:"labels"`
 
 	// For internal use to check if the server is still alive.
 	// Contains timestamp of the last check.
@@ -44,11 +44,12 @@ func (d *Discovery) Bytes() ([]byte, error) {
 	return data, err
 }
 
-// FindLabels returns list of labels with given prefix. For example "service:ns" has prefix "service"
+// FindLabels returns list of labels with given prefix. For example "service:ns" has prefix "service" or "service:".
-func (d *Discovery) FindLabels(prefix string) []string {
+// It doesn't have to be prefix, but for example "service:test" will match "service:test" and also "service:test2".
-	labels := []string{}
+func (d *Discovery) FindLabels(prefix string) Labels {
+	labels := Labels{}
 	for _, label := range d.Labels {
-		if strings.HasPrefix(label, prefix+":") {
+		if strings.HasPrefix(label.String(), prefix) {
 			labels = append(labels, label)
 		}
 	}
@@ -156,7 +157,7 @@ func (d *Discoveries) Filter(labelsFilter []string) []Discovery {
 			found = false
 			for _, label := range discovery.Labels {
 				for _, labelFilter := range labelsFilter {
-					if label == labelFilter {
+					if label.String() == labelFilter {
 						newSet = append(newSet, discovery)
 						found = true
 						break

server/discovery_test.go

@@ -0,0 +1,38 @@
+package server
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDiscovery(t *testing.T) {
+	now := time.Now().Unix()
+	now90 := now - 90
+
+	discovery := Discovery{
+		Hostname: "test.rosti.cz",
+		Labels: Labels{
+			Label("service:test"),
+			Label("test:123"),
+			Label("public_ip:1.2.3.4"),
+		},
+		LastCheck: now,
+	}
+
+	assert.True(t, discovery.IsAlive(), "discovery suppose to be alive")
+	discovery.LastCheck = now90
+	assert.False(t, discovery.IsAlive(), "discovery not suppose to be alive")
+	discovery.LastCheck = now
+
+	assert.Equal(t, Labels{Label("service:test")}, discovery.FindLabels("service"))
+	assert.Equal(t, nil, discovery.Validate()) // TODO: This needs more love
+
+	content, err := json.Marshal(&discovery)
+	assert.Nil(t, err)
+	content2, err := discovery.Bytes()
+	assert.Nil(t, err)
+	assert.Equal(t, content, content2)
+}

server/identification.go

@@ -1,89 +1,80 @@
-package main
+package server
 
 import (
-	"bufio"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
 	"path"
 	"strings"
 
-	"github.com/rosti-cz/server_lobby/server"
 	"github.com/shirou/gopsutil/v3/host"
 )
 
 // getIdentification assembles the discovery packet that contains hotname and set of labels describing a single server, in this case the local server.
-func getIdentification() (server.Discovery, error) {
+// Parameter initialLabels usually coming from configuration of the app.
-	discovery := server.Discovery{}
+// If hostname is empty it will be discovered automatically.
+func GetIdentification(hostname string, initialLabels Labels, labelsPath string) (Discovery, error) {
+	discovery := Discovery{}
 
-	localLabels, err := loadLocalLabels(config.Labels)
+	localLabels, err := loadLocalLabels(initialLabels, labelsPath)
 	if err != nil {
 		return discovery, err
 	}
 
-	if len(config.HostName) == 0 {
+	if len(hostname) == 0 {
 		info, err := host.Info()
 		if err != nil {
 			return discovery, err
 		}
 		discovery.Hostname = info.Hostname
 	} else {
-		discovery.Hostname = config.HostName
+		discovery.Hostname = hostname
 	}
 
-	discovery.Labels = append(config.Labels, localLabels...)
+	discovery.Labels = append(initialLabels, localLabels...)
 
 	return discovery, nil
 }
 
 // loadLocalLabels scans local directory where labels are stored and adds them to the labels configured as environment variables.
 // Filename in LabelsPath is not importent and each file can contain multiple labels, one per each line.
-func loadLocalLabels(skipLabels []string) ([]string, error) {
+func loadLocalLabels(skipLabels Labels, labelsPath string) (Labels, error) {
-	labels := []string{}
+	labels := Labels{}
 	var found bool
 
-	if _, err := os.Stat(config.LabelsPath); !os.IsNotExist(err) {
+	if _, err := os.Stat(labelsPath); !os.IsNotExist(err) {
-		files, err := ioutil.ReadDir(config.LabelsPath)
+		files, err := ioutil.ReadDir(labelsPath)
 		if err != nil {
 			return labels, err
 		}
 
 		for _, filename := range files {
-			fullPath := path.Join(config.LabelsPath, filename.Name())
+			fullPath := path.Join(labelsPath, filename.Name())
-			fp, err := os.OpenFile(fullPath, os.O_RDONLY, os.ModePerm)
+
+			content, err := os.ReadFile(fullPath)
 			if err != nil {
-				return labels, fmt.Errorf("open file error: %v", err)
+				return labels, fmt.Errorf("read file error: %v", err)
 
 			}
-			defer fp.Close()
+			fmt.Println(string(content))
 
-			rd := bufio.NewReader(fp)
+			for _, line := range strings.Split(string(content), "\n") {
-			for {
-				line, err := rd.ReadString('\n')
-				if err != nil {
-					if err == io.EOF {
-						break
-					}
-
-					return labels, fmt.Errorf("read file line error: %v", err)
-				}
 				line = strings.TrimSpace(line)
 				if len(line) > 0 {
 					found = false
 					for _, skipLabel := range skipLabels {
-						if skipLabel == line {
+						if skipLabel == Label(line) {
 							found = true
 							break
 						}
 					}
 					if !found {
-						labels = append(labels, line)
+						labels = append(labels, Label(line))
 					}
 				}
 			}
 		}
 	}
-
+	fmt.Println("LABELS", labels)
 	return labels, nil
 }

server/identification_test.go

@@ -0,0 +1,48 @@
+package server
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const tmpPath = "./tmp"
+const testLabelPath = tmpPath + "/labels"
+
+func TestGetIdentification(t *testing.T) {
+	discovery, err := GetIdentification("test.example.com", Labels{Label("service:test"), Label("test:1")}, testLabelPath)
+	assert.Nil(t, err)
+
+	assert.Equal(t, "test.example.com", discovery.Hostname)
+	assert.Equal(t, "service:test", discovery.Labels[0].String())
+
+	err = os.MkdirAll(testLabelPath, os.ModePerm)
+	assert.Nil(t, err)
+
+	err = os.WriteFile(testLabelPath+"/test", []byte("service:test2\npublic_ip:1.2.3.4"), 0644)
+	assert.Nil(t, err)
+
+	discovery, err = GetIdentification("test.example.com", Labels{Label("service:test"), Label("test:1")}, testLabelPath)
+	assert.Nil(t, err)
+
+	assert.Equal(t, Label("public_ip:1.2.3.4"), discovery.Labels[3])
+
+	os.RemoveAll(tmpPath)
+}
+
+func TestLoadLocalLabels(t *testing.T) {
+	err := os.MkdirAll(testLabelPath, os.ModePerm)
+	assert.Nil(t, err)
+
+	err = os.WriteFile(testLabelPath+"/test", []byte("service:test\npublic_ip:1.2.3.4"), 0644)
+	assert.Nil(t, err)
+
+	labels, err := loadLocalLabels(Labels{Label("service:test")}, testLabelPath)
+	assert.Nil(t, err)
+
+	assert.Equal(t, 1, len(labels))
+	assert.Equal(t, "public_ip:1.2.3.4", labels[0].String())
+
+	os.RemoveAll(tmpPath)
+}

server/types.go

@@ -0,0 +1,11 @@
+package server
+
+// Label keeps one piece of information about a single server
+type Label string
+
+func (l Label) String() string {
+	return string(l)
+}
+
+// Labels stores multiple Label records
+type Labels []Label