Refactoring of identification code, runtime labels

README.md

@@ -76,21 +76,22 @@ service file. It doesn't need to access almost anything in your system.
 
 There are other config directives you can use to fine-tune lobbyd to exactly what you need.
 
-| Environment variable   | Type   | Default           | Required | Note                                                                                                                                                    |
+| Environment variable    | Type   | Default           | Required | Note                                                                                                                                                    |
-| ---------------------- | ------ | ----------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| ----------------------- | ------ | ----------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| TOKEN                  | string |                   | no       | Authentication token for API, if empty auth is disabled                                                                                                 |
+| TOKEN                   | string |                   | no       | Authentication token for API, if empty auth is disabled                                                                                                 |
-| HOST                   | string | 127.0.0.1         | no       | IP address used for the REST server to listen                                                                                                           |
+| HOST                    | string | 127.0.0.1         | no       | IP address used for the REST server to listen                                                                                                           |
-| PORT                   | int    | 1313              | no       | Port related to the address above                                                                                                                       |
+| PORT                    | int    | 1313              | no       | Port related to the address above                                                                                                                       |
-| NATS_URL               | string |                   | yes      | NATS URL used to connect to the NATS server                                                                                                             |
+| NATS_URL                | string |                   | yes      | NATS URL used to connect to the NATS server                                                                                                             |
-| NATS_DISCOVERY_CHANNEL | string | lobby.discovery   | no       | Channel where the keep-alive packets are sent                                                                                                           |
+| NATS_DISCOVERY_CHANNEL  | string | lobby.discovery   | no       | Channel where the keep-alive packets are sent                                                                                                           |
-| LABELS                 | string |                   | no       | List of labels, labels should be separated by comma                                                                                                     |
+| LABELS                  | string |                   | no       | List of labels, labels should be separated by comma                                                                                                     |
-| LABELS_PATH            | string | /etc/lobby/labels | no       | Path where filesystem based labels are located, one label per line, filename is not important for lobby                                                 |
+| LABELS_PATH             | string | /etc/lobby/labels | no       | Path where filesystem based labels are located, one label per line, filename is not important for lobby                                                 |
-| HOSTNAME               | string |                   | no       | Override local machine's hostname                                                                                                                       |
+| RUNTIME_LABELS_FILENAME | string | _runtime          | no       | Filename for file created in LabelsPath where runtime labels will be added                                                                              |
-| CLEAN_EVERY            | int    | 15                | no       | How often to clean the list of discovered servers to get rid of the not alive ones [secs]                                                               |
+| HOSTNAME                | string |                   | no       | Override local machine's hostname                                                                                                                       |
-| KEEP_ALIVE             | int    | 5                 | no       | how often to send the keep-alive discovery message with all available information [secs]                                                                |
+| CLEAN_EVERY             | int    | 15                | no       | How often to clean the list of discovered servers to get rid of the not alive ones [secs]                                                               |
-| TTL                    | int    | 30                | no       | After how many secs is discovery record considered as invalid                                                                                           |
+| KEEP_ALIVE              | int    | 5                 | no       | how often to send the keep-alive discovery message with all available information [secs]                                                                |
-| NODE_EXPORTER_PORT     | int    | 9100              | no       | Default port where node_exporter listens on all registered servers, this is used when the special prometheus labels doesn't contain port                |
+| TTL                     | int    | 30                | no       | After how many secs is discovery record considered as invalid                                                                                           |
-| REGISTER               | bool   | true              | no       | If true (default) then local instance is registered with other instance (discovery packet is sent regularly), if false the daemon runs only as a client |
+| NODE_EXPORTER_PORT      | int    | 9100              | no       | Default port where node_exporter listens on all registered servers, this is used when the special prometheus labels doesn't contain port                |
+| REGISTER                | bool   | true              | no       | If true (default) then local instance is registered with other instance (discovery packet is sent regularly), if false the daemon runs only as a client |
 
 ### Service discovery for Prometheus
 
@@ -141,10 +142,15 @@ At least one prometheus label has to be set to export the monitoring service in
 
 So far the REST API is super simple and it has only two endpoints:
 
-    GET /                     # Returns list of all discovered servers and their labels.
+    GET /                                # Same as /v1/discoveries
-    GET /v1/                  # Same as /
+    GET /v1/discovery                    # Returns current local discovery packet
-    GET /v1/?labels=LABELS    # output will be filtered based on one or multiple labels separated by comma
+    GET /v1/discoveries                  # Returns list of all discovered servers and their labels.
-    GET /v1/prometheus/:name  # Generates output for Prometheus's SD config, name is group of the monitoring services described above.
+    GET /v1/discoveries?labels=LABELS    # output will be filtered based on one or multiple labels separated by comma
+    GET /v1/prometheus/:name             # Generates output for Prometheus's SD config, name is group of the monitoring services described above.
+    POST /v1/labels                      # Add runtime labels that will persist over daemon restarts. Labels should be in the body of the request, one line per one label.
+    DELETE /v1/labels                    # Delete runtime labels. One label per line. Can't affect the labels from environment variables or labels added from the LabelPath.
+
+If there is an error the error message is returned as plain text.
 
 ## TODO
 
@@ -152,6 +158,7 @@ So far the REST API is super simple and it has only two endpoints:
 * [ ] Command hooks - script or list of scripts that are triggered when discovery status has changed
 * [ ] Support for multiple active backend drivers
 * [ ] SNS driver
+* [ ] API to allow add labels at runtime
 
 
 

daemon/config.go

@@ -9,19 +9,20 @@ import (
 
 // Config keeps info about configuration of this daemon
 type Config struct {
-	Token                string        `envconfig:"TOKEN" required:"false"`                                            // Authentication token, if empty auth is disabled
+	Token                 string        `envconfig:"TOKEN" required:"false"`                                            // Authentication token, if empty auth is disabled
-	Host                 string        `envconfig:"HOST" required:"false" default:"127.0.0.1"`                         // IP address used for the REST server to listen
+	Host                  string        `envconfig:"HOST" required:"false" default:"127.0.0.1"`                         // IP address used for the REST server to listen
-	Port                 uint16        `envconfig:"PORT" required:"false" default:"1313"`                              // Port related to the address above
+	Port                  uint16        `envconfig:"PORT" required:"false" default:"1313"`                              // Port related to the address above
-	NATSURL              string        `envconfig:"NATS_URL" required:"true"`                                          // NATS URL used to connect to the NATS server
+	NATSURL               string        `envconfig:"NATS_URL" required:"true"`                                          // NATS URL used to connect to the NATS server
-	NATSDiscoveryChannel string        `envconfig:"NATS_DISCOVERY_CHANNEL" required:"false" default:"lobby.discovery"` // Channel where the kepp alive packets are sent
+	NATSDiscoveryChannel  string        `envconfig:"NATS_DISCOVERY_CHANNEL" required:"false" default:"lobby.discovery"` // Channel where the kepp alive packets are sent
-	Labels               server.Labels `envconfig:"LABELS" required:"false" default:""`                                // List of labels
+	Labels                server.Labels `envconfig:"LABELS" required:"false" default:""`                                // List of labels
-	LabelsPath           string        `envconfig:"LABELS_PATH" required:"false" default:"/etc/lobby/labels"`          // Path where filesystem based labels are located
+	LabelsPath            string        `envconfig:"LABELS_PATH" required:"false" default:"/etc/lobby/labels"`          // Path where filesystem based labels are located
-	HostName             string        `envconfig:"HOSTNAME" required:"false"`                                         // Overrise local machine's hostname
+	RuntimeLabelsFilename string        `envconfig:"RUNTIME_LABELS_FILENAME" required:"false" default:"_runtime"`       // Filename for file created in LabelsPath where runtime labels will be added
-	CleanEvery           uint          `envconfig:"CLEAN_EVERY" required:"false" default:"15"`                         // How often to clean the list of servers to get rid of the not alive ones
+	HostName              string        `envconfig:"HOSTNAME" required:"false"`                                         // Overrise local machine's hostname
-	KeepAlive            uint          `envconfig:"KEEP_ALIVE" required:"false" default:"5"`                           // how often to send the keepalive message with all availabel information [secs]
+	CleanEvery            uint          `envconfig:"CLEAN_EVERY" required:"false" default:"15"`                         // How often to clean the list of servers to get rid of the not alive ones
-	TTL                  uint          `envconfig:"TTL" required:"false" default:"30"`                                 // After how many secs is discovery record considered as invalid
+	KeepAlive             uint          `envconfig:"KEEP_ALIVE" required:"false" default:"5"`                           // how often to send the keepalive message with all availabel information [secs]
-	NodeExporterPort     uint          `envconfig:"NODE_EXPORTER_PORT" required:"false" default:"9100"`                // Default port where node_exporter listens on all registered servers
+	TTL                   uint          `envconfig:"TTL" required:"false" default:"30"`                                 // After how many secs is discovery record considered as invalid
-	Register             bool          `envconfig:"REGISTER" required:"false" default:"true"`                          // If true (default) then local instance is registered with other instance (discovery packet is sent regularly)
+	NodeExporterPort      uint          `envconfig:"NODE_EXPORTER_PORT" required:"false" default:"9100"`                // Default port where node_exporter listens on all registered servers
+	Register              bool          `envconfig:"REGISTER" required:"false" default:"true"`                          // If true (default) then local instance is registered with other instance (discovery packet is sent regularly)
 }
 
 // GetConfig return configuration created based on environment variables

daemon/handlers.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"fmt"
+	"io/ioutil"
 	"net/http"
 	"strings"
 
@@ -30,3 +32,54 @@ func prometheusHandler(c echo.Context) error {
 
 	return c.JSONPretty(http.StatusOK, services, "  ")
 }
+
+func getIdentificationHandler(c echo.Context) error {
+	discovery, err := localHost.GetIdentification()
+	if err != nil {
+		return c.String(http.StatusInternalServerError, fmt.Sprintf("gathering identification info error: %v\n", err))
+	}
+
+	return c.JSONPretty(http.StatusOK, discovery, "  ")
+}
+
+func addLabelsHandler(c echo.Context) error {
+	body, err := ioutil.ReadAll(c.Request().Body)
+	if err != nil {
+		return c.String(http.StatusBadRequest, fmt.Sprintf("reading request body error: %v\n", err))
+	}
+
+	labels := server.Labels{}
+
+	for _, label := range strings.Split(string(body), "\n") {
+		labels = append(labels, server.Label(label))
+	}
+
+	err = localHost.AddLabels(labels)
+
+	if err != nil {
+		return c.String(http.StatusBadRequest, err.Error())
+	}
+
+	return c.String(http.StatusOK, "OK")
+}
+
+func deleteLabelsHandler(c echo.Context) error {
+	body, err := ioutil.ReadAll(c.Request().Body)
+	if err != nil {
+		return c.String(http.StatusBadRequest, fmt.Sprintf("reading request body error: %v\n", err))
+	}
+
+	labels := server.Labels{}
+
+	for _, label := range strings.Split(string(body), "\n") {
+		labels = append(labels, server.Label(label))
+	}
+
+	err = localHost.DeleteLabels(labels)
+
+	if err != nil {
+		return c.String(http.StatusBadRequest, err.Error())
+	}
+
+	return c.String(http.StatusOK, "OK")
+}

daemon/main.go

@@ -18,6 +18,7 @@ import (
 
 var discoveryStorage server.Discoveries = server.Discoveries{}
 var driver common.Driver
+var localHost server.LocalHost
 
 var config Config
 
@@ -31,6 +32,14 @@ func init() {
 	discoveryStorage.LogChannel = make(chan string)
 	discoveryStorage.TTL = config.TTL
 
+	// localhost initization
+	localHost = server.LocalHost{
+		LabelsPath:            config.LabelsPath,
+		HostnameOverride:      config.HostName,
+		InitialLabels:         config.Labels,
+		RuntimeLabelsFilename: config.RuntimeLabelsFilename,
+	}
+
 	// Setup driver
 	driver = &nats_driver.Driver{
 		NATSUrl:              config.NATSURL,
@@ -52,7 +61,7 @@ func cleanDiscoveryPool() {
 // sendGoodbyePacket is almost same as sendDiscoveryPacket but it's not running in loop
 // and it adds goodbye message so other nodes know this node is gonna die.
 func sendGoodbyePacket() {
-	discovery, err := server.GetIdentification(config.HostName, config.Labels, config.LabelsPath)
+	discovery, err := localHost.GetIdentification()
 	if err != nil {
 		log.Printf("sending discovery identification error: %v\n", err)
 	}
@@ -66,7 +75,7 @@ func sendGoodbyePacket() {
 // sendDisoveryPacket sends discovery packet regularly so the network know we exist
 func sendDiscoveryPacket() {
 	for {
-		discovery, err := server.GetIdentification(config.HostName, config.Labels, config.LabelsPath)
+		discovery, err := localHost.GetIdentification()
 		if err != nil {
 			log.Printf("sending discovery identification error: %v\n", err)
 		}
@@ -139,7 +148,10 @@ func main() {
 
 	// Routes
 	e.GET("/", listHandler)
-	e.GET("/v1/", listHandler)
+	e.GET("/v1/discovery", getIdentificationHandler)
+	e.GET("/v1/discoveries", listHandler)
+	e.POST("/v1/labels", addLabelsHandler)
+	e.DELETE("/v1/labels", deleteLabelsHandler)
 	e.GET("/v1/prometheus/:name", prometheusHandler)
 
 	// ------------------------------

server/identification.go

@@ -10,59 +10,164 @@ import (
 	"github.com/shirou/gopsutil/v3/host"
 )
 
-// getIdentification assembles the discovery packet that contains hotname and set of labels describing a single server, in this case the local server.
+type LocalHost struct {
+	LabelsPath            string // Where labels are stored
+	RuntimeLabelsFilename string // Filename under which are runtime labels saved in LabelsPath
+	InitialLabels         Labels // this usually coming from the config
+	HostnameOverride      string // if not empty string hostname in the discovery packet will be replaced by this
+}
+
+// saveRuntimeLabels stores labels in the runtime filesname
+func (l *LocalHost) saveRuntimeLabels(labels Labels) error {
+	stringLabels := []string{}
+
+	for _, label := range labels {
+		stringLabels = append(stringLabels, label.String())
+	}
+
+	content := strings.Join(stringLabels, "\n")
+
+	err := os.WriteFile(path.Join(l.LabelsPath, l.RuntimeLabelsFilename), []byte(content), 0755)
+	return err
+}
+
+// getRuntimeLabels returns labels from the runtime filename
+func (l *LocalHost) getRuntimeLabels() (Labels, error) {
+	labels := Labels{}
+
+	content, err := os.ReadFile(path.Join(l.LabelsPath, l.RuntimeLabelsFilename))
+	if err != nil {
+		if strings.Contains(err.Error(), "no such file or directory") {
+			return labels, nil
+		}
+
+		return labels, err
+	}
+
+	for _, label := range strings.Split(string(content), "\n") {
+		labels = append(labels, Label(strings.TrimSpace(label)))
+	}
+
+	return labels, nil
+}
+
+// AddLabel adds runtime label into the LabelsPath directory
+func (l *LocalHost) AddLabels(labels Labels) error {
+	runtimeLabels, err := l.getRuntimeLabels()
+	if err != nil {
+		return fmt.Errorf("error while loading stored labels: %v", err)
+	}
+
+	var found bool
+
+	for _, label := range labels {
+		found = false
+
+		for _, runtimeLabel := range runtimeLabels {
+			if label == runtimeLabel {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			runtimeLabels = append(runtimeLabels, label)
+		}
+	}
+
+	err = l.saveRuntimeLabels(runtimeLabels)
+	if err != nil {
+		return fmt.Errorf("error while saving new set of labels: %v", err)
+	}
+
+	return nil
+}
+
+// DeleteLabels removed labels from LabelsPath directory. Only labels added this way can be deleted.
+func (l *LocalHost) DeleteLabels(labels Labels) error {
+	runtimeLabels, err := l.getRuntimeLabels()
+	if err != nil {
+		return fmt.Errorf("error while loading stored labels: %v", err)
+	}
+
+	newSet := Labels{}
+	var found bool
+
+	for _, runtimeLabel := range runtimeLabels {
+		found = false
+
+		for _, label := range labels {
+			if label == runtimeLabel {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			newSet = append(newSet, runtimeLabel)
+		}
+	}
+
+	err = l.saveRuntimeLabels(newSet)
+	if err != nil {
+		return fmt.Errorf("error while saving new set of labels: %v", err)
+	}
+
+	return nil
+}
+
+// GetIdentification assembles the discovery packet that contains hotname and set of labels describing a single server, in this case the local server.
 // Parameter initialLabels usually coming from configuration of the app.
 // If hostname is empty it will be discovered automatically.
-func GetIdentification(hostname string, initialLabels Labels, labelsPath string) (Discovery, error) {
+func (l *LocalHost) GetIdentification() (Discovery, error) {
 	discovery := Discovery{}
 
-	localLabels, err := loadLocalLabels(initialLabels, labelsPath)
+	localLabels, err := l.loadLocalLabels()
 	if err != nil {
 		return discovery, err
 	}
 
-	if len(hostname) == 0 {
+	if len(l.HostnameOverride) == 0 {
 		info, err := host.Info()
 		if err != nil {
 			return discovery, err
 		}
 		discovery.Hostname = info.Hostname
 	} else {
-		discovery.Hostname = hostname
+		discovery.Hostname = l.HostnameOverride
 	}
 
-	discovery.Labels = append(initialLabels, localLabels...)
+	discovery.Labels = append(l.InitialLabels, localLabels...)
 
 	return discovery, nil
 }
 
 // loadLocalLabels scans local directory where labels are stored and adds them to the labels configured as environment variables.
 // Filename in LabelsPath is not importent and each file can contain multiple labels, one per each line.
-func loadLocalLabels(skipLabels Labels, labelsPath string) (Labels, error) {
+func (l *LocalHost) loadLocalLabels() (Labels, error) {
 	labels := Labels{}
 	var found bool
 
-	if _, err := os.Stat(labelsPath); !os.IsNotExist(err) {
+	if _, err := os.Stat(l.LabelsPath); !os.IsNotExist(err) {
-		files, err := ioutil.ReadDir(labelsPath)
+		files, err := ioutil.ReadDir(l.LabelsPath)
 		if err != nil {
 			return labels, err
 		}
 
 		for _, filename := range files {
-			fullPath := path.Join(labelsPath, filename.Name())
+			fullPath := path.Join(l.LabelsPath, filename.Name())
 
 			content, err := os.ReadFile(fullPath)
 			if err != nil {
 				return labels, fmt.Errorf("read file error: %v", err)
 
 			}
-			fmt.Println(string(content))
 
 			for _, line := range strings.Split(string(content), "\n") {
 				line = strings.TrimSpace(line)
 				if len(line) > 0 {
 					found = false
-					for _, skipLabel := range skipLabels {
+					for _, skipLabel := range l.InitialLabels {
 						if skipLabel == Label(line) {
 							found = true
 							break
@@ -75,6 +180,5 @@ func loadLocalLabels(skipLabels Labels, labelsPath string) (Labels, error) {
 			}
 		}
 	}
-	fmt.Println("LABELS", labels)
 	return labels, nil
 }

server/identification_test.go

@@ -11,7 +11,13 @@ const tmpPath = "./tmp"
 const testLabelPath = tmpPath + "/labels"
 
 func TestGetIdentification(t *testing.T) {
-	discovery, err := GetIdentification("test.example.com", Labels{Label("service:test"), Label("test:1")}, testLabelPath)
+	localHost := LocalHost{
+		LabelsPath:       testLabelPath,
+		HostnameOverride: "test.example.com",
+		InitialLabels:    Labels{Label("service:test"), Label("test:1")},
+	}
+
+	discovery, err := localHost.GetIdentification()
 	assert.Nil(t, err)
 
 	assert.Equal(t, "test.example.com", discovery.Hostname)
@@ -23,7 +29,7 @@ func TestGetIdentification(t *testing.T) {
 	err = os.WriteFile(testLabelPath+"/test", []byte("service:test2\npublic_ip:1.2.3.4"), 0644)
 	assert.Nil(t, err)
 
-	discovery, err = GetIdentification("test.example.com", Labels{Label("service:test"), Label("test:1")}, testLabelPath)
+	discovery, err = localHost.GetIdentification()
 	assert.Nil(t, err)
 
 	assert.Equal(t, Label("public_ip:1.2.3.4"), discovery.Labels[3])
@@ -32,13 +38,19 @@ func TestGetIdentification(t *testing.T) {
 }
 
 func TestLoadLocalLabels(t *testing.T) {
+	localHost := LocalHost{
+		LabelsPath:       testLabelPath,
+		HostnameOverride: "test.example.com",
+		InitialLabels:    Labels{Label("service:test"), Label("test:1")},
+	}
+
 	err := os.MkdirAll(testLabelPath, os.ModePerm)
 	assert.Nil(t, err)
 
 	err = os.WriteFile(testLabelPath+"/test", []byte("service:test\npublic_ip:1.2.3.4"), 0644)
 	assert.Nil(t, err)
 
-	labels, err := loadLocalLabels(Labels{Label("service:test")}, testLabelPath)
+	labels, err := localHost.loadLocalLabels()
 	assert.Nil(t, err)
 
 	assert.Equal(t, 1, len(labels))

server/runtime.go

@@ -0,0 +1,9 @@
+package server
+
+func AddRuntimeLabel(label Label) error {
+	return nil
+}
+
+func RemoveRuntimeLabel(label Label) error {
+	return nil
+}