From 471f8bb17022d76ca55672c7f4aa24f2609c372f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20=C5=A0trauch?= Date: Sat, 4 Sep 2021 22:17:56 +0200 Subject: [PATCH] Refactoring of identification code, runtime labels --- README.md | 45 +++++++----- daemon/config.go | 27 +++---- daemon/handlers.go | 53 ++++++++++++++ daemon/main.go | 18 ++++- server/identification.go | 130 ++++++++++++++++++++++++++++++---- server/identification_test.go | 18 ++++- server/runtime.go | 9 +++ 7 files changed, 249 insertions(+), 51 deletions(-) create mode 100644 server/runtime.go diff --git a/README.md b/README.md index 5eac61e..dd8d85e 100644 --- a/README.md +++ b/README.md @@ -76,21 +76,22 @@ service file. It doesn't need to access almost anything in your system. There are other config directives you can use to fine-tune lobbyd to exactly what you need. -| Environment variable | Type | Default | Required | Note | -| ---------------------- | ------ | ----------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | -| TOKEN | string | | no | Authentication token for API, if empty auth is disabled | -| HOST | string | 127.0.0.1 | no | IP address used for the REST server to listen | -| PORT | int | 1313 | no | Port related to the address above | -| NATS_URL | string | | yes | NATS URL used to connect to the NATS server | -| NATS_DISCOVERY_CHANNEL | string | lobby.discovery | no | Channel where the keep-alive packets are sent | -| LABELS | string | | no | List of labels, labels should be separated by comma | -| LABELS_PATH | string | /etc/lobby/labels | no | Path where filesystem based labels are located, one label per line, filename is not important for lobby | -| HOSTNAME | string | | no | Override local machine's hostname | -| CLEAN_EVERY | int | 15 | no | How often to clean the list of discovered servers to get rid of the not alive ones [secs] | -| KEEP_ALIVE | int | 5 | no | how often to send the keep-alive discovery message with all available information [secs] | -| TTL | int | 30 | no | After how many secs is discovery record considered as invalid | -| NODE_EXPORTER_PORT | int | 9100 | no | Default port where node_exporter listens on all registered servers, this is used when the special prometheus labels doesn't contain port | -| REGISTER | bool | true | no | If true (default) then local instance is registered with other instance (discovery packet is sent regularly), if false the daemon runs only as a client | +| Environment variable | Type | Default | Required | Note | +| ----------------------- | ------ | ----------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| TOKEN | string | | no | Authentication token for API, if empty auth is disabled | +| HOST | string | 127.0.0.1 | no | IP address used for the REST server to listen | +| PORT | int | 1313 | no | Port related to the address above | +| NATS_URL | string | | yes | NATS URL used to connect to the NATS server | +| NATS_DISCOVERY_CHANNEL | string | lobby.discovery | no | Channel where the keep-alive packets are sent | +| LABELS | string | | no | List of labels, labels should be separated by comma | +| LABELS_PATH | string | /etc/lobby/labels | no | Path where filesystem based labels are located, one label per line, filename is not important for lobby | +| RUNTIME_LABELS_FILENAME | string | _runtime | no | Filename for file created in LabelsPath where runtime labels will be added | +| HOSTNAME | string | | no | Override local machine's hostname | +| CLEAN_EVERY | int | 15 | no | How often to clean the list of discovered servers to get rid of the not alive ones [secs] | +| KEEP_ALIVE | int | 5 | no | how often to send the keep-alive discovery message with all available information [secs] | +| TTL | int | 30 | no | After how many secs is discovery record considered as invalid | +| NODE_EXPORTER_PORT | int | 9100 | no | Default port where node_exporter listens on all registered servers, this is used when the special prometheus labels doesn't contain port | +| REGISTER | bool | true | no | If true (default) then local instance is registered with other instance (discovery packet is sent regularly), if false the daemon runs only as a client | ### Service discovery for Prometheus @@ -141,10 +142,15 @@ At least one prometheus label has to be set to export the monitoring service in So far the REST API is super simple and it has only two endpoints: - GET / # Returns list of all discovered servers and their labels. - GET /v1/ # Same as / - GET /v1/?labels=LABELS # output will be filtered based on one or multiple labels separated by comma - GET /v1/prometheus/:name # Generates output for Prometheus's SD config, name is group of the monitoring services described above. + GET / # Same as /v1/discoveries + GET /v1/discovery # Returns current local discovery packet + GET /v1/discoveries # Returns list of all discovered servers and their labels. + GET /v1/discoveries?labels=LABELS # output will be filtered based on one or multiple labels separated by comma + GET /v1/prometheus/:name # Generates output for Prometheus's SD config, name is group of the monitoring services described above. + POST /v1/labels # Add runtime labels that will persist over daemon restarts. Labels should be in the body of the request, one line per one label. + DELETE /v1/labels # Delete runtime labels. One label per line. Can't affect the labels from environment variables or labels added from the LabelPath. + +If there is an error the error message is returned as plain text. ## TODO @@ -152,6 +158,7 @@ So far the REST API is super simple and it has only two endpoints: * [ ] Command hooks - script or list of scripts that are triggered when discovery status has changed * [ ] Support for multiple active backend drivers * [ ] SNS driver +* [ ] API to allow add labels at runtime diff --git a/daemon/config.go b/daemon/config.go index 0c76431..c9d7899 100644 --- a/daemon/config.go +++ b/daemon/config.go @@ -9,19 +9,20 @@ import ( // Config keeps info about configuration of this daemon type Config struct { - Token string `envconfig:"TOKEN" required:"false"` // Authentication token, if empty auth is disabled - Host string `envconfig:"HOST" required:"false" default:"127.0.0.1"` // IP address used for the REST server to listen - Port uint16 `envconfig:"PORT" required:"false" default:"1313"` // Port related to the address above - NATSURL string `envconfig:"NATS_URL" required:"true"` // NATS URL used to connect to the NATS server - NATSDiscoveryChannel string `envconfig:"NATS_DISCOVERY_CHANNEL" required:"false" default:"lobby.discovery"` // Channel where the kepp alive packets are sent - Labels server.Labels `envconfig:"LABELS" required:"false" default:""` // List of labels - LabelsPath string `envconfig:"LABELS_PATH" required:"false" default:"/etc/lobby/labels"` // Path where filesystem based labels are located - HostName string `envconfig:"HOSTNAME" required:"false"` // Overrise local machine's hostname - CleanEvery uint `envconfig:"CLEAN_EVERY" required:"false" default:"15"` // How often to clean the list of servers to get rid of the not alive ones - KeepAlive uint `envconfig:"KEEP_ALIVE" required:"false" default:"5"` // how often to send the keepalive message with all availabel information [secs] - TTL uint `envconfig:"TTL" required:"false" default:"30"` // After how many secs is discovery record considered as invalid - NodeExporterPort uint `envconfig:"NODE_EXPORTER_PORT" required:"false" default:"9100"` // Default port where node_exporter listens on all registered servers - Register bool `envconfig:"REGISTER" required:"false" default:"true"` // If true (default) then local instance is registered with other instance (discovery packet is sent regularly) + Token string `envconfig:"TOKEN" required:"false"` // Authentication token, if empty auth is disabled + Host string `envconfig:"HOST" required:"false" default:"127.0.0.1"` // IP address used for the REST server to listen + Port uint16 `envconfig:"PORT" required:"false" default:"1313"` // Port related to the address above + NATSURL string `envconfig:"NATS_URL" required:"true"` // NATS URL used to connect to the NATS server + NATSDiscoveryChannel string `envconfig:"NATS_DISCOVERY_CHANNEL" required:"false" default:"lobby.discovery"` // Channel where the kepp alive packets are sent + Labels server.Labels `envconfig:"LABELS" required:"false" default:""` // List of labels + LabelsPath string `envconfig:"LABELS_PATH" required:"false" default:"/etc/lobby/labels"` // Path where filesystem based labels are located + RuntimeLabelsFilename string `envconfig:"RUNTIME_LABELS_FILENAME" required:"false" default:"_runtime"` // Filename for file created in LabelsPath where runtime labels will be added + HostName string `envconfig:"HOSTNAME" required:"false"` // Overrise local machine's hostname + CleanEvery uint `envconfig:"CLEAN_EVERY" required:"false" default:"15"` // How often to clean the list of servers to get rid of the not alive ones + KeepAlive uint `envconfig:"KEEP_ALIVE" required:"false" default:"5"` // how often to send the keepalive message with all availabel information [secs] + TTL uint `envconfig:"TTL" required:"false" default:"30"` // After how many secs is discovery record considered as invalid + NodeExporterPort uint `envconfig:"NODE_EXPORTER_PORT" required:"false" default:"9100"` // Default port where node_exporter listens on all registered servers + Register bool `envconfig:"REGISTER" required:"false" default:"true"` // If true (default) then local instance is registered with other instance (discovery packet is sent regularly) } // GetConfig return configuration created based on environment variables diff --git a/daemon/handlers.go b/daemon/handlers.go index 70f49a9..82a1299 100644 --- a/daemon/handlers.go +++ b/daemon/handlers.go @@ -1,6 +1,8 @@ package main import ( + "fmt" + "io/ioutil" "net/http" "strings" @@ -30,3 +32,54 @@ func prometheusHandler(c echo.Context) error { return c.JSONPretty(http.StatusOK, services, " ") } + +func getIdentificationHandler(c echo.Context) error { + discovery, err := localHost.GetIdentification() + if err != nil { + return c.String(http.StatusInternalServerError, fmt.Sprintf("gathering identification info error: %v\n", err)) + } + + return c.JSONPretty(http.StatusOK, discovery, " ") +} + +func addLabelsHandler(c echo.Context) error { + body, err := ioutil.ReadAll(c.Request().Body) + if err != nil { + return c.String(http.StatusBadRequest, fmt.Sprintf("reading request body error: %v\n", err)) + } + + labels := server.Labels{} + + for _, label := range strings.Split(string(body), "\n") { + labels = append(labels, server.Label(label)) + } + + err = localHost.AddLabels(labels) + + if err != nil { + return c.String(http.StatusBadRequest, err.Error()) + } + + return c.String(http.StatusOK, "OK") +} + +func deleteLabelsHandler(c echo.Context) error { + body, err := ioutil.ReadAll(c.Request().Body) + if err != nil { + return c.String(http.StatusBadRequest, fmt.Sprintf("reading request body error: %v\n", err)) + } + + labels := server.Labels{} + + for _, label := range strings.Split(string(body), "\n") { + labels = append(labels, server.Label(label)) + } + + err = localHost.DeleteLabels(labels) + + if err != nil { + return c.String(http.StatusBadRequest, err.Error()) + } + + return c.String(http.StatusOK, "OK") +} diff --git a/daemon/main.go b/daemon/main.go index f8de886..5398c74 100644 --- a/daemon/main.go +++ b/daemon/main.go @@ -18,6 +18,7 @@ import ( var discoveryStorage server.Discoveries = server.Discoveries{} var driver common.Driver +var localHost server.LocalHost var config Config @@ -31,6 +32,14 @@ func init() { discoveryStorage.LogChannel = make(chan string) discoveryStorage.TTL = config.TTL + // localhost initization + localHost = server.LocalHost{ + LabelsPath: config.LabelsPath, + HostnameOverride: config.HostName, + InitialLabels: config.Labels, + RuntimeLabelsFilename: config.RuntimeLabelsFilename, + } + // Setup driver driver = &nats_driver.Driver{ NATSUrl: config.NATSURL, @@ -52,7 +61,7 @@ func cleanDiscoveryPool() { // sendGoodbyePacket is almost same as sendDiscoveryPacket but it's not running in loop // and it adds goodbye message so other nodes know this node is gonna die. func sendGoodbyePacket() { - discovery, err := server.GetIdentification(config.HostName, config.Labels, config.LabelsPath) + discovery, err := localHost.GetIdentification() if err != nil { log.Printf("sending discovery identification error: %v\n", err) } @@ -66,7 +75,7 @@ func sendGoodbyePacket() { // sendDisoveryPacket sends discovery packet regularly so the network know we exist func sendDiscoveryPacket() { for { - discovery, err := server.GetIdentification(config.HostName, config.Labels, config.LabelsPath) + discovery, err := localHost.GetIdentification() if err != nil { log.Printf("sending discovery identification error: %v\n", err) } @@ -139,7 +148,10 @@ func main() { // Routes e.GET("/", listHandler) - e.GET("/v1/", listHandler) + e.GET("/v1/discovery", getIdentificationHandler) + e.GET("/v1/discoveries", listHandler) + e.POST("/v1/labels", addLabelsHandler) + e.DELETE("/v1/labels", deleteLabelsHandler) e.GET("/v1/prometheus/:name", prometheusHandler) // ------------------------------ diff --git a/server/identification.go b/server/identification.go index 00af0c9..5939bda 100644 --- a/server/identification.go +++ b/server/identification.go @@ -10,59 +10,164 @@ import ( "github.com/shirou/gopsutil/v3/host" ) -// getIdentification assembles the discovery packet that contains hotname and set of labels describing a single server, in this case the local server. +type LocalHost struct { + LabelsPath string // Where labels are stored + RuntimeLabelsFilename string // Filename under which are runtime labels saved in LabelsPath + InitialLabels Labels // this usually coming from the config + HostnameOverride string // if not empty string hostname in the discovery packet will be replaced by this +} + +// saveRuntimeLabels stores labels in the runtime filesname +func (l *LocalHost) saveRuntimeLabels(labels Labels) error { + stringLabels := []string{} + + for _, label := range labels { + stringLabels = append(stringLabels, label.String()) + } + + content := strings.Join(stringLabels, "\n") + + err := os.WriteFile(path.Join(l.LabelsPath, l.RuntimeLabelsFilename), []byte(content), 0755) + return err +} + +// getRuntimeLabels returns labels from the runtime filename +func (l *LocalHost) getRuntimeLabels() (Labels, error) { + labels := Labels{} + + content, err := os.ReadFile(path.Join(l.LabelsPath, l.RuntimeLabelsFilename)) + if err != nil { + if strings.Contains(err.Error(), "no such file or directory") { + return labels, nil + } + + return labels, err + } + + for _, label := range strings.Split(string(content), "\n") { + labels = append(labels, Label(strings.TrimSpace(label))) + } + + return labels, nil +} + +// AddLabel adds runtime label into the LabelsPath directory +func (l *LocalHost) AddLabels(labels Labels) error { + runtimeLabels, err := l.getRuntimeLabels() + if err != nil { + return fmt.Errorf("error while loading stored labels: %v", err) + } + + var found bool + + for _, label := range labels { + found = false + + for _, runtimeLabel := range runtimeLabels { + if label == runtimeLabel { + found = true + break + } + } + + if !found { + runtimeLabels = append(runtimeLabels, label) + } + } + + err = l.saveRuntimeLabels(runtimeLabels) + if err != nil { + return fmt.Errorf("error while saving new set of labels: %v", err) + } + + return nil +} + +// DeleteLabels removed labels from LabelsPath directory. Only labels added this way can be deleted. +func (l *LocalHost) DeleteLabels(labels Labels) error { + runtimeLabels, err := l.getRuntimeLabels() + if err != nil { + return fmt.Errorf("error while loading stored labels: %v", err) + } + + newSet := Labels{} + var found bool + + for _, runtimeLabel := range runtimeLabels { + found = false + + for _, label := range labels { + if label == runtimeLabel { + found = true + break + } + } + + if !found { + newSet = append(newSet, runtimeLabel) + } + } + + err = l.saveRuntimeLabels(newSet) + if err != nil { + return fmt.Errorf("error while saving new set of labels: %v", err) + } + + return nil +} + +// GetIdentification assembles the discovery packet that contains hotname and set of labels describing a single server, in this case the local server. // Parameter initialLabels usually coming from configuration of the app. // If hostname is empty it will be discovered automatically. -func GetIdentification(hostname string, initialLabels Labels, labelsPath string) (Discovery, error) { +func (l *LocalHost) GetIdentification() (Discovery, error) { discovery := Discovery{} - localLabels, err := loadLocalLabels(initialLabels, labelsPath) + localLabels, err := l.loadLocalLabels() if err != nil { return discovery, err } - if len(hostname) == 0 { + if len(l.HostnameOverride) == 0 { info, err := host.Info() if err != nil { return discovery, err } discovery.Hostname = info.Hostname } else { - discovery.Hostname = hostname + discovery.Hostname = l.HostnameOverride } - discovery.Labels = append(initialLabels, localLabels...) + discovery.Labels = append(l.InitialLabels, localLabels...) return discovery, nil } // loadLocalLabels scans local directory where labels are stored and adds them to the labels configured as environment variables. // Filename in LabelsPath is not importent and each file can contain multiple labels, one per each line. -func loadLocalLabels(skipLabels Labels, labelsPath string) (Labels, error) { +func (l *LocalHost) loadLocalLabels() (Labels, error) { labels := Labels{} var found bool - if _, err := os.Stat(labelsPath); !os.IsNotExist(err) { - files, err := ioutil.ReadDir(labelsPath) + if _, err := os.Stat(l.LabelsPath); !os.IsNotExist(err) { + files, err := ioutil.ReadDir(l.LabelsPath) if err != nil { return labels, err } for _, filename := range files { - fullPath := path.Join(labelsPath, filename.Name()) + fullPath := path.Join(l.LabelsPath, filename.Name()) content, err := os.ReadFile(fullPath) if err != nil { return labels, fmt.Errorf("read file error: %v", err) } - fmt.Println(string(content)) for _, line := range strings.Split(string(content), "\n") { line = strings.TrimSpace(line) if len(line) > 0 { found = false - for _, skipLabel := range skipLabels { + for _, skipLabel := range l.InitialLabels { if skipLabel == Label(line) { found = true break @@ -75,6 +180,5 @@ func loadLocalLabels(skipLabels Labels, labelsPath string) (Labels, error) { } } } - fmt.Println("LABELS", labels) return labels, nil } diff --git a/server/identification_test.go b/server/identification_test.go index a45b3c7..336069f 100644 --- a/server/identification_test.go +++ b/server/identification_test.go @@ -11,7 +11,13 @@ const tmpPath = "./tmp" const testLabelPath = tmpPath + "/labels" func TestGetIdentification(t *testing.T) { - discovery, err := GetIdentification("test.example.com", Labels{Label("service:test"), Label("test:1")}, testLabelPath) + localHost := LocalHost{ + LabelsPath: testLabelPath, + HostnameOverride: "test.example.com", + InitialLabels: Labels{Label("service:test"), Label("test:1")}, + } + + discovery, err := localHost.GetIdentification() assert.Nil(t, err) assert.Equal(t, "test.example.com", discovery.Hostname) @@ -23,7 +29,7 @@ func TestGetIdentification(t *testing.T) { err = os.WriteFile(testLabelPath+"/test", []byte("service:test2\npublic_ip:1.2.3.4"), 0644) assert.Nil(t, err) - discovery, err = GetIdentification("test.example.com", Labels{Label("service:test"), Label("test:1")}, testLabelPath) + discovery, err = localHost.GetIdentification() assert.Nil(t, err) assert.Equal(t, Label("public_ip:1.2.3.4"), discovery.Labels[3]) @@ -32,13 +38,19 @@ func TestGetIdentification(t *testing.T) { } func TestLoadLocalLabels(t *testing.T) { + localHost := LocalHost{ + LabelsPath: testLabelPath, + HostnameOverride: "test.example.com", + InitialLabels: Labels{Label("service:test"), Label("test:1")}, + } + err := os.MkdirAll(testLabelPath, os.ModePerm) assert.Nil(t, err) err = os.WriteFile(testLabelPath+"/test", []byte("service:test\npublic_ip:1.2.3.4"), 0644) assert.Nil(t, err) - labels, err := loadLocalLabels(Labels{Label("service:test")}, testLabelPath) + labels, err := localHost.loadLocalLabels() assert.Nil(t, err) assert.Equal(t, 1, len(labels)) diff --git a/server/runtime.go b/server/runtime.go new file mode 100644 index 0000000..a5f8194 --- /dev/null +++ b/server/runtime.go @@ -0,0 +1,9 @@ +package server + +func AddRuntimeLabel(label Label) error { + return nil +} + +func RemoveRuntimeLabel(label Label) error { + return nil +}