cx
/
lobby
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactoring of identification code, runtime labels

This commit is contained in:
Adam Štrauch 2021-09-04 22:17:56 +02:00
parent ff7a26e0d4
commit 471f8bb170
Signed by: cx
GPG Key ID: 018304FFA8988F8D
7 changed files with 249 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
README.md
View File

@ -76,21 +76,22 @@ service file. It doesn't need to access almost anything in your system.
There are other config directives you can use to fine-tune lobbyd to exactly what you need.
| Environment variable | Type | Default | Required | Note |
| ---------------------- | ------ | ----------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- |
| TOKEN | string | | no | Authentication token for API, if empty auth is disabled |
| HOST | string | 127.0.0.1 | no | IP address used for the REST server to listen |
| PORT | int | 1313 | no | Port related to the address above |
| NATS_URL | string | | yes | NATS URL used to connect to the NATS server |
| NATS_DISCOVERY_CHANNEL | string | lobby.discovery | no | Channel where the keep-alive packets are sent |
| LABELS | string | | no | List of labels, labels should be separated by comma |
| LABELS_PATH | string | /etc/lobby/labels | no | Path where filesystem based labels are located, one label per line, filename is not important for lobby |
| HOSTNAME | string | | no | Override local machine's hostname |
| CLEAN_EVERY | int | 15 | no | How often to clean the list of discovered servers to get rid of the not alive ones [secs] |
| KEEP_ALIVE | int | 5 | no | how often to send the keep-alive discovery message with all available information [secs] |
| TTL | int | 30 | no | After how many secs is discovery record considered as invalid |
| NODE_EXPORTER_PORT | int | 9100 | no | Default port where node_exporter listens on all registered servers, this is used when the special prometheus labels doesn't contain port |
| REGISTER | bool | true | no | If true (default) then local instance is registered with other instance (discovery packet is sent regularly), if false the daemon runs only as a client |
| Environment variable | Type | Default | Required | Note |
| ----------------------- | ------ | ----------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- |
| TOKEN | string | | no | Authentication token for API, if empty auth is disabled |
| HOST | string | 127.0.0.1 | no | IP address used for the REST server to listen |
| PORT | int | 1313 | no | Port related to the address above |
| NATS_URL | string | | yes | NATS URL used to connect to the NATS server |
| NATS_DISCOVERY_CHANNEL | string | lobby.discovery | no | Channel where the keep-alive packets are sent |
| LABELS | string | | no | List of labels, labels should be separated by comma |
| LABELS_PATH | string | /etc/lobby/labels | no | Path where filesystem based labels are located, one label per line, filename is not important for lobby |
| RUNTIME_LABELS_FILENAME | string | _runtime | no | Filename for file created in LabelsPath where runtime labels will be added |
| HOSTNAME | string | | no | Override local machine's hostname |
| CLEAN_EVERY | int | 15 | no | How often to clean the list of discovered servers to get rid of the not alive ones [secs] |
| KEEP_ALIVE | int | 5 | no | how often to send the keep-alive discovery message with all available information [secs] |
| TTL | int | 30 | no | After how many secs is discovery record considered as invalid |
| NODE_EXPORTER_PORT | int | 9100 | no | Default port where node_exporter listens on all registered servers, this is used when the special prometheus labels doesn't contain port |
| REGISTER | bool | true | no | If true (default) then local instance is registered with other instance (discovery packet is sent regularly), if false the daemon runs only as a client |
### Service discovery for Prometheus
@ -141,10 +142,15 @@ At least one prometheus label has to be set to export the monitoring service in
So far the REST API is super simple and it has only two endpoints:
GET / # Returns list of all discovered servers and their labels.
GET /v1/ # Same as /
GET /v1/?labels=LABELS # output will be filtered based on one or multiple labels separated by comma
GET /v1/prometheus/:name # Generates output for Prometheus's SD config, name is group of the monitoring services described above.
GET / # Same as /v1/discoveries
GET /v1/discovery # Returns current local discovery packet
GET /v1/discoveries # Returns list of all discovered servers and their labels.
GET /v1/discoveries?labels=LABELS # output will be filtered based on one or multiple labels separated by comma
GET /v1/prometheus/:name # Generates output for Prometheus's SD config, name is group of the monitoring services described above.
POST /v1/labels # Add runtime labels that will persist over daemon restarts. Labels should be in the body of the request, one line per one label.
DELETE /v1/labels # Delete runtime labels. One label per line. Can't affect the labels from environment variables or labels added from the LabelPath.
If there is an error the error message is returned as plain text.
## TODO
@ -152,6 +158,7 @@ So far the REST API is super simple and it has only two endpoints:
* [ ] Command hooks - script or list of scripts that are triggered when discovery status has changed
* [ ] Support for multiple active backend drivers
* [ ] SNS driver
* [ ] API to allow add labels at runtime

27
daemon/config.go
View File

@ -9,19 +9,20 @@ import (
// Config keeps info about configuration of this daemon
type Config struct {
Token string `envconfig:"TOKEN" required:"false"` // Authentication token, if empty auth is disabled
Host string `envconfig:"HOST" required:"false" default:"127.0.0.1"` // IP address used for the REST server to listen
Port uint16 `envconfig:"PORT" required:"false" default:"1313"` // Port related to the address above
NATSURL string `envconfig:"NATS_URL" required:"true"` // NATS URL used to connect to the NATS server
NATSDiscoveryChannel string `envconfig:"NATS_DISCOVERY_CHANNEL" required:"false" default:"lobby.discovery"` // Channel where the kepp alive packets are sent
Labels server.Labels `envconfig:"LABELS" required:"false" default:""` // List of labels
LabelsPath string `envconfig:"LABELS_PATH" required:"false" default:"/etc/lobby/labels"` // Path where filesystem based labels are located
HostName string `envconfig:"HOSTNAME" required:"false"` // Overrise local machine's hostname
CleanEvery uint `envconfig:"CLEAN_EVERY" required:"false" default:"15"` // How often to clean the list of servers to get rid of the not alive ones
KeepAlive uint `envconfig:"KEEP_ALIVE" required:"false" default:"5"` // how often to send the keepalive message with all availabel information [secs]
TTL uint `envconfig:"TTL" required:"false" default:"30"` // After how many secs is discovery record considered as invalid
NodeExporterPort uint `envconfig:"NODE_EXPORTER_PORT" required:"false" default:"9100"` // Default port where node_exporter listens on all registered servers
Register bool `envconfig:"REGISTER" required:"false" default:"true"` // If true (default) then local instance is registered with other instance (discovery packet is sent regularly)
Token string `envconfig:"TOKEN" required:"false"` // Authentication token, if empty auth is disabled
Host string `envconfig:"HOST" required:"false" default:"127.0.0.1"` // IP address used for the REST server to listen
Port uint16 `envconfig:"PORT" required:"false" default:"1313"` // Port related to the address above
NATSURL string `envconfig:"NATS_URL" required:"true"` // NATS URL used to connect to the NATS server
NATSDiscoveryChannel string `envconfig:"NATS_DISCOVERY_CHANNEL" required:"false" default:"lobby.discovery"` // Channel where the kepp alive packets are sent
Labels server.Labels `envconfig:"LABELS" required:"false" default:""` // List of labels
LabelsPath string `envconfig:"LABELS_PATH" required:"false" default:"/etc/lobby/labels"` // Path where filesystem based labels are located
RuntimeLabelsFilename string `envconfig:"RUNTIME_LABELS_FILENAME" required:"false" default:"_runtime"` // Filename for file created in LabelsPath where runtime labels will be added
HostName string `envconfig:"HOSTNAME" required:"false"` // Overrise local machine's hostname
CleanEvery uint `envconfig:"CLEAN_EVERY" required:"false" default:"15"` // How often to clean the list of servers to get rid of the not alive ones
KeepAlive uint `envconfig:"KEEP_ALIVE" required:"false" default:"5"` // how often to send the keepalive message with all availabel information [secs]
TTL uint `envconfig:"TTL" required:"false" default:"30"` // After how many secs is discovery record considered as invalid
NodeExporterPort uint `envconfig:"NODE_EXPORTER_PORT" required:"false" default:"9100"` // Default port where node_exporter listens on all registered servers
Register bool `envconfig:"REGISTER" required:"false" default:"true"` // If true (default) then local instance is registered with other instance (discovery packet is sent regularly)
}
// GetConfig return configuration created based on environment variables

53
daemon/handlers.go
View File

@ -1,6 +1,8 @@
package main
import (
"fmt"
"io/ioutil"
"net/http"
"strings"
@ -30,3 +32,54 @@ func prometheusHandler(c echo.Context) error {
return c.JSONPretty(http.StatusOK, services, " ")
}
func getIdentificationHandler(c echo.Context) error {
discovery, err := localHost.GetIdentification()
if err != nil {
return c.String(http.StatusInternalServerError, fmt.Sprintf("gathering identification info error: %v\n", err))
}
return c.JSONPretty(http.StatusOK, discovery, " ")
}
func addLabelsHandler(c echo.Context) error {
body, err := ioutil.ReadAll(c.Request().Body)
if err != nil {
return c.String(http.StatusBadRequest, fmt.Sprintf("reading request body error: %v\n", err))
}
labels := server.Labels{}
for _, label := range strings.Split(string(body), "\n") {
labels = append(labels, server.Label(label))
}
err = localHost.AddLabels(labels)
if err != nil {
return c.String(http.StatusBadRequest, err.Error())
}
return c.String(http.StatusOK, "OK")
}
func deleteLabelsHandler(c echo.Context) error {
body, err := ioutil.ReadAll(c.Request().Body)
if err != nil {
return c.String(http.StatusBadRequest, fmt.Sprintf("reading request body error: %v\n", err))
}
labels := server.Labels{}
for _, label := range strings.Split(string(body), "\n") {
labels = append(labels, server.Label(label))
}
err = localHost.DeleteLabels(labels)
if err != nil {
return c.String(http.StatusBadRequest, err.Error())
}
return c.String(http.StatusOK, "OK")
}

18
daemon/main.go
View File

@ -18,6 +18,7 @@ import (
var discoveryStorage server.Discoveries = server.Discoveries{}
var driver common.Driver
var localHost server.LocalHost
var config Config
@ -31,6 +32,14 @@ func init() {
discoveryStorage.LogChannel = make(chan string)
discoveryStorage.TTL = config.TTL
// localhost initization
localHost = server.LocalHost{
LabelsPath: config.LabelsPath,
HostnameOverride: config.HostName,
InitialLabels: config.Labels,
RuntimeLabelsFilename: config.RuntimeLabelsFilename,
}
// Setup driver
driver = &nats_driver.Driver{
NATSUrl: config.NATSURL,
@ -52,7 +61,7 @@ func cleanDiscoveryPool() {
// sendGoodbyePacket is almost same as sendDiscoveryPacket but it's not running in loop
// and it adds goodbye message so other nodes know this node is gonna die.
func sendGoodbyePacket() {
discovery, err := server.GetIdentification(config.HostName, config.Labels, config.LabelsPath)
discovery, err := localHost.GetIdentification()
if err != nil {
log.Printf("sending discovery identification error: %v\n", err)
}
@ -66,7 +75,7 @@ func sendGoodbyePacket() {
// sendDisoveryPacket sends discovery packet regularly so the network know we exist
func sendDiscoveryPacket() {
for {
discovery, err := server.GetIdentification(config.HostName, config.Labels, config.LabelsPath)
discovery, err := localHost.GetIdentification()
if err != nil {
log.Printf("sending discovery identification error: %v\n", err)
}
@ -139,7 +148,10 @@ func main() {
// Routes
e.GET("/", listHandler)
e.GET("/v1/", listHandler)
e.GET("/v1/discovery", getIdentificationHandler)
e.GET("/v1/discoveries", listHandler)
e.POST("/v1/labels", addLabelsHandler)
e.DELETE("/v1/labels", deleteLabelsHandler)
e.GET("/v1/prometheus/:name", prometheusHandler)
// ------------------------------

130
server/identification.go
View File

@ -10,59 +10,164 @@ import (
"github.com/shirou/gopsutil/v3/host"
)
// getIdentification assembles the discovery packet that contains hotname and set of labels describing a single server, in this case the local server.
type LocalHost struct {
LabelsPath string // Where labels are stored
RuntimeLabelsFilename string // Filename under which are runtime labels saved in LabelsPath
InitialLabels Labels // this usually coming from the config
HostnameOverride string // if not empty string hostname in the discovery packet will be replaced by this
}
// saveRuntimeLabels stores labels in the runtime filesname
func (l *LocalHost) saveRuntimeLabels(labels Labels) error {
stringLabels := []string{}
for _, label := range labels {
stringLabels = append(stringLabels, label.String())
}
content := strings.Join(stringLabels, "\n")
err := os.WriteFile(path.Join(l.LabelsPath, l.RuntimeLabelsFilename), []byte(content), 0755)
return err
}
// getRuntimeLabels returns labels from the runtime filename
func (l *LocalHost) getRuntimeLabels() (Labels, error) {
labels := Labels{}
content, err := os.ReadFile(path.Join(l.LabelsPath, l.RuntimeLabelsFilename))
if err != nil {
if strings.Contains(err.Error(), "no such file or directory") {
return labels, nil
}
return labels, err
}
for _, label := range strings.Split(string(content), "\n") {
labels = append(labels, Label(strings.TrimSpace(label)))
}
return labels, nil
}
// AddLabel adds runtime label into the LabelsPath directory
func (l *LocalHost) AddLabels(labels Labels) error {
runtimeLabels, err := l.getRuntimeLabels()
if err != nil {
return fmt.Errorf("error while loading stored labels: %v", err)
}
var found bool
for _, label := range labels {
found = false
for _, runtimeLabel := range runtimeLabels {
if label == runtimeLabel {
found = true
break
}
}
if !found {
runtimeLabels = append(runtimeLabels, label)
}
}
err = l.saveRuntimeLabels(runtimeLabels)
if err != nil {
return fmt.Errorf("error while saving new set of labels: %v", err)
}
return nil
}
// DeleteLabels removed labels from LabelsPath directory. Only labels added this way can be deleted.
func (l *LocalHost) DeleteLabels(labels Labels) error {
runtimeLabels, err := l.getRuntimeLabels()
if err != nil {
return fmt.Errorf("error while loading stored labels: %v", err)
}
newSet := Labels{}
var found bool
for _, runtimeLabel := range runtimeLabels {
found = false
for _, label := range labels {
if label == runtimeLabel {
found = true
break
}
}
if !found {
newSet = append(newSet, runtimeLabel)
}
}
err = l.saveRuntimeLabels(newSet)
if err != nil {
return fmt.Errorf("error while saving new set of labels: %v", err)
}
return nil
}
// GetIdentification assembles the discovery packet that contains hotname and set of labels describing a single server, in this case the local server.
// Parameter initialLabels usually coming from configuration of the app.
// If hostname is empty it will be discovered automatically.
func GetIdentification(hostname string, initialLabels Labels, labelsPath string) (Discovery, error) {
func (l *LocalHost) GetIdentification() (Discovery, error) {
discovery := Discovery{}
localLabels, err := loadLocalLabels(initialLabels, labelsPath)
localLabels, err := l.loadLocalLabels()
if err != nil {
return discovery, err
}
if len(hostname) == 0 {
if len(l.HostnameOverride) == 0 {
info, err := host.Info()
if err != nil {
return discovery, err
}
discovery.Hostname = info.Hostname
} else {
discovery.Hostname = hostname
discovery.Hostname = l.HostnameOverride
}
discovery.Labels = append(initialLabels, localLabels...)
discovery.Labels = append(l.InitialLabels, localLabels...)
return discovery, nil
}
// loadLocalLabels scans local directory where labels are stored and adds them to the labels configured as environment variables.
// Filename in LabelsPath is not importent and each file can contain multiple labels, one per each line.
func loadLocalLabels(skipLabels Labels, labelsPath string) (Labels, error) {
func (l *LocalHost) loadLocalLabels() (Labels, error) {
labels := Labels{}
var found bool
if _, err := os.Stat(labelsPath); !os.IsNotExist(err) {
files, err := ioutil.ReadDir(labelsPath)
if _, err := os.Stat(l.LabelsPath); !os.IsNotExist(err) {
files, err := ioutil.ReadDir(l.LabelsPath)
if err != nil {
return labels, err
}
for _, filename := range files {
fullPath := path.Join(labelsPath, filename.Name())
fullPath := path.Join(l.LabelsPath, filename.Name())
content, err := os.ReadFile(fullPath)
if err != nil {
return labels, fmt.Errorf("read file error: %v", err)
}
fmt.Println(string(content))
for _, line := range strings.Split(string(content), "\n") {
line = strings.TrimSpace(line)
if len(line) > 0 {
found = false
for _, skipLabel := range skipLabels {
for _, skipLabel := range l.InitialLabels {
if skipLabel == Label(line) {
found = true
break
@ -75,6 +180,5 @@ func loadLocalLabels(skipLabels Labels, labelsPath string) (Labels, error) {
}
}
}
fmt.Println("LABELS", labels)
return labels, nil
}

18
server/identification_test.go
View File

@ -11,7 +11,13 @@ const tmpPath = "./tmp"
const testLabelPath = tmpPath + "/labels"
func TestGetIdentification(t *testing.T) {
discovery, err := GetIdentification("test.example.com", Labels{Label("service:test"), Label("test:1")}, testLabelPath)
localHost := LocalHost{
LabelsPath: testLabelPath,
HostnameOverride: "test.example.com",
InitialLabels: Labels{Label("service:test"), Label("test:1")},
}
discovery, err := localHost.GetIdentification()
assert.Nil(t, err)
assert.Equal(t, "test.example.com", discovery.Hostname)
@ -23,7 +29,7 @@ func TestGetIdentification(t *testing.T) {
err = os.WriteFile(testLabelPath+"/test", []byte("service:test2\npublic_ip:1.2.3.4"), 0644)
assert.Nil(t, err)
discovery, err = GetIdentification("test.example.com", Labels{Label("service:test"), Label("test:1")}, testLabelPath)
discovery, err = localHost.GetIdentification()
assert.Nil(t, err)
assert.Equal(t, Label("public_ip:1.2.3.4"), discovery.Labels[3])
@ -32,13 +38,19 @@ func TestGetIdentification(t *testing.T) {
}
func TestLoadLocalLabels(t *testing.T) {
localHost := LocalHost{
LabelsPath: testLabelPath,
HostnameOverride: "test.example.com",
InitialLabels: Labels{Label("service:test"), Label("test:1")},
}
err := os.MkdirAll(testLabelPath, os.ModePerm)
assert.Nil(t, err)
err = os.WriteFile(testLabelPath+"/test", []byte("service:test\npublic_ip:1.2.3.4"), 0644)
assert.Nil(t, err)
labels, err := loadLocalLabels(Labels{Label("service:test")}, testLabelPath)
labels, err := localHost.loadLocalLabels()
assert.Nil(t, err)
assert.Equal(t, 1, len(labels))

9
server/runtime.go Normal file
View File

@ -0,0 +1,9 @@
package server
func AddRuntimeLabel(label Label) error {
return nil
}
func RemoveRuntimeLabel(label Label) error {
return nil
}