diff --git a/.gitignore b/.gitignore index 0d83950..ba4761f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ rosti.db node-api .history/ +api-node-17.http diff --git a/main.go b/main.go index 7e6d446..676c035 100644 --- a/main.go +++ b/main.go @@ -9,7 +9,7 @@ import ( "github.com/rosti-cz/node-api/apps" "github.com/rosti-cz/node-api/common" "github.com/rosti-cz/node-api/docker" - "github.com/rosti-cz/node-api/nodes" + "github.com/rosti-cz/node-api/node" ) // JSONIndent Indendation of JSON output format @@ -18,8 +18,6 @@ const JSONIndent = " " func main() { // Close database at the end db := common.GetDBConnection() - // db.AutoMigrate(apps.Label{}) - // db.AutoMigrate(apps.App{}) defer db.Close() // Templating @@ -27,11 +25,24 @@ func main() { // Stats loop go func() { - err := gatherContainerStats() - if err != nil { - log.Println("LOOP ERROR:", err.Error()) + for { + err := gatherContainerStats() + if err != nil { + log.Println("LOOP ERROR:", err.Error()) + } + time.Sleep(5 * time.Minute) + } + }() + + // Node stats + go func() { + for { + err := node.Log() + if err != nil { + log.Println("NODE PERFORMANCE LOG ERROR:", err.Error()) + } + time.Sleep(5 * time.Minute) } - time.Sleep(5 * time.Minute) }() // API @@ -324,7 +335,10 @@ func main() { // Return info about the node including performance index e.GET("/v1/node", func(c echo.Context) error { - node := nodes.GetNodeInfo() + node, err := node.GetNodeInfo() + if err != nil { + return c.JSONPretty(http.StatusInternalServerError, Message{Message: err.Error()}, JSONIndent) + } return c.JSON(http.StatusOK, node) }) diff --git a/node/load.go b/node/load.go new file mode 100644 index 0000000..e26a512 --- /dev/null +++ b/node/load.go @@ -0,0 +1,134 @@ +package node + +import ( + "github.com/rosti-cz/node-api/common" + + "github.com/shirou/gopsutil/cpu" + "github.com/shirou/gopsutil/disk" + "github.com/shirou/gopsutil/load" + "github.com/shirou/gopsutil/mem" +) + +const history = 72 * 3600 / 300 // 3 days, one record every five minutes + +func init() { + db := common.GetDBConnection() + db.AutoMigrate(PerformanceLog{}) +} + +// Log creates a record for all important metrics used as +func Log() error { + performanceLog := PerformanceLog{} + + // Load + loadStats, err := load.Avg() + if err != nil { + return err + } + + performanceLog.Load1 = loadStats.Load1 + performanceLog.Load5 = loadStats.Load5 + performanceLog.Load15 = loadStats.Load15 + + // Memory + memoryStat, err := mem.VirtualMemory() + if err != nil { + return err + } + performanceLog.Memory = memoryStat.UsedPercent / 100.0 + + // Disk space + diskUsage, err := disk.Usage("/srv") + if err != nil { + return err + } + performanceLog.DiskSpaceUsage = diskUsage.UsedPercent / 100.0 + + // Save + db := common.GetDBConnection() + err = db.Create(&performanceLog).Error + if err != nil { + return err + } + + // and clean + // we have to use this stupid approach because DELETE doesn't support ORDER BY and LIMIT + toDeleteLogs := []PerformanceLog{} + err = db.Order("id DESC").Limit("99").Offset(history).Find(&toDeleteLogs).Error + if err != nil { + return err + } + + for _, toDeleteLog := range toDeleteLogs { + err = db.Delete(&toDeleteLog).Error + if err != nil { + return err + } + } + + return nil +} + +// Index returns number from 0 to 1 where 0 means least loaded and 1 maximally loaded. +// It uses history of last 72 hours +func index() (*Node, error) { + node := Node{ + Index: 1.0, + } + + // Number of CPUs + cpus, err := cpu.Counts(true) + if err != nil { + return &node, err + } + + db := common.GetDBConnection() + + logs := []PerformanceLog{} + + err = db.Find(&logs).Error + if err != nil { + return &node, err + } + + var totalLoad1 float64 + var totalLoad5 float64 + var totalLoad15 float64 + var totalDiskSpaceUsage float64 + var totalMemory float64 + + // If there is no record we have to wait until it is, until then the server is "full" + if len(logs) == 0 { + return &node, nil + } + + for _, log := range logs { + totalLoad1 += log.Load1 + totalLoad5 += log.Load5 + totalLoad15 += log.Load15 + totalDiskSpaceUsage += log.DiskSpaceUsage + totalMemory += log.Memory + } + + node.Load1Index = totalLoad1 / float64(len(logs)) / float64(cpus) + node.Load5Index = totalLoad5 / float64(len(logs)) / float64(cpus) + node.Load15Index = totalLoad15 / float64(len(logs)) / float64(cpus) + node.MemoryIndex = totalMemory / float64(len(logs)) + node.DiskSpaceIndex = totalDiskSpaceUsage / float64(len(logs)) + + var indexes []float64 + indexes = append(indexes, node.Load5Index) + indexes = append(indexes, node.MemoryIndex) + indexes = append(indexes, node.DiskSpaceIndex) + + finalIndex := float64(0) + for _, index := range indexes { + if index > finalIndex { + finalIndex = index + } + } + + node.Index = finalIndex + + return &node, nil +} diff --git a/node/main.go b/node/main.go new file mode 100644 index 0000000..fbd8bb5 --- /dev/null +++ b/node/main.go @@ -0,0 +1,7 @@ +package node + +// GetNodeInfo returns information about this node +func GetNodeInfo() (*Node, error) { + node, err := index() + return node, err +} diff --git a/node/types.go b/node/types.go new file mode 100644 index 0000000..858c02e --- /dev/null +++ b/node/types.go @@ -0,0 +1,27 @@ +package node + +import "github.com/jinzhu/gorm" + +// Node keeps info about server +type Node struct { + Index float64 `json:"index"` // 0 is empty server, 1 means full server + Load1Index float64 `json:"load1_index"` // 0 is empty server, 1 means resource fully used + Load5Index float64 `json:"load5_index"` // 0 is empty server, 1 means resource fully used + Load15Index float64 `json:"load15_index"` // 0 is empty server, 1 means resource fully used + MemoryIndex float64 `json:"memory_index"` // 0 is empty server, 1 means resource fully used + DiskSpaceIndex float64 `json:"disk_space_index"` // 0 is empty server, 1 means resource fully used + + // TODO not implemented + SoldMemory int `json:"-"` // allocated to containers +} + +// PerformanceLog keeps track of performance history +type PerformanceLog struct { + gorm.Model + + Load1 float64 + Load5 float64 + Load15 float64 + DiskSpaceUsage float64 + Memory float64 +} diff --git a/nodes/load.go b/nodes/load.go deleted file mode 100644 index cb6f817..0000000 --- a/nodes/load.go +++ /dev/null @@ -1,66 +0,0 @@ -package nodes - -import ( - "log" - - "github.com/jinzhu/gorm" - "github.com/shirou/gopsutil/cpu" - "github.com/shirou/gopsutil/disk" - "github.com/shirou/gopsutil/load" - "github.com/shirou/gopsutil/mem" -) - -// PerformanceLog keeps track of performance history -type PerformanceLog struct { - gorm.Model - - Load1 float64 - Load5 float64 - Load15 float64 - DiskSpaceUsage float64 - Memory float64 -} - -// Log creates a record for all important metrics used as -func Log() error { - performanceLog := PerformanceLog{} - - // Load - loadStats, err := load.Avg() - if err != nil { - return err - } - - performanceLog.Load1 = loadStats.Load1 - performanceLog.Load5 = loadStats.Load5 - performanceLog.Load15 = loadStats.Load15 - - // Memory - memoryStat, err := mem.VirtualMemory() - if err != nil { - return err - } - performanceLog.Memory = memoryStat.UsedPercent / 100.0 - - // Disk space - diskUsage, err := disk.Usage("/srv") - if err != nil { - return err - } - performanceLog.DiskSpaceUsage = diskUsage.UsedPercent / 100.0 - - return nil -} - -// Index returns number from 0 to 1 where 0 means least loaded and 1 maximally loaded. -// It uses history of last 72 hours -func Index() (float64, error) { - // Number of CPUs - cpus, err := cpu.Counts(true) - if err != nil { - return 1.0, err - } - log.Println(cpus) - - return 1.0, err -} diff --git a/nodes/main.go b/nodes/main.go deleted file mode 100644 index 65b1b82..0000000 --- a/nodes/main.go +++ /dev/null @@ -1,19 +0,0 @@ -package nodes - -// Node keeps info about server -type Node struct { - OccupationIndex float32 // bigger number bigger load, number over 1 means full server - Memory int - SoldMemory int // allocated to containers - LoadOverDay float32 // average load 5 in last 24 hours - DiskSpaceUsed int // Usage of overall disk space -} - -// GetNodeInfo returns information about this node -func GetNodeInfo() *Node { - node := Node{ - OccupationIndex: 0, - } - - return &node -}