Metrics: add prometheus monitoring metrics route

This adds a metrics route to monitoring metrics using prometheus.
It exports mirror and files stats, as well as the newly introduced
country downloads counter.

database/utils.go

@@ -41,6 +41,30 @@ func (r *Redis) GetListOfMirrors() (map[int]string, error) {
 	return mirrors, nil
 }
 
+func (r *Redis) GetListOfFiles() ([]string, error) {
+	conn, err := r.Connect()
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+
+	values, err := redis.Values(conn.Do("SMEMBERS", "FILES"))
+	if err != nil {
+		return nil, err
+	}
+
+	files := make([]string, len(values))
+	for i, v := range values {
+		value, okValue := v.([]byte)
+		if !okValue {
+			return nil, errors.New("invalid type for file")
+		}
+		files[i] = string(value)
+	}
+
+	return files, nil
+}
+
 func (r *Redis) GetListOfCountries() ([]string, error) {
 	conn, err := r.Connect()
 	if err != nil {

http/http.go

@@ -72,6 +72,7 @@ func HTTPServer(redis *database.Redis, cache *mirrors.Cache) *HTTP {
 	h.stats = NewStats(redis)
 	h.engine = DefaultEngine{}
 	http.Handle("/", NewGzipHandler(h.requestDispatcher))
+	http.Handle("/metrics", NewGzipHandler(h.metricsHandler))
 
 	// Load the GeoIP databases
 	if err := h.geoip.LoadGeoIP(); err != nil {

http/metrics.go

@@ -0,0 +1,145 @@
+package http
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/gomodule/redigo/redis"
+)
+
+type metrics struct {
+	Day   int
+	Month int
+	Year  int
+	Total int
+}
+
+func statsToPrometheusFormat(metrics metrics, labelName string, labelValue string) string {
+	var output string
+
+	output += fmt.Sprintf("%s_total{%s=\"%s\"} %d\n",
+		labelName, labelName, labelValue, metrics.Total)
+	output += fmt.Sprintf("%s_day{%s=\"%s\"} %d\n",
+		labelName, labelName, labelValue, metrics.Day)
+	output += fmt.Sprintf("%s_month{%s=\"%s\"} %d\n",
+		labelName, labelName, labelValue, metrics.Month)
+	output += fmt.Sprintf("%s_year{%s=\"%s\"} %d\n\n",
+		labelName, labelName, labelValue, metrics.Year)
+
+	return output
+}
+
+func (h *HTTP) metricsHandler(w http.ResponseWriter, r *http.Request) {
+	rconn := h.redis.Get()
+	defer rconn.Close()
+
+	// Get all mirrors ID
+	mirrorsMap, err := h.redis.GetListOfMirrors()
+	if err != nil {
+		log.Error("Cannot fetch the list of mirrors: " + err.Error())
+		return
+	}
+
+	var mirrorsIDs []int
+	for id := range mirrorsMap {
+		// We need a common order to iterate the
+		// results from Redis.
+		mirrorsIDs = append(mirrorsIDs, id)
+	}
+
+	rconn.Send("MULTI")
+
+	today := time.Now()
+	for _, id := range mirrorsIDs {
+		rconn.Send("HGET", "STATS_MIRROR", id)
+		rconn.Send("HGET", "STATS_MIRROR_"+today.Format("2006_01_02"), id)
+		rconn.Send("HGET", "STATS_MIRROR_"+today.Format("2006_01"), id)
+		rconn.Send("HGET", "STATS_MIRROR_"+today.Format("2006"), id)
+	}
+
+	stats, err := redis.Values(rconn.Do("EXEC"))
+	if err != nil {
+		log.Error("Cannot fetch stats: " + err.Error())
+		return
+	}
+
+	var index int64
+	var output string
+	for _, id := range mirrorsIDs {
+		var mirror metrics
+		mirror.Total, _ = redis.Int(stats[index], err)
+		mirror.Day, _ = redis.Int(stats[index+1], err)
+		mirror.Month, _ = redis.Int(stats[index+2], err)
+		mirror.Year, _ = redis.Int(stats[index+3], err)
+		output += statsToPrometheusFormat(mirror, "mirror", mirrorsMap[id])
+		index += 4
+	}
+
+	// Get all files
+	var fileList []string
+	fileList, err = h.redis.GetListOfFiles()
+	if err != nil {
+		log.Error("Cannot fetch list of files: " + err.Error())
+		return
+	}
+
+	rconn.Send("MULTI")
+	for _, file := range fileList {
+		rconn.Send("HGET", "STATS_FILE", file)
+		rconn.Send("HGET", "STATS_FILE_"+today.Format("2006_01_02"), file)
+		rconn.Send("HGET", "STATS_FILE_"+today.Format("2006_01"), file)
+		rconn.Send("HGET", "STATS_FILE_"+today.Format("2006"), file)
+	}
+
+	stats, err = redis.Values(rconn.Do("EXEC"))
+	if err != nil {
+		log.Error("Cannot fetch stats: " + err.Error())
+		return
+	}
+
+	index = 0
+	for _, name := range fileList {
+		var metrics metrics
+		metrics.Total, _ = redis.Int(stats[index], err)
+		metrics.Day, _ = redis.Int(stats[index+1], err)
+		metrics.Month, _ = redis.Int(stats[index+2], err)
+		metrics.Year, _ = redis.Int(stats[index+3], err)
+		output += statsToPrometheusFormat(metrics, "file", name)
+		index += 4
+	}
+
+	// Get all countries
+	var countryList []string
+	countryList, err = h.redis.GetListOfCountries()
+	if err != nil {
+		log.Error("Cannot fetch list of countries: " + err.Error())
+		return
+	}
+
+	rconn.Send("MULTI")
+	for _, country := range countryList {
+		rconn.Send("HGET", "STATS_COUNTRY", country)
+		rconn.Send("HGET", "STATS_COUNTRY_"+today.Format("2006_01_02"), country)
+		rconn.Send("HGET", "STATS_COUNTRY_"+today.Format("2006_01"), country)
+		rconn.Send("HGET", "STATS_COUNTRY_"+today.Format("2006"), country)
+	}
+	stats, err = redis.Values(rconn.Do("EXEC"))
+	if err != nil {
+		log.Error("Cannot fetch stats: " + err.Error())
+		return
+	}
+
+	index = 0
+	for _, name := range countryList {
+		var metrics metrics
+		metrics.Total, _ = redis.Int(stats[index], err)
+		metrics.Day, _ = redis.Int(stats[index+1], err)
+		metrics.Month, _ = redis.Int(stats[index+2], err)
+		metrics.Year, _ = redis.Int(stats[index+3], err)
+		output += statsToPrometheusFormat(metrics, "country", name)
+		index += 4
+	}
+
+	w.Write([]byte(output))
+}