293 lines
7.8 KiB
Go
293 lines
7.8 KiB
Go
// Package metrics provides minimal Prometheus-style HTTP RED and sync counters.
|
|||
|
|
package metrics
|
||
|
|
|
||
|
|
import (
|
||
|
|
"fmt"
|
||
|
|
"net"
|
||
|
|
"net/http"
|
||
|
|
"sort"
|
||
|
|
"strconv"
|
||
|
|
"strings"
|
||
|
|
"sync"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"github.com/go-chi/chi/v5"
|
||
|
|
)
|
||
|
|
|
||
|
|
// Fixed latency buckets (seconds) for HTTP and sync histograms.
|
||
|
|
var durationBuckets = []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30}
|
||
|
|
|
||
|
|
type labelKey struct {
|
||
|
|
a, b, c string
|
||
|
|
}
|
||
|
|
|
||
|
|
type histogram struct {
|
||
|
|
counts []uint64
|
||
|
|
sum float64
|
||
|
|
count uint64
|
||
|
|
}
|
||
|
|
|
||
|
|
func newHistogram() *histogram {
|
||
|
|
return &histogram{counts: make([]uint64, len(durationBuckets))}
|
||
|
|
}
|
||
|
|
|
||
|
|
func (h *histogram) observe(seconds float64) {
|
||
|
|
h.sum += seconds
|
||
|
|
h.count++
|
||
|
|
for i, bound := range durationBuckets {
|
||
|
|
if seconds <= bound {
|
||
|
|
h.counts[i]++
|
||
|
|
return
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
type registry struct {
|
||
|
|
mu sync.Mutex
|
||
|
|
|
||
|
|
httpRequests map[labelKey]uint64
|
||
|
|
httpDuration map[labelKey]*histogram
|
||
|
|
syncDuration map[string]*histogram
|
||
|
|
syncFailures map[string]uint64
|
||
|
|
}
|
||
|
|
|
||
|
|
var defaultRegistry = ®istry{
|
||
|
|
httpRequests: make(map[labelKey]uint64),
|
||
|
|
httpDuration: make(map[labelKey]*histogram),
|
||
|
|
syncDuration: make(map[string]*histogram),
|
||
|
|
syncFailures: make(map[string]uint64),
|
||
|
|
}
|
||
|
|
|
||
|
|
// ObserveHTTP records one finished request (RED: rate via counter, errors via code, duration).
|
||
|
|
func ObserveHTTP(method, path string, status int, d time.Duration) {
|
||
|
|
if path == "" {
|
||
|
|
path = "unmatched"
|
||
|
|
}
|
||
|
|
key := labelKey{method, strconv.Itoa(status), path}
|
||
|
|
sec := d.Seconds()
|
||
|
|
defaultRegistry.mu.Lock()
|
||
|
|
defer defaultRegistry.mu.Unlock()
|
||
|
|
defaultRegistry.httpRequests[key]++
|
||
|
|
h := defaultRegistry.httpDuration[key]
|
||
|
|
if h == nil {
|
||
|
|
h = newHistogram()
|
||
|
|
defaultRegistry.httpDuration[key] = h
|
||
|
|
}
|
||
|
|
h.observe(sec)
|
||
|
|
}
|
||
|
|
|
||
|
|
// ObserveSync records sync job duration and increments failures when err != nil.
|
||
|
|
func ObserveSync(kind string, err error, d time.Duration) {
|
||
|
|
kind = strings.TrimSpace(kind)
|
||
|
|
if kind == "" {
|
||
|
|
kind = "unknown"
|
||
|
|
}
|
||
|
|
sec := d.Seconds()
|
||
|
|
defaultRegistry.mu.Lock()
|
||
|
|
defer defaultRegistry.mu.Unlock()
|
||
|
|
h := defaultRegistry.syncDuration[kind]
|
||
|
|
if h == nil {
|
||
|
|
h = newHistogram()
|
||
|
|
defaultRegistry.syncDuration[kind] = h
|
||
|
|
}
|
||
|
|
h.observe(sec)
|
||
|
|
if err != nil {
|
||
|
|
defaultRegistry.syncFailures[kind]++
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Snapshot returns coarse totals for admin diagnostics (not a full series dump).
|
||
|
|
func Snapshot() map[string]any {
|
||
|
|
defaultRegistry.mu.Lock()
|
||
|
|
defer defaultRegistry.mu.Unlock()
|
||
|
|
|
||
|
|
var httpTotal, syncCount, syncFail uint64
|
||
|
|
var syncSum float64
|
||
|
|
for _, n := range defaultRegistry.httpRequests {
|
||
|
|
httpTotal += n
|
||
|
|
}
|
||
|
|
for _, h := range defaultRegistry.syncDuration {
|
||
|
|
syncCount += h.count
|
||
|
|
syncSum += h.sum
|
||
|
|
}
|
||
|
|
for _, n := range defaultRegistry.syncFailures {
|
||
|
|
syncFail += n
|
||
|
|
}
|
||
|
|
return map[string]any{
|
||
|
|
"http_requests_total": httpTotal,
|
||
|
|
"sync_duration_seconds_sum": syncSum,
|
||
|
|
"sync_duration_seconds_count": syncCount,
|
||
|
|
"sync_failures_total": syncFail,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Reset clears all series (tests only).
|
||
|
|
func Reset() {
|
||
|
|
defaultRegistry.mu.Lock()
|
||
|
|
defer defaultRegistry.mu.Unlock()
|
||
|
|
defaultRegistry.httpRequests = make(map[labelKey]uint64)
|
||
|
|
defaultRegistry.httpDuration = make(map[labelKey]*histogram)
|
||
|
|
defaultRegistry.syncDuration = make(map[string]*histogram)
|
||
|
|
defaultRegistry.syncFailures = make(map[string]uint64)
|
||
|
|
}
|
||
|
|
|
||
|
|
type statusRecorder struct {
|
||
|
|
http.ResponseWriter
|
||
|
|
status int
|
||
|
|
}
|
||
|
|
|
||
|
|
func (r *statusRecorder) WriteHeader(code int) {
|
||
|
|
r.status = code
|
||
|
|
r.ResponseWriter.WriteHeader(code)
|
||
|
|
}
|
||
|
|
|
||
|
|
func (r *statusRecorder) Write(b []byte) (int, error) {
|
||
|
|
if r.status == 0 {
|
||
|
|
r.status = http.StatusOK
|
||
|
|
}
|
||
|
|
return r.ResponseWriter.Write(b)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Middleware records HTTP RED metrics using the chi route pattern (low cardinality).
|
||
|
|
func Middleware(next http.Handler) http.Handler {
|
||
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
|
|
if r.URL.Path == "/metrics" {
|
||
|
|
next.ServeHTTP(w, r)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
start := time.Now()
|
||
|
|
rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
|
||
|
|
next.ServeHTTP(rec, r)
|
||
|
|
path := chi.RouteContext(r.Context()).RoutePattern()
|
||
|
|
if path == "" {
|
||
|
|
path = "unmatched"
|
||
|
|
}
|
||
|
|
ObserveHTTP(r.Method, path, rec.status, time.Since(start))
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
// Handler serves Prometheus text exposition.
|
||
|
|
func Handler() http.Handler {
|
||
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
|
|
if r.Method != http.MethodGet && r.Method != http.MethodHead {
|
||
|
|
w.Header().Set("Allow", "GET, HEAD")
|
||
|
|
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
body := defaultRegistry.render()
|
||
|
|
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||
|
|
w.WriteHeader(http.StatusOK)
|
||
|
|
if r.Method == http.MethodHead {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
_, _ = w.Write(body)
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
// Gate restricts Prometheus scrapes in production: allow when metricsPublic is true
|
||
|
|
// (METRICS_PUBLIC=1) or the peer is loopback. Non-production always allows (local scrapes).
|
||
|
|
func Gate(isProduction, metricsPublic bool) func(http.Handler) http.Handler {
|
||
|
|
return func(next http.Handler) http.Handler {
|
||
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
|
|
if !isProduction || metricsPublic || isLoopbackRemoteAddr(r.RemoteAddr) {
|
||
|
|
next.ServeHTTP(w, r)
|
||
|
|
return
|
||
|
|
}
|
||
|
|
http.NotFound(w, r)
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func isLoopbackRemoteAddr(remoteAddr string) bool {
|
||
|
|
host := strings.TrimSpace(remoteAddr)
|
||
|
|
if host == "" {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
if h, _, err := net.SplitHostPort(host); err == nil {
|
||
|
|
host = h
|
||
|
|
}
|
||
|
|
ip := net.ParseIP(host)
|
||
|
|
return ip != nil && ip.IsLoopback()
|
||
|
|
}
|
||
|
|
|
||
|
|
func (reg *registry) render() []byte {
|
||
|
|
reg.mu.Lock()
|
||
|
|
defer reg.mu.Unlock()
|
||
|
|
|
||
|
|
var b strings.Builder
|
||
|
|
b.WriteString("# HELP http_requests_total Total HTTP requests by method, status code, and route pattern.\n")
|
||
|
|
b.WriteString("# TYPE http_requests_total counter\n")
|
||
|
|
for _, key := range sortedHTTPKeys(reg.httpRequests) {
|
||
|
|
fmt.Fprintf(&b, "http_requests_total{method=%q,code=%q,path=%q} %d\n",
|
||
|
|
key.a, key.b, key.c, reg.httpRequests[key])
|
||
|
|
}
|
||
|
|
|
||
|
|
b.WriteString("# HELP http_request_duration_seconds HTTP request latency in seconds.\n")
|
||
|
|
b.WriteString("# TYPE http_request_duration_seconds histogram\n")
|
||
|
|
for _, key := range sortedHTTPKeys(reg.httpDuration) {
|
||
|
|
writeHistogram(&b, "http_request_duration_seconds",
|
||
|
|
fmt.Sprintf("method=%q,code=%q,path=%q", key.a, key.b, key.c),
|
||
|
|
reg.httpDuration[key])
|
||
|
|
}
|
||
|
|
|
||
|
|
b.WriteString("# HELP sync_duration_seconds Sync job latency in seconds by kind.\n")
|
||
|
|
b.WriteString("# TYPE sync_duration_seconds histogram\n")
|
||
|
|
for _, kind := range sortedStringKeys(reg.syncDuration) {
|
||
|
|
writeHistogram(&b, "sync_duration_seconds",
|
||
|
|
fmt.Sprintf("kind=%q", kind),
|
||
|
|
reg.syncDuration[kind])
|
||
|
|
}
|
||
|
|
|
||
|
|
b.WriteString("# HELP sync_failures_total Sync jobs that returned an error, by kind.\n")
|
||
|
|
b.WriteString("# TYPE sync_failures_total counter\n")
|
||
|
|
for _, kind := range sortedStringKeys(reg.syncFailures) {
|
||
|
|
fmt.Fprintf(&b, "sync_failures_total{kind=%q} %d\n", kind, reg.syncFailures[kind])
|
||
|
|
}
|
||
|
|
return []byte(b.String())
|
||
|
|
}
|
||
|
|
|
||
|
|
func writeHistogram(b *strings.Builder, name, labels string, h *histogram) {
|
||
|
|
var cumulative uint64
|
||
|
|
for i, bound := range durationBuckets {
|
||
|
|
cumulative += h.counts[i]
|
||
|
|
fmt.Fprintf(b, "%s_bucket{%s,le=%q} %d\n", name, labels, formatLE(bound), cumulative)
|
||
|
|
}
|
||
|
|
fmt.Fprintf(b, "%s_bucket{%s,le=\"+Inf\"} %d\n", name, labels, h.count)
|
||
|
|
fmt.Fprintf(b, "%s_sum{%s} %s\n", name, labels, formatFloat(h.sum))
|
||
|
|
fmt.Fprintf(b, "%s_count{%s} %d\n", name, labels, h.count)
|
||
|
|
}
|
||
|
|
|
||
|
|
func formatLE(v float64) string {
|
||
|
|
return strconv.FormatFloat(v, 'f', -1, 64)
|
||
|
|
}
|
||
|
|
|
||
|
|
func formatFloat(v float64) string {
|
||
|
|
return strconv.FormatFloat(v, 'f', -1, 64)
|
||
|
|
}
|
||
|
|
|
||
|
|
func sortedHTTPKeys[T any](m map[labelKey]T) []labelKey {
|
||
|
|
keys := make([]labelKey, 0, len(m))
|
||
|
|
for k := range m {
|
||
|
|
keys = append(keys, k)
|
||
|
|
}
|
||
|
|
sort.Slice(keys, func(i, j int) bool {
|
||
|
|
if keys[i].a != keys[j].a {
|
||
|
|
return keys[i].a < keys[j].a
|
||
|
|
}
|
||
|
|
if keys[i].b != keys[j].b {
|
||
|
|
return keys[i].b < keys[j].b
|
||
|
|
}
|
||
|
|
return keys[i].c < keys[j].c
|
||
|
|
})
|
||
|
|
return keys
|
||
|
|
}
|
||
|
|
|
||
|
|
func sortedStringKeys[T any](m map[string]T) []string {
|
||
|
|
keys := make([]string, 0, len(m))
|
||
|
|
for k := range m {
|
||
|
|
keys = append(keys, k)
|
||
|
|
}
|
||
|
|
sort.Strings(keys)
|
||
|
|
return keys
|
||
|
|
}
|