greeneclipse
8580c996c3
Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
529 lines
17 KiB
Go
529 lines
17 KiB
Go
package httpapi
|
||
|
||
import (
|
||
"fmt"
|
||
"net/http"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/descrybe/descrybe-v2/apps/api/internal/feeds"
|
||
"github.com/go-chi/chi/v5"
|
||
"github.com/google/uuid"
|
||
)
|
||
|
||
// HTTP rate limiters in this file are in-process (per API OS process / replica).
|
||
//
|
||
// MULTI-REPLICA CUTOVER (see docs/production-readiness.md § edge rate limits):
|
||
// - There is no Redis (or other shared store) in the Descrybe stack today.
|
||
// - Without edge caps, effective HTTP budget across N replicas is roughly
|
||
// N× the per-process base RPM.
|
||
// - RATE_LIMIT_REPLICAS=N (optional) divides only the HTTP middleware caps in
|
||
// this file via rateLimitEffectiveCap (ceil) so aggregate under even load
|
||
// approximates the documented RPM. It is not a shared counter and does not
|
||
// affect login email lockout, processing.StartLimiter, support.AIRateLimiter,
|
||
// or email send limiters — those stay per-process until edge/shared infra.
|
||
// - Cutover for multi-replica hard global RPM: enforce cluster caps at the
|
||
// edge (CDN/ingress/WAF). RATE_LIMIT_REPLICAS alone is not a substitute.
|
||
// - RATE_LIMIT_MULTI_REPLICA=true acknowledges multi-replica deploy without a
|
||
// shared backend; the API logs a boot warning (config.RateLimitWarningMessage).
|
||
// - RATE_LIMIT_BACKEND=redis|postgres is accepted as documentation only and
|
||
// forced to memory until a shared backend is implemented — do not assume
|
||
// distributed counters exist.
|
||
|
||
// slidingWindowLimiter is a light in-process rate limiter (per-key).
|
||
// Suitable for a single API instance; not shared across replicas.
|
||
type slidingWindowLimiter struct {
|
||
mu sync.Mutex
|
||
window time.Duration
|
||
limit int
|
||
hits map[string][]time.Time
|
||
lastGC time.Time
|
||
}
|
||
|
||
func newSlidingWindowLimiter(limit int, window time.Duration) *slidingWindowLimiter {
|
||
if limit <= 0 {
|
||
limit = 30
|
||
}
|
||
if window <= 0 {
|
||
window = time.Minute
|
||
}
|
||
return &slidingWindowLimiter{
|
||
window: window,
|
||
limit: limit,
|
||
hits: make(map[string][]time.Time),
|
||
lastGC: time.Now(),
|
||
}
|
||
}
|
||
|
||
func (l *slidingWindowLimiter) allow(key string) bool {
|
||
now := time.Now()
|
||
cutoff := now.Add(-l.window)
|
||
l.mu.Lock()
|
||
defer l.mu.Unlock()
|
||
if now.Sub(l.lastGC) > l.window {
|
||
for k, ts := range l.hits {
|
||
kept := ts[:0]
|
||
for _, t := range ts {
|
||
if t.After(cutoff) {
|
||
kept = append(kept, t)
|
||
}
|
||
}
|
||
if len(kept) == 0 {
|
||
delete(l.hits, k)
|
||
} else {
|
||
l.hits[k] = kept
|
||
}
|
||
}
|
||
l.lastGC = now
|
||
}
|
||
ts := l.hits[key]
|
||
kept := ts[:0]
|
||
for _, t := range ts {
|
||
if t.After(cutoff) {
|
||
kept = append(kept, t)
|
||
}
|
||
}
|
||
if len(kept) >= l.limit {
|
||
l.hits[key] = kept
|
||
return false
|
||
}
|
||
l.hits[key] = append(kept, now)
|
||
return true
|
||
}
|
||
|
||
// writeRateLimited responds 429 with Retry-After plus IETF RateLimit headers
|
||
// (draft-ietf-httpapi-ratelimit-headers) so clients can back off before retrying.
|
||
// On deny, remaining is always 0; t/w use the limiter window in seconds.
|
||
func writeRateLimited(w http.ResponseWriter, limit, windowSec int) {
|
||
if limit < 1 {
|
||
limit = 1
|
||
}
|
||
if windowSec < 1 {
|
||
windowSec = 60
|
||
}
|
||
w.Header().Set("Retry-After", strconv.Itoa(windowSec))
|
||
w.Header().Set("RateLimit", fmt.Sprintf(`"http";r=0;t=%d`, windowSec))
|
||
w.Header().Set("RateLimit-Policy", fmt.Sprintf(`"http";q=%d;w=%d`, limit, windowSec))
|
||
Error(w, http.StatusTooManyRequests, "rate limit exceeded")
|
||
}
|
||
|
||
// rateLimitEffectiveCap divides a per-process HTTP base cap across RATE_LIMIT_REPLICAS
|
||
// (ceil) so aggregate traffic under even load approximates the documented RPM.
|
||
// replicas<=1 leaves the base unchanged (default single-instance behavior).
|
||
// Scope: HTTP middleware in this file only — not lockout / StartLimiter / AI / email.
|
||
func rateLimitEffectiveCap(base, replicas int) int {
|
||
if base <= 0 {
|
||
return 1
|
||
}
|
||
if replicas <= 1 {
|
||
return base
|
||
}
|
||
n := (base + replicas - 1) / replicas
|
||
if n < 1 {
|
||
return 1
|
||
}
|
||
return n
|
||
}
|
||
|
||
func (s *Server) rateLimitReplicas() int {
|
||
if s == nil || s.Config.RateLimitReplicas < 1 {
|
||
return 1
|
||
}
|
||
return s.Config.RateLimitReplicas
|
||
}
|
||
|
||
// heavyMutationRPM is the per-company HTTP budget for sync / process / export mutations.
|
||
// In-process only (not shared across replicas). Counts requests, not products in a bulk body.
|
||
const heavyMutationRPM = 30
|
||
|
||
func isHeavyFeedOrProcessMutation(r *http.Request) bool {
|
||
if r.Method != http.MethodPost {
|
||
return false
|
||
}
|
||
path := strings.TrimSuffix(r.URL.Path, "/")
|
||
switch path {
|
||
case "/api/v1/process", "/api/v1/products/process", "/api/processing/jobs":
|
||
return true
|
||
default:
|
||
if strings.HasSuffix(path, "/sync-process-sample") || strings.HasSuffix(path, "/extract-schema") {
|
||
return true
|
||
}
|
||
// Export generate / selected-product export — heavy CPU + IO per request.
|
||
if strings.Contains(path, "/export-feeds/") &&
|
||
(strings.HasSuffix(path, "/generate") || strings.HasSuffix(path, "/export-products")) {
|
||
return true
|
||
}
|
||
// Process job retries also consume StartLimiter capacity.
|
||
if strings.HasSuffix(path, "/retry") &&
|
||
(strings.Contains(path, "/processing/jobs/") || strings.Contains(path, "/process/")) {
|
||
return true
|
||
}
|
||
// Feed sync downloads/parses remote content — throttle both /api and /api/v1.
|
||
// Store connector syncs (/woocommerce/sync, /shopify/…) are intentionally excluded;
|
||
// they use connector-specific workers and are not part of this shared bucket.
|
||
return strings.HasSuffix(path, "/sync") && strings.Contains(path, "/feeds/")
|
||
}
|
||
}
|
||
|
||
// RateLimitV1Process throttles heavy process / feed sync / export mutations per company.
|
||
// Limit is in-process (per API replica); set RATE_LIMIT_REPLICAS or prefer edge limits when running multiple replicas.
|
||
func (s *Server) RateLimitV1Process(next http.Handler) http.Handler {
|
||
cap := rateLimitEffectiveCap(heavyMutationRPM, s.rateLimitReplicas())
|
||
limiter := newSlidingWindowLimiter(cap, time.Minute)
|
||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
if !isHeavyFeedOrProcessMutation(r) {
|
||
next.ServeHTTP(w, r)
|
||
return
|
||
}
|
||
cid, ok := CompanyIDFromContext(r.Context())
|
||
key := "anon"
|
||
if ok && cid != uuid.Nil {
|
||
key = cid.String()
|
||
}
|
||
if !limiter.allow(key) {
|
||
writeRateLimited(w, cap, 60)
|
||
return
|
||
}
|
||
next.ServeHTTP(w, r)
|
||
})
|
||
}
|
||
|
||
// publicRPM is the per-IP budget for unauthenticated /api/public routes (plans, logos, …).
|
||
const publicRPM = 30
|
||
|
||
// RateLimitPublic throttles unauthenticated /api/public routes per client IP
|
||
// (RemoteAddr; rewritten only via TrustedRealIP when TRUSTED_PROXIES is set).
|
||
func (s *Server) RateLimitPublic(next http.Handler) http.Handler {
|
||
cap := rateLimitEffectiveCap(publicRPM, s.rateLimitReplicas())
|
||
limiter := newSlidingWindowLimiter(cap, time.Minute)
|
||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
key := strings.TrimSpace(r.RemoteAddr)
|
||
if key == "" {
|
||
key = "unknown"
|
||
}
|
||
if !limiter.allow(key) {
|
||
writeRateLimited(w, cap, 60)
|
||
return
|
||
}
|
||
next.ServeHTTP(w, r)
|
||
})
|
||
}
|
||
|
||
// Public export token scraping budgets (in-process; see file header for replicas).
|
||
const (
|
||
publicExportIPRPM = 30 // well-formed export GETs per IP
|
||
publicExportProbeRPM = 15 // invalid-shape token probes per IP (enumeration)
|
||
publicExportTokenRPM = 30 // polls per public_token (known-token scrape)
|
||
)
|
||
|
||
// RateLimitPublicExport throttles tokenized export GETs harder than generic /api/public.
|
||
// Invalid token shapes are rejected here (no DB) and counted against a probe budget.
|
||
func (s *Server) RateLimitPublicExport(next http.Handler) http.Handler {
|
||
ipCap := rateLimitEffectiveCap(publicExportIPRPM, s.rateLimitReplicas())
|
||
probeCap := rateLimitEffectiveCap(publicExportProbeRPM, s.rateLimitReplicas())
|
||
tokenCap := rateLimitEffectiveCap(publicExportTokenRPM, s.rateLimitReplicas())
|
||
ipLimiter := newSlidingWindowLimiter(ipCap, time.Minute)
|
||
probeLimiter := newSlidingWindowLimiter(probeCap, time.Minute)
|
||
tokenLimiter := newSlidingWindowLimiter(tokenCap, time.Minute)
|
||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
ip := strings.TrimSpace(r.RemoteAddr)
|
||
if ip == "" {
|
||
ip = "unknown"
|
||
}
|
||
token := strings.ToLower(strings.TrimSpace(chi.URLParam(r, "token")))
|
||
if !feeds.ValidPublicToken(token) {
|
||
if !probeLimiter.allow(ip) {
|
||
writeRateLimited(w, probeCap, 60)
|
||
return
|
||
}
|
||
// Same body as writePublicExportError — no oracle for token existence.
|
||
Error(w, http.StatusNotFound, "export feed not found")
|
||
return
|
||
}
|
||
if !ipLimiter.allow(ip) {
|
||
writeRateLimited(w, ipCap, 60)
|
||
return
|
||
}
|
||
if !tokenLimiter.allow("t:" + token) {
|
||
writeRateLimited(w, tokenCap, 60)
|
||
return
|
||
}
|
||
next.ServeHTTP(w, r)
|
||
})
|
||
}
|
||
|
||
// API key surface budgets (in-process).
|
||
const (
|
||
apiKeyAttemptRPM = 60 // keyed /api/v1 requests per IP (brute-force / spray)
|
||
apiKeyCompanyRPM = 120 // authenticated /api/v1 requests per company
|
||
)
|
||
|
||
// RateLimitAPIKeyAttempts throttles /api/v1 requests that present an API key, per IP.
|
||
// Mount before RequireAPIKey so invalid keys still consume budget.
|
||
func (s *Server) RateLimitAPIKeyAttempts(next http.Handler) http.Handler {
|
||
cap := rateLimitEffectiveCap(apiKeyAttemptRPM, s.rateLimitReplicas())
|
||
limiter := newSlidingWindowLimiter(cap, time.Minute)
|
||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
if extractAPIKey(r) == "" {
|
||
next.ServeHTTP(w, r)
|
||
return
|
||
}
|
||
key := strings.TrimSpace(r.RemoteAddr)
|
||
if key == "" {
|
||
key = "unknown"
|
||
}
|
||
if !limiter.allow(key) {
|
||
writeRateLimited(w, cap, 60)
|
||
return
|
||
}
|
||
next.ServeHTTP(w, r)
|
||
})
|
||
}
|
||
|
||
// RateLimitAPIKey throttles authenticated /api/v1 traffic per company (API4 abuse cap).
|
||
func (s *Server) RateLimitAPIKey(next http.Handler) http.Handler {
|
||
cap := rateLimitEffectiveCap(apiKeyCompanyRPM, s.rateLimitReplicas())
|
||
limiter := newSlidingWindowLimiter(cap, time.Minute)
|
||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
cid, ok := CompanyIDFromContext(r.Context())
|
||
key := "anon"
|
||
if ok && cid != uuid.Nil {
|
||
key = cid.String()
|
||
}
|
||
if !limiter.allow(key) {
|
||
writeRateLimited(w, cap, 60)
|
||
return
|
||
}
|
||
next.ServeHTTP(w, r)
|
||
})
|
||
}
|
||
|
||
func isMarketingGenerateOrSend(r *http.Request) bool {
|
||
if r.Method != http.MethodPost {
|
||
return false
|
||
}
|
||
path := strings.TrimSuffix(r.URL.Path, "/")
|
||
switch {
|
||
case path == "/api/seo/apply":
|
||
return true
|
||
case path == "/api/campaigns/generate":
|
||
return true
|
||
case strings.HasSuffix(path, "/generate") && strings.Contains(path, "/campaigns/"):
|
||
return true
|
||
case path == "/api/campaigns/send" || path == "/api/campaigns/send-test":
|
||
return true
|
||
case strings.HasSuffix(path, "/send") || strings.HasSuffix(path, "/send-test") || strings.HasSuffix(path, "/schedule"):
|
||
return strings.Contains(path, "/campaigns/")
|
||
case path == "/api/integrations/email/send" || path == "/api/email/send":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// RateLimitMarketing throttles campaign generate/send and SEO AI apply per company.
|
||
// In-process only (per API replica); set RATE_LIMIT_REPLICAS or prefer edge limits when running multiple replicas.
|
||
func (s *Server) RateLimitMarketing(next http.Handler) http.Handler {
|
||
genCap := rateLimitEffectiveCap(10, s.rateLimitReplicas())
|
||
sendCap := rateLimitEffectiveCap(30, s.rateLimitReplicas())
|
||
genLimiter := newSlidingWindowLimiter(genCap, time.Minute)
|
||
sendLimiter := newSlidingWindowLimiter(sendCap, time.Minute)
|
||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
if !isMarketingGenerateOrSend(r) {
|
||
next.ServeHTTP(w, r)
|
||
return
|
||
}
|
||
cid, ok := CompanyIDFromContext(r.Context())
|
||
key := "anon"
|
||
if ok && cid != uuid.Nil {
|
||
key = cid.String()
|
||
}
|
||
path := strings.TrimSuffix(r.URL.Path, "/")
|
||
limiter := sendLimiter
|
||
cap := sendCap
|
||
if strings.Contains(path, "generate") || path == "/api/seo/apply" {
|
||
limiter = genLimiter
|
||
cap = genCap
|
||
}
|
||
if !limiter.allow(key) {
|
||
writeRateLimited(w, cap, 60)
|
||
return
|
||
}
|
||
next.ServeHTTP(w, r)
|
||
})
|
||
}
|
||
|
||
const (
|
||
authLoginRPM = 10 // login / invite / set-password / sales-contact per IP
|
||
authRegisterRPM = 5 // registration spam bucket (stricter than login)
|
||
)
|
||
|
||
func isAuthRegister(r *http.Request) bool {
|
||
return r.Method == http.MethodPost && strings.TrimSuffix(r.URL.Path, "/") == "/api/auth/register"
|
||
}
|
||
|
||
func isAuthMutation(r *http.Request) bool {
|
||
if r.Method != http.MethodPost {
|
||
return false
|
||
}
|
||
switch strings.TrimSuffix(r.URL.Path, "/") {
|
||
case "/api/auth/login",
|
||
"/api/auth/register",
|
||
"/api/auth/forgot-password",
|
||
"/api/auth/reset-password",
|
||
"/api/auth/invite-preview",
|
||
"/api/auth/accept-invite",
|
||
"/api/auth/complete-set-password",
|
||
"/api/sales/contact":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// RateLimitAuth throttles unauthenticated auth POSTs per client IP
|
||
// (RemoteAddr; rewritten only via TrustedRealIP when TRUSTED_PROXIES is set).
|
||
// Register uses a stricter bucket so login brute-force and signup spam do not share budget.
|
||
func (s *Server) RateLimitAuth(next http.Handler) http.Handler {
|
||
loginCap := rateLimitEffectiveCap(authLoginRPM, s.rateLimitReplicas())
|
||
registerCap := rateLimitEffectiveCap(authRegisterRPM, s.rateLimitReplicas())
|
||
loginLimiter := newSlidingWindowLimiter(loginCap, time.Minute)
|
||
registerLimiter := newSlidingWindowLimiter(registerCap, time.Minute)
|
||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
if !isAuthMutation(r) {
|
||
next.ServeHTTP(w, r)
|
||
return
|
||
}
|
||
key := strings.TrimSpace(r.RemoteAddr)
|
||
if key == "" {
|
||
key = "unknown"
|
||
}
|
||
limiter := loginLimiter
|
||
cap := loginCap
|
||
if isAuthRegister(r) {
|
||
limiter = registerLimiter
|
||
cap = registerCap
|
||
}
|
||
if !limiter.allow(key) {
|
||
writeRateLimited(w, cap, 60)
|
||
return
|
||
}
|
||
next.ServeHTTP(w, r)
|
||
})
|
||
}
|
||
|
||
// adminPlanFeaturesGetRPM caps GET /api/admin/plans/{id}/features per user.
|
||
// In-process only; stops client refetch storms from saturating the API.
|
||
const adminPlanFeaturesGetRPM = 60
|
||
|
||
func isAdminPlanFeaturesGet(r *http.Request) bool {
|
||
if r.Method != http.MethodGet {
|
||
return false
|
||
}
|
||
path := strings.TrimSuffix(r.URL.Path, "/")
|
||
if !strings.HasPrefix(path, "/api/admin/plans/") {
|
||
return false
|
||
}
|
||
return strings.HasSuffix(path, "/features")
|
||
}
|
||
|
||
// RateLimitAdminPlanFeatures throttles repeated GET plan-feature matrix fetches per user.
|
||
func (s *Server) RateLimitAdminPlanFeatures(next http.Handler) http.Handler {
|
||
cap := rateLimitEffectiveCap(adminPlanFeaturesGetRPM, s.rateLimitReplicas())
|
||
limiter := newSlidingWindowLimiter(cap, time.Minute)
|
||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
if !isAdminPlanFeaturesGet(r) {
|
||
next.ServeHTTP(w, r)
|
||
return
|
||
}
|
||
uid, ok := UserIDFromContext(r.Context())
|
||
key := "anon"
|
||
if ok && uid != uuid.Nil {
|
||
key = uid.String()
|
||
}
|
||
if !limiter.allow(key) {
|
||
writeRateLimited(w, cap, 60)
|
||
return
|
||
}
|
||
next.ServeHTTP(w, r)
|
||
})
|
||
}
|
||
|
||
// adminAnalyticsGetRPM caps expensive admin diagnostic/analytics GETs per user.
|
||
const adminAnalyticsGetRPM = 20
|
||
|
||
func isAdminAnalyticsGet(r *http.Request) bool {
|
||
if r.Method != http.MethodGet {
|
||
return false
|
||
}
|
||
path := strings.TrimSuffix(r.URL.Path, "/")
|
||
return path == "/api/admin/analytics" || path == "/api/admin/diagnostics"
|
||
}
|
||
|
||
// RateLimitAdminAnalytics throttles expensive platform analytics/diagnostics reads per user.
|
||
func (s *Server) RateLimitAdminAnalytics(next http.Handler) http.Handler {
|
||
cap := rateLimitEffectiveCap(adminAnalyticsGetRPM, s.rateLimitReplicas())
|
||
limiter := newSlidingWindowLimiter(cap, time.Minute)
|
||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
if !isAdminAnalyticsGet(r) {
|
||
next.ServeHTTP(w, r)
|
||
return
|
||
}
|
||
uid, ok := UserIDFromContext(r.Context())
|
||
key := "anon"
|
||
if ok && uid != uuid.Nil {
|
||
key = uid.String()
|
||
}
|
||
if !limiter.allow(key) {
|
||
writeRateLimited(w, cap, 60)
|
||
return
|
||
}
|
||
next.ServeHTTP(w, r)
|
||
})
|
||
}
|
||
|
||
// aiProbeRPM caps LLM/mail credential probes (cost / abuse).
|
||
const aiProbeRPM = 10
|
||
|
||
func isAIOrMailProbePOST(r *http.Request) bool {
|
||
if r.Method != http.MethodPost {
|
||
return false
|
||
}
|
||
path := strings.TrimSuffix(r.URL.Path, "/")
|
||
switch {
|
||
case path == "/api/integrations/ai/test":
|
||
return true
|
||
case path == "/api/admin/settings/mail/test":
|
||
return true
|
||
case strings.HasPrefix(path, "/api/admin/settings/ai-roles/") && strings.HasSuffix(path, "/test"):
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// RateLimitAIProbes throttles AI/mail test probes per user (admin) or company (tenant).
|
||
func (s *Server) RateLimitAIProbes(next http.Handler) http.Handler {
|
||
cap := rateLimitEffectiveCap(aiProbeRPM, s.rateLimitReplicas())
|
||
limiter := newSlidingWindowLimiter(cap, time.Minute)
|
||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
if !isAIOrMailProbePOST(r) {
|
||
next.ServeHTTP(w, r)
|
||
return
|
||
}
|
||
key := "anon"
|
||
if uid, ok := UserIDFromContext(r.Context()); ok && uid != uuid.Nil {
|
||
key = "u:" + uid.String()
|
||
} else if cid, ok := CompanyIDFromContext(r.Context()); ok && cid != uuid.Nil {
|
||
key = "c:" + cid.String()
|
||
}
|
||
if !limiter.allow(key) {
|
||
writeRateLimited(w, cap, 60)
|
||
return
|
||
}
|
||
next.ServeHTTP(w, r)
|
||
})
|
||
}
|