descrybe/apps/api/internal/processing/sanitize.go

package processing

import (
	"regexp"
	"strings"
	"unicode"

	"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
)

const maxPromptFieldRunes = 4000

// maxOutputFieldRunes bounds stored / polled model text (titles + multi-section
// formula HTML descriptions). Must stay well above MaxTokensEnhance (~24k tokens)
// so SanitizeOutput does not chop JSON completion bodies or A1 HTML mid-string.
const maxOutputFieldRunes = 120000

var controlOrInject = regexp.MustCompile(`(?i)(ignore\s+(all\s+)?(previous|prior|above)|disregard\s+(all\s+)?(previous|prior)|forget\s+(all\s+)?(previous|prior)|system\s*:|assistant\s*:|<\s*/?\s*script)`)

// SanitizeText strips control chars, truncates, and soft-neutralizes prompt-injection phrases.
func SanitizeText(s string) string {
	s = strings.TrimSpace(s)
	if s == "" {
		return ""
	}
	var b strings.Builder
	b.Grow(len(s))
	for _, r := range s {
		if r == '\n' || r == '\t' || unicode.IsPrint(r) {
			b.WriteRune(r)
		}
	}
	out := b.String()
	out = controlOrInject.ReplaceAllString(out, "[filtered]")
	return truncateRunes(out, maxPromptFieldRunes)
}

// SanitizeOutput keeps model text printable and bounded for storage/UI.
// Uses a higher rune cap than SanitizeText so formula HTML and full enhance
// JSON completions are not truncated at 4k.
func SanitizeOutput(s string) string {
	s = strings.TrimSpace(s)
	if s == "" {
		return ""
	}
	var b strings.Builder
	b.Grow(len(s))
	for _, r := range s {
		if r == '\n' || r == '\t' || unicode.IsPrint(r) {
			b.WriteRune(r)
		}
	}
	return truncateRunes(b.String(), maxOutputFieldRunes)
}

func truncateRunes(s string, max int) string {
	if max <= 0 {
		return ""
	}
	n := 0
	for i := range s {
		if n == max {
			return s[:i]
		}
		n++
	}
	return s
}

const (
	publicErrProcessingFailed    = "processing_failed"
	publicErrProviderUnavailable = "provider_unavailable"
)

// TruncateError returns a safe, short error string for DB storage / API clients.
// Secret-like substrings and logredact matches become an opaque message so
// job step_progress notes and v1 item errors cannot leak keys/JWTs/DSNs/emails.
// Common provider transport failures are rewritten to short user-facing text
// (no dial URLs / Go net strings). Remaining provider internals map to stable
// public codes (processing_failed / provider_unavailable).
func TruncateError(err error) string {
	if err == nil {
		return ""
	}
	msg := strings.ReplaceAll(err.Error(), "\n", " ")
	lower := strings.ToLower(msg)
	for _, secretHint := range []string{
		"api-key", "api_key", "authorization", "bearer ",
		"sk-", "sk_live", "sk_test", "whsec_", "password", "passwd",
	} {
		if strings.Contains(lower, secretHint) {
			return "provider error (details redacted)"
		}
	}
	redacted := logredact.String(msg)
	if strings.Contains(redacted, logredact.Redacted) {
		return "provider error (details redacted)"
	}
	if friendly := classifyProviderError(redacted); friendly != "" {
		return friendly
	}
	// Drop retry-exhaustion wrapper when the inner message is already clear.
	cleaned := stripOpenAIRetryPrefix(redacted)
	if friendly := classifyProviderError(cleaned); friendly != "" {
		return friendly
	}
	if code := mapPublicErrorCode(cleaned); code != "" {
		return code
	}
	return truncateRunes(cleaned, 500)
}

// PublicV1Error maps a stored job/item error to a stable public v1 string.
// Dashboard notes / step_progress may still carry operator detail; public JSON must not.
func PublicV1Error(msg string) string {
	msg = strings.TrimSpace(msg)
	if msg == "" {
		return ""
	}
	switch msg {
	case publicErrProcessingFailed, publicErrProviderUnavailable,
		"Product data not available", "Processing failed",
		"provider error (details redacted)":
		return msg
	}
	if strings.HasPrefix(msg, "processing.job.error.") {
		return msg
	}
	if strings.Contains(strings.ToLower(msg), "insufficient credits") {
		return msg
	}
	if friendly := classifyProviderError(msg); friendly != "" {
		return friendly
	}
	cleaned := stripOpenAIRetryPrefix(msg)
	if friendly := classifyProviderError(cleaned); friendly != "" {
		return friendly
	}
	if code := mapPublicErrorCode(cleaned); code != "" {
		return code
	}
	if code := mapPublicV1FallbackCode(cleaned); code != "" {
		return code
	}
	if looksLikeInternalProviderError(cleaned) {
		return publicErrProcessingFailed
	}
	return truncateRunes(cleaned, 500)
}

func stripOpenAIRetryPrefix(msg string) string {
	cleaned := msg
	for _, prefix := range []string{
		"openai retries exhausted: ",
		"openai embedding retries exhausted: ",
	} {
		if strings.HasPrefix(strings.ToLower(cleaned), prefix) {
			return strings.TrimSpace(cleaned[len(prefix):])
		}
	}
	return cleaned
}

func mapPublicErrorCode(msg string) string {
	lower := strings.ToLower(strings.TrimSpace(msg))
	if lower == "" {
		return ""
	}
	switch {
	case strings.Contains(lower, "max_tokens"),
		strings.Contains(lower, "length-capped"):
		return publicErrProcessingFailed
	case strings.Contains(lower, "openai unset"),
		strings.Contains(lower, "platform openai"):
		return publicErrProviderUnavailable
	case strings.Contains(lower, "ai_enhance: skipped"):
		return publicErrProcessingFailed
	case strings.Contains(lower, "openai"),
		strings.Contains(lower, "gpt-"),
		strings.Contains(lower, " for model"):
		return publicErrProcessingFailed
	}
	return ""
}

func mapPublicV1FallbackCode(msg string) string {
	lower := strings.ToLower(strings.TrimSpace(msg))
	switch {
	case strings.Contains(lower, "unavailable"),
		strings.Contains(lower, "http 503"),
		strings.Contains(lower, "overloaded"):
		return publicErrProviderUnavailable
	}
	return ""
}

func looksLikeInternalProviderError(msg string) bool {
	lower := strings.ToLower(msg)
	for _, n := range []string{
		"openai", "max_tokens", "length-capped", "gpt-", "ai_enhance",
		"finish_reason", "chat/completions",
	} {
		if strings.Contains(lower, n) {
			return true
		}
	}
	return false
}

// classifyProviderError maps common OpenAI-compatible transport/auth failures
// to short operator-facing text. Returns "" when msg should be kept as-is.
func classifyProviderError(msg string) string {
	lower := strings.ToLower(strings.TrimSpace(msg))
	if lower == "" {
		return ""
	}
	switch {
	case strings.Contains(lower, "connection refused"),
		strings.Contains(lower, "connectex"),
		strings.Contains(lower, "no connection could be made"),
		strings.Contains(lower, "actively refused"),
		strings.Contains(lower, "connection reset"),
		strings.Contains(lower, "no such host"),
		strings.Contains(lower, "dial tcp"):
		return "AI provider unreachable — check base URL and that the service is running"
	case strings.Contains(lower, "deadline exceeded"),
		strings.Contains(lower, "client.timeout"),
		strings.Contains(lower, "i/o timeout"),
		strings.Contains(lower, "timed out"):
		return "AI provider timed out — try again or check provider load"
	case lower == "unauthorized",
		strings.Contains(lower, "http 401"),
		strings.Contains(lower, "invalid api key"),
		strings.Contains(lower, "incorrect api key"),
		strings.Contains(lower, "invalid_api_key"),
		strings.Contains(lower, "rejected the api key"):
		return "AI provider rejected the API key"
	case strings.Contains(lower, "http 403"),
		lower == "forbidden",
		strings.Contains(lower, "forbidden the request"):
		return "AI provider forbidden the request"
	case lower == "rate limited or server error":
		return "AI provider temporarily unavailable (rate limited or server error)"
	case strings.Contains(lower, "http 429"),
		strings.Contains(lower, "too many requests"),
		lower == "rate limited",
		strings.Contains(lower, "rate limited —"),
		strings.Contains(lower, "rate limited -"):
		return "AI provider rate limited — retry later"
	case strings.Contains(lower, "quota exceeded"),
		strings.Contains(lower, "insufficient_quota"),
		strings.Contains(lower, "billing/quota empty"),
		strings.Contains(lower, "prepaid api credits"),
		strings.Contains(lower, "check your plan and billing"),
		strings.Contains(lower, "http 429 — usually empty"):
		return "API billing/quota empty — add prepaid API credits in Platform billing (not ChatGPT Plus)"
	case strings.Contains(lower, "model not found"),
		strings.Contains(lower, "lacks access"):
		return "model not found or API key lacks access — check model name and project permissions"
	}
	return ""
}

func stringFromMap(m map[string]any, keys ...string) string {
	if m == nil {
		return ""
	}
	for _, k := range keys {
		if v, ok := m[k]; ok {
			switch t := v.(type) {
			case string:
				if strings.TrimSpace(t) != "" {
					return SanitizeText(t)
				}
			}
		}
	}
	return ""
}