package processing import ( "regexp" "strings" "unicode" "github.com/descrybe/descrybe-v2/apps/api/internal/logredact" ) const maxPromptFieldRunes = 4000 // maxOutputFieldRunes bounds stored / polled model text (titles + multi-section // formula HTML descriptions). Must stay well above MaxTokensEnhance (~16k tokens) // so SanitizeOutput does not chop JSON completion bodies or A1 HTML mid-string. const maxOutputFieldRunes = 120000 var controlOrInject = regexp.MustCompile(`(?i)(ignore\s+(all\s+)?(previous|prior|above)|disregard\s+(all\s+)?(previous|prior)|forget\s+(all\s+)?(previous|prior)|system\s*:|assistant\s*:|<\s*/?\s*script)`) // SanitizeText strips control chars, truncates, and soft-neutralizes prompt-injection phrases. func SanitizeText(s string) string { s = strings.TrimSpace(s) if s == "" { return "" } var b strings.Builder b.Grow(len(s)) for _, r := range s { if r == '\n' || r == '\t' || unicode.IsPrint(r) { b.WriteRune(r) } } out := b.String() out = controlOrInject.ReplaceAllString(out, "[filtered]") return truncateRunes(out, maxPromptFieldRunes) } // SanitizeOutput keeps model text printable and bounded for storage/UI. // Uses a higher rune cap than SanitizeText so formula HTML and full enhance // JSON completions are not truncated at 4k. func SanitizeOutput(s string) string { s = strings.TrimSpace(s) if s == "" { return "" } var b strings.Builder b.Grow(len(s)) for _, r := range s { if r == '\n' || r == '\t' || unicode.IsPrint(r) { b.WriteRune(r) } } return truncateRunes(b.String(), maxOutputFieldRunes) } func truncateRunes(s string, max int) string { if max <= 0 { return "" } n := 0 for i := range s { if n == max { return s[:i] } n++ } return s } // TruncateError returns a safe, short error string for DB storage / API clients. // Secret-like substrings and logredact matches become an opaque message so // job step_progress notes and v1 item errors cannot leak keys/JWTs/DSNs/emails. // Common provider transport failures are rewritten to short user-facing text // (no dial URLs / Go net strings) while preserving unrelated provider messages. func TruncateError(err error) string { if err == nil { return "" } msg := strings.ReplaceAll(err.Error(), "\n", " ") lower := strings.ToLower(msg) for _, secretHint := range []string{ "api-key", "api_key", "authorization", "bearer ", "sk-", "sk_live", "sk_test", "whsec_", "password", "passwd", } { if strings.Contains(lower, secretHint) { return "provider error (details redacted)" } } redacted := logredact.String(msg) if strings.Contains(redacted, logredact.Redacted) { return "provider error (details redacted)" } if friendly := classifyProviderError(redacted); friendly != "" { return friendly } // Drop retry-exhaustion wrapper when the inner message is already clear. cleaned := redacted for _, prefix := range []string{ "openai retries exhausted: ", "openai embedding retries exhausted: ", } { if strings.HasPrefix(strings.ToLower(cleaned), prefix) { cleaned = strings.TrimSpace(cleaned[len(prefix):]) break } } if friendly := classifyProviderError(cleaned); friendly != "" { return friendly } return truncateRunes(cleaned, 500) } // classifyProviderError maps common OpenAI-compatible transport/auth failures // to short operator-facing text. Returns "" when msg should be kept as-is. func classifyProviderError(msg string) string { lower := strings.ToLower(strings.TrimSpace(msg)) if lower == "" { return "" } switch { case strings.Contains(lower, "connection refused"), strings.Contains(lower, "connectex"), strings.Contains(lower, "no connection could be made"), strings.Contains(lower, "actively refused"), strings.Contains(lower, "connection reset"), strings.Contains(lower, "no such host"), strings.Contains(lower, "dial tcp"): return "AI provider unreachable — check base URL and that the service is running" case strings.Contains(lower, "deadline exceeded"), strings.Contains(lower, "client.timeout"), strings.Contains(lower, "i/o timeout"), strings.Contains(lower, "timed out"): return "AI provider timed out — try again or check provider load" case lower == "unauthorized", strings.Contains(lower, "http 401"), strings.Contains(lower, "invalid api key"), strings.Contains(lower, "incorrect api key"), strings.Contains(lower, "invalid_api_key"): return "AI provider rejected the API key" case strings.Contains(lower, "http 403"), lower == "forbidden": return "AI provider forbidden the request" case strings.Contains(lower, "http 429"), strings.Contains(lower, "too many requests"), lower == "rate limited": return "AI provider rate limited — retry later" case lower == "rate limited or server error": return "AI provider temporarily unavailable (rate limited or server error)" } return "" } func stringFromMap(m map[string]any, keys ...string) string { if m == nil { return "" } for _, k := range keys { if v, ok := m[k]; ok { switch t := v.(type) { case string: if strings.TrimSpace(t) != "" { return SanitizeText(t) } } } } return "" }