package processing import ( "regexp" "strings" "unicode" "github.com/descrybe/descrybe-v2/apps/api/internal/logredact" ) const maxPromptFieldRunes = 4000 // maxOutputFieldRunes bounds stored / polled model text (titles + multi-section // formula HTML descriptions). Must stay well above MaxTokensEnhance (~16k tokens) // so SanitizeOutput does not chop JSON completion bodies or A1 HTML mid-string. const maxOutputFieldRunes = 120000 var controlOrInject = regexp.MustCompile(`(?i)(ignore\s+(all\s+)?(previous|prior|above)|disregard\s+(all\s+)?(previous|prior)|forget\s+(all\s+)?(previous|prior)|system\s*:|assistant\s*:|<\s*/?\s*script)`) // SanitizeText strips control chars, truncates, and soft-neutralizes prompt-injection phrases. func SanitizeText(s string) string { s = strings.TrimSpace(s) if s == "" { return "" } var b strings.Builder b.Grow(len(s)) for _, r := range s { if r == '\n' || r == '\t' || unicode.IsPrint(r) { b.WriteRune(r) } } out := b.String() out = controlOrInject.ReplaceAllString(out, "[filtered]") return truncateRunes(out, maxPromptFieldRunes) } // SanitizeOutput keeps model text printable and bounded for storage/UI. // Uses a higher rune cap than SanitizeText so formula HTML and full enhance // JSON completions are not truncated at 4k. func SanitizeOutput(s string) string { s = strings.TrimSpace(s) if s == "" { return "" } var b strings.Builder b.Grow(len(s)) for _, r := range s { if r == '\n' || r == '\t' || unicode.IsPrint(r) { b.WriteRune(r) } } return truncateRunes(b.String(), maxOutputFieldRunes) } func truncateRunes(s string, max int) string { if max <= 0 { return "" } n := 0 for i := range s { if n == max { return s[:i] } n++ } return s } const ( publicErrProcessingFailed = "processing_failed" publicErrProviderUnavailable = "provider_unavailable" ) // TruncateError returns a safe, short error string for DB storage / API clients. // Secret-like substrings and logredact matches become an opaque message so // job step_progress notes and v1 item errors cannot leak keys/JWTs/DSNs/emails. // Common provider transport failures are rewritten to short user-facing text // (no dial URLs / Go net strings). Remaining provider internals map to stable // public codes (processing_failed / provider_unavailable). func TruncateError(err error) string { if err == nil { return "" } msg := strings.ReplaceAll(err.Error(), "\n", " ") lower := strings.ToLower(msg) for _, secretHint := range []string{ "api-key", "api_key", "authorization", "bearer ", "sk-", "sk_live", "sk_test", "whsec_", "password", "passwd", } { if strings.Contains(lower, secretHint) { return "provider error (details redacted)" } } redacted := logredact.String(msg) if strings.Contains(redacted, logredact.Redacted) { return "provider error (details redacted)" } if friendly := classifyProviderError(redacted); friendly != "" { return friendly } // Drop retry-exhaustion wrapper when the inner message is already clear. cleaned := stripOpenAIRetryPrefix(redacted) if friendly := classifyProviderError(cleaned); friendly != "" { return friendly } if code := mapPublicErrorCode(cleaned); code != "" { return code } return truncateRunes(cleaned, 500) } // PublicV1Error maps a stored job/item error to a stable public v1 string. // Dashboard notes / step_progress may still carry operator detail; public JSON must not. func PublicV1Error(msg string) string { msg = strings.TrimSpace(msg) if msg == "" { return "" } switch msg { case publicErrProcessingFailed, publicErrProviderUnavailable, "Product data not available", "Processing failed", "provider error (details redacted)": return msg } if strings.HasPrefix(msg, "processing.job.error.") { return msg } if strings.Contains(strings.ToLower(msg), "insufficient credits") { return msg } if friendly := classifyProviderError(msg); friendly != "" { return friendly } cleaned := stripOpenAIRetryPrefix(msg) if friendly := classifyProviderError(cleaned); friendly != "" { return friendly } if code := mapPublicErrorCode(cleaned); code != "" { return code } if looksLikeInternalProviderError(cleaned) { return publicErrProcessingFailed } return truncateRunes(cleaned, 500) } func stripOpenAIRetryPrefix(msg string) string { cleaned := msg for _, prefix := range []string{ "openai retries exhausted: ", "openai embedding retries exhausted: ", } { if strings.HasPrefix(strings.ToLower(cleaned), prefix) { return strings.TrimSpace(cleaned[len(prefix):]) } } return cleaned } func mapPublicErrorCode(msg string) string { lower := strings.ToLower(strings.TrimSpace(msg)) if lower == "" { return "" } switch { case strings.Contains(lower, "max_tokens"), strings.Contains(lower, "length-capped"): return publicErrProcessingFailed case strings.Contains(lower, "openai unset"), strings.Contains(lower, "platform openai"): return publicErrProviderUnavailable case strings.Contains(lower, "ai_enhance: skipped"): return publicErrProcessingFailed case strings.Contains(lower, "unavailable"), strings.Contains(lower, "http 503"), strings.Contains(lower, "overloaded"): return publicErrProviderUnavailable case strings.Contains(lower, "openai"), strings.Contains(lower, "gpt-"), strings.Contains(lower, " for model"): return publicErrProcessingFailed } return "" } func looksLikeInternalProviderError(msg string) bool { lower := strings.ToLower(msg) for _, n := range []string{ "openai", "max_tokens", "length-capped", "gpt-", "ai_enhance", "finish_reason", "chat/completions", } { if strings.Contains(lower, n) { return true } } return false } // classifyProviderError maps common OpenAI-compatible transport/auth failures // to short operator-facing text. Returns "" when msg should be kept as-is. func classifyProviderError(msg string) string { lower := strings.ToLower(strings.TrimSpace(msg)) if lower == "" { return "" } switch { case strings.Contains(lower, "connection refused"), strings.Contains(lower, "connectex"), strings.Contains(lower, "no connection could be made"), strings.Contains(lower, "actively refused"), strings.Contains(lower, "connection reset"), strings.Contains(lower, "no such host"), strings.Contains(lower, "dial tcp"): return "AI provider unreachable — check base URL and that the service is running" case strings.Contains(lower, "deadline exceeded"), strings.Contains(lower, "client.timeout"), strings.Contains(lower, "i/o timeout"), strings.Contains(lower, "timed out"): return "AI provider timed out — try again or check provider load" case lower == "unauthorized", strings.Contains(lower, "http 401"), strings.Contains(lower, "invalid api key"), strings.Contains(lower, "incorrect api key"), strings.Contains(lower, "invalid_api_key"): return "AI provider rejected the API key" case strings.Contains(lower, "http 403"), lower == "forbidden": return "AI provider forbidden the request" case strings.Contains(lower, "http 429"), strings.Contains(lower, "too many requests"), lower == "rate limited": return "AI provider rate limited — retry later" case lower == "rate limited or server error": return "AI provider temporarily unavailable (rate limited or server error)" } return "" } func stringFromMap(m map[string]any, keys ...string) string { if m == nil { return "" } for _, k := range keys { if v, ok := m[k]; ok { switch t := v.(type) { case string: if strings.TrimSpace(t) != "" { return SanitizeText(t) } } } } return "" }