fix
This commit is contained in:
@@ -67,11 +67,17 @@ func truncateRunes(s string, max int) string {
|
||||
return s
|
||||
}
|
||||
|
||||
const (
|
||||
publicErrProcessingFailed = "processing_failed"
|
||||
publicErrProviderUnavailable = "provider_unavailable"
|
||||
)
|
||||
|
||||
// TruncateError returns a safe, short error string for DB storage / API clients.
|
||||
// Secret-like substrings and logredact matches become an opaque message so
|
||||
// job step_progress notes and v1 item errors cannot leak keys/JWTs/DSNs/emails.
|
||||
// Common provider transport failures are rewritten to short user-facing text
|
||||
// (no dial URLs / Go net strings) while preserving unrelated provider messages.
|
||||
// (no dial URLs / Go net strings). Remaining provider internals map to stable
|
||||
// public codes (processing_failed / provider_unavailable).
|
||||
func TruncateError(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
@@ -94,20 +100,101 @@ func TruncateError(err error) string {
|
||||
return friendly
|
||||
}
|
||||
// Drop retry-exhaustion wrapper when the inner message is already clear.
|
||||
cleaned := redacted
|
||||
cleaned := stripOpenAIRetryPrefix(redacted)
|
||||
if friendly := classifyProviderError(cleaned); friendly != "" {
|
||||
return friendly
|
||||
}
|
||||
if code := mapPublicErrorCode(cleaned); code != "" {
|
||||
return code
|
||||
}
|
||||
return truncateRunes(cleaned, 500)
|
||||
}
|
||||
|
||||
// PublicV1Error maps a stored job/item error to a stable public v1 string.
|
||||
// Dashboard notes / step_progress may still carry operator detail; public JSON must not.
|
||||
func PublicV1Error(msg string) string {
|
||||
msg = strings.TrimSpace(msg)
|
||||
if msg == "" {
|
||||
return ""
|
||||
}
|
||||
switch msg {
|
||||
case publicErrProcessingFailed, publicErrProviderUnavailable,
|
||||
"Product data not available", "Processing failed",
|
||||
"provider error (details redacted)":
|
||||
return msg
|
||||
}
|
||||
if strings.HasPrefix(msg, "processing.job.error.") {
|
||||
return msg
|
||||
}
|
||||
if strings.Contains(strings.ToLower(msg), "insufficient credits") {
|
||||
return msg
|
||||
}
|
||||
if friendly := classifyProviderError(msg); friendly != "" {
|
||||
return friendly
|
||||
}
|
||||
cleaned := stripOpenAIRetryPrefix(msg)
|
||||
if friendly := classifyProviderError(cleaned); friendly != "" {
|
||||
return friendly
|
||||
}
|
||||
if code := mapPublicErrorCode(cleaned); code != "" {
|
||||
return code
|
||||
}
|
||||
if looksLikeInternalProviderError(cleaned) {
|
||||
return publicErrProcessingFailed
|
||||
}
|
||||
return truncateRunes(cleaned, 500)
|
||||
}
|
||||
|
||||
func stripOpenAIRetryPrefix(msg string) string {
|
||||
cleaned := msg
|
||||
for _, prefix := range []string{
|
||||
"openai retries exhausted: ",
|
||||
"openai embedding retries exhausted: ",
|
||||
} {
|
||||
if strings.HasPrefix(strings.ToLower(cleaned), prefix) {
|
||||
cleaned = strings.TrimSpace(cleaned[len(prefix):])
|
||||
break
|
||||
return strings.TrimSpace(cleaned[len(prefix):])
|
||||
}
|
||||
}
|
||||
if friendly := classifyProviderError(cleaned); friendly != "" {
|
||||
return friendly
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func mapPublicErrorCode(msg string) string {
|
||||
lower := strings.ToLower(strings.TrimSpace(msg))
|
||||
if lower == "" {
|
||||
return ""
|
||||
}
|
||||
return truncateRunes(cleaned, 500)
|
||||
switch {
|
||||
case strings.Contains(lower, "max_tokens"),
|
||||
strings.Contains(lower, "length-capped"):
|
||||
return publicErrProcessingFailed
|
||||
case strings.Contains(lower, "openai unset"),
|
||||
strings.Contains(lower, "platform openai"):
|
||||
return publicErrProviderUnavailable
|
||||
case strings.Contains(lower, "ai_enhance: skipped"):
|
||||
return publicErrProcessingFailed
|
||||
case strings.Contains(lower, "unavailable"),
|
||||
strings.Contains(lower, "http 503"),
|
||||
strings.Contains(lower, "overloaded"):
|
||||
return publicErrProviderUnavailable
|
||||
case strings.Contains(lower, "openai"),
|
||||
strings.Contains(lower, "gpt-"),
|
||||
strings.Contains(lower, " for model"):
|
||||
return publicErrProcessingFailed
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func looksLikeInternalProviderError(msg string) bool {
|
||||
lower := strings.ToLower(msg)
|
||||
for _, n := range []string{
|
||||
"openai", "max_tokens", "length-capped", "gpt-", "ai_enhance",
|
||||
"finish_reason", "chat/completions",
|
||||
} {
|
||||
if strings.Contains(lower, n) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// classifyProviderError maps common OpenAI-compatible transport/auth failures
|
||||
|
||||
@@ -49,6 +49,65 @@ func TestTruncateError_redactsSecrets(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTruncateError_mapsInternalProviderLeaks(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
in: `openai length-capped at max_tokens=16000 for model "gpt-5.4" (prefer a faster non-reasoning product-enhance model): unexpected EOF`,
|
||||
want: "processing_failed",
|
||||
},
|
||||
{
|
||||
in: "ai_enhance: skipped (platform OpenAI unset; configure admin settings or company BYOK)",
|
||||
want: "provider_unavailable",
|
||||
},
|
||||
{
|
||||
in: "openai retries exhausted: green-chat unavailable",
|
||||
want: "provider_unavailable",
|
||||
},
|
||||
{
|
||||
in: "upstream 503: model overloaded",
|
||||
want: "provider_unavailable",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got := TruncateError(errString(tc.in))
|
||||
if got != tc.want {
|
||||
t.Fatalf("TruncateError(%q)=%q want %q", tc.in, got, tc.want)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(got), "openai") ||
|
||||
strings.Contains(strings.ToLower(got), "max_tokens") ||
|
||||
strings.Contains(got, "gpt-") {
|
||||
t.Fatalf("internal leak in TruncateError: %q → %q", tc.in, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicV1Error_stripsProviderInternals(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{in: `openai length-capped at max_tokens=8000 for model "gpt-4o"`, want: "processing_failed"},
|
||||
{in: "ai_enhance: skipped (platform OpenAI unset; configure admin settings or company BYOK)", want: "provider_unavailable"},
|
||||
{in: "processing_failed", want: "processing_failed"},
|
||||
{in: "Product data not available", want: "Product data not available"},
|
||||
{in: "processing.job.error.all_failed|count=3", want: "processing.job.error.all_failed|count=3"},
|
||||
{in: "insufficient credits", want: "insufficient credits"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got := PublicV1Error(tc.in)
|
||||
if got != tc.want {
|
||||
t.Fatalf("PublicV1Error(%q)=%q want %q", tc.in, got, tc.want)
|
||||
}
|
||||
lower := strings.ToLower(got)
|
||||
if strings.Contains(lower, "openai") || strings.Contains(lower, "max_tokens") {
|
||||
t.Fatalf("public v1 leak: %q → %q", tc.in, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTruncateError_classifiesProviderFailures(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
@@ -68,10 +127,8 @@ in: "unauthorized", want: "AI provider rejected the API key"},
|
||||
in: "openai http 401", want: "AI provider rejected the API key"},
|
||||
{in: "rate limited or server error", want: "AI provider temporarily unavailable"},
|
||||
{in: "too many requests", want: "AI provider rate limited"},
|
||||
{
|
||||
in: "upstream 503: model overloaded", want: "upstream 503: model overloaded"},
|
||||
{
|
||||
in: "openai retries exhausted: green-chat unavailable", want: "green-chat unavailable"},
|
||||
{in: "upstream 503: model overloaded", want: "provider_unavailable"},
|
||||
{in: "openai retries exhausted: green-chat unavailable", want: "provider_unavailable"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got := TruncateError(errString(tc.in))
|
||||
|
||||
@@ -46,6 +46,7 @@ func TestV1ProcessJobItemMarshalJSON_order(t *testing.T) {
|
||||
"category": "Hladilniki",
|
||||
"category_id": "11",
|
||||
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
|
||||
"gpt_response": "should-not-ship",
|
||||
"status": "processed",
|
||||
}
|
||||
b, err := item.MarshalJSON()
|
||||
@@ -56,11 +57,15 @@ func TestV1ProcessJobItemMarshalJSON_order(t *testing.T) {
|
||||
eanAt := strings.Index(s, `"ean"`)
|
||||
titleAt := strings.Index(s, `"title"`)
|
||||
attrsAt := strings.Index(s, `"attributes"`)
|
||||
ppAt := strings.Index(s, `"processed_product_id"`)
|
||||
if eanAt < 0 || titleAt < 0 || attrsAt < 0 || ppAt < 0 {
|
||||
if eanAt < 0 || titleAt < 0 || attrsAt < 0 {
|
||||
t.Fatalf("missing keys in %s", s)
|
||||
}
|
||||
if !(eanAt < titleAt && titleAt < attrsAt && attrsAt < ppAt) {
|
||||
if !(eanAt < titleAt && titleAt < attrsAt) {
|
||||
t.Fatalf("bad key order in %s", s)
|
||||
}
|
||||
for _, leak := range []string{`"processed_product_id"`, `"gpt_response"`} {
|
||||
if strings.Contains(s, leak) {
|
||||
t.Fatalf("allowlist MarshalJSON must omit %s: %s", leak, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
// v1ProcessItemKeyOrder is the human-readable LegacyProcessItem property order.
|
||||
// v1ProcessItemKeyOrder is the human-readable ProcessItem property order.
|
||||
// Important catalog fields first; internal ids / SEO last.
|
||||
var v1ProcessItemKeyOrder = []string{
|
||||
"ean",
|
||||
@@ -23,13 +23,39 @@ var v1ProcessItemKeyOrder = []string{
|
||||
"error",
|
||||
"meta_title",
|
||||
"meta_description",
|
||||
"id",
|
||||
"processed_product_id",
|
||||
"raw_product_id",
|
||||
}
|
||||
|
||||
// MarshalJSON emits LegacyProcessItem keys in a stable human-readable order.
|
||||
// ASSUMPTION: JSON object key order is part of the V1 readability contract for A1.
|
||||
// v1ProcessItemInternalKeys must never appear on public ProcessItem JSON.
|
||||
var v1ProcessItemInternalKeys = map[string]struct{}{
|
||||
"id": {},
|
||||
"processed_product_id": {},
|
||||
"raw_product_id": {},
|
||||
"field_sources": {},
|
||||
"enhance_input_hash": {},
|
||||
"ai_enhance": {},
|
||||
"finish_reason": {},
|
||||
"gpt_response": {},
|
||||
"total_tokens": {},
|
||||
"prompt_tokens": {},
|
||||
"completion_tokens": {},
|
||||
"ai_provider_mode": {},
|
||||
"notes": {},
|
||||
"worker": {},
|
||||
"model": {},
|
||||
"provider": {},
|
||||
}
|
||||
|
||||
func stripV1ProcessItemInternalKeys(item V1ProcessJobItem) {
|
||||
if item == nil {
|
||||
return
|
||||
}
|
||||
for k := range v1ProcessItemInternalKeys {
|
||||
delete(item, k)
|
||||
}
|
||||
}
|
||||
|
||||
// MarshalJSON emits only the public ProcessItem allowlist in a stable order.
|
||||
// Unexpected keys (including pipeline internals) cannot ship.
|
||||
func (item V1ProcessJobItem) MarshalJSON() ([]byte, error) {
|
||||
if item == nil {
|
||||
return []byte("null"), nil
|
||||
@@ -55,43 +81,15 @@ func (item V1ProcessJobItem) MarshalJSON() ([]byte, error) {
|
||||
buf.Write(vb)
|
||||
return nil
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
for _, k := range v1ProcessItemKeyOrder {
|
||||
v, ok := item[k]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
seen[k] = struct{}{}
|
||||
if err := writePair(k, v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// Preserve any unexpected keys deterministically (sorted via encoding/json map).
|
||||
extras := map[string]any{}
|
||||
for k, v := range item {
|
||||
if _, ok := seen[k]; ok {
|
||||
continue
|
||||
}
|
||||
extras[k] = v
|
||||
}
|
||||
if len(extras) > 0 {
|
||||
eb, err := json.Marshal(extras)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// eb is `{...}`; splice inner pairs.
|
||||
inner := bytes.TrimSpace(eb)
|
||||
if len(inner) >= 2 && inner[0] == '{' && inner[len(inner)-1] == '}' {
|
||||
inner = inner[1 : len(inner)-1]
|
||||
if len(bytes.TrimSpace(inner)) > 0 {
|
||||
if !first {
|
||||
buf.WriteByte(',')
|
||||
}
|
||||
first = false
|
||||
buf.Write(inner)
|
||||
}
|
||||
}
|
||||
}
|
||||
buf.WriteByte('}')
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
var v1PartialSteps = []string{"category", "title", "description", "attributes"}
|
||||
@@ -274,7 +275,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
|
||||
"error": "Product data not available",
|
||||
}
|
||||
if itemError != nil && *itemError != "" {
|
||||
item["error"] = *itemError
|
||||
item["error"] = PublicV1Error(*itemError)
|
||||
}
|
||||
full = append(full, item)
|
||||
continue
|
||||
@@ -309,6 +310,9 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
|
||||
if descTxt != nil {
|
||||
descOut = v1PreserveDescription(*descTxt)
|
||||
}
|
||||
if isPromptLeakageTitle(descOut) || isPromptLabelTitle(descOut) {
|
||||
descOut = ""
|
||||
}
|
||||
|
||||
eprelVal := extractEPRELFromAttrs(attrs)
|
||||
attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowedAttrs)
|
||||
@@ -453,7 +457,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
|
||||
// Internal ids (id / processed_product_id / raw_product_id) are omitted from
|
||||
// the public V1 process item shape — use catalog APIs when a UUID is needed.
|
||||
if itemError != nil && *itemError != "" {
|
||||
item["error"] = *itemError
|
||||
item["error"] = PublicV1Error(*itemError)
|
||||
}
|
||||
if len(attrs) > 0 {
|
||||
item["attributes"] = attrs
|
||||
@@ -484,35 +488,72 @@ func nullIfEmptyPtr(s *string) any {
|
||||
return *s
|
||||
}
|
||||
|
||||
// companyOmitsSEOMeta is true when V1/process must omit meta_title / meta_description:
|
||||
// A1 cohort (legacy id), Platform Demo (A1-cloned prompts, no legacy id), or any
|
||||
// company whose categories store A1-style role-section enhance prompts.
|
||||
func companyOmitsSEOMeta(ctx context.Context, p *Pipeline, companyID uuid.UUID) bool {
|
||||
if p == nil || p.Pool == nil {
|
||||
// Known v2 companies.id values that omit SEO meta on V1 process payloads.
|
||||
// A1 Slovenija is also matched via billing.A1LegacyCompanyID (legacy_company_id / PK).
|
||||
const (
|
||||
a1V2CompanyID = "604f23a8-b66e-4b21-8b45-0d72b68f4790"
|
||||
platformDemoV2CompanyID = "2b3159b0-fc08-415b-b248-35ed02a6baab"
|
||||
)
|
||||
|
||||
// CompanyOmitsSEOMetaID is true for A1 Slovenija and Platform Demo by companies.id
|
||||
// (or the migrated MySQL company id). Does not grant Legacy plan privileges.
|
||||
func CompanyOmitsSEOMetaID(companyID uuid.UUID) bool {
|
||||
if companyID == uuid.Nil {
|
||||
return false
|
||||
}
|
||||
id := strings.ToLower(companyID.String())
|
||||
switch id {
|
||||
case a1V2CompanyID, platformDemoV2CompanyID, strings.ToLower(billing.A1LegacyCompanyID):
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// CompanyOmitsSEOMetaLookup is the shared A1/Demo SEO-omit decision used by V1 process
|
||||
// and catalog list/get/update. Matches hardcoded companies.id, A1 cohort (legacy id),
|
||||
// Platform Demo by name, or A1-style --- Title --- / --- Description --- / --- Meta ---
|
||||
// category prompts.
|
||||
func CompanyOmitsSEOMetaLookup(companyID uuid.UUID, legacyID, name string, hasA1SectionPrompts bool) bool {
|
||||
if CompanyOmitsSEOMetaID(companyID) {
|
||||
return true
|
||||
}
|
||||
if billing.IsA1CohortCompany(legacyID, "") {
|
||||
return true
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(name), "Platform Demo") {
|
||||
return true
|
||||
}
|
||||
return hasA1SectionPrompts
|
||||
}
|
||||
|
||||
// CompanyOmitsSEOMeta is true when catalog/V1 process must omit meta_title /
|
||||
// meta_description for this company (same rules as CompanyOmitsSEOMetaLookup).
|
||||
func CompanyOmitsSEOMeta(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) bool {
|
||||
if CompanyOmitsSEOMetaID(companyID) {
|
||||
return true
|
||||
}
|
||||
if pool == nil {
|
||||
return false
|
||||
}
|
||||
var legacy, name string
|
||||
err := p.Pool.QueryRow(ctx, `
|
||||
err := pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(legacy_company_id, ''), COALESCE(name, '')
|
||||
FROM companies
|
||||
WHERE id = $1`, companyID).Scan(&legacy, &name)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if billing.IsA1CohortCompany(legacy, "") {
|
||||
return true
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(name), "Platform Demo") {
|
||||
if CompanyOmitsSEOMetaLookup(companyID, legacy, name, false) {
|
||||
return true
|
||||
}
|
||||
var hasA1Prompts bool
|
||||
err = p.Pool.QueryRow(ctx, `
|
||||
err = pool.QueryRow(ctx, `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM categories
|
||||
WHERE company_id = $1
|
||||
AND prompt ILIKE '%--- Title ---%'
|
||||
AND prompt ILIKE '%--- Description ---%'
|
||||
AND prompt ILIKE '%--- Meta ---%'
|
||||
AND prompt ILIKE '%--- Title ---%'
|
||||
AND prompt ILIKE '%--- Description ---%'
|
||||
AND prompt ILIKE '%--- Meta ---%'
|
||||
LIMIT 1
|
||||
)`, companyID).Scan(&hasA1Prompts)
|
||||
if err != nil {
|
||||
@@ -521,6 +562,34 @@ func companyOmitsSEOMeta(ctx context.Context, p *Pipeline, companyID uuid.UUID)
|
||||
return hasA1Prompts
|
||||
}
|
||||
|
||||
// StripV1SEOMeta removes meta_title / meta_description keys (omit, not null).
|
||||
func StripV1SEOMeta(items []V1ProcessJobItem) []V1ProcessJobItem {
|
||||
for _, item := range items {
|
||||
if item == nil {
|
||||
continue
|
||||
}
|
||||
delete(item, "meta_title")
|
||||
delete(item, "meta_description")
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
// ApplyV1SEOMetaPolicy omits SEO meta on A1/Demo process items by companies.id.
|
||||
func ApplyV1SEOMetaPolicy(companyID uuid.UUID, items []V1ProcessJobItem) []V1ProcessJobItem {
|
||||
if !CompanyOmitsSEOMetaID(companyID) {
|
||||
return items
|
||||
}
|
||||
return StripV1SEOMeta(items)
|
||||
}
|
||||
|
||||
// companyOmitsSEOMeta is the pipeline wrapper around CompanyOmitsSEOMeta.
|
||||
func companyOmitsSEOMeta(ctx context.Context, p *Pipeline, companyID uuid.UUID) bool {
|
||||
if p == nil {
|
||||
return CompanyOmitsSEOMeta(ctx, nil, companyID)
|
||||
}
|
||||
return CompanyOmitsSEOMeta(ctx, p.Pool, companyID)
|
||||
}
|
||||
|
||||
func derefStringPtr(s *string) string {
|
||||
if s == nil {
|
||||
return ""
|
||||
|
||||
@@ -190,6 +190,37 @@ func TestApplyV1ProcessItemIDsDualMode(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestV1ProcessJobItemMarshalJSONOmitsInternals(t *testing.T) {
|
||||
item := V1ProcessJobItem{
|
||||
"ean": "1",
|
||||
"status": "processed",
|
||||
"name": "T",
|
||||
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
|
||||
"ai_enhance": true,
|
||||
"finish_reason": "stop",
|
||||
"enhance_input_hash": "deadbeef",
|
||||
"field_sources": map[string]any{"enhance_input_hash": "deadbeef"},
|
||||
"prompt_tokens": 9,
|
||||
"worker": "river",
|
||||
}
|
||||
raw, err := json.Marshal(item)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := string(raw)
|
||||
for _, junk := range []string{
|
||||
`"id"`, `"ai_enhance"`, `"finish_reason"`, `"enhance_input_hash"`,
|
||||
`"field_sources"`, `"prompt_tokens"`, `"worker"`, "deadbeef",
|
||||
} {
|
||||
if strings.Contains(s, junk) {
|
||||
t.Fatalf("internal %q leaked in JSON: %s", junk, s)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(s, `"ean":"1"`) || !strings.Contains(s, `"name":"T"`) {
|
||||
t.Fatalf("public fields missing: %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatJobStatusResponseAddsItems(t *testing.T) {
|
||||
jobID := mustParseTestUUID(t, "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
|
||||
job := Job{ID: jobID, Status: "completed", TotalProducts: 1}
|
||||
@@ -218,3 +249,33 @@ func mustParseTestUUID(t *testing.T, s string) uuid.UUID {
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func TestCompanyOmitsSEOMetaLookup(t *testing.T) {
|
||||
t.Parallel()
|
||||
a1 := mustParseTestUUID(t, "604f23a8-b66e-4b21-8b45-0d72b68f4790")
|
||||
other := mustParseTestUUID(t, "11111111-1111-1111-1111-111111111111")
|
||||
cases := []struct {
|
||||
name string
|
||||
id uuid.UUID
|
||||
legacy string
|
||||
coName string
|
||||
prompts bool
|
||||
wantOmit bool
|
||||
}{
|
||||
{name: "ordinary", id: other, coName: "Acme", wantOmit: false},
|
||||
{name: "a1_uuid", id: a1, wantOmit: true},
|
||||
{name: "demo_by_name", id: other, coName: "Platform Demo", wantOmit: true},
|
||||
{name: "demo_by_name_case", id: other, coName: " platform DEMO ", wantOmit: true},
|
||||
{name: "a1_prompt_markers", id: other, coName: "Acme", prompts: true, wantOmit: true},
|
||||
{name: "a1_legacy_id", id: other, legacy: "97e1a309-3d23-4aa2-b518-8e8d7afdfec7", wantOmit: true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := CompanyOmitsSEOMetaLookup(tc.id, tc.legacy, tc.coName, tc.prompts)
|
||||
if got != tc.wantOmit {
|
||||
t.Fatalf("got %v want %v", got, tc.wantOmit)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
)
|
||||
|
||||
// V1ProcessItemScorecard scores one COMPLETED legacy process item against
|
||||
// OpenAPI LegacyProcessItem + A1 poll expectations.
|
||||
// OpenAPI ProcessItem + A1 poll expectations.
|
||||
type V1ProcessItemScorecard struct {
|
||||
EAN string `json:"ean"`
|
||||
Status string `json:"status"`
|
||||
@@ -184,7 +184,7 @@ func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemO
|
||||
return sc
|
||||
}
|
||||
|
||||
// EnforceV1ProcessCompletedItem fills LegacyProcessItem projection gaps for a successful item:
|
||||
// EnforceV1ProcessCompletedItem fills ProcessItem projection gaps for a successful item:
|
||||
// nonempty description when title exists (formula HTML preserved), readable title spacing,
|
||||
// category display name as primary, optional SEO meta (unless omitSEOMeta), clean attrs.
|
||||
// allowed is the company attribute_key set (canonicalized). When nil, only
|
||||
@@ -209,6 +209,7 @@ func EnforceV1ProcessCompletedItemOpts(item V1ProcessJobItem, opts EnforceV1Opts
|
||||
if item == nil {
|
||||
return item
|
||||
}
|
||||
stripV1ProcessItemInternalKeys(item)
|
||||
status, _ := item["status"].(string)
|
||||
if status != "processed" {
|
||||
return item
|
||||
@@ -230,6 +231,9 @@ func EnforceV1ProcessCompletedItemOpts(item V1ProcessJobItem, opts EnforceV1Opts
|
||||
if title == "" {
|
||||
title = ensureReadableTitleSpacing(stringFromItem(item, "title"))
|
||||
}
|
||||
if isPromptLabelTitle(title) {
|
||||
title = preferredProductTitle(stringFromItem(item, "ean"), title)
|
||||
}
|
||||
if title != "" {
|
||||
item["name"] = title
|
||||
delete(item, "title")
|
||||
@@ -237,9 +241,6 @@ func EnforceV1ProcessCompletedItemOpts(item V1ProcessJobItem, opts EnforceV1Opts
|
||||
item["name"] = nil
|
||||
delete(item, "title")
|
||||
}
|
||||
delete(item, "id")
|
||||
delete(item, "processed_product_id")
|
||||
delete(item, "raw_product_id")
|
||||
|
||||
catID, catName := projectV1CategoryFields(item)
|
||||
catLabel := catName
|
||||
@@ -248,6 +249,9 @@ func EnforceV1ProcessCompletedItemOpts(item V1ProcessJobItem, opts EnforceV1Opts
|
||||
}
|
||||
|
||||
desc, _ := descriptionFromItem(item)
|
||||
if isPromptLeakageTitle(desc) || isPromptLabelTitle(desc) {
|
||||
desc = ""
|
||||
}
|
||||
needsDesc := title != "" && (desc == "" ||
|
||||
isWeakPriorEnhanceDescription(desc, title) ||
|
||||
descriptionEchoesTitle(desc, title) ||
|
||||
|
||||
@@ -4,6 +4,9 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestScoreV1ProcessCompletedItem_processedStructure(t *testing.T) {
|
||||
@@ -149,11 +152,25 @@ func TestEnforceV1ProcessCompletedItem_fillsDescriptionMetaSanitizes(t *testing.
|
||||
|
||||
func TestEnforceV1ProcessCompletedItem_skipsTerminalStatuses(t *testing.T) {
|
||||
t.Parallel()
|
||||
item := V1ProcessJobItem{"ean": "1", "status": "not_found", "error": "missing"}
|
||||
item := V1ProcessJobItem{
|
||||
"ean": "1",
|
||||
"status": "not_found",
|
||||
"error": "missing",
|
||||
"ai_enhance": true,
|
||||
"finish_reason": "stop",
|
||||
"field_sources": map[string]any{"enhance_input_hash": "abc"},
|
||||
"prompt_tokens": 12,
|
||||
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
|
||||
}
|
||||
out := EnforceV1ProcessCompletedItem(item, "", nil)
|
||||
if _, ok := out["title"]; ok {
|
||||
t.Fatalf("should not invent fields for not_found: %v", out)
|
||||
}
|
||||
for _, junk := range []string{"id", "ai_enhance", "finish_reason", "field_sources", "prompt_tokens"} {
|
||||
if _, ok := out[junk]; ok {
|
||||
t.Fatalf("%s must be stripped from public items: %v", junk, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnforceV1ProcessCompletedItem_categoryFromCallerPreserved(t *testing.T) {
|
||||
@@ -328,6 +345,119 @@ func TestEnforceV1ProcessCompletedItem_refreshesPromptLeakageMeta(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompanyOmitsSEOMetaID(t *testing.T) {
|
||||
t.Parallel()
|
||||
a1 := uuid.MustParse("604f23a8-b66e-4b21-8b45-0d72b68f4790")
|
||||
demo := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
|
||||
legacy := uuid.MustParse(billing.A1LegacyCompanyID)
|
||||
other := uuid.MustParse("11111111-1111-1111-1111-111111111111")
|
||||
if !CompanyOmitsSEOMetaID(a1) || !CompanyOmitsSEOMetaID(demo) || !CompanyOmitsSEOMetaID(legacy) {
|
||||
t.Fatal("A1, Demo, and legacy A1 ids must omit SEO meta")
|
||||
}
|
||||
if CompanyOmitsSEOMetaID(other) || CompanyOmitsSEOMetaID(uuid.Nil) {
|
||||
t.Fatal("unrelated company ids must keep SEO meta")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyV1SEOMetaPolicy_omitsOnlyA1(t *testing.T) {
|
||||
t.Parallel()
|
||||
item := V1ProcessJobItem{
|
||||
"ean": "1",
|
||||
"status": "processed",
|
||||
"meta_title": "T",
|
||||
"meta_description": "D",
|
||||
}
|
||||
a1 := uuid.MustParse("604f23a8-b66e-4b21-8b45-0d72b68f4790")
|
||||
out := ApplyV1SEOMetaPolicy(a1, []V1ProcessJobItem{item})
|
||||
if _, ok := out[0]["meta_title"]; ok {
|
||||
t.Fatal("A1 must omit meta_title")
|
||||
}
|
||||
if _, ok := out[0]["meta_description"]; ok {
|
||||
t.Fatal("A1 must omit meta_description")
|
||||
}
|
||||
keep := V1ProcessJobItem{
|
||||
"ean": "1",
|
||||
"status": "processed",
|
||||
"meta_title": "T",
|
||||
"meta_description": "D",
|
||||
}
|
||||
other := uuid.MustParse("11111111-1111-1111-1111-111111111111")
|
||||
kept := ApplyV1SEOMetaPolicy(other, []V1ProcessJobItem{keep})
|
||||
if kept[0]["meta_title"] != "T" || kept[0]["meta_description"] != "D" {
|
||||
t.Fatalf("non-A1 must keep SEO meta: %v", kept[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnforceV1ProcessCompletedItemOpts_omitSEOMeta(t *testing.T) {
|
||||
t.Parallel()
|
||||
item := V1ProcessJobItem{
|
||||
"ean": "1",
|
||||
"status": "processed",
|
||||
"title": "Sony Headphones",
|
||||
"description": "Sony Headphones deliver clear audio with noise cancelling for daily commuting.",
|
||||
"meta_title": "Sony | Headphones",
|
||||
"meta_description": "Sony Headphones deliver clear audio.",
|
||||
"category": "48",
|
||||
"category_name": "Slušalke",
|
||||
"attributes": map[string]any{"brand": "Sony"},
|
||||
"eprel": nil,
|
||||
}
|
||||
out := EnforceV1ProcessCompletedItemOpts(item, EnforceV1Opts{Language: "sl", OmitSEOMeta: true})
|
||||
if _, ok := out["meta_title"]; ok {
|
||||
t.Fatalf("meta_title should be omitted: %v", out["meta_title"])
|
||||
}
|
||||
if _, ok := out["meta_description"]; ok {
|
||||
t.Fatalf("meta_description should be omitted: %v", out["meta_description"])
|
||||
}
|
||||
sc := ScoreV1ProcessCompletedItem(out, ScoreV1ProcessItemOptions{MappedCategory: "48", OmitSEOMeta: true})
|
||||
if !sc.OK {
|
||||
t.Fatalf("omit meta should still score OK, flags=%v", sc.FailFlags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnforceV1ProcessCompletedItemOpts_A1PollScrubsPromptDump(t *testing.T) {
|
||||
t.Parallel()
|
||||
item := V1ProcessJobItem{
|
||||
"ean": "8606019604493",
|
||||
"status": "processed",
|
||||
"name": "--- Title ---\nReply with ONLY JSON\nSchema: {\"name\":\"string\"}",
|
||||
"description": "--- Description ---\nReply with ONLY JSON (no markdown)\n" +
|
||||
"Title formula: Brand + Model. Follow any constraints that follow.\n" +
|
||||
"Schema: {\"description\":\"string\"}",
|
||||
"meta_title": "Sony | short retail title; follow any Title formula constr…",
|
||||
"meta_description": "prefer 1-3 factual paragraphs as ONE string",
|
||||
"category": "50",
|
||||
"category_name": "Cookers",
|
||||
"attributes": map[string]any{"brand": "Vox", "product_model": "EHT6020WG"},
|
||||
}
|
||||
out := EnforceV1ProcessCompletedItemOpts(item, EnforceV1Opts{Language: "en", OmitSEOMeta: true})
|
||||
name := fmt.Sprint(out["name"])
|
||||
desc := fmt.Sprint(out["description"])
|
||||
for _, phrase := range []string{
|
||||
"--- Title ---",
|
||||
"--- Description ---",
|
||||
"Reply with ONLY JSON",
|
||||
"Title formula",
|
||||
"Schema:",
|
||||
} {
|
||||
if strings.Contains(name, phrase) || strings.Contains(desc, phrase) {
|
||||
t.Fatalf("A1 poll must not return prompt dump %q: name=%q desc=%q", phrase, name, desc)
|
||||
}
|
||||
}
|
||||
if isPromptLeakageTitle(name) || isPromptLeakageTitle(desc) {
|
||||
t.Fatalf("A1 poll still looks like prompt leakage: name=%q desc=%q", name, desc)
|
||||
}
|
||||
if desc == "" || desc == "<nil>" || out["description"] == nil {
|
||||
t.Fatalf("A1 poll should replace leaked description, got %v", out["description"])
|
||||
}
|
||||
if _, ok := out["meta_title"]; ok {
|
||||
t.Fatal("A1 poll must omit meta_title")
|
||||
}
|
||||
if _, ok := out["meta_description"]; ok {
|
||||
t.Fatal("A1 poll must omit meta_description")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnforceV1ProcessCompletedItem_preservesFormulaHTML(t *testing.T) {
|
||||
t.Parallel()
|
||||
htmlDesc := `<h1>Anker Soundcore Space One Pro</h1><p>Zložljive ANC slušalke z bogatim zvokom.</p><ul><li>Bluetooth 5.3</li></ul>`
|
||||
|
||||
Reference in New Issue
Block a user