package processing import ( "context" "encoding/json" "fmt" "html" "regexp" "strings" "github.com/descrybe/descrybe-v2/apps/api/internal/catalog" "github.com/descrybe/descrybe-v2/apps/api/internal/eprel" "github.com/google/uuid" ) var v1PartialSteps = []string{"category", "title", "description", "attributes"} const v1MetaDescriptionMaxChars = 155 var ( v1BreakTagRe = regexp.MustCompile(`(?i)`) v1BlockEndRe = regexp.MustCompile(`(?i)`) ) // ParseV1ProcessingType mirrors legacy parseV1ProcessingTypeFromBody. // Accepts string ("full" / step), JSON array of steps, or nil (defaults to full). func ParseV1ProcessingType(raw any) (storageValue string, responseValue any, err error) { if raw == nil { return "full", "full", nil } switch v := raw.(type) { case string: trimmed := strings.TrimSpace(v) if trimmed == "" { return "full", "full", nil } if strings.Contains(trimmed, ",") { return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", ")) } normalized := strings.ToLower(trimmed) if normalized == "full" { return "full", "full", nil } if normalized == "both" || normalized == "search" { return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", ")) } step := normalizeV1Step(normalized) if step != "" { return step, step, nil } // Dual-mode: accept v2 dashboard types (normalize_only, enhance_only, …). if isV2ProcessingType(normalized) { return normalized, normalized, nil } return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", ")) case []any: if len(v) == 0 { return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", ")) } steps := make([]string, 0, len(v)) seen := map[string]struct{}{} for _, entry := range v { step := normalizeV1Step(fmt.Sprint(entry)) if step == "" { return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", ")) } if _, ok := seen[step]; ok { continue } seen[step] = struct{}{} steps = append(steps, step) } if len(steps) == 1 { return steps[0], steps[0], nil } b, _ := json.Marshal(steps) return string(b), steps, nil default: return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", ")) } } func isV2ProcessingType(normalized string) bool { switch normalized { case "normalize_only", "enhance", "enhance_only", "enhance-only", "attributes_only", "specs", "specifications", "eprel", "eprel_only", "categorize", "categorize_only", "categorize_enhance": return true default: return false } } func normalizeV1Step(raw string) string { token := strings.ToLower(strings.TrimSpace(raw)) if token == "name" { token = "title" } for _, s := range v1PartialSteps { if s == token { return s } } return "" } // ProcessingTypeForAPIResponse echoes the stored job type as string or []string. func ProcessingTypeForAPIResponse(stored string) any { normalized := strings.ToLower(strings.TrimSpace(stored)) if normalized == "" || normalized == "full" { return "full" } if normalized == "both" { return "both" } if strings.HasPrefix(normalized, "[") { var parsed []any if err := json.Unmarshal([]byte(stored), &parsed); err == nil { out := make([]string, 0, len(parsed)) for _, e := range parsed { if step := normalizeV1Step(fmt.Sprint(e)); step != "" { out = append(out, step) } else { out = append(out, strings.ToLower(strings.TrimSpace(fmt.Sprint(e)))) } } return out } } if step := normalizeV1Step(normalized); step != "" { return step } return normalized } // MapJobStatusForV1 uppercases pipeline statuses toward the legacy public API. func MapJobStatusForV1(status string) string { switch strings.ToLower(strings.TrimSpace(status)) { case "pending": return "PENDING" case "running", "processing", "queued": return "PROCESSING" case "completed", "success", "done", "finished", "processed": return "COMPLETED" case "failed", "error": return "FAILED" case "cancelled", "canceled": return "CANCELLED" default: return strings.ToUpper(strings.TrimSpace(status)) } } // JobStatusIncludesProducts reports whether a finished job should expose processed product items. func JobStatusIncludesProducts(status string) bool { return MapJobStatusForV1(status) == "COMPLETED" } // FormatJobStatusResponse returns job JSON, optionally enriched with processed product items. // Additive only: existing Job fields are preserved; items/total_items appear when includeItems. func FormatJobStatusResponse(job Job, items []V1ProcessJobItem, includeItems bool) any { if !includeItems { return job } raw, err := json.Marshal(job) if err != nil { return job } out := map[string]any{} if err := json.Unmarshal(raw, &out); err != nil { return job } if items == nil { items = []V1ProcessJobItem{} } out["items"] = items out["total_items"] = len(items) return out } // MapV1JobItemStatus normalizes processing_job_products.status for legacy poll items. func MapV1JobItemStatus(raw string, hasProcessed bool) string { switch strings.ToLower(strings.TrimSpace(raw)) { case "processed", "completed", "success", "done": return "processed" case "failed", "error": return "failed" case "cancelled", "canceled", "skipped": return "cancelled" case "processing", "running": return "processing" case "pending", "queued": return "pending" default: if hasProcessed { return "processed" } return "not_found" } } // V1ProcessJobItem is one projected product in a legacy GET /products/process/{id} response. type V1ProcessJobItem map[string]any // LoadV1ProcessJobItems loads and projects job products for the legacy GET status response. func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID uuid.UUID, processingType string) ([]V1ProcessJobItem, error) { if p == nil || p.Pool == nil { return nil, fmt.Errorf("pipeline not configured") } allowedAttrs := loadCompanyAttributeKeySet(ctx, p, companyID) categoryNames := loadCompanyCategoryNameMap(ctx, p, companyID) rows, err := p.Pool.Query(ctx, ` SELECT COALESCE(r.gtin, p.product_id, '') AS ean, p.category, c.name AS category_name, p.processed_name, p.meta_title, p.meta_description, COALESCE(p.processed_description, p.description) AS description, p.processed_attributes, r.mapped_data, r.raw_data, pjp.status AS item_status, pjp.error AS item_error, p.id AS processed_id, pjp.raw_product_id AS raw_product_id FROM processing_job_products pjp LEFT JOIN raw_products r ON r.id = pjp.raw_product_id LEFT JOIN processed_products p ON p.raw_product_id = pjp.raw_product_id AND p.company_id = $2 LEFT JOIN categories c ON c.unique_id = p.category AND c.company_id = $2 WHERE pjp.job_id = $1 ORDER BY pjp.created_at ASC, pjp.id ASC`, jobID, companyID) if err != nil { return nil, err } defer rows.Close() full := make([]V1ProcessJobItem, 0) for rows.Next() { var ( ean, itemStatus string category, categoryName, title, metaTitle, metaDesc, descTxt *string attrsJSON, mappedJSON, rawJSON []byte itemError *string processedID *uuid.UUID rawProductID *uuid.UUID ) if err := rows.Scan( &ean, &category, &categoryName, &title, &metaTitle, &metaDesc, &descTxt, &attrsJSON, &mappedJSON, &rawJSON, &itemStatus, &itemError, &processedID, &rawProductID, ); err != nil { return nil, err } if processedID == nil { st := MapV1JobItemStatus(itemStatus, false) if st == "processed" || st == "processing" || st == "pending" { st = "not_found" } item := V1ProcessJobItem{ "ean": ean, "status": st, "error": "Product data not available", } if itemError != nil && *itemError != "" { item["error"] = *itemError } applyV1ProcessItemIDs(item, nil, rawProductID) full = append(full, item) continue } attrs := map[string]any{} if len(attrsJSON) > 0 { var raw any if err := json.Unmarshal(attrsJSON, &raw); err == nil { switch t := raw.(type) { case map[string]any: attrs = t case []any: for _, entry := range t { if m, ok := entry.(map[string]any); ok { if k, ok := m["key"].(string); ok && k != "" { attrs[k] = m } } } } } } var mapped, rawData map[string]any _ = json.Unmarshal(mappedJSON, &mapped) _ = json.Unmarshal(rawJSON, &rawData) main, more := catalog.ExtractProductImages(mapped, rawData) // Preserve formula HTML from processed_description; meta synthesis strips tags. descOut := "" if descTxt != nil { descOut = v1PreserveDescription(*descTxt) } eprelVal := extractEPRELFromAttrs(attrs) attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowedAttrs) titleStr := derefStringPtr(title) // Prefer full product name from mapped/raw when processed title is brand-only // (e.g. LLM returned "ANKER" while feed has "Anker Soundcore Space One Pro"). titleStr = preferredProductTitle(ean, titleStr, stringFromAny(mapped["name"]), stringFromAny(mapped["title"]), stringFromAny(rawData["name"]), stringFromAny(rawData["title"]), ) catStr := derefStringPtr(category) catNameStr := derefStringPtr(categoryName) // Projection gap: when processed.category is empty but mapped/raw carries a // unique_id, surface it (and resolve category_name from company taxonomy). if catStr == "" { catStr = categoryUniqueIDFromMaps(mapped, rawData) } // Coerce display names → taxonomy unique_id (same as processOne / d161b28). if catStr != "" && len(categoryNames) > 0 { valid := make(map[string]struct{}, len(categoryNames)) for uid := range categoryNames { valid[uid] = struct{}{} } if resolved := resolveCompanyCategoryUniqueID(catStr, categoryNames, valid); resolved != "" { catStr = resolved } } if catNameStr == "" && catStr != "" { if name, ok := categoryNames[catStr]; ok && name != "" { catNameStr = name } else if nested, ok := mapped["category"].(map[string]any); ok { // Only accept nested display names that resolve in company taxonomy — // never product title / title-formula leftovers from feed objects. for _, k := range []string{"name", "category_name"} { s := strings.TrimSpace(stringFromAny(nested[k])) if s == "" || isUnusableCategoryValue(s, titleStr) { continue } uid := resolveCompanyCategoryUniqueID(s, categoryNames, nil) if uid == "" { continue } if resolvedName := strings.TrimSpace(categoryNames[uid]); resolvedName != "" { catNameStr = resolvedName } else { catNameStr = s } break } } } metaTitleOut := nullIfEmptyPtr(metaTitle) metaDescOut := nullIfEmptyPtr(metaDesc) if s, ok := metaTitleOut.(string); ok && isPoisonedMetaTitle(s) { metaTitleOut = nil // Poisoned title refresh: also drop empty/weak/leakage stub meta_description. if ds, ok := metaDescOut.(string); ok && (ds == "" || isWeakPriorEnhanceDescription(ds, titleStr) || isPromptLeakageTitle(ds)) { metaDescOut = nil } } if ds, ok := metaDescOut.(string); ok && isPromptLeakageTitle(ds) { metaDescOut = nil } catLabel := catNameStr if catLabel == "" { catLabel = catStr } if (metaTitleOut == nil || metaDescOut == nil) && (titleStr != "" || descOut != "" || catLabel != "") { synthTitle, synthDesc := fillMetaFromResult(StepResult{ Name: titleStr, ProcessedName: titleStr, Category: catStr, CategoryName: catNameStr, Description: descOut, ProcessedDescription: descOut, Attributes: attrs, ProcessedAttributes: attrs, }) if metaTitleOut == nil { if synthTitle != "" { metaTitleOut = synthTitle } else { metaTitleOut = nullIfEmptyPtr(title) } } if metaDescOut == nil { if synthDesc != "" { metaDescOut = synthDesc } else if descOut != "" { metaDescOut = truncateMetaDescription(v1PlainDescription(descOut), v1MetaDescriptionMaxChars) } } } else if metaTitleOut == nil { metaTitleOut = nullIfEmptyPtr(title) } var description any if descOut != "" { description = descOut } else { description = nil } var catOut, catNameOut any if catStr != "" { catOut = catStr } if catNameStr != "" { catNameOut = catNameStr } var titleOut any if titleStr != "" { titleOut = titleStr } item := V1ProcessJobItem{ "ean": ean, "status": MapV1JobItemStatus(itemStatus, true), "category": catOut, "category_name": catNameOut, "title": titleOut, "name": titleOut, "meta_title": metaTitleOut, "meta_description": metaDescOut, "description": description, "attributes": nil, "main_image": nil, "more_images": nil, "eprel": eprelVal, } applyV1ProcessItemIDs(item, processedID, rawProductID) if itemError != nil && *itemError != "" { item["error"] = *itemError } if len(attrs) > 0 { item["attributes"] = attrs } if main != "" { item["main_image"] = main } if len(more) > 0 { item["more_images"] = more } item = EnforceV1ProcessCompletedItem(item, "", allowedAttrs) full = append(full, item) } if err := rows.Err(); err != nil { return nil, err } return ProjectV1ProcessJobItems(processingType, full), nil } func nullIfEmptyPtr(s *string) any { if s == nil || strings.TrimSpace(*s) == "" { return nil } return *s } func derefStringPtr(s *string) string { if s == nil { return "" } return strings.TrimSpace(*s) } // loadCompanyAttributeKeySet returns canonicalized attribute_key values for the company. // On query failure returns an empty (non-nil) set so FilterAttributesByAllowed still // restricts to coreCharacteristicAttrKeys only. func loadCompanyAttributeKeySet(ctx context.Context, p *Pipeline, companyID uuid.UUID) map[string]struct{} { out := map[string]struct{}{} if p == nil || p.Pool == nil { return out } rows, err := p.Pool.Query(ctx, ` SELECT attribute_key FROM attributes WHERE company_id = $1 AND COALESCE(attribute_key, '') <> ''`, companyID) if err != nil { return out } defer rows.Close() for rows.Next() { var key string if err := rows.Scan(&key); err != nil { continue } canon := canonicalizeAttrKey(key) if canon == "" { continue } out[canon] = struct{}{} out[strings.ToLower(strings.TrimSpace(key))] = struct{}{} } return out } // loadCompanyCategoryNameMap returns unique_id → display name for the company. func loadCompanyCategoryNameMap(ctx context.Context, p *Pipeline, companyID uuid.UUID) map[string]string { out := map[string]string{} if p == nil || p.Pool == nil { return out } rows, err := p.Pool.Query(ctx, ` SELECT unique_id, name FROM categories WHERE company_id = $1 AND COALESCE(unique_id, '') <> ''`, companyID) if err != nil { return out } defer rows.Close() for rows.Next() { var uid, name string if err := rows.Scan(&uid, &name); err != nil { continue } uid = strings.TrimSpace(uid) name = strings.TrimSpace(name) if uid == "" || name == "" { continue } out[uid] = name } return out } // v1PreserveDescription unwraps legacy JSON-encoded description payloads and // normalizes entities, but keeps formula HTML tags (h1/h2/p/ul/…) for the V1 // process poll and DB-facing normalize. Prefer this over v1PlainDescription when // enhance / category description_template markup must reach API clients. func v1PreserveDescription(s string) string { s = strings.TrimSpace(s) if s == "" { return "" } s = unwrapLegacyDescriptionStored(s) s = strings.TrimSpace(s) if s == "" { return "" } s = html.UnescapeString(s) s = strings.ReplaceAll(s, "\u00a0", " ") s = strings.ReplaceAll(s, `\u00a0`, " ") return strings.TrimSpace(s) } // v1PlainDescription normalizes legacy stored descriptions into a single plain-text // string (meta/SEO, feed normalize). Handles JSON array/string encodings and // strips HTML/
/entity markup. Do not use for V1 item.description when formula // HTML must be preserved — use v1PreserveDescription / DescriptionFromAny. func v1PlainDescription(s string) string { s = strings.TrimSpace(s) if s == "" { return "" } s = unwrapLegacyDescriptionStored(s) s = strings.TrimSpace(s) if s == "" { return "" } s = html.UnescapeString(s) s = strings.ReplaceAll(s, "\u00a0", " ") s = strings.ReplaceAll(s, `\u00a0`, " ") s = v1BreakTagRe.ReplaceAllString(s, "\n") s = v1BlockEndRe.ReplaceAllString(s, "\n") s = specsHTMLTagRe.ReplaceAllString(s, " ") s = html.UnescapeString(s) lines := strings.Split(s, "\n") kept := make([]string, 0, len(lines)) for _, line := range lines { line = strings.Join(strings.Fields(line), " ") line = strings.ReplaceAll(line, " .", ".") line = strings.ReplaceAll(line, " ,", ",") line = strings.ReplaceAll(line, " ;", ";") line = strings.ReplaceAll(line, " :", ":") if line != "" { kept = append(kept, line) } } return strings.TrimSpace(strings.Join(kept, "\n")) } const maxLegacyDescriptionUnwrapDepth = 4 // unwrapLegacyDescriptionStored flattens JSON-encoded legacy description payloads // (one-element arrays, multi-part arrays, nested arrays, JSON-quoted strings). // Non-JSON text is returned unchanged. func unwrapLegacyDescriptionStored(s string) string { return unwrapLegacyDescriptionStoredN(s, 0) } func unwrapLegacyDescriptionStoredN(s string, depth int) string { s = strings.TrimSpace(s) if s == "" || depth > maxLegacyDescriptionUnwrapDepth { return s } if !looksLikeLegacyDescriptionJSON(s) { return s } var raw any if err := json.Unmarshal([]byte(s), &raw); err != nil { return s } flat := flattenLegacyDescriptionAny(raw, depth) if flat == "" { if _, ok := raw.([]any); ok { return "" } return s } return flat } func looksLikeLegacyDescriptionJSON(s string) bool { if len(s) >= 2 && s[0] == '[' && s[len(s)-1] == ']' { return true } return len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' } func flattenLegacyDescriptionAny(v any, depth int) string { switch t := v.(type) { case string: return unwrapLegacyDescriptionStoredN(t, depth+1) case []any: parts := make([]string, 0, len(t)) for _, el := range t { if p := flattenLegacyDescriptionAny(el, depth+1); p != "" { parts = append(parts, p) } } return strings.Join(parts, "\n") case float64: return strings.TrimSpace(fmt.Sprint(t)) case bool: return fmt.Sprint(t) default: return "" } } func extractEPRELFromAttrs(attrs map[string]any) any { return eprel.ExtractFromAttrs(attrs) } // ProjectV1ProcessJobItems applies legacy partial-type field projection. func ProjectV1ProcessJobItems(storedType string, items []V1ProcessJobItem) []V1ProcessJobItem { resolved := resolveV1Steps(storedType) if resolved.isFull { return items } out := make([]V1ProcessJobItem, 0, len(items)) for _, item := range items { if _, hasID := item["id"]; !hasID { if status, _ := item["status"].(string); status == "not_found" || status == "failed" || status == "cancelled" { out = append(out, item) continue } } main, _ := item["main_image"].(string) var more []string if m, ok := item["more_images"].([]string); ok { more = m } else if arr, ok := item["more_images"].([]any); ok { for _, e := range arr { if s, ok := e.(string); ok { more = append(more, s) } } } eprel := item["eprel"] ean, _ := item["ean"].(string) projected := withItemMeta(V1ProcessJobItem{"ean": ean}, item) if len(resolved.steps) == 0 { out = append(out, withAlwaysIncluded(projected, main, more, eprel)) continue } for step := range resolved.steps { switch step { case "category": projected["category"] = item["category"] projected["category_name"] = item["category_name"] case "title": projected["title"] = item["title"] projected["name"] = item["name"] if projected["name"] == nil { projected["name"] = item["title"] } projected["meta_title"] = item["meta_title"] case "description": projected["description"] = item["description"] projected["meta_description"] = item["meta_description"] case "attributes": projected["attributes"] = item["attributes"] } } out = append(out, withAlwaysIncluded(projected, main, more, eprel)) } return out } func withItemMeta(dst, src V1ProcessJobItem) V1ProcessJobItem { for _, k := range []string{"status", "error", "id", "processed_product_id", "raw_product_id"} { if v, ok := src[k]; ok { dst[k] = v } } return dst } // applyV1ProcessItemIDs sets legacy id (= processed UUID) plus additive dual-mode aliases. // id is preserved for existing integrators; processed_product_id mirrors it; raw_product_id is raw_products.id. func applyV1ProcessItemIDs(item V1ProcessJobItem, processedID, rawProductID *uuid.UUID) { if processedID != nil { s := processedID.String() item["id"] = s item["processed_product_id"] = s } if rawProductID != nil { item["raw_product_id"] = rawProductID.String() } } func withAlwaysIncluded(item V1ProcessJobItem, main string, more []string, eprel any) V1ProcessJobItem { if main != "" { item["main_image"] = main } else { item["main_image"] = nil } if len(more) > 0 { item["more_images"] = more } else { item["more_images"] = nil } if eprel == nil { item["eprel"] = nil } else { item["eprel"] = eprel } return item } type v1ResolvedSteps struct { isFull bool steps map[string]struct{} } func resolveV1Steps(stored string) v1ResolvedSteps { normalized := strings.ToLower(strings.TrimSpace(stored)) if normalized == "" || normalized == "full" { return v1ResolvedSteps{isFull: true, steps: map[string]struct{}{}} } if normalized == "both" { return v1ResolvedSteps{steps: map[string]struct{}{"title": {}, "description": {}}} } if strings.HasPrefix(normalized, "[") { var parsed []any if err := json.Unmarshal([]byte(stored), &parsed); err == nil { steps := map[string]struct{}{} for _, e := range parsed { if step := normalizeV1Step(fmt.Sprint(e)); step != "" { steps[step] = struct{}{} } } return v1ResolvedSteps{steps: steps} } } if step := normalizeV1Step(normalized); step != "" { return v1ResolvedSteps{steps: map[string]struct{}{step: {}}} } // Unknown stored types (normalize_only, enhance_only, …) → treat as full projection. return v1ResolvedSteps{isFull: true, steps: map[string]struct{}{}} }