package processing import ( "context" "encoding/json" "fmt" "html" "regexp" "strings" "github.com/descrybe/descrybe-v2/apps/api/internal/catalog" "github.com/google/uuid" ) var v1PartialSteps = []string{"category", "title", "description", "attributes"} const v1MetaDescriptionMaxChars = 155 var ( v1BreakTagRe = regexp.MustCompile(`(?i)`) v1BlockEndRe = regexp.MustCompile(`(?i)`) ) // ParseV1ProcessingType mirrors legacy parseV1ProcessingTypeFromBody. // Accepts string ("full" / step), JSON array of steps, or nil (defaults to full). func ParseV1ProcessingType(raw any) (storageValue string, responseValue any, err error) { if raw == nil { return "full", "full", nil } switch v := raw.(type) { case string: trimmed := strings.TrimSpace(v) if trimmed == "" { return "full", "full", nil } if strings.Contains(trimmed, ",") { return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", ")) } normalized := strings.ToLower(trimmed) if normalized == "full" { return "full", "full", nil } if normalized == "both" || normalized == "search" { return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", ")) } step := normalizeV1Step(normalized) if step != "" { return step, step, nil } // Dual-mode: accept v2 dashboard types (normalize_only, enhance_only, …). if isV2ProcessingType(normalized) { return normalized, normalized, nil } return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", ")) case []any: if len(v) == 0 { return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", ")) } steps := make([]string, 0, len(v)) seen := map[string]struct{}{} for _, entry := range v { step := normalizeV1Step(fmt.Sprint(entry)) if step == "" { return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", ")) } if _, ok := seen[step]; ok { continue } seen[step] = struct{}{} steps = append(steps, step) } if len(steps) == 1 { return steps[0], steps[0], nil } b, _ := json.Marshal(steps) return string(b), steps, nil default: return "", nil, fmt.Errorf("Invalid processing_type. Use \"full\", a single step (%s), or an array of steps, e.g. [\"title\",\"attributes\"].", strings.Join(v1PartialSteps, ", ")) } } func isV2ProcessingType(normalized string) bool { switch normalized { case "normalize_only", "enhance", "enhance_only", "enhance-only", "attributes_only", "specs", "specifications", "eprel", "eprel_only", "categorize", "categorize_only", "categorize_enhance": return true default: return false } } func normalizeV1Step(raw string) string { token := strings.ToLower(strings.TrimSpace(raw)) if token == "name" { token = "title" } for _, s := range v1PartialSteps { if s == token { return s } } return "" } // ProcessingTypeForAPIResponse echoes the stored job type as string or []string. func ProcessingTypeForAPIResponse(stored string) any { normalized := strings.ToLower(strings.TrimSpace(stored)) if normalized == "" || normalized == "full" { return "full" } if normalized == "both" { return "both" } if strings.HasPrefix(normalized, "[") { var parsed []any if err := json.Unmarshal([]byte(stored), &parsed); err == nil { out := make([]string, 0, len(parsed)) for _, e := range parsed { if step := normalizeV1Step(fmt.Sprint(e)); step != "" { out = append(out, step) } else { out = append(out, strings.ToLower(strings.TrimSpace(fmt.Sprint(e)))) } } return out } } if step := normalizeV1Step(normalized); step != "" { return step } return normalized } // MapJobStatusForV1 uppercases pipeline statuses toward the legacy public API. func MapJobStatusForV1(status string) string { switch strings.ToLower(strings.TrimSpace(status)) { case "pending": return "PENDING" case "running", "processing", "queued": return "PROCESSING" case "completed", "success", "done", "finished", "processed": return "COMPLETED" case "failed", "error": return "FAILED" case "cancelled", "canceled": return "CANCELLED" default: return strings.ToUpper(strings.TrimSpace(status)) } } // JobStatusIncludesProducts reports whether a finished job should expose processed product items. func JobStatusIncludesProducts(status string) bool { return MapJobStatusForV1(status) == "COMPLETED" } // FormatJobStatusResponse returns job JSON, optionally enriched with processed product items. // Additive only: existing Job fields are preserved; items/total_items appear when includeItems. func FormatJobStatusResponse(job Job, items []V1ProcessJobItem, includeItems bool) any { if !includeItems { return job } raw, err := json.Marshal(job) if err != nil { return job } out := map[string]any{} if err := json.Unmarshal(raw, &out); err != nil { return job } if items == nil { items = []V1ProcessJobItem{} } out["items"] = items out["total_items"] = len(items) return out } // MapV1JobItemStatus normalizes processing_job_products.status for legacy poll items. func MapV1JobItemStatus(raw string, hasProcessed bool) string { switch strings.ToLower(strings.TrimSpace(raw)) { case "processed", "completed", "success", "done": return "processed" case "failed", "error": return "failed" case "cancelled", "canceled", "skipped": return "cancelled" case "processing", "running": return "processing" case "pending", "queued": return "pending" default: if hasProcessed { return "processed" } return "not_found" } } // V1ProcessJobItem is one projected product in a legacy GET /products/process/{id} response. type V1ProcessJobItem map[string]any // LoadV1ProcessJobItems loads and projects job products for the legacy GET status response. func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID uuid.UUID, processingType string) ([]V1ProcessJobItem, error) { if p == nil || p.Pool == nil { return nil, fmt.Errorf("pipeline not configured") } allowedAttrs := loadCompanyAttributeKeySet(ctx, p, companyID) rows, err := p.Pool.Query(ctx, ` SELECT COALESCE(r.gtin, p.product_id, '') AS ean, p.category, c.name AS category_name, p.processed_name, p.meta_title, p.meta_description, COALESCE(p.processed_description, p.description) AS description, p.processed_attributes, r.mapped_data, r.raw_data, pjp.status AS item_status, pjp.error AS item_error, p.id AS processed_id, pjp.raw_product_id AS raw_product_id FROM processing_job_products pjp LEFT JOIN raw_products r ON r.id = pjp.raw_product_id LEFT JOIN processed_products p ON p.raw_product_id = pjp.raw_product_id AND p.company_id = $2 LEFT JOIN categories c ON c.unique_id = p.category AND c.company_id = $2 WHERE pjp.job_id = $1 ORDER BY pjp.created_at ASC, pjp.id ASC`, jobID, companyID) if err != nil { return nil, err } defer rows.Close() full := make([]V1ProcessJobItem, 0) for rows.Next() { var ( ean, itemStatus string category, categoryName, title, metaTitle, metaDesc, descTxt *string attrsJSON, mappedJSON, rawJSON []byte itemError *string processedID *uuid.UUID rawProductID *uuid.UUID ) if err := rows.Scan( &ean, &category, &categoryName, &title, &metaTitle, &metaDesc, &descTxt, &attrsJSON, &mappedJSON, &rawJSON, &itemStatus, &itemError, &processedID, &rawProductID, ); err != nil { return nil, err } if processedID == nil { st := MapV1JobItemStatus(itemStatus, false) if st == "processed" || st == "processing" || st == "pending" { st = "not_found" } item := V1ProcessJobItem{ "ean": ean, "status": st, "error": "Product data not available", } if itemError != nil && *itemError != "" { item["error"] = *itemError } applyV1ProcessItemIDs(item, nil, rawProductID) full = append(full, item) continue } attrs := map[string]any{} if len(attrsJSON) > 0 { var raw any if err := json.Unmarshal(attrsJSON, &raw); err == nil { switch t := raw.(type) { case map[string]any: attrs = t case []any: for _, entry := range t { if m, ok := entry.(map[string]any); ok { if k, ok := m["key"].(string); ok && k != "" { attrs[k] = m } } } } } } var mapped, rawData map[string]any _ = json.Unmarshal(mappedJSON, &mapped) _ = json.Unmarshal(rawJSON, &rawData) main, more := catalog.ExtractProductImages(mapped, rawData) plainDesc := "" if descTxt != nil { plainDesc = v1PlainDescription(*descTxt) } var description any if plainDesc != "" { description = plainDesc } else { description = nil } eprelVal := extractEPRELFromAttrs(attrs) attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowedAttrs) metaTitleOut := nullIfEmptyPtr(metaTitle) if metaTitleOut == nil { metaTitleOut = nullIfEmptyPtr(title) } metaDescOut := nullIfEmptyPtr(metaDesc) if metaDescOut == nil && plainDesc != "" { metaDescOut = truncateRunes(plainDesc, v1MetaDescriptionMaxChars) } item := V1ProcessJobItem{ "ean": ean, "status": MapV1JobItemStatus(itemStatus, true), "category": nullIfEmptyPtr(category), "category_name": nullIfEmptyPtr(categoryName), "title": nullIfEmptyPtr(title), "meta_title": metaTitleOut, "meta_description": metaDescOut, "description": description, "attributes": nil, "main_image": nil, "more_images": nil, "eprel": eprelVal, } applyV1ProcessItemIDs(item, processedID, rawProductID) if itemError != nil && *itemError != "" { item["error"] = *itemError } if len(attrs) > 0 { item["attributes"] = attrs } if main != "" { item["main_image"] = main } if len(more) > 0 { item["more_images"] = more } full = append(full, item) } if err := rows.Err(); err != nil { return nil, err } return ProjectV1ProcessJobItems(processingType, full), nil } func nullIfEmptyPtr(s *string) any { if s == nil || strings.TrimSpace(*s) == "" { return nil } return *s } // loadCompanyAttributeKeySet returns canonicalized attribute_key values for the company. // On query failure returns an empty (non-nil) set so FilterAttributesByAllowed still // restricts to coreCharacteristicAttrKeys only. func loadCompanyAttributeKeySet(ctx context.Context, p *Pipeline, companyID uuid.UUID) map[string]struct{} { out := map[string]struct{}{} if p == nil || p.Pool == nil { return out } rows, err := p.Pool.Query(ctx, ` SELECT attribute_key FROM attributes WHERE company_id = $1 AND COALESCE(attribute_key, '') <> ''`, companyID) if err != nil { return out } defer rows.Close() for rows.Next() { var key string if err := rows.Scan(&key); err != nil { continue } canon := canonicalizeAttrKey(key) if canon == "" { continue } out[canon] = struct{}{} out[strings.ToLower(strings.TrimSpace(key))] = struct{}{} } return out } // v1PlainDescription normalizes feed HTML into a single plain-text string for the // legacy process poll (description is a string, not a one-element array). func v1PlainDescription(s string) string { s = strings.TrimSpace(s) if s == "" { return "" } s = html.UnescapeString(s) s = strings.ReplaceAll(s, "\u00a0", " ") s = v1BreakTagRe.ReplaceAllString(s, "\n") s = v1BlockEndRe.ReplaceAllString(s, "\n") s = specsHTMLTagRe.ReplaceAllString(s, " ") s = html.UnescapeString(s) lines := strings.Split(s, "\n") kept := make([]string, 0, len(lines)) for _, line := range lines { line = strings.Join(strings.Fields(line), " ") if line != "" { kept = append(kept, line) } } return strings.TrimSpace(strings.Join(kept, "\n")) } func extractEPRELFromAttrs(attrs map[string]any) any { if attrs == nil { return nil } if e, ok := attrs["eprel"]; ok && e != nil { return e } out := map[string]any{} for _, k := range []string{"label", "pdf", "energy_class", "energy_scale"} { if v, ok := attrs["eprel_"+k]; ok && v != nil { out[k] = v } } if len(out) == 0 { return nil } return out } // ProjectV1ProcessJobItems applies legacy partial-type field projection. func ProjectV1ProcessJobItems(storedType string, items []V1ProcessJobItem) []V1ProcessJobItem { resolved := resolveV1Steps(storedType) if resolved.isFull { return items } out := make([]V1ProcessJobItem, 0, len(items)) for _, item := range items { if _, hasID := item["id"]; !hasID { if status, _ := item["status"].(string); status == "not_found" || status == "failed" || status == "cancelled" { out = append(out, item) continue } } main, _ := item["main_image"].(string) var more []string if m, ok := item["more_images"].([]string); ok { more = m } else if arr, ok := item["more_images"].([]any); ok { for _, e := range arr { if s, ok := e.(string); ok { more = append(more, s) } } } eprel := item["eprel"] ean, _ := item["ean"].(string) projected := withItemMeta(V1ProcessJobItem{"ean": ean}, item) if len(resolved.steps) == 0 { out = append(out, withAlwaysIncluded(projected, main, more, eprel)) continue } for step := range resolved.steps { switch step { case "category": projected["category"] = item["category"] projected["category_name"] = item["category_name"] case "title": projected["title"] = item["title"] projected["meta_title"] = item["meta_title"] case "description": projected["description"] = item["description"] projected["meta_description"] = item["meta_description"] case "attributes": projected["attributes"] = item["attributes"] } } out = append(out, withAlwaysIncluded(projected, main, more, eprel)) } return out } func withItemMeta(dst, src V1ProcessJobItem) V1ProcessJobItem { for _, k := range []string{"status", "error", "id", "processed_product_id", "raw_product_id"} { if v, ok := src[k]; ok { dst[k] = v } } return dst } // applyV1ProcessItemIDs sets legacy id (= processed UUID) plus additive dual-mode aliases. // id is preserved for existing integrators; processed_product_id mirrors it; raw_product_id is raw_products.id. func applyV1ProcessItemIDs(item V1ProcessJobItem, processedID, rawProductID *uuid.UUID) { if processedID != nil { s := processedID.String() item["id"] = s item["processed_product_id"] = s } if rawProductID != nil { item["raw_product_id"] = rawProductID.String() } } func withAlwaysIncluded(item V1ProcessJobItem, main string, more []string, eprel any) V1ProcessJobItem { if main != "" { item["main_image"] = main } else { item["main_image"] = nil } if len(more) > 0 { item["more_images"] = more } else { item["more_images"] = nil } if eprel == nil { item["eprel"] = nil } else { item["eprel"] = eprel } return item } type v1ResolvedSteps struct { isFull bool steps map[string]struct{} } func resolveV1Steps(stored string) v1ResolvedSteps { normalized := strings.ToLower(strings.TrimSpace(stored)) if normalized == "" || normalized == "full" { return v1ResolvedSteps{isFull: true, steps: map[string]struct{}{}} } if normalized == "both" { return v1ResolvedSteps{steps: map[string]struct{}{"title": {}, "description": {}}} } if strings.HasPrefix(normalized, "[") { var parsed []any if err := json.Unmarshal([]byte(stored), &parsed); err == nil { steps := map[string]struct{}{} for _, e := range parsed { if step := normalizeV1Step(fmt.Sprint(e)); step != "" { steps[step] = struct{}{} } } return v1ResolvedSteps{steps: steps} } } if step := normalizeV1Step(normalized); step != "" { return v1ResolvedSteps{steps: map[string]struct{}{step: {}}} } // Unknown stored types (normalize_only, enhance_only, …) → treat as full projection. return v1ResolvedSteps{isFull: true, steps: map[string]struct{}{}} }