fixes

2026-08-16 16:57:36 +02:00
parent 96f8c1115c
commit 532d439c41
106 changed files with 10147 additions and 494 deletions
@@ -0,0 +1,456 @@
+package processing
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
+)
+
+// V1ProcessItemScorecard scores one COMPLETED legacy process item against
+// OpenAPI LegacyProcessItem + A1 poll expectations.
+type V1ProcessItemScorecard struct {
+	EAN       string   `json:"ean"`
+	Status    string   `json:"status"`
+	OK        bool     `json:"ok"`
+	Score     int      `json:"score"` // 0–100
+	FailFlags []string `json:"fail_flags,omitempty"`
+
+	HasTitle           bool `json:"has_title"`
+	HasName            bool `json:"has_name"`
+	HasDescription     bool `json:"has_description"`
+	DescriptionIsPlain bool `json:"description_is_plain"`
+	HasMetaTitle       bool `json:"has_meta_title"`
+	HasMetaDescription bool `json:"has_meta_description"`
+	HasCategory        bool `json:"has_category"`
+	HasCategoryName    bool `json:"has_category_name"`
+	AttrsClean         bool `json:"attrs_clean"`
+	EPRELOK            bool `json:"eprel_ok"`
+	HasIDs             bool `json:"has_ids"`
+	ImagesOK           bool `json:"images_ok"`
+}
+
+// ScoreV1ProcessItemOptions tunes structure checks for a poll item.
+type ScoreV1ProcessItemOptions struct {
+	// MappedCategory, when non-empty, requires item.category to equal it
+	// (projection must surface mapped unique_id).
+	MappedCategory string
+	// Language is used only for documentation of synthesize paths in tests.
+	Language string
+}
+
+// ScoreV1ProcessCompletedItem scores a successful (or terminal) V1 process item.
+// For status != processed it only checks ean/status shape and returns OK when those exist.
+func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemOptions) V1ProcessItemScorecard {
+	sc := V1ProcessItemScorecard{
+		EAN:    strings.TrimSpace(fmt.Sprint(item["ean"])),
+		Status: strings.TrimSpace(fmt.Sprint(item["status"])),
+	}
+	if sc.EAN == "" || sc.EAN == "<nil>" {
+		sc.FailFlags = append(sc.FailFlags, "missing_ean")
+	}
+	if sc.Status == "" || sc.Status == "<nil>" {
+		sc.FailFlags = append(sc.FailFlags, "missing_status")
+	}
+	if sc.Status != "processed" {
+		sc.OK = len(sc.FailFlags) == 0
+		if sc.OK {
+			sc.Score = 100
+		}
+		return sc
+	}
+
+	title := stringFromItem(item, "title")
+	name := stringFromItem(item, "name")
+	sc.HasTitle = title != ""
+	sc.HasName = name != ""
+	if sc.HasTitle && !sc.HasName {
+		sc.FailFlags = append(sc.FailFlags, "missing_name")
+	}
+	if sc.HasTitle && sc.HasName && title != name {
+		sc.FailFlags = append(sc.FailFlags, "title_name_mismatch")
+	}
+
+	desc, descOK := plainDescriptionFromItem(item)
+	sc.HasDescription = desc != ""
+	sc.DescriptionIsPlain = descOK
+	if sc.HasTitle && !sc.HasDescription {
+		sc.FailFlags = append(sc.FailFlags, "description_empty_with_title")
+	}
+	if !descOK && item["description"] != nil {
+		sc.FailFlags = append(sc.FailFlags, "description_not_plain_string")
+	}
+	if sc.HasDescription && descriptionEchoesTitle(desc, title) {
+		sc.FailFlags = append(sc.FailFlags, "description_echoes_title")
+	}
+	if sc.HasDescription && containsWeakFillerPhrase(desc) {
+		sc.FailFlags = append(sc.FailFlags, "description_weak_filler")
+	}
+
+	sc.HasMetaTitle = stringFromItem(item, "meta_title") != ""
+	sc.HasMetaDescription = stringFromItem(item, "meta_description") != ""
+	if sc.HasTitle && !sc.HasMetaTitle {
+		sc.FailFlags = append(sc.FailFlags, "meta_title_missing")
+	}
+	if sc.HasTitle && !sc.HasMetaDescription {
+		sc.FailFlags = append(sc.FailFlags, "meta_description_missing")
+	}
+	if md := stringFromItem(item, "meta_description"); md != "" && containsWeakFillerPhrase(md) {
+		sc.FailFlags = append(sc.FailFlags, "meta_description_weak_filler")
+	}
+
+	cat := stringFromItem(item, "category")
+	catName := stringFromItem(item, "category_name")
+	sc.HasCategory = cat != ""
+	sc.HasCategoryName = catName != ""
+	mappedCat := strings.TrimSpace(opts.MappedCategory)
+	if mappedCat != "" {
+		if cat == "" {
+			sc.FailFlags = append(sc.FailFlags, "category_missing_though_mapped")
+		} else if cat != mappedCat {
+			sc.FailFlags = append(sc.FailFlags, "category_mismatch_mapped")
+		}
+		if cat != "" && catName == "" {
+			sc.FailFlags = append(sc.FailFlags, "category_name_missing")
+		}
+	}
+
+	attrs, attrsOK := attrsMapFromItem(item)
+	dirty := dirtyV1AttrKeys(attrs)
+	sc.AttrsClean = attrsOK && len(dirty) == 0
+	if !attrsOK {
+		sc.FailFlags = append(sc.FailFlags, "attributes_invalid_shape")
+	}
+	if len(dirty) > 0 {
+		sc.FailFlags = append(sc.FailFlags, "attributes_dirty:"+strings.Join(dirty, ","))
+	}
+
+	sc.EPRELOK = eprelShapeOK(item["eprel"])
+	if !sc.EPRELOK {
+		sc.FailFlags = append(sc.FailFlags, "eprel_invalid_shape")
+	}
+
+	id := stringFromItem(item, "id")
+	ppID := stringFromItem(item, "processed_product_id")
+	rawID := stringFromItem(item, "raw_product_id")
+	sc.HasIDs = id != "" && ppID != "" && rawID != ""
+	if id == "" {
+		sc.FailFlags = append(sc.FailFlags, "missing_id")
+	}
+	if ppID == "" {
+		sc.FailFlags = append(sc.FailFlags, "missing_processed_product_id")
+	}
+	if rawID == "" {
+		sc.FailFlags = append(sc.FailFlags, "missing_raw_product_id")
+	}
+	if id != "" && ppID != "" && id != ppID {
+		sc.FailFlags = append(sc.FailFlags, "id_processed_product_id_mismatch")
+	}
+
+	sc.ImagesOK = imageFieldsOK(item)
+	if !sc.ImagesOK {
+		sc.FailFlags = append(sc.FailFlags, "images_invalid_shape")
+	}
+
+	// Weighted score: structure gates, not content quality beyond filler/echo.
+	checks := []bool{
+		sc.EAN != "",
+		sc.Status == "processed",
+		sc.HasTitle,
+		sc.HasName || !sc.HasTitle,
+		sc.HasDescription || !sc.HasTitle,
+		sc.DescriptionIsPlain || item["description"] == nil,
+		sc.HasMetaTitle || !sc.HasTitle,
+		sc.HasMetaDescription || !sc.HasTitle,
+		mappedCat == "" || sc.HasCategory,
+		mappedCat == "" || sc.HasCategoryName,
+		sc.AttrsClean,
+		sc.EPRELOK,
+		sc.HasIDs,
+		sc.ImagesOK,
+		!containsWeakFillerPhrase(desc),
+		!descriptionEchoesTitle(desc, title),
+	}
+	pass := 0
+	for _, ok := range checks {
+		if ok {
+			pass++
+		}
+	}
+	sc.Score = (pass * 100) / len(checks)
+	sc.OK = len(sc.FailFlags) == 0
+	return sc
+}
+
+// EnforceV1ProcessCompletedItem fills LegacyProcessItem projection gaps for a
+// successful item: plain nonempty description when title exists, meta_*, clean
+// attributes, eprel object|null, and image key shapes. Category must already be
+// set by the caller when mapped provides a unique_id.
+//
+// allowed is the company attribute_key set (canonicalized). When nil, only
+// coreCharacteristicAttrKeys are kept (never leak feed junk like zavora).
+func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allowed map[string]struct{}) V1ProcessJobItem {
+	if item == nil {
+		return item
+	}
+	status, _ := item["status"].(string)
+	if status != "processed" {
+		return item
+	}
+
+	attrs, _ := attrsMapFromItem(item)
+	if allowed == nil {
+		allowed = map[string]struct{}{}
+	}
+	attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowed)
+	if len(attrs) > 0 {
+		item["attributes"] = attrs
+	} else {
+		item["attributes"] = nil
+	}
+
+	title := stringFromItem(item, "title")
+	if title != "" {
+		item["title"] = title
+		item["name"] = title
+	} else {
+		item["title"] = nil
+		item["name"] = nil
+	}
+	cat := stringFromItem(item, "category")
+	catName := stringFromItem(item, "category_name")
+	catLabel := catName
+	if catLabel == "" {
+		catLabel = cat
+	}
+
+	desc, _ := plainDescriptionFromItem(item)
+	// Empty, weak, or title-echo copy must be replaced — never leave description==title.
+	if title != "" && (desc == "" || isWeakPriorEnhanceDescription(desc, title) || descriptionEchoesTitle(desc, title)) {
+		if synth := synthesizeDescriptionFromTitle(title, catLabel, language, attrs); synth != "" {
+			desc = synth
+		}
+	}
+	if desc != "" {
+		item["description"] = desc
+	} else {
+		item["description"] = nil
+	}
+
+	metaTitle := stringFromItem(item, "meta_title")
+	metaDesc := stringFromItem(item, "meta_description")
+	needMetaTitle := metaTitle == "" || isPoisonedMetaTitle(metaTitle)
+	// Empty, weak stub (Ready for retail…), title-echo, or prompt-leakage — refresh.
+	// When meta_title is poisoned, also force refresh of weak/leakage meta_description.
+	needMetaDesc := metaDesc == "" ||
+		isWeakPriorEnhanceDescription(metaDesc, title) ||
+		containsWeakFillerPhrase(metaDesc) ||
+		isPromptLeakageTitle(metaDesc) ||
+		(title != "" && descriptionEchoesTitle(metaDesc, title))
+	if needMetaTitle && isPoisonedMetaTitle(metaTitle) &&
+		(metaDesc == "" || isWeakPriorEnhanceDescription(metaDesc, title) || isPromptLeakageTitle(metaDesc)) {
+		needMetaDesc = true
+	}
+	if title != "" || desc != "" || cat != "" || catName != "" {
+		synthTitle, synthDesc := fillMetaFromResult(StepResult{
+			Name:                 title,
+			ProcessedName:        title,
+			Category:             cat,
+			CategoryName:         catName,
+			Description:          desc,
+			ProcessedDescription: desc,
+			Attributes:           attrs,
+			ProcessedAttributes:  attrs,
+		})
+		if needMetaTitle {
+			if synthTitle != "" {
+				metaTitle = synthTitle
+			} else {
+				metaTitle = title
+			}
+		}
+		if needMetaDesc {
+			if synthDesc != "" {
+				metaDesc = synthDesc
+			} else if desc != "" {
+				metaDesc = truncateMetaDescription(desc, v1MetaDescriptionMaxChars)
+			}
+		}
+	}
+	if metaTitle != "" {
+		item["meta_title"] = metaTitle
+	} else {
+		item["meta_title"] = nil
+	}
+	if metaDesc != "" {
+		item["meta_description"] = metaDesc
+	} else {
+		item["meta_description"] = nil
+	}
+
+	if cat != "" {
+		item["category"] = cat
+	} else {
+		item["category"] = nil
+	}
+	if catName != "" {
+		item["category_name"] = catName
+	} else {
+		item["category_name"] = nil
+	}
+
+	item["eprel"] = normalizeEPRELValue(item["eprel"])
+
+	main := stringFromItem(item, "main_image")
+	if main != "" {
+		item["main_image"] = main
+	} else {
+		item["main_image"] = nil
+	}
+	more := stringSliceFromItem(item, "more_images")
+	if len(more) > 0 {
+		item["more_images"] = more
+	} else {
+		item["more_images"] = nil
+	}
+
+	return item
+}
+
+func stringFromItem(item V1ProcessJobItem, key string) string {
+	if item == nil {
+		return ""
+	}
+	return strings.TrimSpace(stringFromAny(item[key]))
+}
+
+func plainDescriptionFromItem(item V1ProcessJobItem) (string, bool) {
+	if item == nil {
+		return "", true
+	}
+	raw := item["description"]
+	if raw == nil {
+		return "", true
+	}
+	switch raw.(type) {
+	case string:
+		return PlainDescriptionFromAny(raw), true
+	case []any, []string:
+		// Legacy mistake: description as array — convert to plain string.
+		if s := PlainDescriptionFromAny(raw); s != "" {
+			return s, true
+		}
+		return "", false
+	case map[string]any:
+		if s := PlainDescriptionFromAny(raw); s != "" {
+			return s, true
+		}
+		return "", false
+	default:
+		s := strings.TrimSpace(fmt.Sprint(raw))
+		if s == "" || s == "<nil>" {
+			return "", true
+		}
+		return "", false
+	}
+}
+
+func attrsMapFromItem(item V1ProcessJobItem) (map[string]any, bool) {
+	if item == nil {
+		return map[string]any{}, true
+	}
+	raw := item["attributes"]
+	if raw == nil {
+		return map[string]any{}, true
+	}
+	switch t := raw.(type) {
+	case map[string]any:
+		return t, true
+	default:
+		return map[string]any{}, false
+	}
+}
+
+func dirtyV1AttrKeys(attrs map[string]any) []string {
+	if len(attrs) == 0 {
+		return nil
+	}
+	var dirty []string
+	for k := range attrs {
+		key := strings.TrimSpace(k)
+		lk := strings.ToLower(key)
+		if key == "" || isReservedProductKey(key) || isInvalidAttributeKey(key) ||
+			strings.HasPrefix(lk, "eprel") {
+			dirty = append(dirty, key)
+			continue
+		}
+		// Re-run sanitize: if key disappears, it was dirty.
+		trial := SanitizeV1ProcessAttributes(map[string]any{key: attrs[k]})
+		if len(trial) == 0 {
+			dirty = append(dirty, key)
+		}
+	}
+	return dirty
+}
+
+func eprelShapeOK(v any) bool {
+	if v == nil {
+		return true
+	}
+	_, ok := v.(map[string]any)
+	return ok
+}
+
+func normalizeEPRELValue(v any) any {
+	return eprel.NormalizeShape(v)
+}
+
+func imageFieldsOK(item V1ProcessJobItem) bool {
+	if item == nil {
+		return true
+	}
+	if v := item["main_image"]; v != nil {
+		if _, ok := v.(string); !ok {
+			return false
+		}
+	}
+	if v := item["more_images"]; v != nil {
+		switch v.(type) {
+		case []string, []any:
+			return true
+		default:
+			return false
+		}
+	}
+	return true
+}
+
+func stringSliceFromItem(item V1ProcessJobItem, key string) []string {
+	if item == nil {
+		return nil
+	}
+	switch t := item[key].(type) {
+	case []string:
+		out := make([]string, 0, len(t))
+		for _, s := range t {
+			s = strings.TrimSpace(s)
+			if s != "" {
+				out = append(out, s)
+			}
+		}
+		return out
+	case []any:
+		out := make([]string, 0, len(t))
+		for _, e := range t {
+			if s, ok := e.(string); ok {
+				s = strings.TrimSpace(s)
+				if s != "" {
+					out = append(out, s)
+				}
+			}
+		}
+		return out
+	default:
+		return nil
+	}
+}