descrybe/apps/api/internal/processing/v1_process_item.go

package processing

import (
	"fmt"
	"strings"

	"github.com/descrybe/descrybe-v2/apps/api/internal/company"
	"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
)

// V1ProcessItemScorecard scores one COMPLETED legacy process item against
// OpenAPI ProcessItem + A1 poll expectations.
type V1ProcessItemScorecard struct {
	EAN       string   `json:"ean"`
	Status    string   `json:"status"`
	OK        bool     `json:"ok"`
	Score     int      `json:"score"` // 0–100
	FailFlags []string `json:"fail_flags,omitempty"`

	HasTitle           bool `json:"has_title"`
	HasName            bool `json:"has_name"`
	HasDescription     bool `json:"has_description"`
	DescriptionIsPlain bool `json:"description_is_plain"`
	HasMetaTitle       bool `json:"has_meta_title"`
	HasMetaDescription bool `json:"has_meta_description"`
	HasCategory        bool `json:"has_category"`
	HasCategoryName    bool `json:"has_category_name"`
	AttrsClean         bool `json:"attrs_clean"`
	EPRELOK            bool `json:"eprel_ok"`
	HasIDs             bool `json:"has_ids"`
	ImagesOK           bool `json:"images_ok"`
}

// ScoreV1ProcessItemOptions tunes structure checks for a poll item.
type ScoreV1ProcessItemOptions struct {
	// MappedCategory, when non-empty, requires item.category_id (or legacy
	// item.category unique_id) to equal it.
	MappedCategory string
	// Language is used only for documentation of synthesize paths in tests.
	Language string
	// OmitSEOMeta skips meta_title / meta_description presence checks (A1 cohort).
	OmitSEOMeta bool
}

// ScoreV1ProcessCompletedItem scores a successful (or terminal) V1 process item.
// For status != processed it only checks ean/status shape and returns OK when those exist.
func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemOptions) V1ProcessItemScorecard {
	sc := V1ProcessItemScorecard{
		EAN:    strings.TrimSpace(fmt.Sprint(item["ean"])),
		Status: strings.TrimSpace(fmt.Sprint(item["status"])),
	}
	if sc.EAN == "" || sc.EAN == "<nil>" {
		sc.FailFlags = append(sc.FailFlags, "missing_ean")
	}
	if sc.Status == "" || sc.Status == "<nil>" {
		sc.FailFlags = append(sc.FailFlags, "missing_status")
	}
	if sc.Status != "processed" {
		sc.OK = len(sc.FailFlags) == 0
		if sc.OK {
			sc.Score = 100
		}
		return sc
	}

	title := stringFromItem(item, "name")
	if title == "" {
		title = stringFromItem(item, "title")
	}
	name := stringFromItem(item, "name")
	if name == "" {
		name = title
	}
	sc.HasTitle = title != ""
	sc.HasName = name != ""
	if !sc.HasName && sc.HasTitle {
		sc.FailFlags = append(sc.FailFlags, "missing_name")
	}
	if sc.HasTitle && sc.HasName && title != name {
		sc.FailFlags = append(sc.FailFlags, "title_name_mismatch")
	}

	desc, descOK := plainDescriptionFromItem(item)
	sc.HasDescription = desc != ""
	sc.DescriptionIsPlain = descOK
	if sc.HasTitle && !sc.HasDescription {
		sc.FailFlags = append(sc.FailFlags, "description_empty_with_title")
	}
	if !descOK && item["description"] != nil {
		sc.FailFlags = append(sc.FailFlags, "description_not_plain_string")
	}
	if sc.HasDescription && descriptionEchoesTitle(desc, title) {
		sc.FailFlags = append(sc.FailFlags, "description_echoes_title")
	}
	if sc.HasDescription && containsWeakFillerPhrase(desc) {
		sc.FailFlags = append(sc.FailFlags, "description_weak_filler")
	}

	sc.HasMetaTitle = stringFromItem(item, "meta_title") != ""
	sc.HasMetaDescription = stringFromItem(item, "meta_description") != ""
	if !opts.OmitSEOMeta {
		if sc.HasTitle && !sc.HasMetaTitle {
			sc.FailFlags = append(sc.FailFlags, "meta_title_missing")
		}
		if sc.HasTitle && !sc.HasMetaDescription {
			sc.FailFlags = append(sc.FailFlags, "meta_description_missing")
		}
		if md := stringFromItem(item, "meta_description"); md != "" && containsWeakFillerPhrase(md) {
			sc.FailFlags = append(sc.FailFlags, "meta_description_weak_filler")
		}
	}

	cat := stringFromItem(item, "category")
	catID := stringFromItem(item, "category_id")
	catName := stringFromItem(item, "category_name")
	sc.HasCategory = cat != "" || catID != ""
	sc.HasCategoryName = catName != "" || (cat != "" && catID != "" && cat != catID)
	mappedCat := strings.TrimSpace(opts.MappedCategory)
	if mappedCat != "" {
		gotUID := catID
		if gotUID == "" {
			gotUID = cat // legacy: category held unique_id
		}
		if gotUID == "" {
			sc.FailFlags = append(sc.FailFlags, "category_missing_though_mapped")
		} else if gotUID != mappedCat && cat != mappedCat {
			sc.FailFlags = append(sc.FailFlags, "category_mismatch_mapped")
		}
		if catName == "" && (cat == "" || cat == gotUID) {
			sc.FailFlags = append(sc.FailFlags, "category_name_missing")
		}
	}

	attrs, attrsOK := attrsMapFromItem(item)
	dirty := dirtyV1AttrKeys(attrs)
	sc.AttrsClean = attrsOK && len(dirty) == 0
	if !attrsOK {
		sc.FailFlags = append(sc.FailFlags, "attributes_invalid_shape")
	}
	if len(dirty) > 0 {
		sc.FailFlags = append(sc.FailFlags, "attributes_dirty:"+strings.Join(dirty, ","))
	}

	sc.EPRELOK = eprelShapeOK(item["eprel"])
	if !sc.EPRELOK {
		sc.FailFlags = append(sc.FailFlags, "eprel_invalid_shape")
	}

	// Internal UUIDs are omitted from the public V1 process item shape.
	sc.HasIDs = true

	sc.ImagesOK = imageFieldsOK(item)
	if !sc.ImagesOK {
		sc.FailFlags = append(sc.FailFlags, "images_invalid_shape")
	}

	// Weighted score: structure gates, not content quality beyond filler/echo.
	checks := []bool{
		sc.EAN != "",
		sc.Status == "processed",
		sc.HasTitle,
		sc.HasName || !sc.HasTitle,
		sc.HasDescription || !sc.HasTitle,
		sc.DescriptionIsPlain || item["description"] == nil,
		sc.HasMetaTitle || !sc.HasTitle,
		sc.HasMetaDescription || !sc.HasTitle,
		mappedCat == "" || sc.HasCategory,
		mappedCat == "" || sc.HasCategoryName,
		sc.AttrsClean,
		sc.EPRELOK,
		sc.HasIDs,
		sc.ImagesOK,
		!containsWeakFillerPhrase(desc),
		!descriptionEchoesTitle(desc, title),
	}
	pass := 0
	for _, ok := range checks {
		if ok {
			pass++
		}
	}
	sc.Score = (pass * 100) / len(checks)
	sc.OK = len(sc.FailFlags) == 0
	return sc
}

// EnforceV1ProcessCompletedItem fills ProcessItem projection gaps for a successful item:
// nonempty description when title exists (formula HTML preserved), readable title spacing,
// category display name as primary, optional SEO meta (unless omitSEOMeta), clean attrs.
// allowed is the company attribute_key set (canonicalized). When nil, only
// coreCharacteristicAttrKeys are kept (never leak feed junk like zavora).
func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allowed map[string]struct{}) V1ProcessJobItem {
	return EnforceV1ProcessCompletedItemOpts(item, EnforceV1Opts{
		Language: language,
		Allowed:  allowed,
	})
}

// EnforceV1Opts configures V1 completed-item projection.
type EnforceV1Opts struct {
	Language            string
	Allowed             map[string]struct{}
	OmitSEOMeta         bool
	DescriptionTemplate any
}

// EnforceV1ProcessCompletedItemOpts is the options-aware EnforceV1 entrypoint.
func EnforceV1ProcessCompletedItemOpts(item V1ProcessJobItem, opts EnforceV1Opts) V1ProcessJobItem {
	if item == nil {
		return item
	}
	stripV1ProcessItemInternalKeys(item)
	status, _ := item["status"].(string)
	if status != "processed" {
		return item
	}

	allowed := opts.Allowed
	if allowed == nil {
		allowed = map[string]struct{}{}
	}
	attrs, _ := attrsMapFromItem(item)
	attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowed)
	if len(attrs) > 0 {
		item["attributes"] = attrs
	} else {
		item["attributes"] = nil
	}

	title := ensureReadableTitleSpacing(stringFromItem(item, "name"))
	if title == "" {
		title = ensureReadableTitleSpacing(stringFromItem(item, "title"))
	}
	if isPromptLabelTitle(title) {
		title = preferredProductTitle(stringFromItem(item, "ean"), title)
	}
	if title != "" {
		item["name"] = title
		delete(item, "title")
	} else {
		item["name"] = nil
		delete(item, "title")
	}

	catID, catName := projectV1CategoryFields(item)
	catLabel := catName
	if catLabel == "" {
		catLabel = catID
	}

	desc, _ := descriptionFromItem(item)
	if isPromptLeakageTitle(desc) || isPromptLabelTitle(desc) {
		desc = ""
	}
	needsDesc := title != "" && (desc == "" ||
		isWeakPriorEnhanceDescription(desc, title) ||
		descriptionEchoesTitle(desc, title) ||
		company.LooksLikeHeuristicSynthesize(desc))
	if needsDesc {
		tpl := opts.DescriptionTemplate
		if tpl == nil {
			tpl = inferDescriptionTemplateFromHTML(desc)
		}
		if synth := synthesizeProductDescription(title, catLabel, opts.Language, attrs, tpl); synth != "" {
			desc = synth
		} else if synth := synthesizeDescriptionFromTitle(title, catLabel, opts.Language, attrs); synth != "" {
			desc = synth
		}
	}
	if desc != "" {
		item["description"] = desc
	} else {
		item["description"] = nil
	}

	if opts.OmitSEOMeta {
		delete(item, "meta_title")
		delete(item, "meta_description")
	} else {
		metaTitle := stringFromItem(item, "meta_title")
		metaDesc := stringFromItem(item, "meta_description")
		needMetaTitle := metaTitle == "" || isPoisonedMetaTitle(metaTitle)
		needMetaDesc := metaDesc == "" ||
			isWeakPriorEnhanceDescription(metaDesc, title) ||
			containsWeakFillerPhrase(metaDesc) ||
			isPromptLeakageTitle(metaDesc) ||
			(title != "" && descriptionEchoesTitle(metaDesc, title))
		if needMetaTitle && isPoisonedMetaTitle(metaTitle) &&
			(metaDesc == "" || isWeakPriorEnhanceDescription(metaDesc, title) || isPromptLeakageTitle(metaDesc)) {
			needMetaDesc = true
		}
		if title != "" || desc != "" || catID != "" || catName != "" {
			synthTitle, synthDesc := fillMetaFromResult(StepResult{
				Name:                 title,
				ProcessedName:        title,
				Category:             catID,
				CategoryName:         catName,
				Description:          desc,
				ProcessedDescription: desc,
				Attributes:           attrs,
				ProcessedAttributes:  attrs,
			})
			if needMetaTitle {
				if synthTitle != "" {
					metaTitle = synthTitle
				} else {
					metaTitle = title
				}
			}
			if needMetaDesc {
				if synthDesc != "" {
					metaDesc = synthDesc
				} else if desc != "" {
					metaDesc = truncateMetaDescription(desc, v1MetaDescriptionMaxChars)
				}
			}
		}
		if metaTitle != "" {
			item["meta_title"] = metaTitle
		} else {
			item["meta_title"] = nil
		}
		if metaDesc != "" {
			item["meta_description"] = metaDesc
		} else {
			item["meta_description"] = nil
		}
	}

	item["eprel"] = normalizeEPRELValue(item["eprel"])

	main := stringFromItem(item, "main_image")
	if main != "" {
		item["main_image"] = main
	} else {
		item["main_image"] = nil
	}
	more := stringSliceFromItem(item, "more_images")
	if len(more) > 0 {
		item["more_images"] = more
	} else {
		item["more_images"] = nil
	}

	return item
}

// projectV1CategoryFields sets category=display name, category_id=unique_id,
// category_name=display name. Returns (unique_id, display_name).
func projectV1CategoryFields(item V1ProcessJobItem) (catID, catName string) {
	catID = stringFromItem(item, "category_id")
	catName = stringFromItem(item, "category_name")
	cat := stringFromItem(item, "category")
	if catID == "" {
		// Legacy: category held unique_id when category_name differed.
		if catName != "" && cat != "" && cat != catName {
			catID = cat
		} else if catName == "" && cat != "" {
			catID = cat
		}
	}
	if catName == "" && cat != "" && cat != catID {
		catName = cat
	}
	if catName != "" {
		item["category"] = catName
		item["category_name"] = catName
	} else {
		item["category"] = nil
		item["category_name"] = nil
	}
	if catID != "" {
		item["category_id"] = catID
	} else {
		delete(item, "category_id")
	}
	return catID, catName
}

// inferDescriptionTemplateFromHTML rebuilds a minimal description_template from
// tags already present so invent-garbage HTML can be re-synthesized without a DB lookup.
func inferDescriptionTemplateFromHTML(html string) any {
	lower := strings.ToLower(html)
	var sections []map[string]any
	for _, typ := range []string{"h1", "h2", "h3", "h4", "p", "ul"} {
		if strings.Contains(lower, "<"+typ) {
			sections = append(sections, map[string]any{"type": typ})
		}
	}
	if len(sections) == 0 {
		return map[string]any{
			"sections": []map[string]any{
				{"type": "h1"},
				{"type": "p"},
				{"type": "h2"},
				{"type": "ul"},
			},
		}
	}
	return map[string]any{"sections": sections}
}

func stringFromItem(item V1ProcessJobItem, key string) string {
	if item == nil {
		return ""
	}
	return strings.TrimSpace(stringFromAny(item[key]))
}

func descriptionFromItem(item V1ProcessJobItem) (string, bool) {
	if item == nil {
		return "", true
	}
	raw := item["description"]
	if raw == nil {
		return "", true
	}
	switch raw.(type) {
	case string:
		return DescriptionFromAny(raw), true
	case []any, []string:
		// Legacy mistake: description as array — coerce to string, keep HTML.
		if s := DescriptionFromAny(raw); s != "" {
			return s, true
		}
		return "", false
	case map[string]any:
		if s := DescriptionFromAny(raw); s != "" {
			return s, true
		}
		return "", false
	default:
		s := strings.TrimSpace(fmt.Sprint(raw))
		if s == "" || s == "<nil>" {
			return "", true
		}
		return "", false
	}
}

// plainDescriptionFromItem is kept for callers that need SEO/plain text (meta).
func plainDescriptionFromItem(item V1ProcessJobItem) (string, bool) {
	if item == nil {
		return "", true
	}
	raw := item["description"]
	if raw == nil {
		return "", true
	}
	switch raw.(type) {
	case string:
		return PlainDescriptionFromAny(raw), true
	case []any, []string:
		if s := PlainDescriptionFromAny(raw); s != "" {
			return s, true
		}
		return "", false
	case map[string]any:
		if s := PlainDescriptionFromAny(raw); s != "" {
			return s, true
		}
		return "", false
	default:
		s := strings.TrimSpace(fmt.Sprint(raw))
		if s == "" || s == "<nil>" {
			return "", true
		}
		return "", false
	}
}

func attrsMapFromItem(item V1ProcessJobItem) (map[string]any, bool) {
	if item == nil {
		return map[string]any{}, true
	}
	raw := item["attributes"]
	if raw == nil {
		return map[string]any{}, true
	}
	switch t := raw.(type) {
	case map[string]any:
		return t, true
	default:
		return map[string]any{}, false
	}
}

func dirtyV1AttrKeys(attrs map[string]any) []string {
	if len(attrs) == 0 {
		return nil
	}
	var dirty []string
	for k := range attrs {
		key := strings.TrimSpace(k)
		lk := strings.ToLower(key)
		if key == "" || isReservedProductKey(key) || isInvalidAttributeKey(key) ||
			strings.HasPrefix(lk, "eprel") {
			dirty = append(dirty, key)
			continue
		}
		// Re-run sanitize: if key disappears, it was dirty.
		trial := SanitizeV1ProcessAttributes(map[string]any{key: attrs[k]})
		if len(trial) == 0 {
			dirty = append(dirty, key)
		}
	}
	return dirty
}

func eprelShapeOK(v any) bool {
	if v == nil {
		return true
	}
	_, ok := v.(map[string]any)
	return ok
}

func normalizeEPRELValue(v any) any {
	return eprel.NormalizeShape(v)
}

func imageFieldsOK(item V1ProcessJobItem) bool {
	if item == nil {
		return true
	}
	if v := item["main_image"]; v != nil {
		if _, ok := v.(string); !ok {
			return false
		}
	}
	if v := item["more_images"]; v != nil {
		switch v.(type) {
		case []string, []any:
			return true
		default:
			return false
		}
	}
	return true
}

func stringSliceFromItem(item V1ProcessJobItem, key string) []string {
	if item == nil {
		return nil
	}
	switch t := item[key].(type) {
	case []string:
		out := make([]string, 0, len(t))
		for _, s := range t {
			s = strings.TrimSpace(s)
			if s != "" {
				out = append(out, s)
			}
		}
		return out
	case []any:
		out := make([]string, 0, len(t))
		for _, e := range t {
			if s, ok := e.(string); ok {
				s = strings.TrimSpace(s)
				if s != "" {
					out = append(out, s)
				}
			}
		}
		return out
	default:
		return nil
	}
}