457 lines
12 KiB
Go
457 lines
12 KiB
Go
package processing
|
||
|
||
import (
|
||
"fmt"
|
||
"strings"
|
||
|
||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||
)
|
||
|
||
// V1ProcessItemScorecard scores one COMPLETED legacy process item against
|
||
// OpenAPI LegacyProcessItem + A1 poll expectations.
|
||
type V1ProcessItemScorecard struct {
|
||
EAN string `json:"ean"`
|
||
Status string `json:"status"`
|
||
OK bool `json:"ok"`
|
||
Score int `json:"score"` // 0–100
|
||
FailFlags []string `json:"fail_flags,omitempty"`
|
||
|
||
HasTitle bool `json:"has_title"`
|
||
HasName bool `json:"has_name"`
|
||
HasDescription bool `json:"has_description"`
|
||
DescriptionIsPlain bool `json:"description_is_plain"`
|
||
HasMetaTitle bool `json:"has_meta_title"`
|
||
HasMetaDescription bool `json:"has_meta_description"`
|
||
HasCategory bool `json:"has_category"`
|
||
HasCategoryName bool `json:"has_category_name"`
|
||
AttrsClean bool `json:"attrs_clean"`
|
||
EPRELOK bool `json:"eprel_ok"`
|
||
HasIDs bool `json:"has_ids"`
|
||
ImagesOK bool `json:"images_ok"`
|
||
}
|
||
|
||
// ScoreV1ProcessItemOptions tunes structure checks for a poll item.
|
||
type ScoreV1ProcessItemOptions struct {
|
||
// MappedCategory, when non-empty, requires item.category to equal it
|
||
// (projection must surface mapped unique_id).
|
||
MappedCategory string
|
||
// Language is used only for documentation of synthesize paths in tests.
|
||
Language string
|
||
}
|
||
|
||
// ScoreV1ProcessCompletedItem scores a successful (or terminal) V1 process item.
|
||
// For status != processed it only checks ean/status shape and returns OK when those exist.
|
||
func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemOptions) V1ProcessItemScorecard {
|
||
sc := V1ProcessItemScorecard{
|
||
EAN: strings.TrimSpace(fmt.Sprint(item["ean"])),
|
||
Status: strings.TrimSpace(fmt.Sprint(item["status"])),
|
||
}
|
||
if sc.EAN == "" || sc.EAN == "<nil>" {
|
||
sc.FailFlags = append(sc.FailFlags, "missing_ean")
|
||
}
|
||
if sc.Status == "" || sc.Status == "<nil>" {
|
||
sc.FailFlags = append(sc.FailFlags, "missing_status")
|
||
}
|
||
if sc.Status != "processed" {
|
||
sc.OK = len(sc.FailFlags) == 0
|
||
if sc.OK {
|
||
sc.Score = 100
|
||
}
|
||
return sc
|
||
}
|
||
|
||
title := stringFromItem(item, "title")
|
||
name := stringFromItem(item, "name")
|
||
sc.HasTitle = title != ""
|
||
sc.HasName = name != ""
|
||
if sc.HasTitle && !sc.HasName {
|
||
sc.FailFlags = append(sc.FailFlags, "missing_name")
|
||
}
|
||
if sc.HasTitle && sc.HasName && title != name {
|
||
sc.FailFlags = append(sc.FailFlags, "title_name_mismatch")
|
||
}
|
||
|
||
desc, descOK := plainDescriptionFromItem(item)
|
||
sc.HasDescription = desc != ""
|
||
sc.DescriptionIsPlain = descOK
|
||
if sc.HasTitle && !sc.HasDescription {
|
||
sc.FailFlags = append(sc.FailFlags, "description_empty_with_title")
|
||
}
|
||
if !descOK && item["description"] != nil {
|
||
sc.FailFlags = append(sc.FailFlags, "description_not_plain_string")
|
||
}
|
||
if sc.HasDescription && descriptionEchoesTitle(desc, title) {
|
||
sc.FailFlags = append(sc.FailFlags, "description_echoes_title")
|
||
}
|
||
if sc.HasDescription && containsWeakFillerPhrase(desc) {
|
||
sc.FailFlags = append(sc.FailFlags, "description_weak_filler")
|
||
}
|
||
|
||
sc.HasMetaTitle = stringFromItem(item, "meta_title") != ""
|
||
sc.HasMetaDescription = stringFromItem(item, "meta_description") != ""
|
||
if sc.HasTitle && !sc.HasMetaTitle {
|
||
sc.FailFlags = append(sc.FailFlags, "meta_title_missing")
|
||
}
|
||
if sc.HasTitle && !sc.HasMetaDescription {
|
||
sc.FailFlags = append(sc.FailFlags, "meta_description_missing")
|
||
}
|
||
if md := stringFromItem(item, "meta_description"); md != "" && containsWeakFillerPhrase(md) {
|
||
sc.FailFlags = append(sc.FailFlags, "meta_description_weak_filler")
|
||
}
|
||
|
||
cat := stringFromItem(item, "category")
|
||
catName := stringFromItem(item, "category_name")
|
||
sc.HasCategory = cat != ""
|
||
sc.HasCategoryName = catName != ""
|
||
mappedCat := strings.TrimSpace(opts.MappedCategory)
|
||
if mappedCat != "" {
|
||
if cat == "" {
|
||
sc.FailFlags = append(sc.FailFlags, "category_missing_though_mapped")
|
||
} else if cat != mappedCat {
|
||
sc.FailFlags = append(sc.FailFlags, "category_mismatch_mapped")
|
||
}
|
||
if cat != "" && catName == "" {
|
||
sc.FailFlags = append(sc.FailFlags, "category_name_missing")
|
||
}
|
||
}
|
||
|
||
attrs, attrsOK := attrsMapFromItem(item)
|
||
dirty := dirtyV1AttrKeys(attrs)
|
||
sc.AttrsClean = attrsOK && len(dirty) == 0
|
||
if !attrsOK {
|
||
sc.FailFlags = append(sc.FailFlags, "attributes_invalid_shape")
|
||
}
|
||
if len(dirty) > 0 {
|
||
sc.FailFlags = append(sc.FailFlags, "attributes_dirty:"+strings.Join(dirty, ","))
|
||
}
|
||
|
||
sc.EPRELOK = eprelShapeOK(item["eprel"])
|
||
if !sc.EPRELOK {
|
||
sc.FailFlags = append(sc.FailFlags, "eprel_invalid_shape")
|
||
}
|
||
|
||
id := stringFromItem(item, "id")
|
||
ppID := stringFromItem(item, "processed_product_id")
|
||
rawID := stringFromItem(item, "raw_product_id")
|
||
sc.HasIDs = id != "" && ppID != "" && rawID != ""
|
||
if id == "" {
|
||
sc.FailFlags = append(sc.FailFlags, "missing_id")
|
||
}
|
||
if ppID == "" {
|
||
sc.FailFlags = append(sc.FailFlags, "missing_processed_product_id")
|
||
}
|
||
if rawID == "" {
|
||
sc.FailFlags = append(sc.FailFlags, "missing_raw_product_id")
|
||
}
|
||
if id != "" && ppID != "" && id != ppID {
|
||
sc.FailFlags = append(sc.FailFlags, "id_processed_product_id_mismatch")
|
||
}
|
||
|
||
sc.ImagesOK = imageFieldsOK(item)
|
||
if !sc.ImagesOK {
|
||
sc.FailFlags = append(sc.FailFlags, "images_invalid_shape")
|
||
}
|
||
|
||
// Weighted score: structure gates, not content quality beyond filler/echo.
|
||
checks := []bool{
|
||
sc.EAN != "",
|
||
sc.Status == "processed",
|
||
sc.HasTitle,
|
||
sc.HasName || !sc.HasTitle,
|
||
sc.HasDescription || !sc.HasTitle,
|
||
sc.DescriptionIsPlain || item["description"] == nil,
|
||
sc.HasMetaTitle || !sc.HasTitle,
|
||
sc.HasMetaDescription || !sc.HasTitle,
|
||
mappedCat == "" || sc.HasCategory,
|
||
mappedCat == "" || sc.HasCategoryName,
|
||
sc.AttrsClean,
|
||
sc.EPRELOK,
|
||
sc.HasIDs,
|
||
sc.ImagesOK,
|
||
!containsWeakFillerPhrase(desc),
|
||
!descriptionEchoesTitle(desc, title),
|
||
}
|
||
pass := 0
|
||
for _, ok := range checks {
|
||
if ok {
|
||
pass++
|
||
}
|
||
}
|
||
sc.Score = (pass * 100) / len(checks)
|
||
sc.OK = len(sc.FailFlags) == 0
|
||
return sc
|
||
}
|
||
|
||
// EnforceV1ProcessCompletedItem fills LegacyProcessItem projection gaps for a
|
||
// successful item: plain nonempty description when title exists, meta_*, clean
|
||
// attributes, eprel object|null, and image key shapes. Category must already be
|
||
// set by the caller when mapped provides a unique_id.
|
||
//
|
||
// allowed is the company attribute_key set (canonicalized). When nil, only
|
||
// coreCharacteristicAttrKeys are kept (never leak feed junk like zavora).
|
||
func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allowed map[string]struct{}) V1ProcessJobItem {
|
||
if item == nil {
|
||
return item
|
||
}
|
||
status, _ := item["status"].(string)
|
||
if status != "processed" {
|
||
return item
|
||
}
|
||
|
||
attrs, _ := attrsMapFromItem(item)
|
||
if allowed == nil {
|
||
allowed = map[string]struct{}{}
|
||
}
|
||
attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowed)
|
||
if len(attrs) > 0 {
|
||
item["attributes"] = attrs
|
||
} else {
|
||
item["attributes"] = nil
|
||
}
|
||
|
||
title := stringFromItem(item, "title")
|
||
if title != "" {
|
||
item["title"] = title
|
||
item["name"] = title
|
||
} else {
|
||
item["title"] = nil
|
||
item["name"] = nil
|
||
}
|
||
cat := stringFromItem(item, "category")
|
||
catName := stringFromItem(item, "category_name")
|
||
catLabel := catName
|
||
if catLabel == "" {
|
||
catLabel = cat
|
||
}
|
||
|
||
desc, _ := plainDescriptionFromItem(item)
|
||
// Empty, weak, or title-echo copy must be replaced — never leave description==title.
|
||
if title != "" && (desc == "" || isWeakPriorEnhanceDescription(desc, title) || descriptionEchoesTitle(desc, title)) {
|
||
if synth := synthesizeDescriptionFromTitle(title, catLabel, language, attrs); synth != "" {
|
||
desc = synth
|
||
}
|
||
}
|
||
if desc != "" {
|
||
item["description"] = desc
|
||
} else {
|
||
item["description"] = nil
|
||
}
|
||
|
||
metaTitle := stringFromItem(item, "meta_title")
|
||
metaDesc := stringFromItem(item, "meta_description")
|
||
needMetaTitle := metaTitle == "" || isPoisonedMetaTitle(metaTitle)
|
||
// Empty, weak stub (Ready for retail…), title-echo, or prompt-leakage — refresh.
|
||
// When meta_title is poisoned, also force refresh of weak/leakage meta_description.
|
||
needMetaDesc := metaDesc == "" ||
|
||
isWeakPriorEnhanceDescription(metaDesc, title) ||
|
||
containsWeakFillerPhrase(metaDesc) ||
|
||
isPromptLeakageTitle(metaDesc) ||
|
||
(title != "" && descriptionEchoesTitle(metaDesc, title))
|
||
if needMetaTitle && isPoisonedMetaTitle(metaTitle) &&
|
||
(metaDesc == "" || isWeakPriorEnhanceDescription(metaDesc, title) || isPromptLeakageTitle(metaDesc)) {
|
||
needMetaDesc = true
|
||
}
|
||
if title != "" || desc != "" || cat != "" || catName != "" {
|
||
synthTitle, synthDesc := fillMetaFromResult(StepResult{
|
||
Name: title,
|
||
ProcessedName: title,
|
||
Category: cat,
|
||
CategoryName: catName,
|
||
Description: desc,
|
||
ProcessedDescription: desc,
|
||
Attributes: attrs,
|
||
ProcessedAttributes: attrs,
|
||
})
|
||
if needMetaTitle {
|
||
if synthTitle != "" {
|
||
metaTitle = synthTitle
|
||
} else {
|
||
metaTitle = title
|
||
}
|
||
}
|
||
if needMetaDesc {
|
||
if synthDesc != "" {
|
||
metaDesc = synthDesc
|
||
} else if desc != "" {
|
||
metaDesc = truncateMetaDescription(desc, v1MetaDescriptionMaxChars)
|
||
}
|
||
}
|
||
}
|
||
if metaTitle != "" {
|
||
item["meta_title"] = metaTitle
|
||
} else {
|
||
item["meta_title"] = nil
|
||
}
|
||
if metaDesc != "" {
|
||
item["meta_description"] = metaDesc
|
||
} else {
|
||
item["meta_description"] = nil
|
||
}
|
||
|
||
if cat != "" {
|
||
item["category"] = cat
|
||
} else {
|
||
item["category"] = nil
|
||
}
|
||
if catName != "" {
|
||
item["category_name"] = catName
|
||
} else {
|
||
item["category_name"] = nil
|
||
}
|
||
|
||
item["eprel"] = normalizeEPRELValue(item["eprel"])
|
||
|
||
main := stringFromItem(item, "main_image")
|
||
if main != "" {
|
||
item["main_image"] = main
|
||
} else {
|
||
item["main_image"] = nil
|
||
}
|
||
more := stringSliceFromItem(item, "more_images")
|
||
if len(more) > 0 {
|
||
item["more_images"] = more
|
||
} else {
|
||
item["more_images"] = nil
|
||
}
|
||
|
||
return item
|
||
}
|
||
|
||
func stringFromItem(item V1ProcessJobItem, key string) string {
|
||
if item == nil {
|
||
return ""
|
||
}
|
||
return strings.TrimSpace(stringFromAny(item[key]))
|
||
}
|
||
|
||
func plainDescriptionFromItem(item V1ProcessJobItem) (string, bool) {
|
||
if item == nil {
|
||
return "", true
|
||
}
|
||
raw := item["description"]
|
||
if raw == nil {
|
||
return "", true
|
||
}
|
||
switch raw.(type) {
|
||
case string:
|
||
return PlainDescriptionFromAny(raw), true
|
||
case []any, []string:
|
||
// Legacy mistake: description as array — convert to plain string.
|
||
if s := PlainDescriptionFromAny(raw); s != "" {
|
||
return s, true
|
||
}
|
||
return "", false
|
||
case map[string]any:
|
||
if s := PlainDescriptionFromAny(raw); s != "" {
|
||
return s, true
|
||
}
|
||
return "", false
|
||
default:
|
||
s := strings.TrimSpace(fmt.Sprint(raw))
|
||
if s == "" || s == "<nil>" {
|
||
return "", true
|
||
}
|
||
return "", false
|
||
}
|
||
}
|
||
|
||
func attrsMapFromItem(item V1ProcessJobItem) (map[string]any, bool) {
|
||
if item == nil {
|
||
return map[string]any{}, true
|
||
}
|
||
raw := item["attributes"]
|
||
if raw == nil {
|
||
return map[string]any{}, true
|
||
}
|
||
switch t := raw.(type) {
|
||
case map[string]any:
|
||
return t, true
|
||
default:
|
||
return map[string]any{}, false
|
||
}
|
||
}
|
||
|
||
func dirtyV1AttrKeys(attrs map[string]any) []string {
|
||
if len(attrs) == 0 {
|
||
return nil
|
||
}
|
||
var dirty []string
|
||
for k := range attrs {
|
||
key := strings.TrimSpace(k)
|
||
lk := strings.ToLower(key)
|
||
if key == "" || isReservedProductKey(key) || isInvalidAttributeKey(key) ||
|
||
strings.HasPrefix(lk, "eprel") {
|
||
dirty = append(dirty, key)
|
||
continue
|
||
}
|
||
// Re-run sanitize: if key disappears, it was dirty.
|
||
trial := SanitizeV1ProcessAttributes(map[string]any{key: attrs[k]})
|
||
if len(trial) == 0 {
|
||
dirty = append(dirty, key)
|
||
}
|
||
}
|
||
return dirty
|
||
}
|
||
|
||
func eprelShapeOK(v any) bool {
|
||
if v == nil {
|
||
return true
|
||
}
|
||
_, ok := v.(map[string]any)
|
||
return ok
|
||
}
|
||
|
||
func normalizeEPRELValue(v any) any {
|
||
return eprel.NormalizeShape(v)
|
||
}
|
||
|
||
func imageFieldsOK(item V1ProcessJobItem) bool {
|
||
if item == nil {
|
||
return true
|
||
}
|
||
if v := item["main_image"]; v != nil {
|
||
if _, ok := v.(string); !ok {
|
||
return false
|
||
}
|
||
}
|
||
if v := item["more_images"]; v != nil {
|
||
switch v.(type) {
|
||
case []string, []any:
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
func stringSliceFromItem(item V1ProcessJobItem, key string) []string {
|
||
if item == nil {
|
||
return nil
|
||
}
|
||
switch t := item[key].(type) {
|
||
case []string:
|
||
out := make([]string, 0, len(t))
|
||
for _, s := range t {
|
||
s = strings.TrimSpace(s)
|
||
if s != "" {
|
||
out = append(out, s)
|
||
}
|
||
}
|
||
return out
|
||
case []any:
|
||
out := make([]string, 0, len(t))
|
||
for _, e := range t {
|
||
if s, ok := e.(string); ok {
|
||
s = strings.TrimSpace(s)
|
||
if s != "" {
|
||
out = append(out, s)
|
||
}
|
||
}
|
||
}
|
||
return out
|
||
default:
|
||
return nil
|
||
}
|
||
}
|