This commit is contained in:
2026-08-17 09:33:07 +02:00
parent 0dceb3a404
commit fe94c2fb9c
40 changed files with 2360 additions and 340 deletions
+3 -4
View File
@@ -528,11 +528,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
}
// Pipelines without StepCategorize (enhance_only / normalize_only) still try
// vector categorize when AllowAI + embeddings are available. Full/categorize
// already ran runCategorizeStep (vector then LLM) inside the loop.
// Pipelines without StepCategorize (enhance_only / normalize_only) still run
// vector + LLM categorize when category is empty so enhance is not fed a blank.
if !stepsContain(steps, StepCategorize) {
tryVectorCategorize(ctx, e, companyID, &out, categoryNames, policy)
runCategorizeStep(ctx, e, companyID, &out, in, categoryNames, policy)
noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled())
}
preserveCategoryIfEmpty(&out, in.PriorCategory)
+48 -26
View File
@@ -261,6 +261,8 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
); err != nil {
return nil, err
}
_ = processedID
_ = rawProductID
if processedID == nil {
st := MapV1JobItemStatus(itemStatus, false)
if st == "processed" || st == "processing" || st == "pending" {
@@ -274,7 +276,6 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
if itemError != nil && *itemError != "" {
item["error"] = *itemError
}
applyV1ProcessItemIDs(item, nil, rawProductID)
full = append(full, item)
continue
}
@@ -428,9 +429,9 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
catIDOut = catStr
}
var titleOut any
var nameOut any
if titleStr != "" {
titleOut = titleStr
nameOut = titleStr
}
item := V1ProcessJobItem{
"ean": ean,
@@ -438,8 +439,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
"category": catNameOut,
"category_id": catIDOut,
"category_name": catNameOut,
"title": titleOut,
"name": titleOut,
"name": nameOut,
"description": description,
"attributes": nil,
"main_image": nil,
@@ -450,7 +450,8 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
item["meta_title"] = metaTitleOut
item["meta_description"] = metaDescOut
}
applyV1ProcessItemIDs(item, processedID, rawProductID)
// Internal ids (id / processed_product_id / raw_product_id) are omitted from
// the public V1 process item shape — use catalog APIs when a UUID is needed.
if itemError != nil && *itemError != "" {
item["error"] = *itemError
}
@@ -483,20 +484,41 @@ func nullIfEmptyPtr(s *string) any {
return *s
}
// companyOmitsSEOMeta is true for the A1 cohort (no meta_title / meta_description).
// companyOmitsSEOMeta is true when V1/process must omit meta_title / meta_description:
// A1 cohort (legacy id), Platform Demo (A1-cloned prompts, no legacy id), or any
// company whose categories store A1-style role-section enhance prompts.
func companyOmitsSEOMeta(ctx context.Context, p *Pipeline, companyID uuid.UUID) bool {
if p == nil || p.Pool == nil {
return false
}
var legacy string
var legacy, name string
err := p.Pool.QueryRow(ctx, `
SELECT COALESCE(legacy_company_id, '')
SELECT COALESCE(legacy_company_id, ''), COALESCE(name, '')
FROM companies
WHERE id = $1`, companyID).Scan(&legacy)
WHERE id = $1`, companyID).Scan(&legacy, &name)
if err != nil {
return false
}
return billing.IsA1CohortCompany(legacy, "")
if billing.IsA1CohortCompany(legacy, "") {
return true
}
if strings.EqualFold(strings.TrimSpace(name), "Platform Demo") {
return true
}
var hasA1Prompts bool
err = p.Pool.QueryRow(ctx, `
SELECT EXISTS (
SELECT 1 FROM categories
WHERE company_id = $1
AND prompt ILIKE '%--- Title ---%'
AND prompt ILIKE '%--- Description ---%'
AND prompt ILIKE '%--- Meta ---%'
LIMIT 1
)`, companyID).Scan(&hasA1Prompts)
if err != nil {
return false
}
return hasA1Prompts
}
func derefStringPtr(s *string) string {
@@ -724,10 +746,14 @@ func ProjectV1ProcessJobItems(storedType string, items []V1ProcessJobItem) []V1P
projected["category_id"] = item["category_id"]
projected["category_name"] = item["category_name"]
case "title":
projected["title"] = item["title"]
projected["name"] = item["name"]
if projected["name"] == nil {
projected["name"] = item["title"]
name := item["name"]
if name == nil {
name = item["title"]
}
projected["name"] = name
// Omit duplicate title when it matches name (name is primary).
if title := item["title"]; title != nil && title != name {
projected["title"] = title
}
if _, ok := item["meta_title"]; ok {
projected["meta_title"] = item["meta_title"]
@@ -747,7 +773,7 @@ func ProjectV1ProcessJobItems(storedType string, items []V1ProcessJobItem) []V1P
}
func withItemMeta(dst, src V1ProcessJobItem) V1ProcessJobItem {
for _, k := range []string{"status", "error", "id", "processed_product_id", "raw_product_id"} {
for _, k := range []string{"status", "error"} {
if v, ok := src[k]; ok {
dst[k] = v
}
@@ -755,17 +781,13 @@ func withItemMeta(dst, src V1ProcessJobItem) V1ProcessJobItem {
return dst
}
// applyV1ProcessItemIDs sets legacy id (= processed UUID) plus additive dual-mode aliases.
// id is preserved for existing integrators; processed_product_id mirrors it; raw_product_id is raw_products.id.
// applyV1ProcessItemIDs formerly stamped id / processed_product_id / raw_product_id onto
// V1 process items. Those fields are omitted from the public contract; kept as a no-op
// so older call sites/tests compile until removed.
func applyV1ProcessItemIDs(item V1ProcessJobItem, processedID, rawProductID *uuid.UUID) {
if processedID != nil {
s := processedID.String()
item["id"] = s
item["processed_product_id"] = s
}
if rawProductID != nil {
item["raw_product_id"] = rawProductID.String()
}
_ = item
_ = processedID
_ = rawProductID
}
func withAlwaysIncluded(item V1ProcessJobItem, main string, more []string, eprel any) V1ProcessJobItem {
+23 -32
View File
@@ -72,11 +72,8 @@ func TestMapV1JobItemStatus(t *testing.T) {
func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
items := []V1ProcessJobItem{{
"ean": "123", "id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"status": "processed",
"title": "T", "name": "T", "meta_title": "MT",
"ean": "123", "status": "processed",
"name": "T", "meta_title": "MT",
"description": "D", "attributes": map[string]any{"brand": "X"},
"main_image": "https://example.com/a.jpg", "more_images": []string{"https://example.com/b.jpg"},
"eprel": nil, "category": "cat", "category_name": "Cat",
@@ -88,11 +85,11 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
raw, _ := json.Marshal(out[0])
var got map[string]any
_ = json.Unmarshal(raw, &got)
if got["title"] != "T" || got["ean"] != "123" {
if got["name"] != "T" || got["ean"] != "123" {
t.Fatalf("got=%v", got)
}
if got["name"] != "T" {
t.Fatalf("name dual-mode alias missing or mismatched: %v", got)
if _, ok := got["title"]; ok {
t.Fatalf("duplicate title should be omitted when name exists: %v", got)
}
if _, ok := got["attributes"]; ok {
t.Fatalf("attributes should be projected out: %v", got)
@@ -100,31 +97,29 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
if got["main_image"] != "https://example.com/a.jpg" {
t.Fatalf("images always included: %v", got)
}
if got["status"] != "processed" || got["id"] != "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" {
t.Fatalf("meta should be preserved: %v", got)
if got["status"] != "processed" {
t.Fatalf("status should be preserved: %v", got)
}
if got["processed_product_id"] != "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" {
t.Fatalf("processed_product_id should be preserved: %v", got)
}
if got["raw_product_id"] != "cccccccc-cccc-cccc-cccc-cccccccccccc" {
t.Fatalf("raw_product_id should be preserved: %v", got)
for _, junk := range []string{"id", "processed_product_id", "raw_product_id"} {
if _, ok := got[junk]; ok {
t.Fatalf("%s must be omitted from V1 process items: %v", junk, got)
}
}
}
func TestProjectV1ProcessJobItemsTitleDerivesName(t *testing.T) {
items := []V1ProcessJobItem{{
"ean": "123", "id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"status": "processed", "title": "OnlyTitle", "meta_title": "MT",
"ean": "123", "status": "processed", "title": "OnlyTitle", "meta_title": "MT",
}}
out := ProjectV1ProcessJobItems("title", items)
if len(out) != 1 {
t.Fatalf("len=%d", len(out))
}
if out[0]["title"] != "OnlyTitle" {
t.Fatalf("title=%v", out[0]["title"])
}
if out[0]["name"] != "OnlyTitle" {
t.Fatalf("name should mirror title when absent: %v", out[0]["name"])
t.Fatalf("name should derive from title when absent: %v", out[0]["name"])
}
if _, ok := out[0]["title"]; ok {
t.Fatalf("title omitted when identical to name: %v", out[0])
}
}
@@ -180,22 +175,18 @@ func TestApplyV1ProcessItemIDsDualMode(t *testing.T) {
raw := mustParseTestUUID(t, "cccccccc-cccc-cccc-cccc-cccccccccccc")
item := V1ProcessJobItem{"ean": "1", "status": "processed"}
applyV1ProcessItemIDs(item, &processed, &raw)
if item["id"] != processed.String() {
t.Fatalf("id=%v", item["id"])
}
if item["processed_product_id"] != processed.String() {
t.Fatalf("processed_product_id=%v", item["processed_product_id"])
}
if item["raw_product_id"] != raw.String() {
t.Fatalf("raw_product_id=%v", item["raw_product_id"])
for _, junk := range []string{"id", "processed_product_id", "raw_product_id"} {
if _, ok := item[junk]; ok {
t.Fatalf("%s must stay omitted from V1 process items: %v", junk, item)
}
}
missing := V1ProcessJobItem{"ean": "2", "status": "not_found"}
applyV1ProcessItemIDs(missing, nil, &raw)
if _, ok := missing["id"]; ok {
t.Fatalf("id must stay absent without processed row: %v", missing)
t.Fatalf("id must stay absent: %v", missing)
}
if missing["raw_product_id"] != raw.String() {
t.Fatalf("raw_product_id on not_found: %v", missing)
if _, ok := missing["raw_product_id"]; ok {
t.Fatalf("raw_product_id must stay omitted: %v", missing)
}
}
+23 -25
View File
@@ -63,11 +63,17 @@ func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemO
return sc
}
title := stringFromItem(item, "title")
title := stringFromItem(item, "name")
if title == "" {
title = stringFromItem(item, "title")
}
name := stringFromItem(item, "name")
if name == "" {
name = title
}
sc.HasTitle = title != ""
sc.HasName = name != ""
if sc.HasTitle && !sc.HasName {
if !sc.HasName && sc.HasTitle {
sc.FailFlags = append(sc.FailFlags, "missing_name")
}
if sc.HasTitle && sc.HasName && title != name {
@@ -140,22 +146,8 @@ func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemO
sc.FailFlags = append(sc.FailFlags, "eprel_invalid_shape")
}
id := stringFromItem(item, "id")
ppID := stringFromItem(item, "processed_product_id")
rawID := stringFromItem(item, "raw_product_id")
sc.HasIDs = id != "" && ppID != "" && rawID != ""
if id == "" {
sc.FailFlags = append(sc.FailFlags, "missing_id")
}
if ppID == "" {
sc.FailFlags = append(sc.FailFlags, "missing_processed_product_id")
}
if rawID == "" {
sc.FailFlags = append(sc.FailFlags, "missing_raw_product_id")
}
if id != "" && ppID != "" && id != ppID {
sc.FailFlags = append(sc.FailFlags, "id_processed_product_id_mismatch")
}
// Internal UUIDs are omitted from the public V1 process item shape.
sc.HasIDs = true
sc.ImagesOK = imageFieldsOK(item)
if !sc.ImagesOK {
@@ -234,14 +226,20 @@ func EnforceV1ProcessCompletedItemOpts(item V1ProcessJobItem, opts EnforceV1Opts
item["attributes"] = nil
}
title := ensureReadableTitleSpacing(stringFromItem(item, "title"))
if title != "" {
item["title"] = title
item["name"] = title
} else {
item["title"] = nil
item["name"] = nil
title := ensureReadableTitleSpacing(stringFromItem(item, "name"))
if title == "" {
title = ensureReadableTitleSpacing(stringFromItem(item, "title"))
}
if title != "" {
item["name"] = title
delete(item, "title")
} else {
item["name"] = nil
delete(item, "title")
}
delete(item, "id")
delete(item, "processed_product_id")
delete(item, "raw_product_id")
catID, catName := projectV1CategoryFields(item)
catLabel := catName
@@ -62,7 +62,6 @@ func TestScoreV1ProcessCompletedItem_flagsGaps(t *testing.T) {
}
joined := strings.Join(sc.FailFlags, ",")
for _, want := range []string{
"missing_name",
"description_empty_with_title",
"meta_title_missing",
"meta_description_missing",
@@ -103,7 +102,7 @@ func TestEnforceV1ProcessCompletedItem_fillsDescriptionMetaSanitizes(t *testing.
if desc == "" {
t.Fatal("expected nonempty description")
}
if descriptionEchoesTitle(desc, fmt.Sprint(out["title"])) {
if descriptionEchoesTitle(desc, fmt.Sprint(out["name"])) {
t.Fatalf("EnforceV1 must replace weak/echo desc, got title-echo: %q", desc)
}
if containsWeakFillerPhrase(desc) {
@@ -131,8 +130,16 @@ func TestEnforceV1ProcessCompletedItem_fillsDescriptionMetaSanitizes(t *testing.
if _, bad := attrs["name"]; bad {
t.Fatalf("reserved name kept: %v", attrs)
}
if out["name"] != out["title"] {
t.Fatalf("name should mirror title: name=%v title=%v", out["name"], out["title"])
if out["name"] == nil || strings.TrimSpace(fmt.Sprint(out["name"])) == "" {
t.Fatalf("name missing: %v", out["name"])
}
if _, ok := out["title"]; ok {
t.Fatalf("duplicate title should be omitted when name exists: %v", out["title"])
}
for _, junk := range []string{"id", "processed_product_id", "raw_product_id"} {
if _, ok := out[junk]; ok {
t.Fatalf("%s must be omitted: %v", junk, out)
}
}
sc := ScoreV1ProcessCompletedItem(out, ScoreV1ProcessItemOptions{MappedCategory: "28"})
if !sc.OK {