This commit is contained in:
2026-08-17 09:33:07 +02:00
parent 0dceb3a404
commit fe94c2fb9c
40 changed files with 2360 additions and 340 deletions
+3 -3
View File
@@ -48,11 +48,11 @@ type RepairCategoryEnhancePromptsResult struct {
// RepairA1DemoOptions configures optional seed overlay path for legacy splits.
type RepairA1DemoOptions struct {
// SeedPromptsPath points at a1-category-prompts.json OR wp_product_categories.sql.
// Empty → auto-detect WP SQL (SEED_A1_WP_CATEGORIES / Downloads), else JSON seed.
// Empty → auto-detect WP SQL (SEED_A1_WP_CATEGORIES / scripts/seed), else JSON seed.
SeedPromptsPath string
// WPCategoriesPath forces wp_product_categories.sql (overrides SeedPromptsPath when set).
WPCategoriesPath string
// WPCategoriesSQL is uploaded dump bytes (primary for Admin Sync A1 in prod).
// WPCategoriesSQL is optional dump bytes (legacy Admin Sync A1 upload / API clients).
// When non-empty, takes precedence over filesystem auto-detect / path options.
WPCategoriesSQL []byte
// ForceFromSeed overwrites already-sectioned prompts when a seed match exists
@@ -63,7 +63,7 @@ type RepairA1DemoOptions struct {
// RepairA1DemoCategoryEnhancePrompts replaces non-sectioned / legacy combined
// categories.prompt values for A1 Slovenija + Platform Demo with role-sectioned
// overlays (Title / Description / Meta / Attributes). Prefers wp_product_categories.sql
// (SEED_A1_WP_CATEGORIES / Downloads) as source of truth, then a1-category-prompts.json,
// (SEED_A1_WP_CATEGORIES / scripts/seed) as source of truth, then a1-category-prompts.json,
// splitting combined Name+Description prompts so naming rules land under Title and
// HTML body under Description; otherwise fall back to CategoryEnhanceUserTemplate.
//
@@ -18,8 +18,9 @@ const (
)
// ResolveWPCategoryPromptsPath picks an explicit path, else the first readable
// wp_product_categories.sql candidate (SEED_A1_WP_CATEGORIES + Downloads + scripts/seed).
// Used by Sync A1 / RepairCompanyCategoryEnhancePrompts as the A1 prompt source of truth.
// wp_product_categories.sql candidate (SEED_A1_WP_CATEGORIES, then scripts/seed,
// then optional local Downloads as last-resort). Used by Sync A1 /
// RepairCompanyCategoryEnhancePrompts as the A1 prompt source of truth — no UI upload required.
func ResolveWPCategoryPromptsPath(explicit string) string {
if p := strings.TrimSpace(explicit); p != "" {
if st, err := os.Stat(p); err == nil && !st.IsDir() {
@@ -31,7 +32,7 @@ func ResolveWPCategoryPromptsPath(explicit string) string {
if st, err := os.Stat(v); err == nil && !st.IsDir() {
return v
}
log.Printf("warning: %s=%q not readable — trying Downloads / scripts/seed", envSeedA1WPCategories, v)
log.Printf("warning: %s=%q not readable — trying scripts/seed", envSeedA1WPCategories, v)
}
for _, c := range WPCategoryPromptsCandidates() {
if st, err := os.Stat(c); err == nil && !st.IsDir() {
@@ -42,7 +43,8 @@ func ResolveWPCategoryPromptsPath(explicit string) string {
}
// WPCategoryPromptsCandidates lists local paths Sync A1 / repair try when
// SEED_A1_WP_CATEGORIES is unset. First readable file wins via ResolveWPCategoryPromptsPath.
// SEED_A1_WP_CATEGORIES is unset. Prefer committed scripts/seed first; Downloads
// is last-resort only. First readable file wins via ResolveWPCategoryPromptsPath.
func WPCategoryPromptsCandidates() []string {
var out []string
if v := strings.TrimSpace(os.Getenv(envSeedA1WPCategories)); v != "" {
@@ -53,13 +55,27 @@ func WPCategoryPromptsCandidates() []string {
"wp_product_categories (1).sql",
"wp_product_categories(1).sql",
}
// Repo seed is the default source of truth (no upload / Downloads required).
if root, ok := findMonorepoRootFromCwd(); ok {
for _, n := range names {
out = append(out, filepath.Join(root, "scripts", "seed", n))
}
}
for _, n := range names {
out = append(out,
filepath.Join("scripts", "seed", n),
filepath.Join("..", "..", "scripts", "seed", n),
n,
filepath.Join("..", "..", n),
)
}
// Optional local Downloads fallback (legacy / one-off machines without seed).
home, _ := os.UserHomeDir()
if home != "" {
for _, n := range names {
out = append(out, filepath.Join(home, "Downloads", n))
out = append(out, filepath.Join(home, "downloads", n))
}
// Windows secondary profile Downloads (e.g. D:\Users\…\Downloads).
for _, driveRoot := range []string{`D:\`, `C:\`} {
alt := filepath.Join(driveRoot, "Users", filepath.Base(home), "Downloads")
for _, n := range names {
@@ -67,19 +83,6 @@ func WPCategoryPromptsCandidates() []string {
}
}
}
for _, n := range names {
out = append(out,
n,
filepath.Join("..", "..", n),
filepath.Join("scripts", "seed", n),
filepath.Join("..", "..", "scripts", "seed", n),
)
}
if root, ok := findMonorepoRootFromCwd(); ok {
for _, n := range names {
out = append(out, filepath.Join(root, "scripts", "seed", n))
}
}
return out
}
@@ -81,14 +81,10 @@ func TestParseWPProductCategoriesSQL_SlusalkeSplit(t *testing.T) {
}
}
func TestParseWPProductCategoriesSQL_RealDownloadsFile(t *testing.T) {
path := ResolveWPCategoryPromptsPath(`d:\Users\Green Eclipse\Downloads\wp_product_categories.sql`)
func TestParseWPProductCategoriesSQL_RealSeedFile(t *testing.T) {
path := ResolveWPCategoryPromptsPath("")
if path == "" {
// Also try env / auto-detect without failing CI machines that lack the dump.
path = ResolveWPCategoryPromptsPath("")
}
if path == "" {
t.Skip("wp_product_categories.sql not available on this machine")
t.Skip("wp_product_categories.sql not available (expected under scripts/seed)")
}
byNorm, _, err := loadWPCategoryPromptOverlays(path)
if err != nil {
@@ -109,6 +105,10 @@ func TestParseWPProductCategoriesSQL_RealDownloadsFile(t *testing.T) {
if !strings.Contains(split, "tip izdelka lowercase") && !strings.Contains(split, "znamka") {
t.Fatalf("unexpected Title rules in split: %s", split[:min(400, len(split))])
}
if !strings.Contains(filepath.ToSlash(path), "scripts/seed/wp_product_categories.sql") &&
filepath.Base(path) != "wp_product_categories.sql" {
t.Logf("resolved path %s (prefer scripts/seed when present)", path)
}
}
func TestResolveWPCategoryPromptsPath_Explicit(t *testing.T) {
+55 -11
View File
@@ -32,9 +32,9 @@ func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
}
// heuristicSynthesizePhrases are distinctive invent / formula-skeleton snippets from
// synthesizeDescriptionFromTitle / synthesizeDescriptionFromFormula. These may look
// "strong" enough to pass IsWeakPriorEnhanceDescription but must never hash-skip
// enhance (would leave fallback copy forever on reprocess).
// synthesizeDescriptionFromTitle / synthesizeDescriptionFromFormula / factualDescriptionIntro.
// These may look "strong" enough to pass IsWeakPriorEnhanceDescription but must never
// hash-skip enhance (would leave fallback copy forever on reprocess).
var heuristicSynthesizePhrases = []string{
"is a catalog product with the known attributes",
"is listed in the ",
@@ -43,23 +43,63 @@ var heuristicSynthesizePhrases = []string{
"je izdelek v kategoriji",
"je izdelek znamke",
". ključne specifikacije:",
// factualDescriptionIntro (post-phrase-avoidance invent)
" — znamka ",
", znamka ",
" — katalogski izdelek",
" — from ",
" — catalog product",
}
// LooksLikeHeuristicSynthesize reports invent / formula-skeleton fallback copy.
func LooksLikeHeuristicSynthesize(desc string) bool {
lower := strings.ToLower(desc)
plain := strings.ToLower(plainTextForWeakCheck(desc))
if plain == "" {
return false
}
for _, p := range heuristicSynthesizePhrases {
if p != "" && strings.Contains(lower, p) {
if p != "" && strings.Contains(plain, p) {
return true
}
}
// EN invent: "<title> is a <category> product from <brand>"
if strings.Contains(lower, " is a ") && strings.Contains(lower, " product from ") {
if strings.Contains(plain, " is a ") && strings.Contains(plain, " product from ") {
return true
}
// SL/CS short invent: "<title> je … znamk|kategor|produkt …"
if strings.Contains(plain, " je ") &&
(strings.Contains(plain, "znamk") ||
strings.Contains(plain, "kategor") ||
strings.Contains(plain, "produkt") ||
strings.Contains(plain, "televiz") ||
strings.Contains(plain, "monitor")) {
return true
}
return false
}
// plainTextForWeakCheck strips HTML tags so <p>thin invent</p> is judged on visible copy.
func plainTextForWeakCheck(s string) string {
s = strings.TrimSpace(s)
if s == "" {
return ""
}
var b strings.Builder
b.Grow(len(s))
inTag := false
for _, r := range s {
switch {
case r == '<':
inTag = true
case r == '>':
inTag = false
case !inTag:
b.WriteRune(r)
}
}
return strings.TrimSpace(b.String())
}
// EnhanceHashSkipBlockReason returns a stable reason when prior description must
// not hash-skip the enhance LLM: "weak", "title-echo", "synth", or "".
func EnhanceHashSkipBlockReason(priorDesc string, titles ...string) string {
@@ -67,22 +107,26 @@ func EnhanceHashSkipBlockReason(priorDesc string, titles ...string) string {
if priorDesc == "" || priorDesc == "<nil>" {
return "weak"
}
if len([]rune(priorDesc)) < minUsableProductDescRunes {
plain := plainTextForWeakCheck(priorDesc)
if plain == "" || plain == "<nil>" {
return "weak"
}
if len([]rune(plain)) < minUsableProductDescRunes {
return "weak"
}
for _, title := range titles {
if DescriptionEchoesTitle(priorDesc, title) {
if DescriptionEchoesTitle(plain, title) || DescriptionEchoesTitle(priorDesc, title) {
return "title-echo"
}
}
if ContainsWeakFillerPhrase(priorDesc) {
if ContainsWeakFillerPhrase(plain) || ContainsWeakFillerPhrase(priorDesc) {
return "weak"
}
if LooksLikeHeuristicSynthesize(priorDesc) {
return "synth"
}
if len([]rune(priorDesc)) <= shortBoilerplateDescRunes &&
!descriptionOverlapsProductFacts(priorDesc, titles...) {
if len([]rune(plain)) <= shortBoilerplateDescRunes &&
!descriptionOverlapsProductFacts(plain, titles...) {
return "weak"
}
return ""
@@ -13,6 +13,17 @@ func TestLooksLikeHeuristicSynthesize(t *testing.T) {
if !LooksLikeHeuristicSynthesize(sl) {
t.Fatalf("expected SL invent detected: %q", sl)
}
htmlInvent := "<p>QLED TV 55 je televizor znamke Samsung.</p>"
if !LooksLikeHeuristicSynthesize(htmlInvent) {
t.Fatalf("expected HTML-wrapped invent detected: %q", htmlInvent)
}
if !ShouldRefuseEnhanceHashSkip(htmlInvent, "QLED TV 55") {
t.Fatalf("HTML invent must block hash skip")
}
factual := "QLED TV 55 — Televizorji, znamka Samsung."
if !LooksLikeHeuristicSynthesize(factual) {
t.Fatalf("expected factualDescriptionIntro invent detected: %q", factual)
}
good := "Vogel's WALL 3245 is a sturdy TV mount with clear load ratings and install guidance for wall displays."
if LooksLikeHeuristicSynthesize(good) {
t.Fatalf("retail copy must not look like invent: %q", good)
@@ -27,6 +27,7 @@ func TestMemberForbiddenOnSensitiveMutations(t *testing.T) {
body string
}{
{name: "create_api_key", fn: s.handleCreateAPIKey, body: `{"name":"x"}`},
{name: "update_api_key", fn: s.handleUpdateAPIKey, body: `{"name":"renamed"}`},
{name: "revoke_api_key", fn: s.handleRevokeAPIKey, body: ""},
{name: "put_email", fn: s.handlePutEmailIntegration, body: `{}`},
{name: "verify_email", fn: s.handleVerifyEmailIntegration, body: ""},
@@ -29,11 +29,13 @@ const syncA1MaxBodyBytes = catalog.MaxWPCategorySQLBytes + (2 << 20)
//
// Body (JSON): confirm=true required; backfill_categories (default true);
// reprocess_sample_limit (default 25, max 200); mysql_dump optional path;
// skip_dump_backfill (default false); wp_product_categories_sql_b64 optional
// base64 of wp_product_categories.sql (primary for category prompts).
// skip_dump_backfill (default false). Category prompts load from repo seed
// (scripts/seed/wp_product_categories.sql) via ResolveWPCategoryPromptsPath;
// wp_product_categories_sql_b64 remains accepted but unused by the admin UI.
//
// Body (multipart/form-data): same fields as form values; file field
// wp_product_categories or wp_categories_sql for the SQL dump upload.
// Body (multipart/form-data): same fields as form values; optional file field
// wp_product_categories / wp_categories_sql is still accepted for API clients
// but the admin Sync A1 UI no longer uploads.
//
// Flash (UI): result → flash.admin.syncA1Success.
func (s *Server) handleAdminSyncCompanyA1(w http.ResponseWriter, r *http.Request) {
@@ -116,8 +118,10 @@ func (s *Server) handleAdminSyncCompanyA1(w http.ResponseWriter, r *http.Request
}
if len(parsed.WPCategoriesSQL) > 0 {
note = "Applied uploaded wp_product_categories.sql (Title/Description/Meta/Attributes sections) + catalog hygiene. Reprocess recommended products manually (no mass reprocess)."
} else if strings.TrimSpace(result.WPCategoriesPath) == "" {
note += " Upload wp_product_categories.sql on Sync A1 for force-applied category prompts when the dump is not on the API host."
} else if strings.TrimSpace(result.WPCategoriesPath) != "" {
note = "Applied category prompts from repo seed (" + result.WPCategoriesPath + ") + catalog hygiene. Reprocess recommended products manually (no mass reprocess)."
} else {
note += " Place scripts/seed/wp_product_categories.sql on the API host (or set SEED_A1_WP_CATEGORIES) for force-applied category prompts."
}
JSON(w, http.StatusOK, map[string]any{
@@ -1,11 +1,14 @@
package httpapi
import (
"errors"
"net/http"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/auth"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
func (s *Server) handleListAPIKeys(w http.ResponseWriter, r *http.Request) {
@@ -80,6 +83,61 @@ func (s *Server) handleCreateAPIKey(w http.ResponseWriter, r *http.Request) {
})
}
func (s *Server) handleUpdateAPIKey(w http.ResponseWriter, r *http.Request) {
cid, _ := CompanyIDFromContext(r.Context())
if !s.allowCompanyAdminOrPlatform(w, r) {
return
}
if !s.requireFeatures(w, r, "settings.api_keys", "capability.api_access") {
return
}
id, err := uuid.Parse(chi.URLParam(r, "id"))
if err != nil {
Error(w, http.StatusBadRequest, "invalid id")
return
}
var body struct {
Name string `json:"name"`
}
if err := DecodeJSON(r, &body); err != nil {
Error(w, http.StatusBadRequest, "invalid json")
return
}
name := strings.TrimSpace(body.Name)
if name == "" {
Error(w, http.StatusBadRequest, "name required")
return
}
if len(name) > 120 {
Error(w, http.StatusBadRequest, "name too long")
return
}
var (
outID uuid.UUID
outName string
prefix string
lastUsed any
created any
)
err = s.Pool.QueryRow(r.Context(), `
UPDATE api_keys
SET name = $3, updated_at = now()
WHERE id = $1 AND company_id = $2 AND revoked_at IS NULL
RETURNING id, name, key_prefix, last_used_at, created_at`,
id, cid, name).Scan(&outID, &outName, &prefix, &lastUsed, &created)
if errors.Is(err, pgx.ErrNoRows) {
Error(w, http.StatusNotFound, "not found")
return
}
if err != nil {
Error(w, http.StatusInternalServerError, "update failed")
return
}
JSON(w, http.StatusOK, map[string]any{
"id": outID, "name": outName, "key_prefix": prefix, "last_used_at": lastUsed, "created_at": created,
})
}
func (s *Server) handleRevokeAPIKey(w http.ResponseWriter, r *http.Request) {
cid, _ := CompanyIDFromContext(r.Context())
if !s.allowCompanyAdminOrPlatform(w, r) {
@@ -0,0 +1,57 @@
package httpapi
import (
"bytes"
"context"
"net/http"
"net/http/httptest"
"testing"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
)
func TestHandleUpdateAPIKeyRejectsEmptyName(t *testing.T) {
t.Parallel()
s := &Server{}
cid := uuid.MustParse("11111111-1111-1111-1111-111111111111")
uid := uuid.MustParse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
keyID := uuid.MustParse("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb")
ctx := context.WithValue(context.Background(), ctxUserID, uid)
ctx = context.WithValue(ctx, ctxCompanyID, cid)
ctx = context.WithValue(ctx, ctxRole, "admin")
rctx := chi.NewRouteContext()
rctx.URLParams.Add("id", keyID.String())
ctx = context.WithValue(ctx, chi.RouteCtxKey, rctx)
req := httptest.NewRequest(http.MethodPatch, "/api/api-keys/"+keyID.String(), bytes.NewBufferString(`{"name":" "}`)).WithContext(ctx)
rec := httptest.NewRecorder()
s.handleUpdateAPIKey(rec, req)
if rec.Code != http.StatusBadRequest {
t.Fatalf("status = %d body=%s, want 400", rec.Code, rec.Body.String())
}
}
func TestHandleUpdateAPIKeyRejectsInvalidID(t *testing.T) {
t.Parallel()
s := &Server{}
cid := uuid.MustParse("11111111-1111-1111-1111-111111111111")
uid := uuid.MustParse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
ctx := context.WithValue(context.Background(), ctxUserID, uid)
ctx = context.WithValue(ctx, ctxCompanyID, cid)
ctx = context.WithValue(ctx, ctxRole, "admin")
rctx := chi.NewRouteContext()
rctx.URLParams.Add("id", "not-a-uuid")
ctx = context.WithValue(ctx, chi.RouteCtxKey, rctx)
req := httptest.NewRequest(http.MethodPatch, "/api/api-keys/not-a-uuid", bytes.NewBufferString(`{"name":"ok"}`)).WithContext(ctx)
rec := httptest.NewRecorder()
s.handleUpdateAPIKey(rec, req)
if rec.Code != http.StatusBadRequest {
t.Fatalf("status = %d body=%s, want 400", rec.Code, rec.Body.String())
}
}
@@ -247,6 +247,15 @@ func TestAuthSessionCoreEndpoints(t *testing.T) {
t.Fatalf("create api-key payload=%v", created)
}
rec = do(http.MethodPatch, "/api/api-keys/"+keyID, `{"name":"auth-smoke-key-renamed"}`, true)
if rec.Code != http.StatusOK {
t.Fatalf("rename api-key status=%d body=%s", rec.Code, rec.Body.String())
}
renamed := decode(t, rec)
if fmt.Sprint(renamed["name"]) != "auth-smoke-key-renamed" {
t.Fatalf("rename api-key name=%v", renamed["name"])
}
// Public v1 with the new key (CSRF skipped).
v1 := httptest.NewRecorder()
v1Req := httptest.NewRequest(http.MethodGet, "/api/v1/products?limit=1", nil)
+1
View File
@@ -453,6 +453,7 @@ func (s *Server) Router() http.Handler {
r.Get("/api-keys", s.handleListAPIKeys)
r.Post("/api-keys", s.handleCreateAPIKey)
r.Patch("/api-keys/{id}", s.handleUpdateAPIKey)
r.Delete("/api-keys/{id}", s.handleRevokeAPIKey)
r.Get("/billing/credits", s.handleCreditsOverview)
+37 -51
View File
@@ -118,8 +118,9 @@ info:
- GET /products data[].id = processed_products.id (enriched row)
- GET /products data[].raw_product_id = raw_products.id (use this for raw_product_ids)
- POST ... raw_product_ids[] must be raw_products.id — never PresentProduct.id
- GET /products/process/{id} COMPLETED items[].id = processed_products.id (legacy);
additive processed_product_id (same as id) and raw_product_id (raw_products.id)
- GET /products/process/{id} COMPLETED items[] omit internal UUIDs (id /
processed_product_id / raw_product_id). Use GET /products when a UUID is needed.
Display name is items[].name (title omitted when identical).
Note: Dashboard JSON under /api/* uses session cookies + CSRF and is separate
from this public API-key surface. Other legacy path aliases
@@ -895,9 +896,10 @@ paths:
Completed jobs return items[] (EAN-keyed enrichment). In-progress and failed
jobs omit items. Not the same shape as GET /process/{id} (flat ProcessingJob).
On COMPLETED items, id is the processed_products UUID (legacy). Additive aliases:
processed_product_id (same value as id), raw_product_id (raw_products.id), and
name (same value as title) for dual-mode clients / scorecards.
On COMPLETED items, name is the product display name (title is omitted when
identical). Internal UUIDs (id / processed_product_id / raw_product_id) are
omitted — use GET /products for those. A1 cohort / Platform Demo / A1-prompt
companies omit meta_title and meta_description even if stored.
parameters:
- name: id
in: path
@@ -925,17 +927,11 @@ paths:
processing_type: full
items:
- ean: '8606019604493'
id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
processed_product_id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
raw_product_id: cccccccc-cccc-cccc-cccc-cccccccccccc
status: processed
category: Cookers
category_id: '50'
category_name: Cookers
title: VOX electric cooker EHT 6020 WG
name: VOX electric cooker EHT 6020 WG
meta_title: VOX electric cooker EHT 6020 WG | 50
meta_description: Affordable electric cooker with four plates and a 65 L fan oven.
description: "Vox Electronics electric cooker EHT 6020 WG offers strong value with four electric hobs and a 65 L fan oven."
attributes:
brand: Vox
@@ -6207,12 +6203,24 @@ components:
process_id:
type: string
format: uuid
status:
type: string
description: |
Job lifecycle status on enqueue. Always pending (or processing if the
worker already picked it up). Never COMPLETED on start — poll GET
/products/process/{id} for completion.
example: pending
message:
type: string
total_items:
type: integer
description: Number of products accepted into the job (enqueue size).
processed_items:
type: integer
description: |
Count of items that finished processing. Always 0 on start while the
job is pending/processing; increments as the worker completes products.
example: 0
job_count:
type: integer
description: Present when StartJob auto-splits
@@ -6282,35 +6290,19 @@ components:
expose category as the human-readable display name (category_id holds
categories.unique_id), a description string that may include category formula
HTML (h1/h2/h3/h4, p, ul — never a JSON array), optional SEO meta_title /
meta_description (plain text; omitted for A1 cohort), optional eprel object or
null, clean attributes, images, and dual-mode ids.
Product display name is title; additive name mirrors the same processed title
(dual-mode for scorecards / legacy clients that read name).
Property order prefers human-readable fields first (ean, title, category,
description, attributes, images, eprel) with internal ids last.
meta_description (plain text; omitted for A1 cohort, Platform Demo, and any
company with A1-style category role-section prompts — even if stored in DB),
optional eprel object or null, clean attributes, and images.
Product display name is name (primary). title is omitted when identical to name.
Internal UUIDs (id, processed_product_id, raw_product_id) are omitted from this
public shape — use catalog APIs when a product UUID is required.
Property order prefers human-readable fields first (ean, name, category,
description, attributes, images, eprel).
required:
- ean
properties:
ean:
type: string
id:
type: string
format: uuid
description: |
Legacy field: processed_products.id when enrichment succeeded.
Do not treat as raw_products.id. Same value as processed_product_id.
processed_product_id:
type: string
format: uuid
description: |
Explicit alias of id (processed_products.id). Prefer this name in new
dual-mode clients; id remains for backward compatibility.
raw_product_id:
type: string
format: uuid
description: |
raw_products.id for this job line. Use with POST /process raw_product_ids
or dashboard catalog APIs. Present whenever the job product row exists.
category:
type: string
nullable: true
@@ -6330,32 +6322,32 @@ components:
nullable: true
description: |
Human-readable category display name (mirrors category when both are set).
title:
type: string
nullable: true
description: |
Product display name (processed title). Primary legacy field; same value
as name when present.
name:
type: string
nullable: true
description: |
Additive alias of title (same processed display name). Prefer title in
new clients; name remains for scorecards and legacy readers.
Product display name (processed title). Primary field for clients.
title:
type: string
nullable: true
description: |
Optional legacy alias of name. Omitted when identical to name.
meta_title:
type: string
nullable: true
description: |
SEO title. Filled from processing meta or synthesized from title / category
when empty so successful items are not left with null meta.
Omitted for A1 cohort (SEO meta is not used).
Omitted for A1 cohort, Platform Demo, and companies with A1-style category
prompts (SEO meta is not used).
meta_description:
type: string
nullable: true
description: |
SEO description (word-safe truncate). Distinct from body description when
possible; synthesized from plain description when DB meta is empty.
Omitted for A1 cohort (SEO meta is not used).
Omitted for A1 cohort, Platform Demo, and companies with A1-style category
prompts (SEO meta is not used).
description:
type: string
nullable: true
@@ -6477,17 +6469,11 @@ components:
processing_type: full
items:
- ean: '8606019604493'
id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
processed_product_id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
raw_product_id: cccccccc-cccc-cccc-cccc-cccccccccccc
status: processed
category: Cookers
category_id: '50'
category_name: Cookers
title: VOX electric cooker EHT 6020 WG
name: VOX electric cooker EHT 6020 WG
meta_title: VOX electric cooker EHT 6020 WG | 50
meta_description: Affordable electric cooker with four plates and a 65 L fan oven.
description: "Vox Electronics electric cooker EHT 6020 WG offers strong value with four electric hobs and a 65 L fan oven. Energy class A with practical everyday capacity."
attributes:
brand: Vox
@@ -160,11 +160,14 @@ func (s *Server) handleV1StartProcess(w http.ResponseWriter, r *http.Request) {
}
primary := jobs[0]
// processed_items is the completed count — stay 0 until the worker finishes.
// total_items is the enqueue size; status stays pending/processing until poll COMPLETED.
resp := map[string]any{
"process_id": primary.ID.String(),
"status": "pending",
"message": fmt.Sprintf("Processing started for %d product(s)", len(rawIDs)),
"total_items": totalItems,
"processed_items": len(rawIDs),
"processed_items": 0,
}
if len(jobs) > 1 {
siblings := make([]string, 0, len(jobs)-1)
@@ -194,6 +194,7 @@ func TestHandleV1StartProcessItemsEANReturnsProcessID(t *testing.T) {
var envelope struct {
Data struct {
ProcessID string `json:"process_id"`
Status string `json:"status"`
Message string `json:"message"`
TotalItems int `json:"total_items"`
ProcessedItems int `json:"processed_items"`
@@ -205,8 +206,11 @@ func TestHandleV1StartProcessItemsEANReturnsProcessID(t *testing.T) {
if envelope.Data.ProcessID != jobID.String() {
t.Fatalf("process_id=%q want %s", envelope.Data.ProcessID, jobID)
}
if envelope.Data.TotalItems != 1 || envelope.Data.ProcessedItems != 1 {
t.Fatalf("counts=%+v", envelope.Data)
if envelope.Data.Status != "pending" {
t.Fatalf("status=%q want pending", envelope.Data.Status)
}
if envelope.Data.TotalItems != 1 || envelope.Data.ProcessedItems != 0 {
t.Fatalf("counts=%+v (processed_items must be 0 on start)", envelope.Data)
}
if enqueued != jobID {
t.Fatalf("enqueued=%s", enqueued)
+3 -4
View File
@@ -528,11 +528,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}
}
// Pipelines without StepCategorize (enhance_only / normalize_only) still try
// vector categorize when AllowAI + embeddings are available. Full/categorize
// already ran runCategorizeStep (vector then LLM) inside the loop.
// Pipelines without StepCategorize (enhance_only / normalize_only) still run
// vector + LLM categorize when category is empty so enhance is not fed a blank.
if !stepsContain(steps, StepCategorize) {
tryVectorCategorize(ctx, e, companyID, &out, categoryNames, policy)
runCategorizeStep(ctx, e, companyID, &out, in, categoryNames, policy)
noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled())
}
preserveCategoryIfEmpty(&out, in.PriorCategory)
+48 -26
View File
@@ -261,6 +261,8 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
); err != nil {
return nil, err
}
_ = processedID
_ = rawProductID
if processedID == nil {
st := MapV1JobItemStatus(itemStatus, false)
if st == "processed" || st == "processing" || st == "pending" {
@@ -274,7 +276,6 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
if itemError != nil && *itemError != "" {
item["error"] = *itemError
}
applyV1ProcessItemIDs(item, nil, rawProductID)
full = append(full, item)
continue
}
@@ -428,9 +429,9 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
catIDOut = catStr
}
var titleOut any
var nameOut any
if titleStr != "" {
titleOut = titleStr
nameOut = titleStr
}
item := V1ProcessJobItem{
"ean": ean,
@@ -438,8 +439,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
"category": catNameOut,
"category_id": catIDOut,
"category_name": catNameOut,
"title": titleOut,
"name": titleOut,
"name": nameOut,
"description": description,
"attributes": nil,
"main_image": nil,
@@ -450,7 +450,8 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
item["meta_title"] = metaTitleOut
item["meta_description"] = metaDescOut
}
applyV1ProcessItemIDs(item, processedID, rawProductID)
// Internal ids (id / processed_product_id / raw_product_id) are omitted from
// the public V1 process item shape — use catalog APIs when a UUID is needed.
if itemError != nil && *itemError != "" {
item["error"] = *itemError
}
@@ -483,20 +484,41 @@ func nullIfEmptyPtr(s *string) any {
return *s
}
// companyOmitsSEOMeta is true for the A1 cohort (no meta_title / meta_description).
// companyOmitsSEOMeta is true when V1/process must omit meta_title / meta_description:
// A1 cohort (legacy id), Platform Demo (A1-cloned prompts, no legacy id), or any
// company whose categories store A1-style role-section enhance prompts.
func companyOmitsSEOMeta(ctx context.Context, p *Pipeline, companyID uuid.UUID) bool {
if p == nil || p.Pool == nil {
return false
}
var legacy string
var legacy, name string
err := p.Pool.QueryRow(ctx, `
SELECT COALESCE(legacy_company_id, '')
SELECT COALESCE(legacy_company_id, ''), COALESCE(name, '')
FROM companies
WHERE id = $1`, companyID).Scan(&legacy)
WHERE id = $1`, companyID).Scan(&legacy, &name)
if err != nil {
return false
}
return billing.IsA1CohortCompany(legacy, "")
if billing.IsA1CohortCompany(legacy, "") {
return true
}
if strings.EqualFold(strings.TrimSpace(name), "Platform Demo") {
return true
}
var hasA1Prompts bool
err = p.Pool.QueryRow(ctx, `
SELECT EXISTS (
SELECT 1 FROM categories
WHERE company_id = $1
AND prompt ILIKE '%--- Title ---%'
AND prompt ILIKE '%--- Description ---%'
AND prompt ILIKE '%--- Meta ---%'
LIMIT 1
)`, companyID).Scan(&hasA1Prompts)
if err != nil {
return false
}
return hasA1Prompts
}
func derefStringPtr(s *string) string {
@@ -724,10 +746,14 @@ func ProjectV1ProcessJobItems(storedType string, items []V1ProcessJobItem) []V1P
projected["category_id"] = item["category_id"]
projected["category_name"] = item["category_name"]
case "title":
projected["title"] = item["title"]
projected["name"] = item["name"]
if projected["name"] == nil {
projected["name"] = item["title"]
name := item["name"]
if name == nil {
name = item["title"]
}
projected["name"] = name
// Omit duplicate title when it matches name (name is primary).
if title := item["title"]; title != nil && title != name {
projected["title"] = title
}
if _, ok := item["meta_title"]; ok {
projected["meta_title"] = item["meta_title"]
@@ -747,7 +773,7 @@ func ProjectV1ProcessJobItems(storedType string, items []V1ProcessJobItem) []V1P
}
func withItemMeta(dst, src V1ProcessJobItem) V1ProcessJobItem {
for _, k := range []string{"status", "error", "id", "processed_product_id", "raw_product_id"} {
for _, k := range []string{"status", "error"} {
if v, ok := src[k]; ok {
dst[k] = v
}
@@ -755,17 +781,13 @@ func withItemMeta(dst, src V1ProcessJobItem) V1ProcessJobItem {
return dst
}
// applyV1ProcessItemIDs sets legacy id (= processed UUID) plus additive dual-mode aliases.
// id is preserved for existing integrators; processed_product_id mirrors it; raw_product_id is raw_products.id.
// applyV1ProcessItemIDs formerly stamped id / processed_product_id / raw_product_id onto
// V1 process items. Those fields are omitted from the public contract; kept as a no-op
// so older call sites/tests compile until removed.
func applyV1ProcessItemIDs(item V1ProcessJobItem, processedID, rawProductID *uuid.UUID) {
if processedID != nil {
s := processedID.String()
item["id"] = s
item["processed_product_id"] = s
}
if rawProductID != nil {
item["raw_product_id"] = rawProductID.String()
}
_ = item
_ = processedID
_ = rawProductID
}
func withAlwaysIncluded(item V1ProcessJobItem, main string, more []string, eprel any) V1ProcessJobItem {
+23 -32
View File
@@ -72,11 +72,8 @@ func TestMapV1JobItemStatus(t *testing.T) {
func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
items := []V1ProcessJobItem{{
"ean": "123", "id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"status": "processed",
"title": "T", "name": "T", "meta_title": "MT",
"ean": "123", "status": "processed",
"name": "T", "meta_title": "MT",
"description": "D", "attributes": map[string]any{"brand": "X"},
"main_image": "https://example.com/a.jpg", "more_images": []string{"https://example.com/b.jpg"},
"eprel": nil, "category": "cat", "category_name": "Cat",
@@ -88,11 +85,11 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
raw, _ := json.Marshal(out[0])
var got map[string]any
_ = json.Unmarshal(raw, &got)
if got["title"] != "T" || got["ean"] != "123" {
if got["name"] != "T" || got["ean"] != "123" {
t.Fatalf("got=%v", got)
}
if got["name"] != "T" {
t.Fatalf("name dual-mode alias missing or mismatched: %v", got)
if _, ok := got["title"]; ok {
t.Fatalf("duplicate title should be omitted when name exists: %v", got)
}
if _, ok := got["attributes"]; ok {
t.Fatalf("attributes should be projected out: %v", got)
@@ -100,31 +97,29 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
if got["main_image"] != "https://example.com/a.jpg" {
t.Fatalf("images always included: %v", got)
}
if got["status"] != "processed" || got["id"] != "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" {
t.Fatalf("meta should be preserved: %v", got)
if got["status"] != "processed" {
t.Fatalf("status should be preserved: %v", got)
}
if got["processed_product_id"] != "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" {
t.Fatalf("processed_product_id should be preserved: %v", got)
}
if got["raw_product_id"] != "cccccccc-cccc-cccc-cccc-cccccccccccc" {
t.Fatalf("raw_product_id should be preserved: %v", got)
for _, junk := range []string{"id", "processed_product_id", "raw_product_id"} {
if _, ok := got[junk]; ok {
t.Fatalf("%s must be omitted from V1 process items: %v", junk, got)
}
}
}
func TestProjectV1ProcessJobItemsTitleDerivesName(t *testing.T) {
items := []V1ProcessJobItem{{
"ean": "123", "id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"status": "processed", "title": "OnlyTitle", "meta_title": "MT",
"ean": "123", "status": "processed", "title": "OnlyTitle", "meta_title": "MT",
}}
out := ProjectV1ProcessJobItems("title", items)
if len(out) != 1 {
t.Fatalf("len=%d", len(out))
}
if out[0]["title"] != "OnlyTitle" {
t.Fatalf("title=%v", out[0]["title"])
}
if out[0]["name"] != "OnlyTitle" {
t.Fatalf("name should mirror title when absent: %v", out[0]["name"])
t.Fatalf("name should derive from title when absent: %v", out[0]["name"])
}
if _, ok := out[0]["title"]; ok {
t.Fatalf("title omitted when identical to name: %v", out[0])
}
}
@@ -180,22 +175,18 @@ func TestApplyV1ProcessItemIDsDualMode(t *testing.T) {
raw := mustParseTestUUID(t, "cccccccc-cccc-cccc-cccc-cccccccccccc")
item := V1ProcessJobItem{"ean": "1", "status": "processed"}
applyV1ProcessItemIDs(item, &processed, &raw)
if item["id"] != processed.String() {
t.Fatalf("id=%v", item["id"])
}
if item["processed_product_id"] != processed.String() {
t.Fatalf("processed_product_id=%v", item["processed_product_id"])
}
if item["raw_product_id"] != raw.String() {
t.Fatalf("raw_product_id=%v", item["raw_product_id"])
for _, junk := range []string{"id", "processed_product_id", "raw_product_id"} {
if _, ok := item[junk]; ok {
t.Fatalf("%s must stay omitted from V1 process items: %v", junk, item)
}
}
missing := V1ProcessJobItem{"ean": "2", "status": "not_found"}
applyV1ProcessItemIDs(missing, nil, &raw)
if _, ok := missing["id"]; ok {
t.Fatalf("id must stay absent without processed row: %v", missing)
t.Fatalf("id must stay absent: %v", missing)
}
if missing["raw_product_id"] != raw.String() {
t.Fatalf("raw_product_id on not_found: %v", missing)
if _, ok := missing["raw_product_id"]; ok {
t.Fatalf("raw_product_id must stay omitted: %v", missing)
}
}
+23 -25
View File
@@ -63,11 +63,17 @@ func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemO
return sc
}
title := stringFromItem(item, "title")
title := stringFromItem(item, "name")
if title == "" {
title = stringFromItem(item, "title")
}
name := stringFromItem(item, "name")
if name == "" {
name = title
}
sc.HasTitle = title != ""
sc.HasName = name != ""
if sc.HasTitle && !sc.HasName {
if !sc.HasName && sc.HasTitle {
sc.FailFlags = append(sc.FailFlags, "missing_name")
}
if sc.HasTitle && sc.HasName && title != name {
@@ -140,22 +146,8 @@ func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemO
sc.FailFlags = append(sc.FailFlags, "eprel_invalid_shape")
}
id := stringFromItem(item, "id")
ppID := stringFromItem(item, "processed_product_id")
rawID := stringFromItem(item, "raw_product_id")
sc.HasIDs = id != "" && ppID != "" && rawID != ""
if id == "" {
sc.FailFlags = append(sc.FailFlags, "missing_id")
}
if ppID == "" {
sc.FailFlags = append(sc.FailFlags, "missing_processed_product_id")
}
if rawID == "" {
sc.FailFlags = append(sc.FailFlags, "missing_raw_product_id")
}
if id != "" && ppID != "" && id != ppID {
sc.FailFlags = append(sc.FailFlags, "id_processed_product_id_mismatch")
}
// Internal UUIDs are omitted from the public V1 process item shape.
sc.HasIDs = true
sc.ImagesOK = imageFieldsOK(item)
if !sc.ImagesOK {
@@ -234,14 +226,20 @@ func EnforceV1ProcessCompletedItemOpts(item V1ProcessJobItem, opts EnforceV1Opts
item["attributes"] = nil
}
title := ensureReadableTitleSpacing(stringFromItem(item, "title"))
if title != "" {
item["title"] = title
item["name"] = title
} else {
item["title"] = nil
item["name"] = nil
title := ensureReadableTitleSpacing(stringFromItem(item, "name"))
if title == "" {
title = ensureReadableTitleSpacing(stringFromItem(item, "title"))
}
if title != "" {
item["name"] = title
delete(item, "title")
} else {
item["name"] = nil
delete(item, "title")
}
delete(item, "id")
delete(item, "processed_product_id")
delete(item, "raw_product_id")
catID, catName := projectV1CategoryFields(item)
catLabel := catName
@@ -62,7 +62,6 @@ func TestScoreV1ProcessCompletedItem_flagsGaps(t *testing.T) {
}
joined := strings.Join(sc.FailFlags, ",")
for _, want := range []string{
"missing_name",
"description_empty_with_title",
"meta_title_missing",
"meta_description_missing",
@@ -103,7 +102,7 @@ func TestEnforceV1ProcessCompletedItem_fillsDescriptionMetaSanitizes(t *testing.
if desc == "" {
t.Fatal("expected nonempty description")
}
if descriptionEchoesTitle(desc, fmt.Sprint(out["title"])) {
if descriptionEchoesTitle(desc, fmt.Sprint(out["name"])) {
t.Fatalf("EnforceV1 must replace weak/echo desc, got title-echo: %q", desc)
}
if containsWeakFillerPhrase(desc) {
@@ -131,8 +130,16 @@ func TestEnforceV1ProcessCompletedItem_fillsDescriptionMetaSanitizes(t *testing.
if _, bad := attrs["name"]; bad {
t.Fatalf("reserved name kept: %v", attrs)
}
if out["name"] != out["title"] {
t.Fatalf("name should mirror title: name=%v title=%v", out["name"], out["title"])
if out["name"] == nil || strings.TrimSpace(fmt.Sprint(out["name"])) == "" {
t.Fatalf("name missing: %v", out["name"])
}
if _, ok := out["title"]; ok {
t.Fatalf("duplicate title should be omitted when name exists: %v", out["title"])
}
for _, junk := range []string{"id", "processed_product_id", "raw_product_id"} {
if _, ok := out[junk]; ok {
t.Fatalf("%s must be omitted: %v", junk, out)
}
}
sc := ScoreV1ProcessCompletedItem(out, ScoreV1ProcessItemOptions{MappedCategory: "28"})
if !sc.OK {