This commit is contained in:
2026-08-16 16:57:36 +02:00
parent 96f8c1115c
commit 532d439c41
106 changed files with 10147 additions and 494 deletions
+188
View File
@@ -0,0 +1,188 @@
package main
import (
"context"
"fmt"
"log"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
)
func main() {
log.SetOutput(logredact.Writer(os.Stderr))
eans := []string{
"4548736132597",
"194644207427",
"194644034894",
"195949822384",
"3830061960930",
}
if v := strings.TrimSpace(os.Getenv("PROBE_EANS")); v != "" {
eans = strings.Split(v, ",")
for i := range eans {
eans[i] = strings.TrimSpace(eans[i])
}
}
companyID, err := uuid.Parse("2b3159b0-fc08-415b-b248-35ed02a6baab")
if err != nil {
log.Fatal(err)
}
userID, err := uuid.Parse("0ce3305c-d810-4b56-b1d4-3c1ed510db76")
if err != nil {
log.Fatal(err)
}
cfg, err := config.Load()
if err != nil {
log.Fatalf("config: %v", err)
}
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
MaxConns: int32(cfg.DBMaxConns),
MinConns: int32(cfg.DBMinConns),
MaxConnLifetime: cfg.DBMaxConnLifetime,
MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
MaxConnIdleTime: cfg.DBMaxConnIdleTime,
HealthCheckPeriod: cfg.DBHealthCheckPeriod,
StatementTimeout: cfg.DBStatementTimeout,
})
if err != nil {
log.Fatalf("db: %v", err)
}
defer pool.Close()
pipeline := processing.NewPipeline(pool)
pipeline.BatchSize = cfg.ProcessingBatchSize
// Force mock-llm: platform DB OpenAI binding would otherwise override env.
pipeline.AI = nil
pipeline.Prompts = aiprompts.NewService(pool)
mockKey := strings.TrimSpace(cfg.OpenAIAPIKey)
if mockKey == "" {
mockKey = "local-test"
}
mockBase := strings.TrimSpace(cfg.OpenAIBaseURL)
if mockBase == "" {
mockBase = "http://127.0.0.1:18767/v1"
}
mockModel := strings.TrimSpace(cfg.OpenAIModel)
if mockModel == "" {
mockModel = "mock-llm"
}
log.Printf("forcing completer base=%s model=%s", mockBase, mockModel)
pipeline.Engine = &processing.Engine{
Completer: processing.NewOpenAIClient(mockKey, mockBase, mockModel, cfg.ProcessingRPM, cfg.ProcessingMaxRetries),
Vector: processing.NoopVectorCategorizer{},
EPREL: nil, // skip remote EPREL hangs during local scorecard
ProviderMode: processing.AIProviderInternal,
}
rawIDs := make([]uuid.UUID, 0, len(eans))
for _, ean := range eans {
var id uuid.UUID
err := pool.QueryRow(ctx, `
SELECT id FROM raw_products
WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id)
if err != nil {
log.Fatalf("raw product %s: %v", ean, err)
}
rawIDs = append(rawIDs, id)
log.Printf("ean=%s raw_id=%s", ean, id)
}
// Clear competing queue noise.
_, _ = pool.Exec(ctx, `
UPDATE processing_jobs
SET status = 'failed', error = 'yielded to electronics in-process probe', updated_at = now()
WHERE status IN ('pending','running','processing')`)
jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full")
if err != nil {
log.Fatalf("StartJob: %v", err)
}
if len(jobs) == 0 {
log.Fatal("StartJob returned no jobs")
}
jobID := jobs[0].ID
log.Printf("job_id=%s chunks=%d", jobID, len(jobs))
defend := make(chan struct{})
go func() {
t := time.NewTicker(400 * time.Millisecond)
defer t.Stop()
for {
select {
case <-defend:
return
case <-ctx.Done():
return
case <-t.C:
_, _ = pool.Exec(context.Background(), `
UPDATE processing_jobs
SET status = 'failed', error = 'yielded to electronics in-process probe', updated_at = now()
WHERE status IN ('pending','running','processing') AND id <> $1`, jobID)
_, _ = pool.Exec(context.Background(), `
UPDATE processing_jobs
SET status = 'running', error = NULL, updated_at = now()
WHERE id = $1 AND status IN ('failed','cancelled')
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%' OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%' OR error ILIKE '%abandoned%')`, jobID)
_, _ = pool.Exec(context.Background(), `
UPDATE processing_job_products
SET status = 'pending', error = NULL, updated_at = now()
WHERE job_id = $1 AND status IN ('failed','cancelled')
AND processed_product_id IS NULL
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%' OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%' OR error ILIKE '%abandoned%')`, jobID)
}
}
}()
_, _ = pool.Exec(ctx, `
UPDATE processing_jobs
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
WHERE id = $1`, jobID)
runCtx, runCancel := context.WithTimeout(ctx, 3*time.Minute)
defer runCancel()
if err := pipeline.ProcessJob(runCtx, jobID); err != nil {
close(defend)
log.Fatalf("ProcessJob: %v", err)
}
close(defend)
rows, err := pool.Query(ctx, `
SELECT rp.gtin, pjp.status, COALESCE(pjp.error,''), pjp.processed_product_id IS NOT NULL
FROM processing_job_products pjp
JOIN raw_products rp ON rp.id = pjp.raw_product_id
WHERE pjp.job_id = $1
ORDER BY rp.gtin`, jobID)
if err != nil {
log.Fatalf("query items: %v", err)
}
defer rows.Close()
fmt.Printf("JOB %s\n", jobID)
for rows.Next() {
var gtin, status, errMsg string
var hasPP bool
if err := rows.Scan(&gtin, &status, &errMsg, &hasPP); err != nil {
log.Fatal(err)
}
fmt.Printf("ITEM %s status=%s has_pp=%v err=%s\n", gtin, status, hasPP, errMsg)
}
out := os.Getenv("PROBE_OUT_DIR")
if out == "" {
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_process_probe_electronics`
}
_ = os.MkdirAll(out, 0o755)
_ = os.WriteFile(out+string(os.PathSeparator)+"process_id.txt", []byte(jobID.String()+"\n"), 0o644)
}
+192
View File
@@ -0,0 +1,192 @@
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
)
func main() {
log.SetOutput(logredact.Writer(os.Stderr))
eans := []string{
"4242005342488",
"4242005479948",
"4242002996967",
"4242005327065",
"4242005382330",
}
companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
userID := uuid.MustParse("0ce3305c-d810-4b56-b1d4-3c1ed510db76")
cfg, err := config.Load()
if err != nil {
log.Fatalf("config: %v", err)
}
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
MaxConns: int32(cfg.DBMaxConns),
MinConns: int32(cfg.DBMinConns),
MaxConnLifetime: cfg.DBMaxConnLifetime,
MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
MaxConnIdleTime: cfg.DBMaxConnIdleTime,
HealthCheckPeriod: cfg.DBHealthCheckPeriod,
StatementTimeout: cfg.DBStatementTimeout,
})
if err != nil {
log.Fatalf("db: %v", err)
}
defer pool.Close()
out := os.Getenv("PROBE_OUT_DIR")
if out == "" {
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_probe_batch_a`
}
_ = os.MkdirAll(out, 0o755)
pipeline := processing.NewPipeline(pool)
pipeline.BatchSize = cfg.ProcessingBatchSize
pipeline.AI = nil
pipeline.Prompts = aiprompts.NewService(pool)
mockKey := strings.TrimSpace(cfg.OpenAIAPIKey)
if mockKey == "" {
mockKey = "local-test"
}
mockBase := "http://127.0.0.1:18767/v1"
mockModel := "mock-llm"
log.Printf("forcing completer base=%s model=%s (ModeLabel=internal)", mockBase, mockModel)
client := processing.NewOpenAIClient(mockKey, mockBase, mockModel, cfg.ProcessingRPM, cfg.ProcessingMaxRetries)
client.ModeLabel = processing.AIProviderInternal
eprelClient := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second})
pipeline.Engine = &processing.Engine{
Completer: client,
Vector: processing.NoopVectorCategorizer{},
EPREL: eprelClient,
ProviderMode: processing.AIProviderInternal,
}
rawIDs := make([]uuid.UUID, 0, len(eans))
for _, ean := range eans {
var id uuid.UUID
err := pool.QueryRow(ctx, `
SELECT id FROM raw_products
WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id)
if err != nil {
log.Fatalf("raw product %s: %v", ean, err)
}
rawIDs = append(rawIDs, id)
log.Printf("ean=%s raw_id=%s", ean, id)
}
// Self-heal only: do NOT fail/reclaim other jobs.
if n, err := processing.BackfillMissingMeta(ctx, pool, companyID); err != nil {
log.Printf("BackfillMissingMeta: %v", err)
} else {
log.Printf("BackfillMissingMeta updated=%d", n)
}
jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full")
if err != nil {
log.Fatalf("StartJob: %v", err)
}
if len(jobs) == 0 {
log.Fatal("StartJob returned no jobs")
}
jobID := jobs[0].ID
log.Printf("job_id=%s", jobID)
_ = os.WriteFile(out+string(os.PathSeparator)+"process_id.txt", []byte(jobID.String()+"\n"), 0o644)
defend := make(chan struct{})
go func() {
t := time.NewTicker(500 * time.Millisecond)
defer t.Stop()
for {
select {
case <-defend:
return
case <-ctx.Done():
return
case <-t.C:
// Restore ONLY our job if a sibling parked/yielded it. Never touch other jobs.
_, _ = pool.Exec(context.Background(), `
UPDATE processing_jobs
SET status = 'running', error = NULL, updated_at = now()
WHERE id = $1 AND status IN ('failed','cancelled')
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%')`, jobID)
_, _ = pool.Exec(context.Background(), `
UPDATE processing_job_products
SET status = 'pending', error = NULL, updated_at = now()
WHERE job_id = $1 AND status IN ('failed','cancelled')
AND processed_product_id IS NULL
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%')`, jobID)
}
}
}()
_, _ = pool.Exec(ctx, `
UPDATE processing_jobs
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
WHERE id = $1`, jobID)
runCtx, runCancel := context.WithTimeout(ctx, 12*time.Minute)
defer runCancel()
if err := pipeline.ProcessJob(runCtx, jobID); err != nil {
close(defend)
log.Fatalf("ProcessJob: %v", err)
}
close(defend)
items, err := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
if err != nil {
log.Fatalf("LoadV1ProcessJobItems: %v", err)
}
payload := map[string]any{
"data": map[string]any{
"process_id": jobID.String(),
"status": "COMPLETED",
"processing_type": "full",
"total_items": len(items),
"items": items,
},
}
raw, err := json.MarshalIndent(payload, "", " ")
if err != nil {
log.Fatalf("marshal: %v", err)
}
if err := os.WriteFile(out+string(os.PathSeparator)+"final.json", raw, 0o644); err != nil {
log.Fatalf("write final: %v", err)
}
fmt.Printf("JOB %s items=%d\n", jobID, len(items))
for _, it := range items {
ean, _ := it["ean"].(string)
st, _ := it["status"].(string)
title, _ := it["title"].(string)
if len(title) > 50 {
title = title[:50]
}
_, hasEprel := it["eprel"].(map[string]any)
ai, _ := it["ai_provider_mode"].(string)
fmt.Printf("ITEM %s status=%s eprel=%v ai=%s title=%s\n", ean, st, hasEprel, ai, title)
}
}
+173
View File
@@ -0,0 +1,173 @@
package main
import (
"context"
"fmt"
"log"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
)
// timeoutCompleter forces enhance deadline so synthesis path is exercised live.
type timeoutCompleter struct{}
func (timeoutCompleter) Complete(context.Context, string, string) (processing.Completion, error) {
return processing.Completion{}, context.DeadlineExceeded
}
func main() {
log.SetOutput(logredact.Writer(os.Stderr))
ean := strings.TrimSpace(os.Getenv("PROBE_EAN"))
if ean == "" {
ean = "4719331854218"
}
companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
userID := uuid.MustParse("0ce3305c-d810-4b56-b1d4-3c1ed510db76")
cfg, err := config.Load()
if err != nil {
log.Fatalf("config: %v", err)
}
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
MaxConns: int32(cfg.DBMaxConns),
MinConns: int32(cfg.DBMinConns),
MaxConnLifetime: cfg.DBMaxConnLifetime,
MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
MaxConnIdleTime: cfg.DBMaxConnIdleTime,
HealthCheckPeriod: cfg.DBHealthCheckPeriod,
StatementTimeout: cfg.DBStatementTimeout,
})
if err != nil {
log.Fatalf("db: %v", err)
}
defer pool.Close()
pipeline := processing.NewPipeline(pool)
pipeline.BatchSize = cfg.ProcessingBatchSize
pipeline.AI = nil
pipeline.Prompts = aiprompts.NewService(pool)
pipeline.Engine = &processing.Engine{
Completer: timeoutCompleter{},
Vector: processing.NoopVectorCategorizer{},
EPREL: nil,
ProviderMode: processing.AIProviderInternal,
}
var rawID uuid.UUID
if err := pool.QueryRow(ctx, `
SELECT id FROM raw_products
WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&rawID); err != nil {
log.Fatalf("raw product %s: %v", ean, err)
}
log.Printf("ean=%s raw_id=%s", ean, rawID)
_, _ = pool.Exec(ctx, `
UPDATE processed_products
SET processed_description = '', description = '',
field_sources = COALESCE(field_sources, '{}'::jsonb) - 'enhance_input_hash',
updated_at = now()
WHERE company_id = $1 AND product_id = $2`, companyID, ean)
_, _ = pool.Exec(ctx, `
UPDATE processing_jobs
SET status = 'failed', error = 'yielded to thin-synth probe', updated_at = now()
WHERE status IN ('pending','running','processing')`)
jobs, err := pipeline.StartJob(ctx, companyID, userID, []uuid.UUID{rawID}, "full")
if err != nil {
log.Fatalf("StartJob: %v", err)
}
if len(jobs) == 0 {
log.Fatal("StartJob returned no jobs")
}
jobID := jobs[0].ID
log.Printf("job_id=%s", jobID)
defend := make(chan struct{})
go func() {
t := time.NewTicker(300 * time.Millisecond)
defer t.Stop()
for {
select {
case <-defend:
return
case <-ctx.Done():
return
case <-t.C:
_, _ = pool.Exec(context.Background(), `
UPDATE processing_jobs
SET status = 'failed', error = 'yielded to thin-synth probe', updated_at = now()
WHERE status IN ('pending','running','processing') AND id <> $1`, jobID)
_, _ = pool.Exec(context.Background(), `
UPDATE processing_jobs
SET status = 'running', error = NULL, updated_at = now()
WHERE id = $1 AND status IN ('failed','cancelled')
AND (error ILIKE '%parked%' OR error ILIKE '%yield%' OR error ILIKE '%reclaim%'
OR error ILIKE '%exclusive%' OR error ILIKE '%cleared%' OR error ILIKE '%P0%')`, jobID)
_, _ = pool.Exec(context.Background(), `
UPDATE processing_job_products
SET status = 'pending', error = NULL, updated_at = now()
WHERE job_id = $1 AND status IN ('failed','cancelled')
AND processed_product_id IS NULL`, jobID)
}
}
}()
_, _ = pool.Exec(ctx, `
UPDATE processing_jobs
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
WHERE id = $1`, jobID)
runCtx, runCancel := context.WithTimeout(ctx, 2*time.Minute)
defer runCancel()
if err := pipeline.ProcessJob(runCtx, jobID); err != nil {
close(defend)
log.Fatalf("ProcessJob: %v", err)
}
close(defend)
var name, desc, mode string
var descLen int
if err := pool.QueryRow(ctx, `
SELECT COALESCE(name,''), COALESCE(processed_description,''), length(COALESCE(processed_description,'')),
COALESCE(ai_provider_mode,'')
FROM processed_products
WHERE company_id = $1 AND product_id = $2`, companyID, ean).
Scan(&name, &desc, &descLen, &mode); err != nil {
log.Fatalf("load processed: %v", err)
}
fmt.Printf("EAN %s\n", ean)
fmt.Printf("name=%q\n", name)
fmt.Printf("processed_description_len=%d\n", descLen)
fmt.Printf("processed_description=%q\n", desc)
fmt.Printf("ai_provider_mode=%q\n", mode)
echo := strings.EqualFold(strings.TrimSpace(desc), strings.TrimSpace(name))
pass := descLen > 0 && !echo && !strings.Contains(strings.ToLower(desc), "ready for retail listing")
fmt.Printf("PASS_DESC_NE_TITLE=%v\n", pass)
if !pass {
os.Exit(1)
}
out := os.Getenv("PROBE_OUT_DIR")
if out == "" {
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_process_probe_thin_synth`
}
_ = os.MkdirAll(out, 0o755)
_ = os.WriteFile(out+string(os.PathSeparator)+"process_id.txt", []byte(jobID.String()+"\n"), 0o644)
_ = os.WriteFile(out+string(os.PathSeparator)+"score.txt", []byte(fmt.Sprintf(
"ean=%s\nname=%s\npd_len=%d\npd=%s\nmode=%s\nPASS=%v\n", ean, name, descLen, desc, mode, pass,
)), 0o644)
}
+36
View File
@@ -9,6 +9,7 @@ import (
"strconv" "strconv"
"strings" "strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool" "github.com/jackc/pgx/v5/pgxpool"
@@ -48,6 +49,7 @@ func migrateCatalogAndFeeds(
migrateProcessedProducts(ctx, mysqlDB, pg, companyMap, feedMap, rawMap, allow, report, dryRun) migrateProcessedProducts(ctx, mysqlDB, pg, companyMap, feedMap, rawMap, allow, report, dryRun)
backfillProcessedDescriptionsFromMapped(ctx, pg, companyMap, allow, report, dryRun) backfillProcessedDescriptionsFromMapped(ctx, pg, companyMap, allow, report, dryRun)
backfillProcessedNamesFromMapped(ctx, pg, companyMap, allow, report, dryRun) backfillProcessedNamesFromMapped(ctx, pg, companyMap, allow, report, dryRun)
backfillProcessedCategoriesFromMapped(ctx, pg, companyMap, allow, report, dryRun)
} }
if domains.has("feeds") { if domains.has("feeds") {
migrateExportFeeds(ctx, mysqlDB, pg, companyMap, feedMap, allow, report, dryRun) migrateExportFeeds(ctx, mysqlDB, pg, companyMap, feedMap, allow, report, dryRun)
@@ -956,6 +958,40 @@ func backfillProcessedNamesFromMapped(
} }
} }
// backfillProcessedCategoriesFromMapped fills empty processed.category from mapped unique_ids.
func backfillProcessedCategoriesFromMapped(
ctx context.Context,
pg *pgxpool.Pool,
companyMap map[string]string,
allow map[string]bool,
report map[string]int,
dryRun bool,
) {
if dryRun || pg == nil {
return
}
total := 0
for legacy, cid := range companyMap {
if len(allow) > 0 && !allow[legacy] {
continue
}
u, err := uuid.Parse(cid)
if err != nil {
continue
}
res, err := processing.BackfillProcessedCategoriesFromMapped(ctx, pg, u)
if err != nil {
log.Printf("processed category backfill company=%s: %v", cid, err)
continue
}
total += int(res.Updated)
}
if total > 0 {
report["processed_categories_backfilled"] = total
log.Printf("backfilled %d processed_products.category from mapped_data", total)
}
}
func migrateExportFeeds( func migrateExportFeeds(
ctx context.Context, ctx context.Context,
mysqlDB *sql.DB, mysqlDB *sql.DB,
@@ -0,0 +1,77 @@
// Command repair-category-prompts replaces legacy A1 / Platform Demo categories.prompt
// HTML marketing formulas with aiprompts.CategoryEnhanceUserTemplate (JSON-compatible
// user overlay: {{name}} {{description}} {{attrs}} {{category}} {{language}}; stores
// both prompt->>'sl' and "*"). Does not touch title_template / description_template
// (formulas are appended as plain-text instructions at enhance render time).
//
// Usage (from apps/api):
//
// go run ./cmd/repair-category-prompts -dry-run
// go run ./cmd/repair-category-prompts -apply
//
// DATABASE_URL / -postgres required. Default is dry-run (count only).
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"log"
"os"
"strings"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
"github.com/jackc/pgx/v5/pgxpool"
)
func main() {
postgresURL := flag.String("postgres", os.Getenv("DATABASE_URL"), "Postgres URL (DATABASE_URL)")
dryRun := flag.Bool("dry-run", true, "count rows that would update (default true)")
apply := flag.Bool("apply", false, "write updates (implies not dry-run)")
flag.Parse()
if strings.TrimSpace(*postgresURL) == "" {
log.Fatal("-postgres / DATABASE_URL is required")
}
doDryRun := *dryRun
if *apply {
doDryRun = false
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
pg, err := pgxpool.New(ctx, *postgresURL)
if err != nil {
log.Fatalf("postgres: %v", err)
}
defer pg.Close()
// Always print dry-run counts first when applying, so operators see the delta.
preview, err := catalog.RepairA1DemoCategoryEnhancePrompts(ctx, pg, true)
if err != nil {
log.Fatalf("dry-run: %v", err)
}
printResult("dry-run", preview)
if doDryRun {
return
}
res, err := catalog.RepairA1DemoCategoryEnhancePrompts(ctx, pg, false)
if err != nil {
log.Fatalf("apply: %v", err)
}
printResult("apply", res)
}
func printResult(phase string, res catalog.RepairCategoryEnhancePromptsResult) {
b, err := json.MarshalIndent(res, "", " ")
if err != nil {
log.Printf("%s: %+v", phase, res)
return
}
fmt.Printf("%s:\n%s\n", phase, string(b))
}
@@ -0,0 +1,27 @@
package main
import (
"context"
"fmt"
"log"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// backfillA1ProcessedAttributes rewrites A1 processed_products.attributes and
// processed_attributes with AttrsForPersist + category_attributes allowlists
// (same rules as processOne). Use -mode backfill-attributes for polluted local DBs
// where poll already looked clean but the catalog row still stored feed junk.
func backfillA1ProcessedAttributes(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID) (int64, error) {
n, err := processing.BackfillCompanyProcessedAttributes(ctx, pg, companyID)
if err != nil {
return 0, fmt.Errorf("A1 attributes backfill: %w", err)
}
return n, nil
}
func logAttributesBackfillResult(updated int64) {
log.Printf("attributes backfill (A1): processed_updated=%d", updated)
}
+55 -18
View File
@@ -12,6 +12,7 @@ import (
"strings" "strings"
"unicode" "unicode"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/security" "github.com/descrybe/descrybe-v2/apps/api/internal/security"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool" "github.com/jackc/pgx/v5/pgxpool"
@@ -29,6 +30,7 @@ type categoryPromptsFile struct {
type categoryPromptEntry struct { type categoryPromptEntry struct {
Name string `json:"name"` Name string `json:"name"`
UniqueID string `json:"unique_id,omitempty"`
Prompt string `json:"prompt"` Prompt string `json:"prompt"`
} }
@@ -145,26 +147,43 @@ func applyCategoryPrompts(ctx context.Context, pg *pgxpool.Pool, companyID uuid.
} }
byNorm := make(map[string]string, len(file.Entries)) byNorm := make(map[string]string, len(file.Entries))
byUnique := make(map[string]string, len(file.Entries))
// Overlay sets the shared JSON-compatible enhance user template (not legacy HTML).
// Seed JSON selects which categories get a prompt (prefer unique_id, else name).
// Unique title/description formulas stay in title_template / description_template.
// Stored under "sl" + "*" so prompt->>'sl' and LangPromptAny both resolve;
// language stays via {{language}}.
canonical := prepareCategoryPrompt(aiprompts.CategoryEnhanceUserTemplate)
if canonical == "" {
return out, fmt.Errorf("CategoryEnhanceUserTemplate sanitized to empty")
}
for _, e := range file.Entries { for _, e := range file.Entries {
uid := strings.TrimSpace(e.UniqueID)
name := strings.TrimSpace(e.Name) name := strings.TrimSpace(e.Name)
prompt := prepareCategoryPrompt(e.Prompt) if uid == "" && name == "" {
if name == "" || prompt == "" {
out.Skipped++ out.Skipped++
continue continue
} }
if uid != "" {
byUnique[strings.ToLower(uid)] = canonical
}
if name != "" {
key := normalizeCategoryName(name) key := normalizeCategoryName(name)
if key == "" { if key == "" {
if uid == "" {
out.Skipped++ out.Skipped++
}
continue continue
} }
byNorm[key] = prompt byNorm[key] = canonical
} }
if len(byNorm) == 0 { }
if len(byNorm) == 0 && len(byUnique) == 0 {
return out, fmt.Errorf("no usable category prompts in %s", promptsPath) return out, fmt.Errorf("no usable category prompts in %s", promptsPath)
} }
rows, err := pg.Query(ctx, ` rows, err := pg.Query(ctx, `
SELECT id, name SELECT id, COALESCE(unique_id, ''), name
FROM categories FROM categories
WHERE company_id = $1`, companyID) WHERE company_id = $1`, companyID)
if err != nil { if err != nil {
@@ -172,22 +191,34 @@ func applyCategoryPrompts(ctx context.Context, pg *pgxpool.Pool, companyID uuid.
} }
defer rows.Close() defer rows.Close()
ids := make([]uuid.UUID, 0, len(byNorm)) ids := make([]uuid.UUID, 0, len(byNorm)+len(byUnique))
prompts := make([]string, 0, len(byNorm)) prompts := make([]string, 0, len(byNorm)+len(byUnique))
matchedKeys := make(map[string]struct{}, len(byNorm)) matchedNorm := make(map[string]struct{}, len(byNorm))
matchedUID := make(map[string]struct{}, len(byUnique))
for rows.Next() { for rows.Next() {
var id uuid.UUID var id uuid.UUID
var name string var uniqueID, name string
if err := rows.Scan(&id, &name); err != nil { if err := rows.Scan(&id, &uniqueID, &name); err != nil {
return out, err return out, err
} }
prompt := ""
if uidKey := strings.ToLower(strings.TrimSpace(uniqueID)); uidKey != "" {
if p, ok := byUnique[uidKey]; ok {
prompt = p
matchedUID[uidKey] = struct{}{}
}
}
if prompt == "" {
key := normalizeCategoryName(name) key := normalizeCategoryName(name)
prompt, ok := byNorm[key] if p, ok := byNorm[key]; ok {
if !ok { prompt = p
matchedNorm[key] = struct{}{}
}
}
if prompt == "" {
continue continue
} }
matchedKeys[key] = struct{}{}
ids = append(ids, id) ids = append(ids, id)
prompts = append(prompts, prompt) prompts = append(prompts, prompt)
} }
@@ -195,22 +226,28 @@ func applyCategoryPrompts(ctx context.Context, pg *pgxpool.Pool, companyID uuid.
return out, err return out, err
} }
for key := range byUnique {
if _, ok := matchedUID[key]; !ok {
out.Unmatched = append(out.Unmatched, "uid:"+key)
}
}
for key := range byNorm { for key := range byNorm {
if _, ok := matchedKeys[key]; !ok { if _, ok := matchedNorm[key]; !ok {
out.Unmatched = append(out.Unmatched, key) // Name unmatched is noise when unique_id matched the same row; still report for seed hygiene.
out.Unmatched = append(out.Unmatched, "name:"+key)
} }
} }
sort.Strings(out.Unmatched) sort.Strings(out.Unmatched)
if len(ids) == 0 { if len(ids) == 0 {
return out, fmt.Errorf("no A1 categories matched any of %d seed prompts (check names)", len(byNorm)) return out, fmt.Errorf("no A1 categories matched any of %d seed prompts (check unique_id/names)", len(byNorm)+len(byUnique))
} }
// Single parameterized batch update — company_id gate prevents cross-tenant writes. // Single parameterized batch update — company_id gate prevents cross-tenant writes.
// ASSUMPTION: A1 seed company content language is Slovenian ("sl"). // sl + * = prompt->>'sl' and company.LangPromptAny ({{language}} at render).
tag, err := pg.Exec(ctx, ` tag, err := pg.Exec(ctx, `
UPDATE categories AS c UPDATE categories AS c
SET prompt = jsonb_build_object('sl', v.prompt), updated_at = now() SET prompt = jsonb_build_object('sl', v.prompt, '*', v.prompt), updated_at = now()
FROM ( FROM (
SELECT * FROM unnest($1::uuid[], $2::text[]) AS t(id, prompt) SELECT * FROM unnest($1::uuid[], $2::text[]) AS t(id, prompt)
) AS v ) AS v
+21 -8
View File
@@ -7,11 +7,13 @@
// part of npm run seed:a1 (skipped on import + cleared after). Use // part of npm run seed:a1 (skipped on import + cleared after). Use
// -mode recover-jobs only when restoring legacy job history from a MySQL dump. // -mode recover-jobs only when restoring legacy job history from a MySQL dump.
// //
// After reimport, legacy per-category GPT prompts are overlaid from // After reimport, per-category enhance prompts are overlaid from
// scripts/seed/a1-category-prompts.json (Name→Prompt export), matched by // scripts/seed/a1-category-prompts.json (Name list), matched by normalized
// normalized category name, with v1 placeholders rewritten to {{name}} / // category name, writing aiprompts.CategoryEnhanceUserTemplate (JSON-compatible;
// {{description}}. Use -skip-category-prompts to skip, or // includes {{attrs}}/{{language}}). Legacy HTML bodies in that JSON are ignored.
// -mode apply-category-prompts to overlay without a full wipe/reimport. // Use -skip-category-prompts to skip, or -mode apply-category-prompts to overlay
// without a full wipe/reimport. For polluted A1+Platform Demo prompts without
// reimport, run: go run ./cmd/repair-category-prompts -apply
// //
// Categories: dump/archive store assignment on processed_products.category // Categories: dump/archive store assignment on processed_products.category
// (category unique_id). Feed mappings do not map category. After reimport, // (category unique_id). Feed mappings do not map category. After reimport,
@@ -26,6 +28,9 @@
// exported/imported as-is (skip-processed must not strip them). For a // exported/imported as-is (skip-processed must not strip them). For a
// polluted local DB without a full wipe, use -mode backfill-categories. // polluted local DB without a full wipe, use -mode backfill-categories.
// Fixture EANs are purged from non-A1 tenants automatically. // Fixture EANs are purged from non-A1 tenants automatically.
// Processed attribute junk (poll clean, DB still has zavora/vzmetenje, …): use
// -mode backfill-attributes — same SanitizeProductAttributes + category allowlist
// as processOne/enhance.
// //
// Usage: // Usage:
// //
@@ -34,6 +39,7 @@
// go run ./cmd/seed-a1 -mode apply-category-prompts -file ../../scripts/seed/a1-demo-data.sql.gz // go run ./cmd/seed-a1 -mode apply-category-prompts -file ../../scripts/seed/a1-demo-data.sql.gz
// go run ./cmd/seed-a1 -mode recover-jobs -mysql-dump path/to/descrybe_new.sql // go run ./cmd/seed-a1 -mode recover-jobs -mysql-dump path/to/descrybe_new.sql
// go run ./cmd/seed-a1 -mode backfill-categories -mysql-dump path/to/descrybe_new.sql // go run ./cmd/seed-a1 -mode backfill-categories -mysql-dump path/to/descrybe_new.sql
// go run ./cmd/seed-a1 -mode backfill-attributes
// //
// DATABASE_URL / -postgres required. // DATABASE_URL / -postgres required.
// Day-to-day: `npm run seed:a1` (reimport + category prompts + mapped category backfill). Maintainer snapshot: `npm run seed:a1:export`. // Day-to-day: `npm run seed:a1` (reimport + category prompts + mapped category backfill). Maintainer snapshot: `npm run seed:a1:export`.
@@ -77,7 +83,7 @@ type tableSpec struct {
func main() { func main() {
postgresURL := flag.String("postgres", os.Getenv("DATABASE_URL"), "Postgres URL (DATABASE_URL)") postgresURL := flag.String("postgres", os.Getenv("DATABASE_URL"), "Postgres URL (DATABASE_URL)")
mode := flag.String("mode", "reimport", "export | reimport | recover-jobs | apply-category-prompts | backfill-categories") mode := flag.String("mode", "reimport", "export | reimport | recover-jobs | apply-category-prompts | backfill-categories | backfill-attributes")
file := flag.String("file", "", "gzipped seed archive path (required for export/reimport)") file := flag.String("file", "", "gzipped seed archive path (required for export/reimport)")
mysqlDump := flag.String("mysql-dump", os.Getenv("SEED_A1_MYSQL_DUMP"), "mysqldump path for recover-jobs / backfill-categories (or SEED_A1_MYSQL_DUMP)") mysqlDump := flag.String("mysql-dump", os.Getenv("SEED_A1_MYSQL_DUMP"), "mysqldump path for recover-jobs / backfill-categories (or SEED_A1_MYSQL_DUMP)")
company := flag.String("company", defaultA1CompanyID, "Postgres companies.id for A1") company := flag.String("company", defaultA1CompanyID, "Postgres companies.id for A1")
@@ -95,7 +101,8 @@ func main() {
} }
modeVal := strings.ToLower(strings.TrimSpace(*mode)) modeVal := strings.ToLower(strings.TrimSpace(*mode))
if modeVal != "recover-jobs" && modeVal != "apply-category-prompts" && modeVal != "backfill-categories" && strings.TrimSpace(*file) == "" { needsFile := modeVal != "recover-jobs" && modeVal != "apply-category-prompts" && modeVal != "backfill-categories" && modeVal != "backfill-attributes"
if needsFile && strings.TrimSpace(*file) == "" {
log.Fatal("-file is required (e.g. ../../scripts/seed/a1-demo-data.sql.gz)") log.Fatal("-file is required (e.g. ../../scripts/seed/a1-demo-data.sql.gz)")
} }
dumpPath := resolveMySQLDumpPath(*mysqlDump) dumpPath := resolveMySQLDumpPath(*mysqlDump)
@@ -184,13 +191,19 @@ func main() {
} }
res.PurgedOtherRaw, res.PurgedOtherPP = rawN, ppN res.PurgedOtherRaw, res.PurgedOtherPP = rawN, ppN
logCategoryBackfillResult(res, dumpPath) logCategoryBackfillResult(res, dumpPath)
case "backfill-attributes":
n, err := backfillA1ProcessedAttributes(ctx, pg, companyID)
if err != nil {
log.Fatalf("backfill-attributes: %v", err)
}
logAttributesBackfillResult(n)
case "recover-jobs": case "recover-jobs":
if err := recoverJobsFromMySQLDump(ctx, pg, companyID, dumpPath); err != nil { if err := recoverJobsFromMySQLDump(ctx, pg, companyID, dumpPath); err != nil {
log.Fatalf("recover-jobs: %v", err) log.Fatalf("recover-jobs: %v", err)
} }
log.Printf("recovered A1 processing jobs into company %s from %s", companyID, dumpPath) log.Printf("recovered A1 processing jobs into company %s from %s", companyID, dumpPath)
default: default:
log.Fatalf("unknown -mode %q (want export|reimport|recover-jobs|apply-category-prompts|backfill-categories)", *mode) log.Fatalf("unknown -mode %q (want export|reimport|recover-jobs|apply-category-prompts|backfill-categories|backfill-attributes)", *mode)
} }
} }
@@ -0,0 +1,74 @@
package aiprompts
import (
"context"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/google/uuid"
)
// productEnhanceUpsertLanguages returns ISO 639-1 codes to store product_enhance
// under. Never returns LangPromptAny ("*") — ai_prompt_templates.language has
// CHECK (language ~ '^[a-z]{2}$'). Categories.prompt JSON may still use "*".
func productEnhanceUpsertLanguages(contentLangs []string, primary string) []string {
primary = company.NormalizeLanguage(primary)
if primary == "" || primary == company.LangPromptAny || !company.IsAllowedLanguage(primary) {
primary = company.DefaultLanguage
}
out := make([]string, 0, len(contentLangs)+1)
seen := map[string]struct{}{}
add := func(code string) {
code = company.NormalizeLanguage(code)
if code == "" || code == company.LangPromptAny || !company.IsAllowedLanguage(code) {
return
}
if _, ok := seen[code]; ok {
return
}
seen[code] = struct{}{}
out = append(out, code)
}
add(primary)
for _, lang := range contentLangs {
add(lang)
}
if len(out) == 0 {
return []string{company.DefaultLanguage}
}
return out
}
// ApplyBuiltInProductEnhance upserts BuiltInDefaults for product_enhance under
// the company primary language and each content language. Templates still use
// {{language}} at render time; Resolve falls back requested → primary → built-in.
// Do not store LangPromptAny ("*") in ai_prompt_templates (DB language CHECK).
func (s *Service) ApplyBuiltInProductEnhance(ctx context.Context, companyID uuid.UUID) (languagesApplied int, err error) {
def, ok := DefaultFor(KeyProductEnhance)
if !ok {
return 0, ErrInvalidKey
}
primary := company.LoadLanguage(ctx, s.Pool, companyID)
langs := productEnhanceUpsertLanguages(
company.LoadContentLanguages(ctx, s.Pool, companyID),
primary,
)
enabled := true
items := make([]UpdateItem, 0, len(langs))
for _, lang := range langs {
items = append(items, UpdateItem{
Key: KeyProductEnhance,
Language: lang,
SystemTemplate: def.SystemTemplate,
UserTemplate: def.UserTemplate,
IsEnabled: &enabled,
})
}
_, err = s.Update(ctx, companyID, UpdateInput{
Language: langs[0],
Prompts: items,
})
if err != nil {
return 0, err
}
return len(langs), nil
}
@@ -0,0 +1,71 @@
package aiprompts
import (
"regexp"
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
)
var languageFmt = regexp.MustCompile(`^[a-z]{2}$`)
func TestProductEnhanceUpsertLanguages_neverStar(t *testing.T) {
t.Parallel()
cases := []struct {
name string
contentLangs []string
primary string
want []string
}{
{
name: "primary_only",
contentLangs: nil,
primary: "sl",
want: []string{"sl"},
},
{
name: "primary_plus_content",
contentLangs: []string{"sl", "en", "de"},
primary: "sl",
want: []string{"sl", "en", "de"},
},
{
name: "drops_star_and_invalid",
contentLangs: []string{"*", "sl", "xx", "", "EN"},
primary: "*",
want: []string{"en", "sl"},
},
{
name: "dedupes_primary_in_content",
contentLangs: []string{"sl", "sl", "en"},
primary: "sl",
want: []string{"sl", "en"},
},
{
name: "empty_falls_back_default",
contentLangs: []string{"*", ""},
primary: "",
want: []string{company.DefaultLanguage},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
got := productEnhanceUpsertLanguages(tc.contentLangs, tc.primary)
if len(got) != len(tc.want) {
t.Fatalf("len=%d want %d (%v)", len(got), len(tc.want), got)
}
for i := range got {
if got[i] != tc.want[i] {
t.Fatalf("got %v want %v", got, tc.want)
}
if !languageFmt.MatchString(got[i]) {
t.Fatalf("language %q fails ai_prompt_templates_language_fmt", got[i])
}
if got[i] == company.LangPromptAny {
t.Fatalf("must not upsert LangPromptAny")
}
}
})
}
}
+31 -5
View File
@@ -1,5 +1,7 @@
package aiprompts package aiprompts
import "strings"
// Prompt keys stored in ai_prompt_templates.prompt_key. // Prompt keys stored in ai_prompt_templates.prompt_key.
const ( const (
KeyProductEnhance = "product_enhance" KeyProductEnhance = "product_enhance"
@@ -45,6 +47,32 @@ type DefaultTemplate struct {
UserTemplate string `json:"user_template"` UserTemplate string `json:"user_template"`
} }
// CategoryEnhanceUserTemplate is the shared per-category (and built-in) enhance USER
// message. Includes {{name}} {{description}} {{attrs}} {{category}} {{language}}.
// Compatible with the enhance system JSON schema {"name","description"} — not a
// competing HTML marketing document. Title/description formulas stay in
// title_template / description_template and are appended at render time as plain
// text instructions (see processing.AppendFormulaConstraints).
// Used by local A1/Demo prompt repair and seed-a1 overlays.
const CategoryEnhanceUserTemplate = `Your reply is parsed as JSON {"name":"string","description":"string"} only (system schema). Write name and description in {{language}} (do not hardcode a language).
- name: short retail title; follow any Title formula constraints that follow; use Attrs
- description: prefer 1-3 factual paragraphs as ONE string; limited HTML (<h2><p><ul><li>) is allowed if needed — do NOT emit a competing full HTML document, <name>/<metaDescription> blocks, or separate schema
Category: {{category}}
Name: {{name}}
Desc: {{description}}
Attrs: {{attrs}}`
// CategoryEnhancePromptNeedsRepair reports whether a stored categories.prompt value
// should be replaced by CategoryEnhanceUserTemplate (idempotent equality check).
func CategoryEnhancePromptNeedsRepair(prompt string) bool {
p := strings.TrimSpace(prompt)
if p == "" {
return false
}
return p != strings.TrimSpace(CategoryEnhanceUserTemplate)
}
// BuiltInDefaults match the previous hardcoded system prompts, with structured user templates. // BuiltInDefaults match the previous hardcoded system prompts, with structured user templates.
var BuiltInDefaults = []DefaultTemplate{ var BuiltInDefaults = []DefaultTemplate{
{ {
@@ -56,15 +84,13 @@ Rules:
- Reply with ONLY JSON (no markdown) - Reply with ONLY JSON (no markdown)
- Schema: {"name":"string","description":"string"} - Schema: {"name":"string","description":"string"}
- name: short retail title - name: short retail title
- description: 1-2 factual sentences - description: 1-2 factual sentences; never copy name as description
- When Desc is empty or the same as Name, write 1-2 factual sentences from Category and Attrs only (do not invent specs)
- Write name and description in {{language}} - Write name and description in {{language}}
Example: Example:
{"name":"Acme Widget Pro","description":"Durable widget for everyday use. Clear specs, ready to ship."} {"name":"Acme Widget Pro","description":"Durable widget for everyday use. Clear specs, ready to ship."}
{{brand_voice}}`, {{brand_voice}}`,
UserTemplate: `Category: {{category}} UserTemplate: CategoryEnhanceUserTemplate,
Name: {{name}}
Desc: {{description}}
Attrs: {{attrs}}`,
}, },
{ {
Key: KeySEOMeta, Key: KeySEOMeta,
+59
View File
@@ -0,0 +1,59 @@
package aiprompts
import (
"strings"
"testing"
)
func TestCategoryEnhanceUserTemplateHasRequiredVars(t *testing.T) {
t.Parallel()
vars := ExtractVariables(CategoryEnhanceUserTemplate)
want := []string{"language", "category", "name", "description", "attrs"}
got := map[string]struct{}{}
for _, v := range vars {
got[v] = struct{}{}
}
for _, w := range want {
if _, ok := got[w]; !ok {
t.Fatalf("CategoryEnhanceUserTemplate missing {{%s}}; vars=%v", w, vars)
}
}
lower := strings.ToLower(CategoryEnhanceUserTemplate)
if strings.Contains(lower, "100 besed") || strings.Contains(lower, "gpt predloga") {
t.Fatal("template must not demand legacy long HTML marketing docs")
}
if !strings.Contains(CategoryEnhanceUserTemplate, `{"name":"string","description":"string"}`) {
t.Fatal("template should reference JSON schema shape")
}
if !strings.Contains(lower, "{{language}}") {
t.Fatal("language must come from {{language}}")
}
}
func TestCategoryEnhancePromptNeedsRepair(t *testing.T) {
t.Parallel()
if CategoryEnhancePromptNeedsRepair("") {
t.Fatal("empty should not need repair")
}
if CategoryEnhancePromptNeedsRepair(CategoryEnhanceUserTemplate) {
t.Fatal("canonical template should be idempotent")
}
if CategoryEnhancePromptNeedsRepair(" " + CategoryEnhanceUserTemplate + "\n") {
t.Fatal("whitespace-trimmed canonical should be idempotent")
}
legacy := `Ustvari nov opis\n<H2>foo</H2>\nAttrs missing`
if !CategoryEnhancePromptNeedsRepair(legacy) {
t.Fatal("legacy HTML prompt should need repair")
}
}
func TestBuiltInProductEnhanceUsesSharedUserTemplate(t *testing.T) {
t.Parallel()
def, ok := DefaultFor(KeyProductEnhance)
if !ok {
t.Fatal("missing product_enhance default")
}
if def.UserTemplate != CategoryEnhanceUserTemplate {
t.Fatalf("UserTemplate must be CategoryEnhanceUserTemplate")
}
}
+27 -6
View File
@@ -87,8 +87,11 @@ func (s *Service) GetBundle(ctx context.Context, companyID uuid.UUID, language s
}, nil }, nil
} }
// Resolve returns the effective templates for one key + language // Resolve returns the effective templates for one key + language.
// (custom if enabled for lang, else built-in). No cross-language company fallback. // Fallback: requested lang → "*" → company primary language → built-in.
// Language on the result is always the requested (normalized) lang so {{language}}
// still resolves to the content language being generated — templates are shared,
// not duplicated per language.
func (s *Service) Resolve(ctx context.Context, companyID uuid.UUID, key, language string) (Resolved, error) { func (s *Service) Resolve(ctx context.Context, companyID uuid.UUID, key, language string) (Resolved, error) {
if !ValidPromptKey(key) { if !ValidPromptKey(key) {
return Resolved{}, ErrInvalidKey return Resolved{}, ErrInvalidKey
@@ -101,11 +104,19 @@ func (s *Service) Resolve(ctx context.Context, companyID uuid.UUID, key, languag
if err != nil { if err != nil {
lang = company.DefaultLanguage lang = company.DefaultLanguage
} }
st, err := s.loadOne(ctx, companyID, key, lang) primary := company.LoadLanguage(ctx, s.Pool, companyID)
candidates := []string{lang, company.LangPromptAny}
if primary != "" && primary != lang && primary != company.LangPromptAny {
candidates = append(candidates, primary)
}
for _, cand := range candidates {
st, err := s.loadOne(ctx, companyID, key, cand)
if err != nil && !errors.Is(err, pgx.ErrNoRows) { if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return Resolved{}, err return Resolved{}, err
} }
if err == nil && st.isEnabled { if err != nil || !st.isEnabled {
continue
}
sys := strings.TrimSpace(st.systemTemplate) sys := strings.TrimSpace(st.systemTemplate)
user := strings.TrimSpace(st.userTemplate) user := strings.TrimSpace(st.userTemplate)
if sys == "" { if sys == "" {
@@ -155,10 +166,16 @@ func (s *Service) Update(ctx context.Context, companyID uuid.UUID, in UpdateInpu
if langRaw == "" { if langRaw == "" {
langRaw = defaultLang langRaw = defaultLang
} }
lang, err := company.ParseLanguage(langRaw, false) var lang string
if langRaw == company.LangPromptAny {
lang = company.LangPromptAny
} else {
parsed, err := company.ParseLanguage(langRaw, false)
if err != nil { if err != nil {
return Bundle{}, fmt.Errorf("%w: language %q", ErrInvalidInput, langRaw) return Bundle{}, fmt.Errorf("%w: language %q", ErrInvalidInput, langRaw)
} }
lang = parsed
}
lastLang = lang lastLang = lang
if item.Reset { if item.Reset {
_, err := tx.Exec(ctx, ` _, err := tx.Exec(ctx, `
@@ -196,7 +213,11 @@ func (s *Service) Update(ctx context.Context, companyID uuid.UUID, in UpdateInpu
if err := tx.Commit(ctx); err != nil { if err := tx.Commit(ctx); err != nil {
return Bundle{}, err return Bundle{}, err
} }
return s.GetBundle(ctx, companyID, lastLang) bundleLang := lastLang
if bundleLang == company.LangPromptAny {
bundleLang = defaultLang
}
return s.GetBundle(ctx, companyID, bundleLang)
} }
func (s *Service) loadAll(ctx context.Context, companyID uuid.UUID) ([]stored, error) { func (s *Service) loadAll(ctx context.Context, companyID uuid.UUID) ([]stored, error) {
@@ -61,3 +61,15 @@ func TestResolvePlatformOpenAI_viaPlatformService(t *testing.T) {
t.Fatalf("source=%q", oi.Source) t.Fatalf("source=%q", oi.Source)
} }
} }
func TestPlatformModeLabel(t *testing.T) {
if got := platformModeLabel(platformsettings.SourceDB, "https://llm.overloadedbot.org/v1"); got != ModeInternalLabel {
t.Fatalf("db remote: %q", got)
}
if got := platformModeLabel(platformsettings.SourceEnv, "https://api.openai.com/v1"); got != ModeCustomLabel {
t.Fatalf("env remote: %q", got)
}
if got := platformModeLabel(platformsettings.SourceDB, "http://127.0.0.1:18767/v1"); got != ModeCustomLabel {
t.Fatalf("mock loopback: %q", got)
}
}
+5 -3
View File
@@ -4,6 +4,7 @@ import (
"context" "context"
"strings" "strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing" "github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid" "github.com/google/uuid"
) )
@@ -134,7 +135,7 @@ func (s *Service) resolvePlatformRoleCompleter(ctx context.Context, role string)
BaseURL: cfg.BaseURL, BaseURL: cfg.BaseURL,
Model: model, Model: model,
UsingBYOK: false, UsingBYOK: false,
ModeLabel: ModeInternalLabel, ModeLabel: platformModeLabel(cfg.Source, cfg.BaseURL),
}) })
} }
} }
@@ -143,13 +144,14 @@ func (s *Service) resolvePlatformRoleCompleter(ctx context.Context, role string)
if role == RoleProcessing { if role == RoleProcessing {
key := strings.TrimSpace(s.Env.OpenAIAPIKey) key := strings.TrimSpace(s.Env.OpenAIAPIKey)
model := strings.TrimSpace(s.Env.OpenAIModel) model := strings.TrimSpace(s.Env.OpenAIModel)
baseURL := strings.TrimSpace(s.Env.OpenAIBaseURL)
if key != "" && model != "" { if key != "" && model != "" {
return s.completerFromEndpoint(RoleEndpoint{ return s.completerFromEndpoint(RoleEndpoint{
APIKey: key, APIKey: key,
BaseURL: strings.TrimSpace(s.Env.OpenAIBaseURL), BaseURL: baseURL,
Model: model, Model: model,
UsingBYOK: false, UsingBYOK: false,
ModeLabel: ModeInternalLabel, ModeLabel: platformModeLabel(platformsettings.SourceEnv, baseURL),
}) })
} }
} }
+15 -2
View File
@@ -277,6 +277,7 @@ func (s *Service) Resolve(ctx context.Context, companyID uuid.UUID) (Resolved, e
if strings.TrimSpace(platform.APIKey) == "" { if strings.TrimSpace(platform.APIKey) == "" {
return Resolved{ModeLabel: ModeInternalLabel, UsingBYOK: false}, nil return Resolved{ModeLabel: ModeInternalLabel, UsingBYOK: false}, nil
} }
modeLabel := platformModeLabel(platform.Source, platform.BaseURL)
client := processing.NewOpenAIClient( client := processing.NewOpenAIClient(
platform.APIKey, platform.APIKey,
platform.BaseURL, platform.BaseURL,
@@ -284,17 +285,29 @@ func (s *Service) Resolve(ctx context.Context, companyID uuid.UUID) (Resolved, e
rpm, rpm,
retries, retries,
) )
client.ModeLabel = ModeInternalLabel client.ModeLabel = modeLabel
if s.HTTPClient != nil { if s.HTTPClient != nil {
client.HTTPClient = s.HTTPClient client.HTTPClient = s.HTTPClient
} }
return Resolved{ return Resolved{
Completer: client, Completer: client,
ModeLabel: ModeInternalLabel, ModeLabel: modeLabel,
UsingBYOK: false, UsingBYOK: false,
}, nil }, nil
} }
// platformModeLabel maps platform OpenAI source/base to analytics ModeLabel.
// DB-backed admin settings → internal; env bootstrap or mock/loopback → custom.
func platformModeLabel(source, baseURL string) string {
if processing.IsMockOrLoopbackBaseURL(baseURL) {
return ModeCustomLabel
}
if strings.TrimSpace(source) == platformsettings.SourceEnv {
return ModeCustomLabel
}
return ModeInternalLabel
}
func (s *Service) platformConfigured(ctx context.Context) (bool, error) { func (s *Service) platformConfigured(ctx context.Context) (bool, error) {
oi, err := s.resolvePlatformOpenAI(ctx) oi, err := s.resolvePlatformOpenAI(ctx)
if err != nil { if err != nil {
+13 -1
View File
@@ -9,6 +9,10 @@ import (
"github.com/jackc/pgx/v5/pgxpool" "github.com/jackc/pgx/v5/pgxpool"
) )
// cloneProcessedFieldSourcesSQL copies field_sources but drops enhance_input_hash
// so the destination's first process cannot hash-skip a thin prior description.
const cloneProcessedFieldSourcesSQL = "COALESCE(pp.field_sources, '{}'::jsonb) - 'enhance_input_hash'"
// CloneResult summarizes a company catalog clone (source unchanged). // CloneResult summarizes a company catalog clone (source unchanged).
type CloneResult struct { type CloneResult struct {
SourceCompanyID uuid.UUID `json:"source_company_id"` SourceCompanyID uuid.UUID `json:"source_company_id"`
@@ -382,7 +386,7 @@ func cloneCompanyCatalog(ctx context.Context, pool *pgxpool.Pool, src, dest uuid
pp.meta_description, pp.meta_description,
pp.last_transition_at, pp.last_transition_at,
pp.structured_description, pp.structured_description,
pp.field_sources, ` + cloneProcessedFieldSourcesSQL + `,
pp.created_at, pp.created_at,
now(), now(),
COALESCE(pp.ai_provider_mode, 'internal'), COALESCE(pp.ai_provider_mode, 'internal'),
@@ -452,6 +456,14 @@ func cloneCompanyCatalog(ctx context.Context, pool *pgxpool.Pool, src, dest uuid
return nil, fmt.Errorf("reset raw processing flags: %w", err) return nil, fmt.Errorf("reset raw processing flags: %w", err)
} }
// Drop skip hashes still present on weak / title-echo localized descriptions
// (field_sources hash is already stripped in the INSERT above).
hashRepair, err := repairWeakEnhanceHashesTx(ctx, tx, dest)
if err != nil {
return nil, fmt.Errorf("repair weak enhance hashes: %w", err)
}
add("weak_enhance_hashes_cleared", hashRepair.Cleared)
if err := tx.Commit(ctx); err != nil { if err := tx.Commit(ctx); err != nil {
return nil, err return nil, err
} }
+12
View File
@@ -1,6 +1,7 @@
package catalog package catalog
import ( import (
"strings"
"testing" "testing"
"github.com/google/uuid" "github.com/google/uuid"
@@ -24,3 +25,14 @@ func TestValidateCloneCompanies(t *testing.T) {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
} }
func TestCloneFieldSourcesSQLClearsEnhanceHash(t *testing.T) {
t.Parallel()
const want = "enhance_input_hash"
if !strings.Contains(cloneProcessedFieldSourcesSQL, want) {
t.Fatalf("clone field_sources SQL must remove %q; got %q", want, cloneProcessedFieldSourcesSQL)
}
if !strings.Contains(cloneProcessedFieldSourcesSQL, "-") {
t.Fatalf("expected jsonb key removal operator in %q", cloneProcessedFieldSourcesSQL)
}
}
+76
View File
@@ -0,0 +1,76 @@
package catalog
import (
"context"
"fmt"
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// EnsureCategoryAttributeLinks removes orphan category_attributes rows (missing
// category or attribute) without wiping taxonomy or valid links. Returns orphans
// removed and remaining link count.
func EnsureCategoryAttributeLinks(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (orphansRemoved int, links int, err error) {
if pool == nil {
return 0, 0, fmt.Errorf("nil pool")
}
ct, err := pool.Exec(ctx, `
DELETE FROM category_attributes ca
WHERE ca.company_id = $1
AND (
NOT EXISTS (
SELECT 1 FROM categories c
WHERE c.company_id = ca.company_id AND c.unique_id = ca.category_unique_id
)
OR NOT EXISTS (
SELECT 1 FROM attributes a
WHERE a.id = ca.attribute_id AND a.company_id = ca.company_id
)
)`, companyID)
if err != nil {
return 0, 0, fmt.Errorf("purge orphan category_attributes: %w", err)
}
orphansRemoved = int(ct.RowsAffected())
if err := pool.QueryRow(ctx, `
SELECT count(*)::int FROM category_attributes WHERE company_id = $1`, companyID).Scan(&links); err != nil {
return orphansRemoved, 0, err
}
return orphansRemoved, links, nil
}
// RepairCompanyCategoryEnhancePrompts is the company-scoped variant of
// RepairA1DemoCategoryEnhancePrompts: same repairedCategoryEnhancePromptMap
// (sl + "*" with CategoryEnhanceUserTemplate / {{attrs}}), applied to one company.
// Idempotent — already-OK categories count as already_ok, not updated.
func RepairCompanyCategoryEnhancePrompts(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated, alreadyOK, emptySkipped int, err error) {
if pool == nil {
return 0, 0, 0, fmt.Errorf("nil pool")
}
want, err := repairedCategoryEnhancePromptMap()
if err != nil {
return 0, 0, 0, err
}
summary := &RepairCategoryEnhancePromptsResult{ByCompany: map[string]int{}}
if _, err := repairCompanyCategoryEnhancePrompts(ctx, pool, companyID, companyID.String(), want, false, summary); err != nil {
return 0, 0, 0, err
}
return summary.Updated, summary.AlreadyOK, summary.EmptySkipped, nil
}
// ResolvePlatformDemoCompanyID finds the Platform Demo sandbox (never A1 cohort).
func ResolvePlatformDemoCompanyID(ctx context.Context, pool *pgxpool.Pool) (uuid.UUID, error) {
var id uuid.UUID
err := pool.QueryRow(ctx, `
SELECT c.id
FROM companies c
WHERE c.name = $1
AND COALESCE(c.legacy_company_id, '') <> $2
ORDER BY c.created_at ASC
LIMIT 1`, platformDemoCompanyName, billing.A1LegacyCompanyID).Scan(&id)
if err != nil {
return uuid.Nil, fmt.Errorf("platform demo company not found: %w", err)
}
return id, nil
}
@@ -0,0 +1,45 @@
package catalog
import (
"strings"
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
)
func TestCategoryEnhanceTemplateHasAttrs(t *testing.T) {
t.Parallel()
tpl := aiprompts.CategoryEnhanceUserTemplate
if !strings.Contains(tpl, "{{attrs}}") {
t.Fatalf("template missing {{attrs}}: %q", tpl)
}
if !aiprompts.CategoryEnhancePromptNeedsRepair("legacy HTML prompt without attrs") {
t.Fatal("expected legacy prompt to need repair")
}
if aiprompts.CategoryEnhancePromptNeedsRepair(tpl) {
t.Fatal("canonical template should not need repair")
}
}
func TestPlatformDemoNameConstant(t *testing.T) {
t.Parallel()
if platformDemoCompanyName != "Platform Demo" {
t.Fatalf("got %q", platformDemoCompanyName)
}
}
func TestRepairedCategoryEnhancePromptMapMatchesA1Demo(t *testing.T) {
t.Parallel()
want, err := repairedCategoryEnhancePromptMap()
if err != nil {
t.Fatal(err)
}
tpl := strings.TrimSpace(aiprompts.CategoryEnhanceUserTemplate)
if want["sl"] != tpl || want[company.LangPromptAny] != tpl {
t.Fatalf("want sl+* template, got %#v", want)
}
if !strings.Contains(tpl, "{{attrs}}") {
t.Fatal("template must include {{attrs}}")
}
}
@@ -0,0 +1,66 @@
package catalog
import (
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
)
// alignProductFieldsWithV1 mirrors V1 process-item field semantics on dashboard
// product payloads for the same EAN: title/name, preferred description, and
// structured eprel. Mutates item in place after SQL scan / feed-spec linking.
func alignProductFieldsWithV1(item map[string]any) {
if item == nil {
return
}
name := firstNonEmptyString(
item["processed_name"],
item["name"],
item["title"],
)
if name != "" {
item["name"] = name
item["title"] = name
}
desc := firstNonEmptyString(
item["processed_description"],
item["description"],
)
if desc != "" {
item["description"] = desc
}
eprelVal := eprel.ExtractFromAttrs(asStringAnyMap(item["processed_attributes"]))
if eprelVal == nil {
eprelVal = eprel.ExtractFromAttrs(asStringAnyMap(item["attributes"]))
}
if eprelVal == nil {
if mapped, ok := item["mapped_data"].(map[string]any); ok {
eprelVal = eprel.ExtractFromAttrs(mapped)
}
}
item["eprel"] = eprelVal
if eprelVal != nil {
item["has_eprel"] = true
}
}
func firstNonEmptyString(vals ...any) string {
for _, v := range vals {
switch t := v.(type) {
case string:
if s := strings.TrimSpace(t); s != "" {
return s
}
case *string:
if t != nil {
if s := strings.TrimSpace(*t); s != "" {
return s
}
}
}
}
return ""
}
@@ -0,0 +1,123 @@
package catalog
import (
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
)
func TestAlignProductFieldsWithV1_prefersProcessedAndEprel(t *testing.T) {
item := map[string]any{
"name": "RAW ALLCAPS TITLE",
"processed_name": "Retail Title",
"description": "raw feed description",
"processed_description": "AI retail description",
"attributes": map[string]any{
"vzmetenje": "junk",
"brand": "Ostalo",
},
"processed_attributes": map[string]any{
"brand": "Ostalo",
"product_model": "W53070",
"eprel_id": "1632113",
"eprel_label": "https://eprel.example/label",
"eprel_energy_class": "A",
},
"category_name": "Nosilci za TV",
}
alignProductFieldsWithV1(item)
if item["name"] != "Retail Title" {
t.Fatalf("name=%v want Retail Title", item["name"])
}
if item["title"] != "Retail Title" {
t.Fatalf("title=%v want Retail Title", item["title"])
}
if item["description"] != "AI retail description" {
t.Fatalf("description=%v", item["description"])
}
ep, ok := item["eprel"].(map[string]any)
if !ok {
t.Fatalf("eprel type=%T", item["eprel"])
}
if ep["id"] != "1632113" || ep["energy_class"] != "A" {
t.Fatalf("eprel=%v", ep)
}
if item["has_eprel"] != true {
t.Fatalf("has_eprel=%v", item["has_eprel"])
}
// Feed attrs stay available for dual-pane UI; V1 poll uses processed_attributes.
attrs, _ := item["attributes"].(map[string]any)
if attrs["vzmetenje"] != "junk" {
t.Fatalf("feed attributes should remain: %v", attrs)
}
if item["category_name"] != "Nosilci za TV" {
t.Fatalf("category_name mutated")
}
}
func TestAlignProductFieldsWithV1_matchesProcessItemSemantics(t *testing.T) {
// Same underlying row as a V1 process item would project for one EAN.
processedName := "Pralni stroj Samsung WW90CGC04DAELE, 9kg"
processedDesc := "Tehnologija Ecobubble…"
processedAttrs := map[string]any{
"brand": "Samsung",
"product_model": "WW90CGC04DAELE",
"eprel_id": "1632113",
"eprel_label": "https://eprel.example/label",
"eprel_pdf": "https://eprel.example/fiche.pdf",
"eprel_energy_class": "A",
"eprel_energy_scale": "A_G",
}
dashboard := map[string]any{
"name": "RAW FEED TITLE",
"processed_name": processedName,
"description": "raw description",
"processed_description": processedDesc,
"category": "40",
"category_name": "Pralni stroji",
"attributes": map[string]any{"dodatne-informacije": "AI", "brand": "Samsung"},
"processed_attributes": processedAttrs,
"gtin": "8806095210711",
}
alignProductFieldsWithV1(dashboard)
// V1 process item projection (subset used by poll clients).
v1Title := processedName
v1Desc := processedDesc
v1Eprel := eprel.ExtractFromAttrs(processedAttrs)
if dashboard["name"] != v1Title || dashboard["title"] != v1Title {
t.Fatalf("title/name mismatch: dash=%v/%v v1=%v", dashboard["title"], dashboard["name"], v1Title)
}
if dashboard["description"] != v1Desc {
t.Fatalf("description mismatch: dash=%v v1=%v", dashboard["description"], v1Desc)
}
if dashboard["category_name"] != "Pralni stroji" {
t.Fatalf("category_name=%v", dashboard["category_name"])
}
dashEprel, _ := dashboard["eprel"].(map[string]any)
v1Map, _ := v1Eprel.(map[string]any)
if dashEprel["id"] != v1Map["id"] || dashEprel["energy_class"] != v1Map["energy_class"] {
t.Fatalf("eprel mismatch dash=%v v1=%v", dashEprel, v1Map)
}
// V1 attributes ≈ processed_attributes (allowlist/sanitize aside).
pa, _ := dashboard["processed_attributes"].(map[string]any)
if pa["brand"] != "Samsung" || pa["product_model"] != "WW90CGC04DAELE" {
t.Fatalf("processed_attributes=%v", pa)
}
}
func TestAlignProductFieldsWithV1_eprelFromMappedFallback(t *testing.T) {
item := map[string]any{
"processed_name": "Washer",
"mapped_data": map[string]any{
"eprel_id": "1632113",
},
}
alignProductFieldsWithV1(item)
ep, ok := item["eprel"].(map[string]any)
if !ok || ep["id"] != "1632113" {
t.Fatalf("eprel=%v", item["eprel"])
}
}
+220
View File
@@ -0,0 +1,220 @@
package catalog
import (
"context"
"fmt"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// Canonical Postgres id for the migrated A1 Slovenija tenant (seed-a1 default).
const a1CompanyID = "604f23a8-b66e-4b21-8b45-0d72b68f4790"
// platformDemoCompanyName matches migrator / seed-demo standalone demo tenant.
const platformDemoCompanyName = "Platform Demo"
// RepairCategoryEnhancePromptsResult is the dry-run / apply summary for
// RepairA1DemoCategoryEnhancePrompts.
type RepairCategoryEnhancePromptsResult struct {
CompaniesScanned int `json:"companies_scanned"`
CategoriesSeen int `json:"categories_seen"`
WouldUpdate int `json:"would_update"`
Updated int `json:"updated"`
AlreadyOK int `json:"already_ok"`
EmptySkipped int `json:"empty_skipped"`
ByCompany map[string]int `json:"by_company"`
DryRun bool `json:"dry_run"`
}
// RepairA1DemoCategoryEnhancePrompts replaces legacy HTML marketing categories.prompt
// values for A1 Slovenija + Platform Demo with aiprompts.CategoryEnhanceUserTemplate.
//
// LOCAL repair only (idempotent):
// - Writes JSON-compatible user overlays under both "sl" (prompt->>'sl') and "*"
// (LangPromptAny) so language stays via {{language}}, not hardcoded-only copy.
// - Touches ONLY categories.prompt — never title_template / description_template
// (unique name/description formulas stay intact; AppendFormulaConstraints encodes
// them as plain-text instructions at enhance render time).
// - dryRun=true: count WouldUpdate only; dryRun=false: apply and set Updated.
//
// Entrypoint: go run ./cmd/repair-category-prompts (-dry-run | -apply).
func RepairA1DemoCategoryEnhancePrompts(ctx context.Context, pool *pgxpool.Pool, dryRun bool) (RepairCategoryEnhancePromptsResult, error) {
out := RepairCategoryEnhancePromptsResult{
DryRun: dryRun,
ByCompany: map[string]int{},
}
if pool == nil {
return out, fmt.Errorf("postgres pool is required")
}
a1ID, err := uuid.Parse(a1CompanyID)
if err != nil {
return out, fmt.Errorf("a1 company id: %w", err)
}
companyRows, err := pool.Query(ctx, `
SELECT id, name
FROM companies
WHERE id = $1
OR name = $2
OR COALESCE(legacy_company_id, '') = $3
ORDER BY name`, a1ID, platformDemoCompanyName, billing.A1LegacyCompanyID)
if err != nil {
return out, fmt.Errorf("list target companies: %w", err)
}
defer companyRows.Close()
type co struct {
id uuid.UUID
name string
}
companies := make([]co, 0, 2)
for companyRows.Next() {
var c co
if err := companyRows.Scan(&c.id, &c.name); err != nil {
return out, err
}
companies = append(companies, c)
}
if err := companyRows.Err(); err != nil {
return out, err
}
out.CompaniesScanned = len(companies)
if len(companies) == 0 {
return out, fmt.Errorf("no A1 / Platform Demo companies found")
}
want, err := repairedCategoryEnhancePromptMap()
if err != nil {
return out, err
}
for _, c := range companies {
n, err := repairCompanyCategoryEnhancePrompts(ctx, pool, c.id, c.name, want, dryRun, &out)
if err != nil {
return out, err
}
if n > 0 {
out.ByCompany[c.name] = n
}
}
return out, nil
}
// repairedCategoryEnhancePromptMap is the canonical stored shape: "sl" (prompt->>'sl'
// for A1/Demo) plus LangPromptAny ("*") so PromptForLanguage resolves for any content
// language. Copy stays language-agnostic via {{language}} — not hardcoded Slovenian.
func repairedCategoryEnhancePromptMap() (company.LangPromptMap, error) {
tpl := strings.TrimSpace(aiprompts.CategoryEnhanceUserTemplate)
if tpl == "" {
return nil, fmt.Errorf("CategoryEnhanceUserTemplate is empty")
}
return company.LangPromptMap{
"sl": tpl,
company.LangPromptAny: tpl,
}, nil
}
func categoryEnhancePromptMapOK(m company.LangPromptMap, want company.LangPromptMap) bool {
if !company.HasAnyPrompt(m) || !company.HasAnyPrompt(want) {
return false
}
tpl := strings.TrimSpace(want[company.LangPromptAny])
if tpl == "" {
tpl = strings.TrimSpace(want["sl"])
}
if tpl == "" {
return false
}
// Accept already-repaired maps: every non-empty value equals the shared template
// and LangPromptAny (or legacy sl-only) is present.
hasKey := false
for lang, p := range m {
p = strings.TrimSpace(p)
if p == "" {
continue
}
if p != tpl {
return false
}
if lang == company.LangPromptAny || lang == "sl" {
hasKey = true
}
}
return hasKey
}
func repairCompanyCategoryEnhancePrompts(
ctx context.Context,
pool *pgxpool.Pool,
companyID uuid.UUID,
companyName string,
want company.LangPromptMap,
dryRun bool,
out *RepairCategoryEnhancePromptsResult,
) (int, error) {
rows, err := pool.Query(ctx, `
SELECT id, COALESCE(prompt, '{}'::jsonb)
FROM categories
WHERE company_id = $1`, companyID)
if err != nil {
return 0, fmt.Errorf("list categories for %s: %w", companyName, err)
}
defer rows.Close()
updatedHere := 0
for rows.Next() {
var id uuid.UUID
var raw []byte
if err := rows.Scan(&id, &raw); err != nil {
return updatedHere, err
}
out.CategoriesSeen++
m, err := company.DecodeLangPromptMap(raw)
if err != nil {
return updatedHere, fmt.Errorf("decode prompt category=%s company=%s: %w", id, companyName, err)
}
if !company.HasAnyPrompt(m) {
out.EmptySkipped++
continue
}
if categoryEnhancePromptMapOK(m, want) {
out.AlreadyOK++
continue
}
out.WouldUpdate++
if dryRun {
updatedHere++
continue
}
cleaned, err := company.SanitizeLangPromptMap(want, MaxCategoryPromptRunes)
if err != nil {
return updatedHere, fmt.Errorf("sanitize prompt category=%s: %w", id, err)
}
encoded, err := company.EncodeLangPromptMap(cleaned)
if err != nil {
return updatedHere, err
}
ct, err := pool.Exec(ctx, `
UPDATE categories
SET prompt = $3::jsonb, updated_at = now()
WHERE id = $1 AND company_id = $2`, id, companyID, string(encoded))
if err != nil {
return updatedHere, fmt.Errorf("update prompt category=%s: %w", id, err)
}
if ct.RowsAffected() == 0 {
return updatedHere, fmt.Errorf("category %s not updated (company mismatch?)", id)
}
out.Updated++
updatedHere++
}
return updatedHere, rows.Err()
}
@@ -0,0 +1,52 @@
package catalog
import (
"strings"
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
)
func TestRepairedCategoryEnhancePromptMap(t *testing.T) {
t.Parallel()
want, err := repairedCategoryEnhancePromptMap()
if err != nil {
t.Fatal(err)
}
tpl := strings.TrimSpace(aiprompts.CategoryEnhanceUserTemplate)
if want["sl"] != tpl {
t.Fatalf("want prompt->>'sl' = shared template")
}
if want[company.LangPromptAny] != tpl {
t.Fatalf("want * = shared template")
}
if !categoryEnhancePromptMapOK(want, want) {
t.Fatal("canonical map should be OK")
}
legacy := company.LangPromptMap{"sl": "<H2>legacy HTML marketing</H2>"}
if categoryEnhancePromptMapOK(legacy, want) {
t.Fatal("legacy HTML must need repair")
}
slOnly := company.LangPromptMap{"sl": tpl}
if !categoryEnhancePromptMapOK(slOnly, want) {
t.Fatal("sl-only repaired map should be OK (idempotent)")
}
starOnly := company.LangPromptMap{company.LangPromptAny: tpl}
if !categoryEnhancePromptMapOK(starOnly, want) {
t.Fatal("*-only repaired map should be OK (idempotent)")
}
}
func TestRepairTargetsUseSharedTemplate(t *testing.T) {
t.Parallel()
tpl := strings.TrimSpace(aiprompts.CategoryEnhanceUserTemplate)
for _, v := range []string{"name", "description", "attrs", "category", "language"} {
if !strings.Contains(tpl, "{{"+v+"}}") {
t.Fatalf("shared template missing {{%s}}", v)
}
}
if a1CompanyID == "" || platformDemoCompanyName == "" {
t.Fatal("missing company target constants")
}
}
+9 -5
View File
@@ -161,12 +161,11 @@ func ExtractProductImages(mapped, raw map[string]any) (main string, more []strin
break break
} }
} }
if imgs, ok := merged["images"].([]any); ok && len(imgs) > 0 { if urls := coerceToURLList(merged["images"]); len(urls) > 0 {
urls := coerceToURLList(imgs) if main == "" {
if main == "" && len(urls) > 0 {
main = urls[0] main = urls[0]
urls = urls[1:] urls = urls[1:]
} else if len(urls) > 0 && urls[0] == main { } else if urls[0] == main {
urls = urls[1:] urls = urls[1:]
} }
for _, u := range urls { for _, u := range urls {
@@ -175,6 +174,11 @@ func ExtractProductImages(mapped, raw map[string]any) (main string, more []strin
} }
} }
} }
// Feeds often leave main_image empty while moreimages/images hold URLs.
if main == "" && len(more) > 0 {
main = more[0]
more = more[1:]
}
if main != "" { if main != "" {
filtered := more[:0] filtered := more[:0]
for _, u := range more { for _, u := range more {
@@ -211,7 +215,7 @@ func coerceToURLString(value any) string {
} }
return "" return ""
case map[string]any: case map[string]any:
for _, k := range []string{"#text", "__cdata", "@_href", "@_url", "href", "url"} { for _, k := range []string{"#text", "__cdata", "@_href", "@_url", "href", "url", "src", "link", "image"} {
if u := coerceToURLString(v[k]); u != "" { if u := coerceToURLString(v[k]); u != "" {
return u return u
} }
+35
View File
@@ -54,3 +54,38 @@ func TestBuildMappedDataFromV1ItemOmitsEmptyNulls(t *testing.T) {
t.Fatal("EAN-only must not count as content") t.Fatal("EAN-only must not count as content")
} }
} }
func TestExtractProductImages_promotesMoreimagesWhenMainEmpty(t *testing.T) {
main, more := ExtractProductImages(map[string]any{
"main_image": "",
"moreimages": "https://cdn.example.com/a.jpg,https://cdn.example.com/b.jpg",
}, nil)
if main != "https://cdn.example.com/a.jpg" {
t.Fatalf("main=%q", main)
}
if len(more) != 1 || more[0] != "https://cdn.example.com/b.jpg" {
t.Fatalf("more=%v", more)
}
}
func TestExtractProductImages_imagesStringSliceAndSrcObjects(t *testing.T) {
main, more := ExtractProductImages(map[string]any{
"images": []string{
"https://cdn.example.com/main.jpg",
"https://cdn.example.com/2.jpg",
},
}, nil)
if main != "https://cdn.example.com/main.jpg" || len(more) != 1 || more[0] != "https://cdn.example.com/2.jpg" {
t.Fatalf("string slice main=%q more=%v", main, more)
}
main, more = ExtractProductImages(map[string]any{
"images": []any{
map[string]any{"src": "https://cdn.example.com/from-src.jpg"},
map[string]any{"src": "https://cdn.example.com/from-src-2.jpg"},
},
}, nil)
if main != "https://cdn.example.com/from-src.jpg" || len(more) != 1 || more[0] != "https://cdn.example.com/from-src-2.jpg" {
t.Fatalf("src objects main=%q more=%v", main, more)
}
}
@@ -0,0 +1,188 @@
package catalog
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
const enhanceInputHashKey = "enhance_input_hash"
// RepairWeakEnhanceHashesResult is the detailed report for weak-hash clearing.
type RepairWeakEnhanceHashesResult struct {
Cleared int64
Scanned int
ClearedRawIDs []uuid.UUID
}
// RepairWeakEnhanceHashes clears enhance_input_hash from field_sources and
// localized_content for processed products whose descriptions are weak /
// title-echo / filler (so the next enhance cannot hash-skip thin priors).
// Returns the number of products updated.
func RepairWeakEnhanceHashes(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int64, error) {
res, err := RepairWeakEnhanceHashesDetailed(ctx, pool, companyID)
return res.Cleared, err
}
// RepairWeakEnhanceHashesDetailed is RepairWeakEnhanceHashes plus scan/raw-id detail
// for admin Fix A1 hygiene reporting.
func RepairWeakEnhanceHashesDetailed(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (RepairWeakEnhanceHashesResult, error) {
var out RepairWeakEnhanceHashesResult
if pool == nil {
return out, fmt.Errorf("catalog pool not configured")
}
if companyID == uuid.Nil {
return out, ClientMsg("company_id is required")
}
tx, err := pool.Begin(ctx)
if err != nil {
return out, err
}
defer tx.Rollback(ctx)
out, err = repairWeakEnhanceHashesTx(ctx, tx, companyID)
if err != nil {
return out, err
}
if err := tx.Commit(ctx); err != nil {
return out, err
}
return out, nil
}
// RepairWeakEnhanceHashes clears weak enhance skip hashes for the company.
func (s *Service) RepairWeakEnhanceHashes(ctx context.Context, companyID uuid.UUID) (int64, error) {
if s == nil || s.Pool == nil {
return 0, fmt.Errorf("catalog service not configured")
}
return RepairWeakEnhanceHashes(ctx, s.Pool, companyID)
}
func repairWeakEnhanceHashesTx(ctx context.Context, tx pgx.Tx, companyID uuid.UUID) (RepairWeakEnhanceHashesResult, error) {
var out RepairWeakEnhanceHashesResult
rows, err := tx.Query(ctx, `
SELECT id, raw_product_id,
COALESCE(name, ''),
COALESCE(description, ''),
COALESCE(processed_name, ''),
COALESCE(processed_description, ''),
COALESCE(field_sources, '{}'::jsonb),
COALESCE(localized_content, '{}'::jsonb)
FROM processed_products
WHERE company_id = $1`, companyID)
if err != nil {
return out, fmt.Errorf("list processed products: %w", err)
}
defer rows.Close()
type pending struct {
id uuid.UUID
rawID uuid.UUID
fs []byte
loc []byte
}
var updates []pending
for rows.Next() {
out.Scanned++
var (
id, rawID uuid.UUID
name, desc, processedName, processedDesc string
fsRaw, locRaw []byte
)
if err := rows.Scan(&id, &rawID, &name, &desc, &processedName, &processedDesc, &fsRaw, &locRaw); err != nil {
return out, err
}
fs := map[string]any{}
if len(fsRaw) > 0 {
if err := json.Unmarshal(fsRaw, &fs); err != nil {
return out, err
}
}
if fs == nil {
fs = map[string]any{}
}
loc, err := company.DecodeLocalizedContent(locRaw)
if err != nil {
return out, err
}
changed := false
primaryDesc := strings.TrimSpace(processedDesc)
if primaryDesc == "" {
primaryDesc = desc
}
primaryName := strings.TrimSpace(processedName)
if primaryName == "" {
primaryName = name
}
if _, has := fs[enhanceInputHashKey]; has {
if company.IsWeakPriorEnhanceDescription(primaryDesc, primaryName, name) {
delete(fs, enhanceInputHashKey)
changed = true
}
}
for lang, fields := range loc {
d := strings.TrimSpace(fields.ProcessedDescription)
n := strings.TrimSpace(fields.ProcessedName)
if n == "" {
n = primaryName
}
if strings.TrimSpace(fields.EnhanceInputHash) == "" {
continue
}
if company.IsWeakPriorEnhanceDescription(d, n, primaryName, name) {
fields.EnhanceInputHash = ""
loc[lang] = fields
changed = true
}
}
if !changed {
continue
}
fsBytes, err := json.Marshal(fs)
if err != nil {
return out, err
}
locBytes, err := company.EncodeLocalizedContent(loc)
if err != nil {
return out, err
}
updates = append(updates, pending{id: id, rawID: rawID, fs: fsBytes, loc: locBytes})
}
if err := rows.Err(); err != nil {
return out, err
}
rows.Close()
seenRaw := map[uuid.UUID]struct{}{}
for _, u := range updates {
ct, err := tx.Exec(ctx, `
UPDATE processed_products
SET field_sources = $2::jsonb,
localized_content = $3::jsonb,
updated_at = now()
WHERE id = $1 AND company_id = $4`,
u.id, u.fs, u.loc, companyID)
if err != nil {
return out, fmt.Errorf("clear enhance hashes product=%s: %w", u.id, err)
}
if ct.RowsAffected() > 0 {
out.Cleared++
if _, ok := seenRaw[u.rawID]; !ok {
seenRaw[u.rawID] = struct{}{}
out.ClearedRawIDs = append(out.ClearedRawIDs, u.rawID)
}
}
}
return out, nil
}
@@ -0,0 +1,137 @@
package catalog
import (
"context"
"encoding/json"
"os"
"strings"
"testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestClearWeakEnhanceHashesDecision(t *testing.T) {
t.Parallel()
title := "Acme Widget Pro 24"
weak := title
strong := "Acme Widget Pro 24 is a durable retail widget built for everyday warehouse use with a reinforced frame."
if !company.IsWeakPriorEnhanceDescription(weak, title) {
t.Fatal("title-echo must be weak")
}
if company.IsWeakPriorEnhanceDescription(strong, title) {
t.Fatal("strong description must not be weak")
}
}
func TestRepairWeakEnhanceHashes_nilPool(t *testing.T) {
t.Parallel()
_, err := RepairWeakEnhanceHashes(context.Background(), nil, uuid.New())
if err == nil {
t.Fatal("expected error for nil pool")
}
_, err = RepairWeakEnhanceHashes(context.Background(), &pgxpool.Pool{}, uuid.Nil)
if err == nil {
t.Fatal("expected error for nil company")
}
}
func TestRepairWeakEnhanceHashes_integration(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("DATABASE_URL"))
if dsn == "" {
t.Skip("DATABASE_URL not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatalf("connect: %v", err)
}
defer pool.Close()
var companyID uuid.UUID
err = pool.QueryRow(ctx, `
SELECT id FROM companies
WHERE lower(name) IN ('platform demo', 'demo')
ORDER BY CASE WHEN lower(name) = 'platform demo' THEN 0 ELSE 1 END
LIMIT 1`).Scan(&companyID)
if err != nil {
t.Skipf("Platform Demo company not found: %v", err)
}
title := "Repair Hash Probe Widget X"
weakDesc := title
hash := "deadbeefcafebabe0123456789abcdef0123456789abcdef0123456789abcdef"
loc := company.LocalizedContent{
"en": {
ProcessedName: title,
ProcessedDescription: weakDesc,
EnhanceInputHash: hash,
},
}
locJSON, err := company.EncodeLocalizedContent(loc)
if err != nil {
t.Fatalf("encode loc: %v", err)
}
fs, _ := json.Marshal(map[string]any{enhanceInputHashKey: hash, "name": "ai_enhance"})
tx, err := pool.Begin(ctx)
if err != nil {
t.Fatalf("begin: %v", err)
}
defer tx.Rollback(ctx)
var rawID, ppID uuid.UUID
gtin := "8700999111222"
err = tx.QueryRow(ctx, `
INSERT INTO raw_products (company_id, gtin, raw_data, mapped_data, is_processed, processing_status)
VALUES ($1, $2, '{}'::jsonb, '{}'::jsonb, true, 'processed')
RETURNING id`, companyID, gtin).Scan(&rawID)
if err != nil {
t.Fatalf("insert raw: %v", err)
}
err = tx.QueryRow(ctx, `
INSERT INTO processed_products (
company_id, raw_product_id, product_id, name, processed_name, description, processed_description,
status, attributes, processed_attributes, gpt_response, field_sources, localized_content, ai_provider_mode
) VALUES (
$1, $2, $3, $4, $4, $5, $5, 'needs_review', '{}'::jsonb, '{}'::jsonb, '{}'::jsonb, $6::jsonb, $7::jsonb, 'internal'
) RETURNING id`, companyID, rawID, gtin, title, weakDesc, fs, locJSON).Scan(&ppID)
if err != nil {
t.Fatalf("insert processed: %v", err)
}
res, err := repairWeakEnhanceHashesTx(ctx, tx, companyID)
if err != nil {
t.Fatalf("repair: %v", err)
}
if res.Cleared < 1 {
t.Fatalf("expected at least 1 cleared, got %d (scanned=%d)", res.Cleared, res.Scanned)
}
var fsOut, locOut []byte
err = tx.QueryRow(ctx, `
SELECT COALESCE(field_sources, '{}'::jsonb), COALESCE(localized_content, '{}'::jsonb)
FROM processed_products WHERE id = $1`, ppID).Scan(&fsOut, &locOut)
if err != nil {
t.Fatalf("reload: %v", err)
}
var fsMap map[string]any
_ = json.Unmarshal(fsOut, &fsMap)
if _, ok := fsMap[enhanceInputHashKey]; ok {
t.Fatalf("field_sources hash still present: %v", fsMap[enhanceInputHashKey])
}
gotLoc, err := company.DecodeLocalizedContent(locOut)
if err != nil {
t.Fatalf("decode loc: %v", err)
}
if h := gotLoc["en"].EnhanceInputHash; h != "" {
t.Fatalf("localized hash still present: %q", h)
}
if gotLoc["en"].ProcessedDescription != weakDesc {
t.Fatalf("description should be preserved, got %q", gotLoc["en"].ProcessedDescription)
}
// Roll back probe rows — do not leave smoke data on Platform Demo.
}
+33 -12
View File
@@ -260,7 +260,7 @@ func enrichCategoryPrompts(ctx context.Context, pool *pgxpool.Pool, companyID uu
} }
primary := company.LoadLanguage(ctx, pool, companyID) primary := company.LoadLanguage(ctx, pool, companyID)
item["prompts"] = prompts item["prompts"] = prompts
item["prompt"] = company.PromptForLanguage(prompts, primary) item["prompt"] = company.PromptForLanguage(prompts, primary, primary)
item["has_prompt"] = company.HasAnyPrompt(prompts) item["has_prompt"] = company.HasAnyPrompt(prompts)
return item, nil return item, nil
} }
@@ -668,8 +668,11 @@ func processedProductsCountFromSQL(needsRawJoin bool) string {
// Enrichment coverage SQL predicates (alias p = processed_products, r = raw_products). // Enrichment coverage SQL predicates (alias p = processed_products, r = raw_products).
const ( const (
processedHasNameSQL = `(COALESCE(NULLIF(p.processed_name, ''), NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') <> '')` // Prefer processed_* so dashboard product fields match V1 process items for the same EAN.
processedHasDescriptionSQL = `(COALESCE(NULLIF(p.processed_description, ''), NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '') <> '')` processedPreferredNameSQL = `COALESCE(NULLIF(p.processed_name, ''), NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '')`
processedPreferredDescriptionSQL = `COALESCE(NULLIF(p.processed_description, ''), NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '')`
processedHasNameSQL = `(` + processedPreferredNameSQL + ` <> '')`
processedHasDescriptionSQL = `(` + processedPreferredDescriptionSQL + ` <> '')`
processedHasCategorySQL = `(COALESCE(NULLIF(p.category, ''), '') <> '' AND lower(p.category) <> 'none')` processedHasCategorySQL = `(COALESCE(NULLIF(p.category, ''), '') <> '' AND lower(p.category) <> 'none')`
// Resolve display name / canonical unique_id when products store unique_id, UUID id, or name. // Resolve display name / canonical unique_id when products store unique_id, UUID id, or name.
processedCategoryResolveJoin = ` processedCategoryResolveJoin = `
@@ -1093,7 +1096,7 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID
func(ctx context.Context) ([]map[string]any, error) { func(ctx context.Context) ([]map[string]any, error) {
rows, err := s.Pool.Query(ctx, fmt.Sprintf(` rows, err := s.Pool.Query(ctx, fmt.Sprintf(`
SELECT p.id, p.product_id, SELECT p.id, p.product_id,
COALESCE(NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') AS name, `+processedPreferredNameSQL+` AS name,
COALESCE(NULLIF(p.processed_name, ''), '') AS processed_name, COALESCE(NULLIF(p.processed_name, ''), '') AS processed_name,
p.category, p.category,
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name, COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name,
@@ -1102,9 +1105,9 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID
f.name AS feed_name, f.name AS feed_name,
f.last_synced_at AS feed_last_synced_at, f.last_synced_at AS feed_last_synced_at,
r.updated_at AS raw_updated_at, r.updated_at AS raw_updated_at,
(COALESCE(NULLIF(p.processed_name, ''), NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') <> '') AS has_name, (`+processedPreferredNameSQL+` <> '') AS has_name,
(COALESCE(NULLIF(p.processed_name, ''), '') <> '') AS has_processed_name, (COALESCE(NULLIF(p.processed_name, ''), '') <> '') AS has_processed_name,
(COALESCE(NULLIF(p.processed_description, ''), NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '') <> '') AS has_description, (`+processedPreferredDescriptionSQL+` <> '') AS has_description,
(COALESCE(NULLIF(p.processed_description, ''), '') <> '') AS has_processed_description, (COALESCE(NULLIF(p.processed_description, ''), '') <> '') AS has_processed_description,
(COALESCE(NULLIF(p.category, ''), '') <> '' AND lower(p.category) <> 'none') AS has_category, (COALESCE(NULLIF(p.category, ''), '') <> '' AND lower(p.category) <> 'none') AS has_category,
`+processedHasAttributesSQL+` AS has_attributes, `+processedHasAttributesSQL+` AS has_attributes,
@@ -1120,7 +1123,7 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID
return nil, err return nil, err
} }
defer rows.Close() defer rows.Close()
return scanMaps(rows, []string{ items, err := scanMaps(rows, []string{
"id", "product_id", "name", "processed_name", "category", "category_name", "category_unique_id", "id", "product_id", "name", "processed_name", "category", "category_name", "category_unique_id",
"status", "raw_product_id", "feed_id", "gtin", "status", "raw_product_id", "feed_id", "gtin",
"feed_name", "feed_last_synced_at", "raw_updated_at", "feed_name", "feed_last_synced_at", "raw_updated_at",
@@ -1128,6 +1131,15 @@ func (s *Service) ListProcessedProducts(ctx context.Context, companyID uuid.UUID
"has_eprel", "has_eprel",
"created_at", "updated_at", "created_at", "updated_at",
}) })
if err != nil {
return nil, err
}
for _, item := range items {
if name, ok := item["name"].(string); ok && strings.TrimSpace(name) != "" {
item["title"] = name
}
}
return items, nil
}, },
) )
} }
@@ -1172,12 +1184,12 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u
func(ctx context.Context) ([]map[string]any, error) { func(ctx context.Context) ([]map[string]any, error) {
rows, err := s.Pool.Query(ctx, fmt.Sprintf(` rows, err := s.Pool.Query(ctx, fmt.Sprintf(`
SELECT p.id, p.product_id, SELECT p.id, p.product_id,
COALESCE(NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') AS name, `+processedPreferredNameSQL+` AS name,
p.category, p.category,
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name, COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id, COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id,
p.status, p.raw_product_id, p.feed_id, r.gtin, p.status, p.raw_product_id, p.feed_id, r.gtin,
COALESCE(NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '') AS description, `+processedPreferredDescriptionSQL+` AS description,
p.processed_name, p.processed_description, p.processed_name, p.processed_description,
COALESCE(p.meta_title, ''), COALESCE(p.meta_description, ''), COALESCE(p.meta_title, ''), COALESCE(p.meta_description, ''),
p.attributes, p.processed_attributes, r.mapped_data, p.attributes, p.processed_attributes, r.mapped_data,
@@ -1190,7 +1202,7 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u
return nil, err return nil, err
} }
defer rows.Close() defer rows.Close()
return scanMaps(rows, []string{ items, err := scanMaps(rows, []string{
"id", "product_id", "name", "category", "category_name", "category_unique_id", "id", "product_id", "name", "category", "category_name", "category_unique_id",
"status", "raw_product_id", "feed_id", "gtin", "status", "raw_product_id", "feed_id", "gtin",
"description", "processed_name", "processed_description", "description", "processed_name", "processed_description",
@@ -1198,6 +1210,13 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u
"attributes", "processed_attributes", "mapped_data", "attributes", "processed_attributes", "mapped_data",
"created_at", "updated_at", "created_at", "updated_at",
}) })
if err != nil {
return nil, err
}
for _, item := range items {
alignProductFieldsWithV1(item)
}
return items, nil
}, },
) )
} }
@@ -1205,11 +1224,11 @@ func (s *Service) ListProcessedProductsDetailed(ctx context.Context, companyID u
func (s *Service) GetProcessedProduct(ctx context.Context, companyID, id uuid.UUID) (map[string]any, error) { func (s *Service) GetProcessedProduct(ctx context.Context, companyID, id uuid.UUID) (map[string]any, error) {
row := s.Pool.QueryRow(ctx, ` row := s.Pool.QueryRow(ctx, `
SELECT p.id, p.product_id, SELECT p.id, p.product_id,
COALESCE(NULLIF(p.name, ''), NULLIF(r.mapped_data->>'name', ''), NULLIF(r.mapped_data->>'title', ''), '') AS name, `+processedPreferredNameSQL+` AS name,
p.category, p.category,
COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name, COALESCE(cat.name, NULLIF(BTRIM(p.category), '')) AS category_name,
COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id, COALESCE(cat.unique_id, NULLIF(BTRIM(p.category), '')) AS category_unique_id,
COALESCE(NULLIF(p.description, ''), NULLIF(r.mapped_data->>'description', ''), '') AS description, `+processedPreferredDescriptionSQL+` AS description,
p.processed_name, p.processed_description, p.processed_name, p.processed_description,
p.status, p.attributes, p.processed_attributes, r.gtin, r.mapped_data, p.status, p.attributes, p.processed_attributes, r.gtin, r.mapped_data,
COALESCE(p.feed_id, r.feed_id) AS feed_id, COALESCE(p.feed_id, r.feed_id) AS feed_id,
@@ -1241,6 +1260,7 @@ func (s *Service) GetProcessedProduct(ctx context.Context, companyID, id uuid.UU
return nil, err return nil, err
} }
linkFeedSpecificationsIntoProduct(item) linkFeedSpecificationsIntoProduct(item)
alignProductFieldsWithV1(item)
primary := company.LoadLanguage(ctx, s.Pool, companyID) primary := company.LoadLanguage(ctx, s.Pool, companyID)
item["content_language"] = primary item["content_language"] = primary
item["content_languages"] = company.LoadContentLanguages(ctx, s.Pool, companyID) item["content_languages"] = company.LoadContentLanguages(ctx, s.Pool, companyID)
@@ -1303,6 +1323,7 @@ func (s *Service) GetRawProduct(ctx context.Context, companyID, id uuid.UUID) (m
return nil, err return nil, err
} }
linkFeedSpecificationsIntoProduct(item) linkFeedSpecificationsIntoProduct(item)
alignProductFieldsWithV1(item)
primary := company.LoadLanguage(ctx, s.Pool, companyID) primary := company.LoadLanguage(ctx, s.Pool, companyID)
item["content_language"] = primary item["content_language"] = primary
item["content_languages"] = company.LoadContentLanguages(ctx, s.Pool, companyID) item["content_languages"] = company.LoadContentLanguages(ctx, s.Pool, companyID)
+43 -7
View File
@@ -11,7 +11,12 @@ import (
"github.com/jackc/pgx/v5/pgxpool" "github.com/jackc/pgx/v5/pgxpool"
) )
// LangPromptAny is the wildcard key for a single shared prompt used for every
// content language (avoid duplicating the same text per lang).
const LangPromptAny = "*"
// LangPromptMap is language-code → prompt text for category / template overrides. // LangPromptMap is language-code → prompt text for category / template overrides.
// Optional key LangPromptAny ("*") is a shared overlay for all languages.
type LangPromptMap map[string]string type LangPromptMap map[string]string
// LocalizedFields holds AI/output fields for one content language. // LocalizedFields holds AI/output fields for one content language.
@@ -27,15 +32,16 @@ type LocalizedFields struct {
type LocalizedContent map[string]LocalizedFields type LocalizedContent map[string]LocalizedFields
// SanitizeLangPromptMap validates language codes, sanitizes prompts, and drops empties. // SanitizeLangPromptMap validates language codes, sanitizes prompts, and drops empties.
// Accepts LangPromptAny ("*") as a shared any-language prompt key.
func SanitizeLangPromptMap(in map[string]string, maxRunes int) (LangPromptMap, error) { func SanitizeLangPromptMap(in map[string]string, maxRunes int) (LangPromptMap, error) {
out := make(LangPromptMap) out := make(LangPromptMap)
if len(in) == 0 { if len(in) == 0 {
return out, nil return out, nil
} }
for lang, prompt := range in { for lang, prompt := range in {
code, err := ParseLanguage(lang, false) code, err := parseLangPromptKey(lang)
if err != nil { if err != nil {
return nil, fmt.Errorf("unsupported language %q", lang) return nil, err
} }
p := strings.TrimSpace(security.SanitizePrompt(prompt, maxRunes)) p := strings.TrimSpace(security.SanitizePrompt(prompt, maxRunes))
if p == "" { if p == "" {
@@ -46,17 +52,47 @@ func SanitizeLangPromptMap(in map[string]string, maxRunes int) (LangPromptMap, e
return out, nil return out, nil
} }
// PromptForLanguage returns the prompt for lang, or empty if unset. func parseLangPromptKey(raw string) (string, error) {
func PromptForLanguage(m LangPromptMap, lang string) string { key := strings.TrimSpace(raw)
if key == LangPromptAny {
return LangPromptAny, nil
}
code, err := ParseLanguage(key, false)
if err != nil {
return "", fmt.Errorf("unsupported language %q", raw)
}
return code, nil
}
// PromptForLanguage resolves a prompt with fallback:
// requested lang → LangPromptAny ("*") → primary → "" (caller may then use built-in).
// Empty primary skips the primary step. Does not invent cross-lang text beyond this chain.
func PromptForLanguage(m LangPromptMap, lang, primary string) string {
if len(m) == 0 { if len(m) == 0 {
return "" return ""
} }
code, err := ParseLanguage(lang, true) try := func(code string) string {
if err != nil { code = strings.TrimSpace(code)
code = DefaultLanguage if code == "" {
return ""
} }
return strings.TrimSpace(m[code]) return strings.TrimSpace(m[code])
} }
if code, err := ParseLanguage(lang, true); err == nil {
if p := try(code); p != "" {
return p
}
}
if p := try(LangPromptAny); p != "" {
return p
}
if prim, err := ParseLanguage(primary, false); err == nil {
if p := try(prim); p != "" {
return p
}
}
return ""
}
// HasAnyPrompt reports whether any language has a non-empty prompt. // HasAnyPrompt reports whether any language has a non-empty prompt.
func HasAnyPrompt(m LangPromptMap) bool { func HasAnyPrompt(m LangPromptMap) bool {
+25 -3
View File
@@ -17,6 +17,7 @@ func TestSanitizeLangPromptMap(t *testing.T) {
m, err := SanitizeLangPromptMap(map[string]string{ m, err := SanitizeLangPromptMap(map[string]string{
"SL": " hello {{name}} ", "SL": " hello {{name}} ",
"en": "", "en": "",
"*": " shared ",
}, 100) }, 100)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -27,16 +28,37 @@ func TestSanitizeLangPromptMap(t *testing.T) {
if _, ok := m["en"]; ok { if _, ok := m["en"]; ok {
t.Fatalf("empty en should be dropped: %#v", m) t.Fatalf("empty en should be dropped: %#v", m)
} }
if m[LangPromptAny] != "shared" {
t.Fatalf("wildcard missing: %#v", m)
}
} }
func TestPromptForLanguage(t *testing.T) { func TestPromptForLanguage(t *testing.T) {
t.Parallel() t.Parallel()
m := LangPromptMap{"sl": "slo", "en": "eng"} m := LangPromptMap{"sl": "slo", "en": "eng"}
if got := PromptForLanguage(m, "SL"); got != "slo" { if got := PromptForLanguage(m, "SL", ""); got != "slo" {
t.Fatalf("got %q", got) t.Fatalf("got %q", got)
} }
if got := PromptForLanguage(m, "de"); got != "" { if got := PromptForLanguage(m, "de", ""); got != "" {
t.Fatalf("expected empty, got %q", got) t.Fatalf("expected empty without primary, got %q", got)
}
// sl-only map: exact hit for sl
slOnly := LangPromptMap{"sl": "slo-only"}
if got := PromptForLanguage(slOnly, "sl", "sl"); got != "slo-only" {
t.Fatalf("sl exact: got %q", got)
}
// sl-only map: en request falls back to primary=sl
if got := PromptForLanguage(slOnly, "en", "sl"); got != "slo-only" {
t.Fatalf("en→primary sl: got %q", got)
}
// wildcard before primary
anyMap := LangPromptMap{"*": "shared", "sl": "slo"}
if got := PromptForLanguage(anyMap, "de", "sl"); got != "shared" {
t.Fatalf("wildcard before primary: got %q", got)
}
// explicit lang beats wildcard
if got := PromptForLanguage(anyMap, "sl", "en"); got != "slo" {
t.Fatalf("explicit beats *: got %q", got)
} }
} }
+151
View File
@@ -0,0 +1,151 @@
package company
import "strings"
// minUsableProductDescRunes is the floor for a prior description that may hash-skip
// the enhance LLM. Shorter copy is treated as thin and forced through enhance.
const minUsableProductDescRunes = 40
// shortBoilerplateDescRunes: short copy without product-fact token overlap is weak
// even when longer than minUsableProductDescRunes (generic one-liners).
const shortBoilerplateDescRunes = 96
// weakFillerPhrases are known heuristic / placeholder snippets that must never
// hash-skip enhance (mock-llm / invent fallbacks historically produced these).
var weakFillerPhrases = []string{
"ready for retail listing",
"quality product ready",
"product description",
"based on available specifications",
"with available specifications",
"available catalog details",
"pripravljeno za prodajo",
"na podlagi razpoložljivih specifikacij",
}
// IsWeakPriorEnhanceDescription reports empty, too-short, title-echo, known filler,
// or short boilerplate without overlapping tokens from title/fact sources.
func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
priorDesc = strings.TrimSpace(priorDesc)
if priorDesc == "" || priorDesc == "<nil>" {
return true
}
if len([]rune(priorDesc)) < minUsableProductDescRunes {
return true
}
for _, title := range titles {
if DescriptionEchoesTitle(priorDesc, title) {
return true
}
}
if ContainsWeakFillerPhrase(priorDesc) {
return true
}
if len([]rune(priorDesc)) <= shortBoilerplateDescRunes &&
!descriptionOverlapsProductFacts(priorDesc, titles...) {
return true
}
return false
}
// ContainsWeakFillerPhrase reports known placeholder / invent-fallback snippets.
func ContainsWeakFillerPhrase(desc string) bool {
lower := strings.ToLower(desc)
for _, p := range weakFillerPhrases {
if p != "" && strings.Contains(lower, p) {
return true
}
}
return false
}
var weakDescStopTokens = map[string]struct{}{
"a": {}, "an": {}, "the": {}, "and": {}, "or": {}, "of": {}, "in": {}, "on": {}, "for": {},
"to": {}, "with": {}, "from": {}, "by": {}, "is": {}, "are": {}, "this": {}, "that": {},
"product": {}, "products": {}, "category": {}, "description": {}, "specs": {}, "spec": {},
"key": {}, "je": {}, "v": {}, "za": {}, "z": {}, "iz": {}, "ter": {}, "ali": {},
}
func significantDescTokens(s string) map[string]struct{} {
s = strings.ToLower(s)
out := map[string]struct{}{}
var b strings.Builder
flush := func() {
tok := b.String()
b.Reset()
if len(tok) < 3 {
return
}
if _, stop := weakDescStopTokens[tok]; stop {
return
}
out[tok] = struct{}{}
}
for _, r := range s {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' {
b.WriteRune(r)
continue
}
// Keep Latin-extended letters (Slovenian čšž etc.) via unicode letter-ish: non-space punctuation splits.
if r > 127 && ((r >= 'à' && r <= 'ÿ') || r == 'č' || r == 'š' || r == 'ž' || r == 'ć' || r == 'đ') {
b.WriteRune(r)
continue
}
flush()
}
flush()
return out
}
// descriptionOverlapsProductFacts is true when desc shares significant tokens with
// any fact source (title, brand, model, …).
func descriptionOverlapsProductFacts(desc string, factSources ...string) bool {
descToks := significantDescTokens(desc)
if len(descToks) == 0 {
return false
}
overlap := 0
seen := map[string]struct{}{}
for _, src := range factSources {
for tok := range significantDescTokens(src) {
if _, ok := descToks[tok]; !ok {
continue
}
if _, dup := seen[tok]; dup {
continue
}
seen[tok] = struct{}{}
overlap++
if overlap >= 2 || len(tok) >= 5 {
return true
}
}
}
return overlap >= 1 && len(seen) >= 1 && len([]rune(desc)) >= minUsableProductDescRunes+20
}
func normalizeForEchoCompare(s string) string {
return strings.Join(strings.Fields(strings.ToLower(strings.TrimSpace(s))), " ")
}
// DescriptionEchoesTitle is true when desc is EqualFold or near-equal to title
// (whitespace-normalized, or one contains the other with only a tiny length delta).
func DescriptionEchoesTitle(desc, title string) bool {
d := normalizeForEchoCompare(desc)
t := normalizeForEchoCompare(title)
if d == "" || t == "" {
return false
}
if d == t {
return true
}
shorter, longer := d, t
if len(d) > len(t) {
shorter, longer = t, d
}
if !strings.Contains(longer, shorter) {
return false
}
delta := len(longer) - len(shorter)
return delta <= 8 && float64(len(shorter))/float64(len(longer)) >= 0.85
}
+145
View File
@@ -0,0 +1,145 @@
package eprel
import (
"fmt"
"strings"
)
// ExtractFromAttrs builds the nested EPREL object used by V1 process items and
// the dashboard product API (id / label / pdf / energy_class / energy_scale).
// Returns nil when no usable EPREL fields are present.
//
// Nested attrs["eprel"] may use flat export aliases (eprel_id, eprel_label, …);
// those are normalized onto the public keys. Flat eprel_* keys fill any gaps.
func ExtractFromAttrs(attrs map[string]any) any {
if attrs == nil {
return nil
}
out := map[string]any{}
if e, ok := attrs["eprel"]; ok && e != nil {
switch t := e.(type) {
case map[string]any:
for k, v := range t {
if s := attrString(v); s == "" {
continue
} else {
out[strings.TrimSpace(k)] = s
}
}
default:
if s := attrString(e); s != "" {
return map[string]any{"id": s}
}
return nil
}
}
put := func(dstKey string, keys ...string) {
if attrString(out[dstKey]) != "" {
return
}
for _, k := range keys {
if s := attrString(attrs[k]); s != "" {
out[dstKey] = s
return
}
// Also accept aliases already copied from a nested object.
if s := attrString(out[k]); s != "" {
out[dstKey] = s
return
}
}
}
put("id", "eprel_id", "EPRELID", "eprelId")
put("label", "eprel_label", "eprel_label_url")
put("pdf", "eprel_pdf", "eprel_pdf_url")
put("energy_class", "eprel_energy_class", "energyClass")
put("energy_scale", "eprel_energy_scale", "energyClassRange", "energyScale")
return NormalizeShape(out)
}
// NormalizeShape keeps only the public EPREL keys with non-empty string values.
// Alias keys (eprel_id, …) are folded onto id/label/pdf/energy_class/energy_scale.
func NormalizeShape(v any) any {
if v == nil {
return nil
}
m, ok := v.(map[string]any)
if !ok {
return nil
}
if len(m) == 0 {
return nil
}
out := map[string]any{}
for k, val := range m {
if s := attrString(val); s != "" {
out[strings.TrimSpace(k)] = s
}
}
fold := func(dst string, aliases ...string) {
if attrString(out[dst]) != "" {
for _, a := range aliases {
delete(out, a)
}
return
}
for _, a := range aliases {
if s := attrString(out[a]); s != "" {
out[dst] = s
break
}
}
for _, a := range aliases {
delete(out, a)
}
}
fold("id", "eprel_id", "EPRELID", "eprelId")
fold("label", "eprel_label", "eprel_label_url")
fold("pdf", "eprel_pdf", "eprel_pdf_url")
fold("energy_class", "eprel_energy_class", "energyClass")
fold("energy_scale", "eprel_energy_scale", "energyClassRange", "energyScale")
canonical := map[string]any{}
for _, k := range []string{"id", "label", "pdf", "energy_class", "energy_scale"} {
if s := attrString(out[k]); s != "" {
canonical[k] = s
}
}
if len(canonical) == 0 {
return nil
}
return canonical
}
func attrString(v any) string {
if v == nil {
return ""
}
switch t := v.(type) {
case string:
return strings.TrimSpace(t)
case float64:
// JSON numbers for numeric eprel ids.
if t == float64(int64(t)) {
return strings.TrimSpace(fmt.Sprintf("%.0f", t))
}
return strings.TrimSpace(fmt.Sprint(t))
case int:
return fmt.Sprintf("%d", t)
case int64:
return fmt.Sprintf("%d", t)
case map[string]any:
for _, k := range []string{"value", "#text", "text", "id"} {
if s := attrString(t[k]); s != "" {
return s
}
}
return ""
default:
s := strings.TrimSpace(fmt.Sprint(v))
if s == "" || s == "<nil>" {
return ""
}
return s
}
}
+84
View File
@@ -0,0 +1,84 @@
package eprel
import "testing"
func TestExtractFromAttrs_flatKeys(t *testing.T) {
got := ExtractFromAttrs(map[string]any{
"eprel_id": "1632113",
"eprel_label": "https://eprel.ec.europa.eu/api/product/1632113/labels?format=png",
"eprel_pdf_url": "https://eprel.ec.europa.eu/fiches/x.pdf",
"eprel_energy_class": "A",
"brand": "Samsung",
})
m, ok := got.(map[string]any)
if !ok {
t.Fatalf("type=%T", got)
}
if m["id"] != "1632113" || m["label"] == nil || m["pdf"] == nil || m["energy_class"] != "A" {
t.Fatalf("got=%v", m)
}
if _, leak := m["eprel_id"]; leak {
t.Fatalf("alias leaked: %v", m)
}
}
func TestExtractFromAttrs_nestedAliasesNormalized(t *testing.T) {
got := ExtractFromAttrs(map[string]any{
"eprel": map[string]any{
"eprel_id": "2208111",
"eprel_label": "https://eprel.ec.europa.eu/api/product/2208111/labels?format=png",
"eprel_pdf": "https://eprel.ec.europa.eu/fiches/y.pdf",
"eprel_energy_class": "E",
"source": "eprel_api",
},
})
m, ok := got.(map[string]any)
if !ok {
t.Fatalf("type=%T", got)
}
if m["id"] != "2208111" {
t.Fatalf("id=%v", m["id"])
}
if m["label"] == nil || m["pdf"] == nil {
t.Fatalf("missing label/pdf: %v", m)
}
if m["energy_class"] != "E" {
t.Fatalf("energy_class=%v", m["energy_class"])
}
if _, ok := m["eprel_id"]; ok {
t.Fatalf("eprel_id alias must be folded: %v", m)
}
if _, ok := m["source"]; ok {
t.Fatalf("non-canonical keys must be dropped: %v", m)
}
}
func TestExtractFromAttrs_nestedPartialFilledFromFlat(t *testing.T) {
got := ExtractFromAttrs(map[string]any{
"eprel": map[string]any{
"id": "1632113",
"label": "https://eprel.ec.europa.eu/api/product/1632113/labels?format=png",
},
"eprel_pdf": "https://eprel.ec.europa.eu/fiches/z.pdf",
"eprel_energy_class": "A",
})
m, ok := got.(map[string]any)
if !ok {
t.Fatalf("type=%T", got)
}
if m["id"] != "1632113" || m["pdf"] == nil || m["energy_class"] != "A" {
t.Fatalf("got=%v", m)
}
}
func TestNormalizeShape_empty(t *testing.T) {
if NormalizeShape(nil) != nil {
t.Fatal("nil")
}
if NormalizeShape(map[string]any{}) != nil {
t.Fatal("empty")
}
if NormalizeShape(map[string]any{"eprel_id": ""}) != nil {
t.Fatal("blank")
}
}
+10
View File
@@ -32,6 +32,16 @@ func TestNormalizeAndExtractID(t *testing.T) {
if got := ExtractID(mapped2); got != "777" { if got := ExtractID(mapped2); got != "777" {
t.Fatalf("mapped key=%q", got) t.Fatalf("mapped key=%q", got)
} }
attrsOnly := map[string]any{"eprel_id": "888"}
if got := ExtractID(nil, nil, attrsOnly); got != "888" {
t.Fatalf("attrs extract=%q", got)
}
if got := ExtractID(map[string]any{"eprel_id": "0000"}); got != "" {
t.Fatalf("placeholder must be empty, got %q", got)
}
if got := NormalizeID("0000"); got != "" {
t.Fatalf("NormalizeID placeholder=%q", got)
}
} }
func TestClientFetch_httptest(t *testing.T) { func TestClientFetch_httptest(t *testing.T) {
+44 -12
View File
@@ -16,32 +16,34 @@ var eprelIDKeys = []string{
} }
// NormalizeID coerces XML/API values (string, number, {"#text": ...}) to a trimmed ID. // NormalizeID coerces XML/API values (string, number, {"#text": ...}) to a trimmed ID.
// Placeholder feed values like "0" / "0000" become empty so the EPREL step is not
// falsely "skipped" after a doomed fetch attempt.
func NormalizeID(v any) string { func NormalizeID(v any) string {
if v == nil { if v == nil {
return "" return ""
} }
var s string
switch t := v.(type) { switch t := v.(type) {
case string: case string:
return strings.TrimSpace(t) s = strings.TrimSpace(t)
case json.Number: case json.Number:
s := strings.TrimSpace(t.String()) s = strings.TrimSpace(t.String())
if i := strings.IndexByte(s, '.'); i >= 0 { if i := strings.IndexByte(s, '.'); i >= 0 {
s = s[:i] s = s[:i]
} }
return s
case float64: case float64:
if t != t { // NaN if t != t { // NaN
return "" return ""
} }
return strconv.FormatInt(int64(t), 10) s = strconv.FormatInt(int64(t), 10)
case float32: case float32:
return strconv.FormatInt(int64(t), 10) s = strconv.FormatInt(int64(t), 10)
case int: case int:
return strconv.Itoa(t) s = strconv.Itoa(t)
case int64: case int64:
return strconv.FormatInt(t, 10) s = strconv.FormatInt(t, 10)
case int32: case int32:
return strconv.FormatInt(int64(t), 10) s = strconv.FormatInt(int64(t), 10)
case json.RawMessage: case json.RawMessage:
var decoded any var decoded any
if err := json.Unmarshal(t, &decoded); err != nil { if err := json.Unmarshal(t, &decoded); err != nil {
@@ -55,14 +57,44 @@ func NormalizeID(v any) string {
if text, ok := t["text"]; ok { if text, ok := t["text"]; ok {
return NormalizeID(text) return NormalizeID(text)
} }
return ""
case []any:
for _, item := range t {
if id := NormalizeID(item); id != "" {
return id
}
}
return ""
case []string:
for _, item := range t {
if id := NormalizeID(item); id != "" {
return id
}
}
return ""
default: default:
s := strings.TrimSpace(fmt.Sprint(t)) s = strings.TrimSpace(fmt.Sprint(t))
if s == "" || s == "<nil>" { if s == "" || s == "<nil>" {
return "" return ""
} }
}
if isPlaceholderEPRELID(s) {
return ""
}
return s return s
} }
return ""
func isPlaceholderEPRELID(id string) bool {
id = strings.TrimSpace(id)
if id == "" {
return true
}
for _, r := range id {
if r != '0' {
return false
}
}
return true
} }
// IsValidID reports whether v normalizes to a non-empty EPREL registration id. // IsValidID reports whether v normalizes to a non-empty EPREL registration id.
@@ -70,8 +102,8 @@ func IsValidID(v any) bool {
return NormalizeID(v) != "" return NormalizeID(v) != ""
} }
// ExtractID finds an EPREL ID in mapped and/or raw product field maps // ExtractID finds an EPREL ID in mapped, raw, and/or attribute field maps
// (vendor feeds often use <EPRELID/> / eprel_id). // (vendor feeds often use <EPRELID/> / eprel_id; specs parse may put it only in attrs).
func ExtractID(sources ...map[string]any) string { func ExtractID(sources ...map[string]any) string {
for _, src := range sources { for _, src := range sources {
if src == nil { if src == nil {
@@ -0,0 +1,105 @@
package httpapi
import (
"net/http"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
)
// handleAdminFixCompanyCatalog runs in-place Fix A1 / catalog hygiene for one company.
// Single route: POST /api/admin/companies/{id}/fix-catalog (no separate fix-a1).
// Delegates to processing.FixCompanyCatalog, which:
// 1. Ensures category_attributes links (orphan purge only)
// 2. RepairCompanyCategoryEnhancePrompts (same map as RepairA1DemoCategoryEnhancePrompts)
// 3. Re-applies product_enhance BuiltInDefaults when AIPrompts is set
// 47. FixCatalogHygieneWithIDs + attribute sanitize + reprocess sample
//
// Body: confirm=true required; backfill_categories (default true);
// reprocess_sample_limit (default 25, max 200, 0 = counts only).
// May target A1 in place with confirm — prefer Platform Demo when unsure.
// Refuses system company. Does not use A1 as a clone destination.
//
// Flash (UI): result.prompts / hashes / categories → flash.admin.fixA1Success.
func (s *Server) handleAdminFixCompanyCatalog(w http.ResponseWriter, r *http.Request) {
if s.Pool == nil {
Error(w, http.StatusServiceUnavailable, "database unavailable")
return
}
companyID, err := uuid.Parse(strings.TrimSpace(chi.URLParam(r, "id")))
if err != nil {
Error(w, http.StatusBadRequest, "invalid company id")
return
}
var body struct {
Confirm bool `json:"confirm"`
BackfillCategories *bool `json:"backfill_categories"`
ReprocessSampleLimit *int `json:"reprocess_sample_limit"`
}
if err := DecodeJSON(r, &body); err != nil {
Error(w, http.StatusBadRequest, "invalid json")
return
}
if !body.Confirm {
Error(w, http.StatusBadRequest, "confirm must be true")
return
}
if platformsettings.IsSystemCompany(companyID) {
Error(w, http.StatusBadRequest, "cannot fix the platform settings company")
return
}
var legacy, name string
if err := s.Pool.QueryRow(r.Context(), `
SELECT COALESCE(legacy_company_id, ''), name FROM companies WHERE id = $1`, companyID).
Scan(&legacy, &name); err != nil {
Error(w, http.StatusBadRequest, "company not found")
return
}
backfill := true
if body.BackfillCategories != nil {
backfill = *body.BackfillCategories
}
sampleLimit := 25
if body.ReprocessSampleLimit != nil {
sampleLimit = *body.ReprocessSampleLimit
}
if sampleLimit < 0 {
sampleLimit = 0
}
if sampleLimit > 200 {
sampleLimit = 200
}
result, err := processing.FixCompanyCatalog(
r.Context(),
s.Pool,
companyID,
name,
billing.IsA1CohortCompany(legacy, name),
processing.FixCompanyCatalogOpts{
BackfillCategories: backfill,
ReprocessSampleLimit: sampleLimit,
AIPrompts: s.AIPrompts,
},
)
if err != nil {
ClientOrLog(w, http.StatusBadRequest, "could not fix catalog", err, catalog.ClientError)
return
}
JSON(w, http.StatusOK, map[string]any{
"status": "ok",
"result": result,
"note": "Catalog was repaired in place; reprocess recommended products manually (no mass reprocess).",
})
}
@@ -0,0 +1,91 @@
package httpapi
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/alexedwards/scs/v2"
"github.com/descrybe/descrybe-v2/apps/api/internal/auth"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/google/uuid"
)
func TestHandleAdminFixCompanyCatalogNilPool(t *testing.T) {
t.Parallel()
s := &Server{}
req := httptest.NewRequest(http.MethodPost, "/api/admin/companies/"+uuid.NewString()+"/fix-catalog", strings.NewReader(`{"confirm":true}`))
rec := httptest.NewRecorder()
s.handleAdminFixCompanyCatalog(rec, req)
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("status=%d want 503 body=%s", rec.Code, rec.Body.String())
}
}
func TestHandleAdminFixCompanyCatalogRequiresConfirm(t *testing.T) {
t.Parallel()
// Pool nil short-circuits before confirm — use non-nil Pool stub via missing company path is harder.
// Confirm gate is covered when Pool is set; here we only assert invalid uuid.
s := &Server{Pool: nil}
req := httptest.NewRequest(http.MethodPost, "/api/admin/companies/not-a-uuid/fix-catalog", strings.NewReader(`{"confirm":false}`))
rec := httptest.NewRecorder()
s.handleAdminFixCompanyCatalog(rec, req)
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("status=%d want 503 (nil pool) body=%s", rec.Code, rec.Body.String())
}
}
// TestRouterAdminFixCatalogMounted locks POST /api/admin/companies/{id}/fix-catalog
// after session + CSRF + platform-admin (503 with nil Pool), not chi 404.
func TestRouterAdminFixCatalogMounted(t *testing.T) {
t.Parallel()
sm := scs.New()
sm.Cookie.Name = "descrybe_session"
uid := uuid.MustParse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
s := &Server{
Config: config.Config{
CSRFCookieName: "descrybe_csrf",
WebOrigin: "http://localhost:5173",
},
Sessions: sm,
Auth: &auth.Service{},
testPlatformAdmin: func(_ context.Context, got uuid.UUID) (bool, error) {
return got == uid, nil
},
}
var token string
seed := LoadSession(sm)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
sm.Put(r.Context(), auth.SessionUserIDKey, uid.String())
w.WriteHeader(http.StatusNoContent)
}))
seedRec := httptest.NewRecorder()
seed.ServeHTTP(seedRec, httptest.NewRequest(http.MethodGet, "/seed", nil))
for _, c := range seedRec.Result().Cookies() {
if c.Name == sm.Cookie.Name {
token = c.Value
}
}
if token == "" {
t.Fatal("expected session cookie from seed request")
}
h := s.Router()
path := "/api/admin/companies/" + uuid.NewString() + "/fix-catalog"
csrf := csrfCookieForSession(t, h, sm, token)
mounted := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{"confirm":true}`))
req.AddCookie(&http.Cookie{Name: sm.Cookie.Name, Value: token})
req.AddCookie(csrf)
req.Header.Set("X-CSRF-Token", csrf.Value)
h.ServeHTTP(mounted, req)
if mounted.Code == http.StatusNotFound {
t.Fatalf("route not mounted: status=404 body=%s", mounted.Body.String())
}
if mounted.Code != http.StatusServiceUnavailable {
t.Fatalf("status=%d want 503 (nil pool) body=%s", mounted.Code, mounted.Body.String())
}
}
+1
View File
@@ -391,6 +391,7 @@ func (s *Server) Router() http.Handler {
r.Post("/users/{id}/dev-password", s.handleAdminDevSetPassword) r.Post("/users/{id}/dev-password", s.handleAdminDevSetPassword)
r.Get("/companies", s.handleAdminListCompanies) r.Get("/companies", s.handleAdminListCompanies)
r.Post("/companies/{id}/clone-catalog", s.handleAdminCloneCompanyCatalog) r.Post("/companies/{id}/clone-catalog", s.handleAdminCloneCompanyCatalog)
r.Post("/companies/{id}/fix-catalog", s.handleAdminFixCompanyCatalog)
r.Get("/readiness", s.handleAdminReadiness) r.Get("/readiness", s.handleAdminReadiness)
r.Get("/diagnostics", s.handleAdminDiagnostics) r.Get("/diagnostics", s.handleAdminDiagnostics)
r.Get("/analytics", s.handleAdminAnalytics) r.Get("/analytics", s.handleAdminAnalytics)
+90 -22
View File
@@ -43,7 +43,10 @@ info:
4. Store it securely. Later list/revoke shows only key_prefix (first 10 4. Store it securely. Later list/revoke shows only key_prefix (first 10
characters). Revoked keys fail auth immediately. characters). Revoked keys fail auth immediately.
API keys are created in Settings -> API keys ### Cutover / migration (reissue)
API keys from the previous Descrybe platform were not migrated. After
cutover, integrations must create a new dk_ key in Settings -> API keys
(or Use my API key on /docs). Pre-cutover secrets return the same HTTP 401 (or Use my API key on /docs). Pre-cutover secrets return the same HTTP 401
Unauthorized as unknown keys — there is no separate “legacy key” error. Unauthorized as unknown keys — there is no separate “legacy key” error.
@@ -893,8 +896,8 @@ paths:
jobs omit items. Not the same shape as GET /process/{id} (flat ProcessingJob). jobs omit items. Not the same shape as GET /process/{id} (flat ProcessingJob).
On COMPLETED items, id is the processed_products UUID (legacy). Additive aliases: On COMPLETED items, id is the processed_products UUID (legacy). Additive aliases:
processed_product_id (same value as id) and raw_product_id (raw_products.id) processed_product_id (same value as id), raw_product_id (raw_products.id), and
for dual-mode clients that also call raw_product_ids surfaces. name (same value as title) for dual-mode clients / scorecards.
parameters: parameters:
- name: id - name: id
in: path in: path
@@ -914,20 +917,36 @@ paths:
$ref: "#/components/schemas/LegacyProcessStatusEnvelope" $ref: "#/components/schemas/LegacyProcessStatusEnvelope"
examples: examples:
completed: completed:
summary: Completed summary: Completed A1-shaped item
value: value:
data: data:
status: COMPLETED status: COMPLETED
process_id: aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa process_id: aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
processing_type: full processing_type: full
items: items:
- ean: 0123456789012 - ean: '8606019604493'
id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
processed_product_id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb processed_product_id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
raw_product_id: cccccccc-cccc-cccc-cccc-cccccccccccc raw_product_id: cccccccc-cccc-cccc-cccc-cccccccccccc
status: processed status: processed
category: electronics category: '50'
title: Acme Wireless Earbuds ANC Black category_name: Cookers
title: VOX electric cooker EHT 6020 WG
name: VOX electric cooker EHT 6020 WG
meta_title: VOX electric cooker EHT 6020 WG | 50
meta_description: Affordable electric cooker with four plates and a 65 L fan oven.
description: "Vox Electronics electric cooker EHT 6020 WG offers strong value with four electric hobs and a 65 L fan oven."
attributes:
brand: Vox
product_model: EHT6020WG
main_image: https://images.example.com/products/vox-eht6020wg.jpg
more_images: null
eprel:
id: "1234567"
label: https://eprel.ec.europa.eu/label/Example
pdf: https://eprel.ec.europa.eu/fiches/Example.pdf
energy_class: A
energy_scale: A-G
total_items: 1 total_items: 1
processed_at: '2026-08-04T10:04:12Z' processed_at: '2026-08-04T10:04:12Z'
processing: processing:
@@ -6257,6 +6276,13 @@ components:
nullable: true nullable: true
LegacyProcessItem: LegacyProcessItem:
type: object type: object
description: |
One COMPLETED legacy process line (A1 / public contract). Successful items
expose category as categories.unique_id, a plain-text description string
(never a JSON array; HTML stripped), SEO meta_title / meta_description,
optional eprel object or null, clean attributes, images, and dual-mode ids.
Product display name is title; additive name mirrors the same processed title
(dual-mode for scorecards / legacy clients that read name).
required: required:
- ean - ean
properties: properties:
@@ -6283,25 +6309,50 @@ components:
category: category:
type: string type: string
nullable: true nullable: true
description: |
categories.unique_id for the assigned category. Opaque string — may be
numeric (e.g. "28" or "50") or slug-like. Not a URL path and not a UUID.
category_name: category_name:
type: string type: string
nullable: true nullable: true
description: Human-readable category display name (not the unique_id).
title: title:
type: string type: string
nullable: true nullable: true
description: |
Product display name (processed title). Primary legacy field; same value
as name when present.
name:
type: string
nullable: true
description: |
Additive alias of title (same processed display name). Prefer title in
new clients; name remains for scorecards and legacy readers.
meta_title: meta_title:
type: string type: string
nullable: true nullable: true
description: |
SEO title. Filled from processing meta or synthesized from title / category
when empty so successful items are not left with null meta.
meta_description: meta_description:
type: string type: string
nullable: true nullable: true
description: |
SEO description (word-safe truncate). Distinct from body description when
possible; synthesized from plain description when DB meta is empty.
description: description:
type: string type: string
nullable: true nullable: true
description: Plain-text product description (HTML stripped). attributes: description: |
Plain-text product body description. Always a string — never a one-element
JSON array. Feed HTML tags are stripped; newlines may remain between paragraphs.
attributes:
type: object type: object
additionalProperties: true additionalProperties: true
nullable: true nullable: true
description: |
Characteristic catalog attributes only (brand, model, dims, warranty, …).
Core fields and eprel_* keys are not duplicated here.
main_image: main_image:
type: string type: string
nullable: true nullable: true
@@ -6313,7 +6364,14 @@ components:
eprel: eprel:
nullable: true nullable: true
type: object type: object
description: |
EU energy label payload when EPREL data is available; otherwise null.
Prefer this object over raw eprel_* keys inside attributes.
Live shape keys: id, label, pdf, energy_class, energy_scale.
properties: properties:
id:
type: string
description: EPREL product registration id (same as eprel_id).
label: label:
type: string type: string
pdf: pdf:
@@ -6393,32 +6451,42 @@ components:
started_at: '2026-08-04T10:00:00Z' started_at: '2026-08-04T10:00:00Z'
created_at: '2026-08-04T09:59:50Z' created_at: '2026-08-04T09:59:50Z'
LegacyProcessCompleted: LegacyProcessCompleted:
summary: "Completed legacy poll (GET /products/process/{id})" summary: "Completed legacy poll with A1-shaped item (unique_id category, plain description, meta, eprel)"
value: value:
data: data:
status: COMPLETED status: COMPLETED
process_id: 8f14e45f-ceea-467a-9e5d-9c6b0e8f3a21 process_id: 8f14e45f-ceea-467a-9e5d-9c6b0e8f3a21
processing_type: full processing_type: full
items: items:
- ean: '4548736132174' - ean: '8606019604493'
id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
processed_product_id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb processed_product_id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
raw_product_id: cccccccc-cccc-cccc-cccc-cccccccccccc raw_product_id: cccccccc-cccc-cccc-cccc-cccccccccccc
status: processed status: processed
category: electronics category: '50'
category_name: Electronics category_name: Cookers
title: Sony WH-1000XM5 Wireless Noise Cancelling Headphones Black title: VOX electric cooker EHT 6020 WG
meta_title: Sony WH-1000XM5 | Noise Cancelling Headphones name: VOX electric cooker EHT 6020 WG
meta_description: Industry-leading noise cancellation with up to 30 hours battery life. meta_title: VOX electric cooker EHT 6020 WG | 50
description: Industry-leading noise cancellation with up to 30 hours battery life. meta_description: Affordable electric cooker with four plates and a 65 L fan oven.
description: "Vox Electronics electric cooker EHT 6020 WG offers strong value with four electric hobs and a 65 L fan oven. Energy class A with practical everyday capacity."
attributes: attributes:
color: Black brand: Vox
brand: Sony product_model: EHT6020WG
battery_life_hours: '30' width: 0.6 m
main_image: https://images.example.com/products/wh1000xm5-black.jpg height: 0.85 m
depth: 0.6 m
weight: 42.81 kg
warranty: 60 months
main_image: https://images.example.com/products/vox-eht6020wg.jpg
more_images: more_images:
- https://images.example.com/products/wh1000xm5-black-side.jpg - https://images.example.com/products/vox-eht6020wg-side.jpg
eprel: null eprel:
id: "1234567"
label: https://eprel.ec.europa.eu/label/Example
pdf: https://eprel.ec.europa.eu/fiches/Example.pdf
energy_class: A
energy_scale: A-G
total_items: 1 total_items: 1
processed_at: '2026-08-04T10:04:12Z' processed_at: '2026-08-04T10:04:12Z'
LegacyProcessInProgress: LegacyProcessInProgress:
@@ -415,6 +415,36 @@ func TestV1OpenAPIYAMLProcessDualIDsAndGatesDocumentsContracts(t *testing.T) {
} }
} }
// TestV1OpenAPIYAMLLegacyProcessItemEprelShape locks LegacyProcessItem.eprel
// to the live nested object from eprel.MergeInto / extractEPRELFromAttrs.
func TestV1OpenAPIYAMLLegacyProcessItemEprelShape(t *testing.T) {
t.Parallel()
root := mustParseOpenAPIRoot(t, v1OpenAPIYAML)
comps, _ := root["components"].(map[string]any)
schemas, _ := comps["schemas"].(map[string]any)
item, _ := schemas["LegacyProcessItem"].(map[string]any)
itemProps, _ := item["properties"].(map[string]any)
eprel, _ := itemProps["eprel"].(map[string]any)
eprelProps, _ := eprel["properties"].(map[string]any)
if eprelProps == nil {
t.Fatal("LegacyProcessItem.eprel.properties missing")
}
want := []string{"id", "label", "pdf", "energy_class", "energy_scale"}
for _, key := range want {
if _, ok := eprelProps[key]; !ok {
t.Fatalf("LegacyProcessItem.eprel missing property %q (live shape)", key)
}
}
if len(eprelProps) != len(want) {
keys := make([]string, 0, len(eprelProps))
for k := range eprelProps {
keys = append(keys, k)
}
t.Fatalf("LegacyProcessItem.eprel properties = %v, want exactly %v", keys, want)
}
}
// TestV1OpenAPIYAMLDocumentsAPIKeyCutoverReissue locks cutover honesty: legacy // TestV1OpenAPIYAMLDocumentsAPIKeyCutoverReissue locks cutover honesty: legacy
// API keys were not migrated and clients must create new keys. // API keys were not migrated and clients must create new keys.
func TestV1OpenAPIYAMLDocumentsAPIKeyCutoverReissue(t *testing.T) { func TestV1OpenAPIYAMLDocumentsAPIKeyCutoverReissue(t *testing.T) {
@@ -312,11 +312,13 @@ func (s *Service) ResolveAIConfig(ctx context.Context, role string) (ResolvedAIC
Source: SourceDB, Source: SourceDB,
} }
if role == AIRoleProcessing { if role == AIRoleProcessing {
// Prefer legacy openai JSON fields before OPENAI_* env so a mock .env
// cannot override a configured platform DB endpoint.
if out.BaseURL == "" { if out.BaseURL == "" {
out.BaseURL = strings.TrimSpace(s.Env.OpenAIBaseURL) out.BaseURL = firstNonEmpty(strings.TrimSpace(doc.OpenAI.BaseURL), strings.TrimSpace(s.Env.OpenAIBaseURL))
} }
if out.Model == "" { if out.Model == "" {
out.Model = strings.TrimSpace(s.Env.OpenAIModel) out.Model = firstNonEmpty(strings.TrimSpace(doc.OpenAI.Model), strings.TrimSpace(s.Env.OpenAIModel))
} }
} }
if role == AIRoleVectorization { if role == AIRoleVectorization {
@@ -330,11 +332,26 @@ func (s *Service) ResolveAIConfig(ctx context.Context, role string) (ResolvedAIC
return out, nil return out, nil
} }
if role == AIRoleProcessing && !hasRoleMeta { // processing: prefer legacy openai (DB) then env when the role slot has no key,
// even if provider/model/base meta was partially written.
if role == AIRoleProcessing {
legacy, err := s.resolveLegacyOpenAI(doc) legacy, err := s.resolveLegacyOpenAI(doc)
if err != nil { if err != nil {
return ResolvedAIConfig{}, err return ResolvedAIConfig{}, err
} }
if strings.TrimSpace(legacy.APIKey) != "" {
return ResolvedAIConfig{
Role: role,
Provider: firstNonEmpty(strings.TrimSpace(st.Provider), defaultAIProvider),
APIKey: legacy.APIKey,
BaseURL: firstNonEmpty(strings.TrimSpace(st.BaseURL), legacy.BaseURL),
Model: firstNonEmpty(strings.TrimSpace(st.Model), legacy.Model),
Enabled: true,
Extras: copyStringMap(st.Extras),
Source: legacy.Source,
}, nil
}
if !hasRoleMeta {
return ResolvedAIConfig{ return ResolvedAIConfig{
Role: role, Role: role,
Provider: defaultAIProvider, Provider: defaultAIProvider,
@@ -345,6 +362,7 @@ func (s *Service) ResolveAIConfig(ctx context.Context, role string) (ResolvedAIC
Source: legacy.Source, Source: legacy.Source,
}, nil }, nil
} }
}
if role == AIRoleVectorization && !hasRoleMeta { if role == AIRoleVectorization && !hasRoleMeta {
return s.resolveVectorizationEnv(), nil return s.resolveVectorizationEnv(), nil
@@ -115,6 +115,38 @@ func TestResolveAIConfig_envFallback(t *testing.T) {
} }
} }
func TestResolveAIConfig_dbPreferredOverMockEnv(t *testing.T) {
key := DeriveKey("test-ai-config-secret-material", "fallback")
plain := "sk-db-overloaded-key"
enc, err := EncryptSecret(key, plain)
if err != nil {
t.Fatal(err)
}
svc := &Service{
Key: key,
Env: EnvConfig{
OpenAIAPIKey: "sk-mock-env",
OpenAIBaseURL: "http://127.0.0.1:18767/v1",
OpenAIModel: "mock-llm",
},
}
doc := storedDoc{
OpenAI: openaiStored{
BaseURL: "https://llm.overloadedbot.org/v1",
Model: "code-fast",
APIKeyEnc: enc,
APIKeyLast4: last4(plain),
},
}
got, err := svc.resolveLegacyOpenAI(doc)
if err != nil {
t.Fatal(err)
}
if got.Source != SourceDB || got.APIKey != plain || got.BaseURL != "https://llm.overloadedbot.org/v1" || got.Model != "code-fast" {
t.Fatalf("legacy resolve should prefer DB over mock env: %+v", got)
}
}
func TestPatchAIExtras(t *testing.T) { func TestPatchAIExtras(t *testing.T) {
var extras map[string]string var extras map[string]string
val := "1536" val := "1536"
+36 -5
View File
@@ -87,8 +87,25 @@ type ProductInput struct {
// CategoryEnhancePrompt overrides EnhanceUserTemplate when non-empty // CategoryEnhancePrompt overrides EnhanceUserTemplate when non-empty
// (resolved for the active language before enhance). // (resolved for the active language before enhance).
CategoryEnhancePrompt string CategoryEnhancePrompt string
// CategoryPromptsByLang maps lower(name) → lang → category override prompt. // CategoryPromptsByLang maps lower(name|unique_id) → lang → category override prompt
// (JSON/overlay text with {{attrs}}/{{category}}; works with repaired A1 overlays).
CategoryPromptsByLang map[string]company.LangPromptMap CategoryPromptsByLang map[string]company.LangPromptMap
// TitleTemplate / DescriptionTemplate are language-agnostic category formulas
// (categories.title_template / description_template). Injected into the enhance
// user prompt as structure constraints; not per-lang text maps.
TitleTemplate any
DescriptionTemplate any
// CategoryFormulasByKey maps lower(name|unique_id) → title/description templates.
CategoryFormulasByKey map[string]CategoryFormulas
// CategoryNamesByUID maps category unique_id → display name (e.g. "50"→"Štedilniki").
CategoryNamesByUID map[string]string
// AllowedAttrKeys are company category_attributes keys for this product's
// category unique_id (plus coreCharacteristicAttrKeys via AttrsForEnhance).
// nil = sanitize only (unit tests); empty non-nil = core keys only.
AllowedAttrKeys map[string]struct{}
// CategoryAttrKeys maps category unique_id → attribute keys (job cache).
// When set, enhance resolves allowlist from the step result category.
CategoryAttrKeys map[string]map[string]struct{}
// Prior* are loaded from the last processed_products row for this raw product. // Prior* are loaded from the last processed_products row for this raw product.
PriorEnhanceHash string PriorEnhanceHash string
PriorProcessedName string PriorProcessedName string
@@ -97,6 +114,12 @@ type ProductInput struct {
PriorLocalized company.LocalizedContent PriorLocalized company.LocalizedContent
} }
// CategoryFormulas holds optional title/description templates for one category key.
type CategoryFormulas struct {
TitleTemplate any
DescriptionTemplate any
}
// PromptTemplates is a system+user pair for one language. // PromptTemplates is a system+user pair for one language.
type PromptTemplates struct { type PromptTemplates struct {
System string System string
@@ -105,7 +128,11 @@ type PromptTemplates struct {
// StepResult is the cumulative output for one product. // StepResult is the cumulative output for one product.
type StepResult struct { type StepResult struct {
// Category is the company taxonomy unique_id (DB/API). Prefer CategoryName
// for human-facing copy (meta, synthesize, {{category}}).
Category string Category string
// CategoryName is the display label resolved from categories.name.
CategoryName string
Name string Name string
Description string Description string
ProcessedName string ProcessedName string
@@ -126,6 +153,10 @@ type StepResult struct {
// enhance input hash matched (ai_enhance_unchanged). processOne must not // enhance input hash matched (ai_enhance_unchanged). processOne must not
// ConsumeCredits in that case — no LLM and no meaningful rework. // ConsumeCredits in that case — no LLM and no meaningful rework.
SkipCreditDebit bool SkipCreditDebit bool
// MetaTitle / MetaDescription are free template SEO fields written by
// processOne via fillMetaFromResult (never AI-enhanced by default).
MetaTitle string
MetaDescription string
} }
// StepProgress is a job-level snapshot of pipeline step status. // StepProgress is a job-level snapshot of pipeline step status.
@@ -153,8 +184,8 @@ func (e *Engine) CompleterEnabled() bool {
if c, ok := e.Completer.(EnableChecker); ok { if c, ok := e.Completer.(EnableChecker); ok {
return c.Enabled() return c.Enabled()
} }
// HeuristicCompleter has no Enabled — treat as enabled only if explicitly set. // HeuristicCompleter (local/dev fallback) and other Completers without
// Worker sets Completer=nil when platform OpenAI (admin settings / env) is unset. // EnableChecker are treated as enabled when explicitly wired on the Engine.
_, isHeuristic := e.Completer.(HeuristicCompleter) // Worker still sets Completer=nil when no provider is configured.
return !isHeuristic return true
} }
@@ -56,3 +56,26 @@ func normalizeProviderMode(label string) string {
return AIProviderUnknown return AIProviderUnknown
} }
} }
// isUnknownProviderMode reports empty or the sentinel "unknown" analytics label.
func isUnknownProviderMode(label string) bool {
m := strings.ToLower(strings.TrimSpace(label))
return m == "" || m == AIProviderUnknown
}
// preferKnownProviderMode returns the first non-unknown label, else unknown.
func preferKnownProviderMode(candidates ...string) string {
for _, c := range candidates {
if !isUnknownProviderMode(c) {
return normalizeProviderMode(c)
}
}
return AIProviderUnknown
}
func lastResultAIProviderMode(r *StepResult) string {
if r == nil {
return ""
}
return r.AIProviderMode
}
@@ -2,6 +2,28 @@ package processing
import "testing" import "testing"
func TestPreferKnownProviderMode(t *testing.T) {
t.Parallel()
if got := preferKnownProviderMode(AIProviderUnknown, AIProviderInternal); got != AIProviderInternal {
t.Fatalf("unknown+internal=%q", got)
}
if got := preferKnownProviderMode("", "custom"); got != AIProviderCustom {
t.Fatalf("empty+custom=%q", got)
}
if got := preferKnownProviderMode(AIProviderInternal, AIProviderCustom); got != AIProviderInternal {
t.Fatalf("internal must win first=%q", got)
}
if got := preferKnownProviderMode(AIProviderUnknown, ""); got != AIProviderUnknown {
t.Fatalf("all unknown=%q", got)
}
if isUnknownProviderMode("") != true || isUnknownProviderMode(AIProviderUnknown) != true {
t.Fatal("expected empty/unknown to be unknown")
}
if isUnknownProviderMode(AIProviderInternal) {
t.Fatal("internal must not be unknown")
}
}
func TestNormalizeProviderMode(t *testing.T) { func TestNormalizeProviderMode(t *testing.T) {
t.Parallel() t.Parallel()
cases := []struct { cases := []struct {
+269
View File
@@ -0,0 +1,269 @@
package processing
import (
"fmt"
"strings"
"unicode"
)
// knownAttrKeyAliases maps compact feed/spec tokens onto compact category
// attribute_key tokens (Slovenian catalog keys + common English synonyms).
var knownAttrKeyAliases = map[string]string{
"velikostzaslona": "diagonalazaslona",
"screensize": "diagonalazaslona",
"screen": "diagonalazaslona",
"diagonal": "diagonalazaslona",
"diagonala": "diagonalazaslona",
"nastavljivpovisini": "nastavljivopovisini",
"heightadjustable": "nastavljivopovisini",
"zaslonnadotik": "zaslonobcutljivnadotik",
"touchscreen": "zaslonobcutljivnadotik",
"tipmatrike": "vrstapanela",
"tippanela": "vrstapanela",
"paneltype": "vrstapanela",
"panel": "vrstapanela",
"zvocniki": "vgrajenizvocniki",
"speakers": "vgrajenizvocniki",
"ukrivljenzaslon": "oblikazaslona",
"curved": "oblikazaslona",
"slusalkezmikrofonom": "vgrajenmikrofon",
"mikrofon": "vgrajenmikrofon",
"builtinmic": "vgrajenmikrofon",
"microphone": "vgrajenmikrofon",
"brezzicne": "brezzicnapovezava",
"brezzicna": "brezzicnapovezava",
"wireless": "brezzicnapovezava",
"bluetooth": "brezzicnatehnologija",
"bt": "brezzicnatehnologija",
"anc": "odpravljanjehrupa",
"noisecancelling": "odpravljanjehrupa",
"noisecanceling": "odpravljanjehrupa",
"odpravljanjehrupa": "odpravljanjehrupa",
"resolution": "locljivost",
"responsetime": "odzivnicas",
}
// MapAttrsOntoAllowedKeys remaps feed/spec keys onto category_attributes keys
// when a confident match exists (exact compact, known alias, or unique token
// subset). Existing allowed/core values are never overwritten. When allowed is
// nil or empty, attrs are returned unchanged (caller still filters).
func MapAttrsOntoAllowedKeys(attrs map[string]any, allowed map[string]struct{}) map[string]any {
if len(attrs) == 0 || len(allowed) == 0 {
return attrs
}
prefer := preferredAllowedKeys(allowed)
if len(prefer) == 0 {
return attrs
}
out := make(map[string]any, len(attrs)+len(prefer))
for k, v := range attrs {
out[k] = v
}
for k, v := range attrs {
if !attrValuePresent(v) {
continue
}
canon := canonicalizeAttrKey(k)
if canon != "" {
if _, ok := coreCharacteristicAttrKeys[canon]; ok {
continue
}
if _, ok := allowed[canon]; ok {
continue
}
if _, ok := allowed[strings.ToLower(strings.TrimSpace(k))]; ok {
continue
}
}
target := resolveAllowedAttrKey(k, prefer)
if target == "" || target == canon || target == k {
continue
}
if attrValuePresent(out[target]) {
continue
}
out[target] = v
}
return out
}
func preferredAllowedKeys(allowed map[string]struct{}) map[string]string {
// compact → preferred output key (canonical snake_case when possible)
prefer := make(map[string]string, len(allowed))
for k := range allowed {
c := compactAttrToken(k)
if c == "" {
continue
}
canon := canonicalizeAttrKey(k)
if canon == "" {
canon = strings.ToLower(strings.TrimSpace(k))
}
if prev, ok := prefer[c]; ok {
// Prefer underscore form over hyphen duplicates from loadCategoryAttributeKeySets.
if strings.Contains(canon, "_") && !strings.Contains(prev, "_") {
prefer[c] = canon
}
continue
}
prefer[c] = canon
}
return prefer
}
func resolveAllowedAttrKey(feedKey string, prefer map[string]string) string {
ck := compactAttrToken(feedKey)
if ck == "" {
return ""
}
if target, ok := prefer[ck]; ok {
return target
}
if alias, ok := knownAttrKeyAliases[ck]; ok {
if target, ok := prefer[alias]; ok {
return target
}
}
return softMatchAllowedKey(feedKey, ck, prefer)
}
func softMatchAllowedKey(feedKey, feedCompact string, prefer map[string]string) string {
feedTokens := attrKeyTokens(feedKey)
if len(feedTokens) == 0 && feedCompact == "" {
return ""
}
var hits []string
for compact, target := range prefer {
if feedCompact != "" && (strings.Contains(compact, feedCompact) || strings.Contains(feedCompact, compact)) {
// Require meaningful overlap (avoid tiny tokens matching everything).
shorter, longer := feedCompact, compact
if len(shorter) > len(longer) {
shorter, longer = longer, shorter
}
if len(shorter) >= 6 && len(shorter)*10 >= len(longer)*6 {
hits = append(hits, target)
continue
}
}
if len(feedTokens) == 0 {
continue
}
allowedTokens := attrKeyTokens(target)
if len(allowedTokens) == 0 {
allowedTokens = attrKeyTokens(compact)
}
if tokensSubsetMatch(feedTokens, allowedTokens) {
hits = append(hits, target)
}
}
if len(hits) != 1 {
return ""
}
return hits[0]
}
func tokensSubsetMatch(feed, allowed []string) bool {
significant := 0
for _, ft := range feed {
if len(ft) < 3 {
continue
}
significant++
ok := false
for _, at := range allowed {
if at == ft || strings.HasPrefix(at, ft) || strings.HasPrefix(ft, at) {
ok = true
break
}
}
if !ok {
return false
}
}
return significant >= 2
}
func attrKeyTokens(k string) []string {
k = strings.ToLower(strings.TrimSpace(k))
if k == "" {
return nil
}
var b strings.Builder
var out []string
flush := func() {
if b.Len() == 0 {
return
}
tok := b.String()
b.Reset()
tok = compactAttrToken(tok)
if len(tok) >= 2 {
out = append(out, tok)
}
}
for _, r := range k {
r = foldAttrRune(r)
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
b.WriteRune(r)
default:
flush()
}
}
flush()
return out
}
func compactAttrToken(k string) string {
var b strings.Builder
b.Grow(len(k))
for _, r := range strings.ToLower(k) {
r = foldAttrRune(r)
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
b.WriteRune(r)
}
}
return b.String()
}
func foldAttrRune(r rune) rune {
switch r {
case 'š', 'ś', 'ş':
return 's'
case 'č', 'ć', 'ç':
return 'c'
case 'ž', 'ź', 'ż':
return 'z'
case 'đ':
return 'd'
case 'ä', 'á', 'à', 'â', 'ã', 'å':
return 'a'
case 'ë', 'é', 'è', 'ê':
return 'e'
case 'ï', 'í', 'ì', 'î':
return 'i'
case 'ö', 'ó', 'ò', 'ô', 'õ':
return 'o'
case 'ü', 'ú', 'ù', 'û':
return 'u'
case 'ý', 'ÿ':
return 'y'
}
if unicode.IsLetter(r) {
return unicode.ToLower(r)
}
return r
}
func attrValuePresent(v any) bool {
if v == nil {
return false
}
s := strings.TrimSpace(fmt.Sprint(v))
return s != "" && s != "<nil>"
}
@@ -0,0 +1,220 @@
package processing
import (
"context"
"testing"
)
func TestMapAttrsOntoAllowedKeys_monitorAliases(t *testing.T) {
allowed := map[string]struct{}{
"barva": {},
"diagonala_zaslona": {},
"diagonala-zaslona": {},
"locljivost": {},
"nastavljivo_po_visini": {},
"nastavljivo-po-visini": {},
"odzivni_cas": {},
"odzivni-cas": {},
"zaslon_obcutljiv_na_dotik": {},
"zaslon-obcutljiv-na-dotik": {},
"vrsta_panela": {},
"vrsta-panela": {},
"vgrajeni_zvocniki": {},
"vgrajeni-zvocniki": {},
}
in := map[string]any{
"brand": "Xiaomi",
"barva": "Črna",
"locljivost": "1920 x 1080",
"odzivni-cas": "6 ms",
"velikost-zaslona": `68,58 cm (27,0")`,
"nastavljiv-po-visini": "Ne",
"zaslon-na-dotik": "Ne",
"tip-matrike": "IPS",
"zvocniki": "Ne",
"kontrast": "1000:1", // not on allowlist
}
got := MapAttrsOntoAllowedKeys(in, allowed)
if got["diagonala_zaslona"] != `68,58 cm (27,0")` {
t.Fatalf("diagonala_zaslona=%v keys=%v", got["diagonala_zaslona"], got)
}
if got["nastavljivo_po_visini"] != "Ne" {
t.Fatalf("nastavljivo_po_visini=%v", got["nastavljivo_po_visini"])
}
if got["zaslon_obcutljiv_na_dotik"] != "Ne" {
t.Fatalf("zaslon_obcutljiv_na_dotik=%v", got["zaslon_obcutljiv_na_dotik"])
}
if got["vrsta_panela"] != "IPS" {
t.Fatalf("vrsta_panela=%v", got["vrsta_panela"])
}
if got["vgrajeni_zvocniki"] != "Ne" {
t.Fatalf("vgrajeni_zvocniki=%v", got["vgrajeni_zvocniki"])
}
// Exact keys untouched.
if got["locljivost"] != "1920 x 1080" || got["barva"] != "Črna" {
t.Fatalf("exact keys: %v", got)
}
}
func TestMapAttrsOntoAllowedKeys_headphonesAliases(t *testing.T) {
allowed := map[string]struct{}{
"brezzicna_povezava": {},
"brezzicna-povezava": {},
"brezzicna_tehnologija": {},
"brezzicna-tehnologija": {},
"odpravljanje_hrupa": {},
"odpravljanje-hrupa": {},
"vgrajen_mikrofon": {},
"vgrajen-mikrofon": {},
}
in := map[string]any{
"slusalke-z-mikrofonom": "Da",
"brezzicne": "Da",
"bluetooth": "5.3",
"vrsta-slusalk": "Naglavne",
}
got := MapAttrsOntoAllowedKeys(in, allowed)
if got["vgrajen_mikrofon"] != "Da" {
t.Fatalf("vgrajen_mikrofon=%v got=%v", got["vgrajen_mikrofon"], got)
}
if got["brezzicna_povezava"] != "Da" {
t.Fatalf("brezzicna_povezava=%v", got["brezzicna_povezava"])
}
if got["brezzicna_tehnologija"] != "5.3" {
t.Fatalf("brezzicna_tehnologija=%v", got["brezzicna_tehnologija"])
}
}
func TestAttrsForPersist_keepsMappedFormulaKeys(t *testing.T) {
attrs := map[string]any{
"brand": "Xiaomi",
"velikost-zaslona": "27\"",
"locljivost": "1920 x 1080",
"tip-matrike": "IPS",
"kontrast": "1000:1",
}
allowed := map[string]struct{}{
"diagonala_zaslona": {},
"diagonala-zaslona": {},
"locljivost": {},
"vrsta_panela": {},
"vrsta-panela": {},
"barva": {},
}
got := AttrsForPersist(attrs, allowed)
if got["brand"] != "Xiaomi" {
t.Fatalf("brand=%v", got)
}
if got["diagonala_zaslona"] != "27\"" {
t.Fatalf("formula diagonala_zaslona missing: %v", got)
}
if got["locljivost"] != "1920 x 1080" {
t.Fatalf("locljivost missing: %v", got)
}
if got["vrsta_panela"] != "IPS" {
t.Fatalf("vrsta_panela missing: %v", got)
}
if _, ok := got["kontrast"]; ok {
t.Fatalf("non-formula junk must drop: %v", got)
}
if _, ok := got["velikost_zaslona"]; ok {
t.Fatalf("unmapped alias key should not persist after filter: %v", got)
}
}
func TestRunSteps_parseFillMapsMonitorFormulaAttrs(t *testing.T) {
e := &Engine{}
in := ProductInput{
GTIN: "6941948701199",
Name: "Monitor Xiaomi A27i",
Mapped: map[string]any{
"name": "Monitor Xiaomi A27i",
"category": "25",
"brand": "Xiaomi",
"specifications": map[string]any{
"barva": "Črna",
"locljivost": "1920 x 1080",
"odzivni-cas": "6 ms",
"velikost-zaslona": "27\"",
"nastavljiv-po-visini": "Ne",
"zaslon-na-dotik": "Ne",
"kontrast": "1000:1",
},
},
CategoryAttrKeys: map[string]map[string]struct{}{
"25": {
"barva": {},
"diagonala_zaslona": {},
"diagonala-zaslona": {},
"locljivost": {},
"nastavljivo_po_visini": {},
"nastavljivo-po-visini": {},
"odzivni_cas": {},
"odzivni-cas": {},
"zaslon_obcutljiv_na_dotik": {},
"zaslon-obcutljiv-na-dotik": {},
},
},
}
out, err := e.RunSteps(context.Background(), "co", in, "attributes", nil, StepPolicy{})
if err != nil {
t.Fatal(err)
}
pa := out.ProcessedAttributes
if pa["barva"] != "Črna" || pa["locljivost"] != "1920 x 1080" {
t.Fatalf("exact formula keys: %v", pa)
}
if pa["diagonala_zaslona"] != "27\"" {
t.Fatalf("diagonala_zaslona=%v pa=%v", pa["diagonala_zaslona"], pa)
}
if pa["nastavljivo_po_visini"] != "Ne" {
t.Fatalf("nastavljivo_po_visini=%v", pa["nastavljivo_po_visini"])
}
if pa["zaslon_obcutljiv_na_dotik"] != "Ne" {
t.Fatalf("zaslon_obcutljiv_na_dotik=%v", pa["zaslon_obcutljiv_na_dotik"])
}
if _, ok := pa["kontrast"]; ok {
t.Fatalf("kontrast must not persist: %v", pa)
}
}
func TestRunSteps_parseFillMapsHeadphonesFormulaAttrs(t *testing.T) {
e := &Engine{}
in := ProductInput{
GTIN: "650311612227",
Name: "UVI Gear WRATH V2",
Mapped: map[string]any{
"name": "UVI Gear WRATH V2",
"category": "48",
"brand": "UVI Gear",
"specifications": map[string]any{
"barva": "Črna",
"slusalke-z-mikrofonom": "Da",
"brezzicne": "Ne",
"vrsta-slusalk": "Naglavne",
},
},
CategoryAttrKeys: map[string]map[string]struct{}{
"48": {
"brezzicna_povezava": {},
"brezzicna-povezava": {},
"vgrajen_mikrofon": {},
"vgrajen-mikrofon": {},
},
},
}
out, err := e.RunSteps(context.Background(), "co", in, "attributes", nil, StepPolicy{})
if err != nil {
t.Fatal(err)
}
pa := out.ProcessedAttributes
if pa["vgrajen_mikrofon"] != "Da" {
t.Fatalf("vgrajen_mikrofon=%v pa=%v", pa["vgrajen_mikrofon"], pa)
}
if pa["brezzicna_povezava"] != "Ne" {
t.Fatalf("brezzicna_povezava=%v", pa["brezzicna_povezava"])
}
if _, ok := pa["vrsta_slusalk"]; ok {
t.Fatalf("unlinked feed key must drop: %v", pa)
}
}
@@ -0,0 +1,110 @@
package processing
import (
"bytes"
"context"
"encoding/json"
"fmt"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// BackfillCompanyProcessedAttributes rewrites attributes and processed_attributes
// for every processed row in a company using AttrsForPersist + category_attributes
// allowlists (same rules as processOne / enhance). Returns rows updated.
func BackfillCompanyProcessedAttributes(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int64, error) {
if pool == nil {
return 0, fmt.Errorf("backfill attributes: nil pool")
}
p := &Pipeline{Pool: pool}
sets := p.loadCategoryAttributeKeySets(ctx, companyID, uuid.Nil)
rows, err := pool.Query(ctx, `
SELECT id, COALESCE(category, ''), COALESCE(attributes, '{}'::jsonb), COALESCE(processed_attributes, '{}'::jsonb)
FROM processed_products
WHERE company_id = $1`, companyID)
if err != nil {
return 0, fmt.Errorf("backfill attributes query: %w", err)
}
defer rows.Close()
var updated int64
for rows.Next() {
var (
id uuid.UUID
category string
attrsJSON []byte
procAttrsJSON []byte
)
if err := rows.Scan(&id, &category, &attrsJSON, &procAttrsJSON); err != nil {
return updated, fmt.Errorf("backfill attributes scan: %w", err)
}
attrs := decodeAttrMap(attrsJSON)
procAttrs := decodeAttrMap(procAttrsJSON)
allowed := allowedAttrKeysFromSets(sets, category)
cleanAttrs := AttrsForPersist(attrs, allowed)
cleanProc := AttrsForPersist(procAttrs, allowed)
if len(cleanProc) == 0 {
cleanProc = cleanAttrs
}
newAttrsJSON, err := json.Marshal(cleanAttrs)
if err != nil {
return updated, fmt.Errorf("backfill attributes marshal attrs %s: %w", id, err)
}
newProcJSON, err := json.Marshal(cleanProc)
if err != nil {
return updated, fmt.Errorf("backfill attributes marshal processed %s: %w", id, err)
}
if bytes.Equal(compactJSON(attrsJSON), compactJSON(newAttrsJSON)) &&
bytes.Equal(compactJSON(procAttrsJSON), compactJSON(newProcJSON)) {
continue
}
ct, err := pool.Exec(ctx, `
UPDATE processed_products
SET attributes = $2::jsonb,
processed_attributes = $3::jsonb,
updated_at = now()
WHERE id = $1 AND company_id = $4`, id, string(newAttrsJSON), string(newProcJSON), companyID)
if err != nil {
return updated, fmt.Errorf("backfill attributes update %s: %w", id, err)
}
updated += ct.RowsAffected()
}
if err := rows.Err(); err != nil {
return updated, fmt.Errorf("backfill attributes rows: %w", err)
}
return updated, nil
}
func decodeAttrMap(raw []byte) map[string]any {
out := map[string]any{}
if len(raw) == 0 {
return out
}
var parsed any
if err := json.Unmarshal(raw, &parsed); err != nil {
return out
}
switch t := parsed.(type) {
case map[string]any:
return t
case []any:
for _, entry := range t {
if m, ok := entry.(map[string]any); ok {
if k, ok := m["key"].(string); ok && k != "" {
out[k] = m
}
}
}
}
return out
}
func compactJSON(b []byte) []byte {
var buf bytes.Buffer
if err := json.Compact(&buf, b); err != nil {
return b
}
return buf.Bytes()
}
@@ -0,0 +1,48 @@
package processing
import (
"encoding/json"
"testing"
)
func TestAttrsForPersist_nilAllowlistSanitizeOnly(t *testing.T) {
got := AttrsForPersist(map[string]any{
"name": "TV Mount",
"brand": "Ostalo",
"zavora": "Mehanska",
}, nil)
if _, ok := got["name"]; ok {
t.Fatalf("name must be stripped: %v", got)
}
if got["zavora"] != "Mehanska" {
t.Fatalf("nil allowlist keeps non-reserved: %v", got)
}
}
func TestCompactJSON_stable(t *testing.T) {
a := []byte(`{"b":1,"a":2}`)
b := []byte(`{ "a" : 2 , "b" : 1 }`)
ca := compactJSON(a)
cb := compactJSON(b)
var ma, mb map[string]any
if err := json.Unmarshal(ca, &ma); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(cb, &mb); err != nil {
t.Fatal(err)
}
if ma["a"] != mb["a"] || ma["b"] != mb["b"] {
t.Fatalf("compact mismatch %s vs %s", ca, cb)
}
}
func TestDecodeAttrMap_objectAndKeyedArray(t *testing.T) {
obj := decodeAttrMap([]byte(`{"brand":"Ostalo","zavora":"x"}`))
if obj["brand"] != "Ostalo" {
t.Fatalf("obj=%v", obj)
}
arr := decodeAttrMap([]byte(`[{"key":"brand","value":"Ostalo"},{"key":"zavora","value":"x"}]`))
if _, ok := arr["brand"]; !ok {
t.Fatalf("arr=%v", arr)
}
}
+386
View File
@@ -0,0 +1,386 @@
package processing
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// CatalogFixResult summarizes in-place catalog hygiene (never clears the catalog).
type CatalogFixResult struct {
WeakHashesCleared int `json:"weak_hashes_cleared"`
CategoriesBackfilled int `json:"categories_backfilled"`
DescriptionsNormalized int `json:"descriptions_normalized"`
DescriptionsBackfilled int `json:"descriptions_backfilled"`
NamesBackfilled int `json:"names_backfilled"`
MappedDescriptionsFlat int `json:"mapped_descriptions_flattened"`
MetaBackfilled int `json:"meta_backfilled"`
ProductsScanned int `json:"products_scanned"`
}
// ClearWeakEnhanceHashes removes enhance_input_hash from field_sources and
// localized_content when the product description is weak (poisoned skip hashes).
// Delegates to catalog.RepairWeakEnhanceHashesDetailed (canonical implementation).
func ClearWeakEnhanceHashes(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (cleared int, scanned int, clearedRawIDs []uuid.UUID, err error) {
res, err := catalog.RepairWeakEnhanceHashesDetailed(ctx, pool, companyID)
if err != nil {
return 0, 0, nil, err
}
return int(res.Cleared), res.Scanned, res.ClearedRawIDs, nil
}
// BackfillCategoriesFromMapped copies mapped_data category unique_ids onto
// processed_products.category when the processed category is empty/"none".
// Uses Go-side categoryUniqueIDFromMaps (nested/array shapes) and validates
// against the company taxonomy when categories exist.
func BackfillCategoriesFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
if pool == nil {
return 0, fmt.Errorf("nil pool")
}
valid, err := loadCompanyCategoryUniqueIDs(ctx, pool, companyID)
if err != nil {
return 0, err
}
rows, err := pool.Query(ctx, `
SELECT pp.id,
COALESCE(pp.category, ''),
COALESCE(rp.mapped_data, '{}'::jsonb),
COALESCE(rp.raw_data, '{}'::jsonb)
FROM processed_products pp
JOIN raw_products rp ON rp.id = pp.raw_product_id AND rp.company_id = pp.company_id
WHERE pp.company_id = $1
AND (
pp.category IS NULL
OR btrim(pp.category) = ''
OR lower(btrim(pp.category)) = 'none'
)`, companyID)
if err != nil {
return 0, err
}
defer rows.Close()
for rows.Next() {
var (
id uuid.UUID
prior string
mappedB []byte
rawB []byte
)
if err := rows.Scan(&id, &prior, &mappedB, &rawB); err != nil {
return updated, err
}
mapped, err := decodeJSONObject(mappedB)
if err != nil {
return updated, err
}
raw, err := decodeJSONObject(rawB)
if err != nil {
return updated, err
}
cat := categoryUniqueIDFromMaps(mapped, raw)
if cat == "" {
continue
}
if len(valid) > 0 {
if _, ok := valid[cat]; !ok {
continue
}
}
ct, err := pool.Exec(ctx, `
UPDATE processed_products
SET category = $2,
field_sources = jsonb_set(
COALESCE(field_sources, '{}'::jsonb),
'{category}',
'"mapped_backfill"'::jsonb,
true
),
updated_at = now()
WHERE id = $1 AND company_id = $3
AND (
category IS NULL
OR btrim(category) = ''
OR lower(btrim(category)) = 'none'
)`,
id, cat, companyID)
if err != nil {
return updated, err
}
if ct.RowsAffected() > 0 {
updated++
}
_ = prior
}
return updated, rows.Err()
}
func loadCompanyCategoryUniqueIDs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (map[string]struct{}, error) {
out := map[string]struct{}{}
rows, err := pool.Query(ctx, `
SELECT unique_id FROM categories
WHERE company_id = $1 AND COALESCE(NULLIF(btrim(unique_id), ''), '') <> ''`, companyID)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var uid string
if err := rows.Scan(&uid); err != nil {
return nil, err
}
uid = strings.TrimSpace(uid)
if uid != "" {
out[uid] = struct{}{}
}
}
return out, rows.Err()
}
// FixCatalogHygiene clears poisoned enhance hashes and optionally backfills categories.
func FixCatalogHygiene(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (CatalogFixResult, error) {
out, _, err := FixCatalogHygieneWithIDs(ctx, pool, companyID, true)
return out, err
}
// FixCatalogHygieneWithIDs is FixCatalogHygiene plus raw_product_ids that had weak hashes cleared.
func FixCatalogHygieneWithIDs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID, backfillCategories bool) (CatalogFixResult, []uuid.UUID, error) {
var out CatalogFixResult
cleared, scanned, ids, err := ClearWeakEnhanceHashes(ctx, pool, companyID)
if err != nil {
return out, nil, err
}
out.WeakHashesCleared = cleared
out.ProductsScanned = scanned
flat, err := FlattenMappedDescriptionArrays(ctx, pool, companyID)
if err != nil {
return out, ids, err
}
out.MappedDescriptionsFlat = flat
if backfillCategories {
// Go-side unique_id extraction (nested/array) + taxonomy validation.
cats, err := BackfillCategoriesFromMapped(ctx, pool, companyID)
if err != nil {
return out, ids, err
}
out.CategoriesBackfilled = cats
descN, err := BackfillDescriptionsFromMapped(ctx, pool, companyID)
if err != nil {
return out, ids, err
}
out.DescriptionsBackfilled = descN
nameN, err := BackfillPollutedNamesFromMapped(ctx, pool, companyID)
if err != nil {
return out, ids, err
}
out.NamesBackfilled = nameN
metaN, err := BackfillMissingMeta(ctx, pool, companyID)
if err != nil {
return out, ids, err
}
out.MetaBackfilled = metaN
}
descs, err := NormalizeProcessedDescriptions(ctx, pool, companyID)
if err != nil {
return out, ids, err
}
out.DescriptionsNormalized = descs
return out, ids, nil
}
// NormalizeProcessedDescriptions rewrites processed_description to the plain-text
// form produced by PlainDescriptionFromAny (unwraps JSON arrays, strips HTML). When
// processed_description is empty, normalizes from description. Returns rows updated.
func NormalizeProcessedDescriptions(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (int, error) {
if pool == nil {
return 0, fmt.Errorf("normalize descriptions: nil pool")
}
rows, err := pool.Query(ctx, `
SELECT id,
COALESCE(description, ''),
COALESCE(processed_description, '')
FROM processed_products
WHERE company_id = $1`, companyID)
if err != nil {
return 0, fmt.Errorf("normalize descriptions query: %w", err)
}
defer rows.Close()
updated := 0
for rows.Next() {
var (
id uuid.UUID
desc, processedDesc string
)
if err := rows.Scan(&id, &desc, &processedDesc); err != nil {
return updated, fmt.Errorf("normalize descriptions scan: %w", err)
}
raw := strings.TrimSpace(processedDesc)
if raw == "" {
raw = strings.TrimSpace(desc)
}
if raw == "" {
continue
}
plain := plainDescriptionFromStored(raw)
if plain == "" || plain == strings.TrimSpace(processedDesc) {
continue
}
ct, err := pool.Exec(ctx, `
UPDATE processed_products
SET processed_description = $2,
updated_at = now()
WHERE id = $1 AND company_id = $3`, id, plain, companyID)
if err != nil {
return updated, fmt.Errorf("normalize descriptions update %s: %w", id, err)
}
updated += int(ct.RowsAffected())
}
if err := rows.Err(); err != nil {
return updated, fmt.Errorf("normalize descriptions rows: %w", err)
}
return updated, nil
}
// plainDescriptionFromStored handles DB text that may itself be a JSON array/string.
func plainDescriptionFromStored(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return ""
}
if strings.HasPrefix(raw, "[") || strings.HasPrefix(raw, "{") {
var decoded any
if err := json.Unmarshal([]byte(raw), &decoded); err == nil {
if s := PlainDescriptionFromAny(decoded); s != "" {
return s
}
}
}
return PlainDescriptionFromAny(raw)
}
// RecommendReprocessRawProductIDs returns how many products should be reprocessed
// after Fix A1 (weak desc and/or missing enhance hash and/or empty category), plus
// an optional sample of raw_product_ids (limit 0 = no sample).
func RecommendReprocessRawProductIDs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID, prefer []uuid.UUID, sampleLimit int) (needed int, sample []uuid.UUID, err error) {
if pool == nil {
return 0, nil, fmt.Errorf("nil pool")
}
seen := map[uuid.UUID]struct{}{}
add := func(id uuid.UUID) {
if id == uuid.Nil {
return
}
if _, ok := seen[id]; ok {
return
}
seen[id] = struct{}{}
needed++
if sampleLimit > 0 && len(sample) < sampleLimit {
sample = append(sample, id)
}
}
for _, id := range prefer {
add(id)
}
rows, err := pool.Query(ctx, `
SELECT raw_product_id,
COALESCE(name, ''),
COALESCE(description, ''),
COALESCE(processed_name, ''),
COALESCE(processed_description, ''),
COALESCE(category, ''),
COALESCE(field_sources, '{}'::jsonb),
COALESCE(localized_content, '{}'::jsonb)
FROM processed_products
WHERE company_id = $1`, companyID)
if err != nil {
return 0, nil, err
}
defer rows.Close()
for rows.Next() {
var (
rawID uuid.UUID
name, desc, processedName, processedDesc, category string
fsRaw, locRaw []byte
)
if err := rows.Scan(&rawID, &name, &desc, &processedName, &processedDesc, &category, &fsRaw, &locRaw); err != nil {
return needed, sample, err
}
primaryDesc := strings.TrimSpace(processedDesc)
if primaryDesc == "" {
primaryDesc = desc
}
primaryName := strings.TrimSpace(processedName)
if primaryName == "" {
primaryName = name
}
fs, err := decodeJSONObject(fsRaw)
if err != nil {
return needed, sample, err
}
loc, err := company.DecodeLocalizedContent(locRaw)
if err != nil {
return needed, sample, err
}
needs := false
if strings.TrimSpace(category) == "" {
needs = true
}
if isPromptLabelTitle(primaryName) || isPromptLabelTitle(processedName) || isPromptLabelTitle(name) {
needs = true
}
if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) {
needs = true
}
if _, has := fs[FieldEnhanceInputHash]; !has {
// Missing hash after weak clear / never enhanced — recommend when desc weak or empty category.
if isWeakPriorEnhanceDescription(primaryDesc, primaryName, name) || strings.TrimSpace(category) == "" {
needs = true
}
}
for _, fields := range loc {
d := strings.TrimSpace(fields.ProcessedDescription)
n := strings.TrimSpace(fields.ProcessedName)
if n == "" {
n = primaryName
}
if fields.EnhanceInputHash == "" && isWeakPriorEnhanceDescription(d, n, primaryName, name) {
needs = true
break
}
}
if needs {
add(rawID)
}
}
return needed, sample, rows.Err()
}
func decodeJSONObject(raw []byte) (map[string]any, error) {
out := map[string]any{}
if len(raw) == 0 {
return out, nil
}
if err := json.Unmarshal(raw, &out); err != nil {
return nil, err
}
if out == nil {
out = map[string]any{}
}
return out, nil
}
@@ -0,0 +1,20 @@
package processing
import (
"testing"
)
func TestIsWeakPriorEnhanceDescription_forFixCatalog(t *testing.T) {
t.Parallel()
title := "Acme Widget Pro 12"
if !isWeakPriorEnhanceDescription("", title) {
t.Fatal("empty desc should be weak")
}
if !isWeakPriorEnhanceDescription(title, title) {
t.Fatal("title echo should be weak")
}
good := "Durable widget for everyday kitchen use with clear specs."
if isWeakPriorEnhanceDescription(good, title) {
t.Fatal("factual desc should not be weak")
}
}
+184
View File
@@ -0,0 +1,184 @@
package processing
import (
"strconv"
"strings"
"unicode"
)
// categoryDisplayLabel returns the human category name for meta / synthesize /
// {{category}}. Prefers CategoryName; never falls back to a digits-only unique_id.
func categoryDisplayLabel(result StepResult) string {
if n := strings.TrimSpace(result.CategoryName); n != "" {
return n
}
cat := strings.TrimSpace(result.Category)
if cat == "" || isDigitsOnlyCategoryID(cat) {
return ""
}
return cat
}
// resolveCategoryDisplayName looks up unique_id → name; returns "" when unknown
// so callers do not treat "50" as a human label.
func resolveCategoryDisplayName(uid string, namesByUID map[string]string) string {
uid = strings.TrimSpace(uid)
if uid == "" || len(namesByUID) == 0 {
return ""
}
return strings.TrimSpace(namesByUID[uid])
}
func syncCategoryName(out *StepResult, namesByUID map[string]string) {
if out == nil {
return
}
if n := resolveCategoryDisplayName(out.Category, namesByUID); n != "" {
out.CategoryName = n
}
}
func isDigitsOnlyCategoryID(s string) bool {
if s == "" {
return false
}
for _, r := range s {
if !unicode.IsDigit(r) {
return false
}
}
return true
}
// categoryUniqueIDFromAny extracts a company category unique_id from feed/mapped
// shapes: plain string/number, nested {unique_id|category_unique_id|#text|…}, or
// a one-element array of those. Returns "" for empty/"none"/unusable values.
func categoryUniqueIDFromAny(v any) string {
if v == nil {
return ""
}
switch t := v.(type) {
case string:
return normalizeCategoryUniqueID(t)
case float64:
if t == float64(int64(t)) {
return normalizeCategoryUniqueID(strconv.FormatInt(int64(t), 10))
}
return normalizeCategoryUniqueID(strconv.FormatFloat(t, 'f', -1, 64))
case float32:
return categoryUniqueIDFromAny(float64(t))
case int:
return normalizeCategoryUniqueID(strconv.Itoa(t))
case int64:
return normalizeCategoryUniqueID(strconv.FormatInt(t, 10))
case int32:
return normalizeCategoryUniqueID(strconv.FormatInt(int64(t), 10))
case jsonNumberStringer:
return normalizeCategoryUniqueID(t.String())
case map[string]any:
for _, k := range []string{
"unique_id", "category_unique_id", "categoryUniqueId",
"#text", "text", "id", "value", "code",
} {
if s := categoryUniqueIDFromAny(t[k]); s != "" {
return s
}
}
return ""
case []any:
for _, item := range t {
if s := categoryUniqueIDFromAny(item); s != "" {
return s
}
}
return ""
default:
return ""
}
}
// jsonNumberStringer matches encoding/json.Number without importing encoding/json here.
type jsonNumberStringer interface {
String() string
}
func normalizeCategoryUniqueID(s string) string {
s = strings.TrimSpace(s)
if s == "" || s == "<nil>" || strings.EqualFold(s, "none") {
return ""
}
return s
}
// categoryUniqueIDFromMaps returns the first usable category unique_id from maps
// (normalized/mapped/raw). Prefer canonical keys; mapped should win by call order.
func categoryUniqueIDFromMaps(maps ...map[string]any) string {
for _, m := range maps {
if m == nil {
continue
}
for _, k := range []string{
"category", "category_unique_id", "categoryUniqueId",
"Category", "product_type", "productType",
} {
if s := categoryUniqueIDFromAny(m[k]); s != "" {
return s
}
}
}
return ""
}
// coerceMappedCategoryUniqueID writes a string unique_id onto m["category"] when
// category (or category_unique_id) can be resolved — so RunSteps string paths stay deterministic.
func coerceMappedCategoryUniqueID(m map[string]any) string {
if m == nil {
return ""
}
cat := categoryUniqueIDFromMaps(m)
if cat == "" {
return ""
}
m["category"] = cat
if _, ok := m["category_unique_id"]; !ok {
m["category_unique_id"] = cat
}
return cat
}
// applyCategoryFromMapped sets StepResult.Category from feed/mapped unique_id when present.
// Does not clear an existing Category when maps lack a usable value (caller may preserve prior).
func applyCategoryFromMapped(out *StepResult, maps ...map[string]any) {
if out == nil {
return
}
cat := categoryUniqueIDFromMaps(maps...)
if cat == "" {
return
}
out.Category = SanitizeText(cat)
if out.FieldSources == nil {
out.FieldSources = map[string]any{}
}
out.FieldSources["category"] = "mapped"
}
// filterCategoryIfInvalid clears Category when the company has a known unique_id set
// and the value is not in that set. Empty valid set means "skip validation" (tests / no taxonomy).
func filterCategoryIfInvalid(out *StepResult, valid map[string]struct{}) {
if out == nil || len(valid) == 0 {
return
}
cat := strings.TrimSpace(out.Category)
if cat == "" {
return
}
if _, ok := valid[cat]; ok {
return
}
out.Category = ""
if out.FieldSources != nil {
delete(out.FieldSources, "category")
}
out.Notes = append(out.Notes, "category: ignored (unknown unique_id for company)")
}
@@ -0,0 +1,117 @@
package processing
import (
"context"
"fmt"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// CategoryBackfillResult is the outcome of BackfillProcessedCategoriesFromMapped.
type CategoryBackfillResult struct {
Updated int64 `json:"updated"`
}
// BackfillProcessedCategoriesFromMapped fills empty processed_products.category from
// raw_products.mapped_data when a usable unique_id is present. When the company has
// categories rows, only known unique_ids are applied (same rule as processOne filter).
// Safe for admin "Fix catalog" and migrator reuse. Does not invent categories via vector.
func BackfillProcessedCategoriesFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (CategoryBackfillResult, error) {
var out CategoryBackfillResult
if pool == nil {
return out, fmt.Errorf("category backfill: nil pool")
}
if companyID == uuid.Nil {
return out, fmt.Errorf("category backfill: empty company id")
}
ct, err := pool.Exec(ctx, `
WITH src AS (
SELECT
p.id AS processed_id,
BTRIM(COALESCE(
CASE
WHEN jsonb_typeof(r.mapped_data->'category') IN ('string', 'number')
THEN NULLIF(BTRIM(r.mapped_data->>'category'), '')
ELSE NULL
END,
NULLIF(BTRIM(r.mapped_data->>'category_unique_id'), ''),
NULLIF(BTRIM(r.mapped_data->'category'->>'unique_id'), ''),
NULLIF(BTRIM(r.mapped_data->'category'->>'category_unique_id'), ''),
NULLIF(BTRIM(r.mapped_data->'category'->>'#text'), ''),
NULLIF(BTRIM(r.mapped_data->'category'->>'id'), ''),
NULLIF(BTRIM(r.mapped_data->'category'->>'value'), ''),
NULLIF(BTRIM(r.mapped_data->'category'->>'code'), '')
)) AS cat_raw
FROM processed_products p
INNER JOIN raw_products r ON r.id = p.raw_product_id AND r.company_id = p.company_id
WHERE p.company_id = $1
AND (
COALESCE(NULLIF(BTRIM(p.category), ''), '') = ''
OR LOWER(BTRIM(p.category)) = 'none'
)
),
mapped AS (
SELECT
processed_id,
CASE
WHEN cat_raw IS NULL OR cat_raw = '' OR LOWER(cat_raw) = 'none' THEN NULL
ELSE cat_raw
END AS cat_uid
FROM src
),
eligible AS (
SELECT m.processed_id, m.cat_uid
FROM mapped m
WHERE m.cat_uid IS NOT NULL
AND (
NOT EXISTS (SELECT 1 FROM categories c WHERE c.company_id = $1)
OR EXISTS (
SELECT 1 FROM categories c
WHERE c.company_id = $1
AND BTRIM(c.unique_id) = m.cat_uid
)
)
)
UPDATE processed_products p
SET category = e.cat_uid,
field_sources = jsonb_set(
COALESCE(p.field_sources, '{}'::jsonb),
'{category}',
'"mapped_backfill"'::jsonb,
true
),
updated_at = now()
FROM eligible e
WHERE p.id = e.processed_id`, companyID)
if err != nil {
return out, fmt.Errorf("category backfill: %w", err)
}
out.Updated = ct.RowsAffected()
return out, nil
}
// CountProcessedCategoryCoverage returns how many processed rows for companyID have
// a non-empty category vs empty/"none".
func CountProcessedCategoryCoverage(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (withCat, withoutCat int64, err error) {
if pool == nil {
return 0, 0, fmt.Errorf("category coverage: nil pool")
}
err = pool.QueryRow(ctx, `
SELECT
COUNT(*) FILTER (
WHERE COALESCE(NULLIF(BTRIM(category), ''), '') <> ''
AND LOWER(BTRIM(category)) <> 'none'
),
COUNT(*) FILTER (
WHERE COALESCE(NULLIF(BTRIM(category), ''), '') = ''
OR LOWER(BTRIM(category)) = 'none'
)
FROM processed_products
WHERE company_id = $1`, companyID).Scan(&withCat, &withoutCat)
if err != nil {
return 0, 0, fmt.Errorf("category coverage: %w", err)
}
return withCat, withoutCat, nil
}
@@ -0,0 +1,404 @@
package processing
import (
"context"
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestCategoryUniqueIDFromAny(t *testing.T) {
t.Parallel()
cases := []struct {
in any
want string
}{
{nil, ""},
{"", ""},
{"none", ""},
{"NONE", ""},
{"50", "50"},
{50, "50"},
{float64(50), "50"},
{map[string]any{"unique_id": "28"}, "28"},
{map[string]any{"category_unique_id": 48}, "48"},
{map[string]any{"#text": "120"}, "120"},
{[]any{map[string]any{"unique_id": "7"}}, "7"},
{map[string]any{"name": "OnlyName"}, ""},
}
for i, tc := range cases {
if got := categoryUniqueIDFromAny(tc.in); got != tc.want {
t.Fatalf("case %d: got %q want %q", i, got, tc.want)
}
}
}
func TestRunSteps_full_mappedUniqueIDCategory(t *testing.T) {
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{Text: `{"name":"N","description":"D"}`, TotalTokens: 1}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
GTIN: "8606019604493",
Mapped: map[string]any{
"name": "Cooker",
"description": "stove",
"category": "50",
},
}, "full", nil, StepPolicy{AllowAI: true, AllowEPREL: false})
if err != nil {
t.Fatal(err)
}
if out.Category != "50" {
t.Fatalf("category=%q want 50", out.Category)
}
if src, _ := out.FieldSources["category"].(string); src != "mapped" {
t.Fatalf("field_sources.category=%v want mapped", out.FieldSources["category"])
}
}
func TestRunSteps_full_categoryUniqueIDKey(t *testing.T) {
e := &Engine{Vector: NoopVectorCategorizer{}}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Mapped: map[string]any{
"name": "Headphones",
"category_unique_id": "48",
},
}, "normalize_only", nil, StepPolicy{})
if err != nil {
t.Fatal(err)
}
if out.Category != "48" {
t.Fatalf("category=%q want 48", out.Category)
}
}
func TestRunSteps_full_nestedCategoryUniqueID(t *testing.T) {
e := &Engine{Vector: NoopVectorCategorizer{}}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Mapped: map[string]any{
"name": "Dryer",
"category": map[string]any{
"unique_id": "52",
"name": "Sušilni stroji",
},
},
}, "normalize_only", nil, StepPolicy{})
if err != nil {
t.Fatal(err)
}
if out.Category != "52" {
t.Fatalf("category=%q want 52", out.Category)
}
}
func TestCategoryDisplayLabel_andSync(t *testing.T) {
t.Parallel()
out := StepResult{Category: "50"}
syncCategoryName(&out, map[string]string{"50": "Štedilniki"})
if out.CategoryName != "Štedilniki" {
t.Fatalf("CategoryName=%q", out.CategoryName)
}
if out.Category != "50" {
t.Fatalf("Category unique_id mutated: %q", out.Category)
}
if got := categoryDisplayLabel(out); got != "Štedilniki" {
t.Fatalf("display=%q", got)
}
synth := synthesizeDescriptionFromTitle("Cooker X", categoryDisplayLabel(out), "sl", nil)
if strings.Contains(synth, "kategoriji 50") || strings.Contains(synth, " 50") {
t.Fatalf("synth used unique_id: %q", synth)
}
if !strings.Contains(synth, "Štedilniki") {
t.Fatalf("synth missing display name: %q", synth)
}
}
func TestFilterCategoryIfInvalid(t *testing.T) {
t.Parallel()
out := StepResult{Category: "50", FieldSources: map[string]any{"category": "mapped"}}
filterCategoryIfInvalid(&out, map[string]struct{}{"50": {}})
if out.Category != "50" {
t.Fatalf("valid unique_id cleared: %q", out.Category)
}
filterCategoryIfInvalid(&out, map[string]struct{}{"99": {}})
if out.Category != "" {
t.Fatalf("invalid unique_id kept: %q", out.Category)
}
out.Category = "50"
filterCategoryIfInvalid(&out, nil)
if out.Category != "50" {
t.Fatalf("empty valid set should skip filter: %q", out.Category)
}
}
func TestLoadV1ProcessJobItems_resolvesCategoryName(t *testing.T) {
dsn := os.Getenv("DATABASE_URL")
if dsn == "" {
t.Skip("DATABASE_URL not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
pg, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal(err)
}
defer pg.Close()
companyID := uuid.New()
userID := uuid.New()
rawID := uuid.New()
jobID := uuid.New()
ppID := uuid.New()
gtin := "cat-v1-" + companyID.String()[:8]
if _, err := pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`, companyID, "cat-v1-test"); err != nil {
t.Fatal(err)
}
// users.id may be required for processing_jobs — reuse an existing user when possible.
err = pg.QueryRow(ctx, `SELECT id FROM users ORDER BY created_at DESC LIMIT 1`).Scan(&userID)
if err != nil {
t.Skip("no users rows available")
}
defer func() {
_, _ = pg.Exec(context.Background(), `DELETE FROM processing_job_products WHERE job_id = $1`, jobID)
_, _ = pg.Exec(context.Background(), `DELETE FROM processing_jobs WHERE id = $1`, jobID)
_, _ = pg.Exec(context.Background(), `DELETE FROM processed_products WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM raw_products WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM categories WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
}()
if _, err := pg.Exec(ctx, `
INSERT INTO categories (id, company_id, name, unique_id, is_active)
VALUES (gen_random_uuid(), $1, 'Štedilniki', '50', true)`, companyID); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO raw_products (id, company_id, gtin, raw_data, mapped_data, is_processed, processing_status)
VALUES ($1, $2, $3, '{}'::jsonb, '{"category":"50","name":"Cooker"}'::jsonb, true, 'processed')`,
rawID, companyID, gtin); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO processed_products (
id, company_id, product_id, name, category, description, processed_name, processed_description,
raw_product_id, status, attributes, processed_attributes, field_sources
) VALUES (
$1, $2, $3, 'Cooker', '50', 'stove', 'Cooker', 'stove',
$4, 'needs_review', '{}'::jsonb, '{}'::jsonb, '{"category":"mapped"}'::jsonb
)`, ppID, companyID, gtin, rawID); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO processing_jobs (id, company_id, user_id, status, processing_type, total_products, processed_products)
VALUES ($1, $2, $3, 'completed', 'full', 1, 1)`, jobID, companyID, userID); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO processing_job_products (id, job_id, raw_product_id, processed_product_id, status)
VALUES (gen_random_uuid(), $1, $2, $3, 'processed')`, jobID, rawID, ppID); err != nil {
t.Fatal(err)
}
p := NewPipeline(pg)
items, err := p.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
if err != nil {
t.Fatal(err)
}
if len(items) != 1 {
t.Fatalf("items=%d want 1", len(items))
}
if got := fmt.Sprint(items[0]["category"]); got != "50" {
t.Fatalf("category=%v want 50", items[0]["category"])
}
if got := fmt.Sprint(items[0]["category_name"]); got != "Štedilniki" {
t.Fatalf("category_name=%v want Štedilniki", items[0]["category_name"])
}
}
type stubVectorCategorizer struct {
enabled bool
cat string
err error
calls int
}
func (s *stubVectorCategorizer) Enabled() bool { return s.enabled }
func (s *stubVectorCategorizer) SuggestCategory(context.Context, string, string, []string) (string, error) {
s.calls++
return s.cat, s.err
}
func TestRunSteps_vectorCategoryRequiresAllowAI(t *testing.T) {
vec := &stubVectorCategorizer{enabled: true, cat: "28"}
e := &Engine{Vector: vec}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "TV mount",
Mapped: map[string]any{
"name": "TV mount",
},
}, "normalize_only", nil, StepPolicy{AllowAI: false})
if err != nil {
t.Fatal(err)
}
if out.Category != "" {
t.Fatalf("category=%q want empty when AllowAI=false", out.Category)
}
if vec.calls != 0 {
t.Fatalf("vector calls=%d want 0", vec.calls)
}
found := false
for _, n := range out.Notes {
if strings.Contains(n, "category: unset") && strings.Contains(n, "AI/vector not allowed") {
found = true
break
}
}
if !found {
t.Fatalf("expected unset note, got %v", out.Notes)
}
}
func TestRunSteps_vectorCategoryWhenAllowAI(t *testing.T) {
vec := &stubVectorCategorizer{enabled: true, cat: "28"}
e := &Engine{Vector: vec}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "TV mount",
Mapped: map[string]any{
"name": "TV mount",
},
}, "normalize_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
if out.Category != "28" {
t.Fatalf("category=%q want 28", out.Category)
}
if src, _ := out.FieldSources["category"].(string); src != "vector" {
t.Fatalf("field_sources.category=%v want vector", out.FieldSources["category"])
}
if vec.calls != 1 {
t.Fatalf("vector calls=%d want 1", vec.calls)
}
}
func TestRunSteps_mappedCategorySkipsVector(t *testing.T) {
vec := &stubVectorCategorizer{enabled: true, cat: "99"}
e := &Engine{Vector: vec}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Mapped: map[string]any{
"name": "Cooker",
"category": "50",
},
}, "normalize_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
if out.Category != "50" {
t.Fatalf("category=%q want 50", out.Category)
}
if vec.calls != 0 {
t.Fatalf("vector should not run when mapped unique_id present: calls=%d", vec.calls)
}
}
func TestBackfillProcessedCategoriesFromMapped(t *testing.T) {
dsn := os.Getenv("DATABASE_URL")
if dsn == "" {
t.Skip("DATABASE_URL not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
pg, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal(err)
}
defer pg.Close()
companyID := uuid.New()
rawID := uuid.New()
ppID := uuid.New()
gtin := "cat-bf-" + companyID.String()[:8]
if _, err := pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`, companyID, "cat-backfill-test"); err != nil {
t.Fatal(err)
}
defer func() {
_, _ = pg.Exec(context.Background(), `DELETE FROM processed_products WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM raw_products WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM categories WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
}()
if _, err := pg.Exec(ctx, `
INSERT INTO categories (id, company_id, name, unique_id, is_active)
VALUES (gen_random_uuid(), $1, 'TV mounts', '28', true)`, companyID); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO raw_products (id, company_id, gtin, raw_data, mapped_data, is_processed, processing_status)
VALUES ($1, $2, $3, '{}'::jsonb, '{"name":"Mount","category":{"unique_id":"28","name":"TV"}}'::jsonb, true, 'processed')`,
rawID, companyID, gtin); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO processed_products (
id, company_id, product_id, name, category, description, processed_name, processed_description,
raw_product_id, status, attributes, processed_attributes, field_sources
) VALUES (
$1, $2, $3, 'Mount', '', 'x', 'Mount', 'x',
$4, 'needs_review', '{}'::jsonb, '{}'::jsonb, '{}'::jsonb
)`, ppID, companyID, gtin, rawID); err != nil {
t.Fatal(err)
}
res, err := BackfillProcessedCategoriesFromMapped(ctx, pg, companyID)
if err != nil {
t.Fatal(err)
}
if res.Updated != 1 {
t.Fatalf("updated=%d want 1", res.Updated)
}
var cat, src string
if err := pg.QueryRow(ctx, `
SELECT COALESCE(category, ''), COALESCE(field_sources->>'category', '')
FROM processed_products WHERE id = $1`, ppID).Scan(&cat, &src); err != nil {
t.Fatal(err)
}
if cat != "28" {
t.Fatalf("category=%q want 28", cat)
}
if src != "mapped_backfill" {
t.Fatalf("field_sources.category=%q want mapped_backfill", src)
}
// Idempotent.
res2, err := BackfillProcessedCategoriesFromMapped(ctx, pg, companyID)
if err != nil {
t.Fatal(err)
}
if res2.Updated != 0 {
t.Fatalf("second backfill updated=%d want 0", res2.Updated)
}
withCat, withoutCat, err := CountProcessedCategoryCoverage(ctx, pg, companyID)
if err != nil {
t.Fatal(err)
}
if withCat != 1 || withoutCat != 0 {
t.Fatalf("coverage with=%d without=%d", withCat, withoutCat)
}
}
@@ -4,6 +4,8 @@ import (
"context" "context"
"strings" "strings"
"testing" "testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
) )
func TestHashEnhanceInput_stableAndSensitive(t *testing.T) { func TestHashEnhanceInput_stableAndSensitive(t *testing.T) {
@@ -41,18 +43,19 @@ func TestRunSteps_skipsEnhanceWhenInputHashUnchanged(t *testing.T) {
}}, }},
Vector: NoopVectorCategorizer{}, Vector: NoopVectorCategorizer{},
} }
priorDesc := "Cached description with enough retail detail for listing copy."
in := ProductInput{ in := ProductInput{
Name: "Widget", Name: "Widget",
Description: "A widget", Description: "A widget with enough mapped detail for hashing.",
Mapped: map[string]any{"name": "Widget", "description": "A widget"}, Mapped: map[string]any{"name": "Widget", "description": "A widget with enough mapped detail for hashing."},
PriorProcessedName: "Cached Widget", PriorProcessedName: "Cached Widget",
PriorProcessedDescription: "Cached description.", PriorProcessedDescription: priorDesc,
} }
// First pass without prior hash to compute the hash shape via enhance path is awkward; // First pass without prior hash to compute the hash shape via enhance path is awkward;
// compute the same hash RunSteps will see after normalize (name/desc from mapped). // compute the same hash RunSteps will see after normalize (name/desc from mapped).
// enhance_only: normalize then enhance with out.Name from normalized. // enhance_only: normalize then enhance with out.Name from normalized.
normName := "Widget" normName := "Widget"
normDesc := "A widget" normDesc := "A widget with enough mapped detail for hashing."
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{}) sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{})
in.PriorEnhanceHash = HashEnhanceInput("", normName, normDesc, "", "", sysTpl, userTpl, map[string]any{}) in.PriorEnhanceHash = HashEnhanceInput("", normName, normDesc, "", "", sysTpl, userTpl, map[string]any{})
@@ -66,7 +69,7 @@ func TestRunSteps_skipsEnhanceWhenInputHashUnchanged(t *testing.T) {
if out.ProcessedName != "Cached Widget" { if out.ProcessedName != "Cached Widget" {
t.Fatalf("name=%q", out.ProcessedName) t.Fatalf("name=%q", out.ProcessedName)
} }
if out.ProcessedDescription != "Cached description." { if out.ProcessedDescription != priorDesc {
t.Fatalf("desc=%q", out.ProcessedDescription) t.Fatalf("desc=%q", out.ProcessedDescription)
} }
if out.TotalTokens != 0 { if out.TotalTokens != 0 {
@@ -84,6 +87,15 @@ func TestRunSteps_skipsEnhanceWhenInputHashUnchanged(t *testing.T) {
if shouldDebitProductProcessing(false, out) { if shouldDebitProductProcessing(false, out) {
t.Fatal("processOne must not debit when enhance unchanged") t.Fatal("processOne must not debit when enhance unchanged")
} }
// Hash-skip (0 tokens) must still persist known engine mode, not "unknown".
if out.AIProviderMode != AIProviderInternal {
t.Fatalf("ai_provider_mode=%q want %q on hash-skip", out.AIProviderMode, AIProviderInternal)
}
// processOne treats unknown as empty and prefers job modeLabel.
got := preferKnownProviderMode(AIProviderUnknown, AIProviderInternal)
if got != AIProviderInternal {
t.Fatalf("preferKnownProviderMode(unknown, internal)=%q", got)
}
joined := strings.Join(out.Notes, ";") joined := strings.Join(out.Notes, ";")
if !strings.Contains(joined, "unchanged") { if !strings.Contains(joined, "unchanged") {
t.Fatalf("notes=%v", out.Notes) t.Fatalf("notes=%v", out.Notes)
@@ -128,3 +140,171 @@ func TestRunSteps_callsEnhanceWhenInputHashDiffers(t *testing.T) {
t.Fatalf("expected new hash in field_sources, got %q", h) t.Fatalf("expected new hash in field_sources, got %q", h)
} }
} }
func TestRunSteps_reenhancesWhenPriorDescEqualsTitle(t *testing.T) {
calls := 0
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
calls++
return Completion{Text: `{"name":"Widget Pro","description":"A durable retail widget for everyday use with clear specs."}`, TotalTokens: 5}, nil
}},
Vector: NoopVectorCategorizer{},
}
in := ProductInput{
Name: "Widget",
Description: "Widget",
Mapped: map[string]any{"name": "Widget", "description": "Widget"},
PriorProcessedName: "Widget",
PriorProcessedDescription: "Widget",
}
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{})
in.PriorEnhanceHash = HashEnhanceInput("", "Widget", "Widget", "", "", sysTpl, userTpl, map[string]any{})
out, err := e.RunSteps(context.Background(), "co", in, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if calls != 1 {
t.Fatalf("expected LLM re-enhance when prior desc equals title, calls=%d", calls)
}
if out.ProcessedDescription == "Widget" || out.ProcessedDescription == "" {
t.Fatalf("desc=%q", out.ProcessedDescription)
}
if out.SkipCreditDebit {
t.Fatal("title-echo prior must still debit")
}
}
func TestPreferredProductDescription_skipsTitleEcho(t *testing.T) {
title := "Acme Widget Pro"
if got := preferredProductDescription(title, title, "Acme Widget Pro.", "A durable retail widget for everyday use."); got != "A durable retail widget for everyday use." {
t.Fatalf("got %q", got)
}
if got := preferredProductDescription(title, title, strings.ToUpper(title)); got != "" {
t.Fatalf("expected empty when all candidates echo title, got %q", got)
}
if got := preferredProductDescription(title, "", "<nil>", "Category:"); got != "" {
t.Fatalf("expected empty for unusable candidates, got %q", got)
}
}
func TestRunSteps_emptyDescriptionGetsNonemptyProcessedDesc(t *testing.T) {
calls := 0
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
calls++
return Completion{Text: `{"name":"Gigabyte Monitor","description":""}`, TotalTokens: 4}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "Gigabyte Monitor",
Mapped: map[string]any{"name": "Gigabyte Monitor", "brand": "Gigabyte"},
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if calls != 1 {
t.Fatalf("expected enhance call, calls=%d", calls)
}
if strings.TrimSpace(out.ProcessedDescription) == "" {
t.Fatal("expected nonempty ProcessedDescription when title set")
}
if descriptionEchoesTitle(out.ProcessedDescription, out.ProcessedName) {
t.Fatalf("ProcessedDescription must not echo title: name=%q desc=%q", out.ProcessedName, out.ProcessedDescription)
}
if descriptionEchoesTitle(out.ProcessedDescription, "Gigabyte Monitor") {
t.Fatalf("ProcessedDescription must not echo mapped title: desc=%q", out.ProcessedDescription)
}
}
func TestRunSteps_thinOKDoesNotPersistEnhanceHash(t *testing.T) {
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{Text: `{"name":"Monitor","description":"ok"}`, TotalTokens: 2}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "Monitor",
Description: "Monitor",
Mapped: map[string]any{"name": "Monitor", "description": "Monitor"},
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if h, _ := out.FieldSources[FieldEnhanceInputHash].(string); h != "" && isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName) {
t.Fatalf("must not persist hash with weak desc: hash=%q desc=%q", h, out.ProcessedDescription)
}
for _, lf := range out.LocalizedContent {
if lf.EnhanceInputHash != "" && isWeakPriorEnhanceDescription(lf.ProcessedDescription, lf.ProcessedName) {
t.Fatalf("localized hash poisoned with weak desc: %+v", lf)
}
}
}
func TestRunSteps_weakPriorHashDoesNotSkipReprocess(t *testing.T) {
calls := 0
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
calls++
return Completion{Text: `{"name":"Widget Pro","description":"A durable retail widget for everyday use with clear specs."}`, TotalTokens: 5}, nil
}},
Vector: NoopVectorCategorizer{},
}
normName := "Widget"
normDesc := "Widget"
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{})
priorHash := HashEnhanceInput("", normName, normDesc, "", "", sysTpl, userTpl, map[string]any{})
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "Widget",
Description: "Widget",
Mapped: map[string]any{"name": "Widget", "description": "Widget"},
PriorEnhanceHash: priorHash,
PriorProcessedName: "Widget",
PriorProcessedDescription: "ok",
PriorLocalized: company.LocalizedContent{
"en": {ProcessedName: "Widget", ProcessedDescription: "ok", EnhanceInputHash: priorHash},
},
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if calls != 1 {
t.Fatalf("weak prior desc must force completer, calls=%d", calls)
}
if out.SkipCreditDebit {
t.Fatal("weak prior must not SkipCreditDebit")
}
if out.ProcessedDescription == "ok" || out.ProcessedDescription == "Widget" {
t.Fatalf("desc=%q", out.ProcessedDescription)
}
}
func TestRunSteps_failedEnhanceDoesNotPoisonLocalizedHash(t *testing.T) {
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{}, context.DeadlineExceeded
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Name: "Widget",
Mapped: map[string]any{"name": "Widget", "description": "A widget with enough mapped detail for hashing."},
PriorEnhanceHash: "should-not-survive",
PriorLocalized: company.LocalizedContent{
"en": {ProcessedName: "Old", ProcessedDescription: "Old desc that is long enough to look real.", EnhanceInputHash: "poison"},
},
}, "enhance_only", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if _, ok := out.FieldSources[FieldEnhanceInputHash]; ok {
t.Fatalf("failed enhance must not keep field_sources hash, got %v", out.FieldSources[FieldEnhanceInputHash])
}
for lang, lf := range out.LocalizedContent {
if lf.EnhanceInputHash != "" {
t.Fatalf("lang %s: failed enhance must clear localized hash, got %q", lang, lf.EnhanceInputHash)
}
}
}
+169
View File
@@ -2,7 +2,10 @@ package processing
import ( import (
"context" "context"
"encoding/json"
"strings"
"testing" "testing"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel" "github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
) )
@@ -51,6 +54,12 @@ func TestRunSteps_EPREL_mergesAttributes(t *testing.T) {
if out.ProcessedAttributes["eprel_energy_class"] != "C" { if out.ProcessedAttributes["eprel_energy_class"] != "C" {
t.Fatalf("class missing: %v", out.ProcessedAttributes) t.Fatalf("class missing: %v", out.ProcessedAttributes)
} }
if out.ProcessedAttributes["energy_class"] != "C" {
t.Fatalf("energy_class not promoted: %v", out.ProcessedAttributes)
}
if out.EPREL["id"] != "246834" || out.EPREL["label"] == "" {
t.Fatalf("out.EPREL=%v", out.EPREL)
}
} }
func TestRunSteps_EPREL_disabledOrMissingID(t *testing.T) { func TestRunSteps_EPREL_disabledOrMissingID(t *testing.T) {
@@ -76,3 +85,163 @@ func TestRunSteps_EPREL_disabledOrMissingID(t *testing.T) {
t.Fatal("should not call fetch without id") t.Fatal("should not call fetch without id")
} }
} }
func TestRunSteps_EPREL_readsIDFromParsedAttrs(t *testing.T) {
st := &stubEprel{
enabled: true,
data: &eprel.Data{
ID: "999001",
Label: "https://eprel.ec.europa.eu/api/product/999001/labels?format=png",
PDF: "https://eprel.ec.europa.eu/fiches/y.pdf",
EnergyClass: "B",
EnergyScale: "A-G",
},
}
e := &Engine{EPREL: st, Completer: HeuristicCompleter{}, Vector: NoopVectorCategorizer{}}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Mapped: map[string]any{
"name": "Washer",
"specifications": map[string]any{
"eprel_id": "999001",
"brand": "Samsung",
},
},
}, "eprel_only", nil, StepPolicy{AllowAI: false, AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if st.calls != 1 || st.lastID != "999001" {
t.Fatalf("calls=%d id=%q", st.calls, st.lastID)
}
if out.EPREL["id"] != "999001" || out.EPREL["energy_class"] != "B" {
t.Fatalf("eprel=%v", out.EPREL)
}
if out.ProcessedAttributes["energy_class"] != "B" {
t.Fatalf("energy_class not promoted: %v", out.ProcessedAttributes)
}
if out.ProcessedAttributes["eprel_id"] != "999001" {
t.Fatalf("eprel_id missing: %v", out.ProcessedAttributes)
}
// Nested eprel object must survive sanitize/persist for V1 extract.
nested, _ := out.ProcessedAttributes["eprel"].(map[string]any)
if nested == nil || nested["label"] == nil {
t.Fatalf("nested eprel missing: %v", out.ProcessedAttributes["eprel"])
}
extracted := extractEPRELFromAttrs(out.ProcessedAttributes)
em, ok := extracted.(map[string]any)
if !ok || em["id"] != "999001" || em["label"] == nil || em["pdf"] == nil || em["energy_class"] != "B" {
t.Fatalf("V1 extract=%v", extracted)
}
}
func TestRunSteps_EPREL_skipsPlaceholderID(t *testing.T) {
st := &stubEprel{enabled: true}
e := &Engine{EPREL: st, Completer: HeuristicCompleter{}, Vector: NoopVectorCategorizer{}}
_, err := e.RunSteps(context.Background(), "co", ProductInput{
Mapped: map[string]any{"eprel_id": "0000"},
}, "eprel_only", nil, StepPolicy{AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if st.calls != 0 {
t.Fatalf("placeholder id must not fetch, calls=%d", st.calls)
}
}
func TestExtractEPRELFromAttrs_flatAndNested(t *testing.T) {
got := extractEPRELFromAttrs(map[string]any{
"eprel_id": "1632113",
"eprel_label": "https://eprel.ec.europa.eu/api/product/1632113/labels?format=png",
"eprel_pdf_url": "https://eprel.ec.europa.eu/fiches/x.pdf",
"eprel_energy_class": "A",
"brand": "Samsung",
})
m, ok := got.(map[string]any)
if !ok {
t.Fatalf("type=%T", got)
}
if m["id"] != "1632113" || m["label"] == nil || m["pdf"] == nil || m["energy_class"] != "A" {
t.Fatalf("got=%v", m)
}
// Nested legacy aliases must fold onto public keys.
got2 := extractEPRELFromAttrs(map[string]any{
"eprel": map[string]any{
"eprel_id": "999",
"eprel_label": "https://eprel.example/label",
"eprel_pdf": "https://eprel.example/pdf",
"eprel_energy_class": "B",
},
})
m2, ok := got2.(map[string]any)
if !ok || m2["id"] != "999" || m2["energy_class"] != "B" || m2["label"] == nil || m2["pdf"] == nil {
t.Fatalf("nested aliases: %v", got2)
}
// V1 poll must strip eprel_* from attributes after extract.
clean := SanitizeV1ProcessAttributesAllowed(map[string]any{
"eprel_id": "1632113",
"eprel_label": "https://eprel.example/label",
"brand": "Samsung",
"eprel_energy_class": "A",
}, map[string]struct{}{})
if _, ok := clean["eprel_id"]; ok {
t.Fatalf("eprel leaked into attributes: %v", clean)
}
if clean["brand"] != "Samsung" {
t.Fatalf("brand missing: %v", clean)
}
}
func TestRunSteps_EPREL_liveFetchIfNetwork(t *testing.T) {
if testing.Short() {
t.Skip("short")
}
client := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second})
if !client.Enabled() {
t.Fatal("client disabled")
}
e := &Engine{EPREL: client, Completer: HeuristicCompleter{}, Vector: NoopVectorCategorizer{}}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
Mapped: map[string]any{
"name": "Samsung WW90CGC04DAELE",
"eprel_id": "1632113",
},
}, "eprel_only", nil, StepPolicy{AllowEPREL: true})
if err != nil {
t.Fatal(err)
}
if out.EPREL["id"] != "1632113" {
t.Fatalf("eprel=%v", out.EPREL)
}
if out.EPREL["label"] == nil || out.EPREL["label"] == "" {
t.Fatalf("missing label: %v", out.EPREL)
}
if out.EPREL["pdf"] == nil || out.EPREL["pdf"] == "" {
t.Fatalf("missing pdf: %v", out.EPREL)
}
if out.EPREL["energy_class"] == nil || out.EPREL["energy_class"] == "" {
t.Fatalf("missing energy_class: %v", out.EPREL)
}
if out.ProcessedAttributes["eprel_id"] != "1632113" {
t.Fatalf("attrs=%v", out.ProcessedAttributes)
}
eprelVal := extractEPRELFromAttrs(out.ProcessedAttributes)
m, ok := eprelVal.(map[string]any)
if !ok || m["id"] != "1632113" {
t.Fatalf("extract=%v", eprelVal)
}
if m["label"] == nil || m["label"] == "" || m["pdf"] == nil || m["pdf"] == "" || m["energy_class"] == nil || m["energy_class"] == "" {
t.Fatalf("extract missing fields: %v", m)
}
clean := SanitizeV1ProcessAttributesAllowed(out.ProcessedAttributes, map[string]struct{}{})
for k := range clean {
if strings.HasPrefix(strings.ToLower(k), "eprel") {
t.Fatalf("eprel leaked into V1 attributes: %v", clean)
}
}
sample, _ := json.MarshalIndent(map[string]any{
"eprel": m,
"attributes": clean,
"step_eprel": out.EPREL,
}, "", " ")
t.Logf("sample eprel JSON:\n%s", sample)
}
+8 -19
View File
@@ -1,7 +1,6 @@
package processing package processing
import ( import (
"fmt"
"regexp" "regexp"
"strings" "strings"
) )
@@ -67,10 +66,12 @@ func FillMissingFields(mapped map[string]any, attrs map[string]any) map[string]a
} }
} }
if stringFromAny(out["category"]) == "" { if c := categoryUniqueIDFromAny(out["category"]); c != "" {
if c := stringFromAny(attrs["category"]); c != "" { out["category"] = c
} else if c := categoryUniqueIDFromAny(attrs["category"]); c != "" {
out["category"] = c
} else if c := categoryUniqueIDFromAny(out["category_unique_id"]); c != "" {
out["category"] = c out["category"] = c
}
} }
if stringFromAny(out["stock_status"]) == "" { if stringFromAny(out["stock_status"]) == "" {
@@ -142,19 +143,7 @@ func normalizeStockStatusLabel(s string) string {
} }
func stringFromAny(v any) string { func stringFromAny(v any) string {
if v == nil { // Flatten arrays / #text wrappers; callers that need HTML-stripped descriptions
return "" // should use PlainDescriptionFromAny (normalize + V1 enforce do that).
} return coerceScalarText(v)
switch t := v.(type) {
case string:
return strings.TrimSpace(t)
case float64, float32, int, int64, bool:
s := strings.TrimSpace(fmt.Sprint(t))
if s == "<nil>" {
return ""
}
return s
default:
return ""
}
} }
@@ -0,0 +1,127 @@
package processing
import (
"context"
"fmt"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// FixCompanyCatalogOpts controls optional Fix A1 / catalog hygiene steps.
type FixCompanyCatalogOpts struct {
// BackfillCategories copies mapped_data category unique_ids onto empty
// processed_products.category.
BackfillCategories bool
// ReprocessSampleLimit caps recommended raw_product_ids in the response
// (0 = counts only, no sample ids).
ReprocessSampleLimit int
// AIPrompts optionally re-applies BuiltInDefaults product_enhance per content language.
AIPrompts *aiprompts.Service
}
// FixCompanyCatalogResult is the idempotent admin Fix A1 report.
type FixCompanyCatalogResult struct {
CompanyID uuid.UUID `json:"company_id"`
CompanyName string `json:"company_name"`
A1Cohort bool `json:"a1_cohort"`
CategoryAttributeOrphansRemoved int `json:"category_attribute_orphans_removed"`
CategoryAttributeLinks int `json:"category_attribute_links"`
CategoryPromptsUpdated int `json:"category_prompts_updated"`
CategoryPromptsAlreadyOK int `json:"category_prompts_already_ok"`
CategoryPromptsEmpty int `json:"category_prompts_empty_skipped"`
// Prompts aliases category_prompts_updated for flash.admin.fixA1Success {prompts}.
Prompts int `json:"prompts"`
ProductEnhanceLanguages int `json:"product_enhance_languages"`
WeakHashesCleared int `json:"weak_hashes_cleared"`
CategoriesBackfilled int `json:"categories_backfilled"`
// Hashes / Categories alias the long names for flash.admin.fixA1Success.
Hashes int `json:"hashes"`
Categories int `json:"categories"`
DescriptionsNormalized int `json:"descriptions_normalized"`
DescriptionsBackfilled int `json:"descriptions_backfilled"`
MappedDescriptionsFlat int `json:"mapped_descriptions_flattened"`
MetaBackfilled int `json:"meta_backfilled"`
AttributesSanitized int64 `json:"attributes_sanitized"`
ProductsScanned int `json:"products_scanned"`
ReprocessNeededCount int `json:"reprocess_needed_count"`
ReprocessSampleRawProductIDs []uuid.UUID `json:"reprocess_sample_raw_product_ids,omitempty"`
}
// FixCompanyCatalog runs the admin Fix A1 / catalog hygiene sequence in place.
// Never clears the catalog or uses A1 as a clone destination.
// Step 2 applies RepairCompanyCategoryEnhancePrompts (same repaired map as
// RepairA1DemoCategoryEnhancePrompts / cmd/repair-category-prompts -apply).
// Legacy converters + weak-hash clear live in FixCatalogHygieneWithIDs.
func FixCompanyCatalog(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID, companyName string, a1Cohort bool, opts FixCompanyCatalogOpts) (FixCompanyCatalogResult, error) {
out := FixCompanyCatalogResult{
CompanyID: companyID,
CompanyName: companyName,
A1Cohort: a1Cohort,
}
if pool == nil {
return out, fmt.Errorf("nil pool")
}
orphans, links, err := catalog.EnsureCategoryAttributeLinks(ctx, pool, companyID)
if err != nil {
return out, err
}
out.CategoryAttributeOrphansRemoved = orphans
out.CategoryAttributeLinks = links
updated, alreadyOK, emptySkipped, err := catalog.RepairCompanyCategoryEnhancePrompts(ctx, pool, companyID)
if err != nil {
return out, err
}
out.CategoryPromptsUpdated = updated
out.CategoryPromptsAlreadyOK = alreadyOK
out.CategoryPromptsEmpty = emptySkipped
if opts.AIPrompts != nil {
n, err := opts.AIPrompts.ApplyBuiltInProductEnhance(ctx, companyID)
if err != nil {
return out, fmt.Errorf("product_enhance templates: %w", err)
}
out.ProductEnhanceLanguages = n
}
hygiene, clearedIDs, err := FixCatalogHygieneWithIDs(ctx, pool, companyID, opts.BackfillCategories)
if err != nil {
return out, err
}
out.WeakHashesCleared = hygiene.WeakHashesCleared
out.CategoriesBackfilled = hygiene.CategoriesBackfilled
out.DescriptionsNormalized = hygiene.DescriptionsNormalized
out.DescriptionsBackfilled = hygiene.DescriptionsBackfilled
out.MappedDescriptionsFlat = hygiene.MappedDescriptionsFlat
out.MetaBackfilled = hygiene.MetaBackfilled
out.ProductsScanned = hygiene.ProductsScanned
sanitized, err := BackfillCompanyProcessedAttributes(ctx, pool, companyID)
if err != nil {
return out, fmt.Errorf("sanitize attributes: %w", err)
}
out.AttributesSanitized = sanitized
needed, sample, err := RecommendReprocessRawProductIDs(ctx, pool, companyID, clearedIDs, opts.ReprocessSampleLimit)
if err != nil {
return out, err
}
out.ReprocessNeededCount = needed
out.ReprocessSampleRawProductIDs = sample
// Short keys for flash.admin.fixA1Success {prompts,hashes,categories}.
out.Prompts = out.CategoryPromptsUpdated
out.Hashes = out.WeakHashesCleared
out.Categories = out.CategoriesBackfilled
return out, nil
}
@@ -0,0 +1,23 @@
package processing
import (
"testing"
"github.com/google/uuid"
)
func TestFixCompanyCatalogResultFlashAliases(t *testing.T) {
t.Parallel()
out := FixCompanyCatalogResult{
CompanyID: uuid.New(),
CategoryPromptsUpdated: 3,
WeakHashesCleared: 7,
CategoriesBackfilled: 11,
}
out.Prompts = out.CategoryPromptsUpdated
out.Hashes = out.WeakHashesCleared
out.Categories = out.CategoriesBackfilled
if out.Prompts != 3 || out.Hashes != 7 || out.Categories != 11 {
t.Fatalf("flash aliases mismatch: %+v", out)
}
}
@@ -0,0 +1,216 @@
package processing
import (
"encoding/json"
"fmt"
"strconv"
"strings"
)
// AppendFormulaConstraints appends language-agnostic title/description formula
// guidance to the enhance user template (before {{var}} render). Empty templates
// are no-ops. Shared skeleton stays in company/built-in prompts; formulas only
// constrain structure for the active language via {{language}} elsewhere.
func AppendFormulaConstraints(userTpl string, titleTemplate, descriptionTemplate any) string {
userTpl = strings.TrimSpace(userTpl)
titleBlock := FormatTitleFormulaConstraint(titleTemplate)
descBlock := FormatDescriptionFormulaConstraint(descriptionTemplate)
if titleBlock == "" && descBlock == "" {
return userTpl
}
var b strings.Builder
if userTpl != "" {
b.WriteString(userTpl)
b.WriteString("\n\n")
}
if titleBlock != "" {
b.WriteString(titleBlock)
if descBlock != "" {
b.WriteString("\n\n")
}
}
if descBlock != "" {
b.WriteString(descBlock)
}
return b.String()
}
// FormatTitleFormulaConstraint turns categories.title_template into enhance-prompt
// constraints (element order: literal text + attribute variable keys).
func FormatTitleFormulaConstraint(template any) string {
elements, separator, ok := parseTitleFormula(template)
if !ok || len(elements) == 0 {
return ""
}
sep := separator
if sep == "" {
sep = " "
}
var b strings.Builder
b.WriteString("Title formula (order matters; join with ")
b.WriteString(strconv.Quote(sep))
b.WriteString("). Build name from Attrs using this structure:\n")
for i, el := range elements {
switch el.Type {
case "text":
fmt.Fprintf(&b, "%d. text %s\n", i+1, strconv.Quote(el.Value))
case "variable":
key := strings.TrimSpace(el.Value)
if key == "" {
continue
}
fmt.Fprintf(&b, "%d. attr [%s]\n", i+1, key)
default:
continue
}
}
b.WriteString("Prefer Attrs values for [attr] slots; keep literal text as written; write name in {{language}}.")
return strings.TrimSpace(b.String())
}
// FormatDescriptionFormulaConstraint turns categories.description_template sections
// into bullet instructions for the enhance user prompt.
func FormatDescriptionFormulaConstraint(template any) string {
sections, ok := parseDescriptionFormulaSections(template)
if !ok || len(sections) == 0 {
return ""
}
var b strings.Builder
b.WriteString("Description formula (cover each section in order; write in {{language}}):\n")
for _, s := range sections {
typ := strings.TrimSpace(s.Type)
instr := strings.TrimSpace(s.Instructions)
if typ == "" && instr == "" {
continue
}
if typ == "" {
typ = "section"
}
if instr == "" {
fmt.Fprintf(&b, "- %s\n", typ)
continue
}
fmt.Fprintf(&b, "- %s: %s\n", typ, instr)
}
return strings.TrimSpace(b.String())
}
type titleFormulaElement struct {
Type string `json:"type"`
Value string `json:"value"`
}
type descriptionFormulaSection struct {
Type string `json:"type"`
Instructions string `json:"instructions"`
}
func parseTitleFormula(template any) (elements []titleFormulaElement, separator string, ok bool) {
if template == nil {
return nil, "", false
}
obj, err := asObjectMap(template)
if err != nil || obj == nil {
return nil, "", false
}
sep, _ := obj["separator"].(string)
rawEls, exists := obj["elements"]
if !exists || rawEls == nil {
return nil, "", false
}
b, err := json.Marshal(rawEls)
if err != nil {
return nil, "", false
}
var parsed []titleFormulaElement
if err := json.Unmarshal(b, &parsed); err != nil {
return nil, "", false
}
out := make([]titleFormulaElement, 0, len(parsed))
for _, el := range parsed {
t := strings.ToLower(strings.TrimSpace(el.Type))
v := strings.TrimSpace(el.Value)
if t != "text" && t != "variable" {
continue
}
if v == "" {
continue
}
out = append(out, titleFormulaElement{Type: t, Value: v})
}
if len(out) == 0 {
return nil, "", false
}
return out, sep, true
}
func parseDescriptionFormulaSections(template any) ([]descriptionFormulaSection, bool) {
if template == nil {
return nil, false
}
obj, err := asObjectMap(template)
if err != nil || obj == nil {
return nil, false
}
raw, exists := obj["sections"]
if !exists || raw == nil {
return nil, false
}
b, err := json.Marshal(raw)
if err != nil {
return nil, false
}
var parsed []descriptionFormulaSection
if err := json.Unmarshal(b, &parsed); err != nil {
return nil, false
}
out := make([]descriptionFormulaSection, 0, len(parsed))
for _, s := range parsed {
t := strings.ToLower(strings.TrimSpace(s.Type))
instr := strings.TrimSpace(s.Instructions)
if t == "" && instr == "" {
continue
}
out = append(out, descriptionFormulaSection{Type: t, Instructions: instr})
}
return out, len(out) > 0
}
// categoryFormulasFor resolves title/description templates for a category key
// (unique_id or name). Explicit ProductInput fields win over the job cache map.
func categoryFormulasFor(in ProductInput, category string) (title, description any) {
if in.TitleTemplate != nil || in.DescriptionTemplate != nil {
return in.TitleTemplate, in.DescriptionTemplate
}
key := strings.ToLower(strings.TrimSpace(category))
if key == "" || len(in.CategoryFormulasByKey) == 0 {
return nil, nil
}
f, ok := in.CategoryFormulasByKey[key]
if !ok {
return nil, nil
}
return f.TitleTemplate, f.DescriptionTemplate
}
func asObjectMap(v any) (map[string]any, error) {
switch t := v.(type) {
case map[string]any:
return t, nil
case nil:
return nil, nil
default:
b, err := json.Marshal(t)
if err != nil {
return nil, err
}
if len(b) == 0 || string(b) == "null" {
return nil, nil
}
var obj map[string]any
if err := json.Unmarshal(b, &obj); err != nil {
return nil, err
}
return obj, nil
}
}
@@ -0,0 +1,118 @@
package processing
import (
"strings"
"testing"
)
func TestFormatTitleFormulaConstraint(t *testing.T) {
t.Parallel()
got := FormatTitleFormulaConstraint(map[string]any{
"separator": " ",
"elements": []any{
map[string]any{"type": "variable", "value": "brand"},
map[string]any{"type": "text", "value": "TV"},
map[string]any{"type": "variable", "value": "product_model"},
},
})
if !strings.Contains(got, `join with " "`) {
t.Fatalf("missing separator: %s", got)
}
if !strings.Contains(got, "1. attr [brand]") || !strings.Contains(got, `2. text "TV"`) || !strings.Contains(got, "3. attr [product_model]") {
t.Fatalf("missing ordered elements: %s", got)
}
if !strings.Contains(got, "{{language}}") {
t.Fatalf("missing language placeholder: %s", got)
}
if FormatTitleFormulaConstraint(nil) != "" || FormatTitleFormulaConstraint(map[string]any{}) != "" {
t.Fatal("empty templates should yield empty constraint")
}
}
func TestFormatDescriptionFormulaConstraint(t *testing.T) {
t.Parallel()
got := FormatDescriptionFormulaConstraint(map[string]any{
"sections": []any{
map[string]any{"type": "h1", "instructions": "Main product heading"},
map[string]any{"type": "p", "instructions": "Two factual sentences"},
map[string]any{"type": "ul", "instructions": "3 bullets of specs"},
},
})
if !strings.Contains(got, "- h1: Main product heading") {
t.Fatalf("missing h1: %s", got)
}
if !strings.Contains(got, "- p: Two factual sentences") || !strings.Contains(got, "- ul: 3 bullets of specs") {
t.Fatalf("missing sections: %s", got)
}
if !strings.Contains(got, "{{language}}") {
t.Fatalf("missing language placeholder: %s", got)
}
}
func TestAppendFormulaConstraints_injectedIntoResolve(t *testing.T) {
t.Parallel()
title := map[string]any{
"separator": "-",
"elements": []any{
map[string]any{"type": "variable", "value": "brand"},
map[string]any{"type": "variable", "value": "product_model"},
},
}
desc := map[string]any{
"sections": []any{
map[string]any{"type": "p", "instructions": "Factual summary"},
},
}
_, user := resolveProductPromptTemplates(ProductInput{
EnhanceUserTemplate: "Name: {{name}}\nAttrs: {{attrs}}",
TitleTemplate: title,
DescriptionTemplate: desc,
})
if !strings.Contains(user, "Name: {{name}}") {
t.Fatalf("lost base template: %s", user)
}
if !strings.Contains(user, "Title formula") || !strings.Contains(user, "attr [brand]") {
t.Fatalf("missing title formula: %s", user)
}
if !strings.Contains(user, "Description formula") || !strings.Contains(user, "- p: Factual summary") {
t.Fatalf("missing description formula: %s", user)
}
// Render substitutes {{language}} in formula blocks.
_, rendered := RenderProductEnhancePrompts(
"Write in {{language}}.", user,
"50", "Old Name", "", "", "", "sl",
map[string]any{"brand": "Vox", "product_model": "EHT6020"},
)
if !strings.Contains(rendered, "Slovenian") {
t.Fatalf("formula language not rendered: %s", rendered)
}
if strings.Contains(rendered, "{{language}}") {
t.Fatalf("unrendered language placeholder left: %s", rendered)
}
}
func TestCategoryFormulasFor_explicitWins(t *testing.T) {
t.Parallel()
in := ProductInput{
TitleTemplate: map[string]any{"elements": []any{map[string]any{"type": "text", "value": "X"}}},
CategoryFormulasByKey: map[string]CategoryFormulas{
"50": {TitleTemplate: map[string]any{"elements": []any{map[string]any{"type": "text", "value": "Y"}}}},
},
}
title, _ := categoryFormulasFor(in, "50")
m, _ := title.(map[string]any)
els, _ := m["elements"].([]any)
first, _ := els[0].(map[string]any)
if first["value"] != "X" {
t.Fatalf("explicit should win, got %#v", title)
}
in2 := ProductInput{
CategoryFormulasByKey: map[string]CategoryFormulas{
"50": {DescriptionTemplate: map[string]any{"sections": []any{map[string]any{"type": "p", "instructions": "Hi"}}}},
},
}
_, desc := categoryFormulasFor(in2, "50")
if FormatDescriptionFormulaConstraint(desc) == "" {
t.Fatal("expected cache formula")
}
}
@@ -0,0 +1,382 @@
package processing
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// BackfillDescriptionsFromMapped fills empty processed_products.description from
// linked raw mapped_data (mirrors migrator backfillProcessedDescriptionsFromMapped)
// and coerces array/HTML shapes via PlainDescriptionFromAny.
func BackfillDescriptionsFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
if pool == nil {
return 0, fmt.Errorf("description backfill: nil pool")
}
if companyID == uuid.Nil {
return 0, fmt.Errorf("description backfill: empty company id")
}
rows, err := pool.Query(ctx, `
SELECT p.id,
COALESCE(p.description, ''),
COALESCE(r.mapped_data, '{}'::jsonb),
COALESCE(r.raw_data, '{}'::jsonb)
FROM processed_products p
INNER JOIN raw_products r ON r.id = p.raw_product_id AND r.company_id = p.company_id
WHERE p.company_id = $1
AND COALESCE(NULLIF(BTRIM(p.description), ''), '') = ''`, companyID)
if err != nil {
return 0, fmt.Errorf("description backfill query: %w", err)
}
defer rows.Close()
for rows.Next() {
var (
id uuid.UUID
prior string
mappedB []byte
rawB []byte
)
if err := rows.Scan(&id, &prior, &mappedB, &rawB); err != nil {
return updated, fmt.Errorf("description backfill scan: %w", err)
}
mapped, err := decodeJSONObject(mappedB)
if err != nil {
return updated, err
}
raw, err := decodeJSONObject(rawB)
if err != nil {
return updated, err
}
desc := PlainDescriptionFromAny(mapped["description"])
if desc == "" {
desc = PlainDescriptionFromAny(mapped["desc"])
}
if desc == "" {
desc = PlainDescriptionFromAny(mapped["body"])
}
if desc == "" {
desc = PlainDescriptionFromAny(raw["description"])
}
if desc == "" {
continue
}
ct, err := pool.Exec(ctx, `
UPDATE processed_products
SET description = $2,
updated_at = now()
WHERE id = $1 AND company_id = $3
AND COALESCE(NULLIF(BTRIM(description), ''), '') = ''`,
id, desc, companyID)
if err != nil {
return updated, fmt.Errorf("description backfill update %s: %w", id, err)
}
if ct.RowsAffected() > 0 {
updated++
}
_ = prior
}
return updated, rows.Err()
}
// mappedFeedTitle picks a clean name/title from mapped or raw feed maps.
func mappedFeedTitle(mapped, raw map[string]any) string {
for _, src := range []map[string]any{mapped, raw} {
if src == nil {
continue
}
for _, key := range []string{"name", "title", "product_name", "product_title"} {
v := strings.TrimSpace(stringFromAny(src[key]))
if v == "" || v == "<nil>" || isPromptLabelTitle(v) {
continue
}
return SanitizeOutput(v)
}
}
return ""
}
// BackfillPollutedNamesFromMapped clears processed_name / name when they match
// prompt-leakage patterns (Title formula / short retail title / …) and replaces
// them with mapped_data name/title when available. Next Load/FixCatalog or
// reprocess then shows the feed title instead of instruction text.
func BackfillPollutedNamesFromMapped(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
if pool == nil {
return 0, fmt.Errorf("name pollution backfill: nil pool")
}
if companyID == uuid.Nil {
return 0, fmt.Errorf("name pollution backfill: empty company id")
}
rows, err := pool.Query(ctx, `
SELECT p.id,
COALESCE(p.name, ''),
COALESCE(p.processed_name, ''),
COALESCE(r.mapped_data, '{}'::jsonb),
COALESCE(r.raw_data, '{}'::jsonb)
FROM processed_products p
INNER JOIN raw_products r ON r.id = p.raw_product_id AND r.company_id = p.company_id
WHERE p.company_id = $1`, companyID)
if err != nil {
return 0, fmt.Errorf("name pollution backfill query: %w", err)
}
defer rows.Close()
for rows.Next() {
var (
id uuid.UUID
name, processedName string
mappedB, rawB []byte
)
if err := rows.Scan(&id, &name, &processedName, &mappedB, &rawB); err != nil {
return updated, fmt.Errorf("name pollution backfill scan: %w", err)
}
namePolluted := isPromptLabelTitle(name)
procPolluted := isPromptLabelTitle(processedName)
if !namePolluted && !procPolluted {
continue
}
mapped, err := decodeJSONObject(mappedB)
if err != nil {
return updated, err
}
raw, err := decodeJSONObject(rawB)
if err != nil {
return updated, err
}
feed := mappedFeedTitle(mapped, raw)
if feed == "" {
continue
}
newName := name
newProcessed := processedName
if namePolluted {
newName = feed
}
if procPolluted {
newProcessed = feed
}
if newName == name && newProcessed == processedName {
continue
}
ct, err := pool.Exec(ctx, `
UPDATE processed_products
SET name = $2,
processed_name = $3,
field_sources = jsonb_set(
COALESCE(field_sources, '{}'::jsonb),
'{name}',
'"mapped_pollution_backfill"'::jsonb,
true
),
updated_at = now()
WHERE id = $1 AND company_id = $4`,
id, newName, newProcessed, companyID)
if err != nil {
return updated, fmt.Errorf("name pollution backfill update %s: %w", id, err)
}
if ct.RowsAffected() > 0 {
updated++
}
}
return updated, rows.Err()
}
// BackfillMissingMeta fills empty meta_title / meta_description using the same
// free template fillMetaFromResult uses at process time (no AI / no credits).
func BackfillMissingMeta(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
if pool == nil {
return 0, fmt.Errorf("meta backfill: nil pool")
}
if companyID == uuid.Nil {
return 0, fmt.Errorf("meta backfill: empty company id")
}
rows, err := pool.Query(ctx, `
SELECT id,
COALESCE(name, ''),
COALESCE(processed_name, ''),
COALESCE(category, ''),
COALESCE(description, ''),
COALESCE(processed_description, ''),
COALESCE(meta_title, ''),
COALESCE(meta_description, ''),
COALESCE(attributes, '{}'::jsonb),
COALESCE(processed_attributes, '{}'::jsonb)
FROM processed_products
WHERE company_id = $1
AND (
COALESCE(NULLIF(BTRIM(meta_title), ''), '') = ''
OR COALESCE(NULLIF(BTRIM(meta_description), ''), '') = ''
OR BTRIM(meta_title) ~ '\|\s+[0-9]+$'
OR LOWER(meta_title) LIKE '%short retail title%'
OR LOWER(meta_title) LIKE '%title formula%'
OR LOWER(meta_title) LIKE '%follow any%'
OR LOWER(meta_title) LIKE '%constraints that follow%'
OR LOWER(meta_title) LIKE '%prefer 1-3%'
OR LOWER(meta_title) LIKE '%reply with only json%'
OR LOWER(meta_title) LIKE '%your reply is parsed as json%'
OR LOWER(meta_description) LIKE '%prefer 1-3%'
OR LOWER(meta_description) LIKE '%short retail title%'
OR LOWER(meta_description) LIKE '%title formula%'
OR LOWER(meta_description) LIKE '%do not emit%'
)`, companyID)
if err != nil {
return 0, fmt.Errorf("meta backfill query: %w", err)
}
defer rows.Close()
for rows.Next() {
var (
id uuid.UUID
name, processedName, category string
desc, processedDesc, metaTitle, metaDesc string
attrsJSON, procAttrsJSON []byte
)
if err := rows.Scan(&id, &name, &processedName, &category, &desc, &processedDesc,
&metaTitle, &metaDesc, &attrsJSON, &procAttrsJSON); err != nil {
return updated, fmt.Errorf("meta backfill scan: %w", err)
}
attrs := decodeAttrMap(attrsJSON)
procAttrs := decodeAttrMap(procAttrsJSON)
synthTitle, synthDesc := fillMetaFromResult(StepResult{
Name: name,
ProcessedName: processedName,
Category: category,
Description: PlainDescriptionFromAny(desc),
ProcessedDescription: PlainDescriptionFromAny(processedDesc),
Attributes: attrs,
ProcessedAttributes: procAttrs,
})
newTitle := strings.TrimSpace(metaTitle)
newDesc := strings.TrimSpace(metaDesc)
if newTitle == "" || isPoisonedMetaTitle(newTitle) {
newTitle = synthTitle
}
if newDesc == "" || isPromptLeakageTitle(newDesc) ||
(isPoisonedMetaTitle(strings.TrimSpace(metaTitle)) &&
(isWeakPriorEnhanceDescription(newDesc, name, processedName) || isPromptLeakageTitle(newDesc))) {
newDesc = synthDesc
}
if newTitle == strings.TrimSpace(metaTitle) && newDesc == strings.TrimSpace(metaDesc) {
continue
}
if newTitle == "" && newDesc == "" {
continue
}
ct, err := pool.Exec(ctx, `
UPDATE processed_products
SET meta_title = CASE
WHEN COALESCE(NULLIF(BTRIM(meta_title), ''), '') = ''
OR BTRIM(meta_title) ~ '\|\s+[0-9]+$'
OR LOWER(meta_title) LIKE '%short retail title%'
OR LOWER(meta_title) LIKE '%title formula%'
OR LOWER(meta_title) LIKE '%follow any%'
OR LOWER(meta_title) LIKE '%constraints that follow%'
OR LOWER(meta_title) LIKE '%prefer 1-3%'
OR LOWER(meta_title) LIKE '%reply with only json%'
OR LOWER(meta_title) LIKE '%your reply is parsed as json%'
THEN NULLIF(BTRIM($2), '')
ELSE meta_title
END,
meta_description = CASE
WHEN COALESCE(NULLIF(BTRIM(meta_description), ''), '') = '' THEN NULLIF(BTRIM($3), '')
WHEN LOWER(meta_description) LIKE '%prefer 1-3%'
OR LOWER(meta_description) LIKE '%short retail title%'
OR LOWER(meta_description) LIKE '%title formula%'
OR LOWER(meta_description) LIKE '%do not emit%'
THEN NULLIF(BTRIM($3), '')
WHEN (
BTRIM(meta_title) ~ '\|\s+[0-9]+$'
OR LOWER(meta_title) LIKE '%short retail title%'
OR LOWER(meta_title) LIKE '%title formula%'
OR LOWER(meta_title) LIKE '%follow any%'
OR LOWER(meta_title) LIKE '%constraints that follow%'
OR LOWER(meta_title) LIKE '%prefer 1-3%'
OR LOWER(meta_title) LIKE '%reply with only json%'
OR LOWER(meta_title) LIKE '%your reply is parsed as json%'
)
AND (
char_length(BTRIM(meta_description)) < 40
OR LOWER(meta_description) LIKE '%ready for retail listing%'
OR LOWER(meta_description) LIKE '%quality product ready%'
OR LOWER(meta_description) LIKE '%product description%'
OR LOWER(meta_description) LIKE '%based on available specifications%'
OR LOWER(meta_description) LIKE '%with available specifications%'
OR LOWER(meta_description) LIKE '%available catalog details%'
OR LOWER(meta_description) LIKE '%pripravljeno za prodajo%'
OR LOWER(meta_description) LIKE '%na podlagi razpoložljivih specifikacij%'
OR LOWER(meta_description) LIKE '%prefer 1-3%'
)
THEN NULLIF(BTRIM($3), '')
ELSE meta_description
END,
updated_at = now()
WHERE id = $1 AND company_id = $4`,
id, newTitle, newDesc, companyID)
if err != nil {
return updated, fmt.Errorf("meta backfill update %s: %w", id, err)
}
if ct.RowsAffected() > 0 {
updated++
}
}
return updated, rows.Err()
}
// FlattenMappedDescriptionArrays rewrites mapped_data.description when it is a
// JSON array / HTML blob into a plain string (local admin Fix A1 hygiene).
func FlattenMappedDescriptionArrays(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) (updated int, err error) {
if pool == nil {
return 0, fmt.Errorf("flatten mapped descriptions: nil pool")
}
rows, err := pool.Query(ctx, `
SELECT id, COALESCE(mapped_data, '{}'::jsonb)
FROM raw_products
WHERE company_id = $1
AND mapped_data ? 'description'
AND jsonb_typeof(mapped_data->'description') <> 'string'`, companyID)
if err != nil {
return 0, fmt.Errorf("flatten mapped descriptions query: %w", err)
}
defer rows.Close()
for rows.Next() {
var id uuid.UUID
var mappedB []byte
if err := rows.Scan(&id, &mappedB); err != nil {
return updated, err
}
mapped, err := decodeJSONObject(mappedB)
if err != nil {
return updated, err
}
rawDesc := mapped["description"]
plain := PlainDescriptionFromAny(rawDesc)
if plain == "" {
continue
}
// Skip when already a plain string equal to conversion (shouldn't hit query).
if s, ok := rawDesc.(string); ok && v1PlainDescription(s) == plain {
continue
}
mapped["description"] = plain
encoded, err := json.Marshal(mapped)
if err != nil {
return updated, err
}
ct, err := pool.Exec(ctx, `
UPDATE raw_products
SET mapped_data = $2::jsonb, updated_at = now()
WHERE id = $1 AND company_id = $3`, id, string(encoded), companyID)
if err != nil {
return updated, fmt.Errorf("flatten mapped description %s: %w", id, err)
}
if ct.RowsAffected() > 0 {
updated++
}
}
return updated, rows.Err()
}
@@ -0,0 +1,111 @@
package processing
import (
"fmt"
"strings"
)
// isDescriptionFieldKey reports keys whose values should be plain text scalars
// (not JSON arrays / nested HTML wrappers) for process + V1 poll contracts.
func isDescriptionFieldKey(key string) bool {
switch strings.ToLower(strings.TrimSpace(key)) {
case "description", "desc", "body", "short_description", "shortdescription",
"processed_description", "long_description", "longdescription":
return true
default:
return false
}
}
// PlainDescriptionFromAny coerces legacy description shapes to a single plain
// string: string, {#text}, or arrays of those (joined with newlines). HTML is
// stripped via v1PlainDescription so poll/store never keep ["…"] or markup blobs.
func PlainDescriptionFromAny(v any) string {
if v == nil {
return ""
}
switch t := v.(type) {
case string:
return v1PlainDescription(t)
case []string:
parts := make([]string, 0, len(t))
for _, s := range t {
if p := v1PlainDescription(s); p != "" {
parts = append(parts, p)
}
}
return strings.Join(parts, "\n")
case []any:
parts := make([]string, 0, len(t))
for _, item := range t {
if p := PlainDescriptionFromAny(item); p != "" {
parts = append(parts, p)
}
}
return strings.Join(parts, "\n")
case map[string]any:
for _, k := range []string{"#text", "text", "value", "description", "body"} {
if p := PlainDescriptionFromAny(t[k]); p != "" {
return p
}
}
return ""
default:
s := strings.TrimSpace(fmt.Sprint(t))
if s == "" || s == "<nil>" {
return ""
}
return v1PlainDescription(s)
}
}
// coerceScalarText flattens arrays / #text wrappers into a trimmed string without
// HTML stripping (names, brands, stock labels, …).
func coerceScalarText(v any) string {
if v == nil {
return ""
}
switch t := v.(type) {
case string:
return strings.TrimSpace(t)
case []string:
parts := make([]string, 0, len(t))
for _, s := range t {
s = strings.TrimSpace(s)
if s != "" {
parts = append(parts, s)
}
}
return strings.Join(parts, " ")
case []any:
parts := make([]string, 0, len(t))
for _, item := range t {
if s := coerceScalarText(item); s != "" {
parts = append(parts, s)
}
}
return strings.Join(parts, " ")
case map[string]any:
for _, k := range []string{"#text", "text", "value"} {
if s := coerceScalarText(t[k]); s != "" {
return s
}
}
return ""
case float64, float32, int, int64, int32, bool:
s := strings.TrimSpace(fmt.Sprint(t))
if s == "<nil>" {
return ""
}
return s
default:
if n, ok := t.(jsonNumberStringer); ok {
return strings.TrimSpace(n.String())
}
s := strings.TrimSpace(fmt.Sprint(t))
if s == "" || s == "<nil>" {
return ""
}
return s
}
}
@@ -0,0 +1,61 @@
package processing
import "testing"
func TestPlainDescriptionFromAny_arrayAndHTML(t *testing.T) {
got := PlainDescriptionFromAny([]any{"Hello<br>World", "Second"})
if got != "Hello\nWorld\nSecond" {
t.Fatalf("array+html: %q", got)
}
got = PlainDescriptionFromAny(map[string]any{"#text": "Plain &amp; ok"})
if got != "Plain & ok" {
t.Fatalf("#text: %q", got)
}
if PlainDescriptionFromAny(nil) != "" {
t.Fatal("nil should be empty")
}
}
func TestNormalizeMapped_descriptionArrayBecomesPlainString(t *testing.T) {
got := NormalizeMapped(map[string]any{
"description": []any{"Line1<br/>A", "Line2"},
"name": "Widget",
}, nil)
desc, ok := got["description"].(string)
if !ok {
t.Fatalf("description should be string, got %T %v", got["description"], got["description"])
}
if desc != "Line1\nA\nLine2" {
t.Fatalf("description=%q", desc)
}
}
func TestNormalizeMapped_eprelIDAlias(t *testing.T) {
got := NormalizeMapped(map[string]any{"EPRELID": "246834"}, nil)
if got["eprel_id"] != "246834" {
t.Fatalf("eprel_id=%v", got["eprel_id"])
}
}
func TestStringFromAny_flattensArray(t *testing.T) {
got := stringFromAny([]any{"a", "b"})
if got != "a b" {
t.Fatalf("got %q", got)
}
}
func TestCanonicalizeAttrKey_eprel(t *testing.T) {
if canonicalizeAttrKey("EPRELID") != "eprel_id" {
t.Fatalf("got %q", canonicalizeAttrKey("EPRELID"))
}
if canonicalizeAttrKey("eprel") != "eprel_id" {
t.Fatalf("got %q", canonicalizeAttrKey("eprel"))
}
}
func TestPlainDescriptionFromStored_jsonArrayString(t *testing.T) {
got := plainDescriptionFromStored(`["Hello<br>World"]`)
if got != "Hello\nWorld" {
t.Fatalf("got %q", got)
}
}
@@ -0,0 +1,67 @@
package processing_test
import (
"context"
"os"
"strings"
"testing"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
)
// Live proof against local mock-llm (OPENAI_BASE_URL). Skipped unless LIVE_MOCK_LLM=1.
func TestLive_MockLLM_rejectsCategoryEnhanceInstructionTitle(t *testing.T) {
if strings.TrimSpace(os.Getenv("LIVE_MOCK_LLM")) == "" {
t.Skip("set LIVE_MOCK_LLM=1 with mock-llm on OPENAI_BASE_URL")
}
base := strings.TrimSpace(os.Getenv("OPENAI_BASE_URL"))
key := strings.TrimSpace(os.Getenv("OPENAI_API_KEY"))
model := strings.TrimSpace(os.Getenv("OPENAI_MODEL"))
if base == "" || key == "" {
t.Fatal("OPENAI_BASE_URL and OPENAI_API_KEY required")
}
if model == "" {
model = "mock-llm"
}
client := processing.NewOpenAIClient(key, base, model, 0, 2)
eng := &processing.Engine{
Completer: client,
Vector: processing.NoopVectorCategorizer{},
}
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
out, err := eng.RunSteps(ctx, "live-probe", processing.ProductInput{
GTIN: "8712285326882",
Name: "Vox SWA-8000W",
Description: "Washer dryer combo",
Mapped: map[string]any{"name": "Vox SWA-8000W", "description": "Washer dryer combo", "category": "demo-electronics", "brand": "Vox"},
Language: "en",
EnhanceUserTemplate: aiprompts.CategoryEnhanceUserTemplate,
EnhanceSystemTemplate: `Retail product copywriter. Reply with ONLY JSON. Schema: {"name":"string","description":"string"}`,
TitleTemplate: map[string]any{
"separator": " ",
"elements": []any{
map[string]any{"type": "variable", "value": "brand"},
map[string]any{"type": "text", "value": " "},
map[string]any{"type": "variable", "value": "product_model"},
},
},
}, "enhance_only", nil, processing.StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
t.Logf("ProcessedName=%q AIProviderMode=%q", out.ProcessedName, out.AIProviderMode)
lower := strings.ToLower(out.ProcessedName)
for _, p := range []string{"title formula", "follow any", "short retail title", "use attrs", "schema:", "reply with only json"} {
if strings.Contains(lower, p) {
t.Fatalf("polluted title %q contains %q", out.ProcessedName, p)
}
}
if !strings.Contains(lower, "vox") {
t.Fatalf("ProcessedName=%q want Vox feed title", out.ProcessedName)
}
}
+6 -3
View File
@@ -11,7 +11,10 @@ import (
// Local / weak-model defaults (8k-class context). See docs/local-llm-tuning.md. // Local / weak-model defaults (8k-class context). See docs/local-llm-tuning.md.
const ( const (
DefaultStructuredTemp = 0.2 DefaultStructuredTemp = 0.2
MaxTokensEnhance = 350 // Reasoning-capable OpenAI-compatible models (e.g. code-fast / Qwen3) spend
// hundredsthousands of tokens in reasoning_content before writing message.content.
// 350 capped mid-thought → empty content → "empty model response".
MaxTokensEnhance = 4096
MaxTokensSEO = 180 MaxTokensSEO = 180
MaxTokensCampaign = 650 MaxTokensCampaign = 650
MaxProductDescRunes = 400 MaxProductDescRunes = 400
@@ -152,14 +155,14 @@ func CompactAttrs(attrs map[string]any, maxKeys int) map[string]any {
keys := make([]string, 0, len(attrs)) keys := make([]string, 0, len(attrs))
for k := range attrs { for k := range attrs {
k = strings.TrimSpace(k) k = strings.TrimSpace(k)
if k == "" { if k == "" || isInvalidAttributeKey(k) {
continue continue
} }
keys = append(keys, k) keys = append(keys, k)
} }
sort.Strings(keys) sort.Strings(keys)
// Prefer common retail keys first. // Prefer common retail keys first.
priority := []string{"brand", "Brand", "color", "Color", "material", "Material", "size", "Size", "model", "Model", "gtin", "GTIN", "ean", "EAN"} priority := []string{"brand", "Brand", "product_model", "color", "Color", "material", "Material", "size", "Size", "model", "Model", "gtin", "GTIN", "ean", "EAN"}
ordered := make([]string, 0, len(keys)) ordered := make([]string, 0, len(keys))
seen := map[string]bool{} seen := map[string]bool{}
for _, p := range priority { for _, p := range priority {
+150
View File
@@ -0,0 +1,150 @@
package processing
import (
"regexp"
"strings"
"unicode"
)
// Meta title/description caps mirror seo.MetaTitleMaxChars / MetaDescriptionMaxChars.
// Duplicated here so processing can fill meta without importing seo (seo imports processing).
const (
metaTitleMaxChars = 60
metaDescriptionMaxChars = 155
)
// Bare "| <digits>" suffix = legacy unique_id leak (e.g. "VOX EBR700 | 50").
// Matches upsertProcessedProductSQL refresh predicate.
var poisonedMetaTitleUniqueIDRe = regexp.MustCompile(`\|\s+[0-9]+$`)
// isPoisonedMetaTitle reports meta_title that must be refreshed: bare unique_id
// suffix ("… | 50") or enhance-prompt instruction leakage ("short retail title",
// "Title formula", …). Matches upsert / BackfillMissingMeta SQL predicates.
func isPoisonedMetaTitle(s string) bool {
s = strings.TrimSpace(s)
if s == "" {
return false
}
if poisonedMetaTitleUniqueIDRe.MatchString(s) {
return true
}
return isPromptLeakageTitle(s)
}
// fillMetaFromResult builds free template meta from a StepResult (FillMetaTemplate-equivalent).
// Does not call AI / does not spend credits.
func fillMetaFromResult(result StepResult) (title, description string) {
name := strings.TrimSpace(result.ProcessedName)
if name == "" {
name = strings.TrimSpace(result.Name)
}
if name == "" {
name = "Product"
}
cat := categoryDisplayLabel(result)
brand := metaBrandFromAttrs(result.ProcessedAttributes, result.Attributes)
parts := make([]string, 0, 3)
if brand != "" && !strings.Contains(strings.ToLower(name), strings.ToLower(brand)) {
parts = append(parts, brand)
}
parts = append(parts, name)
if cat != "" && !strings.Contains(strings.ToLower(name), strings.ToLower(cat)) {
parts = append(parts, cat)
}
title = truncateMetaRunes(strings.Join(parts, " | "), metaTitleMaxChars)
body := strings.TrimSpace(result.ProcessedDescription)
if body == "" {
body = strings.TrimSpace(result.Description)
}
body = stripMetaTags(body)
if body == "" {
var bits []string
if brand != "" {
bits = append(bits, brand)
}
bits = append(bits, name)
if cat != "" {
bits = append(bits, "in "+cat)
}
body = strings.Join(bits, " ") + ". Shop quality products with clear specs and fast delivery."
}
description = truncateMetaDescription(body, metaDescriptionMaxChars)
return title, description
}
// truncateMetaDescription collapses whitespace and truncates at a word boundary when possible.
func truncateMetaDescription(s string, max int) string {
s = collapseMetaSpace(s)
if max <= 0 {
return ""
}
r := []rune(s)
if len(r) <= max {
return s
}
cut := r[:max]
lastSpace := -1
for i := len(cut) - 1; i >= 0; i-- {
if unicode.IsSpace(cut[i]) {
lastSpace = i
break
}
}
if lastSpace > max/2 {
return strings.TrimSpace(string(cut[:lastSpace]))
}
return string(cut)
}
func truncateMetaRunes(s string, max int) string {
s = collapseMetaSpace(s)
r := []rune(s)
if max <= 0 || len(r) <= max {
return s
}
if max <= 3 {
return string(r[:max])
}
return string(r[:max-1]) + "…"
}
func collapseMetaSpace(s string) string {
return strings.Join(strings.Fields(s), " ")
}
func stripMetaTags(s string) string {
var b strings.Builder
inTag := false
for _, r := range s {
switch {
case r == '<':
inTag = true
case r == '>':
inTag = false
case !inTag:
b.WriteRune(r)
}
}
return b.String()
}
func metaBrandFromAttrs(bags ...map[string]any) string {
keys := []string{"brand", "Brand", "manufacturer"}
for _, m := range bags {
if m == nil {
continue
}
for _, k := range keys {
if v, ok := m[k]; ok {
if s, ok := v.(string); ok {
if t := strings.TrimSpace(s); t != "" {
return t
}
}
}
}
}
return ""
}
+113
View File
@@ -0,0 +1,113 @@
package processing
import (
"strings"
"testing"
"unicode/utf8"
)
func TestFillMetaFromResult_buildsTitleAndDesc(t *testing.T) {
t.Parallel()
title, desc := fillMetaFromResult(StepResult{
ProcessedName: "Drill 18V",
Category: "Tools",
ProcessedDescription: "Cordless drill with battery pack for DIY projects.",
ProcessedAttributes: map[string]any{"brand": "Bosch"},
})
if title == "" || desc == "" {
t.Fatal("expected non-empty meta")
}
if !strings.Contains(title, "Drill 18V") {
t.Fatalf("title missing name: %q", title)
}
if !strings.Contains(title, "Bosch") {
t.Fatalf("title missing brand: %q", title)
}
if utf8.RuneCountInString(title) > metaTitleMaxChars {
t.Fatalf("title too long: %d", utf8.RuneCountInString(title))
}
if utf8.RuneCountInString(desc) > metaDescriptionMaxChars {
t.Fatalf("desc too long: %d", utf8.RuneCountInString(desc))
}
}
func TestFillMetaFromResult_usesCategoryNameNotUniqueID(t *testing.T) {
t.Parallel()
title, _ := fillMetaFromResult(StepResult{
ProcessedName: "Gorenje GEC5A21WG",
Category: "50",
CategoryName: "Štedilniki",
ProcessedAttributes: map[string]any{
"brand": "Gorenje",
},
})
if strings.Contains(title, "| 50") || strings.HasSuffix(title, "50") {
t.Fatalf("meta title must not use unique_id: %q", title)
}
if !strings.Contains(title, "Štedilniki") {
t.Fatalf("meta title missing display name: %q", title)
}
// Digits-only Category without CategoryName must omit category from title.
title2, _ := fillMetaFromResult(StepResult{
ProcessedName: "Cooker",
Category: "50",
})
if strings.Contains(title2, "50") {
t.Fatalf("digits-only unique_id must not appear in meta: %q", title2)
}
}
func TestTruncateMetaDescription_wordBoundary(t *testing.T) {
t.Parallel()
// 160+ runes with a clear word break near 155.
words := strings.Repeat("word ", 40) // 200 chars with spaces
out := truncateMetaDescription(words, v1MetaDescriptionMaxChars)
n := utf8.RuneCountInString(out)
if n > v1MetaDescriptionMaxChars {
t.Fatalf("len=%d want <= %d (%q)", n, v1MetaDescriptionMaxChars, out)
}
if strings.HasSuffix(out, "wor") || strings.HasSuffix(out, "wo") || strings.HasSuffix(out, "w") {
t.Fatalf("mid-word cut: %q", out)
}
if !strings.HasSuffix(out, "word") {
t.Fatalf("expected last full word, got %q", out)
}
}
func TestTruncateMetaDescription_collapsesSpace(t *testing.T) {
t.Parallel()
out := truncateMetaDescription(" hello \n\t world ", 155)
if out != "hello world" {
t.Fatalf("got %q", out)
}
}
func TestV1PollMetaFallback_usesTemplateWhenEmpty(t *testing.T) {
t.Parallel()
title := "Cordless Drill"
cat := "Tools"
plain := strings.Repeat("quality cordless drill for home and workshop use. ", 10)
synthTitle, synthDesc := fillMetaFromResult(StepResult{
Name: title,
ProcessedName: title,
Category: cat,
Description: plain,
ProcessedDescription: plain,
Attributes: map[string]any{"brand": "Makita"},
})
if synthTitle == "" || synthDesc == "" {
t.Fatal("expected synth meta")
}
if utf8.RuneCountInString(synthDesc) > v1MetaDescriptionMaxChars {
t.Fatalf("poll meta_description too long: %d", utf8.RuneCountInString(synthDesc))
}
// Word-safe: last rune should not be mid-word fragment from a hard cut.
r := []rune(synthDesc)
if len(r) == v1MetaDescriptionMaxChars {
// hard-cut path only when no good space; otherwise last char is letter from a word end
last := r[len(r)-1]
if last == ' ' {
t.Fatalf("trailing space in meta_description: %q", synthDesc)
}
}
}
+57
View File
@@ -85,6 +85,13 @@ func normalizeValue(key string, v any) any {
if v == nil { if v == nil {
return nil return nil
} }
// Descriptions must be plain scalars (never leftover ["…"] / HTML arrays).
if isDescriptionFieldKey(key) {
if s := PlainDescriptionFromAny(v); s != "" {
return SanitizeText(s)
}
return nil
}
switch t := v.(type) { switch t := v.(type) {
case string: case string:
s := strings.TrimSpace(t) s := strings.TrimSpace(t)
@@ -124,6 +131,13 @@ func normalizeValue(key string, v any) any {
if text, ok := t["text"]; ok { if text, ok := t["text"]; ok {
return normalizeValue(key, text) return normalizeValue(key, text)
} }
// Feed/XML category objects often carry unique_id — flatten to a scalar
// so StepResult.Category / processed_products.category get the code.
if key == "category" {
if id := categoryUniqueIDFromAny(t); id != "" {
return id
}
}
nested := make(map[string]any, len(t)) nested := make(map[string]any, len(t))
for nk, nv := range t { for nk, nv := range t {
if nn := normalizeValue(canonicalizeKey(nk), nv); nn != nil { if nn := normalizeValue(canonicalizeKey(nk), nv); nn != nil {
@@ -138,6 +152,14 @@ func normalizeValue(key string, v any) any {
if len(t) == 0 { if len(t) == 0 {
return nil return nil
} }
// Scalar-ish fields (name, brand, eprel_id, …): collapse one-element /
// joinable arrays so process steps never see []any where a string is expected.
if flat := coerceScalarText(t); flat != "" && looksLikeScalarFieldArray(t) {
if isDimensionKey(key) && isZeroishString(flat) {
return nil
}
return SanitizeText(flat)
}
out := make([]any, 0, len(t)) out := make([]any, 0, len(t))
for _, item := range t { for _, item := range t {
if nn := normalizeValue(key, item); nn != nil { if nn := normalizeValue(key, item); nn != nil {
@@ -160,6 +182,41 @@ func normalizeValue(key string, v any) any {
} }
} }
// looksLikeScalarFieldArray is true when every element coerces to a short scalar
// (string/number/#text) — not nested attribute/spec objects.
func looksLikeScalarFieldArray(items []any) bool {
if len(items) == 0 {
return false
}
for _, item := range items {
switch t := item.(type) {
case nil:
continue
case string, float64, float32, int, int64, int32, bool:
continue
case map[string]any:
if _, ok := t["#text"]; ok {
continue
}
if _, ok := t["text"]; ok {
continue
}
if _, ok := t["value"]; ok {
continue
}
return false
case []any, []string:
return false
default:
if _, ok := item.(jsonNumberStringer); ok {
continue
}
return false
}
}
return true
}
func isDimensionKey(key string) bool { func isDimensionKey(key string) bool {
switch key { switch key {
case "width", "height", "depth", "weight", "length", "net_width", "net_height", "net_depth", "net_mass": case "width", "height", "depth", "weight", "length", "net_width", "net_height", "net_depth", "net_mass":
+162 -8
View File
@@ -16,6 +16,7 @@ import (
"sync" "sync"
"time" "time"
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
"github.com/descrybe/descrybe-v2/apps/api/internal/config" "github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/descrybe/descrybe-v2/apps/api/internal/security" "github.com/descrybe/descrybe-v2/apps/api/internal/security"
) )
@@ -56,11 +57,14 @@ func NewOpenAIClient(apiKey, baseURL, model string, rpm, maxRetries int) *OpenAI
} }
// Dial-time SSRF. Loopback when base URL is local; RFC1918 only in non-prod. // Dial-time SSRF. Loopback when base URL is local; RFC1918 only in non-prod.
policy := openAIDialPolicy(baseURL) policy := openAIDialPolicy(baseURL)
// Header/body timeout: code-fast often exceeds 180s; 240s reduces false timeouts.
// Tradeoff: slower fail on hung upstream. Prefer synthesize fallback over infinite wait
// (RunSteps always synthesizes on timeout/error/empty — do not raise unboundedly).
return &OpenAIClient{ return &OpenAIClient{
APIKey: apiKey, APIKey: apiKey,
BaseURL: strings.TrimRight(baseURL, "/"), BaseURL: strings.TrimRight(baseURL, "/"),
Model: model, Model: model,
HTTPClient: security.SafeHTTPClientPolicy(60*time.Second, policy), HTTPClient: security.SafeHTTPClientPolicy(240*time.Second, policy),
MinInterval: interval, MinInterval: interval,
MaxRetries: maxRetries, MaxRetries: maxRetries,
ModeLabel: AIProviderInternal, ModeLabel: AIProviderInternal,
@@ -112,16 +116,31 @@ func (c *OpenAIClient) Enabled() bool {
} }
// ProviderModeLabel implements ProviderLabeler for analytics writes. // ProviderModeLabel implements ProviderLabeler for analytics writes.
// Loopback / mock-llm base URLs must not report as managed "internal".
func (c *OpenAIClient) ProviderModeLabel() string { func (c *OpenAIClient) ProviderModeLabel() string {
if c == nil { if c == nil {
return AIProviderUnknown return AIProviderUnknown
} }
if label := strings.TrimSpace(c.ModeLabel); label != "" { label := strings.TrimSpace(c.ModeLabel)
if IsMockOrLoopbackBaseURL(c.BaseURL) && (label == "" || label == AIProviderInternal) {
return AIProviderCustom
}
if label != "" {
return label return label
} }
return AIProviderInternal return AIProviderInternal
} }
// IsMockOrLoopbackBaseURL reports local mock / loopback OpenAI-compatible bases
// (e.g. mock-llm on 127.0.0.1:18767). Used for analytics ModeLabel accuracy.
func IsMockOrLoopbackBaseURL(baseURL string) bool {
if openAIBaseAllowsLoopback(baseURL) {
return true
}
lower := strings.ToLower(strings.TrimSpace(baseURL))
return strings.Contains(lower, "mock-llm")
}
type chatRequest struct { type chatRequest struct {
Model string `json:"model"` Model string `json:"model"`
Messages []chatMessage `json:"messages"` Messages []chatMessage `json:"messages"`
@@ -137,8 +156,11 @@ type chatMessage struct {
type chatResponse struct { type chatResponse struct {
Model string `json:"model"` Model string `json:"model"`
Choices []struct { Choices []struct {
FinishReason string `json:"finish_reason"`
Message struct { Message struct {
Content string `json:"content"` Content json.RawMessage `json:"content"`
ReasoningContent string `json:"reasoning_content"`
Reasoning string `json:"reasoning"`
} `json:"message"` } `json:"message"`
} `json:"choices"` } `json:"choices"`
Usage struct { Usage struct {
@@ -152,6 +174,12 @@ type chatResponse struct {
} `json:"error"` } `json:"error"`
} }
var errEmptyModelResponse = errors.New("empty model response")
// maxTokensReasoningBudget is used when a capped completion returns empty
// content with finish_reason=length (reasoning models).
const maxTokensReasoningBudget = 4096
func (c *OpenAIClient) Complete(ctx context.Context, system, user string) (Completion, error) { func (c *OpenAIClient) Complete(ctx context.Context, system, user string) (Completion, error) {
return c.CompleteWithOptions(ctx, system, user, CompleteOptions{}) return c.CompleteWithOptions(ctx, system, user, CompleteOptions{})
} }
@@ -169,6 +197,7 @@ func (c *OpenAIClient) CompleteWithOptions(ctx context.Context, system, user str
if temp > 0.3 { if temp > 0.3 {
temp = 0.3 temp = 0.3
} }
maxTok := opts.MaxTokens
var lastErr error var lastErr error
for attempt := 0; attempt <= c.MaxRetries; attempt++ { for attempt := 0; attempt <= c.MaxRetries; attempt++ {
@@ -184,11 +213,15 @@ func (c *OpenAIClient) CompleteWithOptions(ctx context.Context, system, user str
if err := c.waitRate(ctx); err != nil { if err := c.waitRate(ctx); err != nil {
return Completion{}, err return Completion{}, err
} }
comp, retryable, err := c.doComplete(ctx, system, user, temp, opts.MaxTokens) comp, retryable, err := c.doComplete(ctx, system, user, temp, maxTok)
if err == nil { if err == nil {
return comp, nil return comp, nil
} }
lastErr = err lastErr = err
if errors.Is(err, errEmptyModelResponse) && maxTok > 0 && maxTok < maxTokensReasoningBudget {
maxTok = maxTokensReasoningBudget
retryable = true
}
if !retryable { if !retryable {
return Completion{}, err return Completion{}, err
} }
@@ -386,11 +419,15 @@ func (c *OpenAIClient) doComplete(ctx context.Context, system, user string, temp
return Completion{}, false, errors.New(msg) return Completion{}, false, errors.New(msg)
} }
text := "" text := ""
finishReason := ""
if len(parsed.Choices) > 0 { if len(parsed.Choices) > 0 {
text = SanitizeOutput(parsed.Choices[0].Message.Content) finishReason = strings.TrimSpace(parsed.Choices[0].FinishReason)
text = SanitizeOutput(choiceMessageText(parsed.Choices[0].Message.Content, parsed.Choices[0].Message.ReasoningContent, parsed.Choices[0].Message.Reasoning))
} }
if text == "" { if text == "" {
return Completion{}, false, errors.New("empty model response") // Reasoning models often return empty content when max_tokens cuts mid-thought.
retryable := strings.EqualFold(finishReason, "length")
return Completion{}, retryable, errEmptyModelResponse
} }
return Completion{ return Completion{
Text: text, Text: text,
@@ -402,10 +439,58 @@ func (c *OpenAIClient) doComplete(ctx context.Context, system, user string, temp
"model": parsed.Model, "model": parsed.Model,
"usage": parsed.Usage, "usage": parsed.Usage,
"status": res.StatusCode, "status": res.StatusCode,
"finish_reason": finishReason,
}, },
}, false, nil }, false, nil
} }
// choiceMessageText reads assistant text from OpenAI-compatible chat responses.
// Prefer message.content (string or multipart text parts). When empty — common for
// reasoning models that only fill reasoning_content under a tight max_tokens —
// fall back to reasoning fields and prefer an embedded JSON object when present.
func choiceMessageText(content json.RawMessage, reasoningContent, reasoning string) string {
if text := decodeChatContent(content); text != "" {
return text
}
for _, alt := range []string{reasoningContent, reasoning} {
alt = strings.TrimSpace(alt)
if alt == "" {
continue
}
if obj := StripJSONFences(alt); strings.HasPrefix(obj, "{") {
if _, err := ParseJSONObject(obj); err == nil {
return obj
}
}
}
return ""
}
func decodeChatContent(raw json.RawMessage) string {
raw = bytes.TrimSpace(raw)
if len(raw) == 0 || string(raw) == "null" {
return ""
}
var s string
if err := json.Unmarshal(raw, &s); err == nil {
return strings.TrimSpace(s)
}
var parts []struct {
Type string `json:"type"`
Text string `json:"text"`
}
if err := json.Unmarshal(raw, &parts); err == nil {
var b strings.Builder
for _, p := range parts {
if strings.TrimSpace(p.Text) != "" {
b.WriteString(p.Text)
}
}
return strings.TrimSpace(b.String())
}
return ""
}
// HeuristicCompleter is used when OpenAI is not configured (local/dev fallback). // HeuristicCompleter is used when OpenAI is not configured (local/dev fallback).
type HeuristicCompleter struct{} type HeuristicCompleter struct{}
@@ -427,8 +512,8 @@ func (h HeuristicCompleter) Complete(_ context.Context, system, user string) (Co
name = "Product" name = "Product"
} }
desc := labeledPromptValue(user, "desc:", "description:", "current description:") desc := labeledPromptValue(user, "desc:", "description:", "current description:")
if desc == "" { if desc == "" || descriptionEchoesTitle(desc, name) || isWeakPriorEnhanceDescription(desc, name) {
desc = "Product description" desc = inventHeuristicDescription(system, user, name)
} }
b, _ := json.Marshal(map[string]string{"name": name, "description": desc}) b, _ := json.Marshal(map[string]string{"name": name, "description": desc})
text = string(b) text = string(b)
@@ -450,3 +535,72 @@ func (h HeuristicCompleter) Complete(_ context.Context, system, user string) (Co
Raw: map[string]any{"provider": "heuristic"}, Raw: map[string]any{"provider": "heuristic"},
}, nil }, nil
} }
// parseAttrsFromPrompt extracts the Attrs:/Attributes: JSON object from an enhance user message.
func parseAttrsFromPrompt(user string) map[string]any {
lower := strings.ToLower(user)
label := "attrs:"
at := strings.Index(lower, label)
if at < 0 {
label = "attributes:"
at = strings.Index(lower, label)
if at < 0 {
return nil
}
}
rest := strings.TrimSpace(user[at+len(label):])
if i := strings.Index(rest, "{"); i >= 0 {
rest = rest[i:]
if end := strings.Index(rest, "}"); end >= 0 {
rest = rest[:end+1]
}
} else {
rest = firstLine(rest)
}
var m map[string]any
if err := json.Unmarshal([]byte(rest), &m); err != nil || len(m) == 0 {
return nil
}
return m
}
// inventHeuristicDescription builds a short factual description from Category + Attrs
// (+ brand / model / dims) for thin inputs. Matches Slovenian vs English from the
// enhance system/user prompt language hint. Never returns sole retail-filler copy
// when a title or attributes exist.
func inventHeuristicDescription(system, user, name string) string {
name = strings.TrimSpace(name)
if name == "" || name == "<nil>" || isPromptLabelTitle(name) {
name = labeledPromptValue(user, "name:", "current name:")
}
if name == "" || isPromptLabelTitle(name) {
name = "Product"
}
cat := labeledPromptValue(user, "category:")
if isPromptLabelTitle(cat) {
cat = ""
}
lang := languageCodeFromEnhancePrompt(system, user)
attrs := CompactAttrs(parseAttrsFromPrompt(user), MaxAttrKeys)
if out := synthesizeDescriptionFromTitle(name, cat, lang, attrs); out != "" {
return out
}
return SanitizeOutput(fmt.Sprintf("%s is a catalog product with the known attributes.", name))
}
func languageCodeFromEnhancePrompt(system, user string) string {
blob := strings.ToLower(system + "\n" + user)
for _, code := range company.ContentLanguages {
label := strings.ToLower(company.LanguageLabel(code))
if label == "" {
continue
}
if strings.Contains(blob, "in "+label) || strings.Contains(blob, "language: "+label) {
return code
}
}
if code, err := company.ParseLanguage(labeledPromptValue(user, "language:", "content language:"), false); err == nil {
return code
}
return company.DefaultLanguage
}
+119
View File
@@ -2,12 +2,39 @@ package processing
import ( import (
"context" "context"
"encoding/json"
"net/http" "net/http"
"net/http/httptest"
"strings" "strings"
"testing" "testing"
"time" "time"
) )
func TestHeuristicCompleter_thinProductInventFromAttrs(t *testing.T) {
t.Parallel()
h := HeuristicCompleter{}
system := `Retail product copywriter. Schema: {"name":"string","description":"string"}`
user := "Category: Monitors\nName: UltraView 27\nDesc: UltraView 27\nAttrs: {\"brand\":\"Acme\",\"size\":\"27 inch\",\"panel\":\"IPS\"}"
comp, err := h.Complete(context.Background(), system, user)
if err != nil {
t.Fatal(err)
}
var out map[string]string
if err := json.Unmarshal([]byte(comp.Text), &out); err != nil {
t.Fatalf("json: %v text=%q", err, comp.Text)
}
if out["name"] != "UltraView 27" {
t.Fatalf("name=%q", out["name"])
}
desc := out["description"]
if desc == "" || strings.EqualFold(desc, "UltraView 27") || desc == "Product description" {
t.Fatalf("desc=%q want invented non-title copy", desc)
}
if !strings.Contains(strings.ToLower(desc), "acme") && !strings.Contains(strings.ToLower(desc), "ips") && !strings.Contains(strings.ToLower(desc), "27") {
t.Fatalf("desc=%q want attrs-derived facts", desc)
}
}
func TestNewOpenAIClient_capsRetries(t *testing.T) { func TestNewOpenAIClient_capsRetries(t *testing.T) {
c := NewOpenAIClient("k", "https://api.openai.com/v1", "m", 0, 99) c := NewOpenAIClient("k", "https://api.openai.com/v1", "m", 0, 99)
if c.MaxRetries != maxOpenAIRetries { if c.MaxRetries != maxOpenAIRetries {
@@ -107,3 +134,95 @@ func TestNewOpenAIClient_blocksPrivateDialInProduction(t *testing.T) {
t.Fatalf("expected host is not allowed, got: %v", err) t.Fatalf("expected host is not allowed, got: %v", err)
} }
} }
func TestChoiceMessageText_prefersContentThenReasoningJSON(t *testing.T) {
t.Parallel()
if got := choiceMessageText(json.RawMessage(`"{\"name\":\"A\",\"description\":\"B\"}"`), "", ""); got == "" {
t.Fatal("expected content string")
}
reasoning := `Thinking…\nDraft JSON:\n{"name":"NOSILEC W53070","description":"Stenski nosilec za TV."}\nVerify…`
got := choiceMessageText(json.RawMessage(`""`), reasoning, "")
if !strings.Contains(got, `"name"`) || !strings.Contains(got, "NOSILEC") {
t.Fatalf("got=%q want JSON from reasoning_content", got)
}
if got := choiceMessageText(json.RawMessage(`null`), "no json here", ""); got != "" {
t.Fatalf("expected empty without JSON, got %q", got)
}
}
func TestOpenAIClient_doComplete_emptyContentWithReasoningJSON(t *testing.T) {
t.Parallel()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]any{
"model": "code-fast",
"choices": []map[string]any{{
"finish_reason": "length",
"message": map[string]any{
"role": "assistant",
"content": "",
"reasoning_content": `steps… {"name":"TV Mount","description":"A wall mount for TVs."} more`,
},
}},
"usage": map[string]int{"prompt_tokens": 10, "completion_tokens": 50, "total_tokens": 60},
})
}))
defer srv.Close()
c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 1)
c.HTTPClient = srv.Client()
comp, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: 350, Temperature: 0.2})
if err != nil {
t.Fatal(err)
}
if !strings.Contains(comp.Text, "TV Mount") {
t.Fatalf("text=%q", comp.Text)
}
}
func TestOpenAIClient_doComplete_emptyContentLengthRetriesWithBudget(t *testing.T) {
t.Parallel()
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
var req map[string]any
_ = json.NewDecoder(r.Body).Decode(&req)
maxTok, _ := req["max_tokens"].(float64)
if calls == 1 {
if maxTok != 350 {
t.Errorf("first max_tokens=%v want 350", maxTok)
}
_ = json.NewEncoder(w).Encode(map[string]any{
"model": "code-fast",
"choices": []map[string]any{{
"finish_reason": "length",
"message": map[string]any{"role": "assistant", "content": "", "reasoning_content": "still thinking"},
}},
"usage": map[string]int{"prompt_tokens": 1, "completion_tokens": 350, "total_tokens": 351},
})
return
}
if maxTok != float64(maxTokensReasoningBudget) {
t.Errorf("retry max_tokens=%v want %d", maxTok, maxTokensReasoningBudget)
}
_ = json.NewEncoder(w).Encode(map[string]any{
"model": "code-fast",
"choices": []map[string]any{{
"finish_reason": "stop",
"message": map[string]any{"role": "assistant", "content": `{"name":"X","description":"Y product description text here."}`},
}},
"usage": map[string]int{"prompt_tokens": 1, "completion_tokens": 40, "total_tokens": 41},
})
}))
defer srv.Close()
c := NewOpenAIClient("test-key", srv.URL, "code-fast", 0, 2)
c.HTTPClient = srv.Client()
comp, err := c.CompleteWithOptions(context.Background(), "sys", "user", CompleteOptions{MaxTokens: 350, Temperature: 0.2})
if err != nil {
t.Fatal(err)
}
if calls != 2 {
t.Fatalf("calls=%d want 2", calls)
}
if !strings.Contains(comp.Text, "\"name\"") {
t.Fatalf("text=%q", comp.Text)
}
}
+339 -36
View File
@@ -657,10 +657,7 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
if !shouldFlushJobProgress(sinceFlush, progressEvery, batchDone) { if !shouldFlushJobProgress(sinceFlush, progressEvery, batchDone) {
return return
} }
mode := modeLabel mode := preferKnownProviderMode(lastResultAIProviderMode(lastResult), modeLabel)
if lastResult != nil && lastResult.AIProviderMode != "" {
mode = lastResult.AIProviderMode
}
if err := p.flushJobCountersAndProgress(ctx, jobID, processingType, lastResult, processed, tokensSinceFlush, mode); err != nil { if err := p.flushJobCountersAndProgress(ctx, jobID, processingType, lastResult, processed, tokensSinceFlush, mode); err != nil {
log.Printf("processing: flush progress job=%s err=%s", jobID, TruncateError(err)) log.Printf("processing: flush progress job=%s err=%s", jobID, TruncateError(err))
} }
@@ -672,6 +669,7 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
if err := stopOnCancel(p.jobCancelled(ctx, jobID)); err != nil { if err := stopOnCancel(p.jobCancelled(ctx, jobID)); err != nil {
flushProgress(true) flushProgress(true)
if errors.Is(err, errJobCancelled) { if errors.Is(err, errJobCancelled) {
_ = p.reclaimOrphanedProcessingItems(ctx, jobID)
return nil return nil
} }
return fmt.Errorf("processing: check cancel job=%s: %w", jobID, err) return fmt.Errorf("processing: check cancel job=%s: %w", jobID, err)
@@ -682,6 +680,27 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
return err return err
} }
if len(items) == 0 { if len(items) == 0 {
// loadPendingItems only reclaims processing rows older than StuckAgeInterval.
// A crashed peer can leave fresh processing rows; do not mark the job
// completed while open work remains — reset them and keep going.
var open int
if err := p.Pool.QueryRow(ctx, `
SELECT COUNT(*) FROM processing_job_products
WHERE job_id = $1 AND processed_product_id IS NULL
AND status IN ('pending', 'processing')`, jobID).Scan(&open); err != nil {
flushProgress(true)
return fmt.Errorf("processing: count open items job=%s: %w", jobID, err)
}
if open > 0 {
if _, err := p.Pool.Exec(ctx, `
UPDATE processing_job_products
SET status = 'pending', error = NULL, updated_at = now()
WHERE job_id = $1 AND status = 'processing' AND processed_product_id IS NULL`, jobID); err != nil {
flushProgress(true)
return fmt.Errorf("processing: reclaim fresh processing job=%s: %w", jobID, err)
}
continue
}
break break
} }
if err := p.hydrateJobItems(ctx, companyID, items); err != nil { if err := p.hydrateJobItems(ctx, companyID, items); err != nil {
@@ -695,6 +714,7 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
if err := stopOnCancel(p.jobCancelled(ctx, jobID)); err != nil { if err := stopOnCancel(p.jobCancelled(ctx, jobID)); err != nil {
flushProgress(true) flushProgress(true)
if errors.Is(err, errJobCancelled) { if errors.Is(err, errJobCancelled) {
_ = p.reclaimOrphanedProcessingItems(ctx, jobID)
return nil return nil
} }
return fmt.Errorf("processing: check cancel job=%s: %w", jobID, err) return fmt.Errorf("processing: check cancel job=%s: %w", jobID, err)
@@ -766,10 +786,7 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
if finalStatus == "failed" { if finalStatus == "failed" {
finalStep = "failed" finalStep = "failed"
} }
finalMode := modeLabel finalMode := preferKnownProviderMode(lastResultAIProviderMode(lastResult), modeLabel)
if lastResult != nil && lastResult.AIProviderMode != "" {
finalMode = lastResult.AIProviderMode
}
_, err = p.Pool.Exec(ctx, ` _, err = p.Pool.Exec(ctx, `
UPDATE processing_jobs UPDATE processing_jobs
SET status = $2, processed_products = $3, error = $4, completed_at = now(), updated_at = now(), SET status = $2, processed_products = $3, error = $4, completed_at = now(), updated_at = now(),
@@ -818,8 +835,17 @@ type jobScopedCache struct {
enhanceSystemTemplate string enhanceSystemTemplate string
enhanceUserTemplate string enhanceUserTemplate string
enhanceByLang map[string]PromptTemplates enhanceByLang map[string]PromptTemplates
// categoryPromptsByLang maps lower(trim(name)) → lang → sanitized prompt. // categoryPromptsByLang maps lower(trim(name|unique_id)) → lang → sanitized prompt.
categoryPromptsByLang map[string]company.LangPromptMap categoryPromptsByLang map[string]company.LangPromptMap
// categoryFormulasByKey maps lower(trim(name|unique_id)) → title/description templates.
categoryFormulasByKey map[string]CategoryFormulas
// categoryNamesByUID maps unique_id → display name for meta/enhance/synthesize.
categoryNamesByUID map[string]string
// categoryUniqueIDs is the company taxonomy unique_id set (empty = skip validation).
categoryUniqueIDs map[string]struct{}
// categoryAttrKeys maps category unique_id → canonicalized attribute_key set
// from category_attributes (for AI enhance allowlisting).
categoryAttrKeys map[string]map[string]struct{}
contentLanguages []string contentLanguages []string
// Entitlements snapshot — avoids EntitlementsForCompany N+1 per product. // Entitlements snapshot — avoids EntitlementsForCompany N+1 per product.
billingEnabled bool billingEnabled bool
@@ -902,37 +928,153 @@ func (p *Pipeline) loadJobScopedCache(ctx context.Context, companyID, jobID uuid
} }
} }
} }
cache.categoryPromptsByLang = p.loadCategoryEnhancePrompts(ctx, companyID, jobID) cache.categoryPromptsByLang, cache.categoryFormulasByKey, cache.categoryNamesByUID = p.loadCategoryEnhanceOverlays(ctx, companyID, jobID)
cache.categoryUniqueIDs = p.loadCategoryUniqueIDs(ctx, companyID, jobID)
cache.categoryAttrKeys = p.loadCategoryAttributeKeySets(ctx, companyID, jobID)
return cache return cache
} }
func (p *Pipeline) loadCategoryEnhancePrompts(ctx context.Context, companyID, jobID uuid.UUID) map[string]company.LangPromptMap { func (p *Pipeline) loadCategoryUniqueIDs(ctx context.Context, companyID, jobID uuid.UUID) map[string]struct{} {
rows, err := p.Pool.Query(ctx, ` rows, err := p.Pool.Query(ctx, `
SELECT name, COALESCE(prompt, '{}'::jsonb) SELECT unique_id
FROM categories FROM categories
WHERE company_id = $1 AND prompt <> '{}'::jsonb`, companyID) WHERE company_id = $1
AND COALESCE(NULLIF(BTRIM(unique_id), ''), '') <> ''`, companyID)
if err != nil { if err != nil {
log.Printf("processing: load category prompts job=%s company=%s err=%s", jobID, companyID, TruncateError(err)) log.Printf("processing: load category unique_ids job=%s company=%s err=%s", jobID, companyID, TruncateError(err))
return nil return nil
} }
defer rows.Close() defer rows.Close()
out := make(map[string]company.LangPromptMap) out := make(map[string]struct{})
for rows.Next() { for rows.Next() {
var name string var id string
var raw []byte if err := rows.Scan(&id); err != nil {
if err := rows.Scan(&name, &raw); err != nil { log.Printf("processing: scan category unique_id job=%s err=%s", jobID, TruncateError(err))
log.Printf("processing: scan category prompt job=%s err=%s", jobID, TruncateError(err))
continue continue
} }
key := strings.ToLower(strings.TrimSpace(name)) id = strings.TrimSpace(id)
if key == "" { if id == "" {
continue continue
} }
m, err := company.DecodeLangPromptMap(raw) out[id] = struct{}{}
if err != nil || !company.HasAnyPrompt(m) { }
if err := rows.Err(); err != nil {
log.Printf("processing: category unique_ids rows job=%s err=%s", jobID, TruncateError(err))
}
return out
}
// loadCategoryAttributeKeySets returns category unique_id → attribute_key set from
// category_attributes. Empty map on failure (callers still fall back to core keys).
func (p *Pipeline) loadCategoryAttributeKeySets(ctx context.Context, companyID, jobID uuid.UUID) map[string]map[string]struct{} {
out := map[string]map[string]struct{}{}
if p == nil || p.Pool == nil {
return out
}
rows, err := p.Pool.Query(ctx, `
SELECT ca.category_unique_id, a.attribute_key
FROM category_attributes ca
INNER JOIN attributes a ON a.id = ca.attribute_id AND a.company_id = ca.company_id
WHERE ca.company_id = $1
AND COALESCE(NULLIF(BTRIM(a.attribute_key), ''), '') <> ''`, companyID)
if err != nil {
log.Printf("processing: load category attribute keys job=%s company=%s err=%s", jobID, companyID, TruncateError(err))
return out
}
defer rows.Close()
for rows.Next() {
var catUID, key string
if err := rows.Scan(&catUID, &key); err != nil {
log.Printf("processing: scan category attribute key job=%s err=%s", jobID, TruncateError(err))
continue continue
} }
// Re-sanitize with category rune cap. catUID = strings.TrimSpace(catUID)
if catUID == "" {
continue
}
canon := canonicalizeAttrKey(key)
if canon == "" {
continue
}
set := out[catUID]
if set == nil {
set = map[string]struct{}{}
out[catUID] = set
}
set[canon] = struct{}{}
set[strings.ToLower(strings.TrimSpace(key))] = struct{}{}
}
if err := rows.Err(); err != nil {
log.Printf("processing: category attribute keys rows job=%s err=%s", jobID, TruncateError(err))
}
return out
}
// allowedAttrKeysForCategory returns category_attributes keys when known; empty
// non-nil map means core characteristics only. nil cache → nil (sanitize-only).
func allowedAttrKeysForCategory(cache *jobScopedCache, categoryUID string) map[string]struct{} {
if cache == nil {
return nil
}
return allowedAttrKeysFromSets(cache.categoryAttrKeys, categoryUID)
}
func categoryAttrKeysFromCache(cache *jobScopedCache) map[string]map[string]struct{} {
if cache == nil {
return nil
}
return cache.categoryAttrKeys
}
// allowedAttrKeysFromSets picks category keys when present; empty non-nil → core only.
// When sets is nil, returns nil (sanitize-only / unit tests).
func allowedAttrKeysFromSets(sets map[string]map[string]struct{}, categoryUID string) map[string]struct{} {
if sets == nil {
return nil
}
categoryUID = strings.TrimSpace(categoryUID)
if categoryUID != "" {
if keys, ok := sets[categoryUID]; ok && len(keys) > 0 {
return keys
}
}
return map[string]struct{}{}
}
func (p *Pipeline) loadCategoryEnhanceOverlays(ctx context.Context, companyID, jobID uuid.UUID) (map[string]company.LangPromptMap, map[string]CategoryFormulas, map[string]string) {
rows, err := p.Pool.Query(ctx, `
SELECT COALESCE(unique_id, ''), name,
COALESCE(prompt, '{}'::jsonb),
title_template,
description_template
FROM categories
WHERE company_id = $1`, companyID)
if err != nil {
log.Printf("processing: load category overlays job=%s company=%s err=%s", jobID, companyID, TruncateError(err))
return nil, nil, nil
}
defer rows.Close()
prompts := make(map[string]company.LangPromptMap)
formulas := make(map[string]CategoryFormulas)
namesByUID := make(map[string]string)
for rows.Next() {
var uniqueID, name string
var promptRaw []byte
var titleRaw, descRaw []byte
if err := rows.Scan(&uniqueID, &name, &promptRaw, &titleRaw, &descRaw); err != nil {
log.Printf("processing: scan category overlay job=%s err=%s", jobID, TruncateError(err))
continue
}
uid := strings.TrimSpace(uniqueID)
display := strings.TrimSpace(name)
if uid != "" && display != "" {
namesByUID[uid] = display
}
keys := categoryOverlayKeys(uniqueID, name)
if len(keys) == 0 {
continue
}
if m, err := company.DecodeLangPromptMap(promptRaw); err == nil && company.HasAnyPrompt(m) {
cleaned := company.LangPromptMap{} cleaned := company.LangPromptMap{}
for lang, prompt := range m { for lang, prompt := range m {
p := strings.TrimSpace(security.SanitizePrompt(prompt, catalog.MaxCategoryPromptRunes)) p := strings.TrimSpace(security.SanitizePrompt(prompt, catalog.MaxCategoryPromptRunes))
@@ -942,16 +1084,68 @@ func (p *Pipeline) loadCategoryEnhancePrompts(ctx context.Context, companyID, jo
cleaned[lang] = p cleaned[lang] = p
} }
if company.HasAnyPrompt(cleaned) { if company.HasAnyPrompt(cleaned) {
out[key] = cleaned for _, key := range keys {
prompts[key] = cleaned
}
}
}
f := CategoryFormulas{}
if t := decodeOptionalJSONObject(titleRaw); t != nil {
f.TitleTemplate = t
}
if d := decodeOptionalJSONObject(descRaw); d != nil {
f.DescriptionTemplate = d
}
if f.TitleTemplate != nil || f.DescriptionTemplate != nil {
for _, key := range keys {
formulas[key] = f
}
} }
} }
if err := rows.Err(); err != nil { if err := rows.Err(); err != nil {
log.Printf("processing: category prompts rows job=%s err=%s", jobID, TruncateError(err)) log.Printf("processing: category overlays rows job=%s err=%s", jobID, TruncateError(err))
} }
return prompts, formulas, namesByUID
}
func categoryOverlayKeys(uniqueID, name string) []string {
seen := map[string]struct{}{}
var out []string
add := func(raw string) {
key := strings.ToLower(strings.TrimSpace(raw))
if key == "" {
return
}
if _, ok := seen[key]; ok {
return
}
seen[key] = struct{}{}
out = append(out, key)
}
add(uniqueID)
add(name)
return out return out
} }
func categoryEnhancePromptFor(prompts map[string]company.LangPromptMap, category, language string) string { func decodeOptionalJSONObject(raw []byte) map[string]any {
raw = []byte(strings.TrimSpace(string(raw)))
if len(raw) == 0 || string(raw) == "null" || string(raw) == "{}" {
return nil
}
var obj map[string]any
if err := json.Unmarshal(raw, &obj); err != nil || len(obj) == 0 {
return nil
}
return obj
}
// loadCategoryEnhancePrompts is retained for tests / callers that only need prompts.
func (p *Pipeline) loadCategoryEnhancePrompts(ctx context.Context, companyID, jobID uuid.UUID) map[string]company.LangPromptMap {
prompts, _, _ := p.loadCategoryEnhanceOverlays(ctx, companyID, jobID)
return prompts
}
func categoryEnhancePromptFor(prompts map[string]company.LangPromptMap, category, language, primary string) string {
if len(prompts) == 0 { if len(prompts) == 0 {
return "" return ""
} }
@@ -959,7 +1153,7 @@ func categoryEnhancePromptFor(prompts map[string]company.LangPromptMap, category
if key == "" { if key == "" {
return "" return ""
} }
return company.PromptForLanguage(prompts[key], language) return company.PromptForLanguage(prompts[key], language, primary)
} }
func progressFromResult(processingType string, result *StepResult) []StepProgress { func progressFromResult(processingType string, result *StepResult) []StepProgress {
@@ -1180,7 +1374,35 @@ func (p *Pipeline) jobCancelled(ctx context.Context, jobID uuid.UUID) (bool, err
if err != nil { if err != nil {
return false, err return false, err
} }
return status == "cancelled", nil return isTerminalJobStatus(status), nil
}
// isTerminalJobStatus reports statuses that should stop an in-flight ProcessJob
// so worker JobSlots are released (failed/completed parkers included).
func isTerminalJobStatus(status string) bool {
switch strings.ToLower(strings.TrimSpace(status)) {
case "cancelled", "failed", "completed":
return true
default:
return false
}
}
// reclaimOrphanedProcessingItems returns in-flight items to pending when the job
// was marked failed/completed externally so a later ClaimNext can finish them.
func (p *Pipeline) reclaimOrphanedProcessingItems(ctx context.Context, jobID uuid.UUID) error {
var status string
if err := p.Pool.QueryRow(ctx, `SELECT status FROM processing_jobs WHERE id = $1`, jobID).Scan(&status); err != nil {
return err
}
if strings.EqualFold(strings.TrimSpace(status), "cancelled") {
return nil
}
_, err := p.Pool.Exec(ctx, `
UPDATE processing_job_products
SET status = 'pending', error = NULL, updated_at = now()
WHERE job_id = $1 AND status = 'processing' AND processed_product_id IS NULL`, jobID)
return err
} }
func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, it *jobItem, processingType string, engine *Engine, modeLabel string, usingBYOK bool, cache *jobScopedCache) (bool, int, StepResult, error) { func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, it *jobItem, processingType string, engine *Engine, modeLabel string, usingBYOK bool, cache *jobScopedCache) (bool, int, StepResult, error) {
@@ -1215,6 +1437,8 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
var stdDefs []StandardFieldDef var stdDefs []StandardFieldDef
var brandPrompt, language, enhanceSystemTemplate, enhanceUserTemplate string var brandPrompt, language, enhanceSystemTemplate, enhanceUserTemplate string
var categoryPromptsByLang map[string]company.LangPromptMap var categoryPromptsByLang map[string]company.LangPromptMap
var categoryFormulasByKey map[string]CategoryFormulas
var categoryNamesByUID map[string]string
var contentLanguages []string var contentLanguages []string
var enhanceByLang map[string]PromptTemplates var enhanceByLang map[string]PromptTemplates
if cache != nil { if cache != nil {
@@ -1224,6 +1448,8 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
enhanceSystemTemplate = cache.enhanceSystemTemplate enhanceSystemTemplate = cache.enhanceSystemTemplate
enhanceUserTemplate = cache.enhanceUserTemplate enhanceUserTemplate = cache.enhanceUserTemplate
categoryPromptsByLang = cache.categoryPromptsByLang categoryPromptsByLang = cache.categoryPromptsByLang
categoryFormulasByKey = cache.categoryFormulasByKey
categoryNamesByUID = cache.categoryNamesByUID
contentLanguages = cache.contentLanguages contentLanguages = cache.contentLanguages
enhanceByLang = cache.enhanceByLang enhanceByLang = cache.enhanceByLang
} }
@@ -1231,6 +1457,11 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
if len(stdDefs) > 0 { if len(stdDefs) > 0 {
enriched = FillMissingStandardFields(enriched, raw, stdDefs) enriched = FillMissingStandardFields(enriched, raw, stdDefs)
} }
// Deterministic category unique_id from feed/mapped (no LLM required).
coerceMappedCategoryUniqueID(enriched)
if cat := categoryUniqueIDFromMaps(enriched, raw); cat != "" {
enriched["category"] = cat
}
if gtin == "" { if gtin == "" {
gtin = stringFromMap(enriched, "gtin", "ean", "EAN", "upc") gtin = stringFromMap(enriched, "gtin", "ean", "EAN", "upc")
} }
@@ -1249,6 +1480,10 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
EnhanceSystemTemplate: enhanceSystemTemplate, EnhanceSystemTemplate: enhanceSystemTemplate,
EnhanceUserTemplate: enhanceUserTemplate, EnhanceUserTemplate: enhanceUserTemplate,
CategoryPromptsByLang: categoryPromptsByLang, CategoryPromptsByLang: categoryPromptsByLang,
CategoryFormulasByKey: categoryFormulasByKey,
CategoryNamesByUID: categoryNamesByUID,
AllowedAttrKeys: allowedAttrKeysForCategory(cache, stringFromAny(enriched["category"])),
CategoryAttrKeys: categoryAttrKeysFromCache(cache),
} }
if it.hydrated { if it.hydrated {
if it.hasPrior { if it.hasPrior {
@@ -1292,16 +1527,31 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
if err != nil { if err != nil {
return false, 0, StepResult{}, err return false, 0, StepResult{}, err
} }
// Persist mapped unique_id only when it exists in the company taxonomy.
// Empty taxonomy set skips filtering (unit tests / companies without categories).
if cache != nil {
filterCategoryIfInvalid(&result, cache.categoryUniqueIDs)
syncCategoryName(&result, cache.categoryNamesByUID)
}
// Re-apply after category validation so allowlist matches the persisted category.
allowed := allowedAttrKeysForCategory(cache, result.Category)
result.Attributes = AttrsForPersist(result.Attributes, allowed)
result.ProcessedAttributes = AttrsForPersist(result.ProcessedAttributes, allowed)
if len(result.ProcessedAttributes) == 0 {
result.ProcessedAttributes = result.Attributes
}
// Free template SEO meta (no FillMetaAI / no extra credits).
result.MetaTitle, result.MetaDescription = fillMetaFromResult(result)
attrsJSON, procAttrsJSON, gptJSON, sourcesJSON, err := marshalProcessOnePayload(result) attrsJSON, procAttrsJSON, gptJSON, sourcesJSON, err := marshalProcessOnePayload(result)
if err != nil { if err != nil {
return false, 0, result, err return false, 0, result, err
} }
providerMode := result.AIProviderMode // Treat "unknown" like empty so job modeLabel / EngineProviderMode wins on
if providerMode == "" { // hash-skip (TotalTokens==0) paths that previously stamped unknown in RunSteps.
if modeLabel != "" { providerMode := preferKnownProviderMode(result.AIProviderMode, modeLabel)
providerMode = modeLabel if isUnknownProviderMode(providerMode) {
} else if result.TotalTokens > 0 { if result.TotalTokens > 0 {
providerMode = AIProviderInternal providerMode = AIProviderInternal
} else { } else {
providerMode = AIProviderUnknown providerMode = AIProviderUnknown
@@ -1375,8 +1625,9 @@ const upsertProcessedProductSQL = `
INSERT INTO processed_products ( INSERT INTO processed_products (
company_id, raw_product_id, product_id, name, category, description, company_id, raw_product_id, product_id, name, category, description,
processed_name, processed_description, status, attributes, processed_attributes, processed_name, processed_description, status, attributes, processed_attributes,
gpt_response, total_tokens, field_sources, ai_provider_mode, localized_content gpt_response, total_tokens, field_sources, ai_provider_mode, localized_content,
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'needs_review',$9,$10,$11,$12,$13,$14,$15::jsonb) meta_title, meta_description
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'needs_review',$9,$10,$11,$12,$13,$14,$15::jsonb,$16,$17)
ON CONFLICT (company_id, raw_product_id) DO UPDATE SET ON CONFLICT (company_id, raw_product_id) DO UPDATE SET
product_id = EXCLUDED.product_id, product_id = EXCLUDED.product_id,
name = EXCLUDED.name, name = EXCLUDED.name,
@@ -1392,6 +1643,57 @@ const upsertProcessedProductSQL = `
field_sources = EXCLUDED.field_sources, field_sources = EXCLUDED.field_sources,
ai_provider_mode = EXCLUDED.ai_provider_mode, ai_provider_mode = EXCLUDED.ai_provider_mode,
localized_content = EXCLUDED.localized_content, localized_content = EXCLUDED.localized_content,
meta_title = CASE
WHEN NULLIF(BTRIM(processed_products.meta_title), '') IS NULL
THEN NULLIF(BTRIM(EXCLUDED.meta_title), '')
WHEN BTRIM(processed_products.meta_title) ~ '\|\s+[0-9]+$'
THEN NULLIF(BTRIM(EXCLUDED.meta_title), '')
WHEN LOWER(processed_products.meta_title) LIKE '%short retail title%'
OR LOWER(processed_products.meta_title) LIKE '%title formula%'
OR LOWER(processed_products.meta_title) LIKE '%follow any%'
OR LOWER(processed_products.meta_title) LIKE '%constraints that follow%'
OR LOWER(processed_products.meta_title) LIKE '%prefer 1-3%'
OR LOWER(processed_products.meta_title) LIKE '%reply with only json%'
OR LOWER(processed_products.meta_title) LIKE '%your reply is parsed as json%'
THEN NULLIF(BTRIM(EXCLUDED.meta_title), '')
ELSE processed_products.meta_title
END,
meta_description = CASE
WHEN NULLIF(BTRIM(processed_products.meta_description), '') IS NULL
THEN NULLIF(BTRIM(EXCLUDED.meta_description), '')
WHEN LOWER(processed_products.meta_description) LIKE '%prefer 1-3%'
OR LOWER(processed_products.meta_description) LIKE '%short retail title%'
OR LOWER(processed_products.meta_description) LIKE '%title formula%'
OR LOWER(processed_products.meta_description) LIKE '%do not emit%'
OR LOWER(processed_products.meta_description) LIKE '%reply with only json%'
THEN NULLIF(BTRIM(EXCLUDED.meta_description), '')
-- When refreshing a poisoned meta_title (| digits or prompt leakage), also refresh empty/weak stub desc.
WHEN (
BTRIM(processed_products.meta_title) ~ '\|\s+[0-9]+$'
OR LOWER(processed_products.meta_title) LIKE '%short retail title%'
OR LOWER(processed_products.meta_title) LIKE '%title formula%'
OR LOWER(processed_products.meta_title) LIKE '%follow any%'
OR LOWER(processed_products.meta_title) LIKE '%constraints that follow%'
OR LOWER(processed_products.meta_title) LIKE '%prefer 1-3%'
OR LOWER(processed_products.meta_title) LIKE '%reply with only json%'
OR LOWER(processed_products.meta_title) LIKE '%your reply is parsed as json%'
)
AND (
NULLIF(BTRIM(processed_products.meta_description), '') IS NULL
OR char_length(BTRIM(processed_products.meta_description)) < 40
OR LOWER(processed_products.meta_description) LIKE '%ready for retail listing%'
OR LOWER(processed_products.meta_description) LIKE '%quality product ready%'
OR LOWER(processed_products.meta_description) LIKE '%product description%'
OR LOWER(processed_products.meta_description) LIKE '%based on available specifications%'
OR LOWER(processed_products.meta_description) LIKE '%with available specifications%'
OR LOWER(processed_products.meta_description) LIKE '%available catalog details%'
OR LOWER(processed_products.meta_description) LIKE '%pripravljeno za prodajo%'
OR LOWER(processed_products.meta_description) LIKE '%na podlagi razpoložljivih specifikacij%'
OR LOWER(processed_products.meta_description) LIKE '%prefer 1-3%'
)
THEN NULLIF(BTRIM(EXCLUDED.meta_description), '')
ELSE processed_products.meta_description
END,
updated_at = now() updated_at = now()
RETURNING id` RETURNING id`
@@ -1426,6 +1728,7 @@ func (p *Pipeline) upsertProcessedProduct(
err = queryRow(ctx, upsertProcessedProductSQL, err = queryRow(ctx, upsertProcessedProductSQL,
companyID, rawID, gtin, result.Name, result.Category, result.Description, companyID, rawID, gtin, result.Name, result.Category, result.Description,
result.ProcessedName, result.ProcessedDescription, attrsJSON, procAttrsJSON, gptJSON, result.TotalTokens, sourcesJSON, providerMode, string(locJSON), result.ProcessedName, result.ProcessedDescription, attrsJSON, procAttrsJSON, gptJSON, result.TotalTokens, sourcesJSON, providerMode, string(locJSON),
result.MetaTitle, result.MetaDescription,
).Scan(&processedID) ).Scan(&processedID)
return processedID, err return processedID, err
} }
@@ -119,6 +119,12 @@ func TestRunSteps_enhanceMockProviderFailure(t *testing.T) {
if !foundFailed { if !foundFailed {
t.Fatalf("expected ai_enhance failed in progress=%v", prog) t.Fatalf("expected ai_enhance failed in progress=%v", prog)
} }
if strings.TrimSpace(out.ProcessedDescription) == "" {
t.Fatal("provider failure must leave nonempty ProcessedDescription when title exists")
}
if descriptionEchoesTitle(out.ProcessedDescription, out.ProcessedName) {
t.Fatalf("failure synth must not echo title: name=%q desc=%q", out.ProcessedName, out.ProcessedDescription)
}
} }
func TestRunSteps_enhanceMockTimeout(t *testing.T) { func TestRunSteps_enhanceMockTimeout(t *testing.T) {
@@ -150,6 +156,53 @@ func TestRunSteps_enhanceMockTimeout(t *testing.T) {
if !ok { if !ok {
t.Fatalf("expected ai_enhance failed, progress=%v", prog) t.Fatalf("expected ai_enhance failed, progress=%v", prog)
} }
if strings.TrimSpace(out.ProcessedDescription) == "" {
t.Fatal("timeout must leave nonempty ProcessedDescription when title exists")
}
if descriptionEchoesTitle(out.ProcessedDescription, out.ProcessedName) {
t.Fatalf("timeout synth must not echo title: name=%q desc=%q", out.ProcessedName, out.ProcessedDescription)
}
}
func TestRunSteps_enhanceTimeoutSynthesizesDescriptionAndInternalMode(t *testing.T) {
t.Parallel()
e := &Engine{
Completer: stubCompleter{fn: func(_, _ string) (Completion, error) {
return Completion{}, context.DeadlineExceeded
}},
ProviderMode: AIProviderInternal,
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co-timeout-synth", ProductInput{
Mapped: map[string]any{
"name": "VOX EBR700",
"description": "",
"brand": "VOX",
"category": "50",
},
Language: "sl",
CategoryNamesByUID: map[string]string{
"50": "Štedilniki",
},
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatalf("RunSteps should swallow timeout: %v", err)
}
if strings.TrimSpace(out.ProcessedName) == "" {
t.Fatal("expected nonempty ProcessedName")
}
if strings.TrimSpace(out.ProcessedDescription) == "" {
t.Fatal("timeout must synthesize nonempty ProcessedDescription when title exists")
}
if descriptionEchoesTitle(out.ProcessedDescription, out.ProcessedName) {
t.Fatalf("synth must not echo title: name=%q desc=%q", out.ProcessedName, out.ProcessedDescription)
}
if out.AIProviderMode != AIProviderInternal {
t.Fatalf("ai_provider_mode=%q want %q (attempted enhance, not unknown)", out.AIProviderMode, AIProviderInternal)
}
if _, ok := out.FieldSources[FieldEnhanceInputHash]; ok {
t.Fatalf("timeout must clear enhance hash, got %v", out.FieldSources[FieldEnhanceInputHash])
}
} }
func TestOpenAIClient_Complete_httptestHappyPath(t *testing.T) { func TestOpenAIClient_Complete_httptestHappyPath(t *testing.T) {
@@ -28,6 +28,23 @@ func TestStopOnCancel(t *testing.T) {
} }
} }
func TestIsTerminalJobStatus(t *testing.T) {
cases := map[string]bool{
"pending": false,
"running": false,
"cancelled": true,
"failed": true,
"completed": true,
" FAILED ": true,
"": false,
}
for in, want := range cases {
if got := isTerminalJobStatus(in); got != want {
t.Fatalf("isTerminalJobStatus(%q)=%v want %v", in, got, want)
}
}
}
func TestMarshalStepProgress_roundTrip(t *testing.T) { func TestMarshalStepProgress_roundTrip(t *testing.T) {
progress := InitialStepProgress("full") progress := InitialStepProgress("full")
if len(progress) == 0 { if len(progress) == 0 {
@@ -88,6 +88,16 @@ func TestSanitizeProductAttributes(t *testing.T) {
if _, ok := got["netwidth"]; ok { if _, ok := got["netwidth"]; ok {
t.Fatalf("alias netwidth should collapse to width: %v", got) t.Fatalf("alias netwidth should collapse to width: %v", got)
} }
dropped := SanitizeProductAttributes(map[string]any{
"brand": "XIAOMI",
"product_model": "6941948703193",
})
if _, ok := dropped["product_model"]; ok {
t.Fatalf("barcode-like product_model must be dropped: %v", dropped)
}
if dropped["brand"] != "XIAOMI" {
t.Fatalf("brand=%v", dropped["brand"])
}
} }
func TestFilterAttributesByAllowed(t *testing.T) { func TestFilterAttributesByAllowed(t *testing.T) {
@@ -121,6 +131,154 @@ func TestFilterAttributesByAllowed(t *testing.T) {
} }
} }
func TestAttrsForEnhance_cat28DropsZavora(t *testing.T) {
attrs := map[string]any{
"brand": "Ostalo",
"product_model": "W53070",
"zavora": "Mehanska in elektricna",
"vzmetenje": "Spredaj in zadaj",
"nosilnost": "40 kg",
}
// Cat 28 TV mounts: barva/nosilnost — not motorcycle specs.
allowed := map[string]struct{}{
"barva": {},
"nosilnost": {},
}
got := AttrsForEnhance(attrs, allowed)
if got["brand"] != "Ostalo" || got["product_model"] != "W53070" {
t.Fatalf("core keys missing: %v", got)
}
if got["nosilnost"] != "40 kg" {
t.Fatalf("category key missing: %v", got)
}
if _, ok := got["zavora"]; ok {
t.Fatalf("zavora must not reach enhance: %v", got)
}
if _, ok := got["vzmetenje"]; ok {
t.Fatalf("vzmetenje must not reach enhance: %v", got)
}
user := ProductEnhanceUser("Nosilci za TV", "NOSILEC W53070", "", got)
if strings.Contains(strings.ToLower(user), "zavora") {
t.Fatalf("enhance user must not mention zavora:\n%s", user)
}
if !strings.Contains(user, "Ostalo") || !strings.Contains(user, "W53070") {
t.Fatalf("enhance user should keep brand/model:\n%s", user)
}
}
func TestAttrsForEnhance_emptyAllowlistCoreOnly(t *testing.T) {
got := AttrsForEnhance(map[string]any{
"brand": "Ostalo",
"zavora": "Mehanska",
}, map[string]struct{}{})
if got["brand"] != "Ostalo" {
t.Fatalf("brand=%v", got["brand"])
}
if _, ok := got["zavora"]; ok {
t.Fatalf("empty allowlist should drop junk: %v", got)
}
}
func TestAttrsForEnhance_copiesEPRELEnergyClass(t *testing.T) {
t.Parallel()
attrs := map[string]any{
"brand": "Bosch",
"eprel_energy_class": "C",
}
ensureEnergyClassFromEPREL(attrs)
got := AttrsForEnhance(attrs, map[string]struct{}{})
if got["energy_class"] != "C" {
t.Fatalf("energy_class=%v want C (from eprel_energy_class); attrs=%v", got["energy_class"], got)
}
if _, ok := got["eprel_energy_class"]; ok {
t.Fatalf("eprel_energy_class must be stripped by AttrsForEnhance: %v", got)
}
// Do not overwrite an existing energy_class.
attrs2 := map[string]any{
"energy_class": "A",
"eprel_energy_class": "C",
}
ensureEnergyClassFromEPREL(attrs2)
if attrs2["energy_class"] != "A" {
t.Fatalf("existing energy_class overwritten: %v", attrs2["energy_class"])
}
}
func TestAttrsForPersist_tvMountDropsJunkKeepsEPREL(t *testing.T) {
attrs := map[string]any{
"name": "TV Mount",
"description": "html",
"brand": "Ostalo",
"product_model": "W53070",
"zavora": "Mehanska in elektricna",
"vzmetenje": "Spredaj in zadaj",
"nosilnost": "40 kg",
"eprel_id": "12345",
"eprel_label": "https://eprel.example/label",
}
allowed := map[string]struct{}{
"barva": {},
"nosilnost": {},
}
got := AttrsForPersist(attrs, allowed)
if _, ok := got["name"]; ok {
t.Fatalf("reserved name must be stripped: %v", got)
}
if got["brand"] != "Ostalo" || got["product_model"] != "W53070" {
t.Fatalf("core keys missing: %v", got)
}
if got["nosilnost"] != "40 kg" {
t.Fatalf("category key missing: %v", got)
}
if _, ok := got["zavora"]; ok {
t.Fatalf("zavora must not be stored: %v", got)
}
if _, ok := got["vzmetenje"]; ok {
t.Fatalf("vzmetenje must not be stored: %v", got)
}
if got["eprel_id"] != "12345" || got["eprel_label"] != "https://eprel.example/label" {
t.Fatalf("eprel keys must persist for poll projection: %v", got)
}
}
func TestRunSteps_persistsCategoryAllowlist(t *testing.T) {
e := &Engine{}
in := ProductInput{
GTIN: "8712285326882",
Name: "NOSILEC W53070",
Mapped: map[string]any{
"name": "NOSILEC W53070",
"category": "28",
"brand": "Ostalo",
"specifications": map[string]any{
"productmodel": "W53070",
"zavora": "Mehanska",
"vzmetenje": "Spredaj",
"nosilnost": "40 kg",
},
},
CategoryAttrKeys: map[string]map[string]struct{}{
"28": {"nosilnost": {}, "barva": {}},
},
}
out, err := e.RunSteps(context.Background(), "co", in, "attributes", nil, StepPolicy{})
if err != nil {
t.Fatal(err)
}
if out.ProcessedAttributes["brand"] != "Ostalo" {
t.Fatalf("brand=%v", out.ProcessedAttributes["brand"])
}
if out.ProcessedAttributes["nosilnost"] != "40 kg" {
t.Fatalf("nosilnost=%v", out.ProcessedAttributes)
}
if _, ok := out.ProcessedAttributes["zavora"]; ok {
t.Fatalf("zavora must not be stored: %v", out.ProcessedAttributes)
}
if _, ok := out.ProcessedAttributes["vzmetenje"]; ok {
t.Fatalf("vzmetenje must not be stored: %v", out.ProcessedAttributes)
}
}
func TestV1PlainDescription(t *testing.T) { func TestV1PlainDescription(t *testing.T) {
got := v1PlainDescription("Hello<br>World<br/>&amp; more</p><b>bold</b>") got := v1PlainDescription("Hello<br>World<br/>&amp; more</p><b>bold</b>")
if strings.Contains(got, "<") { if strings.Contains(got, "<") {
@@ -1,6 +1,7 @@
package processing package processing
import ( import (
"strings"
"testing" "testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/company" "github.com/descrybe/descrybe-v2/apps/api/internal/company"
@@ -9,14 +10,18 @@ import (
func TestResolvePromptFallbackChain(t *testing.T) { func TestResolvePromptFallbackChain(t *testing.T) {
t.Parallel() t.Parallel()
// Category override for language wins over company user template. // Category override for language wins over company user template.
// Overrides without {{attrs}} get standard context placeholders injected.
sys, user := resolveProductPromptTemplates(ProductInput{ sys, user := resolveProductPromptTemplates(ProductInput{
EnhanceSystemTemplate: "sys", EnhanceSystemTemplate: "sys",
EnhanceUserTemplate: "company-en", EnhanceUserTemplate: "company-en",
CategoryEnhancePrompt: "cat-sl", CategoryEnhancePrompt: "cat-sl",
Language: "sl", Language: "sl",
}) })
if sys != "sys" || user != "cat-sl" { if sys != "sys" {
t.Fatalf("sys=%q user=%q", sys, user) t.Fatalf("sys=%q", sys)
}
if !strings.HasPrefix(user, "cat-sl") || !strings.Contains(user, "{{attrs}}") {
t.Fatalf("sys=%q user=%q want cat-sl + attrs", sys, user)
} }
// Empty category → company template. // Empty category → company template.
@@ -28,12 +33,20 @@ func TestResolvePromptFallbackChain(t *testing.T) {
t.Fatalf("user=%q", user) t.Fatalf("user=%q", user)
} }
// categoryEnhancePromptFor: lang-specific only (no cross-lang fallback). // sl-only category prompt: exact for sl; en falls back to primary=sl.
m := map[string]company.LangPromptMap{"audio": {"sl": "slo-prompt"}} m := map[string]company.LangPromptMap{"audio": {"sl": "slo-prompt"}}
if got := categoryEnhancePromptFor(m, "audio", "sl"); got != "slo-prompt" { if got := categoryEnhancePromptFor(m, "audio", "sl", "sl"); got != "slo-prompt" {
t.Fatalf("got %q", got) t.Fatalf("sl exact: got %q", got)
} }
if got := categoryEnhancePromptFor(m, "audio", "en"); got != "" { if got := categoryEnhancePromptFor(m, "audio", "en", "sl"); got != "slo-prompt" {
t.Fatalf("cross-lang should be empty, got %q", got) t.Fatalf("en→primary sl: got %q", got)
}
if got := categoryEnhancePromptFor(m, "audio", "en", "en"); got != "" {
t.Fatalf("en with primary=en and no en/* prompt should be empty, got %q", got)
}
// wildcard shared overlay
m2 := map[string]company.LangPromptMap{"audio": {"*": "shared-overlay"}}
if got := categoryEnhancePromptFor(m2, "audio", "en", "sl"); got != "shared-overlay" {
t.Fatalf("wildcard: got %q", got)
} }
} }
+62 -4
View File
@@ -12,21 +12,76 @@ func resolveProductPromptTemplates(in ProductInput) (systemTpl, userTpl string)
userTpl = strings.TrimSpace(in.EnhanceUserTemplate) userTpl = strings.TrimSpace(in.EnhanceUserTemplate)
// Per-category prompt wins for the user message (company system keeps JSON schema / brand). // Per-category prompt wins for the user message (company system keeps JSON schema / brand).
if cat := strings.TrimSpace(in.CategoryEnhancePrompt); cat != "" { if cat := strings.TrimSpace(in.CategoryEnhancePrompt); cat != "" {
userTpl = cat userTpl = ensureCategoryEnhanceUserContext(cat)
} }
def, ok := aiprompts.DefaultFor(aiprompts.KeyProductEnhance) def, ok := aiprompts.DefaultFor(aiprompts.KeyProductEnhance)
if !ok { if ok {
return systemTpl, userTpl
}
if systemTpl == "" { if systemTpl == "" {
systemTpl = def.SystemTemplate systemTpl = def.SystemTemplate
} }
if userTpl == "" { if userTpl == "" {
userTpl = def.UserTemplate userTpl = def.UserTemplate
} }
}
// Category formulas are language-agnostic; inject once into the shared user skeleton.
userTpl = AppendFormulaConstraints(userTpl, in.TitleTemplate, in.DescriptionTemplate)
return systemTpl, userTpl return systemTpl, userTpl
} }
// thinEnhanceUserInstruction is prepended when Desc is empty or ≈ Name so the model
// builds copy from Attrs+Category instead of echoing the title.
const thinEnhanceUserInstruction = "Build the description from Attrs and Category only; never copy Name."
// templateHasVar reports whether tpl contains {{name}} (via aiprompts.ExtractVariables).
func templateHasVar(tpl, name string) bool {
for _, v := range aiprompts.ExtractVariables(tpl) {
if v == name {
return true
}
}
return false
}
// ensureCategoryEnhanceUserContext appends standard product context placeholders when a
// category override omits {{attrs}} (common in DB category prompts). Does not rewrite
// category copy that already includes attrs.
func ensureCategoryEnhanceUserContext(userTpl string) string {
userTpl = strings.TrimSpace(userTpl)
if userTpl == "" || templateHasVar(userTpl, "attrs") {
return userTpl
}
var b strings.Builder
b.WriteString(userTpl)
b.WriteString("\n\n")
first := true
appendLine := func(line string) {
if !first {
b.WriteByte('\n')
}
first = false
b.WriteString(line)
}
if !templateHasVar(userTpl, "category") {
appendLine("Category: {{category}}")
}
if !templateHasVar(userTpl, "name") {
appendLine("Name: {{name}}")
}
if !templateHasVar(userTpl, "description") {
appendLine("Desc: {{description}}")
}
appendLine("Attrs: {{attrs}}")
return b.String()
}
func isThinEnhanceDescription(desc, name string) bool {
desc = strings.TrimSpace(desc)
if desc == "" || desc == "<nil>" {
return true
}
return descriptionEchoesTitle(desc, name)
}
// RenderProductEnhancePrompts fills company/built-in templates with product variables. // RenderProductEnhancePrompts fills company/built-in templates with product variables.
func RenderProductEnhancePrompts(systemTpl, userTpl, category, name, description, gtin, brandPrompt, language string, attrs map[string]any) (system, user string) { func RenderProductEnhancePrompts(systemTpl, userTpl, category, name, description, gtin, brandPrompt, language string, attrs map[string]any) (system, user string) {
attrsJSON := "" attrsJSON := ""
@@ -49,5 +104,8 @@ func RenderProductEnhancePrompts(systemTpl, userTpl, category, name, description
// Safety net if a custom user template renders empty. // Safety net if a custom user template renders empty.
user = ProductEnhanceUser(category, name, description, attrs) user = ProductEnhanceUser(category, name, description, attrs)
} }
if isThinEnhanceDescription(description, name) {
user = thinEnhanceUserInstruction + "\n\n" + user
}
return system, user return system, user
} }
@@ -1,8 +1,10 @@
package processing package processing
import ( import (
"strings"
"testing" "testing"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/company" "github.com/descrybe/descrybe-v2/apps/api/internal/company"
) )
@@ -16,8 +18,24 @@ func TestResolveProductPromptTemplates_categoryOverridesUser(t *testing.T) {
if sys != "sys {{brand_voice}}" { if sys != "sys {{brand_voice}}" {
t.Fatalf("system=%q", sys) t.Fatalf("system=%q", sys)
} }
if user != "category user {{description}}" { if !strings.HasPrefix(user, "category user {{description}}") {
t.Fatalf("user=%q want category override", user) t.Fatalf("user=%q want category override prefix", user)
}
if !strings.Contains(user, "{{attrs}}") {
t.Fatalf("user=%q want injected {{attrs}}", user)
}
if !strings.Contains(user, "{{name}}") || !strings.Contains(user, "{{category}}") {
t.Fatalf("user=%q want name/category placeholders", user)
}
}
func TestResolveProductPromptTemplates_categoryWithAttrsUnchanged(t *testing.T) {
t.Parallel()
_, user := resolveProductPromptTemplates(ProductInput{
CategoryEnhancePrompt: "Write copy.\nAttrs: {{attrs}}\nName: {{name}}",
})
if user != "Write copy.\nAttrs: {{attrs}}\nName: {{name}}" {
t.Fatalf("user=%q want unchanged when attrs present", user)
} }
} }
@@ -31,22 +49,105 @@ func TestResolveProductPromptTemplates_fallsBackToCompany(t *testing.T) {
} }
} }
func TestRenderProductEnhancePrompts_categoryOverrideGetsAttrs(t *testing.T) {
t.Parallel()
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{
CategoryEnhancePrompt: "Write a monitor listing. Focus on panel tech.",
})
_, user := RenderProductEnhancePrompts(
sysTpl, userTpl,
"Monitors", "UltraView 27", "", "", "", "en",
map[string]any{"brand": "Acme", "size": "27 inch"},
)
if !strings.Contains(user, "Attrs:") {
t.Fatalf("user missing Attrs block:\n%s", user)
}
if !strings.Contains(user, "Acme") || !strings.Contains(user, "27") {
t.Fatalf("user missing attr values:\n%s", user)
}
if !strings.Contains(user, thinEnhanceUserInstruction) {
t.Fatalf("thin input should prepend instruction:\n%s", user)
}
if !strings.Contains(user, "UltraView 27") {
t.Fatalf("user missing name:\n%s", user)
}
}
func TestCategoryEnhancePromptFor(t *testing.T) { func TestCategoryEnhancePromptFor(t *testing.T) {
t.Parallel() t.Parallel()
m := map[string]company.LangPromptMap{ m := map[string]company.LangPromptMap{
"monitorji": {"sl": "prompt-a", "en": "prompt-a-en"}, "monitorji": {"sl": "prompt-a", "en": "prompt-a-en"},
"televizorji": {"sl": "prompt-b"}, "televizorji": {"sl": "prompt-b"},
} }
if got := categoryEnhancePromptFor(m, " Monitorji ", "sl"); got != "prompt-a" { if got := categoryEnhancePromptFor(m, " Monitorji ", "sl", "sl"); got != "prompt-a" {
t.Fatalf("got %q", got) t.Fatalf("got %q", got)
} }
if got := categoryEnhancePromptFor(m, " Monitorji ", "en"); got != "prompt-a-en" { if got := categoryEnhancePromptFor(m, " Monitorji ", "en", "sl"); got != "prompt-a-en" {
t.Fatalf("got %q", got) t.Fatalf("got %q", got)
} }
if got := categoryEnhancePromptFor(m, "missing", "sl"); got != "" { if got := categoryEnhancePromptFor(m, "missing", "sl", "sl"); got != "" {
t.Fatalf("expected empty, got %q", got) t.Fatalf("expected empty, got %q", got)
} }
if got := categoryEnhancePromptFor(m, "televizorji", "en"); got != "" { // sl-only category: en falls back to primary=sl
t.Fatalf("expected empty fallback, got %q", got) if got := categoryEnhancePromptFor(m, "televizorji", "en", "sl"); got != "prompt-b" {
t.Fatalf("en→primary sl: got %q", got)
}
if got := categoryEnhancePromptFor(m, "televizorji", "en", "en"); got != "" {
t.Fatalf("en with primary=en should stay empty, got %q", got)
}
}
func TestCategoryEnhancePromptFor_uniqueIDAndNameKeys(t *testing.T) {
t.Parallel()
overlay := company.LangPromptMap{"*": aiprompts.CategoryEnhanceUserTemplate}
byName := map[string]company.LangPromptMap{
"štedilniki": overlay,
}
byUID := map[string]company.LangPromptMap{
"50": overlay,
}
if got := categoryEnhancePromptFor(byName, "50", "sl", "sl"); got != "" {
t.Fatalf("name-only map must miss unique_id lookup, got %q", got)
}
if got := categoryEnhancePromptFor(byUID, "50", "sl", "sl"); !strings.Contains(got, "{{attrs}}") {
t.Fatalf("unique_id key should resolve overlay: %q", got)
}
// Dual-index map (as loadCategoryEnhanceOverlays builds): unique_id OR name works.
dual := map[string]company.LangPromptMap{}
for _, k := range categoryOverlayKeys("50", "Štedilniki") {
dual[k] = overlay
}
if got := categoryEnhancePromptFor(dual, "50", "en", "sl"); !strings.Contains(got, "{{category}}") {
t.Fatalf("dual unique_id lookup: %q", got)
}
if got := categoryEnhancePromptFor(dual, "Štedilniki", "en", "sl"); !strings.Contains(got, "{{attrs}}") {
t.Fatalf("dual name lookup: %q", got)
}
}
func TestCategoryOverlayKeys_indexesUIDAndName(t *testing.T) {
t.Parallel()
keys := categoryOverlayKeys("50", "Štedilniki")
want := map[string]bool{"50": true, "štedilniki": true}
if len(keys) != 2 {
t.Fatalf("keys=%v want 2", keys)
}
for _, k := range keys {
if !want[k] {
t.Fatalf("unexpected key %q in %v", k, keys)
}
}
}
func TestRenderProductEnhancePrompts_languageLabelsSLEN(t *testing.T) {
t.Parallel()
sysTpl, userTpl := resolveProductPromptTemplates(ProductInput{})
sysSL, _ := RenderProductEnhancePrompts(sysTpl, userTpl, "Cat", "Name", "Desc", "", "", "sl", nil)
if !strings.Contains(sysSL, "Slovenian") {
t.Fatalf("sl system missing Slovenian label:\n%s", sysSL)
}
sysEN, _ := RenderProductEnhancePrompts(sysTpl, userTpl, "Cat", "Name", "Desc", "", "", "en", nil)
if !strings.Contains(sysEN, "English") {
t.Fatalf("en system missing English label:\n%s", sysEN)
} }
} }
+100
View File
@@ -13,6 +13,8 @@ var (
specsHTMLTagRe = regexp.MustCompile(`(?is)<[^>]+>`) specsHTMLTagRe = regexp.MustCompile(`(?is)<[^>]+>`)
csvLikeRe = regexp.MustCompile(`(?m)^\s*([^:=\n\r]{1,120})\s*[:=]\s*(.+?)\s*$`) csvLikeRe = regexp.MustCompile(`(?m)^\s*([^:=\n\r]{1,120})\s*[:=]\s*(.+?)\s*$`)
bulletLineRe = regexp.MustCompile(`(?m)^\s*[-•*]\s*([^:=\n\r]{1,120})\s*[:=]\s*(.+?)\s*$`) bulletLineRe = regexp.MustCompile(`(?m)^\s*[-•*]\s*([^:=\n\r]{1,120})\s*[:=]\s*(.+?)\s*$`)
// GTIN/EAN/UPC masquerading as product_model (feed mapping mistakes).
barcodeLikeModelRe = regexp.MustCompile(`^\d{8,14}$`)
) )
// Caps for locale/spec parsing — unbounded FindAll on huge CDATA can OOM the worker. // Caps for locale/spec parsing — unbounded FindAll on huge CDATA can OOM the worker.
@@ -173,6 +175,73 @@ func SanitizeV1ProcessAttributesAllowed(attrs map[string]any, allowed map[string
return FilterAttributesByAllowed(sanitizeProductAttributes(attrs, true), allowed) return FilterAttributesByAllowed(sanitizeProductAttributes(attrs, true), allowed)
} }
// AttrsForEnhance prepares attributes for AI enhance prompts / hashes.
// Always drops reserved and invalid keys. When allowed is non-nil (including empty),
// keeps only coreCharacteristicAttrKeys plus the allowlist (category_attributes).
// When allowed is nil, returns sanitized attrs unchanged (unit-test fallback).
func AttrsForEnhance(attrs map[string]any, allowed map[string]struct{}) map[string]any {
cleaned := sanitizeProductAttributes(attrs, true)
if allowed == nil {
return cleaned
}
cleaned = MapAttrsOntoAllowedKeys(cleaned, allowed)
return FilterAttributesByAllowed(cleaned, allowed)
}
// ensureEnergyClassFromEPREL copies eprel_energy_class → energy_class when the
// core energy_class key is empty so AttrsForEnhance (which dropEPREL strips
// eprel_*) still surfaces the label class to the model.
func ensureEnergyClassFromEPREL(attrs map[string]any) {
if attrs == nil {
return
}
if v, ok := attrs["energy_class"]; ok {
if s := strings.TrimSpace(fmt.Sprint(v)); s != "" && s != "<nil>" {
return
}
}
for _, k := range []string{"eprel_energy_class", "eprelEnergyClass"} {
if v, ok := attrs[k]; ok {
s := strings.TrimSpace(fmt.Sprint(v))
if s != "" && s != "<nil>" {
attrs["energy_class"] = s
return
}
}
}
}
// AttrsForPersist prepares attributes for DB storage (attributes / processed_attributes).
// Same category allowlist semantics as AttrsForEnhance, but uses SanitizeProductAttributes
// (keeps eprel_* for poll extractEPRELFromAttrs) then FilterAttributesByAllowed.
// Feed/spec keys are remapped onto category_attributes (formula) keys when possible so
// values under near-miss labels (velikost-zaslona → diagonala_zaslona) are kept.
// When allowed is nil, returns sanitized attrs unchanged (unit-test fallback).
func AttrsForPersist(attrs map[string]any, allowed map[string]struct{}) map[string]any {
cleaned := SanitizeProductAttributes(attrs)
if allowed == nil {
return cleaned
}
cleaned = MapAttrsOntoAllowedKeys(cleaned, allowed)
filtered := FilterAttributesByAllowed(cleaned, allowed)
// Preserve eprel_* that SanitizeProductAttributes kept (not on category allowlists).
for k, v := range cleaned {
if strings.HasPrefix(strings.ToLower(strings.TrimSpace(k)), "eprel") {
filtered[k] = v
}
}
return filtered
}
// enhanceAllowedAttrKeys prefers category_attributes for the resolved category;
// falls back to AllowedAttrKeys (nil = sanitize-only for unit tests).
func enhanceAllowedAttrKeys(in ProductInput, categoryUID string) map[string]struct{} {
if in.CategoryAttrKeys != nil {
return allowedAttrKeysFromSets(in.CategoryAttrKeys, categoryUID)
}
return in.AllowedAttrKeys
}
// FilterAttributesByAllowed keeps coreCharacteristicAttrKeys plus keys present in // FilterAttributesByAllowed keeps coreCharacteristicAttrKeys plus keys present in
// allowed (after canonicalizeAttrKey). When allowed is nil, returns attrs unchanged. // allowed (after canonicalizeAttrKey). When allowed is nil, returns attrs unchanged.
func FilterAttributesByAllowed(attrs map[string]any, allowed map[string]struct{}) map[string]any { func FilterAttributesByAllowed(attrs map[string]any, allowed map[string]struct{}) map[string]any {
@@ -216,6 +285,31 @@ func sanitizeProductAttributes(attrs map[string]any, dropEPREL bool) map[string]
if dropEPREL && strings.HasPrefix(strings.ToLower(key), "eprel") { if dropEPREL && strings.HasPrefix(strings.ToLower(key), "eprel") {
continue continue
} }
// Keep nested eprel object for storage / extractEPRELFromAttrs (stringify would drop maps).
if !dropEPREL && strings.EqualFold(key, "eprel") {
if m, ok := v.(map[string]any); ok && len(m) > 0 {
nested := make(map[string]any, len(m))
for nk, nv := range m {
nk = strings.TrimSpace(nk)
if nk == "" || nv == nil {
continue
}
if s, ok := nv.(string); ok {
s = strings.TrimSpace(SanitizeOutput(s))
if s == "" {
continue
}
nested[nk] = s
continue
}
nested[nk] = nv
}
if len(nested) > 0 {
out["eprel"] = nested
}
}
continue
}
s := stringifySpecValue(v) s := stringifySpecValue(v)
if s == "" || s == "<nil>" { if s == "" || s == "<nil>" {
continue continue
@@ -231,6 +325,10 @@ func sanitizeProductAttributes(attrs map[string]any, dropEPREL bool) map[string]
if isDimensionKey(canon) && isZeroishString(s) { if isDimensionKey(canon) && isZeroishString(s) {
continue continue
} }
// Drop barcode-as-model (e.g. mapped product_model = GTIN).
if canon == "product_model" && barcodeLikeModelRe.MatchString(strings.TrimSpace(s)) {
continue
}
if _, exists := out[canon]; exists && (canon != key) { if _, exists := out[canon]; exists && (canon != key) {
continue continue
} }
@@ -257,6 +355,8 @@ func canonicalizeAttrKey(k string) string {
return "product_model" return "product_model"
case "energijskirazred", "energyclass": case "energijskirazred", "energyclass":
return "energy_class" return "energy_class"
case "eprelid", "eprel":
return "eprel_id"
default: default:
return compact return compact
} }
+482 -93
View File
@@ -4,6 +4,7 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"sort"
"strings" "strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/company" "github.com/descrybe/descrybe-v2/apps/api/internal/company"
@@ -43,14 +44,15 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
in.Name, in.Name,
in.PriorProcessedName, in.PriorProcessedName,
) )
out.Description = preferredProductDescription( out.Description = preferredProductDescription(out.Name,
stringFromAny(normalized["description"]), stringFromAny(normalized["description"]),
in.Description, in.Description,
in.PriorProcessedDescription, in.PriorProcessedDescription,
) )
out.Category = stringFromAny(normalized["category"]) // mapped_data.category / category_unique_id (unique_id codes) win.
// mapped_data.category wins; otherwise keep existing processed category applyCategoryFromMapped(&out, normalized, in.Mapped, in.Raw)
// so enhance_only / reprocess cannot blank A1 legacy categories. // otherwise keep existing processed category so enhance_only / reprocess
// cannot blank A1 legacy categories.
preserveCategoryIfEmpty(&out, in.PriorCategory) preserveCategoryIfEmpty(&out, in.PriorCategory)
out.FieldSources["normalize"] = "mapped+raw" out.FieldSources["normalize"] = "mapped+raw"
appendStepLog(out.GPTResponse, StepNormalize, map[string]any{ appendStepLog(out.GPTResponse, StepNormalize, map[string]any{
@@ -74,10 +76,13 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
} }
} }
} }
attrs = parsed attrs = SanitizeProductAttributes(parsed)
out.Attributes = SanitizeProductAttributes(attrs) // Map feed/spec labels onto category_attributes (formula) keys early.
out.ProcessedAttributes = out.Attributes if allowed := enhanceAllowedAttrKeys(in, out.Category); allowed != nil {
attrs = out.Attributes attrs = MapAttrsOntoAllowedKeys(attrs, allowed)
}
out.Attributes = attrs
out.ProcessedAttributes = attrs
out.FieldSources["attributes"] = "specifications" out.FieldSources["attributes"] = "specifications"
appendStepLog(out.GPTResponse, StepParseSpecs, map[string]any{ appendStepLog(out.GPTResponse, StepParseSpecs, map[string]any{
"count": len(attrs), "count": len(attrs),
@@ -95,12 +100,12 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
in.Name, in.Name,
in.PriorProcessedName, in.PriorProcessedName,
) )
out.Description = preferredProductDescription( out.Description = preferredProductDescription(out.Name,
stringFromAny(normalized["description"]), stringFromAny(normalized["description"]),
out.Description, out.Description,
in.Description, in.Description,
) )
out.Category = stringFromAny(normalized["category"]) applyCategoryFromMapped(&out, normalized, in.Mapped, in.Raw)
// Promote characteristic fields only — never core product identity/content // Promote characteristic fields only — never core product identity/content
// (those belong on the V1 item root: title, description, ean, images, …). // (those belong on the V1 item root: title, description, ean, images, …).
promote := []string{"brand", "width", "height", "depth", "weight", "product_model", "warranty"} promote := []string{"brand", "width", "height", "depth", "weight", "product_model", "warranty"}
@@ -125,31 +130,15 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
} }
} }
attrs = SanitizeProductAttributes(attrs) attrs = SanitizeProductAttributes(attrs)
// Remap again after category may have resolved in applyCategoryFromMapped.
if allowed := enhanceAllowedAttrKeys(in, out.Category); allowed != nil {
attrs = MapAttrsOntoAllowedKeys(attrs, allowed)
}
out.Attributes = attrs out.Attributes = attrs
out.ProcessedAttributes = attrs out.ProcessedAttributes = attrs
appendStepLog(out.GPTResponse, StepFillFields, map[string]any{ appendStepLog(out.GPTResponse, StepFillFields, map[string]any{
"brand": stringFromAny(normalized["brand"]), "brand": stringFromAny(normalized["brand"]),
}) })
if out.Category == "" && e != nil && e.Vector != nil && e.Vector.Enabled() {
text := strings.TrimSpace(out.Name + " " + out.Description)
if text != "" {
if cat, err := e.Vector.SuggestCategory(ctx, companyID, text, categoryNames); err == nil && strings.TrimSpace(cat) != "" {
out.Category = SanitizeOutput(cat)
out.FieldSources["category"] = "vector"
out.Notes = append(out.Notes, "category: vector")
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
"status": "ok",
"category": out.Category,
})
} else if err != nil {
out.Notes = append(out.Notes, "vector_categorize: "+TruncateError(err))
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
"status": "failed",
"error": TruncateError(err),
})
}
}
}
preserveCategoryIfEmpty(&out, in.PriorCategory) preserveCategoryIfEmpty(&out, in.PriorCategory)
case StepEPREL: case StepEPREL:
@@ -161,7 +150,8 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
}) })
break break
} }
id := eprel.ExtractID(normalized, in.Mapped, in.Raw) // Also scan parsed attrs — eprel_id may only appear after parse_specs.
id := eprel.ExtractID(normalized, in.Mapped, in.Raw, attrs)
if id == "" { if id == "" {
out.Notes = append(out.Notes, "eprel: no id") out.Notes = append(out.Notes, "eprel: no id")
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "reason": "no_id"}) appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "reason": "no_id"})
@@ -173,7 +163,7 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
} }
if !enricher.Enabled() { if !enricher.Enabled() {
out.Notes = append(out.Notes, "eprel: enricher disabled") out.Notes = append(out.Notes, "eprel: enricher disabled")
out.EPREL = map[string]any{"eprel_id": id, "status": "skipped"} out.EPREL = map[string]any{"id": id, "status": "skipped"}
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "eprel_id": id, "reason": "disabled"}) appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "eprel_id": id, "reason": "disabled"})
break break
} }
@@ -191,15 +181,22 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
attrs["eprel_id"] = id attrs["eprel_id"] = id
out.Attributes = attrs out.Attributes = attrs
out.ProcessedAttributes = attrs out.ProcessedAttributes = attrs
out.EPREL = map[string]any{"eprel_id": id, "status": "empty"} out.EPREL = map[string]any{"id": id, "status": "empty"}
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "empty", "eprel_id": id}) appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "empty", "eprel_id": id})
break break
} }
attrs = eprel.MergeInto(attrs, data) attrs = eprel.MergeInto(attrs, data)
// Promote energy class onto the characteristic attr key when missing
// (feeds often omit it; EPREL API is the source of truth).
if data.EnergyClass != "" {
if cur := strings.TrimSpace(fmt.Sprint(attrs["energy_class"])); cur == "" || cur == "<nil>" {
attrs["energy_class"] = data.EnergyClass
}
}
out.Attributes = attrs out.Attributes = attrs
out.ProcessedAttributes = attrs out.ProcessedAttributes = attrs
out.EPREL = map[string]any{ out.EPREL = map[string]any{
"eprel_id": data.ID, "id": data.ID,
"label": data.Label, "label": data.Label,
"pdf": data.PDF, "pdf": data.PDF,
"energy_class": data.EnergyClass, "energy_class": data.EnergyClass,
@@ -247,6 +244,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if primary == "" { if primary == "" {
primary = langs[0] primary = langs[0]
} }
syncCategoryName(&out, in.CategoryNamesByUID)
displayCat := categoryDisplayLabel(out)
ensureEnergyClassFromEPREL(attrs)
enhanceAttrs := AttrsForEnhance(attrs, enhanceAllowedAttrKeys(in, out.Category))
localized := company.LocalizedContent{} localized := company.LocalizedContent{}
if in.PriorLocalized != nil { if in.PriorLocalized != nil {
for k, v := range in.PriorLocalized { for k, v := range in.PriorLocalized {
@@ -264,8 +265,9 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
} }
catPrompt := in.CategoryEnhancePrompt catPrompt := in.CategoryEnhancePrompt
if lang != primary || catPrompt == "" { if lang != primary || catPrompt == "" {
catPrompt = categoryEnhancePromptFor(in.CategoryPromptsByLang, out.Category, lang) catPrompt = categoryEnhancePromptFor(in.CategoryPromptsByLang, out.Category, lang, primary)
} }
titleTpl, descTpl := categoryFormulasFor(in, out.Category)
priorFields := company.FieldsForLanguage(in.PriorLocalized, lang) priorFields := company.FieldsForLanguage(in.PriorLocalized, lang)
priorHash := priorFields.EnhanceInputHash priorHash := priorFields.EnhanceInputHash
priorName := priorFields.ProcessedName priorName := priorFields.ProcessedName
@@ -291,10 +293,12 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
EnhanceSystemTemplate: tpl.System, EnhanceSystemTemplate: tpl.System,
EnhanceUserTemplate: tpl.User, EnhanceUserTemplate: tpl.User,
CategoryEnhancePrompt: catPrompt, CategoryEnhancePrompt: catPrompt,
TitleTemplate: titleTpl,
DescriptionTemplate: descTpl,
PriorEnhanceHash: priorHash, PriorEnhanceHash: priorHash,
PriorProcessedName: priorName, PriorProcessedName: priorName,
PriorProcessedDescription: priorDesc, PriorProcessedDescription: priorDesc,
}, out.Category, attrs) }, displayCat, enhanceAttrs)
out.TotalTokens += tokens out.TotalTokens += tokens
status := enhanceStatusFromMeta(raw) status := enhanceStatusFromMeta(raw)
meta := map[string]any{"language": lang, "raw": raw} meta := map[string]any{"language": lang, "raw": raw}
@@ -319,29 +323,59 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
anyOK = true anyOK = true
meta["status"] = status meta["status"] = status
} }
hash := enhanceHashFromMeta(raw) // Prefer title-aware selection + synthesize before deciding hash persistence.
name = preferredProductTitle(in.GTIN, name, out.Name, priorName, in.Name)
desc = preferredProductDescription(name, desc, out.Description, priorDesc)
if isWeakPriorEnhanceDescription(desc, name) {
if synth := synthesizeDescriptionFromTitle(name, displayCat, lang, enhanceAttrs); synth != "" {
desc = synth
}
}
weakDesc := isWeakPriorEnhanceDescription(desc, name)
// Only persist enhance_input_hash for quality ok / hash-skip unchanged.
// Never copy input_hash from error/passthrough/thin meta into localized.
persistHash := ""
rawHash := enhanceHashFromMeta(raw)
switch {
case err != nil:
persistHash = ""
case status == "unchanged" && !weakDesc:
persistHash = rawHash
case status == "ok" && !weakDesc:
persistHash = rawHash
default:
// thin ok / parse_failed / skipped — do not poison reprocess skip
persistHash = ""
if status == "ok" && weakDesc {
allUnchanged = false
}
}
localized[lang] = company.LocalizedFields{ localized[lang] = company.LocalizedFields{
ProcessedName: name, ProcessedName: name,
ProcessedDescription: desc, ProcessedDescription: desc,
EnhanceInputHash: hash, EnhanceInputHash: persistHash,
MetaTitle: company.FieldsForLanguage(localized, lang).MetaTitle, MetaTitle: company.FieldsForLanguage(localized, lang).MetaTitle,
MetaDescription: company.FieldsForLanguage(localized, lang).MetaDescription, MetaDescription: company.FieldsForLanguage(localized, lang).MetaDescription,
} }
// Preserve existing meta when re-enhancing titles only. // Preserve existing meta when re-enhancing titles only.
// Never keep a bare "| <unique_id>" poisoned meta_title.
// When dropping poisoned title, also drop empty/weak stub meta_description.
if prev := company.FieldsForLanguage(in.PriorLocalized, lang); prev.MetaTitle != "" || prev.MetaDescription != "" { if prev := company.FieldsForLanguage(in.PriorLocalized, lang); prev.MetaTitle != "" || prev.MetaDescription != "" {
f := localized[lang] f := localized[lang]
if f.MetaTitle == "" { poisonedTitle := isPoisonedMetaTitle(prev.MetaTitle)
if f.MetaTitle == "" && !poisonedTitle {
f.MetaTitle = prev.MetaTitle f.MetaTitle = prev.MetaTitle
} }
if f.MetaDescription == "" { if f.MetaDescription == "" {
weakStub := poisonedTitle && isWeakPriorEnhanceDescription(prev.MetaDescription, name, priorName)
if !weakStub {
f.MetaDescription = prev.MetaDescription f.MetaDescription = prev.MetaDescription
} }
}
localized[lang] = f localized[lang] = f
} }
langMetas = append(langMetas, meta) langMetas = append(langMetas, meta)
if lang == primary { if lang == primary {
name = preferredProductTitle(in.GTIN, name, out.Name, in.PriorProcessedName)
desc = preferredProductDescription(desc, out.Description, in.PriorProcessedDescription)
out.ProcessedName = name out.ProcessedName = name
out.ProcessedDescription = desc out.ProcessedDescription = desc
if name != "" { if name != "" {
@@ -350,10 +384,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if desc != "" { if desc != "" {
out.Description = desc out.Description = desc
} }
if hash != "" && (status == "ok" || status == "unchanged") { if persistHash != "" {
out.FieldSources[FieldEnhanceInputHash] = hash out.FieldSources[FieldEnhanceInputHash] = persistHash
} else if err != nil { } else {
preservePriorEnhanceHash() delete(out.FieldSources, FieldEnhanceInputHash)
} }
} }
} }
@@ -365,7 +399,34 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
if out.ProcessedDescription == "" { if out.ProcessedDescription == "" {
out.ProcessedDescription = out.Description out.ProcessedDescription = out.Description
} }
preservePriorEnhanceHash() // Timeout/error/empty: always synthesize a factual fallback when a title exists.
if out.ProcessedName != "" && (out.ProcessedDescription == "" ||
isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name)) {
if synth := synthesizeDescriptionFromTitle(out.ProcessedName, displayCat, primary, enhanceAttrs); synth != "" {
out.ProcessedDescription = synth
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) {
out.Description = synth
}
if lf, ok := localized[primary]; ok {
lf.ProcessedDescription = synth
if lf.ProcessedName == "" {
lf.ProcessedName = out.ProcessedName
}
localized[primary] = lf
}
}
}
// Attempted enhance (even on timeout) — record provider mode, not misleading unknown.
out.AIProviderMode = e.EngineProviderMode()
out.FieldSources["name"] = "ai_enhance_failed"
out.FieldSources["description"] = "ai_enhance_failed"
// Failed enhance must not leave a skippable hash behind.
delete(out.FieldSources, FieldEnhanceInputHash)
for lang, lf := range localized {
lf.EnhanceInputHash = ""
localized[lang] = lf
}
out.LocalizedContent = localized
errNote := "" errNote := ""
for _, m := range langMetas { for _, m := range langMetas {
if mm, ok := m.(map[string]any); ok { if mm, ok := m.(map[string]any); ok {
@@ -402,31 +463,50 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
} }
} }
// Paths without fill_fields (enhance_only / normalize_only) still get vector
// categorize when AllowAI + embeddings are available.
tryVectorCategorize(ctx, e, companyID, &out, categoryNames, policy)
preserveCategoryIfEmpty(&out, in.PriorCategory) preserveCategoryIfEmpty(&out, in.PriorCategory)
noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled())
syncCategoryName(&out, in.CategoryNamesByUID)
displayCat := categoryDisplayLabel(out)
out.Name = preferredProductTitle(in.GTIN, out.Name, out.ProcessedName, in.Name, in.PriorProcessedName) out.Name = preferredProductTitle(in.GTIN, out.Name, out.ProcessedName, in.Name, in.PriorProcessedName)
out.ProcessedName = preferredProductTitle(in.GTIN, out.ProcessedName, out.Name, in.PriorProcessedName, in.Name) out.ProcessedName = preferredProductTitle(in.GTIN, out.ProcessedName, out.Name, in.PriorProcessedName, in.Name)
out.Description = preferredProductDescription(out.Description, out.ProcessedDescription, in.Description) out.Description = preferredProductDescription(out.Name, out.Description, out.ProcessedDescription, in.Description)
out.ProcessedDescription = preferredProductDescription(out.ProcessedDescription, out.Description, in.PriorProcessedDescription) out.ProcessedDescription = preferredProductDescription(out.ProcessedName, out.ProcessedDescription, out.Description, in.PriorProcessedDescription)
if out.ProcessedName == "" { if out.ProcessedName == "" {
out.ProcessedName = out.Name out.ProcessedName = out.Name
} }
if out.ProcessedDescription == "" { if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) {
out.ProcessedDescription = out.Description if synth := synthesizeDescriptionFromTitle(out.ProcessedName, displayCat, in.Language, out.Attributes); synth != "" {
out.ProcessedDescription = synth
}
}
if out.Description == "" || isWeakPriorEnhanceDescription(out.Description, out.Name, out.ProcessedName) {
if out.ProcessedDescription != "" && !isWeakPriorEnhanceDescription(out.ProcessedDescription, out.Name, out.ProcessedName) {
out.Description = out.ProcessedDescription
} else if synth := synthesizeDescriptionFromTitle(out.Name, displayCat, in.Language, out.Attributes); synth != "" {
out.Description = synth
if out.ProcessedDescription == "" || isWeakPriorEnhanceDescription(out.ProcessedDescription, out.ProcessedName, out.Name) {
out.ProcessedDescription = synth
}
}
} }
if out.Attributes == nil { if out.Attributes == nil {
out.Attributes = map[string]any{} out.Attributes = map[string]any{}
} }
out.Attributes = SanitizeProductAttributes(out.Attributes) // Persist the same sanitize + category allowlist used by enhance (poll already
out.ProcessedAttributes = SanitizeProductAttributes(out.ProcessedAttributes) // projects clean attrs; DB must not keep feed junk like zavora/vzmetenje).
allowed := enhanceAllowedAttrKeys(in, out.Category)
out.Attributes = AttrsForPersist(out.Attributes, allowed)
out.ProcessedAttributes = AttrsForPersist(out.ProcessedAttributes, allowed)
if len(out.ProcessedAttributes) == 0 { if len(out.ProcessedAttributes) == 0 {
out.ProcessedAttributes = out.Attributes out.ProcessedAttributes = out.Attributes
} }
if out.AIProviderMode == "" { // Prefer EngineProviderMode (job/completer label) over stamping "unknown" on
if out.TotalTokens > 0 { // 0-token paths (hash-skip / AI skipped). processOne also treats unknown as empty.
out.AIProviderMode = e.EngineProviderMode() if isUnknownProviderMode(out.AIProviderMode) {
} else { out.AIProviderMode = preferKnownProviderMode(e.EngineProviderMode(), out.AIProviderMode)
out.AIProviderMode = AIProviderUnknown
}
} }
if len(out.Notes) > 0 { if len(out.Notes) > 0 {
out.GPTResponse["notes"] = out.Notes out.GPTResponse["notes"] = out.Notes
@@ -451,6 +531,66 @@ func preserveCategoryIfEmpty(out *StepResult, prior string) {
out.FieldSources["category"] = "prior_processed" out.FieldSources["category"] = "prior_processed"
} }
// tryVectorCategorize sets Category from embeddings when mapped unique_id is absent.
// Requires policy.AllowAI and a configured/enabled VectorCategorizer.
func tryVectorCategorize(ctx context.Context, e *Engine, companyID string, out *StepResult, categoryNames []string, policy StepPolicy) {
if out == nil || strings.TrimSpace(out.Category) != "" {
return
}
if !policy.AllowAI {
return
}
if e == nil || e.Vector == nil || !e.Vector.Enabled() {
return
}
text := strings.TrimSpace(out.Name + " " + out.Description)
if text == "" {
return
}
cat, err := e.Vector.SuggestCategory(ctx, companyID, text, categoryNames)
if err != nil {
out.Notes = append(out.Notes, "vector_categorize: "+TruncateError(err))
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
"status": "failed",
"error": TruncateError(err),
})
return
}
if strings.TrimSpace(cat) == "" {
return
}
out.Category = SanitizeOutput(cat)
if out.FieldSources == nil {
out.FieldSources = map[string]any{}
}
out.FieldSources["category"] = "vector"
out.Notes = append(out.Notes, "category: vector")
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
"status": "ok",
"category": out.Category,
})
}
// noteMissingCategory records why Category stayed empty (mapped absent; vector skipped or failed).
func noteMissingCategory(out *StepResult, policy StepPolicy, vectorEnabled bool) {
if out == nil || strings.TrimSpace(out.Category) != "" {
return
}
for _, n := range out.Notes {
if strings.HasPrefix(n, "category: unset") || strings.HasPrefix(n, "category: vector") {
return
}
}
switch {
case !policy.AllowAI:
out.Notes = append(out.Notes, "category: unset (no mapped unique_id; AI/vector not allowed)")
case !vectorEnabled:
out.Notes = append(out.Notes, "category: unset (no mapped unique_id; vector embeddings unavailable)")
default:
out.Notes = append(out.Notes, "category: unset (no mapped unique_id; vector did not match)")
}
}
func resolveSteps(processingType string) []string { func resolveSteps(processingType string) []string {
switch strings.ToLower(strings.TrimSpace(processingType)) { switch strings.ToLower(strings.TrimSpace(processingType)) {
case "enhance", "enhance_only", "enhance-only", "title", "description": case "enhance", "enhance_only", "enhance-only", "title", "description":
@@ -458,7 +598,8 @@ func resolveSteps(processingType string) []string {
case "attributes", "attributes_only", "specs", "specifications": case "attributes", "attributes_only", "specs", "specifications":
return []string{StepNormalize, StepParseSpecs, StepFillFields} return []string{StepNormalize, StepParseSpecs, StepFillFields}
case "eprel", "eprel_only": case "eprel", "eprel_only":
return []string{StepNormalize, StepEPREL} // parse_specs first so eprel_id buried in specifications/attributes is visible.
return []string{StepNormalize, StepParseSpecs, StepEPREL}
case "normalize_only": case "normalize_only":
return []string{StepNormalize} return []string{StepNormalize}
case "categorize", "categorize_only", "categorize_enhance": case "categorize", "categorize_only", "categorize_enhance":
@@ -488,15 +629,19 @@ func (e *Engine) enhance(ctx context.Context, in ProductInput, category string,
sysTpl, userTpl := resolveProductPromptTemplates(in) sysTpl, userTpl := resolveProductPromptTemplates(in)
hash := HashEnhanceInput(category, in.Name, in.Description, in.BrandPrompt, in.Language, sysTpl, userTpl, attrs) hash := HashEnhanceInput(category, in.Name, in.Description, in.BrandPrompt, in.Language, sysTpl, userTpl, attrs)
if e == nil || e.Completer == nil { if e == nil || e.Completer == nil {
return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName), name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
preferredProductDescription(in.Description, in.PriorProcessedDescription), return name,
preferredProductDescription(name, in.Description, in.PriorProcessedDescription),
0, map[string]any{"status": "skipped", "input_hash": hash}, nil 0, map[string]any{"status": "skipped", "input_hash": hash}, nil
} }
// Skip LLM when inputs match the last successful enhance (before any credit debit). // Skip LLM when inputs match the last successful enhance (before any credit debit),
// but never reuse a thin / title-echo prior description, or a prompt-leakage title.
if in.PriorEnhanceHash != "" && in.PriorEnhanceHash == hash && if in.PriorEnhanceHash != "" && in.PriorEnhanceHash == hash &&
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") { (in.PriorProcessedName != "" || in.PriorProcessedDescription != "") &&
!isPromptLabelTitle(in.PriorProcessedName) &&
!isWeakPriorEnhanceDescription(in.PriorProcessedDescription, in.PriorProcessedName, in.Name) {
name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name) name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name)
desc := preferredProductDescription(in.PriorProcessedDescription, in.Description) desc := preferredProductDescription(name, in.PriorProcessedDescription, in.Description)
return name, desc, 0, map[string]any{ return name, desc, 0, map[string]any{
"status": "unchanged", "status": "unchanged",
"input_hash": hash, "input_hash": hash,
@@ -509,32 +654,43 @@ func (e *Engine) enhance(ctx context.Context, in ProductInput, category string,
}) })
if err != nil { if err != nil {
// Network/provider failure vs parse failure after retry // Network/provider failure vs parse failure after retry
name := preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName)
desc := preferredProductDescription(name, in.Description, in.PriorProcessedDescription)
if isWeakPriorEnhanceDescription(desc, name) {
if synth := synthesizeDescriptionFromTitle(name, category, in.Language, attrs); synth != "" {
desc = synth
}
}
if obj == nil && comp.Text == "" { if obj == nil && comp.Text == "" {
return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName), return name, desc, 0, map[string]any{
preferredProductDescription(in.Description, in.PriorProcessedDescription),
0, map[string]any{
"provider": "passthrough", "provider": "passthrough",
"error": TruncateError(err), "error": TruncateError(err),
"input_hash": hash,
}, err }, err
} }
// Parse failed after retry — keep original copy (avoid garbage titles) // Parse failed after retry — keep usable copy; synthesize when empty/weak.
return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName), return name, desc, comp.TotalTokens, map[string]any{
preferredProductDescription(in.Description, in.PriorProcessedDescription),
comp.TotalTokens, map[string]any{
"status": "parse_failed", "status": "parse_failed",
"error": "AI returned invalid JSON; kept original title/description", "error": "AI returned invalid JSON; kept original title/description",
"raw": truncateRunes(comp.Text, 200), "raw": truncateRunes(comp.Text, 200),
"input_hash": hash,
}, nil }, nil
} }
name := preferredProductTitle(in.GTIN, SanitizeOutput(fmt.Sprint(obj["name"])), in.Name, in.PriorProcessedName) name := preferredProductTitle(in.GTIN, SanitizeOutput(fmt.Sprint(obj["name"])), in.Name, in.PriorProcessedName)
desc := preferredProductDescription(SanitizeOutput(fmt.Sprint(obj["description"])), in.Description, in.PriorProcessedDescription) desc := preferredProductDescription(name, SanitizeOutput(fmt.Sprint(obj["description"])), in.Description, in.PriorProcessedDescription)
return name, desc, comp.TotalTokens, map[string]any{ if isWeakPriorEnhanceDescription(desc, name) {
if synth := synthesizeDescriptionFromTitle(name, category, in.Language, attrs); synth != "" {
desc = synth
}
}
meta := map[string]any{
"status": "ok", "status": "ok",
"input_hash": hash,
"raw": comp.Raw, "raw": comp.Raw,
}, nil }
// Only attach input_hash when output description is quality-worthy; thin
// title-echo / heuristic filler must not poison field_sources / localized skip hashes.
if !isWeakPriorEnhanceDescription(desc, name) {
meta["input_hash"] = hash
}
return name, desc, comp.TotalTokens, meta, nil
} }
func sanitizeJSON(v any) string { func sanitizeJSON(v any) string {
@@ -556,46 +712,67 @@ func firstLine(s string) string {
return SanitizeOutput(strings.Trim(s, "\"'` ")) return SanitizeOutput(strings.Trim(s, "\"'` "))
} }
// labeledPromptValue returns the first line after any of the given labels // labeledPromptValue returns the first usable line after any of the given labels
// (case-insensitive), e.g. "Name:" / "Desc:" from ProductEnhanceUser. // (case-insensitive), e.g. "Name:" / "Desc:" from ProductEnhanceUser.
// Skips matches whose value is prompt-label / formula-leakage text so instruction
// bullets like "- name: short retail title; follow any Title formula…" do not
// win over the later "Name: <product>" line in CategoryEnhanceUserTemplate.
func labeledPromptValue(user string, labels ...string) string { func labeledPromptValue(user string, labels ...string) string {
lower := strings.ToLower(user) lower := strings.ToLower(user)
bestAt := -1 type hit struct {
bestLabel := "" at int
label string
}
var hits []hit
for _, label := range labels { for _, label := range labels {
label = strings.ToLower(strings.TrimSpace(label)) label = strings.ToLower(strings.TrimSpace(label))
if label == "" { if label == "" {
continue continue
} }
at := strings.Index(lower, label) searchFrom := 0
if at < 0 { for {
continue rel := strings.Index(lower[searchFrom:], label)
if rel < 0 {
break
} }
if bestAt < 0 || at < bestAt { at := searchFrom + rel
bestAt = at hits = append(hits, hit{at: at, label: label})
bestLabel = label searchFrom = at + len(label)
} }
} }
if bestAt < 0 { if len(hits) == 0 {
return "" return ""
} }
rest := user[bestAt+len(bestLabel):] sort.Slice(hits, func(i, j int) bool { return hits[i].at < hits[j].at })
for _, h := range hits {
rest := user[h.at+len(h.label):]
if j := strings.Index(strings.ToLower(rest), "attrs:"); j >= 0 { if j := strings.Index(strings.ToLower(rest), "attrs:"); j >= 0 {
rest = rest[:j] rest = rest[:j]
} }
if j := strings.Index(strings.ToLower(rest), "attributes:"); j >= 0 { if j := strings.Index(strings.ToLower(rest), "attributes:"); j >= 0 {
rest = rest[:j] rest = rest[:j]
} }
return firstLine(rest) val := firstLine(rest)
if val == "" || isPromptLabelTitle(val) {
continue
}
return val
}
return ""
} }
// isPromptLabelTitle detects enhance pollution where the model echoed a // isPromptLabelTitle detects enhance pollution where the model echoed a
// prompt header ("Category:" / "Category: 120") as the product title. // prompt header ("Category:" / "Category: 120") as the product title, or
// leaked instruction scaffolding from CategoryEnhanceUserTemplate /
// AppendFormulaConstraints into name.
func isPromptLabelTitle(s string) bool { func isPromptLabelTitle(s string) bool {
s = strings.TrimSpace(s) s = strings.TrimSpace(s)
if s == "" || s == "<nil>" { if s == "" || s == "<nil>" {
return false return false
} }
if isPromptLeakageTitle(s) {
return true
}
lower := strings.ToLower(s) lower := strings.ToLower(s)
for _, label := range []string{ for _, label := range []string{
"category", "name", "desc", "description", "category", "name", "desc", "description",
@@ -611,8 +788,61 @@ func isPromptLabelTitle(s string) bool {
return false return false
} }
// preferredProductTitle picks the first usable title, skipping empty values and // isPromptLeakageTitle detects when an LLM echoed enhance-prompt instructions
// prompt-label echoes like "Category:" (seen on A1 Elkotex reprocess). // (Title formula / short retail title / Schema / Reply with ONLY JSON / …)
// as the product name instead of a real title.
func isPromptLeakageTitle(s string) bool {
s = strings.TrimSpace(s)
if s == "" || s == "<nil>" {
return false
}
lower := strings.ToLower(s)
for _, phrase := range promptLeakagePhrases {
if strings.Contains(lower, phrase) {
return true
}
}
// Long dumps of the enhance template: any formula/schema keyword is enough.
if len([]rune(s)) > 120 {
for _, kw := range promptLeakageLongKeywords {
if strings.Contains(lower, kw) {
return true
}
}
}
return false
}
// Phrases copied from aiprompts.CategoryEnhanceUserTemplate, BuiltInDefaults,
// and processing.AppendFormulaConstraints / FormatTitleFormulaConstraint.
var promptLeakagePhrases = []string{
"title formula",
"follow any",
"constraints that follow",
"short retail title",
"use attrs",
"write name in",
"schema:",
"reply with only json",
"your reply is parsed as json",
"description formula",
"build name from attrs",
"prefer attrs values",
"order matters; join with",
"do not hardcode a language",
}
var promptLeakageLongKeywords = []string{
"formula",
"constraints",
"schema",
"json",
"attrs",
"retail title",
}
// preferredProductTitle picks the first usable title, skipping empty values,
// prompt-label echoes like "Category:", and instruction-text leakage.
func preferredProductTitle(gtin string, candidates ...string) string { func preferredProductTitle(gtin string, candidates ...string) string {
for _, c := range candidates { for _, c := range candidates {
c = strings.TrimSpace(c) c = strings.TrimSpace(c)
@@ -627,13 +857,172 @@ func preferredProductTitle(gtin string, candidates ...string) string {
return "Product" return "Product"
} }
func preferredProductDescription(candidates ...string) string { // Thin wrappers keep processing call sites stable; logic lives in company so
// catalog.RepairWeakEnhanceHashes can reuse it without an import cycle.
func isWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
return company.IsWeakPriorEnhanceDescription(priorDesc, titles...)
}
func containsWeakFillerPhrase(desc string) bool {
return company.ContainsWeakFillerPhrase(desc)
}
func descriptionEchoesTitle(desc, title string) bool {
return company.DescriptionEchoesTitle(desc, title)
}
// preferredProductDescription picks the first usable description, skipping empty
// values, prompt-label echoes, weak filler phrases, and copy that merely repeats
// the product title.
func preferredProductDescription(title string, candidates ...string) string {
for _, c := range candidates { for _, c := range candidates {
c = strings.TrimSpace(c) c = strings.TrimSpace(c)
if c == "" || c == "<nil>" || isPromptLabelTitle(c) { if c == "" || c == "<nil>" || isPromptLabelTitle(c) {
continue continue
} }
if descriptionEchoesTitle(c, title) {
continue
}
if containsWeakFillerPhrase(c) {
continue
}
return SanitizeOutput(c) return SanitizeOutput(c)
} }
return "" return ""
} }
func attrLookupCI(attrs map[string]any, keys ...string) string {
if attrs == nil {
return ""
}
for _, want := range keys {
want = strings.TrimSpace(want)
if want == "" {
continue
}
if v := strings.TrimSpace(stringFromAny(attrs[want])); v != "" {
return v
}
for k, raw := range attrs {
if strings.EqualFold(strings.TrimSpace(k), want) {
if v := strings.TrimSpace(stringFromAny(raw)); v != "" {
return v
}
}
}
}
return ""
}
func formatAttrDimParts(attrs map[string]any, maxParts int) []string {
if attrs == nil || maxParts <= 0 {
return nil
}
prefer := []string{
"width", "height", "depth", "weight", "max_load", "max load", "load_capacity",
"vesa", "screen_size", "diagonal", "color", "material", "size",
}
parts := make([]string, 0, maxParts)
seen := map[string]struct{}{}
add := func(k, v string) {
k = strings.TrimSpace(k)
v = strings.TrimSpace(v)
if k == "" || v == "" {
return
}
lk := strings.ToLower(k)
if _, ok := seen[lk]; ok {
return
}
seen[lk] = struct{}{}
parts = append(parts, fmt.Sprintf("%s %s", k, v))
}
for _, k := range prefer {
if len(parts) >= maxParts {
break
}
if v := attrLookupCI(attrs, k); v != "" {
add(k, v)
}
}
return parts
}
// synthesizeDescriptionFromTitle builds a short factual fallback when enhance
// returns empty/title-echo/filler copy. Uses title, category, brand, model, and
// key dims. language is a content-language code (en/sl/…) or English label.
func synthesizeDescriptionFromTitle(title, category, language string, attrs map[string]any) string {
title = strings.TrimSpace(title)
if title == "" || title == "<nil>" || isPromptLabelTitle(title) {
return ""
}
cat := strings.TrimSpace(category)
if strings.EqualFold(cat, "general") {
cat = ""
}
brand := attrLookupCI(attrs, "brand")
model := attrLookupCI(attrs, "product_model", "model", "sku")
dims := formatAttrDimParts(attrs, 3)
sl := isSlovenianContentLanguage(language)
var b strings.Builder
if sl {
b.WriteString(title)
switch {
case cat != "" && brand != "":
fmt.Fprintf(&b, " je izdelek v kategoriji %s znamke %s", cat, brand)
case cat != "":
fmt.Fprintf(&b, " je izdelek v kategoriji %s", cat)
case brand != "":
fmt.Fprintf(&b, " je izdelek znamke %s", brand)
default:
b.WriteString(" je katalogski izdelek z znanimi atributi")
}
if model != "" && !strings.Contains(strings.ToLower(title), strings.ToLower(model)) {
fmt.Fprintf(&b, " (model %s)", model)
}
if len(dims) > 0 {
fmt.Fprintf(&b, ". Ključne specifikacije: %s", strings.Join(dims, ", "))
}
b.WriteByte('.')
} else {
b.WriteString(title)
switch {
case cat != "" && brand != "":
fmt.Fprintf(&b, " is a %s product from %s", cat, brand)
case cat != "":
fmt.Fprintf(&b, " is listed in the %s category", cat)
case brand != "":
fmt.Fprintf(&b, " is a product from %s", brand)
default:
b.WriteString(" is a catalog product with the known attributes")
}
if model != "" && !strings.Contains(strings.ToLower(title), strings.ToLower(model)) {
fmt.Fprintf(&b, " (model %s)", model)
}
if len(dims) > 0 {
fmt.Fprintf(&b, ". Key specs: %s", strings.Join(dims, ", "))
}
b.WriteByte('.')
}
out := SanitizeOutput(b.String())
// Never emit sole retail-filler when title/attrs exist — strip legacy phrase if any helper reintroduces it.
if containsWeakFillerPhrase(out) {
out = strings.TrimSpace(strings.ReplaceAll(out, "Ready for retail listing.", ""))
out = strings.TrimSpace(strings.ReplaceAll(out, "ready for retail listing.", ""))
out = strings.TrimSpace(strings.Trim(out, ".")) + "."
}
return out
}
func isSlovenianContentLanguage(raw string) bool {
raw = strings.TrimSpace(raw)
if raw == "" {
return false
}
if code, err := company.ParseLanguage(raw, false); err == nil && code == "sl" {
return true
}
lower := strings.ToLower(raw)
return lower == "slovenian" || strings.Contains(lower, "slovenian")
}
+107 -1
View File
@@ -22,7 +22,7 @@ func TestResolveSteps(t *testing.T) {
"full": {StepNormalize, StepParseSpecs, StepFillFields, StepEPREL, StepAIEnhance}, "full": {StepNormalize, StepParseSpecs, StepFillFields, StepEPREL, StepAIEnhance},
"enhance_only": {StepNormalize, StepAIEnhance}, "enhance_only": {StepNormalize, StepAIEnhance},
"attributes_only": {StepNormalize, StepParseSpecs, StepFillFields}, "attributes_only": {StepNormalize, StepParseSpecs, StepFillFields},
"eprel_only": {StepNormalize, StepEPREL}, "eprel_only": {StepNormalize, StepParseSpecs, StepEPREL},
"normalize_only": {StepNormalize}, "normalize_only": {StepNormalize},
} }
for in, want := range cases { for in, want := range cases {
@@ -187,6 +187,18 @@ func TestPreferredProductTitle_skipsPromptLabels(t *testing.T) {
} }
} }
func TestPreferredProductTitle_skipsFormulaLeakage(t *testing.T) {
leak := "short retail title; follow any Title formula constraints that follow; use Attrs"
got := preferredProductTitle("1", leak, "follow any Title formula constraints that follow; use Attrs", "Vox SWA-8000W")
if got != "Vox SWA-8000W" {
t.Fatalf("got %q want feed title", got)
}
got = preferredProductTitle("", leak, "Schema: {\"name\":\"string\"}", "Reply with ONLY JSON")
if got != "Product" {
t.Fatalf("all-leakage fallback got %q", got)
}
}
func TestIsPromptLabelTitle(t *testing.T) { func TestIsPromptLabelTitle(t *testing.T) {
cases := map[string]bool{ cases := map[string]bool{
"Category:": true, "Category:": true,
@@ -204,6 +216,100 @@ func TestIsPromptLabelTitle(t *testing.T) {
} }
} }
func TestIsPromptLeakageTitle(t *testing.T) {
cases := map[string]bool{
"follow any Title formula constraints that follow; use Attrs": true,
"short retail title; follow any Title formula constraints…": true,
"write name in {{language}}": true,
"Schema: {\"name\":\"string\",\"description\":\"string\"}": true,
"Reply with ONLY JSON (no markdown)": true,
"Vox SWA-8000W dishwasher": false,
"Acme Widget Pro": false,
"": false,
}
for in, want := range cases {
if got := isPromptLeakageTitle(in); got != want {
t.Fatalf("%q: leakage got %v want %v", in, got, want)
}
if want && !isPromptLabelTitle(in) {
t.Fatalf("%q: isPromptLabelTitle should include leakage", in)
}
}
// Long dump with formula keyword (>120 runes).
long := strings.Repeat("x", 100) + " Title formula scaffolding " + strings.Repeat("y", 30)
if !isPromptLeakageTitle(long) {
t.Fatalf("long formula dump should leak: len=%d", len([]rune(long)))
}
}
func TestEnhance_rejectsFormulaLeakageTitle(t *testing.T) {
leak := "short retail title; follow any Title formula constraints that follow; use Attrs"
e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
return Completion{Text: `{"name":"` + leak + `","description":"A solid washer."}`, TotalTokens: 2}, nil
}},
Vector: NoopVectorCategorizer{},
}
out, err := e.RunSteps(context.Background(), "co", ProductInput{
GTIN: "8712285326882",
Name: "Vox SWA-8000W",
Mapped: map[string]any{"name": "Vox SWA-8000W", "description": "Washer", "category": "demo-electronics"},
}, "enhance_only", nil, StepPolicy{AllowAI: true})
if err != nil {
t.Fatal(err)
}
if out.ProcessedName != "Vox SWA-8000W" {
t.Fatalf("ProcessedName=%q want Vox SWA-8000W (reject formula leakage)", out.ProcessedName)
}
if isPromptLeakageTitle(out.Name) || isPromptLabelTitle(out.Name) {
t.Fatalf("Name polluted: %q", out.Name)
}
}
func TestHeuristicCompleter_keepsNameWhenFormulaConstraintsPresent(t *testing.T) {
h := HeuristicCompleter{}
// Shape matches aiprompts.CategoryEnhanceUserTemplate: instruction "- name:"
// appears BEFORE the real "Name:" field — labeledPromptValue must skip leakage.
user := `Your reply is parsed as JSON {"name":"string","description":"string"} only (system schema). Write name and description in English (do not hardcode a language).
- name: short retail title; follow any Title formula constraints that follow; use Attrs
- description: prefer 1-3 factual paragraphs as ONE string
Category: demo-electronics
Name: Vox SWA-8000W
Desc: Washer
Attrs: {"brand":"Vox"}
Title formula (order matters; join with " "). Build name from Attrs using this structure:
1. attr [brand]
Prefer Attrs values for [attr] slots; write name in English.`
comp, err := h.Complete(context.Background(), `Return JSON with "name" and "description".`, user)
if err != nil {
t.Fatal(err)
}
obj, err := ParseJSONObject(comp.Text)
if err != nil {
t.Fatal(err)
}
name := SanitizeOutput(fmt.Sprint(obj["name"]))
if isPromptLeakageTitle(name) || isPromptLabelTitle(name) {
t.Fatalf("heuristic leaked formula into name: %q", name)
}
if !strings.Contains(strings.ToLower(name), "vox") {
t.Fatalf("name=%q want Vox from Name: line", name)
}
}
func TestLabeledPromptValue_skipsInstructionNameBullet(t *testing.T) {
user := `- name: short retail title; follow any Title formula constraints that follow; use Attrs
Name: Vox SWA-8000W
Desc: Washer
Attrs: {}`
got := labeledPromptValue(user, "name:", "current name:")
if got != "Vox SWA-8000W" {
t.Fatalf("got %q want Vox SWA-8000W", got)
}
}
func TestRunSteps_full_mappedCategoryWinsOverPrior(t *testing.T) { func TestRunSteps_full_mappedCategoryWinsOverPrior(t *testing.T) {
e := &Engine{ e := &Engine{
Completer: stubCompleter{fn: func(system, user string) (Completion, error) { Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
@@ -28,6 +28,22 @@ func TestUpsertProcessedProductSQL_usesOnConflict(t *testing.T) {
if !strings.Contains(upsertProcessedProductSQL, "COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category)") { if !strings.Contains(upsertProcessedProductSQL, "COALESCE(NULLIF(BTRIM(EXCLUDED.category), ''), processed_products.category)") {
t.Fatalf("expected category preserve on conflict: got SQL without COALESCE preserve") t.Fatalf("expected category preserve on conflict: got SQL without COALESCE preserve")
} }
// Free template meta must persist; existing non-empty meta must be preserved on reprocess.
if !strings.Contains(upsertProcessedProductSQL, "meta_title") || !strings.Contains(upsertProcessedProductSQL, "meta_description") {
t.Fatalf("expected meta_title/meta_description columns in upsert")
}
if !strings.Contains(upsertProcessedProductSQL, "WHEN NULLIF(BTRIM(processed_products.meta_title), '') IS NULL") {
t.Fatalf("expected meta_title preserve-when-set CASE on conflict")
}
if !strings.Contains(upsertProcessedProductSQL, `~ '\|\s+[0-9]+$'`) {
t.Fatalf("expected meta_title refresh when trailing | <digits-only unique_id>")
}
if !strings.Contains(upsertProcessedProductSQL, "WHEN NULLIF(BTRIM(processed_products.meta_description), '') IS NULL") {
t.Fatalf("expected meta_description preserve-when-set CASE on conflict")
}
if !strings.Contains(upsertProcessedProductSQL, "ready for retail listing") {
t.Fatalf("expected meta_description refresh of weak stubs when meta_title is poisoned")
}
} }
func TestUpsertProcessedProduct_concurrentIdempotent(t *testing.T) { func TestUpsertProcessedProduct_concurrentIdempotent(t *testing.T) {
@@ -0,0 +1,98 @@
package processing
import (
"context"
"os"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestLoadV1ProcessJobItems_promotesMoreimagesWhenMainEmpty(t *testing.T) {
dsn := os.Getenv("DATABASE_URL")
if dsn == "" {
t.Skip("DATABASE_URL not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
pg, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal(err)
}
defer pg.Close()
companyID := uuid.New()
userID := uuid.New()
rawID := uuid.New()
jobID := uuid.New()
ppID := uuid.New()
gtin := "img-v1-" + companyID.String()[:8]
if _, err := pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`, companyID, "img-v1-test"); err != nil {
t.Fatal(err)
}
err = pg.QueryRow(ctx, `SELECT id FROM users ORDER BY created_at DESC LIMIT 1`).Scan(&userID)
if err != nil {
t.Skip("no users rows available")
}
defer func() {
_, _ = pg.Exec(context.Background(), `DELETE FROM processing_job_products WHERE job_id = $1`, jobID)
_, _ = pg.Exec(context.Background(), `DELETE FROM processing_jobs WHERE id = $1`, jobID)
_, _ = pg.Exec(context.Background(), `DELETE FROM processed_products WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM raw_products WHERE company_id = $1`, companyID)
_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
}()
mapped := `{"main_image":"","moreimages":"https://cdn.example.com/a.jpg,https://cdn.example.com/b.jpg","name":"Widget"}`
if _, err := pg.Exec(ctx, `
INSERT INTO raw_products (id, company_id, gtin, raw_data, mapped_data, is_processed, processing_status)
VALUES ($1, $2, $3, '{}'::jsonb, $4::jsonb, true, 'processed')`,
rawID, companyID, gtin, mapped); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO processed_products (
id, company_id, product_id, name, category, description, processed_name, processed_description,
raw_product_id, status, attributes, processed_attributes, field_sources
) VALUES (
$1, $2, $3, 'Widget', NULL, 'desc', 'Widget', 'desc',
$4, 'needs_review', '{}'::jsonb, '{}'::jsonb, '{}'::jsonb
)`, ppID, companyID, gtin, rawID); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO processing_jobs (id, company_id, user_id, status, processing_type, total_products, processed_products)
VALUES ($1, $2, $3, 'completed', 'full', 1, 1)`, jobID, companyID, userID); err != nil {
t.Fatal(err)
}
if _, err := pg.Exec(ctx, `
INSERT INTO processing_job_products (id, job_id, raw_product_id, processed_product_id, status)
VALUES (gen_random_uuid(), $1, $2, $3, 'processed')`, jobID, rawID, ppID); err != nil {
t.Fatal(err)
}
p := NewPipeline(pg)
items, err := p.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
if err != nil {
t.Fatal(err)
}
if len(items) != 1 {
t.Fatalf("items=%d want 1", len(items))
}
main, _ := items[0]["main_image"].(string)
if main != "https://cdn.example.com/a.jpg" {
t.Fatalf("main_image=%v want promoted moreimages[0]", items[0]["main_image"])
}
switch more := items[0]["more_images"].(type) {
case []string:
if len(more) != 1 || more[0] != "https://cdn.example.com/b.jpg" {
t.Fatalf("more_images=%v", more)
}
default:
t.Fatalf("more_images type %T = %v", items[0]["more_images"], items[0]["more_images"])
}
}
+197 -31
View File
@@ -9,6 +9,7 @@ import (
"strings" "strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog" "github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
"github.com/google/uuid" "github.com/google/uuid"
) )
@@ -206,6 +207,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
return nil, fmt.Errorf("pipeline not configured") return nil, fmt.Errorf("pipeline not configured")
} }
allowedAttrs := loadCompanyAttributeKeySet(ctx, p, companyID) allowedAttrs := loadCompanyAttributeKeySet(ctx, p, companyID)
categoryNames := loadCompanyCategoryNameMap(ctx, p, companyID)
rows, err := p.Pool.Query(ctx, ` rows, err := p.Pool.Query(ctx, `
SELECT SELECT
COALESCE(r.gtin, p.product_id, '') AS ean, COALESCE(r.gtin, p.product_id, '') AS ean,
@@ -297,31 +299,98 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
if descTxt != nil { if descTxt != nil {
plainDesc = v1PlainDescription(*descTxt) plainDesc = v1PlainDescription(*descTxt)
} }
eprelVal := extractEPRELFromAttrs(attrs)
attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowedAttrs)
titleStr := derefStringPtr(title)
catStr := derefStringPtr(category)
catNameStr := derefStringPtr(categoryName)
// Projection gap: when processed.category is empty but mapped/raw carries a
// unique_id, surface it (and resolve category_name from company taxonomy).
if catStr == "" {
catStr = categoryUniqueIDFromMaps(mapped, rawData)
}
if catNameStr == "" && catStr != "" {
if name, ok := categoryNames[catStr]; ok && name != "" {
catNameStr = name
} else if nested, ok := mapped["category"].(map[string]any); ok {
for _, k := range []string{"name", "category_name", "title"} {
if s := strings.TrimSpace(stringFromAny(nested[k])); s != "" {
catNameStr = s
break
}
}
}
}
metaTitleOut := nullIfEmptyPtr(metaTitle)
metaDescOut := nullIfEmptyPtr(metaDesc)
if s, ok := metaTitleOut.(string); ok && isPoisonedMetaTitle(s) {
metaTitleOut = nil
// Poisoned title refresh: also drop empty/weak/leakage stub meta_description.
if ds, ok := metaDescOut.(string); ok && (ds == "" || isWeakPriorEnhanceDescription(ds, titleStr) || isPromptLeakageTitle(ds)) {
metaDescOut = nil
}
}
if ds, ok := metaDescOut.(string); ok && isPromptLeakageTitle(ds) {
metaDescOut = nil
}
catLabel := catNameStr
if catLabel == "" {
catLabel = catStr
}
if (metaTitleOut == nil || metaDescOut == nil) && (titleStr != "" || plainDesc != "" || catLabel != "") {
synthTitle, synthDesc := fillMetaFromResult(StepResult{
Name: titleStr,
ProcessedName: titleStr,
Category: catStr,
CategoryName: catNameStr,
Description: plainDesc,
ProcessedDescription: plainDesc,
Attributes: attrs,
ProcessedAttributes: attrs,
})
if metaTitleOut == nil {
if synthTitle != "" {
metaTitleOut = synthTitle
} else {
metaTitleOut = nullIfEmptyPtr(title)
}
}
if metaDescOut == nil {
if synthDesc != "" {
metaDescOut = synthDesc
} else if plainDesc != "" {
metaDescOut = truncateMetaDescription(plainDesc, v1MetaDescriptionMaxChars)
}
}
} else if metaTitleOut == nil {
metaTitleOut = nullIfEmptyPtr(title)
}
var description any var description any
if plainDesc != "" { if plainDesc != "" {
description = plainDesc description = plainDesc
} else { } else {
description = nil description = nil
} }
var catOut, catNameOut any
eprelVal := extractEPRELFromAttrs(attrs) if catStr != "" {
attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowedAttrs) catOut = catStr
metaTitleOut := nullIfEmptyPtr(metaTitle)
if metaTitleOut == nil {
metaTitleOut = nullIfEmptyPtr(title)
} }
metaDescOut := nullIfEmptyPtr(metaDesc) if catNameStr != "" {
if metaDescOut == nil && plainDesc != "" { catNameOut = catNameStr
metaDescOut = truncateRunes(plainDesc, v1MetaDescriptionMaxChars)
} }
titleOut := nullIfEmptyPtr(title)
item := V1ProcessJobItem{ item := V1ProcessJobItem{
"ean": ean, "ean": ean,
"status": MapV1JobItemStatus(itemStatus, true), "status": MapV1JobItemStatus(itemStatus, true),
"category": nullIfEmptyPtr(category), "category": catOut,
"category_name": nullIfEmptyPtr(categoryName), "category_name": catNameOut,
"title": nullIfEmptyPtr(title), "title": titleOut,
"name": titleOut,
"meta_title": metaTitleOut, "meta_title": metaTitleOut,
"meta_description": metaDescOut, "meta_description": metaDescOut,
"description": description, "description": description,
@@ -343,6 +412,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
if len(more) > 0 { if len(more) > 0 {
item["more_images"] = more item["more_images"] = more
} }
item = EnforceV1ProcessCompletedItem(item, "", allowedAttrs)
full = append(full, item) full = append(full, item)
} }
if err := rows.Err(); err != nil { if err := rows.Err(); err != nil {
@@ -358,6 +428,13 @@ func nullIfEmptyPtr(s *string) any {
return *s return *s
} }
func derefStringPtr(s *string) string {
if s == nil {
return ""
}
return strings.TrimSpace(*s)
}
// loadCompanyAttributeKeySet returns canonicalized attribute_key values for the company. // loadCompanyAttributeKeySet returns canonicalized attribute_key values for the company.
// On query failure returns an empty (non-nil) set so FilterAttributesByAllowed still // On query failure returns an empty (non-nil) set so FilterAttributesByAllowed still
// restricts to coreCharacteristicAttrKeys only. // restricts to coreCharacteristicAttrKeys only.
@@ -389,15 +466,52 @@ func loadCompanyAttributeKeySet(ctx context.Context, p *Pipeline, companyID uuid
return out return out
} }
// v1PlainDescription normalizes feed HTML into a single plain-text string for the // loadCompanyCategoryNameMap returns unique_id → display name for the company.
// legacy process poll (description is a string, not a one-element array). func loadCompanyCategoryNameMap(ctx context.Context, p *Pipeline, companyID uuid.UUID) map[string]string {
out := map[string]string{}
if p == nil || p.Pool == nil {
return out
}
rows, err := p.Pool.Query(ctx, `
SELECT unique_id, name
FROM categories
WHERE company_id = $1 AND COALESCE(unique_id, '') <> ''`, companyID)
if err != nil {
return out
}
defer rows.Close()
for rows.Next() {
var uid, name string
if err := rows.Scan(&uid, &name); err != nil {
continue
}
uid = strings.TrimSpace(uid)
name = strings.TrimSpace(name)
if uid == "" || name == "" {
continue
}
out[uid] = name
}
return out
}
// v1PlainDescription normalizes legacy stored descriptions into a single plain-text
// string for the process poll (description is a string, not a one-element array).
// Handles JSON array/string encodings (e.g. mapped_data->>'description' when the
// feed value was an array) and HTML/<br>/entity markup.
func v1PlainDescription(s string) string { func v1PlainDescription(s string) string {
s = strings.TrimSpace(s)
if s == "" {
return ""
}
s = unwrapLegacyDescriptionStored(s)
s = strings.TrimSpace(s) s = strings.TrimSpace(s)
if s == "" { if s == "" {
return "" return ""
} }
s = html.UnescapeString(s) s = html.UnescapeString(s)
s = strings.ReplaceAll(s, "\u00a0", " ") s = strings.ReplaceAll(s, "\u00a0", " ")
s = strings.ReplaceAll(s, `\u00a0`, " ")
s = v1BreakTagRe.ReplaceAllString(s, "\n") s = v1BreakTagRe.ReplaceAllString(s, "\n")
s = v1BlockEndRe.ReplaceAllString(s, "\n") s = v1BlockEndRe.ReplaceAllString(s, "\n")
s = specsHTMLTagRe.ReplaceAllString(s, " ") s = specsHTMLTagRe.ReplaceAllString(s, " ")
@@ -406,6 +520,10 @@ func v1PlainDescription(s string) string {
kept := make([]string, 0, len(lines)) kept := make([]string, 0, len(lines))
for _, line := range lines { for _, line := range lines {
line = strings.Join(strings.Fields(line), " ") line = strings.Join(strings.Fields(line), " ")
line = strings.ReplaceAll(line, " .", ".")
line = strings.ReplaceAll(line, " ,", ",")
line = strings.ReplaceAll(line, " ;", ";")
line = strings.ReplaceAll(line, " :", ":")
if line != "" { if line != "" {
kept = append(kept, line) kept = append(kept, line)
} }
@@ -413,23 +531,67 @@ func v1PlainDescription(s string) string {
return strings.TrimSpace(strings.Join(kept, "\n")) return strings.TrimSpace(strings.Join(kept, "\n"))
} }
const maxLegacyDescriptionUnwrapDepth = 4
// unwrapLegacyDescriptionStored flattens JSON-encoded legacy description payloads
// (one-element arrays, multi-part arrays, nested arrays, JSON-quoted strings).
// Non-JSON text is returned unchanged.
func unwrapLegacyDescriptionStored(s string) string {
return unwrapLegacyDescriptionStoredN(s, 0)
}
func unwrapLegacyDescriptionStoredN(s string, depth int) string {
s = strings.TrimSpace(s)
if s == "" || depth > maxLegacyDescriptionUnwrapDepth {
return s
}
if !looksLikeLegacyDescriptionJSON(s) {
return s
}
var raw any
if err := json.Unmarshal([]byte(s), &raw); err != nil {
return s
}
flat := flattenLegacyDescriptionAny(raw, depth)
if flat == "" {
if _, ok := raw.([]any); ok {
return ""
}
return s
}
return flat
}
func looksLikeLegacyDescriptionJSON(s string) bool {
if len(s) >= 2 && s[0] == '[' && s[len(s)-1] == ']' {
return true
}
return len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"'
}
func flattenLegacyDescriptionAny(v any, depth int) string {
switch t := v.(type) {
case string:
return unwrapLegacyDescriptionStoredN(t, depth+1)
case []any:
parts := make([]string, 0, len(t))
for _, el := range t {
if p := flattenLegacyDescriptionAny(el, depth+1); p != "" {
parts = append(parts, p)
}
}
return strings.Join(parts, "\n")
case float64:
return strings.TrimSpace(fmt.Sprint(t))
case bool:
return fmt.Sprint(t)
default:
return ""
}
}
func extractEPRELFromAttrs(attrs map[string]any) any { func extractEPRELFromAttrs(attrs map[string]any) any {
if attrs == nil { return eprel.ExtractFromAttrs(attrs)
return nil
}
if e, ok := attrs["eprel"]; ok && e != nil {
return e
}
out := map[string]any{}
for _, k := range []string{"label", "pdf", "energy_class", "energy_scale"} {
if v, ok := attrs["eprel_"+k]; ok && v != nil {
out[k] = v
}
}
if len(out) == 0 {
return nil
}
return out
} }
// ProjectV1ProcessJobItems applies legacy partial-type field projection. // ProjectV1ProcessJobItems applies legacy partial-type field projection.
@@ -471,6 +633,10 @@ func ProjectV1ProcessJobItems(storedType string, items []V1ProcessJobItem) []V1P
projected["category_name"] = item["category_name"] projected["category_name"] = item["category_name"]
case "title": case "title":
projected["title"] = item["title"] projected["title"] = item["title"]
projected["name"] = item["name"]
if projected["name"] == nil {
projected["name"] = item["title"]
}
projected["meta_title"] = item["meta_title"] projected["meta_title"] = item["meta_title"]
case "description": case "description":
projected["description"] = item["description"] projected["description"] = item["description"]
+58 -1
View File
@@ -76,7 +76,7 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", "processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc", "raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"status": "processed", "status": "processed",
"title": "T", "meta_title": "MT", "title": "T", "name": "T", "meta_title": "MT",
"description": "D", "attributes": map[string]any{"brand": "X"}, "description": "D", "attributes": map[string]any{"brand": "X"},
"main_image": "https://example.com/a.jpg", "more_images": []string{"https://example.com/b.jpg"}, "main_image": "https://example.com/a.jpg", "more_images": []string{"https://example.com/b.jpg"},
"eprel": nil, "category": "cat", "category_name": "Cat", "eprel": nil, "category": "cat", "category_name": "Cat",
@@ -91,6 +91,9 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
if got["title"] != "T" || got["ean"] != "123" { if got["title"] != "T" || got["ean"] != "123" {
t.Fatalf("got=%v", got) t.Fatalf("got=%v", got)
} }
if got["name"] != "T" {
t.Fatalf("name dual-mode alias missing or mismatched: %v", got)
}
if _, ok := got["attributes"]; ok { if _, ok := got["attributes"]; ok {
t.Fatalf("attributes should be projected out: %v", got) t.Fatalf("attributes should be projected out: %v", got)
} }
@@ -108,6 +111,23 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
} }
} }
func TestProjectV1ProcessJobItemsTitleDerivesName(t *testing.T) {
items := []V1ProcessJobItem{{
"ean": "123", "id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"status": "processed", "title": "OnlyTitle", "meta_title": "MT",
}}
out := ProjectV1ProcessJobItems("title", items)
if len(out) != 1 {
t.Fatalf("len=%d", len(out))
}
if out[0]["title"] != "OnlyTitle" {
t.Fatalf("title=%v", out[0]["title"])
}
if out[0]["name"] != "OnlyTitle" {
t.Fatalf("name should mirror title when absent: %v", out[0]["name"])
}
}
func TestV1PlainDescriptionStandalone(t *testing.T) { func TestV1PlainDescriptionStandalone(t *testing.T) {
got := v1PlainDescription("Line1<br>Line2 &amp; ok") got := v1PlainDescription("Line1<br>Line2 &amp; ok")
if strings.Contains(got, "<") { if strings.Contains(got, "<") {
@@ -118,6 +138,43 @@ func TestV1PlainDescriptionStandalone(t *testing.T) {
} }
} }
func TestV1PlainDescriptionEdgeCases(t *testing.T) {
nbsp := "\u00a0"
cases := []struct {
name string
in string
want string
}{
{name: "empty", in: "", want: ""},
{name: "whitespace", in: " \n\t ", want: ""},
{name: "plain", in: "Hello world", want: "Hello world"},
{name: "html_br", in: "Hello<br>World<br/>&amp; more</p><b>bold</b>", want: "Hello\nWorld\n& more\nbold"},
{name: "one_element_json_array_html", in: `["Hello<br>World"]`, want: "Hello\nWorld"},
{name: "multi_element_array", in: `["Para one.", "Para <b>two</b>."]`, want: "Para one.\nPara two."},
{name: "nested_array", in: `[["inner<br>x"]]`, want: "inner\nx"},
{name: "json_quoted_string", in: `"Plain &amp; quoted"`, want: "Plain & quoted"},
{name: "empty_array", in: `[]`, want: ""},
{name: "array_with_nulls", in: `[null, "keep", null, ""]`, want: "keep"},
{name: "array_with_object_ignored", in: `[{"text":"x"}, "ok"]`, want: "ok"},
{name: "invalid_bracket_text", in: `[not json`, want: "[not json"},
{name: "nbsp_and_entities", in: "A" + nbsp + "B&nbsp;C", want: "A B C"},
{name: "literal_unicode_escape", in: `A\u00a0B`, want: "A B"},
{name: "double_encoded_array", in: `"[\"Hello<br>World\"]"`, want: "Hello\nWorld"},
{name: "whitespace_around_array", in: " [\" spaced \"] ", want: "spaced"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := v1PlainDescription(tc.in)
if got != tc.want {
t.Fatalf("in=%q got=%q want=%q", tc.in, got, tc.want)
}
if strings.Contains(got, "<") {
t.Fatalf("html left: %q", got)
}
})
}
}
func TestApplyV1ProcessItemIDsDualMode(t *testing.T) { func TestApplyV1ProcessItemIDsDualMode(t *testing.T) {
processed := mustParseTestUUID(t, "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb") processed := mustParseTestUUID(t, "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb")
raw := mustParseTestUUID(t, "cccccccc-cccc-cccc-cccc-cccccccccccc") raw := mustParseTestUUID(t, "cccccccc-cccc-cccc-cccc-cccccccccccc")
@@ -0,0 +1,456 @@
package processing
import (
"fmt"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
)
// V1ProcessItemScorecard scores one COMPLETED legacy process item against
// OpenAPI LegacyProcessItem + A1 poll expectations.
type V1ProcessItemScorecard struct {
EAN string `json:"ean"`
Status string `json:"status"`
OK bool `json:"ok"`
Score int `json:"score"` // 0100
FailFlags []string `json:"fail_flags,omitempty"`
HasTitle bool `json:"has_title"`
HasName bool `json:"has_name"`
HasDescription bool `json:"has_description"`
DescriptionIsPlain bool `json:"description_is_plain"`
HasMetaTitle bool `json:"has_meta_title"`
HasMetaDescription bool `json:"has_meta_description"`
HasCategory bool `json:"has_category"`
HasCategoryName bool `json:"has_category_name"`
AttrsClean bool `json:"attrs_clean"`
EPRELOK bool `json:"eprel_ok"`
HasIDs bool `json:"has_ids"`
ImagesOK bool `json:"images_ok"`
}
// ScoreV1ProcessItemOptions tunes structure checks for a poll item.
type ScoreV1ProcessItemOptions struct {
// MappedCategory, when non-empty, requires item.category to equal it
// (projection must surface mapped unique_id).
MappedCategory string
// Language is used only for documentation of synthesize paths in tests.
Language string
}
// ScoreV1ProcessCompletedItem scores a successful (or terminal) V1 process item.
// For status != processed it only checks ean/status shape and returns OK when those exist.
func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemOptions) V1ProcessItemScorecard {
sc := V1ProcessItemScorecard{
EAN: strings.TrimSpace(fmt.Sprint(item["ean"])),
Status: strings.TrimSpace(fmt.Sprint(item["status"])),
}
if sc.EAN == "" || sc.EAN == "<nil>" {
sc.FailFlags = append(sc.FailFlags, "missing_ean")
}
if sc.Status == "" || sc.Status == "<nil>" {
sc.FailFlags = append(sc.FailFlags, "missing_status")
}
if sc.Status != "processed" {
sc.OK = len(sc.FailFlags) == 0
if sc.OK {
sc.Score = 100
}
return sc
}
title := stringFromItem(item, "title")
name := stringFromItem(item, "name")
sc.HasTitle = title != ""
sc.HasName = name != ""
if sc.HasTitle && !sc.HasName {
sc.FailFlags = append(sc.FailFlags, "missing_name")
}
if sc.HasTitle && sc.HasName && title != name {
sc.FailFlags = append(sc.FailFlags, "title_name_mismatch")
}
desc, descOK := plainDescriptionFromItem(item)
sc.HasDescription = desc != ""
sc.DescriptionIsPlain = descOK
if sc.HasTitle && !sc.HasDescription {
sc.FailFlags = append(sc.FailFlags, "description_empty_with_title")
}
if !descOK && item["description"] != nil {
sc.FailFlags = append(sc.FailFlags, "description_not_plain_string")
}
if sc.HasDescription && descriptionEchoesTitle(desc, title) {
sc.FailFlags = append(sc.FailFlags, "description_echoes_title")
}
if sc.HasDescription && containsWeakFillerPhrase(desc) {
sc.FailFlags = append(sc.FailFlags, "description_weak_filler")
}
sc.HasMetaTitle = stringFromItem(item, "meta_title") != ""
sc.HasMetaDescription = stringFromItem(item, "meta_description") != ""
if sc.HasTitle && !sc.HasMetaTitle {
sc.FailFlags = append(sc.FailFlags, "meta_title_missing")
}
if sc.HasTitle && !sc.HasMetaDescription {
sc.FailFlags = append(sc.FailFlags, "meta_description_missing")
}
if md := stringFromItem(item, "meta_description"); md != "" && containsWeakFillerPhrase(md) {
sc.FailFlags = append(sc.FailFlags, "meta_description_weak_filler")
}
cat := stringFromItem(item, "category")
catName := stringFromItem(item, "category_name")
sc.HasCategory = cat != ""
sc.HasCategoryName = catName != ""
mappedCat := strings.TrimSpace(opts.MappedCategory)
if mappedCat != "" {
if cat == "" {
sc.FailFlags = append(sc.FailFlags, "category_missing_though_mapped")
} else if cat != mappedCat {
sc.FailFlags = append(sc.FailFlags, "category_mismatch_mapped")
}
if cat != "" && catName == "" {
sc.FailFlags = append(sc.FailFlags, "category_name_missing")
}
}
attrs, attrsOK := attrsMapFromItem(item)
dirty := dirtyV1AttrKeys(attrs)
sc.AttrsClean = attrsOK && len(dirty) == 0
if !attrsOK {
sc.FailFlags = append(sc.FailFlags, "attributes_invalid_shape")
}
if len(dirty) > 0 {
sc.FailFlags = append(sc.FailFlags, "attributes_dirty:"+strings.Join(dirty, ","))
}
sc.EPRELOK = eprelShapeOK(item["eprel"])
if !sc.EPRELOK {
sc.FailFlags = append(sc.FailFlags, "eprel_invalid_shape")
}
id := stringFromItem(item, "id")
ppID := stringFromItem(item, "processed_product_id")
rawID := stringFromItem(item, "raw_product_id")
sc.HasIDs = id != "" && ppID != "" && rawID != ""
if id == "" {
sc.FailFlags = append(sc.FailFlags, "missing_id")
}
if ppID == "" {
sc.FailFlags = append(sc.FailFlags, "missing_processed_product_id")
}
if rawID == "" {
sc.FailFlags = append(sc.FailFlags, "missing_raw_product_id")
}
if id != "" && ppID != "" && id != ppID {
sc.FailFlags = append(sc.FailFlags, "id_processed_product_id_mismatch")
}
sc.ImagesOK = imageFieldsOK(item)
if !sc.ImagesOK {
sc.FailFlags = append(sc.FailFlags, "images_invalid_shape")
}
// Weighted score: structure gates, not content quality beyond filler/echo.
checks := []bool{
sc.EAN != "",
sc.Status == "processed",
sc.HasTitle,
sc.HasName || !sc.HasTitle,
sc.HasDescription || !sc.HasTitle,
sc.DescriptionIsPlain || item["description"] == nil,
sc.HasMetaTitle || !sc.HasTitle,
sc.HasMetaDescription || !sc.HasTitle,
mappedCat == "" || sc.HasCategory,
mappedCat == "" || sc.HasCategoryName,
sc.AttrsClean,
sc.EPRELOK,
sc.HasIDs,
sc.ImagesOK,
!containsWeakFillerPhrase(desc),
!descriptionEchoesTitle(desc, title),
}
pass := 0
for _, ok := range checks {
if ok {
pass++
}
}
sc.Score = (pass * 100) / len(checks)
sc.OK = len(sc.FailFlags) == 0
return sc
}
// EnforceV1ProcessCompletedItem fills LegacyProcessItem projection gaps for a
// successful item: plain nonempty description when title exists, meta_*, clean
// attributes, eprel object|null, and image key shapes. Category must already be
// set by the caller when mapped provides a unique_id.
//
// allowed is the company attribute_key set (canonicalized). When nil, only
// coreCharacteristicAttrKeys are kept (never leak feed junk like zavora).
func EnforceV1ProcessCompletedItem(item V1ProcessJobItem, language string, allowed map[string]struct{}) V1ProcessJobItem {
if item == nil {
return item
}
status, _ := item["status"].(string)
if status != "processed" {
return item
}
attrs, _ := attrsMapFromItem(item)
if allowed == nil {
allowed = map[string]struct{}{}
}
attrs = SanitizeV1ProcessAttributesAllowed(attrs, allowed)
if len(attrs) > 0 {
item["attributes"] = attrs
} else {
item["attributes"] = nil
}
title := stringFromItem(item, "title")
if title != "" {
item["title"] = title
item["name"] = title
} else {
item["title"] = nil
item["name"] = nil
}
cat := stringFromItem(item, "category")
catName := stringFromItem(item, "category_name")
catLabel := catName
if catLabel == "" {
catLabel = cat
}
desc, _ := plainDescriptionFromItem(item)
// Empty, weak, or title-echo copy must be replaced — never leave description==title.
if title != "" && (desc == "" || isWeakPriorEnhanceDescription(desc, title) || descriptionEchoesTitle(desc, title)) {
if synth := synthesizeDescriptionFromTitle(title, catLabel, language, attrs); synth != "" {
desc = synth
}
}
if desc != "" {
item["description"] = desc
} else {
item["description"] = nil
}
metaTitle := stringFromItem(item, "meta_title")
metaDesc := stringFromItem(item, "meta_description")
needMetaTitle := metaTitle == "" || isPoisonedMetaTitle(metaTitle)
// Empty, weak stub (Ready for retail…), title-echo, or prompt-leakage — refresh.
// When meta_title is poisoned, also force refresh of weak/leakage meta_description.
needMetaDesc := metaDesc == "" ||
isWeakPriorEnhanceDescription(metaDesc, title) ||
containsWeakFillerPhrase(metaDesc) ||
isPromptLeakageTitle(metaDesc) ||
(title != "" && descriptionEchoesTitle(metaDesc, title))
if needMetaTitle && isPoisonedMetaTitle(metaTitle) &&
(metaDesc == "" || isWeakPriorEnhanceDescription(metaDesc, title) || isPromptLeakageTitle(metaDesc)) {
needMetaDesc = true
}
if title != "" || desc != "" || cat != "" || catName != "" {
synthTitle, synthDesc := fillMetaFromResult(StepResult{
Name: title,
ProcessedName: title,
Category: cat,
CategoryName: catName,
Description: desc,
ProcessedDescription: desc,
Attributes: attrs,
ProcessedAttributes: attrs,
})
if needMetaTitle {
if synthTitle != "" {
metaTitle = synthTitle
} else {
metaTitle = title
}
}
if needMetaDesc {
if synthDesc != "" {
metaDesc = synthDesc
} else if desc != "" {
metaDesc = truncateMetaDescription(desc, v1MetaDescriptionMaxChars)
}
}
}
if metaTitle != "" {
item["meta_title"] = metaTitle
} else {
item["meta_title"] = nil
}
if metaDesc != "" {
item["meta_description"] = metaDesc
} else {
item["meta_description"] = nil
}
if cat != "" {
item["category"] = cat
} else {
item["category"] = nil
}
if catName != "" {
item["category_name"] = catName
} else {
item["category_name"] = nil
}
item["eprel"] = normalizeEPRELValue(item["eprel"])
main := stringFromItem(item, "main_image")
if main != "" {
item["main_image"] = main
} else {
item["main_image"] = nil
}
more := stringSliceFromItem(item, "more_images")
if len(more) > 0 {
item["more_images"] = more
} else {
item["more_images"] = nil
}
return item
}
func stringFromItem(item V1ProcessJobItem, key string) string {
if item == nil {
return ""
}
return strings.TrimSpace(stringFromAny(item[key]))
}
func plainDescriptionFromItem(item V1ProcessJobItem) (string, bool) {
if item == nil {
return "", true
}
raw := item["description"]
if raw == nil {
return "", true
}
switch raw.(type) {
case string:
return PlainDescriptionFromAny(raw), true
case []any, []string:
// Legacy mistake: description as array — convert to plain string.
if s := PlainDescriptionFromAny(raw); s != "" {
return s, true
}
return "", false
case map[string]any:
if s := PlainDescriptionFromAny(raw); s != "" {
return s, true
}
return "", false
default:
s := strings.TrimSpace(fmt.Sprint(raw))
if s == "" || s == "<nil>" {
return "", true
}
return "", false
}
}
func attrsMapFromItem(item V1ProcessJobItem) (map[string]any, bool) {
if item == nil {
return map[string]any{}, true
}
raw := item["attributes"]
if raw == nil {
return map[string]any{}, true
}
switch t := raw.(type) {
case map[string]any:
return t, true
default:
return map[string]any{}, false
}
}
func dirtyV1AttrKeys(attrs map[string]any) []string {
if len(attrs) == 0 {
return nil
}
var dirty []string
for k := range attrs {
key := strings.TrimSpace(k)
lk := strings.ToLower(key)
if key == "" || isReservedProductKey(key) || isInvalidAttributeKey(key) ||
strings.HasPrefix(lk, "eprel") {
dirty = append(dirty, key)
continue
}
// Re-run sanitize: if key disappears, it was dirty.
trial := SanitizeV1ProcessAttributes(map[string]any{key: attrs[k]})
if len(trial) == 0 {
dirty = append(dirty, key)
}
}
return dirty
}
func eprelShapeOK(v any) bool {
if v == nil {
return true
}
_, ok := v.(map[string]any)
return ok
}
func normalizeEPRELValue(v any) any {
return eprel.NormalizeShape(v)
}
func imageFieldsOK(item V1ProcessJobItem) bool {
if item == nil {
return true
}
if v := item["main_image"]; v != nil {
if _, ok := v.(string); !ok {
return false
}
}
if v := item["more_images"]; v != nil {
switch v.(type) {
case []string, []any:
return true
default:
return false
}
}
return true
}
func stringSliceFromItem(item V1ProcessJobItem, key string) []string {
if item == nil {
return nil
}
switch t := item[key].(type) {
case []string:
out := make([]string, 0, len(t))
for _, s := range t {
s = strings.TrimSpace(s)
if s != "" {
out = append(out, s)
}
}
return out
case []any:
out := make([]string, 0, len(t))
for _, e := range t {
if s, ok := e.(string); ok {
s = strings.TrimSpace(s)
if s != "" {
out = append(out, s)
}
}
}
return out
default:
return nil
}
}
@@ -0,0 +1,319 @@
package processing
import (
"fmt"
"strings"
"testing"
)
func TestScoreV1ProcessCompletedItem_processedStructure(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{
"ean": "123",
"status": "processed",
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"title": "Sony Headphones",
"name": "Sony Headphones",
"description": "Sony Headphones deliver clear audio with noise cancelling for daily commuting.",
"meta_title": "Sony | Sony Headphones",
"meta_description": "Sony Headphones deliver clear audio with noise cancelling for daily commuting.",
"category": "48",
"category_name": "Slušalke",
"attributes": map[string]any{"brand": "Sony", "color": "Black"},
"main_image": "https://example.com/a.jpg",
"more_images": []string{"https://example.com/b.jpg"},
"eprel": nil,
}
sc := ScoreV1ProcessCompletedItem(item, ScoreV1ProcessItemOptions{MappedCategory: "48"})
if !sc.OK {
t.Fatalf("expected OK, flags=%v score=%d", sc.FailFlags, sc.Score)
}
if sc.Score < 90 {
t.Fatalf("score=%d want >= 90", sc.Score)
}
}
func TestScoreV1ProcessCompletedItem_flagsGaps(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{
"ean": "999",
"status": "processed",
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"title": "Mount",
"description": nil,
"meta_title": nil,
"meta_description": nil,
"category": nil,
"category_name": nil,
"attributes": map[string]any{
"brand": "X",
"eprel_id": "1",
":": "true",
},
"eprel": "not-an-object",
}
sc := ScoreV1ProcessCompletedItem(item, ScoreV1ProcessItemOptions{MappedCategory: "28"})
if sc.OK {
t.Fatal("expected failures")
}
joined := strings.Join(sc.FailFlags, ",")
for _, want := range []string{
"missing_name",
"description_empty_with_title",
"meta_title_missing",
"meta_description_missing",
"category_missing_though_mapped",
"eprel_invalid_shape",
} {
if !strings.Contains(joined, want) {
t.Fatalf("missing flag %q in %v", want, sc.FailFlags)
}
}
}
func TestEnforceV1ProcessCompletedItem_fillsDescriptionMetaSanitizes(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{
"ean": "1",
"status": "processed",
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"title": "Fiksni stenski TV nosilec 32''-55'' MANHATTAN",
"description": "Fiksni stenski TV nosilec 32''-55'' MANHATTAN from Ostalo. Ready for retail listing.",
"meta_title": nil,
"meta_description": nil,
"category": "28",
"category_name": "Nosilci za TV",
"attributes": map[string]any{
"brand": "Ostalo",
"product_model": "460934",
"eprel_label": "A",
"name": "should-drop",
":": "true",
},
"eprel": nil,
}
out := EnforceV1ProcessCompletedItem(item, "sl", nil)
desc, _ := out["description"].(string)
if desc == "" {
t.Fatal("expected nonempty description")
}
if descriptionEchoesTitle(desc, fmt.Sprint(out["title"])) {
t.Fatalf("EnforceV1 must replace weak/echo desc, got title-echo: %q", desc)
}
if containsWeakFillerPhrase(desc) {
t.Fatalf("filler left in description: %q", desc)
}
if strings.TrimSpace(fmt.Sprint(out["meta_title"])) == "" || out["meta_title"] == nil {
t.Fatalf("meta_title empty: %v", out["meta_title"])
}
if strings.TrimSpace(fmt.Sprint(out["meta_description"])) == "" || out["meta_description"] == nil {
t.Fatalf("meta_description empty: %v", out["meta_description"])
}
if containsWeakFillerPhrase(fmt.Sprint(out["meta_description"])) {
t.Fatalf("meta_description still filler: %v", out["meta_description"])
}
attrs, ok := out["attributes"].(map[string]any)
if !ok {
t.Fatalf("attributes type=%T", out["attributes"])
}
if _, bad := attrs["eprel_label"]; bad {
t.Fatalf("eprel_* should be stripped: %v", attrs)
}
if _, bad := attrs[":"]; bad {
t.Fatalf("invalid key kept: %v", attrs)
}
if _, bad := attrs["name"]; bad {
t.Fatalf("reserved name kept: %v", attrs)
}
if out["name"] != out["title"] {
t.Fatalf("name should mirror title: name=%v title=%v", out["name"], out["title"])
}
sc := ScoreV1ProcessCompletedItem(out, ScoreV1ProcessItemOptions{MappedCategory: "28"})
if !sc.OK {
t.Fatalf("enforced item should score OK, flags=%v", sc.FailFlags)
}
}
func TestEnforceV1ProcessCompletedItem_skipsTerminalStatuses(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{"ean": "1", "status": "not_found", "error": "missing"}
out := EnforceV1ProcessCompletedItem(item, "", nil)
if _, ok := out["title"]; ok {
t.Fatalf("should not invent fields for not_found: %v", out)
}
}
func TestEnforceV1ProcessCompletedItem_categoryFromCallerPreserved(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{
"ean": "2",
"status": "processed",
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"title": "Cooker",
"description": "",
"category": "50",
"category_name": "Štedilniki",
"attributes": map[string]any{"brand": "Vox"},
"eprel": nil,
}
out := EnforceV1ProcessCompletedItem(item, "", nil)
if out["category"] != "50" {
t.Fatalf("category=%v", out["category"])
}
if out["category_name"] != "Štedilniki" {
t.Fatalf("category_name=%v", out["category_name"])
}
desc, _ := out["description"].(string)
if desc == "" {
t.Fatal("expected synthesized description when title set")
}
if descriptionEchoesTitle(desc, "Cooker") {
t.Fatalf("synthesized description must not equal title: %q", desc)
}
sc := ScoreV1ProcessCompletedItem(out, ScoreV1ProcessItemOptions{MappedCategory: "50"})
if !sc.OK {
t.Fatalf("flags=%v", sc.FailFlags)
}
}
func TestEnforceV1ProcessCompletedItem_replacesTitleEchoDescription(t *testing.T) {
t.Parallel()
title := "VOX EBR700"
item := V1ProcessJobItem{
"ean": "8806095210711",
"status": "processed",
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"title": title,
"description": title,
"category": "50",
"category_name": "Štedilniki",
"attributes": map[string]any{"brand": "VOX"},
"eprel": nil,
}
out := EnforceV1ProcessCompletedItem(item, "sl", nil)
desc, _ := out["description"].(string)
if desc == "" {
t.Fatal("expected synthesized description")
}
if descriptionEchoesTitle(desc, title) {
t.Fatalf("must not leave title-echo description: %q", desc)
}
if !strings.Contains(desc, "Štedilniki") && !strings.Contains(desc, "VOX") {
t.Fatalf("expected factual synth with category/brand: %q", desc)
}
}
func TestEnforceV1ProcessCompletedItem_refreshesPoisonedMetaTitle(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{
"ean": "8806095210711",
"status": "processed",
"id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
"title": "VOX EBR700",
"description": "VOX EBR700 is a Štedilniki product from VOX.",
"meta_title": "VOX EBR700 | 50",
"meta_description": "Ready for retail listing.",
"category": "50",
"category_name": "Štedilniki",
"attributes": map[string]any{
"brand": "VOX",
"zavora": "junk",
},
"eprel": nil,
}
out := EnforceV1ProcessCompletedItem(item, "sl", nil)
mt := fmt.Sprint(out["meta_title"])
if isPoisonedMetaTitle(mt) || strings.Contains(mt, "| 50") {
t.Fatalf("poisoned meta_title preserved: %q", mt)
}
if !strings.Contains(mt, "Štedilniki") {
t.Fatalf("expected CategoryName in meta_title: %q", mt)
}
md := fmt.Sprint(out["meta_description"])
if md == "" || md == "<nil>" || out["meta_description"] == nil {
t.Fatalf("meta_description empty after poisoned title refresh: %v", out["meta_description"])
}
if containsWeakFillerPhrase(md) || isWeakPriorEnhanceDescription(md, "VOX EBR700") {
t.Fatalf("stub meta_description preserved with poisoned title: %q", md)
}
attrs, ok := out["attributes"].(map[string]any)
if !ok {
t.Fatalf("attributes type=%T", out["attributes"])
}
if _, bad := attrs["zavora"]; bad {
t.Fatalf("feed junk zavora leaked: %v", attrs)
}
if _, ok := attrs["brand"]; !ok {
t.Fatalf("brand should remain: %v", attrs)
}
}
func TestIsPoisonedMetaTitle(t *testing.T) {
t.Parallel()
if !isPoisonedMetaTitle("VOX EBR700 | 50") {
t.Fatal("expected poisoned")
}
if !isPoisonedMetaTitle("Name | 7") {
t.Fatal("expected poisoned with extra spaces")
}
if isPoisonedMetaTitle("VOX | Štedilniki") {
t.Fatal("display name suffix must not count as poisoned")
}
if isPoisonedMetaTitle("") {
t.Fatal("empty not poisoned")
}
if !isPoisonedMetaTitle("Sony | short retail title; follow any Title formula constr…") {
t.Fatal("expected prompt-leakage meta_title poisoned")
}
if !isPoisonedMetaTitle("ANKER | short retail title; follow any Title formula constraints that follow") {
t.Fatal("expected Title formula leakage poisoned")
}
}
func TestEnforceV1ProcessCompletedItem_refreshesPromptLeakageMeta(t *testing.T) {
t.Parallel()
item := V1ProcessJobItem{
"ean": "4548736132597",
"status": "processed",
"id": "dddddddd-dddd-dddd-dddd-dddddddddddd",
"processed_product_id": "dddddddd-dddd-dddd-dddd-dddddddddddd",
"raw_product_id": "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee",
"title": "SONY BT slušalke WH1000XM5S sive",
"description": "SONY BT slušalke WH1000XM5S sive je izdelek v kategoriji Slušalke znamke Sony.",
"meta_title": "Sony | short retail title; follow any Title formula constr…",
"meta_description": "prefer 1-3 factual paragraphs as ONE string; limited HTML () is allowed if needed",
"category": "48",
"category_name": "Slušalke",
"attributes": map[string]any{
"brand": "Sony",
},
"eprel": nil,
}
out := EnforceV1ProcessCompletedItem(item, "sl", nil)
mt := fmt.Sprint(out["meta_title"])
if isPoisonedMetaTitle(mt) || strings.Contains(strings.ToLower(mt), "short retail title") {
t.Fatalf("prompt-leakage meta_title preserved: %q", mt)
}
if !strings.Contains(mt, "Slušalke") && !strings.Contains(mt, "SONY") {
t.Fatalf("expected synth meta_title with product/category: %q", mt)
}
md := fmt.Sprint(out["meta_description"])
if md == "" || md == "<nil>" || out["meta_description"] == nil {
t.Fatalf("meta_description empty after leakage refresh: %v", out["meta_description"])
}
if isPromptLeakageTitle(md) || strings.Contains(strings.ToLower(md), "prefer 1-3") {
t.Fatalf("prompt-leakage meta_description preserved: %q", md)
}
}
@@ -0,0 +1,116 @@
package processing
import (
"strings"
"testing"
)
func TestIsWeakPriorEnhanceDescription_fillerPhrases(t *testing.T) {
title := "Vogel's WALL 3245 TV Wall Mount"
cases := []struct {
desc string
want bool
}{
{"Ready for retail listing with extra padding text here.", true},
{title + " in the Mounts category. Ready for retail listing.", true},
{"Product description placeholder text that is long enough to pass length.", true},
{"A product in the Mounts category based on available specifications.", true},
{"Product with available specifications ready for retail listing.", true},
{"Generic one-liner without product tokens and enough length!!!!!", true},
{
"Vogel's WALL 3245 is a TV Mounts product from Vogel's. Key specs: width 45 cm, max_load 40 kg.",
false,
},
{
"This durable retail mount includes mounting hardware, load ratings, and install guidance for wall displays.",
false,
},
}
for _, tc := range cases {
got := isWeakPriorEnhanceDescription(tc.desc, title)
if got != tc.want {
t.Fatalf("desc=%q got weak=%v want %v", tc.desc, got, tc.want)
}
}
// Long non-filler copy that does not share tokens with this title is still usable
// once past the short-boilerplate window (hash-skip quality floor).
longUnrelated := strings.Repeat("Detailed retail copy covering materials finishes warranty and care. ", 3)
if isWeakPriorEnhanceDescription(longUnrelated, title) {
t.Fatalf("long non-filler unrelated copy should not be weak solely for token miss: %q", longUnrelated)
}
if !isWeakPriorEnhanceDescription("ok", title) {
t.Fatal("too-short must be weak")
}
if !isWeakPriorEnhanceDescription(title, title) {
t.Fatal("title-echo must be weak")
}
}
func TestSynthesizeDescriptionFromTitle_brandModelCategory(t *testing.T) {
title := "Vogel's WALL 3245 TV Wall Mount"
attrs := map[string]any{
"brand": "Vogel's",
"product_model": "WALL 3245",
"width": "45 cm",
"max_load": "40 kg",
}
en := synthesizeDescriptionFromTitle(title, "TV Mounts", "en", attrs)
if en == "" {
t.Fatal("expected nonempty English invent")
}
if strings.Contains(strings.ToLower(en), "ready for retail listing") {
t.Fatalf("must not emit retail filler: %q", en)
}
for _, need := range []string{"Vogel", "TV Mounts", "45 cm", "40 kg"} {
if !strings.Contains(en, need) {
t.Fatalf("English invent missing %q: %q", need, en)
}
}
if isWeakPriorEnhanceDescription(en, title) {
t.Fatalf("factual invent must not be weak: %q", en)
}
sl := synthesizeDescriptionFromTitle(title, "TV Mounts", "sl", attrs)
if sl == "" || !strings.Contains(sl, "kategoriji") {
t.Fatalf("expected Slovenian invent, got %q", sl)
}
if strings.Contains(strings.ToLower(sl), "ready for retail listing") {
t.Fatalf("SL invent must not emit EN filler: %q", sl)
}
if isWeakPriorEnhanceDescription(sl, title) {
t.Fatalf("SL factual invent must not be weak: %q", sl)
}
}
func TestInventHeuristicDescription_fromBrandModelCategory(t *testing.T) {
title := "Vogel's WALL 3245 TV Wall Mount"
user := ProductEnhanceUser("TV Mounts", title, "", map[string]any{
"brand": "Vogel's",
"product_model": "WALL 3245",
"width": "45 cm",
"max_load": "40 kg",
})
system := "Write name and description in English. Return JSON with \"name\" and \"description\"."
got := inventHeuristicDescription(system, user, title)
if got == "" {
t.Fatal("expected invent output")
}
t.Logf("TV mount invent sample: %s", got)
if strings.Contains(strings.ToLower(got), "ready for retail listing") {
t.Fatalf("invent must not return retail filler: %q", got)
}
for _, need := range []string{"Vogel", "TV Mounts", "45 cm"} {
if !strings.Contains(got, need) {
t.Fatalf("invent missing %q: %q", need, got)
}
}
if isWeakPriorEnhanceDescription(got, title) {
t.Fatalf("invent output must not be classified weak: %q", got)
}
slSystem := "Write name and description in Slovenian. Return JSON with \"name\"."
sl := inventHeuristicDescription(slSystem, user, title)
if !strings.Contains(sl, "kategoriji") {
t.Fatalf("expected SL invent from system language, got %q", sl)
}
}
+6 -1
View File
@@ -26,9 +26,14 @@ func SafeHTTPClientPolicy(timeout time.Duration, policy DialPolicy) *http.Client
if timeout <= 0 { if timeout <= 0 {
timeout = 30 * time.Second timeout = 30 * time.Second
} }
tr := SafeHTTPTransportPolicy(policy)
// Match header wait to overall timeout. A fixed 30s ResponseHeaderTimeout
// aborts OpenAI-compatible LLM calls that withhold headers until generation
// finishes (reasoning models often take 60180s).
tr.ResponseHeaderTimeout = timeout
return &http.Client{ return &http.Client{
Timeout: timeout, Timeout: timeout,
Transport: SafeHTTPTransportPolicy(policy), Transport: tr,
} }
} }
+55 -1
View File
@@ -1,5 +1,7 @@
/** /**
* Admin orgs UI client users + companies directory, staff roles, plan assign. * Admin orgs UI client users + companies directory, staff roles, plan assign,
* clone-catalog, and Fix A1 (`POST …/fix-catalog`).
* Flash Fix A1: result.prompts / hashes / categories flash.admin.fixA1Success.
* Contract: docs/admin-roles-support/04-contract.md · Docs: 10-admin-orgs-ui.md * Contract: docs/admin-roles-support/04-contract.md · Docs: 10-admin-orgs-ui.md
*/ */
import { api, ApiError } from "$lib/api"; import { api, ApiError } from "$lib/api";
@@ -18,6 +20,8 @@ export const ADMIN_USERS_PATH = "/api/admin/users";
export const ADMIN_COMPANIES_PATH = "/api/admin/companies"; export const ADMIN_COMPANIES_PATH = "/api/admin/companies";
export const ADMIN_CLONE_CATALOG_PATH = (companyId: string) => export const ADMIN_CLONE_CATALOG_PATH = (companyId: string) =>
`${ADMIN_COMPANIES_PATH}/${encodeURIComponent(companyId)}/clone-catalog`; `${ADMIN_COMPANIES_PATH}/${encodeURIComponent(companyId)}/clone-catalog`;
export const ADMIN_FIX_CATALOG_PATH = (companyId: string) =>
`${ADMIN_COMPANIES_PATH}/${encodeURIComponent(companyId)}/fix-catalog`;
export const ADMIN_STAFF_ROLE_PATH = (userId: string) => export const ADMIN_STAFF_ROLE_PATH = (userId: string) =>
`/api/admin/users/${encodeURIComponent(userId)}/staff-role`; `/api/admin/users/${encodeURIComponent(userId)}/staff-role`;
@@ -226,6 +230,56 @@ export async function cloneAdminCompanyCatalog(
}); });
} }
export type FixCatalogResult = {
company_id: string;
company_name: string;
a1_cohort?: boolean;
category_attribute_orphans_removed?: number;
category_attribute_links?: number;
/** Alias of category_prompts_updated for flash.admin.fixA1Success {prompts}. */
prompts?: number;
category_prompts_updated?: number;
product_enhance_languages?: number;
/** Alias of weak_hashes_cleared for flash {hashes}. */
hashes?: number;
weak_hashes_cleared?: number;
/** Alias of categories_backfilled for flash {categories}. */
categories?: number;
categories_backfilled?: number;
attributes_sanitized?: number;
products_scanned?: number;
reprocess_needed_count?: number;
reprocess_sample_raw_product_ids?: string[];
};
export type FixCatalogResponse = {
status: string;
result: FixCatalogResult;
note?: string;
};
/** In-place Fix A1 / catalog hygiene. Never clears catalog; no mass reprocess. */
export async function fixAdminCompanyCatalog(
companyId: string,
opts?: { backfillCategories?: boolean; reprocessSampleLimit?: number }
): Promise<FixCatalogResponse> {
const body: {
confirm: true;
backfill_categories?: boolean;
reprocess_sample_limit?: number;
} = { confirm: true };
if (opts?.backfillCategories === false) {
body.backfill_categories = false;
}
if (typeof opts?.reprocessSampleLimit === "number") {
body.reprocess_sample_limit = opts.reprocessSampleLimit;
}
return api<FixCatalogResponse>(ADMIN_FIX_CATALOG_PATH(companyId), {
method: "POST",
body
});
}
export function isStaffRoleApiUnavailable(err: unknown): boolean { export function isStaffRoleApiUnavailable(err: unknown): boolean {
return err instanceof ApiError && (err.status === 404 || err.status === 501); return err instanceof ApiError && (err.status === 404 || err.status === 501);
} }
@@ -33,6 +33,7 @@
productFeedSyncLabel, productFeedSyncLabel,
productHasEprel, productHasEprel,
productHasSpecs, productHasSpecs,
productImageUrls,
resolveOriginalDescription, resolveOriginalDescription,
resolveOriginalName resolveOriginalName
} from "./types"; } from "./types";
@@ -438,6 +439,7 @@
const coverageCount = $derived( const coverageCount = $derived(
enrichment ? Object.values(enrichment).filter((s) => s !== "missing").length : 0 enrichment ? Object.values(enrichment).filter((s) => s !== "missing").length : 0
); );
const imageUrls = $derived.by(() => productImageUrls(product));
function attrDisplayValue(key: string, value: string): string { function attrDisplayValue(key: string, value: string): string {
if (key.trim().toLowerCase() !== "category") return value; if (key.trim().toLowerCase() !== "category") return value;
@@ -571,8 +573,9 @@
primary={contentLanguages[0] || DEFAULT_CONTENT_LANGUAGE} primary={contentLanguages[0] || DEFAULT_CONTENT_LANGUAGE}
allowAdd={false} allowAdd={false}
hasOverride={(code) => { hasOverride={(code) => {
const f = fieldsForLang(code); const primary = contentLanguages[0] || DEFAULT_CONTENT_LANGUAGE;
return Boolean(f.processedName.trim() || f.processedDescription.trim()); if (code === primary) return false;
return Boolean(product?.localized_content?.[code]);
}} }}
onChange={(code) => applyContentLang(code)} onChange={(code) => applyContentLang(code)}
/> />
@@ -891,6 +894,34 @@
</div> </div>
</div> </div>
</section> </section>
{#if imageUrls.length > 0}
<section class="space-y-3" data-testid="product-images">
<h3 class="text-sm font-medium">{i18n.t("products.edit.images")}</h3>
<div class="flex flex-wrap gap-3">
{#each imageUrls as src, i}
<a
href={src}
target="_blank"
rel="noopener noreferrer"
class="block overflow-hidden rounded-md border border-border bg-muted/30"
title={src}
>
<img
src={src}
alt={i18n.t("products.edit.imageAlt", {
index: i + 1,
name: product?.processed_name || product?.name || productId
})}
class="h-28 w-28 object-contain"
loading="lazy"
referrerpolicy="no-referrer"
/>
</a>
{/each}
</div>
</section>
{/if}
</TabsContent> </TabsContent>
<TabsContent value="feed" class="mt-4 space-y-4"> <TabsContent value="feed" class="mt-4 space-y-4">
@@ -957,6 +988,19 @@
>{field.value}</pre> >{field.value}</pre>
{/if} {/if}
{:else if /^https?:\/\//i.test(field.value)} {:else if /^https?:\/\//i.test(field.value)}
{@const isImage =
/image|img|photo|picture|thumbnail/i.test(field.key) ||
/\.(jpe?g|png|gif|webp|svg|avif)(\?|$)/i.test(field.value)}
<div class="space-y-2">
{#if isImage}
<img
src={field.value}
alt={field.key}
class="max-h-40 max-w-full rounded border border-border object-contain"
loading="lazy"
referrerpolicy="no-referrer"
/>
{/if}
<a <a
href={field.value} href={field.value}
target="_blank" target="_blank"
@@ -965,6 +1009,7 @@
> >
{field.value} {field.value}
</a> </a>
</div>
{:else} {:else}
<span class="break-words {field.ok ? '' : 'text-chart-amber'}">{field.value}</span> <span class="break-words {field.ok ? '' : 'text-chart-amber'}">{field.value}</span>
{/if} {/if}
@@ -847,6 +847,39 @@ function digAttrBag(
return undefined; return undefined;
} }
/** Collect http(s) image URLs from mapped_data (main_image + more_images). */
export function productImageUrls(product: ProductRow | null | undefined): string[] {
const mapped = asRecord(product?.mapped_data);
if (!mapped) return [];
const out: string[] = [];
const seen = new Set<string>();
const push = (raw: unknown) => {
if (typeof raw === "string") {
const url = raw.trim();
if (!/^https?:\/\//i.test(url) || seen.has(url)) return;
seen.add(url);
out.push(url);
return;
}
if (Array.isArray(raw)) {
for (const item of raw) push(item);
return;
}
const rec = asRecord(raw);
if (!rec) return;
for (const key of ["url", "src", "href", "link"]) {
if (rec[key] != null) push(rec[key]);
}
};
for (const key of ["main_image", "mainImage", "image", "image_url", "imageUrl"]) {
if (mapped[key] != null) push(mapped[key]);
}
for (const key of ["more_images", "moreImages", "images", "additional_images"]) {
if (mapped[key] != null) push(mapped[key]);
}
return out;
}
/** True when product carries an EPREL id or enriched energy-label payload. */ /** True when product carries an EPREL id or enriched energy-label payload. */
export function productHasEprel(product: ProductRow | null | undefined): boolean { export function productHasEprel(product: ProductRow | null | undefined): boolean {
if (!product) return false; if (!product) return false;
+51
View File
@@ -0,0 +1,51 @@
/**
* Focused check: admin Fix A1 i18n keys exist in every UI_LOCALES pack.
*/
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import { UI_LOCALES } from "./locales.ts";
import { en } from "./messages/en.ts";
import { loadAllMessages, messagesFor } from "./messages/catalog.ts";
import { sl } from "./messages/sl.ts";
const FIX_A1_KEYS = [
"admin.users.fixA1",
"admin.users.fixA1Aria",
"admin.users.fixA1Title",
"admin.users.fixA1Desc",
"admin.users.fixA1Company",
"admin.users.fixA1Warning",
"admin.users.fixA1Cancel",
"admin.users.fixA1Confirm",
"flash.admin.fixA1Success",
"flash.admin.fixA1Error"
] as const;
describe("admin Fix A1 i18n keys", () => {
it("English defines every Fix A1 key", () => {
for (const key of FIX_A1_KEYS) {
assert.equal(typeof en[key], "string", key);
assert.ok(en[key].trim().length > 0, key);
}
});
it("every registered UI locale has Fix A1 keys", async () => {
await loadAllMessages();
for (const { code } of UI_LOCALES) {
const pack = code === "en" ? en : messagesFor(code);
for (const key of FIX_A1_KEYS) {
const value = pack[key];
assert.equal(typeof value, "string", `${code}:${key}`);
assert.ok(String(value).trim().length > 0, `${code}:${key}`);
}
}
});
it("Slovenian ready pack has Fix A1 keys (not in UI_LOCALES)", () => {
for (const key of FIX_A1_KEYS) {
assert.equal(typeof sl[key], "string", key);
assert.ok(sl[key].trim().length > 0, key);
}
});
});
+10
View File
@@ -812,6 +812,14 @@ export const de: MessageDict = {
"admin.users.cloneCatalogCancel": "Abbrechen", "admin.users.cloneCatalogCancel": "Abbrechen",
"admin.users.cloneCatalogConfirm": "Katalog kopieren", "admin.users.cloneCatalogConfirm": "Katalog kopieren",
"admin.users.cloneDestFallback": "Ihr Sandbox-Unternehmen", "admin.users.cloneDestFallback": "Ihr Sandbox-Unternehmen",
"admin.users.fixA1": "A1-Katalog reparieren",
"admin.users.fixA1Aria": "KI-Prompts, Kategorien und Enhance-Hashes für {name} reparieren",
"admin.users.fixA1Title": "Unternehmenskatalog reparieren",
"admin.users.fixA1Desc": "Wendet korrigierte KI-Prompts erneut an, löst schwache Enhance-Hashes und füllt Kategorien aus mapped_data nach.",
"admin.users.fixA1Company": "Unternehmen: {name}",
"admin.users.fixA1Warning": "Bevorzugen Sie Platform Demo oder ein explizites Unternehmen. Löscht keine Feeds, Mappings oder Rohprodukte. Schützt A1-Kohorten-Überschreibregeln.",
"admin.users.fixA1Cancel": "Abbrechen",
"admin.users.fixA1Confirm": "Katalog reparieren",
"admin.users.noUsers": "Keine Benutzer entsprechen diesem Filter.", "admin.users.noUsers": "Keine Benutzer entsprechen diesem Filter.",
"admin.users.noCompanies": "Keine Unternehmen entsprechen diesem Filter.", "admin.users.noCompanies": "Keine Unternehmen entsprechen diesem Filter.",
"admin.users.assignRoleTitle": "Mitarbeiterrolle zuweisen", "admin.users.assignRoleTitle": "Mitarbeiterrolle zuweisen",
@@ -2297,6 +2305,8 @@ export const de: MessageDict = {
"flash.admin.staffRoleUnavailable": "Mitarbeiterrollen-Updates sind auf diesem Server noch nicht verfügbar.", "flash.admin.staffRoleUnavailable": "Mitarbeiterrollen-Updates sind auf diesem Server noch nicht verfügbar.",
"flash.admin.planAssigned": "Plan {name} zugewiesen.", "flash.admin.planAssigned": "Plan {name} zugewiesen.",
"flash.admin.catalogCloned": "{source} nach {dest} kopiert: {products} Produkte, {categories} Kategorien.", "flash.admin.catalogCloned": "{source} nach {dest} kopiert: {products} Produkte, {categories} Kategorien.",
"flash.admin.fixA1Success": "Katalogreparatur für {name} abgeschlossen: {prompts} Prompts aktualisiert, {hashes} Hashes gelöscht, {categories} Kategorien nachgefüllt.",
"flash.admin.fixA1Error": "Katalogreparatur für {name} fehlgeschlagen.",
"flash.admin.planAssignedShort": "Plan zugewiesen.", "flash.admin.planAssignedShort": "Plan zugewiesen.",
"flash.admin.creditsUpdated": "Credits aktualisiert.", "flash.admin.creditsUpdated": "Credits aktualisiert.",
"flash.admin.cyclesProcessed": "Abrechnungszyklen verarbeitet: {count}", "flash.admin.cyclesProcessed": "Abrechnungszyklen verarbeitet: {count}",
+16
View File
@@ -355,6 +355,10 @@ export const en: MessageDict = {
"dashboard.lowCreditsTitle": "Credits running low", "dashboard.lowCreditsTitle": "Credits running low",
"dashboard.lowCreditsMessage": "{remaining} of {total} credits left. Top up or upgrade before jobs stall.", "dashboard.lowCreditsMessage": "{remaining} of {total} credits left. Top up or upgrade before jobs stall.",
"dashboard.latestJobs": "Latest processing jobs", "dashboard.latestJobs": "Latest processing jobs",
"dashboard.jobsRunning": "{count} job running",
"dashboard.jobsRunningPlural": "{count} jobs running",
"dashboard.activeJobs": "{count} active job",
"dashboard.activeJobsPlural": "{count} active jobs",
"dashboard.noJobsEmpty": "No jobs yet — import products first.", "dashboard.noJobsEmpty": "No jobs yet — import products first.",
"dashboard.noJobsReady": "No recent jobs. Start one from Products when you are ready.", "dashboard.noJobsReady": "No recent jobs. Start one from Products when you are ready.",
"dashboard.startJob": "Start a job", "dashboard.startJob": "Start a job",
@@ -835,6 +839,14 @@ export const en: MessageDict = {
"admin.users.cloneCatalogCancel": "Cancel", "admin.users.cloneCatalogCancel": "Cancel",
"admin.users.cloneCatalogConfirm": "Copy catalog", "admin.users.cloneCatalogConfirm": "Copy catalog",
"admin.users.cloneDestFallback": "your sandbox company", "admin.users.cloneDestFallback": "your sandbox company",
"admin.users.fixA1": "Fix A1 catalog",
"admin.users.fixA1Aria": "Repair AI prompts, categories, and enhance hashes for {name}",
"admin.users.fixA1Title": "Fix company catalog",
"admin.users.fixA1Desc": "In-place repair: category attribute links, category enhance prompts (with attrs), weak enhance hashes, optional category backfill, attribute sanitize. No mass reprocess.",
"admin.users.fixA1Company": "Company: {name}",
"admin.users.fixA1Warning": "Prefer Platform Demo; A1 may be targeted in place with confirm. Never clears the catalog or uses A1 as a clone destination.",
"admin.users.fixA1Cancel": "Cancel",
"admin.users.fixA1Confirm": "Fix catalog",
"admin.users.noUsers": "No users match this filter.", "admin.users.noUsers": "No users match this filter.",
"admin.users.noCompanies": "No companies match this filter.", "admin.users.noCompanies": "No companies match this filter.",
"admin.users.assignRoleTitle": "Assign staff role", "admin.users.assignRoleTitle": "Assign staff role",
@@ -2322,6 +2334,8 @@ export const en: MessageDict = {
"flash.admin.staffRoleUnavailable": "Staff role updates are not available on this server yet.", "flash.admin.staffRoleUnavailable": "Staff role updates are not available on this server yet.",
"flash.admin.planAssigned": "Plan assigned to {name}.", "flash.admin.planAssigned": "Plan assigned to {name}.",
"flash.admin.catalogCloned": "Copied {source} into {dest}: {products} products, {categories} categories. Open Products and process catalog GTINs (EAN-only API calls need a matching feed product).", "flash.admin.catalogCloned": "Copied {source} into {dest}: {products} products, {categories} categories. Open Products and process catalog GTINs (EAN-only API calls need a matching feed product).",
"flash.admin.fixA1Success": "Catalog repair finished for {name}: {prompts} category prompts updated, {hashes} hashes cleared, {categories} categories backfilled.",
"flash.admin.fixA1Error": "Catalog repair failed for {name}.",
"flash.admin.planAssignedShort": "Plan assigned.", "flash.admin.planAssignedShort": "Plan assigned.",
"flash.admin.creditsUpdated": "Credits updated.", "flash.admin.creditsUpdated": "Credits updated.",
"flash.admin.cyclesProcessed": "Billing cycles processed: {count}", "flash.admin.cyclesProcessed": "Billing cycles processed: {count}",
@@ -4238,6 +4252,8 @@ export const en: MessageDict = {
"products.edit.updatedRelative": "· updated {when}", "products.edit.updatedRelative": "· updated {when}",
"products.edit.viewAllFeedFields": "View all feed fields ({filled}/{total})", "products.edit.viewAllFeedFields": "View all feed fields ({filled}/{total})",
"products.edit.productIdentity": "Product identity", "products.edit.productIdentity": "Product identity",
"products.edit.images": "Images",
"products.edit.imageAlt": "Product image {index}: {name}",
"products.edit.aiProcessedName": "AI-processed name", "products.edit.aiProcessedName": "AI-processed name",
"products.edit.originalName": "Original name", "products.edit.originalName": "Original name",
"products.edit.productIdSku": "Product ID / SKU", "products.edit.productIdSku": "Product ID / SKU",
+10
View File
@@ -812,6 +812,14 @@ export const es: MessageDict = {
"admin.users.cloneCatalogCancel": "Cancelar", "admin.users.cloneCatalogCancel": "Cancelar",
"admin.users.cloneCatalogConfirm": "Copiar catálogo", "admin.users.cloneCatalogConfirm": "Copiar catálogo",
"admin.users.cloneDestFallback": "tu empresa sandbox", "admin.users.cloneDestFallback": "tu empresa sandbox",
"admin.users.fixA1": "Reparar catálogo A1",
"admin.users.fixA1Aria": "Reparar prompts de IA, categorías y hashes de enhance para {name}",
"admin.users.fixA1Title": "Reparar catálogo de la empresa",
"admin.users.fixA1Desc": "Vuelve a aplicar prompts de IA corregidos, limpia hashes de enhance débiles y completa categorías desde mapped_data.",
"admin.users.fixA1Company": "Empresa: {name}",
"admin.users.fixA1Warning": "Prefiera Platform Demo o una empresa explícita. No elimina feeds, mapeos ni productos en bruto. Protege las reglas de sobrescritura de la cohorte A1.",
"admin.users.fixA1Cancel": "Cancelar",
"admin.users.fixA1Confirm": "Reparar catálogo",
"admin.users.noUsers": "Ningún usuario coincide con este filtro.", "admin.users.noUsers": "Ningún usuario coincide con este filtro.",
"admin.users.noCompanies": "Ninguna empresa coincide con este filtro.", "admin.users.noCompanies": "Ninguna empresa coincide con este filtro.",
"admin.users.assignRoleTitle": "Asignar rol de personal", "admin.users.assignRoleTitle": "Asignar rol de personal",
@@ -2296,6 +2304,8 @@ export const es: MessageDict = {
"flash.admin.staffRoleUpdated": "Rol de personal actualizado para {email}.", "flash.admin.staffRoleUpdated": "Rol de personal actualizado para {email}.",
"flash.admin.staffRoleUnavailable": "Las actualizaciones de rol de personal aún no están disponibles en este servidor.", "flash.admin.staffRoleUnavailable": "Las actualizaciones de rol de personal aún no están disponibles en este servidor.",
"flash.admin.catalogCloned": "Se copió {source} en {dest}: {products} productos, {categories} categorías.", "flash.admin.catalogCloned": "Se copió {source} en {dest}: {products} productos, {categories} categorías.",
"flash.admin.fixA1Success": "Reparación del catálogo de {name} finalizada: {prompts} prompts actualizados, {hashes} hashes borrados, {categories} categorías rellenadas.",
"flash.admin.fixA1Error": "Falló la reparación del catálogo de {name}.",
"flash.admin.planAssigned": "Plan asignado a {name}.", "flash.admin.planAssigned": "Plan asignado a {name}.",
"flash.admin.planAssignedShort": "Plan asignado.", "flash.admin.planAssignedShort": "Plan asignado.",
"flash.admin.creditsUpdated": "Créditos actualizados.", "flash.admin.creditsUpdated": "Créditos actualizados.",
+10
View File
@@ -812,6 +812,14 @@ export const fr: MessageDict = {
"admin.users.cloneCatalogCancel": "Annuler", "admin.users.cloneCatalogCancel": "Annuler",
"admin.users.cloneCatalogConfirm": "Copier le catalogue", "admin.users.cloneCatalogConfirm": "Copier le catalogue",
"admin.users.cloneDestFallback": "votre entreprise sandbox", "admin.users.cloneDestFallback": "votre entreprise sandbox",
"admin.users.fixA1": "Réparer le catalogue A1",
"admin.users.fixA1Aria": "Réparer les prompts IA, catégories et hashes enhance pour {name}",
"admin.users.fixA1Title": "Réparer le catalogue entreprise",
"admin.users.fixA1Desc": "Réapplique les prompts IA corrigés, efface les hashes enhance faibles et complète les catégories depuis mapped_data.",
"admin.users.fixA1Company": "Entreprise : {name}",
"admin.users.fixA1Warning": "Préférer Platform Demo ou une entreprise explicite. Ne supprime ni feeds, ni mappings, ni produits bruts. Protège les règles d'écrasement de la cohorte A1.",
"admin.users.fixA1Cancel": "Annuler",
"admin.users.fixA1Confirm": "Réparer le catalogue",
"admin.users.noUsers": "Aucun utilisateur ne correspond à ce filtre.", "admin.users.noUsers": "Aucun utilisateur ne correspond à ce filtre.",
"admin.users.noCompanies": "Aucune entreprise ne correspond à ce filtre.", "admin.users.noCompanies": "Aucune entreprise ne correspond à ce filtre.",
"admin.users.assignRoleTitle": "Assigner un rôle du personnel", "admin.users.assignRoleTitle": "Assigner un rôle du personnel",
@@ -2296,6 +2304,8 @@ export const fr: MessageDict = {
"flash.admin.staffRoleUpdated": "Rôle du personnel mis à jour pour {email}.", "flash.admin.staffRoleUpdated": "Rôle du personnel mis à jour pour {email}.",
"flash.admin.staffRoleUnavailable": "Les mises à jour de rôle personnel ne sont pas encore disponibles sur ce serveur.", "flash.admin.staffRoleUnavailable": "Les mises à jour de rôle personnel ne sont pas encore disponibles sur ce serveur.",
"flash.admin.catalogCloned": "{source} copié vers {dest} : {products} produits, {categories} catégories.", "flash.admin.catalogCloned": "{source} copié vers {dest} : {products} produits, {categories} catégories.",
"flash.admin.fixA1Success": "Réparation du catalogue pour {name} terminée : {prompts} prompts mis à jour, {hashes} hashes effacés, {categories} catégories complétées.",
"flash.admin.fixA1Error": "Échec de la réparation du catalogue pour {name}.",
"flash.admin.planAssigned": "Offre assignée à {name}.", "flash.admin.planAssigned": "Offre assignée à {name}.",
"flash.admin.planAssignedShort": "Offre assignée.", "flash.admin.planAssignedShort": "Offre assignée.",
"flash.admin.creditsUpdated": "Crédits mis à jour.", "flash.admin.creditsUpdated": "Crédits mis à jour.",
+10
View File
@@ -812,6 +812,14 @@ export const it: MessageDict = {
"admin.users.cloneCatalogCancel": "Annulla", "admin.users.cloneCatalogCancel": "Annulla",
"admin.users.cloneCatalogConfirm": "Copia catalogo", "admin.users.cloneCatalogConfirm": "Copia catalogo",
"admin.users.cloneDestFallback": "la tua azienda sandbox", "admin.users.cloneDestFallback": "la tua azienda sandbox",
"admin.users.fixA1": "Ripara catalogo A1",
"admin.users.fixA1Aria": "Ripara prompt IA, categorie e hash enhance per {name}",
"admin.users.fixA1Title": "Ripara catalogo azienda",
"admin.users.fixA1Desc": "Riapplica i prompt IA corretti, cancella hash enhance deboli e completa le categorie da mapped_data.",
"admin.users.fixA1Company": "Azienda: {name}",
"admin.users.fixA1Warning": "Preferisci Platform Demo o un'azienda esplicita. Non elimina feed, mapping o prodotti grezzi. Protegge le regole di sovrascrittura della coorte A1.",
"admin.users.fixA1Cancel": "Annulla",
"admin.users.fixA1Confirm": "Ripara catalogo",
"admin.users.noUsers": "Nessun utente corrisponde a questo filtro.", "admin.users.noUsers": "Nessun utente corrisponde a questo filtro.",
"admin.users.noCompanies": "Nessuna azienda corrisponde a questo filtro.", "admin.users.noCompanies": "Nessuna azienda corrisponde a questo filtro.",
"admin.users.assignRoleTitle": "Assegna ruolo staff", "admin.users.assignRoleTitle": "Assegna ruolo staff",
@@ -2296,6 +2304,8 @@ export const it: MessageDict = {
"flash.admin.staffRoleUpdated": "Ruolo staff aggiornato per {email}.", "flash.admin.staffRoleUpdated": "Ruolo staff aggiornato per {email}.",
"flash.admin.staffRoleUnavailable": "Gli aggiornamenti del ruolo staff non sono ancora disponibili su questo server.", "flash.admin.staffRoleUnavailable": "Gli aggiornamenti del ruolo staff non sono ancora disponibili su questo server.",
"flash.admin.catalogCloned": "Copiato {source} in {dest}: {products} prodotti, {categories} categorie.", "flash.admin.catalogCloned": "Copiato {source} in {dest}: {products} prodotti, {categories} categorie.",
"flash.admin.fixA1Success": "Riparazione catalogo per {name} completata: {prompts} prompt aggiornati, {hashes} hash cancellati, {categories} categorie integrate.",
"flash.admin.fixA1Error": "Riparazione catalogo per {name} non riuscita.",
"flash.admin.planAssigned": "Piano assegnato a {name}.", "flash.admin.planAssigned": "Piano assegnato a {name}.",
"flash.admin.planAssignedShort": "Piano assegnato.", "flash.admin.planAssignedShort": "Piano assegnato.",
"flash.admin.creditsUpdated": "Crediti aggiornati.", "flash.admin.creditsUpdated": "Crediti aggiornati.",
+10
View File
@@ -812,6 +812,14 @@ export const ja: MessageDict = {
"admin.users.cloneCatalogCancel": "キャンセル", "admin.users.cloneCatalogCancel": "キャンセル",
"admin.users.cloneCatalogConfirm": "カタログをコピー", "admin.users.cloneCatalogConfirm": "カタログをコピー",
"admin.users.cloneDestFallback": "サンドボックス会社", "admin.users.cloneDestFallback": "サンドボックス会社",
"admin.users.fixA1": "A1カタログを修復",
"admin.users.fixA1Aria": "{name} のAIプロンプト、カテゴリ、enhanceハッシュを修復",
"admin.users.fixA1Title": "会社カタログを修復",
"admin.users.fixA1Desc": "修正済みAIプロンプトを再適用し、弱いenhanceハッシュを消去し、mapped_dataからカテゴリを補完します。",
"admin.users.fixA1Company": "会社: {name}",
"admin.users.fixA1Warning": "Platform Demoまたは明示的な会社を優先してください。フィード、マッピング、生製品は削除しません。A1コホートの上書きルールを保護します。",
"admin.users.fixA1Cancel": "キャンセル",
"admin.users.fixA1Confirm": "カタログを修復",
"admin.users.noUsers": "このフィルタに一致するユーザーはいません。", "admin.users.noUsers": "このフィルタに一致するユーザーはいません。",
"admin.users.noCompanies": "このフィルタに一致する会社はありません。", "admin.users.noCompanies": "このフィルタに一致する会社はありません。",
"admin.users.assignRoleTitle": "スタッフロールを割り当て", "admin.users.assignRoleTitle": "スタッフロールを割り当て",
@@ -2296,6 +2304,8 @@ export const ja: MessageDict = {
"flash.admin.staffRoleUpdated": "{email} のスタッフロールを更新しました。", "flash.admin.staffRoleUpdated": "{email} のスタッフロールを更新しました。",
"flash.admin.staffRoleUnavailable": "このサーバーではスタッフロールの更新はまだ利用できません。", "flash.admin.staffRoleUnavailable": "このサーバーではスタッフロールの更新はまだ利用できません。",
"flash.admin.catalogCloned": "{source} を {dest} にコピーしました: 商品 {products}、カテゴリ {categories}。", "flash.admin.catalogCloned": "{source} を {dest} にコピーしました: 商品 {products}、カテゴリ {categories}。",
"flash.admin.fixA1Success": "{name} のカタログ修復が完了: プロンプト {prompts} 件更新、ハッシュ {hashes} 件消去、カテゴリ {categories} 件補完。",
"flash.admin.fixA1Error": "{name} のカタログ修復に失敗しました。",
"flash.admin.planAssigned": "{name} にプランを割り当てました。", "flash.admin.planAssigned": "{name} にプランを割り当てました。",
"flash.admin.planAssignedShort": "プランを割り当てました。", "flash.admin.planAssignedShort": "プランを割り当てました。",
"flash.admin.creditsUpdated": "クレジットを更新しました。", "flash.admin.creditsUpdated": "クレジットを更新しました。",

Some files were not shown because too many files have changed in this diff Show More