fix
This commit is contained in:
@@ -78,7 +78,7 @@ var BuiltInDefaults = []DefaultTemplate{
|
||||
{
|
||||
Key: KeyProductEnhance,
|
||||
Label: "Product title & description",
|
||||
Description: "Used when processing products (AI enhance step).",
|
||||
Description: "Used when processing products (AI enhance step). Categorize (taxonomy pick when category is empty) is a separate pipeline step before this.",
|
||||
SystemTemplate: `Retail product copywriter.
|
||||
Rules:
|
||||
- Reply with ONLY JSON (no markdown)
|
||||
|
||||
@@ -9,19 +9,23 @@ import (
|
||||
|
||||
// Pipeline step names (canonical order for "full").
|
||||
const (
|
||||
StepNormalize = "normalize"
|
||||
StepParseSpecs = "parse_specs"
|
||||
StepFillFields = "fill_fields"
|
||||
StepEPREL = "eprel"
|
||||
StepAIEnhance = "ai_enhance"
|
||||
StepNormalize = "normalize"
|
||||
StepParseSpecs = "parse_specs"
|
||||
StepFillFields = "fill_fields"
|
||||
StepEPREL = "eprel"
|
||||
StepCategorize = "categorize"
|
||||
StepAIEnhance = "ai_enhance"
|
||||
)
|
||||
|
||||
// CanonicalSteps is the default full pipeline order.
|
||||
// categorize runs before ai_enhance so category formulas / overlays key correctly
|
||||
// (legacy Descrybe: GPT picks a taxonomy unique_id when mapped category is empty).
|
||||
var CanonicalSteps = []string{
|
||||
StepNormalize,
|
||||
StepParseSpecs,
|
||||
StepFillFields,
|
||||
StepEPREL,
|
||||
StepCategorize,
|
||||
StepAIEnhance,
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
package processing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// MaxCategorizeOptions caps the taxonomy list injected into the categorize LLM
|
||||
// prompt (token/cost bound). Prefer stable sort by display name then unique_id.
|
||||
const MaxCategorizeOptions = 200
|
||||
|
||||
// MaxTokensCategorize budgets a short JSON reply {"categoryId","confidence"}.
|
||||
// code-fast / reasoning-style models often burn internal tokens before JSON;
|
||||
// 256 frequently finishes with empty content and forces a length-cap retry.
|
||||
const MaxTokensCategorize = 4096
|
||||
|
||||
// categoryOption is one company taxonomy row for the categorize prompt.
|
||||
type categoryOption struct {
|
||||
UniqueID string
|
||||
Name string
|
||||
}
|
||||
|
||||
// categoryOptionsFromNames builds a stable, capped list from unique_id → name.
|
||||
// Empty map → no AI categorize (nothing valid to choose).
|
||||
func categoryOptionsFromNames(namesByUID map[string]string) []categoryOption {
|
||||
if len(namesByUID) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]categoryOption, 0, len(namesByUID))
|
||||
for uid, name := range namesByUID {
|
||||
uid = strings.TrimSpace(uid)
|
||||
if uid == "" || strings.EqualFold(uid, "none") {
|
||||
continue
|
||||
}
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
name = uid
|
||||
}
|
||||
out = append(out, categoryOption{UniqueID: uid, Name: name})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Name != out[j].Name {
|
||||
return strings.ToLower(out[i].Name) < strings.ToLower(out[j].Name)
|
||||
}
|
||||
return out[i].UniqueID < out[j].UniqueID
|
||||
})
|
||||
if len(out) > MaxCategorizeOptions {
|
||||
out = out[:MaxCategorizeOptions]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// categoryUniqueIDsList returns sorted unique_ids for vector SuggestCategory candidates.
|
||||
func categoryUniqueIDsList(namesByUID map[string]string) []string {
|
||||
opts := categoryOptionsFromNames(namesByUID)
|
||||
if len(opts) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, len(opts))
|
||||
for i, o := range opts {
|
||||
out[i] = o.UniqueID
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// formatAvailableCategoriesList mirrors legacy Descrybe "Available Categories:"
|
||||
// bullets: "Name (ID: unique_id)".
|
||||
func formatAvailableCategoriesList(opts []categoryOption) string {
|
||||
if len(opts) == 0 {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, o := range opts {
|
||||
b.WriteString("- ")
|
||||
b.WriteString(SanitizeText(o.Name))
|
||||
b.WriteString(" (ID: ")
|
||||
b.WriteString(SanitizeText(o.UniqueID))
|
||||
b.WriteString(")\n")
|
||||
}
|
||||
return strings.TrimSpace(b.String())
|
||||
}
|
||||
|
||||
// ProductCategorizeSystem is the built-in system prompt for taxonomy selection.
|
||||
const ProductCategorizeSystem = `Product categorization expert.
|
||||
Rules:
|
||||
- Reply with ONLY a single JSON object (no markdown, no prose, no reasoning)
|
||||
- Schema: {"categoryId":"string","confidence":0.0}
|
||||
- categoryId MUST be one of the Available Categories IDs exactly (the value in parentheses after ID:)
|
||||
- Never invent IDs or names; inventing IDs fails categorization
|
||||
- Prefer the most specific category that matches the product
|
||||
Example:
|
||||
{"categoryId":"50","confidence":0.9}`
|
||||
|
||||
// ProductCategorizeUser builds the user prompt with product context + taxonomy list.
|
||||
func ProductCategorizeUser(name, description string, opts []categoryOption) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("Select the best category for this product from Available Categories only.\n\n")
|
||||
b.WriteString("Product Information:\n")
|
||||
b.WriteString("Name: ")
|
||||
b.WriteString(SanitizeText(truncateRunes(name, 200)))
|
||||
b.WriteString("\nDesc: ")
|
||||
b.WriteString(SanitizeText(truncateRunes(description, MaxProductDescRunes)))
|
||||
b.WriteString("\n\nAvailable Categories:\n")
|
||||
b.WriteString(formatAvailableCategoriesList(opts))
|
||||
b.WriteString("\n\nImportant:\n")
|
||||
b.WriteString("- Return categoryId as the exact ID from the list\n")
|
||||
b.WriteString("- Do not invent categories outside the list\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// tryAICategorize asks the Completer to pick a company taxonomy unique_id when
|
||||
// Category is still empty after mapped/prior/vector. Never invents outside taxonomy:
|
||||
// responses are coerced then filtered to namesByUID / valid unique_ids.
|
||||
func tryAICategorize(ctx context.Context, e *Engine, out *StepResult, in ProductInput, policy StepPolicy) {
|
||||
if out == nil || strings.TrimSpace(out.Category) != "" {
|
||||
return
|
||||
}
|
||||
if !policy.AllowAI {
|
||||
return
|
||||
}
|
||||
if e == nil || !e.CompleterEnabled() {
|
||||
return
|
||||
}
|
||||
opts := categoryOptionsFromNames(in.CategoryNamesByUID)
|
||||
if len(opts) == 0 {
|
||||
out.Notes = append(out.Notes, "ai_categorize: skipped (no company taxonomy)")
|
||||
appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
|
||||
"status": "skipped",
|
||||
"reason": "no_taxonomy",
|
||||
})
|
||||
return
|
||||
}
|
||||
name := strings.TrimSpace(out.Name)
|
||||
if name == "" {
|
||||
name = strings.TrimSpace(out.ProcessedName)
|
||||
}
|
||||
desc := strings.TrimSpace(out.Description)
|
||||
if desc == "" {
|
||||
desc = strings.TrimSpace(out.ProcessedDescription)
|
||||
}
|
||||
if name == "" && desc == "" {
|
||||
out.Notes = append(out.Notes, "ai_categorize: skipped (empty product text)")
|
||||
appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
|
||||
"status": "skipped",
|
||||
"reason": "empty_product",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
system := ProductCategorizeSystem
|
||||
user := ProductCategorizeUser(name, desc, opts)
|
||||
comp, obj, err := CompleteJSON(ctx, e.Completer, system, user, CompleteOptions{
|
||||
MaxTokens: MaxTokensCategorize,
|
||||
Temperature: DefaultStructuredTemp,
|
||||
})
|
||||
out.TotalTokens += comp.TotalTokens
|
||||
if err != nil {
|
||||
out.Notes = append(out.Notes, "ai_categorize: "+TruncateError(err))
|
||||
appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
|
||||
"status": "failed",
|
||||
"error": TruncateError(err),
|
||||
"tokens": comp.TotalTokens,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
raw := categoryIDFromCategorizeJSON(obj)
|
||||
valid := make(map[string]struct{}, len(opts))
|
||||
for _, o := range opts {
|
||||
valid[o.UniqueID] = struct{}{}
|
||||
}
|
||||
resolved := resolveCompanyCategoryUniqueID(raw, in.CategoryNamesByUID, valid)
|
||||
if resolved == "" || isUnusableCategoryValue(resolved, out.ProcessedName, out.Name) {
|
||||
out.Notes = append(out.Notes, "ai_categorize: ignored (not in company taxonomy)")
|
||||
appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
|
||||
"status": "rejected",
|
||||
"returned": raw,
|
||||
"tokens": comp.TotalTokens,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
out.Category = SanitizeText(resolved)
|
||||
if out.FieldSources == nil {
|
||||
out.FieldSources = map[string]any{}
|
||||
}
|
||||
out.FieldSources["category"] = "llm"
|
||||
syncCategoryName(out, in.CategoryNamesByUID)
|
||||
out.Notes = append(out.Notes, "category: llm")
|
||||
log.Printf("processing: category choice uid=%s name=%s source=llm", out.Category, out.CategoryName)
|
||||
appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
|
||||
"status": "ok",
|
||||
"category": out.Category,
|
||||
"name": out.CategoryName,
|
||||
"source": "llm",
|
||||
"tokens": comp.TotalTokens,
|
||||
"options": len(opts),
|
||||
})
|
||||
}
|
||||
|
||||
func categoryIDFromCategorizeJSON(obj map[string]any) string {
|
||||
if obj == nil {
|
||||
return ""
|
||||
}
|
||||
for _, k := range []string{"categoryId", "category_id", "categoryUniqueId", "category_unique_id", "unique_id", "id", "category"} {
|
||||
if s := categoryUniqueIDFromAny(obj[k]); s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// firstAvailableCategoryIDFromPrompt extracts the first "(ID: …)" token from a
|
||||
// categorize user prompt (HeuristicCompleter / weak-model fallback).
|
||||
func firstAvailableCategoryIDFromPrompt(user string) string {
|
||||
const marker = "(id:"
|
||||
lower := strings.ToLower(user)
|
||||
at := strings.Index(lower, marker)
|
||||
if at < 0 {
|
||||
return ""
|
||||
}
|
||||
rest := strings.TrimSpace(user[at+len(marker):])
|
||||
end := strings.Index(rest, ")")
|
||||
if end <= 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(rest[:end])
|
||||
}
|
||||
|
||||
// runCategorizeStep applies vector then LLM taxonomy selection when Category empty.
|
||||
func runCategorizeStep(ctx context.Context, e *Engine, companyID string, out *StepResult, in ProductInput, categoryNames []string, policy StepPolicy) {
|
||||
if out == nil {
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(out.Category) != "" {
|
||||
appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
|
||||
"status": "skipped",
|
||||
"reason": "already_set",
|
||||
"category": out.Category,
|
||||
})
|
||||
return
|
||||
}
|
||||
if !policy.AllowAI {
|
||||
out.Notes = append(out.Notes, "categorize: skipped (AI not allowed)")
|
||||
appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
|
||||
"status": "skipped",
|
||||
"reason": "entitlement_can_use_ai",
|
||||
})
|
||||
return
|
||||
}
|
||||
tryVectorCategorize(ctx, e, companyID, out, categoryNames, policy)
|
||||
if strings.TrimSpace(out.Category) != "" {
|
||||
return
|
||||
}
|
||||
tryAICategorize(ctx, e, out, in, policy)
|
||||
if strings.TrimSpace(out.Category) == "" {
|
||||
appendStepLog(out.GPTResponse, StepCategorize, map[string]any{
|
||||
"status": "unset",
|
||||
"reason": "no_vector_or_ai_match",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// persistMappedCategorySQL writes a taxonomy unique_id onto mapped_data.category
|
||||
// when the mapped value is still empty (so Products UI + reprocess stick).
|
||||
const persistMappedCategorySQL = `
|
||||
UPDATE raw_products
|
||||
SET mapped_data = jsonb_set(
|
||||
COALESCE(mapped_data, '{}'::jsonb),
|
||||
'{category}',
|
||||
to_jsonb($3::text),
|
||||
true
|
||||
),
|
||||
updated_at = now()
|
||||
WHERE id = $1 AND company_id = $2
|
||||
AND COALESCE(NULLIF(trim(mapped_data->>'category'), ''), '') = ''
|
||||
AND COALESCE(NULLIF(trim($3), ''), '') <> ''
|
||||
AND lower(trim($3)) <> 'none'`
|
||||
@@ -0,0 +1,174 @@
|
||||
package processing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTryAICategorize_picksTaxonomyUniqueID(t *testing.T) {
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
if strings.Contains(strings.ToLower(system), "categoryid") {
|
||||
if !strings.Contains(user, "Available Categories:") {
|
||||
t.Fatalf("expected Available Categories in user prompt")
|
||||
}
|
||||
if !strings.Contains(user, "(ID: 50)") {
|
||||
t.Fatalf("expected ID 50 in list: %s", user)
|
||||
}
|
||||
return Completion{
|
||||
Text: `{"categoryId":"50","confidence":0.91}`,
|
||||
TotalTokens: 12,
|
||||
PromptTokens: 8,
|
||||
OutputTokens: 4,
|
||||
}, nil
|
||||
}
|
||||
return Completion{Text: `{"name":"Gorenje Cooker","description":"Freestanding cooker for the kitchen."}`, TotalTokens: 5}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
Name: "Gorenje stove",
|
||||
Description: "Freestanding cooker",
|
||||
CategoryNamesByUID: map[string]string{
|
||||
"28": "TV",
|
||||
"50": "Štedilniki",
|
||||
},
|
||||
}, "full", nil, StepPolicy{AllowAI: true, AllowEPREL: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Category != "50" {
|
||||
t.Fatalf("Category=%q want 50", out.Category)
|
||||
}
|
||||
if out.CategoryName != "Štedilniki" {
|
||||
t.Fatalf("CategoryName=%q want Štedilniki", out.CategoryName)
|
||||
}
|
||||
if src, _ := out.FieldSources["category"].(string); src != "llm" {
|
||||
t.Fatalf("field_sources.category=%v want llm", out.FieldSources["category"])
|
||||
}
|
||||
if out.TotalTokens < 12 {
|
||||
t.Fatalf("TotalTokens=%d want >=12 from categorize", out.TotalTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTryAICategorize_rejectsInventedID(t *testing.T) {
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
if strings.Contains(strings.ToLower(system), "categoryid") {
|
||||
return Completion{Text: `{"categoryId":"99999","confidence":0.9}`, TotalTokens: 3}, nil
|
||||
}
|
||||
return Completion{Text: `{"name":"X","description":"Y"}`, TotalTokens: 5}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
Name: "Widget",
|
||||
CategoryNamesByUID: map[string]string{
|
||||
"50": "Štedilniki",
|
||||
},
|
||||
}, "full", nil, StepPolicy{AllowAI: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Category != "" {
|
||||
t.Fatalf("Category=%q want empty (invented id rejected)", out.Category)
|
||||
}
|
||||
found := false
|
||||
for _, n := range out.Notes {
|
||||
if strings.Contains(n, "ai_categorize: ignored") {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("expected reject note, got %v", out.Notes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTryAICategorize_resolvesDisplayName(t *testing.T) {
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
if strings.Contains(strings.ToLower(system), "categoryid") {
|
||||
return Completion{Text: `{"categoryId":"Štedilniki","confidence":0.8}`, TotalTokens: 3}, nil
|
||||
}
|
||||
return Completion{Text: `{"name":"X","description":"Y"}`, TotalTokens: 5}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
Name: "Cooker",
|
||||
CategoryNamesByUID: map[string]string{
|
||||
"50": "Štedilniki",
|
||||
},
|
||||
}, "full", nil, StepPolicy{AllowAI: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Category != "50" {
|
||||
t.Fatalf("Category=%q want 50 (name coerced)", out.Category)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTryAICategorize_skippedWithoutTaxonomy(t *testing.T) {
|
||||
calls := 0
|
||||
e := &Engine{
|
||||
Completer: stubCompleter{fn: func(system, user string) (Completion, error) {
|
||||
calls++
|
||||
return Completion{Text: `{"name":"A","description":"B"}`, TotalTokens: 2}, nil
|
||||
}},
|
||||
Vector: NoopVectorCategorizer{},
|
||||
}
|
||||
out, err := e.RunSteps(context.Background(), "co", ProductInput{
|
||||
Name: "Lone product",
|
||||
}, "full", nil, StepPolicy{AllowAI: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Category != "" {
|
||||
t.Fatalf("Category=%q want empty", out.Category)
|
||||
}
|
||||
// Only enhance should call Completer (categorize skips with no taxonomy).
|
||||
if calls != 1 {
|
||||
t.Fatalf("completer calls=%d want 1 (enhance only)", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeuristicCompleter_categorizeReturnsJSON(t *testing.T) {
|
||||
h := HeuristicCompleter{}
|
||||
user := ProductCategorizeUser("Stove", "Cooker", []categoryOption{
|
||||
{UniqueID: "50", Name: "Štedilniki"},
|
||||
{UniqueID: "28", Name: "TV"},
|
||||
})
|
||||
comp, err := h.Complete(context.Background(), ProductCategorizeSystem, user)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, err := ParseJSONObject(comp.Text)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := categoryIDFromCategorizeJSON(obj)
|
||||
if got != "50" && got != "28" {
|
||||
t.Fatalf("categoryId=%q want 50 or 28 from Available Categories", got)
|
||||
}
|
||||
opts := categoryOptionsFromNames(map[string]string{"50": "Štedilniki", "28": "TV"})
|
||||
found := false
|
||||
for _, o := range opts {
|
||||
if o.UniqueID == got {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("categoryId=%q not in options", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCategoryOptionsFromNames_sortAndCap(t *testing.T) {
|
||||
names := map[string]string{"2": "Beta", "1": "Alpha", "3": "Gamma"}
|
||||
opts := categoryOptionsFromNames(names)
|
||||
if len(opts) != 3 || opts[0].UniqueID != "1" || opts[1].UniqueID != "2" {
|
||||
t.Fatalf("opts=%v", opts)
|
||||
}
|
||||
}
|
||||
@@ -644,6 +644,14 @@ func (h HeuristicCompleter) Complete(_ context.Context, system, user string) (Co
|
||||
user = SanitizeText(user)
|
||||
text := "General"
|
||||
switch {
|
||||
case strings.Contains(systemL, "categoryid") || (strings.Contains(systemL, "categor") && strings.Contains(systemL, "available categories")):
|
||||
// Taxonomy pick: return first Available Categories (ID: …) from the user prompt.
|
||||
id := firstAvailableCategoryIDFromPrompt(user)
|
||||
if id == "" {
|
||||
id = "unknown"
|
||||
}
|
||||
b, _ := json.Marshal(map[string]any{"categoryId": id, "confidence": 0.5})
|
||||
text = string(b)
|
||||
case strings.Contains(systemL, `"name"`) || strings.Contains(systemL, "titles and descriptions"):
|
||||
// Prefer explicit Name:/Desc: (ProductEnhanceUser) or Current name: labels.
|
||||
// Never use firstLine(user) alone — that line is often "Category: …".
|
||||
@@ -659,8 +667,6 @@ func (h HeuristicCompleter) Complete(_ context.Context, system, user string) (Co
|
||||
text = string(b)
|
||||
case strings.Contains(systemL, "attributes") && strings.Contains(systemL, "json"):
|
||||
text = `{"material":"unknown","brand":"unknown"}`
|
||||
case strings.Contains(systemL, "categor"):
|
||||
text = "General"
|
||||
default:
|
||||
// Never echo ProductEnhanceUser's first line ("Category: …") as title/output.
|
||||
text = labeledPromptValue(user, "name:", "current name:")
|
||||
|
||||
@@ -1564,7 +1564,8 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
|
||||
engine = &Engine{Vector: NoopVectorCategorizer{}}
|
||||
}
|
||||
policy := cache.stepPolicy()
|
||||
result, err := engine.RunSteps(ctx, companyID.String(), in, processingType, nil, policy)
|
||||
catNames := categoryUniqueIDsList(categoryNamesByUID)
|
||||
result, err := engine.RunSteps(ctx, companyID.String(), in, processingType, catNames, policy)
|
||||
if err != nil {
|
||||
return false, 0, StepResult{}, err
|
||||
}
|
||||
@@ -1644,6 +1645,13 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
|
||||
WHERE id = $1 AND company_id = $2`, it.RawID, companyID); err != nil {
|
||||
return false, 0, result, err
|
||||
}
|
||||
// Stick taxonomy unique_id onto mapped_data.category when still empty so the
|
||||
// Products UI (reads mapped) and reprocess keep the AI/vector/mapped pick.
|
||||
if cat := strings.TrimSpace(result.Category); cat != "" {
|
||||
if _, err := tx.Exec(ctx, persistMappedCategorySQL, it.RawID, companyID, cat); err != nil {
|
||||
return false, 0, result, fmt.Errorf("persist mapped category: %w", err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, 0, result, err
|
||||
}
|
||||
|
||||
@@ -207,6 +207,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
out.FieldSources["eprel"] = "eprel_api"
|
||||
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "ok", "eprel_id": data.ID})
|
||||
|
||||
case StepCategorize:
|
||||
// Vector (if enabled) then LLM taxonomy pick when category still empty.
|
||||
runCategorizeStep(ctx, e, companyID, &out, in, categoryNames, policy)
|
||||
|
||||
case StepAIEnhance:
|
||||
preservePriorEnhanceHash := func() {
|
||||
if in.PriorEnhanceHash != "" {
|
||||
@@ -501,11 +505,14 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
}
|
||||
}
|
||||
|
||||
// Paths without fill_fields (enhance_only / normalize_only) still get vector
|
||||
// categorize when AllowAI + embeddings are available.
|
||||
tryVectorCategorize(ctx, e, companyID, &out, categoryNames, policy)
|
||||
// Pipelines without StepCategorize (enhance_only / normalize_only) still try
|
||||
// vector categorize when AllowAI + embeddings are available. Full/categorize
|
||||
// already ran runCategorizeStep (vector then LLM) inside the loop.
|
||||
if !stepsContain(steps, StepCategorize) {
|
||||
tryVectorCategorize(ctx, e, companyID, &out, categoryNames, policy)
|
||||
noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled())
|
||||
}
|
||||
preserveCategoryIfEmpty(&out, in.PriorCategory)
|
||||
noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled())
|
||||
syncCategoryName(&out, in.CategoryNamesByUID)
|
||||
out.Name = preferredProductTitle(in.GTIN, out.Name, out.ProcessedName, in.Name, in.PriorProcessedName)
|
||||
out.ProcessedName = preferredProductTitle(in.GTIN, out.ProcessedName, out.Name, in.PriorProcessedName, in.Name)
|
||||
@@ -640,12 +647,14 @@ func tryVectorCategorize(ctx context.Context, e *Engine, companyID string, out *
|
||||
}
|
||||
|
||||
// noteMissingCategory records why Category stayed empty (mapped absent; vector skipped or failed).
|
||||
// Used for pipelines that skip StepCategorize (vector-only post-pass).
|
||||
func noteMissingCategory(out *StepResult, policy StepPolicy, vectorEnabled bool) {
|
||||
if out == nil || strings.TrimSpace(out.Category) != "" {
|
||||
return
|
||||
}
|
||||
for _, n := range out.Notes {
|
||||
if strings.HasPrefix(n, "category: unset") || strings.HasPrefix(n, "category: vector") {
|
||||
if strings.HasPrefix(n, "category: unset") || strings.HasPrefix(n, "category: vector") ||
|
||||
strings.HasPrefix(n, "category: llm") || strings.HasPrefix(n, "ai_categorize:") {
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -670,14 +679,25 @@ func resolveSteps(processingType string) []string {
|
||||
return []string{StepNormalize, StepParseSpecs, StepEPREL}
|
||||
case "normalize_only":
|
||||
return []string{StepNormalize}
|
||||
case "categorize", "categorize_only", "categorize_enhance":
|
||||
// Legacy aliases → full deterministic + optional AI
|
||||
case "categorize", "categorize_only":
|
||||
// Taxonomy assign only (vector then LLM) — no title/description rewrite.
|
||||
return []string{StepNormalize, StepParseSpecs, StepFillFields, StepEPREL, StepCategorize}
|
||||
case "categorize_enhance":
|
||||
return append([]string{}, CanonicalSteps...)
|
||||
default: // full
|
||||
return append([]string{}, CanonicalSteps...)
|
||||
}
|
||||
}
|
||||
|
||||
func stepsContain(steps []string, want string) bool {
|
||||
for _, s := range steps {
|
||||
if s == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// InitialStepProgress builds pending step_progress rows for a job.
|
||||
func InitialStepProgress(processingType string) []StepProgress {
|
||||
steps := resolveSteps(processingType)
|
||||
|
||||
@@ -19,11 +19,14 @@ func (s stubCompleter) Complete(_ context.Context, system, user string) (Complet
|
||||
|
||||
func TestResolveSteps(t *testing.T) {
|
||||
cases := map[string][]string{
|
||||
"full": {StepNormalize, StepParseSpecs, StepFillFields, StepEPREL, StepAIEnhance},
|
||||
"full": {StepNormalize, StepParseSpecs, StepFillFields, StepEPREL, StepCategorize, StepAIEnhance},
|
||||
"enhance_only": {StepNormalize, StepAIEnhance},
|
||||
"attributes_only": {StepNormalize, StepParseSpecs, StepFillFields},
|
||||
"eprel_only": {StepNormalize, StepParseSpecs, StepEPREL},
|
||||
"normalize_only": {StepNormalize},
|
||||
"categorize": {StepNormalize, StepParseSpecs, StepFillFields, StepEPREL, StepCategorize},
|
||||
"categorize_only": {StepNormalize, StepParseSpecs, StepFillFields, StepEPREL, StepCategorize},
|
||||
"categorize_enhance": {StepNormalize, StepParseSpecs, StepFillFields, StepEPREL, StepCategorize, StepAIEnhance},
|
||||
}
|
||||
for in, want := range cases {
|
||||
got := resolveSteps(in)
|
||||
|
||||
Reference in New Issue
Block a user