Initial commit of Descrybe v2 without local scratch artifacts.
Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
This commit is contained in:
@@ -0,0 +1,632 @@
|
||||
package processing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/company"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||||
)
|
||||
|
||||
// StepPolicy controls entitlement-gated steps (AI / EPREL).
|
||||
type StepPolicy struct {
|
||||
AllowAI bool
|
||||
AllowEPREL bool
|
||||
}
|
||||
|
||||
// RunSteps executes the multi-step product pipeline.
|
||||
// OpenAI enhance runs only when Completer is configured, Enabled(), and policy.AllowAI.
|
||||
// EPREL runs only when enricher enabled and policy.AllowEPREL.
|
||||
func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput, processingType string, categoryNames []string, policy StepPolicy) (StepResult, error) {
|
||||
steps := resolveSteps(processingType)
|
||||
out := StepResult{
|
||||
Attributes: map[string]any{},
|
||||
ProcessedAttributes: map[string]any{},
|
||||
FieldSources: map[string]any{},
|
||||
EPREL: map[string]any{},
|
||||
GPTResponse: map[string]any{"steps": []any{}},
|
||||
Notes: []string{},
|
||||
}
|
||||
|
||||
normalized := map[string]any{}
|
||||
attrs := map[string]any{}
|
||||
|
||||
for _, step := range steps {
|
||||
switch step {
|
||||
case StepNormalize:
|
||||
normalized = NormalizeMapped(in.Mapped, in.Raw)
|
||||
out.Name = preferredProductTitle(in.GTIN,
|
||||
stringFromAny(normalized["name"]),
|
||||
stringFromAny(normalized["title"]),
|
||||
in.Name,
|
||||
in.PriorProcessedName,
|
||||
)
|
||||
out.Description = preferredProductDescription(
|
||||
stringFromAny(normalized["description"]),
|
||||
in.Description,
|
||||
in.PriorProcessedDescription,
|
||||
)
|
||||
out.Category = stringFromAny(normalized["category"])
|
||||
// mapped_data.category wins; otherwise keep existing processed category
|
||||
// so enhance_only / reprocess cannot blank A1 legacy categories.
|
||||
preserveCategoryIfEmpty(&out, in.PriorCategory)
|
||||
out.FieldSources["normalize"] = "mapped+raw"
|
||||
appendStepLog(out.GPTResponse, StepNormalize, map[string]any{
|
||||
"keys": len(normalized),
|
||||
})
|
||||
|
||||
case StepParseSpecs:
|
||||
specVal := normalized["specifications"]
|
||||
if specVal == nil {
|
||||
specVal = in.Mapped["specifications"]
|
||||
}
|
||||
if specVal == nil {
|
||||
specVal = in.Raw["specifications"]
|
||||
}
|
||||
parsed := ParseSpecifications(specVal)
|
||||
// Also accept pre-mapped attributes map
|
||||
if am, ok := normalized["attributes"].(map[string]any); ok {
|
||||
for k, v := range ParseSpecifications(am) {
|
||||
if _, exists := parsed[k]; !exists {
|
||||
parsed[k] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
attrs = parsed
|
||||
out.Attributes = attrs
|
||||
out.ProcessedAttributes = attrs
|
||||
out.FieldSources["attributes"] = "specifications"
|
||||
appendStepLog(out.GPTResponse, StepParseSpecs, map[string]any{
|
||||
"count": len(attrs),
|
||||
})
|
||||
|
||||
case StepFillFields:
|
||||
normalized = FillMissingFields(normalized, attrs)
|
||||
if len(in.StandardFields) > 0 {
|
||||
normalized = FillMissingStandardFields(normalized, in.Raw, in.StandardFields)
|
||||
}
|
||||
out.Name = preferredProductTitle(in.GTIN,
|
||||
stringFromAny(normalized["name"]),
|
||||
stringFromAny(normalized["title"]),
|
||||
out.Name,
|
||||
in.Name,
|
||||
in.PriorProcessedName,
|
||||
)
|
||||
out.Description = preferredProductDescription(
|
||||
stringFromAny(normalized["description"]),
|
||||
out.Description,
|
||||
in.Description,
|
||||
)
|
||||
out.Category = stringFromAny(normalized["category"])
|
||||
// Promote filled scalar fields into attributes when useful
|
||||
promote := []string{"brand", "width", "height", "depth", "weight", "gtin", "stock_status"}
|
||||
for _, f := range in.StandardFields {
|
||||
if f.Key != "" {
|
||||
promote = append(promote, f.Key)
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, k := range promote {
|
||||
if seen[k] {
|
||||
continue
|
||||
}
|
||||
seen[k] = true
|
||||
if v := stringFromAny(normalized[k]); v != "" {
|
||||
if _, exists := attrs[k]; !exists {
|
||||
attrs[k] = v
|
||||
}
|
||||
out.FieldSources[k] = "fill_fields"
|
||||
}
|
||||
}
|
||||
out.Attributes = attrs
|
||||
out.ProcessedAttributes = attrs
|
||||
appendStepLog(out.GPTResponse, StepFillFields, map[string]any{
|
||||
"brand": stringFromAny(normalized["brand"]),
|
||||
})
|
||||
if out.Category == "" && e != nil && e.Vector != nil && e.Vector.Enabled() {
|
||||
text := strings.TrimSpace(out.Name + " " + out.Description)
|
||||
if text != "" {
|
||||
if cat, err := e.Vector.SuggestCategory(ctx, companyID, text, categoryNames); err == nil && strings.TrimSpace(cat) != "" {
|
||||
out.Category = SanitizeOutput(cat)
|
||||
out.FieldSources["category"] = "vector"
|
||||
out.Notes = append(out.Notes, "category: vector")
|
||||
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
|
||||
"status": "ok",
|
||||
"category": out.Category,
|
||||
})
|
||||
} else if err != nil {
|
||||
out.Notes = append(out.Notes, "vector_categorize: "+TruncateError(err))
|
||||
appendStepLog(out.GPTResponse, "vector_categorize", map[string]any{
|
||||
"status": "failed",
|
||||
"error": TruncateError(err),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
preserveCategoryIfEmpty(&out, in.PriorCategory)
|
||||
|
||||
case StepEPREL:
|
||||
if !policy.AllowEPREL {
|
||||
out.Notes = append(out.Notes, "eprel: skipped (not allowed for this job)")
|
||||
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{
|
||||
"status": "skipped",
|
||||
"reason": "entitlement_can_use_eprel",
|
||||
})
|
||||
break
|
||||
}
|
||||
id := eprel.ExtractID(normalized, in.Mapped, in.Raw)
|
||||
if id == "" {
|
||||
out.Notes = append(out.Notes, "eprel: no id")
|
||||
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "reason": "no_id"})
|
||||
break
|
||||
}
|
||||
enricher := e.EPREL
|
||||
if enricher == nil {
|
||||
enricher = eprel.Disabled{}
|
||||
}
|
||||
if !enricher.Enabled() {
|
||||
out.Notes = append(out.Notes, "eprel: enricher disabled")
|
||||
out.EPREL = map[string]any{"eprel_id": id, "status": "skipped"}
|
||||
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "skipped", "eprel_id": id, "reason": "disabled"})
|
||||
break
|
||||
}
|
||||
data, err := enricher.Fetch(ctx, id)
|
||||
if err != nil {
|
||||
out.Notes = append(out.Notes, "eprel: "+TruncateError(err))
|
||||
attrs["eprel_id"] = id
|
||||
out.Attributes = attrs
|
||||
out.ProcessedAttributes = attrs
|
||||
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "failed", "error": TruncateError(err)})
|
||||
// Non-fatal: continue pipeline
|
||||
break
|
||||
}
|
||||
if data == nil {
|
||||
attrs["eprel_id"] = id
|
||||
out.Attributes = attrs
|
||||
out.ProcessedAttributes = attrs
|
||||
out.EPREL = map[string]any{"eprel_id": id, "status": "empty"}
|
||||
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "empty", "eprel_id": id})
|
||||
break
|
||||
}
|
||||
attrs = eprel.MergeInto(attrs, data)
|
||||
out.Attributes = attrs
|
||||
out.ProcessedAttributes = attrs
|
||||
out.EPREL = map[string]any{
|
||||
"eprel_id": data.ID,
|
||||
"label": data.Label,
|
||||
"pdf": data.PDF,
|
||||
"energy_class": data.EnergyClass,
|
||||
"energy_scale": data.EnergyScale,
|
||||
}
|
||||
out.FieldSources["eprel"] = "eprel_api"
|
||||
appendStepLog(out.GPTResponse, StepEPREL, map[string]any{"status": "ok", "eprel_id": data.ID})
|
||||
|
||||
case StepAIEnhance:
|
||||
preservePriorEnhanceHash := func() {
|
||||
if in.PriorEnhanceHash != "" {
|
||||
out.FieldSources[FieldEnhanceInputHash] = in.PriorEnhanceHash
|
||||
}
|
||||
}
|
||||
if !policy.AllowAI {
|
||||
out.ProcessedName = out.Name
|
||||
out.ProcessedDescription = out.Description
|
||||
out.Notes = append(out.Notes, "ai_enhance: skipped (Free plan — upgrade for AI titles/descriptions)")
|
||||
preservePriorEnhanceHash()
|
||||
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
|
||||
"status": "skipped",
|
||||
"reason": "entitlement_can_use_ai",
|
||||
})
|
||||
break
|
||||
}
|
||||
if !e.CompleterEnabled() {
|
||||
out.ProcessedName = out.Name
|
||||
out.ProcessedDescription = out.Description
|
||||
out.Notes = append(out.Notes, "ai_enhance: skipped (platform OpenAI unset; configure admin settings or company BYOK)")
|
||||
preservePriorEnhanceHash()
|
||||
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
|
||||
"status": "skipped",
|
||||
"reason": "openai_not_configured",
|
||||
})
|
||||
break
|
||||
}
|
||||
langs := in.ContentLanguages
|
||||
if len(langs) == 0 {
|
||||
langs = []string{in.Language}
|
||||
}
|
||||
if len(langs) == 0 {
|
||||
langs = []string{company.DefaultLanguage}
|
||||
}
|
||||
primary := in.Language
|
||||
if primary == "" {
|
||||
primary = langs[0]
|
||||
}
|
||||
localized := company.LocalizedContent{}
|
||||
if in.PriorLocalized != nil {
|
||||
for k, v := range in.PriorLocalized {
|
||||
localized[k] = v
|
||||
}
|
||||
}
|
||||
anyFailed := false
|
||||
anyOK := false
|
||||
allUnchanged := true
|
||||
langMetas := make([]any, 0, len(langs))
|
||||
for _, lang := range langs {
|
||||
tpl := in.EnhanceByLang[lang]
|
||||
if tpl.System == "" && tpl.User == "" && lang == primary {
|
||||
tpl = PromptTemplates{System: in.EnhanceSystemTemplate, User: in.EnhanceUserTemplate}
|
||||
}
|
||||
catPrompt := in.CategoryEnhancePrompt
|
||||
if lang != primary || catPrompt == "" {
|
||||
catPrompt = categoryEnhancePromptFor(in.CategoryPromptsByLang, out.Category, lang)
|
||||
}
|
||||
priorFields := company.FieldsForLanguage(in.PriorLocalized, lang)
|
||||
priorHash := priorFields.EnhanceInputHash
|
||||
priorName := priorFields.ProcessedName
|
||||
priorDesc := priorFields.ProcessedDescription
|
||||
if lang == primary {
|
||||
if priorHash == "" {
|
||||
priorHash = in.PriorEnhanceHash
|
||||
}
|
||||
if priorName == "" {
|
||||
priorName = in.PriorProcessedName
|
||||
}
|
||||
if priorDesc == "" {
|
||||
priorDesc = in.PriorProcessedDescription
|
||||
}
|
||||
}
|
||||
name, desc, tokens, raw, err := e.enhance(ctx, ProductInput{
|
||||
GTIN: in.GTIN,
|
||||
Name: out.Name,
|
||||
Description: out.Description,
|
||||
Mapped: normalized,
|
||||
BrandPrompt: in.BrandPrompt,
|
||||
Language: lang,
|
||||
EnhanceSystemTemplate: tpl.System,
|
||||
EnhanceUserTemplate: tpl.User,
|
||||
CategoryEnhancePrompt: catPrompt,
|
||||
PriorEnhanceHash: priorHash,
|
||||
PriorProcessedName: priorName,
|
||||
PriorProcessedDescription: priorDesc,
|
||||
}, out.Category, attrs)
|
||||
out.TotalTokens += tokens
|
||||
status := enhanceStatusFromMeta(raw)
|
||||
meta := map[string]any{"language": lang, "raw": raw}
|
||||
if err != nil {
|
||||
anyFailed = true
|
||||
allUnchanged = false
|
||||
meta["status"] = "failed"
|
||||
meta["error"] = TruncateError(err)
|
||||
out.Notes = append(out.Notes, "ai_enhance: "+TruncateError(err))
|
||||
if lang == primary {
|
||||
name, desc = out.Name, out.Description
|
||||
} else if priorName != "" || priorDesc != "" {
|
||||
name, desc = priorName, priorDesc
|
||||
} else {
|
||||
langMetas = append(langMetas, meta)
|
||||
continue
|
||||
}
|
||||
} else if status == "unchanged" {
|
||||
meta["status"] = "unchanged"
|
||||
} else {
|
||||
allUnchanged = false
|
||||
anyOK = true
|
||||
meta["status"] = status
|
||||
}
|
||||
hash := enhanceHashFromMeta(raw)
|
||||
localized[lang] = company.LocalizedFields{
|
||||
ProcessedName: name,
|
||||
ProcessedDescription: desc,
|
||||
EnhanceInputHash: hash,
|
||||
MetaTitle: company.FieldsForLanguage(localized, lang).MetaTitle,
|
||||
MetaDescription: company.FieldsForLanguage(localized, lang).MetaDescription,
|
||||
}
|
||||
// Preserve existing meta when re-enhancing titles only.
|
||||
if prev := company.FieldsForLanguage(in.PriorLocalized, lang); prev.MetaTitle != "" || prev.MetaDescription != "" {
|
||||
f := localized[lang]
|
||||
if f.MetaTitle == "" {
|
||||
f.MetaTitle = prev.MetaTitle
|
||||
}
|
||||
if f.MetaDescription == "" {
|
||||
f.MetaDescription = prev.MetaDescription
|
||||
}
|
||||
localized[lang] = f
|
||||
}
|
||||
langMetas = append(langMetas, meta)
|
||||
if lang == primary {
|
||||
name = preferredProductTitle(in.GTIN, name, out.Name, in.PriorProcessedName)
|
||||
desc = preferredProductDescription(desc, out.Description, in.PriorProcessedDescription)
|
||||
out.ProcessedName = name
|
||||
out.ProcessedDescription = desc
|
||||
if name != "" {
|
||||
out.Name = name
|
||||
}
|
||||
if desc != "" {
|
||||
out.Description = desc
|
||||
}
|
||||
if hash != "" && (status == "ok" || status == "unchanged") {
|
||||
out.FieldSources[FieldEnhanceInputHash] = hash
|
||||
} else if err != nil {
|
||||
preservePriorEnhanceHash()
|
||||
}
|
||||
}
|
||||
}
|
||||
out.LocalizedContent = localized
|
||||
if anyFailed && !anyOK {
|
||||
if out.ProcessedName == "" {
|
||||
out.ProcessedName = out.Name
|
||||
}
|
||||
if out.ProcessedDescription == "" {
|
||||
out.ProcessedDescription = out.Description
|
||||
}
|
||||
preservePriorEnhanceHash()
|
||||
errNote := ""
|
||||
for _, m := range langMetas {
|
||||
if mm, ok := m.(map[string]any); ok {
|
||||
if e, ok := mm["error"].(string); ok && e != "" {
|
||||
errNote = e
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
|
||||
"status": "failed",
|
||||
"error": errNote,
|
||||
"languages": langMetas,
|
||||
})
|
||||
break
|
||||
}
|
||||
if allUnchanged {
|
||||
out.SkipCreditDebit = true
|
||||
out.FieldSources["name"] = "ai_enhance_unchanged"
|
||||
out.FieldSources["description"] = "ai_enhance_unchanged"
|
||||
out.Notes = append(out.Notes, "ai_enhance: skipped (inputs unchanged)")
|
||||
} else {
|
||||
out.AIProviderMode = e.EngineProviderMode()
|
||||
out.FieldSources["name"] = "ai_enhance"
|
||||
out.FieldSources["description"] = "ai_enhance"
|
||||
}
|
||||
appendStepLog(out.GPTResponse, StepAIEnhance, map[string]any{
|
||||
"status": map[string]any{"unchanged": allUnchanged, "ok": anyOK, "failed": anyFailed},
|
||||
"languages": langMetas,
|
||||
})
|
||||
|
||||
default:
|
||||
appendStepLog(out.GPTResponse, step, map[string]any{"status": "unknown"})
|
||||
}
|
||||
}
|
||||
|
||||
preserveCategoryIfEmpty(&out, in.PriorCategory)
|
||||
out.Name = preferredProductTitle(in.GTIN, out.Name, out.ProcessedName, in.Name, in.PriorProcessedName)
|
||||
out.ProcessedName = preferredProductTitle(in.GTIN, out.ProcessedName, out.Name, in.PriorProcessedName, in.Name)
|
||||
out.Description = preferredProductDescription(out.Description, out.ProcessedDescription, in.Description)
|
||||
out.ProcessedDescription = preferredProductDescription(out.ProcessedDescription, out.Description, in.PriorProcessedDescription)
|
||||
if out.ProcessedName == "" {
|
||||
out.ProcessedName = out.Name
|
||||
}
|
||||
if out.ProcessedDescription == "" {
|
||||
out.ProcessedDescription = out.Description
|
||||
}
|
||||
if out.Attributes == nil {
|
||||
out.Attributes = map[string]any{}
|
||||
}
|
||||
if out.ProcessedAttributes == nil {
|
||||
out.ProcessedAttributes = out.Attributes
|
||||
}
|
||||
if out.AIProviderMode == "" {
|
||||
if out.TotalTokens > 0 {
|
||||
out.AIProviderMode = e.EngineProviderMode()
|
||||
} else {
|
||||
out.AIProviderMode = AIProviderUnknown
|
||||
}
|
||||
}
|
||||
if len(out.Notes) > 0 {
|
||||
out.GPTResponse["notes"] = out.Notes
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// preserveCategoryIfEmpty keeps an existing processed category when normalize/AI
|
||||
// left Category empty (common for A1 feeds where category lives only on processed).
|
||||
func preserveCategoryIfEmpty(out *StepResult, prior string) {
|
||||
if out == nil || strings.TrimSpace(out.Category) != "" {
|
||||
return
|
||||
}
|
||||
prior = strings.TrimSpace(prior)
|
||||
if prior == "" {
|
||||
return
|
||||
}
|
||||
out.Category = SanitizeText(prior)
|
||||
if out.FieldSources == nil {
|
||||
out.FieldSources = map[string]any{}
|
||||
}
|
||||
out.FieldSources["category"] = "prior_processed"
|
||||
}
|
||||
|
||||
func resolveSteps(processingType string) []string {
|
||||
switch strings.ToLower(strings.TrimSpace(processingType)) {
|
||||
case "enhance", "enhance_only", "enhance-only", "title", "description":
|
||||
return []string{StepNormalize, StepAIEnhance}
|
||||
case "attributes", "attributes_only", "specs", "specifications":
|
||||
return []string{StepNormalize, StepParseSpecs, StepFillFields}
|
||||
case "eprel", "eprel_only":
|
||||
return []string{StepNormalize, StepEPREL}
|
||||
case "normalize_only":
|
||||
return []string{StepNormalize}
|
||||
case "categorize", "categorize_only", "categorize_enhance":
|
||||
// Legacy aliases → full deterministic + optional AI
|
||||
return append([]string{}, CanonicalSteps...)
|
||||
default: // full
|
||||
return append([]string{}, CanonicalSteps...)
|
||||
}
|
||||
}
|
||||
|
||||
// InitialStepProgress builds pending step_progress rows for a job.
|
||||
func InitialStepProgress(processingType string) []StepProgress {
|
||||
steps := resolveSteps(processingType)
|
||||
out := make([]StepProgress, 0, len(steps))
|
||||
for _, s := range steps {
|
||||
out = append(out, StepProgress{Step: s, Status: "pending"})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func appendStepLog(gpt map[string]any, name string, raw any) {
|
||||
steps, _ := gpt["steps"].([]any)
|
||||
gpt["steps"] = append(steps, map[string]any{"step": name, "raw": raw})
|
||||
}
|
||||
|
||||
func (e *Engine) enhance(ctx context.Context, in ProductInput, category string, attrs map[string]any) (string, string, int, any, error) {
|
||||
sysTpl, userTpl := resolveProductPromptTemplates(in)
|
||||
hash := HashEnhanceInput(category, in.Name, in.Description, in.BrandPrompt, in.Language, sysTpl, userTpl, attrs)
|
||||
if e == nil || e.Completer == nil {
|
||||
return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName),
|
||||
preferredProductDescription(in.Description, in.PriorProcessedDescription),
|
||||
0, map[string]any{"status": "skipped", "input_hash": hash}, nil
|
||||
}
|
||||
// Skip LLM when inputs match the last successful enhance (before any credit debit).
|
||||
if in.PriorEnhanceHash != "" && in.PriorEnhanceHash == hash &&
|
||||
(in.PriorProcessedName != "" || in.PriorProcessedDescription != "") {
|
||||
name := preferredProductTitle(in.GTIN, in.PriorProcessedName, in.Name)
|
||||
desc := preferredProductDescription(in.PriorProcessedDescription, in.Description)
|
||||
return name, desc, 0, map[string]any{
|
||||
"status": "unchanged",
|
||||
"input_hash": hash,
|
||||
}, nil
|
||||
}
|
||||
system, user := RenderProductEnhancePrompts(sysTpl, userTpl, category, in.Name, in.Description, in.GTIN, in.BrandPrompt, in.Language, attrs)
|
||||
comp, obj, err := CompleteJSON(ctx, e.Completer, system, user, CompleteOptions{
|
||||
MaxTokens: MaxTokensEnhance,
|
||||
Temperature: DefaultStructuredTemp,
|
||||
})
|
||||
if err != nil {
|
||||
// Network/provider failure vs parse failure after retry
|
||||
if obj == nil && comp.Text == "" {
|
||||
return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName),
|
||||
preferredProductDescription(in.Description, in.PriorProcessedDescription),
|
||||
0, map[string]any{
|
||||
"provider": "passthrough",
|
||||
"error": TruncateError(err),
|
||||
"input_hash": hash,
|
||||
}, err
|
||||
}
|
||||
// Parse failed after retry — keep original copy (avoid garbage titles)
|
||||
return preferredProductTitle(in.GTIN, in.Name, in.PriorProcessedName),
|
||||
preferredProductDescription(in.Description, in.PriorProcessedDescription),
|
||||
comp.TotalTokens, map[string]any{
|
||||
"status": "parse_failed",
|
||||
"error": "AI returned invalid JSON; kept original title/description",
|
||||
"raw": truncateRunes(comp.Text, 200),
|
||||
"input_hash": hash,
|
||||
}, nil
|
||||
}
|
||||
name := preferredProductTitle(in.GTIN, SanitizeOutput(fmt.Sprint(obj["name"])), in.Name, in.PriorProcessedName)
|
||||
desc := preferredProductDescription(SanitizeOutput(fmt.Sprint(obj["description"])), in.Description, in.PriorProcessedDescription)
|
||||
return name, desc, comp.TotalTokens, map[string]any{
|
||||
"status": "ok",
|
||||
"input_hash": hash,
|
||||
"raw": comp.Raw,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func sanitizeJSON(v any) string {
|
||||
if v == nil {
|
||||
return "{}"
|
||||
}
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return "{}"
|
||||
}
|
||||
return SanitizeText(string(b))
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
return SanitizeOutput(strings.Trim(s, "\"'` "))
|
||||
}
|
||||
|
||||
// labeledPromptValue returns the first line after any of the given labels
|
||||
// (case-insensitive), e.g. "Name:" / "Desc:" from ProductEnhanceUser.
|
||||
func labeledPromptValue(user string, labels ...string) string {
|
||||
lower := strings.ToLower(user)
|
||||
bestAt := -1
|
||||
bestLabel := ""
|
||||
for _, label := range labels {
|
||||
label = strings.ToLower(strings.TrimSpace(label))
|
||||
if label == "" {
|
||||
continue
|
||||
}
|
||||
at := strings.Index(lower, label)
|
||||
if at < 0 {
|
||||
continue
|
||||
}
|
||||
if bestAt < 0 || at < bestAt {
|
||||
bestAt = at
|
||||
bestLabel = label
|
||||
}
|
||||
}
|
||||
if bestAt < 0 {
|
||||
return ""
|
||||
}
|
||||
rest := user[bestAt+len(bestLabel):]
|
||||
if j := strings.Index(strings.ToLower(rest), "attrs:"); j >= 0 {
|
||||
rest = rest[:j]
|
||||
}
|
||||
if j := strings.Index(strings.ToLower(rest), "attributes:"); j >= 0 {
|
||||
rest = rest[:j]
|
||||
}
|
||||
return firstLine(rest)
|
||||
}
|
||||
|
||||
// isPromptLabelTitle detects enhance pollution where the model echoed a
|
||||
// prompt header ("Category:" / "Category: 120") as the product title.
|
||||
func isPromptLabelTitle(s string) bool {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" || s == "<nil>" {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(s)
|
||||
for _, label := range []string{
|
||||
"category", "name", "desc", "description",
|
||||
"attrs", "attributes", "current name", "current description",
|
||||
} {
|
||||
if lower == label || lower == label+":" {
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(lower, label+":") || strings.HasPrefix(lower, label+" :") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// preferredProductTitle picks the first usable title, skipping empty values and
|
||||
// prompt-label echoes like "Category:" (seen on A1 Elkotex reprocess).
|
||||
func preferredProductTitle(gtin string, candidates ...string) string {
|
||||
for _, c := range candidates {
|
||||
c = strings.TrimSpace(c)
|
||||
if c == "" || c == "<nil>" || isPromptLabelTitle(c) {
|
||||
continue
|
||||
}
|
||||
return SanitizeOutput(c)
|
||||
}
|
||||
if strings.TrimSpace(gtin) != "" {
|
||||
return SanitizeText("Product " + strings.TrimSpace(gtin))
|
||||
}
|
||||
return "Product"
|
||||
}
|
||||
|
||||
func preferredProductDescription(candidates ...string) string {
|
||||
for _, c := range candidates {
|
||||
c = strings.TrimSpace(c)
|
||||
if c == "" || c == "<nil>" || isPromptLabelTitle(c) {
|
||||
continue
|
||||
}
|
||||
return SanitizeOutput(c)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Reference in New Issue
Block a user