greeneclipse/descrybe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixes

Browse Source
This commit is contained in:
greeneclipse
2026-08-16 16:57:36 +02:00
parent 96f8c1115c
commit 532d439c41
106 changed files with 10147 additions and 494 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/processing/pipeline.go
+352 -49
View File
@@ -657,10 +657,7 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
if !shouldFlushJobProgress(sinceFlush, progressEvery, batchDone) {
return
}
mode := modeLabel
if lastResult != nil && lastResult.AIProviderMode != "" {
mode = lastResult.AIProviderMode
}
mode := preferKnownProviderMode(lastResultAIProviderMode(lastResult), modeLabel)
if err := p.flushJobCountersAndProgress(ctx, jobID, processingType, lastResult, processed, tokensSinceFlush, mode); err != nil {
log.Printf("processing: flush progress job=%s err=%s", jobID, TruncateError(err))
}
@@ -672,6 +669,7 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
if err := stopOnCancel(p.jobCancelled(ctx, jobID)); err != nil {
flushProgress(true)
if errors.Is(err, errJobCancelled) {
_ = p.reclaimOrphanedProcessingItems(ctx, jobID)
return nil
}
return fmt.Errorf("processing: check cancel job=%s: %w", jobID, err)
@@ -682,6 +680,27 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
return err
}
if len(items) == 0 {
// loadPendingItems only reclaims processing rows older than StuckAgeInterval.
// A crashed peer can leave fresh processing rows; do not mark the job
// completed while open work remains — reset them and keep going.
var open int
if err := p.Pool.QueryRow(ctx, `
SELECT COUNT(*) FROM processing_job_products
WHERE job_id = $1 AND processed_product_id IS NULL
AND status IN ('pending', 'processing')`, jobID).Scan(&open); err != nil {
flushProgress(true)
return fmt.Errorf("processing: count open items job=%s: %w", jobID, err)
}
if open > 0 {
if _, err := p.Pool.Exec(ctx, `
UPDATE processing_job_products
SET status = 'pending', error = NULL, updated_at = now()
WHERE job_id = $1 AND status = 'processing' AND processed_product_id IS NULL`, jobID); err != nil {
flushProgress(true)
return fmt.Errorf("processing: reclaim fresh processing job=%s: %w", jobID, err)
}
continue
}
break
}
if err := p.hydrateJobItems(ctx, companyID, items); err != nil {
@@ -695,6 +714,7 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
if err := stopOnCancel(p.jobCancelled(ctx, jobID)); err != nil {
flushProgress(true)
if errors.Is(err, errJobCancelled) {
_ = p.reclaimOrphanedProcessingItems(ctx, jobID)
return nil
}
return fmt.Errorf("processing: check cancel job=%s: %w", jobID, err)
@@ -766,10 +786,7 @@ func (p *Pipeline) ProcessJob(ctx context.Context, jobID uuid.UUID) error {
if finalStatus == "failed" {
finalStep = "failed"
}
finalMode := modeLabel
if lastResult != nil && lastResult.AIProviderMode != "" {
finalMode = lastResult.AIProviderMode
}
finalMode := preferKnownProviderMode(lastResultAIProviderMode(lastResult), modeLabel)
_, err = p.Pool.Exec(ctx, `
UPDATE processing_jobs
SET status = $2, processed_products = $3, error = $4, completed_at = now(), updated_at = now(),
@@ -818,9 +835,18 @@ type jobScopedCache struct {
enhanceSystemTemplate string
enhanceUserTemplate string
enhanceByLang map[string]PromptTemplates
// categoryPromptsByLang maps lower(trim(name)) → lang → sanitized prompt.
// categoryPromptsByLang maps lower(trim(name|unique_id)) → lang → sanitized prompt.
categoryPromptsByLang map[string]company.LangPromptMap
contentLanguages []string
// categoryFormulasByKey maps lower(trim(name|unique_id)) → title/description templates.
categoryFormulasByKey map[string]CategoryFormulas
// categoryNamesByUID maps unique_id → display name for meta/enhance/synthesize.
categoryNamesByUID map[string]string
// categoryUniqueIDs is the company taxonomy unique_id set (empty = skip validation).
categoryUniqueIDs map[string]struct{}
// categoryAttrKeys maps category unique_id → canonicalized attribute_key set
// from category_attributes (for AI enhance allowlisting).
categoryAttrKeys map[string]map[string]struct{}
contentLanguages []string
// Entitlements snapshot — avoids EntitlementsForCompany N+1 per product.
billingEnabled bool
canUseAI bool
@@ -902,56 +928,224 @@ func (p *Pipeline) loadJobScopedCache(ctx context.Context, companyID, jobID uuid
}
}
}
cache.categoryPromptsByLang = p.loadCategoryEnhancePrompts(ctx, companyID, jobID)
cache.categoryPromptsByLang, cache.categoryFormulasByKey, cache.categoryNamesByUID = p.loadCategoryEnhanceOverlays(ctx, companyID, jobID)
cache.categoryUniqueIDs = p.loadCategoryUniqueIDs(ctx, companyID, jobID)
cache.categoryAttrKeys = p.loadCategoryAttributeKeySets(ctx, companyID, jobID)
return cache
}
func (p *Pipeline) loadCategoryEnhancePrompts(ctx context.Context, companyID, jobID uuid.UUID) map[string]company.LangPromptMap {
func (p *Pipeline) loadCategoryUniqueIDs(ctx context.Context, companyID, jobID uuid.UUID) map[string]struct{} {
rows, err := p.Pool.Query(ctx, `
SELECT name, COALESCE(prompt, '{}'::jsonb)
SELECT unique_id
FROM categories
WHERE company_id = $1 AND prompt <> '{}'::jsonb`, companyID)
WHERE company_id = $1
AND COALESCE(NULLIF(BTRIM(unique_id), ''), '') <> ''`, companyID)
if err != nil {
log.Printf("processing: load category prompts job=%s company=%s err=%s", jobID, companyID, TruncateError(err))
log.Printf("processing: load category unique_ids job=%s company=%s err=%s", jobID, companyID, TruncateError(err))
return nil
}
defer rows.Close()
out := make(map[string]company.LangPromptMap)
out := make(map[string]struct{})
for rows.Next() {
var name string
var raw []byte
if err := rows.Scan(&name, &raw); err != nil {
log.Printf("processing: scan category prompt job=%s err=%s", jobID, TruncateError(err))
var id string
if err := rows.Scan(&id); err != nil {
log.Printf("processing: scan category unique_id job=%s err=%s", jobID, TruncateError(err))
continue
}
key := strings.ToLower(strings.TrimSpace(name))
if key == "" {
id = strings.TrimSpace(id)
if id == "" {
continue
}
m, err := company.DecodeLangPromptMap(raw)
if err != nil || !company.HasAnyPrompt(m) {
continue
}
// Re-sanitize with category rune cap.
cleaned := company.LangPromptMap{}
for lang, prompt := range m {
p := strings.TrimSpace(security.SanitizePrompt(prompt, catalog.MaxCategoryPromptRunes))
if p == "" {
continue
}
cleaned[lang] = p
}
if company.HasAnyPrompt(cleaned) {
out[key] = cleaned
}
out[id] = struct{}{}
}
if err := rows.Err(); err != nil {
log.Printf("processing: category prompts rows job=%s err=%s", jobID, TruncateError(err))
log.Printf("processing: category unique_ids rows job=%s err=%s", jobID, TruncateError(err))
}
return out
}
func categoryEnhancePromptFor(prompts map[string]company.LangPromptMap, category, language string) string {
// loadCategoryAttributeKeySets returns category unique_id → attribute_key set from
// category_attributes. Empty map on failure (callers still fall back to core keys).
func (p *Pipeline) loadCategoryAttributeKeySets(ctx context.Context, companyID, jobID uuid.UUID) map[string]map[string]struct{} {
out := map[string]map[string]struct{}{}
if p == nil || p.Pool == nil {
return out
}
rows, err := p.Pool.Query(ctx, `
SELECT ca.category_unique_id, a.attribute_key
FROM category_attributes ca
INNER JOIN attributes a ON a.id = ca.attribute_id AND a.company_id = ca.company_id
WHERE ca.company_id = $1
AND COALESCE(NULLIF(BTRIM(a.attribute_key), ''), '') <> ''`, companyID)
if err != nil {
log.Printf("processing: load category attribute keys job=%s company=%s err=%s", jobID, companyID, TruncateError(err))
return out
}
defer rows.Close()
for rows.Next() {
var catUID, key string
if err := rows.Scan(&catUID, &key); err != nil {
log.Printf("processing: scan category attribute key job=%s err=%s", jobID, TruncateError(err))
continue
}
catUID = strings.TrimSpace(catUID)
if catUID == "" {
continue
}
canon := canonicalizeAttrKey(key)
if canon == "" {
continue
}
set := out[catUID]
if set == nil {
set = map[string]struct{}{}
out[catUID] = set
}
set[canon] = struct{}{}
set[strings.ToLower(strings.TrimSpace(key))] = struct{}{}
}
if err := rows.Err(); err != nil {
log.Printf("processing: category attribute keys rows job=%s err=%s", jobID, TruncateError(err))
}
return out
}
// allowedAttrKeysForCategory returns category_attributes keys when known; empty
// non-nil map means core characteristics only. nil cache → nil (sanitize-only).
func allowedAttrKeysForCategory(cache *jobScopedCache, categoryUID string) map[string]struct{} {
if cache == nil {
return nil
}
return allowedAttrKeysFromSets(cache.categoryAttrKeys, categoryUID)
}
func categoryAttrKeysFromCache(cache *jobScopedCache) map[string]map[string]struct{} {
if cache == nil {
return nil
}
return cache.categoryAttrKeys
}
// allowedAttrKeysFromSets picks category keys when present; empty non-nil → core only.
// When sets is nil, returns nil (sanitize-only / unit tests).
func allowedAttrKeysFromSets(sets map[string]map[string]struct{}, categoryUID string) map[string]struct{} {
if sets == nil {
return nil
}
categoryUID = strings.TrimSpace(categoryUID)
if categoryUID != "" {
if keys, ok := sets[categoryUID]; ok && len(keys) > 0 {
return keys
}
}
return map[string]struct{}{}
}
func (p *Pipeline) loadCategoryEnhanceOverlays(ctx context.Context, companyID, jobID uuid.UUID) (map[string]company.LangPromptMap, map[string]CategoryFormulas, map[string]string) {
rows, err := p.Pool.Query(ctx, `
SELECT COALESCE(unique_id, ''), name,
COALESCE(prompt, '{}'::jsonb),
title_template,
description_template
FROM categories
WHERE company_id = $1`, companyID)
if err != nil {
log.Printf("processing: load category overlays job=%s company=%s err=%s", jobID, companyID, TruncateError(err))
return nil, nil, nil
}
defer rows.Close()
prompts := make(map[string]company.LangPromptMap)
formulas := make(map[string]CategoryFormulas)
namesByUID := make(map[string]string)
for rows.Next() {
var uniqueID, name string
var promptRaw []byte
var titleRaw, descRaw []byte
if err := rows.Scan(&uniqueID, &name, &promptRaw, &titleRaw, &descRaw); err != nil {
log.Printf("processing: scan category overlay job=%s err=%s", jobID, TruncateError(err))
continue
}
uid := strings.TrimSpace(uniqueID)
display := strings.TrimSpace(name)
if uid != "" && display != "" {
namesByUID[uid] = display
}
keys := categoryOverlayKeys(uniqueID, name)
if len(keys) == 0 {
continue
}
if m, err := company.DecodeLangPromptMap(promptRaw); err == nil && company.HasAnyPrompt(m) {
cleaned := company.LangPromptMap{}
for lang, prompt := range m {
p := strings.TrimSpace(security.SanitizePrompt(prompt, catalog.MaxCategoryPromptRunes))
if p == "" {
continue
}
cleaned[lang] = p
}
if company.HasAnyPrompt(cleaned) {
for _, key := range keys {
prompts[key] = cleaned
}
}
}
f := CategoryFormulas{}
if t := decodeOptionalJSONObject(titleRaw); t != nil {
f.TitleTemplate = t
}
if d := decodeOptionalJSONObject(descRaw); d != nil {
f.DescriptionTemplate = d
}
if f.TitleTemplate != nil || f.DescriptionTemplate != nil {
for _, key := range keys {
formulas[key] = f
}
}
}
if err := rows.Err(); err != nil {
log.Printf("processing: category overlays rows job=%s err=%s", jobID, TruncateError(err))
}
return prompts, formulas, namesByUID
}
func categoryOverlayKeys(uniqueID, name string) []string {
seen := map[string]struct{}{}
var out []string
add := func(raw string) {
key := strings.ToLower(strings.TrimSpace(raw))
if key == "" {
return
}
if _, ok := seen[key]; ok {
return
}
seen[key] = struct{}{}
out = append(out, key)
}
add(uniqueID)
add(name)
return out
}
func decodeOptionalJSONObject(raw []byte) map[string]any {
raw = []byte(strings.TrimSpace(string(raw)))
if len(raw) == 0 || string(raw) == "null" || string(raw) == "{}" {
return nil
}
var obj map[string]any
if err := json.Unmarshal(raw, &obj); err != nil || len(obj) == 0 {
return nil
}
return obj
}
// loadCategoryEnhancePrompts is retained for tests / callers that only need prompts.
func (p *Pipeline) loadCategoryEnhancePrompts(ctx context.Context, companyID, jobID uuid.UUID) map[string]company.LangPromptMap {
prompts, _, _ := p.loadCategoryEnhanceOverlays(ctx, companyID, jobID)
return prompts
}
func categoryEnhancePromptFor(prompts map[string]company.LangPromptMap, category, language, primary string) string {
if len(prompts) == 0 {
return ""
}
@@ -959,7 +1153,7 @@ func categoryEnhancePromptFor(prompts map[string]company.LangPromptMap, category
if key == "" {
return ""
}
return company.PromptForLanguage(prompts[key], language)
return company.PromptForLanguage(prompts[key], language, primary)
}
func progressFromResult(processingType string, result *StepResult) []StepProgress {
@@ -1180,7 +1374,35 @@ func (p *Pipeline) jobCancelled(ctx context.Context, jobID uuid.UUID) (bool, err
if err != nil {
return false, err
}
return status == "cancelled", nil
return isTerminalJobStatus(status), nil
}
// isTerminalJobStatus reports statuses that should stop an in-flight ProcessJob
// so worker JobSlots are released (failed/completed parkers included).
func isTerminalJobStatus(status string) bool {
switch strings.ToLower(strings.TrimSpace(status)) {
case "cancelled", "failed", "completed":
return true
default:
return false
}
}
// reclaimOrphanedProcessingItems returns in-flight items to pending when the job
// was marked failed/completed externally so a later ClaimNext can finish them.
func (p *Pipeline) reclaimOrphanedProcessingItems(ctx context.Context, jobID uuid.UUID) error {
var status string
if err := p.Pool.QueryRow(ctx, `SELECT status FROM processing_jobs WHERE id = $1`, jobID).Scan(&status); err != nil {
return err
}
if strings.EqualFold(strings.TrimSpace(status), "cancelled") {
return nil
}
_, err := p.Pool.Exec(ctx, `
UPDATE processing_job_products
SET status = 'pending', error = NULL, updated_at = now()
WHERE job_id = $1 AND status = 'processing' AND processed_product_id IS NULL`, jobID)
return err
}
func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, it *jobItem, processingType string, engine *Engine, modeLabel string, usingBYOK bool, cache *jobScopedCache) (bool, int, StepResult, error) {
@@ -1215,6 +1437,8 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
var stdDefs []StandardFieldDef
var brandPrompt, language, enhanceSystemTemplate, enhanceUserTemplate string
var categoryPromptsByLang map[string]company.LangPromptMap
var categoryFormulasByKey map[string]CategoryFormulas
var categoryNamesByUID map[string]string
var contentLanguages []string
var enhanceByLang map[string]PromptTemplates
if cache != nil {
@@ -1224,6 +1448,8 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
enhanceSystemTemplate = cache.enhanceSystemTemplate
enhanceUserTemplate = cache.enhanceUserTemplate
categoryPromptsByLang = cache.categoryPromptsByLang
categoryFormulasByKey = cache.categoryFormulasByKey
categoryNamesByUID = cache.categoryNamesByUID
contentLanguages = cache.contentLanguages
enhanceByLang = cache.enhanceByLang
}
@@ -1231,6 +1457,11 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
if len(stdDefs) > 0 {
enriched = FillMissingStandardFields(enriched, raw, stdDefs)
}
// Deterministic category unique_id from feed/mapped (no LLM required).
coerceMappedCategoryUniqueID(enriched)
if cat := categoryUniqueIDFromMaps(enriched, raw); cat != "" {
enriched["category"] = cat
}
if gtin == "" {
gtin = stringFromMap(enriched, "gtin", "ean", "EAN", "upc")
}
@@ -1249,6 +1480,10 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
EnhanceSystemTemplate: enhanceSystemTemplate,
EnhanceUserTemplate: enhanceUserTemplate,
CategoryPromptsByLang: categoryPromptsByLang,
CategoryFormulasByKey: categoryFormulasByKey,
CategoryNamesByUID: categoryNamesByUID,
AllowedAttrKeys: allowedAttrKeysForCategory(cache, stringFromAny(enriched["category"])),
CategoryAttrKeys: categoryAttrKeysFromCache(cache),
}
if it.hydrated {
if it.hasPrior {
@@ -1292,16 +1527,31 @@ func (p *Pipeline) processOne(ctx context.Context, companyID, jobID uuid.UUID, i
if err != nil {
return false, 0, StepResult{}, err
}
// Persist mapped unique_id only when it exists in the company taxonomy.
// Empty taxonomy set skips filtering (unit tests / companies without categories).
if cache != nil {
filterCategoryIfInvalid(&result, cache.categoryUniqueIDs)
syncCategoryName(&result, cache.categoryNamesByUID)
}
// Re-apply after category validation so allowlist matches the persisted category.
allowed := allowedAttrKeysForCategory(cache, result.Category)
result.Attributes = AttrsForPersist(result.Attributes, allowed)
result.ProcessedAttributes = AttrsForPersist(result.ProcessedAttributes, allowed)
if len(result.ProcessedAttributes) == 0 {
result.ProcessedAttributes = result.Attributes
}
// Free template SEO meta (no FillMetaAI / no extra credits).
result.MetaTitle, result.MetaDescription = fillMetaFromResult(result)
attrsJSON, procAttrsJSON, gptJSON, sourcesJSON, err := marshalProcessOnePayload(result)
if err != nil {
return false, 0, result, err
}
providerMode := result.AIProviderMode
if providerMode == "" {
if modeLabel != "" {
providerMode = modeLabel
} else if result.TotalTokens > 0 {
// Treat "unknown" like empty so job modeLabel / EngineProviderMode wins on
// hash-skip (TotalTokens==0) paths that previously stamped unknown in RunSteps.
providerMode := preferKnownProviderMode(result.AIProviderMode, modeLabel)
if isUnknownProviderMode(providerMode) {
if result.TotalTokens > 0 {
providerMode = AIProviderInternal
} else {
providerMode = AIProviderUnknown
@@ -1375,8 +1625,9 @@ const upsertProcessedProductSQL = `
INSERT INTO processed_products (
company_id, raw_product_id, product_id, name, category, description,
processed_name, processed_description, status, attributes, processed_attributes,
gpt_response, total_tokens, field_sources, ai_provider_mode, localized_content
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'needs_review',$9,$10,$11,$12,$13,$14,$15::jsonb)
gpt_response, total_tokens, field_sources, ai_provider_mode, localized_content,
meta_title, meta_description
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'needs_review',$9,$10,$11,$12,$13,$14,$15::jsonb,$16,$17)
ON CONFLICT (company_id, raw_product_id) DO UPDATE SET
product_id = EXCLUDED.product_id,
name = EXCLUDED.name,
@@ -1392,6 +1643,57 @@ const upsertProcessedProductSQL = `
field_sources = EXCLUDED.field_sources,
ai_provider_mode = EXCLUDED.ai_provider_mode,
localized_content = EXCLUDED.localized_content,
meta_title = CASE
WHEN NULLIF(BTRIM(processed_products.meta_title), '') IS NULL
THEN NULLIF(BTRIM(EXCLUDED.meta_title), '')
WHEN BTRIM(processed_products.meta_title) ~ '\|\s+[0-9]+$'
THEN NULLIF(BTRIM(EXCLUDED.meta_title), '')
WHEN LOWER(processed_products.meta_title) LIKE '%short retail title%'
OR LOWER(processed_products.meta_title) LIKE '%title formula%'
OR LOWER(processed_products.meta_title) LIKE '%follow any%'
OR LOWER(processed_products.meta_title) LIKE '%constraints that follow%'
OR LOWER(processed_products.meta_title) LIKE '%prefer 1-3%'
OR LOWER(processed_products.meta_title) LIKE '%reply with only json%'
OR LOWER(processed_products.meta_title) LIKE '%your reply is parsed as json%'
THEN NULLIF(BTRIM(EXCLUDED.meta_title), '')
ELSE processed_products.meta_title
END,
meta_description = CASE
WHEN NULLIF(BTRIM(processed_products.meta_description), '') IS NULL
THEN NULLIF(BTRIM(EXCLUDED.meta_description), '')
WHEN LOWER(processed_products.meta_description) LIKE '%prefer 1-3%'
OR LOWER(processed_products.meta_description) LIKE '%short retail title%'
OR LOWER(processed_products.meta_description) LIKE '%title formula%'
OR LOWER(processed_products.meta_description) LIKE '%do not emit%'
OR LOWER(processed_products.meta_description) LIKE '%reply with only json%'
THEN NULLIF(BTRIM(EXCLUDED.meta_description), '')
-- When refreshing a poisoned meta_title (| digits or prompt leakage), also refresh empty/weak stub desc.
WHEN (
BTRIM(processed_products.meta_title) ~ '\|\s+[0-9]+$'
OR LOWER(processed_products.meta_title) LIKE '%short retail title%'
OR LOWER(processed_products.meta_title) LIKE '%title formula%'
OR LOWER(processed_products.meta_title) LIKE '%follow any%'
OR LOWER(processed_products.meta_title) LIKE '%constraints that follow%'
OR LOWER(processed_products.meta_title) LIKE '%prefer 1-3%'
OR LOWER(processed_products.meta_title) LIKE '%reply with only json%'
OR LOWER(processed_products.meta_title) LIKE '%your reply is parsed as json%'
)
AND (
NULLIF(BTRIM(processed_products.meta_description), '') IS NULL
OR char_length(BTRIM(processed_products.meta_description)) < 40
OR LOWER(processed_products.meta_description) LIKE '%ready for retail listing%'
OR LOWER(processed_products.meta_description) LIKE '%quality product ready%'
OR LOWER(processed_products.meta_description) LIKE '%product description%'
OR LOWER(processed_products.meta_description) LIKE '%based on available specifications%'
OR LOWER(processed_products.meta_description) LIKE '%with available specifications%'
OR LOWER(processed_products.meta_description) LIKE '%available catalog details%'
OR LOWER(processed_products.meta_description) LIKE '%pripravljeno za prodajo%'
OR LOWER(processed_products.meta_description) LIKE '%na podlagi razpoložljivih specifikacij%'
OR LOWER(processed_products.meta_description) LIKE '%prefer 1-3%'
)
THEN NULLIF(BTRIM(EXCLUDED.meta_description), '')
ELSE processed_products.meta_description
END,
updated_at = now()
RETURNING id`
@@ -1426,6 +1728,7 @@ func (p *Pipeline) upsertProcessedProduct(
err = queryRow(ctx, upsertProcessedProductSQL,
companyID, rawID, gtin, result.Name, result.Category, result.Description,
result.ProcessedName, result.ProcessedDescription, attrsJSON, procAttrsJSON, gptJSON, result.TotalTokens, sourcesJSON, providerMode, string(locJSON),
result.MetaTitle, result.MetaDescription,
).Scan(&processedID)
return processedID, err
}