This commit is contained in:
2026-08-17 00:39:25 +02:00
parent 92f046b542
commit 93dc70123c
54 changed files with 25528 additions and 20666 deletions
@@ -0,0 +1,662 @@
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"os"
"os/signal"
"path/filepath"
"regexp"
"strings"
"syscall"
"time"
"unicode/utf8"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// Batch A: rich h1/h2/p/ul category formulas (incl. Slusalke/48).
var eans = []string{
"195348253666", // 7 Gaming monitorji
"4548736132597", // 48 Slusalke
"3045380024786", // 36 Pecice
"3838782459856", // 38 Pomivalni stroji
"1200130000638", // 3 Bluetooth zvocniki
}
var eanLabels = map[string]string{
"195348253666": "Gaming monitorji (rich formula)",
"4548736132597": "Slusalke cat48 (rich formula)",
"3045380024786": "Pecice (rich formula)",
"3838782459856": "Pomivalni stroji (rich formula)",
"1200130000638": "Bluetooth zvocniki (rich formula)",
}
var poisonMeta = regexp.MustCompile(`\|\s*\d+\s*$`)
var brandOnlyish = regexp.MustCompile(`(?i)^(bosch|gorenje|gigabyte|xiaomi|lenovo|samsung|lg|anker|jbl|vogels|ostalo)$`)
type dbProduct struct {
EAN string `json:"ean"`
Label string `json:"label"`
ProductID string `json:"product_id"`
Title string `json:"title"`
Description string `json:"description"`
Category string `json:"category"`
CategoryName string `json:"category_name"`
MetaTitle string `json:"meta_title"`
MetaDescription string `json:"meta_description"`
Attributes json.RawMessage `json:"attributes"`
AIProviderMode string `json:"ai_provider_mode"`
DescTemplate any `json:"description_template"`
TitleTemplate any `json:"title_template"`
FormulaConstraint string `json:"formula_constraint"`
FieldSources json.RawMessage `json:"field_sources"`
}
func main() {
out := os.Getenv("PROBE_OUT_DIR")
if out == "" {
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_a1_enhance_examples_A_20260816`
}
_ = os.MkdirAll(out, 0o755)
logPath := filepath.Join(out, "inproc_run.log")
logFile, err := os.Create(logPath)
if err != nil {
log.Fatalf("log file: %v", err)
}
defer logFile.Close()
log.SetOutput(logredact.Writer(io.MultiWriter(os.Stderr, logFile)))
companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
userID := uuid.MustParse("0ce3305c-d810-4b56-b1d4-3c1ed510db76")
cfg, err := config.Load()
if err != nil {
log.Fatalf("config: %v", err)
}
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
MaxConns: int32(cfg.DBMaxConns),
MinConns: int32(cfg.DBMinConns),
MaxConnLifetime: cfg.DBMaxConnLifetime,
MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
MaxConnIdleTime: cfg.DBMaxConnIdleTime,
HealthCheckPeriod: cfg.DBHealthCheckPeriod,
StatementTimeout: cfg.DBStatementTimeout,
})
if err != nil {
log.Fatalf("db: %v", err)
}
defer pool.Close()
waitForeignJobs(ctx, pool, companyID)
platEnv := platformsettings.EnvConfig{
AppEncryptionKey: cfg.AppEncryptionKey,
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
TokenSigningSecret: cfg.TokenSigningSecret,
DatabaseURL: cfg.DatabaseURL,
OpenAIAPIKey: cfg.OpenAIAPIKey,
OpenAIBaseURL: cfg.OpenAIBaseURL,
OpenAIModel: cfg.OpenAIModel,
OpenAIEmbeddingAPIKey: cfg.OpenAIEmbeddingAPIKey,
OpenAIEmbeddingBaseURL: cfg.OpenAIEmbeddingBaseURL,
OpenAIEmbeddingModel: cfg.OpenAIEmbeddingModel,
}
platSettings := platformsettings.NewService(pool, platEnv)
aiSvc := aiprovider.NewService(pool, aiprovider.EnvConfig{
AppEncryptionKey: cfg.AppEncryptionKey,
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
TokenSigningSecret: cfg.TokenSigningSecret,
DatabaseURL: cfg.DatabaseURL,
OpenAIAPIKey: cfg.OpenAIAPIKey,
OpenAIBaseURL: cfg.OpenAIBaseURL,
OpenAIModel: cfg.OpenAIModel,
ProcessingRPM: cfg.ProcessingRPM,
ProcessingMaxRetries: cfg.ProcessingMaxRetries,
})
aiSvc.Platform = platSettings
roleCfg, rerr := platSettings.ResolveAIConfig(ctx, platformsettings.AIRoleProcessing)
if rerr != nil {
failResolve(out, fmt.Sprintf("ResolveAIConfig: %v", rerr))
}
completer, modeLabel, byok, cerr := aiSvc.ResolveCompleterForRole(ctx, companyID, aiprovider.RoleProcessing)
if cerr != nil {
failResolve(out, fmt.Sprintf("ResolveCompleterForRole: %v", cerr))
}
if completer == nil {
failResolve(out, "nil completer")
}
client, ok := completer.(*processing.OpenAIClient)
if !ok {
failResolve(out, fmt.Sprintf("completer type %T", completer))
}
base := strings.TrimSpace(client.BaseURL)
model := strings.TrimSpace(client.Model)
resolveInfo := map[string]any{
"batch": "A",
"eans": eans,
"role_source": roleCfg.Source,
"role_provider": roleCfg.Provider,
"role_base_url": roleCfg.BaseURL,
"role_model": roleCfg.Model,
"role_enabled": roleCfg.Enabled,
"completer_base": base,
"completer_model": model,
"mode_label": modeLabel,
"byok": byok,
"env_base_url": cfg.OpenAIBaseURL,
"env_model": cfg.OpenAIModel,
"live": true,
}
if processing.IsMockOrLoopbackBaseURL(base) || strings.Contains(strings.ToLower(base), "18767") {
resolveInfo["live"] = false
resolveInfo["fail_reason"] = "resolved completer is mock/loopback — refuse silent mock fallback"
writeJSON(out, "resolve.json", resolveInfo)
log.Fatalf("LIVE LLM REQUIRED: resolved base=%s model=%s — refuse mock", base, model)
}
if strings.TrimSpace(client.APIKey) == "" {
resolveInfo["live"] = false
resolveInfo["fail_reason"] = "empty API key"
writeJSON(out, "resolve.json", resolveInfo)
log.Fatal("LIVE LLM REQUIRED: API key empty")
}
keyHint := ""
if k := strings.TrimSpace(client.APIKey); len(k) > 8 {
keyHint = k[:4] + "…" + k[len(k)-4:]
}
resolveInfo["api_key_hint"] = keyHint
modelsURL := strings.TrimRight(base, "/") + "/models"
modelsCtx, modelsCancel := context.WithTimeout(ctx, 30*time.Second)
defer modelsCancel()
modelsReq, merr := http.NewRequestWithContext(modelsCtx, http.MethodGet, modelsURL, nil)
if merr != nil {
failResolve(out, "models req: "+merr.Error())
}
modelsReq.Header.Set("Authorization", "Bearer "+client.APIKey)
modelsReq.Header.Set("Accept", "application/json")
modelsStart := time.Now()
modelsResp, merr := http.DefaultClient.Do(modelsReq)
modelsElapsed := time.Since(modelsStart).Round(time.Millisecond)
if merr != nil {
resolveInfo["probe_ok"] = false
resolveInfo["models_error"] = processing.TruncateError(merr)
writeJSON(out, "resolve.json", resolveInfo)
log.Fatalf("LIVE LLM CONNECTION ERROR: GET models: %v", merr)
}
modelsBody, _ := io.ReadAll(io.LimitReader(modelsResp.Body, 2048))
_ = modelsResp.Body.Close()
resolveInfo["models_url"] = modelsURL
resolveInfo["models_http_status"] = modelsResp.StatusCode
resolveInfo["models_elapsed"] = modelsElapsed.String()
snippet := strings.TrimSpace(string(modelsBody))
if len(snippet) > 200 {
snippet = snippet[:200] + "…"
}
resolveInfo["models_body_snippet"] = snippet
if modelsResp.StatusCode != http.StatusOK {
resolveInfo["probe_ok"] = false
writeJSON(out, "resolve.json", resolveInfo)
log.Fatalf("LIVE LLM CONNECTION ERROR: models HTTP %d", modelsResp.StatusCode)
}
probeCtx, probeCancel := context.WithTimeout(ctx, 3*time.Minute)
defer probeCancel()
probeStart := time.Now()
comp, perr := client.Complete(probeCtx, "Reply with exactly: PONG", "Say PONG")
probeElapsed := time.Since(probeStart).Round(time.Millisecond)
if perr != nil {
// One soft retry — OverloadedBot often needs a cool-down after a prior batch.
log.Printf("chat probe first attempt failed elapsed=%s err=%s — retrying once after 20s", probeElapsed, processing.TruncateError(perr))
select {
case <-ctx.Done():
resolveInfo["probe_ok"] = false
resolveInfo["probe_error"] = processing.TruncateError(perr)
writeJSON(out, "resolve.json", resolveInfo)
log.Fatalf("LIVE LLM CONNECTION ERROR: chat probe: %v", perr)
case <-time.After(20 * time.Second):
}
probeCtx2, probeCancel2 := context.WithTimeout(ctx, 3*time.Minute)
defer probeCancel2()
probeStart = time.Now()
comp, perr = client.Complete(probeCtx2, "Reply with exactly: PONG", "Say PONG")
probeElapsed = time.Since(probeStart).Round(time.Millisecond)
if perr != nil {
resolveInfo["probe_ok"] = false
resolveInfo["probe_error"] = processing.TruncateError(perr)
resolveInfo["models_http_status"] = modelsResp.StatusCode
writeJSON(out, "resolve.json", resolveInfo)
log.Fatalf("LIVE LLM CONNECTION ERROR: chat probe (after retry): %v", perr)
}
}
resolveInfo["probe_ok"] = true
resolveInfo["probe_elapsed"] = probeElapsed.String()
resolveInfo["probe_response_len"] = len(comp.Text)
resolveInfo["started_at_utc"] = time.Now().UTC().Format(time.RFC3339)
writeJSON(out, "resolve.json", resolveInfo)
log.Printf("live resolve ok source=%s base=%s model=%s mode=%s byok=%v", roleCfg.Source, base, model, modeLabel, byok)
pipeline := processing.NewPipeline(pool)
pipeline.BatchSize = cfg.ProcessingBatchSize
pipeline.AI = aiSvc
pipeline.Prompts = aiprompts.NewService(pool)
eprelClient := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second})
pipeline.Engine = &processing.Engine{
Completer: nil,
Vector: processing.NoopVectorCategorizer{},
EPREL: eprelClient,
ProviderMode: processing.AIProviderInternal,
}
rawIDs := make([]uuid.UUID, 0, len(eans))
for _, ean := range eans {
var id uuid.UUID
if err := pool.QueryRow(ctx, `
SELECT id FROM raw_products
WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id); err != nil {
log.Fatalf("raw product %s: %v", ean, err)
}
rawIDs = append(rawIDs, id)
log.Printf("ean=%s raw_id=%s label=%s", ean, id, eanLabels[ean])
}
tag, cerr2 := pool.Exec(ctx, `
UPDATE processed_products pp
SET field_sources = COALESCE(field_sources, '{}'::jsonb) - 'enhance_input_hash',
localized_content = CASE
WHEN localized_content IS NULL OR localized_content = '{}'::jsonb THEN localized_content
ELSE (
SELECT COALESCE(jsonb_object_agg(lang, val - 'enhance_input_hash'), '{}'::jsonb)
FROM jsonb_each(localized_content) AS t(lang, val)
)
END,
updated_at = now()
FROM raw_products rp
WHERE pp.company_id = $1
AND pp.raw_product_id = rp.id
AND rp.gtin = ANY($2::text[])`, companyID, eans)
if cerr2 != nil {
log.Printf("clear enhance hash: %v", cerr2)
} else {
log.Printf("cleared enhance_input_hash rows=%d", tag.RowsAffected())
}
if n, err := processing.BackfillMissingMeta(ctx, pool, companyID); err != nil {
log.Printf("BackfillMissingMeta: %v", err)
} else {
log.Printf("BackfillMissingMeta updated=%d", n)
}
_, _ = pool.Exec(ctx, `
UPDATE processing_jobs
SET status = 'failed', error = 'yielded to live enhance examples batch A', updated_at = now()
WHERE company_id = $1 AND status IN ('pending','running','processing')`, companyID)
jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full")
if err != nil {
log.Fatalf("StartJob: %v", err)
}
if len(jobs) == 0 {
log.Fatal("StartJob returned no jobs")
}
jobID := jobs[0].ID
log.Printf("job_id=%s chunks=%d", jobID, len(jobs))
_ = os.WriteFile(filepath.Join(out, "process_id.txt"), []byte(jobID.String()+"\n"), 0o644)
defend := make(chan struct{})
go func() {
t := time.NewTicker(500 * time.Millisecond)
defer t.Stop()
for {
select {
case <-defend:
return
case <-ctx.Done():
return
case <-t.C:
_, _ = pool.Exec(context.Background(), `
UPDATE processing_jobs
SET status = 'failed', error = 'yielded to live enhance examples batch A', updated_at = now()
WHERE company_id = $1 AND status IN ('pending','running','processing') AND id <> $2`, companyID, jobID)
_, _ = pool.Exec(context.Background(), `
UPDATE processing_jobs
SET status = 'running', error = NULL, updated_at = now()
WHERE id = $1 AND status IN ('failed','cancelled')
AND (error ILIKE '%yield%' OR error ILIKE '%parked%' OR error ILIKE '%reclaim%'
OR error ILIKE '%batch_%' OR error ILIKE '%enhance examples%' OR error ILIKE '%formula%')`, jobID)
_, _ = pool.Exec(context.Background(), `
UPDATE processing_job_products
SET status = 'pending', error = NULL, updated_at = now()
WHERE job_id = $1 AND status IN ('failed','cancelled')
AND processed_product_id IS NULL
AND (error ILIKE '%yield%' OR error ILIKE '%parked%' OR error ILIKE '%reclaim%'
OR error ILIKE '%batch_%' OR error ILIKE '%enhance examples%' OR error ILIKE '%formula%')`, jobID)
}
}
}()
_, _ = pool.Exec(ctx, `
UPDATE processing_jobs
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
WHERE id = $1`, jobID)
runCtx, runCancel := context.WithTimeout(ctx, 25*time.Minute)
defer runCancel()
if err := pipeline.ProcessJob(runCtx, jobID); err != nil {
close(defend)
log.Fatalf("ProcessJob: %v", err)
}
close(defend)
items, err := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
if err != nil {
log.Fatalf("LoadV1ProcessJobItems: %v", err)
}
writeJSON(out, "final.json", map[string]any{
"data": map[string]any{
"process_id": jobID.String(),
"status": "COMPLETED",
"processing_type": "full",
"total_items": len(items),
"items": items,
"llm": map[string]any{
"base": base,
"model": model,
"source": roleCfg.Source,
"mode": modeLabel,
"byok": byok,
},
},
})
rows, err := pool.Query(ctx, `
SELECT rp.gtin,
COALESCE(pp.processed_name, pp.name, ''),
COALESCE(NULLIF(pp.processed_description, ''), pp.description, ''),
COALESCE(pp.category, ''),
COALESCE(c.name, ''),
COALESCE(pp.meta_title, ''),
COALESCE(pp.meta_description, ''),
COALESCE(pp.processed_attributes, pp.attributes, '{}'::jsonb),
COALESCE(pp.ai_provider_mode, ''),
c.description_template,
c.title_template,
pp.id,
COALESCE(pp.field_sources, '{}'::jsonb)
FROM processing_job_products pjp
JOIN raw_products rp ON rp.id = pjp.raw_product_id
LEFT JOIN processed_products pp ON pp.id = pjp.processed_product_id
LEFT JOIN categories c ON c.company_id = pp.company_id AND c.unique_id = pp.category
WHERE pjp.job_id = $1
ORDER BY rp.gtin`, jobID)
if err != nil {
log.Fatalf("db products: %v", err)
}
defer rows.Close()
dbProducts := make([]dbProduct, 0)
for rows.Next() {
var p dbProduct
var ppID *uuid.UUID
var attrs, sources []byte
if err := rows.Scan(&p.EAN, &p.Title, &p.Description, &p.Category, &p.CategoryName,
&p.MetaTitle, &p.MetaDescription, &attrs, &p.AIProviderMode,
&p.DescTemplate, &p.TitleTemplate, &ppID, &sources); err != nil {
log.Fatal(err)
}
if ppID != nil {
p.ProductID = ppID.String()
}
p.Label = eanLabels[p.EAN]
p.Attributes = attrs
p.FieldSources = sources
p.FormulaConstraint = processing.FormatDescriptionFormulaConstraint(p.DescTemplate)
dbProducts = append(dbProducts, p)
}
writeJSON(out, "db_products.json", dbProducts)
scoreAndWrite(out, jobID.String(), resolveInfo, dbProducts)
fmt.Printf("BATCH_A JOB %s products=%d live_base=%s model=%s\n", jobID, len(dbProducts), base, model)
}
func waitForeignJobs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) {
deadline := time.Now().Add(18 * time.Minute)
for time.Now().Before(deadline) {
var n int
err := pool.QueryRow(ctx, `
SELECT count(*) FROM processing_jobs
WHERE company_id = $1 AND status IN ('pending','running','processing')`, companyID).Scan(&n)
if err != nil {
log.Printf("wait foreign jobs: %v", err)
return
}
if n == 0 {
log.Printf("queue clear — starting batch A")
return
}
log.Printf("waiting for %d foreign Demo job(s) before batch A…", n)
select {
case <-ctx.Done():
return
case <-time.After(15 * time.Second):
}
}
log.Printf("wait deadline reached — proceeding (will yield foreign jobs)")
}
func scoreAndWrite(out, jobID string, resolve map[string]any, products []dbProduct) {
scored := make([]map[string]any, 0, len(products))
pass, fail := 0, 0
var md strings.Builder
md.WriteString("# A1 enhance examples — batch A (rich category formulas; includes Slusalke/48)\n\n")
md.WriteString(fmt.Sprintf("- scored_at_utc: `%s`\n", time.Now().UTC().Format(time.RFC3339)))
md.WriteString(fmt.Sprintf("- job_id: `%s`\n", jobID))
md.WriteString(fmt.Sprintf("- live_base: `%v` model: `%v` source: `%v`\n", resolve["completer_base"], resolve["completer_model"], resolve["role_source"]))
md.WriteString(fmt.Sprintf("- mock_fallback: **false** (env mock ignored when DB admin AI resolves)\n\n"))
md.WriteString("Scoring: title quality · HTML formula tags (h1/h2/p/ul) · meta · attrs\n\n")
for _, p := range products {
title := strings.TrimSpace(p.Title)
desc := p.Description
metaT := strings.TrimSpace(p.MetaTitle)
metaD := strings.TrimSpace(p.MetaDescription)
has := map[string]bool{
"h1": strings.Contains(strings.ToLower(desc), "<h1"),
"h2": strings.Contains(strings.ToLower(desc), "<h2"),
"p": strings.Contains(strings.ToLower(desc), "<p"),
"ul": strings.Contains(strings.ToLower(desc), "<ul"),
}
htmlOK := has["h1"] && has["h2"] && has["p"] && has["ul"]
titleOK := title != "" && utf8.RuneCountInString(title) >= 8 && !brandOnlyish.MatchString(title)
titleNotes := []string{}
if title == "" {
titleNotes = append(titleNotes, "empty")
} else if brandOnlyish.MatchString(title) {
titleNotes = append(titleNotes, "brand_only")
} else if utf8.RuneCountInString(title) < 8 {
titleNotes = append(titleNotes, "too_short")
}
if strings.Contains(strings.ToLower(title), "title formula") || strings.Contains(strings.ToLower(title), "category:") {
titleOK = false
titleNotes = append(titleNotes, "prompt_leakage")
}
metaOK := metaT != "" && metaD != "" && !poisonMeta.MatchString(metaT) &&
!strings.Contains(strings.ToLower(metaT), "short retail title")
metaNotes := []string{}
if metaT == "" {
metaNotes = append(metaNotes, "empty_meta_title")
}
if metaD == "" {
metaNotes = append(metaNotes, "empty_meta_description")
}
if poisonMeta.MatchString(metaT) {
metaNotes = append(metaNotes, "poison_|digits")
}
attrMap := map[string]any{}
_ = json.Unmarshal(p.Attributes, &attrMap)
attrKeys := make([]string, 0, len(attrMap))
dirty := []string{}
for k, v := range attrMap {
attrKeys = append(attrKeys, k)
sv := strings.TrimSpace(fmt.Sprint(v))
if sv == "" || strings.EqualFold(sv, "null") || strings.EqualFold(sv, "n/a") {
dirty = append(dirty, k+":empty")
}
if strings.Contains(strings.ToLower(k), "ean") || strings.Contains(strings.ToLower(k), "gtin") {
dirty = append(dirty, k+":id_like_key")
}
}
attrsOK := len(attrKeys) > 0 && len(dirty) == 0
checks := map[string]string{
"title": passFail(titleOK),
"html_tags": passFail(htmlOK),
"meta": passFail(metaOK),
"attrs": passFail(attrsOK),
"category": passFail(p.Category != "" && p.CategoryName != ""),
"desc_ne_title": passFail(strings.TrimSpace(stripTags(desc)) != "" && strings.TrimSpace(stripTags(desc)) != title),
"ai_internal": passFail(strings.EqualFold(p.AIProviderMode, "internal") || p.AIProviderMode == ""),
}
failFlags := []string{}
for k, v := range checks {
if v == "FAIL" {
failFlags = append(failFlags, k)
}
}
verdict := "PASS"
if len(failFlags) > 0 {
verdict = "FAIL"
fail++
} else {
pass++
}
row := map[string]any{
"ean": p.EAN,
"label": p.Label,
"product_id": p.ProductID,
"title": title,
"category": p.Category,
"category_name": p.CategoryName,
"desc_len": len(desc),
"desc_preview": truncate(stripTags(desc), 140),
"meta_title": metaT,
"meta_description": truncate(metaD, 160),
"attr_keys": attrKeys,
"dirty_attrs": dirty,
"attributes": attrMap,
"has_tags": has,
"formula_constraint": p.FormulaConstraint,
"formula_override_hint": strings.Contains(strings.ToLower(fmt.Sprint(p.FieldSources)), "formula"),
"ai_provider_mode": p.AIProviderMode,
"title_notes": titleNotes,
"meta_notes": metaNotes,
"checks": checks,
"fail_flags": failFlags,
"verdict": verdict,
}
scored = append(scored, row)
md.WriteString(fmt.Sprintf("## %s — %s (`%s`)\n\n", p.EAN, p.Label, verdict))
md.WriteString(fmt.Sprintf("- **title:** %s\n", title))
md.WriteString(fmt.Sprintf("- **category:** %s / %s\n", p.Category, p.CategoryName))
md.WriteString(fmt.Sprintf("- **meta_title:** %s\n", metaT))
md.WriteString(fmt.Sprintf("- **meta_description:** %s\n", truncate(metaD, 200)))
md.WriteString(fmt.Sprintf("- **attrs:** %s\n", strings.Join(attrKeys, ", ")))
md.WriteString(fmt.Sprintf("- **HTML tags:** h1=%v h2=%v p=%v ul=%v\n", has["h1"], has["h2"], has["p"], has["ul"]))
md.WriteString(fmt.Sprintf("- **checks:** title=%s html=%s meta=%s attrs=%s\n\n", checks["title"], checks["html_tags"], checks["meta"], checks["attrs"]))
md.WriteString("### Description HTML (raw)\n\n```html\n")
if len(desc) > 4096 {
md.WriteString(desc[:4096])
md.WriteString("\n<!-- truncated -->\n")
} else {
md.WriteString(desc)
md.WriteString("\n")
}
md.WriteString("```\n\n")
}
total := len(products)
rate := 0
if total > 0 {
rate = (pass * 100) / total
}
scorecard := map[string]any{
"batch": "A",
"date": "2026-08-16",
"scored_at_utc": time.Now().UTC().Format(time.RFC3339),
"company_id": "2b3159b0-fc08-415b-b248-35ed02a6baab",
"company_name": "Platform Demo",
"job_id": jobID,
"selection": eanLabels,
"live_llm": resolve,
"mock_as_live": false,
"asserts": []string{"title_quality", "html_h1_h2_p_ul", "meta", "attrs", "category", "desc_ne_title"},
"summary": map[string]any{"pass": pass, "fail": fail, "total": total, "pass_rate": rate},
"verdict": map[bool]string{true: "PASS", false: "FAIL"}[fail == 0 && total > 0],
"products": scored,
"max_tokens_note": fmt.Sprintf("MaxTokensEnhance=%d MaxTokensEnhanceRetry=%d", processing.MaxTokensEnhance, processing.MaxTokensEnhanceRetry),
}
writeJSON(out, "scorecard.json", scorecard)
_ = os.WriteFile(filepath.Join(out, "examples.md"), []byte(md.String()), 0o644)
summary := fmt.Sprintf("# Batch A summary\n\n- verdict: **%s** (%d/%d)\n- live: `%v` / `%v`\n- mock_as_live: false\n- artifacts: scorecard.json, examples.md, db_products.json, final.json, resolve.json\n",
scorecard["verdict"], pass, total, resolve["completer_base"], resolve["completer_model"])
_ = os.WriteFile(filepath.Join(out, "summary.md"), []byte(summary), 0o644)
}
func passFail(ok bool) string {
if ok {
return "PASS"
}
return "FAIL"
}
func stripTags(s string) string {
re := regexp.MustCompile(`<[^>]+>`)
return strings.TrimSpace(re.ReplaceAllString(s, " "))
}
func truncate(s string, n int) string {
r := []rune(strings.TrimSpace(s))
if len(r) <= n {
return string(r)
}
return string(r[:n]) + "…"
}
func failResolve(out, msg string) {
writeJSON(out, "resolve.json", map[string]any{"live": false, "fail_reason": msg, "batch": "A"})
log.Fatalf("LIVE LLM REQUIRED: %s", msg)
}
func writeJSON(out, name string, v any) {
raw, err := json.MarshalIndent(v, "", " ")
if err != nil {
log.Fatalf("marshal %s: %v", name, err)
}
if err := os.WriteFile(filepath.Join(out, name), raw, 0o644); err != nil {
log.Fatalf("write %s: %v", name, err)
}
}
@@ -0,0 +1,664 @@
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"os"
"os/signal"
"path/filepath"
"regexp"
"strings"
"syscall"
"time"
"unicode/utf8"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// Batch B: appliances + monitors + mounts — independent of batch A.
var eans = []string{
"4242005342488", // 41 washer-dryer
"4242005382330", // 38 dishwasher
"4242005327065", // 36 oven
"4719331854218", // 7 monitor GIGABYTE
"6941948703193", // 7 monitor Xiaomi
"8712285326882", // 28 TV mount
}
var eanLabels = map[string]string{
"4242005342488": "appliance washer-dryer",
"4242005382330": "appliance dishwasher",
"4242005327065": "appliance oven",
"4719331854218": "monitor GIGABYTE",
"6941948703193": "monitor Xiaomi",
"8712285326882": "mount Vogels",
}
var poisonMeta = regexp.MustCompile(`\|\s*\d+\s*$`)
var brandOnlyish = regexp.MustCompile(`(?i)^(bosch|gorenje|gigabyte|xiaomi|lenovo|samsung|lg|anker|jbl|vogels|ostalo)$`)
type dbProduct struct {
EAN string `json:"ean"`
Label string `json:"label"`
ProductID string `json:"product_id"`
Title string `json:"title"`
Description string `json:"description"`
Category string `json:"category"`
CategoryName string `json:"category_name"`
MetaTitle string `json:"meta_title"`
MetaDescription string `json:"meta_description"`
Attributes json.RawMessage `json:"attributes"`
AIProviderMode string `json:"ai_provider_mode"`
DescTemplate any `json:"description_template"`
TitleTemplate any `json:"title_template"`
FormulaConstraint string `json:"formula_constraint"`
FieldSources json.RawMessage `json:"field_sources"`
}
func main() {
out := os.Getenv("PROBE_OUT_DIR")
if out == "" {
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_a1_enhance_examples_B_20260816`
}
_ = os.MkdirAll(out, 0o755)
logPath := filepath.Join(out, "inproc_run.log")
logFile, err := os.Create(logPath)
if err != nil {
log.Fatalf("log file: %v", err)
}
defer logFile.Close()
log.SetOutput(logredact.Writer(io.MultiWriter(os.Stderr, logFile)))
companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
userID := uuid.MustParse("0ce3305c-d810-4b56-b1d4-3c1ed510db76")
cfg, err := config.Load()
if err != nil {
log.Fatalf("config: %v", err)
}
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer cancel()
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
MaxConns: int32(cfg.DBMaxConns),
MinConns: int32(cfg.DBMinConns),
MaxConnLifetime: cfg.DBMaxConnLifetime,
MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
MaxConnIdleTime: cfg.DBMaxConnIdleTime,
HealthCheckPeriod: cfg.DBHealthCheckPeriod,
StatementTimeout: cfg.DBStatementTimeout,
})
if err != nil {
log.Fatalf("db: %v", err)
}
defer pool.Close()
waitForeignJobs(ctx, pool, companyID)
platEnv := platformsettings.EnvConfig{
AppEncryptionKey: cfg.AppEncryptionKey,
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
TokenSigningSecret: cfg.TokenSigningSecret,
DatabaseURL: cfg.DatabaseURL,
OpenAIAPIKey: cfg.OpenAIAPIKey,
OpenAIBaseURL: cfg.OpenAIBaseURL,
OpenAIModel: cfg.OpenAIModel,
OpenAIEmbeddingAPIKey: cfg.OpenAIEmbeddingAPIKey,
OpenAIEmbeddingBaseURL: cfg.OpenAIEmbeddingBaseURL,
OpenAIEmbeddingModel: cfg.OpenAIEmbeddingModel,
}
platSettings := platformsettings.NewService(pool, platEnv)
aiSvc := aiprovider.NewService(pool, aiprovider.EnvConfig{
AppEncryptionKey: cfg.AppEncryptionKey,
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
TokenSigningSecret: cfg.TokenSigningSecret,
DatabaseURL: cfg.DatabaseURL,
OpenAIAPIKey: cfg.OpenAIAPIKey,
OpenAIBaseURL: cfg.OpenAIBaseURL,
OpenAIModel: cfg.OpenAIModel,
ProcessingRPM: cfg.ProcessingRPM,
ProcessingMaxRetries: cfg.ProcessingMaxRetries,
})
aiSvc.Platform = platSettings
roleCfg, rerr := platSettings.ResolveAIConfig(ctx, platformsettings.AIRoleProcessing)
if rerr != nil {
failResolve(out, fmt.Sprintf("ResolveAIConfig: %v", rerr))
}
completer, modeLabel, byok, cerr := aiSvc.ResolveCompleterForRole(ctx, companyID, aiprovider.RoleProcessing)
if cerr != nil {
failResolve(out, fmt.Sprintf("ResolveCompleterForRole: %v", cerr))
}
if completer == nil {
failResolve(out, "nil completer")
}
client, ok := completer.(*processing.OpenAIClient)
if !ok {
failResolve(out, fmt.Sprintf("completer type %T", completer))
}
base := strings.TrimSpace(client.BaseURL)
model := strings.TrimSpace(client.Model)
resolveInfo := map[string]any{
"batch": "B",
"eans": eans,
"role_source": roleCfg.Source,
"role_provider": roleCfg.Provider,
"role_base_url": roleCfg.BaseURL,
"role_model": roleCfg.Model,
"role_enabled": roleCfg.Enabled,
"completer_base": base,
"completer_model": model,
"mode_label": modeLabel,
"byok": byok,
"env_base_url": cfg.OpenAIBaseURL,
"env_model": cfg.OpenAIModel,
"live": true,
}
if processing.IsMockOrLoopbackBaseURL(base) || strings.Contains(strings.ToLower(base), "18767") {
resolveInfo["live"] = false
resolveInfo["fail_reason"] = "resolved completer is mock/loopback — refuse silent mock fallback"
writeJSON(out, "resolve.json", resolveInfo)
log.Fatalf("LIVE LLM REQUIRED: resolved base=%s model=%s — refuse mock", base, model)
}
if strings.TrimSpace(client.APIKey) == "" {
resolveInfo["live"] = false
resolveInfo["fail_reason"] = "empty API key"
writeJSON(out, "resolve.json", resolveInfo)
log.Fatal("LIVE LLM REQUIRED: API key empty")
}
keyHint := ""
if k := strings.TrimSpace(client.APIKey); len(k) > 8 {
keyHint = k[:4] + "…" + k[len(k)-4:]
}
resolveInfo["api_key_hint"] = keyHint
modelsURL := strings.TrimRight(base, "/") + "/models"
modelsCtx, modelsCancel := context.WithTimeout(ctx, 30*time.Second)
defer modelsCancel()
modelsReq, merr := http.NewRequestWithContext(modelsCtx, http.MethodGet, modelsURL, nil)
if merr != nil {
failResolve(out, "models req: "+merr.Error())
}
modelsReq.Header.Set("Authorization", "Bearer "+client.APIKey)
modelsReq.Header.Set("Accept", "application/json")
modelsStart := time.Now()
modelsResp, merr := http.DefaultClient.Do(modelsReq)
modelsElapsed := time.Since(modelsStart).Round(time.Millisecond)
if merr != nil {
resolveInfo["probe_ok"] = false
resolveInfo["models_error"] = processing.TruncateError(merr)
writeJSON(out, "resolve.json", resolveInfo)
log.Fatalf("LIVE LLM CONNECTION ERROR: GET models: %v", merr)
}
modelsBody, _ := io.ReadAll(io.LimitReader(modelsResp.Body, 2048))
_ = modelsResp.Body.Close()
resolveInfo["models_url"] = modelsURL
resolveInfo["models_http_status"] = modelsResp.StatusCode
resolveInfo["models_elapsed"] = modelsElapsed.String()
snippet := strings.TrimSpace(string(modelsBody))
if len(snippet) > 200 {
snippet = snippet[:200] + "…"
}
resolveInfo["models_body_snippet"] = snippet
if modelsResp.StatusCode != http.StatusOK {
resolveInfo["probe_ok"] = false
writeJSON(out, "resolve.json", resolveInfo)
log.Fatalf("LIVE LLM CONNECTION ERROR: models HTTP %d", modelsResp.StatusCode)
}
probeCtx, probeCancel := context.WithTimeout(ctx, 3*time.Minute)
defer probeCancel()
probeStart := time.Now()
comp, perr := client.Complete(probeCtx, "Reply with exactly: PONG", "Say PONG")
probeElapsed := time.Since(probeStart).Round(time.Millisecond)
if perr != nil {
// One soft retry — OverloadedBot often needs a cool-down after a prior batch.
log.Printf("chat probe first attempt failed elapsed=%s err=%s — retrying once after 20s", probeElapsed, processing.TruncateError(perr))
select {
case <-ctx.Done():
resolveInfo["probe_ok"] = false
resolveInfo["probe_error"] = processing.TruncateError(perr)
writeJSON(out, "resolve.json", resolveInfo)
log.Fatalf("LIVE LLM CONNECTION ERROR: chat probe: %v", perr)
case <-time.After(20 * time.Second):
}
probeCtx2, probeCancel2 := context.WithTimeout(ctx, 3*time.Minute)
defer probeCancel2()
probeStart = time.Now()
comp, perr = client.Complete(probeCtx2, "Reply with exactly: PONG", "Say PONG")
probeElapsed = time.Since(probeStart).Round(time.Millisecond)
if perr != nil {
resolveInfo["probe_ok"] = false
resolveInfo["probe_error"] = processing.TruncateError(perr)
resolveInfo["models_http_status"] = modelsResp.StatusCode
writeJSON(out, "resolve.json", resolveInfo)
log.Fatalf("LIVE LLM CONNECTION ERROR: chat probe (after retry): %v", perr)
}
}
resolveInfo["probe_ok"] = true
resolveInfo["probe_elapsed"] = probeElapsed.String()
resolveInfo["probe_response_len"] = len(comp.Text)
resolveInfo["started_at_utc"] = time.Now().UTC().Format(time.RFC3339)
writeJSON(out, "resolve.json", resolveInfo)
log.Printf("live resolve ok source=%s base=%s model=%s mode=%s byok=%v", roleCfg.Source, base, model, modeLabel, byok)
pipeline := processing.NewPipeline(pool)
pipeline.BatchSize = cfg.ProcessingBatchSize
pipeline.AI = aiSvc
pipeline.Prompts = aiprompts.NewService(pool)
eprelClient := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second})
pipeline.Engine = &processing.Engine{
Completer: nil,
Vector: processing.NoopVectorCategorizer{},
EPREL: eprelClient,
ProviderMode: processing.AIProviderInternal,
}
rawIDs := make([]uuid.UUID, 0, len(eans))
for _, ean := range eans {
var id uuid.UUID
if err := pool.QueryRow(ctx, `
SELECT id FROM raw_products
WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id); err != nil {
log.Fatalf("raw product %s: %v", ean, err)
}
rawIDs = append(rawIDs, id)
log.Printf("ean=%s raw_id=%s label=%s", ean, id, eanLabels[ean])
}
tag, cerr2 := pool.Exec(ctx, `
UPDATE processed_products pp
SET field_sources = COALESCE(field_sources, '{}'::jsonb) - 'enhance_input_hash',
localized_content = CASE
WHEN localized_content IS NULL OR localized_content = '{}'::jsonb THEN localized_content
ELSE (
SELECT COALESCE(jsonb_object_agg(lang, val - 'enhance_input_hash'), '{}'::jsonb)
FROM jsonb_each(localized_content) AS t(lang, val)
)
END,
updated_at = now()
FROM raw_products rp
WHERE pp.company_id = $1
AND pp.raw_product_id = rp.id
AND rp.gtin = ANY($2::text[])`, companyID, eans)
if cerr2 != nil {
log.Printf("clear enhance hash: %v", cerr2)
} else {
log.Printf("cleared enhance_input_hash rows=%d", tag.RowsAffected())
}
if n, err := processing.BackfillMissingMeta(ctx, pool, companyID); err != nil {
log.Printf("BackfillMissingMeta: %v", err)
} else {
log.Printf("BackfillMissingMeta updated=%d", n)
}
_, _ = pool.Exec(ctx, `
UPDATE processing_jobs
SET status = 'failed', error = 'yielded to live enhance examples batch B', updated_at = now()
WHERE company_id = $1 AND status IN ('pending','running','processing')`, companyID)
jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full")
if err != nil {
log.Fatalf("StartJob: %v", err)
}
if len(jobs) == 0 {
log.Fatal("StartJob returned no jobs")
}
jobID := jobs[0].ID
log.Printf("job_id=%s chunks=%d", jobID, len(jobs))
_ = os.WriteFile(filepath.Join(out, "process_id.txt"), []byte(jobID.String()+"\n"), 0o644)
defend := make(chan struct{})
go func() {
t := time.NewTicker(500 * time.Millisecond)
defer t.Stop()
for {
select {
case <-defend:
return
case <-ctx.Done():
return
case <-t.C:
_, _ = pool.Exec(context.Background(), `
UPDATE processing_jobs
SET status = 'failed', error = 'yielded to live enhance examples batch B', updated_at = now()
WHERE company_id = $1 AND status IN ('pending','running','processing') AND id <> $2`, companyID, jobID)
_, _ = pool.Exec(context.Background(), `
UPDATE processing_jobs
SET status = 'running', error = NULL, updated_at = now()
WHERE id = $1 AND status IN ('failed','cancelled')
AND (error ILIKE '%yield%' OR error ILIKE '%parked%' OR error ILIKE '%reclaim%'
OR error ILIKE '%batch_%' OR error ILIKE '%enhance examples%' OR error ILIKE '%formula%')`, jobID)
_, _ = pool.Exec(context.Background(), `
UPDATE processing_job_products
SET status = 'pending', error = NULL, updated_at = now()
WHERE job_id = $1 AND status IN ('failed','cancelled')
AND processed_product_id IS NULL
AND (error ILIKE '%yield%' OR error ILIKE '%parked%' OR error ILIKE '%reclaim%'
OR error ILIKE '%batch_%' OR error ILIKE '%enhance examples%' OR error ILIKE '%formula%')`, jobID)
}
}
}()
_, _ = pool.Exec(ctx, `
UPDATE processing_jobs
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
WHERE id = $1`, jobID)
runCtx, runCancel := context.WithTimeout(ctx, 25*time.Minute)
defer runCancel()
if err := pipeline.ProcessJob(runCtx, jobID); err != nil {
close(defend)
log.Fatalf("ProcessJob: %v", err)
}
close(defend)
items, err := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
if err != nil {
log.Fatalf("LoadV1ProcessJobItems: %v", err)
}
writeJSON(out, "final.json", map[string]any{
"data": map[string]any{
"process_id": jobID.String(),
"status": "COMPLETED",
"processing_type": "full",
"total_items": len(items),
"items": items,
"llm": map[string]any{
"base": base,
"model": model,
"source": roleCfg.Source,
"mode": modeLabel,
"byok": byok,
},
},
})
rows, err := pool.Query(ctx, `
SELECT rp.gtin,
COALESCE(pp.processed_name, pp.name, ''),
COALESCE(NULLIF(pp.processed_description, ''), pp.description, ''),
COALESCE(pp.category, ''),
COALESCE(c.name, ''),
COALESCE(pp.meta_title, ''),
COALESCE(pp.meta_description, ''),
COALESCE(pp.processed_attributes, pp.attributes, '{}'::jsonb),
COALESCE(pp.ai_provider_mode, ''),
c.description_template,
c.title_template,
pp.id,
COALESCE(pp.field_sources, '{}'::jsonb)
FROM processing_job_products pjp
JOIN raw_products rp ON rp.id = pjp.raw_product_id
LEFT JOIN processed_products pp ON pp.id = pjp.processed_product_id
LEFT JOIN categories c ON c.company_id = pp.company_id AND c.unique_id = pp.category
WHERE pjp.job_id = $1
ORDER BY rp.gtin`, jobID)
if err != nil {
log.Fatalf("db products: %v", err)
}
defer rows.Close()
dbProducts := make([]dbProduct, 0)
for rows.Next() {
var p dbProduct
var ppID *uuid.UUID
var attrs, sources []byte
if err := rows.Scan(&p.EAN, &p.Title, &p.Description, &p.Category, &p.CategoryName,
&p.MetaTitle, &p.MetaDescription, &attrs, &p.AIProviderMode,
&p.DescTemplate, &p.TitleTemplate, &ppID, &sources); err != nil {
log.Fatal(err)
}
if ppID != nil {
p.ProductID = ppID.String()
}
p.Label = eanLabels[p.EAN]
p.Attributes = attrs
p.FieldSources = sources
p.FormulaConstraint = processing.FormatDescriptionFormulaConstraint(p.DescTemplate)
dbProducts = append(dbProducts, p)
}
writeJSON(out, "db_products.json", dbProducts)
scoreAndWrite(out, jobID.String(), resolveInfo, dbProducts)
fmt.Printf("BATCH_B JOB %s products=%d live_base=%s model=%s\n", jobID, len(dbProducts), base, model)
}
func waitForeignJobs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) {
deadline := time.Now().Add(18 * time.Minute)
for time.Now().Before(deadline) {
var n int
err := pool.QueryRow(ctx, `
SELECT count(*) FROM processing_jobs
WHERE company_id = $1 AND status IN ('pending','running','processing')`, companyID).Scan(&n)
if err != nil {
log.Printf("wait foreign jobs: %v", err)
return
}
if n == 0 {
log.Printf("queue clear — starting batch B")
return
}
log.Printf("waiting for %d foreign Demo job(s) before batch B…", n)
select {
case <-ctx.Done():
return
case <-time.After(15 * time.Second):
}
}
log.Printf("wait deadline reached — proceeding (will yield foreign jobs)")
}
func scoreAndWrite(out, jobID string, resolve map[string]any, products []dbProduct) {
scored := make([]map[string]any, 0, len(products))
pass, fail := 0, 0
var md strings.Builder
md.WriteString("# A1 enhance examples — batch B (appliances / monitors / mounts)\n\n")
md.WriteString(fmt.Sprintf("- scored_at_utc: `%s`\n", time.Now().UTC().Format(time.RFC3339)))
md.WriteString(fmt.Sprintf("- job_id: `%s`\n", jobID))
md.WriteString(fmt.Sprintf("- live_base: `%v` model: `%v` source: `%v`\n", resolve["completer_base"], resolve["completer_model"], resolve["role_source"]))
md.WriteString(fmt.Sprintf("- mock_fallback: **false** (env mock ignored when DB admin AI resolves)\n\n"))
md.WriteString("Scoring: title quality · HTML formula tags (h1/h2/p/ul) · meta · attrs\n\n")
for _, p := range products {
title := strings.TrimSpace(p.Title)
desc := p.Description
metaT := strings.TrimSpace(p.MetaTitle)
metaD := strings.TrimSpace(p.MetaDescription)
has := map[string]bool{
"h1": strings.Contains(strings.ToLower(desc), "<h1"),
"h2": strings.Contains(strings.ToLower(desc), "<h2"),
"p": strings.Contains(strings.ToLower(desc), "<p"),
"ul": strings.Contains(strings.ToLower(desc), "<ul"),
}
htmlOK := has["h1"] && has["h2"] && has["p"] && has["ul"]
titleOK := title != "" && utf8.RuneCountInString(title) >= 8 && !brandOnlyish.MatchString(title)
titleNotes := []string{}
if title == "" {
titleNotes = append(titleNotes, "empty")
} else if brandOnlyish.MatchString(title) {
titleNotes = append(titleNotes, "brand_only")
} else if utf8.RuneCountInString(title) < 8 {
titleNotes = append(titleNotes, "too_short")
}
if strings.Contains(strings.ToLower(title), "title formula") || strings.Contains(strings.ToLower(title), "category:") {
titleOK = false
titleNotes = append(titleNotes, "prompt_leakage")
}
metaOK := metaT != "" && metaD != "" && !poisonMeta.MatchString(metaT) &&
!strings.Contains(strings.ToLower(metaT), "short retail title")
metaNotes := []string{}
if metaT == "" {
metaNotes = append(metaNotes, "empty_meta_title")
}
if metaD == "" {
metaNotes = append(metaNotes, "empty_meta_description")
}
if poisonMeta.MatchString(metaT) {
metaNotes = append(metaNotes, "poison_|digits")
}
attrMap := map[string]any{}
_ = json.Unmarshal(p.Attributes, &attrMap)
attrKeys := make([]string, 0, len(attrMap))
dirty := []string{}
for k, v := range attrMap {
attrKeys = append(attrKeys, k)
sv := strings.TrimSpace(fmt.Sprint(v))
if sv == "" || strings.EqualFold(sv, "null") || strings.EqualFold(sv, "n/a") {
dirty = append(dirty, k+":empty")
}
if strings.Contains(strings.ToLower(k), "ean") || strings.Contains(strings.ToLower(k), "gtin") {
dirty = append(dirty, k+":id_like_key")
}
}
attrsOK := len(attrKeys) > 0 && len(dirty) == 0
checks := map[string]string{
"title": passFail(titleOK),
"html_tags": passFail(htmlOK),
"meta": passFail(metaOK),
"attrs": passFail(attrsOK),
"category": passFail(p.Category != "" && p.CategoryName != ""),
"desc_ne_title": passFail(strings.TrimSpace(stripTags(desc)) != "" && strings.TrimSpace(stripTags(desc)) != title),
"ai_internal": passFail(strings.EqualFold(p.AIProviderMode, "internal") || p.AIProviderMode == ""),
}
failFlags := []string{}
for k, v := range checks {
if v == "FAIL" {
failFlags = append(failFlags, k)
}
}
verdict := "PASS"
if len(failFlags) > 0 {
verdict = "FAIL"
fail++
} else {
pass++
}
row := map[string]any{
"ean": p.EAN,
"label": p.Label,
"product_id": p.ProductID,
"title": title,
"category": p.Category,
"category_name": p.CategoryName,
"desc_len": len(desc),
"desc_preview": truncate(stripTags(desc), 140),
"meta_title": metaT,
"meta_description": truncate(metaD, 160),
"attr_keys": attrKeys,
"dirty_attrs": dirty,
"attributes": attrMap,
"has_tags": has,
"formula_constraint": p.FormulaConstraint,
"formula_override_hint": strings.Contains(strings.ToLower(fmt.Sprint(p.FieldSources)), "formula"),
"ai_provider_mode": p.AIProviderMode,
"title_notes": titleNotes,
"meta_notes": metaNotes,
"checks": checks,
"fail_flags": failFlags,
"verdict": verdict,
}
scored = append(scored, row)
md.WriteString(fmt.Sprintf("## %s — %s (`%s`)\n\n", p.EAN, p.Label, verdict))
md.WriteString(fmt.Sprintf("- **title:** %s\n", title))
md.WriteString(fmt.Sprintf("- **category:** %s / %s\n", p.Category, p.CategoryName))
md.WriteString(fmt.Sprintf("- **meta_title:** %s\n", metaT))
md.WriteString(fmt.Sprintf("- **meta_description:** %s\n", truncate(metaD, 200)))
md.WriteString(fmt.Sprintf("- **attrs:** %s\n", strings.Join(attrKeys, ", ")))
md.WriteString(fmt.Sprintf("- **HTML tags:** h1=%v h2=%v p=%v ul=%v\n", has["h1"], has["h2"], has["p"], has["ul"]))
md.WriteString(fmt.Sprintf("- **checks:** title=%s html=%s meta=%s attrs=%s\n\n", checks["title"], checks["html_tags"], checks["meta"], checks["attrs"]))
md.WriteString("### Description HTML (raw)\n\n```html\n")
if len(desc) > 4096 {
md.WriteString(desc[:4096])
md.WriteString("\n<!-- truncated -->\n")
} else {
md.WriteString(desc)
md.WriteString("\n")
}
md.WriteString("```\n\n")
}
total := len(products)
rate := 0
if total > 0 {
rate = (pass * 100) / total
}
scorecard := map[string]any{
"batch": "B",
"date": "2026-08-16",
"scored_at_utc": time.Now().UTC().Format(time.RFC3339),
"company_id": "2b3159b0-fc08-415b-b248-35ed02a6baab",
"company_name": "Platform Demo",
"job_id": jobID,
"selection": eanLabels,
"live_llm": resolve,
"mock_as_live": false,
"asserts": []string{"title_quality", "html_h1_h2_p_ul", "meta", "attrs", "category", "desc_ne_title"},
"summary": map[string]any{"pass": pass, "fail": fail, "total": total, "pass_rate": rate},
"verdict": map[bool]string{true: "PASS", false: "FAIL"}[fail == 0 && total > 0],
"products": scored,
"max_tokens_note": fmt.Sprintf("MaxTokensEnhance=%d MaxTokensEnhanceRetry=%d", processing.MaxTokensEnhance, processing.MaxTokensEnhanceRetry),
}
writeJSON(out, "scorecard.json", scorecard)
_ = os.WriteFile(filepath.Join(out, "examples.md"), []byte(md.String()), 0o644)
summary := fmt.Sprintf("# Batch B summary\n\n- verdict: **%s** (%d/%d)\n- live: `%v` / `%v`\n- mock_as_live: false\n- artifacts: scorecard.json, examples.md, db_products.json, final.json, resolve.json\n",
scorecard["verdict"], pass, total, resolve["completer_base"], resolve["completer_model"])
_ = os.WriteFile(filepath.Join(out, "summary.md"), []byte(summary), 0o644)
}
func passFail(ok bool) string {
if ok {
return "PASS"
}
return "FAIL"
}
func stripTags(s string) string {
re := regexp.MustCompile(`<[^>]+>`)
return strings.TrimSpace(re.ReplaceAllString(s, " "))
}
func truncate(s string, n int) string {
r := []rune(strings.TrimSpace(s))
if len(r) <= n {
return string(r)
}
return string(r[:n]) + "…"
}
func failResolve(out, msg string) {
writeJSON(out, "resolve.json", map[string]any{"live": false, "fail_reason": msg, "batch": "B"})
log.Fatalf("LIVE LLM REQUIRED: %s", msg)
}
func writeJSON(out, name string, v any) {
raw, err := json.MarshalIndent(v, "", " ")
if err != nil {
log.Fatalf("marshal %s: %v", name, err)
}
if err := os.WriteFile(filepath.Join(out, name), raw, 0o644); err != nil {
log.Fatalf("write %s: %v", name, err)
}
}
+19 -7
View File
@@ -1,13 +1,18 @@
// Command repair-category-prompts replaces legacy A1 / Platform Demo categories.prompt
// HTML marketing formulas with aiprompts.CategoryEnhanceUserTemplate (JSON-compatible
// user overlay: {{name}} {{description}} {{attrs}} {{category}} {{language}}; stores
// both prompt->>'sl' and "*"). Does not touch title_template / description_template
// (formulas are appended as plain-text instructions at enhance render time).
// Command repair-category-prompts rewrites A1 / Platform Demo categories.prompt
// values into aiprompts role-sectioned overlays (Title / Description / Meta /
// Attributes). Prefers wp_product_categories.sql (SEED_A1_WP_CATEGORIES /
// Downloads) as source of truth, else a1-category-prompts.json. Legacy combined
// Slovenian name+description blobs are split so naming rules land under Title
// and HTML under Description; otherwise CategoryEnhanceUserTemplate is used.
// Also repairs empty or brand-only title_template and empty description_template
// from legacy <name> / HTML / meta rules (never brand-only).
//
// Usage (from apps/api):
//
// go run ./cmd/repair-category-prompts -dry-run
// go run ./cmd/repair-category-prompts -apply
// go run ./cmd/repair-category-prompts -apply -wp-categories "D:/Users/.../Downloads/wp_product_categories.sql"
// go run ./cmd/repair-category-prompts -apply -prompts ../../scripts/seed/a1-category-prompts.json
//
// DATABASE_URL / -postgres required. Default is dry-run (count only).
package main
@@ -30,6 +35,8 @@ func main() {
postgresURL := flag.String("postgres", os.Getenv("DATABASE_URL"), "Postgres URL (DATABASE_URL)")
dryRun := flag.Bool("dry-run", true, "count rows that would update (default true)")
apply := flag.Bool("apply", false, "write updates (implies not dry-run)")
promptsPath := flag.String("prompts", "", "path to a1-category-prompts.json (optional fallback)")
wpPath := flag.String("wp-categories", os.Getenv("SEED_A1_WP_CATEGORIES"), "path to wp_product_categories.sql (preferred source of truth)")
flag.Parse()
if strings.TrimSpace(*postgresURL) == "" {
@@ -49,8 +56,13 @@ func main() {
}
defer pg.Close()
opts := catalog.RepairA1DemoOptions{
SeedPromptsPath: strings.TrimSpace(*promptsPath),
WPCategoriesPath: strings.TrimSpace(*wpPath),
}
// Always print dry-run counts first when applying, so operators see the delta.
preview, err := catalog.RepairA1DemoCategoryEnhancePrompts(ctx, pg, true)
preview, err := catalog.RepairA1DemoCategoryEnhancePromptsWithOptions(ctx, pg, true, opts)
if err != nil {
log.Fatalf("dry-run: %v", err)
}
@@ -60,7 +72,7 @@ func main() {
return
}
res, err := catalog.RepairA1DemoCategoryEnhancePrompts(ctx, pg, false)
res, err := catalog.RepairA1DemoCategoryEnhancePromptsWithOptions(ctx, pg, false, opts)
if err != nil {
log.Fatalf("apply: %v", err)
}
+136
View File
@@ -0,0 +1,136 @@
// Command seed-a1-teammate upserts a second A1 company member (not the owner)
// with must_set_password=true and prints a one-time accept-invite URL.
//
// Usage:
//
// go run ./cmd/seed-a1-teammate -postgres "$DATABASE_URL"
// go run ./cmd/seed-a1-teammate -email a1-other@descrybe.local -role member
package main
import (
"context"
"flag"
"fmt"
"log"
"os"
"strings"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/auth"
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
"github.com/descrybe/descrybe-v2/apps/api/internal/mail"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
func main() {
postgresURL := flag.String("postgres", os.Getenv("DATABASE_URL"), "Postgres URL")
email := flag.String("email", "a1-other@descrybe.local", "Teammate email to upsert")
name := flag.String("name", "A1 teammate", "Display name")
role := flag.String("role", "member", "Membership role: member|admin")
webOrigin := flag.String("web-origin", firstNonEmpty(os.Getenv("WEB_ORIGIN"), "http://localhost:28472"), "Web origin for accept-invite URL")
flag.Parse()
if strings.TrimSpace(*postgresURL) == "" {
log.Fatal("-postgres / DATABASE_URL is required")
}
emailNorm := strings.ToLower(strings.TrimSpace(*email))
if emailNorm == "" {
log.Fatal("-email is required")
}
if emailNorm == "a1-primary@descrybe.local" {
log.Fatal("-email must not be a1-primary@descrybe.local (that user remains the company owner)")
}
memRole := auth.NormalizeMembershipRole(*role)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
pg, err := pgxpool.New(ctx, *postgresURL)
if err != nil {
log.Fatalf("postgres: %v", err)
}
defer pg.Close()
var companyID uuid.UUID
var companyName string
err = pg.QueryRow(ctx, `
SELECT id, name FROM companies
WHERE legacy_company_id = $1
ORDER BY created_at ASC
LIMIT 1`, billing.A1LegacyCompanyID).Scan(&companyID, &companyName)
if err != nil {
log.Fatalf("find A1 company (legacy_company_id=%s): %v", billing.A1LegacyCompanyID, err)
}
tx, err := pg.Begin(ctx)
if err != nil {
log.Fatalf("begin: %v", err)
}
defer tx.Rollback(ctx)
var userID uuid.UUID
err = tx.QueryRow(ctx, `
INSERT INTO users (email, name, must_set_password, is_platform_admin, is_active, updated_at)
VALUES ($1, $2, true, false, true, now())
ON CONFLICT (email) DO UPDATE SET
name = EXCLUDED.name,
must_set_password = true,
password_hash = NULL,
is_active = true,
updated_at = now()
RETURNING id`, emailNorm, strings.TrimSpace(*name)).Scan(&userID)
if err != nil {
log.Fatalf("upsert user: %v", err)
}
_, err = tx.Exec(ctx, `
INSERT INTO memberships (company_id, user_id, role, status)
VALUES ($1, $2, $3, 'active')
ON CONFLICT (company_id, user_id) DO UPDATE SET
role = EXCLUDED.role,
status = 'active',
updated_at = now()`, companyID, userID, memRole)
if err != nil {
log.Fatalf("upsert membership: %v", err)
}
// Ensure a1-primary remains owner when present.
_, _ = tx.Exec(ctx, `
UPDATE companies c
SET owner_user_id = u.id, updated_at = now()
FROM users u
JOIN memberships m ON m.user_id = u.id AND m.company_id = c.id AND m.status = 'active'
WHERE c.id = $1
AND lower(u.email) = 'a1-primary@descrybe.local'`, companyID)
if err := tx.Commit(ctx); err != nil {
log.Fatalf("commit: %v", err)
}
svc := &auth.Service{Pool: pg}
inv, err := svc.ReissueSetPasswordInvite(ctx, userID, 0)
if err != nil {
log.Fatalf("issue set-password invite: %v", err)
}
acceptURL := mail.AcceptInviteURL(strings.TrimSpace(*webOrigin), inv.Token)
fmt.Printf("A1 teammate ready\n")
fmt.Printf(" company: %s (%s)\n", companyName, companyID)
fmt.Printf(" email: %s\n", emailNorm)
fmt.Printf(" user_id: %s\n", userID)
fmt.Printf(" role: %s\n", memRole)
fmt.Printf(" expires_at: %s\n", inv.ExpiresAt.UTC().Format(time.RFC3339))
fmt.Printf(" accept_url: %s\n", acceptURL)
fmt.Printf("\nShare accept_url with the teammate (or Admin → Users → Send set-password for this user).\n")
fmt.Printf("While impersonating a1-primary, Settings → Team → Invite also returns a copyable link when SMTP is off.\n")
}
func firstNonEmpty(vals ...string) string {
for _, v := range vals {
if strings.TrimSpace(v) != "" {
return strings.TrimSpace(v)
}
}
return ""
}
+59 -8
View File
@@ -13,6 +13,7 @@ import (
"unicode"
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
"github.com/descrybe/descrybe-v2/apps/api/internal/security"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
@@ -41,6 +42,10 @@ var (
)
func defaultCategoryPromptsPath(archivePath string) string {
// Prefer live WP dump when present (Sync A1 / seed share the same source of truth).
if p := catalog.ResolveWPCategoryPromptsPath(""); p != "" {
return p
}
if strings.TrimSpace(archivePath) != "" {
return filepath.Join(filepath.Dir(archivePath), "a1-category-prompts.json")
}
@@ -52,6 +57,10 @@ func loadCategoryPromptsFile(path string) (categoryPromptsFile, error) {
if path == "" || path == "." {
return categoryPromptsFile{}, fmt.Errorf("category prompts path is empty")
}
base := strings.ToLower(filepath.Base(path))
if strings.HasPrefix(base, "wp_product_categories") && strings.HasSuffix(base, ".sql") {
return loadCategoryPromptsFromWPSQL(path)
}
// Allowlist the committed seed filename (blocks accidental reads of unrelated dumps).
if filepath.Base(path) != "a1-category-prompts.json" {
return categoryPromptsFile{}, fmt.Errorf("refusing unexpected category prompts file name %q", filepath.Base(path))
@@ -76,6 +85,36 @@ func loadCategoryPromptsFile(path string) (categoryPromptsFile, error) {
return f, nil
}
func loadCategoryPromptsFromWPSQL(path string) (categoryPromptsFile, error) {
raw, err := os.ReadFile(path)
if err != nil {
return categoryPromptsFile{}, fmt.Errorf("read wp category prompts: %w", err)
}
if len(raw) > 16<<20 {
return categoryPromptsFile{}, fmt.Errorf("wp category prompts file too large (%d bytes)", len(raw))
}
parsed, err := catalog.ParseWPProductCategoriesSQL(string(raw))
if err != nil {
return categoryPromptsFile{}, err
}
entries := make([]categoryPromptEntry, 0, len(parsed))
for _, e := range parsed {
entries = append(entries, categoryPromptEntry{
Name: e.Name,
UniqueID: e.UniqueID,
Prompt: e.Prompt,
})
}
if len(entries) == 0 {
return categoryPromptsFile{}, fmt.Errorf("wp category prompts file has no entries")
}
return categoryPromptsFile{
Version: 1,
Source: filepath.Base(path),
Entries: entries,
}, nil
}
// modernizeLegacyPromptPlaceholders rewrites v1 {""OPIS IZDELKA""} tokens to {{description}} / {{name}}.
func modernizeLegacyPromptPlaceholders(prompt string) string {
prompt = reLegacyDesc.ReplaceAllString(prompt, "{{description}}")
@@ -148,16 +187,11 @@ func applyCategoryPrompts(ctx context.Context, pg *pgxpool.Pool, companyID uuid.
byNorm := make(map[string]string, len(file.Entries))
byUnique := make(map[string]string, len(file.Entries))
// Overlay sets the shared role-sectioned enhance user template
// (aiprompts.CategoryEnhanceUserTemplate — title/description/meta/attributes).
// Seed JSON selects which categories get a prompt (prefer unique_id, else name).
// Unique title/description/meta formulas stay in title_template / description_template.
// Overlay: split each legacy combined Name+Description prompt into role-sectioned
// enhance USER text (Title / Description / Meta / Attributes). Unique title /
// description / meta formulas stay in title_template / description_template.
// Stored under "sl" + "*" so prompt->>'sl' and LangPromptAny both resolve;
// language stays via {{language}}.
canonical := prepareCategoryPrompt(aiprompts.CategoryEnhanceUserTemplate)
if canonical == "" {
return out, fmt.Errorf("CategoryEnhanceUserTemplate sanitized to empty")
}
for _, e := range file.Entries {
uid := strings.TrimSpace(e.UniqueID)
name := strings.TrimSpace(e.Name)
@@ -165,6 +199,23 @@ func applyCategoryPrompts(ctx context.Context, pg *pgxpool.Pool, companyID uuid.
out.Skipped++
continue
}
raw := strings.TrimSpace(e.Prompt)
var canonical string
if raw == "" {
out.Skipped++
continue
}
if aiprompts.IsLegacyCombinedEnhancePrompt(raw) {
canonical = prepareCategoryPrompt(aiprompts.SplitLegacyCombinedEnhancePrompt(raw))
} else if aiprompts.CategoryEnhanceHasRoleSections(raw) {
canonical = prepareCategoryPrompt(raw)
} else {
canonical = prepareCategoryPrompt(aiprompts.CategoryEnhanceUserTemplate)
}
if canonical == "" {
out.Skipped++
continue
}
if uid != "" {
byUnique[strings.ToLower(uid)] = canonical
}