fix
This commit is contained in:
@@ -0,0 +1,662 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// Batch A: rich h1/h2/p/ul category formulas (incl. Slusalke/48).
|
||||
var eans = []string{
|
||||
"195348253666", // 7 Gaming monitorji
|
||||
"4548736132597", // 48 Slusalke
|
||||
"3045380024786", // 36 Pecice
|
||||
"3838782459856", // 38 Pomivalni stroji
|
||||
"1200130000638", // 3 Bluetooth zvocniki
|
||||
}
|
||||
|
||||
var eanLabels = map[string]string{
|
||||
"195348253666": "Gaming monitorji (rich formula)",
|
||||
"4548736132597": "Slusalke cat48 (rich formula)",
|
||||
"3045380024786": "Pecice (rich formula)",
|
||||
"3838782459856": "Pomivalni stroji (rich formula)",
|
||||
"1200130000638": "Bluetooth zvocniki (rich formula)",
|
||||
}
|
||||
|
||||
var poisonMeta = regexp.MustCompile(`\|\s*\d+\s*$`)
|
||||
var brandOnlyish = regexp.MustCompile(`(?i)^(bosch|gorenje|gigabyte|xiaomi|lenovo|samsung|lg|anker|jbl|vogels|ostalo)$`)
|
||||
|
||||
type dbProduct struct {
|
||||
EAN string `json:"ean"`
|
||||
Label string `json:"label"`
|
||||
ProductID string `json:"product_id"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Category string `json:"category"`
|
||||
CategoryName string `json:"category_name"`
|
||||
MetaTitle string `json:"meta_title"`
|
||||
MetaDescription string `json:"meta_description"`
|
||||
Attributes json.RawMessage `json:"attributes"`
|
||||
AIProviderMode string `json:"ai_provider_mode"`
|
||||
DescTemplate any `json:"description_template"`
|
||||
TitleTemplate any `json:"title_template"`
|
||||
FormulaConstraint string `json:"formula_constraint"`
|
||||
FieldSources json.RawMessage `json:"field_sources"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
out := os.Getenv("PROBE_OUT_DIR")
|
||||
if out == "" {
|
||||
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_a1_enhance_examples_A_20260816`
|
||||
}
|
||||
_ = os.MkdirAll(out, 0o755)
|
||||
|
||||
logPath := filepath.Join(out, "inproc_run.log")
|
||||
logFile, err := os.Create(logPath)
|
||||
if err != nil {
|
||||
log.Fatalf("log file: %v", err)
|
||||
}
|
||||
defer logFile.Close()
|
||||
log.SetOutput(logredact.Writer(io.MultiWriter(os.Stderr, logFile)))
|
||||
|
||||
companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
|
||||
userID := uuid.MustParse("0ce3305c-d810-4b56-b1d4-3c1ed510db76")
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("config: %v", err)
|
||||
}
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||||
MaxConns: int32(cfg.DBMaxConns),
|
||||
MinConns: int32(cfg.DBMinConns),
|
||||
MaxConnLifetime: cfg.DBMaxConnLifetime,
|
||||
MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
|
||||
MaxConnIdleTime: cfg.DBMaxConnIdleTime,
|
||||
HealthCheckPeriod: cfg.DBHealthCheckPeriod,
|
||||
StatementTimeout: cfg.DBStatementTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("db: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
waitForeignJobs(ctx, pool, companyID)
|
||||
|
||||
platEnv := platformsettings.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey,
|
||||
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret,
|
||||
DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey,
|
||||
OpenAIBaseURL: cfg.OpenAIBaseURL,
|
||||
OpenAIModel: cfg.OpenAIModel,
|
||||
OpenAIEmbeddingAPIKey: cfg.OpenAIEmbeddingAPIKey,
|
||||
OpenAIEmbeddingBaseURL: cfg.OpenAIEmbeddingBaseURL,
|
||||
OpenAIEmbeddingModel: cfg.OpenAIEmbeddingModel,
|
||||
}
|
||||
platSettings := platformsettings.NewService(pool, platEnv)
|
||||
aiSvc := aiprovider.NewService(pool, aiprovider.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey,
|
||||
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret,
|
||||
DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey,
|
||||
OpenAIBaseURL: cfg.OpenAIBaseURL,
|
||||
OpenAIModel: cfg.OpenAIModel,
|
||||
ProcessingRPM: cfg.ProcessingRPM,
|
||||
ProcessingMaxRetries: cfg.ProcessingMaxRetries,
|
||||
})
|
||||
aiSvc.Platform = platSettings
|
||||
|
||||
roleCfg, rerr := platSettings.ResolveAIConfig(ctx, platformsettings.AIRoleProcessing)
|
||||
if rerr != nil {
|
||||
failResolve(out, fmt.Sprintf("ResolveAIConfig: %v", rerr))
|
||||
}
|
||||
completer, modeLabel, byok, cerr := aiSvc.ResolveCompleterForRole(ctx, companyID, aiprovider.RoleProcessing)
|
||||
if cerr != nil {
|
||||
failResolve(out, fmt.Sprintf("ResolveCompleterForRole: %v", cerr))
|
||||
}
|
||||
if completer == nil {
|
||||
failResolve(out, "nil completer")
|
||||
}
|
||||
client, ok := completer.(*processing.OpenAIClient)
|
||||
if !ok {
|
||||
failResolve(out, fmt.Sprintf("completer type %T", completer))
|
||||
}
|
||||
base := strings.TrimSpace(client.BaseURL)
|
||||
model := strings.TrimSpace(client.Model)
|
||||
resolveInfo := map[string]any{
|
||||
"batch": "A",
|
||||
"eans": eans,
|
||||
"role_source": roleCfg.Source,
|
||||
"role_provider": roleCfg.Provider,
|
||||
"role_base_url": roleCfg.BaseURL,
|
||||
"role_model": roleCfg.Model,
|
||||
"role_enabled": roleCfg.Enabled,
|
||||
"completer_base": base,
|
||||
"completer_model": model,
|
||||
"mode_label": modeLabel,
|
||||
"byok": byok,
|
||||
"env_base_url": cfg.OpenAIBaseURL,
|
||||
"env_model": cfg.OpenAIModel,
|
||||
"live": true,
|
||||
}
|
||||
if processing.IsMockOrLoopbackBaseURL(base) || strings.Contains(strings.ToLower(base), "18767") {
|
||||
resolveInfo["live"] = false
|
||||
resolveInfo["fail_reason"] = "resolved completer is mock/loopback — refuse silent mock fallback"
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM REQUIRED: resolved base=%s model=%s — refuse mock", base, model)
|
||||
}
|
||||
if strings.TrimSpace(client.APIKey) == "" {
|
||||
resolveInfo["live"] = false
|
||||
resolveInfo["fail_reason"] = "empty API key"
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatal("LIVE LLM REQUIRED: API key empty")
|
||||
}
|
||||
keyHint := ""
|
||||
if k := strings.TrimSpace(client.APIKey); len(k) > 8 {
|
||||
keyHint = k[:4] + "…" + k[len(k)-4:]
|
||||
}
|
||||
resolveInfo["api_key_hint"] = keyHint
|
||||
|
||||
modelsURL := strings.TrimRight(base, "/") + "/models"
|
||||
modelsCtx, modelsCancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer modelsCancel()
|
||||
modelsReq, merr := http.NewRequestWithContext(modelsCtx, http.MethodGet, modelsURL, nil)
|
||||
if merr != nil {
|
||||
failResolve(out, "models req: "+merr.Error())
|
||||
}
|
||||
modelsReq.Header.Set("Authorization", "Bearer "+client.APIKey)
|
||||
modelsReq.Header.Set("Accept", "application/json")
|
||||
modelsStart := time.Now()
|
||||
modelsResp, merr := http.DefaultClient.Do(modelsReq)
|
||||
modelsElapsed := time.Since(modelsStart).Round(time.Millisecond)
|
||||
if merr != nil {
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["models_error"] = processing.TruncateError(merr)
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: GET models: %v", merr)
|
||||
}
|
||||
modelsBody, _ := io.ReadAll(io.LimitReader(modelsResp.Body, 2048))
|
||||
_ = modelsResp.Body.Close()
|
||||
resolveInfo["models_url"] = modelsURL
|
||||
resolveInfo["models_http_status"] = modelsResp.StatusCode
|
||||
resolveInfo["models_elapsed"] = modelsElapsed.String()
|
||||
snippet := strings.TrimSpace(string(modelsBody))
|
||||
if len(snippet) > 200 {
|
||||
snippet = snippet[:200] + "…"
|
||||
}
|
||||
resolveInfo["models_body_snippet"] = snippet
|
||||
if modelsResp.StatusCode != http.StatusOK {
|
||||
resolveInfo["probe_ok"] = false
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: models HTTP %d", modelsResp.StatusCode)
|
||||
}
|
||||
|
||||
probeCtx, probeCancel := context.WithTimeout(ctx, 3*time.Minute)
|
||||
defer probeCancel()
|
||||
probeStart := time.Now()
|
||||
comp, perr := client.Complete(probeCtx, "Reply with exactly: PONG", "Say PONG")
|
||||
probeElapsed := time.Since(probeStart).Round(time.Millisecond)
|
||||
if perr != nil {
|
||||
// One soft retry — OverloadedBot often needs a cool-down after a prior batch.
|
||||
log.Printf("chat probe first attempt failed elapsed=%s err=%s — retrying once after 20s", probeElapsed, processing.TruncateError(perr))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["probe_error"] = processing.TruncateError(perr)
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: chat probe: %v", perr)
|
||||
case <-time.After(20 * time.Second):
|
||||
}
|
||||
probeCtx2, probeCancel2 := context.WithTimeout(ctx, 3*time.Minute)
|
||||
defer probeCancel2()
|
||||
probeStart = time.Now()
|
||||
comp, perr = client.Complete(probeCtx2, "Reply with exactly: PONG", "Say PONG")
|
||||
probeElapsed = time.Since(probeStart).Round(time.Millisecond)
|
||||
if perr != nil {
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["probe_error"] = processing.TruncateError(perr)
|
||||
resolveInfo["models_http_status"] = modelsResp.StatusCode
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: chat probe (after retry): %v", perr)
|
||||
}
|
||||
}
|
||||
resolveInfo["probe_ok"] = true
|
||||
resolveInfo["probe_elapsed"] = probeElapsed.String()
|
||||
resolveInfo["probe_response_len"] = len(comp.Text)
|
||||
resolveInfo["started_at_utc"] = time.Now().UTC().Format(time.RFC3339)
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Printf("live resolve ok source=%s base=%s model=%s mode=%s byok=%v", roleCfg.Source, base, model, modeLabel, byok)
|
||||
|
||||
pipeline := processing.NewPipeline(pool)
|
||||
pipeline.BatchSize = cfg.ProcessingBatchSize
|
||||
pipeline.AI = aiSvc
|
||||
pipeline.Prompts = aiprompts.NewService(pool)
|
||||
eprelClient := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second})
|
||||
pipeline.Engine = &processing.Engine{
|
||||
Completer: nil,
|
||||
Vector: processing.NoopVectorCategorizer{},
|
||||
EPREL: eprelClient,
|
||||
ProviderMode: processing.AIProviderInternal,
|
||||
}
|
||||
|
||||
rawIDs := make([]uuid.UUID, 0, len(eans))
|
||||
for _, ean := range eans {
|
||||
var id uuid.UUID
|
||||
if err := pool.QueryRow(ctx, `
|
||||
SELECT id FROM raw_products
|
||||
WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id); err != nil {
|
||||
log.Fatalf("raw product %s: %v", ean, err)
|
||||
}
|
||||
rawIDs = append(rawIDs, id)
|
||||
log.Printf("ean=%s raw_id=%s label=%s", ean, id, eanLabels[ean])
|
||||
}
|
||||
|
||||
tag, cerr2 := pool.Exec(ctx, `
|
||||
UPDATE processed_products pp
|
||||
SET field_sources = COALESCE(field_sources, '{}'::jsonb) - 'enhance_input_hash',
|
||||
localized_content = CASE
|
||||
WHEN localized_content IS NULL OR localized_content = '{}'::jsonb THEN localized_content
|
||||
ELSE (
|
||||
SELECT COALESCE(jsonb_object_agg(lang, val - 'enhance_input_hash'), '{}'::jsonb)
|
||||
FROM jsonb_each(localized_content) AS t(lang, val)
|
||||
)
|
||||
END,
|
||||
updated_at = now()
|
||||
FROM raw_products rp
|
||||
WHERE pp.company_id = $1
|
||||
AND pp.raw_product_id = rp.id
|
||||
AND rp.gtin = ANY($2::text[])`, companyID, eans)
|
||||
if cerr2 != nil {
|
||||
log.Printf("clear enhance hash: %v", cerr2)
|
||||
} else {
|
||||
log.Printf("cleared enhance_input_hash rows=%d", tag.RowsAffected())
|
||||
}
|
||||
|
||||
if n, err := processing.BackfillMissingMeta(ctx, pool, companyID); err != nil {
|
||||
log.Printf("BackfillMissingMeta: %v", err)
|
||||
} else {
|
||||
log.Printf("BackfillMissingMeta updated=%d", n)
|
||||
}
|
||||
|
||||
_, _ = pool.Exec(ctx, `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'failed', error = 'yielded to live enhance examples batch A', updated_at = now()
|
||||
WHERE company_id = $1 AND status IN ('pending','running','processing')`, companyID)
|
||||
|
||||
jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full")
|
||||
if err != nil {
|
||||
log.Fatalf("StartJob: %v", err)
|
||||
}
|
||||
if len(jobs) == 0 {
|
||||
log.Fatal("StartJob returned no jobs")
|
||||
}
|
||||
jobID := jobs[0].ID
|
||||
log.Printf("job_id=%s chunks=%d", jobID, len(jobs))
|
||||
_ = os.WriteFile(filepath.Join(out, "process_id.txt"), []byte(jobID.String()+"\n"), 0o644)
|
||||
|
||||
defend := make(chan struct{})
|
||||
go func() {
|
||||
t := time.NewTicker(500 * time.Millisecond)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-defend:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'failed', error = 'yielded to live enhance examples batch A', updated_at = now()
|
||||
WHERE company_id = $1 AND status IN ('pending','running','processing') AND id <> $2`, companyID, jobID)
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'running', error = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status IN ('failed','cancelled')
|
||||
AND (error ILIKE '%yield%' OR error ILIKE '%parked%' OR error ILIKE '%reclaim%'
|
||||
OR error ILIKE '%batch_%' OR error ILIKE '%enhance examples%' OR error ILIKE '%formula%')`, jobID)
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_job_products
|
||||
SET status = 'pending', error = NULL, updated_at = now()
|
||||
WHERE job_id = $1 AND status IN ('failed','cancelled')
|
||||
AND processed_product_id IS NULL
|
||||
AND (error ILIKE '%yield%' OR error ILIKE '%parked%' OR error ILIKE '%reclaim%'
|
||||
OR error ILIKE '%batch_%' OR error ILIKE '%enhance examples%' OR error ILIKE '%formula%')`, jobID)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
_, _ = pool.Exec(ctx, `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
|
||||
WHERE id = $1`, jobID)
|
||||
|
||||
runCtx, runCancel := context.WithTimeout(ctx, 25*time.Minute)
|
||||
defer runCancel()
|
||||
if err := pipeline.ProcessJob(runCtx, jobID); err != nil {
|
||||
close(defend)
|
||||
log.Fatalf("ProcessJob: %v", err)
|
||||
}
|
||||
close(defend)
|
||||
|
||||
items, err := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
|
||||
if err != nil {
|
||||
log.Fatalf("LoadV1ProcessJobItems: %v", err)
|
||||
}
|
||||
writeJSON(out, "final.json", map[string]any{
|
||||
"data": map[string]any{
|
||||
"process_id": jobID.String(),
|
||||
"status": "COMPLETED",
|
||||
"processing_type": "full",
|
||||
"total_items": len(items),
|
||||
"items": items,
|
||||
"llm": map[string]any{
|
||||
"base": base,
|
||||
"model": model,
|
||||
"source": roleCfg.Source,
|
||||
"mode": modeLabel,
|
||||
"byok": byok,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT rp.gtin,
|
||||
COALESCE(pp.processed_name, pp.name, ''),
|
||||
COALESCE(NULLIF(pp.processed_description, ''), pp.description, ''),
|
||||
COALESCE(pp.category, ''),
|
||||
COALESCE(c.name, ''),
|
||||
COALESCE(pp.meta_title, ''),
|
||||
COALESCE(pp.meta_description, ''),
|
||||
COALESCE(pp.processed_attributes, pp.attributes, '{}'::jsonb),
|
||||
COALESCE(pp.ai_provider_mode, ''),
|
||||
c.description_template,
|
||||
c.title_template,
|
||||
pp.id,
|
||||
COALESCE(pp.field_sources, '{}'::jsonb)
|
||||
FROM processing_job_products pjp
|
||||
JOIN raw_products rp ON rp.id = pjp.raw_product_id
|
||||
LEFT JOIN processed_products pp ON pp.id = pjp.processed_product_id
|
||||
LEFT JOIN categories c ON c.company_id = pp.company_id AND c.unique_id = pp.category
|
||||
WHERE pjp.job_id = $1
|
||||
ORDER BY rp.gtin`, jobID)
|
||||
if err != nil {
|
||||
log.Fatalf("db products: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
dbProducts := make([]dbProduct, 0)
|
||||
for rows.Next() {
|
||||
var p dbProduct
|
||||
var ppID *uuid.UUID
|
||||
var attrs, sources []byte
|
||||
if err := rows.Scan(&p.EAN, &p.Title, &p.Description, &p.Category, &p.CategoryName,
|
||||
&p.MetaTitle, &p.MetaDescription, &attrs, &p.AIProviderMode,
|
||||
&p.DescTemplate, &p.TitleTemplate, &ppID, &sources); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if ppID != nil {
|
||||
p.ProductID = ppID.String()
|
||||
}
|
||||
p.Label = eanLabels[p.EAN]
|
||||
p.Attributes = attrs
|
||||
p.FieldSources = sources
|
||||
p.FormulaConstraint = processing.FormatDescriptionFormulaConstraint(p.DescTemplate)
|
||||
dbProducts = append(dbProducts, p)
|
||||
}
|
||||
writeJSON(out, "db_products.json", dbProducts)
|
||||
scoreAndWrite(out, jobID.String(), resolveInfo, dbProducts)
|
||||
fmt.Printf("BATCH_A JOB %s products=%d live_base=%s model=%s\n", jobID, len(dbProducts), base, model)
|
||||
}
|
||||
|
||||
func waitForeignJobs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) {
|
||||
deadline := time.Now().Add(18 * time.Minute)
|
||||
for time.Now().Before(deadline) {
|
||||
var n int
|
||||
err := pool.QueryRow(ctx, `
|
||||
SELECT count(*) FROM processing_jobs
|
||||
WHERE company_id = $1 AND status IN ('pending','running','processing')`, companyID).Scan(&n)
|
||||
if err != nil {
|
||||
log.Printf("wait foreign jobs: %v", err)
|
||||
return
|
||||
}
|
||||
if n == 0 {
|
||||
log.Printf("queue clear — starting batch A")
|
||||
return
|
||||
}
|
||||
log.Printf("waiting for %d foreign Demo job(s) before batch A…", n)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(15 * time.Second):
|
||||
}
|
||||
}
|
||||
log.Printf("wait deadline reached — proceeding (will yield foreign jobs)")
|
||||
}
|
||||
|
||||
func scoreAndWrite(out, jobID string, resolve map[string]any, products []dbProduct) {
|
||||
scored := make([]map[string]any, 0, len(products))
|
||||
pass, fail := 0, 0
|
||||
var md strings.Builder
|
||||
md.WriteString("# A1 enhance examples — batch A (rich category formulas; includes Slusalke/48)\n\n")
|
||||
md.WriteString(fmt.Sprintf("- scored_at_utc: `%s`\n", time.Now().UTC().Format(time.RFC3339)))
|
||||
md.WriteString(fmt.Sprintf("- job_id: `%s`\n", jobID))
|
||||
md.WriteString(fmt.Sprintf("- live_base: `%v` model: `%v` source: `%v`\n", resolve["completer_base"], resolve["completer_model"], resolve["role_source"]))
|
||||
md.WriteString(fmt.Sprintf("- mock_fallback: **false** (env mock ignored when DB admin AI resolves)\n\n"))
|
||||
md.WriteString("Scoring: title quality · HTML formula tags (h1/h2/p/ul) · meta · attrs\n\n")
|
||||
|
||||
for _, p := range products {
|
||||
title := strings.TrimSpace(p.Title)
|
||||
desc := p.Description
|
||||
metaT := strings.TrimSpace(p.MetaTitle)
|
||||
metaD := strings.TrimSpace(p.MetaDescription)
|
||||
|
||||
has := map[string]bool{
|
||||
"h1": strings.Contains(strings.ToLower(desc), "<h1"),
|
||||
"h2": strings.Contains(strings.ToLower(desc), "<h2"),
|
||||
"p": strings.Contains(strings.ToLower(desc), "<p"),
|
||||
"ul": strings.Contains(strings.ToLower(desc), "<ul"),
|
||||
}
|
||||
htmlOK := has["h1"] && has["h2"] && has["p"] && has["ul"]
|
||||
|
||||
titleOK := title != "" && utf8.RuneCountInString(title) >= 8 && !brandOnlyish.MatchString(title)
|
||||
titleNotes := []string{}
|
||||
if title == "" {
|
||||
titleNotes = append(titleNotes, "empty")
|
||||
} else if brandOnlyish.MatchString(title) {
|
||||
titleNotes = append(titleNotes, "brand_only")
|
||||
} else if utf8.RuneCountInString(title) < 8 {
|
||||
titleNotes = append(titleNotes, "too_short")
|
||||
}
|
||||
if strings.Contains(strings.ToLower(title), "title formula") || strings.Contains(strings.ToLower(title), "category:") {
|
||||
titleOK = false
|
||||
titleNotes = append(titleNotes, "prompt_leakage")
|
||||
}
|
||||
|
||||
metaOK := metaT != "" && metaD != "" && !poisonMeta.MatchString(metaT) &&
|
||||
!strings.Contains(strings.ToLower(metaT), "short retail title")
|
||||
metaNotes := []string{}
|
||||
if metaT == "" {
|
||||
metaNotes = append(metaNotes, "empty_meta_title")
|
||||
}
|
||||
if metaD == "" {
|
||||
metaNotes = append(metaNotes, "empty_meta_description")
|
||||
}
|
||||
if poisonMeta.MatchString(metaT) {
|
||||
metaNotes = append(metaNotes, "poison_|digits")
|
||||
}
|
||||
|
||||
attrMap := map[string]any{}
|
||||
_ = json.Unmarshal(p.Attributes, &attrMap)
|
||||
attrKeys := make([]string, 0, len(attrMap))
|
||||
dirty := []string{}
|
||||
for k, v := range attrMap {
|
||||
attrKeys = append(attrKeys, k)
|
||||
sv := strings.TrimSpace(fmt.Sprint(v))
|
||||
if sv == "" || strings.EqualFold(sv, "null") || strings.EqualFold(sv, "n/a") {
|
||||
dirty = append(dirty, k+":empty")
|
||||
}
|
||||
if strings.Contains(strings.ToLower(k), "ean") || strings.Contains(strings.ToLower(k), "gtin") {
|
||||
dirty = append(dirty, k+":id_like_key")
|
||||
}
|
||||
}
|
||||
attrsOK := len(attrKeys) > 0 && len(dirty) == 0
|
||||
|
||||
checks := map[string]string{
|
||||
"title": passFail(titleOK),
|
||||
"html_tags": passFail(htmlOK),
|
||||
"meta": passFail(metaOK),
|
||||
"attrs": passFail(attrsOK),
|
||||
"category": passFail(p.Category != "" && p.CategoryName != ""),
|
||||
"desc_ne_title": passFail(strings.TrimSpace(stripTags(desc)) != "" && strings.TrimSpace(stripTags(desc)) != title),
|
||||
"ai_internal": passFail(strings.EqualFold(p.AIProviderMode, "internal") || p.AIProviderMode == ""),
|
||||
}
|
||||
failFlags := []string{}
|
||||
for k, v := range checks {
|
||||
if v == "FAIL" {
|
||||
failFlags = append(failFlags, k)
|
||||
}
|
||||
}
|
||||
verdict := "PASS"
|
||||
if len(failFlags) > 0 {
|
||||
verdict = "FAIL"
|
||||
fail++
|
||||
} else {
|
||||
pass++
|
||||
}
|
||||
|
||||
row := map[string]any{
|
||||
"ean": p.EAN,
|
||||
"label": p.Label,
|
||||
"product_id": p.ProductID,
|
||||
"title": title,
|
||||
"category": p.Category,
|
||||
"category_name": p.CategoryName,
|
||||
"desc_len": len(desc),
|
||||
"desc_preview": truncate(stripTags(desc), 140),
|
||||
"meta_title": metaT,
|
||||
"meta_description": truncate(metaD, 160),
|
||||
"attr_keys": attrKeys,
|
||||
"dirty_attrs": dirty,
|
||||
"attributes": attrMap,
|
||||
"has_tags": has,
|
||||
"formula_constraint": p.FormulaConstraint,
|
||||
"formula_override_hint": strings.Contains(strings.ToLower(fmt.Sprint(p.FieldSources)), "formula"),
|
||||
"ai_provider_mode": p.AIProviderMode,
|
||||
"title_notes": titleNotes,
|
||||
"meta_notes": metaNotes,
|
||||
"checks": checks,
|
||||
"fail_flags": failFlags,
|
||||
"verdict": verdict,
|
||||
}
|
||||
scored = append(scored, row)
|
||||
|
||||
md.WriteString(fmt.Sprintf("## %s — %s (`%s`)\n\n", p.EAN, p.Label, verdict))
|
||||
md.WriteString(fmt.Sprintf("- **title:** %s\n", title))
|
||||
md.WriteString(fmt.Sprintf("- **category:** %s / %s\n", p.Category, p.CategoryName))
|
||||
md.WriteString(fmt.Sprintf("- **meta_title:** %s\n", metaT))
|
||||
md.WriteString(fmt.Sprintf("- **meta_description:** %s\n", truncate(metaD, 200)))
|
||||
md.WriteString(fmt.Sprintf("- **attrs:** %s\n", strings.Join(attrKeys, ", ")))
|
||||
md.WriteString(fmt.Sprintf("- **HTML tags:** h1=%v h2=%v p=%v ul=%v\n", has["h1"], has["h2"], has["p"], has["ul"]))
|
||||
md.WriteString(fmt.Sprintf("- **checks:** title=%s html=%s meta=%s attrs=%s\n\n", checks["title"], checks["html_tags"], checks["meta"], checks["attrs"]))
|
||||
md.WriteString("### Description HTML (raw)\n\n```html\n")
|
||||
if len(desc) > 4096 {
|
||||
md.WriteString(desc[:4096])
|
||||
md.WriteString("\n<!-- truncated -->\n")
|
||||
} else {
|
||||
md.WriteString(desc)
|
||||
md.WriteString("\n")
|
||||
}
|
||||
md.WriteString("```\n\n")
|
||||
}
|
||||
|
||||
total := len(products)
|
||||
rate := 0
|
||||
if total > 0 {
|
||||
rate = (pass * 100) / total
|
||||
}
|
||||
scorecard := map[string]any{
|
||||
"batch": "A",
|
||||
"date": "2026-08-16",
|
||||
"scored_at_utc": time.Now().UTC().Format(time.RFC3339),
|
||||
"company_id": "2b3159b0-fc08-415b-b248-35ed02a6baab",
|
||||
"company_name": "Platform Demo",
|
||||
"job_id": jobID,
|
||||
"selection": eanLabels,
|
||||
"live_llm": resolve,
|
||||
"mock_as_live": false,
|
||||
"asserts": []string{"title_quality", "html_h1_h2_p_ul", "meta", "attrs", "category", "desc_ne_title"},
|
||||
"summary": map[string]any{"pass": pass, "fail": fail, "total": total, "pass_rate": rate},
|
||||
"verdict": map[bool]string{true: "PASS", false: "FAIL"}[fail == 0 && total > 0],
|
||||
"products": scored,
|
||||
"max_tokens_note": fmt.Sprintf("MaxTokensEnhance=%d MaxTokensEnhanceRetry=%d", processing.MaxTokensEnhance, processing.MaxTokensEnhanceRetry),
|
||||
}
|
||||
writeJSON(out, "scorecard.json", scorecard)
|
||||
_ = os.WriteFile(filepath.Join(out, "examples.md"), []byte(md.String()), 0o644)
|
||||
|
||||
summary := fmt.Sprintf("# Batch A summary\n\n- verdict: **%s** (%d/%d)\n- live: `%v` / `%v`\n- mock_as_live: false\n- artifacts: scorecard.json, examples.md, db_products.json, final.json, resolve.json\n",
|
||||
scorecard["verdict"], pass, total, resolve["completer_base"], resolve["completer_model"])
|
||||
_ = os.WriteFile(filepath.Join(out, "summary.md"), []byte(summary), 0o644)
|
||||
}
|
||||
|
||||
func passFail(ok bool) string {
|
||||
if ok {
|
||||
return "PASS"
|
||||
}
|
||||
return "FAIL"
|
||||
}
|
||||
|
||||
func stripTags(s string) string {
|
||||
re := regexp.MustCompile(`<[^>]+>`)
|
||||
return strings.TrimSpace(re.ReplaceAllString(s, " "))
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
r := []rune(strings.TrimSpace(s))
|
||||
if len(r) <= n {
|
||||
return string(r)
|
||||
}
|
||||
return string(r[:n]) + "…"
|
||||
}
|
||||
|
||||
func failResolve(out, msg string) {
|
||||
writeJSON(out, "resolve.json", map[string]any{"live": false, "fail_reason": msg, "batch": "A"})
|
||||
log.Fatalf("LIVE LLM REQUIRED: %s", msg)
|
||||
}
|
||||
|
||||
func writeJSON(out, name string, v any) {
|
||||
raw, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("marshal %s: %v", name, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(out, name), raw, 0o644); err != nil {
|
||||
log.Fatalf("write %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,664 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// Batch B: appliances + monitors + mounts — independent of batch A.
|
||||
var eans = []string{
|
||||
"4242005342488", // 41 washer-dryer
|
||||
"4242005382330", // 38 dishwasher
|
||||
"4242005327065", // 36 oven
|
||||
"4719331854218", // 7 monitor GIGABYTE
|
||||
"6941948703193", // 7 monitor Xiaomi
|
||||
"8712285326882", // 28 TV mount
|
||||
}
|
||||
|
||||
var eanLabels = map[string]string{
|
||||
"4242005342488": "appliance washer-dryer",
|
||||
"4242005382330": "appliance dishwasher",
|
||||
"4242005327065": "appliance oven",
|
||||
"4719331854218": "monitor GIGABYTE",
|
||||
"6941948703193": "monitor Xiaomi",
|
||||
"8712285326882": "mount Vogels",
|
||||
}
|
||||
|
||||
var poisonMeta = regexp.MustCompile(`\|\s*\d+\s*$`)
|
||||
var brandOnlyish = regexp.MustCompile(`(?i)^(bosch|gorenje|gigabyte|xiaomi|lenovo|samsung|lg|anker|jbl|vogels|ostalo)$`)
|
||||
|
||||
type dbProduct struct {
|
||||
EAN string `json:"ean"`
|
||||
Label string `json:"label"`
|
||||
ProductID string `json:"product_id"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Category string `json:"category"`
|
||||
CategoryName string `json:"category_name"`
|
||||
MetaTitle string `json:"meta_title"`
|
||||
MetaDescription string `json:"meta_description"`
|
||||
Attributes json.RawMessage `json:"attributes"`
|
||||
AIProviderMode string `json:"ai_provider_mode"`
|
||||
DescTemplate any `json:"description_template"`
|
||||
TitleTemplate any `json:"title_template"`
|
||||
FormulaConstraint string `json:"formula_constraint"`
|
||||
FieldSources json.RawMessage `json:"field_sources"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
out := os.Getenv("PROBE_OUT_DIR")
|
||||
if out == "" {
|
||||
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_a1_enhance_examples_B_20260816`
|
||||
}
|
||||
_ = os.MkdirAll(out, 0o755)
|
||||
|
||||
logPath := filepath.Join(out, "inproc_run.log")
|
||||
logFile, err := os.Create(logPath)
|
||||
if err != nil {
|
||||
log.Fatalf("log file: %v", err)
|
||||
}
|
||||
defer logFile.Close()
|
||||
log.SetOutput(logredact.Writer(io.MultiWriter(os.Stderr, logFile)))
|
||||
|
||||
companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
|
||||
userID := uuid.MustParse("0ce3305c-d810-4b56-b1d4-3c1ed510db76")
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("config: %v", err)
|
||||
}
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||||
MaxConns: int32(cfg.DBMaxConns),
|
||||
MinConns: int32(cfg.DBMinConns),
|
||||
MaxConnLifetime: cfg.DBMaxConnLifetime,
|
||||
MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
|
||||
MaxConnIdleTime: cfg.DBMaxConnIdleTime,
|
||||
HealthCheckPeriod: cfg.DBHealthCheckPeriod,
|
||||
StatementTimeout: cfg.DBStatementTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("db: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
waitForeignJobs(ctx, pool, companyID)
|
||||
|
||||
platEnv := platformsettings.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey,
|
||||
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret,
|
||||
DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey,
|
||||
OpenAIBaseURL: cfg.OpenAIBaseURL,
|
||||
OpenAIModel: cfg.OpenAIModel,
|
||||
OpenAIEmbeddingAPIKey: cfg.OpenAIEmbeddingAPIKey,
|
||||
OpenAIEmbeddingBaseURL: cfg.OpenAIEmbeddingBaseURL,
|
||||
OpenAIEmbeddingModel: cfg.OpenAIEmbeddingModel,
|
||||
}
|
||||
platSettings := platformsettings.NewService(pool, platEnv)
|
||||
aiSvc := aiprovider.NewService(pool, aiprovider.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey,
|
||||
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret,
|
||||
DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey,
|
||||
OpenAIBaseURL: cfg.OpenAIBaseURL,
|
||||
OpenAIModel: cfg.OpenAIModel,
|
||||
ProcessingRPM: cfg.ProcessingRPM,
|
||||
ProcessingMaxRetries: cfg.ProcessingMaxRetries,
|
||||
})
|
||||
aiSvc.Platform = platSettings
|
||||
|
||||
roleCfg, rerr := platSettings.ResolveAIConfig(ctx, platformsettings.AIRoleProcessing)
|
||||
if rerr != nil {
|
||||
failResolve(out, fmt.Sprintf("ResolveAIConfig: %v", rerr))
|
||||
}
|
||||
completer, modeLabel, byok, cerr := aiSvc.ResolveCompleterForRole(ctx, companyID, aiprovider.RoleProcessing)
|
||||
if cerr != nil {
|
||||
failResolve(out, fmt.Sprintf("ResolveCompleterForRole: %v", cerr))
|
||||
}
|
||||
if completer == nil {
|
||||
failResolve(out, "nil completer")
|
||||
}
|
||||
client, ok := completer.(*processing.OpenAIClient)
|
||||
if !ok {
|
||||
failResolve(out, fmt.Sprintf("completer type %T", completer))
|
||||
}
|
||||
base := strings.TrimSpace(client.BaseURL)
|
||||
model := strings.TrimSpace(client.Model)
|
||||
resolveInfo := map[string]any{
|
||||
"batch": "B",
|
||||
"eans": eans,
|
||||
"role_source": roleCfg.Source,
|
||||
"role_provider": roleCfg.Provider,
|
||||
"role_base_url": roleCfg.BaseURL,
|
||||
"role_model": roleCfg.Model,
|
||||
"role_enabled": roleCfg.Enabled,
|
||||
"completer_base": base,
|
||||
"completer_model": model,
|
||||
"mode_label": modeLabel,
|
||||
"byok": byok,
|
||||
"env_base_url": cfg.OpenAIBaseURL,
|
||||
"env_model": cfg.OpenAIModel,
|
||||
"live": true,
|
||||
}
|
||||
if processing.IsMockOrLoopbackBaseURL(base) || strings.Contains(strings.ToLower(base), "18767") {
|
||||
resolveInfo["live"] = false
|
||||
resolveInfo["fail_reason"] = "resolved completer is mock/loopback — refuse silent mock fallback"
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM REQUIRED: resolved base=%s model=%s — refuse mock", base, model)
|
||||
}
|
||||
if strings.TrimSpace(client.APIKey) == "" {
|
||||
resolveInfo["live"] = false
|
||||
resolveInfo["fail_reason"] = "empty API key"
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatal("LIVE LLM REQUIRED: API key empty")
|
||||
}
|
||||
keyHint := ""
|
||||
if k := strings.TrimSpace(client.APIKey); len(k) > 8 {
|
||||
keyHint = k[:4] + "…" + k[len(k)-4:]
|
||||
}
|
||||
resolveInfo["api_key_hint"] = keyHint
|
||||
|
||||
modelsURL := strings.TrimRight(base, "/") + "/models"
|
||||
modelsCtx, modelsCancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer modelsCancel()
|
||||
modelsReq, merr := http.NewRequestWithContext(modelsCtx, http.MethodGet, modelsURL, nil)
|
||||
if merr != nil {
|
||||
failResolve(out, "models req: "+merr.Error())
|
||||
}
|
||||
modelsReq.Header.Set("Authorization", "Bearer "+client.APIKey)
|
||||
modelsReq.Header.Set("Accept", "application/json")
|
||||
modelsStart := time.Now()
|
||||
modelsResp, merr := http.DefaultClient.Do(modelsReq)
|
||||
modelsElapsed := time.Since(modelsStart).Round(time.Millisecond)
|
||||
if merr != nil {
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["models_error"] = processing.TruncateError(merr)
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: GET models: %v", merr)
|
||||
}
|
||||
modelsBody, _ := io.ReadAll(io.LimitReader(modelsResp.Body, 2048))
|
||||
_ = modelsResp.Body.Close()
|
||||
resolveInfo["models_url"] = modelsURL
|
||||
resolveInfo["models_http_status"] = modelsResp.StatusCode
|
||||
resolveInfo["models_elapsed"] = modelsElapsed.String()
|
||||
snippet := strings.TrimSpace(string(modelsBody))
|
||||
if len(snippet) > 200 {
|
||||
snippet = snippet[:200] + "…"
|
||||
}
|
||||
resolveInfo["models_body_snippet"] = snippet
|
||||
if modelsResp.StatusCode != http.StatusOK {
|
||||
resolveInfo["probe_ok"] = false
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: models HTTP %d", modelsResp.StatusCode)
|
||||
}
|
||||
|
||||
probeCtx, probeCancel := context.WithTimeout(ctx, 3*time.Minute)
|
||||
defer probeCancel()
|
||||
probeStart := time.Now()
|
||||
comp, perr := client.Complete(probeCtx, "Reply with exactly: PONG", "Say PONG")
|
||||
probeElapsed := time.Since(probeStart).Round(time.Millisecond)
|
||||
if perr != nil {
|
||||
// One soft retry — OverloadedBot often needs a cool-down after a prior batch.
|
||||
log.Printf("chat probe first attempt failed elapsed=%s err=%s — retrying once after 20s", probeElapsed, processing.TruncateError(perr))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["probe_error"] = processing.TruncateError(perr)
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: chat probe: %v", perr)
|
||||
case <-time.After(20 * time.Second):
|
||||
}
|
||||
probeCtx2, probeCancel2 := context.WithTimeout(ctx, 3*time.Minute)
|
||||
defer probeCancel2()
|
||||
probeStart = time.Now()
|
||||
comp, perr = client.Complete(probeCtx2, "Reply with exactly: PONG", "Say PONG")
|
||||
probeElapsed = time.Since(probeStart).Round(time.Millisecond)
|
||||
if perr != nil {
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["probe_error"] = processing.TruncateError(perr)
|
||||
resolveInfo["models_http_status"] = modelsResp.StatusCode
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: chat probe (after retry): %v", perr)
|
||||
}
|
||||
}
|
||||
resolveInfo["probe_ok"] = true
|
||||
resolveInfo["probe_elapsed"] = probeElapsed.String()
|
||||
resolveInfo["probe_response_len"] = len(comp.Text)
|
||||
resolveInfo["started_at_utc"] = time.Now().UTC().Format(time.RFC3339)
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Printf("live resolve ok source=%s base=%s model=%s mode=%s byok=%v", roleCfg.Source, base, model, modeLabel, byok)
|
||||
|
||||
pipeline := processing.NewPipeline(pool)
|
||||
pipeline.BatchSize = cfg.ProcessingBatchSize
|
||||
pipeline.AI = aiSvc
|
||||
pipeline.Prompts = aiprompts.NewService(pool)
|
||||
eprelClient := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second})
|
||||
pipeline.Engine = &processing.Engine{
|
||||
Completer: nil,
|
||||
Vector: processing.NoopVectorCategorizer{},
|
||||
EPREL: eprelClient,
|
||||
ProviderMode: processing.AIProviderInternal,
|
||||
}
|
||||
|
||||
rawIDs := make([]uuid.UUID, 0, len(eans))
|
||||
for _, ean := range eans {
|
||||
var id uuid.UUID
|
||||
if err := pool.QueryRow(ctx, `
|
||||
SELECT id FROM raw_products
|
||||
WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id); err != nil {
|
||||
log.Fatalf("raw product %s: %v", ean, err)
|
||||
}
|
||||
rawIDs = append(rawIDs, id)
|
||||
log.Printf("ean=%s raw_id=%s label=%s", ean, id, eanLabels[ean])
|
||||
}
|
||||
|
||||
tag, cerr2 := pool.Exec(ctx, `
|
||||
UPDATE processed_products pp
|
||||
SET field_sources = COALESCE(field_sources, '{}'::jsonb) - 'enhance_input_hash',
|
||||
localized_content = CASE
|
||||
WHEN localized_content IS NULL OR localized_content = '{}'::jsonb THEN localized_content
|
||||
ELSE (
|
||||
SELECT COALESCE(jsonb_object_agg(lang, val - 'enhance_input_hash'), '{}'::jsonb)
|
||||
FROM jsonb_each(localized_content) AS t(lang, val)
|
||||
)
|
||||
END,
|
||||
updated_at = now()
|
||||
FROM raw_products rp
|
||||
WHERE pp.company_id = $1
|
||||
AND pp.raw_product_id = rp.id
|
||||
AND rp.gtin = ANY($2::text[])`, companyID, eans)
|
||||
if cerr2 != nil {
|
||||
log.Printf("clear enhance hash: %v", cerr2)
|
||||
} else {
|
||||
log.Printf("cleared enhance_input_hash rows=%d", tag.RowsAffected())
|
||||
}
|
||||
|
||||
if n, err := processing.BackfillMissingMeta(ctx, pool, companyID); err != nil {
|
||||
log.Printf("BackfillMissingMeta: %v", err)
|
||||
} else {
|
||||
log.Printf("BackfillMissingMeta updated=%d", n)
|
||||
}
|
||||
|
||||
_, _ = pool.Exec(ctx, `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'failed', error = 'yielded to live enhance examples batch B', updated_at = now()
|
||||
WHERE company_id = $1 AND status IN ('pending','running','processing')`, companyID)
|
||||
|
||||
jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full")
|
||||
if err != nil {
|
||||
log.Fatalf("StartJob: %v", err)
|
||||
}
|
||||
if len(jobs) == 0 {
|
||||
log.Fatal("StartJob returned no jobs")
|
||||
}
|
||||
jobID := jobs[0].ID
|
||||
log.Printf("job_id=%s chunks=%d", jobID, len(jobs))
|
||||
_ = os.WriteFile(filepath.Join(out, "process_id.txt"), []byte(jobID.String()+"\n"), 0o644)
|
||||
|
||||
defend := make(chan struct{})
|
||||
go func() {
|
||||
t := time.NewTicker(500 * time.Millisecond)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-defend:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'failed', error = 'yielded to live enhance examples batch B', updated_at = now()
|
||||
WHERE company_id = $1 AND status IN ('pending','running','processing') AND id <> $2`, companyID, jobID)
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'running', error = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status IN ('failed','cancelled')
|
||||
AND (error ILIKE '%yield%' OR error ILIKE '%parked%' OR error ILIKE '%reclaim%'
|
||||
OR error ILIKE '%batch_%' OR error ILIKE '%enhance examples%' OR error ILIKE '%formula%')`, jobID)
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_job_products
|
||||
SET status = 'pending', error = NULL, updated_at = now()
|
||||
WHERE job_id = $1 AND status IN ('failed','cancelled')
|
||||
AND processed_product_id IS NULL
|
||||
AND (error ILIKE '%yield%' OR error ILIKE '%parked%' OR error ILIKE '%reclaim%'
|
||||
OR error ILIKE '%batch_%' OR error ILIKE '%enhance examples%' OR error ILIKE '%formula%')`, jobID)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
_, _ = pool.Exec(ctx, `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
|
||||
WHERE id = $1`, jobID)
|
||||
|
||||
runCtx, runCancel := context.WithTimeout(ctx, 25*time.Minute)
|
||||
defer runCancel()
|
||||
if err := pipeline.ProcessJob(runCtx, jobID); err != nil {
|
||||
close(defend)
|
||||
log.Fatalf("ProcessJob: %v", err)
|
||||
}
|
||||
close(defend)
|
||||
|
||||
items, err := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
|
||||
if err != nil {
|
||||
log.Fatalf("LoadV1ProcessJobItems: %v", err)
|
||||
}
|
||||
writeJSON(out, "final.json", map[string]any{
|
||||
"data": map[string]any{
|
||||
"process_id": jobID.String(),
|
||||
"status": "COMPLETED",
|
||||
"processing_type": "full",
|
||||
"total_items": len(items),
|
||||
"items": items,
|
||||
"llm": map[string]any{
|
||||
"base": base,
|
||||
"model": model,
|
||||
"source": roleCfg.Source,
|
||||
"mode": modeLabel,
|
||||
"byok": byok,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT rp.gtin,
|
||||
COALESCE(pp.processed_name, pp.name, ''),
|
||||
COALESCE(NULLIF(pp.processed_description, ''), pp.description, ''),
|
||||
COALESCE(pp.category, ''),
|
||||
COALESCE(c.name, ''),
|
||||
COALESCE(pp.meta_title, ''),
|
||||
COALESCE(pp.meta_description, ''),
|
||||
COALESCE(pp.processed_attributes, pp.attributes, '{}'::jsonb),
|
||||
COALESCE(pp.ai_provider_mode, ''),
|
||||
c.description_template,
|
||||
c.title_template,
|
||||
pp.id,
|
||||
COALESCE(pp.field_sources, '{}'::jsonb)
|
||||
FROM processing_job_products pjp
|
||||
JOIN raw_products rp ON rp.id = pjp.raw_product_id
|
||||
LEFT JOIN processed_products pp ON pp.id = pjp.processed_product_id
|
||||
LEFT JOIN categories c ON c.company_id = pp.company_id AND c.unique_id = pp.category
|
||||
WHERE pjp.job_id = $1
|
||||
ORDER BY rp.gtin`, jobID)
|
||||
if err != nil {
|
||||
log.Fatalf("db products: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
dbProducts := make([]dbProduct, 0)
|
||||
for rows.Next() {
|
||||
var p dbProduct
|
||||
var ppID *uuid.UUID
|
||||
var attrs, sources []byte
|
||||
if err := rows.Scan(&p.EAN, &p.Title, &p.Description, &p.Category, &p.CategoryName,
|
||||
&p.MetaTitle, &p.MetaDescription, &attrs, &p.AIProviderMode,
|
||||
&p.DescTemplate, &p.TitleTemplate, &ppID, &sources); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if ppID != nil {
|
||||
p.ProductID = ppID.String()
|
||||
}
|
||||
p.Label = eanLabels[p.EAN]
|
||||
p.Attributes = attrs
|
||||
p.FieldSources = sources
|
||||
p.FormulaConstraint = processing.FormatDescriptionFormulaConstraint(p.DescTemplate)
|
||||
dbProducts = append(dbProducts, p)
|
||||
}
|
||||
writeJSON(out, "db_products.json", dbProducts)
|
||||
scoreAndWrite(out, jobID.String(), resolveInfo, dbProducts)
|
||||
fmt.Printf("BATCH_B JOB %s products=%d live_base=%s model=%s\n", jobID, len(dbProducts), base, model)
|
||||
}
|
||||
|
||||
func waitForeignJobs(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) {
|
||||
deadline := time.Now().Add(18 * time.Minute)
|
||||
for time.Now().Before(deadline) {
|
||||
var n int
|
||||
err := pool.QueryRow(ctx, `
|
||||
SELECT count(*) FROM processing_jobs
|
||||
WHERE company_id = $1 AND status IN ('pending','running','processing')`, companyID).Scan(&n)
|
||||
if err != nil {
|
||||
log.Printf("wait foreign jobs: %v", err)
|
||||
return
|
||||
}
|
||||
if n == 0 {
|
||||
log.Printf("queue clear — starting batch B")
|
||||
return
|
||||
}
|
||||
log.Printf("waiting for %d foreign Demo job(s) before batch B…", n)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(15 * time.Second):
|
||||
}
|
||||
}
|
||||
log.Printf("wait deadline reached — proceeding (will yield foreign jobs)")
|
||||
}
|
||||
|
||||
func scoreAndWrite(out, jobID string, resolve map[string]any, products []dbProduct) {
|
||||
scored := make([]map[string]any, 0, len(products))
|
||||
pass, fail := 0, 0
|
||||
var md strings.Builder
|
||||
md.WriteString("# A1 enhance examples — batch B (appliances / monitors / mounts)\n\n")
|
||||
md.WriteString(fmt.Sprintf("- scored_at_utc: `%s`\n", time.Now().UTC().Format(time.RFC3339)))
|
||||
md.WriteString(fmt.Sprintf("- job_id: `%s`\n", jobID))
|
||||
md.WriteString(fmt.Sprintf("- live_base: `%v` model: `%v` source: `%v`\n", resolve["completer_base"], resolve["completer_model"], resolve["role_source"]))
|
||||
md.WriteString(fmt.Sprintf("- mock_fallback: **false** (env mock ignored when DB admin AI resolves)\n\n"))
|
||||
md.WriteString("Scoring: title quality · HTML formula tags (h1/h2/p/ul) · meta · attrs\n\n")
|
||||
|
||||
for _, p := range products {
|
||||
title := strings.TrimSpace(p.Title)
|
||||
desc := p.Description
|
||||
metaT := strings.TrimSpace(p.MetaTitle)
|
||||
metaD := strings.TrimSpace(p.MetaDescription)
|
||||
|
||||
has := map[string]bool{
|
||||
"h1": strings.Contains(strings.ToLower(desc), "<h1"),
|
||||
"h2": strings.Contains(strings.ToLower(desc), "<h2"),
|
||||
"p": strings.Contains(strings.ToLower(desc), "<p"),
|
||||
"ul": strings.Contains(strings.ToLower(desc), "<ul"),
|
||||
}
|
||||
htmlOK := has["h1"] && has["h2"] && has["p"] && has["ul"]
|
||||
|
||||
titleOK := title != "" && utf8.RuneCountInString(title) >= 8 && !brandOnlyish.MatchString(title)
|
||||
titleNotes := []string{}
|
||||
if title == "" {
|
||||
titleNotes = append(titleNotes, "empty")
|
||||
} else if brandOnlyish.MatchString(title) {
|
||||
titleNotes = append(titleNotes, "brand_only")
|
||||
} else if utf8.RuneCountInString(title) < 8 {
|
||||
titleNotes = append(titleNotes, "too_short")
|
||||
}
|
||||
if strings.Contains(strings.ToLower(title), "title formula") || strings.Contains(strings.ToLower(title), "category:") {
|
||||
titleOK = false
|
||||
titleNotes = append(titleNotes, "prompt_leakage")
|
||||
}
|
||||
|
||||
metaOK := metaT != "" && metaD != "" && !poisonMeta.MatchString(metaT) &&
|
||||
!strings.Contains(strings.ToLower(metaT), "short retail title")
|
||||
metaNotes := []string{}
|
||||
if metaT == "" {
|
||||
metaNotes = append(metaNotes, "empty_meta_title")
|
||||
}
|
||||
if metaD == "" {
|
||||
metaNotes = append(metaNotes, "empty_meta_description")
|
||||
}
|
||||
if poisonMeta.MatchString(metaT) {
|
||||
metaNotes = append(metaNotes, "poison_|digits")
|
||||
}
|
||||
|
||||
attrMap := map[string]any{}
|
||||
_ = json.Unmarshal(p.Attributes, &attrMap)
|
||||
attrKeys := make([]string, 0, len(attrMap))
|
||||
dirty := []string{}
|
||||
for k, v := range attrMap {
|
||||
attrKeys = append(attrKeys, k)
|
||||
sv := strings.TrimSpace(fmt.Sprint(v))
|
||||
if sv == "" || strings.EqualFold(sv, "null") || strings.EqualFold(sv, "n/a") {
|
||||
dirty = append(dirty, k+":empty")
|
||||
}
|
||||
if strings.Contains(strings.ToLower(k), "ean") || strings.Contains(strings.ToLower(k), "gtin") {
|
||||
dirty = append(dirty, k+":id_like_key")
|
||||
}
|
||||
}
|
||||
attrsOK := len(attrKeys) > 0 && len(dirty) == 0
|
||||
|
||||
checks := map[string]string{
|
||||
"title": passFail(titleOK),
|
||||
"html_tags": passFail(htmlOK),
|
||||
"meta": passFail(metaOK),
|
||||
"attrs": passFail(attrsOK),
|
||||
"category": passFail(p.Category != "" && p.CategoryName != ""),
|
||||
"desc_ne_title": passFail(strings.TrimSpace(stripTags(desc)) != "" && strings.TrimSpace(stripTags(desc)) != title),
|
||||
"ai_internal": passFail(strings.EqualFold(p.AIProviderMode, "internal") || p.AIProviderMode == ""),
|
||||
}
|
||||
failFlags := []string{}
|
||||
for k, v := range checks {
|
||||
if v == "FAIL" {
|
||||
failFlags = append(failFlags, k)
|
||||
}
|
||||
}
|
||||
verdict := "PASS"
|
||||
if len(failFlags) > 0 {
|
||||
verdict = "FAIL"
|
||||
fail++
|
||||
} else {
|
||||
pass++
|
||||
}
|
||||
|
||||
row := map[string]any{
|
||||
"ean": p.EAN,
|
||||
"label": p.Label,
|
||||
"product_id": p.ProductID,
|
||||
"title": title,
|
||||
"category": p.Category,
|
||||
"category_name": p.CategoryName,
|
||||
"desc_len": len(desc),
|
||||
"desc_preview": truncate(stripTags(desc), 140),
|
||||
"meta_title": metaT,
|
||||
"meta_description": truncate(metaD, 160),
|
||||
"attr_keys": attrKeys,
|
||||
"dirty_attrs": dirty,
|
||||
"attributes": attrMap,
|
||||
"has_tags": has,
|
||||
"formula_constraint": p.FormulaConstraint,
|
||||
"formula_override_hint": strings.Contains(strings.ToLower(fmt.Sprint(p.FieldSources)), "formula"),
|
||||
"ai_provider_mode": p.AIProviderMode,
|
||||
"title_notes": titleNotes,
|
||||
"meta_notes": metaNotes,
|
||||
"checks": checks,
|
||||
"fail_flags": failFlags,
|
||||
"verdict": verdict,
|
||||
}
|
||||
scored = append(scored, row)
|
||||
|
||||
md.WriteString(fmt.Sprintf("## %s — %s (`%s`)\n\n", p.EAN, p.Label, verdict))
|
||||
md.WriteString(fmt.Sprintf("- **title:** %s\n", title))
|
||||
md.WriteString(fmt.Sprintf("- **category:** %s / %s\n", p.Category, p.CategoryName))
|
||||
md.WriteString(fmt.Sprintf("- **meta_title:** %s\n", metaT))
|
||||
md.WriteString(fmt.Sprintf("- **meta_description:** %s\n", truncate(metaD, 200)))
|
||||
md.WriteString(fmt.Sprintf("- **attrs:** %s\n", strings.Join(attrKeys, ", ")))
|
||||
md.WriteString(fmt.Sprintf("- **HTML tags:** h1=%v h2=%v p=%v ul=%v\n", has["h1"], has["h2"], has["p"], has["ul"]))
|
||||
md.WriteString(fmt.Sprintf("- **checks:** title=%s html=%s meta=%s attrs=%s\n\n", checks["title"], checks["html_tags"], checks["meta"], checks["attrs"]))
|
||||
md.WriteString("### Description HTML (raw)\n\n```html\n")
|
||||
if len(desc) > 4096 {
|
||||
md.WriteString(desc[:4096])
|
||||
md.WriteString("\n<!-- truncated -->\n")
|
||||
} else {
|
||||
md.WriteString(desc)
|
||||
md.WriteString("\n")
|
||||
}
|
||||
md.WriteString("```\n\n")
|
||||
}
|
||||
|
||||
total := len(products)
|
||||
rate := 0
|
||||
if total > 0 {
|
||||
rate = (pass * 100) / total
|
||||
}
|
||||
scorecard := map[string]any{
|
||||
"batch": "B",
|
||||
"date": "2026-08-16",
|
||||
"scored_at_utc": time.Now().UTC().Format(time.RFC3339),
|
||||
"company_id": "2b3159b0-fc08-415b-b248-35ed02a6baab",
|
||||
"company_name": "Platform Demo",
|
||||
"job_id": jobID,
|
||||
"selection": eanLabels,
|
||||
"live_llm": resolve,
|
||||
"mock_as_live": false,
|
||||
"asserts": []string{"title_quality", "html_h1_h2_p_ul", "meta", "attrs", "category", "desc_ne_title"},
|
||||
"summary": map[string]any{"pass": pass, "fail": fail, "total": total, "pass_rate": rate},
|
||||
"verdict": map[bool]string{true: "PASS", false: "FAIL"}[fail == 0 && total > 0],
|
||||
"products": scored,
|
||||
"max_tokens_note": fmt.Sprintf("MaxTokensEnhance=%d MaxTokensEnhanceRetry=%d", processing.MaxTokensEnhance, processing.MaxTokensEnhanceRetry),
|
||||
}
|
||||
writeJSON(out, "scorecard.json", scorecard)
|
||||
_ = os.WriteFile(filepath.Join(out, "examples.md"), []byte(md.String()), 0o644)
|
||||
|
||||
summary := fmt.Sprintf("# Batch B summary\n\n- verdict: **%s** (%d/%d)\n- live: `%v` / `%v`\n- mock_as_live: false\n- artifacts: scorecard.json, examples.md, db_products.json, final.json, resolve.json\n",
|
||||
scorecard["verdict"], pass, total, resolve["completer_base"], resolve["completer_model"])
|
||||
_ = os.WriteFile(filepath.Join(out, "summary.md"), []byte(summary), 0o644)
|
||||
}
|
||||
|
||||
func passFail(ok bool) string {
|
||||
if ok {
|
||||
return "PASS"
|
||||
}
|
||||
return "FAIL"
|
||||
}
|
||||
|
||||
func stripTags(s string) string {
|
||||
re := regexp.MustCompile(`<[^>]+>`)
|
||||
return strings.TrimSpace(re.ReplaceAllString(s, " "))
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
r := []rune(strings.TrimSpace(s))
|
||||
if len(r) <= n {
|
||||
return string(r)
|
||||
}
|
||||
return string(r[:n]) + "…"
|
||||
}
|
||||
|
||||
func failResolve(out, msg string) {
|
||||
writeJSON(out, "resolve.json", map[string]any{"live": false, "fail_reason": msg, "batch": "B"})
|
||||
log.Fatalf("LIVE LLM REQUIRED: %s", msg)
|
||||
}
|
||||
|
||||
func writeJSON(out, name string, v any) {
|
||||
raw, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("marshal %s: %v", name, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(out, name), raw, 0o644); err != nil {
|
||||
log.Fatalf("write %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,18 @@
|
||||
// Command repair-category-prompts replaces legacy A1 / Platform Demo categories.prompt
|
||||
// HTML marketing formulas with aiprompts.CategoryEnhanceUserTemplate (JSON-compatible
|
||||
// user overlay: {{name}} {{description}} {{attrs}} {{category}} {{language}}; stores
|
||||
// both prompt->>'sl' and "*"). Does not touch title_template / description_template
|
||||
// (formulas are appended as plain-text instructions at enhance render time).
|
||||
// Command repair-category-prompts rewrites A1 / Platform Demo categories.prompt
|
||||
// values into aiprompts role-sectioned overlays (Title / Description / Meta /
|
||||
// Attributes). Prefers wp_product_categories.sql (SEED_A1_WP_CATEGORIES /
|
||||
// Downloads) as source of truth, else a1-category-prompts.json. Legacy combined
|
||||
// Slovenian name+description blobs are split so naming rules land under Title
|
||||
// and HTML under Description; otherwise CategoryEnhanceUserTemplate is used.
|
||||
// Also repairs empty or brand-only title_template and empty description_template
|
||||
// from legacy <name> / HTML / meta rules (never brand-only).
|
||||
//
|
||||
// Usage (from apps/api):
|
||||
//
|
||||
// go run ./cmd/repair-category-prompts -dry-run
|
||||
// go run ./cmd/repair-category-prompts -apply
|
||||
// go run ./cmd/repair-category-prompts -apply -wp-categories "D:/Users/.../Downloads/wp_product_categories.sql"
|
||||
// go run ./cmd/repair-category-prompts -apply -prompts ../../scripts/seed/a1-category-prompts.json
|
||||
//
|
||||
// DATABASE_URL / -postgres required. Default is dry-run (count only).
|
||||
package main
|
||||
@@ -30,6 +35,8 @@ func main() {
|
||||
postgresURL := flag.String("postgres", os.Getenv("DATABASE_URL"), "Postgres URL (DATABASE_URL)")
|
||||
dryRun := flag.Bool("dry-run", true, "count rows that would update (default true)")
|
||||
apply := flag.Bool("apply", false, "write updates (implies not dry-run)")
|
||||
promptsPath := flag.String("prompts", "", "path to a1-category-prompts.json (optional fallback)")
|
||||
wpPath := flag.String("wp-categories", os.Getenv("SEED_A1_WP_CATEGORIES"), "path to wp_product_categories.sql (preferred source of truth)")
|
||||
flag.Parse()
|
||||
|
||||
if strings.TrimSpace(*postgresURL) == "" {
|
||||
@@ -49,8 +56,13 @@ func main() {
|
||||
}
|
||||
defer pg.Close()
|
||||
|
||||
opts := catalog.RepairA1DemoOptions{
|
||||
SeedPromptsPath: strings.TrimSpace(*promptsPath),
|
||||
WPCategoriesPath: strings.TrimSpace(*wpPath),
|
||||
}
|
||||
|
||||
// Always print dry-run counts first when applying, so operators see the delta.
|
||||
preview, err := catalog.RepairA1DemoCategoryEnhancePrompts(ctx, pg, true)
|
||||
preview, err := catalog.RepairA1DemoCategoryEnhancePromptsWithOptions(ctx, pg, true, opts)
|
||||
if err != nil {
|
||||
log.Fatalf("dry-run: %v", err)
|
||||
}
|
||||
@@ -60,7 +72,7 @@ func main() {
|
||||
return
|
||||
}
|
||||
|
||||
res, err := catalog.RepairA1DemoCategoryEnhancePrompts(ctx, pg, false)
|
||||
res, err := catalog.RepairA1DemoCategoryEnhancePromptsWithOptions(ctx, pg, false, opts)
|
||||
if err != nil {
|
||||
log.Fatalf("apply: %v", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
// Command seed-a1-teammate upserts a second A1 company member (not the owner)
|
||||
// with must_set_password=true and prints a one-time accept-invite URL.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// go run ./cmd/seed-a1-teammate -postgres "$DATABASE_URL"
|
||||
// go run ./cmd/seed-a1-teammate -email a1-other@descrybe.local -role member
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/auth"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/mail"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func main() {
|
||||
postgresURL := flag.String("postgres", os.Getenv("DATABASE_URL"), "Postgres URL")
|
||||
email := flag.String("email", "a1-other@descrybe.local", "Teammate email to upsert")
|
||||
name := flag.String("name", "A1 teammate", "Display name")
|
||||
role := flag.String("role", "member", "Membership role: member|admin")
|
||||
webOrigin := flag.String("web-origin", firstNonEmpty(os.Getenv("WEB_ORIGIN"), "http://localhost:28472"), "Web origin for accept-invite URL")
|
||||
flag.Parse()
|
||||
|
||||
if strings.TrimSpace(*postgresURL) == "" {
|
||||
log.Fatal("-postgres / DATABASE_URL is required")
|
||||
}
|
||||
emailNorm := strings.ToLower(strings.TrimSpace(*email))
|
||||
if emailNorm == "" {
|
||||
log.Fatal("-email is required")
|
||||
}
|
||||
if emailNorm == "a1-primary@descrybe.local" {
|
||||
log.Fatal("-email must not be a1-primary@descrybe.local (that user remains the company owner)")
|
||||
}
|
||||
memRole := auth.NormalizeMembershipRole(*role)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
pg, err := pgxpool.New(ctx, *postgresURL)
|
||||
if err != nil {
|
||||
log.Fatalf("postgres: %v", err)
|
||||
}
|
||||
defer pg.Close()
|
||||
|
||||
var companyID uuid.UUID
|
||||
var companyName string
|
||||
err = pg.QueryRow(ctx, `
|
||||
SELECT id, name FROM companies
|
||||
WHERE legacy_company_id = $1
|
||||
ORDER BY created_at ASC
|
||||
LIMIT 1`, billing.A1LegacyCompanyID).Scan(&companyID, &companyName)
|
||||
if err != nil {
|
||||
log.Fatalf("find A1 company (legacy_company_id=%s): %v", billing.A1LegacyCompanyID, err)
|
||||
}
|
||||
|
||||
tx, err := pg.Begin(ctx)
|
||||
if err != nil {
|
||||
log.Fatalf("begin: %v", err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var userID uuid.UUID
|
||||
err = tx.QueryRow(ctx, `
|
||||
INSERT INTO users (email, name, must_set_password, is_platform_admin, is_active, updated_at)
|
||||
VALUES ($1, $2, true, false, true, now())
|
||||
ON CONFLICT (email) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
must_set_password = true,
|
||||
password_hash = NULL,
|
||||
is_active = true,
|
||||
updated_at = now()
|
||||
RETURNING id`, emailNorm, strings.TrimSpace(*name)).Scan(&userID)
|
||||
if err != nil {
|
||||
log.Fatalf("upsert user: %v", err)
|
||||
}
|
||||
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO memberships (company_id, user_id, role, status)
|
||||
VALUES ($1, $2, $3, 'active')
|
||||
ON CONFLICT (company_id, user_id) DO UPDATE SET
|
||||
role = EXCLUDED.role,
|
||||
status = 'active',
|
||||
updated_at = now()`, companyID, userID, memRole)
|
||||
if err != nil {
|
||||
log.Fatalf("upsert membership: %v", err)
|
||||
}
|
||||
|
||||
// Ensure a1-primary remains owner when present.
|
||||
_, _ = tx.Exec(ctx, `
|
||||
UPDATE companies c
|
||||
SET owner_user_id = u.id, updated_at = now()
|
||||
FROM users u
|
||||
JOIN memberships m ON m.user_id = u.id AND m.company_id = c.id AND m.status = 'active'
|
||||
WHERE c.id = $1
|
||||
AND lower(u.email) = 'a1-primary@descrybe.local'`, companyID)
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
log.Fatalf("commit: %v", err)
|
||||
}
|
||||
|
||||
svc := &auth.Service{Pool: pg}
|
||||
inv, err := svc.ReissueSetPasswordInvite(ctx, userID, 0)
|
||||
if err != nil {
|
||||
log.Fatalf("issue set-password invite: %v", err)
|
||||
}
|
||||
acceptURL := mail.AcceptInviteURL(strings.TrimSpace(*webOrigin), inv.Token)
|
||||
|
||||
fmt.Printf("A1 teammate ready\n")
|
||||
fmt.Printf(" company: %s (%s)\n", companyName, companyID)
|
||||
fmt.Printf(" email: %s\n", emailNorm)
|
||||
fmt.Printf(" user_id: %s\n", userID)
|
||||
fmt.Printf(" role: %s\n", memRole)
|
||||
fmt.Printf(" expires_at: %s\n", inv.ExpiresAt.UTC().Format(time.RFC3339))
|
||||
fmt.Printf(" accept_url: %s\n", acceptURL)
|
||||
fmt.Printf("\nShare accept_url with the teammate (or Admin → Users → Send set-password for this user).\n")
|
||||
fmt.Printf("While impersonating a1-primary, Settings → Team → Invite also returns a copyable link when SMTP is off.\n")
|
||||
}
|
||||
|
||||
func firstNonEmpty(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"unicode"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/catalog"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/security"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
@@ -41,6 +42,10 @@ var (
|
||||
)
|
||||
|
||||
func defaultCategoryPromptsPath(archivePath string) string {
|
||||
// Prefer live WP dump when present (Sync A1 / seed share the same source of truth).
|
||||
if p := catalog.ResolveWPCategoryPromptsPath(""); p != "" {
|
||||
return p
|
||||
}
|
||||
if strings.TrimSpace(archivePath) != "" {
|
||||
return filepath.Join(filepath.Dir(archivePath), "a1-category-prompts.json")
|
||||
}
|
||||
@@ -52,6 +57,10 @@ func loadCategoryPromptsFile(path string) (categoryPromptsFile, error) {
|
||||
if path == "" || path == "." {
|
||||
return categoryPromptsFile{}, fmt.Errorf("category prompts path is empty")
|
||||
}
|
||||
base := strings.ToLower(filepath.Base(path))
|
||||
if strings.HasPrefix(base, "wp_product_categories") && strings.HasSuffix(base, ".sql") {
|
||||
return loadCategoryPromptsFromWPSQL(path)
|
||||
}
|
||||
// Allowlist the committed seed filename (blocks accidental reads of unrelated dumps).
|
||||
if filepath.Base(path) != "a1-category-prompts.json" {
|
||||
return categoryPromptsFile{}, fmt.Errorf("refusing unexpected category prompts file name %q", filepath.Base(path))
|
||||
@@ -76,6 +85,36 @@ func loadCategoryPromptsFile(path string) (categoryPromptsFile, error) {
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func loadCategoryPromptsFromWPSQL(path string) (categoryPromptsFile, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return categoryPromptsFile{}, fmt.Errorf("read wp category prompts: %w", err)
|
||||
}
|
||||
if len(raw) > 16<<20 {
|
||||
return categoryPromptsFile{}, fmt.Errorf("wp category prompts file too large (%d bytes)", len(raw))
|
||||
}
|
||||
parsed, err := catalog.ParseWPProductCategoriesSQL(string(raw))
|
||||
if err != nil {
|
||||
return categoryPromptsFile{}, err
|
||||
}
|
||||
entries := make([]categoryPromptEntry, 0, len(parsed))
|
||||
for _, e := range parsed {
|
||||
entries = append(entries, categoryPromptEntry{
|
||||
Name: e.Name,
|
||||
UniqueID: e.UniqueID,
|
||||
Prompt: e.Prompt,
|
||||
})
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return categoryPromptsFile{}, fmt.Errorf("wp category prompts file has no entries")
|
||||
}
|
||||
return categoryPromptsFile{
|
||||
Version: 1,
|
||||
Source: filepath.Base(path),
|
||||
Entries: entries,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// modernizeLegacyPromptPlaceholders rewrites v1 {""OPIS IZDELKA""} tokens to {{description}} / {{name}}.
|
||||
func modernizeLegacyPromptPlaceholders(prompt string) string {
|
||||
prompt = reLegacyDesc.ReplaceAllString(prompt, "{{description}}")
|
||||
@@ -148,16 +187,11 @@ func applyCategoryPrompts(ctx context.Context, pg *pgxpool.Pool, companyID uuid.
|
||||
|
||||
byNorm := make(map[string]string, len(file.Entries))
|
||||
byUnique := make(map[string]string, len(file.Entries))
|
||||
// Overlay sets the shared role-sectioned enhance user template
|
||||
// (aiprompts.CategoryEnhanceUserTemplate — title/description/meta/attributes).
|
||||
// Seed JSON selects which categories get a prompt (prefer unique_id, else name).
|
||||
// Unique title/description/meta formulas stay in title_template / description_template.
|
||||
// Overlay: split each legacy combined Name+Description prompt into role-sectioned
|
||||
// enhance USER text (Title / Description / Meta / Attributes). Unique title /
|
||||
// description / meta formulas stay in title_template / description_template.
|
||||
// Stored under "sl" + "*" so prompt->>'sl' and LangPromptAny both resolve;
|
||||
// language stays via {{language}}.
|
||||
canonical := prepareCategoryPrompt(aiprompts.CategoryEnhanceUserTemplate)
|
||||
if canonical == "" {
|
||||
return out, fmt.Errorf("CategoryEnhanceUserTemplate sanitized to empty")
|
||||
}
|
||||
for _, e := range file.Entries {
|
||||
uid := strings.TrimSpace(e.UniqueID)
|
||||
name := strings.TrimSpace(e.Name)
|
||||
@@ -165,6 +199,23 @@ func applyCategoryPrompts(ctx context.Context, pg *pgxpool.Pool, companyID uuid.
|
||||
out.Skipped++
|
||||
continue
|
||||
}
|
||||
raw := strings.TrimSpace(e.Prompt)
|
||||
var canonical string
|
||||
if raw == "" {
|
||||
out.Skipped++
|
||||
continue
|
||||
}
|
||||
if aiprompts.IsLegacyCombinedEnhancePrompt(raw) {
|
||||
canonical = prepareCategoryPrompt(aiprompts.SplitLegacyCombinedEnhancePrompt(raw))
|
||||
} else if aiprompts.CategoryEnhanceHasRoleSections(raw) {
|
||||
canonical = prepareCategoryPrompt(raw)
|
||||
} else {
|
||||
canonical = prepareCategoryPrompt(aiprompts.CategoryEnhanceUserTemplate)
|
||||
}
|
||||
if canonical == "" {
|
||||
out.Skipped++
|
||||
continue
|
||||
}
|
||||
if uid != "" {
|
||||
byUnique[strings.ToLower(uid)] = canonical
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user