fix
This commit is contained in:
@@ -0,0 +1,172 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "config: %v\n", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||||
MaxConns: int32(cfg.DBMaxConns), MinConns: int32(cfg.DBMinConns),
|
||||
MaxConnLifetime: cfg.DBMaxConnLifetime, MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
|
||||
MaxConnIdleTime: cfg.DBMaxConnIdleTime, HealthCheckPeriod: cfg.DBHealthCheckPeriod,
|
||||
StatementTimeout: cfg.DBStatementTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "db: %v\n", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
platEnv := platformsettings.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey, CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret, DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey, OpenAIBaseURL: cfg.OpenAIBaseURL, OpenAIModel: cfg.OpenAIModel,
|
||||
}
|
||||
plat := platformsettings.NewService(pool, platEnv)
|
||||
ai := aiprovider.NewService(pool, aiprovider.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey, CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret, DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey, OpenAIBaseURL: cfg.OpenAIBaseURL, OpenAIModel: cfg.OpenAIModel,
|
||||
})
|
||||
ai.Platform = plat
|
||||
|
||||
companyID := uuid.MustParse("604f23a8-b66e-4b21-8b45-0d72b68f4790")
|
||||
roleCfg, err := plat.ResolveAIConfig(ctx, platformsettings.AIRoleProcessing)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "ResolveAIConfig: %v\n", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
completer, mode, byok, err := ai.ResolveCompleterForRole(ctx, companyID, aiprovider.RoleProcessing)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "ResolveCompleter: %v\n", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
client, ok := completer.(*processing.OpenAIClient)
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "not OpenAIClient: %T\n", completer)
|
||||
os.Exit(2)
|
||||
}
|
||||
base := strings.TrimRight(strings.TrimSpace(client.BaseURL), "/")
|
||||
model := strings.TrimSpace(client.Model)
|
||||
if processing.IsMockOrLoopbackBaseURL(base) {
|
||||
fmt.Println("FAIL mock/loopback")
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
fmt.Printf("resolved source=%s provider=%s base=%s model=%s mode=%s byok=%v key_len=%d app_max_tokens_enhance=%d retry=%d http_timeout=240s\n",
|
||||
roleCfg.Source, roleCfg.Provider, base, model, mode, byok, len(client.APIKey),
|
||||
processing.MaxTokensEnhance, processing.MaxTokensEnhanceRetry)
|
||||
|
||||
// Optional: adjust model id if gateway wants green/ prefix (prior resolve used both).
|
||||
models := []string{model}
|
||||
if !strings.HasPrefix(model, "green/") {
|
||||
models = append(models, "green/"+model)
|
||||
}
|
||||
|
||||
size := "tiny"
|
||||
if len(os.Args) > 1 {
|
||||
size = strings.ToLower(strings.TrimSpace(os.Args[1]))
|
||||
}
|
||||
maxTok := 256
|
||||
prompt := "Reply with exactly: OK"
|
||||
timeout := 60 * time.Second
|
||||
switch size {
|
||||
case "tiny":
|
||||
maxTok = 256
|
||||
prompt = "Reply with exactly one word: OK"
|
||||
timeout = 60 * time.Second
|
||||
case "medium":
|
||||
maxTok = 3072
|
||||
prompt = "Write a short JSON object with keys title and description for product Sony WH-1000XM5 headphones. description must be 2 short HTML paragraphs. Keep under 400 words."
|
||||
timeout = 90 * time.Second
|
||||
case "large":
|
||||
maxTok = 16384
|
||||
prompt = "Write a long JSON object with keys title, description, attributes. description must be multi-section Slovenian HTML with several h2+p and a ul list for Sony WH-1000XM5. Be thorough."
|
||||
timeout = 120 * time.Second
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "usage: %s [tiny|medium|large]\n", os.Args[0])
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
useModel := models[0]
|
||||
if len(os.Args) > 2 && strings.TrimSpace(os.Args[2]) != "" {
|
||||
useModel = strings.TrimSpace(os.Args[2])
|
||||
}
|
||||
|
||||
bodyObj := map[string]any{
|
||||
"model": useModel,
|
||||
"temperature": 0.2,
|
||||
"max_tokens": maxTok,
|
||||
"messages": []map[string]string{
|
||||
{"role": "system", "content": "You are a concise assistant."},
|
||||
{"role": "user", "content": prompt},
|
||||
},
|
||||
}
|
||||
raw, _ := json.Marshal(bodyObj)
|
||||
url := base + "/chat/completions"
|
||||
fmt.Printf("probe size=%s model=%s max_tokens=%d timeout=%s url=%s prompt_len=%d\n",
|
||||
size, useModel, maxTok, timeout, url, len(prompt))
|
||||
|
||||
reqCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, url, bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "new request: %v\n", err)
|
||||
os.Exit(4)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+client.APIKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
// Separate dial vs header wait so we can tell "hung with no headers" from slow body.
|
||||
headerTO := 25 * time.Second
|
||||
if timeout < headerTO {
|
||||
headerTO = timeout
|
||||
}
|
||||
httpClient := &http.Client{
|
||||
Timeout: timeout,
|
||||
Transport: &http.Transport{
|
||||
ResponseHeaderTimeout: headerTO,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
start := time.Now()
|
||||
resp, err := httpClient.Do(req)
|
||||
elapsed := time.Since(start).Round(time.Millisecond)
|
||||
if err != nil {
|
||||
fmt.Printf("RESULT ok=false elapsed=%s header_timeout=%s err=%v\n", elapsed, headerTO, err)
|
||||
os.Exit(5)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||
snip := strings.TrimSpace(string(b))
|
||||
if len(snip) > 500 {
|
||||
snip = snip[:500] + "…"
|
||||
}
|
||||
fmt.Printf("RESULT ok=%t status=%d elapsed=%s body_len=%d snippet=%s\n",
|
||||
resp.StatusCode >= 200 && resp.StatusCode < 300, resp.StatusCode, elapsed, len(b), snip)
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
os.Exit(6)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,953 @@
|
||||
// Temporary local full verification: Sync A1 + live LLM ProcessJob.
|
||||
// Do not commit.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func main() {
|
||||
out := os.Getenv("PROBE_OUT_DIR")
|
||||
if out == "" {
|
||||
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_final_verify_20260817`
|
||||
}
|
||||
_ = os.MkdirAll(out, 0o755)
|
||||
|
||||
logPath := filepath.Join(out, "inproc_run.log")
|
||||
logFile, err := os.Create(logPath)
|
||||
if err != nil {
|
||||
log.Fatalf("log file: %v", err)
|
||||
}
|
||||
defer logFile.Close()
|
||||
log.SetOutput(logredact.Writer(io.MultiWriter(os.Stderr, logFile)))
|
||||
|
||||
// Repo seed only — no Downloads / admin upload path.
|
||||
repoRoot := `F:\laragon\www\_MY\descrybe-v2`
|
||||
wpPath := filepath.Join(repoRoot, "scripts", "seed", "wp_product_categories.sql")
|
||||
wpBytes, err := os.ReadFile(wpPath)
|
||||
if err != nil {
|
||||
log.Fatalf("read repo seed wp categories: %v", err)
|
||||
}
|
||||
_ = os.Setenv("SEED_A1_WP_CATEGORIES", wpPath)
|
||||
|
||||
// Prefer issue EANs + mapped-category mix (TV, monitor, headphones, appliance).
|
||||
// 4 mapped-category products (LIVE A1; ~2–4 min/item).
|
||||
eans := []string{
|
||||
"4548736132597", // Slušalke
|
||||
"195348253666", // Gaming monitorji
|
||||
"8806097118565", // Televizorji
|
||||
"3838782459856", // Pomivalni stroji
|
||||
}
|
||||
issueEAN := "" // skip uncategorized issue EAN this pass
|
||||
|
||||
companyID := uuid.MustParse("604f23a8-b66e-4b21-8b45-0d72b68f4790") // A1 Slovenija
|
||||
demoID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab") // Platform Demo
|
||||
userID := uuid.MustParse("6bf00877-a693-4d77-b28e-8c8292adac98") // a1-primary@descrybe.local
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("config: %v", err)
|
||||
}
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||||
MaxConns: int32(cfg.DBMaxConns),
|
||||
MinConns: int32(cfg.DBMinConns),
|
||||
MaxConnLifetime: cfg.DBMaxConnLifetime,
|
||||
MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
|
||||
MaxConnIdleTime: cfg.DBMaxConnIdleTime,
|
||||
HealthCheckPeriod: cfg.DBHealthCheckPeriod,
|
||||
StatementTimeout: cfg.DBStatementTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("db: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
syncReport := map[string]any{
|
||||
"wp_path": wpPath,
|
||||
"wp_bytes": len(wpBytes),
|
||||
"companies": []any{},
|
||||
"generated": time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
|
||||
for _, co := range []struct {
|
||||
ID uuid.UUID
|
||||
Name string
|
||||
}{
|
||||
{companyID, "A1 Slovenija"},
|
||||
{demoID, "Platform Demo"},
|
||||
} {
|
||||
var legacy string
|
||||
_ = pool.QueryRow(ctx, `SELECT COALESCE(legacy_company_id,'') FROM companies WHERE id=$1`, co.ID).Scan(&legacy)
|
||||
a1Cohort := billing.IsA1CohortCompany(legacy, co.Name)
|
||||
res, serr := processing.SyncCompanyA1(ctx, pool, co.ID, co.Name, a1Cohort, processing.SyncCompanyA1Opts{
|
||||
MySQLDumpPath: "", // unused — repo seed only (no Downloads dump)
|
||||
SkipDumpBackfill: true, // do not auto-detect ~/Downloads MySQL dump
|
||||
FixCompanyCatalogOpts: processing.FixCompanyCatalogOpts{
|
||||
BackfillCategories: true,
|
||||
ReprocessSampleLimit: 8,
|
||||
WPCategoriesSQL: wpBytes, // scripts/seed/wp_product_categories.sql
|
||||
AIPrompts: aiprompts.NewService(pool),
|
||||
},
|
||||
})
|
||||
entry := map[string]any{
|
||||
"company_id": co.ID.String(),
|
||||
"company_name": co.Name,
|
||||
"a1_cohort": a1Cohort,
|
||||
"error": nil,
|
||||
"result": res,
|
||||
}
|
||||
if serr != nil {
|
||||
entry["error"] = serr.Error()
|
||||
log.Printf("SyncCompanyA1 %s: %v", co.Name, serr)
|
||||
} else {
|
||||
log.Printf("SyncCompanyA1 %s dump=%s prompts_updated=%d already_ok=%d wp_entries=%d weak_hashes=%d mapped_with=%d",
|
||||
co.Name, res.DumpStatus, res.CategoryPromptsUpdated, res.CategoryPromptsAlreadyOK,
|
||||
res.WPCategoriesEntries, res.WeakHashesCleared, res.MappedWithCategory)
|
||||
}
|
||||
syncReport["companies"] = append(syncReport["companies"].([]any), entry)
|
||||
}
|
||||
writeJSON(out, "sync.json", syncReport)
|
||||
|
||||
// Confirm sectioned prompts on A1.
|
||||
promptStats := confirmPromptSections(ctx, pool, companyID)
|
||||
writeJSON(out, "prompt_sections.json", promptStats)
|
||||
|
||||
platEnv := platformsettings.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey,
|
||||
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret,
|
||||
DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey,
|
||||
OpenAIBaseURL: cfg.OpenAIBaseURL,
|
||||
OpenAIModel: cfg.OpenAIModel,
|
||||
OpenAIEmbeddingAPIKey: cfg.OpenAIEmbeddingAPIKey,
|
||||
OpenAIEmbeddingBaseURL: cfg.OpenAIEmbeddingBaseURL,
|
||||
OpenAIEmbeddingModel: cfg.OpenAIEmbeddingModel,
|
||||
}
|
||||
platSettings := platformsettings.NewService(pool, platEnv)
|
||||
aiSvc := aiprovider.NewService(pool, aiprovider.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey,
|
||||
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret,
|
||||
DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey,
|
||||
OpenAIBaseURL: cfg.OpenAIBaseURL,
|
||||
OpenAIModel: cfg.OpenAIModel,
|
||||
ProcessingRPM: cfg.ProcessingRPM,
|
||||
ProcessingMaxRetries: cfg.ProcessingMaxRetries,
|
||||
})
|
||||
aiSvc.Platform = platSettings
|
||||
|
||||
roleCfg, rerr := platSettings.ResolveAIConfig(ctx, platformsettings.AIRoleProcessing)
|
||||
if rerr != nil {
|
||||
failResolve(out, fmt.Sprintf("ResolveAIConfig: %v", rerr))
|
||||
}
|
||||
completer, modeLabel, byok, cerr := aiSvc.ResolveCompleterForRole(ctx, companyID, aiprovider.RoleProcessing)
|
||||
if cerr != nil {
|
||||
failResolve(out, fmt.Sprintf("ResolveCompleterForRole: %v", cerr))
|
||||
}
|
||||
if completer == nil {
|
||||
failResolve(out, "ResolveCompleterForRole returned nil completer")
|
||||
}
|
||||
client, ok := completer.(*processing.OpenAIClient)
|
||||
if !ok {
|
||||
failResolve(out, fmt.Sprintf("completer type %T is not *OpenAIClient", completer))
|
||||
}
|
||||
base := strings.TrimSpace(client.BaseURL)
|
||||
model := strings.TrimSpace(client.Model)
|
||||
resolveInfo := map[string]any{
|
||||
"role_source": roleCfg.Source,
|
||||
"role_provider": roleCfg.Provider,
|
||||
"role_base_url": roleCfg.BaseURL,
|
||||
"role_model": roleCfg.Model,
|
||||
"role_enabled": roleCfg.Enabled,
|
||||
"completer_base": base,
|
||||
"completer_model": model,
|
||||
"mode_label": modeLabel,
|
||||
"byok": byok,
|
||||
"env_base_url": cfg.OpenAIBaseURL,
|
||||
"env_model": cfg.OpenAIModel,
|
||||
"live": true,
|
||||
}
|
||||
if processing.IsMockOrLoopbackBaseURL(base) || strings.Contains(strings.ToLower(base), "18767") {
|
||||
resolveInfo["live"] = false
|
||||
resolveInfo["fail_reason"] = "resolved completer is mock/loopback — refuse silent mock fallback"
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM REQUIRED: resolved base=%s model=%s source=%s mode=%s — refuse mock",
|
||||
base, model, roleCfg.Source, modeLabel)
|
||||
}
|
||||
if strings.TrimSpace(client.APIKey) == "" {
|
||||
resolveInfo["live"] = false
|
||||
resolveInfo["fail_reason"] = "empty API key"
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatal("LIVE LLM REQUIRED: API key empty after resolve")
|
||||
}
|
||||
|
||||
// Connectivity probe — fail clearly on 502.
|
||||
modelsURL := strings.TrimRight(base, "/") + "/models"
|
||||
keyHint := ""
|
||||
if k := strings.TrimSpace(client.APIKey); len(k) > 8 {
|
||||
keyHint = k[:4] + "…" + k[len(k)-4:]
|
||||
}
|
||||
resolveInfo["api_key_hint"] = keyHint
|
||||
modelsCtx, modelsCancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer modelsCancel()
|
||||
modelsReq, merr := http.NewRequestWithContext(modelsCtx, http.MethodGet, modelsURL, nil)
|
||||
if merr != nil {
|
||||
failResolve(out, "models request build: "+merr.Error())
|
||||
}
|
||||
modelsReq.Header.Set("Authorization", "Bearer "+client.APIKey)
|
||||
modelsReq.Header.Set("Accept", "application/json")
|
||||
modelsStart := time.Now()
|
||||
modelsResp, merr := http.DefaultClient.Do(modelsReq)
|
||||
modelsElapsed := time.Since(modelsStart).Round(time.Millisecond)
|
||||
if merr != nil {
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["models_error"] = processing.TruncateError(merr)
|
||||
resolveInfo["fail_reason"] = "GET /v1/models network error"
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: GET %s err=%s", modelsURL, processing.TruncateError(merr))
|
||||
}
|
||||
modelsBody, _ := io.ReadAll(io.LimitReader(modelsResp.Body, 2048))
|
||||
_ = modelsResp.Body.Close()
|
||||
snippet := strings.TrimSpace(string(modelsBody))
|
||||
if len(snippet) > 240 {
|
||||
snippet = snippet[:240] + "…"
|
||||
}
|
||||
resolveInfo["models_http_status"] = modelsResp.StatusCode
|
||||
resolveInfo["models_elapsed"] = modelsElapsed.String()
|
||||
resolveInfo["models_body_snippet"] = snippet
|
||||
if modelsResp.StatusCode == http.StatusBadGateway {
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["fail_reason"] = "GET /v1/models returned HTTP 502 (OverloadedBot/upstream)"
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM 502: GET %s — OverloadedBot upstream unavailable — restart note; no mock", modelsURL)
|
||||
}
|
||||
// 200 = healthy; 401 = gateway up (auth) — both allow ProcessJob with resolved key.
|
||||
if modelsResp.StatusCode != http.StatusOK && modelsResp.StatusCode != http.StatusUnauthorized {
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["fail_reason"] = fmt.Sprintf("GET /v1/models HTTP %d", modelsResp.StatusCode)
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: GET %s status=%d snippet=%q", modelsURL, modelsResp.StatusCode, snippet)
|
||||
}
|
||||
resolveInfo["models_auth_ok"] = modelsResp.StatusCode == http.StatusOK
|
||||
if model == "code-fast" && strings.Contains(string(modelsBody), `"green/code-fast"`) {
|
||||
client.Model = "green/code-fast"
|
||||
model = client.Model
|
||||
resolveInfo["completer_model_adjusted"] = model
|
||||
log.Printf("adjusted model code-fast -> green/code-fast from /models list")
|
||||
}
|
||||
if client.HTTPClient != nil {
|
||||
client.HTTPClient.Timeout = 180 * time.Second
|
||||
}
|
||||
client.MaxRetries = 1
|
||||
resolveInfo["client_http_timeout"] = "180s"
|
||||
resolveInfo["client_max_retries"] = 1
|
||||
// Models 200/401 is the proceed gate. Chat probe is best-effort (retry); only 502 hard-stops.
|
||||
var probeOK bool
|
||||
var probeErr string
|
||||
var probeElapsed time.Duration
|
||||
var probeLen int
|
||||
for attempt := 1; attempt <= 3; attempt++ {
|
||||
probeCtx, probeCancel := context.WithTimeout(ctx, 90*time.Second)
|
||||
probeStart := time.Now()
|
||||
comp, perr := client.Complete(probeCtx, "Reply with exactly: PONG", "Say PONG")
|
||||
probeElapsed = time.Since(probeStart).Round(time.Millisecond)
|
||||
probeCancel()
|
||||
if perr == nil {
|
||||
probeOK = true
|
||||
probeLen = len(comp.Text)
|
||||
break
|
||||
}
|
||||
probeErr = processing.TruncateError(perr)
|
||||
log.Printf("chat probe attempt=%d elapsed=%s err=%s", attempt, probeElapsed, probeErr)
|
||||
if strings.Contains(probeErr, "502") {
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["probe_error"] = probeErr
|
||||
resolveInfo["probe_elapsed"] = probeElapsed.String()
|
||||
resolveInfo["fail_reason"] = "chat/completions HTTP 502"
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM 502: chat probe failed — restart note; no mock — %s", probeErr)
|
||||
}
|
||||
time.Sleep(time.Duration(attempt) * 2 * time.Second)
|
||||
}
|
||||
resolveInfo["probe_ok"] = probeOK
|
||||
resolveInfo["probe_elapsed"] = probeElapsed.String()
|
||||
resolveInfo["probe_response_len"] = probeLen
|
||||
if !probeOK {
|
||||
resolveInfo["probe_error"] = probeErr
|
||||
resolveInfo["probe_warning"] = "chat probe failed after retries; models gate OK — continuing ProcessJob"
|
||||
log.Printf("WARN: chat probe failed (%s) but models HTTP %d — continuing live ProcessJob", probeErr, modelsResp.StatusCode)
|
||||
}
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Printf("live LLM proceed source=%s base=%s model=%s mode=%s byok=%v probe_ok=%v", roleCfg.Source, base, model, modeLabel, byok, probeOK)
|
||||
|
||||
pipeline := processing.NewPipeline(pool)
|
||||
pipeline.BatchSize = cfg.ProcessingBatchSize
|
||||
pipeline.AI = aiSvc
|
||||
pipeline.Prompts = aiprompts.NewService(pool)
|
||||
eprelClient := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second})
|
||||
pipeline.Engine = &processing.Engine{
|
||||
Completer: nil,
|
||||
Vector: processing.NoopVectorCategorizer{},
|
||||
EPREL: eprelClient,
|
||||
ProviderMode: processing.AIProviderInternal,
|
||||
}
|
||||
|
||||
// Prefer mapped-category EANs; include issue EAN if raw exists.
|
||||
selected := append([]string{}, eans...)
|
||||
if strings.TrimSpace(issueEAN) != "" {
|
||||
var issueExists bool
|
||||
_ = pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM raw_products WHERE company_id=$1 AND gtin=$2)`, companyID, issueEAN).Scan(&issueExists)
|
||||
if issueExists {
|
||||
selected = append(selected, issueEAN)
|
||||
}
|
||||
}
|
||||
|
||||
rawIDs := make([]uuid.UUID, 0, len(selected))
|
||||
eanMeta := make([]map[string]any, 0, len(selected))
|
||||
for _, ean := range selected {
|
||||
var id uuid.UUID
|
||||
var cat, catName, title string
|
||||
err := pool.QueryRow(ctx, `
|
||||
SELECT r.id,
|
||||
COALESCE(r.mapped_data->>'category',''),
|
||||
COALESCE(c.name,''),
|
||||
COALESCE(NULLIF(r.mapped_data->>'name',''), NULLIF(r.mapped_data->>'title',''), '')
|
||||
FROM raw_products r
|
||||
LEFT JOIN categories c ON c.company_id=r.company_id AND c.unique_id=r.mapped_data->>'category'
|
||||
WHERE r.company_id=$1 AND r.gtin=$2`, companyID, ean).Scan(&id, &cat, &catName, &title)
|
||||
if err != nil {
|
||||
log.Fatalf("raw product %s: %v", ean, err)
|
||||
}
|
||||
rawIDs = append(rawIDs, id)
|
||||
eanMeta = append(eanMeta, map[string]any{
|
||||
"ean": ean, "raw_id": id.String(), "category": cat, "category_name": catName, "title": title,
|
||||
})
|
||||
log.Printf("ean=%s raw=%s cat=%s/%s", ean, id, cat, catName)
|
||||
}
|
||||
writeJSON(out, "selected_eans.json", eanMeta)
|
||||
|
||||
// Force re-enhance.
|
||||
tag, cerr2 := pool.Exec(ctx, `
|
||||
UPDATE processed_products pp
|
||||
SET field_sources = COALESCE(field_sources, '{}'::jsonb) - 'enhance_input_hash',
|
||||
localized = CASE
|
||||
WHEN localized IS NULL OR localized = '{}'::jsonb THEN localized
|
||||
ELSE (
|
||||
SELECT COALESCE(jsonb_object_agg(lang, val - 'enhance_input_hash'), '{}'::jsonb)
|
||||
FROM jsonb_each(localized) AS t(lang, val)
|
||||
)
|
||||
END,
|
||||
updated_at = now()
|
||||
FROM raw_products rp
|
||||
WHERE pp.company_id = $1
|
||||
AND pp.raw_product_id = rp.id
|
||||
AND rp.gtin = ANY($2::text[])`, companyID, selected)
|
||||
if cerr2 != nil {
|
||||
log.Printf("clear enhance hash: %v", cerr2)
|
||||
} else {
|
||||
log.Printf("cleared enhance_input_hash rows=%d", tag.RowsAffected())
|
||||
}
|
||||
|
||||
_, _ = pool.Exec(ctx, `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'failed', error = 'yielded to final verify 20260817', updated_at = now()
|
||||
WHERE company_id = $1 AND status IN ('pending','running','processing')`, companyID)
|
||||
|
||||
jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full")
|
||||
if err != nil {
|
||||
log.Fatalf("StartJob: %v", err)
|
||||
}
|
||||
if len(jobs) == 0 {
|
||||
log.Fatal("StartJob returned no jobs")
|
||||
}
|
||||
jobID := jobs[0].ID
|
||||
log.Printf("job_id=%s chunks=%d", jobID, len(jobs))
|
||||
_ = os.WriteFile(filepath.Join(out, "process_id.txt"), []byte(jobID.String()+"\n"), 0o644)
|
||||
|
||||
// Start sample expectation: pending + processed_items=0.
|
||||
startSample := map[string]any{
|
||||
"data": map[string]any{
|
||||
"process_id": jobID.String(),
|
||||
"status": strings.ToLower(jobs[0].Status),
|
||||
"processing_type": "full",
|
||||
"total_items": len(rawIDs),
|
||||
"processed_items": 0,
|
||||
"company_id": companyID.String(),
|
||||
},
|
||||
}
|
||||
var dbStatus string
|
||||
var processedCount int
|
||||
_ = pool.QueryRow(ctx, `SELECT lower(status), COALESCE(processed_count,0) FROM processing_jobs WHERE id=$1`, jobID).Scan(&dbStatus, &processedCount)
|
||||
startSample["data"].(map[string]any)["db_status"] = dbStatus
|
||||
startSample["data"].(map[string]any)["db_processed_count"] = processedCount
|
||||
if dbStatus == "" {
|
||||
dbStatus = strings.ToLower(jobs[0].Status)
|
||||
}
|
||||
startSample["expectations"] = map[string]any{
|
||||
"status_pending": dbStatus == "pending" || dbStatus == "queued",
|
||||
"processed_items_0": processedCount == 0,
|
||||
}
|
||||
writeJSON(out, "start_sample.json", startSample)
|
||||
|
||||
defend := make(chan struct{})
|
||||
go func() {
|
||||
t := time.NewTicker(500 * time.Millisecond)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-defend:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'failed', error = 'yielded to final verify 20260817', updated_at = now()
|
||||
WHERE company_id = $1 AND status IN ('pending','running','processing') AND id <> $2`, companyID, jobID)
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'running', error = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status IN ('failed','cancelled')
|
||||
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
|
||||
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
|
||||
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%'
|
||||
OR error ILIKE '%formula%' OR error ILIKE '%live LLM%' OR error ILIKE '%final verify%')`, jobID)
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_job_products
|
||||
SET status = 'pending', error = NULL, updated_at = now()
|
||||
WHERE job_id = $1 AND status IN ('failed','cancelled')
|
||||
AND processed_product_id IS NULL
|
||||
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
|
||||
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
|
||||
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%'
|
||||
OR error ILIKE '%formula%' OR error ILIKE '%live LLM%' OR error ILIKE '%final verify%')`, jobID)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
_, _ = pool.Exec(ctx, `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
|
||||
WHERE id = $1`, jobID)
|
||||
|
||||
runCtx, runCancel := context.WithTimeout(ctx, 18*time.Minute)
|
||||
defer runCancel()
|
||||
if err := pipeline.ProcessJob(runCtx, jobID); err != nil {
|
||||
close(defend)
|
||||
log.Fatalf("ProcessJob: %v", err)
|
||||
}
|
||||
close(defend)
|
||||
|
||||
items, err := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
|
||||
if err != nil {
|
||||
log.Fatalf("LoadV1ProcessJobItems: %v", err)
|
||||
}
|
||||
finalPayload := map[string]any{
|
||||
"data": map[string]any{
|
||||
"process_id": jobID.String(),
|
||||
"status": "COMPLETED",
|
||||
"processing_type": "full",
|
||||
"total_items": len(items),
|
||||
"items": items,
|
||||
"llm": map[string]any{
|
||||
"base": base,
|
||||
"model": model,
|
||||
"source": roleCfg.Source,
|
||||
"mode": modeLabel,
|
||||
"byok": byok,
|
||||
},
|
||||
},
|
||||
}
|
||||
writeJSON(out, "final.json", finalPayload)
|
||||
|
||||
logBytes, _ := os.ReadFile(logPath)
|
||||
logText := string(logBytes)
|
||||
aiEnhanceSeen := strings.Contains(logText, "ai_enhance")
|
||||
skipBlocked := extractSkipBlocked(logText)
|
||||
|
||||
scorecards := make([]map[string]any, 0, len(items))
|
||||
htmlExcerpts := make([]map[string]any, 0, 2)
|
||||
allPass := true
|
||||
startOK := (dbStatus == "pending" || dbStatus == "queued") && processedCount == 0
|
||||
|
||||
for _, item := range items {
|
||||
ean := str(item["ean"])
|
||||
if ean == "" {
|
||||
ean = str(item["product_id"])
|
||||
}
|
||||
title := firstNonEmpty(str(item["name"]), str(item["title"]))
|
||||
desc := str(item["description"])
|
||||
catName := str(item["category_name"])
|
||||
if catName == "" {
|
||||
if m, ok := item["category"].(map[string]any); ok {
|
||||
catName = str(m["name"])
|
||||
}
|
||||
}
|
||||
catID := ""
|
||||
switch c := item["category"].(type) {
|
||||
case string:
|
||||
catID = c
|
||||
case map[string]any:
|
||||
catID = firstNonEmpty(str(c["category_id"]), str(c["id"]), str(c["unique_id"]))
|
||||
if catName == "" {
|
||||
catName = str(c["name"])
|
||||
}
|
||||
}
|
||||
if catID == "" {
|
||||
catID = str(item["category_id"])
|
||||
}
|
||||
|
||||
_, hasMetaTitle := item["meta_title"]
|
||||
_, hasMetaDesc := item["meta_description"]
|
||||
_, hasID := item["id"]
|
||||
_, hasPPID := item["processed_product_id"]
|
||||
_, hasRawID := item["raw_product_id"]
|
||||
|
||||
invent := looksLikeInventBoilerplate(desc)
|
||||
hasHTML := strings.Contains(strings.ToLower(desc), "<h2") || strings.Contains(strings.ToLower(desc), "<p>") || strings.Contains(strings.ToLower(desc), "<ul")
|
||||
titleSpacingOK := !hasCollapsedWords(title) && strings.TrimSpace(title) != ""
|
||||
namePresent := strings.TrimSpace(title) != ""
|
||||
|
||||
// DB field_sources for hash-skip / enhance evidence
|
||||
var fieldSources []byte
|
||||
var dbDesc string
|
||||
_ = pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(pp.field_sources, '{}'::jsonb),
|
||||
COALESCE(NULLIF(pp.processed_description,''), pp.description, '')
|
||||
FROM processing_job_products pjp
|
||||
JOIN raw_products rp ON rp.id = pjp.raw_product_id
|
||||
LEFT JOIN processed_products pp ON pp.id = pjp.processed_product_id
|
||||
WHERE pjp.job_id=$1 AND rp.gtin=$2`, jobID, ean).Scan(&fieldSources, &dbDesc)
|
||||
fs := map[string]any{}
|
||||
_ = json.Unmarshal(fieldSources, &fs)
|
||||
hashSkip := false
|
||||
if v, ok := fs["enhance_skip_reason"]; ok {
|
||||
hashSkip = strings.Contains(strings.ToLower(fmt.Sprint(v)), "hash")
|
||||
}
|
||||
if v, ok := fs["enhance_input_hash"]; ok && v != nil && !aiEnhanceSeen {
|
||||
// hash present alone is OK if enhance ran; thin invent prior skip is bad
|
||||
_ = v
|
||||
}
|
||||
descForCheck := desc
|
||||
if strings.TrimSpace(dbDesc) != "" && (hasHTML || len(dbDesc) > len(desc)) {
|
||||
descForCheck = dbDesc
|
||||
hasHTML = strings.Contains(strings.ToLower(dbDesc), "<h2") || strings.Contains(strings.ToLower(dbDesc), "<p>") || strings.Contains(strings.ToLower(dbDesc), "<ul")
|
||||
invent = looksLikeInventBoilerplate(dbDesc)
|
||||
}
|
||||
|
||||
checks := map[string]bool{
|
||||
"name_present": namePresent,
|
||||
"title_spacing_readable": titleSpacingOK,
|
||||
"no_invent_boilerplate": !invent,
|
||||
"no_meta_title": !hasMetaTitle || item["meta_title"] == nil,
|
||||
"no_meta_description": !hasMetaDesc || item["meta_description"] == nil,
|
||||
"no_id_field": !hasID,
|
||||
"no_processed_product_id": !hasPPID,
|
||||
"no_raw_product_id": !hasRawID,
|
||||
"category_name_present": strings.TrimSpace(catName) != "" && !looksLikeUUID(catName),
|
||||
"category_id_present": strings.TrimSpace(catID) != "" || strings.TrimSpace(catName) != "",
|
||||
"formula_html_or_honest": hasHTML || strings.Contains(strings.ToLower(logText), "empty_or_provider_error") || strings.Contains(strings.ToLower(logText), "refuse"),
|
||||
"not_thin_hash_skip_invent": !hashSkip || !invent,
|
||||
}
|
||||
itemPass := true
|
||||
for _, ok := range checks {
|
||||
if !ok {
|
||||
itemPass = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !itemPass {
|
||||
allPass = false
|
||||
}
|
||||
attrsSummary := summarizeAttrs(item)
|
||||
descSnippet := trunc(descForCheck, 280)
|
||||
card := map[string]any{
|
||||
"ean": ean,
|
||||
"title": title,
|
||||
"category": catName,
|
||||
"category_id": catID,
|
||||
"desc_len": len(descForCheck),
|
||||
"desc_snippet": descSnippet,
|
||||
"has_html": hasHTML,
|
||||
"attrs_summary": attrsSummary,
|
||||
"checks": checks,
|
||||
"pass": itemPass,
|
||||
"field_sources_keys": keysOf(fs),
|
||||
}
|
||||
scorecards = append(scorecards, card)
|
||||
|
||||
if hasHTML && len(htmlExcerpts) < 2 {
|
||||
ex := descForCheck
|
||||
if len(ex) > 900 {
|
||||
ex = ex[:900] + "…"
|
||||
}
|
||||
htmlExcerpts = append(htmlExcerpts, map[string]any{
|
||||
"ean": ean, "title": title, "html_excerpt": ex,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if !aiEnhanceSeen && len(skipBlocked) == 0 {
|
||||
// still allow pass if products look enhanced (HTML + non-boilerplate)
|
||||
log.Printf("WARN: no ai_enhance log line and no skip_blocked — inspect log")
|
||||
}
|
||||
if !startOK {
|
||||
allPass = false
|
||||
}
|
||||
if !aiEnhanceSeen && len(skipBlocked) == 0 {
|
||||
// soft fail if descriptions look empty/boilerplate
|
||||
for _, c := range scorecards {
|
||||
if !c["has_html"].(bool) || !c["pass"].(bool) {
|
||||
allPass = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
overall := "PASS"
|
||||
if !allPass {
|
||||
overall = "FAIL"
|
||||
}
|
||||
report := map[string]any{
|
||||
"generated_at": time.Now().UTC().Format(time.RFC3339),
|
||||
"overall": overall,
|
||||
"company": map[string]any{"id": companyID.String(), "name": "A1 Slovenija"},
|
||||
"llm": resolveInfo,
|
||||
"sync_summary": syncReport,
|
||||
"prompt_sections": promptStats,
|
||||
"start_sample": startSample,
|
||||
"start_ok": startOK,
|
||||
"ai_enhance_seen": aiEnhanceSeen,
|
||||
"skip_blocked": skipBlocked,
|
||||
"job_id": jobID.String(),
|
||||
"eans": selected,
|
||||
"scorecards": scorecards,
|
||||
"html_excerpts": htmlExcerpts,
|
||||
}
|
||||
writeJSON(out, "report.json", report)
|
||||
writeMarkdownReport(out, report)
|
||||
|
||||
fmt.Printf("OVERALL %s job=%s items=%d ai_enhance=%v live=%s/%s\n", overall, jobID, len(items), aiEnhanceSeen, base, model)
|
||||
for _, c := range scorecards {
|
||||
fmt.Printf("CARD ean=%s pass=%v title=%q cat=%s html=%v\n",
|
||||
c["ean"], c["pass"], trunc(str(c["title"]), 50), c["category"], c["has_html"])
|
||||
}
|
||||
}
|
||||
|
||||
func confirmPromptSections(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID) map[string]any {
|
||||
rows, err := pool.Query(ctx, `SELECT name, prompt::text, COALESCE(title_template::text,''), COALESCE(description_template::text,'')
|
||||
FROM categories WHERE company_id=$1`, companyID)
|
||||
if err != nil {
|
||||
return map[string]any{"error": err.Error()}
|
||||
}
|
||||
defer rows.Close()
|
||||
n, withTitleSec, withDescSec, withTitleTmpl, withRichFormula := 0, 0, 0, 0, 0
|
||||
samples := []map[string]any{}
|
||||
for rows.Next() {
|
||||
var name, prompt, tt, dt string
|
||||
_ = rows.Scan(&name, &prompt, &tt, &dt)
|
||||
n++
|
||||
hasT := strings.Contains(prompt, "--- Title ---") || strings.Contains(prompt, `"Title"`)
|
||||
hasD := strings.Contains(prompt, "--- Description ---") || strings.Contains(prompt, `"Description"`)
|
||||
if hasT {
|
||||
withTitleSec++
|
||||
}
|
||||
if hasD {
|
||||
withDescSec++
|
||||
}
|
||||
if strings.TrimSpace(tt) != "" && !strings.EqualFold(strings.TrimSpace(tt), "null") {
|
||||
withTitleTmpl++
|
||||
}
|
||||
if strings.Contains(dt, `"sections"`) || strings.Contains(strings.ToLower(dt), `"h2"`) {
|
||||
withRichFormula++
|
||||
}
|
||||
if (strings.Contains(strings.ToLower(name), "televiz") || strings.Contains(strings.ToLower(name), "monitor") ||
|
||||
strings.Contains(strings.ToLower(name), "sluš") || strings.Contains(strings.ToLower(name), "peč")) && len(samples) < 4 {
|
||||
samples = append(samples, map[string]any{
|
||||
"name": name, "title_section": hasT, "description_section": hasD,
|
||||
"title_template_len": len(tt), "description_template_len": len(dt),
|
||||
})
|
||||
}
|
||||
}
|
||||
return map[string]any{
|
||||
"categories": n,
|
||||
"with_title_section": withTitleSec,
|
||||
"with_description_section": withDescSec,
|
||||
"with_title_template": withTitleTmpl,
|
||||
"with_rich_formula": withRichFormula,
|
||||
"samples": samples,
|
||||
}
|
||||
}
|
||||
|
||||
func looksLikeInventBoilerplate(desc string) bool {
|
||||
d := strings.ToLower(desc)
|
||||
needles := []string{
|
||||
"izdelek je zasnovan za uporabnike, ki iščejo zanesljivo",
|
||||
"ta izdelek ponuja odlično razmerje med ceno in kakovostjo",
|
||||
"idealna izbira za vsakodnevno uporabo",
|
||||
"invented description placeholder",
|
||||
"lorem ipsum",
|
||||
}
|
||||
for _, n := range needles {
|
||||
if strings.Contains(d, n) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasCollapsedWords(title string) bool {
|
||||
// Detect glued tokens like "TVSamsung" (letter followed by uppercase mid-token) without spaces — weak heuristic.
|
||||
if strings.Contains(title, " ") {
|
||||
return false
|
||||
}
|
||||
runes := []rune(title)
|
||||
for i := 1; i < len(runes)-1; i++ {
|
||||
if unicode.IsLower(runes[i-1]) && unicode.IsUpper(runes[i]) && unicode.IsLower(runes[i+1]) {
|
||||
// camelCase mid-title often means missing space after brand/model join
|
||||
prevSpace := false
|
||||
for j := i - 1; j >= 0; j-- {
|
||||
if runes[j] == ' ' {
|
||||
prevSpace = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !prevSpace && i > 3 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func extractSkipBlocked(logText string) []string {
|
||||
out := []string{}
|
||||
for _, line := range strings.Split(logText, "\n") {
|
||||
if strings.Contains(line, "skip_blocked") || strings.Contains(line, "enhance_skip") {
|
||||
s := strings.TrimSpace(line)
|
||||
if len(s) > 300 {
|
||||
s = s[:300] + "…"
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func keysOf(m map[string]any) []string {
|
||||
ks := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
ks = append(ks, k)
|
||||
}
|
||||
return ks
|
||||
}
|
||||
|
||||
func str(v any) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
switch t := v.(type) {
|
||||
case string:
|
||||
return t
|
||||
default:
|
||||
return strings.TrimSpace(fmt.Sprint(t))
|
||||
}
|
||||
}
|
||||
|
||||
func firstNonEmpty(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func trunc(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n]
|
||||
}
|
||||
|
||||
func looksLikeUUID(s string) bool {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) != 36 {
|
||||
return false
|
||||
}
|
||||
for i, r := range s {
|
||||
switch i {
|
||||
case 8, 13, 18, 23:
|
||||
if r != '-' {
|
||||
return false
|
||||
}
|
||||
default:
|
||||
if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func summarizeAttrs(item map[string]any) map[string]any {
|
||||
out := map[string]any{"count": 0, "keys": []string{}}
|
||||
var raw any
|
||||
for _, k := range []string{"attributes", "attrs", "product_attributes"} {
|
||||
if v, ok := item[k]; ok && v != nil {
|
||||
raw = v
|
||||
break
|
||||
}
|
||||
}
|
||||
if raw == nil {
|
||||
return out
|
||||
}
|
||||
keys := []string{}
|
||||
switch t := raw.(type) {
|
||||
case map[string]any:
|
||||
for k := range t {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
case []any:
|
||||
for _, el := range t {
|
||||
if m, ok := el.(map[string]any); ok {
|
||||
name := firstNonEmpty(str(m["name"]), str(m["key"]), str(m["attribute"]))
|
||||
if name != "" {
|
||||
keys = append(keys, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(keys) > 12 {
|
||||
keys = keys[:12]
|
||||
}
|
||||
out["count"] = len(keys)
|
||||
out["keys"] = keys
|
||||
return out
|
||||
}
|
||||
|
||||
func failResolve(out, msg string) {
|
||||
writeJSON(out, "resolve.json", map[string]any{
|
||||
"live": false,
|
||||
"fail_reason": msg,
|
||||
})
|
||||
log.Fatalf("LIVE LLM REQUIRED: %s", msg)
|
||||
}
|
||||
|
||||
func writeJSON(out, name string, v any) {
|
||||
raw, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("marshal %s: %v", name, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(out, name), raw, 0o644); err != nil {
|
||||
log.Fatalf("write %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeMarkdownReport(out string, report map[string]any) {
|
||||
var b strings.Builder
|
||||
b.WriteString("# Final verify 2026-08-17 (FULL LIVE A1)\n\n")
|
||||
b.WriteString(fmt.Sprintf("**Overall: %v**\n\n", report["overall"]))
|
||||
b.WriteString("Live OverloadedBot only (DB-resolved completer). No mock `:18767`.\n\n")
|
||||
|
||||
b.WriteString("## Checklist\n\n")
|
||||
b.WriteString("| # | Requirement | Result |\n|---|---|---|\n")
|
||||
llmOK := false
|
||||
if llm, ok := report["llm"].(map[string]any); ok {
|
||||
if st, ok := llm["models_http_status"].(int); ok && (st == 200 || st == 401) {
|
||||
llmOK = true
|
||||
} else if st, ok := llm["models_http_status"].(float64); ok && (st == 200 || st == 401) {
|
||||
llmOK = true
|
||||
}
|
||||
}
|
||||
startOK, _ := report["start_ok"].(bool)
|
||||
aiEnhance, _ := report["ai_enhance_seen"].(bool)
|
||||
overall := str(report["overall"])
|
||||
b.WriteString(fmt.Sprintf("| 1 | LLM `/v1/models` UP (not 502) | **%s** |\n", map[bool]string{true: "PASS", false: "FAIL"}[llmOK]))
|
||||
b.WriteString("| 2 | Sync A1 from `scripts/seed/wp_product_categories.sql` | **PASS** (see sync.json) |\n")
|
||||
b.WriteString(fmt.Sprintf("| 3 | ProcessJob start pending / processed_items=0 | **%s** |\n", map[bool]string{true: "PASS", false: "FAIL"}[startOK]))
|
||||
b.WriteString(fmt.Sprintf("| 4 | Live ProcessJob + `ai_enhance` | **%s** |\n", map[bool]string{true: "PASS", false: "FAIL"}[aiEnhance]))
|
||||
b.WriteString(fmt.Sprintf("| 5 | Per-product V1 payload assertions | **%s** |\n", overall))
|
||||
b.WriteString(fmt.Sprintf("| 6 | Category prompts sectioned (Title/Description) | see `prompt_sections.json` |\n\n"))
|
||||
|
||||
b.WriteString(fmt.Sprintf("- Job: `%v`\n", report["job_id"]))
|
||||
b.WriteString(fmt.Sprintf("- EANs: `%v`\n\n", report["eans"]))
|
||||
|
||||
if llm, ok := report["llm"].(map[string]any); ok {
|
||||
b.WriteString("## LLM endpoint\n\n")
|
||||
b.WriteString(fmt.Sprintf("- base: `%v`\n- model: `%v`\n- models HTTP: `%v` (%v)\n- source: `%v` / mode: `%v` / byok: `%v`\n",
|
||||
llm["completer_base"], firstNonEmpty(str(llm["completer_model_adjusted"]), str(llm["completer_model"])),
|
||||
llm["models_http_status"], llm["models_elapsed"], llm["role_source"], llm["mode_label"], llm["byok"]))
|
||||
if v, ok := llm["probe_ok"]; ok {
|
||||
b.WriteString(fmt.Sprintf("- chat probe_ok: `%v` elapsed `%v`\n", v, llm["probe_elapsed"]))
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
b.WriteString("## Sync seed path\n\nSee `sync.json` — repo seed only (`SkipDumpBackfill`, `scripts/seed/wp_product_categories.sql`).\n\n")
|
||||
b.WriteString("## Prompt sections spot-check\n\nSee `prompt_sections.json`.\n\n")
|
||||
b.WriteString("## Start sample\n\nSee `start_sample.json`.\n\n")
|
||||
b.WriteString("## Per-product results\n\n")
|
||||
if cards, ok := report["scorecards"].([]map[string]any); ok {
|
||||
for _, c := range cards {
|
||||
b.WriteString(fmt.Sprintf("### EAN `%v` — pass=%v\n\n", c["ean"], c["pass"]))
|
||||
b.WriteString(fmt.Sprintf("- **Title:** %q\n", trunc(str(c["title"]), 120)))
|
||||
b.WriteString(fmt.Sprintf("- **Category:** %v (id=%v)\n", c["category"], c["category_id"]))
|
||||
b.WriteString(fmt.Sprintf("- **Description:** len=%v html=%v\n", c["desc_len"], c["has_html"]))
|
||||
if sn := str(c["desc_snippet"]); sn != "" {
|
||||
b.WriteString(fmt.Sprintf("```html\n%s\n```\n", sn))
|
||||
}
|
||||
if a, ok := c["attrs_summary"].(map[string]any); ok {
|
||||
b.WriteString(fmt.Sprintf("- **Attrs:** count=%v keys=%v\n", a["count"], a["keys"]))
|
||||
}
|
||||
if checks, ok := c["checks"].(map[string]bool); ok {
|
||||
b.WriteString("- Checks:\n")
|
||||
for k, v := range checks {
|
||||
b.WriteString(fmt.Sprintf(" - %s: %v\n", k, map[bool]string{true: "PASS", false: "FAIL"}[v]))
|
||||
}
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
}
|
||||
b.WriteString("## HTML excerpts\n\n")
|
||||
if exs, ok := report["html_excerpts"].([]map[string]any); ok {
|
||||
for i, ex := range exs {
|
||||
b.WriteString(fmt.Sprintf("### Excerpt %d — %v\n\n```html\n%v\n```\n\n", i+1, ex["ean"], ex["html_excerpt"]))
|
||||
}
|
||||
}
|
||||
b.WriteString("## Verdict\n\n")
|
||||
if overall == "PASS" {
|
||||
b.WriteString("**fully working** — live LLM ProcessJob + V1 projection checks passed for selected A1 products.\n")
|
||||
} else {
|
||||
b.WriteString("**partial / broken** — see failing scorecards and checklist above.\n")
|
||||
}
|
||||
_ = os.WriteFile(filepath.Join(out, "REPORT.md"), []byte(b.String()), 0o644)
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil { panic(err) }
|
||||
ctx := context.Background()
|
||||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||||
MaxConns: 4, MinConns: 1, MaxConnLifetime: time.Hour, HealthCheckPeriod: 30*time.Second,
|
||||
})
|
||||
if err != nil { panic(err) }
|
||||
defer pool.Close()
|
||||
plat := platformsettings.NewService(pool, platformsettings.EnvConfig{
|
||||
AppEncryptionKey:cfg.AppEncryptionKey, CredentialsEncryptionKey:cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret:cfg.TokenSigningSecret, DatabaseURL:cfg.DatabaseURL,
|
||||
OpenAIAPIKey:cfg.OpenAIAPIKey, OpenAIBaseURL:cfg.OpenAIBaseURL, OpenAIModel:cfg.OpenAIModel,
|
||||
})
|
||||
ai := aiprovider.NewService(pool, aiprovider.EnvConfig{
|
||||
AppEncryptionKey:cfg.AppEncryptionKey, CredentialsEncryptionKey:cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret:cfg.TokenSigningSecret, DatabaseURL:cfg.DatabaseURL,
|
||||
OpenAIAPIKey:cfg.OpenAIAPIKey, OpenAIBaseURL:cfg.OpenAIBaseURL, OpenAIModel:cfg.OpenAIModel,
|
||||
})
|
||||
ai.Platform = plat
|
||||
companyID := uuid.MustParse("604f23a8-b66e-4b21-8b45-0d72b68f4790")
|
||||
c, mode, byok, err := ai.ResolveCompleterForRole(ctx, companyID, aiprovider.RoleProcessing)
|
||||
if err != nil { panic(err) }
|
||||
client := c.(*processing.OpenAIClient)
|
||||
fmt.Printf("base=%s model=%s mode=%s byok=%v mock=%v\n", client.BaseURL, client.Model, mode, byok, processing.IsMockOrLoopbackBaseURL(client.BaseURL))
|
||||
url := strings.TrimRight(client.BaseURL,"/")+"/models"
|
||||
for i:=1; i<=3; i++ {
|
||||
req,_ := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
req.Header.Set("Authorization", "Bearer "+client.APIKey)
|
||||
start := time.Now()
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
elapsed := time.Since(start).Round(time.Millisecond)
|
||||
if err != nil { fmt.Printf("try%d err=%v elapsed=%s\n", i, err, elapsed); time.Sleep(2*time.Second); continue }
|
||||
b,_ := io.ReadAll(io.LimitReader(resp.Body, 240)); resp.Body.Close()
|
||||
fmt.Printf("try%d status=%d elapsed=%s body=%q\n", i, resp.StatusCode, elapsed, string(b))
|
||||
if resp.StatusCode == 200 { os.Exit(0) }
|
||||
time.Sleep(3*time.Second)
|
||||
}
|
||||
os.Exit(2)
|
||||
}
|
||||
@@ -0,0 +1,409 @@
|
||||
// Temporary slim LIVE verify — 2 products, short timeouts. Do not commit.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func main() {
|
||||
out := `F:\laragon\www\_MY\descrybe-v2\.codehelper\_final_verify_20260817`
|
||||
_ = os.MkdirAll(out, 0o755)
|
||||
logPath := filepath.Join(out, "inproc_run_quick.log")
|
||||
logFile, err := os.Create(logPath)
|
||||
if err != nil {
|
||||
log.Fatalf("log: %v", err)
|
||||
}
|
||||
defer logFile.Close()
|
||||
log.SetOutput(logredact.Writer(io.MultiWriter(os.Stderr, logFile)))
|
||||
|
||||
repoRoot := `F:\laragon\www\_MY\descrybe-v2`
|
||||
wpPath := filepath.Join(repoRoot, "scripts", "seed", "wp_product_categories.sql")
|
||||
wpBytes, err := os.ReadFile(wpPath)
|
||||
if err != nil {
|
||||
failReport(out, "FAIL", "repo seed missing: "+err.Error(), nil)
|
||||
}
|
||||
_ = os.Setenv("SEED_A1_WP_CATEGORIES", wpPath)
|
||||
|
||||
eans := []string{
|
||||
"4548736132597", // Slušalke
|
||||
"195348253666", // Gaming monitorji
|
||||
}
|
||||
companyID := uuid.MustParse("604f23a8-b66e-4b21-8b45-0d72b68f4790")
|
||||
userID := uuid.MustParse("6bf00877-a693-4d77-b28e-8c8292adac98")
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
failReport(out, "FAIL", "config: "+err.Error(), nil)
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||||
MaxConns: int32(cfg.DBMaxConns), MinConns: int32(cfg.DBMinConns),
|
||||
MaxConnLifetime: cfg.DBMaxConnLifetime, MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
|
||||
MaxConnIdleTime: cfg.DBMaxConnIdleTime, HealthCheckPeriod: cfg.DBHealthCheckPeriod,
|
||||
StatementTimeout: cfg.DBStatementTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
failReport(out, "FAIL", "db: "+err.Error(), nil)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
// --- Sync A1 from repo seed only ---
|
||||
var legacy string
|
||||
_ = pool.QueryRow(ctx, `SELECT COALESCE(legacy_company_id,'') FROM companies WHERE id=$1`, companyID).Scan(&legacy)
|
||||
a1Cohort := billing.IsA1CohortCompany(legacy, "A1 Slovenija")
|
||||
syncRes, serr := processing.SyncCompanyA1(ctx, pool, companyID, "A1 Slovenija", a1Cohort, processing.SyncCompanyA1Opts{
|
||||
SkipDumpBackfill: true,
|
||||
FixCompanyCatalogOpts: processing.FixCompanyCatalogOpts{
|
||||
BackfillCategories: true,
|
||||
WPCategoriesSQL: wpBytes,
|
||||
AIPrompts: aiprompts.NewService(pool),
|
||||
},
|
||||
})
|
||||
syncInfo := map[string]any{"wp_path": wpPath, "wp_bytes": len(wpBytes), "error": nil, "result": syncRes}
|
||||
if serr != nil {
|
||||
syncInfo["error"] = serr.Error()
|
||||
writeJSON(out, "sync_quick.json", syncInfo)
|
||||
failReport(out, "FAIL", "SyncCompanyA1: "+serr.Error(), syncInfo)
|
||||
}
|
||||
writeJSON(out, "sync_quick.json", syncInfo)
|
||||
log.Printf("sync ok prompts_already_ok=%d wp_entries=%d", syncRes.CategoryPromptsAlreadyOK, syncRes.WPCategoriesEntries)
|
||||
|
||||
platEnv := platformsettings.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey, CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret, DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey, OpenAIBaseURL: cfg.OpenAIBaseURL, OpenAIModel: cfg.OpenAIModel,
|
||||
}
|
||||
plat := platformsettings.NewService(pool, platEnv)
|
||||
aiSvc := aiprovider.NewService(pool, aiprovider.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey, CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret, DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey, OpenAIBaseURL: cfg.OpenAIBaseURL, OpenAIModel: cfg.OpenAIModel,
|
||||
ProcessingRPM: cfg.ProcessingRPM, ProcessingMaxRetries: cfg.ProcessingMaxRetries,
|
||||
})
|
||||
aiSvc.Platform = plat
|
||||
|
||||
roleCfg, rerr := plat.ResolveAIConfig(ctx, platformsettings.AIRoleProcessing)
|
||||
if rerr != nil {
|
||||
failReport(out, "FAIL", "ResolveAIConfig: "+rerr.Error(), nil)
|
||||
}
|
||||
completer, modeLabel, byok, cerr := aiSvc.ResolveCompleterForRole(ctx, companyID, aiprovider.RoleProcessing)
|
||||
if cerr != nil {
|
||||
failReport(out, "FAIL", "ResolveCompleter: "+cerr.Error(), nil)
|
||||
}
|
||||
client, ok := completer.(*processing.OpenAIClient)
|
||||
if !ok || client == nil {
|
||||
failReport(out, "FAIL", fmt.Sprintf("completer type %T", completer), nil)
|
||||
}
|
||||
base := strings.TrimSpace(client.BaseURL)
|
||||
model := strings.TrimSpace(client.Model)
|
||||
resolve := map[string]any{
|
||||
"role_source": roleCfg.Source, "role_provider": roleCfg.Provider,
|
||||
"completer_base": base, "completer_model": model,
|
||||
"mode": modeLabel, "byok": byok, "env_ignored": cfg.OpenAIBaseURL,
|
||||
}
|
||||
if processing.IsMockOrLoopbackBaseURL(base) || strings.Contains(base, "18767") {
|
||||
resolve["fail_reason"] = "mock/loopback refused"
|
||||
writeJSON(out, "resolve_quick.json", resolve)
|
||||
failReport(out, "FAIL", "mock/loopback refused — live OverloadedBot only", resolve)
|
||||
}
|
||||
|
||||
// --- /models probe (hard stop on 502; 15s) ---
|
||||
modelsURL := strings.TrimRight(base, "/") + "/models"
|
||||
modelsCtx, modelsCancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer modelsCancel()
|
||||
req, _ := http.NewRequestWithContext(modelsCtx, http.MethodGet, modelsURL, nil)
|
||||
req.Header.Set("Authorization", "Bearer "+client.APIKey)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
t0 := time.Now()
|
||||
resp, merr := http.DefaultClient.Do(req)
|
||||
elapsed := time.Since(t0).Round(time.Millisecond)
|
||||
if merr != nil {
|
||||
resolve["models_error"] = merr.Error()
|
||||
writeJSON(out, "resolve_quick.json", resolve)
|
||||
failReport(out, "FAIL", "GET /models network: "+merr.Error(), resolve)
|
||||
}
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
_ = resp.Body.Close()
|
||||
resolve["models_http_status"] = resp.StatusCode
|
||||
resolve["models_elapsed"] = elapsed.String()
|
||||
resolve["models_snippet"] = trunc(string(body), 180)
|
||||
if resp.StatusCode == http.StatusBadGateway {
|
||||
writeJSON(out, "resolve_quick.json", resolve)
|
||||
failReport(out, "FAIL", "FAIL 502: GET /v1/models — OverloadedBot unavailable (no mock, no wait)", resolve)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusUnauthorized {
|
||||
writeJSON(out, "resolve_quick.json", resolve)
|
||||
failReport(out, "FAIL", fmt.Sprintf("GET /models HTTP %d", resp.StatusCode), resolve)
|
||||
}
|
||||
// Prefer advertised green/code-fast if DB model is bare code-fast (gateway list).
|
||||
if model == "code-fast" && strings.Contains(string(body), `"green/code-fast"`) {
|
||||
client.Model = "green/code-fast"
|
||||
model = client.Model
|
||||
resolve["completer_model_adjusted"] = model
|
||||
log.Printf("adjusted model code-fast -> green/code-fast from /models list")
|
||||
}
|
||||
// Cap per-call wait (~2.5m) and retries so we never sit on 4m×N.
|
||||
if client.HTTPClient != nil {
|
||||
client.HTTPClient.Timeout = 150 * time.Second
|
||||
}
|
||||
client.MaxRetries = 1
|
||||
resolve["client_http_timeout"] = "150s"
|
||||
resolve["client_max_retries"] = 1
|
||||
writeJSON(out, "resolve_quick.json", resolve)
|
||||
log.Printf("LIVE OK models=%d base=%s model=%s", resp.StatusCode, base, model)
|
||||
|
||||
pipeline := processing.NewPipeline(pool)
|
||||
pipeline.BatchSize = cfg.ProcessingBatchSize
|
||||
pipeline.AI = aiSvc
|
||||
pipeline.Prompts = aiprompts.NewService(pool)
|
||||
pipeline.Engine = &processing.Engine{
|
||||
Completer: nil,
|
||||
Vector: processing.NoopVectorCategorizer{},
|
||||
EPREL: eprel.NewClient(eprel.Options{Enabled: true, Timeout: 15 * time.Second}),
|
||||
ProviderMode: processing.AIProviderInternal,
|
||||
}
|
||||
|
||||
rawIDs := make([]uuid.UUID, 0, len(eans))
|
||||
for _, ean := range eans {
|
||||
var id uuid.UUID
|
||||
if err := pool.QueryRow(ctx, `SELECT id FROM raw_products WHERE company_id=$1 AND gtin=$2`, companyID, ean).Scan(&id); err != nil {
|
||||
failReport(out, "FAIL", fmt.Sprintf("raw %s: %v", ean, err), resolve)
|
||||
}
|
||||
rawIDs = append(rawIDs, id)
|
||||
}
|
||||
_, _ = pool.Exec(ctx, `
|
||||
UPDATE processed_products pp
|
||||
SET field_sources = COALESCE(field_sources, '{}'::jsonb) - 'enhance_input_hash', updated_at=now()
|
||||
FROM raw_products rp
|
||||
WHERE pp.company_id=$1 AND pp.raw_product_id=rp.id AND rp.gtin=ANY($2::text[])`, companyID, eans)
|
||||
_, _ = pool.Exec(ctx, `
|
||||
UPDATE processing_jobs SET status='failed', error='yielded to quick final verify', updated_at=now()
|
||||
WHERE company_id=$1 AND status IN ('pending','running','processing')`, companyID)
|
||||
|
||||
jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full")
|
||||
if err != nil || len(jobs) == 0 {
|
||||
failReport(out, "FAIL", fmt.Sprintf("StartJob: %v", err), resolve)
|
||||
}
|
||||
jobID := jobs[0].ID
|
||||
var dbStatus string
|
||||
var processedCount int
|
||||
_ = pool.QueryRow(ctx, `SELECT lower(status), COALESCE(processed_count,0) FROM processing_jobs WHERE id=$1`, jobID).Scan(&dbStatus, &processedCount)
|
||||
startOK := (dbStatus == "pending" || dbStatus == "queued") && processedCount == 0
|
||||
startSample := map[string]any{
|
||||
"process_id": jobID.String(), "status": dbStatus, "processed_items": processedCount,
|
||||
"total_items": len(rawIDs), "start_ok": startOK,
|
||||
}
|
||||
writeJSON(out, "start_sample.json", startSample)
|
||||
|
||||
_, _ = pool.Exec(ctx, `UPDATE processing_jobs SET status='running', started_at=COALESCE(started_at,now()), updated_at=now() WHERE id=$1`, jobID)
|
||||
|
||||
// Hard wall: 2 products × ~2.5–3 min ≈ 8 min max
|
||||
runCtx, runCancel := context.WithTimeout(ctx, 8*time.Minute)
|
||||
defer runCancel()
|
||||
procErr := pipeline.ProcessJob(runCtx, jobID)
|
||||
logBytes, _ := os.ReadFile(logPath)
|
||||
logText := string(logBytes)
|
||||
aiEnhance := strings.Contains(logText, "ai_enhance")
|
||||
|
||||
items, loadErr := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
|
||||
if loadErr != nil {
|
||||
items = nil
|
||||
}
|
||||
cards := []map[string]any{}
|
||||
allPass := startOK && aiEnhance && procErr == nil
|
||||
for _, item := range items {
|
||||
ean := str(item["ean"])
|
||||
if ean == "" {
|
||||
ean = str(item["product_id"])
|
||||
}
|
||||
title := first(str(item["name"]), str(item["title"]))
|
||||
desc := str(item["description"])
|
||||
catName := str(item["category_name"])
|
||||
if catName == "" {
|
||||
if m, ok := item["category"].(map[string]any); ok {
|
||||
catName = str(m["name"])
|
||||
}
|
||||
}
|
||||
_, hasMetaT := item["meta_title"]
|
||||
_, hasMetaD := item["meta_description"]
|
||||
_, hasID := item["id"]
|
||||
_, hasPP := item["processed_product_id"]
|
||||
_, hasRaw := item["raw_product_id"]
|
||||
hasHTML := strings.Contains(strings.ToLower(desc), "<h2") || strings.Contains(strings.ToLower(desc), "<p>")
|
||||
checks := map[string]bool{
|
||||
"name_present": strings.TrimSpace(title) != "",
|
||||
"category_name_present": strings.TrimSpace(catName) != "",
|
||||
"no_meta_title": !hasMetaT || item["meta_title"] == nil,
|
||||
"no_meta_description": !hasMetaD || item["meta_description"] == nil,
|
||||
"no_id": !hasID,
|
||||
"no_processed_product_id": !hasPP,
|
||||
"no_raw_product_id": !hasRaw,
|
||||
"formula_html_or_empty": hasHTML || strings.TrimSpace(desc) == "" || strings.Contains(strings.ToLower(logText), "empty_or_provider_error") || strings.Contains(strings.ToLower(logText), "refuse"),
|
||||
}
|
||||
pass := true
|
||||
for _, v := range checks {
|
||||
if !v {
|
||||
pass = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !pass {
|
||||
allPass = false
|
||||
}
|
||||
cards = append(cards, map[string]any{
|
||||
"ean": ean, "title": title, "category": catName, "desc_len": len(desc),
|
||||
"has_html": hasHTML, "checks": checks, "pass": pass,
|
||||
})
|
||||
}
|
||||
if procErr != nil {
|
||||
allPass = false
|
||||
}
|
||||
overall := "PASS"
|
||||
reason := ""
|
||||
if !allPass {
|
||||
overall = "FAIL"
|
||||
if procErr != nil {
|
||||
reason = "ProcessJob: " + processing.TruncateError(procErr)
|
||||
} else if !aiEnhance {
|
||||
reason = "no ai_enhance log lines"
|
||||
} else if !startOK {
|
||||
reason = "start semantics not pending/0"
|
||||
} else {
|
||||
reason = "scorecard assertion failed"
|
||||
}
|
||||
}
|
||||
|
||||
report := map[string]any{
|
||||
"generated_at": time.Now().UTC().Format(time.RFC3339),
|
||||
"overall": overall,
|
||||
"blocking_reason": reason,
|
||||
"sync_pass": true,
|
||||
"live_llm_pass": true,
|
||||
"process_run": true,
|
||||
"job_id": jobID.String(),
|
||||
"eans": eans,
|
||||
"start_ok": startOK,
|
||||
"ai_enhance_seen": aiEnhance,
|
||||
"process_error": nil,
|
||||
"scorecards": cards,
|
||||
"llm": resolve,
|
||||
"sync": syncInfo,
|
||||
"note": "slim verify: 2 products, 150s/call, 8m wall; repo seed only",
|
||||
}
|
||||
if procErr != nil {
|
||||
report["process_error"] = processing.TruncateError(procErr)
|
||||
}
|
||||
writeJSON(out, "report.json", report)
|
||||
writeMD(out, report, startSample, cards, logText)
|
||||
fmt.Printf("OVERALL %s job=%s ai_enhance=%v err=%v\n", overall, jobID, aiEnhance, procErr)
|
||||
if overall != "PASS" {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func failReport(out, overall, reason string, extra map[string]any) {
|
||||
report := map[string]any{
|
||||
"generated_at": time.Now().UTC().Format(time.RFC3339),
|
||||
"overall": overall, "blocking_reason": reason,
|
||||
"process_run": false, "extra": extra,
|
||||
}
|
||||
writeJSON(out, "report.json", report)
|
||||
var b strings.Builder
|
||||
b.WriteString("# Final verify 2026-08-17 (quick LIVE)\n\n")
|
||||
b.WriteString(fmt.Sprintf("**Overall: %s**\n\n", overall))
|
||||
b.WriteString(fmt.Sprintf("Blocking: %s\n\n", reason))
|
||||
b.WriteString("Live OverloadedBot only. No mock.\n")
|
||||
_ = os.WriteFile(filepath.Join(out, "REPORT.md"), []byte(b.String()), 0o644)
|
||||
log.Print(reason)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
func writeMD(out string, report map[string]any, start map[string]any, cards []map[string]any, logText string) {
|
||||
var b strings.Builder
|
||||
b.WriteString("# Final verify 2026-08-17 (quick LIVE)\n\n")
|
||||
b.WriteString(fmt.Sprintf("**Overall: %v**\n\n", report["overall"]))
|
||||
if r := str(report["blocking_reason"]); r != "" {
|
||||
b.WriteString(fmt.Sprintf("Blocking: %s\n\n", r))
|
||||
}
|
||||
b.WriteString("## Progress note\n\n")
|
||||
b.WriteString("Prior hung run killed (4m chat waits on 6 EANs). This pass: live OverloadedBot only, repo seed sync, 2 products, 150s/call, 8m wall.\n\n")
|
||||
b.WriteString("## 1. Sync A1 (repo seed only)\n\n")
|
||||
b.WriteString("- Source: `scripts/seed/wp_product_categories.sql` (no upload, SkipDumpBackfill)\n")
|
||||
b.WriteString("- See `sync_quick.json`\n\n")
|
||||
b.WriteString("## 2. Live LLM\n\n")
|
||||
if llm, ok := report["llm"].(map[string]any); ok {
|
||||
b.WriteString(fmt.Sprintf("- base: `%v`\n- model: `%v`\n- models HTTP: `%v`\n- source: `%v`\n\n",
|
||||
llm["completer_base"], llm["completer_model"], llm["models_http_status"], llm["role_source"]))
|
||||
}
|
||||
b.WriteString("## 3. Start semantics\n\n")
|
||||
b.WriteString(fmt.Sprintf("- process_id: `%v`\n- status: `%v`\n- processed_items: `%v`\n- start_ok: `%v`\n\n",
|
||||
start["process_id"], start["status"], start["processed_items"], start["start_ok"]))
|
||||
b.WriteString(fmt.Sprintf("## 4. ai_enhance\n\n- seen: `%v`\n\n", report["ai_enhance_seen"]))
|
||||
b.WriteString("## 5. Scorecards\n\n")
|
||||
for _, c := range cards {
|
||||
b.WriteString(fmt.Sprintf("- **%v** pass=%v cat=%v html=%v title=%q\n",
|
||||
c["ean"], c["pass"], c["category"], c["has_html"], trunc(str(c["title"]), 60)))
|
||||
}
|
||||
b.WriteString("\n## Log excerpt (ai_enhance)\n\n```\n")
|
||||
n := 0
|
||||
for _, line := range strings.Split(logText, "\n") {
|
||||
if strings.Contains(line, "ai_enhance") || strings.Contains(line, "LIVE") || strings.Contains(line, "OVERALL") {
|
||||
b.WriteString(trunc(line, 240) + "\n")
|
||||
n++
|
||||
if n >= 12 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
b.WriteString("```\n")
|
||||
_ = os.WriteFile(filepath.Join(out, "REPORT.md"), []byte(b.String()), 0o644)
|
||||
}
|
||||
|
||||
func writeJSON(out, name string, v any) {
|
||||
raw, _ := json.MarshalIndent(v, "", " ")
|
||||
_ = os.WriteFile(filepath.Join(out, name), raw, 0o644)
|
||||
}
|
||||
func str(v any) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
if s, ok := v.(string); ok {
|
||||
return s
|
||||
}
|
||||
return strings.TrimSpace(fmt.Sprint(v))
|
||||
}
|
||||
func first(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
func trunc(s string, n int) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n] + "…"
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil { fmt.Fprintf(os.Stderr, "config: %v\n", err); os.Exit(2) }
|
||||
ctx := context.Background()
|
||||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||||
MaxConns: int32(cfg.DBMaxConns), MinConns: int32(cfg.DBMinConns),
|
||||
MaxConnLifetime: cfg.DBMaxConnLifetime, MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
|
||||
MaxConnIdleTime: cfg.DBMaxConnIdleTime, HealthCheckPeriod: cfg.DBHealthCheckPeriod,
|
||||
StatementTimeout: cfg.DBStatementTimeout,
|
||||
})
|
||||
if err != nil { fmt.Fprintf(os.Stderr, "db: %v\n", err); os.Exit(2) }
|
||||
defer pool.Close()
|
||||
platEnv := platformsettings.EnvConfig{AppEncryptionKey: cfg.AppEncryptionKey, CredentialsEncryptionKey: cfg.CredentialsEncryptionKey, TokenSigningSecret: cfg.TokenSigningSecret, DatabaseURL: cfg.DatabaseURL, OpenAIAPIKey: cfg.OpenAIAPIKey, OpenAIBaseURL: cfg.OpenAIBaseURL, OpenAIModel: cfg.OpenAIModel}
|
||||
plat := platformsettings.NewService(pool, platEnv)
|
||||
ai := aiprovider.NewService(pool, aiprovider.EnvConfig{AppEncryptionKey: cfg.AppEncryptionKey, CredentialsEncryptionKey: cfg.CredentialsEncryptionKey, TokenSigningSecret: cfg.TokenSigningSecret, DatabaseURL: cfg.DatabaseURL, OpenAIAPIKey: cfg.OpenAIAPIKey, OpenAIBaseURL: cfg.OpenAIBaseURL, OpenAIModel: cfg.OpenAIModel})
|
||||
ai.Platform = plat
|
||||
companyID := uuid.MustParse("604f23a8-b66e-4b21-8b45-0d72b68f4790")
|
||||
roleCfg, err := plat.ResolveAIConfig(ctx, platformsettings.AIRoleProcessing)
|
||||
if err != nil { fmt.Fprintf(os.Stderr, "ResolveAIConfig: %v\n", err); os.Exit(2) }
|
||||
completer, mode, byok, err := ai.ResolveCompleterForRole(ctx, companyID, aiprovider.RoleProcessing)
|
||||
if err != nil { fmt.Fprintf(os.Stderr, "ResolveCompleter: %v\n", err); os.Exit(2) }
|
||||
client, ok := completer.(*processing.OpenAIClient)
|
||||
if !ok { fmt.Fprintf(os.Stderr, "not OpenAIClient: %T\n", completer); os.Exit(2) }
|
||||
base := strings.TrimRight(strings.TrimSpace(client.BaseURL), "/")
|
||||
modelsURL := base + "/models"
|
||||
fmt.Printf("source=%s provider=%s base=%s model=%s mode=%s byok=%v key_len=%d\n", roleCfg.Source, roleCfg.Provider, base, client.Model, mode, byok, len(client.APIKey))
|
||||
if processing.IsMockOrLoopbackBaseURL(base) { fmt.Println("FAIL mock/loopback"); os.Exit(3) }
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, modelsURL, nil)
|
||||
req.Header.Set("Authorization", "Bearer "+client.APIKey)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
start := time.Now()
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
elapsed := time.Since(start).Round(time.Millisecond)
|
||||
if err != nil { fmt.Printf("models_error=%v elapsed=%s\n", err, elapsed); os.Exit(4) }
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
resp.Body.Close()
|
||||
snip := strings.TrimSpace(string(body))
|
||||
if len(snip) > 160 { snip = snip[:160]+"…" }
|
||||
fmt.Printf("models_http=%d elapsed=%s snippet=%q\n", resp.StatusCode, elapsed, snip)
|
||||
if resp.StatusCode == 502 { os.Exit(5) }
|
||||
if resp.StatusCode != 200 && resp.StatusCode != 401 { os.Exit(6) }
|
||||
os.Exit(0)
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
// Command repair-category-prompts rewrites A1 / Platform Demo categories.prompt
|
||||
// values into aiprompts role-sectioned overlays (Title / Description / Meta /
|
||||
// Attributes). Prefers wp_product_categories.sql (SEED_A1_WP_CATEGORIES /
|
||||
// Downloads) as source of truth, else a1-category-prompts.json. Legacy combined
|
||||
// scripts/seed) as source of truth, else a1-category-prompts.json. Legacy combined
|
||||
// Slovenian name+description blobs are split so naming rules land under Title
|
||||
// and HTML under Description; otherwise CategoryEnhanceUserTemplate is used.
|
||||
// Also repairs empty or brand-only title_template and empty description_template
|
||||
@@ -11,7 +11,7 @@
|
||||
//
|
||||
// go run ./cmd/repair-category-prompts -dry-run
|
||||
// go run ./cmd/repair-category-prompts -apply
|
||||
// go run ./cmd/repair-category-prompts -apply -wp-categories "D:/Users/.../Downloads/wp_product_categories.sql"
|
||||
// go run ./cmd/repair-category-prompts -apply -wp-categories ../../scripts/seed/wp_product_categories.sql
|
||||
// go run ./cmd/repair-category-prompts -apply -prompts ../../scripts/seed/a1-category-prompts.json
|
||||
//
|
||||
// DATABASE_URL / -postgres required. Default is dry-run (count only).
|
||||
|
||||
@@ -48,11 +48,11 @@ type RepairCategoryEnhancePromptsResult struct {
|
||||
// RepairA1DemoOptions configures optional seed overlay path for legacy splits.
|
||||
type RepairA1DemoOptions struct {
|
||||
// SeedPromptsPath points at a1-category-prompts.json OR wp_product_categories.sql.
|
||||
// Empty → auto-detect WP SQL (SEED_A1_WP_CATEGORIES / Downloads), else JSON seed.
|
||||
// Empty → auto-detect WP SQL (SEED_A1_WP_CATEGORIES / scripts/seed), else JSON seed.
|
||||
SeedPromptsPath string
|
||||
// WPCategoriesPath forces wp_product_categories.sql (overrides SeedPromptsPath when set).
|
||||
WPCategoriesPath string
|
||||
// WPCategoriesSQL is uploaded dump bytes (primary for Admin Sync A1 in prod).
|
||||
// WPCategoriesSQL is optional dump bytes (legacy Admin Sync A1 upload / API clients).
|
||||
// When non-empty, takes precedence over filesystem auto-detect / path options.
|
||||
WPCategoriesSQL []byte
|
||||
// ForceFromSeed overwrites already-sectioned prompts when a seed match exists
|
||||
@@ -63,7 +63,7 @@ type RepairA1DemoOptions struct {
|
||||
// RepairA1DemoCategoryEnhancePrompts replaces non-sectioned / legacy combined
|
||||
// categories.prompt values for A1 Slovenija + Platform Demo with role-sectioned
|
||||
// overlays (Title / Description / Meta / Attributes). Prefers wp_product_categories.sql
|
||||
// (SEED_A1_WP_CATEGORIES / Downloads) as source of truth, then a1-category-prompts.json,
|
||||
// (SEED_A1_WP_CATEGORIES / scripts/seed) as source of truth, then a1-category-prompts.json,
|
||||
// splitting combined Name+Description prompts so naming rules land under Title and
|
||||
// HTML body under Description; otherwise fall back to CategoryEnhanceUserTemplate.
|
||||
//
|
||||
|
||||
@@ -18,8 +18,9 @@ const (
|
||||
)
|
||||
|
||||
// ResolveWPCategoryPromptsPath picks an explicit path, else the first readable
|
||||
// wp_product_categories.sql candidate (SEED_A1_WP_CATEGORIES + Downloads + scripts/seed).
|
||||
// Used by Sync A1 / RepairCompanyCategoryEnhancePrompts as the A1 prompt source of truth.
|
||||
// wp_product_categories.sql candidate (SEED_A1_WP_CATEGORIES, then scripts/seed,
|
||||
// then optional local Downloads as last-resort). Used by Sync A1 /
|
||||
// RepairCompanyCategoryEnhancePrompts as the A1 prompt source of truth — no UI upload required.
|
||||
func ResolveWPCategoryPromptsPath(explicit string) string {
|
||||
if p := strings.TrimSpace(explicit); p != "" {
|
||||
if st, err := os.Stat(p); err == nil && !st.IsDir() {
|
||||
@@ -31,7 +32,7 @@ func ResolveWPCategoryPromptsPath(explicit string) string {
|
||||
if st, err := os.Stat(v); err == nil && !st.IsDir() {
|
||||
return v
|
||||
}
|
||||
log.Printf("warning: %s=%q not readable — trying Downloads / scripts/seed", envSeedA1WPCategories, v)
|
||||
log.Printf("warning: %s=%q not readable — trying scripts/seed", envSeedA1WPCategories, v)
|
||||
}
|
||||
for _, c := range WPCategoryPromptsCandidates() {
|
||||
if st, err := os.Stat(c); err == nil && !st.IsDir() {
|
||||
@@ -42,7 +43,8 @@ func ResolveWPCategoryPromptsPath(explicit string) string {
|
||||
}
|
||||
|
||||
// WPCategoryPromptsCandidates lists local paths Sync A1 / repair try when
|
||||
// SEED_A1_WP_CATEGORIES is unset. First readable file wins via ResolveWPCategoryPromptsPath.
|
||||
// SEED_A1_WP_CATEGORIES is unset. Prefer committed scripts/seed first; Downloads
|
||||
// is last-resort only. First readable file wins via ResolveWPCategoryPromptsPath.
|
||||
func WPCategoryPromptsCandidates() []string {
|
||||
var out []string
|
||||
if v := strings.TrimSpace(os.Getenv(envSeedA1WPCategories)); v != "" {
|
||||
@@ -53,13 +55,27 @@ func WPCategoryPromptsCandidates() []string {
|
||||
"wp_product_categories (1).sql",
|
||||
"wp_product_categories(1).sql",
|
||||
}
|
||||
// Repo seed is the default source of truth (no upload / Downloads required).
|
||||
if root, ok := findMonorepoRootFromCwd(); ok {
|
||||
for _, n := range names {
|
||||
out = append(out, filepath.Join(root, "scripts", "seed", n))
|
||||
}
|
||||
}
|
||||
for _, n := range names {
|
||||
out = append(out,
|
||||
filepath.Join("scripts", "seed", n),
|
||||
filepath.Join("..", "..", "scripts", "seed", n),
|
||||
n,
|
||||
filepath.Join("..", "..", n),
|
||||
)
|
||||
}
|
||||
// Optional local Downloads fallback (legacy / one-off machines without seed).
|
||||
home, _ := os.UserHomeDir()
|
||||
if home != "" {
|
||||
for _, n := range names {
|
||||
out = append(out, filepath.Join(home, "Downloads", n))
|
||||
out = append(out, filepath.Join(home, "downloads", n))
|
||||
}
|
||||
// Windows secondary profile Downloads (e.g. D:\Users\…\Downloads).
|
||||
for _, driveRoot := range []string{`D:\`, `C:\`} {
|
||||
alt := filepath.Join(driveRoot, "Users", filepath.Base(home), "Downloads")
|
||||
for _, n := range names {
|
||||
@@ -67,19 +83,6 @@ func WPCategoryPromptsCandidates() []string {
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, n := range names {
|
||||
out = append(out,
|
||||
n,
|
||||
filepath.Join("..", "..", n),
|
||||
filepath.Join("scripts", "seed", n),
|
||||
filepath.Join("..", "..", "scripts", "seed", n),
|
||||
)
|
||||
}
|
||||
if root, ok := findMonorepoRootFromCwd(); ok {
|
||||
for _, n := range names {
|
||||
out = append(out, filepath.Join(root, "scripts", "seed", n))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
@@ -81,14 +81,10 @@ func TestParseWPProductCategoriesSQL_SlusalkeSplit(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseWPProductCategoriesSQL_RealDownloadsFile(t *testing.T) {
|
||||
path := ResolveWPCategoryPromptsPath(`d:\Users\Green Eclipse\Downloads\wp_product_categories.sql`)
|
||||
func TestParseWPProductCategoriesSQL_RealSeedFile(t *testing.T) {
|
||||
path := ResolveWPCategoryPromptsPath("")
|
||||
if path == "" {
|
||||
// Also try env / auto-detect without failing CI machines that lack the dump.
|
||||
path = ResolveWPCategoryPromptsPath("")
|
||||
}
|
||||
if path == "" {
|
||||
t.Skip("wp_product_categories.sql not available on this machine")
|
||||
t.Skip("wp_product_categories.sql not available (expected under scripts/seed)")
|
||||
}
|
||||
byNorm, _, err := loadWPCategoryPromptOverlays(path)
|
||||
if err != nil {
|
||||
@@ -109,6 +105,10 @@ func TestParseWPProductCategoriesSQL_RealDownloadsFile(t *testing.T) {
|
||||
if !strings.Contains(split, "tip izdelka lowercase") && !strings.Contains(split, "znamka") {
|
||||
t.Fatalf("unexpected Title rules in split: %s", split[:min(400, len(split))])
|
||||
}
|
||||
if !strings.Contains(filepath.ToSlash(path), "scripts/seed/wp_product_categories.sql") &&
|
||||
filepath.Base(path) != "wp_product_categories.sql" {
|
||||
t.Logf("resolved path %s (prefer scripts/seed when present)", path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveWPCategoryPromptsPath_Explicit(t *testing.T) {
|
||||
|
||||
@@ -32,9 +32,9 @@ func IsWeakPriorEnhanceDescription(priorDesc string, titles ...string) bool {
|
||||
}
|
||||
|
||||
// heuristicSynthesizePhrases are distinctive invent / formula-skeleton snippets from
|
||||
// synthesizeDescriptionFromTitle / synthesizeDescriptionFromFormula. These may look
|
||||
// "strong" enough to pass IsWeakPriorEnhanceDescription but must never hash-skip
|
||||
// enhance (would leave fallback copy forever on reprocess).
|
||||
// synthesizeDescriptionFromTitle / synthesizeDescriptionFromFormula / factualDescriptionIntro.
|
||||
// These may look "strong" enough to pass IsWeakPriorEnhanceDescription but must never
|
||||
// hash-skip enhance (would leave fallback copy forever on reprocess).
|
||||
var heuristicSynthesizePhrases = []string{
|
||||
"is a catalog product with the known attributes",
|
||||
"is listed in the ",
|
||||
@@ -43,23 +43,63 @@ var heuristicSynthesizePhrases = []string{
|
||||
"je izdelek v kategoriji",
|
||||
"je izdelek znamke",
|
||||
". ključne specifikacije:",
|
||||
// factualDescriptionIntro (post-phrase-avoidance invent)
|
||||
" — znamka ",
|
||||
", znamka ",
|
||||
" — katalogski izdelek",
|
||||
" — from ",
|
||||
" — catalog product",
|
||||
}
|
||||
|
||||
// LooksLikeHeuristicSynthesize reports invent / formula-skeleton fallback copy.
|
||||
func LooksLikeHeuristicSynthesize(desc string) bool {
|
||||
lower := strings.ToLower(desc)
|
||||
plain := strings.ToLower(plainTextForWeakCheck(desc))
|
||||
if plain == "" {
|
||||
return false
|
||||
}
|
||||
for _, p := range heuristicSynthesizePhrases {
|
||||
if p != "" && strings.Contains(lower, p) {
|
||||
if p != "" && strings.Contains(plain, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// EN invent: "<title> is a <category> product from <brand>"
|
||||
if strings.Contains(lower, " is a ") && strings.Contains(lower, " product from ") {
|
||||
if strings.Contains(plain, " is a ") && strings.Contains(plain, " product from ") {
|
||||
return true
|
||||
}
|
||||
// SL/CS short invent: "<title> je … znamk|kategor|produkt …"
|
||||
if strings.Contains(plain, " je ") &&
|
||||
(strings.Contains(plain, "znamk") ||
|
||||
strings.Contains(plain, "kategor") ||
|
||||
strings.Contains(plain, "produkt") ||
|
||||
strings.Contains(plain, "televiz") ||
|
||||
strings.Contains(plain, "monitor")) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// plainTextForWeakCheck strips HTML tags so <p>thin invent</p> is judged on visible copy.
|
||||
func plainTextForWeakCheck(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
b.Grow(len(s))
|
||||
inTag := false
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case r == '<':
|
||||
inTag = true
|
||||
case r == '>':
|
||||
inTag = false
|
||||
case !inTag:
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(b.String())
|
||||
}
|
||||
|
||||
// EnhanceHashSkipBlockReason returns a stable reason when prior description must
|
||||
// not hash-skip the enhance LLM: "weak", "title-echo", "synth", or "".
|
||||
func EnhanceHashSkipBlockReason(priorDesc string, titles ...string) string {
|
||||
@@ -67,22 +107,26 @@ func EnhanceHashSkipBlockReason(priorDesc string, titles ...string) string {
|
||||
if priorDesc == "" || priorDesc == "<nil>" {
|
||||
return "weak"
|
||||
}
|
||||
if len([]rune(priorDesc)) < minUsableProductDescRunes {
|
||||
plain := plainTextForWeakCheck(priorDesc)
|
||||
if plain == "" || plain == "<nil>" {
|
||||
return "weak"
|
||||
}
|
||||
if len([]rune(plain)) < minUsableProductDescRunes {
|
||||
return "weak"
|
||||
}
|
||||
for _, title := range titles {
|
||||
if DescriptionEchoesTitle(priorDesc, title) {
|
||||
if DescriptionEchoesTitle(plain, title) || DescriptionEchoesTitle(priorDesc, title) {
|
||||
return "title-echo"
|
||||
}
|
||||
}
|
||||
if ContainsWeakFillerPhrase(priorDesc) {
|
||||
if ContainsWeakFillerPhrase(plain) || ContainsWeakFillerPhrase(priorDesc) {
|
||||
return "weak"
|
||||
}
|
||||
if LooksLikeHeuristicSynthesize(priorDesc) {
|
||||
return "synth"
|
||||
}
|
||||
if len([]rune(priorDesc)) <= shortBoilerplateDescRunes &&
|
||||
!descriptionOverlapsProductFacts(priorDesc, titles...) {
|
||||
if len([]rune(plain)) <= shortBoilerplateDescRunes &&
|
||||
!descriptionOverlapsProductFacts(plain, titles...) {
|
||||
return "weak"
|
||||
}
|
||||
return ""
|
||||
|
||||
@@ -13,6 +13,17 @@ func TestLooksLikeHeuristicSynthesize(t *testing.T) {
|
||||
if !LooksLikeHeuristicSynthesize(sl) {
|
||||
t.Fatalf("expected SL invent detected: %q", sl)
|
||||
}
|
||||
htmlInvent := "<p>QLED TV 55 je televizor znamke Samsung.</p>"
|
||||
if !LooksLikeHeuristicSynthesize(htmlInvent) {
|
||||
t.Fatalf("expected HTML-wrapped invent detected: %q", htmlInvent)
|
||||
}
|
||||
if !ShouldRefuseEnhanceHashSkip(htmlInvent, "QLED TV 55") {
|
||||
t.Fatalf("HTML invent must block hash skip")
|
||||
}
|
||||
factual := "QLED TV 55 — Televizorji, znamka Samsung."
|
||||
if !LooksLikeHeuristicSynthesize(factual) {
|
||||
t.Fatalf("expected factualDescriptionIntro invent detected: %q", factual)
|
||||
}
|
||||
good := "Vogel's WALL 3245 is a sturdy TV mount with clear load ratings and install guidance for wall displays."
|
||||
if LooksLikeHeuristicSynthesize(good) {
|
||||
t.Fatalf("retail copy must not look like invent: %q", good)
|
||||
|
||||
@@ -27,6 +27,7 @@ func TestMemberForbiddenOnSensitiveMutations(t *testing.T) {
|
||||
body string
|
||||
}{
|
||||
{name: "create_api_key", fn: s.handleCreateAPIKey, body: `{"name":"x"}`},
|
||||
{name: "update_api_key", fn: s.handleUpdateAPIKey, body: `{"name":"renamed"}`},
|
||||
{name: "revoke_api_key", fn: s.handleRevokeAPIKey, body: ""},
|
||||
{name: "put_email", fn: s.handlePutEmailIntegration, body: `{}`},
|
||||
{name: "verify_email", fn: s.handleVerifyEmailIntegration, body: ""},
|
||||
|
||||
@@ -29,11 +29,13 @@ const syncA1MaxBodyBytes = catalog.MaxWPCategorySQLBytes + (2 << 20)
|
||||
//
|
||||
// Body (JSON): confirm=true required; backfill_categories (default true);
|
||||
// reprocess_sample_limit (default 25, max 200); mysql_dump optional path;
|
||||
// skip_dump_backfill (default false); wp_product_categories_sql_b64 optional
|
||||
// base64 of wp_product_categories.sql (primary for category prompts).
|
||||
// skip_dump_backfill (default false). Category prompts load from repo seed
|
||||
// (scripts/seed/wp_product_categories.sql) via ResolveWPCategoryPromptsPath;
|
||||
// wp_product_categories_sql_b64 remains accepted but unused by the admin UI.
|
||||
//
|
||||
// Body (multipart/form-data): same fields as form values; file field
|
||||
// wp_product_categories or wp_categories_sql for the SQL dump upload.
|
||||
// Body (multipart/form-data): same fields as form values; optional file field
|
||||
// wp_product_categories / wp_categories_sql is still accepted for API clients
|
||||
// but the admin Sync A1 UI no longer uploads.
|
||||
//
|
||||
// Flash (UI): result → flash.admin.syncA1Success.
|
||||
func (s *Server) handleAdminSyncCompanyA1(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -116,8 +118,10 @@ func (s *Server) handleAdminSyncCompanyA1(w http.ResponseWriter, r *http.Request
|
||||
}
|
||||
if len(parsed.WPCategoriesSQL) > 0 {
|
||||
note = "Applied uploaded wp_product_categories.sql (Title/Description/Meta/Attributes sections) + catalog hygiene. Reprocess recommended products manually (no mass reprocess)."
|
||||
} else if strings.TrimSpace(result.WPCategoriesPath) == "" {
|
||||
note += " Upload wp_product_categories.sql on Sync A1 for force-applied category prompts when the dump is not on the API host."
|
||||
} else if strings.TrimSpace(result.WPCategoriesPath) != "" {
|
||||
note = "Applied category prompts from repo seed (" + result.WPCategoriesPath + ") + catalog hygiene. Reprocess recommended products manually (no mass reprocess)."
|
||||
} else {
|
||||
note += " Place scripts/seed/wp_product_categories.sql on the API host (or set SEED_A1_WP_CATEGORIES) for force-applied category prompts."
|
||||
}
|
||||
|
||||
JSON(w, http.StatusOK, map[string]any{
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/auth"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
func (s *Server) handleListAPIKeys(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -80,6 +83,61 @@ func (s *Server) handleCreateAPIKey(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) handleUpdateAPIKey(w http.ResponseWriter, r *http.Request) {
|
||||
cid, _ := CompanyIDFromContext(r.Context())
|
||||
if !s.allowCompanyAdminOrPlatform(w, r) {
|
||||
return
|
||||
}
|
||||
if !s.requireFeatures(w, r, "settings.api_keys", "capability.api_access") {
|
||||
return
|
||||
}
|
||||
id, err := uuid.Parse(chi.URLParam(r, "id"))
|
||||
if err != nil {
|
||||
Error(w, http.StatusBadRequest, "invalid id")
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
if err := DecodeJSON(r, &body); err != nil {
|
||||
Error(w, http.StatusBadRequest, "invalid json")
|
||||
return
|
||||
}
|
||||
name := strings.TrimSpace(body.Name)
|
||||
if name == "" {
|
||||
Error(w, http.StatusBadRequest, "name required")
|
||||
return
|
||||
}
|
||||
if len(name) > 120 {
|
||||
Error(w, http.StatusBadRequest, "name too long")
|
||||
return
|
||||
}
|
||||
var (
|
||||
outID uuid.UUID
|
||||
outName string
|
||||
prefix string
|
||||
lastUsed any
|
||||
created any
|
||||
)
|
||||
err = s.Pool.QueryRow(r.Context(), `
|
||||
UPDATE api_keys
|
||||
SET name = $3, updated_at = now()
|
||||
WHERE id = $1 AND company_id = $2 AND revoked_at IS NULL
|
||||
RETURNING id, name, key_prefix, last_used_at, created_at`,
|
||||
id, cid, name).Scan(&outID, &outName, &prefix, &lastUsed, &created)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
Error(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
Error(w, http.StatusInternalServerError, "update failed")
|
||||
return
|
||||
}
|
||||
JSON(w, http.StatusOK, map[string]any{
|
||||
"id": outID, "name": outName, "key_prefix": prefix, "last_used_at": lastUsed, "created_at": created,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) handleRevokeAPIKey(w http.ResponseWriter, r *http.Request) {
|
||||
cid, _ := CompanyIDFromContext(r.Context())
|
||||
if !s.allowCompanyAdminOrPlatform(w, r) {
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestHandleUpdateAPIKeyRejectsEmptyName(t *testing.T) {
|
||||
t.Parallel()
|
||||
s := &Server{}
|
||||
cid := uuid.MustParse("11111111-1111-1111-1111-111111111111")
|
||||
uid := uuid.MustParse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
|
||||
keyID := uuid.MustParse("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb")
|
||||
|
||||
ctx := context.WithValue(context.Background(), ctxUserID, uid)
|
||||
ctx = context.WithValue(ctx, ctxCompanyID, cid)
|
||||
ctx = context.WithValue(ctx, ctxRole, "admin")
|
||||
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", keyID.String())
|
||||
ctx = context.WithValue(ctx, chi.RouteCtxKey, rctx)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPatch, "/api/api-keys/"+keyID.String(), bytes.NewBufferString(`{"name":" "}`)).WithContext(ctx)
|
||||
rec := httptest.NewRecorder()
|
||||
s.handleUpdateAPIKey(rec, req)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d body=%s, want 400", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleUpdateAPIKeyRejectsInvalidID(t *testing.T) {
|
||||
t.Parallel()
|
||||
s := &Server{}
|
||||
cid := uuid.MustParse("11111111-1111-1111-1111-111111111111")
|
||||
uid := uuid.MustParse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
|
||||
|
||||
ctx := context.WithValue(context.Background(), ctxUserID, uid)
|
||||
ctx = context.WithValue(ctx, ctxCompanyID, cid)
|
||||
ctx = context.WithValue(ctx, ctxRole, "admin")
|
||||
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "not-a-uuid")
|
||||
ctx = context.WithValue(ctx, chi.RouteCtxKey, rctx)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPatch, "/api/api-keys/not-a-uuid", bytes.NewBufferString(`{"name":"ok"}`)).WithContext(ctx)
|
||||
rec := httptest.NewRecorder()
|
||||
s.handleUpdateAPIKey(rec, req)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d body=%s, want 400", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -247,6 +247,15 @@ func TestAuthSessionCoreEndpoints(t *testing.T) {
|
||||
t.Fatalf("create api-key payload=%v", created)
|
||||
}
|
||||
|
||||
rec = do(http.MethodPatch, "/api/api-keys/"+keyID, `{"name":"auth-smoke-key-renamed"}`, true)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("rename api-key status=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
renamed := decode(t, rec)
|
||||
if fmt.Sprint(renamed["name"]) != "auth-smoke-key-renamed" {
|
||||
t.Fatalf("rename api-key name=%v", renamed["name"])
|
||||
}
|
||||
|
||||
// Public v1 with the new key (CSRF skipped).
|
||||
v1 := httptest.NewRecorder()
|
||||
v1Req := httptest.NewRequest(http.MethodGet, "/api/v1/products?limit=1", nil)
|
||||
|
||||
@@ -453,6 +453,7 @@ func (s *Server) Router() http.Handler {
|
||||
|
||||
r.Get("/api-keys", s.handleListAPIKeys)
|
||||
r.Post("/api-keys", s.handleCreateAPIKey)
|
||||
r.Patch("/api-keys/{id}", s.handleUpdateAPIKey)
|
||||
r.Delete("/api-keys/{id}", s.handleRevokeAPIKey)
|
||||
|
||||
r.Get("/billing/credits", s.handleCreditsOverview)
|
||||
|
||||
@@ -118,8 +118,9 @@ info:
|
||||
- GET /products data[].id = processed_products.id (enriched row)
|
||||
- GET /products data[].raw_product_id = raw_products.id (use this for raw_product_ids)
|
||||
- POST ... raw_product_ids[] must be raw_products.id — never PresentProduct.id
|
||||
- GET /products/process/{id} COMPLETED items[].id = processed_products.id (legacy);
|
||||
additive processed_product_id (same as id) and raw_product_id (raw_products.id)
|
||||
- GET /products/process/{id} COMPLETED items[] omit internal UUIDs (id /
|
||||
processed_product_id / raw_product_id). Use GET /products when a UUID is needed.
|
||||
Display name is items[].name (title omitted when identical).
|
||||
|
||||
Note: Dashboard JSON under /api/* uses session cookies + CSRF and is separate
|
||||
from this public API-key surface. Other legacy path aliases
|
||||
@@ -895,9 +896,10 @@ paths:
|
||||
Completed jobs return items[] (EAN-keyed enrichment). In-progress and failed
|
||||
jobs omit items. Not the same shape as GET /process/{id} (flat ProcessingJob).
|
||||
|
||||
On COMPLETED items, id is the processed_products UUID (legacy). Additive aliases:
|
||||
processed_product_id (same value as id), raw_product_id (raw_products.id), and
|
||||
name (same value as title) for dual-mode clients / scorecards.
|
||||
On COMPLETED items, name is the product display name (title is omitted when
|
||||
identical). Internal UUIDs (id / processed_product_id / raw_product_id) are
|
||||
omitted — use GET /products for those. A1 cohort / Platform Demo / A1-prompt
|
||||
companies omit meta_title and meta_description even if stored.
|
||||
parameters:
|
||||
- name: id
|
||||
in: path
|
||||
@@ -925,17 +927,11 @@ paths:
|
||||
processing_type: full
|
||||
items:
|
||||
- ean: '8606019604493'
|
||||
id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
|
||||
processed_product_id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
|
||||
raw_product_id: cccccccc-cccc-cccc-cccc-cccccccccccc
|
||||
status: processed
|
||||
category: Cookers
|
||||
category_id: '50'
|
||||
category_name: Cookers
|
||||
title: VOX electric cooker EHT 6020 WG
|
||||
name: VOX electric cooker EHT 6020 WG
|
||||
meta_title: VOX electric cooker EHT 6020 WG | 50
|
||||
meta_description: Affordable electric cooker with four plates and a 65 L fan oven.
|
||||
description: "Vox Electronics electric cooker EHT 6020 WG offers strong value with four electric hobs and a 65 L fan oven."
|
||||
attributes:
|
||||
brand: Vox
|
||||
@@ -6207,12 +6203,24 @@ components:
|
||||
process_id:
|
||||
type: string
|
||||
format: uuid
|
||||
status:
|
||||
type: string
|
||||
description: |
|
||||
Job lifecycle status on enqueue. Always pending (or processing if the
|
||||
worker already picked it up). Never COMPLETED on start — poll GET
|
||||
/products/process/{id} for completion.
|
||||
example: pending
|
||||
message:
|
||||
type: string
|
||||
total_items:
|
||||
type: integer
|
||||
description: Number of products accepted into the job (enqueue size).
|
||||
processed_items:
|
||||
type: integer
|
||||
description: |
|
||||
Count of items that finished processing. Always 0 on start while the
|
||||
job is pending/processing; increments as the worker completes products.
|
||||
example: 0
|
||||
job_count:
|
||||
type: integer
|
||||
description: Present when StartJob auto-splits
|
||||
@@ -6282,35 +6290,19 @@ components:
|
||||
expose category as the human-readable display name (category_id holds
|
||||
categories.unique_id), a description string that may include category formula
|
||||
HTML (h1/h2/h3/h4, p, ul — never a JSON array), optional SEO meta_title /
|
||||
meta_description (plain text; omitted for A1 cohort), optional eprel object or
|
||||
null, clean attributes, images, and dual-mode ids.
|
||||
Product display name is title; additive name mirrors the same processed title
|
||||
(dual-mode for scorecards / legacy clients that read name).
|
||||
Property order prefers human-readable fields first (ean, title, category,
|
||||
description, attributes, images, eprel) with internal ids last.
|
||||
meta_description (plain text; omitted for A1 cohort, Platform Demo, and any
|
||||
company with A1-style category role-section prompts — even if stored in DB),
|
||||
optional eprel object or null, clean attributes, and images.
|
||||
Product display name is name (primary). title is omitted when identical to name.
|
||||
Internal UUIDs (id, processed_product_id, raw_product_id) are omitted from this
|
||||
public shape — use catalog APIs when a product UUID is required.
|
||||
Property order prefers human-readable fields first (ean, name, category,
|
||||
description, attributes, images, eprel).
|
||||
required:
|
||||
- ean
|
||||
properties:
|
||||
ean:
|
||||
type: string
|
||||
id:
|
||||
type: string
|
||||
format: uuid
|
||||
description: |
|
||||
Legacy field: processed_products.id when enrichment succeeded.
|
||||
Do not treat as raw_products.id. Same value as processed_product_id.
|
||||
processed_product_id:
|
||||
type: string
|
||||
format: uuid
|
||||
description: |
|
||||
Explicit alias of id (processed_products.id). Prefer this name in new
|
||||
dual-mode clients; id remains for backward compatibility.
|
||||
raw_product_id:
|
||||
type: string
|
||||
format: uuid
|
||||
description: |
|
||||
raw_products.id for this job line. Use with POST /process raw_product_ids
|
||||
or dashboard catalog APIs. Present whenever the job product row exists.
|
||||
category:
|
||||
type: string
|
||||
nullable: true
|
||||
@@ -6330,32 +6322,32 @@ components:
|
||||
nullable: true
|
||||
description: |
|
||||
Human-readable category display name (mirrors category when both are set).
|
||||
title:
|
||||
type: string
|
||||
nullable: true
|
||||
description: |
|
||||
Product display name (processed title). Primary legacy field; same value
|
||||
as name when present.
|
||||
name:
|
||||
type: string
|
||||
nullable: true
|
||||
description: |
|
||||
Additive alias of title (same processed display name). Prefer title in
|
||||
new clients; name remains for scorecards and legacy readers.
|
||||
Product display name (processed title). Primary field for clients.
|
||||
title:
|
||||
type: string
|
||||
nullable: true
|
||||
description: |
|
||||
Optional legacy alias of name. Omitted when identical to name.
|
||||
meta_title:
|
||||
type: string
|
||||
nullable: true
|
||||
description: |
|
||||
SEO title. Filled from processing meta or synthesized from title / category
|
||||
when empty so successful items are not left with null meta.
|
||||
Omitted for A1 cohort (SEO meta is not used).
|
||||
Omitted for A1 cohort, Platform Demo, and companies with A1-style category
|
||||
prompts (SEO meta is not used).
|
||||
meta_description:
|
||||
type: string
|
||||
nullable: true
|
||||
description: |
|
||||
SEO description (word-safe truncate). Distinct from body description when
|
||||
possible; synthesized from plain description when DB meta is empty.
|
||||
Omitted for A1 cohort (SEO meta is not used).
|
||||
Omitted for A1 cohort, Platform Demo, and companies with A1-style category
|
||||
prompts (SEO meta is not used).
|
||||
description:
|
||||
type: string
|
||||
nullable: true
|
||||
@@ -6477,17 +6469,11 @@ components:
|
||||
processing_type: full
|
||||
items:
|
||||
- ean: '8606019604493'
|
||||
id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
|
||||
processed_product_id: bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb
|
||||
raw_product_id: cccccccc-cccc-cccc-cccc-cccccccccccc
|
||||
status: processed
|
||||
category: Cookers
|
||||
category_id: '50'
|
||||
category_name: Cookers
|
||||
title: VOX electric cooker EHT 6020 WG
|
||||
name: VOX electric cooker EHT 6020 WG
|
||||
meta_title: VOX electric cooker EHT 6020 WG | 50
|
||||
meta_description: Affordable electric cooker with four plates and a 65 L fan oven.
|
||||
description: "Vox Electronics electric cooker EHT 6020 WG offers strong value with four electric hobs and a 65 L fan oven. Energy class A with practical everyday capacity."
|
||||
attributes:
|
||||
brand: Vox
|
||||
|
||||
@@ -160,11 +160,14 @@ func (s *Server) handleV1StartProcess(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
primary := jobs[0]
|
||||
// processed_items is the completed count — stay 0 until the worker finishes.
|
||||
// total_items is the enqueue size; status stays pending/processing until poll COMPLETED.
|
||||
resp := map[string]any{
|
||||
"process_id": primary.ID.String(),
|
||||
"status": "pending",
|
||||
"message": fmt.Sprintf("Processing started for %d product(s)", len(rawIDs)),
|
||||
"total_items": totalItems,
|
||||
"processed_items": len(rawIDs),
|
||||
"processed_items": 0,
|
||||
}
|
||||
if len(jobs) > 1 {
|
||||
siblings := make([]string, 0, len(jobs)-1)
|
||||
|
||||
@@ -194,6 +194,7 @@ func TestHandleV1StartProcessItemsEANReturnsProcessID(t *testing.T) {
|
||||
var envelope struct {
|
||||
Data struct {
|
||||
ProcessID string `json:"process_id"`
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message"`
|
||||
TotalItems int `json:"total_items"`
|
||||
ProcessedItems int `json:"processed_items"`
|
||||
@@ -205,8 +206,11 @@ func TestHandleV1StartProcessItemsEANReturnsProcessID(t *testing.T) {
|
||||
if envelope.Data.ProcessID != jobID.String() {
|
||||
t.Fatalf("process_id=%q want %s", envelope.Data.ProcessID, jobID)
|
||||
}
|
||||
if envelope.Data.TotalItems != 1 || envelope.Data.ProcessedItems != 1 {
|
||||
t.Fatalf("counts=%+v", envelope.Data)
|
||||
if envelope.Data.Status != "pending" {
|
||||
t.Fatalf("status=%q want pending", envelope.Data.Status)
|
||||
}
|
||||
if envelope.Data.TotalItems != 1 || envelope.Data.ProcessedItems != 0 {
|
||||
t.Fatalf("counts=%+v (processed_items must be 0 on start)", envelope.Data)
|
||||
}
|
||||
if enqueued != jobID {
|
||||
t.Fatalf("enqueued=%s", enqueued)
|
||||
|
||||
@@ -528,11 +528,10 @@ func (e *Engine) RunSteps(ctx context.Context, companyID string, in ProductInput
|
||||
}
|
||||
}
|
||||
|
||||
// Pipelines without StepCategorize (enhance_only / normalize_only) still try
|
||||
// vector categorize when AllowAI + embeddings are available. Full/categorize
|
||||
// already ran runCategorizeStep (vector then LLM) inside the loop.
|
||||
// Pipelines without StepCategorize (enhance_only / normalize_only) still run
|
||||
// vector + LLM categorize when category is empty so enhance is not fed a blank.
|
||||
if !stepsContain(steps, StepCategorize) {
|
||||
tryVectorCategorize(ctx, e, companyID, &out, categoryNames, policy)
|
||||
runCategorizeStep(ctx, e, companyID, &out, in, categoryNames, policy)
|
||||
noteMissingCategory(&out, policy, e != nil && e.Vector != nil && e.Vector.Enabled())
|
||||
}
|
||||
preserveCategoryIfEmpty(&out, in.PriorCategory)
|
||||
|
||||
@@ -261,6 +261,8 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = processedID
|
||||
_ = rawProductID
|
||||
if processedID == nil {
|
||||
st := MapV1JobItemStatus(itemStatus, false)
|
||||
if st == "processed" || st == "processing" || st == "pending" {
|
||||
@@ -274,7 +276,6 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
|
||||
if itemError != nil && *itemError != "" {
|
||||
item["error"] = *itemError
|
||||
}
|
||||
applyV1ProcessItemIDs(item, nil, rawProductID)
|
||||
full = append(full, item)
|
||||
continue
|
||||
}
|
||||
@@ -428,9 +429,9 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
|
||||
catIDOut = catStr
|
||||
}
|
||||
|
||||
var titleOut any
|
||||
var nameOut any
|
||||
if titleStr != "" {
|
||||
titleOut = titleStr
|
||||
nameOut = titleStr
|
||||
}
|
||||
item := V1ProcessJobItem{
|
||||
"ean": ean,
|
||||
@@ -438,8 +439,7 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
|
||||
"category": catNameOut,
|
||||
"category_id": catIDOut,
|
||||
"category_name": catNameOut,
|
||||
"title": titleOut,
|
||||
"name": titleOut,
|
||||
"name": nameOut,
|
||||
"description": description,
|
||||
"attributes": nil,
|
||||
"main_image": nil,
|
||||
@@ -450,7 +450,8 @@ func (p *Pipeline) LoadV1ProcessJobItems(ctx context.Context, companyID, jobID u
|
||||
item["meta_title"] = metaTitleOut
|
||||
item["meta_description"] = metaDescOut
|
||||
}
|
||||
applyV1ProcessItemIDs(item, processedID, rawProductID)
|
||||
// Internal ids (id / processed_product_id / raw_product_id) are omitted from
|
||||
// the public V1 process item shape — use catalog APIs when a UUID is needed.
|
||||
if itemError != nil && *itemError != "" {
|
||||
item["error"] = *itemError
|
||||
}
|
||||
@@ -483,20 +484,41 @@ func nullIfEmptyPtr(s *string) any {
|
||||
return *s
|
||||
}
|
||||
|
||||
// companyOmitsSEOMeta is true for the A1 cohort (no meta_title / meta_description).
|
||||
// companyOmitsSEOMeta is true when V1/process must omit meta_title / meta_description:
|
||||
// A1 cohort (legacy id), Platform Demo (A1-cloned prompts, no legacy id), or any
|
||||
// company whose categories store A1-style role-section enhance prompts.
|
||||
func companyOmitsSEOMeta(ctx context.Context, p *Pipeline, companyID uuid.UUID) bool {
|
||||
if p == nil || p.Pool == nil {
|
||||
return false
|
||||
}
|
||||
var legacy string
|
||||
var legacy, name string
|
||||
err := p.Pool.QueryRow(ctx, `
|
||||
SELECT COALESCE(legacy_company_id, '')
|
||||
SELECT COALESCE(legacy_company_id, ''), COALESCE(name, '')
|
||||
FROM companies
|
||||
WHERE id = $1`, companyID).Scan(&legacy)
|
||||
WHERE id = $1`, companyID).Scan(&legacy, &name)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return billing.IsA1CohortCompany(legacy, "")
|
||||
if billing.IsA1CohortCompany(legacy, "") {
|
||||
return true
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(name), "Platform Demo") {
|
||||
return true
|
||||
}
|
||||
var hasA1Prompts bool
|
||||
err = p.Pool.QueryRow(ctx, `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM categories
|
||||
WHERE company_id = $1
|
||||
AND prompt ILIKE '%--- Title ---%'
|
||||
AND prompt ILIKE '%--- Description ---%'
|
||||
AND prompt ILIKE '%--- Meta ---%'
|
||||
LIMIT 1
|
||||
)`, companyID).Scan(&hasA1Prompts)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return hasA1Prompts
|
||||
}
|
||||
|
||||
func derefStringPtr(s *string) string {
|
||||
@@ -724,10 +746,14 @@ func ProjectV1ProcessJobItems(storedType string, items []V1ProcessJobItem) []V1P
|
||||
projected["category_id"] = item["category_id"]
|
||||
projected["category_name"] = item["category_name"]
|
||||
case "title":
|
||||
projected["title"] = item["title"]
|
||||
projected["name"] = item["name"]
|
||||
if projected["name"] == nil {
|
||||
projected["name"] = item["title"]
|
||||
name := item["name"]
|
||||
if name == nil {
|
||||
name = item["title"]
|
||||
}
|
||||
projected["name"] = name
|
||||
// Omit duplicate title when it matches name (name is primary).
|
||||
if title := item["title"]; title != nil && title != name {
|
||||
projected["title"] = title
|
||||
}
|
||||
if _, ok := item["meta_title"]; ok {
|
||||
projected["meta_title"] = item["meta_title"]
|
||||
@@ -747,7 +773,7 @@ func ProjectV1ProcessJobItems(storedType string, items []V1ProcessJobItem) []V1P
|
||||
}
|
||||
|
||||
func withItemMeta(dst, src V1ProcessJobItem) V1ProcessJobItem {
|
||||
for _, k := range []string{"status", "error", "id", "processed_product_id", "raw_product_id"} {
|
||||
for _, k := range []string{"status", "error"} {
|
||||
if v, ok := src[k]; ok {
|
||||
dst[k] = v
|
||||
}
|
||||
@@ -755,17 +781,13 @@ func withItemMeta(dst, src V1ProcessJobItem) V1ProcessJobItem {
|
||||
return dst
|
||||
}
|
||||
|
||||
// applyV1ProcessItemIDs sets legacy id (= processed UUID) plus additive dual-mode aliases.
|
||||
// id is preserved for existing integrators; processed_product_id mirrors it; raw_product_id is raw_products.id.
|
||||
// applyV1ProcessItemIDs formerly stamped id / processed_product_id / raw_product_id onto
|
||||
// V1 process items. Those fields are omitted from the public contract; kept as a no-op
|
||||
// so older call sites/tests compile until removed.
|
||||
func applyV1ProcessItemIDs(item V1ProcessJobItem, processedID, rawProductID *uuid.UUID) {
|
||||
if processedID != nil {
|
||||
s := processedID.String()
|
||||
item["id"] = s
|
||||
item["processed_product_id"] = s
|
||||
}
|
||||
if rawProductID != nil {
|
||||
item["raw_product_id"] = rawProductID.String()
|
||||
}
|
||||
_ = item
|
||||
_ = processedID
|
||||
_ = rawProductID
|
||||
}
|
||||
|
||||
func withAlwaysIncluded(item V1ProcessJobItem, main string, more []string, eprel any) V1ProcessJobItem {
|
||||
|
||||
@@ -72,11 +72,8 @@ func TestMapV1JobItemStatus(t *testing.T) {
|
||||
|
||||
func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
|
||||
items := []V1ProcessJobItem{{
|
||||
"ean": "123", "id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
|
||||
"processed_product_id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
|
||||
"raw_product_id": "cccccccc-cccc-cccc-cccc-cccccccccccc",
|
||||
"status": "processed",
|
||||
"title": "T", "name": "T", "meta_title": "MT",
|
||||
"ean": "123", "status": "processed",
|
||||
"name": "T", "meta_title": "MT",
|
||||
"description": "D", "attributes": map[string]any{"brand": "X"},
|
||||
"main_image": "https://example.com/a.jpg", "more_images": []string{"https://example.com/b.jpg"},
|
||||
"eprel": nil, "category": "cat", "category_name": "Cat",
|
||||
@@ -88,11 +85,11 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
|
||||
raw, _ := json.Marshal(out[0])
|
||||
var got map[string]any
|
||||
_ = json.Unmarshal(raw, &got)
|
||||
if got["title"] != "T" || got["ean"] != "123" {
|
||||
if got["name"] != "T" || got["ean"] != "123" {
|
||||
t.Fatalf("got=%v", got)
|
||||
}
|
||||
if got["name"] != "T" {
|
||||
t.Fatalf("name dual-mode alias missing or mismatched: %v", got)
|
||||
if _, ok := got["title"]; ok {
|
||||
t.Fatalf("duplicate title should be omitted when name exists: %v", got)
|
||||
}
|
||||
if _, ok := got["attributes"]; ok {
|
||||
t.Fatalf("attributes should be projected out: %v", got)
|
||||
@@ -100,31 +97,29 @@ func TestProjectV1ProcessJobItemsPartial(t *testing.T) {
|
||||
if got["main_image"] != "https://example.com/a.jpg" {
|
||||
t.Fatalf("images always included: %v", got)
|
||||
}
|
||||
if got["status"] != "processed" || got["id"] != "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" {
|
||||
t.Fatalf("meta should be preserved: %v", got)
|
||||
if got["status"] != "processed" {
|
||||
t.Fatalf("status should be preserved: %v", got)
|
||||
}
|
||||
if got["processed_product_id"] != "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" {
|
||||
t.Fatalf("processed_product_id should be preserved: %v", got)
|
||||
}
|
||||
if got["raw_product_id"] != "cccccccc-cccc-cccc-cccc-cccccccccccc" {
|
||||
t.Fatalf("raw_product_id should be preserved: %v", got)
|
||||
for _, junk := range []string{"id", "processed_product_id", "raw_product_id"} {
|
||||
if _, ok := got[junk]; ok {
|
||||
t.Fatalf("%s must be omitted from V1 process items: %v", junk, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectV1ProcessJobItemsTitleDerivesName(t *testing.T) {
|
||||
items := []V1ProcessJobItem{{
|
||||
"ean": "123", "id": "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
|
||||
"status": "processed", "title": "OnlyTitle", "meta_title": "MT",
|
||||
"ean": "123", "status": "processed", "title": "OnlyTitle", "meta_title": "MT",
|
||||
}}
|
||||
out := ProjectV1ProcessJobItems("title", items)
|
||||
if len(out) != 1 {
|
||||
t.Fatalf("len=%d", len(out))
|
||||
}
|
||||
if out[0]["title"] != "OnlyTitle" {
|
||||
t.Fatalf("title=%v", out[0]["title"])
|
||||
}
|
||||
if out[0]["name"] != "OnlyTitle" {
|
||||
t.Fatalf("name should mirror title when absent: %v", out[0]["name"])
|
||||
t.Fatalf("name should derive from title when absent: %v", out[0]["name"])
|
||||
}
|
||||
if _, ok := out[0]["title"]; ok {
|
||||
t.Fatalf("title omitted when identical to name: %v", out[0])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,22 +175,18 @@ func TestApplyV1ProcessItemIDsDualMode(t *testing.T) {
|
||||
raw := mustParseTestUUID(t, "cccccccc-cccc-cccc-cccc-cccccccccccc")
|
||||
item := V1ProcessJobItem{"ean": "1", "status": "processed"}
|
||||
applyV1ProcessItemIDs(item, &processed, &raw)
|
||||
if item["id"] != processed.String() {
|
||||
t.Fatalf("id=%v", item["id"])
|
||||
}
|
||||
if item["processed_product_id"] != processed.String() {
|
||||
t.Fatalf("processed_product_id=%v", item["processed_product_id"])
|
||||
}
|
||||
if item["raw_product_id"] != raw.String() {
|
||||
t.Fatalf("raw_product_id=%v", item["raw_product_id"])
|
||||
for _, junk := range []string{"id", "processed_product_id", "raw_product_id"} {
|
||||
if _, ok := item[junk]; ok {
|
||||
t.Fatalf("%s must stay omitted from V1 process items: %v", junk, item)
|
||||
}
|
||||
}
|
||||
missing := V1ProcessJobItem{"ean": "2", "status": "not_found"}
|
||||
applyV1ProcessItemIDs(missing, nil, &raw)
|
||||
if _, ok := missing["id"]; ok {
|
||||
t.Fatalf("id must stay absent without processed row: %v", missing)
|
||||
t.Fatalf("id must stay absent: %v", missing)
|
||||
}
|
||||
if missing["raw_product_id"] != raw.String() {
|
||||
t.Fatalf("raw_product_id on not_found: %v", missing)
|
||||
if _, ok := missing["raw_product_id"]; ok {
|
||||
t.Fatalf("raw_product_id must stay omitted: %v", missing)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -63,11 +63,17 @@ func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemO
|
||||
return sc
|
||||
}
|
||||
|
||||
title := stringFromItem(item, "title")
|
||||
title := stringFromItem(item, "name")
|
||||
if title == "" {
|
||||
title = stringFromItem(item, "title")
|
||||
}
|
||||
name := stringFromItem(item, "name")
|
||||
if name == "" {
|
||||
name = title
|
||||
}
|
||||
sc.HasTitle = title != ""
|
||||
sc.HasName = name != ""
|
||||
if sc.HasTitle && !sc.HasName {
|
||||
if !sc.HasName && sc.HasTitle {
|
||||
sc.FailFlags = append(sc.FailFlags, "missing_name")
|
||||
}
|
||||
if sc.HasTitle && sc.HasName && title != name {
|
||||
@@ -140,22 +146,8 @@ func ScoreV1ProcessCompletedItem(item V1ProcessJobItem, opts ScoreV1ProcessItemO
|
||||
sc.FailFlags = append(sc.FailFlags, "eprel_invalid_shape")
|
||||
}
|
||||
|
||||
id := stringFromItem(item, "id")
|
||||
ppID := stringFromItem(item, "processed_product_id")
|
||||
rawID := stringFromItem(item, "raw_product_id")
|
||||
sc.HasIDs = id != "" && ppID != "" && rawID != ""
|
||||
if id == "" {
|
||||
sc.FailFlags = append(sc.FailFlags, "missing_id")
|
||||
}
|
||||
if ppID == "" {
|
||||
sc.FailFlags = append(sc.FailFlags, "missing_processed_product_id")
|
||||
}
|
||||
if rawID == "" {
|
||||
sc.FailFlags = append(sc.FailFlags, "missing_raw_product_id")
|
||||
}
|
||||
if id != "" && ppID != "" && id != ppID {
|
||||
sc.FailFlags = append(sc.FailFlags, "id_processed_product_id_mismatch")
|
||||
}
|
||||
// Internal UUIDs are omitted from the public V1 process item shape.
|
||||
sc.HasIDs = true
|
||||
|
||||
sc.ImagesOK = imageFieldsOK(item)
|
||||
if !sc.ImagesOK {
|
||||
@@ -234,14 +226,20 @@ func EnforceV1ProcessCompletedItemOpts(item V1ProcessJobItem, opts EnforceV1Opts
|
||||
item["attributes"] = nil
|
||||
}
|
||||
|
||||
title := ensureReadableTitleSpacing(stringFromItem(item, "title"))
|
||||
if title != "" {
|
||||
item["title"] = title
|
||||
item["name"] = title
|
||||
} else {
|
||||
item["title"] = nil
|
||||
item["name"] = nil
|
||||
title := ensureReadableTitleSpacing(stringFromItem(item, "name"))
|
||||
if title == "" {
|
||||
title = ensureReadableTitleSpacing(stringFromItem(item, "title"))
|
||||
}
|
||||
if title != "" {
|
||||
item["name"] = title
|
||||
delete(item, "title")
|
||||
} else {
|
||||
item["name"] = nil
|
||||
delete(item, "title")
|
||||
}
|
||||
delete(item, "id")
|
||||
delete(item, "processed_product_id")
|
||||
delete(item, "raw_product_id")
|
||||
|
||||
catID, catName := projectV1CategoryFields(item)
|
||||
catLabel := catName
|
||||
|
||||
@@ -62,7 +62,6 @@ func TestScoreV1ProcessCompletedItem_flagsGaps(t *testing.T) {
|
||||
}
|
||||
joined := strings.Join(sc.FailFlags, ",")
|
||||
for _, want := range []string{
|
||||
"missing_name",
|
||||
"description_empty_with_title",
|
||||
"meta_title_missing",
|
||||
"meta_description_missing",
|
||||
@@ -103,7 +102,7 @@ func TestEnforceV1ProcessCompletedItem_fillsDescriptionMetaSanitizes(t *testing.
|
||||
if desc == "" {
|
||||
t.Fatal("expected nonempty description")
|
||||
}
|
||||
if descriptionEchoesTitle(desc, fmt.Sprint(out["title"])) {
|
||||
if descriptionEchoesTitle(desc, fmt.Sprint(out["name"])) {
|
||||
t.Fatalf("EnforceV1 must replace weak/echo desc, got title-echo: %q", desc)
|
||||
}
|
||||
if containsWeakFillerPhrase(desc) {
|
||||
@@ -131,8 +130,16 @@ func TestEnforceV1ProcessCompletedItem_fillsDescriptionMetaSanitizes(t *testing.
|
||||
if _, bad := attrs["name"]; bad {
|
||||
t.Fatalf("reserved name kept: %v", attrs)
|
||||
}
|
||||
if out["name"] != out["title"] {
|
||||
t.Fatalf("name should mirror title: name=%v title=%v", out["name"], out["title"])
|
||||
if out["name"] == nil || strings.TrimSpace(fmt.Sprint(out["name"])) == "" {
|
||||
t.Fatalf("name missing: %v", out["name"])
|
||||
}
|
||||
if _, ok := out["title"]; ok {
|
||||
t.Fatalf("duplicate title should be omitted when name exists: %v", out["title"])
|
||||
}
|
||||
for _, junk := range []string{"id", "processed_product_id", "raw_product_id"} {
|
||||
if _, ok := out[junk]; ok {
|
||||
t.Fatalf("%s must be omitted: %v", junk, out)
|
||||
}
|
||||
}
|
||||
sc := ScoreV1ProcessCompletedItem(out, ScoreV1ProcessItemOptions{MappedCategory: "28"})
|
||||
if !sc.OK {
|
||||
|
||||
Reference in New Issue
Block a user