fix
This commit is contained in:
@@ -0,0 +1,200 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func main() {
|
||||
log.SetOutput(logredact.Writer(os.Stderr))
|
||||
eans := []string{
|
||||
"195348253666", // 7 Gaming monitorji (rich)
|
||||
"4548736132597", // 48 Slušalke (rich)
|
||||
"8712285326882", // 28 Nosilci za TV (thin p-only)
|
||||
"3045380024786", // 36 Pečice (rich)
|
||||
"3838782459856", // 38 Pomivalni stroji (rich)
|
||||
"8806095210711", // 40 Pralni stroji (rich)
|
||||
"4242005342488", // 41 Pralno-sušilni (rich)
|
||||
"3838782103889", // 17 Kuhalne plošče (rich)
|
||||
"1200130000638", // 3 Bluetooth zvočniki (rich)
|
||||
"194252029558", // 24 Mobilne naprave (rich)
|
||||
}
|
||||
companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
|
||||
userID := uuid.MustParse("0ce3305c-d810-4b56-b1d4-3c1ed510db76")
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("config: %v", err)
|
||||
}
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||||
MaxConns: int32(cfg.DBMaxConns),
|
||||
MinConns: int32(cfg.DBMinConns),
|
||||
MaxConnLifetime: cfg.DBMaxConnLifetime,
|
||||
MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
|
||||
MaxConnIdleTime: cfg.DBMaxConnIdleTime,
|
||||
HealthCheckPeriod: cfg.DBHealthCheckPeriod,
|
||||
StatementTimeout: cfg.DBStatementTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("db: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
out := os.Getenv("PROBE_OUT_DIR")
|
||||
if out == "" {
|
||||
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_formula_probe_20260816`
|
||||
}
|
||||
_ = os.MkdirAll(out, 0o755)
|
||||
|
||||
pipeline := processing.NewPipeline(pool)
|
||||
pipeline.BatchSize = cfg.ProcessingBatchSize
|
||||
pipeline.AI = nil
|
||||
pipeline.Prompts = aiprompts.NewService(pool)
|
||||
|
||||
mockKey := strings.TrimSpace(cfg.OpenAIAPIKey)
|
||||
if mockKey == "" {
|
||||
mockKey = "local-test"
|
||||
}
|
||||
mockBase := "http://127.0.0.1:18767/v1"
|
||||
mockModel := "mock-llm"
|
||||
log.Printf("forcing completer base=%s model=%s (ModeLabel=internal; OverloadedBot not used)", mockBase, mockModel)
|
||||
client := processing.NewOpenAIClient(mockKey, mockBase, mockModel, cfg.ProcessingRPM, cfg.ProcessingMaxRetries)
|
||||
client.ModeLabel = processing.AIProviderInternal
|
||||
|
||||
eprelClient := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second})
|
||||
pipeline.Engine = &processing.Engine{
|
||||
Completer: client,
|
||||
Vector: processing.NoopVectorCategorizer{},
|
||||
EPREL: eprelClient,
|
||||
ProviderMode: processing.AIProviderInternal,
|
||||
}
|
||||
|
||||
rawIDs := make([]uuid.UUID, 0, len(eans))
|
||||
for _, ean := range eans {
|
||||
var id uuid.UUID
|
||||
err := pool.QueryRow(ctx, `
|
||||
SELECT id FROM raw_products
|
||||
WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id)
|
||||
if err != nil {
|
||||
log.Fatalf("raw product %s: %v", ean, err)
|
||||
}
|
||||
rawIDs = append(rawIDs, id)
|
||||
log.Printf("ean=%s raw_id=%s", ean, id)
|
||||
}
|
||||
|
||||
if n, err := processing.BackfillMissingMeta(ctx, pool, companyID); err != nil {
|
||||
log.Printf("BackfillMissingMeta: %v", err)
|
||||
} else {
|
||||
log.Printf("BackfillMissingMeta updated=%d", n)
|
||||
}
|
||||
|
||||
jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full")
|
||||
if err != nil {
|
||||
log.Fatalf("StartJob: %v", err)
|
||||
}
|
||||
if len(jobs) == 0 {
|
||||
log.Fatal("StartJob returned no jobs")
|
||||
}
|
||||
jobID := jobs[0].ID
|
||||
log.Printf("job_id=%s", jobID)
|
||||
_ = os.WriteFile(out+string(os.PathSeparator)+"process_id.txt", []byte(jobID.String()+"\n"), 0o644)
|
||||
|
||||
defend := make(chan struct{})
|
||||
go func() {
|
||||
t := time.NewTicker(500 * time.Millisecond)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-defend:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'running', error = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status IN ('failed','cancelled')
|
||||
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
|
||||
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
|
||||
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%'
|
||||
OR error ILIKE '%formula%')`, jobID)
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_job_products
|
||||
SET status = 'pending', error = NULL, updated_at = now()
|
||||
WHERE job_id = $1 AND status IN ('failed','cancelled')
|
||||
AND processed_product_id IS NULL
|
||||
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
|
||||
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
|
||||
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%'
|
||||
OR error ILIKE '%formula%')`, jobID)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
_, _ = pool.Exec(ctx, `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
|
||||
WHERE id = $1`, jobID)
|
||||
|
||||
runCtx, runCancel := context.WithTimeout(ctx, 15*time.Minute)
|
||||
defer runCancel()
|
||||
if err := pipeline.ProcessJob(runCtx, jobID); err != nil {
|
||||
close(defend)
|
||||
log.Fatalf("ProcessJob: %v", err)
|
||||
}
|
||||
close(defend)
|
||||
|
||||
items, err := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
|
||||
if err != nil {
|
||||
log.Fatalf("LoadV1ProcessJobItems: %v", err)
|
||||
}
|
||||
payload := map[string]any{
|
||||
"data": map[string]any{
|
||||
"process_id": jobID.String(),
|
||||
"status": "COMPLETED",
|
||||
"processing_type": "full",
|
||||
"total_items": len(items),
|
||||
"items": items,
|
||||
},
|
||||
}
|
||||
raw, err := json.MarshalIndent(payload, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("marshal: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(out+string(os.PathSeparator)+"final.json", raw, 0o644); err != nil {
|
||||
log.Fatalf("write final: %v", err)
|
||||
}
|
||||
fmt.Printf("JOB %s items=%d\n", jobID, len(items))
|
||||
for _, it := range items {
|
||||
ean, _ := it["ean"].(string)
|
||||
st, _ := it["status"].(string)
|
||||
title, _ := it["title"].(string)
|
||||
cat, _ := it["category"].(string)
|
||||
catn, _ := it["category_name"].(string)
|
||||
desc, _ := it["description"].(string)
|
||||
if len(title) > 50 {
|
||||
title = title[:50]
|
||||
}
|
||||
ai, _ := it["ai_provider_mode"].(string)
|
||||
fmt.Printf("ITEM %s status=%s cat=%s/%s ai=%s desc_len=%d title=%s\n",
|
||||
ean, st, cat, catn, ai, len(desc), title)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
fmt.Println("config:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||||
MaxConns: 2,
|
||||
MinConns: 1,
|
||||
HealthCheckPeriod: 30 * time.Second,
|
||||
MaxConnLifetime: time.Hour,
|
||||
MaxConnIdleTime: 30 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println("db:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer pool.Close()
|
||||
plat := platformsettings.NewService(pool, platformsettings.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey,
|
||||
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret,
|
||||
DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey,
|
||||
OpenAIBaseURL: cfg.OpenAIBaseURL,
|
||||
OpenAIModel: cfg.OpenAIModel,
|
||||
})
|
||||
rc, err := plat.ResolveAIConfig(ctx, platformsettings.AIRoleProcessing)
|
||||
if err != nil {
|
||||
fmt.Println("resolve:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
base := strings.TrimRight(strings.TrimSpace(rc.BaseURL), "/")
|
||||
url := base + "/chat/completions"
|
||||
body := map[string]any{
|
||||
"model": rc.Model,
|
||||
"messages": []map[string]string{
|
||||
{"role": "user", "content": "Say PONG"},
|
||||
},
|
||||
"max_tokens": 16,
|
||||
}
|
||||
rawBody, _ := json.Marshal(body)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(rawBody))
|
||||
if err != nil {
|
||||
fmt.Println("req:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+rc.APIKey)
|
||||
cli := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := cli.Do(req)
|
||||
if err != nil {
|
||||
fmt.Println("do:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1200))
|
||||
s := string(raw)
|
||||
s = strings.ReplaceAll(s, rc.APIKey, "[redacted]")
|
||||
fmt.Printf("source=%s provider=%s model=%s\n", rc.Source, rc.Provider, rc.Model)
|
||||
fmt.Printf("url=%s status=%d bytes=%d\n", url, resp.StatusCode, len(raw))
|
||||
fmt.Printf("content_type=%s\n", resp.Header.Get("Content-Type"))
|
||||
fmt.Printf("body_head=%q\n", s)
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func main() {
|
||||
out := `F:\laragon\www\_MY\descrybe-v2\.codehelper\_live_llm_formula_20260816`
|
||||
jobID := uuid.MustParse("3d926dfa-de8a-4afc-b1dd-8707633c673e")
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||||
MaxConns: 4, MinConns: 0,
|
||||
MaxConnLifetime: time.Hour, HealthCheckPeriod: 30 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT rp.gtin,
|
||||
COALESCE(pp.processed_name, pp.name, '') AS title,
|
||||
COALESCE(NULLIF(pp.processed_description, ''), pp.description, '') AS description,
|
||||
COALESCE(pp.category, '') AS category,
|
||||
COALESCE(c.name, '') AS category_name,
|
||||
c.description_template,
|
||||
pp.id
|
||||
FROM processing_job_products pjp
|
||||
JOIN raw_products rp ON rp.id = pjp.raw_product_id
|
||||
LEFT JOIN processed_products pp ON pp.id = pjp.processed_product_id
|
||||
LEFT JOIN categories c ON c.company_id = pp.company_id AND c.unique_id = pp.category
|
||||
WHERE pjp.job_id = $1
|
||||
ORDER BY rp.gtin`, jobID)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
products := []map[string]any{}
|
||||
for rows.Next() {
|
||||
var gtin, title, desc, cat, catName string
|
||||
var tmpl any
|
||||
var ppID *uuid.UUID
|
||||
if err := rows.Scan(>in, &title, &desc, &cat, &catName, &tmpl, &ppID); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
idStr := ""
|
||||
if ppID != nil {
|
||||
idStr = ppID.String()
|
||||
}
|
||||
lower := strings.ToLower(desc)
|
||||
products = append(products, map[string]any{
|
||||
"ean": gtin,
|
||||
"product_id": idStr,
|
||||
"title": title,
|
||||
"description": desc,
|
||||
"category": cat,
|
||||
"category_name": catName,
|
||||
"description_template": tmpl,
|
||||
"formula_constraint": processing.FormatDescriptionFormulaConstraint(tmpl),
|
||||
"has_h1": strings.Contains(lower, "<h1"),
|
||||
"has_h2": strings.Contains(lower, "<h2"),
|
||||
"has_p": strings.Contains(lower, "<p"),
|
||||
"has_ul": strings.Contains(lower, "<ul"),
|
||||
"desc_len": len(desc),
|
||||
"desc_ne_title": strings.TrimSpace(desc) != "" && strings.TrimSpace(desc) != strings.TrimSpace(title),
|
||||
})
|
||||
}
|
||||
raw, _ := json.MarshalIndent(products, "", " ")
|
||||
_ = os.WriteFile(out+`\db_products.json`, raw, 0o644)
|
||||
fmt.Printf("wrote %d products\n", len(products))
|
||||
for _, p := range products {
|
||||
t := p["title"].(string)
|
||||
if len(t) > 40 {
|
||||
t = t[:40]
|
||||
}
|
||||
fmt.Printf("%s cat=%s/%s h1=%v h2=%v p=%v ul=%v desc_ne_title=%v len=%v title=%q\n",
|
||||
p["ean"], p["category"], p["category_name"], p["has_h1"], p["has_h2"], p["has_p"], p["has_ul"], p["desc_ne_title"], p["desc_len"], t)
|
||||
d := p["description"].(string)
|
||||
if len(d) > 220 {
|
||||
d = d[:220]
|
||||
}
|
||||
fmt.Printf(" desc_preview=%q\n", d)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func main() {
|
||||
out := os.Getenv("PROBE_OUT_DIR")
|
||||
if out == "" {
|
||||
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_live_llm_enhance_budget_20260816`
|
||||
}
|
||||
_ = os.MkdirAll(out, 0o755)
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("config: %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||||
MaxConns: 2,
|
||||
MinConns: 1,
|
||||
HealthCheckPeriod: 30 * time.Second,
|
||||
MaxConnLifetime: time.Hour,
|
||||
MaxConnIdleTime: 30 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("db: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
plat := platformsettings.NewService(pool, platformsettings.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey,
|
||||
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret,
|
||||
DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey,
|
||||
OpenAIBaseURL: cfg.OpenAIBaseURL,
|
||||
OpenAIModel: cfg.OpenAIModel,
|
||||
})
|
||||
aiSvc := aiprovider.NewService(pool, aiprovider.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey,
|
||||
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret,
|
||||
DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey,
|
||||
OpenAIBaseURL: cfg.OpenAIBaseURL,
|
||||
OpenAIModel: cfg.OpenAIModel,
|
||||
ProcessingRPM: cfg.ProcessingRPM,
|
||||
ProcessingMaxRetries: cfg.ProcessingMaxRetries,
|
||||
})
|
||||
aiSvc.Platform = plat
|
||||
|
||||
companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
|
||||
completer, mode, byok, err := aiSvc.ResolveCompleterForRole(ctx, companyID, aiprovider.RoleProcessing)
|
||||
if err != nil {
|
||||
log.Fatalf("resolve: %v", err)
|
||||
}
|
||||
client, ok := completer.(*processing.OpenAIClient)
|
||||
if !ok || client == nil {
|
||||
log.Fatalf("completer type %T", completer)
|
||||
}
|
||||
if processing.IsMockOrLoopbackBaseURL(client.BaseURL) {
|
||||
log.Fatalf("resolved mock/loopback base=%s — refuse", client.BaseURL)
|
||||
}
|
||||
|
||||
system := `Reply with ONLY one JSON object: {"name":string,"description":string}.
|
||||
description MUST be HTML using exactly these tags in order: <h1>, <p>, <h2>, <ul><li>…</li></ul>, <p>, <p>, <p>.
|
||||
No markdown. No prose outside JSON.`
|
||||
cases := []struct {
|
||||
ean string
|
||||
user string
|
||||
}{
|
||||
{
|
||||
ean: "1200130000638",
|
||||
user: "Category: Bluetooth zvočniki\nName: JBL Flip 6\nDesc: Portable Bluetooth speaker.\nAttrs: {\"brand\":\"JBL\"}",
|
||||
},
|
||||
{
|
||||
ean: "195348253666",
|
||||
user: "Category: Gaming monitorji\nName: Lenovo G27-20\nDesc: 27-inch gaming monitor.\nAttrs: {\"brand\":\"Lenovo\"}",
|
||||
},
|
||||
}
|
||||
|
||||
results := make([]map[string]any, 0, len(cases))
|
||||
okLLM := 0
|
||||
for _, c := range cases {
|
||||
started := time.Now()
|
||||
comp, obj, cerr := processing.CompleteJSON(ctx, client, system, c.user, processing.CompleteOptions{
|
||||
MaxTokens: processing.MaxTokensEnhance,
|
||||
Temperature: processing.DefaultStructuredTemp,
|
||||
})
|
||||
elapsed := time.Since(started).Round(time.Millisecond)
|
||||
fr := ""
|
||||
if m, ok := comp.Raw.(map[string]any); ok {
|
||||
fr, _ = m["finish_reason"].(string)
|
||||
}
|
||||
row := map[string]any{
|
||||
"ean": c.ean,
|
||||
"elapsed": elapsed.String(),
|
||||
"err": "",
|
||||
"finish_reason": fr,
|
||||
"prompt_tokens": comp.PromptTokens,
|
||||
"completion_tokens": comp.OutputTokens,
|
||||
"total_tokens": comp.TotalTokens,
|
||||
"content_runes": len([]rune(comp.Text)),
|
||||
"parsed": obj != nil,
|
||||
"llm_formula_ok": false,
|
||||
}
|
||||
if cerr != nil {
|
||||
row["err"] = processing.TruncateError(cerr)
|
||||
}
|
||||
if obj != nil {
|
||||
desc := fmt.Sprint(obj["description"])
|
||||
row["name"] = fmt.Sprint(obj["name"])
|
||||
row["description_head"] = truncate(desc, 180)
|
||||
hasTags := strings.Contains(desc, "<h1") && strings.Contains(desc, "<h2") &&
|
||||
strings.Contains(desc, "<p") && strings.Contains(desc, "<ul")
|
||||
row["has_h1_h2_p_ul"] = hasTags
|
||||
row["llm_formula_ok"] = cerr == nil && hasTags && strings.EqualFold(fr, "stop")
|
||||
if row["llm_formula_ok"].(bool) {
|
||||
okLLM++
|
||||
}
|
||||
}
|
||||
results = append(results, row)
|
||||
log.Printf("live enhance ean=%s finish_reason=%s tokens=%d/%d/%d elapsed=%s parsed=%v formula_ok=%v err=%v",
|
||||
c.ean, fr, comp.PromptTokens, comp.OutputTokens, comp.TotalTokens, elapsed, obj != nil, row["llm_formula_ok"], row["err"])
|
||||
}
|
||||
|
||||
summary := map[string]any{
|
||||
"at_utc": time.Now().UTC().Format(time.RFC3339),
|
||||
"base": client.BaseURL,
|
||||
"model": client.Model,
|
||||
"mode": mode,
|
||||
"byok": byok,
|
||||
"max_tokens_enhance": processing.MaxTokensEnhance,
|
||||
"max_tokens_retry": processing.MaxTokensEnhanceRetry,
|
||||
"products": len(cases),
|
||||
"llm_formula_ok_count": okLLM,
|
||||
"live_llm_formula_pass": okLLM == len(cases),
|
||||
"results": results,
|
||||
}
|
||||
raw, _ := json.MarshalIndent(summary, "", " ")
|
||||
_ = os.WriteFile(filepath.Join(out, "summary.json"), raw, 0o644)
|
||||
fmt.Println(string(raw))
|
||||
if okLLM != len(cases) {
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
r := []rune(strings.TrimSpace(s))
|
||||
if len(r) <= n {
|
||||
return string(r)
|
||||
}
|
||||
return string(r[:n]) + "…"
|
||||
}
|
||||
@@ -0,0 +1,443 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func main() {
|
||||
out := os.Getenv("PROBE_OUT_DIR")
|
||||
if out == "" {
|
||||
out = `F:\laragon\www\_MY\descrybe-v2\.codehelper\_live_llm_formula_20260816`
|
||||
}
|
||||
_ = os.MkdirAll(out, 0o755)
|
||||
|
||||
logPath := filepath.Join(out, "inproc_run.log")
|
||||
logFile, err := os.Create(logPath)
|
||||
if err != nil {
|
||||
log.Fatalf("log file: %v", err)
|
||||
}
|
||||
defer logFile.Close()
|
||||
log.SetOutput(logredact.Writer(io.MultiWriter(os.Stderr, logFile)))
|
||||
|
||||
// 5 rich A1 description formulas (h1/h2/p/ul) — Platform Demo.
|
||||
eans := []string{
|
||||
"195348253666", // 7 Gaming monitorji
|
||||
"4548736132597", // 48 Slušalke
|
||||
"3045380024786", // 36 Pečice
|
||||
"3838782459856", // 38 Pomivalni stroji
|
||||
"1200130000638", // 3 Bluetooth zvočniki
|
||||
}
|
||||
companyID := uuid.MustParse("2b3159b0-fc08-415b-b248-35ed02a6baab")
|
||||
userID := uuid.MustParse("0ce3305c-d810-4b56-b1d4-3c1ed510db76")
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("config: %v", err)
|
||||
}
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||||
MaxConns: int32(cfg.DBMaxConns),
|
||||
MinConns: int32(cfg.DBMinConns),
|
||||
MaxConnLifetime: cfg.DBMaxConnLifetime,
|
||||
MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
|
||||
MaxConnIdleTime: cfg.DBMaxConnIdleTime,
|
||||
HealthCheckPeriod: cfg.DBHealthCheckPeriod,
|
||||
StatementTimeout: cfg.DBStatementTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalf("db: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
platEnv := platformsettings.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey,
|
||||
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret,
|
||||
DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey,
|
||||
OpenAIBaseURL: cfg.OpenAIBaseURL,
|
||||
OpenAIModel: cfg.OpenAIModel,
|
||||
OpenAIEmbeddingAPIKey: cfg.OpenAIEmbeddingAPIKey,
|
||||
OpenAIEmbeddingBaseURL: cfg.OpenAIEmbeddingBaseURL,
|
||||
OpenAIEmbeddingModel: cfg.OpenAIEmbeddingModel,
|
||||
}
|
||||
platSettings := platformsettings.NewService(pool, platEnv)
|
||||
aiSvc := aiprovider.NewService(pool, aiprovider.EnvConfig{
|
||||
AppEncryptionKey: cfg.AppEncryptionKey,
|
||||
CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||||
TokenSigningSecret: cfg.TokenSigningSecret,
|
||||
DatabaseURL: cfg.DatabaseURL,
|
||||
OpenAIAPIKey: cfg.OpenAIAPIKey,
|
||||
OpenAIBaseURL: cfg.OpenAIBaseURL,
|
||||
OpenAIModel: cfg.OpenAIModel,
|
||||
ProcessingRPM: cfg.ProcessingRPM,
|
||||
ProcessingMaxRetries: cfg.ProcessingMaxRetries,
|
||||
})
|
||||
aiSvc.Platform = platSettings
|
||||
|
||||
// Resolve like worker (DB admin AI / company BYOK preferred over mock .env).
|
||||
roleCfg, rerr := platSettings.ResolveAIConfig(ctx, platformsettings.AIRoleProcessing)
|
||||
if rerr != nil {
|
||||
failResolve(out, fmt.Sprintf("ResolveAIConfig: %v", rerr))
|
||||
}
|
||||
completer, modeLabel, byok, cerr := aiSvc.ResolveCompleterForRole(ctx, companyID, aiprovider.RoleProcessing)
|
||||
if cerr != nil {
|
||||
failResolve(out, fmt.Sprintf("ResolveCompleterForRole: %v", cerr))
|
||||
}
|
||||
if completer == nil {
|
||||
failResolve(out, "ResolveCompleterForRole returned nil completer (OpenAI not configured in admin/BYOK/env)")
|
||||
}
|
||||
client, ok := completer.(*processing.OpenAIClient)
|
||||
if !ok {
|
||||
failResolve(out, fmt.Sprintf("completer type %T is not *OpenAIClient", completer))
|
||||
}
|
||||
base := strings.TrimSpace(client.BaseURL)
|
||||
model := strings.TrimSpace(client.Model)
|
||||
resolveInfo := map[string]any{
|
||||
"role_source": roleCfg.Source,
|
||||
"role_provider": roleCfg.Provider,
|
||||
"role_base_url": roleCfg.BaseURL,
|
||||
"role_model": roleCfg.Model,
|
||||
"role_enabled": roleCfg.Enabled,
|
||||
"completer_base": base,
|
||||
"completer_model": model,
|
||||
"mode_label": modeLabel,
|
||||
"byok": byok,
|
||||
"env_base_url": cfg.OpenAIBaseURL,
|
||||
"env_model": cfg.OpenAIModel,
|
||||
"live": true,
|
||||
}
|
||||
if processing.IsMockOrLoopbackBaseURL(base) || strings.Contains(strings.ToLower(base), "18767") {
|
||||
resolveInfo["live"] = false
|
||||
resolveInfo["fail_reason"] = "resolved completer is mock/loopback — refuse silent mock fallback"
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM REQUIRED: resolved base=%s model=%s source=%s mode=%s — refuse mock :18767 / loopback",
|
||||
base, model, roleCfg.Source, modeLabel)
|
||||
}
|
||||
if strings.TrimSpace(client.APIKey) == "" {
|
||||
resolveInfo["live"] = false
|
||||
resolveInfo["fail_reason"] = "empty API key"
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatal("LIVE LLM REQUIRED: API key empty after resolve")
|
||||
}
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Printf("live resolve source=%s base=%s model=%s mode=%s byok=%v (env was base=%s model=%s)",
|
||||
roleCfg.Source, base, model, modeLabel, byok, cfg.OpenAIBaseURL, cfg.OpenAIModel)
|
||||
|
||||
// Connectivity probe — GET /v1/models then tiny chat (admin DB key; never log full key).
|
||||
modelsURL := strings.TrimRight(base, "/") + "/models"
|
||||
keyHint := ""
|
||||
if k := strings.TrimSpace(client.APIKey); len(k) > 8 {
|
||||
keyHint = k[:4] + "…" + k[len(k)-4:]
|
||||
} else if k := strings.TrimSpace(client.APIKey); k != "" {
|
||||
keyHint = "(short)"
|
||||
}
|
||||
resolveInfo["api_key_hint"] = keyHint
|
||||
resolveInfo["retried_at_utc"] = time.Now().UTC().Format(time.RFC3339)
|
||||
|
||||
modelsCtx, modelsCancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer modelsCancel()
|
||||
modelsReq, merr := http.NewRequestWithContext(modelsCtx, http.MethodGet, modelsURL, nil)
|
||||
if merr != nil {
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["fail_reason"] = "models request build: " + merr.Error()
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: models req: %v", merr)
|
||||
}
|
||||
modelsReq.Header.Set("Authorization", "Bearer "+client.APIKey)
|
||||
modelsReq.Header.Set("Accept", "application/json")
|
||||
modelsStart := time.Now()
|
||||
modelsResp, merr := http.DefaultClient.Do(modelsReq)
|
||||
modelsElapsed := time.Since(modelsStart).Round(time.Millisecond)
|
||||
if merr != nil {
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["models_url"] = modelsURL
|
||||
resolveInfo["models_error"] = processing.TruncateError(merr)
|
||||
resolveInfo["models_elapsed"] = modelsElapsed.String()
|
||||
resolveInfo["fail_reason"] = "GET /v1/models network error"
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: GET %s elapsed=%s err=%s",
|
||||
modelsURL, modelsElapsed, processing.TruncateError(merr))
|
||||
}
|
||||
modelsBody, _ := io.ReadAll(io.LimitReader(modelsResp.Body, 2048))
|
||||
_ = modelsResp.Body.Close()
|
||||
modelsSnippet := strings.TrimSpace(string(modelsBody))
|
||||
if len(modelsSnippet) > 240 {
|
||||
modelsSnippet = modelsSnippet[:240] + "…"
|
||||
}
|
||||
resolveInfo["models_url"] = modelsURL
|
||||
resolveInfo["models_http_status"] = modelsResp.StatusCode
|
||||
resolveInfo["models_content_type"] = modelsResp.Header.Get("Content-Type")
|
||||
resolveInfo["models_elapsed"] = modelsElapsed.String()
|
||||
resolveInfo["models_body_snippet"] = modelsSnippet
|
||||
if modelsResp.StatusCode != http.StatusOK {
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["http_status"] = modelsResp.StatusCode
|
||||
resolveInfo["fail_reason"] = fmt.Sprintf("GET /v1/models returned HTTP %d (expect 200; 401=wrong/missing key, 502=upstream)", modelsResp.StatusCode)
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: GET %s status=%d ct=%s snippet=%q",
|
||||
modelsURL, modelsResp.StatusCode, modelsResp.Header.Get("Content-Type"), modelsSnippet)
|
||||
}
|
||||
log.Printf("live models ok status=200 elapsed=%s key_hint=%s", modelsElapsed, keyHint)
|
||||
|
||||
probeCtx, probeCancel := context.WithTimeout(ctx, 45*time.Second)
|
||||
defer probeCancel()
|
||||
probeStart := time.Now()
|
||||
comp, perr := client.Complete(probeCtx, "Reply with exactly: PONG", "Say PONG")
|
||||
probeElapsed := time.Since(probeStart).Round(time.Millisecond)
|
||||
if perr != nil {
|
||||
resolveInfo["probe_ok"] = false
|
||||
resolveInfo["probe_error"] = processing.TruncateError(perr)
|
||||
resolveInfo["probe_elapsed"] = probeElapsed.String()
|
||||
resolveInfo["fail_reason"] = "chat/completions probe failed after models 200"
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Fatalf("LIVE LLM CONNECTION ERROR: base=%s model=%s elapsed=%s err=%s",
|
||||
base, model, probeElapsed, processing.TruncateError(perr))
|
||||
}
|
||||
resolveInfo["probe_ok"] = true
|
||||
resolveInfo["probe_elapsed"] = probeElapsed.String()
|
||||
resolveInfo["probe_response_len"] = len(comp.Text)
|
||||
writeJSON(out, "resolve.json", resolveInfo)
|
||||
log.Printf("live probe ok elapsed=%s response_len=%d", probeElapsed, len(comp.Text))
|
||||
|
||||
pipeline := processing.NewPipeline(pool)
|
||||
pipeline.BatchSize = cfg.ProcessingBatchSize
|
||||
pipeline.AI = aiSvc
|
||||
pipeline.Prompts = aiprompts.NewService(pool)
|
||||
eprelClient := eprel.NewClient(eprel.Options{Enabled: true, Timeout: 20 * time.Second})
|
||||
pipeline.Engine = &processing.Engine{
|
||||
Completer: nil, // per-job via pipeline.AI
|
||||
Vector: processing.NoopVectorCategorizer{},
|
||||
EPREL: eprelClient,
|
||||
ProviderMode: processing.AIProviderInternal,
|
||||
}
|
||||
|
||||
rawIDs := make([]uuid.UUID, 0, len(eans))
|
||||
for _, ean := range eans {
|
||||
var id uuid.UUID
|
||||
err := pool.QueryRow(ctx, `
|
||||
SELECT id FROM raw_products
|
||||
WHERE company_id = $1 AND gtin = $2`, companyID, ean).Scan(&id)
|
||||
if err != nil {
|
||||
log.Fatalf("raw product %s: %v", ean, err)
|
||||
}
|
||||
rawIDs = append(rawIDs, id)
|
||||
log.Printf("ean=%s raw_id=%s", ean, id)
|
||||
}
|
||||
|
||||
// Force re-enhance: clear prior enhance_input_hash for these raw products.
|
||||
tag, cerr2 := pool.Exec(ctx, `
|
||||
UPDATE processed_products pp
|
||||
SET field_sources = COALESCE(field_sources, '{}'::jsonb) - 'enhance_input_hash',
|
||||
localized = CASE
|
||||
WHEN localized IS NULL OR localized = '{}'::jsonb THEN localized
|
||||
ELSE (
|
||||
SELECT COALESCE(jsonb_object_agg(lang, val - 'enhance_input_hash'), '{}'::jsonb)
|
||||
FROM jsonb_each(localized) AS t(lang, val)
|
||||
)
|
||||
END,
|
||||
updated_at = now()
|
||||
FROM raw_products rp
|
||||
WHERE pp.company_id = $1
|
||||
AND pp.raw_product_id = rp.id
|
||||
AND rp.gtin = ANY($2::text[])`, companyID, eans)
|
||||
if cerr2 != nil {
|
||||
log.Printf("clear enhance hash: %v", cerr2)
|
||||
} else {
|
||||
log.Printf("cleared enhance_input_hash rows=%d", tag.RowsAffected())
|
||||
}
|
||||
|
||||
if n, err := processing.BackfillMissingMeta(ctx, pool, companyID); err != nil {
|
||||
log.Printf("BackfillMissingMeta: %v", err)
|
||||
} else {
|
||||
log.Printf("BackfillMissingMeta updated=%d", n)
|
||||
}
|
||||
|
||||
// Soft-yield competing queue noise (do not mass-fail the whole company queue).
|
||||
_, _ = pool.Exec(ctx, `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'failed', error = 'yielded to live LLM formula probe', updated_at = now()
|
||||
WHERE company_id = $1 AND status IN ('pending','running','processing')`, companyID)
|
||||
|
||||
jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full")
|
||||
if err != nil {
|
||||
log.Fatalf("StartJob: %v", err)
|
||||
}
|
||||
if len(jobs) == 0 {
|
||||
log.Fatal("StartJob returned no jobs")
|
||||
}
|
||||
jobID := jobs[0].ID
|
||||
log.Printf("job_id=%s chunks=%d", jobID, len(jobs))
|
||||
_ = os.WriteFile(filepath.Join(out, "process_id.txt"), []byte(jobID.String()+"\n"), 0o644)
|
||||
|
||||
defend := make(chan struct{})
|
||||
go func() {
|
||||
t := time.NewTicker(500 * time.Millisecond)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-defend:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'failed', error = 'yielded to live LLM formula probe', updated_at = now()
|
||||
WHERE company_id = $1 AND status IN ('pending','running','processing') AND id <> $2`, companyID, jobID)
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'running', error = NULL, updated_at = now()
|
||||
WHERE id = $1 AND status IN ('failed','cancelled')
|
||||
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
|
||||
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
|
||||
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%'
|
||||
OR error ILIKE '%formula%' OR error ILIKE '%live LLM%')`, jobID)
|
||||
_, _ = pool.Exec(context.Background(), `
|
||||
UPDATE processing_job_products
|
||||
SET status = 'pending', error = NULL, updated_at = now()
|
||||
WHERE job_id = $1 AND status IN ('failed','cancelled')
|
||||
AND processed_product_id IS NULL
|
||||
AND (error ILIKE '%P0%' OR error ILIKE '%parked%' OR error ILIKE '%yield%'
|
||||
OR error ILIKE '%reclaim%' OR error ILIKE '%cleared%' OR error ILIKE '%superseded%'
|
||||
OR error ILIKE '%exclusive%' OR error ILIKE '%batch_%' OR error ILIKE '%inproc%'
|
||||
OR error ILIKE '%formula%' OR error ILIKE '%live LLM%')`, jobID)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
_, _ = pool.Exec(ctx, `
|
||||
UPDATE processing_jobs
|
||||
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
|
||||
WHERE id = $1`, jobID)
|
||||
|
||||
runCtx, runCancel := context.WithTimeout(ctx, 20*time.Minute)
|
||||
defer runCancel()
|
||||
if err := pipeline.ProcessJob(runCtx, jobID); err != nil {
|
||||
close(defend)
|
||||
log.Fatalf("ProcessJob: %v", err)
|
||||
}
|
||||
close(defend)
|
||||
|
||||
items, err := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
|
||||
if err != nil {
|
||||
log.Fatalf("LoadV1ProcessJobItems: %v", err)
|
||||
}
|
||||
payload := map[string]any{
|
||||
"data": map[string]any{
|
||||
"process_id": jobID.String(),
|
||||
"status": "COMPLETED",
|
||||
"processing_type": "full",
|
||||
"total_items": len(items),
|
||||
"items": items,
|
||||
"llm": map[string]any{
|
||||
"base": base,
|
||||
"model": model,
|
||||
"source": roleCfg.Source,
|
||||
"mode": modeLabel,
|
||||
"byok": byok,
|
||||
},
|
||||
},
|
||||
}
|
||||
writeJSON(out, "final.json", payload)
|
||||
|
||||
// DB HTML descriptions (V1 may strip tags).
|
||||
rows, err := pool.Query(ctx, `
|
||||
SELECT rp.gtin,
|
||||
COALESCE(pp.processed_name, pp.name, ''),
|
||||
COALESCE(NULLIF(pp.processed_description, ''), pp.description, ''),
|
||||
COALESCE(pp.category, ''),
|
||||
COALESCE(c.name, ''),
|
||||
COALESCE(c.unique_id, ''),
|
||||
c.description_template,
|
||||
pp.id
|
||||
FROM processing_job_products pjp
|
||||
JOIN raw_products rp ON rp.id = pjp.raw_product_id
|
||||
LEFT JOIN processed_products pp ON pp.id = pjp.processed_product_id
|
||||
LEFT JOIN categories c ON c.company_id = pp.company_id AND c.unique_id = pp.category
|
||||
WHERE pjp.job_id = $1
|
||||
ORDER BY rp.gtin`, jobID)
|
||||
if err != nil {
|
||||
log.Fatalf("db products: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
dbProducts := make([]map[string]any, 0)
|
||||
for rows.Next() {
|
||||
var gtin, name, desc, cat, catName, catUID string
|
||||
var tmpl any
|
||||
var ppID *uuid.UUID
|
||||
if err := rows.Scan(>in, &name, &desc, &cat, &catName, &catUID, &tmpl, &ppID); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
idStr := ""
|
||||
if ppID != nil {
|
||||
idStr = ppID.String()
|
||||
}
|
||||
dbProducts = append(dbProducts, map[string]any{
|
||||
"ean": gtin,
|
||||
"product_id": idStr,
|
||||
"title": name,
|
||||
"description": desc,
|
||||
"category": cat,
|
||||
"category_name": catName,
|
||||
"category_unique_id": catUID,
|
||||
"description_template": tmpl,
|
||||
"formula_constraint": processing.FormatDescriptionFormulaConstraint(tmpl),
|
||||
})
|
||||
}
|
||||
writeJSON(out, "db_products.json", dbProducts)
|
||||
|
||||
fmt.Printf("JOB %s items=%d live_base=%s model=%s\n", jobID, len(items), base, model)
|
||||
for _, p := range dbProducts {
|
||||
ean, _ := p["ean"].(string)
|
||||
title, _ := p["title"].(string)
|
||||
cat, _ := p["category"].(string)
|
||||
catn, _ := p["category_name"].(string)
|
||||
desc, _ := p["description"].(string)
|
||||
if len(title) > 50 {
|
||||
title = title[:50]
|
||||
}
|
||||
fmt.Printf("ITEM %s cat=%s/%s desc_len=%d title=%s\n", ean, cat, catn, len(desc), title)
|
||||
}
|
||||
}
|
||||
|
||||
func failResolve(out, msg string) {
|
||||
writeJSON(out, "resolve.json", map[string]any{
|
||||
"live": false,
|
||||
"fail_reason": msg,
|
||||
})
|
||||
log.Fatalf("LIVE LLM REQUIRED: %s", msg)
|
||||
}
|
||||
|
||||
func writeJSON(out, name string, v any) {
|
||||
raw, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("marshal %s: %v", name, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(out, name), raw, 0o644); err != nil {
|
||||
log.Fatalf("write %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user