410 lines
16 KiB
Go
410 lines
16 KiB
Go
// Temporary slim LIVE verify — 2 products, short timeouts. Do not commit.
|
||
package main
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"log"
|
||
"net/http"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprompts"
|
||
"github.com/descrybe/descrybe-v2/apps/api/internal/aiprovider"
|
||
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||
"github.com/descrybe/descrybe-v2/apps/api/internal/db"
|
||
"github.com/descrybe/descrybe-v2/apps/api/internal/eprel"
|
||
"github.com/descrybe/descrybe-v2/apps/api/internal/logredact"
|
||
"github.com/descrybe/descrybe-v2/apps/api/internal/platformsettings"
|
||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||
"github.com/google/uuid"
|
||
)
|
||
|
||
func main() {
|
||
out := `F:\laragon\www\_MY\descrybe-v2\.codehelper\_final_verify_20260817`
|
||
_ = os.MkdirAll(out, 0o755)
|
||
logPath := filepath.Join(out, "inproc_run_quick.log")
|
||
logFile, err := os.Create(logPath)
|
||
if err != nil {
|
||
log.Fatalf("log: %v", err)
|
||
}
|
||
defer logFile.Close()
|
||
log.SetOutput(logredact.Writer(io.MultiWriter(os.Stderr, logFile)))
|
||
|
||
repoRoot := `F:\laragon\www\_MY\descrybe-v2`
|
||
wpPath := filepath.Join(repoRoot, "scripts", "seed", "wp_product_categories.sql")
|
||
wpBytes, err := os.ReadFile(wpPath)
|
||
if err != nil {
|
||
failReport(out, "FAIL", "repo seed missing: "+err.Error(), nil)
|
||
}
|
||
_ = os.Setenv("SEED_A1_WP_CATEGORIES", wpPath)
|
||
|
||
eans := []string{
|
||
"4548736132597", // Slušalke
|
||
"195348253666", // Gaming monitorji
|
||
}
|
||
companyID := uuid.MustParse("604f23a8-b66e-4b21-8b45-0d72b68f4790")
|
||
userID := uuid.MustParse("6bf00877-a693-4d77-b28e-8c8292adac98")
|
||
|
||
cfg, err := config.Load()
|
||
if err != nil {
|
||
failReport(out, "FAIL", "config: "+err.Error(), nil)
|
||
}
|
||
ctx := context.Background()
|
||
pool, err := db.NewPool(ctx, cfg.DatabaseURL, db.PoolOptions{
|
||
MaxConns: int32(cfg.DBMaxConns), MinConns: int32(cfg.DBMinConns),
|
||
MaxConnLifetime: cfg.DBMaxConnLifetime, MaxConnLifetimeJitter: cfg.DBMaxConnLifetimeJitter,
|
||
MaxConnIdleTime: cfg.DBMaxConnIdleTime, HealthCheckPeriod: cfg.DBHealthCheckPeriod,
|
||
StatementTimeout: cfg.DBStatementTimeout,
|
||
})
|
||
if err != nil {
|
||
failReport(out, "FAIL", "db: "+err.Error(), nil)
|
||
}
|
||
defer pool.Close()
|
||
|
||
// --- Sync A1 from repo seed only ---
|
||
var legacy string
|
||
_ = pool.QueryRow(ctx, `SELECT COALESCE(legacy_company_id,'') FROM companies WHERE id=$1`, companyID).Scan(&legacy)
|
||
a1Cohort := billing.IsA1CohortCompany(legacy, "A1 Slovenija")
|
||
syncRes, serr := processing.SyncCompanyA1(ctx, pool, companyID, "A1 Slovenija", a1Cohort, processing.SyncCompanyA1Opts{
|
||
SkipDumpBackfill: true,
|
||
FixCompanyCatalogOpts: processing.FixCompanyCatalogOpts{
|
||
BackfillCategories: true,
|
||
WPCategoriesSQL: wpBytes,
|
||
AIPrompts: aiprompts.NewService(pool),
|
||
},
|
||
})
|
||
syncInfo := map[string]any{"wp_path": wpPath, "wp_bytes": len(wpBytes), "error": nil, "result": syncRes}
|
||
if serr != nil {
|
||
syncInfo["error"] = serr.Error()
|
||
writeJSON(out, "sync_quick.json", syncInfo)
|
||
failReport(out, "FAIL", "SyncCompanyA1: "+serr.Error(), syncInfo)
|
||
}
|
||
writeJSON(out, "sync_quick.json", syncInfo)
|
||
log.Printf("sync ok prompts_already_ok=%d wp_entries=%d", syncRes.CategoryPromptsAlreadyOK, syncRes.WPCategoriesEntries)
|
||
|
||
platEnv := platformsettings.EnvConfig{
|
||
AppEncryptionKey: cfg.AppEncryptionKey, CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||
TokenSigningSecret: cfg.TokenSigningSecret, DatabaseURL: cfg.DatabaseURL,
|
||
OpenAIAPIKey: cfg.OpenAIAPIKey, OpenAIBaseURL: cfg.OpenAIBaseURL, OpenAIModel: cfg.OpenAIModel,
|
||
}
|
||
plat := platformsettings.NewService(pool, platEnv)
|
||
aiSvc := aiprovider.NewService(pool, aiprovider.EnvConfig{
|
||
AppEncryptionKey: cfg.AppEncryptionKey, CredentialsEncryptionKey: cfg.CredentialsEncryptionKey,
|
||
TokenSigningSecret: cfg.TokenSigningSecret, DatabaseURL: cfg.DatabaseURL,
|
||
OpenAIAPIKey: cfg.OpenAIAPIKey, OpenAIBaseURL: cfg.OpenAIBaseURL, OpenAIModel: cfg.OpenAIModel,
|
||
ProcessingRPM: cfg.ProcessingRPM, ProcessingMaxRetries: cfg.ProcessingMaxRetries,
|
||
})
|
||
aiSvc.Platform = plat
|
||
|
||
roleCfg, rerr := plat.ResolveAIConfig(ctx, platformsettings.AIRoleProcessing)
|
||
if rerr != nil {
|
||
failReport(out, "FAIL", "ResolveAIConfig: "+rerr.Error(), nil)
|
||
}
|
||
completer, modeLabel, byok, cerr := aiSvc.ResolveCompleterForRole(ctx, companyID, aiprovider.RoleProcessing)
|
||
if cerr != nil {
|
||
failReport(out, "FAIL", "ResolveCompleter: "+cerr.Error(), nil)
|
||
}
|
||
client, ok := completer.(*processing.OpenAIClient)
|
||
if !ok || client == nil {
|
||
failReport(out, "FAIL", fmt.Sprintf("completer type %T", completer), nil)
|
||
}
|
||
base := strings.TrimSpace(client.BaseURL)
|
||
model := strings.TrimSpace(client.Model)
|
||
resolve := map[string]any{
|
||
"role_source": roleCfg.Source, "role_provider": roleCfg.Provider,
|
||
"completer_base": base, "completer_model": model,
|
||
"mode": modeLabel, "byok": byok, "env_ignored": cfg.OpenAIBaseURL,
|
||
}
|
||
if processing.IsMockOrLoopbackBaseURL(base) || strings.Contains(base, "18767") {
|
||
resolve["fail_reason"] = "mock/loopback refused"
|
||
writeJSON(out, "resolve_quick.json", resolve)
|
||
failReport(out, "FAIL", "mock/loopback refused — live OverloadedBot only", resolve)
|
||
}
|
||
|
||
// --- /models probe (hard stop on 502; 15s) ---
|
||
modelsURL := strings.TrimRight(base, "/") + "/models"
|
||
modelsCtx, modelsCancel := context.WithTimeout(ctx, 15*time.Second)
|
||
defer modelsCancel()
|
||
req, _ := http.NewRequestWithContext(modelsCtx, http.MethodGet, modelsURL, nil)
|
||
req.Header.Set("Authorization", "Bearer "+client.APIKey)
|
||
req.Header.Set("Accept", "application/json")
|
||
t0 := time.Now()
|
||
resp, merr := http.DefaultClient.Do(req)
|
||
elapsed := time.Since(t0).Round(time.Millisecond)
|
||
if merr != nil {
|
||
resolve["models_error"] = merr.Error()
|
||
writeJSON(out, "resolve_quick.json", resolve)
|
||
failReport(out, "FAIL", "GET /models network: "+merr.Error(), resolve)
|
||
}
|
||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||
_ = resp.Body.Close()
|
||
resolve["models_http_status"] = resp.StatusCode
|
||
resolve["models_elapsed"] = elapsed.String()
|
||
resolve["models_snippet"] = trunc(string(body), 180)
|
||
if resp.StatusCode == http.StatusBadGateway {
|
||
writeJSON(out, "resolve_quick.json", resolve)
|
||
failReport(out, "FAIL", "FAIL 502: GET /v1/models — OverloadedBot unavailable (no mock, no wait)", resolve)
|
||
}
|
||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusUnauthorized {
|
||
writeJSON(out, "resolve_quick.json", resolve)
|
||
failReport(out, "FAIL", fmt.Sprintf("GET /models HTTP %d", resp.StatusCode), resolve)
|
||
}
|
||
// Prefer advertised green/code-fast if DB model is bare code-fast (gateway list).
|
||
if model == "code-fast" && strings.Contains(string(body), `"green/code-fast"`) {
|
||
client.Model = "green/code-fast"
|
||
model = client.Model
|
||
resolve["completer_model_adjusted"] = model
|
||
log.Printf("adjusted model code-fast -> green/code-fast from /models list")
|
||
}
|
||
// Cap per-call wait (~2.5m) and retries so we never sit on 4m×N.
|
||
if client.HTTPClient != nil {
|
||
client.HTTPClient.Timeout = 150 * time.Second
|
||
}
|
||
client.MaxRetries = 1
|
||
resolve["client_http_timeout"] = "150s"
|
||
resolve["client_max_retries"] = 1
|
||
writeJSON(out, "resolve_quick.json", resolve)
|
||
log.Printf("LIVE OK models=%d base=%s model=%s", resp.StatusCode, base, model)
|
||
|
||
pipeline := processing.NewPipeline(pool)
|
||
pipeline.BatchSize = cfg.ProcessingBatchSize
|
||
pipeline.AI = aiSvc
|
||
pipeline.Prompts = aiprompts.NewService(pool)
|
||
pipeline.Engine = &processing.Engine{
|
||
Completer: nil,
|
||
Vector: processing.NoopVectorCategorizer{},
|
||
EPREL: eprel.NewClient(eprel.Options{Enabled: true, Timeout: 15 * time.Second}),
|
||
ProviderMode: processing.AIProviderInternal,
|
||
}
|
||
|
||
rawIDs := make([]uuid.UUID, 0, len(eans))
|
||
for _, ean := range eans {
|
||
var id uuid.UUID
|
||
if err := pool.QueryRow(ctx, `SELECT id FROM raw_products WHERE company_id=$1 AND gtin=$2`, companyID, ean).Scan(&id); err != nil {
|
||
failReport(out, "FAIL", fmt.Sprintf("raw %s: %v", ean, err), resolve)
|
||
}
|
||
rawIDs = append(rawIDs, id)
|
||
}
|
||
_, _ = pool.Exec(ctx, `
|
||
UPDATE processed_products pp
|
||
SET field_sources = COALESCE(field_sources, '{}'::jsonb) - 'enhance_input_hash', updated_at=now()
|
||
FROM raw_products rp
|
||
WHERE pp.company_id=$1 AND pp.raw_product_id=rp.id AND rp.gtin=ANY($2::text[])`, companyID, eans)
|
||
_, _ = pool.Exec(ctx, `
|
||
UPDATE processing_jobs SET status='failed', error='yielded to quick final verify', updated_at=now()
|
||
WHERE company_id=$1 AND status IN ('pending','running','processing')`, companyID)
|
||
|
||
jobs, err := pipeline.StartJob(ctx, companyID, userID, rawIDs, "full")
|
||
if err != nil || len(jobs) == 0 {
|
||
failReport(out, "FAIL", fmt.Sprintf("StartJob: %v", err), resolve)
|
||
}
|
||
jobID := jobs[0].ID
|
||
var dbStatus string
|
||
var processedCount int
|
||
_ = pool.QueryRow(ctx, `SELECT lower(status), COALESCE(processed_count,0) FROM processing_jobs WHERE id=$1`, jobID).Scan(&dbStatus, &processedCount)
|
||
startOK := (dbStatus == "pending" || dbStatus == "queued") && processedCount == 0
|
||
startSample := map[string]any{
|
||
"process_id": jobID.String(), "status": dbStatus, "processed_items": processedCount,
|
||
"total_items": len(rawIDs), "start_ok": startOK,
|
||
}
|
||
writeJSON(out, "start_sample.json", startSample)
|
||
|
||
_, _ = pool.Exec(ctx, `UPDATE processing_jobs SET status='running', started_at=COALESCE(started_at,now()), updated_at=now() WHERE id=$1`, jobID)
|
||
|
||
// Hard wall: 2 products × ~2.5–3 min ≈ 8 min max
|
||
runCtx, runCancel := context.WithTimeout(ctx, 8*time.Minute)
|
||
defer runCancel()
|
||
procErr := pipeline.ProcessJob(runCtx, jobID)
|
||
logBytes, _ := os.ReadFile(logPath)
|
||
logText := string(logBytes)
|
||
aiEnhance := strings.Contains(logText, "ai_enhance")
|
||
|
||
items, loadErr := pipeline.LoadV1ProcessJobItems(ctx, companyID, jobID, "full")
|
||
if loadErr != nil {
|
||
items = nil
|
||
}
|
||
cards := []map[string]any{}
|
||
allPass := startOK && aiEnhance && procErr == nil
|
||
for _, item := range items {
|
||
ean := str(item["ean"])
|
||
if ean == "" {
|
||
ean = str(item["product_id"])
|
||
}
|
||
title := first(str(item["name"]), str(item["title"]))
|
||
desc := str(item["description"])
|
||
catName := str(item["category_name"])
|
||
if catName == "" {
|
||
if m, ok := item["category"].(map[string]any); ok {
|
||
catName = str(m["name"])
|
||
}
|
||
}
|
||
_, hasMetaT := item["meta_title"]
|
||
_, hasMetaD := item["meta_description"]
|
||
_, hasID := item["id"]
|
||
_, hasPP := item["processed_product_id"]
|
||
_, hasRaw := item["raw_product_id"]
|
||
hasHTML := strings.Contains(strings.ToLower(desc), "<h2") || strings.Contains(strings.ToLower(desc), "<p>")
|
||
checks := map[string]bool{
|
||
"name_present": strings.TrimSpace(title) != "",
|
||
"category_name_present": strings.TrimSpace(catName) != "",
|
||
"no_meta_title": !hasMetaT || item["meta_title"] == nil,
|
||
"no_meta_description": !hasMetaD || item["meta_description"] == nil,
|
||
"no_id": !hasID,
|
||
"no_processed_product_id": !hasPP,
|
||
"no_raw_product_id": !hasRaw,
|
||
"formula_html_or_empty": hasHTML || strings.TrimSpace(desc) == "" || strings.Contains(strings.ToLower(logText), "empty_or_provider_error") || strings.Contains(strings.ToLower(logText), "refuse"),
|
||
}
|
||
pass := true
|
||
for _, v := range checks {
|
||
if !v {
|
||
pass = false
|
||
break
|
||
}
|
||
}
|
||
if !pass {
|
||
allPass = false
|
||
}
|
||
cards = append(cards, map[string]any{
|
||
"ean": ean, "title": title, "category": catName, "desc_len": len(desc),
|
||
"has_html": hasHTML, "checks": checks, "pass": pass,
|
||
})
|
||
}
|
||
if procErr != nil {
|
||
allPass = false
|
||
}
|
||
overall := "PASS"
|
||
reason := ""
|
||
if !allPass {
|
||
overall = "FAIL"
|
||
if procErr != nil {
|
||
reason = "ProcessJob: " + processing.TruncateError(procErr)
|
||
} else if !aiEnhance {
|
||
reason = "no ai_enhance log lines"
|
||
} else if !startOK {
|
||
reason = "start semantics not pending/0"
|
||
} else {
|
||
reason = "scorecard assertion failed"
|
||
}
|
||
}
|
||
|
||
report := map[string]any{
|
||
"generated_at": time.Now().UTC().Format(time.RFC3339),
|
||
"overall": overall,
|
||
"blocking_reason": reason,
|
||
"sync_pass": true,
|
||
"live_llm_pass": true,
|
||
"process_run": true,
|
||
"job_id": jobID.String(),
|
||
"eans": eans,
|
||
"start_ok": startOK,
|
||
"ai_enhance_seen": aiEnhance,
|
||
"process_error": nil,
|
||
"scorecards": cards,
|
||
"llm": resolve,
|
||
"sync": syncInfo,
|
||
"note": "slim verify: 2 products, 150s/call, 8m wall; repo seed only",
|
||
}
|
||
if procErr != nil {
|
||
report["process_error"] = processing.TruncateError(procErr)
|
||
}
|
||
writeJSON(out, "report.json", report)
|
||
writeMD(out, report, startSample, cards, logText)
|
||
fmt.Printf("OVERALL %s job=%s ai_enhance=%v err=%v\n", overall, jobID, aiEnhance, procErr)
|
||
if overall != "PASS" {
|
||
os.Exit(1)
|
||
}
|
||
}
|
||
|
||
func failReport(out, overall, reason string, extra map[string]any) {
|
||
report := map[string]any{
|
||
"generated_at": time.Now().UTC().Format(time.RFC3339),
|
||
"overall": overall, "blocking_reason": reason,
|
||
"process_run": false, "extra": extra,
|
||
}
|
||
writeJSON(out, "report.json", report)
|
||
var b strings.Builder
|
||
b.WriteString("# Final verify 2026-08-17 (quick LIVE)\n\n")
|
||
b.WriteString(fmt.Sprintf("**Overall: %s**\n\n", overall))
|
||
b.WriteString(fmt.Sprintf("Blocking: %s\n\n", reason))
|
||
b.WriteString("Live OverloadedBot only. No mock.\n")
|
||
_ = os.WriteFile(filepath.Join(out, "REPORT.md"), []byte(b.String()), 0o644)
|
||
log.Print(reason)
|
||
os.Exit(1)
|
||
}
|
||
|
||
func writeMD(out string, report map[string]any, start map[string]any, cards []map[string]any, logText string) {
|
||
var b strings.Builder
|
||
b.WriteString("# Final verify 2026-08-17 (quick LIVE)\n\n")
|
||
b.WriteString(fmt.Sprintf("**Overall: %v**\n\n", report["overall"]))
|
||
if r := str(report["blocking_reason"]); r != "" {
|
||
b.WriteString(fmt.Sprintf("Blocking: %s\n\n", r))
|
||
}
|
||
b.WriteString("## Progress note\n\n")
|
||
b.WriteString("Prior hung run killed (4m chat waits on 6 EANs). This pass: live OverloadedBot only, repo seed sync, 2 products, 150s/call, 8m wall.\n\n")
|
||
b.WriteString("## 1. Sync A1 (repo seed only)\n\n")
|
||
b.WriteString("- Source: `scripts/seed/wp_product_categories.sql` (no upload, SkipDumpBackfill)\n")
|
||
b.WriteString("- See `sync_quick.json`\n\n")
|
||
b.WriteString("## 2. Live LLM\n\n")
|
||
if llm, ok := report["llm"].(map[string]any); ok {
|
||
b.WriteString(fmt.Sprintf("- base: `%v`\n- model: `%v`\n- models HTTP: `%v`\n- source: `%v`\n\n",
|
||
llm["completer_base"], llm["completer_model"], llm["models_http_status"], llm["role_source"]))
|
||
}
|
||
b.WriteString("## 3. Start semantics\n\n")
|
||
b.WriteString(fmt.Sprintf("- process_id: `%v`\n- status: `%v`\n- processed_items: `%v`\n- start_ok: `%v`\n\n",
|
||
start["process_id"], start["status"], start["processed_items"], start["start_ok"]))
|
||
b.WriteString(fmt.Sprintf("## 4. ai_enhance\n\n- seen: `%v`\n\n", report["ai_enhance_seen"]))
|
||
b.WriteString("## 5. Scorecards\n\n")
|
||
for _, c := range cards {
|
||
b.WriteString(fmt.Sprintf("- **%v** pass=%v cat=%v html=%v title=%q\n",
|
||
c["ean"], c["pass"], c["category"], c["has_html"], trunc(str(c["title"]), 60)))
|
||
}
|
||
b.WriteString("\n## Log excerpt (ai_enhance)\n\n```\n")
|
||
n := 0
|
||
for _, line := range strings.Split(logText, "\n") {
|
||
if strings.Contains(line, "ai_enhance") || strings.Contains(line, "LIVE") || strings.Contains(line, "OVERALL") {
|
||
b.WriteString(trunc(line, 240) + "\n")
|
||
n++
|
||
if n >= 12 {
|
||
break
|
||
}
|
||
}
|
||
}
|
||
b.WriteString("```\n")
|
||
_ = os.WriteFile(filepath.Join(out, "REPORT.md"), []byte(b.String()), 0o644)
|
||
}
|
||
|
||
func writeJSON(out, name string, v any) {
|
||
raw, _ := json.MarshalIndent(v, "", " ")
|
||
_ = os.WriteFile(filepath.Join(out, name), raw, 0o644)
|
||
}
|
||
func str(v any) string {
|
||
if v == nil {
|
||
return ""
|
||
}
|
||
if s, ok := v.(string); ok {
|
||
return s
|
||
}
|
||
return strings.TrimSpace(fmt.Sprint(v))
|
||
}
|
||
func first(vals ...string) string {
|
||
for _, v := range vals {
|
||
if strings.TrimSpace(v) != "" {
|
||
return v
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
func trunc(s string, n int) string {
|
||
s = strings.TrimSpace(s)
|
||
if len(s) <= n {
|
||
return s
|
||
}
|
||
return s[:n] + "…"
|
||
}
|