greeneclipse/descrybe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
greeneclipse
2026-08-16 18:36:56 +02:00
parent d981147ff2
commit a94a5aaad7
26 changed files with 904 additions and 507 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/cmd/seed-a1/category_backfill.go
+19 -196
View File
@@ -1,26 +1,15 @@
package main
import (
"bufio"
"context"
"fmt"
"io"
"log"
"os"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// Classic A1 Postman / Elkotex fixture EANs — must never live on Platform Demo.
var a1FixtureEANs = []string{
"5905575903198",
"6970995789942",
}
type categoryBackfillResult struct {
ProcessedUpdated int64
ProcessedInserted int64
@@ -34,204 +23,38 @@ type categoryBackfillResult struct {
MappedWithoutCat int
}
// backfillMappedCategoriesFromProcessed copies processed_products.category into
// raw_products.mapped_data.category for A1 only. Delegates to processing.
// a1FixtureEANs aliases processing.A1FixtureEANs for seed-a1 tests.
var a1FixtureEANs = processing.A1FixtureEANs
func backfillMappedCategoriesFromProcessed(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID) (int64, error) {
return processing.BackfillMappedCategoriesFromProcessed(ctx, pg, companyID)
}
// backfillCategoriesFromMySQLDump streams dump processed_products for the A1
// legacy company and writes product_id (GTIN) → category onto A1 Postgres
// mapped_data.category (and updates any existing processed_products.category).
// It does not insert processed rows — A1 demo seed stays at processed=0.
func backfillCategoriesFromMySQLDump(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID, dumpPath string) (categoryBackfillResult, error) {
var out categoryBackfillResult
legacyCompany := billing.A1LegacyCompanyID
var legacy string
_ = pg.QueryRow(ctx, `SELECT COALESCE(legacy_company_id::text, '') FROM companies WHERE id = $1`, companyID).Scan(&legacy)
if legacy != "" {
legacyCompany = legacy
}
f, err := os.Open(dumpPath)
res, err := processing.BackfillCategoriesFromMySQLDump(ctx, pg, companyID, dumpPath)
if err != nil {
return out, fmt.Errorf("open mysql dump: %w", err)
return categoryBackfillResult{}, err
}
defer f.Close()
byGTIN, err := scanA1ProcessedCategories(f, legacyCompany)
if err != nil {
return out, err
}
out.DumpPairs = len(byGTIN)
if len(byGTIN) == 0 {
return out, fmt.Errorf("no A1 processed_products categories for legacy %s in dump", legacyCompany)
}
log.Printf("dump: %d A1 gtin→category pairs", len(byGTIN))
gtins := make([]string, 0, len(byGTIN))
cats := make([]string, 0, len(byGTIN))
for g, c := range byGTIN {
gtins = append(gtins, g)
cats = append(cats, c)
}
ct, err := pg.Exec(ctx, `
UPDATE processed_products p
SET category = v.category,
updated_at = now()
FROM unnest($2::text[], $3::text[]) AS v(gtin, category)
WHERE p.company_id = $1
AND p.product_id = v.gtin
AND COALESCE(NULLIF(trim(v.category), ''), '') <> ''
AND (
COALESCE(NULLIF(trim(p.category), ''), '') = ''
OR lower(trim(p.category)) = 'none'
OR p.category IS DISTINCT FROM v.category
)`, companyID, gtins, cats)
if err != nil {
return out, fmt.Errorf("update processed category from dump: %w", err)
}
out.ProcessedUpdated = ct.RowsAffected()
// A1 demo seed keeps processed=0. Do not INSERT processed rows from the dump —
// only refresh mapped_data.category (and any existing processed rows if present).
ct, err = pg.Exec(ctx, `
UPDATE raw_products r
SET mapped_data = jsonb_set(
COALESCE(r.mapped_data, '{}'::jsonb),
'{category}',
to_jsonb(v.category),
true
),
updated_at = now()
FROM unnest($2::text[], $3::text[]) AS v(gtin, category)
WHERE r.company_id = $1
AND r.gtin = v.gtin
AND COALESCE(NULLIF(trim(v.category), ''), '') <> ''
AND (
COALESCE(NULLIF(trim(r.mapped_data->>'category'), ''), '') = ''
OR r.mapped_data->>'category' IS DISTINCT FROM v.category
)`, companyID, gtins, cats)
if err != nil {
return out, fmt.Errorf("update mapped category from dump: %w", err)
}
out.MappedUpdated = ct.RowsAffected()
n, err := backfillMappedCategoriesFromProcessed(ctx, pg, companyID)
if err != nil {
return out, err
}
out.MappedUpdated += n
if err := fillCategoryCoverage(ctx, pg, companyID, &out); err != nil {
return out, err
}
return out, nil
return categoryBackfillResult{
ProcessedUpdated: res.ProcessedUpdated,
ProcessedInserted: res.ProcessedInserted,
MappedUpdated: res.MappedUpdated,
DumpPairs: res.DumpPairs,
PurgedOtherRaw: res.PurgedOtherRaw,
PurgedOtherPP: res.PurgedOtherPP,
ProcessedWithCat: res.ProcessedWithCat,
ProcessedWithoutCat: res.ProcessedWithoutCat,
MappedWithCat: res.MappedWithCat,
MappedWithoutCat: res.MappedWithoutCat,
}, nil
}
func fillCategoryCoverage(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID, out *categoryBackfillResult) error {
err := pg.QueryRow(ctx, `
SELECT
COUNT(*) FILTER (
WHERE COALESCE(NULLIF(trim(category), ''), '') <> ''
AND lower(trim(category)) <> 'none'
),
COUNT(*) FILTER (
WHERE COALESCE(NULLIF(trim(category), ''), '') = ''
OR lower(trim(category)) = 'none'
)
FROM processed_products
WHERE company_id = $1`, companyID).Scan(&out.ProcessedWithCat, &out.ProcessedWithoutCat)
if err != nil {
return fmt.Errorf("count processed categories: %w", err)
}
err = pg.QueryRow(ctx, `
SELECT
COUNT(*) FILTER (
WHERE COALESCE(NULLIF(trim(mapped_data->>'category'), ''), '') <> ''
),
COUNT(*) FILTER (
WHERE COALESCE(NULLIF(trim(mapped_data->>'category'), ''), '') = ''
)
FROM raw_products
WHERE company_id = $1`, companyID).Scan(&out.MappedWithCat, &out.MappedWithoutCat)
if err != nil {
return fmt.Errorf("count mapped categories: %w", err)
}
return nil
}
// purgeA1FixtureEANsFromOtherTenants deletes the Postman Elkotex fixture EANs
// from every company except A1 (Platform Demo must not mirror A1 fixtures).
func purgeA1FixtureEANsFromOtherTenants(ctx context.Context, pg *pgxpool.Pool, a1CompanyID uuid.UUID) (rawN, ppN int64, err error) {
ct, err := pg.Exec(ctx, `
DELETE FROM processing_job_products pjp
WHERE pjp.raw_product_id IN (
SELECT id FROM raw_products
WHERE company_id <> $1 AND gtin = ANY($2::text[])
)
OR pjp.processed_product_id IN (
SELECT id FROM processed_products
WHERE company_id <> $1 AND product_id = ANY($2::text[])
)`, a1CompanyID, a1FixtureEANs)
if err != nil {
return 0, 0, fmt.Errorf("purge fixture job products: %w", err)
}
_ = ct
ct, err = pg.Exec(ctx, `
DELETE FROM processed_products
WHERE company_id <> $1 AND product_id = ANY($2::text[])`, a1CompanyID, a1FixtureEANs)
if err != nil {
return 0, 0, fmt.Errorf("purge fixture processed: %w", err)
}
ppN = ct.RowsAffected()
ct, err = pg.Exec(ctx, `
DELETE FROM raw_products
WHERE company_id <> $1 AND gtin = ANY($2::text[])`, a1CompanyID, a1FixtureEANs)
if err != nil {
return 0, 0, fmt.Errorf("purge fixture raw: %w", err)
}
rawN = ct.RowsAffected()
return rawN, ppN, nil
return processing.PurgeA1FixtureEANsFromOtherTenants(ctx, pg, a1CompanyID)
}
func scanA1ProcessedCategories(r io.Reader, legacyCompany string) (map[string]string, error) {
br := bufio.NewReaderSize(r, 1<<20)
inTable := false
out := make(map[string]string, 4096)
for {
line, err := br.ReadString('\n')
if len(line) > 0 {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "INSERT INTO `processed_products`") ||
strings.HasPrefix(trimmed, "INSERT INTO processed_products") {
inTable = true
} else if inTable && strings.HasPrefix(trimmed, "CREATE TABLE") {
break
} else if inTable && strings.HasPrefix(trimmed, "INSERT INTO `") &&
!strings.Contains(trimmed, "processed_products") {
break
} else if inTable && looksLikeTupleLine(line) && strings.Contains(line, legacyCompany) {
fields := parseMySQLTupleFieldsN(line, 6)
if len(fields) >= 5 {
gtin := strings.TrimSpace(fields[2])
cat := strings.TrimSpace(fields[4])
if gtin != "" && cat != "" && !strings.EqualFold(cat, "NULL") {
out[gtin] = cat
}
}
}
}
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
}
return out, nil
return processing.ScanA1ProcessedCategories(r, legacyCompany)
}
func logCategoryBackfillResult(res categoryBackfillResult, source string) {
apps/api/cmd/seed-a1/dump_resolve.go
+10 -52
View File
@@ -4,70 +4,28 @@ import (
"context"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// resolveMySQLDumpPath picks an explicit path, else the first readable candidate
// under common local locations documented in scripts/seed/README.txt.
// resolveMySQLDumpPath delegates to processing (shared with admin Sync A1).
func resolveMySQLDumpPath(explicit string) string {
if p := strings.TrimSpace(explicit); p != "" {
if st, err := os.Stat(p); err == nil && !st.IsDir() {
return p
}
log.Printf("warning: mysql dump not found at %q — trying auto-detect", p)
}
for _, c := range mysqlDumpCandidates() {
if strings.TrimSpace(explicit) != "" && filepath.Clean(c) == filepath.Clean(strings.TrimSpace(explicit)) {
continue
}
if st, err := os.Stat(c); err == nil && !st.IsDir() {
return c
}
}
return ""
return processing.ResolveMySQLDumpPath(explicit)
}
func mysqlDumpCandidates() []string {
var out []string
if v := strings.TrimSpace(os.Getenv("SEED_A1_MYSQL_DUMP")); v != "" {
out = append(out, v)
}
home, _ := os.UserHomeDir()
names := []string{
"descrybe_new (1).sql",
"descrybe_new.sql",
"descrybe_new(1).sql",
}
if home != "" {
for _, n := range names {
out = append(out, filepath.Join(home, "Downloads", n))
out = append(out, filepath.Join(home, "downloads", n))
}
}
// Repo-relative guesses (cwd may be apps/api or repo root).
for _, n := range names {
out = append(out,
n,
filepath.Join("..", "..", n),
filepath.Join("scripts", "seed", n),
filepath.Join("..", "..", "scripts", "seed", n),
)
}
return out
return processing.MySQLDumpCandidates()
}
type mappedCoverage struct {
Total int
WithDesc int
WithCat int
WithAttrs int
Processed int
Jobs int
Total int
WithDesc int
WithCat int
WithAttrs int
Processed int
Jobs int
}
func (c mappedCoverage) pct(n int) float64 {
apps/api/cmd/seed-a1/main.go
+6 -2
View File
@@ -41,9 +41,10 @@
// go run ./cmd/seed-a1 -mode backfill-categories -mysql-dump path/to/descrybe_new.sql
// go run ./cmd/seed-a1 -mode backfill-attributes
//
// DATABASE_URL / -postgres required.
// DATABASE_URL / -postgres required (also loaded from monorepo-root .env via config.LoadDotEnv).
// Day-to-day: `npm run seed:a1` (reimport + category prompts + mapped category backfill). Maintainer snapshot: `npm run seed:a1:export`.
// recover-jobs is opt-in only and will reintroduce job history.
// Prefer admin UI Sync A1 on the company row (uses API DATABASE_URL; no CLI env needed).
package main
import (
@@ -61,6 +62,7 @@ import (
"compress/gzip"
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
@@ -82,6 +84,8 @@ type tableSpec struct {
}
func main() {
config.LoadDotEnv()
postgresURL := flag.String("postgres", os.Getenv("DATABASE_URL"), "Postgres URL (DATABASE_URL)")
mode := flag.String("mode", "reimport", "export | reimport | recover-jobs | apply-category-prompts | backfill-categories | backfill-attributes")
file := flag.String("file", "", "gzipped seed archive path (required for export/reimport)")
@@ -93,7 +97,7 @@ func main() {
flag.Parse()
if strings.TrimSpace(*postgresURL) == "" {
log.Fatal("-postgres / DATABASE_URL is required")
log.Fatal("-postgres / DATABASE_URL is required (export it, pass -postgres, or put DATABASE_URL in monorepo-root .env — same as the API; prefer admin Sync A1)")
}
companyID, err := uuid.Parse(strings.TrimSpace(*company))
if err != nil {
apps/api/cmd/seed-a1/recover_jobs.go
+4 -53
View File
@@ -12,6 +12,7 @@ import (
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
@@ -419,65 +420,15 @@ func dumpSectionEnded(line, table string) bool {
}
func looksLikeTupleLine(line string) bool {
s := strings.TrimLeft(line, " \t")
return strings.HasPrefix(s, "(")
return processing.LooksLikeMySQLTupleLine(line)
}
func parseMySQLTupleFields(line string) []string {
return parseMySQLTupleFieldsN(line, 0)
return processing.ParseMySQLTupleFields(line)
}
// parseMySQLTupleFieldsN parses up to maxFields (0 = all) from the first (...) tuple on the line.
func parseMySQLTupleFieldsN(line string, maxFields int) []string {
start := strings.Index(line, "(")
if start < 0 {
return nil
}
body := line[start+1:]
var out []string
for i := 0; i < len(body); {
if maxFields > 0 && len(out) >= maxFields {
break
}
for i < len(body) && (body[i] == ' ' || body[i] == '\t' || body[i] == ',') {
i++
}
if i >= len(body) || body[i] == ')' {
break
}
if body[i] == '\'' {
i++
var b strings.Builder
for i < len(body) {
ch := body[i]
if ch == '\\' && i+1 < len(body) {
b.WriteByte(body[i+1])
i += 2
continue
}
if ch == '\'' {
if i+1 < len(body) && body[i+1] == '\'' {
b.WriteByte('\'')
i += 2
continue
}
i++
break
}
b.WriteByte(ch)
i++
}
out = append(out, b.String())
continue
}
j := i
for j < len(body) && body[j] != ',' && body[j] != ')' {
j++
}
out = append(out, strings.TrimSpace(body[i:j]))
i = j
}
return out
return processing.ParseMySQLTupleFieldsN(line, maxFields)
}
func parseDumpJobID(raw string) (uuid.UUID, error) {
apps/api/internal/httpapi/admin_fix_catalog_handlers.go
+38 -23
View File
@@ -12,21 +12,20 @@ import (
"github.com/google/uuid"
)
// handleAdminFixCompanyCatalog runs in-place Fix A1 / catalog hygiene for one company.
// Single route: POST /api/admin/companies/{id}/fix-catalog (no separate fix-a1).
// Delegates to processing.FixCompanyCatalog, which:
// 1. Ensures category_attributes links (orphan purge only)
// 2. RepairCompanyCategoryEnhancePrompts (same map as RepairA1DemoCategoryEnhancePrompts)
// 3. Re-applies product_enhance BuiltInDefaults when AIPrompts is set
// 4–7. FixCatalogHygieneWithIDs + attribute sanitize + reprocess sample
// handleAdminSyncCompanyA1 runs dump category backfill (when dump is on the API
// host filesystem) plus FixCompanyCatalog hygiene. Does not wipe/reimport.
//
// Routes (same handler):
//
// POST /api/admin/companies/{id}/sync-a1
// POST /api/admin/companies/{id}/fix-catalog (compat alias)
//
// Body: confirm=true required; backfill_categories (default true);
// reprocess_sample_limit (default 25, max 200, 0 = counts only).
// May target A1 in place with confirm — prefer Platform Demo when unsure.
// Refuses system company. Does not use A1 as a clone destination.
// reprocess_sample_limit (default 25, max 200); mysql_dump optional path;
// skip_dump_backfill (default false).
//
// Flash (UI): result.prompts / hashes / categories → flash.admin.fixA1Success.
func (s *Server) handleAdminFixCompanyCatalog(w http.ResponseWriter, r *http.Request) {
// Flash (UI): result → flash.admin.syncA1Success.
func (s *Server) handleAdminSyncCompanyA1(w http.ResponseWriter, r *http.Request) {
if s.Pool == nil {
Error(w, http.StatusServiceUnavailable, "database unavailable")
return
@@ -39,9 +38,11 @@ func (s *Server) handleAdminFixCompanyCatalog(w http.ResponseWriter, r *http.Req
}
var body struct {
Confirm bool `json:"confirm"`
BackfillCategories *bool `json:"backfill_categories"`
ReprocessSampleLimit *int `json:"reprocess_sample_limit"`
Confirm bool `json:"confirm"`
BackfillCategories *bool `json:"backfill_categories"`
ReprocessSampleLimit *int `json:"reprocess_sample_limit"`
MySQLDump string `json:"mysql_dump"`
SkipDumpBackfill bool `json:"skip_dump_backfill"`
}
if err := DecodeJSON(r, &body); err != nil {
Error(w, http.StatusBadRequest, "invalid json")
@@ -53,7 +54,7 @@ func (s *Server) handleAdminFixCompanyCatalog(w http.ResponseWriter, r *http.Req
}
if platformsettings.IsSystemCompany(companyID) {
Error(w, http.StatusBadRequest, "cannot fix the platform settings company")
Error(w, http.StatusBadRequest, "cannot sync the platform settings company")
return
}
@@ -80,26 +81,40 @@ func (s *Server) handleAdminFixCompanyCatalog(w http.ResponseWriter, r *http.Req
sampleLimit = 200
}
result, err := processing.FixCompanyCatalog(
result, err := processing.SyncCompanyA1(
r.Context(),
s.Pool,
companyID,
name,
billing.IsA1CohortCompany(legacy, name),
processing.FixCompanyCatalogOpts{
BackfillCategories: backfill,
ReprocessSampleLimit: sampleLimit,
AIPrompts: s.AIPrompts,
processing.SyncCompanyA1Opts{
FixCompanyCatalogOpts: processing.FixCompanyCatalogOpts{
BackfillCategories: backfill,
ReprocessSampleLimit: sampleLimit,
AIPrompts: s.AIPrompts,
},
MySQLDumpPath: strings.TrimSpace(body.MySQLDump),
SkipDumpBackfill: body.SkipDumpBackfill,
},
)
if err != nil {
ClientOrLog(w, http.StatusBadRequest, "could not fix catalog", err, catalog.ClientError)
ClientOrLog(w, http.StatusBadRequest, "could not sync A1 catalog", err, catalog.ClientError)
return
}
note := "Synced in place (dump categories when available + hygiene); reprocess recommended products manually (no mass reprocess)."
if !result.DumpFound {
note = "Hygiene completed without dump backfill. Place descrybe_new.sql on the API host (scripts/seed/ or SEED_A1_MYSQL_DUMP) for mapped category coverage from the legacy dump."
}
JSON(w, http.StatusOK, map[string]any{
"status": "ok",
"result": result,
"note": "Catalog was repaired in place; reprocess recommended products manually (no mass reprocess).",
"note": note,
})
}
// handleAdminFixCompanyCatalog is a compat alias of Sync A1 (same behavior).
func (s *Server) handleAdminFixCompanyCatalog(w http.ResponseWriter, r *http.Request) {
s.handleAdminSyncCompanyA1(w, r)
}
apps/api/internal/httpapi/admin_fix_catalog_handlers_test.go
+63 -4
View File
@@ -13,6 +13,17 @@ import (
"github.com/google/uuid"
)
func TestHandleAdminSyncCompanyA1NilPool(t *testing.T) {
t.Parallel()
s := &Server{}
req := httptest.NewRequest(http.MethodPost, "/api/admin/companies/"+uuid.NewString()+"/sync-a1", strings.NewReader(`{"confirm":true}`))
rec := httptest.NewRecorder()
s.handleAdminSyncCompanyA1(rec, req)
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("status=%d want 503 body=%s", rec.Code, rec.Body.String())
}
}
func TestHandleAdminFixCompanyCatalogNilPool(t *testing.T) {
t.Parallel()
s := &Server{}
@@ -26,8 +37,6 @@ func TestHandleAdminFixCompanyCatalogNilPool(t *testing.T) {
func TestHandleAdminFixCompanyCatalogRequiresConfirm(t *testing.T) {
t.Parallel()
// Pool nil short-circuits before confirm — use non-nil Pool stub via missing company path is harder.
// Confirm gate is covered when Pool is set; here we only assert invalid uuid.
s := &Server{Pool: nil}
req := httptest.NewRequest(http.MethodPost, "/api/admin/companies/not-a-uuid/fix-catalog", strings.NewReader(`{"confirm":false}`))
rec := httptest.NewRecorder()
@@ -37,8 +46,58 @@ func TestHandleAdminFixCompanyCatalogRequiresConfirm(t *testing.T) {
}
}
// TestRouterAdminFixCatalogMounted locks POST /api/admin/companies/{id}/fix-catalog
// after session + CSRF + platform-admin (503 with nil Pool), not chi 404.
func TestRouterAdminSyncA1Mounted(t *testing.T) {
t.Parallel()
sm := scs.New()
sm.Cookie.Name = "descrybe_session"
uid := uuid.MustParse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")
s := &Server{
Config: config.Config{
CSRFCookieName: "descrybe_csrf",
WebOrigin: "http://localhost:5173",
},
Sessions: sm,
Auth: &auth.Service{},
testPlatformAdmin: func(_ context.Context, got uuid.UUID) (bool, error) {
return got == uid, nil
},
}
var token string
seed := LoadSession(sm)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
sm.Put(r.Context(), auth.SessionUserIDKey, uid.String())
w.WriteHeader(http.StatusNoContent)
}))
seedRec := httptest.NewRecorder()
seed.ServeHTTP(seedRec, httptest.NewRequest(http.MethodGet, "/seed", nil))
for _, c := range seedRec.Result().Cookies() {
if c.Name == sm.Cookie.Name {
token = c.Value
}
}
if token == "" {
t.Fatal("expected session cookie from seed request")
}
h := s.Router()
path := "/api/admin/companies/" + uuid.NewString() + "/sync-a1"
csrf := csrfCookieForSession(t, h, sm, token)
mounted := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{"confirm":true}`))
req.AddCookie(&http.Cookie{Name: sm.Cookie.Name, Value: token})
req.AddCookie(csrf)
req.Header.Set("X-CSRF-Token", csrf.Value)
h.ServeHTTP(mounted, req)
if mounted.Code == http.StatusNotFound {
t.Fatalf("route not mounted: status=404 body=%s", mounted.Body.String())
}
if mounted.Code != http.StatusServiceUnavailable {
t.Fatalf("status=%d want 503 (nil pool) body=%s", mounted.Code, mounted.Body.String())
}
}
// TestRouterAdminFixCatalogMounted locks POST …/fix-catalog (compat alias).
func TestRouterAdminFixCatalogMounted(t *testing.T) {
t.Parallel()
sm := scs.New()
apps/api/internal/httpapi/server.go
+2 -1
View File
@@ -391,7 +391,8 @@ func (s *Server) Router() http.Handler {
r.Post("/users/{id}/dev-password", s.handleAdminDevSetPassword)
r.Get("/companies", s.handleAdminListCompanies)
r.Post("/companies/{id}/clone-catalog", s.handleAdminCloneCompanyCatalog)
r.Post("/companies/{id}/fix-catalog", s.handleAdminFixCompanyCatalog)
r.Post("/companies/{id}/sync-a1", s.handleAdminSyncCompanyA1)
r.Post("/companies/{id}/fix-catalog", s.handleAdminFixCompanyCatalog) // compat alias → Sync A1
r.Get("/readiness", s.handleAdminReadiness)
r.Get("/diagnostics", s.handleAdminDiagnostics)
r.Get("/analytics", s.handleAdminAnalytics)
apps/api/internal/processing/dump_category_backfill.go
+234
View File
@@ -0,0 +1,234 @@
package processing
import (
"bufio"
"context"
"fmt"
"io"
"log"
"os"
"strings"
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// Classic A1 Postman / Elkotex fixture EANs — must never live on Platform Demo.
var A1FixtureEANs = []string{
"5905575903198",
"6970995789942",
}
// DumpCategoryBackfillResult is the outcome of BackfillCategoriesFromMySQLDump.
type DumpCategoryBackfillResult struct {
ProcessedUpdated int64 `json:"processed_updated"`
ProcessedInserted int64 `json:"processed_inserted"`
MappedUpdated int64 `json:"mapped_updated"`
DumpPairs int `json:"dump_pairs"`
PurgedOtherRaw int64 `json:"purged_other_raw"`
PurgedOtherPP int64 `json:"purged_other_pp"`
ProcessedWithCat int `json:"processed_with_cat"`
ProcessedWithoutCat int `json:"processed_without_cat"`
MappedWithCat int `json:"mapped_with_cat"`
MappedWithoutCat int `json:"mapped_without_cat"`
}
// BackfillCategoriesFromMySQLDump streams dump processed_products for the company
// legacy id and writes product_id (GTIN) → category onto Postgres mapped_data.category
// (and updates any existing processed_products.category). It does not insert
// processed rows — A1 demo seed stays at processed=0.
func BackfillCategoriesFromMySQLDump(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID, dumpPath string) (DumpCategoryBackfillResult, error) {
var out DumpCategoryBackfillResult
if pg == nil {
return out, fmt.Errorf("dump category backfill: nil pool")
}
if companyID == uuid.Nil {
return out, fmt.Errorf("dump category backfill: empty company id")
}
legacyCompany := billing.A1LegacyCompanyID
var legacy string
_ = pg.QueryRow(ctx, `SELECT COALESCE(legacy_company_id::text, '') FROM companies WHERE id = $1`, companyID).Scan(&legacy)
if legacy != "" {
legacyCompany = legacy
}
f, err := os.Open(dumpPath)
if err != nil {
return out, fmt.Errorf("open mysql dump: %w", err)
}
defer f.Close()
byGTIN, err := ScanA1ProcessedCategories(f, legacyCompany)
if err != nil {
return out, err
}
out.DumpPairs = len(byGTIN)
if len(byGTIN) == 0 {
return out, fmt.Errorf("no A1 processed_products categories for legacy %s in dump", legacyCompany)
}
log.Printf("dump: %d A1 gtin→category pairs", len(byGTIN))
gtins := make([]string, 0, len(byGTIN))
cats := make([]string, 0, len(byGTIN))
for g, c := range byGTIN {
gtins = append(gtins, g)
cats = append(cats, c)
}
ct, err := pg.Exec(ctx, `
UPDATE processed_products p
SET category = v.category,
updated_at = now()
FROM unnest($2::text[], $3::text[]) AS v(gtin, category)
WHERE p.company_id = $1
AND p.product_id = v.gtin
AND COALESCE(NULLIF(trim(v.category), ''), '') <> ''
AND (
COALESCE(NULLIF(trim(p.category), ''), '') = ''
OR lower(trim(p.category)) = 'none'
OR p.category IS DISTINCT FROM v.category
)`, companyID, gtins, cats)
if err != nil {
return out, fmt.Errorf("update processed category from dump: %w", err)
}
out.ProcessedUpdated = ct.RowsAffected()
ct, err = pg.Exec(ctx, `
UPDATE raw_products r
SET mapped_data = jsonb_set(
COALESCE(r.mapped_data, '{}'::jsonb),
'{category}',
to_jsonb(v.category),
true
),
updated_at = now()
FROM unnest($2::text[], $3::text[]) AS v(gtin, category)
WHERE r.company_id = $1
AND r.gtin = v.gtin
AND COALESCE(NULLIF(trim(v.category), ''), '') <> ''
AND (
COALESCE(NULLIF(trim(r.mapped_data->>'category'), ''), '') = ''
OR r.mapped_data->>'category' IS DISTINCT FROM v.category
)`, companyID, gtins, cats)
if err != nil {
return out, fmt.Errorf("update mapped category from dump: %w", err)
}
out.MappedUpdated = ct.RowsAffected()
n, err := BackfillMappedCategoriesFromProcessed(ctx, pg, companyID)
if err != nil {
return out, err
}
out.MappedUpdated += n
if err := fillDumpCategoryCoverage(ctx, pg, companyID, &out); err != nil {
return out, err
}
return out, nil
}
func fillDumpCategoryCoverage(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID, out *DumpCategoryBackfillResult) error {
err := pg.QueryRow(ctx, `
SELECT
COUNT(*) FILTER (
WHERE COALESCE(NULLIF(trim(category), ''), '') <> ''
AND lower(trim(category)) <> 'none'
),
COUNT(*) FILTER (
WHERE COALESCE(NULLIF(trim(category), ''), '') = ''
OR lower(trim(category)) = 'none'
)
FROM processed_products
WHERE company_id = $1`, companyID).Scan(&out.ProcessedWithCat, &out.ProcessedWithoutCat)
if err != nil {
return fmt.Errorf("count processed categories: %w", err)
}
err = pg.QueryRow(ctx, `
SELECT
COUNT(*) FILTER (
WHERE COALESCE(NULLIF(trim(mapped_data->>'category'), ''), '') <> ''
),
COUNT(*) FILTER (
WHERE COALESCE(NULLIF(trim(mapped_data->>'category'), ''), '') = ''
)
FROM raw_products
WHERE company_id = $1`, companyID).Scan(&out.MappedWithCat, &out.MappedWithoutCat)
if err != nil {
return fmt.Errorf("count mapped categories: %w", err)
}
return nil
}
// PurgeA1FixtureEANsFromOtherTenants deletes the Postman Elkotex fixture EANs
// from every company except A1 (Platform Demo must not mirror them).
func PurgeA1FixtureEANsFromOtherTenants(ctx context.Context, pg *pgxpool.Pool, a1CompanyID uuid.UUID) (rawN, ppN int64, err error) {
_, err = pg.Exec(ctx, `
DELETE FROM processing_job_products pjp
WHERE pjp.raw_product_id IN (
SELECT id FROM raw_products
WHERE company_id <> $1 AND gtin = ANY($2::text[])
)
OR pjp.processed_product_id IN (
SELECT id FROM processed_products
WHERE company_id <> $1 AND product_id = ANY($2::text[])
)`, a1CompanyID, A1FixtureEANs)
if err != nil {
return 0, 0, fmt.Errorf("purge fixture job products: %w", err)
}
ct, err := pg.Exec(ctx, `
DELETE FROM processed_products
WHERE company_id <> $1 AND product_id = ANY($2::text[])`, a1CompanyID, A1FixtureEANs)
if err != nil {
return 0, 0, fmt.Errorf("purge fixture processed: %w", err)
}
ppN = ct.RowsAffected()
ct, err = pg.Exec(ctx, `
DELETE FROM raw_products
WHERE company_id <> $1 AND gtin = ANY($2::text[])`, a1CompanyID, A1FixtureEANs)
if err != nil {
return 0, 0, fmt.Errorf("purge fixture raw: %w", err)
}
rawN = ct.RowsAffected()
return rawN, ppN, nil
}
// ScanA1ProcessedCategories reads mysqldump INSERT rows for processed_products
// belonging to legacyCompany and returns gtin → category unique_id.
func ScanA1ProcessedCategories(r io.Reader, legacyCompany string) (map[string]string, error) {
br := bufio.NewReaderSize(r, 1<<20)
inTable := false
out := make(map[string]string, 4096)
for {
line, err := br.ReadString('\n')
if len(line) > 0 {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "INSERT INTO `processed_products`") ||
strings.HasPrefix(trimmed, "INSERT INTO processed_products") {
inTable = true
} else if inTable && strings.HasPrefix(trimmed, "CREATE TABLE") {
break
} else if inTable && strings.HasPrefix(trimmed, "INSERT INTO `") &&
!strings.Contains(trimmed, "processed_products") {
break
} else if inTable && LooksLikeMySQLTupleLine(line) && strings.Contains(line, legacyCompany) {
fields := ParseMySQLTupleFieldsN(line, 6)
if len(fields) >= 5 {
gtin := strings.TrimSpace(fields[2])
cat := strings.TrimSpace(fields[4])
if gtin != "" && cat != "" && !strings.EqualFold(cat, "NULL") {
out[gtin] = cat
}
}
}
}
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
}
return out, nil
}
apps/api/internal/processing/mysql_dump_parse.go
+67
View File
@@ -0,0 +1,67 @@
package processing
import "strings"
// LooksLikeMySQLTupleLine reports whether line starts a mysqldump VALUES tuple.
func LooksLikeMySQLTupleLine(line string) bool {
s := strings.TrimLeft(line, " \t")
return strings.HasPrefix(s, "(")
}
// ParseMySQLTupleFields parses all fields from the first (...) tuple on the line.
func ParseMySQLTupleFields(line string) []string {
return ParseMySQLTupleFieldsN(line, 0)
}
// ParseMySQLTupleFieldsN parses up to maxFields (0 = all) from the first (...) tuple.
func ParseMySQLTupleFieldsN(line string, maxFields int) []string {
start := strings.Index(line, "(")
if start < 0 {
return nil
}
body := line[start+1:]
var out []string
for i := 0; i < len(body); {
if maxFields > 0 && len(out) >= maxFields {
break
}
for i < len(body) && (body[i] == ' ' || body[i] == '\t' || body[i] == ',') {
i++
}
if i >= len(body) || body[i] == ')' {
break
}
if body[i] == '\'' {
i++
var b strings.Builder
for i < len(body) {
ch := body[i]
if ch == '\\' && i+1 < len(body) {
b.WriteByte(body[i+1])
i += 2
continue
}
if ch == '\'' {
if i+1 < len(body) && body[i+1] == '\'' {
b.WriteByte('\'')
i += 2
continue
}
i++
break
}
b.WriteByte(ch)
i++
}
out = append(out, b.String())
continue
}
j := i
for j < len(body) && body[j] != ',' && body[j] != ')' {
j++
}
out = append(out, strings.TrimSpace(body[i:j]))
i = j
}
return out
}
apps/api/internal/processing/mysql_dump_path.go
+91
View File
@@ -0,0 +1,91 @@
package processing
import (
"log"
"os"
"path/filepath"
"strings"
)
// ResolveMySQLDumpPath picks an explicit path, else the first readable candidate
// under common local locations documented in scripts/seed/README.txt.
// Used by seed-a1 CLI and admin Sync A1 (API process filesystem).
func ResolveMySQLDumpPath(explicit string) string {
if p := strings.TrimSpace(explicit); p != "" {
if st, err := os.Stat(p); err == nil && !st.IsDir() {
return p
}
log.Printf("warning: mysql dump not found at %q — trying auto-detect", p)
}
for _, c := range MySQLDumpCandidates() {
if strings.TrimSpace(explicit) != "" && filepath.Clean(c) == filepath.Clean(strings.TrimSpace(explicit)) {
continue
}
if st, err := os.Stat(c); err == nil && !st.IsDir() {
return c
}
}
return ""
}
// MySQLDumpCandidates lists paths seed-a1 / Sync A1 try when SEED_A1_MYSQL_DUMP
// or -mysql-dump is unset. First readable file wins via ResolveMySQLDumpPath.
//
// ASSUMPTION: On servers (e.g. Git-Syncer) the dump must live on the API host
// filesystem — prefer scripts/seed/descrybe_new.sql under the deploy root, or set
// SEED_A1_MYSQL_DUMP in the API/worker environment.
func MySQLDumpCandidates() []string {
var out []string
if v := strings.TrimSpace(os.Getenv("SEED_A1_MYSQL_DUMP")); v != "" {
out = append(out, v)
}
home, _ := os.UserHomeDir()
names := []string{
"descrybe_new (1).sql",
"descrybe_new.sql",
"descrybe_new(1).sql",
}
if home != "" {
for _, n := range names {
out = append(out, filepath.Join(home, "Downloads", n))
out = append(out, filepath.Join(home, "downloads", n))
}
}
// Repo-relative guesses (cwd may be apps/api or repo root).
for _, n := range names {
out = append(out,
n,
filepath.Join("..", "..", n),
filepath.Join("scripts", "seed", n),
filepath.Join("..", "..", "scripts", "seed", n),
)
}
if root, ok := findMonorepoRootFromCwd(); ok {
for _, n := range names {
out = append(out, filepath.Join(root, "scripts", "seed", n))
}
}
return out
}
func findMonorepoRootFromCwd() (string, bool) {
cwd, err := os.Getwd()
if err != nil {
return "", false
}
dir := cwd
for {
api := filepath.Join(dir, "apps", "api")
web := filepath.Join(dir, "apps", "web")
if st, err := os.Stat(api); err == nil && st.IsDir() {
if st, err := os.Stat(web); err == nil && st.IsDir() {
return dir, true
}
}
parent := filepath.Dir(dir)
if parent == dir {
return "", false
}
dir = parent
}
}
apps/api/internal/processing/mysql_dump_path_test.go
+65
View File
@@ -0,0 +1,65 @@
package processing
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestResolveMySQLDumpPathExplicit(t *testing.T) {
t.Parallel()
dir := t.TempDir()
p := filepath.Join(dir, "descrybe_new.sql")
if err := os.WriteFile(p, []byte("-- dump\n"), 0o644); err != nil {
t.Fatal(err)
}
got := ResolveMySQLDumpPath(p)
if got != p {
t.Fatalf("got %q want %q", got, p)
}
}
func TestResolveMySQLDumpPathMissingExplicitFallsBack(t *testing.T) {
dir := t.TempDir()
want := filepath.Join(dir, "descrybe_new (1).sql")
if err := os.WriteFile(want, []byte("-- dump\n"), 0o644); err != nil {
t.Fatal(err)
}
t.Setenv("SEED_A1_MYSQL_DUMP", want)
got := ResolveMySQLDumpPath(filepath.Join(dir, "missing.sql"))
if got != want {
t.Fatalf("got %q want fallback %q", got, want)
}
}
func TestMySQLDumpCandidatesIncludeDownloadsName(t *testing.T) {
t.Parallel()
found := false
for _, c := range MySQLDumpCandidates() {
if filepath.Base(c) == "descrybe_new (1).sql" {
found = true
break
}
}
if !found {
t.Fatal("expected Downloads/descrybe_new (1).sql among candidates")
}
}
func TestScanA1ProcessedCategoriesKeepsDescriptionNullSafe(t *testing.T) {
t.Parallel()
const legacy = "97e1a309-3d23-4aa2-b518-8e8d7afdfec7"
dump := strings.Join([]string{
"INSERT INTO `processed_products` VALUES",
"(1,\t'u',\t'790069217715',\t'Name',\t'103',\tNULL,\t'<p>ai</p>',\tNULL,\tNULL,\t'completed',\tNULL,\t0,\t'2026-01-01 00:00:00',\t'2026-01-01 00:00:00',\t1,\t'" + legacy + "',\t1);",
"CREATE TABLE `x` (",
}, "\n")
got, err := ScanA1ProcessedCategories(strings.NewReader(dump), legacy)
if err != nil {
t.Fatal(err)
}
if got["790069217715"] != "103" {
t.Fatalf("category=%q want 103", got["790069217715"])
}
}
apps/api/internal/processing/sync_company_a1.go
+93
View File
@@ -0,0 +1,93 @@
package processing
import (
"context"
"fmt"
"strings"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// SyncCompanyA1Opts controls admin Sync A1 (dump category backfill + Fix hygiene).
// Default is backfill-categories + hygiene — never a full wipe/reimport.
type SyncCompanyA1Opts struct {
FixCompanyCatalogOpts
// MySQLDumpPath is an explicit dump path; empty triggers auto-detect
// (SEED_A1_MYSQL_DUMP + MySQLDumpCandidates).
MySQLDumpPath string
// SkipDumpBackfill skips dump scan even when a dump is found.
SkipDumpBackfill bool
}
// SyncCompanyA1Result combines dump category backfill with FixCompanyCatalogResult.
type SyncCompanyA1Result struct {
FixCompanyCatalogResult
DumpFound bool `json:"dump_found"`
DumpPath string `json:"dump_path,omitempty"`
DumpSkippedReason string `json:"dump_skipped_reason,omitempty"`
DumpPairs int `json:"dump_pairs"`
DumpMappedUpdated int64 `json:"dump_mapped_updated"`
DumpProcessedUpdated int64 `json:"dump_processed_updated"`
PurgedOtherRaw int64 `json:"purged_other_raw"`
PurgedOtherPP int64 `json:"purged_other_pp"`
// DumpStatus is a short label for flash UI ({dump_status}).
DumpStatus string `json:"dump_status"`
}
// SyncCompanyA1 runs dump→category backfill when a MySQL dump is available, then
// FixCompanyCatalog hygiene. Does not wipe or reimport the catalog.
func SyncCompanyA1(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID, companyName string, a1Cohort bool, opts SyncCompanyA1Opts) (SyncCompanyA1Result, error) {
out := SyncCompanyA1Result{
FixCompanyCatalogResult: FixCompanyCatalogResult{
CompanyID: companyID,
CompanyName: companyName,
A1Cohort: a1Cohort,
},
DumpStatus: "skipped",
}
if pool == nil {
return out, fmt.Errorf("nil pool")
}
dumpPath := strings.TrimSpace(opts.MySQLDumpPath)
if !opts.SkipDumpBackfill {
resolved := ResolveMySQLDumpPath(dumpPath)
if resolved != "" {
out.DumpFound = true
out.DumpPath = resolved
dumpRes, err := BackfillCategoriesFromMySQLDump(ctx, pool, companyID, resolved)
if err != nil {
return out, fmt.Errorf("dump category backfill: %w", err)
}
out.DumpPairs = dumpRes.DumpPairs
out.DumpMappedUpdated = dumpRes.MappedUpdated
out.DumpProcessedUpdated = dumpRes.ProcessedUpdated
out.DumpStatus = "ok"
rawN, ppN, err := PurgeA1FixtureEANsFromOtherTenants(ctx, pool, companyID)
if err != nil {
return out, fmt.Errorf("purge fixture eans: %w", err)
}
out.PurgedOtherRaw = rawN
out.PurgedOtherPP = ppN
} else if dumpPath != "" {
out.DumpSkippedReason = fmt.Sprintf("mysql dump not found at %q and no auto-detect match", dumpPath)
out.DumpStatus = "missing"
} else {
out.DumpSkippedReason = "no MySQL dump (set SEED_A1_MYSQL_DUMP or place descrybe_new.sql in scripts/seed or ~/Downloads); continuing with DB hygiene only"
out.DumpStatus = "missing"
}
} else {
out.DumpSkippedReason = "dump backfill skipped by request"
out.DumpStatus = "skipped"
}
fixRes, err := FixCompanyCatalog(ctx, pool, companyID, companyName, a1Cohort, opts.FixCompanyCatalogOpts)
if err != nil {
return out, err
}
out.FixCompanyCatalogResult = fixRes
return out, nil
}