fix
This commit is contained in:
@@ -1,26 +1,15 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// Classic A1 Postman / Elkotex fixture EANs — must never live on Platform Demo.
|
||||
var a1FixtureEANs = []string{
|
||||
"5905575903198",
|
||||
"6970995789942",
|
||||
}
|
||||
|
||||
type categoryBackfillResult struct {
|
||||
ProcessedUpdated int64
|
||||
ProcessedInserted int64
|
||||
@@ -34,204 +23,38 @@ type categoryBackfillResult struct {
|
||||
MappedWithoutCat int
|
||||
}
|
||||
|
||||
// backfillMappedCategoriesFromProcessed copies processed_products.category into
|
||||
// raw_products.mapped_data.category for A1 only. Delegates to processing.
|
||||
// a1FixtureEANs aliases processing.A1FixtureEANs for seed-a1 tests.
|
||||
var a1FixtureEANs = processing.A1FixtureEANs
|
||||
|
||||
func backfillMappedCategoriesFromProcessed(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID) (int64, error) {
|
||||
return processing.BackfillMappedCategoriesFromProcessed(ctx, pg, companyID)
|
||||
}
|
||||
|
||||
// backfillCategoriesFromMySQLDump streams dump processed_products for the A1
|
||||
// legacy company and writes product_id (GTIN) → category onto A1 Postgres
|
||||
// mapped_data.category (and updates any existing processed_products.category).
|
||||
// It does not insert processed rows — A1 demo seed stays at processed=0.
|
||||
func backfillCategoriesFromMySQLDump(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID, dumpPath string) (categoryBackfillResult, error) {
|
||||
var out categoryBackfillResult
|
||||
legacyCompany := billing.A1LegacyCompanyID
|
||||
var legacy string
|
||||
_ = pg.QueryRow(ctx, `SELECT COALESCE(legacy_company_id::text, '') FROM companies WHERE id = $1`, companyID).Scan(&legacy)
|
||||
if legacy != "" {
|
||||
legacyCompany = legacy
|
||||
}
|
||||
|
||||
f, err := os.Open(dumpPath)
|
||||
res, err := processing.BackfillCategoriesFromMySQLDump(ctx, pg, companyID, dumpPath)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("open mysql dump: %w", err)
|
||||
return categoryBackfillResult{}, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
byGTIN, err := scanA1ProcessedCategories(f, legacyCompany)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.DumpPairs = len(byGTIN)
|
||||
if len(byGTIN) == 0 {
|
||||
return out, fmt.Errorf("no A1 processed_products categories for legacy %s in dump", legacyCompany)
|
||||
}
|
||||
log.Printf("dump: %d A1 gtin→category pairs", len(byGTIN))
|
||||
|
||||
gtins := make([]string, 0, len(byGTIN))
|
||||
cats := make([]string, 0, len(byGTIN))
|
||||
for g, c := range byGTIN {
|
||||
gtins = append(gtins, g)
|
||||
cats = append(cats, c)
|
||||
}
|
||||
|
||||
ct, err := pg.Exec(ctx, `
|
||||
UPDATE processed_products p
|
||||
SET category = v.category,
|
||||
updated_at = now()
|
||||
FROM unnest($2::text[], $3::text[]) AS v(gtin, category)
|
||||
WHERE p.company_id = $1
|
||||
AND p.product_id = v.gtin
|
||||
AND COALESCE(NULLIF(trim(v.category), ''), '') <> ''
|
||||
AND (
|
||||
COALESCE(NULLIF(trim(p.category), ''), '') = ''
|
||||
OR lower(trim(p.category)) = 'none'
|
||||
OR p.category IS DISTINCT FROM v.category
|
||||
)`, companyID, gtins, cats)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("update processed category from dump: %w", err)
|
||||
}
|
||||
out.ProcessedUpdated = ct.RowsAffected()
|
||||
|
||||
// A1 demo seed keeps processed=0. Do not INSERT processed rows from the dump —
|
||||
// only refresh mapped_data.category (and any existing processed rows if present).
|
||||
ct, err = pg.Exec(ctx, `
|
||||
UPDATE raw_products r
|
||||
SET mapped_data = jsonb_set(
|
||||
COALESCE(r.mapped_data, '{}'::jsonb),
|
||||
'{category}',
|
||||
to_jsonb(v.category),
|
||||
true
|
||||
),
|
||||
updated_at = now()
|
||||
FROM unnest($2::text[], $3::text[]) AS v(gtin, category)
|
||||
WHERE r.company_id = $1
|
||||
AND r.gtin = v.gtin
|
||||
AND COALESCE(NULLIF(trim(v.category), ''), '') <> ''
|
||||
AND (
|
||||
COALESCE(NULLIF(trim(r.mapped_data->>'category'), ''), '') = ''
|
||||
OR r.mapped_data->>'category' IS DISTINCT FROM v.category
|
||||
)`, companyID, gtins, cats)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("update mapped category from dump: %w", err)
|
||||
}
|
||||
out.MappedUpdated = ct.RowsAffected()
|
||||
|
||||
n, err := backfillMappedCategoriesFromProcessed(ctx, pg, companyID)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.MappedUpdated += n
|
||||
if err := fillCategoryCoverage(ctx, pg, companyID, &out); err != nil {
|
||||
return out, err
|
||||
}
|
||||
return out, nil
|
||||
return categoryBackfillResult{
|
||||
ProcessedUpdated: res.ProcessedUpdated,
|
||||
ProcessedInserted: res.ProcessedInserted,
|
||||
MappedUpdated: res.MappedUpdated,
|
||||
DumpPairs: res.DumpPairs,
|
||||
PurgedOtherRaw: res.PurgedOtherRaw,
|
||||
PurgedOtherPP: res.PurgedOtherPP,
|
||||
ProcessedWithCat: res.ProcessedWithCat,
|
||||
ProcessedWithoutCat: res.ProcessedWithoutCat,
|
||||
MappedWithCat: res.MappedWithCat,
|
||||
MappedWithoutCat: res.MappedWithoutCat,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func fillCategoryCoverage(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID, out *categoryBackfillResult) error {
|
||||
err := pg.QueryRow(ctx, `
|
||||
SELECT
|
||||
COUNT(*) FILTER (
|
||||
WHERE COALESCE(NULLIF(trim(category), ''), '') <> ''
|
||||
AND lower(trim(category)) <> 'none'
|
||||
),
|
||||
COUNT(*) FILTER (
|
||||
WHERE COALESCE(NULLIF(trim(category), ''), '') = ''
|
||||
OR lower(trim(category)) = 'none'
|
||||
)
|
||||
FROM processed_products
|
||||
WHERE company_id = $1`, companyID).Scan(&out.ProcessedWithCat, &out.ProcessedWithoutCat)
|
||||
if err != nil {
|
||||
return fmt.Errorf("count processed categories: %w", err)
|
||||
}
|
||||
err = pg.QueryRow(ctx, `
|
||||
SELECT
|
||||
COUNT(*) FILTER (
|
||||
WHERE COALESCE(NULLIF(trim(mapped_data->>'category'), ''), '') <> ''
|
||||
),
|
||||
COUNT(*) FILTER (
|
||||
WHERE COALESCE(NULLIF(trim(mapped_data->>'category'), ''), '') = ''
|
||||
)
|
||||
FROM raw_products
|
||||
WHERE company_id = $1`, companyID).Scan(&out.MappedWithCat, &out.MappedWithoutCat)
|
||||
if err != nil {
|
||||
return fmt.Errorf("count mapped categories: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// purgeA1FixtureEANsFromOtherTenants deletes the Postman Elkotex fixture EANs
|
||||
// from every company except A1 (Platform Demo must not mirror A1 fixtures).
|
||||
func purgeA1FixtureEANsFromOtherTenants(ctx context.Context, pg *pgxpool.Pool, a1CompanyID uuid.UUID) (rawN, ppN int64, err error) {
|
||||
ct, err := pg.Exec(ctx, `
|
||||
DELETE FROM processing_job_products pjp
|
||||
WHERE pjp.raw_product_id IN (
|
||||
SELECT id FROM raw_products
|
||||
WHERE company_id <> $1 AND gtin = ANY($2::text[])
|
||||
)
|
||||
OR pjp.processed_product_id IN (
|
||||
SELECT id FROM processed_products
|
||||
WHERE company_id <> $1 AND product_id = ANY($2::text[])
|
||||
)`, a1CompanyID, a1FixtureEANs)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("purge fixture job products: %w", err)
|
||||
}
|
||||
_ = ct
|
||||
|
||||
ct, err = pg.Exec(ctx, `
|
||||
DELETE FROM processed_products
|
||||
WHERE company_id <> $1 AND product_id = ANY($2::text[])`, a1CompanyID, a1FixtureEANs)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("purge fixture processed: %w", err)
|
||||
}
|
||||
ppN = ct.RowsAffected()
|
||||
|
||||
ct, err = pg.Exec(ctx, `
|
||||
DELETE FROM raw_products
|
||||
WHERE company_id <> $1 AND gtin = ANY($2::text[])`, a1CompanyID, a1FixtureEANs)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("purge fixture raw: %w", err)
|
||||
}
|
||||
rawN = ct.RowsAffected()
|
||||
return rawN, ppN, nil
|
||||
return processing.PurgeA1FixtureEANsFromOtherTenants(ctx, pg, a1CompanyID)
|
||||
}
|
||||
|
||||
func scanA1ProcessedCategories(r io.Reader, legacyCompany string) (map[string]string, error) {
|
||||
br := bufio.NewReaderSize(r, 1<<20)
|
||||
inTable := false
|
||||
out := make(map[string]string, 4096)
|
||||
for {
|
||||
line, err := br.ReadString('\n')
|
||||
if len(line) > 0 {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "INSERT INTO `processed_products`") ||
|
||||
strings.HasPrefix(trimmed, "INSERT INTO processed_products") {
|
||||
inTable = true
|
||||
} else if inTable && strings.HasPrefix(trimmed, "CREATE TABLE") {
|
||||
break
|
||||
} else if inTable && strings.HasPrefix(trimmed, "INSERT INTO `") &&
|
||||
!strings.Contains(trimmed, "processed_products") {
|
||||
break
|
||||
} else if inTable && looksLikeTupleLine(line) && strings.Contains(line, legacyCompany) {
|
||||
fields := parseMySQLTupleFieldsN(line, 6)
|
||||
if len(fields) >= 5 {
|
||||
gtin := strings.TrimSpace(fields[2])
|
||||
cat := strings.TrimSpace(fields[4])
|
||||
if gtin != "" && cat != "" && !strings.EqualFold(cat, "NULL") {
|
||||
out[gtin] = cat
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
return processing.ScanA1ProcessedCategories(r, legacyCompany)
|
||||
}
|
||||
|
||||
func logCategoryBackfillResult(res categoryBackfillResult, source string) {
|
||||
|
||||
@@ -4,70 +4,28 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// resolveMySQLDumpPath picks an explicit path, else the first readable candidate
|
||||
// under common local locations documented in scripts/seed/README.txt.
|
||||
// resolveMySQLDumpPath delegates to processing (shared with admin Sync A1).
|
||||
func resolveMySQLDumpPath(explicit string) string {
|
||||
if p := strings.TrimSpace(explicit); p != "" {
|
||||
if st, err := os.Stat(p); err == nil && !st.IsDir() {
|
||||
return p
|
||||
}
|
||||
log.Printf("warning: mysql dump not found at %q — trying auto-detect", p)
|
||||
}
|
||||
for _, c := range mysqlDumpCandidates() {
|
||||
if strings.TrimSpace(explicit) != "" && filepath.Clean(c) == filepath.Clean(strings.TrimSpace(explicit)) {
|
||||
continue
|
||||
}
|
||||
if st, err := os.Stat(c); err == nil && !st.IsDir() {
|
||||
return c
|
||||
}
|
||||
}
|
||||
return ""
|
||||
return processing.ResolveMySQLDumpPath(explicit)
|
||||
}
|
||||
|
||||
func mysqlDumpCandidates() []string {
|
||||
var out []string
|
||||
if v := strings.TrimSpace(os.Getenv("SEED_A1_MYSQL_DUMP")); v != "" {
|
||||
out = append(out, v)
|
||||
}
|
||||
home, _ := os.UserHomeDir()
|
||||
names := []string{
|
||||
"descrybe_new (1).sql",
|
||||
"descrybe_new.sql",
|
||||
"descrybe_new(1).sql",
|
||||
}
|
||||
if home != "" {
|
||||
for _, n := range names {
|
||||
out = append(out, filepath.Join(home, "Downloads", n))
|
||||
out = append(out, filepath.Join(home, "downloads", n))
|
||||
}
|
||||
}
|
||||
// Repo-relative guesses (cwd may be apps/api or repo root).
|
||||
for _, n := range names {
|
||||
out = append(out,
|
||||
n,
|
||||
filepath.Join("..", "..", n),
|
||||
filepath.Join("scripts", "seed", n),
|
||||
filepath.Join("..", "..", "scripts", "seed", n),
|
||||
)
|
||||
}
|
||||
return out
|
||||
return processing.MySQLDumpCandidates()
|
||||
}
|
||||
|
||||
type mappedCoverage struct {
|
||||
Total int
|
||||
WithDesc int
|
||||
WithCat int
|
||||
WithAttrs int
|
||||
Processed int
|
||||
Jobs int
|
||||
Total int
|
||||
WithDesc int
|
||||
WithCat int
|
||||
WithAttrs int
|
||||
Processed int
|
||||
Jobs int
|
||||
}
|
||||
|
||||
func (c mappedCoverage) pct(n int) float64 {
|
||||
|
||||
@@ -41,9 +41,10 @@
|
||||
// go run ./cmd/seed-a1 -mode backfill-categories -mysql-dump path/to/descrybe_new.sql
|
||||
// go run ./cmd/seed-a1 -mode backfill-attributes
|
||||
//
|
||||
// DATABASE_URL / -postgres required.
|
||||
// DATABASE_URL / -postgres required (also loaded from monorepo-root .env via config.LoadDotEnv).
|
||||
// Day-to-day: `npm run seed:a1` (reimport + category prompts + mapped category backfill). Maintainer snapshot: `npm run seed:a1:export`.
|
||||
// recover-jobs is opt-in only and will reintroduce job history.
|
||||
// Prefer admin UI Sync A1 on the company row (uses API DATABASE_URL; no CLI env needed).
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -61,6 +62,7 @@ import (
|
||||
"compress/gzip"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/config"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
@@ -82,6 +84,8 @@ type tableSpec struct {
|
||||
}
|
||||
|
||||
func main() {
|
||||
config.LoadDotEnv()
|
||||
|
||||
postgresURL := flag.String("postgres", os.Getenv("DATABASE_URL"), "Postgres URL (DATABASE_URL)")
|
||||
mode := flag.String("mode", "reimport", "export | reimport | recover-jobs | apply-category-prompts | backfill-categories | backfill-attributes")
|
||||
file := flag.String("file", "", "gzipped seed archive path (required for export/reimport)")
|
||||
@@ -93,7 +97,7 @@ func main() {
|
||||
flag.Parse()
|
||||
|
||||
if strings.TrimSpace(*postgresURL) == "" {
|
||||
log.Fatal("-postgres / DATABASE_URL is required")
|
||||
log.Fatal("-postgres / DATABASE_URL is required (export it, pass -postgres, or put DATABASE_URL in monorepo-root .env — same as the API; prefer admin Sync A1)")
|
||||
}
|
||||
companyID, err := uuid.Parse(strings.TrimSpace(*company))
|
||||
if err != nil {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
|
||||
"github.com/descrybe/descrybe-v2/apps/api/internal/processing"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
@@ -419,65 +420,15 @@ func dumpSectionEnded(line, table string) bool {
|
||||
}
|
||||
|
||||
func looksLikeTupleLine(line string) bool {
|
||||
s := strings.TrimLeft(line, " \t")
|
||||
return strings.HasPrefix(s, "(")
|
||||
return processing.LooksLikeMySQLTupleLine(line)
|
||||
}
|
||||
|
||||
func parseMySQLTupleFields(line string) []string {
|
||||
return parseMySQLTupleFieldsN(line, 0)
|
||||
return processing.ParseMySQLTupleFields(line)
|
||||
}
|
||||
|
||||
// parseMySQLTupleFieldsN parses up to maxFields (0 = all) from the first (...) tuple on the line.
|
||||
func parseMySQLTupleFieldsN(line string, maxFields int) []string {
|
||||
start := strings.Index(line, "(")
|
||||
if start < 0 {
|
||||
return nil
|
||||
}
|
||||
body := line[start+1:]
|
||||
var out []string
|
||||
for i := 0; i < len(body); {
|
||||
if maxFields > 0 && len(out) >= maxFields {
|
||||
break
|
||||
}
|
||||
for i < len(body) && (body[i] == ' ' || body[i] == '\t' || body[i] == ',') {
|
||||
i++
|
||||
}
|
||||
if i >= len(body) || body[i] == ')' {
|
||||
break
|
||||
}
|
||||
if body[i] == '\'' {
|
||||
i++
|
||||
var b strings.Builder
|
||||
for i < len(body) {
|
||||
ch := body[i]
|
||||
if ch == '\\' && i+1 < len(body) {
|
||||
b.WriteByte(body[i+1])
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
if ch == '\'' {
|
||||
if i+1 < len(body) && body[i+1] == '\'' {
|
||||
b.WriteByte('\'')
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
i++
|
||||
break
|
||||
}
|
||||
b.WriteByte(ch)
|
||||
i++
|
||||
}
|
||||
out = append(out, b.String())
|
||||
continue
|
||||
}
|
||||
j := i
|
||||
for j < len(body) && body[j] != ',' && body[j] != ')' {
|
||||
j++
|
||||
}
|
||||
out = append(out, strings.TrimSpace(body[i:j]))
|
||||
i = j
|
||||
}
|
||||
return out
|
||||
return processing.ParseMySQLTupleFieldsN(line, maxFields)
|
||||
}
|
||||
|
||||
func parseDumpJobID(raw string) (uuid.UUID, error) {
|
||||
|
||||
Reference in New Issue
Block a user