fix

2026-08-16 18:36:56 +02:00
parent d981147ff2
commit a94a5aaad7
26 changed files with 904 additions and 507 deletions
@@ -0,0 +1,234 @@
+package processing
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/descrybe/descrybe-v2/apps/api/internal/billing"
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+// Classic A1 Postman / Elkotex fixture EANs — must never live on Platform Demo.
+var A1FixtureEANs = []string{
+	"5905575903198",
+	"6970995789942",
+}
+
+// DumpCategoryBackfillResult is the outcome of BackfillCategoriesFromMySQLDump.
+type DumpCategoryBackfillResult struct {
+	ProcessedUpdated    int64 `json:"processed_updated"`
+	ProcessedInserted   int64 `json:"processed_inserted"`
+	MappedUpdated       int64 `json:"mapped_updated"`
+	DumpPairs           int   `json:"dump_pairs"`
+	PurgedOtherRaw      int64 `json:"purged_other_raw"`
+	PurgedOtherPP       int64 `json:"purged_other_pp"`
+	ProcessedWithCat    int   `json:"processed_with_cat"`
+	ProcessedWithoutCat int   `json:"processed_without_cat"`
+	MappedWithCat       int   `json:"mapped_with_cat"`
+	MappedWithoutCat    int   `json:"mapped_without_cat"`
+}
+
+// BackfillCategoriesFromMySQLDump streams dump processed_products for the company
+// legacy id and writes product_id (GTIN) → category onto Postgres mapped_data.category
+// (and updates any existing processed_products.category). It does not insert
+// processed rows — A1 demo seed stays at processed=0.
+func BackfillCategoriesFromMySQLDump(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID, dumpPath string) (DumpCategoryBackfillResult, error) {
+	var out DumpCategoryBackfillResult
+	if pg == nil {
+		return out, fmt.Errorf("dump category backfill: nil pool")
+	}
+	if companyID == uuid.Nil {
+		return out, fmt.Errorf("dump category backfill: empty company id")
+	}
+	legacyCompany := billing.A1LegacyCompanyID
+	var legacy string
+	_ = pg.QueryRow(ctx, `SELECT COALESCE(legacy_company_id::text, '') FROM companies WHERE id = $1`, companyID).Scan(&legacy)
+	if legacy != "" {
+		legacyCompany = legacy
+	}
+
+	f, err := os.Open(dumpPath)
+	if err != nil {
+		return out, fmt.Errorf("open mysql dump: %w", err)
+	}
+	defer f.Close()
+
+	byGTIN, err := ScanA1ProcessedCategories(f, legacyCompany)
+	if err != nil {
+		return out, err
+	}
+	out.DumpPairs = len(byGTIN)
+	if len(byGTIN) == 0 {
+		return out, fmt.Errorf("no A1 processed_products categories for legacy %s in dump", legacyCompany)
+	}
+	log.Printf("dump: %d A1 gtin→category pairs", len(byGTIN))
+
+	gtins := make([]string, 0, len(byGTIN))
+	cats := make([]string, 0, len(byGTIN))
+	for g, c := range byGTIN {
+		gtins = append(gtins, g)
+		cats = append(cats, c)
+	}
+
+	ct, err := pg.Exec(ctx, `
+		UPDATE processed_products p
+		SET category = v.category,
+			updated_at = now()
+		FROM unnest($2::text[], $3::text[]) AS v(gtin, category)
+		WHERE p.company_id = $1
+		  AND p.product_id = v.gtin
+		  AND COALESCE(NULLIF(trim(v.category), ''), '') <> ''
+		  AND (
+			COALESCE(NULLIF(trim(p.category), ''), '') = ''
+			OR lower(trim(p.category)) = 'none'
+			OR p.category IS DISTINCT FROM v.category
+		  )`, companyID, gtins, cats)
+	if err != nil {
+		return out, fmt.Errorf("update processed category from dump: %w", err)
+	}
+	out.ProcessedUpdated = ct.RowsAffected()
+
+	ct, err = pg.Exec(ctx, `
+		UPDATE raw_products r
+		SET mapped_data = jsonb_set(
+			COALESCE(r.mapped_data, '{}'::jsonb),
+			'{category}',
+			to_jsonb(v.category),
+			true
+		),
+		updated_at = now()
+		FROM unnest($2::text[], $3::text[]) AS v(gtin, category)
+		WHERE r.company_id = $1
+			AND r.gtin = v.gtin
+			AND COALESCE(NULLIF(trim(v.category), ''), '') <> ''
+			AND (
+				COALESCE(NULLIF(trim(r.mapped_data->>'category'), ''), '') = ''
+				OR r.mapped_data->>'category' IS DISTINCT FROM v.category
+			)`, companyID, gtins, cats)
+	if err != nil {
+		return out, fmt.Errorf("update mapped category from dump: %w", err)
+	}
+	out.MappedUpdated = ct.RowsAffected()
+
+	n, err := BackfillMappedCategoriesFromProcessed(ctx, pg, companyID)
+	if err != nil {
+		return out, err
+	}
+	out.MappedUpdated += n
+	if err := fillDumpCategoryCoverage(ctx, pg, companyID, &out); err != nil {
+		return out, err
+	}
+	return out, nil
+}
+
+func fillDumpCategoryCoverage(ctx context.Context, pg *pgxpool.Pool, companyID uuid.UUID, out *DumpCategoryBackfillResult) error {
+	err := pg.QueryRow(ctx, `
+		SELECT
+			COUNT(*) FILTER (
+				WHERE COALESCE(NULLIF(trim(category), ''), '') <> ''
+					AND lower(trim(category)) <> 'none'
+			),
+			COUNT(*) FILTER (
+				WHERE COALESCE(NULLIF(trim(category), ''), '') = ''
+					OR lower(trim(category)) = 'none'
+			)
+		FROM processed_products
+		WHERE company_id = $1`, companyID).Scan(&out.ProcessedWithCat, &out.ProcessedWithoutCat)
+	if err != nil {
+		return fmt.Errorf("count processed categories: %w", err)
+	}
+	err = pg.QueryRow(ctx, `
+		SELECT
+			COUNT(*) FILTER (
+				WHERE COALESCE(NULLIF(trim(mapped_data->>'category'), ''), '') <> ''
+			),
+			COUNT(*) FILTER (
+				WHERE COALESCE(NULLIF(trim(mapped_data->>'category'), ''), '') = ''
+			)
+		FROM raw_products
+		WHERE company_id = $1`, companyID).Scan(&out.MappedWithCat, &out.MappedWithoutCat)
+	if err != nil {
+		return fmt.Errorf("count mapped categories: %w", err)
+	}
+	return nil
+}
+
+// PurgeA1FixtureEANsFromOtherTenants deletes the Postman Elkotex fixture EANs
+// from every company except A1 (Platform Demo must not mirror them).
+func PurgeA1FixtureEANsFromOtherTenants(ctx context.Context, pg *pgxpool.Pool, a1CompanyID uuid.UUID) (rawN, ppN int64, err error) {
+	_, err = pg.Exec(ctx, `
+		DELETE FROM processing_job_products pjp
+		WHERE pjp.raw_product_id IN (
+			SELECT id FROM raw_products
+			WHERE company_id <> $1 AND gtin = ANY($2::text[])
+		)
+		OR pjp.processed_product_id IN (
+			SELECT id FROM processed_products
+			WHERE company_id <> $1 AND product_id = ANY($2::text[])
+		)`, a1CompanyID, A1FixtureEANs)
+	if err != nil {
+		return 0, 0, fmt.Errorf("purge fixture job products: %w", err)
+	}
+
+	ct, err := pg.Exec(ctx, `
+		DELETE FROM processed_products
+		WHERE company_id <> $1 AND product_id = ANY($2::text[])`, a1CompanyID, A1FixtureEANs)
+	if err != nil {
+		return 0, 0, fmt.Errorf("purge fixture processed: %w", err)
+	}
+	ppN = ct.RowsAffected()
+
+	ct, err = pg.Exec(ctx, `
+		DELETE FROM raw_products
+		WHERE company_id <> $1 AND gtin = ANY($2::text[])`, a1CompanyID, A1FixtureEANs)
+	if err != nil {
+		return 0, 0, fmt.Errorf("purge fixture raw: %w", err)
+	}
+	rawN = ct.RowsAffected()
+	return rawN, ppN, nil
+}
+
+// ScanA1ProcessedCategories reads mysqldump INSERT rows for processed_products
+// belonging to legacyCompany and returns gtin → category unique_id.
+func ScanA1ProcessedCategories(r io.Reader, legacyCompany string) (map[string]string, error) {
+	br := bufio.NewReaderSize(r, 1<<20)
+	inTable := false
+	out := make(map[string]string, 4096)
+	for {
+		line, err := br.ReadString('\n')
+		if len(line) > 0 {
+			trimmed := strings.TrimSpace(line)
+			if strings.HasPrefix(trimmed, "INSERT INTO `processed_products`") ||
+				strings.HasPrefix(trimmed, "INSERT INTO processed_products") {
+				inTable = true
+			} else if inTable && strings.HasPrefix(trimmed, "CREATE TABLE") {
+				break
+			} else if inTable && strings.HasPrefix(trimmed, "INSERT INTO `") &&
+				!strings.Contains(trimmed, "processed_products") {
+				break
+			} else if inTable && LooksLikeMySQLTupleLine(line) && strings.Contains(line, legacyCompany) {
+				fields := ParseMySQLTupleFieldsN(line, 6)
+				if len(fields) >= 5 {
+					gtin := strings.TrimSpace(fields[2])
+					cat := strings.TrimSpace(fields[4])
+					if gtin != "" && cat != "" && !strings.EqualFold(cat, "NULL") {
+						out[gtin] = cat
+					}
+				}
+			}
+		}
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+	return out, nil
+}
@@ -0,0 +1,67 @@
+package processing
+
+import "strings"
+
+// LooksLikeMySQLTupleLine reports whether line starts a mysqldump VALUES tuple.
+func LooksLikeMySQLTupleLine(line string) bool {
+	s := strings.TrimLeft(line, " \t")
+	return strings.HasPrefix(s, "(")
+}
+
+// ParseMySQLTupleFields parses all fields from the first (...) tuple on the line.
+func ParseMySQLTupleFields(line string) []string {
+	return ParseMySQLTupleFieldsN(line, 0)
+}
+
+// ParseMySQLTupleFieldsN parses up to maxFields (0 = all) from the first (...) tuple.
+func ParseMySQLTupleFieldsN(line string, maxFields int) []string {
+	start := strings.Index(line, "(")
+	if start < 0 {
+		return nil
+	}
+	body := line[start+1:]
+	var out []string
+	for i := 0; i < len(body); {
+		if maxFields > 0 && len(out) >= maxFields {
+			break
+		}
+		for i < len(body) && (body[i] == ' ' || body[i] == '\t' || body[i] == ',') {
+			i++
+		}
+		if i >= len(body) || body[i] == ')' {
+			break
+		}
+		if body[i] == '\'' {
+			i++
+			var b strings.Builder
+			for i < len(body) {
+				ch := body[i]
+				if ch == '\\' && i+1 < len(body) {
+					b.WriteByte(body[i+1])
+					i += 2
+					continue
+				}
+				if ch == '\'' {
+					if i+1 < len(body) && body[i+1] == '\'' {
+						b.WriteByte('\'')
+						i += 2
+						continue
+					}
+					i++
+					break
+				}
+				b.WriteByte(ch)
+				i++
+			}
+			out = append(out, b.String())
+			continue
+		}
+		j := i
+		for j < len(body) && body[j] != ',' && body[j] != ')' {
+			j++
+		}
+		out = append(out, strings.TrimSpace(body[i:j]))
+		i = j
+	}
+	return out
+}
@@ -0,0 +1,91 @@
+package processing
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// ResolveMySQLDumpPath picks an explicit path, else the first readable candidate
+// under common local locations documented in scripts/seed/README.txt.
+// Used by seed-a1 CLI and admin Sync A1 (API process filesystem).
+func ResolveMySQLDumpPath(explicit string) string {
+	if p := strings.TrimSpace(explicit); p != "" {
+		if st, err := os.Stat(p); err == nil && !st.IsDir() {
+			return p
+		}
+		log.Printf("warning: mysql dump not found at %q — trying auto-detect", p)
+	}
+	for _, c := range MySQLDumpCandidates() {
+		if strings.TrimSpace(explicit) != "" && filepath.Clean(c) == filepath.Clean(strings.TrimSpace(explicit)) {
+			continue
+		}
+		if st, err := os.Stat(c); err == nil && !st.IsDir() {
+			return c
+		}
+	}
+	return ""
+}
+
+// MySQLDumpCandidates lists paths seed-a1 / Sync A1 try when SEED_A1_MYSQL_DUMP
+// or -mysql-dump is unset. First readable file wins via ResolveMySQLDumpPath.
+//
+// ASSUMPTION: On servers (e.g. Git-Syncer) the dump must live on the API host
+// filesystem — prefer scripts/seed/descrybe_new.sql under the deploy root, or set
+// SEED_A1_MYSQL_DUMP in the API/worker environment.
+func MySQLDumpCandidates() []string {
+	var out []string
+	if v := strings.TrimSpace(os.Getenv("SEED_A1_MYSQL_DUMP")); v != "" {
+		out = append(out, v)
+	}
+	home, _ := os.UserHomeDir()
+	names := []string{
+		"descrybe_new (1).sql",
+		"descrybe_new.sql",
+		"descrybe_new(1).sql",
+	}
+	if home != "" {
+		for _, n := range names {
+			out = append(out, filepath.Join(home, "Downloads", n))
+			out = append(out, filepath.Join(home, "downloads", n))
+		}
+	}
+	// Repo-relative guesses (cwd may be apps/api or repo root).
+	for _, n := range names {
+		out = append(out,
+			n,
+			filepath.Join("..", "..", n),
+			filepath.Join("scripts", "seed", n),
+			filepath.Join("..", "..", "scripts", "seed", n),
+		)
+	}
+	if root, ok := findMonorepoRootFromCwd(); ok {
+		for _, n := range names {
+			out = append(out, filepath.Join(root, "scripts", "seed", n))
+		}
+	}
+	return out
+}
+
+func findMonorepoRootFromCwd() (string, bool) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		return "", false
+	}
+	dir := cwd
+	for {
+		api := filepath.Join(dir, "apps", "api")
+		web := filepath.Join(dir, "apps", "web")
+		if st, err := os.Stat(api); err == nil && st.IsDir() {
+			if st, err := os.Stat(web); err == nil && st.IsDir() {
+				return dir, true
+			}
+		}
+		parent := filepath.Dir(dir)
+		if parent == dir {
+			return "", false
+		}
+		dir = parent
+	}
+}
@@ -0,0 +1,65 @@
+package processing
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestResolveMySQLDumpPathExplicit(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+	p := filepath.Join(dir, "descrybe_new.sql")
+	if err := os.WriteFile(p, []byte("-- dump\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	got := ResolveMySQLDumpPath(p)
+	if got != p {
+		t.Fatalf("got %q want %q", got, p)
+	}
+}
+
+func TestResolveMySQLDumpPathMissingExplicitFallsBack(t *testing.T) {
+	dir := t.TempDir()
+	want := filepath.Join(dir, "descrybe_new (1).sql")
+	if err := os.WriteFile(want, []byte("-- dump\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("SEED_A1_MYSQL_DUMP", want)
+	got := ResolveMySQLDumpPath(filepath.Join(dir, "missing.sql"))
+	if got != want {
+		t.Fatalf("got %q want fallback %q", got, want)
+	}
+}
+
+func TestMySQLDumpCandidatesIncludeDownloadsName(t *testing.T) {
+	t.Parallel()
+	found := false
+	for _, c := range MySQLDumpCandidates() {
+		if filepath.Base(c) == "descrybe_new (1).sql" {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Fatal("expected Downloads/descrybe_new (1).sql among candidates")
+	}
+}
+
+func TestScanA1ProcessedCategoriesKeepsDescriptionNullSafe(t *testing.T) {
+	t.Parallel()
+	const legacy = "97e1a309-3d23-4aa2-b518-8e8d7afdfec7"
+	dump := strings.Join([]string{
+		"INSERT INTO `processed_products` VALUES",
+		"(1,\t'u',\t'790069217715',\t'Name',\t'103',\tNULL,\t'<p>ai</p>',\tNULL,\tNULL,\t'completed',\tNULL,\t0,\t'2026-01-01 00:00:00',\t'2026-01-01 00:00:00',\t1,\t'" + legacy + "',\t1);",
+		"CREATE TABLE `x` (",
+	}, "\n")
+	got, err := ScanA1ProcessedCategories(strings.NewReader(dump), legacy)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got["790069217715"] != "103" {
+		t.Fatalf("category=%q want 103", got["790069217715"])
+	}
+}
@@ -0,0 +1,93 @@
+package processing
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+// SyncCompanyA1Opts controls admin Sync A1 (dump category backfill + Fix hygiene).
+// Default is backfill-categories + hygiene — never a full wipe/reimport.
+type SyncCompanyA1Opts struct {
+	FixCompanyCatalogOpts
+	// MySQLDumpPath is an explicit dump path; empty triggers auto-detect
+	// (SEED_A1_MYSQL_DUMP + MySQLDumpCandidates).
+	MySQLDumpPath string
+	// SkipDumpBackfill skips dump scan even when a dump is found.
+	SkipDumpBackfill bool
+}
+
+// SyncCompanyA1Result combines dump category backfill with FixCompanyCatalogResult.
+type SyncCompanyA1Result struct {
+	FixCompanyCatalogResult
+
+	DumpFound            bool   `json:"dump_found"`
+	DumpPath             string `json:"dump_path,omitempty"`
+	DumpSkippedReason    string `json:"dump_skipped_reason,omitempty"`
+	DumpPairs            int    `json:"dump_pairs"`
+	DumpMappedUpdated    int64  `json:"dump_mapped_updated"`
+	DumpProcessedUpdated int64  `json:"dump_processed_updated"`
+	PurgedOtherRaw       int64  `json:"purged_other_raw"`
+	PurgedOtherPP        int64  `json:"purged_other_pp"`
+	// DumpStatus is a short label for flash UI ({dump_status}).
+	DumpStatus string `json:"dump_status"`
+}
+
+// SyncCompanyA1 runs dump→category backfill when a MySQL dump is available, then
+// FixCompanyCatalog hygiene. Does not wipe or reimport the catalog.
+func SyncCompanyA1(ctx context.Context, pool *pgxpool.Pool, companyID uuid.UUID, companyName string, a1Cohort bool, opts SyncCompanyA1Opts) (SyncCompanyA1Result, error) {
+	out := SyncCompanyA1Result{
+		FixCompanyCatalogResult: FixCompanyCatalogResult{
+			CompanyID:   companyID,
+			CompanyName: companyName,
+			A1Cohort:    a1Cohort,
+		},
+		DumpStatus: "skipped",
+	}
+	if pool == nil {
+		return out, fmt.Errorf("nil pool")
+	}
+
+	dumpPath := strings.TrimSpace(opts.MySQLDumpPath)
+	if !opts.SkipDumpBackfill {
+		resolved := ResolveMySQLDumpPath(dumpPath)
+		if resolved != "" {
+			out.DumpFound = true
+			out.DumpPath = resolved
+			dumpRes, err := BackfillCategoriesFromMySQLDump(ctx, pool, companyID, resolved)
+			if err != nil {
+				return out, fmt.Errorf("dump category backfill: %w", err)
+			}
+			out.DumpPairs = dumpRes.DumpPairs
+			out.DumpMappedUpdated = dumpRes.MappedUpdated
+			out.DumpProcessedUpdated = dumpRes.ProcessedUpdated
+			out.DumpStatus = "ok"
+
+			rawN, ppN, err := PurgeA1FixtureEANsFromOtherTenants(ctx, pool, companyID)
+			if err != nil {
+				return out, fmt.Errorf("purge fixture eans: %w", err)
+			}
+			out.PurgedOtherRaw = rawN
+			out.PurgedOtherPP = ppN
+		} else if dumpPath != "" {
+			out.DumpSkippedReason = fmt.Sprintf("mysql dump not found at %q and no auto-detect match", dumpPath)
+			out.DumpStatus = "missing"
+		} else {
+			out.DumpSkippedReason = "no MySQL dump (set SEED_A1_MYSQL_DUMP or place descrybe_new.sql in scripts/seed or ~/Downloads); continuing with DB hygiene only"
+			out.DumpStatus = "missing"
+		}
+	} else {
+		out.DumpSkippedReason = "dump backfill skipped by request"
+		out.DumpStatus = "skipped"
+	}
+
+	fixRes, err := FixCompanyCatalog(ctx, pool, companyID, companyName, a1Cohort, opts.FixCompanyCatalogOpts)
+	if err != nil {
+		return out, err
+	}
+	out.FixCompanyCatalogResult = fixRes
+	return out, nil
+}