package catalog import ( "context" "encoding/csv" "encoding/json" "errors" "fmt" "io" "strings" "github.com/google/uuid" ) const ( importCSVBatchSize = 500 importCSVMaxErrors = 50 ) type ImportResult struct { Created int `json:"created"` Updated int `json:"updated"` Skipped int `json:"skipped"` Errors []string `json:"errors,omitempty"` } func (r *ImportResult) addError(msg string) { if len(r.Errors) >= importCSVMaxErrors { return } r.Errors = append(r.Errors, msg) } func headerIndex(headers []string, names ...string) int { want := map[string]struct{}{} for _, n := range names { want[strings.ToLower(strings.TrimSpace(n))] = struct{}{} } for i, h := range headers { if _, ok := want[strings.ToLower(strings.TrimSpace(h))]; ok { return i } } return -1 } func cell(row []string, idx int) string { if idx < 0 || idx >= len(row) { return "" } return strings.TrimSpace(row[idx]) } type categoryCSVRow struct { name string uniqueID string parent *string desc *string } type categoryPathInfo struct { id uuid.UUID path string level int } func (s *Service) ImportCategoriesCSV(ctx context.Context, companyID uuid.UUID, r io.Reader) (ImportResult, error) { res := ImportResult{} cr := csv.NewReader(r) cr.TrimLeadingSpace = true headers, err := cr.Read() if err != nil { return res, ClientMsg("empty or invalid CSV") } iName := headerIndex(headers, "name") iUID := headerIndex(headers, "unique_id", "id", "category_id") iParent := headerIndex(headers, "parent_unique_id", "parent_id", "parent") iDesc := headerIndex(headers, "description") if iName < 0 || iUID < 0 { return res, ClientMsg("CSV must include name and unique_id columns") } known := map[string]categoryPathInfo{} batch := make([]categoryCSVRow, 0, importCSVBatchSize) for { row, err := cr.Read() if errors.Is(err, io.EOF) { break } if err != nil { res.Skipped++ res.addError(err.Error()) continue } name := cell(row, iName) uid := cell(row, iUID) if name == "" || uid == "" { res.Skipped++ continue } var parent *string if p := cell(row, iParent); p != "" { parent = &p } var desc *string if d := cell(row, iDesc); d != "" { desc = &d } batch = append(batch, categoryCSVRow{name: name, uniqueID: uid, parent: parent, desc: desc}) if len(batch) >= importCSVBatchSize { if err := s.flushCategoryBatch(ctx, companyID, batch, known, &res); err != nil { return res, err } batch = batch[:0] } } if err := s.flushCategoryBatch(ctx, companyID, batch, known, &res); err != nil { return res, err } return res, nil } func (s *Service) flushCategoryBatch(ctx context.Context, companyID uuid.UUID, batch []categoryCSVRow, known map[string]categoryPathInfo, res *ImportResult) error { if len(batch) == 0 { return nil } byUID := make(map[string]categoryCSVRow, len(batch)) order := make([]string, 0, len(batch)) for _, row := range batch { if _, ok := byUID[row.uniqueID]; !ok { order = append(order, row.uniqueID) } byUID[row.uniqueID] = row } lookup := make([]string, 0, len(byUID)*2) seenLookup := map[string]struct{}{} addLookup := func(uid string) { if uid == "" { return } if _, ok := known[uid]; ok { return } if _, ok := seenLookup[uid]; ok { return } seenLookup[uid] = struct{}{} lookup = append(lookup, uid) } for _, row := range byUID { addLookup(row.uniqueID) if row.parent != nil { addLookup(*row.parent) } } if err := s.loadCategoryPaths(ctx, companyID, lookup, known); err != nil { return err } updIDs := make([]uuid.UUID, 0, len(order)) updNames := make([]string, 0, len(order)) updDescs := make([]string, 0, len(order)) pendingInserts := make([]categoryCSVRow, 0, len(order)) for _, uid := range order { row := byUID[uid] if info, ok := known[uid]; ok { updIDs = append(updIDs, info.id) updNames = append(updNames, row.name) updDescs = append(updDescs, deref(row.desc)) continue } pendingInserts = append(pendingInserts, row) } tx, err := s.Pool.Begin(ctx) if err != nil { return err } defer tx.Rollback(ctx) if len(updIDs) > 0 { _, err = tx.Exec(ctx, ` UPDATE categories AS c SET name = v.name, description = CASE WHEN v.description <> '' THEN v.description ELSE c.description END, updated_at = now() FROM unnest($2::uuid[], $3::text[], $4::text[]) AS v(id, name, description) WHERE c.id = v.id AND c.company_id = $1`, companyID, updIDs, updNames, updDescs) if err != nil { return err } res.Updated += len(updIDs) } for len(pendingInserts) > 0 { insNames := make([]string, 0, len(pendingInserts)) insUIDs := make([]string, 0, len(pendingInserts)) insParents := make([]string, 0, len(pendingInserts)) insDescs := make([]string, 0, len(pendingInserts)) insPaths := make([]string, 0, len(pendingInserts)) insLevels := make([]int32, 0, len(pendingInserts)) next := make([]categoryCSVRow, 0, len(pendingInserts)) for _, row := range pendingInserts { path, level, parentVal, err := resolveCategoryPath(row.uniqueID, row.parent, known) if err != nil { next = append(next, row) continue } insNames = append(insNames, row.name) insUIDs = append(insUIDs, row.uniqueID) insParents = append(insParents, parentVal) insDescs = append(insDescs, deref(row.desc)) insPaths = append(insPaths, path) insLevels = append(insLevels, int32(level)) } if len(insUIDs) == 0 { for _, row := range next { res.Skipped++ res.addError(fmt.Sprintf("%s: parent category not found", row.uniqueID)) } break } rows, err := tx.Query(ctx, ` INSERT INTO categories (company_id, name, unique_id, parent_unique_id, description, path, level) SELECT $1, v.name, v.unique_id, NULLIF(v.parent_unique_id, ''), NULLIF(v.description, ''), v.path, v.level FROM unnest($2::text[], $3::text[], $4::text[], $5::text[], $6::text[], $7::int[]) AS v(name, unique_id, parent_unique_id, description, path, level) RETURNING id, unique_id, COALESCE(path, ''), level`, companyID, insNames, insUIDs, insParents, insDescs, insPaths, insLevels) if err != nil { return err } for rows.Next() { var id uuid.UUID var uid, path string var level int if err := rows.Scan(&id, &uid, &path, &level); err != nil { rows.Close() return err } known[uid] = categoryPathInfo{id: id, path: path, level: level} res.Created++ } err = rows.Err() rows.Close() if err != nil { return err } pendingInserts = next } return tx.Commit(ctx) } func (s *Service) loadCategoryPaths(ctx context.Context, companyID uuid.UUID, uids []string, known map[string]categoryPathInfo) error { if len(uids) == 0 { return nil } rows, err := s.Pool.Query(ctx, ` SELECT id, unique_id, COALESCE(path, ''), level FROM categories WHERE company_id = $1 AND unique_id = ANY($2)`, companyID, uids) if err != nil { return err } defer rows.Close() for rows.Next() { var info categoryPathInfo var uid string if err := rows.Scan(&info.id, &uid, &info.path, &info.level); err != nil { return err } known[uid] = info } return rows.Err() } func resolveCategoryPath(uniqueID string, parent *string, known map[string]categoryPathInfo) (path string, level int, parentVal string, err error) { path = uniqueID level = 0 if parent == nil || strings.TrimSpace(*parent) == "" { return path, level, "", nil } p := strings.TrimSpace(*parent) pinfo, ok := known[p] if !ok { return "", 0, "", errors.New("parent category not found") } if pinfo.path != "" { path = pinfo.path + "/" + uniqueID } else { path = p + "/" + uniqueID } return path, pinfo.level + 1, p, nil } func deref(p *string) string { if p == nil { return "" } return *p } type attributeCSVRow struct { key, name, valueType string unit, example, parent *string } func (s *Service) ImportAttributesCSV(ctx context.Context, companyID uuid.UUID, r io.Reader) (ImportResult, error) { res := ImportResult{} cr := csv.NewReader(r) cr.TrimLeadingSpace = true headers, err := cr.Read() if err != nil { return res, ClientMsg("empty or invalid CSV") } iKey := headerIndex(headers, "attribute_key", "key") iName := headerIndex(headers, "name") iType := headerIndex(headers, "value_type", "type") iUnit := headerIndex(headers, "unit") iExample := headerIndex(headers, "example") iParent := headerIndex(headers, "parent_key", "parent") if iKey < 0 || iName < 0 { return res, ClientMsg("CSV must include attribute_key and name columns") } batch := make([]attributeCSVRow, 0, importCSVBatchSize) for { row, err := cr.Read() if errors.Is(err, io.EOF) { break } if err != nil { res.Skipped++ res.addError(err.Error()) continue } key := cell(row, iKey) name := cell(row, iName) if key == "" || name == "" { res.Skipped++ continue } valueType := cell(row, iType) if valueType == "" { valueType = "string" } var unit, example, parent *string if u := cell(row, iUnit); u != "" { unit = &u } if e := cell(row, iExample); e != "" { example = &e } if p := cell(row, iParent); p != "" { parent = &p } batch = append(batch, attributeCSVRow{ key: key, name: name, valueType: valueType, unit: unit, example: example, parent: parent, }) if len(batch) >= importCSVBatchSize { if err := s.flushAttributeBatch(ctx, companyID, batch, &res); err != nil { return res, err } batch = batch[:0] } } if err := s.flushAttributeBatch(ctx, companyID, batch, &res); err != nil { return res, err } return res, nil } func (s *Service) flushAttributeBatch(ctx context.Context, companyID uuid.UUID, batch []attributeCSVRow, res *ImportResult) error { if len(batch) == 0 { return nil } byKey := make(map[string]attributeCSVRow, len(batch)) order := make([]string, 0, len(batch)) for _, row := range batch { if _, ok := byKey[row.key]; !ok { order = append(order, row.key) } byKey[row.key] = row } keys := make([]string, 0, len(byKey)) keys = append(keys, order...) existing := map[string]uuid.UUID{} rows, err := s.Pool.Query(ctx, ` SELECT id, attribute_key FROM attributes WHERE company_id = $1 AND attribute_key = ANY($2)`, companyID, keys) if err != nil { return err } for rows.Next() { var id uuid.UUID var key string if err := rows.Scan(&id, &key); err != nil { rows.Close() return err } existing[key] = id } err = rows.Err() rows.Close() if err != nil { return err } updIDs := make([]uuid.UUID, 0, len(order)) updNames := make([]string, 0, len(order)) updTypes := make([]string, 0, len(order)) insKeys := make([]string, 0, len(order)) insNames := make([]string, 0, len(order)) insTypes := make([]string, 0, len(order)) insUnits := make([]string, 0, len(order)) insExamples := make([]string, 0, len(order)) insParents := make([]string, 0, len(order)) for _, key := range order { row := byKey[key] if id, ok := existing[key]; ok { updIDs = append(updIDs, id) updNames = append(updNames, row.name) updTypes = append(updTypes, row.valueType) continue } insKeys = append(insKeys, key) insNames = append(insNames, row.name) insTypes = append(insTypes, row.valueType) insUnits = append(insUnits, deref(row.unit)) insExamples = append(insExamples, deref(row.example)) insParents = append(insParents, deref(row.parent)) } tx, err := s.Pool.Begin(ctx) if err != nil { return err } defer tx.Rollback(ctx) if len(updIDs) > 0 { _, err = tx.Exec(ctx, ` UPDATE attributes AS a SET name = CASE WHEN v.name <> '' THEN v.name ELSE a.name END, value_type = CASE WHEN v.value_type <> '' THEN v.value_type ELSE a.value_type END, updated_at = now() FROM unnest($2::uuid[], $3::text[], $4::text[]) AS v(id, name, value_type) WHERE a.id = v.id AND a.company_id = $1`, companyID, updIDs, updNames, updTypes) if err != nil { return err } res.Updated += len(updIDs) } if len(insKeys) > 0 { ct, err := tx.Exec(ctx, ` INSERT INTO attributes (company_id, attribute_key, name, value_type, unit, example, parent_key) SELECT $1, v.attribute_key, v.name, v.value_type, NULLIF(v.unit, ''), NULLIF(v.example, ''), NULLIF(v.parent_key, '') FROM unnest($2::text[], $3::text[], $4::text[], $5::text[], $6::text[], $7::text[]) AS v(attribute_key, name, value_type, unit, example, parent_key)`, companyID, insKeys, insNames, insTypes, insUnits, insExamples, insParents) if err != nil { return err } res.Created += int(ct.RowsAffected()) } return tx.Commit(ctx) } func (s *Service) MergeProductsByGTIN(ctx context.Context, companyID uuid.UUID) (bool, error) { var merge bool err := s.Pool.QueryRow(ctx, `SELECT merge_products_by_gtin FROM companies WHERE id = $1`, companyID).Scan(&merge) return merge, err } type productCSVRow struct { gtin, productID, name, category, desc, status string rawJSON string mergeable bool } func (s *Service) ImportProductsCSV(ctx context.Context, companyID uuid.UUID, r io.Reader, fileID *uuid.UUID) (ImportResult, error) { res := ImportResult{} merge, err := s.MergeProductsByGTIN(ctx, companyID) if err != nil { return res, err } cr := csv.NewReader(r) cr.TrimLeadingSpace = true headers, err := cr.Read() if err != nil { return res, ClientMsg("empty or invalid CSV") } iGTIN := headerIndex(headers, "gtin", "ean", "barcode") iPID := headerIndex(headers, "product_id", "sku", "id") iName := headerIndex(headers, "name", "title") iCat := headerIndex(headers, "category") iDesc := headerIndex(headers, "description") iStatus := headerIndex(headers, "status") if iName < 0 && iGTIN < 0 && iPID < 0 { return res, ClientMsg("CSV must include name, gtin, or product_id") } batch := make([]productCSVRow, 0, importCSVBatchSize) for { row, err := cr.Read() if errors.Is(err, io.EOF) { break } if err != nil { res.Skipped++ res.addError(err.Error()) continue } gtin := cell(row, iGTIN) productID := cell(row, iPID) name := cell(row, iName) category := cell(row, iCat) desc := cell(row, iDesc) status := cell(row, iStatus) if status == "" { status = "draft" } if gtin == "" && productID == "" && name == "" { res.Skipped++ continue } if gtin == "" { gtin = "nogtin-" + uuid.NewString() } rawData, _ := json.Marshal(map[string]any{ "product_id": productID, "name": name, "category": category, "description": desc, "status": status, "gtin": gtin, }) batch = append(batch, productCSVRow{ gtin: gtin, productID: productID, name: name, category: category, desc: desc, status: status, rawJSON: string(rawData), mergeable: merge && !strings.HasPrefix(gtin, "nogtin-"), }) if len(batch) >= importCSVBatchSize { if err := s.flushProductBatch(ctx, companyID, fileID, batch, &res); err != nil { return res, err } batch = batch[:0] } } if err := s.flushProductBatch(ctx, companyID, fileID, batch, &res); err != nil { return res, err } return res, nil } func (s *Service) flushProductBatch(ctx context.Context, companyID uuid.UUID, fileID *uuid.UUID, batch []productCSVRow, res *ImportResult) error { if len(batch) == 0 { return nil } // Last row wins per GTIN so a single INSERT cannot hit the same unique key twice. byGTIN := make(map[string]productCSVRow, len(batch)) order := make([]string, 0, len(batch)) for _, row := range batch { if _, ok := byGTIN[row.gtin]; !ok { order = append(order, row.gtin) } byGTIN[row.gtin] = row } deduped := make([]productCSVRow, 0, len(order)) for _, gtin := range order { deduped = append(deduped, byGTIN[gtin]) } batch = deduped mergeGTINs := make([]string, 0, len(batch)) for _, row := range batch { if row.mergeable { mergeGTINs = append(mergeGTINs, row.gtin) } } existingRaw := map[string]uuid.UUID{} if len(mergeGTINs) > 0 { rows, err := s.Pool.Query(ctx, ` SELECT DISTINCT ON (gtin) id, gtin FROM raw_products WHERE company_id = $1 AND gtin = ANY($2) ORDER BY gtin, updated_at DESC`, companyID, mergeGTINs) if err != nil { return err } for rows.Next() { var id uuid.UUID var gtin string if err := rows.Scan(&id, >in); err != nil { rows.Close() return err } existingRaw[gtin] = id } err = rows.Err() rows.Close() if err != nil { return err } } type pendingProcessed struct { rawID uuid.UUID productID, name, category, desc, status string rawWasUpdate bool } tx, err := s.Pool.Begin(ctx) if err != nil { return err } defer tx.Rollback(ctx) updRawIDs := make([]uuid.UUID, 0, len(batch)) updRawJSON := make([]string, 0, len(batch)) updRawMeta := make([]pendingProcessed, 0, len(batch)) insGTINs := make([]string, 0, len(batch)) insJSON := make([]string, 0, len(batch)) insMeta := make([]pendingProcessed, 0, len(batch)) for _, row := range batch { meta := pendingProcessed{ productID: row.productID, name: row.name, category: row.category, desc: row.desc, status: row.status, } if row.mergeable { if id, ok := existingRaw[row.gtin]; ok { updRawIDs = append(updRawIDs, id) updRawJSON = append(updRawJSON, row.rawJSON) meta.rawID = id meta.rawWasUpdate = true updRawMeta = append(updRawMeta, meta) continue } } insGTINs = append(insGTINs, row.gtin) insJSON = append(insJSON, row.rawJSON) insMeta = append(insMeta, meta) } if len(updRawIDs) > 0 { _, err = tx.Exec(ctx, ` UPDATE raw_products AS r SET raw_data = v.raw_data::jsonb, mapped_data = v.raw_data::jsonb, file_id = COALESCE($3, r.file_id), updated_at = now() FROM unnest($2::uuid[], $4::text[]) AS v(id, raw_data) WHERE r.id = v.id AND r.company_id = $1`, companyID, updRawIDs, fileID, updRawJSON) if err != nil { return err } } pending := make([]pendingProcessed, 0, len(batch)) pending = append(pending, updRawMeta...) if len(insGTINs) > 0 { rows, err := tx.Query(ctx, ` INSERT INTO raw_products (company_id, gtin, raw_data, mapped_data, processing_status, file_id) SELECT $1, v.gtin, v.raw_data::jsonb, v.raw_data::jsonb, 'unprocessed', $2 FROM unnest($3::text[], $4::text[]) AS v(gtin, raw_data) ON CONFLICT (company_id, gtin) DO NOTHING RETURNING id, gtin`, companyID, fileID, insGTINs, insJSON) if err != nil { return err } insertedByGTIN := map[string]uuid.UUID{} for rows.Next() { var id uuid.UUID var gtin string if err := rows.Scan(&id, >in); err != nil { rows.Close() return err } insertedByGTIN[gtin] = id } err = rows.Err() rows.Close() if err != nil { return err } // Match inserts back to input order; conflicts (DO NOTHING) are skipped. for i, gtin := range insGTINs { id, ok := insertedByGTIN[gtin] if !ok { res.Skipped++ res.addError(fmt.Sprintf("%s: duplicate gtin", gtin)) continue } meta := insMeta[i] meta.rawID = id pending = append(pending, meta) // Same gtin inserted twice in one batch: second RETURNING miss. delete(insertedByGTIN, gtin) } } if len(pending) == 0 { return tx.Commit(ctx) } rawIDs := make([]uuid.UUID, len(pending)) for i, p := range pending { rawIDs[i] = p.rawID } existingProcessed := map[uuid.UUID]uuid.UUID{} prows, err := tx.Query(ctx, ` SELECT DISTINCT ON (raw_product_id) id, raw_product_id FROM processed_products WHERE company_id = $1 AND raw_product_id = ANY($2) ORDER BY raw_product_id, updated_at DESC`, companyID, rawIDs) if err != nil { return err } for prows.Next() { var id, rawID uuid.UUID if err := prows.Scan(&id, &rawID); err != nil { prows.Close() return err } existingProcessed[rawID] = id } err = prows.Err() prows.Close() if err != nil { return err } updProcIDs := make([]uuid.UUID, 0, len(pending)) updPIDs := make([]string, 0, len(pending)) updNames := make([]string, 0, len(pending)) updCats := make([]string, 0, len(pending)) updDescs := make([]string, 0, len(pending)) updStatuses := make([]string, 0, len(pending)) insRawIDs := make([]uuid.UUID, 0, len(pending)) insPIDs := make([]string, 0, len(pending)) insNames := make([]string, 0, len(pending)) insCats := make([]string, 0, len(pending)) insDescs := make([]string, 0, len(pending)) insStatuses := make([]string, 0, len(pending)) insWasUpdate := make([]bool, 0, len(pending)) for _, p := range pending { if pid, ok := existingProcessed[p.rawID]; ok { updProcIDs = append(updProcIDs, pid) updPIDs = append(updPIDs, p.productID) updNames = append(updNames, p.name) updCats = append(updCats, p.category) updDescs = append(updDescs, p.desc) updStatuses = append(updStatuses, p.status) continue } insRawIDs = append(insRawIDs, p.rawID) insPIDs = append(insPIDs, p.productID) insNames = append(insNames, p.name) insCats = append(insCats, p.category) insDescs = append(insDescs, p.desc) insStatuses = append(insStatuses, p.status) insWasUpdate = append(insWasUpdate, p.rawWasUpdate) } if len(updProcIDs) > 0 { _, err = tx.Exec(ctx, ` UPDATE processed_products AS p SET product_id = COALESCE(NULLIF(v.product_id, ''), p.product_id), name = COALESCE(NULLIF(v.name, ''), p.name), category = COALESCE(NULLIF(v.category, ''), p.category), description = COALESCE(NULLIF(v.description, ''), p.description), status = COALESCE(NULLIF(v.status, ''), p.status), updated_at = now() FROM unnest($2::uuid[], $3::text[], $4::text[], $5::text[], $6::text[], $7::text[]) AS v(id, product_id, name, category, description, status) WHERE p.id = v.id AND p.company_id = $1`, companyID, updProcIDs, updPIDs, updNames, updCats, updDescs, updStatuses) if err != nil { return err } res.Updated += len(updProcIDs) } if len(insRawIDs) > 0 { _, err = tx.Exec(ctx, ` INSERT INTO processed_products (company_id, product_id, name, category, description, status, raw_product_id) SELECT $1, NULLIF(v.product_id, ''), NULLIF(v.name, ''), NULLIF(v.category, ''), NULLIF(v.description, ''), v.status, v.raw_product_id FROM unnest($2::uuid[], $3::text[], $4::text[], $5::text[], $6::text[], $7::text[]) AS v(raw_product_id, product_id, name, category, description, status)`, companyID, insRawIDs, insPIDs, insNames, insCats, insDescs, insStatuses) if err != nil { return err } for _, wasUpdate := range insWasUpdate { if wasUpdate { res.Updated++ } else { res.Created++ } } } return tx.Commit(ctx) }