Initial commit of Descrybe v2 without local scratch artifacts.
Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
This commit is contained in:
@@ -0,0 +1,830 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
importCSVBatchSize = 500
|
||||
importCSVMaxErrors = 50
|
||||
)
|
||||
|
||||
type ImportResult struct {
|
||||
Created int `json:"created"`
|
||||
Updated int `json:"updated"`
|
||||
Skipped int `json:"skipped"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
}
|
||||
|
||||
func (r *ImportResult) addError(msg string) {
|
||||
if len(r.Errors) >= importCSVMaxErrors {
|
||||
return
|
||||
}
|
||||
r.Errors = append(r.Errors, msg)
|
||||
}
|
||||
|
||||
func headerIndex(headers []string, names ...string) int {
|
||||
want := map[string]struct{}{}
|
||||
for _, n := range names {
|
||||
want[strings.ToLower(strings.TrimSpace(n))] = struct{}{}
|
||||
}
|
||||
for i, h := range headers {
|
||||
if _, ok := want[strings.ToLower(strings.TrimSpace(h))]; ok {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func cell(row []string, idx int) string {
|
||||
if idx < 0 || idx >= len(row) {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(row[idx])
|
||||
}
|
||||
|
||||
type categoryCSVRow struct {
|
||||
name string
|
||||
uniqueID string
|
||||
parent *string
|
||||
desc *string
|
||||
}
|
||||
|
||||
type categoryPathInfo struct {
|
||||
id uuid.UUID
|
||||
path string
|
||||
level int
|
||||
}
|
||||
|
||||
func (s *Service) ImportCategoriesCSV(ctx context.Context, companyID uuid.UUID, r io.Reader) (ImportResult, error) {
|
||||
res := ImportResult{}
|
||||
cr := csv.NewReader(r)
|
||||
cr.TrimLeadingSpace = true
|
||||
headers, err := cr.Read()
|
||||
if err != nil {
|
||||
return res, ClientMsg("empty or invalid CSV")
|
||||
}
|
||||
iName := headerIndex(headers, "name")
|
||||
iUID := headerIndex(headers, "unique_id", "id", "category_id")
|
||||
iParent := headerIndex(headers, "parent_unique_id", "parent_id", "parent")
|
||||
iDesc := headerIndex(headers, "description")
|
||||
if iName < 0 || iUID < 0 {
|
||||
return res, ClientMsg("CSV must include name and unique_id columns")
|
||||
}
|
||||
|
||||
known := map[string]categoryPathInfo{}
|
||||
batch := make([]categoryCSVRow, 0, importCSVBatchSize)
|
||||
for {
|
||||
row, err := cr.Read()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
res.Skipped++
|
||||
res.addError(err.Error())
|
||||
continue
|
||||
}
|
||||
name := cell(row, iName)
|
||||
uid := cell(row, iUID)
|
||||
if name == "" || uid == "" {
|
||||
res.Skipped++
|
||||
continue
|
||||
}
|
||||
var parent *string
|
||||
if p := cell(row, iParent); p != "" {
|
||||
parent = &p
|
||||
}
|
||||
var desc *string
|
||||
if d := cell(row, iDesc); d != "" {
|
||||
desc = &d
|
||||
}
|
||||
batch = append(batch, categoryCSVRow{name: name, uniqueID: uid, parent: parent, desc: desc})
|
||||
if len(batch) >= importCSVBatchSize {
|
||||
if err := s.flushCategoryBatch(ctx, companyID, batch, known, &res); err != nil {
|
||||
return res, err
|
||||
}
|
||||
batch = batch[:0]
|
||||
}
|
||||
}
|
||||
if err := s.flushCategoryBatch(ctx, companyID, batch, known, &res); err != nil {
|
||||
return res, err
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (s *Service) flushCategoryBatch(ctx context.Context, companyID uuid.UUID, batch []categoryCSVRow, known map[string]categoryPathInfo, res *ImportResult) error {
|
||||
if len(batch) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
byUID := make(map[string]categoryCSVRow, len(batch))
|
||||
order := make([]string, 0, len(batch))
|
||||
for _, row := range batch {
|
||||
if _, ok := byUID[row.uniqueID]; !ok {
|
||||
order = append(order, row.uniqueID)
|
||||
}
|
||||
byUID[row.uniqueID] = row
|
||||
}
|
||||
|
||||
lookup := make([]string, 0, len(byUID)*2)
|
||||
seenLookup := map[string]struct{}{}
|
||||
addLookup := func(uid string) {
|
||||
if uid == "" {
|
||||
return
|
||||
}
|
||||
if _, ok := known[uid]; ok {
|
||||
return
|
||||
}
|
||||
if _, ok := seenLookup[uid]; ok {
|
||||
return
|
||||
}
|
||||
seenLookup[uid] = struct{}{}
|
||||
lookup = append(lookup, uid)
|
||||
}
|
||||
for _, row := range byUID {
|
||||
addLookup(row.uniqueID)
|
||||
if row.parent != nil {
|
||||
addLookup(*row.parent)
|
||||
}
|
||||
}
|
||||
if err := s.loadCategoryPaths(ctx, companyID, lookup, known); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
updIDs := make([]uuid.UUID, 0, len(order))
|
||||
updNames := make([]string, 0, len(order))
|
||||
updDescs := make([]string, 0, len(order))
|
||||
pendingInserts := make([]categoryCSVRow, 0, len(order))
|
||||
|
||||
for _, uid := range order {
|
||||
row := byUID[uid]
|
||||
if info, ok := known[uid]; ok {
|
||||
updIDs = append(updIDs, info.id)
|
||||
updNames = append(updNames, row.name)
|
||||
updDescs = append(updDescs, deref(row.desc))
|
||||
continue
|
||||
}
|
||||
pendingInserts = append(pendingInserts, row)
|
||||
}
|
||||
|
||||
tx, err := s.Pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
if len(updIDs) > 0 {
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE categories AS c SET
|
||||
name = v.name,
|
||||
description = CASE WHEN v.description <> '' THEN v.description ELSE c.description END,
|
||||
updated_at = now()
|
||||
FROM unnest($2::uuid[], $3::text[], $4::text[]) AS v(id, name, description)
|
||||
WHERE c.id = v.id AND c.company_id = $1`,
|
||||
companyID, updIDs, updNames, updDescs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res.Updated += len(updIDs)
|
||||
}
|
||||
|
||||
for len(pendingInserts) > 0 {
|
||||
insNames := make([]string, 0, len(pendingInserts))
|
||||
insUIDs := make([]string, 0, len(pendingInserts))
|
||||
insParents := make([]string, 0, len(pendingInserts))
|
||||
insDescs := make([]string, 0, len(pendingInserts))
|
||||
insPaths := make([]string, 0, len(pendingInserts))
|
||||
insLevels := make([]int32, 0, len(pendingInserts))
|
||||
next := make([]categoryCSVRow, 0, len(pendingInserts))
|
||||
|
||||
for _, row := range pendingInserts {
|
||||
path, level, parentVal, err := resolveCategoryPath(row.uniqueID, row.parent, known)
|
||||
if err != nil {
|
||||
next = append(next, row)
|
||||
continue
|
||||
}
|
||||
insNames = append(insNames, row.name)
|
||||
insUIDs = append(insUIDs, row.uniqueID)
|
||||
insParents = append(insParents, parentVal)
|
||||
insDescs = append(insDescs, deref(row.desc))
|
||||
insPaths = append(insPaths, path)
|
||||
insLevels = append(insLevels, int32(level))
|
||||
}
|
||||
|
||||
if len(insUIDs) == 0 {
|
||||
for _, row := range next {
|
||||
res.Skipped++
|
||||
res.addError(fmt.Sprintf("%s: parent category not found", row.uniqueID))
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
rows, err := tx.Query(ctx, `
|
||||
INSERT INTO categories (company_id, name, unique_id, parent_unique_id, description, path, level)
|
||||
SELECT $1, v.name, v.unique_id, NULLIF(v.parent_unique_id, ''), NULLIF(v.description, ''), v.path, v.level
|
||||
FROM unnest($2::text[], $3::text[], $4::text[], $5::text[], $6::text[], $7::int[])
|
||||
AS v(name, unique_id, parent_unique_id, description, path, level)
|
||||
RETURNING id, unique_id, COALESCE(path, ''), level`,
|
||||
companyID, insNames, insUIDs, insParents, insDescs, insPaths, insLevels)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var id uuid.UUID
|
||||
var uid, path string
|
||||
var level int
|
||||
if err := rows.Scan(&id, &uid, &path, &level); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
known[uid] = categoryPathInfo{id: id, path: path, level: level}
|
||||
res.Created++
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pendingInserts = next
|
||||
}
|
||||
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (s *Service) loadCategoryPaths(ctx context.Context, companyID uuid.UUID, uids []string, known map[string]categoryPathInfo) error {
|
||||
if len(uids) == 0 {
|
||||
return nil
|
||||
}
|
||||
rows, err := s.Pool.Query(ctx, `
|
||||
SELECT id, unique_id, COALESCE(path, ''), level
|
||||
FROM categories
|
||||
WHERE company_id = $1 AND unique_id = ANY($2)`, companyID, uids)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var info categoryPathInfo
|
||||
var uid string
|
||||
if err := rows.Scan(&info.id, &uid, &info.path, &info.level); err != nil {
|
||||
return err
|
||||
}
|
||||
known[uid] = info
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
func resolveCategoryPath(uniqueID string, parent *string, known map[string]categoryPathInfo) (path string, level int, parentVal string, err error) {
|
||||
path = uniqueID
|
||||
level = 0
|
||||
if parent == nil || strings.TrimSpace(*parent) == "" {
|
||||
return path, level, "", nil
|
||||
}
|
||||
p := strings.TrimSpace(*parent)
|
||||
pinfo, ok := known[p]
|
||||
if !ok {
|
||||
return "", 0, "", errors.New("parent category not found")
|
||||
}
|
||||
if pinfo.path != "" {
|
||||
path = pinfo.path + "/" + uniqueID
|
||||
} else {
|
||||
path = p + "/" + uniqueID
|
||||
}
|
||||
return path, pinfo.level + 1, p, nil
|
||||
}
|
||||
|
||||
func deref(p *string) string {
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return *p
|
||||
}
|
||||
|
||||
type attributeCSVRow struct {
|
||||
key, name, valueType string
|
||||
unit, example, parent *string
|
||||
}
|
||||
|
||||
func (s *Service) ImportAttributesCSV(ctx context.Context, companyID uuid.UUID, r io.Reader) (ImportResult, error) {
|
||||
res := ImportResult{}
|
||||
cr := csv.NewReader(r)
|
||||
cr.TrimLeadingSpace = true
|
||||
headers, err := cr.Read()
|
||||
if err != nil {
|
||||
return res, ClientMsg("empty or invalid CSV")
|
||||
}
|
||||
iKey := headerIndex(headers, "attribute_key", "key")
|
||||
iName := headerIndex(headers, "name")
|
||||
iType := headerIndex(headers, "value_type", "type")
|
||||
iUnit := headerIndex(headers, "unit")
|
||||
iExample := headerIndex(headers, "example")
|
||||
iParent := headerIndex(headers, "parent_key", "parent")
|
||||
if iKey < 0 || iName < 0 {
|
||||
return res, ClientMsg("CSV must include attribute_key and name columns")
|
||||
}
|
||||
|
||||
batch := make([]attributeCSVRow, 0, importCSVBatchSize)
|
||||
for {
|
||||
row, err := cr.Read()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
res.Skipped++
|
||||
res.addError(err.Error())
|
||||
continue
|
||||
}
|
||||
key := cell(row, iKey)
|
||||
name := cell(row, iName)
|
||||
if key == "" || name == "" {
|
||||
res.Skipped++
|
||||
continue
|
||||
}
|
||||
valueType := cell(row, iType)
|
||||
if valueType == "" {
|
||||
valueType = "string"
|
||||
}
|
||||
var unit, example, parent *string
|
||||
if u := cell(row, iUnit); u != "" {
|
||||
unit = &u
|
||||
}
|
||||
if e := cell(row, iExample); e != "" {
|
||||
example = &e
|
||||
}
|
||||
if p := cell(row, iParent); p != "" {
|
||||
parent = &p
|
||||
}
|
||||
batch = append(batch, attributeCSVRow{
|
||||
key: key, name: name, valueType: valueType,
|
||||
unit: unit, example: example, parent: parent,
|
||||
})
|
||||
if len(batch) >= importCSVBatchSize {
|
||||
if err := s.flushAttributeBatch(ctx, companyID, batch, &res); err != nil {
|
||||
return res, err
|
||||
}
|
||||
batch = batch[:0]
|
||||
}
|
||||
}
|
||||
if err := s.flushAttributeBatch(ctx, companyID, batch, &res); err != nil {
|
||||
return res, err
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (s *Service) flushAttributeBatch(ctx context.Context, companyID uuid.UUID, batch []attributeCSVRow, res *ImportResult) error {
|
||||
if len(batch) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
byKey := make(map[string]attributeCSVRow, len(batch))
|
||||
order := make([]string, 0, len(batch))
|
||||
for _, row := range batch {
|
||||
if _, ok := byKey[row.key]; !ok {
|
||||
order = append(order, row.key)
|
||||
}
|
||||
byKey[row.key] = row
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(byKey))
|
||||
keys = append(keys, order...)
|
||||
existing := map[string]uuid.UUID{}
|
||||
rows, err := s.Pool.Query(ctx, `
|
||||
SELECT id, attribute_key FROM attributes
|
||||
WHERE company_id = $1 AND attribute_key = ANY($2)`, companyID, keys)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var id uuid.UUID
|
||||
var key string
|
||||
if err := rows.Scan(&id, &key); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
existing[key] = id
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
updIDs := make([]uuid.UUID, 0, len(order))
|
||||
updNames := make([]string, 0, len(order))
|
||||
updTypes := make([]string, 0, len(order))
|
||||
insKeys := make([]string, 0, len(order))
|
||||
insNames := make([]string, 0, len(order))
|
||||
insTypes := make([]string, 0, len(order))
|
||||
insUnits := make([]string, 0, len(order))
|
||||
insExamples := make([]string, 0, len(order))
|
||||
insParents := make([]string, 0, len(order))
|
||||
|
||||
for _, key := range order {
|
||||
row := byKey[key]
|
||||
if id, ok := existing[key]; ok {
|
||||
updIDs = append(updIDs, id)
|
||||
updNames = append(updNames, row.name)
|
||||
updTypes = append(updTypes, row.valueType)
|
||||
continue
|
||||
}
|
||||
insKeys = append(insKeys, key)
|
||||
insNames = append(insNames, row.name)
|
||||
insTypes = append(insTypes, row.valueType)
|
||||
insUnits = append(insUnits, deref(row.unit))
|
||||
insExamples = append(insExamples, deref(row.example))
|
||||
insParents = append(insParents, deref(row.parent))
|
||||
}
|
||||
|
||||
tx, err := s.Pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
if len(updIDs) > 0 {
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE attributes AS a SET
|
||||
name = CASE WHEN v.name <> '' THEN v.name ELSE a.name END,
|
||||
value_type = CASE WHEN v.value_type <> '' THEN v.value_type ELSE a.value_type END,
|
||||
updated_at = now()
|
||||
FROM unnest($2::uuid[], $3::text[], $4::text[]) AS v(id, name, value_type)
|
||||
WHERE a.id = v.id AND a.company_id = $1`,
|
||||
companyID, updIDs, updNames, updTypes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res.Updated += len(updIDs)
|
||||
}
|
||||
|
||||
if len(insKeys) > 0 {
|
||||
ct, err := tx.Exec(ctx, `
|
||||
INSERT INTO attributes (company_id, attribute_key, name, value_type, unit, example, parent_key)
|
||||
SELECT $1, v.attribute_key, v.name, v.value_type,
|
||||
NULLIF(v.unit, ''), NULLIF(v.example, ''), NULLIF(v.parent_key, '')
|
||||
FROM unnest($2::text[], $3::text[], $4::text[], $5::text[], $6::text[], $7::text[])
|
||||
AS v(attribute_key, name, value_type, unit, example, parent_key)`,
|
||||
companyID, insKeys, insNames, insTypes, insUnits, insExamples, insParents)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res.Created += int(ct.RowsAffected())
|
||||
}
|
||||
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (s *Service) MergeProductsByGTIN(ctx context.Context, companyID uuid.UUID) (bool, error) {
|
||||
var merge bool
|
||||
err := s.Pool.QueryRow(ctx, `SELECT merge_products_by_gtin FROM companies WHERE id = $1`, companyID).Scan(&merge)
|
||||
return merge, err
|
||||
}
|
||||
|
||||
type productCSVRow struct {
|
||||
gtin, productID, name, category, desc, status string
|
||||
rawJSON string
|
||||
mergeable bool
|
||||
}
|
||||
|
||||
func (s *Service) ImportProductsCSV(ctx context.Context, companyID uuid.UUID, r io.Reader, fileID *uuid.UUID) (ImportResult, error) {
|
||||
res := ImportResult{}
|
||||
merge, err := s.MergeProductsByGTIN(ctx, companyID)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
cr := csv.NewReader(r)
|
||||
cr.TrimLeadingSpace = true
|
||||
headers, err := cr.Read()
|
||||
if err != nil {
|
||||
return res, ClientMsg("empty or invalid CSV")
|
||||
}
|
||||
iGTIN := headerIndex(headers, "gtin", "ean", "barcode")
|
||||
iPID := headerIndex(headers, "product_id", "sku", "id")
|
||||
iName := headerIndex(headers, "name", "title")
|
||||
iCat := headerIndex(headers, "category")
|
||||
iDesc := headerIndex(headers, "description")
|
||||
iStatus := headerIndex(headers, "status")
|
||||
if iName < 0 && iGTIN < 0 && iPID < 0 {
|
||||
return res, ClientMsg("CSV must include name, gtin, or product_id")
|
||||
}
|
||||
|
||||
batch := make([]productCSVRow, 0, importCSVBatchSize)
|
||||
for {
|
||||
row, err := cr.Read()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
res.Skipped++
|
||||
res.addError(err.Error())
|
||||
continue
|
||||
}
|
||||
gtin := cell(row, iGTIN)
|
||||
productID := cell(row, iPID)
|
||||
name := cell(row, iName)
|
||||
category := cell(row, iCat)
|
||||
desc := cell(row, iDesc)
|
||||
status := cell(row, iStatus)
|
||||
if status == "" {
|
||||
status = "draft"
|
||||
}
|
||||
if gtin == "" && productID == "" && name == "" {
|
||||
res.Skipped++
|
||||
continue
|
||||
}
|
||||
if gtin == "" {
|
||||
gtin = "nogtin-" + uuid.NewString()
|
||||
}
|
||||
rawData, _ := json.Marshal(map[string]any{
|
||||
"product_id": productID,
|
||||
"name": name,
|
||||
"category": category,
|
||||
"description": desc,
|
||||
"status": status,
|
||||
"gtin": gtin,
|
||||
})
|
||||
batch = append(batch, productCSVRow{
|
||||
gtin: gtin, productID: productID, name: name, category: category,
|
||||
desc: desc, status: status, rawJSON: string(rawData),
|
||||
mergeable: merge && !strings.HasPrefix(gtin, "nogtin-"),
|
||||
})
|
||||
if len(batch) >= importCSVBatchSize {
|
||||
if err := s.flushProductBatch(ctx, companyID, fileID, batch, &res); err != nil {
|
||||
return res, err
|
||||
}
|
||||
batch = batch[:0]
|
||||
}
|
||||
}
|
||||
if err := s.flushProductBatch(ctx, companyID, fileID, batch, &res); err != nil {
|
||||
return res, err
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (s *Service) flushProductBatch(ctx context.Context, companyID uuid.UUID, fileID *uuid.UUID, batch []productCSVRow, res *ImportResult) error {
|
||||
if len(batch) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Last row wins per GTIN so a single INSERT cannot hit the same unique key twice.
|
||||
byGTIN := make(map[string]productCSVRow, len(batch))
|
||||
order := make([]string, 0, len(batch))
|
||||
for _, row := range batch {
|
||||
if _, ok := byGTIN[row.gtin]; !ok {
|
||||
order = append(order, row.gtin)
|
||||
}
|
||||
byGTIN[row.gtin] = row
|
||||
}
|
||||
deduped := make([]productCSVRow, 0, len(order))
|
||||
for _, gtin := range order {
|
||||
deduped = append(deduped, byGTIN[gtin])
|
||||
}
|
||||
batch = deduped
|
||||
|
||||
mergeGTINs := make([]string, 0, len(batch))
|
||||
for _, row := range batch {
|
||||
if row.mergeable {
|
||||
mergeGTINs = append(mergeGTINs, row.gtin)
|
||||
}
|
||||
}
|
||||
|
||||
existingRaw := map[string]uuid.UUID{}
|
||||
if len(mergeGTINs) > 0 {
|
||||
rows, err := s.Pool.Query(ctx, `
|
||||
SELECT DISTINCT ON (gtin) id, gtin
|
||||
FROM raw_products
|
||||
WHERE company_id = $1 AND gtin = ANY($2)
|
||||
ORDER BY gtin, updated_at DESC`, companyID, mergeGTINs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var id uuid.UUID
|
||||
var gtin string
|
||||
if err := rows.Scan(&id, >in); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
existingRaw[gtin] = id
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
type pendingProcessed struct {
|
||||
rawID uuid.UUID
|
||||
productID, name, category, desc, status string
|
||||
rawWasUpdate bool
|
||||
}
|
||||
|
||||
tx, err := s.Pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
updRawIDs := make([]uuid.UUID, 0, len(batch))
|
||||
updRawJSON := make([]string, 0, len(batch))
|
||||
updRawMeta := make([]pendingProcessed, 0, len(batch))
|
||||
|
||||
insGTINs := make([]string, 0, len(batch))
|
||||
insJSON := make([]string, 0, len(batch))
|
||||
insMeta := make([]pendingProcessed, 0, len(batch))
|
||||
|
||||
for _, row := range batch {
|
||||
meta := pendingProcessed{
|
||||
productID: row.productID, name: row.name, category: row.category,
|
||||
desc: row.desc, status: row.status,
|
||||
}
|
||||
if row.mergeable {
|
||||
if id, ok := existingRaw[row.gtin]; ok {
|
||||
updRawIDs = append(updRawIDs, id)
|
||||
updRawJSON = append(updRawJSON, row.rawJSON)
|
||||
meta.rawID = id
|
||||
meta.rawWasUpdate = true
|
||||
updRawMeta = append(updRawMeta, meta)
|
||||
continue
|
||||
}
|
||||
}
|
||||
insGTINs = append(insGTINs, row.gtin)
|
||||
insJSON = append(insJSON, row.rawJSON)
|
||||
insMeta = append(insMeta, meta)
|
||||
}
|
||||
|
||||
if len(updRawIDs) > 0 {
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE raw_products AS r SET
|
||||
raw_data = v.raw_data::jsonb,
|
||||
mapped_data = v.raw_data::jsonb,
|
||||
file_id = COALESCE($3, r.file_id),
|
||||
updated_at = now()
|
||||
FROM unnest($2::uuid[], $4::text[]) AS v(id, raw_data)
|
||||
WHERE r.id = v.id AND r.company_id = $1`,
|
||||
companyID, updRawIDs, fileID, updRawJSON)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
pending := make([]pendingProcessed, 0, len(batch))
|
||||
pending = append(pending, updRawMeta...)
|
||||
|
||||
if len(insGTINs) > 0 {
|
||||
rows, err := tx.Query(ctx, `
|
||||
INSERT INTO raw_products (company_id, gtin, raw_data, mapped_data, processing_status, file_id)
|
||||
SELECT $1, v.gtin, v.raw_data::jsonb, v.raw_data::jsonb, 'unprocessed', $2
|
||||
FROM unnest($3::text[], $4::text[]) AS v(gtin, raw_data)
|
||||
ON CONFLICT (company_id, gtin) DO NOTHING
|
||||
RETURNING id, gtin`,
|
||||
companyID, fileID, insGTINs, insJSON)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
insertedByGTIN := map[string]uuid.UUID{}
|
||||
for rows.Next() {
|
||||
var id uuid.UUID
|
||||
var gtin string
|
||||
if err := rows.Scan(&id, >in); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
insertedByGTIN[gtin] = id
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Match inserts back to input order; conflicts (DO NOTHING) are skipped.
|
||||
for i, gtin := range insGTINs {
|
||||
id, ok := insertedByGTIN[gtin]
|
||||
if !ok {
|
||||
res.Skipped++
|
||||
res.addError(fmt.Sprintf("%s: duplicate gtin", gtin))
|
||||
continue
|
||||
}
|
||||
meta := insMeta[i]
|
||||
meta.rawID = id
|
||||
pending = append(pending, meta)
|
||||
// Same gtin inserted twice in one batch: second RETURNING miss.
|
||||
delete(insertedByGTIN, gtin)
|
||||
}
|
||||
}
|
||||
|
||||
if len(pending) == 0 {
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
rawIDs := make([]uuid.UUID, len(pending))
|
||||
for i, p := range pending {
|
||||
rawIDs[i] = p.rawID
|
||||
}
|
||||
existingProcessed := map[uuid.UUID]uuid.UUID{}
|
||||
prows, err := tx.Query(ctx, `
|
||||
SELECT DISTINCT ON (raw_product_id) id, raw_product_id
|
||||
FROM processed_products
|
||||
WHERE company_id = $1 AND raw_product_id = ANY($2)
|
||||
ORDER BY raw_product_id, updated_at DESC`, companyID, rawIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for prows.Next() {
|
||||
var id, rawID uuid.UUID
|
||||
if err := prows.Scan(&id, &rawID); err != nil {
|
||||
prows.Close()
|
||||
return err
|
||||
}
|
||||
existingProcessed[rawID] = id
|
||||
}
|
||||
err = prows.Err()
|
||||
prows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
updProcIDs := make([]uuid.UUID, 0, len(pending))
|
||||
updPIDs := make([]string, 0, len(pending))
|
||||
updNames := make([]string, 0, len(pending))
|
||||
updCats := make([]string, 0, len(pending))
|
||||
updDescs := make([]string, 0, len(pending))
|
||||
updStatuses := make([]string, 0, len(pending))
|
||||
|
||||
insRawIDs := make([]uuid.UUID, 0, len(pending))
|
||||
insPIDs := make([]string, 0, len(pending))
|
||||
insNames := make([]string, 0, len(pending))
|
||||
insCats := make([]string, 0, len(pending))
|
||||
insDescs := make([]string, 0, len(pending))
|
||||
insStatuses := make([]string, 0, len(pending))
|
||||
insWasUpdate := make([]bool, 0, len(pending))
|
||||
|
||||
for _, p := range pending {
|
||||
if pid, ok := existingProcessed[p.rawID]; ok {
|
||||
updProcIDs = append(updProcIDs, pid)
|
||||
updPIDs = append(updPIDs, p.productID)
|
||||
updNames = append(updNames, p.name)
|
||||
updCats = append(updCats, p.category)
|
||||
updDescs = append(updDescs, p.desc)
|
||||
updStatuses = append(updStatuses, p.status)
|
||||
continue
|
||||
}
|
||||
insRawIDs = append(insRawIDs, p.rawID)
|
||||
insPIDs = append(insPIDs, p.productID)
|
||||
insNames = append(insNames, p.name)
|
||||
insCats = append(insCats, p.category)
|
||||
insDescs = append(insDescs, p.desc)
|
||||
insStatuses = append(insStatuses, p.status)
|
||||
insWasUpdate = append(insWasUpdate, p.rawWasUpdate)
|
||||
}
|
||||
|
||||
if len(updProcIDs) > 0 {
|
||||
_, err = tx.Exec(ctx, `
|
||||
UPDATE processed_products AS p SET
|
||||
product_id = COALESCE(NULLIF(v.product_id, ''), p.product_id),
|
||||
name = COALESCE(NULLIF(v.name, ''), p.name),
|
||||
category = COALESCE(NULLIF(v.category, ''), p.category),
|
||||
description = COALESCE(NULLIF(v.description, ''), p.description),
|
||||
status = COALESCE(NULLIF(v.status, ''), p.status),
|
||||
updated_at = now()
|
||||
FROM unnest($2::uuid[], $3::text[], $4::text[], $5::text[], $6::text[], $7::text[])
|
||||
AS v(id, product_id, name, category, description, status)
|
||||
WHERE p.id = v.id AND p.company_id = $1`,
|
||||
companyID, updProcIDs, updPIDs, updNames, updCats, updDescs, updStatuses)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res.Updated += len(updProcIDs)
|
||||
}
|
||||
|
||||
if len(insRawIDs) > 0 {
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO processed_products (company_id, product_id, name, category, description, status, raw_product_id)
|
||||
SELECT $1,
|
||||
NULLIF(v.product_id, ''), NULLIF(v.name, ''), NULLIF(v.category, ''),
|
||||
NULLIF(v.description, ''), v.status, v.raw_product_id
|
||||
FROM unnest($2::uuid[], $3::text[], $4::text[], $5::text[], $6::text[], $7::text[])
|
||||
AS v(raw_product_id, product_id, name, category, description, status)`,
|
||||
companyID, insRawIDs, insPIDs, insNames, insCats, insDescs, insStatuses)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, wasUpdate := range insWasUpdate {
|
||||
if wasUpdate {
|
||||
res.Updated++
|
||||
} else {
|
||||
res.Created++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
Reference in New Issue
Block a user