831 lines
22 KiB
Go
831 lines
22 KiB
Go
|
package catalog
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"encoding/csv"
|
||
|
|
"encoding/json"
|
||
|
|
"errors"
|
||
|
|
"fmt"
|
||
|
|
"io"
|
||
|
|
"strings"
|
||
|
|
|
||
|
|
"github.com/google/uuid"
|
||
|
|
)
|
||
|
|
|
||
|
|
const (
|
||
|
|
importCSVBatchSize = 500
|
||
|
|
importCSVMaxErrors = 50
|
||
|
|
)
|
||
|
|
|
||
|
|
type ImportResult struct {
|
||
|
|
Created int `json:"created"`
|
||
|
|
Updated int `json:"updated"`
|
||
|
|
Skipped int `json:"skipped"`
|
||
|
|
Errors []string `json:"errors,omitempty"`
|
||
|
|
}
|
||
|
|
|
||
|
|
func (r *ImportResult) addError(msg string) {
|
||
|
|
if len(r.Errors) >= importCSVMaxErrors {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
r.Errors = append(r.Errors, msg)
|
||
|
|
}
|
||
|
|
|
||
|
|
func headerIndex(headers []string, names ...string) int {
|
||
|
|
want := map[string]struct{}{}
|
||
|
|
for _, n := range names {
|
||
|
|
want[strings.ToLower(strings.TrimSpace(n))] = struct{}{}
|
||
|
|
}
|
||
|
|
for i, h := range headers {
|
||
|
|
if _, ok := want[strings.ToLower(strings.TrimSpace(h))]; ok {
|
||
|
|
return i
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return -1
|
||
|
|
}
|
||
|
|
|
||
|
|
func cell(row []string, idx int) string {
|
||
|
|
if idx < 0 || idx >= len(row) {
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
return strings.TrimSpace(row[idx])
|
||
|
|
}
|
||
|
|
|
||
|
|
type categoryCSVRow struct {
|
||
|
|
name string
|
||
|
|
uniqueID string
|
||
|
|
parent *string
|
||
|
|
desc *string
|
||
|
|
}
|
||
|
|
|
||
|
|
type categoryPathInfo struct {
|
||
|
|
id uuid.UUID
|
||
|
|
path string
|
||
|
|
level int
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) ImportCategoriesCSV(ctx context.Context, companyID uuid.UUID, r io.Reader) (ImportResult, error) {
|
||
|
|
res := ImportResult{}
|
||
|
|
cr := csv.NewReader(r)
|
||
|
|
cr.TrimLeadingSpace = true
|
||
|
|
headers, err := cr.Read()
|
||
|
|
if err != nil {
|
||
|
|
return res, ClientMsg("empty or invalid CSV")
|
||
|
|
}
|
||
|
|
iName := headerIndex(headers, "name")
|
||
|
|
iUID := headerIndex(headers, "unique_id", "id", "category_id")
|
||
|
|
iParent := headerIndex(headers, "parent_unique_id", "parent_id", "parent")
|
||
|
|
iDesc := headerIndex(headers, "description")
|
||
|
|
if iName < 0 || iUID < 0 {
|
||
|
|
return res, ClientMsg("CSV must include name and unique_id columns")
|
||
|
|
}
|
||
|
|
|
||
|
|
known := map[string]categoryPathInfo{}
|
||
|
|
batch := make([]categoryCSVRow, 0, importCSVBatchSize)
|
||
|
|
for {
|
||
|
|
row, err := cr.Read()
|
||
|
|
if errors.Is(err, io.EOF) {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
if err != nil {
|
||
|
|
res.Skipped++
|
||
|
|
res.addError(err.Error())
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
name := cell(row, iName)
|
||
|
|
uid := cell(row, iUID)
|
||
|
|
if name == "" || uid == "" {
|
||
|
|
res.Skipped++
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
var parent *string
|
||
|
|
if p := cell(row, iParent); p != "" {
|
||
|
|
parent = &p
|
||
|
|
}
|
||
|
|
var desc *string
|
||
|
|
if d := cell(row, iDesc); d != "" {
|
||
|
|
desc = &d
|
||
|
|
}
|
||
|
|
batch = append(batch, categoryCSVRow{name: name, uniqueID: uid, parent: parent, desc: desc})
|
||
|
|
if len(batch) >= importCSVBatchSize {
|
||
|
|
if err := s.flushCategoryBatch(ctx, companyID, batch, known, &res); err != nil {
|
||
|
|
return res, err
|
||
|
|
}
|
||
|
|
batch = batch[:0]
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if err := s.flushCategoryBatch(ctx, companyID, batch, known, &res); err != nil {
|
||
|
|
return res, err
|
||
|
|
}
|
||
|
|
return res, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) flushCategoryBatch(ctx context.Context, companyID uuid.UUID, batch []categoryCSVRow, known map[string]categoryPathInfo, res *ImportResult) error {
|
||
|
|
if len(batch) == 0 {
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
byUID := make(map[string]categoryCSVRow, len(batch))
|
||
|
|
order := make([]string, 0, len(batch))
|
||
|
|
for _, row := range batch {
|
||
|
|
if _, ok := byUID[row.uniqueID]; !ok {
|
||
|
|
order = append(order, row.uniqueID)
|
||
|
|
}
|
||
|
|
byUID[row.uniqueID] = row
|
||
|
|
}
|
||
|
|
|
||
|
|
lookup := make([]string, 0, len(byUID)*2)
|
||
|
|
seenLookup := map[string]struct{}{}
|
||
|
|
addLookup := func(uid string) {
|
||
|
|
if uid == "" {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if _, ok := known[uid]; ok {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
if _, ok := seenLookup[uid]; ok {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
seenLookup[uid] = struct{}{}
|
||
|
|
lookup = append(lookup, uid)
|
||
|
|
}
|
||
|
|
for _, row := range byUID {
|
||
|
|
addLookup(row.uniqueID)
|
||
|
|
if row.parent != nil {
|
||
|
|
addLookup(*row.parent)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if err := s.loadCategoryPaths(ctx, companyID, lookup, known); err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
updIDs := make([]uuid.UUID, 0, len(order))
|
||
|
|
updNames := make([]string, 0, len(order))
|
||
|
|
updDescs := make([]string, 0, len(order))
|
||
|
|
pendingInserts := make([]categoryCSVRow, 0, len(order))
|
||
|
|
|
||
|
|
for _, uid := range order {
|
||
|
|
row := byUID[uid]
|
||
|
|
if info, ok := known[uid]; ok {
|
||
|
|
updIDs = append(updIDs, info.id)
|
||
|
|
updNames = append(updNames, row.name)
|
||
|
|
updDescs = append(updDescs, deref(row.desc))
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
pendingInserts = append(pendingInserts, row)
|
||
|
|
}
|
||
|
|
|
||
|
|
tx, err := s.Pool.Begin(ctx)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
defer tx.Rollback(ctx)
|
||
|
|
|
||
|
|
if len(updIDs) > 0 {
|
||
|
|
_, err = tx.Exec(ctx, `
|
||
|
|
UPDATE categories AS c SET
|
||
|
|
name = v.name,
|
||
|
|
description = CASE WHEN v.description <> '' THEN v.description ELSE c.description END,
|
||
|
|
updated_at = now()
|
||
|
|
FROM unnest($2::uuid[], $3::text[], $4::text[]) AS v(id, name, description)
|
||
|
|
WHERE c.id = v.id AND c.company_id = $1`,
|
||
|
|
companyID, updIDs, updNames, updDescs)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
res.Updated += len(updIDs)
|
||
|
|
}
|
||
|
|
|
||
|
|
for len(pendingInserts) > 0 {
|
||
|
|
insNames := make([]string, 0, len(pendingInserts))
|
||
|
|
insUIDs := make([]string, 0, len(pendingInserts))
|
||
|
|
insParents := make([]string, 0, len(pendingInserts))
|
||
|
|
insDescs := make([]string, 0, len(pendingInserts))
|
||
|
|
insPaths := make([]string, 0, len(pendingInserts))
|
||
|
|
insLevels := make([]int32, 0, len(pendingInserts))
|
||
|
|
next := make([]categoryCSVRow, 0, len(pendingInserts))
|
||
|
|
|
||
|
|
for _, row := range pendingInserts {
|
||
|
|
path, level, parentVal, err := resolveCategoryPath(row.uniqueID, row.parent, known)
|
||
|
|
if err != nil {
|
||
|
|
next = append(next, row)
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
insNames = append(insNames, row.name)
|
||
|
|
insUIDs = append(insUIDs, row.uniqueID)
|
||
|
|
insParents = append(insParents, parentVal)
|
||
|
|
insDescs = append(insDescs, deref(row.desc))
|
||
|
|
insPaths = append(insPaths, path)
|
||
|
|
insLevels = append(insLevels, int32(level))
|
||
|
|
}
|
||
|
|
|
||
|
|
if len(insUIDs) == 0 {
|
||
|
|
for _, row := range next {
|
||
|
|
res.Skipped++
|
||
|
|
res.addError(fmt.Sprintf("%s: parent category not found", row.uniqueID))
|
||
|
|
}
|
||
|
|
break
|
||
|
|
}
|
||
|
|
|
||
|
|
rows, err := tx.Query(ctx, `
|
||
|
|
INSERT INTO categories (company_id, name, unique_id, parent_unique_id, description, path, level)
|
||
|
|
SELECT $1, v.name, v.unique_id, NULLIF(v.parent_unique_id, ''), NULLIF(v.description, ''), v.path, v.level
|
||
|
|
FROM unnest($2::text[], $3::text[], $4::text[], $5::text[], $6::text[], $7::int[])
|
||
|
|
AS v(name, unique_id, parent_unique_id, description, path, level)
|
||
|
|
RETURNING id, unique_id, COALESCE(path, ''), level`,
|
||
|
|
companyID, insNames, insUIDs, insParents, insDescs, insPaths, insLevels)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
for rows.Next() {
|
||
|
|
var id uuid.UUID
|
||
|
|
var uid, path string
|
||
|
|
var level int
|
||
|
|
if err := rows.Scan(&id, &uid, &path, &level); err != nil {
|
||
|
|
rows.Close()
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
known[uid] = categoryPathInfo{id: id, path: path, level: level}
|
||
|
|
res.Created++
|
||
|
|
}
|
||
|
|
err = rows.Err()
|
||
|
|
rows.Close()
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
pendingInserts = next
|
||
|
|
}
|
||
|
|
|
||
|
|
return tx.Commit(ctx)
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) loadCategoryPaths(ctx context.Context, companyID uuid.UUID, uids []string, known map[string]categoryPathInfo) error {
|
||
|
|
if len(uids) == 0 {
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
rows, err := s.Pool.Query(ctx, `
|
||
|
|
SELECT id, unique_id, COALESCE(path, ''), level
|
||
|
|
FROM categories
|
||
|
|
WHERE company_id = $1 AND unique_id = ANY($2)`, companyID, uids)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
defer rows.Close()
|
||
|
|
for rows.Next() {
|
||
|
|
var info categoryPathInfo
|
||
|
|
var uid string
|
||
|
|
if err := rows.Scan(&info.id, &uid, &info.path, &info.level); err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
known[uid] = info
|
||
|
|
}
|
||
|
|
return rows.Err()
|
||
|
|
}
|
||
|
|
|
||
|
|
func resolveCategoryPath(uniqueID string, parent *string, known map[string]categoryPathInfo) (path string, level int, parentVal string, err error) {
|
||
|
|
path = uniqueID
|
||
|
|
level = 0
|
||
|
|
if parent == nil || strings.TrimSpace(*parent) == "" {
|
||
|
|
return path, level, "", nil
|
||
|
|
}
|
||
|
|
p := strings.TrimSpace(*parent)
|
||
|
|
pinfo, ok := known[p]
|
||
|
|
if !ok {
|
||
|
|
return "", 0, "", errors.New("parent category not found")
|
||
|
|
}
|
||
|
|
if pinfo.path != "" {
|
||
|
|
path = pinfo.path + "/" + uniqueID
|
||
|
|
} else {
|
||
|
|
path = p + "/" + uniqueID
|
||
|
|
}
|
||
|
|
return path, pinfo.level + 1, p, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func deref(p *string) string {
|
||
|
|
if p == nil {
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
return *p
|
||
|
|
}
|
||
|
|
|
||
|
|
type attributeCSVRow struct {
|
||
|
|
key, name, valueType string
|
||
|
|
unit, example, parent *string
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) ImportAttributesCSV(ctx context.Context, companyID uuid.UUID, r io.Reader) (ImportResult, error) {
|
||
|
|
res := ImportResult{}
|
||
|
|
cr := csv.NewReader(r)
|
||
|
|
cr.TrimLeadingSpace = true
|
||
|
|
headers, err := cr.Read()
|
||
|
|
if err != nil {
|
||
|
|
return res, ClientMsg("empty or invalid CSV")
|
||
|
|
}
|
||
|
|
iKey := headerIndex(headers, "attribute_key", "key")
|
||
|
|
iName := headerIndex(headers, "name")
|
||
|
|
iType := headerIndex(headers, "value_type", "type")
|
||
|
|
iUnit := headerIndex(headers, "unit")
|
||
|
|
iExample := headerIndex(headers, "example")
|
||
|
|
iParent := headerIndex(headers, "parent_key", "parent")
|
||
|
|
if iKey < 0 || iName < 0 {
|
||
|
|
return res, ClientMsg("CSV must include attribute_key and name columns")
|
||
|
|
}
|
||
|
|
|
||
|
|
batch := make([]attributeCSVRow, 0, importCSVBatchSize)
|
||
|
|
for {
|
||
|
|
row, err := cr.Read()
|
||
|
|
if errors.Is(err, io.EOF) {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
if err != nil {
|
||
|
|
res.Skipped++
|
||
|
|
res.addError(err.Error())
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
key := cell(row, iKey)
|
||
|
|
name := cell(row, iName)
|
||
|
|
if key == "" || name == "" {
|
||
|
|
res.Skipped++
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
valueType := cell(row, iType)
|
||
|
|
if valueType == "" {
|
||
|
|
valueType = "string"
|
||
|
|
}
|
||
|
|
var unit, example, parent *string
|
||
|
|
if u := cell(row, iUnit); u != "" {
|
||
|
|
unit = &u
|
||
|
|
}
|
||
|
|
if e := cell(row, iExample); e != "" {
|
||
|
|
example = &e
|
||
|
|
}
|
||
|
|
if p := cell(row, iParent); p != "" {
|
||
|
|
parent = &p
|
||
|
|
}
|
||
|
|
batch = append(batch, attributeCSVRow{
|
||
|
|
key: key, name: name, valueType: valueType,
|
||
|
|
unit: unit, example: example, parent: parent,
|
||
|
|
})
|
||
|
|
if len(batch) >= importCSVBatchSize {
|
||
|
|
if err := s.flushAttributeBatch(ctx, companyID, batch, &res); err != nil {
|
||
|
|
return res, err
|
||
|
|
}
|
||
|
|
batch = batch[:0]
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if err := s.flushAttributeBatch(ctx, companyID, batch, &res); err != nil {
|
||
|
|
return res, err
|
||
|
|
}
|
||
|
|
return res, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) flushAttributeBatch(ctx context.Context, companyID uuid.UUID, batch []attributeCSVRow, res *ImportResult) error {
|
||
|
|
if len(batch) == 0 {
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
byKey := make(map[string]attributeCSVRow, len(batch))
|
||
|
|
order := make([]string, 0, len(batch))
|
||
|
|
for _, row := range batch {
|
||
|
|
if _, ok := byKey[row.key]; !ok {
|
||
|
|
order = append(order, row.key)
|
||
|
|
}
|
||
|
|
byKey[row.key] = row
|
||
|
|
}
|
||
|
|
|
||
|
|
keys := make([]string, 0, len(byKey))
|
||
|
|
keys = append(keys, order...)
|
||
|
|
existing := map[string]uuid.UUID{}
|
||
|
|
rows, err := s.Pool.Query(ctx, `
|
||
|
|
SELECT id, attribute_key FROM attributes
|
||
|
|
WHERE company_id = $1 AND attribute_key = ANY($2)`, companyID, keys)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
for rows.Next() {
|
||
|
|
var id uuid.UUID
|
||
|
|
var key string
|
||
|
|
if err := rows.Scan(&id, &key); err != nil {
|
||
|
|
rows.Close()
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
existing[key] = id
|
||
|
|
}
|
||
|
|
err = rows.Err()
|
||
|
|
rows.Close()
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
updIDs := make([]uuid.UUID, 0, len(order))
|
||
|
|
updNames := make([]string, 0, len(order))
|
||
|
|
updTypes := make([]string, 0, len(order))
|
||
|
|
insKeys := make([]string, 0, len(order))
|
||
|
|
insNames := make([]string, 0, len(order))
|
||
|
|
insTypes := make([]string, 0, len(order))
|
||
|
|
insUnits := make([]string, 0, len(order))
|
||
|
|
insExamples := make([]string, 0, len(order))
|
||
|
|
insParents := make([]string, 0, len(order))
|
||
|
|
|
||
|
|
for _, key := range order {
|
||
|
|
row := byKey[key]
|
||
|
|
if id, ok := existing[key]; ok {
|
||
|
|
updIDs = append(updIDs, id)
|
||
|
|
updNames = append(updNames, row.name)
|
||
|
|
updTypes = append(updTypes, row.valueType)
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
insKeys = append(insKeys, key)
|
||
|
|
insNames = append(insNames, row.name)
|
||
|
|
insTypes = append(insTypes, row.valueType)
|
||
|
|
insUnits = append(insUnits, deref(row.unit))
|
||
|
|
insExamples = append(insExamples, deref(row.example))
|
||
|
|
insParents = append(insParents, deref(row.parent))
|
||
|
|
}
|
||
|
|
|
||
|
|
tx, err := s.Pool.Begin(ctx)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
defer tx.Rollback(ctx)
|
||
|
|
|
||
|
|
if len(updIDs) > 0 {
|
||
|
|
_, err = tx.Exec(ctx, `
|
||
|
|
UPDATE attributes AS a SET
|
||
|
|
name = CASE WHEN v.name <> '' THEN v.name ELSE a.name END,
|
||
|
|
value_type = CASE WHEN v.value_type <> '' THEN v.value_type ELSE a.value_type END,
|
||
|
|
updated_at = now()
|
||
|
|
FROM unnest($2::uuid[], $3::text[], $4::text[]) AS v(id, name, value_type)
|
||
|
|
WHERE a.id = v.id AND a.company_id = $1`,
|
||
|
|
companyID, updIDs, updNames, updTypes)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
res.Updated += len(updIDs)
|
||
|
|
}
|
||
|
|
|
||
|
|
if len(insKeys) > 0 {
|
||
|
|
ct, err := tx.Exec(ctx, `
|
||
|
|
INSERT INTO attributes (company_id, attribute_key, name, value_type, unit, example, parent_key)
|
||
|
|
SELECT $1, v.attribute_key, v.name, v.value_type,
|
||
|
|
NULLIF(v.unit, ''), NULLIF(v.example, ''), NULLIF(v.parent_key, '')
|
||
|
|
FROM unnest($2::text[], $3::text[], $4::text[], $5::text[], $6::text[], $7::text[])
|
||
|
|
AS v(attribute_key, name, value_type, unit, example, parent_key)`,
|
||
|
|
companyID, insKeys, insNames, insTypes, insUnits, insExamples, insParents)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
res.Created += int(ct.RowsAffected())
|
||
|
|
}
|
||
|
|
|
||
|
|
return tx.Commit(ctx)
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) MergeProductsByGTIN(ctx context.Context, companyID uuid.UUID) (bool, error) {
|
||
|
|
var merge bool
|
||
|
|
err := s.Pool.QueryRow(ctx, `SELECT merge_products_by_gtin FROM companies WHERE id = $1`, companyID).Scan(&merge)
|
||
|
|
return merge, err
|
||
|
|
}
|
||
|
|
|
||
|
|
type productCSVRow struct {
|
||
|
|
gtin, productID, name, category, desc, status string
|
||
|
|
rawJSON string
|
||
|
|
mergeable bool
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) ImportProductsCSV(ctx context.Context, companyID uuid.UUID, r io.Reader, fileID *uuid.UUID) (ImportResult, error) {
|
||
|
|
res := ImportResult{}
|
||
|
|
merge, err := s.MergeProductsByGTIN(ctx, companyID)
|
||
|
|
if err != nil {
|
||
|
|
return res, err
|
||
|
|
}
|
||
|
|
cr := csv.NewReader(r)
|
||
|
|
cr.TrimLeadingSpace = true
|
||
|
|
headers, err := cr.Read()
|
||
|
|
if err != nil {
|
||
|
|
return res, ClientMsg("empty or invalid CSV")
|
||
|
|
}
|
||
|
|
iGTIN := headerIndex(headers, "gtin", "ean", "barcode")
|
||
|
|
iPID := headerIndex(headers, "product_id", "sku", "id")
|
||
|
|
iName := headerIndex(headers, "name", "title")
|
||
|
|
iCat := headerIndex(headers, "category")
|
||
|
|
iDesc := headerIndex(headers, "description")
|
||
|
|
iStatus := headerIndex(headers, "status")
|
||
|
|
if iName < 0 && iGTIN < 0 && iPID < 0 {
|
||
|
|
return res, ClientMsg("CSV must include name, gtin, or product_id")
|
||
|
|
}
|
||
|
|
|
||
|
|
batch := make([]productCSVRow, 0, importCSVBatchSize)
|
||
|
|
for {
|
||
|
|
row, err := cr.Read()
|
||
|
|
if errors.Is(err, io.EOF) {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
if err != nil {
|
||
|
|
res.Skipped++
|
||
|
|
res.addError(err.Error())
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
gtin := cell(row, iGTIN)
|
||
|
|
productID := cell(row, iPID)
|
||
|
|
name := cell(row, iName)
|
||
|
|
category := cell(row, iCat)
|
||
|
|
desc := cell(row, iDesc)
|
||
|
|
status := cell(row, iStatus)
|
||
|
|
if status == "" {
|
||
|
|
status = "draft"
|
||
|
|
}
|
||
|
|
if gtin == "" && productID == "" && name == "" {
|
||
|
|
res.Skipped++
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
if gtin == "" {
|
||
|
|
gtin = "nogtin-" + uuid.NewString()
|
||
|
|
}
|
||
|
|
rawData, _ := json.Marshal(map[string]any{
|
||
|
|
"product_id": productID,
|
||
|
|
"name": name,
|
||
|
|
"category": category,
|
||
|
|
"description": desc,
|
||
|
|
"status": status,
|
||
|
|
"gtin": gtin,
|
||
|
|
})
|
||
|
|
batch = append(batch, productCSVRow{
|
||
|
|
gtin: gtin, productID: productID, name: name, category: category,
|
||
|
|
desc: desc, status: status, rawJSON: string(rawData),
|
||
|
|
mergeable: merge && !strings.HasPrefix(gtin, "nogtin-"),
|
||
|
|
})
|
||
|
|
if len(batch) >= importCSVBatchSize {
|
||
|
|
if err := s.flushProductBatch(ctx, companyID, fileID, batch, &res); err != nil {
|
||
|
|
return res, err
|
||
|
|
}
|
||
|
|
batch = batch[:0]
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if err := s.flushProductBatch(ctx, companyID, fileID, batch, &res); err != nil {
|
||
|
|
return res, err
|
||
|
|
}
|
||
|
|
return res, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Service) flushProductBatch(ctx context.Context, companyID uuid.UUID, fileID *uuid.UUID, batch []productCSVRow, res *ImportResult) error {
|
||
|
|
if len(batch) == 0 {
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// Last row wins per GTIN so a single INSERT cannot hit the same unique key twice.
|
||
|
|
byGTIN := make(map[string]productCSVRow, len(batch))
|
||
|
|
order := make([]string, 0, len(batch))
|
||
|
|
for _, row := range batch {
|
||
|
|
if _, ok := byGTIN[row.gtin]; !ok {
|
||
|
|
order = append(order, row.gtin)
|
||
|
|
}
|
||
|
|
byGTIN[row.gtin] = row
|
||
|
|
}
|
||
|
|
deduped := make([]productCSVRow, 0, len(order))
|
||
|
|
for _, gtin := range order {
|
||
|
|
deduped = append(deduped, byGTIN[gtin])
|
||
|
|
}
|
||
|
|
batch = deduped
|
||
|
|
|
||
|
|
mergeGTINs := make([]string, 0, len(batch))
|
||
|
|
for _, row := range batch {
|
||
|
|
if row.mergeable {
|
||
|
|
mergeGTINs = append(mergeGTINs, row.gtin)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
existingRaw := map[string]uuid.UUID{}
|
||
|
|
if len(mergeGTINs) > 0 {
|
||
|
|
rows, err := s.Pool.Query(ctx, `
|
||
|
|
SELECT DISTINCT ON (gtin) id, gtin
|
||
|
|
FROM raw_products
|
||
|
|
WHERE company_id = $1 AND gtin = ANY($2)
|
||
|
|
ORDER BY gtin, updated_at DESC`, companyID, mergeGTINs)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
for rows.Next() {
|
||
|
|
var id uuid.UUID
|
||
|
|
var gtin string
|
||
|
|
if err := rows.Scan(&id, >in); err != nil {
|
||
|
|
rows.Close()
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
existingRaw[gtin] = id
|
||
|
|
}
|
||
|
|
err = rows.Err()
|
||
|
|
rows.Close()
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
type pendingProcessed struct {
|
||
|
|
rawID uuid.UUID
|
||
|
|
productID, name, category, desc, status string
|
||
|
|
rawWasUpdate bool
|
||
|
|
}
|
||
|
|
|
||
|
|
tx, err := s.Pool.Begin(ctx)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
defer tx.Rollback(ctx)
|
||
|
|
|
||
|
|
updRawIDs := make([]uuid.UUID, 0, len(batch))
|
||
|
|
updRawJSON := make([]string, 0, len(batch))
|
||
|
|
updRawMeta := make([]pendingProcessed, 0, len(batch))
|
||
|
|
|
||
|
|
insGTINs := make([]string, 0, len(batch))
|
||
|
|
insJSON := make([]string, 0, len(batch))
|
||
|
|
insMeta := make([]pendingProcessed, 0, len(batch))
|
||
|
|
|
||
|
|
for _, row := range batch {
|
||
|
|
meta := pendingProcessed{
|
||
|
|
productID: row.productID, name: row.name, category: row.category,
|
||
|
|
desc: row.desc, status: row.status,
|
||
|
|
}
|
||
|
|
if row.mergeable {
|
||
|
|
if id, ok := existingRaw[row.gtin]; ok {
|
||
|
|
updRawIDs = append(updRawIDs, id)
|
||
|
|
updRawJSON = append(updRawJSON, row.rawJSON)
|
||
|
|
meta.rawID = id
|
||
|
|
meta.rawWasUpdate = true
|
||
|
|
updRawMeta = append(updRawMeta, meta)
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
}
|
||
|
|
insGTINs = append(insGTINs, row.gtin)
|
||
|
|
insJSON = append(insJSON, row.rawJSON)
|
||
|
|
insMeta = append(insMeta, meta)
|
||
|
|
}
|
||
|
|
|
||
|
|
if len(updRawIDs) > 0 {
|
||
|
|
_, err = tx.Exec(ctx, `
|
||
|
|
UPDATE raw_products AS r SET
|
||
|
|
raw_data = v.raw_data::jsonb,
|
||
|
|
mapped_data = v.raw_data::jsonb,
|
||
|
|
file_id = COALESCE($3, r.file_id),
|
||
|
|
updated_at = now()
|
||
|
|
FROM unnest($2::uuid[], $4::text[]) AS v(id, raw_data)
|
||
|
|
WHERE r.id = v.id AND r.company_id = $1`,
|
||
|
|
companyID, updRawIDs, fileID, updRawJSON)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
pending := make([]pendingProcessed, 0, len(batch))
|
||
|
|
pending = append(pending, updRawMeta...)
|
||
|
|
|
||
|
|
if len(insGTINs) > 0 {
|
||
|
|
rows, err := tx.Query(ctx, `
|
||
|
|
INSERT INTO raw_products (company_id, gtin, raw_data, mapped_data, processing_status, file_id)
|
||
|
|
SELECT $1, v.gtin, v.raw_data::jsonb, v.raw_data::jsonb, 'unprocessed', $2
|
||
|
|
FROM unnest($3::text[], $4::text[]) AS v(gtin, raw_data)
|
||
|
|
ON CONFLICT (company_id, gtin) DO NOTHING
|
||
|
|
RETURNING id, gtin`,
|
||
|
|
companyID, fileID, insGTINs, insJSON)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
insertedByGTIN := map[string]uuid.UUID{}
|
||
|
|
for rows.Next() {
|
||
|
|
var id uuid.UUID
|
||
|
|
var gtin string
|
||
|
|
if err := rows.Scan(&id, >in); err != nil {
|
||
|
|
rows.Close()
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
insertedByGTIN[gtin] = id
|
||
|
|
}
|
||
|
|
err = rows.Err()
|
||
|
|
rows.Close()
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
// Match inserts back to input order; conflicts (DO NOTHING) are skipped.
|
||
|
|
for i, gtin := range insGTINs {
|
||
|
|
id, ok := insertedByGTIN[gtin]
|
||
|
|
if !ok {
|
||
|
|
res.Skipped++
|
||
|
|
res.addError(fmt.Sprintf("%s: duplicate gtin", gtin))
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
meta := insMeta[i]
|
||
|
|
meta.rawID = id
|
||
|
|
pending = append(pending, meta)
|
||
|
|
// Same gtin inserted twice in one batch: second RETURNING miss.
|
||
|
|
delete(insertedByGTIN, gtin)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if len(pending) == 0 {
|
||
|
|
return tx.Commit(ctx)
|
||
|
|
}
|
||
|
|
|
||
|
|
rawIDs := make([]uuid.UUID, len(pending))
|
||
|
|
for i, p := range pending {
|
||
|
|
rawIDs[i] = p.rawID
|
||
|
|
}
|
||
|
|
existingProcessed := map[uuid.UUID]uuid.UUID{}
|
||
|
|
prows, err := tx.Query(ctx, `
|
||
|
|
SELECT DISTINCT ON (raw_product_id) id, raw_product_id
|
||
|
|
FROM processed_products
|
||
|
|
WHERE company_id = $1 AND raw_product_id = ANY($2)
|
||
|
|
ORDER BY raw_product_id, updated_at DESC`, companyID, rawIDs)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
for prows.Next() {
|
||
|
|
var id, rawID uuid.UUID
|
||
|
|
if err := prows.Scan(&id, &rawID); err != nil {
|
||
|
|
prows.Close()
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
existingProcessed[rawID] = id
|
||
|
|
}
|
||
|
|
err = prows.Err()
|
||
|
|
prows.Close()
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
updProcIDs := make([]uuid.UUID, 0, len(pending))
|
||
|
|
updPIDs := make([]string, 0, len(pending))
|
||
|
|
updNames := make([]string, 0, len(pending))
|
||
|
|
updCats := make([]string, 0, len(pending))
|
||
|
|
updDescs := make([]string, 0, len(pending))
|
||
|
|
updStatuses := make([]string, 0, len(pending))
|
||
|
|
|
||
|
|
insRawIDs := make([]uuid.UUID, 0, len(pending))
|
||
|
|
insPIDs := make([]string, 0, len(pending))
|
||
|
|
insNames := make([]string, 0, len(pending))
|
||
|
|
insCats := make([]string, 0, len(pending))
|
||
|
|
insDescs := make([]string, 0, len(pending))
|
||
|
|
insStatuses := make([]string, 0, len(pending))
|
||
|
|
insWasUpdate := make([]bool, 0, len(pending))
|
||
|
|
|
||
|
|
for _, p := range pending {
|
||
|
|
if pid, ok := existingProcessed[p.rawID]; ok {
|
||
|
|
updProcIDs = append(updProcIDs, pid)
|
||
|
|
updPIDs = append(updPIDs, p.productID)
|
||
|
|
updNames = append(updNames, p.name)
|
||
|
|
updCats = append(updCats, p.category)
|
||
|
|
updDescs = append(updDescs, p.desc)
|
||
|
|
updStatuses = append(updStatuses, p.status)
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
insRawIDs = append(insRawIDs, p.rawID)
|
||
|
|
insPIDs = append(insPIDs, p.productID)
|
||
|
|
insNames = append(insNames, p.name)
|
||
|
|
insCats = append(insCats, p.category)
|
||
|
|
insDescs = append(insDescs, p.desc)
|
||
|
|
insStatuses = append(insStatuses, p.status)
|
||
|
|
insWasUpdate = append(insWasUpdate, p.rawWasUpdate)
|
||
|
|
}
|
||
|
|
|
||
|
|
if len(updProcIDs) > 0 {
|
||
|
|
_, err = tx.Exec(ctx, `
|
||
|
|
UPDATE processed_products AS p SET
|
||
|
|
product_id = COALESCE(NULLIF(v.product_id, ''), p.product_id),
|
||
|
|
name = COALESCE(NULLIF(v.name, ''), p.name),
|
||
|
|
category = COALESCE(NULLIF(v.category, ''), p.category),
|
||
|
|
description = COALESCE(NULLIF(v.description, ''), p.description),
|
||
|
|
status = COALESCE(NULLIF(v.status, ''), p.status),
|
||
|
|
updated_at = now()
|
||
|
|
FROM unnest($2::uuid[], $3::text[], $4::text[], $5::text[], $6::text[], $7::text[])
|
||
|
|
AS v(id, product_id, name, category, description, status)
|
||
|
|
WHERE p.id = v.id AND p.company_id = $1`,
|
||
|
|
companyID, updProcIDs, updPIDs, updNames, updCats, updDescs, updStatuses)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
res.Updated += len(updProcIDs)
|
||
|
|
}
|
||
|
|
|
||
|
|
if len(insRawIDs) > 0 {
|
||
|
|
_, err = tx.Exec(ctx, `
|
||
|
|
INSERT INTO processed_products (company_id, product_id, name, category, description, status, raw_product_id)
|
||
|
|
SELECT $1,
|
||
|
|
NULLIF(v.product_id, ''), NULLIF(v.name, ''), NULLIF(v.category, ''),
|
||
|
|
NULLIF(v.description, ''), v.status, v.raw_product_id
|
||
|
|
FROM unnest($2::uuid[], $3::text[], $4::text[], $5::text[], $6::text[], $7::text[])
|
||
|
|
AS v(raw_product_id, product_id, name, category, description, status)`,
|
||
|
|
companyID, insRawIDs, insPIDs, insNames, insCats, insDescs, insStatuses)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
for _, wasUpdate := range insWasUpdate {
|
||
|
|
if wasUpdate {
|
||
|
|
res.Updated++
|
||
|
|
} else {
|
||
|
|
res.Created++
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return tx.Commit(ctx)
|
||
|
|
}
|