Initial commit of Descrybe v2 without local scratch artifacts.

Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
This commit is contained in:
2026-08-09 22:47:43 +02:00
commit 8580c996c3
1285 changed files with 325780 additions and 0 deletions
+404
View File
@@ -0,0 +1,404 @@
package feeds
import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"strings"
"sync"
"time"
)
const (
// defaultMaxDownloadBytes caps HTTP downloads and local upload sources.
// Bodies stream to a temp file (not RAM), so hundreds of MiB are safe for
// large merchant catalogs. Keep within ~200500 MiB; raise carefully if disk
// and downloadTimeout remain adequate. Exceeding returns downloadTooLarge().
defaultMaxDownloadBytes int64 = 256 << 20 // 256 MiB
downloadTimeout = 60 * time.Second
dialTimeout = 10 * time.Second
maxRedirects = 5
defaultUserAgent = "DescrybeFeedSync/2.0"
)
// maxDownloadBytes bounds feed downloads (temp file / local source size). Mutable for tests.
var maxDownloadBytes = defaultMaxDownloadBytes
var (
errURLRequired = errors.New("feed url required")
errURLScheme = errors.New("url scheme must be http or https")
errURLPrivate = errors.New("url resolves to a private or blocked address")
errURLFTP = errors.New("ftp/ftps feed sync is not supported yet")
errDownloadTooLarge = errors.New("feed download exceeds size limit")
)
// downloadTooLarge returns errDownloadTooLarge with the active size limit for clients.
func downloadTooLarge() error {
if maxDownloadBytes >= 1<<20 {
return fmt.Errorf("%w (max %d MiB)", errDownloadTooLarge, maxDownloadBytes>>20)
}
return fmt.Errorf("%w (max %d bytes)", errDownloadTooLarge, maxDownloadBytes)
}
var (
allowMu sync.RWMutex
allowConfigured bool
allowHosts map[string]struct{}
allowCIDRs []*net.IPNet
)
// ConfigurePrivateAllowlist sets hostnames and CIDRs that may bypass the private-IP SSRF block.
// Intended for tests and optional startup wiring; production normally uses FEED_URL_PRIVATE_ALLOWLIST
// and/or admin platform setting feeds.private_url_allowlist.
func ConfigurePrivateAllowlist(hosts []string, cidrs []string) error {
h := make(map[string]struct{}, len(hosts))
for _, raw := range hosts {
raw = strings.ToLower(strings.TrimSpace(raw))
if raw != "" {
h[raw] = struct{}{}
}
}
nets := make([]*net.IPNet, 0, len(cidrs))
for _, raw := range cidrs {
raw = strings.TrimSpace(raw)
if raw == "" {
continue
}
_, n, err := net.ParseCIDR(raw)
if err != nil {
return fmt.Errorf("invalid allowlist cidr %q: %w", raw, err)
}
nets = append(nets, n)
}
allowMu.Lock()
allowHosts = h
allowCIDRs = nets
allowConfigured = true
allowMu.Unlock()
return nil
}
// ApplyPrivateAllowlistCSV merges env FEED_URL_PRIVATE_ALLOWLIST with an optional
// admin/settings CSV (settings override by appending unique entries).
func ApplyPrivateAllowlistCSV(settingsCSV string) {
parts := make([]string, 0, 8)
for _, src := range []string{os.Getenv("FEED_URL_PRIVATE_ALLOWLIST"), settingsCSV} {
for _, part := range strings.Split(src, ",") {
part = strings.TrimSpace(part)
if part != "" {
parts = append(parts, part)
}
}
}
hosts := make([]string, 0)
cidrs := make([]string, 0)
seen := map[string]struct{}{}
for _, part := range parts {
key := strings.ToLower(part)
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
if strings.Contains(part, "/") {
cidrs = append(cidrs, part)
} else {
hosts = append(hosts, part)
}
}
_ = ConfigurePrivateAllowlist(hosts, cidrs)
}
func ensureAllowlist() {
allowMu.RLock()
done := allowConfigured
allowMu.RUnlock()
if done {
return
}
allowMu.Lock()
defer allowMu.Unlock()
if allowConfigured {
return
}
allowHosts = map[string]struct{}{}
allowCIDRs = nil
raw := strings.TrimSpace(os.Getenv("FEED_URL_PRIVATE_ALLOWLIST"))
for _, part := range strings.Split(raw, ",") {
part = strings.TrimSpace(part)
if part == "" {
continue
}
if strings.Contains(part, "/") {
if _, n, err := net.ParseCIDR(part); err == nil {
allowCIDRs = append(allowCIDRs, n)
}
continue
}
allowHosts[strings.ToLower(part)] = struct{}{}
}
allowConfigured = true
}
// ValidateFeedURL checks a feed source URL for SSRF before persist.
// Empty URL is allowed (local upload sources). Does not fetch the URL.
func ValidateFeedURL(ctx context.Context, rawURL string) error {
ensureAllowlist()
rawURL = strings.TrimSpace(rawURL)
if rawURL == "" {
return nil
}
if len(rawURL) > 2048 {
return errURLScheme
}
lower := strings.ToLower(rawURL)
if strings.HasPrefix(lower, "ftp://") || strings.HasPrefix(lower, "ftps://") {
return errURLFTP
}
u, err := url.Parse(rawURL)
if err != nil || u.Host == "" {
return ClientMsg("invalid url")
}
if u.Scheme != "http" && u.Scheme != "https" {
return errURLScheme
}
return assertPublicHost(ctx, u.Hostname())
}
// downloadFeed streams an http(s) feed to a temp file with SSRF controls and a hard size cap.
// Callers must Close the returned blob to remove the temp file.
func downloadFeed(ctx context.Context, rawURL string) (*feedBlob, error) {
ensureAllowlist()
rawURL = strings.TrimSpace(rawURL)
if rawURL == "" {
return nil, errURLRequired
}
lower := strings.ToLower(rawURL)
if strings.HasPrefix(lower, "ftp://") || strings.HasPrefix(lower, "ftps://") {
return nil, errURLFTP
}
u, err := url.Parse(rawURL)
if err != nil || u.Host == "" {
return nil, ClientMsg("invalid url")
}
if u.Scheme != "http" && u.Scheme != "https" {
return nil, errURLScheme
}
if err := assertPublicHost(ctx, u.Hostname()); err != nil {
return nil, err
}
client := &http.Client{
Timeout: downloadTimeout,
Transport: ssrfTransport(),
CheckRedirect: func(req *http.Request, via []*http.Request) error {
if len(via) >= maxRedirects {
return ClientMsg("too many redirects")
}
if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
return errURLScheme
}
return assertPublicHost(req.Context(), req.URL.Hostname())
},
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", defaultUserAgent)
req.Header.Set("Accept", "text/csv,application/csv,application/xml,text/xml,application/atom+xml,*/*")
res, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("download failed: %w", err)
}
defer res.Body.Close()
if res.StatusCode < 200 || res.StatusCode >= 300 {
return nil, fmt.Errorf("download status %d", res.StatusCode)
}
if res.ContentLength > maxDownloadBytes {
return nil, downloadTooLarge()
}
tmp, err := os.CreateTemp("", "descrybe-feed-*")
if err != nil {
return nil, fmt.Errorf("temp file: %w", err)
}
tmpPath := tmp.Name()
cleanup := true
defer func() {
_ = tmp.Close()
if cleanup {
_ = os.Remove(tmpPath)
}
}()
limited := io.LimitReader(res.Body, maxDownloadBytes+1)
written, err := io.Copy(tmp, limited)
if err != nil {
return nil, fmt.Errorf("read body: %w", err)
}
if written > maxDownloadBytes {
return nil, downloadTooLarge()
}
if err := tmp.Close(); err != nil {
return nil, fmt.Errorf("close temp: %w", err)
}
cleanup = false
return &feedBlob{
path: tmpPath,
contentType: res.Header.Get("Content-Type"),
size: written,
owned: true,
}, nil
}
func ssrfTransport() *http.Transport {
dialer := &net.Dialer{Timeout: dialTimeout, KeepAlive: 30 * time.Second}
return &http.Transport{
// Never honor HTTP(S)_PROXY: dialing a proxy skips destination SSRF checks.
Proxy: nil,
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
if err := assertPublicHost(ctx, host); err != nil {
return nil, err
}
ips, err := resolveHostIPs(ctx, host)
if err != nil {
return nil, err
}
var lastErr error
for _, ip := range ips {
if isBlockedIP(ip) && !isAllowlistedHostOrIP(host, ip) {
lastErr = errURLPrivate
continue
}
target := net.JoinHostPort(ip.String(), port)
conn, err := dialer.DialContext(ctx, network, target)
if err == nil {
return conn, nil
}
lastErr = err
}
if lastErr == nil {
lastErr = errURLPrivate
}
return nil, lastErr
},
ForceAttemptHTTP2: true,
MaxIdleConns: 10,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
}
}
func resolveHostIPs(ctx context.Context, host string) ([]net.IP, error) {
if ip := net.ParseIP(host); ip != nil {
return []net.IP{ip}, nil
}
addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
if err != nil {
return nil, fmt.Errorf("dns lookup: %w", err)
}
out := make([]net.IP, 0, len(addrs))
for _, a := range addrs {
out = append(out, a.IP)
}
return out, nil
}
func assertPublicHost(ctx context.Context, host string) error {
ensureAllowlist()
host = strings.TrimSpace(host)
if host == "" {
return errURLPrivate
}
lower := strings.ToLower(host)
allowMu.RLock()
_, hostAllowed := allowHosts[lower]
allowMu.RUnlock()
if hostAllowed {
return nil
}
if lower == "localhost" || strings.HasSuffix(lower, ".localhost") || strings.HasSuffix(lower, ".local") {
return errURLPrivate
}
if ip := net.ParseIP(host); ip != nil {
if isBlockedIP(ip) && !isAllowlistedIP(ip) {
return errURLPrivate
}
return nil
}
addrs, err := resolveHostIPs(ctx, host)
if err != nil {
return err
}
if len(addrs) == 0 {
return errURLPrivate
}
for _, ip := range addrs {
if isBlockedIP(ip) && !isAllowlistedIP(ip) {
return errURLPrivate
}
}
return nil
}
func isAllowlistedHostOrIP(host string, ip net.IP) bool {
ensureAllowlist()
allowMu.RLock()
_, ok := allowHosts[strings.ToLower(strings.TrimSpace(host))]
allowMu.RUnlock()
if ok {
return true
}
return isAllowlistedIP(ip)
}
func isAllowlistedIP(ip net.IP) bool {
ensureAllowlist()
if ip == nil {
return false
}
allowMu.RLock()
defer allowMu.RUnlock()
for _, n := range allowCIDRs {
if n.Contains(ip) {
return true
}
}
return false
}
func isBlockedIP(ip net.IP) bool {
if ip == nil {
return true
}
if ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
ip.IsMulticast() || ip.IsUnspecified() {
return true
}
// AWS/GCP/Azure metadata and CGNAT.
if ip4 := ip.To4(); ip4 != nil {
if ip4[0] == 169 && ip4[1] == 254 {
return true
}
if ip4[0] == 100 && ip4[1] >= 64 && ip4[1] <= 127 {
return true
}
}
return false
}
+59
View File
@@ -0,0 +1,59 @@
package feeds
import (
"errors"
"github.com/jackc/pgx/v5"
)
// ErrFormatMismatch is returned when the public export URL extension does not
// match the feed's configured format. Public HTTP handlers must map this to the
// same opaque 404 as an unknown token (no existence oracle).
var ErrFormatMismatch = errors.New("format mismatch")
// ErrNotFound is returned when a company-scoped feed (or related row) is missing.
var ErrNotFound = errors.New("not found")
// clientError is a validation/business message safe to return to API clients.
type clientError struct {
msg string
}
func (e *clientError) Error() string { return e.msg }
// ClientMsg marks a message as safe to expose in HTTP 4xx responses.
func ClientMsg(msg string) error {
return &clientError{msg: msg}
}
// ClientError reports whether err is a known client-facing feeds error.
func ClientError(err error) (msg string, ok bool) {
if err == nil {
return "", false
}
var ce *clientError
if errors.As(err, &ce) {
return ce.msg, true
}
switch {
case errors.Is(err, ErrNotFound), errors.Is(err, pgx.ErrNoRows):
return "not found", true
case errors.Is(err, ErrFormatMismatch),
errors.Is(err, errURLRequired),
errors.Is(err, errURLScheme),
errors.Is(err, errURLPrivate),
errors.Is(err, errURLFTP),
errors.Is(err, errDownloadTooLarge),
errors.Is(err, errParseTooManyRows),
errors.Is(err, errSourceRequired),
errors.Is(err, errLocalSource):
return err.Error(), true
default:
return "", false
}
}
// IsNotFound reports whether err means a missing feed/resource.
func IsNotFound(err error) bool {
return errors.Is(err, ErrNotFound) || errors.Is(err, pgx.ErrNoRows)
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,98 @@
package feeds
import (
"context"
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
// TestRotateExportFeedPublicToken creates an ephemeral sandbox company (never A1 /
// Platform Demo), rotates the export public token, and asserts revoke semantics.
func TestRotateExportFeedPublicToken(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("DATABASE_URL"))
if dsn == "" {
t.Skip("DATABASE_URL not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
pg, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatalf("pool: %v", err)
}
defer pg.Close()
companyID := uuid.New()
prefix := companyID.String()[:8]
_, err = pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`,
companyID, "export-rotate-"+prefix)
if err != nil {
t.Fatalf("seed company: %v", err)
}
t.Cleanup(func() {
_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
})
svc := &Service{Pool: pg}
created, err := svc.CreateExportFeed(ctx, companyID, CreateExportInput{
Name: "rotate-smoke-" + prefix,
Format: "xml",
})
if err != nil {
t.Fatalf("CreateExportFeed: %v", err)
}
oldTok, _ := created["public_token"].(string)
if !validPublicToken(oldTok) || len(oldTok) != 64 {
t.Fatalf("create public_token=%q want 64-hex", oldTok)
}
feedID, ok := created["id"].(uuid.UUID)
if !ok {
idStr := fmt.Sprint(created["id"])
feedID, err = uuid.Parse(idStr)
if err != nil {
t.Fatalf("export id: %v (%v)", err, created["id"])
}
}
rotated, err := svc.RotateExportFeedPublicToken(ctx, companyID, feedID)
if err != nil {
t.Fatalf("RotateExportFeedPublicToken: %v", err)
}
newTok, _ := rotated["public_token"].(string)
if !validPublicToken(newTok) || len(newTok) != 64 {
t.Fatalf("rotated public_token=%q want 64-hex", newTok)
}
if newTok == oldTok {
t.Fatal("rotate must replace public_token")
}
got, err := svc.GetExportFeed(ctx, companyID, feedID)
if err != nil {
t.Fatalf("GetExportFeed: %v", err)
}
if fmt.Sprint(got["public_token"]) != newTok {
t.Fatalf("persisted token=%v want %s", got["public_token"], newTok)
}
var byOld int
err = pg.QueryRow(ctx, `
SELECT COUNT(*) FROM export_feeds
WHERE company_id = $1 AND public_token = $2`, companyID, oldTok).Scan(&byOld)
if err != nil {
t.Fatalf("count old token: %v", err)
}
if byOld != 0 {
t.Fatal("old public_token still present after rotate (not revoked)")
}
otherCompany := uuid.New()
_, err = svc.RotateExportFeedPublicToken(ctx, otherCompany, feedID)
if err == nil || !strings.Contains(err.Error(), "not found") {
t.Fatalf("cross-tenant rotate err=%v want not found", err)
}
}
+382
View File
@@ -0,0 +1,382 @@
package feeds
import (
"bytes"
"errors"
"strings"
"testing"
)
func TestValidPublicToken(t *testing.T) {
if validPublicToken("../etc/passwd") {
t.Fatal("path traversal token must be rejected")
}
if validPublicToken("short") {
t.Fatal("short token must be rejected")
}
if validPublicToken("0123456789abcdef") { // 16 hex / 64-bit — below floor
t.Fatal("undersized token must be rejected")
}
if validPublicToken("0123456789abcdef0123456789abcde") { // odd length
t.Fatal("odd-length hex must be rejected")
}
if !validPublicToken("0123456789abcdef0123456789abcdef") {
t.Fatal("32-hex token should be accepted")
}
tok, err := newPublicExportToken()
if err != nil {
t.Fatalf("newPublicExportToken: %v", err)
}
if len(tok) != 64 {
t.Fatalf("expected 64 hex chars, got %d", len(tok))
}
if !validPublicToken(tok) {
t.Fatal("fresh public export token should be accepted")
}
tok2, err := newPublicExportToken()
if err != nil {
t.Fatalf("newPublicExportToken second: %v", err)
}
if tok == tok2 {
t.Fatal("rotated/fresh tokens must differ (CSPRNG collision)")
}
}
func TestNewPublicExportTokenIs256BitHex(t *testing.T) {
t.Parallel()
for i := 0; i < 8; i++ {
tok, err := newPublicExportToken()
if err != nil {
t.Fatalf("newPublicExportToken: %v", err)
}
if len(tok) != publicExportTokenBytes*2 {
t.Fatalf("len=%d want %d", len(tok), publicExportTokenBytes*2)
}
if !validPublicToken(tok) {
t.Fatalf("token %q rejected by validPublicToken", tok)
}
}
}
func TestSanitizeXMLName(t *testing.T) {
if got := sanitizeXMLName("prod uct!", "product"); got != "prod_uct_" {
t.Fatalf("got %q", got)
}
if got := sanitizeXMLName("", "product"); got != "product" {
t.Fatalf("empty fallback got %q", got)
}
if got := sanitizeXMLName("g:id", "field"); got != "g:id" {
t.Fatalf("namespace colon got %q", got)
}
if got := sanitizeXMLName("g:title", "field"); got != "g:title" {
t.Fatalf("g:title got %q", got)
}
}
func TestParseExportTemplateDefaults(t *testing.T) {
tpl := parseExportTemplate([]byte("{}"))
if tpl.Root != defaultExportRoot || tpl.Item != defaultExportItem {
t.Fatalf("unexpected defaults %#v", tpl)
}
if len(tpl.Fields) < 3 {
t.Fatal("expected default fields")
}
}
func TestParseExportTemplateMappingsSorted(t *testing.T) {
tpl := parseExportTemplate([]byte(`{"mappings":{"z":"status","a":"name"}}`))
if len(tpl.Fields) != 2 {
t.Fatalf("fields=%d", len(tpl.Fields))
}
if tpl.Fields[0].Key != "a" || tpl.Fields[1].Key != "z" {
t.Fatalf("unsorted keys: %#v", tpl.Fields)
}
}
func TestProductFieldValuePrefersProcessed(t *testing.T) {
name := "Widget"
processed := "Widget Pro"
p := exportProduct{Name: &name, ProcessedName: &processed, Attributes: []byte(`{"color":"red"}`)}
if got := productFieldValue(p, "name"); got != "Widget Pro" {
t.Fatalf("name=%q", got)
}
if got := productFieldValue(p, "attr.color"); got != "red" {
t.Fatalf("attr=%q", got)
}
if !strings.Contains(xmlEscape(`a&b<c>`), "&amp;") {
t.Fatal("xmlEscape broken")
}
}
func TestSanitizeExportFileName(t *testing.T) {
if got := sanitizeExportFileName("My Feed!"); got != "my_feed" {
t.Fatalf("got %q", got)
}
if got := sanitizeExportFileName(""); got != "export" {
t.Fatalf("empty got %q", got)
}
}
func TestScalarAttrStringStructuredAndNull(t *testing.T) {
if got := scalarAttrString(nil); got != "" {
t.Fatalf("nil=%q", got)
}
if got := scalarAttrString(map[string]any{"key": "oblika-zaslona-2", "name": "ukrivljen"}); got != "ukrivljen" {
t.Fatalf("name object=%q", got)
}
if got := scalarAttrString(map[string]any{"key": "brand", "value": "CoolCo"}); got != "CoolCo" {
t.Fatalf("value object=%q", got)
}
out := map[string]string{}
putAttrValue(out, "barva", nil)
putAttrValue(out, "oblika-zaslona", map[string]any{"key": "oblika-zaslona-2", "name": "ukrivljen"})
if _, ok := out["barva"]; ok {
t.Fatal("nil attr should be skipped")
}
if out["oblika-zaslona"] != "ukrivljen" {
t.Fatalf("oblika=%q", out["oblika-zaslona"])
}
}
func sampleProcessedProduct() exportProduct {
pid := "4897098683545"
name := "Fridge X"
procName := "Fridge X Energy"
status := "processed"
return exportProduct{
ProductID: &pid,
Name: &name,
ProcessedName: &procName,
Status: &status,
Attributes: []byte(`{"color":"silver"}`),
ProcessedAttributes: []byte(`{
"eprel_id": "246834",
"brand": {"key":"brand","name":"CoolCo","value":"CoolCo"},
"specifications": [
{"key":"battery_life","value":"30 hours"},
{"key":"weight","value":"250g"}
],
"eprel": {
"energy_class": "E",
"energy_scale": "A-G",
"label": "https://eprel.ec.europa.eu/api/product/246834/labels?format=png",
"pdf": "https://eprel.ec.europa.eu/fiches/example.pdf"
}
}`),
MappedData: []byte(`{"eprel_id":"should-not-win"}`),
}
}
func TestFlattenSpecificationsAndEprel(t *testing.T) {
p := sampleProcessedProduct()
if got := productFieldValue(p, "eprel_id"); got != "246834" {
t.Fatalf("eprel_id=%q", got)
}
if got := productFieldValue(p, "energy_class"); got != "E" {
t.Fatalf("energy_class=%q", got)
}
if got := productFieldValue(p, "eprel_energy_class"); got != "E" {
t.Fatalf("eprel_energy_class=%q", got)
}
if got := productFieldValue(p, "eprel_label"); !strings.Contains(got, "eprel.ec.europa.eu") {
t.Fatalf("eprel_label=%q", got)
}
if got := productFieldValue(p, "spec.battery_life"); got != "30 hours" {
t.Fatalf("spec.battery_life=%q", got)
}
if got := productFieldValue(p, "attr.weight"); got != "250g" {
t.Fatalf("attr.weight=%q", got)
}
if got := productFieldValue(p, "attr.brand"); got != "CoolCo" {
t.Fatalf("brand=%q", got)
}
specs := productFieldValue(p, "specifications")
if !strings.Contains(specs, "battery_life: 30 hours") || !strings.Contains(specs, "weight: 250g") {
t.Fatalf("specifications=%q", specs)
}
}
func TestFlattenSpecificationsObject(t *testing.T) {
p := exportProduct{
ProcessedAttributes: []byte(`{"specifications":{"color":"red","size":"L"}}`),
}
if got := productFieldValue(p, "spec.color"); got != "red" {
t.Fatalf("got %q", got)
}
if got := productFieldValue(p, "specifications.size"); got != "L" {
t.Fatalf("got %q", got)
}
}
func TestExportXMLSnippetWithSpecsAndEprel(t *testing.T) {
tpl := exportTemplate{
Root: "products",
Item: "product",
Fields: []exportField{
{Key: "product_id", Source: "product_id"},
{Key: "name", Source: "name"},
{Key: "eprel_id", Source: "eprel_id"},
{Key: "energy_class", Source: "energy_class"},
{Key: "eprel_label", Source: "eprel_label"},
{Key: "specs", Source: "specifications.*"},
},
}
out, n, err := renderExportSnippet("xml", tpl, []exportProduct{sampleProcessedProduct()}, "Demo Feed")
if err != nil {
t.Fatal(err)
}
if n != 1 {
t.Fatalf("count=%d", n)
}
for _, want := range []string{
`<?xml version="1.0" encoding="UTF-8"?>`,
`<products feed="Demo Feed">`,
`<product_id>4897098683545</product_id>`,
`<name>Fridge X Energy</name>`,
`<eprel_id>246834</eprel_id>`,
`<energy_class>E</energy_class>`,
`<battery_life>30 hours</battery_life>`,
`<weight>250g</weight>`,
`</product>`,
`</products>`,
} {
if !strings.Contains(out, want) {
t.Fatalf("missing %q in:\n%s", want, out)
}
}
}
func TestExportCSVSnippetWithEprel(t *testing.T) {
tpl := exportTemplate{
Fields: []exportField{
{Key: "product_id", Source: "product_id"},
{Key: "eprel_id", Source: "eprel_id"},
{Key: "energy_class", Source: "energy_class"},
{Key: "battery_life", Source: "spec.battery_life"},
},
}
out, n, err := renderExportSnippet("csv", tpl, []exportProduct{sampleProcessedProduct()}, "")
if err != nil {
t.Fatal(err)
}
if n != 1 {
t.Fatalf("count=%d", n)
}
lines := strings.Split(strings.TrimSpace(out), "\n")
if len(lines) != 2 {
t.Fatalf("lines=%v", lines)
}
if lines[0] != "product_id,eprel_id,energy_class,battery_life" {
t.Fatalf("header=%q", lines[0])
}
if lines[1] != "4897098683545,246834,E,30 hours" {
t.Fatalf("row=%q", lines[1])
}
}
func TestProcessedAttributesWinOverMappedData(t *testing.T) {
p := sampleProcessedProduct()
if got := productFieldValue(p, "eprel_id"); got != "246834" {
t.Fatalf("expected processed eprel_id, got %q", got)
}
}
func TestStreamXMLUsesProductSeqWithoutCollecting(t *testing.T) {
tpl := exportTemplate{
Root: "products",
Item: "product",
Fields: []exportField{
{Key: "product_id", Source: "product_id"},
{Key: "name", Source: "name"},
},
}
yielded := 0
seq := exportProductSeq(func(yield func(exportProduct) error) error {
for i := 0; i < 3; i++ {
yielded++
if err := yield(sampleProcessedProduct()); err != nil {
return err
}
}
return nil
})
var buf bytes.Buffer
n, err := streamXML(&buf, seq, tpl, "Batch Feed")
if err != nil {
t.Fatal(err)
}
if n != 3 || yielded != 3 {
t.Fatalf("count=%d yielded=%d", n, yielded)
}
out := buf.String()
if !strings.Contains(out, `<products feed="Batch Feed">`) || !strings.Contains(out, "</products>") {
t.Fatalf("bad xml:\n%s", out)
}
if strings.Count(out, "<product>") != 3 {
t.Fatalf("expected 3 products, got:\n%s", out)
}
}
func TestStreamCSVUsesProductSeqWithoutCollecting(t *testing.T) {
tpl := exportTemplate{
Fields: []exportField{
{Key: "product_id", Source: "product_id"},
{Key: "name", Source: "name"},
},
}
seq := exportProductSeq(func(yield func(exportProduct) error) error {
return yield(sampleProcessedProduct())
})
var buf bytes.Buffer
n, err := streamCSV(&buf, seq, tpl)
if err != nil {
t.Fatal(err)
}
if n != 1 {
t.Fatalf("count=%d", n)
}
lines := strings.Split(strings.TrimSpace(buf.String()), "\n")
if len(lines) != 2 {
t.Fatalf("lines=%v", lines)
}
if lines[0] != "product_id,name" {
t.Fatalf("header=%q", lines[0])
}
if lines[1] != "4897098683545,Fridge X Energy" {
t.Fatalf("row=%q", lines[1])
}
}
func TestExportBatchSizeBoundsMemoryPages(t *testing.T) {
if exportBatchSize <= 0 || exportBatchSize > exportMaxProducts {
t.Fatalf("exportBatchSize=%d exportMaxProducts=%d", exportBatchSize, exportMaxProducts)
}
if exportChunkHint <= 0 || exportChunkHint > exportBatchSize {
t.Fatalf("exportChunkHint=%d should be positive and <= batch", exportChunkHint)
}
if exportSelectedMaxProducts <= 0 || exportSelectedMaxProducts > exportMaxProducts {
t.Fatalf("exportSelectedMaxProducts=%d must be in (0, %d]", exportSelectedMaxProducts, exportMaxProducts)
}
}
func TestStreamXMLPropagatesSeqError(t *testing.T) {
tpl := exportTemplate{
Root: "products",
Item: "product",
Fields: []exportField{{Key: "name", Source: "name"}},
}
seq := exportProductSeq(func(yield func(exportProduct) error) error {
if err := yield(sampleProcessedProduct()); err != nil {
return err
}
return errors.New("boom")
})
var buf bytes.Buffer
n, err := streamXML(&buf, seq, tpl, "x")
if err == nil || err.Error() != "boom" {
t.Fatalf("err=%v count=%d", err, n)
}
if n != 1 {
t.Fatalf("expected 1 product written before error, got %d", n)
}
}
+529
View File
@@ -0,0 +1,529 @@
package feeds
import (
"bytes"
"context"
"encoding/csv"
"encoding/xml"
"errors"
"fmt"
"io"
"sort"
"strings"
"unicode"
"github.com/google/uuid"
)
const (
schemaSampleBytes = 512 << 10 // 512 KiB preview window
schemaMaxRows = 25
schemaMaxSamples = 5
schemaMaxFields = 200
previewMaxLines = 80
)
// SchemaField is one discovered source column/xpath with sample values.
type SchemaField struct {
Path string `json:"path"`
FieldName string `json:"field_name"`
DataType string `json:"data_type"`
SampleValues []string `json:"sample_values"`
UniqueValuesCount int `json:"unique_values_count"`
SuggestedTarget string `json:"suggested_target,omitempty"`
}
// SchemaExtractResult is returned by POST /feeds/{id}/extract-schema.
type SchemaExtractResult struct {
FeedID string `json:"feed_id"`
Format string `json:"format"`
SuggestedPath string `json:"suggested_item_path,omitempty"`
ItemPath string `json:"item_path,omitempty"`
Fields []SchemaField `json:"fields"`
SampleRows int `json:"sample_rows"`
Preview string `json:"preview,omitempty"`
PreviewTruncated bool `json:"preview_truncated,omitempty"`
}
// ExtractSchema downloads a bounded sample of the feed and returns field paths + samples.
func (s *Service) ExtractSchema(ctx context.Context, companyID, feedID uuid.UUID, itemPathHint string) (*SchemaExtractResult, error) {
feed, err := s.Get(ctx, companyID, feedID)
if err != nil {
if IsNotFound(err) {
return nil, ErrNotFound
}
return nil, err
}
urlStr, _ := feed["url"].(string)
feedType, _ := feed["feed_type"].(string)
itemPathHint = strings.TrimSpace(itemPathHint)
if itemPathHint == "" {
if m, err := s.GetMappings(ctx, companyID, feedID); err == nil {
itemPathHint = itemPathFromMappings(m["mappings"])
}
if itemPathHint == "" {
if opts, ok := feed["options"].(map[string]any); ok {
if v, ok := opts["item_path"].(string); ok {
itemPathHint = strings.TrimSpace(v)
}
}
}
}
src, err := s.loadFeedSource(ctx, companyID, feed)
if err != nil {
return nil, err
}
defer src.Close()
f, err := src.Open()
if err != nil {
return nil, err
}
defer f.Close()
truncated := src.size > schemaSampleBytes
data, err := io.ReadAll(io.LimitReader(f, schemaSampleBytes))
if err != nil {
return nil, err
}
format := detectFeedFormat(feedType, src.contentType, urlStr, data)
out := &SchemaExtractResult{
FeedID: feedID.String(),
Format: format,
ItemPath: itemPathHint,
PreviewTruncated: truncated,
}
switch format {
case "xml":
suggested := guessXMLItemPath(data)
out.SuggestedPath = suggested
local := itemLocalFromPath(itemPathHint)
if local == "" {
local = itemLocalFromPath(suggested)
out.ItemPath = suggested
} else {
out.ItemPath = itemPathHint
}
fields, rows, err := extractXMLSchema(data, local)
if err != nil {
return nil, err
}
out.Fields = fields
out.SampleRows = rows
out.Preview = buildXMLPreview(data, local)
default:
fields, rows, preview, err := extractCSVSchema(data)
if err != nil {
return nil, err
}
out.Fields = fields
out.SampleRows = rows
out.Preview = preview
out.SuggestedPath = ""
out.ItemPath = ""
}
return out, nil
}
func itemPathFromMappings(raw any) string {
switch t := raw.(type) {
case map[string]any:
if v, ok := t["item_path"].(string); ok {
if p := strings.TrimSpace(v); p != "" {
return p
}
}
if fields, ok := t["fields"]; ok {
if p := itemPathFromMappings(fields); p != "" {
return p
}
}
if nested, ok := t["mappings"]; ok {
return itemPathFromMappings(nested)
}
return ""
case []any, []FieldMapping:
return deriveItemPathFromMappings(parseMappings(t))
default:
if parsed := parseMappings(raw); len(parsed) > 0 {
return deriveItemPathFromMappings(parsed)
}
return ""
}
}
// deriveItemPathFromMappings picks the common parent path of mapping xpaths
// (e.g. Export/Item/ID + Export/Item/name -> Export/Item).
func deriveItemPathFromMappings(mappings []FieldMapping) string {
var partsLists [][]string
for _, m := range mappings {
src := m.sourceKey()
if src == "" {
continue
}
src = strings.Trim(strings.ReplaceAll(src, "\\", "/"), "/")
if !strings.Contains(src, "/") {
continue
}
parts := strings.Split(src, "/")
if len(parts) < 2 {
continue
}
// Drop the leaf field segment.
partsLists = append(partsLists, parts[:len(parts)-1])
}
if len(partsLists) == 0 {
return ""
}
common := partsLists[0]
for _, parts := range partsLists[1:] {
n := len(common)
if len(parts) < n {
n = len(parts)
}
i := 0
for i < n && strings.EqualFold(common[i], parts[i]) {
i++
}
common = common[:i]
if len(common) == 0 {
return ""
}
}
return strings.Join(common, "/")
}
func itemLocalFromPath(path string) string {
path = strings.Trim(strings.TrimSpace(path), "/")
if path == "" {
return ""
}
if i := strings.LastIndex(path, "/"); i >= 0 {
return path[i+1:]
}
return path
}
func guessXMLItemPath(data []byte) string {
sample := string(data)
if len(sample) > 64<<10 {
sample = sample[:64<<10]
}
lower := strings.ToLower(sample)
type cand struct {
local string
full string
}
cands := []cand{
{"item", "rss/channel/item"},
{"product", "products/product"},
{"entry", "feed/entry"},
{"offer", "offers/offer"},
{"row", "rows/row"},
}
for _, c := range cands {
if strings.Contains(lower, "<"+c.local) || strings.Contains(lower, ":"+c.local) {
if path := findFirstTagPath(data, c.local); path != "" {
return path
}
return c.full
}
}
return "rss/channel/item"
}
func findFirstTagPath(data []byte, local string) string {
dec := xml.NewDecoder(bytes.NewReader(data))
dec.Strict = false
var stack []string
for {
tok, err := dec.Token()
if err != nil {
return ""
}
switch t := tok.(type) {
case xml.StartElement:
stack = append(stack, t.Name.Local)
if localNameEquals(t.Name, local) {
return strings.Join(stack, "/")
}
case xml.EndElement:
if len(stack) > 0 {
stack = stack[:len(stack)-1]
}
}
}
}
type fieldAcc struct {
path string
name string
samples []string
seen map[string]struct{}
dataType string
}
func extractXMLSchema(data []byte, itemLocal string) ([]SchemaField, int, error) {
if itemLocal == "" {
itemLocal = guessXMLItemLocal(data)
}
acc := map[string]*fieldAcc{}
rows := 0
_, err := parseXMLItems(bytes.NewReader(data), itemLocal, func(row feedRow) error {
rows++
if rows > schemaMaxRows {
return errStopSchema
}
accumulateRowFields(acc, row)
return nil
})
if err != nil && !errors.Is(err, errStopSchema) {
return nil, rows, err
}
return finalizeSchema(acc), rows, nil
}
func accumulateRowFields(acc map[string]*fieldAcc, row feedRow) {
for k, v := range row {
v = strings.TrimSpace(v)
if v == "" {
continue
}
if strings.HasPrefix(k, "@") && !strings.Contains(k, "/") {
// Bare attribute dupes are noise; path-qualified @ kept below.
continue
}
fa := acc[k]
if fa == nil {
fa = &fieldAcc{
path: k,
name: leafName(k),
seen: map[string]struct{}{},
}
acc[k] = fa
}
if _, ok := fa.seen[v]; !ok {
fa.seen[v] = struct{}{}
if len(fa.samples) < schemaMaxSamples {
fa.samples = append(fa.samples, truncateSample(v))
}
}
if fa.dataType == "" {
fa.dataType = inferDataType(v)
} else if fa.dataType != "string" {
t := inferDataType(v)
if t != fa.dataType {
fa.dataType = "string"
}
}
}
}
var errStopSchema = errors.New("schema sample limit")
func extractCSVSchema(data []byte) ([]SchemaField, int, string, error) {
r := csv.NewReader(bytes.NewReader(data))
r.ReuseRecord = true
r.LazyQuotes = true
r.TrimLeadingSpace = true
r.FieldsPerRecord = -1
header, err := r.Read()
if err != nil {
return nil, 0, "", fmt.Errorf("csv header: %w", err)
}
cols := make([]string, len(header))
for i, h := range header {
cols[i] = strings.TrimSpace(h)
}
acc := map[string]*fieldAcc{}
for _, c := range cols {
if c == "" {
continue
}
acc[c] = &fieldAcc{path: c, name: c, seen: map[string]struct{}{}, dataType: ""}
}
rows := 0
var previewLines []string
previewLines = append(previewLines, strings.Join(cols, ","))
for {
rec, err := r.Read()
if err == io.EOF {
break
}
if err != nil {
return nil, rows, "", fmt.Errorf("csv row %d: %w", rows+1, err)
}
rows++
if rows <= 5 {
previewLines = append(previewLines, strings.Join(rec, ","))
}
if rows > schemaMaxRows {
break
}
row := make(feedRow, len(cols))
for i, col := range cols {
if col == "" || i >= len(rec) {
continue
}
row[col] = strings.TrimSpace(rec[i])
}
expandSpecificationFields(row)
accumulateRowFields(acc, row)
}
preview := strings.Join(previewLines, "\n")
return finalizeSchema(acc), rows, preview, nil
}
func finalizeSchema(acc map[string]*fieldAcc) []SchemaField {
preferNestedFieldPaths(acc)
keys := make([]string, 0, len(acc))
for k := range acc {
keys = append(keys, k)
}
sort.SliceStable(keys, func(i, j int) bool {
di, dj := strings.Count(keys[i], "/"), strings.Count(keys[j], "/")
if di != dj {
return di < dj
}
return keys[i] < keys[j]
})
out := make([]SchemaField, 0, len(keys))
for _, k := range keys {
fa := acc[k]
dt := fa.dataType
if dt == "" {
dt = "string"
}
// Nested CDATA/HTML parent blobs stay as string; children are preferred for mapping.
if isSpecFieldKey(k) && hasPrefixedChildrenAcc(acc, k) {
dt = "object"
}
out = append(out, SchemaField{
Path: fa.path,
FieldName: fa.name,
DataType: dt,
SampleValues: fa.samples,
UniqueValuesCount: len(fa.seen),
SuggestedTarget: SuggestTarget(fa.name),
})
if out[len(out)-1].SuggestedTarget == "" {
out[len(out)-1].SuggestedTarget = SuggestTarget(fa.path)
}
if len(out) >= schemaMaxFields {
break
}
}
return out
}
// preferNestedFieldPaths drops bare leaf keys when a nested path ending with
// the same leaf exists (e.g. keep specifications/Color, drop Color).
func preferNestedFieldPaths(acc map[string]*fieldAcc) {
nestedLeaves := map[string]struct{}{}
for k := range acc {
if strings.Contains(k, "/") {
nestedLeaves[strings.ToLower(leafName(k))] = struct{}{}
}
}
for k := range acc {
if strings.Contains(k, "/") {
continue
}
if _, ok := nestedLeaves[strings.ToLower(k)]; ok {
delete(acc, k)
}
}
}
func hasPrefixedChildrenAcc(acc map[string]*fieldAcc, prefix string) bool {
p := strings.TrimSuffix(prefix, "/") + "/"
for k := range acc {
if strings.HasPrefix(k, p) {
return true
}
}
return false
}
func leafName(path string) string {
path = strings.TrimSpace(path)
if i := strings.LastIndex(path, "/"); i >= 0 {
return path[i+1:]
}
return path
}
func truncateSample(s string) string {
if len(s) > 120 {
return s[:117] + "..."
}
return s
}
func inferDataType(v string) string {
v = strings.TrimSpace(v)
if v == "" {
return "string"
}
lower := strings.ToLower(v)
if lower == "true" || lower == "false" {
return "boolean"
}
dot := 0
digits := 0
for i, r := range v {
if r == '-' && i == 0 {
continue
}
if r == '.' {
dot++
if dot > 1 {
return "string"
}
continue
}
if !unicode.IsDigit(r) {
return "string"
}
digits++
}
if digits == 0 {
return "string"
}
if dot == 1 {
return "number"
}
return "integer"
}
func buildXMLPreview(data []byte, itemLocal string) string {
sample := string(data)
if len(sample) > schemaSampleBytes {
sample = sample[:schemaSampleBytes]
}
lines := strings.Split(sample, "\n")
out := make([]string, 0, previewMaxLines)
for _, line := range lines {
trimmed := strings.TrimSpace(line)
if trimmed == "" {
continue
}
out = append(out, line)
if len(out) >= previewMaxLines {
break
}
}
_ = itemLocal
return strings.Join(out, "\n")
}
@@ -0,0 +1,70 @@
package feeds
import (
"strings"
"testing"
)
func TestExtractCSVSchema(t *testing.T) {
data := []byte("ean,title,price\n123,Widget,9.99\n456,Gadget,12.50\n")
fields, rows, preview, err := extractCSVSchema(data)
if err != nil {
t.Fatal(err)
}
if rows != 2 {
t.Fatalf("rows=%d", rows)
}
if len(fields) != 3 {
t.Fatalf("fields=%d", len(fields))
}
if !strings.Contains(preview, "ean,title,price") {
t.Fatalf("preview missing header: %q", preview)
}
if fields[0].Path != "ean" {
t.Fatalf("first field %q", fields[0].Path)
}
}
func TestExtractXMLSchema(t *testing.T) {
data := []byte(`<?xml version="1.0"?>
<rss><channel>
<item><title>A</title><g:gtin>111</g:gtin></item>
<item><title>B</title><g:gtin>222</g:gtin></item>
</channel></rss>`)
fields, rows, err := extractXMLSchema(data, "item")
if err != nil {
t.Fatal(err)
}
if rows != 2 {
t.Fatalf("rows=%d", rows)
}
if len(fields) == 0 {
t.Fatal("expected fields")
}
foundTitle := false
for _, f := range fields {
if f.FieldName == "title" || f.Path == "title" {
foundTitle = true
}
}
if !foundTitle {
t.Fatalf("title not found in %#v", fields)
}
}
func TestParseMappingsWrapped(t *testing.T) {
raw := map[string]any{
"item_path": "rss/channel/item",
"fields": []any{
map[string]any{"source": "gtin", "target": "gtin"},
map[string]any{"source": "title", "target": "title"},
},
}
got := parseMappings(raw)
if len(got) != 2 {
t.Fatalf("got %d mappings: %#v", len(got), got)
}
if itemPathFromMappings(raw) != "rss/channel/item" {
t.Fatalf("item path: %q", itemPathFromMappings(raw))
}
}
@@ -0,0 +1,96 @@
package feeds
import (
"context"
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestListAndExportFeedsSQLPagination(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("DATABASE_URL"))
if dsn == "" {
t.Skip("DATABASE_URL not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
pg, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatalf("pool: %v", err)
}
defer pg.Close()
companyID := uuid.New()
_, err = pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`,
companyID, "list-page-"+companyID.String()[:8])
if err != nil {
t.Fatalf("seed company: %v", err)
}
t.Cleanup(func() {
_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
})
svc := &Service{Pool: pg}
for i := 0; i < 3; i++ {
_, err := pg.Exec(ctx, `
INSERT INTO input_feeds (company_id, name, url, feed_type, status, sync_interval_minutes, options)
VALUES ($1, $2, '', 'csv', 'unmapped', 60, '{}'::jsonb)`,
companyID, fmt.Sprintf("feed-%d", i))
if err != nil {
t.Fatalf("insert feed: %v", err)
}
}
page, total, _, _, err := svc.List(ctx, companyID, 2, 0, "")
if err != nil {
t.Fatalf("List: %v", err)
}
if total != 3 {
t.Fatalf("total=%d want 3", total)
}
if len(page) != 2 {
t.Fatalf("page len=%d want 2", len(page))
}
for i, item := range page {
v, ok := item["mapping_incomplete"].(bool)
if !ok {
t.Fatalf("page[%d] missing mapping_incomplete bool: %#v", i, item["mapping_incomplete"])
}
if !v {
t.Fatalf("page[%d] mapping_incomplete=false want true for unmapped feed without mappings", i)
}
presented := PresentFeed(item)
if presented["mapping_incomplete"] != true {
t.Fatalf("PresentFeed mapping_incomplete=%v", presented["mapping_incomplete"])
}
}
page2, total2, _, _, err := svc.List(ctx, companyID, 2, 2, "")
if err != nil {
t.Fatalf("List page2: %v", err)
}
if total2 != 3 || len(page2) != 1 {
t.Fatalf("page2 len=%d total=%d", len(page2), total2)
}
matched, matchedTotal, _, _, err := svc.List(ctx, companyID, 10, 0, "feed-1")
if err != nil {
t.Fatalf("List search: %v", err)
}
if matchedTotal != 1 || len(matched) != 1 {
t.Fatalf("search len=%d total=%d want 1", len(matched), matchedTotal)
}
exp, expTotal, err := svc.ListExportFeeds(ctx, companyID, 10, 0)
if err != nil {
t.Fatalf("ListExportFeeds: %v", err)
}
if expTotal != 0 || len(exp) != 0 {
t.Fatalf("export feeds: len=%d total=%d", len(exp), expTotal)
}
}
+323
View File
@@ -0,0 +1,323 @@
package feeds
import (
"encoding/json"
"strings"
)
// FieldMapping maps a feed source column/xpath onto a canonical product field.
type FieldMapping struct {
Source string `json:"source,omitempty"`
Column string `json:"column,omitempty"`
XPath string `json:"xpath,omitempty"`
Target string `json:"target,omitempty"`
FieldName string `json:"fieldName,omitempty"`
Field string `json:"field,omitempty"`
}
func (m FieldMapping) sourceKey() string {
for _, v := range []string{m.Source, m.Column, m.XPath} {
if strings.TrimSpace(v) != "" {
return strings.TrimSpace(v)
}
}
return ""
}
func (m FieldMapping) targetKey() string {
for _, v := range []string{m.Target, m.FieldName, m.Field} {
if strings.TrimSpace(v) != "" {
return strings.TrimSpace(v)
}
}
return ""
}
// parseMappings accepts legacy object maps or array forms.
func parseMappings(raw any) []FieldMapping {
if raw == nil {
return nil
}
switch t := raw.(type) {
case []FieldMapping:
return t
case []any:
out := make([]FieldMapping, 0, len(t))
for _, item := range t {
if m, ok := fieldMappingFromUIEntry(item); ok {
out = append(out, m)
continue
}
b, err := json.Marshal(item)
if err != nil {
continue
}
var m FieldMapping
if json.Unmarshal(b, &m) != nil {
continue
}
if m.sourceKey() != "" && m.targetKey() != "" {
out = append(out, m)
}
}
return out
case map[string]any:
// Prefer wrapped { item_path, fields|mappings: [...] } from the mapping UI.
if fields, ok := t["fields"]; ok {
return parseMappings(fields)
}
if nested, ok := t["mappings"]; ok {
return parseMappings(nested)
}
out := make([]FieldMapping, 0, len(t))
for source, val := range t {
if source == "item_path" {
continue
}
switch v := val.(type) {
case string:
out = append(out, FieldMapping{Source: source, Target: v})
case map[string]any:
b, _ := json.Marshal(v)
var m FieldMapping
_ = json.Unmarshal(b, &m)
if m.sourceKey() == "" {
m.Source = source
}
if m.targetKey() == "" {
continue
}
out = append(out, m)
default:
b, err := json.Marshal(v)
if err != nil {
continue
}
var m FieldMapping
if json.Unmarshal(b, &m) != nil {
continue
}
if m.sourceKey() == "" {
m.Source = source
}
if m.targetKey() != "" {
out = append(out, m)
}
}
}
return out
default:
b, err := json.Marshal(raw)
if err != nil {
return nil
}
var arr []FieldMapping
if json.Unmarshal(b, &arr) == nil && len(arr) > 0 {
return arr
}
var obj map[string]any
if json.Unmarshal(b, &obj) == nil {
return parseMappings(obj)
}
return nil
}
}
// fieldMappingFromUIEntry unwraps dashboard rows shaped like
// {"key":"Export/Item/ID","mapping":{"fieldName":"id","xpath":"Export/Item/ID"}}.
func fieldMappingFromUIEntry(item any) (FieldMapping, bool) {
m, ok := item.(map[string]any)
if !ok {
return FieldMapping{}, false
}
nested, ok := m["mapping"]
if !ok {
return FieldMapping{}, false
}
nestedMap, ok := nested.(map[string]any)
if !ok {
b, err := json.Marshal(nested)
if err != nil {
return FieldMapping{}, false
}
nestedMap = map[string]any{}
if json.Unmarshal(b, &nestedMap) != nil {
return FieldMapping{}, false
}
}
b, err := json.Marshal(nestedMap)
if err != nil {
return FieldMapping{}, false
}
var fm FieldMapping
if json.Unmarshal(b, &fm) != nil {
return FieldMapping{}, false
}
if key, _ := m["key"].(string); strings.TrimSpace(key) != "" {
if fm.XPath == "" {
fm.XPath = strings.TrimSpace(key)
}
if fm.Source == "" && fm.Column == "" {
fm.Source = strings.TrimSpace(key)
}
}
if fm.sourceKey() == "" || fm.targetKey() == "" {
return FieldMapping{}, false
}
return fm, true
}
// applyMappings copies source values into mapped_data as-is (including "0").
// Derivation and zero-dimension cleanup happen later in processing.EnrichMapped.
func applyMappings(row map[string]string, mappings []FieldMapping) (mapped map[string]any, gtin string) {
mapped = make(map[string]any, len(mappings))
for _, m := range mappings {
src := m.sourceKey()
tgt := m.targetKey()
if src == "" || tgt == "" || strings.EqualFold(tgt, "none") {
continue
}
if isSpecificationsTarget(tgt) {
if obj := resolveSpecifications(row, src); obj != nil {
if raw, ok := obj["_raw"]; ok && len(obj) == 1 {
mapped["specifications"] = raw
} else {
delete(obj, "_raw")
mapped["specifications"] = obj
}
if !strings.EqualFold(tgt, "specifications") {
mapped[tgt] = mapped["specifications"]
}
}
continue
}
val, ok := lookupRow(row, src)
if !ok || strings.TrimSpace(val) == "" {
continue
}
val = strings.TrimSpace(val)
mapped[tgt] = val
if strings.EqualFold(tgt, "gtin") || strings.EqualFold(tgt, "ean") || strings.EqualFold(tgt, "upc") {
gtin = val
mapped["gtin"] = val
}
}
if gtin == "" {
for _, k := range []string{"gtin", "ean", "upc", "EAN", "GTIN", "barcode"} {
if v, ok := lookupRow(row, k); ok && strings.TrimSpace(v) != "" {
gtin = strings.TrimSpace(v)
mapped["gtin"] = gtin
break
}
}
}
return mapped, gtin
}
func isSpecificationsTarget(tgt string) bool {
switch strings.ToLower(strings.TrimSpace(tgt)) {
case "specifications", "specs", "specification":
return true
default:
return false
}
}
// resolveSpecifications builds a label→value map from nested XML children,
// CDATA HTML, or flat strings. Empty/missing yields nil.
func resolveSpecifications(row map[string]string, src string) map[string]string {
src = strings.Trim(strings.TrimSpace(src), "/")
if children := collectPrefixed(row, src); len(children) > 0 {
return normalizeSpecKeys(children)
}
if val, ok := lookupRow(row, src); ok {
if pairs := ParseSpecifications(val); len(pairs) > 0 {
return specsToMap(pairs)
}
// Keep non-empty raw blob so UI/backfill can still parse later.
if strings.TrimSpace(val) != "" && !isEmptySpecBlob(val) {
return map[string]string{"_raw": strings.TrimSpace(val)}
}
}
return nil
}
func normalizeSpecKeys(in map[string]string) map[string]string {
if len(in) == 0 {
return nil
}
out := make(map[string]string, len(in))
for k, v := range in {
key := CanonicalAttributeKey(k)
if key == "" || strings.TrimSpace(v) == "" {
continue
}
out[key] = v
}
if len(out) == 0 {
return nil
}
return out
}
func lookupRow(row map[string]string, key string) (string, bool) {
key = strings.Trim(strings.TrimSpace(key), "/")
if key == "" {
return "", false
}
if v, ok := row[key]; ok {
return v, true
}
// Case-insensitive exact path match (prefer longest key).
var bestKey string
for k := range row {
if strings.EqualFold(k, key) {
if len(k) >= len(bestKey) {
bestKey = k
}
}
}
if bestKey != "" {
return row[bestKey], true
}
if !strings.Contains(key, "/") {
return "", false
}
// Nested path: prefer a unique row key that ends with the same path suffix.
leaf := leafName(key)
suffix := "/" + strings.ToLower(key)
var suffixHits []string
var leafHits []string
for k := range row {
lk := strings.ToLower(k)
if strings.HasSuffix(lk, suffix) || lk == strings.ToLower(key) {
suffixHits = append(suffixHits, k)
continue
}
if strings.EqualFold(leafName(k), leaf) {
leafHits = append(leafHits, k)
}
}
if len(suffixHits) == 1 {
return row[suffixHits[0]], true
}
if len(suffixHits) > 1 {
// Prefer the shortest (most specific relative) match.
best := suffixHits[0]
for _, h := range suffixHits[1:] {
if len(h) < len(best) {
best = h
}
}
return row[best], true
}
// Fall back to bare leaf only when unambiguous.
if len(leafHits) == 1 {
return row[leafHits[0]], true
}
if v, ok := row[leaf]; ok && len(leafHits) <= 1 {
return v, true
}
return "", false
}
+233
View File
@@ -0,0 +1,233 @@
package feeds
import (
"bytes"
"encoding/csv"
"encoding/xml"
"errors"
"fmt"
"io"
"strings"
)
// defaultMaxParseRows caps CSV/XML product rows per sync. Parse is streaming
// (one row at a time); the bound limits sync duration and DB write volume for
// oversized catalogs. Exceeding returns parseTooManyRows() with the numeric limit.
const defaultMaxParseRows = 1_000_000
// maxParseRows is the active row cap. Mutable for tests.
var maxParseRows = defaultMaxParseRows
// errParseTooManyRows is the sentinel for ClientError / errors.Is checks.
var errParseTooManyRows = errors.New("feed exceeds max row limit")
// parseTooManyRows returns errParseTooManyRows with the active row limit for clients.
func parseTooManyRows() error {
return fmt.Errorf("%w (%d)", errParseTooManyRows, maxParseRows)
}
// feedRow is a normalized flat record from CSV or XML.
type feedRow map[string]string
func detectFeedFormat(feedType, contentType, urlHint string, sample []byte) string {
ft := strings.ToLower(strings.TrimSpace(feedType))
if ft == "csv" || ft == "xml" {
return ft
}
ct := strings.ToLower(contentType)
u := strings.ToLower(urlHint)
switch {
case strings.Contains(ct, "csv") || strings.HasSuffix(u, ".csv"):
return "csv"
case strings.Contains(ct, "xml") || strings.HasSuffix(u, ".xml"):
return "xml"
}
trimmed := bytes.TrimSpace(sample)
if len(trimmed) > 0 && trimmed[0] == '<' {
return "xml"
}
return "csv"
}
// parseCSV streams rows via callback to avoid holding the full matrix when possible.
// The CSV reader still tokenizes; we only keep one row at a time in the callback path.
func parseCSV(r io.Reader, onRow func(feedRow) error) (int, error) {
cr := csv.NewReader(r)
cr.ReuseRecord = true
cr.LazyQuotes = true
cr.TrimLeadingSpace = true
cr.FieldsPerRecord = -1
header, err := cr.Read()
if err != nil {
return 0, fmt.Errorf("csv header: %w", err)
}
cols := make([]string, len(header))
for i, h := range header {
cols[i] = strings.TrimSpace(h)
}
count := 0
for {
rec, err := cr.Read()
if err == io.EOF {
break
}
if err != nil {
return count, fmt.Errorf("csv row %d: %w", count+1, err)
}
count++
if count > maxParseRows {
return count, parseTooManyRows()
}
row := make(feedRow, len(cols))
for i, col := range cols {
if col == "" {
continue
}
if i < len(rec) {
row[col] = rec[i]
} else {
row[col] = ""
}
}
expandSpecificationFields(row)
if err := onRow(row); err != nil {
return count, err
}
}
return count, nil
}
// parseXMLItems streams element-local text maps for repeating item tags.
// When itemLocal is empty and r is seekable, a small prefix is sniffed then rewound.
func parseXMLItems(r io.Reader, itemLocal string, onRow func(feedRow) error) (int, error) {
itemLocal = strings.TrimSpace(itemLocal)
if itemLocal == "" {
var err error
itemLocal, r, err = resolveXMLItemLocal(r)
if err != nil {
return 0, err
}
}
dec := xml.NewDecoder(r)
dec.Strict = false
count := 0
for {
tok, err := dec.Token()
if err == io.EOF {
break
}
if err != nil {
return count, fmt.Errorf("xml: %w", err)
}
se, ok := tok.(xml.StartElement)
if !ok {
continue
}
if !localNameEquals(se.Name, itemLocal) {
continue
}
row, err := readXMLElementMap(dec, se)
if err != nil {
return count, err
}
expandSpecificationFields(row)
count++
if count > maxParseRows {
return count, parseTooManyRows()
}
if err := onRow(row); err != nil {
return count, err
}
}
return count, nil
}
const xmlItemGuessBytes = 64 << 10
func resolveXMLItemLocal(r io.Reader) (string, io.Reader, error) {
if rs, ok := r.(io.ReadSeeker); ok {
sample := make([]byte, xmlItemGuessBytes)
n, err := io.ReadFull(rs, sample)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
return "", nil, err
}
sample = sample[:n]
local := guessXMLItemLocal(sample)
if _, err := rs.Seek(0, io.SeekStart); err != nil {
return "", nil, err
}
return local, rs, nil
}
sample, err := io.ReadAll(io.LimitReader(r, xmlItemGuessBytes))
if err != nil {
return "", nil, err
}
local := guessXMLItemLocal(sample)
return local, io.MultiReader(bytes.NewReader(sample), r), nil
}
func guessXMLItemLocal(data []byte) string {
sample := string(data)
if len(sample) > xmlItemGuessBytes {
sample = sample[:xmlItemGuessBytes]
}
lower := strings.ToLower(sample)
for _, cand := range []string{"item", "product", "entry", "offer", "row"} {
if strings.Contains(lower, "<"+cand) || strings.Contains(lower, ":"+cand) {
return cand
}
}
return "item"
}
func localNameEquals(n xml.Name, local string) bool {
return strings.EqualFold(n.Local, local)
}
func readXMLElementMap(dec *xml.Decoder, start xml.StartElement) (feedRow, error) {
row := make(feedRow)
for _, a := range start.Attr {
key := "@" + a.Name.Local
row[key] = a.Value
row[start.Name.Local+"/"+key] = a.Value
}
var path []string
for {
tok, err := dec.Token()
if err != nil {
return nil, err
}
switch t := tok.(type) {
case xml.StartElement:
path = append(path, t.Name.Local)
for _, a := range t.Attr {
key := strings.Join(path, "/") + "/@" + a.Name.Local
row["@"+a.Name.Local] = a.Value
row[key] = a.Value
}
case xml.EndElement:
if len(path) == 0 {
return row, nil
}
path = path[:len(path)-1]
case xml.CharData:
text := strings.TrimSpace(string(t))
if text == "" || len(path) == 0 {
continue
}
leaf := path[len(path)-1]
full := strings.Join(path, "/")
// Prefer nested path as source of truth; only set bare leaf when
// unique (no other nested field already owns this leaf name).
if prev, ok := row[full]; ok && prev != "" && prev != text {
row[full] = prev + " " + text
} else {
row[full] = text
}
if prev, ok := row[leaf]; !ok || prev == "" || prev == text || prev == row[full] {
row[leaf] = row[full]
}
}
}
}
@@ -0,0 +1,53 @@
package feeds
import "testing"
func TestParseUIKeyMappingArray(t *testing.T) {
raw := []any{
map[string]any{
"key": "Export/Item/ID",
"mapping": map[string]any{
"fieldName": "id",
"xpath": "Export/Item/ID",
"originalName": "ID",
},
},
map[string]any{
"key": "Export/Item/name",
"mapping": map[string]any{
"fieldName": "name",
"xpath": "Export/Item/name",
},
},
map[string]any{
"key": "Export/Item/EAN",
"mapping": map[string]any{
"fieldName": "gtin",
"xpath": "Export/Item/EAN",
},
},
}
got := parseMappings(raw)
if len(got) != 3 {
t.Fatalf("got %d mappings: %#v", len(got), got)
}
if got[0].targetKey() != "id" || got[0].sourceKey() != "Export/Item/ID" {
t.Fatalf("first=%#v", got[0])
}
path := itemPathFromMappings(raw)
if path != "Export/Item" {
t.Fatalf("item path=%q", path)
}
}
func TestItemPathFromWrappedMappings(t *testing.T) {
raw := map[string]any{
"item_path": "rss/channel/item",
"mappings": []any{
map[string]any{"source": "title", "target": "title"},
},
}
if itemPathFromMappings(raw) != "rss/channel/item" {
t.Fatalf("path=%q", itemPathFromMappings(raw))
}
}
+140
View File
@@ -0,0 +1,140 @@
package feeds
import (
"strings"
"time"
"github.com/google/uuid"
)
// PresentFeed maps an input_feeds row to the public/legacy feed DTO.
// Legacy fields (item_path, is_active, last_synced, product_count) are always set;
// v2 fields (feed_type, sync_interval_minutes, last_synced_at, options) are included for dual-support.
func PresentFeed(feed map[string]any) map[string]any {
if feed == nil {
return nil
}
opts, _ := feed["options"].(map[string]any)
if opts == nil {
opts = map[string]any{}
}
itemPath, _ := opts["item_path"].(string)
if itemPath == "" {
itemPath, _ = feed["item_path"].(string)
}
status, _ := feed["status"].(string)
lastSynced := formatAPITime(feed["last_synced_at"])
productsUpdated := formatAPITime(feed["products_updated_at"])
productCount := intFromAny(feed["product_count"])
mappingFieldCount := intFromAny(feed["mapping_field_count"])
hasMappings := mappingFieldCount > 0
if v, ok := feed["has_mappings"].(bool); ok {
hasMappings = v || mappingFieldCount > 0
}
mappingIncomplete := !hasMappings
if v, ok := feed["mapping_incomplete"].(bool); ok {
mappingIncomplete = v
}
// last_data_at: live sync timestamp when present, else latest raw product update
// (covers MySQL→Postgres imports where last_synced_at was never set).
lastDataAt := lastSynced
if lastDataAt == nil {
lastDataAt = productsUpdated
}
out := map[string]any{
"id": stringifyID(feed["id"]),
"name": feed["name"],
"url": nullish(feed["url"]),
"item_path": itemPath,
"is_active": strings.EqualFold(status, "active"),
"product_count": productCount,
"mapping_field_count": mappingFieldCount,
"has_mappings": hasMappings,
"mapping_incomplete": mappingIncomplete,
"status": status,
"last_synced": lastSynced,
"created_at": formatAPITime(feed["created_at"]),
"updated_at": formatAPITime(feed["updated_at"]),
"feed_type": feed["feed_type"],
"sync_interval_minutes": feed["sync_interval_minutes"],
"last_synced_at": lastSynced,
"products_updated_at": productsUpdated,
"last_data_at": lastDataAt,
"options": opts,
}
if deltas, ok := opts["last_sync_deltas"].(map[string]any); ok && deltas != nil {
out["last_sync_deltas"] = deltas
}
return out
}
func intFromAny(v any) int {
switch t := v.(type) {
case int:
return t
case int32:
return int(t)
case int64:
return int(t)
case float64:
return int(t)
default:
return 0
}
}
// PresentFeeds maps a page of feed rows through PresentFeed.
func PresentFeeds(items []map[string]any) []map[string]any {
out := make([]map[string]any, 0, len(items))
for _, item := range items {
out = append(out, PresentFeed(item))
}
return out
}
func stringifyID(v any) any {
switch t := v.(type) {
case uuid.UUID:
return t.String()
case [16]byte:
return uuid.UUID(t).String()
default:
return v
}
}
func nullish(v any) any {
if v == nil {
return nil
}
if s, ok := v.(string); ok && s == "" {
return nil
}
return v
}
func formatAPITime(v any) any {
if v == nil {
return nil
}
switch t := v.(type) {
case time.Time:
if t.IsZero() {
return nil
}
return t.UTC().Format(time.RFC3339)
case *time.Time:
if t == nil || t.IsZero() {
return nil
}
return t.UTC().Format(time.RFC3339)
case string:
if t == "" {
return nil
}
return t
default:
return v
}
}
+143
View File
@@ -0,0 +1,143 @@
package feeds
import (
"testing"
"time"
)
func TestPresentFeedLegacyFields(t *testing.T) {
t.Parallel()
ts := time.Date(2026, 8, 4, 11, 0, 0, 0, time.UTC)
got := PresentFeed(map[string]any{
"id": "22222222-2222-2222-2222-222222222222",
"name": "Main catalog",
"url": "https://example.com/feed.xml",
"feed_type": "xml",
"status": "active",
"sync_interval_minutes": 60,
"last_synced_at": ts,
"options": map[string]any{"item_path": "channel/item"},
"product_count": int64(12),
"created_at": ts,
"updated_at": ts,
})
if got["item_path"] != "channel/item" {
t.Fatalf("item_path=%v", got["item_path"])
}
if got["is_active"] != true {
t.Fatalf("is_active=%v", got["is_active"])
}
if got["product_count"] != 12 {
t.Fatalf("product_count=%v", got["product_count"])
}
if got["last_synced"] != "2026-08-04T11:00:00Z" {
t.Fatalf("last_synced=%v", got["last_synced"])
}
if got["last_synced_at"] != got["last_synced"] {
t.Fatalf("dual last_synced_at mismatch")
}
if got["feed_type"] != "xml" {
t.Fatalf("feed_type=%v", got["feed_type"])
}
if got["last_data_at"] != got["last_synced_at"] {
t.Fatalf("last_data_at should prefer live sync: %v", got["last_data_at"])
}
}
func TestPresentFeedLastDataFromProducts(t *testing.T) {
t.Parallel()
ts := time.Date(2026, 8, 4, 11, 0, 0, 0, time.UTC)
got := PresentFeed(map[string]any{
"id": "1",
"name": "Imported",
"status": "active",
"product_count": int64(50),
"mapping_field_count": 19,
"has_mappings": true,
"products_updated_at": ts,
"last_synced_at": nil,
})
if got["last_synced_at"] != nil {
t.Fatalf("last_synced_at=%v", got["last_synced_at"])
}
if got["last_data_at"] != "2026-08-04T11:00:00Z" {
t.Fatalf("last_data_at=%v", got["last_data_at"])
}
if got["mapping_field_count"] != 19 {
t.Fatalf("mapping_field_count=%v", got["mapping_field_count"])
}
if got["has_mappings"] != true {
t.Fatalf("has_mappings=%v", got["has_mappings"])
}
if got["mapping_incomplete"] != false {
t.Fatalf("mapping_incomplete=%v want false when has_mappings and unset", got["mapping_incomplete"])
}
if got["product_count"] != 50 {
t.Fatalf("product_count=%v", got["product_count"])
}
}
func TestPresentFeedMappingIncomplete(t *testing.T) {
t.Parallel()
got := PresentFeed(map[string]any{
"id": "1",
"name": "Mapped incomplete",
"status": "mapped",
"mapping_field_count": 2,
"has_mappings": true,
"mapping_incomplete": true,
})
if got["mapping_incomplete"] != true {
t.Fatalf("mapping_incomplete=%v", got["mapping_incomplete"])
}
}
func TestPresentFeedsPreservesMappingIncomplete(t *testing.T) {
t.Parallel()
out := PresentFeeds([]map[string]any{
{
"id": "a",
"name": "Incomplete",
"status": "mapped",
"mapping_field_count": 1,
"has_mappings": true,
"mapping_incomplete": true,
},
{
"id": "b",
"name": "Complete",
"status": "active",
"mapping_field_count": 3,
"has_mappings": true,
"mapping_incomplete": false,
},
})
if len(out) != 2 {
t.Fatalf("len=%d", len(out))
}
if out[0]["mapping_incomplete"] != true || out[1]["mapping_incomplete"] != false {
t.Fatalf("got %#v %#v", out[0]["mapping_incomplete"], out[1]["mapping_incomplete"])
}
}
func TestPresentFeedEmptyURLAndInactive(t *testing.T) {
t.Parallel()
got := PresentFeed(map[string]any{
"id": "1",
"name": "Draft",
"url": "",
"status": "unmapped",
"options": map[string]any{},
"created_at": time.Unix(0, 0).UTC(),
"updated_at": time.Unix(0, 0).UTC(),
})
if got["url"] != nil {
t.Fatalf("url=%v want nil", got["url"])
}
if got["is_active"] != false {
t.Fatalf("is_active=%v", got["is_active"])
}
if got["item_path"] != "" {
t.Fatalf("item_path=%v", got["item_path"])
}
}
+635
View File
@@ -0,0 +1,635 @@
package feeds
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
type Service struct {
Pool *pgxpool.Pool
UploadDir string
}
// CreateInput is the payload for creating an input feed (URL and/or uploaded CSV).
// Legacy clients send name + item_path (url optional). V2 clients send name + url/file
// plus optional feed_type / sync_interval_minutes (or legacy sync_frequency in hours).
type CreateInput struct {
Name string
URL string
ItemPath string
FeedType string
SyncIntervalMinutes int
SyncFrequencyHours int // legacy alias; converted to minutes when SyncIntervalMinutes unset
Options map[string]any
}
func (s *Service) List(ctx context.Context, companyID uuid.UUID, limit, offset int, q string) ([]map[string]any, int64, int64, int64, error) {
q = strings.TrimSpace(q)
where := `company_id = $1`
args := []any{companyID}
if q != "" {
where += ` AND (
COALESCE(name, '') ILIKE '%' || $2 || '%' OR
COALESCE(url, '') ILIKE '%' || $2 || '%' OR
COALESCE(feed_type, '') ILIKE '%' || $2 || '%' OR
COALESCE(status, '') ILIKE '%' || $2 || '%' OR
COALESCE(options->>'source_filename', '') ILIKE '%' || $2 || '%'
)`
args = append(args, q)
}
var total int64
if err := s.Pool.QueryRow(ctx, `SELECT count(*) FROM input_feeds WHERE `+where, args...).Scan(&total); err != nil {
return nil, 0, 0, 0, err
}
// active_total = truly syncing; mapped_total = fields saved but not activated.
var activeTotal, mappedTotal int64
if err := s.Pool.QueryRow(ctx,
`SELECT
count(*) FILTER (WHERE lower(COALESCE(status, '')) = 'active'),
count(*) FILTER (WHERE lower(COALESCE(status, '')) = 'mapped')
FROM input_feeds WHERE `+where,
args...,
).Scan(&activeTotal, &mappedTotal); err != nil {
return nil, 0, 0, 0, err
}
limitArg := len(args) + 1
offsetArg := len(args) + 2
query := fmt.Sprintf(`
SELECT id, name, url, feed_type, status, sync_interval_minutes, last_synced_at, options, created_at, updated_at,
(SELECT count(*)::int FROM raw_products rp WHERE rp.feed_id = input_feeds.id AND rp.company_id = input_feeds.company_id) AS product_count,
(SELECT max(rp.updated_at) FROM raw_products rp WHERE rp.feed_id = input_feeds.id AND rp.company_id = input_feeds.company_id) AS products_updated_at
FROM input_feeds WHERE %s
ORDER BY created_at DESC LIMIT $%d OFFSET $%d`, where, limitArg, offsetArg)
queryArgs := append(append([]any{}, args...), limit, offset)
rows, err := s.Pool.Query(ctx, query, queryArgs...)
if err != nil {
return nil, 0, 0, 0, err
}
defer rows.Close()
items, err := scanMaps(rows, []string{"id", "name", "url", "feed_type", "status", "sync_interval_minutes", "last_synced_at", "options", "created_at", "updated_at", "product_count", "products_updated_at"})
if err != nil {
return nil, 0, 0, 0, err
}
if err := s.attachMappingFieldCounts(ctx, companyID, items); err != nil {
return nil, 0, 0, 0, err
}
return items, total, activeTotal, mappedTotal, nil
}
// ProductTotals is company-scoped catalog counts across all feeds.
type ProductTotals struct {
Total int64
Processed int64
Unprocessed int64
}
// CompanyProductTotals returns company-scoped catalog counts for feeds/dashboard cards.
// ASSUMPTION: Total = count(raw_products); Processed = count(processed_products);
// Unprocessed = count(raw where processing_status='unprocessed'). These are not a
// partition of Total (P+U≠Total by design). Do not redefine without documenting a new ASSUMPTION.
func (s *Service) CompanyProductTotals(ctx context.Context, companyID uuid.UUID) (ProductTotals, error) {
var t ProductTotals
err := s.Pool.QueryRow(ctx, `
SELECT
(SELECT count(*)::bigint FROM raw_products WHERE company_id = $1),
(SELECT count(*)::bigint FROM processed_products WHERE company_id = $1),
(SELECT count(*)::bigint FROM raw_products
WHERE company_id = $1 AND lower(COALESCE(processing_status, '')) = 'unprocessed')`,
companyID,
).Scan(&t.Total, &t.Processed, &t.Unprocessed)
return t, err
}
func (s *Service) Create(ctx context.Context, companyID uuid.UUID, in CreateInput) (map[string]any, error) {
name := strings.TrimSpace(in.Name)
if name == "" {
return nil, ClientMsg("name required")
}
url := strings.TrimSpace(in.URL)
if err := ValidateFeedURL(ctx, url); err != nil {
return nil, err
}
opts := in.Options
if opts == nil {
opts = map[string]any{}
}
itemPath := strings.TrimSpace(in.ItemPath)
if itemPath == "" {
if p, ok := opts["item_path"].(string); ok {
itemPath = strings.TrimSpace(p)
}
}
if itemPath != "" {
opts["item_path"] = itemPath
}
hasLocal := sourcePathFromOptions(opts) != ""
// Dual-support: legacy create allows name + item_path without url/file.
if url == "" && !hasLocal && itemPath == "" {
return nil, errSourceRequired
}
feedType := strings.ToLower(strings.TrimSpace(in.FeedType))
if feedType == "" {
if hasLocal {
feedType = "csv"
} else {
feedType = "xml"
}
}
if feedType != "xml" && feedType != "csv" {
return nil, ClientMsg("feed_type must be xml or csv")
}
interval := in.SyncIntervalMinutes
if interval <= 0 && in.SyncFrequencyHours > 0 {
interval = in.SyncFrequencyHours * 60
}
if interval <= 0 {
interval = 60
}
optsBytes, err := json.Marshal(opts)
if err != nil {
return nil, err
}
var id uuid.UUID
err = s.Pool.QueryRow(ctx, `
INSERT INTO input_feeds (company_id, name, url, feed_type, status, sync_interval_minutes, options)
VALUES ($1, $2, $3, $4, 'unmapped', $5, $6::jsonb) RETURNING id`,
companyID, name, nullStr(url), feedType, interval, optsBytes).Scan(&id)
if err != nil {
return nil, err
}
return s.Get(ctx, companyID, id)
}
func (s *Service) Get(ctx context.Context, companyID, id uuid.UUID) (map[string]any, error) {
row := s.Pool.QueryRow(ctx, `
SELECT id, name, url, feed_type, status, sync_interval_minutes, last_synced_at, options, created_at, updated_at,
(SELECT count(*)::int FROM raw_products rp WHERE rp.feed_id = input_feeds.id AND rp.company_id = input_feeds.company_id) AS product_count,
(SELECT max(rp.updated_at) FROM raw_products rp WHERE rp.feed_id = input_feeds.id AND rp.company_id = input_feeds.company_id) AS products_updated_at
FROM input_feeds WHERE id = $1 AND company_id = $2`, id, companyID)
item, err := scanMap(row, []string{"id", "name", "url", "feed_type", "status", "sync_interval_minutes", "last_synced_at", "options", "created_at", "updated_at", "product_count", "products_updated_at"})
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
return nil, err
}
if err := s.attachMappingFieldCounts(ctx, companyID, []map[string]any{item}); err != nil {
return nil, err
}
return item, nil
}
// attachMappingFieldCounts sets mapping_field_count / has_mappings / mapping_incomplete
// on each feed row from the active feed_mappings document (one query for the page).
// mapping_incomplete mirrors list-chip blocking preflight (empty/required/item_path).
func (s *Service) attachMappingFieldCounts(ctx context.Context, companyID uuid.UUID, items []map[string]any) error {
if len(items) == 0 {
return nil
}
ids := make([]uuid.UUID, 0, len(items))
index := make(map[uuid.UUID]map[string]any, len(items))
for _, item := range items {
id, ok := asUUID(item["id"])
if !ok {
continue
}
ids = append(ids, id)
index[id] = item
item["mapping_field_count"] = 0
item["has_mappings"] = false
item["mapping_incomplete"] = true
}
if len(ids) == 0 {
return nil
}
required, err := s.loadRequiredStandardFields(ctx, companyID)
if err != nil {
return err
}
rows, err := s.Pool.Query(ctx, `
SELECT DISTINCT ON (feed_id) feed_id, mappings
FROM feed_mappings
WHERE company_id = $1 AND is_active = true AND feed_id = ANY($2::uuid[])
ORDER BY feed_id, version DESC`, companyID, ids)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var feedID uuid.UUID
var raw []byte
if err := rows.Scan(&feedID, &raw); err != nil {
return err
}
item := index[feedID]
if item == nil {
continue
}
var parsed any
if err := json.Unmarshal(raw, &parsed); err != nil {
continue
}
mappings := parseMappings(parsed)
n := len(mappings)
item["mapping_field_count"] = n
item["has_mappings"] = n > 0
item["mapping_incomplete"] = mappingDocIncomplete(item, parsed, mappings, required)
}
return rows.Err()
}
func isCSVFeedType(feedType string) bool {
t := strings.ToLower(strings.TrimSpace(feedType))
return t == "csv" || t == "excel"
}
func feedItemPathHint(item map[string]any, mappingsRaw any) string {
if p := itemPathFromMappings(mappingsRaw); p != "" {
return p
}
if opts, ok := item["options"].(map[string]any); ok {
if v, ok := opts["item_path"].(string); ok {
if p := strings.TrimSpace(v); p != "" {
return p
}
}
}
if v, ok := item["item_path"].(string); ok {
return strings.TrimSpace(v)
}
return ""
}
// mappingDocIncomplete reports list-chip blocking gaps (empty mappings, required targets, XML item_path).
func mappingDocIncomplete(item map[string]any, mappingsRaw any, mappings []FieldMapping, required []requiredStandardField) bool {
if err := validateMappingsForSync(mappings, required); err != nil {
return true
}
feedType, _ := item["feed_type"].(string)
if !isCSVFeedType(feedType) && feedItemPathHint(item, mappingsRaw) == "" {
return true
}
return false
}
func asUUID(v any) (uuid.UUID, bool) {
switch t := v.(type) {
case uuid.UUID:
return t, true
case [16]byte:
return uuid.UUID(t), true
case string:
id, err := uuid.Parse(t)
return id, err == nil
default:
return uuid.Nil, false
}
}
func (s *Service) Update(ctx context.Context, companyID, id uuid.UUID, body map[string]any) (map[string]any, error) {
name, _ := body["name"].(string)
url, _ := body["url"].(string)
status, _ := body["status"].(string)
feedType, _ := body["feed_type"].(string)
itemPath, _ := body["item_path"].(string)
if err := ValidateFeedURL(ctx, url); err != nil {
return nil, err
}
feedType = strings.ToLower(strings.TrimSpace(feedType))
if feedType != "" && feedType != "xml" && feedType != "csv" {
return nil, ClientMsg("feed_type must be xml or csv")
}
interval := 0
switch v := body["sync_interval_minutes"].(type) {
case float64:
interval = int(v)
case int:
interval = v
case json.Number:
n, _ := v.Int64()
interval = int(n)
}
if interval <= 0 {
switch v := body["sync_frequency"].(type) {
case float64:
interval = int(v) * 60
case int:
interval = v * 60
}
}
ct, err := s.Pool.Exec(ctx, `
UPDATE input_feeds SET
name = CASE WHEN $3 <> '' THEN $3 ELSE name END,
url = CASE WHEN $4 <> '' THEN $4 ELSE url END,
status = CASE WHEN $5 <> '' THEN $5 ELSE status END,
feed_type = CASE WHEN $6 <> '' THEN $6 ELSE feed_type END,
sync_interval_minutes = CASE WHEN $7 > 0 THEN $7 ELSE sync_interval_minutes END,
options = CASE
WHEN $8 <> '' THEN COALESCE(options, '{}'::jsonb) || jsonb_build_object('item_path', to_jsonb($8::text))
ELSE options
END,
updated_at = now()
WHERE id = $1 AND company_id = $2`,
id, companyID, name, url, status, feedType, interval, strings.TrimSpace(itemPath))
if err != nil {
return nil, err
}
if ct.RowsAffected() == 0 {
return nil, ErrNotFound
}
return s.Get(ctx, companyID, id)
}
func (s *Service) Delete(ctx context.Context, companyID, id uuid.UUID) error {
ct, err := s.Pool.Exec(ctx, `DELETE FROM input_feeds WHERE id = $1 AND company_id = $2`, id, companyID)
if err != nil {
return err
}
if ct.RowsAffected() == 0 {
return ErrNotFound
}
return nil
}
func (s *Service) GetMappings(ctx context.Context, companyID, feedID uuid.UUID) (map[string]any, error) {
var id uuid.UUID
var version int
var mappings []byte
err := s.Pool.QueryRow(ctx, `
SELECT id, version, mappings FROM feed_mappings
WHERE feed_id = $1 AND company_id = $2 AND is_active = true
ORDER BY version DESC LIMIT 1`, feedID, companyID).Scan(&id, &version, &mappings)
if err != nil {
return nil, err
}
var m any
_ = json.Unmarshal(mappings, &m)
return map[string]any{"id": id, "version": version, "mappings": m}, nil
}
func (s *Service) PutMappings(ctx context.Context, companyID, feedID uuid.UUID, mappings any) (map[string]any, error) {
b, err := json.Marshal(mappings)
if err != nil {
return nil, err
}
var version int
_ = s.Pool.QueryRow(ctx, `
SELECT COALESCE(MAX(version), 0) FROM feed_mappings WHERE feed_id = $1`, feedID).Scan(&version)
version++
_, _ = s.Pool.Exec(ctx, `UPDATE feed_mappings SET is_active = false WHERE feed_id = $1`, feedID)
var id uuid.UUID
err = s.Pool.QueryRow(ctx, `
INSERT INTO feed_mappings (feed_id, company_id, version, mappings, is_active)
VALUES ($1, $2, $3, $4, true) RETURNING id`, feedID, companyID, version, b).Scan(&id)
if err != nil {
return nil, err
}
// Keep feed.options.item_path in sync for Sync() XML item selection, and
// flip unmapped -> mapped whenever at least one field mapping is saved.
path := itemPathFromMappings(mappings)
hasFields := len(parseMappings(mappings)) > 0
switch {
case path != "":
_, _ = s.Pool.Exec(ctx, `
UPDATE input_feeds SET
options = COALESCE(options, '{}'::jsonb) || jsonb_build_object('item_path', to_jsonb($3::text)),
status = CASE WHEN status = 'unmapped' THEN 'mapped' ELSE status END,
updated_at = now()
WHERE id = $1 AND company_id = $2`, feedID, companyID, path)
case hasFields:
_, _ = s.Pool.Exec(ctx, `
UPDATE input_feeds SET
status = CASE WHEN status = 'unmapped' THEN 'mapped' ELSE status END,
updated_at = now()
WHERE id = $1 AND company_id = $2`, feedID, companyID)
}
return map[string]any{"id": id, "version": version, "mappings": mappings}, nil
}
func (s *Service) ListExportFeeds(ctx context.Context, companyID uuid.UUID, limit, offset int) ([]map[string]any, int64, error) {
var total int64
if err := s.Pool.QueryRow(ctx, `SELECT count(*) FROM export_feeds WHERE company_id = $1`, companyID).Scan(&total); err != nil {
return nil, 0, err
}
rows, err := s.Pool.Query(ctx, `
SELECT id, name, source_feed_id, format, public_token, is_active, last_generated_at, created_at, updated_at
FROM export_feeds WHERE company_id = $1
ORDER BY created_at DESC LIMIT $2 OFFSET $3`, companyID, limit, offset)
if err != nil {
return nil, 0, err
}
defer rows.Close()
items, err := scanMaps(rows, []string{"id", "name", "source_feed_id", "format", "public_token", "is_active", "last_generated_at", "created_at", "updated_at"})
if err != nil {
return nil, 0, err
}
return items, total, nil
}
func (s *Service) GetExportFeed(ctx context.Context, companyID, id uuid.UUID) (map[string]any, error) {
rows, err := s.Pool.Query(ctx, `
SELECT id, name, source_feed_id, format, public_token, template, filters, is_active, last_generated_at, created_at, updated_at
FROM export_feeds WHERE id = $1 AND company_id = $2`, id, companyID)
if err != nil {
return nil, err
}
defer rows.Close()
items, err := scanMaps(rows, []string{"id", "name", "source_feed_id", "format", "public_token", "template", "filters", "is_active", "last_generated_at", "created_at", "updated_at"})
if err != nil {
return nil, err
}
if len(items) == 0 {
return nil, errors.New("not found")
}
return items[0], nil
}
func (s *Service) DeleteExportFeed(ctx context.Context, companyID, id uuid.UUID) error {
ct, err := s.Pool.Exec(ctx, `DELETE FROM export_feeds WHERE id = $1 AND company_id = $2`, id, companyID)
if err != nil {
return err
}
if ct.RowsAffected() == 0 {
return errors.New("not found")
}
return nil
}
func (s *Service) UpdateExportFeed(ctx context.Context, companyID, id uuid.UUID, name *string, isActive *bool, template, filters any) (map[string]any, error) {
current, err := s.GetExportFeed(ctx, companyID, id)
if err != nil {
return nil, err
}
if name != nil {
n := strings.TrimSpace(*name)
if n == "" {
return nil, ClientMsg("name required")
}
if _, err := s.Pool.Exec(ctx, `
UPDATE export_feeds SET name = $3, updated_at = now() WHERE id = $1 AND company_id = $2`, id, companyID, n); err != nil {
return nil, err
}
}
if isActive != nil {
if _, err := s.Pool.Exec(ctx, `
UPDATE export_feeds SET is_active = $3, updated_at = now() WHERE id = $1 AND company_id = $2`, id, companyID, *isActive); err != nil {
return nil, err
}
}
if template != nil || filters != nil {
tpl := template
flt := filters
if tpl == nil {
tpl = current["template"]
}
if flt == nil {
flt = current["filters"]
}
if _, err := s.UpdateExportFeedTemplate(ctx, companyID, id, tpl, flt); err != nil {
return nil, err
}
}
return s.GetExportFeed(ctx, companyID, id)
}
func (s *Service) CreateExportFeed(ctx context.Context, companyID uuid.UUID, in CreateExportInput) (map[string]any, error) {
if strings.TrimSpace(in.Name) == "" {
return nil, ClientMsg("name required")
}
format := strings.ToLower(strings.TrimSpace(in.Format))
if format == "" {
format = "xml"
}
if format != "xml" && format != "csv" {
return nil, ClientMsg("format must be xml or csv")
}
var src *uuid.UUID
if in.SourceFeedID != nil && *in.SourceFeedID != "" {
id, err := uuid.Parse(*in.SourceFeedID)
if err != nil {
return nil, ClientMsg("invalid source_feed_id")
}
src = &id
}
tplBytes := []byte("{}")
if in.Template != nil {
b, err := json.Marshal(in.Template)
if err != nil {
return nil, err
}
tplBytes = b
}
filterBytes := []byte("{}")
if in.Filters != nil {
b, err := json.Marshal(in.Filters)
if err != nil {
return nil, err
}
filterBytes = b
}
token, err := newPublicExportToken()
if err != nil {
return nil, err
}
var id uuid.UUID
err = s.Pool.QueryRow(ctx, `
INSERT INTO export_feeds (company_id, name, source_feed_id, format, template, filters, public_token)
VALUES ($1, $2, $3, $4, $5::jsonb, $6::jsonb, $7) RETURNING id, public_token`,
companyID, in.Name, src, format, tplBytes, filterBytes, token).Scan(&id, &token)
if err != nil {
return nil, err
}
return map[string]any{
"id": id, "name": in.Name, "format": format, "public_token": token,
"template": in.Template, "filters": in.Filters,
}, nil
}
// RotateExportFeedPublicToken replaces the public URL token (revokes the previous URL).
func (s *Service) RotateExportFeedPublicToken(ctx context.Context, companyID, id uuid.UUID) (map[string]any, error) {
token, err := newPublicExportToken()
if err != nil {
return nil, err
}
ct, err := s.Pool.Exec(ctx, `
UPDATE export_feeds SET public_token = $3, updated_at = now()
WHERE id = $1 AND company_id = $2`, id, companyID, token)
if err != nil {
return nil, err
}
if ct.RowsAffected() == 0 {
return nil, errors.New("not found")
}
return s.GetExportFeed(ctx, companyID, id)
}
func nullStr(s string) *string {
if s == "" {
return nil
}
return &s
}
func xmlEscape(s string) string {
r := strings.NewReplacer("&", "&amp;", "<", "&lt;", ">", "&gt;", `"`, "&quot;")
return r.Replace(s)
}
func scanMaps(rows pgx.Rows, cols []string) ([]map[string]any, error) {
out := make([]map[string]any, 0)
for rows.Next() {
vals := make([]any, len(cols))
ptrs := make([]any, len(cols))
for i := range vals {
ptrs[i] = &vals[i]
}
if err := rows.Scan(ptrs...); err != nil {
return nil, err
}
m := make(map[string]any, len(cols))
for i, c := range cols {
m[c] = normalize(vals[i])
}
out = append(out, m)
}
return out, rows.Err()
}
func scanMap(row pgx.Row, cols []string) (map[string]any, error) {
vals := make([]any, len(cols))
ptrs := make([]any, len(cols))
for i := range vals {
ptrs[i] = &vals[i]
}
if err := row.Scan(ptrs...); err != nil {
return nil, err
}
m := make(map[string]any, len(cols))
for i, c := range cols {
m[c] = normalize(vals[i])
}
return m, nil
}
func normalize(v any) any {
switch t := v.(type) {
case []byte:
var j any
if json.Unmarshal(t, &j) == nil {
return j
}
return string(t)
case [16]byte:
return uuid.UUID(t).String()
default:
return v
}
}
+155
View File
@@ -0,0 +1,155 @@
package feeds
import (
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/google/uuid"
)
var (
errSourceRequired = errors.New("feed url or uploaded CSV source required")
errLocalSource = errors.New("local feed source unavailable")
)
// feedBlob is feed content on disk. Close removes owned temp files (HTTP downloads).
// Local uploads reference the existing path and Close is a no-op.
type feedBlob struct {
path string
contentType string
size int64
owned bool
}
// Close removes the temp file when this blob owns it.
func (b *feedBlob) Close() error {
if b == nil || !b.owned || b.path == "" {
return nil
}
err := os.Remove(b.path)
b.path = ""
b.owned = false
return err
}
// Open returns a new read handle at the start of the blob.
func (b *feedBlob) Open() (*os.File, error) {
if b == nil || b.path == "" {
return nil, errors.New("feed blob closed or empty")
}
return os.Open(b.path)
}
// Sniff reads up to n bytes from the start of the blob (for format detection).
func (b *feedBlob) Sniff(n int) ([]byte, error) {
if n <= 0 {
return nil, nil
}
f, err := b.Open()
if err != nil {
return nil, err
}
defer f.Close()
buf := make([]byte, n)
nr, err := io.ReadFull(f, buf)
if err == io.EOF || err == io.ErrUnexpectedEOF {
err = nil
}
if err != nil {
return nil, err
}
return buf[:nr], nil
}
// loadFeedSource returns on-disk feed content from a local upload or HTTP(S) URL.
// Callers must Close the blob when finished.
func (s *Service) loadFeedSource(ctx context.Context, companyID uuid.UUID, feed map[string]any) (*feedBlob, error) {
if path := sourcePathFromOptions(feed["options"]); path != "" {
return s.readLocalFeed(companyID, path)
}
urlStr, _ := feed["url"].(string)
urlStr = strings.TrimSpace(urlStr)
if urlStr == "" {
return nil, errSourceRequired
}
return downloadFeed(ctx, urlStr)
}
func sourcePathFromOptions(raw any) string {
opts, ok := raw.(map[string]any)
if !ok || opts == nil {
return ""
}
for _, key := range []string{"source_path", "local_path", "file_path"} {
if v, ok := opts[key].(string); ok {
if p := strings.TrimSpace(v); p != "" {
return p
}
}
}
return ""
}
func (s *Service) readLocalFeed(companyID uuid.UUID, rel string) (*feedBlob, error) {
uploadDir := strings.TrimSpace(s.UploadDir)
if uploadDir == "" {
return nil, ClientMsg("upload directory not configured")
}
abs, err := resolveCompanyPath(uploadDir, companyID, rel)
if err != nil {
return nil, err
}
info, err := os.Stat(abs)
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("%w: file missing", errLocalSource)
}
return nil, err
}
if !info.Mode().IsRegular() {
return nil, ClientMsg("invalid source path")
}
if info.Size() > maxDownloadBytes {
return nil, downloadTooLarge()
}
ct := "text/csv"
lower := strings.ToLower(abs)
if strings.HasSuffix(lower, ".xml") {
ct = "application/xml"
}
return &feedBlob{
path: abs,
contentType: ct,
size: info.Size(),
owned: false,
}, nil
}
func resolveCompanyPath(uploadDir string, companyID uuid.UUID, rel string) (string, error) {
rel = filepath.ToSlash(strings.TrimSpace(rel))
if rel == "" || strings.Contains(rel, "..") {
return "", ClientMsg("invalid source path")
}
prefix := companyID.String() + "/"
if !strings.HasPrefix(rel, prefix) {
return "", ClientMsg("forbidden source path")
}
base, err := filepath.Abs(uploadDir)
if err != nil {
return "", err
}
abs, err := filepath.Abs(filepath.Join(uploadDir, filepath.FromSlash(rel)))
if err != nil {
return "", err
}
sep := string(os.PathSeparator)
if abs != base && !strings.HasPrefix(abs, base+sep) {
return "", ClientMsg("forbidden source path")
}
return abs, nil
}
+109
View File
@@ -0,0 +1,109 @@
package feeds
import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
"github.com/google/uuid"
)
func TestResolveCompanyPath(t *testing.T) {
t.Parallel()
base := t.TempDir()
cid := uuid.New()
rel := cid.String() + "/sample.csv"
absWant := filepath.Join(base, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(absWant), 0o750); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(absWant, []byte("a,b\n1,2\n"), 0o640); err != nil {
t.Fatal(err)
}
got, err := resolveCompanyPath(base, cid, rel)
if err != nil {
t.Fatalf("resolve: %v", err)
}
if filepath.Clean(got) != filepath.Clean(absWant) {
t.Fatalf("got %q want %q", got, absWant)
}
if _, err := resolveCompanyPath(base, cid, "../etc/passwd"); err == nil {
t.Fatal("expected traversal reject")
}
other := uuid.New()
if _, err := resolveCompanyPath(base, cid, other.String()+"/x.csv"); err == nil {
t.Fatal("expected company mismatch reject")
}
}
func TestReadLocalFeed(t *testing.T) {
t.Parallel()
base := t.TempDir()
cid := uuid.New()
rel := cid.String() + "/products.csv"
abs := filepath.Join(base, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(abs), 0o750); err != nil {
t.Fatal(err)
}
payload := []byte("ean,title\n123,Widget\n")
if err := os.WriteFile(abs, payload, 0o640); err != nil {
t.Fatal(err)
}
svc := &Service{UploadDir: base}
blob, err := svc.readLocalFeed(cid, rel)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = blob.Close() })
if blob.contentType != "text/csv" {
t.Fatalf("content-type %q", blob.contentType)
}
data, err := os.ReadFile(blob.path)
if err != nil {
t.Fatal(err)
}
if string(data) != string(payload) {
t.Fatalf("payload mismatch")
}
if blob.owned {
t.Fatal("local feed must not own path")
}
}
func TestReadLocalFeedRejectsOversized(t *testing.T) {
t.Parallel()
old := maxDownloadBytes
maxDownloadBytes = 32
t.Cleanup(func() { maxDownloadBytes = old })
base := t.TempDir()
cid := uuid.New()
rel := cid.String() + "/big.csv"
abs := filepath.Join(base, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(abs), 0o750); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(abs, []byte(strings.Repeat("x", 40)), 0o640); err != nil {
t.Fatal(err)
}
svc := &Service{UploadDir: base}
_, err := svc.readLocalFeed(cid, rel)
if !errors.Is(err, errDownloadTooLarge) {
t.Fatalf("err=%v want errDownloadTooLarge", err)
}
}
func TestSourcePathFromOptions(t *testing.T) {
t.Parallel()
if p := sourcePathFromOptions(map[string]any{"source_path": " a/b.csv "}); p != "a/b.csv" {
t.Fatalf("got %q", p)
}
if p := sourcePathFromOptions(nil); p != "" {
t.Fatalf("got %q", p)
}
}
+539
View File
@@ -0,0 +1,539 @@
package feeds
import (
"html"
"regexp"
"strings"
"unicode"
)
const (
maxSpecPairs = 200
maxSpecRawBytes = 64 << 10 // 64 KiB per specifications blob
maxSpecLabelRunes = 120
maxSpecValueRunes = 2000
)
var (
// Accept </li> and broken </> closers seen in A1 feed CDATA.
reHTMLLi = regexp.MustCompile(`(?is)<li\b[^>]*>(.*?)(?:</li\s*>|</\s*>)`)
reHTMLTag = regexp.MustCompile(`(?is)<[^>]+>`)
reMultiSpace = regexp.MustCompile(`\s+`)
)
// SpecPair is one label/value extracted from a specifications blob.
type SpecPair struct {
Label string
Value string
}
// ParseSpecifications accepts nested-expanded text, CDATA HTML lists, or flat
// CSV-like strings. Empty / missing / blank HTML returns nil (not an error).
func ParseSpecifications(raw string) []SpecPair {
raw = strings.TrimSpace(raw)
if raw == "" || len(raw) > maxSpecRawBytes {
return nil
}
if isEmptySpecBlob(raw) {
return nil
}
var pairs []SpecPair
switch {
case looksLikeHTMLList(raw):
pairs = parseHTMLSpecList(raw)
case looksLikeFlatSpecs(raw):
pairs = parseFlatSpecs(raw)
default:
// Single "Label: value" line still counts as flat.
if p, ok := splitLabelValue(raw); ok {
pairs = []SpecPair{p}
}
}
return clampSpecPairs(pairs)
}
// expandSpecificationFields mutates row: for specification-like keys whose
// value is HTML/flat text, add nested paths key/Label → value. Nested XML
// children are already present as key/child from the XML walker.
func expandSpecificationFields(row feedRow) {
if len(row) == 0 {
return
}
keys := make([]string, 0, 8)
for k, v := range row {
if !isSpecFieldKey(k) {
continue
}
if strings.TrimSpace(v) == "" {
continue
}
// Already has nested children — leave tree as-is; still parse text if useful.
if hasPrefixedChildren(row, k) && !looksLikeHTMLList(v) && !looksLikeFlatSpecs(v) {
continue
}
keys = append(keys, k)
}
for _, k := range keys {
pairs := ParseSpecifications(row[k])
for _, p := range pairs {
seg := sanitizePathSegment(p.Label)
if seg == "" {
continue
}
path := k + "/" + seg
if prev, ok := row[path]; ok && strings.TrimSpace(prev) != "" {
continue
}
row[path] = p.Value
}
}
}
func isSpecFieldKey(key string) bool {
leaf := strings.ToLower(leafName(key))
leaf = strings.TrimPrefix(leaf, "@")
switch leaf {
case "specifications", "specification", "specs", "spec", "features", "feature", "attributes_raw":
return true
}
return strings.Contains(leaf, "specification")
}
func hasPrefixedChildren(row feedRow, prefix string) bool {
prefix = strings.TrimSuffix(prefix, "/") + "/"
for k := range row {
if strings.HasPrefix(k, prefix) {
return true
}
}
return false
}
func collectPrefixed(row feedRow, prefix string) map[string]string {
prefix = strings.TrimSuffix(strings.TrimSpace(prefix), "/")
if prefix == "" {
return nil
}
p := prefix + "/"
out := map[string]string{}
for k, v := range row {
if !strings.HasPrefix(k, p) {
continue
}
rest := k[len(p):]
if rest == "" || strings.Contains(rest, "/") {
continue
}
v = strings.TrimSpace(v)
if v == "" {
continue
}
out[rest] = v
}
if len(out) == 0 {
return nil
}
return out
}
func isEmptySpecBlob(raw string) bool {
stripped := strings.TrimSpace(reHTMLTag.ReplaceAllString(raw, " "))
stripped = html.UnescapeString(stripped)
stripped = strings.TrimSpace(reMultiSpace.ReplaceAllString(stripped, " "))
return stripped == ""
}
func looksLikeHTMLList(raw string) bool {
lower := strings.ToLower(raw)
return strings.Contains(lower, "<li") || (strings.Contains(lower, "<ul") && strings.Contains(lower, "</ul>"))
}
func looksLikeFlatSpecs(raw string) bool {
if looksLikeHTMLList(raw) {
return false
}
// Multiple label:value pairs separated by ; | newline or comma between pairs.
if strings.Count(raw, ":") >= 2 {
return true
}
if strings.Count(raw, "=") >= 2 && (strings.Contains(raw, ";") || strings.Contains(raw, "|") || strings.Contains(raw, "\n")) {
return true
}
if strings.Contains(raw, ";") && strings.Contains(raw, ":") {
return true
}
if strings.Contains(raw, "|") && strings.Contains(raw, ":") {
return true
}
if strings.Contains(raw, "\n") && strings.Contains(raw, ":") {
return true
}
// Quoted CSV-ish pairs: "Brand","Acme"; "Model","X1"
if strings.Count(raw, `"`) >= 4 && (strings.Contains(raw, ";") || strings.Contains(raw, ",")) {
return true
}
return false
}
func parseHTMLSpecList(raw string) []SpecPair {
matches := reHTMLLi.FindAllStringSubmatch(raw, maxSpecPairs+1)
if len(matches) == 0 {
// Fallback: strip tags and try flat parse.
plain := strings.TrimSpace(reHTMLTag.ReplaceAllString(raw, "\n"))
plain = html.UnescapeString(plain)
return parseFlatSpecs(plain)
}
out := make([]SpecPair, 0, len(matches))
for _, m := range matches {
inner := strings.TrimSpace(reHTMLTag.ReplaceAllString(m[1], " "))
inner = html.UnescapeString(inner)
inner = strings.TrimSpace(reMultiSpace.ReplaceAllString(inner, " "))
if inner == "" {
continue
}
if p, ok := splitLabelValue(inner); ok {
out = append(out, p)
}
// Bare list items without "Label: value" are skipped — suppliers should
// send explicit pairs; inventing key→"true" produces junk attributes.
if len(out) >= maxSpecPairs {
break
}
}
return out
}
func parseFlatSpecs(raw string) []SpecPair {
raw = strings.ReplaceAll(raw, "\r\n", "\n")
raw = strings.ReplaceAll(raw, "\r", "\n")
chunks := splitSpecChunks(raw)
out := make([]SpecPair, 0, len(chunks))
for _, chunk := range chunks {
chunk = strings.TrimSpace(chunk)
if chunk == "" {
continue
}
// CSV-ish "Label","value" or Label,value
if strings.Contains(chunk, ",") {
if p, ok := parseCSVSpecChunk(chunk); ok {
out = append(out, p)
if len(out) >= maxSpecPairs {
break
}
continue
}
}
if p, ok := splitLabelValue(chunk); ok {
out = append(out, p)
}
if len(out) >= maxSpecPairs {
break
}
}
return out
}
func splitSpecChunks(raw string) []string {
// Prefer strong separators first.
for _, sep := range []string{"\n", ";", "|"} {
if strings.Contains(raw, sep) {
return strings.Split(raw, sep)
}
}
// Comma only when it looks like paired entries (has colon/equals).
if strings.Contains(raw, ",") && (strings.Contains(raw, ":") || strings.Contains(raw, "=")) {
return strings.Split(raw, ",")
}
return []string{raw}
}
func parseCSVSpecChunk(chunk string) (SpecPair, bool) {
parts := strings.SplitN(chunk, ",", 2)
if len(parts) != 2 {
return SpecPair{}, false
}
label := strings.Trim(strings.TrimSpace(parts[0]), `"'`)
value := strings.Trim(strings.TrimSpace(parts[1]), `"'`)
if label == "" || value == "" {
return SpecPair{}, false
}
return SpecPair{
Label: truncateRunes(label, maxSpecLabelRunes),
Value: truncateRunes(value, maxSpecValueRunes),
}, true
}
func splitLabelValue(s string) (SpecPair, bool) {
s = strings.TrimSpace(s)
if s == "" {
return SpecPair{}, false
}
// Prefer "Label: value" / "Labelvalue" / "Label - value" / "Label = value"
for _, sep := range []string{":", "", "=", "", "—"} {
if i := strings.Index(s, sep); i > 0 {
label := strings.TrimSpace(s[:i])
value := strings.TrimSpace(s[i+len(sep):])
if label != "" && value != "" && !looksLikeURLScheme(label) {
return SpecPair{
Label: truncateRunes(label, maxSpecLabelRunes),
Value: truncateRunes(value, maxSpecValueRunes),
}, true
}
}
}
// "Label - value" with spaces (avoid splitting hyphenated words alone)
if i := strings.Index(s, " - "); i > 0 {
label := strings.TrimSpace(s[:i])
value := strings.TrimSpace(s[i+3:])
if label != "" && value != "" {
return SpecPair{
Label: truncateRunes(label, maxSpecLabelRunes),
Value: truncateRunes(value, maxSpecValueRunes),
}, true
}
}
return SpecPair{}, false
}
func looksLikeURLScheme(label string) bool {
lower := strings.ToLower(strings.TrimSpace(label))
return lower == "http" || lower == "https" || lower == "ftp"
}
func sanitizePathSegment(label string) string {
label = strings.TrimSpace(label)
if label == "" {
return ""
}
var b strings.Builder
b.Grow(len(label))
prevUS := false
for _, r := range label {
switch {
case unicode.IsLetter(r) || unicode.IsDigit(r):
b.WriteRune(r)
prevUS = false
case r == '_' || r == '-' || r == '.':
b.WriteRune(r)
prevUS = false
case unicode.IsSpace(r) || r == '/' || r == '\\':
if !prevUS && b.Len() > 0 {
b.WriteByte('_')
prevUS = true
}
default:
// drop punctuation
}
}
out := strings.Trim(b.String(), "._-")
return truncateRunes(out, maxSpecLabelRunes)
}
func clampSpecPairs(pairs []SpecPair) []SpecPair {
if len(pairs) == 0 {
return nil
}
if len(pairs) > maxSpecPairs {
pairs = pairs[:maxSpecPairs]
}
seen := make(map[string]struct{}, len(pairs))
out := make([]SpecPair, 0, len(pairs))
for _, p := range pairs {
p.Label = strings.TrimSpace(p.Label)
p.Value = strings.TrimSpace(p.Value)
if p.Label == "" || p.Value == "" {
continue
}
key := strings.ToLower(p.Label)
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
out = append(out, p)
}
if len(out) == 0 {
return nil
}
return out
}
func truncateRunes(s string, max int) string {
if max <= 0 || s == "" {
return s
}
n := 0
for i := range s {
if n == max {
return s[:i]
}
n++
}
return s
}
func specsToMap(pairs []SpecPair) map[string]string {
if len(pairs) == 0 {
return nil
}
out := make(map[string]string, len(pairs))
for _, p := range pairs {
key := CanonicalAttributeKey(p.Label)
if key == "" || strings.TrimSpace(p.Value) == "" {
continue
}
out[key] = p.Value
}
if len(out) == 0 {
return nil
}
return out
}
// compactAttributeKey strips separators for alias lookup (net_height / net-height / netheight → netheight).
func compactAttributeKey(key string) string {
var b strings.Builder
b.Grow(len(key))
for _, r := range strings.ToLower(key) {
r = foldLatinRune(r)
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
b.WriteRune(r)
}
}
return b.String()
}
// Known supplier / locale labels → Descrybe standard field keys (snake_case).
// Freeform specs keep kebab-case from AttributeKeyFromLabel.
var attributeKeyAliases = map[string]string{
"visina": "net_height",
"height": "net_height",
"netheight": "net_height",
"sirina": "net_width",
"width": "net_width",
"netwidth": "net_width",
"globina": "net_depth",
"depth": "net_depth",
"netdepth": "net_depth",
"netmass": "net_mass",
"mass": "net_mass",
"weight": "net_mass",
"teza": "net_mass",
"productmodel": "product_model",
"model": "product_model",
"eprelid": "eprel_id",
"eprel": "eprel_id",
"energyclass": "energy_class",
"energijskirazred": "energy_class",
}
// IsValidAttributeKey rejects empty / punctuation-only / boolean junk keys.
func IsValidAttributeKey(key string) bool {
key = strings.TrimSpace(key)
if key == "" || len(key) < 2 {
return false
}
compact := compactAttributeKey(key)
if len(compact) < 2 {
return false
}
hasLetter := false
for _, r := range compact {
if r >= 'a' && r <= 'z' {
hasLetter = true
break
}
}
if !hasLetter {
return false
}
switch compact {
case "true", "false", "yes", "no", "null", "undefined", "none", "n", "y":
return false
}
return true
}
// CanonicalAttributeKey normalizes a human or feed label to a stable attribute key.
// Known dimension/identity aliases map to STANDARD_FIELDS snake_case; other labels
// become kebab-case. Invalid / junk labels return "".
func CanonicalAttributeKey(label string) string {
slug := AttributeKeyFromLabel(label)
compact := compactAttributeKey(slug)
if compact == "" {
compact = compactAttributeKey(label)
}
if alias, ok := attributeKeyAliases[compact]; ok {
return alias
}
if slug == "" || !IsValidAttributeKey(slug) {
return ""
}
return slug
}
// AttributeKeyFromLabel turns a human spec label into a kebab-case attribute_key
// (e.g. "Energijski razred" → "energijski-razred") matching company attributes.
func AttributeKeyFromLabel(label string) string {
label = strings.TrimSpace(label)
if label == "" {
return ""
}
var b strings.Builder
b.Grow(len(label))
prevHyphen := false
for _, r := range strings.ToLower(label) {
r = foldLatinRune(r)
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
b.WriteRune(r)
prevHyphen = false
case unicode.IsSpace(r) || r == '_' || r == '/' || r == '\\' || r == '.' || r == ':' || r == '-':
if !prevHyphen && b.Len() > 0 {
b.WriteByte('-')
prevHyphen = true
}
default:
// drop other punctuation
}
}
return strings.Trim(b.String(), "-")
}
func foldLatinRune(r rune) rune {
switch r {
case 'š', 'ś', 'ş':
return 's'
case 'č', 'ć', 'ç':
return 'c'
case 'ž', 'ź', 'ż':
return 'z'
case 'đ':
return 'd'
case 'ň', 'ń':
return 'n'
case 'ř':
return 'r'
case 'ť':
return 't'
case 'ď':
return 'd'
case 'ľ', 'ĺ':
return 'l'
case 'ä', 'á', 'à', 'â', 'ã', 'å':
return 'a'
case 'ë', 'é', 'è', 'ê':
return 'e'
case 'ï', 'í', 'ì', 'î':
return 'i'
case 'ö', 'ó', 'ò', 'ô', 'õ':
return 'o'
case 'ü', 'ú', 'ù', 'û':
return 'u'
case 'ý', 'ÿ':
return 'y'
default:
return r
}
}
+280
View File
@@ -0,0 +1,280 @@
package feeds
import (
"strings"
"testing"
)
func TestParseSpecificationsA1BrokenClosers(t *testing.T) {
raw := "<ul><li>Energijski razred: E</><li>Dimenzije: 605x95x395 </></ul>"
pairs := ParseSpecifications(raw)
if len(pairs) != 2 {
t.Fatalf("got %d pairs: %#v", len(pairs), pairs)
}
if pairs[0].Label != "Energijski razred" || pairs[0].Value != "E" {
t.Fatalf("first=%#v", pairs[0])
}
if pairs[1].Label != "Dimenzije" || pairs[1].Value != "605x95x395" {
t.Fatalf("second=%#v", pairs[1])
}
m := specsToMap(pairs)
if m["energy_class"] != "E" || m["dimenzije"] != "605x95x395" {
t.Fatalf("map=%#v", m)
}
}
func TestAttributeKeyFromLabel(t *testing.T) {
if got := AttributeKeyFromLabel("Energijski razred"); got != "energijski-razred" {
t.Fatalf("got %q", got)
}
if got := AttributeKeyFromLabel("Širina"); got != "sirina" {
t.Fatalf("got %q", got)
}
}
func TestCanonicalAttributeKey(t *testing.T) {
cases := map[string]string{
"Visina": "net_height",
"netheight": "net_height",
"net_height": "net_height",
"Širina": "net_width",
"Globina": "net_depth",
"netMass": "net_mass",
"Teža": "net_mass",
"Energijski razred": "energy_class",
":": "",
"true": "",
"": "",
}
for in, want := range cases {
if got := CanonicalAttributeKey(in); got != want {
t.Fatalf("%q → %q want %q", in, got, want)
}
}
}
func TestParseSpecificationsSkipsBareListItems(t *testing.T) {
raw := `<ul><li>:</li><li>Color: Red</li><li>Waterproof</li></ul>`
pairs := ParseSpecifications(raw)
if len(pairs) != 1 || pairs[0].Label != "Color" || pairs[0].Value != "Red" {
t.Fatalf("got %#v", pairs)
}
m := specsToMap(pairs)
if _, ok := m[":"]; ok {
t.Fatalf("junk key present: %#v", m)
}
if m["color"] != "Red" {
t.Fatalf("map=%#v", m)
}
}
func TestParseSpecificationsHTML(t *testing.T) {
raw := `<ul><li>Color: Red</li><li>Size: Large</li><li>Material: Cotton</li></ul>`
pairs := ParseSpecifications(raw)
if len(pairs) != 3 {
t.Fatalf("got %d pairs: %#v", len(pairs), pairs)
}
if pairs[0].Label != "Color" || pairs[0].Value != "Red" {
t.Fatalf("first=%#v", pairs[0])
}
}
func TestParseSpecificationsHTMLEmpty(t *testing.T) {
for _, raw := range []string{"", " ", "<ul></ul>", "<ul><li></li></ul>", "<![CDATA[]]>"} {
if got := ParseSpecifications(raw); got != nil {
t.Fatalf("raw=%q got %#v", raw, got)
}
}
}
func TestParseSpecificationsFlat(t *testing.T) {
raw := "Color: Red; Size: L; Weight: 1.2 kg"
pairs := ParseSpecifications(raw)
if len(pairs) != 3 {
t.Fatalf("got %d: %#v", len(pairs), pairs)
}
pipe := ParseSpecifications("Voltage: 230V | Frequency: 50Hz")
if len(pipe) != 2 {
t.Fatalf("pipe=%#v", pipe)
}
nl := ParseSpecifications("Width: 10\nHeight: 20")
if len(nl) != 2 {
t.Fatalf("nl=%#v", nl)
}
}
func TestParseSpecificationsCSVLike(t *testing.T) {
raw := `"Brand","Acme","Model","X1"`
// Single chunk without strong separators — treat as one line; may not split.
// Use semicolon CSV-ish pairs:
raw = `"Brand","Acme"; "Model","X1"`
pairs := ParseSpecifications(raw)
if len(pairs) < 2 {
t.Fatalf("got %#v", pairs)
}
}
func TestExpandSpecificationFieldsNestedXML(t *testing.T) {
xmlBody := `<?xml version="1.0"?>
<Export>
<Item>
<EAN>5901234123457</EAN>
<name>Washer</name>
<brand>Janus</brand>
<specifications>
<EnergyClass>A</EnergyClass>
<Capacity>8 kg</Capacity>
</specifications>
<EPRELID></EPRELID>
<netMass>72</netMass>
</Item>
</Export>`
var row feedRow
n, err := parseXMLItems(strings.NewReader(xmlBody), "Item", func(r feedRow) error {
row = r
return nil
})
if err != nil {
t.Fatal(err)
}
if n != 1 {
t.Fatalf("n=%d", n)
}
if row["specifications/EnergyClass"] != "A" {
t.Fatalf("nested energy=%q row=%v", row["specifications/EnergyClass"], row)
}
if row["specifications/Capacity"] != "8 kg" {
t.Fatalf("capacity=%q", row["specifications/Capacity"])
}
v, ok := lookupRow(row, "specifications/EnergyClass")
if !ok || v != "A" {
t.Fatalf("lookup nested: ok=%v v=%q", ok, v)
}
}
func TestExpandSpecificationFieldsCDATA(t *testing.T) {
xmlBody := `<?xml version="1.0"?>
<products>
<item>
<gtin>111</gtin>
<specifications><![CDATA[<ul><li>Color: Blue</li><li>Finish: Matte</li></ul>]]></specifications>
</item>
</products>`
var row feedRow
_, err := parseXMLItems(strings.NewReader(xmlBody), "item", func(r feedRow) error {
row = r
return nil
})
if err != nil {
t.Fatal(err)
}
if row["specifications/Color"] != "Blue" {
t.Fatalf("color=%q row=%v", row["specifications/Color"], row)
}
if row["specifications/Finish"] != "Matte" {
t.Fatalf("finish=%q", row["specifications/Finish"])
}
}
func TestExpandSpecificationFieldsFlatAndEmpty(t *testing.T) {
xmlBody := `<?xml version="1.0"?>
<products>
<item>
<ean>222</ean>
<specifications>Width: 10; Height: 20</specifications>
</item>
<item>
<ean>333</ean>
<specifications><![CDATA[<ul></ul>]]></specifications>
</item>
</products>`
var rows []feedRow
_, err := parseXMLItems(strings.NewReader(xmlBody), "item", func(r feedRow) error {
cp := make(feedRow, len(r))
for k, v := range r {
cp[k] = v
}
rows = append(rows, cp)
return nil
})
if err != nil {
t.Fatal(err)
}
if len(rows) != 2 {
t.Fatalf("rows=%d", len(rows))
}
if rows[0]["specifications/Width"] != "10" {
t.Fatalf("width=%q", rows[0]["specifications/Width"])
}
// Empty HTML must not invent children.
for k := range rows[1] {
if strings.HasPrefix(k, "specifications/") {
t.Fatalf("unexpected child %q", k)
}
}
}
func TestApplyMappingsSpecificationsObject(t *testing.T) {
row := feedRow{
"EAN": "5901234123457",
"specifications": `<ul><li>Color: Red</li></ul>`,
"specifications/Color": "Red",
"specifications/EnergyClass": "B",
}
expandSpecificationFields(row)
mappings := parseMappings([]any{
map[string]any{"source": "EAN", "target": "gtin"},
map[string]any{"source": "specifications", "target": "specifications"},
map[string]any{"source": "specifications/EnergyClass", "target": "energy_class"},
})
mapped, gtin := applyMappings(row, mappings)
if gtin != "5901234123457" {
t.Fatalf("gtin=%q", gtin)
}
specs, ok := mapped["specifications"].(map[string]string)
if !ok || specs["color"] != "Red" {
t.Fatalf("specs=%#v", mapped["specifications"])
}
if mapped["energy_class"] != "B" {
t.Fatalf("energy=%v", mapped["energy_class"])
}
}
func TestExtractXMLSchemaNestedSpecs(t *testing.T) {
data := []byte(`<?xml version="1.0"?>
<Export><Item>
<EAN>1</EAN>
<specifications><Color>Red</Color><Size>M</Size></specifications>
</Item></Export>`)
fields, rows, err := extractXMLSchema(data, "Item")
if err != nil {
t.Fatal(err)
}
if rows != 1 {
t.Fatalf("rows=%d", rows)
}
paths := map[string]bool{}
for _, f := range fields {
paths[f.Path] = true
}
if !paths["specifications/Color"] || !paths["specifications/Size"] {
t.Fatalf("missing nested paths: %#v", paths)
}
// Bare Color leaf should be suppressed when nested exists.
if paths["Color"] {
t.Fatalf("bare Color should be dropped: %#v", paths)
}
}
func TestLookupRowNestedPrefersPath(t *testing.T) {
row := feedRow{
"name": "Product",
"specifications": "ignored",
"specifications/foo": "nested",
"other/foo": "other",
}
v, ok := lookupRow(row, "specifications/foo")
if !ok || v != "nested" {
t.Fatalf("got ok=%v v=%q", ok, v)
}
}
+188
View File
@@ -0,0 +1,188 @@
package feeds
import "strings"
// MappingSuggestion is a suggested source→target pair from schema extraction.
type MappingSuggestion struct {
Source string `json:"source"`
Target string `json:"target"`
Confidence string `json:"confidence"`
Score float64 `json:"score"`
}
var sourceAliases = map[string][]string{
"gtin": {"gtin"},
"ean": {"gtin"},
"upc": {"gtin"},
"barcode": {"gtin"},
"title": {"title"},
"name": {"title"},
"productname": {"title"},
"brand": {"brand"},
"manufacturer": {"brand"},
"description": {"description"},
"desc": {"description"},
"price": {"price"},
"regularprice": {"price"},
"saleprice": {"sale_price", "price"},
"currency": {"currency"},
"image": {"image_url", "main_image", "image"},
"imageurl": {"image_url", "main_image", "image"},
"imagelink": {"image_url", "main_image", "image"},
"mainimage": {"image_url", "main_image", "image"},
"mainimageurl": {"image_url", "main_image", "image"},
"link": {"product_url"},
"url": {"product_url"},
"producturl": {"product_url"},
"sku": {"sku"},
"mpn": {"mpn"},
"category": {"category"},
"availability": {"availability"},
"stockstatus": {"availability", "stock_status"},
"stock": {"stock"},
"quantity": {"stock"},
"qty": {"stock"},
"color": {"color"},
"size": {"size"},
"material": {"material"},
"weight": {"weight"},
"netmass": {"weight"},
"purchaseprice": {"purchase_price", "price"},
"buyprice": {"purchase_price"},
"cost": {"purchase_price"},
"costprice": {"purchase_price"},
"officiallink": {"official_link"},
"warranty": {"warranty"},
"garancija": {"warranty"},
"service": {"service"},
"servis": {"service"},
"productmodel": {"product_model"},
"model": {"product_model"},
"modelnumber": {"product_model"},
"eprelid": {"eprel_id"},
"eprel": {"eprel_id"},
"specifications": {"specs", "specifications"},
"specs": {"specs", "specifications"},
"specification": {"specs", "specifications"},
"techspecs": {"specs", "specifications"},
}
func normalizeSuggestKey(raw string) string {
s := strings.ToLower(strings.TrimSpace(raw))
var b strings.Builder
for _, r := range s {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
b.WriteRune(r)
}
}
return b.String()
}
func leafSuggestKey(path string) string {
leaf := path
if i := strings.LastIndex(path, "/"); i >= 0 {
leaf = path[i+1:]
}
if i := strings.LastIndex(leaf, ":"); i >= 0 {
leaf = leaf[i+1:]
}
return normalizeSuggestKey(leaf)
}
// SuggestTarget returns the best canonical target key for a single source name/path,
// or "" when no alias matches. Used during schema extract to annotate fields.
func SuggestTarget(source string) string {
keys := []string{normalizeSuggestKey(source), leafSuggestKey(source)}
for _, key := range keys {
if key == "" {
continue
}
if aliases, ok := sourceAliases[key]; ok && len(aliases) > 0 {
return aliases[0]
}
// Exact identity for known-looking keys already in alias map as targets.
for _, aliases := range sourceAliases {
for _, a := range aliases {
if a == key {
return a
}
}
}
}
return ""
}
// SuggestMappings fuzzy-matches schema fields onto enabled target keys (1:1).
func SuggestMappings(schema []SchemaField, enabledTargets []string) []MappingSuggestion {
enabled := make(map[string]struct{}, len(enabledTargets))
for _, t := range enabledTargets {
t = strings.TrimSpace(t)
if t != "" {
enabled[t] = struct{}{}
}
}
if len(schema) == 0 || len(enabled) == 0 {
return nil
}
used := map[string]struct{}{}
type scored struct {
MappingSuggestion
order int
}
var candidates []scored
for i, f := range schema {
keys := []string{
normalizeSuggestKey(f.FieldName),
leafSuggestKey(f.Path),
normalizeSuggestKey(f.Path),
}
var hit *MappingSuggestion
for _, key := range keys {
if key == "" {
continue
}
if _, ok := enabled[key]; ok {
if _, taken := used[key]; !taken {
hit = &MappingSuggestion{Source: f.Path, Target: key, Confidence: "exact", Score: 1}
break
}
}
if aliases, ok := sourceAliases[key]; ok {
for _, a := range aliases {
if _, ok := enabled[a]; !ok {
continue
}
if _, taken := used[a]; taken {
continue
}
hit = &MappingSuggestion{Source: f.Path, Target: a, Confidence: "alias", Score: 0.95}
break
}
if hit != nil {
break
}
}
}
if hit != nil {
candidates = append(candidates, scored{MappingSuggestion: *hit, order: i})
}
}
// Prefer higher score, then earlier schema order.
for i := 0; i < len(candidates); i++ {
for j := i + 1; j < len(candidates); j++ {
if candidates[j].Score > candidates[i].Score ||
(candidates[j].Score == candidates[i].Score && candidates[j].order < candidates[i].order) {
candidates[i], candidates[j] = candidates[j], candidates[i]
}
}
}
out := make([]MappingSuggestion, 0, len(candidates))
for _, c := range candidates {
if _, taken := used[c.Target]; taken {
continue
}
used[c.Target] = struct{}{}
out = append(out, c.MappingSuggestion)
}
return out
}
+99
View File
@@ -0,0 +1,99 @@
package feeds
import (
"strings"
"testing"
)
func TestSuggestTarget_b2bFields(t *testing.T) {
cases := map[string]string{
"purchasePrice": "purchase_price",
"stockStatus": "availability",
"officialLink": "official_link",
"warranty": "warranty",
"service": "service",
"productModel": "product_model",
"EPRELID": "eprel_id",
"EAN": "gtin",
"name": "title",
"netMass": "weight",
"mainImage": "image_url",
}
for in, want := range cases {
if got := SuggestTarget(in); got != want {
t.Fatalf("%s: got %q want %q", in, got, want)
}
}
}
func TestNormalizeSuggestKey(t *testing.T) {
cases := map[string]string{
"EAN": "ean",
"purchasePrice": "purchaseprice",
"mainImage": "mainimage",
"EPRELID": "eprelid",
"specifications": "specifications",
}
for in, want := range cases {
got := normalizeSuggestKey(in)
if got != want {
t.Fatalf("%q: got %q want %q", in, got, want)
}
}
if leafSuggestKey("rss/channel/item/g:gtin") != "gtin" {
t.Fatalf("leaf g:gtin → %q", leafSuggestKey("rss/channel/item/g:gtin"))
}
}
func TestSuggestMappingsAliases(t *testing.T) {
schema := []SchemaField{
{Path: "EAN", FieldName: "EAN"},
{Path: "name", FieldName: "name"},
{Path: "brand", FieldName: "brand"},
{Path: "purchasePrice", FieldName: "purchasePrice"},
{Path: "stock", FieldName: "stock"},
{Path: "mainImage", FieldName: "mainImage"},
{Path: "EPRELID", FieldName: "EPRELID"},
{Path: "specifications", FieldName: "specifications"},
}
targets := []string{
"gtin", "title", "brand", "price", "purchase_price", "stock", "image_url", "eprel_id", "specs",
}
got := SuggestMappings(schema, targets)
bySrc := map[string]string{}
for _, s := range got {
bySrc[s.Source] = s.Target
}
want := map[string]string{
"EAN": "gtin",
"name": "title",
"brand": "brand",
"purchasePrice": "purchase_price",
"stock": "stock",
"mainImage": "image_url",
"EPRELID": "eprel_id",
"specifications": "specs",
}
for src, tgt := range want {
if bySrc[src] != tgt {
t.Fatalf("%s → %q, want %q (all=%v)", src, bySrc[src], tgt, bySrc)
}
}
}
func TestSuggestMappingsRespectsEnabled(t *testing.T) {
schema := []SchemaField{{Path: "EAN", FieldName: "EAN"}, {Path: "name", FieldName: "name"}}
got := SuggestMappings(schema, []string{"title"})
if len(got) != 1 || got[0].Target != "title" {
t.Fatalf("expected only title, got %#v", got)
}
}
func TestLeafSuggestKey(t *testing.T) {
if leafSuggestKey("rss/channel/item/g:gtin") != "gtin" {
t.Fatalf("got %q", leafSuggestKey("rss/channel/item/g:gtin"))
}
if !strings.Contains(normalizeSuggestKey("Foo-Bar"), "foobar") {
t.Fatal("normalize")
}
}
+865
View File
@@ -0,0 +1,865 @@
package feeds
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"io"
"strings"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
const upsertChunkSize = 100
type syncStats struct {
Total int
Synced int
Skipped int
Unchanged int
Progress int
ContentHash string
UnchangedFeed bool
Deltas syncDeltaCounts
}
type pendingProduct struct {
GTIN string
RawData map[string]string
MappedData map[string]any
ContentHash string
}
// Sync downloads the feed URL, parses CSV/XML, applies mappings, and upserts raw_products
// in chunks with progress updates on feed_sync_jobs. Replaces SyncStub.
func (s *Service) Sync(ctx context.Context, companyID, feedID uuid.UUID) (map[string]any, error) {
feed, err := s.Get(ctx, companyID, feedID)
if err != nil {
if IsNotFound(err) {
return nil, ErrNotFound
}
return nil, err
}
if err := s.ensureMappingsReadyForSync(ctx, companyID, feedID); err != nil {
return nil, err
}
jobID, err := s.createSyncJob(ctx, companyID, feedID)
if err != nil {
return nil, err
}
if err := s.markJobRunning(ctx, jobID); err != nil {
return nil, err
}
stats, runErr := s.runSync(ctx, companyID, feedID, jobID, feed)
if runErr != nil {
_ = s.failJob(ctx, jobID, runErr.Error(), stats)
return nil, runErr
}
if err := s.completeJob(ctx, jobID, stats); err != nil {
return nil, err
}
_ = s.persistFeedSyncMeta(ctx, companyID, feedID, jobID, stats)
job, err := s.GetSyncJob(ctx, companyID, feedID, jobID)
if err != nil {
return nil, err
}
return enrichJobWithDeltas(job, stats.Deltas), nil
}
// EnqueueSync creates a pending feed_sync_jobs row and wakes the worker via NOTIFY.
// Same-feed pending jobs are reused (no duplicate pending stack). The API process
// does not run sync work (no unbound goroutines). Returns the job id immediately
// for 202/poll (dashboard) or legacy 200 + jobId (v1).
func (s *Service) EnqueueSync(ctx context.Context, companyID, feedID uuid.UUID) (uuid.UUID, error) {
if _, err := s.Get(ctx, companyID, feedID); err != nil {
if IsNotFound(err) {
return uuid.Nil, ErrNotFound
}
return uuid.Nil, err
}
if err := s.ensureMappingsReadyForSync(ctx, companyID, feedID); err != nil {
return uuid.Nil, err
}
jobID, err := s.findOrCreatePendingSyncJob(ctx, companyID, feedID)
if err != nil {
return uuid.Nil, err
}
_, _ = s.Pool.Exec(ctx, `SELECT pg_notify('feed_sync_jobs', $1)`, jobID.String())
return jobID, nil
}
// ClaimNextPendingSyncJob claims one pending feed_sync_jobs row (FOR UPDATE SKIP LOCKED)
// and marks it running for the worker.
func (s *Service) ClaimNextPendingSyncJob(ctx context.Context) (jobID, companyID, feedID uuid.UUID, err error) {
err = s.Pool.QueryRow(ctx, `
WITH candidate AS (
SELECT id FROM feed_sync_jobs
WHERE status = 'pending'
ORDER BY created_at ASC, id ASC
LIMIT 1
FOR UPDATE SKIP LOCKED
)
UPDATE feed_sync_jobs j
SET status = 'running', started_at = now(), updated_at = now()
FROM candidate
WHERE j.id = candidate.id
RETURNING j.id, j.company_id, j.feed_id`).Scan(&jobID, &companyID, &feedID)
if errors.Is(err, pgx.ErrNoRows) {
return uuid.Nil, uuid.Nil, uuid.Nil, pgx.ErrNoRows
}
return jobID, companyID, feedID, err
}
// ProcessSyncJob runs sync for a job already claimed (status=running) by ClaimNextPendingSyncJob.
func (s *Service) ProcessSyncJob(ctx context.Context, companyID, feedID, jobID uuid.UUID) error {
feed, err := s.Get(ctx, companyID, feedID)
if err != nil {
msg := err.Error()
if IsNotFound(err) {
msg = "feed not found"
}
_ = s.failJob(ctx, jobID, msg, syncStats{})
return err
}
stats, runErr := s.runSync(ctx, companyID, feedID, jobID, feed)
if runErr != nil {
_ = s.failJob(ctx, jobID, runErr.Error(), stats)
return runErr
}
if err := s.completeJob(ctx, jobID, stats); err != nil {
_ = s.failJob(ctx, jobID, err.Error(), stats)
return err
}
_ = s.persistFeedSyncMeta(ctx, companyID, feedID, jobID, stats)
return nil
}
// SyncStub is retained as a compatibility alias for Sync.
func (s *Service) SyncStub(ctx context.Context, companyID, feedID uuid.UUID) (map[string]any, error) {
return s.Sync(ctx, companyID, feedID)
}
func (s *Service) GetSyncJob(ctx context.Context, companyID, feedID, jobID uuid.UUID) (map[string]any, error) {
row := s.Pool.QueryRow(ctx, `
SELECT id, feed_id, company_id, status, started_at, completed_at,
products_synced, products_total, products_skipped, products_unchanged,
progress, content_hash, error, created_at, updated_at
FROM feed_sync_jobs
WHERE id = $1 AND company_id = $2 AND feed_id = $3`, jobID, companyID, feedID)
job, err := scanMap(row, []string{
"id", "feed_id", "company_id", "status", "started_at", "completed_at",
"products_synced", "products_total", "products_skipped", "products_unchanged",
"progress", "content_hash", "error", "created_at", "updated_at",
})
if err != nil {
return nil, err
}
if deltas, ok := s.loadFeedLastSyncDeltas(ctx, companyID, feedID); ok {
if matchJobID(deltas["job_id"], jobID) {
return enrichJobWithDeltasMap(job, deltas), nil
}
}
return job, nil
}
func (s *Service) persistFeedSyncMeta(ctx context.Context, companyID, feedID, jobID uuid.UUID, stats syncStats) error {
stats.Deltas.JobID = jobID.String()
stats.Deltas.Unchanged = stats.Unchanged
stats.Deltas.Skipped = stats.Skipped
deltaJSON, err := json.Marshal(stats.Deltas.asMap())
if err != nil {
return err
}
_, err = s.Pool.Exec(ctx, `
UPDATE input_feeds SET last_synced_at = now(), updated_at = now(),
options = COALESCE(options, '{}'::jsonb) || jsonb_build_object(
'last_content_hash', to_jsonb($2::text),
'last_sync_deltas', $3::jsonb
)
WHERE id = $1 AND company_id = $4`, feedID, stats.ContentHash, deltaJSON, companyID)
return err
}
func (s *Service) loadFeedLastSyncDeltas(ctx context.Context, companyID, feedID uuid.UUID) (map[string]any, bool) {
var raw []byte
err := s.Pool.QueryRow(ctx, `
SELECT options->'last_sync_deltas' FROM input_feeds
WHERE id = $1 AND company_id = $2`, feedID, companyID).Scan(&raw)
if err != nil || len(raw) == 0 || string(raw) == "null" {
return nil, false
}
var m map[string]any
if err := json.Unmarshal(raw, &m); err != nil || m == nil {
return nil, false
}
return m, true
}
func enrichJobWithDeltas(job map[string]any, d syncDeltaCounts) map[string]any {
return enrichJobWithDeltasMap(job, d.asMap())
}
func enrichJobWithDeltasMap(job map[string]any, deltas map[string]any) map[string]any {
if job == nil {
return nil
}
out := make(map[string]any, len(job)+1)
for k, v := range job {
out[k] = v
}
out["deltas"] = deltas
out["price_changed"] = deltas["price_changed"]
out["stock_changed"] = deltas["stock_changed"]
out["availability_changed"] = deltas["availability_changed"]
out["title_changed"] = deltas["title_changed"]
out["other_changed"] = deltas["other_changed"]
out["products_new"] = deltas["new"]
return out
}
func matchJobID(v any, id uuid.UUID) bool {
switch t := v.(type) {
case string:
return strings.EqualFold(strings.TrimSpace(t), id.String())
case uuid.UUID:
return t == id
default:
return false
}
}
func (s *Service) ListSyncJobs(ctx context.Context, companyID, feedID uuid.UUID, limit int) ([]map[string]any, error) {
if limit <= 0 {
limit = 50
}
if limit > 200 {
limit = 200
}
rows, err := s.Pool.Query(ctx, `
SELECT id, feed_id, status, started_at, completed_at,
products_synced, products_total, products_skipped, products_unchanged,
progress, content_hash, error, created_at
FROM feed_sync_jobs
WHERE company_id = $1 AND feed_id = $2
ORDER BY created_at DESC LIMIT $3`, companyID, feedID, limit)
if err != nil {
return nil, err
}
defer rows.Close()
return scanMaps(rows, []string{
"id", "feed_id", "status", "started_at", "completed_at",
"products_synced", "products_total", "products_skipped", "products_unchanged",
"progress", "content_hash", "error", "created_at",
})
}
func (s *Service) runSync(ctx context.Context, companyID, feedID, jobID uuid.UUID, feed map[string]any) (syncStats, error) {
var stats syncStats
urlStr, _ := feed["url"].(string)
feedType, _ := feed["feed_type"].(string)
src, err := s.loadFeedSource(ctx, companyID, feed)
if err != nil {
return stats, err
}
defer src.Close()
hash, err := sha256HexFile(src)
if err != nil {
return stats, err
}
stats.ContentHash = hash
prev, _ := s.lastContentHash(ctx, feedID)
if prev != "" && prev == hash {
stats.UnchangedFeed = true
stats.Progress = 100
_ = s.updateJobProgress(ctx, jobID, stats)
return stats, nil
}
mappingsRaw, mapErr := s.loadMappingsRaw(ctx, companyID, feedID)
if mapErr != nil && !errors.Is(mapErr, pgx.ErrNoRows) {
return stats, mapErr
}
mappings := activeMappings(parseMappings(mappingsRaw))
if len(mappings) == 0 {
return stats, ClientMsg("no field mappings defined for feed")
}
itemPath := "item"
if path := itemPathFromMappings(mappingsRaw); path != "" {
itemPath = itemLocalFromPath(path)
} else if opts, ok := feed["options"].(map[string]any); ok {
if v, ok := opts["item_path"].(string); ok && strings.TrimSpace(v) != "" {
itemPath = itemLocalFromPath(v)
}
}
if itemPath == "" {
itemPath = "item"
}
sample, sniffErr := src.Sniff(4096)
if sniffErr != nil {
return stats, sniffErr
}
format := detectFeedFormat(feedType, contentTypeFromBlob(src), urlStr, sample)
chunk := make([]pendingProduct, 0, upsertChunkSize)
flush := func(force bool) error {
if len(chunk) == 0 {
return nil
}
if !force && len(chunk) < upsertChunkSize {
return nil
}
synced, unchanged, skipped, deltas, err := s.upsertChunk(ctx, companyID, feedID, jobID, chunk)
if err != nil {
return err
}
stats.Synced += synced
stats.Unchanged += unchanged
stats.Skipped += skipped
stats.Deltas.New += deltas.New
stats.Deltas.PriceChanged += deltas.PriceChanged
stats.Deltas.StockChanged += deltas.StockChanged
stats.Deltas.AvailabilityChanged += deltas.AvailabilityChanged
stats.Deltas.TitleChanged += deltas.TitleChanged
stats.Deltas.OtherChanged += deltas.OtherChanged
chunk = chunk[:0]
done := stats.Synced + stats.Unchanged + stats.Skipped
if stats.Total > 0 {
stats.Progress = done * 100 / stats.Total
if stats.Progress > 99 {
stats.Progress = 99
}
}
_ = s.updateJobProgress(ctx, jobID, stats)
return nil
}
onRow := func(row feedRow) error {
stats.Total++
mapped, gtin := applyMappings(row, mappings)
if gtin == "" {
stats.Skipped++
return nil
}
rawCopy := make(map[string]string, len(row))
for k, v := range row {
rawCopy[k] = v
}
mh := sha256Hex([]byte(mustJSON(mapped)))
chunk = append(chunk, pendingProduct{
GTIN: gtin, RawData: rawCopy, MappedData: mapped, ContentHash: mh,
})
return flush(false)
}
body, err := src.Open()
if err != nil {
return stats, err
}
defer body.Close()
var parseCount int
switch format {
case "xml":
parseCount, err = parseXMLItems(body, itemPath, onRow)
default:
parseCount, err = parseCSV(body, onRow)
}
if err != nil {
return stats, err
}
if parseCount == 0 {
return stats, ClientMsg("feed contained no rows")
}
if err := flush(true); err != nil {
return stats, err
}
stats.Progress = 100
_ = s.updateJobProgress(ctx, jobID, stats)
return stats, nil
}
type existingProduct struct {
ID uuid.UUID
MappedData []byte
}
const (
// Set-based upserts (UNNEST / ANY) — one statement per op kind, queued in a single
// pgx.Batch round-trip. Mirrors catalog/import_csv.go patterns.
upsertSQLInsertSet = `
INSERT INTO raw_products (
company_id, gtin, feed_id, raw_data, mapped_data, sync_job_id,
is_processed, processing_status, updated_at
)
SELECT $1, v.gtin, $2, v.raw_data::jsonb, v.mapped_data::jsonb, $3, false, 'unprocessed', now()
FROM unnest($4::text[], $5::text[], $6::text[]) AS v(gtin, raw_data, mapped_data)
ON CONFLICT (company_id, gtin) DO UPDATE SET
feed_id = EXCLUDED.feed_id,
raw_data = EXCLUDED.raw_data,
mapped_data = EXCLUDED.mapped_data,
sync_job_id = EXCLUDED.sync_job_id,
is_processed = false,
processing_status = 'unprocessed',
updated_at = now()`
upsertSQLTouchSet = `
UPDATE raw_products SET sync_job_id = $3, feed_id = $4, updated_at = now()
WHERE company_id = $1 AND id = ANY($2::uuid[])`
upsertSQLUpdateSet = `
UPDATE raw_products AS r SET
feed_id = $2,
raw_data = v.raw_data::jsonb,
mapped_data = v.mapped_data::jsonb,
sync_job_id = $3,
is_processed = false,
processing_status = 'unprocessed',
updated_at = now()
FROM unnest($4::uuid[], $5::text[], $6::text[]) AS v(id, raw_data, mapped_data)
WHERE r.id = v.id AND r.company_id = $1`
)
type upsertOpKind int
const (
upsertOpInsert upsertOpKind = iota
upsertOpTouch
upsertOpUpdate
)
type upsertOp struct {
kind upsertOpKind
gtin string
id uuid.UUID
rawJSON []byte
mappedJSON []byte
changes []string
}
// classifyUpsertOps decides insert / touch / update per row without DB I/O so a chunk
// can be applied with set-based UNNEST statements (≤3) in one pgx.Batch round-trip.
// Touch keeps is_processed/processing_status when mapped_data is equal (canonical JSON).
// Updates/inserts stamp mapped_data._sync_changes for seller filters (price/stock/…).
func classifyUpsertOps(chunk []pendingProduct, existing map[string]existingProduct) (ops []upsertOp, skipped int) {
ops = make([]upsertOp, 0, len(chunk))
for _, p := range chunk {
rawJSON, err := json.Marshal(p.RawData)
if err != nil {
skipped++
continue
}
ex, found := existing[p.GTIN]
cleanMapped := stripSyncChanges(p.MappedData)
if !found {
withFlags := withSyncChanges(cleanMapped, []string{"new"})
mappedJSON, err := json.Marshal(withFlags)
if err != nil {
skipped++
continue
}
ops = append(ops, upsertOp{
kind: upsertOpInsert, gtin: p.GTIN, rawJSON: rawJSON, mappedJSON: mappedJSON,
changes: []string{"new"},
})
continue
}
// Category (and similar extras) live outside feed mappings — keep them.
cleanMapped = preserveSyncedExtras(ex.MappedData, cleanMapped)
plainJSON, err := json.Marshal(cleanMapped)
if err != nil {
skipped++
continue
}
if bytesEqualJSON(stripSyncChangesBytes(ex.MappedData), plainJSON) {
ops = append(ops, upsertOp{kind: upsertOpTouch, id: ex.ID})
continue
}
changes := detectMappedChanges(ex.MappedData, cleanMapped)
withFlags := withSyncChanges(cleanMapped, changes)
mappedJSON, err := json.Marshal(withFlags)
if err != nil {
skipped++
continue
}
ops = append(ops, upsertOp{
kind: upsertOpUpdate, id: ex.ID, rawJSON: rawJSON, mappedJSON: mappedJSON,
changes: changes,
})
}
return ops, skipped
}
func stripSyncChangesBytes(raw []byte) []byte {
if len(raw) == 0 {
return raw
}
var m map[string]any
if err := json.Unmarshal(raw, &m); err != nil {
return raw
}
out, err := json.Marshal(stripSyncChanges(m))
if err != nil {
return raw
}
return out
}
// dedupePendingByGTIN keeps first-seen order but last-seen payload per GTIN so a single
// UNNEST INSERT cannot hit "cannot affect row a second time".
func dedupePendingByGTIN(chunk []pendingProduct) []pendingProduct {
if len(chunk) < 2 {
return chunk
}
by := make(map[string]pendingProduct, len(chunk))
order := make([]string, 0, len(chunk))
for _, p := range chunk {
if _, ok := by[p.GTIN]; !ok {
order = append(order, p.GTIN)
}
by[p.GTIN] = p
}
if len(order) == len(chunk) {
return chunk
}
out := make([]pendingProduct, 0, len(order))
for _, gtin := range order {
out = append(out, by[gtin])
}
return out
}
func partitionUpsertOps(ops []upsertOp) (inserts, touches, updates []upsertOp) {
for _, op := range ops {
switch op.kind {
case upsertOpInsert:
inserts = append(inserts, op)
case upsertOpTouch:
touches = append(touches, op)
default:
updates = append(updates, op)
}
}
return inserts, touches, updates
}
func (s *Service) upsertChunk(ctx context.Context, companyID, feedID, jobID uuid.UUID, chunk []pendingProduct) (synced, unchanged, skipped int, deltas syncDeltaCounts, err error) {
chunk = dedupePendingByGTIN(chunk)
if len(chunk) == 0 {
return 0, 0, 0, deltas, nil
}
gtins := make([]string, 0, len(chunk))
for _, p := range chunk {
gtins = append(gtins, p.GTIN)
}
existing, err := s.loadExistingByGTIN(ctx, companyID, gtins)
if err != nil {
return 0, 0, 0, deltas, err
}
ops, skipped := classifyUpsertOps(chunk, existing)
if len(ops) == 0 {
deltas.Skipped = skipped
return 0, 0, skipped, deltas, nil
}
inserts, touches, updates := partitionUpsertOps(ops)
batch := &pgx.Batch{}
queued := 0
if len(inserts) > 0 {
gtinCol := make([]string, len(inserts))
rawCol := make([]string, len(inserts))
mappedCol := make([]string, len(inserts))
for i, op := range inserts {
gtinCol[i] = op.gtin
rawCol[i] = string(op.rawJSON)
mappedCol[i] = string(op.mappedJSON)
deltas.addChanges(op.changes)
}
batch.Queue(upsertSQLInsertSet, companyID, feedID, jobID, gtinCol, rawCol, mappedCol)
queued++
}
if len(touches) > 0 {
ids := make([]uuid.UUID, len(touches))
for i, op := range touches {
ids[i] = op.id
}
batch.Queue(upsertSQLTouchSet, companyID, ids, jobID, feedID)
queued++
}
if len(updates) > 0 {
ids := make([]uuid.UUID, len(updates))
rawCol := make([]string, len(updates))
mappedCol := make([]string, len(updates))
for i, op := range updates {
ids[i] = op.id
rawCol[i] = string(op.rawJSON)
mappedCol[i] = string(op.mappedJSON)
deltas.addChanges(op.changes)
}
batch.Queue(upsertSQLUpdateSet, companyID, feedID, jobID, ids, rawCol, mappedCol)
queued++
}
br := s.Pool.SendBatch(ctx, batch)
defer br.Close()
for i := 0; i < queued; i++ {
if _, err := br.Exec(); err != nil {
return synced, unchanged, skipped, deltas, err
}
}
// Content inserts/updates stamp raw as unprocessed — drop catalog rows so
// processed_products cannot outlive that reset (matches resetRawProducts).
// Touches keep processing_status and must not invalidate catalog.
if len(inserts) > 0 || len(updates) > 0 {
invalidateIDs := make([]uuid.UUID, 0, len(updates))
for _, op := range updates {
invalidateIDs = append(invalidateIDs, op.id)
}
invalidateGTINs := make([]string, 0, len(inserts))
for _, op := range inserts {
invalidateGTINs = append(invalidateGTINs, op.gtin)
}
if _, err := s.Pool.Exec(ctx, `
DELETE FROM processed_products
WHERE company_id = $1
AND (
raw_product_id = ANY($2::uuid[])
OR raw_product_id IN (
SELECT id FROM raw_products
WHERE company_id = $1 AND gtin = ANY($3::text[])
)
)`, companyID, invalidateIDs, invalidateGTINs); err != nil {
return synced, unchanged, skipped, deltas, err
}
}
synced = len(inserts) + len(updates)
unchanged = len(touches)
deltas.Unchanged = unchanged
deltas.Skipped = skipped
return synced, unchanged, skipped, deltas, nil
}
func (s *Service) loadExistingByGTIN(ctx context.Context, companyID uuid.UUID, gtins []string) (map[string]existingProduct, error) {
out := make(map[string]existingProduct, len(gtins))
if len(gtins) == 0 {
return out, nil
}
rows, err := s.Pool.Query(ctx, `
SELECT id, gtin, mapped_data FROM raw_products
WHERE company_id = $1 AND gtin = ANY($2::text[])`, companyID, gtins)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var ex existingProduct
var gtin string
if err := rows.Scan(&ex.ID, &gtin, &ex.MappedData); err != nil {
return nil, err
}
out[gtin] = ex
}
return out, rows.Err()
}
func (s *Service) createSyncJob(ctx context.Context, companyID, feedID uuid.UUID) (uuid.UUID, error) {
var id uuid.UUID
err := s.Pool.QueryRow(ctx, `
INSERT INTO feed_sync_jobs (feed_id, company_id, status)
VALUES ($1, $2, 'pending') RETURNING id`, feedID, companyID).Scan(&id)
return id, err
}
// findOrCreatePendingSyncJob returns an existing pending job for the feed, or inserts one.
// Uses a transaction advisory lock so concurrent enqueues do not stack duplicates.
func (s *Service) findOrCreatePendingSyncJob(ctx context.Context, companyID, feedID uuid.UUID) (uuid.UUID, error) {
tx, err := s.Pool.Begin(ctx)
if err != nil {
return uuid.Nil, err
}
defer tx.Rollback(ctx)
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1::text, 0))`, feedID.String()); err != nil {
return uuid.Nil, err
}
var id uuid.UUID
err = tx.QueryRow(ctx, `
SELECT id FROM feed_sync_jobs
WHERE feed_id = $1 AND company_id = $2 AND status = 'pending'
ORDER BY created_at
LIMIT 1`, feedID, companyID).Scan(&id)
if err == nil {
if err := tx.Commit(ctx); err != nil {
return uuid.Nil, err
}
return id, nil
}
if !errors.Is(err, pgx.ErrNoRows) {
return uuid.Nil, err
}
err = tx.QueryRow(ctx, `
INSERT INTO feed_sync_jobs (feed_id, company_id, status)
VALUES ($1, $2, 'pending') RETURNING id`, feedID, companyID).Scan(&id)
if err != nil {
return uuid.Nil, err
}
if err := tx.Commit(ctx); err != nil {
return uuid.Nil, err
}
return id, nil
}
func (s *Service) markJobRunning(ctx context.Context, jobID uuid.UUID) error {
_, err := s.Pool.Exec(ctx, `
UPDATE feed_sync_jobs SET status = 'running', started_at = now(), updated_at = now()
WHERE id = $1`, jobID)
return err
}
func (s *Service) updateJobProgress(ctx context.Context, jobID uuid.UUID, stats syncStats) error {
_, err := s.Pool.Exec(ctx, `
UPDATE feed_sync_jobs SET
products_synced = $2,
products_total = $3,
products_skipped = $4,
products_unchanged = $5,
progress = $6,
content_hash = NULLIF($7, ''),
updated_at = now()
WHERE id = $1`,
jobID, stats.Synced, stats.Total, stats.Skipped, stats.Unchanged, stats.Progress, stats.ContentHash)
return err
}
func (s *Service) completeJob(ctx context.Context, jobID uuid.UUID, stats syncStats) error {
_, err := s.Pool.Exec(ctx, `
UPDATE feed_sync_jobs SET
status = 'completed',
completed_at = now(),
products_synced = $2,
products_total = $3,
products_skipped = $4,
products_unchanged = $5,
progress = 100,
content_hash = NULLIF($6, ''),
updated_at = now()
WHERE id = $1`,
jobID, stats.Synced, stats.Total, stats.Skipped, stats.Unchanged, stats.ContentHash)
return err
}
func (s *Service) failJob(ctx context.Context, jobID uuid.UUID, msg string, stats syncStats) error {
_, err := s.Pool.Exec(ctx, `
UPDATE feed_sync_jobs SET
status = 'failed',
error = $2,
completed_at = now(),
products_synced = $3,
products_total = $4,
products_skipped = $5,
products_unchanged = $6,
progress = $7,
content_hash = NULLIF($8, ''),
updated_at = now()
WHERE id = $1`,
jobID, truncateErr(msg), stats.Synced, stats.Total, stats.Skipped, stats.Unchanged, stats.Progress, stats.ContentHash)
return err
}
func (s *Service) lastContentHash(ctx context.Context, feedID uuid.UUID) (string, error) {
var hash *string
err := s.Pool.QueryRow(ctx, `
SELECT content_hash FROM feed_sync_jobs
WHERE feed_id = $1 AND status = 'completed' AND content_hash IS NOT NULL AND content_hash <> ''
ORDER BY completed_at DESC NULLS LAST LIMIT 1`, feedID).Scan(&hash)
if errors.Is(err, pgx.ErrNoRows) {
return "", nil
}
if err != nil {
return "", err
}
if hash == nil {
return "", nil
}
return *hash, nil
}
func (s *Service) loadMappingsRaw(ctx context.Context, companyID, feedID uuid.UUID) (any, error) {
var raw []byte
err := s.Pool.QueryRow(ctx, `
SELECT mappings FROM feed_mappings
WHERE feed_id = $1 AND company_id = $2 AND is_active = true
ORDER BY version DESC LIMIT 1`, feedID, companyID).Scan(&raw)
if err != nil {
return nil, err
}
var m any
if err := json.Unmarshal(raw, &m); err != nil {
return nil, err
}
return m, nil
}
func sha256Hex(b []byte) string {
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:])
}
func sha256HexFile(src *feedBlob) (string, error) {
f, err := src.Open()
if err != nil {
return "", err
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
func contentTypeFromBlob(src *feedBlob) string {
if src == nil {
return ""
}
return src.contentType
}
func mustJSON(v any) string {
b, err := json.Marshal(v)
if err != nil {
return ""
}
return string(b)
}
func bytesEqualJSON(a, b []byte) bool {
if len(a) == 0 && len(b) == 0 {
return true
}
var xa, xb any
if json.Unmarshal(a, &xa) != nil || json.Unmarshal(b, &xb) != nil {
return string(a) == string(b)
}
ba, _ := json.Marshal(xa)
bb, _ := json.Marshal(xb)
return string(ba) == string(bb)
}
func truncateErr(msg string) string {
if len(msg) > 2000 {
return msg[:2000]
}
return msg
}
@@ -0,0 +1,112 @@
package feeds
import (
"context"
"errors"
"os"
"strings"
"sync"
"testing"
"time"
"github.com/descrybe/descrybe-v2/apps/api/internal/jobs"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestClaimNextPendingSyncJobConcurrentDistinct(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("DATABASE_URL"))
if dsn == "" {
t.Skip("DATABASE_URL not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
pg, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal(err)
}
// Clean up while the pool is still open (t.Cleanup runs after deferred Close).
defer pg.Close()
companyID := uuid.New()
_, err = pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`,
companyID, "sync-claim-"+companyID.String()[:8])
if err != nil {
t.Fatalf("seed company: %v", err)
}
defer func() {
_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
}()
var feedID uuid.UUID
err = pg.QueryRow(ctx, `
INSERT INTO input_feeds (company_id, name, url, feed_type, status, sync_interval_minutes, options)
VALUES ($1, 'claim-feed', 'https://example.com/feed.csv', 'csv', 'active', 60, '{}'::jsonb)
RETURNING id`, companyID).Scan(&feedID)
if err != nil {
t.Fatalf("seed feed: %v", err)
}
// Live dev workers also ClaimNextPendingSyncJob; seed a buffer so N concurrent
// claimants still succeed when the worker steals a few pending rows.
const n = 4
seedN := n + jobs.MaxSyncWorkers + 4
jobIDs := make([]uuid.UUID, 0, seedN)
for i := 0; i < seedN; i++ {
var id uuid.UUID
err = pg.QueryRow(ctx, `
INSERT INTO feed_sync_jobs (feed_id, company_id, status)
VALUES ($1, $2, 'pending') RETURNING id`, feedID, companyID).Scan(&id)
if err != nil {
t.Fatalf("seed sync job: %v", err)
}
jobIDs = append(jobIDs, id)
}
defer func() {
for _, id := range jobIDs {
_, _ = pg.Exec(context.Background(), `DELETE FROM feed_sync_jobs WHERE id = $1`, id)
}
}()
svc := &Service{Pool: pg}
claimed := make([]uuid.UUID, n)
var wg sync.WaitGroup
wg.Add(n)
for i := 0; i < n; i++ {
go func(i int) {
defer wg.Done()
deadline := time.Now().Add(3 * time.Second)
for {
id, _, _, claimErr := svc.ClaimNextPendingSyncJob(ctx)
if claimErr == nil {
claimed[i] = id
return
}
if !errors.Is(claimErr, pgx.ErrNoRows) {
t.Errorf("ClaimNextPendingSyncJob: %v", claimErr)
return
}
if time.Now().After(deadline) {
t.Errorf("ClaimNextPendingSyncJob: no rows after retries")
return
}
time.Sleep(5 * time.Millisecond)
}
}(i)
}
wg.Wait()
seen := make(map[uuid.UUID]struct{}, n)
for _, id := range claimed {
if id == uuid.Nil {
t.Fatal("nil claim")
}
if _, ok := seen[id]; ok {
t.Fatalf("duplicate claim %s (SKIP LOCKED failed)", id)
}
seen[id] = struct{}{}
}
}
+222
View File
@@ -0,0 +1,222 @@
package feeds
import (
"encoding/json"
"strings"
)
// Reserved mapped_data key holding seller-facing change flags from the latest
// sync that modified the row. Cleared/replaced on the next content update.
const syncChangesMappedKey = "_sync_changes"
// Seller-high-value mapped field groups compared during feed upserts.
var (
syncPriceFields = []string{"price", "sale_price", "purchase_price"}
syncStockFields = []string{"stock", "quantity", "qty"}
syncAvailFields = []string{"availability", "stock_status", "in_stock"}
syncTitleFields = []string{"title", "name", "product_name"}
)
// syncDeltaCounts is the MVP seller summary written to input_feeds.options.last_sync_deltas.
type syncDeltaCounts struct {
JobID string `json:"job_id,omitempty"`
New int `json:"new"`
PriceChanged int `json:"price_changed"`
StockChanged int `json:"stock_changed"`
AvailabilityChanged int `json:"availability_changed"`
TitleChanged int `json:"title_changed"`
OtherChanged int `json:"other_changed"`
Unchanged int `json:"unchanged"`
Skipped int `json:"skipped"`
}
func (d *syncDeltaCounts) addChanges(changes []string) {
if len(changes) == 0 {
d.OtherChanged++
return
}
seen := map[string]struct{}{}
for _, c := range changes {
if _, ok := seen[c]; ok {
continue
}
seen[c] = struct{}{}
switch c {
case "new":
d.New++
case "price":
d.PriceChanged++
case "stock":
d.StockChanged++
case "availability":
d.AvailabilityChanged++
case "title":
d.TitleChanged++
default:
d.OtherChanged++
}
}
}
func (d syncDeltaCounts) asMap() map[string]any {
return map[string]any{
"job_id": d.JobID,
"new": d.New,
"price_changed": d.PriceChanged,
"stock_changed": d.StockChanged,
"availability_changed": d.AvailabilityChanged,
"title_changed": d.TitleChanged,
"other_changed": d.OtherChanged,
"unchanged": d.Unchanged,
"skipped": d.Skipped,
}
}
func mappedScalar(m map[string]any, key string) string {
if m == nil {
return ""
}
v, ok := m[key]
if !ok || v == nil {
return ""
}
switch t := v.(type) {
case string:
return strings.TrimSpace(t)
case float64:
return strings.TrimSpace(strings.TrimRight(strings.TrimRight(
strings.ReplaceAll(jsonNumber(t), "e+0", "e+"), "0"), "."))
case json.Number:
return strings.TrimSpace(t.String())
case bool:
if t {
return "true"
}
return "false"
default:
b, err := json.Marshal(t)
if err != nil {
return ""
}
return strings.TrimSpace(string(b))
}
}
func jsonNumber(f float64) string {
b, err := json.Marshal(f)
if err != nil {
return ""
}
return string(b)
}
func fieldGroupChanged(oldM, newM map[string]any, keys []string) bool {
for _, k := range keys {
if mappedScalar(oldM, k) != mappedScalar(newM, k) {
return true
}
}
return false
}
// detectMappedChanges compares prior mapped_data JSON to the new mapped map.
// Returns change kind tags: price, stock, availability, title, other.
func detectMappedChanges(oldJSON []byte, newMapped map[string]any) []string {
var oldM map[string]any
if len(oldJSON) > 0 {
_ = json.Unmarshal(oldJSON, &oldM)
}
if oldM == nil {
oldM = map[string]any{}
}
cleanNew := stripSyncChanges(newMapped)
var out []string
if fieldGroupChanged(oldM, cleanNew, syncPriceFields) {
out = append(out, "price")
}
if fieldGroupChanged(oldM, cleanNew, syncStockFields) {
out = append(out, "stock")
}
if fieldGroupChanged(oldM, cleanNew, syncAvailFields) {
out = append(out, "availability")
}
if fieldGroupChanged(oldM, cleanNew, syncTitleFields) {
out = append(out, "title")
}
// Any other mapped key change (excluding reserved meta).
if len(out) == 0 && !mappedEqualIgnoringSyncMeta(oldM, cleanNew) {
out = append(out, "other")
}
return out
}
func stripSyncChanges(m map[string]any) map[string]any {
if m == nil {
return map[string]any{}
}
out := make(map[string]any, len(m))
for k, v := range m {
if k == syncChangesMappedKey {
continue
}
out[k] = v
}
return out
}
// preserveSyncedExtras keeps fields that feed mappings never set (notably
// category unique_id from legacy assignment / seed backfill) when a sync
// remaps the row. Without this, upserts replace mapped_data wholesale and
// wipe category even though the feed has no category column.
func preserveSyncedExtras(existingJSON []byte, mapped map[string]any) map[string]any {
out := stripSyncChanges(mapped)
if len(existingJSON) == 0 {
return out
}
var old map[string]any
if err := json.Unmarshal(existingJSON, &old); err != nil || old == nil {
return out
}
old = stripSyncChanges(old)
if mappedScalar(out, "category") == "" {
if cat := mappedScalar(old, "category"); cat != "" && !strings.EqualFold(cat, "none") {
out["category"] = cat
}
}
return out
}
func mappedEqualIgnoringSyncMeta(a, b map[string]any) bool {
aa := stripSyncChanges(a)
bb := stripSyncChanges(b)
ab, err1 := json.Marshal(aa)
bb2, err2 := json.Marshal(bb)
if err1 != nil || err2 != nil {
return false
}
return bytesEqualJSON(ab, bb2)
}
func withSyncChanges(mapped map[string]any, changes []string) map[string]any {
out := stripSyncChanges(mapped)
if len(changes) == 0 {
return out
}
tags := make([]any, 0, len(changes))
seen := map[string]struct{}{}
for _, c := range changes {
c = strings.TrimSpace(strings.ToLower(c))
if c == "" {
continue
}
if _, ok := seen[c]; ok {
continue
}
seen[c] = struct{}{}
tags = append(tags, c)
}
if len(tags) > 0 {
out[syncChangesMappedKey] = tags
}
return out
}
+125
View File
@@ -0,0 +1,125 @@
package feeds
import (
"encoding/json"
"testing"
"github.com/google/uuid"
)
func TestDetectMappedChangesPriceStock(t *testing.T) {
t.Parallel()
old := []byte(`{"price":"10","stock":"5","title":"A"}`)
changes := detectMappedChanges(old, map[string]any{"price": "12", "stock": "5", "title": "A"})
if len(changes) != 1 || changes[0] != "price" {
t.Fatalf("changes=%v", changes)
}
changes = detectMappedChanges(old, map[string]any{"price": "10", "stock": "0", "title": "A"})
if len(changes) != 1 || changes[0] != "stock" {
t.Fatalf("stock changes=%v", changes)
}
changes = detectMappedChanges(old, map[string]any{"price": "11", "stock": "1", "title": "B", "availability": "out"})
want := map[string]bool{"price": true, "stock": true, "title": true, "availability": true}
for _, c := range changes {
if !want[c] {
t.Fatalf("unexpected %s in %v", c, changes)
}
delete(want, c)
}
if len(want) != 0 {
t.Fatalf("missing %v from %v", want, changes)
}
}
func TestClassifyUpsertOpsStampsSyncChanges(t *testing.T) {
t.Parallel()
idUpdate := uuid.MustParse("22222222-2222-2222-2222-222222222222")
existing := map[string]existingProduct{
"update": {ID: idUpdate, MappedData: []byte(`{"price":"10","title":"Old"}`)},
}
chunk := []pendingProduct{
{GTIN: "new", RawData: map[string]string{"EAN": "new"}, MappedData: map[string]any{"title": "N"}},
{GTIN: "update", RawData: map[string]string{"EAN": "update"}, MappedData: map[string]any{"price": "12", "title": "Old"}},
}
ops, skipped := classifyUpsertOps(chunk, existing)
if skipped != 0 || len(ops) != 2 {
t.Fatalf("ops=%d skipped=%d", len(ops), skipped)
}
if ops[0].kind != upsertOpInsert || len(ops[0].changes) != 1 || ops[0].changes[0] != "new" {
t.Fatalf("insert op=%+v", ops[0])
}
var mapped map[string]any
if err := json.Unmarshal(ops[0].mappedJSON, &mapped); err != nil {
t.Fatal(err)
}
tags, _ := mapped[syncChangesMappedKey].([]any)
if len(tags) != 1 || tags[0] != "new" {
t.Fatalf("insert mapped meta=%v", mapped[syncChangesMappedKey])
}
if ops[1].kind != upsertOpUpdate || len(ops[1].changes) != 1 || ops[1].changes[0] != "price" {
t.Fatalf("update op=%+v", ops[1])
}
}
func TestSyncDeltaCountsAddChanges(t *testing.T) {
t.Parallel()
var d syncDeltaCounts
d.addChanges([]string{"price", "stock"})
d.addChanges([]string{"new"})
d.addChanges(nil)
if d.PriceChanged != 1 || d.StockChanged != 1 || d.New != 1 || d.OtherChanged != 1 {
t.Fatalf("%+v", d)
}
}
func TestPreserveSyncedExtrasKeepsCategory(t *testing.T) {
t.Parallel()
existing := []byte(`{"price":"10","category":"46","title":"Roborock"}`)
mapped := map[string]any{"price": "12", "title": "Roborock"}
got := preserveSyncedExtras(existing, mapped)
if got["category"] != "46" {
t.Fatalf("category=%v want 46", got["category"])
}
if got["price"] != "12" {
t.Fatalf("price=%v want 12", got["price"])
}
// Explicit new category wins.
mapped2 := map[string]any{"price": "12", "category": "120"}
got2 := preserveSyncedExtras(existing, mapped2)
if got2["category"] != "120" {
t.Fatalf("category=%v want 120", got2["category"])
}
// Empty / none prior must not invent a category.
got3 := preserveSyncedExtras([]byte(`{"category":"none"}`), map[string]any{"title": "X"})
if _, ok := got3["category"]; ok {
t.Fatalf("unexpected category=%v", got3["category"])
}
}
func TestClassifyUpsertOpsPreservesCategory(t *testing.T) {
t.Parallel()
idUpdate := uuid.MustParse("33333333-3333-3333-3333-333333333333")
existing := map[string]existingProduct{
"keep": {ID: idUpdate, MappedData: []byte(`{"price":"10","category":"46","title":"Old"}`)},
}
chunk := []pendingProduct{
{GTIN: "keep", RawData: map[string]string{"EAN": "keep"}, MappedData: map[string]any{"price": "12", "title": "Old"}},
}
ops, skipped := classifyUpsertOps(chunk, existing)
if skipped != 0 || len(ops) != 1 {
t.Fatalf("ops=%d skipped=%d", len(ops), skipped)
}
if ops[0].kind != upsertOpUpdate {
t.Fatalf("kind=%v want update", ops[0].kind)
}
var mapped map[string]any
if err := json.Unmarshal(ops[0].mappedJSON, &mapped); err != nil {
t.Fatal(err)
}
if mapped["category"] != "46" {
t.Fatalf("category wiped: %v", mapped["category"])
}
if mapped["price"] != "12" {
t.Fatalf("price=%v", mapped["price"])
}
}
@@ -0,0 +1,94 @@
package feeds
import (
"context"
"os"
"strings"
"sync"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestFindOrCreatePendingSyncJobDedupesSameFeed(t *testing.T) {
dsn := strings.TrimSpace(os.Getenv("DATABASE_URL"))
if dsn == "" {
t.Skip("DATABASE_URL not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
pg, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal(err)
}
defer pg.Close()
companyID := uuid.New()
_, err = pg.Exec(ctx, `INSERT INTO companies (id, name) VALUES ($1, $2)`,
companyID, "sync-dedupe-"+companyID.String()[:8])
if err != nil {
t.Fatalf("seed company: %v", err)
}
t.Cleanup(func() {
_, _ = pg.Exec(context.Background(), `DELETE FROM companies WHERE id = $1`, companyID)
})
var feedID uuid.UUID
err = pg.QueryRow(ctx, `
INSERT INTO input_feeds (company_id, name, url, feed_type, status, sync_interval_minutes, options)
VALUES ($1, 'dedupe-feed', 'https://example.com/feed.csv', 'csv', 'active', 60, '{}'::jsonb)
RETURNING id`, companyID).Scan(&feedID)
if err != nil {
t.Fatalf("seed feed: %v", err)
}
svc := &Service{Pool: pg}
first, err := svc.findOrCreatePendingSyncJob(ctx, companyID, feedID)
if err != nil {
t.Fatalf("first: %v", err)
}
second, err := svc.findOrCreatePendingSyncJob(ctx, companyID, feedID)
if err != nil {
t.Fatalf("second: %v", err)
}
if first != second {
t.Fatalf("dedupe failed: first=%s second=%s", first, second)
}
var pendingCount int
if err := pg.QueryRow(ctx, `
SELECT COUNT(*) FROM feed_sync_jobs
WHERE feed_id = $1 AND company_id = $2 AND status = 'pending'`,
feedID, companyID).Scan(&pendingCount); err != nil {
t.Fatal(err)
}
if pendingCount != 1 {
t.Fatalf("pending count=%d want 1", pendingCount)
}
const n = 8
ids := make([]uuid.UUID, n)
var wg sync.WaitGroup
wg.Add(n)
for i := 0; i < n; i++ {
go func(i int) {
defer wg.Done()
id, err := svc.findOrCreatePendingSyncJob(ctx, companyID, feedID)
if err != nil {
t.Errorf("concurrent findOrCreate: %v", err)
return
}
ids[i] = id
}(i)
}
wg.Wait()
for _, id := range ids {
if id != first {
t.Fatalf("concurrent id=%s want %s", id, first)
}
}
}
@@ -0,0 +1,390 @@
package feeds
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"net/url"
"os"
"strings"
"testing"
"github.com/google/uuid"
)
func TestParseCSVAndMappings(t *testing.T) {
csv := "EAN,Title\n123,Widget\n456,\n"
mappings := parseMappings([]any{
map[string]any{"source": "EAN", "target": "gtin"},
map[string]any{"column": "Title", "fieldName": "title"},
})
var rows []map[string]any
n, err := parseCSV(strings.NewReader(csv), func(row feedRow) error {
mapped, gtin := applyMappings(row, mappings)
rows = append(rows, map[string]any{"gtin": gtin, "mapped": mapped})
return nil
})
if err != nil {
t.Fatal(err)
}
if n != 2 {
t.Fatalf("rows=%d", n)
}
if rows[0]["gtin"] != "123" {
t.Fatalf("gtin=%v", rows[0]["gtin"])
}
m := rows[0]["mapped"].(map[string]any)
if m["title"] != "Widget" {
t.Fatalf("mapped=%v", m)
}
}
func TestParseXMLItems(t *testing.T) {
xmlBody := `<?xml version="1.0"?><products><item><gtin>999</gtin><title>X</title></item></products>`
mappings := parseMappings(map[string]any{
"gtin": map[string]any{"fieldName": "gtin"},
"title": map[string]any{"fieldName": "title"},
})
var got string
n, err := parseXMLItems(strings.NewReader(xmlBody), "item", func(row feedRow) error {
_, gtin := applyMappings(row, mappings)
got = gtin
return nil
})
if err != nil {
t.Fatal(err)
}
if n != 1 || got != "999" {
t.Fatalf("n=%d gtin=%q", n, got)
}
}
func TestDownloadRejectsPrivateAndFTP(t *testing.T) {
ctx := context.Background()
if _, err := downloadFeed(ctx, "ftp://example.com/a.csv"); err == nil {
t.Fatal("expected ftp error")
}
if _, err := downloadFeed(ctx, "http://127.0.0.1/x"); err == nil {
t.Fatal("expected private IP error")
}
if _, err := downloadFeed(ctx, "http://localhost/x"); err == nil {
t.Fatal("expected localhost error")
}
}
func TestSSRFTransportDisablesEnvProxy(t *testing.T) {
tr := ssrfTransport()
if tr.Proxy != nil {
t.Fatal("feed SSRF transport must not use ProxyFromEnvironment")
}
}
func TestValidateFeedURLRejectsPrivateAndFTP(t *testing.T) {
ctx := context.Background()
if err := ValidateFeedURL(ctx, ""); err != nil {
t.Fatalf("empty url should be ok: %v", err)
}
if err := ValidateFeedURL(ctx, "ftp://example.com/a.csv"); err == nil {
t.Fatal("expected ftp error")
}
if err := ValidateFeedURL(ctx, "http://127.0.0.1/x"); err == nil {
t.Fatal("expected private IP error")
}
if err := ValidateFeedURL(ctx, "http://169.254.169.254/latest"); err == nil {
t.Fatal("expected metadata IP error")
}
if err := ValidateFeedURL(ctx, "https://8.8.8.8/feed.xml"); err != nil {
t.Fatalf("public IP https should be ok: %v", err)
}
}
func TestDownloadPublicOKWithSizeCap(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/csv")
_, _ = w.Write([]byte("EAN,Title\n1,A\n"))
}))
t.Cleanup(srv.Close)
// httptest uses 127.0.0.1 — should be blocked by SSRF guard.
_, err := downloadFeed(context.Background(), srv.URL)
if err == nil || !strings.Contains(err.Error(), "private") && err != errURLPrivate {
// allow either wrapped or direct
if err == nil {
t.Fatal("expected loopback blocked")
}
}
}
func TestDownloadAllowlistPrivateHost(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/csv")
_, _ = w.Write([]byte("EAN,Title\n1,A\n"))
}))
t.Cleanup(srv.Close)
u, err := url.Parse(srv.URL)
if err != nil {
t.Fatal(err)
}
if err := ConfigurePrivateAllowlist([]string{u.Hostname()}, []string{"127.0.0.0/8"}); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = ConfigurePrivateAllowlist(nil, nil)
})
blob, err := downloadFeed(context.Background(), srv.URL)
if err != nil {
t.Fatalf("allowlisted download: %v", err)
}
t.Cleanup(func() { _ = blob.Close() })
data, err := os.ReadFile(blob.path)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(data), "EAN") {
t.Fatalf("body=%q ct=%s", data, blob.contentType)
}
}
func TestDownloadStreamsToTempFile(t *testing.T) {
payload := "EAN,Title\n1,A\n2,B\n"
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/csv")
_, _ = w.Write([]byte(payload))
}))
t.Cleanup(srv.Close)
u, err := url.Parse(srv.URL)
if err != nil {
t.Fatal(err)
}
if err := ConfigurePrivateAllowlist([]string{u.Hostname()}, []string{"127.0.0.0/8"}); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = ConfigurePrivateAllowlist(nil, nil) })
blob, err := downloadFeed(context.Background(), srv.URL)
if err != nil {
t.Fatal(err)
}
if !blob.owned || blob.path == "" {
t.Fatalf("expected owned temp path, got %+v", blob)
}
if _, err := os.Stat(blob.path); err != nil {
t.Fatalf("temp missing: %v", err)
}
hash, err := sha256HexFile(blob)
if err != nil {
t.Fatal(err)
}
wantHash := sha256Hex([]byte(payload))
if hash != wantHash {
t.Fatalf("hash=%s want=%s", hash, wantHash)
}
f, err := blob.Open()
if err != nil {
t.Fatal(err)
}
var rows int
n, err := parseCSV(f, func(feedRow) error {
rows++
return nil
})
_ = f.Close()
if err != nil {
t.Fatal(err)
}
if n != 2 || rows != 2 {
t.Fatalf("n=%d rows=%d", n, rows)
}
path := blob.path
if err := blob.Close(); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("temp should be removed after Close, err=%v", err)
}
}
func TestDetectFeedFormat(t *testing.T) {
if detectFeedFormat("csv", "", "", nil) != "csv" {
t.Fatal("csv")
}
if detectFeedFormat("", "application/xml", "", []byte("<a/>")) != "xml" {
t.Fatal("xml")
}
}
func TestClassifyUpsertOpsInsertTouchUpdate(t *testing.T) {
idTouch := uuid.MustParse("11111111-1111-1111-1111-111111111111")
idUpdate := uuid.MustParse("22222222-2222-2222-2222-222222222222")
existing := map[string]existingProduct{
"touch": {ID: idTouch, MappedData: []byte(`{"title":"Same"}`)},
"update": {ID: idUpdate, MappedData: []byte(`{"title":"Old"}`)},
}
chunk := []pendingProduct{
{GTIN: "new", RawData: map[string]string{"EAN": "new"}, MappedData: map[string]any{"title": "N"}},
{GTIN: "touch", RawData: map[string]string{"EAN": "touch"}, MappedData: map[string]any{"title": "Same"}},
{GTIN: "update", RawData: map[string]string{"EAN": "update"}, MappedData: map[string]any{"title": "New"}},
}
ops, skipped := classifyUpsertOps(chunk, existing)
if skipped != 0 {
t.Fatalf("skipped=%d", skipped)
}
if len(ops) != 3 {
t.Fatalf("ops=%d", len(ops))
}
if ops[0].kind != upsertOpInsert || ops[0].gtin != "new" {
t.Fatalf("op0=%+v", ops[0])
}
if ops[1].kind != upsertOpTouch || ops[1].id != idTouch {
t.Fatalf("op1=%+v", ops[1])
}
if ops[2].kind != upsertOpUpdate || ops[2].id != idUpdate {
t.Fatalf("op2=%+v", ops[2])
}
inserts, touches, updates := partitionUpsertOps(ops)
if len(inserts) != 1 || len(touches) != 1 || len(updates) != 1 {
t.Fatalf("partition inserts=%d touches=%d updates=%d", len(inserts), len(touches), len(updates))
}
if inserts[0].gtin != "new" || touches[0].id != idTouch || updates[0].id != idUpdate {
t.Fatalf("partition payloads insert=%+v touch=%+v update=%+v", inserts[0], touches[0], updates[0])
}
}
func TestDedupePendingByGTINLastWins(t *testing.T) {
chunk := []pendingProduct{
{GTIN: "a", MappedData: map[string]any{"title": "first"}},
{GTIN: "b", MappedData: map[string]any{"title": "only"}},
{GTIN: "a", MappedData: map[string]any{"title": "last"}},
}
got := dedupePendingByGTIN(chunk)
if len(got) != 2 {
t.Fatalf("len=%d", len(got))
}
if got[0].GTIN != "a" || got[0].MappedData["title"] != "last" {
t.Fatalf("got[0]=%+v", got[0])
}
if got[1].GTIN != "b" {
t.Fatalf("got[1]=%+v", got[1])
}
if dedupePendingByGTIN(nil) != nil {
t.Fatal("nil in")
}
single := []pendingProduct{{GTIN: "x"}}
if out := dedupePendingByGTIN(single); len(out) != 1 || out[0].GTIN != "x" {
t.Fatalf("single=%+v", out)
}
}
func TestUpsertSQLIsSetBased(t *testing.T) {
for name, sql := range map[string]string{
"insert": upsertSQLInsertSet,
"touch": upsertSQLTouchSet,
"update": upsertSQLUpdateSet,
} {
lower := strings.ToLower(sql)
switch name {
case "insert", "update":
if !strings.Contains(lower, "unnest(") {
t.Fatalf("%s missing unnest: %s", name, sql)
}
case "touch":
if !strings.Contains(lower, "any(") {
t.Fatalf("touch missing ANY: %s", sql)
}
}
if strings.Contains(lower, "values ($1") {
t.Fatalf("%s still per-row VALUES form", name)
}
}
}
func TestParseCSVRespectsMaxRows(t *testing.T) {
old := maxParseRows
maxParseRows = 2
t.Cleanup(func() { maxParseRows = old })
body := "EAN\n1\n2\n3\n"
n, err := parseCSV(strings.NewReader(body), func(feedRow) error { return nil })
if err == nil || !errors.Is(err, errParseTooManyRows) {
t.Fatalf("n=%d err=%v want errParseTooManyRows", n, err)
}
if !strings.Contains(err.Error(), "(2)") {
t.Fatalf("err=%v want limit in message", err)
}
if msg, ok := ClientError(err); !ok || !strings.Contains(msg, "max row limit") {
t.Fatalf("ClientError=%q ok=%v", msg, ok)
}
if n != 3 {
t.Fatalf("count=%d want 3 (exceeded after 3rd)", n)
}
}
func TestParseXMLRespectsMaxRows(t *testing.T) {
old := maxParseRows
maxParseRows = 1
t.Cleanup(func() { maxParseRows = old })
body := `<r><item><g>1</g></item><item><g>2</g></item></r>`
n, err := parseXMLItems(strings.NewReader(body), "item", func(feedRow) error { return nil })
if err == nil || !errors.Is(err, errParseTooManyRows) {
t.Fatalf("n=%d err=%v want errParseTooManyRows", n, err)
}
if n != 2 {
t.Fatalf("count=%d want 2", n)
}
}
func TestDownloadRejectsOversizedBody(t *testing.T) {
old := maxDownloadBytes
maxDownloadBytes = 64
t.Cleanup(func() { maxDownloadBytes = old })
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/csv")
_, _ = w.Write([]byte(strings.Repeat("x", int(maxDownloadBytes)+2)))
}))
t.Cleanup(srv.Close)
u, err := url.Parse(srv.URL)
if err != nil {
t.Fatal(err)
}
if err := ConfigurePrivateAllowlist([]string{u.Hostname()}, []string{"127.0.0.0/8"}); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = ConfigurePrivateAllowlist(nil, nil) })
_, err = downloadFeed(context.Background(), srv.URL)
if !errors.Is(err, errDownloadTooLarge) {
t.Fatalf("err=%v want errDownloadTooLarge", err)
}
// Tiny test caps report bytes; production (≥1 MiB) reports MiB.
if !strings.Contains(err.Error(), "max 64 bytes") {
t.Fatalf("err=%v want max bytes in message", err)
}
if msg, ok := ClientError(err); !ok || !strings.Contains(msg, "size limit") {
t.Fatalf("ClientError=%q ok=%v", msg, ok)
}
}
func TestDefaultFeedCaps(t *testing.T) {
const wantDownloadBytes int64 = 256 << 20 // 256 MiB — within 200500 MiB band
const wantParseRows = 1_000_000
if defaultMaxDownloadBytes != wantDownloadBytes {
t.Fatalf("defaultMaxDownloadBytes=%d want %d", defaultMaxDownloadBytes, wantDownloadBytes)
}
if defaultMaxParseRows != wantParseRows {
t.Fatalf("defaultMaxParseRows=%d want %d", defaultMaxParseRows, wantParseRows)
}
if defaultMaxDownloadBytes < 200<<20 || defaultMaxDownloadBytes > 500<<20 {
t.Fatalf("defaultMaxDownloadBytes=%d outside 200500 MiB guidance", defaultMaxDownloadBytes)
}
}
@@ -0,0 +1,112 @@
package feeds
import (
"context"
"errors"
"fmt"
"strings"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
// requiredStandardField is an enabled, required company standard field used for sync preflight.
type requiredStandardField struct {
Key string
Name string
}
// activeMappings returns rows with a non-empty source and a real target (not "none").
// Mirrors the frontend activeMappingRows filter.
func activeMappings(mappings []FieldMapping) []FieldMapping {
out := make([]FieldMapping, 0, len(mappings))
for _, m := range mappings {
src := m.sourceKey()
tgt := m.targetKey()
if src == "" || tgt == "" || strings.EqualFold(tgt, "none") {
continue
}
out = append(out, m)
}
return out
}
// validateMappingsForSync returns a ClientMsg when mappings are empty or required targets are missing.
// Optional (non-required) standard fields may remain unmapped.
func validateMappingsForSync(mappings []FieldMapping, required []requiredStandardField) error {
active := activeMappings(mappings)
if len(active) == 0 {
return ClientMsg("Map at least one source field before syncing.")
}
if len(required) == 0 {
return nil
}
mapped := make(map[string]struct{}, len(active))
for _, m := range active {
mapped[m.targetKey()] = struct{}{}
}
missing := make([]string, 0)
for _, r := range required {
if _, ok := mapped[r.Key]; ok {
continue
}
label := strings.TrimSpace(r.Name)
if label == "" {
label = r.Key
}
missing = append(missing, label)
}
if len(missing) == 0 {
return nil
}
return ClientMsg(fmt.Sprintf("Map required fields before syncing: %s.", strings.Join(missing, ", ")))
}
// loadRequiredStandardFields returns enabled+required standard field keys for the company.
func (s *Service) loadRequiredStandardFields(ctx context.Context, companyID uuid.UUID) ([]requiredStandardField, error) {
if s == nil || s.Pool == nil {
return nil, nil
}
rows, err := s.Pool.Query(ctx, `
SELECT key, COALESCE(NULLIF(TRIM(name), ''), key) AS name
FROM standard_fields
WHERE company_id = $1 AND enabled = true AND is_required = true
ORDER BY sort_order ASC, key ASC`, companyID)
if err != nil {
return nil, err
}
defer rows.Close()
out := make([]requiredStandardField, 0)
for rows.Next() {
var key, name string
if err := rows.Scan(&key, &name); err != nil {
return nil, err
}
key = strings.TrimSpace(key)
if key == "" {
continue
}
out = append(out, requiredStandardField{Key: key, Name: strings.TrimSpace(name)})
}
return out, rows.Err()
}
// ensureMappingsReadyForSync loads active mappings and required standard fields, then validates.
// Called before creating a sync job so clients get a clear 400 without a failed job row.
func (s *Service) ensureMappingsReadyForSync(ctx context.Context, companyID, feedID uuid.UUID) error {
raw, err := s.loadMappingsRaw(ctx, companyID, feedID)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return ClientMsg("Map at least one source field before syncing.")
}
return err
}
required, err := s.loadRequiredStandardFields(ctx, companyID)
if err != nil {
return err
}
return validateMappingsForSync(parseMappings(raw), required)
}
@@ -0,0 +1,130 @@
package feeds
import (
"strings"
"testing"
)
func TestValidateMappingsForSyncEmpty(t *testing.T) {
err := validateMappingsForSync(nil, nil)
if err == nil {
t.Fatal("expected error for empty mappings")
}
msg, ok := ClientError(err)
if !ok || !strings.Contains(msg, "Map at least one source field") {
t.Fatalf("got %v", err)
}
err = validateMappingsForSync([]FieldMapping{
{Source: "col", Target: "none"},
{Source: "", Target: "gtin"},
}, nil)
if err == nil {
t.Fatal("expected error when only inactive rows present")
}
}
func TestValidateMappingsForSyncOptionalUnmappedOK(t *testing.T) {
err := validateMappingsForSync([]FieldMapping{
{Source: "ean", Target: "gtin"},
{Source: "name", Target: "title"},
}, []requiredStandardField{
{Key: "gtin", Name: "GTIN/EAN"},
{Key: "title", Name: "Product name"},
})
if err != nil {
t.Fatalf("optional gaps should be allowed: %v", err)
}
}
func TestValidateMappingsForSyncMissingRequired(t *testing.T) {
err := validateMappingsForSync([]FieldMapping{
{Source: "ean", Target: "gtin"},
}, []requiredStandardField{
{Key: "gtin", Name: "GTIN/EAN"},
{Key: "title", Name: "Product name"},
{Key: "brand", Name: "Brand"},
})
if err == nil {
t.Fatal("expected missing required error")
}
msg, ok := ClientError(err)
if !ok {
t.Fatalf("expected ClientMsg, got %v", err)
}
if !strings.Contains(msg, "Map required fields before syncing") {
t.Fatalf("message=%q", msg)
}
if !strings.Contains(msg, "Product name") || !strings.Contains(msg, "Brand") {
t.Fatalf("expected missing labels in %q", msg)
}
if strings.Contains(msg, "GTIN") {
t.Fatalf("mapped required field should not appear: %q", msg)
}
}
func TestValidateMappingsForSyncNoRequiredConfigured(t *testing.T) {
err := validateMappingsForSync([]FieldMapping{
{Source: "sku", Target: "sku"},
}, nil)
if err != nil {
t.Fatalf("empty required list should pass when mappings exist: %v", err)
}
}
func TestMappingDocIncomplete(t *testing.T) {
required := []requiredStandardField{{Key: "gtin", Name: "GTIN"}}
xmlItem := map[string]any{"feed_type": "xml", "options": map[string]any{}}
csvItem := map[string]any{"feed_type": "csv"}
empty := []FieldMapping{}
if !mappingDocIncomplete(xmlItem, map[string]any{"fields": empty}, empty, required) {
t.Fatal("empty mappings should be incomplete")
}
withGTIN := []FieldMapping{{Source: "id", Target: "gtin"}}
rawNoPath := map[string]any{"fields": []any{map[string]any{"source": "id", "target": "gtin"}}}
if !mappingDocIncomplete(xmlItem, rawNoPath, withGTIN, required) {
t.Fatal("xml without item_path should be incomplete")
}
if mappingDocIncomplete(csvItem, rawNoPath, withGTIN, required) {
t.Fatal("csv without item_path should be complete when required mapped")
}
rawWithPath := map[string]any{
"item_path": "rss/channel/item",
"fields": []any{map[string]any{"source": "id", "target": "gtin"}},
}
if mappingDocIncomplete(xmlItem, rawWithPath, withGTIN, required) {
t.Fatal("xml with item_path + required should be complete")
}
}
func TestActiveMappingsFiltersNone(t *testing.T) {
got := activeMappings([]FieldMapping{
{Source: "a", Target: "gtin"},
{Source: "b", Target: "none"},
{Source: "c", FieldName: "None"},
{Column: "d", Field: "title"},
})
if len(got) != 2 {
t.Fatalf("got %d want 2: %#v", len(got), got)
}
if got[0].targetKey() != "gtin" || got[1].targetKey() != "title" {
t.Fatalf("unexpected: %#v", got)
}
}
func TestApplyMappingsSkipsNoneTarget(t *testing.T) {
row := map[string]string{"col": "value", "ean": "123"}
mapped, gtin := applyMappings(row, []FieldMapping{
{Source: "col", Target: "none"},
{Source: "ean", Target: "gtin"},
})
if _, ok := mapped["none"]; ok {
t.Fatalf("none target must not be applied: %#v", mapped)
}
if gtin != "123" || mapped["gtin"] != "123" {
t.Fatalf("expected gtin mapping, got mapped=%#v gtin=%q", mapped, gtin)
}
}