Initial commit of Descrybe v2 without local scratch artifacts.
Drop one-shot tmp/axe scripts and agent i18n scratch so the Gitea tree is deployable.
This commit is contained in:
@@ -0,0 +1,233 @@
|
||||
package feeds
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// defaultMaxParseRows caps CSV/XML product rows per sync. Parse is streaming
|
||||
// (one row at a time); the bound limits sync duration and DB write volume for
|
||||
// oversized catalogs. Exceeding returns parseTooManyRows() with the numeric limit.
|
||||
const defaultMaxParseRows = 1_000_000
|
||||
|
||||
// maxParseRows is the active row cap. Mutable for tests.
|
||||
var maxParseRows = defaultMaxParseRows
|
||||
|
||||
// errParseTooManyRows is the sentinel for ClientError / errors.Is checks.
|
||||
var errParseTooManyRows = errors.New("feed exceeds max row limit")
|
||||
|
||||
// parseTooManyRows returns errParseTooManyRows with the active row limit for clients.
|
||||
func parseTooManyRows() error {
|
||||
return fmt.Errorf("%w (%d)", errParseTooManyRows, maxParseRows)
|
||||
}
|
||||
|
||||
// feedRow is a normalized flat record from CSV or XML.
|
||||
type feedRow map[string]string
|
||||
|
||||
func detectFeedFormat(feedType, contentType, urlHint string, sample []byte) string {
|
||||
ft := strings.ToLower(strings.TrimSpace(feedType))
|
||||
if ft == "csv" || ft == "xml" {
|
||||
return ft
|
||||
}
|
||||
ct := strings.ToLower(contentType)
|
||||
u := strings.ToLower(urlHint)
|
||||
switch {
|
||||
case strings.Contains(ct, "csv") || strings.HasSuffix(u, ".csv"):
|
||||
return "csv"
|
||||
case strings.Contains(ct, "xml") || strings.HasSuffix(u, ".xml"):
|
||||
return "xml"
|
||||
}
|
||||
trimmed := bytes.TrimSpace(sample)
|
||||
if len(trimmed) > 0 && trimmed[0] == '<' {
|
||||
return "xml"
|
||||
}
|
||||
return "csv"
|
||||
}
|
||||
|
||||
// parseCSV streams rows via callback to avoid holding the full matrix when possible.
|
||||
// The CSV reader still tokenizes; we only keep one row at a time in the callback path.
|
||||
func parseCSV(r io.Reader, onRow func(feedRow) error) (int, error) {
|
||||
cr := csv.NewReader(r)
|
||||
cr.ReuseRecord = true
|
||||
cr.LazyQuotes = true
|
||||
cr.TrimLeadingSpace = true
|
||||
cr.FieldsPerRecord = -1
|
||||
|
||||
header, err := cr.Read()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("csv header: %w", err)
|
||||
}
|
||||
cols := make([]string, len(header))
|
||||
for i, h := range header {
|
||||
cols[i] = strings.TrimSpace(h)
|
||||
}
|
||||
count := 0
|
||||
for {
|
||||
rec, err := cr.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return count, fmt.Errorf("csv row %d: %w", count+1, err)
|
||||
}
|
||||
count++
|
||||
if count > maxParseRows {
|
||||
return count, parseTooManyRows()
|
||||
}
|
||||
row := make(feedRow, len(cols))
|
||||
for i, col := range cols {
|
||||
if col == "" {
|
||||
continue
|
||||
}
|
||||
if i < len(rec) {
|
||||
row[col] = rec[i]
|
||||
} else {
|
||||
row[col] = ""
|
||||
}
|
||||
}
|
||||
expandSpecificationFields(row)
|
||||
if err := onRow(row); err != nil {
|
||||
return count, err
|
||||
}
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// parseXMLItems streams element-local text maps for repeating item tags.
|
||||
// When itemLocal is empty and r is seekable, a small prefix is sniffed then rewound.
|
||||
func parseXMLItems(r io.Reader, itemLocal string, onRow func(feedRow) error) (int, error) {
|
||||
itemLocal = strings.TrimSpace(itemLocal)
|
||||
if itemLocal == "" {
|
||||
var err error
|
||||
itemLocal, r, err = resolveXMLItemLocal(r)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
dec := xml.NewDecoder(r)
|
||||
dec.Strict = false
|
||||
count := 0
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return count, fmt.Errorf("xml: %w", err)
|
||||
}
|
||||
se, ok := tok.(xml.StartElement)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if !localNameEquals(se.Name, itemLocal) {
|
||||
continue
|
||||
}
|
||||
row, err := readXMLElementMap(dec, se)
|
||||
if err != nil {
|
||||
return count, err
|
||||
}
|
||||
expandSpecificationFields(row)
|
||||
count++
|
||||
if count > maxParseRows {
|
||||
return count, parseTooManyRows()
|
||||
}
|
||||
if err := onRow(row); err != nil {
|
||||
return count, err
|
||||
}
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
const xmlItemGuessBytes = 64 << 10
|
||||
|
||||
func resolveXMLItemLocal(r io.Reader) (string, io.Reader, error) {
|
||||
if rs, ok := r.(io.ReadSeeker); ok {
|
||||
sample := make([]byte, xmlItemGuessBytes)
|
||||
n, err := io.ReadFull(rs, sample)
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
return "", nil, err
|
||||
}
|
||||
sample = sample[:n]
|
||||
local := guessXMLItemLocal(sample)
|
||||
if _, err := rs.Seek(0, io.SeekStart); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return local, rs, nil
|
||||
}
|
||||
sample, err := io.ReadAll(io.LimitReader(r, xmlItemGuessBytes))
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
local := guessXMLItemLocal(sample)
|
||||
return local, io.MultiReader(bytes.NewReader(sample), r), nil
|
||||
}
|
||||
|
||||
func guessXMLItemLocal(data []byte) string {
|
||||
sample := string(data)
|
||||
if len(sample) > xmlItemGuessBytes {
|
||||
sample = sample[:xmlItemGuessBytes]
|
||||
}
|
||||
lower := strings.ToLower(sample)
|
||||
for _, cand := range []string{"item", "product", "entry", "offer", "row"} {
|
||||
if strings.Contains(lower, "<"+cand) || strings.Contains(lower, ":"+cand) {
|
||||
return cand
|
||||
}
|
||||
}
|
||||
return "item"
|
||||
}
|
||||
|
||||
func localNameEquals(n xml.Name, local string) bool {
|
||||
return strings.EqualFold(n.Local, local)
|
||||
}
|
||||
|
||||
func readXMLElementMap(dec *xml.Decoder, start xml.StartElement) (feedRow, error) {
|
||||
row := make(feedRow)
|
||||
for _, a := range start.Attr {
|
||||
key := "@" + a.Name.Local
|
||||
row[key] = a.Value
|
||||
row[start.Name.Local+"/"+key] = a.Value
|
||||
}
|
||||
var path []string
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch t := tok.(type) {
|
||||
case xml.StartElement:
|
||||
path = append(path, t.Name.Local)
|
||||
for _, a := range t.Attr {
|
||||
key := strings.Join(path, "/") + "/@" + a.Name.Local
|
||||
row["@"+a.Name.Local] = a.Value
|
||||
row[key] = a.Value
|
||||
}
|
||||
case xml.EndElement:
|
||||
if len(path) == 0 {
|
||||
return row, nil
|
||||
}
|
||||
path = path[:len(path)-1]
|
||||
case xml.CharData:
|
||||
text := strings.TrimSpace(string(t))
|
||||
if text == "" || len(path) == 0 {
|
||||
continue
|
||||
}
|
||||
leaf := path[len(path)-1]
|
||||
full := strings.Join(path, "/")
|
||||
// Prefer nested path as source of truth; only set bare leaf when
|
||||
// unique (no other nested field already owns this leaf name).
|
||||
if prev, ok := row[full]; ok && prev != "" && prev != text {
|
||||
row[full] = prev + " " + text
|
||||
} else {
|
||||
row[full] = text
|
||||
}
|
||||
if prev, ok := row[leaf]; !ok || prev == "" || prev == text || prev == row[full] {
|
||||
row[leaf] = row[full]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user