descrybe/apps/api/internal/feeds/parse.go

package feeds

import (
	"bytes"
	"encoding/csv"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"strings"
)

// defaultMaxParseRows caps CSV/XML product rows per sync. Parse is streaming
// (one row at a time); the bound limits sync duration and DB write volume for
// oversized catalogs. Exceeding returns parseTooManyRows() with the numeric limit.
const defaultMaxParseRows = 1_000_000

// maxParseRows is the active row cap. Mutable for tests.
var maxParseRows = defaultMaxParseRows

// errParseTooManyRows is the sentinel for ClientError / errors.Is checks.
var errParseTooManyRows = errors.New("feed exceeds max row limit")

// parseTooManyRows returns errParseTooManyRows with the active row limit for clients.
func parseTooManyRows() error {
	return fmt.Errorf("%w (%d)", errParseTooManyRows, maxParseRows)
}

// feedRow is a normalized flat record from CSV or XML.
type feedRow map[string]string

func detectFeedFormat(feedType, contentType, urlHint string, sample []byte) string {
	ft := strings.ToLower(strings.TrimSpace(feedType))
	if ft == "csv" || ft == "xml" {
		return ft
	}
	ct := strings.ToLower(contentType)
	u := strings.ToLower(urlHint)
	switch {
	case strings.Contains(ct, "csv") || strings.HasSuffix(u, ".csv"):
		return "csv"
	case strings.Contains(ct, "xml") || strings.HasSuffix(u, ".xml"):
		return "xml"
	}
	trimmed := bytes.TrimSpace(sample)
	if len(trimmed) > 0 && trimmed[0] == '<' {
		return "xml"
	}
	return "csv"
}

// parseCSV streams rows via callback to avoid holding the full matrix when possible.
// The CSV reader still tokenizes; we only keep one row at a time in the callback path.
func parseCSV(r io.Reader, onRow func(feedRow) error) (int, error) {
	cr := csv.NewReader(r)
	cr.ReuseRecord = true
	cr.LazyQuotes = true
	cr.TrimLeadingSpace = true
	cr.FieldsPerRecord = -1

	header, err := cr.Read()
	if err != nil {
		return 0, fmt.Errorf("csv header: %w", err)
	}
	cols := make([]string, len(header))
	for i, h := range header {
		cols[i] = strings.TrimSpace(h)
	}
	count := 0
	for {
		rec, err := cr.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return count, fmt.Errorf("csv row %d: %w", count+1, err)
		}
		count++
		if count > maxParseRows {
			return count, parseTooManyRows()
		}
		row := make(feedRow, len(cols))
		for i, col := range cols {
			if col == "" {
				continue
			}
			if i < len(rec) {
				row[col] = rec[i]
			} else {
				row[col] = ""
			}
		}
		expandSpecificationFields(row)
		if err := onRow(row); err != nil {
			return count, err
		}
	}
	return count, nil
}

// parseXMLItems streams element-local text maps for repeating item tags.
// When itemLocal is empty and r is seekable, a small prefix is sniffed then rewound.
func parseXMLItems(r io.Reader, itemLocal string, onRow func(feedRow) error) (int, error) {
	itemLocal = strings.TrimSpace(itemLocal)
	if itemLocal == "" {
		var err error
		itemLocal, r, err = resolveXMLItemLocal(r)
		if err != nil {
			return 0, err
		}
	}
	dec := xml.NewDecoder(r)
	dec.Strict = false
	count := 0
	for {
		tok, err := dec.Token()
		if err == io.EOF {
			break
		}
		if err != nil {
			return count, fmt.Errorf("xml: %w", err)
		}
		se, ok := tok.(xml.StartElement)
		if !ok {
			continue
		}
		if !localNameEquals(se.Name, itemLocal) {
			continue
		}
		row, err := readXMLElementMap(dec, se)
		if err != nil {
			return count, err
		}
		expandSpecificationFields(row)
		count++
		if count > maxParseRows {
			return count, parseTooManyRows()
		}
		if err := onRow(row); err != nil {
			return count, err
		}
	}
	return count, nil
}

const xmlItemGuessBytes = 64 << 10

func resolveXMLItemLocal(r io.Reader) (string, io.Reader, error) {
	if rs, ok := r.(io.ReadSeeker); ok {
		sample := make([]byte, xmlItemGuessBytes)
		n, err := io.ReadFull(rs, sample)
		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
			return "", nil, err
		}
		sample = sample[:n]
		local := guessXMLItemLocal(sample)
		if _, err := rs.Seek(0, io.SeekStart); err != nil {
			return "", nil, err
		}
		return local, rs, nil
	}
	sample, err := io.ReadAll(io.LimitReader(r, xmlItemGuessBytes))
	if err != nil {
		return "", nil, err
	}
	local := guessXMLItemLocal(sample)
	return local, io.MultiReader(bytes.NewReader(sample), r), nil
}

func guessXMLItemLocal(data []byte) string {
	sample := string(data)
	if len(sample) > xmlItemGuessBytes {
		sample = sample[:xmlItemGuessBytes]
	}
	lower := strings.ToLower(sample)
	for _, cand := range []string{"item", "product", "entry", "offer", "row"} {
		if strings.Contains(lower, "<"+cand) || strings.Contains(lower, ":"+cand) {
			return cand
		}
	}
	return "item"
}

func localNameEquals(n xml.Name, local string) bool {
	return strings.EqualFold(n.Local, local)
}

func readXMLElementMap(dec *xml.Decoder, start xml.StartElement) (feedRow, error) {
	row := make(feedRow)
	for _, a := range start.Attr {
		key := "@" + a.Name.Local
		row[key] = a.Value
		row[start.Name.Local+"/"+key] = a.Value
	}
	var path []string
	for {
		tok, err := dec.Token()
		if err != nil {
			return nil, err
		}
		switch t := tok.(type) {
		case xml.StartElement:
			path = append(path, t.Name.Local)
			for _, a := range t.Attr {
				key := strings.Join(path, "/") + "/@" + a.Name.Local
				row["@"+a.Name.Local] = a.Value
				row[key] = a.Value
			}
		case xml.EndElement:
			if len(path) == 0 {
				return row, nil
			}
			path = path[:len(path)-1]
		case xml.CharData:
			text := strings.TrimSpace(string(t))
			if text == "" || len(path) == 0 {
				continue
			}
			leaf := path[len(path)-1]
			full := strings.Join(path, "/")
			// Prefer nested path as source of truth; only set bare leaf when
			// unique (no other nested field already owns this leaf name).
			if prev, ok := row[full]; ok && prev != "" && prev != text {
				row[full] = prev + " " + text
			} else {
				row[full] = text
			}
			if prev, ok := row[leaf]; !ok || prev == "" || prev == text || prev == row[full] {
				row[leaf] = row[full]
			}
		}
	}
}