descrybe/apps/web/scripts/count-identical-to-en.mjs

/**
 * Count keys identical to English across locale packs.
 * Brands/loanwords/symbols are excluded from the "unfinished" total.
 *
 * Run: node apps/web/scripts/count-identical-to-en.mjs
 * Optional: --json writes apps/web/scripts/_identical-to-en-report.json
 */
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const messagesDir = path.resolve(__dirname, "../src/lib/i18n/messages");
const LOCALES = ["es", "fr", "de", "it", "pt", "nl", "pl", "ja"];

const BRAND_VALUES = new Set([
	" · upgrade",
	" (v{version})",
	"—",
	".",
	"…",
	"· upgrade",
	"(v{version})",
	"{count} file",
	"{count} options",
	"{credits} credits",
	"{used} / {max} SKUs",
	"→",
	"Account",
	"Action",
	"Actions",
	"Admin",
	"AI",
	"Alias",
	"Allowlist",
	"Analytics",
	"API",
	"Assistant",
	"Audience",
	"Azure OpenAI",
	"Base",
	"Black Friday 2026",
	"Business",
	"Client ID",
	"cm",
	"colleague@example.com",
	"Commerce",
	"Compliance",
	"CSV",
	"Date",
	"Description",
	"Descrybe",
	"Dimension",
	"Directory",
	"Docs:",
	"EAN/GTIN",
	"Enterprise",
	"EPREL",
	"EPREL ID",
	"EUR",
	"Exact",
	"Exports",
	"FAQ auto-match",
	"Feeds",
	"Format",
	"Format:",
	"format: csv",
	"Fuzzy",
	"Google OAuth",
	"GPS",
	"Growth",
	"Host",
	"https://…",
	"IA",
	"ID",
	"Insight",
	"Integration",
	"item: {path}",
	"Job {id}",
	"kg",
	"Knowledge",
	"Knowledge base",
	"Last Updated",
	"Legacy",
	"Live",
	"Mail",
	"Mapping",
	"Marketing",
	"Material",
	"Media",
	"Model",
	"Name (A-Z)",
	"Name (Z-A)",
	"Namespace",
	"No",
	"Normalize",
	"Notes",
	"OAuth, EPREL, Pinecone, Stripe, feeds",
	"OK",
	"Ollama",
	"OpenAI",
	"OpenAPI",
	"OpenRouter",
	"Ops",
	"Optional",
	"Parent",
	"pcs",
	"Pinecone",
	"Plan:",
	"Popular",
	"Product",
	"Prompt",
	"Re: {subject}",
	"REST",
	"Reviewer",
	"reviews",
	"Reviews: {status}",
	"SEO",
	"Service",
	"Shopify",
	"Single",
	"SKU",
	"Source",
	"source {source}",
	"Staff",
	"Starter",
	"Stripe",
	"Sync: {status}",
	"Tenant vs platform",
	"Tenants · volume",
	"Test: {status}",
	"Ticket",
	"Tickets",
	"Timeout",
	"TSV",
	"Type",
	"Uploads",
	"URL",
	"via Stripe",
	"Volume",
	"W",
	"WooCommerce",
	"you@company.com",
]);

function parseMessageDict(source) {
	const dict = {};
	const re = /"([^"\\]+)"\s*:\s*((?:"(?:\\.|[^"\\])*")|(?:`(?:\\.|[^`\\])*`))/gs;
	let m;
	while ((m = re.exec(source))) {
		const key = m[1];
		const raw = m[2];
		dict[key] = raw.startsWith("`")
			? raw.slice(1, -1).replace(/\\`/g, "`").replace(/\\\$/g, "$").replace(/\\n/g, "\n")
			: JSON.parse(raw);
	}
	return dict;
}

function isBrandOrLoanword(value) {
	const raw = String(value ?? "");
	const t = raw.trim();
	if (!t) return true;
	if (BRAND_VALUES.has(raw) || BRAND_VALUES.has(t)) return true;
	if (/^\{[^}]+\}$/.test(t)) return true;
	// Short all-caps / code-like tokens kept identical by policy
	if (/^[A-Z0-9][A-Z0-9._/-]{0,14}$/.test(t) && t === t.toUpperCase()) return true;
	return false;
}

function load(code) {
	return parseMessageDict(fs.readFileSync(path.join(messagesDir, `${code}.ts`), "utf8"));
}

const en = load("en");
const enKeys = Object.keys(en);
const byLocale = {};
let totalIdentical = 0;
let totalIdenticalExBrand = 0;
let totalTranslated = 0;
let totalBrandIdentical = 0;

for (const code of LOCALES) {
	const pack = load(code);
	let identical = 0;
	let identicalExBrand = 0;
	let brandIdentical = 0;
	let translated = 0;
	let missing = 0;
	const unfinishedSamples = [];
	for (const key of enKeys) {
		if (!(key in pack)) {
			missing += 1;
			identicalExBrand += 1;
			if (unfinishedSamples.length < 8) unfinishedSamples.push(key);
			continue;
		}
		if (pack[key] === en[key]) {
			identical += 1;
			if (isBrandOrLoanword(en[key])) {
				brandIdentical += 1;
			} else {
				identicalExBrand += 1;
				if (unfinishedSamples.length < 8) unfinishedSamples.push(`${key}=${JSON.stringify(en[key])}`);
			}
		} else {
			translated += 1;
		}
	}
	byLocale[code] = {
		keys: Object.keys(pack).length,
		identical,
		identicalExBrand,
		brandIdentical,
		translated,
		missing,
		unfinishedSamples,
	};
	totalIdentical += identical;
	totalIdenticalExBrand += identicalExBrand;
	totalBrandIdentical += brandIdentical;
	totalTranslated += translated;
	console.log(
		`${code}: identical=${identical} identicalExBrand=${identicalExBrand} brandIdentical=${brandIdentical} translated=${translated} missing=${missing}`,
	);
}

const report = {
	enKeys: enKeys.length,
	locales: LOCALES,
	byLocale,
	totals: {
		identical: totalIdentical,
		identicalExBrand: totalIdenticalExBrand,
		brandIdentical: totalBrandIdentical,
		translated: totalTranslated,
		avgIdenticalExBrandPerLocale: Math.round(totalIdenticalExBrand / LOCALES.length),
	},
	measuredAt: new Date().toISOString(),
};

console.log("---");
console.log(`enKeys=${report.enKeys}`);
console.log(`TOTAL identical=${totalIdentical}`);
console.log(`TOTAL identicalExBrand (excl brands)=${totalIdenticalExBrand}`);
console.log(`AVG identicalExBrand/locale=${report.totals.avgIdenticalExBrandPerLocale}`);
console.log(`TOTAL brandIdentical=${totalBrandIdentical}`);
console.log(`TOTAL translated=${totalTranslated}`);

if (process.argv.includes("--json")) {
	const out = path.join(__dirname, "_identical-to-en-report.json");
	fs.writeFileSync(out, JSON.stringify(report, null, 2), "utf8");
	console.log(`wrote ${out}`);
}