From 593edf35fed40d7798b1715c82459aabb4aa0da5 Mon Sep 17 00:00:00 2001 From: GreenEclipse Date: Mon, 10 Aug 2026 02:04:45 +0200 Subject: [PATCH] update --- Makefile | 6 +- apps/api/Makefile | 12 ++- apps/web/package.json | 1 + apps/web/vite.config.ts | 22 +++++ package.json | 5 +- scripts/build.mjs | 147 ++++++++++++++++++++++++++++++++ scripts/lowmem-env.mjs | 180 ++++++++++++++++++++++++++++++++++++++++ scripts/migrate.mjs | 44 ++++------ scripts/seed-local.mjs | 5 +- scripts/setup.mjs | 5 +- 10 files changed, 394 insertions(+), 33 deletions(-) create mode 100644 scripts/build.mjs create mode 100644 scripts/lowmem-env.mjs diff --git a/Makefile b/Makefile index d1fcfe2..dabaa0a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: up down api worker backend web migrate sqlc test test-api check-web vet seed seed-woo mock-woo health setup cutover-rehearsal +.PHONY: up down api worker backend web migrate sqlc test test-api check-web vet seed seed-woo mock-woo health setup cutover-rehearsal build # Start local Postgres only (:5433 — pairs with host npm run setup / npm run dev) up: @@ -68,3 +68,7 @@ vet: # Type-check SvelteKit (install deps first if needed) check-web: cd apps/web && npm run check + +# Low-memory production build (web then Go api+worker binaries; ~4GB-safe defaults) +build: + node scripts/build.mjs diff --git a/apps/api/Makefile b/apps/api/Makefile index 9572e19..af4d5a8 100644 --- a/apps/api/Makefile +++ b/apps/api/Makefile @@ -2,7 +2,11 @@ DATABASE_URL ?= postgres://descrybe:descrybe@localhost:5433/descrybe?sslmode=dis GOOSE ?= go run github.com/pressly/goose/v3/cmd/goose@v3.24.1 SQLC ?= go run github.com/sqlc-dev/sqlc/cmd/sqlc@v1.29.0 -.PHONY: migrate-up sqlc run worker migrator tidy test vet +.PHONY: migrate-up sqlc run worker migrator tidy test vet build build-lowmem + +# Low-RAM compile defaults (~4GB hosts): one package at a time, capped GOMAXPROCS. +GO_BUILD_P ?= 1 +GO_BIN_DIR ?= ../../bin migrate-up: $(GOOSE) -dir sql/schema postgres "$(DATABASE_URL)" up @@ -27,3 +31,9 @@ test: vet: go vet ./... + +# Sequential api + worker binaries into repo bin/ (see scripts/build.mjs). +build build-lowmem: + mkdir -p "$(GO_BIN_DIR)" + GOMAXPROCS=1 GOMEMLIMIT=768MiB GOGC=50 go build -p=$(GO_BUILD_P) -trimpath -ldflags="-s -w" -o "$(GO_BIN_DIR)/api" ./cmd/api + GOMAXPROCS=1 GOMEMLIMIT=768MiB GOGC=50 go build -p=$(GO_BUILD_P) -trimpath -ldflags="-s -w" -o "$(GO_BIN_DIR)/worker" ./cmd/worker diff --git a/apps/web/package.json b/apps/web/package.json index 368a8f7..72c5969 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -6,6 +6,7 @@ "scripts": { "dev": "vite dev --host --strictPort", "build": "vite build", + "build:lowmem": "node ../../scripts/build.mjs --web", "start": "node build", "preview": "vite preview", "prepare": "svelte-kit sync || echo '' && node scripts/copy-rapidoc-ui.mjs", diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index 5baa837..ad9b0cc 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -4,13 +4,35 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { defineConfig } from "vite"; import { DEV_API_PROXY_TARGET, DEV_WEB_PORT } from "../../scripts/dev-ports.mjs"; +import { detectHostProfile } from "../../scripts/lowmem-env.mjs"; const monorepoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const profile = detectHostProfile(); +/** Env wins; otherwise follow auto RAM/CPU tier. */ +const lowMem = + process.env.BUILD_LOW_MEM != null && process.env.BUILD_LOW_MEM !== "" + ? process.env.BUILD_LOW_MEM !== "0" + : profile.lowMem; + export default defineConfig({ // Load PUBLIC_* from monorepo-root .env (same file as the Go API). envDir: monorepoRoot, plugins: [tailwindcss(), sveltekit()], + build: { + // Gzip size reporting double-reads output and spikes peak RSS on tiny hosts. + reportCompressedSize: !lowMem, + // Production maps are opt-in; maps roughly double transform memory. + sourcemap: process.env.BUILD_SOURCEMAP === "1", + // Smaller inlined assets → less heap during transform on 4GB boxes. + assetsInlineLimit: lowMem ? 1024 : 4096, + rolldownOptions: { + checks: { + // Informative only; sveltekit-guard resolveId dominates wall time (see Rolldown docs). + pluginTimings: process.env.BUILD_PLUGIN_TIMINGS === "1" + } + } + }, server: { // true → all interfaces (IPv4 + IPv6). Default localhost is ::1-only on Windows, // which breaks 127.0.0.1 probes and some browser Happy Eyeballs paths. diff --git a/package.json b/package.json index e81e32a..549e41b 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,10 @@ "cutover:deploy-check": "node scripts/cutover-deploy-check.mjs", "cutover:deploy-check:code": "node scripts/cutover-deploy-check.mjs --skip-goose --skip-readyz", "start:web": "npm run start --workspace=web", - "build": "npm run build --workspace=web && node scripts/run-in-dir.mjs apps/api go build ./...", + "build": "node scripts/build.mjs", + "build:web": "node scripts/build.mjs --web", + "build:api": "node scripts/build.mjs --api", + "build:go:all": "node scripts/run-in-dir.mjs apps/api go build -p=1 ./...", "predev": "node scripts/free-dev-ports.mjs", "dev": "concurrently -k -n api,web,worker -c cyan,magenta,yellow \"npm run dev:api:inner\" \"npm run dev:web:inner\" \"npm run dev:worker:inner\"", "dev:api:inner": "node scripts/with-forced-env.mjs HTTP_ADDR=:28471 WEB_ORIGIN=http://localhost:28472 PUBLIC_API_URL=http://localhost:28471 -- node scripts/run-in-dir.mjs apps/api go run ./cmd/api", diff --git a/scripts/build.mjs b/scripts/build.mjs new file mode 100644 index 0000000..44a828e --- /dev/null +++ b/scripts/build.mjs @@ -0,0 +1,147 @@ +#!/usr/bin/env node +/** + * Production build with auto host detection (Windows/Linux/macOS, RAM, CPU). + * Web then Go — never parallel. Caps cores so rebuilds do not pin 100%. + * + * Env (optional overrides): + * BUILD_PROFILE=low|medium|high + * BUILD_NODE_HEAP_MB / BUILD_GO_P / BUILD_GO_MAXPROCS / BUILD_GO_MEMLIMIT + * BUILD_LOW_MEM=0|1 + * BUILD_SKIP_WEB=1 / BUILD_SKIP_GO=1 / BUILD_GO_ALL=1 + * BUILD_BIN_DIR (default /bin) + * + * Usage: + * node scripts/build.mjs + * npm run build | build:web | build:api + */ +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { + detectHostProfile, + formatHostNote, + withGoLowMem, + withNodeHeap, +} from "./lowmem-env.mjs"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const isWin = process.platform === "win32"; +const exe = isWin ? ".exe" : ""; + +const args = new Set(process.argv.slice(2)); +const onlyWeb = args.has("--web") || args.has("web"); +const onlyGo = args.has("--api") || args.has("api") || args.has("--go") || args.has("go"); +const wantHelp = args.has("-h") || args.has("--help"); + +if (wantHelp) { + console.log(`Usage: node scripts/build.mjs [--web|--api] + +Auto-detects OS/RAM/CPU and scales Node heap + Go -p / GOMAXPROCS. +Alias: npm run build | build:web | build:api`); + process.exit(0); +} + +const profile = detectHostProfile(); + +function section(title) { + console.log(`\n==> ${title}`); +} + +function run(command, cmdArgs, { cwd = root, env = process.env, label, shell } = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, cmdArgs, { + cwd, + env, + stdio: "inherit", + shell: shell ?? (isWin && command === "npm"), + }); + child.on("error", reject); + child.on("exit", (code, signal) => { + if (signal) { + reject(new Error(`${label ?? command}: killed by ${signal}`)); + return; + } + if (code !== 0) { + reject(new Error(`${label ?? command}: exit ${code ?? 1}`)); + return; + } + resolve(); + }); + }); +} + +async function buildWeb() { + section("Web (Vite / SvelteKit)"); + const env = withNodeHeap(process.env, profile.nodeHeapMb, profile.uvThreadpool); + console.log( + `NOTE NODE_OPTIONS max-old-space-size=${profile.nodeHeapMb} UV_THREADPOOL_SIZE=${env.UV_THREADPOOL_SIZE} BUILD_LOW_MEM=${env.BUILD_LOW_MEM}`, + ); + await run( + "npm", + ["run", "build", "--workspace=web"], + { env, label: "web build" }, + ); +} + +async function buildGo() { + section("API + worker build"); + const apiDir = path.join(root, "apps", "api"); + const binDir = path.resolve(root, process.env.BUILD_BIN_DIR || "bin"); + fs.mkdirSync(binDir, { recursive: true }); + const env = withGoLowMem(process.env, profile); + const apiOut = path.join(binDir, `api${exe}`); + const workerOut = path.join(binDir, `worker${exe}`); + const p = profile.goP; + + console.log( + `NOTE GOMAXPROCS=${env.GOMAXPROCS}` + + (env.GOMEMLIMIT ? ` GOMEMLIMIT=${env.GOMEMLIMIT}` : "") + + (env.GOGC ? ` GOGC=${env.GOGC}` : "") + + ` go -p ${p}`, + ); + + // Always sequential api then worker — parallel binaries double peak RAM. + const ldflags = "-s -w"; + await run( + "go", + ["build", `-p=${p}`, "-trimpath", `-ldflags=${ldflags}`, "-o", apiOut, "./cmd/api"], + { cwd: apiDir, env, label: "go build api", shell: false }, + ); + console.log(`OK ${path.relative(root, apiOut)}`); + + await run( + "go", + ["build", `-p=${p}`, "-trimpath", `-ldflags=${ldflags}`, "-o", workerOut, "./cmd/worker"], + { cwd: apiDir, env, label: "go build worker", shell: false }, + ); + console.log(`OK ${path.relative(root, workerOut)}`); + + if (process.env.BUILD_GO_ALL === "1") { + section("Go packages (go build ./...)"); + await run( + "go", + ["build", `-p=${p}`, "./..."], + { cwd: apiDir, env, label: "go build ./..." }, + ); + } +} + +async function main() { + section("Host"); + console.log(`NOTE ${formatHostNote(profile)}`); + + const skipWeb = process.env.BUILD_SKIP_WEB === "1" || onlyGo; + const skipGo = process.env.BUILD_SKIP_GO === "1" || onlyWeb; + + if (!skipWeb) await buildWeb(); + if (!skipGo) await buildGo(); + + section("Done"); + console.log(`Build finished (tier=${profile.tier}, sequential web→go).`); +} + +main().catch((err) => { + console.error(`FAIL ${err?.message || err}`); + process.exit(1); +}); diff --git a/scripts/lowmem-env.mjs b/scripts/lowmem-env.mjs new file mode 100644 index 0000000..cf651d6 --- /dev/null +++ b/scripts/lowmem-env.mjs @@ -0,0 +1,180 @@ +/** + * Cross-platform host profile for build / migrate / seed / Vite. + * Auto-detects OS, RAM, and CPU; caps parallelism so rebuilds do not pin 100% forever. + * + * Override anytime: + * BUILD_PROFILE=low|medium|high + * BUILD_NODE_HEAP_MB / BUILD_GO_P / BUILD_GO_MAXPROCS / BUILD_GO_MEMLIMIT + * BUILD_LOW_MEM=0|1 + */ +import os from "node:os"; + +export function envInt(name, fallback, env = process.env) { + const raw = env[name]; + if (raw == null || raw === "") return fallback; + const n = Number.parseInt(raw, 10); + return Number.isFinite(n) && n > 0 ? n : fallback; +} + +export function envFlag(name, env = process.env) { + const raw = env[name]; + if (raw == null || raw === "") return null; + if (/^(1|true|yes|on)$/i.test(raw)) return true; + if (/^(0|false|no|off)$/i.test(raw)) return false; + return null; +} + +/** Total / free RAM in MiB (Node os.*; works on Windows, Linux, macOS). */ +export function hostRamMiB() { + const total = Math.floor(os.totalmem() / (1024 * 1024)); + const free = Math.floor(os.freemem() / (1024 * 1024)); + return { + totalMiB: Number.isFinite(total) && total > 0 ? total : null, + freeMiB: Number.isFinite(free) && free >= 0 ? free : null, + }; +} + +export function hostCpuCount() { + const n = os.cpus()?.length ?? 1; + return Number.isFinite(n) && n > 0 ? n : 1; +} + +/** + * Pick a resource tier. + * - low: ≤5 GiB RAM or ≤2 CPUs (typical 4GiB VPS) + * - medium: ≤12 GiB + * - high: everything else (dev workstations) + */ +export function detectTier(env = process.env) { + const forced = String(env.BUILD_PROFILE || "").toLowerCase(); + if (forced === "low" || forced === "medium" || forced === "high") return forced; + + const { totalMiB } = hostRamMiB(); + const cpus = hostCpuCount(); + if (totalMiB != null && totalMiB <= 5120) return "low"; + if (cpus <= 2) return "low"; + if (totalMiB != null && totalMiB <= 12288) return "medium"; + return "high"; +} + +/** + * Leave CPU headroom for OS / Postgres / shell so rebuilds do not pin 100%. + * low → 1 · medium → ≤4 and keep 1 free · high → ≤75% and keep ≥2 free when possible + */ +function cappedProcs(cpus, tier) { + if (tier === "low") return 1; + if (tier === "medium") return Math.max(1, Math.min(cpus - 1, 4)); + if (cpus <= 4) return Math.max(1, cpus - 1); + const keepFree = Math.max(2, Math.ceil(cpus * 0.25)); + return Math.max(1, cpus - keepFree); +} + +export function detectHostProfile(env = process.env) { + const platform = process.platform; // win32 | linux | darwin | … + const { totalMiB, freeMiB } = hostRamMiB(); + const cpus = hostCpuCount(); + const tier = detectTier(env); + + let nodeHeapMb; + let goMaxProcs; + let goP; + let goMemLimit; // string or null (unset = Go default) + let goGc; // string or null + let uvThreadpool; + let lowMem; + + if (tier === "low") { + // Keep Node well under total RAM; leave room for Postgres + Go. + const budget = totalMiB != null ? Math.floor(totalMiB * 0.35) : 1536; + nodeHeapMb = Math.min(1536, Math.max(768, budget)); + goMaxProcs = 1; + goP = 1; + goMemLimit = "768MiB"; + goGc = "50"; + uvThreadpool = 2; + lowMem = true; + } else if (tier === "medium") { + nodeHeapMb = Math.min(3072, totalMiB != null ? Math.floor(totalMiB * 0.3) : 3072); + goMaxProcs = cappedProcs(cpus, tier); + goP = goMaxProcs; + goMemLimit = "2048MiB"; + goGc = null; + uvThreadpool = 4; + lowMem = false; + } else { + nodeHeapMb = Math.min(6144, totalMiB != null ? Math.floor(totalMiB * 0.25) : 4096); + goMaxProcs = cappedProcs(cpus, tier); + goP = goMaxProcs; + goMemLimit = null; + goGc = null; + uvThreadpool = Math.min(8, Math.max(4, goMaxProcs)); + lowMem = false; + } + + // Explicit env always wins. + nodeHeapMb = envInt("BUILD_NODE_HEAP_MB", nodeHeapMb, env); + goMaxProcs = envInt("BUILD_GO_MAXPROCS", goMaxProcs, env); + goP = envInt("BUILD_GO_P", goP, env); + if (env.BUILD_GO_MEMLIMIT) goMemLimit = env.BUILD_GO_MEMLIMIT; + const lowMemFlag = envFlag("BUILD_LOW_MEM", env); + if (lowMemFlag != null) lowMem = lowMemFlag; + + return { + platform, + tier, + totalMiB, + freeMiB, + cpus, + nodeHeapMb, + goMaxProcs, + goP, + goMemLimit, + goGc, + uvThreadpool, + lowMem, + }; +} + +/** Merge --max-old-space-size into NODE_OPTIONS without duplicating the flag. */ +export function withNodeHeap(baseEnv = process.env, heapMb, uvThreadpool) { + const profile = heapMb == null || uvThreadpool == null ? detectHostProfile(baseEnv) : null; + const heap = heapMb ?? profile.nodeHeapMb; + const uv = uvThreadpool ?? profile.uvThreadpool; + const out = { ...baseEnv }; + const flag = `--max-old-space-size=${heap}`; + const prev = out.NODE_OPTIONS ?? ""; + if (/\b--max-old-space-size=\d+\b/.test(prev)) { + out.NODE_OPTIONS = prev.replace(/\b--max-old-space-size=\d+\b/g, flag); + } else { + out.NODE_OPTIONS = prev ? `${prev} ${flag}` : flag; + } + if (!out.UV_THREADPOOL_SIZE) out.UV_THREADPOOL_SIZE = String(uv); + // Tell Vite whether to use low-mem knobs (only set when unset). + if (out.BUILD_LOW_MEM == null || out.BUILD_LOW_MEM === "") { + out.BUILD_LOW_MEM = (profile ?? detectHostProfile(baseEnv)).lowMem ? "1" : "0"; + } + return out; +} + +/** Apply Go concurrency / soft memory caps from the detected (or overridden) profile. */ +export function withGoLowMem(baseEnv = process.env, profile = detectHostProfile(baseEnv)) { + const out = { ...baseEnv }; + if (!out.GOMAXPROCS) out.GOMAXPROCS = String(profile.goMaxProcs); + if (profile.goMemLimit && !out.GOMEMLIMIT) out.GOMEMLIMIT = profile.goMemLimit; + if (profile.goGc && !out.GOGC) out.GOGC = profile.goGc; + return out; +} + +export function formatHostNote(profile = detectHostProfile()) { + const ram = + profile.totalMiB != null + ? `${profile.totalMiB}MiB total` + + (profile.freeMiB != null ? `, ~${profile.freeMiB}MiB free` : "") + : "RAM unknown"; + return ( + `${profile.platform} tier=${profile.tier} cpus=${profile.cpus} (${ram}) → ` + + `nodeHeap=${profile.nodeHeapMb} go -p=${profile.goP} GOMAXPROCS=${profile.goMaxProcs}` + + (profile.goMemLimit ? ` GOMEMLIMIT=${profile.goMemLimit}` : "") + + (profile.lowMem ? " lowMem" : "") + ); +} diff --git a/scripts/migrate.mjs b/scripts/migrate.mjs index 32640ed..c1d1b78 100644 --- a/scripts/migrate.mjs +++ b/scripts/migrate.mjs @@ -13,8 +13,12 @@ * prebuilt `sqlc` binary and still prefer ≥6GiB). */ import { spawnSync } from "node:child_process"; -import fs from "node:fs"; import path from "node:path"; +import { + detectHostProfile, + formatHostNote, + withGoLowMem, +} from "./lowmem-env.mjs"; import { loadRootEnv, repoRoot } from "./root-env.mjs"; const args = new Set(process.argv.slice(2)); @@ -26,7 +30,8 @@ const wantSqlc = /** Prefer this much free/total RAM (MiB) before attempting sqlc go-run/wazero. */ const SQLC_MIN_RAM_MIB = Number(process.env.DESCRYBE_SQLC_MIN_RAM_MIB || 6144); -const env = loadRootEnv(process.env); +const profile = detectHostProfile(); +const env = withGoLowMem(loadRootEnv(process.env), profile); if (!env.DATABASE_URL) { console.error( "DATABASE_URL is required (set in monorepo-root .env or the shell; see .env.example)", @@ -41,7 +46,8 @@ function run(command, cmdArgs, opts = {}) { cwd: apiDir, env, stdio: "inherit", - shell: process.platform === "win32", + // Keep go argv intact on Windows (ldflags / module paths). + shell: process.platform === "win32" && command !== "go", ...opts, }); if (r.error) { @@ -53,31 +59,8 @@ function run(command, cmdArgs, opts = {}) { } } -/** Best-effort total RAM in MiB (Linux /proc, Windows wmic, or null). */ function totalRamMiB() { - try { - if (process.platform === "linux") { - const text = fs.readFileSync("/proc/meminfo", "utf8"); - const m = text.match(/^MemTotal:\s+(\d+)\s+kB/m); - if (m) return Math.floor(Number(m[1]) / 1024); - } - if (process.platform === "win32") { - const r = spawnSync( - "powershell", - [ - "-NoProfile", - "-Command", - "[int]((Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory/1MB)", - ], - { encoding: "utf8" }, - ); - const n = Number(String(r.stdout || "").trim()); - if (Number.isFinite(n) && n > 0) return n; - } - } catch { - /* ignore */ - } - return null; + return profile.totalMiB; } function hasSqlcBinary() { @@ -89,6 +72,13 @@ function hasSqlcBinary() { } console.log("==> goose up (apps/api/sql/schema)"); +console.log(`NOTE ${formatHostNote(profile)}`); +console.log( + `NOTE GOMAXPROCS=${env.GOMAXPROCS}` + + (env.GOMEMLIMIT ? ` GOMEMLIMIT=${env.GOMEMLIMIT}` : "") + + (env.GOGC ? ` GOGC=${env.GOGC}` : "") + + " (sqlc skipped by default; needs ≥6GiB)", +); run("go", [ "run", "github.com/pressly/goose/v3/cmd/goose@v3.24.3", diff --git a/scripts/seed-local.mjs b/scripts/seed-local.mjs index fbc1a9a..6d47ca6 100644 --- a/scripts/seed-local.mjs +++ b/scripts/seed-local.mjs @@ -8,9 +8,10 @@ import { spawnSync } from "node:child_process"; import path from "node:path"; import { DEV_API_PORT, DEV_WEB_PORT } from "./dev-ports.mjs"; +import { detectHostProfile, withGoLowMem } from "./lowmem-env.mjs"; import { loadRootEnv, repoRoot } from "./root-env.mjs"; -const env = loadRootEnv(process.env); +const env = withGoLowMem(loadRootEnv(process.env), detectHostProfile()); if (!env.DATABASE_URL) { env.DATABASE_URL = "postgres://descrybe:descrybe@localhost:5433/descrybe?sslmode=disable"; @@ -28,7 +29,7 @@ function run(args) { cwd: apiDir, env, stdio: "inherit", - shell: process.platform === "win32", + shell: false, }); if (r.status !== 0) process.exit(r.status ?? 1); } diff --git a/scripts/setup.mjs b/scripts/setup.mjs index f89e8fc..3b09791 100644 --- a/scripts/setup.mjs +++ b/scripts/setup.mjs @@ -282,5 +282,8 @@ console.log("==> migrate"); runNodeScript("scripts/migrate.mjs"); console.log(""); -console.log("Setup complete. Next: npm install && npm run dev"); +console.log("Setup complete. Next: npm install && npm run build"); console.log("Web http://localhost:28472 · API http://localhost:28471"); +console.log( + "Note: build/migrate auto-detect OS/RAM/CPU; sqlc needs ≥6GiB (npm run migrate -- --sqlc).", +);