import fs from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const OUT_DIR = path.resolve(__dirname, "..", "apps", "web", "public", "data", "subcables"); const GEO_URL = "https://www.submarinecablemap.com/api/v3/cable/cable-geo.json"; const DETAILS_URL_BASE = "https://www.submarinecablemap.com/api/v3/cable/"; const CONCURRENCY = Math.max(1, Math.min(24, Number(process.env.CONCURRENCY || 12))); const TIMEOUT_MS = Math.max(5_000, Math.min(60_000, Number(process.env.TIMEOUT_MS || 20_000))); function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } async function fetchText(url, { timeoutMs = TIMEOUT_MS } = {}) { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), timeoutMs); try { const res = await fetch(url, { signal: controller.signal, headers: { accept: "application/json", }, }); const text = await res.text(); const contentType = res.headers.get("content-type") || ""; if (!res.ok) { throw new Error(`HTTP ${res.status} (${res.statusText})`); } return { text, contentType }; } finally { clearTimeout(timeout); } } async function fetchJson(url) { const { text, contentType } = await fetchText(url); if (!contentType.toLowerCase().includes("application/json")) { const snippet = text.slice(0, 200).replace(/\s+/g, " ").trim(); throw new Error(`Unexpected content-type (${contentType || "unknown"}): ${snippet || ""}`); } try { return JSON.parse(text); } catch (e) { throw new Error(`Invalid JSON: ${e instanceof Error ? e.message : String(e)}`); } } async function fetchJsonWithRetry(url, attempts = 2) { let lastErr = null; for (let i = 0; i < attempts; i += 1) { try { return await fetchJson(url); } catch (e) { lastErr = e; if (i < attempts - 1) { await sleep(250 * (i + 1)); } } } throw lastErr; } function pickCableDetails(raw) { const obj = raw && typeof raw === "object" ? raw : {}; const landingPoints = Array.isArray(obj.landing_points) ? obj.landing_points : []; return { id: String(obj.id || ""), name: String(obj.name || ""), length: obj.length == null ? null : String(obj.length), rfs: obj.rfs == null ? null : String(obj.rfs), rfs_year: typeof obj.rfs_year === "number" ? obj.rfs_year : null, is_planned: Boolean(obj.is_planned), owners: obj.owners == null ? null : String(obj.owners), suppliers: obj.suppliers == null ? null : String(obj.suppliers), landing_points: landingPoints.map((lp) => { const p = lp && typeof lp === "object" ? lp : {}; return { id: String(p.id || ""), name: String(p.name || ""), country: String(p.country || ""), is_tbd: p.is_tbd === true, }; }), notes: obj.notes == null ? null : String(obj.notes), url: obj.url == null ? null : String(obj.url), }; } async function main() { await fs.mkdir(OUT_DIR, { recursive: true }); console.log(`[subcables] fetching geojson: ${GEO_URL}`); const geo = await fetchJsonWithRetry(GEO_URL, 3); const geoPath = path.join(OUT_DIR, "cable-geo.json"); await fs.writeFile(geoPath, JSON.stringify(geo)); const features = Array.isArray(geo?.features) ? geo.features : []; const ids = Array.from( new Set( features .map((f) => f?.properties?.id) .filter((v) => typeof v === "string" && v.trim().length > 0) .map((v) => v.trim()), ), ).sort(); console.log(`[subcables] cables: ${ids.length} (concurrency=${CONCURRENCY}, timeoutMs=${TIMEOUT_MS})`); const byId = {}; const failures = []; let cursor = 0; let completed = 0; const startedAt = Date.now(); const worker = async () => { for (;;) { const idx = cursor; cursor += 1; if (idx >= ids.length) return; const id = ids[idx]; const url = new URL(`${id}.json`, DETAILS_URL_BASE).toString(); try { const raw = await fetchJsonWithRetry(url, 2); const picked = pickCableDetails(raw); if (!picked.id) { throw new Error("Missing id in details response"); } byId[id] = picked; } catch (e) { failures.push({ id, error: e instanceof Error ? e.message : String(e) }); } finally { completed += 1; if (completed % 25 === 0 || completed === ids.length) { const sec = Math.max(1, Math.round((Date.now() - startedAt) / 1000)); const rate = (completed / sec).toFixed(1); console.log(`[subcables] ${completed}/${ids.length} (${rate}/s)`); } } } }; await Promise.all(Array.from({ length: CONCURRENCY }, () => worker())); const detailsOut = { version: 1, generated_at: new Date().toISOString(), by_id: byId, }; const detailsPath = path.join(OUT_DIR, "cable-details.min.json"); await fs.writeFile(detailsPath, JSON.stringify(detailsOut)); if (failures.length > 0) { console.error(`[subcables] failures: ${failures.length}`); for (const f of failures.slice(0, 30)) { console.error(`- ${f.id}: ${f.error}`); } if (failures.length > 30) { console.error(`- ... +${failures.length - 30} more`); } process.exitCode = 1; } console.log(`[subcables] wrote: ${geoPath}`); console.log(`[subcables] wrote: ${detailsPath}`); } main().catch((e) => { console.error(`[subcables] fatal: ${e instanceof Error ? e.stack || e.message : String(e)}`); process.exitCode = 1; });