356 lines
20 KiB
JavaScript
Executable File
356 lines
20 KiB
JavaScript
Executable File
// Explicit maintenance only: importing this module never uses the network or writes.
|
|
import * as fs from 'node:fs/promises';
|
|
import { constants } from 'node:fs';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { safePath } from './files.mjs';
|
|
|
|
const ROOT = fileURLToPath(new URL('../', import.meta.url));
|
|
const CATALOGUE = path.join(ROOT, 'mods/hearth/botany.lua');
|
|
const OUTPUT = path.join(ROOT, 'mods/hearth/wiki_articles.lua');
|
|
export const LIMITS = Object.freeze({ species: 100, batch: 10, concurrency: 2,
|
|
responseBytes: 2 * 1024 * 1024, catalogueBytes: 128 * 1024,
|
|
characters: 8000, requestMs: 15000, totalMs: 90000, redirects: 2 });
|
|
export const USER_AGENT = 'LastHearthSpeciesBundle/1.0 (offline educational game references; explicit manual maintenance)';
|
|
const ORIGIN = 'https://en.wikipedia.org';
|
|
const LICENSE = 'CC BY-SA 4.0';
|
|
const LICENSE_URL = 'https://creativecommons.org/licenses/by-sa/4.0/';
|
|
const NOTE_LICENSE = 'CC0 1.0';
|
|
const NOTE_LICENSE_URL = 'https://creativecommons.org/publicdomain/zero/1.0/';
|
|
const HEADER = '-- Generated by scripts/wiki-species.mjs; Wikipedia contributors, CC BY-SA 4.0.\n'
|
|
+ '-- Plaintext introduction excerpts, not full articles. See each revision/history and docs/species-references.md.\n';
|
|
const FIELDS = Object.freeze(['latin', 'title', 'url', 'pageid', 'revision', 'revision_url',
|
|
'history_url', 'revision_timestamp', 'retrieved', 'text', 'kind', 'excerpt', 'truncated',
|
|
'attribution', 'license', 'license_url', 'changes']);
|
|
|
|
function requireValue(ok, message) { if (!ok) throw new Error(message); }
|
|
function latinName(value) {
|
|
requireValue(typeof value === 'string' && value.length <= 100 &&
|
|
/^[A-Z][a-z]+ [a-z]+(?:-[a-z]+)*$/.test(value), 'Invalid Latin name (not a URL or query).');
|
|
return value;
|
|
}
|
|
function validateCatalogue(records) {
|
|
requireValue(Array.isArray(records) && records.length > 0 && records.length <= LIMITS.species, 'Catalogue scope must be 1..100.');
|
|
const ids = new Set(), names = new Set();
|
|
for (const r of records) {
|
|
requireValue(r && typeof r.id === 'string' && /^[a-z][a-z_]{0,39}$/.test(r.id) &&
|
|
!['constructor', 'prototype', '__proto__'].includes(r.id) && !ids.has(r.id), 'Invalid or duplicate species ID.');
|
|
latinName(r.latin);
|
|
requireValue(!names.has(r.latin), 'Duplicate Latin name.');
|
|
ids.add(r.id); names.add(r.latin);
|
|
}
|
|
return records;
|
|
}
|
|
|
|
// Parse only literal leading add(id,common,latin,...) rows. Never evaluate Lua.
|
|
export function parseCatalogue(source) {
|
|
requireValue(typeof source === 'string' && Buffer.byteLength(source) <= LIMITS.catalogueBytes, 'Catalogue too large.');
|
|
const rows = source.split(/\r?\n/).filter(line => /^\s*add\(/.test(line));
|
|
const records = rows.map(line => {
|
|
const m = /^\s*add\("([a-z_]+)","[^"\\\r\n]{1,100}","([^"\\\r\n]+)",/.exec(line);
|
|
requireValue(m, 'Unsupported catalogue literal.');
|
|
return { id: m[1], latin: m[2] };
|
|
});
|
|
requireValue(records.length === LIMITS.species, 'Expected exactly 100 botany catalogue rows.');
|
|
return validateCatalogue(records);
|
|
}
|
|
|
|
export function apiURL(names) {
|
|
requireValue(Array.isArray(names) && names.length > 0 && names.length <= LIMITS.batch, 'Invalid batch size.');
|
|
names.forEach(latinName);
|
|
const u = new URL('/w/api.php', ORIGIN);
|
|
u.search = new URLSearchParams({ action: 'query', format: 'json', formatversion: '2',
|
|
prop: 'extracts|info|revisions|pageprops', exintro: '1', explaintext: '1',
|
|
inprop: 'url', rvprop: 'ids|timestamp', redirects: '1', titles: names.join('|'), maxlag: '5' });
|
|
return u;
|
|
}
|
|
|
|
export function allowedURL(value) {
|
|
const u = new URL(value);
|
|
requireValue(u.origin === ORIGIN && u.protocol === 'https:' && !u.username && !u.password &&
|
|
!u.hash && !u.port, 'Only credential-free HTTPS en.wikipedia.org URLs are allowed.');
|
|
return u;
|
|
}
|
|
function allowedRequest(value, expected) {
|
|
const u = allowedURL(value);
|
|
requireValue(u.pathname === '/w/api.php' && u.search === expected.search, 'Unexpected API redirect or query.');
|
|
return u;
|
|
}
|
|
function abortable(promise, signal) {
|
|
if (signal.aborted) return Promise.reject(signal.reason);
|
|
return new Promise((resolve, reject) => {
|
|
const abort = () => reject(signal.reason);
|
|
signal.addEventListener('abort', abort, { once: true });
|
|
Promise.resolve(promise).then(resolve, reject).finally(() => signal.removeEventListener('abort', abort));
|
|
});
|
|
}
|
|
async function boundedJSON(response, signal) {
|
|
requireValue(/^application\/json(?:\s*;|$)/i.test(response.headers.get('content-type') || ''), 'Expected JSON response.');
|
|
const length = response.headers.get('content-length');
|
|
requireValue(length === null || (/^\d+$/.test(length) && Number(length) <= LIMITS.responseBytes), 'Response exceeds byte limit.');
|
|
requireValue(response.body && typeof response.body.getReader === 'function', 'Missing response stream.');
|
|
const reader = response.body.getReader(), chunks = [];
|
|
let bytes = 0, done = false;
|
|
try {
|
|
while (true) {
|
|
const part = await abortable(reader.read(), signal);
|
|
if (part.done) { done = true; break; }
|
|
bytes += part.value.byteLength;
|
|
requireValue(bytes <= LIMITS.responseBytes, 'Response exceeds byte limit.');
|
|
chunks.push(part.value);
|
|
}
|
|
return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)));
|
|
} finally {
|
|
if (!done) void reader.cancel().catch(() => {});
|
|
reader.releaseLock();
|
|
}
|
|
}
|
|
export async function fetchBatch(names, { fetchImpl = globalThis.fetch, signal: parent } = {}) {
|
|
const expected = apiURL(names);
|
|
const signal = AbortSignal.any([AbortSignal.timeout(LIMITS.requestMs), ...(parent ? [parent] : [])]);
|
|
let u = expected;
|
|
for (let hop = 0; hop <= LIMITS.redirects; hop++) {
|
|
allowedRequest(u, expected);
|
|
signal.throwIfAborted();
|
|
const pending = Promise.resolve(fetchImpl(u.href, { redirect: 'manual', credentials: 'omit',
|
|
referrerPolicy: 'no-referrer', signal, headers: { 'User-Agent': USER_AGENT, Accept: 'application/json' } })).then(response => {
|
|
if (signal.aborted) {
|
|
void response.body?.cancel().catch(() => {});
|
|
signal.throwIfAborted();
|
|
}
|
|
return response;
|
|
});
|
|
const response = await abortable(pending, signal);
|
|
if (response.url) allowedRequest(response.url, expected);
|
|
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
|
void response.body?.cancel().catch(() => {});
|
|
requireValue(hop < LIMITS.redirects && response.headers.get('location'), 'Too many or invalid redirects.');
|
|
u = allowedRequest(new URL(response.headers.get('location'), u).href, expected);
|
|
continue;
|
|
}
|
|
if (response.status !== 200) {
|
|
void response.body?.cancel().catch(() => {});
|
|
throw new Error(`Wikipedia HTTP ${response.status}; no cache written.`);
|
|
}
|
|
return boundedJSON(response, signal);
|
|
}
|
|
}
|
|
|
|
function positiveInteger(n, label) {
|
|
requireValue(Number.isSafeInteger(n) && n > 0, `Invalid ${label}.`);
|
|
return n;
|
|
}
|
|
function timestamp(s) {
|
|
requireValue(typeof s === 'string' && /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d{3})?Z$/.test(s) &&
|
|
Number.isFinite(Date.parse(s)) && new Date(s).toISOString() === s.replace(/Z$/, s.includes('.') ? 'Z' : '.000Z'), 'Invalid timestamp.');
|
|
return s;
|
|
}
|
|
export function excerpt(text) {
|
|
requireValue(typeof text === 'string' && text.trim().length > 30 &&
|
|
!/[\x00-\x08\x0b-\x1f\x7f\u202a-\u202e\u2066-\u2069]/u.test(text) &&
|
|
!/<\/?[A-Za-z][^>]*>|\[\[|\]\]/u.test(text), 'Missing or non-plaintext introduction.');
|
|
text = text.trim();
|
|
const points = Array.from(text);
|
|
if (points.length <= LIMITS.characters) return { text, truncated: false };
|
|
const head = points.slice(0, LIMITS.characters).join('');
|
|
// Keep the copied prefix verbatim; prefer paragraph, sentence, then word boundary.
|
|
const paragraph = head.lastIndexOf('\n');
|
|
const sentences = [...head.matchAll(/[.!?](?=\s)/g)];
|
|
let end = paragraph >= head.length / 2 ? paragraph : (sentences.at(-1)?.index ?? -1) + 1;
|
|
if (end < head.length / 2) end = head.lastIndexOf(' ');
|
|
requireValue(end > 0, 'No safe excerpt boundary.');
|
|
return { text: head.slice(0, end).trimEnd(), truncated: true };
|
|
}
|
|
function articleTitle(s) {
|
|
requireValue(typeof s === 'string' && s.length > 0 && s.length <= 200 &&
|
|
!/[\x00-\x1f\x7f<>\[\]{}|#]/.test(s) && !s.includes(':'), 'Invalid article title.');
|
|
return s;
|
|
}
|
|
function wikiTitle(s) { return encodeURIComponent(s.replaceAll(' ', '_')); }
|
|
function verifiedArticleURL(value, title) {
|
|
const u = allowedURL(value);
|
|
requireValue(u.pathname.startsWith('/wiki/') && !u.search &&
|
|
decodeURIComponent(u.pathname.slice(6)).replaceAll('_', ' ') === title, 'Canonical title/URL mismatch.');
|
|
return u.href;
|
|
}
|
|
function missingEntry(r, title, url, retrieved) {
|
|
return { id: r.id, latin: r.latin, title, url,
|
|
history_url: `${ORIGIN}/w/index.php?title=${wikiTitle(title)}&action=history`, retrieved,
|
|
text: `No English Wikipedia species article was available for ${r.latin} when this bundle was retrieved. This is an original availability note, not a Wikipedia excerpt. No broader taxon or similarly named species has been substituted. The game catalogue is not evidence for real-world ecological, food or medicinal effects.`,
|
|
kind: 'original_notes', excerpt: false, truncated: false,
|
|
attribution: 'The Last Hearth contributors (original availability note)',
|
|
license: NOTE_LICENSE, license_url: NOTE_LICENSE_URL,
|
|
changes: 'Original availability note dedicated to CC0; no Wikipedia article text copied.' };
|
|
}
|
|
|
|
export function entriesFromResponse(records, data, retrieved) {
|
|
validateCatalogue(records); timestamp(retrieved);
|
|
requireValue(data && !data.error && !data.warnings && data.batchcomplete === true &&
|
|
Array.isArray(data.query?.pages) && data.query.pages.length <= records.length, 'Incomplete or invalid Wikipedia response.');
|
|
const q = data.query, aliases = new Map(), pages = new Map();
|
|
for (const rule of [...(q.normalized || []), ...(q.redirects || [])]) {
|
|
articleTitle(rule.from); articleTitle(rule.to);
|
|
requireValue(!aliases.has(rule.from) && !rule.tofragment, 'Ambiguous or fragment redirect.');
|
|
aliases.set(rule.from, rule.to);
|
|
}
|
|
requireValue(aliases.size <= records.length * 3, 'Too many title aliases.');
|
|
for (const page of q.pages) {
|
|
articleTitle(page.title);
|
|
requireValue(!pages.has(page.title), 'Duplicate article title.');
|
|
pages.set(page.title, page);
|
|
}
|
|
return records.map(r => {
|
|
let title = r.latin;
|
|
const seen = new Set();
|
|
while (aliases.has(title)) {
|
|
requireValue(!seen.has(title) && seen.size < 5, 'Cyclic or excessive title redirects.');
|
|
seen.add(title); title = aliases.get(title);
|
|
}
|
|
const p = pages.get(title);
|
|
requireValue(p && p.ns === 0 && !('invalid' in p) &&
|
|
!Object.hasOwn(p.pageprops || {}, 'disambiguation') && p.contentmodel === 'wikitext' &&
|
|
p.pagelanguage === 'en', `Unverified species article: ${r.latin}.`);
|
|
const url = verifiedArticleURL(p.canonicalurl, title);
|
|
if (p.missing === true) {
|
|
requireValue(!p.pageid && !p.revisions && title === r.latin, 'Ambiguous missing article.');
|
|
return missingEntry(r, title, url, retrieved);
|
|
}
|
|
requireValue(!('missing' in p), 'Unexpected missing marker.');
|
|
const pageid = positiveInteger(p.pageid, 'page ID');
|
|
requireValue(Array.isArray(p.revisions) && p.revisions.length === 1, 'Expected one source revision.');
|
|
const rev = p.revisions[0], revision = positiveInteger(rev.revid, 'revision');
|
|
requireValue(p.lastrevid === revision, 'Revision mismatch; retry maintenance later.');
|
|
timestamp(rev.timestamp);
|
|
requireValue(Date.parse(rev.timestamp) <= Date.parse(retrieved), 'Source revision is in the future.');
|
|
const copied = excerpt(p.extract);
|
|
return { id: r.id, latin: r.latin, title, url, pageid, revision,
|
|
revision_url: `${ORIGIN}/w/index.php?oldid=${revision}`,
|
|
history_url: `${ORIGIN}/w/index.php?title=${wikiTitle(title)}&action=history`,
|
|
revision_timestamp: rev.timestamp, retrieved, ...copied,
|
|
kind: 'wikipedia_intro', excerpt: true, attribution: `Wikipedia contributors, "${title}"`,
|
|
license: LICENSE, license_url: LICENSE_URL,
|
|
changes: copied.truncated ? 'Plaintext introduction; shortened at a text boundary. No translation.'
|
|
: 'Plaintext introduction only; article body omitted. No translation.' };
|
|
});
|
|
}
|
|
|
|
export async function fetchEntries(records, { fetchImpl = globalThis.fetch, now = () => new Date() } = {}) {
|
|
validateCatalogue(records);
|
|
const batches = [];
|
|
for (let i = 0; i < records.length; i += LIMITS.batch) batches.push(records.slice(i, i + LIMITS.batch));
|
|
const controller = new AbortController();
|
|
const signal = AbortSignal.any([controller.signal, AbortSignal.timeout(LIMITS.totalMs)]);
|
|
const results = new Array(batches.length);
|
|
let cursor = 0;
|
|
async function worker() {
|
|
while (!signal.aborted) {
|
|
const i = cursor++;
|
|
if (i >= batches.length) return;
|
|
const batch = batches[i];
|
|
const data = await fetchBatch(batch.map(r => r.latin), { fetchImpl, signal });
|
|
results[i] = entriesFromResponse(batch, data, now().toISOString());
|
|
}
|
|
signal.throwIfAborted();
|
|
}
|
|
try { await Promise.all(Array.from({ length: Math.min(LIMITS.concurrency, batches.length) }, worker)); }
|
|
catch (error) { controller.abort(error); throw error; }
|
|
return results.flat();
|
|
}
|
|
|
|
// Every UTF-8 byte is a fixed-width Lua decimal escape (Lua 5.1+). Article text
|
|
// cannot close a quote, inject a comment/long-string delimiter or execute code.
|
|
export function luaString(value) {
|
|
requireValue(typeof value === 'string', 'Expected string.');
|
|
return '"' + [...Buffer.from(value, 'utf8')].map(b => '\\' + String(b).padStart(3, '0')).join('') + '"';
|
|
}
|
|
export function serializeBundle(entries) {
|
|
validateCatalogue(entries);
|
|
return HEADER + 'return {\n' + entries.map(e => `\t[${luaString(e.id)}] = {\n` +
|
|
FIELDS.map(key => {
|
|
const value = e[key];
|
|
const absent = e.kind === 'original_notes' && ['pageid', 'revision', 'revision_url', 'revision_timestamp'].includes(key) && value === undefined;
|
|
requireValue(absent || (['pageid', 'revision'].includes(key) ? Number.isSafeInteger(value) && value > 0
|
|
: ['excerpt', 'truncated'].includes(key) ? typeof value === 'boolean' : typeof value === 'string'), `Invalid bundle field ${key}.`);
|
|
return `\t\t${key} = ${value === undefined ? 'nil' : typeof value === 'string' ? luaString(value) : value},\n`;
|
|
}).join('') + '\t},\n').join('') + '}\n';
|
|
}
|
|
|
|
// This narrow parser reads ONLY our generated data grammar, not general Lua.
|
|
export function parseBundle(source) {
|
|
requireValue(typeof source === 'string' && Buffer.byteLength(source) <= 16 * 1024 * 1024 &&
|
|
source.startsWith(HEADER + 'return {\n') && source.endsWith('}\n'), 'Invalid generated bundle.');
|
|
const decode = s => new TextDecoder('utf-8', { fatal: true }).decode(Buffer.from(
|
|
[...s.matchAll(/\\(\d{3})/g)].map(m => { const n = Number(m[1]); requireValue(n <= 255, 'Invalid Lua byte.'); return n; })));
|
|
const lines = source.slice((HEADER + 'return {\n').length, -2).split('\n');
|
|
lines.pop();
|
|
const entries = [];
|
|
let i = 0;
|
|
while (i < lines.length) {
|
|
const start = /^\t\["((?:\\\d{3})+)"\] = \{$/.exec(lines[i++]);
|
|
requireValue(start, 'Invalid Lua record.');
|
|
const entry = { id: decode(start[1]) };
|
|
for (const key of FIELDS) {
|
|
const m = new RegExp(`^\\t\\t${key} = ("(?:\\\\\\d{3})*"|true|false|nil|[1-9]\\d*),$`).exec(lines[i++]);
|
|
requireValue(m, 'Invalid Lua field.');
|
|
if (m[1] !== 'nil') entry[key] = m[1].startsWith('"') ? decode(m[1]) : m[1] === 'true' ? true : m[1] === 'false' ? false : Number(m[1]);
|
|
}
|
|
requireValue(lines[i++] === '\t},', 'Invalid Lua record end.');
|
|
entries.push(entry);
|
|
}
|
|
validateCatalogue(entries);
|
|
requireValue(serializeBundle(entries) === source, 'Noncanonical generated bundle.');
|
|
return entries;
|
|
}
|
|
|
|
async function readCatalogue() {
|
|
await safePath(CATALOGUE);
|
|
const stat = await fs.stat(CATALOGUE);
|
|
requireValue(stat.isFile() && stat.size <= LIMITS.catalogueBytes, 'Invalid catalogue file.');
|
|
return parseCatalogue(await fs.readFile(CATALOGUE, 'utf8'));
|
|
}
|
|
export async function main(args = process.argv.slice(2)) {
|
|
requireValue(args.length === 1 && ['--refresh', '--check', '--help'].includes(args[0]),
|
|
'Usage: node scripts/wiki-species.mjs --refresh | --check | --help (no URLs/output paths accepted)');
|
|
if (args[0] === '--help') {
|
|
console.log('--refresh: query all 100 species; replace the fixed Lua cache with verified intros or explicit missing-page notes after full validation.\n--check: verify fixed offline cache/catalogue without network.');
|
|
return;
|
|
}
|
|
const records = await readCatalogue();
|
|
if (args[0] === '--check') {
|
|
await safePath(OUTPUT);
|
|
requireValue((await fs.stat(OUTPUT)).size <= 16 * 1024 * 1024, 'Bundle too large.');
|
|
const entries = parseBundle(await fs.readFile(OUTPUT, 'utf8'));
|
|
requireValue(JSON.stringify(entries.map(({ id, latin }) => ({ id, latin }))) === JSON.stringify(records), 'Catalogue/cache mismatch.');
|
|
for (const e of entries) {
|
|
if (e.kind === 'original_notes') {
|
|
timestamp(e.retrieved); verifiedArticleURL(e.url, e.latin);
|
|
requireValue(serializeBundle([e]) === serializeBundle([missingEntry(e, e.latin, e.url, e.retrieved)]), 'Invalid original availability note.');
|
|
continue;
|
|
}
|
|
entriesFromResponse([{ id: e.id, latin: e.latin }], { batchcomplete: true, query: {
|
|
redirects: e.title === e.latin ? [] : [{ from: e.latin, to: e.title }],
|
|
pages: [{ ns: 0, pageid: e.pageid, title: e.title, canonicalurl: e.url, contentmodel: 'wikitext',
|
|
pagelanguage: 'en', lastrevid: e.revision, revisions: [{ revid: e.revision, timestamp: e.revision_timestamp }], extract: e.text }] } }, e.retrieved);
|
|
requireValue(e.kind === 'wikipedia_intro' && e.excerpt === true && e.license === LICENSE && e.license_url === LICENSE_URL &&
|
|
e.revision_url === `${ORIGIN}/w/index.php?oldid=${e.revision}` &&
|
|
e.history_url === `${ORIGIN}/w/index.php?title=${wikiTitle(e.title)}&action=history` &&
|
|
e.attribution === `Wikipedia contributors, "${e.title}"` && Array.from(e.text).length <= LIMITS.characters, 'Invalid cached attribution/excerpt.');
|
|
}
|
|
console.log(`Verified ${entries.length}/${LIMITS.species} species: ${entries.filter(e => e.kind === 'wikipedia_intro').length} Wikipedia introductions, ${entries.filter(e => e.kind === 'original_notes').length} original availability notes; network requests: 0.`);
|
|
return;
|
|
}
|
|
const entries = await fetchEntries(records);
|
|
const source = serializeBundle(entries);
|
|
requireValue(parseBundle(source).length === LIMITS.species, 'Incomplete generated bundle.');
|
|
// Fixed output only; trusted local parents required, consistent with files.mjs.
|
|
// All network/validation work finishes BEFORE opening/truncating the old cache.
|
|
await safePath(OUTPUT);
|
|
const handle = await fs.open(OUTPUT, constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC | (constants.O_NOFOLLOW || 0), 0o644);
|
|
try { await handle.writeFile(source, 'utf8'); } finally { await handle.close(); }
|
|
console.log(`Cached ${entries.length}/${LIMITS.species} species: ${entries.filter(e => e.kind === 'wikipedia_intro').length} Wikipedia introductions, ${entries.filter(e => e.kind === 'original_notes').length} original availability notes (${entries.filter(e => e.truncated).length} shortened); ${Math.ceil(records.length / LIMITS.batch)} API batches, concurrency <=2.`);
|
|
}
|
|
|
|
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
main().catch(error => { console.error(error.message); process.exitCode = 1; });
|
|
}
|