last-hearth/tests/wiki-species.test.mjs

// Offline only. No test contacts Wikipedia or starts an engine.
import test from 'node:test';
import assert from 'node:assert/strict';
import { readFile } from 'node:fs/promises';
import { setImmediate } from 'node:timers/promises';
import { LIMITS, USER_AGENT, parseCatalogue, apiURL, allowedURL, fetchBatch,
  entriesFromResponse, fetchEntries, excerpt, luaString, serializeBundle, parseBundle, main } from '../scripts/wiki-species.mjs';

const catalogue = parseCatalogue(await readFile(new URL('../mods/hearth/botany.lua', import.meta.url), 'utf8'));
const cached = parseBundle(await readFile(new URL('../mods/hearth/wiki_articles.lua', import.meta.url), 'utf8'));
const retrieved = '2026-11-03T08:00:00.000Z';
const record = catalogue[0];
const intro = 'Quercus robur is a species of flowering plant. This fixture is original test text, not a cached Wikipedia quotation.';
let mockedRequests = 0;
function page(title, changes = {}) {
  return { pageid: 201415, ns: 0, title, extract: intro, canonicalurl: 'https://en.wikipedia.org/wiki/' + encodeURIComponent(title.replaceAll(' ', '_')),
    lastrevid: 12345, revisions: [{ revid: 12345, timestamp: '2026-11-01T10:00:00Z' }],
    contentmodel: 'wikitext', pagelanguage: 'en', ...changes };
}
function payload(pages = [page(record.latin)], changes = {}) {
  return { batchcomplete: true, query: { pages, ...changes } };
}
function jsonResponse(data, headers = {}) {
  return new Response(JSON.stringify(data), { headers: { 'content-type': 'application/json; charset=utf-8', ...headers } });
}
function mock(fn) { return async (...args) => { mockedRequests++; return fn(...args); }; }

test('bounded non-executing Lua parser finds the actual 80 Latin names and IDs', async () => {
  assert.equal(catalogue.length, 80);
  assert.equal(new Set(catalogue.map(r => r.id)).size, 80);
  assert.deepEqual(catalogue.find(r => r.id === 'wood_blewit'), { id: 'wood_blewit', latin: 'Collybia nuda' });
  const source = await readFile(new URL('../mods/hearth/botany.lua', import.meta.url), 'utf8');
  for (const bad of [source.replace('Quercus robur', 'https://127.0.0.1/'),
    source.replace('Quercus robur', 'Quercus robur|User:Private'),
    source.replace('Quercus robur', 'Quercus robur&action=edit'),
    source.replace('add("oak"', 'add("constructor"'),
    source.replace('add("birch"', 'add("oak"'),
    source.replace('"Betula pendula"', '"Quercus robur"'),
    source.replace('add("oak",', 'add(variable,'), source + '\nadd("extra","Extra","Abies alba",',
    source.repeat(10), 'os.execute("not executed")']) assert.throws(() => parseCatalogue(bad));
  assert.equal(parseCatalogue(source + '\nos.execute("never evaluated")').length, 80);
});

test('URL construction rejects URL/query/title injection before any request', async () => {
  for (const name of ['https://evil.example/', '//127.0.0.1/', '../Quercus robur',
    'Quercus robur|Betula pendula', 'Quercus robur&action=edit', 'Quercus robur#x',
    'Quercus robur\n', 'Quercus%20robur', 'User:Somebody', 'Quercus robur?x=1']) {
    assert.throws(() => apiURL([name]));
    await assert.rejects(fetchBatch([name], { fetchImpl: () => { throw new Error('must not fetch'); } }), /Invalid Latin/);
  }
  assert.throws(() => apiURL([]));
  assert.throws(() => apiURL(Array(11).fill('Quercus robur')));
  const u = apiURL(['Quercus robur', 'Collybia nuda']);
  assert.equal(u.origin, 'https://en.wikipedia.org');
  assert.equal(u.pathname, '/w/api.php');
  assert.equal(u.searchParams.get('titles'), 'Quercus robur|Collybia nuda');
  assert.equal(u.searchParams.get('explaintext'), '1');
  assert.equal(u.searchParams.get('exintro'), '1');
  assert.equal(u.searchParams.get('rvprop'), 'ids|timestamp');
});

test('strict HTTPS allowlist excludes credentials, ports, fragments and lookalike hosts', () => {
  for (const u of ['http://en.wikipedia.org/wiki/X', 'https://en.wikipedia.org.evil.example/',
    'https://en.wikipedia.org@evil.example/', 'https://user:pass@en.wikipedia.org/',
    'https://en.wikipedia.org:8443/', 'https://en.wikipedia.org/#x',
    'https://127.0.0.1/', 'https://en.m.wikipedia.org/', 'file:///etc/passwd',
    'https://evil.example/?next=https://en.wikipedia.org']) assert.throws(() => allowedURL(u));
  assert.equal(allowedURL('https://en.wikipedia.org/wiki/Quercus_robur').hostname, 'en.wikipedia.org');
});

test('built-in fetch contract uses manual redirects, omitted credentials and bounded signals', async () => {
  const result = await fetchBatch([record.latin], { fetchImpl: mock((url, options) => {
    assert.equal(new URL(url).searchParams.get('titles'), record.latin);
    assert.equal(options.redirect, 'manual');
    assert.equal(options.credentials, 'omit');
    assert.equal(options.referrerPolicy, 'no-referrer');
    assert.equal(options.headers['User-Agent'], USER_AGENT);
    assert.match(USER_AGENT, /LastHearth.*offline/i);
    assert.equal(Object.keys(options.headers).length, 2);
    assert.ok(options.signal instanceof AbortSignal);
    return jsonResponse(payload());
  }) });
  assert.equal(result.query.pages[0].pageid, 201415);
  assert.equal(LIMITS.requestMs, 15000);
  assert.equal(LIMITS.totalMs, 90000);
});

test('redirect SSRF and query changes are rejected without a second request', async () => {
  for (const location of ['https://evil.example/', 'http://en.wikipedia.org/w/api.php',
    '//127.0.0.1/admin', 'https://en.wikipedia.org@127.0.0.1/',
    '/wiki/Quercus_robur', '/w/api.php?action=edit', 'https://u:p@en.wikipedia.org/w/api.php']) {
    let calls = 0;
    await assert.rejects(fetchBatch([record.latin], { fetchImpl: mock(() => {
      calls++; return new Response(null, { status: 302, headers: { location } });
    }) }));
    assert.equal(calls, 1);
  }
});

test('same allowed endpoint redirects are bounded to two, sharing one timeout', async () => {
  let calls = 0;
  const signals = new Set();
  await assert.rejects(fetchBatch([record.latin], { fetchImpl: mock((url, options) => {
    calls++; signals.add(options.signal);
    return new Response(null, { status: 307, headers: { location: url } });
  }) }), /Too many/);
  assert.equal(calls, 3);
  assert.equal(signals.size, 1);
});

test('unexpected automatic redirect response URL is rejected', async () => {
  await assert.rejects(fetchBatch([record.latin], { fetchImpl: mock(() => {
    const r = jsonResponse(payload());
    Object.defineProperty(r, 'url', { value: 'https://evil.example/' }); return r;
  }) }), /Only credential-free/);
});

test('HTTP errors, API warnings, malformed JSON and HTML are failures, not notes', async () => {
  for (const r of [new Response('rate limit', { status: 429 }), new Response('error', { status: 503 }),
    new Response('<html>error</html>', { headers: { 'content-type': 'text/html' } }),
    new Response('{bad', { headers: { 'content-type': 'application/json' } }),
    new Response(new Uint8Array([0xff]), { headers: { 'content-type': 'application/json' } })]) {
    await assert.rejects(fetchBatch([record.latin], { fetchImpl: mock(() => r) }));
  }
  for (const bad of [{ error: { code: 'maxlag' } }, { ...payload(), warnings: { extracts: 'failure' } },
    { ...payload(), batchcomplete: false }, { query: { pages: [] } }]) {
    assert.throws(() => entriesFromResponse([record], bad, retrieved));
  }
});

test('advertised and streamed response limits both enforce 2 MiB', async () => {
  await assert.rejects(fetchBatch([record.latin], { fetchImpl: mock(() => jsonResponse(payload(), {
    'content-length': String(LIMITS.responseBytes + 1) })) }), /byte limit/);
  let cancelled = false;
  const stream = new ReadableStream({ start(c) { c.enqueue(new Uint8Array(LIMITS.responseBytes + 1)); }, cancel() { cancelled = true; } });
  await assert.rejects(fetchBatch([record.latin], { fetchImpl: mock(() => new Response(stream, {
    headers: { 'content-type': 'application/json' } })) }), /byte limit/);
  assert.equal(cancelled, true);
});

test('aborting a stalled fetch or stalled response body is bounded and cancels body', async () => {
  const a = new AbortController();
  const request = fetchBatch([record.latin], { signal: a.signal, fetchImpl: mock(() => new Promise(() => {})) });
  a.abort(new Error('bounded request')); await assert.rejects(request, /bounded request/);
  const b = new AbortController(); let cancelled = false;
  const body = new ReadableStream({ pull() { b.abort(new Error('bounded body')); }, cancel() { cancelled = true; } }, { highWaterMark: 0 });
  await assert.rejects(fetchBatch([record.latin], { signal: b.signal, fetchImpl: mock(() => new Response(body, {
    headers: { 'content-type': 'application/json' } })) }), /bounded body/);
  assert.equal(cancelled, true);
});

test('source title must match exact Latin or a returned redirect chain; no guessed aliases', () => {
  const r = { id: 'wood_blewit', latin: 'Collybia nuda' };
  const data = payload([page('Lepista nuda')], { redirects: [{ from: 'Collybia nuda', to: 'Lepista nuda' }] });
  const [e] = entriesFromResponse([r], data, retrieved);
  assert.equal(e.latin, 'Collybia nuda'); assert.equal(e.title, 'Lepista nuda');
  assert.equal(e.url, 'https://en.wikipedia.org/wiki/Lepista_nuda');
  assert.equal(e.revision_url, 'https://en.wikipedia.org/w/index.php?oldid=12345');
  assert.match(e.history_url, /title=Lepista_nuda&action=history$/);
  assert.equal(e.attribution, 'Wikipedia contributors, "Lepista nuda"');
  assert.equal(e.license, 'CC BY-SA 4.0');
  assert.throws(() => entriesFromResponse([r], payload([page('Lepista nuda')]), retrieved), /Unverified/);
  for (const redirects of [[{ from: r.latin, to: 'Lepista nuda', tofragment: 'Species' }],
    [{ from: r.latin, to: 'Lepista nuda' }, { from: 'Lepista nuda', to: r.latin }],
    [{ from: r.latin, to: 'Lepista nuda' }, { from: r.latin, to: 'Clitocybe nuda' }]]) {
    assert.throws(() => entriesFromResponse([r], payload([page('Lepista nuda')], { redirects }), retrieved));
  }
});

test('disambiguation, non-mainspace, language, title and canonical URL mismatch fail closed', () => {
  for (const changes of [{ pageprops: { disambiguation: '' } }, { ns: 1 }, { invalid: true },
    { pagelanguage: 'fr' }, { contentmodel: 'json' }, { title: 'Quercus' },
    { canonicalurl: 'https://evil.example/wiki/Quercus_robur' },
    { canonicalurl: 'https://en.wikipedia.org/wiki/Betula_pendula' },
    { canonicalurl: 'https://en.wikipedia.org/wiki/Quercus_robur?x=1' },
    { canonicalurl: 'https://en.wikipedia.org/wiki/Quercus_robur#section' }]) {
    assert.throws(() => entriesFromResponse([record], payload([page(record.latin, changes)]), retrieved));
  }
});

test('page IDs and revisions must be positive finite safe integers with matching current revision', () => {
  for (const n of [0, -1, 1.5, NaN, Infinity, '12345', null, Number.MAX_SAFE_INTEGER + 1]) {
    assert.throws(() => entriesFromResponse([record], payload([page(record.latin, { pageid: n })]), retrieved));
    assert.throws(() => entriesFromResponse([record], payload([page(record.latin, { revisions: [{ revid: n, timestamp: '2026-11-01T10:00:00Z' }] })]), retrieved));
  }
  for (const changes of [{ lastrevid: 54321 }, { revisions: [] }, { revisions: [{ revid: 12345, timestamp: 'not a date' }] },
    { revisions: [{ revid: 12345, timestamp: '2026-02-31T10:00:00Z' }] },
    { revisions: [{ revid: 12345, timestamp: '2027-01-01T10:00:00Z' }] }]) {
    assert.throws(() => entriesFromResponse([record], payload([page(record.latin, changes)]), retrieved));
  }
});

test('only an explicit missing species page becomes original_notes, never a fake excerpt/revision', () => {
  const r = { id: 'old_mans_beard', latin: 'Usnea subfloridana' };
  const missing = { ns: 0, title: r.latin, missing: true, canonicalurl: 'https://en.wikipedia.org/wiki/Usnea_subfloridana', contentmodel: 'wikitext', pagelanguage: 'en' };
  const [e] = entriesFromResponse([r], payload([missing]), retrieved);
  assert.equal(e.kind, 'original_notes'); assert.equal(e.excerpt, false);
  assert.equal(e.revision, undefined); assert.equal(e.pageid, undefined); assert.equal(e.revision_url, undefined);
  assert.match(e.text, /not a Wikipedia excerpt/); assert.equal(e.license, 'CC0 1.0');
  assert.throws(() => entriesFromResponse([r], payload([]), retrieved));
  assert.throws(() => entriesFromResponse([r], payload([{ ...missing, missing: 'true' }]), retrieved));
  assert.deepEqual(parseBundle(serializeBundle([e])), [e]);
});

test('plaintext excerpt cap preserves exact Unicode prefix on a boundary and labels shortening', () => {
  const full = 'A botanical observation.\n' + 'A longer sentence about a species. '.repeat(500);
  const e = excerpt(full);
  assert.ok(Array.from(e.text).length <= 8000); assert.equal(e.truncated, true);
  assert.ok(full.startsWith(e.text)); assert.match(e.text, /[.!?]$/);
  const unicode = 'Species ' + String.fromCodePoint(0x1f333).repeat(1000) + ' ecology. '.repeat(1000);
  const u = excerpt(unicode);
  assert.ok(Array.from(u.text).length <= 8000 && !u.text.includes('\ufffd'));
  assert.equal(excerpt(intro).text, intro); assert.equal(excerpt(intro).truncated, false);
  for (const bad of ['', '<script>alert(1)</script>' + intro, intro + '\0', intro + ' [[article]]']) assert.throws(() => excerpt(bad));
});

test('untrusted Wikipedia text is exclusively UTF-8 byte literals, never executable Lua', () => {
  const attack = 'Species text " }; os.execute("bad"); --\n]=] [[ return function() end \\123 ' + String.fromCodePoint(0xe9, 0x1f333);
  const entry = entriesFromResponse([record], payload(), retrieved)[0];
  // Exercise the serializer directly even for text rejected by plaintext policy.
  entry.text = attack;
  const source = serializeBundle([entry]);
  assert.match(luaString(attack), /^"(?:\\\d{3})*"$/);
  assert.ok(!source.includes('os.execute') && !source.includes('return function'));
  assert.equal(parseBundle(source)[0].text, attack);
  assert.equal(serializeBundle(parseBundle(source)), source);
  assert.throws(() => parseBundle(source.replace('return {', 'os.execute("bad")\nreturn {')));
  assert.throws(() => parseBundle(source.replace('\\083', '\\999')));
  assert.throws(() => serializeBundle([{ ...entry, revision: Infinity }]));
  assert.throws(() => serializeBundle([{ ...entry, revision: '12345' }]));
});

test('80 titles use exactly eight mocked batches at concurrency <=2 and stable catalogue order', async () => {
  let active = 0, peak = 0, calls = 0;
  const seen = [];
  const entries = await fetchEntries(catalogue, { now: () => new Date(retrieved), fetchImpl: mock(async url => {
    calls++; active++; peak = Math.max(peak, active);
    const names = new URL(url).searchParams.get('titles').split('|'); seen.push(...names);
    assert.ok(names.length <= 10); await setImmediate(); active--;
    return jsonResponse(payload(names.map(name => page(name))));
  }) });
  assert.equal(calls, 8); assert.equal(peak, 2);
  assert.deepEqual(seen, catalogue.map(r => r.latin));
  assert.deepEqual(entries.map(r => r.id), catalogue.map(r => r.id));
  assert.equal(entries.length, 80);
  for (const records of [[], [...catalogue, { id: 'extra', latin: 'Abies alba' }]]) {
    await assert.rejects(fetchEntries(records, { fetchImpl: () => { throw new Error('must not fetch'); } }), /scope/);
  }
});

test('batch failure aborts other workers and never launches the remaining catalogue', async () => {
  let calls = 0, otherSignal;
  await assert.rejects(fetchEntries(catalogue, { fetchImpl: mock((url, options) => {
    calls++;
    if (calls === 1) return new Response('error', { status: 503 });
    otherSignal = options.signal; return new Promise(() => {});
  }) }), /HTTP 503/);
  assert.equal(calls, 2); assert.equal(otherSignal.aborted, true);
});

test('shipped bundle contains 78 real intros and two honest missing-page notes for all 80 species', () => {
  assert.equal(cached.length, 80);
  assert.deepEqual(cached.map(({ id, latin }) => ({ id, latin })), catalogue);
  assert.equal(cached.filter(e => e.kind === 'wikipedia_intro').length, 78);
  assert.deepEqual(cached.filter(e => e.kind === 'original_notes').map(e => e.latin), ['Usnea subfloridana', 'Cladonia chlorophaea']);
  for (const e of cached) {
    assert.ok(Array.from(e.text).length <= 8000);
    assert.ok(!/<\/?[A-Za-z][^>]*>/.test(e.text));
    if (e.kind === 'wikipedia_intro') {
      assert.equal(e.license, 'CC BY-SA 4.0');
      assert.ok(Number.isSafeInteger(e.pageid) && e.pageid > 0);
      assert.ok(Number.isSafeInteger(e.revision) && e.revision > 0);
      assert.ok(e.history_url.endsWith('&action=history'));
      assert.ok(e.revision_url.endsWith('?oldid=' + e.revision));
      assert.match(e.attribution, /^Wikipedia contributors/);
    }
  }
  assert.equal(cached.find(e => e.id === 'meadow_fescue').title, 'Lolium pratense');
  assert.equal(cached.find(e => e.id === 'wood_blewit').title, 'Collybia nuda');
});

test('import, help, invalid CLI arguments and offline --check perform zero network requests', async () => {
  const original = globalThis.fetch;
  let network = 0;
  globalThis.fetch = () => { network++; throw new Error('Offline test attempted network'); };
  try {
    await import('../scripts/wiki-species.mjs?offline-import-test');
    await main(['--help']);
    for (const args of [[], ['--refresh', '--url', 'https://evil.example/'], ['https://evil.example/'], ['--output=../other.lua']]) {
      await assert.rejects(main(args), /Usage/);
    }
    await main(['--check']);
    assert.equal(network, 0);
  } finally { globalThis.fetch = original; }
});

test('CLI refresh HTTP failure preserves the existing cache byte for byte', async () => {
  const url = new URL('../mods/hearth/wiki_articles.lua', import.meta.url);
  const before = await readFile(url), original = globalThis.fetch;
  let calls = 0;
  globalThis.fetch = mock(() => { calls++; return new Response('unavailable', { status: 503 }); });
  try { await assert.rejects(main(['--refresh']), /HTTP 503/); }
  finally { globalThis.fetch = original; }
  assert.ok(calls > 0 && calls <= 2);
  assert.deepEqual(await readFile(url), before);
});

test.after(() => console.log(`WIKI_SPECIES_OFFLINE_COUNTS: 21 tests; real network requests: 0; mocked requests: ${mockedRequests}.`));