wikideck/tests/enrichment.test.ts

import test from 'node:test';
import assert from 'node:assert/strict';
import { WikipediaEnrichment } from '../server/enrichment.ts';
import { deriveStats } from '../shared/enrichment.ts';

const JANUARY = new Date('2025-01-15T23:59:59Z');
const MONTHS = ['2024-07', '2024-08', '2024-09', '2024-10', '2024-11', '2024-12'];
function metadata(pageId = 736, title = 'Albert Einstein') {
  return { query: { pages: [{ pageid: pageId, ns: 0, title, lastrevid: 1234, length: 100_000,
    description: '<script>alert("plain text")</script>', pageprops: { wikibase_item: 'Q937' },
    categories: [{ ns: 14, title: 'Category:Physicists' }], langlinks: [{ lang: 'de', title: 'Albert Einstein' }] }] },
  continue: { clcontinue: '736|next', llcontinue: '736|next', continue: '||' } };
}
function views(months = MONTHS, counts = [10, 20, 30, 40, 50, 100]) {
  return { items: months.map((month, index) => ({ timestamp: `${month.replace('-', '')}0100`,
    views: counts[index], project: 'en.wikipedia', access: 'all-access', agent: 'user', granularity: 'monthly' })) };
}
function json(value: unknown) { return new Response(JSON.stringify(value), { headers: { 'Content-Type': 'application/json; charset=utf-8' } }); }
function mockFetch(handler: (url: URL, init: RequestInit) => Response | Promise<Response>) {
  const calls: { url: URL; init: RequestInit }[] = [];
  const fetcher: typeof fetch = async (input, init) => {
    const url = new URL(String(input));
    calls.push({ url, init: init! });
    return handler(url, init!);
  };
  return { fetcher, calls };
}
function fixture(options: { now?: () => Date; ttlMs?: number; maxEntries?: number } = {}) {
  const mock = mockFetch(url => json(url.hostname === 'en.wikipedia.org'
    ? metadata(Number(url.searchParams.get('pageids'))) : views()));
  return { ...mock, service: new WikipediaEnrichment({ fetch: mock.fetcher, now: () => JANUARY, ...options }) };
}

test('fixed HTTPS sources, encoded title, hidden categories, lower bounds and plain strings', async () => {
  const title = 'A / B? #é%';
  const mock = mockFetch(url => json(url.hostname === 'en.wikipedia.org'
    ? metadata(736, title) : { items: views().items.reverse() }));
  const service = new WikipediaEnrichment({ fetch: mock.fetcher, now: () => JANUARY });
  const result = await service.get(736);
  assert.equal(mock.calls.length, 2);
  const query = mock.calls[0].url;
  assert.equal(query.origin, 'https://en.wikipedia.org');
  assert.equal(query.pathname, '/w/api.php');
  assert.equal(query.searchParams.get('pageids'), '736');
  assert.equal(query.searchParams.get('formatversion'), '2');
  assert.equal(query.searchParams.get('prop'), 'info|categories|pageprops|description|langlinks');
  assert.equal(query.searchParams.get('clshow'), '!hidden');
  assert.equal(query.searchParams.get('cllimit'), '50');
  assert.equal(query.searchParams.get('lllimit'), '500');
  const rest = mock.calls[1].url;
  assert.equal(rest.origin, 'https://wikimedia.org');
  assert.equal(rest.pathname, `/api/rest_v1/metrics/pageviews/per-article/en.wikipedia.org/all-access/user/${encodeURIComponent(title.replace(/ /g, '_'))}/monthly/2024070100/2024120100`);
  for (const { init } of mock.calls) {
    assert.equal(init.redirect, 'error');
    assert.ok(init.signal instanceof AbortSignal);
    assert.equal(new Headers(init.headers).get('accept'), 'application/json');
    assert.match(new Headers(init.headers).get('user-agent')!, /Wikideck/);
    assert.equal(new Headers(init.headers).has('origin'), false); // Server-side requests need no CORS workaround.
  }
  assert.equal(result.title, title);
  assert.equal(result.description, '<script>alert("plain text")</script>');
  assert.equal(result.wikidataId, 'Q937');
  assert.equal(result.revisionId, 1234);
  assert.equal(result.pageBytes, 100_000);
  assert.deepEqual(result.categories, ['Physicists']);
  assert.equal(result.categoriesTruncated, true);
  assert.equal(result.languageCount, 1);
  assert.equal(result.languagesTruncated, true);
  assert.equal(result.fetchedAt, JANUARY.toISOString());
  assert.deepEqual(result.pageviews.months.map(item => item.month), MONTHS);
  assert.equal(result.pageviews.total, 250);
  assert.equal(result.pageviews.dailyAverage, 100 / 31);
  assert.equal(result.pageviews.trendPercent, 100);
  assert.deepEqual(result.stats, deriveStats(100_000, 1, 100));
});

test('UTC completed-month windows cross December/January and use leap February day counts', async () => {
  for (const [date, months, days] of [
    ['2024-03-01T00:01:00Z', ['2023-09', '2023-10', '2023-11', '2023-12', '2024-01', '2024-02'], 29],
    ['2025-03-01T00:00:00Z', ['2024-09', '2024-10', '2024-11', '2024-12', '2025-01', '2025-02'], 28],
    ['2025-01-01T00:01:00+01:00', ['2024-06', '2024-07', '2024-08', '2024-09', '2024-10', '2024-11'], 30],
  ] as const) {
    const mock = mockFetch(url => json(url.hostname === 'en.wikipedia.org' ? metadata() : views([...months])));
    const result = await new WikipediaEnrichment({ fetch: mock.fetcher, now: () => new Date(date) }).get(736);
    assert.deepEqual(result.pageviews.months.map(item => item.month), months);
    assert.equal(result.pageviews.dailyAverage, 100 / days);
  }
});

test('missing optional metadata has explicit defaults, not invented descriptions or counts', async () => {
  const mock = mockFetch(url => json(url.hostname === 'en.wikipedia.org'
    ? { query: { pages: [{ pageid: 736, ns: 0, title: 'A', lastrevid: 1, length: 0 }] } } : views()));
  const result = await new WikipediaEnrichment({ fetch: mock.fetcher, now: () => JANUARY }).get(736);
  assert.equal(result.description, null);
  assert.equal(result.wikidataId, null);
  assert.equal(result.languageCount, 0);
  assert.equal(result.languagesTruncated, false);
  assert.equal(result.categoriesTruncated, false);
  assert.deepEqual(result.categories, []);
});

test('reject invalid IDs before fetching and malformed, missing, cross-page or non-article metadata', async () => {
  const { service, calls } = fixture();
  for (const id of [0, -1, 1.2, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1, '736' as unknown as number]) {
    await assert.rejects(service.get(id), /Invalid Wikipedia page ID/);
  }
  assert.equal(calls.length, 0);
  const invalid = [
    { pageid: 737 }, { ns: 14 }, { missing: true }, { invalid: true }, { lastrevid: -1 },
    { length: Number.MAX_SAFE_INTEGER + 1 }, { title: 'a'.repeat(4097) },
    { categories: Array.from({ length: 51 }, () => ({ ns: 14, title: 'Category:A' })) },
    { langlinks: Array.from({ length: 501 }, () => ({ lang: 'en' })) },
    { pageprops: { wikibase_item: 'https://evil.example' } },
  ];
  for (const patch of invalid) {
    const raw = metadata();
    Object.assign(raw.query.pages[0], patch);
    const mock = mockFetch(() => json(raw));
    await assert.rejects(new WikipediaEnrichment({ fetch: mock.fetcher }).get(736), /unavailable/);
    assert.equal(mock.calls.length, 1);
  }
  for (const raw of [{}, { query: { pages: [] } }, { query: { pages: [metadata().query.pages[0], metadata().query.pages[0]] } }]) {
    const mock = mockFetch(() => json(raw));
    await assert.rejects(new WikipediaEnrichment({ fetch: mock.fetcher }).get(736));
  }
});

test('pageviews failure, missing months, duplicates and invalid values retain metadata as unavailable', async () => {
  const failures: (() => Response)[] = [
    () => new Response('', { status: 503 }), () => json({ items: [] }),
    () => json({ items: views().items.slice(1) }),
    () => json({ items: [...views().items.slice(0, 5), views().items[0]] }),
    () => json(views(['2024-06', ...MONTHS.slice(1)])),
    ...[-1, 0.5, Number.MAX_SAFE_INTEGER + 1].map(value => () => json(views(MONTHS, [value, 0, 0, 0, 0, 0]))),
    () => json(views(MONTHS, Array(6).fill(Number.MAX_SAFE_INTEGER))),
    () => json({ items: views().items.map(item => ({ ...item, agent: 'spider' })) }),
    () => json({ items: views().items.map(item => ({ ...item, timestamp: '2024120200' })) }),
    () => { throw new Error('Network failure'); },
  ];
  for (const failure of failures) {
    const mock = mockFetch(url => url.hostname === 'en.wikipedia.org' ? json(metadata()) : failure());
    const result = await new WikipediaEnrichment({ fetch: mock.fetcher, now: () => JANUARY }).get(736);
    assert.equal(result.title, 'Albert Einstein');
    assert.deepEqual(result.pageviews, { status: 'unavailable', months: [], latestMonth: '2024-12',
      total: null, dailyAverage: null, trendPercent: null });
    assert.equal(result.stats.attention, null);
    assert.ok(result.stats.depth > 0);
  }
});

test('six real zero months remain available; zero prior month has no trend', async () => {
  const mock = mockFetch(url => json(url.hostname === 'en.wikipedia.org' ? metadata() : views(MONTHS, Array(6).fill(0))));
  const result = await new WikipediaEnrichment({ fetch: mock.fetcher, now: () => JANUARY }).get(736);
  assert.equal(result.pageviews.status, 'available');
  assert.equal(result.pageviews.total, 0);
  assert.equal(result.pageviews.dailyAverage, 0);
  assert.equal(result.pageviews.trendPercent, null);
  assert.equal(result.stats.attention, 0);
});

test('logarithmic scores are rounded, capped, nullable and reject invalid inputs', () => {
  assert.deepEqual(deriveStats(0, 0, null), { attention: null, depth: 0, reach: 0 });
  assert.deepEqual(deriveStats(999, 9, 99), { attention: 33, depth: 50, reach: 33 });
  assert.deepEqual(deriveStats(Number.MAX_SAFE_INTEGER, 1e9, 1e15), { attention: 100, depth: 100, reach: 100 });
  for (const value of [-1, NaN, Infinity]) {
    assert.throws(() => deriveStats(value, 0, 0), RangeError);
    assert.throws(() => deriveStats(0, value, 0), RangeError);
    assert.throws(() => deriveStats(0, 0, value), RangeError);
  }
});

test('six-hour TTL, deduplication and mutation isolation', async () => {
  let now = JANUARY.getTime();
  const { service, calls } = fixture({ now: () => new Date(now) });
  const [first, second] = await Promise.all([service.get(736), service.get(736)]);
  assert.equal(calls.length, 2);
  first.categories.push('not cached');
  second.pageviews.months[0].views = 999;
  assert.deepEqual((await service.get(736)).categories, ['Physicists']);
  assert.equal((await service.get(736)).pageviews.months[0].views, 10);
  now += 6 * 60 * 60 * 1000 - 1;
  await service.get(736);
  assert.equal(calls.length, 2);
  now++;
  await service.get(736);
  assert.equal(calls.length, 4);
});

test('fresh enrichment bypasses the display cache for mint-time snapshots', async () => {
  const { service, calls } = fixture();
  await service.get(736);
  const pinned = await service.getFresh(736);
  assert.equal(calls.length, 4); // Each lookup makes one metadata and one pageview request.
  assert.equal(pinned.pageId, 736);
  assert.equal(pinned.pageviews.status, 'available');
});

test('LRU bound and configured TTL are enforced', async () => {
  let now = JANUARY.getTime();
  const { service, calls } = fixture({ maxEntries: 2, ttlMs: 100, now: () => new Date(now) });
  await service.get(1); await service.get(2); await service.get(1); await service.get(3);
  assert.equal(calls.length, 6);
  await service.get(1);
  assert.equal(calls.length, 6);
  await service.get(2);
  assert.equal(calls.length, 8);
  now += 100;
  await service.get(2);
  assert.equal(calls.length, 10);
  for (const options of [{ maxEntries: 129 }, { maxEntries: 0 }, { ttlMs: 0 }, { ttlMs: Infinity }]) {
    assert.throws(() => new WikipediaEnrichment(options), RangeError);
  }
});

test('negative cache expires after 30 seconds; unavailable views retry after five minutes', async () => {
  let now = JANUARY.getTime();
  let failPage = true, failViews = true;
  const mock = mockFetch(url => {
    if (url.hostname === 'en.wikipedia.org') return failPage ? json({}) : json(metadata());
    return failViews ? new Response('', { status: 503 }) : json(views());
  });
  const service = new WikipediaEnrichment({ fetch: mock.fetcher, now: () => new Date(now) });
  await assert.rejects(service.get(736));
  failPage = false;
  now += 29_999;
  await assert.rejects(service.get(736));
  assert.equal(mock.calls.length, 1);
  now++;
  assert.equal((await service.get(736)).pageviews.status, 'unavailable');
  failViews = false;
  now += 299_999;
  assert.equal((await service.get(736)).pageviews.status, 'unavailable');
  assert.equal(mock.calls.length, 3);
  now++;
  assert.equal((await service.get(736)).pageviews.status, 'available');
  assert.equal(mock.calls.length, 5);
});

test('cache rolls over at UTC month boundaries even before its TTL', async () => {
  let now = new Date('2024-12-31T23:59:00Z');
  const { service, calls } = fixture({ now: () => now });
  await service.get(736);
  now = new Date('2025-01-01T00:00:00Z');
  const result = await service.get(736);
  assert.equal(calls.length, 4);
  assert.equal(result.pageviews.latestMonth, '2024-12');
  assert.equal(result.pageviews.status, 'available');
});

test('four concurrent jobs, bounded pending requests, busy rejection without a queue and recovery', async () => {
  const releases: (() => void)[] = [];
  const mock = mockFetch(async url => {
    if (url.hostname !== 'en.wikipedia.org') return json(views());
    await new Promise<void>(resolve => releases.push(resolve));
    return json(metadata(Number(url.searchParams.get('pageids'))));
  });
  const service = new WikipediaEnrichment({ fetch: mock.fetcher, now: () => JANUARY });
  const jobs = [1, 2, 3, 4].map(id => service.get(id));
  const duplicate = service.get(1);
  for (let id = 5; id <= 20; id++) await assert.rejects(service.get(id), /busy/);
  assert.equal(mock.calls.length, 4);
  releases.splice(0).forEach(release => release());
  await Promise.all([...jobs, duplicate]);
  assert.equal(mock.calls.length, 8);
  const next = service.get(5);
  releases.splice(0).forEach(release => release());
  await next;
  assert.equal(mock.calls.length, 10);
});

test('JSON-only response contracts, redirects, announced and streamed one-MiB byte limits', async () => {
  const badResponses = [
    () => new Response('{}', { headers: { 'Content-Type': 'text/html' } }),
    () => new Response('{}'),
    () => new Response('{', { headers: { 'Content-Type': 'application/json' } }),
    () => new Response(null, { status: 302, headers: { Location: 'https://evil.example' } }),
    () => new Response('{}', { headers: { 'Content-Type': 'application/json', 'Content-Length': '1048577' } }),
    () => new Response('é'.repeat(600_000), { headers: { 'Content-Type': 'application/json' } }),
    () => { const response = json(metadata()); Object.defineProperty(response, 'redirected', { value: true }); return response; },
  ];
  for (const makeResponse of badResponses) {
    const mock = mockFetch(() => makeResponse());
    await assert.rejects(new WikipediaEnrichment({ fetch: mock.fetcher }).get(736), /unavailable/);
    assert.equal(mock.calls.length, 1);
  }
  let cancelled = false;
  const mock = mockFetch(() => new Response(new ReadableStream<Uint8Array>({
    pull(controller) { controller.enqueue(new Uint8Array(600_000)); },
    cancel() { cancelled = true; },
  }), { headers: { 'Content-Type': 'application/json' } }));
  await assert.rejects(new WikipediaEnrichment({ fetch: mock.fetcher }).get(736));
  assert.equal(cancelled, true);
});

test('every fetch receives a six-second timeout and aborted views retain metadata', async t => {
  const signals: AbortController[] = [];
  t.mock.method(AbortSignal, 'timeout', (ms: number) => {
    assert.equal(ms, 6000);
    const controller = new AbortController();
    signals.push(controller);
    return controller.signal;
  });
  const mock = mockFetch((url, init) => {
    if (url.hostname === 'en.wikipedia.org') return json(metadata());
    return new Promise<Response>((_resolve, reject) => {
      init.signal!.addEventListener('abort', () => reject(init.signal!.reason), { once: true });
      signals[1].abort(new DOMException('Timeout', 'TimeoutError'));
    });
  });
  const result = await new WikipediaEnrichment({ fetch: mock.fetcher, now: () => JANUARY }).get(736);
  assert.equal(signals.length, 2);
  assert.equal(result.pageviews.status, 'unavailable');
  assert.equal(result.pageId, 736);
});