forked from WikiDeck/wikideck
Night 1: env-configurable Wikimedia User-Agent (WIKIDECK_USER_AGENT) #19
@@ -1,10 +1,12 @@
|
||||
# sim-testnet override: publish the fixture on r350's LAN address only (not 0.0.0.0).
|
||||
# WIKIDECK_USER_AGENT identifies the testnet to Wikimedia (UA policy; no personal contact info).
|
||||
# Game-mode env (modes team, #6) is inert until night/modes/hosting merges.
|
||||
services:
|
||||
node-a:
|
||||
ports: !override ["192.168.1.32:3101:3001"]
|
||||
environment:
|
||||
PUBLIC_ORIGIN: http://192.168.1.32:3101
|
||||
WIKIDECK_USER_AGENT: &ua WikiDeck-testnet/0.1 (https://gitea.eom.dev/claude/wikideck; private testnet)
|
||||
WIKIDECK_MODES: classic,trending
|
||||
WIKIDECK_GAME_OPERATOR: Testnet A
|
||||
WIKIDECK_GAME_PEERS: http://192.168.1.32:3102,http://192.168.1.32:3103
|
||||
@@ -12,6 +14,7 @@ services:
|
||||
ports: !override ["192.168.1.32:3102:3001"]
|
||||
environment:
|
||||
PUBLIC_ORIGIN: http://192.168.1.32:3102
|
||||
WIKIDECK_USER_AGENT: *ua
|
||||
WIKIDECK_MODES: draft
|
||||
WIKIDECK_GAME_OPERATOR: Testnet B
|
||||
WIKIDECK_GAME_PEERS: http://192.168.1.32:3101,http://192.168.1.32:3103
|
||||
@@ -19,6 +22,7 @@ services:
|
||||
ports: !override ["192.168.1.32:3103:3001"]
|
||||
environment:
|
||||
PUBLIC_ORIGIN: http://192.168.1.32:3103
|
||||
WIKIDECK_USER_AGENT: *ua
|
||||
WIKIDECK_MODES: classic,trending,draft
|
||||
WIKIDECK_GAME_OPERATOR: Testnet C
|
||||
WIKIDECK_GAME_PEERS: http://192.168.1.32:3101,http://192.168.1.32:3102
|
||||
|
||||
@@ -13,7 +13,8 @@ const blocks = Number(process.argv[2] ?? 45), workers = Number(process.argv[3] ?
|
||||
if (!(workers >= 1 && workers <= 8)) throw new Error('1..8 workers');
|
||||
const NODES = ['http://node-a:3001', 'http://node-b:3001', 'http://node-c:3001'];
|
||||
const WALLET_NAMES = ['alice', 'bob', 'carol', 'dave'];
|
||||
const PASSWORD = 'wikideck-night-testnet';
|
||||
// PUBLIC, TESTNET-ONLY password for the disposable wallets' browser backups (posted on #4). Never reuse.
|
||||
const PASSWORD = process.env.TESTNET_WALLET_PASSWORD ?? 'wikideck-night-testnet';
|
||||
const STATE = process.env.STATE_DIR ?? '/state';
|
||||
const dir = `${STATE}/wallets`;
|
||||
mkdirSync(dir, { recursive: true });
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { z } from 'zod';
|
||||
import { ENRICHMENT_USER_AGENT, wikimediaUserAgent } from './wikimedia-user-agent.ts';
|
||||
import { deriveStats, type PageEnrichment } from '../shared/enrichment.ts';
|
||||
|
||||
const MAX_BYTES = 1024 * 1024;
|
||||
@@ -101,7 +102,7 @@ export class WikipediaEnrichment {
|
||||
private async json(url: URL): Promise<unknown> {
|
||||
const signal = AbortSignal.timeout(6000);
|
||||
const response = await this.fetcher(url, { signal, redirect: 'error',
|
||||
headers: { Accept: 'application/json', 'User-Agent': 'Wikideck/0.1 (Wikipedia off-chain page enrichment)' } });
|
||||
headers: { Accept: 'application/json', 'User-Agent': wikimediaUserAgent(ENRICHMENT_USER_AGENT) } });
|
||||
const length = response.headers.get('content-length');
|
||||
if (!response.ok || response.redirected ||
|
||||
!/^application\/json(?:\s*;|\s*$)/i.test(response.headers.get('content-type') ?? '') ||
|
||||
|
||||
+2
-1
@@ -1,5 +1,6 @@
|
||||
import { z } from 'zod';
|
||||
import type { Article } from '../shared/protocol.ts';
|
||||
import { ARTICLE_USER_AGENT, wikimediaUserAgent } from './wikimedia-user-agent.ts';
|
||||
|
||||
const pageSchema = z.object({
|
||||
pageid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), ns: z.literal(0),
|
||||
@@ -74,7 +75,7 @@ export class WikipediaArticleSource {
|
||||
generator: 'random', grnnamespace: '0', grnlimit: '10', prop: 'extracts|pageimages|info|categories',
|
||||
exintro: '1', explaintext: '1', exchars: '900', piprop: 'thumbnail', pithumbsize: '800', cllimit: '20' }).toString();
|
||||
const response = await this.fetcher(url, { signal: AbortSignal.timeout(3500), redirect: 'error',
|
||||
headers: { 'User-Agent': 'Wikideck/0.1 (experimental Wikipedia article cards)', Accept: 'application/json' } });
|
||||
headers: { 'User-Agent': wikimediaUserAgent(ARTICLE_USER_AGENT), Accept: 'application/json' } });
|
||||
const data = responseSchema.parse(await readJson(response));
|
||||
for (const rawPage of data.query.pages) {
|
||||
const parsedPage = pageSchema.safeParse(rawPage);
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
// User-Agent for Wikimedia API requests. Wikimedia asks automated clients to identify
|
||||
// themselves with contact information; operators can opt in with WIKIDECK_USER_AGENT.
|
||||
// Unset (or blank), each caller keeps its historical default string unchanged.
|
||||
export const ARTICLE_USER_AGENT = 'Wikideck/0.1 (experimental Wikipedia article cards)';
|
||||
export const ENRICHMENT_USER_AGENT = 'Wikideck/0.1 (Wikipedia off-chain page enrichment)';
|
||||
const MAX_LENGTH = 256;
|
||||
|
||||
export function wikimediaUserAgent(fallback: string, env: Record<string, string | undefined> = process.env): string {
|
||||
const configured = env.WIKIDECK_USER_AGENT?.trim();
|
||||
// Reject header-injection/control characters and oversized values; fall back to the default.
|
||||
if (!configured || configured.length > MAX_LENGTH || !/^[\x20-\x7e]+$/.test(configured)) return fallback;
|
||||
return configured;
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { WikipediaArticleSource } from '../server/wiki.ts';
|
||||
import { WikipediaEnrichment } from '../server/enrichment.ts';
|
||||
import { ARTICLE_USER_AGENT, ENRICHMENT_USER_AGENT, wikimediaUserAgent } from '../server/wikimedia-user-agent.ts';
|
||||
|
||||
function capture() {
|
||||
const agents: string[] = [];
|
||||
const fetcher: typeof fetch = async (_input, init) => {
|
||||
agents.push(new Headers(init?.headers).get('User-Agent') ?? '');
|
||||
return new Response('rate limited', { status: 429, headers: { 'Content-Type': 'text/plain' } });
|
||||
};
|
||||
return { agents, fetcher };
|
||||
}
|
||||
async function agentsFor(env: string | undefined) {
|
||||
const previous = process.env.WIKIDECK_USER_AGENT;
|
||||
if (env === undefined) delete process.env.WIKIDECK_USER_AGENT; else process.env.WIKIDECK_USER_AGENT = env;
|
||||
try {
|
||||
const article = capture(), enrichment = capture();
|
||||
await assert.rejects(new WikipediaArticleSource(article.fetcher).random());
|
||||
await new WikipediaEnrichment({ fetch: enrichment.fetcher }).getFresh(12345).catch(() => {});
|
||||
return { article: article.agents, enrichment: enrichment.agents };
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.WIKIDECK_USER_AGENT; else process.env.WIKIDECK_USER_AGENT = previous;
|
||||
}
|
||||
}
|
||||
|
||||
test('default Wikimedia user agents are unchanged when WIKIDECK_USER_AGENT is unset', async () => {
|
||||
assert.equal(ARTICLE_USER_AGENT, 'Wikideck/0.1 (experimental Wikipedia article cards)');
|
||||
assert.equal(ENRICHMENT_USER_AGENT, 'Wikideck/0.1 (Wikipedia off-chain page enrichment)');
|
||||
const { article, enrichment } = await agentsFor(undefined);
|
||||
assert.ok(article.length > 0 && article.every(a => a === ARTICLE_USER_AGENT));
|
||||
assert.ok(enrichment.length > 0 && enrichment.every(a => a === ENRICHMENT_USER_AGENT));
|
||||
});
|
||||
|
||||
test('WIKIDECK_USER_AGENT overrides both Wikimedia clients', async () => {
|
||||
const ua = 'WikiDeck-testnet/0.1 (https://gitea.eom.dev/claude/wikideck; private testnet)';
|
||||
const { article, enrichment } = await agentsFor(ua);
|
||||
assert.ok(article.length > 0 && article.every(a => a === ua));
|
||||
assert.ok(enrichment.length > 0 && enrichment.every(a => a === ua));
|
||||
});
|
||||
|
||||
test('blank, oversized or control-character overrides fall back to the default', () => {
|
||||
for (const bad of ['', ' ', 'x'.repeat(257), 'bad\r\nX-Injected: 1', 'tab\there'])
|
||||
assert.equal(wikimediaUserAgent('fallback', { WIKIDECK_USER_AGENT: bad }), 'fallback');
|
||||
assert.equal(wikimediaUserAgent('fallback', {}), 'fallback');
|
||||
assert.equal(wikimediaUserAgent('fallback', { WIKIDECK_USER_AGENT: ' Custom/1.0 (x) ' }), 'Custom/1.0 (x)');
|
||||
});
|
||||
Reference in New Issue
Block a user