#!/usr/bin/env python3 """Sample random illustrated en.wikipedia articles the way server/wiki.ts selects them and enrich them the way server/enrichment.ts does. Polite: ~4 req/s, sequential, disk cache. Usage: python3 -I card_sample.py [target=320]""" import hashlib, json, os, re, sys, time, urllib.parse, urllib.request, calendar, datetime as dt UA = 'WikiDeckCardDataSampler/0.1 (experimental card-game stat research; python-urllib)' CACHE, OUT = sys.argv[1], sys.argv[2] TARGET = int(sys.argv[3]) if len(sys.argv) > 3 else 320 os.makedirs(CACHE, exist_ok=True) _last = [0.0] def get(url, cache=True): key = os.path.join(CACHE, hashlib.sha1(url.encode()).hexdigest() + '.json') if cache and os.path.exists(key): return json.load(open(key)) if os.environ.get('CACHE_ONLY') and 'wikidata.org' not in url: return {'__missing': True} for attempt in range(6): wait = (1.0 if 'en.wikipedia.org' in url else 0.25) - (time.time() - _last[0]) if wait > 0: time.sleep(wait) _last[0] = time.time() req = urllib.request.Request(url, headers={'User-Agent': UA, 'Accept': 'application/json'}) try: with urllib.request.urlopen(req, timeout=20) as r: data = json.loads(r.read().decode()) break except urllib.error.HTTPError as e: if e.code == 404: data = {'__404': True}; break if e.code == 429 or e.code >= 500: ra = e.headers.get('retry-after'); w = int(ra) + 1 if ra and ra.isdigit() else 5 * (attempt + 1) print('http', e.code, 'sleep', w, file=sys.stderr); time.sleep(min(w, 60)); continue raise except Exception as e: print('retry', attempt, type(e).__name__, e, file=sys.stderr); time.sleep(2 * (attempt + 1)) else: raise RuntimeError('failed ' + url) if cache: json.dump(data, open(key, 'w')) return data def api(params, cache=True): return get('https://en.wikipedia.org/w/api.php?' + urllib.parse.urlencode({'format': 'json', 'formatversion': '2', **params}), cache) def legacy_category(title, cats): t = (title + ' ' + ' '.join(cats)).lower() if re.search(r'astronom|planet|space|galax|moon|star\b', t): return 'Space' if re.search(r'animal|plant|species|ocean|biolog|mollusc', t): return 'Nature' if re.search(r'births|deaths|people|scientist', t): return 'People' if re.search(r'physic|chem|math|science|technolog', t): return 'Science' if re.search(r'histor|ancient|archaeolog|empire', t): return 'History' return 'Culture' def score(v, scale): import math return min(100, round(math.log10(v + 1) / scale * 100)) # 1. random illustrated articles (same filter as wiki.ts: thumbnail + extract >= 40 chars) picked, seen, batches, raw_pages = [], set(), 0, 0 state_file = os.path.join(CACHE, '_random_state.json') if os.path.exists(state_file): picked = json.load(open(state_file)); seen = {p['pageId'] for p in picked} while len(picked) < TARGET: try: d = api({'action': 'query', 'generator': 'random', 'grnnamespace': '0', 'grnlimit': '20', 'prop': 'extracts|pageimages|info|categories', 'exintro': '1', 'explaintext': '1', 'exchars': '900', 'exlimit': '20', 'piprop': 'thumbnail', 'pithumbsize': '800', 'cllimit': '500'}, cache=False) except RuntimeError as e: print(e, file=sys.stderr); time.sleep(5); continue batches += 1 for p in d.get('query', {}).get('pages', []): raw_pages += 1 thumb = (p.get('thumbnail') or {}).get('source', '') ext = (p.get('extract') or '').strip() if not re.match(r'https://(upload|thumb)\.wikimedia\.org/', thumb) or len(ext) < 40 or p['pageid'] in seen: continue seen.add(p['pageid']) picked.append({'pageId': p['pageid'], 'title': p['title'], 'legacyCategory': legacy_category(p['title'], [c['title'] for c in p.get('categories', [])])}) json.dump(picked, open(state_file, 'w')) print(f'random batch {batches}: {len(picked)} illustrated / {raw_pages} raw', file=sys.stderr) picked = picked[:TARGET] # 2. enrichment per article (same params as enrichment.ts) + daily views for drift now = dt.datetime(2026, 11, 16, tzinfo=dt.timezone.utc) months = [] for i in range(6): m = now.month - 6 + i; y = now.year + (m - 1) // 12; m = (m - 1) % 12 + 1 months.append(f'{y:04d}-{m:02d}') # batched metadata (same props as enrichment.ts; per-page cllimit 50 is emulated by truncating) meta = {} ids = [p['pageId'] for p in picked] for i in range(0, len(ids), 50): chunk = ids[i:i + 50]; cont = {}; pages = {} while True: md = api({'action': 'query', 'pageids': '|'.join(map(str, chunk)), 'prop': 'info|categories|pageprops|description|langlinks', 'clshow': '!hidden', 'cllimit': '500', 'lllimit': '500', 'ppprop': 'wikibase_item', **cont}) for pg in md['query']['pages']: cur = pages.setdefault(pg['pageid'], {**pg, 'categories': [], 'langlinks': []}) cur['categories'] += pg.get('categories', []); cur['langlinks'] += pg.get('langlinks', []) if 'continue' not in md: break cont = md['continue'] meta.update(pages) print(f'metadata {len(meta)}/{len(ids)}', file=sys.stderr) cards = [] for n, pk in enumerate(picked): page = meta[pk['pageId']] cont = {'clcontinue': 1} if len(page['categories']) > 50 else {} page['categories'] = page['categories'][:50] langs = len(page.get('langlinks', [])) t = urllib.parse.quote(page['title'].replace(' ', '_'), safe='') daily = get(f'https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia.org/all-access/user/{t}/daily/{months[0].replace("-", "")}01/20261115') if daily.get('__missing'): continue dmap = {it['timestamp'][:8]: it['views'] for it in daily.get('items', [])} pv = {'status': 'unavailable', 'months': [], 'latestMonth': months[5], 'total': None, 'dailyAverage': None, 'trendPercent': None} if dmap: ordered = [] for m in months: y, mo = map(int, m.split('-')); nd = calendar.monthrange(y, mo)[1] ordered.append({'month': m, 'views': sum(dmap.get(f'{y:04d}{mo:02d}{d:02d}', 0) for d in range(1, nd + 1))}) latest, prior = ordered[5]['views'], ordered[4]['views'] nd = calendar.monthrange(*map(int, months[5].split('-')))[1] pv = {'status': 'available', 'months': ordered, 'latestMonth': months[5], 'total': sum(o['views'] for o in ordered), 'dailyAverage': latest / nd, 'trendPercent': (latest - prior) / prior * 100 if prior > 0 else None} # 30-day drift: views 2026-10-17..11-15 vs 2026-09-17..10-16 def window(start, days): s = dt.date.fromisoformat(start) return sum(dmap.get((s + dt.timedelta(d)).strftime('%Y%m%d'), 0) for d in range(days)) last30, prev30 = window('2026-10-17', 30), window('2026-09-17', 30) pb = page['length'] cards.append({'pageId': page['pageid'], 'title': page['title'], 'description': page.get('description'), 'wikidataId': (page.get('pageprops') or {}).get('wikibase_item'), 'revisionId': page['lastrevid'], 'pageBytes': pb, 'languageCount': langs, 'languagesTruncated': 'llcontinue' in cont, 'categories': [c['title'].removeprefix('Category:') for c in page.get('categories', [])], 'categoriesTruncated': 'clcontinue' in cont, 'fetchedAt': now.isoformat(), 'pageviews': pv, 'stats': {'attention': score(pv['months'][5]['views'], 6) if pv['status'] == 'available' else None, 'depth': score(pb, 6), 'reach': score(langs, 3)}, 'legacyCategory': pk['legacyCategory'], 'drift30': {'last30': last30, 'prev30': prev30, 'pct': (last30 - prev30) / prev30 * 100 if prev30 else None}}) if n % 25 == 0: print(f'enriched {n + 1}/{len(picked)}', file=sys.stderr) # 3. Wikidata P31 (instance of) + labels qids = [c['wikidataId'] for c in cards if c['wikidataId']] p31 = {} for i in range(0, len(qids), 50): d = get('https://www.wikidata.org/w/api.php?' + urllib.parse.urlencode({'action': 'wbgetentities', 'format': 'json', 'ids': '|'.join(qids[i:i + 50]), 'props': 'claims'})) for q, ent in d.get('entities', {}).items(): p31[q] = [c['mainsnak']['datavalue']['value']['id'] for c in ent.get('claims', {}).get('P31', []) if c['mainsnak'].get('datavalue')] classes = sorted({c for v in p31.values() for c in v}) labels = {} for i in range(0, len(classes), 50): d = get('https://www.wikidata.org/w/api.php?' + urllib.parse.urlencode({'action': 'wbgetentities', 'format': 'json', 'ids': '|'.join(classes[i:i + 50]), 'props': 'labels', 'languages': 'en'})) for q, ent in d.get('entities', {}).items(): labels[q] = ent.get('labels', {}).get('en', {}).get('value', q) for c in cards: c['instanceOf'] = [{'id': q, 'label': labels.get(q, q)} for q in p31.get(c['wikidataId'], [])] json.dump({'generatedAt': dt.datetime.now(dt.timezone.utc).isoformat(), 'method': 'random ns0 batches of 20, filter as server/wiki.ts (thumbnail + extract>=40); enrichment as server/enrichment.ts; monthly views summed from daily REST pageviews; P31 from Wikidata', 'randomBatches': batches, 'rawPagesSeen': raw_pages, 'cards': cards}, open(OUT, 'w'), indent=1, ensure_ascii=False) print('done', len(cards), file=sys.stderr)