159 lines
9.2 KiB
Python
159 lines
9.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Sample random illustrated en.wikipedia articles the way server/wiki.ts selects them and
|
|
enrich them the way server/enrichment.ts does. Polite: ~4 req/s, sequential, disk cache.
|
|
Usage: python3 -I card_sample.py <cache_dir> <out.json> [target=320]"""
|
|
import hashlib, json, os, re, sys, time, urllib.parse, urllib.request, calendar, datetime as dt
|
|
|
|
UA = 'WikiDeckCardDataSampler/0.1 (experimental card-game stat research; python-urllib)'
|
|
CACHE, OUT = sys.argv[1], sys.argv[2]
|
|
TARGET = int(sys.argv[3]) if len(sys.argv) > 3 else 320
|
|
os.makedirs(CACHE, exist_ok=True)
|
|
_last = [0.0]
|
|
|
|
def get(url, cache=True):
|
|
key = os.path.join(CACHE, hashlib.sha1(url.encode()).hexdigest() + '.json')
|
|
if cache and os.path.exists(key):
|
|
return json.load(open(key))
|
|
if os.environ.get('CACHE_ONLY') and 'wikidata.org' not in url:
|
|
return {'__missing': True}
|
|
for attempt in range(6):
|
|
wait = 1.0 - (time.time() - _last[0])
|
|
if wait > 0: time.sleep(wait)
|
|
_last[0] = time.time()
|
|
req = urllib.request.Request(url, headers={'User-Agent': UA, 'Accept': 'application/json'})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=20) as r:
|
|
data = json.loads(r.read().decode())
|
|
break
|
|
except urllib.error.HTTPError as e:
|
|
if e.code == 404: data = {'__404': True}; break
|
|
if e.code == 429 or e.code >= 500:
|
|
ra = e.headers.get('retry-after'); w = int(ra) + 1 if ra and ra.isdigit() else 5 * (attempt + 1)
|
|
print('http', e.code, 'sleep', w, file=sys.stderr); time.sleep(min(w, 60)); continue
|
|
raise
|
|
except Exception as e:
|
|
print('retry', attempt, type(e).__name__, e, file=sys.stderr); time.sleep(2 * (attempt + 1))
|
|
else:
|
|
raise RuntimeError('failed ' + url)
|
|
if cache: json.dump(data, open(key, 'w'))
|
|
return data
|
|
|
|
def api(params, cache=True):
|
|
return get('https://en.wikipedia.org/w/api.php?' + urllib.parse.urlencode({'format': 'json', 'formatversion': '2', **params}), cache)
|
|
|
|
def legacy_category(title, cats):
|
|
t = (title + ' ' + ' '.join(cats)).lower()
|
|
if re.search(r'astronom|planet|space|galax|moon|star\b', t): return 'Space'
|
|
if re.search(r'animal|plant|species|ocean|biolog|mollusc', t): return 'Nature'
|
|
if re.search(r'births|deaths|people|scientist', t): return 'People'
|
|
if re.search(r'physic|chem|math|science|technolog', t): return 'Science'
|
|
if re.search(r'histor|ancient|archaeolog|empire', t): return 'History'
|
|
return 'Culture'
|
|
|
|
def score(v, scale):
|
|
import math
|
|
return min(100, round(math.log10(v + 1) / scale * 100))
|
|
|
|
# 1. random illustrated articles (same filter as wiki.ts: thumbnail + extract >= 40 chars)
|
|
picked, seen, batches, raw_pages = [], set(), 0, 0
|
|
state_file = os.path.join(CACHE, '_random_state.json')
|
|
if os.path.exists(state_file):
|
|
picked = json.load(open(state_file)); seen = {p['pageId'] for p in picked}
|
|
while len(picked) < TARGET:
|
|
try: d = api({'action': 'query', 'generator': 'random', 'grnnamespace': '0', 'grnlimit': '20',
|
|
'prop': 'extracts|pageimages|info|categories', 'exintro': '1', 'explaintext': '1', 'exchars': '900',
|
|
'exlimit': '20', 'piprop': 'thumbnail', 'pithumbsize': '800', 'cllimit': '500'}, cache=False)
|
|
except RuntimeError as e: print(e, file=sys.stderr); time.sleep(5); continue
|
|
batches += 1
|
|
for p in d.get('query', {}).get('pages', []):
|
|
raw_pages += 1
|
|
thumb = (p.get('thumbnail') or {}).get('source', '')
|
|
ext = (p.get('extract') or '').strip()
|
|
if not re.match(r'https://(upload|thumb)\.wikimedia\.org/', thumb) or len(ext) < 40 or p['pageid'] in seen: continue
|
|
seen.add(p['pageid'])
|
|
picked.append({'pageId': p['pageid'], 'title': p['title'],
|
|
'legacyCategory': legacy_category(p['title'], [c['title'] for c in p.get('categories', [])])})
|
|
json.dump(picked, open(state_file, 'w'))
|
|
print(f'random batch {batches}: {len(picked)} illustrated / {raw_pages} raw', file=sys.stderr)
|
|
picked = picked[:TARGET]
|
|
|
|
# 2. enrichment per article (same params as enrichment.ts) + daily views for drift
|
|
now = dt.datetime(2026, 11, 16, tzinfo=dt.timezone.utc)
|
|
months = []
|
|
for i in range(6):
|
|
m = now.month - 6 + i; y = now.year + (m - 1) // 12; m = (m - 1) % 12 + 1
|
|
months.append(f'{y:04d}-{m:02d}')
|
|
# batched metadata (same props as enrichment.ts; per-page cllimit 50 is emulated by truncating)
|
|
meta = {}
|
|
ids = [p['pageId'] for p in picked]
|
|
for i in range(0, len(ids), 50):
|
|
chunk = ids[i:i + 50]; cont = {}; pages = {}
|
|
while True:
|
|
md = api({'action': 'query', 'pageids': '|'.join(map(str, chunk)), 'prop': 'info|categories|pageprops|description|langlinks',
|
|
'clshow': '!hidden', 'cllimit': '500', 'lllimit': '500', 'ppprop': 'wikibase_item', **cont})
|
|
for pg in md['query']['pages']:
|
|
cur = pages.setdefault(pg['pageid'], {**pg, 'categories': [], 'langlinks': []})
|
|
cur['categories'] += pg.get('categories', []); cur['langlinks'] += pg.get('langlinks', [])
|
|
if 'continue' not in md: break
|
|
cont = md['continue']
|
|
meta.update(pages)
|
|
print(f'metadata {len(meta)}/{len(ids)}', file=sys.stderr)
|
|
cards = []
|
|
for n, pk in enumerate(picked):
|
|
page = meta[pk['pageId']]
|
|
cont = {'clcontinue': 1} if len(page['categories']) > 50 else {}
|
|
page['categories'] = page['categories'][:50]
|
|
langs = len(page.get('langlinks', []))
|
|
t = urllib.parse.quote(page['title'].replace(' ', '_'), safe='')
|
|
daily = get(f'https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia.org/all-access/user/{t}/daily/{months[0].replace("-", "")}01/20261115')
|
|
if daily.get('__missing'): continue
|
|
dmap = {it['timestamp'][:8]: it['views'] for it in daily.get('items', [])}
|
|
pv = {'status': 'unavailable', 'months': [], 'latestMonth': months[5], 'total': None, 'dailyAverage': None, 'trendPercent': None}
|
|
if dmap:
|
|
ordered = []
|
|
for m in months:
|
|
y, mo = map(int, m.split('-')); nd = calendar.monthrange(y, mo)[1]
|
|
ordered.append({'month': m, 'views': sum(dmap.get(f'{y:04d}{mo:02d}{d:02d}', 0) for d in range(1, nd + 1))})
|
|
latest, prior = ordered[5]['views'], ordered[4]['views']
|
|
nd = calendar.monthrange(*map(int, months[5].split('-')))[1]
|
|
pv = {'status': 'available', 'months': ordered, 'latestMonth': months[5], 'total': sum(o['views'] for o in ordered),
|
|
'dailyAverage': latest / nd, 'trendPercent': (latest - prior) / prior * 100 if prior > 0 else None}
|
|
# 30-day drift: views 2026-10-17..11-15 vs 2026-09-17..10-16
|
|
def window(start, days):
|
|
s = dt.date.fromisoformat(start)
|
|
return sum(dmap.get((s + dt.timedelta(d)).strftime('%Y%m%d'), 0) for d in range(days))
|
|
last30, prev30 = window('2026-10-17', 30), window('2026-09-17', 30)
|
|
pb = page['length']
|
|
cards.append({'pageId': page['pageid'], 'title': page['title'], 'description': page.get('description'),
|
|
'wikidataId': (page.get('pageprops') or {}).get('wikibase_item'), 'revisionId': page['lastrevid'], 'pageBytes': pb,
|
|
'languageCount': langs, 'languagesTruncated': 'llcontinue' in cont,
|
|
'categories': [c['title'].removeprefix('Category:') for c in page.get('categories', [])],
|
|
'categoriesTruncated': 'clcontinue' in cont, 'fetchedAt': now.isoformat(), 'pageviews': pv,
|
|
'stats': {'attention': score(pv['months'][5]['views'], 6) if pv['status'] == 'available' else None,
|
|
'depth': score(pb, 6), 'reach': score(langs, 3)},
|
|
'legacyCategory': pk['legacyCategory'],
|
|
'drift30': {'last30': last30, 'prev30': prev30, 'pct': (last30 - prev30) / prev30 * 100 if prev30 else None}})
|
|
if n % 25 == 0: print(f'enriched {n + 1}/{len(picked)}', file=sys.stderr)
|
|
|
|
# 3. Wikidata P31 (instance of) + labels
|
|
qids = [c['wikidataId'] for c in cards if c['wikidataId']]
|
|
p31 = {}
|
|
for i in range(0, len(qids), 50):
|
|
d = get('https://www.wikidata.org/w/api.php?' + urllib.parse.urlencode({'action': 'wbgetentities', 'format': 'json',
|
|
'ids': '|'.join(qids[i:i + 50]), 'props': 'claims'}))
|
|
for q, ent in d.get('entities', {}).items():
|
|
p31[q] = [c['mainsnak']['datavalue']['value']['id'] for c in ent.get('claims', {}).get('P31', [])
|
|
if c['mainsnak'].get('datavalue')]
|
|
classes = sorted({c for v in p31.values() for c in v})
|
|
labels = {}
|
|
for i in range(0, len(classes), 50):
|
|
d = get('https://www.wikidata.org/w/api.php?' + urllib.parse.urlencode({'action': 'wbgetentities', 'format': 'json',
|
|
'ids': '|'.join(classes[i:i + 50]), 'props': 'labels', 'languages': 'en'}))
|
|
for q, ent in d.get('entities', {}).items():
|
|
labels[q] = ent.get('labels', {}).get('en', {}).get('value', q)
|
|
for c in cards:
|
|
c['instanceOf'] = [{'id': q, 'label': labels.get(q, q)} for q in p31.get(c['wikidataId'], [])]
|
|
json.dump({'generatedAt': dt.datetime.now(dt.timezone.utc).isoformat(), 'method': 'random ns0 batches of 20, filter as server/wiki.ts (thumbnail + extract>=40); enrichment as server/enrichment.ts; monthly views summed from daily REST pageviews; P31 from Wikidata',
|
|
'randomBatches': batches, 'rawPagesSeen': raw_pages, 'cards': cards}, open(OUT, 'w'), indent=1, ensure_ascii=False)
|
|
print('done', len(cards), file=sys.stderr)
|