157 lines
13 KiB
Python
157 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
"""Summarise card-sample.json into markdown stats. Usage: python3 -I card_analyze.py sample.json > out.md"""
|
||
import json, sys, math, statistics as st, collections, re
|
||
d = json.load(open(sys.argv[1])); C = d['cards']
|
||
def q(v):
|
||
v = sorted(v); n = len(v)
|
||
def p(x):
|
||
k = (n - 1) * x; f = math.floor(k); c = min(f + 1, n - 1); return v[f] + (v[c] - v[f]) * (k - f)
|
||
return [v[0], p(.1), p(.25), p(.5), p(.75), p(.9), v[-1]]
|
||
def skew(v):
|
||
m, s = st.mean(v), st.pstdev(v); return sum((x - m) ** 3 for x in v) / len(v) / s ** 3 if s else 0
|
||
def rank(v):
|
||
o = sorted(range(len(v)), key=lambda i: v[i]); r = [0] * len(v); i = 0
|
||
while i < len(o):
|
||
j = i
|
||
while j + 1 < len(o) and v[o[j + 1]] == v[o[i]]: j += 1
|
||
for k in range(i, j + 1): r[o[k]] = (i + j) / 2
|
||
i = j + 1
|
||
return r
|
||
def pearson(a, b):
|
||
ma, mb = st.mean(a), st.mean(b); sa = math.sqrt(sum((x - ma) ** 2 for x in a)); sb = math.sqrt(sum((y - mb) ** 2 for y in b))
|
||
return sum((x - ma) * (y - mb) for x, y in zip(a, b)) / (sa * sb) if sa and sb else float('nan')
|
||
pv = [c for c in C if c['pageviews']['status'] == 'available']
|
||
F = {'attention': lambda c: c['stats']['attention'], 'depth': lambda c: c['stats']['depth'], 'reach': lambda c: c['stats']['reach'],
|
||
'pageBytes': lambda c: c['pageBytes'], 'languageCount': lambda c: c['languageCount'],
|
||
'monthlyViews': lambda c: c['pageviews']['months'][5]['views'] if c['pageviews']['status'] == 'available' else None,
|
||
'dailyAverage': lambda c: c['pageviews']['dailyAverage'], 'trendPercent': lambda c: c['pageviews']['trendPercent'],
|
||
'drift30Percent': lambda c: c['drift30']['pct']}
|
||
out = []
|
||
P = out.append
|
||
rate = f"{len(C)/d['rawPagesSeen']*100:.0f}%" if d['rawPagesSeen'] else '~44% (149/340 logged during sampling)'
|
||
P(f"Sample: **{len(C)} cards**; {rate} of random pages pass the illustrated filter. Pageviews available for {len(pv)}.\n")
|
||
P('| stat | n | min | p10 | p25 | median | p75 | p90 | max | mean | skew |'); P('|---|---|---|---|---|---|---|---|---|---|---|')
|
||
for k, f in F.items():
|
||
v = [f(c) for c in C if f(c) is not None]
|
||
qs = q(v); fmt = (lambda x: f'{x:.0f}') if k not in ('trendPercent', 'drift30Percent', 'dailyAverage') else (lambda x: f'{x:.1f}')
|
||
P(f"| {k} | {len(v)} | " + ' | '.join(fmt(x) for x in qs) + f" | {fmt(st.mean(v))} | {skew(v):.2f} |")
|
||
P('\n**Spearman rank correlations** (cards with pageviews):\n')
|
||
keys = ['attention', 'depth', 'reach', 'trendPercent', 'drift30Percent']
|
||
rows = [c for c in pv if c['pageviews']['trendPercent'] is not None and c['drift30']['pct'] is not None]
|
||
R = {k: rank([F[k](c) for c in rows]) for k in keys}
|
||
P('| | ' + ' | '.join(keys) + ' |'); P('|---' * (len(keys) + 1) + '|')
|
||
for a in keys: P(f'| {a} | ' + ' | '.join(f'{pearson(R[a], R[b]):.2f}' for b in keys) + ' |')
|
||
P(f'\n(n={len(rows)}; attention~monthlyViews, depth~pageBytes, reach~languageCount are monotonic so their rank correlations are identical.)\n')
|
||
tp = [c['pageviews']['trendPercent'] for c in pv if c['pageviews']['trendPercent'] is not None]
|
||
P(f"**trendPercent sign:** positive {sum(t>0 for t in tp)} ({sum(t>0 for t in tp)/len(tp)*100:.0f}%), negative {sum(t<0 for t in tp)} ({sum(t<0 for t in tp)/len(tp)*100:.0f}%), zero {sum(t==0 for t in tp)}; null (prior month 0 or no views) {len(C)-len(tp)}. "
|
||
f"|trend| > 50%: {sum(abs(t)>50 for t in tp)}; > 100%: {sum(abs(t)>100 for t in tp)}.\n")
|
||
dr = [c['drift30']['pct'] for c in C if c['drift30']['pct'] is not None]
|
||
P(f"**30-day drift** (views 10-17..11-15 vs 09-17..10-16): median |drift| {st.median([abs(x) for x in dr]):.0f}%, "
|
||
f"p90 |drift| {q([abs(x) for x in dr])[5]:.0f}%. Attention score change implied: "
|
||
f"median {st.median([abs(100*math.log10((c['drift30']['last30']+1)/(c['drift30']['prev30']+1))/6) for c in C]):.1f} points, "
|
||
f"p90 {q([abs(100*math.log10((c['drift30']['last30']+1)/(c['drift30']['prev30']+1))/6) for c in C])[5]:.1f} points.\n")
|
||
for name, f in [('attention', F['attention']), ('depth', F['depth']), ('reach', F['reach'])]:
|
||
v = [f(c) for c in C if f(c) is not None]; h = collections.Counter(min(x // 10, 9) for x in v)
|
||
P(f"**{name} histogram (deciles 0-9,10-19..90-100):** " + ' '.join(str(h.get(i, 0)) for i in range(10)))
|
||
P(f"\nReach = 0 (no other languages): {sum(c['languageCount']==0 for c in C)}; reach 100 (>=999 langs) impossible in practice, max langs {max(c['languageCount'] for c in C)}. "
|
||
f"Attention null: {sum(c['stats']['attention'] is None for c in C)}; attention 0: {sum(c['stats']['attention']==0 for c in C)}.\n")
|
||
s3 = [sum(F[k](c) or 0 for k in ('attention', 'depth', 'reach')) for c in C]
|
||
P(f"**Sum of three stats:** " + ', '.join(f'{n} {x:.0f}' for n, x in zip(['min', 'p10', 'p25', 'median', 'p75', 'p90', 'max'], q(s3))) + '\n')
|
||
P('**Legacy `category()` (server/wiki.ts):** ' + ', '.join(f'{k} {v}' for k, v in collections.Counter(c['legacyCategory'] for c in C).most_common()) + '\n')
|
||
io = collections.Counter(x['label'] for c in C for x in c['instanceOf'])
|
||
P('**Top Wikidata P31 classes:** ' + ', '.join(f'{k} {v}' for k, v in io.most_common(40)) + '\n')
|
||
P(f"No Wikidata item: {sum(not c['wikidataId'] for c in C)}; no P31: {sum(not c['instanceOf'] for c in C)}\n")
|
||
words = collections.Counter()
|
||
for c in C:
|
||
for cat in set(w for t in c['categories'] for w in re.findall(r'[a-z]+', t.lower()) if len(w) > 3): words[cat] += 1
|
||
P('**Most frequent category words (cards containing):** ' + ', '.join(f'{k} {v}' for k, v in words.most_common(60)) + '\n')
|
||
print('\n'.join(out))
|
||
|
||
# ---- rank normalisation table (critic request): score -> 1..10 tier by sample decile
|
||
def tiers(name):
|
||
v = sorted(F[name](c) for c in C if F[name](c) is not None)
|
||
return [v[min(len(v) - 1, int(len(v) * k / 10))] for k in range(1, 10)]
|
||
T = {k: tiers(k) for k in ('attention', 'depth', 'reach')}
|
||
print('\n## Rank-normalised tiers (1–10)\n')
|
||
print('Tier = 1 + number of breakpoints the score is **>=**. Breakpoints are the sample deciles p10..p90 of the 0–100 score. '
|
||
'Ties make some tiers thin (scores are integers).\n')
|
||
print('| stat | p10 | p20 | p30 | p40 | p50 | p60 | p70 | p80 | p90 |'); print('|---' * 10 + '|')
|
||
for k, b in T.items(): print(f'| {k} | ' + ' | '.join(str(x) for x in b) + ' |')
|
||
def tier(k, s): return None if s is None else 1 + sum(s >= x for x in T[k])
|
||
for c in C: c['tiers'] = {k: tier(k, c['stats'][k]) for k in T}
|
||
tc = collections.Counter(c['tiers']['attention'] for c in C)
|
||
print('\nResulting tier counts — attention: ' + ' '.join(f'{t}:{tc.get(t,0)}' for t in [None] + list(range(1, 11))))
|
||
for k in ('depth', 'reach'):
|
||
tc = collections.Counter(c['tiers'][k] for c in C); print(f'; {k}: ' + ' '.join(f'{t}:{tc.get(t,0)}' for t in range(1, 11)))
|
||
json.dump({'breakpoints': T, 'rule': 'tier = 1 + count(breakpoints <= score); attention null -> no tier'},
|
||
open(sys.argv[2] if len(sys.argv) > 2 else '/dev/null', 'w'), indent=1)
|
||
|
||
# ---- starter loaner candidates: 4 per legacy category, spread over combined tier
|
||
if len(sys.argv) > 3:
|
||
pick = []
|
||
for cat in ['Space', 'Nature', 'People', 'Science', 'History', 'Culture']:
|
||
pool = sorted((c for c in C if c['legacyCategory'] == cat and c['stats']['attention'] is not None),
|
||
key=lambda c: sum(c['tiers'].values()))
|
||
if not pool: continue
|
||
n = min(4, len(pool)); idx = sorted({round(i * (len(pool) - 1) / max(1, n - 1)) for i in range(n)})
|
||
for i in idx:
|
||
c = pool[i]
|
||
pick.append({'pageId': c['pageId'], 'title': c['title'], 'category': cat, 'wikidataId': c['wikidataId'],
|
||
'revisionId': c['revisionId'], 'pageBytes': c['pageBytes'], 'languageCount': c['languageCount'],
|
||
'monthlyViews': c['pageviews']['months'][5]['views'], 'latestMonth': c['pageviews']['latestMonth'],
|
||
'trendPercent': None if c['pageviews']['trendPercent'] is None else round(c['pageviews']['trendPercent'], 1),
|
||
'stats': c['stats'], 'tiers': c['tiers']})
|
||
json.dump({'frozenAt': d['generatedAt'][:10], 'note': 'Frozen loaner/starter stats from the council-data random sample. Do not fetch live.',
|
||
'cards': pick}, open(sys.argv[3], 'w'), indent=1, ensure_ascii=False)
|
||
print(f'\nStarter candidates: {len(pick)} written to {sys.argv[3]}: ' + collections.Counter(p['category'] for p in pick).__repr__())
|
||
|
||
# ---- candidate factions from Wikidata P31 (+ categories fallback)
|
||
FAC = [
|
||
('People', r'\bhuman\b|fictional character|person'),
|
||
('Sport', r'season|sports team|football|match|tournament|championship|final\b|club|league|olympic|race\b'),
|
||
('Nature', r'taxon|species|genus|plant|fungus|disease|epiphyte|mineral|bird|fish|insect'),
|
||
('Places', r'village|town|city|commune|municipality|river|lake|mountain|county|district|parish|region|plateau|island|settlement|neighbo|locality|census|hamlet|valley|bay|area|province|state of|capital|suburb|stream|peak|unincorporated|human settlement|local government'),
|
||
('Structures', r'building|castle|mosque|church|cathedral|temple|bridge|station|railway line|road|highway|museum|structure|stadium|dam|tower|house|school|hospital|airport|monument|fort|palace|lighthouse|park'),
|
||
('Works', r'film|album|single|song|musical|television|book|novel|video game|painting|literary|composition|series|episode|magazine|newspaper|poem|opera|play\b|artwork|sculpture'),
|
||
('Organisations', r'company|group|band|organi[sz]ation|university|college|party|business|enterprise|agency|institute|association|brand|union|ministry|military unit'),
|
||
('Events', r'election|protest|battle|war\b|storm|nor.easter|cyclone|event|disaster|treaty|conflict|earthquake|incident|festival|referendum|conference|ceremony'),
|
||
('Ideas & Tech', r'programming|software|language|chemical|compound|gene|protein|theorem|concept|technology|vehicle|aircraft|ship|weapon|class|model|device|algorithm|religion|discipline|theory|food|dish'),
|
||
]
|
||
def faction(c):
|
||
text = ' | '.join(x['label'].lower() for x in c['instanceOf'])
|
||
for name, rx in FAC:
|
||
if text and re.search(rx, text): return name
|
||
cats = ' | '.join(c['categories']).lower()
|
||
if re.search(r'births|deaths|people', cats): return 'People'
|
||
if re.search(r'populated|villages|towns|geography|rivers', cats): return 'Places'
|
||
if re.search(r'films|albums|songs|novels|television|video games', cats): return 'Works'
|
||
return 'Other'
|
||
for c in C: c['faction'] = faction(c)
|
||
print('\n## Candidate factions (Wikidata P31 rules)\n')
|
||
print('| faction | n | share | median attention | median depth | median reach | example |'); print('|---|---|---|---|---|---|---|')
|
||
by = collections.defaultdict(list)
|
||
for c in C: by[c['faction']].append(c)
|
||
for name, cs in sorted(by.items(), key=lambda kv: -len(kv[1])):
|
||
med = lambda k: st.median([x['stats'][k] for x in cs if x['stats'][k] is not None] or [0])
|
||
print(f"| {name} | {len(cs)} | {len(cs)/len(C)*100:.0f}% | {med('attention'):.0f} | {med('depth'):.0f} | {med('reach'):.0f} | {cs[0]['title']} |")
|
||
other = collections.Counter(x['label'] for c in by.get('Other', []) for x in c['instanceOf'])
|
||
print('\nUnclassified P31 labels: ' + ', '.join(f'{k} {v}' for k, v in other.most_common(25)))
|
||
if len(sys.argv) > 4: json.dump({c['pageId']: c['faction'] for c in C}, open(sys.argv[4], 'w'))
|
||
if len(sys.argv) > 3:
|
||
s = json.load(open(sys.argv[3])); fmap = {c['pageId']: c['faction'] for c in C}
|
||
for p in s['cards']: p['faction'] = fmap[p['pageId']]
|
||
json.dump(s, open(sys.argv[3], 'w'), indent=1, ensure_ascii=False)
|
||
if len(sys.argv) > 5:
|
||
d['factionRules'] = [[n, rx] for n, rx in FAC]; d['tierBreakpoints'] = T
|
||
def cuts(k):
|
||
v = sorted(c['stats'][k] for c in C if c['stats'][k] is not None); n = len(v)
|
||
return [v[max(0, math.ceil(i * n / 100) - 1)] for i in range(1, 100)]
|
||
d['percentiles'] = {k: cuts(k) for k in ('attention', 'depth', 'reach')}
|
||
d['percentilesMethod'] = 'cut point i (1..99) = nearest-rank i-th percentile of the sample snapshot score (sorted ascending, index ceil(i*n/100)-1). Use with DESIGN.md §1.1 counting.'
|
||
if not d['rawPagesSeen']:
|
||
d['randomBatches'], d['rawPagesSeen'] = 17, 340
|
||
d['randomBatchesNote'] = 'Sampling was resumed from a saved state, so full batch counts were lost. These are the counts logged at batch 17 (149 illustrated of 340 raw pages, 44% pass rate). The 330 picked cards came from about 38 batches; this sample holds the first N of those that finished enrichment before sampling stopped (Wikimedia 429s).'
|
||
d['note'] = 'docs/game/starter-cards.json is the loaner POOL (real frozen snapshots of random articles), used by the sim and the deck builder to fill slots. The default starter decks are defined in DESIGN.md §2.1; their real pageIds are in starter-cards.json under starterDeckPageIds.'
|
||
for c in d['cards']: c['category'] = c['legacyCategory']
|
||
json.dump(d, open(sys.argv[5], 'w'), indent=1, ensure_ascii=False)
|