Files
wikideck/scripts/night/card_analyze.py
T

140 lines
11 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Summarise card-sample.json into markdown stats. Usage: python3 -I card_analyze.py sample.json > out.md"""
import json, sys, math, statistics as st, collections, re
d = json.load(open(sys.argv[1])); C = d['cards']
def q(v):
v = sorted(v); n = len(v)
def p(x):
k = (n - 1) * x; f = math.floor(k); c = min(f + 1, n - 1); return v[f] + (v[c] - v[f]) * (k - f)
return [v[0], p(.1), p(.25), p(.5), p(.75), p(.9), v[-1]]
def skew(v):
m, s = st.mean(v), st.pstdev(v); return sum((x - m) ** 3 for x in v) / len(v) / s ** 3 if s else 0
def rank(v):
o = sorted(range(len(v)), key=lambda i: v[i]); r = [0] * len(v); i = 0
while i < len(o):
j = i
while j + 1 < len(o) and v[o[j + 1]] == v[o[i]]: j += 1
for k in range(i, j + 1): r[o[k]] = (i + j) / 2
i = j + 1
return r
def pearson(a, b):
ma, mb = st.mean(a), st.mean(b); sa = math.sqrt(sum((x - ma) ** 2 for x in a)); sb = math.sqrt(sum((y - mb) ** 2 for y in b))
return sum((x - ma) * (y - mb) for x, y in zip(a, b)) / (sa * sb) if sa and sb else float('nan')
pv = [c for c in C if c['pageviews']['status'] == 'available']
F = {'attention': lambda c: c['stats']['attention'], 'depth': lambda c: c['stats']['depth'], 'reach': lambda c: c['stats']['reach'],
'pageBytes': lambda c: c['pageBytes'], 'languageCount': lambda c: c['languageCount'],
'monthlyViews': lambda c: c['pageviews']['months'][5]['views'] if c['pageviews']['status'] == 'available' else None,
'dailyAverage': lambda c: c['pageviews']['dailyAverage'], 'trendPercent': lambda c: c['pageviews']['trendPercent'],
'drift30Percent': lambda c: c['drift30']['pct']}
out = []
P = out.append
rate = f"{len(C)/d['rawPagesSeen']*100:.0f}%" if d['rawPagesSeen'] else '~44% (149/340 logged during sampling)'
P(f"Sample: **{len(C)} cards**; {rate} of random pages pass the illustrated filter. Pageviews available for {len(pv)}.\n")
P('| stat | n | min | p10 | p25 | median | p75 | p90 | max | mean | skew |'); P('|---|---|---|---|---|---|---|---|---|---|---|')
for k, f in F.items():
v = [f(c) for c in C if f(c) is not None]
qs = q(v); fmt = (lambda x: f'{x:.0f}') if k not in ('trendPercent', 'drift30Percent', 'dailyAverage') else (lambda x: f'{x:.1f}')
P(f"| {k} | {len(v)} | " + ' | '.join(fmt(x) for x in qs) + f" | {fmt(st.mean(v))} | {skew(v):.2f} |")
P('\n**Spearman rank correlations** (cards with pageviews):\n')
keys = ['attention', 'depth', 'reach', 'trendPercent', 'drift30Percent']
rows = [c for c in pv if c['pageviews']['trendPercent'] is not None and c['drift30']['pct'] is not None]
R = {k: rank([F[k](c) for c in rows]) for k in keys}
P('| | ' + ' | '.join(keys) + ' |'); P('|---' * (len(keys) + 1) + '|')
for a in keys: P(f'| {a} | ' + ' | '.join(f'{pearson(R[a], R[b]):.2f}' for b in keys) + ' |')
P(f'\n(n={len(rows)}; attention~monthlyViews, depth~pageBytes, reach~languageCount are monotonic so their rank correlations are identical.)\n')
tp = [c['pageviews']['trendPercent'] for c in pv if c['pageviews']['trendPercent'] is not None]
P(f"**trendPercent sign:** positive {sum(t>0 for t in tp)} ({sum(t>0 for t in tp)/len(tp)*100:.0f}%), negative {sum(t<0 for t in tp)} ({sum(t<0 for t in tp)/len(tp)*100:.0f}%), zero {sum(t==0 for t in tp)}; null (prior month 0 or no views) {len(C)-len(tp)}. "
f"|trend| > 50%: {sum(abs(t)>50 for t in tp)}; > 100%: {sum(abs(t)>100 for t in tp)}.\n")
dr = [c['drift30']['pct'] for c in C if c['drift30']['pct'] is not None]
P(f"**30-day drift** (views 10-17..11-15 vs 09-17..10-16): median |drift| {st.median([abs(x) for x in dr]):.0f}%, "
f"p90 |drift| {q([abs(x) for x in dr])[5]:.0f}%. Attention score change implied: "
f"median {st.median([abs(100*math.log10((c['drift30']['last30']+1)/(c['drift30']['prev30']+1))/6) for c in C]):.1f} points, "
f"p90 {q([abs(100*math.log10((c['drift30']['last30']+1)/(c['drift30']['prev30']+1))/6) for c in C])[5]:.1f} points.\n")
for name, f in [('attention', F['attention']), ('depth', F['depth']), ('reach', F['reach'])]:
v = [f(c) for c in C if f(c) is not None]; h = collections.Counter(min(x // 10, 9) for x in v)
P(f"**{name} histogram (deciles 0-9,10-19..90-100):** " + ' '.join(str(h.get(i, 0)) for i in range(10)))
P(f"\nReach = 0 (no other languages): {sum(c['languageCount']==0 for c in C)}; reach 100 (>=999 langs) impossible in practice, max langs {max(c['languageCount'] for c in C)}. "
f"Attention null: {sum(c['stats']['attention'] is None for c in C)}; attention 0: {sum(c['stats']['attention']==0 for c in C)}.\n")
s3 = [sum(F[k](c) or 0 for k in ('attention', 'depth', 'reach')) for c in C]
P(f"**Sum of three stats:** " + ', '.join(f'{n} {x:.0f}' for n, x in zip(['min', 'p10', 'p25', 'median', 'p75', 'p90', 'max'], q(s3))) + '\n')
P('**Legacy `category()` (server/wiki.ts):** ' + ', '.join(f'{k} {v}' for k, v in collections.Counter(c['legacyCategory'] for c in C).most_common()) + '\n')
io = collections.Counter(x['label'] for c in C for x in c['instanceOf'])
P('**Top Wikidata P31 classes:** ' + ', '.join(f'{k} {v}' for k, v in io.most_common(40)) + '\n')
P(f"No Wikidata item: {sum(not c['wikidataId'] for c in C)}; no P31: {sum(not c['instanceOf'] for c in C)}\n")
words = collections.Counter()
for c in C:
for cat in set(w for t in c['categories'] for w in re.findall(r'[a-z]+', t.lower()) if len(w) > 3): words[cat] += 1
P('**Most frequent category words (cards containing):** ' + ', '.join(f'{k} {v}' for k, v in words.most_common(60)) + '\n')
print('\n'.join(out))
# ---- rank normalisation table (critic request): score -> 1..10 tier by sample decile
def tiers(name):
v = sorted(F[name](c) for c in C if F[name](c) is not None)
return [v[min(len(v) - 1, int(len(v) * k / 10))] for k in range(1, 10)]
T = {k: tiers(k) for k in ('attention', 'depth', 'reach')}
print('\n## Rank-normalised tiers (1–10)\n')
print('Tier = 1 + number of breakpoints the score is **>=**. Breakpoints are the sample deciles p10..p90 of the 0–100 score. '
'Ties make some tiers thin (scores are integers).\n')
print('| stat | p10 | p20 | p30 | p40 | p50 | p60 | p70 | p80 | p90 |'); print('|---' * 10 + '|')
for k, b in T.items(): print(f'| {k} | ' + ' | '.join(str(x) for x in b) + ' |')
def tier(k, s): return None if s is None else 1 + sum(s >= x for x in T[k])
for c in C: c['tiers'] = {k: tier(k, c['stats'][k]) for k in T}
tc = collections.Counter(c['tiers']['attention'] for c in C)
print('\nResulting tier counts — attention: ' + ' '.join(f'{t}:{tc.get(t,0)}' for t in [None] + list(range(1, 11))))
for k in ('depth', 'reach'):
tc = collections.Counter(c['tiers'][k] for c in C); print(f'; {k}: ' + ' '.join(f'{t}:{tc.get(t,0)}' for t in range(1, 11)))
json.dump({'breakpoints': T, 'rule': 'tier = 1 + count(breakpoints <= score); attention null -> no tier'},
open(sys.argv[2] if len(sys.argv) > 2 else '/dev/null', 'w'), indent=1)
# ---- starter loaner candidates: 4 per legacy category, spread over combined tier
if len(sys.argv) > 3:
pick = []
for cat in ['Space', 'Nature', 'People', 'Science', 'History', 'Culture']:
pool = sorted((c for c in C if c['legacyCategory'] == cat and c['stats']['attention'] is not None),
key=lambda c: sum(c['tiers'].values()))
if not pool: continue
n = min(4, len(pool)); idx = sorted({round(i * (len(pool) - 1) / max(1, n - 1)) for i in range(n)})
for i in idx:
c = pool[i]
pick.append({'pageId': c['pageId'], 'title': c['title'], 'category': cat, 'wikidataId': c['wikidataId'],
'revisionId': c['revisionId'], 'pageBytes': c['pageBytes'], 'languageCount': c['languageCount'],
'monthlyViews': c['pageviews']['months'][5]['views'], 'latestMonth': c['pageviews']['latestMonth'],
'trendPercent': None if c['pageviews']['trendPercent'] is None else round(c['pageviews']['trendPercent'], 1),
'stats': c['stats'], 'tiers': c['tiers']})
json.dump({'frozenAt': d['generatedAt'][:10], 'note': 'Frozen loaner/starter stats from the council-data random sample. Do not fetch live.',
'cards': pick}, open(sys.argv[3], 'w'), indent=1, ensure_ascii=False)
print(f'\nStarter candidates: {len(pick)} written to {sys.argv[3]}: ' + collections.Counter(p['category'] for p in pick).__repr__())
# ---- candidate factions from Wikidata P31 (+ categories fallback)
FAC = [
('People', r'\bhuman\b|fictional character|person'),
('Sport', r'season|sports team|football|match|tournament|championship|final\b|club|league|olympic|race\b'),
('Nature', r'taxon|species|genus|plant|fungus|disease|epiphyte|mineral|bird|fish|insect'),
('Places', r'village|town|city|commune|municipality|river|lake|mountain|county|district|parish|region|plateau|island|settlement|neighbo|locality|census|hamlet|valley|bay|area|province|state of|capital|suburb|stream|peak|unincorporated|human settlement|local government'),
('Structures', r'building|castle|mosque|church|cathedral|temple|bridge|station|railway line|road|highway|museum|structure|stadium|dam|tower|house|school|hospital|airport|monument|fort|palace|lighthouse|park'),
('Works', r'film|album|single|song|musical|television|book|novel|video game|painting|literary|composition|series|episode|magazine|newspaper|poem|opera|play\b|artwork|sculpture'),
('Organisations', r'company|group|band|organi[sz]ation|university|college|party|business|enterprise|agency|institute|association|brand|union|ministry|military unit'),
('Events', r'election|protest|battle|war\b|storm|nor.easter|cyclone|event|disaster|treaty|conflict|earthquake|incident|festival|referendum|conference|ceremony'),
('Ideas & Tech', r'programming|software|language|chemical|compound|gene|protein|theorem|concept|technology|vehicle|aircraft|ship|weapon|class|model|device|algorithm|religion|discipline|theory|food|dish'),
]
def faction(c):
text = ' | '.join(x['label'].lower() for x in c['instanceOf'])
for name, rx in FAC:
if text and re.search(rx, text): return name
cats = ' | '.join(c['categories']).lower()
if re.search(r'births|deaths|people', cats): return 'People'
if re.search(r'populated|villages|towns|geography|rivers', cats): return 'Places'
if re.search(r'films|albums|songs|novels|television|video games', cats): return 'Works'
return 'Other'
for c in C: c['faction'] = faction(c)
print('\n## Candidate factions (Wikidata P31 rules)\n')
print('| faction | n | share | median attention | median depth | median reach | example |'); print('|---|---|---|---|---|---|---|')
by = collections.defaultdict(list)
for c in C: by[c['faction']].append(c)
for name, cs in sorted(by.items(), key=lambda kv: -len(kv[1])):
med = lambda k: st.median([x['stats'][k] for x in cs if x['stats'][k] is not None] or [0])
print(f"| {name} | {len(cs)} | {len(cs)/len(C)*100:.0f}% | {med('attention'):.0f} | {med('depth'):.0f} | {med('reach'):.0f} | {cs[0]['title']} |")
other = collections.Counter(x['label'] for c in by.get('Other', []) for x in c['instanceOf'])
print('\nUnclassified P31 labels: ' + ', '.join(f'{k} {v}' for k, v in other.most_common(25)))
if len(sys.argv) > 4: json.dump({c['pageId']: c['faction'] for c in C}, open(sys.argv[4], 'w'))