forked from WikiDeck/wikideck
140 lines
11 KiB
Python
140 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""Summarise card-sample.json into markdown stats. Usage: python3 -I card_analyze.py sample.json > out.md"""
|
||
import json, sys, math, statistics as st, collections, re
|
||
d = json.load(open(sys.argv[1])); C = d['cards']
|
||
def q(v):
|
||
v = sorted(v); n = len(v)
|
||
def p(x):
|
||
k = (n - 1) * x; f = math.floor(k); c = min(f + 1, n - 1); return v[f] + (v[c] - v[f]) * (k - f)
|
||
return [v[0], p(.1), p(.25), p(.5), p(.75), p(.9), v[-1]]
|
||
def skew(v):
|
||
m, s = st.mean(v), st.pstdev(v); return sum((x - m) ** 3 for x in v) / len(v) / s ** 3 if s else 0
|
||
def rank(v):
|
||
o = sorted(range(len(v)), key=lambda i: v[i]); r = [0] * len(v); i = 0
|
||
while i < len(o):
|
||
j = i
|
||
while j + 1 < len(o) and v[o[j + 1]] == v[o[i]]: j += 1
|
||
for k in range(i, j + 1): r[o[k]] = (i + j) / 2
|
||
i = j + 1
|
||
return r
|
||
def pearson(a, b):
|
||
ma, mb = st.mean(a), st.mean(b); sa = math.sqrt(sum((x - ma) ** 2 for x in a)); sb = math.sqrt(sum((y - mb) ** 2 for y in b))
|
||
return sum((x - ma) * (y - mb) for x, y in zip(a, b)) / (sa * sb) if sa and sb else float('nan')
|
||
pv = [c for c in C if c['pageviews']['status'] == 'available']
|
||
F = {'attention': lambda c: c['stats']['attention'], 'depth': lambda c: c['stats']['depth'], 'reach': lambda c: c['stats']['reach'],
|
||
'pageBytes': lambda c: c['pageBytes'], 'languageCount': lambda c: c['languageCount'],
|
||
'monthlyViews': lambda c: c['pageviews']['months'][5]['views'] if c['pageviews']['status'] == 'available' else None,
|
||
'dailyAverage': lambda c: c['pageviews']['dailyAverage'], 'trendPercent': lambda c: c['pageviews']['trendPercent'],
|
||
'drift30Percent': lambda c: c['drift30']['pct']}
|
||
out = []
|
||
P = out.append
|
||
rate = f"{len(C)/d['rawPagesSeen']*100:.0f}%" if d['rawPagesSeen'] else '~44% (149/340 logged during sampling)'
|
||
P(f"Sample: **{len(C)} cards**; {rate} of random pages pass the illustrated filter. Pageviews available for {len(pv)}.\n")
|
||
P('| stat | n | min | p10 | p25 | median | p75 | p90 | max | mean | skew |'); P('|---|---|---|---|---|---|---|---|---|---|---|')
|
||
for k, f in F.items():
|
||
v = [f(c) for c in C if f(c) is not None]
|
||
qs = q(v); fmt = (lambda x: f'{x:.0f}') if k not in ('trendPercent', 'drift30Percent', 'dailyAverage') else (lambda x: f'{x:.1f}')
|
||
P(f"| {k} | {len(v)} | " + ' | '.join(fmt(x) for x in qs) + f" | {fmt(st.mean(v))} | {skew(v):.2f} |")
|
||
P('\n**Spearman rank correlations** (cards with pageviews):\n')
|
||
keys = ['attention', 'depth', 'reach', 'trendPercent', 'drift30Percent']
|
||
rows = [c for c in pv if c['pageviews']['trendPercent'] is not None and c['drift30']['pct'] is not None]
|
||
R = {k: rank([F[k](c) for c in rows]) for k in keys}
|
||
P('| | ' + ' | '.join(keys) + ' |'); P('|---' * (len(keys) + 1) + '|')
|
||
for a in keys: P(f'| {a} | ' + ' | '.join(f'{pearson(R[a], R[b]):.2f}' for b in keys) + ' |')
|
||
P(f'\n(n={len(rows)}; attention~monthlyViews, depth~pageBytes, reach~languageCount are monotonic so their rank correlations are identical.)\n')
|
||
tp = [c['pageviews']['trendPercent'] for c in pv if c['pageviews']['trendPercent'] is not None]
|
||
P(f"**trendPercent sign:** positive {sum(t>0 for t in tp)} ({sum(t>0 for t in tp)/len(tp)*100:.0f}%), negative {sum(t<0 for t in tp)} ({sum(t<0 for t in tp)/len(tp)*100:.0f}%), zero {sum(t==0 for t in tp)}; null (prior month 0 or no views) {len(C)-len(tp)}. "
|
||
f"|trend| > 50%: {sum(abs(t)>50 for t in tp)}; > 100%: {sum(abs(t)>100 for t in tp)}.\n")
|
||
dr = [c['drift30']['pct'] for c in C if c['drift30']['pct'] is not None]
|
||
P(f"**30-day drift** (views 10-17..11-15 vs 09-17..10-16): median |drift| {st.median([abs(x) for x in dr]):.0f}%, "
|
||
f"p90 |drift| {q([abs(x) for x in dr])[5]:.0f}%. Attention score change implied: "
|
||
f"median {st.median([abs(100*math.log10((c['drift30']['last30']+1)/(c['drift30']['prev30']+1))/6) for c in C]):.1f} points, "
|
||
f"p90 {q([abs(100*math.log10((c['drift30']['last30']+1)/(c['drift30']['prev30']+1))/6) for c in C])[5]:.1f} points.\n")
|
||
for name, f in [('attention', F['attention']), ('depth', F['depth']), ('reach', F['reach'])]:
|
||
v = [f(c) for c in C if f(c) is not None]; h = collections.Counter(min(x // 10, 9) for x in v)
|
||
P(f"**{name} histogram (deciles 0-9,10-19..90-100):** " + ' '.join(str(h.get(i, 0)) for i in range(10)))
|
||
P(f"\nReach = 0 (no other languages): {sum(c['languageCount']==0 for c in C)}; reach 100 (>=999 langs) impossible in practice, max langs {max(c['languageCount'] for c in C)}. "
|
||
f"Attention null: {sum(c['stats']['attention'] is None for c in C)}; attention 0: {sum(c['stats']['attention']==0 for c in C)}.\n")
|
||
s3 = [sum(F[k](c) or 0 for k in ('attention', 'depth', 'reach')) for c in C]
|
||
P(f"**Sum of three stats:** " + ', '.join(f'{n} {x:.0f}' for n, x in zip(['min', 'p10', 'p25', 'median', 'p75', 'p90', 'max'], q(s3))) + '\n')
|
||
P('**Legacy `category()` (server/wiki.ts):** ' + ', '.join(f'{k} {v}' for k, v in collections.Counter(c['legacyCategory'] for c in C).most_common()) + '\n')
|
||
io = collections.Counter(x['label'] for c in C for x in c['instanceOf'])
|
||
P('**Top Wikidata P31 classes:** ' + ', '.join(f'{k} {v}' for k, v in io.most_common(40)) + '\n')
|
||
P(f"No Wikidata item: {sum(not c['wikidataId'] for c in C)}; no P31: {sum(not c['instanceOf'] for c in C)}\n")
|
||
words = collections.Counter()
|
||
for c in C:
|
||
for cat in set(w for t in c['categories'] for w in re.findall(r'[a-z]+', t.lower()) if len(w) > 3): words[cat] += 1
|
||
P('**Most frequent category words (cards containing):** ' + ', '.join(f'{k} {v}' for k, v in words.most_common(60)) + '\n')
|
||
print('\n'.join(out))
|
||
|
||
# ---- rank normalisation table (critic request): score -> 1..10 tier by sample decile
|
||
def tiers(name):
|
||
v = sorted(F[name](c) for c in C if F[name](c) is not None)
|
||
return [v[min(len(v) - 1, int(len(v) * k / 10))] for k in range(1, 10)]
|
||
T = {k: tiers(k) for k in ('attention', 'depth', 'reach')}
|
||
print('\n## Rank-normalised tiers (1–10)\n')
|
||
print('Tier = 1 + number of breakpoints the score is **>=**. Breakpoints are the sample deciles p10..p90 of the 0–100 score. '
|
||
'Ties make some tiers thin (scores are integers).\n')
|
||
print('| stat | p10 | p20 | p30 | p40 | p50 | p60 | p70 | p80 | p90 |'); print('|---' * 10 + '|')
|
||
for k, b in T.items(): print(f'| {k} | ' + ' | '.join(str(x) for x in b) + ' |')
|
||
def tier(k, s): return None if s is None else 1 + sum(s >= x for x in T[k])
|
||
for c in C: c['tiers'] = {k: tier(k, c['stats'][k]) for k in T}
|
||
tc = collections.Counter(c['tiers']['attention'] for c in C)
|
||
print('\nResulting tier counts — attention: ' + ' '.join(f'{t}:{tc.get(t,0)}' for t in [None] + list(range(1, 11))))
|
||
for k in ('depth', 'reach'):
|
||
tc = collections.Counter(c['tiers'][k] for c in C); print(f'; {k}: ' + ' '.join(f'{t}:{tc.get(t,0)}' for t in range(1, 11)))
|
||
json.dump({'breakpoints': T, 'rule': 'tier = 1 + count(breakpoints <= score); attention null -> no tier'},
|
||
open(sys.argv[2] if len(sys.argv) > 2 else '/dev/null', 'w'), indent=1)
|
||
|
||
# ---- starter loaner candidates: 4 per legacy category, spread over combined tier
|
||
if len(sys.argv) > 3:
|
||
pick = []
|
||
for cat in ['Space', 'Nature', 'People', 'Science', 'History', 'Culture']:
|
||
pool = sorted((c for c in C if c['legacyCategory'] == cat and c['stats']['attention'] is not None),
|
||
key=lambda c: sum(c['tiers'].values()))
|
||
if not pool: continue
|
||
n = min(4, len(pool)); idx = sorted({round(i * (len(pool) - 1) / max(1, n - 1)) for i in range(n)})
|
||
for i in idx:
|
||
c = pool[i]
|
||
pick.append({'pageId': c['pageId'], 'title': c['title'], 'category': cat, 'wikidataId': c['wikidataId'],
|
||
'revisionId': c['revisionId'], 'pageBytes': c['pageBytes'], 'languageCount': c['languageCount'],
|
||
'monthlyViews': c['pageviews']['months'][5]['views'], 'latestMonth': c['pageviews']['latestMonth'],
|
||
'trendPercent': None if c['pageviews']['trendPercent'] is None else round(c['pageviews']['trendPercent'], 1),
|
||
'stats': c['stats'], 'tiers': c['tiers']})
|
||
json.dump({'frozenAt': d['generatedAt'][:10], 'note': 'Frozen loaner/starter stats from the council-data random sample. Do not fetch live.',
|
||
'cards': pick}, open(sys.argv[3], 'w'), indent=1, ensure_ascii=False)
|
||
print(f'\nStarter candidates: {len(pick)} written to {sys.argv[3]}: ' + collections.Counter(p['category'] for p in pick).__repr__())
|
||
|
||
# ---- candidate factions from Wikidata P31 (+ categories fallback)
|
||
FAC = [
|
||
('People', r'\bhuman\b|fictional character|person'),
|
||
('Sport', r'season|sports team|football|match|tournament|championship|final\b|club|league|olympic|race\b'),
|
||
('Nature', r'taxon|species|genus|plant|fungus|disease|epiphyte|mineral|bird|fish|insect'),
|
||
('Places', r'village|town|city|commune|municipality|river|lake|mountain|county|district|parish|region|plateau|island|settlement|neighbo|locality|census|hamlet|valley|bay|area|province|state of|capital|suburb|stream|peak|unincorporated|human settlement|local government'),
|
||
('Structures', r'building|castle|mosque|church|cathedral|temple|bridge|station|railway line|road|highway|museum|structure|stadium|dam|tower|house|school|hospital|airport|monument|fort|palace|lighthouse|park'),
|
||
('Works', r'film|album|single|song|musical|television|book|novel|video game|painting|literary|composition|series|episode|magazine|newspaper|poem|opera|play\b|artwork|sculpture'),
|
||
('Organisations', r'company|group|band|organi[sz]ation|university|college|party|business|enterprise|agency|institute|association|brand|union|ministry|military unit'),
|
||
('Events', r'election|protest|battle|war\b|storm|nor.easter|cyclone|event|disaster|treaty|conflict|earthquake|incident|festival|referendum|conference|ceremony'),
|
||
('Ideas & Tech', r'programming|software|language|chemical|compound|gene|protein|theorem|concept|technology|vehicle|aircraft|ship|weapon|class|model|device|algorithm|religion|discipline|theory|food|dish'),
|
||
]
|
||
def faction(c):
|
||
text = ' | '.join(x['label'].lower() for x in c['instanceOf'])
|
||
for name, rx in FAC:
|
||
if text and re.search(rx, text): return name
|
||
cats = ' | '.join(c['categories']).lower()
|
||
if re.search(r'births|deaths|people', cats): return 'People'
|
||
if re.search(r'populated|villages|towns|geography|rivers', cats): return 'Places'
|
||
if re.search(r'films|albums|songs|novels|television|video games', cats): return 'Works'
|
||
return 'Other'
|
||
for c in C: c['faction'] = faction(c)
|
||
print('\n## Candidate factions (Wikidata P31 rules)\n')
|
||
print('| faction | n | share | median attention | median depth | median reach | example |'); print('|---|---|---|---|---|---|---|')
|
||
by = collections.defaultdict(list)
|
||
for c in C: by[c['faction']].append(c)
|
||
for name, cs in sorted(by.items(), key=lambda kv: -len(kv[1])):
|
||
med = lambda k: st.median([x['stats'][k] for x in cs if x['stats'][k] is not None] or [0])
|
||
print(f"| {name} | {len(cs)} | {len(cs)/len(C)*100:.0f}% | {med('attention'):.0f} | {med('depth'):.0f} | {med('reach'):.0f} | {cs[0]['title']} |")
|
||
other = collections.Counter(x['label'] for c in by.get('Other', []) for x in c['instanceOf'])
|
||
print('\nUnclassified P31 labels: ' + ', '.join(f'{k} {v}' for k, v in other.most_common(25)))
|
||
if len(sys.argv) > 4: json.dump({c['pageId']: c['faction'] for c in C}, open(sys.argv[4], 'w'))
|