#!/usr/bin/env python3 """Summarise card-sample.json into markdown stats. Usage: python3 -I card_analyze.py sample.json > out.md""" import json, sys, math, statistics as st, collections, re d = json.load(open(sys.argv[1])); C = d['cards'] def q(v): v = sorted(v); n = len(v) def p(x): k = (n - 1) * x; f = math.floor(k); c = min(f + 1, n - 1); return v[f] + (v[c] - v[f]) * (k - f) return [v[0], p(.1), p(.25), p(.5), p(.75), p(.9), v[-1]] def skew(v): m, s = st.mean(v), st.pstdev(v); return sum((x - m) ** 3 for x in v) / len(v) / s ** 3 if s else 0 def rank(v): o = sorted(range(len(v)), key=lambda i: v[i]); r = [0] * len(v); i = 0 while i < len(o): j = i while j + 1 < len(o) and v[o[j + 1]] == v[o[i]]: j += 1 for k in range(i, j + 1): r[o[k]] = (i + j) / 2 i = j + 1 return r def pearson(a, b): ma, mb = st.mean(a), st.mean(b); sa = math.sqrt(sum((x - ma) ** 2 for x in a)); sb = math.sqrt(sum((y - mb) ** 2 for y in b)) return sum((x - ma) * (y - mb) for x, y in zip(a, b)) / (sa * sb) if sa and sb else float('nan') pv = [c for c in C if c['pageviews']['status'] == 'available'] F = {'attention': lambda c: c['stats']['attention'], 'depth': lambda c: c['stats']['depth'], 'reach': lambda c: c['stats']['reach'], 'pageBytes': lambda c: c['pageBytes'], 'languageCount': lambda c: c['languageCount'], 'monthlyViews': lambda c: c['pageviews']['months'][5]['views'] if c['pageviews']['status'] == 'available' else None, 'dailyAverage': lambda c: c['pageviews']['dailyAverage'], 'trendPercent': lambda c: c['pageviews']['trendPercent'], 'drift30Percent': lambda c: c['drift30']['pct']} out = [] P = out.append rate = f"{len(C)/d['rawPagesSeen']*100:.0f}%" if d['rawPagesSeen'] else '~44% (149/340 logged during sampling)' P(f"Sample: **{len(C)} cards**; {rate} of random pages pass the illustrated filter. Pageviews available for {len(pv)}.\n") P('| stat | n | min | p10 | p25 | median | p75 | p90 | max | mean | skew |'); P('|---|---|---|---|---|---|---|---|---|---|---|') for k, f in F.items(): v = [f(c) for c in C if f(c) is not None] qs = q(v); fmt = (lambda x: f'{x:.0f}') if k not in ('trendPercent', 'drift30Percent', 'dailyAverage') else (lambda x: f'{x:.1f}') P(f"| {k} | {len(v)} | " + ' | '.join(fmt(x) for x in qs) + f" | {fmt(st.mean(v))} | {skew(v):.2f} |") P('\n**Spearman rank correlations** (cards with pageviews):\n') keys = ['attention', 'depth', 'reach', 'trendPercent', 'drift30Percent'] rows = [c for c in pv if c['pageviews']['trendPercent'] is not None and c['drift30']['pct'] is not None] R = {k: rank([F[k](c) for c in rows]) for k in keys} P('| | ' + ' | '.join(keys) + ' |'); P('|---' * (len(keys) + 1) + '|') for a in keys: P(f'| {a} | ' + ' | '.join(f'{pearson(R[a], R[b]):.2f}' for b in keys) + ' |') P(f'\n(n={len(rows)}; attention~monthlyViews, depth~pageBytes, reach~languageCount are monotonic so their rank correlations are identical.)\n') tp = [c['pageviews']['trendPercent'] for c in pv if c['pageviews']['trendPercent'] is not None] P(f"**trendPercent sign:** positive {sum(t>0 for t in tp)} ({sum(t>0 for t in tp)/len(tp)*100:.0f}%), negative {sum(t<0 for t in tp)} ({sum(t<0 for t in tp)/len(tp)*100:.0f}%), zero {sum(t==0 for t in tp)}; null (prior month 0 or no views) {len(C)-len(tp)}. " f"|trend| > 50%: {sum(abs(t)>50 for t in tp)}; > 100%: {sum(abs(t)>100 for t in tp)}.\n") dr = [c['drift30']['pct'] for c in C if c['drift30']['pct'] is not None] P(f"**30-day drift** (views 10-17..11-15 vs 09-17..10-16): median |drift| {st.median([abs(x) for x in dr]):.0f}%, " f"p90 |drift| {q([abs(x) for x in dr])[5]:.0f}%. Attention score change implied: " f"median {st.median([abs(100*math.log10((c['drift30']['last30']+1)/(c['drift30']['prev30']+1))/6) for c in C]):.1f} points, " f"p90 {q([abs(100*math.log10((c['drift30']['last30']+1)/(c['drift30']['prev30']+1))/6) for c in C])[5]:.1f} points.\n") for name, f in [('attention', F['attention']), ('depth', F['depth']), ('reach', F['reach'])]: v = [f(c) for c in C if f(c) is not None]; h = collections.Counter(min(x // 10, 9) for x in v) P(f"**{name} histogram (deciles 0-9,10-19..90-100):** " + ' '.join(str(h.get(i, 0)) for i in range(10))) P(f"\nReach = 0 (no other languages): {sum(c['languageCount']==0 for c in C)}; reach 100 (>=999 langs) impossible in practice, max langs {max(c['languageCount'] for c in C)}. " f"Attention null: {sum(c['stats']['attention'] is None for c in C)}; attention 0: {sum(c['stats']['attention']==0 for c in C)}.\n") s3 = [sum(F[k](c) or 0 for k in ('attention', 'depth', 'reach')) for c in C] P(f"**Sum of three stats:** " + ', '.join(f'{n} {x:.0f}' for n, x in zip(['min', 'p10', 'p25', 'median', 'p75', 'p90', 'max'], q(s3))) + '\n') P('**Legacy `category()` (server/wiki.ts):** ' + ', '.join(f'{k} {v}' for k, v in collections.Counter(c['legacyCategory'] for c in C).most_common()) + '\n') io = collections.Counter(x['label'] for c in C for x in c['instanceOf']) P('**Top Wikidata P31 classes:** ' + ', '.join(f'{k} {v}' for k, v in io.most_common(40)) + '\n') P(f"No Wikidata item: {sum(not c['wikidataId'] for c in C)}; no P31: {sum(not c['instanceOf'] for c in C)}\n") words = collections.Counter() for c in C: for cat in set(w for t in c['categories'] for w in re.findall(r'[a-z]+', t.lower()) if len(w) > 3): words[cat] += 1 P('**Most frequent category words (cards containing):** ' + ', '.join(f'{k} {v}' for k, v in words.most_common(60)) + '\n') print('\n'.join(out)) # ---- rank normalisation table (critic request): score -> 1..10 tier by sample decile def tiers(name): v = sorted(F[name](c) for c in C if F[name](c) is not None) return [v[min(len(v) - 1, int(len(v) * k / 10))] for k in range(1, 10)] T = {k: tiers(k) for k in ('attention', 'depth', 'reach')} print('\n## Rank-normalised tiers (1–10)\n') print('Tier = 1 + number of breakpoints the score is **>=**. Breakpoints are the sample deciles p10..p90 of the 0–100 score. ' 'Ties make some tiers thin (scores are integers).\n') print('| stat | p10 | p20 | p30 | p40 | p50 | p60 | p70 | p80 | p90 |'); print('|---' * 10 + '|') for k, b in T.items(): print(f'| {k} | ' + ' | '.join(str(x) for x in b) + ' |') def tier(k, s): return None if s is None else 1 + sum(s >= x for x in T[k]) for c in C: c['tiers'] = {k: tier(k, c['stats'][k]) for k in T} tc = collections.Counter(c['tiers']['attention'] for c in C) print('\nResulting tier counts — attention: ' + ' '.join(f'{t}:{tc.get(t,0)}' for t in [None] + list(range(1, 11)))) for k in ('depth', 'reach'): tc = collections.Counter(c['tiers'][k] for c in C); print(f'; {k}: ' + ' '.join(f'{t}:{tc.get(t,0)}' for t in range(1, 11))) json.dump({'breakpoints': T, 'rule': 'tier = 1 + count(breakpoints <= score); attention null -> no tier'}, open(sys.argv[2] if len(sys.argv) > 2 else '/dev/null', 'w'), indent=1) # ---- starter loaner candidates: 4 per legacy category, spread over combined tier if len(sys.argv) > 3: pick = [] for cat in ['Space', 'Nature', 'People', 'Science', 'History', 'Culture']: pool = sorted((c for c in C if c['legacyCategory'] == cat and c['stats']['attention'] is not None), key=lambda c: sum(c['tiers'].values())) if not pool: continue n = min(4, len(pool)); idx = sorted({round(i * (len(pool) - 1) / max(1, n - 1)) for i in range(n)}) for i in idx: c = pool[i] pick.append({'pageId': c['pageId'], 'title': c['title'], 'category': cat, 'wikidataId': c['wikidataId'], 'revisionId': c['revisionId'], 'pageBytes': c['pageBytes'], 'languageCount': c['languageCount'], 'monthlyViews': c['pageviews']['months'][5]['views'], 'latestMonth': c['pageviews']['latestMonth'], 'trendPercent': None if c['pageviews']['trendPercent'] is None else round(c['pageviews']['trendPercent'], 1), 'stats': c['stats'], 'tiers': c['tiers']}) json.dump({'frozenAt': d['generatedAt'][:10], 'note': 'Frozen loaner/starter stats from the council-data random sample. Do not fetch live.', 'cards': pick}, open(sys.argv[3], 'w'), indent=1, ensure_ascii=False) print(f'\nStarter candidates: {len(pick)} written to {sys.argv[3]}: ' + collections.Counter(p['category'] for p in pick).__repr__()) # ---- candidate factions from Wikidata P31 (+ categories fallback) FAC = [ ('People', r'\bhuman\b|fictional character|person'), ('Sport', r'season|sports team|football|match|tournament|championship|final\b|club|league|olympic|race\b'), ('Nature', r'taxon|species|genus|plant|fungus|disease|epiphyte|mineral|bird|fish|insect'), ('Places', r'village|town|city|commune|municipality|river|lake|mountain|county|district|parish|region|plateau|island|settlement|neighbo|locality|census|hamlet|valley|bay|area|province|state of|capital|suburb|stream|peak|unincorporated|human settlement|local government'), ('Structures', r'building|castle|mosque|church|cathedral|temple|bridge|station|railway line|road|highway|museum|structure|stadium|dam|tower|house|school|hospital|airport|monument|fort|palace|lighthouse|park'), ('Works', r'film|album|single|song|musical|television|book|novel|video game|painting|literary|composition|series|episode|magazine|newspaper|poem|opera|play\b|artwork|sculpture'), ('Organisations', r'company|group|band|organi[sz]ation|university|college|party|business|enterprise|agency|institute|association|brand|union|ministry|military unit'), ('Events', r'election|protest|battle|war\b|storm|nor.easter|cyclone|event|disaster|treaty|conflict|earthquake|incident|festival|referendum|conference|ceremony'), ('Ideas & Tech', r'programming|software|language|chemical|compound|gene|protein|theorem|concept|technology|vehicle|aircraft|ship|weapon|class|model|device|algorithm|religion|discipline|theory|food|dish'), ] def faction(c): text = ' | '.join(x['label'].lower() for x in c['instanceOf']) for name, rx in FAC: if text and re.search(rx, text): return name cats = ' | '.join(c['categories']).lower() if re.search(r'births|deaths|people', cats): return 'People' if re.search(r'populated|villages|towns|geography|rivers', cats): return 'Places' if re.search(r'films|albums|songs|novels|television|video games', cats): return 'Works' return 'Other' for c in C: c['faction'] = faction(c) print('\n## Candidate factions (Wikidata P31 rules)\n') print('| faction | n | share | median attention | median depth | median reach | example |'); print('|---|---|---|---|---|---|---|') by = collections.defaultdict(list) for c in C: by[c['faction']].append(c) for name, cs in sorted(by.items(), key=lambda kv: -len(kv[1])): med = lambda k: st.median([x['stats'][k] for x in cs if x['stats'][k] is not None] or [0]) print(f"| {name} | {len(cs)} | {len(cs)/len(C)*100:.0f}% | {med('attention'):.0f} | {med('depth'):.0f} | {med('reach'):.0f} | {cs[0]['title']} |") other = collections.Counter(x['label'] for c in by.get('Other', []) for x in c['instanceOf']) print('\nUnclassified P31 labels: ' + ', '.join(f'{k} {v}' for k, v in other.most_common(25))) if len(sys.argv) > 4: json.dump({c['pageId']: c['faction'] for c in C}, open(sys.argv[4], 'w'))