From dbf385d2173a68037b36b7d3ccbd3633e13754d6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 13:07:31 -0500 Subject: [PATCH] Night 2: data analyst scripts behind reviews/data.md Co-Authored-By: Claude Opus 5.5 --- docs/night2/analysis/connected.py | 44 +++++++++++++++++ docs/night2/analysis/fields.py | 23 +++++++++ docs/night2/analysis/stats.py | 59 +++++++++++++++++++++++ docs/night2/analysis/types.py | 79 +++++++++++++++++++++++++++++++ 4 files changed, 205 insertions(+) create mode 100644 docs/night2/analysis/connected.py create mode 100644 docs/night2/analysis/fields.py create mode 100644 docs/night2/analysis/stats.py create mode 100644 docs/night2/analysis/types.py diff --git a/docs/night2/analysis/connected.py b/docs/night2/analysis/connected.py new file mode 100644 index 0000000..fa5f91a --- /dev/null +++ b/docs/night2/analysis/connected.py @@ -0,0 +1,44 @@ +"""How often random card pairs share cheap relations. Usage: python3 -I connected.py """ +import json, sys, re, itertools, collections, importlib.util, os +cards = json.load(open(sys.argv[1]))['cards'] +spec = importlib.util.spec_from_file_location('ty', os.path.join(os.path.dirname(os.path.abspath(__file__)), 'types.py')) +ty = importlib.util.module_from_spec(spec); spec.loader.exec_module(ty) +GENERIC = re.compile(r'stubs|living people|\d+ (births|deaths)$|articles|wikidata|pages', re.I) +COUNTRIES = ['United States','American','England','English','British','France','French','Germany','German','Japan','Japanese','China','Chinese', + 'India','Indian','Brazil','Poland','Polish','Italy','Italian','Spain','Spanish','Austria','Canada','Canadian','Australia','Australian','Russia','Russian', + 'Mexico','Mexican','Iran','Pakistan','Bangladesh','Sweden','Swedish','Norway','Estonia','Ukraine','Croatia','Philippines','Cuba','Israel','Portugal','Scotland','Wales','Ireland','Kazakhstan','Malaysia','Argentin','Belgi','Iceland','Bermuda','Thailand','Singapore','Sri Lanka','Slovenia','Moldova','Cyprus','Macedonia','Mauritania','Congo','Gibraltar','Hong Kong','New Zealand','Korea','Indonesia','Aceh'] +NORM = {'American':'United States','English':'England','British':'United Kingdom','England':'United Kingdom','Scotland':'United Kingdom','Wales':'United Kingdom', + 'French':'France','German':'Germany','Japanese':'Japan','Chinese':'China','Indian':'India','Polish':'Poland','Italian':'Italy','Spanish':'Spain', + 'Canadian':'Canada','Australian':'Australia','Russian':'Russia','Mexican':'Mexico','Swedish':'Sweden'} +Y = re.compile(r'\b(1[0-9]{3}|20[0-2][0-9])') +def feats(c): + txt = (c['description'] or '') + ' | ' + ' | '.join(c['categories']) + ctry = {NORM.get(k, k) for k in COUNTRIES if k in txt} + cents = {int(y) // 100 for y in Y.findall(c['title'] + ' ' + txt)} + cats = {x for x in c['categories'] if not GENERIC.search(x)} + p31 = {i['id'] for i in c['instanceOf']} + return dict(ctry=ctry, cent=cents, cats=cats, allcats=set(c['categories']), p31=p31, type=ty.classify(c)[0]) +F = [feats(c) for c in cards]; pairs = list(itertools.combinations(range(len(cards)), 2)); N = len(pairs) +def rate(f): + k = sum(1 for i, j in pairs if f(F[i], F[j])); return f'{k}/{N} ({100*k/N:.2f}%)' +print('pairs', N) +print('share any category (incl. generic)', rate(lambda a, b: a['allcats'] & b['allcats'])) +print('share specific category', rate(lambda a, b: a['cats'] & b['cats'])) +print('share P31 class', rate(lambda a, b: a['p31'] & b['p31'])) +print('share P31 excl human/taxon', rate(lambda a, b: a['p31'] & b['p31'] - {'Q5', 'Q16521'})) +print('share country (text proxy for P17/P27)', rate(lambda a, b: a['ctry'] & b['ctry'])) +print('share century (text proxy for dates)', rate(lambda a, b: a['cent'] & b['cent'])) +print('share type', rate(lambda a, b: a['type'] == b['type'])) +print('country OR century OR specific cat', rate(lambda a, b: (a['ctry'] & b['ctry']) or (a['cent'] & b['cent']) or (a['cats'] & b['cats']))) +print('cards with >=1 country', sum(1 for f in F if f['ctry']), 'with >=1 century', sum(1 for f in F if f['cent'])) +# deck-level: chance that a random 10-card hand-pair (10 vs 10, own deck) has >=1 connected pair within a deck +import random; random.seed(1) +def deck_hits(pred, size=10, trials=2000): + h = 0 + for _ in range(trials): + s = random.sample(range(len(F)), size) + if any(pred(F[i], F[j]) for i, j in itertools.combinations(s, 2)): h += 1 + return f'{100*h/trials:.0f}%' +pc = lambda a, b: (a['ctry'] & b['ctry']) +print('10-card deck has >=1 same-country pair', deck_hits(pc), '| specific category', deck_hits(lambda a, b: a['cats'] & b['cats']), + '| country-or-century', deck_hits(lambda a, b: (a['ctry'] & b['ctry']) or (a['cent'] & b['cent']))) diff --git a/docs/night2/analysis/fields.py b/docs/night2/analysis/fields.py new file mode 100644 index 0000000..f08b07e --- /dev/null +++ b/docs/night2/analysis/fields.py @@ -0,0 +1,23 @@ +"""Field availability for rules-text generation in card-sample.json. Usage: python3 -I fields.py """ +import json, sys, re, statistics, collections +cards = json.load(open(sys.argv[1]))['cards']; n = len(cards) +Y = re.compile(r'\b(1[0-9]{3}|20[0-2][0-9])\b') +def share(f): k = sum(1 for c in cards if f(c)); return f'{k}/{n} ({100*k/n:.0f}%)' +print('description non-empty', share(lambda c: bool(c['description']))) +print('wikidataId', share(lambda c: bool(c['wikidataId']))) +print('P31 present', share(lambda c: bool(c['instanceOf']))) +print('>=1 non-hidden category', share(lambda c: len(c['categories']) > 0)) +print('categories truncated (cllimit 50)', share(lambda c: c['categoriesTruncated'])) +print('median categories', statistics.median(len(c['categories']) for c in cards)) +print('year in title', share(lambda c: Y.search(c['title']))) +print('year in description', share(lambda c: Y.search(c['description'] or ''))) +print('year in categories', share(lambda c: any(Y.search(x) for x in c['categories']))) +print('year anywhere (title/desc/cats)', share(lambda c: Y.search(c['title'] + ' ' + (c['description'] or '') + ' ' + ' '.join(c['categories'])))) +print('births/deaths category', share(lambda c: any(re.search(r'\d+ (births|deaths)', x) for x in c['categories']))) +print('establishments/completed category', share(lambda c: any(re.search(r'establishments|completed in|disestablish|opened in', x) for x in c['categories']))) +print('living people', share(lambda c: 'Living people' in c['categories'])) +print('stub category', share(lambda c: any('stubs' in x for x in c['categories']))) +print('country/nationality word in desc or cats', share(lambda c: re.search(r'American|British|English|French|German|Japan|China|Chinese|India|Brazil|Poland|Polish|Italy|Italian|Spain|Spanish|Austria|Canada|Australia|Russia|United States', (c['description'] or '') + ' '.join(c['categories'])))) +print('legacy category on-chain', collections.Counter(c['legacyCategory'] for c in cards)) +print('pageviews available', share(lambda c: c['pageviews']['status'] == 'available')) +print('languagesTruncated', share(lambda c: c['languagesTruncated'])) diff --git a/docs/night2/analysis/stats.py b/docs/night2/analysis/stats.py new file mode 100644 index 0000000..4b7d2be --- /dev/null +++ b/docs/night2/analysis/stats.py @@ -0,0 +1,59 @@ +"""Proposed percentile -> cost/power/toughness derivation, evaluated on card-sample.json. +Usage: python3 -I stats.py """ +import json, sys, collections, statistics, itertools +d = json.load(open(sys.argv[1])); cards = d['cards']; P = d['percentiles'] +def pct(s, v): # DESIGN 1.1 counting: cut points strictly below + half of equal + if v is None: return None + cut = P[s]; return int(sum(c < v for c in cut) + sum(c == v for c in cut) / 2) +TH = [26, 39, 51, 63, 78] # v1.1 costThresholds (fame -> cost 1..6) +BUDGET = [None, 3, 4, 6, 8, 10, 12] # P+T per cost, MTG vanilla curve (2/1, 2/2, 3/3, 4/4, 5/5, 6/6) +def derive(c): + a, dp, r = (pct(k, c['stats'][k]) for k in ('attention', 'depth', 'reach')) + vals = [x for x in (a, dp, r) if x is not None] + fame = int(sum(vals) / len(vals) + 0.5) + cost = 1 + sum(t <= fame for t in TH) + b = BUDGET[cost] + kw = 'Multilingual' if r >= 75 else ('Local' if c['stats']['reach'] == 0 else None) + if kw == 'Multilingual': b -= 1 # evasion-like keyword is paid for out of the body + if kw == 'Local': b += 0 # Local = gains +1/+1 next to a same-country Location (conditional, no tax) + share = (a if a is not None else 50) / max(1, (a if a is not None else 50) + dp) # attention vs depth + pw = min(b - 1, max(1, round(b * share))); t = b - pw + return dict(title=c['title'], a=a, d=dp, r=r, fame=fame, cost=cost, P=pw, T=t, kw=kw, + trend=c['pageviews']['trendPercent'], daily=c['pageviews']['dailyAverage']) +rows = [derive(c) for c in cards]; n = len(rows) +def naive(c): # straw man: power = attention tier, toughness = depth tier, cost from their sum + ti = c['tiers']; pw = (ti['attention'] or 1 + 1) // 2 + 0; t = (ti['depth'] + 1) // 2 + return dict(cost=max(1, min(6, round((pw + t) / 2))), P=pw, T=t) +nv = [naive(c) for c in cards] +def dominated_pairs(rs, use_kw=False): + k = 0 + for x, y in itertools.permutations(rs, 2): + if x['cost'] <= y['cost'] and x['P'] >= y['P'] and x['T'] >= y['T'] and \ + (x['cost'] < y['cost'] or x['P'] > y['P'] or x['T'] > y['T']) and (not use_kw or x.get('kw') == y.get('kw')): + k += 1 + return k +print('cost distribution', sorted(collections.Counter(r['cost'] for r in rows).items())) +print('power distribution', sorted(collections.Counter(r['P'] for r in rows).items())) +print('toughness distribution', sorted(collections.Counter(r['T'] for r in rows).items())) +print('keywords', collections.Counter(r['kw'] for r in rows)) +print('P/T by cost:') +for cst in range(1, 7): + g = [r for r in rows if r['cost'] == cst] + if g: print(' ', cst, len(g), collections.Counter(f"{r['P']}/{r['T']}" for r in g).most_common(6)) +pairs = n * (n - 1) +print('strict-dominance pairs proposed', dominated_pairs(rows), 'of', pairs, '; same-keyword only', dominated_pairs(rows, True)) +print('strict-dominance pairs naive ', dominated_pairs(nv), 'of', pairs) +# efficiency by fame quintile +q = sorted(rows, key=lambda r: r['fame']) +print('fame quintile: mean cost, mean (P+T)/cost proposed | naive (P+T)/cost') +idx = sorted(range(n), key=lambda i: rows[i]['fame']) +for k in range(5): + sl = idx[k * n // 5:(k + 1) * n // 5] + print(f" Q{k+1} fame {rows[sl[0]]['fame']}-{rows[sl[-1]]['fame']}: cost {statistics.mean(rows[i]['cost'] for i in sl):.2f} " + f"eff {statistics.mean((rows[i]['P']+rows[i]['T'])/rows[i]['cost'] for i in sl):.2f} | naive eff " + f"{statistics.mean((nv[i]['P']+nv[i]['T'])/nv[i]['cost'] for i in sl):.2f}") +top = sorted(rows, key=lambda r: -r['fame'])[:6] + sorted(rows, key=lambda r: r['fame'])[:3] +print('examples (top fame, bottom fame):') +for r in top: print(f" {r['title'][:34]:34s} a{r['a']} d{r['d']} r{r['r']} fame{r['fame']} cost{r['cost']} {r['P']}/{r['T']} {r['kw'] or ''}") +tr = [r for r in rows if r['trend'] is not None and r['trend'] >= 25 and (r['daily'] or 0) >= 20] +print('trending (+25% and daily>=20):', len(tr), f'{100*len(tr)/n:.1f}%') diff --git a/docs/night2/analysis/types.py b/docs/night2/analysis/types.py new file mode 100644 index 0000000..1d8cd3f --- /dev/null +++ b/docs/night2/analysis/types.py @@ -0,0 +1,79 @@ +"""Classify card-sample.json into Figure/Location/Event/Institution/Doctrine. +Order: P31 label rules (first rule that matches ANY P31 label wins, rule order = priority), +then category keywords, then description keywords, then deterministic fallback. +Usage: python3 -I types.py [--list]""" +import json, re, sys, collections +RULES_P31 = [ # (type, regex on a P31 label). Order matters: specific before generic. + ('Doctrine', r'aspect of|aspect in|disease|clinical sign|list article|chemical entity'), + ('Event', r'election|protest|battle|war\b|storm|nor.easter|cyclone|tornado|earthquake|disaster|event|match|final\b|' + r'season|championship|tournament|\brace\b|grand prix|scandal|annexation|treaty|incident|festival|referendum|' + r'conference|ceremony|time period|delegation'), + ('Institution', r'artificial satellite|space station|space telescope|spacecraft|space probe'), + ('Location', r'planet|natural satellite|\bstar\b|galaxy|nebula|planetary system|astronomical object|rainforest|reef|' + r'civili[sz]ation|empire|historical country|archaeological site|' + r'human settlement|settlement|village|town|city|commune|municipal|parish|county|district|region|province|' + r'township|hamlet|townland|census area|local government|community of|island|river|lake|mountain|plateau|' + r'waterfall|valley|bay|peak|geographical feature|historic district|park|state park|square|road|highway|' + r'building|castle|mosque|church|temple|cathedral|house|home|museum|station|bridge|monument|historic site|' + r'military base|battery|refinery|shopping|stadium|arch|tower|fort|palace|columbari|council'), + ('Figure', r'^human$|fictional character|stock character|person|deity|orisha|loa\b|taxon|species|genus|breed|' + r'^family$|organism'), + ('Institution', r'company|business|organi[sz]ation|association|society|party|agency|studio|band|musical group|team|club|' + r'university|college|school|institute|diocese|ecclesiastical|manufacturer|newspaper|magazine|film|album|' + r'song|single|musical work|composition|television|series|book|novel|video game|painting|award|position|' + r'aircraft|ship|steamship|vehicle|automobile|tram|railway line|route|typeface|software|product|brand'), + ('Doctrine', r'concept|theory|theorem|mathematical|programming language|language|gene|protein|chemical|compound|' + r'disease|clinical sign|ideology|religion|discipline|field|cuisine|food|dish|drink|aspect|list article|' + r'particle|process|genre|form\b|game\b|branch of|class of|type of|tabular'), +] +RULES_CAT = [ # fallback on non-hidden category titles (lowercase) + ('Figure', r'\bbirth|\bdeaths\b|living people|people from|\bspecies\b|described in \d|\bgenera\b|deities|characters'), + ('Event', r'\belections\b|protests|\bbattles\b|disasters|\braces\b|\bseasons\b|\bevents\b|scandals|annexations|competitions'), + ('Location', r'villages|communes|parishes|local government|populated places|geography|rivers|islands|buildings and structures|bridges|' + r'stations|parks|districts|townships|cities and towns|historic places'), + ('Institution', r'companies|organi[sz]ations|\bfilms\b|albums|songs|novels|newspapers|television series|musical groups|' + r'aircraft|ships|typefaces|software|awards|teams|clubs'), + ('Doctrine', r'theory|structures$|algebra|terminology|cuisine|drinks|desserts|ecology|languages|diseases|conditions|' + r'concepts|ranks|etymology|maps|atlases'), +] +RULES_DESC = [ + ('Figure', r'\b(painter|politician|actor|writer|player|born|species|genus)\b'), + ('Event', r'\b(race|election|battle|match|protest|storm|war|competition)\b'), + ('Location', r'\b(village|town|city|river|island|building|church|columbarium|station|park)\b'), + ('Institution', r'\b(company|band|film|album|organi[sz]ation|team|aircraft|ship|newspaper)\b'), + ('Doctrine', r'\b(concept|theory|structure|dessert|drink|dish|technique|class of)\b'), +] +TYPES = ['Figure', 'Location', 'Event', 'Institution', 'Doctrine'] + +def classify(card): + labels = [i['label'].lower() for i in card.get('instanceOf') or []] + for t, rx in RULES_P31: + for l in labels: + if re.search(rx, l): return t, 'p31', l + cats = [c.lower() for c in card.get('categories') or [] if 'stubs' not in c.lower()] + for t, rx in RULES_CAT: + for c in cats: + if re.search(rx, c): return t, 'category', c + desc = (card.get('description') or '').lower() + for t, rx in RULES_DESC: + if re.search(rx, desc): return t, 'description', desc + # deterministic last resort: Doctrine (the residual "idea" class); never hash-random. + return 'Doctrine', 'fallback', '' + +if __name__ == '__main__': + d = json.load(open(sys.argv[1])); cards = d['cards'] + res = [(c, *classify(c)) for c in cards] + n = len(res) + by_t = collections.Counter(r[1] for r in res); by_src = collections.Counter(r[2] for r in res) + print('n', n) + for t in TYPES: print(f'{t:12s} {by_t[t]:4d} {100*by_t[t]/n:5.1f}%') + for s in ['p31', 'category', 'description', 'fallback']: print(f'src {s:12s} {by_src[s]:4d} {100*by_src[s]/n:5.1f}%') + print('labels w/o p31 match but has p31:', sum(1 for c,t,s,w in res if s!='p31' and c['instanceOf'])) + print('cross-tab old faction x new type') + ct = collections.Counter((c['faction'], t) for c,t,s,w in res) + for f in sorted({c['faction'] for c in cards}): + print(f' {f:14s}', {t: ct[(f,t)] for t in TYPES if ct[(f,t)]}) + if '--list' in sys.argv: + for c,t,s,w in res: + if [i['label'] for i in c['instanceOf']] == ['human']: continue + print(f"{t[:5]} {s[:4]} | {c['title'][:42]:42s} | {w[:40]}")