Night 2: data analyst scripts behind reviews/data.md
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
"""How often random card pairs share cheap relations. Usage: python3 -I connected.py <card-sample.json>"""
|
||||
import json, sys, re, itertools, collections, importlib.util, os
|
||||
cards = json.load(open(sys.argv[1]))['cards']
|
||||
spec = importlib.util.spec_from_file_location('ty', os.path.join(os.path.dirname(os.path.abspath(__file__)), 'types.py'))
|
||||
ty = importlib.util.module_from_spec(spec); spec.loader.exec_module(ty)
|
||||
GENERIC = re.compile(r'stubs|living people|\d+ (births|deaths)$|articles|wikidata|pages', re.I)
|
||||
COUNTRIES = ['United States','American','England','English','British','France','French','Germany','German','Japan','Japanese','China','Chinese',
|
||||
'India','Indian','Brazil','Poland','Polish','Italy','Italian','Spain','Spanish','Austria','Canada','Canadian','Australia','Australian','Russia','Russian',
|
||||
'Mexico','Mexican','Iran','Pakistan','Bangladesh','Sweden','Swedish','Norway','Estonia','Ukraine','Croatia','Philippines','Cuba','Israel','Portugal','Scotland','Wales','Ireland','Kazakhstan','Malaysia','Argentin','Belgi','Iceland','Bermuda','Thailand','Singapore','Sri Lanka','Slovenia','Moldova','Cyprus','Macedonia','Mauritania','Congo','Gibraltar','Hong Kong','New Zealand','Korea','Indonesia','Aceh']
|
||||
NORM = {'American':'United States','English':'England','British':'United Kingdom','England':'United Kingdom','Scotland':'United Kingdom','Wales':'United Kingdom',
|
||||
'French':'France','German':'Germany','Japanese':'Japan','Chinese':'China','Indian':'India','Polish':'Poland','Italian':'Italy','Spanish':'Spain',
|
||||
'Canadian':'Canada','Australian':'Australia','Russian':'Russia','Mexican':'Mexico','Swedish':'Sweden'}
|
||||
Y = re.compile(r'\b(1[0-9]{3}|20[0-2][0-9])')
|
||||
def feats(c):
|
||||
txt = (c['description'] or '') + ' | ' + ' | '.join(c['categories'])
|
||||
ctry = {NORM.get(k, k) for k in COUNTRIES if k in txt}
|
||||
cents = {int(y) // 100 for y in Y.findall(c['title'] + ' ' + txt)}
|
||||
cats = {x for x in c['categories'] if not GENERIC.search(x)}
|
||||
p31 = {i['id'] for i in c['instanceOf']}
|
||||
return dict(ctry=ctry, cent=cents, cats=cats, allcats=set(c['categories']), p31=p31, type=ty.classify(c)[0])
|
||||
F = [feats(c) for c in cards]; pairs = list(itertools.combinations(range(len(cards)), 2)); N = len(pairs)
|
||||
def rate(f):
|
||||
k = sum(1 for i, j in pairs if f(F[i], F[j])); return f'{k}/{N} ({100*k/N:.2f}%)'
|
||||
print('pairs', N)
|
||||
print('share any category (incl. generic)', rate(lambda a, b: a['allcats'] & b['allcats']))
|
||||
print('share specific category', rate(lambda a, b: a['cats'] & b['cats']))
|
||||
print('share P31 class', rate(lambda a, b: a['p31'] & b['p31']))
|
||||
print('share P31 excl human/taxon', rate(lambda a, b: a['p31'] & b['p31'] - {'Q5', 'Q16521'}))
|
||||
print('share country (text proxy for P17/P27)', rate(lambda a, b: a['ctry'] & b['ctry']))
|
||||
print('share century (text proxy for dates)', rate(lambda a, b: a['cent'] & b['cent']))
|
||||
print('share type', rate(lambda a, b: a['type'] == b['type']))
|
||||
print('country OR century OR specific cat', rate(lambda a, b: (a['ctry'] & b['ctry']) or (a['cent'] & b['cent']) or (a['cats'] & b['cats'])))
|
||||
print('cards with >=1 country', sum(1 for f in F if f['ctry']), 'with >=1 century', sum(1 for f in F if f['cent']))
|
||||
# deck-level: chance that a random 10-card hand-pair (10 vs 10, own deck) has >=1 connected pair within a deck
|
||||
import random; random.seed(1)
|
||||
def deck_hits(pred, size=10, trials=2000):
|
||||
h = 0
|
||||
for _ in range(trials):
|
||||
s = random.sample(range(len(F)), size)
|
||||
if any(pred(F[i], F[j]) for i, j in itertools.combinations(s, 2)): h += 1
|
||||
return f'{100*h/trials:.0f}%'
|
||||
pc = lambda a, b: (a['ctry'] & b['ctry'])
|
||||
print('10-card deck has >=1 same-country pair', deck_hits(pc), '| specific category', deck_hits(lambda a, b: a['cats'] & b['cats']),
|
||||
'| country-or-century', deck_hits(lambda a, b: (a['ctry'] & b['ctry']) or (a['cent'] & b['cent'])))
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Field availability for rules-text generation in card-sample.json. Usage: python3 -I fields.py <card-sample.json>"""
|
||||
import json, sys, re, statistics, collections
|
||||
cards = json.load(open(sys.argv[1]))['cards']; n = len(cards)
|
||||
Y = re.compile(r'\b(1[0-9]{3}|20[0-2][0-9])\b')
|
||||
def share(f): k = sum(1 for c in cards if f(c)); return f'{k}/{n} ({100*k/n:.0f}%)'
|
||||
print('description non-empty', share(lambda c: bool(c['description'])))
|
||||
print('wikidataId', share(lambda c: bool(c['wikidataId'])))
|
||||
print('P31 present', share(lambda c: bool(c['instanceOf'])))
|
||||
print('>=1 non-hidden category', share(lambda c: len(c['categories']) > 0))
|
||||
print('categories truncated (cllimit 50)', share(lambda c: c['categoriesTruncated']))
|
||||
print('median categories', statistics.median(len(c['categories']) for c in cards))
|
||||
print('year in title', share(lambda c: Y.search(c['title'])))
|
||||
print('year in description', share(lambda c: Y.search(c['description'] or '')))
|
||||
print('year in categories', share(lambda c: any(Y.search(x) for x in c['categories'])))
|
||||
print('year anywhere (title/desc/cats)', share(lambda c: Y.search(c['title'] + ' ' + (c['description'] or '') + ' ' + ' '.join(c['categories']))))
|
||||
print('births/deaths category', share(lambda c: any(re.search(r'\d+ (births|deaths)', x) for x in c['categories'])))
|
||||
print('establishments/completed category', share(lambda c: any(re.search(r'establishments|completed in|disestablish|opened in', x) for x in c['categories'])))
|
||||
print('living people', share(lambda c: 'Living people' in c['categories']))
|
||||
print('stub category', share(lambda c: any('stubs' in x for x in c['categories'])))
|
||||
print('country/nationality word in desc or cats', share(lambda c: re.search(r'American|British|English|French|German|Japan|China|Chinese|India|Brazil|Poland|Polish|Italy|Italian|Spain|Spanish|Austria|Canada|Australia|Russia|United States', (c['description'] or '') + ' '.join(c['categories']))))
|
||||
print('legacy category on-chain', collections.Counter(c['legacyCategory'] for c in cards))
|
||||
print('pageviews available', share(lambda c: c['pageviews']['status'] == 'available'))
|
||||
print('languagesTruncated', share(lambda c: c['languagesTruncated']))
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Proposed percentile -> cost/power/toughness derivation, evaluated on card-sample.json.
|
||||
Usage: python3 -I stats.py <card-sample.json>"""
|
||||
import json, sys, collections, statistics, itertools
|
||||
d = json.load(open(sys.argv[1])); cards = d['cards']; P = d['percentiles']
|
||||
def pct(s, v): # DESIGN 1.1 counting: cut points strictly below + half of equal
|
||||
if v is None: return None
|
||||
cut = P[s]; return int(sum(c < v for c in cut) + sum(c == v for c in cut) / 2)
|
||||
TH = [26, 39, 51, 63, 78] # v1.1 costThresholds (fame -> cost 1..6)
|
||||
BUDGET = [None, 3, 4, 6, 8, 10, 12] # P+T per cost, MTG vanilla curve (2/1, 2/2, 3/3, 4/4, 5/5, 6/6)
|
||||
def derive(c):
|
||||
a, dp, r = (pct(k, c['stats'][k]) for k in ('attention', 'depth', 'reach'))
|
||||
vals = [x for x in (a, dp, r) if x is not None]
|
||||
fame = int(sum(vals) / len(vals) + 0.5)
|
||||
cost = 1 + sum(t <= fame for t in TH)
|
||||
b = BUDGET[cost]
|
||||
kw = 'Multilingual' if r >= 75 else ('Local' if c['stats']['reach'] == 0 else None)
|
||||
if kw == 'Multilingual': b -= 1 # evasion-like keyword is paid for out of the body
|
||||
if kw == 'Local': b += 0 # Local = gains +1/+1 next to a same-country Location (conditional, no tax)
|
||||
share = (a if a is not None else 50) / max(1, (a if a is not None else 50) + dp) # attention vs depth
|
||||
pw = min(b - 1, max(1, round(b * share))); t = b - pw
|
||||
return dict(title=c['title'], a=a, d=dp, r=r, fame=fame, cost=cost, P=pw, T=t, kw=kw,
|
||||
trend=c['pageviews']['trendPercent'], daily=c['pageviews']['dailyAverage'])
|
||||
rows = [derive(c) for c in cards]; n = len(rows)
|
||||
def naive(c): # straw man: power = attention tier, toughness = depth tier, cost from their sum
|
||||
ti = c['tiers']; pw = (ti['attention'] or 1 + 1) // 2 + 0; t = (ti['depth'] + 1) // 2
|
||||
return dict(cost=max(1, min(6, round((pw + t) / 2))), P=pw, T=t)
|
||||
nv = [naive(c) for c in cards]
|
||||
def dominated_pairs(rs, use_kw=False):
|
||||
k = 0
|
||||
for x, y in itertools.permutations(rs, 2):
|
||||
if x['cost'] <= y['cost'] and x['P'] >= y['P'] and x['T'] >= y['T'] and \
|
||||
(x['cost'] < y['cost'] or x['P'] > y['P'] or x['T'] > y['T']) and (not use_kw or x.get('kw') == y.get('kw')):
|
||||
k += 1
|
||||
return k
|
||||
print('cost distribution', sorted(collections.Counter(r['cost'] for r in rows).items()))
|
||||
print('power distribution', sorted(collections.Counter(r['P'] for r in rows).items()))
|
||||
print('toughness distribution', sorted(collections.Counter(r['T'] for r in rows).items()))
|
||||
print('keywords', collections.Counter(r['kw'] for r in rows))
|
||||
print('P/T by cost:')
|
||||
for cst in range(1, 7):
|
||||
g = [r for r in rows if r['cost'] == cst]
|
||||
if g: print(' ', cst, len(g), collections.Counter(f"{r['P']}/{r['T']}" for r in g).most_common(6))
|
||||
pairs = n * (n - 1)
|
||||
print('strict-dominance pairs proposed', dominated_pairs(rows), 'of', pairs, '; same-keyword only', dominated_pairs(rows, True))
|
||||
print('strict-dominance pairs naive ', dominated_pairs(nv), 'of', pairs)
|
||||
# efficiency by fame quintile
|
||||
q = sorted(rows, key=lambda r: r['fame'])
|
||||
print('fame quintile: mean cost, mean (P+T)/cost proposed | naive (P+T)/cost')
|
||||
idx = sorted(range(n), key=lambda i: rows[i]['fame'])
|
||||
for k in range(5):
|
||||
sl = idx[k * n // 5:(k + 1) * n // 5]
|
||||
print(f" Q{k+1} fame {rows[sl[0]]['fame']}-{rows[sl[-1]]['fame']}: cost {statistics.mean(rows[i]['cost'] for i in sl):.2f} "
|
||||
f"eff {statistics.mean((rows[i]['P']+rows[i]['T'])/rows[i]['cost'] for i in sl):.2f} | naive eff "
|
||||
f"{statistics.mean((nv[i]['P']+nv[i]['T'])/nv[i]['cost'] for i in sl):.2f}")
|
||||
top = sorted(rows, key=lambda r: -r['fame'])[:6] + sorted(rows, key=lambda r: r['fame'])[:3]
|
||||
print('examples (top fame, bottom fame):')
|
||||
for r in top: print(f" {r['title'][:34]:34s} a{r['a']} d{r['d']} r{r['r']} fame{r['fame']} cost{r['cost']} {r['P']}/{r['T']} {r['kw'] or ''}")
|
||||
tr = [r for r in rows if r['trend'] is not None and r['trend'] >= 25 and (r['daily'] or 0) >= 20]
|
||||
print('trending (+25% and daily>=20):', len(tr), f'{100*len(tr)/n:.1f}%')
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Classify card-sample.json into Figure/Location/Event/Institution/Doctrine.
|
||||
Order: P31 label rules (first rule that matches ANY P31 label wins, rule order = priority),
|
||||
then category keywords, then description keywords, then deterministic fallback.
|
||||
Usage: python3 -I types.py <card-sample.json> [--list]"""
|
||||
import json, re, sys, collections
|
||||
RULES_P31 = [ # (type, regex on a P31 label). Order matters: specific before generic.
|
||||
('Doctrine', r'aspect of|aspect in|disease|clinical sign|list article|chemical entity'),
|
||||
('Event', r'election|protest|battle|war\b|storm|nor.easter|cyclone|tornado|earthquake|disaster|event|match|final\b|'
|
||||
r'season|championship|tournament|\brace\b|grand prix|scandal|annexation|treaty|incident|festival|referendum|'
|
||||
r'conference|ceremony|time period|delegation'),
|
||||
('Institution', r'artificial satellite|space station|space telescope|spacecraft|space probe'),
|
||||
('Location', r'planet|natural satellite|\bstar\b|galaxy|nebula|planetary system|astronomical object|rainforest|reef|'
|
||||
r'civili[sz]ation|empire|historical country|archaeological site|'
|
||||
r'human settlement|settlement|village|town|city|commune|municipal|parish|county|district|region|province|'
|
||||
r'township|hamlet|townland|census area|local government|community of|island|river|lake|mountain|plateau|'
|
||||
r'waterfall|valley|bay|peak|geographical feature|historic district|park|state park|square|road|highway|'
|
||||
r'building|castle|mosque|church|temple|cathedral|house|home|museum|station|bridge|monument|historic site|'
|
||||
r'military base|battery|refinery|shopping|stadium|arch|tower|fort|palace|columbari|council'),
|
||||
('Figure', r'^human$|fictional character|stock character|person|deity|orisha|loa\b|taxon|species|genus|breed|'
|
||||
r'^family$|organism'),
|
||||
('Institution', r'company|business|organi[sz]ation|association|society|party|agency|studio|band|musical group|team|club|'
|
||||
r'university|college|school|institute|diocese|ecclesiastical|manufacturer|newspaper|magazine|film|album|'
|
||||
r'song|single|musical work|composition|television|series|book|novel|video game|painting|award|position|'
|
||||
r'aircraft|ship|steamship|vehicle|automobile|tram|railway line|route|typeface|software|product|brand'),
|
||||
('Doctrine', r'concept|theory|theorem|mathematical|programming language|language|gene|protein|chemical|compound|'
|
||||
r'disease|clinical sign|ideology|religion|discipline|field|cuisine|food|dish|drink|aspect|list article|'
|
||||
r'particle|process|genre|form\b|game\b|branch of|class of|type of|tabular'),
|
||||
]
|
||||
RULES_CAT = [ # fallback on non-hidden category titles (lowercase)
|
||||
('Figure', r'\bbirth|\bdeaths\b|living people|people from|\bspecies\b|described in \d|\bgenera\b|deities|characters'),
|
||||
('Event', r'\belections\b|protests|\bbattles\b|disasters|\braces\b|\bseasons\b|\bevents\b|scandals|annexations|competitions'),
|
||||
('Location', r'villages|communes|parishes|local government|populated places|geography|rivers|islands|buildings and structures|bridges|'
|
||||
r'stations|parks|districts|townships|cities and towns|historic places'),
|
||||
('Institution', r'companies|organi[sz]ations|\bfilms\b|albums|songs|novels|newspapers|television series|musical groups|'
|
||||
r'aircraft|ships|typefaces|software|awards|teams|clubs'),
|
||||
('Doctrine', r'theory|structures$|algebra|terminology|cuisine|drinks|desserts|ecology|languages|diseases|conditions|'
|
||||
r'concepts|ranks|etymology|maps|atlases'),
|
||||
]
|
||||
RULES_DESC = [
|
||||
('Figure', r'\b(painter|politician|actor|writer|player|born|species|genus)\b'),
|
||||
('Event', r'\b(race|election|battle|match|protest|storm|war|competition)\b'),
|
||||
('Location', r'\b(village|town|city|river|island|building|church|columbarium|station|park)\b'),
|
||||
('Institution', r'\b(company|band|film|album|organi[sz]ation|team|aircraft|ship|newspaper)\b'),
|
||||
('Doctrine', r'\b(concept|theory|structure|dessert|drink|dish|technique|class of)\b'),
|
||||
]
|
||||
TYPES = ['Figure', 'Location', 'Event', 'Institution', 'Doctrine']
|
||||
|
||||
def classify(card):
|
||||
labels = [i['label'].lower() for i in card.get('instanceOf') or []]
|
||||
for t, rx in RULES_P31:
|
||||
for l in labels:
|
||||
if re.search(rx, l): return t, 'p31', l
|
||||
cats = [c.lower() for c in card.get('categories') or [] if 'stubs' not in c.lower()]
|
||||
for t, rx in RULES_CAT:
|
||||
for c in cats:
|
||||
if re.search(rx, c): return t, 'category', c
|
||||
desc = (card.get('description') or '').lower()
|
||||
for t, rx in RULES_DESC:
|
||||
if re.search(rx, desc): return t, 'description', desc
|
||||
# deterministic last resort: Doctrine (the residual "idea" class); never hash-random.
|
||||
return 'Doctrine', 'fallback', ''
|
||||
|
||||
if __name__ == '__main__':
|
||||
d = json.load(open(sys.argv[1])); cards = d['cards']
|
||||
res = [(c, *classify(c)) for c in cards]
|
||||
n = len(res)
|
||||
by_t = collections.Counter(r[1] for r in res); by_src = collections.Counter(r[2] for r in res)
|
||||
print('n', n)
|
||||
for t in TYPES: print(f'{t:12s} {by_t[t]:4d} {100*by_t[t]/n:5.1f}%')
|
||||
for s in ['p31', 'category', 'description', 'fallback']: print(f'src {s:12s} {by_src[s]:4d} {100*by_src[s]/n:5.1f}%')
|
||||
print('labels w/o p31 match but has p31:', sum(1 for c,t,s,w in res if s!='p31' and c['instanceOf']))
|
||||
print('cross-tab old faction x new type')
|
||||
ct = collections.Counter((c['faction'], t) for c,t,s,w in res)
|
||||
for f in sorted({c['faction'] for c in cards}):
|
||||
print(f' {f:14s}', {t: ct[(f,t)] for t in TYPES if ct[(f,t)]})
|
||||
if '--list' in sys.argv:
|
||||
for c,t,s,w in res:
|
||||
if [i['label'] for i in c['instanceOf']] == ['human']: continue
|
||||
print(f"{t[:5]} {s[:4]} | {c['title'][:42]:42s} | {w[:40]}")
|
||||
Reference in New Issue
Block a user