Compare commits

..
Author SHA1 Message Date
eric 5dcdc1cdc6 Merge branch 'main' into v2.1 2026-11-01 22:07:09 +00:00
eric b88dfb1e94 deploy 2026-11-01 16:43:28 -05:00
eric b427fa4217 Mainnet 2026-10-29 11:45:51 -04:00
eric b3a1179703 Remove markets 2026-10-28 18:33:52 -04:00
eric eea2deb1fb Removed markets 2026-10-28 17:09:58 -04:00
eric 216a419c0b Observer integration step 2026-10-23 10:20:58 -04:00
eric bd3d36a7e1 Bounded test succeeded; native RandomX miner accepted one block 2026-10-22 21:25:39 -04:00
eric 176d1927a8 Added pilot-bound Monero observation persistence and the one-shot monero:pilot-observe command 2026-10-22 20:04:05 -04:00
eric 36963fed30 Added the guarded, one-shot monero:pilot-address command 2026-10-22 19:10:08 -04:00
eric 83d3d104c1 operator/library-only Wallet-RPC address creation and recovery 2026-10-22 18:55:05 -04:00
eric 2ecb42c019 Durable, beneficiary-bound subaddress index reservation 2026-10-22 17:26:13 -04:00
eric 84e6e08209 Validation-only pilot contract 2026-10-22 16:11:36 -04:00
eric 359f0df32d Separate identity-only pilot custody foundation 2026-10-22 15:40:37 -04:00
eric d8780701e6 monerod get_info 2026-10-22 15:22:23 -04:00
eric 3f04c3afa9 Real XMR pilot 2026-10-22 14:40:00 -04:00
eric 52559f65d7 Ledger-side credit and withdrawal-intent paths 2026-10-22 13:28:55 -04:00
eric 4fea176877 Opt-in, mainnet-only deposit-address API 2026-10-22 11:44:32 -04:00
eric b865807272 Signed, beneficiary-bount Monero subaddress enrollment 2026-10-22 11:33:38 -04:00
eric 5d607c1521 Removed SHKeeper 2026-10-22 10:45:32 -04:00
eric e835238d05 Hardened systemd deployment contract for read-only Monero observer 2026-10-22 10:08:15 -04:00
eric 158981e965 Implemented and validated the separate Monero observation-worker path 2026-10-22 09:38:20 -04:00
eric 12f80fdc53 Private CLI workflow for reviewed depowit operations 2026-10-22 02:09:07 -04:00
eric 05aead39ae Reviewed release and reorg-hold resolution 2026-10-21 22:24:42 -04:00
eric 9f82f050b4 Verified incoming transfers against Monero Wallet PRC 2026-10-21 21:34:46 -04:00
eric af646dcc32 First deposit integration slice 2026-10-21 20:07:27 -04:00
eric e3b2261da3 Stagenet rehearsal passed 2026-10-21 18:38:25 -04:00
eric 40f4aa6d4e Started Walet RPC auth work without adding MD5 Digest support 2026-10-21 17:33:33 -04:00
eric f00ea55dfd Stagenet wallet rehearsal passed 2026-10-21 16:53:59 -04:00
eric 0534cde1a9 Added wallet-identity check before allocation 2026-10-21 15:25:00 -04:00
eric c8cf8245fc Wallet subaddress creation with retry recovery 2026-10-21 15:12:05 -04:00
eric e70199ecb0 Implemented PostgreSQL v3 allocation-registry groundwork 2026-10-21 14:59:15 -04:00
eric ee5dc35330 Implemented the next deposit-observation step without activating live deposits 2026-10-21 14:09:55 -04:00
eric 79abaa78f2 Read-only Monero wallet-RPC observer 2026-10-21 13:05:57 -04:00
eric c07f3a737b Loopback HTTP benchmark 2026-10-21 12:40:00 -04:00
eric 74f269923c Moved canonical transaction, NFT, and mempool reads into a separate read-only Ledger SQLite connection 2026-10-21 12:24:53 -04:00
eric 4566271ce3 Moved canonical transaction, NFT, and mempool fact reads into the reconciliation worker's own read-only SQLite snapshot 2026-10-21 12:18:33 -04:00
eric 9282c0234d Moved reconciliation row selection and order/listing status evaluation into the worker thread 2026-10-21 11:35:37 -04:00
eric 7b6873921a Worker-owned reconciliation commits 2026-10-21 11:07:36 -04:00
eric a127a40293 Reconciliation uses 32-row SQLite slices 2026-10-21 10:29:45 -04:00
eric 6adfda3cf0 Extended benchmark 2026-10-21 10:14:34 -04:00
eric 687b628424 Market-journal reconciliation benchmark 2026-10-21 10:06:56 -04:00
eric 1f11210b5b Bounded reconciliation increment 2026-10-21 09:57:50 -04:00
eric 8b0e2287a5 Paired-backup verifier now checks more than the manifest hash 2026-10-21 02:39:02 -04:00
eric 0ac2073038 Cold-restore rehearsal 2026-10-20 23:18:14 -04:00
eric 5b21074462 Separate one-shot utility 2026-10-20 23:04:29 -04:00
eric ccc6eef951 Added native RandomX integration check 2026-10-20 20:20:31 -04:00
eric 7025c37390 Added opt-in canonical observer polling lifecycle 2026-10-20 20:11:57 -04:00
eric 8cfcf6c025 Strengthened the disposable reorg rehearsal 2026-10-20 19:31:08 -04:00
eric 22da9764eb Per-order terminal reorg linkage 2026-10-20 19:08:19 -04:00
eric e1fbf5e5af Disposable canonical-event inbox rehearsal 2026-10-20 18:50:02 -04:00
eric ca8ff48628 Paid-sale workflow 2026-10-20 17:38:34 -04:00
eric 621611ecd9 TLS 2026-10-20 17:09:22 -04:00
eric 447314440b Strengthen financial reconciliation checks 2026-10-20 16:56:13 -04:00
eric 7734d8ae90 Environment variables 2026-10-20 16:24:01 -04:00
eric 4cac26d84b Persistent PostgreSQL sandbox deployment 2026-10-20 16:02:18 -04:00
eric 84a505c4a5 NFT sale-lock protocol 2026-10-20 15:38:37 -04:00
eric c254c4c537 Custodial financial storage 2026-10-20 15:21:47 -04:00
eric e48081f041 Canonical indexes and recoverable market observation 2026-10-20 14:35:45 -04:00
eric fc65e497a8 Custodial markets 2026-10-20 14:04:59 -04:00
eric 4e753b35fe Recommendation 2026-10-20 13:08:57 -04:00
eric bcd8630a8e Planning 2026-10-20 13:00:08 -04:00
eric a1de67ea14 XMR native 2026-10-20 12:09:58 -04:00
eric 558427545f Production plan 2026-10-20 11:48:34 -04:00
eric d6f230d6b9 Marketplace documentation 2026-10-20 10:54:50 -04:00
eric 2814b46b73 Documentation and UI 2026-10-20 10:40:18 -04:00
eric f2bd3e38e2 RPC 2026-10-20 10:12:33 -04:00
eric 2dd7382004 SHKeeper 2026-10-20 10:06:38 -04:00
eric 55f26d3fad Deployment monitoring and recovery tooling 2026-10-20 09:49:41 -04:00
eric 9d61c177b7 Consensus calibration 2026-10-20 09:26:12 -04:00
eric 4080114085 encryption 2026-10-20 05:27:18 -04:00
eric 3cee89d023 Improvements 2026-10-20 04:07:37 -04:00
eric 0cc5032857 Security 2026-10-20 03:40:26 -04:00
eric 9850fd8554 Markets 2026-10-20 03:14:08 -04:00
eric b1d2b5c10e Trades 2026-10-19 21:47:38 -04:00
eric f8eeae826f Logo 2026-10-19 20:32:25 -04:00
eric baad477ca5 More scalable data storage 2026-10-19 20:17:10 -04:00
6 changed files with 0 additions and 8020 deletions
-98
View File
@@ -1,98 +0,0 @@
# Card data: real stat distributions (council-data)
Status: **interim**, generated 2026-11-16 23:06 EST. The full 330-card run is still fetching pageviews politely; this file and `card-sample.json` will be regenerated.
Method: `scripts/night/card_sample.py` draws random ns0 articles in batches of 20 and keeps those passing the `server/wiki.ts` filter (thumbnail from upload/thumb.wikimedia.org, intro extract of at least 40 characters). It enriches them with the same API parameters as `server/enrichment.ts`: six completed months Jun–Oct 2026; attention uses October. It adds Wikidata P31 and 30-day drift (views 10-17..11-15 vs 09-17..10-16). `scripts/night/card_analyze.py` produces the tables below.
## Headline findings (INTERIM: 100 fully enriched random illustrated cards; 330 sampled, rest still fetching under Wikimedia 429 rate limits)
1. **Raw stats are compressed and skewed.** Median card: attention 33, depth 63, reach 24 (p10–p90: attention 23–53, depth 54–71, reach 9–48). No card above 81 on anything; 100 is unreachable honestly. **Use the 1–10 tier tables below (sample deciles), not 0–100.**
2. **Correlation is weaker than feared, except attention↔depth (Spearman 0.59).** Reach is nearly independent of both (0.10 / 0.02): stub villages get many bot-made language editions. So reach is a real separate axis; attention+depth partly double-count fame.
3. **Underlying raw values are very long-tailed:** monthly views median 94 (p90 1,438, max 14k); pageBytes median 6 KB; languageCount median 4, and **10% have 0 other languages** (reach 0).
4. **Trend is mostly negative:** 69% of cards have trendPercent < 0 (median −10%), 31% positive, 14% swing by more than 50%. Null is rare (1%). A "Trending" keyword fires for about a third of cards: fine as a small ±1.
5. **Live values are fairly stable in score terms:** 30-day pageview drift median |18%| (p90 55%), but that moves the attention score by only a median of **1.4 points** (p90 5.6). So "live vs. mint" differences are small for random cards; big swings only come from deliberate inflation (see the critic's point A4).
6. **The legacy `category()` buckets are unusable as factions:** Culture 53%, People 30%, Nature 9%, History 5%, Science 3%, **Space 0%**.
7. **Candidate factions from Wikidata P31 (instance of), clean and readable:** People 31%, Places 25%, Nature 10%, Structures 6%, Works 5%, Events 5%, Sport 3%, Ideas & Tech 3%, Organisations 2%, unclassified 10%. Suggest **6 factions**: People, Places, Nature, Works & Culture (Works + Organisations + Sport), Structures, Events & Ideas (Events + Ideas & Tech), plus a neutral fallback. People and Places are big, so faction bonuses should scale with *share in deck*, not raw count, or the small factions need their own perks.
8. **Starter/loaner list:** frozen stats for up to 4 cards per legacy category, spread across tiers, in `docs/game/starter-cards.json` (Space has no sample cards: hand-pick e.g. the Moon from `shared/catalog.ts`).
## Full tables
Sample: **100 cards**; ~44% (149/340 logged during sampling) of random pages pass the illustrated filter. Pageviews available for 100.
| stat | n | min | p10 | p25 | median | p75 | p90 | max | mean | skew |
|---|---|---|---|---|---|---|---|---|---|---|
| attention | 100 | 13 | 23 | 27 | 33 | 41 | 53 | 69 | 35 | 0.75 |
| depth | 100 | 50 | 54 | 59 | 63 | 68 | 71 | 81 | 63 | 0.24 |
| reach | 100 | 0 | 9 | 16 | 24 | 36 | 48 | 59 | 26 | 0.11 |
| pageBytes | 100 | 998 | 1804 | 3407 | 6018 | 12512 | 19161 | 74169 | 9885 | 3.20 |
| languageCount | 100 | 0 | 1 | 2 | 4 | 11 | 27 | 56 | 9 | 1.83 |
| monthlyViews | 100 | 5 | 22 | 43 | 94 | 277 | 1438 | 14374 | 611 | 5.76 |
| dailyAverage | 100 | 0.2 | 0.7 | 1.4 | 3.0 | 8.9 | 46.4 | 463.7 | 19.7 | 5.76 |
| trendPercent | 99 | -86.6 | -45.1 | -26.4 | -10.4 | 4.5 | 21.1 | 187.1 | -9.5 | 1.83 |
| drift30Percent | 99 | -94.7 | -45.9 | -30.7 | -12.7 | -1.7 | 14.8 | 72.1 | -16.1 | -0.20 |
**Spearman rank correlations** (cards with pageviews):
| | attention | depth | reach | trendPercent | drift30Percent |
|---|---|---|---|---|---|
| attention | 1.00 | 0.59 | 0.10 | 0.24 | 0.25 |
| depth | 0.59 | 1.00 | 0.02 | 0.07 | 0.05 |
| reach | 0.10 | 0.02 | 1.00 | -0.10 | 0.24 |
| trendPercent | 0.24 | 0.07 | -0.10 | 1.00 | 0.53 |
| drift30Percent | 0.25 | 0.05 | 0.24 | 0.53 | 1.00 |
(n=99; attention~monthlyViews, depth~pageBytes, reach~languageCount are monotonic so their rank correlations are identical.)
**trendPercent sign:** positive 31 (31%), negative 68 (69%), zero 0; null (prior month 0 or no views) 1. |trend| > 50%: 14; > 100%: 1.
**30-day drift** (views 10-17..11-15 vs 09-17..10-16): median |drift| 18%, p90 |drift| 55%. Attention score change implied: median 1.4 points, p90 5.6 points.
**attention histogram (deciles 0-9,10-19..90-100):** 0 4 30 39 12 11 4 0 0 0
**depth histogram (deciles 0-9,10-19..90-100):** 0 0 0 0 0 30 53 15 2 0
**reach histogram (deciles 0-9,10-19..90-100):** 10 25 23 23 10 9 0 0 0 0
Reach = 0 (no other languages): 10; reach 100 (>=999 langs) impossible in practice, max langs 56. Attention null: 0; attention 0: 0.
**Sum of three stats:** min 84, p10 97, p25 107, median 120, p75 140, p90 155, max 193
**Legacy `category()` (server/wiki.ts):** Culture 53, People 30, Nature 9, History 5, Science 3
**Top Wikidata P31 classes:** human 30, taxon 8, river 4, village 4, commune of France 3, musical group 2, building 2, castle 2, rural municipality of Austria 1, municipality of Austria 1, municipality of Brazil 1, television film 1, nor'easter 1, rail company (Japan) 1, village of Poland 1, municipality 1, civil parish 1, association football match 1, final 1, local government area of Victoria 1, plant disease 1, basketball team season 1, mosque 1, functional programming language 1, purely functional programming language 1, dependently typed programming language 1, programming language 1, free software 1, proof assistant 1, municipality of Croatia 1, municipality of Bangladesh 1, town 1, town in the United States 1, municipality of Georgia 1, epiphyte 1, protest 1, region of Mauritania 1, legislative election 1, musical work/composition 1, plateau 1
No Wikidata item: 0; no P31: 6
**Most frequent category words (cards containing):** stubs 35, births 27, people 26, from 24, century 22, living 16, geography 15, deaths 14, american 13, county 9, university 8, male 7, united 7, film 7, players 7, district 6, states 6, sports 6, biography 6, television 5, alumni 5, genera 5, established 5, villages 5, football 5, city 5, buildings 5, expatriate 5, sportspeople 5, sportsmen 5, towns 4, municipalities 4, writers 4, women 4, faculty 4, communes 4, rivers 4, places 4, establishments 4, actors 4, california 4, russian 4, association 4, structures 4, described 3, british 3, french 3, ohio 3, academics 3, based 3, japanese 3, pennsylvania 3, argentine 3, australia 3, populated 3, directors 3, rock 3, musical 3, conference 3, division 3
## Rank-normalised tiers (1–10)
Tier = 1 + number of breakpoints the score is **>=**. Breakpoints are the sample deciles p10..p90 of the 0–100 score. Ties make some tiers thin (scores are integers).
| stat | p10 | p20 | p30 | p40 | p50 | p60 | p70 | p80 | p90 |
|---|---|---|---|---|---|---|---|---|---|
| attention | 23 | 25 | 28 | 31 | 33 | 36 | 38 | 45 | 53 |
| depth | 54 | 57 | 60 | 61 | 63 | 65 | 67 | 69 | 72 |
| reach | 10 | 10 | 16 | 20 | 26 | 30 | 36 | 39 | 49 |
Resulting tier counts — attention: None:0 1:10 2:9 3:7 4:14 5:8 6:12 7:7 8:12 9:10 10:11
; depth: 1:7 2:11 3:12 4:6 5:11 6:11 7:9 8:9 9:14 10:10
; reach: 1:10 2:0 3:13 4:12 5:15 6:8 7:11 8:10 9:11 10:10
Starter candidates: 19 written to /tmp/claude-1000/-home-eric-worker-jobs-night-council-data/d4e31987-ce7f-4f28-8262-8f0225215102/scratchpad/starter.json: Counter({'Nature': 4, 'People': 4, 'History': 4, 'Culture': 4, 'Science': 3})
## Candidate factions (Wikidata P31 rules)
| faction | n | share | median attention | median depth | median reach | example |
|---|---|---|---|---|---|---|
| People | 31 | 31% | 33 | 65 | 26 | Salahuddin of Aceh |
| Places | 25 | 25% | 28 | 61 | 36 | St. Georgen am Längsee |
| Nature | 10 | 10% | 30 | 56 | 31 | Dichromorpha viridis |
| Other | 10 | 10% | 52 | 64 | 22 | Village (Pennsylvania) |
| Structures | 6 | 6% | 32 | 58 | 16 | Pathrail Mosque |
| Works | 5 | 5% | 40 | 64 | 16 | The Best of Men |
| Events | 5 | 5% | 38 | 72 | 10 | April 2021 nor'easter |
| Sport | 3 | 3% | 25 | 71 | 0 | 1912 Argentine Primera División final |
| Ideas & Tech | 3 | 3% | 37 | 62 | 20 | Agda (programming language) |
| Organisations | 2 | 2% | 50 | 62 | 24 | Hiroshima Electric Railway |
Unclassified P31 labels: typeface 1, annexation 1, position 1, family 1
File diff suppressed because it is too large Load Diff
-424
View File
@@ -1,424 +0,0 @@
{
"frozenAt": "2026-11-17",
"note": "Frozen loaner/starter stats from the council-data random sample. Do not fetch live.",
"cards": [
{
"pageId": 49830722,
"title": "Dicroidium odontopteroides",
"category": "Nature",
"wikidataId": "Q25095289",
"revisionId": 1116073630,
"pageBytes": 1943,
"languageCount": 0,
"monthlyViews": 59,
"latestMonth": "2026-10",
"trendPercent": 5.4,
"stats": {
"attention": 30,
"depth": 55,
"reach": 0
},
"tiers": {
"attention": 4,
"depth": 2,
"reach": 1
}
},
{
"pageId": 25856905,
"title": "Dichromorpha viridis",
"category": "Nature",
"wikidataId": "Q5272562",
"revisionId": 1367363150,
"pageBytes": 1347,
"languageCount": 4,
"monthlyViews": 133,
"latestMonth": "2026-10",
"trendPercent": -43.6,
"stats": {
"attention": 35,
"depth": 52,
"reach": 23
},
"tiers": {
"attention": 6,
"depth": 1,
"reach": 5
}
},
{
"pageId": 66508497,
"title": "Amanita mairei",
"category": "Nature",
"wikidataId": "Q15651951",
"revisionId": 1323515500,
"pageBytes": 998,
"languageCount": 8,
"monthlyViews": 61,
"latestMonth": "2026-10",
"trendPercent": -21.8,
"stats": {
"attention": 30,
"depth": 50,
"reach": 32
},
"tiers": {
"attention": 4,
"depth": 1,
"reach": 7
}
},
{
"pageId": 4289899,
"title": "Anthodon (reptile)",
"category": "Nature",
"wikidataId": "Q2505064",
"revisionId": 1384682008,
"pageBytes": 9037,
"languageCount": 12,
"monthlyViews": 233,
"latestMonth": "2026-10",
"trendPercent": -8.3,
"stats": {
"attention": 39,
"depth": 66,
"reach": 37
},
"tiers": {
"attention": 8,
"depth": 7,
"reach": 8
}
},
{
"pageId": 47519878,
"title": "Jeanne Madden",
"category": "People",
"wikidataId": "Q20974011",
"revisionId": 1382761423,
"pageBytes": 1817,
"languageCount": 4,
"monthlyViews": 26,
"latestMonth": "2026-10",
"trendPercent": -40.9,
"stats": {
"attention": 24,
"depth": 54,
"reach": 23
},
"tiers": {
"attention": 2,
"depth": 2,
"reach": 5
}
},
{
"pageId": 46263191,
"title": "Max Touloute",
"category": "People",
"wikidataId": "Q20641573",
"revisionId": 1384441370,
"pageBytes": 10140,
"languageCount": 2,
"monthlyViews": 47,
"latestMonth": "2026-10",
"trendPercent": 95.8,
"stats": {
"attention": 28,
"depth": 67,
"reach": 16
},
"tiers": {
"attention": 4,
"depth": 8,
"reach": 4
}
},
{
"pageId": 3372934,
"title": "Hermes Ramírez",
"category": "People",
"wikidataId": "Q522855",
"revisionId": 1364063954,
"pageBytes": 8264,
"languageCount": 13,
"monthlyViews": 59,
"latestMonth": "2026-10",
"trendPercent": -9.2,
"stats": {
"attention": 30,
"depth": 65,
"reach": 38
},
"tiers": {
"attention": 4,
"depth": 7,
"reach": 8
}
},
{
"pageId": 1374812,
"title": "Tom Ewell",
"category": "People",
"wikidataId": "Q427388",
"revisionId": 1381663979,
"pageBytes": 17965,
"languageCount": 30,
"monthlyViews": 4548,
"latestMonth": "2026-10",
"trendPercent": -19.8,
"stats": {
"attention": 61,
"depth": 71,
"reach": 50
},
"tiers": {
"attention": 10,
"depth": 9,
"reach": 10
}
},
{
"pageId": 13799810,
"title": "Levophacetoperane",
"category": "Science",
"wikidataId": "Q6535818",
"revisionId": 1374230485,
"pageBytes": 4639,
"languageCount": 2,
"monthlyViews": 165,
"latestMonth": "2026-10",
"trendPercent": -1.8,
"stats": {
"attention": 37,
"depth": 61,
"reach": 16
},
"tiers": {
"attention": 7,
"depth": 5,
"reach": 4
}
},
{
"pageId": 36740675,
"title": "The Best of Men",
"category": "Science",
"wikidataId": "Q7717125",
"revisionId": 1372732350,
"pageBytes": 6002,
"languageCount": 4,
"monthlyViews": 476,
"latestMonth": "2026-10",
"trendPercent": -3.4,
"stats": {
"attention": 45,
"depth": 63,
"reach": 23
},
"tiers": {
"attention": 9,
"depth": 6,
"reach": 5
}
},
{
"pageId": 4426773,
"title": "Agda (programming language)",
"category": "Science",
"wikidataId": "Q20479",
"revisionId": 1384528612,
"pageBytes": 14254,
"languageCount": 11,
"monthlyViews": 2439,
"latestMonth": "2026-10",
"trendPercent": 7.8,
"stats": {
"attention": 56,
"depth": 69,
"reach": 36
},
"tiers": {
"attention": 10,
"depth": 9,
"reach": 8
}
},
{
"pageId": 42770753,
"title": "Château de la Faye (Villexavier)",
"category": "History",
"wikidataId": "Q2971223",
"revisionId": 1375492681,
"pageBytes": 1851,
"languageCount": 1,
"monthlyViews": 14,
"latestMonth": "2026-10",
"trendPercent": 7.7,
"stats": {
"attention": 20,
"depth": 54,
"reach": 10
},
"tiers": {
"attention": 1,
"depth": 2,
"reach": 3
}
},
{
"pageId": 15362336,
"title": "1954 Gold Coast general election",
"category": "History",
"wikidataId": "Q2539674",
"revisionId": 1351181171,
"pageBytes": 4141,
"languageCount": 2,
"monthlyViews": 181,
"latestMonth": "2026-10",
"trendPercent": -10.4,
"stats": {
"attention": 38,
"depth": 60,
"reach": 16
},
"tiers": {
"attention": 8,
"depth": 4,
"reach": 4
}
},
{
"pageId": 9026000,
"title": "Names of Poland",
"category": "History",
"wikidataId": "Q6961536",
"revisionId": 1351068420,
"pageBytes": 18966,
"languageCount": 4,
"monthlyViews": 1431,
"latestMonth": "2026-10",
"trendPercent": -0.9,
"stats": {
"attention": 53,
"depth": 71,
"reach": 23
},
"tiers": {
"attention": 10,
"depth": 9,
"reach": 5
}
},
{
"pageId": 25275009,
"title": "Amur Annexation",
"category": "History",
"wikidataId": "Q4379152",
"revisionId": 1381503728,
"pageBytes": 15490,
"languageCount": 9,
"monthlyViews": 3238,
"latestMonth": "2026-10",
"trendPercent": -6.7,
"stats": {
"attention": 59,
"depth": 70,
"reach": 33
},
"tiers": {
"attention": 10,
"depth": 9,
"reach": 7
}
},
{
"pageId": 7674700,
"title": "Shipley Plateau",
"category": "Culture",
"wikidataId": "Q7498025",
"revisionId": 1384061257,
"pageBytes": 1745,
"languageCount": 1,
"monthlyViews": 22,
"latestMonth": "2026-10",
"trendPercent": -55.1,
"stats": {
"attention": 23,
"depth": 54,
"reach": 10
},
"tiers": {
"attention": 2,
"depth": 2,
"reach": 3
}
},
{
"pageId": 15810707,
"title": "Charmont-en-Beauce",
"category": "Culture",
"wikidataId": "Q667131",
"revisionId": 1381828954,
"pageBytes": 1811,
"languageCount": 32,
"monthlyViews": 23,
"latestMonth": "2026-10",
"trendPercent": 27.8,
"stats": {
"attention": 23,
"depth": 54,
"reach": 51
},
"tiers": {
"attention": 2,
"depth": 2,
"reach": 10
}
},
{
"pageId": 48653101,
"title": "Tornadoes of 1978",
"category": "Culture",
"wikidataId": "Q22093730",
"revisionId": 1381828221,
"pageBytes": 16373,
"languageCount": 0,
"monthlyViews": 511,
"latestMonth": "2026-10",
"trendPercent": 3.2,
"stats": {
"attention": 45,
"depth": 70,
"reach": 0
},
"tiers": {
"attention": 9,
"depth": 9,
"reach": 1
}
},
{
"pageId": 981340,
"title": "Tatsunoko Production",
"category": "Culture",
"wikidataId": "Q1323459",
"revisionId": 1384823392,
"pageBytes": 74169,
"languageCount": 19,
"monthlyViews": 4738,
"latestMonth": "2026-10",
"trendPercent": -9.4,
"stats": {
"attention": 61,
"depth": 81,
"reach": 43
},
"tiers": {
"attention": 10,
"depth": 10,
"reach": 9
}
}
]
}
-98
View File
@@ -1,98 +0,0 @@
# WikiDeck Night 1: team handbook
Read this whole file before doing anything. It's the contract between about 17 agents on 4 machines.
**Coordinator:** the Claude session on tower-plus (Eric's interactive session).
**Lead/integrator:** `night-lead` on tower-plus.
**Window:** 2026-11-16 22:45 until **01:30 EST, hard stop**.
**Milestone:** https://gitea.eom.dev/claude/wikideck/milestone/63
## Goal
Turn WikiDeck into a game:
1. A **fun, robust, balance-tested card game**, playable in the browser against a private testnet.
2. **eom.dev accounts:** log in through Dex SSO, and link a self-custodial wallet to your account.
3. **Peer-hosted game modes:** node operators publish their own modes as an incentive to run a peer.
4. **An Android app**, built and running in an emulator, and an **iOS app** (macOS VM; Xcode sign-in waits for Eric).
## Hard rules (no exceptions)
- **Where to push:** only to `git@gitea.eom.dev:claude/wikideck.git`. Never push to `WikiDeck/wikideck` (upstream) or to `main` on this fork.
- **Branches:** every change goes on a branch named `night/<team>/<topic>`, through a PR into **`night/2026-11-16`**. Only `night-lead` merges.
- **Live services:** don't touch the live wikideck deployment (k8s namespace `wikideck`) or mainnet. The testnet is private, on poweredge-r350.
- **No consensus or protocol changes:**
- no block, transfer or trade format changes
- nothing that changes validation in `server/chain.ts` or `shared/protocol.ts`
- the game runs **off-chain**
- if you think you need a protocol change, write it up in your issue instead
- **Production SSO:** don't change Dex or any other production service yourself. The SSO team prepares the Dex change, and the coordinator reviews and applies it.
- **Nothing public:** only `night-lead` and the scheduled standup post on Discourse (planning topic only). No other public posts, no purchases or sign-ups, and no secrets in git.
- **The printer and CNC:** inspiron-3670 drives a 3D printer and a CNC router. Never start, resume or send a print or cut.
- **Git identity** in every clone:
```
git config user.name Claude
git config user.email claude@eom.dev
git config commit.gpgsign false
```
End commit messages with `Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>`.
- **Before opening a PR:** `npm test` and `npm run build` must pass. If something can't pass yet, say so in the PR.
## How we coordinate
**Gitea is the shared brain.** The machines don't share a filesystem, and workers can't message each other.
- **Your issue is your inbox.**
- Prefix every comment with your agent name, e.g. `[engine-a]`.
- Check your issue and your PRs for new comments **at least every 15 minutes** and whenever you finish a step.
- **Status labels on your issue:**
- `status/in-progress` while you're working
- `status/needs-review` when a PR is up
- `status/blocked` with a comment explaining why
- `status/done` when finished
- **Need another team?** Comment on *their* issue, starting with `[from <you>]`.
- **Contracts beat conversations.** Interfaces live in files on the integration branch (`docs/game/*.md`, TypeScript types). Read them before asking.
- **When the lead (or the coordinator) messages you directly** through herdr, the message is a peer request within this plan. Act on it.
- **Stuck for more than 10 minutes?** Label `status/blocked` and comment. Don't spin.
## Resources (be a good neighbour)
- **inspiron-3670** (6 cores, 60 GB RAM) has 7 agents, the Android emulator and a macOS VM. Wrap heavy commands in a shared lock:
```
flock /tmp/night-heavy.lock <npm ci | npm run build | gradle ...>
```
- **imac** (8 threads, about 4 GB free) and **poweredge-r350** (16 threads, about 6 GB free): keep builds to one at a time.
- **LocalAI** (`https://localai.eom.dev`) is shared production with slow single-GPU inference. Don't use it in loops.
## Timeline (EST)
| Time | Milestone |
|---|---|
| 22:45 | Kickoff. The design council researches. Infra teams set up (testnet, Android SDK, macOS, SSO scaffolding, UI shell, beta fixes). |
| **23:30** | **Design v1 frozen** (`docs/game/DESIGN.md`). The engine team implements it. |
| 23:45 | **Engine API contract** (`docs/game/ENGINE_API.md` plus TypeScript types) on the integration branch, so UI, simulation and modes can build against it. |
| 00:15 | Engine alpha. Simulation starts bot matches. The UI wires up real matches. |
| 00:45 | Balance report v1. The council tunes the rules (v1.1). |
| **01:10** | **Code freeze:** fixes only. The lead integrates. QA plays end to end. |
| 01:25 | Final reports in each issue. The lead writes `docs/night/REPORT.md`. |
| **01:30** | **Stop.** |
## Teams
| Agent | Machine | Issue |
|---|---|---|
| council-designer-a, council-designer-b, council-data | imac | Game design council, card data analysis |
| council-critic, night-lead, night-review, macos-fetch | tower-plus | Design council (critic), integration and QA, iOS (download) |
| engine-a, engine-b | inspiron-3670 | Game engine |
| ui-a, ui-b | inspiron-3670 | Game UI and AI opponent, beta fixes |
| modes | inspiron-3670 | Peer-hosted game modes |
| mobile-android, mobile-ios | inspiron-3670 | Android app, iOS app and macOS VM |
| sim-testnet, sim-balance, sso | poweredge-r350 | Testnet and simulation, eom.dev accounts |
## What the cards hold (game design input)
Each card's data comes from `shared/enrichment.ts`: `PageEnrichment` and `MintStatsSnapshot`.
- **Identity:** `title`, `description`, `wikidataId` (structured facts: instance-of, dates, coordinates...) and `categories[]`
- **Size and history:** `pageBytes` and `revisionId`
- **Reach:** `languageCount`, the number of language editions
- **Pageviews:** monthly views, `total`, `dailyAverage` and `trendPercent`
- **Derived 0–100-ish scores:** `stats.attention`, `stats.depth` and `stats.reach`
- **Two values for everything:** an immutable **at-mint snapshot** and **live current values** that change over time
- **Duplicates:** the same article can be minted more than once, each copy with its own snapshot
Two facts to design around:
- **New players own no cards.** The design must give them a way to play immediately, for example starter decks of preview or loaner cards that don't touch NFTs.
- **Cards are NFTs.** A match must never move or burn cards on-chain tonight.
-139
View File
@@ -1,139 +0,0 @@
#!/usr/bin/env python3
"""Summarise card-sample.json into markdown stats. Usage: python3 -I card_analyze.py sample.json > out.md"""
import json, sys, math, statistics as st, collections, re
d = json.load(open(sys.argv[1])); C = d['cards']
def q(v):
v = sorted(v); n = len(v)
def p(x):
k = (n - 1) * x; f = math.floor(k); c = min(f + 1, n - 1); return v[f] + (v[c] - v[f]) * (k - f)
return [v[0], p(.1), p(.25), p(.5), p(.75), p(.9), v[-1]]
def skew(v):
m, s = st.mean(v), st.pstdev(v); return sum((x - m) ** 3 for x in v) / len(v) / s ** 3 if s else 0
def rank(v):
o = sorted(range(len(v)), key=lambda i: v[i]); r = [0] * len(v); i = 0
while i < len(o):
j = i
while j + 1 < len(o) and v[o[j + 1]] == v[o[i]]: j += 1
for k in range(i, j + 1): r[o[k]] = (i + j) / 2
i = j + 1
return r
def pearson(a, b):
ma, mb = st.mean(a), st.mean(b); sa = math.sqrt(sum((x - ma) ** 2 for x in a)); sb = math.sqrt(sum((y - mb) ** 2 for y in b))
return sum((x - ma) * (y - mb) for x, y in zip(a, b)) / (sa * sb) if sa and sb else float('nan')
pv = [c for c in C if c['pageviews']['status'] == 'available']
F = {'attention': lambda c: c['stats']['attention'], 'depth': lambda c: c['stats']['depth'], 'reach': lambda c: c['stats']['reach'],
'pageBytes': lambda c: c['pageBytes'], 'languageCount': lambda c: c['languageCount'],
'monthlyViews': lambda c: c['pageviews']['months'][5]['views'] if c['pageviews']['status'] == 'available' else None,
'dailyAverage': lambda c: c['pageviews']['dailyAverage'], 'trendPercent': lambda c: c['pageviews']['trendPercent'],
'drift30Percent': lambda c: c['drift30']['pct']}
out = []
P = out.append
rate = f"{len(C)/d['rawPagesSeen']*100:.0f}%" if d['rawPagesSeen'] else '~44% (149/340 logged during sampling)'
P(f"Sample: **{len(C)} cards**; {rate} of random pages pass the illustrated filter. Pageviews available for {len(pv)}.\n")
P('| stat | n | min | p10 | p25 | median | p75 | p90 | max | mean | skew |'); P('|---|---|---|---|---|---|---|---|---|---|---|')
for k, f in F.items():
v = [f(c) for c in C if f(c) is not None]
qs = q(v); fmt = (lambda x: f'{x:.0f}') if k not in ('trendPercent', 'drift30Percent', 'dailyAverage') else (lambda x: f'{x:.1f}')
P(f"| {k} | {len(v)} | " + ' | '.join(fmt(x) for x in qs) + f" | {fmt(st.mean(v))} | {skew(v):.2f} |")
P('\n**Spearman rank correlations** (cards with pageviews):\n')
keys = ['attention', 'depth', 'reach', 'trendPercent', 'drift30Percent']
rows = [c for c in pv if c['pageviews']['trendPercent'] is not None and c['drift30']['pct'] is not None]
R = {k: rank([F[k](c) for c in rows]) for k in keys}
P('| | ' + ' | '.join(keys) + ' |'); P('|---' * (len(keys) + 1) + '|')
for a in keys: P(f'| {a} | ' + ' | '.join(f'{pearson(R[a], R[b]):.2f}' for b in keys) + ' |')
P(f'\n(n={len(rows)}; attention~monthlyViews, depth~pageBytes, reach~languageCount are monotonic so their rank correlations are identical.)\n')
tp = [c['pageviews']['trendPercent'] for c in pv if c['pageviews']['trendPercent'] is not None]
P(f"**trendPercent sign:** positive {sum(t>0 for t in tp)} ({sum(t>0 for t in tp)/len(tp)*100:.0f}%), negative {sum(t<0 for t in tp)} ({sum(t<0 for t in tp)/len(tp)*100:.0f}%), zero {sum(t==0 for t in tp)}; null (prior month 0 or no views) {len(C)-len(tp)}. "
f"|trend| > 50%: {sum(abs(t)>50 for t in tp)}; > 100%: {sum(abs(t)>100 for t in tp)}.\n")
dr = [c['drift30']['pct'] for c in C if c['drift30']['pct'] is not None]
P(f"**30-day drift** (views 10-17..11-15 vs 09-17..10-16): median |drift| {st.median([abs(x) for x in dr]):.0f}%, "
f"p90 |drift| {q([abs(x) for x in dr])[5]:.0f}%. Attention score change implied: "
f"median {st.median([abs(100*math.log10((c['drift30']['last30']+1)/(c['drift30']['prev30']+1))/6) for c in C]):.1f} points, "
f"p90 {q([abs(100*math.log10((c['drift30']['last30']+1)/(c['drift30']['prev30']+1))/6) for c in C])[5]:.1f} points.\n")
for name, f in [('attention', F['attention']), ('depth', F['depth']), ('reach', F['reach'])]:
v = [f(c) for c in C if f(c) is not None]; h = collections.Counter(min(x // 10, 9) for x in v)
P(f"**{name} histogram (deciles 0-9,10-19..90-100):** " + ' '.join(str(h.get(i, 0)) for i in range(10)))
P(f"\nReach = 0 (no other languages): {sum(c['languageCount']==0 for c in C)}; reach 100 (>=999 langs) impossible in practice, max langs {max(c['languageCount'] for c in C)}. "
f"Attention null: {sum(c['stats']['attention'] is None for c in C)}; attention 0: {sum(c['stats']['attention']==0 for c in C)}.\n")
s3 = [sum(F[k](c) or 0 for k in ('attention', 'depth', 'reach')) for c in C]
P(f"**Sum of three stats:** " + ', '.join(f'{n} {x:.0f}' for n, x in zip(['min', 'p10', 'p25', 'median', 'p75', 'p90', 'max'], q(s3))) + '\n')
P('**Legacy `category()` (server/wiki.ts):** ' + ', '.join(f'{k} {v}' for k, v in collections.Counter(c['legacyCategory'] for c in C).most_common()) + '\n')
io = collections.Counter(x['label'] for c in C for x in c['instanceOf'])
P('**Top Wikidata P31 classes:** ' + ', '.join(f'{k} {v}' for k, v in io.most_common(40)) + '\n')
P(f"No Wikidata item: {sum(not c['wikidataId'] for c in C)}; no P31: {sum(not c['instanceOf'] for c in C)}\n")
words = collections.Counter()
for c in C:
for cat in set(w for t in c['categories'] for w in re.findall(r'[a-z]+', t.lower()) if len(w) > 3): words[cat] += 1
P('**Most frequent category words (cards containing):** ' + ', '.join(f'{k} {v}' for k, v in words.most_common(60)) + '\n')
print('\n'.join(out))
# ---- rank normalisation table (critic request): score -> 1..10 tier by sample decile
def tiers(name):
v = sorted(F[name](c) for c in C if F[name](c) is not None)
return [v[min(len(v) - 1, int(len(v) * k / 10))] for k in range(1, 10)]
T = {k: tiers(k) for k in ('attention', 'depth', 'reach')}
print('\n## Rank-normalised tiers (1–10)\n')
print('Tier = 1 + number of breakpoints the score is **>=**. Breakpoints are the sample deciles p10..p90 of the 0–100 score. '
'Ties make some tiers thin (scores are integers).\n')
print('| stat | p10 | p20 | p30 | p40 | p50 | p60 | p70 | p80 | p90 |'); print('|---' * 10 + '|')
for k, b in T.items(): print(f'| {k} | ' + ' | '.join(str(x) for x in b) + ' |')
def tier(k, s): return None if s is None else 1 + sum(s >= x for x in T[k])
for c in C: c['tiers'] = {k: tier(k, c['stats'][k]) for k in T}
tc = collections.Counter(c['tiers']['attention'] for c in C)
print('\nResulting tier counts — attention: ' + ' '.join(f'{t}:{tc.get(t,0)}' for t in [None] + list(range(1, 11))))
for k in ('depth', 'reach'):
tc = collections.Counter(c['tiers'][k] for c in C); print(f'; {k}: ' + ' '.join(f'{t}:{tc.get(t,0)}' for t in range(1, 11)))
json.dump({'breakpoints': T, 'rule': 'tier = 1 + count(breakpoints <= score); attention null -> no tier'},
open(sys.argv[2] if len(sys.argv) > 2 else '/dev/null', 'w'), indent=1)
# ---- starter loaner candidates: 4 per legacy category, spread over combined tier
if len(sys.argv) > 3:
pick = []
for cat in ['Space', 'Nature', 'People', 'Science', 'History', 'Culture']:
pool = sorted((c for c in C if c['legacyCategory'] == cat and c['stats']['attention'] is not None),
key=lambda c: sum(c['tiers'].values()))
if not pool: continue
n = min(4, len(pool)); idx = sorted({round(i * (len(pool) - 1) / max(1, n - 1)) for i in range(n)})
for i in idx:
c = pool[i]
pick.append({'pageId': c['pageId'], 'title': c['title'], 'category': cat, 'wikidataId': c['wikidataId'],
'revisionId': c['revisionId'], 'pageBytes': c['pageBytes'], 'languageCount': c['languageCount'],
'monthlyViews': c['pageviews']['months'][5]['views'], 'latestMonth': c['pageviews']['latestMonth'],
'trendPercent': None if c['pageviews']['trendPercent'] is None else round(c['pageviews']['trendPercent'], 1),
'stats': c['stats'], 'tiers': c['tiers']})
json.dump({'frozenAt': d['generatedAt'][:10], 'note': 'Frozen loaner/starter stats from the council-data random sample. Do not fetch live.',
'cards': pick}, open(sys.argv[3], 'w'), indent=1, ensure_ascii=False)
print(f'\nStarter candidates: {len(pick)} written to {sys.argv[3]}: ' + collections.Counter(p['category'] for p in pick).__repr__())
# ---- candidate factions from Wikidata P31 (+ categories fallback)
FAC = [
('People', r'\bhuman\b|fictional character|person'),
('Sport', r'season|sports team|football|match|tournament|championship|final\b|club|league|olympic|race\b'),
('Nature', r'taxon|species|genus|plant|fungus|disease|epiphyte|mineral|bird|fish|insect'),
('Places', r'village|town|city|commune|municipality|river|lake|mountain|county|district|parish|region|plateau|island|settlement|neighbo|locality|census|hamlet|valley|bay|area|province|state of|capital|suburb|stream|peak|unincorporated|human settlement|local government'),
('Structures', r'building|castle|mosque|church|cathedral|temple|bridge|station|railway line|road|highway|museum|structure|stadium|dam|tower|house|school|hospital|airport|monument|fort|palace|lighthouse|park'),
('Works', r'film|album|single|song|musical|television|book|novel|video game|painting|literary|composition|series|episode|magazine|newspaper|poem|opera|play\b|artwork|sculpture'),
('Organisations', r'company|group|band|organi[sz]ation|university|college|party|business|enterprise|agency|institute|association|brand|union|ministry|military unit'),
('Events', r'election|protest|battle|war\b|storm|nor.easter|cyclone|event|disaster|treaty|conflict|earthquake|incident|festival|referendum|conference|ceremony'),
('Ideas & Tech', r'programming|software|language|chemical|compound|gene|protein|theorem|concept|technology|vehicle|aircraft|ship|weapon|class|model|device|algorithm|religion|discipline|theory|food|dish'),
]
def faction(c):
text = ' | '.join(x['label'].lower() for x in c['instanceOf'])
for name, rx in FAC:
if text and re.search(rx, text): return name
cats = ' | '.join(c['categories']).lower()
if re.search(r'births|deaths|people', cats): return 'People'
if re.search(r'populated|villages|towns|geography|rivers', cats): return 'Places'
if re.search(r'films|albums|songs|novels|television|video games', cats): return 'Works'
return 'Other'
for c in C: c['faction'] = faction(c)
print('\n## Candidate factions (Wikidata P31 rules)\n')
print('| faction | n | share | median attention | median depth | median reach | example |'); print('|---|---|---|---|---|---|---|')
by = collections.defaultdict(list)
for c in C: by[c['faction']].append(c)
for name, cs in sorted(by.items(), key=lambda kv: -len(kv[1])):
med = lambda k: st.median([x['stats'][k] for x in cs if x['stats'][k] is not None] or [0])
print(f"| {name} | {len(cs)} | {len(cs)/len(C)*100:.0f}% | {med('attention'):.0f} | {med('depth'):.0f} | {med('reach'):.0f} | {cs[0]['title']} |")
other = collections.Counter(x['label'] for c in by.get('Other', []) for x in c['instanceOf'])
print('\nUnclassified P31 labels: ' + ', '.join(f'{k} {v}' for k, v in other.most_common(25)))
if len(sys.argv) > 4: json.dump({c['pageId']: c['faction'] for c in C}, open(sys.argv[4], 'w'))
-158
View File
@@ -1,158 +0,0 @@
#!/usr/bin/env python3
"""Sample random illustrated en.wikipedia articles the way server/wiki.ts selects them and
enrich them the way server/enrichment.ts does. Polite: ~4 req/s, sequential, disk cache.
Usage: python3 -I card_sample.py <cache_dir> <out.json> [target=320]"""
import hashlib, json, os, re, sys, time, urllib.parse, urllib.request, calendar, datetime as dt
UA = 'WikiDeckCardDataSampler/0.1 (experimental card-game stat research; python-urllib)'
CACHE, OUT = sys.argv[1], sys.argv[2]
TARGET = int(sys.argv[3]) if len(sys.argv) > 3 else 320
os.makedirs(CACHE, exist_ok=True)
_last = [0.0]
def get(url, cache=True):
key = os.path.join(CACHE, hashlib.sha1(url.encode()).hexdigest() + '.json')
if cache and os.path.exists(key):
return json.load(open(key))
if os.environ.get('CACHE_ONLY') and 'wikidata.org' not in url:
return {'__missing': True}
for attempt in range(6):
wait = (1.0 if 'en.wikipedia.org' in url else 0.25) - (time.time() - _last[0])
if wait > 0: time.sleep(wait)
_last[0] = time.time()
req = urllib.request.Request(url, headers={'User-Agent': UA, 'Accept': 'application/json'})
try:
with urllib.request.urlopen(req, timeout=20) as r:
data = json.loads(r.read().decode())
break
except urllib.error.HTTPError as e:
if e.code == 404: data = {'__404': True}; break
if e.code == 429 or e.code >= 500:
ra = e.headers.get('retry-after'); w = int(ra) + 1 if ra and ra.isdigit() else 5 * (attempt + 1)
print('http', e.code, 'sleep', w, file=sys.stderr); time.sleep(min(w, 60)); continue
raise
except Exception as e:
print('retry', attempt, type(e).__name__, e, file=sys.stderr); time.sleep(2 * (attempt + 1))
else:
raise RuntimeError('failed ' + url)
if cache: json.dump(data, open(key, 'w'))
return data
def api(params, cache=True):
return get('https://en.wikipedia.org/w/api.php?' + urllib.parse.urlencode({'format': 'json', 'formatversion': '2', **params}), cache)
def legacy_category(title, cats):
t = (title + ' ' + ' '.join(cats)).lower()
if re.search(r'astronom|planet|space|galax|moon|star\b', t): return 'Space'
if re.search(r'animal|plant|species|ocean|biolog|mollusc', t): return 'Nature'
if re.search(r'births|deaths|people|scientist', t): return 'People'
if re.search(r'physic|chem|math|science|technolog', t): return 'Science'
if re.search(r'histor|ancient|archaeolog|empire', t): return 'History'
return 'Culture'
def score(v, scale):
import math
return min(100, round(math.log10(v + 1) / scale * 100))
# 1. random illustrated articles (same filter as wiki.ts: thumbnail + extract >= 40 chars)
picked, seen, batches, raw_pages = [], set(), 0, 0
state_file = os.path.join(CACHE, '_random_state.json')
if os.path.exists(state_file):
picked = json.load(open(state_file)); seen = {p['pageId'] for p in picked}
while len(picked) < TARGET:
try: d = api({'action': 'query', 'generator': 'random', 'grnnamespace': '0', 'grnlimit': '20',
'prop': 'extracts|pageimages|info|categories', 'exintro': '1', 'explaintext': '1', 'exchars': '900',
'exlimit': '20', 'piprop': 'thumbnail', 'pithumbsize': '800', 'cllimit': '500'}, cache=False)
except RuntimeError as e: print(e, file=sys.stderr); time.sleep(5); continue
batches += 1
for p in d.get('query', {}).get('pages', []):
raw_pages += 1
thumb = (p.get('thumbnail') or {}).get('source', '')
ext = (p.get('extract') or '').strip()
if not re.match(r'https://(upload|thumb)\.wikimedia\.org/', thumb) or len(ext) < 40 or p['pageid'] in seen: continue
seen.add(p['pageid'])
picked.append({'pageId': p['pageid'], 'title': p['title'],
'legacyCategory': legacy_category(p['title'], [c['title'] for c in p.get('categories', [])])})
json.dump(picked, open(state_file, 'w'))
print(f'random batch {batches}: {len(picked)} illustrated / {raw_pages} raw', file=sys.stderr)
picked = picked[:TARGET]
# 2. enrichment per article (same params as enrichment.ts) + daily views for drift
now = dt.datetime(2026, 11, 16, tzinfo=dt.timezone.utc)
months = []
for i in range(6):
m = now.month - 6 + i; y = now.year + (m - 1) // 12; m = (m - 1) % 12 + 1
months.append(f'{y:04d}-{m:02d}')
# batched metadata (same props as enrichment.ts; per-page cllimit 50 is emulated by truncating)
meta = {}
ids = [p['pageId'] for p in picked]
for i in range(0, len(ids), 50):
chunk = ids[i:i + 50]; cont = {}; pages = {}
while True:
md = api({'action': 'query', 'pageids': '|'.join(map(str, chunk)), 'prop': 'info|categories|pageprops|description|langlinks',
'clshow': '!hidden', 'cllimit': '500', 'lllimit': '500', 'ppprop': 'wikibase_item', **cont})
for pg in md['query']['pages']:
cur = pages.setdefault(pg['pageid'], {**pg, 'categories': [], 'langlinks': []})
cur['categories'] += pg.get('categories', []); cur['langlinks'] += pg.get('langlinks', [])
if 'continue' not in md: break
cont = md['continue']
meta.update(pages)
print(f'metadata {len(meta)}/{len(ids)}', file=sys.stderr)
cards = []
for n, pk in enumerate(picked):
page = meta[pk['pageId']]
cont = {'clcontinue': 1} if len(page['categories']) > 50 else {}
page['categories'] = page['categories'][:50]
langs = len(page.get('langlinks', []))
t = urllib.parse.quote(page['title'].replace(' ', '_'), safe='')
daily = get(f'https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia.org/all-access/user/{t}/daily/{months[0].replace("-", "")}01/20261115')
if daily.get('__missing'): continue
dmap = {it['timestamp'][:8]: it['views'] for it in daily.get('items', [])}
pv = {'status': 'unavailable', 'months': [], 'latestMonth': months[5], 'total': None, 'dailyAverage': None, 'trendPercent': None}
if dmap:
ordered = []
for m in months:
y, mo = map(int, m.split('-')); nd = calendar.monthrange(y, mo)[1]
ordered.append({'month': m, 'views': sum(dmap.get(f'{y:04d}{mo:02d}{d:02d}', 0) for d in range(1, nd + 1))})
latest, prior = ordered[5]['views'], ordered[4]['views']
nd = calendar.monthrange(*map(int, months[5].split('-')))[1]
pv = {'status': 'available', 'months': ordered, 'latestMonth': months[5], 'total': sum(o['views'] for o in ordered),
'dailyAverage': latest / nd, 'trendPercent': (latest - prior) / prior * 100 if prior > 0 else None}
# 30-day drift: views 2026-10-17..11-15 vs 2026-09-17..10-16
def window(start, days):
s = dt.date.fromisoformat(start)
return sum(dmap.get((s + dt.timedelta(d)).strftime('%Y%m%d'), 0) for d in range(days))
last30, prev30 = window('2026-10-17', 30), window('2026-09-17', 30)
pb = page['length']
cards.append({'pageId': page['pageid'], 'title': page['title'], 'description': page.get('description'),
'wikidataId': (page.get('pageprops') or {}).get('wikibase_item'), 'revisionId': page['lastrevid'], 'pageBytes': pb,
'languageCount': langs, 'languagesTruncated': 'llcontinue' in cont,
'categories': [c['title'].removeprefix('Category:') for c in page.get('categories', [])],
'categoriesTruncated': 'clcontinue' in cont, 'fetchedAt': now.isoformat(), 'pageviews': pv,
'stats': {'attention': score(pv['months'][5]['views'], 6) if pv['status'] == 'available' else None,
'depth': score(pb, 6), 'reach': score(langs, 3)},
'legacyCategory': pk['legacyCategory'],
'drift30': {'last30': last30, 'prev30': prev30, 'pct': (last30 - prev30) / prev30 * 100 if prev30 else None}})
if n % 25 == 0: print(f'enriched {n + 1}/{len(picked)}', file=sys.stderr)
# 3. Wikidata P31 (instance of) + labels
qids = [c['wikidataId'] for c in cards if c['wikidataId']]
p31 = {}
for i in range(0, len(qids), 50):
d = get('https://www.wikidata.org/w/api.php?' + urllib.parse.urlencode({'action': 'wbgetentities', 'format': 'json',
'ids': '|'.join(qids[i:i + 50]), 'props': 'claims'}))
for q, ent in d.get('entities', {}).items():
p31[q] = [c['mainsnak']['datavalue']['value']['id'] for c in ent.get('claims', {}).get('P31', [])
if c['mainsnak'].get('datavalue')]
classes = sorted({c for v in p31.values() for c in v})
labels = {}
for i in range(0, len(classes), 50):
d = get('https://www.wikidata.org/w/api.php?' + urllib.parse.urlencode({'action': 'wbgetentities', 'format': 'json',
'ids': '|'.join(classes[i:i + 50]), 'props': 'labels', 'languages': 'en'}))
for q, ent in d.get('entities', {}).items():
labels[q] = ent.get('labels', {}).get('en', {}).get('value', q)
for c in cards:
c['instanceOf'] = [{'id': q, 'label': labels.get(q, q)} for q in p31.get(c['wikidataId'], [])]
json.dump({'generatedAt': dt.datetime.now(dt.timezone.utc).isoformat(), 'method': 'random ns0 batches of 20, filter as server/wiki.ts (thumbnail + extract>=40); enrichment as server/enrichment.ts; monthly views summed from daily REST pageviews; P31 from Wikidata',
'randomBatches': batches, 'rawPagesSeen': raw_pages, 'cards': cards}, open(OUT, 'w'), indent=1, ensure_ascii=False)
print('done', len(cards), file=sys.stderr)