"""Field availability for rules-text generation in card-sample.json. Usage: python3 -I fields.py """ import json, sys, re, statistics, collections cards = json.load(open(sys.argv[1]))['cards']; n = len(cards) Y = re.compile(r'\b(1[0-9]{3}|20[0-2][0-9])\b') def share(f): k = sum(1 for c in cards if f(c)); return f'{k}/{n} ({100*k/n:.0f}%)' print('description non-empty', share(lambda c: bool(c['description']))) print('wikidataId', share(lambda c: bool(c['wikidataId']))) print('P31 present', share(lambda c: bool(c['instanceOf']))) print('>=1 non-hidden category', share(lambda c: len(c['categories']) > 0)) print('categories truncated (cllimit 50)', share(lambda c: c['categoriesTruncated'])) print('median categories', statistics.median(len(c['categories']) for c in cards)) print('year in title', share(lambda c: Y.search(c['title']))) print('year in description', share(lambda c: Y.search(c['description'] or ''))) print('year in categories', share(lambda c: any(Y.search(x) for x in c['categories']))) print('year anywhere (title/desc/cats)', share(lambda c: Y.search(c['title'] + ' ' + (c['description'] or '') + ' ' + ' '.join(c['categories'])))) print('births/deaths category', share(lambda c: any(re.search(r'\d+ (births|deaths)', x) for x in c['categories']))) print('establishments/completed category', share(lambda c: any(re.search(r'establishments|completed in|disestablish|opened in', x) for x in c['categories']))) print('living people', share(lambda c: 'Living people' in c['categories'])) print('stub category', share(lambda c: any('stubs' in x for x in c['categories']))) print('country/nationality word in desc or cats', share(lambda c: re.search(r'American|British|English|French|German|Japan|China|Chinese|India|Brazil|Poland|Polish|Italy|Italian|Spain|Spanish|Austria|Canada|Australia|Russia|United States', (c['description'] or '') + ' '.join(c['categories'])))) print('legacy category on-chain', collections.Counter(c['legacyCategory'] for c in cards)) print('pageviews available', share(lambda c: c['pageviews']['status'] == 'available')) print('languagesTruncated', share(lambda c: c['languagesTruncated']))