forked from WikiDeck/wikideck
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2ad4d022aa | ||
|
|
f544019bfe |
@@ -28,7 +28,3 @@ dist-ssr
|
||||
*.njsproj
|
||||
*.sln
|
||||
*.sw?
|
||||
|
||||
# Native RandomX engine for the app (scripts/build-randomx-android.sh)
|
||||
native/build-android-*/
|
||||
android/app/src/main/jniLibs/
|
||||
|
||||
@@ -16,13 +16,6 @@ android {
|
||||
ignoreAssetsPattern '!.svn:!.git:!.ds_store:!*.scc:.*:!CVS:!thumbs.db:!picasa.ini:!*~'
|
||||
}
|
||||
}
|
||||
packagingOptions {
|
||||
jniLibs {
|
||||
// The RandomX engine (jniLibs/<abi>/libwikideck_randomx.so) is an executable, not a JNI library.
|
||||
// Android may exec it only from the extracted nativeLibraryDir, so keep legacy extraction.
|
||||
useLegacyPackaging true
|
||||
}
|
||||
}
|
||||
buildTypes {
|
||||
release {
|
||||
minifyEnabled false
|
||||
|
||||
@@ -1,14 +1,5 @@
|
||||
package dev.eom.wikideck;
|
||||
|
||||
import android.os.Bundle;
|
||||
|
||||
import com.getcapacitor.BridgeActivity;
|
||||
|
||||
public class MainActivity extends BridgeActivity {
|
||||
@Override
|
||||
public void onCreate(Bundle savedInstanceState) {
|
||||
// Local plugins must be registered before the bridge starts.
|
||||
registerPlugin(MinerPlugin.class);
|
||||
super.onCreate(savedInstanceState);
|
||||
}
|
||||
}
|
||||
public class MainActivity extends BridgeActivity {}
|
||||
|
||||
@@ -1,264 +0,0 @@
|
||||
package dev.eom.wikideck;
|
||||
|
||||
import android.content.BroadcastReceiver;
|
||||
import android.content.Context;
|
||||
import android.content.Intent;
|
||||
import android.content.IntentFilter;
|
||||
import android.os.BatteryManager;
|
||||
import android.os.Build;
|
||||
import android.os.PowerManager;
|
||||
|
||||
import com.getcapacitor.JSObject;
|
||||
import com.getcapacitor.Plugin;
|
||||
import com.getcapacitor.PluginCall;
|
||||
import com.getcapacitor.PluginMethod;
|
||||
import com.getcapacitor.annotation.CapacitorPlugin;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Native RandomX engine for the WikiDeck app (sideload build only, see docs/night2/MOBILE_MINING.md).
|
||||
*
|
||||
* The plugin only runs the pinned `wikideck-randomx` executable (shipped as jniLibs/<abi>/libwikideck_randomx.so)
|
||||
* in LIGHT mode and speaks its line protocol (docs/randomx-native.md). It never sees wallet keys: the WebView builds
|
||||
* the work blob, signs the block claim with the web wallet, and submits it to the node. This class also enforces the
|
||||
* device safeguards: engines are killed when the app goes to the background, on battery saver, when the device is
|
||||
* hot, or when the battery is low and not charging. Each stop is reported to the WebView as a `halted` event.
|
||||
*/
|
||||
@CapacitorPlugin(name = "WikiDeckMiner")
|
||||
public class MinerPlugin extends Plugin {
|
||||
private static final String BINARY = "libwikideck_randomx.so";
|
||||
private static final int MAX_THREADS = 4;
|
||||
/** At or above Android's THERMAL_STATUS_MODERATE we stop. */
|
||||
private static final int THERMAL_LIMIT = 2;
|
||||
/** Battery temperature limit in tenths of a degree Celsius. */
|
||||
private static final int BATTERY_TEMP_LIMIT = 400;
|
||||
private static final int LOW_BATTERY_PERCENT = 20;
|
||||
private static final Pattern HEX = Pattern.compile("^[0-9a-f]+$");
|
||||
|
||||
private final List<Engine> engines = new ArrayList<>();
|
||||
private BroadcastReceiver receiver;
|
||||
private Object thermalListener;
|
||||
|
||||
/** One long-lived engine process with exactly one outstanding command (it is strictly request/response). */
|
||||
private static final class Engine {
|
||||
final Process process;
|
||||
final OutputStream in;
|
||||
final BufferedReader out;
|
||||
final ExecutorService worker = Executors.newSingleThreadExecutor();
|
||||
Engine(Process process) {
|
||||
this.process = process;
|
||||
this.in = process.getOutputStream();
|
||||
this.out = new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.US_ASCII));
|
||||
}
|
||||
String command(String line) throws Exception {
|
||||
in.write((line + "\n").getBytes(StandardCharsets.US_ASCII));
|
||||
in.flush();
|
||||
String reply = out.readLine();
|
||||
if (reply == null) throw new Exception("RandomX engine exited");
|
||||
return reply;
|
||||
}
|
||||
void kill() {
|
||||
worker.shutdownNow();
|
||||
try { in.close(); } catch (Exception ignored) { }
|
||||
process.destroy();
|
||||
try { if (!process.waitFor(1, TimeUnit.SECONDS)) process.destroyForcibly(); } catch (InterruptedException ignored) { }
|
||||
}
|
||||
}
|
||||
|
||||
private File binary() {
|
||||
return new File(getContext().getApplicationInfo().nativeLibraryDir, BINARY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void load() {
|
||||
receiver = new BroadcastReceiver() {
|
||||
@Override public void onReceive(Context context, Intent intent) { checkAndHalt(); }
|
||||
};
|
||||
IntentFilter filter = new IntentFilter();
|
||||
filter.addAction(PowerManager.ACTION_POWER_SAVE_MODE_CHANGED);
|
||||
filter.addAction(Intent.ACTION_BATTERY_CHANGED);
|
||||
getContext().registerReceiver(receiver, filter);
|
||||
if (Build.VERSION.SDK_INT >= 29) {
|
||||
PowerManager pm = (PowerManager) getContext().getSystemService(Context.POWER_SERVICE);
|
||||
PowerManager.OnThermalStatusChangedListener l = status -> checkAndHalt();
|
||||
pm.addThermalStatusListener(l);
|
||||
thermalListener = l;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void handleOnPause() {
|
||||
halt("background");
|
||||
super.handleOnPause();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void handleOnDestroy() {
|
||||
halt("closed");
|
||||
try { getContext().unregisterReceiver(receiver); } catch (Exception ignored) { }
|
||||
if (Build.VERSION.SDK_INT >= 29 && thermalListener != null) {
|
||||
PowerManager pm = (PowerManager) getContext().getSystemService(Context.POWER_SERVICE);
|
||||
pm.removeThermalStatusListener((PowerManager.OnThermalStatusChangedListener) thermalListener);
|
||||
}
|
||||
super.handleOnDestroy();
|
||||
}
|
||||
|
||||
/** Current device state and, if mining must not run, the reason. */
|
||||
private JSObject conditionsObject() {
|
||||
PowerManager pm = (PowerManager) getContext().getSystemService(Context.POWER_SERVICE);
|
||||
Intent battery = getContext().registerReceiver(null, new IntentFilter(Intent.ACTION_BATTERY_CHANGED));
|
||||
int temp = battery == null ? -1 : battery.getIntExtra(BatteryManager.EXTRA_TEMPERATURE, -1);
|
||||
int level = battery == null ? -1 : battery.getIntExtra(BatteryManager.EXTRA_LEVEL, -1);
|
||||
int scale = battery == null ? -1 : battery.getIntExtra(BatteryManager.EXTRA_SCALE, -1);
|
||||
int plugged = battery == null ? 0 : battery.getIntExtra(BatteryManager.EXTRA_PLUGGED, 0);
|
||||
int percent = level >= 0 && scale > 0 ? Math.round(100f * level / scale) : -1;
|
||||
int thermal = Build.VERSION.SDK_INT >= 29 ? pm.getCurrentThermalStatus() : 0;
|
||||
boolean powerSave = pm.isPowerSaveMode();
|
||||
String block = null;
|
||||
if (powerSave) block = "battery-saver";
|
||||
else if (thermal >= THERMAL_LIMIT) block = "thermal";
|
||||
else if (temp >= BATTERY_TEMP_LIMIT) block = "battery-hot";
|
||||
else if (plugged == 0 && percent >= 0 && percent < LOW_BATTERY_PERCENT) block = "battery-low";
|
||||
JSObject o = new JSObject();
|
||||
o.put("powerSave", powerSave);
|
||||
o.put("thermalStatus", thermal);
|
||||
o.put("batteryTempC", temp < 0 ? null : temp / 10.0);
|
||||
o.put("batteryPercent", percent);
|
||||
o.put("charging", plugged != 0);
|
||||
o.put("cores", Runtime.getRuntime().availableProcessors());
|
||||
o.put("abi", Build.SUPPORTED_ABIS.length > 0 ? Build.SUPPORTED_ABIS[0] : "unknown");
|
||||
o.put("engineInstalled", binary().canExecute());
|
||||
o.put("blocked", block);
|
||||
return o;
|
||||
}
|
||||
|
||||
private void checkAndHalt() {
|
||||
String block = conditionsObject().getString("blocked");
|
||||
if (block != null) halt(block);
|
||||
}
|
||||
|
||||
private synchronized void halt(String reason) {
|
||||
if (engines.isEmpty()) return;
|
||||
for (Engine e : engines) e.kill();
|
||||
engines.clear();
|
||||
JSObject event = new JSObject();
|
||||
event.put("reason", reason);
|
||||
notifyListeners("halted", event);
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public void conditions(PluginCall call) {
|
||||
call.resolve(conditionsObject());
|
||||
}
|
||||
|
||||
/** Runs upstream's known-vector test (Hash test 1a/2a) on this device. */
|
||||
@PluginMethod
|
||||
public void selfTest(PluginCall call) {
|
||||
new Thread(() -> {
|
||||
try {
|
||||
Process p = new ProcessBuilder(binary().getAbsolutePath(), "--self-test").redirectErrorStream(true).start();
|
||||
p.getOutputStream().close();
|
||||
StringBuilder text = new StringBuilder();
|
||||
BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream(), StandardCharsets.UTF_8));
|
||||
for (String line; (line = r.readLine()) != null && text.length() < 2000; ) text.append(line).append('\n');
|
||||
boolean done = p.waitFor(120, TimeUnit.SECONDS);
|
||||
if (!done) p.destroyForcibly();
|
||||
JSObject o = new JSObject();
|
||||
o.put("ok", done && p.exitValue() == 0);
|
||||
o.put("output", text.toString().trim());
|
||||
call.resolve(o);
|
||||
} catch (Exception e) {
|
||||
call.reject("Self-test failed to run: " + e.getMessage());
|
||||
}
|
||||
}).start();
|
||||
}
|
||||
|
||||
/** Starts 1..4 light-mode engines (more than 2 only while charging) (each needs about 256 MiB). Refuses if a safeguard is active. */
|
||||
@PluginMethod
|
||||
public synchronized void start(PluginCall call) {
|
||||
int threads = call.getInt("threads", 1);
|
||||
if (threads < 1 || threads > MAX_THREADS || threads > Runtime.getRuntime().availableProcessors()) { call.reject("threads must be 1 to " + MAX_THREADS + " and at most the core count"); return; }
|
||||
String block = conditionsObject().getString("blocked");
|
||||
if (block == null && threads > 2 && !conditionsObject().getBool("charging")) block = "max-needs-charging";
|
||||
if (block != null) { call.reject("Mining is paused: " + block, block); return; }
|
||||
if (!engines.isEmpty()) { for (Engine e : engines) e.kill(); engines.clear(); }
|
||||
try {
|
||||
for (int i = 0; i < threads; i++) {
|
||||
// No arguments = LIGHT mode. `--fast` (about 2 GiB) is never used on phones.
|
||||
Process p = new ProcessBuilder(binary().getAbsolutePath()).redirectError(ProcessBuilder.Redirect.to(new File("/dev/null"))).start();
|
||||
engines.add(new Engine(p));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
for (Engine en : engines) en.kill();
|
||||
engines.clear();
|
||||
call.reject("Could not start the RandomX engine: " + e.getMessage());
|
||||
return;
|
||||
}
|
||||
JSObject o = new JSObject();
|
||||
o.put("threads", threads);
|
||||
call.resolve(o);
|
||||
}
|
||||
|
||||
@PluginMethod
|
||||
public synchronized void stop(PluginCall call) {
|
||||
for (Engine e : engines) e.kill();
|
||||
engines.clear();
|
||||
call.resolve();
|
||||
}
|
||||
|
||||
/**
|
||||
* One bounded batch on engine `lane`: `mine <key> <blob> <target> <nonceOffset> <maxAttempts>`.
|
||||
* Resolves {found:false} on exhaustion or {found:true, hash, nonce} (nonce as a decimal string).
|
||||
*/
|
||||
@PluginMethod
|
||||
public void mine(PluginCall call) {
|
||||
String key = call.getString("key", ""), blob = call.getString("blob", ""), target = call.getString("target", "");
|
||||
int lane = call.getInt("lane", 0), offset = call.getInt("nonceOffset", -1), attempts = call.getInt("maxAttempts", 0);
|
||||
if (key.length() != 64 || target.length() != 64 || blob.length() < 2 || blob.length() > 16384 || blob.length() % 2 != 0
|
||||
|| !HEX.matcher(key).matches() || !HEX.matcher(target).matches() || !HEX.matcher(blob).matches()
|
||||
|| offset < 0 || offset + 8 > blob.length() / 2 || attempts < 1 || attempts > 10000) {
|
||||
call.reject("Invalid mining request");
|
||||
return;
|
||||
}
|
||||
final Engine engine;
|
||||
synchronized (this) {
|
||||
if (lane < 0 || lane >= engines.size()) { call.reject("Miner is not running", "stopped"); return; }
|
||||
engine = engines.get(lane);
|
||||
}
|
||||
String line = "mine " + key + " " + blob + " " + target + " " + offset + " " + attempts;
|
||||
try {
|
||||
engine.worker.execute(() -> {
|
||||
try {
|
||||
String reply = engine.command(line);
|
||||
JSObject o = new JSObject();
|
||||
if (reply.equals("exhausted")) {
|
||||
o.put("found", false);
|
||||
} else {
|
||||
String[] parts = reply.split(" ");
|
||||
if (parts.length != 3 || !parts[0].equals("found") || parts[1].length() != 64 || !HEX.matcher(parts[1]).matches()
|
||||
|| !parts[2].matches("^[0-9]{1,20}$")) throw new Exception("Malformed engine reply");
|
||||
o.put("found", true);
|
||||
o.put("hash", parts[1]);
|
||||
o.put("nonce", parts[2]);
|
||||
}
|
||||
call.resolve(o);
|
||||
} catch (Exception e) {
|
||||
call.reject("Miner stopped: " + e.getMessage(), "stopped");
|
||||
}
|
||||
});
|
||||
} catch (Exception e) {
|
||||
call.reject("Miner is not running", "stopped");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,15 +1,11 @@
|
||||
#!/bin/sh
|
||||
# Build the WikiDeck debug APK against the private Night-1 testnet.
|
||||
# Needs ANDROID_HOME (with an NDK and CMake for the RandomX engine) and a JDK 21 (JAVA_HOME). Output: android/app/build/outputs/apk/debug/app-debug.apk
|
||||
# Needs ANDROID_HOME and a JDK 21 (JAVA_HOME). Output: android/app/build/outputs/apk/debug/app-debug.apk
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
export VITE_WIKIDECK_NETWORK="${VITE_WIKIDECK_NETWORK:-testnet}"
|
||||
export VITE_WIKIDECK_GENESIS_TIME="${VITE_WIKIDECK_GENESIS_TIME:-1792411200}"
|
||||
export VITE_NODE_URL="${VITE_NODE_URL:-http://192.168.1.32:3101}"
|
||||
# Native RandomX engine (jniLibs/<abi>/libwikideck_randomx.so); built once, reused after (scripts/build-randomx-android.sh).
|
||||
for abi in ${ABIS:-arm64-v8a x86_64}; do
|
||||
[ -x "android/app/src/main/jniLibs/$abi/libwikideck_randomx.so" ] || ABIS="$abi" sh scripts/build-randomx-android.sh
|
||||
done
|
||||
npm run build
|
||||
npx cap sync android
|
||||
cd android && ./gradlew --no-daemon -q assembleDebug
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
# Mobile mining (Android, sideload build)
|
||||
|
||||
**Status (Night 2, 2026-11-17):** it works. The emulator mined and got testnet block #92 accepted with the in-app native RandomX engine (see "Results tonight"). Issue: claude/wikideck#59.
|
||||
|
||||
BRIEF decision 4: the mobile app must be able to mine, because otherwise mobile players can never earn the 40 owned cards a deck needs. Night 1's `isNativeApp()` hid the mining panel and the app had no miner at all.
|
||||
|
||||
## App stores: why this is a sideloaded APK
|
||||
**Google Play and the Apple App Store both prohibit on-device cryptocurrency mining**, so the mining build cannot ship through either store. It is distributed as a sideloaded APK only.
|
||||
- **Google Play**, Developer Program Policy, *Device and Network Abuse*: "We don't allow apps that mine cryptocurrency on devices." Apps that *remotely manage* mining that happens elsewhere are allowed.
|
||||
- **Apple**, App Review Guidelines **3.1.5(b)(ii) Cryptocurrencies, Mining**: "Apps may not mine for cryptocurrencies unless the processing is performed off device (e.g. cloud-based mining)." Guideline 2.4.2 also bans unrelated background processes such as mining.
|
||||
|
||||
A WikiDeck block is a RandomX proof-of-work that mints an NFT on WikiDeck's own chain. That is cryptocurrency mining under both policies, whatever the reward looks like. Re-check the policy wording before any store submission; these quotes are from memory, not verified tonight (tests and agents make no web calls).
|
||||
|
||||
## How it works
|
||||
```
|
||||
WebView (React) Native (Java, Capacitor plugin "WikiDeckMiner")
|
||||
src/MobileMiningPanel.tsx consent, intensity, stats
|
||||
src/mobile-miner.ts same loop as server/testnet/miner.ts
|
||||
POST /api/mining/template {owner, publicKey} ──► node (picks the article and stats; the phone never calls Wikipedia)
|
||||
build blob = sha256(headerMessage) || nonceLE8
|
||||
WikiDeckMiner.mine(lane, key, blob, target, 32, 16) ──► MinerPlugin.java ──stdin/stdout──► libwikideck_randomx.so
|
||||
(the pinned native/ engine, LIGHT mode, line protocol)
|
||||
found → hash, tokenId (mintId), sign(blockClaimMessage) with the web wallet
|
||||
POST /api/blocks ──► node validates as usual
|
||||
```
|
||||
- **Same engine as desktop.** `native/` (RandomX v1.2.1, pinned SHA `102f8acf…`) is cross-compiled with the Android NDK by `scripts/build-randomx-android.sh` for `arm64-v8a` and `x86_64` (the emulator). It is packaged as `jniLibs/<abi>/libwikideck_randomx.so`: an executable named like a library, so Android extracts it to `nativeLibraryDir`, the one place an app may `exec` from (`useLegacyPackaging true` in `android/app/build.gradle`). The binaries are build outputs, so they're gitignored, and `android/build-debug.sh` builds them when they're missing.
|
||||
- **Light mode only** (256 MiB cache per engine process). Fast mode (about 2 GiB) is never started on a phone: the plugin runs the binary with no arguments.
|
||||
- **Known-vector check on the device.** Before the first start in each session, the app runs `libwikideck_randomx.so --self-test` (upstream Hash test 1a/2a). If the hash doesn't match, mining is disabled. A wrong hash would get every block rejected.
|
||||
- **Keys never leave the web wallet.** The plugin only receives the public work (key, blob, target). The WebView builds and signs the block with the existing wallet code (`src/wallet.ts` `sign`). The private key and backup password are never passed to native code or to the node.
|
||||
- **No protocol, consensus or server changes.** `src/mobile-miner.ts` rebuilds `miningBlob` and `mintId` with Web Crypto. `tests/mobile-mining-miner.test.ts` checks that they're byte-identical to `server/testnet/consensus.ts` and that the consensus validator accepts the assembled block.
|
||||
- **Testnet only tonight.** In a mainnet build (`IS_MAINNET`), the panel only says mainnet mining isn't enabled. The debug build points at the private testnet `http://192.168.1.32:3101`.
|
||||
|
||||
## Safeguards
|
||||
| Safeguard | How |
|
||||
|---|---|
|
||||
| Explicit consent | An unchecked consent box ("uses this phone's CPU and battery and can make it warm"). Start stays disabled until it's ticked. The choice is stored locally (`wikideck.mobileMining.consent.v1`). |
|
||||
| Intensity | Low (1 thread, rests 400 ms between 16-hash batches), **Normal (1 thread, default)**, High (2 threads, about 512 MiB, offered only on 4+ cores). |
|
||||
| Foreground only | `MinerPlugin.handleOnPause` kills the engines as soon as the activity pauses, and the WebView also stops on `visibilitychange`. There is no foreground service, no wake lock and no background mining. The web wallet also locks when the app is hidden. |
|
||||
| Battery saver | `PowerManager.isPowerSaveMode()`, checked before start, every 2 s while mining, and on `ACTION_POWER_SAVE_MODE_CHANGED`. |
|
||||
| Heat | `PowerManager.getCurrentThermalStatus() >= THERMAL_STATUS_MODERATE` (API 29+, with a listener), or a battery temperature of 40 °C or more (`ACTION_BATTERY_CHANGED`). |
|
||||
| Low battery | Below 20 % while not charging. |
|
||||
| Visible activity | Live hashrate, hashes tried, the block height being mined, accepted blocks, battery % and °C, and the self-test result. |
|
||||
| Wallet change | If the wallet locks or changes, mining stops, so it never keeps mining to an old address. |
|
||||
|
||||
Each automatic stop shows its reason ("Paused: battery saver is on"). Mining never restarts by itself: the player taps Start again.
|
||||
|
||||
**Wallet idle lock:** a found block must be signed by the unlocked wallet, and the wallet locks after 5 minutes without taps. While mining is active (which means foreground and consented), each status update calls `touchWallet()` to keep the session alive. The 4-hour absolute limit and lock-on-hide still apply. night-review accepted this trade-off on PR #78. The session can't stay unlocked in the background: hiding the app locks the wallet, and `handleOnPause` halts mining, so the keep-alive applies only to consented foreground mining. Eric may prefer a stricter rule, such as re-prompting for the password when a block is found.
|
||||
|
||||
## Measured hashrate (emulator)
|
||||
Measured on inspiron-3670 (6 cores, shared with ~7 agents), using the AVD `wikideck` (Android 15, x86_64 google_apis, KVM, `-cores 4`, 2 GB RAM):
|
||||
|
||||
| Measurement | Result |
|
||||
|---|---|
|
||||
| Known vector (`--self-test`), x86_64 | passed, 1.45 s including the 256 MiB cache init |
|
||||
| Known vector, arm64-v8a (through the emulator's ARM translation) | passed |
|
||||
| Raw engine, 1 process, 2000 hashes | about 53 s → **~37 H/s per thread** |
|
||||
| In the app, Max (4 threads) | **~125 H/s** shown in the panel, with 4 engine processes at ~260 MB RSS each |
|
||||
|
||||
Light mode is about 10× slower per thread than desktop fast mode. At tonight's testnet target (`00002514…`, about 452 K expected hashes), 125 H/s means **one block per hour on average**, with high variance. On a real phone, expect roughly 20–60 H/s per big core (not measured tonight: no device).
|
||||
|
||||
## Results tonight
|
||||
Verified in the emulator with the debug APK (`~/worker/jobs/night/mobile-mining/apk/wikideck-mining-testnet-debug.apk` on inspiron):
|
||||
- The APK contains `lib/{arm64-v8a,x86_64}/libwikideck_randomx.so`. The app runs the self-test before its first start ("known vector OK"), then mines on the testnet (:3101) with live hashrate and block height.
|
||||
- **Every safeguard halts the engines** (the engine process count drops to 0, and the panel shows the reason):
|
||||
- thermal: `cmd thermalservice override-status 3` → "Paused: the device is warm"
|
||||
- battery saver (unplugged, `settings put global low_power 1`) → "Paused: battery saver is on"
|
||||
- low battery (`dumpsys battery set level 15`, unplugged) → halted (Android's low-battery dialog paused the activity first)
|
||||
- background (HOME) → "Paused: the app went to the background"; the web wallet locks too, so restarting needs an unlock
|
||||
- Max intensity is refused while unplugged
|
||||
- **Accepted block:** at 22:37 EST the emulator (Max, ~136 H/s, 4 light threads) mined **testnet block #92** ("International FJ"; block `edaf8ac3…a8cb0`, NFT `acf778b1…7ee58`) to the in-app wallet `3b2362c7…5e054`. All three nodes (:3101–:3103) accepted it as their tip. That one was lucky: about 41 K hashes against ~452 K expected.
|
||||
|
||||

|
||||
|
||||
## Options for Eric
|
||||
1. **Keep it sideload-only** (what exists now). Publish a signed release APK from a WikiDeck download page with clear battery and heat copy. This needs a release keystore (not in git) and an "unknown sources" install guide. F-Droid is a possible channel, but its anti-features policy would flag it, and its build server needs a reproducible NDK build.
|
||||
2. **Store builds without mining.** Ship the store APK/AAB and the iOS app without the engine (a build flag that drops jniLibs and the panel), with the panel replaced by "Mine on desktop, or with the sideload app", plus a remote-management view of a desktop miner (allowed by Google's policy).
|
||||
3. **Make mobile mining worthwhile.** Phones in light mode are roughly 10× slower per core than desktop fast mode. Options that need **consensus changes** (out of scope tonight): a lower-difficulty mobile lane, or pooled/shared mining where a phone's partial shares earn a card.
|
||||
4. **Later engineering:** a JNI wrapper instead of exec (needed if a future Android blocks exec from `nativeLibraryDir`), an arm64 device test on real hardware, and battery-drain measurements.
|
||||
@@ -0,0 +1,72 @@
|
||||
# Polite Wikimedia client (Night 2, issue #62)
|
||||
|
||||
Every server-side Wikimedia request goes through one process-wide `WikimediaClient`
|
||||
(`server/wikimedia/client.ts`). The design comes from engineer review §4 and data review §5.
|
||||
|
||||
## What it does
|
||||
|
||||
| Behaviour | Default |
|
||||
|---|---|
|
||||
| Rate per upstream host (`en.wikipedia.org`, `wikimedia.org`, `www.wikidata.org`) | token bucket, 1 req/s, burst 3 |
|
||||
| Concurrency | 1 request in flight per host |
|
||||
| Priority lanes | `mint` → `interactive` → `background`; the queue holds 64 waiters, and background work is shed first |
|
||||
| Longest queue wait | mint 10 s, interactive 5 s, background 120 s |
|
||||
| `maxlag` | `maxlag=5` on every Action API request; a maxlag reply waits for `Retry-After` and retries up to 2 times |
|
||||
| 429 / 503 | honour `Retry-After` (seconds or HTTP date), else back off 2ⁿ s with jitter, capped at 10 min, by pausing the whole host |
|
||||
| Circuit breaker | 3 consecutive 429s stop that host for 10 min. Requests are refused at once without touching Wikimedia |
|
||||
| Daily cap | 20,000 requests a day per egress IP. Background work stops at 80 %, interactive at 100 %; mint has a 10 % reserve |
|
||||
| Disk cache | `$WIKIDECK_DATA_DIR/wikimedia-cache.sqlite`: ETag / Last-Modified, `If-None-Match`, stale-while-revalidate on the background lane, stale copy served on refusal |
|
||||
| User-Agent | `WikiDeck/<version> (+https://gitea.eom.dev/WikiDeck/wikideck; <operator contact>) node/<version>` |
|
||||
|
||||
The rate is deliberately 1 req/s for **every** host, including the `wikimedia.org` REST (pageviews) host.
|
||||
The engineer review suggested 5 req/s there; we are stricter on purpose after the Night 1 rate limiting, because the
|
||||
budget is per egress IP. Raise it only with a reason and a per-host setting.
|
||||
|
||||
Cache lifetimes: article metadata 24 h, monthly pageviews 7 days (the URL names six closed
|
||||
months, which do not change). Random articles and all mint reads are never cached.
|
||||
|
||||
## Operator settings
|
||||
|
||||
| Variable | Meaning |
|
||||
|---|---|
|
||||
| `WIKIDECK_OPERATOR_CONTACT` | **Set this.** An email address or URL, put in the User-Agent. Without it (and without `PUBLIC_ORIGIN`), the node logs a warning at start-up. Wikimedia policy requires a contact, and contact-less clients are the first to be blocked. |
|
||||
| `PUBLIC_ORIGIN` | Used as the contact when `WIKIDECK_OPERATOR_CONTACT` is unset. |
|
||||
| `WIKIDECK_USER_AGENT` | Overrides the whole User-Agent string (sanitized; kept from Night 1). |
|
||||
| `WIKIDECK_WIKIMEDIA_RPS` | Requests per second per host for the **whole egress IP** (default 1). |
|
||||
| `WIKIDECK_WIKIMEDIA_BURST` | Token-bucket burst (default 3). |
|
||||
| `WIKIDECK_WIKIMEDIA_CONCURRENCY` | In-flight requests per host (default 1). |
|
||||
| `WIKIDECK_WIKIMEDIA_DAILY_CAP` | Requests per UTC day for the whole egress IP (default 20000). |
|
||||
| `WIKIDECK_WIKIMEDIA_NODES_PER_IP` | How many nodes share this egress IP. The rate and the daily cap are divided by it. **The testnet runs several nodes behind one IP: set it to the node count** (e.g. 3 for `deploy/night/compose.lan.yaml`). |
|
||||
|
||||
The testnet's `WIKIDECK_USER_AGENT` already names the project URL, which satisfies the policy.
|
||||
|
||||
Counters (`lane.*`, `http.429`, `breaker.open`, `cache.hit`, `refused.*`, …), per-host pause state and the
|
||||
mint pool's state are in the private operations listener's `/status`, under `wikimedia`.
|
||||
|
||||
## Mint path: transport only
|
||||
|
||||
- `random()` (W1) and `getFresh()` (W2 and W3) keep their request parameters byte-identical. The only
|
||||
addition is `maxlag=5`, which changes only error behaviour. `tests/wikimedia-mint.test.ts` pins the exact URLs.
|
||||
- Mint reads use the `mint` lane and **never** read or write the disk cache, and never join a display read.
|
||||
- **Fixed:** a 429, 5xx, timeout, network error or non-JSON reply on the pageviews call used to mint a card with
|
||||
`attention: null` forever (`enrichment.ts`). Now `getFresh()` rejects, the template returns 503 and the miner
|
||||
retries. `unavailable` is minted only for a genuine 404 (no data) or missing months.
|
||||
- No block, transfer, trade or validation code changed. Validation never calls Wikimedia.
|
||||
|
||||
## Mining prefetch pool (`server/wikimedia/pool.ts`)
|
||||
|
||||
Each template used to cost 3+ requests, all in a burst at each new tip, so load scaled with the number of miners.
|
||||
Now every template on the same tip gets the same fresh article+snapshot pair (miners race for one block anyway),
|
||||
and a new tip takes the next prefetched pair. Load is about one pair, or 3 requests, per block, whatever the number
|
||||
of miners.
|
||||
|
||||
- Pairs come from the mint-path reads above, and the snapshot is stamped when the pair is fetched, so `capturedAt` is true.
|
||||
- Pairs older than 5 minutes are dropped, never served.
|
||||
- The pool (2 pairs) refills only after use, so an idle node makes no requests.
|
||||
|
||||
## Card faces
|
||||
|
||||
`/api/wikipedia/:id` → `get()`: memory LRU, then the disk cache (fresh, or stale with a background
|
||||
revalidation), and only then the interactive lane. A grid of cards therefore no longer means a burst
|
||||
of 2 upstream requests per card per render. `TrendCache` can use `getBackground()` to stay off the
|
||||
interactive lane.
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 17 KiB |
@@ -1,35 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Cross-compile the pinned native RandomX engine (native/, RandomX v1.2.1) for the Android app.
|
||||
# Output: android/app/src/main/jniLibs/<abi>/libwikideck_randomx.so, an executable named like a
|
||||
# library so Android extracts it to nativeLibraryDir, where an app may exec it (docs/night2/MOBILE_MINING.md).
|
||||
#
|
||||
# Needs ANDROID_HOME with an NDK (ndk;<version>) and CMake 3.24+ (cmake;<version>, or cmake on PATH).
|
||||
# ABIS="x86_64 arm64-v8a" JOBS=2 sh scripts/build-randomx-android.sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${ANDROID_HOME:?set ANDROID_HOME}"
|
||||
ABIS="${ABIS:-arm64-v8a x86_64}"
|
||||
JOBS="${JOBS:-2}"
|
||||
API="${ANDROID_API:-24}"
|
||||
NDK="${ANDROID_NDK_HOME:-$(ls -d "$ANDROID_HOME"/ndk/* 2>/dev/null | sort -V | tail -n1)}"
|
||||
[ -f "$NDK/build/cmake/android.toolchain.cmake" ] || { echo "No NDK under $ANDROID_HOME/ndk" >&2; exit 1; }
|
||||
CMAKE_DIR="$(ls -d "$ANDROID_HOME"/cmake/* 2>/dev/null | sort -V | tail -n1 || true)"
|
||||
[ -n "$CMAKE_DIR" ] && PATH="$CMAKE_DIR/bin:$PATH"
|
||||
command -v cmake >/dev/null || { echo "cmake not found" >&2; exit 1; }
|
||||
GEN=""
|
||||
command -v ninja >/dev/null && GEN="-G Ninja"
|
||||
|
||||
for abi in $ABIS; do
|
||||
build="native/build-android-$abi"
|
||||
# shellcheck disable=SC2086
|
||||
cmake -S native -B "$build" $GEN \
|
||||
-DCMAKE_TOOLCHAIN_FILE="$NDK/build/cmake/android.toolchain.cmake" \
|
||||
-DANDROID_ABI="$abi" -DANDROID_PLATFORM="android-$API" \
|
||||
-DCMAKE_BUILD_TYPE=Release -DWIKIDECK_RANDOMX_TESTS=OFF
|
||||
cmake --build "$build" --config Release --target wikideck-randomx --parallel "$JOBS"
|
||||
out="android/app/src/main/jniLibs/$abi"
|
||||
mkdir -p "$out"
|
||||
cp "$build/wikideck-randomx" "$out/libwikideck_randomx.so"
|
||||
"$NDK"/toolchains/llvm/prebuilt/*/bin/llvm-strip "$out/libwikideck_randomx.so"
|
||||
ls -l "$out/libwikideck_randomx.so"
|
||||
done
|
||||
+72
-27
@@ -1,5 +1,5 @@
|
||||
import { z } from 'zod';
|
||||
import { ENRICHMENT_USER_AGENT, wikimediaUserAgent } from './wikimedia-user-agent.ts';
|
||||
import { WikimediaClient, WikimediaUnavailable, wikimediaClient, type CachePolicy, type Lane } from './wikimedia/client.ts';
|
||||
import { deriveStats, type PageEnrichment } from '../shared/enrichment.ts';
|
||||
|
||||
const MAX_BYTES = 1024 * 1024;
|
||||
@@ -25,7 +25,23 @@ const viewsSchema = z.object({ items: z.array(z.object({
|
||||
})).max(6) });
|
||||
|
||||
type CacheEntry = { expires: number; value: PageEnrichment } | { expires: number; error: Error };
|
||||
type Options = { fetch?: typeof globalThis.fetch; now?: () => Date; ttlMs?: number; maxEntries?: number };
|
||||
type Options = { fetch?: typeof globalThis.fetch; client?: WikimediaClient; now?: () => Date; ttlMs?: number; maxEntries?: number };
|
||||
type Mode = 'mint' | 'display' | 'background';
|
||||
|
||||
/** Disk-cache lifetimes (engineer review §4). Mint reads never use the cache. */
|
||||
const METADATA_CACHE: CachePolicy = { ttlMs: 24 * 3600_000, staleWhileRevalidate: true, maxBytes: MAX_BYTES };
|
||||
// The URL names its six completed months, which are immutable; a week bounds late corrections.
|
||||
const PAGEVIEWS_CACHE: CachePolicy = { ttlMs: 7 * 24 * 3600_000, staleWhileRevalidate: true, maxBytes: MAX_BYTES };
|
||||
const NOT_FOUND = Symbol('not found');
|
||||
|
||||
/** The upstream did not answer properly (throttled, 5xx, timeout, network, non-JSON). Not "no data". */
|
||||
export class UpstreamFailure extends Error {
|
||||
constructor(message: string) { super(message); this.name = 'UpstreamFailure'; }
|
||||
}
|
||||
function transportFailure(error: unknown): boolean {
|
||||
return error instanceof WikimediaUnavailable || error instanceof UpstreamFailure ||
|
||||
(error instanceof DOMException && (error.name === 'TimeoutError' || error.name === 'AbortError'));
|
||||
}
|
||||
|
||||
function monthWindow(now: Date): string[] {
|
||||
return Array.from({ length: 6 }, (_, index) =>
|
||||
@@ -34,16 +50,17 @@ function monthWindow(now: Date): string[] {
|
||||
|
||||
/** Fixed-source read-only enrichment; no consensus, persistence, URL or header inputs. */
|
||||
export class WikipediaEnrichment {
|
||||
private readonly fetcher: typeof globalThis.fetch;
|
||||
private readonly client: WikimediaClient;
|
||||
private readonly now: () => Date;
|
||||
private readonly ttlMs: number;
|
||||
private readonly maxEntries: number;
|
||||
private readonly cache = new Map<number, CacheEntry>();
|
||||
private readonly pending = new Map<number, Promise<PageEnrichment>>();
|
||||
private readonly pending = new Map<string, Promise<PageEnrichment>>();
|
||||
private active = 0;
|
||||
private mintActive = 0;
|
||||
|
||||
constructor(options: Options = {}) {
|
||||
this.fetcher = options.fetch ?? globalThis.fetch;
|
||||
this.client = options.client ?? (options.fetch ? WikimediaClient.unthrottled(options.fetch) : wikimediaClient);
|
||||
this.now = options.now ?? (() => new Date());
|
||||
this.ttlMs = options.ttlMs ?? 6 * 60 * 60 * 1000;
|
||||
this.maxEntries = options.maxEntries ?? 128;
|
||||
@@ -53,16 +70,27 @@ export class WikipediaEnrichment {
|
||||
}
|
||||
}
|
||||
|
||||
/** Card faces (`/api/wikipedia/:id`): memory, then the disk cache, then the interactive lane. */
|
||||
async get(pageId: number): Promise<PageEnrichment> {
|
||||
return this.loadCached(pageId, false);
|
||||
return this.loadCached(pageId, 'display');
|
||||
}
|
||||
|
||||
/** Bypass cached values when the result is being pinned into a newly minted block. */
|
||||
/** Like get(), on the background lane: for trends and other work nobody is waiting on. */
|
||||
async getBackground(pageId: number): Promise<PageEnrichment> {
|
||||
return this.loadCached(pageId, 'background');
|
||||
}
|
||||
|
||||
/**
|
||||
* Mint data: bypasses every cache and uses the mint lane. If pageviews fail in transport
|
||||
* (429, 5xx, timeout, throttled), this rejects so the template fails and the miner retries;
|
||||
* `unavailable` is recorded only for a genuine 404 or missing data.
|
||||
*/
|
||||
async getFresh(pageId: number): Promise<PageEnrichment> {
|
||||
return this.loadCached(pageId, true);
|
||||
return this.loadCached(pageId, 'mint');
|
||||
}
|
||||
|
||||
private async loadCached(pageId: number, fresh: boolean): Promise<PageEnrichment> {
|
||||
private async loadCached(pageId: number, mode: Mode): Promise<PageEnrichment> {
|
||||
const fresh = mode === 'mint';
|
||||
if (!Number.isSafeInteger(pageId) || pageId <= 0) throw new RangeError('Invalid Wikipedia page ID');
|
||||
const now = this.now();
|
||||
const cached = this.cache.get(pageId);
|
||||
@@ -73,23 +101,29 @@ export class WikipediaEnrichment {
|
||||
if ('error' in cached) throw cached.error;
|
||||
return structuredClone(cached.value);
|
||||
}
|
||||
this.cache.delete(pageId);
|
||||
const existing = this.pending.get(pageId);
|
||||
if (!fresh) this.cache.delete(pageId);
|
||||
// Mint jobs never join a display job: that one may be served from the disk cache.
|
||||
const pendingKey = `${fresh ? 'mint' : 'cached'}:${pageId}`;
|
||||
const existing = this.pending.get(pendingKey);
|
||||
if (existing) return structuredClone(await existing);
|
||||
// Reject immediately: no unbounded waiting queue. Four active jobs also keep
|
||||
// the pending-map ceiling of eight conservative (one upstream request/job).
|
||||
if (this.pending.size >= 8 || this.active >= 4) throw new Error('Wikipedia enrichment busy');
|
||||
this.active++;
|
||||
const job = this.load(pageId, now).then(value => {
|
||||
// Mint jobs have their own small allowance so card faces can never starve a template.
|
||||
if (fresh ? this.mintActive >= 2 : this.pending.size - this.mintActive >= 8 || this.active >= 4) {
|
||||
throw new Error('Wikipedia enrichment busy');
|
||||
}
|
||||
if (fresh) this.mintActive++; else this.active++;
|
||||
const job = this.load(pageId, now, mode).then(value => {
|
||||
const ttl = value.pageviews.status === 'available' ? this.ttlMs : Math.min(this.ttlMs, 5 * 60 * 1000);
|
||||
this.remember(pageId, { value, expires: this.now().getTime() + ttl });
|
||||
return value;
|
||||
}, () => {
|
||||
const error = new Error('Wikipedia page enrichment unavailable');
|
||||
this.remember(pageId, { error, expires: this.now().getTime() + 30_000 });
|
||||
// A failed mint read says nothing new about display data; keep any cached value.
|
||||
if (!fresh) this.remember(pageId, { error, expires: this.now().getTime() + 30_000 });
|
||||
throw error;
|
||||
}).finally(() => { this.active--; this.pending.delete(pageId); });
|
||||
this.pending.set(pageId, job);
|
||||
}).finally(() => { if (fresh) this.mintActive--; else this.active--; this.pending.delete(pendingKey); });
|
||||
this.pending.set(pendingKey, job);
|
||||
return structuredClone(await job);
|
||||
}
|
||||
|
||||
@@ -99,16 +133,19 @@ export class WikipediaEnrichment {
|
||||
while (this.cache.size > this.maxEntries) this.cache.delete(this.cache.keys().next().value!);
|
||||
}
|
||||
|
||||
private async json(url: URL): Promise<unknown> {
|
||||
const signal = AbortSignal.timeout(6000);
|
||||
const response = await this.fetcher(url, { signal, redirect: 'error',
|
||||
headers: { Accept: 'application/json', 'User-Agent': wikimediaUserAgent(ENRICHMENT_USER_AGENT) } });
|
||||
private async json(url: URL, lane: Lane, cache: CachePolicy | undefined, allowNotFound = false): Promise<unknown> {
|
||||
const { response, signal } = await this.client.request(url, { lane, timeoutMs: 6000, cache });
|
||||
if (allowNotFound && response.status === 404) { await response.body?.cancel(); return NOT_FOUND; }
|
||||
if (!response.ok || response.redirected) {
|
||||
await response.body?.cancel();
|
||||
throw new UpstreamFailure(`Upstream HTTP ${response.status}`);
|
||||
}
|
||||
const length = response.headers.get('content-length');
|
||||
if (!response.ok || response.redirected ||
|
||||
if (
|
||||
!/^application\/json(?:\s*;|\s*$)/i.test(response.headers.get('content-type') ?? '') ||
|
||||
(length !== null && (!/^\d+$/.test(length) || Number(length) > MAX_BYTES)) || !response.body) {
|
||||
await response.body?.cancel();
|
||||
throw new Error('Invalid upstream response');
|
||||
throw new UpstreamFailure('Invalid upstream response');
|
||||
}
|
||||
const reader = response.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
@@ -130,12 +167,14 @@ export class WikipediaEnrichment {
|
||||
}
|
||||
}
|
||||
|
||||
private async load(pageId: number, now: Date): Promise<PageEnrichment> {
|
||||
private async load(pageId: number, now: Date, mode: Mode): Promise<PageEnrichment> {
|
||||
const lane: Lane = mode === 'mint' ? 'mint' : mode === 'display' ? 'interactive' : 'background';
|
||||
const fresh = mode === 'mint';
|
||||
const url = new URL('https://en.wikipedia.org/w/api.php');
|
||||
url.search = new URLSearchParams({ action: 'query', format: 'json', formatversion: '2',
|
||||
pageids: String(pageId), prop: 'info|categories|pageprops|description|langlinks',
|
||||
clshow: '!hidden', cllimit: '50', lllimit: '500', ppprop: 'wikibase_item' }).toString();
|
||||
const metadata = metadataSchema.parse(await this.json(url));
|
||||
const metadata = metadataSchema.parse(await this.json(url, lane, fresh ? undefined : METADATA_CACHE));
|
||||
const page = metadata.query.pages[0];
|
||||
if (page.pageid !== pageId) throw new Error('Wikipedia page ID mismatch');
|
||||
const months = monthWindow(now);
|
||||
@@ -146,7 +185,9 @@ export class WikipediaEnrichment {
|
||||
const title = encodeURIComponent(page.title.replace(/ /g, '_'));
|
||||
const stamp = (month: string) => `${month.replace('-', '')}0100`;
|
||||
const endpoint = new URL(`https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia.org/all-access/user/${title}/monthly/${stamp(months[0])}/${stamp(latestMonth)}`);
|
||||
const result = viewsSchema.parse(await this.json(endpoint));
|
||||
const raw = await this.json(endpoint, lane, fresh ? undefined : PAGEVIEWS_CACHE, true);
|
||||
if (raw === NOT_FOUND) throw new Error('No pageview data');
|
||||
const result = viewsSchema.parse(raw);
|
||||
const byMonth = new Map<string, number>();
|
||||
for (const item of result.items) {
|
||||
const month = `${item.timestamp.slice(0, 4)}-${item.timestamp.slice(4, 6)}`;
|
||||
@@ -161,7 +202,11 @@ export class WikipediaEnrichment {
|
||||
const days = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 0)).getUTCDate();
|
||||
pageviews = { status: 'available', months: ordered, latestMonth, total,
|
||||
dailyAverage: latest / days, trendPercent: prior > 0 ? (latest - prior) / prior * 100 : null };
|
||||
} catch { /* Metadata remains useful; unknown views must not become fake zeroes. */ }
|
||||
} catch (error) {
|
||||
// Mint: a throttled or failed upstream must not pin `attention: null` into a block forever.
|
||||
if (fresh && transportFailure(error)) throw error;
|
||||
/* Metadata remains useful; unknown views must not become fake zeroes. */
|
||||
}
|
||||
return { pageId, title: page.title, description: page.description ?? null,
|
||||
wikidataId: page.pageprops?.wikibase_item ?? null, revisionId: page.lastrevid, pageBytes: page.length,
|
||||
languageCount: page.langlinks?.length ?? 0, languagesTruncated: metadata.continue?.llcontinue !== undefined,
|
||||
|
||||
@@ -11,8 +11,7 @@ import { RandomX } from './randomx.ts';
|
||||
import { Ledger } from './ledger.ts';
|
||||
import { PeerNetwork } from './peers.ts';
|
||||
import { wikipediaEnrichment } from '../enrichment.ts';
|
||||
import { snapshotStats } from '../../shared/enrichment.ts';
|
||||
import { wikipediaArticles } from '../wiki.ts';
|
||||
import { configureWikimedia, mintPool, wikimediaStatus } from '../wikimedia/index.ts';
|
||||
import { IS_MAINNET, PROFILE } from '../../shared/network.ts';
|
||||
import { assertLaunchAllowed, launchManifest } from '../launch.ts';
|
||||
import { resourceConfig, RpcBudget, RpcError } from '../resources.ts';
|
||||
@@ -28,6 +27,7 @@ const apiBudget = new RpcBudget(limits);
|
||||
const peerBudget = new RpcBudget({ ...limits, requestsPerSecond: 40, writesPerSecond: 10, active: 16 });
|
||||
const dataDir = process.env.WIKIDECK_DATA_DIR ?? resolve(root,'data');
|
||||
await mkdir(dataDir,{recursive:true});
|
||||
configureWikimedia({dataDir});
|
||||
// Never permit two processes to rewrite the same ledger. A stale lock requires operator review.
|
||||
const lockPath = resolve(dataDir,`${PROFILE}-node.lock`);
|
||||
const lock = await open(lockPath,'wx',0o600).catch(()=>{throw new Error('Data directory locked. Stop its other node; remove stale testnet-node.lock only after verifying no node is running.');});
|
||||
@@ -119,9 +119,9 @@ const templateLimit=rateLimit({windowMs:60000,limit:20,standardHeaders:'draft-8'
|
||||
app.post('/api/mining/template', templateLimit, async (req,res)=>{
|
||||
const body=z.object({owner:z.string().regex(/^[a-f0-9]{64}$/),publicKey:z.string().max(256)}).strict().parse(req.body);
|
||||
try {
|
||||
const article=await wikipediaArticles.random();
|
||||
const metrics=await wikipediaEnrichment.getFresh(article.pageId);
|
||||
res.json(ledger.template(body.owner,body.publicKey,article,snapshotStats(metrics)));
|
||||
// One fresh article+snapshot per tip, shared by every miner (server/wikimedia/pool.ts).
|
||||
const {article,statsSnapshot}=await mintPool.take(ledger.summary().tip);
|
||||
res.json(ledger.template(body.owner,body.publicKey,article,statsSnapshot));
|
||||
} catch {
|
||||
res.status(503).json({error:'Could not prepare a Wikipedia article and current statistics. Retry shortly.'});
|
||||
}
|
||||
@@ -170,7 +170,7 @@ const server=app.listen(port,host,()=>{
|
||||
const operations=startOperations(opsPort,()=>({network:NETWORK,
|
||||
ledger:ledger.diagnostics(),native:engine.diagnostics(),
|
||||
peers:{configured:peers.status().length,connected:peers.status().filter(p=>p.connected).length},
|
||||
rpc:{api:apiBudget.status(),p2p:peerBudget.status()}}),resolve(dataDir,PROFILE));
|
||||
rpc:{api:apiBudget.status(),p2p:peerBudget.status()},wikimedia:wikimediaStatus()}),resolve(dataDir,PROFILE));
|
||||
server.maxConnections=256;
|
||||
server.headersTimeout=10000;
|
||||
server.requestTimeout=15000;
|
||||
|
||||
+22
-10
@@ -1,6 +1,6 @@
|
||||
import { z } from 'zod';
|
||||
import type { Article } from '../shared/protocol.ts';
|
||||
import { ARTICLE_USER_AGENT, wikimediaUserAgent } from './wikimedia-user-agent.ts';
|
||||
import { WikimediaClient, WikimediaUnavailable, wikimediaClient } from './wikimedia/client.ts';
|
||||
|
||||
const pageSchema = z.object({
|
||||
pageid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), ns: z.literal(0),
|
||||
@@ -58,11 +58,23 @@ function pictureUrl(value: string | undefined): string | undefined {
|
||||
} catch { return undefined; }
|
||||
}
|
||||
|
||||
/** The exact W1 request. Its parameters decide minted article content: keep them byte-identical. */
|
||||
export function randomArticlesUrl(): URL {
|
||||
const url = new URL('https://en.wikipedia.org/w/api.php');
|
||||
url.search = new URLSearchParams({ action: 'query', format: 'json', formatversion: '2',
|
||||
generator: 'random', grnnamespace: '0', grnlimit: '10', prop: 'extracts|pageimages|info|categories',
|
||||
exintro: '1', explaintext: '1', exchars: '900', piprop: 'thumbnail', pithumbsize: '800', cllimit: '20' }).toString();
|
||||
return url;
|
||||
}
|
||||
|
||||
export class WikipediaArticleSource {
|
||||
private readonly fetcher: typeof globalThis.fetch;
|
||||
private readonly client: WikimediaClient;
|
||||
private active = 0;
|
||||
|
||||
constructor(fetcher: typeof globalThis.fetch = globalThis.fetch) { this.fetcher = fetcher; }
|
||||
/** A bare fetch (tests) gets an unthrottled pass-through client; production uses the shared client. */
|
||||
constructor(source: typeof globalThis.fetch | WikimediaClient = wikimediaClient) {
|
||||
this.client = source instanceof WikimediaClient ? source : WikimediaClient.unthrottled(source);
|
||||
}
|
||||
|
||||
async random(): Promise<Article> {
|
||||
if (this.active >= 4) throw new Error('Wikipedia article service is busy');
|
||||
@@ -70,12 +82,8 @@ export class WikipediaArticleSource {
|
||||
try {
|
||||
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
||||
try {
|
||||
const url = new URL('https://en.wikipedia.org/w/api.php');
|
||||
url.search = new URLSearchParams({ action: 'query', format: 'json', formatversion: '2',
|
||||
generator: 'random', grnnamespace: '0', grnlimit: '10', prop: 'extracts|pageimages|info|categories',
|
||||
exintro: '1', explaintext: '1', exchars: '900', piprop: 'thumbnail', pithumbsize: '800', cllimit: '20' }).toString();
|
||||
const response = await this.fetcher(url, { signal: AbortSignal.timeout(3500), redirect: 'error',
|
||||
headers: { 'User-Agent': wikimediaUserAgent(ARTICLE_USER_AGENT), Accept: 'application/json' } });
|
||||
// Mint lane, never cached: random articles become immutable chain data.
|
||||
const { response } = await this.client.request(randomArticlesUrl(), { lane: 'mint', timeoutMs: 3500 });
|
||||
const data = responseSchema.parse(await readJson(response));
|
||||
for (const rawPage of data.query.pages) {
|
||||
const parsedPage = pageSchema.safeParse(rawPage);
|
||||
@@ -88,7 +96,11 @@ export class WikipediaArticleSource {
|
||||
url: `https://en.wikipedia.org/?curid=${page.pageid}`, revisionId: page.lastrevid,
|
||||
source: 'live', category: category(page.title, page.categories?.map(c => c.title) ?? []) };
|
||||
}
|
||||
} catch { /* Try another random API response, but keep the number of attempts bounded. */ }
|
||||
} catch (error) {
|
||||
// Throttled, paused or failing upstream: stop now rather than hammer it with retries.
|
||||
if (error instanceof WikimediaUnavailable) throw error;
|
||||
/* Otherwise try another random API response, but keep the number of attempts bounded. */
|
||||
}
|
||||
}
|
||||
throw new Error('Wikipedia did not return an article with a picture after three attempts');
|
||||
} finally {
|
||||
|
||||
@@ -1,13 +1,48 @@
|
||||
// User-Agent for Wikimedia API requests. Wikimedia asks automated clients to identify
|
||||
// themselves with contact information; operators can opt in with WIKIDECK_USER_AGENT.
|
||||
// Unset (or blank), each caller keeps its historical default string unchanged.
|
||||
export const ARTICLE_USER_AGENT = 'Wikideck/0.1 (experimental Wikipedia article cards)';
|
||||
export const ENRICHMENT_USER_AGENT = 'Wikideck/0.1 (Wikipedia off-chain page enrichment)';
|
||||
// User-Agent for Wikimedia API requests. Wikimedia's User-Agent policy asks automated clients for a
|
||||
// descriptive agent with contact details: `Tool/version (+https://site; contact) lib/version`.
|
||||
// Operators set WIKIDECK_OPERATOR_CONTACT (an email address or URL); PUBLIC_ORIGIN is the fallback.
|
||||
// WIKIDECK_USER_AGENT still overrides the whole string.
|
||||
import { readFileSync } from 'node:fs';
|
||||
|
||||
export const PROJECT_URL = 'https://gitea.eom.dev/WikiDeck/wikideck';
|
||||
const MAX_LENGTH = 256;
|
||||
const SAFE = /^[\x20-\x7e]+$/;
|
||||
|
||||
function packageVersion(): string {
|
||||
try {
|
||||
const raw = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')) as { version?: unknown };
|
||||
return typeof raw.version === 'string' && /^[\w.+-]{1,32}$/.test(raw.version) ? raw.version : '0';
|
||||
} catch { return '0'; }
|
||||
}
|
||||
const VERSION = packageVersion();
|
||||
|
||||
/** The operator contact, or undefined when none (or only an unsafe value) is configured. */
|
||||
export function operatorContact(env: Record<string, string | undefined> = process.env): string | undefined {
|
||||
for (const value of [env.WIKIDECK_OPERATOR_CONTACT, env.PUBLIC_ORIGIN]) {
|
||||
const trimmed = value?.trim();
|
||||
if (trimmed && trimmed.length <= 128 && SAFE.test(trimmed) && !/[();]/.test(trimmed)) return trimmed;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** `WikiDeck/<version> (+<project URL>; <contact>) node/<version>`. */
|
||||
export function defaultUserAgent(env: Record<string, string | undefined> = process.env): string {
|
||||
const contact = operatorContact(env);
|
||||
return `WikiDeck/${VERSION} (+${PROJECT_URL}${contact ? `; ${contact}` : ''}) node/${process.versions.node}`;
|
||||
}
|
||||
|
||||
export function wikimediaUserAgent(fallback: string, env: Record<string, string | undefined> = process.env): string {
|
||||
const configured = env.WIKIDECK_USER_AGENT?.trim();
|
||||
// Reject header-injection/control characters and oversized values; fall back to the default.
|
||||
if (!configured || configured.length > MAX_LENGTH || !/^[\x20-\x7e]+$/.test(configured)) return fallback;
|
||||
if (!configured || configured.length > MAX_LENGTH || !SAFE.test(configured)) return fallback;
|
||||
return configured;
|
||||
}
|
||||
|
||||
/** Start-up check: Wikimedia throttles or blocks clients without contact details first. */
|
||||
export function warnIfNoContact(env: Record<string, string | undefined> = process.env,
|
||||
warn: (message: string) => void = console.warn): boolean {
|
||||
if (wikimediaUserAgent('', env) || operatorContact(env)) return false;
|
||||
warn('[wikimedia] WARNING: no operator contact in the Wikimedia User-Agent. Set WIKIDECK_OPERATOR_CONTACT '
|
||||
+ '(an email address or URL) or PUBLIC_ORIGIN; Wikimedia policy requires it and may block this node.');
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
/**
|
||||
* On-disk HTTP cache for off-chain Wikimedia reads (`$WIKIDECK_DATA_DIR/wikimedia-cache.sqlite`).
|
||||
* It stores the raw body with its ETag / Last-Modified so stale entries can be revalidated with
|
||||
* conditional requests. Mint-path requests never use it (see client.ts).
|
||||
*/
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
|
||||
export type StoredEntry = {
|
||||
status: number; body: Buffer; contentType: string; etag: string | null; lastModified: string | null;
|
||||
expiresAt: number; storedAt: number;
|
||||
};
|
||||
export type CacheEntry = StoredEntry & { toResponse(): Response };
|
||||
|
||||
/** Canonical key: origin + path + sorted query, without `maxlag` (it changes only error behaviour). */
|
||||
export function cacheKey(url: URL): string {
|
||||
const params = [...url.searchParams].filter(([name]) => name !== 'maxlag')
|
||||
.sort(([a, x], [b, y]) => a < b ? -1 : a > b ? 1 : x < y ? -1 : x > y ? 1 : 0);
|
||||
return `${url.origin}${url.pathname}?${new URLSearchParams(params).toString()}`;
|
||||
}
|
||||
|
||||
function withResponse(entry: StoredEntry): CacheEntry {
|
||||
return { ...entry, toResponse: () => new Response(new Uint8Array(entry.body), { status: entry.status,
|
||||
headers: { 'Content-Type': entry.contentType, 'Content-Length': String(entry.body.byteLength),
|
||||
...(entry.etag ? { ETag: entry.etag } : {}) } }) };
|
||||
}
|
||||
|
||||
type Row = { status: number; body: Uint8Array; content_type: string; etag: string | null;
|
||||
last_modified: string | null; expires_at: number; stored_at: number };
|
||||
|
||||
export class WikimediaCache {
|
||||
private readonly db: DatabaseSync;
|
||||
private readonly maxRows: number;
|
||||
private writes = 0;
|
||||
|
||||
/** `path` may be ':memory:' (tests). */
|
||||
constructor(path: string, options: { maxRows?: number } = {}) {
|
||||
this.maxRows = options.maxRows ?? 100_000;
|
||||
this.db = new DatabaseSync(path);
|
||||
this.db.exec(`PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;
|
||||
CREATE TABLE IF NOT EXISTS http_cache (key TEXT PRIMARY KEY, status INTEGER NOT NULL, body BLOB NOT NULL,
|
||||
content_type TEXT NOT NULL, etag TEXT, last_modified TEXT, expires_at INTEGER NOT NULL, stored_at INTEGER NOT NULL);
|
||||
CREATE INDEX IF NOT EXISTS http_cache_stored ON http_cache(stored_at);`);
|
||||
}
|
||||
|
||||
get(key: string): CacheEntry | undefined {
|
||||
const row = this.db.prepare('SELECT * FROM http_cache WHERE key = ?').get(key) as Row | undefined;
|
||||
if (!row) return undefined;
|
||||
return withResponse({ status: row.status, body: Buffer.from(row.body), contentType: row.content_type,
|
||||
etag: row.etag, lastModified: row.last_modified, expiresAt: row.expires_at, storedAt: row.stored_at });
|
||||
}
|
||||
|
||||
put(key: string, entry: StoredEntry): CacheEntry {
|
||||
this.db.prepare(`INSERT OR REPLACE INTO http_cache (key, status, body, content_type, etag, last_modified, expires_at, stored_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(key, entry.status, entry.body, entry.contentType, entry.etag,
|
||||
entry.lastModified, entry.expiresAt, entry.storedAt);
|
||||
if (++this.writes % 500 === 0) this.prune();
|
||||
return withResponse(entry);
|
||||
}
|
||||
|
||||
touch(key: string, expiresAt: number): void {
|
||||
this.db.prepare('UPDATE http_cache SET expires_at = ? WHERE key = ?').run(expiresAt, key);
|
||||
}
|
||||
|
||||
delete(key: string): void { this.db.prepare('DELETE FROM http_cache WHERE key = ?').run(key); }
|
||||
|
||||
size(): number { return (this.db.prepare('SELECT COUNT(*) AS n FROM http_cache').get() as { n: number }).n; }
|
||||
|
||||
prune(): void {
|
||||
this.db.prepare(`DELETE FROM http_cache WHERE key IN (SELECT key FROM http_cache ORDER BY stored_at DESC LIMIT -1 OFFSET ?)`)
|
||||
.run(this.maxRows);
|
||||
}
|
||||
|
||||
close(): void { this.db.close(); }
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
/**
|
||||
* One process-wide, polite Wikimedia HTTP client (engineer review §4, data review §5).
|
||||
*
|
||||
* Every server-side Wikimedia request goes through `WikimediaClient.request()`:
|
||||
* - a token bucket per upstream host (default 1 req/s, burst 3) and 1 request in flight per host;
|
||||
* - priority lanes: `mint` before `interactive` before `background`, with a bounded queue;
|
||||
* - `maxlag=5` on Action API requests (it changes only error behaviour, never content);
|
||||
* - `Retry-After` honoured on 429/503/maxlag by pausing the whole host, with exponential backoff;
|
||||
* - a circuit breaker: 3 consecutive 429s stop all requests to that host for 10 minutes;
|
||||
* - a daily request cap, and a budget shared by the nodes behind one egress IP;
|
||||
* - an optional on-disk cache (see cache.ts) with ETag / If-None-Match and stale-while-revalidate.
|
||||
*
|
||||
* The client is transport only. It never rewrites request parameters other than adding `maxlag`,
|
||||
* and requests made with `cache: undefined` (all mint-path requests) never read or write the cache.
|
||||
* Nothing here is reachable from block validation, so it cannot affect consensus.
|
||||
*/
|
||||
import { WikimediaCache, cacheKey } from './cache.ts';
|
||||
import { wikimediaUserAgent, defaultUserAgent } from '../wikimedia-user-agent.ts';
|
||||
|
||||
export type Lane = 'mint' | 'interactive' | 'background';
|
||||
const LANES: Lane[] = ['mint', 'interactive', 'background'];
|
||||
|
||||
export type Clock = {
|
||||
now(): number;
|
||||
setTimeout(fn: () => void, ms: number): unknown;
|
||||
clearTimeout(handle: unknown): void;
|
||||
};
|
||||
const realClock: Clock = {
|
||||
now: () => Date.now(),
|
||||
setTimeout: (fn, ms) => { const t = setTimeout(fn, ms); t.unref?.(); return t; },
|
||||
clearTimeout: handle => clearTimeout(handle as ReturnType<typeof setTimeout>),
|
||||
};
|
||||
|
||||
export type ClientConfig = {
|
||||
/** Sustained requests per second per upstream host, for this whole egress IP. */
|
||||
requestsPerSecond: number;
|
||||
burst: number;
|
||||
concurrencyPerHost: number;
|
||||
/** Requests per UTC day for this whole egress IP. */
|
||||
dailyCap: number;
|
||||
/** Nodes sharing one egress IP; the rate and daily cap are divided between them. */
|
||||
nodesPerIp: number;
|
||||
queueLimit: number;
|
||||
/** Longest wait in the queue before a request fails, per lane. */
|
||||
maxWaitMs: Record<Lane, number>;
|
||||
breakerThreshold: number;
|
||||
breakerMs: number;
|
||||
maxBackoffMs: number;
|
||||
maxlag: number;
|
||||
/** Retries of a single request after a maxlag reply (HTTP 200/503 with MediaWiki-API-Error: maxlag). */
|
||||
maxlagRetries: number;
|
||||
};
|
||||
|
||||
export const DEFAULT_CONFIG: ClientConfig = {
|
||||
requestsPerSecond: 1, burst: 3, concurrencyPerHost: 1, dailyCap: 20_000, nodesPerIp: 1, queueLimit: 64,
|
||||
maxWaitMs: { mint: 10_000, interactive: 5_000, background: 120_000 },
|
||||
breakerThreshold: 3, breakerMs: 10 * 60_000, maxBackoffMs: 10 * 60_000, maxlag: 5, maxlagRetries: 2,
|
||||
};
|
||||
|
||||
/** Unthrottled settings for tests and tools that inject their own fetch. */
|
||||
export const UNTHROTTLED: Partial<ClientConfig> = {
|
||||
requestsPerSecond: 1e9, burst: 1e9, concurrencyPerHost: 1e9, dailyCap: Number.MAX_SAFE_INTEGER, queueLimit: 1e9,
|
||||
breakerThreshold: Number.MAX_SAFE_INTEGER, maxlagRetries: 0, maxBackoffMs: 0,
|
||||
};
|
||||
|
||||
function positive(value: string | undefined, fallback: number, integer = false): number {
|
||||
if (value === undefined || value.trim() === '') return fallback;
|
||||
const n = Number(value);
|
||||
if (!Number.isFinite(n) || n <= 0 || (integer && !Number.isSafeInteger(n))) {
|
||||
console.warn(`[wikimedia] ignoring invalid setting ${JSON.stringify(value)}; using ${fallback}`);
|
||||
return fallback;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/** Operator settings. Documented in docs/night2/WIKIMEDIA.md. */
|
||||
export function configFromEnv(env: Record<string, string | undefined> = process.env): ClientConfig {
|
||||
return { ...DEFAULT_CONFIG,
|
||||
requestsPerSecond: positive(env.WIKIDECK_WIKIMEDIA_RPS, DEFAULT_CONFIG.requestsPerSecond),
|
||||
burst: positive(env.WIKIDECK_WIKIMEDIA_BURST, DEFAULT_CONFIG.burst),
|
||||
concurrencyPerHost: positive(env.WIKIDECK_WIKIMEDIA_CONCURRENCY, DEFAULT_CONFIG.concurrencyPerHost, true),
|
||||
dailyCap: positive(env.WIKIDECK_WIKIMEDIA_DAILY_CAP, DEFAULT_CONFIG.dailyCap, true),
|
||||
nodesPerIp: positive(env.WIKIDECK_WIKIMEDIA_NODES_PER_IP, DEFAULT_CONFIG.nodesPerIp, true),
|
||||
};
|
||||
}
|
||||
|
||||
/** Thrown for every refusal or transport failure: queue full, cap reached, circuit open, timeout, network. */
|
||||
export class WikimediaUnavailable extends Error {
|
||||
readonly reason: string;
|
||||
readonly status?: number;
|
||||
constructor(reason: string, status?: number) {
|
||||
super(`Wikimedia unavailable: ${reason}`);
|
||||
this.name = 'WikimediaUnavailable';
|
||||
this.reason = reason; this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
export type CachePolicy = {
|
||||
ttlMs: number;
|
||||
/** Serve a stale entry at once and revalidate it on the background lane. */
|
||||
staleWhileRevalidate?: boolean;
|
||||
/** Largest body stored, in bytes. */
|
||||
maxBytes?: number;
|
||||
};
|
||||
|
||||
export type RequestOptions = {
|
||||
lane: Lane;
|
||||
/** Starts when the request is dispatched, not while it waits in the queue. */
|
||||
timeoutMs: number;
|
||||
headers?: Record<string, string>;
|
||||
/** Omit for mint data: the request then bypasses the cache completely. */
|
||||
cache?: CachePolicy;
|
||||
};
|
||||
|
||||
export type ClientResponse = { response: Response; signal: AbortSignal; fromCache: boolean };
|
||||
|
||||
type Waiter = {
|
||||
lane: Lane; host: string; enqueuedAt: number;
|
||||
run: () => void; fail: (error: Error) => void; timer?: unknown;
|
||||
};
|
||||
type HostState = {
|
||||
tokens: number; refilledAt: number; inFlight: number;
|
||||
pausedUntil: number; breakerUntil: number; consecutive429: number; backoffExponent: number;
|
||||
};
|
||||
|
||||
export type ClientStatus = {
|
||||
day: string; requestsToday: number; dailyCap: number; requestsPerSecond: number; queued: number;
|
||||
counters: Record<string, number>;
|
||||
hosts: Record<string, { inFlight: number; pausedForMs: number; circuitOpen: boolean; consecutive429: number }>;
|
||||
};
|
||||
|
||||
function parseRetryAfter(value: string | null, now: number): number | undefined {
|
||||
if (!value) return undefined;
|
||||
if (/^\d+$/.test(value.trim())) return Number(value.trim()) * 1000;
|
||||
const at = Date.parse(value);
|
||||
return Number.isNaN(at) ? undefined : Math.max(0, at - now);
|
||||
}
|
||||
|
||||
function isActionApi(url: URL): boolean {
|
||||
return url.pathname === '/w/api.php' && /(^|\.)(wikipedia|wikidata|wikimedia)\.org$/.test(url.hostname);
|
||||
}
|
||||
|
||||
export class WikimediaClient {
|
||||
readonly config: ClientConfig;
|
||||
private readonly fetcher: typeof globalThis.fetch;
|
||||
private readonly clock: Clock;
|
||||
private readonly userAgent: () => string;
|
||||
private cache: WikimediaCache | undefined;
|
||||
private readonly hosts = new Map<string, HostState>();
|
||||
private readonly queues = new Map<Lane, Waiter[]>(LANES.map(lane => [lane, []]));
|
||||
private readonly revalidating = new Set<string>();
|
||||
private pumpTimer: unknown;
|
||||
private pumpAt = Infinity;
|
||||
private day = '';
|
||||
private requestsToday = 0;
|
||||
readonly counters: Record<string, number> = {};
|
||||
|
||||
constructor(options: { fetch?: typeof globalThis.fetch; config?: Partial<ClientConfig>; clock?: Clock;
|
||||
cache?: WikimediaCache; userAgent?: () => string } = {}) {
|
||||
this.fetcher = options.fetch ?? ((...args) => globalThis.fetch(...args));
|
||||
this.config = { ...DEFAULT_CONFIG, ...options.config,
|
||||
maxWaitMs: { ...DEFAULT_CONFIG.maxWaitMs, ...options.config?.maxWaitMs } };
|
||||
this.clock = options.clock ?? realClock;
|
||||
this.userAgent = options.userAgent ?? (() => wikimediaUserAgent(defaultUserAgent()));
|
||||
this.cache = options.cache;
|
||||
}
|
||||
|
||||
/** A pass-through client for injected fetchers (tests): no rate limit, no breaker, no cache. */
|
||||
static unthrottled(fetcher: typeof globalThis.fetch): WikimediaClient {
|
||||
return new WikimediaClient({ fetch: fetcher, config: UNTHROTTLED });
|
||||
}
|
||||
|
||||
attachCache(cache: WikimediaCache | undefined): void { this.cache = cache; }
|
||||
|
||||
private count(name: string): void { this.counters[name] = (this.counters[name] ?? 0) + 1; }
|
||||
|
||||
private get rate(): number { return this.config.requestsPerSecond / this.config.nodesPerIp; }
|
||||
private get cap(): number { return Math.max(1, Math.floor(this.config.dailyCap / this.config.nodesPerIp)); }
|
||||
|
||||
private host(name: string): HostState {
|
||||
let state = this.hosts.get(name);
|
||||
if (!state) {
|
||||
state = { tokens: this.config.burst, refilledAt: this.clock.now(), inFlight: 0,
|
||||
pausedUntil: 0, breakerUntil: 0, consecutive429: 0, backoffExponent: 0 };
|
||||
this.hosts.set(name, state);
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
private refill(state: HostState, now: number): void {
|
||||
state.tokens = Math.min(this.config.burst, state.tokens + (now - state.refilledAt) / 1000 * this.rate);
|
||||
state.refilledAt = now;
|
||||
}
|
||||
|
||||
/** Daily budget by lane: background stops at 80 %, interactive at 100 %, mint has a 10 % reserve. */
|
||||
private overCap(lane: Lane): boolean {
|
||||
const today = new Date(this.clock.now()).toISOString().slice(0, 10);
|
||||
if (today !== this.day) { this.day = today; this.requestsToday = 0; }
|
||||
const limit = lane === 'background' ? this.cap * 0.8 : lane === 'interactive' ? this.cap : this.cap * 1.1;
|
||||
return this.requestsToday >= limit;
|
||||
}
|
||||
|
||||
status(): ClientStatus {
|
||||
const now = this.clock.now();
|
||||
return { day: this.day, requestsToday: this.requestsToday, dailyCap: this.cap, requestsPerSecond: this.rate,
|
||||
queued: LANES.reduce((n, lane) => n + this.queues.get(lane)!.length, 0), counters: { ...this.counters },
|
||||
hosts: Object.fromEntries([...this.hosts].map(([name, s]) => [name,
|
||||
{ inFlight: s.inFlight, pausedForMs: Math.max(0, s.pausedUntil - now), circuitOpen: s.breakerUntil > now,
|
||||
consecutive429: s.consecutive429 }])) };
|
||||
}
|
||||
|
||||
async request(input: URL, options: RequestOptions): Promise<ClientResponse> {
|
||||
const url = new URL(input);
|
||||
if (url.protocol !== 'https:') throw new RangeError('Wikimedia requests must use HTTPS');
|
||||
if (isActionApi(url) && !url.searchParams.has('maxlag')) url.searchParams.append('maxlag', String(this.config.maxlag));
|
||||
const key = options.cache && this.cache ? cacheKey(url) : undefined;
|
||||
const entry = key ? this.cache?.get(key) : undefined;
|
||||
const now = this.clock.now();
|
||||
if (entry && entry.expiresAt > now) {
|
||||
this.count('cache.hit');
|
||||
return { response: entry.toResponse(), signal: new AbortController().signal, fromCache: true };
|
||||
}
|
||||
if (entry && options.cache?.staleWhileRevalidate) {
|
||||
this.count('cache.stale');
|
||||
this.revalidate(url, key!, options);
|
||||
return { response: entry.toResponse(), signal: new AbortController().signal, fromCache: true };
|
||||
}
|
||||
try {
|
||||
return await this.network(url, options, key);
|
||||
} catch (error) {
|
||||
// Prefer a stale copy over an error for anything that is allowed to use the cache.
|
||||
if (entry && error instanceof WikimediaUnavailable) {
|
||||
this.count('cache.staleOnError');
|
||||
return { response: entry.toResponse(), signal: new AbortController().signal, fromCache: true };
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private revalidate(url: URL, key: string, options: RequestOptions): void {
|
||||
if (this.revalidating.has(key)) return;
|
||||
this.revalidating.add(key);
|
||||
void this.network(url, { ...options, lane: 'background' }, key)
|
||||
.then(({ response }) => response.body?.cancel(), () => this.count('cache.revalidateFailed'))
|
||||
.finally(() => this.revalidating.delete(key));
|
||||
}
|
||||
|
||||
private async network(url: URL, options: RequestOptions, key: string | undefined): Promise<ClientResponse> {
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
const result = await this.dispatch(url, options, key);
|
||||
if (result !== 'maxlag') return result;
|
||||
if (attempt >= this.config.maxlagRetries) throw new WikimediaUnavailable('maxlag');
|
||||
}
|
||||
}
|
||||
|
||||
/** Wait for a slot, send one request, and classify the reply. */
|
||||
private async dispatch(url: URL, options: RequestOptions, key: string | undefined): Promise<ClientResponse | 'maxlag'> {
|
||||
const hostName = url.hostname;
|
||||
const wait = this.acquire(hostName, options.lane);
|
||||
if (wait) await wait; // No await on the fast path: an idle host dispatches synchronously.
|
||||
const state = this.host(hostName);
|
||||
const headers = new Headers({ Accept: 'application/json', 'Accept-Encoding': 'gzip', ...options.headers,
|
||||
'User-Agent': this.userAgent() });
|
||||
const entry = key ? this.cache?.get(key) : undefined;
|
||||
if (entry?.etag) headers.set('If-None-Match', entry.etag);
|
||||
if (entry?.lastModified) headers.set('If-Modified-Since', entry.lastModified);
|
||||
const signal = AbortSignal.timeout(options.timeoutMs);
|
||||
this.requestsToday++;
|
||||
this.count(`lane.${options.lane}`);
|
||||
let response: Response;
|
||||
try {
|
||||
response = await this.fetcher(url, { signal, redirect: 'error', headers });
|
||||
} catch (error) {
|
||||
this.release(state);
|
||||
this.count('error.network');
|
||||
throw new WikimediaUnavailable(signal.aborted ? 'timeout' : `network: ${(error as Error).message}`);
|
||||
}
|
||||
const now = this.clock.now();
|
||||
const retryAfter = parseRetryAfter(response.headers.get('retry-after'), now);
|
||||
const apiError = response.headers.get('mediawiki-api-error');
|
||||
if (apiError === 'maxlag') {
|
||||
await response.body?.cancel();
|
||||
this.count('maxlag');
|
||||
this.pause(state, Math.min(this.config.maxBackoffMs, retryAfter ?? 5000), false);
|
||||
this.release(state);
|
||||
return 'maxlag';
|
||||
}
|
||||
if (response.status === 429 || response.status === 503) {
|
||||
await response.body?.cancel();
|
||||
this.count(`http.${response.status}`);
|
||||
if (response.status === 429) state.consecutive429++;
|
||||
const backoff = Math.min(this.config.maxBackoffMs, 2 ** state.backoffExponent * 1000 * (1 + Math.random() * 0.25));
|
||||
state.backoffExponent = Math.min(state.backoffExponent + 1, 20);
|
||||
if (state.consecutive429 >= this.config.breakerThreshold) {
|
||||
this.pause(state, this.config.breakerMs, true);
|
||||
this.count('breaker.open');
|
||||
} else {
|
||||
this.pause(state, Math.min(this.config.maxBackoffMs, Math.max(retryAfter ?? 0, backoff)), false);
|
||||
}
|
||||
this.release(state);
|
||||
throw new WikimediaUnavailable(`HTTP ${response.status}`, response.status);
|
||||
}
|
||||
state.consecutive429 = 0;
|
||||
state.backoffExponent = 0;
|
||||
this.release(state);
|
||||
if (response.status === 304 && entry && key) {
|
||||
this.count('cache.revalidated');
|
||||
this.cache?.touch(key, now + options.cache!.ttlMs);
|
||||
return { response: entry.toResponse(), signal, fromCache: true };
|
||||
}
|
||||
if (key && this.cache && response.status === 200 && !response.redirected && options.cache &&
|
||||
/^application\/json(?:\s*;|\s*$)/i.test(response.headers.get('content-type') ?? '')) {
|
||||
// Store the raw body; callers re-validate it exactly as they would a network reply.
|
||||
const body = await this.readLimited(response, options.cache.maxBytes ?? 1024 * 1024, signal);
|
||||
const stored = this.cache.put(key, { status: response.status, body,
|
||||
contentType: response.headers.get('content-type') ?? '', etag: response.headers.get('etag'),
|
||||
lastModified: response.headers.get('last-modified'), expiresAt: now + options.cache.ttlMs, storedAt: now });
|
||||
return { response: stored.toResponse(), signal, fromCache: false };
|
||||
}
|
||||
if (response.status >= 500) this.count(`http.${response.status}`);
|
||||
return { response, signal, fromCache: false };
|
||||
}
|
||||
|
||||
private async readLimited(response: Response, maxBytes: number, signal: AbortSignal): Promise<Buffer> {
|
||||
const length = response.headers.get('content-length');
|
||||
if (length !== null && (!/^\d+$/.test(length) || Number(length) > maxBytes) || !response.body) {
|
||||
await response.body?.cancel();
|
||||
throw new WikimediaUnavailable('oversized or empty response');
|
||||
}
|
||||
const reader = response.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let size = 0;
|
||||
try {
|
||||
for (;;) {
|
||||
if (signal.aborted) throw new WikimediaUnavailable('timeout');
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
size += value.byteLength;
|
||||
if (size > maxBytes) throw new WikimediaUnavailable('oversized response');
|
||||
chunks.push(value);
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
} catch (error) {
|
||||
throw error instanceof WikimediaUnavailable ? error : new WikimediaUnavailable('read failed');
|
||||
} finally {
|
||||
await reader.cancel().catch(() => {});
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
private pause(state: HostState, ms: number, breaker: boolean): void {
|
||||
const until = this.clock.now() + ms;
|
||||
if (until > state.pausedUntil) {
|
||||
if (state.pausedUntil <= this.clock.now()) {
|
||||
console.warn(`[wikimedia] ${breaker ? 'circuit open' : 'pausing'} for ${Math.ceil(ms / 1000)} s`);
|
||||
}
|
||||
state.pausedUntil = until;
|
||||
}
|
||||
if (breaker) { state.consecutive429 = 0; state.breakerUntil = until; }
|
||||
}
|
||||
|
||||
private release(state: HostState): void {
|
||||
state.inFlight--;
|
||||
this.pump();
|
||||
}
|
||||
|
||||
/** Undefined when a slot was taken at once; otherwise a promise for the slot. Throws on refusal. */
|
||||
private acquire(hostName: string, lane: Lane): Promise<void> | undefined {
|
||||
const state = this.host(hostName);
|
||||
const now = this.clock.now();
|
||||
if (this.overCap(lane)) { this.count('refused.cap'); throw new WikimediaUnavailable('daily cap reached'); }
|
||||
// A paused host refuses at once when the pause outlasts this lane's patience.
|
||||
if (state.pausedUntil - now > this.config.maxWaitMs[lane]) {
|
||||
this.count('refused.paused');
|
||||
throw new WikimediaUnavailable(state.breakerUntil > now ? 'circuit open' : 'host paused');
|
||||
}
|
||||
this.refill(state, now);
|
||||
if (state.pausedUntil <= now && state.inFlight < this.config.concurrencyPerHost && state.tokens >= 1 &&
|
||||
!LANES.some(l => this.queues.get(l)!.some(w => w.host === hostName))) {
|
||||
state.tokens -= 1;
|
||||
state.inFlight++;
|
||||
return undefined;
|
||||
}
|
||||
const queued = LANES.reduce((n, l) => n + this.queues.get(l)!.length, 0);
|
||||
if (queued >= this.config.queueLimit) {
|
||||
// Background work is shed first; a mint request evicts the newest background waiter.
|
||||
const background = this.queues.get('background')!;
|
||||
if (lane === 'background' || background.length === 0) {
|
||||
this.count('refused.queue');
|
||||
throw new WikimediaUnavailable('queue full');
|
||||
}
|
||||
const evicted = background.pop()!;
|
||||
this.clock.clearTimeout(evicted.timer);
|
||||
evicted.fail(new WikimediaUnavailable('queue full'));
|
||||
}
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const waiter: Waiter = { lane, host: hostName, enqueuedAt: now,
|
||||
run: () => { this.clock.clearTimeout(waiter.timer); resolve(); },
|
||||
fail: error => reject(error) };
|
||||
waiter.timer = this.clock.setTimeout(() => {
|
||||
const queue = this.queues.get(lane)!;
|
||||
const index = queue.indexOf(waiter);
|
||||
if (index >= 0) { queue.splice(index, 1); this.count('refused.wait'); reject(new WikimediaUnavailable('queue wait')); }
|
||||
}, this.config.maxWaitMs[lane]);
|
||||
this.queues.get(lane)!.push(waiter);
|
||||
this.pump();
|
||||
});
|
||||
}
|
||||
|
||||
/** Start every waiter that may run now, highest lane first; otherwise arm a timer for the next slot. */
|
||||
private pump(): void {
|
||||
const now = this.clock.now();
|
||||
let next = Infinity;
|
||||
for (const lane of LANES) {
|
||||
const queue = this.queues.get(lane)!;
|
||||
for (let i = 0; i < queue.length;) {
|
||||
const waiter = queue[i];
|
||||
// A host is served strictly by priority: lower lanes wait while a higher lane waits for the same host.
|
||||
if (LANES.slice(0, LANES.indexOf(lane)).some(l => this.queues.get(l)!.some(w => w.host === waiter.host))) { i++; continue; }
|
||||
const state = this.host(waiter.host);
|
||||
this.refill(state, now);
|
||||
if (state.pausedUntil > now) { next = Math.min(next, state.pausedUntil); i++; continue; }
|
||||
if (state.inFlight >= this.config.concurrencyPerHost) { i++; continue; }
|
||||
if (state.tokens < 1) { next = Math.min(next, now + (1 - state.tokens) / this.rate * 1000); i++; continue; }
|
||||
state.tokens -= 1;
|
||||
state.inFlight++;
|
||||
queue.splice(i, 1);
|
||||
waiter.run();
|
||||
}
|
||||
}
|
||||
if (next < Infinity && next < this.pumpAt) {
|
||||
if (this.pumpTimer !== undefined) this.clock.clearTimeout(this.pumpTimer);
|
||||
this.pumpAt = next;
|
||||
this.pumpTimer = this.clock.setTimeout(() => { this.pumpTimer = undefined; this.pumpAt = Infinity; this.pump(); },
|
||||
Math.max(1, Math.ceil(next - now)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The process-wide client. `configureWikimedia()` (server start-up) attaches the disk cache. */
|
||||
export const wikimediaClient = new WikimediaClient({ config: configFromEnv() });
|
||||
@@ -0,0 +1,36 @@
|
||||
/**
|
||||
* Node start-up wiring for the polite Wikimedia client: the disk cache, the User-Agent warning,
|
||||
* the mining prefetch pool and an ops status snapshot. See docs/night2/WIKIMEDIA.md.
|
||||
*/
|
||||
import { resolve } from 'node:path';
|
||||
import { WikimediaCache } from './cache.ts';
|
||||
import { wikimediaClient } from './client.ts';
|
||||
import { MintTemplatePool } from './pool.ts';
|
||||
import { warnIfNoContact } from '../wikimedia-user-agent.ts';
|
||||
import { wikipediaArticles } from '../wiki.ts';
|
||||
import { wikipediaEnrichment } from '../enrichment.ts';
|
||||
|
||||
export { wikimediaClient } from './client.ts';
|
||||
|
||||
/** Serves every mining template; shared by all miners of this node. */
|
||||
export const mintPool = new MintTemplatePool({ articles: wikipediaArticles, enrichment: wikipediaEnrichment });
|
||||
|
||||
let cache: WikimediaCache | undefined;
|
||||
|
||||
/** Call once at start-up. Without it the client still works, with in-memory caching only. */
|
||||
export function configureWikimedia(options: { dataDir: string }): void {
|
||||
warnIfNoContact();
|
||||
if (cache) return;
|
||||
try {
|
||||
cache = new WikimediaCache(resolve(options.dataDir, 'wikimedia-cache.sqlite'));
|
||||
wikimediaClient.attachCache(cache);
|
||||
} catch (error) {
|
||||
console.warn(`[wikimedia] disk cache disabled: ${(error as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function closeWikimedia(): void { wikimediaClient.attachCache(undefined); cache?.close(); cache = undefined; }
|
||||
|
||||
export function wikimediaStatus() {
|
||||
return { client: wikimediaClient.status(), cacheEntries: cache?.size() ?? null, mintPool: mintPool.status() };
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
/**
|
||||
* Mining-template prefetch pool (data review §5, "Mining load").
|
||||
*
|
||||
* Without it every miner costs 3+ Wikimedia requests per template, all in a burst at each new tip.
|
||||
* The pool keeps a few fresh article + stats snapshots ready, and serves the SAME pair to every
|
||||
* template built on the same tip (miners race for one block anyway). Wikimedia load is then about
|
||||
* one pair (3 requests) per block, whatever the number of miners.
|
||||
*
|
||||
* Mint data stays transport-only: each pair comes from the mint-path reads (`random()` and
|
||||
* `getFresh()`, byte-identical parameters, never the disk cache), and the snapshot is stamped when it
|
||||
* is fetched, so `capturedAt` stays true. Pairs older than `maxAgeMs` are discarded, never served.
|
||||
* The pool refills only after it is used, so an idle node makes no requests.
|
||||
*/
|
||||
import type { Article } from '../../shared/protocol.ts';
|
||||
import { snapshotStats, type MintStatsSnapshot, type PageEnrichment } from '../../shared/enrichment.ts';
|
||||
|
||||
export type MintPair = { article: Article; statsSnapshot: MintStatsSnapshot };
|
||||
type Entry = MintPair & { fetchedAt: number };
|
||||
type Sources = {
|
||||
articles: { random(): Promise<Article> };
|
||||
enrichment: { getFresh(pageId: number): Promise<PageEnrichment> };
|
||||
};
|
||||
|
||||
export class MintTemplatePool {
|
||||
private readonly sources: Sources;
|
||||
private readonly size: number;
|
||||
private readonly maxAgeMs: number;
|
||||
private readonly now: () => number;
|
||||
private readonly ready: Entry[] = [];
|
||||
private assigned: { tip: string; entry: Entry } | undefined;
|
||||
private filling: Promise<void> | undefined;
|
||||
private direct: Promise<Entry> | undefined;
|
||||
readonly counters = { served: 0, shared: 0, fetched: 0, expired: 0, failed: 0 };
|
||||
|
||||
constructor(sources: Sources, options: { size?: number; maxAgeMs?: number; now?: () => number } = {}) {
|
||||
this.sources = sources;
|
||||
this.size = options.size ?? 2;
|
||||
this.maxAgeMs = options.maxAgeMs ?? 5 * 60_000;
|
||||
this.now = options.now ?? Date.now;
|
||||
if (!Number.isSafeInteger(this.size) || this.size < 0 || this.size > 16 ||
|
||||
!Number.isFinite(this.maxAgeMs) || this.maxAgeMs <= 0) throw new RangeError('Invalid mint pool options');
|
||||
}
|
||||
|
||||
private fresh(entry: Entry): boolean { return this.now() - entry.fetchedAt < this.maxAgeMs; }
|
||||
|
||||
private async fetchOne(): Promise<Entry> {
|
||||
const article = await this.sources.articles.random();
|
||||
const metrics = await this.sources.enrichment.getFresh(article.pageId);
|
||||
this.counters.fetched++;
|
||||
return { article, statsSnapshot: snapshotStats(metrics), fetchedAt: this.now() };
|
||||
}
|
||||
|
||||
/** The article and snapshot for a template on `tip`. Rejects when Wikimedia is unavailable. */
|
||||
async take(tip: string): Promise<MintPair> {
|
||||
let entry: Entry | undefined;
|
||||
if (this.assigned && this.assigned.tip === tip && this.fresh(this.assigned.entry)) {
|
||||
entry = this.assigned.entry;
|
||||
this.counters.shared++;
|
||||
} else {
|
||||
while (this.ready.length && !this.fresh(this.ready[0])) { this.ready.shift(); this.counters.expired++; }
|
||||
entry = this.ready.shift();
|
||||
if (!entry) {
|
||||
// Cold: fetch directly, and let concurrent templates share that one fetch.
|
||||
this.direct ??= this.fetchOne().finally(() => { this.direct = undefined; });
|
||||
try { entry = await this.direct; }
|
||||
catch (error) { this.counters.failed++; throw error; }
|
||||
if (this.assigned && this.assigned.tip === tip && this.fresh(this.assigned.entry)) entry = this.assigned.entry;
|
||||
}
|
||||
this.assigned = { tip, entry };
|
||||
}
|
||||
this.counters.served++;
|
||||
this.refill();
|
||||
return structuredClone({ article: entry.article, statsSnapshot: entry.statsSnapshot });
|
||||
}
|
||||
|
||||
/** Top the pool up one pair at a time, in the background. Exposed for tests. */
|
||||
refill(): Promise<void> {
|
||||
this.filling ??= (async () => {
|
||||
while (this.ready.filter(e => this.fresh(e)).length < this.size) {
|
||||
try { this.ready.push(await this.fetchOne()); }
|
||||
catch { this.counters.failed++; break; } // The next take() tries again; no retry loop.
|
||||
}
|
||||
})().finally(() => { this.filling = undefined; });
|
||||
return this.filling;
|
||||
}
|
||||
|
||||
status() { return { ready: this.ready.length, size: this.size, maxAgeMs: this.maxAgeMs, ...this.counters }; }
|
||||
}
|
||||
+1
-3
@@ -16,7 +16,6 @@ import { IS_MAINNET, NETWORK_GENESIS_TIME, NETWORK_LABEL } from '../shared/netwo
|
||||
import { StateClient } from '../shared/rpc-client.ts';
|
||||
import { PlaySection } from './game/PlaySection.tsx';
|
||||
import { isNativeApp } from './native.ts';
|
||||
import { MobileMiningPanel } from './MobileMiningPanel.tsx';
|
||||
import { Landing } from './Landing.tsx';
|
||||
import './App.css';
|
||||
import './testnet.css';
|
||||
@@ -191,8 +190,7 @@ export default function App() {
|
||||
<div><span className="metric-icon"><WalletIcon size={19} /></span><div><span>Confirmed ownership</span><strong>{wallet && snapshot ? owned.length : '—'}<small>in your wallet</small></strong></div></div>
|
||||
<div><span className="metric-icon"><Cpu size={19} /></span><div><span>Proof of work</span><strong className="algorithm">{snapshot?.algorithm ?? 'RandomX'}<small>native CPU mining</small></strong></div></div>
|
||||
</section>
|
||||
{isNativeApp() && <MobileMiningPanel ref={minerPanel} wallet={wallet} onAccepted={() => { void refresh().catch(() => {}); }} />}
|
||||
<section className="mining-panel native-mining" ref={isNativeApp() ? undefined : minerPanel} hidden={isNativeApp()}>
|
||||
<section className="mining-panel native-mining" ref={minerPanel} hidden={isNativeApp()}>
|
||||
<div className="mining-description"><div className="section-eyebrow"><span className="idle-dot" /> RANDOMX · DESIGNED FOR GENERAL-PURPOSE CPUS</div><h2>Mining should be for more people.</h2><p>Wikideck and Monero use RandomX, an algorithm designed to resist specialized ASICs and keep general-purpose CPUs competitive. That can make mining more accessible—not make every machine equally powerful or promise rewards. Each accepted Wikideck block earns its miner a distinct illustrated Wikipedia card. Wikideck is its own independent network.</p><div className="mining-meta"><span><Cpu size={13} /> Mine with a CPU</span><span><Shield size={13} /> Independently validating peers</span></div><details className="mining-details"><summary>Hardware, rewards & network details</summary><div className="native-memory"><strong>Light · default</strong><span>256 MiB cache plus scratchpad and process overhead.</span><strong>Fast · --fast</strong><span>About 2 GiB dataset plus cache; budget over 2.3 GiB per process. Mining uses CPU, memory, electricity and time. ASIC resistance is a design goal, not a guarantee; results depend on hardware and network difficulty. Wikideck does not share Monero’s network, hashpower or security.</span></div></details></div>
|
||||
<div className="miner-controls"><div className="miner-display"><div><span>NEXT WIKIPEDIA ARTICLE</span><strong>{snapshot ? 'Chosen when the miner requests work' : `Waiting for ${NETWORK_LABEL} RPC…`}</strong></div></div><p className="muted">The miner selects a random illustrated English Wikipedia article. The same article can appear again; each card keeps its own mint-time statistics.</p><details className="mining-details"><summary>Want to run a miner? Setup & safety details</summary><ol className="native-steps"><li>Build <code>wikideck-randomx</code> using the repository’s native build instructions.</li><li>Save your encrypted backup and keep its password separate.</li><li>Replace <code><backup.json></code> with your local file path (quote paths with spaces), then run:</li></ol><pre className="native-command"><code>{nativeCommand}</code></pre><p className="miner-footnote">Node: {window.location.origin}. The command connects to this node; use HTTPS for remote nodes. Password entry stays local—never put it in the command. It defaults to one worker and at most one block. Ctrl+C stops the native miner; closing this page does not.</p></details><button className="button dark full" onClick={openWallet}><Download size={16} /> Download wallet backup</button><button className="text-button" onClick={() => setDialog('guide')}>Read the {NETWORK_LABEL.toLowerCase()} field guide <ArrowRight size={14} /></button></div>
|
||||
</section>
|
||||
|
||||
@@ -1,144 +0,0 @@
|
||||
// Android (sideload build) mining panel. Rendered only inside the native app; see docs/night2/MOBILE_MINING.md.
|
||||
import { forwardRef, useEffect, useRef, useState } from 'react';
|
||||
import { registerPlugin, type PluginListenerHandle } from '@capacitor/core';
|
||||
import { Cpu, Pickaxe, Square } from 'lucide-react';
|
||||
import { IS_MAINNET, NETWORK_LABEL } from '../shared/network.ts';
|
||||
import { expectedChainIdentity } from '../shared/chain-identity.ts';
|
||||
import { sign, touchWallet, type Wallet } from './wallet.ts';
|
||||
import { nodeHref } from './native.ts';
|
||||
import { MobileMiner, type MinerEngine, type MinerStatus, type MineResult } from './mobile-miner.ts';
|
||||
import './mobile-mining.css';
|
||||
|
||||
type Conditions = { powerSave: boolean; thermalStatus: number; batteryTempC: number | null; batteryPercent: number;
|
||||
charging: boolean; cores: number; abi: string; engineInstalled: boolean; blocked: string | null };
|
||||
interface MinerPlugin {
|
||||
conditions(): Promise<Conditions>;
|
||||
selfTest(): Promise<{ ok: boolean; output: string }>;
|
||||
start(o: { threads: number }): Promise<{ threads: number }>;
|
||||
stop(): Promise<void>;
|
||||
mine(o: { lane: number; key: string; blob: string; target: string; nonceOffset: number; maxAttempts: number }): Promise<MineResult>;
|
||||
addListener(event: 'halted', fn: (e: { reason: string }) => void): Promise<PluginListenerHandle>;
|
||||
}
|
||||
const Native = registerPlugin<MinerPlugin>('WikiDeckMiner');
|
||||
const engine: MinerEngine = {
|
||||
start: async threads => { await Native.start({ threads }); },
|
||||
stop: () => Native.stop(),
|
||||
mine: (lane, key, blob, target, nonceOffset, maxAttempts) => Native.mine({ lane, key, blob, target, nonceOffset, maxAttempts }),
|
||||
};
|
||||
|
||||
const CONSENT_KEY = 'wikideck.mobileMining.consent.v1';
|
||||
const INTENSITY_KEY = 'wikideck.mobileMining.intensity';
|
||||
const INTENSITIES = {
|
||||
low: { label: 'Low: 1 thread, rests between batches', threads: 1, idleMs: 400 },
|
||||
normal: { label: 'Normal: 1 thread', threads: 1, idleMs: 0 },
|
||||
high: { label: 'High: 2 threads (about 512 MiB)', threads: 2, idleMs: 0 },
|
||||
max: { label: 'Max: 4 threads (about 1 GiB), only while charging', threads: 4, idleMs: 0 },
|
||||
} as const;
|
||||
type Intensity = keyof typeof INTENSITIES;
|
||||
const REASONS: Record<string, string> = {
|
||||
background: 'the app went to the background', 'battery-saver': 'battery saver is on', thermal: 'the device is warm',
|
||||
'battery-hot': 'the battery is hot', 'battery-low': 'the battery is below 20% and not charging', 'max-needs-charging': 'Max intensity needs the charger', closed: 'the app closed',
|
||||
};
|
||||
const explain = (reason: string) => REASONS[reason] ?? reason;
|
||||
|
||||
async function api(path: string, body?: unknown): Promise<unknown> {
|
||||
const url = path.startsWith('/p2p/') ? nodeHref(path) : path; // /api/... is routed to the node by native.ts
|
||||
const res = await fetch(url, { method: body ? 'POST' : 'GET', cache: 'no-store', signal: AbortSignal.timeout(30000),
|
||||
headers: body ? { 'Content-Type': 'application/json' } : {}, body: body ? JSON.stringify(body) : undefined });
|
||||
const text = await res.text();
|
||||
if (text.length > 262144) throw new Error('Node response too large');
|
||||
const data = text ? JSON.parse(text) : {};
|
||||
if (!res.ok) throw new Error(data.error ?? `Node rejected request (${res.status})`);
|
||||
return data;
|
||||
}
|
||||
|
||||
export const MobileMiningPanel = forwardRef<HTMLElement, { wallet?: Wallet; onAccepted?: () => void }>(function MobileMiningPanel({ wallet, onAccepted }, ref) {
|
||||
const [consent, setConsent] = useState(() => localStorage.getItem(CONSENT_KEY) === 'yes');
|
||||
const [intensity, setIntensity] = useState<Intensity>(() => (localStorage.getItem(INTENSITY_KEY) as Intensity) in INTENSITIES ? localStorage.getItem(INTENSITY_KEY) as Intensity : 'normal');
|
||||
const [conditions, setConditions] = useState<Conditions>();
|
||||
const [status, setStatus] = useState<MinerStatus>();
|
||||
const [check, setCheck] = useState<{ ok: boolean; output: string }>();
|
||||
const [error, setError] = useState('');
|
||||
const miner = useRef<MobileMiner | undefined>(undefined);
|
||||
const accepted = useRef(0);
|
||||
const active = !!status && ['starting', 'requesting', 'mining', 'submitting'].includes(status.phase);
|
||||
|
||||
useEffect(() => {
|
||||
let live = true;
|
||||
const poll = () => Native.conditions().then(c => { if (live) setConditions(c); }).catch(() => {});
|
||||
void poll();
|
||||
const timer = setInterval(poll, 5000);
|
||||
const halted = Native.addListener('halted', e => { void miner.current?.stop(`Paused: ${explain(e.reason)}`); });
|
||||
const hide = () => { if (document.visibilityState !== 'visible') void miner.current?.stop(`Paused: ${explain('background')}`); };
|
||||
document.addEventListener('visibilitychange', hide);
|
||||
return () => { live = false; clearInterval(timer); void halted.then(h => h.remove()); document.removeEventListener('visibilitychange', hide); void miner.current?.stop(); };
|
||||
}, []);
|
||||
// A different (or locked) wallet must never keep mining to the previous address.
|
||||
useEffect(() => { if (miner.current?.active) void miner.current.stop('Stopped: the wallet changed or locked.'); }, [wallet]);
|
||||
|
||||
const start = async () => {
|
||||
setError('');
|
||||
if (!wallet) { setError('Unlock your wallet first: cards are mined to its address.'); return; }
|
||||
try {
|
||||
let result = check;
|
||||
if (!result?.ok) { result = await Native.selfTest(); setCheck(result); }
|
||||
if (!result.ok) { setError('The RandomX engine failed its known-vector self-test on this device, so mining is disabled.'); return; }
|
||||
const { chainId } = await expectedChainIdentity();
|
||||
const preset = INTENSITIES[intensity];
|
||||
const current = new MobileMiner({
|
||||
engine, api, chainId, idleMs: preset.idleMs,
|
||||
sign: message => sign(wallet, message),
|
||||
blocked: async () => {
|
||||
const c = await Native.conditions(); setConditions(c);
|
||||
if (c.blocked) return explain(c.blocked);
|
||||
if (preset.threads > 2 && !c.charging) return explain('max-needs-charging');
|
||||
return document.visibilityState !== 'visible' ? explain('background') : undefined;
|
||||
},
|
||||
onStatus: s => {
|
||||
// Mining runs only in the foreground with consent; keep the unlocked session alive so a found block can be signed.
|
||||
touchWallet();
|
||||
setStatus(s);
|
||||
if (s.accepted > accepted.current) { accepted.current = s.accepted; onAccepted?.(); }
|
||||
},
|
||||
}, wallet);
|
||||
miner.current = current;
|
||||
void current.start(preset.threads);
|
||||
} catch (e) { setError((e as Error).message); }
|
||||
};
|
||||
const stop = () => { void miner.current?.stop('Stopped.'); };
|
||||
|
||||
if (IS_MAINNET) return <section className="mining-panel mobile-mining" ref={ref}><h2>Mining</h2>
|
||||
<p className="muted">This build mines on the private testnet only. Mainnet mining from the app is not enabled.</p></section>;
|
||||
|
||||
const blockedNow = conditions?.blocked;
|
||||
return <section className="mining-panel mobile-mining" ref={ref} aria-label="Mine on this phone">
|
||||
<div className="section-eyebrow"><Cpu size={14} /> RANDOMX LIGHT · {NETWORK_LABEL.toUpperCase()} · SIDELOAD BUILD</div>
|
||||
<h2>Mine cards on this phone</h2>
|
||||
<p>Each accepted block mints one Wikipedia card to your wallet. The phone runs the same pinned RandomX engine as the desktop miner, in light mode (about 256 MiB per thread). Mining runs only while this screen is open, and it stops by itself on battery saver, when the phone is warm, or below 20% battery when not charging.</p>
|
||||
<label className="mobile-mining-consent"><input type="checkbox" checked={consent} disabled={active}
|
||||
onChange={e => { setConsent(e.target.checked); localStorage.setItem(CONSENT_KEY, e.target.checked ? 'yes' : 'no'); }} />
|
||||
<span>I understand that mining uses this phone's CPU and battery and can make it warm, and I want to mine to my wallet.</span></label>
|
||||
<label className="mobile-mining-intensity">Intensity
|
||||
<select value={intensity} disabled={active} onChange={e => { setIntensity(e.target.value as Intensity); localStorage.setItem(INTENSITY_KEY, e.target.value); }}>
|
||||
{Object.entries(INTENSITIES).map(([k, v]) => <option key={k} value={k} disabled={(k === 'high' && (conditions?.cores ?? 0) < 4) || (k === 'max' && ((conditions?.cores ?? 0) < 4 || !conditions?.charging))}>{v.label}</option>)}
|
||||
</select></label>
|
||||
<div className="mobile-mining-actions">
|
||||
{active ? <button className="button outline" onClick={stop}><Square size={15} /> Stop mining</button>
|
||||
: <button className="button" onClick={() => void start()} disabled={!consent || !wallet || !!blockedNow || conditions?.engineInstalled === false}><Pickaxe size={15} /> Start mining</button>}
|
||||
</div>
|
||||
{!wallet && <p className="muted">Unlock or create your wallet to mine.</p>}
|
||||
{conditions?.engineInstalled === false && <p className="error-text">This APK has no RandomX engine for {conditions.abi}.</p>}
|
||||
{blockedNow && !active && <p className="muted">Paused: {explain(blockedNow)}.</p>}
|
||||
{error && <p className="error-text" role="alert">{error}</p>}
|
||||
<dl className="mobile-mining-stats" aria-live="polite">
|
||||
<div><dt>Hashrate</dt><dd>{status ? `${status.hashrate.toFixed(1)} H/s` : '—'}</dd></div>
|
||||
<div><dt>Status</dt><dd>{status?.phase ?? 'idle'}{status?.height ? ` · block #${status.height}` : ''}</dd></div>
|
||||
<div><dt>Hashes</dt><dd>{status?.attempts.toLocaleString() ?? 0}</dd></div>
|
||||
<div><dt>Accepted blocks</dt><dd>{status?.accepted ?? 0}</dd></div>
|
||||
<div><dt>Battery</dt><dd>{conditions ? `${conditions.batteryPercent}%${conditions.charging ? ' charging' : ''}${conditions.batteryTempC !== null ? ` · ${conditions.batteryTempC.toFixed(1)} °C` : ''}` : '—'}</dd></div>
|
||||
<div><dt>Engine check</dt><dd>{check ? check.ok ? 'known vector OK' : 'FAILED' : 'runs before first start'}</dd></div>
|
||||
</dl>
|
||||
{status?.article && active && <p className="muted">Mining for: {status.article}</p>}
|
||||
{status?.message && <p className="muted">{status.message}</p>}
|
||||
</section>;
|
||||
});
|
||||
@@ -1,173 +0,0 @@
|
||||
// Mobile (Android sideload) mining: the same loop as server/testnet/miner.ts, run in the WebView.
|
||||
// The native WikiDeckMiner plugin (android/.../MinerPlugin.java) only runs the pinned RandomX engine in
|
||||
// LIGHT mode; this module builds work from the node's template, signs the block claim with the local web
|
||||
// wallet and submits it. The plugin never sees a private key or backup password.
|
||||
// Browser-safe: Web Crypto only, no node: imports. See docs/night2/MOBILE_MINING.md.
|
||||
import { CONSENSUS, NETWORK, blockClaimMessage, headerMessage, statsSnapshotTuple,
|
||||
type BlockHeader, type MiningTemplate, type TestnetBlock } from '../shared/testnet.ts';
|
||||
|
||||
export type MineResult = { found: false } | { found: true; hash: string; nonce: string };
|
||||
/** The RandomX engine as seen from the WebView (the Capacitor plugin, or a fake in tests). */
|
||||
export interface MinerEngine {
|
||||
start(threads: number): Promise<void>;
|
||||
stop(): Promise<void>;
|
||||
mine(lane: number, key: string, blobHex: string, target: string, nonceOffset: number, maxAttempts: number): Promise<MineResult>;
|
||||
}
|
||||
export interface MinerIdentity { address: string; publicKey: string }
|
||||
export interface MinerDeps {
|
||||
engine: MinerEngine;
|
||||
/** POST/GET JSON against the node (relative paths such as /api/mining/template). */
|
||||
api: (path: string, body?: unknown) => Promise<unknown>;
|
||||
/** Sign a message with the unlocked web wallet. */
|
||||
sign: (message: string) => Promise<string>;
|
||||
chainId: string;
|
||||
/** Checked before every batch; return a reason to stop (battery saver, thermal, background...). */
|
||||
blocked?: () => string | undefined | Promise<string | undefined>;
|
||||
onStatus?: (status: MinerStatus) => void;
|
||||
now?: () => number;
|
||||
/** Pause between batches per lane (ms): intensity control. */
|
||||
idleMs?: number;
|
||||
random?: () => number;
|
||||
}
|
||||
export type MinerPhase = 'idle' | 'starting' | 'requesting' | 'mining' | 'submitting' | 'stopped' | 'error';
|
||||
export interface MinerStatus {
|
||||
phase: MinerPhase; hashrate: number; attempts: number; height?: number; article?: string;
|
||||
accepted: number; rejected: number; lastBlock?: { height: number; tokenId: string; title: string };
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export const BATCH = 16;
|
||||
const hex = (bytes: Uint8Array) => Array.from(bytes, b => b.toString(16).padStart(2, '0')).join('');
|
||||
export function hexToBytes(value: string): Uint8Array {
|
||||
if (!/^([0-9a-f]{2})*$/i.test(value)) throw new Error('Invalid hex');
|
||||
const out = new Uint8Array(value.length / 2);
|
||||
for (let i = 0; i < out.length; i++) out[i] = parseInt(value.slice(2 * i, 2 * i + 2), 16);
|
||||
return out;
|
||||
}
|
||||
export async function sha256Hex(data: string | Uint8Array): Promise<string> {
|
||||
const bytes = typeof data === 'string' ? new TextEncoder().encode(data) : data;
|
||||
return hex(new Uint8Array(await crypto.subtle.digest('SHA-256', bytes as BufferSource)));
|
||||
}
|
||||
/** Same bytes as consensus `miningBlob`: sha256(headerMessage) || nonce as uint64 little-endian. */
|
||||
export async function miningBlobHex(header: BlockHeader): Promise<string> {
|
||||
const nonce = new Uint8Array(8);
|
||||
new DataView(nonce.buffer).setBigUint64(0, BigInt(header.nonce), true);
|
||||
return await sha256Hex(headerMessage(header)) + hex(nonce);
|
||||
}
|
||||
/** Same value as consensus `mintId`, with the chain id taken from the (verified) header. */
|
||||
export function mintIdOf(h: BlockHeader): Promise<string> {
|
||||
return sha256Hex(JSON.stringify([NETWORK, h.chainId, 'mint', h.previousHash, h.height, h.owner,
|
||||
h.article.pageId, statsSnapshotTuple(h.statsSnapshot)]));
|
||||
}
|
||||
export function meetsTarget(hash: string, target: string): boolean {
|
||||
return BigInt(`0x${hash}`) <= BigInt(`0x${target}`);
|
||||
}
|
||||
/** Check a node template before spending CPU on it. */
|
||||
export async function checkTemplate(t: MiningTemplate, chainId: string, me: MinerIdentity): Promise<void> {
|
||||
const h = t?.header;
|
||||
if (!h || h.network !== NETWORK || h.chainId !== chainId || t.nonceOffset !== 32) throw new Error('Node returned work for another network');
|
||||
if (h.owner !== me.address || h.publicKey !== me.publicKey) throw new Error('Node returned work for another wallet');
|
||||
if (!/^[0-9a-f]{64}$/.test(h.key) || !/^[0-9a-f]{64}$/.test(h.target)) throw new Error('Node returned malformed work');
|
||||
if (typeof t.blob === 'string' && t.blob.slice(0, 64) !== (await miningBlobHex({ ...h, nonce: 0 })).slice(0, 64)) {
|
||||
throw new Error('Node work blob does not match its header');
|
||||
}
|
||||
}
|
||||
/** Assemble the block exactly as server/testnet/miner.ts does. */
|
||||
export async function buildBlock(t: MiningTemplate, nonce: number, powHash: string, sign: (m: string) => Promise<string>): Promise<TestnetBlock> {
|
||||
const header: BlockHeader = { ...t.header, nonce };
|
||||
const blob = await miningBlobHex(header);
|
||||
const hash = await sha256Hex(hexToBytes(blob));
|
||||
const signature = await sign(blockClaimMessage(hash));
|
||||
return { ...header, hash, tokenId: await mintIdOf(header), powHash, signature, transactions: t.transactions };
|
||||
}
|
||||
|
||||
/** Runs until stop() or a safeguard trips. One instance per app. */
|
||||
export class MobileMiner {
|
||||
private stopping = false;
|
||||
private running?: Promise<void>;
|
||||
status: MinerStatus = { phase: 'idle', hashrate: 0, attempts: 0, accepted: 0, rejected: 0 };
|
||||
private readonly deps: MinerDeps;
|
||||
private readonly me: MinerIdentity;
|
||||
constructor(deps: MinerDeps, me: MinerIdentity) { this.deps = deps; this.me = me; }
|
||||
private set(patch: Partial<MinerStatus>) { this.status = { ...this.status, ...patch }; this.deps.onStatus?.(this.status); }
|
||||
get active() { return !!this.running; }
|
||||
|
||||
start(threads = 1, maxBlocks = Infinity): Promise<void> {
|
||||
if (this.running) return this.running;
|
||||
this.stopping = false;
|
||||
this.running = this.loop(Math.max(1, Math.min(4, Math.floor(threads))), maxBlocks)
|
||||
.finally(() => { this.running = undefined; });
|
||||
return this.running;
|
||||
}
|
||||
async stop(reason?: string): Promise<void> {
|
||||
this.stopping = true;
|
||||
if (reason) this.set({ message: reason });
|
||||
await this.deps.engine.stop().catch(() => {});
|
||||
await this.running?.catch(() => {});
|
||||
}
|
||||
|
||||
private async loop(threads: number, maxBlocks: number) {
|
||||
const { engine, api } = this.deps;
|
||||
const now = this.deps.now ?? Date.now, random = this.deps.random ?? Math.random;
|
||||
this.set({ phase: 'starting', message: undefined, hashrate: 0 });
|
||||
try {
|
||||
const reason = await this.deps.blocked?.();
|
||||
if (reason) throw new Error(`Mining is paused: ${reason}`);
|
||||
await engine.start(threads);
|
||||
let accepted = 0;
|
||||
while (!this.stopping && accepted < maxBlocks) {
|
||||
this.set({ phase: 'requesting' });
|
||||
const template = await api('/api/mining/template', { owner: this.me.address, publicKey: this.me.publicKey }) as MiningTemplate;
|
||||
await checkTemplate(template, this.deps.chainId, this.me);
|
||||
const h = template.header;
|
||||
this.set({ phase: 'mining', height: h.height, article: h.article.title });
|
||||
const started = now();
|
||||
let attempts = 0, lastCheck = now(), stale = false;
|
||||
let solution: { nonce: number; hash: string } | undefined;
|
||||
const lane = async (index: number) => {
|
||||
let nonce = Math.floor(random() * 2 ** 32);
|
||||
while (!this.stopping && !solution && !stale) {
|
||||
const blob = await miningBlobHex({ ...h, nonce });
|
||||
const result = await engine.mine(index, h.key, blob, h.target, 32, BATCH);
|
||||
attempts += BATCH;
|
||||
if (result.found) {
|
||||
const n = Number(result.nonce);
|
||||
if (!Number.isSafeInteger(n) || !meetsTarget(result.hash, h.target)) throw new Error('Engine returned an invalid solution');
|
||||
solution ??= { nonce: n, hash: result.hash };
|
||||
return;
|
||||
}
|
||||
nonce += BATCH;
|
||||
const elapsed = (now() - started) / 1000;
|
||||
this.set({ attempts: this.status.attempts + BATCH, hashrate: elapsed > 0 ? attempts / elapsed : 0 });
|
||||
if (index === 0 && now() - lastCheck >= 2000) {
|
||||
lastCheck = now();
|
||||
const reason = await this.deps.blocked?.();
|
||||
if (reason) { this.stopping = true; this.set({ message: `Paused: ${reason}` }); return; }
|
||||
const status = await api('/p2p/status') as { tip?: string };
|
||||
if (status.tip !== h.previousHash || now() / 1000 - h.createdAt > CONSENSUS.targetSeconds * 2) stale = true;
|
||||
}
|
||||
if (this.deps.idleMs) await new Promise(r => setTimeout(r, this.deps.idleMs));
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: threads }, (_, i) => lane(i)));
|
||||
if (this.stopping || !solution) continue;
|
||||
this.set({ phase: 'submitting' });
|
||||
const block = await buildBlock(template, solution.nonce, solution.hash, this.deps.sign);
|
||||
try {
|
||||
const result = await api('/api/blocks', block) as { accepted?: boolean; reason?: string };
|
||||
if (result.accepted === false) throw new Error(result.reason ?? 'Block not accepted');
|
||||
accepted++;
|
||||
this.set({ accepted: this.status.accepted + 1, lastBlock: { height: block.height, tokenId: block.tokenId, title: block.article.title },
|
||||
message: `Accepted block #${block.height}: ${block.article.title}` });
|
||||
} catch (error) {
|
||||
this.set({ rejected: this.status.rejected + 1, message: (error as Error).message });
|
||||
}
|
||||
}
|
||||
this.set({ phase: 'stopped' });
|
||||
} catch (error) {
|
||||
this.set({ phase: this.stopping ? 'stopped' : 'error', message: this.stopping ? this.status.message : (error as Error).message });
|
||||
} finally {
|
||||
await engine.stop().catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
.mobile-mining { display: grid; gap: 12px; }
|
||||
.mobile-mining h2 { margin: 0; }
|
||||
.mobile-mining-consent { display: flex; gap: 10px; align-items: flex-start; }
|
||||
.mobile-mining-consent input { margin-top: 4px; min-width: 20px; min-height: 20px; }
|
||||
.mobile-mining-intensity { display: grid; gap: 6px; }
|
||||
.mobile-mining-intensity select { min-height: 44px; }
|
||||
.mobile-mining-actions .button { min-height: 44px; display: inline-flex; gap: 8px; align-items: center; }
|
||||
.mobile-mining-stats { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px 16px; margin: 0; }
|
||||
.mobile-mining-stats dt { font-size: 0.75rem; opacity: 0.7; }
|
||||
.mobile-mining-stats dd { margin: 0; font-variant-numeric: tabular-nums; }
|
||||
@@ -54,7 +54,7 @@ test('fixed HTTPS sources, encoded title, hidden categories, lower bounds and pl
|
||||
assert.equal(init.redirect, 'error');
|
||||
assert.ok(init.signal instanceof AbortSignal);
|
||||
assert.equal(new Headers(init.headers).get('accept'), 'application/json');
|
||||
assert.match(new Headers(init.headers).get('user-agent')!, /Wikideck/);
|
||||
assert.match(new Headers(init.headers).get('user-agent')!, /WikiDeck/i);
|
||||
assert.equal(new Headers(init.headers).has('origin'), false); // Server-side requests need no CORS workaround.
|
||||
}
|
||||
assert.equal(result.title, title);
|
||||
|
||||
@@ -1,121 +0,0 @@
|
||||
// mobile-mining: the WebView mining loop must build blocks byte-identical to server/testnet/miner.ts.
|
||||
// No network calls: the node API and the RandomX engine are in-process fakes.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, generateKeyPairSync, sign } from 'node:crypto';
|
||||
import { CHAIN_ID, MAX_TARGET, address, makeTemplate, miningBlob, mintId, sha256, validateBlock } from '../server/testnet/consensus.ts';
|
||||
import { type MiningTemplate, type TestnetBlock } from '../shared/testnet.ts';
|
||||
import { deriveStats } from '../shared/enrichment.ts';
|
||||
import type { PowEngine } from '../server/testnet/randomx.ts';
|
||||
import { MobileMiner, buildBlock, checkTemplate, meetsTarget, miningBlobHex, mintIdOf, type MinerEngine, type MineResult } from '../src/mobile-miner.ts';
|
||||
|
||||
const keys = generateKeyPairSync('ec', { namedCurve: 'prime256v1' });
|
||||
const publicKey = keys.publicKey.export({ format: 'der', type: 'spki' }).toString('base64');
|
||||
const me = { publicKey, address: address(publicKey) };
|
||||
const signer = async (message: string) => sign('sha256', Buffer.from(message), { key: keys.privateKey, dsaEncoding: 'ieee-p1363' }).toString('base64');
|
||||
const fakeHash = (key: string, blob: Uint8Array) => createHash('sha256').update(Buffer.concat([Buffer.from(key, 'hex'), Buffer.from(blob)])).digest('hex');
|
||||
// Test-only stand-in for RandomX on both sides (production has no fake PoW).
|
||||
const pow: PowEngine = { hash: async (key, blob) => fakeHash(key, blob), close() {} };
|
||||
|
||||
function template(target = MAX_TARGET): MiningTemplate {
|
||||
const pageBytes = 2048, languageCount = 3;
|
||||
const stats = { capturedAt: '2026-11-17T00:00:00.000Z', revisionId: 7, pageBytes, languageCount, languagesTruncated: false,
|
||||
pageviews: { status: 'unavailable' as const, months: [], latestMonth: '2026-10', total: null, dailyAverage: null, trendPercent: null },
|
||||
stats: deriveStats(pageBytes, languageCount, null) };
|
||||
const article = { pageId: 4242, title: 'Tetrated dodecahedron', extract: 'A deterministic test article.', image: 'https://upload.wikimedia.org/wikipedia/commons/test.jpg',
|
||||
url: 'https://en.wikipedia.org/?curid=4242', revisionId: 7, source: 'live' as const, category: 'Science' };
|
||||
const t = makeTemplate([], {}, me.address, me.publicKey, [], article, stats, Math.floor(Date.now() / 1000));
|
||||
t.header.target = target;
|
||||
t.blob = miningBlob(t.header).toString('hex');
|
||||
return t;
|
||||
}
|
||||
|
||||
/** Fake engine that speaks the plugin's mine contract with the fake hash. */
|
||||
class FakeEngine implements MinerEngine {
|
||||
started = 0; stopped = 0; calls = 0;
|
||||
async start() { this.started++; }
|
||||
async stop() { this.stopped++; }
|
||||
async mine(_lane: number, key: string, blobHex: string, target: string, offset: number, max: number): Promise<MineResult> {
|
||||
this.calls++;
|
||||
const blob = Buffer.from(blobHex, 'hex');
|
||||
let nonce = blob.readBigUInt64LE(offset);
|
||||
for (let i = 0; i < max; i++, nonce++) {
|
||||
blob.writeBigUInt64LE(nonce, offset);
|
||||
const hash = fakeHash(key, blob);
|
||||
if (meetsTarget(hash, target)) return { found: true, hash, nonce: nonce.toString() };
|
||||
}
|
||||
return { found: false };
|
||||
}
|
||||
}
|
||||
|
||||
test('browser blob and mint id match consensus', async () => {
|
||||
const t = template();
|
||||
for (const nonce of [0, 1, 2 ** 32 + 5, Number.MAX_SAFE_INTEGER]) {
|
||||
const header = { ...t.header, nonce };
|
||||
assert.equal(await miningBlobHex(header), miningBlob(header).toString('hex'));
|
||||
}
|
||||
assert.equal(await mintIdOf(t.header), mintId(t.header));
|
||||
assert.equal(t.header.chainId, CHAIN_ID);
|
||||
});
|
||||
|
||||
test('buildBlock produces a block the consensus validator accepts', async () => {
|
||||
const t = template();
|
||||
const header = { ...t.header, nonce: 0 };
|
||||
while (!meetsTarget(fakeHash(header.key, miningBlob(header)), header.target)) header.nonce++;
|
||||
const powHash = fakeHash(header.key, miningBlob(header));
|
||||
const block = await buildBlock(t, header.nonce, powHash, signer);
|
||||
assert.equal(block.hash, sha256(miningBlob(header)));
|
||||
assert.equal(block.signature.length > 0, true);
|
||||
const { block: valid } = await validateBlock(structuredClone(block), [], {}, pow);
|
||||
assert.equal(valid.tokenId, mintId(header));
|
||||
});
|
||||
|
||||
test('checkTemplate rejects work for another network or wallet', async () => {
|
||||
const t = template();
|
||||
await checkTemplate(t, CHAIN_ID, me);
|
||||
await assert.rejects(checkTemplate(t, 'f'.repeat(64), me), /another network/);
|
||||
await assert.rejects(checkTemplate(t, CHAIN_ID, { ...me, address: '0'.repeat(64) }), /another wallet/);
|
||||
await assert.rejects(checkTemplate({ ...t, blob: '00'.repeat(40) }, CHAIN_ID, me), /does not match/);
|
||||
});
|
||||
|
||||
test('MobileMiner mines, signs and submits one accepted block, then stops the engine', async () => {
|
||||
const engine = new FakeEngine();
|
||||
const submitted: TestnetBlock[] = [];
|
||||
const api = async (path: string, body?: unknown) => {
|
||||
if (path === '/api/mining/template') { assert.deepEqual(body, { owner: me.address, publicKey: me.publicKey }); return template('0' + 'f'.repeat(63)); }
|
||||
if (path === '/api/blocks') { submitted.push(body as TestnetBlock); await validateBlock(structuredClone(body), [], {}, pow); return { accepted: true }; }
|
||||
if (path === '/p2p/status') return { tip: 'x' };
|
||||
throw new Error(`unexpected ${path}`);
|
||||
};
|
||||
const miner = new MobileMiner({ engine, api, sign: signer, chainId: CHAIN_ID }, me);
|
||||
await miner.start(1, 1);
|
||||
assert.equal(miner.status.phase, 'stopped', miner.status.message);
|
||||
assert.equal(miner.status.accepted, 1);
|
||||
assert.equal(submitted.length, 1);
|
||||
assert.equal(submitted[0].owner, me.address);
|
||||
assert.ok(engine.stopped >= 1);
|
||||
});
|
||||
|
||||
test('a tripped safeguard stops mining before the engine starts', async () => {
|
||||
const engine = new FakeEngine();
|
||||
const miner = new MobileMiner({ engine, api: async () => { throw new Error('no network'); }, sign: signer, chainId: CHAIN_ID,
|
||||
blocked: () => 'battery saver is on' }, me);
|
||||
await miner.start(1, 1);
|
||||
assert.equal(miner.status.phase, 'error');
|
||||
assert.match(miner.status.message ?? '', /battery saver/);
|
||||
assert.equal(engine.started, 0);
|
||||
assert.equal(engine.calls, 0);
|
||||
});
|
||||
|
||||
test('stop() halts an in-progress search', async () => {
|
||||
const engine = new FakeEngine();
|
||||
const api = async (path: string) => path === '/api/mining/template' ? template('0'.repeat(64)) : { tip: template().header.previousHash };
|
||||
const miner = new MobileMiner({ engine, api, sign: signer, chainId: CHAIN_ID }, me);
|
||||
const run = miner.start(1);
|
||||
for (let i = 0; engine.calls < 3 && i < 2000; i++) await new Promise(r => setTimeout(r, 1));
|
||||
assert.ok(engine.calls >= 3, miner.status.message);
|
||||
await miner.stop('Stopped.');
|
||||
await run;
|
||||
assert.equal(miner.status.phase, 'stopped');
|
||||
assert.equal(miner.status.accepted, 0);
|
||||
});
|
||||
@@ -2,7 +2,7 @@ import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { WikipediaArticleSource } from '../server/wiki.ts';
|
||||
import { WikipediaEnrichment } from '../server/enrichment.ts';
|
||||
import { ARTICLE_USER_AGENT, ENRICHMENT_USER_AGENT, wikimediaUserAgent } from '../server/wikimedia-user-agent.ts';
|
||||
import { PROJECT_URL, defaultUserAgent, wikimediaUserAgent } from '../server/wikimedia-user-agent.ts';
|
||||
|
||||
function capture() {
|
||||
const agents: string[] = [];
|
||||
@@ -12,9 +12,10 @@ function capture() {
|
||||
};
|
||||
return { agents, fetcher };
|
||||
}
|
||||
async function agentsFor(env: string | undefined) {
|
||||
const previous = process.env.WIKIDECK_USER_AGENT;
|
||||
async function agentsFor(env: string | undefined, contact?: string) {
|
||||
const previous = process.env.WIKIDECK_USER_AGENT, previousContact = process.env.WIKIDECK_OPERATOR_CONTACT;
|
||||
if (env === undefined) delete process.env.WIKIDECK_USER_AGENT; else process.env.WIKIDECK_USER_AGENT = env;
|
||||
if (contact === undefined) delete process.env.WIKIDECK_OPERATOR_CONTACT; else process.env.WIKIDECK_OPERATOR_CONTACT = contact;
|
||||
try {
|
||||
const article = capture(), enrichment = capture();
|
||||
await assert.rejects(new WikipediaArticleSource(article.fetcher).random());
|
||||
@@ -22,15 +23,22 @@ async function agentsFor(env: string | undefined) {
|
||||
return { article: article.agents, enrichment: enrichment.agents };
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.WIKIDECK_USER_AGENT; else process.env.WIKIDECK_USER_AGENT = previous;
|
||||
if (previousContact === undefined) delete process.env.WIKIDECK_OPERATOR_CONTACT;
|
||||
else process.env.WIKIDECK_OPERATOR_CONTACT = previousContact;
|
||||
}
|
||||
}
|
||||
|
||||
test('default Wikimedia user agents are unchanged when WIKIDECK_USER_AGENT is unset', async () => {
|
||||
assert.equal(ARTICLE_USER_AGENT, 'Wikideck/0.1 (experimental Wikipedia article cards)');
|
||||
assert.equal(ENRICHMENT_USER_AGENT, 'Wikideck/0.1 (Wikipedia off-chain page enrichment)');
|
||||
const { article, enrichment } = await agentsFor(undefined);
|
||||
assert.ok(article.length > 0 && article.every(a => a === ARTICLE_USER_AGENT));
|
||||
assert.ok(enrichment.length > 0 && enrichment.every(a => a === ENRICHMENT_USER_AGENT));
|
||||
test('default Wikimedia user agent names the project, the operator contact and node', async () => {
|
||||
const { article, enrichment } = await agentsFor(undefined, 'ops@example.org');
|
||||
const expected = `WikiDeck/0.1.0 (+${PROJECT_URL}; ops@example.org) node/${process.versions.node}`;
|
||||
assert.equal(defaultUserAgent({ WIKIDECK_OPERATOR_CONTACT: 'ops@example.org' }), expected);
|
||||
assert.ok(article.length > 0 && article.every(a => a === expected));
|
||||
assert.ok(enrichment.length > 0 && enrichment.every(a => a === expected));
|
||||
assert.equal(defaultUserAgent({ PUBLIC_ORIGIN: 'https://node.example' }),
|
||||
`WikiDeck/0.1.0 (+${PROJECT_URL}; https://node.example) node/${process.versions.node}`);
|
||||
assert.equal(defaultUserAgent({}), `WikiDeck/0.1.0 (+${PROJECT_URL}) node/${process.versions.node}`);
|
||||
// Unsafe contacts are dropped rather than breaking the header's comment syntax.
|
||||
assert.equal(defaultUserAgent({ WIKIDECK_OPERATOR_CONTACT: 'a); evil(' }), defaultUserAgent({}));
|
||||
});
|
||||
|
||||
test('WIKIDECK_USER_AGENT overrides both Wikimedia clients', async () => {
|
||||
|
||||
+4
-1
@@ -56,13 +56,16 @@ test('random article fetch rejects unsafe response types, redirects, and oversiz
|
||||
() => new Response('{}', { headers: { 'Content-Type': 'text/html' } }),
|
||||
() => new Response(null, { status: 302, headers: { Location: 'https://evil.example' } }),
|
||||
() => new Response(' '.repeat(512 * 1024 + 1), { headers: { 'Content-Type': 'application/json' } }),
|
||||
() => { throw new Error('offline'); },
|
||||
];
|
||||
for (const fail of failures) {
|
||||
const { fetcher, calls } = mockFetch(() => fail());
|
||||
await assert.rejects(new WikipediaArticleSource(fetcher).random());
|
||||
assert.equal(calls.length, 3);
|
||||
}
|
||||
// A network failure (or throttling) stops at once instead of retrying against an unreachable upstream.
|
||||
const { fetcher, calls } = mockFetch(() => { throw new Error('offline'); });
|
||||
await assert.rejects(new WikipediaArticleSource(fetcher).random(), /Wikimedia unavailable/);
|
||||
assert.equal(calls.length, 1);
|
||||
});
|
||||
|
||||
test('bad page records are skipped while later valid candidates are considered', async () => {
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { WikimediaClient, WikimediaUnavailable, configFromEnv, type ClientConfig } from '../server/wikimedia/client.ts';
|
||||
import { WikimediaCache, cacheKey } from '../server/wikimedia/cache.ts';
|
||||
import { warnIfNoContact } from '../server/wikimedia-user-agent.ts';
|
||||
import { FakeClock, flush, json, recorder } from './wikimedia-helpers.ts';
|
||||
|
||||
const API = 'https://en.wikipedia.org/w/api.php?action=query&format=json&pageids=1';
|
||||
const REST = 'https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/x';
|
||||
|
||||
function setup(handler: Parameters<typeof recorder>[1] = () => json({ ok: true }), config: Partial<ClientConfig> = {},
|
||||
cache?: WikimediaCache) {
|
||||
const clock = new FakeClock();
|
||||
const rec = recorder(clock, handler);
|
||||
const client = new WikimediaClient({ fetch: rec.fetcher, clock, config, cache, userAgent: () => 'Test/1 (+x; y)' });
|
||||
return { clock, client, ...rec };
|
||||
}
|
||||
|
||||
test('token bucket: burst of 3, then 1 request per second, one in flight per host', async () => {
|
||||
const { clock, client, calls } = setup();
|
||||
const jobs = Array.from({ length: 6 }, () => client.request(new URL(API), { lane: 'interactive', timeoutMs: 1000 })
|
||||
.then(r => r.response.body?.cancel()));
|
||||
await clock.advance(0);
|
||||
assert.equal(calls.length, 3);
|
||||
await clock.advance(999);
|
||||
assert.equal(calls.length, 3);
|
||||
await clock.advance(1);
|
||||
assert.equal(calls.length, 4);
|
||||
await clock.advance(2000);
|
||||
await Promise.all(jobs);
|
||||
assert.deepEqual(calls.map(c => c.at - calls[0].at), [0, 0, 0, 1000, 2000, 3000]);
|
||||
// Different hosts have separate buckets.
|
||||
await client.request(new URL(REST), { lane: 'interactive', timeoutMs: 1000 });
|
||||
assert.equal(calls.at(-1)!.url.hostname, 'wikimedia.org');
|
||||
});
|
||||
|
||||
test('only one request is in flight per host, even with tokens to spare', async () => {
|
||||
const releases: (() => void)[] = [];
|
||||
const { clock, client, calls } = setup(() => new Promise(resolve => releases.push(() => resolve(json({})))));
|
||||
const a = client.request(new URL(API), { lane: 'interactive', timeoutMs: 1000 });
|
||||
const b = client.request(new URL(API), { lane: 'interactive', timeoutMs: 1000 });
|
||||
await clock.advance(0);
|
||||
assert.equal(calls.length, 1);
|
||||
releases.shift()!();
|
||||
await a;
|
||||
await clock.advance(0);
|
||||
assert.equal(calls.length, 2);
|
||||
releases.shift()!();
|
||||
await b;
|
||||
});
|
||||
|
||||
test('priority lanes: mint, then interactive, then background', async () => {
|
||||
const { clock, client, calls } = setup(url => json({ lane: url.searchParams.get('n') }), { burst: 1 });
|
||||
await client.request(new URL(`${API}&n=warm`), { lane: 'background', timeoutMs: 1000 }); // Uses the only token.
|
||||
const order: string[] = [];
|
||||
const go = (lane: 'mint' | 'interactive' | 'background', n: string) =>
|
||||
client.request(new URL(`${API}&n=${n}`), { lane, timeoutMs: 1000 }).then(() => order.push(n));
|
||||
const jobs = [go('background', 'b1'), go('interactive', 'i1'), go('background', 'b2'), go('mint', 'm1'), go('interactive', 'i2')];
|
||||
await clock.advance(5000);
|
||||
await Promise.all(jobs);
|
||||
assert.deepEqual(order, ['m1', 'i1', 'i2', 'b1', 'b2']);
|
||||
assert.equal(calls.length, 6);
|
||||
});
|
||||
|
||||
test('maxlag=5 is added to Action API requests only, and a maxlag reply waits for Retry-After', async () => {
|
||||
let lagged = 1;
|
||||
const { clock, client, calls } = setup(url => url.pathname === '/w/api.php' && lagged-- > 0
|
||||
? json({ error: { code: 'maxlag' } }, { 'MediaWiki-API-Error': 'maxlag', 'Retry-After': '5' })
|
||||
: json({ ok: true }));
|
||||
const job = client.request(new URL(API), { lane: 'background', timeoutMs: 1000 });
|
||||
await clock.advance(0);
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(calls[0].url.searchParams.get('maxlag'), '5');
|
||||
await clock.advance(4999);
|
||||
assert.equal(calls.length, 1);
|
||||
await clock.advance(1);
|
||||
const { response } = await job;
|
||||
assert.deepEqual(await response.json(), { ok: true });
|
||||
assert.equal(calls.length, 2);
|
||||
await client.request(new URL(REST), { lane: 'background', timeoutMs: 1000 });
|
||||
assert.equal(calls[2].url.searchParams.has('maxlag'), false);
|
||||
assert.equal(calls[0].headers.get('user-agent'), 'Test/1 (+x; y)');
|
||||
});
|
||||
|
||||
test('429 honours Retry-After by pausing the host; three in a row open the circuit for 10 minutes', async () => {
|
||||
let status = 429;
|
||||
const { clock, client, calls } = setup(() => status === 200 ? json({}) : new Response('slow down',
|
||||
{ status, headers: { 'Retry-After': '2' } }));
|
||||
const req = (lane: 'mint' | 'background' = 'background') => client.request(new URL(API), { lane, timeoutMs: 1000 });
|
||||
await assert.rejects(req(), (e: unknown) => e instanceof WikimediaUnavailable && e.status === 429);
|
||||
// The next request waits out Retry-After (2 s) instead of going straight back.
|
||||
const second = req().catch(e => e);
|
||||
await clock.advance(1999);
|
||||
assert.equal(calls.length, 1);
|
||||
await clock.advance(1);
|
||||
assert.equal(((await second) as WikimediaUnavailable).status, 429);
|
||||
const third = req().catch(e => e);
|
||||
await clock.advance(10_000);
|
||||
assert.equal(((await third) as WikimediaUnavailable).status, 429);
|
||||
assert.equal(calls.length, 3);
|
||||
assert.equal(client.status().hosts['en.wikipedia.org'].circuitOpen, true);
|
||||
// Open circuit: refused at once without touching Wikimedia, even on the mint lane.
|
||||
status = 200;
|
||||
await assert.rejects(req('mint'), /circuit open/);
|
||||
assert.equal(calls.length, 3);
|
||||
await clock.advance(10 * 60_000);
|
||||
await req('mint');
|
||||
assert.equal(calls.length, 4);
|
||||
assert.equal(client.status().counters['breaker.open'], 1);
|
||||
});
|
||||
|
||||
test('a success resets the consecutive-429 count', async () => {
|
||||
const replies = [429, 200, 429, 200, 429];
|
||||
const { clock, client } = setup(() => {
|
||||
const status = replies.shift()!;
|
||||
return status === 200 ? json({}) : new Response('', { status });
|
||||
});
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const job = client.request(new URL(API), { lane: 'background', timeoutMs: 1000 }).catch(() => {});
|
||||
await clock.advance(60_000);
|
||||
await job;
|
||||
}
|
||||
assert.equal(client.status().hosts['en.wikipedia.org'].circuitOpen, false);
|
||||
});
|
||||
|
||||
test('daily cap: background stops at 80 %, interactive at 100 %, mint keeps a 10 % reserve; resets at UTC midnight', async () => {
|
||||
const { clock, client } = setup(() => json({}), { dailyCap: 10, requestsPerSecond: 1000, burst: 1000 });
|
||||
const req = (lane: 'mint' | 'interactive' | 'background') => client.request(new URL(API), { lane, timeoutMs: 1000 });
|
||||
for (let i = 0; i < 8; i++) await req('background');
|
||||
await assert.rejects(req('background'), /daily cap/);
|
||||
await req('interactive'); await req('interactive');
|
||||
await assert.rejects(req('interactive'), /daily cap/);
|
||||
await req('mint');
|
||||
await assert.rejects(req('mint'), /daily cap/);
|
||||
await clock.advance(24 * 3600_000);
|
||||
await req('background');
|
||||
});
|
||||
|
||||
test('nodes sharing one egress IP split the rate and the daily cap', () => {
|
||||
const config = configFromEnv({ WIKIDECK_WIKIMEDIA_RPS: '0.9', WIKIDECK_WIKIMEDIA_NODES_PER_IP: '3',
|
||||
WIKIDECK_WIKIMEDIA_DAILY_CAP: '3000' });
|
||||
const client = new WikimediaClient({ fetch: async () => json({}), config });
|
||||
assert.ok(Math.abs(client.status().requestsPerSecond - 0.3) < 1e-9);
|
||||
assert.equal(client.status().dailyCap, 1000);
|
||||
assert.equal(configFromEnv({ WIKIDECK_WIKIMEDIA_RPS: 'abc' }).requestsPerSecond, 1);
|
||||
});
|
||||
|
||||
test('a full queue sheds background work first and a mint request evicts a background waiter', async () => {
|
||||
const { clock, client } = setup(() => json({}), { queueLimit: 2, burst: 1 });
|
||||
await client.request(new URL(API), { lane: 'interactive', timeoutMs: 1000 });
|
||||
const b1 = client.request(new URL(API), { lane: 'background', timeoutMs: 1000 }).then(() => 'ok', e => e.message);
|
||||
const b2 = client.request(new URL(API), { lane: 'background', timeoutMs: 1000 }).then(() => 'ok', e => e.message);
|
||||
await assert.rejects(client.request(new URL(API), { lane: 'background', timeoutMs: 1000 }), /queue full/);
|
||||
const mint = client.request(new URL(API), { lane: 'mint', timeoutMs: 1000 });
|
||||
await clock.advance(5000);
|
||||
await mint;
|
||||
assert.deepEqual([await b1, await b2], ['ok', 'Wikimedia unavailable: queue full']);
|
||||
});
|
||||
|
||||
test('waiting longer than the lane allows fails instead of queueing forever', async () => {
|
||||
const { clock, client } = setup(() => json({}), { burst: 1, requestsPerSecond: 0.01, maxWaitMs: { interactive: 3000 } as ClientConfig['maxWaitMs'] });
|
||||
await client.request(new URL(API), { lane: 'interactive', timeoutMs: 1000 });
|
||||
const late = client.request(new URL(API), { lane: 'interactive', timeoutMs: 1000 }).catch(e => e);
|
||||
await clock.advance(3000);
|
||||
assert.match(String(await late), /queue wait/);
|
||||
});
|
||||
|
||||
test('disk cache: ETag revalidation with If-None-Match, 304 refresh, stale-while-revalidate on the background lane', async () => {
|
||||
const cache = new WikimediaCache(':memory:');
|
||||
let body = { v: 1 };
|
||||
const { clock, client, calls } = setup((_url, headers) => headers.get('if-none-match') === '"v1"' && body.v === 1
|
||||
? new Response(null, { status: 304 }) : json(body, { ETag: `"v${body.v}"` }), {}, cache);
|
||||
const policy = { ttlMs: 1000, staleWhileRevalidate: false };
|
||||
const read = async (lane: 'interactive' | 'background' = 'interactive', cachePolicy = policy) =>
|
||||
(await (await client.request(new URL(API), { lane, timeoutMs: 1000, cache: cachePolicy })).response.json());
|
||||
assert.deepEqual(await read(), { v: 1 });
|
||||
assert.deepEqual(await read(), { v: 1 });
|
||||
assert.equal(calls.length, 1); // Fresh hit.
|
||||
await clock.advance(1000);
|
||||
assert.deepEqual(await read(), { v: 1 });
|
||||
assert.equal(calls.length, 2);
|
||||
assert.equal(calls[1].headers.get('if-none-match'), '"v1"'); // Conditional request, answered 304.
|
||||
assert.equal(client.status().counters['cache.revalidated'], 1);
|
||||
// Stale-while-revalidate: the stale copy is served at once; the refresh happens on the background lane.
|
||||
await clock.advance(1000);
|
||||
body = { v: 2 };
|
||||
assert.deepEqual(await read('interactive', { ttlMs: 1000, staleWhileRevalidate: true }), { v: 1 });
|
||||
await clock.advance(0); await flush();
|
||||
assert.equal(calls.length, 3);
|
||||
assert.equal(client.status().counters['lane.background'], 1);
|
||||
assert.deepEqual(await read(), { v: 2 });
|
||||
assert.equal(calls.length, 3);
|
||||
// The key ignores maxlag and parameter order.
|
||||
assert.equal(cacheKey(new URL('https://h.org/p?b=2&a=1&maxlag=5')), cacheKey(new URL('https://h.org/p?a=1&b=2')));
|
||||
});
|
||||
|
||||
test('requests without a cache policy (mint) neither read nor write the cache', async () => {
|
||||
const cache = new WikimediaCache(':memory:');
|
||||
const { client, calls } = setup(() => json({ n: calls.length }), {}, cache);
|
||||
await client.request(new URL(API), { lane: 'interactive', timeoutMs: 1000, cache: { ttlMs: 60_000 } });
|
||||
const minted = await client.request(new URL(API), { lane: 'mint', timeoutMs: 1000 });
|
||||
assert.equal(minted.fromCache, false);
|
||||
assert.equal(calls.length, 2);
|
||||
assert.equal(cache.size(), 1);
|
||||
});
|
||||
|
||||
test('a stale copy is served when the upstream refuses, and non-JSON replies are never cached', async () => {
|
||||
const cache = new WikimediaCache(':memory:');
|
||||
let fail = false, html = false;
|
||||
const { clock, client } = setup(() => fail ? new Response('', { status: 429 })
|
||||
: html ? new Response('<html>', { headers: { 'Content-Type': 'text/html' } }) : json({ v: 1 }), {}, cache);
|
||||
const policy = { ttlMs: 1000 };
|
||||
await client.request(new URL(API), { lane: 'interactive', timeoutMs: 1000, cache: policy });
|
||||
await clock.advance(2000);
|
||||
fail = true;
|
||||
const stale = await client.request(new URL(API), { lane: 'interactive', timeoutMs: 1000, cache: policy });
|
||||
assert.equal(stale.fromCache, true);
|
||||
fail = false; html = true;
|
||||
await clock.advance(60_000);
|
||||
await client.request(new URL(`${API}&other=1`), { lane: 'interactive', timeoutMs: 1000, cache: policy });
|
||||
assert.equal(cache.size(), 1);
|
||||
});
|
||||
|
||||
test('start-up warns when there is no operator contact', () => {
|
||||
const messages: string[] = [];
|
||||
assert.equal(warnIfNoContact({}, m => messages.push(m)), true);
|
||||
assert.match(messages[0], /WIKIDECK_OPERATOR_CONTACT/);
|
||||
assert.equal(warnIfNoContact({ WIKIDECK_OPERATOR_CONTACT: 'ops@example.org' }, m => messages.push(m)), false);
|
||||
assert.equal(warnIfNoContact({ WIKIDECK_USER_AGENT: 'X/1 (+https://x; y)' }, m => messages.push(m)), false);
|
||||
assert.equal(messages.length, 1);
|
||||
});
|
||||
|
||||
test('only HTTPS is allowed', async () => {
|
||||
const { client } = setup();
|
||||
await assert.rejects(client.request(new URL('http://en.wikipedia.org/w/api.php'), { lane: 'mint', timeoutMs: 1 }), RangeError);
|
||||
});
|
||||
@@ -0,0 +1,52 @@
|
||||
// Shared fakes for tests/wikimedia-*.test.ts: a manual clock and a recording fetch. Zero network.
|
||||
import type { Clock } from '../server/wikimedia/client.ts';
|
||||
|
||||
export class FakeClock implements Clock {
|
||||
time: number;
|
||||
private timers: { at: number; fn: () => void; id: number }[] = [];
|
||||
private nextId = 1;
|
||||
constructor(start = Date.parse('2026-11-17T12:00:00Z')) { this.time = start; }
|
||||
now(): number { return this.time; }
|
||||
setTimeout(fn: () => void, ms: number): unknown {
|
||||
const id = this.nextId++;
|
||||
this.timers.push({ at: this.time + Math.max(0, ms), fn, id });
|
||||
return id;
|
||||
}
|
||||
clearTimeout(handle: unknown): void { this.timers = this.timers.filter(t => t.id !== handle); }
|
||||
/** Advance time, firing due timers in order and letting promise chains settle between them. */
|
||||
async advance(ms: number): Promise<void> {
|
||||
const target = this.time + ms;
|
||||
await flush();
|
||||
for (;;) {
|
||||
this.timers.sort((a, b) => a.at - b.at || a.id - b.id);
|
||||
const next = this.timers[0];
|
||||
if (!next || next.at > target) break;
|
||||
this.timers.shift();
|
||||
this.time = Math.max(this.time, next.at);
|
||||
next.fn();
|
||||
await flush();
|
||||
}
|
||||
this.time = target;
|
||||
await flush();
|
||||
}
|
||||
}
|
||||
|
||||
export async function flush(): Promise<void> {
|
||||
for (let i = 0; i < 10; i++) await new Promise(resolve => setImmediate(resolve));
|
||||
}
|
||||
|
||||
export function json(value: unknown, headers: Record<string, string> = {}, status = 200): Response {
|
||||
return new Response(JSON.stringify(value), { status, headers: { 'Content-Type': 'application/json; charset=utf-8', ...headers } });
|
||||
}
|
||||
|
||||
export type Call = { url: URL; headers: Headers; at: number };
|
||||
export function recorder(clock: { now(): number }, handler: (url: URL, headers: Headers) => Response | Promise<Response>) {
|
||||
const calls: Call[] = [];
|
||||
const fetcher: typeof fetch = async (input, init) => {
|
||||
const url = new URL(String(input));
|
||||
const headers = new Headers(init?.headers);
|
||||
calls.push({ url, headers, at: clock.now() });
|
||||
return handler(url, headers);
|
||||
};
|
||||
return { calls, fetcher };
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { WikipediaEnrichment } from '../server/enrichment.ts';
|
||||
import { WikipediaArticleSource, randomArticlesUrl } from '../server/wiki.ts';
|
||||
import { WikimediaClient } from '../server/wikimedia/client.ts';
|
||||
import { WikimediaCache } from '../server/wikimedia/cache.ts';
|
||||
import { UNTHROTTLED } from '../server/wikimedia/client.ts';
|
||||
import { json } from './wikimedia-helpers.ts';
|
||||
|
||||
const JANUARY = new Date('2025-01-15T23:59:59Z');
|
||||
const MONTHS = ['2024-07', '2024-08', '2024-09', '2024-10', '2024-11', '2024-12'];
|
||||
const metadata = (pageId = 736) => ({ query: { pages: [{ pageid: pageId, ns: 0, title: 'Albert Einstein', lastrevid: 1234,
|
||||
length: 100_000, pageprops: { wikibase_item: 'Q937' }, categories: [], langlinks: [{ lang: 'de' }] }] } });
|
||||
const views = (counts = [10, 20, 30, 40, 50, 100]) => ({ items: MONTHS.map((month, i) => ({ timestamp: `${month.replace('-', '')}0100`,
|
||||
views: counts[i], project: 'en.wikipedia', access: 'all-access', agent: 'user', granularity: 'monthly' })) });
|
||||
|
||||
function service(viewsReply: () => Response | Promise<Response>, cache?: WikimediaCache) {
|
||||
const urls: URL[] = [];
|
||||
const fetcher: typeof fetch = async input => {
|
||||
const url = new URL(String(input));
|
||||
urls.push(url);
|
||||
return url.hostname === 'en.wikipedia.org' ? json(metadata(Number(url.searchParams.get('pageids')))) : viewsReply();
|
||||
};
|
||||
const client = new WikimediaClient({ fetch: fetcher, config: UNTHROTTLED, cache });
|
||||
return { urls, client, enrichment: new WikipediaEnrichment({ client, now: () => JANUARY }) };
|
||||
}
|
||||
|
||||
test('mint fix: a 429, 5xx, timeout or network failure on pageviews fails the mint read instead of pinning attention: null', async () => {
|
||||
const failures: (() => Response | Promise<Response>)[] = [
|
||||
() => new Response('', { status: 429, headers: { 'Retry-After': '0' } }),
|
||||
() => new Response('', { status: 503 }),
|
||||
() => new Response('', { status: 500 }),
|
||||
() => new Response('<html>busy</html>', { headers: { 'Content-Type': 'text/html' } }),
|
||||
() => { throw new TypeError('fetch failed'); },
|
||||
() => Promise.reject(new DOMException('The operation timed out.', 'TimeoutError')),
|
||||
];
|
||||
for (const failure of failures) {
|
||||
const { enrichment } = service(failure);
|
||||
await assert.rejects(enrichment.getFresh(736), /unavailable/);
|
||||
// Display reads keep their old behaviour: metadata with unavailable views.
|
||||
const shown = await enrichment.get(736);
|
||||
assert.equal(shown.pageviews.status, 'unavailable');
|
||||
assert.equal(shown.stats.attention, null);
|
||||
}
|
||||
});
|
||||
|
||||
test('mint records unavailable views only for a genuine 404 or missing data', async () => {
|
||||
for (const reply of [() => json({ type: 'not found' }, {}, 404), () => json({ items: [] }), () => json({ items: views().items.slice(1) })]) {
|
||||
const { enrichment } = service(reply);
|
||||
const minted = await enrichment.getFresh(736);
|
||||
assert.equal(minted.pageviews.status, 'unavailable');
|
||||
assert.equal(minted.stats.attention, null);
|
||||
}
|
||||
const { enrichment } = service(() => json(views()));
|
||||
assert.equal((await enrichment.getFresh(736)).stats.attention, 33);
|
||||
});
|
||||
|
||||
test('mint reads never use the disk cache or join a display read', async () => {
|
||||
const cache = new WikimediaCache(':memory:');
|
||||
let count = 100;
|
||||
const { enrichment, urls } = service(() => json(views([1, 1, 1, 1, 1, count++])), cache);
|
||||
const shown = await enrichment.get(736);
|
||||
assert.equal(cache.size(), 2);
|
||||
const [display, minted] = await Promise.all([enrichment.get(736), enrichment.getFresh(736)]);
|
||||
assert.equal(display.pageviews.months[5].views, shown.pageviews.months[5].views);
|
||||
assert.equal(minted.pageviews.months[5].views, 101); // Fetched again, not the cached 100.
|
||||
assert.equal(urls.length, 4);
|
||||
// A later display read (memory cache cleared by a new service) is served from disk: no upstream call.
|
||||
const second = new WikipediaEnrichment({ client: new WikimediaClient({ fetch: async () => { throw new Error('no network'); },
|
||||
config: UNTHROTTLED, cache }), now: () => JANUARY });
|
||||
assert.equal((await second.get(736)).pageviews.months[5].views, 100);
|
||||
});
|
||||
|
||||
test('golden mint URLs: request parameters are byte-identical apart from the added maxlag', async () => {
|
||||
assert.equal(randomArticlesUrl().toString(), 'https://en.wikipedia.org/w/api.php?action=query&format=json&formatversion=2'
|
||||
+ '&generator=random&grnnamespace=0&grnlimit=10&prop=extracts%7Cpageimages%7Cinfo%7Ccategories&exintro=1&explaintext=1'
|
||||
+ '&exchars=900&piprop=thumbnail&pithumbsize=800&cllimit=20');
|
||||
const urls: string[] = [];
|
||||
const fetcher: typeof fetch = async input => {
|
||||
const url = new URL(String(input));
|
||||
urls.push(url.toString());
|
||||
if (url.searchParams.get('generator') === 'random') return json({ query: { pages: [{ pageid: 9, ns: 0, title: 'T',
|
||||
extract: 'An extract that is comfortably longer than forty characters.', lastrevid: 2,
|
||||
thumbnail: { source: 'https://upload.wikimedia.org/a.jpg' } }] } });
|
||||
return url.hostname === 'en.wikipedia.org' ? json(metadata(9)) : json(views());
|
||||
};
|
||||
const client = new WikimediaClient({ fetch: fetcher, config: UNTHROTTLED });
|
||||
const article = await new WikipediaArticleSource(client).random();
|
||||
await new WikipediaEnrichment({ client, now: () => JANUARY }).getFresh(article.pageId);
|
||||
assert.deepEqual(urls, [
|
||||
`${randomArticlesUrl()}&maxlag=5`,
|
||||
'https://en.wikipedia.org/w/api.php?action=query&format=json&formatversion=2&pageids=9'
|
||||
+ '&prop=info%7Ccategories%7Cpageprops%7Cdescription%7Clanglinks&clshow=%21hidden&cllimit=50&lllimit=500'
|
||||
+ '&ppprop=wikibase_item&maxlag=5',
|
||||
'https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia.org/all-access/user/Albert_Einstein/monthly/2024070100/2024120100',
|
||||
]);
|
||||
});
|
||||
|
||||
test('a throttled upstream stops random() at once instead of retrying three times', async () => {
|
||||
let calls = 0;
|
||||
const source = new WikipediaArticleSource(async () => { calls++; return new Response('', { status: 429 }); });
|
||||
await assert.rejects(source.random(), /HTTP 429/);
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
@@ -0,0 +1,74 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { MintTemplatePool } from '../server/wikimedia/pool.ts';
|
||||
import type { Article } from '../shared/protocol.ts';
|
||||
import type { PageEnrichment } from '../shared/enrichment.ts';
|
||||
|
||||
function article(pageId: number): Article {
|
||||
return { pageId, title: `Page ${pageId}`, extract: 'An extract that is comfortably longer than forty characters.',
|
||||
image: 'https://upload.wikimedia.org/a.jpg', url: `https://en.wikipedia.org/?curid=${pageId}`, revisionId: pageId + 1,
|
||||
source: 'live', category: 'Culture' };
|
||||
}
|
||||
function enrichment(pageId: number): PageEnrichment {
|
||||
return { pageId, title: `Page ${pageId}`, description: null, wikidataId: null, revisionId: pageId + 1, pageBytes: 10,
|
||||
languageCount: 1, languagesTruncated: false, categories: [], categoriesTruncated: false, fetchedAt: '2026-11-17T00:00:00Z',
|
||||
pageviews: { status: 'available', months: [], latestMonth: '2026-10', total: 1, dailyAverage: 1, trendPercent: null },
|
||||
stats: { attention: 1, depth: 1, reach: 1 } };
|
||||
}
|
||||
|
||||
function fixture(options: { size?: number; maxAgeMs?: number } = {}) {
|
||||
let next = 1, now = 0, fail = false;
|
||||
const log: string[] = [];
|
||||
const sources = {
|
||||
articles: { random: async () => { log.push('random'); if (fail) throw new Error('Wikimedia unavailable: HTTP 429'); return article(next++); } },
|
||||
enrichment: { getFresh: async (id: number) => { log.push(`fresh:${id}`); return enrichment(id); } },
|
||||
};
|
||||
const pool = new MintTemplatePool(sources, { now: () => now, ...options });
|
||||
return { pool, log, advance: (ms: number) => { now += ms; }, setFail: (value: boolean) => { fail = value; } };
|
||||
}
|
||||
|
||||
test('every template on one tip shares one article; a new tip gets a different one', async () => {
|
||||
const { pool, log } = fixture({ size: 1 });
|
||||
const first = await Promise.all(Array.from({ length: 20 }, () => pool.take('tip-a')));
|
||||
assert.ok(first.every(pair => pair.article.pageId === first[0].article.pageId));
|
||||
await pool.refill();
|
||||
const requestsForTwentyMiners = log.length;
|
||||
assert.equal(requestsForTwentyMiners, 4); // One direct pair, one prefetched pair: 2 calls each.
|
||||
const second = await pool.take('tip-b');
|
||||
assert.notEqual(second.article.pageId, first[0].article.pageId);
|
||||
assert.equal(second.article.pageId, 2); // Served from the prefetched pair, no wait.
|
||||
await pool.refill();
|
||||
assert.equal(log.length, 6); // ...and that use alone triggers the next prefetch.
|
||||
// The stats snapshot comes from a mint-path read of the same page.
|
||||
assert.equal(second.statsSnapshot.revisionId, second.article.pageId + 1);
|
||||
});
|
||||
|
||||
test('pairs older than maxAge are dropped, never served', async () => {
|
||||
const { pool, advance } = fixture({ size: 1, maxAgeMs: 1000 });
|
||||
const a = await pool.take('t1');
|
||||
await pool.refill();
|
||||
advance(1000);
|
||||
const b = await pool.take('t1'); // Same tip, but its pair expired; the ready pair expired too.
|
||||
assert.notEqual(b.article.pageId, a.article.pageId);
|
||||
assert.equal(pool.status().expired, 1);
|
||||
});
|
||||
|
||||
test('an idle pool makes no requests, and a failure rejects the template without a retry loop', async () => {
|
||||
const { pool, log, setFail } = fixture({ size: 2 });
|
||||
assert.equal(log.length, 0);
|
||||
setFail(true);
|
||||
await assert.rejects(pool.take('t1'), /429/);
|
||||
await pool.refill();
|
||||
assert.equal(log.filter(entry => entry === 'random').length, 2); // The direct attempt and one refill attempt.
|
||||
setFail(false);
|
||||
assert.ok((await pool.take('t1')).article.pageId > 0);
|
||||
});
|
||||
|
||||
test('callers get copies they cannot mutate', async () => {
|
||||
const { pool } = fixture();
|
||||
const a = await pool.take('t');
|
||||
a.article.title = 'changed';
|
||||
assert.equal((await pool.take('t')).article.title.startsWith('Page'), true);
|
||||
assert.throws(() => new MintTemplatePool({ articles: { random: async () => article(1) },
|
||||
enrichment: { getFresh: async () => enrichment(1) } }, { size: 99 }), RangeError);
|
||||
});
|
||||
Reference in New Issue
Block a user