commit 14def28b688bb9eaa255782e564afd5f39abc44a Author: Eric Meehan Date: Wed May 28 19:24:37 2025 -0400 Initial commit Created basic application design and data schemas. diff --git a/.env_example b/.env_example new file mode 100644 index 0000000..e69de29 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..09dd1da --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +venv/* diff --git a/db/createChainDatabase.sql b/db/createChainDatabase.sql new file mode 100644 index 0000000..cc9bb20 --- /dev/null +++ b/db/createChainDatabase.sql @@ -0,0 +1,34 @@ +CREATE DATABASE IF NOT EXISTS chain; +USE chain; +CREATE TABLE IF NOT EXISTS blocks( + rowId INT PRIMARY KEY AUTO INCREMENT, + blockId VARCHAR(32) UNIQUE NOT NULL, + blockHash VARCHAR(64) UNIQUE NOT NULL, + previousHash VARCHAR(64) UNIQUE NOT NULL, + timestamp DATETIME NOT NULL, + height INT UNIQUE NOT NULL, + nonce INT NOT NULL +); +CREATE TABLE IF NOT EXISTS cards( + rowId INT PRIMARY KEY AUTO INCREMENT, + cardId VARCHAR(32) UNIQUE NOT NULL, + pageId INT NOT NULL, + FOREIGN KEY (blockId) REFERENCES blocks(blockId) +); +CREATE TABLE IF NOT EXISTS transactions( + rowId INT PRIMARY KEY AUTO INCREMENT, + transactionId VARCHAR(32) UNIQUE NOT NULL, + timestamp DATETIME NOT NULL, + receiver VARCHAR(128) NOT NULL, + isPending BOOLEAN NOT NULL DEFAULT true, + isAbandoned BOOLEAN NOT NULL DEFAULT false, + FOREIGN KEY (blockId) REFERENCES blocks(blockId), + FOREIGN KEY (cardId) REFERENCES cards(cardId) +); +CREATE TABLE IF NOT EXISTS peers( + peerId INT PRIMARY KEY AUTO INCREMENT, + baseUrl VARCHAR(128) UNIQUE NOT NULL, + isUp BOOLEAN NOT NULL, + downCount INT DEFAULT 0, + lastTry TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP +); diff --git a/db/createMarketDatabase.sql b/db/createMarketDatabase.sql new file mode 100644 index 0000000..70a935c --- /dev/null +++ b/db/createMarketDatabase.sql @@ -0,0 +1,40 @@ +CREATE DATABASE IF NOT EXISTS market; +USE market; +CREATE TABLE IF NOT EXISTS users( + rowId INT PRIMARY KEY AUTO INCREMENT, + userId VARCHAR(36) UNIQUE NOT NULL, + userName VARCHAR(64) UNIQUE NOT NULL, + passwordHash VARCHAR(64) NOT NULL, + email VARCHAR(64) UNIQUE NOT NULL, + balance FLOAT NOT NULL DEFAULT 0, + public_key VARCHAR(128) NOT NULL +); +CREATE TABLE IF NOT EXISTS buyOrders( + rowId INT PRIMARY KEY AUTO INCREMENT, + buyOrderId VARCHAR(36) UNIQUE NOT NULL, + pageId INT NOT NULL, + price FLOAT NOT NULL, + volume INT NOT NULL, + fee FLOAT NOT NULL DEFAULT 0, + FOREIGN KEY (userId) REFERENCES users(userId) +); +CREATE TABLE IF NOT EXISTS sellOrders( + rowId INT PRIMARY KEY AUTO INCREMENT, + sellOrderId VARCHAR(36) UNIQUE NOT NULL, + price FLOAT NOT NULL, + fee FLOAT NOT NULL, + FOREIGN KEY (userId) REFERENCES users(userId) +); +CREATE TABLE IF NOT EXISTS sellOrdersItems( + rowId INT PRIMARY KEY AUTO INCREMENT, + cardId VARCHAR(36) NOT NULL, + FOREIGN KEY (sellOrderId) REFERENCES sellOrders(sellOrderId) NOT NULL, +); +CREATE TABLE IF NOT EXISTS transactions( + rowId INT PRIMARY KEY AUTO INCREMENT, + transactionId VARCHAR(36) UNIQUE NOT NULL, + isPending BOOLEAN NOT NULL DEFAULT true, + isAbandoned BOOLEAN NOT NULL DEFAULT false, + FOREIGN KEY (buyOrderId) REFERENCES buyOrders(buyOrderId) NOT NULL, + FOREIGN KEY (sellOrderId) REFERENCES sellOrders(sellOrderId) NOT NULL +); diff --git a/db/createPeersDatabase.sql b/db/createPeersDatabase.sql new file mode 100644 index 0000000..be04f5b --- /dev/null +++ b/db/createPeersDatabase.sql @@ -0,0 +1,3 @@ +CREATE DATABASE IF NOT EXISTS peers; +USE peers; +CREATE TABLE IF NOT EXISTS peers diff --git a/db/getBuyOrders.sql b/db/getBuyOrders.sql new file mode 100644 index 0000000..c2ab0e3 --- /dev/null +++ b/db/getBuyOrders.sql @@ -0,0 +1,12 @@ +SELECT buy_order_id +FROM buy_orders +JOIN cards +ON buy_orders.title = cards.title +WHERE title == desired_title +AND price >= desired_price +AND volume > ( + SELECT count(*) + FROM transactions + WHERE buy_order_id == buy_order_id; +) +ORDER BY price DESC; diff --git a/db/getSellOrders.sql b/db/getSellOrders.sql new file mode 100644 index 0000000..332cde6 --- /dev/null +++ b/db/getSellOrders.sql @@ -0,0 +1,11 @@ +SELECT sell_orders.sell_order_id, sell_order_items.card_id +FROM sell_orders +JOIN sell_order_items +ON sell_orders.sell_order_id = sell_order_items.sell_order_id +WHERE price <= desired_price +AND sell_order_items.card_id NOT IN ( + SELECT card_id + FROM transactions + WHERE sell_order_id == sell_order_id; +) +ORDER BY price ASC; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e2589d2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +wikipedia diff --git a/static/client.js b/static/client.js new file mode 100644 index 0000000..e69de29 diff --git a/wikideck/Market/BuyOrder.py b/wikideck/Market/BuyOrder.py new file mode 100644 index 0000000..f87b936 --- /dev/null +++ b/wikideck/Market/BuyOrder.py @@ -0,0 +1,33 @@ +import json + +from Market.Order import Order + +class BuyOrder(Order): + def __init__(self, cardId=None, price=0, volume=0, fee=0): + super().__init__( + orderType = "buy", + cardId = cardId, + fee = fee + ) + self.price = price + self.volume = volume + + def add_transaction(self, transaction): + if len(self.transactions) < self.volume: + # TODO: update database + self.transactions.append(transaction) + else: + raise self.Fulfilled("Order volume has already been fulfilled.") + + def __str__(self): + return json.dumps({ + "orderId": str(self.orderId), + "timestamp": str(self.timestamp) + "userId": str(self.userId) + "orderType": str(self.orderType), + "cardId": self.cardId, + "price": self.price, + "volume": self.volume, + "fee": self.fee, + "transactions": [str(each) for each in self.transactions] + }) diff --git a/wikideck/Market/Market.py b/wikideck/Market/Market.py new file mode 100644 index 0000000..035aef2 --- /dev/null +++ b/wikideck/Market/Market.py @@ -0,0 +1,39 @@ +from flask import Flask + +market = Flask(__name__) + +# TODO: figure out db + +# TODO: users and logins + +### +# Submit an order to the market. +# This method calls the Order().validate() method to validate the order schema. +# This method performs additional validations against user table. +# User balances are reduced immediately upon receiving a buy order from their account. +### +@app.post('/orders') +def order_post(): + try: + new_order = Order(data=request.get_json()) + new_order.validate() + # TODO: validate order against the Mine + # TODO: write order to db + if newOrder.orderType == 'buy': + # TODO: reduce user balance by ((price + fee) * volume) + continue + return 200 + # TODO: exceptions + except mariadb.Error as e: + return e, 500 + +### +# Retrieve one or more orders. +### +@app.get('/orders') +def orders_get(): + # TODO: order queries + return 200 + +if __name__ == "__main__": + app.run() diff --git a/wikideck/Market/Order.py b/wikideck/Market/Order.py new file mode 100644 index 0000000..fec0819 --- /dev/null +++ b/wikideck/Market/Order.py @@ -0,0 +1,32 @@ +import json +import uuid + +from abc import ABC, abstractmethod + +class Order(ABC): + def __init__(self, orderId=uuid.uuid4(), + timestamp=datetime.datetime.now(datetime.timezone.utc) + userId=None, orderType=None, cardId=None, fee=0, transactions=[], data={}): + self.orderId = orderId + self.timestamp = timestamp + self.userId = userId + self.orderType = orderType + self.cardId = cardId + self.fee = fee + self.transactions = transactions + + @abstractmethod + def add_transaction(self, transaction): + continue + + @abstractmethod + def validate(self): + continue + + @abstractmethod + def load_from_data(self): + continue + + @abstractmethod + def __str__(self): + continue diff --git a/wikideck/Market/SellOrder.py b/wikideck/Market/SellOrder.py new file mode 100644 index 0000000..677f5de --- /dev/null +++ b/wikideck/Market/SellOrder.py @@ -0,0 +1,30 @@ +import json + +class SellOrder(): + def __init__(self, cardId=None, items=[], price=0, fee=0): + super().__init__( + orderType = "sell", + cardId = cardId, + fee = fee + ) + self.items = items + self.price = price + + def add_transaction(self, transaction): + if transaction.cardId in self.items: + # TODO: update database + self.items.pop(transaction.cardId) + self.transactions.append(transaction) + + def __str__(self): + return json.dumps({ + "orderId": str(self.orderId), + "timestamp": str(self.timestamp), + "userId": str(self.userId), + "orderType": str(self.orderType), + "cardId": self.cardId, + "items": self.items, + "price": self.price, + "fee": self.fee, + "transactions": [str(each) for each in self.transactions] + }) diff --git a/wikideck/Mine/Block.py b/wikideck/Mine/Block.py new file mode 100644 index 0000000..9219e33 --- /dev/null +++ b/wikideck/Mine/Block.py @@ -0,0 +1,107 @@ +from card import Card + +import datetime +import hashlib + +class Block(): + def __init__(self, blockId=uuid.uuid4(), previousHash=None, + timestamp=datetime.datetime.now(datetime.timezone.utc), height=0, nonce=0, + card=Card(), transactions=[], authorPublicKey=None, data=None): + self.blockId = blockId + self.previousHash = previousHash + self.timestamp = timestamp + self.height = height + self.difficulty = difficulty + self.nonce = nonce + self.card = card + self.transactions = transactions.append( + Transaction( + cardId = self.card.id, + receiver = authorPublicKey + ) + ) + if data: + self.load_from_data(data) + self.update() + + def update(self): + self.blockHash = hashlib.sha256(str(self).encode('utf-8')) + + def mine(self): + while int(self.blockHash.hexdigest(), 16) > 2**(256-self.difficulty): + self.nonce += 1 + self.update() + + ### + # Validate the internal block structure. + # This method confirms that the correct data types have been used and that + # values are basically valid. + # Further validations against the existing chain must be done elsewhere. + # This method calls the Card.validate() and Transaction.validate() methods to validate the + # respective schemas. + ### + def validate(self): + # TODO: validate blockId is uuid + # TODO: validate previousHash is sha256 hash + if not int(self.blockHash.hexdigest(), 16) > 2**(256-self.difficulty): + raise self.Invalid("Hash does not meet difficulty requirement.") + # TODO: validate timestamp is timestamp + if not self.timestamp < datetime.datetime.now(): + raise self.Invalid("Timestamp in the future.") + if not self.height > 0: + raise self.Invalid("Height less than 0.") + if not self.difficulty > 0: + raise self.Invalid("Difficulty less than 0.") + if not self.nonce > 0: + raise self.Invalid("Nonce less than 0.") + self.card.validate() + for transaction in self.transactions: + transaction.validate() + + def load_from_data(self, data): + self.blockId = uuid.UUID(data['blockId']) + self.previousHash = data['previousHash'] + self.timestamp = datetime.datetime.strptime( + data['timestamp'], + "%Y-%m-%d %H:%M:%S.%f%z" + ) + self.height = data['height'] + self.difficulty = data['difficulty'] + self.nonce = data['nonce'] + self.card = Card( + cardId = uuid.UUID(data['card']['cardId']), + pageId = data['card']['pageId'] + ) + self.transactions = [ + Transaction( + transactionId = uuid.UUID(each['transactionId']), + timestamp = datetime.datetime.strptime( + each['timestamp'], + "%Y-%m-%d %H:%M:%S.%f" + ), + cardId = uuid.UUID(each['cardId']), + sender = each['sender'], + receiver = each['receiver'], + signature = each['signature'] + ) for each in data['transactions'] + ] + self.update() + + def __str__(self): + # The hash of the block is the SHA256 hash of what this method returns. + return data.dumps({ + "blockId": str(self.blockId) + "previousHash": self.previousHash, + "timestamp": str(self.timestamp) + "height": self.height, + "difficulty": self.difficulty, + "nonce": self.nonce, + "author": self.author, + "card": str(self.card), + "transactions": [str(each) for each in transactions] + }) + + class Invalid(Exception): + def __init__(message, status_code=406): + super().__init__(message) + self.status_code = status_code diff --git a/wikideck/Mine/Card.py b/wikideck/Mine/Card.py new file mode 100644 index 0000000..fd73b47 --- /dev/null +++ b/wikideck/Mine/Card.py @@ -0,0 +1,25 @@ +import requests +import wikipedia +import uuid + +class Card(): + def __init__(self, cardId=uuid.uuid4(), pageId=wikipedia.page(wikipedia.random()).pageid): + self.cardId = cardId + self.pageId = pageId + + def validate(self): + try: + wikipedia.page(pageid=self.pageId) + except Exception as e: + raise self.Invalid("Page ID does not match a Wikipedia page.") + + def __str__(self): + return json.dumps({ + "cardId": str(self.cardId), + "pageId": self.pageId + }) + + class Invalid(Exception): + def __init__(self, message, status_code=406): + super().__init__(message) + self.status_code = status_code diff --git a/wikideck/Mine/Mine.py b/wikideck/Mine/Mine.py new file mode 100644 index 0000000..5534e61 --- /dev/null +++ b/wikideck/Mine/Mine.py @@ -0,0 +1,158 @@ +from flask import Flask +import json + +mine = Flask(__name__) +db = mariadb.connect(**config) + +@app.get('/') +def mine(): + # TODO: return a page to mine through the browser + return 200 + +### +# Submit a block to the chain. +# This method calls the Block().validate() method to validate the block schema. +# This method performs additional validations against the rest of the chain. +### +@app.post('/blocks') +def blocks_post(): + previousBlock = get_last_block() + try: + newBlock = Block(data=request.get_json()) + newBlock.validate() + previousBlock = db.get_last_block() + if newBlock.previousHash != previousBlock.blockHash: + raise Block.Invalid( + f"Incorrect previous hash - should be {previousBlock.blockHash}." + ) + if newBlock.timestamp <= previousBlock.timestamp: + raise Block.Invalid( + "Timestamp is later than previous block." + ) + if newBlock.height != previousBlock.height + 1: + raise Block.Invalid( + f"Incorrect block height - should be {previousBlock.height + 1}." + ) + if newBlock.difficulty != DIFFICULTY_REQUIREMENT: + raise Block.Invalid( + f"Incorrect difficulty - should be {DIFFICULTY_REQUIREMENT}." + ) + if len(newBlock.transactions) == 0: + raise Block.Invalid( + "Block contains no transactions." + ) + for transaction in newBlock.transactions: + pendingTransaction = db.get_transaction(transaction.transactionId) + if not pendingTransaction: + raise Transaction.Invalid( + f"No matching pending transaction for {transaction.transactionId}." + ) + if not pendingTransaction.pending: + raise Transaction.AlreadyFulfilled( + f"Transaction {transaction.transactionId} has already been fulfilled." + ) + if transaction.timestamp != pendingTransaction.timestamp: + raise Transaction.Invalid( + f"Incorrect timestamp on {transaction.transactionId}." + ) + if transaction.cardId != pendingTransaction.cardId: + raise Transaction.Invalid( + f"Incorrect cardId on {transaction.transactionId}." + ) + if transaction.sender != pendingTransaction.sender: + raise Transaction.Invalid( + f"Incorrect sender on {transaction.transactionId}." + ) + if transaction.recipient != pendingTransaction.recipient: + raise Transaction.Invalid( + f"Incorrect recipient on {transaction.transactionId}." + ) + if transaction.signature != pendingTransaction.signature: + raise Transaction.Invalid( + f"Incorrect signature on {transaction.transactionId}." + ) + # TODO: write to database + with open(f"{DATA_PATH}/{newBlock.blockId}.json", 'w') as f: + f.write(str(newBlock)) + # TODO: update peers + return str(newBlock), 200 + except: Transaction.Invalid as e: + return e, e.statusCode + except: Transaction.AlreadyFulfilled as e: + return e, e.statusCode + except: Card.Invalid as e: + return e, e.statusCode + except: Block.Invalid as e: + return e, e.statusCode + +### +# Retrieve blocks and block data. +# Returns a skeleton block to be mined by default. +# Queries for a specific block when given parameters. +### +@app.get('/blocks') +def blocks_get(): + blockHash = request.args.get('blockHash', None) + height = request.args.get('height', None) + # TODO: block queries + return str( + Block( + previousHash = lastBlock.blockHash, + height = lastBlock.height + 1, + difficulty = DIFFICULTY_REQUIREMENT, + transactions = [ + Transaction(data=each) for each in get_pending_transactions() + ] + ) + ) + +### +# Retrieve card data +### +@app.get('/cards') +def cards_get(): + # TODO: query cards + return 200 + +### +# Submit a transaction to be mined in a block. +# This method performs a number of validations on the submitted transaction and returns +# a status code result. +### +@app.put('/transactions') +def transactions_put(): + try: + newTransaction = Transaction(data=request.get_json()) + newTransaction.validate() + # TODO: validate transaction against blockchain + # TODO: add transaction to database + # TODO: update peers? + return 200 + except Transaction.Unauthorized as e: + return e, e.statusCode + except Transaction.Invalid as e: + return e, e.statusCode + +### +# Retrieve a transaction. +### +@app.get('/transactions') +def transactions_get(): + transactionId = request.args.get('transactionId', None) + return get_transaction_by_id(transactionId) if transactionId else json.dumps( + db.get_pending_transactions() + ) + +@app.post('/peers') +def peers_post(): + # TODO: validate peer + # TODO: add peers to database + return 200 + +@app.get('/peers') +def peers_get(): + # TODO: query peers + return 200 + +if __name__ == "__main__": + app.run() diff --git a/wikideck/Mine/Transaction.py b/wikideck/Mine/Transaction.py new file mode 100644 index 0000000..6efa999 --- /dev/null +++ b/wikideck/Mine/Transaction.py @@ -0,0 +1,74 @@ +import datetime +import data +import uuid + +class Transaction(): + def __init__(self, transactionId=uuid.uuid4(), + timestamp=datetime.datetime.now(datetime.timezone.utc), + cardId=None, sender=None, receiver=None, signature=None, data=None): + self.transactionId = transactionId + self.timestamp = timestamp + self.cardId = cardId + self.sender = sender + self.receiver = receiver + self.signature = signature + if data: + self.load_from_data(data) + + def validate(self): + # TODO: validate transactionId + # TODO: validate timestamp + # TODO: validate cardId + # TODO: validate sender + # TODO: validate receiver + # TODO: validate signature + if False: + raise self.Unauthorized() + elif False: + raise self.Invalid() + + def sign(self, privateKey): + # TODO: use rsa private key to sign block + return None + + def load_from_data(self, data): + self.transactionId = uuid.UUID(data['transactionId']) + self.timestamp = datetime.datetime.strptime( + data['timestamp'], + "%Y-%m-%d %H:%M:%S.%f%z" + ) + self.cardId = uuid.UUID(data['cardId']) + self.sender = data['sender'] + self.receiver = data['receiver'] + self.signature = data['signature'] + + + def __str__(self): + return data.dumps({ + "transactionId": str(self.id), + "timestamp": str(self.timestamp), + "cardId": self.cardId, + "sender": self.sender, + "receiver": self.receiver, + "signature": self.signature + }) + + class Unauthorized(Exception): + def __init__(self, message, statusCode=403): + super().__init__(message) + self.statusCode = statusCode + + class Invalid(Exception): + def __init__(self, message, statusCode=400): + super().__init__(message) + self.statusCode = statusCode + + class AlreadyFulfilled(Exception): + def __init__(self, message, statusCode=400): + super().__init__(message) + self.statusCode = statusCode + + class Abandoned(Exception): + def __init__(self, message, statusCode=500): + super().__init__(message) + self.statusCode = statusCode diff --git a/wikideck/OrderMatcher/OrderMatcher.py b/wikideck/OrderMatcher/OrderMatcher.py new file mode 100644 index 0000000..ef9611f --- /dev/null +++ b/wikideck/OrderMatcher/OrderMatcher.py @@ -0,0 +1,21 @@ +### +# The OrderMatcher converts orders to transactions. +# When new orders arrive on the market, the order matcher is triggered to search the +# database for an existing order to fulfill the new one. +# If the order matcher finds a match for the order, it creates a transaction and +# sends it to the mine. +### +class OrderMatcher(): + def __init__(self): + return + + def match_orders(self): + # TODO: find matching orders + # TODO: generate transactions + # TODO: send transactions to mine + +if __name__ == '__main__': + orderMatcher = OrderMatcher() + while True: + orderMatcher.match_orders() + time.sleep(10) diff --git a/wikideck/StatusChecker/StatusChecker.py b/wikideck/StatusChecker/StatusChecker.py new file mode 100644 index 0000000..db4f28a --- /dev/null +++ b/wikideck/StatusChecker/StatusChecker.py @@ -0,0 +1,39 @@ +### +# The StatusChecker updates the market database based on the mine database. +# It will periodically query for pending transactions, and then confirm their status against +# the Mine API. +# It will also update user balances upon transaction completion. +### + +class StatusChecker(): + def __init__(self): + return + + def update_pending_transactions(self): + for pendingTransaction in self.db.get_pending_transactions(): + self.check_transaction_status(pendingTransaction) + + def check_transaction_status(transaction): + transaction = Transaction( + data = transaction + ) + # TODO: get transaction data from mine + if not mine_transaction.is_pending: + db.update_transaction_pending_status( + transactionId=transactionId, + is_pending=False + ) + if not mine_transaction.block_id: + raise Transaction.Abandoned( + f"Transaction {pendingTransaction.transactionId} was abandoned." + ) + else: + # TODO: increase seller balance by (price - fee) + # TODO: increase market balance by (fee) + return + +if __name__ == '__main__': + statusChecker = StatusChecker() + while True: + transactionStatusChecker.update_pending_transactions() + time.sleep(INTERVAL) diff --git a/wikideck/app.py b/wikideck/app.py new file mode 100644 index 0000000..d6ac1f0 --- /dev/null +++ b/wikideck/app.py @@ -0,0 +1,30 @@ +import os +import dotenv + +ROLE = os.getenv("ROLE", None) +INTERVAL = os.getenv("INTERVAL", 15) + +if __name__ == "__main__": + dotenv.load_dotenv() + if ROLE == "market": + from Market.Market import market + market.run() + elif ROLE == "mine": + from Mine.Mine import mine + mine.run() + elif ROLE == "order_matcher": + from OrderMatcher.OrderMatcher import OrderMatcher + import time + orderMatcher = OrderMatcher() + while True: + orderMatcher.match_orders() + time.sleep(INTERVAL) + elif ROLE == "status_checker": + from StatusChecker.StatusChecker import StatusChecker + import time + statusChecker = StatusChecker() + while True: + statusChecker.update_pending_transactions() + time.sleep(INTERVAL) + else: + raise Exception("Role must be one of: market, mine, order_matcher, status_checker")