From c83f5feb4b98f2834f9ed7f36298300b5b87ed5b Mon Sep 17 00:00:00 2001 From: Eric Meehan Date: Fri, 30 May 2025 00:28:08 -0400 Subject: [PATCH] Developing components. * Created database classes * Wrote initial queries * Added validations --- db/createChainDatabase.sql | 11 ++- db/createMarketDatabase.sql | 29 ++++--- db/getDeck.sql | 10 +++ db/getLastTransactionForCard.sql | 0 wikideck/Market/BuyOrder.py | 14 ---- wikideck/Market/Database.py | 12 +++ wikideck/Market/Market.py | 15 ++-- wikideck/Market/Order.py | 4 - wikideck/Market/SellOrder.py | 15 +--- wikideck/Mine/Card.py | 10 ++- wikideck/Mine/Database.py | 111 ++++++++++++++++++++++++++ wikideck/Mine/Mine.py | 32 +++++--- wikideck/Mine/Transaction.py | 4 +- wikideck/OrderMatcher/Database.py | 64 +++++++++++++++ wikideck/OrderMatcher/OrderMatcher.py | 38 +++++++-- wikideck/StatusChecker/Database.py | 12 +++ wikideck/app.py | 21 +++-- 17 files changed, 320 insertions(+), 82 deletions(-) create mode 100644 db/getDeck.sql create mode 100644 db/getLastTransactionForCard.sql create mode 100644 wikideck/Market/Database.py create mode 100644 wikideck/Mine/Database.py create mode 100644 wikideck/OrderMatcher/Database.py create mode 100644 wikideck/StatusChecker/Database.py diff --git a/db/createChainDatabase.sql b/db/createChainDatabase.sql index cc9bb20..70265ba 100644 --- a/db/createChainDatabase.sql +++ b/db/createChainDatabase.sql @@ -1,7 +1,7 @@ CREATE DATABASE IF NOT EXISTS chain; USE chain; CREATE TABLE IF NOT EXISTS blocks( - rowId INT PRIMARY KEY AUTO INCREMENT, + sqlId INT PRIMARY KEY AUTO INCREMENT, blockId VARCHAR(32) UNIQUE NOT NULL, blockHash VARCHAR(64) UNIQUE NOT NULL, previousHash VARCHAR(64) UNIQUE NOT NULL, @@ -10,23 +10,26 @@ CREATE TABLE IF NOT EXISTS blocks( nonce INT NOT NULL ); CREATE TABLE IF NOT EXISTS cards( - rowId INT PRIMARY KEY AUTO INCREMENT, + sqlId INT PRIMARY KEY AUTO INCREMENT, cardId VARCHAR(32) UNIQUE NOT NULL, pageId INT NOT NULL, FOREIGN KEY (blockId) REFERENCES blocks(blockId) ); CREATE TABLE IF NOT EXISTS transactions( - rowId INT PRIMARY KEY AUTO INCREMENT, + sqlId INT PRIMARY KEY AUTO INCREMENT, transactionId VARCHAR(32) UNIQUE NOT NULL, timestamp DATETIME NOT NULL, + sender VARCHAR(128) NOT NULL, receiver VARCHAR(128) NOT NULL, + signature VARCHAR(128) NOT NULL, isPending BOOLEAN NOT NULL DEFAULT true, isAbandoned BOOLEAN NOT NULL DEFAULT false, FOREIGN KEY (blockId) REFERENCES blocks(blockId), FOREIGN KEY (cardId) REFERENCES cards(cardId) ); CREATE TABLE IF NOT EXISTS peers( - peerId INT PRIMARY KEY AUTO INCREMENT, + sqlId INT PRIMARY KEY AUTO INCREMENT, + peerId VARCHAR(32) UNIQUE NOT NULL, baseUrl VARCHAR(128) UNIQUE NOT NULL, isUp BOOLEAN NOT NULL, downCount INT DEFAULT 0, diff --git a/db/createMarketDatabase.sql b/db/createMarketDatabase.sql index 70a935c..16d7337 100644 --- a/db/createMarketDatabase.sql +++ b/db/createMarketDatabase.sql @@ -1,40 +1,45 @@ CREATE DATABASE IF NOT EXISTS market; USE market; CREATE TABLE IF NOT EXISTS users( - rowId INT PRIMARY KEY AUTO INCREMENT, + sqlId INT PRIMARY KEY AUTO INCREMENT, userId VARCHAR(36) UNIQUE NOT NULL, userName VARCHAR(64) UNIQUE NOT NULL, passwordHash VARCHAR(64) NOT NULL, email VARCHAR(64) UNIQUE NOT NULL, balance FLOAT NOT NULL DEFAULT 0, - public_key VARCHAR(128) NOT NULL + publicKey VARCHAR(128) NOT NULL ); -CREATE TABLE IF NOT EXISTS buyOrders( - rowId INT PRIMARY KEY AUTO INCREMENT, +CREATE TABLE IF NOT EXISTS buy_orders( + sqlId INT PRIMARY KEY AUTO INCREMENT, buyOrderId VARCHAR(36) UNIQUE NOT NULL, + timestamp DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP(), + expires DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP() + INTERVAL 3 DAY, pageId INT NOT NULL, price FLOAT NOT NULL, volume INT NOT NULL, fee FLOAT NOT NULL DEFAULT 0, + sold INT NOT NULL DEFAULT 0, FOREIGN KEY (userId) REFERENCES users(userId) ); -CREATE TABLE IF NOT EXISTS sellOrders( - rowId INT PRIMARY KEY AUTO INCREMENT, +CREATE TABLE IF NOT EXISTS sell_orders( + sqlId INT PRIMARY KEY AUTO INCREMENT, sellOrderId VARCHAR(36) UNIQUE NOT NULL, + timestamp DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP(), + expires DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP() + INTERVAL 3 DAY, price FLOAT NOT NULL, fee FLOAT NOT NULL, FOREIGN KEY (userId) REFERENCES users(userId) ); -CREATE TABLE IF NOT EXISTS sellOrdersItems( - rowId INT PRIMARY KEY AUTO INCREMENT, +CREATE TABLE IF NOT EXISTS sell_orders_items( + sqlId INT PRIMARY KEY AUTO INCREMENT, cardId VARCHAR(36) NOT NULL, - FOREIGN KEY (sellOrderId) REFERENCES sellOrders(sellOrderId) NOT NULL, + FOREIGN KEY (sellOrderId) REFERENCES sell_orders(sellOrderId) NOT NULL ); CREATE TABLE IF NOT EXISTS transactions( - rowId INT PRIMARY KEY AUTO INCREMENT, + sqlId INT PRIMARY KEY AUTO INCREMENT, transactionId VARCHAR(36) UNIQUE NOT NULL, isPending BOOLEAN NOT NULL DEFAULT true, isAbandoned BOOLEAN NOT NULL DEFAULT false, - FOREIGN KEY (buyOrderId) REFERENCES buyOrders(buyOrderId) NOT NULL, - FOREIGN KEY (sellOrderId) REFERENCES sellOrders(sellOrderId) NOT NULL + FOREIGN KEY (buyOrderId) REFERENCES buy_orders(buyOrderId) NOT NULL, + FOREIGN KEY (sellOrderItemSqlId) REFERENCES sell_order_items(sqlId) NOT NULL ); diff --git a/db/getDeck.sql b/db/getDeck.sql new file mode 100644 index 0000000..7ea9a40 --- /dev/null +++ b/db/getDeck.sql @@ -0,0 +1,10 @@ +SELECT cardId, pageId, transactionId +FROM ( + SELECT cardId, pageId, transactionId, ROW_NUMBER() OVER ( + PARTITION BY cardId ORDER BY timestamp DESC + ) AS rowNumber + FROM transactions + JOIN cards + WHERE receiver == {userId} +) AS subquery +WHERE rowNumber == 0; diff --git a/db/getLastTransactionForCard.sql b/db/getLastTransactionForCard.sql new file mode 100644 index 0000000..e69de29 diff --git a/wikideck/Market/BuyOrder.py b/wikideck/Market/BuyOrder.py index f87b936..1aa7da5 100644 --- a/wikideck/Market/BuyOrder.py +++ b/wikideck/Market/BuyOrder.py @@ -15,19 +15,5 @@ class BuyOrder(Order): def add_transaction(self, transaction): if len(self.transactions) < self.volume: # TODO: update database - self.transactions.append(transaction) else: raise self.Fulfilled("Order volume has already been fulfilled.") - - def __str__(self): - return json.dumps({ - "orderId": str(self.orderId), - "timestamp": str(self.timestamp) - "userId": str(self.userId) - "orderType": str(self.orderType), - "cardId": self.cardId, - "price": self.price, - "volume": self.volume, - "fee": self.fee, - "transactions": [str(each) for each in self.transactions] - }) diff --git a/wikideck/Market/Database.py b/wikideck/Market/Database.py new file mode 100644 index 0000000..92fecf0 --- /dev/null +++ b/wikideck/Market/Database.py @@ -0,0 +1,12 @@ +class Database(): + def __init__(self, host, port, user, password): + self.host = host + self.port = port + self.user = user + self.password = password + self.conn = mariadb.connect( + host = self.host, + port = self.port, + user = self.user, + password = self.password + ) diff --git a/wikideck/Market/Market.py b/wikideck/Market/Market.py index 035aef2..64c46cd 100644 --- a/wikideck/Market/Market.py +++ b/wikideck/Market/Market.py @@ -1,11 +1,17 @@ -from flask import Flask +import flask -market = Flask(__name__) +from Market.Database import Database -# TODO: figure out db +market = flask.Blueprint("market", __name__) +db = Database() # TODO: users and logins +@app.get('/') +def market(): + # TODO: return a page to view market stats + return 200 + ### # Submit an order to the market. # This method calls the Order().validate() method to validate the order schema. @@ -34,6 +40,3 @@ def order_post(): def orders_get(): # TODO: order queries return 200 - -if __name__ == "__main__": - app.run() diff --git a/wikideck/Market/Order.py b/wikideck/Market/Order.py index fec0819..831f283 100644 --- a/wikideck/Market/Order.py +++ b/wikideck/Market/Order.py @@ -26,7 +26,3 @@ class Order(ABC): @abstractmethod def load_from_data(self): continue - - @abstractmethod - def __str__(self): - continue diff --git a/wikideck/Market/SellOrder.py b/wikideck/Market/SellOrder.py index 677f5de..258f2e9 100644 --- a/wikideck/Market/SellOrder.py +++ b/wikideck/Market/SellOrder.py @@ -16,15 +16,6 @@ class SellOrder(): self.items.pop(transaction.cardId) self.transactions.append(transaction) - def __str__(self): - return json.dumps({ - "orderId": str(self.orderId), - "timestamp": str(self.timestamp), - "userId": str(self.userId), - "orderType": str(self.orderType), - "cardId": self.cardId, - "items": self.items, - "price": self.price, - "fee": self.fee, - "transactions": [str(each) for each in self.transactions] - }) + def validate(self): + # TODO: no duplicate items + return diff --git a/wikideck/Mine/Card.py b/wikideck/Mine/Card.py index fd73b47..374e580 100644 --- a/wikideck/Mine/Card.py +++ b/wikideck/Mine/Card.py @@ -3,16 +3,24 @@ import wikipedia import uuid class Card(): - def __init__(self, cardId=uuid.uuid4(), pageId=wikipedia.page(wikipedia.random()).pageid): + def __init__(self, cardId=uuid.uuid4(), + pageId=wikipedia.page(wikipedia.random()).pageid, data=None): self.cardId = cardId self.pageId = pageId + if data: + self.load_from_data(data) def validate(self): try: + # TODO: cardId is UUID wikipedia.page(pageid=self.pageId) except Exception as e: raise self.Invalid("Page ID does not match a Wikipedia page.") + def load_from_data(self, data): + self.cardId = uuid.UUID(data['cardId']) + self.pageId = data['pageId'] + def __str__(self): return json.dumps({ "cardId": str(self.cardId), diff --git a/wikideck/Mine/Database.py b/wikideck/Mine/Database.py new file mode 100644 index 0000000..39a0d74 --- /dev/null +++ b/wikideck/Mine/Database.py @@ -0,0 +1,111 @@ +class Database(): + SQL_GET_LAST_BLOCK = """ + SELECT * + FROM blocks + ORDER BY timestamp DESC + LIMIT 1; + """ + + SQL_GET_CARD = """ + SELECT * + FROM cards + WHERE cardId == {}; + """ + + SQL_GET_CARD_OWNER = """ + SELECT receiver + FROM transactions + WHERE cardId == {} + AND isPending == false + ORDER BY timestamp DESC + LIMIT 1; + """ + + SQL_GET_PENDING_TRANSACTIONS = """ + SELECT * + FROM transactions + WHERE isPending == true; + """ + + SQL_GET_TRANSACTION = """ + SELECT * + FROM transactions + WHERE transactionId == {} + """ + + SQL_INSERT_BLOCK = """ + INSERT INTO blocks (blockId, blockHash, previousHash, timestamp, height, nonce) + VALUES ({}, {}, {}, {}, {}, {}); + """ + + SQL_INSERT_TRANSACTION = """ + INSERT INTO transactions ( + transactionId, + timestamp, + sender, + receiver, + signature, + blockId, + cardId + ) + VALUES ({}, {}, {}, {}, {}); + """ + + def __init__(self, host, port, user, password): + self.host = host + self.port = port + self.user = user + self.password = password + self.conn = mariadb.connect( + host = self.host, + port = self.port, + user = self.user, + password = self.password + ) + + def get_last_block(self): + return Block( + data=self.conn.cursor().execute(SQL_GET_LAST_BLOCK)[0] + ) + + def get_card(self, cardId): + return Card( + data=self.conn.cursor().execute(SQL_GET_CARD.format(cardId))[0] + ) + + def get_card_owner(self, cardId): + return self.conn.cursor().execute( + SQL_GET_CARD_OWNER.format(cardId) + )[0] + + def get_pending_transactions(self): + return [ + Transaction( + data=each + ) for each in self.conn.cursor().execute(SQL_GET_PENDING_TRANSACTIONS) + ] + + def get_transaction(self, transactionId): + return Transaction( + data=self.conn.cursor().execute(SQL_GET_TRANSACTION.format(transactionId)) + ) + + def insert_block(self, block): + return self.conn.cursor().execute(SQL_INSERT_BLOCK.format( + str(block.blockId), + block.blockHash.hexdigest(), + block.previousHash.hexdigest(), + str(block.timestamp), + block.height, + block.nonce + )) + + def insert_transaction(self, transaction): + return self.conn.cursor().execute(SQL_INSERT_TRANSACTION.format( + str(transaction.transactionId), + str(transaction.timestamp), + transaction.sender, + transaction.receiver, + transaction.signature, + str(transaction.cardId) + )) diff --git a/wikideck/Mine/Mine.py b/wikideck/Mine/Mine.py index 5534e61..20c6364 100644 --- a/wikideck/Mine/Mine.py +++ b/wikideck/Mine/Mine.py @@ -1,8 +1,10 @@ -from flask import Flask +import flask import json -mine = Flask(__name__) -db = mariadb.connect(**config) +from Mine.Database import Database + +mine = flask.Blueprint("mine", __name__) +db = Database() @app.get('/') def mine(): @@ -71,9 +73,10 @@ def blocks_post(): raise Transaction.Invalid( f"Incorrect signature on {transaction.transactionId}." ) - # TODO: write to database + db.insert_block(block) with open(f"{DATA_PATH}/{newBlock.blockId}.json", 'w') as f: f.write(str(newBlock)) + # TODO: update transactions # TODO: update peers return str(newBlock), 200 except: Transaction.Invalid as e: @@ -101,7 +104,7 @@ def blocks_get(): height = lastBlock.height + 1, difficulty = DIFFICULTY_REQUIREMENT, transactions = [ - Transaction(data=each) for each in get_pending_transactions() + Transaction(data=each) for each in db.get_pending_transactions() ] ) ) @@ -111,7 +114,9 @@ def blocks_get(): ### @app.get('/cards') def cards_get(): + # TODO: render cards in html # TODO: query cards + # TODO: get decks return 200 ### @@ -124,10 +129,20 @@ def transactions_put(): try: newTransaction = Transaction(data=request.get_json()) newTransaction.validate() - # TODO: validate transaction against blockchain - # TODO: add transaction to database + if not get_card(newTransaction.cardId): + raise Transaction.Invalid( + f"Card {newTransaction.cardId} does not exist.", + 404 + ) + if newTransaction.sender != get_card_owner(newTransaction.cardId): + raise Transaction.Unauthorized( + f"{newTransaction.sender} does not own {newTransaction.cardId}." + ) + insert_transaction(newTransaction) # TODO: update peers? return 200 + except mariadb.Error as e: + return e, 500 except Transaction.Unauthorized as e: return e, e.statusCode except Transaction.Invalid as e: @@ -153,6 +168,3 @@ def peers_post(): def peers_get(): # TODO: query peers return 200 - -if __name__ == "__main__": - app.run() diff --git a/wikideck/Mine/Transaction.py b/wikideck/Mine/Transaction.py index 6efa999..a60c69b 100644 --- a/wikideck/Mine/Transaction.py +++ b/wikideck/Mine/Transaction.py @@ -23,9 +23,7 @@ class Transaction(): # TODO: validate receiver # TODO: validate signature if False: - raise self.Unauthorized() - elif False: - raise self.Invalid() + raise self.Unauthorized("Failed signature check.") def sign(self, privateKey): # TODO: use rsa private key to sign block diff --git a/wikideck/OrderMatcher/Database.py b/wikideck/OrderMatcher/Database.py new file mode 100644 index 0000000..c9983c8 --- /dev/null +++ b/wikideck/OrderMatcher/Database.py @@ -0,0 +1,64 @@ +class Database(): + SQL_GET_BUY_ORDERS = """ + SELECT * + FROM orders + WHERE orderType == 'buy'; + """ + + SQL_GET_UNFULFILLED_BUY_ORDERS = """ + SELECT b.buyOrderId, b.price, b.volume - COUNT(t.sqlId) AS remainingVolume + FROM buy_orders b + JOIN transactions t + ON b.buyOrderId = t.buyOrderId + WHERE b.expires > CURRENT_TIMESTAMP() + GROUP BY b.buyOrderId + HAVING COUNT(t.sqlId) < b.volume + ORDER BY b.timestamp ASC + """ + + SQL_GET_AVAILABLE_SELL_ORDER_ITEMS_FOR_BUY_ORDER = """ + SELECT o.sellOrderId, o.price, o.fee, o.userId, i.sqlId, i.cardId + FROM sell_orders o + JOIN sell_order_items i + ON o.sellOrderId = i.sellOrderId + WHERE i.cardId == {} + AND o.price <= {} + AND o.expires > CURRENT_TIMESTAMP() + AND i.sqlId NOT IN ( + SELECT sellOrderItemSqlId + FROM transactions + WHERE isAbandoned == false; + ) + ORDER BY price DESC; + """ + + SQL_INSERT_TRANSACTION = """ + INSERT INTO transactions (transactionId, buyOrderId, sellOrderItemSqlId) + VALUES ({}, {}, {}); + """ + + def __init__(self): + self.conn = mariadb.connect( + # TODO: default values + host = os.getenv("ORDER_MATCHER_DB_HOST", None), + port = os.getenv("ORDER_MATCHER_DB_PORT", 3342), + user = os.getenv("ORDER_MATCHER_DB_USER", None), + password = os.getenv("ORDER_MATCHER_DB_PASSWORD", None) + ) + + def get_available_sell_order_items_for_buy_order(self, buyOrderId, price): + return self.conn.cursor().execute( + SQL_GET_AVAILABLE_SELL_ORDER_ITEMS_FOR_BUY_ORDER.format( + buyOrderId, + price + ) + ) + + def insert_transaction(self, transactionId, buyOrderId, sellOrderItemSqlId): + self.conn.cursor().execute( + SQL_INSERT_TRANSACTION.format( + transactionId, + buyOrderId, + sellOrderItemSqlId + ) + ) diff --git a/wikideck/OrderMatcher/OrderMatcher.py b/wikideck/OrderMatcher/OrderMatcher.py index ef9611f..5983499 100644 --- a/wikideck/OrderMatcher/OrderMatcher.py +++ b/wikideck/OrderMatcher/OrderMatcher.py @@ -5,17 +5,45 @@ # If the order matcher finds a match for the order, it creates a transaction and # sends it to the mine. ### + +import requests + +from Mine.Transaction import Transaction +from OrderMatcher.Database import Database + class OrderMatcher(): def __init__(self): - return + self.db = Database() def match_orders(self): - # TODO: find matching orders - # TODO: generate transactions - # TODO: send transactions to mine + try: + for buyOrder in self.db.get_unfulfilled_buy_orders(): + for item in self.db.get_available_sell_order_items_for_buy_order( + buyOrder[0], + buyOrder[1] + )[:buyOrder[2]]: + newTransaction = Transaction( + cardId = item[5], + sender = item[3], + receiver = currentUser, # TODO: get current currentUser + ) + newTransaction.sign(userPrivateKey) # TODO: get userPrivateKey + r = requests.post( + f"{MINE_URL}/transactions", + data=str(newTransaction) + ) + # TODO: only insert on success + self.db.insert_transaction( + newTransaction.transactionId, + buyOrder[0], + item[5] + ) + # TODO: error handling + except Exception as e: + print("crap!") if __name__ == '__main__': orderMatcher = OrderMatcher() while True: orderMatcher.match_orders() - time.sleep(10) + time.sleep(INTERVAL) diff --git a/wikideck/StatusChecker/Database.py b/wikideck/StatusChecker/Database.py new file mode 100644 index 0000000..92fecf0 --- /dev/null +++ b/wikideck/StatusChecker/Database.py @@ -0,0 +1,12 @@ +class Database(): + def __init__(self, host, port, user, password): + self.host = host + self.port = port + self.user = user + self.password = password + self.conn = mariadb.connect( + host = self.host, + port = self.port, + user = self.user, + password = self.password + ) diff --git a/wikideck/app.py b/wikideck/app.py index d6ac1f0..df45a6a 100644 --- a/wikideck/app.py +++ b/wikideck/app.py @@ -1,30 +1,29 @@ -import os import dotenv +import time +import os + +dotenv.load_dotenv() ROLE = os.getenv("ROLE", None) INTERVAL = os.getenv("INTERVAL", 15) if __name__ == "__main__": - dotenv.load_dotenv() - if ROLE == "market": + if ROLE == "api": from Market.Market import market - market.run() - elif ROLE == "mine": from Mine.Mine import mine - mine.run() + app = Flask(__name__) + app.register_blueprint(market, url_prefix="/market") + app.register_blueprint(mine, url_prefix="/mine") + app.run() elif ROLE == "order_matcher": from OrderMatcher.OrderMatcher import OrderMatcher - import time orderMatcher = OrderMatcher() while True: orderMatcher.match_orders() time.sleep(INTERVAL) elif ROLE == "status_checker": from StatusChecker.StatusChecker import StatusChecker - import time statusChecker = StatusChecker() while True: - statusChecker.update_pending_transactions() + statusChecker.check_pending_transactions() time.sleep(INTERVAL) - else: - raise Exception("Role must be one of: market, mine, order_matcher, status_checker")