Initial commit

Created basic application design and data schemas.

.gitignore

@@ -0,0 +1 @@
+venv/*

db/createChainDatabase.sql

@@ -0,0 +1,34 @@
+CREATE DATABASE IF NOT EXISTS chain;
+USE chain;
+CREATE TABLE IF NOT EXISTS blocks(
+    rowId INT PRIMARY KEY AUTO INCREMENT,
+    blockId VARCHAR(32) UNIQUE NOT NULL,
+    blockHash VARCHAR(64) UNIQUE NOT NULL,
+    previousHash VARCHAR(64) UNIQUE NOT NULL,
+    timestamp DATETIME NOT NULL,
+    height INT UNIQUE NOT NULL,
+    nonce INT NOT NULL
+);
+CREATE TABLE IF NOT EXISTS cards(
+    rowId INT PRIMARY KEY AUTO INCREMENT,
+    cardId VARCHAR(32) UNIQUE NOT NULL,
+    pageId INT NOT NULL,
+    FOREIGN KEY (blockId) REFERENCES blocks(blockId)
+);
+CREATE TABLE IF NOT EXISTS transactions(
+    rowId INT PRIMARY KEY AUTO INCREMENT,
+    transactionId VARCHAR(32) UNIQUE NOT NULL,
+    timestamp DATETIME NOT NULL,
+    receiver VARCHAR(128) NOT NULL,
+    isPending BOOLEAN NOT NULL DEFAULT true,
+    isAbandoned BOOLEAN NOT NULL DEFAULT false,
+    FOREIGN KEY (blockId) REFERENCES blocks(blockId),
+    FOREIGN KEY (cardId) REFERENCES cards(cardId)
+);
+CREATE TABLE IF NOT EXISTS peers(
+    peerId INT PRIMARY KEY AUTO INCREMENT,
+    baseUrl VARCHAR(128) UNIQUE NOT NULL,
+    isUp BOOLEAN NOT NULL,
+    downCount INT DEFAULT 0,
+    lastTry TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+);

db/createMarketDatabase.sql

@@ -0,0 +1,40 @@
+CREATE DATABASE IF NOT EXISTS market;
+USE market;
+CREATE TABLE IF NOT EXISTS users(
+    rowId INT PRIMARY KEY AUTO INCREMENT,
+    userId VARCHAR(36) UNIQUE NOT NULL,
+    userName VARCHAR(64) UNIQUE NOT NULL,
+    passwordHash VARCHAR(64) NOT NULL,
+    email VARCHAR(64) UNIQUE NOT NULL,
+    balance FLOAT NOT NULL DEFAULT 0,
+    public_key VARCHAR(128) NOT NULL
+);
+CREATE TABLE IF NOT EXISTS buyOrders(
+    rowId INT PRIMARY KEY AUTO INCREMENT,
+    buyOrderId VARCHAR(36) UNIQUE NOT NULL,
+    pageId INT NOT NULL,
+    price FLOAT NOT NULL,
+    volume INT NOT NULL,
+    fee FLOAT NOT NULL DEFAULT 0,
+    FOREIGN KEY (userId) REFERENCES users(userId)
+);
+CREATE TABLE IF NOT EXISTS sellOrders(
+    rowId INT PRIMARY KEY AUTO INCREMENT,
+    sellOrderId VARCHAR(36) UNIQUE NOT NULL,
+    price FLOAT NOT NULL,
+    fee FLOAT NOT NULL,
+    FOREIGN KEY (userId) REFERENCES users(userId)
+);
+CREATE TABLE IF NOT EXISTS sellOrdersItems(
+    rowId INT PRIMARY KEY AUTO INCREMENT,
+    cardId VARCHAR(36) NOT NULL,
+    FOREIGN KEY (sellOrderId) REFERENCES sellOrders(sellOrderId) NOT NULL,
+);
+CREATE TABLE IF NOT EXISTS transactions(
+    rowId INT PRIMARY KEY AUTO INCREMENT,
+    transactionId VARCHAR(36) UNIQUE NOT NULL,
+    isPending BOOLEAN NOT NULL DEFAULT true,
+    isAbandoned BOOLEAN NOT NULL DEFAULT false,
+    FOREIGN KEY (buyOrderId) REFERENCES buyOrders(buyOrderId) NOT NULL,
+    FOREIGN KEY (sellOrderId) REFERENCES sellOrders(sellOrderId) NOT NULL
+);

db/createPeersDatabase.sql

@@ -0,0 +1,3 @@
+CREATE DATABASE IF NOT EXISTS peers;
+USE peers;
+CREATE TABLE IF NOT EXISTS peers

db/getBuyOrders.sql

@@ -0,0 +1,12 @@
+SELECT buy_order_id
+FROM buy_orders
+JOIN cards
+ON buy_orders.title = cards.title
+WHERE title == desired_title
+AND price >= desired_price
+AND volume > (
+    SELECT count(*)
+    FROM transactions
+    WHERE buy_order_id == buy_order_id;
+)
+ORDER BY price DESC;

db/getSellOrders.sql

@@ -0,0 +1,11 @@
+SELECT sell_orders.sell_order_id, sell_order_items.card_id
+FROM sell_orders
+JOIN sell_order_items
+ON sell_orders.sell_order_id = sell_order_items.sell_order_id
+WHERE price <= desired_price
+AND sell_order_items.card_id NOT IN (
+    SELECT card_id
+    FROM transactions
+    WHERE sell_order_id == sell_order_id;
+)
+ORDER BY price ASC;

requirements.txt

@@ -0,0 +1 @@
+wikipedia

wikideck/Market/BuyOrder.py

@@ -0,0 +1,33 @@
+import json
+
+from Market.Order import Order
+
+class BuyOrder(Order):
+    def __init__(self, cardId=None, price=0, volume=0, fee=0):
+        super().__init__(
+            orderType = "buy",
+            cardId = cardId,
+            fee = fee
+            )
+        self.price = price
+        self.volume = volume
+
+    def add_transaction(self, transaction):
+        if len(self.transactions) < self.volume:
+            # TODO: update database
+            self.transactions.append(transaction)
+        else:
+            raise self.Fulfilled("Order volume has already been fulfilled.")
+
+    def __str__(self):
+        return json.dumps({
+            "orderId": str(self.orderId),
+            "timestamp": str(self.timestamp)
+            "userId": str(self.userId)
+            "orderType": str(self.orderType),
+            "cardId": self.cardId,
+            "price": self.price,
+            "volume": self.volume,
+            "fee": self.fee,
+            "transactions": [str(each) for each in self.transactions]
+            })

wikideck/Market/Market.py

@@ -0,0 +1,39 @@
+from flask import Flask
+
+market = Flask(__name__)
+
+# TODO: figure out db
+
+# TODO: users and logins
+
+###
+# Submit an order to the market.
+# This method calls the Order().validate() method to validate the order schema.
+# This method performs additional validations against user table.
+# User balances are reduced immediately upon receiving a buy order from their account.
+###
+@app.post('/orders')
+def order_post():
+    try:
+        new_order = Order(data=request.get_json())
+        new_order.validate()
+        # TODO: validate order against the Mine
+        # TODO: write order to db
+        if newOrder.orderType == 'buy':
+            # TODO: reduce user balance by ((price + fee) * volume)
+            continue
+        return 200
+    # TODO: exceptions
+    except mariadb.Error as e:
+        return e, 500
+
+###
+# Retrieve one or more orders.
+###
+@app.get('/orders')
+def orders_get():
+    # TODO: order queries
+    return 200
+
+if __name__ == "__main__":
+    app.run()

wikideck/Market/Order.py

@@ -0,0 +1,32 @@
+import json
+import uuid
+
+from abc import ABC, abstractmethod
+
+class Order(ABC):
+    def __init__(self, orderId=uuid.uuid4(),
+                 timestamp=datetime.datetime.now(datetime.timezone.utc)
+                 userId=None, orderType=None, cardId=None, fee=0, transactions=[], data={}):
+        self.orderId = orderId
+        self.timestamp = timestamp
+        self.userId = userId
+        self.orderType = orderType
+        self.cardId = cardId
+        self.fee = fee
+        self.transactions = transactions
+
+    @abstractmethod
+    def add_transaction(self, transaction):
+        continue
+
+    @abstractmethod
+    def validate(self):
+        continue
+
+    @abstractmethod
+    def load_from_data(self):
+        continue
+
+    @abstractmethod
+    def __str__(self):
+        continue

wikideck/Market/SellOrder.py

@@ -0,0 +1,30 @@
+import json
+
+class SellOrder():
+    def __init__(self, cardId=None, items=[], price=0, fee=0):
+        super().__init__(
+            orderType = "sell",
+            cardId = cardId,
+            fee = fee
+            )
+        self.items = items
+        self.price = price
+
+    def add_transaction(self, transaction):
+        if transaction.cardId in self.items:
+            # TODO: update database
+            self.items.pop(transaction.cardId)
+            self.transactions.append(transaction)
+
+    def __str__(self):
+        return json.dumps({
+            "orderId": str(self.orderId),
+            "timestamp": str(self.timestamp),
+            "userId": str(self.userId),
+            "orderType": str(self.orderType),
+            "cardId": self.cardId,
+            "items": self.items,
+            "price": self.price,
+            "fee": self.fee,
+            "transactions": [str(each) for each in self.transactions]
+            })

wikideck/Mine/Block.py

@@ -0,0 +1,107 @@
+from card import Card
+
+import datetime
+import hashlib
+
+class Block():
+    def __init__(self, blockId=uuid.uuid4(), previousHash=None,
+                 timestamp=datetime.datetime.now(datetime.timezone.utc), height=0, nonce=0,
+                 card=Card(), transactions=[], authorPublicKey=None, data=None):
+        self.blockId = blockId
+        self.previousHash = previousHash
+        self.timestamp = timestamp
+        self.height = height
+        self.difficulty = difficulty
+        self.nonce = nonce
+        self.card = card
+        self.transactions = transactions.append(
+            Transaction(
+                cardId = self.card.id,
+                receiver = authorPublicKey
+                )
+            )
+        if data:
+            self.load_from_data(data)
+        self.update()
+
+    def update(self):
+        self.blockHash = hashlib.sha256(str(self).encode('utf-8'))
+
+    def mine(self):
+        while int(self.blockHash.hexdigest(), 16) > 2**(256-self.difficulty):
+            self.nonce += 1
+            self.update()
+
+    ###
+    # Validate the internal block structure.
+    # This method confirms that the correct data types have been used and that
+    # values are basically valid.
+    # Further validations against the existing chain must be done elsewhere.
+    # This method calls the Card.validate() and Transaction.validate() methods to validate the
+    # respective schemas.
+    ###
+    def validate(self):
+        # TODO: validate blockId is uuid
+        # TODO: validate previousHash is sha256 hash
+        if not int(self.blockHash.hexdigest(), 16) > 2**(256-self.difficulty):
+            raise self.Invalid("Hash does not meet difficulty requirement.")
+        # TODO: validate timestamp is timestamp
+        if not self.timestamp < datetime.datetime.now():
+            raise self.Invalid("Timestamp in the future.")
+        if not self.height > 0:
+            raise self.Invalid("Height less than 0.")
+        if not self.difficulty > 0:
+            raise self.Invalid("Difficulty less than 0.")
+        if not self.nonce > 0:
+            raise self.Invalid("Nonce less than 0.")
+        self.card.validate()
+        for transaction in self.transactions:
+            transaction.validate()
+
+    def load_from_data(self, data):
+        self.blockId = uuid.UUID(data['blockId'])
+        self.previousHash = data['previousHash']
+        self.timestamp = datetime.datetime.strptime(
+            data['timestamp'],
+            "%Y-%m-%d %H:%M:%S.%f%z"
+            )
+        self.height = data['height']
+        self.difficulty = data['difficulty']
+        self.nonce = data['nonce']
+        self.card = Card(
+            cardId = uuid.UUID(data['card']['cardId']),
+            pageId = data['card']['pageId']
+            )
+        self.transactions = [
+            Transaction(
+                transactionId = uuid.UUID(each['transactionId']),
+                timestamp = datetime.datetime.strptime(
+                    each['timestamp'],
+                    "%Y-%m-%d %H:%M:%S.%f"
+                    ),
+                cardId = uuid.UUID(each['cardId']),
+                sender = each['sender'],
+                receiver = each['receiver'],
+                signature = each['signature']
+                ) for each in data['transactions']
+            ]
+        self.update()
+
+    def __str__(self):
+        # The hash of the block is the SHA256 hash of what this method returns.
+        return data.dumps({
+            "blockId": str(self.blockId)
+            "previousHash": self.previousHash,
+            "timestamp": str(self.timestamp)
+            "height": self.height,
+            "difficulty": self.difficulty,
+            "nonce": self.nonce,
+            "author": self.author,
+            "card": str(self.card),
+            "transactions": [str(each) for each in transactions]
+            })
+
+    class Invalid(Exception):
+        def __init__(message, status_code=406):
+            super().__init__(message)
+            self.status_code = status_code

wikideck/Mine/Card.py

@@ -0,0 +1,25 @@
+import requests
+import wikipedia
+import uuid
+
+class Card():
+    def __init__(self, cardId=uuid.uuid4(), pageId=wikipedia.page(wikipedia.random()).pageid):
+        self.cardId = cardId
+        self.pageId = pageId
+
+    def validate(self):
+        try:
+            wikipedia.page(pageid=self.pageId)
+        except Exception as e:
+            raise self.Invalid("Page ID does not match a Wikipedia page.")
+
+    def __str__(self):
+        return json.dumps({
+            "cardId": str(self.cardId),
+            "pageId": self.pageId
+            })
+
+    class Invalid(Exception):
+        def __init__(self, message, status_code=406):
+            super().__init__(message)
+            self.status_code = status_code

wikideck/Mine/Mine.py

@@ -0,0 +1,158 @@
+from flask import Flask
+import json
+
+mine = Flask(__name__)
+db = mariadb.connect(**config)
+
+@app.get('/')
+def mine():
+    # TODO: return a page to mine through the browser
+    return 200
+
+###
+# Submit a block to the chain.
+# This method calls the Block().validate() method to validate the block schema.
+# This method performs additional validations against the rest of the chain.
+###
+@app.post('/blocks')
+def blocks_post():
+    previousBlock = get_last_block()
+    try:
+        newBlock = Block(data=request.get_json())
+        newBlock.validate()
+        previousBlock = db.get_last_block()
+        if newBlock.previousHash != previousBlock.blockHash:
+            raise Block.Invalid(
+                f"Incorrect previous hash - should be {previousBlock.blockHash}."
+                )
+        if newBlock.timestamp <= previousBlock.timestamp:
+            raise Block.Invalid(
+                "Timestamp is later than previous block."
+                )
+        if newBlock.height != previousBlock.height + 1:
+            raise Block.Invalid(
+                f"Incorrect block height - should be {previousBlock.height + 1}."
+                )
+        if newBlock.difficulty != DIFFICULTY_REQUIREMENT:
+            raise Block.Invalid(
+                f"Incorrect difficulty - should be {DIFFICULTY_REQUIREMENT}."
+                )
+        if len(newBlock.transactions) == 0:
+            raise Block.Invalid(
+                "Block contains no transactions."
+                )
+        for transaction in newBlock.transactions:
+            pendingTransaction = db.get_transaction(transaction.transactionId)
+            if not pendingTransaction:
+                raise Transaction.Invalid(
+                    f"No matching pending transaction for {transaction.transactionId}."
+                    )
+            if not pendingTransaction.pending:
+                raise Transaction.AlreadyFulfilled(
+                    f"Transaction {transaction.transactionId} has already been fulfilled."
+                    )
+            if transaction.timestamp != pendingTransaction.timestamp:
+                raise Transaction.Invalid(
+                    f"Incorrect timestamp on {transaction.transactionId}."
+                    )
+            if transaction.cardId != pendingTransaction.cardId:
+                raise Transaction.Invalid(
+                    f"Incorrect cardId on {transaction.transactionId}."
+                    )
+            if transaction.sender != pendingTransaction.sender:
+                raise Transaction.Invalid(
+                    f"Incorrect sender on {transaction.transactionId}."
+                    )
+            if transaction.recipient != pendingTransaction.recipient:
+                raise Transaction.Invalid(
+                    f"Incorrect recipient on {transaction.transactionId}."
+                    )
+            if transaction.signature != pendingTransaction.signature:
+                raise Transaction.Invalid(
+                    f"Incorrect signature on {transaction.transactionId}."
+                    )
+        # TODO: write to database
+        with open(f"{DATA_PATH}/{newBlock.blockId}.json", 'w') as f:
+            f.write(str(newBlock))
+        # TODO: update peers
+        return str(newBlock), 200
+    except: Transaction.Invalid as e:
+        return e, e.statusCode
+    except: Transaction.AlreadyFulfilled as e:
+        return e, e.statusCode
+    except: Card.Invalid as e:
+        return e, e.statusCode
+    except: Block.Invalid as e:
+        return e, e.statusCode
+
+###
+# Retrieve blocks and block data.
+# Returns a skeleton block to be mined by default.
+# Queries for a specific block when given parameters.
+###
+@app.get('/blocks')
+def blocks_get():
+    blockHash = request.args.get('blockHash', None)
+    height = request.args.get('height', None)
+    # TODO: block queries
+    return str(
+        Block(
+            previousHash = lastBlock.blockHash,
+            height = lastBlock.height + 1,
+            difficulty = DIFFICULTY_REQUIREMENT,
+            transactions = [
+                Transaction(data=each) for each in get_pending_transactions()
+                ]
+            )
+        )
+
+###
+# Retrieve card data
+###
+@app.get('/cards')
+def cards_get():
+    # TODO: query cards
+    return 200
+
+###
+# Submit a transaction to be mined in a block.
+# This method performs a number of validations on the submitted transaction and returns
+# a status code result.
+###
+@app.put('/transactions')
+def transactions_put():
+    try:
+        newTransaction = Transaction(data=request.get_json())
+        newTransaction.validate()
+        # TODO: validate transaction against blockchain
+        # TODO: add transaction to database
+        # TODO: update peers?
+        return 200
+    except Transaction.Unauthorized as e:
+        return e, e.statusCode
+    except Transaction.Invalid as e:
+        return e, e.statusCode
+
+###
+# Retrieve a transaction.
+###
+@app.get('/transactions')
+def transactions_get():
+    transactionId = request.args.get('transactionId', None)
+    return get_transaction_by_id(transactionId) if transactionId else json.dumps(
+        db.get_pending_transactions()
+        )
+
+@app.post('/peers')
+def peers_post():
+    # TODO: validate peer
+    # TODO: add peers to database
+    return 200
+
+@app.get('/peers')
+def peers_get():
+    # TODO: query peers
+    return 200
+
+if __name__ == "__main__":
+    app.run()

wikideck/Mine/Transaction.py

@@ -0,0 +1,74 @@
+import datetime
+import data
+import uuid
+
+class Transaction():
+    def __init__(self, transactionId=uuid.uuid4(),
+                 timestamp=datetime.datetime.now(datetime.timezone.utc),
+                 cardId=None, sender=None, receiver=None, signature=None, data=None):
+        self.transactionId = transactionId
+        self.timestamp = timestamp
+        self.cardId = cardId
+        self.sender = sender
+        self.receiver = receiver
+        self.signature = signature
+        if data:
+            self.load_from_data(data)
+
+    def validate(self):
+        # TODO: validate transactionId
+        # TODO: validate timestamp
+        # TODO: validate cardId
+        # TODO: validate sender
+        # TODO: validate receiver
+        # TODO: validate signature
+        if False:
+            raise self.Unauthorized()
+        elif False:
+            raise self.Invalid()
+
+    def sign(self, privateKey):
+        # TODO: use rsa private key to sign block
+        return None
+
+    def load_from_data(self, data):
+        self.transactionId = uuid.UUID(data['transactionId'])
+        self.timestamp = datetime.datetime.strptime(
+            data['timestamp'],
+            "%Y-%m-%d %H:%M:%S.%f%z"
+            )
+        self.cardId = uuid.UUID(data['cardId'])
+        self.sender = data['sender']
+        self.receiver = data['receiver']
+        self.signature = data['signature']
+
+
+    def __str__(self):
+        return data.dumps({
+            "transactionId": str(self.id),
+            "timestamp": str(self.timestamp),
+            "cardId": self.cardId,
+            "sender": self.sender,
+            "receiver": self.receiver,
+            "signature": self.signature
+            })
+
+    class Unauthorized(Exception):
+        def __init__(self, message, statusCode=403):
+            super().__init__(message)
+            self.statusCode = statusCode
+
+    class Invalid(Exception):
+        def __init__(self, message, statusCode=400):
+            super().__init__(message)
+            self.statusCode = statusCode
+
+    class AlreadyFulfilled(Exception):
+        def __init__(self, message, statusCode=400):
+            super().__init__(message)
+            self.statusCode = statusCode
+
+    class Abandoned(Exception):
+        def __init__(self, message, statusCode=500):
+            super().__init__(message)
+            self.statusCode = statusCode

wikideck/OrderMatcher/OrderMatcher.py

@@ -0,0 +1,21 @@
+###
+# The OrderMatcher converts orders to transactions.
+# When new orders arrive on the market, the order matcher is triggered to search the
+# database for an existing order to fulfill the new one.
+# If the order matcher finds a match for the order, it creates a transaction and
+# sends it to the mine.
+###
+class OrderMatcher():
+    def __init__(self):
+        return
+
+    def match_orders(self):
+        # TODO: find matching orders
+        # TODO: generate transactions
+        # TODO: send transactions to mine
+
+if __name__ == '__main__':
+    orderMatcher = OrderMatcher()
+    while True:
+        orderMatcher.match_orders()
+        time.sleep(10)

wikideck/StatusChecker/StatusChecker.py

@@ -0,0 +1,39 @@
+###
+# The StatusChecker updates the market database based on the mine database.
+# It will periodically query for pending transactions, and then confirm their status against
+# the Mine API.
+# It will also update user balances upon transaction completion.
+###
+
+class StatusChecker():
+    def __init__(self):
+        return
+
+    def update_pending_transactions(self):
+        for pendingTransaction in self.db.get_pending_transactions():
+            self.check_transaction_status(pendingTransaction)
+
+    def check_transaction_status(transaction):
+        transaction = Transaction(
+            data = transaction
+            )
+        # TODO: get transaction data from mine
+        if not mine_transaction.is_pending:
+            db.update_transaction_pending_status(
+                transactionId=transactionId,
+                is_pending=False
+                )
+            if not mine_transaction.block_id:
+                raise Transaction.Abandoned(
+                    f"Transaction {pendingTransaction.transactionId} was abandoned."
+                    )
+            else:
+                # TODO: increase seller balance by (price - fee)
+                # TODO: increase market balance by (fee)
+                return
+
+if __name__ == '__main__':
+    statusChecker = StatusChecker()
+    while True:
+        transactionStatusChecker.update_pending_transactions()
+        time.sleep(INTERVAL)

wikideck/app.py

@@ -0,0 +1,30 @@
+import os
+import dotenv
+
+ROLE = os.getenv("ROLE", None)
+INTERVAL = os.getenv("INTERVAL", 15)
+
+if __name__ == "__main__":
+    dotenv.load_dotenv()
+    if ROLE == "market":
+        from Market.Market import market
+        market.run()
+    elif ROLE == "mine":
+        from Mine.Mine import mine
+        mine.run()
+    elif ROLE == "order_matcher":
+        from OrderMatcher.OrderMatcher import OrderMatcher
+        import time
+        orderMatcher = OrderMatcher()
+        while True:
+            orderMatcher.match_orders()
+            time.sleep(INTERVAL)
+    elif ROLE == "status_checker":
+        from StatusChecker.StatusChecker import StatusChecker
+        import time
+        statusChecker = StatusChecker()
+        while True:
+            statusChecker.update_pending_transactions()
+            time.sleep(INTERVAL)
+    else:
+        raise Exception("Role must be one of: market, mine, order_matcher, status_checker")