From f4749d63e4a92aeb3587b4e3305e0a4bd9e28a34 Mon Sep 17 00:00:00 2001 From: Leonard G Date: Mon, 31 Aug 2020 20:52:51 +0100 Subject: [PATCH] Added Market Structure. (#8) Added market structure, implemented the `load` factory function. --- .pylintrc | 1 - Pipfile.lock | 8 -- notebooks/market.ipynb | 70 +++++++++++++ src/layouts/account_flags.py | 8 +- src/layouts/slab.py | 6 +- src/market.py | 160 +++++++++++++++++++++++++++++ tests/market_test.py | 15 +++ tests/test_account_flags_layout.py | 4 +- 8 files changed, 253 insertions(+), 19 deletions(-) create mode 100644 notebooks/market.ipynb create mode 100644 src/market.py create mode 100644 tests/market_test.py diff --git a/.pylintrc b/.pylintrc index d363332..849b685 100644 --- a/.pylintrc +++ b/.pylintrc @@ -223,7 +223,6 @@ spelling-store-unknown-words=no # List of note tags to take in consideration, separated by a comma. notes=FIXME, XXX, - TODO # Regular expression of note tags to take in consideration. #notes-rgx= diff --git a/Pipfile.lock b/Pipfile.lock index bddf84e..3fea267 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -165,14 +165,6 @@ ], "version": "==1.4.4" }, - "appnope": { - "hashes": [ - "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0", - "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71" - ], - "markers": "sys_platform == 'darwin' and platform_system == 'Darwin'", - "version": "==0.1.0" - }, "argon2-cffi": { "hashes": [ "sha256:05a8ac07c7026542377e38389638a8a1e9b78f1cd8439cd7493b39f08dd75fbf", diff --git a/notebooks/market.ipynb b/notebooks/market.ipynb new file mode 100644 index 0000000..117ec05 --- /dev/null +++ b/notebooks/market.ipynb @@ -0,0 +1,70 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "from src.market import MARKET_FORMAT, Market\n", + "\n", + "market = Market.load(\"https://api.mainnet-beta.solana.com\", \"CAgAeMD7quTdnr6RPa7JySQpjf3irAmefYNdTb6anemq\", None)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "assert market is not None\n", + "assert isinstance(market, Market)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "market \n" + ] + } + ], + "source": [ + "print(\"market\", market)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/src/layouts/account_flags.py b/src/layouts/account_flags.py index 00076e0..fbf2099 100644 --- a/src/layouts/account_flags.py +++ b/src/layouts/account_flags.py @@ -1,9 +1,9 @@ from typing import Dict -from construct import BitsInteger, BitStruct, BitsSwapped, Flag # type: ignore +from construct import BitsInteger, BitsSwapped, BitStruct, Flag # type: ignore # We will use a bitstruct with 64 bits instead of the widebits implementation in serum-js. -_ACCOUNT_FLAGS_LAYOUT = BitsSwapped( # Swap to little endian +ACCOUNT_FLAGS_LAYOUT = BitsSwapped( # Swap to little endian BitStruct( "initialized" / Flag, "market" / Flag, @@ -19,10 +19,10 @@ _ACCOUNT_FLAGS_LAYOUT = BitsSwapped( # Swap to little endian def decode_account_flags(raw_flags: bytes) -> Dict: """Parse account flags from bytes.""" - return _ACCOUNT_FLAGS_LAYOUT.parse(raw_flags) + return ACCOUNT_FLAGS_LAYOUT.parse(raw_flags) def encode_account_flags(flag_params: Dict) -> bytes: """Serialize account flags to bytes.""" flag_params[None] = False # Set padding to false - return _ACCOUNT_FLAGS_LAYOUT.build(flag_params) + return ACCOUNT_FLAGS_LAYOUT.build(flag_params) diff --git a/src/layouts/slab.py b/src/layouts/slab.py index 9aa7904..e4b75fd 100644 --- a/src/layouts/slab.py +++ b/src/layouts/slab.py @@ -3,7 +3,7 @@ from construct import Bytes, Int8ul, Int32ul, Int64ul, Padding # type: ignore from construct import Struct as cStruct from construct import Switch -from .account_flags import _ACCOUNT_FLAGS_LAYOUT +from .account_flags import ACCOUNT_FLAGS_LAYOUT KEY = cStruct( "key" / Bytes(16), @@ -52,6 +52,4 @@ SLAB_NODE_LAYOUT = cStruct( SLAB_LAYOUT = cStruct("header" / SLAB_HEADER_LAYOUT, "nodes" / SLAB_NODE_LAYOUT[lambda this: this.header.bump_index]) -ORDER_BOOK_LAYOUT = cStruct( - Padding(5), "account_flags" / _ACCOUNT_FLAGS_LAYOUT, "slab_layout" / SLAB_LAYOUT, Padding(7) -) +ORDER_BOOK_LAYOUT = cStruct(Padding(5), "account_flags" / ACCOUNT_FLAGS_LAYOUT, "slab_layout" / SLAB_LAYOUT, Padding(7)) diff --git a/src/market.py b/src/market.py new file mode 100644 index 0000000..804f246 --- /dev/null +++ b/src/market.py @@ -0,0 +1,160 @@ +"""Market module to interact with Serum DEX.""" +import base64 +from typing import Any + +from construct import Bytes, Int8ul, Int64ul, Padding # type: ignore +from construct import Struct as cStruct # type: ignore +from solana.publickey import PublicKey +from solana.rpc.api import Client + +from .layouts.account_flags import ACCOUNT_FLAGS_LAYOUT + +DEFAULT_DEX_PROGRAM_ID = PublicKey( + "4ckmDgGdxQoPDLUkDT3vHgSAkzA3QRdNq5ywwY4sUSJn", +) + +MARKET_FORMAT = cStruct( + Padding(5), + "account_flags" / ACCOUNT_FLAGS_LAYOUT, + "own_address" / Bytes(32), + "vault_signer_nonce" / Int64ul, + "base_mint" / Bytes(32), + "quote_mint" / Bytes(32), + "base_vault" / Bytes(32), + "base_deposits_total" / Int64ul, + "base_fees_accrued" / Int64ul, + "quote_vault" / Bytes(32), + "quote_deposits_total" / Int64ul, + "quote_fees_accrued" / Int64ul, + "quote_dust_threshold" / Int64ul, + "request_queue" / Bytes(32), + "event_queue" / Bytes(32), + "bids" / Bytes(32), + "asks" / Bytes(32), + "base_lot_size" / Int64ul, + "quote_lot_size" / Int64ul, + "fee_rate_bps" / Int64ul, + Padding(7), +) + +# TODO: probably need to change the amount of padding since they recently changed it. +# See here: https://github.com/project-serum/serum-js/commit/87c25716c0f2f1092cf27467dd8bb06aabb83fdb +MINT_LAYOUT = cStruct(Padding(36), "decimals" / Int8ul, Padding(3)) + + +class Market: + """Represents a Serum Market.""" + + _decode: Any + _baseSplTokenDecimals: int + _quoteSolTokenDecimals: int + _skipPreflight: bool + _confirmations: int + _porgram_id: PublicKey + + # pylint: disable=too-many-arguments + def __init__( + self, + decoded: Any, + base_mint_decimals: int, + quote_mint_decimals: int, + options: Any, # pylint: disable=unused-argument + program_id: PublicKey = DEFAULT_DEX_PROGRAM_ID, + ): + # TODO: add options + if not decoded.account_flags.initialized or not decoded.account_flags.market: + raise Exception("Invalid market state") + self._decode = decoded + self._base_spl_token_decimals = base_mint_decimals + self._quote_spl_token_decimals = quote_mint_decimals + self._skip_preflight = False + self._confirmations = 10 + self._program_id = program_id + + @staticmethod + # pylint: disable=unused-argument + def load(endpoint: str, market_address: str, options: Any, program_id: PublicKey = DEFAULT_DEX_PROGRAM_ID): + """Factory method to create a Market.""" + http_client = Client(endpoint) + base64_res = http_client.get_account_info(market_address)["result"]["value"]["data"][0] + bytes_data = base64.decodebytes(base64_res.encode("ascii")) + market_state = MARKET_FORMAT.parse(bytes_data) + + # TODO: add ownAddress check! + if not market_state.account_flags.initialized or not market_state.account_flags.market: + raise Exception("Invalid market") + + base_mint_decimals = Market.get_mint_decimals(endpoint, PublicKey(market_state.base_mint)) + quote_mint_decimals = Market.get_mint_decimals(endpoint, PublicKey(market_state.quote_mint)) + + return Market(market_state, base_mint_decimals, quote_mint_decimals, options) + + def address(self): + """Return market address.""" + raise NotImplementedError("address is not implemented yet") + + def base_mint_address(self) -> PublicKey: + """Returns base mint address.""" + raise NotImplementedError("base_mint_address is not implemented yet") + + def quote_mint_address(self) -> PublicKey: + """Returns quote mint address.""" + raise NotImplementedError("quote_mint_address is not implemented yet") + + @staticmethod + def get_mint_decimals(endpoint: str, mint_pub_key: PublicKey) -> int: + """Get the mint decimals from given public key.""" + data = Client(endpoint).get_account_info(mint_pub_key)["result"]["value"]["data"][0] + bytes_data = base64.decodebytes(data.encode("ascii")) + return MINT_LAYOUT.parse(bytes_data).decimals + + def load_bids(self, endpoint: str): + """Load the bid order book""" + raise NotImplementedError("load_bids is not implemented yet") + + def load_asks(self, endpoint: str): + """Load the Ask order book.""" + raise NotImplementedError("load_asks is not implemented yet") + + +class Slab: + """Slab data structure.""" + + _header: Any + _nodes: Any + + def __init__(self, header, nodes): + self._header = header + self._nodes = nodes + + def get(self, key: int): + """Return slab node with the given key.""" + raise NotImplementedError("get is not implemented yet") + + def __iter__(self): + pass + + +class OrderBook: + """Represents an order book.""" + + market: Market + is_bids: bool + slab: Slab + + def __init__(self, market: Market, account_flags: Any, slab: Slab): + self.market = market + self.is_bids = account_flags + self.slab = slab + + @staticmethod + def decode(market: Market, buffer): + """Decode the given buffer into an order book.""" + raise NotImplementedError("decode is not implemented yet") + + def get_l2(self, depth: int): + """Get the Level 2 market information.""" + raise NotImplementedError("get_l2 is not implemented yet") + + def __iter__(self): + pass diff --git a/tests/market_test.py b/tests/market_test.py new file mode 100644 index 0000000..7602dc5 --- /dev/null +++ b/tests/market_test.py @@ -0,0 +1,15 @@ +from src.market import MARKET_FORMAT + +MARKET_DATA_HEX = "736572756d030000000000000054f23cbc4d93795ce75ecd8173bcf436923112f7b6b024f3afd9b6789124b9680000000000000000c73690f1d4b87aa9337848369c20a682b37e8dcb33f4237a2a8f8b0abd64bd1cd9c75c9a58645ff02ab0741cd6d1790067957d2266165b767259b358ded270fb2dbc6d44f2ab58dce432b5bb31d98517366d6e24e69c0bf5b926ead9ec658935408ffd71c50000000000000000000000e1a461a046199877c4cd3cbafc61c3dfdb088e737b7193a3d28e72b709421fc4544fec927c000000a4fc4c12000000006400000000000000f422ea23ada9e1a9d100ba8443deb041c231e0f79ee6d07d1c1f7042fe4a1ade3b236ea4ba636227dfa22773f41fa02cc91842c2e9330e2ac0a987dc68b520e8c58676b5751c48e22c3bcc6edda8f75f76b1596b9874bd5714366e32d84e3bc0400501895361982c4be67d03af519ac7fd96a8a79f5b15ec7af79f6b70290bf740420f00000000001027000000000000000000000000000070616464696e67" # noqa: E501 # pylint: disable=line-too-long +DATA = bytes.fromhex(MARKET_DATA_HEX) + + +def test_parse_market_state(): + parsed_market = MARKET_FORMAT.parse(DATA) + assert parsed_market.account_flags.initialized + assert parsed_market.account_flags.market + assert not parsed_market.account_flags.openOrders + assert parsed_market.vault_signer_nonce == 0 + assert parsed_market.base_fees_accrued == 0 + assert parsed_market.quote_dust_threshold == 100 + assert parsed_market.fee_rate_bps == 0 diff --git a/tests/test_account_flags_layout.py b/tests/test_account_flags_layout.py index 7e40db4..3d968a5 100644 --- a/tests/test_account_flags_layout.py +++ b/tests/test_account_flags_layout.py @@ -1,6 +1,6 @@ """Tests for account flags layout.""" -from src.layouts.account_flags import decode_account_flags, encode_account_flags, _ACCOUNT_FLAGS_LAYOUT +from src.layouts.account_flags import ACCOUNT_FLAGS_LAYOUT, decode_account_flags, encode_account_flags def default_flags(): @@ -17,7 +17,7 @@ def default_flags(): def test_correct_size(): """Test account flags layout has 8 bytes.""" - assert _ACCOUNT_FLAGS_LAYOUT.sizeof() == 8 + assert ACCOUNT_FLAGS_LAYOUT.sizeof() == 8 def test_decode():