Added Market Structure. (#8)

Added market structure, implemented the `load` factory function.

.pylintrc

@@ -223,7 +223,6 @@ spelling-store-unknown-words=no
 # List of note tags to take in consideration, separated by a comma.
 notes=FIXME,
       XXX,
-      TODO
 
 # Regular expression of note tags to take in consideration.
 #notes-rgx=

Pipfile.lock

@@ -165,14 +165,6 @@
             ],
             "version": "==1.4.4"
         },
-        "appnope": {
-            "hashes": [
-                "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0",
-                "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71"
-            ],
-            "markers": "sys_platform == 'darwin' and platform_system == 'Darwin'",
-            "version": "==0.1.0"
-        },
         "argon2-cffi": {
             "hashes": [
                 "sha256:05a8ac07c7026542377e38389638a8a1e9b78f1cd8439cd7493b39f08dd75fbf",

notebooks/market.ipynb

@@ -0,0 +1,70 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from src.market import MARKET_FORMAT, Market\n",
+    "\n",
+    "market = Market.load(\"https://api.mainnet-beta.solana.com\", \"CAgAeMD7quTdnr6RPa7JySQpjf3irAmefYNdTb6anemq\", None)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert market is not None\n",
+    "assert isinstance(market, Market)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "market <src.market.Market object at 0x7f663d1fa590>\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"market\", market)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

src/layouts/account_flags.py

@@ -1,9 +1,9 @@
 from typing import Dict
 
-from construct import BitsInteger, BitStruct, BitsSwapped, Flag  # type: ignore
+from construct import BitsInteger, BitsSwapped, BitStruct, Flag  # type: ignore
 
 # We will use a bitstruct with 64 bits instead of the widebits implementation in serum-js.
-_ACCOUNT_FLAGS_LAYOUT = BitsSwapped(  # Swap to little endian
+ACCOUNT_FLAGS_LAYOUT = BitsSwapped(  # Swap to little endian
     BitStruct(
         "initialized" / Flag,
         "market" / Flag,
@@ -19,10 +19,10 @@ _ACCOUNT_FLAGS_LAYOUT = BitsSwapped(  # Swap to little endian
 
 def decode_account_flags(raw_flags: bytes) -> Dict:
     """Parse account flags from bytes."""
-    return _ACCOUNT_FLAGS_LAYOUT.parse(raw_flags)
+    return ACCOUNT_FLAGS_LAYOUT.parse(raw_flags)
 
 
 def encode_account_flags(flag_params: Dict) -> bytes:
     """Serialize account flags to bytes."""
     flag_params[None] = False  # Set padding to false
-    return _ACCOUNT_FLAGS_LAYOUT.build(flag_params)
+    return ACCOUNT_FLAGS_LAYOUT.build(flag_params)

src/layouts/slab.py

@@ -3,7 +3,7 @@ from construct import Bytes, Int8ul, Int32ul, Int64ul, Padding  # type: ignore
 from construct import Struct as cStruct
 from construct import Switch
 
-from .account_flags import _ACCOUNT_FLAGS_LAYOUT
+from .account_flags import ACCOUNT_FLAGS_LAYOUT
 
 KEY = cStruct(
     "key" / Bytes(16),
@@ -52,6 +52,4 @@ SLAB_NODE_LAYOUT = cStruct(
 
 SLAB_LAYOUT = cStruct("header" / SLAB_HEADER_LAYOUT, "nodes" / SLAB_NODE_LAYOUT[lambda this: this.header.bump_index])
 
-ORDER_BOOK_LAYOUT = cStruct(
-    Padding(5), "account_flags" / _ACCOUNT_FLAGS_LAYOUT, "slab_layout" / SLAB_LAYOUT, Padding(7)
-)
+ORDER_BOOK_LAYOUT = cStruct(Padding(5), "account_flags" / ACCOUNT_FLAGS_LAYOUT, "slab_layout" / SLAB_LAYOUT, Padding(7))

src/market.py

@@ -0,0 +1,160 @@
+"""Market module to interact with Serum DEX."""
+import base64
+from typing import Any
+
+from construct import Bytes, Int8ul, Int64ul, Padding  # type: ignore
+from construct import Struct as cStruct  # type: ignore
+from solana.publickey import PublicKey
+from solana.rpc.api import Client
+
+from .layouts.account_flags import ACCOUNT_FLAGS_LAYOUT
+
+DEFAULT_DEX_PROGRAM_ID = PublicKey(
+    "4ckmDgGdxQoPDLUkDT3vHgSAkzA3QRdNq5ywwY4sUSJn",
+)
+
+MARKET_FORMAT = cStruct(
+    Padding(5),
+    "account_flags" / ACCOUNT_FLAGS_LAYOUT,
+    "own_address" / Bytes(32),
+    "vault_signer_nonce" / Int64ul,
+    "base_mint" / Bytes(32),
+    "quote_mint" / Bytes(32),
+    "base_vault" / Bytes(32),
+    "base_deposits_total" / Int64ul,
+    "base_fees_accrued" / Int64ul,
+    "quote_vault" / Bytes(32),
+    "quote_deposits_total" / Int64ul,
+    "quote_fees_accrued" / Int64ul,
+    "quote_dust_threshold" / Int64ul,
+    "request_queue" / Bytes(32),
+    "event_queue" / Bytes(32),
+    "bids" / Bytes(32),
+    "asks" / Bytes(32),
+    "base_lot_size" / Int64ul,
+    "quote_lot_size" / Int64ul,
+    "fee_rate_bps" / Int64ul,
+    Padding(7),
+)
+
+# TODO: probably need to change the amount of padding since they recently changed it.
+# See here: https://github.com/project-serum/serum-js/commit/87c25716c0f2f1092cf27467dd8bb06aabb83fdb
+MINT_LAYOUT = cStruct(Padding(36), "decimals" / Int8ul, Padding(3))
+
+
+class Market:
+    """Represents a Serum Market."""
+
+    _decode: Any
+    _baseSplTokenDecimals: int
+    _quoteSolTokenDecimals: int
+    _skipPreflight: bool
+    _confirmations: int
+    _porgram_id: PublicKey
+
+    # pylint: disable=too-many-arguments
+    def __init__(
+        self,
+        decoded: Any,
+        base_mint_decimals: int,
+        quote_mint_decimals: int,
+        options: Any,  # pylint: disable=unused-argument
+        program_id: PublicKey = DEFAULT_DEX_PROGRAM_ID,
+    ):
+        # TODO: add options
+        if not decoded.account_flags.initialized or not decoded.account_flags.market:
+            raise Exception("Invalid market state")
+        self._decode = decoded
+        self._base_spl_token_decimals = base_mint_decimals
+        self._quote_spl_token_decimals = quote_mint_decimals
+        self._skip_preflight = False
+        self._confirmations = 10
+        self._program_id = program_id
+
+    @staticmethod
+    # pylint: disable=unused-argument
+    def load(endpoint: str, market_address: str, options: Any, program_id: PublicKey = DEFAULT_DEX_PROGRAM_ID):
+        """Factory method to create a Market."""
+        http_client = Client(endpoint)
+        base64_res = http_client.get_account_info(market_address)["result"]["value"]["data"][0]
+        bytes_data = base64.decodebytes(base64_res.encode("ascii"))
+        market_state = MARKET_FORMAT.parse(bytes_data)
+
+        # TODO: add ownAddress check!
+        if not market_state.account_flags.initialized or not market_state.account_flags.market:
+            raise Exception("Invalid market")
+
+        base_mint_decimals = Market.get_mint_decimals(endpoint, PublicKey(market_state.base_mint))
+        quote_mint_decimals = Market.get_mint_decimals(endpoint, PublicKey(market_state.quote_mint))
+
+        return Market(market_state, base_mint_decimals, quote_mint_decimals, options)
+
+    def address(self):
+        """Return market address."""
+        raise NotImplementedError("address is not implemented yet")
+
+    def base_mint_address(self) -> PublicKey:
+        """Returns base mint address."""
+        raise NotImplementedError("base_mint_address is not implemented yet")
+
+    def quote_mint_address(self) -> PublicKey:
+        """Returns quote mint address."""
+        raise NotImplementedError("quote_mint_address is not implemented yet")
+
+    @staticmethod
+    def get_mint_decimals(endpoint: str, mint_pub_key: PublicKey) -> int:
+        """Get the mint decimals from given public key."""
+        data = Client(endpoint).get_account_info(mint_pub_key)["result"]["value"]["data"][0]
+        bytes_data = base64.decodebytes(data.encode("ascii"))
+        return MINT_LAYOUT.parse(bytes_data).decimals
+
+    def load_bids(self, endpoint: str):
+        """Load the bid order book"""
+        raise NotImplementedError("load_bids is not implemented yet")
+
+    def load_asks(self, endpoint: str):
+        """Load the Ask order book."""
+        raise NotImplementedError("load_asks is not implemented yet")
+
+
+class Slab:
+    """Slab data structure."""
+
+    _header: Any
+    _nodes: Any
+
+    def __init__(self, header, nodes):
+        self._header = header
+        self._nodes = nodes
+
+    def get(self, key: int):
+        """Return slab node with the given key."""
+        raise NotImplementedError("get is not implemented yet")
+
+    def __iter__(self):
+        pass
+
+
+class OrderBook:
+    """Represents an order book."""
+
+    market: Market
+    is_bids: bool
+    slab: Slab
+
+    def __init__(self, market: Market, account_flags: Any, slab: Slab):
+        self.market = market
+        self.is_bids = account_flags
+        self.slab = slab
+
+    @staticmethod
+    def decode(market: Market, buffer):
+        """Decode the given buffer into an order book."""
+        raise NotImplementedError("decode is not implemented yet")
+
+    def get_l2(self, depth: int):
+        """Get the Level 2 market information."""
+        raise NotImplementedError("get_l2 is not implemented yet")
+
+    def __iter__(self):
+        pass

tests/market_test.py

@@ -0,0 +1,15 @@
+from src.market import MARKET_FORMAT
+
+MARKET_DATA_HEX = "736572756d030000000000000054f23cbc4d93795ce75ecd8173bcf436923112f7b6b024f3afd9b6789124b9680000000000000000c73690f1d4b87aa9337848369c20a682b37e8dcb33f4237a2a8f8b0abd64bd1cd9c75c9a58645ff02ab0741cd6d1790067957d2266165b767259b358ded270fb2dbc6d44f2ab58dce432b5bb31d98517366d6e24e69c0bf5b926ead9ec658935408ffd71c50000000000000000000000e1a461a046199877c4cd3cbafc61c3dfdb088e737b7193a3d28e72b709421fc4544fec927c000000a4fc4c12000000006400000000000000f422ea23ada9e1a9d100ba8443deb041c231e0f79ee6d07d1c1f7042fe4a1ade3b236ea4ba636227dfa22773f41fa02cc91842c2e9330e2ac0a987dc68b520e8c58676b5751c48e22c3bcc6edda8f75f76b1596b9874bd5714366e32d84e3bc0400501895361982c4be67d03af519ac7fd96a8a79f5b15ec7af79f6b70290bf740420f00000000001027000000000000000000000000000070616464696e67"  # noqa: E501 # pylint: disable=line-too-long
+DATA = bytes.fromhex(MARKET_DATA_HEX)
+
+
+def test_parse_market_state():
+    parsed_market = MARKET_FORMAT.parse(DATA)
+    assert parsed_market.account_flags.initialized
+    assert parsed_market.account_flags.market
+    assert not parsed_market.account_flags.openOrders
+    assert parsed_market.vault_signer_nonce == 0
+    assert parsed_market.base_fees_accrued == 0
+    assert parsed_market.quote_dust_threshold == 100
+    assert parsed_market.fee_rate_bps == 0

tests/test_account_flags_layout.py

@@ -1,6 +1,6 @@
 """Tests for account flags layout."""
 
-from src.layouts.account_flags import decode_account_flags, encode_account_flags, _ACCOUNT_FLAGS_LAYOUT
+from src.layouts.account_flags import ACCOUNT_FLAGS_LAYOUT, decode_account_flags, encode_account_flags
 
 
 def default_flags():
@@ -17,7 +17,7 @@ def default_flags():
 
 def test_correct_size():
     """Test account flags layout has 8 bytes."""
-    assert _ACCOUNT_FLAGS_LAYOUT.sizeof() == 8
+    assert ACCOUNT_FLAGS_LAYOUT.sizeof() == 8
 
 
 def test_decode():