Support Serum v3. (#65)

Added support for DEX v3.

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   localnet:
-    image: "solanalabs/solana:stable"
+    image: "solanalabs/solana:v1.4.18"
     ports:
       - "8899:8899"
       - "8900:8900"

pyserum/_layouts/instructions.py

@@ -16,6 +16,9 @@ class InstructionType(IntEnum):
     CancelOrder = 4
     SettleFunds = 5
     CancelOrderByClientID = 6
+    NewOrderV3 = 10
+    CancelOrderV2 = 11
+    CancelOrderByClientIdV2 = 12
 
 
 _VERSION = 0
@@ -49,6 +52,24 @@ _CANCEL_ORDER = cStruct(
 
 _CANCEL_ORDER_BY_CLIENTID = cStruct("client_id" / Int64ul)
 
+_NEW_ORDER_V3 = cStruct(
+    "side" / Int32ul,  # Enum
+    "limit_price" / Int64ul,
+    "max_base_quantity" / Int64ul,
+    "max_quote_quantity" / Int64ul,
+    "self_trade_behavior" / Int32ul,
+    "order_type" / Int32ul,  # Enum
+    "client_id" / Int64ul,
+    "limit" / Int16ul,
+)
+
+_CANCEL_ORDER_V2 = cStruct(
+    "side" / Int32ul,  # Enum
+    "order_id" / KEY,
+)
+
+_CANCEL_ORDER_BY_CLIENTID_V2 = cStruct("client_id" / Int64ul)
+
 INSTRUCTIONS_LAYOUT = cStruct(
     "version" / Const(_VERSION, Int8ul),
     "instruction_type" / Int32ul,
@@ -63,6 +84,9 @@ INSTRUCTIONS_LAYOUT = cStruct(
             InstructionType.CancelOrder: _CANCEL_ORDER,
             InstructionType.SettleFunds: Pass,  # Empty list
             InstructionType.CancelOrderByClientID: _CANCEL_ORDER_BY_CLIENTID,
+            InstructionType.NewOrderV3: _NEW_ORDER_V3,
+            InstructionType.CancelOrderV2: _CANCEL_ORDER_V2,
+            InstructionType.CancelOrderByClientIdV2: _CANCEL_ORDER_BY_CLIENTID_V2,
         },
     ),
 )

pyserum/enums.py

@@ -21,3 +21,9 @@ class OrderType(IntEnum):
     """"""
     PostOnly = 2
     """"""
+
+
+class SelfTradeBehavior(IntEnum):
+    DecrementTake = 0
+    CancelProvide = 1
+    AbortTransaction = 2

pyserum/instructions.py

@@ -1,5 +1,5 @@
 """Serum Dex Instructions."""
-from typing import Any, Dict, List, NamedTuple
+from typing import Any, Dict, List, NamedTuple, Optional
 
 from solana.publickey import PublicKey
 from solana.sysvar import SYSVAR_RENT_PUBKEY
@@ -8,7 +8,7 @@ from solana.utils.validate import validate_instruction_keys, validate_instructio
 from spl.token.constants import TOKEN_PROGRAM_ID  # type: ignore # TODO: Fix and remove ignore.
 
 from ._layouts.instructions import INSTRUCTIONS_LAYOUT, InstructionType
-from .enums import OrderType, Side
+from .enums import OrderType, SelfTradeBehavior, Side
 
 # V2
 DEFAULT_DEX_PROGRAM_ID = PublicKey("EUqojwWA2rd19FZrzeBncJsm38Jm1hEhE3zsmX3bRc2o")
@@ -177,6 +177,96 @@ class SettleFundsParams(NamedTuple):
     program_id: PublicKey = DEFAULT_DEX_PROGRAM_ID
 
 
+class NewOrderV3Params(NamedTuple):
+    """New order params."""
+
+    market: PublicKey
+    """"""
+    open_orders: PublicKey
+    """"""
+    payer: PublicKey
+    """"""
+    owner: PublicKey
+    """"""
+    request_queue: PublicKey
+    """"""
+    event_queue: PublicKey
+    """"""
+    bids: PublicKey
+    """"""
+    asks: PublicKey
+    """"""
+    base_vault: PublicKey
+    """"""
+    quote_vault: PublicKey
+    """"""
+    side: Side
+    """"""
+    limit_price: int
+    """"""
+    max_base_quantity: int
+    """"""
+    max_quote_quantity: int
+    """"""
+    order_type: OrderType
+    """"""
+    self_trade_behavior: SelfTradeBehavior
+    """"""
+    limit: Optional[int]
+    """"""
+    client_id: int = 0
+    """"""
+    program_id: PublicKey = DEFAULT_DEX_PROGRAM_ID
+    """"""
+    fee_discount_pubkey: Optional[PublicKey] = None
+
+
+class CancelOrderV2Params(NamedTuple):
+    """Cancel order params."""
+
+    market: PublicKey
+    """"""
+    bids: PublicKey
+    """"""
+    asks: PublicKey
+    """"""
+    event_queue: PublicKey
+    """"""
+    open_orders: PublicKey
+    """"""
+    owner: PublicKey
+    """"""
+    side: Side
+    """"""
+    order_id: int
+    """"""
+    open_orders_slot: int
+    """"""
+    program_id: PublicKey = DEFAULT_DEX_PROGRAM_ID
+    """"""
+
+
+class CancelOrderByClientIDV2Params(NamedTuple):
+    """Cancel order by client ID params."""
+
+    market: PublicKey
+    """"""
+    bids: PublicKey
+    """"""
+    asks: PublicKey
+    """"""
+    event_queue: PublicKey
+    """"""
+    open_orders: PublicKey
+    """"""
+    owner: PublicKey
+    """"""
+    client_id: int
+    """"""
+    program_id: PublicKey = DEFAULT_DEX_PROGRAM_ID
+    """"""
+
+
 def __parse_and_validate_instruction(instruction: TransactionInstruction, instruction_type: InstructionType) -> Any:
     instruction_type_to_length_map: Dict[InstructionType, int] = {
         InstructionType.InitializeMarket: 9,
@@ -186,6 +276,9 @@ def __parse_and_validate_instruction(instruction: TransactionInstruction, instru
         InstructionType.CancelOrder: 4,
         InstructionType.CancelOrderByClientID: 4,
         InstructionType.SettleFunds: 9,
+        InstructionType.NewOrderV3: 12,
+        InstructionType.CancelOrderV2: 6,
+        InstructionType.CancelOrderByClientIdV2: 6,
     }
     validate_instruction_keys(instruction, instruction_type_to_length_map[instruction_type])
     data = INSTRUCTIONS_LAYOUT.parse(instruction.data)
@@ -297,6 +390,58 @@ def decode_cancel_order_by_client_id(instruction: TransactionInstruction) -> Can
     )
 
 
+def decode_new_order_v3(instruction: TransactionInstruction) -> NewOrderV3Params:
+    data = __parse_and_validate_instruction(instruction, InstructionType.NewOrderV3)
+    return NewOrderV3Params(
+        market=instruction.keys[0].pubkey,
+        open_orders=instruction.keys[1].pubkey,
+        request_queue=instruction.keys[2].pubkey,
+        event_queue=instruction.keys[3].pubkey,
+        bids=instruction.keys[4].pubkey,
+        asks=instruction.keys[5].pubkey,
+        payer=instruction.keys[6].pubkey,
+        owner=instruction.keys[7].pubkey,
+        base_vault=instruction.keys[8].pubkey,
+        quote_vault=instruction.keys[9].pubkey,
+        side=data.args.side,
+        limit_price=data.args.limit_price,
+        max_base_quantity=data.args.max_base_quantity,
+        max_quote_quantity=data.args.max_quote_quantity,
+        self_trade_behavior=SelfTradeBehavior(data.args.self_trade_behavior),
+        order_type=OrderType(data.args.order_type),
+        client_id=data.args.client_id,
+        limit=data.args.limit,
+    )
+
+
+def decode_cancel_order_v2(instruction: TransactionInstruction) -> CancelOrderV2Params:
+    data = __parse_and_validate_instruction(instruction, InstructionType.CancelOrderV2)
+    return CancelOrderV2Params(
+        market=instruction.keys[0].pubkey,
+        bids=instruction.keys[1].pubkey,
+        asks=instruction.keys[2].pubkey,
+        open_orders=instruction.keys[3].pubkey,
+        owner=instruction.keys[4].pubkey,
+        event_queue=instruction.keys[5].pubkey,
+        side=Side(data.args.side),
+        order_id=int.from_bytes(data.args.order_id, "little"),
+        open_orders_slot=data.args.open_orders_slot,
+    )
+
+
+def decode_cancel_order_by_client_id_v2(instruction: TransactionInstruction) -> CancelOrderByClientIDV2Params:
+    data = __parse_and_validate_instruction(instruction, InstructionType.CancelOrderByClientIdV2)
+    return CancelOrderByClientIDV2Params(
+        market=instruction.keys[0].pubkey,
+        bids=instruction.keys[1].pubkey,
+        asks=instruction.keys[2].pubkey,
+        open_orders=instruction.keys[3].pubkey,
+        owner=instruction.keys[4].pubkey,
+        event_queue=instruction.keys[5].pubkey,
+        client_id=data.args.client_id,
+    )
+
+
 def initialize_market(params: InitializeMarketParams) -> TransactionInstruction:
     """Generate a transaction instruction to initialize a Serum market."""
     return TransactionInstruction(
@@ -453,3 +598,91 @@ def cancel_order_by_client_id(params: CancelOrderByClientIDParams) -> Transactio
             )
         ),
     )
+
+
+def new_order_v3(params: NewOrderV3Params) -> TransactionInstruction:
+    """Generate a transaction instruction to place new order."""
+    touched_keys = [
+        AccountMeta(pubkey=params.market, is_signer=False, is_writable=True),
+        AccountMeta(pubkey=params.open_orders, is_signer=False, is_writable=True),
+        AccountMeta(pubkey=params.request_queue, is_signer=False, is_writable=True),
+        AccountMeta(pubkey=params.event_queue, is_signer=False, is_writable=True),
+        AccountMeta(pubkey=params.bids, is_signer=False, is_writable=True),
+        AccountMeta(pubkey=params.asks, is_signer=False, is_writable=True),
+        AccountMeta(pubkey=params.payer, is_signer=False, is_writable=True),
+        AccountMeta(pubkey=params.owner, is_signer=True, is_writable=False),
+        AccountMeta(pubkey=params.base_vault, is_signer=False, is_writable=True),
+        AccountMeta(pubkey=params.quote_vault, is_signer=False, is_writable=True),
+        AccountMeta(pubkey=TOKEN_PROGRAM_ID, is_signer=False, is_writable=False),
+        AccountMeta(pubkey=SYSVAR_RENT_PUBKEY, is_signer=False, is_writable=False),
+    ]
+    if params.fee_discount_pubkey:
+        touched_keys.append(
+            AccountMeta(pubkey=params.fee_discount_pubkey, is_signer=False, is_writable=False),
+        )
+    return TransactionInstruction(
+        keys=touched_keys,
+        program_id=params.program_id,
+        data=INSTRUCTIONS_LAYOUT.build(
+            dict(
+                instruction_type=InstructionType.NewOrderV3,
+                args=dict(
+                    side=params.side,
+                    limit_price=params.limit_price,
+                    max_base_quantity=params.max_base_quantity,
+                    max_quote_quantity=params.max_quote_quantity,
+                    self_trade_behavior=params.self_trade_behavior,
+                    order_type=params.order_type,
+                    client_id=params.client_id,
+                    limit=65535,
+                ),
+            )
+        ),
+    )
+
+
+def cancel_order_v2(params: CancelOrderV2Params) -> TransactionInstruction:
+    """Generate a transaction instruction to cancel order."""
+    return TransactionInstruction(
+        keys=[
+            AccountMeta(pubkey=params.market, is_signer=False, is_writable=False),
+            AccountMeta(pubkey=params.bids, is_signer=False, is_writable=True),
+            AccountMeta(pubkey=params.asks, is_signer=False, is_writable=True),
+            AccountMeta(pubkey=params.open_orders, is_signer=False, is_writable=True),
+            AccountMeta(pubkey=params.owner, is_signer=True, is_writable=False),
+            AccountMeta(pubkey=params.event_queue, is_signer=False, is_writable=True),
+        ],
+        program_id=params.program_id,
+        data=INSTRUCTIONS_LAYOUT.build(
+            dict(
+                instruction_type=InstructionType.CancelOrderV2,
+                args=dict(
+                    side=params.side,
+                    order_id=params.order_id.to_bytes(16, byteorder="little"),
+                ),
+            )
+        ),
+    )
+
+
+def cancel_order_by_client_id_v2(params: CancelOrderByClientIDV2Params) -> TransactionInstruction:
+    """Generate a transaction instruction to cancel order by client id."""
+    return TransactionInstruction(
+        keys=[
+            AccountMeta(pubkey=params.market, is_signer=False, is_writable=False),
+            AccountMeta(pubkey=params.bids, is_signer=False, is_writable=True),
+            AccountMeta(pubkey=params.asks, is_signer=False, is_writable=True),
+            AccountMeta(pubkey=params.open_orders, is_signer=False, is_writable=True),
+            AccountMeta(pubkey=params.owner, is_signer=True, is_writable=False),
+            AccountMeta(pubkey=params.event_queue, is_signer=False, is_writable=True),
+        ],
+        program_id=params.program_id,
+        data=INSTRUCTIONS_LAYOUT.build(
+            dict(
+                instruction_type=InstructionType.CancelOrderByClientIdV2,
+                args=dict(
+                    client_id=params.client_id,
+                ),
+            )
+        ),
+    )

pyserum/market/market.py

@@ -20,7 +20,7 @@ import pyserum.instructions as instructions
 import pyserum.market.types as t
 
 from .._layouts.open_orders import OPEN_ORDERS_LAYOUT
-from ..enums import OrderType, Side
+from ..enums import OrderType, SelfTradeBehavior, Side
 from ..open_orders_account import OpenOrdersAccount, make_create_account_instruction
 from ..utils import load_bytes_data
 from ._internal.queue import decode_event_queue, decode_request_queue
@@ -36,13 +36,10 @@ class Market:
 
     logger = logging.getLogger("pyserum.market.Market")
 
-    def __init__(
+    def __init__(self, conn: Client, market_state: MarketState, force_use_request_queue: bool = False) -> None:
-        self,
-        conn: Client,
-        market_state: MarketState,
-    ) -> None:
         self._conn = conn
         self.state = market_state
+        self.force_use_request_queue = force_use_request_queue
 
     @staticmethod
     # pylint: disable=unused-argument
@@ -50,6 +47,7 @@ class Market:
         conn: Client,
         market_address: PublicKey,
         program_id: PublicKey = instructions.DEFAULT_DEX_PROGRAM_ID,
+        force_use_request_queue: bool = False,
     ) -> Market:
         """Factory method to create a Market.
 
@@ -58,7 +56,20 @@ class Market:
         :param program_id: The program id of the given market, it will use the default value if not provided.
         """
         market_state = MarketState.load(conn, market_address, program_id)
-        return Market(conn, market_state)
+        return Market(conn, market_state, force_use_request_queue)
+
+    def _use_request_queue(self) -> bool:
+        return (
+            # DEX Version 1
+            self.state.program_id == PublicKey("4ckmDgGdxQoPDLUkDT3vHgSAkzA3QRdNq5ywwY4sUSJn")
+            or
+            # DEX Version 1
+            self.state.program_id == PublicKey("BJ3jrUzddfuSrZHXSCxMUUQsjKEyLmuuyZebkcaFp2fg")
+            or
+            # DEX Version 2
+            self.state.program_id == PublicKey("EUqojwWA2rd19FZrzeBncJsm38Jm1hEhE3zsmX3bRc2o")
+            or self.force_use_request_queue
+        )
 
     def support_srm_fee_discounts(self) -> bool:
         raise NotImplementedError("support_srm_fee_discounts not implemented")
@@ -158,8 +169,8 @@ class Market:
         owner: Account,
         order_type: OrderType,
         side: Side,
-        limit_price: int,
+        limit_price: float,
-        max_quantity: int,
+        max_quantity: float,
         client_id: int = 0,
         opts: TxOpts = TxOpts(),
     ) -> RPCResponse:  # TODO: Add open_orders_address_key param and fee_discount_pubkey
@@ -245,7 +256,7 @@ class Market:
 
     @staticmethod
     def _get_lamport_need_for_sol_wrapping(
-        price: int, size: int, side: Side, open_orders_accounts: List[OpenOrdersAccount]
+        price: float, size: float, side: Side, open_orders_accounts: List[OpenOrdersAccount]
     ) -> int:
         lamports = 0
         if side == Side.Buy:
@@ -265,30 +276,56 @@ class Market:
         owner: Account,
         order_type: OrderType,
         side: Side,
-        limit_price: int,
+        limit_price: float,
-        max_quantity: int,
+        max_quantity: float,
         client_id: int,
         open_order_account: PublicKey,
+        fee_discount_pubkey: PublicKey = None,
     ) -> TransactionInstruction:
         if self.state.base_size_number_to_lots(max_quantity) < 0:
             raise Exception("Size lot %d is too small" % max_quantity)
         if self.state.price_number_to_lots(limit_price) < 0:
             raise Exception("Price lot %d is too small" % limit_price)
-        return instructions.new_order(
+        if self._use_request_queue():
-            instructions.NewOrderParams(
+            return instructions.new_order(
+                instructions.NewOrderParams(
+                    market=self.state.public_key(),
+                    open_orders=open_order_account,
+                    payer=payer,
+                    owner=owner.public_key(),
+                    request_queue=self.state.request_queue(),
+                    base_vault=self.state.base_vault(),
+                    quote_vault=self.state.quote_vault(),
+                    side=side,
+                    limit_price=self.state.price_number_to_lots(limit_price),
+                    max_quantity=self.state.base_size_number_to_lots(max_quantity),
+                    order_type=order_type,
+                    client_id=client_id,
+                    program_id=self.state.program_id(),
+                )
+            )
+        return instructions.new_order_v3(
+            instructions.NewOrderV3Params(
                 market=self.state.public_key(),
                 open_orders=open_order_account,
                 payer=payer,
                 owner=owner.public_key(),
                 request_queue=self.state.request_queue(),
+                event_queue=self.state.event_queue(),
+                bids=self.state.bids(),
+                asks=self.state.asks(),
                 base_vault=self.state.base_vault(),
                 quote_vault=self.state.quote_vault(),
                 side=side,
-                limit_price=limit_price,
+                limit_price=self.state.price_number_to_lots(limit_price),
-                max_quantity=max_quantity,
+                max_base_quantity=self.state.base_size_number_to_lots(max_quantity),
+                max_quote_quantity=self.state.quote_size_number_to_lots(max_quantity * limit_price),
                 order_type=order_type,
                 client_id=client_id,
                 program_id=self.state.program_id(),
+                self_trade_behavior=SelfTradeBehavior.DecrementTake,
+                fee_discount_pubkey=fee_discount_pubkey,
+                limit=65535,
             )
         )
 
@@ -301,12 +338,25 @@ class Market:
     def make_cancel_order_by_client_id_instruction(
         self, owner: Account, open_orders_account: PublicKey, client_id: int
     ) -> TransactionInstruction:
-        return instructions.cancel_order_by_client_id(
+        if self._use_request_queue():
-            instructions.CancelOrderByClientIDParams(
+            return instructions.cancel_order_by_client_id(
+                instructions.CancelOrderByClientIDParams(
+                    market=self.state.public_key(),
+                    owner=owner.public_key(),
+                    open_orders=open_orders_account,
+                    request_queue=self.state.request_queue(),
+                    client_id=client_id,
+                    program_id=self.state.program_id(),
+                )
+            )
+        return instructions.cancel_order_by_client_id_v2(
+            instructions.CancelOrderByClientIDV2Params(
                 market=self.state.public_key(),
                 owner=owner.public_key(),
                 open_orders=open_orders_account,
-                request_queue=self.state.request_queue(),
+                bids=self.state.bids(),
+                asks=self.state.asks(),
+                event_queue=self.state.event_queue(),
                 client_id=client_id,
                 program_id=self.state.program_id(),
             )
@@ -316,23 +366,39 @@ class Market:
         txn = Transaction().add(self.make_cancel_order_instruction(owner.public_key(), order))
         return self._conn.send_transaction(txn, owner, opts=opts)
 
+    def make_cancel_order_instruction(self, owner: PublicKey, order: t.Order) -> TransactionInstruction:
+        if self._use_request_queue():
+            return instructions.cancel_order(
+                instructions.CancelOrderParams(
+                    market=self.state.public_key(),
+                    owner=owner,
+                    open_orders=order.open_order_address,
+                    request_queue=self.state.request_queue(),
+                    side=order.side,
+                    order_id=order.order_id,
+                    open_orders_slot=order.open_order_slot,
+                    program_id=self.state.program_id(),
+                )
+            )
+        return instructions.cancel_order_v2(
+            instructions.CancelOrderV2Params(
+                market=self.state.public_key(),
+                owner=owner,
+                open_orders=order.open_order_address,
+                bids=self.state.bids(),
+                asks=self.state.asks(),
+                event_queue=self.state.event_queue(),
+                side=order.side,
+                order_id=order.order_id,
+                open_orders_slot=order.open_order_slot,
+                program_id=self.state.program_id(),
+            )
+        )
+
     def match_orders(self, fee_payer: Account, limit: int, opts: TxOpts = TxOpts()) -> RPCResponse:
         txn = Transaction().add(self.make_match_orders_instruction(limit))
         return self._conn.send_transaction(txn, fee_payer, opts=opts)
 
-    def make_cancel_order_instruction(self, owner: PublicKey, order: t.Order) -> TransactionInstruction:
-        params = instructions.CancelOrderParams(
-            market=self.state.public_key(),
-            owner=owner,
-            open_orders=order.open_order_address,
-            request_queue=self.state.request_queue(),
-            side=order.side,
-            order_id=order.order_id,
-            open_orders_slot=order.open_order_slot,
-            program_id=self.state.program_id(),
-        )
-        return instructions.cancel_order(params)
-
     def make_match_orders_instruction(self, limit: int) -> TransactionInstruction:
         params = instructions.MatchOrdersParams(
             market=self.state.public_key(),

pyserum/market/state.py

@@ -147,4 +147,10 @@ class MarketState:  # pylint: disable=too-many-public-methods
         return float(size * self.base_lot_size()) / self.base_spl_token_multiplier()
 
     def base_size_number_to_lots(self, size: float) -> int:
-        return int(math.floor(size * 10 ** self._base_mint_decimals) / self.base_lot_size())
+        return int(math.floor(size * self.base_spl_token_multiplier()) / self.base_lot_size())
+
+    def quote_size_lots_to_number(self, size: int) -> float:
+        return float(size * self.quote_lot_size()) / self.quote_spl_token_multiplier()
+
+    def quote_size_number_to_lots(self, size: float) -> int:
+        return int(math.floor(size * self.quote_spl_token_multiplier()) / self.quote_lot_size())

scripts/bootstrap_dex.sh

@@ -13,15 +13,17 @@ else
     exit 1
 fi
 
-docker-compose up -d
+
 if ! hash solana 2>/dev/null; then
     echo Installing Solana tool suite ...
-    curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.3.9/install/solana-install-init.sh | sh -s - v1.3.9
+    curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v1.5.8/install/solana-install-init.sh | SOLANA_RELEASE=v1.5.8 sh -s - v1.5.8
     export PATH="/home/runner/.local/share/solana/install/active_release/bin:$PATH"
     echo Generating keypair ...
     solana-keygen new -o ~/.config/solana/id.json --no-passphrase --silent
 fi
-solana config set --url "http://localhost:8899"
+
+solana-test-validator &
+solana config set --url "http://127.0.0.1:8899"
 curl -s -L "https://github.com/serum-community/serum-dex/releases/download/v2/serum_dex-$os_type.so" > serum_dex.so
 sleep 1
 solana airdrop 10000

tests/integration/test_market.py

@@ -13,7 +13,7 @@ from pyserum.market import Market
 @pytest.mark.integration
 @pytest.fixture(scope="module")
 def bootstrapped_market(http_client: Client, stubbed_market_pk: PublicKey, stubbed_dex_program_pk: PublicKey) -> Market:
-    return Market.load(http_client, stubbed_market_pk, stubbed_dex_program_pk)
+    return Market.load(http_client, stubbed_market_pk, stubbed_dex_program_pk, force_use_request_queue=True)
 
 
 @pytest.mark.integration