Better logging of failing instructions.

mango/account.py

@@ -96,7 +96,9 @@ class Account(AddressableAccount):
     def __init__(self, account_info: AccountInfo, version: Version,
                  meta_data: Metadata, group: Group, owner: PublicKey,
                  shared_quote_token: AccountBasketToken,
-                 basket_indices: typing.Sequence[bool], basket: typing.Sequence[AccountBasketBaseToken],
+                 in_margin_basket: typing.Sequence[bool],
+                 basket_indices: typing.Sequence[bool],
+                 basket: typing.Sequence[AccountBasketBaseToken],
                  msrm_amount: Decimal, being_liquidated: bool, is_bankrupt: bool):
         super().__init__(account_info)
         self.version: Version = version
@@ -105,6 +107,7 @@ class Account(AddressableAccount):
         self.group: Group = group
         self.owner: PublicKey = owner
         self.shared_quote_token: AccountBasketToken = shared_quote_token
+        self.in_margin_basket: typing.Sequence[bool] = in_margin_basket
         self.basket_indices: typing.Sequence[bool] = basket_indices
         self.basket: typing.Sequence[AccountBasketBaseToken] = basket
         self.msrm_amount: Decimal = msrm_amount
@@ -116,9 +119,10 @@ class Account(AddressableAccount):
         meta_data = Metadata.from_layout(layout.meta_data)
         owner: PublicKey = layout.owner
         in_margin_basket: typing.Sequence[bool] = list([bool(in_basket) for in_basket in layout.in_margin_basket])
+        active_in_basket: typing.List[bool] = []
         basket: typing.List[AccountBasketBaseToken] = []
         for index, token_info in enumerate(group.tokens[:-1]):
-            if token_info and in_margin_basket[index]:
+            if token_info:
                 intrinsic_deposit = token_info.root_bank.deposit_index * layout.deposits[index]
                 deposit = TokenValue(token_info.token, token_info.token.shift_to_decimals(intrinsic_deposit))
                 intrinsic_borrow = token_info.root_bank.borrow_index * layout.borrows[index]
@@ -128,6 +132,9 @@ class Account(AddressableAccount):
                 basket_item: AccountBasketBaseToken = AccountBasketBaseToken(
                     token_info, deposit, borrow, spot_open_orders, perp_account)
                 basket += [basket_item]
+                active_in_basket += [True]
+            else:
+                active_in_basket += [False]
 
         quote_token_info: typing.Optional[TokenInfo] = group.tokens[-1]
         if quote_token_info is None:
@@ -145,7 +152,7 @@ class Account(AddressableAccount):
         being_liquidated: bool = bool(layout.being_liquidated)
         is_bankrupt: bool = bool(layout.is_bankrupt)
 
-        return Account(account_info, version, meta_data, group, owner, quote, in_margin_basket, basket, msrm_amount, being_liquidated, is_bankrupt)
+        return Account(account_info, version, meta_data, group, owner, quote, in_margin_basket, active_in_basket, basket, msrm_amount, being_liquidated, is_bankrupt)
 
     @staticmethod
     def parse(account_info: AccountInfo, group: Group) -> "Account":

mango/combinableinstructions.py

@@ -16,19 +16,42 @@
 import logging
 import typing
 
+from pyserum._layouts.instructions import InstructionType as SerumInstructionType
 from solana.account import Account as SolanaAccount
 from solana.blockhash import Blockhash
 from solana.publickey import PublicKey
 from solana.transaction import Transaction, TransactionInstruction
 
 from .context import Context
+from .layouts import layouts
+from .transactionscout import MangoInstruction, InstructionType
 from .wallet import Wallet
 
 _MAXIMUM_TRANSACTION_LENGTH = 1280 - 40 - 8
 _SIGNATURE_LENGTH = 64
 
 
-def _instruction_to_str(instruction: TransactionInstruction) -> str:
+def _mango_instruction_to_str(instruction: TransactionInstruction) -> str:
+    initial = layouts.MANGO_INSTRUCTION_VARIANT_FINDER.parse(instruction.data)
+    parser = layouts.InstructionParsersByVariant[initial.variant]
+    if parser is None:
+        raise Exception(
+            f"Could not find instruction parser for variant {initial.variant} / {InstructionType(initial.variant)}.")
+
+    accounts: typing.List[PublicKey] = list(map(lambda meta: meta.pubkey, instruction.keys))
+    parsed = parser.parse(instruction.data)
+    instruction_type = InstructionType(int(parsed.variant))
+
+    return str(MangoInstruction(instruction_type, parsed, accounts))
+
+
+def _serum_instruction_to_str(instruction: TransactionInstruction) -> str:
+    initial = layouts.SERUM_INSTRUCTION_VARIANT_FINDER.parse(instruction.data)
+    instruction_type = SerumInstructionType(initial.variant)
+    return f"« Serum Instruction: {instruction_type.name}: " + "".join("{:02x}".format(x) for x in instruction.data) + "»"
+
+
+def _raw_instruction_to_str(instruction: TransactionInstruction) -> str:
     report: typing.List[str] = []
     for index, key in enumerate(instruction.keys):
         report += [f"Key[{index}]: {key.pubkey} {key.is_signer: <5} {key.is_writable: <5}"]
@@ -37,6 +60,14 @@ def _instruction_to_str(instruction: TransactionInstruction) -> str:
     return "\n".join(report)
 
 
+def _instruction_to_str(context: Context, instruction: TransactionInstruction) -> str:
+    if instruction.program_id == context.program_id:
+        return _mango_instruction_to_str(instruction)
+    elif instruction.program_id == context.dex_program_id:
+        return _serum_instruction_to_str(instruction)
+    return _raw_instruction_to_str(instruction)
+
+
 def _split_instructions_into_chunks(signers: typing.Sequence[SolanaAccount], instructions: typing.Sequence[TransactionInstruction]) -> typing.Sequence[typing.Sequence[TransactionInstruction]]:
     vetted_chunks: typing.List[typing.List[TransactionInstruction]] = []
     current_chunk: typing.List[TransactionInstruction] = []
@@ -151,7 +182,7 @@ class CombinableInstructions():
                 response = context.client.send_transaction(transaction, *self.signers, opts=context.transaction_options)
                 results += [context.unwrap_or_raise_exception(response)]
             except Exception as exception:
-                instruction_str: str = _instruction_to_str(instruction)
+                instruction_str: str = _instruction_to_str(context, instruction)
                 self.logger.error(f"""Error executing individual instruction: {exception}
 {instruction_str}""")
 
@@ -175,10 +206,12 @@ class CombinableInstructions():
             try:
                 response = context.client.send_transaction(transaction, *self.signers, opts=context.transaction_options)
                 results += [context.unwrap_or_raise_exception(response)]
-            except:
+            except Exception as exception:
                 starts_at = sum(len(ch) for ch in chunks[0:index])
-                instruction_text = list(map(_instruction_to_str, chunk))
-                self.logger.error(f"""Error executing chunk {index} (instructions {starts_at} to {starts_at + len(chunk)}) of CombinableInstruction. Failing instruction(s):
+                instruction_text = "\n".join(list(map(lambda ins: _instruction_to_str(context, ins), chunk)))
+                self.logger.error(f"""Error executing chunk {index} (instructions {starts_at} to {starts_at + len(chunk)}) of CombinableInstruction.
+Exception: {exception}
+Failing instruction(s):
 {instruction_text}""")
 
         return results
@@ -192,7 +225,7 @@ class CombinableInstructions():
             report += [f"Signer[{index}]: {signer.public_key()}"]
 
         for instruction in self.instructions:
-            report += _instruction_to_str(instruction)
+            report += _raw_instruction_to_str(instruction)
 
         return "\n".join(report)
 

mango/inventory.py

@@ -77,12 +77,8 @@ class InventoryAccountWatcher:
         self.account_watcher: Watcher[Account] = account_watcher
         account: Account = account_watcher.latest
         base_value = TokenValue.find_by_symbol(account.net_assets, market.base.symbol)
-        if base_value is None:
-            raise Exception(f"Could not find net assets in account {account.address} for base token {market.base}.")
         self.base_index: int = account.net_assets.index(base_value)
         quote_value = TokenValue.find_by_symbol(account.net_assets, market.quote.symbol)
-        if quote_value is None:
-            raise Exception(f"Could not find net assets in account {account.address} for quote token {market.quote}.")
         self.quote_index: int = account.net_assets.index(quote_value)
 
     @property

mango/layouts/layouts.py

@@ -1101,6 +1101,12 @@ MANGO_INSTRUCTION_VARIANT_FINDER = construct.Struct(
     "variant" / construct.BytesInteger(4, swapped=True)
 )
 
+
+SERUM_INSTRUCTION_VARIANT_FINDER = construct.Struct(
+    "version" / construct.BytesInteger(1, swapped=True),
+    "variant" / construct.BytesInteger(4, swapped=True)
+)
+
 # /// Place an order on a perp market
 # /// Accounts expected by this instruction (6):
 # /// 0. `[]` mango_group_ai - TODO

mango/orders.py

@@ -41,6 +41,11 @@ class Side(enum.Enum):
     BUY = "BUY"
     SELL = "SELL"
 
+    @staticmethod
+    def from_value(value: Decimal) -> "Side":
+        converted: pyserum.enums.Side = pyserum.enums.Side(int(value))
+        return Side.BUY if converted == pyserum.enums.Side.BUY else Side.SELL
+
     def __str__(self) -> str:
         return self.value
 
@@ -63,6 +68,17 @@ class OrderType(enum.Enum):
     IOC = "IOC"
     POST_ONLY = "POST_ONLY"
 
+    @staticmethod
+    def from_value(value: Decimal) -> "OrderType":
+        converted: pyserum.enums.OrderType = pyserum.enums.OrderType(int(value))
+        if converted == pyserum.enums.OrderType.IOC:
+            return OrderType.IOC
+        elif converted == pyserum.enums.OrderType.POST_ONLY:
+            return OrderType.POST_ONLY
+        elif converted == pyserum.enums.OrderType.LIMIT:
+            return OrderType.LIMIT
+        return OrderType.UNKNOWN
+
     def __str__(self) -> str:
         return self.value
 
@@ -99,7 +115,7 @@ class Order(typing.NamedTuple):
     def from_serum_order(serum_order: SerumOrder) -> "Order":
         price = Decimal(serum_order.info.price)
         quantity = Decimal(serum_order.info.size)
-        side = Side.BUY if serum_order.side == pyserum.enums.Side.BUY else Side.SELL
+        side = Side.from_value(serum_order.side)
         order = Order(id=serum_order.order_id, side=side, price=price, quantity=quantity,
                       client_id=serum_order.client_id, owner=serum_order.open_order_address,
                       order_type=OrderType.UNKNOWN)

mango/transactionscout.py

@@ -26,6 +26,7 @@ from solana.publickey import PublicKey
 from .context import Context
 from .instructiontype import InstructionType
 from .layouts import layouts
+from .orders import OrderType, Side
 from .ownedtokenvalue import OwnedTokenValue
 from .tokenvalue import TokenValue
 
@@ -182,8 +183,6 @@ _target_indices: typing.Dict[InstructionType, int] = {
 # This class packages up Mango instruction data, which can come from disparate parts of the
 # transaction. Keeping it all together here makes many things simpler.
 #
-
-
 class MangoInstruction:
     def __init__(self, instruction_type: InstructionType, instruction_data: typing.Any, accounts: typing.Sequence[PublicKey]):
         self.instruction_type = instruction_type
@@ -245,17 +244,17 @@ class MangoInstruction:
         elif instruction_type == InstructionType.CacheRootBanks:
             pass
         elif instruction_type == InstructionType.PlaceSpotOrder:
-            additional_data = f"side: {self.instruction_data.side}, order_type: {self.instruction_data.order_type}, limit_price: {self.instruction_data.limit_price}, max_base_quantity: {self.instruction_data.max_base_quantity}, max_quote_quantity: {self.instruction_data.max_quote_quantity}, self_trade_behavior: {self.instruction_data.self_trade_behavior}, client_id: {self.instruction_data.client_id}, limit: {self.instruction_data.limit}"
+            additional_data = f"side: {Side.from_value(self.instruction_data.side)}, order_type: {OrderType.from_value(self.instruction_data.order_type)}, limit_price: {self.instruction_data.limit_price}, max_base_quantity: {self.instruction_data.max_base_quantity}, max_quote_quantity: {self.instruction_data.max_quote_quantity}, self_trade_behavior: {self.instruction_data.self_trade_behavior}, client_id: {self.instruction_data.client_id}, limit: {self.instruction_data.limit}"
         elif instruction_type == InstructionType.AddOracle:
             pass
         elif instruction_type == InstructionType.AddPerpMarket:
             pass
         elif instruction_type == InstructionType.PlacePerpOrder:
-            additional_data = f"side: {self.instruction_data.side}, order_type: {self.instruction_data.order_type}, price: {self.instruction_data.price}, quantity: {self.instruction_data.quantity}, client_order_id: {self.instruction_data.client_order_id}"
+            additional_data = f"side: {Side.from_value(self.instruction_data.side)}, order_type: {OrderType.from_value(self.instruction_data.order_type)}, price: {self.instruction_data.price}, quantity: {self.instruction_data.quantity}, client_order_id: {self.instruction_data.client_order_id}"
         elif instruction_type == InstructionType.CancelPerpOrderByClientId:
             additional_data = f"client ID: {self.instruction_data.client_order_id}"
         elif instruction_type == InstructionType.CancelPerpOrder:
-            additional_data = f"order ID: {self.instruction_data.order_id}, side: {self.instruction_data.side}"
+            additional_data = f"order ID: {self.instruction_data.order_id}, side: {Side.from_value(self.instruction_data.side)}"
         elif instruction_type == InstructionType.ConsumeEvents:
             additional_data = f"limit: {self.instruction_data.limit}"
         elif instruction_type == InstructionType.CachePerpMarkets:
@@ -267,7 +266,7 @@ class MangoInstruction:
         elif instruction_type == InstructionType.SettleFunds:
             pass
         elif instruction_type == InstructionType.CancelSpotOrder:
-            additional_data = f"order ID: {self.instruction_data.order_id}, side: {self.instruction_data.side}"
+            additional_data = f"order ID: {self.instruction_data.order_id}, side: {Side.from_value(self.instruction_data.side)}"
         elif instruction_type == InstructionType.UpdateRootBank:
             pass
         elif instruction_type == InstructionType.SettlePnl:
@@ -310,6 +309,10 @@ class MangoInstruction:
 
         return MangoInstruction(instruction_type, parsed, accounts)
 
+    @staticmethod
+    def _order_type_enum(side: Decimal):
+        return Side.BUY if side == Decimal(0) else Side.SELL
+
     def __str__(self) -> str:
         parameters = self.describe_parameters() or "None"
         return f"« {self.instruction_type.name}: {parameters} »"

tests/test_account.py

@@ -10,7 +10,8 @@ def test_construction():
     meta_data = mango.Metadata(layouts.DATA_TYPE.Group, mango.Version.V1, True)
     group = fake_seeded_public_key("group")
     owner = fake_seeded_public_key("owner")
-    in_margin_basket = [False, True, False, True, True]
+    in_margin_basket = [False, False, False, False, False]
+    active_in_basket = [False, True, False, True, True]
     quote_deposit = fake_token_value(Decimal(50))
     quote_borrow = fake_token_value(Decimal(5))
     quote = mango.AccountBasketToken(fake_token_info(), quote_deposit, quote_borrow)
@@ -33,7 +34,8 @@ def test_construction():
     is_bankrupt = False
 
     actual = mango.Account(account_info, mango.Version.V1, meta_data, group, owner, quote,
-                           in_margin_basket, basket, msrm_amount, being_liquidated, is_bankrupt)
+                           in_margin_basket, active_in_basket, basket, msrm_amount, being_liquidated,
+                           is_bankrupt)
 
     assert actual is not None
     assert actual.logger is not None
@@ -41,7 +43,8 @@ def test_construction():
     assert actual.meta_data == meta_data
     assert actual.group == group
     assert actual.owner == owner
-    assert actual.basket_indices == in_margin_basket
+    assert actual.basket_indices == active_in_basket
+    assert actual.in_margin_basket == in_margin_basket
     assert actual.deposits == [None, deposit1, None, deposit2, deposit3, quote_deposit]
     assert actual.borrows == [None, borrow1, None, borrow2, borrow3, quote_borrow]
     assert actual.net_assets == [None, deposit1 - borrow1, None,