2021-05-06 11:06:47 -07:00
|
|
|
#!/usr/bin/env pyston3
|
2021-04-26 09:09:35 -07:00
|
|
|
|
2021-04-27 02:53:43 -07:00
|
|
|
import argparse
|
2021-04-29 02:53:01 -07:00
|
|
|
import logging
|
2021-06-07 07:10:18 -07:00
|
|
|
import os
|
2021-04-26 09:09:35 -07:00
|
|
|
import os.path
|
2021-06-17 08:30:40 -07:00
|
|
|
import rx
|
2021-06-21 03:06:26 -07:00
|
|
|
import rx.core.typing
|
2021-06-17 08:30:40 -07:00
|
|
|
import rx.operators as ops
|
2021-06-07 07:10:18 -07:00
|
|
|
import sys
|
2021-06-18 05:50:33 -07:00
|
|
|
import threading
|
2021-04-26 09:09:35 -07:00
|
|
|
import traceback
|
2021-06-21 03:06:26 -07:00
|
|
|
import typing
|
2021-04-26 09:09:35 -07:00
|
|
|
|
2021-05-06 06:21:40 -07:00
|
|
|
from decimal import Decimal
|
|
|
|
|
2021-06-07 07:10:18 -07:00
|
|
|
sys.path.insert(0, os.path.abspath(
|
|
|
|
os.path.join(os.path.dirname(__file__), '..')))
|
|
|
|
import mango # nopep8
|
2021-04-26 09:09:35 -07:00
|
|
|
|
2021-04-28 10:36:48 -07:00
|
|
|
# We explicitly want argument parsing to be outside the main try-except block because some arguments
|
2021-05-11 11:24:02 -07:00
|
|
|
# (like --help) will cause an exit, which our except: block traps.
|
2021-05-07 02:47:01 -07:00
|
|
|
parser = argparse.ArgumentParser(description="Run a liquidator for a Mango Markets group.")
|
2021-06-17 04:06:39 -07:00
|
|
|
mango.Context.add_command_line_parameters(parser)
|
|
|
|
mango.Wallet.add_command_line_parameters(parser)
|
2021-05-21 12:02:54 -07:00
|
|
|
parser.add_argument("--name", type=str, default="Mango Markets Liquidator",
|
|
|
|
help="Name of the liquidator (used in reports and alerts)")
|
2021-05-27 09:07:02 -07:00
|
|
|
parser.add_argument("--throttle-reload-to-seconds", type=Decimal, default=Decimal(60),
|
2021-05-12 02:52:26 -07:00
|
|
|
help="minimum number of seconds between each full margin account reload loop (including time taken processing accounts)")
|
2021-05-27 09:07:02 -07:00
|
|
|
parser.add_argument("--throttle-ripe-update-to-seconds", type=Decimal, default=Decimal(5),
|
2021-05-12 02:52:26 -07:00
|
|
|
help="minimum number of seconds between each ripe update loop (including time taken processing accounts)")
|
2021-05-07 02:47:01 -07:00
|
|
|
parser.add_argument("--target", type=str, action="append",
|
2021-05-06 06:21:40 -07:00
|
|
|
help="token symbol plus target value or percentage, separated by a colon (e.g. 'ETH:2.5' or 'ETH:33%')")
|
|
|
|
parser.add_argument("--action-threshold", type=Decimal, default=Decimal("0.01"),
|
|
|
|
help="fraction of total wallet value a trade must be above to be carried out")
|
2021-06-24 08:36:11 -07:00
|
|
|
parser.add_argument("--worthwhile-threshold", type=Decimal, default=Decimal("0.01"),
|
|
|
|
help="value a liquidation must be above to be carried out")
|
2021-05-06 06:21:40 -07:00
|
|
|
parser.add_argument("--adjustment-factor", type=Decimal, default=Decimal("0.05"),
|
|
|
|
help="factor by which to adjust the SELL price (akin to maximum slippage)")
|
2021-06-07 07:10:18 -07:00
|
|
|
parser.add_argument("--notify-liquidations", type=mango.parse_subscription_target, action="append", default=[],
|
2021-05-18 10:08:38 -07:00
|
|
|
help="The notification target for liquidation events")
|
2021-06-07 07:10:18 -07:00
|
|
|
parser.add_argument("--notify-successful-liquidations", type=mango.parse_subscription_target,
|
2021-05-21 12:02:54 -07:00
|
|
|
action="append", default=[], help="The notification target for successful liquidation events")
|
2021-06-07 07:10:18 -07:00
|
|
|
parser.add_argument("--notify-failed-liquidations", type=mango.parse_subscription_target,
|
2021-05-21 12:02:54 -07:00
|
|
|
action="append", default=[], help="The notification target for failed liquidation events")
|
2021-06-07 07:10:18 -07:00
|
|
|
parser.add_argument("--notify-errors", type=mango.parse_subscription_target, action="append", default=[],
|
2021-05-18 10:08:38 -07:00
|
|
|
help="The notification target for error events")
|
2021-05-07 02:47:01 -07:00
|
|
|
parser.add_argument("--dry-run", action="store_true", default=False,
|
2021-05-06 07:27:25 -07:00
|
|
|
help="runs as read-only and does not perform any transactions")
|
2021-04-28 10:36:48 -07:00
|
|
|
args = parser.parse_args()
|
2021-04-26 09:09:35 -07:00
|
|
|
|
2021-05-06 06:21:40 -07:00
|
|
|
logging.getLogger().setLevel(args.log_level)
|
2021-05-18 10:08:38 -07:00
|
|
|
for notify in args.notify_errors:
|
2021-06-07 07:10:18 -07:00
|
|
|
handler = mango.NotificationHandler(notify)
|
2021-05-18 10:08:38 -07:00
|
|
|
handler.setLevel(logging.ERROR)
|
|
|
|
logging.getLogger().addHandler(handler)
|
|
|
|
|
2021-06-07 07:10:18 -07:00
|
|
|
logging.warning(mango.WARNING_DISCLAIMER_TEXT)
|
2021-04-29 02:53:01 -07:00
|
|
|
|
2021-06-21 03:06:26 -07:00
|
|
|
|
|
|
|
def start_subscriptions(context: mango.Context, liquidation_processor: mango.LiquidationProcessor, fetch_prices: typing.Callable[[typing.Any], typing.Any], fetch_margin_accounts: typing.Callable[[typing.Any], typing.Any], throttle_reload_to_seconds: Decimal, throttle_ripe_update_to_seconds: Decimal):
|
|
|
|
liquidation_processor.state = mango.LiquidationProcessorState.STARTING
|
|
|
|
|
|
|
|
logging.info("Starting margin account fetcher subscription")
|
|
|
|
margin_account_subscription = rx.interval(float(throttle_reload_to_seconds)).pipe(
|
|
|
|
ops.subscribe_on(context.pool_scheduler),
|
|
|
|
ops.start_with(-1),
|
|
|
|
ops.map(fetch_margin_accounts(context)),
|
|
|
|
ops.catch(mango.observable_pipeline_error_reporter),
|
|
|
|
ops.retry()
|
|
|
|
).subscribe(mango.create_backpressure_skipping_observer(on_next=liquidation_processor.update_margin_accounts, on_error=mango.log_subscription_error))
|
|
|
|
|
|
|
|
logging.info("Starting price fetcher subscription")
|
|
|
|
price_subscription = rx.interval(float(throttle_ripe_update_to_seconds)).pipe(
|
|
|
|
ops.subscribe_on(context.pool_scheduler),
|
|
|
|
ops.map(fetch_prices(context)),
|
|
|
|
ops.catch(mango.observable_pipeline_error_reporter),
|
|
|
|
ops.retry()
|
|
|
|
).subscribe(mango.create_backpressure_skipping_observer(on_next=lambda piped: liquidation_processor.update_prices(piped[0], piped[1]), on_error=mango.log_subscription_error))
|
|
|
|
|
|
|
|
return margin_account_subscription, price_subscription
|
|
|
|
|
|
|
|
|
2021-04-28 10:36:48 -07:00
|
|
|
try:
|
2021-06-17 04:06:39 -07:00
|
|
|
context = mango.Context.from_command_line_parameters(args)
|
|
|
|
wallet = mango.Wallet.from_command_line_parameters_or_raise(args)
|
|
|
|
|
2021-05-06 06:21:40 -07:00
|
|
|
action_threshold = args.action_threshold
|
2021-06-24 08:36:11 -07:00
|
|
|
worthwhile_threshold = args.worthwhile_threshold
|
2021-05-06 06:21:40 -07:00
|
|
|
adjustment_factor = args.adjustment_factor
|
2021-05-12 02:52:26 -07:00
|
|
|
throttle_reload_to_seconds = args.throttle_reload_to_seconds
|
|
|
|
throttle_ripe_update_to_seconds = args.throttle_ripe_update_to_seconds
|
2021-05-21 12:02:54 -07:00
|
|
|
liquidator_name = args.name
|
2021-04-28 10:36:48 -07:00
|
|
|
|
2021-04-29 08:33:04 -07:00
|
|
|
logging.info(f"Context: {context}")
|
2021-04-28 10:36:48 -07:00
|
|
|
logging.info(f"Wallet address: {wallet.address}")
|
|
|
|
|
2021-06-07 07:10:18 -07:00
|
|
|
group = mango.Group.load(context)
|
2021-05-11 12:32:20 -07:00
|
|
|
tokens = [basket_token.token for basket_token in group.basket_tokens]
|
2021-04-28 10:36:48 -07:00
|
|
|
|
|
|
|
logging.info("Checking wallet accounts.")
|
2021-06-07 07:10:18 -07:00
|
|
|
scout = mango.AccountScout()
|
2021-04-29 08:33:04 -07:00
|
|
|
report = scout.verify_account_prepared_for_group(context, group, wallet.address)
|
|
|
|
logging.info(f"Wallet account report: {report}")
|
|
|
|
if report.has_errors:
|
|
|
|
raise Exception(f"Account '{wallet.address}' is not prepared for group '{group.address}'.")
|
|
|
|
|
2021-04-28 10:36:48 -07:00
|
|
|
logging.info("Wallet accounts OK.")
|
|
|
|
|
2021-06-07 07:10:18 -07:00
|
|
|
liquidations_publisher = mango.EventSource[mango.LiquidationEvent]()
|
|
|
|
liquidations_publisher.subscribe(on_next=lambda event: logging.info(
|
|
|
|
str(mango.TransactionScout.load(context, event.signature))))
|
2021-05-21 06:28:42 -07:00
|
|
|
for notification_target in args.notify_liquidations:
|
|
|
|
liquidations_publisher.subscribe(on_next=notification_target.send)
|
2021-06-22 01:43:31 -07:00
|
|
|
for successful_notification_target in args.notify_successful_liquidations:
|
|
|
|
captured_successful_notification_target = successful_notification_target
|
|
|
|
filtering_successful = mango.FilteringNotificationTarget(
|
|
|
|
captured_successful_notification_target, lambda item: isinstance(item, mango.LiquidationEvent) and item.succeeded)
|
|
|
|
liquidations_publisher.subscribe(on_next=filtering_successful.send)
|
|
|
|
for failed_notification_target in args.notify_failed_liquidations:
|
|
|
|
captured_failed_notification_target = failed_notification_target
|
|
|
|
filtering_failed = mango.FilteringNotificationTarget(
|
|
|
|
captured_failed_notification_target, lambda item: isinstance(item, mango.LiquidationEvent) and not item.succeeded)
|
|
|
|
liquidations_publisher.subscribe(on_next=filtering_failed.send)
|
2021-05-21 06:28:42 -07:00
|
|
|
|
2021-05-06 07:27:25 -07:00
|
|
|
if args.dry_run:
|
2021-06-19 11:55:23 -07:00
|
|
|
intermediate: mango.AccountLiquidator = mango.NullAccountLiquidator()
|
2021-05-06 07:27:25 -07:00
|
|
|
else:
|
2021-06-07 07:10:18 -07:00
|
|
|
intermediate = mango.ForceCancelOrdersAccountLiquidator(context, wallet)
|
2021-06-19 11:55:23 -07:00
|
|
|
|
|
|
|
account_liquidator: mango.AccountLiquidator = mango.ReportingAccountLiquidator(intermediate,
|
|
|
|
context,
|
|
|
|
wallet,
|
|
|
|
liquidations_publisher,
|
|
|
|
liquidator_name)
|
2021-05-06 07:27:25 -07:00
|
|
|
|
|
|
|
if args.dry_run or (args.target is None) or (len(args.target) == 0):
|
2021-06-07 07:10:18 -07:00
|
|
|
wallet_balancer: mango.WalletBalancer = mango.NullWalletBalancer()
|
2021-05-06 06:21:40 -07:00
|
|
|
else:
|
2021-06-07 07:10:18 -07:00
|
|
|
balance_parser = mango.TargetBalanceParser(tokens)
|
2021-05-06 06:21:40 -07:00
|
|
|
targets = list(map(balance_parser.parse, args.target))
|
2021-06-18 05:50:33 -07:00
|
|
|
trade_executor = mango.SerumImmediateTradeExecutor(context, wallet, adjustment_factor)
|
2021-06-08 06:38:28 -07:00
|
|
|
wallet_balancer = mango.LiveWalletBalancer(
|
|
|
|
context, wallet, group, trade_executor, action_threshold, tokens, targets)
|
2021-05-06 06:21:40 -07:00
|
|
|
|
2021-06-22 01:43:31 -07:00
|
|
|
# These (along with `context`) are captured and read by `load_updated_price_details()`.
|
|
|
|
group_address = group.address
|
|
|
|
oracle_addresses = list([market.oracle for market in group.markets])
|
|
|
|
|
|
|
|
def load_updated_price_details() -> typing.Tuple[mango.Group, typing.List[mango.TokenValue]]:
|
|
|
|
all_addresses = [group_address, *oracle_addresses]
|
|
|
|
all_account_infos = mango.AccountInfo.load_multiple(context, all_addresses)
|
|
|
|
group_account_info = all_account_infos[0]
|
|
|
|
oracle_account_infos = all_account_infos[1:]
|
|
|
|
group = mango.Group.parse(context, group_account_info)
|
|
|
|
|
|
|
|
oracles = map(lambda oracle_account_info: mango.Aggregator.parse(
|
|
|
|
context, oracle_account_info), oracle_account_infos)
|
|
|
|
prices = list(map(lambda oracle: oracle.price, oracles)) + [Decimal(1)]
|
|
|
|
token_prices = []
|
|
|
|
for index, price in enumerate(prices):
|
|
|
|
token_prices += [mango.TokenValue(group.basket_tokens[index].token, price)]
|
|
|
|
|
|
|
|
return group, token_prices
|
|
|
|
|
2021-06-17 08:30:40 -07:00
|
|
|
def fetch_prices(context):
|
|
|
|
def _fetch_prices(_):
|
|
|
|
with mango.retry_context("Price Fetch",
|
2021-06-22 01:43:31 -07:00
|
|
|
load_updated_price_details,
|
2021-06-17 08:30:40 -07:00
|
|
|
context.retry_pauses) as retrier:
|
|
|
|
return retrier.run()
|
|
|
|
|
|
|
|
return _fetch_prices
|
2021-05-17 12:43:16 -07:00
|
|
|
|
2021-06-17 08:30:40 -07:00
|
|
|
def fetch_margin_accounts(context):
|
2021-06-22 01:43:31 -07:00
|
|
|
def _actual_fetch():
|
|
|
|
group = mango.Group.load(context)
|
|
|
|
return mango.MarginAccount.load_ripe(context, group)
|
2021-06-17 08:30:40 -07:00
|
|
|
|
|
|
|
def _fetch_margin_accounts(_):
|
2021-06-07 07:10:18 -07:00
|
|
|
with mango.retry_context("Margin Account Fetch",
|
2021-06-22 01:43:31 -07:00
|
|
|
_actual_fetch,
|
2021-06-17 08:30:40 -07:00
|
|
|
context.retry_pauses) as retrier:
|
|
|
|
return retrier.run()
|
|
|
|
return _fetch_margin_accounts
|
|
|
|
|
2021-06-21 03:06:26 -07:00
|
|
|
class LiquidationProcessorSubscriptions:
|
|
|
|
def __init__(self, margin_account: rx.core.typing.Disposable, price: rx.core.typing.Disposable):
|
|
|
|
self.margin_account: rx.core.typing.Disposable = margin_account
|
|
|
|
self.price: rx.core.typing.Disposable = price
|
|
|
|
|
2021-06-24 08:36:11 -07:00
|
|
|
liquidation_processor = mango.LiquidationProcessor(
|
|
|
|
context, liquidator_name, account_liquidator, wallet_balancer, worthwhile_threshold)
|
2021-06-21 03:06:26 -07:00
|
|
|
margin_account_subscription, price_subscription = start_subscriptions(
|
|
|
|
context, liquidation_processor, fetch_prices, fetch_margin_accounts, throttle_reload_to_seconds, throttle_ripe_update_to_seconds)
|
|
|
|
|
|
|
|
subscriptions = LiquidationProcessorSubscriptions(margin_account=margin_account_subscription,
|
|
|
|
price=price_subscription)
|
|
|
|
|
|
|
|
def on_unhealthy(liquidation_processor: mango.LiquidationProcessor):
|
|
|
|
if liquidation_processor.state != mango.LiquidationProcessorState.UNHEALTHY:
|
|
|
|
logging.info(
|
|
|
|
f"Ignoring LiquidationProcessor state change - state is: {liquidation_processor.state}")
|
|
|
|
return
|
|
|
|
|
|
|
|
logging.warning("Liquidation processor has been marked as unhealthy so recreating subscriptions.")
|
|
|
|
try:
|
|
|
|
subscriptions.margin_account.dispose()
|
|
|
|
except Exception as exception:
|
|
|
|
logging.warning(f"Ignoring problem disposing of margin account subscription: {exception}")
|
|
|
|
try:
|
|
|
|
subscriptions.price.dispose()
|
|
|
|
except Exception as exception:
|
|
|
|
logging.warning(f"Ignoring problem disposing of margin account subscription: {exception}")
|
|
|
|
|
|
|
|
margin_account_subscription, price_subscription = start_subscriptions(
|
|
|
|
context, liquidation_processor, fetch_prices, fetch_margin_accounts, throttle_reload_to_seconds, throttle_ripe_update_to_seconds)
|
|
|
|
subscriptions.margin_account = margin_account_subscription
|
|
|
|
subscriptions.price = price_subscription
|
|
|
|
|
|
|
|
liquidation_processor.state_change.subscribe(on_next=on_unhealthy)
|
2021-06-17 08:30:40 -07:00
|
|
|
|
2021-06-18 05:50:33 -07:00
|
|
|
# Wait - don't exit. Exiting will be handled by signals/interrupts.
|
|
|
|
waiter = threading.Event()
|
|
|
|
waiter.wait()
|
2021-06-17 08:30:40 -07:00
|
|
|
except KeyboardInterrupt:
|
|
|
|
logging.info("Liquidator stopping...")
|
2021-04-26 09:09:35 -07:00
|
|
|
except Exception as exception:
|
|
|
|
logging.critical(f"Liquidator stopped because of exception: {exception} - {traceback.format_exc()}")
|
|
|
|
except:
|
|
|
|
logging.critical(f"Liquidator stopped because of uncatchable error: {traceback.format_exc()}")
|
|
|
|
finally:
|
|
|
|
logging.info("Liquidator completed.")
|