From a941f486eded534a29f7911ab49193f9954a2d30 Mon Sep 17 00:00:00 2001 From: Christian Kamm Date: Thu, 25 Nov 2021 10:24:06 +0100 Subject: [PATCH] Fix bug introduced by deleting older slots Deleting older slots meant that older (but still most recent) account writes that had their slot deleted would no longer show up in views. Now the guarantee is that an account write for a missing slot that is smaller than the top slot in the slots table is rooted. To guarantee that, account writes for non-rooted slots that preceed rooted slots are deleted now. --- README.md | 4 +- connector-mango/scripts/create_views.sql | 74 ++++++++++++++++-------- connector-mango/scripts/drop_views.sql | 6 +- connector-raw/scripts/create_views.sql | 18 ++++-- lib/src/postgres_target.rs | 27 +++++++-- 5 files changed, 88 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index ab484c2..bba7743 100644 --- a/README.md +++ b/README.md @@ -147,9 +147,9 @@ Example for querying confirmed data: SELECT DISTINCT ON(pubkey_id) pubkey, account_write.* FROM account_write -INNER JOIN slot USING(slot) +LEFT JOIN slot USING(slot) INNER JOIN pubkey USING(pubkey_id) -WHERE status = 'Rooted' OR (uncle = FALSE AND status = 'Confirmed') +WHERE status = 'Rooted' OR status IS NULL OR (uncle = FALSE AND status = 'Confirmed') ORDER BY pubkey_id, slot DESC, write_version DESC; ``` diff --git a/connector-mango/scripts/create_views.sql b/connector-mango/scripts/create_views.sql index 24e4eb6..0dd1d09 100644 --- a/connector-mango/scripts/create_views.sql +++ b/connector-mango/scripts/create_views.sql @@ -4,27 +4,33 @@ CREATE VIEW account_rooted AS DISTINCT ON(pubkey_id) pubkey, account_write.* FROM account_write - INNER JOIN slot USING(slot) + LEFT JOIN slot USING(slot) INNER JOIN pubkey USING(pubkey_id) - WHERE slot.status = 'Rooted' + CROSS JOIN (SELECT max(slot) FROM slot) ms + WHERE slot <= ms.max + AND (slot.status = 'Rooted' OR slot.status is NULL) ORDER BY pubkey_id, slot DESC, write_version DESC; CREATE VIEW account_confirmed AS SELECT DISTINCT ON(pubkey_id) pubkey, account_write.* FROM account_write - INNER JOIN slot USING(slot) + LEFT JOIN slot USING(slot) INNER JOIN pubkey USING(pubkey_id) - WHERE (slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' + CROSS JOIN (SELECT max(slot) FROM slot) ms + WHERE slot <= ms.max + AND ((slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL) ORDER BY pubkey_id, slot DESC, write_version DESC; CREATE VIEW account_processed AS SELECT DISTINCT ON(pubkey_id) pubkey, account_write.* FROM account_write - INNER JOIN slot USING(slot) + LEFT JOIN slot USING(slot) INNER JOIN pubkey USING(pubkey_id) - WHERE ((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' + CROSS JOIN (SELECT max(slot) FROM slot) ms + WHERE slot <= ms.max + AND (((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL) ORDER BY pubkey_id, slot DESC, write_version DESC; @@ -33,27 +39,33 @@ CREATE VIEW mango_account_rooted AS DISTINCT ON(pubkey_id) pubkey, mango_account_write.* FROM mango_account_write - INNER JOIN slot USING(slot) + LEFT JOIN slot USING(slot) INNER JOIN pubkey using(pubkey_id) - WHERE slot.status = 'Rooted' + CROSS JOIN (SELECT max(slot) FROM slot) ms + WHERE slot <= ms.max + AND (slot.status = 'Rooted' OR slot.status is NULL) ORDER BY pubkey_id, slot DESC, write_version DESC; CREATE VIEW mango_account_confirmed AS SELECT DISTINCT ON(pubkey_id) pubkey, mango_account_write.* FROM mango_account_write - INNER JOIN slot USING(slot) + LEFT JOIN slot USING(slot) INNER JOIN pubkey using(pubkey_id) - WHERE (slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' + CROSS JOIN (SELECT max(slot) FROM slot) ms + WHERE slot <= ms.max + AND ((slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL) ORDER BY pubkey_id, slot DESC, write_version DESC; CREATE VIEW mango_account_processed AS SELECT DISTINCT ON(pubkey_id) pubkey, mango_account_write.* FROM mango_account_write - INNER JOIN slot USING(slot) + LEFT JOIN slot USING(slot) INNER JOIN pubkey using(pubkey_id) - WHERE ((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' + CROSS JOIN (SELECT max(slot) FROM slot) ms + WHERE slot <= ms.max + AND (((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL) ORDER BY pubkey_id, slot DESC, write_version DESC; CREATE VIEW mango_account_processed_balance AS @@ -82,27 +94,33 @@ CREATE VIEW mango_group_rooted AS DISTINCT ON(pubkey_id) pubkey, mango_group_write.* FROM mango_group_write - INNER JOIN slot USING(slot) + LEFT JOIN slot USING(slot) INNER JOIN pubkey using(pubkey_id) - WHERE slot.status = 'Rooted' + CROSS JOIN (SELECT max(slot) FROM slot) ms + WHERE slot <= ms.max + AND (slot.status = 'Rooted' OR slot.status is NULL) ORDER BY pubkey_id, slot DESC, write_version DESC; CREATE VIEW mango_group_confirmed AS SELECT DISTINCT ON(pubkey_id) pubkey, mango_group_write.* FROM mango_group_write - INNER JOIN slot USING(slot) + LEFT JOIN slot USING(slot) INNER JOIN pubkey using(pubkey_id) - WHERE (slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' + CROSS JOIN (SELECT max(slot) FROM slot) ms + WHERE slot <= ms.max + AND ((slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL) ORDER BY pubkey_id, slot DESC, write_version DESC; CREATE VIEW mango_group_processed AS SELECT DISTINCT ON(pubkey_id) pubkey, mango_group_write.* FROM mango_group_write - INNER JOIN slot USING(slot) + LEFT JOIN slot USING(slot) INNER JOIN pubkey using(pubkey_id) - WHERE ((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' + CROSS JOIN (SELECT max(slot) FROM slot) ms + WHERE slot <= ms.max + AND (((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL) ORDER BY pubkey_id, slot DESC, write_version DESC; CREATE VIEW mango_cache_rooted AS @@ -110,25 +128,31 @@ CREATE VIEW mango_cache_rooted AS DISTINCT ON(pubkey_id) pubkey, mango_cache_write.* FROM mango_cache_write - INNER JOIN slot USING(slot) + LEFT JOIN slot USING(slot) INNER JOIN pubkey using(pubkey_id) - WHERE slot.status = 'Rooted' + CROSS JOIN (SELECT max(slot) FROM slot) ms + WHERE slot <= ms.max + AND (slot.status = 'Rooted' OR slot.status is NULL) ORDER BY pubkey_id, slot DESC, write_version DESC; CREATE VIEW mango_cache_confirmed AS SELECT DISTINCT ON(pubkey_id) pubkey, mango_cache_write.* FROM mango_cache_write - INNER JOIN slot USING(slot) + LEFT JOIN slot USING(slot) INNER JOIN pubkey using(pubkey_id) - WHERE (slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' + CROSS JOIN (SELECT max(slot) FROM slot) ms + WHERE slot <= ms.max + AND ((slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL) ORDER BY pubkey_id, slot DESC, write_version DESC; CREATE VIEW mango_cache_processed AS SELECT DISTINCT ON(pubkey_id) pubkey, mango_cache_write.* FROM mango_cache_write - INNER JOIN slot USING(slot) + LEFT JOIN slot USING(slot) INNER JOIN pubkey using(pubkey_id) - WHERE ((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' - ORDER BY pubkey_id, slot DESC, write_version DESC; \ No newline at end of file + CROSS JOIN (SELECT max(slot) FROM slot) ms + WHERE slot <= ms.max + AND (((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL) + ORDER BY pubkey_id, slot DESC, write_version DESC; diff --git a/connector-mango/scripts/drop_views.sql b/connector-mango/scripts/drop_views.sql index 358601a..ba6a3c5 100644 --- a/connector-mango/scripts/drop_views.sql +++ b/connector-mango/scripts/drop_views.sql @@ -2,13 +2,13 @@ DROP VIEW account_rooted; DROP VIEW account_confirmed; DROP VIEW account_processed; +DROP VIEW mango_account_processed_balance; +DROP VIEW mango_account_processed_perp; + DROP VIEW mango_account_rooted; DROP VIEW mango_account_confirmed; DROP VIEW mango_account_processed; -DROP VIEW mango_account_processed_balance; -DROP VIEW mango_account_processed_perp; - DROP VIEW mango_group_rooted; DROP VIEW mango_group_confirmed; DROP VIEW mango_group_processed; diff --git a/connector-raw/scripts/create_views.sql b/connector-raw/scripts/create_views.sql index 5dcb192..2657ba9 100644 --- a/connector-raw/scripts/create_views.sql +++ b/connector-raw/scripts/create_views.sql @@ -4,25 +4,31 @@ CREATE VIEW account_rooted AS DISTINCT ON(pubkey_id) pubkey, account_write.* FROM account_write - INNER JOIN slot USING(slot) + LEFT JOIN slot USING(slot) INNER JOIN pubkey USING(pubkey_id) - WHERE slot.status = 'Rooted' + CROSS JOIN (SELECT max(slot) FROM slot) ms + WHERE slot <= ms.max + AND (slot.status = 'Rooted' OR slot.status is NULL) ORDER BY pubkey_id, slot DESC, write_version DESC; CREATE VIEW account_confirmed AS SELECT DISTINCT ON(pubkey_id) pubkey, account_write.* FROM account_write - INNER JOIN slot USING(slot) + LEFT JOIN slot USING(slot) INNER JOIN pubkey USING(pubkey_id) - WHERE (slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' + CROSS JOIN (SELECT max(slot) FROM slot) ms + WHERE slot <= ms.max + AND ((slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL) ORDER BY pubkey_id, slot DESC, write_version DESC; CREATE VIEW account_processed AS SELECT DISTINCT ON(pubkey_id) pubkey, account_write.* FROM account_write - INNER JOIN slot USING(slot) + LEFT JOIN slot USING(slot) INNER JOIN pubkey USING(pubkey_id) - WHERE ((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' + CROSS JOIN (SELECT max(slot) FROM slot) ms + WHERE slot <= ms.max + AND (((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL) ORDER BY pubkey_id, slot DESC, write_version DESC; diff --git a/lib/src/postgres_target.rs b/lib/src/postgres_target.rs index b6d796a..a79bf11 100644 --- a/lib/src/postgres_target.rs +++ b/lib/src/postgres_target.rs @@ -171,6 +171,10 @@ struct SlotsProcessing { impl SlotsProcessing { fn new(tables: &Vec) -> Self { + // Delete: + // 1. account writes that came before the newest rooted write + // 2. account writes that came after the newest rooted write but before + // the newest rooted slot (like processed writes that never confirmed) let mut cleanup_table_sql: Vec = tables .iter() .map(|table_name| { @@ -179,14 +183,26 @@ impl SlotsProcessing { USING ( SELECT DISTINCT ON(pubkey_id) pubkey_id, slot, write_version FROM {table} - INNER JOIN slot USING(slot) - WHERE slot <= $newest_final_slot AND status = 'Rooted' + LEFT JOIN slot USING(slot) + WHERE slot <= $newest_final_slot AND (status = 'Rooted' OR status is NULL) ORDER BY pubkey_id, slot DESC, write_version DESC ) latest_write WHERE data.pubkey_id = latest_write.pubkey_id - AND (data.slot < latest_write.slot - OR (data.slot = latest_write.slot - AND data.write_version < latest_write.write_version + AND ( + (data.slot < latest_write.slot + OR (data.slot = latest_write.slot + AND data.write_version < latest_write.write_version + ) + ) + OR + ( + data.slot < $newest_final_slot + AND + (data.slot > latest_write.slot + OR (data.slot = latest_write.slot + AND data.write_version > latest_write.write_version + ) + ) ) )", table = table_name @@ -194,6 +210,7 @@ impl SlotsProcessing { }) .collect(); + // Delete old slots cleanup_table_sql.push("DELETE FROM slot WHERE slot + 100000 < $newest_final_slot".into()); Self { cleanup_table_sql }