Fix bug introduced by deleting older slots
Deleting older slots meant that older (but still most recent) account writes that had their slot deleted would no longer show up in views. Now the guarantee is that an account write for a missing slot that is smaller than the top slot in the slots table is rooted. To guarantee that, account writes for non-rooted slots that preceed rooted slots are deleted now.
This commit is contained in:
parent
3cdbeddf8e
commit
a941f486ed
|
@ -147,9 +147,9 @@ Example for querying confirmed data:
|
||||||
SELECT DISTINCT ON(pubkey_id)
|
SELECT DISTINCT ON(pubkey_id)
|
||||||
pubkey, account_write.*
|
pubkey, account_write.*
|
||||||
FROM account_write
|
FROM account_write
|
||||||
INNER JOIN slot USING(slot)
|
LEFT JOIN slot USING(slot)
|
||||||
INNER JOIN pubkey USING(pubkey_id)
|
INNER JOIN pubkey USING(pubkey_id)
|
||||||
WHERE status = 'Rooted' OR (uncle = FALSE AND status = 'Confirmed')
|
WHERE status = 'Rooted' OR status IS NULL OR (uncle = FALSE AND status = 'Confirmed')
|
||||||
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
@ -4,27 +4,33 @@ CREATE VIEW account_rooted AS
|
||||||
DISTINCT ON(pubkey_id)
|
DISTINCT ON(pubkey_id)
|
||||||
pubkey, account_write.*
|
pubkey, account_write.*
|
||||||
FROM account_write
|
FROM account_write
|
||||||
INNER JOIN slot USING(slot)
|
LEFT JOIN slot USING(slot)
|
||||||
INNER JOIN pubkey USING(pubkey_id)
|
INNER JOIN pubkey USING(pubkey_id)
|
||||||
WHERE slot.status = 'Rooted'
|
CROSS JOIN (SELECT max(slot) FROM slot) ms
|
||||||
|
WHERE slot <= ms.max
|
||||||
|
AND (slot.status = 'Rooted' OR slot.status is NULL)
|
||||||
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
||||||
CREATE VIEW account_confirmed AS
|
CREATE VIEW account_confirmed AS
|
||||||
SELECT
|
SELECT
|
||||||
DISTINCT ON(pubkey_id)
|
DISTINCT ON(pubkey_id)
|
||||||
pubkey, account_write.*
|
pubkey, account_write.*
|
||||||
FROM account_write
|
FROM account_write
|
||||||
INNER JOIN slot USING(slot)
|
LEFT JOIN slot USING(slot)
|
||||||
INNER JOIN pubkey USING(pubkey_id)
|
INNER JOIN pubkey USING(pubkey_id)
|
||||||
WHERE (slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted'
|
CROSS JOIN (SELECT max(slot) FROM slot) ms
|
||||||
|
WHERE slot <= ms.max
|
||||||
|
AND ((slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
|
||||||
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
||||||
CREATE VIEW account_processed AS
|
CREATE VIEW account_processed AS
|
||||||
SELECT
|
SELECT
|
||||||
DISTINCT ON(pubkey_id)
|
DISTINCT ON(pubkey_id)
|
||||||
pubkey, account_write.*
|
pubkey, account_write.*
|
||||||
FROM account_write
|
FROM account_write
|
||||||
INNER JOIN slot USING(slot)
|
LEFT JOIN slot USING(slot)
|
||||||
INNER JOIN pubkey USING(pubkey_id)
|
INNER JOIN pubkey USING(pubkey_id)
|
||||||
WHERE ((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted'
|
CROSS JOIN (SELECT max(slot) FROM slot) ms
|
||||||
|
WHERE slot <= ms.max
|
||||||
|
AND (((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
|
||||||
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
||||||
|
|
||||||
|
|
||||||
|
@ -33,27 +39,33 @@ CREATE VIEW mango_account_rooted AS
|
||||||
DISTINCT ON(pubkey_id)
|
DISTINCT ON(pubkey_id)
|
||||||
pubkey, mango_account_write.*
|
pubkey, mango_account_write.*
|
||||||
FROM mango_account_write
|
FROM mango_account_write
|
||||||
INNER JOIN slot USING(slot)
|
LEFT JOIN slot USING(slot)
|
||||||
INNER JOIN pubkey using(pubkey_id)
|
INNER JOIN pubkey using(pubkey_id)
|
||||||
WHERE slot.status = 'Rooted'
|
CROSS JOIN (SELECT max(slot) FROM slot) ms
|
||||||
|
WHERE slot <= ms.max
|
||||||
|
AND (slot.status = 'Rooted' OR slot.status is NULL)
|
||||||
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
||||||
CREATE VIEW mango_account_confirmed AS
|
CREATE VIEW mango_account_confirmed AS
|
||||||
SELECT
|
SELECT
|
||||||
DISTINCT ON(pubkey_id)
|
DISTINCT ON(pubkey_id)
|
||||||
pubkey, mango_account_write.*
|
pubkey, mango_account_write.*
|
||||||
FROM mango_account_write
|
FROM mango_account_write
|
||||||
INNER JOIN slot USING(slot)
|
LEFT JOIN slot USING(slot)
|
||||||
INNER JOIN pubkey using(pubkey_id)
|
INNER JOIN pubkey using(pubkey_id)
|
||||||
WHERE (slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted'
|
CROSS JOIN (SELECT max(slot) FROM slot) ms
|
||||||
|
WHERE slot <= ms.max
|
||||||
|
AND ((slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
|
||||||
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
||||||
CREATE VIEW mango_account_processed AS
|
CREATE VIEW mango_account_processed AS
|
||||||
SELECT
|
SELECT
|
||||||
DISTINCT ON(pubkey_id)
|
DISTINCT ON(pubkey_id)
|
||||||
pubkey, mango_account_write.*
|
pubkey, mango_account_write.*
|
||||||
FROM mango_account_write
|
FROM mango_account_write
|
||||||
INNER JOIN slot USING(slot)
|
LEFT JOIN slot USING(slot)
|
||||||
INNER JOIN pubkey using(pubkey_id)
|
INNER JOIN pubkey using(pubkey_id)
|
||||||
WHERE ((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted'
|
CROSS JOIN (SELECT max(slot) FROM slot) ms
|
||||||
|
WHERE slot <= ms.max
|
||||||
|
AND (((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
|
||||||
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
||||||
|
|
||||||
CREATE VIEW mango_account_processed_balance AS
|
CREATE VIEW mango_account_processed_balance AS
|
||||||
|
@ -82,27 +94,33 @@ CREATE VIEW mango_group_rooted AS
|
||||||
DISTINCT ON(pubkey_id)
|
DISTINCT ON(pubkey_id)
|
||||||
pubkey, mango_group_write.*
|
pubkey, mango_group_write.*
|
||||||
FROM mango_group_write
|
FROM mango_group_write
|
||||||
INNER JOIN slot USING(slot)
|
LEFT JOIN slot USING(slot)
|
||||||
INNER JOIN pubkey using(pubkey_id)
|
INNER JOIN pubkey using(pubkey_id)
|
||||||
WHERE slot.status = 'Rooted'
|
CROSS JOIN (SELECT max(slot) FROM slot) ms
|
||||||
|
WHERE slot <= ms.max
|
||||||
|
AND (slot.status = 'Rooted' OR slot.status is NULL)
|
||||||
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
||||||
CREATE VIEW mango_group_confirmed AS
|
CREATE VIEW mango_group_confirmed AS
|
||||||
SELECT
|
SELECT
|
||||||
DISTINCT ON(pubkey_id)
|
DISTINCT ON(pubkey_id)
|
||||||
pubkey, mango_group_write.*
|
pubkey, mango_group_write.*
|
||||||
FROM mango_group_write
|
FROM mango_group_write
|
||||||
INNER JOIN slot USING(slot)
|
LEFT JOIN slot USING(slot)
|
||||||
INNER JOIN pubkey using(pubkey_id)
|
INNER JOIN pubkey using(pubkey_id)
|
||||||
WHERE (slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted'
|
CROSS JOIN (SELECT max(slot) FROM slot) ms
|
||||||
|
WHERE slot <= ms.max
|
||||||
|
AND ((slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
|
||||||
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
||||||
CREATE VIEW mango_group_processed AS
|
CREATE VIEW mango_group_processed AS
|
||||||
SELECT
|
SELECT
|
||||||
DISTINCT ON(pubkey_id)
|
DISTINCT ON(pubkey_id)
|
||||||
pubkey, mango_group_write.*
|
pubkey, mango_group_write.*
|
||||||
FROM mango_group_write
|
FROM mango_group_write
|
||||||
INNER JOIN slot USING(slot)
|
LEFT JOIN slot USING(slot)
|
||||||
INNER JOIN pubkey using(pubkey_id)
|
INNER JOIN pubkey using(pubkey_id)
|
||||||
WHERE ((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted'
|
CROSS JOIN (SELECT max(slot) FROM slot) ms
|
||||||
|
WHERE slot <= ms.max
|
||||||
|
AND (((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
|
||||||
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
||||||
|
|
||||||
CREATE VIEW mango_cache_rooted AS
|
CREATE VIEW mango_cache_rooted AS
|
||||||
|
@ -110,25 +128,31 @@ CREATE VIEW mango_cache_rooted AS
|
||||||
DISTINCT ON(pubkey_id)
|
DISTINCT ON(pubkey_id)
|
||||||
pubkey, mango_cache_write.*
|
pubkey, mango_cache_write.*
|
||||||
FROM mango_cache_write
|
FROM mango_cache_write
|
||||||
INNER JOIN slot USING(slot)
|
LEFT JOIN slot USING(slot)
|
||||||
INNER JOIN pubkey using(pubkey_id)
|
INNER JOIN pubkey using(pubkey_id)
|
||||||
WHERE slot.status = 'Rooted'
|
CROSS JOIN (SELECT max(slot) FROM slot) ms
|
||||||
|
WHERE slot <= ms.max
|
||||||
|
AND (slot.status = 'Rooted' OR slot.status is NULL)
|
||||||
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
||||||
CREATE VIEW mango_cache_confirmed AS
|
CREATE VIEW mango_cache_confirmed AS
|
||||||
SELECT
|
SELECT
|
||||||
DISTINCT ON(pubkey_id)
|
DISTINCT ON(pubkey_id)
|
||||||
pubkey, mango_cache_write.*
|
pubkey, mango_cache_write.*
|
||||||
FROM mango_cache_write
|
FROM mango_cache_write
|
||||||
INNER JOIN slot USING(slot)
|
LEFT JOIN slot USING(slot)
|
||||||
INNER JOIN pubkey using(pubkey_id)
|
INNER JOIN pubkey using(pubkey_id)
|
||||||
WHERE (slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted'
|
CROSS JOIN (SELECT max(slot) FROM slot) ms
|
||||||
|
WHERE slot <= ms.max
|
||||||
|
AND ((slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
|
||||||
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
||||||
CREATE VIEW mango_cache_processed AS
|
CREATE VIEW mango_cache_processed AS
|
||||||
SELECT
|
SELECT
|
||||||
DISTINCT ON(pubkey_id)
|
DISTINCT ON(pubkey_id)
|
||||||
pubkey, mango_cache_write.*
|
pubkey, mango_cache_write.*
|
||||||
FROM mango_cache_write
|
FROM mango_cache_write
|
||||||
INNER JOIN slot USING(slot)
|
LEFT JOIN slot USING(slot)
|
||||||
INNER JOIN pubkey using(pubkey_id)
|
INNER JOIN pubkey using(pubkey_id)
|
||||||
WHERE ((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted'
|
CROSS JOIN (SELECT max(slot) FROM slot) ms
|
||||||
|
WHERE slot <= ms.max
|
||||||
|
AND (((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
|
||||||
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
|
@ -2,13 +2,13 @@ DROP VIEW account_rooted;
|
||||||
DROP VIEW account_confirmed;
|
DROP VIEW account_confirmed;
|
||||||
DROP VIEW account_processed;
|
DROP VIEW account_processed;
|
||||||
|
|
||||||
|
DROP VIEW mango_account_processed_balance;
|
||||||
|
DROP VIEW mango_account_processed_perp;
|
||||||
|
|
||||||
DROP VIEW mango_account_rooted;
|
DROP VIEW mango_account_rooted;
|
||||||
DROP VIEW mango_account_confirmed;
|
DROP VIEW mango_account_confirmed;
|
||||||
DROP VIEW mango_account_processed;
|
DROP VIEW mango_account_processed;
|
||||||
|
|
||||||
DROP VIEW mango_account_processed_balance;
|
|
||||||
DROP VIEW mango_account_processed_perp;
|
|
||||||
|
|
||||||
DROP VIEW mango_group_rooted;
|
DROP VIEW mango_group_rooted;
|
||||||
DROP VIEW mango_group_confirmed;
|
DROP VIEW mango_group_confirmed;
|
||||||
DROP VIEW mango_group_processed;
|
DROP VIEW mango_group_processed;
|
||||||
|
|
|
@ -4,25 +4,31 @@ CREATE VIEW account_rooted AS
|
||||||
DISTINCT ON(pubkey_id)
|
DISTINCT ON(pubkey_id)
|
||||||
pubkey, account_write.*
|
pubkey, account_write.*
|
||||||
FROM account_write
|
FROM account_write
|
||||||
INNER JOIN slot USING(slot)
|
LEFT JOIN slot USING(slot)
|
||||||
INNER JOIN pubkey USING(pubkey_id)
|
INNER JOIN pubkey USING(pubkey_id)
|
||||||
WHERE slot.status = 'Rooted'
|
CROSS JOIN (SELECT max(slot) FROM slot) ms
|
||||||
|
WHERE slot <= ms.max
|
||||||
|
AND (slot.status = 'Rooted' OR slot.status is NULL)
|
||||||
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
||||||
CREATE VIEW account_confirmed AS
|
CREATE VIEW account_confirmed AS
|
||||||
SELECT
|
SELECT
|
||||||
DISTINCT ON(pubkey_id)
|
DISTINCT ON(pubkey_id)
|
||||||
pubkey, account_write.*
|
pubkey, account_write.*
|
||||||
FROM account_write
|
FROM account_write
|
||||||
INNER JOIN slot USING(slot)
|
LEFT JOIN slot USING(slot)
|
||||||
INNER JOIN pubkey USING(pubkey_id)
|
INNER JOIN pubkey USING(pubkey_id)
|
||||||
WHERE (slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted'
|
CROSS JOIN (SELECT max(slot) FROM slot) ms
|
||||||
|
WHERE slot <= ms.max
|
||||||
|
AND ((slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
|
||||||
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
||||||
CREATE VIEW account_processed AS
|
CREATE VIEW account_processed AS
|
||||||
SELECT
|
SELECT
|
||||||
DISTINCT ON(pubkey_id)
|
DISTINCT ON(pubkey_id)
|
||||||
pubkey, account_write.*
|
pubkey, account_write.*
|
||||||
FROM account_write
|
FROM account_write
|
||||||
INNER JOIN slot USING(slot)
|
LEFT JOIN slot USING(slot)
|
||||||
INNER JOIN pubkey USING(pubkey_id)
|
INNER JOIN pubkey USING(pubkey_id)
|
||||||
WHERE ((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted'
|
CROSS JOIN (SELECT max(slot) FROM slot) ms
|
||||||
|
WHERE slot <= ms.max
|
||||||
|
AND (((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
|
||||||
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
ORDER BY pubkey_id, slot DESC, write_version DESC;
|
||||||
|
|
|
@ -171,6 +171,10 @@ struct SlotsProcessing {
|
||||||
|
|
||||||
impl SlotsProcessing {
|
impl SlotsProcessing {
|
||||||
fn new(tables: &Vec<String>) -> Self {
|
fn new(tables: &Vec<String>) -> Self {
|
||||||
|
// Delete:
|
||||||
|
// 1. account writes that came before the newest rooted write
|
||||||
|
// 2. account writes that came after the newest rooted write but before
|
||||||
|
// the newest rooted slot (like processed writes that never confirmed)
|
||||||
let mut cleanup_table_sql: Vec<String> = tables
|
let mut cleanup_table_sql: Vec<String> = tables
|
||||||
.iter()
|
.iter()
|
||||||
.map(|table_name| {
|
.map(|table_name| {
|
||||||
|
@ -179,14 +183,26 @@ impl SlotsProcessing {
|
||||||
USING (
|
USING (
|
||||||
SELECT DISTINCT ON(pubkey_id) pubkey_id, slot, write_version
|
SELECT DISTINCT ON(pubkey_id) pubkey_id, slot, write_version
|
||||||
FROM {table}
|
FROM {table}
|
||||||
INNER JOIN slot USING(slot)
|
LEFT JOIN slot USING(slot)
|
||||||
WHERE slot <= $newest_final_slot AND status = 'Rooted'
|
WHERE slot <= $newest_final_slot AND (status = 'Rooted' OR status is NULL)
|
||||||
ORDER BY pubkey_id, slot DESC, write_version DESC
|
ORDER BY pubkey_id, slot DESC, write_version DESC
|
||||||
) latest_write
|
) latest_write
|
||||||
WHERE data.pubkey_id = latest_write.pubkey_id
|
WHERE data.pubkey_id = latest_write.pubkey_id
|
||||||
AND (data.slot < latest_write.slot
|
AND (
|
||||||
OR (data.slot = latest_write.slot
|
(data.slot < latest_write.slot
|
||||||
AND data.write_version < latest_write.write_version
|
OR (data.slot = latest_write.slot
|
||||||
|
AND data.write_version < latest_write.write_version
|
||||||
|
)
|
||||||
|
)
|
||||||
|
OR
|
||||||
|
(
|
||||||
|
data.slot < $newest_final_slot
|
||||||
|
AND
|
||||||
|
(data.slot > latest_write.slot
|
||||||
|
OR (data.slot = latest_write.slot
|
||||||
|
AND data.write_version > latest_write.write_version
|
||||||
|
)
|
||||||
|
)
|
||||||
)
|
)
|
||||||
)",
|
)",
|
||||||
table = table_name
|
table = table_name
|
||||||
|
@ -194,6 +210,7 @@ impl SlotsProcessing {
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
|
// Delete old slots
|
||||||
cleanup_table_sql.push("DELETE FROM slot WHERE slot + 100000 < $newest_final_slot".into());
|
cleanup_table_sql.push("DELETE FROM slot WHERE slot + 100000 < $newest_final_slot".into());
|
||||||
|
|
||||||
Self { cleanup_table_sql }
|
Self { cleanup_table_sql }
|
||||||
|
|
Loading…
Reference in New Issue