blockworks-foundation
/
solana-accountsdb-connector
mirror of https://github.com/blockworks-foundation/solana-accountsdb-connector.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix bug introduced by deleting older slots 

Deleting older slots meant that older (but still most recent) account
writes that had their slot deleted would no longer show up in views.

Now the guarantee is that an account write for a missing slot that is
smaller than the top slot in the slots table is rooted.

To guarantee that, account writes for non-rooted slots that preceed
rooted slots are deleted now.
This commit is contained in:
Christian Kamm 2021-11-25 10:24:06 +01:00
parent 3cdbeddf8e
commit a941f486ed
5 changed files with 88 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -147,9 +147,9 @@ Example for querying confirmed data:
SELECT DISTINCT ON(pubkey_id)
pubkey, account_write.*
FROM account_write
INNER JOIN slot USING(slot)
LEFT JOIN slot USING(slot)
INNER JOIN pubkey USING(pubkey_id)
WHERE status = 'Rooted' OR (uncle = FALSE AND status = 'Confirmed')
WHERE status = 'Rooted' OR status IS NULL OR (uncle = FALSE AND status = 'Confirmed')
ORDER BY pubkey_id, slot DESC, write_version DESC;
```

74
connector-mango/scripts/create_views.sql
View File

@ -4,27 +4,33 @@ CREATE VIEW account_rooted AS
DISTINCT ON(pubkey_id)
pubkey, account_write.*
FROM account_write
INNER JOIN slot USING(slot)
LEFT JOIN slot USING(slot)
INNER JOIN pubkey USING(pubkey_id)
WHERE slot.status = 'Rooted'
CROSS JOIN (SELECT max(slot) FROM slot) ms
WHERE slot <= ms.max
AND (slot.status = 'Rooted' OR slot.status is NULL)
ORDER BY pubkey_id, slot DESC, write_version DESC;
CREATE VIEW account_confirmed AS
SELECT
DISTINCT ON(pubkey_id)
pubkey, account_write.*
FROM account_write
INNER JOIN slot USING(slot)
LEFT JOIN slot USING(slot)
INNER JOIN pubkey USING(pubkey_id)
WHERE (slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted'
CROSS JOIN (SELECT max(slot) FROM slot) ms
WHERE slot <= ms.max
AND ((slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
ORDER BY pubkey_id, slot DESC, write_version DESC;
CREATE VIEW account_processed AS
SELECT
DISTINCT ON(pubkey_id)
pubkey, account_write.*
FROM account_write
INNER JOIN slot USING(slot)
LEFT JOIN slot USING(slot)
INNER JOIN pubkey USING(pubkey_id)
WHERE ((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted'
CROSS JOIN (SELECT max(slot) FROM slot) ms
WHERE slot <= ms.max
AND (((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
ORDER BY pubkey_id, slot DESC, write_version DESC;
@ -33,27 +39,33 @@ CREATE VIEW mango_account_rooted AS
DISTINCT ON(pubkey_id)
pubkey, mango_account_write.*
FROM mango_account_write
INNER JOIN slot USING(slot)
LEFT JOIN slot USING(slot)
INNER JOIN pubkey using(pubkey_id)
WHERE slot.status = 'Rooted'
CROSS JOIN (SELECT max(slot) FROM slot) ms
WHERE slot <= ms.max
AND (slot.status = 'Rooted' OR slot.status is NULL)
ORDER BY pubkey_id, slot DESC, write_version DESC;
CREATE VIEW mango_account_confirmed AS
SELECT
DISTINCT ON(pubkey_id)
pubkey, mango_account_write.*
FROM mango_account_write
INNER JOIN slot USING(slot)
LEFT JOIN slot USING(slot)
INNER JOIN pubkey using(pubkey_id)
WHERE (slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted'
CROSS JOIN (SELECT max(slot) FROM slot) ms
WHERE slot <= ms.max
AND ((slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
ORDER BY pubkey_id, slot DESC, write_version DESC;
CREATE VIEW mango_account_processed AS
SELECT
DISTINCT ON(pubkey_id)
pubkey, mango_account_write.*
FROM mango_account_write
INNER JOIN slot USING(slot)
LEFT JOIN slot USING(slot)
INNER JOIN pubkey using(pubkey_id)
WHERE ((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted'
CROSS JOIN (SELECT max(slot) FROM slot) ms
WHERE slot <= ms.max
AND (((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
ORDER BY pubkey_id, slot DESC, write_version DESC;
CREATE VIEW mango_account_processed_balance AS
@ -82,27 +94,33 @@ CREATE VIEW mango_group_rooted AS
DISTINCT ON(pubkey_id)
pubkey, mango_group_write.*
FROM mango_group_write
INNER JOIN slot USING(slot)
LEFT JOIN slot USING(slot)
INNER JOIN pubkey using(pubkey_id)
WHERE slot.status = 'Rooted'
CROSS JOIN (SELECT max(slot) FROM slot) ms
WHERE slot <= ms.max
AND (slot.status = 'Rooted' OR slot.status is NULL)
ORDER BY pubkey_id, slot DESC, write_version DESC;
CREATE VIEW mango_group_confirmed AS
SELECT
DISTINCT ON(pubkey_id)
pubkey, mango_group_write.*
FROM mango_group_write
INNER JOIN slot USING(slot)
LEFT JOIN slot USING(slot)
INNER JOIN pubkey using(pubkey_id)
WHERE (slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted'
CROSS JOIN (SELECT max(slot) FROM slot) ms
WHERE slot <= ms.max
AND ((slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
ORDER BY pubkey_id, slot DESC, write_version DESC;
CREATE VIEW mango_group_processed AS
SELECT
DISTINCT ON(pubkey_id)
pubkey, mango_group_write.*
FROM mango_group_write
INNER JOIN slot USING(slot)
LEFT JOIN slot USING(slot)
INNER JOIN pubkey using(pubkey_id)
WHERE ((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted'
CROSS JOIN (SELECT max(slot) FROM slot) ms
WHERE slot <= ms.max
AND (((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
ORDER BY pubkey_id, slot DESC, write_version DESC;
CREATE VIEW mango_cache_rooted AS
@ -110,25 +128,31 @@ CREATE VIEW mango_cache_rooted AS
DISTINCT ON(pubkey_id)
pubkey, mango_cache_write.*
FROM mango_cache_write
INNER JOIN slot USING(slot)
LEFT JOIN slot USING(slot)
INNER JOIN pubkey using(pubkey_id)
WHERE slot.status = 'Rooted'
CROSS JOIN (SELECT max(slot) FROM slot) ms
WHERE slot <= ms.max
AND (slot.status = 'Rooted' OR slot.status is NULL)
ORDER BY pubkey_id, slot DESC, write_version DESC;
CREATE VIEW mango_cache_confirmed AS
SELECT
DISTINCT ON(pubkey_id)
pubkey, mango_cache_write.*
FROM mango_cache_write
INNER JOIN slot USING(slot)
LEFT JOIN slot USING(slot)
INNER JOIN pubkey using(pubkey_id)
WHERE (slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted'
CROSS JOIN (SELECT max(slot) FROM slot) ms
WHERE slot <= ms.max
AND ((slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
ORDER BY pubkey_id, slot DESC, write_version DESC;
CREATE VIEW mango_cache_processed AS
SELECT
DISTINCT ON(pubkey_id)
pubkey, mango_cache_write.*
FROM mango_cache_write
INNER JOIN slot USING(slot)
LEFT JOIN slot USING(slot)
INNER JOIN pubkey using(pubkey_id)
WHERE ((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted'
ORDER BY pubkey_id, slot DESC, write_version DESC;
CROSS JOIN (SELECT max(slot) FROM slot) ms
WHERE slot <= ms.max
AND (((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
ORDER BY pubkey_id, slot DESC, write_version DESC;

6
connector-mango/scripts/drop_views.sql
View File

@ -2,13 +2,13 @@ DROP VIEW account_rooted;
DROP VIEW account_confirmed;
DROP VIEW account_processed;
DROP VIEW mango_account_processed_balance;
DROP VIEW mango_account_processed_perp;
DROP VIEW mango_account_rooted;
DROP VIEW mango_account_confirmed;
DROP VIEW mango_account_processed;
DROP VIEW mango_account_processed_balance;
DROP VIEW mango_account_processed_perp;
DROP VIEW mango_group_rooted;
DROP VIEW mango_group_confirmed;
DROP VIEW mango_group_processed;

18
connector-raw/scripts/create_views.sql
View File

@ -4,25 +4,31 @@ CREATE VIEW account_rooted AS
DISTINCT ON(pubkey_id)
pubkey, account_write.*
FROM account_write
INNER JOIN slot USING(slot)
LEFT JOIN slot USING(slot)
INNER JOIN pubkey USING(pubkey_id)
WHERE slot.status = 'Rooted'
CROSS JOIN (SELECT max(slot) FROM slot) ms
WHERE slot <= ms.max
AND (slot.status = 'Rooted' OR slot.status is NULL)
ORDER BY pubkey_id, slot DESC, write_version DESC;
CREATE VIEW account_confirmed AS
SELECT
DISTINCT ON(pubkey_id)
pubkey, account_write.*
FROM account_write
INNER JOIN slot USING(slot)
LEFT JOIN slot USING(slot)
INNER JOIN pubkey USING(pubkey_id)
WHERE (slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted'
CROSS JOIN (SELECT max(slot) FROM slot) ms
WHERE slot <= ms.max
AND ((slot.status = 'Confirmed' AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
ORDER BY pubkey_id, slot DESC, write_version DESC;
CREATE VIEW account_processed AS
SELECT
DISTINCT ON(pubkey_id)
pubkey, account_write.*
FROM account_write
INNER JOIN slot USING(slot)
LEFT JOIN slot USING(slot)
INNER JOIN pubkey USING(pubkey_id)
WHERE ((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted'
CROSS JOIN (SELECT max(slot) FROM slot) ms
WHERE slot <= ms.max
AND (((slot.status = 'Confirmed' OR slot.status = 'Processed') AND NOT slot.uncle) OR slot.status = 'Rooted' OR slot.status is NULL)
ORDER BY pubkey_id, slot DESC, write_version DESC;

27
lib/src/postgres_target.rs
View File

@ -171,6 +171,10 @@ struct SlotsProcessing {
impl SlotsProcessing {
fn new(tables: &Vec<String>) -> Self {
// Delete:
// 1. account writes that came before the newest rooted write
// 2. account writes that came after the newest rooted write but before
// the newest rooted slot (like processed writes that never confirmed)
let mut cleanup_table_sql: Vec<String> = tables
.iter()
.map(|table_name| {
@ -179,14 +183,26 @@ impl SlotsProcessing {
USING (
SELECT DISTINCT ON(pubkey_id) pubkey_id, slot, write_version
FROM {table}
INNER JOIN slot USING(slot)
WHERE slot <= $newest_final_slot AND status = 'Rooted'
LEFT JOIN slot USING(slot)
WHERE slot <= $newest_final_slot AND (status = 'Rooted' OR status is NULL)
ORDER BY pubkey_id, slot DESC, write_version DESC
) latest_write
WHERE data.pubkey_id = latest_write.pubkey_id
AND (data.slot < latest_write.slot
OR (data.slot = latest_write.slot
AND data.write_version < latest_write.write_version
AND (
(data.slot < latest_write.slot
OR (data.slot = latest_write.slot
AND data.write_version < latest_write.write_version
)
)
OR
(
data.slot < $newest_final_slot
AND
(data.slot > latest_write.slot
OR (data.slot = latest_write.slot
AND data.write_version > latest_write.write_version
)
)
)
)",
table = table_name
@ -194,6 +210,7 @@ impl SlotsProcessing {
})
.collect();
// Delete old slots
cleanup_table_sql.push("DELETE FROM slot WHERE slot + 100000 < $newest_final_slot".into());
Self { cleanup_table_sql }