csv - binary comparison
Signed-off-by: microwavedcola1 <microwavedcola@gmail.com>
This commit is contained in:
microwavedcola1
parent
2af0718a09
commit
f409f10f4a
Binary file not shown.
File diff suppressed because it is too large
Load Diff
|
|
@ -24,6 +24,7 @@
|
|||
"@types/chai": "^4.3.0",
|
||||
"@types/mocha": "^9.0.0",
|
||||
"chai": "^4.3.4",
|
||||
"fast-csv": "^4.3.6",
|
||||
"mocha": "^9.0.3",
|
||||
"prettier": "^2.6.2",
|
||||
"ts-mocha": "^10.0.0",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,151 @@
|
|||
import fs from "fs";
|
||||
import * as path from "path";
|
||||
import * as csv from "fast-csv";
|
||||
import {
|
||||
Cluster,
|
||||
Config,
|
||||
MangoClient,
|
||||
} from "@blockworks-foundation/mango-client";
|
||||
import { Connection, Keypair, PublicKey } from "@solana/web3.js";
|
||||
import { Wallet, AnchorProvider, Program } from "@project-serum/anchor";
|
||||
import { ID, MangoV3ReimbursementClient } from "./client";
|
||||
import { MangoV3Reimbursement, IDL } from "./mango_v3_reimbursement";
|
||||
|
||||
/// Env
|
||||
const CLUSTER_URL =
|
||||
process.env.CLUSTER_URL_OVERRIDE || process.env.MB_CLUSTER_URL;
|
||||
const PAYER_KEYPAIR =
|
||||
process.env.PAYER_KEYPAIR_OVERRIDE || process.env.MB_PAYER_KEYPAIR;
|
||||
const GROUP_NUM = Number(process.env.GROUP_NUM || 20);
|
||||
const MANGO_V3_CLUSTER: Cluster =
|
||||
(process.env.MANGO_V3_CLUSTER_OVERRIDE as Cluster) || "mainnet";
|
||||
const MANGO_V3_GROUP_NAME: any =
|
||||
(process.env.MANGO_V3_GROUP_NAME_OVERRIDE as Cluster) || "mainnet.1";
|
||||
|
||||
const options = AnchorProvider.defaultOptions();
|
||||
const connection = new Connection(CLUSTER_URL!, options);
|
||||
|
||||
// Mango v3 client setup
|
||||
const config = Config.ids();
|
||||
const groupIds = config.getGroup(MANGO_V3_CLUSTER, MANGO_V3_GROUP_NAME);
|
||||
if (!groupIds) {
|
||||
throw new Error(`Group ${MANGO_V3_GROUP_NAME} not found`);
|
||||
}
|
||||
const mangoProgramId = groupIds.mangoProgramId;
|
||||
const mangoGroupKey = groupIds.publicKey;
|
||||
const mangoV3Client = new MangoClient(connection, mangoProgramId);
|
||||
|
||||
async function main() {
|
||||
// Load IDL for binary data decoding
|
||||
const admin = Keypair.fromSecretKey(
|
||||
Buffer.from(JSON.parse(fs.readFileSync(PAYER_KEYPAIR!, "utf-8")))
|
||||
);
|
||||
const adminWallet = new Wallet(admin);
|
||||
const provider = new AnchorProvider(connection, adminWallet, options);
|
||||
const program = new Program<MangoV3Reimbursement>(
|
||||
IDL as MangoV3Reimbursement,
|
||||
ID,
|
||||
provider
|
||||
);
|
||||
|
||||
const rowsFromCsv = [];
|
||||
fs.createReadStream(
|
||||
path.resolve(
|
||||
__dirname,
|
||||
"assets",
|
||||
"../../../../assets/reimbursement-20221018-v8.csv"
|
||||
)
|
||||
)
|
||||
.pipe(csv.parse({ headers: true }))
|
||||
.on("data", (row) => rowsFromCsv.push(row))
|
||||
.on("end", async (rowCount: number) => {
|
||||
if (rowCount != rowsFromCsv.length) {
|
||||
throw new Error("Error in aggregating all rows from the csv!");
|
||||
}
|
||||
console.log(`Loaded ${rowCount} rows from csv file`);
|
||||
|
||||
//
|
||||
// Collect accounts together, map is indexed by owner
|
||||
//
|
||||
const rowsFromCsvMapByOwner = new Map();
|
||||
for (const row of rowsFromCsv) {
|
||||
!rowsFromCsvMapByOwner.get(row.owner) &&
|
||||
rowsFromCsvMapByOwner.set(row.owner, []);
|
||||
rowsFromCsvMapByOwner.get(row.owner).push(row);
|
||||
}
|
||||
console.log(
|
||||
`Created map for ${
|
||||
Array.from(rowsFromCsvMapByOwner.keys()).length
|
||||
} owner from csv file`
|
||||
);
|
||||
|
||||
//
|
||||
// Read table
|
||||
//
|
||||
var table = fs.readFileSync(
|
||||
"/Users/mc/repos/mango-v3-reimbursement/assets/reimb-mixed-v8.binary"
|
||||
);
|
||||
const rowSize = 160;
|
||||
const tableHeaderSize = 3;
|
||||
const rowsFromBinaryTable = [
|
||||
...Array((table.length - tableHeaderSize) / rowSize).keys(),
|
||||
].map((i) => {
|
||||
const start = tableHeaderSize + i * rowSize;
|
||||
const end = start + rowSize;
|
||||
return (program as any)._coder.types.typeLayouts
|
||||
.get("Row")
|
||||
.decode(table.subarray(start, end));
|
||||
});
|
||||
console.log(`Loaded ${rowsFromBinaryTable.length} rows from binary file`);
|
||||
|
||||
const groupTokens = (await mangoV3Client.getMangoGroup(mangoGroupKey))
|
||||
.tokens;
|
||||
//
|
||||
// Compare for a owner
|
||||
//
|
||||
let errorCount = 0;
|
||||
for (const rowFromBinaryTable of rowsFromBinaryTable) {
|
||||
const balancesFromBinaryTable = rowFromBinaryTable.balances;
|
||||
const accountsFromCsv = rowsFromCsvMapByOwner.get(
|
||||
rowFromBinaryTable.owner.toBase58()
|
||||
);
|
||||
|
||||
Array.from(groupTokens.entries())
|
||||
.filter((entry) => entry[1].oracleInactive === false)
|
||||
.filter((entry) => !entry[1].mint.equals(PublicKey.default))
|
||||
.forEach((entry) => {
|
||||
const tokenIndex = entry[0];
|
||||
|
||||
const symbol = groupIds?.tokens.find((token) =>
|
||||
token.mintKey.equals(entry[1].mint)
|
||||
).symbol;
|
||||
|
||||
const balanceFromBinaryTable =
|
||||
balancesFromBinaryTable[tokenIndex].toNumber();
|
||||
|
||||
const sumOfbalancesAcrossAccountsFromBinaryCsv = accountsFromCsv
|
||||
.map((account) => account[symbol])
|
||||
.reduce((a, b) => {
|
||||
return a + parseInt(b);
|
||||
}, 0);
|
||||
|
||||
if (
|
||||
balanceFromBinaryTable !==
|
||||
sumOfbalancesAcrossAccountsFromBinaryCsv
|
||||
) {
|
||||
console.error(
|
||||
`Error: Entries mismatched for ${symbol} ${rowFromBinaryTable.owner} ${balanceFromBinaryTable} ${sumOfbalancesAcrossAccountsFromBinaryCsv}`
|
||||
);
|
||||
errorCount += 1;
|
||||
} else {
|
||||
// console.log(
|
||||
// `Success: Entries matched for ${symbol} ${rowFromBinaryTable.owner} ${balanceFromBinaryTable} ${sumOfbalancesAcrossAccountsFromBinaryCsv}`
|
||||
// );
|
||||
}
|
||||
});
|
||||
}
|
||||
console.log(`Finished comparing errorCount - ${errorCount}!`);
|
||||
});
|
||||
}
|
||||
|
||||
main();
|
||||
78
yarn.lock
78
yarn.lock
|
|
@ -27,6 +27,31 @@
|
|||
toformat "^2.0.0"
|
||||
yargs "^17.0.1"
|
||||
|
||||
"@fast-csv/format@4.3.5":
|
||||
version "4.3.5"
|
||||
resolved "https://registry.yarnpkg.com/@fast-csv/format/-/format-4.3.5.tgz#90d83d1b47b6aaf67be70d6118f84f3e12ee1ff3"
|
||||
integrity sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==
|
||||
dependencies:
|
||||
"@types/node" "^14.0.1"
|
||||
lodash.escaperegexp "^4.1.2"
|
||||
lodash.isboolean "^3.0.3"
|
||||
lodash.isequal "^4.5.0"
|
||||
lodash.isfunction "^3.0.9"
|
||||
lodash.isnil "^4.0.0"
|
||||
|
||||
"@fast-csv/parse@4.3.6":
|
||||
version "4.3.6"
|
||||
resolved "https://registry.yarnpkg.com/@fast-csv/parse/-/parse-4.3.6.tgz#ee47d0640ca0291034c7aa94039a744cfb019264"
|
||||
integrity sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA==
|
||||
dependencies:
|
||||
"@types/node" "^14.0.1"
|
||||
lodash.escaperegexp "^4.1.2"
|
||||
lodash.groupby "^4.6.0"
|
||||
lodash.isfunction "^3.0.9"
|
||||
lodash.isnil "^4.0.0"
|
||||
lodash.isundefined "^3.0.1"
|
||||
lodash.uniq "^4.5.0"
|
||||
|
||||
"@noble/ed25519@^1.7.0":
|
||||
version "1.7.1"
|
||||
resolved "https://registry.yarnpkg.com/@noble/ed25519/-/ed25519-1.7.1.tgz#6899660f6fbb97798a6fbd227227c4589a454724"
|
||||
|
|
@ -271,6 +296,11 @@
|
|||
resolved "https://registry.yarnpkg.com/@types/node/-/node-12.20.55.tgz#c329cbd434c42164f846b909bd6f85b5537f6240"
|
||||
integrity sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==
|
||||
|
||||
"@types/node@^14.0.1":
|
||||
version "14.18.32"
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.32.tgz#8074f7106731f1a12ba993fe8bad86ee73905014"
|
||||
integrity sha512-Y6S38pFr04yb13qqHf8uk1nHE3lXgQ30WZbv1mLliV9pt0NjvqdWttLcrOYLnXbOafknVYRHZGoMSpR9UwfYow==
|
||||
|
||||
"@types/ws@^7.4.4":
|
||||
version "7.4.7"
|
||||
resolved "https://registry.yarnpkg.com/@types/ws/-/ws-7.4.7.tgz#f7c390a36f7a0679aa69de2d501319f4f8d9b702"
|
||||
|
|
@ -638,6 +668,14 @@ eyes@^0.1.8:
|
|||
resolved "https://registry.yarnpkg.com/eyes/-/eyes-0.1.8.tgz#62cf120234c683785d902348a800ef3e0cc20bc0"
|
||||
integrity sha512-GipyPsXO1anza0AOZdy69Im7hGFCNB7Y/NGjDlZGJ3GJJLtwNSb2vrzYrTYJRrRloVx7pl+bhUaTB8yiccPvFQ==
|
||||
|
||||
fast-csv@^4.3.6:
|
||||
version "4.3.6"
|
||||
resolved "https://registry.yarnpkg.com/fast-csv/-/fast-csv-4.3.6.tgz#70349bdd8fe4d66b1130d8c91820b64a21bc4a63"
|
||||
integrity sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw==
|
||||
dependencies:
|
||||
"@fast-csv/format" "4.3.5"
|
||||
"@fast-csv/parse" "4.3.6"
|
||||
|
||||
fast-stable-stringify@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/fast-stable-stringify/-/fast-stable-stringify-1.0.0.tgz#5c5543462b22aeeefd36d05b34e51c78cb86d313"
|
||||
|
|
@ -851,6 +889,46 @@ locate-path@^6.0.0:
|
|||
dependencies:
|
||||
p-locate "^5.0.0"
|
||||
|
||||
lodash.escaperegexp@^4.1.2:
|
||||
version "4.1.2"
|
||||
resolved "https://registry.yarnpkg.com/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz#64762c48618082518ac3df4ccf5d5886dae20347"
|
||||
integrity sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==
|
||||
|
||||
lodash.groupby@^4.6.0:
|
||||
version "4.6.0"
|
||||
resolved "https://registry.yarnpkg.com/lodash.groupby/-/lodash.groupby-4.6.0.tgz#0b08a1dcf68397c397855c3239783832df7403d1"
|
||||
integrity sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==
|
||||
|
||||
lodash.isboolean@^3.0.3:
|
||||
version "3.0.3"
|
||||
resolved "https://registry.yarnpkg.com/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6"
|
||||
integrity sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==
|
||||
|
||||
lodash.isequal@^4.5.0:
|
||||
version "4.5.0"
|
||||
resolved "https://registry.yarnpkg.com/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0"
|
||||
integrity sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==
|
||||
|
||||
lodash.isfunction@^3.0.9:
|
||||
version "3.0.9"
|
||||
resolved "https://registry.yarnpkg.com/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz#06de25df4db327ac931981d1bdb067e5af68d051"
|
||||
integrity sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==
|
||||
|
||||
lodash.isnil@^4.0.0:
|
||||
version "4.0.0"
|
||||
resolved "https://registry.yarnpkg.com/lodash.isnil/-/lodash.isnil-4.0.0.tgz#49e28cd559013458c814c5479d3c663a21bfaa6c"
|
||||
integrity sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==
|
||||
|
||||
lodash.isundefined@^3.0.1:
|
||||
version "3.0.1"
|
||||
resolved "https://registry.yarnpkg.com/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz#23ef3d9535565203a66cefd5b830f848911afb48"
|
||||
integrity sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==
|
||||
|
||||
lodash.uniq@^4.5.0:
|
||||
version "4.5.0"
|
||||
resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
|
||||
integrity sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==
|
||||
|
||||
lodash@^4.17.20:
|
||||
version "4.17.21"
|
||||
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
|
||||
|
|
|
|||
Loading…
Reference in New Issue