btc-spv transaction parsing (#5858)

* Transaction and input parsing/decoding + utils

* Transaction input & output parsing

* public struct members, tx parsing test

* format and clippy fixes

* update block data/test material fetching utils

* update tx parsing tests

* format changes

* rename for consistency

programs/btc_spv_api/Cargo.toml

@@ -17,6 +17,7 @@ num-traits = "0.2"
 serde = "1.0.100"
 serde_derive = "1.0.100"
 solana-sdk = { path = "../../sdk", version = "0.19.0-pre0"}
+hex = "0.3.2"
 
 [lib]
 crate-type = ["lib"]

programs/btc_spv_api/src/spv_instruction.rs

@@ -57,7 +57,7 @@ pub fn submit_proof(
     submitter: &Pubkey,
     proof: MerkleProof,
     headers: HeaderChain,
-    transaction: Transaction,
+    transaction: BitcoinTransaction,
     request: &Pubkey,
 ) -> Instruction {
     let account_meta = vec![

programs/btc_spv_api/src/spv_processor.rs

@@ -3,7 +3,8 @@
 use crate::spv_instruction::*;
 use crate::spv_state::*;
 #[allow(unused_imports)]
-use crate::utils::decode_hex;
+use crate::utils::*;
+use hex;
 use log::*;
 use solana_sdk::account::KeyedAccount;
 use solana_sdk::instruction::InstructionError;
@@ -129,8 +130,8 @@ mod test {
     fn test_parse_header_hex() -> Result<(), SpvError> {
         let testheader = "010000008a730974ac39042e95f82d719550e224c1a680a8dc9e8df9d007000000000000f50b20e8720a552dd36eb2ebdb7dceec9569e0395c990c1eb8a4292eeda05a931e1fce4e9a110e1a7a58aeb0";
         let testhash = "0000000000000bae09a7a393a8acded75aa67e46cb81f7acaa5ad94f9eacd103";
-        let testheaderbytes = decode_hex(&testheader)?;
-        let testhashbytes = decode_hex(&testhash)?;
+        let testheaderbytes = hex::decode(&testheader)?;
+        let testhashbytes = hex::decode(&testhash)?;
 
         let mut blockhash: [u8; 32] = [0; 32];
         blockhash.copy_from_slice(&testhashbytes[..32]);
@@ -163,4 +164,27 @@ mod test {
 
         Ok(())
     }
+
+    #[test]
+    fn test_parse_transaction_hex() {
+        let testblockhash = "0000000000000bae09a7a393a8acded75aa67e46cb81f7acaa5ad94f9eacd103";
+        let testtxhash = "5b09bbb8d3cb2f8d4edbcf30664419fb7c9deaeeb1f62cb432e7741c80dbe5ba";
+
+        let mut testdatabytes = include_bytes!("testblock.in");
+        let mut headerbytes = hex::encode(&testdatabytes[0..]);
+        let hbc = &headerbytes[0..80];
+
+        let mut txdata = &testdatabytes[80..];
+
+        let vilen = measure_variable_int(&txdata[0..9]).unwrap();
+        let txnum = decode_variable_int(&txdata[0..9]).unwrap();
+
+        txdata = &txdata[vilen..];
+        let tx = BitcoinTransaction::new(txdata.to_vec());
+
+        assert_eq!(tx.inputs.len(), 1);
+        assert_eq!(txnum, 22);
+        assert_eq!(tx.outputs.len(), 1);
+        assert_eq!(tx.version, 1);
+    }
 }

programs/btc_spv_api/src/spv_state.rs

@@ -71,9 +71,9 @@ impl BlockHeader {
             return Err(SpvError::InvalidBlockHeader);
         }
 
-        match decode_hex(header) {
+        match hex::decode(header) {
             Ok(header) => {
-                let bhbytes = decode_hex(blockhash)?;
+                let bhbytes = hex::decode(blockhash)?;
                 const SIZE: usize = 80;
                 let mut hh = [0; SIZE];
                 hh.copy_from_slice(&header[..header.len()]);
@@ -95,33 +95,142 @@ impl BlockHeader {
 }
 
 #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
-pub struct Transaction {
-    inputs: Vec<Input>,
+pub struct BitcoinTransaction {
+    pub inputs: Vec<Input>,
+
+    pub inputs_num: u64,
     //input utxos
-    outputs: Vec<Output>,
+    pub outputs: Vec<Output>,
+
+    pub outputs_num: u64,
     //output utxos
-    version: u32,
+    pub version: u32,
     //bitcoin network version
-    locktime: u32,
+    pub lock_time: u32,
+
+    pub bytes_len: usize,
 }
 
-// impl Transaction {
-//     fn new(bytes: Vec<u8>) -> Self {
-//         //reinsert later
-//     }
-//     fn hexnew(hex: String) -> Self {
-//         //reinsert later
-//     }
-// }
+impl BitcoinTransaction {
+    pub fn new(txbytes: Vec<u8>) -> Self {
+        let mut ver: [u8; 4] = [0; 4];
+        ver.copy_from_slice(&txbytes[..4]);
+        let version = u32::from_le_bytes(ver);
+
+        let inputs_num: u64 = decode_variable_int(&txbytes[4..13]).unwrap();
+        let vinlen: usize = measure_variable_int(&txbytes[4..13]).unwrap();
+        let mut inputstart: usize = 4 + vinlen;
+        let mut inputs = Vec::new();
+
+        if inputs_num > 0 {
+            for i in 0..inputs_num {
+                let mut input = Input::new(txbytes[inputstart..].to_vec());
+                inputstart += input.bytes_len;
+                inputs.push(input);
+            }
+            inputs.to_vec();
+        }
+
+        let outputs_num: u64 = decode_variable_int(&txbytes[inputstart..9 + inputstart]).unwrap();
+        let voutlen: usize = measure_variable_int(&txbytes[inputstart..9 + inputstart]).unwrap();
+
+        let mut outputstart: usize = inputstart + voutlen;
+        let mut outputs = Vec::new();
+        for i in 0..outputs_num {
+            let mut output = Output::new(txbytes[outputstart..].to_vec());
+            outputstart += output.bytes_len;
+            outputs.push(output);
+        }
+
+        let mut lt: [u8; 4] = [0; 4];
+        lt.copy_from_slice(&txbytes[outputstart..4 + outputstart]);
+        let lock_time = u32::from_le_bytes(lt);
+
+        assert_eq!(inputs.len(), inputs_num as usize);
+        assert_eq!(outputs.len(), outputs_num as usize);
+
+        BitcoinTransaction {
+            inputs,
+            inputs_num,
+            outputs,
+            outputs_num,
+            version,
+            lock_time,
+            bytes_len: 4 + outputstart,
+        }
+    }
+    pub fn hexnew(hex: String) -> Result<BitcoinTransaction, SpvError> {
+        match hex::decode(&hex) {
+            Ok(txbytes) => Ok(BitcoinTransaction::new(txbytes)),
+            Err(e) => Err(SpvError::ParseError),
+        }
+    }
+}
 
 #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
 pub struct Input {
-    r#type: InputType,
+    pub input_type: InputType,
     // Type of the input
-    position: u32,
+    pub position: u32,
     // position of the tx in its Block
-    txhash: BitcoinTxHash,
+    pub txhash: BitcoinTxHash,
     // hash of the transaction
+    pub script_length: u64,
+    // length of the spend script
+    pub script: Vec<u8>,
+    // script bytes
+    pub sequence: [u8; 4],
+    // length of the input in bytes
+    pub bytes_len: usize,
+}
+
+impl Input {
+    fn new(ibytes: Vec<u8>) -> Self {
+        let mut txhash: [u8; 32] = [0; 32];
+        txhash.copy_from_slice(&ibytes[..32]);
+
+        let mut tx_out_index: [u8; 4] = [0; 4];
+        tx_out_index.copy_from_slice(&ibytes[32..36]);
+        let position = u32::from_le_bytes(tx_out_index);
+
+        let script_length: u64 = decode_variable_int(&ibytes[36..45]).unwrap();
+        let script_length_len: usize = measure_variable_int(&ibytes[36..45]).unwrap();
+        let script_start = 36 + script_length_len; //checkc for correctness
+        let script_end = script_start + script_length as usize;
+        let input_end = script_end + 4;
+
+        let script: Vec<u8> = ibytes[script_start..script_length as usize].to_vec();
+
+        let mut sequence: [u8; 4] = [0; 4];
+        sequence.copy_from_slice(&ibytes[script_end..input_end]);
+
+        let input_type: InputType = InputType::NONE; // testing measure
+
+        Self {
+            input_type,
+            position,
+            txhash,
+            script_length,
+            script,
+            sequence,
+            bytes_len: input_end,
+        }
+    }
+
+    fn default() -> Self {
+        let txh: [u8; 32] = [0; 32];
+        let seq: [u8; 4] = [0; 4];
+
+        Self {
+            input_type: InputType::NONE,
+            position: 55,
+            txhash: txh,
+            script_length: 45,
+            script: txh.to_vec(),
+            sequence: seq,
+            bytes_len: 123,
+        }
+    }
 }
 
 #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
@@ -134,11 +243,53 @@ pub enum InputType {
 
 #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
 pub struct Output {
-    r#type: OutputType,
+    pub output_type: OutputType,
     // type of the output
-    value: u64,
+    pub value: u64,
     // amount of btc in sats
-    payload: Vec<u8>, // data sent with the transaction
+    pub script: Vec<u8>,
+
+    pub script_length: u64,
+
+    pub bytes_len: usize,
+    // payload: Option<Vec<u8>>,
+    // // data sent with the transaction (Op return)
+}
+
+impl Output {
+    fn new(obytes: Vec<u8>) -> Self {
+        let mut val: [u8; 8] = [0; 8];
+        val.copy_from_slice(&obytes[..8]);
+        let value: u64 = u64::from_le_bytes(val);
+
+        let script_start: usize = 8 + measure_variable_int(&obytes[8..17]).unwrap();
+        let script_length = decode_variable_int(&obytes[8..script_start]).unwrap();
+        let script_end: usize = script_start + script_length as usize;
+
+        let script = obytes[script_start..script_end].to_vec();
+
+        let output_type = OutputType::WPKH; // temporary hardcode
+
+        Self {
+            output_type,
+            value,
+            script,
+            script_length,
+            bytes_len: script_end,
+        }
+    }
+
+    fn default() -> Self {
+        let transaction_hash: [u8; 32] = [0; 32];
+
+        Self {
+            output_type: OutputType::WPKH,
+            value: 55,
+            script: transaction_hash.to_vec(),
+            script_length: 45,
+            bytes_len: 123,
+        }
+    }
 }
 
 #[allow(non_camel_case_types)]
@@ -150,6 +301,7 @@ pub enum OutputType {
     PKH,
     SH,
     NONSTANDARD,
+    // https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md
 }
 
 pub type HeaderChain = Vec<BlockHeader>;
@@ -221,7 +373,7 @@ pub struct Proof {
     // chain of bitcoin headers provifing context for the proof
     pub headers: HeaderChain,
     // transaction associated with the Proof
-    pub transaction: Transaction,
+    pub transaction: BitcoinTransaction,
     // public key of the request this proof corresponds to
     pub request: Pubkey,
 }
@@ -232,6 +384,8 @@ pub enum AccountState {
     Request(ClientRequestInfo),
     // Verified Proof
     Verification(Proof),
+    // Account holds a HeaderStore structure
+    Headers(HeaderAccountInfo),
     // Account's userdata is Unallocated
     Unallocated,
     // Invalid
@@ -253,6 +407,7 @@ pub enum SpvError {
     // header store write/read result is invalid
     ParseError,
     // other errors with parsing inputs
+    InvalidAccount,
 }
 
 impl error::Error for SpvError {
@@ -274,6 +429,12 @@ impl From<DecodeHexError> for SpvError {
     }
 }
 
+impl From<hex::FromHexError> for SpvError {
+    fn from(e: hex::FromHexError) -> Self {
+        SpvError::ParseError
+    }
+}
+
 // impl fmt::Debug for SpvError {
 //     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result{
 //         match self {
@@ -290,6 +451,7 @@ impl fmt::Display for SpvError {
             SpvError::InvalidBlockHeader => "BlockHeader is malformed or does not apply ".fmt(f),
             SpvError::HeaderStoreError => "Placeholder headerstore error text".fmt(f),
             SpvError::ParseError => "Error parsing blockheaders placceholder text".fmt(f),
+            SpvError::InvalidAccount => "Provided account is not usable or does not exist".fmt(f),
         }
     }
 }

programs/btc_spv_api/src/utils.rs

@@ -1,8 +1,9 @@
+use serde_derive::{Deserialize, Serialize};
 use std::{fmt, num::ParseIntError};
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum DecodeHexError {
-    OddLength,
+    InvalidLength(LengthError),
     ParseInt(ParseIntError),
 }
 
@@ -15,15 +16,30 @@ impl From<ParseIntError> for DecodeHexError {
 impl fmt::Display for DecodeHexError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
-            DecodeHexError::OddLength => "input hex string length is odd ".fmt(f),
+            DecodeHexError::InvalidLength(LengthError::OddLength) => {
+                "input hex string length is odd ".fmt(f)
+            }
+            DecodeHexError::InvalidLength(LengthError::Maximum(e)) => {
+                "input exceeds the maximum length".fmt(f)
+            }
+            DecodeHexError::InvalidLength(LengthError::Minimum(e)) => {
+                "input does not meet the minimum length".fmt(f)
+            }
             DecodeHexError::ParseInt(e) => e.fmt(f),
         }
     }
 }
 
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub enum LengthError {
+    OddLength,
+    Maximum(u32),
+    Minimum(u32),
+}
+
 pub fn decode_hex(s: &str) -> Result<Vec<u8>, DecodeHexError> {
     if s.len() % 2 != 0 {
-        Err(DecodeHexError::OddLength)
+        Err(DecodeHexError::InvalidLength(LengthError::OddLength))
     } else {
         (0..s.len())
             .step_by(2)
@@ -31,3 +47,41 @@ pub fn decode_hex(s: &str) -> Result<Vec<u8>, DecodeHexError> {
             .collect()
     }
 }
+
+pub fn measure_variable_int(vint: &[u8]) -> Result<usize, DecodeHexError> {
+    let ln = vint.len();
+    if ln > 9 {
+        return Err(DecodeHexError::InvalidLength(LengthError::Maximum(9)));
+    }
+
+    let val: usize = match vint[0] {
+        0..=252 => 1,
+        253 => 2,
+        254 => 5,
+        255 => 9,
+    };
+    Ok(val)
+}
+
+pub fn decode_variable_int(vint: &[u8]) -> Result<u64, DecodeHexError> {
+    let ln = vint.len();
+    if ln > 9 {
+        return Err(DecodeHexError::InvalidLength(LengthError::Maximum(9)));
+    }
+
+    let val: u64 = match vint[0] {
+        0..=252 => u64::from(vint[0]),
+        253 => u64::from(vint[1]),
+        254 => {
+            let mut val: [u8; 4] = [0; 4];
+            val.copy_from_slice(&vint[1..5]);
+            u64::from(u32::from_le_bytes(val))
+        }
+        255 => {
+            let mut val: [u8; 8] = [0; 8];
+            val.copy_from_slice(&vint[1..9]);
+            u64::from_le_bytes(val)
+        }
+    };
+    Ok(val)
+}

programs/btc_spv_bin/Cargo.toml

@@ -10,7 +10,12 @@ serde="1.0.100"
 serde_derive="1.0.100"
 serde_json = "1.0.40"
 ureq = { version = "0.11.1", features = ["json"] }
+hex = "0.3.2"
 
 [[bin]]
 name = "blockheaders"
 path = "src/blockheade.rs"
+
+[[bin]]
+name = "blocks"
+path = "src/block.rs"

programs/btc_spv_bin/src/block.rs

@@ -0,0 +1,46 @@
+use clap;
+use clap::{App, Arg};
+use hex;
+use std::fs::File;
+use std::io::prelude::*;
+
+fn get_block_raw(hash: &str) -> String {
+    let qs = format!("https://blockchain.info/block/{}?format=hex", hash);
+    let body = ureq::get(&qs).call();
+    if body.error() {
+        panic!("request failed");
+    } else {
+        let textbh: String = body.into_string().unwrap();
+        textbh
+    }
+}
+
+fn write_file(fname: String, bytes: &[u8]) -> std::io::Result<()> {
+    let mut buffer = File::create(fname)?;
+    buffer.write_all(bytes)?;
+    Ok(())
+}
+
+fn main() {
+    let matches = App::new("header fetch util")
+        .arg(Arg::with_name("blockhash"))
+        .arg(Arg::with_name("output"))
+        .help("block hash to get header from")
+        .get_matches();
+
+    let default_output = "file";
+    let output = matches.value_of("output").unwrap_or(default_output);
+
+    let testhash = "0000000000000bae09a7a393a8acded75aa67e46cb81f7acaa5ad94f9eacd103";
+    let blockhash = matches.value_of("blockhash").unwrap_or(testhash);
+    let blockraw = get_block_raw(&blockhash);
+
+    if default_output == output {
+        let fname = format!("block-{}.in", blockhash);
+        let outf = hex::decode(&blockraw).unwrap();
+        let arr = &outf[0..];
+        write_file(fname, arr).unwrap();
+    } else {
+        println!("{}", blockraw);
+    }
+}