Add BpfLoader (#34)

feat: Add BpfLoader

web3.js/flow-typed/elfy.js

@@ -0,0 +1,4 @@
+declare module 'elfy' {
+  // TODO: Fill in types
+  declare module.exports: any;
+}

web3.js/flow-typed/mz_vx.x.x.js

@@ -0,0 +1,73 @@
+// flow-typed signature: ed29f42bf4f4916e4f3ba1f5e7343c9d
+// flow-typed version: <<STUB>>/mz_v2.7.0/flow_v0.81.0
+
+/**
+ * This is an autogenerated libdef stub for:
+ *
+ *   'mz'
+ *
+ * Fill this stub out by replacing all the `any` types.
+ *
+ * Once filled out, we encourage you to share your work with the
+ * community by sending a pull request to:
+ * https://github.com/flowtype/flow-typed
+ */
+
+declare module 'mz' {
+  declare module.exports: any;
+}
+
+/**
+ * We include stubs for each file inside this npm package in case you need to
+ * require those files directly. Feel free to delete any files that aren't
+ * needed.
+ */
+declare module 'mz/child_process' {
+  declare module.exports: any;
+}
+
+declare module 'mz/crypto' {
+  declare module.exports: any;
+}
+
+declare module 'mz/dns' {
+  declare module.exports: any;
+}
+
+declare module 'mz/fs' {
+  declare module.exports: any;
+}
+
+declare module 'mz/readline' {
+  declare module.exports: any;
+}
+
+declare module 'mz/zlib' {
+  declare module.exports: any;
+}
+
+// Filename aliases
+declare module 'mz/child_process.js' {
+  declare module.exports: $Exports<'mz/child_process'>;
+}
+declare module 'mz/crypto.js' {
+  declare module.exports: $Exports<'mz/crypto'>;
+}
+declare module 'mz/dns.js' {
+  declare module.exports: $Exports<'mz/dns'>;
+}
+declare module 'mz/fs.js' {
+  declare module.exports: $Exports<'mz/fs'>;
+}
+declare module 'mz/index' {
+  declare module.exports: $Exports<'mz'>;
+}
+declare module 'mz/index.js' {
+  declare module.exports: $Exports<'mz'>;
+}
+declare module 'mz/readline.js' {
+  declare module.exports: $Exports<'mz/readline'>;
+}
+declare module 'mz/zlib.js' {
+  declare module.exports: $Exports<'mz/zlib'>;
+}

web3.js/package.json

@@ -55,7 +55,9 @@
     "bn.js": "^4.11.8",
     "bs58": "^4.0.1",
     "buffer-layout": "^1.2.0",
+    "elfy": "^0.1.0",
     "jayson": "^2.0.6",
+    "mz": "^2.7.0",
     "node-fetch": "^2.2.0",
     "superstruct": "^0.6.0",
     "tweetnacl": "^1.0.0"

web3.js/src/bpf-loader.js

@@ -0,0 +1,55 @@
+// @flow
+
+import fs from 'mz/fs';
+import elfy from 'elfy';
+
+import {Account, PublicKey, Loader, SystemProgram} from '.';
+import {sendAndConfirmTransaction} from './util/send-and-confirm-transaction';
+import type {Connection} from '.';
+
+/**
+ * Factory class for transactions to interact with a program loader
+ */
+export class BpfLoader {
+  /**
+   * Public key that identifies the NativeLoader
+   */
+  static get programId(): PublicKey {
+    return new PublicKey('0x0606060606060606060606060606060606060606060606060606060606060606');
+  }
+
+  /**
+   * Loads a BPF program
+   *
+   * @param connection The connection to use
+   * @param owner User account to load the program with
+   * @param programName Name of the BPF program
+   */
+  static async load(
+    connection: Connection,
+    owner: Account,
+    programName: string,
+  ): Promise<PublicKey> {
+    const programAccount = new Account();
+
+    const data = await fs.readFile(programName);
+    const elf = elfy.parse(data);
+    const section = elf.body.sections.find(section => section.name === '.text.entrypoint');
+
+    // Allocate memory for the program account
+    const transaction = SystemProgram.createAccount(
+      owner.publicKey,
+      programAccount.publicKey,
+      1,
+      section.data.length,
+      BpfLoader.programId,
+    );
+    await sendAndConfirmTransaction(connection, owner, transaction);
+
+    const loader = new Loader(connection, BpfLoader.programId);
+    await loader.load(programAccount, section.data);
+    await loader.finalize(programAccount);
+
+    return programAccount.publicKey;
+  }
+}

web3.js/src/index.js

@@ -1,5 +1,6 @@
 // @flow
 export {Account} from './account';
+export {BpfLoader} from './bpf-loader';
 export {BudgetProgram} from './budget-program';
 export {Connection} from './connection';
 export {Loader} from './loader';

web3.js/src/loader.js

@@ -32,10 +32,9 @@ export class Loader {
    * Load program data
    *
    * @param program Account to load the program info
-   * @param offset Account userdata offset to write `bytes` into
-   * @param bytes Program data
+   * @param data Program data
    */
-  async load(program: Account, offset: number, bytes: Array<number>) {
+  async load(program: Account, data: Array<number>) {
     const userdataLayout = BufferLayout.struct([
       BufferLayout.u32('instruction'),
       BufferLayout.u32('offset'),
@@ -43,27 +42,35 @@ export class Loader {
       BufferLayout.u32('bytesLengthPadding'),
       BufferLayout.seq(
         BufferLayout.u8('byte'),
-        BufferLayout.offset(BufferLayout.u32(), -8),
-        'bytes'
-      ),
+        BufferLayout.offset(BufferLayout.u32(), -8), 'bytes'),
     ]);
 
-    let userdata = Buffer.alloc(bytes.length + 16);
-    userdataLayout.encode(
-      {
-        instruction: 0, // Load instruction
-        offset,
-        bytes,
-      },
-      userdata,
-    );
+    const chunkSize = 256;
+    let userdata = Buffer.alloc(chunkSize + 16);
+    let offset = 0;
+    let array = data;
+    while (array.length > 0) {
+      const bytes = array.slice(0, chunkSize);
 
-    const transaction = new Transaction().add({
-      keys: [program.publicKey],
-      programId: this.programId,
-      userdata,
-    });
-    await sendAndConfirmTransaction(this.connection, program, transaction);
+      userdataLayout.encode(
+        {
+          instruction: 0, // Load instruction
+          offset,
+          bytes,
+        },
+        userdata,
+      );
+
+      const transaction = new Transaction().add({
+        keys: [program.publicKey],
+        programId: this.programId,
+        userdata,
+      });
+      await sendAndConfirmTransaction(this.connection, program, transaction);
+
+      offset += chunkSize;
+      array = array.slice(chunkSize);
+    }
   }
 
   /**

web3.js/src/native-loader.js

@@ -42,7 +42,7 @@ export class NativeLoader {
     await sendAndConfirmTransaction(connection, owner, transaction);
 
     const loader = new Loader(connection, NativeLoader.programId);
-    await loader.load(programAccount, 0, bytes);
+    await loader.load(programAccount, bytes);
     await loader.finalize(programAccount);
 
     return programAccount.publicKey;

web3.js/test/bin/build.sh

@@ -0,0 +1,4 @@
+#!/bin/bash -ex
+
+/usr/local/opt/llvm/bin/clang -Werror -target bpf -O2 -emit-llvm -fno-builtin -o noop_c.bc -c noop.c
+/usr/local/opt/llvm/bin/llc -march=bpf -filetype=obj -function-sections -o noop_c.o noop_c.bc

web3.js/test/bin/noop.c

@@ -0,0 +1,133 @@
+
+//#include <stdint.h>
+//#include <stddef.h>
+
+#if 1
+// one way to define a helper function is with index as a fixed value
+#define BPF_TRACE_PRINTK_IDX 6
+static int (*sol_print)(int, int, int, int, int) = (void *)BPF_TRACE_PRINTK_IDX;
+#else
+// relocation is another option
+extern int sol_print(int, int, int, int, int);
+#endif
+
+typedef long long unsigned int uint64_t;
+typedef long long int int64_t;
+typedef unsigned char uint8_t;
+
+typedef enum { false = 0, true } bool;
+
+#define SIZE_PUBKEY 32
+typedef struct {
+    uint8_t x[SIZE_PUBKEY];
+} SolPubkey;
+
+typedef struct {
+    SolPubkey *key;
+    int64_t* tokens;
+    uint64_t userdata_len;
+    uint8_t *userdata;
+    SolPubkey *program_id;
+} SolKeyedAccounts;
+
+// TODO support BPF function calls rather then forcing everything to be inlined
+#define SOL_FN_PREFIX __attribute__((always_inline)) static
+
+// TODO move this to a registered helper
+SOL_FN_PREFIX void sol_memcpy(void *dst, void *src, int len) {
+    for (int i = 0; i < len; i++) {
+        *((uint8_t *)dst + i) = *((uint8_t *)src + i);
+    }
+}
+
+#define sol_panic() _sol_panic(__LINE__)
+SOL_FN_PREFIX void _sol_panic(uint64_t line) {
+    sol_print(0, 0, 0xFF, 0xFF, line);
+    char *pv = (char *)1;
+    *pv = 1;
+}
+
+SOL_FN_PREFIX int sol_deserialize(uint8_t *src, uint64_t num_ka, SolKeyedAccounts *ka,
+                              uint8_t **userdata, uint64_t *userdata_len) {
+    if (num_ka != *(uint64_t *)src) {
+        return 0;
+    }
+    src += sizeof(uint64_t);
+
+    // TODO fixed iteration loops ok? unrolled?
+    for (int i = 0; i < num_ka; i++) { // TODO this should end up unrolled, confirm
+        // key
+        ka[i].key = (SolPubkey *)src;
+        src += SIZE_PUBKEY;
+
+        // tokens
+        ka[i].tokens = (int64_t *)src;
+        src += sizeof(int64_t);
+
+        // account userdata
+        ka[i].userdata_len = *(uint64_t *)src;
+        src += sizeof(uint64_t);
+        ka[i].userdata = src;
+        src += ka[i].userdata_len;
+
+        // program_id
+        ka[i].program_id = (SolPubkey *)src;
+        src += SIZE_PUBKEY;
+    }
+    // tx userdata
+    *userdata_len = *(uint64_t *)src;
+    src += sizeof(uint64_t);
+    *userdata = src;
+
+    return 1;
+}
+
+
+// -- Debug --
+
+SOL_FN_PREFIX void print_key(SolPubkey *key) {
+    for (int j = 0; j < SIZE_PUBKEY; j++) {
+        sol_print(0, 0, 0, j, key->x[j]);
+    }
+}
+
+SOL_FN_PREFIX void print_userdata(uint8_t *data, int len) {
+    for (int j = 0; j < len; j++) {
+        sol_print(0, 0, 0, j, data[j]);
+    }
+}
+
+SOL_FN_PREFIX void print_params(uint64_t num_ka, SolKeyedAccounts *ka,
+                                uint8_t *userdata, uint64_t userdata_len) {
+    sol_print(0, 0, 0, 0, num_ka);
+    for (int i = 0; i < num_ka; i++) {
+        // key
+        print_key(ka[i].key);
+
+        // tokens
+        sol_print(0, 0, 0, 0, *ka[i].tokens);
+
+        // account userdata
+        print_userdata(ka[i].userdata, ka[i].userdata_len);
+
+        // program_id
+        print_key(ka[i].program_id);
+    }
+    // tx userdata
+    print_userdata(userdata, userdata_len);
+}
+
+// -- Program entrypoint --
+
+uint64_t entrypoint(char *buf) {
+    SolKeyedAccounts ka[1];
+    uint64_t userdata_len;
+    uint8_t *userdata;
+
+    if (1 != sol_deserialize((uint8_t *)buf, 1, ka, &userdata, &userdata_len)) {
+        return 0;
+    }
+    print_params(1, ka, userdata, userdata_len);
+    return 1;
+}
+

web3.js/test/bin/noop_c.readme.txt

@@ -0,0 +1,9 @@
+
+The Solana SDK Test Binary
+
+One of the functions that the SDK tests exercises is the loading and calling of Berkley Packet Filter programs (BPF).
+
+The test file noop_c.o is an ELF object file containing a small BPF program contained in the ELF section named ".text.entrypoint".
+
+The C source file for noop_c.o is noop_c.c and it can be rebuilt using the build.sh script.
+The build.sh script depends on LLVM 6.0 to be installed.

web3.js/test/bpf-loader.test.js

@@ -0,0 +1,33 @@
+// @flow
+
+import {
+  Connection,
+  BpfLoader,
+  Transaction,
+  sendAndConfirmTransaction,
+} from '../src';
+import {mockRpcEnabled} from './__mocks__/node-fetch';
+import {url} from './url';
+import {newAccountWithTokens} from './new-account-with-tokens';
+
+if (!mockRpcEnabled) {
+  // The default of 5 seconds is too slow for live testing sometimes
+  jest.setTimeout(10000);
+}
+
+test('load BPF program', async () => {
+  if (mockRpcEnabled) {
+    console.log('non-live test skipped');
+    return;
+  }
+
+  const connection = new Connection(url);
+  const from = await newAccountWithTokens(connection);
+  const programId = await BpfLoader.load(connection, from, 'test/bin/noop_c.o');
+  const transaction = new Transaction().add({
+    keys: [from.publicKey],
+    programId,
+  });
+  await sendAndConfirmTransaction(connection, from, transaction);
+});
+

web3.js/test/native-loader.test.js

@@ -15,8 +15,7 @@ if (!mockRpcEnabled) {
   jest.setTimeout(10000);
 }
 
-
-test('load noop program', async () => {
+test('load native program', async () => {
   if (mockRpcEnabled) {
     console.log('non-live test skipped');
     return;
@@ -24,12 +23,12 @@ test('load noop program', async () => {
 
   const connection = new Connection(url);
   const from = await newAccountWithTokens(connection);
-
-  const noopProgramId = await NativeLoader.load(connection, from, 'noop');
-  const noopTransaction = new Transaction().add({
+  const programId = await NativeLoader.load(connection, from, 'noop');
+  const transaction = new Transaction().add({
     keys: [from.publicKey],
-    programId: noopProgramId,
+    programId,
   });
-  await expect(sendAndConfirmTransaction(connection, from, noopTransaction)).resolves.toBeUndefined();
+
+  await sendAndConfirmTransaction(connection, from, transaction);
 });