From 19b9d8a0129cb5fa79ac87de5a3d8c8c36bc585f Mon Sep 17 00:00:00 2001 From: ben devlin Date: Sat, 29 Jun 2019 10:49:42 -0400 Subject: [PATCH] updates to zcash class --- aws/cl_zcash/software/runtime/Makefile | 4 +- aws/cl_zcash/software/runtime/test_zcash.cpp | 126 ++++++++++ aws/cl_zcash/software/runtime/zcash_fpga.cpp | 246 ++++++++++++++++++- aws/cl_zcash/software/runtime/zcash_fpga.hpp | 154 ++++++++++++ 4 files changed, 519 insertions(+), 11 deletions(-) create mode 100644 aws/cl_zcash/software/runtime/test_zcash.cpp create mode 100644 aws/cl_zcash/software/runtime/zcash_fpga.hpp diff --git a/aws/cl_zcash/software/runtime/Makefile b/aws/cl_zcash/software/runtime/Makefile index b68d710..1b3c266 100644 --- a/aws/cl_zcash/software/runtime/Makefile +++ b/aws/cl_zcash/software/runtime/Makefile @@ -20,11 +20,11 @@ INCLUDES += -I $(HDK_DIR)/common/software/include INCLUDES += -I ./include CC = gcc -CFLAGS = -DCONFIG_LOGLEVEL=4 -g -Wall $(INCLUDES) +CFLAGS = -DCONFIG_LOGLEVEL=4 -g -Wall $(INCLUDES) -lstdc++ LDLIBS = -lfpga_mgmt -lrt -lpthread -SRC = ${SDK_DIR}/userspace/utils/sh_dpi_tasks.c test_zcash.c +SRC = zcash_fpga.cpp test_zcash.cpp ${SDK_DIR}/userspace/utils/sh_dpi_tasks.c OBJ = $(SRC:.c=.o) BIN = test_zcash diff --git a/aws/cl_zcash/software/runtime/test_zcash.cpp b/aws/cl_zcash/software/runtime/test_zcash.cpp new file mode 100644 index 0000000..84ead07 --- /dev/null +++ b/aws/cl_zcash/software/runtime/test_zcash.cpp @@ -0,0 +1,126 @@ +// +// ZCash FPGA test. +// +// Copyright (C) 2019 Benjamin Devlin and Zcash Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// + +#define _XOPEN_SOURCE 500 + +#include +#include +#include +#include +#include +#include + +#include +#include + + +#include +#include +#include +#include + +#include "zcash_fpga.hpp" + +/* use the stdout logger for printing debug information */ + +const struct logger *logger = &logger_stdout; +/* + * check if the corresponding AFI for hello_world is loaded + */ +int check_afi_ready(int slot_id); + + +void usage(char* program_name) { + printf("usage: %s [--slot ][]\n", program_name); +} + +uint32_t byte_swap(uint32_t value); + + +uint32_t byte_swap(uint32_t value) { + uint32_t swapped_value = 0; + int b; + for (b = 0; b < 4; b++) { + swapped_value |= ((value >> (b * 8)) & 0xff) << (8 * (3-b)); + } + return swapped_value; +} + + +int main(int argc, char **argv) { + + int slot_id = 0; + int rc; + uint32_t value = 0; + + // Process command line args + { + int i; + int value_set = 0; + for (i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--slot")) { + i++; + if (i >= argc) { + printf("error: missing slot-id\n"); + usage(argv[0]); + return 1; + } + sscanf(argv[i], "%d", &slot_id); + } else if (!value_set) { + sscanf(argv[i], "%x", &value); + value_set = 1; + } else { + printf("error: Invalid arg: %s", argv[i]); + usage(argv[0]); + return 1; + } + } + } + + zcash_fpga& zfpga = zcash_fpga::get_instance(); + zfpga.init_fpga(); + + // Get FPGA status + zcash_fpga::fpga_status_rpl_t status_rpl; + rc = zfpga.get_status(status_rpl); + fail_on(rc, out, "Unable toget FPGA status!"); + + // Read and write a data slot in BLS12_381 + zcash_fpga::bls12_381_slot_t data_slot; + rc = zfpga.bls12_381_read_data_slot(0, data_slot); + printf("Data slot type was: %i, data is 0x", data_slot.point_type); + for (int i = 47; i >= 0; i--) + printf("%x", data_slot.dat[i]); + printf("\n"); + + printf("Writing to data slot...\n"); + data_slot.point_type = zcash_fpga::FE; + memset(&data_slot.dat, 0x0a, 48); + rc = zfpga.bls12_381_read_data_slot(0, data_slot); + + rc = zfpga.bls12_381_read_data_slot(0, data_slot); + printf("Data slot type was: %i, data is 0x", data_slot.point_type); + for (int i = 47; i >= 0; i--) + printf("%x", data_slot.dat[i]); + printf("\n"); + + return rc; +out: + return 1; +} diff --git a/aws/cl_zcash/software/runtime/zcash_fpga.cpp b/aws/cl_zcash/software/runtime/zcash_fpga.cpp index cc6be9c..d1893d8 100644 --- a/aws/cl_zcash/software/runtime/zcash_fpga.cpp +++ b/aws/cl_zcash/software/runtime/zcash_fpga.cpp @@ -1,16 +1,15 @@ -#include zcash_fpga.h +#include "zcash_fpga.hpp" #include -#include -#include -#include -#include -#include +#include +#include #include -#include -#include -#include + +zcash_fpga& zcash_fpga::get_instance() { + static zcash_fpga instance; + return instance; +} int zcash_fpga::init_fpga(int slot_id) { // Initialize the FPGA @@ -90,3 +89,232 @@ int zcash_fpga::init_fpga(int slot_id) { } return 1; } + +int zcash_fpga::check_afi_ready(int slot_id) { + struct fpga_mgmt_image_info info = {0}; + int rc; + + /* get local image description, contains status, vendor id, and device id. */ + rc = fpga_mgmt_describe_local_image(slot_id, &info,0); + fail_on(rc, out, "Unable to get AFI information from slot %d. Are you running as root?",slot_id); + + /* check to see if the slot is ready */ + if (info.status != FPGA_STATUS_LOADED) { + rc = 1; + fail_on(rc, out, "AFI in Slot %d is not in READY state !", slot_id); + } + + printf("AFI PCI Vendor ID: 0x%x, Device ID 0x%x\n", + info.spec.map[FPGA_APP_PF].vendor_id, + info.spec.map[FPGA_APP_PF].device_id); + + /* confirm that the AFI that we expect is in fact loaded */ + if (info.spec.map[FPGA_APP_PF].vendor_id != pci_vendor_id || + info.spec.map[FPGA_APP_PF].device_id != pci_device_id) { + printf("AFI does not show expected PCI vendor id and device ID. If the AFI " + "was just loaded, it might need a rescan. Rescanning now.\n"); + + rc = fpga_pci_rescan_slot_app_pfs(slot_id); + fail_on(rc, out, "Unable to update PF for slot %d",slot_id); + /* get local image description, contains status, vendor id, and device id. */ + rc = fpga_mgmt_describe_local_image(slot_id, &info,0); + fail_on(rc, out, "Unable to get AFI information from slot %d",slot_id); + + printf("AFI PCI Vendor ID: 0x%x, Device ID 0x%x\n", + info.spec.map[FPGA_APP_PF].vendor_id, + info.spec.map[FPGA_APP_PF].device_id); + + /* confirm that the AFI that we expect is in fact loaded after rescan */ + if (info.spec.map[FPGA_APP_PF].vendor_id != pci_vendor_id || + info.spec.map[FPGA_APP_PF].device_id != pci_device_id) { + rc = 1; + fail_on(rc, out, "The PCI vendor id and device of the loaded AFI are not " + "the expected values."); + } + } + + return rc; + out: + return 1; +} + +int zcash_fpga::get_status(fpga_status_rpl_t& status_rpl) { + // Test: send status message + int rc; + unsigned int timeout = 0; + unsigned int read_len = 0; + + header_t hdr; + hdr.cmd = FPGA_STATUS; + hdr.len = 8; + rc = write_stream((uint8_t*)&hdr, sizeof(hdr)); + fail_on(rc, out, "Unable to read from FPGA!"); + + // Try read reply + uint8_t reply[256]; + while ((read_len = read_stream(reply, 256)) == 0) { + usleep(1); + timeout++; + if (timeout > 1000) { + printf("ERROR: No reply received, timeout\n"); + rc = 1; + goto out; + } + } + + status_rpl = *(fpga_status_rpl_t*)reply; + printf("INFO: Received FPGA reply, FPGA version: 0x%x", status_rpl.version); // TODO print more + + + return rc; +out: + return 1; +} + +int zcash_fpga::write_stream(uint8_t* data, unsigned int len) { + int rc; + uint32_t rdata; + unsigned int len_send = 0; + + if (~initialized) { + printf("INFO: FPGA not initialized!"); + goto out; + } + + + rc = fpga_pci_peek(pci_bar_handle_bar0, AXI_FIFO_OFFSET + 0xCULL, &rdata); + fail_on(rc, out, "Unable to read from FPGA!"); + if (len > rdata) { + printf("ERROR: write_stream does not have enough space to write %d bytes! (%d free)\n", len, rdata); + goto out; + } + + + while(len_send < len) { + if (AXI4_enabled) { + fpga_pci_poke64(pci_bar_handle_bar4, 0, *(uint64_t*)(&data[len_send])); + len_send += 8; + } else { + rc = fpga_pci_poke(pci_bar_handle_bar0, AXI_FIFO_OFFSET+0x10ULL, *(uint32_t*)(&data[len_send])); // Reset ISR + fail_on(rc, out, "Unable to write to FPGA!"); + len_send += 4; + } + } + + rc = fpga_pci_poke(pci_bar_handle_bar0, AXI_FIFO_OFFSET+0x14ULL, len); // Reset ISR + fail_on(rc, out, "Unable to write to FPGA!"); + + + printf("INFO: write_stream::Wrote %d bytes of data\n", len); + + // Check transmit complete bit and reset it + rc = fpga_pci_peek(pci_bar_handle_bar0, AXI_FIFO_OFFSET, &rdata); + fail_on(rc, out, "Unable to read from FPGA!"); + if ((rdata & (1 << 27)) == 0) { + printf("WARNING: write_stream transmit bit not set, register returned 0x%x\n", rdata); + } + + rc = fpga_pci_poke(pci_bar_handle_bar0, AXI_FIFO_OFFSET, 0x08000000); // Reset ISR + fail_on(rc, out, "Unable to write to FPGA!"); + + return rc; + out: + return 1; +} + +int zcash_fpga::read_stream(uint8_t* data, unsigned int size) { + + uint32_t rdata; + unsigned int read_len = 0; + int rc; + + if (~initialized) { + printf("INFO: FPGA not initialized!"); + goto out; + } + + + rc = fpga_pci_peek(pci_bar_handle_bar0, AXI_FIFO_OFFSET, &rdata); + fail_on(rc, out, "Unable to read from FPGA!"); + if ((rdata & (1 << 26)) == 0) return 0; // Nothing to read + + rc = fpga_pci_poke(pci_bar_handle_bar0, AXI_FIFO_OFFSET, 0x04000000); // clear ISR + fail_on(rc, out, "Unable to write to FPGA!"); + + rc = fpga_pci_peek(pci_bar_handle_bar0, AXI_FIFO_OFFSET + 0x1CULL, &rdata); //RDFO should be non-zero (slots used in FIFO) + fail_on(rc, out, "Unable to read from FPGA!"); + if (rdata == 0) { + printf("WARNING: Read FIFO shows data but length was 0!\n"); + return 0; + } + + rc = fpga_pci_peek(pci_bar_handle_bar0, AXI_FIFO_OFFSET + 0x24ULL, &rdata); //RLR - length of packet in bytes + fail_on(rc, out, "Unable to read from FPGA!"); + printf("INFO: Read FIFO shows %d waiting to be read from FPGA\n", rdata); + + if (size < rdata) { + printf("ERROR: Size of buffer (%d bytes) not big enough to read data!\n", size); + return 0; + } + + while(read_len < rdata) { + if (AXI4_enabled) { + rc = fpga_pci_peek(pci_bar_handle_bar4, 0x1000, (uint32_t*)(&data[read_len])); + fail_on(rc, out, "Unable to read from FPGA PCIS!"); + read_len += 8; + } else { + rc = fpga_pci_peek(pci_bar_handle_bar0, AXI_FIFO_OFFSET + 0x20ULL, (uint32_t*)(&data[read_len])); + fail_on(rc, out, "Unable to read from FPGA!"); + read_len += 4; + } + } + + printf("INFO: Read %d bytes from read_stream()\n", read_len); + + return read_len; + out: + return -1; +} + +int zcash_fpga::bls12_381_write_data_slot(unsigned int id, bls12_381_slot_t slot_data) { + char data[48]; + int rc = 0; + if (~initialized) { + printf("INFO: FPGA not initialized!"); + goto out; + } + + *(bls12_381_slot_t*)data = slot_data; + // Set the top 3 bits to the point type + data[47] &= 0x1F; + data[47] |= (slot_data.point_type << 5); + + for(int i = 0; i < 48/4; i=i+4) { + rc = fpga_pci_poke(pci_bar_handle_bar0, BLS12_381_OFFSET + BLS12_381_DATA_OFFSET + id*64 + i, *((uint32_t*)&data[i])); + fail_on(rc, out, "Unable to write to FPGA!"); + } + return 0; + out: + return rc; +} + +int zcash_fpga::bls12_381_read_data_slot(unsigned int id, bls12_381_slot_t& slot_data) { + int rc = 0; + if (~initialized) { + printf("INFO: FPGA not initialized!"); + goto out; + } + + for(int i = 0; i < 48/4; i=i+4) { + rc = fpga_pci_peek(pci_bar_handle_bar0, BLS12_381_OFFSET + BLS12_381_DATA_OFFSET + id*64 + i, (uint32_t*)(&slot_data)); + fail_on(rc, out, "Unable to read from FPGA!"); + } + + slot_data.point_type = (point_type_t)(*((uint8_t*)&slot_data + 47) >> 5); + // Clear top 3 bits + *((char*)&slot_data + 47) &= 0x1F; + + return 0; + out: + return rc; +} diff --git a/aws/cl_zcash/software/runtime/zcash_fpga.hpp b/aws/cl_zcash/software/runtime/zcash_fpga.hpp new file mode 100644 index 0000000..a438819 --- /dev/null +++ b/aws/cl_zcash/software/runtime/zcash_fpga.hpp @@ -0,0 +1,154 @@ +// +// ZCash FPGA library. +// +// Copyright (C) 2019 Benjamin Devlin and Zcash Foundation +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// + +#ifndef ZCASH_FPGA_H_ /* Include guard */ +#define ZCASH_FPGA_H_ + +#include + +#include +#include +#include +#include + +#define AXI_FIFO_OFFSET UINT64_C(0x0) +#define BLS12_381_OFFSET UINT64_C(0x1000) +#define BLS12_381_INST_OFFSET UINT64_C(0x1000) +#define BLS12_381_DATA_OFFSET UINT64_C(0x2000) + + +// These match the structs and commands defined in zcash_fpga_pkg.sv +class zcash_fpga { + + public: + + enum uint64_t { + ENB_BLS12_381 = 1 << 3, + ENB_VERIFY_SECP256K1_SIG = 1 << 2, + ENB_VERIFY_EQUIHASH_144_5 = 1 << 1, + ENB_VERIFY_EQUIHASH_200_9 = 1 << 0 + } command_cap_e; + + typedef enum uin32_t { + RESET_FPGA = 0x00000000, + FPGA_STATUS = 0x00000001, + VERIFY_EQUIHASH = 0x00000100, + VERIFY_SECP256K1_SIG = 0x00000101, + + // Replies from the FPGA + RESET_FPGA_RPL = 0x80000000, + FPGA_STATUS_RPL = 0x80000001, + FPGA_IGNORE_RPL = 0x80000002, + VERIFY_EQUIHASH_RPL = 0x80000100, + VERIFY_SECP256K1_SIG_RPL = 0x80000101, + BLS12_381_INTERRUPT_RPL = 0x80000200 + } command_t; + + typedef enum uint8_t { + SCALAR = 0, + FE = 1, + FE2 = 2, + FE12 = 3, + FP_AF = 4, + FP_JB = 5, + FP2_AF = 6, + FP2_JB = 7 + } point_type_t; + + // On the FPGA only the first 381 bits of dat are stored + typedef struct __attribute__((__packed__)) { + uint8_t dat[48]; + point_type_t point_type; + } bls12_381_slot_t; + + typedef struct __attribute__((__packed__)) { + uint32_t len; + command_t cmd; + } header_t; + + typedef struct __attribute__((__packed__)) { + header_t hdr; + } fpga_reset_rpl_t; + + typedef struct __attribute__((__packed__)) { + header_t hdr; + uint64_t ignore_hdr; + } fpga_ignore_rpl_t; + + typedef struct __attribute__((__packed__)) { + uint8_t typ1_state; + } fpga_state_t; + + typedef struct __attribute__((__packed__)) { + header_t hdr; + } fpga_status_rq_t; + + + typedef struct __attribute__((__packed__)) { + header_t hdr; + uint32_t version; + uint64_t build_date; + uint64_t build_host; + uint64_t cmd_cap; + fpga_state_t fpga_state; + } fpga_status_rpl_t; + + private: + static const uint16_t pci_vendor_id = 0x1D0F; /* Amazon PCI Vendor ID */ + static const uint16_t pci_device_id = 0xF000; /* PCI Device ID preassigned by Amazon for F1 applications */ + + pci_bar_handle_t pci_bar_handle_bar0 = PCI_BAR_HANDLE_INIT; + pci_bar_handle_t pci_bar_handle_bar4 = PCI_BAR_HANDLE_INIT; + + bool AXI4_enabled = false; + bool initialized = false; + + public: + static zcash_fpga& get_instance(); + zcash_fpga(zcash_fpga const&) = delete; + void operator=(zcash_fpga const&) = delete; + + /* + * This connects to the FPGA and must be called at least once + */ + int init_fpga(int slot_id = 0); + /* + * This sends a status request to the FPGA and waits for the reply, + * checking for any errors. + */ + int get_status(fpga_status_rpl_t& status_rpl); + + /* + * Functions for writing and reading data slots in the BLS12_381 coprocessor + */ + int bls12_381_write_data_slot(unsigned int id, bls12_381_slot_t slot_data); + int bls12_381_read_data_slot(unsigned int id, bls12_381_slot_t& slot_data); + + private: + zcash_fpga(){}; + ~zcash_fpga(){}; + + int check_afi_ready(int slot_id); + int read_stream(uint8_t* data, unsigned int size); + int write_stream(uint8_t* data, unsigned int len); + + +}; // zcash_fpga + +#endif // ZCASH_FPGA_H_