From b729bdbcbf736a8317aded45757d179f953e74a3 Mon Sep 17 00:00:00 2001 From: Matthew Kennedy Date: Sun, 7 Mar 2021 16:25:34 -0800 Subject: [PATCH] Store ini filesystem in compressed block device (#2420) * add module * implement compressed block device * enable for f7/h7 * script * stub compressed image * implement * call the script * headers, comments, fix init bug * set msd options * update chibios-contrib Co-authored-by: Matthew Kennedy --- .gitmodules | 3 + firmware/ChibiOS-Contrib | 2 +- firmware/config/stm32f7ems/efifeatures.h | 2 + firmware/config/stm32h7ems/efifeatures.h | 4 +- firmware/controllers/thread_priority.h | 2 + firmware/ext/uzlib | 1 + firmware/gen_config_board.sh | 1 + .../mass_storage/compressed_block_device.cpp | 149 ++++++++++++++++++ .../mass_storage/compressed_block_device.h | 25 +++ .../create_ini_image_compressed.sh | 28 ++++ .../hw_layer/mass_storage/mass_storage.mk | 9 +- .../hw_layer/mass_storage/null_device.cpp | 20 ++- .../mass_storage/ramdisk_image_compressed.h | 4 + firmware/hw_layer/ports/rusefi_halconf.h | 7 + 14 files changed, 251 insertions(+), 6 deletions(-) create mode 160000 firmware/ext/uzlib create mode 100644 firmware/hw_layer/mass_storage/compressed_block_device.cpp create mode 100644 firmware/hw_layer/mass_storage/compressed_block_device.h create mode 100755 firmware/hw_layer/mass_storage/create_ini_image_compressed.sh create mode 100644 firmware/hw_layer/mass_storage/ramdisk_image_compressed.h diff --git a/.gitmodules b/.gitmodules index 0a91ea4390..93b647f893 100644 --- a/.gitmodules +++ b/.gitmodules @@ -21,3 +21,6 @@ [submodule "firmware/controllers/can/wideband_firmware"] path = firmware/controllers/can/wideband_firmware url = https://github.com/mck1117/wideband +[submodule "firmware/ext/uzlib"] + path = firmware/ext/uzlib + url = https://github.com/pfalcon/uzlib diff --git a/firmware/ChibiOS-Contrib b/firmware/ChibiOS-Contrib index e7cdad3d1a..fdcbac6a8c 160000 --- a/firmware/ChibiOS-Contrib +++ b/firmware/ChibiOS-Contrib @@ -1 +1 @@ -Subproject commit e7cdad3d1a041caea83729fd2c709fd963fb40b0 +Subproject commit fdcbac6a8c13dc5ec55a31cbe4f217edd7dfca02 diff --git a/firmware/config/stm32f7ems/efifeatures.h b/firmware/config/stm32f7ems/efifeatures.h index b848f63217..30d850be97 100644 --- a/firmware/config/stm32f7ems/efifeatures.h +++ b/firmware/config/stm32f7ems/efifeatures.h @@ -63,3 +63,5 @@ // todo: start using consoleSerialRxPin? Not sure #undef EFI_CONSOLE_RX_BRAIN_PIN #define EFI_CONSOLE_RX_BRAIN_PIN GPIOD_9 + +#define EFI_USE_COMPRESSED_INI_MSD diff --git a/firmware/config/stm32h7ems/efifeatures.h b/firmware/config/stm32h7ems/efifeatures.h index 4395403e13..b25b17c876 100644 --- a/firmware/config/stm32h7ems/efifeatures.h +++ b/firmware/config/stm32h7ems/efifeatures.h @@ -40,4 +40,6 @@ #define EFI_MAX_31855 FALSE #undef BOARD_EXT_GPIOCHIPS - #define BOARD_EXT_GPIOCHIPS (BOARD_TLE6240_COUNT + BOARD_MC33972_COUNT + BOARD_TLE8888_COUNT + BOARD_DRV8860_COUNT + BOARD_MC33810_COUNT) +#define BOARD_EXT_GPIOCHIPS (BOARD_TLE6240_COUNT + BOARD_MC33972_COUNT + BOARD_TLE8888_COUNT + BOARD_DRV8860_COUNT + BOARD_MC33810_COUNT) + +#define EFI_USE_COMPRESSED_INI_MSD diff --git a/firmware/controllers/thread_priority.h b/firmware/controllers/thread_priority.h index 7be9c6706f..e150c35840 100644 --- a/firmware/controllers/thread_priority.h +++ b/firmware/controllers/thread_priority.h @@ -29,6 +29,8 @@ // Less important things #define PRIO_MMC (NORMALPRIO - 1) +// USB mass storage +#define MSD_THD_PRIO LOWPRIO // These can get starved without too much adverse effect #define PRIO_AUX_SERIAL NORMALPRIO diff --git a/firmware/ext/uzlib b/firmware/ext/uzlib new file mode 160000 index 0000000000..27e4f4c15b --- /dev/null +++ b/firmware/ext/uzlib @@ -0,0 +1 @@ +Subproject commit 27e4f4c15ba30c2cfc89575159e8efb50f95037e diff --git a/firmware/gen_config_board.sh b/firmware/gen_config_board.sh index f68e4117d7..8aef056741 100755 --- a/firmware/gen_config_board.sh +++ b/firmware/gen_config_board.sh @@ -39,5 +39,6 @@ java -DSystemOut.name=gen_config_board \ [ $? -eq 0 ] || { echo "ERROR generating TunerStudio config for ${BOARDNAME}"; exit 1; } ./hw_layer/mass_storage/create_ini_image.sh ./tunerstudio/generated/rusefi_${SHORT_BOARDNAME}.ini ./hw_layer/mass_storage/ramdisk_image.h +./hw_layer/mass_storage/create_ini_image_compressed.sh ./tunerstudio/generated/rusefi_${SHORT_BOARDNAME}.ini ./hw_layer/mass_storage/ramdisk_image_compressed.h exit 0 diff --git a/firmware/hw_layer/mass_storage/compressed_block_device.cpp b/firmware/hw_layer/mass_storage/compressed_block_device.cpp new file mode 100644 index 0000000000..193b9d723a --- /dev/null +++ b/firmware/hw_layer/mass_storage/compressed_block_device.cpp @@ -0,0 +1,149 @@ +/** + * @file compressed_block_device.cpp + * @brief This file implements a ChibiOS block device backed by a compressed (gzip) store. + * + * @date Mar 4, 2021 + * @author Matthew Kennedy, (c) 2021 + * + * This works by decompressing one block (512 bytes) at a time. + * + * For sequential reads, the performance is great - the gzip decompress can only go forwards in the file. + * If a block later in the file (but with a gap) is requested, we decompress (and discard) the blocks in the gap, + * returning the block requested. + * + * If a block is requested from before the previous block, we discard decompression state, + * reinitialize, and decompress up to that block. + * + * NOTE: This means performance is terrible for "true" random access! Things go best when you have a few + * big files in the filesystem with no fragmentation, so they can be read out in large sequential chunks. + * + */ + +#include "compressed_block_device.h" + +#include + +#define BLOCK_SIZE 512 + +static bool is_inserted(void*) { + // Device is always inserted + return true; +} + +static bool is_protected(void*) { + // Write protected - we can't do random access writes to a compressed volume + return true; +} + +static bool connect(void* instance) { + CompressedBlockDevice* cbd = reinterpret_cast(instance); + if (BLK_STOP == cbd->state) { + cbd->state = BLK_READY; + } + return HAL_SUCCESS; +} + +static bool disconnect(void* instance) { + CompressedBlockDevice* cbd = reinterpret_cast(instance); + if (BLK_STOP != cbd->state) { + cbd->state = BLK_STOP; + } + return HAL_SUCCESS; +} + +static bool read(void* instance, uint32_t startblk, uint8_t* buffer, uint32_t n) { + CompressedBlockDevice* cbd = reinterpret_cast(instance); + + // If we just initialized, or trying to seek backwards, (re)initialize the decompressor + if (cbd->lastBlock == -1 || startblk <= cbd->lastBlock) { + uzlib_uncompress_init(&cbd->d, cbd->dictionary, sizeof(cbd->dictionary)); + + cbd->d.source = cbd->source; + cbd->d.source_limit = cbd->d.source + cbd->sourceSize; + cbd->d.source_read_cb = NULL; + + uzlib_gzip_parse_header(&cbd->d); + + cbd->lastBlock = -1; + } + + // How many blocks do we need to decompress to get to the one requested? + size_t blocks_ahead = startblk - cbd->lastBlock; + + // Decompress blocks until we get to the block we need + for (size_t i = 0; i < blocks_ahead; i++) { + cbd->d.dest = cbd->d.dest_start = buffer; + cbd->d.dest_limit = buffer + BLOCK_SIZE; + + // Decompress one chunk + uzlib_uncompress(&cbd->d); + } + + // Save the current position in the stream so we can efficiently seek forward later + cbd->lastBlock = startblk; + + return HAL_SUCCESS; +} + +static bool write(void*, uint32_t, const uint8_t*, uint32_t) { + // you shouldn't be able to do this anyway, so just swallow it, I guess? + return HAL_SUCCESS; +} + +constexpr size_t gzSize(const uint8_t* image, size_t imageSize) { + // The last 4 bytes of the gzip stream encode the total size in bytes + const uint8_t* pSize = image + imageSize - 1; + size_t size = *pSize--; + size = 256 * size + *pSize--; + size = 256 * size + *pSize--; + return 256 * size + *pSize--; +} + +static bool get_info(void* instance, BlockDeviceInfo* bdip) { + CompressedBlockDevice* cbd = reinterpret_cast(instance); + if (cbd->state != BLK_READY) { + return HAL_FAILED; + } + + // The last 4 bytes of the gzip stream encode the total size in bytes + size_t size = gzSize(cbd->source, cbd->sourceSize); + + bdip->blk_num = size / BLOCK_SIZE; + bdip->blk_size = BLOCK_SIZE; + return HAL_SUCCESS; +} + +static bool sync(void* instance) { + CompressedBlockDevice* cbd = reinterpret_cast(instance); + if (BLK_READY != cbd->state) { + return HAL_FAILED; + } + else { + return HAL_SUCCESS; + } +} + +static const BaseBlockDeviceVMT cbdVmt = { + (size_t)0, // instanceOffset + is_inserted, + is_protected, + connect, + disconnect, + read, + write, + sync, + get_info, +}; + +void compressedBlockDeviceObjectInit(CompressedBlockDevice* cbd) { + cbd->vmt = &cbdVmt; + memset(cbd->dictionary, 0, sizeof(cbd->dictionary)); + cbd->state = BLK_STOP; +} + +void compressedBlockDeviceStart(CompressedBlockDevice* cbd, const uint8_t* source, size_t sourceSize) { + cbd->source = source; + cbd->sourceSize = sourceSize; + cbd->state = BLK_READY; + cbd->lastBlock = -1; +} diff --git a/firmware/hw_layer/mass_storage/compressed_block_device.h b/firmware/hw_layer/mass_storage/compressed_block_device.h new file mode 100644 index 0000000000..aea27074e7 --- /dev/null +++ b/firmware/hw_layer/mass_storage/compressed_block_device.h @@ -0,0 +1,25 @@ +/** + * @file compressed_block_device.h + * @brief This file implements a ChibiOS block device backed by a compressed (gzip) store. + * + * @date Mar 4, 2021 + * @author Matthew Kennedy, (c) 2021 + */ + +#pragma once + +#include "hal.h" +#include "uzlib.h" + +struct CompressedBlockDevice { + const BaseBlockDeviceVMT* vmt; + _base_block_device_data + int32_t lastBlock; + uzlib_uncomp d; + uint8_t dictionary[32768]; + const uint8_t* source; + size_t sourceSize; +}; + +void compressedBlockDeviceObjectInit(CompressedBlockDevice* cbd); +void compressedBlockDeviceStart(CompressedBlockDevice* cbd, const uint8_t* source, size_t sourceSize); diff --git a/firmware/hw_layer/mass_storage/create_ini_image_compressed.sh b/firmware/hw_layer/mass_storage/create_ini_image_compressed.sh new file mode 100755 index 0000000000..bc114a07c6 --- /dev/null +++ b/firmware/hw_layer/mass_storage/create_ini_image_compressed.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# fail on error +set -e + +rm -f rusefi.zip ramdisk_image.h + +# copy 1MB of zeroes +dd if=/dev/zero of=ramdisk.image bs=1024 count=1024 + +# create a FAT filesystem inside, name it RUSEFI +mkfs.fat ramdisk.image +fatlabel ramdisk.image RUSEFI + +# Put the zip inside the filesystem +mcopy -i ramdisk.image $1 :: +# Put a readme text file in there too +mcopy -i ramdisk.image hw_layer/mass_storage/README.txt :: + +# Compress the image as DEFLATE with gzip +gzip ramdisk.image + +# write out as a C array, with "static const" tacked on the front +xxd -i ramdisk.image.gz \ + | cat <(echo -n "static const ") - \ + > $2 + +rm ramdisk.image.gz diff --git a/firmware/hw_layer/mass_storage/mass_storage.mk b/firmware/hw_layer/mass_storage/mass_storage.mk index 0f1c2b6d64..61b20fcb3e 100644 --- a/firmware/hw_layer/mass_storage/mass_storage.mk +++ b/firmware/hw_layer/mass_storage/mass_storage.mk @@ -1,4 +1,9 @@ -HW_MASS_STORAGE_SRC_C = $(PROJECT_DIR)/ChibiOS-Contrib/os/various/lib_scsi.c +HW_MASS_STORAGE_SRC_C = $(PROJECT_DIR)/ChibiOS-Contrib/os/various/lib_scsi.c \ + ext/uzlib/src/tinflate.c \ + ext/uzlib/src/tinfgzip.c -ALLCPPSRC += $(PROJECT_DIR)/hw_layer/mass_storage/null_device.cpp +ALLINC += $(PROJECT_DIR)/ext/uzlib/src + +ALLCPPSRC += $(PROJECT_DIR)/hw_layer/mass_storage/null_device.cpp \ + $(PROJECT_DIR)/hw_layer/mass_storage/compressed_block_device.cpp diff --git a/firmware/hw_layer/mass_storage/null_device.cpp b/firmware/hw_layer/mass_storage/null_device.cpp index af389d68c8..0d290deb77 100644 --- a/firmware/hw_layer/mass_storage/null_device.cpp +++ b/firmware/hw_layer/mass_storage/null_device.cpp @@ -12,7 +12,13 @@ #if EFI_EMBED_INI_MSD #include "ramdisk.h" +#include "compressed_block_device.h" + +#ifdef EFI_USE_COMPRESSED_INI_MSD +#include "ramdisk_image_compressed.h" +#else #include "ramdisk_image.h" +#endif // If the ramdisk image told us not to use it, don't use it. #ifdef RAMDISK_INVALID @@ -78,7 +84,11 @@ static const struct BaseBlockDeviceVMT ndVmt = { }; #if EFI_EMBED_INI_MSD +#ifdef EFI_USE_COMPRESSED_INI_MSD +static CompressedBlockDevice cbd; +#else static RamDisk ramdisk; +#endif #else // This device is always ready and has no state static NullDevice nd = { &ndVmt, BLK_READY }; @@ -89,6 +99,12 @@ void msdMountNullDevice(USBMassStorageDriver* msdp, USBDriver *usbp, uint8_t* bl // TODO: implement multi-LUN so we can mount the ini image and SD card at the same time #if EFI_EMBED_INI_MSD +#ifdef EFI_USE_COMPRESSED_INI_MSD + uzlib_init(); + compressedBlockDeviceObjectInit(&cbd); + compressedBlockDeviceStart(&cbd, ramdisk_image_gz, sizeof(ramdisk_image_gz)); + msdStart(msdp, usbp, (BaseBlockDevice*)&cbd, blkbuf, inquiry, nullptr); +#else // not EFI_USE_COMPRESSED_INI_MSD ramdiskObjectInit(&ramdisk); constexpr size_t ramdiskSize = sizeof(ramdisk_image); @@ -101,8 +117,8 @@ void msdMountNullDevice(USBMassStorageDriver* msdp, USBDriver *usbp, uint8_t* bl ramdiskStart(&ramdisk, const_cast(ramdisk_image), blockSize, blockCount, /*readonly =*/ true); msdStart(msdp, usbp, (BaseBlockDevice*)&ramdisk, blkbuf, inquiry, nullptr); - -#else +#endif // EFI_USE_COMPRESSED_INI_MSD +#else // not EFI_EMBED_INI_MSD // No embedded ini file, just mount the null device instead msdStart(msdp, usbp, (BaseBlockDevice*)&nd, blkbuf, inquiry, nullptr); #endif diff --git a/firmware/hw_layer/mass_storage/ramdisk_image_compressed.h b/firmware/hw_layer/mass_storage/ramdisk_image_compressed.h new file mode 100644 index 0000000000..3bdbd9a758 --- /dev/null +++ b/firmware/hw_layer/mass_storage/ramdisk_image_compressed.h @@ -0,0 +1,4 @@ +// This file will be replaced by a ramdisk image containing the corresponding ini +// Defining this macro tells the ramdisk to instead mount a null device instead of +// the filesystem image, since we don't have one. +#define RAMDISK_INVALID diff --git a/firmware/hw_layer/ports/rusefi_halconf.h b/firmware/hw_layer/ports/rusefi_halconf.h index ba416348b4..45ed76c0c2 100644 --- a/firmware/hw_layer/ports/rusefi_halconf.h +++ b/firmware/hw_layer/ports/rusefi_halconf.h @@ -1,6 +1,7 @@ #pragma once #include "efifeatures.h" +#include "thread_priority.h" /*===========================================================================*/ /* Conditional EFI feature settings */ @@ -71,6 +72,12 @@ #define SERIAL_USB_BUFFERS_SIZE 320 #define SERIAL_USB_BUFFERS_NUMBER 2 +// USB Mass Storage +#ifdef EFI_USE_COMPRESSED_INI_MSD +// if enabled, we do gzip decompression on the MSD thread - it requires more stack space +#define USB_MSD_THREAD_WA_SIZE 512 +#endif + // SPI #define SPI_USE_WAIT TRUE #define SPI_USE_MUTUAL_EXCLUSION TRUE