Store ini filesystem in compressed block device (#2420)

* add module

* implement compressed block device

* enable for f7/h7

* script

* stub compressed image

* implement

* call the script

* headers, comments, fix init bug

* set msd options

* update chibios-contrib

Co-authored-by: Matthew Kennedy <makenne@microsoft.com>

.gitmodules

@@ -21,3 +21,6 @@
 [submodule "firmware/controllers/can/wideband_firmware"]
 	path = firmware/controllers/can/wideband_firmware
 	url = https://github.com/mck1117/wideband
+[submodule "firmware/ext/uzlib"]
+	path = firmware/ext/uzlib
+	url = https://github.com/pfalcon/uzlib

firmware/ChibiOS-Contrib

@@ -1 +1 @@
-Subproject commit e7cdad3d1a041caea83729fd2c709fd963fb40b0
+Subproject commit fdcbac6a8c13dc5ec55a31cbe4f217edd7dfca02

firmware/config/stm32f7ems/efifeatures.h

@@ -63,3 +63,5 @@
 // todo: start using consoleSerialRxPin? Not sure
 #undef EFI_CONSOLE_RX_BRAIN_PIN
 #define EFI_CONSOLE_RX_BRAIN_PIN GPIOD_9
+
+#define EFI_USE_COMPRESSED_INI_MSD

firmware/config/stm32h7ems/efifeatures.h

@@ -40,4 +40,6 @@
 #define EFI_MAX_31855 FALSE
 
 #undef BOARD_EXT_GPIOCHIPS
- #define BOARD_EXT_GPIOCHIPS			(BOARD_TLE6240_COUNT + BOARD_MC33972_COUNT + BOARD_TLE8888_COUNT + BOARD_DRV8860_COUNT + BOARD_MC33810_COUNT)
+#define BOARD_EXT_GPIOCHIPS			(BOARD_TLE6240_COUNT + BOARD_MC33972_COUNT + BOARD_TLE8888_COUNT + BOARD_DRV8860_COUNT + BOARD_MC33810_COUNT)
+
+#define EFI_USE_COMPRESSED_INI_MSD

firmware/controllers/thread_priority.h

@@ -29,6 +29,8 @@
 
 // Less important things
 #define PRIO_MMC (NORMALPRIO - 1)
+// USB mass storage
+#define MSD_THD_PRIO LOWPRIO
 
 // These can get starved without too much adverse effect
 #define PRIO_AUX_SERIAL NORMALPRIO

firmware/ext/uzlib

@@ -0,0 +1 @@
+Subproject commit 27e4f4c15ba30c2cfc89575159e8efb50f95037e

firmware/gen_config_board.sh

@@ -39,5 +39,6 @@ java -DSystemOut.name=gen_config_board \
 [ $? -eq 0 ] || { echo "ERROR generating TunerStudio config for ${BOARDNAME}"; exit 1; }
 
 ./hw_layer/mass_storage/create_ini_image.sh ./tunerstudio/generated/rusefi_${SHORT_BOARDNAME}.ini ./hw_layer/mass_storage/ramdisk_image.h
+./hw_layer/mass_storage/create_ini_image_compressed.sh ./tunerstudio/generated/rusefi_${SHORT_BOARDNAME}.ini ./hw_layer/mass_storage/ramdisk_image_compressed.h
 
 exit 0

firmware/hw_layer/mass_storage/compressed_block_device.cpp

@@ -0,0 +1,149 @@
+/**
+ * @file	compressed_block_device.cpp
+ * @brief	This file implements a ChibiOS block device backed by a compressed (gzip) store.
+ *
+ * @date	Mar 4, 2021
+ * @author	Matthew Kennedy, (c) 2021
+ *
+ * This works by decompressing one block (512 bytes) at a time.
+ *
+ * For sequential reads, the performance is great - the gzip decompress can only go forwards in the file.
+ * If a block later in the file (but with a gap) is requested, we decompress (and discard) the blocks in the gap,
+ * returning the block requested.
+ *
+ * If a block is requested from before the previous block, we discard decompression state,
+ * reinitialize, and decompress up to that block.
+ *
+ * NOTE: This means performance is terrible for "true" random access!  Things go best when you have a few
+ * big files in the filesystem with no fragmentation, so they can be read out in large sequential chunks.
+ *
+ */
+
+#include "compressed_block_device.h"
+
+#include <cstring>
+
+#define BLOCK_SIZE 512
+
+static bool is_inserted(void*) {
+	// Device is always inserted
+	return true;
+}
+
+static bool is_protected(void*) {
+	// Write protected - we can't do random access writes to a compressed volume
+	return true;
+}
+
+static bool connect(void* instance) {
+  CompressedBlockDevice* cbd = reinterpret_cast<CompressedBlockDevice*>(instance);
+  if (BLK_STOP == cbd->state) {
+    cbd->state = BLK_READY;
+  }
+  return HAL_SUCCESS;
+}
+
+static bool disconnect(void* instance) {
+	CompressedBlockDevice* cbd = reinterpret_cast<CompressedBlockDevice*>(instance);
+  if (BLK_STOP != cbd->state) {
+    cbd->state = BLK_STOP;
+  }
+  return HAL_SUCCESS;
+}
+
+static bool read(void* instance, uint32_t startblk, uint8_t* buffer, uint32_t n) {
+	CompressedBlockDevice* cbd = reinterpret_cast<CompressedBlockDevice*>(instance);
+
+	// If we just initialized, or trying to seek backwards, (re)initialize the decompressor
+	if (cbd->lastBlock == -1 || startblk <= cbd->lastBlock) {
+		uzlib_uncompress_init(&cbd->d, cbd->dictionary, sizeof(cbd->dictionary));
+
+		cbd->d.source = cbd->source;
+		cbd->d.source_limit = cbd->d.source + cbd->sourceSize;
+		cbd->d.source_read_cb = NULL;
+
+		uzlib_gzip_parse_header(&cbd->d);
+
+		cbd->lastBlock = -1;
+	}
+
+	// How many blocks do we need to decompress to get to the one requested?
+	size_t blocks_ahead = startblk - cbd->lastBlock;
+
+	// Decompress blocks until we get to the block we need
+	for (size_t i = 0; i < blocks_ahead; i++) {
+		cbd->d.dest = cbd->d.dest_start = buffer;
+		cbd->d.dest_limit = buffer + BLOCK_SIZE;
+
+		// Decompress one chunk
+		uzlib_uncompress(&cbd->d);
+	}
+
+	// Save the current position in the stream so we can efficiently seek forward later
+	cbd->lastBlock = startblk;
+
+	return HAL_SUCCESS;
+}
+
+static bool write(void*, uint32_t, const uint8_t*, uint32_t) {
+	// you shouldn't be able to do this anyway, so just swallow it, I guess?
+	return HAL_SUCCESS;
+}
+
+constexpr size_t gzSize(const uint8_t* image, size_t imageSize) {
+	// The last 4 bytes of the gzip stream encode the total size in bytes
+	const uint8_t* pSize = image + imageSize - 1;
+	size_t size = *pSize--;
+	size = 256 * size + *pSize--;
+	size = 256 * size + *pSize--;
+	return 256 * size + *pSize--;
+}
+
+static bool get_info(void* instance, BlockDeviceInfo* bdip) {
+	CompressedBlockDevice* cbd = reinterpret_cast<CompressedBlockDevice*>(instance);
+	if (cbd->state != BLK_READY) {
+		return HAL_FAILED;
+	}
+
+	// The last 4 bytes of the gzip stream encode the total size in bytes
+	size_t size = gzSize(cbd->source, cbd->sourceSize);
+
+	bdip->blk_num = size / BLOCK_SIZE;
+	bdip->blk_size = BLOCK_SIZE;
+	return HAL_SUCCESS;
+}
+
+static bool sync(void* instance) {
+	CompressedBlockDevice* cbd = reinterpret_cast<CompressedBlockDevice*>(instance);
+  if (BLK_READY != cbd->state) {
+    return HAL_FAILED;
+  }
+  else {
+    return HAL_SUCCESS;
+  }
+}
+
+static const BaseBlockDeviceVMT cbdVmt = {
+	(size_t)0, // instanceOffset
+	is_inserted,
+	is_protected,
+	connect,
+	disconnect,
+	read,
+	write,
+	sync,
+	get_info,
+};
+
+void compressedBlockDeviceObjectInit(CompressedBlockDevice* cbd) {
+	cbd->vmt = &cbdVmt;
+	memset(cbd->dictionary, 0, sizeof(cbd->dictionary));
+	cbd->state = BLK_STOP;
+}
+
+void compressedBlockDeviceStart(CompressedBlockDevice* cbd, const uint8_t* source, size_t sourceSize) {
+	cbd->source = source;
+	cbd->sourceSize = sourceSize;
+	cbd->state = BLK_READY;
+	cbd->lastBlock = -1;
+}

firmware/hw_layer/mass_storage/compressed_block_device.h

@@ -0,0 +1,25 @@
+/**
+ * @file	compressed_block_device.h
+ * @brief	This file implements a ChibiOS block device backed by a compressed (gzip) store.
+ *
+ * @date	Mar 4, 2021
+ * @author	Matthew Kennedy, (c) 2021
+ */
+
+#pragma once
+
+#include "hal.h"
+#include "uzlib.h"
+
+struct CompressedBlockDevice {
+	const BaseBlockDeviceVMT* vmt;
+	_base_block_device_data
+	int32_t lastBlock;
+	uzlib_uncomp d;
+	uint8_t dictionary[32768];
+	const uint8_t* source;
+	size_t sourceSize;
+};
+
+void compressedBlockDeviceObjectInit(CompressedBlockDevice* cbd);
+void compressedBlockDeviceStart(CompressedBlockDevice* cbd, const uint8_t* source, size_t sourceSize);

firmware/hw_layer/mass_storage/create_ini_image_compressed.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# fail on error
+set -e
+
+rm -f rusefi.zip ramdisk_image.h
+
+# copy 1MB of zeroes
+dd if=/dev/zero of=ramdisk.image bs=1024 count=1024
+
+# create a FAT filesystem inside, name it RUSEFI
+mkfs.fat ramdisk.image
+fatlabel ramdisk.image RUSEFI
+
+# Put the zip inside the filesystem
+mcopy -i ramdisk.image $1 ::
+# Put a readme text file in there too
+mcopy -i ramdisk.image hw_layer/mass_storage/README.txt ::
+
+# Compress the image as DEFLATE with gzip
+gzip ramdisk.image
+
+# write out as a C array, with "static const" tacked on the front
+xxd -i ramdisk.image.gz \
+    | cat <(echo -n "static const ") - \
+    > $2
+
+rm ramdisk.image.gz

firmware/hw_layer/mass_storage/mass_storage.mk

@@ -1,4 +1,9 @@
 
-HW_MASS_STORAGE_SRC_C = $(PROJECT_DIR)/ChibiOS-Contrib/os/various/lib_scsi.c
+HW_MASS_STORAGE_SRC_C = $(PROJECT_DIR)/ChibiOS-Contrib/os/various/lib_scsi.c \
+						ext/uzlib/src/tinflate.c \
+						ext/uzlib/src/tinfgzip.c
 
-ALLCPPSRC += $(PROJECT_DIR)/hw_layer/mass_storage/null_device.cpp
+ALLINC += $(PROJECT_DIR)/ext/uzlib/src
+
+ALLCPPSRC += $(PROJECT_DIR)/hw_layer/mass_storage/null_device.cpp \
+			 $(PROJECT_DIR)/hw_layer/mass_storage/compressed_block_device.cpp

firmware/hw_layer/mass_storage/null_device.cpp

@@ -12,7 +12,13 @@
 
 #if EFI_EMBED_INI_MSD
 #include "ramdisk.h"
+#include "compressed_block_device.h"
+
+#ifdef EFI_USE_COMPRESSED_INI_MSD
+#include "ramdisk_image_compressed.h"
+#else
 #include "ramdisk_image.h"
+#endif
 
 // If the ramdisk image told us not to use it, don't use it.
 #ifdef RAMDISK_INVALID
@@ -78,7 +84,11 @@ static const struct BaseBlockDeviceVMT ndVmt = {
 };
 
 #if EFI_EMBED_INI_MSD
+#ifdef EFI_USE_COMPRESSED_INI_MSD
+static CompressedBlockDevice cbd;
+#else
 static RamDisk ramdisk;
+#endif
 #else
 // This device is always ready and has no state
 static NullDevice nd = { &ndVmt, BLK_READY };
@@ -89,6 +99,12 @@ void msdMountNullDevice(USBMassStorageDriver* msdp, USBDriver *usbp, uint8_t* bl
 	// TODO: implement multi-LUN so we can mount the ini image and SD card at the same time
 
 #if EFI_EMBED_INI_MSD
+#ifdef EFI_USE_COMPRESSED_INI_MSD
+	uzlib_init();
+	compressedBlockDeviceObjectInit(&cbd);
+	compressedBlockDeviceStart(&cbd, ramdisk_image_gz, sizeof(ramdisk_image_gz));
+	msdStart(msdp, usbp, (BaseBlockDevice*)&cbd, blkbuf, inquiry, nullptr);
+#else // not EFI_USE_COMPRESSED_INI_MSD
 	ramdiskObjectInit(&ramdisk);
 
 	constexpr size_t ramdiskSize = sizeof(ramdisk_image);
@@ -101,8 +117,8 @@ void msdMountNullDevice(USBMassStorageDriver* msdp, USBDriver *usbp, uint8_t* bl
 	ramdiskStart(&ramdisk, const_cast<uint8_t*>(ramdisk_image), blockSize, blockCount, /*readonly =*/ true);
 
 	msdStart(msdp, usbp, (BaseBlockDevice*)&ramdisk, blkbuf, inquiry, nullptr);
-
-#else
+#endif // EFI_USE_COMPRESSED_INI_MSD
+#else // not EFI_EMBED_INI_MSD
 	// No embedded ini file, just mount the null device instead
 	msdStart(msdp, usbp, (BaseBlockDevice*)&nd, blkbuf, inquiry, nullptr);
 #endif

firmware/hw_layer/mass_storage/ramdisk_image_compressed.h

@@ -0,0 +1,4 @@
+// This file will be replaced by a ramdisk image containing the corresponding ini
+// Defining this macro tells the ramdisk to instead mount a null device instead of
+// the filesystem image, since we don't have one.
+#define RAMDISK_INVALID

firmware/hw_layer/ports/rusefi_halconf.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "efifeatures.h"
+#include "thread_priority.h"
 
 /*===========================================================================*/
 /* Conditional EFI feature settings                                          */
@@ -71,6 +72,12 @@
 #define SERIAL_USB_BUFFERS_SIZE     320
 #define SERIAL_USB_BUFFERS_NUMBER   2
 
+// USB Mass Storage
+#ifdef EFI_USE_COMPRESSED_INI_MSD
+// if enabled, we do gzip decompression on the MSD thread - it requires more stack space
+#define USB_MSD_THREAD_WA_SIZE 512
+#endif
+
 // SPI
 #define SPI_USE_WAIT                TRUE
 #define SPI_USE_MUTUAL_EXCLUSION    TRUE