use csv::WriterBuilder; use hashbrown::HashMap; use serde::{ ser::{SerializeMap}, Serialize, Serializer, }; use std::{ fs::File, io::{LineWriter, Read, Write}, path::PathBuf, str, usize, }; const FORMAT_LENGTH: usize = 6; const LOGGER_FIELD_LENGTH: i16 = 55; const FIELD_NAME_LENGTH: usize = 34; const FIELD_UNITS_LENGTH: usize = 10; const MARKER_MESSAGE_LENGTH: usize = 50; const TYPE_FIELD: &str = "field"; const TYPE_MARKER: &str = "marker"; const BLOCK_TYPE_FIELD: i8 = 0; const BLOCK_TYPE_MARKER: i8 = 1; const FIELD_DISPLAY_STYLE_FLOAT: &str = "Float"; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct Parsed { file_format: String, format_version: i16, timestamp: i32, info_data_start: i16, data_begin_index: i32, record_length: i16, num_logger_fields: i16, fields: Vec, bit_field_names: String, info_data: String, data_blocks: Vec, } #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct DataBlockField { block_type: i8, counter: i8, timestamp: u16, } type Records = HashMap; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct BlockHeader { block_type: i8, counter: i8, timestamp: u16, } #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct LoggerFieldScalar { field_type: i8, name: String, units: String, display_style: String, scale: f32, transform: f32, digits: i8, } #[derive(Debug)] struct DataBlock { block_type: i8, counter: i8, timestamp: u16, records: Records, message: String, // marker block } impl Serialize for DataBlock { fn serialize(&self, serializer: S) -> Result where S: Serializer, { let mut map = serializer.serialize_map(Some(self.records.len() + 2))?; // serialize normal fields map.serialize_entry(&"timestamp", &self.timestamp)?; map.serialize_entry( &"type", match self.block_type { BLOCK_TYPE_FIELD => TYPE_FIELD, BLOCK_TYPE_MARKER => TYPE_MARKER, _ => panic!("Unsupported Block Type"), }, )?; // serialize either message (marker) or hashmap (records) match self.block_type { BLOCK_TYPE_FIELD => { // serialize hash map for (k, v) in &self.records { map.serialize_entry(&k.to_string(), &v)?; } } BLOCK_TYPE_MARKER => map.serialize_entry(&"message", &self.message)?, _ => (), } map.end() } } pub enum Formats { Csv, Json, } pub fn parse(paths: Vec<&PathBuf>, format: Formats) { for path in paths { let parsed = parse_single_file(path); match &parsed { Ok(_) => {} Err(e) => return println!("Error in [{}]: {}", path.display(), e), } match format { Formats::Csv => { let filepath = path.with_extension("csv"); save_csv(&parsed.unwrap(), &filepath); println!("Generated: {}", filepath.display()); } Formats::Json => { let json = serde_json::to_string(&parsed).expect("Unable to serialize the result"); let filepath = path.with_extension("json"); File::create(&filepath) .unwrap() .write_all(json.as_bytes()) .expect("Unable to save output file"); println!("Generated: {}", &filepath.display()); } } } } fn parse_single_file(path: &PathBuf) -> Result { let mut file = File::open(path).expect("Unable to open file"); let mut buff = Vec::new(); let mut offset: usize = 0; file.read_to_end(&mut buff).expect("Unable to read file"); let mut result = Parsed { file_format: "".to_string(), format_version: 0, timestamp: 0, info_data_start: 0, data_begin_index: 0, record_length: 0, num_logger_fields: 0, fields: Vec::new(), bit_field_names: "".to_string(), info_data: "".to_string(), data_blocks: Vec::new(), }; result.file_format = parse_string(&buff, &mut offset, FORMAT_LENGTH); if result.file_format != "MLVLG" { return Err("Unsupported file format"); } result.format_version = parse_i16(&buff, &mut offset); if result.format_version != 1 { return Err("Unsupported file format version"); } result.timestamp = parse_i32(&buff, &mut offset); result.info_data_start = parse_i16(&buff, &mut offset); result.data_begin_index = parse_i32(&buff, &mut offset); result.record_length = parse_i16(&buff, &mut offset); result.num_logger_fields = parse_i16(&buff, &mut offset); let logger_fields_length = offset + (result.num_logger_fields * LOGGER_FIELD_LENGTH) as usize; while offset < logger_fields_length { result.fields.push(LoggerFieldScalar { field_type: parse_i8(&buff, &mut offset), name: parse_string(&buff, &mut offset, FIELD_NAME_LENGTH), units: parse_string(&buff, &mut offset, FIELD_UNITS_LENGTH), display_style: match parse_i8(&buff, &mut offset) { 0 => "Float".to_string(), 1 => "Hex".to_string(), 2 => "bits".to_string(), 3 => "Date".to_string(), 4 => "On/Off".to_string(), 5 => "Yes/No".to_string(), 6 => "High/Low".to_string(), 7 => "Active/Inactive".to_string(), _ => panic!("Unsupported Field Display Style"), }, scale: parse_f32(&buff, &mut offset), transform: parse_f32(&buff, &mut offset), digits: parse_i8(&buff, &mut offset), }); } result.bit_field_names = parse_string( &buff, &mut offset, result.info_data_start as usize - logger_fields_length, ); jump(&mut offset, result.info_data_start as usize); result.info_data = parse_string( &buff, &mut offset, (result.data_begin_index - result.info_data_start as i32) as usize, ); jump(&mut offset, result.data_begin_index as usize); while offset < buff.len() { // TODO: report progress every X record let mut records: Records = HashMap::new(); let header = BlockHeader { block_type: parse_i8(&buff, &mut offset), counter: parse_i8(&buff, &mut offset), timestamp: parse_u16(&buff, &mut offset), }; match header.block_type { BLOCK_TYPE_FIELD => { for field in result.fields.iter() { records.insert( field.name.to_string(), match field.field_type { // Logger Field – scalar 0 => parse_u8(&buff, &mut offset) as f64, 1 => parse_i8(&buff, &mut offset) as f64, 2 => parse_u16(&buff, &mut offset) as f64, 3 => parse_i16(&buff, &mut offset) as f64, 4 => parse_u32(&buff, &mut offset) as f64, 5 => parse_i32(&buff, &mut offset) as f64, 6 => parse_i64(&buff, &mut offset) as f64, 7 => parse_f32(&buff, &mut offset) as f64, // Logger Field - Bit 10 => parse_u8(&buff, &mut offset) as f64, 11 => parse_u16(&buff, &mut offset) as f64, 12 => parse_u32(&buff, &mut offset) as f64, _ => panic!("Unsupported Field Type"), }, ); } // don't parse "crc" (not needed for now), just advance offset advance(&mut offset, std::mem::size_of::()); result.data_blocks.push(DataBlock { block_type: header.block_type, counter: header.counter, timestamp: header.timestamp, records, message: "".to_string(), }); } BLOCK_TYPE_MARKER => result.data_blocks.push(DataBlock { block_type: header.block_type, counter: header.counter, timestamp: header.timestamp, records, message: parse_string(&buff, &mut offset, MARKER_MESSAGE_LENGTH), }), _ => panic!("Unsupported Block Type"), }; } Ok(result) } fn save_csv(parsed: &Parsed, path: &PathBuf) { let line_writer = LineWriter::new(File::create(path).unwrap()); let mut writer = WriterBuilder::new() .delimiter(b'\t') .from_writer(line_writer); let mut header: Vec = Vec::new(); parsed .fields .iter() .for_each(|field| header.push(field.name.to_string())); writer.write_record(header).unwrap(); let mut units: Vec = Vec::new(); parsed .fields .iter() .for_each(|field| units.push(field.units.to_string())); writer.write_record(units).unwrap(); for block in parsed.data_blocks.iter() { let mut row: Vec = Vec::new(); if block.block_type == BLOCK_TYPE_FIELD { for field in parsed.fields.iter() { let value = (block.records.get(&field.name).unwrap() + field.transform as f64) * field.scale as f64; if field.display_style == FIELD_DISPLAY_STYLE_FLOAT { row.push(format!("{:.1$}", value, field.digits as usize)); } else { row.push(value.to_string()); } } writer.write_record(row).unwrap(); } } writer.flush().unwrap(); } fn advance(offset: &mut usize, length: usize) { *offset += length; } fn jump(offset: &mut usize, to: usize) { *offset = to; } fn parse_string(buff: &[u8], offset: &mut usize, length: usize) -> String { let val = str::from_utf8(&buff[*offset..(*offset + length)]) .expect("Unable to parse string") .trim_matches(char::from(0)) .to_string(); advance(offset, length); val } fn parse_i8(buff: &[u8], offset: &mut usize) -> i8 { let length = std::mem::size_of::(); let buff = &buff[*offset..(*offset + length)]; advance(offset, length); i8::from_be_bytes([buff[0]]) } fn parse_u8(buff: &[u8], offset: &mut usize) -> u8 { let length = std::mem::size_of::(); let buff = &buff[*offset..(*offset + length)]; advance(offset, length); u8::from_be_bytes([buff[0]]) } fn parse_i16(buff: &[u8], offset: &mut usize) -> i16 { let length = std::mem::size_of::(); let buff = &buff[*offset..(*offset + length)]; advance(offset, length); i16::from_be_bytes([buff[0], buff[1]]) } fn parse_u16(buff: &[u8], offset: &mut usize) -> u16 { let length = std::mem::size_of::(); let buff = &buff[*offset..(*offset + length)]; advance(offset, length); u16::from_be_bytes([buff[0], buff[1]]) } fn parse_i32(buff: &[u8], offset: &mut usize) -> i32 { let length = std::mem::size_of::(); let buff = &buff[*offset..(*offset + length)]; advance(offset, length); i32::from_be_bytes([buff[0], buff[1], buff[2], buff[3]]) } fn parse_u32(buff: &[u8], offset: &mut usize) -> u32 { let length = std::mem::size_of::(); let buff = &buff[*offset..(*offset + length)]; advance(offset, length); u32::from_be_bytes([buff[0], buff[1], buff[2], buff[3]]) } fn parse_f32(buff: &[u8], offset: &mut usize) -> f32 { let length = std::mem::size_of::(); let buff = &buff[*offset..(*offset + length)]; advance(offset, length); f32::from_be_bytes([buff[0], buff[1], buff[2], buff[3]]) } fn parse_i64(buff: &[u8], offset: &mut usize) -> i64 { let length = std::mem::size_of::(); let buff = &buff[*offset..(*offset + length)]; advance(offset, length); i64::from_be_bytes([ buff[0], buff[1], buff[2], buff[3], buff[4], buff[5], buff[6], buff[7], ]) }