400 lines
12 KiB
Rust
400 lines
12 KiB
Rust
|
use csv::WriterBuilder;
|
|||
|
use hashbrown::HashMap;
|
|||
|
use serde::{
|
|||
|
ser::{SerializeMap},
|
|||
|
Serialize, Serializer,
|
|||
|
};
|
|||
|
use std::{
|
|||
|
fs::File,
|
|||
|
io::{LineWriter, Read, Write},
|
|||
|
path::PathBuf,
|
|||
|
str, usize,
|
|||
|
};
|
|||
|
|
|||
|
const FORMAT_LENGTH: usize = 6;
|
|||
|
const LOGGER_FIELD_LENGTH: i16 = 55;
|
|||
|
const FIELD_NAME_LENGTH: usize = 34;
|
|||
|
const FIELD_UNITS_LENGTH: usize = 10;
|
|||
|
const MARKER_MESSAGE_LENGTH: usize = 50;
|
|||
|
const TYPE_FIELD: &str = "field";
|
|||
|
const TYPE_MARKER: &str = "marker";
|
|||
|
const BLOCK_TYPE_FIELD: i8 = 0;
|
|||
|
const BLOCK_TYPE_MARKER: i8 = 1;
|
|||
|
const FIELD_DISPLAY_STYLE_FLOAT: &str = "Float";
|
|||
|
|
|||
|
#[derive(Debug, Serialize)]
|
|||
|
#[serde(rename_all = "camelCase")]
|
|||
|
struct Parsed {
|
|||
|
file_format: String,
|
|||
|
format_version: i16,
|
|||
|
timestamp: i32,
|
|||
|
info_data_start: i16,
|
|||
|
data_begin_index: i32,
|
|||
|
record_length: i16,
|
|||
|
num_logger_fields: i16,
|
|||
|
fields: Vec<LoggerFieldScalar>,
|
|||
|
bit_field_names: String,
|
|||
|
info_data: String,
|
|||
|
data_blocks: Vec<DataBlock>,
|
|||
|
}
|
|||
|
|
|||
|
#[derive(Debug, Serialize)]
|
|||
|
#[serde(rename_all = "camelCase")]
|
|||
|
struct DataBlockField {
|
|||
|
block_type: i8,
|
|||
|
counter: i8,
|
|||
|
timestamp: u16,
|
|||
|
}
|
|||
|
|
|||
|
type Records = HashMap<String, f64>;
|
|||
|
|
|||
|
#[derive(Debug, Serialize)]
|
|||
|
#[serde(rename_all = "camelCase")]
|
|||
|
struct BlockHeader {
|
|||
|
block_type: i8,
|
|||
|
counter: i8,
|
|||
|
timestamp: u16,
|
|||
|
}
|
|||
|
|
|||
|
#[derive(Debug, Serialize)]
|
|||
|
#[serde(rename_all = "camelCase")]
|
|||
|
struct LoggerFieldScalar {
|
|||
|
field_type: i8,
|
|||
|
name: String,
|
|||
|
units: String,
|
|||
|
display_style: String,
|
|||
|
scale: f32,
|
|||
|
transform: f32,
|
|||
|
digits: i8,
|
|||
|
}
|
|||
|
|
|||
|
#[derive(Debug)]
|
|||
|
struct DataBlock {
|
|||
|
block_type: i8,
|
|||
|
counter: i8,
|
|||
|
timestamp: u16,
|
|||
|
records: Records,
|
|||
|
message: String, // marker block
|
|||
|
}
|
|||
|
|
|||
|
impl Serialize for DataBlock {
|
|||
|
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
|||
|
where
|
|||
|
S: Serializer,
|
|||
|
{
|
|||
|
let mut map = serializer.serialize_map(Some(self.records.len() + 2))?;
|
|||
|
|
|||
|
// serialize normal fields
|
|||
|
map.serialize_entry(&"timestamp", &self.timestamp)?;
|
|||
|
map.serialize_entry(
|
|||
|
&"type",
|
|||
|
match self.block_type {
|
|||
|
BLOCK_TYPE_FIELD => TYPE_FIELD,
|
|||
|
BLOCK_TYPE_MARKER => TYPE_MARKER,
|
|||
|
_ => panic!("Unsupported Block Type"),
|
|||
|
},
|
|||
|
)?;
|
|||
|
|
|||
|
// serialize either message (marker) or hashmap (records)
|
|||
|
match self.block_type {
|
|||
|
BLOCK_TYPE_FIELD => {
|
|||
|
// serialize hash map
|
|||
|
for (k, v) in &self.records {
|
|||
|
map.serialize_entry(&k.to_string(), &v)?;
|
|||
|
}
|
|||
|
}
|
|||
|
BLOCK_TYPE_MARKER => map.serialize_entry(&"message", &self.message)?,
|
|||
|
_ => (),
|
|||
|
}
|
|||
|
|
|||
|
map.end()
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
pub enum Formats {
|
|||
|
Csv,
|
|||
|
Json,
|
|||
|
}
|
|||
|
|
|||
|
pub fn parse(paths: Vec<&PathBuf>, format: Formats) {
|
|||
|
for path in paths {
|
|||
|
let parsed = parse_single_file(path);
|
|||
|
|
|||
|
match &parsed {
|
|||
|
Ok(_) => {}
|
|||
|
Err(e) => return println!("Error in [{}]: {}", path.display(), e),
|
|||
|
}
|
|||
|
|
|||
|
match format {
|
|||
|
Formats::Csv => {
|
|||
|
let filepath = path.with_extension("csv");
|
|||
|
save_csv(&parsed.unwrap(), &filepath);
|
|||
|
println!("Generated: {}", filepath.display());
|
|||
|
}
|
|||
|
Formats::Json => {
|
|||
|
let json = serde_json::to_string(&parsed).expect("Unable to serialize the result");
|
|||
|
let filepath = path.with_extension("json");
|
|||
|
File::create(&filepath)
|
|||
|
.unwrap()
|
|||
|
.write_all(json.as_bytes())
|
|||
|
.expect("Unable to save output file");
|
|||
|
println!("Generated: {}", &filepath.display());
|
|||
|
}
|
|||
|
}
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
fn parse_single_file(path: &PathBuf) -> Result<Parsed, &str> {
|
|||
|
let mut file = File::open(path).expect("Unable to open file");
|
|||
|
let mut buff = Vec::new();
|
|||
|
let mut offset: usize = 0;
|
|||
|
|
|||
|
file.read_to_end(&mut buff).expect("Unable to read file");
|
|||
|
|
|||
|
let mut result = Parsed {
|
|||
|
file_format: "".to_string(),
|
|||
|
format_version: 0,
|
|||
|
timestamp: 0,
|
|||
|
info_data_start: 0,
|
|||
|
data_begin_index: 0,
|
|||
|
record_length: 0,
|
|||
|
num_logger_fields: 0,
|
|||
|
fields: Vec::new(),
|
|||
|
bit_field_names: "".to_string(),
|
|||
|
info_data: "".to_string(),
|
|||
|
data_blocks: Vec::new(),
|
|||
|
};
|
|||
|
|
|||
|
result.file_format = parse_string(&buff, &mut offset, FORMAT_LENGTH);
|
|||
|
|
|||
|
if result.file_format != "MLVLG" {
|
|||
|
return Err("Unsupported file format");
|
|||
|
}
|
|||
|
|
|||
|
result.format_version = parse_i16(&buff, &mut offset);
|
|||
|
|
|||
|
if result.format_version != 1 {
|
|||
|
return Err("Unsupported file format version");
|
|||
|
}
|
|||
|
|
|||
|
result.timestamp = parse_i32(&buff, &mut offset);
|
|||
|
result.info_data_start = parse_i16(&buff, &mut offset);
|
|||
|
result.data_begin_index = parse_i32(&buff, &mut offset);
|
|||
|
result.record_length = parse_i16(&buff, &mut offset);
|
|||
|
result.num_logger_fields = parse_i16(&buff, &mut offset);
|
|||
|
|
|||
|
let logger_fields_length = offset + (result.num_logger_fields * LOGGER_FIELD_LENGTH) as usize;
|
|||
|
|
|||
|
while offset < logger_fields_length {
|
|||
|
result.fields.push(LoggerFieldScalar {
|
|||
|
field_type: parse_i8(&buff, &mut offset),
|
|||
|
name: parse_string(&buff, &mut offset, FIELD_NAME_LENGTH),
|
|||
|
units: parse_string(&buff, &mut offset, FIELD_UNITS_LENGTH),
|
|||
|
display_style: match parse_i8(&buff, &mut offset) {
|
|||
|
0 => "Float".to_string(),
|
|||
|
1 => "Hex".to_string(),
|
|||
|
2 => "bits".to_string(),
|
|||
|
3 => "Date".to_string(),
|
|||
|
4 => "On/Off".to_string(),
|
|||
|
5 => "Yes/No".to_string(),
|
|||
|
6 => "High/Low".to_string(),
|
|||
|
7 => "Active/Inactive".to_string(),
|
|||
|
_ => panic!("Unsupported Field Display Style"),
|
|||
|
},
|
|||
|
scale: parse_f32(&buff, &mut offset),
|
|||
|
transform: parse_f32(&buff, &mut offset),
|
|||
|
digits: parse_i8(&buff, &mut offset),
|
|||
|
});
|
|||
|
}
|
|||
|
|
|||
|
result.bit_field_names = parse_string(
|
|||
|
&buff,
|
|||
|
&mut offset,
|
|||
|
result.info_data_start as usize - logger_fields_length,
|
|||
|
);
|
|||
|
|
|||
|
jump(&mut offset, result.info_data_start as usize);
|
|||
|
|
|||
|
result.info_data = parse_string(
|
|||
|
&buff,
|
|||
|
&mut offset,
|
|||
|
(result.data_begin_index - result.info_data_start as i32) as usize,
|
|||
|
);
|
|||
|
|
|||
|
jump(&mut offset, result.data_begin_index as usize);
|
|||
|
|
|||
|
while offset < buff.len() {
|
|||
|
// TODO: report progress every X record
|
|||
|
let mut records: Records = HashMap::new();
|
|||
|
let header = BlockHeader {
|
|||
|
block_type: parse_i8(&buff, &mut offset),
|
|||
|
counter: parse_i8(&buff, &mut offset),
|
|||
|
timestamp: parse_u16(&buff, &mut offset),
|
|||
|
};
|
|||
|
match header.block_type {
|
|||
|
BLOCK_TYPE_FIELD => {
|
|||
|
for field in result.fields.iter() {
|
|||
|
records.insert(
|
|||
|
field.name.to_string(),
|
|||
|
match field.field_type {
|
|||
|
// Logger Field – scalar
|
|||
|
0 => parse_u8(&buff, &mut offset) as f64,
|
|||
|
1 => parse_i8(&buff, &mut offset) as f64,
|
|||
|
2 => parse_u16(&buff, &mut offset) as f64,
|
|||
|
3 => parse_i16(&buff, &mut offset) as f64,
|
|||
|
4 => parse_u32(&buff, &mut offset) as f64,
|
|||
|
5 => parse_i32(&buff, &mut offset) as f64,
|
|||
|
6 => parse_i64(&buff, &mut offset) as f64,
|
|||
|
7 => parse_f32(&buff, &mut offset) as f64,
|
|||
|
// Logger Field - Bit
|
|||
|
10 => parse_u8(&buff, &mut offset) as f64,
|
|||
|
11 => parse_u16(&buff, &mut offset) as f64,
|
|||
|
12 => parse_u32(&buff, &mut offset) as f64,
|
|||
|
_ => panic!("Unsupported Field Type"),
|
|||
|
},
|
|||
|
);
|
|||
|
}
|
|||
|
|
|||
|
// don't parse "crc" (not needed for now), just advance offset
|
|||
|
advance(&mut offset, std::mem::size_of::<u8>());
|
|||
|
|
|||
|
result.data_blocks.push(DataBlock {
|
|||
|
block_type: header.block_type,
|
|||
|
counter: header.counter,
|
|||
|
timestamp: header.timestamp,
|
|||
|
records,
|
|||
|
message: "".to_string(),
|
|||
|
});
|
|||
|
}
|
|||
|
BLOCK_TYPE_MARKER => result.data_blocks.push(DataBlock {
|
|||
|
block_type: header.block_type,
|
|||
|
counter: header.counter,
|
|||
|
timestamp: header.timestamp,
|
|||
|
records,
|
|||
|
message: parse_string(&buff, &mut offset, MARKER_MESSAGE_LENGTH),
|
|||
|
}),
|
|||
|
_ => panic!("Unsupported Block Type"),
|
|||
|
};
|
|||
|
}
|
|||
|
|
|||
|
Ok(result)
|
|||
|
}
|
|||
|
|
|||
|
fn save_csv(parsed: &Parsed, path: &PathBuf) {
|
|||
|
let line_writer = LineWriter::new(File::create(path).unwrap());
|
|||
|
let mut writer = WriterBuilder::new()
|
|||
|
.delimiter(b'\t')
|
|||
|
.from_writer(line_writer);
|
|||
|
|
|||
|
let mut header: Vec<String> = Vec::new();
|
|||
|
parsed
|
|||
|
.fields
|
|||
|
.iter()
|
|||
|
.for_each(|field| header.push(field.name.to_string()));
|
|||
|
|
|||
|
writer.write_record(header).unwrap();
|
|||
|
|
|||
|
let mut units: Vec<String> = Vec::new();
|
|||
|
parsed
|
|||
|
.fields
|
|||
|
.iter()
|
|||
|
.for_each(|field| units.push(field.units.to_string()));
|
|||
|
|
|||
|
writer.write_record(units).unwrap();
|
|||
|
|
|||
|
for block in parsed.data_blocks.iter() {
|
|||
|
let mut row: Vec<String> = Vec::new();
|
|||
|
|
|||
|
if block.block_type == BLOCK_TYPE_FIELD {
|
|||
|
for field in parsed.fields.iter() {
|
|||
|
let value = (block.records.get(&field.name).unwrap() + field.transform as f64)
|
|||
|
* field.scale as f64;
|
|||
|
|
|||
|
if field.display_style == FIELD_DISPLAY_STYLE_FLOAT {
|
|||
|
row.push(format!("{:.1$}", value, field.digits as usize));
|
|||
|
} else {
|
|||
|
row.push(value.to_string());
|
|||
|
}
|
|||
|
}
|
|||
|
writer.write_record(row).unwrap();
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
writer.flush().unwrap();
|
|||
|
}
|
|||
|
|
|||
|
fn advance(offset: &mut usize, length: usize) {
|
|||
|
*offset += length;
|
|||
|
}
|
|||
|
|
|||
|
fn jump(offset: &mut usize, to: usize) {
|
|||
|
*offset = to;
|
|||
|
}
|
|||
|
|
|||
|
fn parse_string(buff: &[u8], offset: &mut usize, length: usize) -> String {
|
|||
|
let val = str::from_utf8(&buff[*offset..(*offset + length)])
|
|||
|
.expect("Unable to parse string")
|
|||
|
.trim_matches(char::from(0))
|
|||
|
.to_string();
|
|||
|
advance(offset, length);
|
|||
|
val
|
|||
|
}
|
|||
|
|
|||
|
fn parse_i8(buff: &[u8], offset: &mut usize) -> i8 {
|
|||
|
let length = std::mem::size_of::<i8>();
|
|||
|
let buff = &buff[*offset..(*offset + length)];
|
|||
|
advance(offset, length);
|
|||
|
i8::from_be_bytes([buff[0]])
|
|||
|
}
|
|||
|
|
|||
|
fn parse_u8(buff: &[u8], offset: &mut usize) -> u8 {
|
|||
|
let length = std::mem::size_of::<u8>();
|
|||
|
let buff = &buff[*offset..(*offset + length)];
|
|||
|
advance(offset, length);
|
|||
|
u8::from_be_bytes([buff[0]])
|
|||
|
}
|
|||
|
|
|||
|
fn parse_i16(buff: &[u8], offset: &mut usize) -> i16 {
|
|||
|
let length = std::mem::size_of::<i16>();
|
|||
|
let buff = &buff[*offset..(*offset + length)];
|
|||
|
advance(offset, length);
|
|||
|
i16::from_be_bytes([buff[0], buff[1]])
|
|||
|
}
|
|||
|
|
|||
|
fn parse_u16(buff: &[u8], offset: &mut usize) -> u16 {
|
|||
|
let length = std::mem::size_of::<u16>();
|
|||
|
let buff = &buff[*offset..(*offset + length)];
|
|||
|
advance(offset, length);
|
|||
|
u16::from_be_bytes([buff[0], buff[1]])
|
|||
|
}
|
|||
|
|
|||
|
fn parse_i32(buff: &[u8], offset: &mut usize) -> i32 {
|
|||
|
let length = std::mem::size_of::<i32>();
|
|||
|
let buff = &buff[*offset..(*offset + length)];
|
|||
|
advance(offset, length);
|
|||
|
i32::from_be_bytes([buff[0], buff[1], buff[2], buff[3]])
|
|||
|
}
|
|||
|
|
|||
|
fn parse_u32(buff: &[u8], offset: &mut usize) -> u32 {
|
|||
|
let length = std::mem::size_of::<u32>();
|
|||
|
let buff = &buff[*offset..(*offset + length)];
|
|||
|
advance(offset, length);
|
|||
|
u32::from_be_bytes([buff[0], buff[1], buff[2], buff[3]])
|
|||
|
}
|
|||
|
|
|||
|
fn parse_f32(buff: &[u8], offset: &mut usize) -> f32 {
|
|||
|
let length = std::mem::size_of::<f32>();
|
|||
|
let buff = &buff[*offset..(*offset + length)];
|
|||
|
advance(offset, length);
|
|||
|
f32::from_be_bytes([buff[0], buff[1], buff[2], buff[3]])
|
|||
|
}
|
|||
|
|
|||
|
fn parse_i64(buff: &[u8], offset: &mut usize) -> i64 {
|
|||
|
let length = std::mem::size_of::<i64>();
|
|||
|
let buff = &buff[*offset..(*offset + length)];
|
|||
|
advance(offset, length);
|
|||
|
i64::from_be_bytes([
|
|||
|
buff[0], buff[1], buff[2], buff[3], buff[4], buff[5], buff[6], buff[7],
|
|||
|
])
|
|||
|
}
|