Parse MSL logs (#823)

package-lock.json

@@ -17,7 +17,7 @@
         "antd": "^4.23.6",
         "kbar": "^0.1.0-beta.36",
         "lodash.debounce": "^4.0.8",
-        "mlg-converter": "^0.5.1",
+        "mlg-converter": "^0.6.0",
         "nanoid": "^4.0.0",
         "pako": "^2.0.4",
         "pocketbase": "^0.7.4",
@@ -7525,9 +7525,9 @@
       }
     },
     "node_modules/mlg-converter": {
-      "version": "0.5.1",
-      "resolved": "https://registry.npmjs.org/mlg-converter/-/mlg-converter-0.5.1.tgz",
-      "integrity": "sha512-t9jvCRmOpGLKBPHaSjnJt+83rBicWVV7qHLDtZ0M651zl8IMrOJ9jPp8W5zGqukKP5sOgxYWbgejIBSdJ1SAXQ=="
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/mlg-converter/-/mlg-converter-0.6.0.tgz",
+      "integrity": "sha512-hk5o+JP9bjb0qQkCIjkivXOuvyZxEY2ZJaeNUYatZUGovub8edFyWH0H5ZGdsyusNHi4FDboBUwsadSbUTxYjQ=="
     },
     "node_modules/moment": {
       "version": "2.29.4",
@@ -16034,9 +16034,9 @@
       "dev": true
     },
     "mlg-converter": {
-      "version": "0.5.1",
-      "resolved": "https://registry.npmjs.org/mlg-converter/-/mlg-converter-0.5.1.tgz",
-      "integrity": "sha512-t9jvCRmOpGLKBPHaSjnJt+83rBicWVV7qHLDtZ0M651zl8IMrOJ9jPp8W5zGqukKP5sOgxYWbgejIBSdJ1SAXQ=="
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/mlg-converter/-/mlg-converter-0.6.0.tgz",
+      "integrity": "sha512-hk5o+JP9bjb0qQkCIjkivXOuvyZxEY2ZJaeNUYatZUGovub8edFyWH0H5ZGdsyusNHi4FDboBUwsadSbUTxYjQ=="
     },
     "moment": {
       "version": "2.29.4",

package.json

@@ -28,7 +28,7 @@
     "antd": "^4.23.6",
     "kbar": "^0.1.0-beta.36",
     "lodash.debounce": "^4.0.8",
-    "mlg-converter": "^0.5.1",
+    "mlg-converter": "^0.6.0",
     "nanoid": "^4.0.0",
     "pako": "^2.0.4",
     "pocketbase": "^0.7.4",

src/pages/Logs.tsx

@@ -38,7 +38,7 @@ import {
   DatalogEntry,
 } from '@hyper-tuner/types';
 // eslint-disable-next-line import/no-unresolved
-import MlgParserWorker from '../workers/mlgParser?worker';
+import LogParserWorker from '../workers/logParserWorker?worker';
 import LogCanvas from '../components/Logs/LogCanvas';
 import store from '../store';
 import {
@@ -63,7 +63,7 @@ import useServerStorage from '../hooks/useServerStorage';
 import { Routes } from '../routes';
 import { removeFilenameSuffix } from '../pocketbase';
 import { isAbortedRequest } from '../utils/error';
-import { WorkerOutput } from '../workers/mlgParser';
+import { WorkerOutput } from '../workers/logParserWorker';
 
 const { Content } = Layout;
 const { Step } = Steps;
@@ -168,11 +168,10 @@ const Logs = ({
         format,
       };
     }).filter((val) => !!val);
-
   }, [config?.datalog, findOutputChannel, isConfigReady]);
 
   useEffect(() => {
-    const worker = new MlgParserWorker();
+    const worker = new LogParserWorker();
     const controller = new AbortController();
     const { signal } = controller;
 

src/pages/Upload.tsx

@@ -53,7 +53,7 @@ import { useAuth } from '../contexts/AuthContext';
 import { Routes } from '../routes';
 import TuneParser from '../utils/tune/TuneParser';
 import TriggerLogsParser from '../utils/logs/TriggerLogsParser';
-import LogParser from '../utils/logs/LogParser';
+import LogValidator from '../utils/logs/LogValidator';
 import useDb from '../hooks/useDb';
 import useServerStorage from '../hooks/useServerStorage';
 import { buildFullUrl } from '../utils/url';
@@ -314,7 +314,7 @@ const UploadPage = () => {
 
       let valid = true;
       const extension = file.name.split('.').pop();
-      const parser = new LogParser(await file.arrayBuffer());
+      const parser = new LogValidator(await file.arrayBuffer());
 
       switch (extension) {
         case 'mlg':

src/utils/logs/LogValidator.ts

@@ -1,6 +1,6 @@
 import { ParserInterface } from '../ParserInterface';
 
-class LogParser implements ParserInterface {
+class LogValidator implements ParserInterface {
   private MLG_FORMAT_LENGTH = 6;
 
   private isMLGLogs: boolean = false;
@@ -11,7 +11,7 @@ class LogParser implements ParserInterface {
 
   private raw: string = '';
 
-  constructor(buffer: ArrayBuffer) {
+  public constructor(buffer: ArrayBuffer) {
     this.buffer = buffer;
     this.raw = (new TextDecoder()).decode(buffer);
 
@@ -19,15 +19,15 @@ class LogParser implements ParserInterface {
     this.checkMSL();
   }
 
-  parse(): this {
+  public parse(): this {
     return this;
   }
 
-  isMLG(): boolean {
+  public isMLG(): boolean {
     return this.isMLGLogs;
   }
 
-  isMSL(): boolean {
+  public isMSL(): boolean {
     return this.isMSLLogs;
   }
 
@@ -49,8 +49,12 @@ class LogParser implements ParserInterface {
         this.isMSLLogs = true;
         break;
       }
+
+      if (index > 10) {
+        break;
+      }
     }
   }
 }
 
-export default LogParser;
+export default LogValidator;

src/utils/logs/MslLogParser.ts

@@ -0,0 +1,77 @@
+/* eslint-disable no-continue */
+import { Result } from 'mlg-converter/dist/types';
+import { ParserInterface } from '../ParserInterface';
+
+class MslLogParser implements ParserInterface {
+  private buffer: ArrayBuffer = new ArrayBuffer(0);
+
+  private raw: string = '';
+
+  private result: Result = {
+    fileFormat: 'MSL',
+    formatVersion: 1,
+    timestamp: new Date(),
+    info: '',
+    bitFieldNames: '',
+    fields: [],
+    records: [],
+  };
+
+  public constructor(buffer: ArrayBuffer) {
+    this.buffer = buffer;
+    this.raw = (new TextDecoder()).decode(buffer);
+  }
+
+  public parse(): this {
+    let unitsIndex = 999;
+    const lines = this.raw.trim().split('\n');
+    for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) {
+      const line = lines[lineIndex].trim();
+
+      if (line.startsWith('"')) {
+        this.result.info += `${line.replaceAll('"', '').trim()}\n`;
+
+        continue;
+      }
+
+      if (line.startsWith('Time')) {
+        unitsIndex = lineIndex + 1;
+        const fields = line.split('\t');
+        const units = lines[unitsIndex].trim().split('\t');
+
+        this.result.fields = fields.map((name, fieldIndex) => ({
+          name: name.trim(),
+          units: (units[fieldIndex] || '').trim(),
+          displayStyle: 'Float',
+          scale: 1,
+          transform: 0,
+          digits: 0,
+        }));
+
+        continue;
+      }
+
+      if (lineIndex > unitsIndex) {
+        const fields = line.split('\t');
+        const record = {
+          type: 'field' as const,
+          timestamp: 0,
+        };
+
+        fields.forEach((value, fieldIndex) => {
+          (record as any)[this.result.fields[fieldIndex].name] = parseFloat(value);
+        });
+
+        this.result.records.push(record);
+      }
+    };
+
+    return this;
+  }
+
+  public getResult(): Result {
+    return this.result;
+  }
+}
+
+export default MslLogParser;

src/utils/logs/TriggerLogsParser.ts

@@ -41,11 +41,11 @@ class TriggerLogsParser implements ParserInterface {
 
   private raw: string = '';
 
-  constructor(buffer: ArrayBuffer) {
+  public constructor(buffer: ArrayBuffer) {
     this.raw = (new TextDecoder()).decode(buffer);
   }
 
-  parse(): this {
+  public parse(): this {
     this.parseCompositeLogs(this.raw);
     this.parseToothLogs(this.raw);
 
@@ -60,15 +60,15 @@ class TriggerLogsParser implements ParserInterface {
     return this;
   }
 
-  getCompositeLogs(): CompositeLogEntry[] {
+  public getCompositeLogs(): CompositeLogEntry[] {
     return this.resultComposite;
   }
 
-  getToothLogs(): ToothLogEntry[] {
+  public getToothLogs(): ToothLogEntry[] {
     return this.resultTooth;
   }
 
-  isTooth(): boolean {
+  public isTooth(): boolean {
     if (!this.alreadyParsed) {
       this.parse();
     }
@@ -76,7 +76,7 @@ class TriggerLogsParser implements ParserInterface {
     return this.isToothLogs;
   }
 
-  isComposite(): boolean {
+  public isComposite(): boolean {
     if (!this.alreadyParsed) {
       this.parse();
     }

src/workers/logParserWorker.ts

@@ -0,0 +1,67 @@
+import { Parser } from 'mlg-converter';
+import { Result } from 'mlg-converter/dist/types';
+import Pako from 'pako';
+import LogValidator from '../utils/logs/LogValidator';
+import MslLogParser from '../utils/logs/MslLogParser';
+
+// eslint-disable-next-line no-restricted-globals
+const ctx: Worker = self as any;
+
+export interface WorkerOutput {
+  type: 'progress' | 'metrics' | 'result' | 'error';
+  progress?: number;
+  result?: Result;
+  error?: Error;
+  elapsed?: number;
+  records?: number;
+}
+
+// eslint-disable-next-line no-bitwise
+const elapsed = (t0: number): number => ~~(performance.now() - t0);
+
+const parseMsl = (raw: ArrayBufferLike, t0: number): Result => new MslLogParser(raw).parse().getResult();
+
+const parseMlg = (raw: ArrayBufferLike, t0: number): Result => new Parser(raw).parse((progress) => {
+  ctx.postMessage({
+    type: 'progress',
+    progress,
+    elapsed: elapsed(t0),
+  } as WorkerOutput);
+});
+
+ctx.addEventListener('message', ({ data }: { data: ArrayBuffer }) => {
+  try {
+    const t0 = performance.now();
+    const raw = Pako.inflate(new Uint8Array(data)).buffer;
+    const logParser = new LogValidator(raw);
+
+    if (logParser.isMLG()) {
+      const mlgResult = parseMlg(raw, t0);
+      ctx.postMessage({
+        type: 'metrics',
+        elapsed: elapsed(t0),
+        records: mlgResult.records.length,
+      } as WorkerOutput);
+      ctx.postMessage({ type: 'result', result: mlgResult } as WorkerOutput);
+
+      return;
+    }
+
+    if (logParser.isMSL()) {
+      const mslResult = parseMsl(raw, t0);
+      ctx.postMessage({
+        type: 'metrics',
+        elapsed: elapsed(t0),
+        records: mslResult.records.length,
+      } as WorkerOutput);
+      ctx.postMessage({ type: 'result', result: mslResult } as WorkerOutput);
+
+      return;
+    }
+
+    throw new Error('Unsupported file format');
+  } catch (error) {
+    ctx.postMessage({ type: 'error', error } as WorkerOutput);
+    throw error;
+  }
+});

src/workers/mlgParser.ts

@@ -1,39 +0,0 @@
-/* eslint-disable no-bitwise */
-
-import { Parser } from 'mlg-converter';
-import { Result } from 'mlg-converter/dist/types';
-import Pako from 'pako';
-
-// eslint-disable-next-line no-restricted-globals
-const ctx: Worker = self as any;
-
-export interface WorkerOutput {
-  type: 'progress' | 'metrics' | 'result' | 'error' ;
-  progress?: number;
-  result?: Result;
-  error?: Error;
-  elapsed?: number;
-  records?: number;
-}
-
-ctx.addEventListener('message', ({ data }: { data: ArrayBuffer }) => {
-  try {
-    const t0 = performance.now();
-    const result = new Parser(Pako.inflate(new Uint8Array(data)).buffer).parse((progress) => {
-      ctx.postMessage({
-        type: 'progress',
-        progress,
-        elapsed: ~~(performance.now() - t0),
-      } as WorkerOutput);
-    });
-    ctx.postMessage({
-      type: 'metrics',
-      elapsed: ~~(performance.now() - t0),
-      records: result.records.length,
-    } as WorkerOutput);
-    ctx.postMessage({ type: 'result', result } as WorkerOutput);
-  } catch (error) {
-    ctx.postMessage({ type: 'error', error } as WorkerOutput);
-    throw error;
-  }
-});

vite.config.ts

@@ -20,6 +20,7 @@ export default defineConfig({
           kbar: ['kbar'],
           perfectScrollbar: ['perfect-scrollbar'],
           pako: ['pako'],
+          mlgConverter: ['mlg-converter'],
         },
       },
     },