Improve log parsing - move parser to web worker (#44)

craco.config.js

@@ -13,4 +13,20 @@ module.exports = {
       },
     },
   ],
+  webpack: {
+    configure: (config) => {
+      config.resolve.extensions.push('.wasm');
+
+      config.module.rules.forEach(rule => {
+        (rule.oneOf || []).forEach(oneOf => {
+          if (oneOf.loader && oneOf.loader.indexOf('file-loader') >= 0) {
+            // Make file-loader ignore WASM files
+            oneOf.exclude.push(/\.wasm$/);
+          }
+        });
+      });
+
+      return config;
+    },
+  },
 };

package.json

@@ -39,7 +39,7 @@
     "antd": "^4.15.0",
     "electron-squirrel-startup": "^1.0.0",
     "js-yaml": "^4.0.0 ",
-    "mlg-converter": "^0.3.0",
+    "mlg-converter": "^0.4.0",
     "parsimmon": "^1.16.0",
     "react": "^17.0.1",
     "react-dom": "^17.0.2",
@@ -77,7 +77,8 @@
     "less-loader": "^6.1.0",
     "prettier": "^2.2.1",
     "typescript": "^4.1.5",
-    "wait-on": "^5.3.0"
+    "wait-on": "^5.3.0",
+    "worker-loader": "^3.0.8"
   },
   "config": {
     "forge": {

src/@types/worker.loader.d.ts

@@ -0,0 +1,10 @@
+declare module 'worker-loader!*' {
+  // You need to change `Worker`, if you specified a different value for the `workerType` option
+  class WebpackWorker extends Worker {
+    constructor();
+  }
+
+  // Uncomment this if you set the `esModule` option to `false`
+  // export = WebpackWorker;
+  export default WebpackWorker;
+}

src/components/Dialog.tsx

@@ -77,7 +77,8 @@ const containerStyle = {
 };
 
 const skeleton = (<div style={containerStyle}>
-  <Skeleton /><Skeleton />
+  <Skeleton active />
+  <Skeleton active />
 </div>);
 
 // TODO: refactor this

src/components/Log.tsx

@@ -1,3 +1,4 @@
+/* eslint-disable import/no-webpack-loader-syntax */
 import {
   useCallback,
   useEffect,
@@ -5,12 +6,15 @@ import {
   useState,
 } from 'react';
 import {
-  Spin,
   Layout,
   Tabs,
   Checkbox,
   Row,
   Skeleton,
+  Progress,
+  Steps,
+  Space,
+  Divider,
 } from 'antd';
 import {
   FileTextOutlined,
@@ -18,19 +22,22 @@ import {
   DashboardOutlined,
 } from '@ant-design/icons';
 import { CheckboxValueType } from 'antd/lib/checkbox/Group';
+import useBreakpoint from 'antd/lib/grid/hooks/useBreakpoint';
 import { connect } from 'react-redux';
-import { Parser } from 'mlg-converter';
 import { Field, Result as ParserResult } from 'mlg-converter/dist/types';
 import PerfectScrollbar from 'react-perfect-scrollbar';
+// eslint-disable-next-line import/no-unresolved
+import MlgParserWorker from 'worker-loader!../workers/mlgParser.worker';
 import { loadLogs } from '../utils/api';
 import Canvas, { LogEntry } from './Log/Canvas';
 import { AppState, UIState } from '../types/state';
 import { Config } from '../types/config';
 import store from '../store';
+import { formatBytes, msToTime } from '../utils/number';
 
-// const { SubMenu } = Menu;
 const { TabPane } = Tabs;
 const { Content } = Layout;
+const { Step } = Steps;
 
 const mapStateToProps = (state: AppState) => ({
   ui: state.ui,
@@ -39,7 +46,13 @@ const mapStateToProps = (state: AppState) => ({
 });
 
 const Log = ({ ui, config }: { ui: UIState, config: Config }) => {
+  const { lg } = useBreakpoint();
   const { Sider } = Layout;
+  const [progress, setProgress] = useState(0);
+  const [fileSize, setFileSize] = useState<string>();
+  const [parseElapsed, setParseElapsed] = useState<string>();
+  const [samplesCount, setSamplesCount] = useState();
+  const [step, setStep] = useState(0);
   const contentRef = useRef<HTMLDivElement | null>(null);
   const margin = 30;
   const [canvasWidth, setCanvasWidth] = useState(0);
@@ -55,7 +68,6 @@ const Log = ({ ui, config }: { ui: UIState, config: Config }) => {
       setTimeout(calculateCanvasWidth, 1);
     },
   } as any;
-  const [isLoading, setIsLoading] = useState(true);
   const [logs, setLogs] = useState<ParserResult>();
   const [fields, setFields] = useState<Field[]>([]);
   const [selectedFields, setSelectedFields] = useState<CheckboxValueType[]>([
@@ -67,28 +79,49 @@ const Log = ({ ui, config }: { ui: UIState, config: Config }) => {
   ]);
 
   useEffect(() => {
-    loadLogs()
-      .then((data) => {
-        setIsLoading(true);
-        const parsed = new Parser(data).parse();
-        setLogs(parsed);
-        setIsLoading(false);
-        setFields(parsed.fields);
+    const worker = new MlgParserWorker();
+    const loadData = async () => {
+      const raw = await loadLogs();
+      setFileSize(formatBytes(raw.byteLength));
 
-        console.log(parsed);
-      });
+      worker.postMessage(raw);
+      worker.onmessage = ({ data }) => {
+        switch (data.type) {
+          case 'progress':
+            setStep(1);
+            setProgress(data.progress);
+            break;
+          case 'result':
+            setSamplesCount(data.result.records.length);
+            setStep(2);
+            setLogs(data.result);
+            setFields(data.result.fields);
+            break;
+          case 'metrics':
+            setParseElapsed(msToTime(data.metrics.elapsedMs));
+            break;
+          default:
+            break;
+        }
+      };
+    };
 
-    window.addEventListener('resize', calculateCanvasWidth);
+    loadData();
     calculateCanvasWidth();
 
-    return () => window.removeEventListener('resize', calculateCanvasWidth);
+    window.addEventListener('resize', calculateCanvasWidth);
+
+    return () => {
+      window.removeEventListener('resize', calculateCanvasWidth);
+      worker.terminate();
+    };
   }, [calculateCanvasWidth]);
 
   return (
     <>
       <Sider {...siderProps} className="app-sidebar">
-        {isLoading ?
-          <Skeleton />
+        {!logs ?
+          <div style={{ padding: 20 }}><Skeleton active /></div>
           :
           !ui.sidebarCollapsed &&
           <Tabs defaultActiveKey="fields" style={{ marginLeft: 20 }}>
@@ -117,8 +150,36 @@ const Log = ({ ui, config }: { ui: UIState, config: Config }) => {
       <Layout style={{ width: '100%', textAlign: 'center', marginTop: 50 }}>
         <Content>
           <div ref={contentRef} style={{ width: '100%', marginRight: margin }}>
-            {isLoading ?
-              <Spin size="large" />
+            {!logs ?
+              <Space
+                direction="vertical"
+                size="large"
+                style={{ width: '80%', maxWidth: 1000 }}
+              >
+                <Progress
+                  type="circle"
+                  percent={progress}
+                  width={170}
+                />
+                <Divider />
+                <Steps current={step} direction={lg ? 'horizontal' : 'vertical'}>
+                  <Step
+                    title="Downloading"
+                    description="From the closest server"
+                    subTitle={fileSize}
+                  />
+                  <Step
+                    title="Decoding"
+                    description="Reading ones and zeros"
+                    subTitle={parseElapsed}
+                  />
+                  <Step
+                    title="Rendering"
+                    description="Putting pixels on your screen"
+                    subTitle={samplesCount && `${samplesCount} samples`}
+                  />
+                </Steps>
+              </Space>
               :
               <Canvas
                 data={logs!.records as LogEntry[]}

src/components/SideBar.tsx

@@ -1,4 +1,8 @@
-import { Layout, Menu, Skeleton } from 'antd';
+import {
+  Layout,
+  Menu,
+  Skeleton,
+} from 'antd';
 import { connect } from 'react-redux';
 import {
   generatePath,
@@ -126,7 +130,11 @@ const SideBar = ({
     return (
       <Sider {...siderProps} className="app-sidebar" >
         <div style={{ paddingLeft: 10 }}>
-          <Skeleton /><Skeleton /><Skeleton /><Skeleton /><Skeleton />
+          <Skeleton active />
+          <Skeleton active />
+          <Skeleton active />
+          <Skeleton active />
+          <Skeleton active />
         </div>
       </Sider>
     );

src/utils/number.ts

@@ -0,0 +1,25 @@
+export const formatBytes = (bytes: number, decimals = 2): string => {
+  if (bytes === 0) return '0 Bytes';
+
+  const k = 1024;
+  const dm = decimals < 0 ? 0 : decimals;
+  const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+  return `${parseFloat((bytes / k**i).toFixed(dm))  } ${  sizes[i]}`;
+};
+
+export const leftPad = (n: number, z = 2) => (`00${n}`).slice(-z);
+
+export const msToTime = (input: number) => {
+  let s = input;
+  const ms = s % 1000;
+  s = (s - ms) / 1000;
+  const secs = s % 60;
+  s = (s - secs) / 60;
+  const mins = s % 60;
+  const hrs = (s - mins) / 60;
+
+  return `${leftPad(hrs)}:${leftPad(mins)}:${leftPad(secs)}.${ms}`;
+};

src/workers/mlgParser.worker.ts

@@ -0,0 +1,22 @@
+import { Parser } from 'mlg-converter';
+
+// eslint-disable-next-line no-restricted-globals
+const ctx: Worker = self as any;
+
+ctx.addEventListener('message', ({ data }: { data: ArrayBuffer }) => {
+  const t0 = performance.now();
+
+  const result = new Parser(data).parse((progress) => {
+    ctx.postMessage({
+      type: 'progress',
+      progress,
+    });
+  });
+  ctx.postMessage({
+    type: 'metrics',
+    metrics: {
+      elapsedMs: Math.round(performance.now() - t0),
+    },
+  });
+  ctx.postMessage({ type: 'result', result });
+});