logging improvements and proper command line launcher

reader/build.gradle

@@ -6,4 +6,12 @@ plugins {
 dependencies {
     implementation libs.snakeyaml
     testImplementation libs.junit
+}
+
+shadowJar {
+    manifest {
+        attributes(
+                'Main-Class': 'com.rusefi.can.Launcher'
+        )
+    }
 }

reader/src/main/java/com/rusefi/can/Launcher.java

@@ -0,0 +1,52 @@
+package com.rusefi.can;
+
+import com.rusefi.can.analysis.ByteRateOfChangeReports;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+public class Launcher {
+    public static final String FILENAME_SUFFIX_PROPERTY = "-suffix";
+    public static String fileNameSuffixValue = ".trc";
+    public static final String FILENAME_FILTER_PROPERTY = "-filter";
+    public static String fileNameFilter;
+    public static final String DBC_FILENAME_PROPERTY = "-dbc";
+    public static String dbcFileName;
+
+    public static void main(String[] args) throws IOException {
+        if (args.length < 1) {
+            System.err.println("At least folder name is expected");
+            System.err.println("Supported arguments: " + Arrays.toString(new String[]{
+                    FILENAME_SUFFIX_PROPERTY,
+                    FILENAME_FILTER_PROPERTY,
+                    DBC_FILENAME_PROPERTY,
+            }));
+            System.exit(-1);
+        }
+        String inputFolderName = args[0];
+        for (int i = 1; i < args.length; i++) {
+            switch (args[i]) {
+                case FILENAME_SUFFIX_PROPERTY:
+                    i += 1;
+                    fileNameSuffixValue = args[i];
+                    break;
+                case FILENAME_FILTER_PROPERTY:
+                    i += 1;
+                    fileNameFilter = args[i];
+                    break;
+                case DBC_FILENAME_PROPERTY:
+                    i += 1;
+                    dbcFileName = args[i];
+                    break;
+                default:
+                    throw new IllegalStateException("Unexpected argument " + args[i]);
+            }
+        }
+
+        System.out.println("Running");
+        System.out.println("\tinputFolderName=" + inputFolderName);
+        System.out.println("\tfileNameSuffixValue=" + fileNameSuffixValue);
+        System.out.println("\tdbcFileName=" + dbcFileName);
+        ByteRateOfChangeReports.scanInputFolder(inputFolderName, fileNameSuffixValue);
+    }
+}

reader/src/main/java/com/rusefi/can/TrcToMlq.java

@@ -11,7 +11,7 @@ import static com.rusefi.can.reader.CANLineReader.getReader;
 
 public class TrcToMlq {
     public static ReaderType parseCurrentReaderTypeSetting() {
-        String property = System.getProperty("TRACE_READER", ReaderType.PCAN.name());
+        String property = System.getProperty("TRACE_READER", ReaderType.AUTO.name());
         ReaderType readerType = ReaderType.valueOf(property);
         System.out.println("getCurrentReaderType: " + readerType + " for [" + property + "]");
         return readerType;

reader/src/main/java/com/rusefi/can/analysis/ByteRateOfChange.java

@@ -40,7 +40,7 @@ public class ByteRateOfChange {
     }
 
     private static void writeByteReport(List<ByteStatistics> allStats, String fileName) throws FileNotFoundException {
-        System.out.println("Writing byte report to " + fileName);
+        System.out.println(new Date() + " Writing byte report to " + fileName);
         PrintStream ps = new PrintStream(new FileOutputStream(fileName, false));
 
         for (ByteStatistics byteStatistics : allStats) {

reader/src/main/java/com/rusefi/can/analysis/ByteRateOfChangeReports.java

@@ -1,6 +1,7 @@
 package com.rusefi.can.analysis;
 
 import com.rusefi.can.CANPacket;
+import com.rusefi.can.Launcher;
 import com.rusefi.can.reader.CANLineReader;
 import com.rusefi.util.FolderUtil;
 
@@ -97,8 +98,8 @@ public class ByteRateOfChangeReports {
         List<ByteRateOfChange.TraceReport> reports = new ArrayList<>();
 
         FolderUtil.handleFolder(inputFolderName, (simpleFileName, fullInputFileName) -> {
-//            if (!simpleFileName.contains("belt"))
+            if (Launcher.fileNameFilter != null && !simpleFileName.contains(Launcher.fileNameFilter))
-//                return;
+                return;
 
             List<CANPacket> logFileContent = CANLineReader.getReader().readFile(fullInputFileName);
 
@@ -110,7 +111,7 @@ public class ByteRateOfChangeReports {
             CanToMegaLogViewer.createMegaLogViewer(reportDestinationFolder, logFileContent, simpleFileName);
 
             ByteRateOfChange.TraceReport report = ByteRateOfChange.process(reportDestinationFolder, simpleFileName, logFileContent);
-            report.save("temp.txt");
+            report.save("temp-ByteRateOfChange.txt");
 
             reports.add(report);
         }, fileNameSuffix);

reader/src/main/java/com/rusefi/can/analysis/CanToMegaLogViewer.java

@@ -1,19 +1,33 @@
 package com.rusefi.can.analysis;
 
 import com.rusefi.can.CANPacket;
+import com.rusefi.can.Launcher;
+import com.rusefi.can.reader.dbc.DbcFile;
 import com.rusefi.mlv.LoggingContext;
 import com.rusefi.mlv.LoggingStrategy;
 import com.rusefi.sensor_logs.BinaryLogEntry;
 import com.rusefi.sensor_logs.BinarySensorLog;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
 public class CanToMegaLogViewer {
-    public static void createMegaLogViewer(String reportDestinationFolder, List<CANPacket> canPackets, String simpleFileName) {
+    public static void createMegaLogViewer(String reportDestinationFolder, List<CANPacket> canPackets, String simpleFileName) throws IOException {
+
+        if (Launcher.dbcFileName != null) {
+            DbcFile dbc = DbcFile.readFromFile(Launcher.dbcFileName);
+            String outputFileName = reportDestinationFolder + File.separator + simpleFileName + ".by_dbc.mlg";
+            LoggingStrategy.writeLog(dbc, canPackets, outputFileName);
+        }
+
+        writeByIds(reportDestinationFolder, canPackets, simpleFileName);
+    }
+
+    private static void writeByIds(String reportDestinationFolder, List<CANPacket> canPackets, String simpleFileName) {
         List<BinaryLogEntry> entries = new ArrayList<>();
         Set<ByteRateOfChange.ByteId> byteIds = new HashSet<>();
 

reader/src/main/java/com/rusefi/can/analysis/ChecksumScanner.java

@@ -44,7 +44,7 @@ public class ChecksumScanner {
         withChecksum.sort(Comparator.naturalOrder());
         Yaml yaml = new Yaml();
         String yamlCountersReportFileName = reportDestinationFolder + File.separator + CHECKSUM_YAML;
-        System.out.println("Writing report to " + yamlCountersReportFileName);
+        System.out.println(new Date() + " Writing report to " + yamlCountersReportFileName);
         yaml.dump(withChecksum, new FileWriter(yamlCountersReportFileName));
     }
 }

reader/src/main/java/com/rusefi/can/analysis/CounterScanner.java

@@ -63,7 +63,7 @@ public class CounterScanner {
             lengthByStartIndex.put(counterWithWidth.getStart().getTotalBitIndex(), counterWithWidth.getTotalNumberOfBits());
         }
         String yamlCountersReportFileName = reportDestinationFolder + File.separator + COUNTERS_YAML;
-        System.out.println("Writing report to " + yamlCountersReportFileName);
+        System.out.println(new Date() + " Writing report to " + yamlCountersReportFileName);
         yaml.dump(map, new FileWriter(yamlCountersReportFileName));
         pw.close();
     }

reader/src/main/java/com/rusefi/can/reader/CANLineReader.java

@@ -31,6 +31,7 @@ public interface CANLineReader {
             case CANHACKER:
                 return CanHackerReader.INSTANCE;
             case PCAN:
+            case AUTO:
             default:
                 return AutoFormatReader.INSTANCE;
         }

reader/src/main/java/com/rusefi/can/reader/ReaderType.java

@@ -1,6 +1,8 @@
 package com.rusefi.can.reader;
 
 public enum ReaderType {
+    AUTO,
+    // todo: do we even need all these types? shall we always auto-detect?
     PCAN,
     CANOE,
     CANHACKER,

reader/src/main/java/com/rusefi/can/reader/dbc/DbcFile.java

@@ -6,10 +6,7 @@ import com.rusefi.sensor_logs.BinaryLogEntry;
 import java.io.BufferedReader;
 import java.io.FileReader;
 import java.io.IOException;
-import java.util.ArrayList;
+import java.util.*;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Objects;
 
 public class DbcFile {
     public final LinkedHashMap<Integer, DbcPacket> packets = new LinkedHashMap<>();
@@ -25,7 +22,7 @@ public class DbcFile {
     }
 
     public static DbcFile readFromFile(String fileName) throws IOException {
-        System.out.println("Reading DBC file from " + fileName + "...");
+        System.out.println(new Date() + " Reading DBC file from " + fileName + "...");
         DbcFile dbc = new DbcFile(LoggingStrategy.LOG_ONLY_TRANSLATED_FIELDS);
         {
             BufferedReader reader = new BufferedReader(new FileReader(fileName));
@@ -115,7 +112,6 @@ public class DbcFile {
         return packets.get(canId);
     }
 
-
     public List<BinaryLogEntry> getFieldNameEntries() {
         if (list == null) {
             list = LoggingStrategy.getFieldNameEntries(this, logOnlyTranslatedFields);

reader/src/main/java/com/rusefi/mlv/LoggingStrategy.java

@@ -8,6 +8,7 @@ import com.rusefi.sensor_logs.BinaryLogEntry;
 import com.rusefi.sensor_logs.BinarySensorLog;
 
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 
 public class LoggingStrategy {
@@ -29,6 +30,7 @@ public class LoggingStrategy {
     public static void writeLog(DbcFile dbc, List<CANPacket> packets, String outputFileName) {
         List<BinaryLogEntry> entries = dbc.getFieldNameEntries();
 
+        System.out.println(new Date() + " writeLog... " + outputFileName);
         LoggingContext context = new LoggingContext();
         BinarySensorLog<BinaryLogEntry> log = context.getBinaryLogEntryBinarySensorLog(entries, outputFileName);
 
@@ -44,6 +46,7 @@ public class LoggingStrategy {
         };
 
         context.writeLogContent(packets, log, logger);
+        System.out.println(new Date() + " writeLog " + outputFileName + " done!");
     }
 
     public interface PacketLogger {

reader/src/main/java/com/rusefi/sensor_logs/BinarySensorLog.java

@@ -106,7 +106,7 @@ public class BinarySensorLog<T extends BinaryLogEntry> implements SensorLog, Aut
         stream.writeInt((int) (System.currentTimeMillis() / 1000));
         // 000ch
         int infoDataStart = Fields.MLQ_HEADER_SIZE + Fields.MLQ_FIELD_HEADER_SIZE * entries.size();
-        System.out.println("Total " + entries.size() + " fields");
+        System.out.println(new Date() + " Total " + entries.size() + " fields");
         stream.writeInt(infoDataStart);
         stream.writeInt(infoDataStart + headerText.length());
         // 0012h
@@ -140,7 +140,7 @@ public class BinarySensorLog<T extends BinaryLogEntry> implements SensorLog, Aut
 
     @Override
     public void close() {
-        System.out.println("Finishing " + outputFileName);
+        System.out.println(new Date() + " Finishing " + outputFileName);
         close(stream);
         stream = null;
     }

reader/src/test/java/com/rusefi/can/ByteRateOfChangeVagSandbox.java

@@ -1,15 +1,11 @@
 package com.rusefi.can;
 
 import com.rusefi.can.analysis.ByteRateOfChangeReports;
-import com.rusefi.can.reader.ReaderType;
-import com.rusefi.can.reader.ReaderTypeHolder;
 
 import java.io.IOException;
 
 public class ByteRateOfChangeVagSandbox {
     public static void main(String[] args) throws IOException {
-        ReaderTypeHolder.INSTANCE.type = ReaderType.PCAN;
-
         String inputFolderName = "C:\\stuff\\rusefi_documentation\\OEM-Docs\\VAG\\2006-Passat-B6";
 
         ByteRateOfChangeReports.scanInputFolder(inputFolderName, "fast-acceleration3.trc");

reader/src/test/java/com/rusefi/can/CanAmMG1Sandbox.java

@@ -1,24 +1,11 @@
 package com.rusefi.can;
 
-import com.rusefi.can.analysis.ByteRateOfChangeReports;
-import com.rusefi.can.reader.CANLineReader;
-import com.rusefi.can.reader.ReaderType;
-import com.rusefi.can.reader.ReaderTypeHolder;
-import com.rusefi.can.reader.impl.PcanTrcReader2_0;
-
 import java.io.IOException;
 
 public class CanAmMG1Sandbox {
     public static void main(String[] args) throws IOException {
-        ReaderTypeHolder.INSTANCE.type = ReaderType.PCAN;
-        CANLineReader reader = PcanTrcReader2_0.INSTANCE;
-
-
         String inputFolderName = "C:\\stuff\\rusefi_documentation\\OEM-Docs\\CanAm\\maverick-x3-xrs-turbo-rr-max-2021\\";
-//        String revvingCltWentUp = inputFolderName + "engine-revving-CLT-increased.trc";
-        //      List<CANPacket> idling = reader.readFile(revvingCltWentUp);
-        //printStats(idling, "IDLING");
 
-        ByteRateOfChangeReports.scanInputFolder(inputFolderName, ".trc");
+        Launcher.main(new String[]{inputFolderName});
     }
 }

reader/src/test/java/com/rusefi/can/KiaSandbox.java

@@ -1,17 +1,11 @@
 package com.rusefi.can;
 
-import com.rusefi.can.analysis.ByteRateOfChangeReports;
-import com.rusefi.can.reader.ReaderType;
-import com.rusefi.can.reader.ReaderTypeHolder;
-
 import java.io.IOException;
 
 public class KiaSandbox {
     public static void main(String[] args) throws IOException {
-        ReaderTypeHolder.INSTANCE.type = ReaderType.PCAN;
-
         String inputFolderName = "C:\\stuff\\rusefi_documentation\\OEM-Docs\\Kia\\2013-CAN-logs";
 
-        ByteRateOfChangeReports.scanInputFolder(inputFolderName, ".trc");
+        Launcher.main(new String[]{inputFolderName});
     }
 }

reader/src/test/java/com/rusefi/can/NissanSandbox.java

@@ -1,14 +1,11 @@
 package com.rusefi.can;
 
-import com.rusefi.can.analysis.ByteRateOfChangeReports;
-
 import java.io.IOException;
 
 public class NissanSandbox {
     public static void main(String[] args) throws IOException {
         String inputFolderName = "C:\\stuff\\rusefi_documentation\\OEM-Docs\\Nissan\\2011_Xterra\\2011-nissan-CAN-June-2021";
 
-        ByteRateOfChangeReports.scanInputFolder(inputFolderName, ".trc");
+        Launcher.main(new String[]{inputFolderName});
-
     }
 }

reader/src/test/java/com/rusefi/can/VagB6Sandbox.java

@@ -1,45 +1,42 @@
 package com.rusefi.can;
 
-import com.rusefi.can.analysis.ByteRateOfChangeReports;
-import com.rusefi.can.analysis.PerSidDump;
-import com.rusefi.can.reader.CANLineReader;
-import com.rusefi.can.reader.ReaderType;
-import com.rusefi.can.reader.ReaderTypeHolder;
-import com.rusefi.can.reader.dbc.DbcFile;
-import com.rusefi.can.reader.impl.PcanTrcReader1_1;
 import com.rusefi.can.reader.impl.ReadFullVagDbc;
-import com.rusefi.mlv.LoggingStrategy;
 
 import java.io.IOException;
-import java.util.List;
-
-import static com.rusefi.can.analysis.ByteRateOfChangeReports.createOutputFolder;
 
 public class VagB6Sandbox {
     public static void main(String[] args) throws IOException {
-        DbcFile dbc = DbcFile.readFromFile(ReadFullVagDbc.VAG_DBC_FILE);
+//        DbcFile dbc = DbcFile.readFromFile(ReadFullVagDbc.VAG_DBC_FILE);
 
-        CANLineReader reader = new PcanTrcReader1_1();
         String inputFolderName = "C:\\stuff\\rusefi_documentation\\OEM-Docs\\VAG\\2006-Passat-B6\\";
 
-        {
+        Launcher.main(new String[]{inputFolderName,
-            ReaderTypeHolder.INSTANCE.type = ReaderType.PCAN;
+                Launcher.FILENAME_FILTER_PROPERTY,
-            ByteRateOfChangeReports.scanInputFolder(inputFolderName + "2023.10-tcu-side2", ".trc");
+                "passat-back-and-forth-60-seconds",
-        }
+                Launcher.DBC_FILENAME_PROPERTY,
+                ReadFullVagDbc.VAG_DBC_FILE
+        });
+
+//        {
+//            ReaderTypeHolder.INSTANCE.type = ReaderType.PCAN;
+//            ByteRateOfChangeReports.scanInputFolder(inputFolderName + "2023.10-tcu-side2", ".trc");
+//        }
+
+//        String name = "passat-back-and-forth-60-seconds";
+//        String simpleFileName = name + ".trc";
 
 
-        String simpleFileName = "passat-back-and-forth-60-seconds.trc1";
+//
-        String file = inputFolderName + simpleFileName;
+//        CANLineReader reader = new AutoFormatReader();
-
+//        String file = inputFolderName + simpleFileName;
-        String reportDestinationFolder = createOutputFolder(inputFolderName);
+//
-
+//        String reportDestinationFolder = createOutputFolder(inputFolderName);
-
+//
-        List<CANPacket> packets = reader.readFile(file);
+//
-
+//        List<CANPacket> packets = reader.readFile(file);
-        PerSidDump.handle(reportDestinationFolder, simpleFileName, packets);
+//
-
+//        PerSidDump.handle(reportDestinationFolder, simpleFileName, packets);
-
+//
-        String outputFileName = "vag.mlg";
+//
-        LoggingStrategy.writeLog(dbc, packets, outputFileName);
     }
 }