Compare commits

...

8 Commits

Author SHA1 Message Date
rusefillc ec535cfb66 SomethingLinuxReader 2024-02-09 23:00:42 -05:00
rusefillc 516a2b740c BusMasterReader 2024-02-09 22:47:08 -05:00
rusefillc 4067f21a05 logging improvements and proper command line launcher 2024-02-09 21:25:39 -05:00
rusefillc 574efe1347
Update README.md 2024-02-09 21:24:16 -05:00
rusefillc 8199b5bb53
Update README.md 2024-02-09 20:41:42 -05:00
rusefillc c2dc31672b
Update README.md 2024-02-09 20:40:10 -05:00
rusefillc 7a994c7f5f
Update README.md 2024-02-09 20:39:54 -05:00
rusefillc 624cda9519 fresh pointer 2024-02-09 19:39:05 -05:00
26 changed files with 290 additions and 80 deletions

View File

@ -13,10 +13,18 @@ CAN log file utilities to help me work with https://github.com/brent-stone/CAN_R
* ignition on, engine not running, throttle pedal from 0% to 50%, to 0%, to 100%, to 0%
* engine running, rev from 1500 rpm to 3000 rpm
# CAN file processing
```
gradlew :reader:shadowJar
java -jar reader/build/libs/reader-all.jar "C:\stuff\rusefi_documentation\OEM-Docs\VAG\2006-Passat-B6" -filter passat-back-and-forth-60-seconds -dbc opendbc/vw_golf_mk4.dbc
```
# CAN playback
``
```
gradlew :playback:shadowJar
java -jar playback/build/libs/playback-all.jar playback/src/main/resources/atlas.trc
``
```

@ -1 +1 @@
Subproject commit 8526f52ad32de4cd7a17de4cca96d9774f278af4
Subproject commit b697e9304892c4aa63a1e1ab5bee3dff4eaf0f9c

View File

@ -6,4 +6,12 @@ plugins {
dependencies {
implementation libs.snakeyaml
testImplementation libs.junit
}
shadowJar {
manifest {
attributes(
'Main-Class': 'com.rusefi.can.Launcher'
)
}
}

View File

@ -0,0 +1,52 @@
package com.rusefi.can;
import com.rusefi.can.analysis.ByteRateOfChangeReports;
import java.io.IOException;
import java.util.Arrays;
public class Launcher {
public static final String FILENAME_SUFFIX_PROPERTY = "-suffix";
public static String fileNameSuffixValue = ".trc";
public static final String FILENAME_FILTER_PROPERTY = "-filter";
public static String fileNameFilter;
public static final String DBC_FILENAME_PROPERTY = "-dbc";
public static String dbcFileName;
public static void main(String[] args) throws IOException {
if (args.length < 1) {
System.err.println("At least folder name is expected");
System.err.println("Supported arguments: " + Arrays.toString(new String[]{
FILENAME_SUFFIX_PROPERTY,
FILENAME_FILTER_PROPERTY,
DBC_FILENAME_PROPERTY,
}));
System.exit(-1);
}
String inputFolderName = args[0];
for (int i = 1; i < args.length; i++) {
switch (args[i]) {
case FILENAME_SUFFIX_PROPERTY:
i += 1;
fileNameSuffixValue = args[i];
break;
case FILENAME_FILTER_PROPERTY:
i += 1;
fileNameFilter = args[i];
break;
case DBC_FILENAME_PROPERTY:
i += 1;
dbcFileName = args[i];
break;
default:
throw new IllegalStateException("Unexpected argument " + args[i]);
}
}
System.out.println("Running");
System.out.println("\tinputFolderName=" + inputFolderName);
System.out.println("\tfileNameSuffixValue=" + fileNameSuffixValue);
System.out.println("\tdbcFileName=" + dbcFileName);
ByteRateOfChangeReports.scanInputFolder(inputFolderName, fileNameSuffixValue);
}
}

View File

@ -11,7 +11,7 @@ import static com.rusefi.can.reader.CANLineReader.getReader;
public class TrcToMlq {
public static ReaderType parseCurrentReaderTypeSetting() {
String property = System.getProperty("TRACE_READER", ReaderType.PCAN.name());
String property = System.getProperty("TRACE_READER", ReaderType.AUTO.name());
ReaderType readerType = ReaderType.valueOf(property);
System.out.println("getCurrentReaderType: " + readerType + " for [" + property + "]");
return readerType;

View File

@ -40,7 +40,7 @@ public class ByteRateOfChange {
}
private static void writeByteReport(List<ByteStatistics> allStats, String fileName) throws FileNotFoundException {
System.out.println("Writing byte report to " + fileName);
System.out.println(new Date() + " Writing byte report to " + fileName);
PrintStream ps = new PrintStream(new FileOutputStream(fileName, false));
for (ByteStatistics byteStatistics : allStats) {

View File

@ -1,6 +1,7 @@
package com.rusefi.can.analysis;
import com.rusefi.can.CANPacket;
import com.rusefi.can.Launcher;
import com.rusefi.can.reader.CANLineReader;
import com.rusefi.util.FolderUtil;
@ -97,8 +98,8 @@ public class ByteRateOfChangeReports {
List<ByteRateOfChange.TraceReport> reports = new ArrayList<>();
FolderUtil.handleFolder(inputFolderName, (simpleFileName, fullInputFileName) -> {
// if (!simpleFileName.contains("belt"))
// return;
if (Launcher.fileNameFilter != null && !simpleFileName.contains(Launcher.fileNameFilter))
return;
List<CANPacket> logFileContent = CANLineReader.getReader().readFile(fullInputFileName);
@ -110,7 +111,7 @@ public class ByteRateOfChangeReports {
CanToMegaLogViewer.createMegaLogViewer(reportDestinationFolder, logFileContent, simpleFileName);
ByteRateOfChange.TraceReport report = ByteRateOfChange.process(reportDestinationFolder, simpleFileName, logFileContent);
report.save("temp.txt");
report.save("temp-ByteRateOfChange.txt");
reports.add(report);
}, fileNameSuffix);

View File

@ -1,19 +1,33 @@
package com.rusefi.can.analysis;
import com.rusefi.can.CANPacket;
import com.rusefi.can.Launcher;
import com.rusefi.can.reader.dbc.DbcFile;
import com.rusefi.mlv.LoggingContext;
import com.rusefi.mlv.LoggingStrategy;
import com.rusefi.sensor_logs.BinaryLogEntry;
import com.rusefi.sensor_logs.BinarySensorLog;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
public class CanToMegaLogViewer {
public static void createMegaLogViewer(String reportDestinationFolder, List<CANPacket> canPackets, String simpleFileName) {
public static void createMegaLogViewer(String reportDestinationFolder, List<CANPacket> canPackets, String simpleFileName) throws IOException {
if (Launcher.dbcFileName != null) {
DbcFile dbc = DbcFile.readFromFile(Launcher.dbcFileName);
String outputFileName = reportDestinationFolder + File.separator + simpleFileName + ".by_dbc.mlg";
LoggingStrategy.writeLog(dbc, canPackets, outputFileName);
}
writeByIds(reportDestinationFolder, canPackets, simpleFileName);
}
private static void writeByIds(String reportDestinationFolder, List<CANPacket> canPackets, String simpleFileName) {
List<BinaryLogEntry> entries = new ArrayList<>();
Set<ByteRateOfChange.ByteId> byteIds = new HashSet<>();

View File

@ -44,7 +44,7 @@ public class ChecksumScanner {
withChecksum.sort(Comparator.naturalOrder());
Yaml yaml = new Yaml();
String yamlCountersReportFileName = reportDestinationFolder + File.separator + CHECKSUM_YAML;
System.out.println("Writing report to " + yamlCountersReportFileName);
System.out.println(new Date() + " Writing report to " + yamlCountersReportFileName);
yaml.dump(withChecksum, new FileWriter(yamlCountersReportFileName));
}
}

View File

@ -63,7 +63,7 @@ public class CounterScanner {
lengthByStartIndex.put(counterWithWidth.getStart().getTotalBitIndex(), counterWithWidth.getTotalNumberOfBits());
}
String yamlCountersReportFileName = reportDestinationFolder + File.separator + COUNTERS_YAML;
System.out.println("Writing report to " + yamlCountersReportFileName);
System.out.println(new Date() + " Writing report to " + yamlCountersReportFileName);
yaml.dump(map, new FileWriter(yamlCountersReportFileName));
pw.close();
}

View File

@ -31,6 +31,7 @@ public interface CANLineReader {
case CANHACKER:
return CanHackerReader.INSTANCE;
case PCAN:
case AUTO:
default:
return AutoFormatReader.INSTANCE;
}

View File

@ -1,6 +1,8 @@
package com.rusefi.can.reader;
public enum ReaderType {
AUTO,
// todo: do we even need all these types? shall we always auto-detect?
PCAN,
CANOE,
CANHACKER,

View File

@ -6,10 +6,7 @@ import com.rusefi.sensor_logs.BinaryLogEntry;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Objects;
import java.util.*;
public class DbcFile {
public final LinkedHashMap<Integer, DbcPacket> packets = new LinkedHashMap<>();
@ -25,7 +22,7 @@ public class DbcFile {
}
public static DbcFile readFromFile(String fileName) throws IOException {
System.out.println("Reading DBC file from " + fileName + "...");
System.out.println(new Date() + " Reading DBC file from " + fileName + "...");
DbcFile dbc = new DbcFile(LoggingStrategy.LOG_ONLY_TRANSLATED_FIELDS);
{
BufferedReader reader = new BufferedReader(new FileReader(fileName));
@ -115,7 +112,6 @@ public class DbcFile {
return packets.get(canId);
}
public List<BinaryLogEntry> getFieldNameEntries() {
if (list == null) {
list = LoggingStrategy.getFieldNameEntries(this, logOnlyTranslatedFields);

View File

@ -26,10 +26,18 @@ public class AutoFormatReader implements CANLineReader {
@Override
public List<CANPacket> readFile(String fileName) throws IOException {
String firstLine = Files.lines(Paths.get(fileName)).findFirst().get();
if (!firstLine.contains(PcanTrcReader2_0.FILEVERSION) && !firstLine.contains(CanHackerReader.HEADER))
if (!firstLine.contains(PcanTrcReader2_0.FILEVERSION)
&& !firstLine.contains(CanHackerReader.HEADER)
&& !firstLine.contains(SomethingLinuxReader.HEADER)
&& !firstLine.contains(BusMasterReader.HEADER)
)
throw new IllegalStateException(PcanTrcReader2_0.FILEVERSION + " expected in first line");
if (firstLine.contains(CanHackerReader.HEADER)) {
delegate = CanHackerReader.INSTANCE;
} else if (firstLine.contains(SomethingLinuxReader.HEADER)) {
delegate = SomethingLinuxReader.INSTANCE;
} else if (firstLine.contains(BusMasterReader.HEADER)) {
delegate = BusMasterReader.INSTANCE;
} else if (firstLine.contains("1.1")) {
delegate = PcanTrcReader1_1.INSTANCE;
} else if (firstLine.contains("2.0")) {
@ -37,7 +45,11 @@ public class AutoFormatReader implements CANLineReader {
} else {
throw new IllegalStateException("Unsupported version in " + firstLine);
}
return delegate.readFile(fileName);
try {
return delegate.readFile(fileName);
} catch (Throwable e) {
throw new IllegalStateException("While " + fileName, e);
}
}
@Override

View File

@ -0,0 +1,45 @@
package com.rusefi.can.reader.impl;
import com.rusefi.can.CANPacket;
import com.rusefi.can.reader.CANLineReader;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
public enum BusMasterReader implements CANLineReader {
INSTANCE;
public static final CharSequence HEADER = "BUSMASTER Ver 3";
final String FORMAT = "HH:mm:ss:SSS";
DateFormat formatter = new SimpleDateFormat(FORMAT);
@Override
public CANPacket readLine(String line, String fileName, int lineIndex) {
if (line.startsWith("***"))
return null;
String trimmed = line.trim();
if (trimmed.isEmpty())
return null;
String[] tokens = trimmed.split("\\s+");
if (tokens.length < 7)
throw new IllegalStateException("Unexpected " + Arrays.toString(tokens));
String hexId = tokens[3];
String lenghtString = tokens[5];
int sid = Integer.parseInt(hexId.substring(2), 16);
int size = Integer.parseInt(lenghtString);
long timeStamp;
byte[] data = CANLineReader.readHexArray(tokens, 6, size);
try {
Date date = formatter.parse(tokens[0]);
timeStamp = date.getTime();
} catch (ParseException e) {
throw new RuntimeException(e);
}
return new CANPacket(timeStamp, sid, data);
}
}

View File

@ -0,0 +1,38 @@
package com.rusefi.can.reader.impl;
import com.rusefi.can.CANPacket;
import com.rusefi.can.reader.CANLineReader;
public enum SomethingLinuxReader implements CANLineReader {
INSTANCE;
public static final CharSequence HEADER = ") can0 ";
@Override
public CANPacket readLine(String line, String fileName, int lineIndex) {
String trimmed = line.trim();
if (trimmed.isEmpty())
return null;
String[] tokens = trimmed.split("\\s+");
if (tokens.length != 3)
throw new IllegalStateException("Three tokens expected [" + trimmed + "]");
String time = tokens[0].substring(1, tokens[0].length() - 2);
String mainToken = tokens[2];
int poundIndex = mainToken.indexOf('#');
int sid = Integer.parseInt(mainToken.substring(0, poundIndex), 16);
String hex = mainToken.substring(poundIndex + 1);
if (hex.length() % 2 != 0)
throw new IllegalStateException("Even length expected " + hex);
byte[] data = new byte[hex.length() / 2];
for (int i = 0 ;i < data.length; i++) {
String twoSymbols = hex.substring(2 * i, 2 * i + 2);
data[i] = (byte) Integer.parseInt(twoSymbols, 16);
}
return new CANPacket(Double.parseDouble(time), sid, data);
}
}

View File

@ -8,6 +8,7 @@ import com.rusefi.sensor_logs.BinaryLogEntry;
import com.rusefi.sensor_logs.BinarySensorLog;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
public class LoggingStrategy {
@ -29,6 +30,7 @@ public class LoggingStrategy {
public static void writeLog(DbcFile dbc, List<CANPacket> packets, String outputFileName) {
List<BinaryLogEntry> entries = dbc.getFieldNameEntries();
System.out.println(new Date() + " writeLog... " + outputFileName);
LoggingContext context = new LoggingContext();
BinarySensorLog<BinaryLogEntry> log = context.getBinaryLogEntryBinarySensorLog(entries, outputFileName);
@ -44,6 +46,7 @@ public class LoggingStrategy {
};
context.writeLogContent(packets, log, logger);
System.out.println(new Date() + " writeLog " + outputFileName + " done!");
}
public interface PacketLogger {

View File

@ -106,7 +106,7 @@ public class BinarySensorLog<T extends BinaryLogEntry> implements SensorLog, Aut
stream.writeInt((int) (System.currentTimeMillis() / 1000));
// 000ch
int infoDataStart = Fields.MLQ_HEADER_SIZE + Fields.MLQ_FIELD_HEADER_SIZE * entries.size();
System.out.println("Total " + entries.size() + " fields");
System.out.println(new Date() + " Total " + entries.size() + " fields");
stream.writeInt(infoDataStart);
stream.writeInt(infoDataStart + headerText.length());
// 0012h
@ -140,7 +140,7 @@ public class BinarySensorLog<T extends BinaryLogEntry> implements SensorLog, Aut
@Override
public void close() {
System.out.println("Finishing " + outputFileName);
System.out.println(new Date() + " Finishing " + outputFileName);
close(stream);
stream = null;
}

View File

@ -1,15 +1,11 @@
package com.rusefi.can;
import com.rusefi.can.analysis.ByteRateOfChangeReports;
import com.rusefi.can.reader.ReaderType;
import com.rusefi.can.reader.ReaderTypeHolder;
import java.io.IOException;
public class ByteRateOfChangeVagSandbox {
public static void main(String[] args) throws IOException {
ReaderTypeHolder.INSTANCE.type = ReaderType.PCAN;
String inputFolderName = "C:\\stuff\\rusefi_documentation\\OEM-Docs\\VAG\\2006-Passat-B6";
ByteRateOfChangeReports.scanInputFolder(inputFolderName, "fast-acceleration3.trc");

View File

@ -1,24 +1,11 @@
package com.rusefi.can;
import com.rusefi.can.analysis.ByteRateOfChangeReports;
import com.rusefi.can.reader.CANLineReader;
import com.rusefi.can.reader.ReaderType;
import com.rusefi.can.reader.ReaderTypeHolder;
import com.rusefi.can.reader.impl.PcanTrcReader2_0;
import java.io.IOException;
public class CanAmMG1Sandbox {
public static void main(String[] args) throws IOException {
ReaderTypeHolder.INSTANCE.type = ReaderType.PCAN;
CANLineReader reader = PcanTrcReader2_0.INSTANCE;
String inputFolderName = "C:\\stuff\\rusefi_documentation\\OEM-Docs\\CanAm\\maverick-x3-xrs-turbo-rr-max-2021\\";
// String revvingCltWentUp = inputFolderName + "engine-revving-CLT-increased.trc";
// List<CANPacket> idling = reader.readFile(revvingCltWentUp);
//printStats(idling, "IDLING");
ByteRateOfChangeReports.scanInputFolder(inputFolderName, ".trc");
Launcher.main(new String[]{inputFolderName});
}
}

View File

@ -1,17 +1,11 @@
package com.rusefi.can;
import com.rusefi.can.analysis.ByteRateOfChangeReports;
import com.rusefi.can.reader.ReaderType;
import com.rusefi.can.reader.ReaderTypeHolder;
import java.io.IOException;
public class KiaSandbox {
public static void main(String[] args) throws IOException {
ReaderTypeHolder.INSTANCE.type = ReaderType.PCAN;
String inputFolderName = "C:\\stuff\\rusefi_documentation\\OEM-Docs\\Kia\\2013-CAN-logs";
ByteRateOfChangeReports.scanInputFolder(inputFolderName, ".trc");
Launcher.main(new String[]{inputFolderName});
}
}

View File

@ -1,14 +1,11 @@
package com.rusefi.can;
import com.rusefi.can.analysis.ByteRateOfChangeReports;
import java.io.IOException;
public class NissanSandbox {
public static void main(String[] args) throws IOException {
String inputFolderName = "C:\\stuff\\rusefi_documentation\\OEM-Docs\\Nissan\\2011_Xterra\\2011-nissan-CAN-June-2021";
ByteRateOfChangeReports.scanInputFolder(inputFolderName, ".trc");
Launcher.main(new String[]{inputFolderName});
}
}

View File

@ -0,0 +1,15 @@
package com.rusefi.can;
import java.io.IOException;
public class RallySandbox {
public static void main(String[] args) throws IOException {
Launcher.main(new String[]{
"C:\\stuff\\rusefi_documentation\\OEM-Docs\\Mitsubishi\\2009-rallyart",
Launcher.FILENAME_SUFFIX_PROPERTY,
".log",
// Launcher.FILENAME_FILTER_PROPERTY,
// "auto",
});
}
}

View File

@ -1,45 +1,42 @@
package com.rusefi.can;
import com.rusefi.can.analysis.ByteRateOfChangeReports;
import com.rusefi.can.analysis.PerSidDump;
import com.rusefi.can.reader.CANLineReader;
import com.rusefi.can.reader.ReaderType;
import com.rusefi.can.reader.ReaderTypeHolder;
import com.rusefi.can.reader.dbc.DbcFile;
import com.rusefi.can.reader.impl.PcanTrcReader1_1;
import com.rusefi.can.reader.impl.ReadFullVagDbc;
import com.rusefi.mlv.LoggingStrategy;
import java.io.IOException;
import java.util.List;
import static com.rusefi.can.analysis.ByteRateOfChangeReports.createOutputFolder;
public class VagB6Sandbox {
public static void main(String[] args) throws IOException {
DbcFile dbc = DbcFile.readFromFile(ReadFullVagDbc.VAG_DBC_FILE);
// DbcFile dbc = DbcFile.readFromFile(ReadFullVagDbc.VAG_DBC_FILE);
CANLineReader reader = new PcanTrcReader1_1();
String inputFolderName = "C:\\stuff\\rusefi_documentation\\OEM-Docs\\VAG\\2006-Passat-B6\\";
{
ReaderTypeHolder.INSTANCE.type = ReaderType.PCAN;
ByteRateOfChangeReports.scanInputFolder(inputFolderName + "2023.10-tcu-side2", ".trc");
}
Launcher.main(new String[]{inputFolderName,
Launcher.FILENAME_FILTER_PROPERTY,
"passat-back-and-forth-60-seconds",
Launcher.DBC_FILENAME_PROPERTY,
ReadFullVagDbc.VAG_DBC_FILE
});
// {
// ReaderTypeHolder.INSTANCE.type = ReaderType.PCAN;
// ByteRateOfChangeReports.scanInputFolder(inputFolderName + "2023.10-tcu-side2", ".trc");
// }
// String name = "passat-back-and-forth-60-seconds";
// String simpleFileName = name + ".trc";
String simpleFileName = "passat-back-and-forth-60-seconds.trc1";
String file = inputFolderName + simpleFileName;
String reportDestinationFolder = createOutputFolder(inputFolderName);
List<CANPacket> packets = reader.readFile(file);
PerSidDump.handle(reportDestinationFolder, simpleFileName, packets);
String outputFileName = "vag.mlg";
LoggingStrategy.writeLog(dbc, packets, outputFileName);
//
// CANLineReader reader = new AutoFormatReader();
// String file = inputFolderName + simpleFileName;
//
// String reportDestinationFolder = createOutputFolder(inputFolderName);
//
//
// List<CANPacket> packets = reader.readFile(file);
//
// PerSidDump.handle(reportDestinationFolder, simpleFileName, packets);
//
//
}
}

View File

@ -0,0 +1,22 @@
package com.rusefi.can.reader.impl;
import com.rusefi.can.CANPacket;
import org.junit.Test;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
import static org.junit.Assert.assertEquals;
public class BusMasterReaderTest {
@Test
public void test() throws IOException {
String line = "21:27:52:3456 Rx 1 0x309 s 7 47 D8 7F D4 A8 00 05 ";
BufferedReader reader = new BufferedReader(new StringReader(line));
CANPacket packet = BusMasterReader.INSTANCE.readLine(reader.readLine());
assertEquals(packet.getData().length, 7);
assertEquals(packet.getData()[0], 0x47);
assertEquals(packet.getData()[6], 0x05);
}
}

View File

@ -0,0 +1,22 @@
package com.rusefi.can.reader.impl;
import com.rusefi.can.CANPacket;
import org.junit.Test;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
import static org.junit.Assert.assertEquals;
public class SomethingLinuxReaderTest {
@Test
public void test() throws IOException {
String line = "(1666123717.446553) can0 215#09344FA34F8E01";
BufferedReader reader = new BufferedReader(new StringReader(line));
CANPacket packet = SomethingLinuxReader.INSTANCE.readLine(reader.readLine());
assertEquals(packet.getData().length, 7);
assertEquals(packet.getData()[0], 0x09);
assertEquals(packet.getData()[6], 0x01);
}
}