From 3635d0077a2c2f7685163b0b9b920620d477a910 Mon Sep 17 00:00:00 2001 From: andreika-git Date: Sun, 28 Jun 2020 17:00:11 +0300 Subject: [PATCH] gen_config cache (#1543) * use cache for gen_config * create dummy cache files to store them in github * fix typo * comment * text Co-authored-by: Andrei --- .../boards/kinetis/config/!gen_config.bat | 2 +- .../boards/kinetis/config/gen_config.sh | 3 +- firmware/gen_config.sh | 1 + firmware/gen_config_board.sh | 1 + firmware/tunerstudio/cache/firing_order.h | 0 .../cache/frankenso/firing_order.h | 0 .../cache/frankenso/frankenso_prefix.txt | 0 .../tunerstudio/cache/frankenso/prepend.txt | 0 .../tunerstudio/cache/frankenso/rusefi.input | 0 .../cache/frankenso/rusefi_config.txt | 0 .../tunerstudio/cache/kinetis/firing_order.h | 0 .../cache/kinetis/kinetis_prefix.txt | 0 .../tunerstudio/cache/kinetis/rusefi.input | 0 .../cache/kinetis/rusefi_config.txt | 0 .../cache/kinetis/rusefi_config_kinetis.txt | 0 .../cache/microrusefi/firing_order.h | 0 .../cache/microrusefi/microrusefi_prefix.txt | 0 .../tunerstudio/cache/microrusefi/prepend.txt | 0 .../cache/microrusefi/rusefi.input | 0 .../cache/microrusefi/rusefi_config.txt | 0 .../cache/prometheus/firing_order.h | 0 .../tunerstudio/cache/prometheus/prepend.txt | 0 .../cache/prometheus/prometheus_prefix.txt | 0 .../tunerstudio/cache/prometheus/rusefi.input | 0 .../cache/prometheus/rusefi_config.txt | 0 .../tunerstudio/cache/proteus/firing_order.h | 0 .../tunerstudio/cache/proteus/prepend.txt | 0 .../cache/proteus/proteus_prefix.txt | 0 .../tunerstudio/cache/proteus/rusefi.input | 0 .../cache/proteus/rusefi_config.txt | 0 firmware/tunerstudio/cache/rusefi.input | 0 firmware/tunerstudio/cache/rusefi_config.txt | 0 .../tunerstudio/cache/rusefi_template.xml | 0 .../src/com/rusefi/ConfigDefinition.java | 90 ++++++++++++------- 34 files changed, 62 insertions(+), 35 deletions(-) create mode 100644 firmware/tunerstudio/cache/firing_order.h create mode 100644 firmware/tunerstudio/cache/frankenso/firing_order.h create mode 100644 firmware/tunerstudio/cache/frankenso/frankenso_prefix.txt create mode 100644 firmware/tunerstudio/cache/frankenso/prepend.txt create mode 100644 firmware/tunerstudio/cache/frankenso/rusefi.input create mode 100644 firmware/tunerstudio/cache/frankenso/rusefi_config.txt create mode 100644 firmware/tunerstudio/cache/kinetis/firing_order.h create mode 100644 firmware/tunerstudio/cache/kinetis/kinetis_prefix.txt create mode 100644 firmware/tunerstudio/cache/kinetis/rusefi.input create mode 100644 firmware/tunerstudio/cache/kinetis/rusefi_config.txt create mode 100644 firmware/tunerstudio/cache/kinetis/rusefi_config_kinetis.txt create mode 100644 firmware/tunerstudio/cache/microrusefi/firing_order.h create mode 100644 firmware/tunerstudio/cache/microrusefi/microrusefi_prefix.txt create mode 100644 firmware/tunerstudio/cache/microrusefi/prepend.txt create mode 100644 firmware/tunerstudio/cache/microrusefi/rusefi.input create mode 100644 firmware/tunerstudio/cache/microrusefi/rusefi_config.txt create mode 100644 firmware/tunerstudio/cache/prometheus/firing_order.h create mode 100644 firmware/tunerstudio/cache/prometheus/prepend.txt create mode 100644 firmware/tunerstudio/cache/prometheus/prometheus_prefix.txt create mode 100644 firmware/tunerstudio/cache/prometheus/rusefi.input create mode 100644 firmware/tunerstudio/cache/prometheus/rusefi_config.txt create mode 100644 firmware/tunerstudio/cache/proteus/firing_order.h create mode 100644 firmware/tunerstudio/cache/proteus/prepend.txt create mode 100644 firmware/tunerstudio/cache/proteus/proteus_prefix.txt create mode 100644 firmware/tunerstudio/cache/proteus/rusefi.input create mode 100644 firmware/tunerstudio/cache/proteus/rusefi_config.txt create mode 100644 firmware/tunerstudio/cache/rusefi.input create mode 100644 firmware/tunerstudio/cache/rusefi_config.txt create mode 100644 firmware/tunerstudio/cache/rusefi_template.xml diff --git a/firmware/config/boards/kinetis/config/!gen_config.bat b/firmware/config/boards/kinetis/config/!gen_config.bat index e516105bc3..26cd37ac21 100644 --- a/firmware/config/boards/kinetis/config/!gen_config.bat +++ b/firmware/config/boards/kinetis/config/!gen_config.bat @@ -23,7 +23,7 @@ java ^ -jar ../java_tools/ConfigDefinition.jar ^ -definition integration/rusefi_config.txt ^ -ts_destination tunerstudio ^ - -tool kineris_gen_config.bat ^ + -tool kinetis_gen_config.bat ^ -firing_order controllers/algo/firing_order.h ^ -with_c_defines false ^ -initialize_to_zero false ^ diff --git a/firmware/config/boards/kinetis/config/gen_config.sh b/firmware/config/boards/kinetis/config/gen_config.sh index 2800f371cc..e4bc45ded1 100644 --- a/firmware/config/boards/kinetis/config/gen_config.sh +++ b/firmware/config/boards/kinetis/config/gen_config.sh @@ -20,8 +20,9 @@ java \ -Drusefi.generator.lazyfile.enabled=true \ -jar ../java_tools/ConfigDefinition.jar \ -definition integration/rusefi_config.txt \ + -cache tunerstudio/cache/kinetis \ -ts_destination tunerstudio \ - -tool kineris_gen_config.bat \ + -tool kinetis_gen_config.bat \ -firing_order controllers/algo/firing_order.h \ -with_c_defines false \ -initialize_to_zero false \ diff --git a/firmware/gen_config.sh b/firmware/gen_config.sh index 33bbfe5786..019adf76d2 100644 --- a/firmware/gen_config.sh +++ b/firmware/gen_config.sh @@ -17,6 +17,7 @@ java -DSystemOut.name=gen_config \ -definition integration/rusefi_config.txt \ -romraider integration \ -ts_destination tunerstudio \ + -cache tunerstudio/cache \ -with_c_defines false \ -initialize_to_zero false \ -tool gen_config.sh \ diff --git a/firmware/gen_config_board.sh b/firmware/gen_config_board.sh index b43aca517a..b60706b672 100644 --- a/firmware/gen_config_board.sh +++ b/firmware/gen_config_board.sh @@ -31,6 +31,7 @@ java -DSystemOut.name=gen_config_board \ -definition integration/rusefi_config.txt \ -tool gen_config.sh \ -ts_destination tunerstudio \ + -cache tunerstudio/cache/${BOARDNAME} \ -firing_order controllers/algo/firing_order.h \ -ts_output_name rusefi_${BOARDNAME}.ini \ -prepend tunerstudio/${BOARDNAME}_prefix.txt \ diff --git a/firmware/tunerstudio/cache/firing_order.h b/firmware/tunerstudio/cache/firing_order.h new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/frankenso/firing_order.h b/firmware/tunerstudio/cache/frankenso/firing_order.h new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/frankenso/frankenso_prefix.txt b/firmware/tunerstudio/cache/frankenso/frankenso_prefix.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/frankenso/prepend.txt b/firmware/tunerstudio/cache/frankenso/prepend.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/frankenso/rusefi.input b/firmware/tunerstudio/cache/frankenso/rusefi.input new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/frankenso/rusefi_config.txt b/firmware/tunerstudio/cache/frankenso/rusefi_config.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/kinetis/firing_order.h b/firmware/tunerstudio/cache/kinetis/firing_order.h new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/kinetis/kinetis_prefix.txt b/firmware/tunerstudio/cache/kinetis/kinetis_prefix.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/kinetis/rusefi.input b/firmware/tunerstudio/cache/kinetis/rusefi.input new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/kinetis/rusefi_config.txt b/firmware/tunerstudio/cache/kinetis/rusefi_config.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/kinetis/rusefi_config_kinetis.txt b/firmware/tunerstudio/cache/kinetis/rusefi_config_kinetis.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/microrusefi/firing_order.h b/firmware/tunerstudio/cache/microrusefi/firing_order.h new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/microrusefi/microrusefi_prefix.txt b/firmware/tunerstudio/cache/microrusefi/microrusefi_prefix.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/microrusefi/prepend.txt b/firmware/tunerstudio/cache/microrusefi/prepend.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/microrusefi/rusefi.input b/firmware/tunerstudio/cache/microrusefi/rusefi.input new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/microrusefi/rusefi_config.txt b/firmware/tunerstudio/cache/microrusefi/rusefi_config.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/prometheus/firing_order.h b/firmware/tunerstudio/cache/prometheus/firing_order.h new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/prometheus/prepend.txt b/firmware/tunerstudio/cache/prometheus/prepend.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/prometheus/prometheus_prefix.txt b/firmware/tunerstudio/cache/prometheus/prometheus_prefix.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/prometheus/rusefi.input b/firmware/tunerstudio/cache/prometheus/rusefi.input new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/prometheus/rusefi_config.txt b/firmware/tunerstudio/cache/prometheus/rusefi_config.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/proteus/firing_order.h b/firmware/tunerstudio/cache/proteus/firing_order.h new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/proteus/prepend.txt b/firmware/tunerstudio/cache/proteus/prepend.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/proteus/proteus_prefix.txt b/firmware/tunerstudio/cache/proteus/proteus_prefix.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/proteus/rusefi.input b/firmware/tunerstudio/cache/proteus/rusefi.input new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/proteus/rusefi_config.txt b/firmware/tunerstudio/cache/proteus/rusefi_config.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/rusefi.input b/firmware/tunerstudio/cache/rusefi.input new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/rusefi_config.txt b/firmware/tunerstudio/cache/rusefi_config.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/firmware/tunerstudio/cache/rusefi_template.xml b/firmware/tunerstudio/cache/rusefi_template.xml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/java_tools/configuration_definition/src/com/rusefi/ConfigDefinition.java b/java_tools/configuration_definition/src/com/rusefi/ConfigDefinition.java index f9aab8409f..d2924bc614 100644 --- a/java_tools/configuration_definition/src/com/rusefi/ConfigDefinition.java +++ b/java_tools/configuration_definition/src/com/rusefi/ConfigDefinition.java @@ -8,6 +8,8 @@ import com.rusefi.util.SystemOut; import java.io.*; import java.lang.reflect.Array; import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.text.SimpleDateFormat; @@ -42,6 +44,7 @@ public class ConfigDefinition { private static final String KEY_FIRING = "-firing_order"; public static final String KEY_PREPEND = "-prepend"; public static final String KEY_SIGNATURE = "-signature"; + public static final String KEY_CACHE = "-cache"; private static final String KEY_SKIP = "-skip"; private static final String KEY_ZERO_INIT = "-initialize_to_zero"; public static boolean needZeroInit = true; @@ -89,14 +92,13 @@ public class ConfigDefinition { String skipRebuildFile = null; String romRaiderInputFile = null; String firingEnumFileName = null; + String cachePath = null; CHeaderConsumer.withC_Defines = true; // used to update .ini files List inputTsFiles = new ArrayList<>(); - List outputTsFiles = new ArrayList<>(); // used to update other files List inputFiles = new ArrayList<>(); - List outputFiles = new ArrayList<>(); // disable the lazy checks because we use timestamps to detect changes LazyFile.setLazyFileEnabled(false); @@ -111,42 +113,37 @@ public class ConfigDefinition { tsPath = args[i + 1]; } else if (key.equals(KEY_C_DESTINATION)) { destCHeaderFileName = args[i + 1]; - outputFiles.add(destCHeaderFileName); } else if (key.equals(KEY_C_FSIO_GETTERS)) { destCFsioGettersFileName = args[i + 1]; - outputFiles.add(destCFsioGettersFileName); } else if (key.equals(KEY_C_FSIO_STRING)) { stringsCFileName = args[i + 1]; - outputFiles.add(stringsCFileName); } else if (key.equals(KEY_C_FSIO_NAMES)) { namesCFileName = args[i + 1]; - outputFiles.add(namesCFileName); } else if (key.equals(KEY_C_FSIO_CONSTANTS)) { destCFsioConstantsFileName = args[i + 1]; - outputFiles.add(destCFsioConstantsFileName); } else if (key.equals(KEY_ZERO_INIT)) { needZeroInit = Boolean.parseBoolean(args[i + 1]); } else if (key.equals(KEY_WITH_C_DEFINES)) { CHeaderConsumer.withC_Defines = Boolean.parseBoolean(args[i + 1]); } else if (key.equals(KEY_C_DEFINES)) { destCDefinesFileName = args[i + 1]; - outputFiles.add(destCDefinesFileName); } else if (key.equals(KEY_JAVA_DESTINATION)) { javaDestinationFileName = args[i + 1]; - outputFiles.add(javaDestinationFileName); } else if (key.equals(KEY_FIRING)) { firingEnumFileName = args[i + 1]; inputFiles.add(firingEnumFileName); } else if (key.equals(KEY_ROMRAIDER_DESTINATION)) { romRaiderDestination = args[i + 1]; - outputFiles.add(romRaiderDestination); } else if (key.equals(KEY_PREPEND)) { prependFiles.add(args[i + 1]); inputFiles.add(args[i + 1]); } else if (key.equals(KEY_SIGNATURE)) { prependFiles.add(args[i + 1]); // don't add this file to the 'inputFiles' - } else if (key.equals(KEY_SKIP)) { + } else if (key.equals(KEY_CACHE)) { + cachePath = args[i + 1]; + } + else if (key.equals(KEY_SKIP)) { // is this now not needed in light if LazyFile surving the same goal of not changing output unless needed? skipRebuildFile = args[i + 1]; } else if (key.equals("-ts_output_name")) { @@ -161,13 +158,12 @@ public class ConfigDefinition { if (tsPath != null) { inputTsFiles = new ArrayList<>(inputFiles); inputTsFiles.add(TSProjectConsumer.getTsFileInputName(tsPath)); - outputTsFiles.add(TSProjectConsumer.getTsFileOutputName(tsPath)); } - SystemOut.println("Check the input/output TS file timestamps:"); - boolean needToUpdateTsFiles = checkIfOutputFilesAreOutdated(inputTsFiles, outputTsFiles); - SystemOut.println("Check the input/output other file timestamps:"); - boolean needToUpdateOtherFiles = checkIfOutputFilesAreOutdated(inputFiles, outputFiles); + SystemOut.println("Check the input/output TS files:"); + boolean needToUpdateTsFiles = checkIfOutputFilesAreOutdated(inputTsFiles, cachePath); + SystemOut.println("Check the input/output other files:"); + boolean needToUpdateOtherFiles = checkIfOutputFilesAreOutdated(inputFiles, cachePath); if (!needToUpdateTsFiles && !needToUpdateOtherFiles) { SystemOut.println("All output files are up-to-date, nothing to do here!"); @@ -238,6 +234,8 @@ public class ConfigDefinition { writer.write(currentMD5); writer.close(); } + + saveCachedInputFiles(inputTsFiles, cachePath); } private static boolean needToSkipRebuild(String skipRebuildFile, String currentMD5) throws IOException { @@ -368,26 +366,52 @@ public class ConfigDefinition { } } - private static boolean checkIfOutputFilesAreOutdated(List inputFiles, List outputFiles) { - // first get the newest modified input file - long iFileLastModified = 0; - for (String iFile : inputFiles) { - File file = new File(iFile); - iFileLastModified = Math.max(iFileLastModified, file.lastModified()); - } - - // now check if any of the output files is older (or absent) - for (String oFile : outputFiles) { - File file = new File(oFile); - long oFileLastModified = file.lastModified(); - if (oFileLastModified < iFileLastModified) { // lastModified=0 for absent files - System.out.println("* the file " + oFile + " is outdated:"); - System.out.println(" inputModified = " + (new SimpleDateFormat("dd/MM/yyyy hh:mm:ss")).format(iFileLastModified)); - System.out.println(" outputModified = " + (new SimpleDateFormat("dd/MM/yyyy hh:mm:ss")).format(oFileLastModified)); + private static boolean checkIfOutputFilesAreOutdated(List inputFiles, String cachePath) { + if (cachePath == null) + return true; + // find if any input file was changed from the cached version + for (String iFile : inputFiles) { + File newFile = new File(iFile); + File cachedFile = new File(getCachedInputFile(newFile.getName(), cachePath)); + //boolean isEqual = FileUtils.contentEquals(newFile, cachedFile); + try { + byte[] f1 = Files.readAllBytes(newFile.toPath()); + byte[] f2 = Files.readAllBytes(cachedFile.toPath()); + boolean isEqual = Arrays.equals(f1, f2); + if (!isEqual) { + SystemOut.println("* the file " + iFile + " is changed!"); + return true; + } + } catch(java.io.IOException e) { + SystemOut.println("* cannot validate the file " + iFile + ", so assuming it's changed."); return true; } } - System.out.println("* all the files are up-to-date!"); + SystemOut.println("* all the files are up-to-date!"); return false; } + + private static boolean saveCachedInputFiles(List inputFiles, String cachePath) throws IOException { + if (cachePath == null) { + SystemOut.println("* cache storage is disabled."); + return false; + } + // copy all input files to the cache + for (String iFile : inputFiles) { + File newFile = new File(iFile); + File cachedFile = new File(getCachedInputFile(newFile.getName(), cachePath)); + cachedFile.mkdirs(); + try { + Files.copy(newFile.toPath(), cachedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } catch(java.io.IOException e) { + SystemOut.println("* cannot store the cached file for " + iFile); + throw e; + } + } + return true; + } + + private static String getCachedInputFile(String inputFile, String cachePath) { + return cachePath + File.separator + inputFile; + } }