gen_config is no more lazy (#1541)

* gen_config: replace lazyFile with timestamp checks

* board_generator: use LazyFile

* fix romraider and TOTAL_CONFIG_SIZE

* merge

Co-authored-by: Andrei <andreikagit@users.noreply.github.com>

firmware/gen_config_board.sh

@@ -15,6 +15,7 @@ BOARDNAME=$1
 echo "BOARDNAME=${BOARDNAME}"
 
 java -DSystemOut.name=gen_config_board \
+	-Drusefi.generator.lazyfile.enabled=true \
 	-cp ../java_tools/ConfigDefinition.jar \
 	com.rusefi.board_generator.BoardReader \
 	-board ${BOARDNAME} \

firmware/integration/rusefi_template.xml

@@ -10,7 +10,7 @@
             <ecuid>RUSEFI</ecuid>
             <make>rusEfi</make>
             <model>rusEfi</model>
-            <filesize>@@total_config_size@@</filesize>
+            <filesize>@@TOTAL_CONFIG_SIZE@@</filesize>
         </romid>
 
         <table type="3D" name="Ignition Advance"

java_console/logging/src/com/rusefi/util/LazyFile.java

@@ -12,6 +12,7 @@ public class LazyFile implements Output {
     public static final String LAZY_FILE_TAG = "was generated automatically by rusEfi tool ";
     private static final String PROPERTY_NAME = "rusefi.generator.lazyfile.enabled";
     private static boolean ENABLED = Boolean.getBoolean(PROPERTY_NAME);
+    private static boolean isLazyCheckEnabled = true;
 
     static {
         SystemOut.println(PROPERTY_NAME + "=" + ENABLED);
@@ -41,7 +42,8 @@ public class LazyFile implements Output {
     public void close() throws IOException {
         String fileContent = unifySpaces(readCurrentContent(filename));
         String newContent = unifySpaces(contentWithoutTag.toString());
-        if (fileContent.equals(newContent)) {
+
+        if (fileContent.equals(newContent) && LazyFile.isLazyCheckEnabled()) {
             SystemOut.println(getClass().getSimpleName() + ": Not updating " + filename + " since looks to be the same content, new content size=" + contentWithoutTag.length());
             return;
         }
@@ -60,6 +62,14 @@ public class LazyFile implements Output {
         fw.close();
     }
 
+    public static void setLazyFileEnabled(boolean isEnabled) {
+        LazyFile.isLazyCheckEnabled = isEnabled;
+    }
+
+    public static boolean isLazyCheckEnabled() {
+        return LazyFile.isLazyCheckEnabled;
+    }
+
     public static String unifySpaces(String line) {
         line = line.replace("\r", "");
         return line.replaceAll("\n[\n]*", "");

java_tools/configuration_definition/src/com/rusefi/ConfigDefinition.java

@@ -10,6 +10,7 @@ import java.lang.reflect.Array;
 import java.math.BigInteger;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
@@ -40,6 +41,7 @@ public class ConfigDefinition {
     private static final String KEY_ROMRAIDER_DESTINATION = "-romraider_destination";
     private static final String KEY_FIRING = "-firing_order";
     public static final String KEY_PREPEND = "-prepend";
+    public static final String KEY_SIGNATURE = "-signature";
     private static final String KEY_SKIP = "-skip";
     private static final String KEY_ZERO_INIT = "-initialize_to_zero";
     public static boolean needZeroInit = true;
@@ -86,52 +88,96 @@ public class ConfigDefinition {
         List<String> prependFiles = new ArrayList<>();
         String skipRebuildFile = null;
         String romRaiderInputFile = null;
+        String firingEnumFileName = null;
         CHeaderConsumer.withC_Defines = true;
 
+        // used to update .ini files
+        List<String> inputTsFiles = new ArrayList<>();
+        List<String> outputTsFiles = new ArrayList<>();
+        // used to update other files
+        List<String> inputFiles = new ArrayList<>();
+        List<String> outputFiles = new ArrayList<>();
+        // disable the lazy checks because we use timestamps to detect changes
+        LazyFile.setLazyFileEnabled(false);
+
         for (int i = 0; i < args.length - 1; i += 2) {
             String key = args[i];
             if (key.equals("-tool")) {
                 ConfigDefinition.TOOL = args[i + 1];
             } else if (key.equals(KEY_DEFINITION)) {
                 definitionInputFile = args[i + 1];
+                inputFiles.add(definitionInputFile);
             } else if (key.equals(KEY_TS_DESTINATION)) {
                 tsPath = args[i + 1];
             } else if (key.equals(KEY_C_DESTINATION)) {
                 destCHeaderFileName = args[i + 1];
+                outputFiles.add(destCHeaderFileName);
             } else if (key.equals(KEY_C_FSIO_GETTERS)) {
                 destCFsioGettersFileName = args[i + 1];
+                outputFiles.add(destCFsioGettersFileName);
             } else if (key.equals(KEY_C_FSIO_STRING)) {
                 stringsCFileName = args[i + 1];
+                outputFiles.add(stringsCFileName);
             } else if (key.equals(KEY_C_FSIO_NAMES)) {
                 namesCFileName = args[i + 1];
+                outputFiles.add(namesCFileName);
             } else if (key.equals(KEY_C_FSIO_CONSTANTS)) {
                 destCFsioConstantsFileName = args[i + 1];
+                outputFiles.add(destCFsioConstantsFileName);
             } else if (key.equals(KEY_ZERO_INIT)) {
                 needZeroInit = Boolean.parseBoolean(args[i + 1]);
             } else if (key.equals(KEY_WITH_C_DEFINES)) {
                 CHeaderConsumer.withC_Defines = Boolean.parseBoolean(args[i + 1]);
             } else if (key.equals(KEY_C_DEFINES)) {
                 destCDefinesFileName = args[i + 1];
+                outputFiles.add(destCDefinesFileName);
             } else if (key.equals(KEY_JAVA_DESTINATION)) {
                 javaDestinationFileName = args[i + 1];
+                outputFiles.add(javaDestinationFileName);
             } else if (key.equals(KEY_FIRING)) {
-                String firingEnumFileName = args[i + 1];
+                firingEnumFileName = args[i + 1];
-                SystemOut.println("Reading firing from " + firingEnumFileName);
+                inputFiles.add(firingEnumFileName);
-                VariableRegistry.INSTANCE.register("FIRINGORDER", FiringOrderTSLogic.invoke(firingEnumFileName));
             } else if (key.equals(KEY_ROMRAIDER_DESTINATION)) {
                 romRaiderDestination = args[i + 1];
+                outputFiles.add(romRaiderDestination);
             } else if (key.equals(KEY_PREPEND)) {
                 prependFiles.add(args[i + 1]);
+                inputFiles.add(args[i + 1]);
+            } else if (key.equals(KEY_SIGNATURE)) {
+                prependFiles.add(args[i + 1]);
+                // don't add this file to the 'inputFiles'
             } else if (key.equals(KEY_SKIP)) {
                 // is this now not needed in light if LazyFile surving the same goal of not changing output unless needed?
                 skipRebuildFile = args[i + 1];
             } else if (key.equals("-ts_output_name")) {
                 TSProjectConsumer.TS_FILE_OUTPUT_NAME = args[i + 1];
             } else if (key.equals(KEY_ROM_INPUT)) {
-                romRaiderInputFile = args[i + 1];
+                String inputFilePath = args[i + 1];
+                romRaiderInputFile  = inputFilePath + File.separator + ROM_RAIDER_XML_TEMPLATE;
+                inputFiles.add(romRaiderInputFile);
             }
         }
 
+        if (tsPath != null) {
+            inputTsFiles = new ArrayList<>(inputFiles);
+            inputTsFiles.add(TSProjectConsumer.getTsFileInputName(tsPath));
+            outputTsFiles.add(TSProjectConsumer.getTsFileOutputName(tsPath));
+        }
+
+        SystemOut.println("Check the input/output TS file timestamps:");
+        boolean needToUpdateTsFiles = checkIfOutputFilesAreOutdated(inputTsFiles, outputTsFiles);
+        SystemOut.println("Check the input/output other file timestamps:");
+        boolean needToUpdateOtherFiles = checkIfOutputFilesAreOutdated(inputFiles, outputFiles);
+        if (!needToUpdateTsFiles && !needToUpdateOtherFiles)
+        {
+            SystemOut.println("All output files are up-to-date, nothing to do here!");
+            return;
+        }
+
+        if (firingEnumFileName != null) {
+            SystemOut.println("Reading firing from " + firingEnumFileName);
+            VariableRegistry.INSTANCE.register("FIRINGORDER", FiringOrderTSLogic.invoke(firingEnumFileName));
+        }
         MESSAGE = getGeneratedAutomaticallyTag() + definitionInputFile + " " + new Date();
 
         SystemOut.println("Reading definition from " + definitionInputFile);
@@ -153,13 +199,14 @@ public class ConfigDefinition {
         ReaderState state = new ReaderState();
 
         List<ConfigurationConsumer> destinations = new ArrayList<>();
-        if (destCHeaderFileName != null) {
+        if (tsPath != null && needToUpdateTsFiles) {
-            destinations.add(new CHeaderConsumer(destCHeaderFileName));
-        }
-        if (tsPath != null) {
             CharArrayWriter tsWriter = new CharArrayWriter();
             destinations.add(new TSProjectConsumer(tsWriter, tsPath, state));
         }
+        if (needToUpdateOtherFiles) {
+            if (destCHeaderFileName != null) {
+                destinations.add(new CHeaderConsumer(destCHeaderFileName));
+            }
             if (javaDestinationFileName != null) {
                 destinations.add(new FileJavaFieldsConsumer(state, javaDestinationFileName));
             }
@@ -171,6 +218,7 @@ public class ConfigDefinition {
                         namesCFileName,
                         stringsCFileName));
             }
+        }
 
         if (destinations.isEmpty())
             throw new IllegalArgumentException("No destinations specified");
@@ -178,12 +226,11 @@ public class ConfigDefinition {
 
 
 
-        if (destCDefinesFileName != null)
+        if (destCDefinesFileName != null && needToUpdateOtherFiles)
             VariableRegistry.INSTANCE.writeDefinesToFile(destCDefinesFileName);
 
-        if (romRaiderDestination != null && romRaiderInputFile != null) {
+        if (romRaiderDestination != null && romRaiderInputFile != null && needToUpdateOtherFiles) {
-            String inputFileName = romRaiderInputFile + File.separator + ROM_RAIDER_XML_TEMPLATE;
+            processTextTemplate(romRaiderInputFile, romRaiderDestination);
-            processTextTemplate(inputFileName, romRaiderDestination);
         }
         if (skipRebuildFile != null) {
             SystemOut.println("Writing " + currentMD5 + " to " + skipRebuildFile);
@@ -320,4 +367,27 @@ public class ConfigDefinition {
             throw new RuntimeException(e);
         }
     }
+
+    private static boolean checkIfOutputFilesAreOutdated(List<String> inputFiles, List<String> outputFiles) {
+        // first get the newest modified input file
+        long iFileLastModified = 0;
+            for (String iFile : inputFiles) {
+            File file = new File(iFile);
+            iFileLastModified = Math.max(iFileLastModified, file.lastModified());
+        }
+
+        // now check if any of the output files is older (or absent)
+        for (String oFile : outputFiles) {
+            File file = new File(oFile);
+            long oFileLastModified = file.lastModified();
+            if (oFileLastModified < iFileLastModified) { // lastModified=0 for absent files
+                System.out.println("* the file " + oFile + " is outdated:");
+                System.out.println("  inputModified = " + (new SimpleDateFormat("dd/MM/yyyy hh:mm:ss")).format(iFileLastModified));
+                System.out.println("  outputModified = " + (new SimpleDateFormat("dd/MM/yyyy hh:mm:ss")).format(oFileLastModified));
+                return true;
+            }
+        }
+        System.out.println("* all the files are up-to-date!");
+        return false;
+    }
 }

java_tools/configuration_definition/src/com/rusefi/board_generator/BoardReader.java

@@ -2,6 +2,8 @@ package com.rusefi.board_generator;
 
 import com.rusefi.EnumsReader;
 import com.rusefi.enum_reader.Value;
+import com.rusefi.util.LazyFile;
+import com.rusefi.util.Output;
 import com.rusefi.util.SystemOut;
 import org.yaml.snakeyaml.Yaml;
 
@@ -60,7 +62,7 @@ public class BoardReader {
         } else {
             SystemOut.println(data);
 
-            BufferedWriter bw = new BufferedWriter(new FileWriter(outputPath + File.separator + boardName + OUTPUT_FILE_PREFIX));
+            Output bw = new LazyFile(outputPath + File.separator + boardName + OUTPUT_FILE_PREFIX);
 
             bw.write(processSection(data, "brain_pin_e", "output_pin_e", "outputs", "GPIO_UNASSIGNED"));
             bw.write(processSection(data, "adc_channel_e", "adc_channel_e", "analog_inputs", "EFI_ADC_NONE"));

java_tools/configuration_definition/src/com/rusefi/output/TSProjectConsumer.java

@@ -108,7 +108,7 @@ public class TSProjectConsumer implements ConfigurationConsumer {
         SystemOut.println("Got " + tsContent.getPrefix().length() + "/" + tsContent.getPostfix().length() + " of " + TS_FILE_INPUT_NAME);
 
         // File.getPath() would eliminate potential separator at the end of the path
-        String fileName = new File(tsPath).getPath() + File.separator + TS_FILE_OUTPUT_NAME;
+        String fileName = getTsFileOutputName(new File(tsPath).getPath());
         Output tsHeader = new LazyFile(fileName);
         writeContent(fieldsSection, tsContent, tsHeader);
     }
@@ -132,7 +132,7 @@ public class TSProjectConsumer implements ConfigurationConsumer {
     }
 
     private static TsFileContent readTsFile(String tsPath) throws IOException {
-        String fileName = tsPath + File.separator + TS_FILE_INPUT_NAME;
+        String fileName = getTsFileInputName(tsPath);
         BufferedReader r = new BufferedReader(new InputStreamReader(new FileInputStream(fileName), CHARSET.name()));
 
         StringBuilder prefix = new StringBuilder();
@@ -192,6 +192,14 @@ public class TSProjectConsumer implements ConfigurationConsumer {
         return token;
     }
 
+    public static String getTsFileOutputName(String tsPath) {
+        return tsPath + File.separator + TS_FILE_OUTPUT_NAME;
+    }
+
+    public static String getTsFileInputName(String tsPath) {
+        return tsPath + File.separator + TS_FILE_INPUT_NAME;
+    }
+
     @Override
     public void startFile() {
     }