rusefi
/
rusefi-full
mirror of https://github.com/rusefi/rusefi-full.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

remove cache zip, rely on LazyFile instead (#3639) 

* don't use cache.zip

* delete cache zip

* withConstructor

* I can't type

* bad merge

* jar

* jar
This commit is contained in:
Matthew Kennedy 2021-12-01 20:00:54 -08:00 committed by GitHub
parent bf98a53872
commit 1a618c9d6f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 14 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
firmware/tunerstudio/generated/cache.zip
View File

Binary file not shown.

BIN
java_tools/ConfigDefinition.jar
View File

Binary file not shown.

60
java_tools/configuration_definition/src/main/java/com/rusefi/ConfigDefinition.java
View File

@ -6,7 +6,6 @@ import com.rusefi.generated.RusefiConfigGrammarParser;
import com.rusefi.newparse.ParseState; import com.rusefi.newparse.ParseState;
import com.rusefi.newparse.parsing.Definition; import com.rusefi.newparse.parsing.Definition;
import com.rusefi.output.*; import com.rusefi.output.*;
import com.rusefi.util.CachingStrategy;
import com.rusefi.util.IoUtils; import com.rusefi.util.IoUtils;
import com.rusefi.util.LazyFile; import com.rusefi.util.LazyFile;
import com.rusefi.util.SystemOut; import com.rusefi.util.SystemOut;
@ -49,8 +48,6 @@ public class ConfigDefinition {
public static final String KEY_PREPEND = "-prepend"; public static final String KEY_PREPEND = "-prepend";
public static final String KEY_SIGNATURE = "-signature"; public static final String KEY_SIGNATURE = "-signature";
public static final String KEY_SIGNATURE_DESTINATION = "-signature_destination"; public static final String KEY_SIGNATURE_DESTINATION = "-signature_destination";
public static final String KEY_CACHE = "-cache";
public static final String KEY_CACHE_ZIP_FILE = "-cache_zip_file";
private static final String KEY_ZERO_INIT = "-initialize_to_zero"; private static final String KEY_ZERO_INIT = "-initialize_to_zero";
private static final String KEY_BOARD_NAME = "-board"; private static final String KEY_BOARD_NAME = "-board";
/** /**
@ -95,8 +92,6 @@ public class ConfigDefinition {
List<String> prependFiles = new ArrayList<>(); List<String> prependFiles = new ArrayList<>();
String romRaiderInputFile = null; String romRaiderInputFile = null;
String firingEnumFileName = null; String firingEnumFileName = null;
String cachePath = null;
String cacheZipFile = null;
String signatureDestination = null; String signatureDestination = null;
String signaturePrependFile = null; String signaturePrependFile = null;
List<String> enumInputFiles = new ArrayList<>(); List<String> enumInputFiles = new ArrayList<>();
@ -106,7 +101,7 @@ public class ConfigDefinition {
// used to update other files // used to update other files
List<String> inputFiles = new ArrayList<>(); List<String> inputFiles = new ArrayList<>();
// disable the lazy checks because we use timestamps to detect changes // disable the lazy checks because we use timestamps to detect changes
LazyFile.setLazyFileEnabled(false); LazyFile.setLazyFileEnabled(true);
ReaderState state = new ReaderState(); ReaderState state = new ReaderState();
@ -169,12 +164,6 @@ public class ConfigDefinition {
case EnumToString.KEY_ENUM_INPUT_FILE: case EnumToString.KEY_ENUM_INPUT_FILE:
enumInputFiles.add(args[i + 1]); enumInputFiles.add(args[i + 1]);
break; break;
case KEY_CACHE:
cachePath = args[i + 1];
break;
case KEY_CACHE_ZIP_FILE:
cacheZipFile = args[i + 1];
break;
case "-ts_output_name": case "-ts_output_name":
TSProjectConsumer.TS_FILE_OUTPUT_NAME = args[i + 1]; TSProjectConsumer.TS_FILE_OUTPUT_NAME = args[i + 1];
break; break;
@ -198,18 +187,9 @@ public class ConfigDefinition {
} }
} }
List<String> inputAllFiles = new ArrayList<>(inputFiles);
if (tsInputFileFolder != null) { if (tsInputFileFolder != null) {
// used to update .ini files // used to update .ini files
inputAllFiles.add(TSProjectConsumer.getTsFileInputName(tsInputFileFolder)); inputFiles.add(TSProjectConsumer.getTsFileInputName(tsInputFileFolder));
}
boolean needToUpdateTsFiles = isNeedToUpdateTsFiles(tsInputFileFolder, cachePath, cacheZipFile, inputAllFiles);
boolean needToUpdateOtherFiles = CachingStrategy.checkIfOutputFilesAreOutdated(inputFiles, cachePath, cacheZipFile);
if (!needToUpdateTsFiles && !needToUpdateOtherFiles) {
SystemOut.println("All output files are up-to-date, nothing to do here!");
return;
} }
if (!enumInputFiles.isEmpty()) { if (!enumInputFiles.isEmpty()) {
@ -222,7 +202,7 @@ public class ConfigDefinition {
ParseState parseState = new ParseState(state.enumsReader); ParseState parseState = new ParseState(state.enumsReader);
// Add the variable for the config signature // Add the variable for the config signature
long crc32 = signatureHash(state, parseState, tsInputFileFolder, inputAllFiles); long crc32 = signatureHash(state, parseState, tsInputFileFolder, inputFiles);
handleFiringOrder(firingEnumFileName, state.variableRegistry, parseState); handleFiringOrder(firingEnumFileName, state.variableRegistry, parseState);
@ -276,7 +256,7 @@ public class ConfigDefinition {
destinations.add(new DataLogConsumer(TS_OUTPUTS_DESTINATION + File.separator + "generated/data_logs.ini", state)); destinations.add(new DataLogConsumer(TS_OUTPUTS_DESTINATION + File.separator + "generated/data_logs.ini", state));
destinations.add(new GaugeConsumer(TS_OUTPUTS_DESTINATION + File.separator + "generated/gauges.ini", state)); destinations.add(new GaugeConsumer(TS_OUTPUTS_DESTINATION + File.separator + "generated/gauges.ini", state));
} }
if (tsInputFileFolder != null && needToUpdateTsFiles) { if (tsInputFileFolder != null) {
CharArrayWriter tsWriter = new CharArrayWriter(); CharArrayWriter tsWriter = new CharArrayWriter();
destinations.add(new TSProjectConsumer(tsWriter, tsInputFileFolder, state)); destinations.add(new TSProjectConsumer(tsWriter, tsInputFileFolder, state));
@ -286,13 +266,12 @@ public class ConfigDefinition {
tmpRegistry.readPrependValues(signaturePrependFile); tmpRegistry.readPrependValues(signaturePrependFile);
destinations.add(new SignatureConsumer(signatureDestination, tmpRegistry)); destinations.add(new SignatureConsumer(signatureDestination, tmpRegistry));
} }
if (needToUpdateOtherFiles) {
if (destCHeaderFileName != null) { if (destCHeaderFileName != null) {
destinations.add(new CHeaderConsumer(state.variableRegistry, destCHeaderFileName)); destinations.add(new CHeaderConsumer(state.variableRegistry, destCHeaderFileName));
} }
if (javaDestinationFileName != null) { if (javaDestinationFileName != null) {
destinations.add(new FileJavaFieldsConsumer(state, javaDestinationFileName)); destinations.add(new FileJavaFieldsConsumer(state, javaDestinationFileName));
}
} }
if (destinations.isEmpty()) if (destinations.isEmpty())
@ -303,15 +282,13 @@ public class ConfigDefinition {
*/ */
state.readBufferedReader(definitionReader, destinations); state.readBufferedReader(definitionReader, destinations);
if (destCDefinesFileName != null) {
if (destCDefinesFileName != null && needToUpdateOtherFiles)
writeDefinesToFile(state.variableRegistry, destCDefinesFileName); writeDefinesToFile(state.variableRegistry, destCDefinesFileName);
if (romRaiderDestination != null && romRaiderInputFile != null && needToUpdateOtherFiles) {
processTextTemplate(state, romRaiderInputFile, romRaiderDestination);
} }
CachingStrategy.saveCachedInputFiles(inputAllFiles, cachePath, cacheZipFile); if (romRaiderDestination != null && romRaiderInputFile != null) {
processTextTemplate(state, romRaiderInputFile, romRaiderDestination);
}
} }
private static String readFile(String fileName) { private static String readFile(String fileName) {
@ -363,15 +340,6 @@ public class ConfigDefinition {
return crc32; return crc32;
} }
private static boolean isNeedToUpdateTsFiles(String tsPath, String cachePath, String cacheZipFile, List<String> inputAllFiles) {
boolean needToUpdateTsFiles = false;
if (tsPath != null) {
SystemOut.println("Check the input/output TS files:");
needToUpdateTsFiles = CachingStrategy.checkIfOutputFilesAreOutdated(inputAllFiles, cachePath, cacheZipFile);
}
return needToUpdateTsFiles;
}
public static void processYamls(VariableRegistry registry, File[] yamlFiles, ReaderState state) throws IOException { public static void processYamls(VariableRegistry registry, File[] yamlFiles, ReaderState state) throws IOException {
ArrayList<Map<String, Object>> listPins = new ArrayList<>(); ArrayList<Map<String, Object>> listPins = new ArrayList<>();
for (File yamlFile : yamlFiles) { for (File yamlFile : yamlFiles) {

76
java_tools/configuration_definition/src/main/java/com/rusefi/util/CachingStrategy.java
View File

@ -1,76 +0,0 @@
package com.rusefi.util;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.Arrays;
import java.util.List;
public class CachingStrategy {
public static boolean checkIfOutputFilesAreOutdated(List<String> inputFileNames, String cachePath, String cacheZipFile) {
if (cachePath == null)
return true;
SystemOut.println("Check the input/output other files:");
// find if any input file was changed from the cached version
for (String inputFileName : inputFileNames) {
File inputFile = new File(inputFileName);
try {
byte[] inputFileContent = Files.readAllBytes(inputFile.toPath());
byte[] f2;
if (cacheZipFile != null) {
f2 = ZipUtil.unzipFileContents(cacheZipFile, cachePath + File.separator + inputFileName);
} else {
String cachedFileName = getCachedInputFileName(cachePath, inputFile.getName());
SystemOut.println("* cache ZIP file not specified, reading " + cachedFileName + " vs " + inputFileName);
/**
* todo: do we have a bug in this branch? how often do we simply read same 'inputFile'?
*/
File cachedFile = new File(cachedFileName);
f2 = Files.readAllBytes(cachedFile.toPath());
}
boolean isEqual = Arrays.equals(inputFileContent, f2);
if (!isEqual) {
SystemOut.println("* the file " + inputFileName + " is changed!");
return true;
} else {
SystemOut.println("* the file " + inputFileName + " is NOT changed!");
}
} catch (IOException e) {
SystemOut.println("* cannot validate the file " + inputFileName + ", so assuming it's changed.");
return true;
}
}
SystemOut.println("* all the files are up-to-date!");
return false;
}
public static boolean saveCachedInputFiles(List<String> inputFiles, String cachePath, String cacheZipFile) throws IOException {
if (cachePath == null) {
SystemOut.println("* cache storage is disabled.");
return false;
}
// copy all input files to the cache
if (cacheZipFile != null) {
ZipUtil.zipAddFiles(cacheZipFile, inputFiles, cachePath);
} else {
for (String iFile : inputFiles) {
File newFile = new File(iFile);
File cachedFile = new File(getCachedInputFileName(cachePath, newFile.getName()));
cachedFile.mkdirs();
try {
Files.copy(newFile.toPath(), cachedFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
SystemOut.println("* cannot store the cached file for " + iFile);
throw e;
}
}
}
SystemOut.println("* input files copied to the cached folder");
return true;
}
private static String getCachedInputFileName(String cachePath, String inputFile) {
return cachePath + File.separator + inputFile;
}
}

58
java_tools/configuration_definition/src/main/java/com/rusefi/util/ZipUtil.java
View File

@ -1,58 +0,0 @@
package com.rusefi.util;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URI;
import java.nio.file.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
public class ZipUtil {
public static boolean zipAddFiles(String zipFileName, List<String> fileNames, String zipPath) throws IOException {
// requires Java7+
Map<String, String> env = new HashMap<>();
env.put("create", "true");
Path path = Paths.get(zipFileName);
URI uri = URI.create("jar:" + path.toUri());
FileSystem fs = FileSystems.newFileSystem(uri, env);
for (String fileName : fileNames) {
String fileNameInZip = zipPath + File.separator + fileName;
Path extFile = Paths.get(fileName);
Path zippedFile = fs.getPath(fileNameInZip);
Files.createDirectories(zippedFile.getParent());
//fs.provider().checkAccess(zippedFile, AccessMode.READ);
Files.copy(extFile, zippedFile, StandardCopyOption.REPLACE_EXISTING);
}
fs.close();
return true;
}
public static byte[] unzipFileContents(String zipFileName, String fileName) throws IOException {
ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFileName));
ZipEntry zipEntry;
byte[] data = null;
while ((zipEntry = zis.getNextEntry()) != null) {
Path zippedName = Paths.get(zipEntry.getName()).normalize();
Path searchName = Paths.get(fileName).normalize();
if (zippedName.equals(searchName)) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
byte[] buffer = new byte[4096];
int read;
while ((read = zis.read(buffer)) != -1) {
baos.write(buffer, 0, read);
}
data = baos.toByteArray();
break;
}
}
zis.closeEntry();
zis.close();
System.out.println("Unzip " + zipFileName + ": " + fileName + (data != null ? " extracted!" : " failed!"));
return data;
}
}