mirror of https://github.com/rusefi/rusefi-1.git
remove cache zip, rely on LazyFile instead (#3639)
* don't use cache.zip * delete cache zip * withConstructor * I can't type * bad merge * jar * jar
This commit is contained in:
parent
bf98a53872
commit
1a618c9d6f
Binary file not shown.
Binary file not shown.
|
@ -6,7 +6,6 @@ import com.rusefi.generated.RusefiConfigGrammarParser;
|
||||||
import com.rusefi.newparse.ParseState;
|
import com.rusefi.newparse.ParseState;
|
||||||
import com.rusefi.newparse.parsing.Definition;
|
import com.rusefi.newparse.parsing.Definition;
|
||||||
import com.rusefi.output.*;
|
import com.rusefi.output.*;
|
||||||
import com.rusefi.util.CachingStrategy;
|
|
||||||
import com.rusefi.util.IoUtils;
|
import com.rusefi.util.IoUtils;
|
||||||
import com.rusefi.util.LazyFile;
|
import com.rusefi.util.LazyFile;
|
||||||
import com.rusefi.util.SystemOut;
|
import com.rusefi.util.SystemOut;
|
||||||
|
@ -49,8 +48,6 @@ public class ConfigDefinition {
|
||||||
public static final String KEY_PREPEND = "-prepend";
|
public static final String KEY_PREPEND = "-prepend";
|
||||||
public static final String KEY_SIGNATURE = "-signature";
|
public static final String KEY_SIGNATURE = "-signature";
|
||||||
public static final String KEY_SIGNATURE_DESTINATION = "-signature_destination";
|
public static final String KEY_SIGNATURE_DESTINATION = "-signature_destination";
|
||||||
public static final String KEY_CACHE = "-cache";
|
|
||||||
public static final String KEY_CACHE_ZIP_FILE = "-cache_zip_file";
|
|
||||||
private static final String KEY_ZERO_INIT = "-initialize_to_zero";
|
private static final String KEY_ZERO_INIT = "-initialize_to_zero";
|
||||||
private static final String KEY_BOARD_NAME = "-board";
|
private static final String KEY_BOARD_NAME = "-board";
|
||||||
/**
|
/**
|
||||||
|
@ -95,8 +92,6 @@ public class ConfigDefinition {
|
||||||
List<String> prependFiles = new ArrayList<>();
|
List<String> prependFiles = new ArrayList<>();
|
||||||
String romRaiderInputFile = null;
|
String romRaiderInputFile = null;
|
||||||
String firingEnumFileName = null;
|
String firingEnumFileName = null;
|
||||||
String cachePath = null;
|
|
||||||
String cacheZipFile = null;
|
|
||||||
String signatureDestination = null;
|
String signatureDestination = null;
|
||||||
String signaturePrependFile = null;
|
String signaturePrependFile = null;
|
||||||
List<String> enumInputFiles = new ArrayList<>();
|
List<String> enumInputFiles = new ArrayList<>();
|
||||||
|
@ -106,7 +101,7 @@ public class ConfigDefinition {
|
||||||
// used to update other files
|
// used to update other files
|
||||||
List<String> inputFiles = new ArrayList<>();
|
List<String> inputFiles = new ArrayList<>();
|
||||||
// disable the lazy checks because we use timestamps to detect changes
|
// disable the lazy checks because we use timestamps to detect changes
|
||||||
LazyFile.setLazyFileEnabled(false);
|
LazyFile.setLazyFileEnabled(true);
|
||||||
|
|
||||||
ReaderState state = new ReaderState();
|
ReaderState state = new ReaderState();
|
||||||
|
|
||||||
|
@ -169,12 +164,6 @@ public class ConfigDefinition {
|
||||||
case EnumToString.KEY_ENUM_INPUT_FILE:
|
case EnumToString.KEY_ENUM_INPUT_FILE:
|
||||||
enumInputFiles.add(args[i + 1]);
|
enumInputFiles.add(args[i + 1]);
|
||||||
break;
|
break;
|
||||||
case KEY_CACHE:
|
|
||||||
cachePath = args[i + 1];
|
|
||||||
break;
|
|
||||||
case KEY_CACHE_ZIP_FILE:
|
|
||||||
cacheZipFile = args[i + 1];
|
|
||||||
break;
|
|
||||||
case "-ts_output_name":
|
case "-ts_output_name":
|
||||||
TSProjectConsumer.TS_FILE_OUTPUT_NAME = args[i + 1];
|
TSProjectConsumer.TS_FILE_OUTPUT_NAME = args[i + 1];
|
||||||
break;
|
break;
|
||||||
|
@ -198,18 +187,9 @@ public class ConfigDefinition {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
List<String> inputAllFiles = new ArrayList<>(inputFiles);
|
|
||||||
if (tsInputFileFolder != null) {
|
if (tsInputFileFolder != null) {
|
||||||
// used to update .ini files
|
// used to update .ini files
|
||||||
inputAllFiles.add(TSProjectConsumer.getTsFileInputName(tsInputFileFolder));
|
inputFiles.add(TSProjectConsumer.getTsFileInputName(tsInputFileFolder));
|
||||||
}
|
|
||||||
|
|
||||||
boolean needToUpdateTsFiles = isNeedToUpdateTsFiles(tsInputFileFolder, cachePath, cacheZipFile, inputAllFiles);
|
|
||||||
|
|
||||||
boolean needToUpdateOtherFiles = CachingStrategy.checkIfOutputFilesAreOutdated(inputFiles, cachePath, cacheZipFile);
|
|
||||||
if (!needToUpdateTsFiles && !needToUpdateOtherFiles) {
|
|
||||||
SystemOut.println("All output files are up-to-date, nothing to do here!");
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!enumInputFiles.isEmpty()) {
|
if (!enumInputFiles.isEmpty()) {
|
||||||
|
@ -222,7 +202,7 @@ public class ConfigDefinition {
|
||||||
|
|
||||||
ParseState parseState = new ParseState(state.enumsReader);
|
ParseState parseState = new ParseState(state.enumsReader);
|
||||||
// Add the variable for the config signature
|
// Add the variable for the config signature
|
||||||
long crc32 = signatureHash(state, parseState, tsInputFileFolder, inputAllFiles);
|
long crc32 = signatureHash(state, parseState, tsInputFileFolder, inputFiles);
|
||||||
|
|
||||||
handleFiringOrder(firingEnumFileName, state.variableRegistry, parseState);
|
handleFiringOrder(firingEnumFileName, state.variableRegistry, parseState);
|
||||||
|
|
||||||
|
@ -276,7 +256,7 @@ public class ConfigDefinition {
|
||||||
destinations.add(new DataLogConsumer(TS_OUTPUTS_DESTINATION + File.separator + "generated/data_logs.ini", state));
|
destinations.add(new DataLogConsumer(TS_OUTPUTS_DESTINATION + File.separator + "generated/data_logs.ini", state));
|
||||||
destinations.add(new GaugeConsumer(TS_OUTPUTS_DESTINATION + File.separator + "generated/gauges.ini", state));
|
destinations.add(new GaugeConsumer(TS_OUTPUTS_DESTINATION + File.separator + "generated/gauges.ini", state));
|
||||||
}
|
}
|
||||||
if (tsInputFileFolder != null && needToUpdateTsFiles) {
|
if (tsInputFileFolder != null) {
|
||||||
CharArrayWriter tsWriter = new CharArrayWriter();
|
CharArrayWriter tsWriter = new CharArrayWriter();
|
||||||
destinations.add(new TSProjectConsumer(tsWriter, tsInputFileFolder, state));
|
destinations.add(new TSProjectConsumer(tsWriter, tsInputFileFolder, state));
|
||||||
|
|
||||||
|
@ -286,13 +266,12 @@ public class ConfigDefinition {
|
||||||
tmpRegistry.readPrependValues(signaturePrependFile);
|
tmpRegistry.readPrependValues(signaturePrependFile);
|
||||||
destinations.add(new SignatureConsumer(signatureDestination, tmpRegistry));
|
destinations.add(new SignatureConsumer(signatureDestination, tmpRegistry));
|
||||||
}
|
}
|
||||||
if (needToUpdateOtherFiles) {
|
|
||||||
if (destCHeaderFileName != null) {
|
if (destCHeaderFileName != null) {
|
||||||
destinations.add(new CHeaderConsumer(state.variableRegistry, destCHeaderFileName));
|
destinations.add(new CHeaderConsumer(state.variableRegistry, destCHeaderFileName));
|
||||||
}
|
}
|
||||||
if (javaDestinationFileName != null) {
|
if (javaDestinationFileName != null) {
|
||||||
destinations.add(new FileJavaFieldsConsumer(state, javaDestinationFileName));
|
destinations.add(new FileJavaFieldsConsumer(state, javaDestinationFileName));
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (destinations.isEmpty())
|
if (destinations.isEmpty())
|
||||||
|
@ -303,15 +282,13 @@ public class ConfigDefinition {
|
||||||
*/
|
*/
|
||||||
state.readBufferedReader(definitionReader, destinations);
|
state.readBufferedReader(definitionReader, destinations);
|
||||||
|
|
||||||
|
if (destCDefinesFileName != null) {
|
||||||
if (destCDefinesFileName != null && needToUpdateOtherFiles)
|
|
||||||
writeDefinesToFile(state.variableRegistry, destCDefinesFileName);
|
writeDefinesToFile(state.variableRegistry, destCDefinesFileName);
|
||||||
|
|
||||||
if (romRaiderDestination != null && romRaiderInputFile != null && needToUpdateOtherFiles) {
|
|
||||||
processTextTemplate(state, romRaiderInputFile, romRaiderDestination);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
CachingStrategy.saveCachedInputFiles(inputAllFiles, cachePath, cacheZipFile);
|
if (romRaiderDestination != null && romRaiderInputFile != null) {
|
||||||
|
processTextTemplate(state, romRaiderInputFile, romRaiderDestination);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static String readFile(String fileName) {
|
private static String readFile(String fileName) {
|
||||||
|
@ -363,15 +340,6 @@ public class ConfigDefinition {
|
||||||
return crc32;
|
return crc32;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static boolean isNeedToUpdateTsFiles(String tsPath, String cachePath, String cacheZipFile, List<String> inputAllFiles) {
|
|
||||||
boolean needToUpdateTsFiles = false;
|
|
||||||
if (tsPath != null) {
|
|
||||||
SystemOut.println("Check the input/output TS files:");
|
|
||||||
needToUpdateTsFiles = CachingStrategy.checkIfOutputFilesAreOutdated(inputAllFiles, cachePath, cacheZipFile);
|
|
||||||
}
|
|
||||||
return needToUpdateTsFiles;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void processYamls(VariableRegistry registry, File[] yamlFiles, ReaderState state) throws IOException {
|
public static void processYamls(VariableRegistry registry, File[] yamlFiles, ReaderState state) throws IOException {
|
||||||
ArrayList<Map<String, Object>> listPins = new ArrayList<>();
|
ArrayList<Map<String, Object>> listPins = new ArrayList<>();
|
||||||
for (File yamlFile : yamlFiles) {
|
for (File yamlFile : yamlFiles) {
|
||||||
|
|
|
@ -1,76 +0,0 @@
|
||||||
package com.rusefi.util;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.nio.file.Files;
|
|
||||||
import java.nio.file.StandardCopyOption;
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
public class CachingStrategy {
|
|
||||||
public static boolean checkIfOutputFilesAreOutdated(List<String> inputFileNames, String cachePath, String cacheZipFile) {
|
|
||||||
if (cachePath == null)
|
|
||||||
return true;
|
|
||||||
SystemOut.println("Check the input/output other files:");
|
|
||||||
// find if any input file was changed from the cached version
|
|
||||||
for (String inputFileName : inputFileNames) {
|
|
||||||
File inputFile = new File(inputFileName);
|
|
||||||
try {
|
|
||||||
byte[] inputFileContent = Files.readAllBytes(inputFile.toPath());
|
|
||||||
byte[] f2;
|
|
||||||
if (cacheZipFile != null) {
|
|
||||||
f2 = ZipUtil.unzipFileContents(cacheZipFile, cachePath + File.separator + inputFileName);
|
|
||||||
} else {
|
|
||||||
String cachedFileName = getCachedInputFileName(cachePath, inputFile.getName());
|
|
||||||
SystemOut.println("* cache ZIP file not specified, reading " + cachedFileName + " vs " + inputFileName);
|
|
||||||
/**
|
|
||||||
* todo: do we have a bug in this branch? how often do we simply read same 'inputFile'?
|
|
||||||
*/
|
|
||||||
File cachedFile = new File(cachedFileName);
|
|
||||||
f2 = Files.readAllBytes(cachedFile.toPath());
|
|
||||||
}
|
|
||||||
boolean isEqual = Arrays.equals(inputFileContent, f2);
|
|
||||||
if (!isEqual) {
|
|
||||||
SystemOut.println("* the file " + inputFileName + " is changed!");
|
|
||||||
return true;
|
|
||||||
} else {
|
|
||||||
SystemOut.println("* the file " + inputFileName + " is NOT changed!");
|
|
||||||
}
|
|
||||||
} catch (IOException e) {
|
|
||||||
SystemOut.println("* cannot validate the file " + inputFileName + ", so assuming it's changed.");
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
SystemOut.println("* all the files are up-to-date!");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static boolean saveCachedInputFiles(List<String> inputFiles, String cachePath, String cacheZipFile) throws IOException {
|
|
||||||
if (cachePath == null) {
|
|
||||||
SystemOut.println("* cache storage is disabled.");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
// copy all input files to the cache
|
|
||||||
if (cacheZipFile != null) {
|
|
||||||
ZipUtil.zipAddFiles(cacheZipFile, inputFiles, cachePath);
|
|
||||||
} else {
|
|
||||||
for (String iFile : inputFiles) {
|
|
||||||
File newFile = new File(iFile);
|
|
||||||
File cachedFile = new File(getCachedInputFileName(cachePath, newFile.getName()));
|
|
||||||
cachedFile.mkdirs();
|
|
||||||
try {
|
|
||||||
Files.copy(newFile.toPath(), cachedFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
|
|
||||||
} catch (IOException e) {
|
|
||||||
SystemOut.println("* cannot store the cached file for " + iFile);
|
|
||||||
throw e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
SystemOut.println("* input files copied to the cached folder");
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String getCachedInputFileName(String cachePath, String inputFile) {
|
|
||||||
return cachePath + File.separator + inputFile;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,58 +0,0 @@
|
||||||
package com.rusefi.util;
|
|
||||||
|
|
||||||
import java.io.ByteArrayOutputStream;
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.FileInputStream;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.net.URI;
|
|
||||||
import java.nio.file.*;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.zip.ZipEntry;
|
|
||||||
import java.util.zip.ZipInputStream;
|
|
||||||
|
|
||||||
public class ZipUtil {
|
|
||||||
public static boolean zipAddFiles(String zipFileName, List<String> fileNames, String zipPath) throws IOException {
|
|
||||||
// requires Java7+
|
|
||||||
Map<String, String> env = new HashMap<>();
|
|
||||||
env.put("create", "true");
|
|
||||||
Path path = Paths.get(zipFileName);
|
|
||||||
URI uri = URI.create("jar:" + path.toUri());
|
|
||||||
FileSystem fs = FileSystems.newFileSystem(uri, env);
|
|
||||||
for (String fileName : fileNames) {
|
|
||||||
String fileNameInZip = zipPath + File.separator + fileName;
|
|
||||||
Path extFile = Paths.get(fileName);
|
|
||||||
Path zippedFile = fs.getPath(fileNameInZip);
|
|
||||||
Files.createDirectories(zippedFile.getParent());
|
|
||||||
//fs.provider().checkAccess(zippedFile, AccessMode.READ);
|
|
||||||
Files.copy(extFile, zippedFile, StandardCopyOption.REPLACE_EXISTING);
|
|
||||||
}
|
|
||||||
fs.close();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static byte[] unzipFileContents(String zipFileName, String fileName) throws IOException {
|
|
||||||
ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFileName));
|
|
||||||
ZipEntry zipEntry;
|
|
||||||
byte[] data = null;
|
|
||||||
while ((zipEntry = zis.getNextEntry()) != null) {
|
|
||||||
Path zippedName = Paths.get(zipEntry.getName()).normalize();
|
|
||||||
Path searchName = Paths.get(fileName).normalize();
|
|
||||||
if (zippedName.equals(searchName)) {
|
|
||||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
|
||||||
byte[] buffer = new byte[4096];
|
|
||||||
int read;
|
|
||||||
while ((read = zis.read(buffer)) != -1) {
|
|
||||||
baos.write(buffer, 0, read);
|
|
||||||
}
|
|
||||||
data = baos.toByteArray();
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
zis.closeEntry();
|
|
||||||
zis.close();
|
|
||||||
System.out.println("Unzip " + zipFileName + ": " + fileName + (data != null ? " extracted!" : " failed!"));
|
|
||||||
return data;
|
|
||||||
}
|
|
||||||
}
|
|
Loading…
Reference in New Issue