test new parser TS layout (#4220)

* layout tests

* code analysis

* LOL java changes its line ending behavior based on your platform

* jar

java_tools/configuration_definition/src/main/java/com/rusefi/newparse/layout/ArrayLayout.java

@@ -58,7 +58,7 @@ public class ArrayLayout extends Layout {
 
     @Override
     public String toString() {
-        return "Array of " + this.prototypeLayout.toString() + " length " + this.length[0] + " " + super.toString();
+        return "Array of " + this.prototypeLayout + " length " + this.length[0] + " " + super.toString();
     }
 
     @Override

java_tools/configuration_definition/src/main/java/com/rusefi/newparse/layout/BitGroupLayout.java

@@ -2,16 +2,13 @@ package com.rusefi.newparse.layout;
 
 import com.rusefi.newparse.outputs.TsMetadata;
 import com.rusefi.newparse.parsing.BitGroup;
-import com.rusefi.newparse.parsing.EnumField;
-import com.rusefi.newparse.parsing.Type;
-import com.rusefi.newparse.parsing.UnusedField;
 
 import java.io.PrintStream;
 import java.util.List;
 import java.util.stream.Collectors;
 
 public class BitGroupLayout extends Layout {
-    private class BitLayout {
+    private static class BitLayout {
         public final String name;
         public final String comment;
         public final String trueValue;
@@ -55,7 +52,7 @@ public class BitGroupLayout extends Layout {
 
             String name = prefixer.get(bit.name);
 
-            ps.print(prefixer.get(bit.name));
+            ps.print(name);
             ps.print(" = bits, U32, ");
             ps.print(actualOffset);
             ps.print(", [");

java_tools/configuration_definition/src/main/java/com/rusefi/newparse/layout/ScalarLayout.java

@@ -9,10 +9,10 @@ import com.rusefi.newparse.parsing.Type;
 import java.io.PrintStream;
 
 public class ScalarLayout extends Layout {
-    private String name;
+    public final String name;
-    private Type type;
+    private final Type type;
-    private FieldOptions options;
+    private final FieldOptions options;
-    private boolean autoscale;
+    private final boolean autoscale;
 
     public ScalarLayout(ScalarField field) {
         this.name = field.name;

java_tools/configuration_definition/src/main/java/com/rusefi/newparse/layout/StringLayout.java

@@ -2,7 +2,6 @@ package com.rusefi.newparse.layout;
 
 import com.rusefi.newparse.outputs.TsMetadata;
 import com.rusefi.newparse.parsing.StringField;
-import com.rusefi.newparse.parsing.UnusedField;
 
 import java.io.PrintStream;
 

java_tools/configuration_definition/src/main/java/com/rusefi/newparse/layout/StructLayout.java

@@ -4,12 +4,11 @@ import com.rusefi.newparse.outputs.TsMetadata;
 import com.rusefi.newparse.parsing.*;
 
 import java.io.PrintStream;
-import java.lang.reflect.Array;
 import java.util.ArrayList;
 import java.util.List;
 
 public class StructLayout extends Layout {
-    /*private*/public List<Layout> children = new ArrayList<>();
+    /*private*/public final List<Layout> children = new ArrayList<>();
 
     public final String typeName;
     private final String name;
@@ -93,7 +92,7 @@ public class StructLayout extends Layout {
             return addStruct(offset, sf.struct, sf.name);
         }
 
-        Layout l = null;
+        Layout l;
         if (f instanceof ScalarField) {
             l = new ScalarLayout((ScalarField)f);
         } else if (f instanceof EnumField) {

java_tools/configuration_definition/src/main/java/com/rusefi/newparse/layout/StructNamePrefixer.java

@@ -1,10 +1,9 @@
 package com.rusefi.newparse.layout;
 
 import java.util.Stack;
-import java.util.stream.Collectors;
 
 public class StructNamePrefixer {
-    private Stack<String> stack = new Stack<>();
+    private final Stack<String> stack = new Stack<>();
     private int idx = -1;
 
     public void pop() {
@@ -36,7 +35,7 @@ public class StructNamePrefixer {
 
     String get(String name) {
         // stack has no prefixes, just return the plain name (no join necessary)
-        name = stack.stream().collect(Collectors.joining()) + name;
+        name = String.join("", stack) + name;
 
         // Append the array index if necessary (for iterated arrays)
         if (this.idx != -1) {

java_tools/configuration_definition/src/main/java/com/rusefi/newparse/layout/UnionLayout.java

@@ -7,12 +7,11 @@ import com.rusefi.newparse.parsing.ScalarField;
 import com.rusefi.newparse.parsing.Union;
 
 import java.io.PrintStream;
-import java.lang.reflect.Array;
 import java.util.ArrayList;
 import java.util.List;
 
 public class UnionLayout extends Layout {
-    private List<Layout> children = new ArrayList<>();
+    private final List<Layout> children = new ArrayList<>();
 
     public UnionLayout(Union u) {
         for (Field f : u.fields) {
@@ -36,24 +35,24 @@ public class UnionLayout extends Layout {
     @Override
     public void setOffset(int offset) {
         super.setOffset(offset);
-        this.children.stream().forEach(c -> c.setOffset(offset));
+        this.children.forEach(c -> c.setOffset(offset));
     }
 
     @Override
     public void setOffsetWithinStruct(int offset) {
         super.setOffsetWithinStruct(offset);
-        this.children.stream().forEach(c -> c.setOffsetWithinStruct(offset));
+        this.children.forEach(c -> c.setOffsetWithinStruct(offset));
     }
 
     @Override
     public int getSize() {
-        return this.children.stream().map(l -> l.getSize()).max(Integer::compare).get();
+        return this.children.stream().map(Layout::getSize).max(Integer::compare).get();
     }
 
     @Override
     public int getAlignment() {
         // The alignment of the union is the largest alignment required by one of the members
-        return this.children.stream().map(l -> l.getAlignment()).max(Integer::compare).get();
+        return this.children.stream().map(Layout::getAlignment).max(Integer::compare).get();
     }
 
     @Override

java_tools/configuration_definition/src/main/java/com/rusefi/newparse/layout/UnusedLayout.java

@@ -1,12 +1,9 @@
 package com.rusefi.newparse.layout;
 
 import com.rusefi.newparse.outputs.TsMetadata;
-import com.rusefi.newparse.parsing.EnumField;
-import com.rusefi.newparse.parsing.Type;
 import com.rusefi.newparse.parsing.UnusedField;
 
 import java.io.PrintStream;
-import java.util.Random;
 
 public class UnusedLayout extends Layout {
     private final int size;

java_tools/configuration_definition/src/main/java/com/rusefi/newparse/outputs/CStructWriter.java

@@ -13,8 +13,10 @@ import java.io.PrintStream;
  */
 public class CStructWriter {
     public void writeCStructs(ParseState parser, String outputFile) throws FileNotFoundException {
-        PrintStream ps = new PrintStream(new FileOutputStream(outputFile));
+        writeCStructs(parser, new PrintStream(new FileOutputStream(outputFile)));
+    }
 
+    public void writeCStructs(ParseState parser, PrintStream ps) {
         ps.println(
                 "// begin\n" +
                 "#pragma once\n" +

java_tools/configuration_definition/src/main/java/com/rusefi/newparse/outputs/TsMetadata.java

@@ -5,7 +5,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 public class TsMetadata {
-    private List<String> comments = new ArrayList<>();
+    private final List<String> comments = new ArrayList<>();
     public void addComment(String name, String comment) {
         if (comment == null) {
             return;
@@ -27,6 +27,6 @@ public class TsMetadata {
     }
 
     public void writeComments(PrintStream ps) {
-        this.comments.stream().forEach(ps::println);
+        this.comments.forEach(ps::println);
     }
 }

java_tools/configuration_definition/src/main/java/com/rusefi/newparse/outputs/TsWriter.java

@@ -17,9 +17,14 @@ public class TsWriter {
 
     private static final Pattern OPTIONAL_LINE = Pattern.compile("@@if_([a-zA-Z0-9_]+)");
 
-    public void writeTunerstudio(ParseState parser, String inputFile, String outputFile) throws FileNotFoundException, IOException {
+    public void writeTunerstudio(ParseState parser, String inputFile, String outputFile) throws IOException {
-        BufferedReader is = new BufferedReader(new FileReader(inputFile));
         PrintStream ps = new PrintStream(new FileOutputStream(outputFile));
+        writeTunerstudio(parser, inputFile, ps);
+        ps.close();
+    }
+
+    public void writeTunerstudio(ParseState parser, String inputFile, PrintStream ps) throws IOException {
+        BufferedReader is = new BufferedReader(new FileReader(inputFile));
 
         while (is.ready()) {
             String line = is.readLine();
@@ -80,10 +85,9 @@ public class TsWriter {
         }
 
         is.close();
-        ps.close();
     }
 
-    private void writeLayoutAndComments(ParseState parser, PrintStream ps) {
+    public void writeLayoutAndComments(ParseState parser, PrintStream ps) {
         StructLayout root = new StructLayout(0, "root", parser.getLastStruct());
         TsMetadata meta = new TsMetadata();
 

java_tools/configuration_definition/src/main/java/com/rusefi/newparse/parsing/Struct.java

@@ -1,6 +1,5 @@
 package com.rusefi.newparse.parsing;
 
-import java.io.PrintStream;
 import java.util.List;
 
 public class Struct implements Field {

java_tools/configuration_definition/src/test/java/com/rusefi/test/newParse/LayoutTest.java

@@ -0,0 +1,232 @@
+package com.rusefi.test.newParse;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static com.rusefi.test.newParse.NewParseHelper.parseToTs;
+
+public class LayoutTest {
+    @Test
+    public void singleField() throws IOException {
+        String ts = parseToTs("struct_no_prefix myStruct\n" +
+                "int8_t xyz;comment;\"units\", 1, 2, 3, 4, 5\n" +
+                "end_struct");
+
+        Assert.assertEquals(
+                "pageSize            = 4\n" +
+                "page = 1\n" +
+                "xyz = scalar, S08, 0, \"units\", 1, 2, 3, 4, 5\n" +
+                "; unused 3 bytes at offset 1\n" +
+                "; total TS size = 4\n" +
+                "[SettingContextHelp]\n" +
+                "\txyz = \"comment\"\n", ts);
+    }
+
+    @Test
+    public void twoFieldsSameSize() throws IOException {
+        String ts = parseToTs("struct_no_prefix myStruct\n" +
+                "int32_t abc;;\"\", 1, 2, 3, 4, 5\n" +
+                "int32_t xyz;;\"\", 6, 7, 8, 9, 10\n" +
+                "end_struct");
+
+        Assert.assertEquals(
+                "pageSize            = 8\n" +
+                        "page = 1\n" +
+                        "abc = scalar, S32, 0, \"\", 1, 2, 3, 4, 5\n" +
+                        "xyz = scalar, S32, 4, \"\", 6, 7, 8, 9, 10\n" +
+                        "; total TS size = 8\n" +
+                        "[SettingContextHelp]\n", ts);
+    }
+
+    @Test
+    public void twoFieldsBigThenSmall() throws IOException {
+        String ts = parseToTs("struct_no_prefix myStruct\n" +
+                "int32_t abc;;\"\", 1, 2, 3, 4, 5\n" +
+                "int8_t xyz;;\"\", 6, 7, 8, 9, 10\n" +
+                "end_struct");
+
+        Assert.assertEquals(
+                "pageSize            = 8\n" +
+                        "page = 1\n" +
+                        "abc = scalar, S32, 0, \"\", 1, 2, 3, 4, 5\n" +
+                        "xyz = scalar, S08, 4, \"\", 6, 7, 8, 9, 10\n" +
+                        "; unused 3 bytes at offset 5\n" +
+                        "; total TS size = 8\n" +
+                        "[SettingContextHelp]\n", ts);
+    }
+
+    @Test
+    public void twoFieldsSmallThenBig() throws IOException {
+        String ts = parseToTs("struct_no_prefix myStruct\n" +
+                "int8_t abc;;\"\", 1, 2, 3, 4, 5\n" +
+                "int32_t xyz;;\"\", 6, 7, 8, 9, 10\n" +
+                "end_struct");
+
+        Assert.assertEquals(
+                "pageSize            = 8\n" +
+                        "page = 1\n" +
+                        "abc = scalar, S08, 0, \"\", 1, 2, 3, 4, 5\n" +
+                        "; unused 3 bytes at offset 1\n" +
+                        "xyz = scalar, S32, 4, \"\", 6, 7, 8, 9, 10\n" +
+                        "; total TS size = 8\n" +
+                        "[SettingContextHelp]\n", ts);
+    }
+
+    @Test
+    public void threeFieldsSmallInMisalignment() throws IOException {
+        String ts = parseToTs("struct_no_prefix myStruct\n" +
+                "int16_t abc\n" +
+                "int8_t def\n" +
+                "int32_t xyz\n" +
+                "end_struct");
+
+        Assert.assertEquals(
+                "pageSize            = 8\n" +
+                        "page = 1\n" +
+                        "abc = scalar, S16, 0, \"\", 1, 0, 0, 0, 0\n" +
+                        "def = scalar, S08, 2, \"\", 1, 0, 0, 0, 0\n" +
+                        "; unused 1 bytes at offset 3\n" +
+                        "xyz = scalar, S32, 4, \"\", 1, 0, 0, 0, 0\n" +
+                        "; total TS size = 8\n" +
+                        "[SettingContextHelp]\n", ts);
+    }
+
+    @Test
+    public void twoFieldsArrayThenBig() throws IOException {
+        String ts = parseToTs("struct_no_prefix myStruct\n" +
+                "int8_t[10] abc;;\"\", 1, 2, 3, 4, 5\n" +
+                "int32_t xyz;;\"\", 6, 7, 8, 9, 10\n" +
+                "end_struct");
+
+        Assert.assertEquals(
+                "pageSize            = 16\n" +
+                        "page = 1\n" +
+                        "abc = array, S08, 0, [10], \"\", 1, 2, 3, 4, 5\n" +
+                        "; unused 2 bytes at offset 10\n" +
+                        "xyz = scalar, S32, 12, \"\", 6, 7, 8, 9, 10\n" +
+                        "; total TS size = 16\n" +
+                        "[SettingContextHelp]\n", ts);
+    }
+
+    @Test
+    public void singleFieldMultiDimArray() throws IOException {
+        String ts = parseToTs("struct_no_prefix myStruct\n" +
+                "int8_t[10 x 5] abc;;\"\", 1, 2, 3, 4, 5\n" +
+                "end_struct");
+
+        Assert.assertEquals(
+                "pageSize            = 52\n" +
+                        "page = 1\n" +
+                        "abc = array, S08, 0, [10x5], \"\", 1, 2, 3, 4, 5\n" +
+                        "; unused 2 bytes at offset 50\n" +
+                        "; total TS size = 52\n" +
+                        "[SettingContextHelp]\n", ts);
+    }
+
+    @Test
+    public void structs() throws IOException {
+        String ts = parseToTs(
+                "struct s1\n" +
+                        "int32_t abc\n" +
+                        "uint8_t def\n" +
+                        "end_struct\n" +
+                "struct_no_prefix rootStruct\n" +
+                        "s1 a\n" +
+                        "s1 b\n" +
+                "end_struct");
+
+        Assert.assertEquals(
+                "pageSize            = 16\n" +
+                        "page = 1\n" +
+                        "a_abc = scalar, S32, 0, \"\", 1, 0, 0, 0, 0\n" +
+                        "a_def = scalar, U08, 4, \"\", 1, 0, 0, 0, 0\n" +
+                        "; unused 3 bytes at offset 5\n" +
+                        "b_abc = scalar, S32, 8, \"\", 1, 0, 0, 0, 0\n" +
+                        "b_def = scalar, U08, 12, \"\", 1, 0, 0, 0, 0\n" +
+                        "; unused 3 bytes at offset 13\n" +
+                        "; total TS size = 16\n" +
+                        "[SettingContextHelp]\n", ts);
+    }
+
+    @Test
+    public void arrayIterate() throws IOException {
+        String ts = parseToTs(
+                        "struct_no_prefix rootStruct\n" +
+                        "int32_t[4 iterate] myArr\n" +
+                        "end_struct");
+
+        Assert.assertEquals(
+                "pageSize            = 16\n" +
+                        "page = 1\n" +
+                        "myArr1 = scalar, S32, 0, \"\", 1, 0, 0, 0, 0\n" +
+                        "myArr2 = scalar, S32, 4, \"\", 1, 0, 0, 0, 0\n" +
+                        "myArr3 = scalar, S32, 8, \"\", 1, 0, 0, 0, 0\n" +
+                        "myArr4 = scalar, S32, 12, \"\", 1, 0, 0, 0, 0\n" +
+                        "; total TS size = 16\n" +
+                        "[SettingContextHelp]\n", ts);
+    }
+
+    @Test
+    public void arrayZeroLength() throws IOException {
+        String ts = parseToTs(
+                "struct_no_prefix rootStruct\n" +
+                        "int32_t[0] myArr\n" +
+                        "end_struct");
+
+        Assert.assertEquals(
+                "pageSize            = 0\n" +
+                        "page = 1\n" +
+                        "; total TS size = 0\n" +
+                        "[SettingContextHelp]\n", ts);
+    }
+
+    @Test
+    public void arrayOfStructsIterate() throws IOException {
+        String ts = parseToTs(
+                "struct s1\n" +
+                "uint8_t var1\n" +
+                "uint16_t var2\n" +
+                "end_struct\n" +
+                "struct_no_prefix rootStruct\n" +
+                "s1[4 iterate] arr\n" +
+                "end_struct");
+
+        Assert.assertEquals(
+                "pageSize            = 16\n" +
+                        "page = 1\n" +
+                        "arr1_var1 = scalar, U08, 0, \"\", 1, 0, 0, 0, 0\n" +
+                        "; unused 1 bytes at offset 1\n" +
+                        "arr1_var2 = scalar, U16, 2, \"\", 1, 0, 0, 0, 0\n" +
+                        "arr2_var1 = scalar, U08, 4, \"\", 1, 0, 0, 0, 0\n" +
+                        "; unused 1 bytes at offset 5\n" +
+                        "arr2_var2 = scalar, U16, 6, \"\", 1, 0, 0, 0, 0\n" +
+                        "arr3_var1 = scalar, U08, 8, \"\", 1, 0, 0, 0, 0\n" +
+                        "; unused 1 bytes at offset 9\n" +
+                        "arr3_var2 = scalar, U16, 10, \"\", 1, 0, 0, 0, 0\n" +
+                        "arr4_var1 = scalar, U08, 12, \"\", 1, 0, 0, 0, 0\n" +
+                        "; unused 1 bytes at offset 13\n" +
+                        "arr4_var2 = scalar, U16, 14, \"\", 1, 0, 0, 0, 0\n" +
+                        "; total TS size = 16\n" +
+                        "[SettingContextHelp]\n", ts);
+    }
+
+    @Test
+    public void bits() throws IOException {
+        String ts = parseToTs("struct_no_prefix myStruct\n" +
+                "bit first\n" +
+                "bit second\n" +
+                "bit withOpt,\"a\",\"b\"\n" +
+                "end_struct");
+
+        Assert.assertEquals(
+                "pageSize            = 4\n" +
+                        "page = 1\n" +
+                        "first = bits, U32, 0, [0:0], \"false\", \"true\"\n" +
+                        "second = bits, U32, 0, [1:1], \"false\", \"true\"\n" +
+                        "withOpt = bits, U32, 0, [2:2], \"b\", \"a\"\n" +
+                        "; total TS size = 4\n" +
+                        "[SettingContextHelp]\n", ts);
+    }
+}

java_tools/configuration_definition/src/test/java/com/rusefi/test/newParse/NewParseHelper.java

@@ -2,8 +2,11 @@ package com.rusefi.test.newParse;
 
 import com.rusefi.RusefiParseErrorStrategy;
 import com.rusefi.newparse.ParseState;
+import com.rusefi.newparse.outputs.CStructWriter;
+import com.rusefi.newparse.outputs.TsWriter;
 
-import java.io.IOException;
+import java.io.*;
+import java.nio.charset.StandardCharsets;
 
 public class NewParseHelper {
     public static ParseState parse(String input) throws IOException {
@@ -11,4 +14,51 @@ public class NewParseHelper {
         RusefiParseErrorStrategy.parseDefinitionString(parseState.getListener(), input);
         return parseState;
     }
+
+    private static class PrintStreamAlwaysUnix extends PrintStream {
+        public PrintStreamAlwaysUnix(OutputStream out, boolean autoFlush, String encoding) throws UnsupportedEncodingException {
+            super(out, autoFlush, encoding);
+        }
+
+        @Override
+        public void println() {
+            print('\n');
+        }
+
+        @Override
+        public void println(String s) {
+            print(s);
+            println();
+        }
+    }
+
+    public static String parseToTs(String input) throws IOException {
+        ParseState state = parse(input);
+
+        TsWriter writer = new TsWriter();
+
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        final String utf8 = StandardCharsets.UTF_8.name();
+
+        PrintStream ps = new PrintStreamAlwaysUnix(baos, true, utf8);
+
+        writer.writeLayoutAndComments(state, ps);
+
+        return baos.toString(utf8);
+    }
+
+    public static String parseToC(String input) throws IOException {
+        ParseState state = parse(input);
+
+        CStructWriter writer = new CStructWriter();
+
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        final String utf8 = StandardCharsets.UTF_8.name();
+
+        PrintStream ps = new PrintStreamAlwaysUnix(baos, true, utf8);
+
+        writer.writeCStructs(state, ps);
+
+        return baos.toString(utf8);
+    }
 }