parse enums (#2832)

* enums and definitions

* bad merge

* txt format

* endBit

* handle 1-size arrays

* strings

* fix auto enum

* hard code 0 and 1

* jar

* jar

firmware/integration/rusefi_config.txt

@@ -32,7 +32,7 @@
 !
 ! Q: how does 'custom' work?
 ! A: let's look at
-!  #define can_baudrate_e_enum "100kbps", "250kbps" , "500kbps", "1Mbps"
+!  #define can_baudrate_e_enum "100kbps", "250kbps", "500kbps", "1Mbps"
 !   custom can_baudrate_e 1 bits, U08, @OFFSET@, [0:1], @@can_baudrate_e_enum@@
 !   can_baudrate_e canBaudRate; set can_baudrate
 !
@@ -646,7 +646,7 @@ float fanOffTemperature;+Cooling fan turn-off temperature threshold, in Celsius;
 
 float vehicleSpeedCoef;+This coefficient translates vehicle speed input frequency (in Hz) into vehicle speed, km/h;"coef", 1, 0, 0.01, 2000, 2
 
-custom can_nbc_e 4 bits, U32, @OFFSET@, [0:4], "None", "FIAT", "VAG" , "MAZDA RX8", "BMW", "W202", "BMW E90", "Haltech", "VAG MQB", "type 9", "type 10", "INVALID", "INVALID"
+custom can_nbc_e 4 bits, U32, @OFFSET@, [0:4], "None", "FIAT", "VAG", "MAZDA RX8", "BMW", "W202", "BMW E90", "Haltech", "VAG MQB", "type 9", "type 10", "INVALID", "INVALID"
 can_nbc_e canNbcType;set can_mode X
 
 int canSleepPeriodMs;CANbus thread period, ms;"ms", 1, 0, 0, 1000, 2
@@ -1264,7 +1264,7 @@ int16_t tps2Max;Full throttle#2. tpsMax value as 10 bit ADC value. Not Voltage!\
 	float throttlePedalSecondaryUpVoltage;;"voltage", 1, 0, -6, 6, 2
 	float throttlePedalSecondaryWOTVoltage;+Pedal in the floor;"voltage", 1, 0, -6, 6, 2
 
-	#define can_baudrate_e_enum "100kbps", "250kbps" , "500kbps", "1Mbps"
+	#define can_baudrate_e_enum "100kbps", "250kbps", "500kbps", "1Mbps"
 	custom can_baudrate_e 1 bits, U08, @OFFSET@, [0:1], @@can_baudrate_e_enum@@
 	can_baudrate_e canBaudRate; set can_baudrate
 

java_tools/configuration_definition/RusefiConfigGrammar.g4

@@ -72,7 +72,9 @@ restOfLine
     | 'false';
 
 definition
-    : Definition identifier numexpr
+    : Definition identifier integer
+    | Definition identifier floatNum
+    | Definition identifier numexpr
     | Definition identifier restOfLine;
 struct: (Struct | StructNoPrefix) identifier ('@brief' restOfLine)? ENDL+ statements EndStruct;
 

java_tools/configuration_definition/src/com/rusefi/ConfigDefinition.java

@@ -250,7 +250,7 @@ public class ConfigDefinition {
 
         // Parse the input files
         {
-            ParseState listener = new ParseState();
+            ParseState listener = new ParseState(state.enumsReader);
 
             // First process yaml files
             //processYamls(listener, yamlFiles);

java_tools/configuration_definition/src/com/rusefi/newparse/ParseState.java

@@ -1,10 +1,14 @@
 package com.rusefi.newparse;
 
+import com.rusefi.EnumsReader;
+import com.rusefi.enum_reader.Value;
 import com.rusefi.generated.RusefiConfigGrammarBaseListener;
 import com.rusefi.generated.RusefiConfigGrammarParser;
 import com.rusefi.newparse.parsing.*;
 import jdk.nashorn.internal.runtime.regexp.joni.constants.StringType;
+import org.jetbrains.annotations.Nullable;
 
+import java.io.PrintStream;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -14,6 +18,50 @@ public class ParseState extends RusefiConfigGrammarBaseListener {
     List<Struct> structList = new ArrayList<>();
     Map<String, Typedef> typedefs = new HashMap<>();
 
+    private final EnumsReader enumsReader;
+
+    public ParseState(EnumsReader enumsReader) {
+        this.enumsReader = enumsReader;
+    }
+
+    private static boolean isNumeric(String str) {
+        try {
+            Integer.parseInt(str);
+            return true;
+        } catch (NumberFormatException e) {
+            return false;
+        }
+    }
+
+    private String[] resolveEnumValues(String enumName) {
+        TreeMap<Integer, String> valueNameById = new TreeMap<>();
+
+        Map<String, Value> stringValueMap = this.enumsReader.getEnums().get(enumName);
+        if (stringValueMap == null)
+            return null;
+        for (Value value : stringValueMap.values()) {
+            if (value.getValue().contains("ENUM_32_BITS"))
+                continue;
+
+            if (isNumeric(value.getValue())) {
+                valueNameById.put(value.getIntValue(), value.getName());
+            } else {
+                Definition def = this.definitions.get(value.getValue());
+                if (def == null)
+                    throw new IllegalStateException("No value for " + value);;
+                valueNameById.put((Integer)def.value, value.getName());
+            }
+        }
+
+        // Now iterate over all values, assembling as an array in-order
+        String[] result = new String[valueNameById.lastKey() + 1];
+        for (int i = 0; i < result.length; i++) {
+            result[i] = valueNameById.getOrDefault(i, "INVALID");
+        }
+
+        return result;
+    }
+
     public List<Struct> getStructs() {
         return structList;
     }
@@ -38,7 +86,11 @@ public class ParseState extends RusefiConfigGrammarBaseListener {
 
     private Definition.OverwritePolicy definitionOverwritePolicy = Definition.OverwritePolicy.NotAllowed;
 
-    public void addDefinition(String name, String value, Definition.OverwritePolicy overwritePolicy) {
+    private void addDefinition(String name, Object value) {
+        addDefinition(name, value, this.definitionOverwritePolicy);
+    }
+
+    public void addDefinition(String name, Object value, Definition.OverwritePolicy overwritePolicy) {
         Definition existingDefinition = definitions.getOrDefault(name, null);
 
         if (existingDefinition != null) {
@@ -64,16 +116,25 @@ public class ParseState extends RusefiConfigGrammarBaseListener {
     public void exitDefinition(RusefiConfigGrammarParser.DefinitionContext ctx) {
         String name = ctx.identifier().getText();
 
-        String value;
+        if (ctx.integer() != null) {
+            addDefinition(name, Integer.parseInt(ctx.integer().getText()));
+        } else if (ctx.floatNum() != null) {
+            addDefinition(name, Double.parseDouble(ctx.floatNum().getText()));
+        } else if (ctx.numexpr() != null) {
+            double evalResult = this.evalResults.remove();
+            double floored = Math.floor(evalResult);
 
-        if (!this.evalResults.isEmpty()) {
+            if (Math.abs(floored - evalResult) < 0.001) {
-            value = this.evalResults.remove().toString();
+                // value is an int, process as such
+                addDefinition(name, (int)floored);
+            } else {
+                // Value is a double, add it
+                addDefinition(name, evalResult);
+            }
         } else {
             // glue the list of definitions back together
-            value = ctx.restOfLine().getText();
+            addDefinition(name, ctx.restOfLine().getText());
         }
-
-        addDefinition(name, value, this.definitionOverwritePolicy);
     }
 
     String typedefName = null;
@@ -100,25 +161,45 @@ public class ParseState extends RusefiConfigGrammarBaseListener {
 
     @Override
     public void enterEnumTypedefSuffix(RusefiConfigGrammarParser.EnumTypedefSuffixContext ctx) {
-        int startBit = Integer.parseInt(ctx.integer(1).getText());
         int endBit = Integer.parseInt(ctx.integer(2).getText());
         Type datatype = Type.findByTsType(ctx.Datatype().getText());
 
-        String values = ctx.enumRhs().getText();
+        String rhs = ctx.enumRhs().getText();
 
-        // TODO: many enum defs are missing so this doesn't work yet
+        String[] values = null;
-        /*
-        if (values.startsWith("@@")) {
-            Definition def = this.definitions.get(values.replaceAll("@", ""));
 
-            if (def == null) {
+        if (rhs.startsWith("@@")) {
-                throw new RuntimeException("couldn't find definition for " + values);
+            String defName = rhs.replaceAll("@", "");
+
+            if (defName.endsWith("_auto_enum")) {
+                // clip off the "_auto_enum" part
+                defName = defName.substring(0, defName.length() - 10);
+                values = resolveEnumValues(defName);
+            } else {
+                Definition def = this.definitions.get(defName);
+
+                if (def == null) {
+                    throw new RuntimeException("couldn't find definition for " + rhs);
+                }
+
+                Object value = def.value;
+
+                if (!(value instanceof String)) {
+                    throw new RuntimeException("Found definition for " + rhs + " but it wasn't a string as expected");
+                }
+
+                rhs = (String)value;
             }
+        }
 
-            values = def.value;
+        if (values == null) {
-        }*/
+            values = Arrays.stream(rhs.split(","))                    // Split on commas
+                        .map(s -> s.trim())                                 // trim whitespace
+                        .map(s -> s.replaceAll("\"", ""))   // Remove quotes
+                        .toArray(n -> new String[n]);                       // Convert back to array
+        }
 
-        this.typedefs.put(this.typedefName, new EnumTypedef(this.typedefName, datatype, startBit, endBit, values));
+        this.typedefs.put(this.typedefName, new EnumTypedef(this.typedefName, datatype, endBit, values));
     }
 
     @Override
@@ -135,7 +216,7 @@ public class ParseState extends RusefiConfigGrammarBaseListener {
 
     @Override
     public void exitStringTypedefSuffix(RusefiConfigGrammarParser.StringTypedefSuffixContext ctx) {
-        Float stringLength = this.evalResults.remove();
+        Double stringLength = this.evalResults.remove();
 
         this.typedefs.put(this.typedefName, new StringTypedef(this.typedefName, stringLength.intValue()));
     }
@@ -176,10 +257,10 @@ public class ParseState extends RusefiConfigGrammarBaseListener {
             // this is a legacy field option list, parse it as such
             if (!ctx.numexpr().isEmpty()) {
                 options.units = ctx.QuotedString().getText();
-                options.scale = evalResults.remove();
+                options.scale = evalResults.remove().floatValue();
-                options.offset = evalResults.remove();
+                options.offset = evalResults.remove().floatValue();
-                options.min = evalResults.remove();
+                options.min = evalResults.remove().floatValue();
-                options.max = evalResults.remove();
+                options.max = evalResults.remove().floatValue();
                 options.digits = Integer.parseInt(ctx.integer().getText());
 
                 // we should have consumed everything on the results list
@@ -201,13 +282,13 @@ public class ParseState extends RusefiConfigGrammarBaseListener {
             } else if (key.equals("digits")) {
                 options.digits = Integer.parseInt(sValue);
             } else {
-                Float value = evalResults.remove();
+                Double value = evalResults.remove();
 
                 switch (key) {
-                    case "min": options.min = value; break;
+                    case "min": options.min = value.floatValue(); break;
-                    case "max": options.max = value; break;
+                    case "max": options.max = value.floatValue(); break;
-                    case "scale": options.scale = value; break;
+                    case "scale": options.scale = value.floatValue(); break;
-                    case "offset": options.offset = value; break;
+                    case "offset": options.offset = value.floatValue(); break;
                 }
             }
         }
@@ -257,7 +338,7 @@ public class ParseState extends RusefiConfigGrammarBaseListener {
                 // Merge the read-in options list with the default from the typedef (if exists)
                 handleFieldOptionsList(options, ctx.fieldOptionsList());
 
-                scope.structFields.add(new EnumField(bTypedef.type, type, name, bTypedef.values, options));
+                scope.structFields.add(new EnumField(bTypedef.type, type, name, bTypedef.endBit, bTypedef.values, options));
                 return;
             } else if (typedef instanceof StringTypedef) {
                 StringTypedef sTypedef = (StringTypedef) typedef;
@@ -349,7 +430,7 @@ public class ParseState extends RusefiConfigGrammarBaseListener {
                 options = new FieldOptions();
                 handleFieldOptionsList(options, ctx.fieldOptionsList());
 
-                EnumField prototype = new EnumField(bTypedef.type, type, name, bTypedef.values, options);
+                EnumField prototype = new EnumField(bTypedef.type, type, name, bTypedef.endBit, bTypedef.values, options);
 
                 scope.structFields.add(new ArrayField<EnumField>(prototype, length, iterate));
                 return;
@@ -444,12 +525,12 @@ public class ParseState extends RusefiConfigGrammarBaseListener {
         scope.structFields.add(u);
     }
 
-    private Stack<Float> evalStack = new Stack<>();
+    private Stack<Double> evalStack = new Stack<>();
 
 
     @Override
     public void exitEvalNumber(RusefiConfigGrammarParser.EvalNumberContext ctx) {
-        evalStack.push(Float.parseFloat(ctx.floatNum().getText()));
+        evalStack.push(Double.parseDouble(ctx.floatNum().getText()));
     }
 
     @Override
@@ -461,43 +542,49 @@ public class ParseState extends RusefiConfigGrammarBaseListener {
             throw new RuntimeException("Definition not found for " + ctx.getText());
         }
 
-        // Find the matching definition, parse it, and push on to the eval stack
+        // Find the matching definition and push on to the eval stack
-        evalStack.push(Float.parseFloat(this.definitions.get(defName).value));
+        Definition def = this.definitions.get(defName);
+
+        if (!def.isNumeric()) {
+            throw new RuntimeException("Tried to use symbol " + defName + " in an expression, but it wasn't a number");
+        }
+
+        evalStack.push(def.asDouble());
     }
 
     @Override
     public void exitEvalMul(RusefiConfigGrammarParser.EvalMulContext ctx) {
-        Float right = evalStack.pop();
+        Double right = evalStack.pop();
-        Float left = evalStack.pop();
+        Double left = evalStack.pop();
 
         evalStack.push(left * right);
     }
 
     @Override
     public void exitEvalDiv(RusefiConfigGrammarParser.EvalDivContext ctx) {
-        Float right = evalStack.pop();
+        Double right = evalStack.pop();
-        Float left = evalStack.pop();
+        Double left = evalStack.pop();
 
         evalStack.push(left / right);
     }
 
     @Override
     public void exitEvalAdd(RusefiConfigGrammarParser.EvalAddContext ctx) {
-        Float right = evalStack.pop();
+        Double right = evalStack.pop();
-        Float left = evalStack.pop();
+        Double left = evalStack.pop();
 
         evalStack.push(left + right);
     }
 
     @Override
     public void exitEvalSub(RusefiConfigGrammarParser.EvalSubContext ctx) {
-        Float right = evalStack.pop();
+        Double right = evalStack.pop();
-        Float left = evalStack.pop();
+        Double left = evalStack.pop();
 
         evalStack.push(left - right);
     }
 
-    private Queue<Float> evalResults = new LinkedList<>();
+    private Queue<Double> evalResults = new LinkedList<>();
 
     @Override
     public void exitNumexpr(RusefiConfigGrammarParser.NumexprContext ctx) {

java_tools/configuration_definition/src/com/rusefi/newparse/layout/EnumLayout.java

@@ -10,13 +10,15 @@ public class EnumLayout extends Layout {
     private final String name;
     private final Type type;
     private final String enumType;
-    private final String values;
+    private final int endBit;
+    private final String[] values;
     private final FieldOptions options;
 
     public EnumLayout(EnumField field) {
         this.name = field.name;
         this.type = field.type;
         this.enumType = field.enumType;
+        this.endBit = field.endBit;
         this.values = field.values;
         this.options = field.options;
     }
@@ -26,6 +28,12 @@ public class EnumLayout extends Layout {
         return this.type.size;
     }
 
+    private static void writeEnumVal(PrintStream ps, String enumVal) {
+        ps.print('"');
+        ps.print(enumVal);
+        ps.print('"');
+    }
+
     @Override
     public void writeTunerstudioLayout(PrintStream ps, StructNamePrefixer prefixer) {
         ps.print(prefixer.get(this.name));
@@ -35,11 +43,23 @@ public class EnumLayout extends Layout {
         ps.print(this.offset);
         ps.print(", ");
 
-        // TODO: automatically compute number of bits required?
+        ps.print("[0:");
-        ps.print("[0:7], ");
+        ps.print(this.endBit);
+        ps.print("], ");
 
-        // TODO: where should value define resolution happen?
+        writeEnumVal(ps, this.values[0]);
-        ps.print(this.values);
+
+        for (int i = 1; i < this.values.length; i++) {
+            ps.print(", ");
+            writeEnumVal(ps, this.values[i]);
+        }
+
+        // Pad out the rest of the enum's values with "INVALID"
+        int expectedNumber = 2 << this.endBit;
+        for (int i = this.values.length; i < expectedNumber; i++) {
+            ps.print(", ");
+            writeEnumVal(ps, "INVALID");
+        }
 
         ps.println();
     }

java_tools/configuration_definition/src/com/rusefi/newparse/layout/ScalarLayout.java

@@ -48,6 +48,10 @@ public class ScalarLayout extends Layout {
         if (arrayLength == 0) {
             // Skip zero length arrays, they may be used for dynamic padding but TS doesn't like them
             return;
+        } else if (arrayLength == 1) {
+            // For 1-length arrays, emit as a plain scalar instead
+            writeTunerstudioLayout(ps, prefixer);
+            return;
         }
 
         printBeforeArrayLength(ps, prefixer, "array");

java_tools/configuration_definition/src/com/rusefi/newparse/layout/StringLayout.java

@@ -33,7 +33,7 @@ public class StringLayout extends Layout {
     @Override
     public void writeTunerstudioLayout(PrintStream ps, StructNamePrefixer prefixer) {
         ps.print(prefixer.get(this.name));
-        ps.print(" = scalar, ASCII, ");
+        ps.print(" = string, ASCII, ");
         ps.print(this.offset);
         ps.print(", ");
         ps.print(size);

java_tools/configuration_definition/src/com/rusefi/newparse/parsing/Definition.java

@@ -2,7 +2,7 @@ package com.rusefi.newparse.parsing;
 
 public class Definition {
     public final String name;
-    public final String value;
+    public final Object value;
     public final OverwritePolicy overwritePolicy;
 
     public enum OverwritePolicy {
@@ -11,9 +11,21 @@ public class Definition {
         IgnoreNew
     }
 
-    public Definition(String name, String value, OverwritePolicy overwritePolicy) {
+    public Definition(String name, Object value, OverwritePolicy overwritePolicy) {
         this.name = name;
         this.value = value;
         this.overwritePolicy = overwritePolicy;
     }
+
+    public boolean isNumeric() {
+        return this.value instanceof Double || this.value instanceof Integer;
+    }
+
+    public double asDouble() {
+        if (this.value instanceof Double) {
+            return ((Double)this.value);
+        } else {
+            return ((Integer)this.value).doubleValue();
+        }
+    }
 }

java_tools/configuration_definition/src/com/rusefi/newparse/parsing/EnumField.java

@@ -3,14 +3,16 @@ package com.rusefi.newparse.parsing;
 public class EnumField extends PrototypeField {
     public final Type type;
     public final String enumType;
-    public final String values;
+    public final int endBit;
+    public final String[] values;
     public final FieldOptions options;
 
-    public EnumField(Type type, String enumType, String name, String values, FieldOptions options) {
+    public EnumField(Type type, String enumType, String name, int endBit, String[] values, FieldOptions options) {
         super(name);
 
         this.type = type;
         this.enumType = enumType;
+        this.endBit = endBit;
         this.values = values;
         this.options = options;
     }

java_tools/configuration_definition/src/com/rusefi/newparse/parsing/EnumTypedef.java

@@ -2,15 +2,13 @@ package com.rusefi.newparse.parsing;
 
 public class EnumTypedef extends Typedef {
     public final Type type;
-    public final int startBit;
     public final int endBit;
-    public final String values;
+    public final String[] values;
 
-    public EnumTypedef(String name, Type type, int startBit, int endBit, String values) {
+    public EnumTypedef(String name, Type type, int endBit, String[] values) {
         super(name);
 
         this.type = type;
-        this.startBit = startBit;
         this.endBit = endBit;
         this.values = values;
     }

java_tools/configuration_definition/src/com/rusefi/output/TSProjectConsumer.java

@@ -106,7 +106,14 @@ public class TSProjectConsumer implements ConfigurationConsumer {
                  * https://github.com/rusefi/web_backend/issues/97
                  */
                 double val = IniField.parseDouble(fields[mutliplierIndex]);
-                fields[mutliplierIndex] = " " + (val == 1 ? "1" : val);
+
+                if (val == 0) {
+                    fields[mutliplierIndex] = " 0";
+                } else if (val == 1) {
+                    fields[mutliplierIndex] = " 1";
+                } else {
+                    fields[mutliplierIndex] = " " + val;
+                }
             }
             StringBuilder sb = new StringBuilder();
             for (String f : fields) {