Simpler handling of Byte Order Mark (BOM)

lauxlib.c

@@ -740,17 +740,18 @@ static int errfile (lua_State *L, const char *what, int fnameindex) {
 }
 
 
-static int skipBOM (LoadF *lf) {
+/*
-  const char *p = "\xEF\xBB\xBF";  /* UTF-8 BOM mark */
+** Skip an optional BOM at the start of a stream. If there is an
-  int c;
+** incomplete BOM (the first character is correct but the rest is
-  lf->n = 0;
+** not), returns the first character anyway to force an error
-  do {
+** (as no chunk can start with 0xEF).
-    c = getc(lf->f);
+*/
-    if (c == EOF || c != *(const unsigned char *)p++) return c;
+static int skipBOM (FILE *f) {
-    lf->buff[lf->n++] = c;  /* to be read by the parser */
+  int c = getc(f);  /* read first character */
-  } while (*p != '\0');
+  if (c == 0xEF && getc(f) == 0xBB && getc(f) == 0xBF)  /* correct BOM? */
-  lf->n = 0;  /* prefix matched; discard it */
+    return getc(f);  /* ignore BOM and return next char */
-  return getc(lf->f);  /* return next character */
+  else  /* no (valid) BOM */
+    return c;  /* return first character */
 }
 
 
@@ -761,13 +762,13 @@ static int skipBOM (LoadF *lf) {
 ** first "valid" character of the file (after the optional BOM and
 ** a first-line comment).
 */
-static int skipcomment (LoadF *lf, int *cp) {
+static int skipcomment (FILE *f, int *cp) {
-  int c = *cp = skipBOM(lf);
+  int c = *cp = skipBOM(f);
   if (c == '#') {  /* first line is a comment (Unix exec. file)? */
     do {  /* skip first line */
-      c = getc(lf->f);
+      c = getc(f);
     } while (c != EOF && c != '\n');
-    *cp = getc(lf->f);  /* skip end-of-line, if present */
+    *cp = getc(f);  /* next character after comment, if present */
     return 1;  /* there was a comment */
   }
   else return 0;  /* no comment */
@@ -789,12 +790,16 @@ LUALIB_API int luaL_loadfilex (lua_State *L, const char *filename,
     lf.f = fopen(filename, "r");
     if (lf.f == NULL) return errfile(L, "open", fnameindex);
   }
-  if (skipcomment(&lf, &c))  /* read initial portion */
+  lf.n = 0;
-    lf.buff[lf.n++] = '\n';  /* add line to correct line numbers */
+  if (skipcomment(lf.f, &c))  /* read initial portion */
-  if (c == LUA_SIGNATURE[0] && filename) {  /* binary file? */
+    lf.buff[lf.n++] = '\n';  /* add newline to correct line numbers */
-    lf.f = freopen(filename, "rb", lf.f);  /* reopen in binary mode */
+  if (c == LUA_SIGNATURE[0]) {  /* binary file? */
-    if (lf.f == NULL) return errfile(L, "reopen", fnameindex);
+    lf.n = 0;  /* remove possible newline */
-    skipcomment(&lf, &c);  /* re-read initial portion */
+    if (filename) {  /* "real" file? */
+      lf.f = freopen(filename, "rb", lf.f);  /* reopen in binary mode */
+      if (lf.f == NULL) return errfile(L, "reopen", fnameindex);
+      skipcomment(lf.f, &c);  /* re-read initial portion */
+    }
   }
   if (c != EOF)
     lf.buff[lf.n++] = c;  /* 'c' is the first character of the stream */

testes/main.lua

@@ -94,6 +94,33 @@ RUN('echo "print(10)\nprint(2)\n" | lua > %s', out)
 checkout("10\n2\n")
 
 
+-- testing BOM
+prepfile("\xEF\xBB\xBF")
+RUN('lua %s > %s', prog, out)
+checkout("")
+
+prepfile("\xEF\xBB\xBFprint(3)")
+RUN('lua %s > %s', prog, out)
+checkout("3\n")
+
+prepfile("\xEF\xBB\xBF# comment!!\nprint(3)")
+RUN('lua %s > %s', prog, out)
+checkout("3\n")
+
+-- bad BOMs
+prepfile("\xEF")
+NoRun("unexpected symbol", 'lua %s > %s', prog, out)
+
+prepfile("\xEF\xBB")
+NoRun("unexpected symbol", 'lua %s > %s', prog, out)
+
+prepfile("\xEFprint(3)")
+NoRun("unexpected symbol", 'lua %s > %s', prog, out)
+
+prepfile("\xEF\xBBprint(3)")
+NoRun("unexpected symbol", 'lua %s > %s', prog, out)
+
+
 -- test option '-'
 RUN('echo "print(arg[1])" | lua - -h > %s', out)
 checkout("-h\n")
@@ -385,12 +412,10 @@ checkprogout("101\n13\t22\n\n")
 prepfile[[#comment in 1st line without \n at the end]]
 RUN('lua %s', prog)
 
-prepfile[[#test line number when file starts with comment line
+-- first-line comment with binary file
-debug = require"debug"
+prepfile("#comment\n" .. string.dump(load("print(3)")))
-print(debug.getinfo(1).currentline)
-]]
 RUN('lua %s > %s', prog, out)
-checkprogout('3\n')
+checkout('3\n')
 
 -- close Lua with an open file
 prepfile(string.format([[io.output(%q); io.write('alo')]], out))