[m-rev.] diff: fix bug #430

[Julien Fischer]

Fix bug #430.

The lexer was not allowing underscores between leading zeros in decimal integer
literals and float literals (e.g. 0_0 or 0_0.0).

library/lexer.m:
      Allow underscores in the above cases.

tests/hard_coded/parse_number_from_io.{m,exp,exp2,exp3}:
tests/hard_coded/parse_number_from_string.{m,exp,exp2,exp3}:
      Extend these tests to cover the above cases.

tests/invalid/invalid_binary_literal.err_exp:
tests/invalid/invalid_octal_literal.err_exp:
tests/invalid/invalid_hex_literal.err_exp:
      Conform to the above change.

Julien.

diff --git a/library/lexer.m b/library/lexer.m
index 0e89668..1cb2427 100644
--- a/library/lexer.m
+++ b/library/lexer.m
@@ -1961,6 +1961,9 @@ get_zero(Stream, Token, !IO) :-
          ( if char.is_digit(Char) then
              LastDigit = last_digit_is_not_underscore,
              get_number(Stream, LastDigit, [Char], Token, !IO)
+        else if Char = '_' then
+            LastDigit = last_digit_is_underscore,
+            get_number(Stream, LastDigit, [], Token, !IO)
          else if Char = '''' then
              get_char_code(Stream, Token, !IO)
          else if Char = 'b' then
@@ -2001,6 +2004,10 @@ string_get_zero(String, Len, Posn0, Token, Context, !Posn) :-
              LastDigit = last_digit_is_not_underscore,
              string_get_number(String, LastDigit, Len, Posn0, Token, Context,
                  !Posn)
+        else if Char = '_' then
+            LastDigit = last_digit_is_underscore,
+            string_get_number(String, LastDigit, Len, Posn0, Token, Context,
+                !Posn)
          else if Char = '''' then
              string_get_char_code(String, Len, Posn0, Token, Context, !Posn)
          else if Char = 'b' then
diff --git a/tests/hard_coded/parse_number_from_io.exp b/tests/hard_coded/parse_number_from_io.exp
index b4aa73e..cef3e65 100644
--- a/tests/hard_coded/parse_number_from_io.exp
+++ b/tests/hard_coded/parse_number_from_io.exp
@@ -1,10 +1,16 @@
  Decimal:
  0
  0
+0
+0
  10
  -10
  10
  -10
+1
+1
+-1
+-1

  Binary:
  0
@@ -50,3 +56,5 @@ Float:
  1200000000000.0
  1.2e-10
  1.2e-10
+0.0
+1.01
diff --git a/tests/hard_coded/parse_number_from_io.exp2 b/tests/hard_coded/parse_number_from_io.exp2
index 33a2286..0a8763c 100644
--- a/tests/hard_coded/parse_number_from_io.exp2
+++ b/tests/hard_coded/parse_number_from_io.exp2
@@ -1,10 +1,16 @@
  Decimal:
  0
  0
+0
+0
  10
  -10
  10
  -10
+1
+1
+-1
+-1

  Binary:
  0
@@ -50,3 +56,5 @@ Float:
  1.2E12
  1.2E-10
  1.2E-10
+0.0
+1.01
diff --git a/tests/hard_coded/parse_number_from_io.exp3 b/tests/hard_coded/parse_number_from_io.exp3
index b6c52a6..5ca44b3 100644
--- a/tests/hard_coded/parse_number_from_io.exp3
+++ b/tests/hard_coded/parse_number_from_io.exp3
@@ -1,10 +1,16 @@
  Decimal:
  0
  0
+0
+0
  10
  -10
  10
  -10
+1
+1
+-1
+-1

  Binary:
  0
@@ -50,3 +56,5 @@ Float:
  1200000000000.0
  1.2E-10
  1.2E-10
+0.0
+1.01
diff --git a/tests/hard_coded/parse_number_from_io.m b/tests/hard_coded/parse_number_from_io.m
index e51b9c0..11e48d7 100644
--- a/tests/hard_coded/parse_number_from_io.m
+++ b/tests/hard_coded/parse_number_from_io.m
@@ -15,10 +15,16 @@ main(!IO) :-
      io.print_line("Decimal:", !IO),
      io.print_line(0, !IO),
      io.print_line(-0, !IO),
+    io.print_line(00, !IO),
+    io.print_line(0_0, !IO),
      io.print_line(10, !IO),
      io.print_line(-10, !IO),
      io.print_line(1_0, !IO),
      io.print_line(-1_0, !IO),
+    io.print_line(01, !IO),
+    io.print_line(0_1, !IO),
+    io.print_line(-01, !IO),
+    io.print_line(-0_1, !IO),
      io.nl(!IO),

      io.print_line("Binary:", !IO),
@@ -67,4 +73,6 @@ main(!IO) :-
      io.print_line(1_2e+1_1, !IO),
      io.print_line(1_2E+1_1, !IO),
      io.print_line(1_2e-1_1, !IO),
-    io.print_line(1_2E-1_1, !IO).
+    io.print_line(1_2E-1_1, !IO),
+    io.print_line(0_0.0, !IO),
+    io.print_line(0_1.0_1, !IO).
diff --git a/tests/hard_coded/parse_number_from_string.exp b/tests/hard_coded/parse_number_from_string.exp
index dc8623f..01eb941 100644
--- a/tests/hard_coded/parse_number_from_string.exp
+++ b/tests/hard_coded/parse_number_from_string.exp
@@ -1,10 +1,14 @@
  Valid decimal literals:
  read_term("0.") = functor(integer(0), [], context("", 1))
  read_term("-0.") = functor(integer(0), [], context("", 1))
+read_term("00.") = functor(integer(0), [], context("", 1))
+read_term("0_0.") = functor(integer(0), [], context("", 1))
  read_term("10.") = functor(integer(10), [], context("", 1))
  read_term("-10.") = functor(integer(-10), [], context("", 1))
  read_term("1_0.") = functor(integer(10), [], context("", 1))
  read_term("-1_0.") = functor(integer(-10), [], context("", 1))
+read_term("01.") = functor(integer(1), [], context("", 1))
+read_term("0_1.") = functor(integer(1), [], context("", 1))
  read_term("1_000_000_000_000_000_000_000.") = functor(big_integer(base_10, i(5, [13877, 12907, 7261, 14976, 0])), [], context("", 1))
  read_term("-1_000_000_000_000_000_000_000.") = functor(atom("-"), [functor(big_integer(base_10, i(5, [13877, 12907, 7261, 14976, 0])), [], context("", 1))], context("", 1))

@@ -84,6 +88,10 @@ read_term("1_2e+1_1.") = functor(float(1200000000000.0), [], context("", 1))
  read_term("1_2E+1_1.") = functor(float(1200000000000.0), [], context("", 1))
  read_term("1_2e-1_1.") = functor(float(1.2e-10), [], context("", 1))
  read_term("1_2E-1_1.") = functor(float(1.2e-10), [], context("", 1))
+read_term("00.0.") = functor(float(0.0), [], context("", 1))
+read_term("0_0.0.") = functor(float(0.0), [], context("", 1))
+read_term("01.0.") = functor(float(1.0), [], context("", 1))
+read_term("0_1.0.") = functor(float(1.0), [], context("", 1))

  Invalid float literals:
  read_term("1_2_3.1_2_3_.") = Syntax error: fractional part of float terminated by underscore
diff --git a/tests/hard_coded/parse_number_from_string.exp2 b/tests/hard_coded/parse_number_from_string.exp2
index 68f1c63..ad2e577 100644
--- a/tests/hard_coded/parse_number_from_string.exp2
+++ b/tests/hard_coded/parse_number_from_string.exp2
@@ -1,10 +1,14 @@
  Valid decimal literals:
  read_term("0.") = functor(integer(0), [], context("", 1))
  read_term("-0.") = functor(integer(0), [], context("", 1))
+read_term("00.") = functor(integer(0), [], context("", 1))
+read_term("0_0.") = functor(integer(0), [], context("", 1))
  read_term("10.") = functor(integer(10), [], context("", 1))
  read_term("-10.") = functor(integer(-10), [], context("", 1))
  read_term("1_0.") = functor(integer(10), [], context("", 1))
  read_term("-1_0.") = functor(integer(-10), [], context("", 1))
+read_term("01.") = functor(integer(1), [], context("", 1))
+read_term("0_1.") = functor(integer(1), [], context("", 1))
  read_term("1_000_000_000_000_000_000_000.") = functor(big_integer(base_10, i(5, [13877, 12907, 7261, 14976, 0])), [], context("", 1))
  read_term("-1_000_000_000_000_000_000_000.") = functor(atom("-"), [functor(big_integer(base_10, i(5, [13877, 12907, 7261, 14976, 0])), [], context("", 1))], context("", 1))

@@ -84,6 +88,10 @@ read_term("1_2e+1_1.") = functor(float(1.2E12), [], context("", 1))
  read_term("1_2E+1_1.") = functor(float(1.2E12), [], context("", 1))
  read_term("1_2e-1_1.") = functor(float(1.2E-10), [], context("", 1))
  read_term("1_2E-1_1.") = functor(float(1.2E-10), [], context("", 1))
+read_term("00.0.") = functor(float(0.0), [], context("", 1))
+read_term("0_0.0.") = functor(float(0.0), [], context("", 1))
+read_term("01.0.") = functor(float(1.0), [], context("", 1))
+read_term("0_1.0.") = functor(float(1.0), [], context("", 1))

  Invalid float literals:
  read_term("1_2_3.1_2_3_.") = Syntax error: fractional part of float terminated by underscore
diff --git a/tests/hard_coded/parse_number_from_string.exp3 b/tests/hard_coded/parse_number_from_string.exp3
index 258a76f..d518db3 100644
--- a/tests/hard_coded/parse_number_from_string.exp3
+++ b/tests/hard_coded/parse_number_from_string.exp3
@@ -1,10 +1,14 @@
  Valid decimal literals:
  read_term("0.") = functor(integer(0), [], context("", 1))
  read_term("-0.") = functor(integer(0), [], context("", 1))
+read_term("00.") = functor(integer(0), [], context("", 1))
+read_term("0_0.") = functor(integer(0), [], context("", 1))
  read_term("10.") = functor(integer(10), [], context("", 1))
  read_term("-10.") = functor(integer(-10), [], context("", 1))
  read_term("1_0.") = functor(integer(10), [], context("", 1))
  read_term("-1_0.") = functor(integer(-10), [], context("", 1))
+read_term("01.") = functor(integer(1), [], context("", 1))
+read_term("0_1.") = functor(integer(1), [], context("", 1))
  read_term("1_000_000_000_000_000_000_000.") = functor(big_integer(base_10, i(5, [13877, 12907, 7261, 14976, 0])), [], context("", 1))
  read_term("-1_000_000_000_000_000_000_000.") = functor(atom("-"), [functor(big_integer(base_10, i(5, [13877, 12907, 7261, 14976, 0])), [], context("", 1))], context("", 1))

@@ -84,6 +88,10 @@ read_term("1_2e+1_1.") = functor(float(1200000000000.0), [], context("", 1))
  read_term("1_2E+1_1.") = functor(float(1200000000000.0), [], context("", 1))
  read_term("1_2e-1_1.") = functor(float(1.2E-10), [], context("", 1))
  read_term("1_2E-1_1.") = functor(float(1.2E-10), [], context("", 1))
+read_term("00.0.") = functor(float(0.0), [], context("", 1))
+read_term("0_0.0.") = functor(float(0.0), [], context("", 1))
+read_term("01.0.") = functor(float(1.0), [], context("", 1))
+read_term("0_1.0.") = functor(float(1.0), [], context("", 1))

  Invalid float literals:
  read_term("1_2_3.1_2_3_.") = Syntax error: fractional part of float terminated by underscore
diff --git a/tests/hard_coded/parse_number_from_string.m b/tests/hard_coded/parse_number_from_string.m
index 527532a..2635791 100644
--- a/tests/hard_coded/parse_number_from_string.m
+++ b/tests/hard_coded/parse_number_from_string.m
@@ -69,10 +69,14 @@ run_test(String, !IO) :-
  valid_decimal_cases = [
      "0.",
      "-0.",
+    "00.",
+    "0_0.",
      "10.",
      "-10.",
      "1_0.",
      "-1_0.",
+    "01.",
+    "0_1.",
      "1_000_000_000_000_000_000_000.",
      "-1_000_000_000_000_000_000_000."
  ].
@@ -175,7 +179,11 @@ valid_float_cases = [
      "1_2e+1_1.",
      "1_2E+1_1.",
      "1_2e-1_1.",
-    "1_2E-1_1."
+    "1_2E-1_1.",
+    "00.0.",
+    "0_0.0.",
+    "01.0.",
+    "0_1.0."
  ].

  :- func invalid_float_cases = list(string).
diff --git a/tests/invalid/invalid_binary_literal.err_exp b/tests/invalid/invalid_binary_literal.err_exp
index 216d695..27e0d3e 100644
--- a/tests/invalid/invalid_binary_literal.err_exp
+++ b/tests/invalid/invalid_binary_literal.err_exp
@@ -24,7 +24,9 @@ invalid_binary_literal.m:025: Syntax error: unterminated binary literal.
  invalid_binary_literal.m:027: Syntax error at token `. ': unexpected token at
  invalid_binary_literal.m:027:   start of (sub)term.
  invalid_binary_literal.m:027: Syntax error: unterminated binary literal.
-invalid_binary_literal.m:029: Syntax error at variable `_b11': operator or `.'
-invalid_binary_literal.m:029:   expected.
-invalid_binary_literal.m:031: Syntax error at variable `_b11': operator or `.'
-invalid_binary_literal.m:031:   expected.
+invalid_binary_literal.m:029: Syntax error: unterminated decimal literal.
+invalid_binary_literal.m:029: Error: clause for predicate
+invalid_binary_literal.m:029:   `invalid_binary_literal.b11'/0
+invalid_binary_literal.m:029:   without corresponding `:- pred' declaration.
+invalid_binary_literal.m:029: Inferred :- pred b11.
+invalid_binary_literal.m:031: Syntax error: unterminated decimal literal.
diff --git a/tests/invalid/invalid_hex_literal.err_exp b/tests/invalid/invalid_hex_literal.err_exp
index b2127ca..38bda41 100644
--- a/tests/invalid/invalid_hex_literal.err_exp
+++ b/tests/invalid/invalid_hex_literal.err_exp
@@ -24,7 +24,9 @@ invalid_hex_literal.m:025: Syntax error: unterminated hexadecimal literal.
  invalid_hex_literal.m:027: Syntax error at token `. ': unexpected token at
  invalid_hex_literal.m:027:   start of (sub)term.
  invalid_hex_literal.m:027: Syntax error: unterminated hexadecimal literal.
-invalid_hex_literal.m:029: Syntax error at variable `_xff': operator or `.'
-invalid_hex_literal.m:029:   expected.
-invalid_hex_literal.m:031: Syntax error at variable `_xff': operator or `.'
-invalid_hex_literal.m:031:   expected.
+invalid_hex_literal.m:029: Syntax error: unterminated decimal literal.
+invalid_hex_literal.m:029: Error: clause for predicate
+invalid_hex_literal.m:029:   `invalid_hex_literal.xff'/0
+invalid_hex_literal.m:029:   without corresponding `:- pred' declaration.
+invalid_hex_literal.m:029: Inferred :- pred xff.
+invalid_hex_literal.m:031: Syntax error: unterminated decimal literal.
diff --git a/tests/invalid/invalid_octal_literal.err_exp b/tests/invalid/invalid_octal_literal.err_exp
index 6c4ecd8..12a14d8 100644
--- a/tests/invalid/invalid_octal_literal.err_exp
+++ b/tests/invalid/invalid_octal_literal.err_exp
@@ -24,7 +24,9 @@ invalid_octal_literal.m:025: Syntax error: unterminated octal literal.
  invalid_octal_literal.m:027: Syntax error at token `. ': unexpected token at
  invalid_octal_literal.m:027:   start of (sub)term.
  invalid_octal_literal.m:027: Syntax error: unterminated octal literal.
-invalid_octal_literal.m:029: Syntax error at variable `_o77': operator or `.'
-invalid_octal_literal.m:029:   expected.
-invalid_octal_literal.m:031: Syntax error at variable `_o77': operator or `.'
-invalid_octal_literal.m:031:   expected.
+invalid_octal_literal.m:029: Syntax error: unterminated decimal literal.
+invalid_octal_literal.m:029: Error: clause for predicate
+invalid_octal_literal.m:029:   `invalid_octal_literal.o77'/0
+invalid_octal_literal.m:029:   without corresponding `:- pred' declaration.
+invalid_octal_literal.m:029: Inferred :- pred o77.
+invalid_octal_literal.m:031: Syntax error: unterminated decimal literal.