[m-rev.] for review: extend the lexer to recognise additional integer literals
Julien Fischer
jfischer at opturion.com
Tue Apr 25 21:10:14 AEST 2017
For review by anyone.
This builds on my uncommitted earlier change to merge integer token
representations; once both of those are committed using uint literals
in programs will work, although I wouldn't suggest using them in anger
just yet.
-----------------
Extend the lexer to recognise additional integer literals.
Extend the lexer to recognise uint literals, optional signedness suffixes on
ints and the literals for all of the proposed fixed size integer types.
Fix XXX UINTs in the library and compiler.
library/lexer.m:
Uncomment the other alternatives in the integer_size/0 type.
Handle signedness and size suffixes in integer literals.
library/parser.m
library/term.m:
Conform to the above changes.
library/stream.string_writer.m:
Fix an XXX UINT: make write handle uints properly.
library/term_io.m:
Fix an XXX UINT: output integer signedness and size suffixes for
integers when appropriate.
compiler/superhomogeneous.m:
Print an error message if we encounter a fixed size integer literal
as the rest of the compiler does not yet support them.
compiler/hlds_out_util.m:
compiler/parse_tree_out_info.m:
Output the 'u' suffix on uint values.
test/hard_coded/lexer_zero.{m,inp,exp*}:
Extend this test to cover zeros of varying signedness and size.
Prefix each line of the output with the input line number of the
token -- this makes it easier to relate the output back to the
input.
tests/hard_coded/Mmakefile:
Add the new test case.
tests/hard_coded/lexer_ints.{m,inp,exp*}:
Test the lexer on non-zero integer literals.
Julien.
diff --git a/compiler/hlds_out_util.m b/compiler/hlds_out_util.m
index 70e8794..69acf9d 100644
--- a/compiler/hlds_out_util.m
+++ b/compiler/hlds_out_util.m
@@ -834,7 +834,7 @@ cons_id_and_vars_or_arity_to_string(VarSet, Qual, ConsId, MaybeArgVars)
string.int_to_string(Int, String)
;
ConsId = uint_const(UInt),
- String = uint_to_string(UInt)
+ String = uint_to_string(UInt) ++ "u"
;
ConsId = float_const(Float),
String = float_to_string(Float)
diff --git a/compiler/parse_tree_out_info.m b/compiler/parse_tree_out_info.m
index 2ffc617..4634de6 100644
--- a/compiler/parse_tree_out_info.m
+++ b/compiler/parse_tree_out_info.m
@@ -191,7 +191,7 @@ maybe_unqualify_sym_name(Info, SymName, OutSymName) :-
pred(add_strings/3) is io.write_strings,
pred(add_char/3) is io.write_char,
pred(add_int/3) is io.write_int,
- pred(add_uint/3) is io.write_uint, % XXX UINT - literal syntax.
+ pred(add_uint/3) is write_uint_literal,
pred(add_float/3) is io.write_float,
pred(add_purity_prefix/3) is prog_out.write_purity_prefix,
pred(add_quoted_atom/3) is term_io.quote_atom,
@@ -224,6 +224,14 @@ maybe_unqualify_sym_name(Info, SymName, OutSymName) :-
%---------------------------------------------------------------------------%
+:- pred write_uint_literal(uint::in, io::di, io::uo) is det.
+
+write_uint_literal(UInt, !IO) :-
+ io.write_uint(UInt, !IO),
+ io.write_char('u', !IO).
+
+%---------------------------------------------------------------------------%
+
:- pred write_eval_eval_method(eval_method::in, io::di, io::uo) is det.
write_eval_eval_method(EvalMethod, !IO) :-
@@ -264,8 +272,7 @@ output_int(I, Str0, Str) :-
:- pred output_uint(uint::in, string::di, string::uo) is det.
output_uint(U, Str0, Str) :-
- % XXX UINT - literal syntax.
- S = uint_to_string(U),
+ S = uint_to_string(U) ++ "u",
string.append(Str0, S, Str).
:- pred output_float(float::in, string::di, string::uo) is det.
diff --git a/compiler/superhomogeneous.m b/compiler/superhomogeneous.m
index ccff047..7953300 100644
--- a/compiler/superhomogeneous.m
+++ b/compiler/superhomogeneous.m
@@ -734,41 +734,62 @@ parse_ordinary_cons_id(Functor, ArgTerms, Context, ConsId, !Specs) :-
list.length(ArgTerms, Arity),
ConsId = cons(unqualified(Name), Arity, cons_id_dummy_type_ctor)
;
- Functor = term.integer(Base, Integer, Signedness, size_word),
+ Functor = term.integer(Base, Integer, Signedness, Size),
(
- Signedness = signed,
- ( if source_integer_to_int(Base, Integer, Int) then
- ConsId = int_const(Int)
- else
- BasePrefix = integer_base_prefix(Base),
- IntString =
- integer.to_base_string(Integer, integer_base_int(Base)),
- Pieces = [words("Error: the integer literal"),
- quote(BasePrefix ++ IntString),
- words("is too big to be represented on this machine."), nl],
- Msg = simple_msg(Context, [always(Pieces)]),
- Spec = error_spec(severity_error, phase_parse_tree_to_hlds, [Msg]),
- !:Specs = [Spec | !.Specs],
- % This is a dummy.
- ConsId = int_const(0)
+ Size = size_word,
+ (
+ Signedness = signed,
+ ( if source_integer_to_int(Base, Integer, Int) then
+ ConsId = int_const(Int)
+ else
+ BasePrefix = integer_base_prefix(Base),
+ IntString = integer.to_base_string(Integer,
+ integer_base_int(Base)),
+ Pieces = [words("Error: the integer literal"),
+ quote(BasePrefix ++ IntString),
+ words("is too big to be represented on this machine."),
+ nl],
+ Msg = simple_msg(Context, [always(Pieces)]),
+ Spec = error_spec(severity_error, phase_parse_tree_to_hlds,
+ [Msg]),
+ !:Specs = [Spec | !.Specs],
+ % This is a dummy.
+ ConsId = int_const(0)
+ )
+ ;
+ Signedness = unsigned,
+ ( if integer.to_uint(Integer, UInt) then
+ ConsId = uint_const(UInt)
+ else
+ BasePrefix = integer_base_prefix(Base),
+ IntString = integer.to_base_string(Integer,
+ integer_base_int(Base)),
+ Pieces = [words("Error: the unsigned integer literal"),
+ quote(BasePrefix ++ IntString ++ "u"),
+ words("is too big to be represented on this machine."),
+ nl],
+ Msg = simple_msg(Context, [always(Pieces)]),
+ Spec = error_spec(severity_error, phase_parse_tree_to_hlds,
+ [Msg]),
+ !:Specs = [Spec | !.Specs],
+ % This is a dummy.
+ ConsId = int_const(0)
+ )
)
;
- Signedness = unsigned,
- ( if integer.to_uint(Integer, UInt) then
- ConsId = uint_const(UInt)
- else
- BasePrefix = integer_base_prefix(Base),
- IntString =
- integer.to_base_string(Integer, integer_base_int(Base)),
- Pieces = [words("Error: the unsigned integer literal"),
- quote(BasePrefix ++ IntString ++ "u"),
- words("is too big to be represented on this machine."), nl],
- Msg = simple_msg(Context, [always(Pieces)]),
- Spec = error_spec(severity_error, phase_parse_tree_to_hlds, [Msg]),
- !:Specs = [Spec | !.Specs],
- % This is a dummy.
- ConsId = int_const(0)
- )
+ ( Size = size_8_bit
+ ; Size = size_16_bit
+ ; Size = size_32_bit
+ ; Size = size_64_bit
+ ),
+ Pieces = [words("Error: fixed size integers"),
+ words("are not (yet) supported.")],
+ Msg = simple_msg(Context, [always(Pieces)]),
+ Spec = error_spec(severity_error, phase_parse_tree_to_hlds,
+ [Msg]),
+ !:Specs = [Spec | !.Specs],
+ % This is a dummy.
+ ConsId = int_const(0)
)
;
Functor = term.string(String),
diff --git a/library/lexer.m b/library/lexer.m
index c48a800..1249570 100644
--- a/library/lexer.m
+++ b/library/lexer.m
@@ -67,11 +67,11 @@
; unsigned.
:- type integer_size
- ---> size_word.
- %; size_8_bit
- %; size_16_bit
- %; size_32_bit
- %; size_64_bit
+ ---> size_word
+ ; size_8_bit
+ ; size_16_bit
+ ; size_32_bit
+ ; size_64_bit.
% For every token, we record the line number of the line on
% which the token occurred.
@@ -1969,7 +1969,7 @@ get_zero(Stream, Token, !IO) :-
get_number(Stream, LastDigit, [Char], Token, !IO)
else if Char = '_' then
LastDigit = last_digit_is_underscore,
- get_number(Stream, LastDigit, [], Token, !IO)
+ get_number(Stream, LastDigit, ['0'], Token, !IO)
else if Char = '''' then
get_char_code(Stream, Token, !IO)
else if Char = 'b' then
@@ -1978,6 +1978,12 @@ get_zero(Stream, Token, !IO) :-
get_octal(Stream, Token, !IO)
else if Char = 'x' then
get_hex(Stream, Token, !IO)
+ else if Char = 'u' then
+ get_integer_size_suffix(Stream, ['0'], base_10, unsigned,
+ Token, !IO)
+ else if Char = 'i' then
+ get_integer_size_suffix(Stream, ['0'], base_10, signed,
+ Token, !IO)
else if Char = ('.') then
LastDigit = last_digit_is_not_underscore,
get_int_dot(Stream, LastDigit, ['0'], Token, !IO)
@@ -2005,6 +2011,7 @@ get_zero(Stream, Token, !IO) :-
string_token_context::out, posn::in, posn::out) is det.
string_get_zero(String, Len, Posn0, Token, Context, !Posn) :-
+ Posn1 = !.Posn,
( if string_read_char(String, Len, Char, !Posn) then
( if char.is_digit(Char) then
LastDigit = last_digit_is_not_underscore,
@@ -2022,6 +2029,14 @@ string_get_zero(String, Len, Posn0, Token, Context, !Posn) :-
string_get_octal(String, Len, Posn0, Token, Context, !Posn)
else if Char = 'x' then
string_get_hex(String, Len, Posn0, Token, Context, !Posn)
+ else if Char = 'u' then
+ string_get_integer_size_suffix(String, Len, Posn0, Posn1, base_10,
+ unsigned, Token, !Posn),
+ string_get_context(Posn0, Context, !Posn)
+ else if Char = 'i' then
+ string_get_integer_size_suffix(String, Len, Posn0, Posn1, base_10,
+ signed, Token, !Posn),
+ string_get_context(Posn0, Context, !Posn)
else if Char = ('.') then
LastDigit = last_digit_is_not_underscore,
string_get_int_dot(String, LastDigit, Len, Posn0, Token, Context,
@@ -2125,7 +2140,8 @@ get_binary_2(Stream, !.LastDigit, !.RevChars, Token, !IO) :-
Result = eof,
(
!.LastDigit = last_digit_is_not_underscore,
- rev_char_list_to_int(!.RevChars, base_2, Token)
+ rev_char_list_to_int(!.RevChars, base_2, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated binary literal")
@@ -2139,11 +2155,18 @@ get_binary_2(Stream, !.LastDigit, !.RevChars, Token, !IO) :-
else if Char = '_' then
!:LastDigit = last_digit_is_underscore,
get_binary_2(Stream, !.LastDigit, !.RevChars, Token, !IO)
+ else if Char = 'u' then
+ get_integer_size_suffix(Stream, !.RevChars, base_2, unsigned,
+ Token, !IO)
+ else if Char = 'i' then
+ get_integer_size_suffix(Stream, !.RevChars, base_2, signed,
+ Token, !IO)
else
io.putback_char(Stream, Char, !IO),
(
!.LastDigit = last_digit_is_not_underscore,
- rev_char_list_to_int(!.RevChars, base_2, Token)
+ rev_char_list_to_int(!.RevChars, base_2, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated binary literal")
@@ -2156,6 +2179,7 @@ get_binary_2(Stream, !.LastDigit, !.RevChars, Token, !IO) :-
posn::in, posn::out) is det.
string_get_binary_2(String, !.LastDigit, Len, Posn1, Token, Context, !Posn) :-
+ Posn2 = !.Posn,
( if string_read_char(String, Len, Char, !Posn) then
( if char.is_binary_digit(Char) then
!:LastDigit = last_digit_is_not_underscore,
@@ -2165,12 +2189,21 @@ string_get_binary_2(String, !.LastDigit, Len, Posn1, Token, Context, !Posn) :-
!:LastDigit = last_digit_is_underscore,
string_get_binary_2(String, !.LastDigit, Len, Posn1, Token,
Context, !Posn)
+ else if Char = 'u' then
+ string_get_integer_size_suffix(String, Len, Posn1, Posn2, base_2,
+ unsigned, Token, !Posn),
+ string_get_context(Posn1, Context, !Posn)
+ else if Char = 'i' then
+ string_get_integer_size_suffix(String, Len, Posn1, Posn2, base_2,
+ signed, Token, !Posn),
+ string_get_context(Posn1, Context, !Posn)
else
string_ungetchar(String, !Posn),
(
!.LastDigit = last_digit_is_not_underscore,
grab_string(String, Posn1, BinaryString, !Posn),
- conv_string_to_int(BinaryString, base_2, Token)
+ conv_string_to_int(BinaryString, base_2, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated binary literal")
@@ -2181,7 +2214,8 @@ string_get_binary_2(String, !.LastDigit, Len, Posn1, Token, Context, !Posn) :-
(
!.LastDigit = last_digit_is_not_underscore,
grab_string(String, Posn1, BinaryString, !Posn),
- conv_string_to_int(BinaryString, base_2, Token)
+ conv_string_to_int(BinaryString, base_2, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated binary literal")
@@ -2246,7 +2280,8 @@ get_octal_2(Stream, !.LastDigit, !.RevChars, Token, !IO) :-
Result = eof,
(
!.LastDigit = last_digit_is_not_underscore,
- rev_char_list_to_int(!.RevChars, base_8, Token)
+ rev_char_list_to_int(!.RevChars, base_8, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated octal literal")
@@ -2260,11 +2295,18 @@ get_octal_2(Stream, !.LastDigit, !.RevChars, Token, !IO) :-
else if Char = '_' then
!:LastDigit = last_digit_is_underscore,
get_octal_2(Stream, !.LastDigit, !.RevChars, Token, !IO)
+ else if Char = 'u' then
+ get_integer_size_suffix(Stream, !.RevChars, base_8, unsigned,
+ Token, !IO)
+ else if Char = 'i' then
+ get_integer_size_suffix(Stream, !.RevChars, base_8, signed,
+ Token, !IO)
else
io.putback_char(Stream, Char, !IO),
(
!.LastDigit = last_digit_is_not_underscore,
- rev_char_list_to_int(!.RevChars, base_8, Token)
+ rev_char_list_to_int(!.RevChars, base_8, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated octal literal")
@@ -2277,6 +2319,7 @@ get_octal_2(Stream, !.LastDigit, !.RevChars, Token, !IO) :-
posn::in, posn::out) is det.
string_get_octal_2(String, !.LastDigit, Len, Posn1, Token, Context, !Posn) :-
+ Posn2 = !.Posn,
( if string_read_char(String, Len, Char, !Posn) then
( if char.is_octal_digit(Char) then
!:LastDigit = last_digit_is_not_underscore,
@@ -2286,12 +2329,21 @@ string_get_octal_2(String, !.LastDigit, Len, Posn1, Token, Context, !Posn) :-
!:LastDigit = last_digit_is_underscore,
string_get_octal_2(String, !.LastDigit, Len, Posn1, Token, Context,
!Posn)
+ else if Char = 'u' then
+ string_get_integer_size_suffix(String, Len, Posn1, Posn2, base_8,
+ unsigned, Token, !Posn),
+ string_get_context(Posn1, Context, !Posn)
+ else if Char = 'i' then
+ string_get_integer_size_suffix(String, Len, Posn1, Posn2, base_8,
+ signed, Token, !Posn),
+ string_get_context(Posn1, Context, !Posn)
else
string_ungetchar(String, !Posn),
(
!.LastDigit = last_digit_is_not_underscore,
grab_string(String, Posn1, BinaryString, !Posn),
- conv_string_to_int(BinaryString, base_8, Token)
+ conv_string_to_int(BinaryString, base_8, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated octal literal")
@@ -2302,7 +2354,8 @@ string_get_octal_2(String, !.LastDigit, Len, Posn1, Token, Context, !Posn) :-
(
!.LastDigit = last_digit_is_not_underscore,
grab_string(String, Posn1, BinaryString, !Posn),
- conv_string_to_int(BinaryString, base_8, Token)
+ conv_string_to_int(BinaryString, base_8, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated octal literal")
@@ -2367,7 +2420,8 @@ get_hex_2(Stream, !.LastDigit, !.RevChars, Token, !IO) :-
Result = eof,
(
!.LastDigit = last_digit_is_not_underscore,
- rev_char_list_to_int(!.RevChars, base_16, Token)
+ rev_char_list_to_int(!.RevChars, base_16, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated hexadecimal literal")
@@ -2381,11 +2435,18 @@ get_hex_2(Stream, !.LastDigit, !.RevChars, Token, !IO) :-
else if Char = '_' then
!:LastDigit = last_digit_is_underscore,
get_hex_2(Stream, !.LastDigit, !.RevChars, Token, !IO)
+ else if Char = 'u' then
+ get_integer_size_suffix(Stream, !.RevChars, base_16, unsigned,
+ Token, !IO)
+ else if Char = 'i' then
+ get_integer_size_suffix(Stream, !.RevChars, base_16, signed,
+ Token, !IO)
else
io.putback_char(Stream, Char, !IO),
(
!.LastDigit = last_digit_is_not_underscore,
- rev_char_list_to_int(!.RevChars, base_16, Token)
+ rev_char_list_to_int(!.RevChars, base_16, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated hexadecimal literal")
@@ -2398,6 +2459,7 @@ get_hex_2(Stream, !.LastDigit, !.RevChars, Token, !IO) :-
posn::in, posn::out) is det.
string_get_hex_2(String, !.LastDigit, Len, Posn1, Token, Context, !Posn) :-
+ Posn2 = !.Posn,
( if string_read_char(String, Len, Char, !Posn) then
( if char.is_hex_digit(Char) then
!:LastDigit = last_digit_is_not_underscore,
@@ -2407,12 +2469,21 @@ string_get_hex_2(String, !.LastDigit, Len, Posn1, Token, Context, !Posn) :-
!:LastDigit = last_digit_is_underscore,
string_get_hex_2(String, !.LastDigit, Len, Posn1, Token, Context,
!Posn)
+ else if Char = 'u' then
+ string_get_integer_size_suffix(String, Len, Posn1, Posn2, base_16,
+ unsigned, Token, !Posn),
+ string_get_context(Posn1, Context, !Posn)
+ else if Char = 'i' then
+ string_get_integer_size_suffix(String, Len, Posn1, Posn2, base_16,
+ signed, Token, !Posn),
+ string_get_context(Posn1, Context, !Posn)
else
(
!.LastDigit = last_digit_is_not_underscore,
string_ungetchar(String, !Posn),
grab_string(String, Posn1, BinaryString, !Posn),
- conv_string_to_int(BinaryString, base_16, Token)
+ conv_string_to_int(BinaryString, base_16, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated hexadecimal literal")
@@ -2423,7 +2494,8 @@ string_get_hex_2(String, !.LastDigit, Len, Posn1, Token, Context, !Posn) :-
(
!.LastDigit = last_digit_is_not_underscore,
grab_string(String, Posn1, BinaryString, !Posn),
- conv_string_to_int(BinaryString, base_16, Token)
+ conv_string_to_int(BinaryString, base_16, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated hexadecimal literal")
@@ -2443,7 +2515,8 @@ get_number(Stream, !.LastDigit, !.RevChars, Token, !IO) :-
Result = eof,
(
!.LastDigit = last_digit_is_not_underscore,
- rev_char_list_to_int(!.RevChars, base_10, Token)
+ rev_char_list_to_int(!.RevChars, base_10, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated decimal literal")
@@ -2465,6 +2538,12 @@ get_number(Stream, !.LastDigit, !.RevChars, Token, !IO) :-
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated decimal literal")
)
+ else if Char = 'u' then
+ get_integer_size_suffix(Stream, !.RevChars, base_10, unsigned,
+ Token, !IO)
+ else if Char = 'i' then
+ get_integer_size_suffix(Stream, !.RevChars, base_10, signed,
+ Token, !IO)
else if ( Char = 'e' ; Char = 'E' ) then
(
!.LastDigit = last_digit_is_not_underscore,
@@ -2478,7 +2557,8 @@ get_number(Stream, !.LastDigit, !.RevChars, Token, !IO) :-
io.putback_char(Stream, Char, !IO),
(
!.LastDigit = last_digit_is_not_underscore,
- rev_char_list_to_int(!.RevChars, base_10, Token)
+ rev_char_list_to_int(!.RevChars, base_10, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated decimal literal")
@@ -2491,6 +2571,7 @@ get_number(Stream, !.LastDigit, !.RevChars, Token, !IO) :-
posn::in, posn::out) is det.
string_get_number(String, !.LastDigit, Len, Posn0, Token, Context, !Posn) :-
+ Posn1 = !.Posn,
( if string_read_char(String, Len, Char, !Posn) then
( if char.is_digit(Char) then
!:LastDigit = last_digit_is_not_underscore,
@@ -2510,6 +2591,14 @@ string_get_number(String, !.LastDigit, Len, Posn0, Token, Context, !Posn) :-
Token = error("unterminated decimal literal"),
string_get_context(Posn0, Context, !Posn)
)
+ else if Char = 'u' then
+ string_get_integer_size_suffix(String, Len, Posn0, Posn1, base_10,
+ unsigned, Token, !Posn),
+ string_get_context(Posn0, Context, !Posn)
+ else if Char = 'i' then
+ string_get_integer_size_suffix(String, Len, Posn0, Posn1, base_10,
+ signed, Token, !Posn),
+ string_get_context(Posn0, Context, !Posn)
else if ( Char = 'e' ; Char = 'E' ) then
(
!.LastDigit = last_digit_is_not_underscore,
@@ -2525,7 +2614,8 @@ string_get_number(String, !.LastDigit, Len, Posn0, Token, Context, !Posn) :-
(
!.LastDigit = last_digit_is_not_underscore,
grab_string(String, Posn0, NumberString, !Posn),
- conv_string_to_int(NumberString, base_10, Token)
+ conv_string_to_int(NumberString, base_10, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated decimal literal")
@@ -2536,7 +2626,8 @@ string_get_number(String, !.LastDigit, Len, Posn0, Token, Context, !Posn) :-
(
!.LastDigit = last_digit_is_not_underscore,
grab_string(String, Posn0, NumberString, !Posn),
- conv_string_to_int(NumberString, base_10, Token)
+ conv_string_to_int(NumberString, base_10, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated decimal literal")
@@ -2544,6 +2635,111 @@ string_get_number(String, !.LastDigit, Len, Posn0, Token, Context, !Posn) :-
string_get_context(Posn0, Context, !Posn)
).
+:- pred get_integer_size_suffix(io.input_stream::in, list(char)::in, integer_base::in,
+ signedness::in, token::out, io::di, io::uo) is det.
+
+get_integer_size_suffix(Stream, RevChars, Base, Signedness, Token, !IO) :-
+ io.read_char_unboxed(Stream, Result, Char, !IO),
+ (
+ Result = error(Error),
+ Token = io_error(Error)
+ ;
+ Result = eof,
+ rev_char_list_to_int(RevChars, Base, Signedness, size_word, Token)
+ ;
+ Result = ok,
+ ( if Char = '8' then
+ rev_char_list_to_int(RevChars, Base, Signedness, size_8_bit, Token)
+ else if Char = '1' then
+ get_integer_size_suffix_2(Stream, RevChars, Base, Signedness,
+ '6', size_16_bit, Token, !IO)
+ else if Char = '3' then
+ get_integer_size_suffix_2(Stream, RevChars, Base, Signedness,
+ '2', size_32_bit, Token, !IO)
+ else if Char = '6' then
+ get_integer_size_suffix_2(Stream, RevChars, Base, Signedness,
+ '4', size_64_bit, Token, !IO)
+ else if char.is_digit(Char) then
+ Token = error("invalid integer size suffix")
+ else
+ io.putback_char(Stream, Char, !IO),
+ rev_char_list_to_int(RevChars, Base, Signedness, size_word, Token)
+ )
+ ).
+
+:- pred get_integer_size_suffix_2(io.input_stream::in, list(char)::in,
+ integer_base::in, signedness::in, char::in, integer_size::in,
+ token::out, io::di, io::uo) is det.
+
+get_integer_size_suffix_2(Stream, RevChars, Base, Signedness, ExpectedNextChar,
+ ExpectedSize, Token, !IO) :-
+ io.read_char_unboxed(Stream, Result, Char, !IO),
+ (
+ Result = error(Error),
+ Token = io_error(Error)
+ ;
+ Result = eof,
+ Token = error("invalid integer size suffix")
+ ;
+ Result = ok,
+ ( if Char = ExpectedNextChar then
+ rev_char_list_to_int(RevChars, Base, Signedness, ExpectedSize, Token)
+ else
+ Token = error("invalid integer size suffix")
+ )
+ ).
+
+:- pred string_get_integer_size_suffix(string::in, int::in, posn::in,
+ posn::in, integer_base::in, signedness::in, token::out,
+ posn::in, posn::out) is det.
+
+string_get_integer_size_suffix(String, Len, Posn1, Posn2, Base, Signedness,
+ Token, !Posn) :-
+ ( if string_read_char(String, Len, Char, !Posn) then
+ ( if Char = '8' then
+ grab_string(String, Posn1, DigitString, Posn2, _),
+ conv_string_to_int(DigitString, Base, Signedness, size_8_bit,
+ Token)
+ else if Char = '1' then
+ string_get_integer_size_suffix_2(String, Len, Posn1, Posn2,
+ Base, Signedness, '6', size_16_bit, Token, !Posn)
+ else if Char = '3' then
+ string_get_integer_size_suffix_2(String, Len, Posn1, Posn2,
+ Base, Signedness, '2', size_32_bit, Token, !Posn)
+ else if Char = '6' then
+ string_get_integer_size_suffix_2(String, Len, Posn1, Posn2,
+ Base, Signedness, '4', size_64_bit, Token, !Posn)
+ else if char.is_digit(Char) then
+ Token = error("invalid integer size suffix")
+ else
+ string_ungetchar(String, !Posn),
+ grab_string(String, Posn1, DigitString, Posn2, _),
+ conv_string_to_int(DigitString, Base, Signedness, size_word,
+ Token)
+ )
+ else
+ grab_string(String, Posn1, DigitString, Posn2, _),
+ conv_string_to_int(DigitString, Base, Signedness, size_word,
+ Token)
+ ).
+
+:- pred string_get_integer_size_suffix_2(string::in, int::in,
+ posn::in, posn::in, integer_base::in, signedness::in, char::in,
+ integer_size::in, token::out, posn::in, posn::out) is det.
+
+string_get_integer_size_suffix_2(String, Len, Posn1, Posn2,
+ Base, Signedness, ExpectedChar, Size, Token, !Posn) :-
+ ( if string_read_char(String, Len, Char, !Posn) then
+ ( if Char = ExpectedChar then
+ grab_string(String, Posn1, DigitString, Posn2, _),
+ conv_string_to_int(DigitString, Base, Signedness, Size, Token)
+ else
+ Token = error("invalid integer size suffix")
+ )
+ else
+ Token = error("invalid integer size suffix")
+ ).
+
:- pred get_int_dot(io.input_stream::in, last_digit_is_underscore::in,
list(char)::in, token::out, io::di, io::uo) is det.
@@ -2558,7 +2754,8 @@ get_int_dot(Stream, !.LastDigit, !.RevChars, Token, !IO) :-
io.putback_char(Stream, '.', !IO),
(
!.LastDigit = last_digit_is_not_underscore,
- rev_char_list_to_int(!.RevChars, base_10, Token)
+ rev_char_list_to_int(!.RevChars, base_10, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated decimal literal")
@@ -2579,7 +2776,8 @@ get_int_dot(Stream, !.LastDigit, !.RevChars, Token, !IO) :-
% handle this appropriately.
(
!.LastDigit = last_digit_is_not_underscore,
- rev_char_list_to_int(!.RevChars, base_10, Token0),
+ rev_char_list_to_int(!.RevChars, base_10, signed, size_word,
+ Token0),
( if Token0 = integer(_, Integer, _, _) then
Token = integer_dot(Integer)
else
@@ -2611,7 +2809,8 @@ string_get_int_dot(String, !.LastDigit, Len, Posn0, Token, Context, !Posn) :-
(
!.LastDigit = last_digit_is_not_underscore,
grab_string(String, Posn0, NumberString, !Posn),
- conv_string_to_int(NumberString, base_10, Token)
+ conv_string_to_int(NumberString, base_10, signed,
+ size_word, Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated decimal literal")
@@ -2623,7 +2822,8 @@ string_get_int_dot(String, !.LastDigit, Len, Posn0, Token, Context, !Posn) :-
(
!.LastDigit = last_digit_is_not_underscore,
grab_string(String, Posn0, NumberString, !Posn),
- conv_string_to_int(NumberString, base_10, Token)
+ conv_string_to_int(NumberString, base_10, signed, size_word,
+ Token)
;
!.LastDigit = last_digit_is_underscore,
Token = error("unterminated decimal literal")
@@ -2902,22 +3102,24 @@ string_get_float_exponent_3(String, !.LastDigit, Len, Posn0, Token, Context,
% Utility routines.
%
-:- pred rev_char_list_to_int(list(char)::in, integer_base::in, token::out)
+:- pred rev_char_list_to_int(list(char)::in, integer_base::in,
+ signedness::in, integer_size::in, token::out)
is det.
-rev_char_list_to_int(RevChars, Base, Token) :-
+rev_char_list_to_int(RevChars, Base, Signedness, Size, Token) :-
( if rev_char_list_to_string(RevChars, String) then
- conv_string_to_int(String, Base, Token)
+ conv_string_to_int(String, Base, Signedness, Size, Token)
else
Token = error("invalid character in int")
).
-:- pred conv_string_to_int(string::in, integer_base::in, token::out) is det.
+:- pred conv_string_to_int(string::in, integer_base::in, signedness::in,
+ integer_size::in, token::out) is det.
-conv_string_to_int(String, Base, Token) :-
+conv_string_to_int(String, Base, Signedness, Size, Token) :-
BaseInt = integer_base_int(Base),
( if integer.from_base_string_underscore(BaseInt, String, Integer) then
- Token = integer(Base, Integer, signed, size_word)
+ Token = integer(Base, Integer, Signedness, Size)
else
Token = error("invalid character in int")
).
diff --git a/library/parser.m b/library/parser.m
index 39edb59..02c8eae 100644
--- a/library/parser.m
+++ b/library/parser.m
@@ -1146,6 +1146,10 @@ lexer_signedness_to_term_signedness(signed) = signed.
:- func lexer_size_to_term_size(lexer.integer_size) = term.integer_size.
lexer_size_to_term_size(size_word) = size_word.
+lexer_size_to_term_size(size_8_bit) = size_8_bit.
+lexer_size_to_term_size(size_16_bit) = size_16_bit.
+lexer_size_to_term_size(size_32_bit) = size_32_bit.
+lexer_size_to_term_size(size_64_bit) = size_64_bit.
%---------------------------------------------------------------------------%
%---------------------------------------------------------------------------%
diff --git a/library/stream.string_writer.m b/library/stream.string_writer.m
index f72bd62..efdb35f 100644
--- a/library/stream.string_writer.m
+++ b/library/stream.string_writer.m
@@ -287,11 +287,13 @@ print_cc(Stream, Term, !State) :-
print(Stream, include_details_cc, Term, !State).
print(Stream, NonCanon, Term, !State) :-
- % `string', `char' and `univ' are special cases for print
+ % `string', `char', `uint' and `univ' are special cases for print
( if dynamic_cast(Term, String : string) then
put(Stream, String, !State)
else if dynamic_cast(Term, Char : char) then
put(Stream, Char, !State)
+ else if dynamic_cast(Term, UInt : uint) then
+ put(Stream, uint_to_string(UInt), !State)
else if dynamic_cast(Term, OrigUniv) then
write_univ(Stream, OrigUniv, !State)
else if dynamic_cast(Term, BigInt) then
@@ -382,7 +384,7 @@ do_write_univ(Stream, NonCanon, Univ, !State) :-
do_write_univ_prio(Stream, NonCanon, Univ, Priority, !State) :-
% We need to special-case the builtin types:
- % int, char, float, string
+ % int, uint, char, float, string
% type_info, univ, c_pointer, array
% and private_builtin.type_info
%
@@ -393,9 +395,8 @@ do_write_univ_prio(Stream, NonCanon, Univ, Priority, !State) :-
else if univ_to_type(Univ, Int) then
put_int(Stream, Int, !State)
else if univ_to_type(Univ, UInt) then
- % XXX UINT -- write should emit an unsigned literal
- % print should just emit a decimal
- put_uint(Stream, UInt, !State)
+ put_uint(Stream, UInt, !State),
+ put_char(Stream, 'u', !State)
else if univ_to_type(Univ, Float) then
put_float(Stream, Float, !State)
else if univ_to_type(Univ, Bitmap) then
diff --git a/library/term.m b/library/term.m
index a36b29a..e45bc66 100644
--- a/library/term.m
+++ b/library/term.m
@@ -76,11 +76,11 @@
; unsigned.
:- type integer_size
- ---> size_word.
- %; size_8_bit
- %; size_16_bit
- %; size_32_bit
- %; size_64_bit
+ ---> size_word
+ ; size_8_bit
+ ; size_16_bit
+ ; size_32_bit
+ ; size_64_bit.
:- type generic
---> generic.
diff --git a/library/term_io.m b/library/term_io.m
index 5985605..2eff73f 100644
--- a/library/term_io.m
+++ b/library/term_io.m
@@ -606,12 +606,13 @@ write_constant(OutStream, Const, !IO) :-
write_constant(OutStream, Const, NextToGraphicToken, !IO) :-
(
- Const = term.integer(Base, I, _Signedness, _Size),
- % XXX UINT handle signedness and size.
+ Const = term.integer(Base, I, Signedness, Size),
Prefix = integer_base_prefix(Base),
IntString = integer.to_base_string(I, integer_base_int(Base)),
+ Suffix = integer_signedness_and_size_suffix(Signedness, Size),
io.write_string(OutStream, Prefix, !IO),
- io.write_string(OutStream, IntString, !IO)
+ io.write_string(OutStream, IntString, !IO),
+ io.write_string(OutStream, Suffix, !IO)
;
Const = term.float(F),
io.write_float(OutStream, F, !IO)
@@ -630,11 +631,11 @@ write_constant(OutStream, Const, NextToGraphicToken, !IO) :-
format_constant(Const) =
term_io.format_constant_agt(Const, not_adjacent_to_graphic_token).
-:- func term_io.format_constant_agt(const, adjacent_to_graphic_token) = string.
+:- func format_constant_agt(const, adjacent_to_graphic_token) = string.
- % XXX UINT handle Signedness and Size.
-format_constant_agt(term.integer(Base, I, _Signedness, _Size), _) =
- integer_base_prefix(Base) ++ to_base_string(I, integer_base_int(Base)).
+format_constant_agt(term.integer(Base, I, Signedness, Size), _) =
+ integer_base_prefix(Base) ++ to_base_string(I, integer_base_int(Base)) ++
+ integer_signedness_and_size_suffix(Signedness, Size).
format_constant_agt(term.float(F), _) =
string.float_to_string(F).
format_constant_agt(term.atom(A), NextToGraphicToken) =
@@ -654,6 +655,20 @@ integer_base_prefix(base_8) = "0o".
integer_base_prefix(base_10) = "".
integer_base_prefix(base_16) = "0x".
+:- func integer_signedness_and_size_suffix(term.signedness,
+ term.integer_size) = string.
+
+integer_signedness_and_size_suffix(signed, size_word) = "".
+integer_signedness_and_size_suffix(signed, size_8_bit) = "i8".
+integer_signedness_and_size_suffix(signed, size_16_bit) = "i16".
+integer_signedness_and_size_suffix(signed, size_32_bit) = "i32".
+integer_signedness_and_size_suffix(signed, size_64_bit) = "i64".
+integer_signedness_and_size_suffix(unsigned, size_word) = "u".
+integer_signedness_and_size_suffix(unsigned, size_8_bit) = "u8".
+integer_signedness_and_size_suffix(unsigned, size_16_bit) = "u16".
+integer_signedness_and_size_suffix(unsigned, size_32_bit) = "u32".
+integer_signedness_and_size_suffix(unsigned, size_64_bit) = "u64".
+
%---------------------------------------------------------------------------%
quote_char(C, !IO) :-
diff --git a/tests/hard_coded/Mmakefile b/tests/hard_coded/Mmakefile
index 4393750..a51681e 100644
--- a/tests/hard_coded/Mmakefile
+++ b/tests/hard_coded/Mmakefile
@@ -183,6 +183,7 @@ ORDINARY_PROGS = \
lco_no_inline \
lco_pack_args \
lexer_bigint \
+ lexer_ints \
lexer_zero \
list_series_int \
list_split_take_drop \
diff --git a/tests/hard_coded/lexer_ints.exp b/tests/hard_coded/lexer_ints.exp
index e69de29..25fc520 100644
--- a/tests/hard_coded/lexer_ints.exp
+++ b/tests/hard_coded/lexer_ints.exp
@@ -0,0 +1,313 @@
+1: string("Signed binary integers")
+2: integer(base_2, i(1, [2]), signed, size_word)
+3: integer(base_2, i(1, [4]), signed, size_word)
+4: integer(base_2, i(1, [2]), signed, size_8_bit)
+5: integer(base_2, i(1, [4]), signed, size_16_bit)
+6: integer(base_2, i(1, [4]), signed, size_32_bit)
+7: integer(base_2, i(1, [5]), signed, size_64_bit)
+9: string("Signed binary integers (with underscores)")
+10: integer(base_2, i(1, [2184]), signed, size_word)
+11: integer(base_2, i(1, [2184]), signed, size_word)
+12: integer(base_2, i(1, [15]), signed, size_8_bit)
+13: integer(base_2, i(1, [2184]), signed, size_16_bit)
+14: integer(base_2, i(1, [2184]), signed, size_32_bit)
+15: integer(base_2, i(1, [2184]), signed, size_64_bit)
+17: string("Signed binary integers (with leading zeros)")
+18: integer(base_2, i(1, [2]), signed, size_word)
+19: integer(base_2, i(1, [4]), signed, size_word)
+20: integer(base_2, i(1, [2]), signed, size_8_bit)
+21: integer(base_2, i(1, [4]), signed, size_16_bit)
+22: integer(base_2, i(1, [4]), signed, size_32_bit)
+23: integer(base_2, i(1, [5]), signed, size_64_bit)
+25: string("Unsigned binary integers")
+26: integer(base_2, i(1, [4]), unsigned, size_word)
+27: integer(base_2, i(1, [2]), unsigned, size_8_bit)
+28: integer(base_2, i(1, [4]), unsigned, size_16_bit)
+29: integer(base_2, i(1, [4]), unsigned, size_32_bit)
+30: integer(base_2, i(1, [5]), unsigned, size_64_bit)
+32: string("Unsigned binary integers (with underscores)")
+33: integer(base_2, i(1, [2184]), unsigned, size_word)
+34: integer(base_2, i(1, [15]), unsigned, size_8_bit)
+35: integer(base_2, i(1, [2184]), unsigned, size_16_bit)
+36: integer(base_2, i(1, [2184]), unsigned, size_32_bit)
+37: integer(base_2, i(1, [2184]), unsigned, size_64_bit)
+39: string("Unsigned binary integers (with leading zeros)")
+40: integer(base_2, i(1, [4]), unsigned, size_word)
+41: integer(base_2, i(1, [2]), unsigned, size_8_bit)
+42: integer(base_2, i(1, [4]), unsigned, size_16_bit)
+43: integer(base_2, i(1, [4]), unsigned, size_32_bit)
+44: integer(base_2, i(1, [5]), unsigned, size_64_bit)
+46: string("Signed octal integers")
+47: integer(base_8, i(1, [63]), signed, size_word)
+48: integer(base_8, i(1, [63]), signed, size_word)
+49: integer(base_8, i(1, [63]), signed, size_8_bit)
+50: integer(base_8, i(1, [63]), signed, size_16_bit)
+51: integer(base_8, i(1, [63]), signed, size_32_bit)
+52: integer(base_8, i(1, [63]), signed, size_64_bit)
+54: string("Signed octal integers (with underscores)")
+55: integer(base_8, i(1, [63]), signed, size_word)
+56: integer(base_8, i(1, [63]), signed, size_word)
+57: integer(base_8, i(1, [63]), signed, size_8_bit)
+58: integer(base_8, i(1, [63]), signed, size_16_bit)
+59: integer(base_8, i(1, [63]), signed, size_32_bit)
+60: integer(base_8, i(1, [63]), signed, size_64_bit)
+62: string("Signed octal integers")
+63: integer(base_8, i(1, [63]), signed, size_word)
+64: integer(base_8, i(1, [63]), signed, size_word)
+65: integer(base_8, i(1, [63]), signed, size_8_bit)
+66: integer(base_8, i(1, [63]), signed, size_16_bit)
+67: integer(base_8, i(1, [63]), signed, size_32_bit)
+68: integer(base_8, i(1, [63]), signed, size_64_bit)
+70: string("Unsigned octal integers")
+71: integer(base_8, i(1, [63]), unsigned, size_word)
+72: integer(base_8, i(1, [63]), unsigned, size_8_bit)
+73: integer(base_8, i(1, [63]), unsigned, size_16_bit)
+74: integer(base_8, i(1, [63]), unsigned, size_32_bit)
+75: integer(base_8, i(1, [63]), unsigned, size_64_bit)
+77: string("Unsigned octal integers (with underscores)")
+78: integer(base_8, i(1, [63]), unsigned, size_word)
+79: integer(base_8, i(1, [63]), unsigned, size_8_bit)
+80: integer(base_8, i(1, [63]), unsigned, size_16_bit)
+81: integer(base_8, i(1, [63]), unsigned, size_32_bit)
+82: integer(base_8, i(1, [63]), unsigned, size_64_bit)
+84: string("Unsigned octal integers (with leading zeros)")
+85: integer(base_8, i(1, [63]), unsigned, size_word)
+86: integer(base_8, i(1, [63]), unsigned, size_8_bit)
+87: integer(base_8, i(1, [63]), unsigned, size_16_bit)
+88: integer(base_8, i(1, [63]), unsigned, size_32_bit)
+89: integer(base_8, i(1, [63]), unsigned, size_64_bit)
+91: string("Signed decimal integers")
+92: integer(base_10, i(1, [99]), signed, size_word)
+93: integer(base_10, i(1, [99]), signed, size_word)
+94: integer(base_10, i(1, [99]), signed, size_8_bit)
+95: integer(base_10, i(1, [99]), signed, size_16_bit)
+96: integer(base_10, i(1, [99]), signed, size_32_bit)
+97: integer(base_10, i(1, [99]), signed, size_64_bit)
+99: string("Signed decimal integers (with underscores)")
+100: integer(base_10, i(1, [99]), signed, size_word)
+101: integer(base_10, i(1, [99]), signed, size_word)
+102: integer(base_10, i(1, [99]), signed, size_8_bit)
+103: integer(base_10, i(1, [99]), signed, size_16_bit)
+104: integer(base_10, i(1, [99]), signed, size_32_bit)
+105: integer(base_10, i(1, [99]), signed, size_64_bit)
+107: string("Signed decimal integers (with leading zeros)")
+108: integer(base_10, i(1, [99]), signed, size_word)
+109: integer(base_10, i(1, [99]), signed, size_word)
+110: integer(base_10, i(1, [99]), signed, size_8_bit)
+111: integer(base_10, i(1, [99]), signed, size_16_bit)
+112: integer(base_10, i(1, [99]), signed, size_32_bit)
+113: integer(base_10, i(1, [99]), signed, size_64_bit)
+115: string("Unsigned decimal integers")
+116: integer(base_10, i(1, [99]), unsigned, size_word)
+117: integer(base_10, i(1, [99]), unsigned, size_8_bit)
+118: integer(base_10, i(1, [99]), unsigned, size_16_bit)
+119: integer(base_10, i(1, [99]), unsigned, size_32_bit)
+120: integer(base_10, i(1, [99]), unsigned, size_64_bit)
+122: string("Unsigned decimal integers (with underscores)")
+123: integer(base_10, i(1, [99]), unsigned, size_word)
+124: integer(base_10, i(1, [99]), unsigned, size_8_bit)
+125: integer(base_10, i(1, [99]), unsigned, size_16_bit)
+126: integer(base_10, i(1, [99]), unsigned, size_32_bit)
+127: integer(base_10, i(1, [99]), unsigned, size_64_bit)
+129: string("Unsigned decimal integers (with leading zeros)")
+130: integer(base_10, i(1, [99]), unsigned, size_word)
+131: integer(base_10, i(1, [99]), unsigned, size_8_bit)
+132: integer(base_10, i(1, [99]), unsigned, size_16_bit)
+133: integer(base_10, i(1, [99]), unsigned, size_32_bit)
+134: integer(base_10, i(1, [99]), unsigned, size_64_bit)
+136: string("Signed hexadecimal integers")
+137: integer(base_16, i(1, [31]), signed, size_word)
+138: integer(base_16, i(1, [31]), signed, size_word)
+139: integer(base_16, i(1, [31]), signed, size_8_bit)
+140: integer(base_16, i(1, [31]), signed, size_16_bit)
+141: integer(base_16, i(1, [31]), signed, size_32_bit)
+142: integer(base_16, i(1, [31]), signed, size_64_bit)
+144: string("Signed hexadecimal integers (with underscores)")
+145: integer(base_16, i(1, [31]), signed, size_word)
+146: integer(base_16, i(1, [31]), signed, size_word)
+147: integer(base_16, i(1, [31]), signed, size_8_bit)
+148: integer(base_16, i(1, [31]), signed, size_16_bit)
+149: integer(base_16, i(1, [31]), signed, size_32_bit)
+150: integer(base_16, i(1, [31]), signed, size_64_bit)
+152: string("Signed hexadecimal integers (with leading zeros)")
+153: integer(base_16, i(1, [31]), signed, size_word)
+154: integer(base_16, i(1, [31]), signed, size_word)
+155: integer(base_16, i(1, [31]), signed, size_8_bit)
+156: integer(base_16, i(1, [31]), signed, size_16_bit)
+157: integer(base_16, i(1, [31]), signed, size_32_bit)
+158: integer(base_16, i(1, [31]), signed, size_64_bit)
+160: string("Unsigned hexadecimal integers")
+161: integer(base_16, i(1, [31]), unsigned, size_word)
+162: integer(base_16, i(1, [31]), unsigned, size_8_bit)
+163: integer(base_16, i(1, [31]), unsigned, size_16_bit)
+164: integer(base_16, i(1, [31]), unsigned, size_32_bit)
+165: integer(base_16, i(1, [31]), unsigned, size_64_bit)
+167: string("Unsigned hexadecimal integers (with underscores)")
+168: integer(base_16, i(1, [31]), unsigned, size_word)
+169: integer(base_16, i(1, [31]), unsigned, size_8_bit)
+170: integer(base_16, i(1, [31]), unsigned, size_16_bit)
+171: integer(base_16, i(1, [31]), unsigned, size_32_bit)
+172: integer(base_16, i(1, [31]), unsigned, size_64_bit)
+174: string("Unsigned hexadecimal integers (with leading zeros)")
+175: integer(base_16, i(1, [31]), unsigned, size_word)
+176: integer(base_16, i(1, [31]), unsigned, size_8_bit)
+177: integer(base_16, i(1, [31]), unsigned, size_16_bit)
+178: integer(base_16, i(1, [31]), unsigned, size_32_bit)
+179: integer(base_16, i(1, [31]), unsigned, size_64_bit)
+
+1: string("Signed binary integers")
+2: integer(base_2, i(1, [2]), signed, size_word)
+3: integer(base_2, i(1, [4]), signed, size_word)
+4: integer(base_2, i(1, [2]), signed, size_8_bit)
+5: integer(base_2, i(1, [4]), signed, size_16_bit)
+6: integer(base_2, i(1, [4]), signed, size_32_bit)
+7: integer(base_2, i(1, [5]), signed, size_64_bit)
+9: string("Signed binary integers (with underscores)")
+10: integer(base_2, i(1, [2184]), signed, size_word)
+11: integer(base_2, i(1, [2184]), signed, size_word)
+12: integer(base_2, i(1, [15]), signed, size_8_bit)
+13: integer(base_2, i(1, [2184]), signed, size_16_bit)
+14: integer(base_2, i(1, [2184]), signed, size_32_bit)
+15: integer(base_2, i(1, [2184]), signed, size_64_bit)
+17: string("Signed binary integers (with leading zeros)")
+18: integer(base_2, i(1, [2]), signed, size_word)
+19: integer(base_2, i(1, [4]), signed, size_word)
+20: integer(base_2, i(1, [2]), signed, size_8_bit)
+21: integer(base_2, i(1, [4]), signed, size_16_bit)
+22: integer(base_2, i(1, [4]), signed, size_32_bit)
+23: integer(base_2, i(1, [5]), signed, size_64_bit)
+25: string("Unsigned binary integers")
+26: integer(base_2, i(1, [4]), unsigned, size_word)
+27: integer(base_2, i(1, [2]), unsigned, size_8_bit)
+28: integer(base_2, i(1, [4]), unsigned, size_16_bit)
+29: integer(base_2, i(1, [4]), unsigned, size_32_bit)
+30: integer(base_2, i(1, [5]), unsigned, size_64_bit)
+32: string("Unsigned binary integers (with underscores)")
+33: integer(base_2, i(1, [2184]), unsigned, size_word)
+34: integer(base_2, i(1, [15]), unsigned, size_8_bit)
+35: integer(base_2, i(1, [2184]), unsigned, size_16_bit)
+36: integer(base_2, i(1, [2184]), unsigned, size_32_bit)
+37: integer(base_2, i(1, [2184]), unsigned, size_64_bit)
+39: string("Unsigned binary integers (with leading zeros)")
+40: integer(base_2, i(1, [4]), unsigned, size_word)
+41: integer(base_2, i(1, [2]), unsigned, size_8_bit)
+42: integer(base_2, i(1, [4]), unsigned, size_16_bit)
+43: integer(base_2, i(1, [4]), unsigned, size_32_bit)
+44: integer(base_2, i(1, [5]), unsigned, size_64_bit)
+46: string("Signed octal integers")
+47: integer(base_8, i(1, [63]), signed, size_word)
+48: integer(base_8, i(1, [63]), signed, size_word)
+49: integer(base_8, i(1, [63]), signed, size_8_bit)
+50: integer(base_8, i(1, [63]), signed, size_16_bit)
+51: integer(base_8, i(1, [63]), signed, size_32_bit)
+52: integer(base_8, i(1, [63]), signed, size_64_bit)
+54: string("Signed octal integers (with underscores)")
+55: integer(base_8, i(1, [63]), signed, size_word)
+56: integer(base_8, i(1, [63]), signed, size_word)
+57: integer(base_8, i(1, [63]), signed, size_8_bit)
+58: integer(base_8, i(1, [63]), signed, size_16_bit)
+59: integer(base_8, i(1, [63]), signed, size_32_bit)
+60: integer(base_8, i(1, [63]), signed, size_64_bit)
+62: string("Signed octal integers")
+63: integer(base_8, i(1, [63]), signed, size_word)
+64: integer(base_8, i(1, [63]), signed, size_word)
+65: integer(base_8, i(1, [63]), signed, size_8_bit)
+66: integer(base_8, i(1, [63]), signed, size_16_bit)
+67: integer(base_8, i(1, [63]), signed, size_32_bit)
+68: integer(base_8, i(1, [63]), signed, size_64_bit)
+70: string("Unsigned octal integers")
+71: integer(base_8, i(1, [63]), unsigned, size_word)
+72: integer(base_8, i(1, [63]), unsigned, size_8_bit)
+73: integer(base_8, i(1, [63]), unsigned, size_16_bit)
+74: integer(base_8, i(1, [63]), unsigned, size_32_bit)
+75: integer(base_8, i(1, [63]), unsigned, size_64_bit)
+77: string("Unsigned octal integers (with underscores)")
+78: integer(base_8, i(1, [63]), unsigned, size_word)
+79: integer(base_8, i(1, [63]), unsigned, size_8_bit)
+80: integer(base_8, i(1, [63]), unsigned, size_16_bit)
+81: integer(base_8, i(1, [63]), unsigned, size_32_bit)
+82: integer(base_8, i(1, [63]), unsigned, size_64_bit)
+84: string("Unsigned octal integers (with leading zeros)")
+85: integer(base_8, i(1, [63]), unsigned, size_word)
+86: integer(base_8, i(1, [63]), unsigned, size_8_bit)
+87: integer(base_8, i(1, [63]), unsigned, size_16_bit)
+88: integer(base_8, i(1, [63]), unsigned, size_32_bit)
+89: integer(base_8, i(1, [63]), unsigned, size_64_bit)
+91: string("Signed decimal integers")
+92: integer(base_10, i(1, [99]), signed, size_word)
+93: integer(base_10, i(1, [99]), signed, size_word)
+94: integer(base_10, i(1, [99]), signed, size_8_bit)
+95: integer(base_10, i(1, [99]), signed, size_16_bit)
+96: integer(base_10, i(1, [99]), signed, size_32_bit)
+97: integer(base_10, i(1, [99]), signed, size_64_bit)
+99: string("Signed decimal integers (with underscores)")
+100: integer(base_10, i(1, [99]), signed, size_word)
+101: integer(base_10, i(1, [99]), signed, size_word)
+102: integer(base_10, i(1, [99]), signed, size_8_bit)
+103: integer(base_10, i(1, [99]), signed, size_16_bit)
+104: integer(base_10, i(1, [99]), signed, size_32_bit)
+105: integer(base_10, i(1, [99]), signed, size_64_bit)
+107: string("Signed decimal integers (with leading zeros)")
+108: integer(base_10, i(1, [99]), signed, size_word)
+109: integer(base_10, i(1, [99]), signed, size_word)
+110: integer(base_10, i(1, [99]), signed, size_8_bit)
+111: integer(base_10, i(1, [99]), signed, size_16_bit)
+112: integer(base_10, i(1, [99]), signed, size_32_bit)
+113: integer(base_10, i(1, [99]), signed, size_64_bit)
+115: string("Unsigned decimal integers")
+116: integer(base_10, i(1, [99]), unsigned, size_word)
+117: integer(base_10, i(1, [99]), unsigned, size_8_bit)
+118: integer(base_10, i(1, [99]), unsigned, size_16_bit)
+119: integer(base_10, i(1, [99]), unsigned, size_32_bit)
+120: integer(base_10, i(1, [99]), unsigned, size_64_bit)
+122: string("Unsigned decimal integers (with underscores)")
+123: integer(base_10, i(1, [99]), unsigned, size_word)
+124: integer(base_10, i(1, [99]), unsigned, size_8_bit)
+125: integer(base_10, i(1, [99]), unsigned, size_16_bit)
+126: integer(base_10, i(1, [99]), unsigned, size_32_bit)
+127: integer(base_10, i(1, [99]), unsigned, size_64_bit)
+129: string("Unsigned decimal integers (with leading zeros)")
+130: integer(base_10, i(1, [99]), unsigned, size_word)
+131: integer(base_10, i(1, [99]), unsigned, size_8_bit)
+132: integer(base_10, i(1, [99]), unsigned, size_16_bit)
+133: integer(base_10, i(1, [99]), unsigned, size_32_bit)
+134: integer(base_10, i(1, [99]), unsigned, size_64_bit)
+136: string("Signed hexadecimal integers")
+137: integer(base_16, i(1, [31]), signed, size_word)
+138: integer(base_16, i(1, [31]), signed, size_word)
+139: integer(base_16, i(1, [31]), signed, size_8_bit)
+140: integer(base_16, i(1, [31]), signed, size_16_bit)
+141: integer(base_16, i(1, [31]), signed, size_32_bit)
+142: integer(base_16, i(1, [31]), signed, size_64_bit)
+144: string("Signed hexadecimal integers (with underscores)")
+145: integer(base_16, i(1, [31]), signed, size_word)
+146: integer(base_16, i(1, [31]), signed, size_word)
+147: integer(base_16, i(1, [31]), signed, size_8_bit)
+148: integer(base_16, i(1, [31]), signed, size_16_bit)
+149: integer(base_16, i(1, [31]), signed, size_32_bit)
+150: integer(base_16, i(1, [31]), signed, size_64_bit)
+152: string("Signed hexadecimal integers (with leading zeros)")
+153: integer(base_16, i(1, [31]), signed, size_word)
+154: integer(base_16, i(1, [31]), signed, size_word)
+155: integer(base_16, i(1, [31]), signed, size_8_bit)
+156: integer(base_16, i(1, [31]), signed, size_16_bit)
+157: integer(base_16, i(1, [31]), signed, size_32_bit)
+158: integer(base_16, i(1, [31]), signed, size_64_bit)
+160: string("Unsigned hexadecimal integers")
+161: integer(base_16, i(1, [31]), unsigned, size_word)
+162: integer(base_16, i(1, [31]), unsigned, size_8_bit)
+163: integer(base_16, i(1, [31]), unsigned, size_16_bit)
+164: integer(base_16, i(1, [31]), unsigned, size_32_bit)
+165: integer(base_16, i(1, [31]), unsigned, size_64_bit)
+167: string("Unsigned hexadecimal integers (with underscores)")
+168: integer(base_16, i(1, [31]), unsigned, size_word)
+169: integer(base_16, i(1, [31]), unsigned, size_8_bit)
+170: integer(base_16, i(1, [31]), unsigned, size_16_bit)
+171: integer(base_16, i(1, [31]), unsigned, size_32_bit)
+172: integer(base_16, i(1, [31]), unsigned, size_64_bit)
+174: string("Unsigned hexadecimal integers (with leading zeros)")
+175: integer(base_16, i(1, [31]), unsigned, size_word)
+176: integer(base_16, i(1, [31]), unsigned, size_8_bit)
+177: integer(base_16, i(1, [31]), unsigned, size_16_bit)
+178: integer(base_16, i(1, [31]), unsigned, size_32_bit)
+179: integer(base_16, i(1, [31]), unsigned, size_64_bit)
diff --git a/tests/hard_coded/lexer_ints.inp b/tests/hard_coded/lexer_ints.inp
index e69de29..b3fb7dc 100644
--- a/tests/hard_coded/lexer_ints.inp
+++ b/tests/hard_coded/lexer_ints.inp
@@ -0,0 +1,179 @@
+"Signed binary integers"
+0b10
+0b100i
+0b10i8
+0b100i16
+0b100i32
+0b101i64
+
+"Signed binary integers (with underscores)"
+0b_1000_1000_1000
+0b_1000_1000_1000_i
+0b_11_11_i8
+0b_1000_1000_1000_i16
+0b_1000_1000_1000_i32
+0b_1000_1000_1000_i64
+
+"Signed binary integers (with leading zeros)"
+0b0010
+0b00100i
+0b0010i8
+0b00100i16
+0b00100i32
+0b00101i64
+
+"Unsigned binary integers"
+0b100u
+0b10u8
+0b100u16
+0b100u32
+0b101u64
+
+"Unsigned binary integers (with underscores)"
+0b_1000_1000_1000_u
+0b_11_11_u8
+0b_1000_1000_1000_u16
+0b_1000_1000_1000_u32
+0b_1000_1000_1000_u64
+
+"Unsigned binary integers (with leading zeros)"
+0b00100u
+0b0010u8
+0b00100u16
+0b00100u32
+0b00101u64
+
+"Signed octal integers"
+0o77
+0o77i
+0o77i8
+0o77i16
+0o77i32
+0o77i64
+
+"Signed octal integers (with underscores)"
+0o_77
+0o_77_i
+0o_77_i8
+0o_77_i16
+0o_77_i32
+0o_77_i64
+
+"Signed octal integers"
+0o0077
+0o0077i
+0o0077i8
+0o0077i16
+0o0077i32
+0o0077i64
+
+"Unsigned octal integers"
+0o77u
+0o77u8
+0o77u16
+0o77u32
+0o77u64
+
+"Unsigned octal integers (with underscores)"
+0o_77_u
+0o_77_u8
+0o_77_u16
+0o_77_u32
+0o_77_u64
+
+"Unsigned octal integers (with leading zeros)"
+0o0077u
+0o0077u8
+0o0077u16
+0o0077u32
+0o0077u64
+
+"Signed decimal integers"
+99
+99i
+99i8
+99i16
+99i32
+99i64
+
+"Signed decimal integers (with underscores)"
+99
+99_i
+99_i8
+99_i16
+99_i32
+99_i64
+
+"Signed decimal integers (with leading zeros)"
+0099
+0099i
+0099i8
+0099i16
+0099i32
+0099i64
+
+"Unsigned decimal integers"
+99u
+99u8
+99u16
+99u32
+99u64
+
+"Unsigned decimal integers (with underscores)"
+99_u
+99_u8
+99_u16
+99_u32
+99_u64
+
+"Unsigned decimal integers (with leading zeros)"
+0099u
+0099u8
+0099u16
+0099u32
+0099u64
+
+"Signed hexadecimal integers"
+0x1f
+0x1fi
+0x1fi8
+0x1fi16
+0x1fi32
+0x1fi64
+
+"Signed hexadecimal integers (with underscores)"
+0x_1f
+0x_1f_i
+0x_1f_i8
+0x_1f_i16
+0x_1f_i32
+0x_1f_i64
+
+"Signed hexadecimal integers (with leading zeros)"
+0x001f
+0x001fi
+0x001fi8
+0x01fi16
+0x001fi32
+0x001fi64
+
+"Unsigned hexadecimal integers"
+0x1fu
+0x1fu8
+0x1fu16
+0x1fu32
+0x1fu64
+
+"Unsigned hexadecimal integers (with underscores)"
+0x_1f_u
+0x_1f_u8
+0x_1f_u16
+0x_1f_u32
+0x_1f_u64
+
+"Unsigned hexadecimal integers (with leading zeros)"
+0x001fu
+0x001fu8
+0x001fu16
+0x001fu32
+0x001fu64
diff --git a/tests/hard_coded/lexer_ints.m b/tests/hard_coded/lexer_ints.m
index e69de29..8032b60 100644
--- a/tests/hard_coded/lexer_ints.m
+++ b/tests/hard_coded/lexer_ints.m
@@ -0,0 +1,53 @@
+%---------------------------------------------------------------------------%
+% vim: ts=4 sw=4 et ft=mercury
+%---------------------------------------------------------------------------%
+
+:- module lexer_ints.
+:- interface.
+
+:- import_module io.
+
+:- pred main(io::di, io::uo) is det.
+
+%---------------------------------------------------------------------------%
+
+:- implementation.
+
+:- import_module lexer.
+
+%---------------------------------------------------------------------------%
+
+main(!IO) :-
+ % Read from the current input stream.
+ lexer.get_token_list(Tokens, !IO),
+ write_token_list(Tokens, !IO),
+ io.nl(!IO),
+
+ % Read from a string.
+ io.open_input("lexer_ints.inp", OpenRes, !IO),
+ (
+ OpenRes = ok(Stream),
+ io.read_file_as_string(Stream, ReadRes, !IO),
+ (
+ ReadRes = ok(String),
+ Posn0 = posn(1, 0, 0),
+ lexer.string_get_token_list(String, StringTokens, Posn0, _Posn),
+ write_token_list(StringTokens, !IO)
+ ;
+ ReadRes = error(_, Error),
+ io.write_line(Error, !IO)
+ ),
+ io.close_input(Stream, !IO)
+ ;
+ OpenRes = error(Error),
+ io.write_line(Error, !IO)
+ ).
+
+:- pred write_token_list(token_list::in, io::di, io::uo) is det.
+
+write_token_list(token_nil, !IO).
+write_token_list(token_cons(Token, Context, List), !IO) :-
+ io.write_int(Context, !IO),
+ io.write_string(": ", !IO),
+ io.write_line(Token, !IO),
+ write_token_list(List, !IO).
diff --git a/tests/hard_coded/lexer_zero.exp b/tests/hard_coded/lexer_zero.exp
index 5e08feb..bc3e725 100644
--- a/tests/hard_coded/lexer_zero.exp
+++ b/tests/hard_coded/lexer_zero.exp
@@ -1,47 +1,367 @@
-integer(base_10, i(0, []), signed, size_word)
-integer(base_10, i(2, [753, 8526]), signed, size_word)
-integer(base_10, i(1, [10]), signed, size_word)
-integer(base_10, i(1, [97]), signed, size_word)
-integer(base_10, i(1, [97]), signed, size_word)
-name("bc")
-integer(base_2, i(0, []), signed, size_word)
-integer(base_2, i(1, [1]), signed, size_word)
-integer(base_2, i(1, [3]), signed, size_word)
-integer(base_2, i(1, [7]), signed, size_word)
-integer(base_8, i(0, []), signed, size_word)
-integer(base_8, i(1, [1]), signed, size_word)
-integer(base_8, i(2, [20, 14711]), signed, size_word)
-integer(base_16, i(1, [1]), signed, size_word)
-integer(base_16, i(3, [1, 2257, 5752]), signed, size_word)
-integer(base_16, i(2, [9903, 3567]), signed, size_word)
-integer(base_16, i(2, [9903, 3567]), signed, size_word)
-float(0.123)
-float(1.23e+44)
-float(0.0)
-float(0.0)
-float(0.0)
-float(0.0)
+1: integer(base_10, i(0, []), signed, size_word)
+2: integer(base_10, i(2, [753, 8526]), signed, size_word)
+3: integer(base_10, i(1, [10]), signed, size_word)
+4: integer(base_10, i(1, [97]), signed, size_word)
+5: integer(base_10, i(1, [97]), signed, size_word)
+5: name("bc")
+7: integer(base_2, i(0, []), signed, size_word)
+8: integer(base_2, i(1, [1]), signed, size_word)
+9: integer(base_2, i(1, [3]), signed, size_word)
+10: integer(base_2, i(1, [7]), signed, size_word)
+14: integer(base_8, i(0, []), signed, size_word)
+15: integer(base_8, i(1, [1]), signed, size_word)
+16: integer(base_8, i(2, [20, 14711]), signed, size_word)
+18: integer(base_16, i(1, [1]), signed, size_word)
+19: integer(base_16, i(3, [1, 2257, 5752]), signed, size_word)
+20: integer(base_16, i(2, [9903, 3567]), signed, size_word)
+21: integer(base_16, i(2, [9903, 3567]), signed, size_word)
+25: float(0.123)
+26: float(1.23e+44)
+28: float(0.0)
+29: float(0.0)
+30: float(0.0)
+31: float(0.0)
+33: string("Decimal signed zeros")
+34: integer(base_10, i(0, []), signed, size_word)
+35: integer(base_10, i(0, []), signed, size_8_bit)
+36: integer(base_10, i(0, []), signed, size_16_bit)
+37: integer(base_10, i(0, []), signed, size_32_bit)
+38: integer(base_10, i(0, []), signed, size_64_bit)
+40: string("Decimal signed zeros (with underscores)")
+41: integer(base_10, i(0, []), signed, size_word)
+42: integer(base_10, i(0, []), signed, size_8_bit)
+43: integer(base_10, i(0, []), signed, size_16_bit)
+44: integer(base_10, i(0, []), signed, size_32_bit)
+45: integer(base_10, i(0, []), signed, size_64_bit)
+47: string("Decimal signed zeros (with leading zeros)")
+48: integer(base_10, i(0, []), signed, size_word)
+49: integer(base_10, i(0, []), signed, size_8_bit)
+50: integer(base_10, i(0, []), signed, size_16_bit)
+51: integer(base_10, i(0, []), signed, size_32_bit)
+52: integer(base_10, i(0, []), signed, size_64_bit)
+54: string("Decimal unsigned zeros")
+55: integer(base_10, i(0, []), unsigned, size_word)
+56: integer(base_10, i(0, []), unsigned, size_8_bit)
+57: integer(base_10, i(0, []), unsigned, size_16_bit)
+58: integer(base_10, i(0, []), unsigned, size_32_bit)
+59: integer(base_10, i(0, []), unsigned, size_64_bit)
+61: string("Decimal unsigned zeros (with underscores)")
+62: integer(base_10, i(0, []), unsigned, size_word)
+63: integer(base_10, i(0, []), unsigned, size_8_bit)
+64: integer(base_10, i(0, []), unsigned, size_16_bit)
+65: integer(base_10, i(0, []), unsigned, size_32_bit)
+66: integer(base_10, i(0, []), unsigned, size_64_bit)
+68: string("Decimal unsigned zeros (with leading zeros)")
+69: integer(base_10, i(0, []), unsigned, size_word)
+70: integer(base_10, i(0, []), unsigned, size_8_bit)
+71: integer(base_10, i(0, []), unsigned, size_16_bit)
+72: integer(base_10, i(0, []), unsigned, size_32_bit)
+73: integer(base_10, i(0, []), unsigned, size_64_bit)
+75: string("Binary signed zeros")
+76: integer(base_2, i(0, []), signed, size_word)
+77: integer(base_2, i(0, []), signed, size_word)
+78: integer(base_2, i(0, []), signed, size_8_bit)
+79: integer(base_2, i(0, []), signed, size_16_bit)
+80: integer(base_2, i(0, []), signed, size_32_bit)
+81: integer(base_2, i(0, []), signed, size_64_bit)
+83: string("Binary signed zeros (with underscores)")
+84: integer(base_2, i(0, []), signed, size_word)
+85: integer(base_2, i(0, []), signed, size_word)
+86: integer(base_2, i(0, []), signed, size_8_bit)
+87: integer(base_2, i(0, []), signed, size_16_bit)
+88: integer(base_2, i(0, []), signed, size_32_bit)
+89: integer(base_2, i(0, []), signed, size_64_bit)
+91: string("Binary signed zeros (with leading zeros)")
+92: integer(base_2, i(0, []), signed, size_word)
+93: integer(base_2, i(0, []), signed, size_word)
+94: integer(base_2, i(0, []), signed, size_8_bit)
+95: integer(base_2, i(0, []), signed, size_16_bit)
+96: integer(base_2, i(0, []), signed, size_32_bit)
+97: integer(base_2, i(0, []), signed, size_64_bit)
+99: string("Binary unsigned zeros")
+100: integer(base_2, i(0, []), unsigned, size_word)
+101: integer(base_2, i(0, []), unsigned, size_8_bit)
+102: integer(base_2, i(0, []), unsigned, size_16_bit)
+103: integer(base_2, i(0, []), unsigned, size_32_bit)
+104: integer(base_2, i(0, []), unsigned, size_64_bit)
+106: string("Binary unsigned zeros (with underscores)")
+107: integer(base_2, i(0, []), unsigned, size_word)
+108: integer(base_2, i(0, []), unsigned, size_8_bit)
+109: integer(base_2, i(0, []), unsigned, size_16_bit)
+110: integer(base_2, i(0, []), unsigned, size_32_bit)
+111: integer(base_2, i(0, []), unsigned, size_64_bit)
+113: string("Binary unsigned zeros (with leading zeros)")
+114: integer(base_2, i(0, []), unsigned, size_word)
+115: integer(base_2, i(0, []), unsigned, size_8_bit)
+116: integer(base_2, i(0, []), unsigned, size_16_bit)
+117: integer(base_2, i(0, []), unsigned, size_32_bit)
+118: integer(base_2, i(0, []), unsigned, size_64_bit)
+120: string("Octal signed zeros")
+121: integer(base_8, i(0, []), signed, size_word)
+122: integer(base_8, i(0, []), signed, size_word)
+123: integer(base_8, i(0, []), signed, size_8_bit)
+124: integer(base_8, i(0, []), signed, size_16_bit)
+125: integer(base_8, i(0, []), signed, size_32_bit)
+126: integer(base_8, i(0, []), signed, size_64_bit)
+128: string("Octal signed zeros (with underscores)")
+129: integer(base_8, i(0, []), signed, size_word)
+130: integer(base_8, i(0, []), signed, size_word)
+131: integer(base_8, i(0, []), signed, size_8_bit)
+132: integer(base_8, i(0, []), signed, size_16_bit)
+133: integer(base_8, i(0, []), signed, size_32_bit)
+134: integer(base_8, i(0, []), signed, size_64_bit)
+136: string("Octal signed zeros (with leading zeros)")
+137: integer(base_8, i(0, []), signed, size_word)
+138: integer(base_8, i(0, []), signed, size_word)
+139: integer(base_8, i(0, []), signed, size_8_bit)
+140: integer(base_8, i(0, []), signed, size_16_bit)
+141: integer(base_8, i(0, []), signed, size_32_bit)
+142: integer(base_8, i(0, []), signed, size_64_bit)
+144: string("Octal unsigned zeros")
+145: integer(base_8, i(0, []), unsigned, size_word)
+146: integer(base_8, i(0, []), unsigned, size_8_bit)
+147: integer(base_8, i(0, []), unsigned, size_16_bit)
+148: integer(base_8, i(0, []), unsigned, size_32_bit)
+149: integer(base_8, i(0, []), unsigned, size_64_bit)
+151: string("Octal unsigned zeros (with underscores)")
+152: integer(base_8, i(0, []), unsigned, size_word)
+153: integer(base_8, i(0, []), unsigned, size_8_bit)
+154: integer(base_8, i(0, []), unsigned, size_16_bit)
+155: integer(base_8, i(0, []), unsigned, size_32_bit)
+156: integer(base_8, i(0, []), unsigned, size_64_bit)
+158: string("Octal unsigned zeros (with leading zeros)")
+159: integer(base_8, i(0, []), unsigned, size_word)
+160: integer(base_8, i(0, []), unsigned, size_8_bit)
+161: integer(base_8, i(0, []), unsigned, size_16_bit)
+162: integer(base_8, i(0, []), unsigned, size_32_bit)
+163: integer(base_8, i(0, []), unsigned, size_64_bit)
+165: string("Hexadecimal signed zeros")
+166: integer(base_16, i(0, []), signed, size_word)
+167: integer(base_16, i(0, []), signed, size_word)
+168: integer(base_16, i(0, []), signed, size_8_bit)
+169: integer(base_16, i(0, []), signed, size_16_bit)
+170: integer(base_16, i(0, []), signed, size_32_bit)
+171: integer(base_16, i(0, []), signed, size_64_bit)
+173: string("Hexadecimal signed zeros (with underscores)")
+174: integer(base_16, i(0, []), signed, size_word)
+175: integer(base_16, i(0, []), signed, size_word)
+176: integer(base_16, i(0, []), signed, size_8_bit)
+177: integer(base_16, i(0, []), signed, size_16_bit)
+178: integer(base_16, i(0, []), signed, size_32_bit)
+179: integer(base_16, i(0, []), signed, size_64_bit)
+181: string("Hexadecimal signed zeros (with leading zeros)")
+182: integer(base_16, i(0, []), signed, size_word)
+183: integer(base_16, i(0, []), signed, size_word)
+184: integer(base_16, i(0, []), signed, size_8_bit)
+185: integer(base_16, i(0, []), signed, size_16_bit)
+186: integer(base_16, i(0, []), signed, size_32_bit)
+187: integer(base_16, i(0, []), signed, size_64_bit)
+189: string("Hexadecimal unsigned zeros")
+190: integer(base_16, i(0, []), unsigned, size_word)
+191: integer(base_16, i(0, []), unsigned, size_8_bit)
+192: integer(base_16, i(0, []), unsigned, size_16_bit)
+193: integer(base_16, i(0, []), unsigned, size_32_bit)
+194: integer(base_16, i(0, []), unsigned, size_64_bit)
+196: string("Hexadecimal unsigned zeros (with underscores)")
+197: integer(base_16, i(0, []), unsigned, size_word)
+198: integer(base_16, i(0, []), unsigned, size_8_bit)
+199: integer(base_16, i(0, []), unsigned, size_16_bit)
+200: integer(base_16, i(0, []), unsigned, size_32_bit)
+201: integer(base_16, i(0, []), unsigned, size_64_bit)
+203: string("Hexadecimal unsigned zeros (with leading zeros)")
+204: integer(base_16, i(0, []), unsigned, size_word)
+205: integer(base_16, i(0, []), unsigned, size_8_bit)
+206: integer(base_16, i(0, []), unsigned, size_16_bit)
+207: integer(base_16, i(0, []), unsigned, size_32_bit)
+208: integer(base_16, i(0, []), unsigned, size_64_bit)
+210: string("Miscellaneous underscores")
+211: integer(base_2, i(0, []), unsigned, size_8_bit)
+212: integer(base_16, i(0, []), unsigned, size_16_bit)
+213: integer(base_8, i(0, []), signed, size_32_bit)
+214: integer(base_10, i(0, []), signed, size_64_bit)
+215: integer(base_2, i(0, []), signed, size_word)
+216: integer(base_2, i(0, []), unsigned, size_word)
-integer(base_10, i(0, []), signed, size_word)
-integer(base_10, i(2, [753, 8526]), signed, size_word)
-integer(base_10, i(1, [10]), signed, size_word)
-integer(base_10, i(1, [97]), signed, size_word)
-integer(base_10, i(1, [97]), signed, size_word)
-name("bc")
-integer(base_2, i(0, []), signed, size_word)
-integer(base_2, i(1, [1]), signed, size_word)
-integer(base_2, i(1, [3]), signed, size_word)
-integer(base_2, i(1, [7]), signed, size_word)
-integer(base_8, i(0, []), signed, size_word)
-integer(base_8, i(1, [1]), signed, size_word)
-integer(base_8, i(2, [20, 14711]), signed, size_word)
-integer(base_16, i(1, [1]), signed, size_word)
-integer(base_16, i(3, [1, 2257, 5752]), signed, size_word)
-integer(base_16, i(2, [9903, 3567]), signed, size_word)
-integer(base_16, i(2, [9903, 3567]), signed, size_word)
-float(0.123)
-float(1.23e+44)
-float(0.0)
-float(0.0)
-float(0.0)
-float(0.0)
+1: integer(base_10, i(0, []), signed, size_word)
+2: integer(base_10, i(2, [753, 8526]), signed, size_word)
+3: integer(base_10, i(1, [10]), signed, size_word)
+4: integer(base_10, i(1, [97]), signed, size_word)
+5: integer(base_10, i(1, [97]), signed, size_word)
+5: name("bc")
+7: integer(base_2, i(0, []), signed, size_word)
+8: integer(base_2, i(1, [1]), signed, size_word)
+9: integer(base_2, i(1, [3]), signed, size_word)
+10: integer(base_2, i(1, [7]), signed, size_word)
+14: integer(base_8, i(0, []), signed, size_word)
+15: integer(base_8, i(1, [1]), signed, size_word)
+16: integer(base_8, i(2, [20, 14711]), signed, size_word)
+18: integer(base_16, i(1, [1]), signed, size_word)
+19: integer(base_16, i(3, [1, 2257, 5752]), signed, size_word)
+20: integer(base_16, i(2, [9903, 3567]), signed, size_word)
+21: integer(base_16, i(2, [9903, 3567]), signed, size_word)
+25: float(0.123)
+26: float(1.23e+44)
+28: float(0.0)
+29: float(0.0)
+30: float(0.0)
+31: float(0.0)
+33: string("Decimal signed zeros")
+34: integer(base_10, i(0, []), signed, size_word)
+35: integer(base_10, i(0, []), signed, size_8_bit)
+36: integer(base_10, i(0, []), signed, size_16_bit)
+37: integer(base_10, i(0, []), signed, size_32_bit)
+38: integer(base_10, i(0, []), signed, size_64_bit)
+40: string("Decimal signed zeros (with underscores)")
+41: integer(base_10, i(0, []), signed, size_word)
+42: integer(base_10, i(0, []), signed, size_8_bit)
+43: integer(base_10, i(0, []), signed, size_16_bit)
+44: integer(base_10, i(0, []), signed, size_32_bit)
+45: integer(base_10, i(0, []), signed, size_64_bit)
+47: string("Decimal signed zeros (with leading zeros)")
+48: integer(base_10, i(0, []), signed, size_word)
+49: integer(base_10, i(0, []), signed, size_8_bit)
+50: integer(base_10, i(0, []), signed, size_16_bit)
+51: integer(base_10, i(0, []), signed, size_32_bit)
+52: integer(base_10, i(0, []), signed, size_64_bit)
+54: string("Decimal unsigned zeros")
+55: integer(base_10, i(0, []), unsigned, size_word)
+56: integer(base_10, i(0, []), unsigned, size_8_bit)
+57: integer(base_10, i(0, []), unsigned, size_16_bit)
+58: integer(base_10, i(0, []), unsigned, size_32_bit)
+59: integer(base_10, i(0, []), unsigned, size_64_bit)
+61: string("Decimal unsigned zeros (with underscores)")
+62: integer(base_10, i(0, []), unsigned, size_word)
+63: integer(base_10, i(0, []), unsigned, size_8_bit)
+64: integer(base_10, i(0, []), unsigned, size_16_bit)
+65: integer(base_10, i(0, []), unsigned, size_32_bit)
+66: integer(base_10, i(0, []), unsigned, size_64_bit)
+68: string("Decimal unsigned zeros (with leading zeros)")
+69: integer(base_10, i(0, []), unsigned, size_word)
+70: integer(base_10, i(0, []), unsigned, size_8_bit)
+71: integer(base_10, i(0, []), unsigned, size_16_bit)
+72: integer(base_10, i(0, []), unsigned, size_32_bit)
+73: integer(base_10, i(0, []), unsigned, size_64_bit)
+75: string("Binary signed zeros")
+76: integer(base_2, i(0, []), signed, size_word)
+77: integer(base_2, i(0, []), signed, size_word)
+78: integer(base_2, i(0, []), signed, size_8_bit)
+79: integer(base_2, i(0, []), signed, size_16_bit)
+80: integer(base_2, i(0, []), signed, size_32_bit)
+81: integer(base_2, i(0, []), signed, size_64_bit)
+83: string("Binary signed zeros (with underscores)")
+84: integer(base_2, i(0, []), signed, size_word)
+85: integer(base_2, i(0, []), signed, size_word)
+86: integer(base_2, i(0, []), signed, size_8_bit)
+87: integer(base_2, i(0, []), signed, size_16_bit)
+88: integer(base_2, i(0, []), signed, size_32_bit)
+89: integer(base_2, i(0, []), signed, size_64_bit)
+91: string("Binary signed zeros (with leading zeros)")
+92: integer(base_2, i(0, []), signed, size_word)
+93: integer(base_2, i(0, []), signed, size_word)
+94: integer(base_2, i(0, []), signed, size_8_bit)
+95: integer(base_2, i(0, []), signed, size_16_bit)
+96: integer(base_2, i(0, []), signed, size_32_bit)
+97: integer(base_2, i(0, []), signed, size_64_bit)
+99: string("Binary unsigned zeros")
+100: integer(base_2, i(0, []), unsigned, size_word)
+101: integer(base_2, i(0, []), unsigned, size_8_bit)
+102: integer(base_2, i(0, []), unsigned, size_16_bit)
+103: integer(base_2, i(0, []), unsigned, size_32_bit)
+104: integer(base_2, i(0, []), unsigned, size_64_bit)
+106: string("Binary unsigned zeros (with underscores)")
+107: integer(base_2, i(0, []), unsigned, size_word)
+108: integer(base_2, i(0, []), unsigned, size_8_bit)
+109: integer(base_2, i(0, []), unsigned, size_16_bit)
+110: integer(base_2, i(0, []), unsigned, size_32_bit)
+111: integer(base_2, i(0, []), unsigned, size_64_bit)
+113: string("Binary unsigned zeros (with leading zeros)")
+114: integer(base_2, i(0, []), unsigned, size_word)
+115: integer(base_2, i(0, []), unsigned, size_8_bit)
+116: integer(base_2, i(0, []), unsigned, size_16_bit)
+117: integer(base_2, i(0, []), unsigned, size_32_bit)
+118: integer(base_2, i(0, []), unsigned, size_64_bit)
+120: string("Octal signed zeros")
+121: integer(base_8, i(0, []), signed, size_word)
+122: integer(base_8, i(0, []), signed, size_word)
+123: integer(base_8, i(0, []), signed, size_8_bit)
+124: integer(base_8, i(0, []), signed, size_16_bit)
+125: integer(base_8, i(0, []), signed, size_32_bit)
+126: integer(base_8, i(0, []), signed, size_64_bit)
+128: string("Octal signed zeros (with underscores)")
+129: integer(base_8, i(0, []), signed, size_word)
+130: integer(base_8, i(0, []), signed, size_word)
+131: integer(base_8, i(0, []), signed, size_8_bit)
+132: integer(base_8, i(0, []), signed, size_16_bit)
+133: integer(base_8, i(0, []), signed, size_32_bit)
+134: integer(base_8, i(0, []), signed, size_64_bit)
+136: string("Octal signed zeros (with leading zeros)")
+137: integer(base_8, i(0, []), signed, size_word)
+138: integer(base_8, i(0, []), signed, size_word)
+139: integer(base_8, i(0, []), signed, size_8_bit)
+140: integer(base_8, i(0, []), signed, size_16_bit)
+141: integer(base_8, i(0, []), signed, size_32_bit)
+142: integer(base_8, i(0, []), signed, size_64_bit)
+144: string("Octal unsigned zeros")
+145: integer(base_8, i(0, []), unsigned, size_word)
+146: integer(base_8, i(0, []), unsigned, size_8_bit)
+147: integer(base_8, i(0, []), unsigned, size_16_bit)
+148: integer(base_8, i(0, []), unsigned, size_32_bit)
+149: integer(base_8, i(0, []), unsigned, size_64_bit)
+151: string("Octal unsigned zeros (with underscores)")
+152: integer(base_8, i(0, []), unsigned, size_word)
+153: integer(base_8, i(0, []), unsigned, size_8_bit)
+154: integer(base_8, i(0, []), unsigned, size_16_bit)
+155: integer(base_8, i(0, []), unsigned, size_32_bit)
+156: integer(base_8, i(0, []), unsigned, size_64_bit)
+158: string("Octal unsigned zeros (with leading zeros)")
+159: integer(base_8, i(0, []), unsigned, size_word)
+160: integer(base_8, i(0, []), unsigned, size_8_bit)
+161: integer(base_8, i(0, []), unsigned, size_16_bit)
+162: integer(base_8, i(0, []), unsigned, size_32_bit)
+163: integer(base_8, i(0, []), unsigned, size_64_bit)
+165: string("Hexadecimal signed zeros")
+166: integer(base_16, i(0, []), signed, size_word)
+167: integer(base_16, i(0, []), signed, size_word)
+168: integer(base_16, i(0, []), signed, size_8_bit)
+169: integer(base_16, i(0, []), signed, size_16_bit)
+170: integer(base_16, i(0, []), signed, size_32_bit)
+171: integer(base_16, i(0, []), signed, size_64_bit)
+173: string("Hexadecimal signed zeros (with underscores)")
+174: integer(base_16, i(0, []), signed, size_word)
+175: integer(base_16, i(0, []), signed, size_word)
+176: integer(base_16, i(0, []), signed, size_8_bit)
+177: integer(base_16, i(0, []), signed, size_16_bit)
+178: integer(base_16, i(0, []), signed, size_32_bit)
+179: integer(base_16, i(0, []), signed, size_64_bit)
+181: string("Hexadecimal signed zeros (with leading zeros)")
+182: integer(base_16, i(0, []), signed, size_word)
+183: integer(base_16, i(0, []), signed, size_word)
+184: integer(base_16, i(0, []), signed, size_8_bit)
+185: integer(base_16, i(0, []), signed, size_16_bit)
+186: integer(base_16, i(0, []), signed, size_32_bit)
+187: integer(base_16, i(0, []), signed, size_64_bit)
+189: string("Hexadecimal unsigned zeros")
+190: integer(base_16, i(0, []), unsigned, size_word)
+191: integer(base_16, i(0, []), unsigned, size_8_bit)
+192: integer(base_16, i(0, []), unsigned, size_16_bit)
+193: integer(base_16, i(0, []), unsigned, size_32_bit)
+194: integer(base_16, i(0, []), unsigned, size_64_bit)
+196: string("Hexadecimal unsigned zeros (with underscores)")
+197: integer(base_16, i(0, []), unsigned, size_word)
+198: integer(base_16, i(0, []), unsigned, size_8_bit)
+199: integer(base_16, i(0, []), unsigned, size_16_bit)
+200: integer(base_16, i(0, []), unsigned, size_32_bit)
+201: integer(base_16, i(0, []), unsigned, size_64_bit)
+203: string("Hexadecimal unsigned zeros (with leading zeros)")
+204: integer(base_16, i(0, []), unsigned, size_word)
+205: integer(base_16, i(0, []), unsigned, size_8_bit)
+206: integer(base_16, i(0, []), unsigned, size_16_bit)
+207: integer(base_16, i(0, []), unsigned, size_32_bit)
+208: integer(base_16, i(0, []), unsigned, size_64_bit)
+210: string("Miscellaneous underscores")
+211: integer(base_2, i(0, []), unsigned, size_8_bit)
+212: integer(base_16, i(0, []), unsigned, size_16_bit)
+213: integer(base_8, i(0, []), signed, size_32_bit)
+214: integer(base_10, i(0, []), signed, size_64_bit)
+215: integer(base_2, i(0, []), signed, size_word)
+216: integer(base_2, i(0, []), unsigned, size_word)
diff --git a/tests/hard_coded/lexer_zero.exp2 b/tests/hard_coded/lexer_zero.exp2
index 2f27022..a1cf369 100644
--- a/tests/hard_coded/lexer_zero.exp2
+++ b/tests/hard_coded/lexer_zero.exp2
@@ -1,47 +1,367 @@
-integer(base_10, i(0, []), signed, size_word)
-integer(base_10, i(2, [753, 8526]), signed, size_word)
-integer(base_10, i(1, [10]), signed, size_word)
-integer(base_10, i(1, [97]), signed, size_word)
-integer(base_10, i(1, [97]), signed, size_word)
-name("bc")
-integer(base_2, i(0, []), signed, size_word)
-integer(base_2, i(1, [1]), signed, size_word)
-integer(base_2, i(1, [3]), signed, size_word)
-integer(base_2, i(1, [7]), signed, size_word)
-integer(base_8, i(0, []), signed, size_word)
-integer(base_8, i(1, [1]), signed, size_word)
-integer(base_8, i(2, [20, 14711]), signed, size_word)
-integer(base_16, i(1, [1]), signed, size_word)
-integer(base_16, i(3, [1, 2257, 5752]), signed, size_word)
-integer(base_16, i(2, [9903, 3567]), signed, size_word)
-integer(base_16, i(2, [9903, 3567]), signed, size_word)
-float(0.123)
-float(1.23E44)
-float(0.0)
-float(0.0)
-float(0.0)
-float(0.0)
+1: integer(base_10, i(0, []), signed, size_word)
+2: integer(base_10, i(2, [753, 8526]), signed, size_word)
+3: integer(base_10, i(1, [10]), signed, size_word)
+4: integer(base_10, i(1, [97]), signed, size_word)
+5: integer(base_10, i(1, [97]), signed, size_word)
+5: name("bc")
+7: integer(base_2, i(0, []), signed, size_word)
+8: integer(base_2, i(1, [1]), signed, size_word)
+9: integer(base_2, i(1, [3]), signed, size_word)
+10: integer(base_2, i(1, [7]), signed, size_word)
+14: integer(base_8, i(0, []), signed, size_word)
+15: integer(base_8, i(1, [1]), signed, size_word)
+16: integer(base_8, i(2, [20, 14711]), signed, size_word)
+18: integer(base_16, i(1, [1]), signed, size_word)
+19: integer(base_16, i(3, [1, 2257, 5752]), signed, size_word)
+20: integer(base_16, i(2, [9903, 3567]), signed, size_word)
+21: integer(base_16, i(2, [9903, 3567]), signed, size_word)
+25: float(0.123)
+26: float(1.23E44)
+28: float(0.0)
+29: float(0.0)
+30: float(0.0)
+31: float(0.0)
+33: string("Decimal signed zeros")
+34: integer(base_10, i(0, []), signed, size_word)
+35: integer(base_10, i(0, []), signed, size_8_bit)
+36: integer(base_10, i(0, []), signed, size_16_bit)
+37: integer(base_10, i(0, []), signed, size_32_bit)
+38: integer(base_10, i(0, []), signed, size_64_bit)
+40: string("Decimal signed zeros (with underscores)")
+41: integer(base_10, i(0, []), signed, size_word)
+42: integer(base_10, i(0, []), signed, size_8_bit)
+43: integer(base_10, i(0, []), signed, size_16_bit)
+44: integer(base_10, i(0, []), signed, size_32_bit)
+45: integer(base_10, i(0, []), signed, size_64_bit)
+47: string("Decimal signed zeros (with leading zeros)")
+48: integer(base_10, i(0, []), signed, size_word)
+49: integer(base_10, i(0, []), signed, size_8_bit)
+50: integer(base_10, i(0, []), signed, size_16_bit)
+51: integer(base_10, i(0, []), signed, size_32_bit)
+52: integer(base_10, i(0, []), signed, size_64_bit)
+54: string("Decimal unsigned zeros")
+55: integer(base_10, i(0, []), unsigned, size_word)
+56: integer(base_10, i(0, []), unsigned, size_8_bit)
+57: integer(base_10, i(0, []), unsigned, size_16_bit)
+58: integer(base_10, i(0, []), unsigned, size_32_bit)
+59: integer(base_10, i(0, []), unsigned, size_64_bit)
+61: string("Decimal unsigned zeros (with underscores)")
+62: integer(base_10, i(0, []), unsigned, size_word)
+63: integer(base_10, i(0, []), unsigned, size_8_bit)
+64: integer(base_10, i(0, []), unsigned, size_16_bit)
+65: integer(base_10, i(0, []), unsigned, size_32_bit)
+66: integer(base_10, i(0, []), unsigned, size_64_bit)
+68: string("Decimal unsigned zeros (with leading zeros)")
+69: integer(base_10, i(0, []), unsigned, size_word)
+70: integer(base_10, i(0, []), unsigned, size_8_bit)
+71: integer(base_10, i(0, []), unsigned, size_16_bit)
+72: integer(base_10, i(0, []), unsigned, size_32_bit)
+73: integer(base_10, i(0, []), unsigned, size_64_bit)
+75: string("Binary signed zeros")
+76: integer(base_2, i(0, []), signed, size_word)
+77: integer(base_2, i(0, []), signed, size_word)
+78: integer(base_2, i(0, []), signed, size_8_bit)
+79: integer(base_2, i(0, []), signed, size_16_bit)
+80: integer(base_2, i(0, []), signed, size_32_bit)
+81: integer(base_2, i(0, []), signed, size_64_bit)
+83: string("Binary signed zeros (with underscores)")
+84: integer(base_2, i(0, []), signed, size_word)
+85: integer(base_2, i(0, []), signed, size_word)
+86: integer(base_2, i(0, []), signed, size_8_bit)
+87: integer(base_2, i(0, []), signed, size_16_bit)
+88: integer(base_2, i(0, []), signed, size_32_bit)
+89: integer(base_2, i(0, []), signed, size_64_bit)
+91: string("Binary signed zeros (with leading zeros)")
+92: integer(base_2, i(0, []), signed, size_word)
+93: integer(base_2, i(0, []), signed, size_word)
+94: integer(base_2, i(0, []), signed, size_8_bit)
+95: integer(base_2, i(0, []), signed, size_16_bit)
+96: integer(base_2, i(0, []), signed, size_32_bit)
+97: integer(base_2, i(0, []), signed, size_64_bit)
+99: string("Binary unsigned zeros")
+100: integer(base_2, i(0, []), unsigned, size_word)
+101: integer(base_2, i(0, []), unsigned, size_8_bit)
+102: integer(base_2, i(0, []), unsigned, size_16_bit)
+103: integer(base_2, i(0, []), unsigned, size_32_bit)
+104: integer(base_2, i(0, []), unsigned, size_64_bit)
+106: string("Binary unsigned zeros (with underscores)")
+107: integer(base_2, i(0, []), unsigned, size_word)
+108: integer(base_2, i(0, []), unsigned, size_8_bit)
+109: integer(base_2, i(0, []), unsigned, size_16_bit)
+110: integer(base_2, i(0, []), unsigned, size_32_bit)
+111: integer(base_2, i(0, []), unsigned, size_64_bit)
+113: string("Binary unsigned zeros (with leading zeros)")
+114: integer(base_2, i(0, []), unsigned, size_word)
+115: integer(base_2, i(0, []), unsigned, size_8_bit)
+116: integer(base_2, i(0, []), unsigned, size_16_bit)
+117: integer(base_2, i(0, []), unsigned, size_32_bit)
+118: integer(base_2, i(0, []), unsigned, size_64_bit)
+120: string("Octal signed zeros")
+121: integer(base_8, i(0, []), signed, size_word)
+122: integer(base_8, i(0, []), signed, size_word)
+123: integer(base_8, i(0, []), signed, size_8_bit)
+124: integer(base_8, i(0, []), signed, size_16_bit)
+125: integer(base_8, i(0, []), signed, size_32_bit)
+126: integer(base_8, i(0, []), signed, size_64_bit)
+128: string("Octal signed zeros (with underscores)")
+129: integer(base_8, i(0, []), signed, size_word)
+130: integer(base_8, i(0, []), signed, size_word)
+131: integer(base_8, i(0, []), signed, size_8_bit)
+132: integer(base_8, i(0, []), signed, size_16_bit)
+133: integer(base_8, i(0, []), signed, size_32_bit)
+134: integer(base_8, i(0, []), signed, size_64_bit)
+136: string("Octal signed zeros (with leading zeros)")
+137: integer(base_8, i(0, []), signed, size_word)
+138: integer(base_8, i(0, []), signed, size_word)
+139: integer(base_8, i(0, []), signed, size_8_bit)
+140: integer(base_8, i(0, []), signed, size_16_bit)
+141: integer(base_8, i(0, []), signed, size_32_bit)
+142: integer(base_8, i(0, []), signed, size_64_bit)
+144: string("Octal unsigned zeros")
+145: integer(base_8, i(0, []), unsigned, size_word)
+146: integer(base_8, i(0, []), unsigned, size_8_bit)
+147: integer(base_8, i(0, []), unsigned, size_16_bit)
+148: integer(base_8, i(0, []), unsigned, size_32_bit)
+149: integer(base_8, i(0, []), unsigned, size_64_bit)
+151: string("Octal unsigned zeros (with underscores)")
+152: integer(base_8, i(0, []), unsigned, size_word)
+153: integer(base_8, i(0, []), unsigned, size_8_bit)
+154: integer(base_8, i(0, []), unsigned, size_16_bit)
+155: integer(base_8, i(0, []), unsigned, size_32_bit)
+156: integer(base_8, i(0, []), unsigned, size_64_bit)
+158: string("Octal unsigned zeros (with leading zeros)")
+159: integer(base_8, i(0, []), unsigned, size_word)
+160: integer(base_8, i(0, []), unsigned, size_8_bit)
+161: integer(base_8, i(0, []), unsigned, size_16_bit)
+162: integer(base_8, i(0, []), unsigned, size_32_bit)
+163: integer(base_8, i(0, []), unsigned, size_64_bit)
+165: string("Hexadecimal signed zeros")
+166: integer(base_16, i(0, []), signed, size_word)
+167: integer(base_16, i(0, []), signed, size_word)
+168: integer(base_16, i(0, []), signed, size_8_bit)
+169: integer(base_16, i(0, []), signed, size_16_bit)
+170: integer(base_16, i(0, []), signed, size_32_bit)
+171: integer(base_16, i(0, []), signed, size_64_bit)
+173: string("Hexadecimal signed zeros (with underscores)")
+174: integer(base_16, i(0, []), signed, size_word)
+175: integer(base_16, i(0, []), signed, size_word)
+176: integer(base_16, i(0, []), signed, size_8_bit)
+177: integer(base_16, i(0, []), signed, size_16_bit)
+178: integer(base_16, i(0, []), signed, size_32_bit)
+179: integer(base_16, i(0, []), signed, size_64_bit)
+181: string("Hexadecimal signed zeros (with leading zeros)")
+182: integer(base_16, i(0, []), signed, size_word)
+183: integer(base_16, i(0, []), signed, size_word)
+184: integer(base_16, i(0, []), signed, size_8_bit)
+185: integer(base_16, i(0, []), signed, size_16_bit)
+186: integer(base_16, i(0, []), signed, size_32_bit)
+187: integer(base_16, i(0, []), signed, size_64_bit)
+189: string("Hexadecimal unsigned zeros")
+190: integer(base_16, i(0, []), unsigned, size_word)
+191: integer(base_16, i(0, []), unsigned, size_8_bit)
+192: integer(base_16, i(0, []), unsigned, size_16_bit)
+193: integer(base_16, i(0, []), unsigned, size_32_bit)
+194: integer(base_16, i(0, []), unsigned, size_64_bit)
+196: string("Hexadecimal unsigned zeros (with underscores)")
+197: integer(base_16, i(0, []), unsigned, size_word)
+198: integer(base_16, i(0, []), unsigned, size_8_bit)
+199: integer(base_16, i(0, []), unsigned, size_16_bit)
+200: integer(base_16, i(0, []), unsigned, size_32_bit)
+201: integer(base_16, i(0, []), unsigned, size_64_bit)
+203: string("Hexadecimal unsigned zeros (with leading zeros)")
+204: integer(base_16, i(0, []), unsigned, size_word)
+205: integer(base_16, i(0, []), unsigned, size_8_bit)
+206: integer(base_16, i(0, []), unsigned, size_16_bit)
+207: integer(base_16, i(0, []), unsigned, size_32_bit)
+208: integer(base_16, i(0, []), unsigned, size_64_bit)
+210: string("Miscellaneous underscores")
+211: integer(base_2, i(0, []), unsigned, size_8_bit)
+212: integer(base_16, i(0, []), unsigned, size_16_bit)
+213: integer(base_8, i(0, []), signed, size_32_bit)
+214: integer(base_10, i(0, []), signed, size_64_bit)
+215: integer(base_2, i(0, []), signed, size_word)
+216: integer(base_2, i(0, []), unsigned, size_word)
-integer(base_10, i(0, []), signed, size_word)
-integer(base_10, i(2, [753, 8526]), signed, size_word)
-integer(base_10, i(1, [10]), signed, size_word)
-integer(base_10, i(1, [97]), signed, size_word)
-integer(base_10, i(1, [97]), signed, size_word)
-name("bc")
-integer(base_2, i(0, []), signed, size_word)
-integer(base_2, i(1, [1]), signed, size_word)
-integer(base_2, i(1, [3]), signed, size_word)
-integer(base_2, i(1, [7]), signed, size_word)
-integer(base_8, i(0, []), signed, size_word)
-integer(base_8, i(1, [1]), signed, size_word)
-integer(base_8, i(2, [20, 14711]), signed, size_word)
-integer(base_16, i(1, [1]), signed, size_word)
-integer(base_16, i(3, [1, 2257, 5752]), signed, size_word)
-integer(base_16, i(2, [9903, 3567]), signed, size_word)
-integer(base_16, i(2, [9903, 3567]), signed, size_word)
-float(0.123)
-float(1.23E44)
-float(0.0)
-float(0.0)
-float(0.0)
-float(0.0)
+1: integer(base_10, i(0, []), signed, size_word)
+2: integer(base_10, i(2, [753, 8526]), signed, size_word)
+3: integer(base_10, i(1, [10]), signed, size_word)
+4: integer(base_10, i(1, [97]), signed, size_word)
+5: integer(base_10, i(1, [97]), signed, size_word)
+5: name("bc")
+7: integer(base_2, i(0, []), signed, size_word)
+8: integer(base_2, i(1, [1]), signed, size_word)
+9: integer(base_2, i(1, [3]), signed, size_word)
+10: integer(base_2, i(1, [7]), signed, size_word)
+14: integer(base_8, i(0, []), signed, size_word)
+15: integer(base_8, i(1, [1]), signed, size_word)
+16: integer(base_8, i(2, [20, 14711]), signed, size_word)
+18: integer(base_16, i(1, [1]), signed, size_word)
+19: integer(base_16, i(3, [1, 2257, 5752]), signed, size_word)
+20: integer(base_16, i(2, [9903, 3567]), signed, size_word)
+21: integer(base_16, i(2, [9903, 3567]), signed, size_word)
+25: float(0.123)
+26: float(1.23E44)
+28: float(0.0)
+29: float(0.0)
+30: float(0.0)
+31: float(0.0)
+33: string("Decimal signed zeros")
+34: integer(base_10, i(0, []), signed, size_word)
+35: integer(base_10, i(0, []), signed, size_8_bit)
+36: integer(base_10, i(0, []), signed, size_16_bit)
+37: integer(base_10, i(0, []), signed, size_32_bit)
+38: integer(base_10, i(0, []), signed, size_64_bit)
+40: string("Decimal signed zeros (with underscores)")
+41: integer(base_10, i(0, []), signed, size_word)
+42: integer(base_10, i(0, []), signed, size_8_bit)
+43: integer(base_10, i(0, []), signed, size_16_bit)
+44: integer(base_10, i(0, []), signed, size_32_bit)
+45: integer(base_10, i(0, []), signed, size_64_bit)
+47: string("Decimal signed zeros (with leading zeros)")
+48: integer(base_10, i(0, []), signed, size_word)
+49: integer(base_10, i(0, []), signed, size_8_bit)
+50: integer(base_10, i(0, []), signed, size_16_bit)
+51: integer(base_10, i(0, []), signed, size_32_bit)
+52: integer(base_10, i(0, []), signed, size_64_bit)
+54: string("Decimal unsigned zeros")
+55: integer(base_10, i(0, []), unsigned, size_word)
+56: integer(base_10, i(0, []), unsigned, size_8_bit)
+57: integer(base_10, i(0, []), unsigned, size_16_bit)
+58: integer(base_10, i(0, []), unsigned, size_32_bit)
+59: integer(base_10, i(0, []), unsigned, size_64_bit)
+61: string("Decimal unsigned zeros (with underscores)")
+62: integer(base_10, i(0, []), unsigned, size_word)
+63: integer(base_10, i(0, []), unsigned, size_8_bit)
+64: integer(base_10, i(0, []), unsigned, size_16_bit)
+65: integer(base_10, i(0, []), unsigned, size_32_bit)
+66: integer(base_10, i(0, []), unsigned, size_64_bit)
+68: string("Decimal unsigned zeros (with leading zeros)")
+69: integer(base_10, i(0, []), unsigned, size_word)
+70: integer(base_10, i(0, []), unsigned, size_8_bit)
+71: integer(base_10, i(0, []), unsigned, size_16_bit)
+72: integer(base_10, i(0, []), unsigned, size_32_bit)
+73: integer(base_10, i(0, []), unsigned, size_64_bit)
+75: string("Binary signed zeros")
+76: integer(base_2, i(0, []), signed, size_word)
+77: integer(base_2, i(0, []), signed, size_word)
+78: integer(base_2, i(0, []), signed, size_8_bit)
+79: integer(base_2, i(0, []), signed, size_16_bit)
+80: integer(base_2, i(0, []), signed, size_32_bit)
+81: integer(base_2, i(0, []), signed, size_64_bit)
+83: string("Binary signed zeros (with underscores)")
+84: integer(base_2, i(0, []), signed, size_word)
+85: integer(base_2, i(0, []), signed, size_word)
+86: integer(base_2, i(0, []), signed, size_8_bit)
+87: integer(base_2, i(0, []), signed, size_16_bit)
+88: integer(base_2, i(0, []), signed, size_32_bit)
+89: integer(base_2, i(0, []), signed, size_64_bit)
+91: string("Binary signed zeros (with leading zeros)")
+92: integer(base_2, i(0, []), signed, size_word)
+93: integer(base_2, i(0, []), signed, size_word)
+94: integer(base_2, i(0, []), signed, size_8_bit)
+95: integer(base_2, i(0, []), signed, size_16_bit)
+96: integer(base_2, i(0, []), signed, size_32_bit)
+97: integer(base_2, i(0, []), signed, size_64_bit)
+99: string("Binary unsigned zeros")
+100: integer(base_2, i(0, []), unsigned, size_word)
+101: integer(base_2, i(0, []), unsigned, size_8_bit)
+102: integer(base_2, i(0, []), unsigned, size_16_bit)
+103: integer(base_2, i(0, []), unsigned, size_32_bit)
+104: integer(base_2, i(0, []), unsigned, size_64_bit)
+106: string("Binary unsigned zeros (with underscores)")
+107: integer(base_2, i(0, []), unsigned, size_word)
+108: integer(base_2, i(0, []), unsigned, size_8_bit)
+109: integer(base_2, i(0, []), unsigned, size_16_bit)
+110: integer(base_2, i(0, []), unsigned, size_32_bit)
+111: integer(base_2, i(0, []), unsigned, size_64_bit)
+113: string("Binary unsigned zeros (with leading zeros)")
+114: integer(base_2, i(0, []), unsigned, size_word)
+115: integer(base_2, i(0, []), unsigned, size_8_bit)
+116: integer(base_2, i(0, []), unsigned, size_16_bit)
+117: integer(base_2, i(0, []), unsigned, size_32_bit)
+118: integer(base_2, i(0, []), unsigned, size_64_bit)
+120: string("Octal signed zeros")
+121: integer(base_8, i(0, []), signed, size_word)
+122: integer(base_8, i(0, []), signed, size_word)
+123: integer(base_8, i(0, []), signed, size_8_bit)
+124: integer(base_8, i(0, []), signed, size_16_bit)
+125: integer(base_8, i(0, []), signed, size_32_bit)
+126: integer(base_8, i(0, []), signed, size_64_bit)
+128: string("Octal signed zeros (with underscores)")
+129: integer(base_8, i(0, []), signed, size_word)
+130: integer(base_8, i(0, []), signed, size_word)
+131: integer(base_8, i(0, []), signed, size_8_bit)
+132: integer(base_8, i(0, []), signed, size_16_bit)
+133: integer(base_8, i(0, []), signed, size_32_bit)
+134: integer(base_8, i(0, []), signed, size_64_bit)
+136: string("Octal signed zeros (with leading zeros)")
+137: integer(base_8, i(0, []), signed, size_word)
+138: integer(base_8, i(0, []), signed, size_word)
+139: integer(base_8, i(0, []), signed, size_8_bit)
+140: integer(base_8, i(0, []), signed, size_16_bit)
+141: integer(base_8, i(0, []), signed, size_32_bit)
+142: integer(base_8, i(0, []), signed, size_64_bit)
+144: string("Octal unsigned zeros")
+145: integer(base_8, i(0, []), unsigned, size_word)
+146: integer(base_8, i(0, []), unsigned, size_8_bit)
+147: integer(base_8, i(0, []), unsigned, size_16_bit)
+148: integer(base_8, i(0, []), unsigned, size_32_bit)
+149: integer(base_8, i(0, []), unsigned, size_64_bit)
+151: string("Octal unsigned zeros (with underscores)")
+152: integer(base_8, i(0, []), unsigned, size_word)
+153: integer(base_8, i(0, []), unsigned, size_8_bit)
+154: integer(base_8, i(0, []), unsigned, size_16_bit)
+155: integer(base_8, i(0, []), unsigned, size_32_bit)
+156: integer(base_8, i(0, []), unsigned, size_64_bit)
+158: string("Octal unsigned zeros (with leading zeros)")
+159: integer(base_8, i(0, []), unsigned, size_word)
+160: integer(base_8, i(0, []), unsigned, size_8_bit)
+161: integer(base_8, i(0, []), unsigned, size_16_bit)
+162: integer(base_8, i(0, []), unsigned, size_32_bit)
+163: integer(base_8, i(0, []), unsigned, size_64_bit)
+165: string("Hexadecimal signed zeros")
+166: integer(base_16, i(0, []), signed, size_word)
+167: integer(base_16, i(0, []), signed, size_word)
+168: integer(base_16, i(0, []), signed, size_8_bit)
+169: integer(base_16, i(0, []), signed, size_16_bit)
+170: integer(base_16, i(0, []), signed, size_32_bit)
+171: integer(base_16, i(0, []), signed, size_64_bit)
+173: string("Hexadecimal signed zeros (with underscores)")
+174: integer(base_16, i(0, []), signed, size_word)
+175: integer(base_16, i(0, []), signed, size_word)
+176: integer(base_16, i(0, []), signed, size_8_bit)
+177: integer(base_16, i(0, []), signed, size_16_bit)
+178: integer(base_16, i(0, []), signed, size_32_bit)
+179: integer(base_16, i(0, []), signed, size_64_bit)
+181: string("Hexadecimal signed zeros (with leading zeros)")
+182: integer(base_16, i(0, []), signed, size_word)
+183: integer(base_16, i(0, []), signed, size_word)
+184: integer(base_16, i(0, []), signed, size_8_bit)
+185: integer(base_16, i(0, []), signed, size_16_bit)
+186: integer(base_16, i(0, []), signed, size_32_bit)
+187: integer(base_16, i(0, []), signed, size_64_bit)
+189: string("Hexadecimal unsigned zeros")
+190: integer(base_16, i(0, []), unsigned, size_word)
+191: integer(base_16, i(0, []), unsigned, size_8_bit)
+192: integer(base_16, i(0, []), unsigned, size_16_bit)
+193: integer(base_16, i(0, []), unsigned, size_32_bit)
+194: integer(base_16, i(0, []), unsigned, size_64_bit)
+196: string("Hexadecimal unsigned zeros (with underscores)")
+197: integer(base_16, i(0, []), unsigned, size_word)
+198: integer(base_16, i(0, []), unsigned, size_8_bit)
+199: integer(base_16, i(0, []), unsigned, size_16_bit)
+200: integer(base_16, i(0, []), unsigned, size_32_bit)
+201: integer(base_16, i(0, []), unsigned, size_64_bit)
+203: string("Hexadecimal unsigned zeros (with leading zeros)")
+204: integer(base_16, i(0, []), unsigned, size_word)
+205: integer(base_16, i(0, []), unsigned, size_8_bit)
+206: integer(base_16, i(0, []), unsigned, size_16_bit)
+207: integer(base_16, i(0, []), unsigned, size_32_bit)
+208: integer(base_16, i(0, []), unsigned, size_64_bit)
+210: string("Miscellaneous underscores")
+211: integer(base_2, i(0, []), unsigned, size_8_bit)
+212: integer(base_16, i(0, []), unsigned, size_16_bit)
+213: integer(base_8, i(0, []), signed, size_32_bit)
+214: integer(base_10, i(0, []), signed, size_64_bit)
+215: integer(base_2, i(0, []), signed, size_word)
+216: integer(base_2, i(0, []), unsigned, size_word)
diff --git a/tests/hard_coded/lexer_zero.exp3 b/tests/hard_coded/lexer_zero.exp3
index 399d836..6073a2f 100644
--- a/tests/hard_coded/lexer_zero.exp3
+++ b/tests/hard_coded/lexer_zero.exp3
@@ -1,47 +1,367 @@
-integer(base_10, i(0, []), signed, size_word)
-integer(base_10, i(2, [753, 8526]), signed, size_word)
-integer(base_10, i(1, [10]), signed, size_word)
-integer(base_10, i(1, [97]), signed, size_word)
-integer(base_10, i(1, [97]), signed, size_word)
-name("bc")
-integer(base_2, i(0, []), signed, size_word)
-integer(base_2, i(1, [1]), signed, size_word)
-integer(base_2, i(1, [3]), signed, size_word)
-integer(base_2, i(1, [7]), signed, size_word)
-integer(base_8, i(0, []), signed, size_word)
-integer(base_8, i(1, [1]), signed, size_word)
-integer(base_8, i(2, [20, 14711]), signed, size_word)
-integer(base_16, i(1, [1]), signed, size_word)
-integer(base_16, i(3, [1, 2257, 5752]), signed, size_word)
-integer(base_16, i(2, [9903, 3567]), signed, size_word)
-integer(base_16, i(2, [9903, 3567]), signed, size_word)
-float(0.123)
-float(1.23E+44)
-float(0.0)
-float(0.0)
-float(0.0)
-float(0.0)
+1: integer(base_10, i(0, []), signed, size_word)
+2: integer(base_10, i(2, [753, 8526]), signed, size_word)
+3: integer(base_10, i(1, [10]), signed, size_word)
+4: integer(base_10, i(1, [97]), signed, size_word)
+5: integer(base_10, i(1, [97]), signed, size_word)
+5: name("bc")
+7: integer(base_2, i(0, []), signed, size_word)
+8: integer(base_2, i(1, [1]), signed, size_word)
+9: integer(base_2, i(1, [3]), signed, size_word)
+10: integer(base_2, i(1, [7]), signed, size_word)
+14: integer(base_8, i(0, []), signed, size_word)
+15: integer(base_8, i(1, [1]), signed, size_word)
+16: integer(base_8, i(2, [20, 14711]), signed, size_word)
+18: integer(base_16, i(1, [1]), signed, size_word)
+19: integer(base_16, i(3, [1, 2257, 5752]), signed, size_word)
+20: integer(base_16, i(2, [9903, 3567]), signed, size_word)
+21: integer(base_16, i(2, [9903, 3567]), signed, size_word)
+25: float(0.123)
+26: float(1.23E+44)
+28: float(0.0)
+29: float(0.0)
+30: float(0.0)
+31: float(0.0)
+33: string("Decimal signed zeros")
+34: integer(base_10, i(0, []), signed, size_word)
+35: integer(base_10, i(0, []), signed, size_8_bit)
+36: integer(base_10, i(0, []), signed, size_16_bit)
+37: integer(base_10, i(0, []), signed, size_32_bit)
+38: integer(base_10, i(0, []), signed, size_64_bit)
+40: string("Decimal signed zeros (with underscores)")
+41: integer(base_10, i(0, []), signed, size_word)
+42: integer(base_10, i(0, []), signed, size_8_bit)
+43: integer(base_10, i(0, []), signed, size_16_bit)
+44: integer(base_10, i(0, []), signed, size_32_bit)
+45: integer(base_10, i(0, []), signed, size_64_bit)
+47: string("Decimal signed zeros (with leading zeros)")
+48: integer(base_10, i(0, []), signed, size_word)
+49: integer(base_10, i(0, []), signed, size_8_bit)
+50: integer(base_10, i(0, []), signed, size_16_bit)
+51: integer(base_10, i(0, []), signed, size_32_bit)
+52: integer(base_10, i(0, []), signed, size_64_bit)
+54: string("Decimal unsigned zeros")
+55: integer(base_10, i(0, []), unsigned, size_word)
+56: integer(base_10, i(0, []), unsigned, size_8_bit)
+57: integer(base_10, i(0, []), unsigned, size_16_bit)
+58: integer(base_10, i(0, []), unsigned, size_32_bit)
+59: integer(base_10, i(0, []), unsigned, size_64_bit)
+61: string("Decimal unsigned zeros (with underscores)")
+62: integer(base_10, i(0, []), unsigned, size_word)
+63: integer(base_10, i(0, []), unsigned, size_8_bit)
+64: integer(base_10, i(0, []), unsigned, size_16_bit)
+65: integer(base_10, i(0, []), unsigned, size_32_bit)
+66: integer(base_10, i(0, []), unsigned, size_64_bit)
+68: string("Decimal unsigned zeros (with leading zeros)")
+69: integer(base_10, i(0, []), unsigned, size_word)
+70: integer(base_10, i(0, []), unsigned, size_8_bit)
+71: integer(base_10, i(0, []), unsigned, size_16_bit)
+72: integer(base_10, i(0, []), unsigned, size_32_bit)
+73: integer(base_10, i(0, []), unsigned, size_64_bit)
+75: string("Binary signed zeros")
+76: integer(base_2, i(0, []), signed, size_word)
+77: integer(base_2, i(0, []), signed, size_word)
+78: integer(base_2, i(0, []), signed, size_8_bit)
+79: integer(base_2, i(0, []), signed, size_16_bit)
+80: integer(base_2, i(0, []), signed, size_32_bit)
+81: integer(base_2, i(0, []), signed, size_64_bit)
+83: string("Binary signed zeros (with underscores)")
+84: integer(base_2, i(0, []), signed, size_word)
+85: integer(base_2, i(0, []), signed, size_word)
+86: integer(base_2, i(0, []), signed, size_8_bit)
+87: integer(base_2, i(0, []), signed, size_16_bit)
+88: integer(base_2, i(0, []), signed, size_32_bit)
+89: integer(base_2, i(0, []), signed, size_64_bit)
+91: string("Binary signed zeros (with leading zeros)")
+92: integer(base_2, i(0, []), signed, size_word)
+93: integer(base_2, i(0, []), signed, size_word)
+94: integer(base_2, i(0, []), signed, size_8_bit)
+95: integer(base_2, i(0, []), signed, size_16_bit)
+96: integer(base_2, i(0, []), signed, size_32_bit)
+97: integer(base_2, i(0, []), signed, size_64_bit)
+99: string("Binary unsigned zeros")
+100: integer(base_2, i(0, []), unsigned, size_word)
+101: integer(base_2, i(0, []), unsigned, size_8_bit)
+102: integer(base_2, i(0, []), unsigned, size_16_bit)
+103: integer(base_2, i(0, []), unsigned, size_32_bit)
+104: integer(base_2, i(0, []), unsigned, size_64_bit)
+106: string("Binary unsigned zeros (with underscores)")
+107: integer(base_2, i(0, []), unsigned, size_word)
+108: integer(base_2, i(0, []), unsigned, size_8_bit)
+109: integer(base_2, i(0, []), unsigned, size_16_bit)
+110: integer(base_2, i(0, []), unsigned, size_32_bit)
+111: integer(base_2, i(0, []), unsigned, size_64_bit)
+113: string("Binary unsigned zeros (with leading zeros)")
+114: integer(base_2, i(0, []), unsigned, size_word)
+115: integer(base_2, i(0, []), unsigned, size_8_bit)
+116: integer(base_2, i(0, []), unsigned, size_16_bit)
+117: integer(base_2, i(0, []), unsigned, size_32_bit)
+118: integer(base_2, i(0, []), unsigned, size_64_bit)
+120: string("Octal signed zeros")
+121: integer(base_8, i(0, []), signed, size_word)
+122: integer(base_8, i(0, []), signed, size_word)
+123: integer(base_8, i(0, []), signed, size_8_bit)
+124: integer(base_8, i(0, []), signed, size_16_bit)
+125: integer(base_8, i(0, []), signed, size_32_bit)
+126: integer(base_8, i(0, []), signed, size_64_bit)
+128: string("Octal signed zeros (with underscores)")
+129: integer(base_8, i(0, []), signed, size_word)
+130: integer(base_8, i(0, []), signed, size_word)
+131: integer(base_8, i(0, []), signed, size_8_bit)
+132: integer(base_8, i(0, []), signed, size_16_bit)
+133: integer(base_8, i(0, []), signed, size_32_bit)
+134: integer(base_8, i(0, []), signed, size_64_bit)
+136: string("Octal signed zeros (with leading zeros)")
+137: integer(base_8, i(0, []), signed, size_word)
+138: integer(base_8, i(0, []), signed, size_word)
+139: integer(base_8, i(0, []), signed, size_8_bit)
+140: integer(base_8, i(0, []), signed, size_16_bit)
+141: integer(base_8, i(0, []), signed, size_32_bit)
+142: integer(base_8, i(0, []), signed, size_64_bit)
+144: string("Octal unsigned zeros")
+145: integer(base_8, i(0, []), unsigned, size_word)
+146: integer(base_8, i(0, []), unsigned, size_8_bit)
+147: integer(base_8, i(0, []), unsigned, size_16_bit)
+148: integer(base_8, i(0, []), unsigned, size_32_bit)
+149: integer(base_8, i(0, []), unsigned, size_64_bit)
+151: string("Octal unsigned zeros (with underscores)")
+152: integer(base_8, i(0, []), unsigned, size_word)
+153: integer(base_8, i(0, []), unsigned, size_8_bit)
+154: integer(base_8, i(0, []), unsigned, size_16_bit)
+155: integer(base_8, i(0, []), unsigned, size_32_bit)
+156: integer(base_8, i(0, []), unsigned, size_64_bit)
+158: string("Octal unsigned zeros (with leading zeros)")
+159: integer(base_8, i(0, []), unsigned, size_word)
+160: integer(base_8, i(0, []), unsigned, size_8_bit)
+161: integer(base_8, i(0, []), unsigned, size_16_bit)
+162: integer(base_8, i(0, []), unsigned, size_32_bit)
+163: integer(base_8, i(0, []), unsigned, size_64_bit)
+165: string("Hexadecimal signed zeros")
+166: integer(base_16, i(0, []), signed, size_word)
+167: integer(base_16, i(0, []), signed, size_word)
+168: integer(base_16, i(0, []), signed, size_8_bit)
+169: integer(base_16, i(0, []), signed, size_16_bit)
+170: integer(base_16, i(0, []), signed, size_32_bit)
+171: integer(base_16, i(0, []), signed, size_64_bit)
+173: string("Hexadecimal signed zeros (with underscores)")
+174: integer(base_16, i(0, []), signed, size_word)
+175: integer(base_16, i(0, []), signed, size_word)
+176: integer(base_16, i(0, []), signed, size_8_bit)
+177: integer(base_16, i(0, []), signed, size_16_bit)
+178: integer(base_16, i(0, []), signed, size_32_bit)
+179: integer(base_16, i(0, []), signed, size_64_bit)
+181: string("Hexadecimal signed zeros (with leading zeros)")
+182: integer(base_16, i(0, []), signed, size_word)
+183: integer(base_16, i(0, []), signed, size_word)
+184: integer(base_16, i(0, []), signed, size_8_bit)
+185: integer(base_16, i(0, []), signed, size_16_bit)
+186: integer(base_16, i(0, []), signed, size_32_bit)
+187: integer(base_16, i(0, []), signed, size_64_bit)
+189: string("Hexadecimal unsigned zeros")
+190: integer(base_16, i(0, []), unsigned, size_word)
+191: integer(base_16, i(0, []), unsigned, size_8_bit)
+192: integer(base_16, i(0, []), unsigned, size_16_bit)
+193: integer(base_16, i(0, []), unsigned, size_32_bit)
+194: integer(base_16, i(0, []), unsigned, size_64_bit)
+196: string("Hexadecimal unsigned zeros (with underscores)")
+197: integer(base_16, i(0, []), unsigned, size_word)
+198: integer(base_16, i(0, []), unsigned, size_8_bit)
+199: integer(base_16, i(0, []), unsigned, size_16_bit)
+200: integer(base_16, i(0, []), unsigned, size_32_bit)
+201: integer(base_16, i(0, []), unsigned, size_64_bit)
+203: string("Hexadecimal unsigned zeros (with leading zeros)")
+204: integer(base_16, i(0, []), unsigned, size_word)
+205: integer(base_16, i(0, []), unsigned, size_8_bit)
+206: integer(base_16, i(0, []), unsigned, size_16_bit)
+207: integer(base_16, i(0, []), unsigned, size_32_bit)
+208: integer(base_16, i(0, []), unsigned, size_64_bit)
+210: string("Miscellaneous underscores")
+211: integer(base_2, i(0, []), unsigned, size_8_bit)
+212: integer(base_16, i(0, []), unsigned, size_16_bit)
+213: integer(base_8, i(0, []), signed, size_32_bit)
+214: integer(base_10, i(0, []), signed, size_64_bit)
+215: integer(base_2, i(0, []), signed, size_word)
+216: integer(base_2, i(0, []), unsigned, size_word)
-integer(base_10, i(0, []), signed, size_word)
-integer(base_10, i(2, [753, 8526]), signed, size_word)
-integer(base_10, i(1, [10]), signed, size_word)
-integer(base_10, i(1, [97]), signed, size_word)
-integer(base_10, i(1, [97]), signed, size_word)
-name("bc")
-integer(base_2, i(0, []), signed, size_word)
-integer(base_2, i(1, [1]), signed, size_word)
-integer(base_2, i(1, [3]), signed, size_word)
-integer(base_2, i(1, [7]), signed, size_word)
-integer(base_8, i(0, []), signed, size_word)
-integer(base_8, i(1, [1]), signed, size_word)
-integer(base_8, i(2, [20, 14711]), signed, size_word)
-integer(base_16, i(1, [1]), signed, size_word)
-integer(base_16, i(3, [1, 2257, 5752]), signed, size_word)
-integer(base_16, i(2, [9903, 3567]), signed, size_word)
-integer(base_16, i(2, [9903, 3567]), signed, size_word)
-float(0.123)
-float(1.23E+44)
-float(0.0)
-float(0.0)
-float(0.0)
-float(0.0)
+1: integer(base_10, i(0, []), signed, size_word)
+2: integer(base_10, i(2, [753, 8526]), signed, size_word)
+3: integer(base_10, i(1, [10]), signed, size_word)
+4: integer(base_10, i(1, [97]), signed, size_word)
+5: integer(base_10, i(1, [97]), signed, size_word)
+5: name("bc")
+7: integer(base_2, i(0, []), signed, size_word)
+8: integer(base_2, i(1, [1]), signed, size_word)
+9: integer(base_2, i(1, [3]), signed, size_word)
+10: integer(base_2, i(1, [7]), signed, size_word)
+14: integer(base_8, i(0, []), signed, size_word)
+15: integer(base_8, i(1, [1]), signed, size_word)
+16: integer(base_8, i(2, [20, 14711]), signed, size_word)
+18: integer(base_16, i(1, [1]), signed, size_word)
+19: integer(base_16, i(3, [1, 2257, 5752]), signed, size_word)
+20: integer(base_16, i(2, [9903, 3567]), signed, size_word)
+21: integer(base_16, i(2, [9903, 3567]), signed, size_word)
+25: float(0.123)
+26: float(1.23E+44)
+28: float(0.0)
+29: float(0.0)
+30: float(0.0)
+31: float(0.0)
+33: string("Decimal signed zeros")
+34: integer(base_10, i(0, []), signed, size_word)
+35: integer(base_10, i(0, []), signed, size_8_bit)
+36: integer(base_10, i(0, []), signed, size_16_bit)
+37: integer(base_10, i(0, []), signed, size_32_bit)
+38: integer(base_10, i(0, []), signed, size_64_bit)
+40: string("Decimal signed zeros (with underscores)")
+41: integer(base_10, i(0, []), signed, size_word)
+42: integer(base_10, i(0, []), signed, size_8_bit)
+43: integer(base_10, i(0, []), signed, size_16_bit)
+44: integer(base_10, i(0, []), signed, size_32_bit)
+45: integer(base_10, i(0, []), signed, size_64_bit)
+47: string("Decimal signed zeros (with leading zeros)")
+48: integer(base_10, i(0, []), signed, size_word)
+49: integer(base_10, i(0, []), signed, size_8_bit)
+50: integer(base_10, i(0, []), signed, size_16_bit)
+51: integer(base_10, i(0, []), signed, size_32_bit)
+52: integer(base_10, i(0, []), signed, size_64_bit)
+54: string("Decimal unsigned zeros")
+55: integer(base_10, i(0, []), unsigned, size_word)
+56: integer(base_10, i(0, []), unsigned, size_8_bit)
+57: integer(base_10, i(0, []), unsigned, size_16_bit)
+58: integer(base_10, i(0, []), unsigned, size_32_bit)
+59: integer(base_10, i(0, []), unsigned, size_64_bit)
+61: string("Decimal unsigned zeros (with underscores)")
+62: integer(base_10, i(0, []), unsigned, size_word)
+63: integer(base_10, i(0, []), unsigned, size_8_bit)
+64: integer(base_10, i(0, []), unsigned, size_16_bit)
+65: integer(base_10, i(0, []), unsigned, size_32_bit)
+66: integer(base_10, i(0, []), unsigned, size_64_bit)
+68: string("Decimal unsigned zeros (with leading zeros)")
+69: integer(base_10, i(0, []), unsigned, size_word)
+70: integer(base_10, i(0, []), unsigned, size_8_bit)
+71: integer(base_10, i(0, []), unsigned, size_16_bit)
+72: integer(base_10, i(0, []), unsigned, size_32_bit)
+73: integer(base_10, i(0, []), unsigned, size_64_bit)
+75: string("Binary signed zeros")
+76: integer(base_2, i(0, []), signed, size_word)
+77: integer(base_2, i(0, []), signed, size_word)
+78: integer(base_2, i(0, []), signed, size_8_bit)
+79: integer(base_2, i(0, []), signed, size_16_bit)
+80: integer(base_2, i(0, []), signed, size_32_bit)
+81: integer(base_2, i(0, []), signed, size_64_bit)
+83: string("Binary signed zeros (with underscores)")
+84: integer(base_2, i(0, []), signed, size_word)
+85: integer(base_2, i(0, []), signed, size_word)
+86: integer(base_2, i(0, []), signed, size_8_bit)
+87: integer(base_2, i(0, []), signed, size_16_bit)
+88: integer(base_2, i(0, []), signed, size_32_bit)
+89: integer(base_2, i(0, []), signed, size_64_bit)
+91: string("Binary signed zeros (with leading zeros)")
+92: integer(base_2, i(0, []), signed, size_word)
+93: integer(base_2, i(0, []), signed, size_word)
+94: integer(base_2, i(0, []), signed, size_8_bit)
+95: integer(base_2, i(0, []), signed, size_16_bit)
+96: integer(base_2, i(0, []), signed, size_32_bit)
+97: integer(base_2, i(0, []), signed, size_64_bit)
+99: string("Binary unsigned zeros")
+100: integer(base_2, i(0, []), unsigned, size_word)
+101: integer(base_2, i(0, []), unsigned, size_8_bit)
+102: integer(base_2, i(0, []), unsigned, size_16_bit)
+103: integer(base_2, i(0, []), unsigned, size_32_bit)
+104: integer(base_2, i(0, []), unsigned, size_64_bit)
+106: string("Binary unsigned zeros (with underscores)")
+107: integer(base_2, i(0, []), unsigned, size_word)
+108: integer(base_2, i(0, []), unsigned, size_8_bit)
+109: integer(base_2, i(0, []), unsigned, size_16_bit)
+110: integer(base_2, i(0, []), unsigned, size_32_bit)
+111: integer(base_2, i(0, []), unsigned, size_64_bit)
+113: string("Binary unsigned zeros (with leading zeros)")
+114: integer(base_2, i(0, []), unsigned, size_word)
+115: integer(base_2, i(0, []), unsigned, size_8_bit)
+116: integer(base_2, i(0, []), unsigned, size_16_bit)
+117: integer(base_2, i(0, []), unsigned, size_32_bit)
+118: integer(base_2, i(0, []), unsigned, size_64_bit)
+120: string("Octal signed zeros")
+121: integer(base_8, i(0, []), signed, size_word)
+122: integer(base_8, i(0, []), signed, size_word)
+123: integer(base_8, i(0, []), signed, size_8_bit)
+124: integer(base_8, i(0, []), signed, size_16_bit)
+125: integer(base_8, i(0, []), signed, size_32_bit)
+126: integer(base_8, i(0, []), signed, size_64_bit)
+128: string("Octal signed zeros (with underscores)")
+129: integer(base_8, i(0, []), signed, size_word)
+130: integer(base_8, i(0, []), signed, size_word)
+131: integer(base_8, i(0, []), signed, size_8_bit)
+132: integer(base_8, i(0, []), signed, size_16_bit)
+133: integer(base_8, i(0, []), signed, size_32_bit)
+134: integer(base_8, i(0, []), signed, size_64_bit)
+136: string("Octal signed zeros (with leading zeros)")
+137: integer(base_8, i(0, []), signed, size_word)
+138: integer(base_8, i(0, []), signed, size_word)
+139: integer(base_8, i(0, []), signed, size_8_bit)
+140: integer(base_8, i(0, []), signed, size_16_bit)
+141: integer(base_8, i(0, []), signed, size_32_bit)
+142: integer(base_8, i(0, []), signed, size_64_bit)
+144: string("Octal unsigned zeros")
+145: integer(base_8, i(0, []), unsigned, size_word)
+146: integer(base_8, i(0, []), unsigned, size_8_bit)
+147: integer(base_8, i(0, []), unsigned, size_16_bit)
+148: integer(base_8, i(0, []), unsigned, size_32_bit)
+149: integer(base_8, i(0, []), unsigned, size_64_bit)
+151: string("Octal unsigned zeros (with underscores)")
+152: integer(base_8, i(0, []), unsigned, size_word)
+153: integer(base_8, i(0, []), unsigned, size_8_bit)
+154: integer(base_8, i(0, []), unsigned, size_16_bit)
+155: integer(base_8, i(0, []), unsigned, size_32_bit)
+156: integer(base_8, i(0, []), unsigned, size_64_bit)
+158: string("Octal unsigned zeros (with leading zeros)")
+159: integer(base_8, i(0, []), unsigned, size_word)
+160: integer(base_8, i(0, []), unsigned, size_8_bit)
+161: integer(base_8, i(0, []), unsigned, size_16_bit)
+162: integer(base_8, i(0, []), unsigned, size_32_bit)
+163: integer(base_8, i(0, []), unsigned, size_64_bit)
+165: string("Hexadecimal signed zeros")
+166: integer(base_16, i(0, []), signed, size_word)
+167: integer(base_16, i(0, []), signed, size_word)
+168: integer(base_16, i(0, []), signed, size_8_bit)
+169: integer(base_16, i(0, []), signed, size_16_bit)
+170: integer(base_16, i(0, []), signed, size_32_bit)
+171: integer(base_16, i(0, []), signed, size_64_bit)
+173: string("Hexadecimal signed zeros (with underscores)")
+174: integer(base_16, i(0, []), signed, size_word)
+175: integer(base_16, i(0, []), signed, size_word)
+176: integer(base_16, i(0, []), signed, size_8_bit)
+177: integer(base_16, i(0, []), signed, size_16_bit)
+178: integer(base_16, i(0, []), signed, size_32_bit)
+179: integer(base_16, i(0, []), signed, size_64_bit)
+181: string("Hexadecimal signed zeros (with leading zeros)")
+182: integer(base_16, i(0, []), signed, size_word)
+183: integer(base_16, i(0, []), signed, size_word)
+184: integer(base_16, i(0, []), signed, size_8_bit)
+185: integer(base_16, i(0, []), signed, size_16_bit)
+186: integer(base_16, i(0, []), signed, size_32_bit)
+187: integer(base_16, i(0, []), signed, size_64_bit)
+189: string("Hexadecimal unsigned zeros")
+190: integer(base_16, i(0, []), unsigned, size_word)
+191: integer(base_16, i(0, []), unsigned, size_8_bit)
+192: integer(base_16, i(0, []), unsigned, size_16_bit)
+193: integer(base_16, i(0, []), unsigned, size_32_bit)
+194: integer(base_16, i(0, []), unsigned, size_64_bit)
+196: string("Hexadecimal unsigned zeros (with underscores)")
+197: integer(base_16, i(0, []), unsigned, size_word)
+198: integer(base_16, i(0, []), unsigned, size_8_bit)
+199: integer(base_16, i(0, []), unsigned, size_16_bit)
+200: integer(base_16, i(0, []), unsigned, size_32_bit)
+201: integer(base_16, i(0, []), unsigned, size_64_bit)
+203: string("Hexadecimal unsigned zeros (with leading zeros)")
+204: integer(base_16, i(0, []), unsigned, size_word)
+205: integer(base_16, i(0, []), unsigned, size_8_bit)
+206: integer(base_16, i(0, []), unsigned, size_16_bit)
+207: integer(base_16, i(0, []), unsigned, size_32_bit)
+208: integer(base_16, i(0, []), unsigned, size_64_bit)
+210: string("Miscellaneous underscores")
+211: integer(base_2, i(0, []), unsigned, size_8_bit)
+212: integer(base_16, i(0, []), unsigned, size_16_bit)
+213: integer(base_8, i(0, []), signed, size_32_bit)
+214: integer(base_10, i(0, []), signed, size_64_bit)
+215: integer(base_2, i(0, []), signed, size_word)
+216: integer(base_2, i(0, []), unsigned, size_word)
diff --git a/tests/hard_coded/lexer_zero.inp b/tests/hard_coded/lexer_zero.inp
index 7be608b..eb28f5b 100644
--- a/tests/hard_coded/lexer_zero.inp
+++ b/tests/hard_coded/lexer_zero.inp
@@ -29,3 +29,188 @@
0e-12
0E12
0E-12
+
+"Decimal signed zeros"
+0i
+0i8
+0i16
+0i32
+0i64
+
+"Decimal signed zeros (with underscores)"
+0_i
+0_i8
+0_i16
+0_i32
+0_i64
+
+"Decimal signed zeros (with leading zeros)"
+00i
+00i8
+00i16
+00i32
+00i64
+
+"Decimal unsigned zeros"
+0u
+0u8
+0u16
+0u32
+0u64
+
+"Decimal unsigned zeros (with underscores)"
+0_u
+0_u8
+0_u16
+0_u32
+0_u64
+
+"Decimal unsigned zeros (with leading zeros)"
+00u
+00u8
+00u16
+00u32
+00u64
+
+"Binary signed zeros"
+0b0
+0b0i
+0b0i8
+0b0i16
+0b0i32
+0b0i64
+
+"Binary signed zeros (with underscores)"
+0b_0
+0b_0_i
+0b_0_i8
+0b_0_i16
+0b_0_i32
+0b_0_i64
+
+"Binary signed zeros (with leading zeros)"
+0b00
+0b00i
+0b00i8
+0b00i16
+0b00i32
+0b00i64
+
+"Binary unsigned zeros"
+0b0u
+0b0u8
+0b0u16
+0b0u32
+0b0u64
+
+"Binary unsigned zeros (with underscores)"
+0b_0_u
+0b_0_u8
+0b_0_u16
+0b_0_u32
+0b_0_u64
+
+"Binary unsigned zeros (with leading zeros)"
+0b00u
+0b00u8
+0b00u16
+0b00u32
+0b00u64
+
+"Octal signed zeros"
+0o0
+0o0i
+0o0i8
+0o0i16
+0o0i32
+0o0i64
+
+"Octal signed zeros (with underscores)"
+0o_0
+0o_0_i
+0o_0_i8
+0o_0_i16
+0o_0_i32
+0o_0_i64
+
+"Octal signed zeros (with leading zeros)"
+0o00
+0o00i
+0o00i8
+0o00i16
+0o00i32
+0o00i64
+
+"Octal unsigned zeros"
+0o0u
+0o0u8
+0o0u16
+0o0u32
+0o0u64
+
+"Octal unsigned zeros (with underscores)"
+0o_0_u
+0o_0_u8
+0o_0_u16
+0o_0_u32
+0o_0_u64
+
+"Octal unsigned zeros (with leading zeros)"
+0o00u
+0o00u8
+0o00u16
+0o00u32
+0o00u64
+
+"Hexadecimal signed zeros"
+0x0
+0x0i
+0x0i8
+0x0i16
+0x0i32
+0x0i64
+
+"Hexadecimal signed zeros (with underscores)"
+0x_0
+0x_0_i
+0x_0_i8
+0x_0_i16
+0x_0_i32
+0x_0_i64
+
+"Hexadecimal signed zeros (with leading zeros)"
+0x00
+0x00i
+0x00i8
+0x00i16
+0x00i32
+0x00i64
+
+"Hexadecimal unsigned zeros"
+0x0u
+0x0u8
+0x0u16
+0x0u32
+0x0u64
+
+"Hexadecimal unsigned zeros (with underscores)"
+0x_0_u
+0x_0_u8
+0x_0_u16
+0x_0_u32
+0x_0_u64
+
+"Hexadecimal unsigned zeros (with leading zeros)"
+0x00u
+0x00u8
+0x00u16
+0x00u32
+0x00u64
+
+"Miscellaneous underscores"
+0b_0_u8
+0x_00_u16
+0o_00_i32
+0_0_0_i64
+0b_0_0_0__i
+0b_0_0_00__u
diff --git a/tests/hard_coded/lexer_zero.m b/tests/hard_coded/lexer_zero.m
index d35043e..946e560 100644
--- a/tests/hard_coded/lexer_zero.m
+++ b/tests/hard_coded/lexer_zero.m
@@ -35,20 +35,19 @@ main(!IO) :-
write_token_list(StringTokens, !IO)
;
ReadRes = error(_, Error),
- io.write(Error, !IO),
- io.nl(!IO)
+ io.write_line(Error, !IO)
),
io.close_input(Stream, !IO)
;
OpenRes = error(Error),
- io.write(Error, !IO),
- io.nl(!IO)
+ io.write_line(Error, !IO)
).
:- pred write_token_list(token_list::in, io::di, io::uo) is det.
write_token_list(token_nil, !IO).
-write_token_list(token_cons(Token, _Context, List), !IO) :-
- io.write(Token, !IO),
- io.nl(!IO),
+write_token_list(token_cons(Token, Context, List), !IO) :-
+ io.write_int(Context, !IO),
+ io.write_string(": ", !IO),
+ io.write_line(Token, !IO),
write_token_list(List, !IO).
More information about the reviews
mailing list