[m-rev.] diff: fix tokenisation of bin/oct/hex literals in strings
Peter Wang
novalazy at gmail.com
Fri May 11 10:14:33 AEST 2012
Branches: main, 11.07
library/lexer.m:
Fix tokenisation of binary, octal and hexadecimal integer literals
in strings. It did not work at all.
tests/hard_coded/Mmakefile:
tests/hard_coded/lexer_zero.exp:
tests/hard_coded/lexer_zero.inp:
tests/hard_coded/lexer_zero.m:
Add test case.
diff --git a/library/lexer.m b/library/lexer.m
index d60a71a..8c387b2 100644
--- a/library/lexer.m
+++ b/library/lexer.m
@@ -1932,9 +1932,10 @@ get_binary(Stream, Token, !IO) :-
string_token_context::out, posn::in, posn::out) is det.
string_get_binary(String, Len, Posn0, Token, Context, !Posn) :-
+ Posn1 = !.Posn,
( string_read_char(String, Len, Char, !Posn) ->
( char.is_binary_digit(Char) ->
- string_get_binary_2(String, Len, Posn0, Token, Context, !Posn)
+ string_get_binary_2(String, Len, Posn1, Token, Context, !Posn)
;
string_ungetchar(String, !Posn),
Token = error("unterminated binary constant"),
@@ -1969,20 +1970,20 @@ get_binary_2(Stream, Chars, Token, !IO) :-
:- pred string_get_binary_2(string::in, int::in, posn::in, token::out,
string_token_context::out, posn::in, posn::out) is det.
-string_get_binary_2(String, Len, Posn0, Token, Context, !Posn) :-
+string_get_binary_2(String, Len, Posn1, Token, Context, !Posn) :-
( string_read_char(String, Len, Char, !Posn) ->
( char.is_binary_digit(Char) ->
- string_get_binary_2(String, Len, Posn0, Token, Context, !Posn)
+ string_get_binary_2(String, Len, Posn1, Token, Context, !Posn)
;
string_ungetchar(String, !Posn),
- grab_string(String, Posn0, BinaryString, !Posn),
+ grab_string(String, Posn1, BinaryString, !Posn),
conv_string_to_int(BinaryString, 2, Token),
- string_get_context(Posn0, Context, !Posn)
+ string_get_context(Posn1, Context, !Posn)
)
;
- grab_string(String, Posn0, BinaryString, !Posn),
+ grab_string(String, Posn1, BinaryString, !Posn),
conv_string_to_int(BinaryString, 2, Token),
- string_get_context(Posn0, Context, !Posn)
+ string_get_context(Posn1, Context, !Posn)
).
:- pred get_octal(io.input_stream::in, token::out, io::di, io::uo) is det.
@@ -2010,9 +2011,10 @@ get_octal(Stream, Token, !IO) :-
posn::in, posn::out) is det.
string_get_octal(String, Len, Posn0, Token, Context, !Posn) :-
+ Posn1 = !.Posn,
( string_read_char(String, Len, Char, !Posn) ->
( char.is_octal_digit(Char) ->
- string_get_octal_2(String, Len, Posn0, Token, Context, !Posn)
+ string_get_octal_2(String, Len, Posn1, Token, Context, !Posn)
;
string_ungetchar(String, !Posn),
Token = error("unterminated octal constant"),
@@ -2047,20 +2049,20 @@ get_octal_2(Stream, Chars, Token, !IO) :-
:- pred string_get_octal_2(string::in, int::in, posn::in, token::out,
string_token_context::out, posn::in, posn::out) is det.
-string_get_octal_2(String, Len, Posn0, Token, Context, !Posn) :-
+string_get_octal_2(String, Len, Posn1, Token, Context, !Posn) :-
( string_read_char(String, Len, Char, !Posn) ->
( char.is_octal_digit(Char) ->
- string_get_octal_2(String, Len, Posn0, Token, Context, !Posn)
+ string_get_octal_2(String, Len, Posn1, Token, Context, !Posn)
;
string_ungetchar(String, !Posn),
- grab_string(String, Posn0, BinaryString, !Posn),
+ grab_string(String, Posn1, BinaryString, !Posn),
conv_string_to_int(BinaryString, 8, Token),
- string_get_context(Posn0, Context, !Posn)
+ string_get_context(Posn1, Context, !Posn)
)
;
- grab_string(String, Posn0, BinaryString, !Posn),
+ grab_string(String, Posn1, BinaryString, !Posn),
conv_string_to_int(BinaryString, 8, Token),
- string_get_context(Posn0, Context, !Posn)
+ string_get_context(Posn1, Context, !Posn)
).
:- pred get_hex(io.input_stream::in, token::out, io::di, io::uo) is det.
@@ -2088,9 +2090,10 @@ get_hex(Stream, Token, !IO) :-
posn::in, posn::out) is det.
string_get_hex(String, Len, Posn0, Token, Context, !Posn) :-
+ Posn1 = !.Posn,
( string_read_char(String, Len, Char, !Posn) ->
( char.is_hex_digit(Char) ->
- string_get_hex_2(String, Len, Posn0, Token, Context, !Posn)
+ string_get_hex_2(String, Len, Posn1, Token, Context, !Posn)
;
string_ungetchar(String, !Posn),
Token = error("unterminated hex constant"),
@@ -2125,20 +2128,20 @@ get_hex_2(Stream, Chars, Token, !IO) :-
:- pred string_get_hex_2(string::in, int::in, posn::in, token::out,
string_token_context::out, posn::in, posn::out) is det.
-string_get_hex_2(String, Len, Posn0, Token, Context, !Posn) :-
+string_get_hex_2(String, Len, Posn1, Token, Context, !Posn) :-
( string_read_char(String, Len, Char, !Posn) ->
( char.is_hex_digit(Char) ->
- string_get_hex_2(String, Len, Posn0, Token, Context, !Posn)
+ string_get_hex_2(String, Len, Posn1, Token, Context, !Posn)
;
string_ungetchar(String, !Posn),
- grab_string(String, Posn0, BinaryString, !Posn),
+ grab_string(String, Posn1, BinaryString, !Posn),
conv_string_to_int(BinaryString, 16, Token),
- string_get_context(Posn0, Context, !Posn)
+ string_get_context(Posn1, Context, !Posn)
)
;
- grab_string(String, Posn0, BinaryString, !Posn),
+ grab_string(String, Posn1, BinaryString, !Posn),
conv_string_to_int(BinaryString, 16, Token),
- string_get_context(Posn0, Context, !Posn)
+ string_get_context(Posn1, Context, !Posn)
).
:- pred get_number(io.input_stream::in, list(char)::in, token::out,
diff --git a/tests/hard_coded/Mmakefile b/tests/hard_coded/Mmakefile
index 46c610a..033412a 100644
--- a/tests/hard_coded/Mmakefile
+++ b/tests/hard_coded/Mmakefile
@@ -169,6 +169,7 @@ ORDINARY_PROGS= \
lco_mday_bug_2 \
lco_no_inline \
lco_pack_args \
+ lexer_zero \
list_series_int \
lookup_disj \
lookup_switch_simple \
diff --git a/tests/hard_coded/lexer_zero.exp b/tests/hard_coded/lexer_zero.exp
new file mode 100644
index 0000000..d947834
--- /dev/null
+++ b/tests/hard_coded/lexer_zero.exp
@@ -0,0 +1,47 @@
+integer(0)
+integer(12345678)
+integer(10)
+integer(97)
+integer(97)
+name("bc")
+integer(0)
+integer(1)
+integer(3)
+integer(7)
+integer(0)
+integer(1)
+integer(342391)
+integer(1)
+integer(305419896)
+integer(162254319)
+integer(162254319)
+float(0.123)
+float(1.23e+44)
+float(0.0)
+float(0.0)
+float(0.0)
+float(0.0)
+
+integer(0)
+integer(12345678)
+integer(10)
+integer(97)
+integer(97)
+name("bc")
+integer(0)
+integer(1)
+integer(3)
+integer(7)
+integer(0)
+integer(1)
+integer(342391)
+integer(1)
+integer(305419896)
+integer(162254319)
+integer(162254319)
+float(0.123)
+float(1.23e+44)
+float(0.0)
+float(0.0)
+float(0.0)
+float(0.0)
diff --git a/tests/hard_coded/lexer_zero.inp b/tests/hard_coded/lexer_zero.inp
new file mode 100644
index 0000000..7be608b
--- /dev/null
+++ b/tests/hard_coded/lexer_zero.inp
@@ -0,0 +1,31 @@
+0
+012345678
+0'
+0'a
+0'abc
+% 0b
+0b0
+0b1
+0b11
+0b111
+% 0b2
+% 0x
+% 0o
+0o0
+0o1
+0o1234567
+% 0o8
+0x1
+0x12345678
+0x9abcdef
+0x9ABCDEF
+% 0xg
+% 0xG
+% 0.
+0.123
+0.123e45
+% 0e
+0e12
+0e-12
+0E12
+0E-12
diff --git a/tests/hard_coded/lexer_zero.m b/tests/hard_coded/lexer_zero.m
new file mode 100644
index 0000000..f71cac5
--- /dev/null
+++ b/tests/hard_coded/lexer_zero.m
@@ -0,0 +1,56 @@
+%-----------------------------------------------------------------------------%
+
+:- module lexer_zero.
+:- interface.
+
+:- import_module io.
+
+:- pred main(io::di, io::uo) is det.
+
+%-----------------------------------------------------------------------------%
+%-----------------------------------------------------------------------------%
+
+:- implementation.
+
+:- import_module lexer.
+
+%-----------------------------------------------------------------------------%
+
+main(!IO) :-
+ % Read from the current input stream.
+ lexer.get_token_list(Tokens, !IO),
+ write_token_list(Tokens, !IO),
+ io.nl(!IO),
+
+ % Read from a string.
+ io.open_input("lexer_zero.inp", OpenRes, !IO),
+ (
+ OpenRes = ok(Stream),
+ io.read_file_as_string(Stream, ReadRes, !IO),
+ (
+ ReadRes = ok(String),
+ Posn0 = posn(1, 0, 0),
+ lexer.string_get_token_list(String, StringTokens, Posn0, _Posn),
+ write_token_list(StringTokens, !IO)
+ ;
+ ReadRes = error(_, Error),
+ io.write(Error, !IO),
+ io.nl(!IO)
+ ),
+ io.close_input(Stream, !IO)
+ ;
+ OpenRes = error(Error),
+ io.write(Error, !IO),
+ io.nl(!IO)
+ ).
+
+:- pred write_token_list(token_list::in, io::di, io::uo) is det.
+
+write_token_list(token_nil, !IO).
+write_token_list(token_cons(Token, _Context, List), !IO) :-
+ io.write(Token, !IO),
+ io.nl(!IO),
+ write_token_list(List, !IO).
+
+%-----------------------------------------------------------------------------%
+% vim: ft=mercury ts=4 sts=4 sw=4 et
--------------------------------------------------------------------------
mercury-reviews mailing list
Post messages to: mercury-reviews at csse.unimelb.edu.au
Administrative Queries: owner-mercury-reviews at csse.unimelb.edu.au
Subscriptions: mercury-reviews-request at csse.unimelb.edu.au
--------------------------------------------------------------------------
More information about the reviews
mailing list