<div dir="ltr">This looks fine.</div> <div class="gmail_quote"><div dir="ltr" class="gmail_attr">On Wed, Oct 16, 2019 at 5:06 PM Peter Wang <<a href="mailto:novalazy@gmail.com">novalazy@gmail.com</a>> wrote: </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">library/string.m: As above. tests/hard_coded/Mmakefile: tests/hard_coded/string_fold_ilseq.exp: tests/hard_coded/string_fold_ilseq.exp2: tests/hard_coded/string_fold_ilseq.m: Add test case. --- library/string.m | 27 +++++--------- tests/hard_coded/Mmakefile | 1 + tests/hard_coded/string_fold_ilseq.exp | 5 +++ tests/hard_coded/string_fold_ilseq.exp2 | 5 +++ tests/hard_coded/string_fold_ilseq.m | 49 +++++++++++++++++++++++++ 5 files changed, 69 insertions(+), 18 deletions(-) create mode 100644 tests/hard_coded/string_fold_ilseq.exp create mode 100644 tests/hard_coded/string_fold_ilseq.exp2 create mode 100644 tests/hard_coded/string_fold_ilseq.m diff --git a/library/string.m b/library/string.m index 884363c82..10bbf6f5b 100644 --- a/library/string.m +++ b/library/string.m @@ -1006,8 +1006,15 @@ % foldl(Closure, String, !Acc): % % `Closure' is an accumulator predicate which is to be called for each - % character (code point) of the string `String' in turn. The initial - % value of the accumulator is `!.Acc' and the final value is `!:Acc'. + % character (code point) of the string `String' in turn. + % If `String' contains ill-formed sequences, `Closure' is called for each + % code unit in an ill-formed sequences. If strings use UTF-8 encoding, + % U+FFFD is passed to `Closure' in place each such code unit. + % If strings use UTF-16 encoding, each code unit in an ill-formed sequence + % is an unpaired surrogate code point, which will be passed to `Closure'. + % + % The initial value of the accumulator is `!.Acc' and the final value is + % `!:Acc'. % (foldl(Closure, String, !Acc) is equivalent to % to_char_list(String, Chars), % list.foldl(Closure, Chars, !Acc) @@ -5202,20 +5209,6 @@ break_up_string_reverse(Str, N, Prev) = Strs :- % Folds over the characters in strings. % -% XXX ILSEQ The behaviour of foldl depends on unsafe_index_next. -% For UTF-16, we can call Closure for unpaired surrogate code points like any -% other code point. For UTF-8, bytes in ill-formed sequences cannot be passed -% as `char's since they are not code points. Perhaps foldl should throw an -% exception, or just should pass replacement char. -% -% We may want to introduce fold predicates with an accumulator of type: -% pred(char_or_code_unit, A, A) -% or -% pred(char, maybe(int), A, A) -% For the latter, the second argument would be set to `yes(CodeUnit)' for each -% code unit in an ill-formed sequence, and the first argument could be set to -% U+FFFD (replacement char). - foldl(F, S, A) = B :- P = ( pred(X::in, Y::in, Z::out) is det :- Z = F(X, Y) ), foldl(P, S, A, B). @@ -5308,8 +5301,6 @@ foldl2_substring(Closure, String, Start, Count, !Acc1, !Acc2) :- %---------------------% -% XXX ILSEQ Behaviour depends on unsafe_prev_index. - foldr(F, String, Acc0) = Acc :- Closure = ( pred(X::in, Y::in, Z::out) is det :- Z = F(X, Y)), foldr(Closure, String, Acc0, Acc). diff --git a/tests/hard_coded/Mmakefile b/tests/hard_coded/Mmakefile index 7906dcbf3..626f60c6a 100644 --- a/tests/hard_coded/Mmakefile +++ b/tests/hard_coded/Mmakefile @@ -360,6 +360,7 @@ ORDINARY_PROGS = \ string_codepoint \ string_count_codepoints_ilseq \ string_first_char \ + string_fold_ilseq \ string_index_ilseq \ string_index_next_ilseq \ string_loop \ diff --git a/tests/hard_coded/string_fold_ilseq.exp b/tests/hard_coded/string_fold_ilseq.exp new file mode 100644 index 000000000..bfc1435ba --- /dev/null +++ b/tests/hard_coded/string_fold_ilseq.exp @@ -0,0 +1,5 @@ +string.foldl: +a b c 😀 0xfffd 0xfffd 0xfffd x y z + +string.foldr: +z y x 0xfffd 0xfffd 0xfffd 😀 c b a diff --git a/tests/hard_coded/string_fold_ilseq.exp2 b/tests/hard_coded/string_fold_ilseq.exp2 new file mode 100644 index 000000000..d28c67096 --- /dev/null +++ b/tests/hard_coded/string_fold_ilseq.exp2 @@ -0,0 +1,5 @@ +string.foldl: +a b c 😀 0xd83d x y z + +string.foldr: +z y x 0xd83d 😀 c b a diff --git a/tests/hard_coded/string_fold_ilseq.m b/tests/hard_coded/string_fold_ilseq.m new file mode 100644 index 000000000..ba8a64bcb --- /dev/null +++ b/tests/hard_coded/string_fold_ilseq.m @@ -0,0 +1,49 @@ +%---------------------------------------------------------------------------% +% vim: ts=4 sw=4 et ft=mercury +%---------------------------------------------------------------------------% +% +% The .exp file is for backends using UTF-8 string encoding. +% The .exp2 file is for backends using UTF-16 string encoding. +% +%---------------------------------------------------------------------------% + +:- module string_fold_ilseq. +:- interface. + +:- import_module io. + +:- pred main(io::di, io::uo) is det. + +%---------------------------------------------------------------------------% + +:- implementation. + +:- import_module char. +:- import_module int. +:- import_module list. +:- import_module string. + +%---------------------------------------------------------------------------% + +main(!IO) :- + S0 = "😀", + S1 = string.between(S0, 0, count_code_units(S0) - 1), + S = "abc" ++ S0 ++ S1 ++ "xyz", + + io.write_string("string.foldl:\n", !IO), + string.foldl(write_char_or_hex, S, !IO), + io.write_string("\n\n", !IO), + + io.write_string("string.foldr:\n", !IO), + string.foldr(write_char_or_hex, S, !IO), + io.write_string("\n", !IO). + +:- pred write_char_or_hex(char::in, io::di, io::uo) is det. + +write_char_or_hex(Char, !IO) :- + ( if Char = '\ufffd' ; char.is_surrogate(Char) then + io.format("%#x", [i(char.to_int(Char))], !IO) + else + io.write_char(Char, !IO) + ), + io.write_char(' ', !IO). -- 2.23.0 _______________________________________________ reviews mailing list <a href="mailto:reviews@lists.mercurylang.org" target="_blank">reviews@lists.mercurylang.org</a> <a href="https://lists.mercurylang.org/listinfo/reviews" rel="noreferrer" target="_blank">https://lists.mercurylang.org/listinfo/reviews</a> </blockquote></div>