Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Unicode regex #32

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Unicode regex #32

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
af1e838
Correct usage of unicode binaries in test_combinators.erl.
Nov 3, 2014
3ecb836
Added test showing that neotoma cannot handle unicode symbols in rege…
Nov 3, 2014
299a3c4
Handle unicode symbols in regexps correctly.
Nov 3, 2014
1f5ead2
Fixed syntax error.
dimitarp Nov 9, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions priv/neotoma_parse.peg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ regexp_string <- '#' string:(!'#' ('\\#' / .))+ '#'
% \ -> \\
% " -> \"
re:replace(proplists:get_value(string, Node), "\"|\\\\", "\\\\&", [{return, binary}, global]),
"\">>)"]
"\"/utf8>>)"]
`;

quoted_string <- single_quoted_string / double_quoted_string
Expand All @@ -176,7 +176,7 @@ character_class <- '[' characters:(!']' ('\\\\' . / !'\\\\' .))+ ']'
used_combinator(p_charclass),
["p_charclass(<<\"[",
escape_string(unicode:characters_to_list(proplists:get_value(characters, Node))),
"]\">>)"]
"]\"/utf8>>)"]
`;

anything_symbol <- '.' ` used_combinator(p_anything), <<"p_anything()">> `;
Expand Down
4 changes: 2 additions & 2 deletions src/neotoma_parse.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,7 +306,7 @@ end
% \ -> \\
% " -> \"
re:replace(proplists:get_value(string, Node), "\"|\\\\", "\\\\&", [{return, binary}, global]),
"\">>)"]
"\"/utf8>>)"]
end).

-spec 'quoted_string'(input(), index()) -> parse_result().
Expand All @@ -332,7 +332,7 @@ end
used_combinator(p_charclass),
["p_charclass(<<\"[",
escape_string(unicode:characters_to_list(proplists:get_value(characters, Node))),
"]\">>)"]
"]\"/utf8>>)"]
end).

-spec 'anything_symbol'(input(), index()) -> parse_result().
Expand Down
6 changes: 3 additions & 3 deletions test/test_combinators.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ anything_test_() ->
charclass_test_() ->
[
?_assertEqual({<<"+">>,<<"----">>,{{line,1},{column,2}}}, (neotoma_peg:p_charclass(<<"[+]">>))(<<"+----">>,?STARTINDEX)),
?_assertEqual({fail,{expected, {character_class, "[+]"}, ?STARTINDEX}}, (neotoma_peg:p_charclass(<<"[+]">>))(<<"----">>,?STARTINDEX))
?_assertEqual({fail,{expected, {character_class, "[+]"}, ?STARTINDEX}}, (neotoma_peg:p_charclass(<<"[+]">>))(<<"----">>,?STARTINDEX)),
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You have a syntax error here, trailing comma.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, sorry about that... I'll fix the patch.

].

regexp_test_() ->
Expand All @@ -101,6 +101,6 @@ column_test() ->

utf8_string_test_() ->
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test fails, as below:

  test_combinators:104: utf8_string_test_...*failed*
in function test_combinators:'-utf8_string_test_/0-fun-0-'/1 (test/test_combinators.erl, line 104)
**error:{assertEqual_failed,[{module,test_combinators},
                     {line,104},
                     {expression,"( neotoma_peg : p_string ( << \"�\\226�\\225\\214\" / utf8 >> ) ) ( << \"�\\226�\\225\\214def\" / utf8 >> , ? STARTINDEX )"},
                     {expected,{<<195,164,194,184,194,150,195,167,194,149,...>>,
                                <<"def">>,
                                {{line,1},{column,3}}}},
                     {value,{<<195,164,194,184,194,150,195,167,194,...>>,
                             <<"def">>,
                             {{line,1},{column,7}}}}]}

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems that you need to update the p_advance_index function so that it properly accounts for UTF8 characters.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The test is ok when I execute it... I can't reproduce the error.
What is the problem with p_advance_index?

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are only 3 codepoints in the matched string but they are multi-byte characters, so it advances too far.

[
?_assertEqual({<<"世界">>, <<"def">>,{{line,1},{column,3}}}, (neotoma_peg:p_string(<<"世界">>))(<<"世界def">>,?STARTINDEX)),
?_assertEqual({fail,{expected, {string, <<"世界">>}, ?STARTINDEX}}, (neotoma_peg:p_string(<<"世界">>))(<<"界世abc">>,?STARTINDEX))
?_assertEqual({<<"世界"/utf8>>, <<"def">>,{{line,1},{column,3}}}, (neotoma_peg:p_string(<<"世界"/utf8>>))(<<"世界def"/utf8>>,?STARTINDEX)),
?_assertEqual({fail,{expected, {string, <<"世界"/utf8>>}, ?STARTINDEX}}, (neotoma_peg:p_string(<<"世界"/utf8>>))(<<"界世abc"/utf8>>,?STARTINDEX))
].
16 changes: 15 additions & 1 deletion test/test_parse.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ parser_test() ->
file:write_file("test_parser.peg", io_lib:fwrite("~s\n", [Data])),
neotoma:file("test_parser.peg"),
compile:file("test_parser.erl", []),
try
try
TestString = [19990,30028,32,102,111,111],
Result = test_parser:parse(TestString),
?assertEqual(6, length(Result)),
Expand All @@ -17,3 +17,17 @@ parser_test() ->
_:_ -> ?assert(false)
end.

unicode_parser_test() ->
Data = "rule <- [\\x{1}-\\x{D7FF}]+;",
file:write_file("test_unicode_parser.peg", io_lib:fwrite("~s\n", [Data])),
neotoma:file("test_unicode_parser.peg"),
compile:file("test_unicode_parser.erl", []),
try
TestString = [19990,30028,32,102,111,111],
Result = test_unicode_parser:parse(TestString),
?assertEqual(6, length(Result)),
StringResult = lists:flatten(io_lib:format("~ts", [Result])),
?assertEqual(TestString, StringResult)
catch
_:_ -> ?assert(false)
end.