File 5413-compiler-Make-leex-respect-deterministic.p... of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 5413-compiler-Make-leex-respect-deterministic.patch of Package erlang (Revision f9f387618fe39a6cb7be110b2c705b4a)

Currently displaying revision f9f387618fe39a6cb7be110b2c705b4a , Show latest

From d8014fb184ef7a64bec94baae59dcdc89b1f66b3 Mon Sep 17 00:00:00 2001
From: Tom Davies <todavies5@gmail.com>
Date: Thu, 28 Apr 2022 06:06:51 -0700
Subject: [PATCH 3/7] compiler: Make leex respect +deterministic

Makes generated leex scanners only use basenames in generated -file
attributes, rather than absolute paths when +deterministic is set.
---
 lib/parsetools/doc/src/leex.xml    |  5 +++
 lib/parsetools/src/leex.erl        | 58 +++++++++++++++++++-----------
 lib/parsetools/test/leex_SUITE.erl | 49 +++++++++++++++++++++++--
 3 files changed, 89 insertions(+), 23 deletions(-)

diff --git a/lib/parsetools/doc/src/leex.xml b/lib/parsetools/doc/src/leex.xml
index 357ed86f44..d802e46b59 100644
--- a/lib/parsetools/doc/src/leex.xml
+++ b/lib/parsetools/doc/src/leex.xml
@@ -110,6 +110,11 @@
 <item>
 Causes warnings to be treated as errors.
 </item>
+ <tag><c>{deterministic, boolean()}</c></tag>
+ <item>
+ Causes generated -file() attributes to only include
+ the basename of the file path.
+ </item>
 </taglist>
 Any of the Boolean options can be set to <c>true</c> by 
 stating the name of the option. For example, <c>verbose</c>
diff --git a/lib/parsetools/src/leex.erl b/lib/parsetools/src/leex.erl
index 15803b3b82..b764678516 100644
--- a/lib/parsetools/src/leex.erl
+++ b/lib/parsetools/src/leex.erl
@@ -78,9 +78,10 @@ compile(Input0, Output0,
 Output = assure_extension(shorten_filename(Output0), ".erl"),
 Includefile = lists:sublist(Includes, 1),
 Werror = proplists:get_bool(warnings_as_errors, Specific),
+ Deterministic = proplists:get_bool(deterministic, Specific),
 Opts = [{scannerfile,Output},{includefile,Includefile},{verbose,Verbose},
 {report_errors,true},{report_warnings,WarnLevel > 0},
-	 {warnings_as_errors, Werror}],
+	 {warnings_as_errors, Werror}, {deterministic, Deterministic}],
 case file(Input, Opts) of
 {ok, _} ->
 ok;
@@ -117,6 +118,7 @@ file(File) -> file(File, []).
 | {'scannerfile', Scannerfile :: file:filename()}
 | {'verbose', boolean()}
 | {'warnings_as_errors', boolean()}
+ | {'deterministic', boolean()}
 | 'dfa_graph'
 | 'report_errors' | 'report_warnings' | 'report'
 | 'return_errors' | 'return_warnings' | 'return'
@@ -287,7 +289,7 @@ check_options(_Options, _, _L) ->
 all_options() ->
 [dfa_graph,includefile,report_errors,report_warnings,
 return_errors,return_warnings,scannerfile,verbose,
- warnings_as_errors].
+ warnings_as_errors, deterministic].
 
 default_option(dfa_graph) -> false;
 default_option(includefile) -> [];
@@ -297,7 +299,8 @@ default_option(return_errors) -> false;
 default_option(return_warnings) -> false;
 default_option(scannerfile) -> [];
 default_option(verbose) -> false;
-default_option(warnings_as_errors) -> false.
+default_option(warnings_as_errors) -> false;
+default_option(deterministic) -> false.
 
 atom_option(dfa_graph) -> {dfa_graph,true};
 atom_option(report_errors) -> {report_errors,true};
@@ -306,6 +309,7 @@ atom_option(warnings_as_errors) -> {warnings_as_errors,true};
 atom_option(return_errors) -> {return_errors,true};
 atom_option(verbose) -> {verbose,true};
 atom_option(return_warnings) -> {return_warnings,true};
+atom_option(deterministic) -> {deterministic,true};
 atom_option(Key) -> Key.
 
 is_filename(T) ->
@@ -1362,7 +1366,8 @@ out_file(St0, DFA, DF, Actions, Code) ->
 set_encoding(St0, Ofile),
 try 
 output_encoding_comment(Ofile, St0),
- output_file_directive(Ofile, St0#leex.ifile, 0),
+ Deterministic = proplists:get_bool(deterministic, St0#leex.opts),
+ output_file_directive(Ofile, St0#leex.ifile, Deterministic, 0),
 out_file(Ifile, Ofile, St0, DFA, DF, Actions,
 Code, 1),
 verbose_print(St0, "ok~n", []),
@@ -1400,15 +1405,18 @@ inc_file_name(Filename) ->
 %% characters.
 
 out_file(Ifile, Ofile, St, DFA, DF, Actions, Code, L) ->
+ Deterministic = proplists:get_bool(deterministic, St#leex.opts),
 case io:get_line(Ifile, leex) of
- eof -> output_file_directive(Ofile, St#leex.ifile, L);
- {error, _} -> add_error(St#leex.ifile, {L, leex, cannot_parse}, St);
+ eof ->
+ output_file_directive(Ofile, St#leex.ifile, Deterministic, L);
+ {error, _} ->
+ add_error(St#leex.ifile, {L, leex, cannot_parse}, St);
 Line ->
 case string:slice(Line, 0, 5) of
 "##mod" -> out_module(Ofile, St);
 "##cod" -> out_erlang_code(Ofile, St, Code, L);
 "##dfa" -> out_dfa(Ofile, St, DFA, Code, DF, L);
- "##act" -> out_actions(Ofile, St#leex.xfile, Actions);
+ "##act" -> out_actions(Ofile, St#leex.xfile, Deterministic, Actions);
 _ -> io:put_chars(Ofile, Line)
 end,
 out_file(Ifile, Ofile, St, DFA, DF, Actions, Code, L+1)
@@ -1419,7 +1427,8 @@ out_module(File, St) ->
 
 out_erlang_code(File, St, Code, L) ->
 {CodeL,CodePos,_NCodeLines} = Code,
- output_file_directive(File, St#leex.xfile, CodeL),
+ Deterministic = proplists:get_bool(deterministic, St#leex.opts),
+ output_file_directive(File, St#leex.xfile, Deterministic, CodeL),
 {ok,Xfile} = file:open(St#leex.xfile, [read]),
 try
 set_encoding(St, Xfile),
@@ -1429,7 +1438,7 @@ out_erlang_code(File, St, Code, L) ->
 ok = file:close(Xfile)
 end,
 io:nl(File),
- output_file_directive(File, St#leex.ifile, L).
+ output_file_directive(File, St#leex.ifile, Deterministic, L).
 
 file_copy(From, To) ->
 case io:get_line(From, leex) of
@@ -1441,8 +1450,9 @@ file_copy(From, To) ->
 
 out_dfa(File, St, DFA, Code, DF, L) ->
 {_CodeL,_CodePos,NCodeLines} = Code,
+ Deterministic = proplists:get_bool(deterministic, St#leex.opts),
 %% Three file attributes before this one...
- output_file_directive(File, St#leex.efile, L+(NCodeLines-1)+3),
+ output_file_directive(File, St#leex.efile, Deterministic, L+(NCodeLines-1)+3),
 io:fwrite(File, "yystate() -> ~w.~n~n", [DF]),
 foreach(fun (S) -> out_trans(File, S) end, DFA),
 io:fwrite(File, "yystate(S, Ics, Line, Tlen, Action, Alen) ->~n", []),
@@ -1565,14 +1575,14 @@ pack_trans([Tr|Trs], Pt) -> % The default uninteresting case
 pack_trans(Trs, Pt ++ [Tr]);
 pack_trans([], Pt) -> Pt.
 
-%% out_actions(File, XrlFile, ActionList) -> ok.
+%% out_actions(File, XrlFile, Deterministic, ActionList) -> ok.
 %% Write out the action table.
 
-out_actions(File, XrlFile, As) ->
+out_actions(File, XrlFile, Deterministic, As) ->
 As1 = prep_out_actions(As),
 foreach(fun (A) -> out_action(File, A) end, As1),
 io:fwrite(File, "yyaction(_, _, _, _) -> error.~n", []),
- foreach(fun (A) -> out_action_code(File, XrlFile, A) end, As1).
+ foreach(fun (A) -> out_action_code(File, XrlFile, Deterministic, A) end, As1).
 
 prep_out_actions(As) ->
 map(fun ({A,empty_action}) ->
@@ -1603,14 +1613,14 @@ out_action(File, {A,_Code,Vars,Name,_Args,ArgsChars}) ->
 end,
 io:fwrite(File, " ~s(~s);~n", [Name, ArgsChars]).
 
-out_action_code(_File, _XrlFile, {_A,empty_action}) ->
+out_action_code(_File, _XrlFile, _Deterministic, {_A,empty_action}) ->
 ok;
-out_action_code(File, XrlFile, {_A,Code,_Vars,Name,Args,ArgsChars}) ->
+out_action_code(File, XrlFile, Deterministic, {_A,Code,_Vars,Name,Args,ArgsChars}) ->
 %% Should set the file to the .erl file, but instead assumes that
 %% ?LEEXINC is syntactically correct.
 io:fwrite(File, "\n-compile({inline,~w/~w}).\n", [Name, length(Args)]),
 L = erl_scan:line(hd(Code)),
- output_file_directive(File, XrlFile, L-2),
+ output_file_directive(File, XrlFile, Deterministic, L-2),
 io:fwrite(File, "~s(~s) ->~n", [Name, ArgsChars]),
 io:fwrite(File, " ~ts\n", [pp_tokens(Code, L, File)]).
 
@@ -1710,12 +1720,18 @@ output_encoding_comment(_File, #leex{encoding = none}) ->
 output_encoding_comment(File, #leex{encoding = Encoding}) ->
 io:fwrite(File, <<"%% ~s\n">>, [epp:encoding_to_string(Encoding)]).
 
-output_file_directive(File, Filename, Line) ->
+output_file_directive(File, Filename, Deterministic, Line) ->
 io:fwrite(File, <<"-file(~ts, ~w).\n">>,
- [format_filename(Filename, File), Line]).
-
-format_filename(Filename0, File) ->
- Filename = filename:flatten(Filename0),
+ [format_filename(Filename, File, Deterministic), Line]).
+
+format_filename(Filename0, File, Deterministic) ->
+ Filename =
+ case Deterministic of
+ true ->
+ filename:basename(filename:flatten(Filename0));
+ false ->
+ filename:flatten(Filename0)
+ end,
 case enc(File) of
 unicode -> io_lib:write_string(Filename);
 latin1 -> io_lib:write_string_as_latin1(Filename)
diff --git a/lib/parsetools/test/leex_SUITE.erl b/lib/parsetools/test/leex_SUITE.erl
index 09a6e026bd..2d73b58e3c 100644
--- a/lib/parsetools/test/leex_SUITE.erl
+++ b/lib/parsetools/test/leex_SUITE.erl
@@ -22,6 +22,7 @@
 %-define(debug, true).
 
 -include_lib("stdlib/include/erl_compile.hrl").
+-include_lib("stdlib/include/assert.hrl").
 -include_lib("kernel/include/file.hrl").
 
 -ifdef(debug).
@@ -39,7 +40,7 @@
 	 init_per_testcase/2, end_per_testcase/2]).
 
 -export([
-	 file/1, compile/1, syntax/1,
+	 file/1, compile/1, syntax/1, deterministic/1,
 	 
 	 pt/1, man/1, ex/1, ex2/1, not_yet/1,
 	 line_wrap/1,
@@ -64,7 +65,7 @@ all() ->
 [{group, checks}, {group, examples}, {group, tickets}, {group, bugs}].
 
 groups() -> 
- [{checks, [], [file, compile, syntax]},
+ [{checks, [], [file, compile, syntax, deterministic]},
 {examples, [], [pt, man, ex, ex2, not_yet, unicode]},
 {tickets, [], [otp_10302, otp_11286, otp_13916, otp_14285, otp_17023,
 compiler_warnings]},
@@ -368,6 +369,40 @@ syntax(Config) when is_list(Config) ->
 leex:file(Filename, Ret),
 ok.
 
+deterministic(doc) ->
+ "Check leex respects the +deterministic flag.";
+deterministic(suite) -> [];
+deterministic(Config) when is_list(Config) ->
+ Dir = ?privdir,
+ Filename = filename:join(Dir, "file.xrl"),
+ Scannerfile = filename:join(Dir, "file.erl"),
+ Mini = <<"Definitions.\n"
+ "D = [0-9]\n"
+ "Rules.\n"
+ "{L}+ : {token,{word,TokenLine,TokenChars}}.\n"
+ "Erlang code.\n">>,
+ ok = file:write_file(Filename, Mini),
+
+ %% Generated leex scanners include the leexinc.hrl header file by default,
+ %% so we'll get a -file attribute corresponding to that include. In
+ %% deterministic mode, that include should only use the basename,
+ %% "leexinc.hrl", but otherwise, it should contain the full path.
+
+ AbsolutePathSuffix = "/lib/parsetools/include/leexinc.hrl",
+
+ ok = leex:compile(Filename, Scannerfile, #options{specific=[deterministic]}),
+ {ok, FormsDet} = epp:parse_file(Scannerfile,[]),
+ ?assertMatch(false, search_for_file_attr(AbsolutePathSuffix, FormsDet)),
+ ?assertMatch({value, _}, search_for_file_attr("leexinc.hrl", FormsDet)),
+ file:delete(Scannerfile),
+
+ ok = leex:compile(Filename, Scannerfile, #options{}),
+ {ok, Forms} = epp:parse_file(Scannerfile,[]),
+ ?assertMatch({value, _}, search_for_file_attr(AbsolutePathSuffix, Forms)),
+ file:delete(Scannerfile),
+
+ file:delete(Filename),
+ ok.
 
 pt(doc) ->
 "Pushing back characters.";
@@ -1272,3 +1307,13 @@ extract(File, {error, Es, Ws}) ->
 {errors, extract(File, Es), extract(File, Ws)}; 
 extract(File, Ts) ->
 lists:append([T || {F, T} <- Ts, F =:= File]).
+
+search_for_file_attr(PartialFilePath, Forms) ->
+ lists:search(fun
+ ({attribute, _, file, {FileAttr, _}}) ->
+ case string:find(FileAttr, PartialFilePath) of
+ nomatch -> false;
+ _ -> true
+ end;
+ (_) -> false end,
+ Forms).
-- 
2.35.3

Places

File 5413-compiler-Make-leex-respect-deterministic.patch of Package erlang (Revision f9f387618fe39a6cb7be110b2c705b4a)

Places