[m-rev.] for review [juliensf]: Avoid command line buffer limits
Peter Ross
pro at missioncriticalit.com
Mon Oct 6 15:23:30 AEDT 2008
Hi,
===================================================================
Estimated hours taken: 4
Branches: main
Windows has a command line length limit, and I have started to hit
it when using a MinGW version of the compiler, so use files to
pass argument lists which are potentially very long to programs.
compiler/compile_target_code.m:
Make all calls to mkinit use -f to pass the list
of files to process.
For linking use the @file syntax to pass all the
arguments to a command as when linking we
can have very long lists of object files.
compiler/handle_options.m:
Describe how mmc handles @files.
compiler/mercury_compile.m:
Handle @file arguments on the command line.
Index: compiler/compile_target_code.m
===================================================================
RCS file: /home/mercury1/repository/mercury/compiler/compile_target_code.m,v
retrieving revision 1.131
diff -u -r1.131 compile_target_code.m
--- compiler/compile_target_code.m 1 Oct 2008 04:09:13 -0000 1.131
+++ compiler/compile_target_code.m 6 Oct 2008 04:03:26 -0000
@@ -1052,34 +1052,10 @@
list.map_foldl(
module_name_to_file_name_ext(TargetExt, do_not_create_dirs),
AllModules, AllTargetFilesList, !IO),
- join_quoted_string_list(AllTargetFilesList, "", "\n", "",
- TargetFileNames),
-
- io.make_temp(TmpFile, !IO),
- io.open_output(TmpFile, OpenResult, !IO),
- (
- OpenResult = ok(TmpStream),
- io.write_string(TmpStream, TargetFileNames, !IO),
- io.close_output(TmpStream, !IO),
- MkInitCmd = string.append_list([MkInit, " -k -f ", TmpFile]),
- invoke_system_command(InitFileStream, cmd_verbose_commands,
- MkInitCmd, MkInitOK0, !IO),
+ invoke_mkinit(InitFileStream, cmd_verbose_commands,
+ MkInit, " -k ", AllTargetFilesList, MkInitOK, !IO),
- io.remove_file(TmpFile, RemoveResult, !IO),
- (
- RemoveResult = ok,
- MkInitOK = MkInitOK0
- ;
- RemoveResult = error(_),
- MkInitOK = no
- )
- ;
- OpenResult = error(_),
- MkInitOK = no
- ),
-
-
(
MkInitOK = yes,
globals.io_lookup_maybe_string_option(extra_init_command,
@@ -1152,6 +1128,38 @@
module_name_to_file_name_ext(Ext, MkDir, ModuleName, FileName, !IO) :-
module_name_to_file_name(ModuleName, Ext, MkDir, FileName, !IO).
+:- pred invoke_mkinit(io.output_stream::in, command_verbosity::in,
+ string::in, string::in, list(file_name)::in, bool::out,
+ io::di, io::uo) is det.
+
+invoke_mkinit(InitFileStream, Verbosity,
+ MkInit, Args, FileNames, MkInitOK, !IO) :-
+ join_quoted_string_list(FileNames, "", "\n", "", TargetFileNames),
+
+ io.make_temp(TmpFile, !IO),
+ io.open_output(TmpFile, OpenResult, !IO),
+ (
+ OpenResult = ok(TmpStream),
+ io.write_string(TmpStream, TargetFileNames, !IO),
+ io.close_output(TmpStream, !IO),
+
+ MkInitCmd = string.append_list([MkInit, " ", Args, " -f ", TmpFile]),
+ invoke_system_command(InitFileStream, Verbosity,
+ MkInitCmd, MkInitOK0, !IO),
+
+ io.remove_file(TmpFile, RemoveResult, !IO),
+ (
+ RemoveResult = ok,
+ MkInitOK = MkInitOK0
+ ;
+ RemoveResult = error(_),
+ MkInitOK = no
+ )
+ ;
+ OpenResult = error(_),
+ MkInitOK = no
+ ).
+
%-----------------------------------------------------------------------------%
link_module_list(Modules, FactTableObjFiles, Succeeded, !IO) :-
@@ -1359,7 +1367,6 @@
list.map_foldl(module_name_to_file_name_ext(TargetExt, do_not_create_dirs),
ModuleNames, TargetFileNameList, !IO),
- join_quoted_string_list(TargetFileNameList, "", "", " ", TargetFileNames),
globals.io_lookup_accumulating_option(init_file_directories,
InitFileDirsList, !IO),
@@ -1390,8 +1397,6 @@
InitFileNamesList = InitFileNamesList2
),
- join_quoted_string_list(InitFileNamesList, "", "", " ", InitFileNames),
-
globals.io_lookup_accumulating_option(runtime_flags, RuntimeFlagsList,
!IO),
join_quoted_string_list(RuntimeFlagsList, "-r ", "", " ", RuntimeFlags),
@@ -1424,9 +1429,8 @@
),
TmpInitTargetFileName = InitTargetFileName ++ ".tmp",
- MkInitCmd = string.append_list(
- [ MkInit,
- " -g ", Grade,
+ MkInitArgs = string.append_list(
+ [ " -g ", Grade,
" ", TraceOpt,
" ", ExtraInitsOpt,
" ", NoMainOpt,
@@ -1434,12 +1438,13 @@
" ", RuntimeFlags,
" -o ", quote_arg(TmpInitTargetFileName),
" ", InitFileDirs,
- " ", InitFileNames,
- " ", TargetFileNames,
ModuleNameOption
]),
- invoke_system_command(ErrorStream, cmd_verbose_commands, MkInitCmd,
+
+ invoke_mkinit(ErrorStream, cmd_verbose_commands,
+ MkInit, MkInitArgs, TargetFileNameList ++ InitFileNamesList,
MkInitOk, !IO),
+
maybe_report_stats(Stats, !IO),
(
MkInitOk = yes,
@@ -1810,7 +1815,6 @@
% after Objects.
globals.io_lookup_string_option(CommandOpt, Command, !IO),
string.append_list([
- Command, " ",
StaticOpts, " ",
StripOpt, " ",
UndefOpt, " ",
@@ -1826,7 +1830,7 @@
LDFlags, " ",
LinkLibraries, " ",
MercuryStdLibs, " ",
- SystemLibs], LinkCmd),
+ SystemLibs], Args),
globals.io_lookup_bool_option(demangle, Demangle, !IO),
(
@@ -1839,9 +1843,9 @@
MaybeDemangleCmd = no
),
- invoke_system_command_maybe_filter_output(ErrorStream,
- cmd_verbose_commands, LinkCmd, MaybeDemangleCmd, LinkSucceeded,
- !IO)
+ invoke_long_system_command_maybe_filter_output(ErrorStream,
+ cmd_verbose_commands, Command, Args, MaybeDemangleCmd,
+ LinkSucceeded, !IO)
;
LibrariesSucceeded = no,
LinkSucceeded = no
@@ -2203,45 +2207,14 @@
% Quoting would prevent that.
join_string_list(ObjectList, "", "", " ", Objects)
),
- ( ArCmd = "lib" ->
-
- %
- % If we are using lib, we are on windows and windows doesn't
- % handle long command lines, so place the list of object
- % files in a file and pass that file as an argument to lib.
- %
- io.make_temp(TmpFile, !IO),
- io.open_output(TmpFile, OpenResult, !IO),
- (
- OpenResult = ok(TmpStream),
- io.write_string(TmpStream, Objects, !IO),
- io.close_output(TmpStream, !IO),
- MakeLibCmd = string.append_list([
- ArCmd, " ", ArFlags, " ", ArOutputFlag,
- LibFileName, " @", TmpFile]),
- invoke_system_command(ErrorStream, cmd_verbose_commands,
- MakeLibCmd, MakeLibCmdSucceeded0, !IO),
+ MakeLibCmdArgs = string.append_list([
+ ArFlags, " ", ArOutputFlag, " ",
+ LibFileName, " ", Objects]),
+
+ invoke_long_system_command(ErrorStream, cmd_verbose_commands,
+ ArCmd, MakeLibCmdArgs, MakeLibCmdSucceeded, !IO),
- io.remove_file(TmpFile, RemoveResult, !IO),
- (
- RemoveResult = ok,
- MakeLibCmdSucceeded = MakeLibCmdSucceeded0
- ;
- RemoveResult = error(_),
- MakeLibCmdSucceeded = no
- )
- ;
- OpenResult = error(_),
- MakeLibCmdSucceeded = no
- )
- ;
- MakeLibCmd = string.append_list([
- ArCmd, " ", ArFlags, " ", ArOutputFlag, " ",
- LibFileName, " ", Objects]),
- invoke_system_command(ErrorStream, cmd_verbose_commands,
- MakeLibCmd, MakeLibCmdSucceeded, !IO)
- ),
(
( RanLib = ""
; MakeLibCmdSucceeded = no
@@ -2610,7 +2583,6 @@
SourceDebug = no,
InitFiles = InitFiles2
),
- join_string_list(InitFiles, "", "", " ", InitFilesList),
globals.lookup_accumulating_option(Globals, runtime_flags,
RuntimeFlagsList),
join_quoted_string_list(RuntimeFlagsList, "-r ", "", " ", RuntimeFlags),
@@ -2628,18 +2600,18 @@
globals.lookup_string_option(Globals, mkinit_command, MkInit),
CFileName = Basename ++ ".c",
io.output_stream(ErrorStream, !IO),
- MkInitCmd = string.append_list(
- [ MkInit,
- " -g ", Grade,
+ MkInitArgs = string.append_list(
+ [ " -g ", Grade,
" ", TraceOpt,
" ", ExperimentalComplexityOpt,
" ", RuntimeFlags,
" -o ", quote_arg(CFileName),
" ", InitFileDirs,
- " -s ", InitFilesList
+ " -s "
]),
- invoke_system_command(ErrorStream, cmd_verbose_commands,
- MkInitCmd, MkInitCmdOk, !IO),
+
+ invoke_mkinit(ErrorStream, cmd_verbose_commands,
+ MkInit, MkInitArgs, InitFiles, MkInitCmdOk, !IO),
(
MkInitCmdOk = yes,
get_object_code_type(executable, PIC, !IO),
@@ -2667,6 +2639,73 @@
).
%-----------------------------------------------------------------------------%
+
+ %
+ % invoke_long_system_command attempts to use the @file style of
+ % calling to avoid command line length arguments on various systems.
+ % If the underlying tool chain doesn't support this it just calls
+ % the normal invoke_system_command and hopes the command isn't too
+ % long.
+ %
+:- pred invoke_long_system_command(io.output_stream::in,
+ command_verbosity::in, string::in, string::in,
+ bool::out, io::di, io::uo) is det.
+
+invoke_long_system_command(ErrorStream, Verbosity, Cmd, Args, Succeeded, !IO) :-
+ invoke_long_system_command_maybe_filter_output(ErrorStream, Verbosity,
+ Cmd, Args, no, Succeeded, !IO).
+
+:- pred invoke_long_system_command_maybe_filter_output(io.output_stream::in,
+ command_verbosity::in, string::in, string::in, maybe(string)::in,
+ bool::out, io::di, io::uo) is det.
+
+invoke_long_system_command_maybe_filter_output(ErrorStream, Verbosity,
+ Cmd, Args, MaybeProcessOutput, Succeeded, !IO) :-
+ support_at_file(SupportAtFile, !IO),
+ ( SupportAtFile = yes,
+ %
+ % Avoid generating very long command lines by using @files
+ %
+ io.make_temp(TmpFile, !IO),
+ io.open_output(TmpFile, OpenResult, !IO),
+ (
+ OpenResult = ok(TmpStream),
+ io.write_string(TmpStream, Args, !IO),
+ io.close_output(TmpStream, !IO),
+
+ FullCmd = string.append_list([Cmd, " @", TmpFile]),
+ invoke_system_command(ErrorStream, Verbosity,
+ FullCmd, Succeeded0, !IO),
+
+ io.remove_file(TmpFile, RemoveResult, !IO),
+ (
+ RemoveResult = ok,
+ Succeeded = Succeeded0
+ ;
+ RemoveResult = error(_),
+ Succeeded = no
+ )
+ ;
+ OpenResult = error(_),
+ Succeeded = no
+ )
+
+ ; SupportAtFile = no,
+ FullCmd = Cmd ++ " " ++ Args,
+ invoke_system_command_maybe_filter_output(ErrorStream, Verbosity,
+ FullCmd, MaybeProcessOutput, Succeeded, !IO)
+ ).
+
+ % Does the underlying tool chain support using "@file" to pass
+ % arguments?
+ %
+ % XXX This should be made a conditional option, so that toolchains
+ % which don't support this can be used.
+:- pred support_at_file(bool::out, io::di, io::uo) is det.
+
+support_at_file(yes, !IO).
+
+%-----------------------------------------------------------------------------%
%
% C compiler flags
%
Index: compiler/handle_options.m
===================================================================
RCS file: /home/mercury1/repository/mercury/compiler/handle_options.m,v
retrieving revision 1.325
diff -u -r1.325 handle_options.m
--- compiler/handle_options.m 20 Sep 2008 11:38:04 -0000 1.325
+++ compiler/handle_options.m 6 Oct 2008 04:03:27 -0000
@@ -2282,6 +2282,8 @@
"are assumed to be source file names.\n", !IO),
io.write_string("\tArguments that do not end in `.m' " ++
"are assumed to be module names.\n", !IO),
+ io.write_string("\tArguments in the form @file are replaced " ++
+ "with the contents of the file.\n", !IO),
io.write_string("Options:\n", !IO),
options_help(!IO).
Index: compiler/mercury_compile.m
===================================================================
RCS file: /home/mercury1/repository/mercury/compiler/mercury_compile.m,v
retrieving revision 1.480
diff -u -r1.480 mercury_compile.m
--- compiler/mercury_compile.m 30 Sep 2008 04:30:27 -0000 1.480
+++ compiler/mercury_compile.m 6 Oct 2008 04:03:28 -0000
@@ -232,6 +232,21 @@
io.set_output_stream(StdErr, _, !IO),
io.command_line_arguments(Args0, !IO),
+ % Replace all @file arguments with the contents of the file
+ expand_at_file_arguments(Args0, Res, !IO),
+ ( Res = ok(Args),
+ real_main_2(Args, !IO)
+ ; Res = error(E),
+ io.set_exit_status(1, !IO),
+
+ io.write_string(io.error_message(E), !IO),
+ io.nl(!IO)
+ ).
+
+
+:- pred real_main_2(list(string)::in, io::di, io::uo) is det.
+
+real_main_2(Args0, !IO) :-
% read_args_file and globals.io_printing_usage may attempt
% to look up options, so we need to initialize the globals.
handle_options([], _, _, _, _, !IO),
@@ -261,14 +276,13 @@
Variables = options_variables_init,
MaybeMCFlags = no
;
- Errors0 = [],
- read_options_files(options_variables_init, MaybeVariables0, !IO),
+ Errors0 = [], read_options_files(options_variables_init, MaybeVariables0, !IO),
(
MaybeVariables0 = yes(Variables0),
lookup_mmc_options(Variables0, MaybeMCFlags0, !IO),
(
MaybeMCFlags0 = yes(MCFlags0),
- real_main_2(MCFlags0, MaybeMCFlags, Args0,
+ real_main_3(MCFlags0, MaybeMCFlags, Args0,
Variables0, Variables, !IO)
;
@@ -295,11 +309,11 @@
io.set_exit_status(1, !IO)
).
-:- pred real_main_2(list(string)::in, maybe(list(string))::out,
+:- pred real_main_3(list(string)::in, maybe(list(string))::out,
list(string)::in, options_variables::in, options_variables::out,
io::di, io::uo) is det.
-real_main_2(MCFlags0, MaybeMCFlags, Args0, Variables0, Variables, !IO) :-
+real_main_3(MCFlags0, MaybeMCFlags, Args0, Variables0, Variables, !IO) :-
% Process the options again to find out which configuration file to read.
handle_options(MCFlags0 ++ Args0, Errors, _, _, _, !IO),
(
@@ -5393,6 +5407,73 @@
%-----------------------------------------------------------------------------%
+ %
+ % Expand @File arguments.
+ % Each argument in the above form is replaced with a list of arguments
+ % where each arg is each line in the file File which is not just whitespace.
+ %
+:- pred expand_at_file_arguments(list(string)::in, io.res(list(string))::out,
+ io::di, io::uo) is det.
+
+expand_at_file_arguments([], ok([]), !IO).
+expand_at_file_arguments([Arg | Args], Result, !IO) :-
+ ( string.remove_prefix("@", Arg, File) ->
+ io.open_input(File, OpenRes, !IO),
+ ( OpenRes = ok(S),
+ expand_file_into_arg_list(S, ReadRes, !IO),
+ ( ReadRes = ok(FileArgs),
+ expand_at_file_arguments(FileArgs ++ Args, Result, !IO)
+ ; ReadRes = error(E),
+ Result = error(at_file_error(File, E))
+ )
+ ; OpenRes = error(_E),
+ Msg = "mercury_compile: cannot open '" ++ File ++ "'",
+ Result = error(io.make_io_error(Msg))
+ )
+ ;
+ expand_at_file_arguments(Args, Result0, !IO),
+ ( Result0 = ok(ExpandedArgs),
+ Result = ok([Arg | ExpandedArgs])
+ ; Result0 = error(E),
+ Result = error(E)
+ )
+ ).
+
+:- func at_file_error(string, io.error) = io.error.
+
+at_file_error(File, E) =
+ io.make_io_error("While attempting to process '" ++ File ++
+ "' the following error occurred: " ++ io.error_message(E)).
+
+ %
+ % Read each of the command line arguments from the given input file.
+ % Note lines which consist purely of whitespace are ignored.
+ %
+:- pred expand_file_into_arg_list(io.input_stream::in, io.res(list(string))::out,
+ io::di, io::uo) is det.
+
+expand_file_into_arg_list(S, Res, !IO) :-
+ io.read_line_as_string(S, LineRes, !IO),
+ ( LineRes = ok(Line),
+ expand_file_into_arg_list(S, Res0, !IO),
+ ( Res0 = ok(Lines),
+ StrippedLine = strip(Line),
+ ( StrippedLine = "" ->
+ Res = ok(Lines)
+ ;
+ Res = ok([StrippedLine | Lines])
+ )
+ ; Res0 = error(_E),
+ Res = Res0
+ )
+ ; LineRes = eof,
+ Res = ok([])
+ ; LineRes = error(E),
+ Res = error(E)
+ ).
+
+%-----------------------------------------------------------------------------%
+
:- func this_file = string.
this_file = "mercury_compile.m".
--------------------------------------------------------------------------
mercury-reviews mailing list
Post messages to: mercury-reviews at csse.unimelb.edu.au
Administrative Queries: owner-mercury-reviews at csse.unimelb.edu.au
Subscriptions: mercury-reviews-request at csse.unimelb.edu.au
--------------------------------------------------------------------------
More information about the reviews
mailing list