[m-rev.] for review [juliensf]: Avoid command line buffer limits

Peter Ross pro at missioncriticalit.com
Mon Oct 6 15:23:30 AEDT 2008


Hi,


===================================================================


Estimated hours taken: 4
Branches: main

Windows has a command line length limit, and I have started to hit
it when using a MinGW version of the compiler, so use files to
pass argument lists which are potentially very long to programs.


compiler/compile_target_code.m:
	Make all calls to mkinit use -f to pass the list 
	of files to process.
	For linking use the @file syntax to pass all the
	arguments to a command as when linking we
	can have very long lists of object files.

compiler/handle_options.m:
	Describe how mmc handles @files.
	
compiler/mercury_compile.m:
	Handle @file arguments on the command line.


Index: compiler/compile_target_code.m
===================================================================
RCS file: /home/mercury1/repository/mercury/compiler/compile_target_code.m,v
retrieving revision 1.131
diff -u -r1.131 compile_target_code.m
--- compiler/compile_target_code.m	1 Oct 2008 04:09:13 -0000	1.131
+++ compiler/compile_target_code.m	6 Oct 2008 04:03:26 -0000
@@ -1052,34 +1052,10 @@
         list.map_foldl(
             module_name_to_file_name_ext(TargetExt, do_not_create_dirs),
             AllModules, AllTargetFilesList, !IO),
-        join_quoted_string_list(AllTargetFilesList, "", "\n", "",
-            TargetFileNames),
-        
-        io.make_temp(TmpFile, !IO),
-        io.open_output(TmpFile, OpenResult, !IO),
-        (
-            OpenResult = ok(TmpStream),
-            io.write_string(TmpStream, TargetFileNames, !IO),
-            io.close_output(TmpStream, !IO),
 
-            MkInitCmd = string.append_list([MkInit, " -k -f ", TmpFile]),
-            invoke_system_command(InitFileStream, cmd_verbose_commands,
-                MkInitCmd, MkInitOK0, !IO),
+        invoke_mkinit(InitFileStream, cmd_verbose_commands,
+            MkInit, " -k ", AllTargetFilesList, MkInitOK, !IO),
 
-            io.remove_file(TmpFile, RemoveResult, !IO),
-            (
-                RemoveResult = ok,
-                MkInitOK = MkInitOK0
-            ;
-                RemoveResult = error(_),
-                MkInitOK = no
-            )
-        ;
-            OpenResult = error(_),
-            MkInitOK = no
-        ),
-
-       
         ( 
             MkInitOK =  yes,
             globals.io_lookup_maybe_string_option(extra_init_command,
@@ -1152,6 +1128,38 @@
 module_name_to_file_name_ext(Ext, MkDir, ModuleName, FileName, !IO) :-
     module_name_to_file_name(ModuleName, Ext, MkDir, FileName, !IO).
 
+:- pred invoke_mkinit(io.output_stream::in, command_verbosity::in,
+    string::in, string::in, list(file_name)::in, bool::out,
+    io::di, io::uo) is det.
+
+invoke_mkinit(InitFileStream, Verbosity,
+        MkInit, Args, FileNames, MkInitOK, !IO) :-
+    join_quoted_string_list(FileNames, "", "\n", "", TargetFileNames),
+
+    io.make_temp(TmpFile, !IO),
+    io.open_output(TmpFile, OpenResult, !IO),
+    (
+        OpenResult = ok(TmpStream),
+        io.write_string(TmpStream, TargetFileNames, !IO),
+        io.close_output(TmpStream, !IO),
+
+        MkInitCmd = string.append_list([MkInit, " ", Args, " -f ", TmpFile]),
+        invoke_system_command(InitFileStream, Verbosity,
+            MkInitCmd, MkInitOK0, !IO),
+
+        io.remove_file(TmpFile, RemoveResult, !IO),
+        (
+            RemoveResult = ok,
+            MkInitOK = MkInitOK0
+        ;
+            RemoveResult = error(_),
+            MkInitOK = no
+        )
+    ;
+        OpenResult = error(_),
+        MkInitOK = no
+    ).
+
 %-----------------------------------------------------------------------------%
 
 link_module_list(Modules, FactTableObjFiles, Succeeded, !IO) :-
@@ -1359,7 +1367,6 @@
 
     list.map_foldl(module_name_to_file_name_ext(TargetExt, do_not_create_dirs),
         ModuleNames, TargetFileNameList, !IO),
-    join_quoted_string_list(TargetFileNameList, "", "", " ", TargetFileNames),
 
     globals.io_lookup_accumulating_option(init_file_directories,
         InitFileDirsList, !IO),
@@ -1390,8 +1397,6 @@
         InitFileNamesList = InitFileNamesList2
     ),
 
-    join_quoted_string_list(InitFileNamesList, "", "", " ", InitFileNames),
-
     globals.io_lookup_accumulating_option(runtime_flags, RuntimeFlagsList,
         !IO),
     join_quoted_string_list(RuntimeFlagsList, "-r ", "", " ", RuntimeFlags),
@@ -1424,9 +1429,8 @@
     ),
 
     TmpInitTargetFileName = InitTargetFileName ++ ".tmp",
-    MkInitCmd = string.append_list(
-        [   MkInit,
-            " -g ", Grade,
+    MkInitArgs = string.append_list(
+        [   " -g ", Grade,
             " ", TraceOpt,
             " ", ExtraInitsOpt,
             " ", NoMainOpt,
@@ -1434,12 +1438,13 @@
             " ", RuntimeFlags,
             " -o ", quote_arg(TmpInitTargetFileName),
             " ", InitFileDirs,
-            " ", InitFileNames, 
-            " ", TargetFileNames,
             ModuleNameOption
         ]),
-    invoke_system_command(ErrorStream, cmd_verbose_commands, MkInitCmd,
+
+    invoke_mkinit(ErrorStream, cmd_verbose_commands,
+        MkInit, MkInitArgs, TargetFileNameList ++ InitFileNamesList,
         MkInitOk, !IO),
+
     maybe_report_stats(Stats, !IO),
     (
         MkInitOk = yes,
@@ -1810,7 +1815,6 @@
         % after Objects.
         globals.io_lookup_string_option(CommandOpt, Command, !IO),
         string.append_list([
-                Command, " ",
                 StaticOpts, " ",
                 StripOpt, " ",
                 UndefOpt, " ",
@@ -1826,7 +1830,7 @@
                 LDFlags, " ",
                 LinkLibraries, " ",
                 MercuryStdLibs, " ",
-                SystemLibs], LinkCmd),
+                SystemLibs], Args),
 
         globals.io_lookup_bool_option(demangle, Demangle, !IO),
         (
@@ -1839,9 +1843,9 @@
             MaybeDemangleCmd = no
         ),
 
-        invoke_system_command_maybe_filter_output(ErrorStream,
-            cmd_verbose_commands, LinkCmd, MaybeDemangleCmd, LinkSucceeded,
-            !IO)
+        invoke_long_system_command_maybe_filter_output(ErrorStream,
+            cmd_verbose_commands, Command, Args, MaybeDemangleCmd,
+            LinkSucceeded, !IO)
     ;
         LibrariesSucceeded = no,
         LinkSucceeded = no
@@ -2203,45 +2207,14 @@
         % Quoting would prevent that.
         join_string_list(ObjectList, "", "", " ", Objects)
     ),
-    ( ArCmd = "lib" ->
-            
-        %
-        % If we are using lib, we are on windows and windows doesn't
-        % handle long command lines, so place the list of object
-        % files in a file and pass that file as an argument to lib.
-        %
-        io.make_temp(TmpFile, !IO),
-        io.open_output(TmpFile, OpenResult, !IO),
-        (
-            OpenResult = ok(TmpStream),
-            io.write_string(TmpStream, Objects, !IO),
-            io.close_output(TmpStream, !IO),
 
-            MakeLibCmd = string.append_list([
-                ArCmd, " ", ArFlags, " ", ArOutputFlag,
-                LibFileName, " @", TmpFile]),
-            invoke_system_command(ErrorStream, cmd_verbose_commands,
-                MakeLibCmd, MakeLibCmdSucceeded0, !IO),
+    MakeLibCmdArgs = string.append_list([
+        ArFlags, " ", ArOutputFlag, " ",
+        LibFileName, " ", Objects]),
+
+    invoke_long_system_command(ErrorStream, cmd_verbose_commands,
+        ArCmd, MakeLibCmdArgs, MakeLibCmdSucceeded, !IO),
 
-            io.remove_file(TmpFile, RemoveResult, !IO),
-            (
-                RemoveResult = ok,
-                MakeLibCmdSucceeded = MakeLibCmdSucceeded0
-            ;
-                RemoveResult = error(_),
-                MakeLibCmdSucceeded = no
-            )
-        ;
-            OpenResult = error(_),
-            MakeLibCmdSucceeded = no
-        )
-    ;
-        MakeLibCmd = string.append_list([
-            ArCmd, " ", ArFlags, " ", ArOutputFlag, " ",
-            LibFileName, " ", Objects]),
-        invoke_system_command(ErrorStream, cmd_verbose_commands,
-            MakeLibCmd, MakeLibCmdSucceeded, !IO)
-    ),
     (
         ( RanLib = ""
         ; MakeLibCmdSucceeded = no
@@ -2610,7 +2583,6 @@
         SourceDebug = no,
         InitFiles = InitFiles2
     ),
-    join_string_list(InitFiles, "", "", " ", InitFilesList),
     globals.lookup_accumulating_option(Globals, runtime_flags,
         RuntimeFlagsList),
     join_quoted_string_list(RuntimeFlagsList, "-r ", "", " ", RuntimeFlags),
@@ -2628,18 +2600,18 @@
     globals.lookup_string_option(Globals, mkinit_command, MkInit),
     CFileName = Basename ++ ".c",
     io.output_stream(ErrorStream, !IO),
-    MkInitCmd = string.append_list(
-        [   MkInit,
-            " -g ", Grade,
+    MkInitArgs = string.append_list(
+        [   " -g ", Grade,
             " ", TraceOpt,
             " ", ExperimentalComplexityOpt,
             " ", RuntimeFlags,
             " -o ", quote_arg(CFileName),
             " ", InitFileDirs,
-            " -s ", InitFilesList
+            " -s "
         ]),
-    invoke_system_command(ErrorStream, cmd_verbose_commands,
-        MkInitCmd, MkInitCmdOk, !IO),
+
+    invoke_mkinit(ErrorStream, cmd_verbose_commands,
+        MkInit, MkInitArgs, InitFiles, MkInitCmdOk, !IO),
     (
         MkInitCmdOk = yes,
         get_object_code_type(executable, PIC, !IO),
@@ -2667,6 +2639,73 @@
     ).
 
 %-----------------------------------------------------------------------------%
+
+    %
+    % invoke_long_system_command attempts to use the @file style of
+    % calling to avoid command line length arguments on various systems.
+    % If the underlying tool chain doesn't support this it just calls
+    % the normal invoke_system_command and hopes the command isn't too
+    % long.
+    %
+:- pred invoke_long_system_command(io.output_stream::in,
+    command_verbosity::in, string::in, string::in,
+    bool::out, io::di, io::uo) is det.
+
+invoke_long_system_command(ErrorStream, Verbosity, Cmd, Args, Succeeded, !IO) :-
+    invoke_long_system_command_maybe_filter_output(ErrorStream, Verbosity,
+        Cmd, Args, no, Succeeded, !IO).
+
+:- pred invoke_long_system_command_maybe_filter_output(io.output_stream::in,
+    command_verbosity::in, string::in, string::in, maybe(string)::in,
+    bool::out, io::di, io::uo) is det.
+
+invoke_long_system_command_maybe_filter_output(ErrorStream, Verbosity,
+        Cmd, Args, MaybeProcessOutput, Succeeded, !IO) :-
+    support_at_file(SupportAtFile, !IO),
+    ( SupportAtFile = yes,
+        %
+        % Avoid generating very long command lines by using @files
+        %
+        io.make_temp(TmpFile, !IO),
+        io.open_output(TmpFile, OpenResult, !IO),
+        (
+            OpenResult = ok(TmpStream),
+            io.write_string(TmpStream, Args, !IO),
+            io.close_output(TmpStream, !IO),
+
+            FullCmd = string.append_list([Cmd, " @", TmpFile]),
+            invoke_system_command(ErrorStream, Verbosity,
+                FullCmd, Succeeded0, !IO),
+
+            io.remove_file(TmpFile, RemoveResult, !IO),
+            (
+                RemoveResult = ok,
+                Succeeded = Succeeded0
+            ;
+                RemoveResult = error(_),
+                Succeeded = no
+            )
+        ;
+            OpenResult = error(_),
+            Succeeded = no
+        )
+        
+    ; SupportAtFile = no,
+        FullCmd = Cmd ++ " " ++ Args,
+        invoke_system_command_maybe_filter_output(ErrorStream, Verbosity,
+            FullCmd, MaybeProcessOutput, Succeeded, !IO)
+    ).
+
+    % Does the underlying tool chain support using "@file" to pass
+    % arguments?
+    %
+    % XXX This should be made a conditional option, so that toolchains
+    % which don't support this can be used.
+:- pred support_at_file(bool::out, io::di, io::uo) is det.
+
+support_at_file(yes, !IO).
+
+%-----------------------------------------------------------------------------%
 %
 % C compiler flags
 %
Index: compiler/handle_options.m
===================================================================
RCS file: /home/mercury1/repository/mercury/compiler/handle_options.m,v
retrieving revision 1.325
diff -u -r1.325 handle_options.m
--- compiler/handle_options.m	20 Sep 2008 11:38:04 -0000	1.325
+++ compiler/handle_options.m	6 Oct 2008 04:03:27 -0000
@@ -2282,6 +2282,8 @@
         "are assumed to be source file names.\n", !IO),
     io.write_string("\tArguments that do not end in `.m' " ++
         "are assumed to be module names.\n", !IO),
+    io.write_string("\tArguments in the form @file are replaced " ++
+        "with the contents of the file.\n", !IO),
     io.write_string("Options:\n", !IO),
     options_help(!IO).
 
Index: compiler/mercury_compile.m
===================================================================
RCS file: /home/mercury1/repository/mercury/compiler/mercury_compile.m,v
retrieving revision 1.480
diff -u -r1.480 mercury_compile.m
--- compiler/mercury_compile.m	30 Sep 2008 04:30:27 -0000	1.480
+++ compiler/mercury_compile.m	6 Oct 2008 04:03:28 -0000
@@ -232,6 +232,21 @@
     io.set_output_stream(StdErr, _, !IO),
     io.command_line_arguments(Args0, !IO),
 
+        % Replace all @file arguments with the contents of the file
+    expand_at_file_arguments(Args0, Res, !IO),
+    ( Res = ok(Args),
+        real_main_2(Args, !IO)
+    ; Res = error(E),
+        io.set_exit_status(1, !IO),
+
+        io.write_string(io.error_message(E), !IO),
+        io.nl(!IO)
+    ).
+
+
+:- pred real_main_2(list(string)::in, io::di, io::uo) is det.
+
+real_main_2(Args0, !IO) :-
     % read_args_file and globals.io_printing_usage may attempt
     % to look up options, so we need to initialize the globals.
     handle_options([], _, _, _, _, !IO),
@@ -261,14 +276,13 @@
             Variables = options_variables_init,
             MaybeMCFlags = no
         ;
-            Errors0 = [],
-            read_options_files(options_variables_init, MaybeVariables0, !IO),
+            Errors0 = [], read_options_files(options_variables_init, MaybeVariables0, !IO),
             (
                 MaybeVariables0 = yes(Variables0),
                 lookup_mmc_options(Variables0, MaybeMCFlags0, !IO),
                 (
                     MaybeMCFlags0 = yes(MCFlags0),
-                    real_main_2(MCFlags0, MaybeMCFlags, Args0,
+                    real_main_3(MCFlags0, MaybeMCFlags, Args0,
                         Variables0, Variables, !IO)
 
                 ;
@@ -295,11 +309,11 @@
         io.set_exit_status(1, !IO)
     ).
 
-:- pred real_main_2(list(string)::in, maybe(list(string))::out,
+:- pred real_main_3(list(string)::in, maybe(list(string))::out,
     list(string)::in, options_variables::in, options_variables::out,
     io::di, io::uo) is det.
 
-real_main_2(MCFlags0, MaybeMCFlags, Args0, Variables0, Variables, !IO) :-
+real_main_3(MCFlags0, MaybeMCFlags, Args0, Variables0, Variables, !IO) :-
     % Process the options again to find out which configuration file to read.
     handle_options(MCFlags0 ++ Args0, Errors, _, _, _, !IO),
     (
@@ -5393,6 +5407,73 @@
 
 %-----------------------------------------------------------------------------%
 
+    %
+    % Expand @File arguments.
+    % Each argument in the above form is replaced with a list of arguments
+    % where each arg is each line in the file File which is not just whitespace.
+    %
+:- pred expand_at_file_arguments(list(string)::in, io.res(list(string))::out,
+    io::di, io::uo) is det.
+
+expand_at_file_arguments([], ok([]), !IO).
+expand_at_file_arguments([Arg | Args], Result, !IO) :-
+    ( string.remove_prefix("@", Arg, File) ->
+        io.open_input(File, OpenRes, !IO),
+        ( OpenRes = ok(S),
+            expand_file_into_arg_list(S, ReadRes, !IO),
+            ( ReadRes = ok(FileArgs),
+                expand_at_file_arguments(FileArgs ++ Args, Result, !IO)
+            ; ReadRes = error(E),
+                Result = error(at_file_error(File, E))
+            )
+        ; OpenRes = error(_E),
+            Msg = "mercury_compile: cannot open '" ++ File ++ "'",
+            Result = error(io.make_io_error(Msg))
+        )
+    ;
+        expand_at_file_arguments(Args, Result0, !IO),
+        ( Result0 = ok(ExpandedArgs),
+            Result = ok([Arg | ExpandedArgs])
+        ; Result0 = error(E),
+            Result = error(E)
+        )
+    ).
+
+:- func at_file_error(string, io.error) = io.error.
+
+at_file_error(File, E) = 
+    io.make_io_error("While attempting to process '" ++ File ++
+            "' the following error occurred: " ++ io.error_message(E)).
+  
+    %
+    % Read each of the command line arguments from the given input file.
+    % Note lines which consist purely of whitespace are ignored.
+    %
+:- pred expand_file_into_arg_list(io.input_stream::in, io.res(list(string))::out,
+    io::di, io::uo) is det.
+
+expand_file_into_arg_list(S, Res, !IO) :-
+    io.read_line_as_string(S, LineRes, !IO),
+    ( LineRes = ok(Line),
+        expand_file_into_arg_list(S, Res0, !IO),
+        ( Res0 = ok(Lines),
+            StrippedLine = strip(Line),
+            ( StrippedLine = "" ->
+                Res = ok(Lines)
+            ;
+                Res = ok([StrippedLine | Lines])
+            )
+        ; Res0 = error(_E),
+            Res = Res0
+        )
+    ; LineRes = eof,
+        Res = ok([])
+    ; LineRes = error(E),
+        Res = error(E)
+    ).
+
+%-----------------------------------------------------------------------------%
+
 :- func this_file = string.
 
 this_file = "mercury_compile.m".

--------------------------------------------------------------------------
mercury-reviews mailing list
Post messages to:       mercury-reviews at csse.unimelb.edu.au
Administrative Queries: owner-mercury-reviews at csse.unimelb.edu.au
Subscriptions:          mercury-reviews-request at csse.unimelb.edu.au
--------------------------------------------------------------------------



More information about the reviews mailing list