[m-rev.] for review: speed up .opt file reading and deps
Peter Wang
novalazy at gmail.com
Mon Sep 1 16:27:24 AEST 2008
Branches: main
Change some data representations to avoid repeatedly appending lists when
reading optimisation interface files and calculating dependencies.
compiler/module_imports.m:
Change `module_imports' to hold items in a queue instead of a list.
Add getter/setter for items that work with lists, as most code will
continue to use lists.
Make `get_dependencies' and `get_dependencies_int_imp' accumulate
sets of modules instead of lists.
compiler/intermod.m:
Conform to `module_imports' changes.
Make `read_optimization_interfaces' accumulate lists of lists of items
so that appending a list of items doesn't rebuild as big a list.
compiler/make.module_dep_file.m:
compiler/mercury_compile.m:
compiler/modules.m:
compiler/trans_opt.m:
compiler/write_deps_file.m:
Conform to `module_imports' changes.
diff --git a/compiler/intermod.m b/compiler/intermod.m
index 38a41c8..3df99dd 100644
--- a/compiler/intermod.m
+++ b/compiler/intermod.m
@@ -128,6 +128,7 @@
:- import_module maybe.
:- import_module multi_map.
:- import_module pair.
+:- import_module queue.
:- import_module set.
:- import_module solutions.
:- import_module string.
@@ -2289,14 +2290,16 @@ grab_opt_files(!Module, FoundError, !IO) :-
ModulesProcessed = set.insert(set.sorted_list_to_set(OptFiles),
ModuleName),
read_optimization_interfaces(Transitive, ModuleName, OptFiles,
- ModulesProcessed, [], OptItems, no, OptError, !IO),
+ ModulesProcessed, [], OptItemsLists, no, OptError, !IO),
+ OptItems = list.condense(OptItemsLists),
% Append the items to the current item list, using a `opt_imported'
% psuedo-declaration to let make_hlds know the opt_imported stuff
% is coming.
module_imports_get_items(!.Module, Items0),
- Items1 = Items0 ++ [make_pseudo_decl(md_opt_imported) | OptItems],
- module_imports_set_items(Items1, !Module),
+ queue.put(Items0, make_pseudo_decl(md_opt_imported), Items1),
+ queue.put_list(Items1, OptItems, Items2),
+ module_imports_set_items(Items2, !Module),
% Get the :- pragma unused_args(...) declarations created when writing
% the .opt file for the current module. These are needed because we can
@@ -2318,7 +2321,8 @@ grab_opt_files(!Module, FoundError, !IO) :-
)
->
read_optimization_interfaces(no, ModuleName, [ModuleName],
- set.init, [], LocalItems, no, UA_SR_Error, !IO),
+ set.init, [], LocalItemsLists, no, UA_SR_Error, !IO),
+ LocalItems = list.condense(LocalItemsLists),
KeepPragma = (pred(Item::in) is semidet :-
Item = item_pragma(ItemPragma),
ItemPragma = item_pragma_info(_, Pragma, _, _),
@@ -2332,8 +2336,8 @@ grab_opt_files(!Module, FoundError, !IO) :-
),
list.filter(KeepPragma, LocalItems, PragmaItems),
- module_imports_get_items(!.Module, Items2),
- list.append(Items2, PragmaItems, Items),
+ module_imports_get_items(!.Module, Items3),
+ queue.put_list(Items3, PragmaItems, Items),
module_imports_set_items(Items, !Module)
;
UA_SR_Error = no
@@ -2386,12 +2390,12 @@ grab_opt_files(!Module, FoundError, !IO) :-
:- pred read_optimization_interfaces(bool::in, module_name::in,
list(module_name)::in, set(module_name)::in,
- list(item)::in, list(item)::out, bool::in, bool::out,
+ list(list(item))::in, list(list(item))::out, bool::in, bool::out,
io::di, io::uo) is det.
-read_optimization_interfaces(_, _, [], _, !Items, !Error, !IO).
+read_optimization_interfaces(_, _, [], _, !ItemsLists, !Error, !IO).
read_optimization_interfaces(Transitive, ModuleName,
- [ModuleToRead | ModulesToRead], ModulesProcessed0, !Items, !Error,
+ [ModuleToRead | ModulesToRead], ModulesProcessed0, !ItemsLists, !Error,
!IO) :-
globals.io_lookup_bool_option(very_verbose, VeryVerbose, !IO),
maybe_write_string(VeryVerbose,
@@ -2406,7 +2410,7 @@ read_optimization_interfaces(Transitive, ModuleName,
prog_io.read_opt_file(FileName, ModuleToRead, OptItems, Specs, ModuleError,
!IO),
update_error_status(opt_file, FileName, ModuleError, Specs, !Error, !IO),
- !:Items = !.Items ++ OptItems,
+ !:ItemsLists = !.ItemsLists ++ [OptItems],
maybe_write_string(VeryVerbose, "% done.\n", !IO),
globals.io_get_globals(Globals, !IO),
@@ -2427,7 +2431,7 @@ read_optimization_interfaces(Transitive, ModuleName,
NewDeps = []
),
read_optimization_interfaces(Transitive, ModuleName,
- NewDeps ++ ModulesToRead, ModulesProcessed, !Items, !Error, !IO).
+ NewDeps ++ ModulesToRead, ModulesProcessed, !ItemsLists, !Error, !IO).
update_error_status(FileType, FileName, ModuleError, Specs,
!Error, !IO) :-
diff --git a/compiler/make.module_dep_file.m b/compiler/make.module_dep_file.m
index 90806c3..56858d2 100644
--- a/compiler/make.module_dep_file.m
+++ b/compiler/make.module_dep_file.m
@@ -56,6 +56,7 @@
:- import_module dir.
:- import_module getopt_io.
:- import_module parser.
+:- import_module queue.
:- import_module term.
:- import_module term_io.
@@ -244,7 +245,8 @@ module_dependencies_version_number = 1.
write_module_dep_file(Imports0, !IO) :-
% Make sure all the required fields are filled in.
globals.io_get_globals(Globals, !IO),
- strip_imported_items(Imports0 ^ items, Items),
+ module_imports_get_items_list(Imports0, Items0),
+ strip_imported_items(Items0, Items),
init_dependencies(Imports0 ^ source_file_name,
Imports0 ^ source_file_module_name,
Imports0 ^ nested_children, no_module_errors, Globals,
@@ -476,7 +478,7 @@ read_module_dependencies_2(RebuildModuleDeps, SearchDirs, ModuleName, !Info,
Imports ^ has_foreign_code = ContainsForeignCode,
Imports ^ foreign_import_modules = ForeignImports,
Imports ^ contains_foreign_export = ContainsForeignExport,
- Imports ^ items = [], % not used.
+ Imports ^ items = queue.init, % not used.
Imports ^ error = no_module_errors, % not used.
Imports ^ maybe_timestamps = no, % not used.
Imports ^ has_main = HasMain,
diff --git a/compiler/mercury_compile.m b/compiler/mercury_compile.m
index 530e1ec..ab714a9 100644
--- a/compiler/mercury_compile.m
+++ b/compiler/mercury_compile.m
@@ -1864,7 +1864,7 @@ pre_hlds_pass(ModuleImports0, DontWriteDFile0, HLDS1, QualInfo,
)
),
- module_imports_get_items(ModuleImports1, Items1),
+ module_imports_get_items_list(ModuleImports1, Items1),
MaybeTimestamps = ModuleImports1 ^ maybe_timestamps,
invoke_module_qualify_items(Items1, Items2, EventSpecMap1, EventSpecMap2,
diff --git a/compiler/module_imports.m b/compiler/module_imports.m
index 7967fb1..3a15cc8 100644
--- a/compiler/module_imports.m
+++ b/compiler/module_imports.m
@@ -29,6 +29,9 @@
:- import_module map.
:- import_module maybe.
:- import_module pair.
+:- import_module queue.
+
+%-----------------------------------------------------------------------------%
% When doing smart recompilation record for each module the suffix of
% the file that was read and the modification time of the file.
@@ -109,7 +112,7 @@
contains_foreign_export :: contains_foreign_export,
% The contents of the module and its imports.
- items :: list(item),
+ items :: queue(item),
% Whether an error has been encountered when reading in
% this module.
@@ -132,10 +135,14 @@
is det.
:- pred module_imports_get_impl_deps(module_imports::in,
list(module_name)::out) is det.
-:- pred module_imports_get_items(module_imports::in, list(item)::out) is det.
+:- pred module_imports_get_items(module_imports::in, queue(item)::out) is det.
+:- pred module_imports_get_items_list(module_imports::in, list(item)::out)
+ is det.
:- pred module_imports_get_error(module_imports::in, module_error::out) is det.
-:- pred module_imports_set_items(list(item)::in,
+:- pred module_imports_set_items(queue(item)::in,
+ module_imports::in, module_imports::out) is det.
+:- pred module_imports_set_items_list(list(item)::in,
module_imports::in, module_imports::out) is det.
:- pred module_imports_set_error(module_error::in,
module_imports::in, module_imports::out) is det.
@@ -226,6 +233,7 @@
:- pred get_fact_table_dependencies(list(item)::in, list(string)::out) is det.
%-----------------------------------------------------------------------------%
+%-----------------------------------------------------------------------------%
:- implementation.
@@ -236,14 +244,20 @@
:- import_module bool.
:- import_module dir.
:- import_module set.
+:- import_module svset.
:- import_module term.
+%-----------------------------------------------------------------------------%
+
module_imports_get_source_file_name(Module, Module ^ source_file_name).
module_imports_get_module_name(Module, Module ^ module_name).
module_imports_get_impl_deps(Module, Module ^ impl_deps).
module_imports_get_items(Module, Module ^ items).
+module_imports_get_items_list(Module, queue.to_list(Module ^ items)).
module_imports_get_error(Module, Module ^ error).
module_imports_set_items(Items, Module, Module ^ items := Items).
+module_imports_set_items_list(Items, Module, Module ^ items := Queue) :-
+ Queue = queue.from_list(Items).
module_imports_set_error(Error, Module, Module ^ error := Error).
module_imports_set_int_deps(IntDeps, Module, Module ^ int_deps := IntDeps).
module_imports_set_impl_deps(ImplDeps, Module,
@@ -332,26 +346,31 @@ init_dependencies(FileName, SourceFileModuleName, NestedModuleNames,
ImplementationDeps, IndirectDeps, IncludeDeps,
InterfaceIncludeDeps, NestedDeps, FactTableDeps,
ContainsForeignCode, ForeignImports, ContainsPragmaExport,
- [], Error, no, HasMain, dir.this_directory).
+ queue.init, Error, no, HasMain, dir.this_directory).
%-----------------------------------------------------------------------------%
get_dependencies(Items, ImportDeps, UseDeps) :-
- get_dependencies_implementation(Items,
- [], IntImportDeps, [], IntUseDeps, [], ImpImportDeps, [], ImpUseDeps),
+ get_dependencies_int_imp(Items, IntImportDeps, IntUseDeps,
+ ImpImportDeps, ImpUseDeps),
ImportDeps = IntImportDeps ++ ImpImportDeps,
UseDeps = IntUseDeps ++ ImpUseDeps.
get_dependencies_int_imp(Items, IntImportDeps, IntUseDeps,
ImpImportDeps, ImpUseDeps) :-
get_dependencies_implementation(Items,
- [], IntImportDeps, [], IntUseDeps, [], ImpImportDeps, [], ImpUseDeps).
+ set.init, IntImportDepsSet, set.init, IntUseDepsSet,
+ set.init, ImpImportDepsSet, set.init, ImpUseDepsSet),
+ IntImportDeps = set.to_sorted_list(IntImportDepsSet),
+ ImpImportDeps = set.to_sorted_list(ImpImportDepsSet),
+ IntUseDeps = set.to_sorted_list(IntUseDepsSet),
+ ImpUseDeps = set.to_sorted_list(ImpUseDepsSet).
:- pred get_dependencies_implementation(list(item)::in,
- list(module_name)::in, list(module_name)::out,
- list(module_name)::in, list(module_name)::out,
- list(module_name)::in, list(module_name)::out,
- list(module_name)::in, list(module_name)::out) is det.
+ set(module_name)::in, set(module_name)::out,
+ set(module_name)::in, set(module_name)::out,
+ set(module_name)::in, set(module_name)::out,
+ set(module_name)::in, set(module_name)::out) is det.
get_dependencies_implementation([],
!IntImportDeps, !IntUseDeps, !ImpImportDeps, !ImpUseDeps).
@@ -364,9 +383,9 @@ get_dependencies_implementation([Item | Items],
!IntImportDeps, !IntUseDeps, !ImpImportDeps, !ImpUseDeps)
;
( ModuleDefn = md_import(Modules) ->
- !:ImpImportDeps = !.ImpImportDeps ++ Modules
+ svset.insert_list(Modules, !ImpImportDeps)
; ModuleDefn = md_use(Modules) ->
- !:ImpUseDeps = !.ImpUseDeps ++ Modules
+ svset.insert_list(Modules, !ImpUseDeps)
;
true
),
@@ -379,10 +398,10 @@ get_dependencies_implementation([Item | Items],
).
:- pred get_dependencies_interface(list(item)::in,
- list(module_name)::in, list(module_name)::out,
- list(module_name)::in, list(module_name)::out,
- list(module_name)::in, list(module_name)::out,
- list(module_name)::in, list(module_name)::out) is det.
+ set(module_name)::in, set(module_name)::out,
+ set(module_name)::in, set(module_name)::out,
+ set(module_name)::in, set(module_name)::out,
+ set(module_name)::in, set(module_name)::out) is det.
get_dependencies_interface([],
!IntImportDeps, !IntUseDeps, !ImpImportDeps, !ImpUseDeps).
@@ -395,9 +414,9 @@ get_dependencies_interface([Item | Items],
!IntImportDeps, !IntUseDeps, !ImpImportDeps, !ImpUseDeps)
;
( ModuleDefn = md_import(Modules) ->
- !:IntImportDeps = !.IntImportDeps ++ Modules
+ svset.insert_list(Modules, !IntImportDeps)
; ModuleDefn = md_use(Modules) ->
- !:IntUseDeps = !.IntUseDeps ++ Modules
+ svset.insert_list(Modules, !IntUseDeps)
;
true
),
diff --git a/compiler/modules.m b/compiler/modules.m
index d0903e8..483a6fb 100644
--- a/compiler/modules.m
+++ b/compiler/modules.m
@@ -384,6 +384,7 @@
:- import_module map.
:- import_module multi_map.
:- import_module pair.
+:- import_module queue.
:- import_module set.
:- import_module solutions.
:- import_module sparse_bitset.
@@ -417,7 +418,7 @@ make_private_interface(SourceFileName, SourceFileModuleName, ModuleName,
"`", FileName, "' not written.\n"], !IO)
;
% Module-qualify all items.
- module_imports_get_items(Module, Items1),
+ module_imports_get_items_list(Module, Items1),
globals.io_get_globals(Globals, !IO),
module_name_to_file_name(ModuleName, ".m", do_not_create_dirs,
FileName, !IO),
@@ -531,7 +532,7 @@ make_interface(SourceFileName, SourceFileModuleName, ModuleName,
ModuleName, !.InterfaceItems, Module0, Error, !IO),
% Check whether we succeeded.
- module_imports_get_items(Module0, !:InterfaceItems),
+ module_imports_get_items_list(Module0, !:InterfaceItems),
% XXX zs: why does this code not check for fatal_module_errors?
( Error = some_module_errors ->
module_name_to_file_name(ModuleName, ".int", do_not_create_dirs,
@@ -1741,7 +1742,7 @@ grab_imported_modules(SourceFileName, SourceFileModuleName, ModuleName,
[[make_pseudo_decl(md_interface) | InterfaceItems],
[make_pseudo_decl(md_private_interface) | ImplDecls],
[make_pseudo_decl(md_implementation) | Clauses]], Items1),
- module_imports_set_items(Items1, !Module)
+ module_imports_set_items_list(Items1, !Module)
),
% Add `builtin' and `private_builtin' to the list of imported modules.
@@ -1830,7 +1831,7 @@ grab_imported_modules(SourceFileName, SourceFileModuleName, ModuleName,
make_pseudo_decl(md_abstract_imported),
!Module, !IO),
- module_imports_get_items(!.Module, Items),
+ module_imports_get_items_list(!.Module, Items),
check_imports_accessibility(ModuleName,
IntImportedModules ++ IntUsedModules ++
ImpImportedModules ++ ImpUsedModules, Items, !Specs),
@@ -1928,7 +1929,7 @@ grab_unqual_imported_modules(SourceFileName, SourceFileModuleName, ModuleName,
some [!Specs] (
!:Specs = [],
- module_imports_get_items(!.Module, Items),
+ module_imports_get_items_list(!.Module, Items),
check_imports_accessibility(ModuleName,
IntImportDeps ++ IntUseDeps ++ ImpImportDeps ++ ImpUseDeps,
Items, !Specs),
@@ -1943,7 +1944,7 @@ grab_unqual_imported_modules(SourceFileName, SourceFileModuleName, ModuleName,
append_pseudo_decl(PseudoDecl, Module0, Module) :-
Items0 = Module0 ^ items,
- list.append(Items0, [make_pseudo_decl(PseudoDecl)], Items),
+ queue.put(Items0, make_pseudo_decl(PseudoDecl), Items),
Module = Module0 ^ items := Items.
make_pseudo_decl(PseudoDecl) = Item :-
@@ -2787,7 +2788,7 @@ process_module_private_interfaces(ReadModules, [Ancestor | Ancestors],
get_dependencies(Items, AncDirectImports, AncDirectUses),
!:DirectImports = !.DirectImports ++ AncDirectImports,
!:DirectUses = !.DirectUses ++ AncDirectUses,
- ModItems = ModItems0 ++ Items,
+ queue.put_list(ModItems0, Items, ModItems),
!:Module = !.Module ^ items := ModItems,
!:Module = !.Module ^ parent_deps := ModAncestors,
!:Module = !.Module ^ error := ModError,
@@ -2846,7 +2847,7 @@ process_module_long_interfaces(ReadModules, NeedQualifier, [Import | Imports],
++ IndirectUses1,
!:ImplIndirectImports = !.ImplIndirectImports
++ ImplIndirectImports1 ++ ImplIndirectUses1,
- ModItems = ModItems0 ++ Items,
+ queue.put_list(ModItems0, Items, ModItems),
!:Module = !.Module ^ impl_deps := ModImplementationImports,
!:Module = !.Module ^ items := ModItems,
!:Module = !.Module ^ error := ModError,
@@ -3021,7 +3022,7 @@ process_module_short_interfaces(ReadModules, [Import | Imports], Ext,
ModIndirectImports = [Import | ModIndirectImports0],
!:IndirectImports = !.IndirectImports ++ IntImports1 ++ IntUses1,
!:ImpIndirectImports = !.ImpIndirectImports ++ ImpImports1 ++ ImpUses1,
- ModItems = ModItems0 ++ Items,
+ queue.put_list(ModItems0, Items, ModItems),
!:Module = !.Module ^ indirect_deps := ModIndirectImports,
!:Module = !.Module ^ items := ModItems,
!:Module = !.Module ^ error := ModError,
@@ -3411,10 +3412,11 @@ init_module_imports(SourceFileName, SourceFileModuleName, ModuleName,
% module_imports.m is this call. This should be fixed, preferably
% by changing the module_imports structure.
maybe_add_foreign_import_module(ModuleName, Items0, Items),
+ ItemsQueue = queue.from_list(Items),
Module = module_imports(SourceFileName, SourceFileModuleName,
ModuleName, [], [], [], [], [], PublicChildren,
NestedChildren, FactDeps, contains_foreign_code_unknown, [],
- contains_no_foreign_export, Items, no_module_errors,
+ contains_no_foreign_export, ItemsQueue, no_module_errors,
MaybeTimestamps, no_main, dir.this_directory).
:- pred maybe_add_foreign_import_module(module_name::in,
diff --git a/compiler/trans_opt.m b/compiler/trans_opt.m
index cf5bbb8..c53cf0e 100644
--- a/compiler/trans_opt.m
+++ b/compiler/trans_opt.m
@@ -110,6 +110,7 @@
:- import_module list.
:- import_module map.
:- import_module pair.
+:- import_module queue.
:- import_module set.
:- import_module term.
@@ -201,7 +202,7 @@ grab_trans_opt_files(TransOptDeps, !Module, FoundError, !IO) :-
append_pseudo_decl(md_opt_imported, !Module),
module_imports_get_items(!.Module, Items0),
- list.append(Items0, OptItems, Items),
+ queue.put_list(Items0, OptItems, Items),
module_imports_set_items(Items, !Module),
module_imports_set_error(no_module_errors, !Module),
diff --git a/compiler/write_deps_file.m b/compiler/write_deps_file.m
index dee2a1b..7fce5cb 100644
--- a/compiler/write_deps_file.m
+++ b/compiler/write_deps_file.m
@@ -75,6 +75,7 @@
string::in, list(module_name)::out, io::di, io::uo) is det.
%-----------------------------------------------------------------------------%
+%-----------------------------------------------------------------------------%
:- implementation.
@@ -96,13 +97,16 @@
:- import_module map.
:- import_module string.
:- import_module pair.
+:- import_module queue.
+
+%-----------------------------------------------------------------------------%
write_dependency_file(Module, AllDepsSet, MaybeTransOptDeps, !IO) :-
Module = module_imports(SourceFileName, SourceFileModuleName,
ModuleName, ParentDeps, IntDeps, ImplDeps, IndirectDeps,
_Children, InclDeps, NestedDeps, FactDeps0,
ContainsForeignCode, ForeignImports0, _ContainsForeignExport,
- Items, _Error, _Timestamps, _HasMain, _Dir),
+ ItemsQueue, _Error, _Timestamps, _HasMain, _Dir),
globals.io_lookup_bool_option(verbose, Verbose, !IO),
module_name_to_make_var_name(ModuleName, MakeVarName),
@@ -513,6 +517,7 @@ write_dependency_file(Module, AllDepsSet, MaybeTransOptDeps, !IO) :-
ForeignImports = ForeignImports0
;
ContainsForeignCode = contains_foreign_code_unknown,
+ Items = queue.to_list(ItemsQueue),
get_item_list_foreign_code(Globals, Items, LangSet,
ForeignImports1, _),
% If we're generating the `.dep' file, ForeignImports0 will contain
--------------------------------------------------------------------------
mercury-reviews mailing list
Post messages to: mercury-reviews at csse.unimelb.edu.au
Administrative Queries: owner-mercury-reviews at csse.unimelb.edu.au
Subscriptions: mercury-reviews-request at csse.unimelb.edu.au
--------------------------------------------------------------------------
More information about the reviews
mailing list