Skip to content

Commit

Permalink
Merge pull request #2217 from ferd/merge-analysis-phases
Browse files Browse the repository at this point in the history 
Split up the compiler DAG
  • Loading branch information
@ferd
ferd authored Feb 5, 2020
2 parents fe837e8 + 3b3201a commit a524fe3
Show file tree
Hide file tree
Showing 3 changed files with 344 additions and 238 deletions.
257 changes: 54 additions & 203 deletions src/rebar_compiler.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,18 +30,35 @@
ok | {ok, [string()]} | {ok, [string()], [string()]}.
-callback clean([file:filename()], rebar_app_info:t()) -> _.

-define(DAG_VSN, 2).
-define(DAG_ROOT, "source").
-define(DAG_EXT, ".dag").
-type dag_v() :: {digraph:vertex(), term()} | 'false'.
-type dag_e() :: {digraph:vertex(), digraph:vertex()}.
-type dag() :: {list(dag_v()), list(dag_e()), list(string())}.
-record(dag, {vsn = ?DAG_VSN :: pos_integer(),
info = {[], [], []} :: dag()}).

-define(RE_PREFIX, "^(?!\\._)").

compile_all(Compilers, AppInfo) ->
-spec compile_all([{module(), digraph:graph()}, ...], rebar_app_info:t()) -> ok
; ([module(), ...], rebar_app_info:t()) -> ok.
compile_all(DAGs, AppInfo) when is_tuple(hd(DAGs)) -> % > 3.13.0
prepare_compiler_env(AppInfo),
lists:foreach(fun({Compiler, G}) ->
run(G, Compiler, AppInfo),
%% TODO: disable default recursivity in extra_src_dirs compiling to
%% prevent compiling sample modules in _SUITE_data/ directories
%% in CT.
ExtraApps = annotate_extras(AppInfo),
[run(G, Compiler, ExtraAppInfo) || ExtraAppInfo <- ExtraApps],
ok
end,
DAGs);
compile_all(Compilers, AppInfo) -> % =< 3.13.0 interface; plugins use this!
%% Support the old-style API by re-declaring a local DAG for the
%% compile steps needed.
lists:foreach(fun(Compiler) ->
OutDir = rebar_app_info:out_dir(AppInfo),
G = rebar_compiler_dag:init(OutDir, Compiler, undefined, []),
compile_all([{Compiler, G}], AppInfo),
rebar_compiler_dag:maybe_store(G, OutDir, Compiler, undefined, []),
rebar_compiler_dag:terminate(G)
end, Compilers).

prepare_compiler_env(AppInfo) ->
EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)),
%% Make sure that outdir is on the path
ok = rebar_file_utils:ensure_dir(EbinDir),
Expand All @@ -51,15 +68,9 @@ compile_all(Compilers, AppInfo) ->
%% called here for clarity as it's required by both opts_changed/2
%% and erl_compiler_opts_set/0 in needed_files
_ = code:ensure_loaded(compile),

lists:foreach(fun(CompilerMod) ->
run(CompilerMod, AppInfo, undefined),
run_on_extra_src_dirs(CompilerMod, AppInfo,
fun(Mod, App) -> run(Mod, App, "extra") end)
end, Compilers),
ok.

run(CompilerMod, AppInfo, Label) ->
run(G, CompilerMod, AppInfo) ->
#{src_dirs := SrcDirs,
include_dirs := InclDirs,
src_ext := SrcExt,
Expand All @@ -72,12 +83,14 @@ run(CompilerMod, AppInfo, Label) ->
AbsInclDirs = [filename:join(BaseDir, InclDir) || InclDir <- InclDirs],
FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, BaseOpts),

OutDir = rebar_app_info:out_dir(AppInfo),
AbsSrcDirs = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs],
G = init_dag(CompilerMod, AbsInclDirs, AbsSrcDirs, FoundFiles, OutDir, EbinDir, Label),
{{FirstFiles, FirstFileOpts}, {RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles,
Mappings, AppInfo),
true = digraph:delete(G),

InDirs = lists:usort(AbsInclDirs ++ AbsSrcDirs),

rebar_compiler_dag:prune(G, AbsSrcDirs, EbinDir, FoundFiles),
rebar_compiler_dag:update(G, CompilerMod, InDirs, FoundFiles),
{{FirstFiles, FirstFileOpts},
{RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles, Mappings, AppInfo),

compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod),
case RestFiles of
Expand Down Expand Up @@ -167,20 +180,19 @@ compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod) ->
clean(Compilers, AppInfo) ->
lists:foreach(fun(CompilerMod) ->
clean_(CompilerMod, AppInfo, undefined),
run_on_extra_src_dirs(CompilerMod, AppInfo,
fun(Mod, App) -> clean_(Mod, App, "extra") end)
Extras = annotate_extras(AppInfo),
[clean_(CompilerMod, ExtraApp, "extra") || ExtraApp <- Extras]
end, Compilers).

clean_(CompilerMod, AppInfo, Label) ->
clean_(CompilerMod, AppInfo, _Label) ->
#{src_dirs := SrcDirs,
src_ext := SrcExt} = CompilerMod:context(AppInfo),
BaseDir = rebar_app_info:dir(AppInfo),
Opts = rebar_app_info:opts(AppInfo),
EbinDir = rebar_app_info:ebin_dir(AppInfo),

FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, Opts),
CompilerMod:clean(FoundFiles, AppInfo),
rebar_file_utils:rm_rf(dag_file(CompilerMod, EbinDir, Label)).
ok.

-spec needs_compile(filename:all(), extension(), [{extension(), file:dirname()}]) -> boolean().
needs_compile(Source, OutExt, Mappings) ->
Expand All @@ -190,30 +202,23 @@ needs_compile(Source, OutExt, Mappings) ->
Target = filename:join(OutDir, BaseName++OutExt),
filelib:last_modified(Source) > filelib:last_modified(Target).

run_on_extra_src_dirs(CompilerMod, AppInfo, Fun) ->
annotate_extras(AppInfo) ->
ExtraDirs = rebar_dir:extra_src_dirs(rebar_app_info:opts(AppInfo), []),
run_on_extra_src_dirs(ExtraDirs, CompilerMod, AppInfo, Fun).

run_on_extra_src_dirs([], _CompilerMod, _AppInfo, _Fun) ->
ok;
run_on_extra_src_dirs([Dir | Rest], CompilerMod, AppInfo, Fun) ->
case filelib:is_dir(filename:join(rebar_app_info:dir(AppInfo), Dir)) of
true ->
OldSrcDirs = rebar_app_info:get(AppInfo, src_dirs, ["src"]),
AppDir = rebar_app_info:dir(AppInfo),
EbinDir = filename:join(rebar_app_info:out_dir(AppInfo), Dir),
AppInfo1 = rebar_app_info:ebin_dir(AppInfo, EbinDir),
AppInfo2 = rebar_app_info:set(AppInfo1, src_dirs, [Dir]),
AppInfo3 = rebar_app_info:set(AppInfo2, extra_src_dirs, OldSrcDirs),
AppInfo4 = add_to_includes( % give access to .hrl in app's src/
AppInfo3,
[filename:join([AppDir, D]) || D <- OldSrcDirs]
),
Fun(CompilerMod, AppInfo4);
_ ->
ok
OldSrcDirs = rebar_app_info:get(AppInfo, src_dirs, ["src"]),
AppDir = rebar_app_info:dir(AppInfo),
lists:map(fun(Dir) ->
EbinDir = filename:join(rebar_app_info:out_dir(AppInfo), Dir),
AppInfo1 = rebar_app_info:ebin_dir(AppInfo, EbinDir),
AppInfo2 = rebar_app_info:set(AppInfo1, src_dirs, [Dir]),
AppInfo3 = rebar_app_info:set(AppInfo2, extra_src_dirs, OldSrcDirs),
add_to_includes( % give access to .hrl in app's src/
AppInfo3,
[filename:join([AppDir, D]) || D <- OldSrcDirs]
)
end,
run_on_extra_src_dirs(Rest, CompilerMod, AppInfo, Fun).
[ExtraDir || ExtraDir <- ExtraDirs,
filelib:is_dir(filename:join(AppDir, ExtraDir))]
).

%% These functions are here for the ultimate goal of getting rid of
%% rebar_base_compiler. This can't be done because of existing plugins.
Expand All @@ -233,7 +238,7 @@ format_error_source(Path, Opts) ->
report(Messages) ->
rebar_base_compiler:report(Messages).

%% private functions
%%% private functions

find_source_files(BaseDir, SrcExt, SrcDirs, Opts) ->
SourceExtRe = "^(?!\\._).*\\" ++ SrcExt ++ [$$],
Expand All @@ -242,160 +247,6 @@ find_source_files(BaseDir, SrcExt, SrcDirs, Opts) ->
rebar_utils:find_files_in_dirs([filename:join(BaseDir, SrcDir)], SourceExtRe, Recursive)
end, SrcDirs).

%% @private generate the name for the DAG based on the compiler module and
%% a custom label, both of which are used to prevent various compiler runs
%% from clobbering each other. The label `undefined' is kept for a default
%% run of the compiler, to keep in line with previous versions of the file.
dag_file(CompilerMod, Dir, undefined) ->
filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod,
?DAG_ROOT ++ ?DAG_EXT]);
dag_file(CompilerMod, Dir, Label) ->
filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod,
?DAG_ROOT ++ "_" ++ Label ++ ?DAG_EXT]).

%% private graph functions

%% Get dependency graph of given Erls files and their dependencies (header files,
%% parse transforms, behaviours etc.) located in their directories or given
%% InclDirs. Note that last modification times stored in vertices already respect
%% dependencies induced by given graph G.
init_dag(Compiler, InclDirs, SrcDirs, Erls, Dir, EbinDir, Label) ->
G = digraph:new([acyclic]),
try restore_dag(Compiler, G, InclDirs, Dir, Label)
catch
_:_ ->
?WARN("Failed to restore ~ts file. Discarding it.~n", [dag_file(Compiler, Dir, Label)]),
file:delete(dag_file(Compiler, Dir, Label))
end,
Dirs = lists:usort(InclDirs ++ SrcDirs),
%% A source file may have been renamed or deleted. Remove it from the graph
%% and remove any beam file for that source if it exists.
Modified = maybe_rm_beams_and_edges(G, EbinDir, Erls),
Modified1 = lists:foldl(update_dag_fun(G, Compiler, Dirs), Modified, Erls),
if Modified1 -> store_dag(Compiler, G, InclDirs, Dir, Label);
not Modified1 -> ok
end,
G.

maybe_rm_beams_and_edges(G, Dir, Files) ->
Vertices = digraph:vertices(G),
case lists:filter(fun(File) ->
case filename:extension(File) =:= ".erl" of
true ->
maybe_rm_beam_and_edge(G, Dir, File);
false ->
false
end
end, lists:sort(Vertices) -- lists:sort(Files)) of
[] ->
false;
_ ->
true
end.

maybe_rm_beam_and_edge(G, OutDir, Source) ->
%% This is NOT a double check it is the only check that the source file is actually gone
case filelib:is_regular(Source) of
true ->
%% Actually exists, don't delete
false;
false ->
Target = target_base(OutDir, Source) ++ ".beam",
?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]),
file:delete(Target),
digraph:del_vertex(G, Source),
true
end.


target_base(OutDir, Source) ->
filename:join(OutDir, filename:basename(Source, ".erl")).

restore_dag(Compiler, G, InclDirs, Dir, Label) ->
case file:read_file(dag_file(Compiler, Dir, Label)) of
{ok, Data} ->
% Since externally passed InclDirs can influence dependency graph (see
% modify_dag), we have to check here that they didn't change.
#dag{vsn=?DAG_VSN, info={Vs, Es, InclDirs}} =
binary_to_term(Data),
lists:foreach(
fun({V, LastUpdated}) ->
digraph:add_vertex(G, V, LastUpdated)
end, Vs),
lists:foreach(
fun({_, V1, V2, _}) ->
digraph:add_edge(G, V1, V2)
end, Es);
{error, _} ->
ok
end.

store_dag(Compiler, G, InclDirs, Dir, Label) ->
Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)),
Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)),
File = dag_file(Compiler, Dir, Label),
ok = filelib:ensure_dir(File),
Data = term_to_binary(#dag{info={Vs, Es, InclDirs}}, [{compressed, 2}]),
file:write_file(File, Data).

update_dag(G, Compiler, Dirs, Source) ->
case digraph:vertex(G, Source) of
{_, LastUpdated} ->
case filelib:last_modified(Source) of
0 ->
%% The file doesn't exist anymore,
%% erase it from the graph.
%% All the edges will be erased automatically.
digraph:del_vertex(G, Source),
modified;
LastModified when LastUpdated < LastModified ->
modify_dag(G, Compiler, Source, LastModified, filename:dirname(Source), Dirs);
_ ->
Modified = lists:foldl(
update_dag_fun(G, Compiler, Dirs),
false, digraph:out_neighbours(G, Source)),
MaxModified = update_max_modified_deps(G, Source),
case Modified orelse MaxModified > LastUpdated of
true -> modified;
false -> unmodified
end
end;
false ->
modify_dag(G, Compiler, Source, filelib:last_modified(Source), filename:dirname(Source), Dirs)
end.

modify_dag(G, Compiler, Source, LastModified, SourceDir, Dirs) ->
AbsIncls = Compiler:dependencies(Source, SourceDir, Dirs),
digraph:add_vertex(G, Source, LastModified),
digraph:del_edges(G, digraph:out_edges(G, Source)),
lists:foreach(
fun(Incl) ->
update_dag(G, Compiler, Dirs, Incl),
digraph:add_edge(G, Source, Incl)
end, AbsIncls),
modified.

update_dag_fun(G, Compiler, Dirs) ->
fun(Erl, Modified) ->
case update_dag(G, Compiler, Dirs, Erl) of
modified -> true;
unmodified -> Modified
end
end.

update_max_modified_deps(G, Source) ->
MaxModified =
lists:foldl(fun(File, Acc) ->
case digraph:vertex(G, File) of
{_, MaxModified} when MaxModified > Acc ->
MaxModified;
_ ->
Acc
end
end, 0, [Source | digraph:out_neighbours(G, Source)]),
digraph:add_vertex(G, Source, MaxModified),
MaxModified.

add_to_includes(AppInfo, Dirs) ->
Opts = rebar_app_info:opts(AppInfo),
List = rebar_opts:get(Opts, erl_opts, []),
Expand Down
Loading

0 comments on commit a524fe3

Please sign in to comment.