Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Split up the compiler DAG #2217

Merged
merged 1 commit into from
Feb 5, 2020
Merged

Split up the compiler DAG #2217

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
257 changes: 54 additions & 203 deletions src/rebar_compiler.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,18 +30,35 @@
ok | {ok, [string()]} | {ok, [string()], [string()]}.
-callback clean([file:filename()], rebar_app_info:t()) -> _.

-define(DAG_VSN, 2).
-define(DAG_ROOT, "source").
-define(DAG_EXT, ".dag").
-type dag_v() :: {digraph:vertex(), term()} | 'false'.
-type dag_e() :: {digraph:vertex(), digraph:vertex()}.
-type dag() :: {list(dag_v()), list(dag_e()), list(string())}.
-record(dag, {vsn = ?DAG_VSN :: pos_integer(),
info = {[], [], []} :: dag()}).

-define(RE_PREFIX, "^(?!\\._)").

compile_all(Compilers, AppInfo) ->
-spec compile_all([{module(), digraph:graph()}, ...], rebar_app_info:t()) -> ok
; ([module(), ...], rebar_app_info:t()) -> ok.
compile_all(DAGs, AppInfo) when is_tuple(hd(DAGs)) -> % > 3.13.0
prepare_compiler_env(AppInfo),
lists:foreach(fun({Compiler, G}) ->
run(G, Compiler, AppInfo),
%% TODO: disable default recursivity in extra_src_dirs compiling to
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we really merge this without this disabled by default?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah it's what we have today. See the complaints about modules auto-compiled by CT. I just added the note in comments to fix this long-standing bug.

%% prevent compiling sample modules in _SUITE_data/ directories
%% in CT.
ExtraApps = annotate_extras(AppInfo),
[run(G, Compiler, ExtraAppInfo) || ExtraAppInfo <- ExtraApps],
ok
end,
DAGs);
compile_all(Compilers, AppInfo) -> % =< 3.13.0 interface; plugins use this!
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should decide on how many versions back of rebar3 to support and figure out a way to give useful error messages when plugins are outdated.

Doesn't need to be a part of this PR, just bringing it up because this is adding to the already extensive technical debt for plugin support.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah. To some extent I have never taken the time to update the rebar3_proper plugin myself so I feel in a bad place to force others to do it as well. It still displays the cool warning every time you run it.

%% Support the old-style API by re-declaring a local DAG for the
%% compile steps needed.
lists:foreach(fun(Compiler) ->
OutDir = rebar_app_info:out_dir(AppInfo),
G = rebar_compiler_dag:init(OutDir, Compiler, undefined, []),
compile_all([{Compiler, G}], AppInfo),
rebar_compiler_dag:maybe_store(G, OutDir, Compiler, undefined, []),
rebar_compiler_dag:terminate(G)
end, Compilers).

prepare_compiler_env(AppInfo) ->
EbinDir = rebar_utils:to_list(rebar_app_info:ebin_dir(AppInfo)),
%% Make sure that outdir is on the path
ok = rebar_file_utils:ensure_dir(EbinDir),
Expand All @@ -51,15 +68,9 @@ compile_all(Compilers, AppInfo) ->
%% called here for clarity as it's required by both opts_changed/2
%% and erl_compiler_opts_set/0 in needed_files
_ = code:ensure_loaded(compile),

lists:foreach(fun(CompilerMod) ->
run(CompilerMod, AppInfo, undefined),
run_on_extra_src_dirs(CompilerMod, AppInfo,
fun(Mod, App) -> run(Mod, App, "extra") end)
end, Compilers),
ok.

run(CompilerMod, AppInfo, Label) ->
run(G, CompilerMod, AppInfo) ->
#{src_dirs := SrcDirs,
include_dirs := InclDirs,
src_ext := SrcExt,
Expand All @@ -72,12 +83,14 @@ run(CompilerMod, AppInfo, Label) ->
AbsInclDirs = [filename:join(BaseDir, InclDir) || InclDir <- InclDirs],
FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, BaseOpts),

OutDir = rebar_app_info:out_dir(AppInfo),
AbsSrcDirs = [filename:join(BaseDir, SrcDir) || SrcDir <- SrcDirs],
G = init_dag(CompilerMod, AbsInclDirs, AbsSrcDirs, FoundFiles, OutDir, EbinDir, Label),
{{FirstFiles, FirstFileOpts}, {RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles,
Mappings, AppInfo),
true = digraph:delete(G),

InDirs = lists:usort(AbsInclDirs ++ AbsSrcDirs),

rebar_compiler_dag:prune(G, AbsSrcDirs, EbinDir, FoundFiles),
rebar_compiler_dag:update(G, CompilerMod, InDirs, FoundFiles),
{{FirstFiles, FirstFileOpts},
{RestFiles, Opts}} = CompilerMod:needed_files(G, FoundFiles, Mappings, AppInfo),

compile_each(FirstFiles, FirstFileOpts, BaseOpts, Mappings, CompilerMod),
case RestFiles of
Expand Down Expand Up @@ -167,20 +180,19 @@ compile_queue(Targets, Pids, Opts, Config, Outs, CompilerMod) ->
clean(Compilers, AppInfo) ->
lists:foreach(fun(CompilerMod) ->
clean_(CompilerMod, AppInfo, undefined),
run_on_extra_src_dirs(CompilerMod, AppInfo,
fun(Mod, App) -> clean_(Mod, App, "extra") end)
Extras = annotate_extras(AppInfo),
[clean_(CompilerMod, ExtraApp, "extra") || ExtraApp <- Extras]
end, Compilers).

clean_(CompilerMod, AppInfo, Label) ->
clean_(CompilerMod, AppInfo, _Label) ->
#{src_dirs := SrcDirs,
src_ext := SrcExt} = CompilerMod:context(AppInfo),
BaseDir = rebar_app_info:dir(AppInfo),
Opts = rebar_app_info:opts(AppInfo),
EbinDir = rebar_app_info:ebin_dir(AppInfo),

FoundFiles = find_source_files(BaseDir, SrcExt, SrcDirs, Opts),
CompilerMod:clean(FoundFiles, AppInfo),
rebar_file_utils:rm_rf(dag_file(CompilerMod, EbinDir, Label)).
ok.

-spec needs_compile(filename:all(), extension(), [{extension(), file:dirname()}]) -> boolean().
needs_compile(Source, OutExt, Mappings) ->
Expand All @@ -190,30 +202,23 @@ needs_compile(Source, OutExt, Mappings) ->
Target = filename:join(OutDir, BaseName++OutExt),
filelib:last_modified(Source) > filelib:last_modified(Target).

run_on_extra_src_dirs(CompilerMod, AppInfo, Fun) ->
annotate_extras(AppInfo) ->
ExtraDirs = rebar_dir:extra_src_dirs(rebar_app_info:opts(AppInfo), []),
run_on_extra_src_dirs(ExtraDirs, CompilerMod, AppInfo, Fun).

run_on_extra_src_dirs([], _CompilerMod, _AppInfo, _Fun) ->
ok;
run_on_extra_src_dirs([Dir | Rest], CompilerMod, AppInfo, Fun) ->
case filelib:is_dir(filename:join(rebar_app_info:dir(AppInfo), Dir)) of
true ->
OldSrcDirs = rebar_app_info:get(AppInfo, src_dirs, ["src"]),
AppDir = rebar_app_info:dir(AppInfo),
EbinDir = filename:join(rebar_app_info:out_dir(AppInfo), Dir),
AppInfo1 = rebar_app_info:ebin_dir(AppInfo, EbinDir),
AppInfo2 = rebar_app_info:set(AppInfo1, src_dirs, [Dir]),
AppInfo3 = rebar_app_info:set(AppInfo2, extra_src_dirs, OldSrcDirs),
AppInfo4 = add_to_includes( % give access to .hrl in app's src/
AppInfo3,
[filename:join([AppDir, D]) || D <- OldSrcDirs]
),
Fun(CompilerMod, AppInfo4);
_ ->
ok
OldSrcDirs = rebar_app_info:get(AppInfo, src_dirs, ["src"]),
AppDir = rebar_app_info:dir(AppInfo),
lists:map(fun(Dir) ->
EbinDir = filename:join(rebar_app_info:out_dir(AppInfo), Dir),
AppInfo1 = rebar_app_info:ebin_dir(AppInfo, EbinDir),
AppInfo2 = rebar_app_info:set(AppInfo1, src_dirs, [Dir]),
AppInfo3 = rebar_app_info:set(AppInfo2, extra_src_dirs, OldSrcDirs),
add_to_includes( % give access to .hrl in app's src/
AppInfo3,
[filename:join([AppDir, D]) || D <- OldSrcDirs]
)
end,
run_on_extra_src_dirs(Rest, CompilerMod, AppInfo, Fun).
[ExtraDir || ExtraDir <- ExtraDirs,
filelib:is_dir(filename:join(AppDir, ExtraDir))]
).

%% These functions are here for the ultimate goal of getting rid of
%% rebar_base_compiler. This can't be done because of existing plugins.
Expand All @@ -233,7 +238,7 @@ format_error_source(Path, Opts) ->
report(Messages) ->
rebar_base_compiler:report(Messages).

%% private functions
%%% private functions

find_source_files(BaseDir, SrcExt, SrcDirs, Opts) ->
SourceExtRe = "^(?!\\._).*\\" ++ SrcExt ++ [$$],
Expand All @@ -242,160 +247,6 @@ find_source_files(BaseDir, SrcExt, SrcDirs, Opts) ->
rebar_utils:find_files_in_dirs([filename:join(BaseDir, SrcDir)], SourceExtRe, Recursive)
end, SrcDirs).

%% @private generate the name for the DAG based on the compiler module and
%% a custom label, both of which are used to prevent various compiler runs
%% from clobbering each other. The label `undefined' is kept for a default
%% run of the compiler, to keep in line with previous versions of the file.
dag_file(CompilerMod, Dir, undefined) ->
filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod,
?DAG_ROOT ++ ?DAG_EXT]);
dag_file(CompilerMod, Dir, Label) ->
filename:join([rebar_dir:local_cache_dir(Dir), CompilerMod,
?DAG_ROOT ++ "_" ++ Label ++ ?DAG_EXT]).

%% private graph functions

%% Get dependency graph of given Erls files and their dependencies (header files,
%% parse transforms, behaviours etc.) located in their directories or given
%% InclDirs. Note that last modification times stored in vertices already respect
%% dependencies induced by given graph G.
init_dag(Compiler, InclDirs, SrcDirs, Erls, Dir, EbinDir, Label) ->
G = digraph:new([acyclic]),
try restore_dag(Compiler, G, InclDirs, Dir, Label)
catch
_:_ ->
?WARN("Failed to restore ~ts file. Discarding it.~n", [dag_file(Compiler, Dir, Label)]),
file:delete(dag_file(Compiler, Dir, Label))
end,
Dirs = lists:usort(InclDirs ++ SrcDirs),
%% A source file may have been renamed or deleted. Remove it from the graph
%% and remove any beam file for that source if it exists.
Modified = maybe_rm_beams_and_edges(G, EbinDir, Erls),
Modified1 = lists:foldl(update_dag_fun(G, Compiler, Dirs), Modified, Erls),
if Modified1 -> store_dag(Compiler, G, InclDirs, Dir, Label);
not Modified1 -> ok
end,
G.

maybe_rm_beams_and_edges(G, Dir, Files) ->
Vertices = digraph:vertices(G),
case lists:filter(fun(File) ->
case filename:extension(File) =:= ".erl" of
true ->
maybe_rm_beam_and_edge(G, Dir, File);
false ->
false
end
end, lists:sort(Vertices) -- lists:sort(Files)) of
[] ->
false;
_ ->
true
end.

maybe_rm_beam_and_edge(G, OutDir, Source) ->
%% This is NOT a double check it is the only check that the source file is actually gone
case filelib:is_regular(Source) of
true ->
%% Actually exists, don't delete
false;
false ->
Target = target_base(OutDir, Source) ++ ".beam",
?DEBUG("Source ~ts is gone, deleting previous beam file if it exists ~ts", [Source, Target]),
file:delete(Target),
digraph:del_vertex(G, Source),
true
end.


target_base(OutDir, Source) ->
filename:join(OutDir, filename:basename(Source, ".erl")).

restore_dag(Compiler, G, InclDirs, Dir, Label) ->
case file:read_file(dag_file(Compiler, Dir, Label)) of
{ok, Data} ->
% Since externally passed InclDirs can influence dependency graph (see
% modify_dag), we have to check here that they didn't change.
#dag{vsn=?DAG_VSN, info={Vs, Es, InclDirs}} =
binary_to_term(Data),
lists:foreach(
fun({V, LastUpdated}) ->
digraph:add_vertex(G, V, LastUpdated)
end, Vs),
lists:foreach(
fun({_, V1, V2, _}) ->
digraph:add_edge(G, V1, V2)
end, Es);
{error, _} ->
ok
end.

store_dag(Compiler, G, InclDirs, Dir, Label) ->
Vs = lists:map(fun(V) -> digraph:vertex(G, V) end, digraph:vertices(G)),
Es = lists:map(fun(E) -> digraph:edge(G, E) end, digraph:edges(G)),
File = dag_file(Compiler, Dir, Label),
ok = filelib:ensure_dir(File),
Data = term_to_binary(#dag{info={Vs, Es, InclDirs}}, [{compressed, 2}]),
file:write_file(File, Data).

update_dag(G, Compiler, Dirs, Source) ->
case digraph:vertex(G, Source) of
{_, LastUpdated} ->
case filelib:last_modified(Source) of
0 ->
%% The file doesn't exist anymore,
%% erase it from the graph.
%% All the edges will be erased automatically.
digraph:del_vertex(G, Source),
modified;
LastModified when LastUpdated < LastModified ->
modify_dag(G, Compiler, Source, LastModified, filename:dirname(Source), Dirs);
_ ->
Modified = lists:foldl(
update_dag_fun(G, Compiler, Dirs),
false, digraph:out_neighbours(G, Source)),
MaxModified = update_max_modified_deps(G, Source),
case Modified orelse MaxModified > LastUpdated of
true -> modified;
false -> unmodified
end
end;
false ->
modify_dag(G, Compiler, Source, filelib:last_modified(Source), filename:dirname(Source), Dirs)
end.

modify_dag(G, Compiler, Source, LastModified, SourceDir, Dirs) ->
AbsIncls = Compiler:dependencies(Source, SourceDir, Dirs),
digraph:add_vertex(G, Source, LastModified),
digraph:del_edges(G, digraph:out_edges(G, Source)),
lists:foreach(
fun(Incl) ->
update_dag(G, Compiler, Dirs, Incl),
digraph:add_edge(G, Source, Incl)
end, AbsIncls),
modified.

update_dag_fun(G, Compiler, Dirs) ->
fun(Erl, Modified) ->
case update_dag(G, Compiler, Dirs, Erl) of
modified -> true;
unmodified -> Modified
end
end.

update_max_modified_deps(G, Source) ->
MaxModified =
lists:foldl(fun(File, Acc) ->
case digraph:vertex(G, File) of
{_, MaxModified} when MaxModified > Acc ->
MaxModified;
_ ->
Acc
end
end, 0, [Source | digraph:out_neighbours(G, Source)]),
digraph:add_vertex(G, Source, MaxModified),
MaxModified.

add_to_includes(AppInfo, Dirs) ->
Opts = rebar_app_info:opts(AppInfo),
List = rebar_opts:get(Opts, erl_opts, []),
Expand Down
Loading