From 11538ac791ef2f396167688b79716554ca53e0a5 Mon Sep 17 00:00:00 2001 From: thomasdavid Date: Fri, 12 Feb 2021 07:29:38 -0800 Subject: [PATCH 01/26] {xref_ignores} now properly works on module, by taking into account function calls. Summary: Before, the xref_ignores would properly ignores function calls on specified modules. That is, even though {xref_ignores, [ModName,...]} would be in rebar3.config, those unassigned function calls to and from the ModName module would be displayed. This was due to a mismatch between the format of the ignore (a module name), and the format of the warning ({MFA, MFA} or MFA) emmited by xref. --- src/rebar_prv_xref.erl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/rebar_prv_xref.erl b/src/rebar_prv_xref.erl index a720d8465..bae505f79 100644 --- a/src/rebar_prv_xref.erl +++ b/src/rebar_prv_xref.erl @@ -180,9 +180,6 @@ get_behaviour_callbacks(exports_not_used, Attributes) -> get_behaviour_callbacks(_XrefCheck, _Attributes) -> []. -parse_xref_result({_, MFAt}) -> MFAt; -parse_xref_result(MFAt) -> MFAt. - filter_xref_results(XrefCheck, XrefIgnores, XrefResults) -> SearchModules = lists:usort( lists:map( @@ -194,10 +191,16 @@ filter_xref_results(XrefCheck, XrefIgnores, XrefResults) -> Ignores = XrefIgnores ++ lists:flatmap(fun(Module) -> get_xref_ignorelist(Module, XrefCheck) end, SearchModules), + lists:filter( fun(Result) -> pred_xref_result(Result, Ignores) end, XrefResults). + +pred_xref_result({Src, Dest}, Ignores) -> pred_xref_result1(Src, Ignores) + andalso pred_xref_result1(Dest, Ignores); +pred_xref_result(Vertex, Ignores) -> pred_xref_result1(Vertex, Ignores). - [Result || Result <- XrefResults, - not lists:member(element(1, Result), Ignores) - andalso not lists:member(parse_xref_result(Result), Ignores)]. +pred_xref_result1(Vertex, Ignores) -> + Mod = case Vertex of {Module, _Func, _Arity} -> Module; + _ -> Vertex end, + not lists:member(Vertex, Ignores) andalso not lists:member(Mod, Ignores). display_results(XrefResults, QueryResults) -> [lists:map(fun display_xref_results_for_type/1, XrefResults), From 5e0d71de71a25c4b3685f509b08c518b4eedbe81 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 26 May 2021 14:00:24 +0000 Subject: [PATCH 02/26] Fix optimistic registry update of plugins being upgraded This apparently never worked properly, the call being made swapped arguments order, but kept the type correct enough that no type warning would be raised. This isn't a problem though, because before this optimization, people were already used to calling 'rebar3 update' before upgrading a plugin. This being fixed makes that call optional now. --- src/rebar_prv_plugins_upgrade.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rebar_prv_plugins_upgrade.erl b/src/rebar_prv_plugins_upgrade.erl index 944791138..7b2481535 100644 --- a/src/rebar_prv_plugins_upgrade.erl +++ b/src/rebar_prv_plugins_upgrade.erl @@ -101,7 +101,7 @@ build_plugin(ToBuild, State) -> maybe_update_pkg(Tup, State) when is_tuple(Tup) -> maybe_update_pkg(element(1, Tup), State); maybe_update_pkg(Name, State) -> - try rebar_app_utils:parse_dep(root, unicode:characters_to_binary(?DEFAULT_PLUGINS_DIR), Name, State, [], 0) of + try rebar_app_utils:parse_dep(Name, root, unicode:characters_to_list(?DEFAULT_PLUGINS_DIR), State, [], 0) of AppInfo -> Source = rebar_app_info:source(AppInfo), case element(1, Source) of From e033a9d4dfbd96c6b61310db661c03b3060f1ebc Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 24 May 2021 09:26:00 -0400 Subject: [PATCH 03/26] back to git-based versioning --- src/rebar.app.src.script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rebar.app.src.script b/src/rebar.app.src.script index 1bf95282d..2f1a6a68a 100644 --- a/src/rebar.app.src.script +++ b/src/rebar.app.src.script @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "3.16.1"}, + {vsn, "git"}, {modules, []}, {registered, []}, {applications, [kernel, From bb79156cd1c644fd255ba5432703b05b7719e9b0 Mon Sep 17 00:00:00 2001 From: Dave Bryson Date: Thu, 27 May 2021 05:30:49 -0500 Subject: [PATCH 04/26] added SSL cacert patch for corporate mitm --- src/rebar_utils.erl | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index 922034581..cb73eb325 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -1042,6 +1042,24 @@ ssl_opts(Url) -> [{verify, verify_none}] end. +%% @private Determines which CA Certs to use for the HTTPS request. +%% If the user sets the value {cacert_path, "path to pem"} in their +%% global rebar.config file, the pem will be encoded and used for the +%% SSL connection. Otherwise CA Certs from `certifi` will be used. +%% This is functionaity is useful for Corporate Proxies that rewrite Certs. +%% See ssl_opts/2 +get_cacerts() -> + GlobalConfigFile = rebar_dir:global_config(), + Config = rebar_config:consult_file(GlobalConfigFile), + case proplists:get_value(cacert_path, Config) of + undefined -> + certifi:cacerts(); + Path -> + {ok, Bin} = file:read_file(Path), + Pems = public_key:pem_decode(Bin), + [Der || {'Certificate', Der, _} <- Pems] + end. + %%------------------------------------------------------------------------------ %% @doc %% Return the SSL options adequate for the project based on @@ -1058,7 +1076,7 @@ ssl_opts(ssl_verify_enabled, Url) -> #{host := Hostname} = rebar_uri:parse(rebar_utils:to_list(Url)), VerifyFun = {fun ssl_verify_hostname:verify_fun/3, [{check_hostname, Hostname}]}, - CACerts = certifi:cacerts(), + CACerts = get_cacerts(), [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts}, {partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}]; false -> From 9a189b52eb99af51dd9c77537857cd58cd3fd015 Mon Sep 17 00:00:00 2001 From: Dave Bryson Date: Fri, 28 May 2021 06:32:44 -0500 Subject: [PATCH 05/26] changed cacerts_path to ssl_cacerts_path --- src/rebar_utils.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index cb73eb325..fb04876ff 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -1043,15 +1043,15 @@ ssl_opts(Url) -> end. %% @private Determines which CA Certs to use for the HTTPS request. -%% If the user sets the value {cacert_path, "path to pem"} in their +%% If the user sets the value {ssl_cacerts_path, "path to pem"} in their %% global rebar.config file, the pem will be encoded and used for the -%% SSL connection. Otherwise CA Certs from `certifi` will be used. -%% This is functionaity is useful for Corporate Proxies that rewrite Certs. +%% SSL connection. Otherwise, CA Certs from `certifi` will be used. +%% This functionality is useful (needed) for Corporate Proxies that rewrite Certs. %% See ssl_opts/2 get_cacerts() -> GlobalConfigFile = rebar_dir:global_config(), Config = rebar_config:consult_file(GlobalConfigFile), - case proplists:get_value(cacert_path, Config) of + case proplists:get_value(ssl_cacerts_path, Config) of undefined -> certifi:cacerts(); Path -> From f0652199e4bdde2c9b413a2109e4636917a11c4c Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 9 Jun 2021 13:29:27 +0000 Subject: [PATCH 06/26] Normalize Dialyzer PLT paths this takes the current path formatting (which reuses compiler-specific functions) and makes it contextual to knowing whether files exist or not. The compiler path printing assumes all directory paths are project-local, but PLTs only occasionally fit this description. On first copies, they don't. This currently yields output such as: ===> Building with 204 files in ../.cache/rebar3/rebar3_24.0.2_plt... ===> Copying ../.cache/rebar3/rebar3_24.0.2_plt to _build/default/rebar3_24.0.2_plt... ===> Checking 204 files in _build/default/rebar3_24.0.2_plt... ===> Adding 29 files to _build/default/rebar3_24.0.2_plt... ===> Doing success typing analysis... ===> Resolving files... ===> Analyzing 13 files with _build/default/rebar3_24.0.2_plt... Whereas this patch instead allows the following formatting: ===> Dialyzer starting, this may take a while... ===> Updating plt... ===> Resolving files... ===> Updating base plt... ===> Resolving files... ===> Checking 204 files in /home/ferd/.cache/rebar3/rebar3_24.0_plt... ===> Copying /home/ferd/.cache/rebar3/rebar3_24.0_plt to /tmp/chk/_build/default/rebar3_24.0_plt... ===> Checking 204 files in _build/default/rebar3_24.0_plt... ===> Doing success typing analysis... ===> Resolving files... ===> Analyzing 2 files with _build/default/rebar3_24.0_plt... That `Copying to ` message is always going to have the `` value be absolute just because I can check whether a relative path exists before normalizing on the absolute path, which is the simplest way to make this work without major rework of output functions to attach them all to the calling context to know when a path may or may not be absolute. Fixes https://github.com/erlang/rebar3/issues/2577 --- src/rebar_prv_dialyzer.erl | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/rebar_prv_dialyzer.erl b/src/rebar_prv_dialyzer.erl index 9c2fee5ae..f4a4d0cbc 100644 --- a/src/rebar_prv_dialyzer.erl +++ b/src/rebar_prv_dialyzer.erl @@ -352,7 +352,7 @@ read_plt(_State, Plt) -> {error, not_valid} -> error; {error, read_error} -> - Error = io_lib:format("Could not read the PLT file ~ts", [rebar_dir:format_source_file_name(Plt)]), + Error = io_lib:format("Could not read the PLT file ~ts", [format_path(Plt)]), throw({dialyzer_error, Error}) end. @@ -370,7 +370,7 @@ read_plt_files(Plt, Files) -> [] -> {ok, Files}; Missing -> - ?INFO("Could not find ~p files in ~ts...", [length(Missing), rebar_dir:format_source_file_name(Plt)]), + ?INFO("Could not find ~p files in ~ts...", [length(Missing), format_path(Plt)]), ?DEBUG("Could not find files: ~p", [Missing]), error end. @@ -389,19 +389,19 @@ check_plt(State, Plt, Output, OldList, FilesList) -> remove_plt(State, _Plt, _Output, []) -> {0, State}; remove_plt(State, Plt, Output, Files) -> - ?INFO("Removing ~b files from ~ts...", [length(Files), rebar_dir:format_source_file_name(Plt)]), + ?INFO("Removing ~b files from ~ts...", [length(Files), format_path(Plt)]), run_plt(State, Plt, Output, plt_remove, Files). check_plt(State, _Plt, _Output, []) -> {0, State}; check_plt(State, Plt, Output, Files) -> - ?INFO("Checking ~b files in ~ts...", [length(Files), rebar_dir:format_source_file_name(Plt)]), + ?INFO("Checking ~b files in ~ts...", [length(Files), format_path(Plt)]), run_plt(State, Plt, Output, plt_check, Files). add_plt(State, _Plt, _Output, []) -> {0, State}; add_plt(State, Plt, Output, Files) -> - ?INFO("Adding ~b files to ~ts...", [length(Files), rebar_dir:format_source_file_name(Plt)]), + ?INFO("Adding ~b files to ~ts...", [length(Files), format_path(Plt)]), run_plt(State, Plt, Output, plt_add, Files). run_plt(State, Plt, Output, Analysis, Files) -> @@ -419,8 +419,7 @@ build_proj_plt(Args, State, Plt, Output, Files) -> ?INFO("Updating base plt...", []), BaseFiles = base_plt_files(State), {BaseWarnings, State1} = update_base_plt(State, BasePlt, Output, BaseFiles), - ?INFO("Copying ~ts to ~ts...", [rebar_dir:format_source_file_name(BasePlt), - rebar_dir:format_source_file_name(Plt)]), + ?INFO("Copying ~ts to ~ts...", [format_path(BasePlt), format_path(Plt)]), _ = filelib:ensure_dir(Plt), case file:copy(BasePlt, Plt) of {ok, _} -> @@ -462,7 +461,7 @@ update_base_plt(State, BasePlt, Output, BaseFiles) -> end. build_plt(State, Plt, _, []) -> - ?INFO("Building with no files in ~ts...", [rebar_dir:format_source_file_name(Plt)]), + ?INFO("Building with no files in ~ts...", [format_path(Plt)]), Opts = [{get_warnings, false}, {output_plt, Plt}, {apps, [erts]}], @@ -473,7 +472,7 @@ build_plt(State, Plt, _, []) -> _ = dialyzer:run([{analysis_type, plt_remove}, {init_plt, Plt} | Opts]), {0, State}; build_plt(State, Plt, Output, Files) -> - ?INFO("Building with ~b files in ~ts...", [length(Files), rebar_dir:format_source_file_name(Plt)]), + ?INFO("Building with ~b files in ~ts...", [length(Files), format_path(Plt)]), GetWarnings = get_config(State, get_warnings, false), Opts = [{analysis_type, plt_build}, {get_warnings, GetWarnings}, @@ -492,10 +491,10 @@ succ_typings(Args, State, Plt, Output) -> end. succ_typings_(State, Plt, _, []) -> - ?INFO("Analyzing no files with ~ts...", [rebar_dir:format_source_file_name(Plt)]), + ?INFO("Analyzing no files with ~ts...", [format_path(Plt)]), {0, State}; succ_typings_(State, Plt, Output, Files) -> - ?INFO("Analyzing ~b files with ~ts...", [length(Files), rebar_dir:format_source_file_name(Plt)]), + ?INFO("Analyzing ~b files with ~ts...", [length(Files), format_path(Plt)]), Opts = [{analysis_type, succ_typings}, {get_warnings, true}, {from, byte_code}, @@ -660,3 +659,13 @@ dialyzer_version() -> version_tuple(Major, Minor, Patch) -> {list_to_integer(Major), list_to_integer(Minor), list_to_integer(Patch)}. + +format_path(Path) -> + Normalized = rebar_dir:format_source_file_name(Path), + case filelib:is_file(Normalized) of + true -> Normalized; + false -> rebar_dir:format_source_file_name(Path, abs_path_opts()) + end. + +abs_path_opts() -> + dict:from_list([{compiler_source_format, absolute}]). From 7476e804091caa1411b6bf6e047fb53de763dcc4 Mon Sep 17 00:00:00 2001 From: thomasdavid Date: Fri, 12 Feb 2021 07:29:38 -0800 Subject: [PATCH 07/26] {xref_ignores} now properly works on module, by taking into account function calls. Summary: Before, the xref_ignores would properly ignores function calls on specified modules. That is, even though {xref_ignores, [ModName,...]} would be in rebar3.config, those unassigned function calls to and from the ModName module would be displayed. This was due to a mismatch between the format of the ignore (a module name), and the format of the warning ({MFA, MFA} or MFA) emmited by xref. --- test/rebar_xref_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/rebar_xref_SUITE.erl b/test/rebar_xref_SUITE.erl index 9f4bc7dec..090808859 100644 --- a/test/rebar_xref_SUITE.erl +++ b/test/rebar_xref_SUITE.erl @@ -231,7 +231,7 @@ get_module_body(ignoremod, AppName, IgnoreXref) -> "localfunc1(A, B) -> {A, B}.\n", % used local "localfunc2() -> ok.\n", % unused local "fdeprecated() -> ok.\n" % deprecated function - + "undef_call() -> non_existent_mod:func().\n" % undef function call ]; get_module_body(ignoremod2, AppName, IgnoreXref) -> ["-module(", AppName, "_ignoremod2).\n", @@ -241,7 +241,7 @@ get_module_body(ignoremod2, AppName, IgnoreXref) -> "localfunc1(A, B) -> {A, B}.\n", % used local "localfunc2() -> ok.\n", % unused local "fdeprecated() -> ok.\n" % deprecated function - + "undef_call() -> non_existent_mod:func().\n" % undef function call ]; get_module_body(othermod, AppName, IgnoreXref) -> ["-module(", AppName, "_othermod).\n", From 4bf81cb44e665983089f759047dffade166c3810 Mon Sep 17 00:00:00 2001 From: belltoy Date: Sat, 12 Jun 2021 18:42:47 +0800 Subject: [PATCH 08/26] Fix ssl check hostname options for wildcard certificate --- rebar.config | 1 + src/rebar_utils.erl | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/rebar.config b/rebar.config index 55e748f6e..faf47cb83 100644 --- a/rebar.config +++ b/rebar.config @@ -34,6 +34,7 @@ {erl_opts, [warnings_as_errors, {platform_define, "^(2[1-9])|(20\\\\.3)", filelib_find_source}, + {platform_define, "^(1|(20))", no_customize_hostname_check}, {platform_define, "^(20)", fun_stacktrace} ]}. diff --git a/src/rebar_utils.erl b/src/rebar_utils.erl index fb04876ff..d5a6b4b8b 100644 --- a/src/rebar_utils.erl +++ b/src/rebar_utils.erl @@ -1077,14 +1077,24 @@ ssl_opts(ssl_verify_enabled, Url) -> VerifyFun = {fun ssl_verify_hostname:verify_fun/3, [{check_hostname, Hostname}]}, CACerts = get_cacerts(), - [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts}, - {partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}]; + SslOpts = [{verify, verify_peer}, {depth, 2}, {cacerts, CACerts}, + {partial_chain, fun partial_chain/1}, {verify_fun, VerifyFun}], + check_hostname_opt(SslOpts); false -> ?WARN("Insecure HTTPS request (peer verification disabled), " "please update to OTP 17.4 or later", []), [{verify, verify_none}] end. +-ifdef(no_customize_hostname_check). +check_hostname_opt(Opts) -> + Opts. +-else. +check_hostname_opt(Opts) -> + MatchFun = public_key:pkix_verify_hostname_match_fun(https), + [{customize_hostname_check, [{match_fun, MatchFun}]} | Opts]. +-endif. + -spec partial_chain(Certs) -> Res when Certs :: list(any()), Res :: unknown_ca | {trusted_ca, any()}. From 30ca8055d6268d6310e35332ba3c85ea5c8e6948 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 18 Jun 2021 15:03:36 +0000 Subject: [PATCH 09/26] Prevent crashes on recursive src_dir definitions in deps In some rare cases (see https://github.com/erlang/rebar3/issues/2581 for a sample interplay between plugins and recursive dep definitions with apps) there can be some interplay where a dependency declares recursive directories but the lookup for app files does not acknowledges these and causes problems. This patch simply ignores such cases by stripping options; this should have no backward compatibility impact since the value was just ignored before anyway, but will prevent crashes in hard-to-reproduce cases. --- src/rebar_app_discover.erl | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 3070b5170..0b4a48e13 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -344,14 +344,18 @@ find_app(AppInfo, AppDir, SrcDirs, Validate, State) -> {true, rebar_app_info:t()} | false. find_app_(AppInfo, AppDir, SrcDirs, Validate, State) -> Extensions = rebar_state:get(State, application_resource_extensions, ?DEFAULT_APP_RESOURCE_EXT), + NormSrcDirs = [case SrcDir of + {SrcDir, _Opts} -> SrcDir; + _ -> SrcDir + end || SrcDir <- SrcDirs], ResourceFiles = [ {app, filelib:wildcard(filename:join([AppDir, "ebin", "*.app"]))}, - {mix_exs, filelib:wildcard(filename:join([AppDir, "src", "mix.exs"]))} | - [ - {extension_type(Ext), lists:append([ filelib:wildcard(filename:join([AppDir, SrcDir, "*" ++ Ext])) - || SrcDir <- SrcDirs])} - || Ext <- Extensions - ]], + {mix_exs, filelib:wildcard(filename:join([AppDir, "src", "mix.exs"]))} + | [{extension_type(Ext), + lists:append([filelib:wildcard(filename:join([AppDir, SrcDir, "*" ++ Ext])) + || SrcDir <- NormSrcDirs])} + || Ext <- Extensions] + ], FlattenedResourceFiles = flatten_resource_files(ResourceFiles), try_handle_resource_files(AppInfo, AppDir, FlattenedResourceFiles, Validate). From a0951e34a080a8b9c98f25908989c415c9d84894 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 21 Jun 2021 11:06:29 -0400 Subject: [PATCH 10/26] Apply suggestions from code review Co-authored-by: Denis Anufriev --- src/rebar_app_discover.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index 0b4a48e13..e4c047c66 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -345,7 +345,7 @@ find_app(AppInfo, AppDir, SrcDirs, Validate, State) -> find_app_(AppInfo, AppDir, SrcDirs, Validate, State) -> Extensions = rebar_state:get(State, application_resource_extensions, ?DEFAULT_APP_RESOURCE_EXT), NormSrcDirs = [case SrcDir of - {SrcDir, _Opts} -> SrcDir; + {ActualSrcDir, _Opts} -> ActualSrcDir; _ -> SrcDir end || SrcDir <- SrcDirs], ResourceFiles = [ From b74912ea73d3d5dac820aac76b6372fd437db7f2 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 25 Jun 2021 12:57:32 +0000 Subject: [PATCH 11/26] Drop bootstrap URI handling functions warnings bootstrap is always on an immediate run and should be skippable safely when dealing with compiler versions. As suggested by @saleyn in https://github.com/erlang/rebar3/issues/2584 --- bootstrap | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/bootstrap b/bootstrap index 5b54876f8..b3f9e86a7 100755 --- a/bootstrap +++ b/bootstrap @@ -747,30 +747,32 @@ chr([], _C, _I) -> 0. %% These two functions are ports of rebar_uri module calls that %% can't be called before the app is built; they're utility functions for %% forwards and backwards compat so we need them to be current. +-ifdef(OTP_RELEASE). rebar_uri_parse(URIString) -> %% we drop rebar_uri:apply_opts since it's a noop as called here - try uri_string:parse(URIString) of + case uri_string:parse(URIString) of Map = #{userinfo := _} -> Map; Map when is_map(Map) -> Map#{userinfo => ""}; Res -> Res - catch - error:undef -> - %% Taken from rebar_uri and trimmed down. - case http_uri:parse(URIString, []) of - {error, Reason} -> - {error, "", Reason}; - {ok, {Scheme, UserInfo, Host, Port, Path, Query}} -> - #{ - scheme => atom_to_list(Scheme), - host => Host, port => Port, path => Path, - query => case Query of - [] -> ""; - _ -> string:substr(Query, 2) - end, - userinfo => UserInfo - } - end end. +-else. +rebar_uri_parse(URIString) -> + %% Taken from rebar_uri and trimmed down. + case http_uri:parse(URIString, []) of + {error, Reason} -> + {error, "", Reason}; + {ok, {Scheme, UserInfo, Host, Port, Path, Query}} -> + #{ + scheme => atom_to_list(Scheme), + host => Host, port => Port, path => Path, + query => case Query of + [] -> ""; + _ -> string:substr(Query, 2) + end, + userinfo => UserInfo + } + end. +-endif. %% Taken from rebar_uri.erl rebar_uri_percent_decode(URIMap) when is_map(URIMap)-> From b2834147818a4fbca53d53bb9c185bdffa943108 Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sun, 27 Jun 2021 08:17:46 -0600 Subject: [PATCH 12/26] add run of shelltestrunner to CI --- .github/workflows/shelltests.yml | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/shelltests.yml diff --git a/.github/workflows/shelltests.yml b/.github/workflows/shelltests.yml new file mode 100644 index 000000000..e89cec0bd --- /dev/null +++ b/.github/workflows/shelltests.yml @@ -0,0 +1,33 @@ +name: Shell tests + +on: + pull_request: + branches: + - 'master' + push: + branches: + - 'master' + +jobs: + shelltests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: erlef/setup-beam@v1 + with: + otp-version: '24.0.1' + elixir-version: '1.12' + - name: Compile + run: ./bootstrap + - name: Install + run: cp ./rebar3 /usr/bin/ + + - name: Checkout shell tests + run: git checkout https://github.com/tsloughter/rebar3_tests + + - name: Install and run shelltestrunner + run: | + apt-get update + apt-get install -y shelltestrunner build-essential + mix local.hex --force + ./run_tests.sh From 9b132f06054660a7bb95145ebd755cd8ac09a0fd Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Sun, 27 Jun 2021 08:18:08 -0600 Subject: [PATCH 13/26] fix app discover to prefer erlang app to mix project --- src/rebar_app_discover.erl | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/rebar_app_discover.erl b/src/rebar_app_discover.erl index e4c047c66..a1400c688 100644 --- a/src/rebar_app_discover.erl +++ b/src/rebar_app_discover.erl @@ -256,7 +256,7 @@ app_dirs(LibDir, SrcDirs, State) -> ]), EbinPath = filename:join([LibDir, "ebin", "*.app"]), - MixExsPath = filename:join([LibDir, "src", "mix.exs"]), + MixExsPath = filename:join([LibDir, "mix.exs"]), lists:usort(lists:foldl(fun(Path, Acc) -> Files = filelib:wildcard(rebar_utils:to_list(Path)), @@ -350,7 +350,7 @@ find_app_(AppInfo, AppDir, SrcDirs, Validate, State) -> end || SrcDir <- SrcDirs], ResourceFiles = [ {app, filelib:wildcard(filename:join([AppDir, "ebin", "*.app"]))}, - {mix_exs, filelib:wildcard(filename:join([AppDir, "src", "mix.exs"]))} + {mix_exs, filelib:wildcard(filename:join([AppDir, "mix.exs"]))} | [{extension_type(Ext), lists:append([filelib:wildcard(filename:join([AppDir, SrcDir, "*" ++ Ext])) || SrcDir <- NormSrcDirs])} @@ -451,8 +451,14 @@ try_handle_resource_files(AppInfo, AppDir, [{app, AppFile} | Rest], Validate) -> try_handle_resource_files(AppInfo, AppDir, [{Type, AppSrcFile} | _Rest], Validate) when Type =:= app_src orelse Type =:= script -> try_handle_app_src_file(AppInfo, AppDir, AppSrcFile, Validate); -try_handle_resource_files(AppInfo, _AppDir, [{mix_exs, _AppSrcFile} | _Rest], _Validate) -> - {true, rebar_app_info:project_type(AppInfo, mix)}; +try_handle_resource_files(AppInfo, AppDir, [{mix_exs, _MixExs} | Rest], Validate) -> + %% prefer a rebar3 buildable app if both are found + case try_handle_resource_files(AppInfo, AppDir, Rest, Validate) of + false -> + {true, rebar_app_info:project_type(AppInfo, mix)}; + {true, _}=Result -> + Result + end; try_handle_resource_files(_AppInfo, _AppDir, [], _Validate) -> false. From 25d66c582354aa212ab8bb16a5535f307e410b21 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 27 Jun 2021 12:02:13 -0400 Subject: [PATCH 14/26] fix yaml syntax on shelltest workflow --- .github/workflows/shelltests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/shelltests.yml b/.github/workflows/shelltests.yml index e89cec0bd..81d51b3c4 100644 --- a/.github/workflows/shelltests.yml +++ b/.github/workflows/shelltests.yml @@ -14,7 +14,7 @@ jobs: steps: - uses: actions/checkout@v2 - uses: erlef/setup-beam@v1 - with: + with: otp-version: '24.0.1' elixir-version: '1.12' - name: Compile From 35c00865dfa5c107c37ff39b3134e099f6f07ffc Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 27 Jun 2021 12:07:33 -0400 Subject: [PATCH 15/26] fixing workflow items, debugging --- .github/workflows/shelltests.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/shelltests.yml b/.github/workflows/shelltests.yml index 81d51b3c4..55edb2f05 100644 --- a/.github/workflows/shelltests.yml +++ b/.github/workflows/shelltests.yml @@ -20,14 +20,16 @@ jobs: - name: Compile run: ./bootstrap - name: Install - run: cp ./rebar3 /usr/bin/ + run: | + sudo cp ./rebar3 /usr/local/bin/ - name: Checkout shell tests - run: git checkout https://github.com/tsloughter/rebar3_tests + run: git clone https://github.com/tsloughter/rebar3_tests - name: Install and run shelltestrunner run: | - apt-get update - apt-get install -y shelltestrunner build-essential + sudo apt-get update + sudo apt-get install -y shelltestrunner build-essential + cd rebar3_tests mix local.hex --force ./run_tests.sh From 519c3fbc029e75b215950b3549287c5b97e11e5d Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 7 Jul 2021 16:59:52 +0200 Subject: [PATCH 16/26] Handle abstract code starting with a non-file attribute --- src/rebar_prv_xref.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rebar_prv_xref.erl b/src/rebar_prv_xref.erl index 3b3f28224..d72c1d06e 100644 --- a/src/rebar_prv_xref.erl +++ b/src/rebar_prv_xref.erl @@ -296,7 +296,7 @@ find_function_source(M, F, A, Bin) -> find_function_source_in_abstract_code(F, A, AbstractCode) -> %% Extract the original source filename from the abstract code - [{attribute, _, file, {Source0, _}} | _] = AbstractCode, + [{attribute, _, file, {Source0, _}} | _] = [Attr || Attr = {attribute, _, file, _} <- AbstractCode], Source = rebar_dir:make_relative_path(Source0, rebar_dir:get_cwd()), %% Extract the line number for a given function def Fn = [E || E <- AbstractCode, From d43a5fd82e4eb335dd9e5888f782b0755dfdc803 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomek=20Ma=C5=84ko?= Date: Tue, 13 Jul 2021 18:52:06 +0200 Subject: [PATCH 17/26] Make `rebar_file_utils:system_tmpdir/1` take `TMPDIR` envvar into account on *nix --- src/rebar_file_utils.erl | 2 +- test/rebar_file_utils_SUITE.erl | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 425f4c520..1b9916f5a 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -406,7 +406,7 @@ system_tmpdir(PathComponents) -> "win32" -> "./tmp"; _SysArch -> - "/tmp" + os:getenv("TMPDIR", "/tmp") end, filename:join([Tmp|PathComponents]). diff --git a/test/rebar_file_utils_SUITE.erl b/test/rebar_file_utils_SUITE.erl index d771a82fa..443c6231e 100644 --- a/test/rebar_file_utils_SUITE.erl +++ b/test/rebar_file_utils_SUITE.erl @@ -71,26 +71,30 @@ end_per_testcase(_Test, Config) -> Config. raw_tmpdir(_Config) -> + UnixTmpdir = os:getenv("TMPDIR", "/tmp"), case rebar_file_utils:system_tmpdir() of - "/tmp" -> ok; - "./tmp" -> ok + UnixTmpdir -> ok; + "./tmp" -> ok end. empty_tmpdir(_Config) -> + UnixTmpdir = os:getenv("TMPDIR", "/tmp"), case rebar_file_utils:system_tmpdir([]) of - "/tmp" -> ok; - "./tmp" -> ok + UnixTmpdir -> ok; + "./tmp" -> ok end. simple_tmpdir(_Config) -> + UnixTmpdir = os:getenv("TMPDIR", "/tmp") ++ "/test", case rebar_file_utils:system_tmpdir(["test"]) of - "/tmp/test" -> ok; + UnixTmpdir -> ok; "./tmp/test" -> ok end. multi_tmpdir(_Config) -> + UnixTmpdir = os:getenv("TMPDIR", "/tmp") ++ "/a/b/c", case rebar_file_utils:system_tmpdir(["a", "b", "c"]) of - "/tmp/a/b/c" -> ok; + UnixTmpdir -> ok; "./tmp/a/b/c" -> ok end. From a3f4f26e05bb474216ac5e6ac9c06f622822be3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomek=20Ma=C5=84ko?= Date: Tue, 13 Jul 2021 20:32:38 +0200 Subject: [PATCH 18/26] Mock the `TMPDIR` envvar value for `rebar_file_utils:system_tmpdir/1` tests --- test/rebar_file_utils_SUITE.erl | 35 +++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/test/rebar_file_utils_SUITE.erl b/test/rebar_file_utils_SUITE.erl index 443c6231e..9b97bd412 100644 --- a/test/rebar_file_utils_SUITE.erl +++ b/test/rebar_file_utils_SUITE.erl @@ -31,6 +31,8 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("kernel/include/file.hrl"). +-define(TMPDIR, "/test"). + all() -> [{group, tmpdir}, @@ -52,8 +54,19 @@ groups() -> init_per_group(reset_dir, Config) -> TmpDir = rebar_file_utils:system_tmpdir(["rebar_file_utils_SUITE", "resetable"]), - [{tmpdir, TmpDir}|Config]; + [{tmpdir, TmpDir} | Config]; +init_per_group(tmpdir, Config) -> + PreviousTmpDir = os:getenv("TMPDIR"), + os:putenv("TMPDIR", ?TMPDIR), + [{previous_tmp, PreviousTmpDir} | Config]; init_per_group(_, Config) -> Config. + +end_per_group(tmpdir, Config) -> + case ?config(previous_tmp, Config) of + false -> os:unsetenv("TMPDIR"); + Val -> os:putenv("TMPDIR", Val) + end, + Config; end_per_group(_, Config) -> Config. init_per_testcase(Test, Config) -> @@ -71,31 +84,27 @@ end_per_testcase(_Test, Config) -> Config. raw_tmpdir(_Config) -> - UnixTmpdir = os:getenv("TMPDIR", "/tmp"), case rebar_file_utils:system_tmpdir() of - UnixTmpdir -> ok; - "./tmp" -> ok + ?TMPDIR -> ok; + "./tmp" -> ok end. empty_tmpdir(_Config) -> - UnixTmpdir = os:getenv("TMPDIR", "/tmp"), case rebar_file_utils:system_tmpdir([]) of - UnixTmpdir -> ok; - "./tmp" -> ok + ?TMPDIR -> ok; + "./tmp" -> ok end. simple_tmpdir(_Config) -> - UnixTmpdir = os:getenv("TMPDIR", "/tmp") ++ "/test", case rebar_file_utils:system_tmpdir(["test"]) of - UnixTmpdir -> ok; - "./tmp/test" -> ok + ?TMPDIR ++ "/test" -> ok; + "./tmp/test" -> ok end. multi_tmpdir(_Config) -> - UnixTmpdir = os:getenv("TMPDIR", "/tmp") ++ "/a/b/c", case rebar_file_utils:system_tmpdir(["a", "b", "c"]) of - UnixTmpdir -> ok; - "./tmp/a/b/c" -> ok + ?TMPDIR ++ "/a/b/c" -> ok; + "./tmp/a/b/c" -> ok end. reset_nonexistent_dir(Config) -> From 15ce0a8a35675b8b6155292edfb7ba3ea779638b Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sat, 31 Jul 2021 12:26:37 -0400 Subject: [PATCH 19/26] Interleave DAG app orders Assume that the original app order respects the dependency order as supplied by the rebar3 dependency order. Then what we do is interleave the hard compile- time dependencies as found by analysis to maintain their ordering constraints into those given by the dependency order. This should lower the chance of hitting conflicts when runtime dependencies become compile-time dependencies via interactions in parse transform calls. --- src/rebar_compiler_dag.erl | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/rebar_compiler_dag.erl b/src/rebar_compiler_dag.erl index fcb418cc8..77e751c12 100644 --- a/src/rebar_compiler_dag.erl +++ b/src/rebar_compiler_dag.erl @@ -271,8 +271,35 @@ compile_order(G, AppDefs, SrcExt, ArtifactExt) -> end, new_cache(), digraph:edges(G)), Sorted = lists:reverse(digraph_utils:topsort(AppDAG)), digraph:delete(AppDAG), - Standalone = [Name || {Name, _} <- AppDefs] -- Sorted, - Standalone ++ Sorted. + Standalone = [Name || {Name, _} <- AppDefs], + interleave(Standalone, Sorted). + +%% assume that the standalone app list respects the +%% rebar.config deps order, and enforce the sorted app +%% constraints onto it such that we're always respecting +%% the hard dependency but augment the list with the others +%% +%% [a,b,c,d] + [d,a] -> [d,a,b,c] +%% [a,b,c,d] + [d,c] -> [a,b,d,c] +%% [a,b,c,d] + [a,d] -> [a,b,c,d] +%% [a,b,c,d] + [b,d] -> [a,b,c,d] +%% [a,b,c,d] + [c,a,d] -> [c,a,b,d] +interleave(L, Sorted) -> interleave(L, Sorted, 1). + +interleave([], _, _) -> []; +interleave(L, [], _) -> L; +interleave([H|T], Sorted, N) -> + case find_at(H, Sorted) of + undefined -> [H|interleave(T, Sorted, N)]; + N -> [H|interleave(T, Sorted, N+1)]; + M when N > M -> interleave(T, Sorted, N); + M -> lists:sublist(Sorted, N, 1+M-N) ++ interleave(T, Sorted, M+1) + end. + +find_at(X, Sorted) -> find_at(X, Sorted, 1). +find_at(_, [], _) -> undefined; +find_at(X, [X|_], N) -> N; +find_at(X, [_|T], N) -> find_at(X, T, N+1). add_one_dependency_to_digraph(V1, V2, Cache, AppDefs, AppDAG) -> %% First resolve the file we depend on so that we can shortcut resolution From 6d78d19787ee2bbd4cdf5784f07c975cc5822cfa Mon Sep 17 00:00:00 2001 From: Max Kukartsev Date: Sat, 31 Jul 2021 16:56:29 -0700 Subject: [PATCH 20/26] Make escript_incl_extra work with _checkouts When developing `relx` and testing changes with `rebar3`, it is convenient to make `relx` a checkout dependency. However, the current implementation of `escript_incl_extra` expects the output directory of `relx` to be under `_build//lib/`, which is not the case if `relx` is a checkout dependency. Instead of making a breaking change to `escript_incl_extra`, introduce a new config parameter instead. --- rebar.config | 10 +++------- src/rebar_prv_escriptize.erl | 20 ++++++++++++++++++-- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/rebar.config b/rebar.config index faf47cb83..37e6fe14f 100644 --- a/rebar.config +++ b/rebar.config @@ -25,10 +25,10 @@ {escript_wrappers_windows, ["cmd", "powershell"]}. {escript_comment, "%%Rebar3 3.16.1\n"}. {escript_emu_args, "%%! +sbtu +A1\n"}. -%% escript_incl_extra is for internal rebar-private use only. +%% escript_incl_priv is for internal rebar-private use only. %% Do not use outside rebar. Config interface is not stable. -{escript_incl_extra, [{"relx/priv/templates/*", "_build/default/lib/"}, - {"rebar/priv/templates/*", "_build/default/lib/"}]}. +{escript_incl_priv, [{relx, "templates/*"}, + {rebar, "templates/*"}]}. {overrides, [{add, relx, [{erl_opts, [{d, 'RLX_LOG', rebar_log}]}]}]}. @@ -67,10 +67,6 @@ {bootstrap, []}, {prod, [ - {escript_incl_extra, [ - {"relx/priv/templates/*", "_build/prod/lib/"}, - {"rebar/priv/templates/*", "_build/prod/lib/"} - ]}, {erl_opts, [no_debug_info]}, {overrides, [ {override, erlware_commons, [ diff --git a/src/rebar_prv_escriptize.erl b/src/rebar_prv_escriptize.erl index 69ee4041c..bad8d67ee 100644 --- a/src/rebar_prv_escriptize.erl +++ b/src/rebar_prv_escriptize.erl @@ -206,10 +206,26 @@ get_app_beams(App, Path) -> load_files(Prefix, "*.app", Path). get_extra(State) -> - Extra = rebar_state:get(State, escript_incl_extra, []), + AllApps = rebar_state:all_deps(State) ++ rebar_state:project_apps(State), + InclPriv = rebar_state:get(State, escript_incl_priv, []), + InclPrivPaths = lists:map(fun(Entry) -> + resolve_incl_priv(Entry, AllApps) + end, InclPriv), + % `escript_incl_extra` is kept for historical reasons as its internal use in + % rebar3 has been replaced with `escript_incl_priv`. + InclExtraPaths = rebar_state:get(State, escript_incl_extra, []), lists:foldl(fun({Wildcard, Dir}, Files) -> load_files(Wildcard, Dir) ++ Files - end, [], Extra). + end, [], InclPrivPaths ++ InclExtraPaths). + +% Converts a wildcard path relative to an app (e.g., `priv/*`) into a wildcard +% path with with the app name included (e.g., `relx/priv/*`). +resolve_incl_priv({AppName, PrivWildcard}, AllApps) when is_atom(AppName) -> + {ok, AppInfo} = + rebar_app_utils:find(rebar_utils:to_binary(AppName), AllApps), + AppOutDir = rebar_app_info:out_dir(AppInfo), + Wildcard = filename:join([atom_to_list(AppName), "priv", PrivWildcard]), + {Wildcard, filename:dirname(AppOutDir)}. load_files(Wildcard, Dir) -> load_files("", Wildcard, Dir). From 48b0640d5ea750103a1f0e435b0d5f0e70cdc276 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 16 Aug 2021 08:55:23 -0400 Subject: [PATCH 21/26] Build compiler order DAG without topological sort As described in https://github.com/erlang/rebar3/issues/2563, part of the previous approach's weakness is that we need to seed the compile order with the dependency order for some invisible compile-time dependencies (runtime ones turned to build-time ones via calls to parse-transforms). Unfortunately, topological sorting is insufficient due to how it flattens apps. Assume the DAG contains: a -> b c -> d then a valid topological sort would be `[a,c,b,d]`. However, if we introduce the hidden dep (and decide that `c` calls `b`in a parse_transform) that the DAG doesn't contain, then the sorting order is broken. However, if we initially carried the `[x,b,a,c,f,e,d]` dep list, we can use a run over it to build `[x,a,b,c,f,e,d]` final compile order. --- src/rebar_compiler_dag.erl | 64 ++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/src/rebar_compiler_dag.erl b/src/rebar_compiler_dag.erl index 77e751c12..23f56943c 100644 --- a/src/rebar_compiler_dag.erl +++ b/src/rebar_compiler_dag.erl @@ -269,37 +269,53 @@ compile_order(G, AppDefs, SrcExt, ArtifactExt) -> end end end, new_cache(), digraph:edges(G)), - Sorted = lists:reverse(digraph_utils:topsort(AppDAG)), - digraph:delete(AppDAG), Standalone = [Name || {Name, _} <- AppDefs], - interleave(Standalone, Sorted). + Sorted = interleave(Standalone, AppDAG), + digraph:delete(AppDAG), + Sorted. -%% assume that the standalone app list respects the +%% Assume that the standalone app list respects the %% rebar.config deps order, and enforce the sorted app %% constraints onto it such that we're always respecting -%% the hard dependency but augment the list with the others +%% the hard dependencies. %% -%% [a,b,c,d] + [d,a] -> [d,a,b,c] -%% [a,b,c,d] + [d,c] -> [a,b,d,c] -%% [a,b,c,d] + [a,d] -> [a,b,c,d] -%% [a,b,c,d] + [b,d] -> [a,b,c,d] -%% [a,b,c,d] + [c,a,d] -> [c,a,b,d] -interleave(L, Sorted) -> interleave(L, Sorted, 1). - -interleave([], _, _) -> []; -interleave(L, [], _) -> L; -interleave([H|T], Sorted, N) -> - case find_at(H, Sorted) of - undefined -> [H|interleave(T, Sorted, N)]; - N -> [H|interleave(T, Sorted, N+1)]; - M when N > M -> interleave(T, Sorted, N); - M -> lists:sublist(Sorted, N, 1+M-N) ++ interleave(T, Sorted, M+1) +%% What we do here is a sort of run-length reordering based +%% on DAG information, which preserves the original dependency +%% order as declared, but successfully interleaves hard deps +%% to come first. +%% +%% Note that this approach is required as opposed to topsort +%% because when the original DAG reports two distinct set of +%% app dependencies that are joined by an invisible compile-time +%% one (e.g. a parse_transform runtime dep between both sets), +%% then the topological sort can't provide the right ordering +%% information because it's flattened into one list, but +%% this one can. +interleave(Apps, DAG) -> + interleave(Apps, DAG, #{}). + +interleave([], _, _) -> + []; +interleave([App|Apps], DAG, Expanded) -> + case Expanded of + #{App := _} -> + [App|interleave(Apps, DAG, Expanded)]; + _ -> + %% The DAG functions don't make it easy on insert to check for + %% duplicate edges across apps, so we clean them up here. + Deps = dedupe(digraph:out_neighbours(DAG, App)) -- maps:keys(Expanded), + interleave(Deps ++ [App|Apps -- Deps], DAG, Expanded#{App => true}) end. -find_at(X, Sorted) -> find_at(X, Sorted, 1). -find_at(_, [], _) -> undefined; -find_at(X, [X|_], N) -> N; -find_at(X, [_|T], N) -> find_at(X, T, N+1). +dedupe(L) -> dedupe(L, #{}). + +dedupe([], _) -> + []; +dedupe([H|T], Map) -> + case Map of + #{H := _} -> dedupe(T, Map); + _ -> [H|dedupe(T, Map#{H => true})] + end. add_one_dependency_to_digraph(V1, V2, Cache, AppDefs, AppDAG) -> %% First resolve the file we depend on so that we can shortcut resolution From ec8464168b7acd5dea4244e2296a49c003468f45 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Tue, 24 Aug 2021 21:50:45 +0000 Subject: [PATCH 22/26] switch from maps to sets in DAG dedupe ops --- src/rebar_compiler_dag.erl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/rebar_compiler_dag.erl b/src/rebar_compiler_dag.erl index 23f56943c..0cd75aea0 100644 --- a/src/rebar_compiler_dag.erl +++ b/src/rebar_compiler_dag.erl @@ -292,29 +292,29 @@ compile_order(G, AppDefs, SrcExt, ArtifactExt) -> %% information because it's flattened into one list, but %% this one can. interleave(Apps, DAG) -> - interleave(Apps, DAG, #{}). + interleave(Apps, DAG, sets:new()). interleave([], _, _) -> []; interleave([App|Apps], DAG, Expanded) -> - case Expanded of - #{App := _} -> + case sets:is_element(App, Expanded) of + true -> [App|interleave(Apps, DAG, Expanded)]; - _ -> + false -> %% The DAG functions don't make it easy on insert to check for %% duplicate edges across apps, so we clean them up here. - Deps = dedupe(digraph:out_neighbours(DAG, App)) -- maps:keys(Expanded), - interleave(Deps ++ [App|Apps -- Deps], DAG, Expanded#{App => true}) + Deps = dedupe(digraph:out_neighbours(DAG, App)) -- sets:to_list(Expanded), + interleave(Deps ++ [App|Apps -- Deps], DAG, sets:add_element(App, Expanded)) end. -dedupe(L) -> dedupe(L, #{}). +dedupe(L) -> dedupe(L, sets:new()). dedupe([], _) -> []; -dedupe([H|T], Map) -> - case Map of - #{H := _} -> dedupe(T, Map); - _ -> [H|dedupe(T, Map#{H => true})] +dedupe([H|T], Set) -> + case sets:is_element(H, Set) of + true -> dedupe(T, Set); + false -> [H|dedupe(T, sets:add_element(H, Set))] end. add_one_dependency_to_digraph(V1, V2, Cache, AppDefs, AppDAG) -> From 623a52923234a82faa3ed0af07a02788b6deaaf2 Mon Sep 17 00:00:00 2001 From: acw224 Date: Fri, 27 Aug 2021 07:15:47 -0700 Subject: [PATCH 23/26] rebar3 release to honour the ignore-xref attribute This comes in coniunction with https://github.com/erlware/relx/pull/882, that allows RELX to accept in its state a method to filter xref warnings. We pass to it the method that rebar3 currently uses to filter them based on the ignore-xref attributes in erlang file. --- src/rebar_prv_xref.erl | 3 ++- src/rebar_relx.erl | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/rebar_prv_xref.erl b/src/rebar_prv_xref.erl index d72c1d06e..ead9408a5 100644 --- a/src/rebar_prv_xref.erl +++ b/src/rebar_prv_xref.erl @@ -7,7 +7,8 @@ -export([init/1, do/1, - format_error/1]). + format_error/1, + filter_xref_results/3]). -include("rebar.hrl"). -include_lib("providers/include/providers.hrl"). diff --git a/src/rebar_relx.erl b/src/rebar_relx.erl index 05ef6de4a..2216eb001 100644 --- a/src/rebar_relx.erl +++ b/src/rebar_relx.erl @@ -42,7 +42,11 @@ do(Provider, State) -> Args = [include_erts, system_libs, vm_args, sys_config], RelxConfig2 = maybe_obey_command_args(RelxConfig1, Opts, Args), - {ok, RelxState} = rlx_config:to_state(RelxConfig2), + {ok, RelxState_0} = rlx_config:to_state(RelxConfig2), + XrefIgnores = rebar_state:get(State, xref_ignores, []), + RelxState = rlx_state:filter_xref_warning(RelxState_0, + fun(Warnings) -> + rebar_prv_xref:filter_xref_results(undefined_function_calls, XrefIgnores, Warnings) end), Providers = rebar_state:providers(State), Cwd = rebar_state:dir(State), From c46b429b923ad322568b4eb57157404a28ba8f99 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 2 Sep 2021 16:52:08 +0000 Subject: [PATCH 24/26] Bump relx to 4.5.0 https://github.com/erlware/relx/releases/tag/v4.5.0 - use copied erts dir when tar'ing even when the user sets the erts - improve error message when a symlink creation fails - State extended with a filter method to filter xref_warnings - Do not require logger - Add default time warp mode of multi - optionally allow static node name prefixes - use random:uniform instead of os:pid when constructing node name in nodetool - Fix eval command to use ERL_DIST_PORT, consistency with rpc command - Fix bin script arguments to erlexec - avoid quoting $@ in bin script's exec --- rebar.config | 2 +- rebar.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rebar.config b/rebar.config index 37e6fe14f..23cecb493 100644 --- a/rebar.config +++ b/rebar.config @@ -7,7 +7,7 @@ {providers, "1.8.1"}, {getopt, "1.0.1"}, {bbmustache, "1.10.0"}, - {relx, "4.4.0"}, + {relx, "4.5.0"}, {cf, "0.3.1"}, {cth_readable, "1.5.1"}, {eunit_formatters, "0.5.0"}]}. diff --git a/rebar.lock b/rebar.lock index f8e951c9f..e98ff18bf 100644 --- a/rebar.lock +++ b/rebar.lock @@ -7,7 +7,7 @@ {<<"eunit_formatters">>,{pkg,<<"eunit_formatters">>,<<"0.5.0">>},0}, {<<"getopt">>,{pkg,<<"getopt">>,<<"1.0.1">>},0}, {<<"providers">>,{pkg,<<"providers">>,<<"1.8.1">>},0}, - {<<"relx">>,{pkg,<<"relx">>,<<"4.4.0">>},0}, + {<<"relx">>,{pkg,<<"relx">>,<<"4.5.0">>},0}, {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.6">>},0}]}. [ {pkg_hash,[ @@ -19,7 +19,7 @@ {<<"eunit_formatters">>, <<"6A9133943D36A465D804C1C5B6E6839030434B8879C5600D7DDB5B3BAD4CCB59">>}, {<<"getopt">>, <<"C73A9FA687B217F2FF79F68A3B637711BB1936E712B521D8CE466B29CBF7808A">>}, {<<"providers">>, <<"70B4197869514344A8A60E2B2A4EF41CA03DEF43CFB1712ECF076A0F3C62F083">>}, - {<<"relx">>, <<"A7483BF5B82821D9CC992EF51F0B2DBFE05ADACD976807D42907C02C0C038B9C">>}, + {<<"relx">>, <<"2BF90A855023023EDD000641033D1AB9F4EBD4314D1739F691E16FE03CB35B85">>}, {<<"ssl_verify_fun">>, <<"CF344F5692C82D2CD7554F5EC8FD961548D4FD09E7D22F5B62482E5AEAEBD4B0">>}]}, {pkg_hash_ext,[ {<<"bbmustache">>, <<"43EFFA3FD4BB9523157AF5A9E2276C493495B8459FC8737144AA186CB13CE2EE">>}, @@ -30,6 +30,6 @@ {<<"eunit_formatters">>, <<"D6C8BA213424944E6E05BBC097C32001CDD0ABE3925D02454F229B20D68763C9">>}, {<<"getopt">>, <<"53E1AB83B9CEB65C9672D3E7A35B8092E9BDC9B3EE80721471A161C10C59959C">>}, {<<"providers">>, <<"E45745ADE9C476A9A469EA0840E418AB19360DC44F01A233304E118A44486BA0">>}, - {<<"relx">>, <<"55C0ED63BB5D55EB983A19EB94D7F3075DF6D126DBDFF43102A6660A91FCE925">>}, + {<<"relx">>, <<"DDB58F20CCE6CA63F5A2725E925816DD7213476698A3F3C3CB4FA8C272202C52">>}, {<<"ssl_verify_fun">>, <<"BDB0D2471F453C88FF3908E7686F86F9BE327D065CC1EC16FA4540197EA04680">>}]} ]. From 513577c1dc7802ac2a65e79d9ae28d0508b9c5fd Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 2 Sep 2021 21:15:12 -0400 Subject: [PATCH 25/26] bump to 3.17.0 --- bootstrap | 2 +- rebar.config | 2 +- src/rebar.app.src.script | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bootstrap b/bootstrap index b3f9e86a7..40524da09 100755 --- a/bootstrap +++ b/bootstrap @@ -44,7 +44,7 @@ main(_) -> bootstrap_rebar3(), %% Build rebar.app from rebar.app.src - {ok, App} = rebar_app_info:new(rebar, "3.16.1", filename:absname("_build/default/lib/rebar/")), +f {ok, App} = rebar_app_info:new(rebar, "3.17.0", filename:absname("_build/default/lib/rebar/")), rebar_otp_app:compile(rebar_state:new(), App), %% Because we are compiling files that are loaded already we want to silence diff --git a/rebar.config b/rebar.config index 23cecb493..0704380f3 100644 --- a/rebar.config +++ b/rebar.config @@ -23,7 +23,7 @@ {escript_name, rebar3}. {escript_wrappers_windows, ["cmd", "powershell"]}. -{escript_comment, "%%Rebar3 3.16.1\n"}. +{escript_comment, "%%Rebar3 3.17.0\n"}. {escript_emu_args, "%%! +sbtu +A1\n"}. %% escript_incl_priv is for internal rebar-private use only. %% Do not use outside rebar. Config interface is not stable. diff --git a/src/rebar.app.src.script b/src/rebar.app.src.script index 2f1a6a68a..f2f63458f 100644 --- a/src/rebar.app.src.script +++ b/src/rebar.app.src.script @@ -3,7 +3,7 @@ {application, rebar, [{description, "Rebar: Erlang Build Tool"}, - {vsn, "git"}, + {vsn, "3.17.0"}, {modules, []}, {registered, []}, {applications, [kernel, From e008a2ff40f708494a7c259a1fd8c2e77f7cd6a0 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 2 Sep 2021 21:17:08 -0400 Subject: [PATCH 26/26] Actually run the 3.17.0 release fine --- bootstrap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap b/bootstrap index 40524da09..9632b6c77 100755 --- a/bootstrap +++ b/bootstrap @@ -44,7 +44,7 @@ main(_) -> bootstrap_rebar3(), %% Build rebar.app from rebar.app.src -f {ok, App} = rebar_app_info:new(rebar, "3.17.0", filename:absname("_build/default/lib/rebar/")), + {ok, App} = rebar_app_info:new(rebar, "3.17.0", filename:absname("_build/default/lib/rebar/")), rebar_otp_app:compile(rebar_state:new(), App), %% Because we are compiling files that are loaded already we want to silence