From fe9837b9a6ab4fffc91ca38f68be4765e9d95121 Mon Sep 17 00:00:00 2001 From: Tyler Hughes Date: Fri, 12 May 2023 21:58:37 +0100 Subject: [PATCH 01/38] Implement 'hex' for erlang.mk qa: Remove test log files qa: Remove 'unique' on elixir modules qa: Remove 'LOCAL_DEPS_DIRS' changes qa: Remove duplicate targets qa: Remove 'fix' of setting the elixir commit to 'main' qa: Move 'GetDeps' fun to its own macro within hex.mk Some cleanup + if crypto isn't installed, fail earlier Add suites to CI --- .github/workflows/ci.yaml | 1 + build.config | 1 + core/core.mk | 2 + core/deps-tools.mk | 3 +- core/deps.mk | 96 ++++++++++++-- core/docs.mk | 2 +- core/elixir.mk | 269 ++++++++++++++++++++++++++++++++++++++ core/erlc.mk | 12 +- core/rel.mk | 2 +- core/test.mk | 2 +- plugins/ct.mk | 2 +- plugins/escript.mk | 4 + plugins/eunit.mk | 2 +- plugins/shell.mk | 2 +- test/core_elixir.mk | 150 +++++++++++++++++++++ 15 files changed, 528 insertions(+), 22 deletions(-) create mode 100644 core/elixir.mk create mode 100644 test/core_elixir.mk diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 850f8cb96..45ead96bb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,6 +31,7 @@ jobs: - core-compat # Trick GH runners into picking this slower job early. - ACME=1 c=core-deps + - core-elixir - core-makedep - core-misc - core-plugins diff --git a/build.config b/build.config index 5f1c1932a..27b3dc9a1 100644 --- a/build.config +++ b/build.config @@ -15,6 +15,7 @@ core/deps # Core modules, continued. core/beam-cache core/erlc +core/elixir core/docs core/rel core/test diff --git a/core/core.mk b/core/core.mk index 5d5eca00f..32cfc39c3 100644 --- a/core/core.mk +++ b/core/core.mk @@ -98,6 +98,8 @@ endif export PLATFORM endif +UNIQUE = $(if $1,$(firstword $1) $(call UNIQUE,$(filter-out $(firstword $1),$1))) + # Core targets. all:: deps app rel diff --git a/core/deps-tools.mk b/core/deps-tools.mk index 47f3b8e84..5c74ef108 100644 --- a/core/deps-tools.mk +++ b/core/deps-tools.mk @@ -71,6 +71,7 @@ endif $$dep/GNUmakefile $$dep/makefile $$dep/Makefile; then \ $(MAKE) -C $$dep fetch-deps \ IS_DEP=1 \ + ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM) \ ERLANG_MK_RECURSIVE_TMP_LIST=$(ERLANG_MK_RECURSIVE_TMP_LIST); \ fi \ fi \ @@ -119,7 +120,7 @@ endif :; \ else \ echo $$$$dep >> $(ERLANG_MK_TMP)/query.log; \ - $(MAKE) -C $(DEPS_DIR)/$$$$dep $$@ QUERY="$(QUERY)" IS_DEP=1 || true; \ + $(MAKE) -C $(DEPS_DIR)/$$$$dep $$@ QUERY="$(QUERY)" IS_DEP=1 ELIXIR_USE_SYSTEM=$$(ELIXIR_USE_SYSTEM) || true; \ fi \ done) ifeq ($(IS_APP)$(IS_DEP),) diff --git a/core/deps.mk b/core/deps.mk index d557957df..fcbbf6492 100644 --- a/core/deps.mk +++ b/core/deps.mk @@ -178,7 +178,7 @@ ifneq ($(ALL_APPS_DIRS_TO_BUILD),) :; \ else \ echo $$dep >> $(ERLANG_MK_TMP)/apps.log; \ - $(MAKE) -C $$dep $(if $(IS_TEST),test-build-app) IS_APP=1; \ + $(MAKE) -C $$dep $(if $(IS_TEST),test-build-app) IS_APP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); \ fi \ done endif @@ -220,10 +220,10 @@ ifneq ($(ALL_DEPS_DIRS),) if [ -z "$(strip $(FULL))" ] $(if $(force_rebuild_dep),&& ! ($(call force_rebuild_dep,$$dep)),) && [ ! -L $$dep ] && [ -f $$dep/ebin/dep_built ]; then \ :; \ elif [ "$$dep" = "$(DEPS_DIR)/hut" -a "$(HUT_PATCH)" ]; then \ - $(MAKE) -C $$dep app IS_DEP=1; \ + $(MAKE) -C $$dep app IS_DEP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); \ if [ ! -L $$dep ] && [ -d $$dep/ebin ]; then touch $$dep/ebin/dep_built; fi; \ elif [ -f $$dep/GNUmakefile ] || [ -f $$dep/makefile ] || [ -f $$dep/Makefile ]; then \ - $(MAKE) -C $$dep IS_DEP=1; \ + $(MAKE) -C $$dep IS_DEP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); \ if [ ! -L $$dep ] && [ -d $$dep/ebin ]; then touch $$dep/ebin/dep_built; fi; \ else \ echo "Error: No Makefile to build dependency $$dep." >&2; \ @@ -236,6 +236,79 @@ endif # Deps related targets. +define elixir_get_deps.erl +(fun(Deps) -> + GetVer = fun(Name, Req) -> + application:ensure_all_started(ssl), + application:ensure_all_started(inets), + {ok, PackageInfo} = + case hex_repo:get_package(hex_core:default_config(), atom_to_binary(Name)) of + {ok, {200, _RespHeaders, Decoded}} -> + LFirst(Decoded, fun(#{version := Vsn}) -> 'Elixir.Version':'match?'(Vsn, Req) end); + Other -> + io:format(standard_error, "Unexpected response for Dep ~p", [Name]), + erlang:halt(1) + end, + maps:get(version, PackageInfo) + end, + (fun + F([], DEPS_Acc0, DEP_Acc0) -> + [DEPS_Acc0, "\n", DEP_Acc0]; + F([H|T], DEPS_Acc0, DEP_Acc0) -> + {DEPS_Acc1, DEP_Acc1} = + case H of + {Name, Req} when is_binary(Req) -> + { + [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], + [DEP_Acc0, io_lib:format("dep_~p = hex ~s ~p~n", [Name, GetVer(Name, Req), Name])] + }; + {Name, Opts} when is_list(Opts) -> + Path = proplists:get_value(path, Opts), + IsRequired = proplists:get_value(optional, Opts) =/= true, + IsProdOnly = case proplists:get_value(only, Opts, prod) of + prod -> true; + L when is_list(L) -> lists:member(prod, L); + _ -> false + end, + case IsRequired andalso IsProdOnly of + true when Path =/= undefined -> + { + [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], + [DEP_Acc0, io_lib:format("dep_~p = ln ~s~n", [Name, Path])] + }; + true when Path =:= undefined -> + io:format(standard_error, "Skipping 'dep_~p' as no vsn given.", [Name]), + { + [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], + DEP_Acc0 + }; + false -> + {DEPS_Acc0, DEP_Acc0} + end; + {Name, Req, Opts} -> + IsRequired = proplists:get_value(optional, Opts) =/= true, + IsProdOnly = case proplists:get_value(only, Opts, prod) of + prod -> true; + L when is_list(L) -> lists:member(prod, L); + _ -> false + end, + case IsRequired andalso IsProdOnly of + true -> + { + [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], + [DEP_Acc0, io_lib:format("dep_~p = hex ~s ~p~n", [Name, GetVer(Name, Req), Name])] + }; + false -> + {DEPS_Acc0, DEP_Acc0} + end; + _ -> + {DEPS_Acc0, DEP_Acc0} + end, + F(T, DEPS_Acc1, DEP_Acc1) + end)(Deps, [], []) +end)($1) +endef + # @todo rename GNUmakefile and makefile into Makefile first, if they exist # While Makefile file could be GNUmakefile or makefile, # in practice only Makefile is needed so far. @@ -244,6 +317,8 @@ define dep_autopatch rm -rf $(DEPS_DIR)/$1/ebin/; \ $(call erlang,$(call dep_autopatch_appsrc.erl,$1)); \ $(call dep_autopatch_erlang_mk,$1); \ + elif [ -f $(DEPS_DIR)/$1/mix.exs ]; then \ + $(call dep_autopatch_mix,$1); \ elif [ -f $(DEPS_DIR)/$1/Makefile ]; then \ if [ -f $(DEPS_DIR)/$1/rebar.lock ]; then \ $(call dep_autopatch2,$1); \ @@ -661,7 +736,7 @@ define dep_autopatch_rebar.erl _ -> Path = "$(call core_native_path,$(DEPS_DIR)/)" ++ atom_to_list(P), io:format("~s", [os:cmd("$(MAKE) -C $(call core_native_path,$(DEPS_DIR)/$1) " ++ Path)]), - io:format("~s", [os:cmd("$(MAKE) -C " ++ Path ++ " IS_DEP=1")]), + io:format("~s", [os:cmd("$(MAKE) -C " ++ Path ++ " IS_DEP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM)")]), code:add_patha(Path ++ "/ebin") end end @@ -755,7 +830,7 @@ endef else define dep_fetch_git - git clone -q -n -- $(call query_repo_git,$1) $(DEPS_DIR)/$(call query_name,$1); \ + git clone -q -n $(if $(filter elixir,$1), --depth 1 ,) -- $(call query_repo_git,$1) $(DEPS_DIR)/$(call query_name,$1); \ cd $(DEPS_DIR)/$(call query_name,$1) && git checkout -q $(call query_version_git,$1); endef @@ -835,7 +910,10 @@ define dep_fetch_fail endef define dep_target -$(DEPS_DIR)/$(call query_name,$1): | $(if $(filter hex,$(call query_fetch_method,$1)),hex-core) $(ERLANG_MK_TMP) +prefetch-$1:: + @ + +$(DEPS_DIR)/$(call query_name,$1): prefetch-$1 | $(if $(filter hex,$(call query_fetch_method,$1)),hex-core) $(ERLANG_MK_TMP) $(eval DEP_NAME := $(call query_name,$1)) $(eval DEP_STR := $(if $(filter $1,$(DEP_NAME)),$1,"$1 ($(DEP_NAME))")) $(verbose) if test -d $(APPS_DIR)/$(DEP_NAME); then \ @@ -860,7 +938,7 @@ endif .PHONY: autopatch-$(call query_name,$1) autopatch-$(call query_name,$1):: - $(verbose) if [ "$1" = "elixir" -a "$(ELIXIR_PATCH)" ]; then \ + $(verbose) if [ "$1" = "elixir" ]; then \ ln -s lib/elixir/ebin $(DEPS_DIR)/elixir/; \ else \ $$(call dep_autopatch,$(call query_name,$1)) \ @@ -886,14 +964,14 @@ clean:: clean-apps clean-apps: $(verbose) set -e; for dep in $(ALL_APPS_DIRS) ; do \ - $(MAKE) -C $$dep clean IS_APP=1; \ + $(MAKE) -C $$dep clean IS_APP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); \ done distclean:: distclean-apps distclean-apps: $(verbose) set -e; for dep in $(ALL_APPS_DIRS) ; do \ - $(MAKE) -C $$dep distclean IS_APP=1; \ + $(MAKE) -C $$dep distclean IS_APP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); \ done endif diff --git a/core/docs.mk b/core/docs.mk index 6ef2ce6fe..cb9471447 100644 --- a/core/docs.mk +++ b/core/docs.mk @@ -16,5 +16,5 @@ ifneq ($(SKIP_DEPS),) doc-deps: else doc-deps: $(ALL_DOC_DEPS_DIRS) - $(verbose) set -e; for dep in $(ALL_DOC_DEPS_DIRS) ; do $(MAKE) -C $$dep IS_DEP=1; done + $(verbose) set -e; for dep in $(ALL_DOC_DEPS_DIRS) ; do $(MAKE) -C $$dep IS_DEP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); done endif diff --git a/core/elixir.mk b/core/elixir.mk new file mode 100644 index 000000000..44cb7b507 --- /dev/null +++ b/core/elixir.mk @@ -0,0 +1,269 @@ +ELIXIR_USE_SYSTEM ?= 1 + +ifeq ($(ELIXIR_USE_SYSTEM),1) +ELIXIRC := $(shell which elixirc) +ifneq ($(ELIXIRC),) +ELIXIR_PATH := $(shell $(dir $(ELIXIRC))/elixir -e 'IO.puts(:code.lib_dir(:elixir))')/../../ +endif +endif + +ifeq ($(ELIXIRC),) +ELIXIR_USE_SYSTEM = 0 +ELIXIRC := $(DEPS_DIR)/$(call dep_name,elixir)/bin/elixirc +ELIXIR_PATH = $(DEPS_DIR)/$(call dep_name,elixir) +endif + +ELIXIRC := $(abspath $(ELIXIRC)) +ELIXIR_PATH := $(abspath $(ELIXIR_PATH)) + +ELIXIR_COMPILE_FIRST_PATHS = $(addprefix src/,$(addsuffix .ex,$(COMPILE_FIRST))) $(addprefix lib/,$(addsuffix .ex,$(COMPILE_FIRST))) +ELIXIRC_EXCLUDE_PATHS = $(addprefix src/,$(addsuffix .ex,$(ERLC_EXCLUDE))) $(addprefix lib/,$(addsuffix .ex,$(ERLC_EXCLUDE))) +ELIXIRC_OPTS += $(ERLC_OPTS) + +elixirc_verbose_0 = @echo " ELIXIRC " $(filter-out $(patsubst %,%.ex,$(ERLC_EXCLUDE)),\ + $(filter %.ex %.core,$(?F))); +elixirc_verbose_2 = set -x; +elixirc_verbose = $(elixirc_verbose_$(V)) + +ALL_LIB_FILES := $(sort $(call core_find,lib/,*)) + +EX_FILES := $(filter-out lib/mix/%,$(filter %.ex,$(ALL_SRC_FILES) $(ALL_LIB_FILES))) +ELIXIR_BUILTINS = $(addprefix $(ELIXIR_PATH)/lib/,eex elixir logger mix) +USES_ELIXIR = $(if $(EX_FILES)$(shell find $(DEPS_DIR) -name '*.ex' 2>/dev/null),1,) + +ifneq ($(USES_ELIXIR),) +ERL_LIBS := $(ERL_LIBS):$(ELIXIR_PATH)/lib/ +export ERL_LIBS + +define app_file +{application, '$(PROJECT)', [ + {description, "$(PROJECT_DESCRIPTION)"}, + {vsn, "$(PROJECT_VERSION)"},$(if $(IS_DEP), + {id$(comma)$(space)"$(1)"}$(comma)) + {modules, [$(call comma_list,$(2))]}, + {registered, [$(if $(wildcard src/*/$(PROJECT_SUP).erl),$(call comma_list,$(PROJECT)_sup $(PROJECT_REGISTERED)),)]}, + {applications, [$(call comma_list,kernel stdlib $(OTP_DEPS) $(LOCAL_DEPS) $(foreach dep,$(DEPS),$(call dep_name,$(dep))))]}, + $(if $(PROJECT_MOD),{mod$(comma) {'$(PROJECT_MOD)'$(comma) []}}$(comma),) + {env, $(subst \,\\,$(PROJECT_ENV))}$(if $(findstring {,$(PROJECT_APP_EXTRA_KEYS)),$(comma)$(newline)$(tab)$(subst \,\\,$(PROJECT_APP_EXTRA_KEYS)),) +]}. +endef + +app:: $(if $(wildcard ebin/test),clean) deps + +define validate_app_file + case file:consult("ebin/$(PROJECT).app") of + {ok, _} -> halt(); + _ -> halt(1) + end +endef +endif + +define Mix_Makefile.erl +{ok, _} = application:ensure_all_started(elixir), +{ok, _} = application:ensure_all_started(mix), +File = <<"$(DEPS_DIR)/$(1)/mix.exs">>, +[{Mod, Bin}] = elixir_compiler:file(File, fun(_File, _LexerPid) -> ok end), +{module, Mod} = code:load_binary(Mod, binary_to_list(File), Bin), +Project = Mod:project(), +Application = try Mod:application() catch error:undef -> [] end, +Fmt = + "PROJECT = ~p~n" + "PROJECT_DESCRIPTION = ~s~n" + "PROJECT_VERSION = ~s~n" + "PROJECT_MOD = ~s~n" + "define PROJECT_ENV~n" + "~p~n" + "endef~n" + "~n~n~s~n~n" + "~n~n~s~n~n" + "~n~s~n" + "ERLC_OPTS = +debug_info~n" + "include $$(if $$(ERLANG_MK_FILENAME),$$(ERLANG_MK_FILENAME),erlang.mk)", +LFirst = fun + F([], Pred) -> + false; + F([H|T], Pred) -> + case catch Pred(H) of + true -> + {ok, H}; + false -> + F(T, Pred) + end +end, +StartMod = + case lists:keyfind(mod, 1, Application) of + {mod, {StartMod_, _StartArgs}} -> + atom_to_list(StartMod_); + _ -> + "" + end, +ExtraApps = [io_lib:format("LOCAL_DEPS += ~p~n", [App]) || App <- proplists:get_value(extra_applications, Application, [])], +ProjectCompilers = proplists:get_value(compilers, Project, []), +"https://hexdocs.pm/elixir_make/Mix.Tasks.Compile.ElixirMake.html", +ExtraMakeLines = + case lists:member(elixir_make, ProjectCompilers) of + false -> + ""; + true -> + Fetch = fun(Key, Proplist, DefaultVal, DefaultReplacement) -> + case proplists:get_value(Key, Proplist, DefaultVal) of + DefaultVal -> DefaultReplacement; + Value -> Value + end + end, + case file:copy("$(DEPS_DIR)/$(1)/" ++ Fetch(make_makefile, Project, default, "Makefile"), "$(DEPS_DIR)/$(1)/elixir_make.mk") of + {ok, _} -> ok; + Err = {error, _} -> + io:format(standard_error, "Failed to copy Makefile with error ~p~n", [Err]), + halt(1) + end, + [ + io_lib:format("app::~n\t~s -C \"~s\" -f \"$(DEPS_DIR)/$(1)/elixir_make.mk\" ~s ~s~n", [ + Fetch(make_executable, Project, default, "$(MAKE)"), + Fetch(make_cwd, Project, undefined, <<".">>), + lists:join(" ", Fetch(make_targets, Project, [], [])), + lists:join(" ", Fetch(make_args, Project, undefined, [])) + ]), + "\n", + case Fetch(make_clean, Project, nil, undefined) of + undefined -> + ""; + Clean -> + io_lib:format("clean::~n\t~s~n", [Clean]) + end + ] + end, +Deps = + lists:foldl(fun(DepToRemove, Acc) -> + lists:keydelete(DepToRemove, 1, Acc) + end, proplists:get_value(deps, Project, []), [elixir_make]), +Args = [ + proplists:get_value(app, Project), + proplists:get_value(description, Project, ""), + proplists:get_value(version, Project, ""), + StartMod, + proplists:get_value(env, Application, []), + ExtraApps, + $(call elixir_get_deps.erl, Deps), + ExtraMakeLines +], +Str = io_lib:format(Fmt, Args), +case file:write_file("$(DEPS_DIR)/$(1)/Makefile", Str) of + ok -> + halt(0); + {error, Reason} -> + io:format(standard_error, "Failed to create '$(DEPS_DIR)/$(1)/Makefile' with reason ~p~n", [Reason]), + halt(1) +end +endef + +define SHELL_ASSERT_ +if ! $(1); then \ + echo "$(2)" >&2; \ + exit 1; \ +fi +endef + +define SHELL_ASSERT +$(if $(verbose),,$(info [SHELL_ASSERT] $(call SHELL_ASSERT_,$(1),$(2)))) $(call SHELL_ASSERT_,$(1),$(2)) +endef + +define dep_autopatch_mix + $(MAKE) $(ELIXIRC) hex-core || exit 1; \ + sed 's|\(defmodule.*do\)|\1\ntry do\nCode.compiler_options(on_undefined_variable: :warn)\nrescue _ -> :ok\nend|g' -i $(DEPS_DIR)/$(1)/mix.exs; \ + MIX_ENV="$(if $(MIX_ENV),$(strip $(MIX_ENV)),prod)" \ + $(ERL) -pa $(DEPS_DIR)/hex_core $(addprefix -pa ,$(addsuffix /ebin,$(ELIXIR_BUILTINS))) \ + -eval "$(subst ",\",$(subst $(newline), ,$(subst $$,\$$,$(call Mix_Makefile.erl,$(1)))))." \ + -eval "halt(0)." || exit 1 \ + mkdir $(DEPS_DIR)/$1/src || exit 1 +endef + +ifneq ($(USES_ELIXIR),) +LOCAL_DEPS += eex elixir logger mix + +ebin/$(PROJECT).app:: $(ELIXIR_PATH)/lib/elixir/ebin/elixir.app +endif + +$(ELIXIRC): $(ELIXIR_PATH)/lib/elixir/ebin/elixir.app + +$(addsuffix /ebin,$(ELIXIR_BUILTINS)): $(ELIXIR_PATH)/lib/elixir/ebin/elixir.app + $(verbose) $(if $(ELIXIR_USE_SYSTEM),@,$(MAKE) -C $(DEPS_DIR)/elixir IS_DEP=1) + +define compile_ex +$(elixirc_verbose) ERL_COMPILER_OPTIONS="[$(call comma_list,$(patsubst '%',%,$(patsubst +%,%,$(filter +%,$(ELIXIRC_OPTS))))), {pa, \"ebin/\"}, {i, \"include/\"}]" $(ELIXIRC) \ + --verbose $(if $(IS_DEP),,$(if $(filter -Werror,$(ELIXIRC_OPTS)),--warnings-as-errors)) \ + -o ebin/ \ + $(filter-out $(ELIXIRC_EXCLUDE_PATHS),$(ELIXIR_COMPILE_FIRST_PATHS)) $(1) +endef + +# Currently doesn't account for nested modules +define get_elixir_mod +$(foreach module,$(strip \ + $(patsubst %do,%,$(patsubst defmodule%,%,\ + $(shell grep 'defmodule' $(1))\ + ))),'Elixir.$(module)') +endef + +ebin/$(PROJECT).app:: $(EX_FILES) + $(if $(strip $(EX_FILES)),$(call compile_ex,$(EX_FILES))) +# Older git versions do not have the --first-parent flag. Do without in that case. + $(eval GITDESCRIBE := $(shell git describe --dirty --abbrev=7 --tags --always --first-parent 2>/dev/null \ + || git describe --dirty --abbrev=7 --tags --always 2>/dev/null || true)) + $(eval MODULES := $(patsubst %,'%',$(sort $(notdir $(basename \ + $(filter-out $(ELIXIRC_EXCLUDE_PATHS), $(ERL_FILES) $(CORE_FILES) $(BEAM_FILES))))))) + $(eval MODULES := $(MODULES) $(foreach file, \ + $(EX_FILES), \ + $(call get_elixir_mod,$(file)) \ + )) +ifeq ($(wildcard src/$(PROJECT).app.src),) + $(app_verbose) printf "$(subst %,%%,$(subst $(newline),\n,$(subst ",\",$(call app_file,$(GITDESCRIBE),$(MODULES)))))" \ + > ebin/$(PROJECT).app + $(verbose) if ! $(call erlang,$(call validate_app_file)); then \ + echo "The .app file produced is invalid. Please verify the value of PROJECT_ENV." >&2; \ + exit 1; \ + fi +else + $(verbose) if [ -z "$$(grep -e '^[^%]*{\s*modules\s*,' src/$(PROJECT).app.src)" ]; then \ + echo "Empty modules entry not found in $(PROJECT).app.src. Please consult the erlang.mk documentation for instructions." >&2; \ + exit 1; \ + fi + $(appsrc_verbose) cat src/$(PROJECT).app.src \ + | sed "s/{[[:space:]]*modules[[:space:]]*,[[:space:]]*\[\]}/{modules, \[$(call comma_list,$(MODULES))\]}/" \ + | sed "s/{id,[[:space:]]*\"git\"}/{id, \"$(subst /,\/,$(GITDESCRIBE))\"}/" \ + > ebin/$(PROJECT).app +endif +ifneq ($(wildcard src/$(PROJECT).appup),) + $(verbose) cp src/$(PROJECT).appup ebin/ +endif + +$(ELIXIR_PATH)/lib/elixir/ebin/elixir.app: $(ELIXIR_PATH) +ifeq ($(ELIXIR_USE_SYSTEM),1) + @ +else + $(verbose) $(MAKE) -C $(DEPS_DIR)/elixir -f Makefile.orig compile Q="$(verbose)" +endif + +# We need the original makefile so that we can compile the elixir compiler +autopatch-elixir:: + $(verbose) cp $(DEPS_DIR)/elixir/Makefile $(DEPS_DIR)/elixir/Makefile.orig + $(verbose) sed 's|"$$(MAKE)"|"$$(MAKE)" -f $$(CURDIR)/Makefile.orig|g' -i $(DEPS_DIR)/elixir/Makefile.orig + +ifneq ($(USES_ELIXIR),) +elixir-check-crypto: + $(verbose) OUTPUT=`$(ERL) -eval 'code:load_file(crypto), init:stop().'`; \ + if test -n "$$OUTPUT"; then \ + echo " Crypto is required to use Elixir! "; \ + echo "$$OUTPUT"; \ + exit 1; \ + fi; + +prefetch-elixir:: elixir-check-crypto + $(verbose) echo " Fetching Elixir (this may take a while) " + +ebin/$(PROJECT).app:: $(eval $(call dep_target,elixir)) $(addsuffix /ebin,$(ELIXIR_BUILTINS)) + @ + +iex: + $(verbose) $(dir $(ELIXIRC))/iex +#$(addprefix -pa ,$(CURDIR)/ebin $(wildcard $(ELIXIR_PATH)/lib/*/ebin) $(wildcard $(DEPS_DIR)/*/ebin)) +endif diff --git a/core/erlc.mk b/core/erlc.mk index f3b499a63..3862870b7 100644 --- a/core/erlc.mk +++ b/core/erlc.mk @@ -49,7 +49,7 @@ mib_verbose_0 = @echo " MIB " $(filter %.bin %.mib,$(?F)); mib_verbose_2 = set -x; mib_verbose = $(mib_verbose_$(V)) -ifneq ($(wildcard src/),) +ifneq ($(wildcard src/)$(wildcard lib/),) # Targets. @@ -86,7 +86,7 @@ define app_file endef endif -app-build: ebin/$(PROJECT).app +app-build:: ebin/$(PROJECT).app $(verbose) : # Source files. @@ -271,21 +271,21 @@ define makedep.erl endef ifeq ($(if $(NO_MAKEDEP),$(wildcard $(PROJECT).d),),) -$(PROJECT).d:: $(ERL_FILES) $(call core_find,include/,*.hrl) $(MAKEFILE_LIST) +$(PROJECT).d:: $(ERL_FILES) $(EX_FILES) $(call core_find,include/,*.hrl) $(MAKEFILE_LIST) $(makedep_verbose) $(call erlang,$(call makedep.erl,$@)) endif ifeq ($(IS_APP)$(IS_DEP),) -ifneq ($(words $(ERL_FILES) $(CORE_FILES) $(ASN1_FILES) $(MIB_FILES) $(XRL_FILES) $(YRL_FILES)),0) +ifneq ($(words $(ERL_FILES) $(EX_FILES) $(CORE_FILES) $(ASN1_FILES) $(MIB_FILES) $(XRL_FILES) $(YRL_FILES)),0) # Rebuild everything when the Makefile changes. $(ERLANG_MK_TMP)/last-makefile-change: $(MAKEFILE_LIST) | $(ERLANG_MK_TMP) $(verbose) if test -f $@; then \ - touch $(ERL_FILES) $(CORE_FILES) $(ASN1_FILES) $(MIB_FILES) $(XRL_FILES) $(YRL_FILES); \ + touch $(ERL_FILES) $(EX_FILES) $(CORE_FILES) $(ASN1_FILES) $(MIB_FILES) $(XRL_FILES) $(YRL_FILES); \ touch -c $(PROJECT).d; \ fi $(verbose) touch $@ -$(ERL_FILES) $(CORE_FILES) $(ASN1_FILES) $(MIB_FILES) $(XRL_FILES) $(YRL_FILES):: $(ERLANG_MK_TMP)/last-makefile-change +$(ERL_FILES) $(EX_FILES) $(CORE_FILES) $(ASN1_FILES) $(MIB_FILES) $(XRL_FILES) $(YRL_FILES):: $(ERLANG_MK_TMP)/last-makefile-change ebin/$(PROJECT).app:: $(ERLANG_MK_TMP)/last-makefile-change endif endif diff --git a/core/rel.mk b/core/rel.mk index f64278c2a..451c989c9 100644 --- a/core/rel.mk +++ b/core/rel.mk @@ -15,5 +15,5 @@ ifneq ($(SKIP_DEPS),) rel-deps: else rel-deps: $(ALL_REL_DEPS_DIRS) - $(verbose) set -e; for dep in $(ALL_REL_DEPS_DIRS) ; do $(MAKE) -C $$dep; done + $(verbose) set -e; for dep in $(ALL_REL_DEPS_DIRS) ; do $(MAKE) -C $$dep ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); done endif diff --git a/core/test.mk b/core/test.mk index d576619c1..7a884fb6e 100644 --- a/core/test.mk +++ b/core/test.mk @@ -24,7 +24,7 @@ test-deps: $(ALL_TEST_DEPS_DIRS) if [ -z "$(strip $(FULL))" ] && [ ! -L $$dep ] && [ -f $$dep/ebin/dep_built ]; then \ :; \ else \ - $(MAKE) -C $$dep IS_DEP=1; \ + $(MAKE) -C $$dep IS_DEP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); \ if [ ! -L $$dep ] && [ -d $$dep/ebin ]; then touch $$dep/ebin/dep_built; fi; \ fi \ done diff --git a/plugins/ct.mk b/plugins/ct.mk index 820af1608..b11ecdba2 100644 --- a/plugins/ct.mk +++ b/plugins/ct.mk @@ -54,7 +54,7 @@ endif ifneq ($(ALL_APPS_DIRS),) define ct_app_target apps-ct-$1: test-build - $$(MAKE) -C $1 ct IS_APP=1 + $$(MAKE) -C $1 ct IS_APP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM) endef $(foreach app,$(ALL_APPS_DIRS),$(eval $(call ct_app_target,$(app)))) diff --git a/plugins/escript.mk b/plugins/escript.mk index 1790dcbcf..b80482b2e 100644 --- a/plugins/escript.mk +++ b/plugins/escript.mk @@ -37,6 +37,10 @@ ifneq ($(DEPS),) $(subst $(DEPS_DIR)/,,$(addsuffix /*,$(wildcard \ $(addsuffix /ebin,$(shell cat $(ERLANG_MK_TMP)/deps.log))))) endif +ifneq ($(USES_ELIXIR),) + $(verbose) cd $(DEPS_DIR) && $(ESCRIPT_ZIP) $(abspath $(ESCRIPT_ZIP_FILE)) \ + $(addsuffix /*,$(wildcard $(addsuffix /ebin,$(ELIXIR_BUILTINS)))) +endif escript:: escript-zip $(gen_verbose) printf "%s\n" \ diff --git a/plugins/eunit.mk b/plugins/eunit.mk index 211a7442d..f7fc81ed0 100644 --- a/plugins/eunit.mk +++ b/plugins/eunit.mk @@ -56,7 +56,7 @@ endif ifneq ($(ALL_APPS_DIRS),) apps-eunit: test-build - $(verbose) eunit_retcode=0 ; for app in $(ALL_APPS_DIRS); do $(MAKE) -C $$app eunit IS_APP=1; \ + $(verbose) eunit_retcode=0 ; for app in $(ALL_APPS_DIRS); do $(MAKE) -C $$app eunit IS_APP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); \ [ $$? -ne 0 ] && eunit_retcode=1 ; done ; \ exit $$eunit_retcode endif diff --git a/plugins/shell.mk b/plugins/shell.mk index 7c1f8e6e9..985ab70e9 100644 --- a/plugins/shell.mk +++ b/plugins/shell.mk @@ -31,7 +31,7 @@ build-shell-deps: $(ALL_SHELL_DEPS_DIRS) if [ -z "$(strip $(FULL))" ] && [ ! -L $$dep ] && [ -f $$dep/ebin/dep_built ]; then \ :; \ else \ - $(MAKE) -C $$dep IS_DEP=1; \ + $(MAKE) -C $$dep IS_DEP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); \ if [ ! -L $$dep ] && [ -d $$dep/ebin ]; then touch $$dep/ebin/dep_built; fi; \ fi \ done diff --git a/test/core_elixir.mk b/test/core_elixir.mk new file mode 100644 index 000000000..4475334ab --- /dev/null +++ b/test/core_elixir.mk @@ -0,0 +1,150 @@ +# Core: Miscellaneous. +# +# The miscellaneous tests use the prefix "core-", not "core-misc-". + +CORE_ELIXIR_TARGETS = $(call list_targets,core-elixir) + +.PHONY: core-elixir $(CORE_ELIXIR_TARGETS) + +core-elixir: $(CORE_ELIXIR_TARGETS) + +core-elixir-test-project_library: init + $i "Bootstrap a new OTP library named $(APP)" + $t mkdir $(APP)/ + $t cp ../erlang.mk $(APP)/ + $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v + + $i "Configure Makefile" + $t echo "ERLC_OPTS += +debug_info +'{parse_transform,lager_transform}'" >> $(APP)/Makefile + $t echo "ELIXIR_USE_SYSTEM = 0" >> $(APP)/Makefile + $t echo "DEPS += lager" >> $(APP)/Makefile + $t echo "DEPS += jason" >> $(APP)/Makefile + $t echo "DEPS += phoenix" >> $(APP)/Makefile + $t echo "dep_jason = git https://github.com/michalmuskala/jason.git master" >> $(APP)/Makefile + $t echo "dep_phoenix = hex 1.7.2" >> $(APP)/Makefile + $t echo "$$(grep -v 'include erlang.mk' $(APP)/Makefile)" > $(APP)/Makefile + $t echo "include erlang.mk" >> $(APP)/Makefile + + $i "Make deps" + $t $(MAKE) -C $(APP) deps $v + + $i "Check deps have compiled" + $t test -d $(APP)/deps/lager/ebin + $t test -d $(APP)/deps/jason/ebin + $t test -d $(APP)/deps/phoenix/ebin + + $i "Make the app" + $t $(MAKE) -C $(APP) app $v + + $i "Get started apps" + $t $(MAKE) -C $(APP) shell SHELL_OPTS="$(filter-out erl,$(ERL)) -pa $(APP)/deps/*/ebin/ $(APP)/ebin/ $(APP)/apps/*/ebin/ -eval \" \ + {ok, Apps} = application:ensure_all_started('$(APP)'), \ + true = lists:member(lager, Apps), \ + true = lists:member(jason, Apps), \ + true = lists:member(phoenix, Apps), \ + halt()\"" + + $i "Check modules aren't duplicated" + $t $(MAKE) -C $(APP) shell SHELL_OPTS="$(filter-out erl,$(ERL)) -pa $(APP)/deps/*/ebin/ $(APP)/ebin/ $(APP)/apps/*/ebin/ -eval \" \ + {ok, Apps} = application:ensure_all_started('$(APP)'), \ + [begin \ + {ok, Mods} = application:get_key(App, modules), \ + true = lists:sort(Mods) =:= lists:usort(Mods) \ + end || App <- Apps], \ + halt()\"" + +core-elixir-test-project_system: init +ifneq ($(shell which elixirc),) + $i "Bootstrap a new OTP library named $(APP)" + $t mkdir $(APP)/ + $t cp ../erlang.mk $(APP)/ + $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v + + $i "Configure Makefile" + $t echo "ERLC_OPTS += +debug_info +'{parse_transform,lager_transform}'" >> $(APP)/Makefile + $t echo "ELIXIR_USE_SYSTEM = 1" >> $(APP)/Makefile + $t echo "DEPS += lager" >> $(APP)/Makefile + $t echo "DEPS += jason" >> $(APP)/Makefile + $t echo "DEPS += phoenix" >> $(APP)/Makefile + $t echo "dep_jason = git https://github.com/michalmuskala/jason.git master" >> $(APP)/Makefile + $t echo "dep_phoenix = hex 1.7.2" >> $(APP)/Makefile + $t echo "$$(grep -v 'include erlang.mk' $(APP)/Makefile)" > $(APP)/Makefile + $t echo "include erlang.mk" >> $(APP)/Makefile + + $i "Make deps" + $t $(MAKE) -C $(APP) deps $v + + $i "Check deps have compiled" + $t test -d $(APP)/deps/lager/ebin + $t test -d $(APP)/deps/jason/ebin + $t test -d $(APP)/deps/phoenix/ebin + + $i "Make the app" + $t $(MAKE) -C $(APP) app $v + + $i "Get started apps" + $t $(MAKE) -C $(APP) shell SHELL_OPTS="$(filter-out erl,$(ERL)) -pa $(APP)/deps/*/ebin/ $(APP)/ebin/ $(APP)/apps/*/ebin/ -eval \" \ + {ok, Apps} = application:ensure_all_started('$(APP)'), \ + true = lists:member(lager, Apps), \ + true = lists:member(jason, Apps), \ + true = lists:member(phoenix, Apps), \ + halt()\"" + + $i "Check modules aren't duplicated" + $t $(MAKE) -C $(APP) shell SHELL_OPTS="$(filter-out erl,$(ERL)) -pa $(APP)/deps/*/ebin/ $(APP)/ebin/ $(APP)/apps/*/ebin/ -eval \" \ + {ok, Apps} = application:ensure_all_started('$(APP)'), \ + [begin \ + {ok, Mods} = application:get_key(App, modules), \ + true = lists:sort(Mods) =:= lists:usort(Mods) \ + end || App <- Apps], \ + halt()\"" +else + $i "Test depends on a System Install of Elixir, skipping." +endif + +core-elixir-test-project-rel: init + $i "Bootstrap a new OTP library named $(APP)" + $t mkdir $(APP)/ + $t cp ../erlang.mk $(APP)/ + $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v + + $i "Configure Makefile" + $t echo "ERLC_OPTS += +debug_info +'{parse_transform,lager_transform}'" >> $(APP)/Makefile + $t echo "ELIXIR_USE_SYSTEM = 1" >> $(APP)/Makefile + $t echo "DEPS += lager" >> $(APP)/Makefile + $t echo "DEPS += jason" >> $(APP)/Makefile + $t echo "DEPS += phoenix" >> $(APP)/Makefile + $t echo "dep_jason = git https://github.com/michalmuskala/jason.git master" >> $(APP)/Makefile + $t echo "dep_phoenix = hex 1.7.2" >> $(APP)/Makefile + $t echo "$$(grep -v 'include erlang.mk' $(APP)/Makefile)" > $(APP)/Makefile + $t echo "include erlang.mk" >> $(APP)/Makefile + + $i "Make deps" + $t $(MAKE) -C $(APP) deps $v + + $i "Check a release can be made" + $t $(MAKE) -C $(APP) bootstrap-rel + $t $(MAKE) -C $(APP) rel + +core-elixir-nif: init +ifneq ($(shell which cpp >/dev/null && echo '#include "sodium.h"' | cpp -H -o /dev/null 2>&1 | head -n1 | grep -v 'No such file or directory'),) + $i "Bootstrap a new OTP library named $(APP)" + $t mkdir $(APP)/ + $t cp ../erlang.mk $(APP)/ + $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v + + $i "Configure Makefile" + $t echo "DEPS += libsalty2" >> $(APP)/Makefile + $t echo "dep_libsalty2 = git https://github.com/Ianleeclark/libsalty2.git b11e544" >> $(APP)/Makefile + $t echo "$$(grep -v 'include erlang.mk' $(APP)/Makefile)" > $(APP)/Makefile + $t echo "include erlang.mk" >> $(APP)/Makefile + + $i "Make deps" + $t $(MAKE) -C $(APP) deps $v + + $i "Check libsalty2 has compiled" + $t test -f $(APP)/deps/libsalty2/ebin/libsalty2.app + $t test -f $(APP)/deps/libsalty2/priv/salty_nif.so +else + $i "Test depends on libsodium-dev, skipping." +endif \ No newline at end of file From 530f6690c123119ff4b12113fe974ab96fbf1f7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Tue, 3 Dec 2024 14:36:35 +0100 Subject: [PATCH 02/38] Get modules from Elixir compiler --- core/deps.mk | 109 +++++++++++++++---------------------------------- core/elixir.mk | 102 +++++++++++++++++++++++++++++++-------------- 2 files changed, 103 insertions(+), 108 deletions(-) diff --git a/core/deps.mk b/core/deps.mk index fcbbf6492..e1a6e2001 100644 --- a/core/deps.mk +++ b/core/deps.mk @@ -236,79 +236,6 @@ endif # Deps related targets. -define elixir_get_deps.erl -(fun(Deps) -> - GetVer = fun(Name, Req) -> - application:ensure_all_started(ssl), - application:ensure_all_started(inets), - {ok, PackageInfo} = - case hex_repo:get_package(hex_core:default_config(), atom_to_binary(Name)) of - {ok, {200, _RespHeaders, Decoded}} -> - LFirst(Decoded, fun(#{version := Vsn}) -> 'Elixir.Version':'match?'(Vsn, Req) end); - Other -> - io:format(standard_error, "Unexpected response for Dep ~p", [Name]), - erlang:halt(1) - end, - maps:get(version, PackageInfo) - end, - (fun - F([], DEPS_Acc0, DEP_Acc0) -> - [DEPS_Acc0, "\n", DEP_Acc0]; - F([H|T], DEPS_Acc0, DEP_Acc0) -> - {DEPS_Acc1, DEP_Acc1} = - case H of - {Name, Req} when is_binary(Req) -> - { - [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], - [DEP_Acc0, io_lib:format("dep_~p = hex ~s ~p~n", [Name, GetVer(Name, Req), Name])] - }; - {Name, Opts} when is_list(Opts) -> - Path = proplists:get_value(path, Opts), - IsRequired = proplists:get_value(optional, Opts) =/= true, - IsProdOnly = case proplists:get_value(only, Opts, prod) of - prod -> true; - L when is_list(L) -> lists:member(prod, L); - _ -> false - end, - case IsRequired andalso IsProdOnly of - true when Path =/= undefined -> - { - [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], - [DEP_Acc0, io_lib:format("dep_~p = ln ~s~n", [Name, Path])] - }; - true when Path =:= undefined -> - io:format(standard_error, "Skipping 'dep_~p' as no vsn given.", [Name]), - { - [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], - DEP_Acc0 - }; - false -> - {DEPS_Acc0, DEP_Acc0} - end; - {Name, Req, Opts} -> - IsRequired = proplists:get_value(optional, Opts) =/= true, - IsProdOnly = case proplists:get_value(only, Opts, prod) of - prod -> true; - L when is_list(L) -> lists:member(prod, L); - _ -> false - end, - case IsRequired andalso IsProdOnly of - true -> - { - [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], - [DEP_Acc0, io_lib:format("dep_~p = hex ~s ~p~n", [Name, GetVer(Name, Req), Name])] - }; - false -> - {DEPS_Acc0, DEP_Acc0} - end; - _ -> - {DEPS_Acc0, DEP_Acc0} - end, - F(T, DEPS_Acc1, DEP_Acc1) - end)(Deps, [], []) -end)($1) -endef - # @todo rename GNUmakefile and makefile into Makefile first, if they exist # While Makefile file could be GNUmakefile or makefile, # in practice only Makefile is needed so far. @@ -881,6 +808,37 @@ define hex_get_tarball.erl end endef +# Unfortunately this currently requires Elixir. +define hex_version_resolver.erl + HexVersionResolve = fun(Name, Req) -> + application:ensure_all_started(ssl), + application:ensure_all_started(inets), + Config = $(hex_config.erl), + case hex_repo:get_package(Config, Name) of + {ok, {200, _RespHeaders, Package}} -> + #{releases := List} = Package, + {value, #{version := Version}} = lists:search(fun(#{version := Vsn}) -> + M = list_to_atom("Elixir.Version"), + F = list_to_atom("match?"), + M:F(Vsn, Req) + end, List), + {ok, Version}; + {ok, {Status, _, Errors}} -> + {error, Status, Errors} + end + end, + HexVersionResolveAndPrint = fun(Name, Req) -> + case HexVersionResolve(Name, Req) of + {ok, Version} -> + io:format("~s", [Version]), + halt(0); + {error, Status, Errors} -> + io:format("Error ~b: ~0p~n", [Status, Errors]), + halt(77) + end + end +endef + ifeq ($(CACHE_DEPS),1) # Hex only has a package version. No need to look in the Erlang.mk packages. @@ -910,10 +868,7 @@ define dep_fetch_fail endef define dep_target -prefetch-$1:: - @ - -$(DEPS_DIR)/$(call query_name,$1): prefetch-$1 | $(if $(filter hex,$(call query_fetch_method,$1)),hex-core) $(ERLANG_MK_TMP) +$(DEPS_DIR)/$(call query_name,$1): | $(if $(filter hex,$(call query_fetch_method,$1)),hex-core) $(ERLANG_MK_TMP) $(eval DEP_NAME := $(call query_name,$1)) $(eval DEP_STR := $(if $(filter $1,$(DEP_NAME)),$1,"$1 ($(DEP_NAME))")) $(verbose) if test -d $(APPS_DIR)/$(DEP_NAME); then \ diff --git a/core/elixir.mk b/core/elixir.mk index 44cb7b507..ad71c29c2 100644 --- a/core/elixir.mk +++ b/core/elixir.mk @@ -58,6 +58,67 @@ define validate_app_file endef endif +define elixir_get_deps.erl +(fun(Deps) -> + $(call hex_version_resolver.erl), + (fun + F([], DEPS_Acc0, DEP_Acc0) -> + [DEPS_Acc0, "\n", DEP_Acc0]; + F([H|T], DEPS_Acc0, DEP_Acc0) -> + {DEPS_Acc1, DEP_Acc1} = + case H of + {Name, Req} when is_binary(Req) -> + { + [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], + [DEP_Acc0, io_lib:format("dep_~p = hex ~s ~p~n", [Name, HexVersionResolve(Name, Req), Name])] + }; + {Name, Opts} when is_list(Opts) -> + Path = proplists:get_value(path, Opts), + IsRequired = proplists:get_value(optional, Opts) =/= true, + IsProdOnly = case proplists:get_value(only, Opts, prod) of + prod -> true; + L when is_list(L) -> lists:member(prod, L); + _ -> false + end, + case IsRequired andalso IsProdOnly of + true when Path =/= undefined -> + { + [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], + [DEP_Acc0, io_lib:format("dep_~p = ln ~s~n", [Name, Path])] + }; + true when Path =:= undefined -> + io:format(standard_error, "Skipping 'dep_~p' as no vsn given.", [Name]), + { + [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], + DEP_Acc0 + }; + false -> + {DEPS_Acc0, DEP_Acc0} + end; + {Name, Req, Opts} -> + IsRequired = proplists:get_value(optional, Opts) =/= true, + IsProdOnly = case proplists:get_value(only, Opts, prod) of + prod -> true; + L when is_list(L) -> lists:member(prod, L); + _ -> false + end, + case IsRequired andalso IsProdOnly of + true -> + { + [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], + [DEP_Acc0, io_lib:format("dep_~p = hex ~s ~p~n", [Name, HexVersionResolve(Name, Req), Name])] + }; + false -> + {DEPS_Acc0, DEP_Acc0} + end; + _ -> + {DEPS_Acc0, DEP_Acc0} + end, + F(T, DEPS_Acc1, DEP_Acc1) + end)(Deps, [], []) +end)($1) +endef + define Mix_Makefile.erl {ok, _} = application:ensure_all_started(elixir), {ok, _} = application:ensure_all_started(mix), @@ -189,32 +250,22 @@ $(ELIXIRC): $(ELIXIR_PATH)/lib/elixir/ebin/elixir.app $(addsuffix /ebin,$(ELIXIR_BUILTINS)): $(ELIXIR_PATH)/lib/elixir/ebin/elixir.app $(verbose) $(if $(ELIXIR_USE_SYSTEM),@,$(MAKE) -C $(DEPS_DIR)/elixir IS_DEP=1) -define compile_ex -$(elixirc_verbose) ERL_COMPILER_OPTIONS="[$(call comma_list,$(patsubst '%',%,$(patsubst +%,%,$(filter +%,$(ELIXIRC_OPTS))))), {pa, \"ebin/\"}, {i, \"include/\"}]" $(ELIXIRC) \ - --verbose $(if $(IS_DEP),,$(if $(filter -Werror,$(ELIXIRC_OPTS)),--warnings-as-errors)) \ - -o ebin/ \ - $(filter-out $(ELIXIRC_EXCLUDE_PATHS),$(ELIXIR_COMPILE_FIRST_PATHS)) $(1) -endef - -# Currently doesn't account for nested modules -define get_elixir_mod -$(foreach module,$(strip \ - $(patsubst %do,%,$(patsubst defmodule%,%,\ - $(shell grep 'defmodule' $(1))\ - ))),'Elixir.$(module)') +define compile_ex.erl + {ok, _} = application:ensure_all_started(elixir), + Mod = list_to_atom("Elixir.Kernel.ParallelCompiler"), + {ok, Modules, _} = Mod:compile_to_path([$(call comma_list,$(patsubst %,<<"%">>,$(EX_FILES)))], <<"ebin/">>), + lists:foreach(fun(E) -> io:format("~p ", [E]) end, Modules), + halt() endef ebin/$(PROJECT).app:: $(EX_FILES) - $(if $(strip $(EX_FILES)),$(call compile_ex,$(EX_FILES))) + $(if $(strip $(EX_FILES)),$(eval MODULES := $(shell $(call erlang,$(call compile_ex.erl,$(EX_FILES)),-pa $(ELIXIR_PATH)/lib/elixir/ebin)))) # Older git versions do not have the --first-parent flag. Do without in that case. $(eval GITDESCRIBE := $(shell git describe --dirty --abbrev=7 --tags --always --first-parent 2>/dev/null \ || git describe --dirty --abbrev=7 --tags --always 2>/dev/null || true)) - $(eval MODULES := $(patsubst %,'%',$(sort $(notdir $(basename \ - $(filter-out $(ELIXIRC_EXCLUDE_PATHS), $(ERL_FILES) $(CORE_FILES) $(BEAM_FILES))))))) - $(eval MODULES := $(MODULES) $(foreach file, \ - $(EX_FILES), \ - $(call get_elixir_mod,$(file)) \ - )) + $(eval MODULES2 := $(patsubst %,'%',$(sort $(notdir $(basename \ + $(filter-out $(ELIXIRC_EXCLUDE_PATHS),$(ERL_FILES) $(CORE_FILES) $(BEAM_FILES)) \ + $(shell find ebin -type f -name Elixir.\*.beam)))))) ifeq ($(wildcard src/$(PROJECT).app.src),) $(app_verbose) printf "$(subst %,%%,$(subst $(newline),\n,$(subst ",\",$(call app_file,$(GITDESCRIBE),$(MODULES)))))" \ > ebin/$(PROJECT).app @@ -249,17 +300,6 @@ autopatch-elixir:: $(verbose) sed 's|"$$(MAKE)"|"$$(MAKE)" -f $$(CURDIR)/Makefile.orig|g' -i $(DEPS_DIR)/elixir/Makefile.orig ifneq ($(USES_ELIXIR),) -elixir-check-crypto: - $(verbose) OUTPUT=`$(ERL) -eval 'code:load_file(crypto), init:stop().'`; \ - if test -n "$$OUTPUT"; then \ - echo " Crypto is required to use Elixir! "; \ - echo "$$OUTPUT"; \ - exit 1; \ - fi; - -prefetch-elixir:: elixir-check-crypto - $(verbose) echo " Fetching Elixir (this may take a while) " - ebin/$(PROJECT).app:: $(eval $(call dep_target,elixir)) $(addsuffix /ebin,$(ELIXIR_BUILTINS)) @ From 823207ddee8d0fac77d00e34925410a19339d6ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Thu, 5 Dec 2024 14:39:51 +0100 Subject: [PATCH 03/38] Resolve recompile issues, merge app_file, and delete a bunch --- core/core.mk | 2 +- core/deps.mk | 2 +- core/elixir.mk | 60 +++++--------------------------------------------- core/erlc.mk | 28 +++++++---------------- 4 files changed, 16 insertions(+), 76 deletions(-) diff --git a/core/core.mk b/core/core.mk index 32cfc39c3..ee32cb5bb 100644 --- a/core/core.mk +++ b/core/core.mk @@ -36,7 +36,7 @@ PROJECT ?= $(notdir $(CURDIR)) PROJECT := $(strip $(PROJECT)) PROJECT_VERSION ?= rolling -PROJECT_MOD ?= $(PROJECT)_app +PROJECT_MOD ?= PROJECT_ENV ?= [] # Verbosity. diff --git a/core/deps.mk b/core/deps.mk index e1a6e2001..3e2a1e832 100644 --- a/core/deps.mk +++ b/core/deps.mk @@ -244,7 +244,7 @@ define dep_autopatch rm -rf $(DEPS_DIR)/$1/ebin/; \ $(call erlang,$(call dep_autopatch_appsrc.erl,$1)); \ $(call dep_autopatch_erlang_mk,$1); \ - elif [ -f $(DEPS_DIR)/$1/mix.exs ]; then \ + elif [ -f $(DEPS_DIR)/$1/mix.exs -a -f $(DEPS_DIR)/$1/lib ]; then \ $(call dep_autopatch_mix,$1); \ elif [ -f $(DEPS_DIR)/$1/Makefile ]; then \ if [ -f $(DEPS_DIR)/$1/rebar.lock ]; then \ diff --git a/core/elixir.mk b/core/elixir.mk index ad71c29c2..02870db85 100644 --- a/core/elixir.mk +++ b/core/elixir.mk @@ -16,8 +16,6 @@ endif ELIXIRC := $(abspath $(ELIXIRC)) ELIXIR_PATH := $(abspath $(ELIXIR_PATH)) -ELIXIR_COMPILE_FIRST_PATHS = $(addprefix src/,$(addsuffix .ex,$(COMPILE_FIRST))) $(addprefix lib/,$(addsuffix .ex,$(COMPILE_FIRST))) -ELIXIRC_EXCLUDE_PATHS = $(addprefix src/,$(addsuffix .ex,$(ERLC_EXCLUDE))) $(addprefix lib/,$(addsuffix .ex,$(ERLC_EXCLUDE))) ELIXIRC_OPTS += $(ERLC_OPTS) elixirc_verbose_0 = @echo " ELIXIRC " $(filter-out $(patsubst %,%.ex,$(ERLC_EXCLUDE)),\ @@ -29,25 +27,12 @@ ALL_LIB_FILES := $(sort $(call core_find,lib/,*)) EX_FILES := $(filter-out lib/mix/%,$(filter %.ex,$(ALL_SRC_FILES) $(ALL_LIB_FILES))) ELIXIR_BUILTINS = $(addprefix $(ELIXIR_PATH)/lib/,eex elixir logger mix) -USES_ELIXIR = $(if $(EX_FILES)$(shell find $(DEPS_DIR) -name '*.ex' 2>/dev/null),1,) +USES_ELIXIR = $(if $(EX_FILES),1,) ifneq ($(USES_ELIXIR),) ERL_LIBS := $(ERL_LIBS):$(ELIXIR_PATH)/lib/ export ERL_LIBS -define app_file -{application, '$(PROJECT)', [ - {description, "$(PROJECT_DESCRIPTION)"}, - {vsn, "$(PROJECT_VERSION)"},$(if $(IS_DEP), - {id$(comma)$(space)"$(1)"}$(comma)) - {modules, [$(call comma_list,$(2))]}, - {registered, [$(if $(wildcard src/*/$(PROJECT_SUP).erl),$(call comma_list,$(PROJECT)_sup $(PROJECT_REGISTERED)),)]}, - {applications, [$(call comma_list,kernel stdlib $(OTP_DEPS) $(LOCAL_DEPS) $(foreach dep,$(DEPS),$(call dep_name,$(dep))))]}, - $(if $(PROJECT_MOD),{mod$(comma) {'$(PROJECT_MOD)'$(comma) []}}$(comma),) - {env, $(subst \,\\,$(PROJECT_ENV))}$(if $(findstring {,$(PROJECT_APP_EXTRA_KEYS)),$(comma)$(newline)$(tab)$(subst \,\\,$(PROJECT_APP_EXTRA_KEYS)),) -]}. -endef - app:: $(if $(wildcard ebin/test),clean) deps define validate_app_file @@ -230,7 +215,7 @@ $(if $(verbose),,$(info [SHELL_ASSERT] $(call SHELL_ASSERT_,$(1),$(2)))) $(call endef define dep_autopatch_mix - $(MAKE) $(ELIXIRC) hex-core || exit 1; \ + $(MAKE) $(ELIXIR_PATH)/lib/elixir/ebin/elixir.app hex-core || exit 1; \ sed 's|\(defmodule.*do\)|\1\ntry do\nCode.compiler_options(on_undefined_variable: :warn)\nrescue _ -> :ok\nend|g' -i $(DEPS_DIR)/$(1)/mix.exs; \ MIX_ENV="$(if $(MIX_ENV),$(strip $(MIX_ENV)),prod)" \ $(ERL) -pa $(DEPS_DIR)/hex_core $(addprefix -pa ,$(addsuffix /ebin,$(ELIXIR_BUILTINS))) \ @@ -245,48 +230,19 @@ LOCAL_DEPS += eex elixir logger mix ebin/$(PROJECT).app:: $(ELIXIR_PATH)/lib/elixir/ebin/elixir.app endif -$(ELIXIRC): $(ELIXIR_PATH)/lib/elixir/ebin/elixir.app - $(addsuffix /ebin,$(ELIXIR_BUILTINS)): $(ELIXIR_PATH)/lib/elixir/ebin/elixir.app $(verbose) $(if $(ELIXIR_USE_SYSTEM),@,$(MAKE) -C $(DEPS_DIR)/elixir IS_DEP=1) define compile_ex.erl {ok, _} = application:ensure_all_started(elixir), - Mod = list_to_atom("Elixir.Kernel.ParallelCompiler"), - {ok, Modules, _} = Mod:compile_to_path([$(call comma_list,$(patsubst %,<<"%">>,$(EX_FILES)))], <<"ebin/">>), + ModCode = list_to_atom("Elixir.Code"), + ModCode:put_compiler_option(ignore_module_conflict, true), + ModComp = list_to_atom("Elixir.Kernel.ParallelCompiler"), + {ok, Modules, _} = ModComp:compile_to_path([$(call comma_list,$(patsubst %,<<"%">>,$(EX_FILES)))], <<"ebin/">>), lists:foreach(fun(E) -> io:format("~p ", [E]) end, Modules), halt() endef -ebin/$(PROJECT).app:: $(EX_FILES) - $(if $(strip $(EX_FILES)),$(eval MODULES := $(shell $(call erlang,$(call compile_ex.erl,$(EX_FILES)),-pa $(ELIXIR_PATH)/lib/elixir/ebin)))) -# Older git versions do not have the --first-parent flag. Do without in that case. - $(eval GITDESCRIBE := $(shell git describe --dirty --abbrev=7 --tags --always --first-parent 2>/dev/null \ - || git describe --dirty --abbrev=7 --tags --always 2>/dev/null || true)) - $(eval MODULES2 := $(patsubst %,'%',$(sort $(notdir $(basename \ - $(filter-out $(ELIXIRC_EXCLUDE_PATHS),$(ERL_FILES) $(CORE_FILES) $(BEAM_FILES)) \ - $(shell find ebin -type f -name Elixir.\*.beam)))))) -ifeq ($(wildcard src/$(PROJECT).app.src),) - $(app_verbose) printf "$(subst %,%%,$(subst $(newline),\n,$(subst ",\",$(call app_file,$(GITDESCRIBE),$(MODULES)))))" \ - > ebin/$(PROJECT).app - $(verbose) if ! $(call erlang,$(call validate_app_file)); then \ - echo "The .app file produced is invalid. Please verify the value of PROJECT_ENV." >&2; \ - exit 1; \ - fi -else - $(verbose) if [ -z "$$(grep -e '^[^%]*{\s*modules\s*,' src/$(PROJECT).app.src)" ]; then \ - echo "Empty modules entry not found in $(PROJECT).app.src. Please consult the erlang.mk documentation for instructions." >&2; \ - exit 1; \ - fi - $(appsrc_verbose) cat src/$(PROJECT).app.src \ - | sed "s/{[[:space:]]*modules[[:space:]]*,[[:space:]]*\[\]}/{modules, \[$(call comma_list,$(MODULES))\]}/" \ - | sed "s/{id,[[:space:]]*\"git\"}/{id, \"$(subst /,\/,$(GITDESCRIBE))\"}/" \ - > ebin/$(PROJECT).app -endif -ifneq ($(wildcard src/$(PROJECT).appup),) - $(verbose) cp src/$(PROJECT).appup ebin/ -endif - $(ELIXIR_PATH)/lib/elixir/ebin/elixir.app: $(ELIXIR_PATH) ifeq ($(ELIXIR_USE_SYSTEM),1) @ @@ -302,8 +258,4 @@ autopatch-elixir:: ifneq ($(USES_ELIXIR),) ebin/$(PROJECT).app:: $(eval $(call dep_target,elixir)) $(addsuffix /ebin,$(ELIXIR_BUILTINS)) @ - -iex: - $(verbose) $(dir $(ELIXIRC))/iex -#$(addprefix -pa ,$(CURDIR)/ebin $(wildcard $(ELIXIR_PATH)/lib/*/ebin) $(wildcard $(DEPS_DIR)/*/ebin)) endif diff --git a/core/erlc.mk b/core/erlc.mk index 3862870b7..9b2f4b384 100644 --- a/core/erlc.mk +++ b/core/erlc.mk @@ -57,34 +57,21 @@ app:: $(if $(wildcard ebin/test),beam-cache-restore-app) deps $(verbose) $(MAKE) --no-print-directory $(PROJECT).d $(verbose) $(MAKE) --no-print-directory app-build -ifeq ($(wildcard src/$(PROJECT_MOD).erl),) -define app_file -{application, '$(PROJECT)', [ - {description, "$(PROJECT_DESCRIPTION)"}, - {vsn, "$(PROJECT_VERSION)"},$(if $(IS_DEP), - {id$(comma)$(space)"$1"}$(comma)) - {modules, [$(call comma_list,$2)]}, - {registered, []}, - {applications, [$(call comma_list,kernel stdlib $(OTP_DEPS) $(LOCAL_DEPS) $(OPTIONAL_DEPS) $(foreach dep,$(DEPS),$(call query_name,$(dep))))]}, - {optional_applications, [$(call comma_list,$(OPTIONAL_DEPS))]}, - {env, $(subst \,\\,$(PROJECT_ENV))}$(if $(findstring {,$(PROJECT_APP_EXTRA_KEYS)),$(comma)$(newline)$(tab)$(subst \,\\,$(PROJECT_APP_EXTRA_KEYS)),) -]}. -endef -else +PROJECT_MOD := $(if $(PROJECT_MOD),$(PROJECT_MOD),$(if $(wildcard src/$(PROJECT_MOD)_app.erl),$(PROJECT_MOD)_app)) + define app_file {application, '$(PROJECT)', [ {description, "$(PROJECT_DESCRIPTION)"}, {vsn, "$(PROJECT_VERSION)"},$(if $(IS_DEP), {id$(comma)$(space)"$1"}$(comma)) {modules, [$(call comma_list,$2)]}, - {registered, [$(call comma_list,$(PROJECT)_sup $(PROJECT_REGISTERED))]}, + {registered, [$(if $(PROJECT_MOD),$(call comma_list,$(PROJECT)_sup $(PROJECT_REGISTERED)))]}, {applications, [$(call comma_list,kernel stdlib $(OTP_DEPS) $(LOCAL_DEPS) $(OPTIONAL_DEPS) $(foreach dep,$(DEPS),$(call query_name,$(dep))))]}, - {optional_applications, [$(call comma_list,$(OPTIONAL_DEPS))]}, - {mod, {$(PROJECT_MOD), []}}, + {optional_applications, [$(call comma_list,$(OPTIONAL_DEPS))]},$(if $(PROJECT_MOD), + {mod$(comma)$(space){$(PROJECT_MOD)$(comma)$(space)[]}}$(comma)) {env, $(subst \,\\,$(PROJECT_ENV))}$(if $(findstring {,$(PROJECT_APP_EXTRA_KEYS)),$(comma)$(newline)$(tab)$(subst \,\\,$(PROJECT_APP_EXTRA_KEYS)),) ]}. endef -endif app-build:: ebin/$(PROJECT).app $(verbose) : @@ -312,13 +299,14 @@ define validate_app_file end endef -ebin/$(PROJECT).app:: $(ERL_FILES) $(CORE_FILES) $(wildcard src/$(PROJECT).app.src) +ebin/$(PROJECT).app:: $(ERL_FILES) $(CORE_FILES) $(wildcard src/$(PROJECT).app.src) $(EX_FILES) $(eval FILES_TO_COMPILE := $(filter-out src/$(PROJECT).app.src,$?)) $(if $(strip $(FILES_TO_COMPILE)),$(call compile_erl,$(FILES_TO_COMPILE))) + $(if $(strip $(EX_FILES)),$(elixirc_verbose) $(eval MODULES := $(shell $(call erlang,$(call compile_ex.erl,$(EX_FILES)),-pa $(ELIXIR_PATH)/lib/elixir/ebin)))) # Older git versions do not have the --first-parent flag. Do without in that case. $(eval GITDESCRIBE := $(shell git describe --dirty --abbrev=7 --tags --always --first-parent 2>/dev/null \ || git describe --dirty --abbrev=7 --tags --always 2>/dev/null || true)) - $(eval MODULES := $(patsubst %,'%',$(sort $(notdir $(basename \ + $(eval MODULES := $(MODULES) $(patsubst %,'%',$(sort $(notdir $(basename \ $(filter-out $(ERLC_EXCLUDE_PATHS),$(ERL_FILES) $(CORE_FILES) $(BEAM_FILES))))))) ifeq ($(wildcard src/$(PROJECT).app.src),) $(app_verbose) printf '$(subst %,%%,$(subst $(newline),\n,$(subst ','\'',$(call app_file,$(GITDESCRIBE),$(MODULES)))))' \ From bdee4756f781c1b3bc526b24605d02dbe19c81b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Thu, 5 Dec 2024 16:50:37 +0100 Subject: [PATCH 04/38] More --- core/core.mk | 2 -- core/deps.mk | 2 +- core/elixir.mk | 76 ++++++++++++-------------------------------------- 3 files changed, 19 insertions(+), 61 deletions(-) diff --git a/core/core.mk b/core/core.mk index ee32cb5bb..222faebfb 100644 --- a/core/core.mk +++ b/core/core.mk @@ -98,8 +98,6 @@ endif export PLATFORM endif -UNIQUE = $(if $1,$(firstword $1) $(call UNIQUE,$(filter-out $(firstword $1),$1))) - # Core targets. all:: deps app rel diff --git a/core/deps.mk b/core/deps.mk index 3e2a1e832..e946a6e23 100644 --- a/core/deps.mk +++ b/core/deps.mk @@ -244,7 +244,7 @@ define dep_autopatch rm -rf $(DEPS_DIR)/$1/ebin/; \ $(call erlang,$(call dep_autopatch_appsrc.erl,$1)); \ $(call dep_autopatch_erlang_mk,$1); \ - elif [ -f $(DEPS_DIR)/$1/mix.exs -a -f $(DEPS_DIR)/$1/lib ]; then \ + elif [ -f $(DEPS_DIR)/$1/mix.exs -a -d $(DEPS_DIR)/$1/lib ]; then \ $(call dep_autopatch_mix,$1); \ elif [ -f $(DEPS_DIR)/$1/Makefile ]; then \ if [ -f $(DEPS_DIR)/$1/rebar.lock ]; then \ diff --git a/core/elixir.mk b/core/elixir.mk index 02870db85..63ae73931 100644 --- a/core/elixir.mk +++ b/core/elixir.mk @@ -1,47 +1,32 @@ -ELIXIR_USE_SYSTEM ?= 1 - -ifeq ($(ELIXIR_USE_SYSTEM),1) -ELIXIRC := $(shell which elixirc) -ifneq ($(ELIXIRC),) -ELIXIR_PATH := $(shell $(dir $(ELIXIRC))/elixir -e 'IO.puts(:code.lib_dir(:elixir))')/../../ -endif -endif +ALL_LIB_FILES := $(sort $(call core_find,lib/,*)) +EX_FILES := $(filter-out lib/mix/%,$(filter %.ex,$(ALL_SRC_FILES) $(ALL_LIB_FILES))) -ifeq ($(ELIXIRC),) -ELIXIR_USE_SYSTEM = 0 -ELIXIRC := $(DEPS_DIR)/$(call dep_name,elixir)/bin/elixirc -ELIXIR_PATH = $(DEPS_DIR)/$(call dep_name,elixir) -endif +#ifneq ($(strip $(EX_FILES)),) -ELIXIRC := $(abspath $(ELIXIRC)) -ELIXIR_PATH := $(abspath $(ELIXIR_PATH)) +ELIXIR ?= system +export ELIXIR -ELIXIRC_OPTS += $(ERLC_OPTS) +ifeq ($(ELIXIR),system) +# We expect 'elixir' to be on the path. +ELIXIR_LIBS := $(dir $(shell elixir -e 'IO.puts(:code.lib_dir(:elixir))')) +ERL_LIBS := $(ERL_LIBS):$(ELIXIR_LIBS) +endif elixirc_verbose_0 = @echo " ELIXIRC " $(filter-out $(patsubst %,%.ex,$(ERLC_EXCLUDE)),\ $(filter %.ex %.core,$(?F))); elixirc_verbose_2 = set -x; elixirc_verbose = $(elixirc_verbose_$(V)) -ALL_LIB_FILES := $(sort $(call core_find,lib/,*)) +ELIXIR_PATH := $(dir $(ELIXIR_LIBS)) -EX_FILES := $(filter-out lib/mix/%,$(filter %.ex,$(ALL_SRC_FILES) $(ALL_LIB_FILES))) -ELIXIR_BUILTINS = $(addprefix $(ELIXIR_PATH)/lib/,eex elixir logger mix) -USES_ELIXIR = $(if $(EX_FILES),1,) +#endif -ifneq ($(USES_ELIXIR),) -ERL_LIBS := $(ERL_LIBS):$(ELIXIR_PATH)/lib/ -export ERL_LIBS +ELIXIR_USE_SYSTEM ?= $(if $(filter system,$(ELIXIR)),1) -app:: $(if $(wildcard ebin/test),clean) deps +ELIXIRC_OPTS += $(ERLC_OPTS) -define validate_app_file - case file:consult("ebin/$(PROJECT).app") of - {ok, _} -> halt(); - _ -> halt(1) - end -endef -endif +ELIXIR_BUILTINS = $(addprefix $(ELIXIR_PATH)/lib/,eex elixir logger mix) +USES_ELIXIR = $(if $(EX_FILES),1,) define elixir_get_deps.erl (fun(Deps) -> @@ -203,19 +188,8 @@ case file:write_file("$(DEPS_DIR)/$(1)/Makefile", Str) of end endef -define SHELL_ASSERT_ -if ! $(1); then \ - echo "$(2)" >&2; \ - exit 1; \ -fi -endef - -define SHELL_ASSERT -$(if $(verbose),,$(info [SHELL_ASSERT] $(call SHELL_ASSERT_,$(1),$(2)))) $(call SHELL_ASSERT_,$(1),$(2)) -endef - define dep_autopatch_mix - $(MAKE) $(ELIXIR_PATH)/lib/elixir/ebin/elixir.app hex-core || exit 1; \ + $(MAKE) hex-core || exit 1; \ sed 's|\(defmodule.*do\)|\1\ntry do\nCode.compiler_options(on_undefined_variable: :warn)\nrescue _ -> :ok\nend|g' -i $(DEPS_DIR)/$(1)/mix.exs; \ MIX_ENV="$(if $(MIX_ENV),$(strip $(MIX_ENV)),prod)" \ $(ERL) -pa $(DEPS_DIR)/hex_core $(addprefix -pa ,$(addsuffix /ebin,$(ELIXIR_BUILTINS))) \ @@ -226,11 +200,9 @@ endef ifneq ($(USES_ELIXIR),) LOCAL_DEPS += eex elixir logger mix - -ebin/$(PROJECT).app:: $(ELIXIR_PATH)/lib/elixir/ebin/elixir.app endif -$(addsuffix /ebin,$(ELIXIR_BUILTINS)): $(ELIXIR_PATH)/lib/elixir/ebin/elixir.app +$(addsuffix /ebin,$(ELIXIR_BUILTINS)): $(verbose) $(if $(ELIXIR_USE_SYSTEM),@,$(MAKE) -C $(DEPS_DIR)/elixir IS_DEP=1) define compile_ex.erl @@ -243,18 +215,6 @@ define compile_ex.erl halt() endef -$(ELIXIR_PATH)/lib/elixir/ebin/elixir.app: $(ELIXIR_PATH) -ifeq ($(ELIXIR_USE_SYSTEM),1) - @ -else - $(verbose) $(MAKE) -C $(DEPS_DIR)/elixir -f Makefile.orig compile Q="$(verbose)" -endif - -# We need the original makefile so that we can compile the elixir compiler -autopatch-elixir:: - $(verbose) cp $(DEPS_DIR)/elixir/Makefile $(DEPS_DIR)/elixir/Makefile.orig - $(verbose) sed 's|"$$(MAKE)"|"$$(MAKE)" -f $$(CURDIR)/Makefile.orig|g' -i $(DEPS_DIR)/elixir/Makefile.orig - ifneq ($(USES_ELIXIR),) ebin/$(PROJECT).app:: $(eval $(call dep_target,elixir)) $(addsuffix /ebin,$(ELIXIR_BUILTINS)) @ From 7340500e6044ef3c8952d7fd71059860616389b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Fri, 6 Dec 2024 11:27:43 +0100 Subject: [PATCH 05/38] Obvious fixes --- core/deps.mk | 2 +- core/elixir.mk | 10 ++++------ core/erlc.mk | 4 ++-- test/core_elixir.mk | 5 ++++- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/core/deps.mk b/core/deps.mk index e946a6e23..b0e2033ec 100644 --- a/core/deps.mk +++ b/core/deps.mk @@ -814,7 +814,7 @@ define hex_version_resolver.erl application:ensure_all_started(ssl), application:ensure_all_started(inets), Config = $(hex_config.erl), - case hex_repo:get_package(Config, Name) of + case hex_repo:get_package(Config, atom_to_binary(Name)) of {ok, {200, _RespHeaders, Package}} -> #{releases := List} = Package, {value, #{version := Version}} = lists:search(fun(#{version := Vsn}) -> diff --git a/core/elixir.mk b/core/elixir.mk index 63ae73931..dbdcd49bc 100644 --- a/core/elixir.mk +++ b/core/elixir.mk @@ -38,9 +38,10 @@ define elixir_get_deps.erl {DEPS_Acc1, DEP_Acc1} = case H of {Name, Req} when is_binary(Req) -> + {ok, Vsn} = HexVersionResolve(Name, Req), { [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], - [DEP_Acc0, io_lib:format("dep_~p = hex ~s ~p~n", [Name, HexVersionResolve(Name, Req), Name])] + [DEP_Acc0, io_lib:format("dep_~p = hex ~s ~p~n", [Name, Vsn, Name])] }; {Name, Opts} when is_list(Opts) -> Path = proplists:get_value(path, Opts), @@ -74,9 +75,10 @@ define elixir_get_deps.erl end, case IsRequired andalso IsProdOnly of true -> + {ok, Vsn} = HexVersionResolve(Name, Req), { [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], - [DEP_Acc0, io_lib:format("dep_~p = hex ~s ~p~n", [Name, HexVersionResolve(Name, Req), Name])] + [DEP_Acc0, io_lib:format("dep_~p = hex ~s ~p~n", [Name, Vsn, Name])] }; false -> {DEPS_Acc0, DEP_Acc0} @@ -198,10 +200,6 @@ define dep_autopatch_mix mkdir $(DEPS_DIR)/$1/src || exit 1 endef -ifneq ($(USES_ELIXIR),) -LOCAL_DEPS += eex elixir logger mix -endif - $(addsuffix /ebin,$(ELIXIR_BUILTINS)): $(verbose) $(if $(ELIXIR_USE_SYSTEM),@,$(MAKE) -C $(DEPS_DIR)/elixir IS_DEP=1) diff --git a/core/erlc.mk b/core/erlc.mk index 9b2f4b384..6bb42e8b2 100644 --- a/core/erlc.mk +++ b/core/erlc.mk @@ -65,10 +65,10 @@ define app_file {vsn, "$(PROJECT_VERSION)"},$(if $(IS_DEP), {id$(comma)$(space)"$1"}$(comma)) {modules, [$(call comma_list,$2)]}, - {registered, [$(if $(PROJECT_MOD),$(call comma_list,$(PROJECT)_sup $(PROJECT_REGISTERED)))]}, + {registered, [$(if $(PROJECT_MOD),$(call comma_list,$(if $(filter $(PROJECT_MOD),$(PROJECT)_app),$(PROJECT)_sup) $(PROJECT_REGISTERED)))]}, {applications, [$(call comma_list,kernel stdlib $(OTP_DEPS) $(LOCAL_DEPS) $(OPTIONAL_DEPS) $(foreach dep,$(DEPS),$(call query_name,$(dep))))]}, {optional_applications, [$(call comma_list,$(OPTIONAL_DEPS))]},$(if $(PROJECT_MOD), - {mod$(comma)$(space){$(PROJECT_MOD)$(comma)$(space)[]}}$(comma)) + {mod$(comma)$(space){$(patsubst %,'%',$(PROJECT_MOD))$(comma)$(space)[]}}$(comma)) {env, $(subst \,\\,$(PROJECT_ENV))}$(if $(findstring {,$(PROJECT_APP_EXTRA_KEYS)),$(comma)$(newline)$(tab)$(subst \,\\,$(PROJECT_APP_EXTRA_KEYS)),) ]}. endef diff --git a/test/core_elixir.mk b/test/core_elixir.mk index 4475334ab..d0083a6ac 100644 --- a/test/core_elixir.mk +++ b/test/core_elixir.mk @@ -20,6 +20,7 @@ core-elixir-test-project_library: init $t echo "DEPS += lager" >> $(APP)/Makefile $t echo "DEPS += jason" >> $(APP)/Makefile $t echo "DEPS += phoenix" >> $(APP)/Makefile + $t echo "dep_lager = git https://github.com/erlang-lager/lager master" >> $(APP)/Makefile $t echo "dep_jason = git https://github.com/michalmuskala/jason.git master" >> $(APP)/Makefile $t echo "dep_phoenix = hex 1.7.2" >> $(APP)/Makefile $t echo "$$(grep -v 'include erlang.mk' $(APP)/Makefile)" > $(APP)/Makefile @@ -66,6 +67,7 @@ ifneq ($(shell which elixirc),) $t echo "DEPS += lager" >> $(APP)/Makefile $t echo "DEPS += jason" >> $(APP)/Makefile $t echo "DEPS += phoenix" >> $(APP)/Makefile + $t echo "dep_lager = git https://github.com/erlang-lager/lager master" >> $(APP)/Makefile $t echo "dep_jason = git https://github.com/michalmuskala/jason.git master" >> $(APP)/Makefile $t echo "dep_phoenix = hex 1.7.2" >> $(APP)/Makefile $t echo "$$(grep -v 'include erlang.mk' $(APP)/Makefile)" > $(APP)/Makefile @@ -114,6 +116,7 @@ core-elixir-test-project-rel: init $t echo "DEPS += lager" >> $(APP)/Makefile $t echo "DEPS += jason" >> $(APP)/Makefile $t echo "DEPS += phoenix" >> $(APP)/Makefile + $t echo "dep_lager = git https://github.com/erlang-lager/lager master" >> $(APP)/Makefile $t echo "dep_jason = git https://github.com/michalmuskala/jason.git master" >> $(APP)/Makefile $t echo "dep_phoenix = hex 1.7.2" >> $(APP)/Makefile $t echo "$$(grep -v 'include erlang.mk' $(APP)/Makefile)" > $(APP)/Makefile @@ -147,4 +150,4 @@ ifneq ($(shell which cpp >/dev/null && echo '#include "sodium.h"' | cpp -H -o /d $t test -f $(APP)/deps/libsalty2/priv/salty_nif.so else $i "Test depends on libsodium-dev, skipping." -endif \ No newline at end of file +endif From 60ed0f7e1f88ec30665fa3b90522504dfdad1e06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Fri, 6 Dec 2024 12:54:39 +0100 Subject: [PATCH 06/38] Restore optional usage of Elixir as dep --- core/deps-tools.mk | 3 +-- core/deps.mk | 14 +++++++------- core/elixir.mk | 18 ++++-------------- core/erlc.mk | 2 +- core/rel.mk | 2 +- core/test.mk | 2 +- index/elixir.mk | 7 +++++++ plugins/ct.mk | 2 +- plugins/eunit.mk | 2 +- plugins/shell.mk | 2 +- test/core_elixir.mk | 5 ++--- 11 files changed, 27 insertions(+), 32 deletions(-) create mode 100644 index/elixir.mk diff --git a/core/deps-tools.mk b/core/deps-tools.mk index 5c74ef108..47f3b8e84 100644 --- a/core/deps-tools.mk +++ b/core/deps-tools.mk @@ -71,7 +71,6 @@ endif $$dep/GNUmakefile $$dep/makefile $$dep/Makefile; then \ $(MAKE) -C $$dep fetch-deps \ IS_DEP=1 \ - ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM) \ ERLANG_MK_RECURSIVE_TMP_LIST=$(ERLANG_MK_RECURSIVE_TMP_LIST); \ fi \ fi \ @@ -120,7 +119,7 @@ endif :; \ else \ echo $$$$dep >> $(ERLANG_MK_TMP)/query.log; \ - $(MAKE) -C $(DEPS_DIR)/$$$$dep $$@ QUERY="$(QUERY)" IS_DEP=1 ELIXIR_USE_SYSTEM=$$(ELIXIR_USE_SYSTEM) || true; \ + $(MAKE) -C $(DEPS_DIR)/$$$$dep $$@ QUERY="$(QUERY)" IS_DEP=1 || true; \ fi \ done) ifeq ($(IS_APP)$(IS_DEP),) diff --git a/core/deps.mk b/core/deps.mk index b0e2033ec..4b4c2b92b 100644 --- a/core/deps.mk +++ b/core/deps.mk @@ -178,7 +178,7 @@ ifneq ($(ALL_APPS_DIRS_TO_BUILD),) :; \ else \ echo $$dep >> $(ERLANG_MK_TMP)/apps.log; \ - $(MAKE) -C $$dep $(if $(IS_TEST),test-build-app) IS_APP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); \ + $(MAKE) -C $$dep $(if $(IS_TEST),test-build-app) IS_APP=1; \ fi \ done endif @@ -220,10 +220,10 @@ ifneq ($(ALL_DEPS_DIRS),) if [ -z "$(strip $(FULL))" ] $(if $(force_rebuild_dep),&& ! ($(call force_rebuild_dep,$$dep)),) && [ ! -L $$dep ] && [ -f $$dep/ebin/dep_built ]; then \ :; \ elif [ "$$dep" = "$(DEPS_DIR)/hut" -a "$(HUT_PATCH)" ]; then \ - $(MAKE) -C $$dep app IS_DEP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); \ + $(MAKE) -C $$dep app IS_DEP=1; \ if [ ! -L $$dep ] && [ -d $$dep/ebin ]; then touch $$dep/ebin/dep_built; fi; \ elif [ -f $$dep/GNUmakefile ] || [ -f $$dep/makefile ] || [ -f $$dep/Makefile ]; then \ - $(MAKE) -C $$dep IS_DEP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); \ + $(MAKE) -C $$dep IS_DEP=1; \ if [ ! -L $$dep ] && [ -d $$dep/ebin ]; then touch $$dep/ebin/dep_built; fi; \ else \ echo "Error: No Makefile to build dependency $$dep." >&2; \ @@ -663,7 +663,7 @@ define dep_autopatch_rebar.erl _ -> Path = "$(call core_native_path,$(DEPS_DIR)/)" ++ atom_to_list(P), io:format("~s", [os:cmd("$(MAKE) -C $(call core_native_path,$(DEPS_DIR)/$1) " ++ Path)]), - io:format("~s", [os:cmd("$(MAKE) -C " ++ Path ++ " IS_DEP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM)")]), + io:format("~s", [os:cmd("$(MAKE) -C " ++ Path ++ " IS_DEP=1")]), code:add_patha(Path ++ "/ebin") end end @@ -757,7 +757,7 @@ endef else define dep_fetch_git - git clone -q -n $(if $(filter elixir,$1), --depth 1 ,) -- $(call query_repo_git,$1) $(DEPS_DIR)/$(call query_name,$1); \ + git clone -q -n -- $(call query_repo_git,$1) $(DEPS_DIR)/$(call query_name,$1); \ cd $(DEPS_DIR)/$(call query_name,$1) && git checkout -q $(call query_version_git,$1); endef @@ -919,14 +919,14 @@ clean:: clean-apps clean-apps: $(verbose) set -e; for dep in $(ALL_APPS_DIRS) ; do \ - $(MAKE) -C $$dep clean IS_APP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); \ + $(MAKE) -C $$dep clean IS_APP=1; \ done distclean:: distclean-apps distclean-apps: $(verbose) set -e; for dep in $(ALL_APPS_DIRS) ; do \ - $(MAKE) -C $$dep distclean IS_APP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); \ + $(MAKE) -C $$dep distclean IS_APP=1; \ done endif diff --git a/core/elixir.mk b/core/elixir.mk index dbdcd49bc..d7dcd465c 100644 --- a/core/elixir.mk +++ b/core/elixir.mk @@ -3,13 +3,15 @@ EX_FILES := $(filter-out lib/mix/%,$(filter %.ex,$(ALL_SRC_FILES) $(ALL_LIB_FILE #ifneq ($(strip $(EX_FILES)),) -ELIXIR ?= system +ELIXIR ?= $(if $(filter elixir,$(BUILD_DEPS) $(DEPS)),dep,system) export ELIXIR ifeq ($(ELIXIR),system) # We expect 'elixir' to be on the path. ELIXIR_LIBS := $(dir $(shell elixir -e 'IO.puts(:code.lib_dir(:elixir))')) ERL_LIBS := $(ERL_LIBS):$(ELIXIR_LIBS) +else +ERL_LIBS := $(ERL_LIBS):$(DEPS_DIR)/elixir/lib/ endif elixirc_verbose_0 = @echo " ELIXIRC " $(filter-out $(patsubst %,%.ex,$(ERLC_EXCLUDE)),\ @@ -17,15 +19,10 @@ elixirc_verbose_0 = @echo " ELIXIRC " $(filter-out $(patsubst %,%.ex,$(ERLC_EXC elixirc_verbose_2 = set -x; elixirc_verbose = $(elixirc_verbose_$(V)) -ELIXIR_PATH := $(dir $(ELIXIR_LIBS)) - #endif -ELIXIR_USE_SYSTEM ?= $(if $(filter system,$(ELIXIR)),1) - ELIXIRC_OPTS += $(ERLC_OPTS) -ELIXIR_BUILTINS = $(addprefix $(ELIXIR_PATH)/lib/,eex elixir logger mix) USES_ELIXIR = $(if $(EX_FILES),1,) define elixir_get_deps.erl @@ -191,6 +188,7 @@ end endef define dep_autopatch_mix + if test -d $(DEPS_DIR)/elixir; then $(MAKE) -C $(DEPS_DIR)/elixir || exit 1; fi; \ $(MAKE) hex-core || exit 1; \ sed 's|\(defmodule.*do\)|\1\ntry do\nCode.compiler_options(on_undefined_variable: :warn)\nrescue _ -> :ok\nend|g' -i $(DEPS_DIR)/$(1)/mix.exs; \ MIX_ENV="$(if $(MIX_ENV),$(strip $(MIX_ENV)),prod)" \ @@ -200,9 +198,6 @@ define dep_autopatch_mix mkdir $(DEPS_DIR)/$1/src || exit 1 endef -$(addsuffix /ebin,$(ELIXIR_BUILTINS)): - $(verbose) $(if $(ELIXIR_USE_SYSTEM),@,$(MAKE) -C $(DEPS_DIR)/elixir IS_DEP=1) - define compile_ex.erl {ok, _} = application:ensure_all_started(elixir), ModCode = list_to_atom("Elixir.Code"), @@ -212,8 +207,3 @@ define compile_ex.erl lists:foreach(fun(E) -> io:format("~p ", [E]) end, Modules), halt() endef - -ifneq ($(USES_ELIXIR),) -ebin/$(PROJECT).app:: $(eval $(call dep_target,elixir)) $(addsuffix /ebin,$(ELIXIR_BUILTINS)) - @ -endif diff --git a/core/erlc.mk b/core/erlc.mk index 6bb42e8b2..39e2ce134 100644 --- a/core/erlc.mk +++ b/core/erlc.mk @@ -302,7 +302,7 @@ endef ebin/$(PROJECT).app:: $(ERL_FILES) $(CORE_FILES) $(wildcard src/$(PROJECT).app.src) $(EX_FILES) $(eval FILES_TO_COMPILE := $(filter-out src/$(PROJECT).app.src,$?)) $(if $(strip $(FILES_TO_COMPILE)),$(call compile_erl,$(FILES_TO_COMPILE))) - $(if $(strip $(EX_FILES)),$(elixirc_verbose) $(eval MODULES := $(shell $(call erlang,$(call compile_ex.erl,$(EX_FILES)),-pa $(ELIXIR_PATH)/lib/elixir/ebin)))) + $(if $(strip $(EX_FILES)),$(elixirc_verbose) $(eval MODULES := $(shell $(call erlang,$(call compile_ex.erl,$(EX_FILES)))))) # Older git versions do not have the --first-parent flag. Do without in that case. $(eval GITDESCRIBE := $(shell git describe --dirty --abbrev=7 --tags --always --first-parent 2>/dev/null \ || git describe --dirty --abbrev=7 --tags --always 2>/dev/null || true)) diff --git a/core/rel.mk b/core/rel.mk index 451c989c9..f64278c2a 100644 --- a/core/rel.mk +++ b/core/rel.mk @@ -15,5 +15,5 @@ ifneq ($(SKIP_DEPS),) rel-deps: else rel-deps: $(ALL_REL_DEPS_DIRS) - $(verbose) set -e; for dep in $(ALL_REL_DEPS_DIRS) ; do $(MAKE) -C $$dep ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); done + $(verbose) set -e; for dep in $(ALL_REL_DEPS_DIRS) ; do $(MAKE) -C $$dep; done endif diff --git a/core/test.mk b/core/test.mk index 7a884fb6e..d576619c1 100644 --- a/core/test.mk +++ b/core/test.mk @@ -24,7 +24,7 @@ test-deps: $(ALL_TEST_DEPS_DIRS) if [ -z "$(strip $(FULL))" ] && [ ! -L $$dep ] && [ -f $$dep/ebin/dep_built ]; then \ :; \ else \ - $(MAKE) -C $$dep IS_DEP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); \ + $(MAKE) -C $$dep IS_DEP=1; \ if [ ! -L $$dep ] && [ -d $$dep/ebin ]; then touch $$dep/ebin/dep_built; fi; \ fi \ done diff --git a/index/elixir.mk b/index/elixir.mk new file mode 100644 index 000000000..86f89e79d --- /dev/null +++ b/index/elixir.mk @@ -0,0 +1,7 @@ +PACKAGES += elixir +pkg_elixir_name = elixir +pkg_elixir_description = Elixir is a dynamic, functional language for building scalable and maintainable applications. +pkg_elixir_homepage = https://elixir-lang.org +pkg_elixir_fetch = git +pkg_elixir_repo = https://github.com/elixir-lang/elixir +pkg_elixir_commit = main diff --git a/plugins/ct.mk b/plugins/ct.mk index b11ecdba2..820af1608 100644 --- a/plugins/ct.mk +++ b/plugins/ct.mk @@ -54,7 +54,7 @@ endif ifneq ($(ALL_APPS_DIRS),) define ct_app_target apps-ct-$1: test-build - $$(MAKE) -C $1 ct IS_APP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM) + $$(MAKE) -C $1 ct IS_APP=1 endef $(foreach app,$(ALL_APPS_DIRS),$(eval $(call ct_app_target,$(app)))) diff --git a/plugins/eunit.mk b/plugins/eunit.mk index f7fc81ed0..211a7442d 100644 --- a/plugins/eunit.mk +++ b/plugins/eunit.mk @@ -56,7 +56,7 @@ endif ifneq ($(ALL_APPS_DIRS),) apps-eunit: test-build - $(verbose) eunit_retcode=0 ; for app in $(ALL_APPS_DIRS); do $(MAKE) -C $$app eunit IS_APP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); \ + $(verbose) eunit_retcode=0 ; for app in $(ALL_APPS_DIRS); do $(MAKE) -C $$app eunit IS_APP=1; \ [ $$? -ne 0 ] && eunit_retcode=1 ; done ; \ exit $$eunit_retcode endif diff --git a/plugins/shell.mk b/plugins/shell.mk index 985ab70e9..7c1f8e6e9 100644 --- a/plugins/shell.mk +++ b/plugins/shell.mk @@ -31,7 +31,7 @@ build-shell-deps: $(ALL_SHELL_DEPS_DIRS) if [ -z "$(strip $(FULL))" ] && [ ! -L $$dep ] && [ -f $$dep/ebin/dep_built ]; then \ :; \ else \ - $(MAKE) -C $$dep IS_DEP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); \ + $(MAKE) -C $$dep IS_DEP=1; \ if [ ! -L $$dep ] && [ -d $$dep/ebin ]; then touch $$dep/ebin/dep_built; fi; \ fi \ done diff --git a/test/core_elixir.mk b/test/core_elixir.mk index d0083a6ac..492c10d3b 100644 --- a/test/core_elixir.mk +++ b/test/core_elixir.mk @@ -16,10 +16,11 @@ core-elixir-test-project_library: init $i "Configure Makefile" $t echo "ERLC_OPTS += +debug_info +'{parse_transform,lager_transform}'" >> $(APP)/Makefile - $t echo "ELIXIR_USE_SYSTEM = 0" >> $(APP)/Makefile + $t echo "DEPS += elixir" >> $(APP)/Makefile $t echo "DEPS += lager" >> $(APP)/Makefile $t echo "DEPS += jason" >> $(APP)/Makefile $t echo "DEPS += phoenix" >> $(APP)/Makefile + $t echo "dep_elixir_commit = v1.17.3" >> $(APP)/Makefile $t echo "dep_lager = git https://github.com/erlang-lager/lager master" >> $(APP)/Makefile $t echo "dep_jason = git https://github.com/michalmuskala/jason.git master" >> $(APP)/Makefile $t echo "dep_phoenix = hex 1.7.2" >> $(APP)/Makefile @@ -63,7 +64,6 @@ ifneq ($(shell which elixirc),) $i "Configure Makefile" $t echo "ERLC_OPTS += +debug_info +'{parse_transform,lager_transform}'" >> $(APP)/Makefile - $t echo "ELIXIR_USE_SYSTEM = 1" >> $(APP)/Makefile $t echo "DEPS += lager" >> $(APP)/Makefile $t echo "DEPS += jason" >> $(APP)/Makefile $t echo "DEPS += phoenix" >> $(APP)/Makefile @@ -112,7 +112,6 @@ core-elixir-test-project-rel: init $i "Configure Makefile" $t echo "ERLC_OPTS += +debug_info +'{parse_transform,lager_transform}'" >> $(APP)/Makefile - $t echo "ELIXIR_USE_SYSTEM = 1" >> $(APP)/Makefile $t echo "DEPS += lager" >> $(APP)/Makefile $t echo "DEPS += jason" >> $(APP)/Makefile $t echo "DEPS += phoenix" >> $(APP)/Makefile From da5f16e069d7f2c7f14e21fd3621024caa526292 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Fri, 6 Dec 2024 13:26:22 +0100 Subject: [PATCH 07/38] More cleanup --- core/deps.mk | 11 ++++++----- core/elixir.mk | 11 +++++------ plugins/escript.mk | 4 ---- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/core/deps.mk b/core/deps.mk index 4b4c2b92b..aedd983b6 100644 --- a/core/deps.mk +++ b/core/deps.mk @@ -892,12 +892,13 @@ endif .PHONY: autopatch-$(call query_name,$1) +ifeq ($1,elixir) +autopatch-elixir:: + ln -s lib/elixir/ebin $(DEPS_DIR)/elixir/ +else autopatch-$(call query_name,$1):: - $(verbose) if [ "$1" = "elixir" ]; then \ - ln -s lib/elixir/ebin $(DEPS_DIR)/elixir/; \ - else \ - $$(call dep_autopatch,$(call query_name,$1)) \ - fi + $$(call dep_autopatch,$(call query_name,$1)) +endif endef # We automatically depend on hex_core when the project isn't already. diff --git a/core/elixir.mk b/core/elixir.mk index d7dcd465c..f8d99ebdf 100644 --- a/core/elixir.mk +++ b/core/elixir.mk @@ -21,10 +21,6 @@ elixirc_verbose = $(elixirc_verbose_$(V)) #endif -ELIXIRC_OPTS += $(ERLC_OPTS) - -USES_ELIXIR = $(if $(EX_FILES),1,) - define elixir_get_deps.erl (fun(Deps) -> $(call hex_version_resolver.erl), @@ -188,11 +184,14 @@ end endef define dep_autopatch_mix - if test -d $(DEPS_DIR)/elixir; then $(MAKE) -C $(DEPS_DIR)/elixir || exit 1; fi; \ + if test -d $(DEPS_DIR)/elixir; then \ + $(MAKE) -C $(DEPS_DIR)/elixir; \ + touch $(DEPS_DIR)/hex_core/ebin/dep_built; \ + fi; \ $(MAKE) hex-core || exit 1; \ sed 's|\(defmodule.*do\)|\1\ntry do\nCode.compiler_options(on_undefined_variable: :warn)\nrescue _ -> :ok\nend|g' -i $(DEPS_DIR)/$(1)/mix.exs; \ MIX_ENV="$(if $(MIX_ENV),$(strip $(MIX_ENV)),prod)" \ - $(ERL) -pa $(DEPS_DIR)/hex_core $(addprefix -pa ,$(addsuffix /ebin,$(ELIXIR_BUILTINS))) \ + $(ERL) \ -eval "$(subst ",\",$(subst $(newline), ,$(subst $$,\$$,$(call Mix_Makefile.erl,$(1)))))." \ -eval "halt(0)." || exit 1 \ mkdir $(DEPS_DIR)/$1/src || exit 1 diff --git a/plugins/escript.mk b/plugins/escript.mk index b80482b2e..1790dcbcf 100644 --- a/plugins/escript.mk +++ b/plugins/escript.mk @@ -37,10 +37,6 @@ ifneq ($(DEPS),) $(subst $(DEPS_DIR)/,,$(addsuffix /*,$(wildcard \ $(addsuffix /ebin,$(shell cat $(ERLANG_MK_TMP)/deps.log))))) endif -ifneq ($(USES_ELIXIR),) - $(verbose) cd $(DEPS_DIR) && $(ESCRIPT_ZIP) $(abspath $(ESCRIPT_ZIP_FILE)) \ - $(addsuffix /*,$(wildcard $(addsuffix /ebin,$(ELIXIR_BUILTINS)))) -endif escript:: escript-zip $(gen_verbose) printf "%s\n" \ From 41a9b86fa0f0da8ac9e2f43a5f17ec2912e4eaf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Fri, 6 Dec 2024 13:43:42 +0100 Subject: [PATCH 08/38] Missed one --- core/docs.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/docs.mk b/core/docs.mk index cb9471447..6ef2ce6fe 100644 --- a/core/docs.mk +++ b/core/docs.mk @@ -16,5 +16,5 @@ ifneq ($(SKIP_DEPS),) doc-deps: else doc-deps: $(ALL_DOC_DEPS_DIRS) - $(verbose) set -e; for dep in $(ALL_DOC_DEPS_DIRS) ; do $(MAKE) -C $$dep IS_DEP=1 ELIXIR_USE_SYSTEM=$(ELIXIR_USE_SYSTEM); done + $(verbose) set -e; for dep in $(ALL_DOC_DEPS_DIRS) ; do $(MAKE) -C $$dep IS_DEP=1; done endif From 9df18e467292c6aa764c3846adda35ceaf43c182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Fri, 6 Dec 2024 13:45:49 +0100 Subject: [PATCH 09/38] Don't need this I think --- core/erlc.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/erlc.mk b/core/erlc.mk index 39e2ce134..cb4c8c93d 100644 --- a/core/erlc.mk +++ b/core/erlc.mk @@ -73,7 +73,7 @@ define app_file ]}. endef -app-build:: ebin/$(PROJECT).app +app-build: ebin/$(PROJECT).app $(verbose) : # Source files. From 544e71220b528173a3e39b7c96cc704576626710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 9 Dec 2024 13:51:25 +0100 Subject: [PATCH 10/38] Make Elixir autopatch -j compatible --- core/deps.mk | 89 +++++++++++++++++++++++++++++--------------------- core/elixir.mk | 8 +---- plugins/hex.mk | 24 +++++++------- 3 files changed, 64 insertions(+), 57 deletions(-) diff --git a/core/deps.mk b/core/deps.mk index aedd983b6..b77b88dc2 100644 --- a/core/deps.mk +++ b/core/deps.mk @@ -236,35 +236,38 @@ endif # Deps related targets. -# @todo rename GNUmakefile and makefile into Makefile first, if they exist -# While Makefile file could be GNUmakefile or makefile, -# in practice only Makefile is needed so far. -define dep_autopatch +define dep_autopatch_detect if [ -f $(DEPS_DIR)/$1/erlang.mk ]; then \ - rm -rf $(DEPS_DIR)/$1/ebin/; \ - $(call erlang,$(call dep_autopatch_appsrc.erl,$1)); \ - $(call dep_autopatch_erlang_mk,$1); \ + echo erlang.mk; \ + elif [ -f $(DEPS_DIR)/$1/rebar.config -o -f $(DEPS_DIR)/$1/rebar.config.script -o -f $(DEPS_DIR)/$1/rebar.lock ]; then \ + echo rebar3; \ elif [ -f $(DEPS_DIR)/$1/mix.exs -a -d $(DEPS_DIR)/$1/lib ]; then \ - $(call dep_autopatch_mix,$1); \ - elif [ -f $(DEPS_DIR)/$1/Makefile ]; then \ - if [ -f $(DEPS_DIR)/$1/rebar.lock ]; then \ - $(call dep_autopatch2,$1); \ - elif [ 0 != `grep -c "include ../\w*\.mk" $(DEPS_DIR)/$1/Makefile` ]; then \ - $(call dep_autopatch2,$1); \ - elif [ 0 != `grep -ci "^[^#].*rebar" $(DEPS_DIR)/$1/Makefile` ]; then \ - $(call dep_autopatch2,$1); \ - elif [ -n "`find $(DEPS_DIR)/$1/ -type f -name \*.mk -not -name erlang.mk -exec grep -i "^[^#].*rebar" '{}' \;`" ]; then \ - $(call dep_autopatch2,$1); \ - fi \ + echo mix; \ + elif [ ! -f $(DEPS_DIR)/$1/Makefile ]; then \ + echo noop; \ else \ - if [ ! -d $(DEPS_DIR)/$1/src/ ]; then \ - $(call dep_autopatch_noop,$1); \ - else \ - $(call dep_autopatch2,$1); \ - fi \ + echo rebar3; \ fi endef +define dep_autopatch_for_erlang.mk + rm -rf $(DEPS_DIR)/$1/ebin/; \ + $(call erlang,$(call dep_autopatch_appsrc.erl,$1)); \ + $(call dep_autopatch_erlang_mk,$1) +endef + +define dep_autopatch_for_rebar3 + $(call dep_autopatch2,$1) +endef + +define dep_autopatch_for_mix + $(call dep_autopatch_mix,$1) +endef + +define dep_autopatch_for_noop + $(call dep_autopatch_noop,$1) +endef + define dep_autopatch2 ! test -f $(DEPS_DIR)/$1/ebin/$1.app || \ mv -n $(DEPS_DIR)/$1/ebin/$1.app $(DEPS_DIR)/$1/src/$1.app.src; \ @@ -307,17 +310,21 @@ define dep_autopatch_gen "include ../../erlang.mk" > $(DEPS_DIR)/$1/Makefile endef -# We use flock/lockf when available to avoid concurrency issues. -define dep_autopatch_fetch_rebar +define maybe_flock if command -v flock >/dev/null; then \ - flock $(ERLANG_MK_TMP)/rebar.lock sh -c "$(call dep_autopatch_fetch_rebar2)"; \ + flock $1 sh -c "$2"; \ elif command -v lockf >/dev/null; then \ - lockf $(ERLANG_MK_TMP)/rebar.lock sh -c "$(call dep_autopatch_fetch_rebar2)"; \ + lockf $1 sh -c "$2"; \ else \ - $(call dep_autopatch_fetch_rebar2); \ + $2; \ fi endef +# We use flock/lockf when available to avoid concurrency issues. +define dep_autopatch_fetch_rebar + $(call maybe_flock,$(ERLANG_MK_TMP)/rebar.lock,$(call dep_autopatch_fetch_rebar2)) +endef + define dep_autopatch_fetch_rebar2 if [ ! -d $(ERLANG_MK_TMP)/rebar3 ]; then \ git clone -q -n -- $(REBAR3_GIT) $(ERLANG_MK_TMP)/rebar3; \ @@ -868,7 +875,7 @@ define dep_fetch_fail endef define dep_target -$(DEPS_DIR)/$(call query_name,$1): | $(if $(filter hex,$(call query_fetch_method,$1)),hex-core) $(ERLANG_MK_TMP) +$(DEPS_DIR)/$(call query_name,$1): $(if $(filter elixir,$(BUILD_DEPS) $(DEPS)),$(if $(filter-out elixir,$1),$(DEPS_DIR)/elixir/ebin/dep_built)) $(if $(filter hex,$(call query_fetch_method,$1)),$(DEPS_DIR)/hex_core/ebin/dep_built) | $(ERLANG_MK_TMP) $(eval DEP_NAME := $(call query_name,$1)) $(eval DEP_STR := $(if $(filter $1,$(DEP_NAME)),$1,"$1 ($(DEP_NAME))")) $(verbose) if test -d $(APPS_DIR)/$(DEP_NAME); then \ @@ -887,7 +894,7 @@ $(DEPS_DIR)/$(call query_name,$1): | $(if $(filter hex,$(call query_fetch_method cd $(DEPS_DIR)/$(DEP_NAME) && ./configure; \ fi ifeq ($(filter $1,$(NO_AUTOPATCH)),) - $(verbose) $$(MAKE) --no-print-directory autopatch-$(DEP_NAME) + $(verbose) $$(MAKE) --no-print-directory autopatch-$(DEP_NAME) AUTOPATCH_METHOD=`$(call dep_autopatch_detect,$1)` endif .PHONY: autopatch-$(call query_name,$1) @@ -897,7 +904,7 @@ autopatch-elixir:: ln -s lib/elixir/ebin $(DEPS_DIR)/elixir/ else autopatch-$(call query_name,$1):: - $$(call dep_autopatch,$(call query_name,$1)) + $$(call dep_autopatch_for_$(AUTOPATCH_METHOD),$(call query_name,$1)) endif endef @@ -905,13 +912,19 @@ endef $(if $(filter hex_core,$(DEPS) $(BUILD_DEPS) $(DOC_DEPS) $(REL_DEPS) $(TEST_DEPS)),,\ $(eval $(call dep_target,hex_core))) -.PHONY: hex-core - -hex-core: $(DEPS_DIR)/hex_core - $(verbose) if [ ! -e $(DEPS_DIR)/hex_core/ebin/dep_built ]; then \ - $(MAKE) -C $(DEPS_DIR)/hex_core IS_DEP=1; \ - touch $(DEPS_DIR)/hex_core/ebin/dep_built; \ - fi +$(DEPS_DIR)/hex_core/ebin/dep_built: $(DEPS_DIR)/hex_core | $(ERLANG_MK_TMP) + $(verbose) $(call maybe_flock,$(ERLANG_MK_TMP)/hex_core.lock,\ + if [ ! -e $(DEPS_DIR)/hex_core/ebin/dep_built ]; then \ + $(MAKE) -C $(DEPS_DIR)/hex_core IS_DEP=1; \ + touch $(DEPS_DIR)/hex_core/ebin/dep_built; \ + fi) + +$(DEPS_DIR)/elixir/ebin/dep_built: $(DEPS_DIR)/elixir | $(ERLANG_MK_TMP) + $(verbose) $(call maybe_flock,$(ERLANG_MK_TMP)/elixir.lock,\ + if [ ! -e $(DEPS_DIR)/elixir/ebin/dep_built ]; then \ + $(MAKE) -C $(DEPS_DIR)/elixir; \ + touch $(DEPS_DIR)/elixir/ebin/dep_built; \ + fi) $(foreach dep,$(BUILD_DEPS) $(DEPS),$(eval $(call dep_target,$(dep)))) diff --git a/core/elixir.mk b/core/elixir.mk index f8d99ebdf..3302bf5ea 100644 --- a/core/elixir.mk +++ b/core/elixir.mk @@ -184,17 +184,11 @@ end endef define dep_autopatch_mix - if test -d $(DEPS_DIR)/elixir; then \ - $(MAKE) -C $(DEPS_DIR)/elixir; \ - touch $(DEPS_DIR)/hex_core/ebin/dep_built; \ - fi; \ - $(MAKE) hex-core || exit 1; \ sed 's|\(defmodule.*do\)|\1\ntry do\nCode.compiler_options(on_undefined_variable: :warn)\nrescue _ -> :ok\nend|g' -i $(DEPS_DIR)/$(1)/mix.exs; \ MIX_ENV="$(if $(MIX_ENV),$(strip $(MIX_ENV)),prod)" \ $(ERL) \ -eval "$(subst ",\",$(subst $(newline), ,$(subst $$,\$$,$(call Mix_Makefile.erl,$(1)))))." \ - -eval "halt(0)." || exit 1 \ - mkdir $(DEPS_DIR)/$1/src || exit 1 + -eval "halt(0)." endef define compile_ex.erl diff --git a/plugins/hex.mk b/plugins/hex.mk index 44beb6583..2b52f4e52 100644 --- a/plugins/hex.mk +++ b/plugins/hex.mk @@ -19,7 +19,7 @@ define hex_user_create.erl endef # The $(info ) call inserts a new line after the password prompt. -hex-user-create: hex-core +hex-user-create: $(DEPS_DIR)/hex_core/ebin/dep_built $(if $(HEX_USERNAME),,$(eval HEX_USERNAME := $(shell read -p "Username: " username; echo $$username))) $(if $(HEX_PASSWORD),,$(eval HEX_PASSWORD := $(shell stty -echo; read -p "Password: " password; stty echo; echo $$password) $(info ))) $(if $(HEX_EMAIL),,$(eval HEX_EMAIL := $(shell read -p "Email: " email; echo $$email))) @@ -49,7 +49,7 @@ define hex_key_add.erl end endef -hex-key-add: hex-core +hex-key-add: $(DEPS_DIR)/hex_core/ebin/dep_built $(if $(HEX_USERNAME),,$(eval HEX_USERNAME := $(shell read -p "Username: " username; echo $$username))) $(if $(HEX_PASSWORD),,$(eval HEX_PASSWORD := $(shell stty -echo; read -p "Password: " password; stty echo; echo $$password) $(info ))) $(gen_verbose) $(call erlang,$(call hex_key_add.erl,$(HEX_USERNAME),$(HEX_PASSWORD),\ @@ -127,7 +127,7 @@ hex_tar_verbose_0 = @echo " TAR $(notdir $(ERLANG_MK_TMP))/$(@F)"; hex_tar_verbose_2 = set -x; hex_tar_verbose = $(hex_tar_verbose_$(V)) -$(HEX_TARBALL_OUTPUT_FILE): hex-core app +$(HEX_TARBALL_OUTPUT_FILE): $(DEPS_DIR)/hex_core/ebin/dep_built app $(hex_tar_verbose) $(call erlang,$(call hex_tarball_create.erl)) hex-tarball-create: $(HEX_TARBALL_OUTPUT_FILE) @@ -178,14 +178,14 @@ define hex_release_publish.erl end endef -hex-release-tarball: hex-core $(HEX_TARBALL_OUTPUT_FILE) +hex-release-tarball: $(DEPS_DIR)/hex_core/ebin/dep_built $(HEX_TARBALL_OUTPUT_FILE) $(verbose) $(call erlang,$(call hex_release_publish_summary.erl)) -hex-release-publish: hex-core hex-release-tarball +hex-release-publish: $(DEPS_DIR)/hex_core/ebin/dep_built hex-release-tarball $(if $(HEX_SECRET),,$(eval HEX_SECRET := $(shell stty -echo; read -p "Secret: " secret; stty echo; echo $$secret) $(info ))) $(gen_verbose) $(call erlang,$(call hex_release_publish.erl,$(HEX_SECRET),false)) -hex-release-replace: hex-core hex-release-tarball +hex-release-replace: $(DEPS_DIR)/hex_core/ebin/dep_built hex-release-tarball $(if $(HEX_SECRET),,$(eval HEX_SECRET := $(shell stty -echo; read -p "Secret: " secret; stty echo; echo $$secret) $(info ))) $(gen_verbose) $(call erlang,$(call hex_release_publish.erl,$(HEX_SECRET),true)) @@ -204,7 +204,7 @@ define hex_release_delete.erl end endef -hex-release-delete: hex-core +hex-release-delete: $(DEPS_DIR)/hex_core/ebin/dep_built $(if $(HEX_SECRET),,$(eval HEX_SECRET := $(shell stty -echo; read -p "Secret: " secret; stty echo; echo $$secret) $(info ))) $(gen_verbose) $(call erlang,$(call hex_release_delete.erl,$(HEX_SECRET))) @@ -224,7 +224,7 @@ define hex_release_retire.erl end endef -hex-release-retire: hex-core +hex-release-retire: $(DEPS_DIR)/hex_core/ebin/dep_built $(if $(HEX_SECRET),,$(eval HEX_SECRET := $(shell stty -echo; read -p "Secret: " secret; stty echo; echo $$secret) $(info ))) $(gen_verbose) $(call erlang,$(call hex_release_retire.erl,$(HEX_SECRET),\ $(if $(HEX_VERSION),$(HEX_VERSION),$(PROJECT_VERSION)),\ @@ -246,7 +246,7 @@ define hex_release_unretire.erl end endef -hex-release-unretire: hex-core +hex-release-unretire: $(DEPS_DIR)/hex_core/ebin/dep_built $(if $(HEX_SECRET),,$(eval HEX_SECRET := $(shell stty -echo; read -p "Secret: " secret; stty echo; echo $$secret) $(info ))) $(gen_verbose) $(call erlang,$(call hex_release_unretire.erl,$(HEX_SECRET),\ $(if $(HEX_VERSION),$(HEX_VERSION),$(PROJECT_VERSION)))) @@ -255,7 +255,7 @@ HEX_DOCS_DOC_DIR ?= doc/ HEX_DOCS_TARBALL_FILES ?= $(sort $(call core_find,$(HEX_DOCS_DOC_DIR),*)) HEX_DOCS_TARBALL_OUTPUT_FILE ?= $(ERLANG_MK_TMP)/$(PROJECT)-docs.tar.gz -$(HEX_DOCS_TARBALL_OUTPUT_FILE): hex-core app docs +$(HEX_DOCS_TARBALL_OUTPUT_FILE): $(DEPS_DIR)/hex_core/ebin/dep_built app docs $(hex_tar_verbose) tar czf $(HEX_DOCS_TARBALL_OUTPUT_FILE) -C $(HEX_DOCS_DOC_DIR) \ $(HEX_DOCS_TARBALL_FILES:$(HEX_DOCS_DOC_DIR)%=%) @@ -279,7 +279,7 @@ define hex_docs_publish.erl end endef -hex-docs-publish: hex-core hex-docs-tarball-create +hex-docs-publish: $(DEPS_DIR)/hex_core/ebin/dep_built hex-docs-tarball-create $(if $(HEX_SECRET),,$(eval HEX_SECRET := $(shell stty -echo; read -p "Secret: " secret; stty echo; echo $$secret) $(info ))) $(gen_verbose) $(call erlang,$(call hex_docs_publish.erl,$(HEX_SECRET))) @@ -299,7 +299,7 @@ define hex_docs_delete.erl end endef -hex-docs-delete: hex-core +hex-docs-delete: $(DEPS_DIR)/hex_core/ebin/dep_built $(if $(HEX_SECRET),,$(eval HEX_SECRET := $(shell stty -echo; read -p "Secret: " secret; stty echo; echo $$secret) $(info ))) $(gen_verbose) $(call erlang,$(call hex_docs_delete.erl,$(HEX_SECRET),\ $(if $(HEX_VERSION),$(HEX_VERSION),$(PROJECT_VERSION)))) From 330ad83c0fa41d9d9c6cc3767b6a7214f66bb082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 9 Dec 2024 14:31:04 +0100 Subject: [PATCH 11/38] Fix autopatch --- core/deps.mk | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/core/deps.mk b/core/deps.mk index b77b88dc2..ec71ce510 100644 --- a/core/deps.mk +++ b/core/deps.mk @@ -239,11 +239,21 @@ endif define dep_autopatch_detect if [ -f $(DEPS_DIR)/$1/erlang.mk ]; then \ echo erlang.mk; \ - elif [ -f $(DEPS_DIR)/$1/rebar.config -o -f $(DEPS_DIR)/$1/rebar.config.script -o -f $(DEPS_DIR)/$1/rebar.lock ]; then \ - echo rebar3; \ elif [ -f $(DEPS_DIR)/$1/mix.exs -a -d $(DEPS_DIR)/$1/lib ]; then \ echo mix; \ - elif [ ! -f $(DEPS_DIR)/$1/Makefile ]; then \ + elif [ -f $(DEPS_DIR)/$1/Makefile ]; then \ + if [ -f $(DEPS_DIR)/$1/rebar.lock ]; then \ + echo rebar3; \ + elif [ 0 != \`grep -c "include ../\w*\.mk" $(DEPS_DIR)/$1/Makefile\` ]; then \ + echo rebar3; \ + elif [ 0 != \`grep -ci "^[^#].*rebar" $(DEPS_DIR)/$1/Makefile\` ]; then \ + echo rebar3; \ + elif [ -n "\`find $(DEPS_DIR)/$1/ -type f -name \*.mk -not -name erlang.mk -exec grep -i "^[^#].*rebar" '{}' \;\`" ]; then \ + echo rebar3; \ + else \ + echo noop; \ + fi \ + elif [ ! -d $(DEPS_DIR)/$1/src/ ]; then \ echo noop; \ else \ echo rebar3; \ @@ -285,7 +295,7 @@ define dep_autopatch2 endef define dep_autopatch_noop - printf "noop:\n" > $(DEPS_DIR)/$1/Makefile + test -f $(DEPS_DIR)/$1/Makefile || printf "noop:\n" > $(DEPS_DIR)/$1/Makefile endef # Replace "include erlang.mk" with a line that will load the parent Erlang.mk From e8ceee428e267b876520586110567349ca3a12aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 9 Dec 2024 14:38:58 +0100 Subject: [PATCH 12/38] Debug --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 45ead96bb..a80187423 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,7 +31,7 @@ jobs: - core-compat # Trick GH runners into picking this slower job early. - ACME=1 c=core-deps - - core-elixir + - core-elixir V=3 - core-makedep - core-misc - core-plugins From 524511eb7f751806172b46dd6aa82aec9b454b1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 9 Dec 2024 14:49:52 +0100 Subject: [PATCH 13/38] Debug --- .github/workflows/ci.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a80187423..2bfd69d63 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -75,6 +75,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - run: which lockf || true + + - run: which flock || true + - name: Install Erlang/OTP uses: erlef/setup-beam@v1 with: From 5bc07e5c8361c5d116d6bafeabfc270fdb806433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 9 Dec 2024 15:12:54 +0100 Subject: [PATCH 14/38] Debug --- .github/workflows/ci.yaml | 72 ++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2bfd69d63..71d3381b3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -25,40 +25,40 @@ jobs: - ubuntu-latest # - windows-latest suite: - - core-app - - core-apps - - core-autopatch - - core-compat - # Trick GH runners into picking this slower job early. - - ACME=1 c=core-deps - - core-elixir V=3 - - core-makedep - - core-misc - - core-plugins - - core-query - - core-upgrade - - asciidoc - - bootstrap - - concuerror - - cover - - c-src - - ct - - dialyzer SET=1 - - dialyzer SET=2 - - dialyzer SET=3 - - dialyzer SET=4 - - edoc - - erlydtl - - escript - - eunit - - proper - - protobuffs - # Trick GH runners into picking this slower job early. - - ACME=1 c=relx - - shell - - sphinx - - triq - - xref +# - core-app +# - core-apps +# - core-autopatch +# - core-compat +# # Trick GH runners into picking this slower job early. +# - ACME=1 c=core-deps + - core-elixir-test-project_library V=3 +# - core-makedep +# - core-misc +# - core-plugins +# - core-query +# - core-upgrade +# - asciidoc +# - bootstrap +# - concuerror +# - cover +# - c-src +# - ct +# - dialyzer SET=1 +# - dialyzer SET=2 +# - dialyzer SET=3 +# - dialyzer SET=4 +# - edoc +# - erlydtl +# - escript +# - eunit +# - proper +# - protobuffs +# # Trick GH runners into picking this slower job early. +# - ACME=1 c=relx +# - shell +# - sphinx +# - triq +# - xref extra: ['', 'CACHE_DEPS=1', 'LEGACY=1'] exclude: # We disable proper/triq tests until ubuntu-latest includes GNU Make 4.4 or later. @@ -75,10 +75,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - run: which lockf || true - - - run: which flock || true - - name: Install Erlang/OTP uses: erlef/setup-beam@v1 with: From f21d95f38c0f80122c01dc9cdd33188924ff5b5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 9 Dec 2024 15:16:25 +0100 Subject: [PATCH 15/38] Debug --- test/core_elixir.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/test/core_elixir.mk b/test/core_elixir.mk index 492c10d3b..503245c8d 100644 --- a/test/core_elixir.mk +++ b/test/core_elixir.mk @@ -41,6 +41,7 @@ core-elixir-test-project_library: init $i "Get started apps" $t $(MAKE) -C $(APP) shell SHELL_OPTS="$(filter-out erl,$(ERL)) -pa $(APP)/deps/*/ebin/ $(APP)/ebin/ $(APP)/apps/*/ebin/ -eval \" \ {ok, Apps} = application:ensure_all_started('$(APP)'), \ + erlang:display(Apps), \ true = lists:member(lager, Apps), \ true = lists:member(jason, Apps), \ true = lists:member(phoenix, Apps), \ From 71e23fa745c58e95900eef443f0bd03930695812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 9 Dec 2024 15:25:04 +0100 Subject: [PATCH 16/38] Debug --- .github/workflows/ci.yaml | 2 +- test/core_elixir.mk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 71d3381b3..7efe37882 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,7 +31,7 @@ jobs: # - core-compat # # Trick GH runners into picking this slower job early. # - ACME=1 c=core-deps - - core-elixir-test-project_library V=3 + - core-elixir-test-project_system V=3 # - core-makedep # - core-misc # - core-plugins diff --git a/test/core_elixir.mk b/test/core_elixir.mk index 503245c8d..417f3f2d7 100644 --- a/test/core_elixir.mk +++ b/test/core_elixir.mk @@ -8,6 +8,7 @@ CORE_ELIXIR_TARGETS = $(call list_targets,core-elixir) core-elixir: $(CORE_ELIXIR_TARGETS) +# @todo This one doesn't work with LEGACY=1 core-elixir-test-project_library: init $i "Bootstrap a new OTP library named $(APP)" $t mkdir $(APP)/ @@ -41,7 +42,6 @@ core-elixir-test-project_library: init $i "Get started apps" $t $(MAKE) -C $(APP) shell SHELL_OPTS="$(filter-out erl,$(ERL)) -pa $(APP)/deps/*/ebin/ $(APP)/ebin/ $(APP)/apps/*/ebin/ -eval \" \ {ok, Apps} = application:ensure_all_started('$(APP)'), \ - erlang:display(Apps), \ true = lists:member(lager, Apps), \ true = lists:member(jason, Apps), \ true = lists:member(phoenix, Apps), \ From c5b753a49f557bb7ca708cd618002c1514ac3de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 9 Dec 2024 15:28:04 +0100 Subject: [PATCH 17/38] Debug --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7efe37882..85ec5d955 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,7 +31,7 @@ jobs: # - core-compat # # Trick GH runners into picking this slower job early. # - ACME=1 c=core-deps - - core-elixir-test-project_system V=3 + - core-elixir-test-project-rel V=3 # - core-makedep # - core-misc # - core-plugins From 3933501e967eeadcb6225fc4ad8810846ba1213d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 9 Dec 2024 15:39:13 +0100 Subject: [PATCH 18/38] Debug --- .github/workflows/ci.yaml | 5 ++++- test/core_elixir.mk | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 85ec5d955..df75a25ca 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,6 +20,8 @@ jobs: matrix: erlang: - '27' + elixir: + - '1.17' os: # - macos-latest - ubuntu-latest @@ -79,6 +81,7 @@ jobs: uses: erlef/setup-beam@v1 with: otp-version: ${{ matrix.erlang }} + elixir-version: ${{ matrix.elixir }} version-type: loose - name: Setup MSYS2 (Windows) @@ -144,7 +147,7 @@ jobs: uses: erlef/setup-beam@v1 with: otp-version: ${{ matrix.erlang }} - elixir-version: 1.17 + elixir-version: '1.17' version-type: loose - name: Setup MSYS2 (Windows) diff --git a/test/core_elixir.mk b/test/core_elixir.mk index 417f3f2d7..1c6fc4676 100644 --- a/test/core_elixir.mk +++ b/test/core_elixir.mk @@ -57,6 +57,7 @@ core-elixir-test-project_library: init halt()\"" core-elixir-test-project_system: init +# @todo ifneq ($(shell which elixirc),) $i "Bootstrap a new OTP library named $(APP)" $t mkdir $(APP)/ @@ -130,6 +131,7 @@ core-elixir-test-project-rel: init $t $(MAKE) -C $(APP) rel core-elixir-nif: init +# @todo ifneq ($(shell which cpp >/dev/null && echo '#include "sodium.h"' | cpp -H -o /dev/null 2>&1 | head -n1 | grep -v 'No such file or directory'),) $i "Bootstrap a new OTP library named $(APP)" $t mkdir $(APP)/ From 60fbc0f19facfcdc5fdf708461f55f602d2fbc6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 9 Dec 2024 15:58:22 +0100 Subject: [PATCH 19/38] Debug --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index df75a25ca..4ea8eeb3f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,8 +20,8 @@ jobs: matrix: erlang: - '27' - elixir: - - '1.17' + elixir: + - '1.17' os: # - macos-latest - ubuntu-latest From 560e8a830f87f501c7462dcdb2c34fd31842b9c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 9 Dec 2024 16:07:00 +0100 Subject: [PATCH 20/38] Debug end? --- .github/workflows/ci.yaml | 68 +++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4ea8eeb3f..830c19151 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,40 +27,40 @@ jobs: - ubuntu-latest # - windows-latest suite: -# - core-app -# - core-apps -# - core-autopatch -# - core-compat -# # Trick GH runners into picking this slower job early. -# - ACME=1 c=core-deps - - core-elixir-test-project-rel V=3 -# - core-makedep -# - core-misc -# - core-plugins -# - core-query -# - core-upgrade -# - asciidoc -# - bootstrap -# - concuerror -# - cover -# - c-src -# - ct -# - dialyzer SET=1 -# - dialyzer SET=2 -# - dialyzer SET=3 -# - dialyzer SET=4 -# - edoc -# - erlydtl -# - escript -# - eunit -# - proper -# - protobuffs -# # Trick GH runners into picking this slower job early. -# - ACME=1 c=relx -# - shell -# - sphinx -# - triq -# - xref + - core-app + - core-apps + - core-autopatch + - core-compat + # Trick GH runners into picking this slower job early. + - ACME=1 c=core-deps + - core-elixir + - core-makedep + - core-misc + - core-plugins + - core-query + - core-upgrade + - asciidoc + - bootstrap + - concuerror + - cover + - c-src + - ct + - dialyzer SET=1 + - dialyzer SET=2 + - dialyzer SET=3 + - dialyzer SET=4 + - edoc + - erlydtl + - escript + - eunit + - proper + - protobuffs + # Trick GH runners into picking this slower job early. + - ACME=1 c=relx + - shell + - sphinx + - triq + - xref extra: ['', 'CACHE_DEPS=1', 'LEGACY=1'] exclude: # We disable proper/triq tests until ubuntu-latest includes GNU Make 4.4 or later. From 08aba48f6902d6bd970ec7b22ed4de30c65cb68c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 9 Dec 2024 20:43:57 +0100 Subject: [PATCH 21/38] Fix the failing LEGACY=1 test --- test/core_elixir.mk | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/core_elixir.mk b/test/core_elixir.mk index 1c6fc4676..64eed6b3f 100644 --- a/test/core_elixir.mk +++ b/test/core_elixir.mk @@ -28,6 +28,11 @@ core-elixir-test-project_library: init $t echo "$$(grep -v 'include erlang.mk' $(APP)/Makefile)" > $(APP)/Makefile $t echo "include erlang.mk" >> $(APP)/Makefile +ifdef LEGACY + $i "Add Elixir, Lager, Jason and Phoenix to the applications key in the .app.src file" + $t perl -ni.bak -e 'print;if ($$.==7) {print "\t\telixir,\n\t\tlager,\n\t\tjason,\n\t\tphoenix,\n"}' $(APP)/src/$(APP).app.src +endif + $i "Make deps" $t $(MAKE) -C $(APP) deps $v From ca7a7777cbfacbca30bc0e01a827593b65b0968d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Tue, 10 Dec 2024 11:42:28 +0100 Subject: [PATCH 22/38] Rework tests --- core/elixir.mk | 3 +- test/core_elixir.mk | 227 +++++++++++++++++++++----------------------- 2 files changed, 111 insertions(+), 119 deletions(-) diff --git a/core/elixir.mk b/core/elixir.mk index 3302bf5ea..b0b4f8912 100644 --- a/core/elixir.mk +++ b/core/elixir.mk @@ -123,7 +123,8 @@ StartMod = _ -> "" end, -ExtraApps = [io_lib:format("LOCAL_DEPS += ~p~n", [App]) || App <- proplists:get_value(extra_applications, Application, [])], +ExtraApps0 = lists:usort([eex, elixir, logger, mix] ++ proplists:get_value(extra_applications, Application, [])), +ExtraApps = [io_lib:format("LOCAL_DEPS += ~p~n", [App]) || App <- ExtraApps0], ProjectCompilers = proplists:get_value(compilers, Project, []), "https://hexdocs.pm/elixir_make/Mix.Tasks.Compile.ElixirMake.html", ExtraMakeLines = diff --git a/test/core_elixir.mk b/test/core_elixir.mk index 64eed6b3f..19899f5f4 100644 --- a/test/core_elixir.mk +++ b/test/core_elixir.mk @@ -8,153 +8,144 @@ CORE_ELIXIR_TARGETS = $(call list_targets,core-elixir) core-elixir: $(CORE_ELIXIR_TARGETS) -# @todo This one doesn't work with LEGACY=1 -core-elixir-test-project_library: init +core-elixir-from-dep: init + $i "Bootstrap a new OTP library named $(APP)" $t mkdir $(APP)/ $t cp ../erlang.mk $(APP)/ - $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v - - $i "Configure Makefile" - $t echo "ERLC_OPTS += +debug_info +'{parse_transform,lager_transform}'" >> $(APP)/Makefile - $t echo "DEPS += elixir" >> $(APP)/Makefile - $t echo "DEPS += lager" >> $(APP)/Makefile - $t echo "DEPS += jason" >> $(APP)/Makefile - $t echo "DEPS += phoenix" >> $(APP)/Makefile - $t echo "dep_elixir_commit = v1.17.3" >> $(APP)/Makefile - $t echo "dep_lager = git https://github.com/erlang-lager/lager master" >> $(APP)/Makefile - $t echo "dep_jason = git https://github.com/michalmuskala/jason.git master" >> $(APP)/Makefile - $t echo "dep_phoenix = hex 1.7.2" >> $(APP)/Makefile - $t echo "$$(grep -v 'include erlang.mk' $(APP)/Makefile)" > $(APP)/Makefile - $t echo "include erlang.mk" >> $(APP)/Makefile + $t $(MAKE) -C $(APP) -f erlang.mk bootstrap-lib $v + + $i "Add Elixir, Lager, Jason, Phoenix to the list of dependencies" + $t perl -ni.bak -e 'print;if ($$.==1) {print "DEPS = elixir lager jason phoenix\ndep_elixir_commit = v1.17.3\ndep_lager = git https://github.com/erlang-lager/lager master\ndep_jason = git https://github.com/michalmuskala/jason.git master\ndep_phoenix = hex 1.7.2\n"}' $(APP)/Makefile + + $i "Add the lager_transform parse_transform to ERLC_OPTS" + $t echo "ERLC_OPTS += +'{parse_transform, lager_transform}'" >> $(APP)/Makefile ifdef LEGACY $i "Add Elixir, Lager, Jason and Phoenix to the applications key in the .app.src file" $t perl -ni.bak -e 'print;if ($$.==7) {print "\t\telixir,\n\t\tlager,\n\t\tjason,\n\t\tphoenix,\n"}' $(APP)/src/$(APP).app.src endif - $i "Make deps" - $t $(MAKE) -C $(APP) deps $v + $i "Build the application" + $t $(MAKE) -C $(APP) $v - $i "Check deps have compiled" - $t test -d $(APP)/deps/lager/ebin - $t test -d $(APP)/deps/jason/ebin - $t test -d $(APP)/deps/phoenix/ebin + $i "Check that all dependencies were fetched and built" + $t test -f $(APP)/deps/elixir/ebin/dep_built + $t test -f $(APP)/deps/lager/ebin/dep_built + $t test -f $(APP)/deps/jason/ebin/dep_built + $t test -f $(APP)/deps/phoenix/ebin/dep_built - $i "Make the app" - $t $(MAKE) -C $(APP) app $v - - $i "Get started apps" - $t $(MAKE) -C $(APP) shell SHELL_OPTS="$(filter-out erl,$(ERL)) -pa $(APP)/deps/*/ebin/ $(APP)/ebin/ $(APP)/apps/*/ebin/ -eval \" \ + $i "Check that the application was compiled correctly" + $t $(ERL) -pa $(APP)/ebin/ -pa $(APP)/deps/*/ebin -pa $(APP)/deps/elixir/lib/*/ebin -eval " \ {ok, Apps} = application:ensure_all_started('$(APP)'), \ + true = lists:member(elixir, Apps), \ true = lists:member(lager, Apps), \ true = lists:member(jason, Apps), \ true = lists:member(phoenix, Apps), \ - halt()\"" + halt()" + + $i "Check that the Jason application depends on Elixir builtins" + $t $(ERL) -pa $(APP)/ebin/ -pa $(APP)/deps/*/ebin -pa $(APP)/deps/elixir/lib/*/ebin -eval " \ + {ok, Apps} = application:ensure_all_started(jason), \ + true = lists:member(elixir, Apps), \ + true = lists:member(eex, Apps), \ + true = lists:member(logger, Apps), \ + true = lists:member(mix, Apps), \ + halt()" + +core-elixir-from-system: init - $i "Check modules aren't duplicated" - $t $(MAKE) -C $(APP) shell SHELL_OPTS="$(filter-out erl,$(ERL)) -pa $(APP)/deps/*/ebin/ $(APP)/ebin/ $(APP)/apps/*/ebin/ -eval \" \ - {ok, Apps} = application:ensure_all_started('$(APP)'), \ - [begin \ - {ok, Mods} = application:get_key(App, modules), \ - true = lists:sort(Mods) =:= lists:usort(Mods) \ - end || App <- Apps], \ - halt()\"" - -core-elixir-test-project_system: init -# @todo -ifneq ($(shell which elixirc),) $i "Bootstrap a new OTP library named $(APP)" $t mkdir $(APP)/ $t cp ../erlang.mk $(APP)/ - $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v - - $i "Configure Makefile" - $t echo "ERLC_OPTS += +debug_info +'{parse_transform,lager_transform}'" >> $(APP)/Makefile - $t echo "DEPS += lager" >> $(APP)/Makefile - $t echo "DEPS += jason" >> $(APP)/Makefile - $t echo "DEPS += phoenix" >> $(APP)/Makefile - $t echo "dep_lager = git https://github.com/erlang-lager/lager master" >> $(APP)/Makefile - $t echo "dep_jason = git https://github.com/michalmuskala/jason.git master" >> $(APP)/Makefile - $t echo "dep_phoenix = hex 1.7.2" >> $(APP)/Makefile - $t echo "$$(grep -v 'include erlang.mk' $(APP)/Makefile)" > $(APP)/Makefile - $t echo "include erlang.mk" >> $(APP)/Makefile - - $i "Make deps" - $t $(MAKE) -C $(APP) deps $v - - $i "Check deps have compiled" - $t test -d $(APP)/deps/lager/ebin - $t test -d $(APP)/deps/jason/ebin - $t test -d $(APP)/deps/phoenix/ebin - - $i "Make the app" - $t $(MAKE) -C $(APP) app $v - - $i "Get started apps" - $t $(MAKE) -C $(APP) shell SHELL_OPTS="$(filter-out erl,$(ERL)) -pa $(APP)/deps/*/ebin/ $(APP)/ebin/ $(APP)/apps/*/ebin/ -eval \" \ + $t $(MAKE) -C $(APP) -f erlang.mk bootstrap-lib $v + + $i "Add Lager, Jason, Phoenix to the list of dependencies" + $t perl -ni.bak -e 'print;if ($$.==1) {print "DEPS = lager jason phoenix\ndep_lager = git https://github.com/erlang-lager/lager master\ndep_jason = git https://github.com/michalmuskala/jason.git master\ndep_phoenix = hex 1.7.2\n"}' $(APP)/Makefile + + $i "Add the lager_transform parse_transform to ERLC_OPTS" + $t echo "ERLC_OPTS += +'{parse_transform, lager_transform}'" >> $(APP)/Makefile + +ifdef LEGACY + $i "Add Lager, Jason and Phoenix to the applications key in the .app.src file" + $t perl -ni.bak -e 'print;if ($$.==7) {print "\t\telixir,\n\t\tlager,\n\t\tjason,\n\t\tphoenix,\n"}' $(APP)/src/$(APP).app.src +endif + + $i "Build the application" + $t $(MAKE) -C $(APP) $v + + $i "Check that all dependencies were fetched and built" + $t ! test -e $(APP)/deps/elixir + $t test -f $(APP)/deps/lager/ebin/dep_built + $t test -f $(APP)/deps/jason/ebin/dep_built + $t test -f $(APP)/deps/phoenix/ebin/dep_built + + $i "Check that the application was compiled correctly" + $t $(ERL) -pa $(APP)/ebin/ -pa $(APP)/deps/*/ebin -pa $(dir $(shell elixir -e 'IO.puts(:code.lib_dir(:elixir))'))/*/ebin -eval " \ {ok, Apps} = application:ensure_all_started('$(APP)'), \ true = lists:member(lager, Apps), \ true = lists:member(jason, Apps), \ true = lists:member(phoenix, Apps), \ - halt()\"" + halt()" - $i "Check modules aren't duplicated" - $t $(MAKE) -C $(APP) shell SHELL_OPTS="$(filter-out erl,$(ERL)) -pa $(APP)/deps/*/ebin/ $(APP)/ebin/ $(APP)/apps/*/ebin/ -eval \" \ - {ok, Apps} = application:ensure_all_started('$(APP)'), \ - [begin \ - {ok, Mods} = application:get_key(App, modules), \ - true = lists:sort(Mods) =:= lists:usort(Mods) \ - end || App <- Apps], \ - halt()\"" -else - $i "Test depends on a System Install of Elixir, skipping." -endif + $i "Check that the Jason application depends on Elixir builtins" + $t $(ERL) -pa $(APP)/ebin/ -pa $(APP)/deps/*/ebin -pa $(dir $(shell elixir -e 'IO.puts(:code.lib_dir(:elixir))'))/*/ebin -eval " \ + {ok, Apps} = application:ensure_all_started(jason), \ + true = lists:member(elixir, Apps), \ + true = lists:member(eex, Apps), \ + true = lists:member(logger, Apps), \ + true = lists:member(mix, Apps), \ + halt()" + +core-elixir-nif: init -core-elixir-test-project-rel: init $i "Bootstrap a new OTP library named $(APP)" $t mkdir $(APP)/ $t cp ../erlang.mk $(APP)/ - $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v - - $i "Configure Makefile" - $t echo "ERLC_OPTS += +debug_info +'{parse_transform,lager_transform}'" >> $(APP)/Makefile - $t echo "DEPS += lager" >> $(APP)/Makefile - $t echo "DEPS += jason" >> $(APP)/Makefile - $t echo "DEPS += phoenix" >> $(APP)/Makefile - $t echo "dep_lager = git https://github.com/erlang-lager/lager master" >> $(APP)/Makefile - $t echo "dep_jason = git https://github.com/michalmuskala/jason.git master" >> $(APP)/Makefile - $t echo "dep_phoenix = hex 1.7.2" >> $(APP)/Makefile - $t echo "$$(grep -v 'include erlang.mk' $(APP)/Makefile)" > $(APP)/Makefile - $t echo "include erlang.mk" >> $(APP)/Makefile - - $i "Make deps" - $t $(MAKE) -C $(APP) deps $v - - $i "Check a release can be made" - $t $(MAKE) -C $(APP) bootstrap-rel - $t $(MAKE) -C $(APP) rel + $t $(MAKE) -C $(APP) -f erlang.mk bootstrap-lib $v -core-elixir-nif: init -# @todo -ifneq ($(shell which cpp >/dev/null && echo '#include "sodium.h"' | cpp -H -o /dev/null 2>&1 | head -n1 | grep -v 'No such file or directory'),) - $i "Bootstrap a new OTP library named $(APP)" + $i "Add Libsalty2 to the list of dependencies" + $t perl -ni.bak -e 'print;if ($$.==1) {print "DEPS = libsalty2\ndep_libsalty2 = git https://github.com/Ianleeclark/libsalty2.git b11e544\n"}' $(APP)/Makefile + +ifdef LEGACY + $i "Add Libsalty2 to the applications key in the .app.src file" + $t perl -ni.bak -e 'print;if ($$.==7) {print "\t\tlibsalty2,\n"}' $(APP)/src/$(APP).app.src +endif + + $i "Build the application" + $t $(MAKE) -C $(APP) $v + + $i "Check that the application was compiled correctly" + $t $(ERL) -pa $(APP)/ebin/ -pa $(APP)/deps/*/ebin -pa $(dir $(shell elixir -e 'IO.puts(:code.lib_dir(:elixir))'))/*/ebin -eval " \ + {ok, Apps} = application:ensure_all_started('$(APP)'), \ + true = lists:member(libsalty2, Apps), \ + halt()" + +core-elixir-rel: init + + $i "Bootstrap a new release named $(APP)" $t mkdir $(APP)/ $t cp ../erlang.mk $(APP)/ - $t $(MAKE) -C $(APP) -f erlang.mk bootstrap $v - - $i "Configure Makefile" - $t echo "DEPS += libsalty2" >> $(APP)/Makefile - $t echo "dep_libsalty2 = git https://github.com/Ianleeclark/libsalty2.git b11e544" >> $(APP)/Makefile - $t echo "$$(grep -v 'include erlang.mk' $(APP)/Makefile)" > $(APP)/Makefile - $t echo "include erlang.mk" >> $(APP)/Makefile - - $i "Make deps" - $t $(MAKE) -C $(APP) deps $v - - $i "Check libsalty2 has compiled" - $t test -f $(APP)/deps/libsalty2/ebin/libsalty2.app - $t test -f $(APP)/deps/libsalty2/priv/salty_nif.so -else - $i "Test depends on libsodium-dev, skipping." + $t $(MAKE) -C $(APP) -f erlang.mk bootstrap-lib bootstrap-rel $v + + $i "Add Lager, Jason, Phoenix to the list of dependencies" + $t perl -ni.bak -e 'print;if ($$.==1) {print "DEPS = lager jason phoenix\ndep_lager = git https://github.com/erlang-lager/lager master\ndep_jason = git https://github.com/michalmuskala/jason.git master\ndep_phoenix = hex 1.7.2\n"}' $(APP)/Makefile + + $i "Add the lager_transform parse_transform to ERLC_OPTS" + $t echo "ERLC_OPTS += +'{parse_transform, lager_transform}'" >> $(APP)/Makefile + +ifdef LEGACY + $i "Add Lager, Jason and Phoenix to the applications key in the .app.src file" + $t perl -ni.bak -e 'print;if ($$.==7) {print "\t\tlager,\n\t\tjason,\n\t\tphoenix,\n"}' $(APP)/src/$(APP).app.src endif + + $i "Build the release" + $t $(MAKE) -C $(APP) $v + + $i "Check that the release was built" + $t test -d $(APP)/_rel + $t test -d $(APP)/_rel/$(APP)_release + $t test -d $(APP)/_rel/$(APP)_release/bin + $t test -d $(APP)/_rel/$(APP)_release/lib + $t test -d $(APP)/_rel/$(APP)_release/releases + $t test -d $(APP)/_rel/$(APP)_release/releases/1 From 91636d9306a419e449c560bda99669c2f8cbf563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Tue, 10 Dec 2024 11:52:41 +0100 Subject: [PATCH 23/38] Install libsodium in CI when needed --- .github/workflows/ci.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 830c19151..040a6ec4b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -98,6 +98,10 @@ jobs: gnu-netcat diffutils + - name: Install libsodium (Linux, Elixir) + if: matrix.os == 'ubuntu-latest' && matrix.suite == 'core-elixir' + run: sudo apt-get -y install libsodium-dev + - name: Run tests (Linux) if: matrix.os == 'ubuntu-latest' run: make check c=${{ matrix.suite }} -j4 -k ${{ matrix.extra }} From 84e404414f9ea8af37a9d25423eb2dd370c54087 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Tue, 10 Dec 2024 12:03:21 +0100 Subject: [PATCH 24/38] Debug --- .github/workflows/ci.yaml | 68 +++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 040a6ec4b..983b6ec32 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,40 +27,40 @@ jobs: - ubuntu-latest # - windows-latest suite: - - core-app - - core-apps - - core-autopatch - - core-compat - # Trick GH runners into picking this slower job early. - - ACME=1 c=core-deps - - core-elixir - - core-makedep - - core-misc - - core-plugins - - core-query - - core-upgrade - - asciidoc - - bootstrap - - concuerror - - cover - - c-src - - ct - - dialyzer SET=1 - - dialyzer SET=2 - - dialyzer SET=3 - - dialyzer SET=4 - - edoc - - erlydtl - - escript - - eunit - - proper - - protobuffs - # Trick GH runners into picking this slower job early. - - ACME=1 c=relx - - shell - - sphinx - - triq - - xref +# - core-app +# - core-apps +# - core-autopatch +# - core-compat +# # Trick GH runners into picking this slower job early. +# - ACME=1 c=core-deps + - core-elixir-rel V=3 +# - core-makedep +# - core-misc +# - core-plugins +# - core-query +# - core-upgrade +# - asciidoc +# - bootstrap +# - concuerror +# - cover +# - c-src +# - ct +# - dialyzer SET=1 +# - dialyzer SET=2 +# - dialyzer SET=3 +# - dialyzer SET=4 +# - edoc +# - erlydtl +# - escript +# - eunit +# - proper +# - protobuffs +# # Trick GH runners into picking this slower job early. +# - ACME=1 c=relx +# - shell +# - sphinx +# - triq +# - xref extra: ['', 'CACHE_DEPS=1', 'LEGACY=1'] exclude: # We disable proper/triq tests until ubuntu-latest includes GNU Make 4.4 or later. From 56360f7967985b293f9a5bd8172dc071e52e21d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Tue, 10 Dec 2024 12:07:03 +0100 Subject: [PATCH 25/38] Debug --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 983b6ec32..74c6c4ff6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -33,7 +33,7 @@ jobs: # - core-compat # # Trick GH runners into picking this slower job early. # - ACME=1 c=core-deps - - core-elixir-rel V=3 + - core-elixir V=3 # - core-makedep # - core-misc # - core-plugins From 81934daceff11b62524eca0525e701a0f2c4ba6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Tue, 10 Dec 2024 12:13:16 +0100 Subject: [PATCH 26/38] Fix more concurrency issues? --- core/deps.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/deps.mk b/core/deps.mk index ec71ce510..434367e55 100644 --- a/core/deps.mk +++ b/core/deps.mk @@ -922,14 +922,14 @@ endef $(if $(filter hex_core,$(DEPS) $(BUILD_DEPS) $(DOC_DEPS) $(REL_DEPS) $(TEST_DEPS)),,\ $(eval $(call dep_target,hex_core))) -$(DEPS_DIR)/hex_core/ebin/dep_built: $(DEPS_DIR)/hex_core | $(ERLANG_MK_TMP) +$(DEPS_DIR)/hex_core/ebin/dep_built: | $(DEPS_DIR)/hex_core $(ERLANG_MK_TMP) $(verbose) $(call maybe_flock,$(ERLANG_MK_TMP)/hex_core.lock,\ if [ ! -e $(DEPS_DIR)/hex_core/ebin/dep_built ]; then \ $(MAKE) -C $(DEPS_DIR)/hex_core IS_DEP=1; \ touch $(DEPS_DIR)/hex_core/ebin/dep_built; \ fi) -$(DEPS_DIR)/elixir/ebin/dep_built: $(DEPS_DIR)/elixir | $(ERLANG_MK_TMP) +$(DEPS_DIR)/elixir/ebin/dep_built: | $(DEPS_DIR)/elixir $(ERLANG_MK_TMP) $(verbose) $(call maybe_flock,$(ERLANG_MK_TMP)/elixir.lock,\ if [ ! -e $(DEPS_DIR)/elixir/ebin/dep_built ]; then \ $(MAKE) -C $(DEPS_DIR)/elixir; \ From 1c281a6fb4c09a5086e955f9d8bcf21f88fdd1e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Tue, 10 Dec 2024 12:50:07 +0100 Subject: [PATCH 27/38] Try again --- core/deps.mk | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/deps.mk b/core/deps.mk index 434367e55..a5ad2d2c4 100644 --- a/core/deps.mk +++ b/core/deps.mk @@ -922,16 +922,18 @@ endef $(if $(filter hex_core,$(DEPS) $(BUILD_DEPS) $(DOC_DEPS) $(REL_DEPS) $(TEST_DEPS)),,\ $(eval $(call dep_target,hex_core))) -$(DEPS_DIR)/hex_core/ebin/dep_built: | $(DEPS_DIR)/hex_core $(ERLANG_MK_TMP) +$(DEPS_DIR)/hex_core/ebin/dep_built: | $(ERLANG_MK_TMP) $(verbose) $(call maybe_flock,$(ERLANG_MK_TMP)/hex_core.lock,\ if [ ! -e $(DEPS_DIR)/hex_core/ebin/dep_built ]; then \ + $(MAKE) $(DEPS_DIR)/hex_core; \ $(MAKE) -C $(DEPS_DIR)/hex_core IS_DEP=1; \ touch $(DEPS_DIR)/hex_core/ebin/dep_built; \ fi) -$(DEPS_DIR)/elixir/ebin/dep_built: | $(DEPS_DIR)/elixir $(ERLANG_MK_TMP) +$(DEPS_DIR)/elixir/ebin/dep_built: | $(ERLANG_MK_TMP) $(verbose) $(call maybe_flock,$(ERLANG_MK_TMP)/elixir.lock,\ if [ ! -e $(DEPS_DIR)/elixir/ebin/dep_built ]; then \ + $(MAKE) $(DEPS_DIR)/elixir; \ $(MAKE) -C $(DEPS_DIR)/elixir; \ touch $(DEPS_DIR)/elixir/ebin/dep_built; \ fi) From 95e6cdfc379866cc50d9bbfbc898f26510d5393a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Tue, 10 Dec 2024 13:04:38 +0100 Subject: [PATCH 28/38] Put it back --- .github/workflows/ci.yaml | 68 +++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 74c6c4ff6..040a6ec4b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,40 +27,40 @@ jobs: - ubuntu-latest # - windows-latest suite: -# - core-app -# - core-apps -# - core-autopatch -# - core-compat -# # Trick GH runners into picking this slower job early. -# - ACME=1 c=core-deps - - core-elixir V=3 -# - core-makedep -# - core-misc -# - core-plugins -# - core-query -# - core-upgrade -# - asciidoc -# - bootstrap -# - concuerror -# - cover -# - c-src -# - ct -# - dialyzer SET=1 -# - dialyzer SET=2 -# - dialyzer SET=3 -# - dialyzer SET=4 -# - edoc -# - erlydtl -# - escript -# - eunit -# - proper -# - protobuffs -# # Trick GH runners into picking this slower job early. -# - ACME=1 c=relx -# - shell -# - sphinx -# - triq -# - xref + - core-app + - core-apps + - core-autopatch + - core-compat + # Trick GH runners into picking this slower job early. + - ACME=1 c=core-deps + - core-elixir + - core-makedep + - core-misc + - core-plugins + - core-query + - core-upgrade + - asciidoc + - bootstrap + - concuerror + - cover + - c-src + - ct + - dialyzer SET=1 + - dialyzer SET=2 + - dialyzer SET=3 + - dialyzer SET=4 + - edoc + - erlydtl + - escript + - eunit + - proper + - protobuffs + # Trick GH runners into picking this slower job early. + - ACME=1 c=relx + - shell + - sphinx + - triq + - xref extra: ['', 'CACHE_DEPS=1', 'LEGACY=1'] exclude: # We disable proper/triq tests until ubuntu-latest includes GNU Make 4.4 or later. From 09cee52e2b8157cc1a99ba1b55758968e374a301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Tue, 10 Dec 2024 14:04:36 +0100 Subject: [PATCH 29/38] Move things around --- core/deps.mk | 38 +++++++++++++++----------------------- core/elixir.mk | 8 ++++---- 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/core/deps.mk b/core/deps.mk index a5ad2d2c4..55cd58dc2 100644 --- a/core/deps.mk +++ b/core/deps.mk @@ -267,18 +267,6 @@ define dep_autopatch_for_erlang.mk endef define dep_autopatch_for_rebar3 - $(call dep_autopatch2,$1) -endef - -define dep_autopatch_for_mix - $(call dep_autopatch_mix,$1) -endef - -define dep_autopatch_for_noop - $(call dep_autopatch_noop,$1) -endef - -define dep_autopatch2 ! test -f $(DEPS_DIR)/$1/ebin/$1.app || \ mv -n $(DEPS_DIR)/$1/ebin/$1.app $(DEPS_DIR)/$1/src/$1.app.src; \ rm -f $(DEPS_DIR)/$1/ebin/$1.app; \ @@ -294,10 +282,24 @@ define dep_autopatch2 fi endef -define dep_autopatch_noop +define dep_autopatch_for_mix + $(call dep_autopatch_mix,$1) +endef + +define dep_autopatch_for_noop test -f $(DEPS_DIR)/$1/Makefile || printf "noop:\n" > $(DEPS_DIR)/$1/Makefile endef +define maybe_flock + if command -v flock >/dev/null; then \ + flock $1 sh -c "$2"; \ + elif command -v lockf >/dev/null; then \ + lockf $1 sh -c "$2"; \ + else \ + $2; \ + fi +endef + # Replace "include erlang.mk" with a line that will load the parent Erlang.mk # if given. Do it for all 3 possible Makefile file names. ifeq ($(NO_AUTOPATCH_ERLANG_MK),) @@ -320,16 +322,6 @@ define dep_autopatch_gen "include ../../erlang.mk" > $(DEPS_DIR)/$1/Makefile endef -define maybe_flock - if command -v flock >/dev/null; then \ - flock $1 sh -c "$2"; \ - elif command -v lockf >/dev/null; then \ - lockf $1 sh -c "$2"; \ - else \ - $2; \ - fi -endef - # We use flock/lockf when available to avoid concurrency issues. define dep_autopatch_fetch_rebar $(call maybe_flock,$(ERLANG_MK_TMP)/rebar.lock,$(call dep_autopatch_fetch_rebar2)) diff --git a/core/elixir.mk b/core/elixir.mk index b0b4f8912..a6f21b378 100644 --- a/core/elixir.mk +++ b/core/elixir.mk @@ -1,8 +1,10 @@ +# Copyright (c) 2024, Tyler Hughes +# Copyright (c) 2024, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + ALL_LIB_FILES := $(sort $(call core_find,lib/,*)) EX_FILES := $(filter-out lib/mix/%,$(filter %.ex,$(ALL_SRC_FILES) $(ALL_LIB_FILES))) -#ifneq ($(strip $(EX_FILES)),) - ELIXIR ?= $(if $(filter elixir,$(BUILD_DEPS) $(DEPS)),dep,system) export ELIXIR @@ -19,8 +21,6 @@ elixirc_verbose_0 = @echo " ELIXIRC " $(filter-out $(patsubst %,%.ex,$(ERLC_EXC elixirc_verbose_2 = set -x; elixirc_verbose = $(elixirc_verbose_$(V)) -#endif - define elixir_get_deps.erl (fun(Deps) -> $(call hex_version_resolver.erl), From 8df75ed82aa87aed7ad97b9e824fcb7bc4e474ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Wed, 11 Dec 2024 13:07:36 +0100 Subject: [PATCH 30/38] Rework Mix autopatch and fix another concurrency issue --- core/deps.mk | 4 +- core/elixir.mk | 242 +++++++++++++++++++------------------------------ 2 files changed, 95 insertions(+), 151 deletions(-) diff --git a/core/deps.mk b/core/deps.mk index 55cd58dc2..21eac4fde 100644 --- a/core/deps.mk +++ b/core/deps.mk @@ -118,7 +118,9 @@ dep_name = $(call query_name,$(1)) # Application directories. LOCAL_DEPS_DIRS = $(foreach a,$(LOCAL_DEPS),$(if $(wildcard $(APPS_DIR)/$a),$(APPS_DIR)/$a)) -ALL_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(foreach dep,$(filter-out $(IGNORE_DEPS),$(BUILD_DEPS) $(DEPS)),$(call query_name,$(dep)))) +# Elixir is handled specially as it must be built before all other deps +# when Mix autopatching is necessary. +ALL_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(foreach dep,$(filter-out elixir $(IGNORE_DEPS),$(BUILD_DEPS) $(DEPS)),$(call query_name,$(dep)))) # When we are calling an app directly we don't want to include it here # otherwise it'll be treated both as an apps and a top-level project. diff --git a/core/elixir.mk b/core/elixir.mk index a6f21b378..ca9b369c6 100644 --- a/core/elixir.mk +++ b/core/elixir.mk @@ -21,175 +21,117 @@ elixirc_verbose_0 = @echo " ELIXIRC " $(filter-out $(patsubst %,%.ex,$(ERLC_EXC elixirc_verbose_2 = set -x; elixirc_verbose = $(elixirc_verbose_$(V)) -define elixir_get_deps.erl -(fun(Deps) -> +define dep_autopatch_mix.erl $(call hex_version_resolver.erl), - (fun - F([], DEPS_Acc0, DEP_Acc0) -> - [DEPS_Acc0, "\n", DEP_Acc0]; - F([H|T], DEPS_Acc0, DEP_Acc0) -> - {DEPS_Acc1, DEP_Acc1} = - case H of - {Name, Req} when is_binary(Req) -> - {ok, Vsn} = HexVersionResolve(Name, Req), - { - [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], - [DEP_Acc0, io_lib:format("dep_~p = hex ~s ~p~n", [Name, Vsn, Name])] - }; - {Name, Opts} when is_list(Opts) -> - Path = proplists:get_value(path, Opts), - IsRequired = proplists:get_value(optional, Opts) =/= true, - IsProdOnly = case proplists:get_value(only, Opts, prod) of - prod -> true; - L when is_list(L) -> lists:member(prod, L); - _ -> false - end, - case IsRequired andalso IsProdOnly of - true when Path =/= undefined -> - { - [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], - [DEP_Acc0, io_lib:format("dep_~p = ln ~s~n", [Name, Path])] - }; - true when Path =:= undefined -> - io:format(standard_error, "Skipping 'dep_~p' as no vsn given.", [Name]), - { - [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], - DEP_Acc0 - }; - false -> - {DEPS_Acc0, DEP_Acc0} - end; - {Name, Req, Opts} -> - IsRequired = proplists:get_value(optional, Opts) =/= true, - IsProdOnly = case proplists:get_value(only, Opts, prod) of - prod -> true; - L when is_list(L) -> lists:member(prod, L); - _ -> false - end, - case IsRequired andalso IsProdOnly of - true -> - {ok, Vsn} = HexVersionResolve(Name, Req), - { - [DEPS_Acc0, io_lib:format("DEPS += ~p~n", [Name])], - [DEP_Acc0, io_lib:format("dep_~p = hex ~s ~p~n", [Name, Vsn, Name])] - }; - false -> - {DEPS_Acc0, DEP_Acc0} - end; - _ -> - {DEPS_Acc0, DEP_Acc0} - end, - F(T, DEPS_Acc1, DEP_Acc1) - end)(Deps, [], []) -end)($1) -endef - -define Mix_Makefile.erl -{ok, _} = application:ensure_all_started(elixir), -{ok, _} = application:ensure_all_started(mix), -File = <<"$(DEPS_DIR)/$(1)/mix.exs">>, -[{Mod, Bin}] = elixir_compiler:file(File, fun(_File, _LexerPid) -> ok end), -{module, Mod} = code:load_binary(Mod, binary_to_list(File), Bin), -Project = Mod:project(), -Application = try Mod:application() catch error:undef -> [] end, -Fmt = - "PROJECT = ~p~n" - "PROJECT_DESCRIPTION = ~s~n" - "PROJECT_VERSION = ~s~n" - "PROJECT_MOD = ~s~n" - "define PROJECT_ENV~n" - "~p~n" - "endef~n" - "~n~n~s~n~n" - "~n~n~s~n~n" - "~n~s~n" - "ERLC_OPTS = +debug_info~n" - "include $$(if $$(ERLANG_MK_FILENAME),$$(ERLANG_MK_FILENAME),erlang.mk)", -LFirst = fun - F([], Pred) -> - false; - F([H|T], Pred) -> - case catch Pred(H) of - true -> - {ok, H}; - false -> - F(T, Pred) - end -end, -StartMod = - case lists:keyfind(mod, 1, Application) of - {mod, {StartMod_, _StartArgs}} -> - atom_to_list(StartMod_); + {ok, _} = application:ensure_all_started(elixir), + {ok, _} = application:ensure_all_started(mix), + MixFile = <<"$(call core_native_path,$(DEPS_DIR)/$1/mix.exs)">>, + [{Mod, Bin}] = elixir_compiler:file(MixFile, fun(_File, _LexerPid) -> ok end), + {module, Mod} = code:load_binary(Mod, binary_to_list(MixFile), Bin), + Project = Mod:project(), + Application = try Mod:application() catch error:undef -> [] end, + StartMod = case lists:keyfind(mod, 1, Application) of + {mod, {StartMod0, _StartArgs}} -> + StartMod0; _ -> "" end, -ExtraApps0 = lists:usort([eex, elixir, logger, mix] ++ proplists:get_value(extra_applications, Application, [])), -ExtraApps = [io_lib:format("LOCAL_DEPS += ~p~n", [App]) || App <- ExtraApps0], -ProjectCompilers = proplists:get_value(compilers, Project, []), -"https://hexdocs.pm/elixir_make/Mix.Tasks.Compile.ElixirMake.html", -ExtraMakeLines = - case lists:member(elixir_make, ProjectCompilers) of + Write = fun (Text) -> + file:write_file("$(call core_native_path,$(DEPS_DIR)/$1/Makefile)", Text, [append]) + end, + Write([ + "PROJECT = ", atom_to_list(proplists:get_value(app, Project)), "\n" + "PROJECT_DESCRIPTION = ", proplists:get_value(description, Project, ""), "\n" + "PROJECT_VERSION = ", proplists:get_value(version, Project, ""), "\n" + "PROJECT_MOD = ", StartMod, "\n" + "define PROJECT_ENV\n", + io_lib:format("~p", [proplists:get_value(env, Application, [])]), "\n" + "endef\n\n"]), + ExtraApps = lists:usort([eex, elixir, logger, mix] ++ proplists:get_value(extra_applications, Application, [])), + Write(["LOCAL_DEPS += ", lists:join(" ", [atom_to_list(App) || App <- ExtraApps]), "\n\n"]), + Deps = proplists:get_value(deps, Project, []) -- [elixir_make], + IsRequiredProdDep = fun(Opts) -> + (proplists:get_value(optional, Opts) =/= true) + andalso + case proplists:get_value(only, Opts, prod) of + prod -> true; + L when is_list(L) -> lists:member(prod, L); + _ -> false + end + end, + lists:foreach(fun + ({Name, Req}) when is_binary(Req) -> + {ok, Vsn} = HexVersionResolve(Name, Req), + Write(["DEPS += ", atom_to_list(Name), "\n"]), + Write(["dep_", atom_to_list(Name), " = hex ", Vsn, " ", atom_to_list(Name), "\n"]); + ({Name, Opts}) when is_list(Opts) -> + Path = proplists:get_value(path, Opts), + case IsRequiredProdDep(Opts) of + true when Path =/= undefined -> + Write(["DEPS += ", atom_to_list(Name), "\n"]), + Write(["dep_", atom_to_list(Name), " = ln ", Path, "\n"]); + true when Path =:= undefined -> + Write(["DEPS += ", atom_to_list(Name), "\n"]), + io:format(standard_error, "Warning: No version given for ~p.", [Name]); + false -> + ok + end; + ({Name, Req, Opts}) -> + case IsRequiredProdDep(Opts) of + true -> + {ok, Vsn} = HexVersionResolve(Name, Req), + Write(["DEPS += ", atom_to_list(Name), "\n"]), + Write(["dep_", atom_to_list(Name), " = hex ", Vsn, " ", atom_to_list(Name), "\n"]); + false -> + ok + end; + (_) -> + ok + end, Deps), + case lists:member(elixir_make, proplists:get_value(compilers, Project, [])) of false -> - ""; + ok; true -> - Fetch = fun(Key, Proplist, DefaultVal, DefaultReplacement) -> + Write("# https://hexdocs.pm/elixir_make/Mix.Tasks.Compile.ElixirMake.html\n"), + MakeVal = fun(Key, Proplist, DefaultVal, DefaultReplacement) -> case proplists:get_value(Key, Proplist, DefaultVal) of DefaultVal -> DefaultReplacement; Value -> Value end end, - case file:copy("$(DEPS_DIR)/$(1)/" ++ Fetch(make_makefile, Project, default, "Makefile"), "$(DEPS_DIR)/$(1)/elixir_make.mk") of - {ok, _} -> ok; + MakeMakefile = binary_to_list(MakeVal(make_makefile, Project, default, <<"Makefile">>)), + MakeExe = MakeVal(make_executable, Project, default, "$$\(MAKE)"), + MakeCwd = MakeVal(make_cwd, Project, undefined, <<".">>), + MakeTargets = MakeVal(make_targets, Project, [], []), + MakeArgs = MakeVal(make_args, Project, undefined, []), + case file:rename("$(DEPS_DIR)/$1/" ++ MakeMakefile, "$(DEPS_DIR)/$1/elixir_make.mk") of + ok -> ok; Err = {error, _} -> io:format(standard_error, "Failed to copy Makefile with error ~p~n", [Err]), - halt(1) + halt(90) end, - [ - io_lib:format("app::~n\t~s -C \"~s\" -f \"$(DEPS_DIR)/$(1)/elixir_make.mk\" ~s ~s~n", [ - Fetch(make_executable, Project, default, "$(MAKE)"), - Fetch(make_cwd, Project, undefined, <<".">>), - lists:join(" ", Fetch(make_targets, Project, [], [])), - lists:join(" ", Fetch(make_args, Project, undefined, [])) - ]), - "\n", - case Fetch(make_clean, Project, nil, undefined) of - undefined -> - ""; - Clean -> - io_lib:format("clean::~n\t~s~n", [Clean]) - end - ] + Write(["app::\n" + "\t", MakeExe, " -C ", MakeCwd, " -f $(DEPS_DIR)/$1/elixir_make.mk", + lists:join(" ", MakeTargets), + lists:join(" ", MakeArgs), + "\n\n"]), + case MakeVal(make_clean, Project, nil, undefined) of + undefined -> + ok; + Clean -> + Write(["clean::\n\t", Clean, "\n\n"]) + end end, -Deps = - lists:foldl(fun(DepToRemove, Acc) -> - lists:keydelete(DepToRemove, 1, Acc) - end, proplists:get_value(deps, Project, []), [elixir_make]), -Args = [ - proplists:get_value(app, Project), - proplists:get_value(description, Project, ""), - proplists:get_value(version, Project, ""), - StartMod, - proplists:get_value(env, Application, []), - ExtraApps, - $(call elixir_get_deps.erl, Deps), - ExtraMakeLines -], -Str = io_lib:format(Fmt, Args), -case file:write_file("$(DEPS_DIR)/$(1)/Makefile", Str) of - ok -> - halt(0); - {error, Reason} -> - io:format(standard_error, "Failed to create '$(DEPS_DIR)/$(1)/Makefile' with reason ~p~n", [Reason]), - halt(1) -end + Write("ERLC_OPTS = +debug_info\n\n"), + Write("include $$\(if $$\(ERLANG_MK_FILENAME),$$\(ERLANG_MK_FILENAME),erlang.mk)"), + halt() endef define dep_autopatch_mix - sed 's|\(defmodule.*do\)|\1\ntry do\nCode.compiler_options(on_undefined_variable: :warn)\nrescue _ -> :ok\nend|g' -i $(DEPS_DIR)/$(1)/mix.exs; \ + sed 's|\(defmodule.*do\)|\1\n try do\n Code.compiler_options(on_undefined_variable: :warn)\n rescue _ -> :ok\n end\n|g' -i $(DEPS_DIR)/$(1)/mix.exs; \ + $(MAKE) $(DEPS_DIR)/hex_core/ebin/dep_built; \ MIX_ENV="$(if $(MIX_ENV),$(strip $(MIX_ENV)),prod)" \ - $(ERL) \ - -eval "$(subst ",\",$(subst $(newline), ,$(subst $$,\$$,$(call Mix_Makefile.erl,$(1)))))." \ - -eval "halt(0)." + $(call erlang,$(call dep_autopatch_mix.erl,$1)) endef define compile_ex.erl From 12f19876d86240e8e6f2b77ddf42a717f39b238c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Wed, 11 Dec 2024 13:24:27 +0100 Subject: [PATCH 31/38] Don't handle elixir specially if it's the only dep --- core/deps.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/deps.mk b/core/deps.mk index 21eac4fde..1daee3f98 100644 --- a/core/deps.mk +++ b/core/deps.mk @@ -120,7 +120,7 @@ dep_name = $(call query_name,$(1)) LOCAL_DEPS_DIRS = $(foreach a,$(LOCAL_DEPS),$(if $(wildcard $(APPS_DIR)/$a),$(APPS_DIR)/$a)) # Elixir is handled specially as it must be built before all other deps # when Mix autopatching is necessary. -ALL_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(foreach dep,$(filter-out elixir $(IGNORE_DEPS),$(BUILD_DEPS) $(DEPS)),$(call query_name,$(dep)))) +ALL_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(foreach dep,$(filter-out $(if $(filter-out elixir,$(BUILD_DEPS) $(DEPS)),elixir) $(IGNORE_DEPS),$(BUILD_DEPS) $(DEPS)),$(call query_name,$(dep)))) # When we are calling an app directly we don't want to include it here # otherwise it'll be treated both as an apps and a top-level project. From bfa73c43261a3ced5fc26a97fae44988e28728e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Thu, 12 Dec 2024 16:26:16 +0100 Subject: [PATCH 32/38] Resolve ELIXIR_LIBS only once - it's slow --- core/elixir.mk | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/elixir.mk b/core/elixir.mk index ca9b369c6..2145b57e3 100644 --- a/core/elixir.mk +++ b/core/elixir.mk @@ -10,14 +10,15 @@ export ELIXIR ifeq ($(ELIXIR),system) # We expect 'elixir' to be on the path. -ELIXIR_LIBS := $(dir $(shell elixir -e 'IO.puts(:code.lib_dir(:elixir))')) +ELIXIR_LIBS ?= $(dir $(shell elixir -e 'IO.puts(:code.lib_dir(:elixir))')) +ELIXIR_LIBS := $(ELIXIR_LIBS) +export ELIXIR_LIBS ERL_LIBS := $(ERL_LIBS):$(ELIXIR_LIBS) else ERL_LIBS := $(ERL_LIBS):$(DEPS_DIR)/elixir/lib/ endif -elixirc_verbose_0 = @echo " ELIXIRC " $(filter-out $(patsubst %,%.ex,$(ERLC_EXCLUDE)),\ - $(filter %.ex %.core,$(?F))); +elixirc_verbose_0 = @echo " EXC $(words $(EX_FILES)) files" elixirc_verbose_2 = set -x; elixirc_verbose = $(elixirc_verbose_$(V)) From 6be6f4d5c74994b814df7fa9cf34248a4898f2ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Thu, 12 Dec 2024 16:35:40 +0100 Subject: [PATCH 33/38] Don't FULL=1 when building escripts --- plugins/escript.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/escript.mk b/plugins/escript.mk index 1790dcbcf..16ae1a97d 100644 --- a/plugins/escript.mk +++ b/plugins/escript.mk @@ -27,15 +27,15 @@ help:: # Plugin-specific targets. -escript-zip:: FULL=1 escript-zip:: deps app $(verbose) mkdir -p $(dir $(abspath $(ESCRIPT_ZIP_FILE))) $(verbose) rm -f $(abspath $(ESCRIPT_ZIP_FILE)) $(gen_verbose) cd .. && $(ESCRIPT_ZIP) $(abspath $(ESCRIPT_ZIP_FILE)) $(PROJECT)/ebin/* ifneq ($(DEPS),) + $(MAKE) query-deps QUERY=name $(verbose) cd $(DEPS_DIR) && $(ESCRIPT_ZIP) $(abspath $(ESCRIPT_ZIP_FILE)) \ $(subst $(DEPS_DIR)/,,$(addsuffix /*,$(wildcard \ - $(addsuffix /ebin,$(shell cat $(ERLANG_MK_TMP)/deps.log))))) + $(addsuffix /ebin,$(shell cat $(ERLANG_MK_TMP)/query.log))))) endif escript:: escript-zip From cd0efb9fc9ee031ad6ad847f5b9e7c43bf1896ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Thu, 12 Dec 2024 20:27:31 +0100 Subject: [PATCH 34/38] Fix escript again --- core/deps-tools.mk | 3 ++- plugins/escript.mk | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/core/deps-tools.mk b/core/deps-tools.mk index 47f3b8e84..4b649ff48 100644 --- a/core/deps-tools.mk +++ b/core/deps-tools.mk @@ -113,7 +113,7 @@ ifeq ($(IS_APP)$(IS_DEP),) endif $(verbose) $(foreach dep,$3,\ echo $(PROJECT): $(foreach q,$(QUERY),$(call query_$(q),$(dep))) >> $4 ;) - $(if $(filter-out query-deps,$1),,\ + $(if $(filter-out query-deps query-runtime-deps,$1),,\ $(verbose) set -e; for dep in $3 ; do \ if grep -qs ^$$$$dep$$$$ $(ERLANG_MK_TMP)/query.log; then \ :; \ @@ -134,6 +134,7 @@ ifeq ($(IS_DEP),) endif $(eval $(call query_target,query-deps,$(ERLANG_MK_RECURSIVE_DEPS_LIST),$(BUILD_DEPS) $(DEPS),$(ERLANG_MK_QUERY_DEPS_FILE))) +$(eval $(call query_target,query-runtime-deps,$(ERLANG_MK_RECURSIVE_DEPS_LIST),$(DEPS),$(ERLANG_MK_QUERY_DEPS_FILE))) $(eval $(call query_target,query-doc-deps,$(ERLANG_MK_RECURSIVE_DOC_DEPS_LIST),$(DOC_DEPS),$(ERLANG_MK_QUERY_DOC_DEPS_FILE))) $(eval $(call query_target,query-rel-deps,$(ERLANG_MK_RECURSIVE_REL_DEPS_LIST),$(REL_DEPS),$(ERLANG_MK_QUERY_REL_DEPS_FILE))) $(eval $(call query_target,query-test-deps,$(ERLANG_MK_RECURSIVE_TEST_DEPS_LIST),$(TEST_DEPS),$(ERLANG_MK_QUERY_TEST_DEPS_FILE))) diff --git a/plugins/escript.mk b/plugins/escript.mk index 16ae1a97d..2f2d9204d 100644 --- a/plugins/escript.mk +++ b/plugins/escript.mk @@ -27,15 +27,18 @@ help:: # Plugin-specific targets. -escript-zip:: deps app +escript-prepare: deps app + $(MAKE) query-runtime-deps QUERY=name + +escript-zip:: escript-prepare $(verbose) mkdir -p $(dir $(abspath $(ESCRIPT_ZIP_FILE))) $(verbose) rm -f $(abspath $(ESCRIPT_ZIP_FILE)) $(gen_verbose) cd .. && $(ESCRIPT_ZIP) $(abspath $(ESCRIPT_ZIP_FILE)) $(PROJECT)/ebin/* ifneq ($(DEPS),) - $(MAKE) query-deps QUERY=name + $(eval ESCRIPT_DEPS := $(shell cat $(ERLANG_MK_TMP)/query.log)) $(verbose) cd $(DEPS_DIR) && $(ESCRIPT_ZIP) $(abspath $(ESCRIPT_ZIP_FILE)) \ $(subst $(DEPS_DIR)/,,$(addsuffix /*,$(wildcard \ - $(addsuffix /ebin,$(shell cat $(ERLANG_MK_TMP)/query.log))))) + $(addprefix $(DEPS_DIR)/,$(addsuffix /ebin,$(ESCRIPT_DEPS)))))) endif escript:: escript-zip From 8d4112fc37fa11baf89cb6a4f232ed9d5629621c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Fri, 13 Dec 2024 13:14:30 +0100 Subject: [PATCH 35/38] More improvements including better escript building --- core/deps-tools.mk | 7 ++----- core/deps.mk | 10 ++++++---- core/elixir.mk | 1 + plugins/escript.mk | 32 ++++++++++++++++++++++++++++---- 4 files changed, 37 insertions(+), 13 deletions(-) diff --git a/core/deps-tools.mk b/core/deps-tools.mk index 4b649ff48..b29bf2a51 100644 --- a/core/deps-tools.mk +++ b/core/deps-tools.mk @@ -78,9 +78,7 @@ endif ifeq ($(IS_APP)$(IS_DEP),) $(verbose) sort < $(ERLANG_MK_RECURSIVE_TMP_LIST) | \ uniq > $(ERLANG_MK_RECURSIVE_TMP_LIST).sorted - $(verbose) cmp -s $(ERLANG_MK_RECURSIVE_TMP_LIST).sorted $@ \ - || mv $(ERLANG_MK_RECURSIVE_TMP_LIST).sorted $@ - $(verbose) rm -f $(ERLANG_MK_RECURSIVE_TMP_LIST).sorted + $(verbose) mv $(ERLANG_MK_RECURSIVE_TMP_LIST).sorted $@ $(verbose) rm $(ERLANG_MK_RECURSIVE_TMP_LIST) endif endif # ifneq ($(SKIP_DEPS),) @@ -113,7 +111,7 @@ ifeq ($(IS_APP)$(IS_DEP),) endif $(verbose) $(foreach dep,$3,\ echo $(PROJECT): $(foreach q,$(QUERY),$(call query_$(q),$(dep))) >> $4 ;) - $(if $(filter-out query-deps query-runtime-deps,$1),,\ + $(if $(filter-out query-deps,$1),,\ $(verbose) set -e; for dep in $3 ; do \ if grep -qs ^$$$$dep$$$$ $(ERLANG_MK_TMP)/query.log; then \ :; \ @@ -134,7 +132,6 @@ ifeq ($(IS_DEP),) endif $(eval $(call query_target,query-deps,$(ERLANG_MK_RECURSIVE_DEPS_LIST),$(BUILD_DEPS) $(DEPS),$(ERLANG_MK_QUERY_DEPS_FILE))) -$(eval $(call query_target,query-runtime-deps,$(ERLANG_MK_RECURSIVE_DEPS_LIST),$(DEPS),$(ERLANG_MK_QUERY_DEPS_FILE))) $(eval $(call query_target,query-doc-deps,$(ERLANG_MK_RECURSIVE_DOC_DEPS_LIST),$(DOC_DEPS),$(ERLANG_MK_QUERY_DOC_DEPS_FILE))) $(eval $(call query_target,query-rel-deps,$(ERLANG_MK_RECURSIVE_REL_DEPS_LIST),$(REL_DEPS),$(ERLANG_MK_QUERY_REL_DEPS_FILE))) $(eval $(call query_target,query-test-deps,$(ERLANG_MK_RECURSIVE_TEST_DEPS_LIST),$(TEST_DEPS),$(ERLANG_MK_QUERY_TEST_DEPS_FILE))) diff --git a/core/deps.mk b/core/deps.mk index 1daee3f98..b7b183d41 100644 --- a/core/deps.mk +++ b/core/deps.mk @@ -120,7 +120,7 @@ dep_name = $(call query_name,$(1)) LOCAL_DEPS_DIRS = $(foreach a,$(LOCAL_DEPS),$(if $(wildcard $(APPS_DIR)/$a),$(APPS_DIR)/$a)) # Elixir is handled specially as it must be built before all other deps # when Mix autopatching is necessary. -ALL_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(foreach dep,$(filter-out $(if $(filter-out elixir,$(BUILD_DEPS) $(DEPS)),elixir) $(IGNORE_DEPS),$(BUILD_DEPS) $(DEPS)),$(call query_name,$(dep)))) +ALL_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(foreach dep,$(filter-out $(IGNORE_DEPS),$(BUILD_DEPS) $(DEPS)),$(call query_name,$(dep)))) # When we are calling an app directly we don't want to include it here # otherwise it'll be treated both as an apps and a top-level project. @@ -212,9 +212,11 @@ endif ifneq ($(SKIP_DEPS),) deps:: else -deps:: $(ALL_DEPS_DIRS) apps clean-tmp-deps.log | $(ERLANG_MK_TMP) -ifneq ($(ALL_DEPS_DIRS),) - $(verbose) set -e; for dep in $(ALL_DEPS_DIRS); do \ +ALL_DEPS_DIRS_TO_BUILD = $(if $(filter-out $(DEPS_DIR)/elixir,$(ALL_DEPS_DIRS)),$(filter-out $(DEPS_DIR)/elixir,$(ALL_DEPS_DIRS)),$(ALL_DEPS_DIRS)) + +deps:: $(ALL_DEPS_DIRS_TO_BUILD) apps clean-tmp-deps.log | $(ERLANG_MK_TMP) +ifneq ($(ALL_DEPS_DIRS_TO_BUILD),) + $(verbose) set -e; for dep in $(ALL_DEPS_DIRS_TO_BUILD); do \ if grep -qs ^$$dep$$ $(ERLANG_MK_TMP)/deps.log; then \ :; \ else \ diff --git a/core/elixir.mk b/core/elixir.mk index 2145b57e3..672874ae5 100644 --- a/core/elixir.mk +++ b/core/elixir.mk @@ -137,6 +137,7 @@ endef define compile_ex.erl {ok, _} = application:ensure_all_started(elixir), + {ok, _} = application:ensure_all_started(mix), ModCode = list_to_atom("Elixir.Code"), ModCode:put_compiler_option(ignore_module_conflict, true), ModComp = list_to_atom("Elixir.Kernel.ParallelCompiler"), diff --git a/plugins/escript.mk b/plugins/escript.mk index 2f2d9204d..f810b637d 100644 --- a/plugins/escript.mk +++ b/plugins/escript.mk @@ -27,18 +27,42 @@ help:: # Plugin-specific targets. +ALL_ESCRIPT_DEPS_DIRS = $(LOCAL_DEPS_DIRS) $(addprefix $(DEPS_DIR)/,$(foreach dep,$(filter-out $(IGNORE_DEPS),$(DEPS)),$(call query_name,$(dep)))) + +ESCRIPT_RUNTIME_DEPS_FILE ?= $(ERLANG_MK_TMP)/escript-deps.log + +escript-list-runtime-deps: +ifeq ($(IS_DEP),) + $(verbose) rm -f $(ESCRIPT_RUNTIME_DEPS_FILE) +endif + $(verbose) touch $(ESCRIPT_RUNTIME_DEPS_FILE) + $(verbose) set -e; for dep in $(ALL_ESCRIPT_DEPS_DIRS) ; do \ + if ! grep -qs ^$$dep$$ $(ESCRIPT_RUNTIME_DEPS_FILE); then \ + echo $$dep >> $(ESCRIPT_RUNTIME_DEPS_FILE); \ + if grep -qs -E "^[[:blank:]]*include[[:blank:]]+(erlang\.mk|.*/erlang\.mk|.*ERLANG_MK_FILENAME.*)$$" \ + $$dep/GNUmakefile $$dep/makefile $$dep/Makefile; then \ + $(MAKE) -C $$dep escript-list-runtime-deps \ + IS_DEP=1 \ + ESCRIPT_RUNTIME_DEPS_FILE=$(ESCRIPT_RUNTIME_DEPS_FILE); \ + fi \ + fi \ + done +ifeq ($(IS_DEP),) + $(verbose) sort < $(ESCRIPT_RUNTIME_DEPS_FILE) | uniq > $(ESCRIPT_RUNTIME_DEPS_FILE).sorted + $(verbose) mv $(ESCRIPT_RUNTIME_DEPS_FILE).sorted $(ESCRIPT_RUNTIME_DEPS_FILE) +endif + escript-prepare: deps app - $(MAKE) query-runtime-deps QUERY=name + $(MAKE) escript-list-runtime-deps escript-zip:: escript-prepare $(verbose) mkdir -p $(dir $(abspath $(ESCRIPT_ZIP_FILE))) $(verbose) rm -f $(abspath $(ESCRIPT_ZIP_FILE)) - $(gen_verbose) cd .. && $(ESCRIPT_ZIP) $(abspath $(ESCRIPT_ZIP_FILE)) $(PROJECT)/ebin/* + $(gen_verbose) cd .. && $(ESCRIPT_ZIP) $(abspath $(ESCRIPT_ZIP_FILE)) $(notdir $(CURDIR))/ebin/* ifneq ($(DEPS),) - $(eval ESCRIPT_DEPS := $(shell cat $(ERLANG_MK_TMP)/query.log)) $(verbose) cd $(DEPS_DIR) && $(ESCRIPT_ZIP) $(abspath $(ESCRIPT_ZIP_FILE)) \ $(subst $(DEPS_DIR)/,,$(addsuffix /*,$(wildcard \ - $(addprefix $(DEPS_DIR)/,$(addsuffix /ebin,$(ESCRIPT_DEPS)))))) + $(addsuffix /ebin,$(shell cat $(ESCRIPT_RUNTIME_DEPS_FILE)))))) endif escript:: escript-zip From 91e552d9bfc11af164dcbe721e58d3f4ff8e0226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Fri, 13 Dec 2024 14:47:08 +0100 Subject: [PATCH 36/38] Probably good changes --- core/elixir.mk | 4 +--- core/erlc.mk | 12 ++++++++---- plugins/escript.mk | 1 + 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/core/elixir.mk b/core/elixir.mk index 672874ae5..5b71430d6 100644 --- a/core/elixir.mk +++ b/core/elixir.mk @@ -2,14 +2,12 @@ # Copyright (c) 2024, Loïc Hoguin # This file is part of erlang.mk and subject to the terms of the ISC License. -ALL_LIB_FILES := $(sort $(call core_find,lib/,*)) -EX_FILES := $(filter-out lib/mix/%,$(filter %.ex,$(ALL_SRC_FILES) $(ALL_LIB_FILES))) - ELIXIR ?= $(if $(filter elixir,$(BUILD_DEPS) $(DEPS)),dep,system) export ELIXIR ifeq ($(ELIXIR),system) # We expect 'elixir' to be on the path. +# @todo Only if there are EX_FILES ELIXIR_LIBS ?= $(dir $(shell elixir -e 'IO.puts(:code.lib_dir(:elixir))')) ELIXIR_LIBS := $(ELIXIR_LIBS) export ELIXIR_LIBS diff --git a/core/erlc.mk b/core/erlc.mk index cb4c8c93d..4a87092e3 100644 --- a/core/erlc.mk +++ b/core/erlc.mk @@ -83,6 +83,10 @@ ALL_SRC_FILES := $(sort $(call core_find,src/,*)) ERL_FILES := $(filter %.erl,$(ALL_SRC_FILES)) CORE_FILES := $(filter %.core,$(ALL_SRC_FILES)) +# @todo Must be defined first. +ALL_LIB_FILES := $(sort $(call core_find,lib/,*)) +EX_FILES := $(filter-out lib/mix/%,$(filter %.ex,$(ALL_SRC_FILES) $(ALL_LIB_FILES))) + # ASN.1 files. ifneq ($(wildcard asn1/),) @@ -263,11 +267,11 @@ $(PROJECT).d:: $(ERL_FILES) $(EX_FILES) $(call core_find,include/,*.hrl) $(MAKEF endif ifeq ($(IS_APP)$(IS_DEP),) -ifneq ($(words $(ERL_FILES) $(EX_FILES) $(CORE_FILES) $(ASN1_FILES) $(MIB_FILES) $(XRL_FILES) $(YRL_FILES)),0) +ifneq ($(words $(ERL_FILES) $(EX_FILES) $(CORE_FILES) $(ASN1_FILES) $(MIB_FILES) $(XRL_FILES) $(YRL_FILES) $(EX_FILES)),0) # Rebuild everything when the Makefile changes. $(ERLANG_MK_TMP)/last-makefile-change: $(MAKEFILE_LIST) | $(ERLANG_MK_TMP) $(verbose) if test -f $@; then \ - touch $(ERL_FILES) $(EX_FILES) $(CORE_FILES) $(ASN1_FILES) $(MIB_FILES) $(XRL_FILES) $(YRL_FILES); \ + touch $(ERL_FILES) $(EX_FILES) $(CORE_FILES) $(ASN1_FILES) $(MIB_FILES) $(XRL_FILES) $(YRL_FILES) $(EX_FILES); \ touch -c $(PROJECT).d; \ fi $(verbose) touch $@ @@ -300,9 +304,9 @@ define validate_app_file endef ebin/$(PROJECT).app:: $(ERL_FILES) $(CORE_FILES) $(wildcard src/$(PROJECT).app.src) $(EX_FILES) - $(eval FILES_TO_COMPILE := $(filter-out src/$(PROJECT).app.src,$?)) + $(eval FILES_TO_COMPILE := $(filter-out $(EX_FILES) src/$(PROJECT).app.src,$?)) $(if $(strip $(FILES_TO_COMPILE)),$(call compile_erl,$(FILES_TO_COMPILE))) - $(if $(strip $(EX_FILES)),$(elixirc_verbose) $(eval MODULES := $(shell $(call erlang,$(call compile_ex.erl,$(EX_FILES)))))) + $(if $(filter $?,$(EX_FILES)),$(elixirc_verbose) $(eval MODULES := $(shell $(call erlang,$(call compile_ex.erl,$(EX_FILES)))))) # Older git versions do not have the --first-parent flag. Do without in that case. $(eval GITDESCRIBE := $(shell git describe --dirty --abbrev=7 --tags --always --first-parent 2>/dev/null \ || git describe --dirty --abbrev=7 --tags --always 2>/dev/null || true)) diff --git a/plugins/escript.mk b/plugins/escript.mk index f810b637d..4eac352fa 100644 --- a/plugins/escript.mk +++ b/plugins/escript.mk @@ -65,6 +65,7 @@ ifneq ($(DEPS),) $(addsuffix /ebin,$(shell cat $(ESCRIPT_RUNTIME_DEPS_FILE)))))) endif +# @todo Only generate the zip file if there were changes. escript:: escript-zip $(gen_verbose) printf "%s\n" \ "#!$(ESCRIPT_SHEBANG)" \ From ef047978ff76db1290bf59ec498736f787913e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 16 Dec 2024 15:31:25 +0100 Subject: [PATCH 37/38] More improvements and fixes --- core/deps.mk | 11 ++++++----- core/elixir.mk | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/deps.mk b/core/deps.mk index b7b183d41..6e4b7cc1b 100644 --- a/core/deps.mk +++ b/core/deps.mk @@ -240,6 +240,10 @@ endif # Deps related targets. +autopatch_verbose_0 = @echo " PATCH " $(subst autopatch-,,$@) "(method: $(AUTOPATCH_METHOD))"; +autopatch_verbose_2 = set -x; +autopatch_verbose = $(autopatch_verbose_$(V)) + define dep_autopatch_detect if [ -f $(DEPS_DIR)/$1/erlang.mk ]; then \ echo erlang.mk; \ @@ -412,7 +416,6 @@ define dep_autopatch_rebar.erl GetHexVsn2 = fun(N, NP) -> case file:consult("$(call core_native_path,$(DEPS_DIR)/$1/rebar.lock)") of {ok, Lock} -> - io:format("~p~n", [Lock]), LockPkgs = case lists:keyfind("1.2.0", 1, Lock) of {_, LP} -> LP; @@ -426,10 +429,8 @@ define dep_autopatch_rebar.erl end, if is_list(LockPkgs) -> - io:format("~p~n", [LockPkgs]), case lists:keyfind(atom_to_binary(N, latin1), 1, LockPkgs) of {_, {pkg, _, Vsn}, _} -> - io:format("~p~n", [Vsn]), {N, {hex, NP, binary_to_list(Vsn)}}; _ -> false @@ -881,7 +882,7 @@ define dep_fetch_fail endef define dep_target -$(DEPS_DIR)/$(call query_name,$1): $(if $(filter elixir,$(BUILD_DEPS) $(DEPS)),$(if $(filter-out elixir,$1),$(DEPS_DIR)/elixir/ebin/dep_built)) $(if $(filter hex,$(call query_fetch_method,$1)),$(DEPS_DIR)/hex_core/ebin/dep_built) | $(ERLANG_MK_TMP) +$(DEPS_DIR)/$(call query_name,$1): $(if $(filter elixir,$(BUILD_DEPS) $(DEPS)),$(if $(filter-out elixir,$1),$(DEPS_DIR)/elixir/ebin/dep_built)) $(if $(filter hex,$(call query_fetch_method,$1)),$(if $(wildcard $(DEPS_DIR)/$(call query_name,$1)),,$(DEPS_DIR)/hex_core/ebin/dep_built)) | $(ERLANG_MK_TMP) $(eval DEP_NAME := $(call query_name,$1)) $(eval DEP_STR := $(if $(filter $1,$(DEP_NAME)),$1,"$1 ($(DEP_NAME))")) $(verbose) if test -d $(APPS_DIR)/$(DEP_NAME); then \ @@ -910,7 +911,7 @@ autopatch-elixir:: ln -s lib/elixir/ebin $(DEPS_DIR)/elixir/ else autopatch-$(call query_name,$1):: - $$(call dep_autopatch_for_$(AUTOPATCH_METHOD),$(call query_name,$1)) + $$(autopatch_verbose) $$(call dep_autopatch_for_$(AUTOPATCH_METHOD),$(call query_name,$1)) endif endef diff --git a/core/elixir.mk b/core/elixir.mk index 5b71430d6..fb6b5a147 100644 --- a/core/elixir.mk +++ b/core/elixir.mk @@ -8,7 +8,7 @@ export ELIXIR ifeq ($(ELIXIR),system) # We expect 'elixir' to be on the path. # @todo Only if there are EX_FILES -ELIXIR_LIBS ?= $(dir $(shell elixir -e 'IO.puts(:code.lib_dir(:elixir))')) +ELIXIR_LIBS ?= $(dir $(shell readlink -f `which elixir`))/../lib ELIXIR_LIBS := $(ELIXIR_LIBS) export ELIXIR_LIBS ERL_LIBS := $(ERL_LIBS):$(ELIXIR_LIBS) From fa90971c2cbcc0411db92c61846296dde31d1f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Hoguin?= Date: Mon, 16 Dec 2024 16:32:56 +0100 Subject: [PATCH 38/38] Fix default PROJECT_MOD --- core/erlc.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/erlc.mk b/core/erlc.mk index 4a87092e3..b42334125 100644 --- a/core/erlc.mk +++ b/core/erlc.mk @@ -57,7 +57,7 @@ app:: $(if $(wildcard ebin/test),beam-cache-restore-app) deps $(verbose) $(MAKE) --no-print-directory $(PROJECT).d $(verbose) $(MAKE) --no-print-directory app-build -PROJECT_MOD := $(if $(PROJECT_MOD),$(PROJECT_MOD),$(if $(wildcard src/$(PROJECT_MOD)_app.erl),$(PROJECT_MOD)_app)) +PROJECT_MOD := $(if $(PROJECT_MOD),$(PROJECT_MOD),$(if $(wildcard src/$(PROJECT)_app.erl),$(PROJECT)_app)) define app_file {application, '$(PROJECT)', [