BaseTools: ninja build system for faster builds

The current makefiles are executed sequentially. The VfrCompile build also includes an antlr build that generates code over a long period of time. Ninja allows to parallelise builds of different utilities and build them 2.5 times faster. Signed-off-by: Alexander Gryanko <[email protected]>
tianocore · Jan 9, 2025 · 5f54ef5 · 5f54ef5
1 parent 25ce25e
commit 5f54ef5
Show file tree

Hide file tree

Showing 21 changed files with 251 additions and 10 deletions.
diff --git a/BaseTools/GNUmakefile b/BaseTools/GNUmakefile
@@ -13,13 +13,16 @@ SOURCE_SUBDIRS := $(patsubst %,Source/%,$(sort $(LANGUAGES)))
 SUBDIRS := $(SOURCE_SUBDIRS) Tests
 CLEAN_SUBDIRS := $(patsubst %,%-clean,$(sort $(SUBDIRS)))
 
-.PHONY: subdirs $(SUBDIRS)
+.PHONY: subdirs $(SUBDIRS) ninja
 subdirs: $(SUBDIRS)
 $(SUBDIRS):
 	$(MAKE) -C $@
 
 Tests: $(SOURCE_SUBDIRS)
 
+ninja:
+	for dir in $(SUBDIRS); do $(MAKE) -C $$dir ninja || exit 1; done
+
 .PHONY: $(CLEAN_SUBDIRS)
 $(CLEAN_SUBDIRS):
 	-$(MAKE) -C $(@:-clean=) clean

diff --git a/BaseTools/Source/C/.gitignore b/BaseTools/Source/C/.gitignore
@@ -0,0 +1,3 @@
+**/.ninja_deps
+**/.ninja_log
+**/build.ninja
diff --git a/BaseTools/Source/C/BrotliCompress/GNUmakefile b/BaseTools/Source/C/BrotliCompress/GNUmakefile
@@ -42,7 +42,43 @@ OBJECTS = \
   brotli/c/enc/static_dict.o \
   brotli/c/enc/utf8_util.o
 
+override NINJA_CSRCS := \
+  BrotliCompress.c \
+  brotli/c/common/platform.c \
+  brotli/c/common/shared_dictionary.c \
+  brotli/c/common/constants.c \
+  brotli/c/common/context.c \
+  brotli/c/enc/command.c \
+  brotli/c/enc/compound_dictionary.c \
+  brotli/c/enc/fast_log.c \
+  brotli/c/common/dictionary.c \
+  brotli/c/common/transform.c \
+  brotli/c/dec/bit_reader.c \
+  brotli/c/dec/decode.c \
+  brotli/c/dec/huffman.c \
+  brotli/c/dec/state.c \
+  brotli/c/enc/backward_references.c \
+  brotli/c/enc/backward_references_hq.c \
+  brotli/c/enc/bit_cost.c \
+  brotli/c/enc/block_splitter.c \
+  brotli/c/enc/brotli_bit_stream.c \
+  brotli/c/enc/cluster.c \
+  brotli/c/enc/compress_fragment.c \
+  brotli/c/enc/compress_fragment_two_pass.c \
+  brotli/c/enc/dictionary_hash.c \
+  brotli/c/enc/encode.c \
+  brotli/c/enc/encoder_dict.c \
+  brotli/c/enc/entropy_encode.c \
+  brotli/c/enc/histogram.c \
+  brotli/c/enc/literal_cost.c \
+  brotli/c/enc/memory.c \
+  brotli/c/enc/metablock.c \
+  brotli/c/enc/static_dict.c \
+  brotli/c/enc/utf8_util.c
+
+override NINJA_CXXSRCS :=
+
 include $(MAKEROOT)/Makefiles/app.makefile
 
-TOOL_INCLUDE = -I ./brotli/c/include
+TOOL_INCLUDE = -I $(MAKEROOT)/$(APPNAME)/brotli/c/include
 LIBS += -lm
diff --git a/BaseTools/Source/C/DevicePath/GNUmakefile b/BaseTools/Source/C/DevicePath/GNUmakefile
@@ -11,6 +11,8 @@ APPNAME = DevicePath
 
 OBJECTS = DevicePath.o UefiDevicePathLib.o DevicePathFromText.o  DevicePathUtilities.o
 
+NINJA_LIBS_DEPS = Common
+
 include $(MAKEROOT)/Makefiles/app.makefile
 
 GCCVERSION = $(shell $(CC) -dumpversion | awk -F'.' '{print $$1}')

diff --git a/BaseTools/Source/C/EfiRom/GNUmakefile b/BaseTools/Source/C/EfiRom/GNUmakefile
@@ -12,4 +12,6 @@ LIBS = -lCommon
 
 OBJECTS = EfiRom.o
 
+NINJA_LIBS_DEPS = Common
+
 include $(MAKEROOT)/Makefiles/app.makefile
diff --git a/BaseTools/Source/C/GNUmakefile b/BaseTools/Source/C/GNUmakefile
@@ -41,7 +41,15 @@ endif
 
 export HOST_ARCH
 
-MAKEROOT = .
+MAKEROOT ?= $(CURDIR)
+export MAKEROOT
+
+NINJA := $(shell which ninja-build 2>/dev/null || which ninja 2>/dev/null)
+NINJAFLAGS =
+ifneq (,$(filter -j%,$(MAKEFLAGS)))
+  NUMJOBS = $(patsubst -j%,%,$(filter -j%,$(MAKEFLAGS)))
+  NINJAFLAGS += -j$(NUMJOBS)
+endif
 
 include Makefiles/header.makefile
 
@@ -85,6 +93,17 @@ $(patsubst %,%-clean,$(sort $(SUBDIRS))):
 $(VFRAUTOGEN): VfrCompile/VfrSyntax.g
 	$(MAKE) -C VfrCompile VfrLexer.h
 
+.PHONY: ninja
+ninja:
+	for dir in $(LIBRARIES); do $(MAKE) -C $$dir ninja_lib; done
+	for dir in $(APPLICATIONS); do $(MAKE) -C $$dir ninja_app; done
+	@echo "pool link_pool" > build.ninja
+	@echo "  depth = 10" >> build.ninja
+	@echo "" >> build.ninja
+	for dir in $(LIBRARIES); do echo "subninja $${dir}/build.ninja" >> build.ninja; done
+	for dir in $(APPLICATIONS); do echo "subninja $${dir}/build.ninja" >> build.ninja; done
+	$(NINJA) $(NINJAFLAGS)
+
 clean:  $(patsubst %,%-clean,$(sort $(SUBDIRS)))
 
 clean: localClean

diff --git a/BaseTools/Source/C/GenCrc32/GNUmakefile b/BaseTools/Source/C/GenCrc32/GNUmakefile
@@ -12,4 +12,6 @@ LIBS = -lCommon
 
 OBJECTS = GenCrc32.o
 
+NINJA_LIBS_DEPS = Common
+
 include $(MAKEROOT)/Makefiles/app.makefile
diff --git a/BaseTools/Source/C/GenFfs/GNUmakefile b/BaseTools/Source/C/GenFfs/GNUmakefile
@@ -10,7 +10,8 @@ APPNAME = GenFfs
 
 OBJECTS = GenFfs.o
 
+NINJA_LIBS_DEPS = Common
+
 include $(MAKEROOT)/Makefiles/app.makefile
 
 LIBS = -lCommon
-
diff --git a/BaseTools/Source/C/GenFv/GNUmakefile b/BaseTools/Source/C/GenFv/GNUmakefile
@@ -10,6 +10,8 @@ APPNAME = GenFv
 
 OBJECTS = GenFv.o GenFvInternalLib.o
 
+NINJA_LIBS_DEPS = Common
+
 include $(MAKEROOT)/Makefiles/app.makefile
 
 LIBS = -lCommon

diff --git a/BaseTools/Source/C/GenFw/GNUmakefile b/BaseTools/Source/C/GenFw/GNUmakefile
@@ -10,6 +10,8 @@ APPNAME = GenFw
 
 OBJECTS = GenFw.o ElfConvert.o Elf32Convert.o Elf64Convert.o
 
+NINJA_LIBS_DEPS = Common
+
 include $(MAKEROOT)/Makefiles/app.makefile
 
 LIBS = -lCommon

diff --git a/BaseTools/Source/C/GenSec/GNUmakefile b/BaseTools/Source/C/GenSec/GNUmakefile
@@ -10,6 +10,8 @@ APPNAME = GenSec
 
 OBJECTS = GenSec.o
 
+NINJA_LIBS_DEPS = Common
+
 include $(MAKEROOT)/Makefiles/app.makefile
 
 LIBS = -lCommon

diff --git a/BaseTools/Source/C/LzmaCompress/GNUmakefile b/BaseTools/Source/C/LzmaCompress/GNUmakefile
@@ -22,6 +22,20 @@ OBJECTS = \
   $(SDK_C)/7zStream.o \
   $(SDK_C)/Bra86.o
 
+override NINJA_CSRCS := \
+  LzmaCompress.c \
+  $(SDK_C)/Alloc.c \
+  $(SDK_C)/LzFind.c \
+  $(SDK_C)/LzmaDec.c \
+  $(SDK_C)/LzmaEnc.c \
+  $(SDK_C)/7zFile.c \
+  $(SDK_C)/7zStream.c \
+  $(SDK_C)/Bra86.c
+
+override NINJA_CXXSRCS :=
+
+NINJA_LIBS_DEPS = Common
+
 include $(MAKEROOT)/Makefiles/app.makefile
 
 CFLAGS += -D_7ZIP_ST
diff --git a/BaseTools/Source/C/Makefiles/app.makefile b/BaseTools/Source/C/Makefiles/app.makefile
@@ -11,12 +11,25 @@ include $(MAKEROOT)/Makefiles/header.makefile
 
 APPLICATION = $(MAKEROOT)/bin/$(APPNAME)
 
-.PHONY:all
+NINJA_BUILD_DEPS =
+ifneq ($(NINJA_LIBS_DEPS),)
+  NINJA_BUILD_DEPS +=  || $(foreach lib,$(NINJA_LIBS_DEPS),"$(MAKEROOT)/libs/lib$(subst -l,,$(lib)).a")
+endif
+
+.PHONY:all ninja
 all: $(MAKEROOT)/bin $(APPLICATION)
 
 $(APPLICATION): $(OBJECTS)
 	$(LINKER) -o $(APPLICATION) $(LDFLAGS) $(OBJECTS) -L$(MAKEROOT)/libs $(LIBS)
 
 $(OBJECTS): $(MAKEROOT)/Include/Common/BuildVersion.h
 
+ninja_app: ninja_build
+	@echo "" >> build.ninja
+	@echo "rule link_$(NINJA_ID)" >> build.ninja
+	@echo "  pool = link_pool" >> build.ninja
+	@echo "  command = $(LINKER) -o \044out $(LDFLAGS) \044in -L$(MAKEROOT)/libs $(LIBS)" >> build.ninja
+	@echo "" >> build.ninja
+	@echo "build $(APPLICATION): link_$(NINJA_ID) $(NINJA_OBJECTS)$(NINJA_BUILD_DEPS)" >> build.ninja
+
 include $(MAKEROOT)/Makefiles/footer.makefile
diff --git a/BaseTools/Source/C/Makefiles/footer.makefile b/BaseTools/Source/C/Makefiles/footer.makefile
@@ -10,7 +10,12 @@ DEPFILES = $(OBJECTS:%.o=%.d)
 $(MAKEROOT)/libs-$(HOST_ARCH):
 	mkdir -p $(MAKEROOT)/libs-$(HOST_ARCH)
 
-.PHONY: install
+NINJA_CSRCS = $(wildcard *.c)
+NINJA_CXXSRCS = $(wildcard *.cpp)
+NINJA_COBJECTS := $(foreach file,$(NINJA_CSRCS),"build $(NINJA_DIR)/$(subst .c,,$(file)).o: cc_$(NINJA_ID) $(NINJA_DIR)/$(file)")
+NINJA_CXXOBJECTS := $(foreach file,$(NINJA_CXXSRCS),"build $(NINJA_DIR)/$(subst .c,,$(file)).o: cxx_$(NINJA_ID) $(NINJA_DIR)/$(file)")
+
+.PHONY: install ninja
 install: $(MAKEROOT)/libs-$(HOST_ARCH) $(LIBRARY)
 	cp $(LIBRARY) $(MAKEROOT)/libs-$(HOST_ARCH)
 
@@ -23,8 +28,25 @@ $(LIBRARY): $(OBJECTS)
 %.o : %.cpp
 	$(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $< -o $@
 
+ninja_build:
+	@echo "" > build.ninja
+	@echo "rule cc_$(NINJA_ID)" >> build.ninja
+	@echo "  deps = $(CC)" >> build.ninja
+	@echo "  depfile = \044out.d" >> build.ninja
+	@echo "  command = $(CC) -c $(CPPFLAGS) $(CFLAGS) \044in -o \044out" >> build.ninja
+	@echo "" >> build.ninja
+	@echo "rule cxx_$(NINJA_ID)" >> build.ninja
+	@echo "  deps = $(CC)" >> build.ninja
+	@echo "  depfile = \044out.d" >> build.ninja
+	@echo "  command = $(CC) -c $(CPPFLAGS) $(CXXFLAGS) \044in -o \044out" >> build.ninja
+	@echo "" >> build.ninja
+	@printf "%s\n" $(NINJA_COBJECTS) >> build.ninja
+	@printf "%s\n" $(NINJA_CXXOBJECTS) >> build.ninja
+	@echo "" >> build.ninja
+
 .PHONY: clean
 clean:
 	@rm -f $(OBJECTS) $(LIBRARY) $(DEPFILES)
+	@rm -f build.ninja
 
 -include $(DEPFILES)
diff --git a/BaseTools/Source/C/Makefiles/header.makefile b/BaseTools/Source/C/Makefiles/header.makefile
@@ -134,6 +134,19 @@ CXXFLAGS += $(BUILD_OPTFLAGS)
 # keep EXTRA_LDFLAGS last
 LDFLAGS += $(EXTRA_LDFLAGS)
 
+NINJA_DIR =
+ifneq ($(APPNAME),)
+  NINJA_DIR = $(APPNAME)
+else ifneq ($(LIBNAME),)
+  NINJA_DIR = $(LIBNAME)
+else ifneq ($(APPLICATIONS),)
+# The root makefile cannot define APPNAME/LIBNAME. APPLICATIONS is a unique
+#   variable for the root makefile that is unlikely to be used in subfolders.
+  $(error "Neither APPNAME nor LIBNAME is defined. At least one of them must be defined")
+endif
+NINJA_ID := $(shell tr -dc A-Za-z0-9 </dev/random | head -c 5)
+NINJA_OBJECTS := $(foreach obj,$(OBJECTS),"$(NINJA_DIR)/$(obj)")
+
 .PHONY: all
 .PHONY: install
 .PHONY: clean

diff --git a/BaseTools/Source/C/Makefiles/lib.makefile b/BaseTools/Source/C/Makefiles/lib.makefile
@@ -11,4 +11,12 @@ LIBRARY = $(MAKEROOT)/libs/lib$(LIBNAME).a
 
 all: $(MAKEROOT)/libs $(LIBRARY)
 
+ninja_lib: ninja_build
+	@echo "" >> build.ninja
+	@echo "rule link_$(NINJA_ID)" >> build.ninja
+	@echo "  pool = link_pool" >> build.ninja
+	@echo "  command = $(AR) crs \044out \044in" >> build.ninja
+	@echo "" >> build.ninja
+	@echo "build $(LIBRARY): link_$(NINJA_ID) $(NINJA_OBJECTS)" >> build.ninja
+
 include $(MAKEROOT)/Makefiles/footer.makefile
diff --git a/BaseTools/Source/C/TianoCompress/GNUmakefile b/BaseTools/Source/C/TianoCompress/GNUmakefile
@@ -12,4 +12,6 @@ LIBS = -lCommon
 
 OBJECTS = TianoCompress.o
 
+NINJA_LIBS_DEPS = Common
+
 include $(MAKEROOT)/Makefiles/app.makefile
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,4 +12,6 @@ LIBS = -lCommon

		OBJECTS = EfiRom.o

		NINJA_LIBS_DEPS = Common

		include $(MAKEROOT)/Makefiles/app.makefile
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,4 +12,6 @@ LIBS = -lCommon

		OBJECTS = GenCrc32.o

		NINJA_LIBS_DEPS = Common

		include $(MAKEROOT)/Makefiles/app.makefile
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,4 +12,6 @@ LIBS = -lCommon

		OBJECTS = TianoCompress.o

		NINJA_LIBS_DEPS = Common

		include $(MAKEROOT)/Makefiles/app.makefile