diff --git a/Makefile b/Makefile index ee0a4b35..49cc92fc 100644 --- a/Makefile +++ b/Makefile @@ -3,206 +3,81 @@ # This program is free software: you can redistribute/modify under the terms of the GPL-v3 (https://www.gnu.org/licenses/gpl-3.0.html). # This program is distributed WITHOUT ANY WARRANTY, without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -ifeq ($(origin CC),default) -undefine CC -endif - # -# build configuration -#vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - +# user configurable MAKEFLAGS := -j$(shell nproc||echo 1) -k - -BASH ?= bash -CC ?= gcc -GIT ?= git -PYTHON2 ?= python2 -PYTHON ?= python3 - -#^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -# - -# ensure we are insitive to path from now on /!\ putting comment on var def lines put spaces in var -override BASH := $(shell $(BASH) -c 'type -p $(BASH) ') -override CC := $(shell $(BASH) -c 'type -p $(CC) ') -override GIT := $(shell $(BASH) -c 'type -p $(GIT) ') -override PYTHON2 := $(shell $(BASH) -c 'type -p $(PYTHON2)') -override PYTHON := $(shell $(BASH) -c 'type -p $(PYTHON) ') - -STD_PATH := $(shell env -i $(BASH) -c 'echo $$PATH') - +# mandatory MAKEFLAGS += -r -R -# syntax for LMAKE_OPT : [0123][gG][aA][tT] -# - [0123] : compiler optimization level -# - [gG] : -g flag if G -# - [dD] : -DNDEBUG if d -# - [tT] : -DNO_TRACE if t -ifeq ($(LMAKE_OPT),) -LMAKE_OPT := 3GDT -endif -ifneq ($(findstring 0,$(LMAKE_OPT)),) -OPT_FLAGS := -O0 -else ifneq ($(findstring 1,$(LMAKE_OPT)),) -OPT_FLAGS := -O1 -else ifneq ($(findstring 2,$(LMAKE_OPT)),) -OPT_FLAGS := -O2 -else ifneq ($(findstring 3,$(LMAKE_OPT)),) -OPT_FLAGS := -O3 -else -$(error $$LMAKE_OPT must be [0123][gG][dD][tT]) -endif -ifneq ($(findstring G,$(LMAKE_OPT)),) -OPT_FLAGS += -g -else ifneq ($(findstring g,$(LMAKE_OPT)),) -else -$(error $$LMAKE_OPT must be [0123][gG][dD][tT]) -endif -ifneq ($(findstring D,$(LMAKE_OPT)),) -else ifneq ($(findstring d,$(LMAKE_OPT)),) -OPT_FLAGS += -DNDEBUG -else -$(error $$LMAKE_OPT must be [0123][gG][dD][tT]) -endif -ifneq ($(findstring T,$(LMAKE_OPT)),) -else ifneq ($(findstring t,$(LMAKE_OPT)),) -OPT_FLAGS += -DNO_TRACE -else -$(error $$LMAKE_OPT must be [0123][gG][dD][tT]) -endif - -COVERAGE := -ifneq ($(LMAKE_COVERAGE),) -COVERAGE := --coverage # XXX : not operational yet -endif - -WARNING_FLAGS := -Wall -Wextra -Wno-cast-function-type -Wno-type-limits - -LANG := c++20 - -include sys_config.h.inc_stamp # sys_config.h is used in this makefile - -# python configuration (python2 is optional) -ifeq ($(PYTHON2),) -else ifeq ($(shell $(PYTHON2) -c 'import sys ; print(sys.version_info.major==2 and sys.version_info.minor>=7)'),False) -override PYTHON2 := -endif -ifeq ($(PYTHON),) -$(error cannot find python3) -else ifeq ($(shell $(PYTHON) -c 'import sys ; print(sys.version_info.major==3 and sys.version_info.minor>=6)'),False) -$(error python3 version must be at least 3.6) -endif - -# CC configuration -USE_GCC := -USE_CLANG := -ifeq ($(CC),) -$(error cannot find c compiler) -else ifneq ($(findstring gcc,$(CC)),) -ifeq ($(intcmp $(shell $(CC) -dumpversion),11,lt,eq,gt),lt) -$(error gcc version must be at least 11) -endif -USE_GCC := 1 -else ifneq ($(findstring clang,$(CC)),) -ifeq ($(intcmp $(shell $(CC) -dumpversion),15,lt,eq,gt),lt) -$(error clang version must be at least 15) -endif -USE_CLANG := 1 -else -$(error cannot recognize c compiler $(CC)) -endif +.DEFAULT_GOAL := DFLT -ifneq ($(USE_GCC),) # for an unknown reason, clang is incompatible with -fsanitize -# -fsanitize=address and -fsanitize=thread are exclusive of one another -ifeq ($(LMAKE_SAN),A) -SAN_FLAGS := -fsanitize=address -fsanitize=undefined -endif -ifeq ($(LMAKE_SAN),T) -SAN_FLAGS := -fsanitize=thread -endif -endif +sys_config.log : _bin/sys_config + ./$< $(@:%.log=%.mk) $(@:%.log=%.h) 2>$@ || cat $@ +sys_config.mk : sys_config.log ;+@[ -f $@ ] || { echo "cannot find $@" ; exit 1 ; } +sys_config.h : sys_config.log ;+@[ -f $@ ] || { echo "cannot find $@" ; exit 1 ; } -ifneq ($(USE_CLANG),) -WARNING_FLAGS += -Wno-misleading-indentation -Wno-unknown-warning-option -Wno-c2x-extensions -Wno-c++2b-extensions -endif +# defines HAS_SECCOMP & HAS_SLURM +include sys_config.mk # this is the recommanded way to insert a , when calling functions # /!\ cannot put a comment on the following line or a lot of spaces will be inserted in the variable definition COMMA := , -.DEFAULT_GOAL := DFLT - -SAN := $(if $(SAN_FLAGS),.san,) -LINK_LIB_PATH := $(shell $(CC) -v -E /dev/null 2>&1 | grep LIBRARY_PATH=) # e.g. : LIBARY_PATH=/usr/lib/x:/a/b:/c:/a/b/c/.. -LINK_LIB_PATH := $(subst LIBRARY_PATH=,,$(LINK_LIB_PATH)) # e.g. : /usr/lib/x:/a/b:/c:/a/b/c/.. -LINK_LIB_PATH := $(subst :, ,$(LINK_LIB_PATH)) # e.g. : /usr/lib/x /a/b /c /a/b/c/.. -LINK_LIB_PATH := $(realpath $(LINK_LIB_PATH)) # e.g. : /usr/lib/x /a/b /c /a/b -LINK_LIB_PATH := $(sort $(LINK_LIB_PATH)) # e.g. : /a/b /c /usr/lib/x -LINK_LIB_PATH := $(filter-out /usr/lib /usr/lib64 /usr/lib/% /usr/lib64/%,$(LINK_LIB_PATH)) # e.g. : /a/b /c (suppress standard dirs as required in case of installed package) -LINK_OPTS := $(patsubst %,-Wl$(COMMA)-rpath=%,$(LINK_LIB_PATH)) -pthread # e.g. : -Wl,-rpath=/a/b -Wl,-rpath=/c -pthread -LINK_O := $(CC) $(COVERAGE) -r -LINK_SO := $(CC) $(COVERAGE) $(LINK_OPTS) -shared # some usage may have specific libs, avoid dependencies -LINK_BIN := $(CC) $(COVERAGE) $(LINK_OPTS) -LINK_LIB := -ldl -lstdc++ -lm +# syntax for LMAKE_FLAGS : O[0123]G?D?T?S[AT]C? +# - O[0123] : compiler optimization level, defaults to 3 +# - G : -g +# - D : -DNDEBUG +# - T : -DNO_TRACE +# - SA : -fsanitize address +# - ST : -fsanitize threads +# - C : coverage (not operational yet) +OPT_FLAGS := -O3 +OPT_FLAGS := $(if $(findstring O2,$(LMAKE_FLAGS)),-O2,$(OPT_FLAGS)) +OPT_FLAGS := $(if $(findstring O1,$(LMAKE_FLAGS)),-O1,$(OPT_FLAGS)) +OPT_FLAGS := $(if $(findstring O0,$(LMAKE_FLAGS)),-O0,$(OPT_FLAGS)) +EXTRA_FLAGS += $(if $(findstring G, $(LMAKE_FLAGS)),-g) +HIDDEN_FLAGS += $(if $(findstring G, $(LMAKE_FLAGS)),-fno-omit-frame-pointer) +EXTRA_FLAGS += $(if $(findstring d, $(LMAKE_FLAGS)),-DNDEBUG) +EXTRA_FLAGS += $(if $(findstring t, $(LMAKE_FLAGS)),-DNO_TRACE) +SAN_FLAGS += $(if $(findstring SA,$(LMAKE_FLAGS)),-fsanitize=address -fsanitize=undefined) +SAN_FLAGS += $(if $(findstring ST,$(LMAKE_FLAGS)),-fsanitize=thread) +COVERAGE += $(if $(findstring C, $(LMAKE_FLAGS)),--coverage) # -STD_INC_DIRS := $(shell $(CC) -E -v -std=$(LANG) -xc++ /dev/null 2>&1 | sed -e '1,/<.*>.*search starts/d' -e '/End of search/,$$d' ) -# -ifneq ($(PYTHON2),) -PY2_INCLUDEDIR := $(shell $(PYTHON2) -c 'import sysconfig ; print(sysconfig.get_config_var("INCLUDEDIR"))') -PY2_INCLUDEPY := $(shell $(PYTHON2) -c 'import sysconfig ; print(sysconfig.get_config_var("INCLUDEPY" ))') -PY2_INC_DIRS := $(filter-out $(STD_INC_DIRS),$(PY2_INCLUDEDIR) $(PY2_INCLUDEPY)) # for some reasons, compilation does not work if standard inc dirs are given with -isystem -PY2_CC_OPTS := $(patsubst %,-isystem %,$(PY2_INC_DIRS)) -Wno-register -PY2_LIB_BASE := $(shell $(PYTHON2) -c 'import sysconfig ; print(sysconfig.get_config_var("LDLIBRARY" ))') # transform lib.so -> -PY2_LIB_DIR := $(shell $(PYTHON2) -c 'import sysconfig ; print(sysconfig.get_config_var("LIBDIR" ))') -# ensure we can compile and link with Python2 or exclude its support -ifneq ($(shell [ -f $(PY2_INCLUDEPY_DIR)/Python.h ] && file -L $(PY2_LIB_DIR)/$(PY2_LIB_BASE) | grep -q shared && echo 1),1) -override PYTHON2 := -endif -# suppress standard dirs as required in case of installed package /!\ comments on variable definitions insert blanks ! -PY2_LIB_DIR := $(strip $(filter-out /usr/lib /usr/lib64 /usr/lib/% /usr/lib64/%,$(PY2_LIB_DIR))) -PY2_LINK_OPTS := $(patsubst %,-L%,$(PY2_LIB_DIR)) $(patsubst %,-Wl$(COMMA)-rpath=%,$(PY2_LIB_DIR)) -l:$(PY2_LIB_BASE) -endif +WARNING_FLAGS := -Wall -Wextra -Wno-cast-function-type -Wno-type-limits # -PY_INCLUDEDIR := $(shell $(PYTHON) -c 'import sysconfig ; print(sysconfig.get_config_var("INCLUDEDIR"))') -PY_INCLUDEPY := $(shell $(PYTHON) -c 'import sysconfig ; print(sysconfig.get_config_var("INCLUDEPY") )') -PY_INC_DIRS := $(filter-out $(STD_INC_DIRS),$(PY_INCLUDEDIR) $(PY_INCLUDEPY)) # for some reasons, compilation does not work if standard inc dirs are given with -isystem -PY_CC_OPTS := $(patsubst %,-isystem %,$(PY_INC_DIRS)) -Wno-register -PY_LIB_BASE := $(shell $(PYTHON) -c 'import sysconfig ; print(sysconfig.get_config_var("LDLIBRARY") )') # transform lib.so -> -PY_LIB_DIR := $(shell $(PYTHON) -c 'import sysconfig ; print(sysconfig.get_config_var("LIBDIR" ) )') -# suppress standard dirs as required in case of installed package /!\ comments on variable definitions insert blanks ! -PY_LIB_DIR := $(strip $(filter-out /usr/lib /usr/lib64 /usr/lib/% /usr/lib64/%,$(PY_LIB_DIR))) -PY_LINK_OPTS := $(patsubst %,-L%,$(PY_LIB_DIR)) $(patsubst %,-Wl$(COMMA)-rpath=%,$(PY_LIB_DIR)) -l:$(PY_LIB_BASE) +LANG := c++20 # -PY_VERSION := $(shell $(PYTHON) -c 'import sysconfig ; print(sysconfig.get_config_var("VERSION"))') +SAN := $(if $(strip $(SAN_FLAGS)),.san,) +LINK_OPTS := $(patsubst %,-Wl$(COMMA)-rpath=%,$(LINK_LIB_PATH)) -pthread # e.g. : -Wl,-rpath=/a/b -Wl,-rpath=/c -pthread +LINK_O := $(CXX) $(COVERAGE) -r +LINK_SO := $(CXX) $(COVERAGE) $(LINK_OPTS) -shared # some usage may have specific libs, avoid dependencies +LINK_BIN := $(CXX) $(COVERAGE) $(LINK_OPTS) +LINK_LIB := -ldl # -CXX_FLAGS := $(OPT_FLAGS) -std=$(LANG) -COMPILE := $(CC) -ftabstop=4 $(COVERAGE) -fvisibility=hidden -ftemplate-backtrace-limit=0 -fno-strict-aliasing -pthread -pedantic $(WARNING_FLAGS) -Werror -ROOT_DIR := $(abspath .) -LIB := lib -SLIB := _lib -BIN := bin -SBIN := _bin -DOC := doc -SRC := src -LMAKE_ENV := lmake_env -STORE_LIB := $(SRC)/store - -sys_config.h : _bin/sys_config - CC=$(CC) PYTHON=$(PYTHON) PY_LD_LIBRARY_PATH=$(PY_LD_LIBRARY_PATH) ./$< >$@ 2>$@.err - -HAS_SECCOMP := $(shell grep -q 'HAS_SECCOMP *1' sys_config.h 2>/dev/null && echo 1) -HAS_SLURM := $(shell grep -q 'HAS_SLURM *1' sys_config.h 2>/dev/null && echo 1) +ifeq ($(CXX_FLAVOR),clang) + WARNING_FLAGS += -Wno-misleading-indentation -Wno-unknown-warning-option -Wno-c2x-extensions -Wno-c++2b-extensions +endif # -PY_LD_LIBRARY_PATH := $(PY_LIB_DIR) +USER_FLAGS := $(OPT_FLAGS) $(EXTRA_FLAGS) -std=$(LANG) +COMPILE := $(CXX) -ftabstop=4 $(COVERAGE) $(USER_FLAGS) $(HIDDEN_FLAGS) -fvisibility=hidden -ftemplate-backtrace-limit=0 -fno-strict-aliasing -pthread -pedantic $(WARNING_FLAGS) -Werror +ROOT_DIR := $(abspath .) +LIB := lib +SLIB := _lib +BIN := bin +SBIN := _bin +DOC := doc +SRC := src +LMAKE_ENV := lmake_env +STORE_LIB := $(SRC)/store + ifneq ($(PYTHON2),) -ifneq ($(PY2_LIB_DIR),) -ifeq ($(PY_LD_LIBRARY_PATH),) -PY_LD_LIBRARY_PATH := $(PY2_LIB_DIR) -else -PY_LD_LIBRARY_PATH := $(PY_LD_LIBRARY_PATH):$(PY2_LIB_DIR) -endif -endif + PY2_INC_DIRS := $(filter-out $(STD_INC_DIRS),$(PY2_INCLUDEDIR) $(PY2_INCLUDEPY)) # for some reasons, compilation breaks if standard inc dirs are given with -isystem + PY2_CC_OPTS := $(patsubst %,-isystem %,$(PY2_INC_DIRS)) -Wno-register + PY2_LINK_OPTS := $(patsubst %,-L%,$(PY2_LIB_DIR)) $(patsubst %,-Wl$(COMMA)-rpath=%,$(PY2_LIB_DIR)) -l:$(PY2_LIB_BASE) endif +PY_INC_DIRS := $(filter-out $(STD_INC_DIRS),$(PY_INCLUDEDIR) $(PY_INCLUDEPY)) # for some reasons, compilation does not work if standard inc dirs are given with -isystem +PY_CC_OPTS := $(patsubst %,-isystem %,$(PY_INC_DIRS)) -Wno-register +PY_LINK_OPTS := $(patsubst %,-L%,$(PY_LIB_DIR)) $(patsubst %,-Wl$(COMMA)-rpath=%,$(PY_LIB_DIR)) -l:$(PY_LIB_BASE) # Engine SRC_ENGINE := $(SRC)/lmakeserver @@ -252,7 +127,7 @@ LMAKE_REMOTE_FILES := \ $(BIN)/ltarget \ $(LIB)/clmake.so ifneq ($(PYTHON2),) -LMAKE_REMOTE_FILES := $(LMAKE_REMOTE_FILES) $(LIB)/clmake2.so + LMAKE_REMOTE_FILES := $(LMAKE_REMOTE_FILES) $(LIB)/clmake2.so endif LMAKE_BASIC_SAN_OBJS := \ @@ -391,18 +266,18 @@ ALL_H := version.hh sys_config.h ext/xxhash.h # On ubuntu, seccomp.h is in /usr/include. On CenOS7, it is in /usr/include/linux, but beware that otherwise, /usr/include must be prefered, hence -idirafter CPP_OPTS := -iquote ext -iquote $(SRC) -iquote $(SRC_ENGINE) -iquote . -idirafter /usr/include/linux -%_py2.san.o : %.cc $(ALL_H) ; @echo $(CC) -c $(CXX_FLAGS) $(SAN_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -c $(SAN_FLAGS) -frtti -fPIC $(PY2_CC_OPTS) $(CPP_OPTS) -o $@ $< -%_py2.i : %.cc $(ALL_H) ; @echo $(CC) -E $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -E $(PY2_CC_OPTS) $(CPP_OPTS) -o $@ $< -%_py2.s : %.cc $(ALL_H) ; @echo $(CC) -S $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -S $(PY2_CC_OPTS) $(CPP_OPTS) -o $@ $< -%_py2.o : %.cc $(ALL_H) ; @echo $(CC) -c $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -c -frtti -fPIC $(PY2_CC_OPTS) $(CPP_OPTS) -o $@ $< +%_py2.san.o : %.cc $(ALL_H) ; @echo $(CXX) -c $(USER_FLAGS) $(SAN_FLAGS) to $@ ; $(COMPILE) -c $(SAN_FLAGS) -frtti -fPIC $(PY2_CC_OPTS) $(CPP_OPTS) -o $@ $< +%_py2.i : %.cc $(ALL_H) ; @echo $(CXX) -E $(USER_FLAGS) to $@ ; $(COMPILE) -E $(PY2_CC_OPTS) $(CPP_OPTS) -o $@ $< +%_py2.s : %.cc $(ALL_H) ; @echo $(CXX) -S $(USER_FLAGS) to $@ ; $(COMPILE) -S $(PY2_CC_OPTS) $(CPP_OPTS) -o $@ $< +%_py2.o : %.cc $(ALL_H) ; @echo $(CXX) -c $(USER_FLAGS) to $@ ; $(COMPILE) -c -frtti -fPIC $(PY2_CC_OPTS) $(CPP_OPTS) -o $@ $< -%.san.o : %.cc $(ALL_H) ; @echo $(CC) -c $(CXX_FLAGS) $(SAN_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -c $(SAN_FLAGS) -frtti -fPIC $(PY_CC_OPTS) $(CPP_OPTS) -o $@ $< -%.i : %.cc $(ALL_H) ; @echo $(CC) -E $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -E $(PY_CC_OPTS) $(CPP_OPTS) -o $@ $< -%.s : %.cc $(ALL_H) ; @echo $(CC) -S $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -S $(PY_CC_OPTS) $(CPP_OPTS) -o $@ $< -%.o : %.cc $(ALL_H) ; @echo $(CC) -c $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -c -frtti -fPIC $(PY_CC_OPTS) $(CPP_OPTS) -o $@ $< +%.san.o : %.cc $(ALL_H) ; @echo $(CXX) -c $(USER_FLAGS) $(SAN_FLAGS) to $@ ; $(COMPILE) -c $(SAN_FLAGS) -frtti -fPIC $(PY_CC_OPTS) $(CPP_OPTS) -o $@ $< +%.i : %.cc $(ALL_H) ; @echo $(CXX) -E $(USER_FLAGS) to $@ ; $(COMPILE) -E $(PY_CC_OPTS) $(CPP_OPTS) -o $@ $< +%.s : %.cc $(ALL_H) ; @echo $(CXX) -S $(USER_FLAGS) to $@ ; $(COMPILE) -S $(PY_CC_OPTS) $(CPP_OPTS) -o $@ $< +%.o : %.cc $(ALL_H) ; @echo $(CXX) -c $(USER_FLAGS) to $@ ; $(COMPILE) -c -frtti -fPIC $(PY_CC_OPTS) $(CPP_OPTS) -o $@ $< -%_py2.d : %.cc $(ALL_H) ; @$(COMPILE) $(CXX_FLAGS) -MM -MT '$(@:%.d=%.i) $(@:%.d=%.s) $(@:%.d=%.o) $(@:%.d=%.san.o) $@ ' -MF $@ $(PY2_CC_OPTS) $(CPP_OPTS) $< 2>/dev/null || : -%.d : %.cc $(ALL_H) ; @$(COMPILE) $(CXX_FLAGS) -MM -MT '$(@:%.d=%.i) $(@:%.d=%.s) $(@:%.d=%.o) $(@:%.d=%.san.o) $@ ' -MF $@ $(PY_CC_OPTS) $(CPP_OPTS) $< 2>/dev/null || : +%_py2.d : %.cc $(ALL_H) ; @$(COMPILE) -MM -MT '$(@:%.d=%.i) $(@:%.d=%.s) $(@:%.d=%.o) $(@:%.d=%.san.o) $@ ' -MF $@ $(PY2_CC_OPTS) $(CPP_OPTS) $< 2>/dev/null || : +%.d : %.cc $(ALL_H) ; @$(COMPILE) -MM -MT '$(@:%.d=%.i) $(@:%.d=%.s) $(@:%.d=%.o) $(@:%.d=%.san.o) $@ ' -MF $@ $(PY_CC_OPTS) $(CPP_OPTS) $< 2>/dev/null || : include $(patsubst %.cc,%.d, $(filter-out %.x.cc,$(filter %.cc,$(shell git ls-files))) ) include src/py_py2.d src/autodep/clmake_py2.d @@ -525,21 +400,22 @@ $(SBIN)/ldump_job : \ @echo link to $@ @$(LINK_BIN) $(SAN_FLAGS) -o $@ $^ $(PY_LINK_OPTS) $(LINK_LIB) +# XXX : why job_exec does not support sanitize thread ? $(SBIN)/job_exec : \ - $(LMAKE_BASIC_SAN_OBJS) \ - $(SRC)/app$(SAN).o \ - $(SRC)/py$(SAN).o \ - $(SRC)/rpc_job$(SAN).o \ - $(SRC)/trace$(SAN).o \ - $(SRC)/autodep/env$(SAN).o \ - $(SRC)/autodep/gather$(SAN).o \ - $(SRC)/autodep/ptrace$(SAN).o \ - $(SRC)/autodep/record$(SAN).o \ - $(SRC)/autodep/syscall_tab$(SAN).o \ - $(SRC)/job_exec$(SAN).o + $(LMAKE_BASIC_OBJS) \ + $(SRC)/app.o \ + $(SRC)/py.o \ + $(SRC)/rpc_job.o \ + $(SRC)/trace.o \ + $(SRC)/autodep/env.o \ + $(SRC)/autodep/gather.o \ + $(SRC)/autodep/ptrace.o \ + $(SRC)/autodep/record.o \ + $(SRC)/autodep/syscall_tab.o \ + $(SRC)/job_exec.o @mkdir -p $(@D) @echo link to $@ - @@$(LINK_BIN) $(SAN_FLAGS) -o $@ $^ $(PY_LINK_OPTS) $(LIB_SECCOMP) $(LINK_LIB) + @@$(LINK_BIN) -o $@ $^ $(PY_LINK_OPTS) $(LIB_SECCOMP) $(LINK_LIB) $(SBIN)/align_comments : \ $(LMAKE_BASIC_SAN_OBJS) \ @@ -603,12 +479,13 @@ $(BIN)/lmark : \ @echo link to $@ @$(LINK_BIN) $(SAN_FLAGS) -o $@ $^ $(LINK_LIB) +# XXX : why xxhsum does not support sanitize thread ? $(BIN)/xxhsum : \ - $(LMAKE_BASIC_SAN_OBJS) \ + $(LMAKE_BASIC_OBJS) \ $(SRC)/xxhsum.o @mkdir -p $(BIN) @echo link to $@ - @$(LINK_BIN) $(SAN_FLAGS) -o $@ $^ $(LINK_LIB) + @$(LINK_BIN) -o $@ $^ $(LINK_LIB) $(BIN)/autodep : \ $(LMAKE_BASIC_SAN_OBJS) \ @@ -805,7 +682,7 @@ $(LMAKE_ENV)/stamp : $(LMAKE_ALL_FILES) $(LMAKE_ENV)/Manifest $(patsubst %,$(LMA @touch $@ @echo init $(LMAKE_ENV)-cache $(LMAKE_ENV)/tok : $(LMAKE_ENV)/stamp $(LMAKE_ENV)/Lmakefile.py - @set -e ; cd $(LMAKE_ENV) ; export CC=$(CC) ; $(ROOT_DIR)/bin/lmake lmake.tar.gz -Vn & sleep 1 ; $(ROOT_DIR)/bin/lmake lmake.tar.gz >$(@F) || rm -f $(@F) ; wait $$! || rm -f $(@F) + @set -e ; cd $(LMAKE_ENV) ; export CXX=$(CXX) ; $(ROOT_DIR)/bin/lmake lmake.tar.gz -Vn & sleep 1 ; $(ROOT_DIR)/bin/lmake lmake.tar.gz >$(@F) || rm -f $(@F) ; wait $$! || rm -f $(@F) # # archive diff --git a/README.md b/README.md index a591f899..c14b2986 100644 --- a/README.md +++ b/README.md @@ -75,19 +75,19 @@ it has been tested with the dockers listed in the docker directory - the relative positions of these 4 dirs must remain the same, i.e. they must stay in the same directory with the same names. - specialization - you can specialize the build process to better suit your needs : - - this can be done by setting variables on the command line or in the environment - - for example, you can run : make CC=/my/gcc or CC=/my/gcc make + - this can be done by setting variables + - for example, you can run : CXX=/my/g++ make - PYTHON2 can be set to your preferred Python 2 (defaults to python2). You will be told if it is not supported. - PYTHON can be set to your preferred Python 3 (defaults to python3). You will be told if it is not supported. - - CC can be set to your preferred compiler. You will be told if it is not supported. - - LMAKE_OP can be defined as [0123][gG][dD][tT] (default is 3GDT) - - [0123] controls the -O option. - - [gG] controls the -g option : G passes it to enable debug, g does not. - - [dD] controls -DNDEBUG : D passes it to enable asserts, d does not. - - [tT] controls -DNO_TRACE : T does not pass it to enable traces, t passes it. + - CXX can be set to your preferred C++ compiler. You will be told if it is not supported. + - LMAKE_FLAGS can be defined as O[0123]G?D?T?S[AB]C? + - O[0123] controls the -O option (default : 3 ) + - G controls the -g option (default : no debug ) + - d controls -DNDEBUG (default : asserts are enabled) + - t controls -DNO_TRACE (default : traces are enabled ) - the -j flag of make is automatically set to the number of processors, you may want to override this, though - it is up to you to provide a suitable LD\_LIBRARY\_PATH value. - it will be transferred as a default value for rules, to the extent it is necessary to provide lmake semantic (mostly this means we keep what is necessary to run python). + it will be transferred as a default value for rules, to the extent it is necessary to provide the lmake semantic - if you modify these variables, you should execute git clean as make will not detect such modifications automatically. # coding rules @@ -182,6 +182,17 @@ When there is a choice between "if (cond) branch1 else branch2" and "if (!cond) - prefer "true" case to "false" case (at semantic level) - prefer positive test (i.e. prefer == to !=, x or +x to !x, etc.) +## bool values and if +Most objects have a natural "empty" value, such as empty strings, empty vectors, the first value of an enum, etc. +- It is extremely practical to write if (err\_msg) process\_err() ; rather than if (!err\+msg.empty()) process\_err() ; +- This suggests to have casts to bool mostly everywhere, but + - this does not apply to enum nor to STL classes + - this creates a lot of ambiguities + - this is actually pretty dangerous as this weakens static type checking (as bool can in turn be converted to int...) +- The pefect balance is to define the prefix operators + (non empty) and ! (empty) : + - we can write if (+err\_msg) process\_err() ; or if (!err\_msg) process\_ok() ; which is still very light + - it can apply to any type + ## goto's - goto's are used when they allow the code to be easier to read and understand - they are always forward unless specifically flagged with a `BACKWARD` comment, which is exceptional diff --git a/TO_DO b/TO_DO index 9c383360..51c1a9f1 100644 --- a/TO_DO +++ b/TO_DO @@ -74,6 +74,7 @@ items : * ROBUSTNESS & MAINTENABILITY (fragile/difficult to read or maintain as long as not implemented) **************************************************************************************************** +* fix store to be compliant with strict aliasing rules * support 64-bits id - configure with NBits rather than types - in store/file.hh, reserve address space after NBits instead of type diff --git a/_bin/sys_config b/_bin/sys_config index 490b334d..be4b3e62 100755 --- a/_bin/sys_config +++ b/_bin/sys_config @@ -5,11 +5,82 @@ # This program is free software: you can redistribute/modify under the terms of the GPL-v3 (https://www.gnu.org/licenses/gpl-3.0.html). # This program is distributed WITHOUT ANY WARRANTY, without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +MK_FILE=$1 +H_FILE=$2 + unset CDPATH # ensure cd goes to the right directory and writes nothing to stdout +START_DIR=$(pwd) mkdir -p trial cd trial +# +# Python configuration (Python2 is optional) +# + +PYTHON2=${PYTHON2:-python2} ; PYTHON2=$(type -p $PYTHON2||:) # ok if we have no python2 +PYTHON=${PYTHON:-python3} ; PYTHON=$( type -p $PYTHON ) +for python in $PYTHON2 $PYTHON ; do + if [ -x $python ] ; then + eval $($python -c 'if True : + import sys + v = sys.version_info + if v.major==2 : + if v.minor<7 : exit(0) # Python2 is optional + pfx = "PY2" + elif v.major==3 : + if v.minor<6 : + print("echo python3 version must be at least 3.6 ; exit 1") + exit(1) + pfx = "PY" + else : + print("echo unexpected python version",v.major," ; exit 1") + exit(1) + # + import sysconfig + print(pfx+"_INCLUDEDIR="+sysconfig.get_config_var("INCLUDEDIR")) + print(pfx+"_INCLUDEPY=" +sysconfig.get_config_var("INCLUDEPY" )) + print(pfx+"_LIB_BASE=" +sysconfig.get_config_var("LDLIBRARY" )) + lib_dir = sysconfig.get_config_var("LIBDIR") + if not ( (lib_dir+"/").startswith("/usr/lib/") or (lib_dir+"/").startswith("/usr/lib64/") ) : + print(pfx+"_LIB_DIR="+lib_dir) # suppress dir as required in case of installed package + ') + fi +done +if [ "$PY2_LIB_DIR" -a "$PY_LIB_DIR" ] ; then PY_LD_LIBRARY_PATH=$PY_LIB_DIR:$PY2_LIB_DIR # both present, separate with : +else PY_LD_LIBRARY_PATH=$PY_LIB_DIR$PY2_LIB_DIR # only one is present +fi +# cancel Python2 if unusable as a library +{ [ -f $PY2_INCLUDEPY/Python.h ] && file -L $PY2_LIB_DIR/$PY2_LIB_BASE | grep -q shared ; } || PYTHON2= + +# +# CXX configuration +# +CXX=${CXX:-g++} +[ "$CXX" != "${CXX%++}" ] || { echo cannot recognize c++ compiler $CXX ; exit 1 ; } +type -p $CXX >/dev/null || { echo cannot find c++ compiler $CXX ; exit 1 ; } + +case "$($CXX --version|head -1)" in + *clang* ) CXX_FLAVOR=clang ; [ "$($CXX -dumpversion)" -lt 15 ] && { echo clang version must be at least 15 ; exit 1 ; } ;; + *g++* ) CXX_FLAVOR=gcc ; [ "$($CXX -dumpversion)" -lt 11 ] && { echo gcc version must be at least 11 ; exit 1 ; } ;; + * ) echo cannot recognize c++ compiler $CXX ; exit 1 ;; +esac + +LLP="$($CXX -v -E /dev/null 2>&1 | grep LIBRARY_PATH=)" # e.g. : LIBARY_PATH=/usr/lib/x:/a/b:/c:/a/b/c/.. +LLP="$(echo $LLP | sed 's/LIBRARY_PATH=//' )" # e.g. : /usr/lib/x:/a/b:/c:/a/b/c/.. +LLP="$(echo $LLP | sed 's/:/ /' )" # e.g. : /usr/lib/x /a/b /c /a/b/c/.. +LLP="$(echo $LLP | sort -u )" # e.g. : /usr/lib/x /a/b /c /a/b +LINK_LIB_PATH= # e.g. : /a/b /c /usr/lib/x +for l in $LLP ; do # e.g. : /a/b /c (suppress standard dirs as required in case of installed package) + case $l/ in + /usr/lib/* ) ;; + /usr/lib64/*) ;; + * ) LINK_LIB_PATH="$LINK_LIB_PATH $l" ;; + esac +done + +STD_INC_DIRS="$(echo $( $CXX -E -v -std=c++20 -xc++ /dev/null 2>&1 | sed -e '1,/<.*>.*search starts/d' -e '/End of search/,$d' ) )" + # # HAS_SECCOMP # test whether we have seccomp : warning, include is not enough to test, check its content too @@ -18,8 +89,8 @@ cat <<"EOF" > seccomp.c #include struct scmp_version v ; EOF -# usually seccomp.h is in /usr/include (with an internal file /usr/include/linux/seccompt.h) but some systems have seccomp.h directly installed in /usr/include/linux -if $CC -c -idirafter /usr/include/linux -o ptrace.o seccomp.c +# usually seccomp.h is in /usr/include (with an internal file /usr/include/linux/seccomp.h) but some systems have seccomp.h directly installed in /usr/include/linux +if $CXX -c -idirafter /usr/include/linux -o ptrace.o -xc seccomp.c then HAS_SECCOMP=1 else HAS_SECCOMP=0 fi @@ -35,7 +106,7 @@ cat <<"EOF" > undef_ptrace_macros.c long rx = ptrace(PTRACE_TRACEME,0,0,0) ; } EOF -if $CC -c -idirafter /usr/include/linux -o undef_ptrace_macros.o undef_ptrace_macros.c +if $CXX -c -idirafter /usr/include/linux -o undef_ptrace_macros.o -xc undef_ptrace_macros.c then MUST_UNDEF_PTRACE_MACROS=1 else MUST_UNDEF_PTRACE_MACROS=0 fi @@ -53,7 +124,7 @@ cat <<"EOF" > get_syscall_info.c long rx = ptrace(PTRACE_GET_SYSCALL_INFO,0,0,0) ; } EOF -if $CC -c -idirafter /usr/include/linux -DMUST_UNDEF_PTRACE_MACROS=$MUST_UNDEF_PTRACE_MACROS -o get_syscall_info.o get_syscall_info.c +if $CXX -c -idirafter /usr/include/linux -DMUST_UNDEF_PTRACE_MACROS=$MUST_UNDEF_PTRACE_MACROS -o get_syscall_info.o -xc get_syscall_info.c then HAS_PTRACE_GET_SYSCALL_INFO=1 else HAS_PTRACE_GET_SYSCALL_INFO=0 fi @@ -62,7 +133,7 @@ fi # HAS_CLOSE_RANGE # test whether we can include linux/close_range.h # -if $CC -E --include 'linux/close_range.h' -xc - /dev/null 2>/dev/null +if $CXX -E --include 'linux/close_range.h' -xc - /dev/null 2>/dev/null then HAS_CLOSE_RANGE=1 else HAS_CLOSE_RANGE=0 fi @@ -94,8 +165,8 @@ if [ -z "$HAS_LD_AUDIT" ] ; then #include unsigned int la_version(unsigned int) { printf("1") ; exit(0) ; } EOF - $CC -o audited audited.c - $CC -o ld_audit.so -shared -fPIC ld_audit.c + $CXX -o audited -xc audited.c + $CXX -o ld_audit.so -shared -fPIC -xc ld_audit.c HAS_LD_AUDIT=$(LD_AUDIT=./ld_audit.so ./audited) fi @@ -108,7 +179,7 @@ cat <<"EOF" > stat.c struct stat buf ; int main() { stat("",&buf) ; } EOF -$CC -o stat_trial stat.c +$CXX -o stat_trial -xc stat.c if nm -D stat_trial | grep -wq stat then NEED_STAT_WRAPPERS=0 else NEED_STAT_WRAPPERS=1 @@ -122,7 +193,7 @@ cat <<"EOF" > ostringstream_view.cc #include std::string_view ossv = std::ostringstream().view() ; EOF -if $CC -c -std=c++20 -o ostringstream_view.o ostringstream_view.cc 2>/dev/null +if $CXX -c -std=c++20 -o ostringstream_view.o -xc ostringstream_view.cc 2>/dev/null then HAS_OSTRINGSTREAM_VIEW=1 else HAS_OSTRINGSTREAM_VIEW=0 fi @@ -130,7 +201,7 @@ fi # # HAS_SLURM # -if $CC -E --include slurm/slurm.h -xc - /dev/null 2>/dev/null +if $CXX -E --include slurm/slurm.h -xc - /dev/null 2>/dev/null then HAS_SLURM=1 else HAS_SLURM=0 fi @@ -138,9 +209,9 @@ fi # # addr2line # -if [ -x $(dirname $CC)/addr2line ] ; then ADDR2LINE=$(dirname $CC)/addr2line -else ADDR2LINE=$(type -p addr2line) -fi +[ "" ] || ADDR2LINE=$(find $(dirname $(realpath $(type -p $CXX))) -name llvm-addr2line) +[ -x "$ADDR2LINE" ] || ADDR2LINE=$(find $(dirname $(realpath $(type -p $CXX))) -name addr2line ) +[ -x "$ADDR2LINE" ] || ADDR2LINE=$(type -p addr2line) # # stacktrace @@ -149,7 +220,7 @@ cat <<"EOF" > stacktrace.cc #include std::stacktrace st ; EOF -if $CC -c -std=c++23 -o stacktrace.o stacktrace.cc 2>/dev/null +if $CXX -c -std=c++23 -o stacktrace.o stacktrace.cc 2>/dev/null then HAS_STACKTRACE=1 else HAS_STACKTRACE=0 fi @@ -158,11 +229,13 @@ fi # HAS_MEMFD # cat <<"EOF" > memfd.c - #define _GNU_SOURCE + #ifndef _GNU_SOURCE + #define _GNU_SOURCE + #endif #include int main() { memfd_create("trial",0) ; } EOF -if $CC -Werror -o memfd memfd.c +if $CXX -Werror -o memfd -xc memfd.c then HAS_MEMFD=1 else HAS_MEMFD=0 fi @@ -178,10 +251,37 @@ cat <<"EOF" > start_main.c } int main() { write(1,"0",1) ; } EOF -$CC -o start_main start_main.c +$CXX -o start_main -xc start_main.c USE_LIBC_START_MAIN=$(./start_main) -cat <$MK_FILE <$H_FILE < struct Syntax { // SWEAR(!( has_dflt_key && +ks && ks.at(Key::None).short_name )) ; // - for( auto const& [k,s] : ks ) keys [+k] = s ; - for( auto const& [f,s] : fs ) flags[+f] = s ; + for( auto const& [k,s] : ks ) { SWEAR(!keys [+k].short_name) ; keys [+k] = s ; } // ensure no short name conflicts + for( auto const& [f,s] : fs ) { SWEAR(!flags[+f].short_name) ; flags[+f] = s ; } // . } // [[noreturn]] void usage(::string const& msg={}) const ; diff --git a/src/autodep/record.hh b/src/autodep/record.hh index c46934b0..f8ff8ee8 100644 --- a/src/autodep/record.hh +++ b/src/autodep/record.hh @@ -153,7 +153,7 @@ public : _Path( ::string const& f , bool steal=false ) : file{f.c_str()} { if (!steal) allocate(f.size()) ; } _Path( Fd a , ::string const& f , bool steal=false ) : has_at{true} , at{a} , file{f.c_str()} { if (!steal) allocate(f.size()) ; } // - _Path(_Path && p) { *this = ::move(p) ; } + _Path(_Path && p) { *this = ::move(p) ; } // XXX : use copy&swap idiom _Path& operator=(_Path&& p) { deallocate() ; has_at = p.has_at ; diff --git a/src/client.cc b/src/client.cc index ba5de9e5..f986f304 100644 --- a/src/client.cc +++ b/src/client.cc @@ -17,8 +17,8 @@ using namespace Time ; AutoCloseFdPair g_server_fds ; -static bool server_ok( Fd fd , ::string const& tag ) { - Trace trace("server_ok",tag,fd) ; +static bool _server_ok( Fd fd , ::string const& tag ) { + Trace trace("_server_ok",tag,fd) ; bool ok = false ; int cnt = ::read( fd , &ok , sizeof(bool) ) ; if (cnt!=sizeof(bool)) { trace("bad_answer",cnt) ; return false ; } @@ -26,8 +26,8 @@ static bool server_ok( Fd fd , ::string const& tag ) { return ok ; } -static void connect_to_server(bool refresh) { - Trace trace("connect_to_server",STR(refresh)) ; +static pid_t _connect_to_server( bool refresh , bool sync ) { // if sync, ensure we launch our own server + Trace trace("_connect_to_server",STR(refresh)) ; ::string server_service ; bool server_is_local = false ; pid_t server_pid = 0 ; @@ -48,9 +48,10 @@ static void connect_to_server(bool refresh) { server_is_local = true ; } ClientSockFd req_fd { server_service , 3/*n_trials*/ , Delay(3)/*timeout*/ } ; - if (server_ok(req_fd,"old")) { + if (_server_ok(req_fd,"old")) { g_server_fds = ::move(req_fd) ; - return ; + if (sync) exit(Rc::Format,"server already exists") ; + return 0 ; } } catch(::string const&) { trace("cannot_connect",server_service) ; } // @@ -75,10 +76,11 @@ static void connect_to_server(bool refresh) { client_to_server.read .close() ; server_to_client.write.close() ; // - if (server_ok(server_to_client.read,"new")) { + if (_server_ok(server_to_client.read,"new")) { g_server_fds = AutoCloseFdPair{ server_to_client.read , client_to_server.write } ; + pid_t pid = server.pid ; server.mk_daemon() ; - return ; + return pid ; } client_to_server.write.close() ; server_to_client.read .close() ; @@ -172,12 +174,14 @@ static Bool3 is_reverse_video( Fd in_fd , Fd out_fd ) { return res ; } -Bool3/*ok*/ out_proc( ::ostream& os , ReqProc proc , bool refresh , ReqSyntax const& syntax , ReqCmdLine const& cmd_line , ::function const& started_cb ) { +Bool3/*ok*/ _out_proc( ::vector_s* files , ReqProc proc , bool refresh , ReqSyntax const& syntax , ReqCmdLine const& cmd_line , ::function const& started_cb ) { Trace trace("out_proc") ; // if ( cmd_line.flags[ReqFlag::Job] && cmd_line.args.size()!=1 ) syntax.usage("can process several files, but a single job" ) ; if ( !cmd_line.flags[ReqFlag::Job] && cmd_line.flags[ReqFlag::Rule] ) syntax.usage("can only force a rule to identify a job, not a file") ; // + bool sync = cmd_line.flags[ReqFlag::Sync] ; + // Bool3 rv = Maybe/*garbage*/ ; ::string rv_str ; if (!rv_str) { rv_str = cmd_line.flag_args[+ReqFlag::Video] ; trace("cmd_line",rv_str) ; } @@ -193,21 +197,26 @@ Bool3/*ok*/ out_proc( ::ostream& os , ReqProc proc , bool refresh , ReqSyntax co } trace("reverse_video",rv) ; // - ReqRpcReq rrr { proc , cmd_line.files() , { rv , cmd_line } } ; - connect_to_server(refresh) ; + ReqRpcReq rrr { proc , cmd_line.files() , { rv , cmd_line } } ; + Bool3 rc = Maybe ; + pid_t server_pid = _connect_to_server(refresh,sync) ; started_cb() ; OMsgBuf().send(g_server_fds.out,rrr) ; try { for(;;) { + using Proc = ReqRpcReplyProc ; ReqRpcReply report = IMsgBuf().receive(g_server_fds.in) ; - switch (report.kind) { - case ReqKind::None : trace("none" ) ; return Maybe ; - case ReqKind::Status : trace("done",STR(report.ok)) ; return report.ok?Yes:No ; - case ReqKind::Txt : os << report.txt << flush ; break ; + switch (report.proc) { + case Proc::None : trace("done" ) ; goto Return ; + case Proc::Status : trace("status",STR(report.ok)) ; rc = No|report.ok ; break ; + case Proc::File : trace("file" ,report.txt ) ; SWEAR(files) ; files->push_back(report.txt) ; break ; + case Proc::Txt : ::cout << report.txt << flush ; break ; DF} } } catch(...) { trace("disconnected") ; - return Maybe ; // input has been closed by peer before reporting a status } +Return : + if (sync) waitpid( server_pid , nullptr , 0 ) ; + return rc ; } diff --git a/src/client.hh b/src/client.hh index 9deaaaf6..7acc0293 100644 --- a/src/client.hh +++ b/src/client.hh @@ -32,4 +32,10 @@ inline Rc mk_rc(Bool3 ok) { DF} } -Bool3/*ok*/ out_proc( ::ostream& , ReqProc , bool refresh , ReqSyntax const& , ReqCmdLine const& , ::function const& started_cb = []()->void{} ) ; +#define RSC ReqSyntax const +#define RCLC ReqCmdLine const +/**/ Bool3/*ok*/ _out_proc( ::vector_s* files , ReqProc , bool refresh , RSC& , RCLC& , ::function const& started_cb ) ; +inline Bool3/*ok*/ out_proc ( ::vector_s& fs , ReqProc p , bool r , RSC& s , RCLC& cl , ::function const& cb=[]()->void{} ) { return _out_proc(&fs ,p,r,s,cl,cb) ; } +inline Bool3/*ok*/ out_proc ( ReqProc p , bool r , RSC& s , RCLC& cl , ::function const& cb=[]()->void{} ) { return _out_proc(nullptr,p,r,s,cl,cb) ; } +#undef RCLC +#undef RSC diff --git a/src/fd.hh b/src/fd.hh index de38287c..04c65866 100644 --- a/src/fd.hh +++ b/src/fd.hh @@ -22,7 +22,7 @@ struct Fd { static const Fd Std ; // the highest standard fd // cxtors & casts constexpr Fd( ) = default ; - constexpr Fd( Fd const& fd_ ) { *this = fd_ ; } + constexpr Fd( Fd const& fd_ ) { *this = fd_ ; } // XXX : use copy&swap idiom constexpr Fd( Fd && fd_ ) { *this = ::move(fd_) ; } constexpr Fd( int fd_ ) : fd{fd_} { } /**/ Fd( int fd_ , bool no_std_ ) : fd{fd_} { if (no_std_) no_std() ; } @@ -71,7 +71,7 @@ struct AutoCloseFd : Fd { // cxtors & casts using Fd::Fd ; AutoCloseFd(Fd const& fd_ ) : Fd{ fd_ } { } - AutoCloseFd(AutoCloseFd && acfd) : Fd{::move(acfd)} { SWEAR(!acfd) ; } // ensure acfd has been detached + AutoCloseFd(AutoCloseFd && acfd) : Fd{::move(acfd)} { SWEAR(!acfd) ; } // ensure acfd has been detached XXX : use copy&swap idiom AutoCloseFd(AutoCloseFd const& acfd) = delete ; // ~AutoCloseFd() { close() ; } @@ -244,7 +244,7 @@ struct Epoll { void add_write( Fd fd_ ) { add(true /*write*/,fd_ ) ; } void del(Fd fd_) { int rc = ::epoll_ctl( fd , EPOLL_CTL_DEL , fd_ , nullptr ) ; - swear_prod(rc==0,"cannot del ",fd_," from epoll ",fd," (",strerror(errno),')') ; + swear_prod(rc==0,"cannot del",fd_,"from epoll",fd,'(',strerror(errno),')') ; cnt-- ; } void close(Fd fd_) { SWEAR(+fd_) ; del(fd_) ; fd_.close() ; } diff --git a/src/job_exec.cc b/src/job_exec.cc index ce0a079f..f708822e 100644 --- a/src/job_exec.cc +++ b/src/job_exec.cc @@ -239,6 +239,8 @@ Digest analyze( bool at_end , bool killed=false ) { res.targets.emplace_back(file,td) ; //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ trace("target",ad,td,STR(unlnk),file) ; + } else if (!is_dep) { + trace("ignore",ad,file) ; } } for( ::string const& t : g_washed ) if (!g_gather.access_map.contains(t)) { @@ -419,8 +421,8 @@ int main( int argc , char* argv[] ) { // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv Status status = g_gather.exec_child( cmd_line() , child_stdin , child_stdout , Child::Pipe ) ; // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - bool killed = g_killed ; // sample g_killed to ensure coherence (even if status is correct, it may mean we were waiting for stdout/stderr) Pdate end_job = New ; // as early as possible after child ends + bool killed = g_killed ; // sample g_killed to ensure coherence (even if status is correct, it may mean we were waiting for stdout/stderr) struct rusage rsrcs ; getrusage(RUSAGE_CHILDREN,&rsrcs) ; trace("start_job",start_job,"end_job",end_job) ; // diff --git a/src/ldebug.cc b/src/ldebug.cc index dc357284..a824bb27 100644 --- a/src/ldebug.cc +++ b/src/ldebug.cc @@ -12,34 +12,34 @@ using namespace Disk ; int main( int argc , char* argv[] ) { - app_init() ; Trace trace("main") ; - + // ReqSyntax syntax{{},{ { ReqFlag::Graphic , { .short_name='g' , .doc="launch execution under pudb control" } } , { ReqFlag::Vscode , { .short_name='c' , .doc="launch execution under vscode control" } } }} ; ReqCmdLine cmd_line{syntax,argc,argv} ; - + // if ( cmd_line.args.size()<1 ) syntax.usage( "need a target to debug" ) ; if ( cmd_line.args.size()>1 ) syntax.usage(to_string("cannot debug ",cmd_line.args.size()," targets at once")) ; if ( cmd_line.flags[ReqFlag::Graphic] && cmd_line.flags[ReqFlag::Vscode] ) syntax.usage( "cannot debug with pudb and vscode simultaneously" ) ; cmd_line.flags |= ReqFlag::Debug ; - - ::ostringstream script_file_stream ; - // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - Bool3 ok = out_proc( script_file_stream , ReqProc::Debug , false/*refresh_makefiles*/ , syntax , cmd_line ) ; - // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ::string script_file = ensure_no_nl(script_file_stream.str()) ; - if ( Rc rc=mk_rc(ok) ; +rc ) exit(rc,script_file) ; - + // + ::vector_s script_files ; + // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + Bool3 ok = out_proc( script_files , ReqProc::Debug , false/*refresh_makefiles*/ , syntax , cmd_line ) ; + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + if ( Rc rc=mk_rc(ok) ; +rc ) exit(rc) ; + SWEAR(script_files.size()==1,script_files) ; + ::string& script_file = script_files[0] ; + // char* exec_args[] = { script_file.data() , nullptr } ; - + // ::cerr << "executing : " << script_file << endl ; //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv ::execv(script_file.c_str(),exec_args) ; //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - + // exit(Rc::System,"could not run ",script_file) ; } diff --git a/src/lforget.cc b/src/lforget.cc index b3ce4455..a6c0131b 100644 --- a/src/lforget.cc +++ b/src/lforget.cc @@ -26,8 +26,8 @@ int main( int argc , char* argv[] ) { case ReqKey::Resources : if (+cmd_line.args) syntax.usage("must not have targets when forgetting errors" ) ; break ; default : ; } - // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - Bool3 ok = out_proc( ::cout , ReqProc::Forget , false/*refresh_makefiles*/ , syntax , cmd_line ) ; - // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + Bool3 ok = out_proc( ReqProc::Forget , false/*refresh_makefiles*/ , syntax , cmd_line ) ; + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exit(mk_rc(ok)) ; } diff --git a/src/lmake.cc b/src/lmake.cc index ead598b1..2a06603d 100644 --- a/src/lmake.cc +++ b/src/lmake.cc @@ -82,8 +82,8 @@ int main( int argc , char* argv[] ) { try { from_string(n_jobs,true/*empty_ok*/) ; } catch (::string const& e) { syntax.usage(to_string("cannot understand max number of jobs (",e,") : ",n_jobs)) ; } // start interrupt handling thread once server is started - //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - Bool3 ok = out_proc( ::cout , ReqProc::Make , true/*refresh_makefiles*/ , syntax , cmd_line , _handle_int ) ; - //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + Bool3 ok = out_proc( ReqProc::Make , true/*refresh_makefiles*/ , syntax , cmd_line , _handle_int ) ; + //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exit(mk_rc(ok)) ; } diff --git a/src/lmakeserver/backend.cc b/src/lmakeserver/backend.cc index 5bb0a218..b3235325 100644 --- a/src/lmakeserver/backend.cc +++ b/src/lmakeserver/backend.cc @@ -135,14 +135,15 @@ namespace Backends { } void Backend::_s_handle_deferred_wakeup(DeferredEntry&& de) { + Trace trace(BeChnl,"_s_handle_deferred_wakeup",de) ; { Lock lock { _s_mutex } ; // lock _s_start_tab for minimal time to avoid dead-locks auto it = _s_start_tab.find(+de.job_exec) ; if (it==_s_start_tab.end() ) return ; // too late, job has ended if (it->second.conn.seq_id!=de.seq_id) return ; // too late, job has ended and restarted } - Trace trace(BeChnl,"_s_handle_deferred_wakeup",de) ; JobDigest jd { .status=Status::LateLost } ; // job is still present, must be really lost if (+de.job_exec.start_date.p) jd.stats.total = Pdate(New)-de.job_exec.start_date.p ; + trace("lost",jd) ; _s_handle_job_end( JobRpcReq( JobProc::End , de.seq_id , +de.job_exec , ::move(jd) ) ) ; } @@ -388,7 +389,7 @@ namespace Backends { entry.conn.port = jrr.port ; entry.conn.small_id = reply.small_id ; // - trace("started",reply) ; + trace("started",job_exec,reply) ; } bool report_now = +pre_actions.second || +start_msg_err.second || Delay(job->exec_time)>=start_none_attrs.start_delay ; // dont defer long jobs or if a message is to be delivered to user //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv @@ -484,7 +485,7 @@ namespace Backends { //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv g_engine_queue.emplace( JobProc::GiveUp , JobExec(j,New,New) , req , false/*report*/ ) ; //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - trace("waiting",j) ; + trace("queued_in_backend",j) ; } for( auto jit = _s_start_tab.begin() ; jit!=_s_start_tab.end() ;) { // /!\ we erase entries while iterating JobIdx j = jit->first ; @@ -509,7 +510,7 @@ namespace Backends { to_wakeup.emplace_back(j,::pair(e.conn,e.start_date)) ; jit++ ; } else { - trace("not_started",j) ; + trace("queued_in_slurm",j) ; s_tab[+e.tag]->kill_job(j) ; //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv g_engine_queue.emplace( JobProc::GiveUp , JobExec(j,e.start_date,Pdate(New)) ) ; diff --git a/src/lmakeserver/backends/generic.hh b/src/lmakeserver/backends/generic.hh index cd84b1af..5cbb7657 100644 --- a/src/lmakeserver/backends/generic.hh +++ b/src/lmakeserver/backends/generic.hh @@ -52,8 +52,7 @@ namespace Backends { else it->second-- ; // data is shared, just decrement refcount } // - Shared& operator=(Shared const& r) { this->~Shared() ; new(this) Shared( r ) ; return *this ; } - Shared& operator=(Shared && r) { this->~Shared() ; new(this) Shared(::move(r)) ; return *this ; } + Shared& operator=(Shared s) { swap(*this,s) ; return *this ; } // bool operator==(Shared const&) const = default ; // access @@ -65,6 +64,9 @@ namespace Backends { Data const* data = nullptr ; } ; template< class Data , ::unsigned_integral RefCnt > ::umap Shared::_s_store ; + template< class Data , ::unsigned_integral RefCnt > void swap( Shared& s1 , Shared& s2 ) { + ::swap(s1.data,s2.data) ; + } } @@ -167,14 +169,14 @@ namespace Backends { return IsLocal ; } virtual void open_req( ReqIdx req , JobIdx n_jobs ) { - Trace trace("open_req",req,n_jobs) ; + Trace trace(BeChnl,"open_req",req,n_jobs) ; Lock lock { Req::s_reqs_mutex } ; // taking Req::s_reqs_mutex is compulsery to derefence req bool inserted = reqs.insert({ req , {n_jobs,Req(req)->options.flags[ReqFlag::Verbose]} }).second ; SWEAR(inserted) ; } virtual void close_req(ReqIdx req) { auto it = reqs.find(req) ; - Trace trace("close_req",req,STR(it==reqs.end())) ; + Trace trace(BeChnl,"close_req",req,STR(it==reqs.end())) ; if (it==reqs.end()) return ; // req has been killed ReqEntry const& re = it->second ; SWEAR(!re.waiting_jobs) ; @@ -193,31 +195,31 @@ namespace Backends { SWEAR(!waiting_jobs .contains(job)) ; // job must be a new one SWEAR(!re.waiting_jobs.contains(job)) ; // in particular for this req CoarseDelay pressure = submit_attrs.pressure ; - Trace trace("submit",rsa,pressure) ; + Trace trace(BeChnl,"submit",rsa,pressure) ; // re.waiting_jobs[job] = pressure ; waiting_jobs.emplace( job , WaitingEntry(rsa,submit_attrs,re.verbose) ) ; re.waiting_queues[rsa].insert({pressure,job}) ; } virtual void add_pressure( JobIdx job , ReqIdx req , SubmitAttrs const& submit_attrs ) { - Trace trace("add_pressure",job,req,submit_attrs) ; + Trace trace(BeChnl,"add_pressure",job,req,submit_attrs) ; ReqEntry& re = reqs.at(req) ; auto wit = waiting_jobs.find(job) ; if (wit==waiting_jobs.end()) { // job is not waiting anymore, mostly ignore auto sit = spawned_jobs.find(job) ; if (sit==spawned_jobs.end()) { // job is already ended - trace("ended") ; + trace(BeChnl,"ended") ; } else { SpawnedEntry& se = sit->second ; // if not waiting, it must be spawned if add_pressure is called if (re.verbose ) se.verbose = true ; // mark it verbose, though - trace("queued") ; + trace(BeChnl,"queued") ; } return ; } WaitingEntry& we = wit->second ; SWEAR(!re.waiting_jobs.contains(job)) ; // job must be new for this req CoarseDelay pressure = submit_attrs.pressure ; - trace("adjusted_pressure",pressure) ; + trace(BeChnl,"adjusted_pressure",pressure) ; // re.waiting_jobs[job] = pressure ; re.waiting_queues[we.rsrcs_ask].insert({pressure,job}) ; // job must be known @@ -234,7 +236,7 @@ namespace Backends { CoarseDelay & old_pressure = re.waiting_jobs .at(job ) ; // job must be known ::set& q = re.waiting_queues.at(we.rsrcs_ask) ; // including for this req CoarseDelay pressure = submit_attrs.pressure ; - Trace trace("set_pressure","pressure",pressure) ; + Trace trace(BeChnl,"set_pressure","pressure",pressure) ; we.submit_attrs |= submit_attrs ; q.erase ({old_pressure,job}) ; q.insert({pressure ,job}) ; @@ -268,7 +270,7 @@ namespace Backends { ::pair_s digest = heartbeat_queued_job(j,se) ; // if (digest.second!=HeartbeatState::Alive) { - Trace trace("heartbeat",j,se.id) ; + Trace trace(BeChnl,"heartbeat",j,se.id) ; /**/ spawned_jobs .erase(it) ; for( auto& [r,re] : reqs ) re.queued_jobs.erase(j ) ; } @@ -277,7 +279,7 @@ namespace Backends { // kill all if req==0 virtual ::vector kill_waiting_jobs(ReqIdx req=0) { ::vector res ; - Trace trace("kill_req",T,req,reqs.size()) ; + Trace trace(BeChnl,"kill_req",T,req,reqs.size()) ; if ( !req || reqs.size()<=1 ) { if (req) SWEAR( reqs.size()==1 && req==reqs.begin()->first ) ; // ensure the last req is the right one // kill waiting jobs @@ -299,7 +301,7 @@ namespace Backends { return res ; } virtual void kill_job(JobIdx j) { - Trace trace("kill_job",j) ; + Trace trace(BeChnl,"kill_job",j) ; auto it = spawned_jobs.find(j) ; SWEAR(it!=spawned_jobs.end()) ; SWEAR(!it->second.started) ; // if job is started, it is not our responsibility any more @@ -308,7 +310,7 @@ namespace Backends { } virtual void launch( ) { launch(Yes,{}) ; } // using default arguments is not recognized to override virtual methods virtual void launch( Bool3 go , Rsrcs rsrcs ) { - Trace trace("launch",T,go,rsrcs) ; + Trace trace(BeChnl,"launch",T,go,rsrcs) ; RsrcsAsk rsrcs_ask ; switch (go) { case No : return ; @@ -356,7 +358,7 @@ namespace Backends { try { SpawnId id = launch_job( j , prio , cmd_line , rsrcs , wit->second.verbose ) ; spawned_jobs[j] = { .rsrcs=rsrcs , .id=id , .verbose=wit->second.verbose } ; - trace("child",req,j,prio,id,cmd_line) ; + trace(BeChnl,"child",req,j,prio,id,cmd_line) ; } catch (::string const& e) { err_jobs.push_back({j,{e,rsrcs_map}}) ; ok = false ; diff --git a/src/lmakeserver/backends/slurm.cc b/src/lmakeserver/backends/slurm.cc index 7291532a..1e35b3de 100644 --- a/src/lmakeserver/backends/slurm.cc +++ b/src/lmakeserver/backends/slurm.cc @@ -249,8 +249,8 @@ namespace Backends::Slurm { virtual uint32_t/*id*/ launch_job( JobIdx j , Pdate prio , ::vector_s const& cmd_line , Rsrcs const& rs , bool verbose ) const { int32_t nice = use_nice ? int32_t((prio-daemon.time_origin).sec()*daemon.nice_factor) : 0 ; nice &= 0x7fffffff ; // slurm will not accept negative values, default values overflow in ... 2091 - Trace trace(Channel::Backend,"Slurm::launch_job",repo_key,j,nice,cmd_line,rs,STR(verbose)) ; uint32_t id = slurm_spawn_job( repo_key , j , nice , cmd_line , *rs , verbose ) ; + Trace trace(BeChnl,"Slurm::launch_job",repo_key,j,id,nice,cmd_line,rs,STR(verbose)) ; spawned_rsrcs.inc(rs) ; // only reserv resources once we are sure job is launched return id ; } @@ -398,7 +398,7 @@ namespace Backends::Slurm { if (!dst) throw to_string("cannot find ",name," in ",LibSlurm) ; } void slurm_init() { - Trace trace("slurm_init") ; + Trace trace(BeChnl,"slurm_init") ; void* handler = ::dlopen(LibSlurm,RTLD_NOW|RTLD_GLOBAL) ; if (!handler) throw to_string("cannot find ",LibSlurm) ; // @@ -443,7 +443,7 @@ namespace Backends::Slurm { RsrcsData parse_args(::string const& args) { static ::string slurm = "slurm" ; // apparently "slurm"s.data() does not work as memory is freed right away - Trace trace(Channel::Backend,"parse_args",args) ; + Trace trace(BeChnl,"parse_args",args) ; // if (!args) return {} ; // fast path // @@ -492,7 +492,7 @@ namespace Backends::Slurm { //This for loop with a retry comes from the scancel Slurm utility code //Normally we kill mainly waiting jobs, but some "just started jobs" could be killed like that also //Running jobs are killed by lmake/job_exec - Trace trace(Channel::Backend,"slurm_cancel",slurm_id) ; + Trace trace(BeChnl,"slurm_cancel",slurm_id) ; int i = 0/*garbage*/ ; for( i=0 ; i<10/*MAX_CANCEL_RETRY*/ ; i++ ) { if (SlurmApi::kill_job(slurm_id,SIGKILL,KILL_FULL_JOB)==SLURM_SUCCESS) { trace("done") ; return ; } @@ -508,7 +508,7 @@ namespace Backends::Slurm { } ::pair_s slurm_job_state(uint32_t slurm_id) { // Maybe means job has not completed - Trace trace(Channel::Backend,"slurm_job_state",slurm_id) ; + Trace trace(BeChnl,"slurm_job_state",slurm_id) ; job_info_msg_t* resp = nullptr/*garbage*/ ; // if (SlurmApi::load_job(&resp,slurm_id,SHOW_LOCAL)!=SLURM_SUCCESS) return { "cannot load job info : "+slurm_err() , Yes/*job_ok*/ } ; // no info on job -> retry @@ -521,7 +521,7 @@ namespace Backends::Slurm { completed &= js>=JOB_COMPLETE ; if (js<=JOB_COMPLETE) continue ; // we only search errors const char* on_nodes = !ji.nodes||::strchr(ji.nodes,' ')==nullptr?" on node : ":" on nodes : " ; - int exit_code = ji.exit_code ; + int exit_code = ji.exit_code ; // when job_exec receives a signal, the bash process which launches it (which the process seen by slurm) exits with an exit code > 128 // however, the user is interested in the received signal, not mapped bash exit code, so undo mapping // signaled wstatus are barely the signal number @@ -554,7 +554,7 @@ namespace Backends::Slurm { static ::string _get_stdout_path(JobIdx job) { return _get_log_dir(job) + "/stdout" ; } ::string read_stderr(JobIdx job) { - Trace trace(Channel::Backend,"Slurm::read_stderr",job) ; + Trace trace(BeChnl,"Slurm::read_stderr",job) ; ::string err_file = _get_stderr_path(job) ; try { ::string res = read_content(err_file) ; @@ -574,7 +574,7 @@ namespace Backends::Slurm { } uint32_t slurm_spawn_job( ::string const& key , JobIdx job , int32_t nice , ::vector_s const& cmd_line , RsrcsData const& rsrcs , bool verbose ) { static char* env[1] = {const_cast("")} ; - Trace trace(Channel::Backend,"slurm_spawn_job",key,job,nice,cmd_line,rsrcs,STR(verbose)) ; + Trace trace(BeChnl,"slurm_spawn_job",key,job,nice,cmd_line,rsrcs,STR(verbose)) ; // SWEAR(rsrcs.size()> 0) ; SWEAR(nice >=0) ; @@ -636,7 +636,7 @@ namespace Backends::Slurm { } Daemon slurm_sense_daemon() { - Trace trace("slurm_sense_daemon") ; + Trace trace(BeChnl,"slurm_sense_daemon") ; slurm_conf_t* conf = nullptr ; // XXX : remember last conf read so as to pass a real update_time param & optimize call if (!is_target("/etc/slurm/slurm.conf") ) throw "no slurm config file /etc/slur/slurm.conf"s ; diff --git a/src/lmakeserver/cmd.cc b/src/lmakeserver/cmd.cc index 1ea2dad4..ef365a93 100644 --- a/src/lmakeserver/cmd.cc +++ b/src/lmakeserver/cmd.cc @@ -467,7 +467,7 @@ R"({ OFStream(dir_guard(cmd_file )) << cmd ; ::chmod(cmd_file .c_str(),0755) ; // . OFStream(dir_guard(vscode_file)) << vscode ; // - audit( fd , ro , script_file , true/*as_is*/ ) ; + audit_file( fd , ::move(script_file) ) ; return true ; } @@ -509,8 +509,8 @@ R"({ Trace trace("show_job",ro.key,job) ; Rule rule = job->rule ; JobInfo job_info = job->job_info() ; - bool has_start = false ; - bool has_end = false ; + bool has_start = +job_info.start.start.proc ; + bool has_end = +job_info.end .end .proc ; bool verbose = ro.flags[ReqFlag::Verbose] ; JobDigest const& digest = job_info.end.end.digest ; switch (ro.key) { @@ -537,6 +537,7 @@ R"({ } else { JobRpcReq const& pre_start = job_info.start.pre_start ; JobRpcReply const& start = job_info.start.start ; + JobRpcReq const& end = job_info.end .end ; bool redirected = +start.stdin || +start.stdout ; // if (pre_start.job) SWEAR(pre_start.job==+job,pre_start.job,+job) ; @@ -544,7 +545,7 @@ R"({ switch (ro.key) { case ReqKey::Env : { if (!has_start) { audit( fd , ro , Color::Err , "no info available" , true/*as_is*/ , lvl ) ; break ; } - ::vmap_ss env = _mk_env(start.env,job_info.end.end.dynamic_env) ; + ::vmap_ss env = _mk_env(start.env,end.dynamic_env) ; size_t w = 0 ; for( auto const& [k,v] : env ) w = ::max(w,k.size()) ; for( auto const& [k,v] : env ) audit( fd , ro , to_string(::setw(w),k," : ",v) , true/*as_is*/ , lvl ) ; @@ -553,12 +554,12 @@ R"({ if (!has_start) audit( fd , ro , Color::Err , "no info available" , true , lvl ) ; else audit( fd , ro , _mk_script(job,ro.flags,job_info.start,job_info.end,ro.flag_args[+ReqFlag::Debug],false/*with_cmd*/) , true , lvl ) ; break ; - case ReqKey::Cmd : { //! as_is + case ReqKey::Cmd : //! as_is if (!has_start) audit( fd , ro , Color::Err , "no info available" , true , lvl ) ; else audit( fd , ro , _mk_cmd(job,ro.flags,start,{},redirected) , true , lvl ) ; - } break ; + break ; case ReqKey::Stdout : - if (!has_end ) { audit( fd , ro , Color::Err , "no info available" , true/*as_is*/ , lvl ) ; break ; } + if (!has_end) { audit( fd , ro , Color::Err , "no info available" , true/*as_is*/ , lvl ) ; break ; } _send_job( fd , ro , No/*show_deps*/ , false/*hide*/ , job , lvl ) ; audit ( fd , ro , digest.stdout , lvl+1 ) ; break ; @@ -568,9 +569,9 @@ R"({ if (has_start) { if (verbose) audit( fd , ro , Color::Note , pre_start.msg , false/*as_is*/ , lvl+1 ) ; } - if (has_end) { //! as_is - if (verbose) audit( fd , ro , Color::Note , job_info.end.end.msg , false , lvl+1 ) ; - /**/ audit( fd , ro , digest.stderr , true , lvl+1 ) ; + if (has_end) { //! as_is + if (verbose) audit( fd , ro , Color::Note , end.msg , false , lvl+1 ) ; + /**/ audit( fd , ro , digest.stderr , true , lvl+1 ) ; } break ; case ReqKey::Info : { @@ -608,34 +609,34 @@ R"({ else push_entry("required by",localize(mk_file( n ->name()),ro.startup_dir_s)) ; } if (has_start) { - JobInfoStart const& rs = job_info.start ; - SubmitAttrs const& sa = rs.submit_attrs ; - ::string cwd = rs.start.cwd_s.substr(0,rs.start.cwd_s.size()-1) ; - ::string tmp_dir = rs.start.autodep_env.tmp_dir ; - ::string pressure = sa.pressure.short_str() ; + JobInfoStart const& rs = job_info.start ; + SubmitAttrs const& sa = rs.submit_attrs ; + ::string cwd = start.cwd_s.substr(0,start.cwd_s.size()-1) ; + ::string tmp_dir = start.autodep_env.tmp_dir ; + ::string pressure = sa.pressure.short_str() ; // - if (!rs.start.keep_tmp) - for( auto const& [k,v] : rs.start.env ) + if (!start.keep_tmp) + for( auto const& [k,v] : start.env ) if (k=="TMPDIR") { tmp_dir = v==EnvPassMrkr ? "..." : v ; break ; } // - if (+sa.reason ) push_entry("reason",localize(reason_str(sa.reason),ro.startup_dir_s)) ; - if (rs.host!=NoSockAddr) push_entry("host" ,SockFd::s_host(rs.host) ) ; + if (+sa.reason ) push_entry( "reason" , localize(reason_str(sa.reason) , ro.startup_dir_s) ) ; + if (rs.host!=NoSockAddr) push_entry( "host" , SockFd::s_host(rs.host) ) ; // if (+rs.eta) { - if (porcelaine) push_entry("scheduling",to_string("( ",mk_py_str(rs.eta.str())," , ",double(sa.pressure) ," )"),Color::None,true/*as_is*/) ; - else push_entry("scheduling",to_string( rs.eta.str() ," - ", sa.pressure.short_str() ) ) ; + if (porcelaine) push_entry( "scheduling" , to_string("( ",mk_py_str(rs.eta.str())," , ",double(sa.pressure) ," )") , Color::None,true/*as_is*/ ) ; + else push_entry( "scheduling" , to_string( rs.eta.str() ," - ", sa.pressure.short_str() ) ) ; } // - if (+tmp_dir ) push_entry("tmp dir" ,localize(mk_file(tmp_dir),ro.startup_dir_s)) ; - if (+rs.start.autodep_env.tmp_view ) push_entry("tmp view" ,rs.start.autodep_env.tmp_view ) ; - if ( sa.live_out ) push_entry("live_out" ,"true" ) ; - if (+rs.start.chroot ) push_entry("chroot" ,rs.start.chroot ) ; - if (+rs.start.cwd_s ) push_entry("cwd" ,cwd ) ; - if ( rs.start.autodep_env.auto_mkdir ) push_entry("auto_mkdir" ,"true" ) ; - if ( rs.start.autodep_env.ignore_stat) push_entry("ignore_stat","true" ) ; - /**/ push_entry("autodep" ,snake_str(rs.start.method) ) ; - if (+rs.start.timeout ) push_entry("timeout" ,rs.start.timeout.short_str() ) ; - if (sa.tag!=BackendTag::Local ) push_entry("backend" ,snake_str(sa.tag) ) ; + if (+tmp_dir ) push_entry( "tmp dir" , localize(mk_file(tmp_dir),ro.startup_dir_s) ) ; + if (+start.autodep_env.tmp_view ) push_entry( "tmp view" , start.autodep_env.tmp_view ) ; + if ( sa.live_out ) push_entry( "live_out" , "true" ) ; + if (+start.chroot ) push_entry( "chroot" , start.chroot ) ; + if (+start.cwd_s ) push_entry( "cwd" , cwd ) ; + if ( start.autodep_env.auto_mkdir ) push_entry( "auto_mkdir" , "true" ) ; + if ( start.autodep_env.ignore_stat) push_entry( "ignore_stat" , "true" ) ; + /**/ push_entry( "autodep" , snake_str(start.method) ) ; + if (+start.timeout ) push_entry( "timeout" , start.timeout.short_str() ) ; + if (sa.tag!=BackendTag::Local ) push_entry( "backend" , snake_str(sa.tag) ) ; } // ::map_ss allocated_rsrcs = mk_map(job_info.start.rsrcs) ; @@ -646,29 +647,29 @@ R"({ } catch(::pair_ss const&) {} // if (has_end) { - push_entry( "end date" , digest.end_date.str() ) ; - push_entry( "rc" , wstatus_str(digest.wstatus) , WIFEXITED(digest.wstatus) && WEXITSTATUS(digest.wstatus)==0 ? Color::None : Color::Err ) ; - if (porcelaine) { - push_entry("cpu time" ,to_string(double(digest.stats.cpu )),Color::None,true/*as_is*/) ; - push_entry("elapsed in job",to_string(double(digest.stats.job )),Color::None,true/*as_is*/) ; - push_entry("elapsed total" ,to_string(double(digest.stats.total)),Color::None,true/*as_is*/) ; - push_entry("used mem" ,to_string( digest.stats.mem ),Color::None,true/*as_is*/) ; + push_entry( "end date" , digest.end_date.str() ) ; + push_entry( "rc" , wstatus_str(digest.wstatus) , WIFEXITED(digest.wstatus)&&WEXITSTATUS(digest.wstatus)==0?Color::None:Color::Err ) ; + if (porcelaine) { //! as_is + push_entry( "cpu time" , to_string(double(digest.stats.cpu )) , Color::None , true ) ; + push_entry( "elapsed in job" , to_string(double(digest.stats.job )) , Color::None , true ) ; + push_entry( "elapsed total" , to_string(double(digest.stats.total)) , Color::None , true ) ; + push_entry( "used mem" , to_string( digest.stats.mem ) , Color::None , true ) ; } else { ::string const& mem_rsrc_str = allocated_rsrcs.contains("mem") ? allocated_rsrcs.at("mem") : required_rsrcs.contains("mem") ? required_rsrcs.at("mem") : ""s ; size_t mem_rsrc = +mem_rsrc_str?from_string_with_units(mem_rsrc_str):0 ; bool overflow = digest.stats.mem > mem_rsrc ; ::string mem_str = to_string_with_units<'M'>(digest.stats.mem>>20)+'B' ; if ( overflow && mem_rsrc ) mem_str += " > "+mem_rsrc_str+'B' ; - push_entry("cpu time" ,digest.stats.cpu .short_str() ) ; - push_entry("elapsed in job",digest.stats.job .short_str() ) ; - push_entry("elapsed total" ,digest.stats.total.short_str() ) ; - push_entry("used mem" ,mem_str ,overflow?Color::Warning:Color::None) ; + push_entry( "cpu time" , digest.stats.cpu .short_str() ) ; + push_entry( "elapsed in job" , digest.stats.job .short_str() ) ; + push_entry( "elapsed total" , digest.stats.total.short_str() ) ; + push_entry( "used mem" , mem_str , overflow?Color::Warning:Color::None ) ; } } // - if (+job_info.start.pre_start.msg) push_entry("start message",localize(job_info.start.pre_start.msg,ro.startup_dir_s) ) ; - if (+job_info.start.stderr ) push_entry("start stderr" ,job_info.start.stderr ,Color::Warning) ; - if (+job_info.end.end.msg ) push_entry("message" ,localize(job_info.end.end .msg,ro.startup_dir_s) ) ; + if (+pre_start.msg ) push_entry( "start message" , localize(pre_start.msg,ro.startup_dir_s) ) ; + if (+job_info.start.stderr) push_entry( "start stderr" , job_info.start.stderr , Color::Warning ) ; + if (+end.msg ) push_entry( "message" , localize(end .msg,ro.startup_dir_s) ) ; // generate output if (porcelaine) { auto audit_rsrcs = [&]( ::string const& k , ::map_ss const& rsrcs , bool allocated )->void { @@ -689,15 +690,15 @@ R"({ audit( fd , ro , to_string(::setw(w),mk_py_str("job")," : ",mk_py_str(jn)) , false/*as_is*/ , lvl+1 , '{' ) ; for( auto const& [k,e] : tab ) audit( fd , ro , to_string(::setw(w),mk_py_str(k)," : ",e.as_is?e.txt:mk_py_str(e.txt)) , false/*as_is*/ , lvl+1 , ',' ) ; - if (+required_rsrcs ) audit_rsrcs("required resources" ,required_rsrcs ,false/*allocated*/) ; - if (+allocated_rsrcs) audit_rsrcs("allocated resources",allocated_rsrcs,true /*allocated*/) ; + if (+required_rsrcs ) audit_rsrcs( "required resources" , required_rsrcs , false/*allocated*/ ) ; + if (+allocated_rsrcs) audit_rsrcs( "allocated resources" , allocated_rsrcs , true /*allocated*/ ) ; audit( fd , ro , "}" , true/*as_is*/ , lvl ) ; } else { size_t w = 0 ; for( auto const& [k,e] : tab ) if (e.txt.find('\n')==Npos) w = ::max(w,k.size()) ; _send_job( fd , ro , No/*show_deps*/ , false/*hide*/ , job , lvl ) ; - for( auto const& [k,e] : tab ) - if (e.txt.find('\n')==Npos) audit( fd , ro , e.color , to_string(::setw(w),k," : ",e.txt) , false/*as_is*/ , lvl+1 ) ; - else { audit( fd , ro , e.color , to_string( k," :" ) , false/*as_is*/ , lvl+1 ) ; audit(fd,ro,e.txt,true/*as_is*/,lvl+2) ; } + for( auto const& [k,e] : tab ) //! as_is + if (e.txt.find('\n')==Npos) audit( fd , ro , e.color , to_string(::setw(w),k," : ",e.txt) , false , lvl+1 ) ; + else { audit( fd , ro , e.color , to_string( k," :" ) , false , lvl+1 ) ; audit(fd,ro,e.txt,true/*as_is*/,lvl+2) ; } if ( +required_rsrcs || +allocated_rsrcs ) { size_t w2 = 0 ; for( auto const& [k,_] : required_rsrcs ) w2 = ::max(w2,k.size()) ; @@ -705,20 +706,20 @@ R"({ ::string hdr = "resources :" ; if (!+allocated_rsrcs) hdr = "required " +hdr ; else if (!+required_rsrcs ) hdr = "allocated "+hdr ; - audit( fd , ro , hdr , true/*as_is*/ , lvl+1 ) ; - if (!required_rsrcs ) for( auto const& [k,v] : allocated_rsrcs ) audit( fd , ro , to_string(::setw(w2),k," : ",v) , true/*as_is*/ , lvl+2 ) ; - else if (!allocated_rsrcs ) for( auto const& [k,v] : required_rsrcs ) audit( fd , ro , to_string(::setw(w2),k," : ",v) , true/*as_is*/ , lvl+2 ) ; - else if (required_rsrcs==allocated_rsrcs) for( auto const& [k,v] : required_rsrcs ) audit( fd , ro , to_string(::setw(w2),k," : ",v) , true/*as_is*/ , lvl+2 ) ; + audit( fd , ro , hdr , true/*as_is*/ , lvl+1 ) ; //! as_is + if (!required_rsrcs ) for( auto const& [k,v] : allocated_rsrcs ) audit( fd , ro , to_string(::setw(w2),k," : ",v) , true , lvl+2 ) ; + else if (!allocated_rsrcs ) for( auto const& [k,v] : required_rsrcs ) audit( fd , ro , to_string(::setw(w2),k," : ",v) , true , lvl+2 ) ; + else if (required_rsrcs==allocated_rsrcs) for( auto const& [k,v] : required_rsrcs ) audit( fd , ro , to_string(::setw(w2),k," : ",v) , true , lvl+2 ) ; else { - for( auto const& [k,rv] : required_rsrcs ) { - if (!allocated_rsrcs.contains(k)) { audit( fd , ro , to_string(::setw(w2),k,"(required )"," : ",rv) , true/*as_is*/ , lvl+2 ) ; continue ; } + for( auto const& [k,rv] : required_rsrcs ) { //! as_is + if (!allocated_rsrcs.contains(k)) { audit( fd , ro , to_string(::setw(w2),k,"(required )"," : ",rv) , true , lvl+2 ) ; continue ; } ::string const& av = allocated_rsrcs.at(k) ; - if (rv==av ) { audit( fd , ro , to_string(::setw(w2),k," "," : ",rv) , true/*as_is*/ , lvl+2 ) ; continue ; } - /**/ audit( fd , ro , to_string(::setw(w2),k,"(required )"," : ",rv) , true/*as_is*/ , lvl+2 ) ; - /**/ audit( fd , ro , to_string(::setw(w2),k,"(allocated)"," : ",av) , true/*as_is*/ , lvl+2 ) ; + if (rv==av ) { audit( fd , ro , to_string(::setw(w2),k," "," : ",rv) , true , lvl+2 ) ; continue ; } + /**/ audit( fd , ro , to_string(::setw(w2),k,"(required )"," : ",rv) , true , lvl+2 ) ; + /**/ audit( fd , ro , to_string(::setw(w2),k,"(allocated)"," : ",av) , true , lvl+2 ) ; } for( auto const& [k,av] : allocated_rsrcs ) - if (!required_rsrcs.contains(k)) audit( fd , ro , to_string(::setw(w2),k,"(allocated)"," : ",av) , true/*as_is*/ , lvl+2 ) ; + if (!required_rsrcs.contains(k)) audit( fd , ro , to_string(::setw(w2),k,"(allocated)"," : ",av) , true , lvl+2 ) ; } } } diff --git a/src/lmakeserver/global.cc b/src/lmakeserver/global.cc index 1a082530..a675a33c 100644 --- a/src/lmakeserver/global.cc +++ b/src/lmakeserver/global.cc @@ -24,7 +24,7 @@ namespace Engine { return res ; } - void audit( Fd out_fd, ::ostream& log , ReqOptions const& ro , Color c , ::string const& txt , bool as_is , DepDepth lvl , char sep ) { + void _audit( Fd out_fd , ::ostream* log , ReqOptions const& ro , Color c , ::string const& txt , bool as_is , DepDepth lvl , char sep ) { if (!txt) return ; // ::string report_txt = color_pfx(ro,c) ; @@ -33,10 +33,34 @@ namespace Engine { /**/ report_txt += color_sfx(ro,c) ; /**/ report_txt += '\n' ; // - // if we lose connection, there is nothing much we can do about it (hoping that we can still trace) - /**/ try { OMsgBuf().send( out_fd , ReqRpcReply(_audit_indent(::move(report_txt) ,lvl,sep)) ) ; } catch (::string const& e) { Trace("audit","lost_client",e) ; } - if (as_is) try { log << _audit_indent(ensure_nl( txt ),lvl,sep) << ::flush ; } catch (::string const& e) { Trace("audit","lost_log" ,e) ; } - else try { log << _audit_indent(ensure_nl(localize(txt,{})),lvl,sep) << ::flush ; } catch (::string const& e) { Trace("audit","lost_log" ,e) ; } + using Proc = ReqRpcReplyProc ; + try { OMsgBuf().send( out_fd , ReqRpcReply(Proc::Txt,_audit_indent(::move(report_txt),lvl,sep)) ) ; } // if we lose connection, there is nothing much we can do ... + catch (::string const& e) { Trace("audit","lost_client",e) ; } // ... about it (hoping that we can still trace) + if (log) + try { *log << _audit_indent(ensure_nl(as_is?txt:localize(txt,{})),lvl,sep) << ::flush ; } // . + catch (::string const& e) { Trace("audit","lost_log",e) ; } + } + + void audit_file( Fd out_fd , ::string&& file ) { + using Proc = ReqRpcReplyProc ; + try { OMsgBuf().send( out_fd , ReqRpcReply(Proc::File,::move(file)) ) ; } // if we lose connection, there is nothing much we can do ... + catch (::string const& e) { Trace("audit_file","lost_client",e) ; } // ... about it (hoping that we can still trace) + } + + void _audit_status( Fd out_fd , ::ostream* log , ReqOptions const& , bool ok ) { + using Proc = ReqRpcReplyProc ; + try { OMsgBuf().send( out_fd , ReqRpcReply(Proc::Status,ok) ) ; } // if we lose connection, there is nothing much we can do ... + catch (::string const& e) { Trace("audit_status","lost_client",e) ; } // ... about it (hoping that we can still trace) + if (log) + try { *log << "status : " << (ok?"ok":"failed") <<'\n'<< ::flush ; } // . + catch (::string const& e) { Trace("audit_status","lost_log",e) ; } + } + + void _audit_ctrl_c( Fd , ::ostream* log , ReqOptions const& ) { + // lmake echos a \n as soon as it sees ^C (and it does that much faster than we could), no need to do it here + if (log) + try { *log << "^C\n"<< ::flush ; } + catch (::string const& e) { Trace("audit_ctrl_c","lost_log",e) ; } } // @@ -95,8 +119,8 @@ namespace Engine { } } - Config::Config(Dict const& py_map) : booted{true} { // if config is read from makefiles, it is booted - db_version = Version::Db ; // record current version + Config::Config(Dict const& py_map) : booted{true} { // if config is read from makefiles, it is booted + db_version = Version::Db ; // record current version // generate a random key char buf_char[8] ; IFStream("/dev/urandom").read(buf_char,sizeof(buf_char)) ; uint64_t buf_int ; ::memcpy( &buf_int , buf_char , sizeof(buf_int) ) ; diff --git a/src/lmakeserver/global.x.hh b/src/lmakeserver/global.x.hh index 96ddffa5..ac5ab47a 100644 --- a/src/lmakeserver/global.x.hh +++ b/src/lmakeserver/global.x.hh @@ -221,14 +221,21 @@ namespace Engine { } ; // sep is put before the last indent level, useful for porcelaine output - #define ROC ReqOptions const - #define SC ::string const - /**/ void audit( Fd out , ::ostream& log , ROC& , Color , SC& , bool as_is=false , DepDepth =0 , char sep=0 ) ; - inline void audit( Fd out , ROC& ro , Color c , SC& t , bool a =false , DepDepth l=0 , char sep=0 ) { static OFakeStream fs ; audit(out,fs ,ro,c ,t,a,l,sep) ; } - inline void audit( Fd out , ::ostream& log , ROC& ro , SC& t , bool a =false , DepDepth l=0 , char sep=0 ) { audit(out,log,ro,Color::None,t,a,l,sep) ; } - inline void audit( Fd out , ROC& ro , SC& t , bool a =false , DepDepth l=0 , char sep=0 ) { static OFakeStream fs ; audit(out,fs ,ro,Color::None,t,a,l,sep) ; } - #undef SC - #undef ROC + /**/ void _audit( Fd out , ::ostream* log , ReqOptions const& , Color , ::string const& , bool as_is , DepDepth , char sep ) ; + inline void audit ( Fd out , ::ostream& log , ReqOptions const& ro , Color c , ::string const& t , bool a=false , DepDepth l=0 , char sep=0 ) { _audit(out,&log ,ro,c ,t,a,l,sep) ; } + inline void audit ( Fd out , ReqOptions const& ro , Color c , ::string const& t , bool a=false , DepDepth l=0 , char sep=0 ) { _audit(out,nullptr,ro,c ,t,a,l,sep) ; } + inline void audit ( Fd out , ::ostream& log , ReqOptions const& ro , ::string const& t , bool a=false , DepDepth l=0 , char sep=0 ) { _audit(out,&log ,ro,Color::None,t,a,l,sep) ; } + inline void audit ( Fd out , ReqOptions const& ro , ::string const& t , bool a=false , DepDepth l=0 , char sep=0 ) { _audit(out,nullptr,ro,Color::None,t,a,l,sep) ; } + // + /**/ void audit_file( Fd out , ::string&& f ) ; + // + /**/ void _audit_status( Fd out , ::ostream* log , ReqOptions const& , bool ) ; + inline void audit_status ( Fd out , ::ostream& log , ReqOptions const& ro , bool ok ) { _audit_status(out,&log ,ro,ok) ; } + inline void audit_status ( Fd out , ReqOptions const& ro , bool ok ) { _audit_status(out,nullptr,ro,ok) ; } + // + /**/ void _audit_ctrl_c( Fd out , ::ostream* log , ReqOptions const& ) ; + inline void audit_ctrl_c ( Fd out , ::ostream& log , ReqOptions const& ro ) { _audit_ctrl_c(out,&log ,ro) ; } + inline void audit_ctrl_c ( Fd out , ReqOptions const& ro ) { _audit_ctrl_c(out,nullptr,ro) ; } template ::string title ( ReqOptions const& , A&&... ) ; inline ::string color_pfx( ReqOptions const& , Color ) ; diff --git a/src/lmakeserver/job.x.hh b/src/lmakeserver/job.x.hh index 6d0c6241..a71278dc 100644 --- a/src/lmakeserver/job.x.hh +++ b/src/lmakeserver/job.x.hh @@ -419,7 +419,7 @@ namespace Engine { inline JobData::JobData (JobData&& jd) : JobData(jd) { jd.targets.forget() ; jd.deps.forget() ; } inline JobData::~JobData ( ) { targets.pop () ; deps.pop () ; } - inline JobData& JobData::operator=(JobData&& jd) { SWEAR(rule==jd.rule,rule,jd.rule) ; *this = jd ; jd.targets.forget() ; jd.deps.forget() ; return *this ; } + inline JobData& JobData::operator=(JobData&& jd) { SWEAR(rule==jd.rule,rule,jd.rule) ; *this = jd ; jd.targets.forget() ; jd.deps.forget() ; return *this ; } // XXX : use copy&swap idiom inline ::string JobData::special_stderr ( ) const { return special_stderr ( {}) ; } inline void JobData::audit_end_special( Req r , SpecialStep s , Bool3 m ) const { return audit_end_special(r,s,m,{}) ; } diff --git a/src/lmakeserver/main.cc b/src/lmakeserver/main.cc index f1e255b8..107adf01 100644 --- a/src/lmakeserver/main.cc +++ b/src/lmakeserver/main.cc @@ -199,16 +199,17 @@ void reqs_thread_func( ::stop_token stop , Fd in_fd , Fd out_fd ) { case ReqProc::Forget : case ReqProc::Mark : case ReqProc::Show : + epoll.del(fd) ; // must precede close(fd) which may occur as soon as we push to g_engine_queue + in_tab.erase(fd) ; //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv g_engine_queue.emplace( rrr.proc , fd , ofd , rrr.files , rrr.options ) ; //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ break ; - case ReqProc::Kill : - case ReqProc::Close : - case ReqProc::None : { - epoll.del(fd) ; // must precede close(fd) - auto it = in_tab.find(fd) ; - Req r = it->second.second ; + case ReqProc::Kill : + case ReqProc::None : { + epoll.del(fd) ; // must precede close(fd) which may occur as soon as we push to g_engine_queue + auto it=in_tab.find(fd) ; + Req r = it->second.second ; trace("eof",fd) ; if (+r) { trace("zombie",r) ; r.zombie(true) ; } // make req zombie immediately to optimize reaction time //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv @@ -288,7 +289,7 @@ bool/*interrupted*/ engine_loop() { audit( req.out_fd , req.options , Color::Note , "startup dir : "+startup_dir_s.substr(0,startup_dir_s.size()-1) , true/*as_is*/ ) ; try { ok = g_cmd_tab[+req.proc](req) ; } catch (::string const& e) { ok = false ; if (+e) audit(req.out_fd,req.options,Color::Err,e) ; } - OMsgBuf().send( req.out_fd , ReqRpcReply(ok) ) ; + OMsgBuf().send( req.out_fd , ReqRpcReply(ReqRpcReplyProc::Status,ok) ) ; /**/ req.in_fd .close() ; if (req.out_fd!=req.in_fd) req.out_fd.close() ; } break ; @@ -310,8 +311,8 @@ bool/*interrupted*/ engine_loop() { fd_tab[req.req] = { .in=req.in_fd , .out=req.out_fd } ; break ; } catch(::string const& e) { - audit( req.out_fd , req.options , Color::Err , e ) ; - OMsgBuf().send( req.out_fd , ReqRpcReply(false/*ok*/) ) ; + audit ( req.out_fd , req.options , Color::Err , e ) ; + audit_status( req.out_fd , req.options , false/*ok*/ ) ; // cannot make, process as if followed by Close trace("no_make") ; } diff --git a/src/lmakeserver/req.cc b/src/lmakeserver/req.cc index 052d38ef..d349630a 100644 --- a/src/lmakeserver/req.cc +++ b/src/lmakeserver/req.cc @@ -85,6 +85,7 @@ namespace Engine { void Req::kill() { Trace trace("kill",*this) ; SWEAR(zombie()) ; // zombie has already been set + audit_ctrl_c( (*this)->audit_fd , (*this)->log_stream , (*this)->options ) ; Backend::s_kill_req(+*this) ; } @@ -453,11 +454,8 @@ namespace Engine { last_info = {} ; } - void ReqData::audit_status(bool ok) const { - try { OMsgBuf().send( audit_fd , ReqRpcReply(ok) ) ; } - catch (::string const&) { } // if client has disappeared, well, we cannot do much - log_stream << "status : " << (ok?"ok":"failed") << '\n' ; - } + static void _audit_status( Fd out_fd, ::ostream& log , ReqOptions const& ro , bool ok ) { audit_status (out_fd ,log ,ro ,ok) ; } // allow access to global function ... + /**/ void ReqData::audit_status ( bool ok ) const { _audit_status(audit_fd,log_stream,options,ok) ; } // ... w/o naming namespace bool/*seen*/ ReqData::audit_stderr( ::string const& msg , ::string const& stderr , size_t max_stderr_lines , DepDepth lvl ) const { if (+msg ) audit_info( Color::Note , msg , lvl ) ; @@ -476,16 +474,19 @@ namespace Engine { void ReqData::audit_stats() const { try { - ReqRpcReply rrr{ title( - options - , stats.ended(JobReport::Failed)==0 ? ""s : to_string( "failed:" , stats.ended(JobReport::Failed),' ') - , "done:" , stats.ended(JobReport::Done )+stats.ended(JobReport::Steady) - , !g_config.caches || !stats.ended(JobReport::Hit) ? ""s : to_string(" hit:" , stats.ended(JobReport::Hit )) - , stats.ended(JobReport::Rerun )==0 ? ""s : to_string(" rerun:" , stats.ended(JobReport::Rerun )) - , " running:" , stats.cur (JobStep ::Exec ) - , stats.cur (JobStep ::Queued)==0 ? ""s : to_string(" queued:" , stats.cur (JobStep ::Queued)) - , stats.cur (JobStep ::Dep )==0 ? ""s : to_string(" waiting:" , stats.cur (JobStep ::Dep )) - ) } ; + ReqRpcReply rrr{ + ReqRpcReplyProc::Txt + , title( + options + , stats.ended(JobReport::Failed)==0 ? ""s : to_string( "failed:" , stats.ended(JobReport::Failed),' ') + , "done:" , stats.ended(JobReport::Done )+stats.ended(JobReport::Steady) + , !g_config.caches || !stats.ended(JobReport::Hit) ? ""s : to_string(" hit:" , stats.ended(JobReport::Hit )) + , stats.ended(JobReport::Rerun )==0 ? ""s : to_string(" rerun:" , stats.ended(JobReport::Rerun )) + , " running:" , stats.cur (JobStep ::Exec ) + , stats.cur (JobStep ::Queued)==0 ? ""s : to_string(" queued:" , stats.cur (JobStep ::Queued)) + , stats.cur (JobStep ::Dep )==0 ? ""s : to_string(" waiting:" , stats.cur (JobStep ::Dep )) + ) + } ; OMsgBuf().send( audit_fd , rrr ) ; } catch (::string const&) {} // if client has disappeared, well, we cannot do much } diff --git a/src/lmark.cc b/src/lmark.cc index 324e3f9d..cfa3e999 100644 --- a/src/lmark.cc +++ b/src/lmark.cc @@ -30,8 +30,8 @@ int main( int argc , char* argv[] ) { if ( cmd_line.flags[ReqFlag::Freeze] + cmd_line.flags[ReqFlag::NoTrigger] !=1 ) syntax.usage("can only process a single attribute : freeze or no-trigger") ; // - // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - Bool3 ok = out_proc( ::cout , ReqProc::Mark , true/*refresh_makefiles*/ , syntax , cmd_line ) ; - // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + Bool3 ok = out_proc( ReqProc::Mark , true/*refresh_makefiles*/ , syntax , cmd_line ) ; + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exit(mk_rc(ok)) ; } diff --git a/src/lshow.cc b/src/lshow.cc index 3685a914..ae8d5a84 100644 --- a/src/lshow.cc +++ b/src/lshow.cc @@ -39,8 +39,8 @@ int main( int argc , char* argv[] ) { if ( cmd_line.flags[ReqFlag::Job ] && cmd_line.key==ReqKey::InvDeps ) syntax.usage("dependents cannot be shown for jobs" ) ; if ( cmd_line.flags[ReqFlag::Job ] && cmd_line.key==ReqKey::InvTargets ) syntax.usage("producing jobs cannot be shown for jobs" ) ; if ( cmd_line.flags[ReqFlag::Porcelaine] && cmd_line.key!=ReqKey::Info ) syntax.usage("porcelaine output is only valid with --info" ) ; - // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - Bool3 ok = out_proc( ::cout , ReqProc::Show , false/*refresh_makefiles*/ , syntax , cmd_line ) ; - // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + Bool3 ok = out_proc( ReqProc::Show , false/*refresh_makefiles*/ , syntax , cmd_line ) ; + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ exit(mk_rc(ok)) ; } diff --git a/src/py.hh b/src/py.hh index 6657b19a..4a1df482 100644 --- a/src/py.hh +++ b/src/py.hh @@ -134,7 +134,7 @@ namespace Py { // ~Ptr() { unboost() ; } // - Ptr& operator=(Object* o) { unboost() ; ptr = o ; boost() ; return *this ; } + Ptr& operator=(Object* o) { unboost() ; ptr = o ; boost() ; return *this ; } // XXX : use copy&swap idiom Ptr& operator=(Ptr const& p) { unboost() ; ptr = p.ptr ; boost() ; return *this ; } Ptr& operator=(Ptr && p) { unboost() ; ptr = p.ptr ; p.detach() ; return *this ; } // accesses diff --git a/src/rpc_client.cc b/src/rpc_client.cc index 0943c5b8..0e24f16b 100644 --- a/src/rpc_client.cc +++ b/src/rpc_client.cc @@ -23,11 +23,13 @@ ::ostream& operator<<( ::ostream& os , ReqRpcReq const& rrr ) { } ::ostream& operator<<( ::ostream& os , ReqRpcReply const& rrr ) { - os << "ReqRpcReply("< ; +ENUM( ReqRpcReplyProc +, None +, File +, Status +, Txt +) + struct ReqSyntax : Syntax { ReqSyntax() = default ; ReqSyntax( ::umap const& fs ) : ReqSyntax{{},fs} {} @@ -83,6 +91,7 @@ struct ReqSyntax : Syntax { // add standard options flags[+ReqFlag::Quiet] = { .short_name='q' , .has_arg=false , .doc="do not generate user oriented messages" } ; flags[+ReqFlag::Job ] = { .short_name='J' , .has_arg=false , .doc="interpret (unique) arg as a job name" } ; + flags[+ReqFlag::Sync ] = { .short_name='S' , .has_arg=false , .doc="synchronous : start server and wait for its end" } ; flags[+ReqFlag::Rule ] = { .short_name='R' , .has_arg=true , .doc="force rule when interpreting arg as job" } ; flags[+ReqFlag::Video] = { .short_name='V' , .has_arg=true , .doc="assume output video : n(ormal), r(everse) or f(ile)" } ; } @@ -149,31 +158,26 @@ struct ReqRpcReq { ReqOptions options ; } ; -ENUM( ReqKind -, None -, Txt -, Status -) - struct ReqRpcReply { friend ::ostream& operator<<( ::ostream& , ReqRpcReply const& ) ; - using Kind = ReqKind ; + using Proc = ReqRpcReplyProc ; // cxtors & casts - ReqRpcReply( ) = default ; - ReqRpcReply(bool ok_ ) : kind{Kind::Status} , ok {ok_ } {} - ReqRpcReply(::string&& txt_) : kind{Kind::Txt } , txt{::move(txt_)} {} + ReqRpcReply( ) = default ; + ReqRpcReply( Proc p , bool ok_ ) : proc{p} , ok {ok_ } { SWEAR( p==Proc::Status ) ; } + ReqRpcReply( Proc p , ::string&& txt_ ) : proc{p} , txt{::move(txt_)} { SWEAR( p==Proc::File || p==Proc::Txt ) ; } // template void serdes(T& s) { - if (::is_base_of_v<::istream,T>) *this = ReqRpcReply() ; - ::serdes(s,kind) ; - switch (kind) { - case Kind::None : break ; - case Kind::Status : ::serdes(s,ok ) ; break ; - case Kind::Txt : ::serdes(s,txt) ; break ; + if (::is_base_of_v<::istream,T>) *this = {} ; + ::serdes(s,proc) ; + switch (proc) { + case Proc::None : break ; + case Proc::Status : ::serdes(s,ok ) ; break ; + case Proc::File : + case Proc::Txt : ::serdes(s,txt) ; break ; DF} } // data - Kind kind = Kind::None ; + Proc proc = Proc::None ; bool ok = false ; ::string txt ; } ; diff --git a/src/rpc_job.hh b/src/rpc_job.hh index fda2ae32..6636b05f 100644 --- a/src/rpc_job.hh +++ b/src/rpc_job.hh @@ -358,7 +358,7 @@ struct CrcDate { using Crc = Hash::Crc ; using Ddate = Time::Ddate ; //cxtors & casts - /**/ CrcDate( ) : _crc{} { } + /**/ CrcDate( ) : _crc{} { } // XXX : implement cxtors and just provide copy assignment /**/ CrcDate(Crc c ) { *this = c ; } /**/ CrcDate(Ddate d ) { *this = d ; } /**/ CrcDate(CrcDate const& cd ) { *this = cd ; } @@ -556,7 +556,7 @@ struct JobRpcReq { JobRpcReq( SI si , JI j , JobExecRpcReq&& jerr ) ; // services template void serdes(T& s) { - if (::is_base_of_v<::istream,T>) *this = JobRpcReq() ; + if (::is_base_of_v<::istream,T>) *this = {} ; ::serdes(s,proc ) ; ::serdes(s,seq_id) ; ::serdes(s,job ) ; @@ -641,7 +641,7 @@ struct JobRpcReply { JobRpcReply( Proc p , ::string const& t , Crc c , Bool3 o ) : proc{p} , ok{o} , txt{t} , crc{c} { SWEAR( proc==Proc::Decode || proc==Proc::Encode ) ; } // services template void serdes(S& s) { - if (is_base_of_v<::istream,S>) *this = JobRpcReply() ; + if (is_base_of_v<::istream,S>) *this = {} ; ::serdes(s,proc) ; switch (proc) { case Proc::None : @@ -864,7 +864,7 @@ struct JobExecRpcReply { JobExecRpcReply( JobRpcReply const& jrr ) ; // services template void serdes(S& s) { - if (::is_base_of_v<::istream,S>) *this = JobExecRpcReply() ; + if (::is_base_of_v<::istream,S>) *this = {} ; ::serdes(s,proc) ; switch (proc) { case Proc::Access : break ; diff --git a/src/time.hh b/src/time.hh index eb077f7e..315b9615 100644 --- a/src/time.hh +++ b/src/time.hh @@ -99,10 +99,10 @@ namespace Time { constexpr ::strong_ordering operator<=>(Delay const& other) const { return _val<=>other._val ; } // using Base::operator+ ; - constexpr Delay operator+ (Delay other) const { return Delay(New,_val+other._val) ; } - constexpr Delay operator- (Delay other) const { return Delay(New,_val-other._val) ; } - constexpr Delay& operator+=(Delay other) { *this = *this + other ; return *this ; } - constexpr Delay& operator-=(Delay other) { *this = *this - other ; return *this ; } + constexpr Delay operator+ (Delay other) const { return Delay(New,_val+other._val) ; } + constexpr Delay operator- (Delay other) const { return Delay(New,_val-other._val) ; } + constexpr Delay& operator+=(Delay other) { *this = *this+other ; return *this ; } + constexpr Delay& operator-=(Delay other) { *this = *this-other ; return *this ; } constexpr Date operator+ (Date ) const ; // template requires(::is_arithmetic_v) constexpr Delay operator* (T f) const ; @@ -139,7 +139,7 @@ namespace Time { explicit constexpr CoarseDelay( NewType , Val v ) : _val{v} {} public : constexpr CoarseDelay() = default ; - constexpr CoarseDelay(Delay d) { *this = d ; } + constexpr CoarseDelay(Delay d) { *this = d ; } // XXX : implemment cxtor and remove operator= constexpr CoarseDelay& operator=(Delay d) { uint32_t t = ::logf(d._val)*(1<= (1<)+Scale ) _val = -1 ; @@ -184,10 +184,10 @@ namespace Time { Date(::string_view const&) ; // read a reasonable approximation of ISO8601 // services using Base::operator+ ; - constexpr Date operator+ (Delay other) const { return Date(New,_val+other._val) ; } - constexpr Date operator- (Delay other) const { return Date(New,_val-other._val) ; } - constexpr Date& operator+=(Delay other) { *this = *this + other ; return *this ; } - constexpr Date& operator-=(Delay other) { *this = *this - other ; return *this ; } + constexpr Date operator+ (Delay other) const { return Date(New,_val+other._val) ; } + constexpr Date operator- (Delay other) const { return Date(New,_val-other._val) ; } + constexpr Date& operator+=(Delay other) { *this = *this+other ; return *this ; } + constexpr Date& operator-=(Delay other) { *this = *this-other ; return *this ; } // ::string str ( uint8_t prec=0 , bool in_day=false ) const ; size_t hash( ) const { return _val ; } @@ -216,10 +216,10 @@ namespace Time { constexpr ::strong_ordering operator<=>(Pdate const& other) const { return _val<=>other._val ; } // using Base::operator+ ; - constexpr Pdate operator+ (Delay other) const { return Pdate(New,_val+other._val) ; } - constexpr Pdate operator- (Delay other) const { return Pdate(New,_val-other._val) ; } - constexpr Pdate& operator+=(Delay other) { *this = *this + other ; return *this ; } - constexpr Pdate& operator-=(Delay other) { *this = *this - other ; return *this ; } + constexpr Pdate operator+ (Delay other) const { return Pdate(New,_val+other._val) ; } + constexpr Pdate operator- (Delay other) const { return Pdate(New,_val-other._val) ; } + constexpr Pdate& operator+=(Delay other) { *this = *this+other ; return *this ; } + constexpr Pdate& operator-=(Delay other) { *this = *this-other ; return *this ; } constexpr Delay operator- (Pdate ) const ; // bool/*slept*/ sleep_until(::stop_token) const ; diff --git a/src/trace.cc b/src/trace.cc index c41adfb0..22c34040 100644 --- a/src/trace.cc +++ b/src/trace.cc @@ -19,10 +19,11 @@ Channels Trace::s_channels = DfltChannels ; // by default, trace de #ifndef NO_TRACE - size_t Trace::_s_pos = 0 ; - bool Trace::_s_ping = false ; - Fd Trace::_s_fd ; - Mutex Trace::_s_mutex ; + size_t Trace::_s_pos = 0 ; + bool Trace::_s_ping = false ; + Fd Trace::_s_fd ; + ::atomic Trace::_s_has_fd = false ; + Mutex Trace::_s_mutex ; thread_local int Trace::_t_lvl = 0 ; thread_local bool Trace::_t_hide = false ; @@ -41,6 +42,8 @@ Channels Trace::s_channels = DfltChannels ; // by default, trace de // Lock lock{_s_mutex} ; // + _s_has_fd = false ; + fence() ; _s_fd.close() ; *g_trace_file = trace_file ; _s_open() ; @@ -57,8 +60,10 @@ Channels Trace::s_channels = DfltChannels ; // by default, trace de unlnk(*g_trace_file) ; // avoid write clashes if trace is still being written by another process Fd fd = open_write(*g_trace_file) ; fd.no_std() ; - _s_pos = 0 ; - _s_fd = fd ; // ensure _s_fd is updated once everything is ok as tracing may be called from other threads while being initialized + _s_pos = 0 ; + _s_fd = fd ; // ensure _s_fd is updated once everything is ok as tracing may be called from other threads while being initialized + fence() ; + _s_has_fd = +fd ; } #endif diff --git a/src/trace.hh b/src/trace.hh index 9956b82a..a9ff06e5 100644 --- a/src/trace.hh +++ b/src/trace.hh @@ -60,10 +60,11 @@ static constexpr Channels DfltChannels = ~Channels() ; static ::atomic s_sz ; // max overall size of trace, beyond, trace wraps static Channels s_channels ; private : - static size_t _s_pos ; // current line number - static bool _s_ping ; // ping-pong to distinguish where trace stops in the middle of a trace - static Fd _s_fd ; - static Mutex _s_mutex ; + static size_t _s_pos ; // current line number + static bool _s_ping ; // ping-pong to distinguish where trace stops in the middle of a trace + static Fd _s_fd ; + static ::atomic _s_has_fd ; + static Mutex _s_mutex ; // static thread_local int _t_lvl ; static thread_local bool _t_hide ; // if true <=> do not generate trace @@ -79,9 +80,9 @@ static constexpr Channels DfltChannels = ~Channels() ; /**/ Trace( ) : Trace{Channel::Default } {} template Trace( const char* tag , Ts const&... args ) : Trace{Channel::Default,tag,args...} {} // services - /**/ void hide (bool h=true ) { _t_hide = h ; } - template void operator()(Ts const&... args) { if ( +_s_fd && _active && !_sav_hide.saved ) _record(args...) ; } - template void protect (Ts const&... args) { if ( +_s_fd && _active && !_sav_hide.saved ) _record(args...) ; } + /**/ void hide (bool h=true ) { _t_hide = h ; } + template void operator()(Ts const&... args) { if ( _s_has_fd && _active && !_sav_hide.saved ) _record(args...) ; } + template void protect (Ts const&... args) { if ( _s_has_fd && _active && !_sav_hide.saved ) _record(args...) ; } private : template void _record(Ts const&... ) ; template void _output(T const& x) { *_t_buf << x ; } diff --git a/src/utils.hh b/src/utils.hh index b1d4fd10..c1c8ba0b 100644 --- a/src/utils.hh +++ b/src/utils.hh @@ -506,12 +506,16 @@ template constexpr void swear_prod( bool cond , A const&... args ) { #define DF default : FAIL() ; // for use at end of switch statements inline bool/*done*/ kill_process( pid_t pid , int sig , bool as_group=false ) { - swear_prod(pid>1,"killing process ",pid) ; // ensure no system wide catastrophe ! - bool proc_killed = ::kill( pid,sig)==0 ; // kill process before process group as maybe, setpgid(0,0) has not been called in the child yet - bool group_killed = as_group && ::kill(-pid,sig)==0 ; // kill group if asked so, whether proc was killed or not + swear_prod(pid>1,"killing process ",pid) ; // /!\ ::kill(-1) sends signal to all possible processes, ensure no system wide catastrophe + // + if (!as_group ) return ::kill(pid,sig)==0 ; + if (::kill(-pid,sig)==0) return true ; // fast path : group exists, nothing else to do + bool proc_killed = ::kill( pid,sig)==0 ; // else, there may be another possibility : the process to kill might not have had enough time to call setpgid(0,0) ... + bool group_killed = ::kill(-pid,sig)==0 ; // ... that makes it be a group, so kill it as a process, and kill the group again in case it was created inbetween return proc_killed || group_killed ; } -inline bool/*done*/ kill_self(int sig) { // raise kills the thread, not the process + +inline bool/*done*/ kill_self(int sig) { // raise kills the thread, not the process return kill_process(::getpid(),sig) ; } @@ -568,7 +572,7 @@ template struct vector_view { ViewC subvec( size_t start , size_t sz=Npos ) const requires(!IsConst) { return ViewC( begin()+start , ::min(sz,_sz-start) ) ; } View subvec( size_t start , size_t sz=Npos ) requires(!IsConst) { return View ( begin()+start , ::min(sz,_sz-start) ) ; } // - void clear() { *this = vector_view() ; } + void clear() { *this = {} ; } // data protected : T* _data = nullptr ; @@ -815,15 +819,15 @@ template struct BitMap { constexpr bool operator!() const { return !_val ; } // services constexpr bool operator==( BitMap const& ) const = default ; - constexpr bool operator<=( BitMap other ) const { return !( _val & ~other._val ) ; } - constexpr bool operator>=( BitMap other ) const { return !( ~_val & other._val ) ; } - constexpr BitMap operator~ ( ) const { return BitMap(lsb_msk(N)&~_val) ; } - constexpr BitMap operator& ( BitMap other ) const { return BitMap(_val&other._val) ; } - constexpr BitMap operator| ( BitMap other ) const { return BitMap(_val|other._val) ; } - constexpr BitMap& operator&=( BitMap other ) { *this = *this & other ; return *this ; } - constexpr BitMap& operator|=( BitMap other ) { *this = *this | other ; return *this ; } - constexpr bool operator[]( E bit_ ) const { return bit(_val,+bit_) ; } - constexpr uint8_t popcount ( ) const { return ::popcount(_val) ; } + constexpr bool operator<=( BitMap other ) const { return !( _val & ~other._val ) ; } + constexpr bool operator>=( BitMap other ) const { return !( ~_val & other._val ) ; } + constexpr BitMap operator~ ( ) const { return BitMap(lsb_msk(N)&~_val) ; } + constexpr BitMap operator& ( BitMap other ) const { return BitMap(_val&other._val) ; } + constexpr BitMap operator| ( BitMap other ) const { return BitMap(_val|other._val) ; } + constexpr BitMap& operator&=( BitMap other ) { *this = *this&other ; return *this ; } + constexpr BitMap& operator|=( BitMap other ) { *this = *this|other ; return *this ; } + constexpr bool operator[]( E bit_ ) const { return bit(_val,+bit_) ; } + constexpr uint8_t popcount ( ) const { return ::popcount(_val) ; } constexpr void set ( E flag , bool val ) { if (val) *this |= flag ; else *this &= ~BitMap(flag) ; } // operator~(E) is not always recognized because of namespace's // data private : @@ -931,11 +935,11 @@ extern thread_local MutexLvl t_mutex_lvl ; template struct Mutex : ::conditional_t<::is_void_v,::conditional_t,M> { using Base = ::conditional_t<::is_void_v,::conditional_t,M> ; static constexpr MutexLvl Lvl = Lvl_ ; + // services void lock (MutexLvl& lvl) { SWEAR(t_mutex_lvl< Lvl,t_mutex_lvl) ; lvl = t_mutex_lvl ; t_mutex_lvl = Lvl ; Base::lock () ; } void unlock (MutexLvl lvl) { SWEAR(t_mutex_lvl==Lvl,t_mutex_lvl) ; t_mutex_lvl = lvl ; Base::unlock () ; } void lock_shared (MutexLvl& lvl) requires(S) { SWEAR(t_mutex_lvl< Lvl,t_mutex_lvl) ; lvl = t_mutex_lvl ; t_mutex_lvl = Lvl ; Base::lock_shared () ; } void unlock_shared(MutexLvl lvl) requires(S) { SWEAR(t_mutex_lvl==Lvl,t_mutex_lvl) ; t_mutex_lvl = lvl ; Base::unlock_shared() ; } - // services #ifndef NDEBUG void swear_locked () { SWEAR(t_mutex_lvl>=Lvl) ; SWEAR(!Base::try_lock ()) ; } void swear_locked_shared() requires(S) { SWEAR(t_mutex_lvl>=Lvl) ; SWEAR(!Base::try_lock_shared()) ; } diff --git a/unit_tests/hello_world.py b/unit_tests/hello_world.py index 477412c8..fc435c26 100644 --- a/unit_tests/hello_world.py +++ b/unit_tests/hello_world.py @@ -36,6 +36,9 @@ def cmd() : else : + import os + import os.path as osp + import ut print('hello',file=open('hello','w')) @@ -44,3 +47,7 @@ def cmd() : ut.lmake( 'hello+world_sh' , 'hello+world_py' , done=2 , new=2 ) # check targets are out of date ut.lmake( 'hello+world_sh' , 'hello+world_py' , done=0 , new=0 ) # check targets are up to date ut.lmake( 'hello+hello_sh' , 'world+world_py' , done=2 ) # check reconvergence + + os.system('ldebug hello+world_sh') # ensure no crash and server is dead + + assert not osp.exists('LMAKE/server'),'server is still alive'