improve auto-config + improve socket management + fix bug with tmp ma…

…pping with autodep=ld_audit + protect agains chdir in dynamic attributes + fix hanging server + fixed rerun loop + add maybe_unused attributes in syscall_tab
G-P-S · Apr 12, 2024 · 3ec8007 · 3ec8007
1 parent 826b8e7
commit 3ec8007
Show file tree

Hide file tree

Showing 52 changed files with 1,464 additions and 1,116 deletions.
diff --git a/Makefile b/Makefile
@@ -48,9 +48,9 @@ WARNING_FLAGS := -Wall -Wextra -Wno-cast-function-type -Wno-type-limits
 LANG := c++20
 #
 SAN       := $(if $(strip $(SAN_FLAGS)),.san,)
-LINK_OPTS := $(patsubst %,-Wl$(COMMA)-rpath=%,$(LINK_LIB_PATH)) -pthread                                    # e.g. : -Wl,-rpath=/a/b -Wl,-rpath=/c -pthread
+LINK_OPTS := $(patsubst %,-Wl$(COMMA)-rpath=%,$(LINK_LIB_PATH)) -pthread # e.g. : -Wl,-rpath=/a/b -Wl,-rpath=/c -pthread
 LINK_O    := $(CXX) $(COVERAGE) -r
-LINK_SO   := $(CXX) $(COVERAGE) $(LINK_OPTS) -shared                                                        # some usage may have specific libs, avoid dependencies
+LINK_SO   := $(CXX) $(COVERAGE) $(LINK_OPTS) -shared                     # some usage may have specific libs, avoid dependencies
 LINK_BIN  := $(CXX) $(COVERAGE) $(LINK_OPTS)
 LINK_LIB  := -ldl
 #
@@ -71,11 +71,11 @@ LMAKE_ENV  := lmake_env
 STORE_LIB  := $(SRC)/store
 
 ifneq ($(PYTHON2),)
-    PY2_INC_DIRS  := $(filter-out $(STD_INC_DIRS),$(PY2_INCLUDEDIR) $(PY2_INCLUDEPY))                          # for some reasons, compilation breaks if standard inc dirs are given with -isystem
+    PY2_INC_DIRS  := $(filter-out $(STD_INC_DIRS),$(PY2_INCLUDEDIR) $(PY2_INCLUDEPY)) # for some reasons, compilation breaks if standard inc dirs are given with -isystem
     PY2_CC_OPTS   := $(patsubst %,-isystem %,$(PY2_INC_DIRS)) -Wno-register
     PY2_LINK_OPTS := $(patsubst %,-L%,$(PY2_LIB_DIR)) $(patsubst %,-Wl$(COMMA)-rpath=%,$(PY2_LIB_DIR)) -l:$(PY2_LIB_BASE)
 endif
-PY_INC_DIRS  := $(filter-out $(STD_INC_DIRS),$(PY_INCLUDEDIR) $(PY_INCLUDEPY))                           # for some reasons, compilation does not work if standard inc dirs are given with -isystem
+PY_INC_DIRS  := $(filter-out $(STD_INC_DIRS),$(PY_INCLUDEDIR) $(PY_INCLUDEPY))        # for some reasons, compilation does not work if standard inc dirs are given with -isystem
 PY_CC_OPTS   := $(patsubst %,-isystem %,$(PY_INC_DIRS)) -Wno-register
 PY_LINK_OPTS := $(patsubst %,-L%,$(PY_LIB_DIR))  $(patsubst %,-Wl$(COMMA)-rpath=%,$(PY_LIB_DIR))  -l:$(PY_LIB_BASE)
 
@@ -646,14 +646,18 @@ UNIT_TESTS : UNIT_TESTS1 UNIT_TESTS2
 	@( cd $(@D) ; git clean -ffdxq >/dev/null 2>/dev/null ) ; : # keep $(@D) to ease debugging, ignore rc as old versions of git work but generate an error
 	@for f in $$(grep '^$(UT_DIR)/base/' Manifest) ; do df=$(@D)/$${f#$(UT_DIR)/base/} ; mkdir -p $$(dirname $$df) ; cp $$f $$df ; done
 	@cd $(@D) ; find . -type f -printf '%P\n' > Manifest
-	@( cd $(@D) ; PATH=$(ROOT_DIR)/bin:$(ROOT_DIR)/_bin:$$PATH $(ROOT_DIR)/$< ) >$@.out 2>$@.err && mv $@.out $@ || ( cat $@ $@.err ; exit 1 )
+	@	( cd $(@D) ; PATH=$(ROOT_DIR)/bin:$(ROOT_DIR)/_bin:$$PATH $(ROOT_DIR)/$< ) >$@.out 2>$@.err \
+	&&	mv $@.out $@                                                                                \
+	||	( cat $@.out $@.err ; exit 1 )
 
 %.dir/tok : %.py $(LMAKE_FILES) _lib/ut.py
 	@echo py test to $@
 	@mkdir -p $(@D)
 	@( cd $(@D) ; git clean -ffdxq >/dev/null 2>/dev/null ) ; : # keep $(@D) to ease debugging, ignore rc as old versions of git work but generate an error
 	@cp $< $(@D)/Lmakefile.py
-	@( cd $(@D) ; PATH=$(ROOT_DIR)/bin:$(ROOT_DIR)/_bin:$$PATH PYTHONPATH=$(ROOT_DIR)/lib:$(ROOT_DIR)/_lib HOME= $(PYTHON) Lmakefile.py ) >$@.out 2>$@.err && mv $@.out $@ || ( cat $@ $@.err ; exit 1 )
+	@	( cd $(@D) ; PATH=$(ROOT_DIR)/bin:$(ROOT_DIR)/_bin:$$PATH PYTHONPATH=$(ROOT_DIR)/lib:$(ROOT_DIR)/_lib HOME= $(PYTHON) Lmakefile.py ) >$@.out 2>$@.err \
+	&&	mv $@.out $@                                                                                                                                          \
+	||	( cat $@.out $@.err ; exit 1 )
 
 #
 # lmake env

diff --git a/Manifest b/Manifest
@@ -116,6 +116,7 @@ src/lmakeserver/store.x.hh
 src/lmark.cc
 src/lrepair.cc
 src/lshow.cc
+src/msg.hh
 src/non_portable.cc
 src/non_portable.hh
 src/process.cc
@@ -237,6 +238,7 @@ unit_tests/test17.script
 unit_tests/test18.script
 unit_tests/test19.script
 unit_tests/tmp.py
+unit_tests/tmp2.py
 unit_tests/trancient.py
 unit_tests/unstable.script
 unit_tests/uphill.py

diff --git a/TO_DO b/TO_DO
@@ -12,6 +12,8 @@ items :
 * BUGS (implemented but does not work)
 ****************************************************************************************************
 
+* crash if python() dynamic attribute returns None
+* crash in regexpr (cf mail NM 2024/04/08 21:29)
 * fix compilation with LMAKE_FLAGS=ST
 * missing some deps when reading elf
 	- it seems that libc.so is missing at least in some occasions
@@ -34,14 +36,6 @@ items :
 * generate meaningful message in case of I/O error such as disk full
 ! before erasing a dir, check for phony targets, not only files on disk
 	- then dir markers for gcc include dirs could be phony, which is more elegant
-? mimic slurm killing procedure
-	- use PR_SET_CHILD_SUBREAPER (cf man 2 prctl) to ensure we get all sub-processes
-	- follow process hierarchy
-	- recognize stdin, stdout, stderr
-	- do it in 2 passes
-		- as process hierarchy moves while we walk through it
-		- stop during discovery pass
-		- kill when everything is steady
 ? improve hard link management
 	- when ln a b
 	- followed by write to b
@@ -51,10 +45,6 @@ items :
 * COSMETIC (ugly as long as not implemented)
 ****************************************************************************************************
 
-* report "killed while waiting for stdout or stderr"
-	- when a job is killed while the child is terminated
-	- so as to indicate to the user that something weird was on going
-	- could also report this time in lshow -i
 * generate an error when calling depend on a target
 	- either known at that time
 	- or when it becomes a target if later
@@ -76,18 +66,17 @@ items :
 * ROBUSTNESS & MAINTENABILITY (fragile/difficult to read or maintain as long as not implemented)
 ****************************************************************************************************
 
-* close socket in the management thread (M) while request is being processed
-	- such requests must go through the engine_loop
-	- this may take a while
-	- this means during this time, all jobs may connect (and actually do)
-	- requiring a socket for each slurm slot
-	- defeating the purpose of disconnecting jobs during execution
-	- in all cases, code must resist to an error in accept
-*	fix store to be compliant with strict aliasing rules
+* add size check to date check
+	- create a file version by aggregating Ddate, FileTag & size
+	- keep FileTag accessible from such version
+	- this improves protection against continuing writing without change mtime
+		- this is not perfect, though, as one can write w/o change the size, but decreases probability (already low) by a significant factor
+	- as granularity is several ms
 * support 64-bits id
 	- configure with NBits rather than types
 	- in store/file.hh, reserve address space after NBits instead of type
 	- first candidate would be deps, by far the most demanding
+* fix store to be compliant with strict aliasing rules
 * add a warning when ids reach 15/16 of their limits
 * implement noexcept/except everywhere pertinent
 * use the Path struct instead of at/file
@@ -113,9 +102,13 @@ items :
 * FEATURES (not implemented and can work without)
 ****************************************************************************************************
 
-* improve job isolation by using namespaces
+*3 improve job isolation by using namespaces
 	- much like faketree : https://github.com/enfabrica/enkit/tree/master/faketree
 	- generalize tmp mapping
+	- provide a container with just : system + repo + tmp
+		- this way, no harm can be done outside repo
+* provide a reasonable default value when dynamic functions return None
+	- at least, dont crash
 * implement cache v2 (copy & link) :
 	- 2 levels : disk level, global level
 	- use link instead of copy
@@ -171,6 +164,11 @@ items :
 * OPTIMIZATIONS
 ****************************************************************************************************
 
+*2 regexprs
+	- consider using PCRE2 instead of std::regex
+	- consider handling prefix/suffix apart
+		- most importantly sufixes
+		? maybe a pre-pass searching for infix is advisable
 * when rounding resources using config.backends.precisions
 	- round to prioritize jobs
 	- do not round for actual allocation
@@ -186,11 +184,6 @@ items :
 	- total number of deps
 	- total number of pressure updates
 * gprof
-* regexprs
-	- consider using PCRE2 instead of std::regex
-	- consider handling prefix/suffix apart
-		- most importantly sufixes
-		? maybe a pre-pass searching for infix is advisable
 * no req_info for non-buildable nodes
 * there can be 1 backend thread per backend
 	- with one master socket
@@ -204,25 +197,15 @@ items :
 	- does not work straight forward
 	- may be use a giant map to reserve address space and allocate a growing map in there using HUGE pages
 	- need to do some trials
-? implement deps compression
-	- store words (NodeIdx)
-	- MSB=1, no flag and crc=None, use a single word
-	- else if 2nd word does not look like a flag word (with a marker inside flag word), no flag and crc : use 3 words
-	- else if 2nd indicates no presence of crc, flag and crc=None, use 2 words
-	- else flag and crc, use 4 words
-	- there is marginal case where crc looks like flag, in this case use 4 words instead of 3 when no flag, not a drama, even with a 8 bits marker
-	- should bring about 3x reduction in deps size
-	- targets can be compressed similarly when no flag
 ? record dep crcs in Job rather than in Dep
 	- store crc just before dep error parallel chunk
 	- dont store deps after first dep error parallel chunk as they are useless
 	- this way we can severely reduce the Deps size (4x if we can manage accesses)
 ? avoid ack at job start
 	- pass all infos in command line
-		- or pass a tmp file that contains all info if too large
-		- specially if it becomes necessary to pass targets & deps
+		- pass JobRpcReply as argument through mk_shell_str
+		- fall back to current implementation if command line would be too large
 	- find a way to be certain that seq_id is not necessary to qualify job start
-	- either wash targets at acquire_cmd_line time or wash in job_exec
 ? launch deps as soon as they are discovered
 	- w/o waiting for dependent job to complete
 	- for non-phony non-existent deps, else there is a risk that a steady target is modified while job is on going

diff --git a/_bin/sys_config b/_bin/sys_config
@@ -18,15 +18,15 @@ cd       trial
 # Python configuration (Python2 is optional)
 #
 
-PYTHON2=${PYTHON2:-python2} ; PYTHON2=$(type -p $PYTHON2||:) # ok if we have no python2
+PYTHON2=${PYTHON2:-python2} ; PYTHON2=$(type -p $PYTHON2||:)                                              # ok if we have no python2
 PYTHON=${PYTHON:-python3}   ; PYTHON=$( type -p $PYTHON    )
 for python in $PYTHON2 $PYTHON ; do
 	if [ -x $python ] ; then
 		eval $($python -c 'if True :
 			import sys
 			v = sys.version_info
 			if v.major==2 :
-				if v.minor<7 : exit(0) # Python2 is optional
+				if v.minor<7 : exit(0)                                                                    # Python2 is optional
 				pfx = "PY2"
 			elif v.major==3 :
 				if v.minor<6 :
@@ -43,12 +43,12 @@ for python in $PYTHON2 $PYTHON ; do
 			print(pfx+"_LIB_BASE="  +sysconfig.get_config_var("LDLIBRARY" ))
 			lib_dir = sysconfig.get_config_var("LIBDIR")
 			if not ( (lib_dir+"/").startswith("/usr/lib/") or (lib_dir+"/").startswith("/usr/lib64/") ) :
-				print(pfx+"_LIB_DIR="+lib_dir) # suppress dir as required in case of installed package
+				print(pfx+"_LIB_DIR="+lib_dir)                                                            # suppress dir as required in case of installed package
 		')
 	fi
 done
-if [ "$PY2_LIB_DIR" -a "$PY_LIB_DIR" ] ; then PY_LD_LIBRARY_PATH=$PY_LIB_DIR:$PY2_LIB_DIR # both present, separate with :
-else                                          PY_LD_LIBRARY_PATH=$PY_LIB_DIR$PY2_LIB_DIR  # only one is present
+if [ "$PY2_LIB_DIR" -a "$PY_LIB_DIR" ] ; then PY_LD_LIBRARY_PATH=$PY_LIB_DIR:$PY2_LIB_DIR                 # both present, separate with :
+else                                          PY_LD_LIBRARY_PATH=$PY_LIB_DIR$PY2_LIB_DIR                  # only one is present
 fi
 # cancel Python2 if unusable as a library
 { [ -f $PY2_INCLUDEPY/Python.h ] && file -L $PY2_LIB_DIR/$PY2_LIB_BASE | grep -q shared ; } || PYTHON2=
@@ -66,23 +66,33 @@ case "$($CXX --version|head -1)" in
 	*       )                                                                      echo cannot recognize c++ compiler $CXX ; exit 1     ;;
 esac
 
-# XXX : do the equivalent probe with clang
+$CXX -v -o /dev/null -std=c++20 -xc++ -<<< "int main(){}" 2>compiler_info
 if [ $CXX_FLAVOR = gcc ] ; then
-	LLP="$($CXX -v -E /dev/null 2>&1 | grep LIBRARY_PATH=)" # e.g. : LIBRARY_PATH=/usr/lib/x:/a/b:/c:/a/b/c/..
+	LLP="$(grep LIBRARY_PATH= compiler_info)" # e.g. : LIBRARY_PATH=/usr/lib/x:/a/b:/c:/a/b/c/..
+	LLP="${LLP#LIBRARY_PATH=}"                # e.g. : /usr/lib/x:/a/b:/c:/a/b/c/..
+	LLP="$(sed 's/:/ /' -<<< $LLP)"           # e.g. : /usr/lib/x /a/b /c /a/b/c/..
+elif [ $CXX_FLAVOR = clang ] ; then
+	set $(grep -- -L compiler_info)
+	while [ $# != 0 ] ; do
+		case $1 in
+			-L  ) LLP="$LLP $2" ; shift ;;
+			-L* ) LLP="$LLP ${1#-L}"    ;;
+			*   )                       ;;
+		esac
+		shift
+	done
 fi
-LLP="$(echo $LLP | sed 's/LIBRARY_PATH=//' )"     # e.g. : /usr/lib/x:/a/b:/c:/a/b/c/..
-LLP="$(echo $LLP | sed 's/:/ /'            )"     # e.g. : /usr/lib/x /a/b /c /a/b/c/..
-LLP="$(echo $LLP | sort -u                 )"     # e.g. : /usr/lib/x /a/b /c /a/b
-LINK_LIB_PATH=                                    # e.g. : /a/b /c /usr/lib/x
-for l in $LLP ; do                                # e.g. : /a/b /c (suppress standard dirs as required in case of installed package)
+LLP="$(realpath $LLP | sort -u)"              # e.g. : /a/b /c /usr/lib/x
+LINK_LIB_PATH=
+for l in $LLP ; do                            # e.g. : /a/b /c (suppress standard dirs as required in case of installed package)
 	case $l/ in
 		/usr/lib/*  )                                   ;;
 		/usr/lib64/*)                                   ;;
 		*           ) LINK_LIB_PATH="$LINK_LIB_PATH $l" ;;
 	esac
 done
 
-STD_INC_DIRS="$(echo $( $CXX -E -v -std=c++20 -xc++ /dev/null 2>&1 | sed -e '1,/<.*>.*search starts/d' -e '/End of search/,$d' ) )"
+STD_INC_DIRS="$(echo $(realpath $(sed -e '1,/<.*>.*search starts/d' -e '/End of search/,$d' compiler_info)) )" # echo is used to replace \n by space
 
 #
 # HAS_SECCOMP

diff --git a/_lib/lmake/utils.py b/_lib/lmake/utils.py
@@ -24,10 +24,11 @@ def __delattr__(self,attr) :
 
 def multi_strip(txt) :
 	'in addition to stripping its input, this function also suppresses the common blank prefix of all lines'
-	ls = txt.strip().split('\n')
-	if not ls: return ''
+	ls = txt.split('\n')
+	while ls and ( not ls[ 0] or ls[ 0].isspace() ) : ls = ls[1:  ]
+	while ls and ( not ls[-1] or ls[-1].isspace() ) : ls = ls[ :-1]
+	if not ls : return ''
 	l0 = ls[0]
-	assert l0,'first line cannot be empty because input was stripped'
 	while l0[0].isspace() and all(not l or l[0]==l0[0] for l in ls) :
 		ls = [ l[1:] for l in ls ]
 		l0 = ls[0]

diff --git a/_lib/read_makefiles.py b/_lib/read_makefiles.py
@@ -290,33 +290,31 @@ def _is_simple_fstr(self,fstr) :
 	def _fstring(self,x,mk_fstring=True,for_deps=False) :
 		if callable(x) : return True,x
 		if isinstance(x,(tuple,list,set)) :
-			res_id  = False
+			res_is_dyn  = False
 			res_val = []
 			first = True
 			for c in x :
-				id,v = self._fstring(c,mk_fstring,first and for_deps) # only transmit for_deps to first item of dep when it is a tuple
-				first   = False
-				res_id |= id
+				is_dyn,v    = self._fstring(c,mk_fstring,first and for_deps) # only transmit for_deps to first item of dep when it is a tuple
+				first       = False
+				res_is_dyn |= is_dyn
 				res_val.append(v)
-			return res_id,tuple(res_val)
+			return res_is_dyn,tuple(res_val)
 		if isinstance(x,dict) :
-			res_id  = False
-			res_dct = {}
+			res_is_dyn = False
+			res_dct    = {}
 			for key,val in x.items() :
-				id_k,k = self._fstring(key,False     )
-				id_v,v = self._fstring(val,mk_fstring)
-				res_id |= id_k or id_v
-				res_dct[k] = v
-			return res_id,res_dct
-		if not mk_fstring  or not isinstance(x,str) :
+				is_dyn_k,k  = self._fstring(key,False     )
+				is_dyn_v,v  = self._fstring(val,mk_fstring)
+				res_is_dyn |= is_dyn_k or is_dyn_v
+				res_dct[k]  = v
+			return res_is_dyn,res_dct
+		if not mk_fstring or not isinstance(x,str) :
 			return False,x
 		if for_deps :
-			if self._is_simple_fstr(x) :
-				return False,x
+			if self._is_simple_fstr(x) : return False,x
 		else :
-			if SimpleStrRe.match(x)  :
-				return False,static_fstring(x)                        # v has no variable parts, can be interpreted statically as an f-string
-		return True,serialize.f_str(x)                                # x is made an f-string
+			if SimpleStrRe.match(x)    : return False,static_fstring(x)      # v has no variable parts, can be interpreted statically as an f-string
+		return True,serialize.f_str(x)                                       # x is made an f-string
 
 	def _handle_val(self,key,rep_key=None,for_deps=False) :
 		if not rep_key               : rep_key = key
@@ -329,16 +327,16 @@ def _handle_val(self,key,rep_key=None,for_deps=False) :
 			sv = {}
 			dv = {}
 			for k,v in val.items() :
-				id_k,k = self._fstring(k,False                  )
-				id_v,v = self._fstring(v      ,for_deps=for_deps)
-				if   id_k or id_v : dv[k],sv[k] = self._fstring(v)[1],None # static_val must have an entry for each dynamic one, simple dep stems are only interpreted by engine if static
-				else              : sv[k]       = v
+				is_dyn_k,k = self._fstring(k,False                  )
+				is_dyn_v,v = self._fstring(v      ,for_deps=for_deps)
+				if   is_dyn_k or is_dyn_v : dv[k],sv[k] = self._fstring(v)[1],None # static_val must have an entry for each dynamic one, simple dep stems are only interpreted by engine if static
+				else                      : sv[k]       = v
 			if sv : self.static_val [key] = sv
 			if dv : self.dynamic_val[key] = dv
 		else :
-			id,v = self._fstring(val)
-			if id : self.dynamic_val[key] = v
-			else  : self.static_val [key] = v
+			is_dyn,v = self._fstring(val,for_deps=for_deps)
+			if is_dyn : self.dynamic_val[key] = v
+			else      : self.static_val [key] = v
 
 	def _finalize(self) :
 		static_val  = self.static_val
@@ -350,7 +348,7 @@ def _finalize(self) :
 		code,ctx,names,dbg = serialize.get_expr(
 			dynamic_val
 		,	ctx            = serialize_ctx
-		,	no_imports     = in_repo_re                                                      # non-static deps are forbidden, transport all local modules by value
+		,	no_imports     = in_repo_re    # non-static deps are forbidden, transport all local modules by value
 		,	call_callables = True
 		)
 		return ( static_val , tuple(names) , ctx , code , *mk_dbg_info(dbg,serialize_ctx) )
@@ -500,7 +498,7 @@ def handle_cmd(self) :
 			)
 			if multi :
 				cmd += 'def cmd() : \n'
-				x = avoid_ctx('x',serialize_ctx)                                                                         # find a non-conflicting name
+				x = avoid_ctx('x',serialize_ctx) # find a non-conflicting name
 				for i,c in enumerate(cmd_lst) :
 					a = '' if c.__code__.co_argcount==0 else 'None' if i==0 else x
 					if   i==len(self.attrs.cmd)-1          : cmd += f'\treturn {c.__name__}({a})\n'
@@ -509,7 +507,7 @@ def handle_cmd(self) :
 			if dbg : self.rule_rep.cmd = ( pdict(cmd=cmd) , tuple(names)  , "" , "" , *mk_dbg_info(dbg,serialize_ctx) )
 			else   : self.rule_rep.cmd = ( pdict(cmd=cmd) , tuple(names)                                              )
 		else :
-			self.attrs.cmd = cmd = '\n'.join(self.attrs.cmd)
+			self.attrs.cmd = '\n'.join(self.attrs.cmd)
 			self._init()
 			self._handle_val('cmd',for_deps=True)
 			if 'cmd' in self.dynamic_val : self.dynamic_val = self.dynamic_val['cmd']