diff --git a/Makefile b/Makefile index b548d989..af4baa2d 100644 --- a/Makefile +++ b/Makefile @@ -48,9 +48,9 @@ WARNING_FLAGS := -Wall -Wextra -Wno-cast-function-type -Wno-type-limits LANG := c++20 # SAN := $(if $(strip $(SAN_FLAGS)),.san,) -LINK_OPTS := $(patsubst %,-Wl$(COMMA)-rpath=%,$(LINK_LIB_PATH)) -pthread # e.g. : -Wl,-rpath=/a/b -Wl,-rpath=/c -pthread +LINK_OPTS := $(patsubst %,-Wl$(COMMA)-rpath=%,$(LINK_LIB_PATH)) -pthread # e.g. : -Wl,-rpath=/a/b -Wl,-rpath=/c -pthread LINK_O := $(CXX) $(COVERAGE) -r -LINK_SO := $(CXX) $(COVERAGE) $(LINK_OPTS) -shared # some usage may have specific libs, avoid dependencies +LINK_SO := $(CXX) $(COVERAGE) $(LINK_OPTS) -shared # some usage may have specific libs, avoid dependencies LINK_BIN := $(CXX) $(COVERAGE) $(LINK_OPTS) LINK_LIB := -ldl # @@ -71,11 +71,11 @@ LMAKE_ENV := lmake_env STORE_LIB := $(SRC)/store ifneq ($(PYTHON2),) - PY2_INC_DIRS := $(filter-out $(STD_INC_DIRS),$(PY2_INCLUDEDIR) $(PY2_INCLUDEPY)) # for some reasons, compilation breaks if standard inc dirs are given with -isystem + PY2_INC_DIRS := $(filter-out $(STD_INC_DIRS),$(PY2_INCLUDEDIR) $(PY2_INCLUDEPY)) # for some reasons, compilation breaks if standard inc dirs are given with -isystem PY2_CC_OPTS := $(patsubst %,-isystem %,$(PY2_INC_DIRS)) -Wno-register PY2_LINK_OPTS := $(patsubst %,-L%,$(PY2_LIB_DIR)) $(patsubst %,-Wl$(COMMA)-rpath=%,$(PY2_LIB_DIR)) -l:$(PY2_LIB_BASE) endif -PY_INC_DIRS := $(filter-out $(STD_INC_DIRS),$(PY_INCLUDEDIR) $(PY_INCLUDEPY)) # for some reasons, compilation does not work if standard inc dirs are given with -isystem +PY_INC_DIRS := $(filter-out $(STD_INC_DIRS),$(PY_INCLUDEDIR) $(PY_INCLUDEPY)) # for some reasons, compilation does not work if standard inc dirs are given with -isystem PY_CC_OPTS := $(patsubst %,-isystem %,$(PY_INC_DIRS)) -Wno-register PY_LINK_OPTS := $(patsubst %,-L%,$(PY_LIB_DIR)) $(patsubst %,-Wl$(COMMA)-rpath=%,$(PY_LIB_DIR)) -l:$(PY_LIB_BASE) @@ -646,14 +646,18 @@ UNIT_TESTS : UNIT_TESTS1 UNIT_TESTS2 @( cd $(@D) ; git clean -ffdxq >/dev/null 2>/dev/null ) ; : # keep $(@D) to ease debugging, ignore rc as old versions of git work but generate an error @for f in $$(grep '^$(UT_DIR)/base/' Manifest) ; do df=$(@D)/$${f#$(UT_DIR)/base/} ; mkdir -p $$(dirname $$df) ; cp $$f $$df ; done @cd $(@D) ; find . -type f -printf '%P\n' > Manifest - @( cd $(@D) ; PATH=$(ROOT_DIR)/bin:$(ROOT_DIR)/_bin:$$PATH $(ROOT_DIR)/$< ) >$@.out 2>$@.err && mv $@.out $@ || ( cat $@ $@.err ; exit 1 ) + @ ( cd $(@D) ; PATH=$(ROOT_DIR)/bin:$(ROOT_DIR)/_bin:$$PATH $(ROOT_DIR)/$< ) >$@.out 2>$@.err \ + && mv $@.out $@ \ + || ( cat $@.out $@.err ; exit 1 ) %.dir/tok : %.py $(LMAKE_FILES) _lib/ut.py @echo py test to $@ @mkdir -p $(@D) @( cd $(@D) ; git clean -ffdxq >/dev/null 2>/dev/null ) ; : # keep $(@D) to ease debugging, ignore rc as old versions of git work but generate an error @cp $< $(@D)/Lmakefile.py - @( cd $(@D) ; PATH=$(ROOT_DIR)/bin:$(ROOT_DIR)/_bin:$$PATH PYTHONPATH=$(ROOT_DIR)/lib:$(ROOT_DIR)/_lib HOME= $(PYTHON) Lmakefile.py ) >$@.out 2>$@.err && mv $@.out $@ || ( cat $@ $@.err ; exit 1 ) + @ ( cd $(@D) ; PATH=$(ROOT_DIR)/bin:$(ROOT_DIR)/_bin:$$PATH PYTHONPATH=$(ROOT_DIR)/lib:$(ROOT_DIR)/_lib HOME= $(PYTHON) Lmakefile.py ) >$@.out 2>$@.err \ + && mv $@.out $@ \ + || ( cat $@.out $@.err ; exit 1 ) # # lmake env diff --git a/Manifest b/Manifest index 1e68a301..a35c1f3f 100644 --- a/Manifest +++ b/Manifest @@ -116,6 +116,7 @@ src/lmakeserver/store.x.hh src/lmark.cc src/lrepair.cc src/lshow.cc +src/msg.hh src/non_portable.cc src/non_portable.hh src/process.cc @@ -237,6 +238,7 @@ unit_tests/test17.script unit_tests/test18.script unit_tests/test19.script unit_tests/tmp.py +unit_tests/tmp2.py unit_tests/trancient.py unit_tests/unstable.script unit_tests/uphill.py diff --git a/TO_DO b/TO_DO index 42db2d03..decdc3be 100644 --- a/TO_DO +++ b/TO_DO @@ -12,6 +12,8 @@ items : * BUGS (implemented but does not work) **************************************************************************************************** +* crash if python() dynamic attribute returns None +* crash in regexpr (cf mail NM 2024/04/08 21:29) * fix compilation with LMAKE_FLAGS=ST * missing some deps when reading elf - it seems that libc.so is missing at least in some occasions @@ -34,14 +36,6 @@ items : * generate meaningful message in case of I/O error such as disk full ! before erasing a dir, check for phony targets, not only files on disk - then dir markers for gcc include dirs could be phony, which is more elegant -? mimic slurm killing procedure - - use PR_SET_CHILD_SUBREAPER (cf man 2 prctl) to ensure we get all sub-processes - - follow process hierarchy - - recognize stdin, stdout, stderr - - do it in 2 passes - - as process hierarchy moves while we walk through it - - stop during discovery pass - - kill when everything is steady ? improve hard link management - when ln a b - followed by write to b @@ -51,10 +45,6 @@ items : * COSMETIC (ugly as long as not implemented) **************************************************************************************************** -* report "killed while waiting for stdout or stderr" - - when a job is killed while the child is terminated - - so as to indicate to the user that something weird was on going - - could also report this time in lshow -i * generate an error when calling depend on a target - either known at that time - or when it becomes a target if later @@ -76,18 +66,17 @@ items : * ROBUSTNESS & MAINTENABILITY (fragile/difficult to read or maintain as long as not implemented) **************************************************************************************************** -* close socket in the management thread (M) while request is being processed - - such requests must go through the engine_loop - - this may take a while - - this means during this time, all jobs may connect (and actually do) - - requiring a socket for each slurm slot - - defeating the purpose of disconnecting jobs during execution - - in all cases, code must resist to an error in accept -* fix store to be compliant with strict aliasing rules +* add size check to date check + - create a file version by aggregating Ddate, FileTag & size + - keep FileTag accessible from such version + - this improves protection against continuing writing without change mtime + - this is not perfect, though, as one can write w/o change the size, but decreases probability (already low) by a significant factor + - as granularity is several ms * support 64-bits id - configure with NBits rather than types - in store/file.hh, reserve address space after NBits instead of type - first candidate would be deps, by far the most demanding +* fix store to be compliant with strict aliasing rules * add a warning when ids reach 15/16 of their limits * implement noexcept/except everywhere pertinent * use the Path struct instead of at/file @@ -113,9 +102,13 @@ items : * FEATURES (not implemented and can work without) **************************************************************************************************** -* improve job isolation by using namespaces +*3 improve job isolation by using namespaces - much like faketree : https://github.com/enfabrica/enkit/tree/master/faketree - generalize tmp mapping + - provide a container with just : system + repo + tmp + - this way, no harm can be done outside repo +* provide a reasonable default value when dynamic functions return None + - at least, dont crash * implement cache v2 (copy & link) : - 2 levels : disk level, global level - use link instead of copy @@ -171,6 +164,11 @@ items : * OPTIMIZATIONS **************************************************************************************************** +*2 regexprs + - consider using PCRE2 instead of std::regex + - consider handling prefix/suffix apart + - most importantly sufixes + ? maybe a pre-pass searching for infix is advisable * when rounding resources using config.backends.precisions - round to prioritize jobs - do not round for actual allocation @@ -186,11 +184,6 @@ items : - total number of deps - total number of pressure updates * gprof -* regexprs - - consider using PCRE2 instead of std::regex - - consider handling prefix/suffix apart - - most importantly sufixes - ? maybe a pre-pass searching for infix is advisable * no req_info for non-buildable nodes * there can be 1 backend thread per backend - with one master socket @@ -204,25 +197,15 @@ items : - does not work straight forward - may be use a giant map to reserve address space and allocate a growing map in there using HUGE pages - need to do some trials -? implement deps compression - - store words (NodeIdx) - - MSB=1, no flag and crc=None, use a single word - - else if 2nd word does not look like a flag word (with a marker inside flag word), no flag and crc : use 3 words - - else if 2nd indicates no presence of crc, flag and crc=None, use 2 words - - else flag and crc, use 4 words - - there is marginal case where crc looks like flag, in this case use 4 words instead of 3 when no flag, not a drama, even with a 8 bits marker - - should bring about 3x reduction in deps size - - targets can be compressed similarly when no flag ? record dep crcs in Job rather than in Dep - store crc just before dep error parallel chunk - dont store deps after first dep error parallel chunk as they are useless - this way we can severely reduce the Deps size (4x if we can manage accesses) ? avoid ack at job start - pass all infos in command line - - or pass a tmp file that contains all info if too large - - specially if it becomes necessary to pass targets & deps + - pass JobRpcReply as argument through mk_shell_str + - fall back to current implementation if command line would be too large - find a way to be certain that seq_id is not necessary to qualify job start - - either wash targets at acquire_cmd_line time or wash in job_exec ? launch deps as soon as they are discovered - w/o waiting for dependent job to complete - for non-phony non-existent deps, else there is a risk that a steady target is modified while job is on going diff --git a/_bin/sys_config b/_bin/sys_config index 12bfa302..cad6c0fe 100755 --- a/_bin/sys_config +++ b/_bin/sys_config @@ -18,7 +18,7 @@ cd trial # Python configuration (Python2 is optional) # -PYTHON2=${PYTHON2:-python2} ; PYTHON2=$(type -p $PYTHON2||:) # ok if we have no python2 +PYTHON2=${PYTHON2:-python2} ; PYTHON2=$(type -p $PYTHON2||:) # ok if we have no python2 PYTHON=${PYTHON:-python3} ; PYTHON=$( type -p $PYTHON ) for python in $PYTHON2 $PYTHON ; do if [ -x $python ] ; then @@ -26,7 +26,7 @@ for python in $PYTHON2 $PYTHON ; do import sys v = sys.version_info if v.major==2 : - if v.minor<7 : exit(0) # Python2 is optional + if v.minor<7 : exit(0) # Python2 is optional pfx = "PY2" elif v.major==3 : if v.minor<6 : @@ -43,12 +43,12 @@ for python in $PYTHON2 $PYTHON ; do print(pfx+"_LIB_BASE=" +sysconfig.get_config_var("LDLIBRARY" )) lib_dir = sysconfig.get_config_var("LIBDIR") if not ( (lib_dir+"/").startswith("/usr/lib/") or (lib_dir+"/").startswith("/usr/lib64/") ) : - print(pfx+"_LIB_DIR="+lib_dir) # suppress dir as required in case of installed package + print(pfx+"_LIB_DIR="+lib_dir) # suppress dir as required in case of installed package ') fi done -if [ "$PY2_LIB_DIR" -a "$PY_LIB_DIR" ] ; then PY_LD_LIBRARY_PATH=$PY_LIB_DIR:$PY2_LIB_DIR # both present, separate with : -else PY_LD_LIBRARY_PATH=$PY_LIB_DIR$PY2_LIB_DIR # only one is present +if [ "$PY2_LIB_DIR" -a "$PY_LIB_DIR" ] ; then PY_LD_LIBRARY_PATH=$PY_LIB_DIR:$PY2_LIB_DIR # both present, separate with : +else PY_LD_LIBRARY_PATH=$PY_LIB_DIR$PY2_LIB_DIR # only one is present fi # cancel Python2 if unusable as a library { [ -f $PY2_INCLUDEPY/Python.h ] && file -L $PY2_LIB_DIR/$PY2_LIB_BASE | grep -q shared ; } || PYTHON2= @@ -66,15 +66,25 @@ case "$($CXX --version|head -1)" in * ) echo cannot recognize c++ compiler $CXX ; exit 1 ;; esac -# XXX : do the equivalent probe with clang +$CXX -v -o /dev/null -std=c++20 -xc++ -<<< "int main(){}" 2>compiler_info if [ $CXX_FLAVOR = gcc ] ; then - LLP="$($CXX -v -E /dev/null 2>&1 | grep LIBRARY_PATH=)" # e.g. : LIBRARY_PATH=/usr/lib/x:/a/b:/c:/a/b/c/.. + LLP="$(grep LIBRARY_PATH= compiler_info)" # e.g. : LIBRARY_PATH=/usr/lib/x:/a/b:/c:/a/b/c/.. + LLP="${LLP#LIBRARY_PATH=}" # e.g. : /usr/lib/x:/a/b:/c:/a/b/c/.. + LLP="$(sed 's/:/ /' -<<< $LLP)" # e.g. : /usr/lib/x /a/b /c /a/b/c/.. +elif [ $CXX_FLAVOR = clang ] ; then + set $(grep -- -L compiler_info) + while [ $# != 0 ] ; do + case $1 in + -L ) LLP="$LLP $2" ; shift ;; + -L* ) LLP="$LLP ${1#-L}" ;; + * ) ;; + esac + shift + done fi -LLP="$(echo $LLP | sed 's/LIBRARY_PATH=//' )" # e.g. : /usr/lib/x:/a/b:/c:/a/b/c/.. -LLP="$(echo $LLP | sed 's/:/ /' )" # e.g. : /usr/lib/x /a/b /c /a/b/c/.. -LLP="$(echo $LLP | sort -u )" # e.g. : /usr/lib/x /a/b /c /a/b -LINK_LIB_PATH= # e.g. : /a/b /c /usr/lib/x -for l in $LLP ; do # e.g. : /a/b /c (suppress standard dirs as required in case of installed package) +LLP="$(realpath $LLP | sort -u)" # e.g. : /a/b /c /usr/lib/x +LINK_LIB_PATH= +for l in $LLP ; do # e.g. : /a/b /c (suppress standard dirs as required in case of installed package) case $l/ in /usr/lib/* ) ;; /usr/lib64/*) ;; @@ -82,7 +92,7 @@ for l in $LLP ; do # e.g. : /a/b /c (suppress sta esac done -STD_INC_DIRS="$(echo $( $CXX -E -v -std=c++20 -xc++ /dev/null 2>&1 | sed -e '1,/<.*>.*search starts/d' -e '/End of search/,$d' ) )" +STD_INC_DIRS="$(echo $(realpath $(sed -e '1,/<.*>.*search starts/d' -e '/End of search/,$d' compiler_info)) )" # echo is used to replace \n by space # # HAS_SECCOMP diff --git a/_lib/lmake/utils.py b/_lib/lmake/utils.py index e5c69ad1..42e5a454 100644 --- a/_lib/lmake/utils.py +++ b/_lib/lmake/utils.py @@ -24,10 +24,11 @@ def __delattr__(self,attr) : def multi_strip(txt) : 'in addition to stripping its input, this function also suppresses the common blank prefix of all lines' - ls = txt.strip().split('\n') - if not ls: return '' + ls = txt.split('\n') + while ls and ( not ls[ 0] or ls[ 0].isspace() ) : ls = ls[1: ] + while ls and ( not ls[-1] or ls[-1].isspace() ) : ls = ls[ :-1] + if not ls : return '' l0 = ls[0] - assert l0,'first line cannot be empty because input was stripped' while l0[0].isspace() and all(not l or l[0]==l0[0] for l in ls) : ls = [ l[1:] for l in ls ] l0 = ls[0] diff --git a/_lib/read_makefiles.py b/_lib/read_makefiles.py index 3235b6f8..5f9ef542 100755 --- a/_lib/read_makefiles.py +++ b/_lib/read_makefiles.py @@ -290,33 +290,31 @@ def _is_simple_fstr(self,fstr) : def _fstring(self,x,mk_fstring=True,for_deps=False) : if callable(x) : return True,x if isinstance(x,(tuple,list,set)) : - res_id = False + res_is_dyn = False res_val = [] first = True for c in x : - id,v = self._fstring(c,mk_fstring,first and for_deps) # only transmit for_deps to first item of dep when it is a tuple - first = False - res_id |= id + is_dyn,v = self._fstring(c,mk_fstring,first and for_deps) # only transmit for_deps to first item of dep when it is a tuple + first = False + res_is_dyn |= is_dyn res_val.append(v) - return res_id,tuple(res_val) + return res_is_dyn,tuple(res_val) if isinstance(x,dict) : - res_id = False - res_dct = {} + res_is_dyn = False + res_dct = {} for key,val in x.items() : - id_k,k = self._fstring(key,False ) - id_v,v = self._fstring(val,mk_fstring) - res_id |= id_k or id_v - res_dct[k] = v - return res_id,res_dct - if not mk_fstring or not isinstance(x,str) : + is_dyn_k,k = self._fstring(key,False ) + is_dyn_v,v = self._fstring(val,mk_fstring) + res_is_dyn |= is_dyn_k or is_dyn_v + res_dct[k] = v + return res_is_dyn,res_dct + if not mk_fstring or not isinstance(x,str) : return False,x if for_deps : - if self._is_simple_fstr(x) : - return False,x + if self._is_simple_fstr(x) : return False,x else : - if SimpleStrRe.match(x) : - return False,static_fstring(x) # v has no variable parts, can be interpreted statically as an f-string - return True,serialize.f_str(x) # x is made an f-string + if SimpleStrRe.match(x) : return False,static_fstring(x) # v has no variable parts, can be interpreted statically as an f-string + return True,serialize.f_str(x) # x is made an f-string def _handle_val(self,key,rep_key=None,for_deps=False) : if not rep_key : rep_key = key @@ -329,16 +327,16 @@ def _handle_val(self,key,rep_key=None,for_deps=False) : sv = {} dv = {} for k,v in val.items() : - id_k,k = self._fstring(k,False ) - id_v,v = self._fstring(v ,for_deps=for_deps) - if id_k or id_v : dv[k],sv[k] = self._fstring(v)[1],None # static_val must have an entry for each dynamic one, simple dep stems are only interpreted by engine if static - else : sv[k] = v + is_dyn_k,k = self._fstring(k,False ) + is_dyn_v,v = self._fstring(v ,for_deps=for_deps) + if is_dyn_k or is_dyn_v : dv[k],sv[k] = self._fstring(v)[1],None # static_val must have an entry for each dynamic one, simple dep stems are only interpreted by engine if static + else : sv[k] = v if sv : self.static_val [key] = sv if dv : self.dynamic_val[key] = dv else : - id,v = self._fstring(val) - if id : self.dynamic_val[key] = v - else : self.static_val [key] = v + is_dyn,v = self._fstring(val,for_deps=for_deps) + if is_dyn : self.dynamic_val[key] = v + else : self.static_val [key] = v def _finalize(self) : static_val = self.static_val @@ -350,7 +348,7 @@ def _finalize(self) : code,ctx,names,dbg = serialize.get_expr( dynamic_val , ctx = serialize_ctx - , no_imports = in_repo_re # non-static deps are forbidden, transport all local modules by value + , no_imports = in_repo_re # non-static deps are forbidden, transport all local modules by value , call_callables = True ) return ( static_val , tuple(names) , ctx , code , *mk_dbg_info(dbg,serialize_ctx) ) @@ -500,7 +498,7 @@ def handle_cmd(self) : ) if multi : cmd += 'def cmd() : \n' - x = avoid_ctx('x',serialize_ctx) # find a non-conflicting name + x = avoid_ctx('x',serialize_ctx) # find a non-conflicting name for i,c in enumerate(cmd_lst) : a = '' if c.__code__.co_argcount==0 else 'None' if i==0 else x if i==len(self.attrs.cmd)-1 : cmd += f'\treturn {c.__name__}({a})\n' @@ -509,7 +507,7 @@ def handle_cmd(self) : if dbg : self.rule_rep.cmd = ( pdict(cmd=cmd) , tuple(names) , "" , "" , *mk_dbg_info(dbg,serialize_ctx) ) else : self.rule_rep.cmd = ( pdict(cmd=cmd) , tuple(names) ) else : - self.attrs.cmd = cmd = '\n'.join(self.attrs.cmd) + self.attrs.cmd = '\n'.join(self.attrs.cmd) self._init() self._handle_val('cmd',for_deps=True) if 'cmd' in self.dynamic_val : self.dynamic_val = self.dynamic_val['cmd'] diff --git a/lmake_env/Lmakefile.py b/lmake_env/Lmakefile.py index 1b155dee..6ff104f9 100644 --- a/lmake_env/Lmakefile.py +++ b/lmake_env/Lmakefile.py @@ -10,9 +10,10 @@ import sysconfig import tarfile import zipfile -from subprocess import run,DEVNULL,STDOUT +from subprocess import run,check_output,DEVNULL,STDOUT -gxx = os.environ.get('CXX','g++') +gxx = os.environ.get('CXX','g++') +gxx_is_clang = 'clang' in check_output( (gxx,"--version") , universal_newlines=True ) import lmake from lmake import config,pdict @@ -140,10 +141,10 @@ class ConfigH(BaseRule) : deps = { 'CONFIGURE' : 'ext/{DirS}configure' } cmd = 'cd ext/{DirS} ; ./configure' -class SysConfigH(PathRule) : +class SysConfig(PathRule) : targets = { - 'MK' : 'sys_config.mk' - , 'H' : 'sys_config.h' + 'H' : 'sys_config.h' + , 'MK' : 'sys_config.mk' , 'TRIAL' : 'trial/{*:.*}' } deps = { 'EXE' : '_bin/sys_config' } @@ -209,8 +210,8 @@ def cmd() : seen_inc = x in ('-I','-iquote','-isystem','-idirafter') lmake.depend(*mrkrs) lmake.check_deps() - if 'clang' in gxx : clang_opts = ('-Wno-misleading-indentation','-Wno-unknown-warning-option','-Wno-c2x-extensions','-Wno-unused-function','-Wno-c++2b-extensions') - else : clang_opts = () + if gxx_is_clang : clang_opts = ('-Wno-misleading-indentation','-Wno-unknown-warning-option','-Wno-c2x-extensions','-Wno-unused-function','-Wno-c++2b-extensions') + else : clang_opts = () run_gxx( OBJ , '-c' , '-fPIC' , '-pthread' , f'-frandom-seed={OBJ}' , '-fvisibility=hidden' , *basic_opts diff --git a/src/autodep/autodep.cc b/src/autodep/autodep.cc index 9775aad3..62be9a2c 100644 --- a/src/autodep/autodep.cc +++ b/src/autodep/autodep.cc @@ -36,7 +36,7 @@ int main( int argc , char* argv[] ) { // if (!cmd_line.flags[CmdFlag::AutodepMethod]) syntax.usage("must have both autodep-method and link-support options") ; // - Gather gather{ New } ; + Gather gather ; // try { /**/ gather.method = mk_enum(cmd_line.flag_args[+CmdFlag::AutodepMethod]) ; diff --git a/src/autodep/clmake.cc b/src/autodep/clmake.cc index 6be07c05..fde92a23 100644 --- a/src/autodep/clmake.cc +++ b/src/autodep/clmake.cc @@ -18,7 +18,7 @@ using namespace Hash ; using namespace Py ; using namespace Time ; -using Proc = JobExecRpcProc ; +using Proc = JobExecProc ; static Record _g_record ; @@ -123,7 +123,7 @@ static PyObject* target( PyObject* /*null*/ , PyObject* args , PyObject* kwds ) try { files = _get_files(py_args) ; } catch (::string const& e) { return py_err_set(Exception::TypeErr,e) ; } // - JobExecRpcReq jerr { JobExecRpcProc::Access , ::move(files) , ad , no_follow , false/*sync*/ , "target" } ; + JobExecRpcReq jerr { Proc::Access , ::move(files) , ad , no_follow , false/*sync*/ , "target" } ; _g_record.direct(::move(jerr)) ; // return None.to_py_boost() ; diff --git a/src/autodep/gather.cc b/src/autodep/gather.cc index 970e73c7..454efa57 100644 --- a/src/autodep/gather.cc +++ b/src/autodep/gather.cc @@ -77,12 +77,12 @@ void Gather::_new_access( Fd fd , PD pd , ::string&& file , AccessDigest ad , CD } else { info = &accesses[it->second].second ; } - if (!parallel) parallel_id++ ; - AccessInfo old_info = *info ; // for tracing only + if (!parallel) _parallel_id++ ; + AccessInfo old_info = *info ; // for tracing only //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - info->update( pd , ad , cd , parallel_id ) ; + info->update( pd , ad , cd , _parallel_id ) ; //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - if ( is_new || *info!=old_info ) Trace("_new_access", fd , STR(is_new) , pd , ad , cd , parallel_id , comment , old_info , "->" , *info , it->first ) ; // only trace if something changes + if ( is_new || *info!=old_info ) Trace("_new_access", fd , STR(is_new) , pd , ad , cd , _parallel_id , comment , old_info , "->" , *info , it->first ) ; // only trace if something changes } void Gather::new_deps( PD pd , ::vmap_s&& deps , ::string const& stdin ) { @@ -108,31 +108,41 @@ void Gather::new_exec( PD pd , ::string const& exe , ::string const& c ) { } } -ENUM( GatherKind , Stdout , Stderr , ServerReply , ChildEnd , Master , Slave ) - -void _child_wait_thread_func( int* wstatus , pid_t pid , Fd fd ) { +void _child_wait_thread_func( int* wstatus , Pdate* end_time , pid_t pid , Fd fd ) { static constexpr uint64_t One = 1 ; do { ::waitpid(pid,wstatus,0) ; } while (WIFSTOPPED(*wstatus)) ; + *end_time = New ; swear_prod(::write(fd,&One,8)==8,"cannot report child wstatus",wstatus) ; } -bool/*done*/ Gather::kill(int sig) { - Trace trace("kill",sig,pid,STR(as_session),child_stdout,child_stderr) ; - Lock lock { _pid_mutex } ; - int kill_sig = sig>=0 ? sig : SIGKILL ; - bool killed_ = false ; - killed = true ; // prevent child from starting if killed before - // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - if (pid>1) killed_ = kill_process(pid,kill_sig,as_session/*as_group*/) ; - // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - if (sig<0) { // kill all processes (or process groups) connected to a stream we wait for - pid_t ctl_pid = as_session ? ::getpgrp() : ::getpid() ; - ::umap_s< GatherKind> fd_strs ; - ::umap to_kill ; - trace("ctl",ctl_pid,mk_key_uset(slaves)) ; - if (+child_stdout) fd_strs[ read_lnk(to_string("/proc/self/fd/",child_stdout.fd)) ] = GatherKind::Stdout ; - if (+child_stderr) fd_strs[ read_lnk(to_string("/proc/self/fd/",child_stderr.fd)) ] = GatherKind::Stderr ; - trace("fds",fd_strs) ; +void Gather::_kill( KillStep kill_step , Child const& child ) { + Trace trace("kill",kill_step,STR(as_session),child.pid) ; + SWEAR(kill_step>=KillStep::Kill) ; + uint8_t kill_idx = kill_step-KillStep::Kill ; + bool last = kill_idx>=kill_sigs.size() ; + if (!_kill_reported) { + if (!_wait[Kind::ChildEnd] ) { + trace("no_child",_wait) ; + const char* pfx = "killed while waiting" ; + if (_wait[Kind::Stdout]) { append_to_string( msg , pfx , " stdout" ) ; pfx = " and" ; } + if (_wait[Kind::Stderr]) append_to_string( msg , pfx , " stderr" ) ; + msg.push_back('\n') ; + } + _kill_reported = true ; + } + if (_wait[Kind::ChildEnd]) { + int sig = kill_sigs[kill_idx] ; + trace("kill",sig) ; + // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + if ( sig && child.pid>1 ) kill_process(child.pid,sig,as_session/*as_group*/) ; + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + } + if (last) { + pid_t ctl_pid = as_session ? ::getpgrp() : ::getpid() ; + ::umap_s< Kind> fd_strs ; + ::umap to_kill ; + if (_wait[Kind::Stdout]) fd_strs[ read_lnk(to_string("/proc/self/fd/",child.stdout.fd)) ] = Kind::Stdout ; + if (_wait[Kind::Stderr]) fd_strs[ read_lnk(to_string("/proc/self/fd/",child.stderr.fd)) ] = Kind::Stderr ; for( ::string const& proc_entry : lst_dir("/proc") ) { for( char c : proc_entry ) if (c>'9'||c<'0') goto NextProc ; try { @@ -151,13 +161,12 @@ bool/*done*/ Gather::kill(int sig) { } catch(::string const&) {} // if we cannot read /proc/pid, process is dead, ignore NextProc : ; } - trace("to_kill",to_kill) ; - // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - for( auto [p,_] : to_kill ) killed_ |= kill_process(p,kill_sig,as_session/*as_group*/) ; - // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + trace("last_kill",ctl_pid,child.stdout,child.stderr,fd_strs,to_kill) ; + // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + for( auto [p,_] : to_kill ) kill_process(p,SIGKILL,as_session/*as_group*/) ; + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ } - trace("done",STR(killed_)) ; - return killed_ ; + trace("done") ; } void Gather::_solve( Fd fd , JobExecRpcReq& jerr ) { @@ -184,173 +193,198 @@ void Gather::_solve( Fd fd , JobExecRpcReq& jerr ) { jerr.solve = false ; // files are now real and dated } -Status Gather::exec_child( ::vector_s const& args , Fd cstdin , Fd cstdout , Fd cstderr ) { - using Kind = GatherKind ; - Trace trace("exec_child",STR(as_session),method,autodep_env,args) ; - if (env) trace("env",*env) ; - Child child ; +void Gather::_send_to_server ( Fd fd , Jerr&& jerr ) { + Trace trace("_send_to_server",fd,jerr) ; // - if (env) swear_prod( !env->contains("LMAKE_AUTODEP_ENV") , "cannot run lmake under lmake" ) ; - else swear_prod( !has_env ("LMAKE_AUTODEP_ENV") , "cannot run lmake under lmake" ) ; - autodep_env.service = master_fd.service(addr) ; - trace("autodep_env",::string(autodep_env)) ; + Proc proc = jerr.proc ; // capture essential info before moving to server_cb + size_t sz = jerr.files.size() ; // . + switch (proc) { + case Proc::ChkDeps : reorder(false/*at_end*/) ; break ; // ensure server sees a coherent view + case Proc::DepInfos : _new_accesses(fd,::copy(jerr)) ; break ; + // + case Proc::Decode : SWEAR(jerr.files.size()==1) ; _codec_files[fd] = Codec::mk_decode_node( jerr.files[0].first , jerr.ctx , jerr.txt ) ; break ; + case Proc::Encode : SWEAR(jerr.files.size()==1) ; _codec_files[fd] = Codec::mk_encode_node( jerr.files[0].first , jerr.ctx , jerr.txt ) ; break ; + default : ; + } + try { + JobMngtRpcReq jmrr ; + if (jerr.proc==JobExecProc::ChkDeps) jmrr = { JobMngtProc::ChkDeps , seq_id , job , fd , cur_deps_cb() } ; + else jmrr = { seq_id , job , fd , ::move(jerr) } ; + //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + OMsgBuf().send( ClientSockFd(service_mngt) , jmrr ) ; + //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + } catch (...) { + trace("no_server") ; + JobExecRpcReply sync_reply ; + sync_reply.proc = proc ; + sync_reply.ok = Yes ; // try to mimic server as much as possible when none is available + sync_reply.dep_infos = ::vector>(sz,{Yes,{}}) ; // . + sync(fd,::move(sync_reply)) ; + } +} + +void Gather::_spawn_child( Child& child , ::vector_s const& args , Fd cstdin , Fd cstdout , Fd cstderr ) { + Trace trace("_spawn_child",args,cstdin,cstdout,cstderr) ; // - ::map_ss add_env {{"LMAKE_AUTODEP_ENV",autodep_env}} ; // required even with method==None or ptrace to allow support (ldepend, lmake module, ...) to work - { Lock lock{_pid_mutex} ; - if (killed ) return Status::Killed ; // dont start if we are already killed before starting - if (method==AutodepMethod::Ptrace) { // PER_AUTODEP_METHOD : handle case - // XXX : splitting responsibility is no more necessary. Can directly report child termination from within autodep_ptrace.process using same ifce as _child_wait_thread_func - // we split the responsability into 2 processes : - // - parent watches for data (stdin, stdout, stderr & incoming connections to report deps) - // - child launches target process using ptrace and watches it using direct wait (without signalfd) then report deps using normal socket report - bool in_parent = child.spawn( as_session , {} , cstdin , cstdout , cstderr ) ; - if (!in_parent) { - Child grand_child ; - AutodepPtrace::s_autodep_env = new AutodepEnv{autodep_env} ; - try { - //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - grand_child.spawn( - as_session , args - , Fd::Stdin , Fd::Stdout , Fd::Stderr - , env , &add_env - , chroot - , cwd - , AutodepPtrace::s_prepare_child - ) ; - //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - } catch(::string const& e) { - exit(Rc::System,e) ; - } - trace("pid",grand_child.pid) ; - AutodepPtrace autodep_ptrace { grand_child.pid } ; - int wstatus = autodep_ptrace.process() ; - grand_child.waited() ; // grand_child has already been waited - if (WIFEXITED (wstatus)) ::_exit(WEXITSTATUS(wstatus)) ; - else if (WIFSIGNALED(wstatus)) ::_exit(+Rc::System ) ; - fail_prod("ptraced child did not exit and was not signaled : wstatus : ",wstatus) ; - } + ::map_ss add_env { {"LMAKE_AUTODEP_ENV",autodep_env} } ; // required even with method==None or ptrace to allow support (ldepend, lmake module, ...) to work + if (method==AutodepMethod::Ptrace) { // PER_AUTODEP_METHOD : handle case + // we split the responsability into 2 processes : + // - parent watches for data (stdin, stdout, stderr & incoming connections to report deps) + // - child launches target process using ptrace and watches it using direct wait (without signalfd) then report deps using normal socket report + bool in_parent = child.spawn( as_session , {} , cstdin , cstdout , cstderr ) ; + if (in_parent) { + start_time = New ; // record job start time as late as possible } else { - if (method>=AutodepMethod::Ld) { // PER_AUTODEP_METHOD : handle case - ::string env_var ; - // - switch (method) { // PER_AUTODEP_METHOD : handle case - case AutodepMethod::LdAudit : env_var = "LD_AUDIT" ; add_env[env_var] = *g_lmake_dir+"/_lib/ld_audit.so" ; break ; - case AutodepMethod::LdPreload : env_var = "LD_PRELOAD" ; add_env[env_var] = *g_lmake_dir+"/_lib/ld_preload.so" ; break ; - case AutodepMethod::LdPreloadJemalloc : env_var = "LD_PRELOAD" ; add_env[env_var] = *g_lmake_dir+"/_lib/ld_preload_jemalloc.so" ; break ; - DF} - if (env) { if (env->contains(env_var)) add_env[env_var] += ':' + env->at(env_var) ; } - else { if (has_env (env_var)) add_env[env_var] += ':' + get_env(env_var) ; } - } - new_exec( New , args[0] ) ; + Child grand_child ; + AutodepPtrace::s_autodep_env = new AutodepEnv{autodep_env} ; try { - //vvvvvvvvvvvvvvvvvvvvvvvvvvvv - child.spawn( - as_session , args - , cstdin , cstdout , cstderr + //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + grand_child.spawn( + false/*as_group*/ , args // first level child has created the group + , Fd::Stdin , Fd::Stdout , Fd::Stderr , env , &add_env , chroot , cwd + , AutodepPtrace::s_prepare_child ) ; - //^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ } catch(::string const& e) { - if (cstderr==Child::Pipe) stderr = e ; - else cstderr.write(e) ; - return Status::EarlyErr ; + exit(Rc::System,e) ; } - trace("pid",child.pid) ; + trace("grand_child_pid",grand_child.pid) ; + AutodepPtrace autodep_ptrace { grand_child.pid } ; + int wstatus = autodep_ptrace.process() ; + grand_child.waited() ; // grand_child has already been waited + if (WIFEXITED (wstatus)) ::_exit(WEXITSTATUS(wstatus)) ; + else if (WIFSIGNALED(wstatus)) ::_exit(+Rc::System ) ; + fail_prod("ptraced child did not exit and was not signaled : wstatus : ",wstatus) ; + } + } else { + if (method>=AutodepMethod::Ld) { // PER_AUTODEP_METHOD : handle case + ::string env_var ; + // + switch (method) { // PER_AUTODEP_METHOD : handle case + case AutodepMethod::LdAudit : env_var = "LD_AUDIT" ; add_env[env_var] = *g_lmake_dir+"/_lib/ld_audit.so" ; break ; + case AutodepMethod::LdPreload : env_var = "LD_PRELOAD" ; add_env[env_var] = *g_lmake_dir+"/_lib/ld_preload.so" ; break ; + case AutodepMethod::LdPreloadJemalloc : env_var = "LD_PRELOAD" ; add_env[env_var] = *g_lmake_dir+"/_lib/ld_preload_jemalloc.so" ; break ; + DF} + if (env) { if (env->contains(env_var)) add_env[env_var] += ':' + env->at(env_var) ; } + else { if (has_env (env_var)) add_env[env_var] += ':' + get_env(env_var) ; } } - pid = child.pid ; + new_exec( New , args[0] ) ; + start_time = New ; // record job start time as late as possible + //vvvvvvvvvvvvvvvvvvvvvvvvvvvv + child.spawn( + as_session , args + , cstdin , cstdout , cstderr + , env , &add_env + , chroot + , cwd + ) ; + //^^^^^^^^^^^^^^^^^^^^^^^^^^^^ } + trace("child_pid",child.pid) ; +} +Status Gather::exec_child( ::vector_s const& args , Fd cstdin , Fd cstdout , Fd cstderr ) { + Trace trace("exec_child",STR(as_session),method,autodep_env,args) ; + if (env) trace("env",*env) ; + ServerSockFd job_master_fd { New } ; // - Fd child_fd = ::eventfd(0,EFD_CLOEXEC) ; - ::jthread wait_jt { _child_wait_thread_func , &wstatus , child.pid , child_fd } ; // thread dedicated to wating child - Epoll epoll { New } ; - Status status = Status::New ; - uint8_t n_active = 0 ; - ::umap server_replies ; - ::umap delayed_check_deps ; // check_deps events are delayed to ensure all previous deps are taken into account - Pdate job_end ; - Pdate reporting_end ; + if (env) swear_prod( !env->contains("LMAKE_AUTODEP_ENV") , "cannot run lmake under lmake" ) ; + else swear_prod( !has_env ("LMAKE_AUTODEP_ENV") , "cannot run lmake under lmake" ) ; + autodep_env.service = job_master_fd.service(addr) ; + trace("autodep_env",::string(autodep_env)) ; + // + Child child ; + AutoCloseFd child_fd ; + ::jthread wait_jt ; // thread dedicated to wating child + Epoll epoll { New } ; + Status status = Status::New ; + ::umap delayed_check_deps ; // check_deps events are delayed to ensure all previous deps are received + KillStep kill_step = {} ; + Pdate event_date ; + size_t live_out_pos = 0 ; + ::umap>> slaves ; // Jerr's are waiting for confirmation // - auto handle_req_to_server = [&]( Fd fd , Jerr&& jerr ) -> bool/*still_sync*/ { - trace("slave",fd,jerr) ; - Proc proc = jerr.proc ; // capture essential info before moving to server_cb - size_t sz = jerr.files.size() ; // . - bool sync_ = jerr.sync ; // . - ::string codec_file ; - switch (proc) { - case Proc::ChkDeps : reorder(false/*at_end*/) ; break ; // ensure server sees a coherent view - case Proc::DepInfos : _new_accesses(fd,::copy(jerr)) ; break ; - case Proc::Decode : codec_file = Codec::mk_decode_node( jerr.files[0].first/*file*/ , jerr.ctx , jerr.txt ) ; break ; - case Proc::Encode : codec_file = Codec::mk_encode_node( jerr.files[0].first/*file*/ , jerr.ctx , jerr.txt ) ; break ; - default : ; - } - Fd reply_fd = server_cb(::move(jerr)) ; - if (+reply_fd) { - epoll.add_read(reply_fd,Kind::ServerReply) ; - trace("read_reply",reply_fd) ; - ServerReply& sr = server_replies[reply_fd] ; - if (sync_) sr.fd = fd ; - /**/ sr.codec_file = codec_file ; - } else if (sync_) { - JobExecRpcReply sync_reply ; - sync_reply.proc = proc ; - sync_reply.ok = Yes ; // try to mimic server as much as possible when none is available - sync_reply.dep_infos = ::vector>(sz,{Yes,{}}) ; // . - sync(fd,sync_reply) ; - } - return false ; - } ; auto set_status = [&]( Status status_ , ::string const& msg_={} )->void { if ( status==Status::New || status==Status::Ok ) status = status_ ; // else there is already another reason if ( +msg_ ) append_line_to_string(msg,msg_) ; } ; - auto dec_active = [&]()->void { - SWEAR(n_active) ; - n_active-- ; - if ( !n_active && +network_delay ) reporting_end = Pdate(New)+network_delay ; // once job is dead, wait at most network_delay (if set) for reporting to calm down + auto done = [&](Kind k)->void { + SWEAR(_wait[k]) ; + _wait &= ~k ; + if (!_wait) { // if job is dead for good + event_date = Pdate(New)+network_delay ; // wait at most network_delay for reporting to settle down + /**/ epoll.cnt-- ; // dont wait for new connections from job (but process those that come) + if (+server_master_fd) epoll.cnt-- ; // idem for connections from server + } } ; // - SWEAR(!slaves) ; - // if (+timeout) { - job_end = Pdate(New) + timeout ; - trace("set_timeout",timeout,job_end) ; + event_date = Pdate(New) + timeout ; + trace("set_timeout",timeout,event_date) ; + } + if (+server_master_fd) { + epoll.add_read(server_master_fd,Kind::ServerMaster) ; + trace("read_server_master",server_master_fd) ; } - if (cstdout==Child::Pipe) { epoll.add_read(child_stdout=child.stdout,Kind::Stdout ) ; n_active++ ; trace("read_stdout",child_stdout) ; } - if (cstderr==Child::Pipe) { epoll.add_read(child_stderr=child.stderr,Kind::Stderr ) ; n_active++ ; trace("read_stderr",child_stderr) ; } - /**/ { epoll.add_read(child_fd ,Kind::ChildEnd) ; n_active++ ; trace("read_child ",child_fd ) ; } - /**/ { epoll.add_read(master_fd ,Kind::Master ) ; trace("read_master",master_fd ) ; } - while (epoll.cnt) { - uint64_t wait_ns = Epoll::Forever ; - if (+reporting_end) { - Pdate now = {New} ; - if (now=event_date ) { + switch (kill_step) { + case KillStep::Report : + goto Return ; + case KillStep::None : + trace("fire_timeout") ; + if (+_wait) set_status(Status::Err,to_string("timout after " ,timeout .short_str())) ; + else set_status(Status::Err,to_string("still active after being dead for ",network_delay.short_str())) ; + kill_step = KillStep::Kill ; + [[fallthrough]] ; + default : + SWEAR(kill_step>=KillStep::Kill) ; + if (_wait[Kind::ChildStart]) goto Return ; // killed before job start + _kill(kill_step,child) ; + if (!event_date) event_date = now ; + if (uint8_t(kill_step-KillStep::Kill)==kill_sigs.size()) { kill_step = KillStep::Report ; event_date += network_delay ; } + else { kill_step++ ; SWEAR(+kill_step) ; event_date += Delay(1) ; } // ensure no wrap around + } } + wait_for = event_date-now ; } - if (+delayed_check_deps) wait_ns = 0 ; - ::vector events = epoll.wait(wait_ns) ; - if ( !events && +delayed_check_deps ) { // process delayed check deps after all other events - trace("delayed_chk_deps") ; - for( auto& [fd,jerr] : delayed_check_deps ) handle_req_to_server(fd,::move(jerr)) ; - delayed_check_deps.clear() ; - continue ; + if ( +delayed_check_deps || _wait[Kind::ChildStart] ) wait_for = {} ; + ::vector events = epoll.wait(wait_for) ; + if (!events) { + if (+delayed_check_deps) { // process delayed check deps after all other events + trace("delayed_chk_deps") ; + for( auto& [fd,jerr] : delayed_check_deps ) _send_to_server(fd,::move(jerr)) ; + delayed_check_deps.clear() ; + continue ; + } + if (_wait[Kind::ChildStart]) { // handle case where we are killed before starting : create child when we have processed waiting connections from server + try { + _spawn_child( child , args , cstdin , cstdout , cstderr ) ; + _wait &= ~Kind::ChildStart ; + } catch(::string const& e) { + if (cstderr==Child::Pipe) stderr = e ; + else cstderr.write(e) ; + status = Status::EarlyErr ; + goto Return ; + } + child_fd = ::eventfd(0,EFD_CLOEXEC) ; + wait_jt = ::jthread( _child_wait_thread_func , &wstatus , &end_time , child.pid , Fd(child_fd) ) ; // thread dedicated to wating child + if (cstdout==Child::Pipe) { epoll.add_read(child.stdout ,Kind::Stdout ) ; _wait |= Kind::Stdout ; trace("read_stdout" ,child.stdout) ; } + if (cstderr==Child::Pipe) { epoll.add_read(child.stderr ,Kind::Stderr ) ; _wait |= Kind::Stderr ; trace("read_stderr" ,child.stderr) ; } + /**/ epoll.add_read(child_fd ,Kind::ChildEnd ) ; _wait |= Kind::ChildEnd ; trace("read_child " ,child_fd ) ; + /**/ epoll.add_read(job_master_fd,Kind::JobMaster) ; trace("read_job_master",job_master_fd) ; + } } for( Epoll::Event const& event : events ) { Kind kind = event.data() ; Fd fd = event.fd() ; - if (kind!=Kind::Slave) trace(kind,fd,epoll.cnt) ; + if (kind!=Kind::JobSlave) trace(kind,fd,epoll.cnt) ; switch (kind) { case Kind::Stdout : case Kind::Stderr : { @@ -358,68 +392,85 @@ Status Gather::exec_child( ::vector_s const& args , Fd cstdin , Fd cstdout , Fd int cnt = ::read( fd , buf , sizeof(buf) ) ; SWEAR( cnt>=0 , cnt ) ; ::string_view buf_view { buf , size_t(cnt) } ; if (cnt) { - if (kind==Kind::Stderr) stderr.append(buf_view) ; - else { stdout.append(buf_view) ; live_out_cb(buf_view) ; } + if (kind==Kind::Stderr) { + stderr.append(buf_view) ; + } else { + stdout.append(buf_view) ; + if (live_out) { + if ( size_t pos = buf_view.rfind('\n') ; pos!=Npos ) { + size_t old_sz = stdout.size() ; + pos++ ; + trace("live_out",old_sz-live_out_pos,'+',pos) ; + //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + OMsgBuf().send( ClientSockFd(service_mngt) , JobMngtRpcReq( JobMngtProc::LiveOut , seq_id , job , stdout.substr(live_out_pos,old_sz+pos-live_out_pos) ) ) ; + //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + live_out_pos = old_sz+pos ; + } + } + } } else { - epoll.del(fd) ; // /!\ dont close as fd is closed upon child destruction + epoll.close(fd) ; trace("close",kind,fd) ; - if (kind==Kind::Stderr) child_stderr = {} ; // tell kill not to wait for this one - else child_stdout = {} ; - dec_active() ; + done(kind) ; } } break ; case Kind::ChildEnd : { uint64_t one = 0/*garbage*/ ; int cnt = ::read( fd , &one , 8 ) ; SWEAR( cnt==8 && one==1 , cnt , one ) ; - { Lock lock{_pid_mutex} ; - child.pid = -1 ; // too late to kill job - } if (WIFEXITED (wstatus)) set_status( WEXITSTATUS(wstatus)!=0 ? Status::Err : Status::Ok ) ; - else if (WIFSIGNALED(wstatus)) set_status( is_sig_sync(WTERMSIG(wstatus)) ? Status::Err : Status::LateLost ) ; // synchronous signals are actually errors + else if (WIFSIGNALED(wstatus)) set_status( is_sig_sync(WTERMSIG(wstatus)) ? Status::Err : Status::LateLost ) ; // synchronous signals are actually errors else fail("unexpected wstatus : ",wstatus) ; epoll.close(fd) ; - epoll.cnt-- ; // do not wait for new connections, but if one arrives before all flows are closed, process it, so wait Master no more - dec_active() ; + done(kind) ; trace("close",kind,status,::hex,wstatus,::dec) ; } break ; - case Kind::Master : { - SWEAR(fd==master_fd) ; - Fd slave = master_fd.accept() ; - epoll.add_read(slave,Kind::Slave) ; - trace("read_slave",slave) ; - slaves[slave] ; // allocate entry + case Kind::JobMaster : + case Kind::ServerMaster : { + bool is_job = kind==Kind::JobMaster ; + SWEAR( fd==(is_job?job_master_fd:server_master_fd) , fd , is_job , job_master_fd , server_master_fd ) ; + Fd slave = (kind==Kind::JobMaster?job_master_fd:server_master_fd).accept() ; + epoll.add_read(slave,is_job?Kind::JobSlave:Kind::ServerSlave) ; + trace("read_slave",STR(is_job),slave) ; + slaves[slave] ; // allocate entry } break ; - case Kind::ServerReply : { - JobRpcReply jrr ; - auto it = server_replies.find(fd) ; - SWEAR(it!=server_replies.end()) ; - try { if (!it->second.buf.receive_step(fd,jrr)) continue ; } - catch (...) { } // server disappeared, give up - trace(jrr) ; - Fd rfd = it->second.fd ; // capture before move - switch (jrr.proc) { - case JobProc::ChkDeps : if (jrr.ok==Maybe) { set_status(Status::ChkDeps) ; kill_job_cb() ; } break ; - case JobProc::DepInfos : break ; - case JobProc::Decode : - case JobProc::Encode : _codec(::move(it->second),jrr) ; break ; - case JobProc::None : kill_job_cb() ; break ; // server died - DF} - // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - if (+rfd) sync( rfd , JobExecRpcReply(jrr) ) ; - // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - server_replies.erase(it) ; + case Kind::ServerSlave : { + JobMngtRpcReply jmrr ; + auto it = slaves.find(fd) ; + auto& slave_entry = it->second ; + try { if (!slave_entry.first.receive_step(fd,jmrr)) continue ; } + catch (...) { trace("no_jmrr",jmrr) ; jmrr.proc = {} ; } // fd was closed, ensure no partially received jmrr + trace(kind,jmrr) ; + Fd rfd = jmrr.fd ; // capture before move + if (jmrr.seq_id==seq_id) { + switch (jmrr.proc) { + case JobMngtProc::DepInfos : + case JobMngtProc::Heartbeat : break ; + case JobMngtProc::Kill : + case JobMngtProc::None : set_status(Status::Killed ) ; kill_step = KillStep::Kill ; break ; // server died + case JobMngtProc::ChkDeps : if (jmrr.ok==Maybe) { set_status(Status::ChkDeps) ; kill_step = KillStep::Kill ; rfd = {} ; } break ; + case JobMngtProc::Decode : + case JobMngtProc::Encode : { + auto it = _codec_files.find(jmrr.fd) ; + _new_access( rfd , PD(New) , ::move(it->second) , {.accesses=Access::Reg} , jmrr.crc , false/*parallel*/ , ::string(snake(jmrr.proc)) ) ; + _codec_files.erase(it) ; + } break ; + DF} + // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + if (+rfd) sync( rfd , JobExecRpcReply(::move(jmrr)) ) ; + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + } epoll.close(fd) ; trace("close",kind,fd) ; } break ; - case Kind::Slave : { + case Kind::JobSlave : { Jerr jerr ; auto it = slaves.find(fd) ; auto& slave_entry = it->second ; try { if (!slave_entry.first.receive_step(fd,jerr)) continue ; } - catch (...) { trace("no_jerr",jerr) ; jerr.proc = Proc::None ; } // fd was closed, ensure no partially received jerr - Proc proc = jerr.proc ; // capture essential info so as to be able to move jerr - bool sync_ = jerr.sync ; // . - if ( proc!=Proc::Access ) trace(kind,fd,epoll.cnt,proc) ; // there may be too many Access'es, only trace within _new_accesses + catch (...) { trace("no_jerr",jerr) ; jerr.proc = Proc::None ; } // fd was closed, ensure no partially received jerr + Proc proc = jerr.proc ; // capture essential info so as to be able to move jerr + bool sync_ = jerr.sync ; // . + if ( proc!=Proc::Access ) trace(kind,fd,epoll.cnt,proc) ; // there may be too many Access'es, only trace within _new_accesses if ( proc>=Proc::HasFiles && jerr.solve ) _solve(fd,jerr) ; switch (proc) { case Proc::Confirm : @@ -429,22 +480,22 @@ Status Gather::exec_child( ::vector_s const& args , Fd cstdin , Fd cstdout , Fd case Proc::None : epoll.close(fd) ; trace("close",kind,fd) ; - for( Jerr& j : slave_entry.second ) _new_accesses(fd,::move(j)) ; // process deferred entries although with uncertain outcome + for( Jerr& j : slave_entry.second ) _new_accesses(fd,::move(j)) ; // process deferred entries although with uncertain outcome slaves.erase(it) ; break ; case Proc::Access : // for read accesses, trying is enough to trigger a dep, so confirm is useless - if ( jerr.digest.write==Maybe ) slave_entry.second.push_back(::move(jerr)) ; // defer until confirm resolution + if ( jerr.digest.write==Maybe ) slave_entry.second.push_back(::move(jerr)) ; // defer until confirm resolution else _new_accesses(fd,::move(jerr)) ; break ; - case Proc::Tmp : seen_tmp = true ; break ; - case Proc::Guard : _new_guards(fd,::move(jerr)) ; break ; + case Proc::Tmp : seen_tmp = true ; break ; + case Proc::Guard : _new_guards(fd,::move(jerr)) ; break ; case Proc::DepInfos : case Proc::Decode : - case Proc::Encode : handle_req_to_server(fd,::move(jerr)) ; goto NoReply ; - case Proc::ChkDeps : delayed_check_deps[fd] = ::move(jerr) ; goto NoReply ; // if sync, reply is delayed as well - case Proc::Panic : set_status(Status::Err,jerr.txt) ; kill_job_cb() ; [[fallthrough]] ; - case Proc::Trace : trace(jerr.txt) ; break ; + case Proc::Encode : _send_to_server(fd,::move(jerr)) ; goto NoReply ; + case Proc::ChkDeps : delayed_check_deps[fd] = ::move(jerr) ; goto NoReply ; // if sync, reply is delayed as well + case Proc::Panic : set_status(Status::Err,jerr.txt) ; kill_step = KillStep::Kill ; [[fallthrough]] ; + case Proc::Trace : trace(jerr.txt) ; break ; DF} if (sync_) sync( fd , JobExecRpcReply(proc) ) ; NoReply : ; @@ -452,8 +503,11 @@ Status Gather::exec_child( ::vector_s const& args , Fd cstdin , Fd cstdout , Fd DF} } } - trace("done") ; - reorder(true/*at_end*/) ; // ensure server sees a coherent view +Return : + child.waited() ; + trace("done",status) ; + SWEAR(status!=Status::New) ; + reorder(true/*at_end*/) ; // ensure server sees a coherent view return status ; } diff --git a/src/autodep/gather.hh b/src/autodep/gather.hh index 9f3ce4c5..32395564 100644 --- a/src/autodep/gather.hh +++ b/src/autodep/gather.hh @@ -9,6 +9,7 @@ #include "disk.hh" #include "hash.hh" +#include "msg.hh" #include "process.hh" #include "rpc_job.hh" #include "time.hh" @@ -22,13 +23,32 @@ // This way, we do not generate spurious errors. // To do so, we maintain, for each access entry (i.e. a file), a list of sockets that are unordered, i.e. for which a following Write could actually have been done before by the user. +ENUM( GatherKind // epoll events +, Stdout +, Stderr +, ServerReply +, ChildStart // just a marker, not actually used as epoll event +, ChildEnd +, JobMaster +, JobSlave +, ServerMaster +, ServerSlave +) + +ENUM( KillStep +, None +, Report +, Kill // must be last as following values are used +) + struct Gather { friend ::ostream& operator<<( ::ostream& os , Gather const& ad ) ; - using Proc = JobExecRpcProc ; - using Jerr = JobExecRpcReq ; - using Crc = Hash::Crc ; - using PD = Time::Pdate ; - using CD = CrcDate ; + using Kind = GatherKind ; + using Proc = JobExecProc ; + using Jerr = JobExecRpcReq ; + using Crc = Hash::Crc ; + using PD = Time::Pdate ; + using CD = CrcDate ; struct AccessInfo { friend ::ostream& operator<<( ::ostream& , AccessInfo const& ) ; // cxtors & casts @@ -56,17 +76,6 @@ struct Gather { NodeIdx parallel_id = 0 ; AccessDigest digest ; } ; - struct ServerReply { - IMsgBuf buf ; // buf to assemble the reply - Fd fd ; // fd to forward reply to - ::string codec_file ; - } ; - // cxtors & casts -public : - Gather( ) = default ; - Gather(NewType) { init() ; } - // - void init() { master_fd.listen() ; } // services private : void _solve( Fd , Jerr& jerr) ; @@ -82,10 +91,8 @@ private : Trace trace("_new_guards",fd,jerr.txt) ; for( auto& [f,_] : jerr.files ) { trace(f) ; guards.insert(::move(f)) ; } } - void _codec( ServerReply&& sr , JobRpcReply const& jrr , ::string const& comment="codec" ) { - Trace trace("_codec",jrr) ; - _new_access( sr.fd , PD(New) , ::move(sr.codec_file) , {.accesses=Access::Reg} , jrr.crc , false/*parallel*/ , comment ) ; - } + void _kill ( KillStep , Child const& ) ; + void _send_to_server( Fd fd , Jerr&& jerr ) ; public : //! crc_date parallel void new_target( PD pd , ::string const& t , ::string const& c="s_target" ) { _new_access(pd,::copy(t),{.write=Yes},{} ,false ,c) ; } void new_unlnk ( PD pd , ::string const& t , ::string const& c="s_unlnk" ) { _new_access(pd,::copy(t),{.write=Yes},{} ,false ,c) ; } // new_unlnk is used for internal wash @@ -95,43 +102,46 @@ public : //! void new_exec( PD , ::string const& exe , ::string const& ="s_exec" ) ; // void sync( Fd sock , JobExecRpcReply const& jerr ) { - try { OMsgBuf().send(sock,jerr) ; } catch (::string const&) {} // dont care if we cannot report the reply to job + try { OMsgBuf().send(sock,jerr) ; } catch (::string const&) {} // dont care if we cannot report the reply to job } // Status exec_child( ::vector_s const& args , Fd child_stdin=Fd::Stdin , Fd child_stdout=Fd::Stdout , Fd child_stderr=Fd::Stderr ) ; // - bool/*done*/ kill(int sig=-1) ; // is sig==-1, use best effort to kill job - // - void reorder(bool at_end) ; // reorder accesses by first read access and suppress superfluous accesses + void reorder(bool at_end) ; // reorder accesses by first read access and suppress superfluous accesses +private : + void _spawn_child( Child& , ::vector_s const& args , Fd child_stdin=Fd::Stdin , Fd child_stdout=Fd::Stdout , Fd child_stderr=Fd::Stderr ) ; // data - ::function server_cb = [](Jerr && )->Fd { return {} ; } ; // function to contact server when necessary, return error by default - ::function live_out_cb = [](::string_view const&)->void { } ; // function to report live output, dont report by default - ::function kill_job_cb = []( )->void { } ; // function to kill job - ServerSockFd master_fd ; - in_addr_t addr = NoSockAddr ; // local addr to which we can be contacted by running job - ::atomic as_session = false ; // if true <=> process is launched in its own group - AutodepMethod method = AutodepMethod::Dflt ; - AutodepEnv autodep_env ; - Time::Delay timeout ; - Time::Delay network_delay ; - pid_t pid = -1 ; // pid to kill - bool killed = false ; // do not start as child is supposed to be already killed - ::vector kill_sigs ; // signals used to kill job - ::string chroot ; - ::string cwd ; - ::map_ss const* env = nullptr ; - vmap_s accesses ; - umap_s access_map ; - uset_s guards ; // dir creation/deletion that must be guarded against NFS - NodeIdx parallel_id = 0 ; // id to identify parallel deps - bool seen_tmp = false ; - int wstatus = 0/*garbage*/ ; - Fd child_stdout ; // fd used to gather stdout - Fd child_stderr ; // fd used to gather stderr - ::string stdout ; // contains child stdout if child_stdout==Pipe - ::string stderr ; // contains child stderr if child_stderr==Pipe - ::string msg ; // contains error messages not from job - ::umap>> slaves ; // Jerr's are waiting for confirmation +public : + umap_s access_map ; + vmap_s accesses ; + in_addr_t addr = NoSockAddr ; // local addr to which we can be contacted by running job + ::atomic as_session = false ; // if true <=> process is launched in its own group + AutodepEnv autodep_env ; + ::string chroot ; + ::function<::vmap_s()> cur_deps_cb ; + ::string cwd ; + Time::Pdate end_time ; + ::map_ss const* env = nullptr ; + uset_s guards ; // dir creation/deletion that must be guarded against NFS + JobIdx job = 0 ; + ::vector kill_sigs ; // signals used to kill job + bool live_out = false ; + AutodepMethod method = AutodepMethod::Dflt ; + ::string msg ; // contains error messages not from job + Time::Delay network_delay ; + pid_t pid = -1 ; // pid to kill + bool seen_tmp = false ; + SeqId seq_id = 0 ; + ServerSockFd server_master_fd ; + ::string service_mngt ; + Time::Pdate start_time ; + ::string stdout ; // contains child stdout if child_stdout==Pipe + ::string stderr ; // contains child stderr if child_stderr==Pipe + Time::Delay timeout ; + int wstatus = 0 ; private : - Mutex mutable _pid_mutex ; + ::umap _codec_files ; + bool _kill_reported = false ; + NodeIdx _parallel_id = 0 ; // id to identify parallel deps + BitMap _wait ; // events we are waiting for } ; diff --git a/src/autodep/lcheck_deps.cc b/src/autodep/lcheck_deps.cc index d7ff51b0..235a7aa5 100644 --- a/src/autodep/lcheck_deps.cc +++ b/src/autodep/lcheck_deps.cc @@ -19,6 +19,6 @@ int main( int argc , char* argv[]) { CmdLine cmd_line { syntax , argc , argv } ; bool verbose = cmd_line.flags[Flag::Verbose] ; if (cmd_line.args.size()!=0 ) syntax.usage("must have no argument") ; - JobExecRpcReply reply = Record(New,Yes/*enable*/).direct( JobExecRpcReq( JobExecRpcProc::ChkDeps , cmd_line.flags[Flag::Verbose]/*sync*/ ) ) ; + JobExecRpcReply reply = Record(New,Yes/*enable*/).direct( JobExecRpcReq( JobExecProc::ChkDeps , cmd_line.flags[Flag::Verbose]/*sync*/ ) ) ; return verbose && reply.ok!=Yes ? 1 : 0 ; } diff --git a/src/autodep/ld_audit.cc b/src/autodep/ld_audit.cc index d2df8807..e75d3001 100644 --- a/src/autodep/ld_audit.cc +++ b/src/autodep/ld_audit.cc @@ -49,6 +49,7 @@ static bool started() { return true ; } ::umap_s const* const g_syscall_tab = new ::umap_s{ { "chdir" , { reinterpret_cast(Audited::chdir ) } } +, { "chmod" , { reinterpret_cast(Audited::chmod ) } } , { "close" , { reinterpret_cast(Audited::close ) } } , { "__close" , { reinterpret_cast(Audited::__close ) } } , { "creat" , { reinterpret_cast(Audited::creat ) } } @@ -66,6 +67,8 @@ ::umap_s const* const g_syscall_tab = new ::umap_s{ , { "execvp" , { reinterpret_cast(Audited::execvp ) } } , { "execvpe" , { reinterpret_cast(Audited::execvpe ) } } , { "fchdir" , { reinterpret_cast(Audited::fchdir ) } } +, { "fchmod" , { reinterpret_cast(Audited::fchmod ) } } +, { "fchmodat" , { reinterpret_cast(Audited::fchmodat ) } } , { "fopen" , { reinterpret_cast(Audited::fopen ) } } , { "fopen64" , { reinterpret_cast(Audited::fopen64 ) } } , { "fork" , { reinterpret_cast(Audited::fork ) } } diff --git a/src/autodep/ld_common.x.cc b/src/autodep/ld_common.x.cc index 396ed241..ca5a3bde 100644 --- a/src/autodep/ld_common.x.cc +++ b/src/autodep/ld_common.x.cc @@ -62,7 +62,10 @@ static thread_local bool _t_loop = false ; // prevent recu // In that case, they may come before our own Audit is constructed if declared global (in the case of LD_PRELOAD). // To face this order problem, we declare our Audit as a static within a funciton which will be constructed upon first call. // As all statics with cxtor/dxtor, we define it through new so as to avoid destruction during finalization. -static Record& auditer() { +#ifndef IN_SERVER + static // in server, we want to have direct access to recorder (no risk of name pollution as we masterize the code) +#endif +Record& auditer() { static Record* s_res = new Record{New} ; return *s_res ; } diff --git a/src/autodep/ld_server.cc b/src/autodep/ld_server.cc index 133335dc..d479adba 100644 --- a/src/autodep/ld_server.cc +++ b/src/autodep/ld_server.cc @@ -12,3 +12,20 @@ static bool started() { return AutodepLock::t_active ; } // no auto-start for se #define IN_SERVER #include "ld.x.cc" + +AutodepLock::AutodepLock(::vmap_s* deps) : lock{_s_mutex} { + // SWEAR(cwd()==Record::s_autodep_env().root_dir) ; // too expensive + SWEAR( !Record::s_deps && !Record::s_deps_err ) ; + SWEAR( !*Record::s_access_cache ) ; + Record::s_deps = deps ; + Record::s_deps_err = &err ; + t_active = true ; +} + +AutodepLock::~AutodepLock() { + Record::s_deps = nullptr ; + Record::s_deps_err = nullptr ; + t_active = false ; + Record::s_access_cache->clear() ; + if (auditer().seen_chdir) swear_prod(::fchdir(Record::s_root_fd())==0) ; // restore cwd in case it has been modified during user Python code execution +} diff --git a/src/autodep/ld_server.hh b/src/autodep/ld_server.hh index df8c9026..25770328 100644 --- a/src/autodep/ld_server.hh +++ b/src/autodep/ld_server.hh @@ -7,6 +7,8 @@ #pragma once +Record& auditer() ; + struct AutodepLock { // static data static thread_local bool t_active ; @@ -15,21 +17,9 @@ private : // cxtors & casts public : AutodepLock( ) = default ; - AutodepLock(::vmap_s* deps=nullptr) : lock{_s_mutex} { - // SWEAR(cwd()==Record::s_autodep_env().root_dir) ; // too expensive - SWEAR( !Record::s_deps && !Record::s_deps_err ) ; - SWEAR( !*Record::s_access_cache ) ; - Record::s_deps = deps ; - Record::s_deps_err = &err ; - t_active = true ; - } - ~AutodepLock() { - Record::s_deps = nullptr ; - Record::s_deps_err = nullptr ; - t_active = false ; - Record::s_access_cache->clear() ; - if (Record::s_seen_chdir) swear_prod(::fchdir(Record::s_root_fd())) ; // restore cwd in case it has been modified during user Python code execution - } + AutodepLock(::vmap_s* deps=nullptr) ; + // + ~AutodepLock() ; // data Lock> lock ; ::string err ; diff --git a/src/autodep/ldecode.cc b/src/autodep/ldecode.cc index 099cbf77..7738d0e3 100644 --- a/src/autodep/ldecode.cc +++ b/src/autodep/ldecode.cc @@ -29,7 +29,7 @@ int main( int argc , char* argv[]) { if (!cmd_line.flags[Flag::Context]) syntax.usage("must have context to retrieve associated value") ; // JobExecRpcReply reply = Record(New,Yes/*enable*/).direct(JobExecRpcReq( - JobExecRpcProc::Decode + JobExecProc::Decode , ::move(cmd_line.flag_args[+Flag::File ]) , ::move(cmd_line.flag_args[+Flag::Code ]) , ::move(cmd_line.flag_args[+Flag::Context]) diff --git a/src/autodep/ldepend.cc b/src/autodep/ldepend.cc index 7bf248f2..14e70eda 100644 --- a/src/autodep/ldepend.cc +++ b/src/autodep/ldepend.cc @@ -56,7 +56,7 @@ int main( int argc , char* argv[]) { if ( cmd_line.flags[Flag::StatReadData]) ad.extra_dflags |= ExtraDflag::StatReadData ; // if (verbose) { - JobExecRpcReply reply = Record(New).direct(JobExecRpcReq( JobExecRpcProc::DepInfos , ::copy(cmd_line.args) , ad , no_follow , true/*sync*/ , "ldepend" )) ; + JobExecRpcReply reply = Record(New).direct(JobExecRpcReq( JobExecProc::DepInfos , ::copy(cmd_line.args) , ad , no_follow , true/*sync*/ , "ldepend" )) ; // SWEAR( reply.dep_infos.size()==cmd_line.args.size() , reply.dep_infos.size() , cmd_line.args.size() ) ; // @@ -70,7 +70,7 @@ int main( int argc , char* argv[]) { } // } else { - Record(New,Yes/*enable*/).direct(JobExecRpcReq( JobExecRpcProc::Access , ::move(cmd_line.args) , ad , no_follow , "ldepend" )) ; + Record(New,Yes/*enable*/).direct(JobExecRpcReq( JobExecProc::Access , ::move(cmd_line.args) , ad , no_follow , "ldepend" )) ; } return err ? 1 : 0 ; } diff --git a/src/autodep/lencode.cc b/src/autodep/lencode.cc index 0063a5f4..03045f3b 100644 --- a/src/autodep/lencode.cc +++ b/src/autodep/lencode.cc @@ -40,7 +40,7 @@ int main( int argc , char* argv[]) { syntax.usage(to_string("bad min len value : ",e)) ; } JobExecRpcReply reply = Record(New,Yes/*enable*/).direct(JobExecRpcReq( - JobExecRpcProc::Encode + JobExecProc::Encode , ::move(cmd_line.flag_args[+Flag::File ]) , to_string(::cin.rdbuf()) , ::move(cmd_line.flag_args[+Flag::Context]) diff --git a/src/autodep/ltarget.cc b/src/autodep/ltarget.cc index 41f8ba20..2f5cd561 100644 --- a/src/autodep/ltarget.cc +++ b/src/autodep/ltarget.cc @@ -54,7 +54,7 @@ int main( int argc , char* argv[]) { if (!cmd_line.flags[Flag::NoAllow ]) ad.extra_tflags |= ExtraTflag::Allow ; if ( cmd_line.flags[Flag::SourceOk ]) ad.extra_tflags |= ExtraTflag::SourceOk ; // - JobExecRpcReq jerr { JobExecRpcProc::Access , ::move(cmd_line.args) , ad , no_follow , false/*sync*/ , "ltarget" } ; + JobExecRpcReq jerr { JobExecProc::Access , ::move(cmd_line.args) , ad , no_follow , false/*sync*/ , "ltarget" } ; Record(New,Yes/*enable*/).direct(::move(jerr)) ; // return 0 ; diff --git a/src/autodep/record.cc b/src/autodep/record.cc index 8cac55f0..40de2ec3 100644 --- a/src/autodep/record.cc +++ b/src/autodep/record.cc @@ -26,7 +26,6 @@ bool Record::s_static_report = ::vmap_s * Record::s_deps = nullptr ; ::string * Record::s_deps_err = nullptr ; ::umap_s>* Record::s_access_cache = nullptr ; // map file to read accesses -bool Record::s_seen_chdir = false ; AutodepEnv* Record::_s_autodep_env = nullptr ; // declare as pointer to avoid late initialization Fd Record::_s_root_fd ; @@ -72,7 +71,7 @@ bool Record::s_is_simple(const char* file) { void Record::_static_report(JobExecRpcReq&& jerr) const { switch (jerr.proc) { - case JobExecRpcProc::Access : + case Proc::Access : if (jerr.digest.write!=No) for( auto& [f,dd] : jerr.files ) append_to_string(*s_deps_err,"unexpected write/unlink to " ,f,'\n') ; // can have only deps from within server else if (!s_deps ) for( auto& [f,dd] : jerr.files ) append_to_string(*s_deps_err,"unexpected access of " ,f,'\n') ; // can have no deps when no way to record them else { @@ -80,16 +79,16 @@ void Record::_static_report(JobExecRpcReq&& jerr) const { if (+jerr.files) s_deps->back().second.parallel = false ; // parallel bit is marked false on last of a series of parallel accesses } break ; - case JobExecRpcProc::Confirm : - case JobExecRpcProc::Guard : - case JobExecRpcProc::Tmp : - case JobExecRpcProc::Trace : break ; - default : append_to_string(*s_deps_err,"unexpected proc ",jerr.proc,'\n') ; + case Proc::Confirm : + case Proc::Guard : + case Proc::Tmp : + case Proc::Trace : break ; + default : append_to_string(*s_deps_err,"unexpected proc ",jerr.proc,'\n') ; } } void Record::_report_access( JobExecRpcReq&& jerr ) const { - SWEAR( jerr.proc==JobExecRpcProc::Access , jerr.proc ) ; + SWEAR( jerr.proc==Proc::Access , jerr.proc ) ; if (s_autodep_env().disabled) return ; // dont update cache as report is not actually done if (!jerr.sync) { bool miss = false ; @@ -123,8 +122,8 @@ JobExecRpcReply Record::direct(JobExecRpcReq&& jerr) { } else { // not under lmake, try to mimic server as much as possible, but of course no real info available // XXX : for Encode/Decode, we should interrogate the server or explore association file directly so as to allow jobs to run with reasonable data - if ( jerr.sync && jerr.proc==JobExecRpcProc::DepInfos) return { jerr.proc , ::vector>(jerr.files.size(),{Yes,{}}) } ; - else return { } ; + if ( jerr.sync && jerr.proc==Proc::DepInfos) return { jerr.proc , ::vector>(jerr.files.size(),{Yes,{}}) } ; + else return { } ; } } diff --git a/src/autodep/record.hh b/src/autodep/record.hh index 2e48dea2..ef0d60ea 100644 --- a/src/autodep/record.hh +++ b/src/autodep/record.hh @@ -14,7 +14,7 @@ struct Record { using Crc = Hash::Crc ; using Ddate = Time::Ddate ; using SolveReport = Disk::RealPath::SolveReport ; - using Proc = JobExecRpcProc ; + using Proc = JobExecProc ; using GetReplyCb = ::function ; using ReportCb = ::function ; // statics @@ -54,7 +54,6 @@ public : static ::vmap_s * s_deps ; static ::string * s_deps_err ; static ::umap_s>* s_access_cache ; // map file to read accesses - static bool s_seen_chdir ; private : static AutodepEnv* _s_autodep_env ; static Fd _s_root_fd ; // a file descriptor to repo root dir @@ -99,7 +98,7 @@ private : // void _report_access( JobExecRpcReq&& jerr ) const ; void _report_access( ::string&& f , Ddate d , Accesses a , bool write , ::string&& c={} ) const { - _report_access({ JobExecRpcProc::Access , {{::move(f),d}} , {.write=Maybe&write,.accesses=a} , ::move(c) }) ; + _report_access({ Proc::Access , {{::move(f),d}} , {.write=Maybe&write,.accesses=a} , ::move(c) }) ; } // for modifying accesses (_report_update, _report_target, _report_unlnk, _report_targets) : // - if we report after the access, it may be that job is interrupted inbetween and repo is modified without server being notified and we have a manual case @@ -121,27 +120,27 @@ private : void _report_deps( ::vector_s const& fs , Accesses a , bool u , ::string&& c={} ) const { ::vmap_s fds ; for( ::string const& f : fs ) fds.emplace_back( f , Disk::file_date(s_root_fd(),f) ) ; - _report_access({ JobExecRpcProc::Access , ::move(fds) , {.write=Maybe&u,.accesses=a} , ::move(c) }) ; + _report_access({ Proc::Access , ::move(fds) , {.write=Maybe&u,.accesses=a} , ::move(c) }) ; } void _report_targets( ::vector_s&& fs , ::string&& c={} ) const { vmap_s mdd ; for( ::string& f : fs ) mdd.emplace_back(::move(f),Ddate()) ; - _report_access({ JobExecRpcProc::Access , ::move(mdd) , {.write=Maybe} , ::move(c) }) ; + _report_access({ Proc::Access , ::move(mdd) , {.write=Maybe} , ::move(c) }) ; } void _report_tmp( bool sync=false , ::string&& c={} ) const { if (!_tmp_cache) _tmp_cache = true ; else if (!sync ) return ; - _report({JobExecRpcProc::Tmp,sync,::move(c)}) ; + _report({Proc::Tmp,sync,::move(c)}) ; } void _report_confirm( FileLoc fl , bool ok ) const { - if (fl==FileLoc::Repo) _report({ JobExecRpcProc::Confirm , ok }) ; + if (fl==FileLoc::Repo) _report({ Proc::Confirm , ok }) ; } void _report_guard( ::string&& f , ::string&& c={} ) const { - _report({ JobExecRpcProc::Guard , {::move(f)} , ::move(c) }) ; + _report({ Proc::Guard , {::move(f)} , ::move(c) }) ; } public : - template [[noreturn]] void report_panic(A const&... args) { _report({JobExecRpcProc::Panic,to_string(args...)}) ; exit(Rc::Usage) ; } // continuing is meaningless - template void report_trace(A const&... args) { _report({JobExecRpcProc::Trace,to_string(args...)}) ; } + template [[noreturn]] void report_panic(A const&... args) { _report({Proc::Panic,to_string(args...)}) ; exit(Rc::Usage) ; } // continuing is meaningless + template void report_trace(A const&... args) { _report({Proc::Trace,to_string(args...)}) ; } JobExecRpcReply direct( JobExecRpcReq&& jerr) ; // template struct _Path { @@ -154,7 +153,7 @@ public : _Path( ::string const& f , bool steal=false ) : file{f.c_str()} { if (!steal) allocate(f.size()) ; } _Path( Fd a , ::string const& f , bool steal=false ) : has_at{true} , at{a} , file{f.c_str()} { if (!steal) allocate(f.size()) ; } // - _Path(_Path && p) { *this = ::move(p) ; } // XXX : use copy&swap idiom + _Path(_Path && p) { *this = ::move(p) ; } _Path& operator=(_Path&& p) { deallocate() ; has_at = p.has_at ; @@ -336,11 +335,12 @@ public : } ; // void chdir(const char* dir) { - s_seen_chdir = true ; + seen_chdir = true ; _real_path.chdir(dir) ; } // // data + bool seen_chdir = false ; private : Disk::RealPath _real_path ; mutable AutoCloseFd _report_fd ; diff --git a/src/autodep/syscall_tab.cc b/src/autodep/syscall_tab.cc index 21fa2abc..8925fccc 100644 --- a/src/autodep/syscall_tab.cc +++ b/src/autodep/syscall_tab.cc @@ -37,7 +37,7 @@ ::pair fix_cwd( char* buf , size_t buf_sz , ssize_t sz , Bool3 al } // return null terminated string pointed by src in process pid's space -static ::string _get_str( pid_t pid , uint64_t src ) { +[[maybe_unused]] static ::string _get_str( pid_t pid , uint64_t src ) { if (!pid) return {reinterpret_cast(src)} ; ::string res ; errno = 0 ; @@ -53,7 +53,7 @@ static ::string _get_str( pid_t pid , uint64_t src ) { } // copy src to process pid's space @ dst -static void _poke( pid_t pid , uint64_t dst , const char* src , size_t sz ) { +[[maybe_unused]] static void _poke( pid_t pid , uint64_t dst , const char* src , size_t sz ) { SWEAR(pid) ; errno = 0 ; for( size_t chunk ; sz ; src+=chunk , dst+=chunk , sz-=chunk) { // invariant : copy src[i:sz] to dst @@ -71,7 +71,7 @@ static void _poke( pid_t pid , uint64_t dst , const char* src , size_t sz ) { } // copy src to process pid's space @ dst -static void _peek( pid_t pid , char* dst , uint64_t src , size_t sz ) { +[[maybe_unused]] static void _peek( pid_t pid , char* dst , uint64_t src , size_t sz ) { SWEAR(pid) ; errno = 0 ; for( size_t chunk ; sz ; src+=chunk , dst+=chunk , sz-=chunk) { // invariant : copy src[i:sz] to dst @@ -83,14 +83,14 @@ static void _peek( pid_t pid , char* dst , uint64_t src , size_t sz ) { } } -template static Record::Path _path( pid_t pid , uint64_t const* args ) { +template [[maybe_unused]] static Record::Path _path( pid_t pid , uint64_t const* args ) { if (At) return { Fd(args[0]) , _get_str(pid,args[1]) } ; else return { _get_str(pid,args[0]) } ; } // updating args is meaningful only when processing calls to the syscall function with ld_audit & ld_preload // when autodep is ptrace, tmp mapping is not supported and such args updating is ignored as args have been copied from tracee and are not copied back to it -template void _update( uint64_t* args , Record::Path const& p ) { +template [[maybe_unused]] static void _update( uint64_t* args , Record::Path const& p ) { SWEAR(p.has_at==At) ; if (At) args[0 ] = p.at ; /**/ args[At] = reinterpret_cast(p.file) ; @@ -98,7 +98,7 @@ template void _update( uint64_t* args , Record::Path const& p ) { static constexpr int FlagAlways = -1 ; static constexpr int FlagNever = -2 ; -template bool _flag( uint64_t args[6] , int flag ) { +template [[maybe_unused]] static bool _flag( uint64_t args[6] , int flag ) { switch (FlagArg) { case FlagAlways : return true ; case FlagNever : return false ; @@ -107,13 +107,13 @@ template bool _flag( uint64_t args[6] , int flag ) { } // chdir -template void _entry_chdir( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { +template [[maybe_unused]] static void _entry_chdir( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { try { if (At) { Record::Chdir* cd = new Record::Chdir( r , {Fd(args[0]) } , comment ) ; ctx = cd ; } else { Record::Chdir* cd = new Record::Chdir( r , {_path(pid,args+0)} , comment ) ; ctx = cd ; _update(args+0,*cd) ; } } catch (int) {} } -static int64_t/*res*/ _exit_chdir( void* ctx , Record& r , pid_t pid , int64_t res ) { +[[maybe_unused]] static int64_t/*res*/ _exit_chdir( void* ctx , Record& r , pid_t pid , int64_t res ) { if (!ctx) return res ; Record::Chdir* cd = static_cast(ctx) ; (*cd)(r,res,pid) ; @@ -122,14 +122,14 @@ static int64_t/*res*/ _exit_chdir( void* ctx , Record& r , pid_t pid , int64_t r } // chmod -template void _entry_chmod( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { +template [[maybe_unused]] static void _entry_chmod( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { try { Record::Chmod* cm = new Record::Chmod( r , _path(pid,args+0) , args[1+At]&S_IXUSR , _flag(args,AT_SYMLINK_NOFOLLOW) , comment ) ; ctx = cm ; _update(args+0,*cm) ; } catch (int) {} } -static int64_t/*res*/ _exit_chmod( void* ctx , Record& r , pid_t , int64_t res ) { +[[maybe_unused]] static int64_t/*res*/ _exit_chmod( void* ctx , Record& r , pid_t , int64_t res ) { if (!ctx) return res ; Record::Chmod* cm = static_cast(ctx) ; (*cm)(r,res) ; @@ -138,7 +138,7 @@ static int64_t/*res*/ _exit_chmod( void* ctx , Record& r , pid_t , int64_t res ) } // creat -static void _entry_creat( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { +[[maybe_unused]] static void _entry_creat( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { try { Record::Open* o = new Record::Open( r , _path(pid,args+0) , O_WRONLY|O_CREAT|O_TRUNC , comment ) ; ctx = o ; @@ -150,7 +150,7 @@ static void _entry_creat( void* & ctx , Record& r , pid_t pid , uint64_t args[6] // execve // must be called before actual syscall execution as after execution, info is no more available -template static void _entry_execve( void* & /*ctx*/ , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { +template [[maybe_unused]] static void _entry_execve( void* & /*ctx*/ , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { try { Record::Exec e{ r , _path(pid,args+0) , _flag(args,AT_SYMLINK_NOFOLLOW) , comment } ; _update(args+0,e) ; @@ -159,11 +159,11 @@ template static void _entry_execve( void* & /*ctx*/ , Recor // getcwd // getcwd is only necessary if tmp is mapped (not in table with ptrace) -static void _entry_getcwd( void* & ctx , Record& , pid_t , uint64_t args[6] , const char* /*comment*/ ) { +[[maybe_unused]] static void _entry_getcwd( void* & ctx , Record& , pid_t , uint64_t args[6] , const char* /*comment*/ ) { size_t* sz = new size_t{args[1]} ; ctx = sz ; } -static int64_t/*res*/ _exit_getcwd( void* ctx , Record& , pid_t pid , int64_t res ) { +[[maybe_unused]] static int64_t/*res*/ _exit_getcwd( void* ctx , Record& , pid_t pid , int64_t res ) { if (!res ) return res ; // in case of error, man getcwd says buffer is undefined => nothing to do if (!Record::s_has_tmp_view()) return res ; // no tmp mapping => nothing to do SWEAR(pid==0,pid) ; // tmp mapping is not supported with ptrace (need to report fixed result to caller) @@ -175,7 +175,7 @@ static int64_t/*res*/ _exit_getcwd( void* ctx , Record& , pid_t pid , int64_t re } // hard link -template void _entry_lnk( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { +template [[maybe_unused]] static void _entry_lnk( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { try { Record::Lnk* l = new Record::Lnk( r , _path(pid,args+0) , _path(pid,args+1+At) , _flag(args,AT_SYMLINK_NOFOLLOW) , comment ) ; ctx = l ; @@ -183,7 +183,7 @@ template void _entry_lnk( void* & ctx , Record& r , pid_t p _update(args+2,l->dst) ; } catch (int) {} } -static int64_t/*res*/ _exit_lnk( void* ctx , Record& r , pid_t /*pid */, int64_t res ) { +[[maybe_unused]] static int64_t/*res*/ _exit_lnk( void* ctx , Record& r , pid_t /*pid */, int64_t res ) { if (!ctx) return res ; Record::Lnk* l = static_cast(ctx) ; (*l)(r,res) ; @@ -192,7 +192,7 @@ static int64_t/*res*/ _exit_lnk( void* ctx , Record& r , pid_t /*pid */, int64_t } // mkdir -template void _entry_mkdir( void* & /*ctx*/ , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { +template [[maybe_unused]] static void _entry_mkdir( void* & /*ctx*/ , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { try { Record::Mkdir m{ r , _path(pid,args+0) , comment } ; _update(args+0,m) ; @@ -200,7 +200,7 @@ template void _entry_mkdir( void* & /*ctx*/ , Record& r , pid_t pid , u } // open -template void _entry_open( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { +template [[maybe_unused]] static void _entry_open( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { try { Record::Open* o = new Record::Open( r , _path(pid,args+0) , args[1+At]/*flags*/ , comment ) ; ctx = o ; @@ -208,7 +208,7 @@ template void _entry_open( void* & ctx , Record& r , pid_t pid , uint64 } catch (int) {} } -static int64_t/*res*/ _exit_open( void* ctx , Record& r , pid_t /*pid*/ , int64_t res ) { +[[maybe_unused]] static int64_t/*res*/ _exit_open( void* ctx , Record& r , pid_t /*pid*/ , int64_t res ) { if (!ctx) return res ; Record::Open* o = static_cast(ctx) ; (*o)( r , res ) ; @@ -218,7 +218,7 @@ static int64_t/*res*/ _exit_open( void* ctx , Record& r , pid_t /*pid*/ , int64_ // read_lnk using RLB = ::pair ; -template void _entry_read_lnk( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { +template [[maybe_unused]] static void _entry_read_lnk( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { try { uint64_t orig_buf = args[At+1] ; size_t sz = args[At+2] ; @@ -228,7 +228,7 @@ template void _entry_read_lnk( void* & ctx , Record& r , pid_t pid , ui _update(args+0,rlb->first) ; } catch (int) {} } -static int64_t/*res*/ _exit_read_lnk( void* ctx , Record& r , pid_t pid , int64_t res ) { +[[maybe_unused]] static int64_t/*res*/ _exit_read_lnk( void* ctx , Record& r , pid_t pid , int64_t res ) { if (!ctx) return res ; RLB* rlb = static_cast(ctx) ; SWEAR( res<=ssize_t(rlb->first.sz) , res , rlb->first.sz ) ; @@ -243,7 +243,7 @@ static int64_t/*res*/ _exit_read_lnk( void* ctx , Record& r , pid_t pid , int64_ } // rename -template void _entry_rename( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { +template [[maybe_unused]] static void _entry_rename( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { try { #ifdef RENAME_EXCHANGE bool exchange = _flag(args,RENAME_EXCHANGE) ; @@ -261,7 +261,7 @@ template void _entry_rename( void* & ctx , Record& r , pid_ _update(args+2,rn->dst) ; } catch (int) {} } -static int64_t/*res*/ _exit_rename( void* ctx , Record& r , pid_t /*pid*/ , int64_t res ) { +[[maybe_unused]] static int64_t/*res*/ _exit_rename( void* ctx , Record& r , pid_t /*pid*/ , int64_t res ) { if (!ctx) return res ; Record::Rename* rn = static_cast(ctx) ; (*rn)(r,res) ; @@ -270,14 +270,14 @@ static int64_t/*res*/ _exit_rename( void* ctx , Record& r , pid_t /*pid*/ , int6 } // symlink -template void _entry_sym_lnk( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { +template [[maybe_unused]] static void _entry_sym_lnk( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { try { Record::Symlnk* sl = new Record::Symlnk( r , _path(pid,args+1) , comment ) ; ctx = sl ; _update(args+1,*sl) ; } catch (int) {} } -static int64_t/*res*/ _exit_sym_lnk( void* ctx , Record& r , pid_t , int64_t res ) { +[[maybe_unused]] static int64_t/*res*/ _exit_sym_lnk( void* ctx , Record& r , pid_t , int64_t res ) { if (!ctx) return res ; Record::Symlnk* sl = static_cast(ctx) ; (*sl)(r,res) ; @@ -286,7 +286,7 @@ static int64_t/*res*/ _exit_sym_lnk( void* ctx , Record& r , pid_t , int64_t res } // unlink -template void _entry_unlnk( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { +template [[maybe_unused]] static void _entry_unlnk( void* & ctx , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { try { bool rmdir = _flag(args,AT_REMOVEDIR) ; Record::Unlnk* u = new Record::Unlnk( r , _path(pid,args+0) , rmdir , comment ) ; @@ -294,7 +294,7 @@ template void _entry_unlnk( void* & ctx , Record& r , pid_t _update(args+0,*u) ; } catch (int) {} } -static int64_t/*res*/ _exit_unlnk( void* ctx , Record& r , pid_t , int64_t res ) { +[[maybe_unused]] static int64_t/*res*/ _exit_unlnk( void* ctx , Record& r , pid_t , int64_t res ) { if (!ctx) return res ; Record::Unlnk* u = static_cast(ctx) ; (*u)(r,res) ; @@ -303,7 +303,7 @@ static int64_t/*res*/ _exit_unlnk( void* ctx , Record& r , pid_t , int64_t res ) } // access -template void _entry_stat( void* & /*ctx*/ , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { +template [[maybe_unused]] static void _entry_stat( void* & /*ctx*/ , Record& r , pid_t pid , uint64_t args[6] , const char* comment ) { try { Record::Stat s{ r , _path(pid,args+0) , _flag(args,AT_SYMLINK_NOFOLLOW) , comment } ; _update(args+0,s) ; diff --git a/src/client.cc b/src/client.cc index fc094543..861ce1e4 100644 --- a/src/client.cc +++ b/src/client.cc @@ -5,6 +5,7 @@ #include "app.hh" #include "disk.hh" +#include "msg.hh" #include "process.hh" #include "rpc_client.hh" #include "time.hh" @@ -144,8 +145,8 @@ static Bool3 is_reverse_video( Fd in_fd , Fd out_fd ) { if (::write(out_fd,&c,1)!=1) throw "cannot send request"s ; trace("sent",STR(fg),mk_printable(reqs[fg])) ; for(;;) { - char c = 0/*garbage*/ ; - ::vector events = epoll.wait(500'000'000/*ns*/) ; // 500ms, normal reaction time is 20-50ms + char c = 0/*garbage*/ ; + ::vector events = epoll.wait(Delay(0.5)) ; // normal reaction time is 20-50ms SWEAR( events.size()<=1 , events.size() ) ; if (!events.size() ) throw "timeout"s ; // there is a single fd, there may not be more than 1 event SWEAR( events[0].fd()==in_fd , events[0].fd() , in_fd ) ; @@ -208,7 +209,7 @@ Bool3/*ok*/ _out_proc( ::vector_s* files , ReqProc proc , bool refresh , ReqSynt ReqRpcReply report = IMsgBuf().receive(g_server_fds.in) ; switch (report.proc) { case Proc::None : trace("done" ) ; goto Return ; - case Proc::Status : trace("status",STR(report.ok)) ; rc = No|report.ok ; goto Return ; // XXX : why is it necessary goto Return here ? ... + case Proc::Status : trace("status",STR(report.ok)) ; rc = No|report.ok ; goto Return ; // XXX : why is it necessary to goto Return here ? ... case Proc::File : trace("file" ,report.txt ) ; SWEAR(files) ; files->push_back(report.txt) ; break ; // ... we should receive None when server closes stream case Proc::Txt : ::cout << report.txt << flush ; break ; DF} diff --git a/src/fd.cc b/src/fd.cc index ae02c92c..d5b21c2d 100644 --- a/src/fd.cc +++ b/src/fd.cc @@ -15,15 +15,19 @@ ostream& operator<<( ostream& os , Epoll::Event const& e ) { return os << "Event(" << e.fd() <<','<< e.data() <<')' ; } -::vector Epoll::wait(uint64_t timeout_ns) const { +::vector Epoll::wait(Delay timeout) const { + if (!cnt) { + SWEAR(timeout0 && timeout_nsDelay() && timeout!=Delay::Forever ; if (has_timeout) { ::clock_gettime(CLOCK_MONOTONIC,&now) ; - end.tv_sec = now.tv_sec + timeout_ns/1'000'000'000l ; - end.tv_nsec = now.tv_nsec + timeout_ns%1'000'000'000l ; + end.tv_sec = now.tv_sec + timeout.sec() ; + end.tv_nsec = now.tv_nsec + timeout.nsec_in_s() ; if (end.tv_nsec>=1'000'000'000l) { end.tv_nsec -= 1'000'000'000l ; end.tv_sec += 1 ; @@ -41,7 +45,7 @@ ::vector Epoll::wait(uint64_t timeout_ns) const { wait_ms = wait_s * 1'000 ; wait_ms += (end.tv_nsec-now.tv_nsec) / 1'000'000l ; // protect against possible conversion to time_t which may be unsigned } else { - wait_ms = timeout_ns ? -1 : 0 ; + wait_ms = +timeout ? -1 : 0 ; } cnt_ = ::epoll_wait( fd , events.data() , cnt , wait_ms ) ; switch (cnt_) { diff --git a/src/fd.hh b/src/fd.hh index e5aa7178..c11a1533 100644 --- a/src/fd.hh +++ b/src/fd.hh @@ -9,6 +9,7 @@ #include #include +#include "serialize.hh" #include "time.hh" ::string host() ; @@ -56,6 +57,9 @@ struct Fd { void cloexec(bool set=true) { ::fcntl(fd,F_SETFD,set?FD_CLOEXEC:0) ; } + template void serdes(T& s) { + ::serdes(s,fd) ; + } // data int fd = -1 ; } ; @@ -213,7 +217,6 @@ namespace std { // struct Epoll { - static constexpr uint64_t Forever = -1 ; struct Event : epoll_event { friend ::ostream& operator<<( ::ostream& , Event const& ) ; // cxtors & casts @@ -244,7 +247,7 @@ struct Epoll { swear_prod(rc==0,"cannot del",fd_,"from epoll",fd,'(',strerror(errno),')') ; cnt -= wait ; } - ::vector wait(uint64_t timeout_ns=Forever) const ; + ::vector wait(Time::Delay timeout=Time::Delay::Forever) const ; void close() { fd .close() ; } diff --git a/src/job_exec.cc b/src/job_exec.cc index 0407cfc4..940eebbf 100644 --- a/src/job_exec.cc +++ b/src/job_exec.cc @@ -42,60 +42,17 @@ struct PatternDict { ::vmap patterns = {} ; } ; -ServerSockFd g_server_fd ; -Gather g_gather { New } ; +Gather g_gather ; JobRpcReply g_start_info ; ::string g_service_start ; ::string g_service_mngt ; ::string g_service_end ; SeqId g_seq_id = 0/*garbage*/ ; JobIdx g_job = 0/*garbage*/ ; -::atomic g_killed = false ; // written by thread S and read by main thread PatternDict g_match_dct ; NfsGuard g_nfs_guard ; ::vector_s g_washed ; -void kill_thread_func(::stop_token stop) { - t_thread_key = 'K' ; - Trace trace("kill_thread_func",g_start_info.kill_sigs) ; - for( size_t i=0 ;; i++ ) { - int sig = i ) if (info.read[+a]>info.write) ad.accesses &= ~a ; bool is_dep = ad.dflags[Dflag::Static] ; - if (flags.is_target!=Yes) // if a target or a side target, it is a target since the beginning + Access dep_access = {} ; + Pdate dep_date = Pdate::Future ; + if (flags.is_target!=Yes) { // if a target or a side target, it is a target since the beginning for( Access a : All ) - if ( ad.accesses[a] && info.read[+a]<=info.target ) { is_dep = true ; break ; } + if ( ad.accesses[a] && info.read[+a] cur_deps_cb() { return analyze(false/*at_end*/).deps ; } ::vector_s cmd_line() { ::vector_s cmd_line = ::move(g_start_info.interpreter) ; // avoid copying as interpreter is used only here @@ -279,20 +235,6 @@ ::vector_s cmd_line() { return cmd_line ; } -void live_out_cb(::string_view const& txt) { - static ::string live_out_buf ; // used to store incomplete last line to have line coherent chunks - // could be slightly optimized, but when generating live output, we have very few jobs, no need to optimize - live_out_buf.append(txt) ; - size_t pos = live_out_buf.rfind('\n') ; - Trace trace("live_out_cb",live_out_buf.size(),txt.size(),pos+1) ; - if (pos==Npos) return ; - pos++ ; - //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - OMsgBuf().send( ClientSockFd(g_service_mngt) , JobRpcReq( JobProc::LiveOut , g_seq_id , g_job , live_out_buf.substr(0,pos) ) ) ; - //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - live_out_buf = live_out_buf.substr(pos) ; -} - void crc_thread_func( size_t id , vmap_s* targets , ::vector const* crcs , ::string* msg , Mutex* msg_mutex ) { static ::atomic crc_idx = 0 ; t_thread_key = '0'+id ; @@ -328,7 +270,8 @@ ::string compute_crcs(Digest& digest) { } int main( int argc , char* argv[] ) { - Pdate start_overhead = Pdate(New) ; + Pdate start_overhead = Pdate(New) ; + ServerSockFd server_fd { New } ; // server socket must be listening before connecting to server and last to the very end to ensure we can handle heartbeats // swear_prod(argc==8,argc) ; // syntax is : job_exec server:port/*start*/ server:port/*mngt*/ server:port/*end*/ seq_id job_idx root_dir trace_file g_service_start = argv[1] ; @@ -353,17 +296,14 @@ int main( int argc , char* argv[] ) { trace("pid",::getpid(),::getpgrp()) ; trace("start_overhead",start_overhead) ; // - ServerThread server_thread{'S',handle_server_req} ; - // - JobRpcReq req_info { JobProc::Start , g_seq_id , g_job , server_thread.fd.port() } ; - bool found_server = false ; + bool found_server = false ; try { ClientSockFd fd {g_service_start,NConnectionTrials} ; found_server = true ; - // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - /**/ OMsgBuf().send ( fd , req_info ) ; - g_start_info = IMsgBuf().receive( fd ) ; - // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + /**/ OMsgBuf().send ( fd , JobRpcReq{JobProc::Start,g_seq_id,g_job,server_fd.port()} ) ; + g_start_info = IMsgBuf().receive( fd ) ; + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ } catch (::string const& e) { trace("no_server",g_service_start,STR(found_server),e) ; if (found_server) exit(Rc::Fail ) ; // this is typically a ^C @@ -371,8 +311,8 @@ int main( int argc , char* argv[] ) { } trace("g_start_info",g_start_info) ; switch (g_start_info.proc) { - case JobProc::None : return 0 ; // server ask us to give up - case JobProc::Start : break ; // normal case + case JobProc::None : return 0 ; // server ask us to give up + case JobProc::Start : break ; // normal case DF} g_nfs_guard.reliable_dirs = g_start_info.autodep_env.reliable_dirs ; // @@ -384,19 +324,22 @@ int main( int argc , char* argv[] ) { try { cmd_env = prepare_env(end_report) ; } catch (::string const& e) { end_report.msg += e ; goto End ; } // - /**/ g_gather.addr = g_start_info.addr ; - /**/ g_gather.autodep_env = g_start_info.autodep_env ; - /**/ g_gather.chroot = g_start_info.chroot ; - /**/ g_gather.as_session = true ; - /**/ g_gather.cwd = g_start_info.cwd_s ; if (+g_gather.cwd) g_gather.cwd.pop_back() ; - /**/ g_gather.env = &cmd_env ; - /**/ g_gather.kill_sigs = g_start_info.kill_sigs ; - if (g_start_info.live_out) g_gather.live_out_cb = live_out_cb ; - /**/ g_gather.method = g_start_info.method ; - /**/ g_gather.server_cb = server_cb ; - /**/ g_gather.timeout = g_start_info.timeout ; - /**/ g_gather.network_delay = g_start_info.network_delay ; - /**/ g_gather.kill_job_cb = kill_job ; + g_gather.addr = g_start_info.addr ; + g_gather.as_session = true ; + g_gather.autodep_env = g_start_info.autodep_env ; + g_gather.chroot = g_start_info.chroot ; + g_gather.cur_deps_cb = cur_deps_cb ; + g_gather.cwd = g_start_info.cwd_s ; if (+g_gather.cwd) g_gather.cwd.pop_back() ; + g_gather.env = &cmd_env ; + g_gather.job = g_job ; + g_gather.kill_sigs = g_start_info.kill_sigs ; + g_gather.live_out = g_start_info.live_out ; + g_gather.method = g_start_info.method ; + g_gather.network_delay = g_start_info.network_delay ; + g_gather.seq_id = g_seq_id ; + g_gather.server_master_fd = ::move(server_fd) ; + g_gather.service_mngt = g_service_mngt ; + g_gather.timeout = g_start_info.timeout ; // trace("wash",g_start_info.pre_actions) ; ::pair_s wash_report = do_file_actions( g_washed , ::move(g_start_info.pre_actions) , g_nfs_guard , g_start_info.hash_algo ) ; @@ -415,32 +358,28 @@ int main( int argc , char* argv[] ) { g_gather.new_target( start_overhead , g_start_info.stdout , "" ) ; child_stdout.no_std() ; } + // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + Status status = g_gather.exec_child( cmd_line() , child_stdin , child_stdout , Child::Pipe ) ; + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + struct rusage rsrcs ; getrusage(RUSAGE_CHILDREN,&rsrcs) ; // - Pdate start_job = New ; // as late as possible before child starts - // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - Status status = g_gather.exec_child( cmd_line() , child_stdin , child_stdout , Child::Pipe ) ; - // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Pdate end_job = New ; // as early as possible after child ends - bool killed = g_killed ; // sample g_killed to ensure coherence (even if status is correct, it may mean we were waiting for stdout/stderr) - struct rusage rsrcs ; getrusage(RUSAGE_CHILDREN,&rsrcs) ; - trace("start_job",start_job,"end_job",end_job) ; - // - Digest digest = analyze(true/*at_end*/,killed) ; + Digest digest = analyze(true/*at_end*/,status==Status::Killed) ; + trace("analysis",g_gather.start_time,g_gather.end_time,status,g_gather.msg,digest.msg) ; // end_report.msg += compute_crcs(digest) ; // - if (!g_start_info.autodep_env.reliable_dirs) { // fast path : avoid listing targets & guards if reliable_dirs - for( auto const& [t,_] : digest.targets ) g_nfs_guard.change(t) ; // protect against NFS strange notion of coherence while computing crcs - for( auto const& f : g_gather.guards ) g_nfs_guard.change(f) ; // . + if (!g_start_info.autodep_env.reliable_dirs) { // fast path : avoid listing targets & guards if reliable_dirs + for( auto const& [t,_] : digest.targets ) g_nfs_guard.change(t) ; // protect against NFS strange notion of coherence while computing crcs + for( auto const& f : g_gather.guards ) g_nfs_guard.change(f) ; // . g_nfs_guard.close() ; } // if ( g_gather.seen_tmp && !g_start_info.keep_tmp ) - try { unlnk_inside(g_start_info.autodep_env.tmp_dir) ; } catch (::string const&) {} // cleaning is done at job start any way, so no harm + try { unlnk_inside(g_start_info.autodep_env.tmp_dir) ; } catch (::string const&) {} // cleaning is done at job start any way, so no harm // - if ( killed ) { trace("killed" ) ; status = Status::Killed ; } - else if ( status==Status::Ok && +digest.msg ) { trace("analysis_err") ; status = Status::Err ; } - append_to_string( end_report.msg , g_gather.msg , digest.msg ) ; + if ( status==Status::Ok && +digest.msg ) status = Status::Err ; + /**/ end_report.msg += g_gather.msg ; + if (status!=Status::Killed) end_report.msg += digest .msg ; end_report.digest = { .status = status , .targets { ::move(digest.targets ) } @@ -448,10 +387,10 @@ int main( int argc , char* argv[] ) { , .stderr { ::move(g_gather.stderr) } , .stdout { ::move(g_gather.stdout) } , .wstatus = g_gather.wstatus - , .end_date = end_job + , .end_date = g_gather.end_time , .stats{ .cpu { Delay(rsrcs.ru_utime) + Delay(rsrcs.ru_stime) } - , .job { end_job - start_job } + , .job { g_gather.end_time-g_gather.start_time } , .mem = size_t(rsrcs.ru_maxrss<<10) } } ; @@ -461,7 +400,7 @@ End : try { ClientSockFd fd { g_service_end , NConnectionTrials } ; Pdate end_overhead = New ; - end_report.digest.stats.total = end_overhead - start_overhead ; // measure overhead as late as possible + end_report.digest.stats.total = end_overhead - start_overhead ; // measure overhead as late as possible //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv OMsgBuf().send( fd , end_report ) ; //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/src/lmakeserver/backend.cc b/src/lmakeserver/backend.cc index 7db1cf24..3cde504a 100644 --- a/src/lmakeserver/backend.cc +++ b/src/lmakeserver/backend.cc @@ -14,6 +14,14 @@ using namespace Engine ; namespace Backends { + void send_reply( JobIdx job , JobMngtRpcReply&& jmrr ) { + Lock lock { Backend::_s_mutex } ; + Backend::StartEntry::Conn const& conn = Backend::_s_start_tab.at(job).conn ; + ClientSockFd fd ( conn.host , conn.port ) ; + jmrr.seq_id = conn.seq_id ; + OMsgBuf().send( fd , jmrr ) ; // XXX : straighten out Fd : Fd must not detach on mv and Epoll must take an AutoCloseFd as arg to take close resp. + } + // // Backend::* // @@ -53,9 +61,9 @@ namespace Backends { ::atomic Backend::_s_starting_job ; ::map Backend::_s_start_tab ; SmallIds Backend::_s_small_ids ; - Backend::JobExecThread * Backend::_s_job_start_thread = nullptr ; - Backend::JobExecThread * Backend::_s_job_mngt_thread = nullptr ; - Backend::JobExecThread * Backend::_s_job_end_thread = nullptr ; + Backend::JobThread * Backend::_s_job_start_thread = nullptr ; + Backend::JobMngtThread * Backend::_s_job_mngt_thread = nullptr ; + Backend::JobThread * Backend::_s_job_end_thread = nullptr ; Backend::DeferredThread* Backend::_s_deferred_report_thread = nullptr ; Backend::DeferredThread* Backend::_s_deferred_wakeup_thread = nullptr ; @@ -147,11 +155,11 @@ namespace Backends { _s_handle_job_end( JobRpcReq( JobProc::End , de.seq_id , +de.job_exec , ::move(jd) ) ) ; } - void Backend::_s_wakeup_remote( JobIdx job , StartEntry::Conn const& conn , FullDate const& start_date , JobServerRpcProc proc ) { + void Backend::_s_wakeup_remote( JobIdx job , StartEntry::Conn const& conn , FullDate const& start_date , JobMngtProc proc ) { Trace trace(BeChnl,"_s_wakeup_remote",job,conn,proc) ; try { ClientSockFd fd(conn.host,conn.port) ; - OMsgBuf().send( fd , JobServerRpcReq(proc,conn.seq_id,job) ) ; // XXX : straighten out Fd : Fd must not detach on mv and Epoll must take an AutoCloseFd as arg to take close resp. + OMsgBuf().send( fd , JobMngtRpcReply(proc,conn.seq_id) ) ; // XXX : straighten out Fd : Fd must not detach on mv and Epoll must take an AutoCloseFd as arg to take close resp. } catch (::string const& e) { trace("no_job",job,e) ; // if job cannot be connected to, assume it is dead and pretend it died if it still exists after network delay @@ -402,31 +410,33 @@ namespace Backends { return false/*keep_fd*/ ; } - bool/*keep_fd*/ Backend::_s_handle_job_mngt( JobRpcReq&& jrr , SlaveSockFd const& fd ) { - switch (jrr.proc) { - case JobProc::None : return false ; // if connection is lost, ignore it - case JobProc::ChkDeps : - case JobProc::DepInfos : - case JobProc::Decode : - case JobProc::Encode : SWEAR(+fd,jrr.proc) ; break ; // fd is needed to reply - case JobProc::LiveOut : break ; // no reply + bool/*keep_fd*/ Backend::_s_handle_job_mngt( JobMngtRpcReq&& jmrr , SlaveSockFd const& fd ) { + switch (jmrr.proc) { + case JobMngtProc::None : return false ; // if connection is lost, ignore it + case JobMngtProc::ChkDeps : + case JobMngtProc::DepInfos : + case JobMngtProc::Decode : + case JobMngtProc::Encode : SWEAR(+fd,jmrr.proc) ; break ; // fd is needed to reply + case JobMngtProc::LiveOut : break ; // no reply DF} - Job job { jrr.job } ; - Trace trace(BeChnl,"_s_handle_job_mngt",jrr) ; - { Lock lock { _s_mutex } ; // prevent sub-backend from manipulating _s_start_tab from main thread, lock for minimal time - // keep_fd - auto it = _s_start_tab.find(+job) ; if (it==_s_start_tab.end() ) { trace("not_in_tab" ) ; return false ; } - StartEntry& entry = it->second ; if (entry.conn.seq_id!=jrr.seq_id) { trace("bad_seq_id",entry.conn.seq_id,jrr.seq_id) ; return false ; } + Job job { jmrr.job } ; + Trace trace(BeChnl,"_s_handle_job_mngt",jmrr) ; + { Lock lock { _s_mutex } ; // prevent sub-backend from manipulating _s_start_tab from main thread, lock for minimal time + // keep_fd + auto it = _s_start_tab.find(+job) ; if (it==_s_start_tab.end() ) { trace("not_in_tab" ) ; return false ; } + StartEntry& entry = it->second ; if (entry.conn.seq_id!=jmrr.seq_id) { trace("bad_seq_id",entry.conn.seq_id,jmrr.seq_id) ; return false ; } trace("entry",job,entry) ; - switch (jrr.proc) { //! vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv keep_fd - case JobProc::ChkDeps : - case JobProc::DepInfos : g_engine_queue . emplace( jrr.proc , JobExec(job,entry.conn.host,entry.start_date,New) , ::move(jrr.digest.deps) , fd ) ; return true ; - case JobProc::LiveOut : g_engine_queue . emplace( jrr.proc , JobExec(job,entry.conn.host,entry.start_date,New) , ::move(jrr.msg) ) ; return false ; - case JobProc::Decode : Codec::g_codec_queue->emplace( jrr.proc , ::move(jrr.msg) , ::move(jrr.file) , ::move(jrr.ctx) , entry.reqs , fd ) ; return true ; - case JobProc::Encode : Codec::g_codec_queue->emplace( jrr.proc , ::move(jrr.msg) , ::move(jrr.file) , ::move(jrr.ctx) , jrr.min_len , entry.reqs , fd ) ; return true ; - // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + switch (jmrr.proc) { //! vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + case JobMngtProc::ChkDeps : + case JobMngtProc::DepInfos : g_engine_queue.emplace( jmrr.proc , JobExec(job,entry.conn.host,entry.start_date,New) , jmrr.fd , ::move(jmrr.deps) ) ; break ; + case JobMngtProc::LiveOut : g_engine_queue.emplace( jmrr.proc , JobExec(job,entry.conn.host,entry.start_date,New) , ::move(jmrr.txt) ) ; break ; + // + case JobMngtProc::Decode : Codec::g_codec_queue->emplace( jmrr.proc , +job , jmrr.fd , ::move(jmrr.txt) , ::move(jmrr.file) , ::move(jmrr.ctx) , entry.reqs ) ; break ; + case JobMngtProc::Encode : Codec::g_codec_queue->emplace( jmrr.proc , +job , jmrr.fd , ::move(jmrr.txt) , ::move(jmrr.file) , ::move(jmrr.ctx) , jmrr.min_len , entry.reqs ) ; break ; + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ DF} } + return false/*keep_fd*/ ; } bool/*keep_fd*/ Backend::_s_handle_job_end( JobRpcReq&& jrr , SlaveSockFd const& ) { @@ -519,9 +529,9 @@ namespace Backends { } } } - // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - for( auto const& [j,c] : to_wakeup ) _s_wakeup_remote( j , c.first , c.second , JobServerRpcProc::Kill ) ; - // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + for( auto const& [j,c] : to_wakeup ) _s_wakeup_remote( j , c.first , c.second , JobMngtProc::Kill ) ; + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ } void Backend::_s_heartbeat_thread_func(::stop_token stop) { @@ -584,7 +594,7 @@ namespace Backends { goto Next ; } Wakeup : - _s_wakeup_remote(job,conn,start_date,JobServerRpcProc::Heartbeat) ; + _s_wakeup_remote(job,conn,start_date,JobMngtProc::Heartbeat) ; Next : if (!g_config.heartbeat_tick.sleep_for(stop)) break ; // limit job checks continue ; @@ -599,9 +609,9 @@ namespace Backends { void Backend::s_config( ::array> const& config , bool dynamic ) { static ::jthread heartbeat_thread { _s_heartbeat_thread_func } ; - static JobExecThread job_start_thread {'S',_s_handle_job_start ,1000/*backlog*/} ; _s_job_start_thread = &job_start_thread ; - static JobExecThread job_mngt_thread {'M',_s_handle_job_mngt ,1000/*backlog*/} ; _s_job_mngt_thread = &job_mngt_thread ; - static JobExecThread job_end_thread {'E',_s_handle_job_end ,1000/*backlog*/} ; _s_job_end_thread = &job_end_thread ; + static JobThread job_start_thread {'S',_s_handle_job_start ,1000/*backlog*/} ; _s_job_start_thread = &job_start_thread ; + static JobMngtThread job_mngt_thread {'M',_s_handle_job_mngt ,1000/*backlog*/} ; _s_job_mngt_thread = &job_mngt_thread ; + static JobThread job_end_thread {'E',_s_handle_job_end ,1000/*backlog*/} ; _s_job_end_thread = &job_end_thread ; static DeferredThread deferred_report_thread{'R',_s_handle_deferred_report } ; _s_deferred_report_thread = &deferred_report_thread ; static DeferredThread deferred_wakeup_thread{'W',_s_handle_deferred_wakeup } ; _s_deferred_wakeup_thread = &deferred_wakeup_thread ; Trace trace(BeChnl,"s_config",STR(dynamic)) ; diff --git a/src/lmakeserver/backend.x.hh b/src/lmakeserver/backend.x.hh index 97602a23..e17a05d9 100644 --- a/src/lmakeserver/backend.x.hh +++ b/src/lmakeserver/backend.x.hh @@ -39,7 +39,12 @@ using Backends::Backend ; namespace Backends { + void send_reply( JobIdx , JobMngtRpcReply&& ) ; + struct Backend { + + friend void send_reply( JobIdx , JobMngtRpcReply&& ) ; + using SmallId = uint32_t ; using CoarseDelay = Time::CoarseDelay ; using Pdate = Time::Pdate ; @@ -116,17 +121,18 @@ namespace Backends { s_tab[+t] = &be ; } private : - static void _s_kill_req ( ReqIdx=0 ) ; // kill all if req==0 - static void _s_wakeup_remote ( JobIdx , StartEntry::Conn const& , FullDate const& start , JobServerRpcProc ) ; - static void _s_heartbeat_thread_func ( ::stop_token ) ; - static bool/*keep_fd*/ _s_handle_job_start ( JobRpcReq && , SlaveSockFd const& ={} ) ; - static bool/*keep_fd*/ _s_handle_job_mngt ( JobRpcReq && , SlaveSockFd const& ={} ) ; - static bool/*keep_fd*/ _s_handle_job_end ( JobRpcReq && , SlaveSockFd const& ={} ) ; - static void _s_handle_deferred_report( DeferredEntry&& ) ; - static void _s_handle_deferred_wakeup( DeferredEntry&& ) ; - static Status _s_release_start_entry ( ::map::iterator , Status ) ; + static void _s_kill_req ( ReqIdx=0 ) ; // kill all if req==0 + static void _s_wakeup_remote ( JobIdx , StartEntry::Conn const& , FullDate const& start , JobMngtProc ) ; + static void _s_heartbeat_thread_func ( ::stop_token ) ; + static bool/*keep_fd*/ _s_handle_job_start ( JobRpcReq && , SlaveSockFd const& ={} ) ; + static bool/*keep_fd*/ _s_handle_job_mngt ( JobMngtRpcReq&& , SlaveSockFd const& ={} ) ; + static bool/*keep_fd*/ _s_handle_job_end ( JobRpcReq && , SlaveSockFd const& ={} ) ; + static void _s_handle_deferred_report( DeferredEntry&& ) ; + static void _s_handle_deferred_wakeup( DeferredEntry&& ) ; + static Status _s_release_start_entry ( ::map::iterator , Status ) ; // - using JobExecThread = ServerThread ; + using JobThread = ServerThread ; + using JobMngtThread = ServerThread ; using DeferredThread = QueueThread ; // static data public : @@ -134,9 +140,9 @@ namespace Backends { static Backend* s_tab[N] ; private : - static JobExecThread * _s_job_start_thread ; - static JobExecThread * _s_job_mngt_thread ; - static JobExecThread * _s_job_end_thread ; + static JobThread * _s_job_start_thread ; + static JobMngtThread * _s_job_mngt_thread ; + static JobThread * _s_job_end_thread ; static DeferredThread* _s_deferred_report_thread ; static DeferredThread* _s_deferred_wakeup_thread ; static Mutex _s_mutex ; diff --git a/src/lmakeserver/codec.cc b/src/lmakeserver/codec.cc index 81813607..a9b0967f 100644 --- a/src/lmakeserver/codec.cc +++ b/src/lmakeserver/codec.cc @@ -32,9 +32,9 @@ namespace Codec { void codec_thread_func(Closure const& cc) ; ::ostream& operator<<( ::ostream& os , Closure const& cc ) { - /**/ os << "Closure(" << cc.proc ; - if (cc.proc==JobProc::Encode) os <<','<< cc.min_len() ; - return os <<','<< cc.file <<','<< cc.ctx << ',' << cc.txt <<','<< cc.reqs <<')' ; + /**/ os << "Closure(" << cc.proc ; + if (cc.proc==JobMngtProc::Encode) os <<','<< cc.min_len() ; + return os <<','<< cc.file <<','<< cc.ctx << ',' << cc.txt <<','<< cc.reqs <<')' ; } void Closure::s_init() { @@ -211,35 +211,35 @@ namespace Codec { return true/*ok*/ ; } - JobRpcReply Closure::decode() const { + JobMngtRpcReply Closure::decode() const { Trace trace("decode",*this) ; - SWEAR(proc==JobProc::Decode,proc) ; + SWEAR(proc==JobMngtProc::Decode,proc) ; Node decode_node { mk_decode_node(file,ctx,txt) , true/*no_dir*/ } ; bool refreshed = s_refresh( file , +decode_node , reqs ) ; - if (refreshed) { // else codec file not available + if (refreshed) { // else codec file not available if (_buildable_ok(file,decode_node)) { ::string val { decode_node->codec_val().str_view() } ; trace("found",val) ; - return JobRpcReply( JobProc::Decode , val , decode_node->crc , Yes ) ; + return { JobMngtProc::Decode , {}/*seq_id*/ , {}/*fd*/ , val , decode_node->crc , Yes } ; // seq_id and fd will be filled in later } } trace("fail",STR(refreshed)) ; - return JobRpcReply(JobProc::Decode,{}/*val*/,Crc::None,No) ; + return {JobMngtProc::Decode , {}/*seq_id*/ , {}/*fd*/ , {}/*val*/ , Crc::None , No } ; // seq_id and fd will be filled in later } - JobRpcReply Closure::encode() const { + JobMngtRpcReply Closure::encode() const { Trace trace("encode",*this) ; - SWEAR(proc==JobProc::Encode,proc) ; + SWEAR(proc==JobMngtProc::Encode,proc) ; Node encode_node { mk_encode_node(file,ctx,txt) , true/*no_dir*/ } ; if ( !s_refresh( file , +encode_node , reqs ) ) { trace("no_refresh") ; - return JobRpcReply(JobProc::Encode,{}/*code*/,Crc::None,No) ; // codec file not available + return { JobMngtProc::Encode , {}/*seq_id*/ , {}/*fd*/ , {}/*code*/ , Crc::None , No } ; // codec file not available, seq_id and fd will be filled in later } // if (_buildable_ok(file,encode_node)) { ::string code { encode_node->codec_code().str_view() } ; trace("found",code) ; - return JobRpcReply( JobProc::Encode , code , encode_node->crc , Yes ) ; + return { JobMngtProc::Encode , {}/*seq_id*/ , {}/*fd*/ , code , encode_node->crc , Yes } ; // seq_id and fd will be filled in later } // ::string full_code = ::string(Xxh(txt).digest()) ; @@ -250,7 +250,7 @@ namespace Codec { if (!_buildable_ok(file,decode_node)) goto NewCode ; } trace("clash") ; - return JobRpcReply(JobProc::Encode,"crc clash",{},No) ; // this is a true full crc clash + return { JobMngtProc::Encode , {}/*seq_id*/ , {}/*fd*/ , "crc clash" , {} , No } ; // this is a true full crc clash, seq_id and fd will be filled in later NewCode : trace("new_code",code) ; OFStream(file,::ios::app) << _codec_line(ctx,code,txt,true/*with_nl*/) ; @@ -259,10 +259,10 @@ namespace Codec { _create_pair( file , decode_node , txt , encode_node , code ) ; decode_node->date = {entry.log_date,now} ; encode_node->date = {entry.log_date,now} ; - entry.phys_date = file_date(file) ; // we have touched the file but not the semantic, update phys_date but not log_date + entry.phys_date = file_date(file) ; // we have touched the file but not the semantic, update phys_date but not log_date // trace("found",code) ; - return JobRpcReply( JobProc::Encode , code , encode_node->crc , Yes ) ; + return { JobMngtProc::Encode , {}/*seq_id*/ , {}/*fd*/ , code , encode_node->crc , Yes } ; } bool/*ok*/ refresh( NodeIdx ni , ReqIdx r ) { @@ -276,11 +276,13 @@ namespace Codec { } void codec_thread_func(Closure const& cc) { + JobMngtRpcReply jmrr ; switch (cc.proc) { - case JobProc::Decode : OMsgBuf().send( cc.reply_fd , cc.decode() ) ; break ; - case JobProc::Encode : OMsgBuf().send( cc.reply_fd , cc.encode() ) ; break ; + case JobMngtProc::Decode : jmrr = cc.decode() ; break ; + case JobMngtProc::Encode : jmrr = cc.encode() ; break ; DF} - ::close(cc.reply_fd) ; + jmrr.fd = cc.fd ; // seq_id will be filled in by send_reply + Backends::send_reply( cc.job , ::move(jmrr) ) ; } } diff --git a/src/lmakeserver/codec.hh b/src/lmakeserver/codec.hh index b5be9172..a04f9329 100644 --- a/src/lmakeserver/codec.hh +++ b/src/lmakeserver/codec.hh @@ -20,6 +20,7 @@ namespace Codec { struct Closure { friend ::ostream& operator<<( ::ostream& , Closure const& ) ; + using Proc = JobMngtProc ; struct Entry { // log_date is the semantic date, i.e. : // - all decode & encode nodes for this file have this common date @@ -41,54 +42,59 @@ namespace Codec { // cxtors & casts Closure() = default ; Closure( - JobProc p + Proc p + , JobIdx j + , Fd fd_ , ::string&& code , ::string&& f , ::string&& c , ::vector const& rs - , Fd rfd ) : - proc { p } - , reply_fd{ rfd } - , file { ::move(f ) } - , ctx { ::move(c ) } - , txt { ::move(code) } - , reqs { rs } - { SWEAR(p==JobProc::Decode) ; } + proc { p } + , job { j } + , fd { fd_ } + , file { ::move(f ) } + , ctx { ::move(c ) } + , txt { ::move(code) } + , reqs { rs } + { SWEAR(p==Proc::Decode) ; } Closure( - JobProc p + Proc p + , JobIdx j + , Fd fd_ , ::string&& val , ::string&& f , ::string&& c , uint8_t ml , ::vector const& rs - , Fd rfd ) : proc { p } , _min_len { ml } - , reply_fd { rfd } + , job { j } + , fd { fd_ } , file { ::move(f ) } , ctx { ::move(c ) } , txt { ::move(val) } , reqs { rs } - { SWEAR(p==JobProc::Encode) ; } + { SWEAR(p==Proc::Encode) ; } // accesses - uint8_t min_len() const { SWEAR(proc==JobProc::Encode) ; return _min_len ; } + uint8_t min_len() const { SWEAR(proc==Proc::Encode) ; return _min_len ; } // services private : public : - JobRpcReply decode() const ; - JobRpcReply encode() const ; + JobMngtRpcReply decode() const ; + JobMngtRpcReply encode() const ; // data - JobProc proc = {}/*garbage*/ ; + Proc proc = {}/*garbage*/ ; private : uint8_t _min_len = 0/*garbage*/ ; public : - Fd reply_fd ; - ::string file ; - ::string ctx ; - ::string txt ; - ::vector reqs ; + JobIdx job ; + Fd fd ; + ::string file ; + ::string ctx ; + ::string txt ; + ::vector reqs ; } ; struct ValMrkr {} ; diff --git a/src/lmakeserver/core.hh b/src/lmakeserver/core.hh index 0e19c1e6..5cbe796d 100644 --- a/src/lmakeserver/core.hh +++ b/src/lmakeserver/core.hh @@ -10,8 +10,8 @@ #include "config.hh" #include "disk.hh" -#include "fd.hh" #include "hash.hh" +#include "msg.hh" #include "process.hh" #include "thread.hh" #include "time.hh" diff --git a/src/lmakeserver/global.cc b/src/lmakeserver/global.cc index a675a33c..b7c40835 100644 --- a/src/lmakeserver/global.cc +++ b/src/lmakeserver/global.cc @@ -354,11 +354,17 @@ namespace Engine { static ::string _set_dir( ::string const& user_dir , ::string const& key , ::string const& file ) { ::string std_file = to_string(PrivateAdminDir,'/',file) ; - if (!user_dir) return std_file ; + if (!user_dir) { + if (!is_dir(std_file)) unlnk(std_file) ; + return std_file ; + } ::string special_file = to_string(user_dir,'/',key) ; ::string lnk_target = mk_rel(special_file,dir_name(std_file)+'/') ; - if ( ::string t=read_lnk(std_file) ; +t ) swear_prod(t==lnk_target) ; - else lnk( dir_guard(std_file) , lnk_target ) ; + ::string t = read_lnk(std_file) ; + if (t!=lnk_target) { + unlnk( std_file , true/*dir_ok*/ ) ; + lnk ( dir_guard(std_file) , lnk_target ) ; + } return special_file ; } void Config::open(bool dynamic) { @@ -381,41 +387,54 @@ namespace Engine { // EngineClosure // + ::ostream& operator<<( ::ostream& os , EngineClosureGlobal const& ecg ) { + return os << "Glb(" << ecg.proc <<')' ; + } + ::ostream& operator<<( ::ostream& os , EngineClosureReq const& ecr ) { - os << "Req(" << ecr.proc <<',' ; + /**/ os << "Req(" << ecr.proc <<',' ; switch (ecr.proc) { case ReqProc::Debug : // PER_CMD : format for tracing case ReqProc::Forget : case ReqProc::Mark : case ReqProc::Make : case ReqProc::Show : os << ecr.in_fd <<','<< ecr.out_fd <<','<< ecr.options <<','<< ecr.files ; break ; - case ReqProc::Kill : os << ecr.in_fd <<','<< ecr.out_fd ; break ; - case ReqProc::Close : os << ecr.req ; break ; + case ReqProc::Kill : os << ecr.in_fd <<','<< ecr.out_fd ; break ; + case ReqProc::Close : os << ecr.req ; break ; DF} - return os << ')' ; + return os << ')' ; } ::ostream& operator<<( ::ostream& os , EngineClosureJob const& ecj ) { - os << "Job(" << ecj.proc <<','<< ecj.exec ; + /**/ os << "Job(" << ecj.proc <<','<< ecj.job_exec ; switch (ecj.proc) { case JobProc::Start : if (ecj.report) os <<",report" ; break ; - case JobProc::LiveOut : os <<','<< ecj.txt.size() ; break ; - case JobProc::GiveUp : os <<','<< ecj.req ; break ; + case JobProc::GiveUp : os <<','<< ecj.req ; break ; case JobProc::ReportStart : break ; - case JobProc::End : os <<','<< ecj.digest ; break ; - case JobProc::ChkDeps : os <<','<< ecj.digest.deps ; break ; + case JobProc::End : os <<','<< ecj.digest ; break ; + DF} + return os << ')' ; + } + + ::ostream& operator<<( ::ostream& os , EngineClosureJobMngt const& ecjm ) { + /**/ os << "JobMngt(" << ecjm.proc <<','<< ecjm.job_exec ; + switch (ecjm.proc) { + case JobMngtProc::LiveOut : os <<','<< ecjm.txt.size() ; break ; + case JobMngtProc::DepInfos : os <<','<< ecjm.deps ; break ; + case JobMngtProc::ChkDeps : os <<','<< ecjm.deps ; break ; DF} - return os << ')' ; + return os << ')' ; } ::ostream& operator<<( ::ostream& os , EngineClosure const& ec ) { - os << "EngineClosure(" << ec.kind <<',' ; + /**/ os << "EngineClosure(" << ec.kind <<',' ; switch (ec.kind) { - case EngineClosure::Kind::Global : os << ec.global_proc ; break ; - case EngineClosure::Kind::Job : os << ec.job ; break ; - case EngineClosure::Kind::Req : os << ec.req ; break ; + case EngineClosure::Kind::Global : os << ec.ecg ; break ; + case EngineClosure::Kind::Req : os << ec.ecr ; break ; + case EngineClosure::Kind::Job : os << ec.ecj ; break ; + case EngineClosure::Kind::JobMngt : os << ec.ecjm ; break ; DF} - return os << ')' ; + return os << ')' ; } ::vector EngineClosureReq::targets(::string const& startup_dir_s) const { diff --git a/src/lmakeserver/global.x.hh b/src/lmakeserver/global.x.hh index ac5ab47a..19dcabae 100644 --- a/src/lmakeserver/global.x.hh +++ b/src/lmakeserver/global.x.hh @@ -36,6 +36,7 @@ ENUM( EngineClosureKind , Global , Req , Job +, JobMngt ) ENUM( GlobalProc @@ -248,6 +249,10 @@ namespace Engine { namespace Engine { + struct EngineClosureGlobal { + GlobalProc proc = {} ; + } ; + struct EngineClosureReq { friend ::ostream& operator<<( ::ostream& , EngineClosureReq const& ) ; // accesses @@ -269,29 +274,40 @@ namespace Engine { struct EngineClosureJob { friend ::ostream& operator<<( ::ostream& , EngineClosureJob const& ) ; - JobProc proc = JobProc::None ; - JobExec exec = {} ; - bool report = false ; // if proc == Start | GiveUp - ::vector report_unlnks = {} ; // if proc == Start - ::string txt = {} ; // if proc == Start | LiveOut - Req req = {} ; // if proc == GiveUp - ::vmap_ss rsrcs = {} ; // if proc == End - JobDigest digest = {} ; // if proc == End - ::string backend_msg = {} ; // if proc == End - Fd reply_fd = {} ; // if proc == ChkDeps + JobProc proc = {} ; + JobExec job_exec = {} ; + bool report = false ; // proc==Start |GiveUp + ::vector report_unlnks = {} ; // proc==Start + ::string txt = {} ; // proc==Start |LiveOut + Req req = {} ; // proc== GiveUp + ::vmap_ss rsrcs = {} ; // proc==End + JobDigest digest = {} ; // proc==End + ::string msg = {} ; // proc==End + } ; + + struct EngineClosureJobMngt { + friend ::ostream& operator<<( ::ostream& , EngineClosureJobMngt const& ) ; + JobMngtProc proc = {} ; + JobExec job_exec = {} ; + Fd fd = {} ; + ::vmap_s deps = {} ; // proc==ChkDeps|DepsInfo + ::string txt = {} ; // proc==LiveOut } ; struct EngineClosure { friend ::ostream& operator<<( ::ostream& , EngineClosure const& ) ; // - using Kind = EngineClosureKind ; - using Req_ = EngineClosureReq ; - using Job_ = EngineClosureJob ; + using Kind = EngineClosureKind ; + using ECG = EngineClosureGlobal ; + using ECR = EngineClosureReq ; + using ECJ = EngineClosureJob ; + using ECJM = EngineClosureJobMngt ; // using GP = GlobalProc ; using RP = ReqProc ; - using J = Engine::Job ; using JP = JobProc ; + using JMP = JobMngtProc ; + using J = Engine::Job ; using JD = JobDigest ; using JE = JobExec ; using K = Kind ; @@ -300,53 +316,57 @@ namespace Engine { using VS = ::vector_s ; // // cxtors & casts - EngineClosure(GlobalProc p) : kind{Kind::Global} , global_proc{p} {} - // - EngineClosure(RP p,R r,Fd ifd,Fd ofd,VS const& fs,RO const& ro) : kind{K::Req},req{.proc=p,.req=r,.in_fd=ifd,.out_fd=ofd,.files=fs,.options=ro} { SWEAR(p==RP::Make ) ; } - EngineClosure(RP p, Fd ifd,Fd ofd,VS const& fs,RO const& ro) : kind{K::Req},req{.proc=p, .in_fd=ifd,.out_fd=ofd,.files=fs,.options=ro} { SWEAR(p!=RP::Make&&p>=RP::HasArgs) ; } - EngineClosure(RP p,R r,Fd ifd,Fd ofd ) : kind{K::Req},req{.proc=p,.req=r,.in_fd=ifd,.out_fd=ofd } { SWEAR(p==RP::Kill ) ; } - EngineClosure(RP p,R r ) : kind{K::Req},req{.proc=p,.req=r } { SWEAR(p==RP::Close ) ; } - // + // Global + EngineClosure(GP p) : kind{Kind::Global} , ecg{.proc=p} {} + // Req + EngineClosure(RP p,R r,Fd ifd,Fd ofd,VS const& fs,RO const& ro) : kind{K::Req},ecr{.proc=p,.req=r,.in_fd=ifd,.out_fd=ofd,.files=fs,.options=ro} { SWEAR(p==RP::Make ) ; } + EngineClosure(RP p, Fd ifd,Fd ofd,VS const& fs,RO const& ro) : kind{K::Req},ecr{.proc=p, .in_fd=ifd,.out_fd=ofd,.files=fs,.options=ro} { SWEAR(p!=RP::Make&&p>=RP::HasArgs) ; } + EngineClosure(RP p,R r,Fd ifd,Fd ofd ) : kind{K::Req},ecr{.proc=p,.req=r,.in_fd=ifd,.out_fd=ofd } { SWEAR(p==RP::Kill ) ; } + EngineClosure(RP p,R r ) : kind{K::Req},ecr{.proc=p,.req=r } { SWEAR(p==RP::Close ) ; } + // Job EngineClosure( JP p , JE&& je , bool r , ::vector&& rus={} , ::string&& t={} , ::string&& bem={} ) : - kind { K::Job } - , job { .proc=p , .exec=::move(je) , .report=r , .report_unlnks=::move(rus) , .txt=::move(t) , .backend_msg=::move(bem) } + kind { K::Job } + , ecj { .proc=p , .job_exec=::move(je) , .report=r , .report_unlnks=::move(rus) , .txt=::move(t) , .msg=::move(bem) } { SWEAR(p==JP::Start) ; } // - EngineClosure( JP p , JE&& je , ::string&& t ) : kind{K::Job} , job{.proc=p,.exec=::move(je),.txt=::move(t) } { SWEAR( p==JP::LiveOut ) ; } - EngineClosure( JP p , JE&& je , R rq , bool rpt ) : kind{K::Job} , job{.proc=p,.exec=::move(je),.report=rpt,.req=rq} { SWEAR( p==JP::GiveUp ) ; } - EngineClosure( JP p , JE&& je ) : kind{K::Job} , job{.proc=p,.exec=::move(je) } { SWEAR( p==JP::GiveUp || p==JP::ReportStart ) ; } + EngineClosure( JP p , JE&& je , R rq , bool rpt ) : kind{K::Job} , ecj{.proc=p,.job_exec=::move(je),.report=rpt,.req=rq} { SWEAR( p==JP::GiveUp ) ; } + EngineClosure( JP p , JE&& je ) : kind{K::Job} , ecj{.proc=p,.job_exec=::move(je) } { SWEAR( p==JP::GiveUp || p==JP::ReportStart ) ; } // EngineClosure( JP p , JE&& je , ::vmap_ss&& r , JD&& jd , ::string&& bem={} ) : kind { K::Job } - , job { .proc=p , .exec=::move(je) , .rsrcs=::move(r) , .digest=::move(jd) , .backend_msg=::move(bem) } + , ecj { .proc=p , .job_exec=::move(je) , .rsrcs=::move(r) , .digest=::move(jd) , .msg=::move(bem) } { SWEAR(p==JP::End) ; } - // - EngineClosure( JP p , JE&& je , ::vmap_s&& dds , Fd rfd ) : kind{K::Job} , job{.proc=p,.exec=::move(je),.digest={.deps{::move(dds)}},.reply_fd=rfd} { - SWEAR( p==JP::DepInfos || p==JP::ChkDeps ) ; + // JobMngt + EngineClosure( JMP p , JE&& je , ::string&& t ) : kind{K::JobMngt} , ecjm{.proc=p,.job_exec=::move(je),.txt=::move(t) } { SWEAR(p==JMP::LiveOut) ; } + EngineClosure( JMP p , JE&& je , Fd fd_ , ::vmap_s&& dds ) : kind{K::JobMngt} , ecjm{.proc=p,.job_exec=::move(je),.fd{fd_},.deps{::move(dds)}} { + SWEAR( p==JMP::DepInfos || p==JMP::ChkDeps ) ; } // EngineClosure(EngineClosure&& ec) : kind(ec.kind) { switch (ec.kind) { - case K::Global : new(&global_proc) GlobalProc{::move(ec.global_proc)} ; break ; - case K::Req : new(&req ) Req_ {::move(ec.req )} ; break ; - case K::Job : new(&job ) Job_ {::move(ec.job )} ; break ; + case K::Global : new(&ecg ) ECG {::move(ec.ecg )} ; break ; + case K::Req : new(&ecr ) ECR {::move(ec.ecr )} ; break ; + case K::Job : new(&ecj ) ECJ {::move(ec.ecj )} ; break ; + case K::JobMngt : new(&ecjm) ECJM{::move(ec.ecjm)} ; break ; DF} } ~EngineClosure() { switch (kind) { - case K::Global : global_proc.~GlobalProc() ; break ; - case K::Req : req .~Req_ () ; break ; - case K::Job : job .~Job_ () ; break ; + case K::Global : ecg .~ECG () ; break ; + case K::Req : ecr .~ECR () ; break ; + case K::Job : ecj .~ECJ () ; break ; + case K::JobMngt : ecjm.~ECJM() ; break ; DF} } - EngineClosure& operator=(EngineClosure const& ec) = delete ; - EngineClosure& operator=(EngineClosure && ec) = delete ; + EngineClosure& operator=(EngineClosure const& ec) = delete ; + EngineClosure& operator=(EngineClosure && ec) = delete ; // data Kind kind = K::Global ; union { - GlobalProc global_proc = GP::None ; // if kind==Global - Req_ req ; - Job_ job ; + ECG ecg ; + ECR ecr ; + ECJ ecj ; + ECJM ecjm ; } ; } ; diff --git a/src/lmakeserver/job.cc b/src/lmakeserver/job.cc index 4098b33a..dfec0bc1 100644 --- a/src/lmakeserver/job.cc +++ b/src/lmakeserver/job.cc @@ -241,13 +241,13 @@ namespace Engine { } // answer to job execution requests - JobRpcReply JobExec::job_info( JobProc proc , ::vector const& deps ) const { + JobMngtRpcReply JobExec::job_info( JobMngtProc proc , ::vector const& deps ) const { ::vector reqs = (*this)->running_reqs(false/*with_zombies*/) ; Trace trace("job_info",proc,deps.size()) ; - if (!reqs) return proc ; // if job is not running, it is too late // switch (proc) { - case JobProc::DepInfos : { + case JobMngtProc::DepInfos : { + if (!reqs) return {proc,{}/*seq_id*/,{}/*fd*/,Maybe} ; // if job is not running, it is too late, seq_id will be filled in later ::vector> res ; res.reserve(deps.size()) ; for( Dep const& dep : deps ) { Node(dep)->full_refresh(false/*report_no_file*/,{},dep->name()) ; @@ -260,20 +260,22 @@ namespace Engine { trace("dep_info",dep,dep_ok) ; res.emplace_back(dep_ok,dep->crc) ; } - return {proc,res} ; + return {proc,{}/*seq_id*/,{}/*fd*/,res} ; } - case JobProc::ChkDeps : + case JobMngtProc::ChkDeps : + if (!reqs) return {proc,{}/*seq_id*/,{}/*fd*/,{}} ; // if job is not running, it is too late, seq_id will be filled in later for( Dep const& dep : deps ) { Node(dep)->full_refresh(false/*report_no_file*/,{},dep->name()) ; for( Req req : reqs ) { - NodeReqInfo const& cdri = dep->c_req_info(req) ; - if (!cdri.done(NodeGoal::Dsk) ) { trace("waiting",dep,req) ; return {proc,Maybe} ; } - if (dep->ok(cdri,dep.accesses)==No) { trace("bad" ,dep,req) ; return {proc,No } ; } + NodeReqInfo const& cdri = dep->c_req_info(req) ; + NodeGoal goal = +dep.accesses ? NodeGoal::Dsk : NodeGoal::Status ; // if no access, we do not care about file on disk + if (!cdri.done(goal) ) { trace("waiting",dep,req) ; return {proc,{}/*seq_id*/,{}/*fd*/,Maybe} ; } // seq_id will be filled in later + if (dep->ok(cdri,dep.accesses)==No) { trace("bad" ,dep,req) ; return {proc,{}/*seq_id*/,{}/*fd*/,No } ; } // . } trace("ok",dep) ; } trace("done") ; - return {proc,Yes} ; + return {proc,{}/*seq_id*/,{}/*fd*/,Yes} ; // seq_id will be filled in later DF} } @@ -602,6 +604,7 @@ namespace Engine { req->stats.ended(jr)++ ; req->stats.jobs_time[jr<=JR::Useful] += exec_time ; if (with_stderr) req->audit_stderr(msg,stderr,max_stderr_len,1) ; + else req->audit_stderr(msg,{} ,max_stderr_len,1) ; return res ; } @@ -677,7 +680,7 @@ namespace Engine { } ; auto need_run = [&](ReqInfo::State const& s)->bool { SWEAR( pre_reason.tag const& deps ) const ; // answer to requests from job execution + JobMngtRpcReply job_info( JobMngtProc , ::vector const& deps ) const ; // answer to requests from job execution bool/*modified*/ end ( ::vmap_ss const& rsrcs , JobDigest const& , ::string&& backend_msg ) ; // hit indicates that result is from a cache hit void give_up ( Req={} , bool report=true ) ; // Req (all if 0) was killed and job was not killed (not started or continue) // @@ -419,7 +419,7 @@ namespace Engine { inline JobData::JobData (JobData&& jd) : JobData(jd) { jd.targets.forget() ; jd.deps.forget() ; } inline JobData::~JobData ( ) { targets.pop () ; deps.pop () ; } - inline JobData& JobData::operator=(JobData&& jd) { SWEAR(rule==jd.rule,rule,jd.rule) ; *this = jd ; jd.targets.forget() ; jd.deps.forget() ; return *this ; } // XXX : use copy&swap idiom + inline JobData& JobData::operator=(JobData&& jd) { SWEAR(rule==jd.rule,rule,jd.rule) ; *this = jd ; jd.targets.forget() ; jd.deps.forget() ; return *this ; } inline ::string JobData::special_stderr ( ) const { return special_stderr ( {}) ; } inline void JobData::audit_end_special( Req r , SpecialStep s , Bool3 m ) const { return audit_end_special(r,s,m,{}) ; } diff --git a/src/lmakeserver/main.cc b/src/lmakeserver/main.cc index 7d7b6fdf..8104cfe5 100644 --- a/src/lmakeserver/main.cc +++ b/src/lmakeserver/main.cc @@ -263,7 +263,7 @@ bool/*interrupted*/ engine_loop() { EngineClosure closure = g_engine_queue.pop() ; switch (closure.kind) { case EngineClosureKind::Global : { - switch (closure.global_proc) { + switch (closure.ecg.proc) { case GlobalProc::Int : trace("int") ; // vvvvvvvvvvvv @@ -276,22 +276,23 @@ bool/*interrupted*/ engine_loop() { DF} } break ; case EngineClosureKind::Req : { - EngineClosureReq& req = closure.req ; - ::string const& startup_dir_s = req.options.startup_dir_s ; - switch (req.proc) { + EngineClosureReq& ecr = closure.ecr ; + Req req = ecr.req ; + ::string const& startup_dir_s = ecr.options.startup_dir_s ; + switch (ecr.proc) { case ReqProc::Debug : // PER_CMD : handle request coming from receiving thread, just add your Proc here if the request is answered immediately case ReqProc::Forget : case ReqProc::Mark : case ReqProc::Show : { - trace(req) ; + trace(ecr) ; bool ok = true/*garbage*/ ; - if ( !req.options.flags[ReqFlag::Quiet] && +startup_dir_s ) - audit( req.out_fd , req.options , Color::Note , "startup dir : "+startup_dir_s.substr(0,startup_dir_s.size()-1) , true/*as_is*/ ) ; - try { ok = g_cmd_tab[+req.proc](req) ; } - catch (::string const& e) { ok = false ; if (+e) audit(req.out_fd,req.options,Color::Err,e) ; } - OMsgBuf().send( req.out_fd , ReqRpcReply(ReqRpcReplyProc::Status,ok) ) ; - /**/ req.in_fd .close() ; - if (req.out_fd!=req.in_fd) req.out_fd.close() ; + if ( !ecr.options.flags[ReqFlag::Quiet] && +startup_dir_s ) + audit( ecr.out_fd , ecr.options , Color::Note , "startup dir : "+startup_dir_s.substr(0,startup_dir_s.size()-1) , true/*as_is*/ ) ; + try { ok = g_cmd_tab[+ecr.proc](ecr) ; } + catch (::string const& e) { ok = false ; if (+e) audit(ecr.out_fd,ecr.options,Color::Err,e) ; } + OMsgBuf().send( ecr.out_fd , ReqRpcReply(ReqRpcReplyProc::Status,ok) ) ; + /**/ ecr.in_fd .close() ; + if (ecr.out_fd!=ecr.in_fd) ecr.out_fd.close() ; } break ; // Make, Kill and Close management : // there is exactly one Kill and one Close and one Make for each with only one guarantee : Close comes after Make @@ -299,74 +300,86 @@ bool/*interrupted*/ engine_loop() { // read side is closed upon Kill (cannot be upon Close as epoll.del must be called before close) // write side is closed upon Close (cannot be upon Kill as this may trigger lmake command termination, which, in turn, will trigger eof on the read side case ReqProc::Make : - if (!req.req.zombie()) // if already zombie, dont make req.req + if (req.zombie()) // if already zombie, dont make req + trace("already_killed",req) ; + else try { ::string msg = Makefiles::dynamic_refresh(startup_dir_s) ; - if (+msg) audit( req.out_fd , req.options , Color::Note , msg ) ; + if (+msg) audit( ecr.out_fd , ecr.options , Color::Note , msg ) ; trace("new_req",req) ; - //vvvvvvvvvvvvvvv - req.req.make(req) ; - //^^^^^^^^^^^^^^^ - if (!req.as_job()) record_targets(req.req->job) ; - fd_tab[req.req] = { .in=req.in_fd , .out=req.out_fd } ; + //vvvvvvvvvvv + req.make(ecr) ; + //^^^^^^^^^^^ + if (!ecr.as_job()) record_targets(req->job) ; + fd_tab[req] = { .in=ecr.in_fd , .out=ecr.out_fd } ; break ; } catch(::string const& e) { - audit ( req.out_fd , req.options , Color::Err , e ) ; - audit_status( req.out_fd , req.options , false/*ok*/ ) ; - // cannot make, process as if followed by Close - trace("no_make") ; + audit ( ecr.out_fd , ecr.options , Color::Err , e ) ; + audit_status( ecr.out_fd , ecr.options , false/*ok*/ ) ; + trace("no_make",req) ; } - /**/ req.in_fd .close() ; - if (req.out_fd!=req.in_fd) req.out_fd.close() ; + // cannot make, process as if followed by Close + req.close() ; + /**/ ecr.in_fd .close() ; + if (ecr.out_fd!=ecr.in_fd) ecr.out_fd.close() ; break ; case ReqProc::Close : { - auto it = fd_tab.find(req.req) ; - trace("close_req",req,it->second.in,it->second.out,STR(it->second.killed)) ; - //vvvvvvvvvvvvv - req.req.close() ; - //^^^^^^^^^^^^^ + auto it = fd_tab.find(req) ; + trace("close_req",ecr,it->second.in,it->second.out,STR(it->second.killed)) ; + //vvvvvvvvv + req.close() ; + //^^^^^^^^^ if (it->second.in!=it->second.out) ::close (it->second.out ) ; else if (it->second.killed ) ::close (it->second.out ) ; // we are after Kill, finalize close of file descriptor else ::shutdown(it->second.out,SHUT_WR) ; // we are before Kill, shutdown until final close upon Close fd_tab.erase(it) ; } break ; case ReqProc::Kill : { - Req r = req.req ; - auto it = fd_tab.find(r) ; - bool req_active = it!=fd_tab.end() && it->second.out==req.out_fd ; // out_fd is held until now, and if it does not coincide with it->second, req id was reused for a new Req + auto it = fd_tab.find(req) ; + bool req_active = it!=fd_tab.end() && it->second.out==ecr.out_fd ; // out_fd is held until now, and if it does not coincide with it->second, req id was reused for a new Req // - if (it==fd_tab.end()) trace("kill_req",req ) ; - else trace("kill_req",req,it->second.in,it->second.out,STR(it->second.killed)) ; - // vvvvvvvv - if ( +r && +*r && req_active ) r.kill() ; - // ^^^^^^^^ + if (it==fd_tab.end()) trace("kill_req",ecr ) ; + else trace("kill_req",ecr,it->second.in,it->second.out,STR(it->second.killed)) ; + // vvvvvvvvvv + if ( +req && +*req && req_active ) req.kill() ; + // ^^^^^^^^^^ if (req_active ) it->second.killed = true ; - if (req.in_fd!=req.out_fd) ::close (req.in_fd ) ; - else if (!req_active ) ::close (req.in_fd ) ; // we are after Close, finalize close of file descriptor - else ::shutdown(req.in_fd,SHUT_RD) ; // we are before Close, shutdown until final close upon Close + if (ecr.in_fd!=ecr.out_fd) ::close (ecr.in_fd ) ; + else if (!req_active ) ::close (ecr.in_fd ) ; // we are after Close, finalize close of file descriptor + else ::shutdown(ecr.in_fd,SHUT_RD) ; // we are before Close, shutdown until final close upon Close } break ; DF} } break ; case EngineClosureKind::Job : { - EngineClosureJob& job = closure.job ; - JobExec & je = job.exec ; - trace("job",job.proc,je) ; - switch (job.proc) { - // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - case JobProc::Start : je.started (job.report,job.report_unlnks,job.txt,job.backend_msg) ; break ; - case JobProc::ReportStart : je.report_start( ) ; break ; - case JobProc::LiveOut : je.live_out (job.txt ) ; break ; - case JobProc::GiveUp : je.give_up (job.req , job.report ) ; break ; - case JobProc::End : je.end (job.rsrcs,job.digest,::move(job.backend_msg) ) ; break ; - // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - case JobProc::ChkDeps : - case JobProc::DepInfos : { - ::vector deps ; deps.reserve(job.digest.deps.size()) ; - for( auto const& [dn,dd] : job.digest.deps ) deps.emplace_back(Node(dn),dd) ; - //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - OMsgBuf().send( job.reply_fd , je.job_info(job.proc,deps) ) ; - //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - ::close(job.reply_fd) ; + EngineClosureJob& ecj = closure.ecj ; + JobExec & je = ecj.job_exec ; + trace("job",ecj.proc,je) ; + switch (ecj.proc) { + // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + case JobProc::Start : je.started (ecj.report,ecj.report_unlnks,ecj.txt,ecj.msg) ; break ; + case JobProc::ReportStart : je.report_start( ) ; break ; + case JobProc::GiveUp : je.give_up (ecj.req , ecj.report ) ; break ; + case JobProc::End : je.end (ecj.rsrcs,ecj.digest,::move(ecj.msg) ) ; break ; + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + DF} + } break ; + case EngineClosureKind::JobMngt : { + EngineClosureJobMngt& ecjm = closure.ecjm ; + JobExec & je = ecjm.job_exec ; + trace("job_mngt",ecjm.proc,je) ; + switch (ecjm.proc) { + // vvvvvvvvvvvvvvvvvvvv + case JobMngtProc::LiveOut : je.live_out(ecjm.txt) ; break ; + // ^^^^^^^^^^^^^^^^^^^^ + case JobMngtProc::ChkDeps : + case JobMngtProc::DepInfos : { + ::vector deps ; deps.reserve(ecjm.deps.size()) ; + for( auto const& [dn,dd] : ecjm.deps ) deps.emplace_back(Node(dn),dd) ; + JobMngtRpcReply jmrr = je.job_info(ecjm.proc,deps) ; + jmrr.fd = ecjm.fd ; // seq_id will be filled in by send_reply + //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + Backends::send_reply( +je , ::move(jmrr) ) ; + //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ } break ; DF} } break ; diff --git a/src/lmakeserver/makefiles.cc b/src/lmakeserver/makefiles.cc index bf897d7c..70bbe823 100644 --- a/src/lmakeserver/makefiles.cc +++ b/src/lmakeserver/makefiles.cc @@ -161,7 +161,7 @@ namespace Engine::Makefiles { // static RegExpr pyc_re { R"(((.*/)?)(?:__pycache__/)?(\w+)(?:\.\w+-\d+)?\.pyc)" , true/*fast*/ } ; // dir_s is \1, module is \3, matches both python 2 & 3 // - Gather gather { New } ; + Gather gather ; ::string data = to_string(PrivateAdminDir,'/',action,"_data.py") ; dir_guard(data) ; ::vector_s cmd_line = { PYTHON , *g_lmake_dir+"/_lib/read_makefiles.py" , data , action , module } ; gather.autodep_env.src_dirs_s = {"/"} ; diff --git a/src/lmakeserver/rule.cc b/src/lmakeserver/rule.cc index 49346de1..9490f8bf 100644 --- a/src/lmakeserver/rule.cc +++ b/src/lmakeserver/rule.cc @@ -495,7 +495,7 @@ namespace Engine { ::string interpreter0 = res.interpreter[0] ; m.rule->add_cwd(interpreter0) ; AutodepLock lock{deps} ; - AutoCloseFd(::open(interpreter0.c_str(),O_RDONLY)) ; // speed up dep acquisition by accessing interpreter, XXX : pretend access rather than make a real one for perf + Record::Read( auditer() , interpreter0.c_str() , false/*no_follow*/ , false/*keep_real*/ , true/*allow_tmp_map*/ , "dyn_attr_eval" ) ; return res ; } diff --git a/src/msg.hh b/src/msg.hh new file mode 100644 index 00000000..e386670e --- /dev/null +++ b/src/msg.hh @@ -0,0 +1,97 @@ +// This file is part of the open-lmake distribution (git@github.com:cesar-douady/open-lmake.git) +// Copyright (c) 2023 Doliam +// This program is free software: you can redistribute/modify under the terms of the GPL-v3 (https://www.gnu.org/licenses/gpl-3.0.html). +// This program is distributed WITHOUT ANY WARRANTY, without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +#pragma once + +#include "fd.hh" +#include "serialize.hh" + +// +// MsgBuf +// + +struct MsgBuf { + friend ::ostream& operator<<( ::ostream& , MsgBuf const& ) ; + using Len = size_t ; + // statics + static Len s_sz(const char* str) { + Len len = 0 ; ::memcpy( &len , str , sizeof(Len) ) ; + return len ; + } +protected : + // data + Len _len = 0 ; // data sent/received so far, reading : may also apply to len accumulated in buf + ::string _buf ; // reading : sized after expected size, but actuall filled up only with len char's // writing : contains len+data to be sent + bool _data_pass = false ; // reading : if true <=> buf contains partial data, else it contains partial data len // writing : if true <=> buf contains data +} ; +inline ::ostream& operator<<( ::ostream& os , MsgBuf const& mb ) { return os<<"MsgBuf("< static T s_receive(const char* str) { + Len len = s_sz(str) ; + // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + return deserialize(IStringStream(::string( str+sizeof(Len) , len ))) ; // XXX : avoid copy (use string_view in C++26) + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + } + // cxtors & casts + IMsgBuf() { _buf.resize(sizeof(Len)) ; } // prepare to receive len + // services + template T receive(Fd fd) { + T res ; + while (!receive_step(fd,res)) ; + return res ; + } + template bool/*complete*/ receive_step( Fd fd , T& res ) { + ssize_t cnt = ::read( fd , &_buf[_len] , _buf.size()-_len ) ; + if (cnt<=0) throw to_string("cannot receive over fd ",fd) ; + _len += cnt ; + if (_len<_buf.size()) return false/*complete*/ ; // _buf is still partial + if (_data_pass) { + // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + res = deserialize(IStringStream(::move(_buf))) ; + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + *this = {} ; + return true/*complete*/ ; + } else { + SWEAR( _buf.size()==sizeof(Len) , _buf.size() ) ; + Len len = s_sz(_buf.data()) ; + // we now expect the data + _buf.resize(len) ; + _data_pass = true ; + _len = 0 ; + return false/*complete*/ ; + } + } +} ; + +struct OMsgBuf : MsgBuf { + // statics + template static ::string s_send(T const& x) { + ::string res = serialize(::pair(0,x)) ; SWEAR(res.size()>=sizeof(Len)) ; // directly serialize in res to avoid copy : serialize a pair with length+data + Len len = res.size()-sizeof(Len) ; + ::memcpy( res.data() , &len , sizeof(Len) ) ; // overwrite len + return res ; + } + // services + template void send( Fd fd , T const& x ) { + if (send_step(fd,x)) return ; + while (!send_step(fd)) ; + } + template bool/*complete*/ send_step( Fd fd , T const& x ) { + SWEAR(!_data_pass) ; + _buf = s_send(x) ; + _data_pass = true ; + return send_step(fd) ; + } + bool/*complete*/ send_step(Fd fd) { + SWEAR(_data_pass) ; + ssize_t cnt = ::write( fd , &_buf[_len] , _buf.size()-_len ) ; + if (cnt<=0) throw to_string("cannot send over ",fd) ; + _len += cnt ; + if (_len<_buf.size()) { return false/*complete*/ ; } // _buf is still partial + else { *this = {} ; return true /*complete*/ ; } + } +} ; diff --git a/src/non_portable.cc b/src/non_portable.cc index 7d1c65e2..1c468452 100644 --- a/src/non_portable.cc +++ b/src/non_portable.cc @@ -123,9 +123,9 @@ void np_ptrace_set_res( int pid , long val ) { ptrace( PTRACE_GETREGSET , pid , (void*)NT_PRSTATUS , &iov ) ; if (errno) throw 0 ; #if __arm__ - regs.r0 = val ; // XXX : to be validated + regs.r0 = val ; #elif __aarch64__ - regs.regs[0] = val ; // XXX : to be validated + regs.regs[0] = val ; #endif ptrace( PTRACE_SETREGSET , pid , (void*)NT_PRSTATUS , &iov ) ; if (errno) throw 0 ; diff --git a/src/process.cc b/src/process.cc index fa6c7a3c..05804d90 100644 --- a/src/process.cc +++ b/src/process.cc @@ -72,7 +72,7 @@ bool/*parent*/ Child::spawn( child_args[args.size()] = nullptr ; if (env) ::execve( child_args[0] , const_cast(child_args) , const_cast(child_env) ) ; else ::execv ( child_args[0] , const_cast(child_args) ) ; - pid = -1 ; + pid = 0 ; exit(Rc::System,"cannot exec (",strerror(errno),") : ",args) ; // in case exec fails } if (stdin_fd ==Pipe) { stdin = p2c .write ; p2c .read .close() ; } diff --git a/src/process.hh b/src/process.hh index e4abec81..d1ebf58f 100644 --- a/src/process.hh +++ b/src/process.hh @@ -96,11 +96,11 @@ struct Child { spawn(as_session,args,stdin_fd,stdout_fd,stderr_fd,env,add_env,chroot,cwd,pre_exec) ; } ~Child() { - swear_prod(pid==-1,"bad pid ",pid) ; + swear_prod(pid==0,"bad pid ",pid) ; } // accesses - bool operator +() const { return pid!=-1 ; } - bool operator !() const { return !+*this ; } + bool operator+() const { return pid ; } + bool operator!() const { return !+*this ; } // services bool/*parent*/ spawn( bool as_session , ::vector_s const& args @@ -111,16 +111,16 @@ struct Child { , void (*pre_exec)() =nullptr ) ; void mk_daemon() { - pid = -1 ; + pid = 0 ; stdin .detach() ; stdout.detach() ; stderr.detach() ; } void waited() { - pid = -1 ; + pid = 0 ; } int/*wstatus*/ wait() { - SWEAR(pid!=-1) ; + SWEAR(pid!=0) ; int wstatus ; int rc = ::waitpid(pid,&wstatus,0) ; swear_prod(rc==pid,"cannot wait for pid ",pid) ; @@ -134,7 +134,7 @@ struct Child { bool/*done*/ kill (int sig) { return kill_process(pid,sig,as_session/*as_group*/) ; } bool is_alive( ) const { return kill_process(pid,0 ) ; } //data - pid_t pid = -1 ; + pid_t pid = 0 ; AutoCloseFd stdin ; AutoCloseFd stdout ; AutoCloseFd stderr ; diff --git a/src/py.hh b/src/py.hh index 4a1df482..92af81a5 100644 --- a/src/py.hh +++ b/src/py.hh @@ -134,7 +134,6 @@ namespace Py { // ~Ptr() { unboost() ; } // - Ptr& operator=(Object* o) { unboost() ; ptr = o ; boost() ; return *this ; } // XXX : use copy&swap idiom Ptr& operator=(Ptr const& p) { unboost() ; ptr = p.ptr ; boost() ; return *this ; } Ptr& operator=(Ptr && p) { unboost() ; ptr = p.ptr ; p.detach() ; return *this ; } // accesses @@ -157,7 +156,8 @@ namespace Py { } ; template requires(!::is_same_v) struct PtrBase : Ptr { - using Base = Ptr ; + using TBase = typename T::Base ; + using Base = Ptr ; // cxtors & casts PtrBase() = default ; // @@ -165,25 +165,20 @@ namespace Py { PtrBase(Object* o) : Base{ o } { _chk(&**this) ; } PtrBase(Base const& o) : Base{ o } { _chk(&**this) ; } PtrBase(Base && o) : Base{::move(o)} { _chk(&**this) ; } + PtrBase(T* o) : Base{ o } { } // - PtrBase(T* o) : Base{ o } {} - PtrBase(PtrBase const& o) : Base{ o } {} - PtrBase(PtrBase && o) : Base{::move(o)} {} - // - PtrBase& operator=(PtrBase const&) = default ; // XXX : why is this necessary ? - PtrBase& operator=(PtrBase &&) = default ; // . // accesses T & operator* () { return *static_cast(this->ptr) ; } T const& operator* () const { return *static_cast(this->ptr) ; } T * operator->() { return &**this ; } T const* operator->() const { return &**this ; } // - operator Object *() { return &**this ; } - operator Object const*() const { return &**this ; } - operator typename T::Base *() requires(!::is_same_v) { return &**this ; } - operator typename T::Base const*() const requires(!::is_same_v) { return &**this ; } - operator T *() { return &**this ; } - operator T const*() const { return &**this ; } + operator Object *() { return &**this ; } + operator Object const*() const { return &**this ; } + operator TBase *() requires(!::is_same_v) { return &**this ; } + operator TBase const*() const requires(!::is_same_v) { return &**this ; } + operator T *() { return &**this ; } + operator T const*() const { return &**this ; } } ; // diff --git a/src/rpc_job.cc b/src/rpc_job.cc index 069e4247..f3c73d51 100644 --- a/src/rpc_job.cc +++ b/src/rpc_job.cc @@ -112,27 +112,13 @@ ::ostream& operator<<( ::ostream& os , JobDigest const& jd ) { } ::ostream& operator<<( ::ostream& os , JobRpcReq const& jrr ) { - os << "JobRpcReq(" << jrr.proc <<','<< jrr.seq_id <<','<< jrr.job ; + /**/ os << "JobRpcReq(" << jrr.proc <<','<< jrr.seq_id <<','<< jrr.job ; switch (jrr.proc) { - case JobProc::LiveOut : os <<','<< jrr.msg ; break ; - case JobProc::DepInfos : os <<','<< jrr.digest.deps ; break ; - case JobProc::End : os <<','<< jrr.digest ; break ; - default : break ; + case JobProc::Start : os <<','<< jrr.port ; break ; + case JobProc::End : os <<','<< jrr.digest <<','<< jrr.dynamic_env ; break ; + default : break ; } - return os << ')' ; -} - -JobRpcReq::JobRpcReq( SI si , JI j , JobExecRpcReq&& jerr ) : seq_id{si} , job{j} { - switch (jerr.proc) { - case JobExecRpcProc::Decode : proc = P::Decode ; msg = ::move(jerr.txt) ; file = ::move(jerr.files[0].first) ; ctx = ::move(jerr.ctx) ; break ; - case JobExecRpcProc::Encode : proc = P::Encode ; msg = ::move(jerr.txt) ; file = ::move(jerr.files[0].first) ; ctx = ::move(jerr.ctx) ; min_len = jerr.min_len ; break ; - case JobExecRpcProc::DepInfos : { - ::vmap_s ds ; ds.reserve(jerr.files.size()) ; - for( auto&& [dep,date] : jerr.files ) ds.emplace_back( ::move(dep) , DepDigest(jerr.digest.accesses,date,{}/*dflags*/,true/*parallel*/) ) ; // no need for flags to ask info - proc = P::DepInfos ; - digest.deps = ::move(ds) ; - } break ; - DF} + return os <<','<< jrr.msg <<')' ; } // @@ -140,22 +126,18 @@ JobRpcReq::JobRpcReq( SI si , JI j , JobExecRpcReq&& jerr ) : seq_id{si} , job{j // ::ostream& operator<<( ::ostream& os , MatchFlags const& mf ) { - os << "MatchFlags(" ; + /**/ os << "MatchFlags(" ; switch (mf.is_target) { - case Yes : os << "target," << mf.tflags() ; break ; - case No : os << "dep," << mf.dflags() ; break ; - case Maybe : break ; + case Yes : os << "target" ; if (+mf.tflags()) os<<','<=JobExecRpcProc::HasFiles) { + if (jerr.proc>=JobExecProc::HasFiles) { if ( +jerr.digest.accesses && !jerr.solve ) os <<','<< jerr.files ; else os <<','<< mk_key_vector(jerr.files) ; } @@ -232,35 +259,25 @@ AccessDigest& AccessDigest::operator|=(AccessDigest const& other) { ::ostream& operator<<( ::ostream& os , JobExecRpcReply const& jerr ) { os << "JobExecRpcReply(" << jerr.proc ; switch (jerr.proc) { - case JobExecRpcProc::None : ; break ; - case JobExecRpcProc::ChkDeps : os <<','<< jerr.ok ; break ; - case JobExecRpcProc::DepInfos : os <<','<< jerr.dep_infos ; break ; - case JobExecRpcProc::Decode : - case JobExecRpcProc::Encode : os <<','<< jerr.txt <<','<< jerr.ok ; break ; + case JobExecProc::None : ; break ; + case JobExecProc::ChkDeps : os <<','<< jerr.ok ; break ; + case JobExecProc::DepInfos : os <<','<< jerr.dep_infos ; break ; + case JobExecProc::Decode : + case JobExecProc::Encode : os <<','<< jerr.txt <<','<< jerr.ok ; break ; DF} return os << ')' ; } -JobExecRpcReply::JobExecRpcReply( JobRpcReply const& jrr ) { - switch (jrr.proc) { - case JobProc::None : proc = Proc::None ; break ; - case JobProc::ChkDeps : SWEAR(jrr.ok!=Maybe) ; proc = Proc::ChkDeps ; ok = jrr.ok ; break ; - case JobProc::DepInfos : proc = Proc::DepInfos ; dep_infos = jrr.dep_infos ; break ; - case JobProc::Decode : proc = Proc::Decode ; ok = jrr.ok ; txt = jrr.txt ; break ; - case JobProc::Encode : proc = Proc::Encode ; ok = jrr.ok ; txt = jrr.txt ; break ; +JobExecRpcReply::JobExecRpcReply( JobMngtRpcReply&& jmrr ) { + switch (jmrr.proc) { + case JobMngtProc::None : proc = Proc::None ; break ; + case JobMngtProc::ChkDeps : SWEAR(jmrr.ok!=Maybe) ; proc = Proc::ChkDeps ; ok = jmrr.ok ; break ; + case JobMngtProc::DepInfos : proc = Proc::DepInfos ; dep_infos = ::move(jmrr.dep_infos) ; break ; + case JobMngtProc::Decode : proc = Proc::Decode ; ok = jmrr.ok ; txt = ::move(jmrr.txt ) ; break ; + case JobMngtProc::Encode : proc = Proc::Encode ; ok = jmrr.ok ; txt = ::move(jmrr.txt ) ; break ; DF} } -// -// JobSserverRpcReq -// - -::ostream& operator<<( ::ostream& os , JobServerRpcReq const& jsrr ) { - /**/ os << "JobServerRpcReq(" << jsrr.proc <<','<< jsrr.seq_id ; - if (jsrr.proc==JobServerRpcProc::Heartbeat) os <<','<< jsrr.job ; - return os <<')' ; -} - // // JobInfo // diff --git a/src/rpc_job.hh b/src/rpc_job.hh index c5116d30..4b350854 100644 --- a/src/rpc_job.hh +++ b/src/rpc_job.hh @@ -15,14 +15,15 @@ #include "autodep/env.hh" // START_OF_VERSIONING - ENUM_1( BackendTag // PER_BACKEND : add a tag for each backend , Dflt = Local , Unknown // must be first , Local , Slurm ) +// END_OF_VERSIONING +// START_OF_VERSIONING ENUM_1( FileActionTag , HasFile = Uniquify // <=HasFile means action acts on file , None // no action, just check integrity @@ -32,6 +33,7 @@ ENUM_1( FileActionTag , Mkdir , Rmdir ) +// END_OF_VERSIONING struct FileAction { friend ::ostream& operator<<( ::ostream& , FileAction const& ) ; // cxtors & casts @@ -45,6 +47,7 @@ struct FileAction { inline ::pair_s do_file_actions( ::vector_s& unlnks/*out*/ , ::vmap_s&& pa , Disk::NfsGuard& ng , Algo a ) { return do_file_actions(&unlnks,::move(pa),ng,a) ; } inline ::pair_s do_file_actions( ::vmap_s&& pa , Disk::NfsGuard& ng , Algo a ) { return do_file_actions(nullptr,::move(pa),ng,a) ; } +// START_OF_VERSIONING ENUM_2( Dflag // flags for deps , NRule = Required // number of Dflag's allowed in rule definition , NDyn = Static // number of Dflag's allowed in lside flags @@ -55,6 +58,7 @@ ENUM_2( Dflag // flags for deps , Required // dep must be buildable , Static // is static dep, for internal use only ) +// END_OF_VERSIONING static constexpr char DflagChars[] = { 'E' // Essential , 'c' // Critical @@ -65,12 +69,14 @@ static constexpr char DflagChars[] = { static_assert(::size(DflagChars)==N) ; using Dflags = BitMap ; +// START_OF_VERSIONING ENUM_1( ExtraDflag , NRule // all flags allowed , Top , Ignore , StatReadData ) +// END_OF_VERSIONING static constexpr char ExtraDflagChars[] = { 0 // Top , 'I' // Ignore @@ -79,6 +85,7 @@ static constexpr char ExtraDflagChars[] = { static_assert(::size(ExtraDflagChars)==N) ; using ExtraDflags = BitMap ; +// START_OF_VERSIONING ENUM_2( Tflag // flags for targets , NRule = Static // number of Tflag's allowed in rule definition , NDyn = Phony // number of Tflag's allowed inlside flags @@ -90,6 +97,7 @@ ENUM_2( Tflag // flags for targets , Static // is static , for internal use only, only if also a Target , Target // is a target, for internal use only ) +// END_OF_VERSIONING static constexpr char TflagChars[] = { 'E' // Essential , 'i' // Incremental @@ -105,6 +113,7 @@ inline bool static_phony(Tflags tf) { return tf[Tflag::Target] && (tf[Tflag::Static]||tf[Tflag::Phony]) ; } +// START_OF_VERSIONING ENUM_1( ExtraTflag , NRule = Allow // number of Tflag's allowed in rule definition , Top @@ -113,6 +122,7 @@ ENUM_1( ExtraTflag , Allow // writing to this target is allowed (for use in clmake.target and ltarget) , Wash // target was unlinked when washing before job execution ) +// END_OF_VERSIONING static constexpr char ExtraTflagChars[] = { 0 // Top , 'I' // Ignore @@ -123,19 +133,30 @@ static constexpr char ExtraTflagChars[] = { static_assert(::size(ExtraTflagChars)==N) ; using ExtraTflags = BitMap ; -ENUM( JobProc +// START_OF_VERSIONING +ENUM( JobMngtProc , None -, Start -, ReportStart -, GiveUp // Req (all if 0) was killed and job was not (either because of other Req's or it did not start yet) , ChkDeps , DepInfos +, LiveOut , Decode , Encode -, LiveOut +, Heartbeat +, Kill +) +// END_OF_VERSIONING + +// START_OF_VERSIONING +ENUM( JobProc +, None +, Start +, ReportStart +, GiveUp // Req (all if 0) was killed and job was not (either because of other Req's or it did not start yet) , End ) +// END_OF_VERSIONING +// START_OF_VERSIONING ENUM_3( JobReasonTag // see explanations in table below , HasNode = NoTarget // if >=HasNode, a node is associated , Err = DepOverwritten @@ -169,6 +190,7 @@ ENUM_3( JobReasonTag // see explanations in table // with missing , DepMissingStatic // this prevents the job from being selected ) +// END_OF_VERSIONING static constexpr const char* JobReasonTagStrs[] = { "no reason" // None // with reason @@ -231,12 +253,15 @@ static constexpr uint8_t JobReasonTagPrios[] = { } ; static_assert(::size(JobReasonTagPrios)==N) ; +// START_OF_VERSIONING ENUM( MatchKind , Target , SideTargets , SideDeps ) +// END_OF_VERSIONING +// START_OF_VERSIONING ENUM_3( Status // result of job execution , Early = EarlyLostErr // <=Early means output has not been modified , Async = Killed // <=Async means job was interrupted asynchronously @@ -254,6 +279,7 @@ ENUM_3( Status // result of job ex , Ok // job execution ended successfully , Err // job execution ended in error ) +// END_OF_VERSIONING inline bool is_lost(Status s) { return s<=Status::LateLostErr && s>=Status::EarlyLost ; } inline Bool3 is_ok (Status s) { static constexpr Bool3 IsOkTab[] = { @@ -292,8 +318,10 @@ struct AccDflags { AccDflags operator| (AccDflags other) const { return { accesses|other.accesses , dflags|other.dflags } ; } AccDflags& operator|=(AccDflags other) { *this = *this | other ; return *this ; } // data + // START_OF_VERSIONING Accesses accesses ; Dflags dflags ; + // END_OF_VERSIONING } ; struct JobReason { @@ -317,17 +345,21 @@ struct JobReason { return JobReasonTagStrs[+tag] ; } // data + // START_OF_VERSIONING Tag tag = JobReasonTag::None ; NodeIdx node = 0 ; + // END_OF_VERSIONING } ; struct JobStats { using Delay = Time::Delay ; // data + // START_OF_VERSIONING Delay cpu = {} ; Delay job = {} ; // elapsed in job Delay total = {} ; // elapsed including overhead size_t mem = 0 ; // in bytes + // END_OF_VERSIONING } ; template struct DepDigestBase ; @@ -337,19 +369,14 @@ struct CrcDate { using Crc = Hash::Crc ; using Ddate = Time::Ddate ; //cxtors & casts - /**/ CrcDate( ) : _crc{} { } // XXX : implement cxtors and just provide copy assignment - /**/ CrcDate(Crc c ) { *this = c ; } - /**/ CrcDate(Ddate d ) { *this = d ; } - /**/ CrcDate(CrcDate const& cd ) { *this = cd ; } - template CrcDate(DepDigestBase const& ddb) { *this = ddb ; } + CrcDate( ) : _crc { } {} + CrcDate(Crc c ) : is_date{false} , _crc {c} {} + CrcDate(Ddate d ) : is_date{true } , _date{d} {} // - /**/ CrcDate& operator=(Crc c ) { is_date = false ; _crc = c ; return *this ; } - /**/ CrcDate& operator=(Ddate d ) { is_date = true ; _date = d ; return *this ; } - /**/ CrcDate& operator=(CrcDate const& cd ) { { if (cd .is_date) *this=cd.date() ; else *this=cd .crc() ; } return *this ; } - template CrcDate& operator=(DepDigestBase const& ddb) { - if (!ddb.accesses) return *this = Crc() ; - if ( ddb.is_date ) return *this = ddb.date() ; - /**/ return *this = ddb.crc () ; + template CrcDate(DepDigestBase const& ddb) { + if (!ddb.accesses) *this = Crc() ; + else if ( ddb.is_date ) *this = ddb.date() ; + else *this = ddb.crc () ; } // accesses bool operator==(CrcDate const& other) const { @@ -369,12 +396,14 @@ struct CrcDate { else return !Crc::None.match( _crc , a ) ; } // data + // START_OF_VERSIONING bool is_date = false ; private : union { Crc _crc ; // ~46<64 bits Ddate _date ; // ~45<64 bits } ; + // END_OF_VERSIONING } ; // for Dep recording in book-keeping, we want to derive from Node @@ -457,6 +486,7 @@ template struct DepDigestBase : NoVoid { } } // data + // START_OF_VERSIONING static constexpr uint8_t NSzBits = 6 ; Accesses accesses ; // 3< 8 bits Dflags dflags ; // 6< 8 bits @@ -469,8 +499,8 @@ private : Crc _crc = {} ; // ~45<64 bits Ddate _date ; // ~40<64 bits } ; + // END_OF_VERSIONING } ; -// END_OF_VERSIONING template ::ostream& operator<<( ::ostream& os , DepDigestBase const& dd ) { const char* sep = "" ; /**/ os << "D(" ; @@ -482,7 +512,6 @@ template ::ostream& operator<<( ::ostream& os , DepDigestBase const& else if ( +dd.accesses && +dd.crc() ) { os < ; static_assert(::is_trivially_copyable_v) ; // as long as this holds, we do not have to bother about union member cxtor/dxtor @@ -491,16 +520,19 @@ struct TargetDigest { friend ::ostream& operator<<( ::ostream& , TargetDigest const& ) ; using Crc = Hash::Crc ; // data + // START_OF_VERSIONING Tflags tflags = {} ; ExtraTflags extra_tflags = {} ; bool polluted = false ; // if true <= file was seen as existing while not incremental Crc crc = {} ; // if None <=> file was unlinked, if Unknown => file is idle (not written, not unlinked) Time::Ddate date = {} ; + // END_OF_VERSIONING } ; struct JobDigest { friend ::ostream& operator<<( ::ostream& , JobDigest const& ) ; // data + // START_OF_VERSIONING Status status = Status::New ; ::vmap_s targets = {} ; ::vmap_s deps = {} ; // INVARIANT : sorted in first access order @@ -509,6 +541,7 @@ struct JobDigest { int wstatus = 0 ; Time::Pdate end_date = {} ; JobStats stats = {} ; + // END_OF_VERSIONING } ; struct JobExecRpcReq ; @@ -522,15 +555,9 @@ struct JobRpcReq { // statics // cxtors & casts JobRpcReq() = default ; - JobRpcReq( P p , SI si , JI j ) : proc{p} , seq_id{si} , job{j} { SWEAR( p==P::None ) ; } - JobRpcReq( P p , SI si , JI j , in_port_t pt , ::string&& m={} ) : proc{p} , seq_id{si} , job{j} , port {pt } , msg{::move(m)} { SWEAR( p==P::Start ) ; } - JobRpcReq( P p , SI si , JI j , Status s , ::string&& m={} ) : proc{p} , seq_id{si} , job{j} , digest{.status=s } , msg{::move(m)} { SWEAR( p==P::End && s<=Status::Garbage ) ; } - JobRpcReq( P p , SI si , JI j , JobDigest&& d , ::string&& m={} ) : proc{p} , seq_id{si} , job{j} , digest{::move(d) } , msg{::move(m)} { SWEAR( p==P::End ) ; } - JobRpcReq( P p , SI si , JI j , ::string&& m ) : proc{p} , seq_id{si} , job{j} , msg{::move(m)} { SWEAR( p==P::LiveOut ) ; } - JobRpcReq( P p , SI si , JI j , MDD&& ds ) : proc{p} , seq_id{si} , job{j} , digest{.deps=::move(ds)} { SWEAR( p==P::ChkDeps || p==P::DepInfos ) ; } - // - JobRpcReq( P p , SI si , JI j , ::string&& code , ::string&& f , ::string&& c ) : proc{p} , seq_id{si} , job{j} , msg{code} , file{f} , ctx{c} { SWEAR(p==P::Decode) ; } - JobRpcReq( P p , SI si , JI j , ::string&& val , ::string&& f , ::string&& c , uint8_t ml ) : proc{p} , seq_id{si} , job{j} , msg{val } , file{f} , ctx{c} , min_len{ml} { SWEAR(p==P::Encode) ; } + JobRpcReq( P p , SI si , JI j ) : proc{p} , seq_id{si} , job{j} { SWEAR(p==P::None ) ; } + JobRpcReq( P p , SI si , JI j , in_port_t pt , ::string&& m={} ) : proc{p} , seq_id{si} , job{j} , port {pt } , msg{::move(m)} { SWEAR(p==P::Start) ; } + JobRpcReq( P p , SI si , JI j , JobDigest&& d , ::string&& m={} ) : proc{p} , seq_id{si} , job{j} , digest{::move(d)} , msg{::move(m)} { SWEAR(p==P::End ) ; } // JobRpcReq( SI si , JI j , JobExecRpcReq&& jerr ) ; // services @@ -545,17 +572,6 @@ struct JobRpcReq { ::serdes(s,port) ; ::serdes(s,msg ) ; break ; - case P::LiveOut : ::serdes(s,msg ) ; break ; - case P::ChkDeps : ::serdes(s,digest) ; break ; - case P::DepInfos : ::serdes(s,digest) ; break ; - case P::Encode : - ::serdes(s,min_len) ; - [[fallthrough]] ; - case P::Decode : - ::serdes(s,msg ) ; - ::serdes(s,file) ; - ::serdes(s,ctx ) ; - break ; case P::End : ::serdes(s,digest ) ; ::serdes(s,dynamic_env) ; @@ -564,16 +580,15 @@ struct JobRpcReq { DF} } // data + // START_OF_VERSIONING P proc = P::None ; SI seq_id = 0 ; JI job = 0 ; - in_port_t port = 0 ; // if proc == Start - JobDigest digest ; // if proc == ChkDeps | DepInfos | End - ::vmap_ss dynamic_env ; // if proc == End env variables computed in job_exec - ::string msg ; // if proc == Start | LiveOut | Decode | Encode | End - ::string file ; // if proc == Decode | Encode - ::string ctx ; // if proc == Decode | Encode - uint8_t min_len = 0 ; // if proc == Encode + in_port_t port = 0 ; // if proc==Start + JobDigest digest ; // if proc== End + ::vmap_ss dynamic_env ; // if proc== End, env variables computed in job_exec + ::string msg ; + // END_OF_VERSIONING } ; struct MatchFlags { @@ -592,12 +607,15 @@ struct MatchFlags { // data Bool3 is_target = Maybe ; private : + // START_OF_VERSIONING Tflags _tflags ; // if is_target Dflags _dflags ; // if !is_target ExtraTflags _extra_tflags ; // if is_target ExtraDflags _extra_dflags ; // if !is_target + // END_OF_VERSIONING } ; +// START_OF_VERSIONING ENUM_2( AutodepMethod // PER_AUTODEP_METHOD : add entry here , Ld = LdAudit // >=Ld means a lib is pre-loaded (through LD_AUDIT or LD_PRELOAD) , Dflt = HAS_LD_AUDIT?LdAudit:LdPreload // by default, use a compromize between speed an reliability @@ -607,32 +625,22 @@ ENUM_2( AutodepMethod // PER_AUTODEP_METHOD : add entry here , LdPreload , LdPreloadJemalloc ) +// END_OF_VERSIONING struct JobRpcReply { friend ::ostream& operator<<( ::ostream& , JobRpcReply const& ) ; using Crc = Hash::Crc ; using Proc = JobProc ; // cxtors & casts - JobRpcReply( ) = default ; - JobRpcReply( Proc p ) : proc{p} { } - JobRpcReply( Proc p , Bool3 o ) : proc{p} , ok{o} { SWEAR( proc==Proc::ChkDeps ) ; } - JobRpcReply( Proc p , ::vector> const& is ) : proc{p} , dep_infos{is} { SWEAR( proc==Proc::DepInfos ) ; } - JobRpcReply( Proc p , ::string const& t , Crc c , Bool3 o ) : proc{p} , ok{o} , txt{t} , crc{c} { SWEAR( proc==Proc::Decode || proc==Proc::Encode ) ; } + JobRpcReply( ) = default ; + JobRpcReply(Proc p) : proc{p} {} // services template void serdes(S& s) { if (is_base_of_v<::istream,S>) *this = {} ; ::serdes(s,proc) ; switch (proc) { - case Proc::None : - case Proc::End : break ; - case Proc::DepInfos : ::serdes(s,dep_infos) ; break ; - case Proc::ChkDeps : ::serdes(s,ok ) ; break ; - case Proc::Decode : - case Proc::Encode : - ::serdes(s,ok ) ; - ::serdes(s,txt) ; - ::serdes(s,crc) ; - break ; + case Proc::None : + case Proc::End : break ; case Proc::Start : ::serdes(s,addr ) ; ::serdes(s,autodep_env ) ; @@ -661,39 +669,143 @@ struct JobRpcReply { DF} } // data - Proc proc = Proc::None ; - in_addr_t addr = 0 ; // proc == Start , the address at which server and subproccesses can contact job_exec - AutodepEnv autodep_env ; // proc == Start - ::string chroot ; // proc == Start - ::pair_ss/*script,call*/ cmd ; // proc == Start - ::string cwd_s ; // proc == Start - ::vmap_s deps ; // proc == Start , deps already accessed (always includes static deps) - ::vmap_ss env ; // proc == Start - Algo hash_algo = Algo::Xxh ; // proc == Start - ::vector_s interpreter ; // proc == Start , actual interpreter used to execute cmd - bool keep_tmp = false ; // proc == Start - vector kill_sigs ; // proc == Start - bool live_out = false ; // proc == Start - AutodepMethod method = {}/*garbage*/ ; // proc == Start - Time::Delay network_delay ; // proc == Start - ::vmap_s pre_actions ; // proc == Start - ::string remote_admin_dir ; // proc == Start - SmallId small_id = 0 ; // proc == Start - ::vmap_s star_matches ; // proc == Start , maps regexprs to flags - ::vmap_s static_matches ; // proc == Start - ::string stdin ; // proc == Start - ::string stdout ; // proc == Start - Time::Delay timeout ; // proc == Start - bool use_script = false ; // proc == Start - ::vector> dep_infos ; // proc == DepInfos - Bool3 ok = Maybe ; // proc == ChkDeps|Decode|Encode , if No <=> deps in error, if Maybe <=> deps not ready - ::string txt ; // proc == Decode|Encode , value for Decode, code for Encode - Crc crc ; // proc == Decode|Encode , crc of txt + // START_OF_VERSIONING + Proc proc = Proc::None ; + in_addr_t addr = 0 ; // proc == Start , the address at which server and subproccesses can contact job_exec + AutodepEnv autodep_env ; // proc == Start + ::string chroot ; // proc == Start + ::pair_ss/*script,call*/ cmd ; // proc == Start + ::string cwd_s ; // proc == Start + ::vmap_s deps ; // proc == Start , deps already accessed (always includes static deps) + ::vmap_ss env ; // proc == Start + Algo hash_algo = Algo::Xxh ; // proc == Start + ::vector_s interpreter ; // proc == Start , actual interpreter used to execute cmd + bool keep_tmp = false ; // proc == Start + vector kill_sigs ; // proc == Start + bool live_out = false ; // proc == Start + AutodepMethod method = {} ; // proc == Start + Time::Delay network_delay ; // proc == Start + ::vmap_s pre_actions ; // proc == Start + ::string remote_admin_dir ; // proc == Start + SmallId small_id = 0 ; // proc == Start + ::vmap_s star_matches ; // proc == Start , maps regexprs to flags + ::vmap_s static_matches ; // proc == Start + ::string stdin ; // proc == Start + ::string stdout ; // proc == Start + Time::Delay timeout ; // proc == Start + bool use_script = false ; // proc == Start + // END_OF_VERSIONING } ; -// END_OF_VERSIONING +struct JobMngtRpcReq { + using JMMR = JobMngtRpcReq ; + using P = JobMngtProc ; + using SI = SeqId ; + using JI = JobIdx ; + using MDD = ::vmap_s ; + friend ::ostream& operator<<( ::ostream& , JobMngtRpcReq const& ) ; + // statics + // cxtors & casts + #define S ::string + #define M ::move + JobMngtRpcReq( ) = default ; + JobMngtRpcReq( P p , SI si , JI j , Fd fd_ ) : proc{p} , seq_id{si} , job{j} , fd{fd_} {} + // + JobMngtRpcReq(P p,SI si,JI j, S&& t ) : proc{p},seq_id{si},job{j}, txt{M(t)} { SWEAR(p==P::LiveOut ) ; } + JobMngtRpcReq(P p,SI si,JI j,Fd fd_,MDD&& ds) : proc{p},seq_id{si},job{j},fd{fd_},deps{M(ds)} { SWEAR(p==P::ChkDeps||p==P::DepInfos) ; } + // + JobMngtRpcReq(P p,SI si,JI j,Fd fd_,S&& code,S&& f,S&& c ) : proc{p},seq_id{si},job{j},fd{fd_},ctx{M(c)},file{M(f)},txt{M(code)} { SWEAR(p==P::Decode) ; } + JobMngtRpcReq(P p,SI si,JI j,Fd fd_,S&& val ,S&& f,S&& c,uint8_t ml) : proc{p},seq_id{si},job{j},fd{fd_},ctx{M(c)},file{M(f)},txt{M(val )},min_len{ml} { SWEAR(p==P::Encode) ; } + // + JobMngtRpcReq( SI , JI , Fd , JobExecRpcReq&& ) ; + #undef M + #undef S + // services + template void serdes(T& s) { + if (::is_base_of_v<::istream,T>) *this = {} ; + ::serdes(s,proc ) ; + ::serdes(s,seq_id) ; + ::serdes(s,job ) ; + switch (proc) { + case P::None : break ; + case P::LiveOut : ::serdes(s,txt ) ; break ; + case P::ChkDeps : + case P::DepInfos : + ::serdes(s,fd ) ; + ::serdes(s,deps) ; + break ; + case P::Encode : + ::serdes(s,min_len) ; + [[fallthrough]] ; + case P::Decode : + ::serdes(s,fd ) ; + ::serdes(s,ctx ) ; + ::serdes(s,file) ; + ::serdes(s,txt ) ; + break ; + DF} + } + // data + P proc = P::None ; + SI seq_id = 0 ; + JI job = 0 ; + Fd fd ; // fd to which reply must be forwarded + ::vmap_s deps ; // proc==ChkDeps|DepInfos + ::string ctx ; // proc== Decode|Encode + ::string file ; // proc== Decode|Encode + ::string txt ; // proc== LiveOut|Decode|Encode + uint8_t min_len = 0 ; // proc== Encode +} ; + +struct JobMngtRpcReply { + friend ::ostream& operator<<( ::ostream& , JobMngtRpcReply const& ) ; + using Crc = Hash::Crc ; + using Proc = JobMngtProc ; + // cxtors & casts + JobMngtRpcReply() = default ; + // + JobMngtRpcReply( Proc p , SeqId si ) : proc{p} , seq_id{si} { SWEAR(proc==Proc::Kill||proc==Proc::Heartbeat) ; } + // + JobMngtRpcReply( Proc p , SeqId si , Fd fd_ , Bool3 o ) : proc{p},seq_id{si},fd{fd_},ok{o} { SWEAR(proc==Proc::ChkDeps ) ; } + JobMngtRpcReply( Proc p , SeqId si , Fd fd_ , ::vector> const& is ) : proc{p},seq_id{si},fd{fd_}, dep_infos{is} { SWEAR(proc==Proc::DepInfos ) ; } + JobMngtRpcReply( Proc p , SeqId si , Fd fd_ , ::string const& t , Crc c , Bool3 o ) : proc{p},seq_id{si},fd{fd_},ok{o},txt{t},crc{c} { SWEAR(proc==Proc::Decode||proc==Proc::Encode) ; } + // services + template void serdes(S& s) { + if (is_base_of_v<::istream,S>) *this = {} ; + ::serdes(s,proc ) ; + ::serdes(s,seq_id) ; + switch (proc) { + case Proc::None : + case Proc::Kill : + case Proc::Heartbeat : break ; + case Proc::DepInfos : + ::serdes(s,fd ) ; + ::serdes(s,dep_infos) ; + break ; + case Proc::ChkDeps : + ::serdes(s,fd) ; + ::serdes(s,ok) ; + break ; + case Proc::Decode : + case Proc::Encode : + ::serdes(s,fd ) ; + ::serdes(s,ok ) ; + ::serdes(s,txt) ; + ::serdes(s,crc) ; + break ; + DF} + } + // data + Proc proc = {} ; + SeqId seq_id = 0 ; + Fd fd ; // proc == ChkDeps|DepInfos|Decode|Encode , fd to which reply must be forwarded + ::vector> dep_infos ; // proc == DepInfos + Bool3 ok = Maybe ; // proc == ChkDeps| Decode|Encode , if No <=> deps in error, if Maybe <=> deps not ready + ::string txt ; // proc == Decode|Encode , value for Decode, code for Encode + Crc crc ; // proc == Decode|Encode , crc of txt +} ; -ENUM_1( JobExecRpcProc +ENUM_1( JobExecProc , HasFiles = Access // >=HasFiles means files field is significative , None , ChkDeps @@ -729,10 +841,10 @@ struct AccessDigest { // order struct JobExecRpcReq { friend ::ostream& operator<<( ::ostream& , JobExecRpcReq const& ) ; // make short lines - using AD = AccessDigest ; - using P = JobExecRpcProc ; - using PD = Time::Pdate ; - using DD = Time::Ddate ; + using AD = AccessDigest ; + using P = JobExecProc ; + using PD = Time::Pdate ; + using DD = Time::Ddate ; // statics private : static ::vmap_s
_s_mk_mdd(::vector_s&& fs) { ::vmap_s
res ; for( ::string& f : fs ) res.emplace_back(::move(f),DD()) ; return res ; } @@ -831,8 +943,8 @@ public : struct JobExecRpcReply { friend ::ostream& operator<<( ::ostream& , JobExecRpcReply const& ) ; - using Proc = JobExecRpcProc ; - using Crc = Hash::Crc ; + using Proc = JobExecProc ; + using Crc = Hash::Crc ; // cxtors & casts JobExecRpcReply( ) = default ; JobExecRpcReply( Proc p ) : proc{p} { SWEAR( proc!=Proc::ChkDeps && proc!=Proc::DepInfos ) ; } @@ -840,7 +952,7 @@ struct JobExecRpcReply { JobExecRpcReply( Proc p , ::vector> const& is ) : proc{p} , dep_infos{is} { SWEAR( proc==Proc::DepInfos ) ; } JobExecRpcReply( Proc p , ::string const& t ) : proc{p} , txt {t } { SWEAR( proc==Proc::Decode || proc==Proc::Encode ) ; } // - JobExecRpcReply( JobRpcReply const& jrr ) ; + JobExecRpcReply( JobMngtRpcReply&& jrr ) ; // services template void serdes(S& s) { if (::is_base_of_v<::istream,S>) *this = {} ; @@ -858,42 +970,9 @@ struct JobExecRpcReply { } // data Proc proc = Proc::None ; - Bool3 ok = Maybe ; // if proc==ChkDeps + Bool3 ok = Maybe ; // if proc==ChkDeps |Decode|Encode ::vector> dep_infos ; // if proc==DepInfos - ::string txt ; // if proc==Decode|Encode (value for Decode, code for Encode) -} ; - -// START_OF_VERSIONING - -// -// JobSserverRpcReq -// - -ENUM( JobServerRpcProc -, Heartbeat -, Kill -) - -struct JobServerRpcReq { - friend ::ostream& operator<<( ::ostream& , JobServerRpcReq const& ) ; - using Proc = JobServerRpcProc ; - // cxtors & casts - JobServerRpcReq( ) = default ; - JobServerRpcReq( Proc p , SeqId si ) : proc{p} , seq_id{si} { SWEAR(proc==Proc::Kill ) ; } - JobServerRpcReq( Proc p , SeqId si , JobIdx j ) : proc{p} , seq_id{si} , job{j} { } // need a job for heartbeat as we may have to reply on its behalf - // services - template void serdes(S& s) { - ::serdes(s,proc ) ; - ::serdes(s,seq_id) ; - switch (proc) { - case Proc::Heartbeat : ::serdes(s,job) ; break ; - case Proc::Kill : if (::is_base_of_v<::istream,S>) job = 0 ; break ; - DF} - } - // data - Proc proc = {} ; - SeqId seq_id = 0 ; - JobIdx job = 0 ; + ::string txt ; // if proc== Decode|Encode (value for Decode, code for Encode) } ; struct SubmitAttrs { @@ -915,17 +994,20 @@ struct SubmitAttrs { return res ; } // data + // START_OF_VERSIONING BackendTag tag = {} ; bool live_out = false ; uint8_t n_retries = 0 ; Time::CoarseDelay pressure = {} ; ::vmap_s deps = {} ; JobReason reason = {} ; + // END_OF_VERSIONING } ; struct JobInfoStart { friend ::ostream& operator<<( ::ostream& , JobInfoStart const& ) ; // data + // START_OF_VERSIONING Hash::Crc rule_cmd_crc = {} ; ::vector_s stems = {} ; Time::Pdate eta = {} ; @@ -935,12 +1017,15 @@ struct JobInfoStart { JobRpcReq pre_start = {} ; JobRpcReply start = {} ; ::string stderr = {} ; + // END_OF_VERSIONING } ; struct JobInfoEnd { friend ::ostream& operator<<( ::ostream& , JobInfoEnd const& ) ; // data + // START_OF_VERSIONING JobRpcReq end = {} ; + // END_OF_VERSIONING } ; struct JobInfo { @@ -951,8 +1036,10 @@ struct JobInfo { // ervices void write(::string const& filename) const ; // data + // START_OF_VERSIONING JobInfoStart start ; JobInfoEnd end ; + // END_OF_VERSIONING } ; // @@ -969,5 +1056,3 @@ namespace Codec { ::string mk_file(::string const& node) ; // node may have been obtained from mk_decode_node or mk_encode_node } - -// END_OF_VERSIONING diff --git a/src/serialize.hh b/src/serialize.hh index 03e30f34..b8aad04d 100644 --- a/src/serialize.hh +++ b/src/serialize.hh @@ -5,7 +5,7 @@ #pragma once -#include "fd.hh" +#include "utils.hh" template struct Serdeser ; @@ -131,91 +131,3 @@ template struct Serdeser<::pair> { static void s_serdes( ::ostream& s , ::pair const& p ) { serdes(s,p.first ) ; serdes(s,p.second) ; } static void s_serdes( ::istream& s , ::pair & p ) { serdes(s,p.first ) ; serdes(s,p.second) ; } } ; - -// -// MsgBuf -// - -struct MsgBuf { - friend ::ostream& operator<<( ::ostream& , MsgBuf const& ) ; - using Len = size_t ; - // statics - static Len s_sz(const char* str) { - Len len = 0 ; ::memcpy( &len , str , sizeof(Len) ) ; - return len ; - } -protected : - // data - Len _len = 0 ; // data sent/received so far, reading : may also apply to len accumulated in buf - ::string _buf ; // reading : sized after expected size, but actuall filled up only with len char's // writing : contains len+data to be sent - bool _data_pass = false ; // reading : if true <=> buf contains partial data, else it contains partial data len // writing : if true <=> buf contains data -} ; -inline ::ostream& operator<<( ::ostream& os , MsgBuf const& mb ) { return os<<"MsgBuf("< static T s_receive(const char* str) { - Len len = s_sz(str) ; - // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - return deserialize(IStringStream(::string( str+sizeof(Len) , len ))) ; // XXX : avoid copy (use string_view in C++26) - // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - } - // cxtors & casts - IMsgBuf() { _buf.resize(sizeof(Len)) ; } // prepare to receive len - // services - template T receive(Fd fd) { - T res ; - while (!receive_step(fd,res)) ; - return res ; - } - template bool/*complete*/ receive_step( Fd fd , T& res ) { - ssize_t cnt = ::read( fd , &_buf[_len] , _buf.size()-_len ) ; - if (cnt<=0) throw to_string("cannot receive over fd ",fd) ; - _len += cnt ; - if (_len<_buf.size()) return false/*complete*/ ; // _buf is still partial - if (_data_pass) { - // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - res = deserialize(IStringStream(::move(_buf))) ; - // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - *this = {} ; - return true/*complete*/ ; - } else { - SWEAR( _buf.size()==sizeof(Len) , _buf.size() ) ; - Len len = s_sz(_buf.data()) ; - // we now expect the data - _buf.resize(len) ; - _data_pass = true ; - _len = 0 ; - return false/*complete*/ ; - } - } -} ; - -struct OMsgBuf : MsgBuf { - // statics - template static ::string s_send(T const& x) { - ::string res = serialize(::pair(0,x)) ; SWEAR(res.size()>=sizeof(Len)) ; // directly serialize in res to avoid copy : serialize a pair with length+data - Len len = res.size()-sizeof(Len) ; - ::memcpy( res.data() , &len , sizeof(Len) ) ; // overwrite len - return res ; - } - // services - template void send( Fd fd , T const& x ) { - if (send_step(fd,x)) return ; - while (!send_step(fd)) ; - } - template bool/*complete*/ send_step( Fd fd , T const& x ) { - SWEAR(!_data_pass) ; - _buf = s_send(x) ; - _data_pass = true ; - return send_step(fd) ; - } - bool/*complete*/ send_step(Fd fd) { - SWEAR(_data_pass) ; - ssize_t cnt = ::write( fd , &_buf[_len] , _buf.size()-_len ) ; - if (cnt<=0) throw to_string("cannot send over ",fd) ; - _len += cnt ; - if (_len<_buf.size()) { return false/*complete*/ ; } // _buf is still partial - else { *this = {} ; return true /*complete*/ ; } - } -} ; diff --git a/src/thread.hh b/src/thread.hh index b47dbd3f..d161a1a5 100644 --- a/src/thread.hh +++ b/src/thread.hh @@ -10,9 +10,9 @@ #include #include +#include "msg.hh" #include "time.hh" #include "trace.hh" -#include "serialize.hh" template struct ThreadQueue : private ::deque { using ThreadMutex = Mutex ; @@ -131,10 +131,12 @@ private : switch (kind) { case EventKind::Master : { SWEAR(efd==self->fd) ; - SlaveSockFd slave_fd{self->fd.accept()} ; - trace("new_req",slave_fd) ; - epoll.add_read(slave_fd,EventKind::Slave) ; - slaves.try_emplace(::move(slave_fd)) ; + try { + SlaveSockFd slave_fd { self->fd.accept() } ; + trace("new_req",slave_fd) ; + epoll.add_read(slave_fd,EventKind::Slave) ; + slaves.try_emplace(::move(slave_fd)) ; + } catch (::string const& e) { trace("cannot_accept",e) ; } // ignore error as this may be fd starvation and client will retry } break ; case EventKind::Stop : { uint64_t one ; @@ -150,12 +152,12 @@ private : try { if (!slaves.at(efd).receive_step(efd,r)) { trace("partial") ; continue ; } } catch (...) { trace("bad_msg") ; continue ; } // ignore malformed messages // - epoll.del(efd) ; // Func may trigger efd being closed by another thread, hence epoll.del must be done before + epoll.del(efd) ; // Func may trigger efd being closed by another thread, hence epoll.del must be done before slaves.erase(efd) ; SlaveSockFd ssfd { efd } ; bool keep = false/*garbage*/ ; keep=func(::move(r),ssfd) ; - if (keep) ssfd.detach() ; // dont close ssfd if requested to keep it + if (keep) ssfd.detach() ; // dont close ssfd if requested to keep it trace("called",STR(keep)) ; } break ; DF} @@ -165,7 +167,7 @@ private : } // cxtors & casts public : - ServerThread(char key, ::function func ,int backlog=0) : fd{New,backlog} , _thread{_s_thread_func,key,this,func} {} + ServerThread( char key , ::function func , int backlog=0 ) : fd{New,backlog} , _thread{_s_thread_func,key,this,func} {} // services void wait_started() { _ready.wait() ; @@ -174,5 +176,5 @@ public : ServerSockFd fd ; private : ::latch _ready {1} ; - ::jthread _thread ; // ensure _thread is last so other fields are constructed when it starts + ::jthread _thread ; // ensure _thread is last so other fields are constructed when it starts } ; diff --git a/src/time.hh b/src/time.hh index 315b9615..fe4ceee4 100644 --- a/src/time.hh +++ b/src/time.hh @@ -87,6 +87,7 @@ namespace Time { friend CoarseDelay ; static const Delay Lowest ; static const Delay Highest ; + static const Delay Forever ; // statics private : static bool/*slept*/ _s_sleep( ::stop_token tkn , Delay sleep , Pdate until ) ; @@ -119,6 +120,7 @@ namespace Time { } ; constexpr Delay Delay::Lowest { New , ::numeric_limits::min() } ; constexpr Delay Delay::Highest { New , ::numeric_limits::max() } ; + constexpr Delay Delay::Forever { New , ::numeric_limits::max() } ; // short float representation of time (positive) // when exp<=0, representation is linear after TicksPerSecond @@ -138,14 +140,12 @@ namespace Time { // cxtors & casts explicit constexpr CoarseDelay( NewType , Val v ) : _val{v} {} public : - constexpr CoarseDelay() = default ; - constexpr CoarseDelay(Delay d) { *this = d ; } // XXX : implemment cxtor and remove operator= - constexpr CoarseDelay& operator=(Delay d) { + constexpr CoarseDelay( ) = default ; + constexpr CoarseDelay(Delay d) { uint32_t t = ::logf(d._val)*(1<= (1<)+Scale ) _val = -1 ; else if ( t < Scale ) _val = 0 ; else _val = t-Scale ; - return *this ; } constexpr operator Delay() const { if (!_val) return Delay() ; diff --git a/unit_tests/conflict2.py b/unit_tests/conflict2.py index 2d35f521..d04586c1 100644 --- a/unit_tests/conflict2.py +++ b/unit_tests/conflict2.py @@ -15,8 +15,8 @@ class A(Rule): def cmd(): pass class B(Rule): - targets = { 'B' : 'b' } - side_deps = { 'A' : ('a{*:.*}','Ignore') } + targets = { 'B' : 'b' } + side_deps = { 'A' : (r'a{*:.*}','Ignore') } def cmd(): open(A('') ) open(B ,'w') diff --git a/unit_tests/tmp2.py b/unit_tests/tmp2.py new file mode 100644 index 00000000..dd172c77 --- /dev/null +++ b/unit_tests/tmp2.py @@ -0,0 +1,137 @@ +# This file is part of the open-lmake distribution (git@github.com:cesar-douady/open-lmake.git) +# Copyright (c) 2023 Doliam +# This program is free software: you can redistribute/modify under the terms of the GPL-v3 (https://www.gnu.org/licenses/gpl-3.0.html). +# This program is distributed WITHOUT ANY WARRANTY, without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +import lmake + +if __name__!='__main__' : + + from lmake import multi_strip + from lmake.rules import Rule + + lmake.manifest = ( + 'Lmakefile.py' + , 'step.py' + , 'cp_dut.ref' + , 'exec_dut.ref' + , 'lnk_dut.ref' + , 'touch_dut.ref' + ) + + from step import autodep + + class TmpRule(Rule) : + tmp = '/tmp' + autodep = autodep + cmd = f'#{autodep}' # force cmd modification + + class Src(Rule) : + target = 'src' + cmd = 'echo src_content' + + class Chk(Rule) : + target = '{File:.*}.ok' + deps = { + 'DUT' : '{File}' + , 'REF' : '{File}.ref' + } + cmd = 'diff {REF} {DUT} >&2' + + class Lnk(TmpRule) : + target = 'lnk_dut' + side_targets = { 'SIDE' : 'lnk_dut.tmp' } + # make a chain of links with all potential cases + cmd = multi_strip(''' + ln -s /tmp/a lnk_dut.tmp # link from repo to tmp + cd /tmp + ln -s b a # relative link within tmp + ln -s /tmp/c b # absolute link within tmp + ln -s $ROOT_DIR/src c # link from tmp to repo + readlink a b c $ROOT_DIR/lnk_dut.tmp + cd $ROOT_DIR + readlink /tmp/a /tmp/b /tmp/c lnk_dut.tmp + cat lnk_dut.tmp + ''') + + class Cp(TmpRule) : + target = 'cp_dut' + side_targets = { 'SIDE' : 'cp_dut.tmp' } + # make a chain of copies with all potential cases + cmd = multi_strip(''' + cp $ROOT_DIR/src /tmp/c + cd /tmp + cp /tmp/c b + cp b a + cd $ROOT_DIR + cp /tmp/a cp_dut.tmp + cat cp_dut.tmp + ''') + + class Touch(TmpRule) : + target = 'touch_dut' + side_targets = { 'SIDE' : 'touch_dut.tmp' } + cmd = multi_strip(''' + cd /tmp + mkdir d + cp $ROOT_DIR/src d/a + cd d + pwd > $ROOT_DIR/{SIDE} + mv a b + sleep 0.1 # ensure a and b have different dates + cp b c + ls -l --full-time c > date1 + touch -r b c + ls -l --full-time c > date2 + cmp date1 date2 >/dev/null && {{ echo 'touch did not change date' >&2 ; exit 1 ; }} + cd .. + cat d/c + rm -r d + [ -f d/c ] && {{ echo 'rm did not rm' >&2 ; exit 1 ; }} + cd $ROOT_DIR + cat {SIDE} + ''') + + class Exec(TmpRule) : + target = 'exec_dut' + side_targets = { 'SIDE' : 'exec_dut.tmp' } + cmd = multi_strip(''' + PATH=/tmp:$PATH + echo '#!/bin/cat' > /tmp/dut_exe #ensure there is an execve as bash optimizes cases where it calls itself + echo 'dut_exe_content' >> /tmp/dut_exe + chmod +x /tmp/dut_exe + dut_exe > exec_dut.tmp + cat exec_dut.tmp + ''') + +else : + + import ut + + with open('lnk_dut.ref','w') as f : + for p in range(2) : + print('b' ,file=f) + print('/tmp/c' ,file=f) + print(f'{lmake.root_dir}/src',file=f) + print('/tmp/a' ,file=f) + print('src_content',file=f) + + with open('cp_dut.ref','w') as f : + print('src_content',file=f) + + with open('touch_dut.ref','w') as f : + print('src_content',file=f) + print('/tmp/d' ,file=f) + + with open('exec_dut.ref','w') as f : + print('#!/bin/cat' ,file=f) + print('dut_exe_content',file=f) + + print("autodep='ld_preload'",file=open('step.py','w')) + res = ut.lmake( 'lnk_dut.ok' , 'cp_dut.ok' , 'touch_dut.ok' , 'exec_dut.ok' , new=4 , may_rerun=... , rerun=... , done=... , steady=... ) + assert 1<=res['may_rerun']+res['rerun'] and res['may_rerun']+res['rerun']<=4 + assert res['done']+res['steady']==9 + + if lmake.has_ld_audit : + print("autodep='ld_audit'",file=open('step.py','w')) + res = ut.lmake( 'lnk_dut.ok' , 'cp_dut.ok' , 'touch_dut.ok' , 'exec_dut.ok' , new=0 , steady=4 ) diff --git a/unit_tests/wine.py b/unit_tests/wine.py index ecf3845f..520985d4 100644 --- a/unit_tests/wine.py +++ b/unit_tests/wine.py @@ -23,6 +23,8 @@ lmake.manifest = ('Lmakefile.py',) + lmake.config.network_delay = 10 # WineInit is still alive after job end for ~1s but may last more than 5s + class Base(Rule) : stems = { 'Method' : r'\w+' } @@ -34,7 +36,7 @@ class WineRule(Rule) : class WineInit(WineRule) : target = '.wine/init' targets = { 'WINE' : '.wine/{*:.*}' } # for init wine env is not incremental - side_targets = { 'WINE' : None } + side_targets = { 'WINE' : None } allow_stderr = True cmd = 'wine64 cmd >$TMPDIR/out 2>$TMPDIR/err ; cat $TMPDIR/out ; cat $TMPDIR/err >&2' # do nothing, just to init support files (in targets), avoid waiting for stdout/stderr