reworked crc computation + fix PyRule w/ ptrace + fixed dep managemen…

…t bug in engine + fixed cycle detection

.gitignore

@@ -25,7 +25,6 @@ __pycache__
 *.d
 *.hh.gch
 *.so
-/ext/**.patched*
 *.stamp
 */stamp
 *.inc_stamp

Makefile

@@ -299,14 +299,6 @@ ext/%.dir.stamp : ext/%.zip
 	@mkdir -p $(@:%.stamp=%)
 	unzip -d $(@:%.stamp=%) $<
 	touch $@
-ext/%.patched.stamp : ext/%.dir.stamp ext/%.patch_script
-	rm -rf $(@:%.stamp=%)
-	cp -r $(@:%.patched.stamp=%.dir) $(@:%.patched.stamp=%.patched)
-	cd $(@:%.patched.stamp=%.patched) ; $(ROOT_DIR)/$(@:%.patched.stamp=%.patch_script)
-	touch $@
-ext/%.patched.h : ext/%.h ext/%.patch_script
-	cp $< $@
-	cd $(@D) ; $(ROOT_DIR)/$(@:%.patched.h=%.patch_script) $(@F)
 
 .SECONDARY :
 
@@ -394,7 +386,7 @@ $(STORE_LIB)/big_test.dir/tok : $(STORE_LIB)/big_test.py LMAKE
 # engine
 #
 
-ALL_H := version.hh sys_config.h ext/xxhash.patched.h
+ALL_H := version.hh sys_config.h ext/xxhash.h
 
 # On ubuntu, seccomp.h is in /usr/include. On CenOS7, it is in /usr/include/linux, but beware that otherwise, /usr/include must be prefered, hence -idirafter
 CPP_OPTS := -iquote ext -iquote $(SRC) -iquote $(SRC_ENGINE) -iquote . -idirafter /usr/include/linux

Manifest

@@ -36,7 +36,6 @@ docker/ubuntu20.docker
 docker/ubuntu22.docker
 ext/cxxopts.hpp
 ext/xxhash.h
-ext/xxhash.patch_script
 lmake_env/Lmakefile.py
 lmake_env/ext_lnk
 src/align_comments.cc
@@ -193,6 +192,7 @@ unit_tests/misc5.py
 unit_tests/misc6.py
 unit_tests/misc7.py
 unit_tests/misc8.py
+unit_tests/misc9.py
 unit_tests/modules.py
 unit_tests/name.py
 unit_tests/numba.py

TO_DO

@@ -104,6 +104,9 @@ items :
 * implement a lshow -r to see what is running now and -B to see BOM (list of sources)
 	- implement a generic walk through deps
 	- use it for check_deps, which will prevent jobs post critical deps to be run
+* improve job isolation by using namespaces
+	- much like faketree : https://github.com/enfabrica/enkit/tree/master/faketree
+	- generalize tmp mapping
 * implement cache v2 (copy & link) :
 	- put typeid(StartInfo,EndInfo,...) as version tag to ensure no inter-version clashes
 	- 2 levels : disk level, global level

lmake_env/Lmakefile.py

@@ -92,8 +92,8 @@ class Unpack(BaseRule) :
 # it is unacceptable to have a pack inside a pack, as this creates ambiguities
 class AntiPackPack(BaseRule,AntiRule) :
 	targets = {
-		'TAR' : '{Dir}.{DirExt:(patched_)?dir}/{File}.tar.gz'
-	,	'ZIP' : '{Dir}.{DirExt               }/{File}.zip'
+		'TAR' : '{Dir}..dir/{File}.tar.gz'
+	,	'ZIP' : '{Dir}..dir/{File}.zip'
 	}
 
 class Untar(Unpack) :
@@ -134,38 +134,6 @@ def cmd() :
 					n_files += 1
 		print(f'unzip {n_files} files',file=sys.stderr)
 
-class PatchDir(BaseRule) :
-	targets = {
-		'PATCHED_FILE'     : 'ext/{Dir}.patched_dir/{File*}'
-	,	'PATCHED_MANIFEST' : 'ext/{Dir}.patched_manifest'
-	}
-	deps = {
-		'MANIFEST'     : 'ext/{Dir}.manifest'
-	,	'PATCH_SCRIPT' : 'ext/{Dir}.patch_script'
-	}
-	def cmd() :
-		sdir = f'ext/{Dir}.dir'
-		pdir = f'ext/{Dir}.patched_dir'
-		with open(PATCHED_MANIFEST,'w') as pm :
-			for f in open(MANIFEST).read().splitlines() :
-				src  = f'{sdir}/{f}'
-				dest = f'{pdir}/{f}'
-				dir_guard(dest)
-				open(dest,'w').write(open(src,'r').read())
-				print(f,file=pm)
-		run((osp.abspath(PATCH_SCRIPT),),cwd=pdir,stdin=DEVNULL,stderr=STDOUT,check=True)
-
-class PatchFile(BaseRule) :
-	targets = { 'DST' : 'ext/{DirS}{ File}.patched.{Ext}' }
-	deps = {
-		'SRC'          : 'ext/{DirS}{File}.{Ext}'
-	,	'PATCH_SCRIPT' : 'ext/{DirS}{File}.patch_script'
-	}
-	def cmd() :
-		dir = ('ext/'+DirS)[:-1]
-		open(DST,'w').write(open(SRC,'r').read())
-		run((osp.abspath(PATCH_SCRIPT),File+'.patched.'+Ext),cwd=dir,stdin=DEVNULL,stderr=STDOUT,check=True)
-
 class ConfigH(BaseRule) :
 	targets      = { 'CONFIG_H'   : 'ext/{DirS}config.h'  }
 	side_targets = { 'SCRATCHPAD' : 'ext/{DirS}{File*}'   }

src/autodep/clmake.cc

@@ -245,11 +245,12 @@ static PyObject* get_autodep( PyObject* /*null*/ , PyObject* args , PyObject* kw
 	size_t       n_args  = py_args.size()              ;
 	if (kwds    ) return py_err_set(Exception::TypeErr,"expected no keyword args") ;
 	if (n_args>0) return py_err_set(Exception::TypeErr,"expected no args"        ) ;
-	char c[2] ;
+	char c = 0/*garbage*/ ;
 	// we have a private Record with a private AutodepEnv, so we must go through the backdoor to alter the regular AutodepEnv
-	int rc [[maybe_unused]] ;                                                                     // avoid compiler warning
-	rc = ::readlink( (PrivateAdminDir+"/backdoor/autodep"s ).c_str() , c , 2 ) ;
-	return Ptr<Bool>(c[0]!='0')->to_py_boost() ;
+	int rc [[maybe_unused]] = ::readlink( (PrivateAdminDir+"/backdoor/autodep"s ).c_str() , &c , 1 ) ;
+	SWEAR( c=='0' || c=='1' , int(c) ) ;
+	SWEAR( rc==1            , rc     ) ;
+	return Ptr<Bool>(c!='0')->to_py_boost() ;
 }
 
 static PyObject* set_autodep( PyObject* /*null*/ , PyObject* args , PyObject* kwds ) {

src/autodep/ld_common.x.cc

@@ -82,20 +82,20 @@ template<class Action,int NP=1> struct AuditAction : Ctx,Action {
 	template<class T> T operator()(T res) { save_errno() ; return Action::operator()(auditer(),res) ; }
 } ;
 //                                         n paths
-using Chdir   = AuditAction<Record::Chdir        > ;
-using Chmod   = AuditAction<Record::Chmod        > ;
-using Hide    = AuditAction<Record::Hide   ,0    > ;
-using Mkdir   = AuditAction<Record::Mkdir        > ;
-using Lnk     = AuditAction<Record::Lnk    ,2    > ;
-using Open    = AuditAction<Record::Open         > ;
-using Read    = AuditAction<Record::Read         > ;
-using Readlnk = AuditAction<Record::Readlnk      > ;
-using Rename  = AuditAction<Record::Rename ,2    > ;
-using Solve   = AuditAction<Record::Solve        > ;
-using Stat    = AuditAction<Record::Stat         > ;
-using Symlnk  = AuditAction<Record::Symlnk       > ;
-using Unlnk   = AuditAction<Record::Unlnk        > ;
-using WSolve  = AuditAction<Record::WSolve       > ;
+using Chdir    = AuditAction<Record::Chdir         > ;
+using Chmod    = AuditAction<Record::Chmod         > ;
+using Hide     = AuditAction<Record::Hide    ,0    > ;
+using Mkdir    = AuditAction<Record::Mkdir         > ;
+using Lnk      = AuditAction<Record::Lnk     ,2    > ;
+using Open     = AuditAction<Record::Open          > ;
+using Read     = AuditAction<Record::Read          > ;
+using Readlink = AuditAction<Record::Readlink      > ;
+using Rename   = AuditAction<Record::Rename  ,2    > ;
+using Solve    = AuditAction<Record::Solve         > ;
+using Stat     = AuditAction<Record::Stat          > ;
+using Symlnk   = AuditAction<Record::Symlnk        > ;
+using Unlnk    = AuditAction<Record::Unlnk         > ;
+using WSolve   = AuditAction<Record::WSolve        > ;
 
 #ifdef LD_PRELOAD
 
@@ -458,14 +458,14 @@ struct Mkstemp : WSolve {
 		// once init phase is passed, we proceed normally
 		ssize_t readlink(CC* p,char* b,size_t sz) NE {
 			if (!started()) return __readlink_chk(p,b,sz,sz) ;
-			HEADER1(readlink,p,(p,b,sz)) ; Readlnk r{p ,b,sz,"readlink"} ; return r(orig(F(r),b,sz)) ;
+			HEADER1(readlink,p,(p,b,sz)) ; Readlink r{p ,b,sz,"readlink"} ; return r(orig(F(r),b,sz)) ;
 		}
 	#else
-		ssize_t readlink      (CC* p,char* b,size_t sz           ) NE { HEADER1(readlink      ,p,(p,b,sz    )) ; Readlnk r{p ,b,sz,"readlink"       } ; return r(orig(F(r),b,sz    )) ; }
-		ssize_t __readlink_chk(CC* p,char* b,size_t sz,size_t bsz) NE { HEADER1(__readlink_chk,p,(p,b,sz,bsz)) ; Readlnk r{p ,b,sz,"__readlink__chk"} ; return r(orig(F(r),b,sz,bsz)) ; }
+		ssize_t readlink      (CC* p,char* b,size_t sz           ) NE { HEADER1(readlink      ,p,(p,b,sz    )) ; Readlink r{p ,b,sz,"readlink"       } ; return r(orig(F(r),b,sz    )) ; }
+		ssize_t __readlink_chk(CC* p,char* b,size_t sz,size_t bsz) NE { HEADER1(__readlink_chk,p,(p,b,sz,bsz)) ; Readlink r{p ,b,sz,"__readlink__chk"} ; return r(orig(F(r),b,sz,bsz)) ; }
 	#endif
-	ssize_t readlinkat      (int d,CC* p,char* b,size_t sz           ) NE { HEADER1(readlinkat      ,p,(d,p,b,sz    )) ; Readlnk r{{d,p},b,sz,"readlinkat"      } ; return r(orig(P(r),b,sz    )) ; }
-	ssize_t __readlinkat_chk(int d,CC* p,char* b,size_t sz,size_t bsz) NE { HEADER1(__readlinkat_chk,p,(d,p,b,sz,bsz)) ; Readlnk r{{d,p},b,sz,"__readlinkat_chk"} ; return r(orig(P(r),b,sz,bsz)) ; }
+	ssize_t readlinkat      (int d,CC* p,char* b,size_t sz           ) NE { HEADER1(readlinkat      ,p,(d,p,b,sz    )) ; Readlink r{{d,p},b,sz,"readlinkat"      } ; return r(orig(P(r),b,sz    )) ; }
+	ssize_t __readlinkat_chk(int d,CC* p,char* b,size_t sz,size_t bsz) NE { HEADER1(__readlinkat_chk,p,(d,p,b,sz,bsz)) ; Readlink r{{d,p},b,sz,"__readlinkat_chk"} ; return r(orig(P(r),b,sz,bsz)) ; }
 
 	// rename
 	#ifdef RENAME_EXCHANGE

src/autodep/ptrace.cc

@@ -91,54 +91,54 @@ ::pair<bool/*done*/,int/*wstatus*/> AutodepPtrace::_changed( pid_t pid , int wst
 	if (WIFSTOPPED(wstatus)) {
 		int sig   = WSTOPSIG(wstatus) ;
 		int event = wstatus>>16       ;
-		if (sig==(SIGTRAP|0x80)) goto DoSyscall ;                                                    // if HAS_SECCOMP => syscall exit, else syscall enter or exit
+		if (sig==(SIGTRAP|0x80)) goto DoSyscall ;                                        // if HAS_SECCOMP => syscall exit, else syscall enter or exit
 		switch (event) {
 			#if HAS_SECCOMP
-				case PTRACE_EVENT_SECCOMP : SWEAR(!info.on_going) ; goto DoSyscall ;                 // syscall enter
+				case PTRACE_EVENT_SECCOMP : SWEAR(!info.on_going) ; goto DoSyscall ;     // syscall enter
 			#endif
-			case 0  : if (sig==SIGTRAP)         sig = 0 ; goto NextSyscall ;                         // other stop reasons, wash spurious SIGTRAP, ignore
-			default : SWEAR(sig==SIGTRAP,sig) ; sig = 0 ; goto NextSyscall ;                         // ignore other events
+			case 0  : if (sig==SIGTRAP)         sig = 0 ; goto NextSyscall ;             // other stop reasons, wash spurious SIGTRAP, ignore
+			default : SWEAR(sig==SIGTRAP,sig) ; sig = 0 ; goto NextSyscall ;             // ignore other events
 		}
 	DoSyscall :
 		{	static SyscallDescr::Tab const& tab = SyscallDescr::s_tab(true/*for_ptrace*/) ;
 			sig = 0 ;
-			#if HAS_PTRACE_GET_SYSCALL_INFO                                                          // use portable calls if implemented
+			#if HAS_PTRACE_GET_SYSCALL_INFO                                              // use portable calls if implemented
 				struct ptrace_syscall_info syscall_info ;
 				::ptrace( PTRACE_GET_SYSCALL_INFO , pid , sizeof(struct ptrace_syscall_info) , &syscall_info ) ;
 			#endif
 			if (!info.on_going) {
 				#if HAS_PTRACE_GET_SYSCALL_INFO
 					SWEAR( syscall_info.op==(HAS_SECCOMP?PTRACE_SYSCALL_INFO_SECCOMP:PTRACE_SYSCALL_INFO_ENTRY) ) ;
 					#if HAS_SECCOMP
-						auto& entry_info = syscall_info.seccomp ;                                    // access available info upon syscall entry, i.e. seccomp field when seccomp is     used
+						auto& entry_info = syscall_info.seccomp ;                        // access available info upon syscall entry, i.e. seccomp field when seccomp is     used
 					#else
-						auto& entry_info = syscall_info.entry   ;                                    // access available info upon syscall entry, i.e. entry   field when seccomp is not used
+						auto& entry_info = syscall_info.entry   ;                        // access available info upon syscall entry, i.e. entry   field when seccomp is not used
 					#endif
 					int syscall = entry_info.nr ;
 				#else
-					int syscall = np_ptrace_get_syscall(pid) ;                                       // use non-portable calls if portable accesses are not implemented
+					int syscall = np_ptrace_get_nr(pid) ;                           // use non-portable calls if portable accesses are not implemented
 				#endif
 				SWEAR( syscall>=0 && syscall<SyscallDescr::NSyscalls ) ;
 				SyscallDescr const& descr = tab[syscall] ;
 				if (HAS_SECCOMP) SWEAR(+descr,"should not be awaken for nothing") ;
 				if (+descr) {
 					info.idx = syscall ;
-					#if HAS_PTRACE_GET_SYSCALL_INFO                                                  // use portable calls if implemented
+					#if HAS_PTRACE_GET_SYSCALL_INFO                                      // use portable calls if implemented
 						// ensure entry_info is actually an array of uint64_t although one is declared as unsigned long and the other is unesigned long long
 						static_assert( sizeof(entry_info.args[0])==sizeof(uint64_t) && ::is_unsigned_v<remove_reference_t<decltype(entry_info.args[0])>> ) ;
 						uint64_t* args = reinterpret_cast<uint64_t*>(entry_info.args) ;
 					#else
-						::array<uint64_t,6> arg_array = np_ptrace_get_args(pid) ;                    // use non-portable calls if portable accesses are not implemented
-						uint64_t *          args      = arg_array.data()        ;                    // we need a variable to hold the data while we pass the pointer
+						::array<uint64_t,6> arg_array = np_ptrace_get_args(pid) ;        // use non-portable calls if portable accesses are not implemented
+						uint64_t *          args      = arg_array.data()        ;        // we need a variable to hold the data while we pass the pointer
 					#endif
-					SWEAR(!info.ctx,syscall) ;                                                       // ensure following SWEAR on info.ctx is pertinent
+					SWEAR(!info.ctx,syscall) ;                                           // ensure following SWEAR on info.ctx is pertinent
 					descr.entry( info.ctx , info.record , pid , args , descr.comment ) ;
 					info.has_exit_proc = descr.exit ;
-					if (!info.has_exit_proc) SWEAR(!info.ctx,syscall) ;                              // no need for a context if we are not called at exit
+					if (!info.has_exit_proc) SWEAR(!info.ctx,syscall) ;                  // no need for a context if we are not called at exit
 				} else {
 					info.has_exit_proc = false ;
 				}
-				info.on_going = !HAS_SECCOMP || info.has_exit_proc ;                                 // if using seccomp and we have no exit proc, we skip the syscall-exit
+				info.on_going = !HAS_SECCOMP || info.has_exit_proc ;                     // if using seccomp and we have no exit proc, we skip the syscall-exit
 				if (info.has_exit_proc) goto SyscallExit ;
 				else                    goto NextSyscall ;
 			} else {
@@ -147,14 +147,14 @@ ::pair<bool/*done*/,int/*wstatus*/> AutodepPtrace::_changed( pid_t pid , int wst
 				#endif
 				if (HAS_SECCOMP) SWEAR(info.has_exit_proc,"should not have been stopped on exit") ;
 				if (info.has_exit_proc) {
-					#if HAS_PTRACE_GET_SYSCALL_INFO                                                                         // use portable calls if implemented
+					#if HAS_PTRACE_GET_SYSCALL_INFO                                      // use portable calls if implemented
 						int64_t res = syscall_info.exit.rval ;
 					#else
-						int64_t res = np_ptrace_get_res(pid) ;                                                              // use non-portable calls if portable accesses are not implemented
+						int64_t res = np_ptrace_get_res(pid) ;                           // use non-portable calls if portable accesses are not implemented
 					#endif
 					int64_t new_res = tab[info.idx].exit( info.ctx , info.record , pid , res ) ;
-					if (new_res!=res) FAIL("modified syscall result ",new_res,"!=",res," not yet implemented for ptrace") ; // there is no such cases for now, if it arises, new_res must be reported
-					info.ctx = nullptr ;                                                                                    // ctx is used to retain some info between syscall entry and exit
+					if (new_res!=res) np_ptrace_set_res( pid , new_res ) ;
+					info.ctx = nullptr ;                                                 // ctx is used to retain some info between syscall entry and exit
 				}
 				info.on_going = false ;
 				goto NextSyscall ;
@@ -166,9 +166,9 @@ ::pair<bool/*done*/,int/*wstatus*/> AutodepPtrace::_changed( pid_t pid , int wst
 	SyscallExit :
 		::ptrace( StopAtSyscallExit      , pid , 0/*addr*/ , sig ) ;
 		goto Done ;
-	} else if ( WIFEXITED(wstatus) || WIFSIGNALED(wstatus) ) {                                                              // not sure we are informed that a process exits
+	} else if ( WIFEXITED(wstatus) || WIFSIGNALED(wstatus) ) {                           // not sure we are informed that a process exits
 		if (pid==child_pid) return {true/*done*/,wstatus} ;
-		PidInfo::s_tab.erase(pid) ;                                                                                         // free resources if possible
+		PidInfo::s_tab.erase(pid) ;                                                      // free resources if possible
 	} else {
 		fail("unrecognized wstatus ",wstatus," for pid ",pid) ;
 	}