From 051d0440277162223aed4a43478e7074826c9774 Mon Sep 17 00:00:00 2001 From: Cesar Douady Date: Mon, 15 Apr 2024 21:10:17 +0200 Subject: [PATCH] add flag optional to targets + support utimensat with tmp mapping --- doc/lmake.texi | 3 ++ src/autodep/ld_audit.cc | 6 ++- src/autodep/ld_common.x.cc | 89 ++++++++++++++++++++------------------ src/lmakeserver/rule.cc | 73 ++++++++++++++++++------------- src/rpc_job.hh | 12 ++--- unit_tests/star.py | 12 +++-- unit_tests/tmp2.py | 2 +- 7 files changed, 117 insertions(+), 80 deletions(-) diff --git a/doc/lmake.texi b/doc/lmake.texi index 9ba32148..8d719a8c 100644 --- a/doc/lmake.texi +++ b/doc/lmake.texi @@ -1827,6 +1827,9 @@ The flags may be any combination of the following flags, optionally preceded by @item Incremental @tab incremental @tab No @tab Previous content may be used to produce these targets. In that case, these are not unlinked before execution. However, if the link count is more than 1, they are uniquified, i.e. they are copied in place to ensure modification to the link does not alter other links. +@item Optional @tab optional @tab No @tab If this target is not generated, it is not deemed to be produced by the job. +@lmake will try to find an alternative rule. +This is equivalent to being a star target, except that there is no star stem. @item Phony @tab phony @tab No @tab Accept that this target is not generated, this target is deemed generated even not physically on disk. If a star target, do not search for an alternative rule to produce the file. @item SourceOk @tab source_ok @tab No @tab Do not generate an error if target is actually a source diff --git a/src/autodep/ld_audit.cc b/src/autodep/ld_audit.cc index e75d3001..e98d0553 100644 --- a/src/autodep/ld_audit.cc +++ b/src/autodep/ld_audit.cc @@ -67,7 +67,6 @@ ::umap_s const* const g_syscall_tab = new ::umap_s{ , { "execvp" , { reinterpret_cast(Audited::execvp ) } } , { "execvpe" , { reinterpret_cast(Audited::execvpe ) } } , { "fchdir" , { reinterpret_cast(Audited::fchdir ) } } -, { "fchmod" , { reinterpret_cast(Audited::fchmod ) } } , { "fchmodat" , { reinterpret_cast(Audited::fchmodat ) } } , { "fopen" , { reinterpret_cast(Audited::fopen ) } } , { "fopen64" , { reinterpret_cast(Audited::fopen64 ) } } @@ -75,12 +74,14 @@ ::umap_s const* const g_syscall_tab = new ::umap_s{ , { "__fork" , { reinterpret_cast(Audited::__fork ) } } , { "freopen" , { reinterpret_cast(Audited::freopen ) } } , { "freopen64" , { reinterpret_cast(Audited::freopen64 ) } } +, { "futimesat" , { reinterpret_cast(Audited::futimesat ) } } , { "getcwd" , { reinterpret_cast(Audited::getcwd ) } } // necessary when tmp_view is not empty to map back to job view , { "getwd" , { reinterpret_cast(Audited::getwd ) } } // . , { "get_current_dir_name", { reinterpret_cast(Audited::get_current_dir_name) } } // . , { "__libc_fork" , { reinterpret_cast(Audited::__libc_fork ) } } , { "link" , { reinterpret_cast(Audited::link ) } } , { "linkat" , { reinterpret_cast(Audited::linkat ) } } +, { "lutimes" , { reinterpret_cast(Audited::lutimes ) } } , { "mkdir" , { reinterpret_cast(Audited::mkdir ) } } // necessary against NFS strange notion of coherence as this touches containing dir , { "mkostemp" , { reinterpret_cast(Audited::mkostemp ) } } , { "mkostemp64" , { reinterpret_cast(Audited::mkostemp64 ) } } @@ -118,6 +119,9 @@ ::umap_s const* const g_syscall_tab = new ::umap_s{ , { "truncate64" , { reinterpret_cast(Audited::truncate64 ) } } , { "unlink" , { reinterpret_cast(Audited::unlink ) } } , { "unlinkat" , { reinterpret_cast(Audited::unlinkat ) } } +, { "utime" , { reinterpret_cast(Audited::utime ) } } +, { "utimensat" , { reinterpret_cast(Audited::utimensat ) } } +, { "utimes" , { reinterpret_cast(Audited::utimes ) } } , { "vfork" , { reinterpret_cast(Audited::vfork ) } } // because vfork semantic does not allow instrumentation of following exec , { "__vfork" , { reinterpret_cast(Audited::__vfork ) } } // . // diff --git a/src/autodep/ld_common.x.cc b/src/autodep/ld_common.x.cc index ca5a3bde..4ff39847 100644 --- a/src/autodep/ld_common.x.cc +++ b/src/autodep/ld_common.x.cc @@ -9,7 +9,9 @@ #include #include #include +#include #include +#include #include "disk.hh" @@ -294,16 +296,15 @@ struct Mkstemp : WSolve { // chdir // chdir must be tracked as we must tell Record of the new cwd // /!\ chdir manipulates cwd, which mandates an exclusive lock - int chdir (CC* pth) NE { HEADER0(chdir ,(pth)) ; NO_SERVER(chdir ) ; Chdir r{pth ,"chdir" } ; return r(orig(F(r))) ; } - int fchdir(int fd ) NE { HEADER0(fchdir,(fd )) ; NO_SERVER(fchdir) ; Chdir r{Fd(fd),"fchdir"} ; return r(orig(A(r))) ; } + int chdir (CC* p ) NE { HEADER0(chdir ,(p )) ; NO_SERVER(chdir ) ; Chdir r{p ,"chdir" } ; return r(orig(F(r))) ; } + int fchdir(int fd) NE { HEADER0(fchdir,(fd)) ; NO_SERVER(fchdir) ; Chdir r{Fd(fd),"fchdir"} ; return r(orig(A(r))) ; } // chmod // although file is not modified, resulting file after chmod depends on its previous content, much like a copy - // exe no_follow - int chmod ( CC* pth,mode_t mod ) NE { HEADER1(chmod ,pth,( pth,mod )) ; Chmod r{ pth ,EXE(mod),false ,"chmod" } ; return r(orig(F(r),mod )) ; } - int fchmod (int fd , mode_t mod ) NE { HEADER0(fchmod , (fd ,mod )) ; Chmod r{Fd(fd) ,EXE(mod),false ,"fchmod" } ; return r(orig(A(r),mod )) ; } - int fchmodat(int dfd,CC* pth,mode_t mod, int flgs) NE { HEADER1(fchmodat,pth,(dfd,pth,mod,flgs)) ; Chmod r{{dfd,pth},EXE(mod),ASLNF(flgs),"fchmodat"} ; return r(orig(P(r),mod,flgs)) ; } + // exe no_follow + int chmod ( CC* p,mode_t m ) NE { HEADER1(chmod ,p,( p,m )) ; Chmod r{ p ,EXE(m),false ,"chmod" } ; return r(orig(F(r),m )) ; } + int fchmodat(int d,CC* p,mode_t m,int f) NE { HEADER1(fchmodat,p,(d,p,m,f)) ; Chmod r{{d,p},EXE(m),ASLNF(f),"fchmodat"} ; return r(orig(P(r),m,f)) ; } #ifndef IN_SERVER // close @@ -313,23 +314,23 @@ struct Mkstemp : WSolve { int close (int fd ) { HEADER0(close ,(fd)) ; Hide r{fd} ; return r(orig(fd)) ; } int __close(int fd ) { HEADER0(__close,(fd)) ; Hide r{fd} ; return r(orig(fd)) ; } #if HAS_CLOSE_RANGE - int close_range(uint fd1,uint fd2,int flgs) NE { HEADER0(close_range,(fd1,fd2,flgs)) ; Hide r{fd1,fd2,flgs} ; return r(orig(fd1,fd2,flgs)) ; } + int close_range(uint fd1,uint fd2,int f) NE { HEADER0(close_range,(fd1,fd2,f)) ; Hide r{fd1,fd2,f} ; return r(orig(fd1,fd2,f)) ; } #endif #endif #ifdef LD_PRELOAD // dlopen // not necessary with ld_audit as auditing mechanism provides a reliable way of finding indirect deps - void* dlopen ( CC* pth,int fs) NE { HEADER(dlopen ,!pth||!*pth,( pth,fs)) ; Dlopen r{pth,"dlopen" } ; return r(orig( pth,fs)) ; } // we do not support tmp mapping for indirect ... - void* dlmopen(Lmid_t lm,CC* pth,int fs) NE { HEADER(dlmopen,!pth||!*pth,(lm,pth,fs)) ; Dlopen r{pth,"dlmopen"} ; return r(orig(lm,pth,fs)) ; } // ... deps, so we can pass pth to orig + void* dlopen ( CC* p,int f) NE { HEADER(dlopen ,!p||!*p,( p,f)) ; Dlopen r{p,"dlopen" } ; return r(orig( p,f)) ; } // we do not support tmp mapping for indirect ... + void* dlmopen(Lmid_t lm,CC* p,int f) NE { HEADER(dlmopen,!p||!*p,(lm,p,f)) ; Dlopen r{p,"dlmopen"} ; return r(orig(lm,p,f)) ; } // ... deps, so we can pass pth to orig #endif #ifndef IN_SERVER // dup2 // in case dup2/3 is called with one our fd's, we must hide somewhere else (unless in server) - int dup2 (int oldfd,int newfd ) NE { HEADER0(dup2 ,(oldfd,newfd )) ; Hide r{newfd} ; return r(orig(oldfd,newfd )) ; } - int dup3 (int oldfd,int newfd,int flgs) NE { HEADER0(dup3 ,(oldfd,newfd,flgs)) ; Hide r{newfd} ; return r(orig(oldfd,newfd,flgs)) ; } - int __dup2(int oldfd,int newfd ) NE { HEADER0(__dup2,(oldfd,newfd )) ; Hide r{newfd} ; return r(orig(oldfd,newfd )) ; } + int dup2 (int ofd,int nfd ) NE { HEADER0(dup2 ,(ofd,nfd )) ; Hide r{nfd} ; return r(orig(ofd,nfd )) ; } + int dup3 (int ofd,int nfd,int f) NE { HEADER0(dup3 ,(ofd,nfd,f)) ; Hide r{nfd} ; return r(orig(ofd,nfd,f)) ; } + int __dup2(int ofd,int nfd ) NE { HEADER0(__dup2,(ofd,nfd )) ; Hide r{nfd} ; return r(orig(ofd,nfd )) ; } #endif #ifdef LD_PRELOAD @@ -343,11 +344,11 @@ struct Mkstemp : WSolve { // execv // execv*p cannot be simple as we do not know which file will be accessed - // exec may not support tmp mapping if it is involved along the interpreter path no_follow - int execv (CC* pth,char* const argv[] ) NE { HEADER0(execv ,(pth,argv )) ; NO_SERVER(execv ) ; Exec r{pth,false ,environ,"execv" } ; return r(orig(F(r),argv )) ; } - int execve (CC* pth,char* const argv[],char* const envp[]) NE { HEADER0(execve ,(pth,argv,envp)) ; NO_SERVER(execve ) ; Exec r{pth,false ,envp ,"execve" } ; return r(orig(F(r),argv,envp)) ; } - int execvp (CC* pth,char* const argv[] ) NE { HEADER0(execvp ,(pth,argv )) ; NO_SERVER(execvp ) ; Execp r{pth, environ,"execvp" } ; return r(orig(F(r),argv )) ; } - int execvpe(CC* pth,char* const argv[],char* const envp[]) NE { HEADER0(execvpe,(pth,argv,envp)) ; NO_SERVER(execvpe) ; Execp r{pth, envp ,"execvpe"} ; return r(orig(F(r),argv,envp)) ; } + // exec may not support tmp mapping if it is involved along the interpreter path no_follow + int execv (CC* p,char* const argv[] ) NE { HEADER0(execv ,(p,argv )) ; NO_SERVER(execv ) ; Exec r{p,false ,environ,"execv" } ; return r(orig(F(r),argv )) ; } + int execve (CC* p,char* const argv[],char* const envp[]) NE { HEADER0(execve ,(p,argv,envp)) ; NO_SERVER(execve ) ; Exec r{p,false ,envp ,"execve" } ; return r(orig(F(r),argv,envp)) ; } + int execvp (CC* p,char* const argv[] ) NE { HEADER0(execvp ,(p,argv )) ; NO_SERVER(execvp ) ; Execp r{p, environ,"execvp" } ; return r(orig(F(r),argv )) ; } + int execvpe(CC* p,char* const argv[],char* const envp[]) NE { HEADER0(execvpe,(p,argv,envp)) ; NO_SERVER(execvpe) ; Execp r{p, envp ,"execvpe"} ; return r(orig(F(r),argv,envp)) ; } // int execveat( int dfd , CC* pth , char* const argv[] , char *const envp[] , int flgs ) NE { HEADER1(execveat,pth,(dfd,pth,argv,envp,flgs)) ; @@ -371,17 +372,16 @@ struct Mkstemp : WSolve { int rc = value ; \ delete[] args ; \ return rc - int execl (CC* pth,CC* arg,...) NE { MK_ARGS( , execv (pth,args ) ) ; } - int execle(CC* pth,CC* arg,...) NE { MK_ARGS( char* const* envp = va_arg(args_lst,char**) , execve(pth,args,envp) ) ; } - int execlp(CC* pth,CC* arg,...) NE { MK_ARGS( , execvp(pth,args ) ) ; } + int execl (CC* p,CC* arg,...) NE { MK_ARGS( , execv (p,args ) ) ; } + int execle(CC* p,CC* arg,...) NE { MK_ARGS( char* const* envp = va_arg(args_lst,char**) , execve(p,args,envp) ) ; } + int execlp(CC* p,CC* arg,...) NE { MK_ARGS( , execvp(p,args ) ) ; } #undef MK_ARGS // fopen - FILE* fopen (CC* pth,CC* mod ) { HEADER1(fopen ,pth,(pth,mod )) ; Fopen r{pth ,mod,"fopen" } ; return r(orig(F(r),mod )) ; } - FILE* fopen64 (CC* pth,CC* mod ) { HEADER1(fopen64 ,pth,(pth,mod )) ; Fopen r{pth ,mod,"fopen64" } ; return r(orig(F(r),mod )) ; } - FILE* freopen (CC* pth,CC* mod,FILE* fp) { HEADER1(freopen ,pth,(pth,mod,fp)) ; Fopen r{pth ,mod,"freopen" } ; return r(orig(F(r),mod,fp)) ; } - FILE* freopen64(CC* pth,CC* mod,FILE* fp) { HEADER1(freopen64,pth,(pth,mod,fp)) ; Fopen r{pth ,mod,"freopen64"} ; return r(orig(F(r),mod,fp)) ; } - FILE* fdopen (int fd ,CC* mod ) { HEADER0(fdopen , (fd ,mod )) ; Fopen r{Fd(fd),mod,"fdopen" } ; return r(orig(A(r),mod )) ; } + FILE* fopen (CC* p,CC* m ) { HEADER1(fopen ,p,(p,m )) ; Fopen r{p,m,"fopen" } ; return r(orig(F(r),m )) ; } + FILE* fopen64 (CC* p,CC* m ) { HEADER1(fopen64 ,p,(p,m )) ; Fopen r{p,m,"fopen64" } ; return r(orig(F(r),m )) ; } + FILE* freopen (CC* p,CC* m,FILE* fp) { HEADER1(freopen ,p,(p,m,fp)) ; Fopen r{p,m,"freopen" } ; return r(orig(F(r),m,fp)) ; } + FILE* freopen64(CC* p,CC* m,FILE* fp) { HEADER1(freopen64,p,(p,m,fp)) ; Fopen r{p,m,"freopen64"} ; return r(orig(F(r),m,fp)) ; } // fork // not recursively called by auditing code @@ -402,8 +402,8 @@ struct Mkstemp : WSolve { // cf man 3 getcwd (Linux) // needed for tmp mapping (not available in server) // nothing to hide, but calling Hide guarantees all invariants (in particular errno management and auditer initialization) - char* getcwd (char* buf,size_t sz) NE { HEADER0(getcwd ,(buf,sz)) ; Getcwd r{sz ,buf?No:sz?Maybe:Yes} ; return r(orig(buf,sz)) ; } - char* get_current_dir_name( ) NE { HEADER0(get_current_dir_name,( )) ; Getcwd r{PATH_MAX,Yes } ; return r(orig( )) ; } + char* getcwd (char* b,size_t sz) NE { HEADER0(getcwd ,(b,sz)) ; Getcwd r{sz ,b?No:sz?Maybe:Yes} ; return r(orig(b,sz)) ; } + char* get_current_dir_name( ) NE { HEADER0(get_current_dir_name,( )) ; Getcwd r{PATH_MAX,Yes } ; return r(orig( )) ; } #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" char* getwd (char* buf ) NE { HEADER0(getwd ,(buf )) ; Getcwd r{PATH_MAX,No } ; return r(orig(buf )) ; } @@ -419,14 +419,14 @@ struct Mkstemp : WSolve { int mkdirat(int d,CC* p,mode_t m) NE { HEADER1(mkdirat,p,(d,p,m)) ; Mkdir r{{d,p},"mkdir" } ; return r(orig(P(r),m)) ; } // mkstemp - int mkstemp (char* tmpl ) { HEADER0(mkstemp ,(tmpl )) ; Mkstemp r{tmpl, "mkstemp" } ; return r(orig(F(r) )) ; } - int mkostemp (char* tmpl,int flgs ) { HEADER0(mkostemp ,(tmpl,flgs )) ; Mkstemp r{tmpl, "mkostemp" } ; return r(orig(F(r),flgs )) ; } - int mkstemps (char* tmpl, int sfx_len) { HEADER0(mkstemps ,(tmpl, sfx_len)) ; Mkstemp r{tmpl,sfx_len,"mkstemps" } ; return r(orig(F(r), sfx_len)) ; } - int mkostemps (char* tmpl,int flgs,int sfx_len) { HEADER0(mkostemps ,(tmpl,flgs,sfx_len)) ; Mkstemp r{tmpl,sfx_len,"mkostemps" } ; return r(orig(F(r),flgs,sfx_len)) ; } - int mkstemp64 (char* tmpl ) { HEADER0(mkstemp64 ,(tmpl )) ; Mkstemp r{tmpl, "mkstemp64" } ; return r(orig(F(r) )) ; } - int mkostemp64 (char* tmpl,int flgs ) { HEADER0(mkostemp64 ,(tmpl,flgs )) ; Mkstemp r{tmpl, "mkostemp64" } ; return r(orig(F(r),flgs )) ; } - int mkstemps64 (char* tmpl, int sfx_len) { HEADER0(mkstemps64 ,(tmpl, sfx_len)) ; Mkstemp r{tmpl,sfx_len,"mkstemps64" } ; return r(orig(F(r), sfx_len)) ; } - int mkostemps64(char* tmpl,int flgs,int sfx_len) { HEADER0(mkostemps64,(tmpl,flgs,sfx_len)) ; Mkstemp r{tmpl,sfx_len,"mkostemps64"} ; return r(orig(F(r),flgs,sfx_len)) ; } + int mkstemp (char* t ) { HEADER0(mkstemp ,(t )) ; Mkstemp r{t, "mkstemp" } ; return r(orig(F(r) )) ; } + int mkostemp (char* t,int f ) { HEADER0(mkostemp ,(t,f )) ; Mkstemp r{t, "mkostemp" } ; return r(orig(F(r),f )) ; } + int mkstemps (char* t, int sl) { HEADER0(mkstemps ,(t, sl)) ; Mkstemp r{t,sl,"mkstemps" } ; return r(orig(F(r), sl)) ; } + int mkostemps (char* t,int f,int sl) { HEADER0(mkostemps ,(t,f,sl)) ; Mkstemp r{t,sl,"mkostemps" } ; return r(orig(F(r),f,sl)) ; } + int mkstemp64 (char* t ) { HEADER0(mkstemp64 ,(t )) ; Mkstemp r{t, "mkstemp64" } ; return r(orig(F(r) )) ; } + int mkostemp64 (char* t,int f ) { HEADER0(mkostemp64 ,(t,f )) ; Mkstemp r{t, "mkostemp64" } ; return r(orig(F(r),f )) ; } + int mkstemps64 (char* t, int sl) { HEADER0(mkstemps64 ,(t, sl)) ; Mkstemp r{t,sl,"mkstemps64" } ; return r(orig(F(r), sl)) ; } + int mkostemps64(char* t,int f,int sl) { HEADER0(mkostemps64,(t,f,sl)) ; Mkstemp r{t,sl,"mkostemps64"} ; return r(orig(F(r),f,sl)) ; } // open #define MOD mode_t m = 0 ; if ( f & (O_CREAT|O_TMPFILE) ) { va_list lst ; va_start(lst,f) ; m = va_arg(lst,mode_t) ; va_end(lst) ; } @@ -491,16 +491,23 @@ struct Mkstemp : WSolve { int rmdir(CC* p) NE { HEADER1(rmdir,p,(p)) ; Unlnk r{p,true/*rmdir*/,"rmdir"} ; return r(orig(F(r))) ; } // symlink - int symlink (CC* target, CC* pth) NE { HEADER1(symlink ,pth,(target, pth)) ; Symlnk r{ pth ,"symlink" } ; return r(orig(target,F(r))) ; } - int symlinkat(CC* target,int dfd,CC* pth) NE { HEADER1(symlinkat,pth,(target,dfd,pth)) ; Symlnk r{{dfd,pth},"symlinkat"} ; return r(orig(target,P(r))) ; } + int symlink (CC* t, CC* p) NE { HEADER1(symlink ,p,(t, p)) ; Symlnk r{ p ,"symlink" } ; return r(orig(t,F(r))) ; } + int symlinkat(CC* t,int d,CC* p) NE { HEADER1(symlinkat,p,(t,d,p)) ; Symlnk r{{d,p},"symlinkat"} ; return r(orig(t,P(r))) ; } // truncate - int truncate (CC* pth,off_t len) NE { HEADER1(truncate ,pth,(pth,len)) ; Open r{pth,len?O_RDWR:O_WRONLY,"truncate" } ; return r(orig(F(r),len)) ; } - int truncate64(CC* pth,off_t len) NE { HEADER1(truncate64,pth,(pth,len)) ; Open r{pth,len?O_RDWR:O_WRONLY,"truncate64"} ; return r(orig(F(r),len)) ; } + int truncate (CC* p,off_t l) NE { HEADER1(truncate ,p,(p,l)) ; Open r{p,l?O_RDWR:O_WRONLY,"truncate" } ; return r(orig(F(r),l)) ; } + int truncate64(CC* p,off_t l) NE { HEADER1(truncate64,p,(p,l)) ; Open r{p,l?O_RDWR:O_WRONLY,"truncate64"} ; return r(orig(F(r),l)) ; } // unlink - int unlink ( CC* pth ) NE { HEADER1(unlink ,pth,( pth )) ; Unlnk r{ pth ,false/*rmdir*/ ,"unlink" } ; return r(orig(F(r) )) ; } - int unlinkat(int dfd,CC* pth,int flgs) NE { HEADER1(unlinkat,pth,(dfd,pth,flgs)) ; Unlnk r{{dfd,pth},bool(flgs&AT_REMOVEDIR),"unlinkat"} ; return r(orig(P(r),flgs)) ; } + int unlink ( CC* p ) NE { HEADER1(unlink ,p,( p )) ; Unlnk r{ p ,false/*rmdir*/ ,"unlink" } ; return r(orig(F(r) )) ; } + int unlinkat(int d,CC* p,int f) NE { HEADER1(unlinkat,p,(d,p,f)) ; Unlnk r{{d,p},bool(f&AT_REMOVEDIR),"unlinkat"} ; return r(orig(P(r),f)) ; } + + // utime no_follow read allow_tmp_map + int utime ( CC* p,const struct utimbuf* t ) { HEADER1(utime ,p,( p,t )) ; Solve r{ p ,false ,false,true ,"utime" } ; return r(orig(F(r),t )) ; } + int utimes ( CC* p,const struct timeval t[2] ) { HEADER1(utimes ,p,( p,t )) ; Solve r{ p ,false ,false,true ,"utimes" } ; return r(orig(F(r),t )) ; } + int futimesat(int d,CC* p,const struct timeval t[2] ) { HEADER1(futimesat,p,(d,p,t )) ; Solve r{{d,p},false ,false,true ,"futimesat"} ; return r(orig(P(r),t )) ; } + int lutimes ( CC* p,const struct timeval t[2] ) { HEADER1(lutimes ,p,( p,t )) ; Solve r{ p ,true ,false,true ,"lutimes" } ; return r(orig(F(r),t )) ; } + int utimensat(int d,CC* p,const struct timespec t[2],int f) { HEADER1(utimensat,p,(d,p,t,f)) ; Solve r{{d,p},ASLNF(f),false,true ,"utimensat"} ; return r(orig(P(r),t,f)) ; } // mere path accesses (neeed to solve path, but no actual access to file data) // no_follow read allow_tmp_map diff --git a/src/lmakeserver/rule.cc b/src/lmakeserver/rule.cc index 9490f8bf..5c5aeddd 100644 --- a/src/lmakeserver/rule.cc +++ b/src/lmakeserver/rule.cc @@ -271,10 +271,10 @@ namespace Engine { case VarCmd::Dep : cb_str(vc,i,deps() [i].first,deps ()[i].second) ; break ; case VarCmd::Rsrc : { auto it = rsrcs().find(rsrcs_spec[i].first) ; if (it!=rsrcs().end()) cb_str(vc,i,it->first ,it->second ) ; } break ; // - case VarCmd::Stems : for( VarIdx j=0 ; jn_static_stems ; j++ ) dct.emplace_back(r->stems [j].first,stems ()[j]) ; cb_dct(vc,i,"stems" ,dct ) ; break ; - case VarCmd::Targets : for( VarIdx j=0 ; jn_static_targets; j++ ) dct.emplace_back(r->matches[j].first,matches()[j]) ; cb_dct(vc,i,"targets" ,dct ) ; break ; - case VarCmd::Deps : for( auto const& [k,d] : deps() ) dct.emplace_back(k ,d ) ; cb_dct(vc,i,"deps" ,dct ) ; break ; - case VarCmd::Rsrcs : cb_dct(vc,i,"resources",rsrcs_) ; break ; + case VarCmd::Stems : for( VarIdx j=0 ; jn_static_stems ; j++ ) dct.emplace_back(r->stems [j].first,stems ()[j]) ; cb_dct(vc,i,"stems" ,dct ) ; break ; + case VarCmd::Targets : for( VarIdx j=0 ; jn_static_targets ; j++ ) dct.emplace_back(r->matches[j].first,matches()[j]) ; cb_dct(vc,i,"targets" ,dct ) ; break ; + case VarCmd::Deps : for( auto const& [k,d] : deps() ) dct.emplace_back(k ,d ) ; cb_dct(vc,i,"deps" ,dct ) ; break ; + case VarCmd::Rsrcs : cb_dct(vc,i,"resources",rsrcs_) ; break ; DF} } } @@ -424,7 +424,7 @@ namespace Engine { } deps.emplace_back( key , DepSpec{ ::move(parsed_dep) , df , edf } ) ; } - if (deps.size()>=Rule::NoVar-1) throw to_string("too many static deps : ",deps.size()) ; // -1 to leave some room to the interpreter, if any + if (deps.size()>=Rule::NoVar-1) throw to_string("too many static deps : ",deps.size()) ; // -1 to leave some room to the interpreter, if any } void DepsAttrs::add_interpreter(RuleData const& rd) { @@ -695,7 +695,7 @@ namespace Engine { stem_defs.emplace( ::string(py_k.as_a()) , ::string(py_v.as_a()) ) ; // // augment stems with definitions found in job_name and targets - size_t unnamed_star_idx = 1 ; // free running while walking over job_name + targets + size_t unnamed_star_idx = 1 ; // free running while walking over job_name + targets auto augment_stems = [&]( ::string const& k , bool star , ::string const* re , bool star_only ) -> void { if (re) { auto [it,inserted] = stem_defs.emplace(k,*re) ; @@ -703,7 +703,7 @@ namespace Engine { } if ( !star_only || star ) { auto [it,inserted] = stem_stars.emplace(k,No|star) ; - if ( !inserted && (No|star)!=it->second ) it->second = Maybe ; // stem is used both as static and star + if ( !inserted && (No|star)!=it->second ) it->second = Maybe ; // stem is used both as static and star } } ; field = "job_name" ; @@ -718,8 +718,8 @@ namespace Engine { ::string job_name_msg = "job_name" ; for( auto const& [py_k,py_tkfs] : dct[field].as_a() ) { field = py_k.as_a() ; - ::string target = py_tkfs.as_a()[0].as_a() ; // . - MatchKind kind = mk_enum(py_tkfs.as_a()[1].as_a()) ; // targets are a tuple (target_pattern,kind,flags...) + ::string target = py_tkfs.as_a()[0].as_a() ; // . + MatchKind kind = mk_enum(py_tkfs.as_a()[1].as_a()) ; // targets are a tuple (target_pattern,kind,flags...) // avoid processing target if it is identical to job_name : this is not an optimization, it is to ensure unnamed_star_idx's match if (target!=job_name) { _parse_py( target , &unnamed_star_idx , @@ -736,7 +736,7 @@ namespace Engine { // // gather job_name and targets field = "job_name" ; - unnamed_star_idx = 1 ; // reset free running at each pass over job_name+targets + unnamed_star_idx = 1 ; // reset free running at each pass over job_name+targets VarIdx n_static_unnamed_stems = 0 ; bool job_name_is_star = false ; auto stem_words = []( ::string const& k , bool star , bool unnamed ) -> ::string { @@ -752,25 +752,25 @@ namespace Engine { ) ; // field = "matches" ; - { ::vmap_s star_matches ; // defer star matches so that static targets are put first - ::vmap_s static_matches[N] ; // defer star matches so that static targets are put first + { ::vmap_s star_matches ; // defer star matches so that static targets are put first + ::vmap_s static_matches[N] ; // defer star matches so that static targets are put first bool seen_top = false ; bool seen_target = false ; - for( auto const& [py_k,py_tkfs] : dct[field].as_a() ) { // targets are a tuple (target_pattern,flags...) + for( auto const& [py_k,py_tkfs] : dct[field].as_a() ) { // targets are a tuple (target_pattern,flags...) field = py_k.as_a() ; Sequence const& pyseq_tkfs = py_tkfs.as_a() ; - ::string target = pyseq_tkfs[0].as_a() ; // . - MatchKind kind = mk_enum(pyseq_tkfs[1].as_a()) ; // targets are a tuple (target_pattern,kind,flags...) + ::string target = pyseq_tkfs[0].as_a() ; // . + MatchKind kind = mk_enum(pyseq_tkfs[1].as_a()) ; // targets are a tuple (target_pattern,kind,flags...) bool is_star = false ; ::set_s missing_stems ; bool is_target = kind!=MatchKind::SideDeps ; bool is_official_target = kind==MatchKind::Target ; bool is_stdout = field=="" ; + MatchFlags flags ; Tflags tflags ; Dflags dflags ; ExtraTflags extra_tflags ; ExtraDflags extra_dflags ; - MatchFlags flags ; // // avoid processing target if it is identical to job_name : this is not an optimization, it is to ensure unnamed_star_idx's match if (target==job_name) { @@ -793,20 +793,25 @@ namespace Engine { } ) ; } - if ( !is_star ) tflags |= Tflag::Static ; - if ( is_official_target ) tflags |= Tflag::Target ; - if ( !is_star && is_official_target ) tflags |= Tflag::Essential ; // static targets are essential by default - if ( is_target ) { _split_flags( snake_str(kind) , pyseq_tkfs , 2/*n_skip*/ , tflags , extra_tflags ) ; flags = {tflags,extra_tflags} ; } - else { _split_flags( snake_str(kind) , pyseq_tkfs , 2/*n_skip*/ , dflags , extra_dflags ) ; flags = {dflags,extra_dflags} ; } + bool is_native_star = is_star ; + if ( is_official_target ) tflags |= Tflag::Target ; + if ( !is_star && is_official_target ) tflags |= Tflag::Essential ; // static targets are essential by default + if ( is_target ) _split_flags( snake_str(kind) , pyseq_tkfs , 2/*n_skip*/ , tflags , extra_tflags ) ; + else _split_flags( snake_str(kind) , pyseq_tkfs , 2/*n_skip*/ , dflags , extra_dflags ) ; + if ( extra_tflags[ExtraTflag::Optional] ) is_star = true ; + if ( !is_star ) tflags |= Tflag::Static ; + if (is_target ) flags = {tflags,extra_tflags} ; + else flags = {dflags,extra_dflags} ; // check - if ( target.starts_with(root_dir_s) ) throw to_string(snake(kind)," must be relative to root dir : " ,target) ; - if ( !is_lcl(target) ) throw to_string(snake(kind)," must be local : " ,target) ; - if ( !is_canon(target) ) throw to_string(snake(kind)," must be canonical : " ,target) ; - if ( +missing_stems ) throw to_string("missing stems ",missing_stems," in ",snake(kind)," : ",target) ; - if ( !is_official_target && is_special() ) throw "flags are meaningless for source and anti-rules"s ; - if ( is_star && is_special() ) throw to_string("star ",kind,"s are meaningless for source and anti-rules") ; - if ( is_star && is_stdout ) throw "stdout cannot be directed to a star target"s ; - if ( tflags[Tflag::Incremental] && is_stdout ) throw "stdout cannot be directed to an incremental target"s ; + if ( target.starts_with(root_dir_s) ) throw to_string(snake(kind)," must be relative to root dir : " ,target) ; + if ( !is_lcl(target) ) throw to_string(snake(kind)," must be local : " ,target) ; + if ( !is_canon(target) ) throw to_string(snake(kind)," must be canonical : " ,target) ; + if ( +missing_stems ) throw to_string("missing stems ",missing_stems," in ",snake(kind)," : ",target) ; + if ( !is_official_target && is_special() ) throw "flags are meaningless for source and anti-rules"s ; + if ( is_star && is_special() ) throw to_string("star ",kind,"s are meaningless for source and anti-rules") ; + if ( is_star && is_stdout ) throw "stdout cannot be directed to a star target"s ; + if ( tflags [Tflag ::Incremental] && is_stdout ) throw "stdout cannot be directed to an incremental target"s ; + if ( extra_tflags[ExtraTflag::Optional ] && is_native_star ) throw "star targets are natively optional" ; bool is_top = is_target ? extra_tflags[ExtraTflag::Top] : extra_dflags[ExtraDflag::Top] ; seen_top |= is_top ; seen_target |= is_official_target ; @@ -1046,12 +1051,22 @@ namespace Engine { flags << (first?" : ":" , ") << snake(df) ; first = false ; } + for( ExtraDflag edf : ExtraDflag::NRule ) { + if (!me.flags.extra_dflags()[edf]) continue ; + flags << (first?" : ":" , ") << snake(edf) ; + first = false ; + } } else { for( Tflag tf : Tflag::NRule ) { if (!me.flags.tflags()[tf]) continue ; flags << (first?" : ":" , ") << snake(tf) ; first = false ; } + for( ExtraTflag etf : ExtraTflag::NRule ) { + if (!me.flags.extra_tflags()[etf]) continue ; + flags << (first?" : ":" , ") << snake(etf) ; + first = false ; + } if (me.flags.tflags()[Tflag::Target]) { bool first_conflict = true ; for( VarIdx c : me.conflicts ) { diff --git a/src/rpc_job.hh b/src/rpc_job.hh index 0f6918ca..505ac511 100644 --- a/src/rpc_job.hh +++ b/src/rpc_job.hh @@ -118,6 +118,7 @@ ENUM_1( ExtraTflag , NRule = Allow // number of Tflag's allowed in rule definition , Top , Ignore +, Optional , SourceOk // ok to overwrite source files , Allow // writing to this target is allowed (for use in clmake.target and ltarget) , Wash // target was unlinked when washing before job execution @@ -126,6 +127,7 @@ ENUM_1( ExtraTflag static constexpr char ExtraTflagChars[] = { 0 // Top , 'I' // Ignore +, 0 // Optional , 's' // SourceOk , 'a' // Allow , 0 // Wash @@ -419,9 +421,9 @@ struct DepInfo { DepInfoKind kind = Kind::Crc ; private : union { - Crc _crc ; // ~46< 64 bits - FileSig _sig ; // 64 bits - FileInfo _info ; // 128 bits + Crc _crc ; // ~46< 64 bits + FileSig _sig ; // 64 bits + FileInfo _info ; // 128 bits } ; // END_OF_VERSIONING } ; @@ -611,8 +613,8 @@ struct JobRpcReq { JobDigest digest ; // if proc== End ::vmap_ss dynamic_env ; // if proc== End, env variables computed in job_exec ::string msg ; - // END_OF_VERSIONING -} ; + // END_OF_VERSIONING) +} ; struct MatchFlags { friend ::ostream& operator<<( ::ostream& , MatchFlags const& ) ; diff --git a/unit_tests/star.py b/unit_tests/star.py index 93e549fc..80a0e596 100644 --- a/unit_tests/star.py +++ b/unit_tests/star.py @@ -15,6 +15,10 @@ , 'hello' ) + class Opt(Rule) : + targets = { 'DST' : ('{File:.*}.opt','Optional') } + cmd = '[ {File} != ok ] || echo > {DST}' + class Star(Rule) : targets = { 'DST' : ('{File:.*}.star{D*:\\d+}',) } dep = '{File}' @@ -34,7 +38,9 @@ class Cpy(Rule) : print( 'hello' , file=open('hello','w') ) - ut.lmake( 'hello.star1.cpy' , done=2 , new=1 ) - ut.lmake( 'hello.star2' , done=0 , new=0 ) - ut.lmake( 'hello.star3' , rc=1 , done=0 , new=0 ) + ut.lmake( 'hello.star1.cpy' , done=2 , new=1 ) + ut.lmake( 'hello.star2' , done=0 , new=0 ) + ut.lmake( 'hello.star3' , rc=1 , done=0 , new=0 ) + ut.lmake( 'ok.opt' , done=1 , new=0 ) + ut.lmake( 'ko.opt' , rc=1 , steady=1 , new=0 ) # no rule to make target diff --git a/unit_tests/tmp2.py b/unit_tests/tmp2.py index dd172c77..5a25ef02 100644 --- a/unit_tests/tmp2.py +++ b/unit_tests/tmp2.py @@ -81,7 +81,7 @@ class Touch(TmpRule) : sleep 0.1 # ensure a and b have different dates cp b c ls -l --full-time c > date1 - touch -r b c + touch -r /tmp/d/b /tmp/d/c ls -l --full-time c > date2 cmp date1 date2 >/dev/null && {{ echo 'touch did not change date' >&2 ; exit 1 ; }} cd ..