Skip to content

Commit

Permalink
Merge branch 'cesar-douady:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
@nmoriniere-gpfw
nmoriniere-gpfw authored May 14, 2024
2 parents d2c8fe5 + 1426ecb commit a632cc9
Show file tree
Hide file tree
Showing 19 changed files with 194 additions and 87 deletions.
2 changes: 2 additions & 0 deletions Manifest
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,8 @@ unit_tests/manual_target.py
unit_tests/misc1.py
unit_tests/misc10.py
unit_tests/misc11.py
unit_tests/misc13.py
unit_tests/misc14.py
unit_tests/misc2.py
unit_tests/misc3.py
unit_tests/misc4.py
Expand Down
4 changes: 1 addition & 3 deletions _lib/lmake_dbg.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,7 @@ def run_vscode(dbg_dir,redirected,func,*args,**kwds) :
if os.path.exists(workspace) :
data = json.load(open(workspace))
for elmt in data['launch']['configurations'] :
if elmt.get('type') == 'by-gdb' :
if 'processId' in elmt : elmt['processId'] = os.getpid()
if 'program' in elmt : elmt['program' ] = open('/proc/self/cmdline').read().split('\x00')[:-1]
if elmt.get('type')=='by-gdb' and 'processId' in elmt : elmt['processId'] = os.getpid()
with open(workspace,'w') as out :
json.dump(data,out,indent='\t')
out.write('\n')
Expand Down
1 change: 1 addition & 0 deletions src/autodep/clmake.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,7 @@ static PyObject* get_autodep( PyObject* /*null*/ , PyObject* args , PyObject* kw
char c = 0/*garbage*/ ;
// we have a private Record with a private AutodepEnv, so we must go through the backdoor to alter the regular AutodepEnv
int rc [[maybe_unused]] = ::readlinkat( Record::s_root_fd() , (PrivateAdminDir+"/backdoor/autodep"s ).c_str() , &c , 1 ) ;
if (rc<0) return None.to_py_boost() ;
SWEAR( c=='0' || c=='1' , int(c) ) ;
SWEAR( rc==1 , rc ) ;
return Ptr<Bool>(c!='0')->to_py_boost() ;
Expand Down
6 changes: 4 additions & 2 deletions src/autodep/gather.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,10 +202,11 @@ void Gather::_send_to_server ( Fd fd , Jerr&& jerr ) {
case Proc::ChkDeps : reorder(false/*at_end*/) ; break ; // ensure server sees a coherent view
case Proc::DepVerbose : _new_accesses(fd,::copy(jerr)) ; break ;
//
case Proc::Decode : SWEAR(jerr.files.size()==1) ; _codec_files[fd] = Codec::mk_decode_node( jerr.files[0].first , jerr.ctx , jerr.txt ) ; break ;
case Proc::Encode : SWEAR(jerr.files.size()==1) ; _codec_files[fd] = Codec::mk_encode_node( jerr.files[0].first , jerr.ctx , jerr.txt ) ; break ;
case Proc::Decode : SWEAR( jerr.sync && jerr.files.size()==1 ) ; _codec_files[fd] = Codec::mk_decode_node( jerr.files[0].first , jerr.ctx , jerr.txt ) ; break ;
case Proc::Encode : SWEAR( jerr.sync && jerr.files.size()==1 ) ; _codec_files[fd] = Codec::mk_encode_node( jerr.files[0].first , jerr.ctx , jerr.txt ) ; break ;
default : ;
}
if (!jerr.sync) fd = {} ; // dont reply if not sync
try {
JobMngtRpcReq jmrr ;
if (jerr.proc==JobExecProc::ChkDeps) jmrr = { JobMngtProc::ChkDeps , seq_id , job , fd , cur_deps_cb() } ;
Expand Down Expand Up @@ -451,6 +452,7 @@ Status Gather::exec_child( ::vector_s const& args , Fd cstdin , Fd cstdout , Fd
case JobMngtProc::ChkDeps : if (jmrr.ok==Maybe) { set_status(Status::ChkDeps) ; kill_step = KillStep::Kill ; rfd = {} ; } break ;
case JobMngtProc::Decode :
case JobMngtProc::Encode : {
SWEAR(+jmrr.fd) ;
auto it = _codec_files.find(jmrr.fd) ;
_new_access( rfd , PD(New) , ::move(it->second) , {.accesses=Access::Reg} , jmrr.crc , false/*parallel*/ , ::string(snake(jmrr.proc)) ) ;
_codec_files.erase(it) ;
Expand Down
46 changes: 22 additions & 24 deletions src/autodep/record.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ using namespace Time ;
//

// if strict, user might look at the st_size field which gives access to regular and link content
static constexpr Accesses UserStatAccess = StrictUserAccesses ? ~Accesses() : Accesses(Access::Stat) ;
static constexpr Accesses UserStatAccesses = StrictUserAccesses ? ~Accesses() : Accesses(Access::Stat) ;

bool Record::s_static_report = false ;
::vmap_s<DepDigest> * Record::s_deps = nullptr ;
Expand Down Expand Up @@ -177,30 +177,28 @@ Record::Mkdir::Mkdir( Record& r , Path&& path , ::string&& c ) : Solve{r,::move(
// However :
// - if it is an official target, it is not a dep, whether you declare reading it or not
// - else, we do not compute a CRC on it and its actual content is not guaranteed. What is important in this case is that the execution of the job does not see the content.
static bool _do_stat (int flags) { return flags&O_PATH ; }
static bool _do_read (int flags) { return !_do_stat(flags) && (flags&O_ACCMODE)!=O_WRONLY && !(flags&O_TRUNC) ; }
static bool _do_write(int flags) { return !_do_stat(flags) && (flags&O_ACCMODE)!=O_RDONLY ; }
Record::Open::Open( Record& r , Path&& path , int flags , ::string&& c ) :
Solve{ r , ::move(path) , bool(flags&O_NOFOLLOW) , _do_read(flags) , true/*allow_tmp_map*/ , to_string(c,::hex,'.',flags) }
, do_write{_do_write(flags)}
{
bool do_read = _do_read(flags) ;
bool do_stat = _do_stat(flags) ;
//
static bool _no_follow(int flags) { return (flags&O_NOFOLLOW) || ( (flags&O_CREAT) && (flags&O_EXCL) ) ; }
static bool _do_stat (int flags) { return flags&O_PATH || ( (flags&O_CREAT) && (flags&O_EXCL) ) ; }
static bool _do_read (int flags) { return !(flags&O_PATH) && (flags&O_ACCMODE)!=O_WRONLY && !(flags&O_TRUNC) ; }
static bool _do_write (int flags) { return !(flags&O_PATH) && (flags&O_ACCMODE)!=O_RDONLY ; }
//
Record::Open::Open( Record& r , Path&& path , int flags , ::string&& c ) : Solve{ r , ::move(path) , _no_follow(flags) , _do_read(flags) , true/*allow_tmp_map*/ , to_string(c,::hex,'.',flags) } {
if ( flags&(O_DIRECTORY|O_TMPFILE) ) return ; // we already solved, this is enough
if ( !(flags&O_PATH) && s_autodep_env().ignore_stat ) return ;
if ( file_loc>FileLoc::Dep ) return ;
//
if ( flags&(O_DIRECTORY|O_TMPFILE) ) { file_loc = FileLoc::Ext ; return ; } // we already solved, this is enough
if ( do_stat && s_autodep_env().ignore_stat ) { file_loc = FileLoc::Ext ; return ; }
if ( file_loc>FileLoc::Dep ) return ;
bool do_stat = _do_stat (flags) ;
bool do_read = _do_read (flags) ;
bool do_write = _do_write(flags) && file_loc==FileLoc::Repo ;
//
if (!do_write) {
if (do_read) r._report_dep ( ::move(real) , accesses|Access::Reg|UserStatAccess , c+".rd" ) ; // user might do fstat on returned fd and if strict user might look at st_size field
else if (do_stat) r._report_dep ( ::move(real) , accesses |UserStatAccess , c+".path" ) ; // .
} else if (file_loc==FileLoc::Repo) {
if (do_read) r._report_update( ::move(real) , accesses|Access::Reg|UserStatAccess , c+".upd" ) ; // file date may be updated if created, user might do a meaningful fstat
else r._report_update( ::move(real) , accesses , c+".wr" ) ; // file date may be updated if created, user cannot see file stat before the write
} else {
if (do_read) r._report_dep ( ::move(real) , accesses|Access::Reg|UserStatAccess , c+".upd" ) ; // in src dirs, only the read side is reported
else r._report_dep ( ::move(real) , accesses , c+".wr" ) ; // .
}
c += '.' ;
if (do_stat ) { c += 'S' ; accesses |= UserStatAccesses ; }
if (do_read ) { c += 'R' ; accesses |= UserStatAccesses|Access::Reg ; }
if (do_write) c += 'W' ;
//
if ( do_write ) { r._report_update( ::move(real) , accesses , ::move(c) ) ; confirm = true ; }
else if ( do_read || do_stat ) r._report_dep ( ::move(real) , accesses , ::move(c) ) ;
}

Record::Read::Read( Record& r , Path&& path , bool no_follow , bool keep_real , bool allow_tmp_map , ::string&& c ) : Solve{r,::move(path),no_follow,true/*read*/,allow_tmp_map,c} {
Expand Down Expand Up @@ -303,7 +301,7 @@ Record::Rename::Rename( Record& r , Path&& src_ , Path&& dst_ , bool exchange ,
}

Record::Stat::Stat( Record& r , Path&& path , bool no_follow , ::string&& c ) : Solve{r,::move(path),no_follow,true/*read*/,true/*allow_tmp_map*/,c} {
if ( !s_autodep_env().ignore_stat && file_loc<=FileLoc::Dep ) r._report_dep( ::move(real) , accesses|UserStatAccess , ::move(c) ) ;
if ( !s_autodep_env().ignore_stat && file_loc<=FileLoc::Dep ) r._report_dep( ::move(real) , accesses|UserStatAccesses , ::move(c) ) ;
}

Record::Symlnk::Symlnk( Record& r , Path&& p , ::string&& c ) : Solve{r,::move(p),true/*no_follow*/,false/*read*/,true/*allow_tmp_map*/,c} {
Expand Down
6 changes: 3 additions & 3 deletions src/autodep/record.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ private :
else return IMsgBuf().receive<JobExecRpcReply>(report_fd()) ;
}
//
void _report_access( JobExecRpcReq&& jerr ) const ;
void _report_access( JobExecRpcReq&& jerr ) const ;
void _report_access( ::string&& f , FileInfo fi , Accesses a , bool write , ::string&& c={} ) const {
_report_access({ Proc::Access , {{::move(f),fi}} , {.write=Maybe&write,.accesses=a} , ::move(c) }) ;
}
Expand Down Expand Up @@ -279,9 +279,9 @@ public :
Open() = default ;
Open( Record& , Path&& , int flags , ::string&& comment ) ;
// services
int operator()( Record& r , int rc ) { { if (do_write) r._report_confirm(file_loc,rc>=0) ; } return rc ; }
int operator()( Record& r , int rc ) { { if (confirm) r._report_confirm(file_loc,rc>=0) ; } return rc ; }
// data
bool do_write = false ;
bool confirm = false ;
} ;
struct Read : Solve {
Read() = default ;
Expand Down
24 changes: 17 additions & 7 deletions src/lmakeserver/backend.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,19 @@ namespace Backends {
void send_reply( JobIdx job , JobMngtRpcReply&& jmrr ) {
Lock lock { Backend::_s_mutex } ;
auto it = Backend::_s_start_tab.find(job) ;
if (it==Backend::_s_start_tab.end()) return ; // job is dead without waiting for reply, curious but possible
Backend::StartEntry::Conn const& conn = it->second.conn ;
ClientSockFd fd ( conn.host , conn.port ) ;
jmrr.seq_id = conn.seq_id ;
OMsgBuf().send( fd , jmrr ) ; // XXX : straighten out Fd : Fd must not detach on mv and Epoll must take an AutoCloseFd as arg to take close resp.
if (it==Backend::_s_start_tab.end()) return ; // job is dead without waiting for reply, curious but possible
Backend::StartEntry const& e = it->second ;
if (!e ) return ; // .
try {
jmrr.seq_id = e.conn.seq_id ;
ClientSockFd fd( e.conn.host , e.conn.port , 3/*n_trials*/ ) ;
OMsgBuf().send( fd , jmrr ) ; // XXX : straighten out Fd : Fd must not detach on mv and Epoll must take an AutoCloseFd as arg to take close resp.
} catch (...) { // if we cannot connect to job, assume it is dead while we processed the request
Backend::_s_deferred_wakeup_thread->emplace_after(
g_config.network_delay
, Backend::DeferredEntry { e.conn.seq_id , JobExec(Job(job),e.conn.host,e.start_date,New) }
) ;
}
}

//
Expand Down Expand Up @@ -159,6 +167,7 @@ namespace Backends {

void Backend::_s_wakeup_remote( JobIdx job , StartEntry::Conn const& conn , SigDate const& start_date , JobMngtProc proc ) {
Trace trace(BeChnl,"_s_wakeup_remote",job,conn,proc) ;
SWEAR(conn.seq_id,job,conn) ;
try {
ClientSockFd fd(conn.host,conn.port) ;
OMsgBuf().send( fd , JobMngtRpcReply(proc,conn.seq_id) ) ; // XXX : straighten out Fd : Fd must not detach on mv and Epoll must take an AutoCloseFd as arg to take close resp.
Expand Down Expand Up @@ -424,7 +433,7 @@ namespace Backends {
DF}
Job job { jmrr.job } ;
Trace trace(BeChnl,"_s_handle_job_mngt",jmrr) ;
{ Lock lock { _s_mutex } ; // prevent sub-backend from manipulating _s_start_tab from main thread, lock for minimal time
{ Lock lock { _s_mutex } ; // prevent sub-backend from manipulating _s_start_tab from main thread, lock for minimal time
// keep_fd
auto it = _s_start_tab.find(+job) ; if (it==_s_start_tab.end() ) { trace("not_in_tab" ) ; return false ; }
StartEntry& entry = it->second ; if (entry.conn.seq_id!=jmrr.seq_id) { trace("bad_seq_id",entry.conn.seq_id,jmrr.seq_id) ; return false ; }
Expand Down Expand Up @@ -503,6 +512,7 @@ namespace Backends {
for( auto jit = _s_start_tab.begin() ; jit!=_s_start_tab.end() ;) { // /!\ we erase entries while iterating
JobIdx j = jit->first ;
StartEntry& e = jit->second ;
if (!e) { jit++ ; continue ; }
if (ri) {
if ( e.reqs.size()==1 && e.reqs[0]==ri ) goto Kill ;
for( auto it = e.reqs.begin() ; it!=e.reqs.end() ; it++ ) { // e.reqs is a non-sorted vector, we must search ri by hand
Expand Down Expand Up @@ -642,7 +652,7 @@ namespace Backends {
be->addr = ServerSockFd::s_addr(ifce) ;
}
try { be->config(cfg.dct,dynamic) ; be->config_err.clear() ; trace("ready",t ) ; }
catch (::string const& e) { SWEAR(+e) ; be->config_err = e ; trace("err" ,t,e) ; } // empty config_err means ready
catch (::string const& e) { SWEAR(+e) ; be->config_err = e ; trace("err" ,t,e) ; } // empty config_err means ready
}
job_start_thread.wait_started() ;
job_mngt_thread .wait_started() ;
Expand Down
8 changes: 6 additions & 2 deletions src/lmakeserver/backend.x.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ namespace Backends {
friend ::ostream& operator<<( ::ostream& , StartEntry const& ) ;
struct Conn {
friend ::ostream& operator<<( ::ostream& , Conn const& ) ;
// accesses
bool operator+() const { return seq_id ; }
bool operator!() const { return !+*this ; }
// data
in_addr_t host = NoSockAddr ;
in_port_t port = 0 ;
SeqId seq_id = 0 ;
Expand All @@ -63,8 +67,8 @@ namespace Backends {
StartEntry( ) = default ;
StartEntry(NewType) { open() ; }
// accesses
bool operator+() const { return conn.seq_id ; }
bool operator!() const { return !+*this ; }
bool operator+() const { return +conn ; }
bool operator!() const { return !+*this ; }
bool useful () const ;
// services
void open() {
Expand Down
17 changes: 9 additions & 8 deletions src/lmakeserver/cmd.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ R"({
"type" : "by-gdb"
, "request" : "attach"
, "name" : "Attach C/C++"
, "program" : ""
, "program" : $interpreter
, "cwd" : $g_root_dir
, "processId" : 0
}
Expand All @@ -285,10 +285,11 @@ R"({
/**/ append_to_string( extensions , '"',ext,'"' ) ;
first = false ;
}
res = ::regex_replace( res , ::regex("\\$extensions") , extensions ) ;
res = ::regex_replace( res , ::regex("\\$name" ) , mk_json_str( j->name() ) ) ;
res = ::regex_replace( res , ::regex("\\$g_root_dir") , mk_json_str( *g_root_dir ) ) ;
res = ::regex_replace( res , ::regex("\\$program" ) , mk_json_str(to_string(*g_root_dir,'/',dbg_dir,"/cmd")) ) ;
res = ::regex_replace( res , ::regex("\\$extensions" ) , extensions ) ;
res = ::regex_replace( res , ::regex("\\$name" ) , mk_json_str( j->name() ) ) ;
res = ::regex_replace( res , ::regex("\\$g_root_dir" ) , mk_json_str( *g_root_dir ) ) ;
res = ::regex_replace( res , ::regex("\\$program" ) , mk_json_str(to_string(*g_root_dir,'/',dbg_dir,"/cmd")) ) ;
res = ::regex_replace( res , ::regex("\\$interpreter") , mk_json_str(to_string(start.interpreter[0] )) ) ;
//
::vmap_ss env = _mk_env(start.env,report_end.end.dynamic_env) ;
size_t kw = 13/*SEQUENCE_ID*/ ; for( auto&& [k,v] : env ) if (k!="TMPDIR") kw = ::max(kw,mk_json_str(k).size()) ;
Expand Down Expand Up @@ -370,10 +371,10 @@ R"({
append_to_string( script , "args=()\n" ) ;
append_to_string( script , "type code | grep -q .vscode-server || args+=( \"--user-data-dir ${DEBUG_DIR}/vscode/user\" )\n" ) ;
for( Dep const& dep : j->deps )
if (dep.dflags[Dflag::Static]) append_to_string( script , "args+=( ",mk_shell_str("-g "+dep->name()),")\n" ) ; // list dependences file to open in vscode
append_to_string( script , "args+=(\"-g ${DEBUG_DIR}/cmd\")\n" ) ;
if (dep.dflags[Dflag::Static]) append_to_string( script , "args+=( ",mk_shell_str(dep->name()),")\n" ) ; // list dependences file to open in vscode
append_to_string( script , "args+=(\"${DEBUG_DIR}/cmd\")\n" ) ;
append_to_string( script , "args+=(\"${DEBUG_DIR}/vscode/ldebug.code-workspace\")\n" ) ;
append_to_string( script , "code -n -w ${args[@]} &" ) ;
append_to_string( script , "code -n -w --password-store=basic ${args[@]} &" ) ;
} else {
::vmap_ss env = _mk_env(start.env,report_end.end.dynamic_env) ;
/**/ append_to_string( script , "exec env -i" , " \\\n" ) ;
Expand Down
Loading

0 comments on commit a632cc9

Please sign in to comment.