From 757fec390f56b20d4a3c307476de815cd8e0c8da Mon Sep 17 00:00:00 2001 From: Cesar Douady Date: Sat, 6 Apr 2024 11:11:34 +0200 Subject: [PATCH] implemented lshow -r (running) & -b (bom) --- .gitignore | 3 +- Manifest | 1 + TO_DO | 6 + doc/lmake.texi | 38 ++++-- lmake_env/Lmakefile.py | 20 +-- src/lmakeserver/cmd.cc | 142 ++++++++++++++++--- src/lmakeserver/job.cc | 272 ++++++++++++++++++++++--------------- src/lmakeserver/job.x.hh | 56 ++++---- src/lmakeserver/node.cc | 83 ++++++----- src/lmakeserver/node.x.hh | 50 ++++--- src/lmakeserver/req.cc | 2 +- src/lmakeserver/store.x.hh | 34 ++--- src/lshow.cc | 14 +- src/rpc_client.hh | 2 + src/rpc_job.hh | 87 +++++------- unit_tests/misc10.py | 56 ++++++++ unit_tests/wine.py | 8 +- 17 files changed, 554 insertions(+), 320 deletions(-) create mode 100644 unit_tests/misc10.py diff --git a/.gitignore b/.gitignore index 363f8684..f197c441 100644 --- a/.gitignore +++ b/.gitignore @@ -14,8 +14,9 @@ __pycache__ #auto-configuration /trial/ +/sys_config.log +/sys_config.mk /sys_config.h -/sys_config.h.err /version.hh # compiled files diff --git a/Manifest b/Manifest index cd2c3400..5fc2df4c 100644 --- a/Manifest +++ b/Manifest @@ -185,6 +185,7 @@ unit_tests/mandelbrot.py unit_tests/mandelbrot.zip unit_tests/manual_target.py unit_tests/misc1.py +unit_tests/misc10.py unit_tests/misc2.py unit_tests/misc3.py unit_tests/misc4.py diff --git a/TO_DO b/TO_DO index 51c1a9f1..57daf612 100644 --- a/TO_DO +++ b/TO_DO @@ -74,6 +74,12 @@ items : * ROBUSTNESS & MAINTENABILITY (fragile/difficult to read or maintain as long as not implemented) **************************************************************************************************** +* close socket in the management thread (M) while request is being processed + - such requests must go through the engine_loop + - this may take a while + - this means during this time, all jobs may connect (and actually do) + - requiring a coket for each slurm slot + - defeating the purpose of disconnecting jobs during execution * fix store to be compliant with strict aliasing rules * support 64-bits id - configure with NBits rather than types diff --git a/doc/lmake.texi b/doc/lmake.texi index 1b9a5265..b47cb688 100644 --- a/doc/lmake.texi +++ b/doc/lmake.texi @@ -23,8 +23,8 @@ @end macro @set TITLE The lmake Manual -@set UPDATED 22 April 2023 -@set UPDATED-MONTH April 2023 +@set UPDATED 22 April 2024 +@set UPDATED-MONTH April 2024 @set EDITION @dfn{work in progress} @set VERSION 0.1 @@ -594,16 +594,22 @@ Once the build process is complete, a summary is generated with : @section @code{lshow} -Usage : @code{lshow [ [-d|--deps] | [-D|-inv-deps] | [-T|-inv-targets] | [-i|--info] | [-s|--script] | [-S|--exec-script] | [-E|--env] | [-e|--stderr] |[-o|--stdout] [-b|--backend] -[[-d|--debug] debug_dir] [-t|--targets] [-v|--verbose] [-p|--porcelaine] targets...} +Usage : @code{lshow -short_cmd|--long_cmd [ -short_option|--long_option option_arg ]* files or job} @code{lshow} provides various information about jobs. The jobs are those officially generating the targets (i.e. the jobs that would be selected by @lmake if needing to generate it) if any, else it may be the jobs that actually generated the target. -Options : +There must be a single command while there may be 0 or more options. + +Arguments can be files or a single job as described in generic arguments description. + +Commands : @itemize @bullet -@item @code{-d}|@code{--deps} : output the list of all the deps of the jobs. +@item @code{-b}|@code{--bom} : Output the list of all source files necessary to build the arguments. +If verbose (option @code{-v} or @code{--verbose}), some intermediate files are shown in gray, enough to justify why all listed source files. +@item @code{-c}|@code{--cmd} : Output the @code{cmd} used to execute the job. If dynamic, it shows the specialized @code{cmd} for this job. +@item @code{-d}|@code{--deps} : Output the list of all the deps of the jobs. Unless the verbose flag is specified, only existing deps are shown. Each line is composed of 6 fields separated by spaces : @itemize @minus @@ -626,18 +632,19 @@ Each line is composed of 6 fields separated by spaces : @item Name of the dep. @end itemize If a dep does not exist, it is deemed secondary information and is shown in gray. -@item @code{-D}|@code{--inv-deps} : show jobs that depend on targets. -@item @code{-T}|@code{--inv-targets} : show jobs that produce targets either officially (listed in @code{targets} attribute) or not (listed in @code{side_targets} attribute or not) -@item @code{-i}|@code{--info} : show various self-reading info about jobs, such as reason to be launched, why it was needed, execution time, host that executed it, ... +@item @code{-D}|@code{--inv-deps} : Show jobs that depend on targets. +@item @code{-E}|@code{--env} : Show the environment used to run the script +@item @code{-i}|@code{--info} : Show various self-reading info about jobs, such as reason to be launched, why it was needed, execution time, host that executed it, ... If porcelaine mode (using the @code{--porcelaine} or @code{-p} option), the same information is reported as as an easy to parse Python @code{dict}. -@item @code{-s}|@code{--script} : show the scripts used to execute the job. +@item @code{-r}|@code{--running} : Show the list of jobs currently running to build the arguments. +If verbose (option @code{-v} or @code{--verbose}), some waiting jobs are shown in gray, enough to justify why all running jobs are running. +Queued jobs are shown in blue, actively running jobs are uncolored. +@item @code{-s}|@code{--script} : Show the scripts used to execute the job. If debug is required through the @code{-d} or @code{--debug} option, a dir must be passed and generated script will write info to it using the autodep mechanism. Note that if job requires tmp mapping or auto mkdir, autodep is used in all cases (as it is functionally required) although minimally configured if debug is not asked. -@item @code{-E}|@code{--env} : show the environment used to run the script -@item @code{-e}|@code{--stderr} : show the stderr of the jobs. -@item @code{-o}|@code{--stdout} : show the stdout of the jobs. -@item @code{-b}|@code{--backend} : show some execution information from backend generated when @lmake is run with the @code{-v} flag. -@item @code{-t}|@code{--targets} : show the targets of the jobs. +@item @code{-e}|@code{--stderr} : Show the stderr of the jobs. +@item @code{-o}|@code{--stdout} : Show the stdout of the jobs. +@item @code{-t}|@code{--targets} : Show the targets of the jobs. Unless the verbose flag is specified, only existing targets are shown. Each line is composed of 3 fields separated by spaces : @itemize @minus @@ -655,6 +662,7 @@ Each line is composed of 3 fields separated by spaces : @item The name of the target. @end itemize If a target does not exist, it is deemed secondary information and is shown in gray. +@item @code{-T}|@code{--inv-targets} : Show jobs that produce targets either officially (listed in @code{targets} attribute) or not (listed in @code{side_targets} attribute or not) @end itemize @section @code{ldebug} diff --git a/lmake_env/Lmakefile.py b/lmake_env/Lmakefile.py index 5d81c50f..1b155dee 100644 --- a/lmake_env/Lmakefile.py +++ b/lmake_env/Lmakefile.py @@ -69,8 +69,8 @@ class BaseRule(Rule) : start_delay = 2 n_tokens = config.backends.local.cpu -class Centos7Rule(BaseRule) : - environ_cmd = { 'PATH' : '...:/opt/rh/devtoolset-11/root/usr/bin' } +class PathRule(BaseRule) : # compiler must be accessed using the PATH as it must find its sub-binaries + environ_cmd = { 'PATH' : os.getenv('PATH') } cache = 'dir' class Html(BaseRule) : @@ -140,14 +140,14 @@ class ConfigH(BaseRule) : deps = { 'CONFIGURE' : 'ext/{DirS}configure' } cmd = 'cd ext/{DirS} ; ./configure' -class SysConfigH(Centos7Rule) : +class SysConfigH(PathRule) : targets = { 'MK' : 'sys_config.mk' , 'H' : 'sys_config.h' , 'TRIAL' : 'trial/{*:.*}' } deps = { 'EXE' : '_bin/sys_config' } - cmd = 'CC={gxx} PYTHON={sys.executable} ./{EXE} {MK} {H} 2>&1' + cmd = 'CXX={gxx} PYTHON={sys.executable} ./{EXE} {MK} {H} 2>&1' class VersionH(BaseRule) : target = 'version.hh' @@ -184,12 +184,12 @@ def cmd() : } def run_gxx(target,*args) : cmd_line = ( gxx , '-o' , target , '-fdiagnostics-color=always' , *args ) - if '/' in gxx : os.environ['PATH'] = ':'.join((osp.dirname(gxx),os.environ['PATH'])) # gxx calls its subprograms (e.g. as) using PATH, ensure it points to gxx dir + if '/' in gxx : os.environ['PATH'] = ':'.join((osp.dirname(gxx),os.environ['PATH'])) # gxx calls its subprograms (e.g. as) using PATH, ensure it points to gxx dir for k,v in os.environ.items() : print(f'{k}={v}') print(' '.join(cmd_line)) run( cmd_line , check=True ) for ext,basic_opts in basic_opts_tab.items() : - class Compile(Centos7Rule) : # note that although class is overwritten at each iteration, each is recorded at definition time by the metaclass + class Compile(PathRule) : # note that although class is overwritten at each iteration, each is recorded at definition time by the metaclass name = f'compile {ext}' targets = { 'OBJ' : '{File}.o' } deps = { @@ -223,7 +223,7 @@ def cmd() : if True : resources.mem = '512M' if backend=='local' : resources.cc = 1 -class GxxRule(Centos7Rule) : +class LinkRule(PathRule) : combine = ('pre_opts','rev_post_opts') pre_opts = [] # options before inputs & outputs rev_post_opts = [] # options after inputs & outputs, combine appends at each level, but here we want to prepend @@ -234,14 +234,14 @@ def cmd() : , *reversed(rev_post_opts) ) -class LinkO(GxxRule) : +class LinkO(LinkRule) : pre_opts = ('-r','-fPIC') -class LinkSo(GxxRule) : +class LinkSo(LinkRule) : pre_opts = ('-shared-libgcc','-shared','-pthread') rev_post_opts = () -class LinkExe(GxxRule) : +class LinkExe(LinkRule) : pre_opts = '-pthread' rev_post_opts = () diff --git a/src/lmakeserver/cmd.cc b/src/lmakeserver/cmd.cc index ef365a93..4795f891 100644 --- a/src/lmakeserver/cmd.cc +++ b/src/lmakeserver/cmd.cc @@ -505,6 +505,97 @@ R"({ throw "no mark specified"s ; } + template struct Show { + // cxtors & casts + Show( ) = default ; + Show( Fd fd_ , ReqOptions const& ro_ , DepDepth lvl_=0 ) : fd{fd_} , ro{&ro_} , lvl{lvl_} , verbose{ro_.flags[ReqFlag::Verbose]} {} + // data + Fd fd ; + ReqOptions const* ro = nullptr ; + DepDepth lvl = 0 ; + ::uset job_seen = {} ; + ::uset node_seen = {} ; + ::vector backlog = {} ; + bool verbose = false/*garbage*/ ; + } ; + + struct ShowBom : Show { + using Show::Show ; + // services + void show_job(Job job) { + if (!job_seen.insert(job).second) return ; + for ( Dep const& dep : job->deps ) show_node(dep) ; + } + void show_node(Node node) { + if (!node_seen.insert(node).second) return ; + node->set_buildable() ; + // + if (!node->is_src_anti()) { + if (verbose) backlog.push_back(node) ; + lvl += verbose ; + for( Job j : node->candidate_job_tgts() ) show_job(j) ; + lvl -= verbose ; + if (+backlog) backlog.pop_back() ; + } else if (node->status()<=NodeStatus::Makable) { + Color c = node->buildable==Buildable::Src ? Color::None : Color::Warning ; + DepDepth l = lvl - backlog.size() ; + for( Node n : backlog ) audit( fd , *ro , Color::HiddenNote , mk_file(n ->name()) , false/*as_is*/ , l++ ) ; + /**/ audit( fd , *ro , c , mk_file(node->name()) , false/*as_is*/ , lvl ) ; + backlog = {} ; + } + } + } ; + + struct ShowRunning : Show { + static constexpr BitMap InterestingSteps = { JobStep::Dep/*waiting*/ , JobStep::Queued , JobStep::Exec } ; + using Show::Show ; + // services + void show_job(Job job) { + JobStep step = {} ; + for( Req r : Req::s_reqs_by_start ) + if ( JobStep s = job->c_req_info(r).step() ; InterestingSteps[s] && step!=s ) { // process job as soon as one Req is waiting/running, and this must be coherent + SWEAR(!step,step,s) ; + step = s ; + } + Color color = {} /*garbage*/ ; + char hdr = '?'/*garbage*/ ; + switch (step) { + case JobStep::Dep : break ; + case JobStep::Queued : color = Color::Note ; hdr = 'Q' ; break ; + case JobStep::Exec : color = Color::None ; hdr = 'R' ; break ; + default : return ; + } + if (!job_seen.insert(job).second) return ; + // + switch (step) { + case JobStep::Dep : + if (verbose) backlog.push_back(job) ; + break ; + case JobStep::Queued : + case JobStep::Exec : { + SWEAR( lvl>=backlog.size() , lvl , backlog.size() ) ; + DepDepth l = lvl - backlog.size() ; + for( Job j : backlog ) audit( fd , *ro , Color::HiddenNote , to_string('W',' ',j ->rule->name,' ',mk_file(j ->name())) , false/*as_is*/ , l++ ) ; + /**/ audit( fd , *ro , color , to_string(hdr,' ',job->rule->name,' ',mk_file(job->name())) , false/*as_is*/ , lvl ) ; + backlog = {} ; + return ; + } + DF} + lvl += verbose ; + for( Dep const& dep : job->deps ) show_node(dep) ; + lvl -= verbose ; + if (+backlog) backlog.pop_back() ; + } + void show_node(Node node) { + for( Req r : Req::s_reqs_by_start ) + if ( NodeReqInfo const& cri = node->c_req_info(r) ; cri.waiting() ) { // process node as soon as one Req is waiting + if (!node_seen.insert(node).second) return ; + for( Job j : node->conform_job_tgts(cri) ) show_job(j) ; + return ; + } + } + } ; + static void _show_job( Fd fd , ReqOptions const& ro , Job job , DepDepth lvl=0 ) { Trace trace("show_job",ro.key,job) ; Rule rule = job->rule ; @@ -522,12 +613,13 @@ R"({ case ReqKey::Stdout : { if (rule->is_special()) { switch (ro.key) { - case ReqKey::Info : + case ReqKey::Info : case ReqKey::Stderr : { _send_job( fd , ro , No/*show_deps*/ , false/*hide*/ , job , lvl ) ; audit ( fd , ro , job->special_stderr() , false/*as_is*/ , lvl+1 ) ; } break ; case ReqKey::Cmd : + case ReqKey::Env : case ReqKey::ExecScript : case ReqKey::Stdout : _send_job( fd , ro , No/*show_deps*/ , false/*hide*/ , job , lvl ) ; @@ -560,8 +652,8 @@ R"({ break ; case ReqKey::Stdout : if (!has_end) { audit( fd , ro , Color::Err , "no info available" , true/*as_is*/ , lvl ) ; break ; } - _send_job( fd , ro , No/*show_deps*/ , false/*hide*/ , job , lvl ) ; - audit ( fd , ro , digest.stdout , lvl+1 ) ; + _send_job( fd , ro , No/*show_deps*/ , false/*hide*/ , job , lvl ) ; + audit ( fd , ro , digest.stdout , lvl+1 ) ; break ; case ReqKey::Stderr : if (!has_end && !(has_start&&verbose) ) { audit( fd , ro , Color::Err , "no info available" , true/*as_is*/ , lvl ) ; break ; } @@ -727,9 +819,9 @@ R"({ DF} } } break ; - case ReqKey::Deps : - _send_job( fd , ro , Maybe|verbose , false/*hide*/ , job , lvl ) ; - break ; + case ReqKey::Bom : ShowBom (fd,ro,lvl).show_job(job) ; break ; + case ReqKey::Running : ShowRunning(fd,ro,lvl).show_job(job) ; break ; + case ReqKey::Deps : _send_job( fd , ro , Maybe|verbose , false/*hide*/ , job , lvl ) ; break ; case ReqKey::Targets : { Rule::SimpleMatch match = job->simple_match() ; for( auto const& [tn,td] : digest.targets ) { @@ -759,6 +851,11 @@ R"({ ::vector targets = ecr.targets() ; bool porcelaine = ro.flags[ReqFlag::Porcelaine] ; char sep = ' ' ; + switch (ro.key) { + case ReqKey::Bom : { ShowBom sb {fd,ro} ; for( Node t : targets ) sb.show_node(t) ; goto Return ; } + case ReqKey::Running : { ShowRunning sr {fd,ro} ; for( Node t : targets ) sr.show_node(t) ; goto Return ; } + default : ; + } if (porcelaine) audit( fd , ro , "{" , true/*as_is*/ ) ; for( Node target : targets ) { trace("target",target) ; @@ -767,11 +864,12 @@ R"({ else if (targets.size()>1) _send_node( fd , ro , true/*always*/ , Maybe/*hide*/ , {} , target ) ; else lvl-- ; sep = ',' ; - bool for_job = false/*garbage*/ ; + bool for_job = true ; switch (ro.key) { case ReqKey::InvDeps : - case ReqKey::InvTargets : for_job = false ; break ; - default : for_job = true ; + case ReqKey::InvTargets : + case ReqKey::Running : for_job = false ; break ; + default : ; } Job job ; if (for_job) { @@ -779,14 +877,22 @@ R"({ if ( !job && ro.key!=ReqKey::Info ) { ok = false ; continue ; } } switch (ro.key) { + case ReqKey::Cmd : + case ReqKey::Env : + case ReqKey::ExecScript : + case ReqKey::Stderr : + case ReqKey::Stdout : + case ReqKey::Targets : + _show_job(fd,ro,job,lvl) ; + break ; case ReqKey::Info : if (target->status()==NodeStatus::Plain) { Job cj = target->conform_job() ; size_t w = 0 ; bool seen_candidate = false ; for( Job j : target->conform_job_tgts() ) { - /**/ w = ::max(w,j->rule->name.size()) ; - if (j!=cj) seen_candidate = true ; + w = ::max(w,j->rule->name.size()) ; + seen_candidate |= j!=cj ; } for( Job j : target->conform_job_tgts() ) { if (j==job ) continue ; @@ -798,17 +904,12 @@ R"({ if (!job) { Node n = target ; while ( +n->asking && n->asking.is_a() ) n = Node(n->asking) ; - if (+n->asking) audit( fd , ro , to_string("required by : ",mk_file(Job(n->asking)->name())) ) ; - else audit( fd , ro , to_string("required by : ",mk_file( n ->name())) ) ; + if (n!=target) { + if (+n->asking) audit( fd , ro , to_string("required by : ",mk_file(Job(n->asking)->name())) ) ; + else audit( fd , ro , to_string("required by : ",mk_file( n ->name())) ) ; + } continue ; } - [[fallthrough]] ; - case ReqKey::Cmd : - case ReqKey::Env : - case ReqKey::ExecScript : - case ReqKey::Stderr : - case ReqKey::Stdout : - case ReqKey::Targets : _show_job(fd,ro,job,lvl) ; break ; case ReqKey::Deps : { @@ -840,6 +941,7 @@ R"({ DF} } if (porcelaine) audit( fd , ro , "}" , true/*as_is*/ ) ; + Return : trace(STR(ok)) ; return ok ; } diff --git a/src/lmakeserver/job.cc b/src/lmakeserver/job.cc index 6e2fb37d..8bb83c23 100644 --- a/src/lmakeserver/job.cc +++ b/src/lmakeserver/job.cc @@ -124,8 +124,25 @@ namespace Engine { // JobReqInfo // + ::ostream& operator<<( ::ostream& os , JobReqInfo::State const& ris ) { + const char* sep = "" ; + /**/ os <<'(' ; + if (+ris.reason ) { os <"<"< old_deps = mk_uset((*this)->deps.view()) ; - ::vector deps ; deps.reserve(digest.deps.size()) ; + ::uset old_deps ; for( Dep const& d : (*this)->deps ) old_deps.insert(d) ; + ::vector deps ; deps.reserve(digest.deps.size()) ; for( auto const& [dn,dd] : digest.deps ) { Dep dep { Node(dn) , dd } ; if (!old_deps.contains(dep)) has_new_deps = true ; @@ -492,15 +509,15 @@ namespace Engine { req->missing_audits.erase(*this) ; // old missing audit is obsolete as soon as we have rerun the job // we call wakeup_watchers ourselves once reports are done to avoid anti-intuitive report order // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - JobReason job_reason = (*this)->make( ri , end_action , target_reason , Yes/*speculate*/ , &old_exec_time , false/*wakeup_watchers*/ ) ; + JobReason err_reason = (*this)->make( ri , end_action , target_reason , Yes/*speculate*/ , &old_exec_time , false/*wakeup_watchers*/ ) ; // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ bool full_report = ri.done() || !has_new_deps ; // if not done, does a full report anyway if this is not due to new deps ::string job_msg ; if (full_report) { - bool job_err = job_reason.tag>=JobReasonTag::Err ; + bool job_err = err_reason.tag>=JobReasonTag::Err ; job_msg = msg ; if (!job_err) append_line_to_string( job_msg , local_msg ) ; // report local_msg if nothing more import to report - else { append_line_to_string( job_msg , reason_str(job_reason),'\n' ) ; err_reason |= job_reason ; stderr = {} ; } // dont report user stderr if analysis made it meaningless + else { append_line_to_string( job_msg , reason_str(err_reason),'\n' ) ; err_reason |= err_reason ; stderr = {} ; } // dont report user stderr if analysis made it meaningless /**/ append_line_to_string( job_msg , severe_msg ) ; } // @@ -621,11 +638,32 @@ namespace Engine { } } - JobReason JobData::make( ReqInfo& ri , MakeAction make_action , JobReason reason , Bool3 speculate , CoarseDelay const* old_exec_time , bool wakeup_watchers ) { + static JobReasonTag _mk_reason(Status s) { + static constexpr JobReasonTag ReasonTab[] = { + JobReasonTag::New // New + , JobReasonTag::ChkDeps // EarlyChkDeps + , JobReasonTag::None // EarlyErr + , JobReasonTag::Lost // EarlyLost + , JobReasonTag::None // EarlyLostErr + , JobReasonTag::Lost // LateLost + , JobReasonTag::None // LateLostErr + , JobReasonTag::Killed // Killed + , JobReasonTag::ChkDeps // ChkDeps + , JobReasonTag::PollutedTargets // BadTarget + , JobReasonTag::None // Ok + , JobReasonTag::None // Err + } ; + static_assert(sizeof(ReasonTab)==N) ; + JobReasonTag res = ReasonTab[+s] ; + /**/ SWEAR( resspecial ; @@ -633,15 +671,24 @@ namespace Engine { bool submit_loop = false ; CoarseDelay dep_pressure = ri.pressure + best_exec_time().first ; bool archive = req->options.flags[ReqFlag::Archive] ; + JobReason pre_reason ; // reason to run job when deps are ready before deps analysis // - Trace trace("Jmake",idx(),ri,ri.reasons,make_action,reason,speculate,STR(stop_speculate),old_exec_time?*old_exec_time:CoarseDelay(),STR(wakeup_watchers),prev_step) ; + auto reason = [&]()->JobReason { + if (ri.force) return pre_reason | ri.reason | ri.state.reason ; + else return pre_reason | ri.state.reason | ri.reason ; + } ; + auto need_run = [&](ReqInfo::State const& s)->bool { + SWEAR( pre_reason.tagn_submits && ri.n_submits>rule->n_submits ) { SWEAR(is_ok(status)==No) ; goto Done ; } // we do not analyze, ensure we have an error state + if ( idx().frozen() ) goto Run ; // ensure crc are updated, akin sources + if ( special==Special::Infinite ) goto Run ; // special case : Infinite actually has no dep, just a list of node showing infinity + if ( rule->n_submits && ri.n_submits>rule->n_submits ) { SWEAR(is_ok(status)==No) ; goto Done ; } // we do not analyze, ensure we have an error state } - if ( is_ok(status)==Maybe && +mk_reason(status) ) ri.reasons.first |= mk_reason(status) ; - if ( ri.step()==Step::None ) { + if (ri.step()==Step::None) { + ri.step(Step::Dep) ; if (ri.full) { - if ( rule->force ) ri.reasons.first |= JobReasonTag::Force ; - else if ( !cmd_ok() ) { ri.new_cmd = true ; ri.reasons.first |= JobReasonTag::Cmd ; } - else if ( req->options.flags[ReqFlag::ForgetOldErrors] && err() ) { ri.new_cmd = true ; ri.reasons.first |= JobReasonTag::OldErr ; } // probably a transcient error, do as cmd modif - else if ( !rsrcs_ok() ) { ri.new_cmd = true ; ri.reasons.first |= JobReasonTag::Rsrcs ; } // probably a resource error, do as cmd modif + JobReasonTag jrt = {} ; + if ( rule->force ) jrt = JobReasonTag::Force ; + else if ( !cmd_ok() ) jrt = JobReasonTag::Cmd ; + else if ( req->options.flags[ReqFlag::ForgetOldErrors] && err() ) jrt = JobReasonTag::OldErr ; // probably a transcient error + else if ( !rsrcs_ok() ) jrt = JobReasonTag::Rsrcs ; // probably a resource error + if (+jrt) { + ri.reason = jrt ; + ri.force = true ; + ri.state.stamped_modif = true ; + } } - ri.step(Step::Dep) ; } SWEAR(ri.step()==Step::Dep) ; - // - { auto need_dsk = [&]( bool care , JobReason r=JobReasonTag::Force )->bool { - bool need_run = r.need_run() || ri.reason().need_run() ; - return ri.full && care && (need_run||archive) ; - } ; + { RestartAnalysis : // restart analysis here when it is discovered we need deps to run the job bool stamped_seen_waiting = false ; bool proto_seen_waiting = false ; @@ -693,21 +744,17 @@ namespace Engine { bool critical_waiting = false ; bool sure = true ; // - ri.speculative_deps = false ; // initially, we are not speculatively waiting - ri.stamped_modif = ri.proto_modif = ri.new_cmd ; + ri.speculative_wait = false ; // initially, we are not waiting at all // - JobReason cur_reason = ri.reasons.first ; - for ( DepsIter iter {deps,ri.iter} ; true ; iter++ ) { - if (!proto_seen_waiting) { - ri.iter = iter.digest(deps) ; // fast path : info is recorded in ri, next time, restart analysis here - ri.reasons.first = cur_reason ; - } + ReqInfo::State state = ri.state ; + pre_reason = _mk_reason(status) ; + for ( DepsIter iter {deps,ri.iter} ;; iter++ ) { // bool seen_all = iter==deps.end() ; Dep const& dep = seen_all ? Sentinel : *iter ; // use empty dep as sentinel - if ( !dep.parallel || seen_all ) { - ri.stamped_err = ri.proto_err ; // proto become stamped upon sequential dep - ri.stamped_modif = ri.proto_modif ; // . + if (!dep.parallel) { + state.stamped_err = state.proto_err ; // proto become stamped upon sequential dep + state.stamped_modif = state.proto_modif ; // . if ( critical_modif && !seen_all ) { // suppress deps following modified critical one, except static deps as no-access ::vector static_deps ; for( DepsIter it=iter ; it!=deps.end() ; it++ ) if (it->dflags[Dflag::Static]) { @@ -717,9 +764,13 @@ namespace Engine { deps.replace_tail(iter,static_deps) ; seen_all = !static_deps ; } - if ( seen_all || critical_waiting ) break ; stamped_seen_waiting = proto_seen_waiting ; } + if (!proto_seen_waiting) { + ri.iter = iter.digest(deps) ; // fast path : info is recorded in ri, next time, restart analysis here + ri.state = state ; // . + } + if ( seen_all || (!dep.parallel&&critical_waiting) ) break ; NodeData & dnd = *Node(dep) ; bool dep_modif = false ; RunStatus dep_err = RunStatus::Ok ; @@ -728,13 +779,14 @@ namespace Engine { bool is_critical = dep.dflags[Dflag::Critical ] && care ; bool sense_err = !dep.dflags[Dflag::IgnoreError] ; bool required = dep.dflags[Dflag::Required ] || is_static ; + bool modif = state.stamped_modif || ri.force ; NodeReqInfo const* cdri = &dep->c_req_info(req) ; // avoid allocating req_info as long as not necessary NodeReqInfo * dri = nullptr ; // . NodeGoal dg = - need_dsk(care,cur_reason) ? NodeGoal::Dsk - : ri.full && ( care || sense_err ) ? NodeGoal::Status - : required ? NodeGoal::Makable - : NodeGoal::None + ri.full && care && (need_run(state)||archive) ? NodeGoal::Dsk + : ri.full && (care||sense_err) ? NodeGoal::Status + : required ? NodeGoal::Makable + : NodeGoal::None ; if (!dg ) continue ; // this is not a dep (not static while asked for makable only) if (!cdri->waiting()) { @@ -742,10 +794,10 @@ namespace Engine { if (!dri ) cdri = dri = &dep->req_info(*cdri) ; // refresh cdri in case dri allocated a new one if (dep_live_out) dri->live_out = true ; // ask live output for last level if user asked it Bool3 speculate_dep = - is_static ? ri.speculate // static deps do not disappear - : stamped_seen_waiting || ri.stamped_modif ? Yes // this dep may disappear - : +ri.stamped_err ? ri.speculate|Maybe // this dep is not the origin of the error - : ri.speculate // this dep will not disappear from us + is_static ? ri.speculate // static deps do not disappear + : stamped_seen_waiting || modif ? Yes // this dep may disappear + : +state.stamped_err ? ri.speculate|Maybe // this dep is not the origin of the error + : ri.speculate // this dep will not disappear from us ; if (special!=Special::Req) dnd.asking = idx() ; // Req jobs are fugitive, dont record them //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv @@ -754,76 +806,76 @@ namespace Engine { } if ( is_static && dnd.buildablewaiting()) { - if ( is_static ) ri.speculative_deps = false ; // we are non-speculatively waiting, even if after a speculative wait - else if ( !stamped_seen_waiting && (+ri.stamped_err||ri.stamped_modif) ) ri.speculative_deps = true ; + if ( is_static ) ri.speculative_wait = false ; // we are non-speculatively waiting, even if after a speculative wait + else if ( !stamped_seen_waiting && (+state.stamped_err||modif) ) ri.speculative_wait = true ; proto_seen_waiting = true ; - if (!dri) cdri = dri = &dnd.req_info(*cdri) ; // refresh cdri in case dri allocated a new one + if (!dri) cdri = dri = &dnd.req_info(*cdri) ; // refresh cdri in case dri allocated a new one dnd.add_watcher(*dri,idx(),ri,dep_pressure) ; critical_waiting |= is_critical ; goto Continue ; } - SWEAR(dnd.done(*cdri,dg)) ; // after having called make, dep must be either waiting or done - if ( need_dsk(care) && !dnd.done(*cdri,NodeGoal::Dsk) ) ri.missing_dsk = true ; // job needs this dep if it must run - if (dg==NodeGoal::Makable ) continue ; // dont check modifs and errors + SWEAR(dnd.done(*cdri,dg)) ; // after having called make, dep must be either waiting or done + if ( ri.full && care && !dnd.done(*cdri,NodeGoal::Dsk) ) state.missing_dsk = true ; // job needs this dep if it must run + if (dg==NodeGoal::Makable ) continue ; // dont check modifs and errors dep_modif = care && !dep.up_to_date() ; - if ( dep_modif && status==Status::Ok && dep.no_trigger() ) { // no_trigger only applies to successful jobs + if ( dep_modif && status==Status::Ok && dep.no_trigger() ) { // no_trigger only applies to successful jobs trace("no_trigger",dep) ; - req->no_triggers.emplace(dep,req->no_triggers.size()) ; // record to repeat in summary, value is just to order summary in discovery order + req->no_triggers.emplace(dep,req->no_triggers.size()) ; // record to repeat in summary, value is just to order summary in discovery order dep_modif = false ; } - if (dep_modif) cur_reason |= {JobReasonTag::DepOutOfDate,+dep} ; - if ( +ri.stamped_err ) goto Continue ; // we are already in error, no need to analyze errors any further - if ( !is_static && ri.stamped_modif ) goto Continue ; // if not static, errors may be washed by previous modifs, dont record them + if (dep_modif) state.reason |= {JobReasonTag::DepOutOfDate,+dep} ; + if ( +state.stamped_err ) goto Continue ; // we are already in error, no need to analyze errors any further + if ( !is_static && modif ) goto Continue ; // if not static, errors may be washed by previous modifs, dont record them // { Bool3 dep_ok = dnd.ok(*cdri,dep.accesses) ; switch (dep_ok) { case Yes : break ; case Maybe : if (required) { - if (is_static) { dep_err = RunStatus::MissingStatic ; cur_reason |= {JobReasonTag::DepMissingStatic ,+dep} ; } - else { dep_err = RunStatus::DepErr ; cur_reason |= {JobReasonTag::DepMissingRequired,+dep} ; } + if (is_static) { dep_err = RunStatus::MissingStatic ; state.reason |= {JobReasonTag::DepMissingStatic ,+dep} ; } + else { dep_err = RunStatus::DepErr ; state.reason |= {JobReasonTag::DepMissingRequired,+dep} ; } trace("missing",STR(is_static),dep) ; goto Continue ; } break ; case No : trace("dep_err",dep,STR(sense_err)) ; - if (+(cdri->overwritten&dep.accesses)) { dep_err = RunStatus::DepErr ; cur_reason |= {JobReasonTag::DepOverwritten,+dep} ; goto Continue ; } // even with !sense_err - if (sense_err ) { dep_err = RunStatus::DepErr ; cur_reason |= {JobReasonTag::DepErr ,+dep} ; goto Continue ; } + if (+(cdri->overwritten&dep.accesses)) { dep_err = RunStatus::DepErr ; state.reason |= {JobReasonTag::DepOverwritten,+dep} ; goto Continue ; } // even with !sense_err + if (sense_err ) { dep_err = RunStatus::DepErr ; state.reason |= {JobReasonTag::DepErr ,+dep} ; goto Continue ; } break ; DF} - if ( care && dep.is_date && +dep.date() ) { // if still waiting for a crc here, it will never come + if ( care && dep.is_date && +dep.date() ) { // if still waiting for a crc here, it will never come SWEAR(!dnd.running(*cdri)) ; - SWEAR(dep_modif ) ; // this dep was moving, retry job - if (!dri) cdri = dri = &dnd.req_info(*cdri) ; // refresh cdri in case dri allocated a new one + SWEAR(dep_modif ) ; // this dep was moving, retry job + if (!dri) cdri = dri = &dnd.req_info(*cdri) ; // refresh cdri in case dri allocated a new one Manual manual = dnd.manual_wash(*dri) ; - if (manual>=Manual::Changed) { trace("dangling",dep ) ; dep_err = RunStatus::DepErr ; cur_reason |= {JobReasonTag::DepDangling,+dep} ; } - else if (manual==Manual::Unlnked) { trace("washed" ,dep ) ; cur_reason |= {JobReasonTag::DepUnlnked ,+dep} ; } + if (manual>=Manual::Changed) { trace("dangling",dep ) ; dep_err = RunStatus::DepErr ; state.reason |= {JobReasonTag::DepDangling,+dep} ; } + else if (manual==Manual::Unlnked) { trace("washed" ,dep ) ; state.reason |= {JobReasonTag::DepUnlnked ,+dep} ; } else trace("unstable",dep,manual) ; } } Continue : - trace("dep",ri,dep,*cdri,STR(dnd.done(*cdri)),STR(dnd.ok()),dnd.crc,dep_err,STR(dep_modif),STR(critical_modif),STR(critical_waiting),cur_reason) ; + trace("dep",ri,dep,*cdri,STR(dnd.done(*cdri)),STR(dnd.ok()),dnd.crc,dep_err,STR(dep_modif),STR(critical_modif),STR(critical_waiting),state.reason) ; // - if ( ri.missing_dsk && cur_reason.need_run() ) { + if ( state.missing_dsk && need_run(state) ) { trace("restart_analysis") ; ri.reset() ; - SWEAR(!ri.reasons.second) ; // we should have asked for dep on disk if we had a reason to run - ri.reasons.second = cur_reason ; // record that we must ask for dep on disk - goto RestartAnalysis ; // BACKWARD + SWEAR(!ri.reason,ri.reason) ; // we should have asked for dep on disk if we had a reason to run + ri.reason = state.reason ; // record that we must ask for dep on disk + goto RestartAnalysis ; // BACKWARD } - SWEAR( !dep_err || !ri.stamped_modif || is_static ) ; // if earlier modifs have been seen, we do not want to record errors as they can be washed, unless static - ri.proto_err = ri.proto_err | dep_err ; // |= is forbidden for bit fields - ri.proto_modif = ri.proto_modif | dep_modif ; // . - critical_modif |= dep_modif && is_critical ; + SWEAR(!( +dep_err && modif && !is_static )) ; // if earlier modifs have been seen, we do not want to record errors as they can be washed, unless static + state.proto_err = state.proto_err | dep_err ; // |= is forbidden for bit fields + state.proto_modif = state.proto_modif | dep_modif ; // . + critical_modif |= dep_modif && is_critical ; } - if ( ri.waiting() ) goto Wait ; - if ( sure ) mk_sure() ; // improve sure (sure is pessimistic) - if ( +(run_status=ri.stamped_err) ) goto Done ; - if ( !ri.reason() ) goto Done ; + if ( ri.waiting() ) goto Wait ; + if ( sure ) mk_sure() ; // improve sure (sure is pessimistic) + if ( +(run_status=ri.state.stamped_err) ) goto Done ; + if ( !need_run(ri.state) ) goto Done ; } Run : - trace("run",ri,run_status,ri.reasons) ; + trace("run",pre_reason,ri,run_status) ; if (rule->n_submits) { if (ri.n_submits>=rule->n_submits) { trace("submit_loop") ; @@ -833,17 +885,18 @@ namespace Engine { } ri.n_submits++ ; } - // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - if (is_special()) { if (!_submit_special(ri )) goto Done ; } // if no new deps, we cannot be waiting and we are done - else { if (!_submit_plain (ri,ri.reason(),dep_pressure)) goto Done ; } // . - // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + // vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + if (is_special()) { if (!_submit_special( ri )) goto Done ; } // if no new deps, we cannot be waiting and we are done + else { if (!_submit_plain ( ri , reason() , dep_pressure )) goto Done ; } // . + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ if (ri.waiting()) goto Wait ; SWEAR(!ri.running()) ; - make_action = MakeAction::End ; // restart analysis as if called by end() as in case of flash execution, submit has called end() - if (is_ok(status)!=Maybe) ri.reasons = { {} , {} } ; // . - else ri.reasons = { mk_reason(status) , {} } ; // . + make_action = MakeAction::End ; // restart analysis as if called by end() as in case of flash execution, submit has called end() + ri.inc_wait() ; // . + asked_reason = {} ; // . + ri.reason = {} ; // . trace("restart_analysis",ri) ; - goto RestartFullAnalysis ; // BACKWARD + goto RestartFullAnalysis ; // BACKWARD Done : SWEAR( !ri.running() && !ri.waiting() , idx() , ri ) ; ri.step(Step::Done) ; @@ -854,12 +907,12 @@ namespace Engine { JobInfo ji = job_info() ; ::string const& stderr = ji.end.end.digest.stderr ; // - if (ja.report!=JobReport::Hit) { // if not Hit, then job was rerun and ja.report is the report that would have been done w/o rerun + if (ja.report!=JobReport::Hit) { // if not Hit, then job was rerun and ja.report is the report that would have been done w/o rerun SWEAR(req->stats.ended(JobReport::Rerun)>0) ; - req->stats.ended(JobReport::Rerun)-- ; // we tranform a rerun into a completed job, subtract what was accumulated as rerun - req->stats.ended(ja.report )++ ; // we tranform a rerun into a completed job, subtract what was accumulated as rerun - req->stats.jobs_time[false/*useful*/] -= exec_time ; // exec time is not added to useful as it is not provided to audit_end - req->stats.jobs_time[true /*useful*/] += exec_time ; // . + req->stats.ended(JobReport::Rerun)-- ; // we tranform a rerun into a completed job, subtract what was accumulated as rerun + req->stats.ended(ja.report )++ ; // we tranform a rerun into a completed job, subtract what was accumulated as rerun + req->stats.jobs_time[false/*useful*/] -= exec_time ; // exec time is not added to useful as it is not provided to audit_end + req->stats.jobs_time[true /*useful*/] += exec_time ; // . } // size_t max_stderr_len ; @@ -872,8 +925,9 @@ namespace Engine { req->audit_job(Color::Note,"dynamic",idx()) ; req->audit_stderr( ensure_nl(rule->end_none_attrs.s_exc_msg(true/*using_static*/))+msg_err.first , msg_err.second , -1 , 1 ) ; } - if (ri.reason().tag>=JobReasonTag::Err) audit_end( ja.report==JobReport::Hit?"hit_":"was_" , ri , reason_str(ri.reason()) , stderr , max_stderr_len , ja.modified ) ; - else audit_end( ja.report==JobReport::Hit?"hit_":"was_" , ri , ja.backend_msg , stderr , max_stderr_len , ja.modified ) ; + JobReason r = reason() ; + if (r.tag>=JobReasonTag::Err) audit_end( ja.report==JobReport::Hit?"hit_":"was_" , ri , reason_str(r) , stderr , max_stderr_len , ja.modified ) ; + else audit_end( ja.report==JobReport::Hit?"hit_":"was_" , ri , ja.backend_msg , stderr , max_stderr_len , ja.modified ) ; req->missing_audits.erase(it) ; } trace("wakeup",ri) ; @@ -891,7 +945,7 @@ namespace Engine { bool add_new = ri.step()>=Step::MinCurStats && ri.step()(deps)) ; - for( Req r : running_reqs() ) { (void)r ; return false ; } // ensure job is not running + Trace trace("Jforget",idx(),STR(targets_),STR(deps_)) ; + for( Req r : running_reqs() ) { (void)r ; return false ; } // ensure job is not running status = Status::New ; - fence() ; // once status is New, we are sure target is not up to date, we can safely modify it + fence() ; // once status is New, we are sure target is not up to date, we can safely modify it run_status = RunStatus::Ok ; if (deps_) { ::vector static_deps ; diff --git a/src/lmakeserver/job.x.hh b/src/lmakeserver/job.x.hh index a71278dc..6407a825 100644 --- a/src/lmakeserver/job.x.hh +++ b/src/lmakeserver/job.x.hh @@ -185,19 +185,15 @@ namespace Engine { default : return false ; } } - bool done (bool is_full=false) const { return step()>=Step::Done && (!is_full||full) ; } - JobReason reason( ) const { return reasons.first | reasons.second ; } + bool done(bool is_full=false) const { return step()>=Step::Done && (!is_full||full) ; } // Step step( ) const { return _step ; } void step(Step s) ; // services void reset() { if (step()>Step::Dep) step(Step::Dep) ; - missing_dsk = false ; - iter = {} ; - reasons.first = {} ; - stamped_err = proto_err = {} ; // no errors in dep as no dep analyzed yet - stamped_modif = proto_modif = false ; + iter = {} ; + state = State() ; } void add_watcher( Node watcher , NodeReqInfo& watcher_req_info ) { ReqInfo::add_watcher(watcher,watcher_req_info) ; @@ -213,26 +209,30 @@ namespace Engine { DF} } // data - // req independent (identical for all Req's) : these fields are there as there is no Req-independent non-persistent table - ::pair reasons ; // 36+36<=128 bits, reasons to run job when deps are ready - DepsIter::Digest iter ; // ~28 <= 64 bits, deps up to this one statisfy required action - uint8_t n_submits = 0 ; // 8 bits, number of times job has been submitted to avoid infinite loop - bool new_cmd :1 = false ; // 1 bit , if true <=> cmd has been modified - bool full :1 = false ; // 1 bit , if true <=>, job result is asked, else only makable - bool missing_dsk :1 = false ; // 1 bit , if true <=>, a dep has been checked but not on disk - RunStatus stamped_err :2 = {} ; // 2 bits, errors seen in dep until iter before last parallel chunk, Maybe means missing static - RunStatus proto_err :2 = {} ; // 2 bits, errors seen in dep until iter including last parallel chunk, Maybe means missing static - bool stamped_modif :1 = false ; // 1 bit , modifs seen in dep until iter before last parallel chunk - bool proto_modif :1 = false ; // 1 bit , modifs seen in dep until iter including last parallel chunk - bool start_reported :1 = false ; // 1 bit , if true <=> start message has been reported to user - bool speculative_deps:1 = false ; // 1 bit , if true <=> job is waiting for speculative deps only - Bool3 speculate :2 = Yes ; // 2 bits, Yes : prev dep not ready, Maybe : prev dep in error (percolated) - bool reported :1 = false ; // 1 bit , used for delayed report when speculating - BackendTag backend :2 = {} ; // 2 bits + struct State { + friend ::ostream& operator<<( ::ostream& , State const& ) ; + JobReason reason = {} ; // 36 <= 64 bits, reason to run job when deps are ready, due to dep analysis + bool missing_dsk :1 = false ; // 1 bit , if true <=>, a dep has been checked but not on disk and analysis must be redone if job has to run + RunStatus stamped_err :2 = {} ; // 2 bits, errors seen in dep until iter before last parallel chunk, Maybe means missing static + RunStatus proto_err :2 = {} ; // 2 bits, errors seen in dep until iter including last parallel chunk, Maybe means missing static + bool stamped_modif:1 = false ; // 1 bit , modifs seen in dep until iter before last parallel chunk + bool proto_modif :1 = false ; // 1 bit , modifs seen in dep until iter including last parallel chunk + } ; + DepsIter::Digest iter ; // ~20+6<= 64 bits, deps up to this one statisfy required action + State state ; // 43 <= 96 bits, dep analysis state + JobReason reason ; // 36 <= 64 bits, reason to run job when deps are ready, forced (before deps) or asked by caller (after deps) + uint8_t n_submits = 0 ; // 8 bits, number of times job has been submitted to avoid infinite loop + bool force :1 = false ; // 1 bit , if true <=> job must run because reason + bool full :1 = false ; // 1 bit , if true <=>, job result is asked, else only makable + bool start_reported :1 = false ; // 1 bit , if true <=> start message has been reported to user + bool speculative_wait:1 = false ; // 1 bit , if true <=> job is waiting for speculative deps only + Bool3 speculate :2 = Yes ; // 2 bits, Yes : prev dep not ready, Maybe : prev dep in error (percolated) + bool reported :1 = false ; // 1 bit , used for delayed report when speculating + BackendTag backend :2 = {} ; // 2 bits private : - Step _step :3 = {} ; // 3 bits + Step _step:3 = {} ; // 3 bits } ; - static_assert(sizeof(JobReqInfo)==48) ; // check expected size + static_assert(sizeof(JobReqInfo)==48) ; // check expected size XXX : optimize size, can be 32 } @@ -311,12 +311,12 @@ namespace Engine { // Tflags tflags(Node target) const ; // - void end_exec ( ) const ; // thread-safe + void end_exec ( ) const ; // thread-safe ::string ancillary_file(AncillaryTag=AncillaryTag::Data) const ; JobInfo job_info ( ) const { return { ancillary_file() } ; } void write_job_info(JobInfo const& ji ) const { ji.write(Disk::dir_guard(ancillary_file())) ; } ::string special_stderr(Node ) const ; - ::string special_stderr( ) const ; // cannot declare a default value for incomplete type Node + ::string special_stderr( ) const ; // cannot declare a default value for incomplete type Node // void invalidate_old() ; Rule::SimpleMatch simple_match () const ; // thread-safe @@ -331,7 +331,7 @@ namespace Engine { _propag_speculate(ri) ; } // - JobReason make( ReqInfo& , MakeAction , JobReason={} , Bool3 speculate=Yes , CoarseDelay const* old_exec_time=nullptr , bool wakeup_watchers=true ) ; + JobReason/*err*/ make( ReqInfo& , MakeAction , JobReason={} , Bool3 speculate=Yes , CoarseDelay const* old_exec_time=nullptr , bool wakeup_watchers=true ) ; // void wakeup(ReqInfo& ri) { make(ri,MakeAction::Wakeup) ; } // diff --git a/src/lmakeserver/node.cc b/src/lmakeserver/node.cc index 2354978b..34466619 100644 --- a/src/lmakeserver/node.cc +++ b/src/lmakeserver/node.cc @@ -153,6 +153,19 @@ namespace Engine { return sjts.subvec(0,sz) ; } + ::c_vector_view NodeData::candidate_job_tgts() const { + RuleIdx ci = conform_idx() ; + if (ci==NoIdx) return {} ; + JobTgts const& jts = job_tgts() ; // /!\ jts is a CrunchVector, so if single element, a subvec would point to it, so it *must* be a ref + Prio prio = jts[ci]->rule->prio ; + RuleIdx idx = ci ; + for( JobTgt jt : jts.subvec(ci) ) { + if (jt->rule->prio(ds) ; - } - - static void _append_dep( ::vector& deps , Dep const& dep , size_t& hole ) { + static void _append_dep( ::vector& deps , Dep const& dep , size_t& hole ) { bool can_compress = !dep.is_date && +dep.accesses && dep.crc()==Crc::None && !dep.dflags && !dep.parallel ; if (hole==Npos) { if (can_compress) { // create new open chunk - /**/ hole = deps.size() ; - Dep& hdr = deps.emplace_back() ; - /**/ hdr.sz = 1 ; - /**/ hdr.chunk_accesses = dep.accesses ; - /**/ static_cast(&deps.emplace_back())[0] = dep ; + /**/ hole = deps.size() ; + Dep& hdr = deps.emplace_back().hdr ; + /**/ hdr.sz = 1 ; + /**/ hdr.chunk_accesses = dep.accesses ; + /**/ deps.emplace_back().chunk[0] = dep ; } else { // create a chunk just for dep deps.push_back(dep) ; } } else { - Dep& hdr = deps[hole] ; + Dep& hdr = deps[hole].hdr ; if ( can_compress && dep.accesses==hdr.chunk_accesses && hdr.sz(&deps.back())[i] = dep ; + deps.back().chunk[i] = dep ; hdr.sz++ ; } else { // close chunk : copy dep to hdr, excetp sz and chunk_accesses fields uint8_t sz = hdr.sz ; @@ -746,40 +755,40 @@ namespace Engine { } } } - static void _fill_hole(Dep& hdr) { - SWEAR(hdr.sz!=0) ; - uint8_t sz = hdr.sz-1 ; - Accesses chunk_accesses = hdr.chunk_accesses ; - /**/ hdr = { static_cast(&hdr+1)[sz] , hdr.chunk_accesses , Crc::None } ; - /**/ hdr.sz = sz ; - /**/ hdr.chunk_accesses = chunk_accesses ; + static void _fill_hole(GenericDep& hdr) { + SWEAR(hdr.hdr.sz!=0) ; + uint8_t sz = hdr.hdr.sz-1 ; + Accesses chunk_accesses = hdr.hdr.chunk_accesses ; + /**/ hdr.hdr = { (&hdr)[1].chunk[sz] , hdr.hdr.chunk_accesses , Crc::None } ; + /**/ hdr.hdr.sz = sz ; + /**/ hdr.hdr.chunk_accesses = chunk_accesses ; } - static void _fill_hole( ::vector& deps , size_t hole ) { + static void _fill_hole( ::vector& deps , size_t hole ) { if (hole==Npos) return ; - Dep& d = deps[hole] ; + GenericDep& d = deps[hole] ; _fill_hole(d) ; - if (d.sz%Dep::NodesPerDep==0) deps.pop_back() ; + if (d.hdr.sz%GenericDep::NodesPerDep==0) deps.pop_back() ; } Deps::Deps(::vmap const& deps , Accesses accesses , bool parallel ) { - ::vector ds ; ds.reserve(deps.size()) ; // reserving deps.size() is comfortable and guarantees no reallocaiton - size_t hole = Npos ; + ::vector ds ; ds.reserve(deps.size()) ; // reserving deps.size() is comfortable and guarantees no reallocaiton + size_t hole = Npos ; for( auto const& [d,df] : deps ) _append_dep( ds , {d,accesses,df,parallel} , hole ) ; _fill_hole(ds,hole) ; *this = {ds} ; } Deps::Deps( ::vector const& deps , Accesses accesses , Dflags dflags , bool parallel ) { - ::vector ds ; ds.reserve(deps.size()) ; // reserving deps.size() is comfortable and guarantees no reallocaiton - size_t hole = Npos ; + ::vector ds ; ds.reserve(deps.size()) ; // reserving deps.size() is comfortable and guarantees no reallocaiton + size_t hole = Npos ; for( auto const& d : deps ) _append_dep( ds , {d,accesses,dflags,parallel} , hole ) ; _fill_hole(ds,hole) ; *this = {ds} ; } void Deps::assign(::vector const& deps) { - ::vector ds ; ds.reserve(deps.size()) ; // reserving deps.size() is comfortable and guarantees no reallocaiton - size_t hole = Npos ; + ::vector ds ; ds.reserve(deps.size()) ; // reserving deps.size() is comfortable and guarantees no reallocaiton + size_t hole = Npos ; for( auto const& d : deps ) _append_dep( ds , d , hole ) ; _fill_hole(ds,hole) ; DepsBase::assign(ds) ; @@ -787,25 +796,25 @@ namespace Engine { void Deps::replace_tail( DepsIter it , ::vector const& deps ) { // close current chunk - Dep* cur_dep = const_cast(it.hdr) ; + GenericDep* cur_dep = const_cast(it.hdr) ; if (it.i_chunk!=0) { - cur_dep->sz = it.i_chunk ; + cur_dep->hdr.sz = it.i_chunk ; _fill_hole(*cur_dep) ; cur_dep = cur_dep->next() ; } // create new tail - ::vector ds ; - size_t hole = Npos ; + ::vector ds ; + size_t hole = Npos ; for( auto const& d : deps ) _append_dep( ds , d , hole ) ; _fill_hole(ds,hole) ; // splice it NodeIdx tail_sz = cur_dep-items() ; if (ds.size()<=tail_sz) { - for( Dep const& d : ds ) *cur_dep++ = d ; // copy all - shorten_by(tail_sz-ds.size()) ; // and shorten + for( GenericDep const& d : ds ) *cur_dep++ = d ; // copy all + shorten_by(tail_sz-ds.size()) ; // and shorten } else { - for( Dep const& d : ::vector_view(ds.data(),tail_sz) ) *cur_dep++ = d ; // copy what can be fitted - append(::vector_view( &ds[tail_sz] , ds.size()-tail_sz ) ) ; // and append for the remaining + for( GenericDep const& d : ::vector_view(ds.data(),tail_sz) ) *cur_dep++ = d ; // copy what can be fitted + append(::vector_view( &ds[tail_sz] , ds.size()-tail_sz ) ) ; // and append for the remaining } } diff --git a/src/lmakeserver/node.x.hh b/src/lmakeserver/node.x.hh index f8560c53..8caefc30 100644 --- a/src/lmakeserver/node.x.hh +++ b/src/lmakeserver/node.x.hh @@ -151,12 +151,9 @@ namespace Engine { struct Dep : DepDigestBase { friend ::ostream& operator<<( ::ostream& , Dep const& ) ; using Base = DepDigestBase ; - static const uint8_t NodesPerDep ; // cxtors & casts using Base::Base ; // accesses - Dep const* next() const { return this+1+div_up(sz,NodesPerDep) ; } - Dep * next() { return this+1+div_up(sz,NodesPerDep) ; } ::string accesses_str() const ; ::string dflags_str () const ; // services @@ -164,7 +161,18 @@ namespace Engine { void acquire_crc() ; } ; static_assert(sizeof(Dep)==16) ; - inline constexpr uint8_t Dep::NodesPerDep = sizeof(Dep)/sizeof(Node) ; static_assert(sizeof(Dep)%sizeof(Node)==0) ; + + union GenericDep { + static constexpr uint8_t NodesPerDep = sizeof(Dep)/sizeof(Node) ; + // cxtors & casts + GenericDep(Dep const& d={}) : hdr{d} {} + // services + GenericDep const* next() const { return this+1+div_up(hdr.sz,GenericDep::NodesPerDep) ; } + GenericDep * next() { return this+1+div_up(hdr.sz,GenericDep::NodesPerDep) ; } + // data + Dep hdr = {} ; + Node chunk[NodesPerDep] ; + } ; // // Deps @@ -179,7 +187,7 @@ namespace Engine { // cxtors & casts DepsIter( ) = default ; DepsIter( DepsIter const& dit ) : hdr{dit.hdr} , i_chunk{dit.i_chunk} {} - DepsIter( Dep const* d ) : hdr{d } {} + DepsIter( GenericDep const* d ) : hdr{d } {} DepsIter( Deps , Digest ) ; // DepsIter& operator=(DepsIter const& dit) { @@ -196,40 +204,37 @@ namespace Engine { // Node's in chunk are semanticly located before header so : // - if i_chunk< hdr->sz : refer to dep with no crc, flags nor parallel // - if i_chunk==hdr->sz : refer to header - if (i_chunk==hdr->sz) return *hdr ; - static_cast(tmpl) = static_cast(hdr+1)[i_chunk] ; - tmpl.accesses = hdr->chunk_accesses ; + if (i_chunk==hdr->hdr.sz) return hdr->hdr ; + static_cast(tmpl) = hdr[1].chunk[i_chunk] ; + tmpl.accesses = hdr->hdr.chunk_accesses ; return tmpl ; } DepsIter& operator++(int) { return ++*this ; } DepsIter& operator++( ) { - if (i_chunksz) i_chunk++ ; // go to next item in chunk - else { i_chunk = 0 ; hdr = hdr->next() ; } // go to next chunk } + if (i_chunkhdr.sz) i_chunk++ ; // go to next item in chunk + else { i_chunk = 0 ; hdr = hdr->next() ; } // go to next chunk } return *this ; } // data - Dep const* hdr = nullptr ; // pointer to current chunk header - uint8_t i_chunk = 0 ; // current index in chunk - mutable Dep tmpl = {{}/*accesses*/,Crc::None} ; // template to store uncompressed Dep's + GenericDep const* hdr = nullptr ; // pointer to current chunk header + uint8_t i_chunk = 0 ; // current index in chunk + mutable Dep tmpl = {{}/*accesses*/,Crc::None} ; // template to store uncompressed Dep's } ; struct Deps : DepsBase { - friend ::ostream& operator<<( ::ostream& , Deps const& ) ; // cxtors & casts using DepsBase::DepsBase ; Deps( ::vmap const& , Accesses , bool parallel ) ; Deps( ::vector const& , Accesses , Dflags , bool parallel ) ; // accesses - Dep const& operator[](size_t i) const = delete ; // deps are compressed, cannot do random accesses - Dep & operator[](size_t i) = delete ; // . - NodeIdx size ( ) const = delete ; // . + NodeIdx size() const = delete ; // deps are compressed // services DepsIter begin() const { - Dep const* first = items() ; + GenericDep const* first = items() ; return {first} ; } DepsIter end() const { - Dep const* last1 = items()+DepsBase::size() ; + GenericDep const* last1 = items()+DepsBase::size() ; return {last1} ; } void assign ( ::vector const& ) ; @@ -416,9 +421,10 @@ namespace Engine { void mk_src (FileTag=FileTag::Err) ; // Err means no crc update void mk_no_src( ) ; // - ::c_vector_view prio_job_tgts (RuleIdx prio_idx) const ; - ::c_vector_view conform_job_tgts(ReqInfo const& ) const ; - ::c_vector_view conform_job_tgts( ) const ; + ::c_vector_view prio_job_tgts (RuleIdx prio_idx) const ; + ::c_vector_view conform_job_tgts (ReqInfo const& ) const ; + ::c_vector_view conform_job_tgts ( ) const ; + ::c_vector_view candidate_job_tgts( ) const ; // all jobs above prio provided in conform_idx // void set_buildable( Req={} , DepDepth lvl=0 ) ; // data independent, may be pessimistic (Maybe instead of Yes), req is for error reporing only void set_pressure ( ReqInfo& , CoarseDelay pressure ) const ; diff --git a/src/lmakeserver/req.cc b/src/lmakeserver/req.cc index d349630a..b7c6e417 100644 --- a/src/lmakeserver/req.cc +++ b/src/lmakeserver/req.cc @@ -175,7 +175,7 @@ namespace Engine { for( Job j : d->conform_job_tgts(d->c_req_info(*this)) ) { // 2nd pass to find the loop JobReqInfo const& cjri = j->c_req_info(*this) ; if (cjri.done() ) continue ; - if (cjri.speculative_deps) to_forget.push_back(d) ; + if (cjri.speculative_wait) to_forget.push_back(d) ; for( Node dd : j->deps ) { if (dd->done(*this)) continue ; d = dd ; diff --git a/src/lmakeserver/store.x.hh b/src/lmakeserver/store.x.hh index 989b8e65..a9fe5b1a 100644 --- a/src/lmakeserver/store.x.hh +++ b/src/lmakeserver/store.x.hh @@ -45,17 +45,17 @@ namespace Engine { struct StoreMrkr {} ; // just a marker to disambiguate file association - struct Dep ; - struct Target ; + union GenericDep ; + struct Target ; extern SeqId* g_seq_id ; // used to identify launched jobs. Persistent so that we keep as many old traces as possible } namespace Engine { - namespace Persistent { using RuleStr = Vector::Simple ; } - /**/ using DepsBase = Vector::Simple ; - /**/ using TargetsBase = Vector::Simple ; + namespace Persistent { using RuleStr = Vector::Simple ; } + /**/ using DepsBase = Vector::Simple ; + /**/ using TargetsBase = Vector::Simple ; } #endif @@ -261,22 +261,22 @@ namespace Engine::Persistent { Targets no_triggers ; // these nodes do not trigger rebuild } ; - // autolock header index key data misc + // autolock header index key data misc // jobs - using JobFile = Store::AllocFile < false , JobHdr , Job , JobData > ; - using DepsFile = Store::VectorFile < false , void , Deps , Dep , NodeIdx , 4 > ; // Deps are compressed when Crc==None - using TargetsFile = Store::VectorFile < false , void , Targets , Target > ; + using JobFile = Store::AllocFile < false , JobHdr , Job , JobData > ; + using DepsFile = Store::VectorFile < false , void , Deps , GenericDep , NodeIdx , 4 > ; // Deps are compressed when Crc==None + using TargetsFile = Store::VectorFile < false , void , Targets , Target > ; // nodes - using NodeFile = Store::AllocFile < false , NodeHdr , Node , NodeData > ; - using JobTgtsFile = Store::VectorFile < false , void , JobTgts::Vector , JobTgt , RuleIdx > ; + using NodeFile = Store::AllocFile < false , NodeHdr , Node , NodeData > ; + using JobTgtsFile = Store::VectorFile < false , void , JobTgts::Vector , JobTgt , RuleIdx > ; // rules - using RuleStrFile = Store::VectorFile < false , void , RuleStr , char , uint32_t > ; - using RuleFile = Store::AllocFile < false , MatchGen , Rule , RuleStr > ; - using RuleTgtsFile = Store::SinglePrefixFile< false , void , RuleTgts , RuleTgt , void , true /*Reverse*/ > ; - using SfxFile = Store::SinglePrefixFile< false , void , PsfxIdx , char , PsfxIdx , true /*Reverse*/ > ; // map sfxes to root of pfxes, no lock : static - using PfxFile = Store::MultiPrefixFile < false , void , PsfxIdx , char , RuleTgts , false/*Reverse*/ > ; + using RuleStrFile = Store::VectorFile < false , void , RuleStr , char , uint32_t > ; + using RuleFile = Store::AllocFile < false , MatchGen , Rule , RuleStr > ; + using RuleTgtsFile = Store::SinglePrefixFile< false , void , RuleTgts , RuleTgt , void , true /*Reverse*/ > ; + using SfxFile = Store::SinglePrefixFile< false , void , PsfxIdx , char , PsfxIdx , true /*Reverse*/ > ; // map sfxes to root of pfxes, no lock : static + using PfxFile = Store::MultiPrefixFile < false , void , PsfxIdx , char , RuleTgts , false/*Reverse*/ > ; // commons - using NameFile = Store::SinglePrefixFile< true , void , Name , char , JobNode > ; // for Job's & Node's + using NameFile = Store::SinglePrefixFile< true , void , Name , char , JobNode > ; // for Job's & Node's static constexpr char StartMrkr = 0x0 ; // used to indicate a single match suffix (i.e. a suffix which actually is an entire file name) diff --git a/src/lshow.cc b/src/lshow.cc index ae8d5a84..a12d1bfa 100644 --- a/src/lshow.cc +++ b/src/lshow.cc @@ -13,13 +13,15 @@ int main( int argc , char* argv[] ) { Trace trace("main") ; // ReqSyntax syntax{{ - { ReqKey::Cmd , { .short_name='c' , .doc="show cmd" } } + { ReqKey::Bom , { .short_name='b' , .doc="show necessary sources" } } + , { ReqKey::Cmd , { .short_name='c' , .doc="show cmd" } } , { ReqKey::Deps , { .short_name='d' , .doc="show existing deps" } } , { ReqKey::Env , { .short_name='E' , .doc="show envionment variables to execute job" } } , { ReqKey::ExecScript , { .short_name='s' , .doc="show a sh-executable script" } } , { ReqKey::Info , { .short_name='i' , .doc="show info about jobs leading to files" } } , { ReqKey::InvDeps , { .short_name='D' , .doc="show dependents" } } , { ReqKey::InvTargets , { .short_name='T' , .doc="show producing jobs" } } + , { ReqKey::Running , { .short_name='r' , .doc="show running jobs" } } , { ReqKey::Stderr , { .short_name='e' , .doc="show stderr" } } , { ReqKey::Stdout , { .short_name='o' , .doc="show stdout" } } , { ReqKey::Targets , { .short_name='t' , .doc="show targets of jobs leading to files" } } @@ -31,7 +33,15 @@ int main( int argc , char* argv[] ) { }} ; ReqCmdLine cmd_line{syntax,argc,argv} ; // - bool may_verbose = cmd_line.key==ReqKey::Deps || cmd_line.key==ReqKey::Targets || cmd_line.key==ReqKey::Stderr ; + bool may_verbose = false ; + switch (cmd_line.key) { + case ReqKey::Bom : + case ReqKey::Deps : + case ReqKey::Targets : + case ReqKey::Running : + case ReqKey::Stderr : may_verbose = true ; + default : ; + } // if ( cmd_line.key==ReqKey::ExecScript && cmd_line.args.size()!=1 ) syntax.usage("must have a single argument to generate an executable script") ; if ( cmd_line.flags[ReqFlag::Verbose ] && !may_verbose ) syntax.usage("verbose is only for showing deps, targets or stderr" ) ; diff --git a/src/rpc_client.hh b/src/rpc_client.hh index 7097f28d..a3a93bca 100644 --- a/src/rpc_client.hh +++ b/src/rpc_client.hh @@ -25,6 +25,7 @@ ENUM_1( ReqProc // PER_CMD : add a value that represents your command, above ENUM( ReqKey // PER_CMD : add key as necessary (you may share with other commands) : there must be a single key on the command line , None // must stay first , Add // if proc==Mark +, Bom // if proc==Show , Clear // if proc==Mark , Cmd // if proc==Show , Delete // if proc==Mark @@ -37,6 +38,7 @@ ENUM( ReqKey // PER_CMD : add key as necessary (you may share with other comma , InvTargets // if proc==Show , List // if proc==Mark , Resources // if proc==Forget, redo everything that were not redone when resources changed, to ensure reproducibility +, Running // if proc==Show , Stderr // if proc==Show , Stdout // if proc==Show , Targets // if proc==Show diff --git a/src/rpc_job.hh b/src/rpc_job.hh index 6636b05f..c5116d30 100644 --- a/src/rpc_job.hh +++ b/src/rpc_job.hh @@ -237,38 +237,38 @@ ENUM( MatchKind , SideDeps ) -ENUM_3( Status // result of job execution -, Early = EarlyLostErr // <=Early means output has not been modified -, Async = Killed // <=Async means job was interrupted asynchronously -, Garbage = BadTarget // <=Garbage means job has not run reliably -, New // job was never run -, EarlyChkDeps // dep check failed before job actually started -, EarlyErr // job was not started because of error -, EarlyLost // job was lost before starting , retry -, EarlyLostErr // job was lost before starting , do not retry -, LateLost // job was lost after having started, retry -, LateLostErr // job was lost after having started, do not retry -, Killed // job was killed -, ChkDeps // dep check failed -, BadTarget // target was not correctly initialized or simultaneously written by another job -, Ok // job execution ended successfully -, Err // job execution ended in error +ENUM_3( Status // result of job execution +, Early = EarlyLostErr // <=Early means output has not been modified +, Async = Killed // <=Async means job was interrupted asynchronously +, Garbage = BadTarget // <=Garbage means job has not run reliably +, New // job was never run +, EarlyChkDeps // dep check failed before job actually started +, EarlyErr // job was not started because of error +, EarlyLost // job was lost before starting , retry +, EarlyLostErr // job was lost before starting , do not retry +, LateLost // job was lost after having started, retry +, LateLostErr // job was lost after having started, do not retry +, Killed // job was killed +, ChkDeps // dep check failed +, BadTarget // target was not correctly initialized or simultaneously written by another job +, Ok // job execution ended successfully +, Err // job execution ended in error ) inline bool is_lost(Status s) { return s<=Status::LateLostErr && s>=Status::EarlyLost ; } inline Bool3 is_ok (Status s) { static constexpr Bool3 IsOkTab[] = { - Maybe // New - , Maybe // EarlyChkDeps - , No // EarlyErr - , Maybe // EarlyLost - , No // EarlyLostEr - , Maybe // LateLost - , No // LateLostErr - , Maybe // Killed - , Maybe // ChkDeps - , Maybe // BadTarget - , Yes // Ok - , No // Err + Maybe // New + , Maybe // EarlyChkDeps + , No // EarlyErr + , Maybe // EarlyLost + , No // EarlyLostErr + , Maybe // LateLost + , No // LateLostErr + , Maybe // Killed + , Maybe // ChkDeps + , Maybe // BadTarget + , Yes // Ok + , No // Err } ; static_assert(sizeof(IsOkTab)==N) ; return IsOkTab[+s] ; @@ -281,26 +281,6 @@ inline Status mk_err(Status s) { case Status::Ok : return Status::Err ; DF} } -inline JobReasonTag mk_reason(Status s) { - static constexpr JobReasonTag ReasonTab[] = { - JobReasonTag::New // New - , JobReasonTag::ChkDeps // EarlyChkDeps - , JobReasonTag::None // EarlyErr - , JobReasonTag::Lost // EarlyLost - , JobReasonTag::Lost // EarlyLostEr - , JobReasonTag::Lost // LateLost - , JobReasonTag::Lost // LateLostErr - , JobReasonTag::Killed // Killed - , JobReasonTag::ChkDeps // ChkDeps - , JobReasonTag::PollutedTargets // BadTarget - , JobReasonTag::None // Ok - , JobReasonTag::None // Err - } ; - static_assert(sizeof(ReasonTab)==N) ; - JobReasonTag res = ReasonTab[+s] ; - SWEAR( res=Tag::HasNode && +n , t , n ) ; } // accesses - bool operator+() const { return +tag ; } - bool operator!() const { return !tag ; } - bool need_run () const { return +tag && tag=JobReasonTagPrios[+jr.tag]) return *this ; // at equal level, prefer older reason @@ -383,7 +362,7 @@ struct CrcDate { Crc crc () const { SWEAR(!is_date) ; return _crc ; } Ddate date () const { SWEAR( is_date) ; return _date ; } // - bool seen(Accesses a) const { // return true if accesses could perceive the existence of file + bool seen(Accesses a) const { // return true if accesses could perceive the existence of file if (!a) return false ; SWEAR(+*this,*this,a) ; if (is_date) return +_date && !Crc::None.match( Crc(_date.tag()) , a ) ; @@ -393,8 +372,8 @@ struct CrcDate { bool is_date = false ; private : union { - Crc _crc ; // ~46<64 bits - Ddate _date ; // ~45<64 bits + Crc _crc ; // ~46<64 bits + Ddate _date ; // ~45<64 bits } ; } ; diff --git a/unit_tests/misc10.py b/unit_tests/misc10.py new file mode 100644 index 00000000..dde68d7c --- /dev/null +++ b/unit_tests/misc10.py @@ -0,0 +1,56 @@ +# This file is part of the open-lmake distribution (git@github.com:cesar-douady/open-lmake.git) +# Copyright (c) 2023 Doliam +# This program is free software: you can redistribute/modify under the terms of the GPL-v3 (https://www.gnu.org/licenses/gpl-3.0.html). +# This program is distributed WITHOUT ANY WARRANTY, without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +if __name__!='__main__' : + + import sys + import time + + import lmake + from lmake.rules import Rule + + lmake.manifest = ('Lmakefile.py',) + + class BaseRule(Rule) : + start_delay = 0 + + class Good(BaseRule) : + target = 'good' + cmd = 'echo good_content' + + class Bad(BaseRule) : + targets = { 'Bad' : 'bad1{*:}' } + cmd = 'sleep 3' # dont produce bad + + class Ptr(BaseRule) : + target = 'ptr' + cmd = ' echo good ; sleep 2 ' + + class Dut(BaseRule) : + target = 'dut' + allow_stderr = True + cmd = ''' + deps="$( cat ptr 2>/dev/null || echo bad1 bad2 )" + ldepend $deps # makes deps required + echo $deps $(cat $deps) >&2 + echo $(cat $deps) + sleep 2 + ''' + + class Side(BaseRule) : + target = 'side' + cmd = ' sleep 3 ; cat ptr good ' + + class chk(BaseRule) : + target = 'chk' + deps = { 'DUT':'dut' , 'SIDE':'side' } + cmd = '[ "$(cat {DUT})" = good_content ]' + +else : + + import ut + +# ut.lmake( 'bad' , failed=1 , rc=1 ) + ut.lmake( 'chk' , done=5 , may_rerun=2 , steady=1 ) diff --git a/unit_tests/wine.py b/unit_tests/wine.py index ec8a3d01..26ed7116 100644 --- a/unit_tests/wine.py +++ b/unit_tests/wine.py @@ -33,17 +33,17 @@ class WineRule(Rule) : class WineInit(WineRule) : target = '.wine/init' - targets = { 'WINE' : '.wine/{*:.*}' } # for init wine env is not incremental + targets = { 'WINE' : '.wine/{*:.*}' } # for init wine env is not incremental side_targets = { 'WINE' : None } allow_stderr = True - cmd = 'wine64 cmd' # do nothing, just to init support files (in targets) + cmd = 'wine64 cmd >$TMPDIR/out ; cat $TMPDIR/out' # do nothing, just to init support files (in targets), avoid waiting for stdout class Dut(Base,WineRule) : target = 'dut.{Method}' deps = { 'WINE_INIT' : '.wine/init' } autodep = '{Method}' - allow_stderr = True # in some systems, there are fixme messages - cmd = f'wine64 {hostname_exe}' + allow_stderr = True # in some systems, there are fixme messages + cmd = f'wine64 {hostname_exe} > $TMPDIR/out ; cat $TMPDIR/out' # avoid waiting for stdout class Chk(Base) : target = r'test.{Method}'