implemented lshow -r (running) & -b (bom)

.gitignore

@@ -14,8 +14,9 @@ __pycache__
 
 #auto-configuration
 /trial/
+/sys_config.log
+/sys_config.mk
 /sys_config.h
-/sys_config.h.err
 /version.hh
 
 # compiled files

Manifest

@@ -185,6 +185,7 @@ unit_tests/mandelbrot.py
 unit_tests/mandelbrot.zip
 unit_tests/manual_target.py
 unit_tests/misc1.py
+unit_tests/misc10.py
 unit_tests/misc2.py
 unit_tests/misc3.py
 unit_tests/misc4.py

TO_DO

@@ -74,6 +74,12 @@ items :
 * ROBUSTNESS & MAINTENABILITY (fragile/difficult to read or maintain as long as not implemented)
 ****************************************************************************************************
 
+* close socket in the management thread (M) while request is being processed
+	- such requests must go through the engine_loop
+	- this may take a while
+	- this means during this time, all jobs may connect (and actually do)
+	- requiring a coket for each slurm slot
+	- defeating the purpose of disconnecting jobs during execution
 *	fix store to be compliant with strict aliasing rules
 * support 64-bits id
 	- configure with NBits rather than types

doc/lmake.texi

@@ -23,8 +23,8 @@
 @end macro
 
 @set TITLE The lmake Manual
-@set UPDATED 22 April 2023
-@set UPDATED-MONTH April 2023
+@set UPDATED 22 April 2024
+@set UPDATED-MONTH April 2024
 @set EDITION @dfn{work in progress}
 @set VERSION 0.1
 
@@ -594,16 +594,22 @@ Once the build process is complete, a summary is generated with :
 
 @section @code{lshow}
 
-Usage : @code{lshow [ [-d|--deps] | [-D|-inv-deps] | [-T|-inv-targets] | [-i|--info] | [-s|--script] | [-S|--exec-script] | [-E|--env] | [-e|--stderr] |[-o|--stdout] [-b|--backend]
-[[-d|--debug] debug_dir] [-t|--targets] [-v|--verbose] [-p|--porcelaine] targets...}
+Usage : @code{lshow -short_cmd|--long_cmd [ -short_option|--long_option option_arg ]* files or job}
 
 @code{lshow} provides various information about jobs.
 The jobs are those officially generating the targets (i.e. the jobs that would be selected by @lmake if needing to generate it) if any,
 else it may be the jobs that actually generated the target.
 
-Options :
+There must be a single command while there may be 0 or more options.
+
+Arguments can be files or a single job as described in generic arguments description.
+
+Commands :
 @itemize @bullet
-@item @code{-d}|@code{--deps} : output the list of all the deps of the jobs.
+@item @code{-b}|@code{--bom} : Output the list of all source files necessary to build the arguments.
+If verbose (option @code{-v} or @code{--verbose}), some intermediate files are shown in gray, enough to justify why all listed source files.
+@item @code{-c}|@code{--cmd} : Output the @code{cmd} used to execute the job. If dynamic, it shows the specialized @code{cmd} for this job.
+@item @code{-d}|@code{--deps} : Output the list of all the deps of the jobs.
 Unless the verbose flag is specified, only existing deps are shown.
 Each line is composed of 6 fields  separated by spaces :
 @itemize @minus
@@ -626,18 +632,19 @@ Each line is composed of 6 fields  separated by spaces :
 @item Name of the dep.
 @end itemize
 If a dep does not exist, it is deemed secondary information and is shown in gray.
-@item @code{-D}|@code{--inv-deps}    : show jobs that depend on targets.
-@item @code{-T}|@code{--inv-targets} : show jobs that produce targets either officially (listed in @code{targets} attribute) or not (listed in @code{side_targets} attribute or not)
-@item @code{-i}|@code{--info}        : show various self-reading info about jobs, such as reason to be launched, why it was needed, execution time, host that executed it, ...
+@item @code{-D}|@code{--inv-deps} : Show jobs that depend on targets.
+@item @code{-E}|@code{--env}      : Show the environment used to run the script
+@item @code{-i}|@code{--info}     : Show various self-reading info about jobs, such as reason to be launched, why it was needed, execution time, host that executed it, ...
 If porcelaine mode (using the @code{--porcelaine} or @code{-p} option), the same information is reported as as an easy to parse Python @code{dict}.
-@item @code{-s}|@code{--script} : show the scripts used to execute the job.
+@item @code{-r}|@code{--running}  : Show the list of jobs currently running to build the arguments.
+If verbose (option @code{-v} or @code{--verbose}), some waiting jobs are shown in gray, enough to justify why all running jobs are running.
+Queued jobs are shown in blue, actively running jobs are uncolored.
+@item @code{-s}|@code{--script}   : Show the scripts used to execute the job.
 If debug is required through the @code{-d} or @code{--debug} option, a dir must be passed and generated script will write info to it using the autodep mechanism.
 Note that if job requires tmp mapping or auto mkdir, autodep is used in all cases (as it is functionally required) although minimally configured if debug is not asked.
-@item @code{-E}|@code{--env}     : show the environment used to run the script
-@item @code{-e}|@code{--stderr}  : show the stderr of the jobs.
-@item @code{-o}|@code{--stdout}  : show the stdout of the jobs.
-@item @code{-b}|@code{--backend} : show some execution information from backend generated when @lmake is run with the @code{-v} flag.
-@item @code{-t}|@code{--targets} : show the targets of the jobs.
+@item @code{-e}|@code{--stderr}   : Show the stderr of the jobs.
+@item @code{-o}|@code{--stdout}   : Show the stdout of the jobs.
+@item @code{-t}|@code{--targets}  : Show the targets of the jobs.
 Unless the verbose flag is specified, only existing targets are shown.
 Each line is composed of 3 fields  separated by spaces :
 @itemize @minus
@@ -655,6 +662,7 @@ Each line is composed of 3 fields  separated by spaces :
 @item The name of the target.
 @end itemize
 If a target does not exist, it is deemed secondary information and is shown in gray.
+@item @code{-T}|@code{--inv-targets} : Show jobs that produce targets either officially (listed in @code{targets} attribute) or not (listed in @code{side_targets} attribute or not)
 @end itemize
 
 @section @code{ldebug}

lmake_env/Lmakefile.py

@@ -69,8 +69,8 @@ class BaseRule(Rule) :
 	start_delay = 2
 	n_tokens    = config.backends.local.cpu
 
-class Centos7Rule(BaseRule) :
-	environ_cmd = { 'PATH' : '...:/opt/rh/devtoolset-11/root/usr/bin' }
+class PathRule(BaseRule) :                       # compiler must be accessed using the PATH as it must find its sub-binaries
+	environ_cmd = { 'PATH' : os.getenv('PATH') }
 	cache       = 'dir'
 
 class Html(BaseRule) :
@@ -140,14 +140,14 @@ class ConfigH(BaseRule) :
 	deps         = { 'CONFIGURE'  : 'ext/{DirS}configure' }
 	cmd          = 'cd ext/{DirS} ; ./configure'
 
-class SysConfigH(Centos7Rule) :
+class SysConfigH(PathRule) :
     targets = {
 		'MK'    : 'sys_config.mk'
 	,	'H'     : 'sys_config.h'
 	,	'TRIAL' : 'trial/{*:.*}'
 	}
     deps = { 'EXE' : '_bin/sys_config' }
-    cmd  = 'CC={gxx} PYTHON={sys.executable} ./{EXE} {MK} {H} 2>&1'
+    cmd  = 'CXX={gxx} PYTHON={sys.executable} ./{EXE} {MK} {H} 2>&1'
 
 class VersionH(BaseRule) :
     target = 'version.hh'
@@ -184,12 +184,12 @@ def cmd() :
 }
 def run_gxx(target,*args) :
 		cmd_line = ( gxx , '-o' , target , '-fdiagnostics-color=always' , *args )
-		if '/' in gxx : os.environ['PATH'] = ':'.join((osp.dirname(gxx),os.environ['PATH']))                            # gxx calls its subprograms (e.g. as) using PATH, ensure it points to gxx dir
+		if '/' in gxx : os.environ['PATH'] = ':'.join((osp.dirname(gxx),os.environ['PATH'])) # gxx calls its subprograms (e.g. as) using PATH, ensure it points to gxx dir
 		for k,v in os.environ.items() : print(f'{k}={v}')
 		print(' '.join(cmd_line))
 		run( cmd_line , check=True )
 for ext,basic_opts in basic_opts_tab.items() :
-	class Compile(Centos7Rule) :                                         # note that although class is overwritten at each iteration, each is recorded at definition time by the metaclass
+	class Compile(PathRule) :                                            # note that although class is overwritten at each iteration, each is recorded at definition time by the metaclass
 		name    = f'compile {ext}'
 		targets = { 'OBJ' : '{File}.o' }
 		deps    = {
@@ -223,7 +223,7 @@ def cmd() :
 		if True             : resources.mem = '512M'
 		if backend=='local' : resources.cc  = 1
 
-class GxxRule(Centos7Rule) :
+class LinkRule(PathRule) :
 	combine       = ('pre_opts','rev_post_opts')
 	pre_opts      = []                           # options before inputs & outputs
 	rev_post_opts = []                           # options after  inputs & outputs, combine appends at each level, but here we want to prepend
@@ -234,14 +234,14 @@ def cmd() :
 		,	*reversed(rev_post_opts)
 		)
 
-class LinkO(GxxRule) :
+class LinkO(LinkRule) :
 	pre_opts = ('-r','-fPIC')
 
-class LinkSo(GxxRule) :
+class LinkSo(LinkRule) :
 	pre_opts      = ('-shared-libgcc','-shared','-pthread')
 	rev_post_opts = ()
 
-class LinkExe(GxxRule) :
+class LinkExe(LinkRule) :
 	pre_opts      = '-pthread'
 	rev_post_opts = ()
 

src/lmakeserver/cmd.cc

@@ -505,6 +505,97 @@ R"({
 		throw "no mark specified"s ;
 	}
 
+	template<class T> struct Show {
+		// cxtors & casts
+		Show(                                                  ) = default ;
+		Show( Fd fd_ , ReqOptions const& ro_ , DepDepth lvl_=0 ) : fd{fd_} , ro{&ro_} , lvl{lvl_} , verbose{ro_.flags[ReqFlag::Verbose]} {}
+		// data
+		Fd                fd        ;
+		ReqOptions const* ro        = nullptr          ;
+		DepDepth          lvl       = 0                ;
+		::uset<Job >      job_seen  = {}               ;
+		::uset<Node>      node_seen = {}               ;
+		::vector<T>       backlog   = {}               ;
+		bool              verbose   = false/*garbage*/ ;
+	} ;
+
+	struct ShowBom : Show<Node> {
+		using Show<Node>::Show ;
+		// services
+		void show_job(Job job) {
+			if (!job_seen.insert(job).second) return ;
+			for ( Dep const& dep : job->deps ) show_node(dep) ;
+		}
+		void show_node(Node node) {
+			if (!node_seen.insert(node).second) return ;
+			node->set_buildable() ;
+			//
+			if (!node->is_src_anti()) {
+				if (verbose) backlog.push_back(node) ;
+				lvl += verbose ;
+				for( Job j : node->candidate_job_tgts() ) show_job(j) ;
+				lvl -= verbose ;
+				if (+backlog) backlog.pop_back() ;
+			} else if (node->status()<=NodeStatus::Makable) {
+				Color c = node->buildable==Buildable::Src ? Color::None : Color::Warning ;
+				DepDepth l = lvl - backlog.size() ;
+				for( Node n : backlog ) audit( fd , *ro , Color::HiddenNote , mk_file(n   ->name()) , false/*as_is*/ , l++ ) ;
+				/**/                    audit( fd , *ro , c                 , mk_file(node->name()) , false/*as_is*/ , lvl ) ;
+				backlog = {} ;
+			}
+		}
+	} ;
+
+	struct ShowRunning : Show<Job> {
+		static constexpr BitMap<JobStep> InterestingSteps = { JobStep::Dep/*waiting*/ , JobStep::Queued , JobStep::Exec } ;
+		using Show<Job>::Show ;
+		// services
+		void show_job(Job job) {
+			JobStep step = {} ;
+			for( Req r : Req::s_reqs_by_start )
+				if ( JobStep s = job->c_req_info(r).step() ; InterestingSteps[s] && step!=s ) { // process job as soon as one Req is waiting/running, and this must be coherent
+					SWEAR(!step,step,s) ;
+					step = s ;
+				}
+			Color color = {} /*garbage*/ ;
+			char  hdr   = '?'/*garbage*/ ;
+			switch (step) {
+				case JobStep::Dep    :                                   break ;
+				case JobStep::Queued : color = Color::Note ; hdr = 'Q' ; break ;
+				case JobStep::Exec   : color = Color::None ; hdr = 'R' ; break ;
+				default : return ;
+			}
+			if (!job_seen.insert(job).second) return ;
+			//
+			switch (step) {
+				case JobStep::Dep    :
+					if (verbose) backlog.push_back(job) ;
+				break ;
+				case JobStep::Queued :
+				case JobStep::Exec   : {
+					SWEAR( lvl>=backlog.size() , lvl , backlog.size() ) ;
+					DepDepth l = lvl - backlog.size() ;
+					for( Job j : backlog ) audit( fd , *ro , Color::HiddenNote , to_string('W',' ',j  ->rule->name,' ',mk_file(j  ->name())) , false/*as_is*/ , l++ ) ;
+					/**/                   audit( fd , *ro , color             , to_string(hdr,' ',job->rule->name,' ',mk_file(job->name())) , false/*as_is*/ , lvl ) ;
+					backlog = {} ;
+					return ;
+				}
+			DF}
+			lvl += verbose ;
+			for( Dep const& dep : job->deps ) show_node(dep) ;
+			lvl -= verbose ;
+			if (+backlog) backlog.pop_back() ;
+		}
+		void show_node(Node node) {
+			for( Req r : Req::s_reqs_by_start )
+				if ( NodeReqInfo const& cri = node->c_req_info(r) ; cri.waiting() ) {           // process node as soon as one Req is waiting
+					if (!node_seen.insert(node).second) return ;
+					for( Job j : node->conform_job_tgts(cri) ) show_job(j) ;
+					return ;
+				}
+		}
+	} ;
+
 	static void _show_job( Fd fd , ReqOptions const& ro , Job job , DepDepth lvl=0 ) {
 		Trace trace("show_job",ro.key,job) ;
 		Rule             rule         = job->rule                  ;
@@ -522,12 +613,13 @@ R"({
 			case ReqKey::Stdout     : {
 				if (rule->is_special()) {
 					switch (ro.key) {
-						case ReqKey::Info :
+						case ReqKey::Info   :
 						case ReqKey::Stderr : {
 							_send_job( fd , ro , No/*show_deps*/ , false/*hide*/ , job  , lvl   ) ;
 							audit    ( fd , ro , job->special_stderr() , false/*as_is*/ , lvl+1 ) ;
 						} break ;
 						case ReqKey::Cmd        :
+						case ReqKey::Env        :
 						case ReqKey::ExecScript :
 						case ReqKey::Stdout     :
 							_send_job( fd , ro , No/*show_deps*/ , false/*hide*/ , job                                    , lvl   ) ;
@@ -560,8 +652,8 @@ R"({
 						break ;
 						case ReqKey::Stdout :
 							if (!has_end) { audit( fd , ro , Color::Err , "no info available" , true/*as_is*/ , lvl ) ; break ; }
-							_send_job( fd , ro , No/*show_deps*/ , false/*hide*/ , job , lvl ) ;
-							audit    ( fd , ro , digest.stdout , lvl+1 )         ;
+							_send_job( fd , ro , No/*show_deps*/ , false/*hide*/ , job , lvl   ) ;
+							audit    ( fd , ro , digest.stdout                         , lvl+1 ) ;
 						break ;
 						case ReqKey::Stderr :
 							if (!has_end && !(has_start&&verbose) ) { audit( fd , ro , Color::Err , "no info available" , true/*as_is*/ , lvl ) ; break ; }
@@ -727,9 +819,9 @@ R"({
 					DF}
 				}
 			} break ;
-			case ReqKey::Deps :
-				_send_job( fd , ro , Maybe|verbose , false/*hide*/ , job , lvl ) ;
-			break ;
+			case ReqKey::Bom     : ShowBom    (fd,ro,lvl).show_job(job) ;                             break ;
+			case ReqKey::Running : ShowRunning(fd,ro,lvl).show_job(job) ;                             break ;
+			case ReqKey::Deps    : _send_job( fd , ro , Maybe|verbose , false/*hide*/ , job , lvl ) ; break ;
 			case ReqKey::Targets : {
 				Rule::SimpleMatch match = job->simple_match() ;
 				for( auto const& [tn,td] : digest.targets ) {
@@ -759,6 +851,11 @@ R"({
 		::vector<Node> targets    = ecr.targets()                 ;
 		bool           porcelaine = ro.flags[ReqFlag::Porcelaine] ;
 		char           sep        = ' '                           ;
+		switch (ro.key) {
+			case ReqKey::Bom     : { ShowBom     sb {fd,ro} ; for( Node t : targets ) sb.show_node(t) ; goto Return ; }
+			case ReqKey::Running : { ShowRunning sr {fd,ro} ; for( Node t : targets ) sr.show_node(t) ; goto Return ; }
+			default : ;
+		}
 		if (porcelaine) audit( fd , ro , "{" , true/*as_is*/ ) ;
 		for( Node target : targets ) {
 			trace("target",target) ;
@@ -767,26 +864,35 @@ R"({
 			else if (targets.size()>1) _send_node( fd , ro , true/*always*/ , Maybe/*hide*/ , {} , target                                                          ) ;
 			else                       lvl-- ;
 			sep = ',' ;
-			bool for_job = false/*garbage*/ ;
+			bool for_job = true ;
 			switch (ro.key) {
 				case ReqKey::InvDeps    :
-				case ReqKey::InvTargets : for_job = false ; break ;
-				default                 : for_job = true  ;
+				case ReqKey::InvTargets :
+				case ReqKey::Running    : for_job = false ; break ;
+				default                 : ;
 			}
 			Job job ;
 			if (for_job) {
 				job = _job_from_target(fd,ro,target) ;
 				if ( !job && ro.key!=ReqKey::Info ) { ok = false ; continue ; }
 			}
 			switch (ro.key) {
+				case ReqKey::Cmd        :
+				case ReqKey::Env        :
+				case ReqKey::ExecScript :
+				case ReqKey::Stderr     :
+				case ReqKey::Stdout     :
+				case ReqKey::Targets    :
+					_show_job(fd,ro,job,lvl) ;
+				break ;
 				case ReqKey::Info :
 					if (target->status()==NodeStatus::Plain) {
 						Job    cj             = target->conform_job() ;
 						size_t w              = 0                     ;
 						bool   seen_candidate = false                 ;
 						for( Job j : target->conform_job_tgts() ) {
-							/**/       w              = ::max(w,j->rule->name.size()) ;
-							if (j!=cj) seen_candidate = true                          ;
+							w               = ::max(w,j->rule->name.size()) ;
+							seen_candidate |= j!=cj                         ;
 						}
 						for( Job j : target->conform_job_tgts() ) {
 							if      (j==job         ) continue ;
@@ -798,17 +904,12 @@ R"({
 					if (!job) {
 						Node n = target ;
 						while ( +n->asking && n->asking.is_a<Node>() ) n = Node(n->asking) ;
-						if (+n->asking) audit( fd , ro , to_string("required by : ",mk_file(Job(n->asking)->name())) ) ;
-						else            audit( fd , ro , to_string("required by : ",mk_file(    n         ->name())) ) ;
+						if (n!=target) {
+							if (+n->asking) audit( fd , ro , to_string("required by : ",mk_file(Job(n->asking)->name())) ) ;
+							else            audit( fd , ro , to_string("required by : ",mk_file(    n         ->name())) ) ;
+						}
 						continue ;
 					}
-				[[fallthrough]] ;
-				case ReqKey::Cmd        :
-				case ReqKey::Env        :
-				case ReqKey::ExecScript :
-				case ReqKey::Stderr     :
-				case ReqKey::Stdout     :
-				case ReqKey::Targets    :
 					_show_job(fd,ro,job,lvl) ;
 				break ;
 				case ReqKey::Deps    : {
@@ -840,6 +941,7 @@ R"({
 			DF}
 		}
 		if (porcelaine) audit( fd , ro , "}" , true/*as_is*/ ) ;
+	Return :
 		trace(STR(ok)) ;
 		return ok ;
 	}