fixed optional target flag + fixed Python2 support + fixed ldebug + i…

…mproved build

Makefile

@@ -11,8 +11,12 @@ MAKEFLAGS += -r -R
 
 .DEFAULT_GOAL := DFLT
 
-sys_config.log : _bin/sys_config
-	./$< $(@:%.log=%.mk) $(@:%.log=%.h) 2>$@ || cat $@
+$(shell { echo CXX=$$CXX ; echo PYTHON2=$$PYTHON2 ; echo PYTHON=$$PYTHON ; } >sys_config_env.tmp                                                     )
+$(shell cmp sys_config_env sys_config_env.tmp 2>/dev/null || { cp sys_config_env.tmp sys_config_env ; echo new env : >&2 ; cat sys_config_env >&2 ; })
+$(shell rm -f sys_config_env.tmp                                                                                                                     )
+
+sys_config.log : _bin/sys_config sys_config_env
+	. ./sys_config_env ; ./$< $(@:%.log=%.mk) $(@:%.log=%.h) 2>$@ || cat $@
 sys_config.mk : sys_config.log ;+@[ -f $@ ] || { echo "cannot find $@" ; exit 1 ; }
 sys_config.h  : sys_config.log ;+@[ -f $@ ] || { echo "cannot find $@" ; exit 1 ; }
 
@@ -217,6 +221,8 @@ $(LIB)/%.py : $(SLIB)/%.src.py
 	@mkdir -p $(@D)
 	sed \
 		-e 's!\$$BASH!$(BASH)!'                          \
+		-e 's!\$$PYTHON2!$(PYTHON2)!'                    \
+		-e 's!\$$PYTHON!$(PYTHON)!'                      \
 		-e 's!\$$GIT!$(GIT)!'                            \
 		-e 's!\$$LD_LIBRARY_PATH!$(PY_LD_LIBRARY_PATH)!' \
 		-e 's!\$$STD_PATH!$(STD_PATH)!'                  \
@@ -649,17 +655,17 @@ UNIT_TESTS : UNIT_TESTS1 UNIT_TESTS2
 	@( cd $(@D) ; git clean -ffdxq >/dev/null 2>/dev/null ) ; : # keep $(@D) to ease debugging, ignore rc as old versions of git work but generate an error
 	@for f in $$(grep '^$(UT_DIR)/base/' Manifest) ; do df=$(@D)/$${f#$(UT_DIR)/base/} ; mkdir -p $$(dirname $$df) ; cp $$f $$df ; done
 	@cd $(@D) ; find . -type f -printf '%P\n' > Manifest
-	@	( cd $(@D) ; PATH=$(ROOT_DIR)/bin:$(ROOT_DIR)/_bin:$$PATH $(ROOT_DIR)/$< ) >$@.out 2>$@.err \
-	&&	mv $@.out $@                                                                                \
+	@	( cd $(@D) ; PATH=$(ROOT_DIR)/bin:$(ROOT_DIR)/_bin:$$PATH CXX=$(CXX) $(ROOT_DIR)/$< ) >$@.out 2>$@.err \
+	&&	mv $@.out $@                                                                                           \
 	||	( cat $@.out $@.err ; exit 1 )
 
 %.dir/tok : %.py $(LMAKE_FILES) _lib/ut.py
 	@echo py test to $@
 	@mkdir -p $(@D)
 	@( cd $(@D) ; git clean -ffdxq >/dev/null 2>/dev/null ) ; : # keep $(@D) to ease debugging, ignore rc as old versions of git work but generate an error
 	@cp $< $(@D)/Lmakefile.py
-	@	( cd $(@D) ; PATH=$(ROOT_DIR)/bin:$(ROOT_DIR)/_bin:$$PATH PYTHONPATH=$(ROOT_DIR)/lib:$(ROOT_DIR)/_lib HOME= $(PYTHON) Lmakefile.py ) >$@.out 2>$@.err \
-	&&	mv $@.out $@                                                                                                                                          \
+	@	( cd $(@D) ; PATH=$(ROOT_DIR)/bin:$(ROOT_DIR)/_bin:$$PATH PYTHONPATH=$(ROOT_DIR)/lib:$(ROOT_DIR)/_lib HOME= CXX=$(CXX) $(PYTHON) Lmakefile.py ) >$@.out 2>$@.err \
+	&&	mv $@.out $@                                                                                                                                                     \
 	||	( cat $@.out $@.err ; exit 1 )
 
 #

Manifest

@@ -153,6 +153,7 @@ unit_tests/base/Lmakefile.py
 unit_tests/base/hello.py
 unit_tests/base/src1
 unit_tests/base/src2
+unit_tests/bench.py
 unit_tests/cache.py
 unit_tests/cargo.py
 unit_tests/chain.py

TO_DO

@@ -108,6 +108,10 @@ items :
 		- this way, no harm can be done outside repo
 * provide a reasonable default value when dynamic functions return None
 	- at least, dont crash
+* provide more options to stderr management :
+	- redirect to stdout
+	- hide it (still accessible in lshow -e)
+	? rename allow_stderr to stderr
 * implement cache v2 (copy & link) :
 	- 2 levels : disk level, global level
 	- use link instead of copy

_bin/sys_config

@@ -14,12 +14,16 @@ START_DIR=$(pwd)
 mkdir -p trial
 cd       trial
 
+PYTHON2=${PYTHON2:-python2} ; PYTHON2=$(type -p $PYTHON2||:)                                              # ok if we have no python2
+PYTHON=${PYTHON:-python3}   ; PYTHON=$( type -p $PYTHON    )
+CXX=${CXX:-g++}             ; CXX=$(    type -p $CXX       )
+
+echo /$PYTHON2/ /$PYTHON/ /$CXX/
+
 #
 # Python configuration (Python2 is optional)
 #
 
-PYTHON2=${PYTHON2:-python2} ; PYTHON2=$(type -p $PYTHON2||:)                                              # ok if we have no python2
-PYTHON=${PYTHON:-python3}   ; PYTHON=$( type -p $PYTHON    )
 for python in $PYTHON2 $PYTHON ; do
 	if [ -x $python ] ; then
 		eval $($python -c 'if True :
@@ -51,12 +55,19 @@ if [ "$PY2_LIB_DIR" -a "$PY_LIB_DIR" ] ; then PY_LD_LIBRARY_PATH=$PY_LIB_DIR:$PY
 else                                          PY_LD_LIBRARY_PATH=$PY_LIB_DIR$PY2_LIB_DIR                  # only one is present
 fi
 # cancel Python2 if unusable as a library
-{ [ -f $PY2_INCLUDEPY/Python.h ] && file -L $PY2_LIB_DIR/$PY2_LIB_BASE | grep -q shared ; } || PYTHON2=
+cat <<"EOF" > python2.c
+	#include <Python.h>
+	int main() {
+		PyObject* none = Py_None ;
+	}
+EOF
+p2ld=$PY2_LIB_DIR
+[ "$p2ld" ] && p2ld="-L$p2ld"
+$CXX -o python2 -I$PY2_INCLUDEPY $p2ld python2.c -l:$PY2_LIB_BASE || PYTHON2=
 
 #
 # CXX configuration
 #
-CXX=${CXX:-g++}
 [ "$CXX" != "${CXX%++*}" ] || { echo cannot recognize c++ compiler $CXX ; exit 1 ; } # ensure we can find "++" in cxx, for example gcc or clang will not work, we must use g++ and clang++
 type -p $CXX >/dev/null    || { echo cannot find      c++ compiler $CXX ; exit 1 ; }
 
@@ -99,7 +110,7 @@ STD_INC_DIRS="$(echo $(realpath $(sed -e '1,/<.*>.*search starts/d' -e '/End of
 # test whether we have seccomp : warning, include <seccomp.h> is not enough to test, check its content too
 #
 cat <<"EOF" > seccomp.c
-	#include<seccomp.h>
+	#include <seccomp.h>
 	struct scmp_version v ;
 EOF
 # usually seccomp.h is in /usr/include (with an internal file /usr/include/linux/seccomp.h) but some systems have seccomp.h directly installed in /usr/include/linux
@@ -222,8 +233,8 @@ fi
 #
 # addr2line
 #
-[ ""              ] || ADDR2LINE=$(find $(dirname $(realpath $(type -p $CXX))) -name llvm-addr2line)
-[ -x "$ADDR2LINE" ] || ADDR2LINE=$(find $(dirname $(realpath $(type -p $CXX))) -name addr2line     )
+[ ""              ] || ADDR2LINE=$(find $(dirname $(realpath $CXX)) -name llvm-addr2line)
+[ -x "$ADDR2LINE" ] || ADDR2LINE=$(find $(dirname $(realpath $CXX)) -name addr2line     )
 [ -x "$ADDR2LINE" ] || ADDR2LINE=$(type -p addr2line)
 
 #
@@ -296,7 +307,7 @@ PY_INCLUDEPY   := $PY_INCLUDEPY
 PY_LIB_DIR     := $PY_LIB_DIR
 PY_LIB_BASE    := $PY_LIB_BASE
 #
-CXX            := $(type -p $CXX)
+CXX            := $CXX
 CXX_FLAVOR     := $CXX_FLAVOR
 LINK_LIB_PATH  := $LINK_LIB_PATH
 STD_INC_DIRS   := $STD_INC_DIRS

_lib/lmake/import_machinery.py

@@ -19,9 +19,12 @@ def _mask_python_deps() :
 	# - if a file can be generated, there will be no dep (as the file is not accessed), this may lead to a non-existing module without job rerun or worse, a following file may be used
 	# - a lot of sibling files will be accessed, triggering potentially unwanted deps
 	# to prevent that, autodep is deactivated during import
-	if _sys.version_info.major==2 : import __builtins__ as builtins
-	else                          : import   builtins
-	orig_import = builtins.__import__
+	if _sys.version_info.major==2 :
+		builtins_dct = __builtins__
+	else :
+		import builtins
+		builtins_dct = builtins.__dict__
+	orig_import = builtins_dct['__import__']
 	def new_import(*args,**kwds) :
 		with Autodep(False) :
 			return orig_import(*args,**kwds)
@@ -35,13 +38,16 @@ def new_exec_module(*args,**kwds) :
 		_LoaderBasics.exec_module = orig_exec_module
 	except :
 		raise RuntimeError('masking python deps during import is not available for python%d.%d'%_sys.version_info[:2])
-	builtins.__import__ = new_import # wrap at the end to avoid wraping our own imports
+	builtins_dct['__import__'] = new_import                                                                            # wrap at the end to avoid wraping our own imports
 
 def _maybe_local(file) :
 	'fast check for local files, avoiding full absolute path generation'
 	return not file or file[0]!='/' or file.startswith(root_dir)
-import importlib.machinery as _machinery
-_std_suffixes = _machinery.all_suffixes()+['/__init__.py'] # account for packages, not included in all_suffixes()
+if _sys.version_info.major==2 :
+	_std_suffixes = ['.py','.so','/__init__.py']               # standard suffixes are not available with Python2
+else :
+	import importlib.machinery as _machinery
+	_std_suffixes = _machinery.all_suffixes()+['/__init__.py'] # account for packages, not included in all_suffixes()
 
 def _gen_module_deps() :
 	'''fixes imports so as to be sure all files needed to do an import is correctly reported (not merely those that exist)'''

_lib/lmake/rules.src.py

@@ -11,10 +11,10 @@
 import lmake
 from . import has_ld_audit,pdict,root_dir # if not in an lmake repo, root_dir is not set to current dir
 
-shell  = '$BASH'         # .
-python = _sys.executable
-
-_std_path        = '$STD_PATH'                   # substituted at installation time
+shell            = '$BASH'                       # substituted at installation time
+python2          = '$PYTHON2'                    # .
+python           = '$PYTHON'                     # .
+_std_path        = '$STD_PATH'                   # .
 _ld_library_path = '$LD_LIBRARY_PATH'            # .
 _lmake_dir       = __file__.rsplit('/lib/',1)[0]
 
@@ -88,8 +88,8 @@ class Rule(_RuleBase) :
 	n_retries        = 1                                     # number of retries in case of job lost. 1 is a reasonable value
 #	n_tokens         = 1                                     # number of jobs likely to run in parallel for this rule (used for ETA estimation)
 #	prio             = 0                                     # in case of ambiguity, rules are selected with highest prio first
-	python           = (python,)                            # python used for callable cmd
-	shell            = (shell ,)                            # shell  used for str      cmd (_sh is usually /bin/sh which may test for dir existence before chdir, which defeats auto_mkdir)
+	python           = (python,)                             # python used for callable cmd
+	shell            = (shell ,)                             # shell  used for str      cmd (_sh is usually /bin/sh which may test for dir existence before chdir, which defeats auto_mkdir)
 	start_delay      = 3                                     # delay before sending a start message if job is not done by then, 3 is a reasonable compromise
 	max_stderr_len   = 100                                   # maximum number of stderr lines shown in output (full content is accessible with lshow -e), 100 is a reasonable compromise
 #	timeout          = None                                  # timeout allocated to job execution (in s), must be None or an int
@@ -140,7 +140,7 @@ class _PyRule(Rule) :
 	def cmd() :                                                                           # this will be executed before cmd() of concrete subclasses as cmd() are chained in case of inheritance
 		if gen_module_deps or mask_python_deps :                                          # fast path :if nothing to do, do nothing
 			try : import lmake
-			except ModuleNotFoundError :
+			except ImportError :
 				import sys
 				sys.path[0:0] = (_lmake_dir+'/lib',)
 			from lmake.import_machinery import fix_import
@@ -150,6 +150,7 @@ class Py2Rule(_PyRule) :
 	'base rule that handle pyc creation when importing modules in Python'
 	side_targets    = { '__PYC__' : ( r'{*:(.+/)?}{*:\w+}.pyc' , 'Incremental'  ) }
 	gen_module_deps = True
+	python          = python2
 class Py3Rule(_PyRule) :
 	'base rule that handle pyc creation when importing modules in Python'
 	side_targets     = { '__PYC__' : ( r'{*:(.+/)?}__pycache__/{*:\w+}.{*:\w+-\d+}.pyc' , 'Incremental'  ) }

_lib/lmake_dbg.py

@@ -79,7 +79,8 @@ def run_pdb(dbg_dir,redirected,func,*args,**kwds) :
 		debugger.runcall(func,*args,**kwds)
 	except BaseException as e :
 		traceback.print_exc()
-		debugger.interaction(None,sys.exc_traceback)
+		debugger.reset()
+		debugger.interaction(None,sys.exc_info()[2])
 
 def run_vscode(dbg_dir,redirected,func,*args,**kwds) :
 	import json

src/autodep/clmake.cc

@@ -68,11 +68,11 @@ static PyObject* depend( PyObject* /*null*/ , PyObject* args , PyObject* kwds )
 	AccessDigest ad        { .accesses=~Accesses() , .dflags=Dflag::Required } ;
 	if (py_kwds) {
 		size_t n_kwds = py_kwds->size() ;
-		/**/                                    if ( const char* s="follow_symlinks" ;                          py_kwds->contains(s) ) { n_kwds-- ; no_follow =             !(*py_kwds)[s]  ; }
-		/**/                                    if ( const char* s="verbose"         ;                          py_kwds->contains(s) ) { n_kwds-- ; verbose   =             +(*py_kwds)[s]  ; }
-		/**/                                    if ( const char* s="read"            ;                          py_kwds->contains(s) ) { n_kwds-- ; read      =             +(*py_kwds)[s]  ; }
-		for( Dflag      df  : Dflag::NDyn     ) if ( ::string    s=snake_str(df )    ;                          py_kwds->contains(s) ) { n_kwds-- ; ad.dflags      .set(df ,+(*py_kwds)[s]) ; }
-		for( ExtraDflag edf : All<ExtraDflag> ) if ( ::string    s=snake_str(edf)    ; ExtraDflagChars[+edf] && py_kwds->contains(s) ) { n_kwds-- ; ad.extra_dflags.set(edf,+(*py_kwds)[s]) ; }
+		/**/                                    if ( const char* s="follow_symlinks" ;                                 py_kwds->contains(s) ) { n_kwds-- ; no_follow =             !(*py_kwds)[s]  ; }
+		/**/                                    if ( const char* s="verbose"         ;                                 py_kwds->contains(s) ) { n_kwds-- ; verbose   =             +(*py_kwds)[s]  ; }
+		/**/                                    if ( const char* s="read"            ;                                 py_kwds->contains(s) ) { n_kwds-- ; read      =             +(*py_kwds)[s]  ; }
+		for( Dflag      df  : Dflag::NDyn     ) if ( ::string    s=snake_str(df )    ;                                 py_kwds->contains(s) ) { n_kwds-- ; ad.dflags      .set(df ,+(*py_kwds)[s]) ; }
+		for( ExtraDflag edf : All<ExtraDflag> ) if ( ::string    s=snake_str(edf)    ; ExtraDflagChars[+edf].second && py_kwds->contains(s) ) { n_kwds-- ; ad.extra_dflags.set(edf,+(*py_kwds)[s]) ; }
 		//
 		if (n_kwds) return py_err_set(Exception::TypeErr,"unexpected keyword arg") ;
 	}
@@ -112,10 +112,10 @@ static PyObject* target( PyObject* /*null*/ , PyObject* args , PyObject* kwds )
 	AccessDigest ad        { .write=Yes , .extra_tflags=ExtraTflag::Allow } ;
 	if (py_kwds) {
 		size_t n_kwds = py_kwds->size() ;
-		/**/                                    if ( const char* s="follow_symlinks" ;                          py_kwds->contains(s) ) { n_kwds-- ; no_follow =             !(*py_kwds)[s]  ; }
-		/**/                                    if ( const char* s="write"           ;                          py_kwds->contains(s) ) { n_kwds-- ; ad.write  = No |        +(*py_kwds)[s]  ; }
-		for( Tflag      tf  : Tflag::NDyn     ) if ( ::string    s=snake_str(tf )    ;                          py_kwds->contains(s) ) { n_kwds-- ; ad.tflags      .set(tf ,+(*py_kwds)[s]) ; }
-		for( ExtraTflag etf : All<ExtraTflag> ) if ( ::string    s=snake_str(etf)    ; ExtraTflagChars[+etf] && py_kwds->contains(s) ) { n_kwds-- ; ad.extra_tflags.set(etf,+(*py_kwds)[s]) ; }
+		/**/                                    if ( const char* s="follow_symlinks" ;                                 py_kwds->contains(s) ) { n_kwds-- ; no_follow =             !(*py_kwds)[s]  ; }
+		/**/                                    if ( const char* s="write"           ;                                 py_kwds->contains(s) ) { n_kwds-- ; ad.write  = No |        +(*py_kwds)[s]  ; }
+		for( Tflag      tf  : Tflag::NDyn     ) if ( ::string    s=snake_str(tf )    ;                                 py_kwds->contains(s) ) { n_kwds-- ; ad.tflags      .set(tf ,+(*py_kwds)[s]) ; }
+		for( ExtraTflag etf : All<ExtraTflag> ) if ( ::string    s=snake_str(etf)    ; ExtraTflagChars[+etf].second && py_kwds->contains(s) ) { n_kwds-- ; ad.extra_tflags.set(etf,+(*py_kwds)[s]) ; }
 		//
 		if (n_kwds) return py_err_set(Exception::TypeErr,"unexpected keyword arg") ;
 	}

src/autodep/ldepend.cc

@@ -31,12 +31,12 @@ int main( int argc , char* argv[]) {
 	,	{ Flag::Verbose        , { .short_name='v' , .has_arg=false , .doc="write dep crcs on stdout"           } }
 	,	{ Flag::NoRead         , { .short_name='R' , .has_arg=false , .doc="does not report a read, only flags" } }
 	//
-	,	{ Flag::Critical     , { .short_name=DflagChars     [+Dflag     ::Critical    ] , .has_arg=false , .doc="report critical deps"                            } }
-	,	{ Flag::Essential    , { .short_name=DflagChars     [+Dflag     ::Essential   ] , .has_arg=false , .doc="ask that deps be seen in graphical flow"         } }
-	,	{ Flag::IgnoreError  , { .short_name=DflagChars     [+Dflag     ::IgnoreError ] , .has_arg=false , .doc="accept that deps are in error"                   } }
-	,	{ Flag::NoRequired   , { .short_name=DflagChars     [+Dflag     ::Required    ] , .has_arg=false , .doc="accept that deps cannot be built"                } }
-	,	{ Flag::Ignore       , { .short_name=ExtraDflagChars[+ExtraDflag::Ignore      ] , .has_arg=false , .doc="ignore reads"                                    } }
-	,	{ Flag::StatReadData , { .short_name=ExtraDflagChars[+ExtraDflag::StatReadData] , .has_arg=false , .doc="stat access implies access to full file content" } }
+	,	{ Flag::Critical     , { .short_name=DflagChars     [+Dflag     ::Critical    ].second , .has_arg=false , .doc="report critical deps"                            } }
+	,	{ Flag::Essential    , { .short_name=DflagChars     [+Dflag     ::Essential   ].second , .has_arg=false , .doc="ask that deps be seen in graphical flow"         } }
+	,	{ Flag::IgnoreError  , { .short_name=DflagChars     [+Dflag     ::IgnoreError ].second , .has_arg=false , .doc="accept that deps are in error"                   } }
+	,	{ Flag::NoRequired   , { .short_name=DflagChars     [+Dflag     ::Required    ].second , .has_arg=false , .doc="accept that deps cannot be built"                } }
+	,	{ Flag::Ignore       , { .short_name=ExtraDflagChars[+ExtraDflag::Ignore      ].second , .has_arg=false , .doc="ignore reads"                                    } }
+	,	{ Flag::StatReadData , { .short_name=ExtraDflagChars[+ExtraDflag::StatReadData].second , .has_arg=false , .doc="stat access implies access to full file content" } }
 	}} ;
 	CmdLine<Key,Flag> cmd_line { syntax , argc , argv } ;
 	//