Skip to content

Commit

Permalink
improve target/dep definition + bring back clang support + update xxh…
Browse files Browse the repository at this point in the history 
… + improve compilation control
  • Loading branch information
@cesar-douady
cesar-douady committed Mar 27, 2024
1 parent f493c71 commit aad3d51
Show file tree
Hide file tree
Showing 52 changed files with 1,836 additions and 980 deletions.
97 changes: 62 additions & 35 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,20 +33,42 @@ STD_PATH := $(shell env -i $(BASH) -c 'echo $$PATH')

MAKEFLAGS += -r -R

ifeq ($(strip $(LMAKE_OPT_LVL)),)
OPT_FLAGS := -O3
else ifeq ($(strip $(LMAKE_OPT_LVL)),0)
OPT_FLAGS := -O0 -g
else ifeq ($(strip $(LMAKE_OPT_LVL)),1)
OPT_FLAGS := -O1
else ifeq ($(strip $(LMAKE_OPT_LVL)),2)
OPT_FLAGS := -O2
else ifeq ($(strip $(LMAKE_OPT_LVL)),3)
OPT_FLAGS := -O3
else ifeq ($(strip $(LMAKE_OPT_LVL)),4)
OPT_FLAGS := -O3 -DNDEBUG
else ifeq ($(strip $(LMAKE_OPT_LVL)),5)
OPT_FLAGS := -O3 -DNDEBUG -DNO_TRACE
# syntax for LMAKE_OPT : [0123][gG][aA][tT]
# - [0123] : compiler optimization level
# - [gG] : -g flag if G
# - [dD] : -DNDEBUG if d
# - [tT] : -DNO_TRACE if t
ifeq ($(LMAKE_OPT),)
LMAKE_OPT := 3GDT
endif
ifneq ($(findstring 0,$(LMAKE_OPT)),)
OPT_FLAGS := -O0
else ifneq ($(findstring 1,$(LMAKE_OPT)),)
OPT_FLAGS := -O1
else ifneq ($(findstring 2,$(LMAKE_OPT)),)
OPT_FLAGS := -O2
else ifneq ($(findstring 3,$(LMAKE_OPT)),)
OPT_FLAGS := -O3
else
$(error $$LMAKE_OPT must be [0123][gG][dD][tT])
endif
ifneq ($(findstring G,$(LMAKE_OPT)),)
OPT_FLAGS += -g
else ifneq ($(findstring g,$(LMAKE_OPT)),)
else
$(error $$LMAKE_OPT must be [0123][gG][dD][tT])
endif
ifneq ($(findstring D,$(LMAKE_OPT)),)
else ifneq ($(findstring d,$(LMAKE_OPT)),)
OPT_FLAGS += -DNDEBUG
else
$(error $$LMAKE_OPT must be [0123][gG][dD][tT])
endif
ifneq ($(findstring T,$(LMAKE_OPT)),)
else ifneq ($(findstring t,$(LMAKE_OPT)),)
OPT_FLAGS += -DNO_TRACE
else
$(error $$LMAKE_OPT must be [0123][gG][dD][tT])
endif

COVERAGE :=
Expand All @@ -72,31 +94,36 @@ $(error python3 version must be at least 3.6)
endif

# CC configuration
USE_GCC :=
USE_CLANG :=
ifeq ($(CC),)

$(error cannot find c compiler)

else ifeq ($(findstring gcc,$(CC)),gcc)

else ifneq ($(findstring gcc,$(CC)),)
ifeq ($(intcmp $(shell $(CC) -dumpversion),11,lt,eq,gt),lt)
$(error gcc version must be at least 11)
USE_GCC := 1
endif
else ifneq ($(findstring clang,$(CC)),)
ifeq ($(intcmp $(shell $(CC) -dumpversion),15,lt,eq,gt),lt)
$(error clang version must be at least 15)
endif
USE_CLANG := 1
else
$(error cannot recognize c compiler $(CC))
endif

ifneq ($(USE_GCC),) # for an unknown reason, clang is incompatible with -fsanitize
# -fsanitize=address and -fsanitize=thread are exclusive of one another
# for an unknown reason, clang is incompatible with -fsanitize
ifeq ($(LMAKE_SAN),A)
SAN_FLAGS := -fsanitize=address -fsanitize=undefined
endif
ifeq ($(LMAKE_SAN),T)
SAN_FLAGS := -fsanitize=thread
endif

else ifeq ($(findstring clang,$(CC)),clang)

ifeq ($(intcmp $(shell $(CC) -dumpversion),15,lt,eq,gt),lt)
$(error clang version must be at least 15)
endif
WARNING_FLAGS += -Wno-misleading-indentation -Wno-unknown-warning-option -Wno-c2x-extensions

ifneq ($(USE_CLANG),)
WARNING_FLAGS += -Wno-misleading-indentation -Wno-unknown-warning-option -Wno-c2x-extensions -Wno-unused-function -Wno-c++2b-extensions
endif

# this is the recommanded way to insert a , when calling functions
Expand Down Expand Up @@ -371,18 +398,18 @@ ALL_H := version.hh sys_config.h ext/xxhash.patched.h
# On ubuntu, seccomp.h is in /usr/include. On CenOS7, it is in /usr/include/linux, but beware that otherwise, /usr/include must be prefered, hence -idirafter
CPP_OPTS := -iquote ext -iquote $(SRC) -iquote $(SRC_ENGINE) -iquote . -idirafter /usr/include/linux

%_py2.san.o : %.cc $(ALL_H) ; @echo compile $(CXX_FLAGS) $(SAN_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -c $(SAN_FLAGS) -frtti -fPIC $(PY2_CC_OPTS) $(CPP_OPTS) -o $@ $<
%_py2.i : %.cc $(ALL_H) ; @echo preproc $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -E $(PY2_CC_OPTS) $(CPP_OPTS) -o $@ $<
%_py2.s : %.cc $(ALL_H) ; @echo compile $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -S $(PY2_CC_OPTS) $(CPP_OPTS) -o $@ $<
%_py2.o : %.cc $(ALL_H) ; @echo compile $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -c -frtti -fPIC $(PY2_CC_OPTS) $(CPP_OPTS) -o $@ $<
%_py2.san.o : %.cc $(ALL_H) ; @echo $(CC) -c $(CXX_FLAGS) $(SAN_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -c $(SAN_FLAGS) -frtti -fPIC $(PY2_CC_OPTS) $(CPP_OPTS) -o $@ $<
%_py2.i : %.cc $(ALL_H) ; @echo $(CC) -E $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -E $(PY2_CC_OPTS) $(CPP_OPTS) -o $@ $<
%_py2.s : %.cc $(ALL_H) ; @echo $(CC) -S $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -S $(PY2_CC_OPTS) $(CPP_OPTS) -o $@ $<
%_py2.o : %.cc $(ALL_H) ; @echo $(CC) -c $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -c -frtti -fPIC $(PY2_CC_OPTS) $(CPP_OPTS) -o $@ $<

%.san.o : %.cc $(ALL_H) ; @echo compile $(CXX_FLAGS) $(SAN_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -c $(SAN_FLAGS) -frtti -fPIC $(PY_CC_OPTS) $(CPP_OPTS) -o $@ $<
%.i : %.cc $(ALL_H) ; @echo preproc $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -E $(PY_CC_OPTS) $(CPP_OPTS) -o $@ $<
%.s : %.cc $(ALL_H) ; @echo compile $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -S $(PY_CC_OPTS) $(CPP_OPTS) -o $@ $<
%.o : %.cc $(ALL_H) ; @echo compile $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -c -frtti -fPIC $(PY_CC_OPTS) $(CPP_OPTS) -o $@ $<
%.san.o : %.cc $(ALL_H) ; @echo $(CC) -c $(CXX_FLAGS) $(SAN_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -c $(SAN_FLAGS) -frtti -fPIC $(PY_CC_OPTS) $(CPP_OPTS) -o $@ $<
%.i : %.cc $(ALL_H) ; @echo $(CC) -E $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -E $(PY_CC_OPTS) $(CPP_OPTS) -o $@ $<
%.s : %.cc $(ALL_H) ; @echo $(CC) -S $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -S $(PY_CC_OPTS) $(CPP_OPTS) -o $@ $<
%.o : %.cc $(ALL_H) ; @echo $(CC) -c $(CXX_FLAGS) to $@ ; $(COMPILE) $(CXX_FLAGS) -c -frtti -fPIC $(PY_CC_OPTS) $(CPP_OPTS) -o $@ $<

%_py2.d : %.cc $(ALL_H) ; @$(COMPILE) $(CXX_FLAGS) -MM -MT '$(@:%.d=%.i) $(@:%.d=%.s) $(@:%.d=%.o) $(@:%.d=%.san.o) $@ ' -MF $@ $(PY2_CC_OPTS) $(CPP_OPTS) $<
%.d : %.cc $(ALL_H) ; @$(COMPILE) $(CXX_FLAGS) -MM -MT '$(@:%.d=%.i) $(@:%.d=%.s) $(@:%.d=%.o) $(@:%.d=%.san.o) $@ ' -MF $@ $(PY_CC_OPTS) $(CPP_OPTS) $<
%_py2.d : %.cc $(ALL_H) ; @$(COMPILE) $(CXX_FLAGS) -MM -MT '$(@:%.d=%.i) $(@:%.d=%.s) $(@:%.d=%.o) $(@:%.d=%.san.o) $@ ' -MF $@ $(PY2_CC_OPTS) $(CPP_OPTS) $< 2>/dev/null || :
%.d : %.cc $(ALL_H) ; @$(COMPILE) $(CXX_FLAGS) -MM -MT '$(@:%.d=%.i) $(@:%.d=%.s) $(@:%.d=%.o) $(@:%.d=%.san.o) $@ ' -MF $@ $(PY_CC_OPTS) $(CPP_OPTS) $< 2>/dev/null || :

include $(patsubst %.cc,%.d, $(filter-out %.x.cc,$(filter %.cc,$(shell git ls-files))) )
include src/py_py2.d src/autodep/clmake_py2.d
Expand Down
1 change: 1 addition & 0 deletions Manifest
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,7 @@ unit_tests/misc4.py
unit_tests/misc5.py
unit_tests/misc6.py
unit_tests/misc7.py
unit_tests/misc8.py
unit_tests/modules.py
unit_tests/name.py
unit_tests/numba.py
Expand Down
43 changes: 28 additions & 15 deletions _bin/sys_config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
# This program is free software: you can redistribute/modify under the terms of the GPL-v3 (https://www.gnu.org/licenses/gpl-3.0.html).
# This program is distributed WITHOUT ANY WARRANTY, without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

unset CDPATH # ensure cd goes to the right directory and writes nothing to stdout
unset CDPATH # ensure cd goes to the right directory and writes nothing to stdout

mkdir -p trial
cd trial
Expand Down Expand Up @@ -71,20 +71,33 @@ fi
# HAS_LD_AUDIT
# test whether LD_AUDIT environment variable is managed by dynamic linker
#
cat <<"EOF" > audited.c
#include<stdio.h>
int main() { printf("0") ; }
EOF
cat <<"EOF" > ld_audit.c
#include<stdio.h>
#include<stdlib.h>
unsigned int la_version(unsigned int) { printf("1") ; exit(0) ; }
EOF
$CC -o audited audited.c
$CC -o ld_audit.so -shared -fPIC ld_audit.c
HAS_LD_AUDIT=$(LD_AUDIT=./ld_audit.so ./audited)
# CentOS7 fix : This is what would answer a real centos-7 system, but we may be running in a docker which has a more recent kernel.
grep -q 'release 7' /etc/redhat-release 2>/dev/null && HAS_LD_AUDIT=0
source /etc/os-release
case "$ID" in
ubuntu)
case "$VERSION_ID" in
20.04) HAS_LD_AUDIT=0 ;; # Ubuntu fix : Ubuntu 20.04 seems to miss calling auditing code upon ldopen
esac
;;
centos)
case "$VERSION_ID" in
7) HAS_LD_AUDIT=0 ;; # CentOS7 fix : This is what would answer a real centos-7 system, but we may be running in a docker which has a more recent kernel.
esac
;;
esac
if [ -z "$HAS_LD_AUDIT" ] ; then
cat <<-"EOF" > audited.c
#include<stdio.h>
int main() { printf("0") ; }
EOF
cat <<-"EOF" > ld_audit.c
#include<stdio.h>
#include<stdlib.h>
unsigned int la_version(unsigned int) { printf("1") ; exit(0) ; }
EOF
$CC -o audited audited.c
$CC -o ld_audit.so -shared -fPIC ld_audit.c
HAS_LD_AUDIT=$(LD_AUDIT=./ld_audit.so ./audited)
fi

#
# NEED_STAT_WRAPPERS
Expand Down
23 changes: 8 additions & 15 deletions _lib/lmake/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,23 +112,16 @@ def version(major,minor) :
)
)

class SuspendAutodep :
class Autodep :
"""context version of the set_autodep function (applies to this process and all processes started in the protected core)
usage :
with SuspendAutodep() :
<code with autodep deactivated>
with Autodep(enable) :
<code with autodep activate(enable=True) or deactivated (enable=False)>
"""
level = 0
def __init__(self) :
pass
def __init__(self,enable) :
self.cur = enable
def __enter__(self) :
if not self.__class__.level :
self.ld_preload = _os.environ.pop('LD_PRELOAD',None)
set_autodep(False)
self.__class__.level += 1
self.prev = get_autodep()
set_autodep(self.cur)
def __exit__(self,*args) :
self.__class__.level -= 1
if not self.__class__.level :
if self.ld_preload is not None : _os.environ['LD_PRELOAD'] = self.ld_preload
set_autodep(True)

set_autodep(self.prev)
139 changes: 59 additions & 80 deletions _lib/lmake/import_machinery.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,90 +6,69 @@
import os.path as _osp
import sys as _sys

from . import depend,SuspendAutodep
from . import depend,Autodep

module_suffixes = ('.so','.py','/__init__.py') # can be tailored to suit application needs, the lesser the better (less spurious dependencies)

_method = 1
from . import root_dir

if _method==1 :
def _mask_python_deps() :
'''replace __import__ by a semantically equivalent function (that actually calls the original one) to suppress python generated deps'''
# Instead of trying to access file candidates implementing the module, Python optimizes by first reading the englobing dir and only access existing files.
# This is severely anti-lmake :
# - if a file can be generated, there will be no dep (as the file is not accessed), this may lead to a non-existing module without job rerun or worse, a following file may be used
# - a lot of sibling files will be accessed, triggering potentially unwanted deps
# to prevent that, autodep is deactivated during import
if _sys.version_info.major==2 : import __builtins__ as builtins
else : import builtins
orig_import = builtins.__import__
def new_import(*args,**kwds) :
with Autodep(False) :
return orig_import(*args,**kwds)
# try to re-enable autodep when actually executing the module as the purpose was to disable autodep during the search phase, not the execute phase
try :
from importlib._bootstrap_external import _LoaderBasics
orig_exec_module = _LoaderBasics.exec_module
def new_exec_module(*args,**kwds) :
with Autodep(True) :
return orig_exec_module(*args,**kwds)
_LoaderBasics.exec_module = orig_exec_module
except :
raise RuntimeError('masking python deps during import is not available for python%d.%d'%_sys.version_info[:2])
builtins.__import__ = new_import # wrap at the end to avoid wraping our own imports

#
# this version is more pythonic, but does only half of the job : it generates adequate deps, but does not suppress the superfluous ones
#
from . import root_dir
def _maybe_local(file) :
'fast check for local files, avoiding full absolute path generation'
return not file or file[0]!='/' or file.startswith(root_dir)
import importlib.machinery as _machinery
_std_suffixes = _machinery.all_suffixes()+['/__init__.py'] # account for packages, not included in all_suffixes()
def _maybe_local(file) :
'fast check for local files, avoiding full absolute path generation'
return not file or file[0]!='/' or file.startswith(root_dir)
import importlib.machinery as _machinery
_std_suffixes = _machinery.all_suffixes()+['/__init__.py'] # account for packages, not included in all_suffixes()

def fix_import() :
'''fixes imports so as to be sure all files needed to do an import is correctly reported (not merely those which exist)'''
class Depend :
@staticmethod
def find_spec(module_name,path,target=None) :
if path==None : path = _sys.path
tail = module_name.rsplit('.',1)[-1]
for dir in path :
if dir : dir += '/'
base = dir+tail
if _maybe_local(base) :
for suffix in module_suffixes :
file = base+suffix
depend(file,required=False,essential=False)
if _osp.exists(file) : return
else :
for suffix in _std_suffixes :
if _osp.exists(base+suffix) : return

# give priority to system so as to avoid too numerous dependencies
_sys.path = (
[ d for d in _sys.path if not _maybe_local(d+'/') ]
+ [ d for d in _sys.path if _maybe_local(d+'/') ]
)

# put dependency checker before the first path based finder
for i in range(len(_sys.meta_path)) :
if _sys.meta_path[i]==_machinery.PathFinder :
_sys.meta_path.insert(i,Depend)
break
else :
_sys.meta_path.append(Depend)

elif method==2 :

def fix_import() :
'''replace __import__ by a semantically equivalent function (that actually call the original one) to generate clean deps'''
if _sys.version_info.major==2 : import __builtins__ as builtins
else : import builtins
orig_import = builtins.__import__
# Instead of trying to access file candidates implementing the module, Python optimizes by first reading the englobing dir and only access existing files.
# This is severely anti-lmake :
# - if a file can be generated, there will be no dep (as the file is not accessed), this may lead to a non-existing module without job rerun or worse, a following file may be used
# - a lot of sibling files will be accessed, triggering potentially unwanted deps
# to prevent that, deps are managed explicitly
def new_import(name,glbs={},*args,**kwds) :
if name.startswith('.') :
pkg = glbs['__package__']
while name.startswith('..') :
pkg = pkg.rsplit('.',1)[0]
name = name[1:]
full_name = pkg+name
else :
full_name = name
if full_name not in _sys.modules :
file_name = full_name.replace('.','/')
for d in _sys.path :
if d : b = d+'/'+file_name
else : b = file_name
for sfx in module_suffixes :
f = b+sfx
depend(f,required=False)
if _osp.exists(f) : break
else : continue
break
with SuspendAutodep() :
return orig_import(name,glbs,*args,**kwds)
builtins.__import__ = new_import
def _gen_module_deps() :
'''fixes imports so as to be sure all files needed to do an import is correctly reported (not merely those that exist)'''
class Depend :
@staticmethod
def find_spec(module_name,path,target=None) :
if path==None : path = _sys.path
tail = module_name.rsplit('.',1)[-1]
for dir in path :
if dir : dir += '/'
base = dir+tail
if _maybe_local(base) :
for suffix in module_suffixes :
file = base+suffix
depend(file,required=False,read=True)
if _osp.exists(file) : return
else :
for suffix in _std_suffixes :
if _osp.exists(base+suffix) : return
# put dependency checker before the first path based finder
for i in range(len(_sys.meta_path)) :
if _sys.meta_path[i]==_machinery.PathFinder :
_sys.meta_path.insert(i,Depend)
break
else :
_sys.meta_path.append(Depend)

def fix_import(gen_module_deps=True,mask_python_deps=True) :
if (mask_python_deps) : _mask_python_deps()
if (gen_module_deps ) : _gen_module_deps ()
Loading

0 comments on commit aad3d51

Please sign in to comment.