From 3f36e19d552cbbf2bdcf79c34850906f4d061013 Mon Sep 17 00:00:00 2001 From: nmoriniere-gpfw Date: Sat, 30 Mar 2024 16:49:00 +0100 Subject: [PATCH] Add custom Python import machinery (#16) --- Makefile | 1 + Manifest | 1 + _lib/lmake/custom_importer.py | 269 ++++++++++++++++++++++++++++++++++ _lib/lmake/rules.src.py | 4 + 4 files changed, 275 insertions(+) create mode 100644 _lib/lmake/custom_importer.py diff --git a/Makefile b/Makefile index 8ed2f37b..c11d2647 100644 --- a/Makefile +++ b/Makefile @@ -215,6 +215,7 @@ LMAKE_SERVER_PY_FILES := \ $(LIB)/lmake/__init__.py \ $(LIB)/lmake/auto_sources.py \ $(LIB)/lmake/import_machinery.py \ + $(LIB)/lmake/custom_importer.py \ $(LIB)/lmake/rules.py \ $(LIB)/lmake/sources.py \ $(LIB)/lmake/utils.py \ diff --git a/Manifest b/Manifest index ba89b509..56a2d21a 100644 --- a/Manifest +++ b/Manifest @@ -9,6 +9,7 @@ _bin/ut_launch _bin/version _lib/lmake/__init__.py _lib/lmake/auto_sources.py +_lib/lmake/custom_importer.py _lib/lmake/import_machinery.py _lib/lmake/rules.src.py _lib/lmake/sources.src.py diff --git a/_lib/lmake/custom_importer.py b/_lib/lmake/custom_importer.py new file mode 100644 index 00000000..7bb3263a --- /dev/null +++ b/_lib/lmake/custom_importer.py @@ -0,0 +1,269 @@ +''' + Handle local modules such that : + - we do not access dirs, only candidate files + - no caching as this is anti-lmake by nature + - accept to try modules in non-existent directories as they can be dynamically created + System modules (not found in repository) and modules imported by system modules are processed as is. + Besides that, the semantic is conserved except that : + - sys.path is split into sys.path for system and sys.local_path for modules in repository + - sys.local_path is augmented with the content of the environment variable PYTHON_LOCAL_PATH + - sys.import_map is augmented with the content of the environment variable PYTHON_IMPORT_MAP + - this split is performed at the time this module is executed, but both sys.path and sys.local_path + may be updated afterwards, however the split semantic must be enforced (system modules in sys.path + and repository modules in sys.local_path) + - sys.import_map contains a dict mapping module/package names to physical file/dir + - keys are names in dot notation (e.g. 'a.b') + - values are physical file/dir or path (e.g. 'products/a/tag/b') + - keys can be specialized, for example there may be an entry for 'a.b' and another for 'a.b.c' + - modules are seached with and without translation + - e.g. the translated can contain computed modules and the untranslated can contain python modules + - for each search mapped to , the following files are tried in order : + - .so + - .so.py + - .py + - /__init__.py + - idem with instead of + - else, module is deemed to be a so called namespace package that need not exist on disk + ModuleNotFoundError is reported if a namespace module is about to be returned as this is non-sens. +''' + +# because this file may be exec'ed rather than imported (as the import machinery may not be reliable +# before execution), code is protected against multiple execution + +import sys +import os +import site + +import os.path as osp + +from os import environ + +def mkLocal(fileName) : + if not fileName : return '' + if fileName[0]!='/' : return fileName + # ROOT_DIR is typically set after import of this module as importer is meant to be imported very early. + try : root = environ['ROOT_DIR'] + except KeyError : root = os.getcwd() + rootSlash = root+'/' + if (fileName+'/').startswith(rootSlash) : return fileName[len(rootSlash):] +def isLocal(fileName) : return mkLocal(fileName) is not None + +if 'importer_done' not in sys.__dict__ : + sys.importer_done = True + + def fromSystemModule() : + # determine caller of __import__ + # caller is the deepest non frozen frame other than this very module + # as the Python import machinery is frozen + frame = sys._getframe() + here = frame.f_code.co_filename + while frame : + fileName = frame.f_code.co_filename + if fileName!=here and not re.match('^$',fileName) : + return not isLocal(fileName) + frame = frame.f_back + assert False,'spontaneous call to import' + + def splitPaths() : + if not any(isLocal(f) for f in sys.path) : return # fast path + sysPath = sys.path + sys.path = [] + for f in sysPath : + lf = mkLocal(f) + if lf is None : sys.path .append(f) + elif lf not in sys.local_path : sys.local_path.append(lf) + + if 'local_path' not in sys.__dict__ : sys.local_path = [] + localPath = environ.get('PYTHON_LOCAL_PATH') + if localPath : # stupid split gives [''] instead of [] for an empty string + for f in localPath.split(':') : + sys.local_path.append(f) + splitPaths() + + # add top level dir if not already present + if 'import_map' not in sys.__dict__ : sys.import_map = {} + importMap = environ.get('PYTHON_IMPORT_MAP') + if importMap : # stupid split gives [''] instead of [] for an empty string + for modMap in importMap.split(':') : + name,file = modMap.split('=',1) + sys.import_map[name] = file + + import re + import builtins + import importlib.machinery + from importlib.metadata import MetadataPathFinder + + origImport = builtins.__import__ + def __import__(name,globals=None,locals=None,fromlist=(),level=0) : + def chkModule(m,p=False) : + try : l = m.__loader__ # if there is no loader, next condition does not make sense + except: return + if isinstance(l,NamespaceLoader) : + raise ModuleNotFoundError('module %s not found\nglobal path : %s\nlocal path : %s'%( + m + , sys.path + , sys.local_path + )) + splitPaths() + mod = origImport(name,globals,locals,fromlist,level) + # catch return namespace modules as these are unusable in code and mask ModuleNotFoundError's + # in case of attribute error, it means we do not recognize a namespace module, and we let go + # be careful that for the last form, we must test all values, so we cannot put a global try/except + if not fromlist : # form : import foo.bar.baz, mod is foo and we must verify baz is not a namespace module + v = mod + try : + for n in name.split('.')[1:] : v = getattr(v,n) + chkModule(v) # v is necessarily a module + except AttributeError : pass + elif fromlist[0]=='*' : # form : from foo.bar.baz import *, mod is baz and we must verify it is not a namespace module + try : chkModule(mod) + except AttributeError : pass + else : # form : from foo.bar import baz,zee, mod is bar and we must verify baz nor zee are namespace modules + for n in fromlist : + try : chkModule(getattr(mod,n)) # if v does not exist, this is not really our problem + except AttributeError : pass + return mod + builtins.__import__ = __import__ + + # reimplement import_module to call __import__ if called from user code (dont touch system code, it is too complex) + origImportModule = importlib.import_module + def import_module(name,package=None) : + if fromSystemModule() : return origImportModule(name,package) + for lvl in range(len(name)) : + if name[lvl]!='.' : break + else : raise NameError('cannot import module whose name is only dots') + if lvl : + if not package : raise TypeError('cannot do relative import of %s without a package'%name) + bits = package.rsplit('.',lvl-1) + if len(bits)