diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..61e03e9a --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,39 @@ +Help us to make autowrap better and become part of the OpenMS open-source community. + +This document is displayed because you either opened an issue or you want to provide your code as a pull request for inclusion into autowrap. Please take a look at the appropriate section below to find some details on how we handle this process. +When interacting with other developers, users or anyone else from our community, please adhere to +[the OpenMS CODE OF CONDUCT](https://github.com/OpenMS/OpenMS/blob/develop/CODE_OF_CONDUCT.md) + +# Reporting an Issue: + +You most likely came here to: + - report bugs or annoyances + - pose questions + - point out missing documentation + - request new features + +If you found a bug, e.g. an autowrap tool crashes during code generation, it is essential to provide some basic information: + - the autowrap version you are running + - the platform you are running autowrap on (Windows 10, ...) + - how you installed autowrap (e.g., from source, pip) + - a description on how to reproduce the bug + - relevant tool output (e.g., error messages) + - data to repoduce the bug (If possible as a GitHub gist. Other platforms like Dropbox, Google Drive links also work. If you can't share the data publicly please indicate this and we will contact you in private.) + +If you are an official OpenMS team meber: + - label your issue using github labels (e.g. as: question, defect) that indicate the type of issue and which components of autowrap (blue labels) are affected. The severity is usually assigned by OpenMS maintainers and used internally to e.g. indicate if a bug is a blocker for a new release. + +# Opening a Pull Request + +Before getting started we recommend taking a look at the OpenMS GitHub-Wiki: https://github.com/OpenMS/OpenMS/wiki#-for-developers + +Before you open the pull request, make sure you + - adhere to [our coding conventions](https://github.com/OpenMS/OpenMS/wiki/Coding-conventions) + - have [unit tests and functional tests](https://github.com/OpenMS/OpenMS/wiki/Write-tests) + - Have [proper documentation](https://github.com/OpenMS/OpenMS/wiki/Coding-conventions#doxygen) + +A core developer will review your changes to the main development branch (develop) and approve them (or ask for modifications). You may indicate the prefered reviewer(s) by adding links to them in a comment section (e.g., @cbielow @hendrikweisser @hroest @jpfeuffer @timosachsenberg) + +Also consider getting in contact with the core developers early. They might provide additional guidance and valuable information on how your specific aim is achieved. This might give you a head start in, for example, developing novel tools or algorithms. + +Happy coding! diff --git a/autowrap/CodeGenerator.py b/autowrap/CodeGenerator.py index e362081b..25824c90 100644 --- a/autowrap/CodeGenerator.py +++ b/autowrap/CodeGenerator.py @@ -149,7 +149,7 @@ def __init__(self, resolved, instance_mapping, pyx_target_path=None, self.all_classes = self.classes self.all_resolved = self.resolved if len(allDecl) > 0: - + self.all_typedefs = [] self.all_enums = [] self.all_functions = [] @@ -191,6 +191,7 @@ def setup_cimport_paths(self): pxd_dirs = set() for inst in self.all_classes + self.all_enums + self.all_functions + self.all_typedefs: + # Could this not simply be `for inst in self.all_resolved:` ? pxd_path = os.path.abspath(inst.cpp_decl.pxd_path) pxd_dir = os.path.dirname(pxd_path) pxd_dirs.add(pxd_dir) @@ -395,7 +396,7 @@ def create_wrapper_for_enum(self, decl, out_codes): def create_wrapper_for_class(self, r_class, out_codes): """Create Cython code for a single class - + Note that the cdef class definition and the member variables go into the .pxd file while the Python-level implementation goes into the .pyx file. This allows us to cimport these classes later across modules. @@ -718,7 +719,7 @@ def create_wrapper_for_method(self, cdcl, py_name, methods): assert len(methods) == 1, "overloaded operator/= not supported" code = self.create_special_itruediv_method(cdcl, methods[0]) return [code], Code.Code() - + if len(methods) == 1: code, typestubs = self.create_wrapper_for_nonoverloaded_method(cdcl, py_name, methods[0]) @@ -1155,7 +1156,7 @@ def create_special_mul_method(self, cdcl, mdcl): | return result """, locals()) return code - + def create_special_truediv_method(self, cdcl, mdcl): L.info(" create wrapper for operator/") assert len(mdcl.arguments) == 1, "operator/ has wrong signature" @@ -1197,7 +1198,7 @@ def create_special_add_method(self, cdcl, mdcl): | return result """, locals()) return code - + def create_special_sub_method(self, cdcl, mdcl): L.info(" create wrapper for operator-") assert len(mdcl.arguments) == 1, "operator- has wrong signature" @@ -1302,7 +1303,7 @@ def create_special_imul_method(self, cdcl, mdcl): self.top_level_code.append(tl) return code - + def create_special_itruediv_method(self, cdcl, mdcl): L.info(" create wrapper for operator/") assert len(mdcl.arguments) == 1, "operator/ has wrong signature" @@ -1535,7 +1536,7 @@ def create_foreign_cimports(self): we may be using in this compilation unit. Since we are passing objects as arguments quite frequently, we need to know about all other wrapped classes and we need to cimport them. - + E.g. if we have module1 containing classA, classB and want to access it through the pxd header, then we need to add: @@ -1555,7 +1556,7 @@ def create_foreign_cimports(self): for resolved in self.allDecl[module]["decls"]: # We need to import classes and enums that could be used in - # the Cython code in the current module + # the Cython code in the current module # use Cython name, which correctly imports template classes (instead of C name) name = resolved.name @@ -1578,7 +1579,7 @@ def create_foreign_cimports(self): else: code.add("from $mname cimport $name", locals()) - else: + else: L.info("Skip imports from self (own module %s)" % module) self.top_level_code.append(code) diff --git a/autowrap/DeclResolver.py b/autowrap/DeclResolver.py index 2fb1a6f9..39ebb945 100644 --- a/autowrap/DeclResolver.py +++ b/autowrap/DeclResolver.py @@ -79,7 +79,7 @@ class is the same as the name of the C++ class. - wrap-manual-memory: will allow the user to provide manual memory management of self.inst, therefore the class will not provide the automated __dealloc__ and inst - attribute (but their presence is still expected). + attribute (but their presence is still expected). This is useful if you cannot use the shared-ptr approach to store a reference to the C++ class (as with singletons for example). @@ -229,23 +229,24 @@ class ResolvedFunction(ResolvedMethod): pass +def resolve_decls_from_files(paths, root, num_processes = 1, cython_warn_level = 1): + if num_processes > 1: + return resolve_decls_from_files_multi_thread(paths, root, num_processes, cython_warn_level) + else: + return resolve_decls_from_files_single_thread(paths, root, cython_warn_level) + -def resolve_decls_from_files_single_thread(pathes, root, cython_warn_level = 1): +def resolve_decls_from_files_single_thread(paths, root, cython_warn_level = 1): decls = [] - for k, path in enumerate(pathes): + for k, path in enumerate(paths): full_path = os.path.join(root, path) - if k % 50 == 0: - logger.log(25, "parsing progress %s out of %s" % (k, len(pathes))) + if k % 50 == 0: + logger.log(25, "parsing progress %s out of %s" % (k, len(paths))) decls.extend(PXDParser.parse_pxd_file(full_path, cython_warn_level)) return _resolve_decls(decls) -def resolve_decls_from_files(pathes, root, num_processes = 1, cython_warn_level = 1): - if num_processes > 1: - return resolve_decls_from_files_multi_thread(pathes, root, num_processes, cython_warn_level) - else: - return resolve_decls_from_files_single_thread(pathes, root, cython_warn_level) -def resolve_decls_from_files_multi_thread(pathes, root, num_processes, cython_warn_level = 1): +def resolve_decls_from_files_multi_thread(paths, root, num_processes, cython_warn_level = 1): """Perform parsing with multiple threads This function distributes the work on `num_processes` processes and each @@ -257,20 +258,20 @@ def resolve_decls_from_files_multi_thread(pathes, root, num_processes, cython_wa CONCURRENT_FILES_PER_CORE = 10 pool = mp.Pool(processes=num_processes) - full_pathes = [os.path.join(root, path) for path in pathes] + full_paths = [os.path.join(root, path) for path in paths] decls = [] - while len(full_pathes) > 0: - n_work = len(full_pathes) + while len(full_paths) > 0: + n_work = len(full_paths) remaining = max(0, n_work - num_processes * CONCURRENT_FILES_PER_CORE) - args = [full_pathes.pop() for k in range(n_work - remaining)] - logger.log(25, "parsing progress %s out of %s with %s processes" % (len(pathes)-remaining, len(pathes), num_processes)) + args = [full_paths.pop() for k in range(n_work - remaining)] + logger.log(25, "parsing progress %s out of %s with %s processes" % (len(paths)-remaining, len(paths), num_processes)) parse_pxd_file_warn = partial(PXDParser.parse_pxd_file, warn_level = cython_warn_level) res = pool.map(parse_pxd_file_warn, args) for r in res: decls.extend(r) - return _resolve_decls(decls) + return _resolve_decls(decls) def resolve_decls_from_string(pxd_in_a_string): @@ -280,7 +281,7 @@ def resolve_decls_from_string(pxd_in_a_string): def _resolve_decls(decls): """ input: - class_decls ist list of instances of PXDParser.BaseDecl. + class_decls is a list of instances of PXDParser.BaseDecl. (contains annotations - about instance names for template parameterized classes - about inheritance of methods from other classes in class_decls diff --git a/autowrap/PXDParser.py b/autowrap/PXDParser.py index ecca1183..88eaf0be 100644 --- a/autowrap/PXDParser.py +++ b/autowrap/PXDParser.py @@ -150,7 +150,7 @@ def parse_line_annotations(node, lines): result[key] += value except Exception as e: raise ValueError("Cannot parse '{}'".format(line)) from e - + # check for multi line annotations after method declaration additional_annotations = _parse_multiline_annotations(lines[end:]) # add multi line doc string to result (overwrites single line wrap-doc, if exists) diff --git a/docs/README.md b/docs/README.md index bd01608f..08762238 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,7 +20,7 @@ class IntHolder { int i_; IntHolder(int i): i_(i) {}; IntHolder(const IntHolder & i): i_(i.i_) {}; - int add(const IntHolder & other) + int add(const IntHolder & other) { return i_ + other.i_; } @@ -44,7 +44,7 @@ which will generate Cython code that allows direct access to the public internal variable `i_` as well as to the two constructors and the public `add` method. -Compiling +Compiling ------------- To compile the above examples to .pyx and .cpp files, go to the `./example` @@ -129,7 +129,7 @@ you could generate the following .pyx file and run autowrap (see below for a lis # TemplatedWithDouble := TemplateClassName[double] # # wrap-doc: - # TemplatedClass for double and float, + # TemplatedClass for double and float, # useful for processing foobars TemplateType myInner_ @@ -164,7 +164,7 @@ directives are: - `wrap-constant`: Useful for constant properties that should have `__get__` but no `__set__` - `wrap-upper-limit`: Can be used to check input argument of type int (e.g. for operator[]) to make sure the input integer does not exceed the limit - `wrap-cast`: Wrap casting functions such as `double operator()(MyObject)` -- `wrap-inherits`: Inherit methods from parent classes (see below for example) +- `wrap-inherits`: Inherit methods from parent classes (see below for example) - `wrap-instances`: Wrap specific template instances (see below for example) - `wrap-manual-memory`: will allow the user to provide manual memory management of `self.inst`, therefore the class will not provide the automated @@ -179,7 +179,7 @@ directives are: - `wrap-with-no-gil`: Autowrap will release the GIL (Global interpreter lock) before calling this method, so that it does not block other Python threads. It is advised to release the GIL for long running, expensive calls into - native code which does not manipulate python objects. + native code which does not manipulate python objects. ### Method Directives @@ -200,10 +200,10 @@ Example declaration for releasing the GIL (in the pxd): Example for multiple wrap statements in a method directive: ``` - size_t countSomething(libcpp_vector[double] inpVec) nogil # wrap-attach:Counter wrap-as:count + size_t countSomething(libcpp_vector[double] inpVec) nogil # wrap-attach:Counter wrap-as:count ``` - -In addition you have to declare the function as nogil. For further details see + +In addition you have to declare the function as nogil. For further details see http://docs.cython.org/src/userguide/external_C_code.html @@ -248,24 +248,24 @@ taken with priority. # wrap-doc: # Multi line docstring for class # with indentation - # + # # and empty line size_t count(libcpp_vector[double] inpVec) # wrap-doc:Single line docstring that will be overwritten by multi line wrap-doc # wrap-doc: # Multi line docstring for method # with indentation - # + # # and empty line ``` Test examples ------------- -The tests provide several examples on how to wrap tricky C++ constructs, see for example +The tests provide several examples on how to wrap tricky C++ constructs, see for example - [minimal.pxd](../tests/test_files/minimal.pxd) for a class example that uses static methods, pointers, operators ([], +=, \*, etc) and iterators -- [libcpp_stl_test.pxd](../tests/test_files/libcpp_stl_test.pxd) for examples using the STL +- [libcpp_stl_test.pxd](../tests/test_files/libcpp_stl_test.pxd) for examples using the STL - [libcpp_test.pxd](../tests/test_files/libcpp_test.pxd) for a set of C++ functions that use abstract base classes - [libcpp_utf8_string_test.pxd](../tests/test_files/libcpp_utf8_string_test.pxd) for an example using UTF8 strings - [templated.pxd](../tests/test_files/templated.pxd) for an example using templated classes @@ -293,3 +293,120 @@ autowrap and split up the compilation into multiple units where each unit contains some of the projects classes. For an example on how to do this, see `./tests/test_full_library.py`. +High Level Overview +-------------- +Broadly speaking, the autowrap process consists of two steps: +1. `DeclResolver` uses the `PXDParser` to parse files +2. `CodeGenerator` generates code + +## Parsing + +The process is kicked off by the autowrap `parse` function. This method takes +a list of files and a root directory as required parameters and optional parameters +for designating the number of prcesses to use and the cython warning level. The +method calls `DeclResolver#resolve_decls_from_files` with the given parameters. + +Depending on the number of processes passed to the `parse` function, the +`resolve_decls_from_files` method calls either a single or multi threaded method +which calls the `PXDParser#parse_pxd_file` method on each given file. + +The method `parse_pxd_file` uses the file path passed ot it to build Cython +`Context` and `Pipeline` objects. These objects are used to create a `root` +object. This object is passed to an `iter_bodies` method which extracts all the +Cython `node` objects present in the .pyx file. These objects are used as keys +to find the appropriate `BaseDecl` class for each node. These `BaseDecl` objects +are put into a Python list and passed back to the calling `DeclResolver` method (either +the `resolve_decls_from_files_single_thread` or `resolve_decls_from_files_multi_thread` +method), which then passes `decls` to the private method `DeclResolver#_resolve_decls`. + +The `_resolve_decls` method starts by organizing each `BaseDecl` object by its +specific subclass (e.g., `CTypeDefDecl`, `EnumDecl`, etc). The method then handles +the data processing specific to each type of declaration and puts them into a list +of `Resolved*` objects (`ResolvedEnum` for example). The method also creates a dictionary +object called `instance_mapping` which contains class instantiations. Finally, the +`_resolve_decls` method returns a tuple of the resolved declaration objects and +`instance_mapping` dict. + +## Code Generation + +### Building a `CodeGenerator` object + +The second part of the process is to call the autowrap `generate_code` method. This +method takes the list of resolved declaration objects and `instance_mapping` dict +generated from the `parse` method. The `generate_code` method also requires a +`target` argument for designating the name and path of the .pyx file to be generated. + +The `gerneate_code` method builds a `CodeGenerator` object with its inputs. This object's +constructor first does some preprocessing to file paths and set other configurations. +Then, it gets all the classes, enums, functions, and typedefs of resolved declarations +and puts them into a Python list called `resolved`. It then builds an `instance_mapping` +instance attribute from both the `instance_mapping` argument passed to it, as well as +instances extracted from the resolved typedefs. + +The constructor then checks for items in the `allDecl` argument. This argument will +have items in it if the user is parsing many files instead of just one. When this is +the case, the `CodeGenerator` constructor builds an `all_resolved` list which contains +instances of resolved classes, enums, functions, and typdefs from both the `resolved` +list, and the `allDecl` object. + +Finally, the `CodeGenerator` constructor instantiates an instance variable called `cr`, +which stands for _container registry_ by calling the +`ConversionProvider#setup_converter_registry` method. This method takes lists of classes +and enums to wrap, as well as an instance map. From these values, the method creates +a new instance of the `ConverterRegistry` method and calls its `register` method on +each supported type of converter supported by autowrap. + +The `register` method builds a hash containing the base type of declarations to be +converted as keys and maps the appropriate converter class to that key. For example, +a `void` declaration maps to a `VoidConverter`. + +### Creating a .pyx file +Once the `CodeGenerator` object has been built, the `generate_code` method t hen calls +the object's `create_pyx_file` method. This method is responsible for actually generating +the Cython code. It starts by setting up the cimport paths by calling its own +`setup_cimport_paths` method. This method finds the exact location of each .pxd file +of the `cpp_decl` objects of each resolved declaration object of the `CodeGenerator` class. + +This method sets the `CodeGenerator` object's `pxd_import_path` instance attribute to the +location of the .pxd file. The method also ensures that all .pxd files are located in one +directory, then sets the `pxd_dir` instance attribute as that directory. + +With the cimport paths set, the `create_pyx_file` can then create the cimports by calling +the `create_cimports` method. This method first imports standard and extra modules if +they were designated in the `CodeGenerator` constructor. + +Then, the `create_cimports` method instantiates a `Code` object. Then, the method iterates +through each item in its `all_resolved` attribute and adds lines of code to the `Code` +object, which interpolates local variables into parts of the pseudo code strings prefixed +with a `$` character. Finally, the `create_cimports` method appends that code to the +`CodeGenerator`'s `top_level_code` attribute. + +With the cimports created, the `create_pyx_file` method creates the foreign cimports, +other classes created by autowrap, by callings the `create_foreign_cimports` method. +This method creates a `Code` object and generates a line of code for every extra +module that needs to be imported. + +The next step in the `create_pyx_file` is to create includes, which it does by calling +the `create_includes` method. This method again builds a `Code` object that appends a +line for including a cdef line in the `top_level_code` attribute. + +Next, the `create_pyx_file` method calls the appropriate wrapper method for classes, +enums, and free functions. These methods generate the Cython code necessary to +wrap those items and does so by creating `Code` objects and appending interpolated +strings into them. + +As one last bit of preparation, the `create_pyx_file` method then resolves any +extra classes not resolved in the previous steps. + +Finally, the `create_pyx_file` begins writing code to the .pyx file. It does this +by calling the `render` function on each `Code` object in its `top_level_code` and +`top_level_pyx_code` attributes. This `render` method builds a list of Cython code +expressions with proper indentation. + +Finally, the `create_pyx_file` method creates files objects and writes the generated +pyi, pyx, and pxd code to their relevant files. + +Finally, back in the autowrap `__init__` file, the `generate_code` method gathers +the include directories by calling `CodeGenerator`'s `get_include_dirs` method. This +method returns a list of packaged resources from the Cython code generated in the previous +steps. This value is then returned by the `generate_code` method, completing the second step.