Move all information related to a population item into a dedicated cl…

…ass (#248)

Add Individual and DefaultIndividual classes defining the API of
such individuals and an actual implementation of it maintaining
file-based populations. The patch doesn't change any semantics, but
separates ideas.

grammarinator/runtime/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2017-2023 Renata Hodovan, Akos Kiss.
+# Copyright (c) 2017-2024 Renata Hodovan, Akos Kiss.
 #
 # Licensed under the BSD 3-Clause License
 # <LICENSE.rst or https://opensource.org/licenses/BSD-3-Clause>.
@@ -12,6 +12,6 @@
 from .generator import AlternationContext, Generator, QuantifiedContext, QuantifierContext, RuleSize, UnlexerRuleContext, UnparserRuleContext
 from .listener import Listener
 from .model import Model
-from .population import Population
+from .population import Annotations, Individual, Population
 from .rule import ParentRule, Rule, UnlexerRule, UnparserRule, UnparserRuleAlternative, UnparserRuleQuantified, UnparserRuleQuantifier
 from .serializer import simple_space_serializer

grammarinator/runtime/population.py

@@ -1,10 +1,13 @@
-# Copyright (c) 2023 Renata Hodovan, Akos Kiss.
+# Copyright (c) 2023-2024 Renata Hodovan, Akos Kiss.
 #
 # Licensed under the BSD 3-Clause License
 # <LICENSE.rst or https://opensource.org/licenses/BSD-3-Clause>.
 # This file may not be copied, modified, or distributed except
 # according to those terms.
 
+from .rule import ParentRule, UnlexerRule, UnparserRule, UnparserRuleAlternative, UnparserRuleQuantifier
+
+
 class Population:
     """
     Abstract base class of populations that store test cases in tree form (i.e.,
@@ -22,16 +25,13 @@ def __bool__(self):
         """
         raise NotImplementedError()
 
-    def add_individual(self, root, annotations=None, path=None):
+    def add_individual(self, root, path=None):
         """
         Add a tree to the population.
 
         Raises :exc:`NotImplementedError` by default.
 
         :param ~grammarinator.runtime.Rule root: Root of the tree to be added.
-        :param object annotations: Data to be stored along the tree, if
-            possible. No assumption should be made about the structure or the
-            contents of the data, it should be treated as opaque.
         :param str path: The pathname of the test case corresponding to the
             tree, if it exists. May be used for debugging.
         """
@@ -43,9 +43,125 @@ def select_individual(self):
 
         Raises :exc:`NotImplementedError` by default.
 
-        :return: Root of the selected tree, and any associated information that
-            was stored along the tree when it was added (if storing/restoring
-            that information was possible).
-        :rtype: tuple[~grammarinator.runtime.Rule,object]
+        :return: A single individual of the population.
+        :rtype: Individual
+        """
+        raise NotImplementedError()
+
+
+class Individual:
+    """
+    Abstract base class of population individuals.
+    """
+    def __init__(self, name):
+        """
+        :param str name: Name or identifier of the individual.
+        """
+        self.name = name
+        self._annot = None
+
+    @property
+    def root(self):
+        """
+        Return the root node of the tree of the individual.
+
+        :return: Root of the tree.
+        :rtype: ~grammarinator.runtime.Rule
         """
         raise NotImplementedError()
+
+    @property
+    def annotations(self):
+        """
+        Return the associated annotations if available, otherwise compute them immediately.
+
+        :return: The annotations associated with the tree.
+        :rtype: Annotations
+        """
+        if not self._annot:
+            self._annot = Annotations(self.root)
+        return self._annot
+
+
+class Annotations:
+    """
+    Class for calculating and managing additional metadata needed by the
+    mutators, particularly to enforce size constraints and facilitate node
+    filtering by rule types.
+    """
+
+    def __init__(self, root):
+        """
+        :param ~grammarinator.runtime.Rule root: Root of the tree to be annotated.
+        """
+        def _annotate(current, level):
+            nonlocal current_rule_name
+            self.node_levels[current] = level
+
+            if isinstance(current, (UnlexerRule, UnparserRule)):
+                if current.name and current.name != '<INVALID>':
+                    current_rule_name = (current.name,)
+                    if not isinstance(current, UnlexerRule) or not current.immutable:
+                        if current_rule_name not in self.rules_by_name:
+                            self.rules_by_name[current_rule_name] = []
+                        self.rules_by_name[current_rule_name].append(current)
+                else:
+                    current_rule_name = None
+            elif current_rule_name:
+                if isinstance(current, UnparserRuleQuantifier):
+                    node_name = current_rule_name + ('q', current.idx,)
+                    if node_name not in self.quants_by_name:
+                        self.quants_by_name[node_name] = []
+                    self.quants_by_name[node_name].append(current)
+                elif isinstance(current, UnparserRuleAlternative):
+                    node_name = current_rule_name + ('a', current.alt_idx,)
+                    if node_name not in self.alts_by_name:
+                        self.alts_by_name[node_name] = []
+                    self.alts_by_name[node_name].append(current)
+
+            self.node_depths[current] = 0
+            self.token_counts[current] = 0
+            if isinstance(current, ParentRule):
+                for child in current.children:
+                    _annotate(child, level + 1)
+                    self.node_depths[current] = max(self.node_depths[current], self.node_depths[child] + 1)
+                    self.token_counts[current] += self.token_counts[child] if isinstance(child, ParentRule) else child.size.tokens + 1
+
+        current_rule_name = None
+        self.rules_by_name = {}
+        self.alts_by_name = {}
+        self.quants_by_name = {}
+        self.node_levels = {}
+        self.node_depths = {}
+        self.token_counts = {}
+        _annotate(root, 0)
+
+    @property
+    def rules(self):
+        """
+        Get nodes created from rule nodes.
+
+        :return: List of rule nodes.
+        :rtype: list[~grammarinator.runtime.Rule]
+        """
+        return [rule for rules in self.rules_by_name.values() for rule in rules]
+
+    @property
+    def alts(self):
+        """
+        Get nodes created from alternatives.
+
+        :return: List of alternative nodes.
+        :rtype: list[UnparserRuleAlternative]
+        """
+        return [alt for alts in self.alts_by_name.values() for alt in alts]
+
+    @property
+    def quants(self):
+        """
+        Get nodes created from quantified expressions.
+
+        :return: List of quantifier nodes.
+        :rtype: list[UnparserRuleQuantifier]
+        """
+        return [quant for quants in self.quants_by_name.values() for quant in quants]

grammarinator/tool/__init__.py

@@ -5,7 +5,7 @@
 # This file may not be copied, modified, or distributed except
 # according to those terms.
 
-from .default_population import DefaultPopulation
+from .default_population import DefaultIndividual, DefaultPopulation
 from .generator import DefaultGeneratorFactory, GeneratorFactory, GeneratorTool
 from .parser import ParserTool
 from .processor import ProcessorTool

grammarinator/tool/default_population.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2023 Renata Hodovan, Akos Kiss.
+# Copyright (c) 2023-2024 Renata Hodovan, Akos Kiss.
 #
 # Licensed under the BSD 3-Clause License
 # <LICENSE.rst or https://opensource.org/licenses/BSD-3-Clause>.
@@ -13,7 +13,7 @@
 from os.path import basename, join
 from uuid import uuid4
 
-from ..runtime import Population, Rule
+from ..runtime import Annotations, Individual, Population, Rule
 from .tree_codec import AnnotatedTreeCodec, PickleTreeCodec
 
 logger = logging.getLogger(__name__)
@@ -45,7 +45,7 @@ def __bool__(self):
         """
         return len(self._files) > 0
 
-    def add_individual(self, root, annotations=None, path=None):
+    def add_individual(self, root, path=None):
         """
         Save the tree to a new file. The name of the tree file is determined
         based on the pathname of the corresponding test case. From the pathname
@@ -62,22 +62,63 @@ def add_individual(self, root, annotations=None, path=None):
             path = type(self).__name__
 
         fn = join(self._directory, f'{path}.{uuid4().hex}.{self._extension}')
-        with open(fn, 'wb') as f:
-            if isinstance(self._codec, AnnotatedTreeCodec):
-                f.write(self._codec.encode_annotated(root, annotations))
-            else:
-                f.write(self._codec.encode(root))
+        self._save(fn, root)
         self._files.append(fn)
 
     def select_individual(self):
         """
-        Randomly select an individual of the population.
+        Randomly select an individual of the population and create a
+        DefaultIndividual instance from it.
+
+        :return: DefaultIndividual instance created from a randomly selected population item.
+        :rtype: DefaultIndividual
         """
-        fn = random.sample(self._files, k=1)[0]
+        return DefaultIndividual(self, random.sample(self._files, k=1)[0])
+
+    def _save(self, fn, root):
+        with open(fn, 'wb') as f:
+            if isinstance(self._codec, AnnotatedTreeCodec):
+                f.write(self._codec.encode_annotated(root, Annotations(root)))
+            else:
+                f.write(self._codec.encode(root))
+
+    def _load(self, fn):
         with open(fn, 'rb') as f:
             if isinstance(self._codec, AnnotatedTreeCodec):
                 root, annot = self._codec.decode_annotated(f.read())
             else:
                 root, annot = self._codec.decode(f.read()), None
             assert isinstance(root, Rule), root
         return root, annot
+
+
+class DefaultIndividual(Individual):
+    """
+    Individual subclass presenting a file-based population individual, which
+    maintains both the tree and the associated annotations. It is responsible
+    for loading and storing the tree and its annotations with the appropriate
+    tree codec in a lazy manner.
+    """
+
+    def __init__(self, population, name):
+        """
+        :param DefaultPopulation population: The population this individual
+            belongs to.
+        :param str name: Path to the encoded tree file.
+        """
+        super().__init__(name)
+        self._population = population
+        self._root = None
+
+    @property
+    def root(self):
+        """
+        Get the root of the tree. Return the root if it is already loaded,
+        otherwise load it immediately.
+
+        :return: The root of the tree.
+        :rtype: ~grammarinator.runtime.Rule
+        """
+        if not self._root:
+            self._root, self._annot = self._population._load(self.name)
+        return self._root