fix bugs, remove text-utils-grammar and text-utils-prefix

ad-freiburg · Nov 22, 2024 · b6e14f6 · b6e14f6
1 parent 1bf0d76
commit b6e14f6
Show file tree

Hide file tree

Showing 66 changed files with 58,108 additions and 139,537 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 publish = false
-name = "text-utils-rs"
+name = "text-utils"
 version = "0.6.0"
 edition = "2021"
 
@@ -13,7 +13,7 @@ log = "0.4"
 env_logger = "0.11"
 rayon = "1.8"
 indicatif = { version = "0.17", features = ["rayon"] }
-pyo3 = { version = "0.22", features = ["anyhow", "extension-module", "abi3-py310", "auto-initialize", "macros"] }
+pyo3 = { version = "0.22", features = ["anyhow", "extension-module", "abi3-py310", "auto-initialize"]}
 itertools = "0.13"
 rand = "0.8"
 rand_distr = "0.4"
@@ -30,8 +30,6 @@ num_cpus = "1.14"
 numpy = "0.22"
 clap = { version = "4", features = ["derive"] }
 tokenizers = "0.20"
-text-utils-prefix = { path = "text-utils-prefix" }
-text-utils-grammar = { path = "text-utils-grammar" }
 lru = "0.12"
 
 [dev-dependencies]

diff --git a/pyproject.toml b/pyproject.toml
@@ -29,7 +29,8 @@ dependencies = [
     "tensorboard>=2.16.0",
     "flask>=3.0",
     "requests>=2.31",
-    "termcolor>=2.4.0"
+    "termcolor>=2.4.0",
+    "grammar-utils>=0.1.0"
 ]
 
 [project.scripts]

diff --git a/python/text_utils/__init__.py b/python/text_utils/__init__.py
@@ -9,15 +9,5 @@
     windows,
     metrics,
     unicode,
-    continuations,
-    grammar
-)
-from text_utils import (
-    api,
-    inference,
-    modules,
-    logging,
-    distributed,
-    mask,
-    io
 )
+from text_utils import api, inference, modules, logging, distributed, mask, io
diff --git a/python/text_utils/api/trainer.py b/python/text_utils/api/trainer.py
@@ -921,7 +921,7 @@ def train_slurm(cls, work_dir: str, experiment_dir: str, config_path: str):
 
     @classmethod
     def train_local(
-        cls, work_dir: str, experiment_dir: str, config_path: str, profile: bool
+        cls, work_dir: str, experiment_dir: str, config_path: str, profile: bool = False
     ):
         logging.setup_logging()
         logger = logging.get_logger("LOCAL_INITIALIZATION")

diff --git a/python/text_utils/constraints.py b/python/text_utils/constraints.py