Truncate seed if too large during seqio caching jobs.

PiperOrigin-RevId: 633765044
google · May 15, 2024 · f09ed20 · f09ed20
1 parent 20e5f45
commit f09ed20
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/seqio/beam_utils.py b/seqio/beam_utils.py
@@ -136,6 +136,8 @@ def _emit_examples(self, shard: Tuple[int, str]):
       shard_preprocessors_seed = int.from_bytes(md5_digest, "little") + (
           self._preprocessors_seed or 0
       )
+    # Truncate if still a large number.
+    shard_preprocessors_seed %= self._int64_max
 
     ds = task.source.get_dataset(
         split=self._split,