Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(interpolation): allow any versions of t4 dataset and make timestamp uinque #147

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

fix(interpolation): allow any versions of t4 dataset and make timestamp uinque #147

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 24 additions & 7 deletions perception_dataset/t4_dataset/data_interpolator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@
import multiprocessing as mp
import os
import os.path as osp
import re
from secrets import token_hex
import subprocess
from typing import Any, Dict, List, Optional, Tuple

import matplotlib.pyplot as plt
Expand All @@ -21,7 +23,6 @@

from perception_dataset.abstract_converter import AbstractConverter
from perception_dataset.utils.logger import configure_logger
import subprocess


class DataInterpolator(AbstractConverter):
Expand Down Expand Up @@ -90,20 +91,31 @@ def convert(self) -> None:
def _convert_single(self, data_root: str) -> None:
self.logger.info(f"Start processing: {data_root}")

# an integer larger than or eaqual to 0
version_pattern = re.compile(r"^\d+$")

version_dirs = [d for d in os.listdir(data_root) if version_pattern.match(d)]

dataset_id = osp.basename(data_root)

if version_dirs:
version_id = sorted(version_dirs, key=int)[-1]
data_root = osp.join(data_root, version_id)
nusc = NuScenes(version="annotation", dataroot=data_root, verbose=False)

output_path = osp.join(self._output_base, dataset_id)
os.makedirs(output_path, exist_ok=True)

command: list[str] = ["rsync", "-ahuv"]
if self.copy_excludes is not None:
assert isinstance(self.copy_excludes, list), f"Unexpected type of excludes: {type(self.copy_excludes)}"
assert isinstance(
self.copy_excludes, list
), f"Unexpected type of excludes: {type(self.copy_excludes)}"
for e in self.copy_excludes:
command.extend(["--exclude", e])
command.extend([f"{data_root}", f"{self._output_base}"])
command.extend([f"{data_root}/", f"{output_path}"])
subprocess.run(command)

nusc = NuScenes(version="annotation", dataroot=data_root, verbose=False)

all_samples, all_sample_data = self.interpolate_sample(nusc)
self.logger.info("Finish interpolating sample and sample data")

Expand Down Expand Up @@ -178,7 +190,9 @@ def interpolate_sample(
for sd_record in all_sample_data:
if sd_record["is_key_frame"]:
continue
filename: str = osp.splitext(osp.basename(sd_record["filename"].replace(".pcd.bin", ".pcd")))[0]
filename: str = osp.splitext(
osp.basename(sd_record["filename"].replace(".pcd.bin", ".pcd"))
)[0]
if filename in interpolated_samples_dict.keys():
sample_token: str = interpolated_samples_dict[filename]["token"]
sd_record["sample_token"] = sample_token
Expand Down Expand Up @@ -223,11 +237,14 @@ def interpolate_sample_annotation(
accelerations = []
original_timestamps = []
for ann in sample_anns:
ann_timestamp = original_timestamps_info[ann["sample_token"]]
if ann_timestamp in original_timestamps:
continue
original_timestamps.append(ann_timestamp)
translations.append(ann["translation"])
rotations.append(ann["rotation"])
velocities.append(ann["velocity"])
accelerations.append(ann["acceleration"])
original_timestamps.append(original_timestamps_info[ann["sample_token"]])

# skip if there is only one annotation because interpolation is unavailable
if len(original_timestamps) < 2:
Expand Down
Loading