style(conf): set train and val with diff config path in conf.

for afterward easy ssl train

Author: Kin-Zhang

1_train.py

@@ -31,14 +31,14 @@ def main(cfg):
     pl.seed_everything(cfg.seed, workers=True)
     output_dir = HydraConfig.get().runtime.output_dir
 
-    train_dataset = HDF5Dataset(cfg.dataset_path + "/train")
+    train_dataset = HDF5Dataset(cfg.train_data)
     train_loader = DataLoader(train_dataset,
                               batch_size=cfg.batch_size,
                               shuffle=True,
                               num_workers=cfg.num_workers,
                               collate_fn=collate_fn_pad,
                               pin_memory=True)
-    val_loader = DataLoader(HDF5Dataset(cfg.dataset_path + "/val"),
+    val_loader = DataLoader(HDF5Dataset(cfg.val_data),
                             batch_size=cfg.batch_size,
                             shuffle=False,
                             num_workers=cfg.num_workers,

conf/config.yaml

@@ -7,7 +7,9 @@ slurm_id: 00000
 
 wandb_mode: offline # [offline, disabled, online]
 wandb_project_name: deflow
-dataset_path: /home/kin/data/av2/preprocess/sensor
+
+train_data: /home/kin/data/dataset/av2/preprocess/sensor/train
+val_data: /home/kin/data/dataset/av2/preprocess/sensor/val
 
 output: ${model.name}-${slurm_id}
 

dataprocess/extract_av2.py

@@ -265,10 +265,14 @@ def main(
     av2_type: str = "sensor",
     data_mode: str = "test",
     mask_dir: str = "/home/kin/data/av2/3d_scene_flow",
-    nproc: int = (multiprocessing.cpu_count() - 1)
+    nproc: int = (multiprocessing.cpu_count() - 1),
+    only_index: bool = False,
 ):
     data_root_ = Path(argo_dir) / av2_type/ data_mode
     output_dir_ = Path(output_dir) / av2_type / data_mode
+    if only_index:
+        create_reading_index(output_dir_)
+        return
     output_dir_.mkdir(exist_ok=True, parents=True)
     process_logs(data_root_, output_dir_, nproc)
     create_reading_index(output_dir_)