docs(README): update deflow bib cite. add seflow arch

update o3d_view scripts also.

feat(env): install CUDA through conda to free users.

Author: Kin-Zhang

README.md

@@ -6,8 +6,10 @@ SeFlow: A Self-Supervised Scene Flow Method in Autonomous Driving
 [poster comming soon]
 [video coming soon]
 
+![](assets/docs/seflow_arch.png)
+
 2024/07/16 17:18: Most of codes already uploaded and tested. You can to try training directly by [downloading](https://zenodo.org/records/12751363) demo data or pretrained weight for evaluation. 
-The process script will be public when the paper published. 
+The process script will be public when the paper is published officially. 
 
 Pre-trained weights for models are available in [Zenodo](https://zenodo.org/records/12751363) link. Check usage in [2. Evaluation](#2-evaluation) or [3. Visualization](#3-visualization).
 
@@ -55,10 +57,7 @@ cd SeFlow && mamba env create -f environment.yaml
 CUDA package (need install nvcc compiler), the compile time is around 1-5 minutes:
 ```bash
 mamba activate seflow
-# change it if you use different cuda version (I tested 11.3, 11.4, 11.7, 11.8 all works)
-export PATH=/usr/local/cuda-11.7/bin:$PATH
-export LD_LIBRARY_PATH=/usr/local/cuda-11.7/lib64:$LD_LIBRARY_PATH
-
+# CUDA already install in python environment. I also tested others version like 11.3, 11.4, 11.7, 11.8 all works
 cd assets/cuda/mmcv && python ./setup.py install && cd ../../..
 cd assets/cuda/chamfer3D && python ./setup.py install && cd ../../..
 ```
@@ -179,11 +178,13 @@ https://github.com/user-attachments/assets/f031d1a2-2d2f-4947-a01f-834ed1c146e6
   journal={arXiv preprint arXiv:2407.01702},
   year={2024}
 }
-@article{zhang2024deflow,
+@inproceedings{zhang2024deflow,
   author={Zhang, Qingwen and Yang, Yi and Fang, Heng and Geng, Ruoyu and Jensfelt, Patric},
-  title={DeFlow: Decoder of Scene Flow Network in Autonomous Driving},
-  journal={arXiv preprint arXiv:2401.16122},
-  year={2024}
+  booktitle={2024 IEEE International Conference on Robotics and Automation (ICRA)}, 
+  title={{DeFlow}: Decoder of Scene Flow Network in Autonomous Driving}, 
+  year={2024},
+  pages={2105-2111},
+  doi={10.1109/ICRA57147.2024.10610278}
 }
 ```
 

assets/docs/seflow_arch.png

@@ -8,8 +8,8 @@ slurm_id: 00000
 wandb_mode: offline # [offline, disabled, online]
 wandb_project_name: seflow
 
-train_data: /home/kin/data/dataset/av2/demo/preprocess_v2/sensor/train
-val_data: /home/kin/data/dataset/av2/demo/preprocess_v2/sensor/val
+train_data: /home/kin/data/av2/preprocess_v2/demo/sensor/train
+val_data: /home/kin/data/av2/preprocess_v2/demo/sensor/val
 
 output: ${model.name}-${slurm_id}
 

conf/config.yaml

@@ -7,8 +7,8 @@ dependencies:
   - pytorch::pytorch=2.0.0
   - pytorch::torchvision
   - pytorch::pytorch-cuda=11.7
+  - nvidia/label/cuda-11.7.0::cuda
   - lightning==2.0.1
-  - nvidia::cudatoolkit=11.7
   - mkl==2024.0.0
   - numba
   - numpy

environment.yaml

@@ -1,101 +1,57 @@
 '''
-# Created: 2023-1-26 16:38
-# Updated: 2024-04-15 12:06
+# @date: 2023-1-26 16:38
+# @author: Qingwen Zhang  (https://kin-zhang.github.io/)
 # Copyright (C) 2023-now, RPL, KTH Royal Institute of Technology
-# Author: Qingwen ZHANG  (https://kin-zhang.github.io/)
+# @detail:
+#  1. Play the data you want in open3d, and save the view control to json file.
+#  2. Use the json file to view the data again.
+#  3. Save the screen shot and view file for later check and animation.
 # 
 # code gits: https://gist.github.com/Kin-Zhang/77e8aa77a998f1a4f7495357843f24ef
 # 
-# Description as follows:
-
-This file is for open3d view control set from view_file, which should be json
-1. use normal way to open any geometry and set view by mouse you want
-2. `CTRL+C` it will copy the view detail at this moment.
-3. `CTRL+V` to json file, you can create new one
-4. give the json file path
-
-Check this part: http://www.open3d.org/docs/release/tutorial/visualization/visualization.html#Store-view-point
-
-Test if you want by run this script: by press 'V' on keyboard, will set from json
-
 # CHANGELOG:
+# 2024-08-23 21:41(Qingwen): remove totally on view setting from scratch but use open3d>=0.18.0 version for set_view from json text func.
 # 2024-04-15 12:06(Qingwen): show a example json text. add hex_to_rgb, color_map_hex, color_map (for color points if needed)
-# 2024-01-27 0:41(Qingwen): update MyVisualizer class, reference from kiss-icp 
-[python/kiss-icp/tools/visualizer.py](https://github.com/PRBonn/kiss-icp/blob/main/python/kiss_icp/tools/visualizer.py)
+# 2024-01-27 0:41(Qingwen): update MyVisualizer class, reference from kiss-icp: https://github.com/PRBonn/kiss-icp/blob/main/python/kiss_icp/tools/visualizer.py
 '''
 
 import open3d as o3d
-import json
-import os, sys
+import os, time
 from typing import List, Callable
 from functools import partial
 
 def hex_to_rgb(hex_color):
     hex_color = hex_color.lstrip("#")
     return tuple(int(hex_color[i:i + 2], 16) / 255.0 for i in (0, 2, 4))
 
-color_map_hex = ['#a6cee3', '#de2d26', '#1f78b4','#b2df8a','#33a02c','#fb9a99','#e31a1c','#fdbf6f','#ff7f00','#cab2d6','#6a3d9a','#ffff99','#b15928',\
-                 '#8dd3c7','#ffffb3','#bebada','#fb8072','#80b1d3','#fdb462','#b3de69','#fccde5','#d9d9d9','#bc80bd','#ccebc5','#ffed6f']
-color_map = [hex_to_rgb(color) for color in color_map_hex]
+color_map_hex = ['#a6cee3', '#de2d26', '#1f78b4','#b2df8a','#33a02c','#fb9a99','#e31a1c','#fdbf6f','#ff7f00',\
+                 '#cab2d6','#6a3d9a','#ffff99','#b15928', '#8dd3c7','#ffffb3','#bebada','#fb8072','#80b1d3',\
+                 '#fdb462','#b3de69','#fccde5','#d9d9d9','#bc80bd','#ccebc5','#ffed6f']
 
-class ViewControl:
-    def __init__(self, vctrl: o3d.visualization.ViewControl, view_file=None):
-        self.vctrl = vctrl
-        self.params = None
-        if view_file is not None:
-            print(f"Init with view_file from: {view_file}")
-            self.parse_file(view_file)
-            self.set_param()
-        else:
-            print("Init without view_file")
-
-    def read_viewTfile(self, view_file):
-        if view_file is None:
-            return
-        self.parse_file(view_file)
-        self.set_param()
-
-    def save_viewTfile(self, view_file):
-        return
-    
-    def parse_file(self, view_file):
-        if view_file is None:
-            print(f"\033[91mNo specific view file. Skip to setup viewpoint in open3d. \033[0m")
-            return
-        if(os.path.exists(view_file)):
-            with open((view_file)) as user_file:
-                file_contents = user_file.read()
-                self.params = json.loads(file_contents)
-        else:
-            print(f"\033[91mDidn't find the file, please check it again: {view_file} \033[0m")
-            print(f"NOTE: If you still have this error, please give the absulote path for view_file")
-            sys.exit()
-
-    def set_param(self):
-        self.vctrl.change_field_of_view(self.params['trajectory'][0]['field_of_view'])
-        self.vctrl.set_front(self.params['trajectory'][0]['front'])
-        self.vctrl.set_lookat(self.params['trajectory'][0]['lookat'])
-        self.vctrl.set_up(self.params['trajectory'][0]['up'])
-        self.vctrl.set_zoom(self.params['trajectory'][0]['zoom'])
+color_map = [hex_to_rgb(color) for color in color_map_hex]
 
+        
 class MyVisualizer:
-    def __init__(self, view_file=None, window_title="Default"):
+    def __init__(self, view_file=None, window_title="Default", save_folder="logs/imgs"):
         self.params = None
         self.vis = o3d.visualization.VisualizerWithKeyCallback()
         self.vis.create_window(window_name=window_title)
-        self.o3d_vctrl = ViewControl(self.vis.get_view_control(), view_file=view_file)
         self.view_file = view_file
 
         self.block_vis = True
         self.play_crun = False
         self.reset_bounding_box = True
+        self.save_img_folder = save_folder
+        os.makedirs(self.save_img_folder, exist_ok=True)
         print(
             f"\n{window_title.capitalize()} initialized. Press:\n"
             "\t[SPACE] to pause/start\n"
-            "\t  [ESC] to exit\n"
+            "\t[ESC/Q] to exit\n"
+            "\t    [P] to save screen and viewpoint\n"
             "\t    [N] to step\n"
         )
         self._register_key_callback(["Ā", "Q", "\x1b"], self._quit)
+        self._register_key_callback(["P"], self._save_screen)
         self._register_key_callback([" "], self._start_stop)
         self._register_key_callback(["N"], self._next_frame)
 
@@ -104,7 +60,8 @@ def show(self, assets: List):
 
         for asset in assets:
             self.vis.add_geometry(asset)
-            self.o3d_vctrl.read_viewTfile(self.view_file)
+            if self.view_file is not None:
+                self.vis.set_view_status(open(self.view_file).read())
 
         self.vis.update_renderer()
         self.vis.poll_events()
@@ -122,7 +79,7 @@ def update(self, assets: List, clear: bool = True):
         if self.reset_bounding_box:
             self.vis.reset_view_point(True)
             if self.view_file is not None:
-                self.o3d_vctrl.read_viewTfile(self.view_file)
+                self.vis.set_view_status(open(self.view_file).read())
             self.reset_bounding_box = False
 
         self.vis.update_renderer()
@@ -135,15 +92,27 @@ def update(self, assets: List, clear: bool = True):
     def _register_key_callback(self, keys: List, callback: Callable):
         for key in keys:
             self.vis.register_key_callback(ord(str(key)), partial(callback))
+
     def _next_frame(self, vis):
         self.block_vis = not self.block_vis
+
     def _start_stop(self, vis):
         self.play_crun = not self.play_crun
+
     def _quit(self, vis):
         print("Destroying Visualizer. Thanks for using ^v^.")
         vis.destroy_window()
         os._exit(0)
 
+    def _save_screen(self, vis):
+        timestamp = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
+        png_file = f"{self.save_img_folder}/ScreenShot_{timestamp}.png"
+        view_json_file = f"{self.save_img_folder}/ScreenView_{timestamp}.json"
+        with open(view_json_file, 'w') as f:
+            f.write(vis.get_view_status())
+        vis.capture_screen_image(png_file)
+        print(f"ScreenShot saved to: {png_file}, Please check it.")
+
 if __name__ == "__main__":
     json_content = """{
 	"class_name" : "ViewTrajectory",
@@ -171,29 +140,6 @@ def _quit(self, vis):
         f.write(json_content)
     sample_ply_data = o3d.data.PLYPointCloud()
     pcd = o3d.io.read_point_cloud(sample_ply_data.path)
-    # 1. define
-    viz = o3d.visualization.VisualizerWithKeyCallback()
-    # 2. create
-    viz.create_window(window_name="TEST ON Change View point through JSON, Press V Please")
-    # 3. add geometry
-    viz.add_geometry(pcd)
-    # 4. get control !!! must step by step
-    ctr = viz.get_view_control()
-
-    o3d_vctrl = ViewControl(ctr)
-
-    def set_view(viz):
-        #Your update routine
-        o3d_vctrl.read_viewTfile(view_json_file)
-        viz.update_renderer()
-        viz.poll_events()
-        viz.run()
-
-    viz.register_key_callback(ord('V'), set_view)
-    viz.run()
-    viz.destroy_window()
-    print("\033[92mAll o3d_view codes run successfully, Close now..\033[0m See you!")
-
-    # or:
-    # viz = MyVisualizer(view_file, window_title="Check Pose")
-    # viz.show([*pcds, *draw_tfs])
+
+    viz = MyVisualizer(view_json_file, window_title="Qingwen's View")
+    viz.show([pcd])