Skip to content

Commit 103478a

Browse files
Kin-ZhangKin-Zhang
committed
docs(slurm): update slurm run scripts.
1 parent 6659c89 commit 103478a

3 files changed

Lines changed: 23 additions & 33 deletions

File tree

‎assets/slurm/0_process.sh‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,34 +2,34 @@
22
#SBATCH -J pack_data
33
#SBATCH --gpus 0
44
#SBATCH --cpus-per-task 64
5-
#SBATCH --mem 512G
5+
#SBATCH --mem 256G
66
#SBATCH --mincpus=64
7-
#SBATCH -t 3-00:00:00
7+
#SBATCH -t 1-00:00:00
88
#SBATCH --mail-type=END,FAIL
99
#SBATCH --mail-user=qingwen@kth.se
10-
#SBATCH --output /proj/berzelius-2023-154/users/x_qinzh/deflow/logs/slurm/%J_data.out
11-
#SBATCH --error /proj/berzelius-2023-154/users/x_qinzh/deflow/logs/slurm/%J_data.err
10+
#SBATCH --output /proj/berzelius-2023-154/users/x_qinzh/seflow/logs/slurm/%J_data.out
11+
#SBATCH --error /proj/berzelius-2023-154/users/x_qinzh/seflow/logs/slurm/%J_data.err
1212

13-
cd /proj/berzelius-2023-154/users/x_qinzh/deflow
13+
cd /proj/berzelius-2023-154/users/x_qinzh/seflow
1414
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/proj/berzelius-2023-154/users/x_qinzh/mambaforge/lib
15-
export HYDRA_FULL_ERROR=1
15+
# export HYDRA_FULL_ERROR=1
1616

17-
/proj/berzelius-2023-154/users/x_qinzh/mambaforge/envs/deflow/bin/python 0_preprocess.py \
17+
/proj/berzelius-2023-154/users/x_qinzh/mambaforge/envs/seflow/bin/python dataprocess/extract_av2.py \
1818
--av2_type sensor \
1919
--data_mode train \
2020
--argo_dir /proj/berzelius-2023-154/users/x_qinzh/av2 \
21-
--output_dir /proj/berzelius-2023-154/users/x_qinzh/av2/deflow_preprocess
21+
--output_dir /proj/berzelius-2023-154/users/x_qinzh/av2/preprocess_v2
2222

23-
/proj/berzelius-2023-154/users/x_qinzh/mambaforge/envs/deflow/bin/python 0_preprocess.py \
23+
/proj/berzelius-2023-154/users/x_qinzh/mambaforge/envs/seflow/bin/python dataprocess/extract_av2.py \
2424
--av2_type sensor \
2525
--data_mode val \
2626
--argo_dir /proj/berzelius-2023-154/users/x_qinzh/av2 \
27-
--output_dir /proj/berzelius-2023-154/users/x_qinzh/av2/deflow_preprocess \
27+
--output_dir /proj/berzelius-2023-154/users/x_qinzh/av2/preprocess_v2 \
2828
--mask_dir /proj/berzelius-2023-154/users/x_qinzh/av2/3d_scene_flow
2929

30-
/proj/berzelius-2023-154/users/x_qinzh/mambaforge/envs/deflow/bin/python 0_preprocess.py \
30+
/proj/berzelius-2023-154/users/x_qinzh/mambaforge/envs/seflow/bin/python dataprocess/extract_av2.py \
3131
--av2_type sensor \
3232
--data_mode test \
3333
--argo_dir /proj/berzelius-2023-154/users/x_qinzh/av2 \
34-
--output_dir /proj/berzelius-2023-154/users/x_qinzh/av2/deflow_preprocess \
34+
--output_dir /proj/berzelius-2023-154/users/x_qinzh/av2/preprocess_v2 \
3535
--mask_dir /proj/berzelius-2023-154/users/x_qinzh/av2/3d_scene_flow

‎assets/slurm/1_train.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
cd /proj/berzelius-2023-154/users/x_qinzh/seflow
1111

12-
SOURCE="/proj/berzelius-2023-154/users/x_qinzh/data/av2/seflow_preprocess"
12+
SOURCE="/proj/berzelius-2023-154/users/x_qinzh/data/av2/preprocess_v2"
1313
DEST="/scratch/local/av2"
1414
SUBDIRS=("sensor/train" "sensor/val")
1515

‎assets/slurm/2_eval.sh‎

Lines changed: 10 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,13 @@
11
#!/bin/bash
22
#SBATCH -J eval
33
#SBATCH --gpus 1
4-
#SBATCH -t 3-00:00:00
5-
#SBATCH --mail-type=END,FAIL
6-
#SBATCH --mail-user=qingwen@kth.se
7-
#SBATCH --output /proj/berzelius-2023-154/users/x_qinzh/deflow/logs/slurm/%J_eval.out
8-
#SBATCH --error /proj/berzelius-2023-154/users/x_qinzh/deflow/logs/slurm/%J_eval.err
4+
#SBATCH -t 01:00:00
5+
#SBATCH --output /proj/berzelius-2023-154/users/x_qinzh/seflow/logs/slurm/%J_eval.out
6+
#SBATCH --error /proj/berzelius-2023-154/users/x_qinzh/seflow/logs/slurm/%J_eval.err
97

10-
cd /proj/berzelius-2023-154/users/x_qinzh/deflow
8+
cd /proj/berzelius-2023-154/users/x_qinzh/seflow
119

12-
SOURCE="/proj/berzelius-2023-154/users/x_qinzh/av2/deflow_preprocess"
10+
SOURCE="/proj/berzelius-2023-154/users/x_qinzh/av2/preprocess_v2"
1311
DEST="/scratch/local/av2"
1412
SUBDIRS=("sensor/val")
1513

@@ -24,20 +22,12 @@ elapsed=$((end_time - start_time))
2422
echo "Copy ${SOURCE} to ${DEST} Total time: ${elapsed} seconds"
2523
echo "Start training..."
2624

27-
# ====> leaderboard model = [fastflow3d, deflow]
28-
# /proj/berzelius-2023-154/users/x_qinzh/mambaforge/envs/deflow/bin/python 2_eval.py \
25+
# ====> leaderboard model
26+
# /proj/berzelius-2023-154/users/x_qinzh/mambaforge/envs/seflow/bin/python 2_eval.py \
2927
# wandb_mode=online dataset_path=/scratch/local/av2/sensor \
28+
# checkpoint=/proj/berzelius-2023-154/users/x_qinzh/seflow/logs/wandb/seflow-10086990/checkpoints/epoch_19_seflow.ckpt \
3029
# av2_mode=test save_res=True
3130

32-
33-
/proj/berzelius-2023-154/users/x_qinzh/mambaforge/envs/deflow/bin/python 2_eval.py \
34-
wandb_mode=online dataset_path=/scratch/local/av2/sensor av2_mode=val \
35-
checkpoint=/proj/berzelius-2023-154/users/x_qinzh/deflow/logs/wandb/fastflow3d-10086990/checkpoints/epoch_49_fastflow3d.ckpt
36-
37-
/proj/berzelius-2023-154/users/x_qinzh/mambaforge/envs/deflow/bin/python 2_eval.py \
38-
wandb_mode=online dataset_path=/scratch/local/av2/sensor av2_mode=val \
39-
checkpoint=/proj/berzelius-2023-154/users/x_qinzh/deflow/logs/wandb/fastflow3d-10088873/checkpoints/epoch_49_fastflow3d.ckpt
40-
41-
/proj/berzelius-2023-154/users/x_qinzh/mambaforge/envs/deflow/bin/python 2_eval.py \
31+
/proj/berzelius-2023-154/users/x_qinzh/mambaforge/envs/seflow/bin/python 2_eval.py \
4232
wandb_mode=online dataset_path=/scratch/local/av2/sensor av2_mode=val \
43-
checkpoint=/proj/berzelius-2023-154/users/x_qinzh/deflow/logs/wandb/fastflow3d-10088874/checkpoints/epoch_49_fastflow3d.ckpt
33+
checkpoint=/proj/berzelius-2023-154/users/x_qinzh/seflow/logs/wandb/seflow-10086990/checkpoints/epoch_19_seflow.ckpt

0 commit comments

Comments
 (0)