-
Notifications
You must be signed in to change notification settings - Fork 20
/
Copy pathcodalab_commands.bash
103 lines (81 loc) · 4.02 KB
/
codalab_commands.bash
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
# Lower bound experiments (Section 3).
# CIFAR:
cl run :experiments :calibration :data \
'export PYTHONPATH="."; \
python3 experiments/platt_not_calibrated/lower_bounds.py \
--probs_path=data/cifar_probs.dat --bins_list="2, 4, 8, 16, 32, 64, 128" --lp=2 \
--calibration_data_size=1000 --bin_data_size=1000 --plot_save_file=l2_lower_bound_cifar_plot.png \
--binning=equal_bins --num_samples=1000' \
--request-queue tag=nlp -n cifar_l2_lower_bound
cl run :experiments :calibration :data \
'export PYTHONPATH="."; \
python3 experiments/platt_not_calibrated/lower_bounds.py \
--probs_path=data/cifar_probs.dat --bins_list="2, 4, 8, 16, 32, 64, 128" --lp=1 \
--calibration_data_size=1000 --bin_data_size=1000 --plot_save_file=l1_lower_bound_cifar_plot.png \
--binning=equal_bins --num_samples=1000' \
--request-queue tag=nlp -n cifar_l1_lower_bound
cl run :experiments :calibration :data \
'export PYTHONPATH="."; \
python3 experiments/platt_not_calibrated/lower_bounds.py \
--probs_path=data/cifar_probs.dat --bins_list="2, 4, 8, 16, 32, 64, 128" --lp=1 \
--calibration_data_size=1000 --bin_data_size=1000 --plot_save_file=prob_lower_bound_cifar_plot.png \
--binning=equal_prob_bins --num_samples=1000' \
--request-queue tag=nlp -n cifar_prob_lower_bound
# ImageNet:
cl run :experiments :calibration :data \
'export PYTHONPATH="."; \
python3 experiments/platt_not_calibrated/lower_bounds.py \
--probs_path=data/imagenet_probs.dat --bins_list="2, 4, 8, 16, 32, 64, 128, 256, 512" --lp=2 \
--calibration_data_size=20000 --bin_data_size=5000 --plot_save_file=l2_lower_bound_imnet_plot.png \
--binning=equal_bins --num_samples=1000' \
--request-queue tag=nlp -n imagenet_l2_lower_bound
cl run :experiments :calibration :data \
'export PYTHONPATH="."; \
python3 experiments/platt_not_calibrated/lower_bounds.py \
--probs_path=data/imagenet_probs.dat --bins_list="2, 4, 8, 16, 32, 64, 128, 256, 512" --lp=1 \
--calibration_data_size=20000 --bin_data_size=5000 --plot_save_file=l1_lower_bound_imnet_plot.png \
--binning=equal_bins --num_samples=1000' \
--request-queue tag=nlp -n imagenet_l1_lower_bound
cl run :experiments :calibration :data \
'export PYTHONPATH="."; \
python3 experiments/platt_not_calibrated/lower_bounds.py \
--probs_path=data/imagenet_probs.dat --bins_list="2, 4, 8, 16, 32, 64, 128, 256, 512" --lp=1 \
--calibration_data_size=20000 --bin_data_size=5000 --plot_save_file=prob_lower_bound_imnet_plot.png \
--binning=equal_prob_bins --num_samples=1000' \
--request-queue tag=nlp -n imagenet_prob_lower_bound
# Comparing calibrators experiment (Section 4).
cl run :experiments :calibration :data \
'export PYTHONPATH="."; \
python3 experiments/scaling_binning_calibrator/compare_calibrators.py' \
--request-queue tag=nlp -n compare_calibrators
# Synthetic experiments (Section 4).
cl run :experiments :calibration :data \
'export PYTHONPATH="."; \
python3 experiments/synthetic/synthetic.py \
--experiment_name=vary_n_a1_b0' \
--request-queue tag=nlp -n vary_n_a1_b0
cl run :experiments :calibration :data \
'export PYTHONPATH="."; \
python3 experiments/synthetic/synthetic.py \
--experiment_name=vary_b_a1_b0' \
--request-queue tag=nlp -n vary_b_a1_b0
cl run :experiments :calibration :data \
'export PYTHONPATH="."; \
python3 experiments/synthetic/synthetic.py \
--experiment_name=vary_n_a2_b1' \
--request-queue tag=nlp -n vary_n_a2_b1
cl run :experiments :calibration :data \
'export PYTHONPATH="."; \
python3 experiments/synthetic/synthetic.py \
--experiment_name=vary_b_a2_b1' \
--request-queue tag=nlp -n vary_b_a2_b1
cl run :experiments :calibration :data \
'export PYTHONPATH="."; \
python3 experiments/synthetic/synthetic.py \
--experiment_name=noisy_vary_n_a2_b1' \
--request-queue tag=nlp -n noisy_vary_n_a2_b1
# Comparing plugin with debiased calibration error estimators (Section 5).
cl run :experiments :calibration :data \
'export PYTHONPATH="."; \
python3 experiments/debiased_estimator/estimation_error_vs_bins.py' \
--request-queue tag=nlp -n estimator_comparison