-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathMakefile
392 lines (350 loc) · 15.7 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
## ==== Core Arguments and Parameters ====
MAJOR ?= 1
MINOR ?= 0
VERSION = $(MAJOR).$(MINOR)
APP_NAME ?= xai-nav-neurips21
# Core Names
MAZE_XAI_BASENAME ?= maze_xai_pretrained
FLOORPLAN_XAI_BASENAME ?= floorplan_xai_pretrained
EXPERIMENT_NAME ?= base
SP_LIMIT_NUM ?= -1
# Handle Optional GPU
USE_GPU ?= true
ifeq ($(USE_GPU),true)
DOCKER_GPU_ARG = --gpus all
endif
# Docker args
DISPLAY ?= :0.0
DATA_BASE_DIR = $(shell pwd)/data
UNITY_DIR = $(DATA_BASE_DIR)/unity
UNITY_BASENAME = xai_unity
XPASSTHROUGH ?= false
DOCKER_FILE_DIR = "."
DOCKERFILE = ${DOCKER_FILE_DIR}/Dockerfile
IMAGE_NAME = ${APP_NAME}
DOCKER_CORE_VOLUMES = \
--env XPASSTHROUGH=$(XPASSTHROUGH) \
--env DISPLAY=$(DISPLAY) \
$(DOCKER_GPU_ARG) \
--volume="$(UNITY_DIR):/unity/:rw" \
--volume="$(DATA_BASE_DIR):/data/:rw" \
--volume="/tmp/.X11-unix:/tmp/.X11-unix:rw"
DOCKER_PYTHON = docker run --init --ipc=host \
$(DOCKER_ARGS) $(DOCKER_CORE_VOLUMES) \
${IMAGE_NAME}:${VERSION} python3
.PHONY: help
help:
@echo ''
@echo 'Usage: make [TARGET] [EXTRA_ARGUMENTS]'
@echo 'Targets:'
@echo ' help display this help message'
@echo ' build build docker image (incremental)'
@echo ' rebuild build docker image from scratch'
@echo ' kill close all project-related docker containers'
@echo ' test run pytest in docker container'
@echo 'Extra Arguments:'
@echo ' DATA_BASE_DIR local path to directory for storing data'
@echo ''
## ==== Helpers for setting up the environment ====
define arg_check_unity
@[ "${UNITY_DIR}" ] && true || \
( echo "ERROR: Environment variable 'UNITY_DIR' must be set." 1>&2; exit 1 )
endef
define xhost_activate
@echo "Enabling local xhost sharing:"
@echo " Display: $(DISPLAY)"
@-DISPLAY=$(DISPLAY) xhost +
@-xhost +
endef
arg-check-unity:
$(call arg_check_unity)
arg-check-data:
@[ "${DATA_BASE_DIR}" ] && true || \
( echo "ERROR: Environment variable 'DATA_BASE_DIR' must be set." 1>&2; exit 1 )
xhost-activate:
$(call xhost_activate)
## ==== Build targets ====
.PHONY: build
build:
@echo "Building the Docker container"
@docker build -t ${IMAGE_NAME}:${VERSION} \
$(DOCKER_ARGS) -f ./${DOCKERFILE} .
@echo "Creating source directory: $(DATA_BASE_DIR)"
@mkdir -p $(DATA_BASE_DIR)
.PHONY: rebuild
rebuild:
@docker build -t ${IMAGE_NAME}:${VERSION} --no-cache \
$(DOCKER_ARGS) -f ./${DOCKERFILE} .
.PHONY: kill
kill:
@echo "Closing all running docker containers:"
@docker kill $(shell docker ps -q --filter ancestor=${IMAGE_NAME}:${VERSION})
.PHONY: fix-target-timestamps
fix-target-timestamps:
@echo "Fixing relative timestamps after unzipping."
@- touch $(shell ls $(maze-dir)/data_collect_plots/*.png)
@- touch $(shell ls $(maze-dir)/training/base_allSG/*.pt)
@- touch $(shell ls $(maze-dir)/results/base_allSG/*.png)
@- touch $(shell ls $(maze-dir)/training/base_4SG/*.pt)
@- touch $(shell ls $(maze-dir)/results/base_4SG/*.png)
@- touch $(shell ls $(maze-dir)/training/base_0SG/*.pt)
@- touch $(shell ls $(maze-dir)/results/base_0SG/*.png)
@- touch $(shell ls $(floorplan-dir)/data_collect_plots/*.png)
@- touch $(shell ls $(floorplan-dir)/training/base_allSG/*.pt)
@- touch $(shell ls $(floorplan-dir)/results/base_allSG/*.png)
@- touch $(shell ls $(floorplan-dir)/training/base_4SG/*.pt)
@- touch $(shell ls $(floorplan-dir)/results/base_4SG/*110*.png)
@- touch $(shell ls $(floorplan-dir)/results/base_4SG/*111*.png)
@- touch $(shell ls $(floorplan-dir)/results/base_4SG/*112*.png)
@- touch $(shell ls $(floorplan-dir)/results/base_4SG/*113*.png)
@- touch $(shell ls $(floorplan-dir)/results/base_4SG/*114*.png)
@- touch $(shell ls $(floorplan-dir)/results/base_4SG/*115*.png)
@- touch $(shell ls $(floorplan-dir)/results/base_4SG/*116*.png)
@- touch $(shell ls $(floorplan-dir)/results/base_4SG/*117*.png)
@- touch $(shell ls $(floorplan-dir)/results/base_4SG/*118*.png)
@- touch $(shell ls $(floorplan-dir)/results/base_4SG/*119*.png)
@- touch $(shell ls $(floorplan-dir)/training/base_0SG/*.pt)
@- touch $(shell ls $(floorplan-dir)/results/base_0SG/*.png)
# Shrink images (for submission file size limit)
.PHONY: shrink-images
# shrink-images: shr-dir = $(maze-dir)/data_collect_plots
shrink-images: shr-dir = $(maze-dir)/results/base_4SG
shrink-images: shr-args = -dither None -colors 10 -set filename:base "%[basename]"
shrink-images:
- sudo convert $(shr-dir)/*110*.png[240x] $(shr-args) "$(shr-dir)/%[filename:base].png"
- sudo convert $(shr-dir)/*111*.png[240x] $(shr-args) "$(shr-dir)/%[filename:base].png"
- sudo convert $(shr-dir)/*112*.png[240x] $(shr-args) "$(shr-dir)/%[filename:base].png"
- sudo convert $(shr-dir)/*113*.png[240x] $(shr-args) "$(shr-dir)/%[filename:base].png"
- sudo convert $(shr-dir)/*114*.png[240x] $(shr-args) "$(shr-dir)/%[filename:base].png"
- sudo convert $(shr-dir)/*115*.png[240x] $(shr-args) "$(shr-dir)/%[filename:base].png"
- sudo convert $(shr-dir)/*116*.png[240x] $(shr-args) "$(shr-dir)/%[filename:base].png"
- sudo convert $(shr-dir)/*117*.png[240x] $(shr-args) "$(shr-dir)/%[filename:base].png"
- sudo convert $(shr-dir)/*118*.png[240x] $(shr-args) "$(shr-dir)/%[filename:base].png"
- sudo convert $(shr-dir)/*119*.png[240x] $(shr-args) "$(shr-dir)/%[filename:base].png"
## ==== Running tests ====
.PHONY: test
test: DOCKER_ARGS ?= -it
test: PYTEST_FILTER ?= "py"
test:
@$(call xhost_activate)
@$(call arg_check_unity)
@mkdir -p $(DATA_BASE_DIR)/test_logs
@$(DOCKER_PYTHON) \
-m py.test -vk $(PYTEST_FILTER) \
-rsx \
--full-trace \
--html=/data/test_logs/report.html \
--xpassthrough=$(XPASSTHROUGH) \
--unity-path=/unity/$(UNITY_BASENAME).x86_64 \
--maze-interp-network-path /data/maze_xai_pretrained/training/base_0SG/ExpNavVisLSP.final.pt \
tests/
## ==== Core arguments ====
SIM_ROBOT_ARGS ?= --step_size 1.8 \
--num_primitives 32 \
--field_of_view_deg 360
INTERP_ARGS ?= --summary_frequency 100 \
--num_epochs 1 \
--learning_rate 2.0e-2 \
--batch_size 4
## ==== Maze Arguments and Experiments ====
MAZE_CORE_ARGS ?= --unity_path /unity/$(UNITY_BASENAME).x86_64 \
--map_type maze \
--base_resolution 1.0 \
--inflation_rad 2.5 \
--laser_max_range_m 60 \
--save_dir /data/$(MAZE_XAI_BASENAME)/
MAZE_DATA_GEN_ARGS = $(MAZE_CORE_ARGS) --logdir /data/$(MAZE_XAI_BASENAME)/training/data_gen
MAZE_EVAL_ARGS = $(MAZE_CORE_ARGS) --logdir /data/$(MAZE_XAI_BASENAME)/training/$(EXPERIMENT_NAME)
maze-dir = $(DATA_BASE_DIR)/$(MAZE_XAI_BASENAME)
# Initialize the Learning
xai-maze-init-learning = $(maze-dir)/training/data_gen/ExpNavVisLSP.init.pt
$(xai-maze-init-learning):
@echo "Writing the 'initial' neural network [Maze: $(MAZE_XAI_BASENAME)]"
@mkdir -p $(maze-dir)/training/$(EXPERIMENT_NAME)
@$(DOCKER_PYTHON) -m scripts.explainability_train_eval \
$(MAZE_DATA_GEN_ARGS) \
$(SIM_ROBOT_ARGS) \
$(INTERP_ARGS) \
--do_init_learning
# Generate Data
xai-maze-data-gen-seeds = $(shell for ii in $$(seq 1000 1999); do echo "$(maze-dir)/data_collect_plots/learned_planner_$$ii.png"; done)
$(xai-maze-data-gen-seeds): $(xai-maze-init-learning)
@echo "Generating Data [$(MAZE_XAI_BASENAME) | seed: $(shell echo $@ | grep -Eo '[0-9]+' | tail -1)"]
@$(call xhost_activate)
@rm -f $(maze-dir)/lsp_data_$(shell echo $@ | grep -Eo '[0-9]+' | tail -1).*.csv
@mkdir -p $(maze-dir)/data
@mkdir -p $(maze-dir)/data_collect_plots
@$(DOCKER_PYTHON) -m scripts.explainability_train_eval \
$(MAZE_DATA_GEN_ARGS) \
$(SIM_ROBOT_ARGS) \
$(INTERP_ARGS) \
--do_data_gen \
--current_seed $(shell echo $@ | grep -Eo '[0-9]+' | tail -1) \
# Train the Network
xai-maze-train-learning = $(maze-dir)/training/$(EXPERIMENT_NAME)/ExpNavVisLSP.final.pt
$(xai-maze-train-learning): $(xai-maze-data-gen-seeds)
@$(DOCKER_PYTHON) -m scripts.explainability_train_eval \
$(MAZE_EVAL_ARGS) \
$(SIM_ROBOT_ARGS) \
$(INTERP_ARGS) \
--sp_limit_num $(SP_LIMIT_NUM) \
--do_train
# Evaluate Performance
xai-maze-eval-seeds = $(shell for ii in $$(seq 11000 11999); do echo "$(DATA_BASE_DIR)/$(MAZE_XAI_BASENAME)/results/$(EXPERIMENT_NAME)/learned_planner_$$ii.png"; done)
$(xai-maze-eval-seeds): $(xai-maze-train-learning)
@echo "Evaluating Performance [$(MAZE_XAI_BASENAME) | seed: $(shell echo $@ | grep -Eo '[0-9]+ | tail -1')]"
@$(call xhost_activate)
@$(call arg_check_unity)
@mkdir -p $(maze-dir)/results/$(EXPERIMENT_NAME)
$(DOCKER_PYTHON) -m scripts.explainability_train_eval \
$(MAZE_EVAL_ARGS) \
$(SIM_ROBOT_ARGS) \
$(INTERP_ARGS) \
--do_eval \
--current_seed $(shell echo $@ | grep -Eo '[0-9]+' | tail -1) \
--save_dir /data/$(MAZE_XAI_BASENAME)/results/$(EXPERIMENT_NAME) \
--logfile_name logfile_final.txt
## ==== University Building (Floorplan) Environment Experiments ====
FLOORPLAN_CORE_ARGS ?= --unity_path /unity/$(UNITY_BASENAME).x86_64 \
--map_type ploader \
--base_resolution 0.6 \
--inflation_radius_m 1.5 \
--laser_max_range_m 72 \
--save_dir /data/$(FLOORPLAN_XAI_BASENAME)/
FLOORPLAN_DATA_GEN_ARGS ?= $(FLOORPLAN_CORE_ARGS) \
--map_file /data/university_building_floorplans/train/*.pickle \
--logdir /data/$(FLOORPLAN_XAI_BASENAME)/training/data_gen
FLOORPLAN_EVAL_ARGS ?= $(FLOORPLAN_CORE_ARGS) \
--map_file /data/university_building_floorplans/test/*.pickle \
--logdir /data/$(FLOORPLAN_XAI_BASENAME)/training/$(EXPERIMENT_NAME)
floorplan-dir = $(DATA_BASE_DIR)/$(FLOORPLAN_XAI_BASENAME)
# Initialize the Learning
xai-floorplan-init-learning = $(floorplan-dir)/training/data_gen/ExpNavVisLSP.init.pt
$(xai-floorplan-init-learning):
@echo "Writing the 'initial' neural network [Floorplan: $(FLOORPLAN_XAI_BASENAME)]"
@mkdir -p $(floorplan-dir)/training/$(EXPERIMENT_NAME)
@$(DOCKER_PYTHON) -m scripts.explainability_train_eval \
$(FLOORPLAN_DATA_GEN_ARGS) \
$(SIM_ROBOT_ARGS) \
$(INTERP_ARGS) \
--do_init_learning
# Generate Data
xai-floorplan-data-gen-seeds = $(shell for ii in $$(seq 1000 1009); do echo "$(floorplan-dir)/data_collect_plots/learned_planner_$$ii.png"; done)
$(xai-floorplan-data-gen-seeds): $(xai-floorplan-init-learning)
@echo "Generating Data [$(FLOORPLAN_XAI_BASENAME) | seed: $(shell echo $@ | grep -Eo '[0-9]+' | tail -1)"]
@$(call xhost_activate)
@rm -f $(floorplan-dir)/lsp_data_$(shell echo $@ | grep -Eo '[0-9]+' | tail -1).*.csv
@mkdir -p $(floorplan-dir)/data
@mkdir -p $(floorplan-dir)/data_collect_plots
@$(DOCKER_PYTHON) -m scripts.explainability_train_eval \
$(FLOORPLAN_DATA_GEN_ARGS) \
$(SIM_ROBOT_ARGS) \
$(INTERP_ARGS) \
--do_data_gen \
--current_seed $(shell echo $@ | grep -Eo '[0-9]+' | tail -1) \
# Train the Network
xai-floorplan-train-learning = $(floorplan-dir)/training/$(EXPERIMENT_NAME)/ExpNavVisLSP.final.pt
$(xai-floorplan-train-learning): $(xai-floorplan-data-gen-seeds)
@$(DOCKER_PYTHON) -m scripts.explainability_train_eval \
$(FLOORPLAN_EVAL_ARGS) \
$(SIM_ROBOT_ARGS) \
$(INTERP_ARGS) \
--sp_limit_num $(SP_LIMIT_NUM) \
--do_train
# Evaluate Performance
xai-floorplan-eval-seeds = $(shell for ii in $$(seq 11000 11009); do echo "$(DATA_BASE_DIR)/$(FLOORPLAN_XAI_BASENAME)/results/$(EXPERIMENT_NAME)/learned_planner_$$ii.png"; done)
$(xai-floorplan-eval-seeds): $(xai-floorplan-train-learning)
@echo "Evaluating Performance [$(FLOORPLAN_XAI_BASENAME) | seed: $(shell echo $@ | grep -Eo '[0-9]+ | tail -1')]"
@$(call xhost_activate)
@$(call arg_check_unity)
@mkdir -p $(floorplan-dir)/results/$(EXPERIMENT_NAME)
$(DOCKER_PYTHON) -m scripts.explainability_train_eval \
$(FLOORPLAN_EVAL_ARGS) \
$(SIM_ROBOT_ARGS) \
$(INTERP_ARGS) \
--do_eval \
--current_seed $(shell echo $@ | grep -Eo '[0-9]+' | tail -1) \
--save_dir /data/$(FLOORPLAN_XAI_BASENAME)/results/$(EXPERIMENT_NAME) \
--logfile_name logfile_final.txt
# Some helper targets to run code individually
xai-floorplan-intervene-seeds-4SG = $(shell for ii in 11304 11591 11870 11336 11245 11649 11891 11315 11069 11202 11614 11576 11100 11979 11714 11430 11267 11064 11278 11367 11193 11670 11385 11180 11923 11195 11642 11462 11010 11386 11913 11103 11474 11855 11823 11641 11408 11899 11449 11393 11041 11435 11101 11610 11422 11546 11048 11070 11699 11618; do echo "$(floorplan-dir)/results/$(EXPERIMENT_NAME)/learned_planner_$${ii}_intervened_4SG.png"; done)
xai-floorplan-intervene-seeds-allSG = $(shell for ii in 11304 11591 11870 11336 11245 11649 11891 11315 11069 11202 11614 11576 11100 11979 11714 11430 11267 11064 11278 11367 11193 11670 11385 11180 11923 11195 11642 11462 11010 11386 11913 11103 11474 11855 11823 11641 11408 11899 11449 11393 11041 11435 11101 11610 11422 11546 11048 11070 11699 11618; do echo "$(floorplan-dir)/results/$(EXPERIMENT_NAME)/learned_planner_$${ii}_intervened_allSG.png"; done)
$(xai-floorplan-intervene-seeds-4SG): $(xai-floorplan-train-learning)
@mkdir -p $(DATA_BASE_DIR)/$(FLOORPLAN_XAI_BASENAME)/results/$(EXPERIMENT_NAME)
@$(DOCKER_PYTHON) -m scripts.explainability_train_eval \
$(FLOORPLAN_EVAL_ARGS) \
$(SIM_ROBOT_ARGS) \
$(INTERP_ARGS) \
--do_intervene \
--sp_limit_num 4 \
--current_seed $(shell echo $@ | grep -Eo '[0-9]+' | tail -2 | head -1) \
--save_dir /data/$(FLOORPLAN_XAI_BASENAME)/results/$(EXPERIMENT_NAME) \
--logfile_name logfile_intervene_4SG.txt
$(xai-floorplan-intervene-seeds-allSG): $(xai-floorplan-train-learning)
@mkdir -p $(DATA_BASE_DIR)/$(FLOORPLAN_XAI_BASENAME)/results/$(EXPERIMENT_NAME)
@$(DOCKER_PYTHON) -m scripts.explainability_train_eval \
$(FLOORPLAN_EVAL_ARGS) \
$(SIM_ROBOT_ARGS) \
$(INTERP_ARGS) \
--do_intervene \
--current_seed $(shell echo $@ | grep -Eo '[0-9]+' | tail -1) \
--save_dir /data/$(FLOORPLAN_XAI_BASENAME)/results/$(EXPERIMENT_NAME) \
--logfile_name logfile_intervene_allSG.txt \
## ==== Results & Plotting ====
.PHONY: xai-process-results
xai-process-results:
@echo "==== Maze Results ===="
@$(DOCKER_PYTHON) -m scripts.explainability_results \
--data_file /data/$(MAZE_XAI_BASENAME)/results/base_allSG/logfile_final.txt \
/data/$(MAZE_XAI_BASENAME)/results/base_4SG/logfile_final.txt \
/data/$(MAZE_XAI_BASENAME)/results/base_0SG/logfile_final.txt \
--output_image_file /data/maze_results.png
@echo "==== Floorplan Results ===="
@$(DOCKER_PYTHON) -m scripts.explainability_results \
--data_file /data/$(FLOORPLAN_XAI_BASENAME)/results/base_allSG/logfile_final.txt \
/data/$(FLOORPLAN_XAI_BASENAME)/results/base_4SG/logfile_final.txt \
/data/$(FLOORPLAN_XAI_BASENAME)/results/base_0SG/logfile_final.txt \
--output_image_file /data/floorplan_results.png
@echo "==== Floorplan Intervention Results ===="
@$(DOCKER_PYTHON) -m scripts.explainability_results \
--data_file /data/$(FLOORPLAN_XAI_BASENAME)/results/base_4SG/logfile_intervene_4SG.txt \
/data/$(FLOORPLAN_XAI_BASENAME)/results/base_4SG/logfile_intervene_allSG.txt \
--do_intervene \
--xpassthrough $(XPASSTHROUGH) \
--output_image_file /data/floorplan_intervene_results.png \
.PHONY: xai-explanations
xai-explanations:
@mkdir -p $(DATA_BASE_DIR)/explanations/
@$(DOCKER_PYTHON) -m scripts.explainability_train_eval \
$(MAZE_EVAL_ARGS) \
$(SIM_ROBOT_ARGS) \
$(INTERP_ARGS) \
--do_explain \
--explain_at 20 \
--sp_limit_num 4 \
--current_seed 1037 \
--save_dir /data/explanations/ \
--logdir /data/$(MAZE_XAI_BASENAME)/training/base_0SG
@$(DOCKER_PYTHON) -m scripts.explainability_train_eval \
$(FLOORPLAN_EVAL_ARGS) \
$(SIM_ROBOT_ARGS) \
$(INTERP_ARGS) \
--do_explain \
--explain_at 289 \
--sp_limit_num 4 \
--current_seed 11591 \
--save_dir /data/explanations/ \
--logdir /data/$(FLOORPLAN_XAI_BASENAME)/training/base_4SG
## ==== Some helper targets to run code individually ====
# Maze
xai-maze-data-gen: $(xai-maze-data-gen-seeds)
xai-maze-train: $(xai-maze-train-learning)
xai-maze-eval: $(xai-maze-eval-seeds)
xai-maze: xai-maze-eval
# Floorplan
xai-floorplan-data-gen: $(xai-floorplan-data-gen-seeds)
xai-floorplan-train: $(xai-floorplan-train-learning)
xai-floorplan-eval: $(xai-floorplan-eval-seeds)
xai-floorplan: xai-floorplan-eval
xai-floorplan-data-gen: $(xai-floorplan-data-gen-seeds)
xai-floorplan-intervene: $(xai-floorplan-intervene-seeds-allSG) $(xai-floorplan-intervene-seeds-4SG)