build: make docs build incremental and idempotent (determined-ai#4116)

Because docs/ depends on examples/, model_hub/, and helm/, and recursive
make is never quite correct, incremental builds will not be perfect.
But they should be correct enough for efficient local development.

Only clean builds are fully correct, which CI already does.

.circleci/config.yml

@@ -930,6 +930,7 @@ jobs:
           extra-requirements-file: "docs/requirements.txt"
           executor: <<pipeline.parameters.docker-image>>
       - run: make -C examples build
+      - run: make -C model_hub examples
       - run: make -C helm build
       - attach_workspace:
           at: .

docs/Makefile

@@ -5,34 +5,62 @@ SPHINXOPTS    = -W
 SPHINXBUILD   = sphinx-build
 SPHINXWRAPPER = ./sphinx-wrapper.py
 
-.PHONY: build-examples
-build-examples:
-	$(MAKE) -C ../examples build
-	$(MAKE) -C ../model_hub examples
+# These are "shallow" dependencies, where we check the output of another module
+# but not the dependencies of that output.  This will catch the "nothing has
+# been built" and the "module was rebuilt" cases but not the "module needs
+# rebuilding" case.
+../helm/build/stamp:
+	$(MAKE) -C ../helm build/stamp
 
-.PHONY: build-helm
-build-helm:
-	$(MAKE) -C ../helm build
+../examples/build/stamp:
+	$(MAKE) -C ../examples build/stamp
+
+../model_hub/build-examples/stamp:
+	$(MAKE) -C ../model_hub build-examples/stamp
+
+../proto/build/swagger/determined/api/v1/api.swagger.json:
+	$(MAKE) -C ../proto build
+
+build/helm.stamp: ../helm/build/stamp
+	rm -rf site/downloads/helm
+	mkdir -p site/downloads/helm
+	cp ../helm/build/*.tgz site/downloads/helm
+	mkdir -p build
+	touch $@
+
+build/examples.stamp: ../examples/build/stamp ../model_hub/build-examples/stamp
+	rm -rf site/downloads/examples
+	mkdir -p site/downloads/examples
+	cp ../examples/build/*.tgz site/downloads/examples
+	cp ../model_hub/build-examples/*.tgz site/downloads/examples
+	mkdir -p build
+	touch $@
 
 attributions.txt: $(shell find ../tools/scripts/licenses -type f)
 	../tools/scripts/gen-attributions.py sphinx $@
 
-.PHONY: build
-build: build-examples build-helm attributions.txt
-	mkdir -p site/downloads/examples
-	cp ../examples/build/* site/downloads/examples
-	cp ../model_hub/build-examples/* site/downloads/examples
-	mkdir -p site/downloads/helm
-	cp ../helm/build/* site/downloads/helm
-	$(MAKE) sp-html
-	# pull in swagger-ui
+build/sphinx.stamp: $(SPHINXWRAPPER) conf.py attributions.txt $(shell find . -name site -prune -o -type f -name '*.txt' -print)
+	@$(SPHINXWRAPPER) $(SPHINXBUILD) -M html . site $(SPHINXOPTS)
+	mkdir -p build
+	touch $@
+
+build/swagger.stamp: ../proto/build/swagger/determined/api/v1/api.swagger.json swagger-ui/*
+	rm -rf site/html/rest-api
+	mkdir -p site/html
 	cp -r ./swagger-ui site/html/rest-api
-	cp ../proto/build/swagger/determined/api/v1/api.swagger.json  site/html/rest-api/
+	cp $< site/html/rest-api/
+	mkdir -p build
+	touch $@
+
+build/stamp: build/examples.stamp build/helm.stamp build/swagger.stamp build/sphinx.stamp
+	touch $@
+
+.PHONY: build
+build: build/stamp
 
 .PHONY: clean
 clean:
-	rm -rf site
-	rm -f attributions.txt
+	rm -rf site build attributions.txt
 
 .PHONY: pre-publish
 pre-publish:
@@ -43,7 +71,7 @@ publish: pre-publish
 	$(MAKE) -C deploy publish
 
 live:
-	npx nodemon --ext txt --exec "$(MAKE) build" --ignore site
+	npx nodemon --ext txt --exec "$(MAKE) build.stamp" --ignore site
 
 RSTFMT_WIDTH = 100
 
@@ -54,9 +82,3 @@ fmt:
 .PHONY: check
 check:
 	git ls-files -z '*.txt' ':!:requirements.txt' | xargs -0 rstfmt -w $(RSTFMT_WIDTH) --check
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-.PHONY: ALWAYS
-sp-%: ALWAYS
-	@$(SPHINXWRAPPER) $(SPHINXBUILD) -M $* . site $(SPHINXOPTS) $(O)

examples/Makefile

@@ -31,15 +31,19 @@ DT_EXAMPLES_DIRS := $(patsubst decision_trees/%/., build/%.tgz, $(DT_EXAMPLES))
 FEATURES_EXAMPLES := $(wildcard features/*/.)
 FEATURES_EXAMPLES_DIRS := $(patsubst features/%/., build/%.tgz, $(FEATURES_EXAMPLES))
 
+build/stamp: $(TUTORIAL_EXAMPLES_DIRS) $(CV_EXAMPLES_DIRS) $(NLP_EXAMPLES_DIRS) $(DEEPSPEED_EXAMPLES_DIRS) $(HPSEARCH_EXAMPLES_DIRS) $(NAS_EXAMPLES_DIRS) $(META_LEARNING_EXAMPLES_DIRS) $(GAN_EXAMPLES_DIRS) $(GRAPHS_EXAMPLES_DIRS) $(DT_EXAMPLES_DIRS) $(FEATURES_EXAMPLES_DIRS)
+	touch $@
+
 .PHONY: build
-build: $(TUTORIAL_EXAMPLES_DIRS) $(CV_EXAMPLES_DIRS) $(NLP_EXAMPLES_DIRS) $(DEEPSPEED_EXAMPLES_DIRS) $(HPSEARCH_EXAMPLES_DIRS) $(NAS_EXAMPLES_DIRS) $(META_LEARNING_EXAMPLES_DIRS) $(GAN_EXAMPLES_DIRS) $(GRAPHS_EXAMPLES_DIRS) $(DT_EXAMPLES_DIRS) $(FEATURES_EXAMPLES_DIRS)
+build: build/stamp
 
 .PHONY: clean
 clean:
 	find . \( -name __pycache__ -o -name \*.pyc -o -name .mypy_cache \) -print0 | xargs -0 rm -rf
 	rm -rf build/
 
-build/%.tgz: */%/
+# The first `*/%/` defines '$<', the second `*/%/*` defines dependencies.
+build/%.tgz: */% */%/*
 	find "$<" \( -name __pycache__ -o -name \*.pyc \) -delete
 	mkdir -p $$(dirname "$@")
 	tar -czf "$@" -C $$(dirname "$<") $$(basename "$<")

helm/Makefile

@@ -1,8 +1,12 @@
-.PHONY: build
-build: clean
-	mkdir -p build/
+build/stamp: $(shell find charts -type f)
+	mkdir -p build
+	rm -rf build/*.tgz
 	helm package --destination build charts/determined
 	cd build/ && ln -s determined-*.tgz determined-latest.tgz && cd ..
+	touch $@
+
+.PHONY: build
+build: build/stamp
 
 fmt:
 	helm lint charts/determined/

model_hub/Makefile

@@ -45,15 +45,19 @@ clean:
 build:
 	python setup.py -q bdist_wheel
 
+build-examples/stamp: $(HF_EXAMPLES_DIRS) $(MMDET_EXAMPLE_DIR)
+	touch $@
+
 .PHONY: examples
-examples: $(HF_EXAMPLES_DIRS) $(MMDET_EXAMPLE_DIR)
+examples: build-examples/stamp
 
-build-examples/%.tgz: examples/*/%/
+# the first `examples/*/%` defines `$<` and the second sets file dependencies
+build-examples/%.tgz: examples/*/% examples/*/%/*
 	find "$<" \( -name __pycache__ -o -name \*.pyc \) -delete
 	mkdir -p $$(dirname "$@")
 	tar -czf "$@" -C $$(dirname "$<") $$(basename "$<")
 
-build-examples/%.tgz: examples/%/
+build-examples/%.tgz: examples/%/ examples/%/*
 	find "$<" \( -name __pycache__ -o -name \*.pyc \) -delete
 	mkdir -p $$(dirname "$@")
 	tar -czf "$@" -C $$(dirname "$<") $$(basename "$<")