Reduce build requirements for non-developers, neaten build system

.gitignore

@@ -8,7 +8,6 @@
 *.gcno
 *.dSYM
 *.rej
-*.po
 *.pyc
 .cppcheck-suppress
 .mypy_cache

.travis.yml

@@ -9,6 +9,7 @@ before_install:
 
 env:
   # These run superficial checks that don't take too long on their own
+  - ARCH=64 TEST_CMD="make" NO_PYTHON=1
   - ARCH=64 TEST_CMD="make check-source check-units installcheck" COPTFLAGS="-O3"
   - ARCH=32 TEST_CMD="make check-source check-units installcheck" DEVELOPER=0
   - ARCH=64 TEST_CMD="make check-source check-units installcheck" DEVELOPER=0 COMPILER=gcc-4.8

.travis/build.sh

@@ -6,12 +6,12 @@ export SLOW_MACHINE=1
 export CC=${COMPILER:-gcc}
 export DEVELOPER=${DEVELOPER:-1}
 export EXPERIMENTAL_FEATURES=${EXPERIMENTAL_FEATURES:-0}
-export SOURCE_CHECK_ONLY=${SOURCE_CHECK_ONLY:-"false"}
 export COMPAT=${COMPAT:-1}
 export PATH=$CWD/dependencies/bin:"$HOME"/.local/bin:"$PATH"
 export PYTEST_PAR=2
 export PYTEST_SENTRY_ALWAYS_REPORT=1
 export BOLTDIR=lightning-rfc
+
 # Allow up to 4 concurrent tests when not under valgrind, which might run out of memory.
 if [ "$VALGRIND" = 0 ]; then
     PYTEST_PAR=4
@@ -28,34 +28,35 @@ if [ ! -f dependencies/bin/bitcoind ]; then
     rm -rf bitcoin-0.20.1-x86_64-linux-gnu.tar.gz bitcoin-0.20.1
 fi
 
-pyenv global 3.7
+if [ "$NO_PYTHON" != 1 ]; then
+    pyenv global 3.7
+
+    pip3 install --user -U --quiet --progress-bar off \
+	 pip \
+	 pytest-test-groups==1.0.3
 
-# Update pip first, may save us the compilation of binary packages in the next call
-pip3 install --user -U --quiet --progress-bar off \
-     pip \
-     pytest-test-groups==1.0.3
+    pip3 install --user -U --quiet --progress-bar off \
+	 -r requirements.txt \
+	 -r contrib/pyln-client/requirements.txt \
+	 -r contrib/pyln-proto/requirements.txt \
+	 -r contrib/pyln-testing/requirements.txt
 
-pip3 install --user -U --quiet --progress-bar off \
-     -r requirements.txt \
-     -r contrib/pyln-client/requirements.txt \
-     -r contrib/pyln-proto/requirements.txt \
-     -r contrib/pyln-testing/requirements.txt
+    pip3 install --user -U --quiet --progress-bar off \
+	 pytest-sentry \
+	 pytest-rerunfailures
 
-pip3 install --user -U --quiet --progress-bar off \
-     pytest-sentry \
-     pytest-rerunfailures
+   cat > pytest.ini << EOF
+[pytest]
+addopts=-p no:logging --color=no --reruns=5
+EOF
+fi
 
 echo "Configuration which is going to be built:"
 echo -en 'travis_fold:start:script.1\\r'
 ./configure CC="$CC"
 cat config.vars
 echo -en 'travis_fold:end:script.1\\r'
 
-cat > pytest.ini << EOF
-[pytest]
-addopts=-p no:logging --color=no --reruns=5
-EOF
-
 git clone https://github.com/lightningnetwork/lightning-rfc.git
 
 if [ "$TARGET_HOST" == "arm-linux-gnueabihf" ] || [ "$TARGET_HOST" == "aarch64-linux-gnu" ]
@@ -108,7 +109,7 @@ then
     #echo -en 'travis_fold:end:script.3\\r'
 else
     echo -en 'travis_fold:start:script.2\\r'
-    make -j8 > /dev/null
+    make -j8
     echo -en 'travis_fold:end:script.2\\r'
 
     echo -en 'travis_fold:start:script.3\\r'

Makefile

@@ -200,9 +200,15 @@ CDUMP_OBJS := ccan-cdump.o ccan-strmap.o
 
 BOLT_GEN := tools/generate-wire.py
 WIRE_GEN := $(BOLT_GEN)
-BOLT_DEPS := $(BOLT_GEN)
 
-ALL_PROGRAMS =
+# If you use wiregen, you're dependent on the tool, its templates, and Makefile
+WIRE_GEN_DEPS := $(WIRE_GEN) Makefile $(wildcard tools/gen/*_template)
+
+# These are filled by individual Makefiles
+ALL_PROGRAMS :=
+ALL_TEST_PROGRAMS :=
+ALL_C_SOURCES :=
+ALL_C_HEADERS := gen_header_versions.h gen_list_of_builtin_plugins.h gen_version.h
 
 CPPFLAGS += -DBINTOPKGLIBEXECDIR="\"$(shell sh tools/rel.sh $(bindir) $(pkglibexecdir))\""
 CFLAGS = $(CPPFLAGS) $(CWARNFLAGS) $(CDEBUGFLAGS) $(COPTFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . -I/usr/local/include $(SQLITE3_CFLAGS) $(POSTGRES_INCLUDE) $(FEATURES) $(COVFLAGS) $(DEV_CFLAGS) -DSHACHAIN_BITS=48 -DJSMN_PARENT_LINKS $(PIE_CFLAGS) $(COMPAT_CFLAGS) -DBUILD_ELEMENTS=1
@@ -247,12 +253,25 @@ config.vars:
 %.o: %.c
 	@$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<)
 
+# Git doesn't maintain timestamps, so we only regen if sources actually changed:
+# We place the SHA inside some generated files so we can tell if they need updating.
+# Usage: $(call SHA256STAMP_CHANGED,prefix)
+SHA256STAMP_CHANGED = [ x"`sed -n 's/.*SHA256STAMP://p' $@ 2>/dev/null`" != x"$(1)`cat $(filter-out FORCE,$^) | sha256sum | cut -c1-64`" ]
+# Usage: $(call SHA256STAMP,prefix,commentprefix)
+SHA256STAMP = echo '$(2) SHA256STAMP:$(1)'`cat $(filter-out FORCE,$^) | sha256sum | cut -c1-64` >> $@
+
 # generate-wire.py --page [header|impl] hdrfilename wirename < csv > file
-%_wiregen.h: %_wire.csv
-	@if $(CHANGED_FROM_GIT); then $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page header $($@_args) $@ `basename $< .csv` < $< > $@); fi
+%_wiregen.h: %_wire.csv $(WIRE_GEN_DEPS)
+	@if $(call SHA256STAMP_CHANGED,exp-$(EXPERIMENTAL_FEATURES)-); then $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page header $($@_args) $@ `basename $< .csv` < $< > $@ && $(call SHA256STAMP,exp-$(EXPERIMENTAL_FEATURES)-,//)); fi
+
+%_wiregen.c: %_wire.csv $(WIRE_GEN_DEPS)
+	@if $(call SHA256STAMP_CHANGED,exp-$(EXPERIMENTAL_FEATURES)-); then $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page impl $($@_args) ${@:.c=.h} `basename $< .csv` < $< > $@ && $(call SHA256STAMP,exp-$(EXPERIMENTAL_FEATURES)-,//)); fi
 
-%_wiregen.c: %_wire.csv
-	@if $(CHANGED_FROM_GIT); then $(call VERBOSE,"wiregen $@",tools/generate-wire.py --page impl $($@_args) ${@:.c=.h} `basename $< .csv` < $< > $@); fi
+%_printgen.h: %_wire.csv $(WIRE_GEN_DEPS)
+	@if $(call SHA256STAMP_CHANGED,exp-$(EXPERIMENTAL_FEATURES)-); then $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page header $($@_args) $@ `basename $< .csv` < $< > $@ && $(call SHA256STAMP,exp-$(EXPERIMENTAL_FEATURES)-,//)); fi
+
+%_printgen.c: %_wire.csv $(WIRE_GEN_DEPS)
+	@if $(call SHA256STAMP_CHANGED,exp-$(EXPERIMENTAL_FEATURES)-); then $(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page impl $($@_args) ${@:.c=.h} `basename $< .csv` < $< > $@ && $(call SHA256STAMP,exp-$(EXPERIMENTAL_FEATURES)-,//)); fi
 
 include external/Makefile
 include bitcoin/Makefile
@@ -273,6 +292,21 @@ include tools/Makefile
 include plugins/Makefile
 include tests/plugins/Makefile
 
+# We make pretty much everything depend on these.
+ALL_GEN_HEADERS := $(filter gen%.h %printgen.h %wiregen.h,$(ALL_C_HEADERS))
+ALL_GEN_SOURCES := $(filter gen%.c %printgen.c %wiregen.c,$(ALL_C_SOURCES))
+ALL_NONGEN_SRCFILES := $(filter-out gen%.h %printgen.h %wiregen.h,$(ALL_C_HEADERS)) $(filter-out gen%.c %printgen.c %wiregen.c,$(ALL_C_SOURCES))
+
+# Don't delete these intermediaries.
+.PRECIOUS: $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES)
+
+# Every single object file.
+ALL_OBJS := $(ALL_C_SOURCES:.c=.o)
+
+# We always regen wiregen and printgen files, since SHA256STAMP protects against
+# spurious rebuilds.
+$(filter %printgen.h %printgen.c %wiregen.h %wiregen.c, $(ALL_C_HEADERS) $(ALL_C_SOURCES)): FORCE
+
 # Generated from PLUGINS definition in plugins/Makefile
 gen_list_of_builtin_plugins.h : plugins/Makefile Makefile
 	@echo GEN $@
@@ -282,10 +316,6 @@ gen_list_of_builtin_plugins.h : plugins/Makefile Makefile
 	@echo 'NULL' >> $@
 	@echo '};' >> $@
 
-# Git doesn't maintain timestamps, so we only regen if git says we should.
-# If neither is in git, we generate (note: one combines stderr, one discards)
-CHANGED_FROM_GIT = [ x"`git log $@ 2>&1 | head -n1`" != x"`git log $< 2>/dev/null | head -n1`" -o x"`git diff $<`" != x"" ]
-
 ifneq ($(TEST_GROUP_COUNT),)
 PYTEST_OPTS += --test-group=$(TEST_GROUP) --test-group-count=$(TEST_GROUP_COUNT)
 endif
@@ -342,12 +372,12 @@ LOCAL_BOLTDIR=.tmp.lightningrfc
 bolt-precheck:
 	@[ -d $(BOLTDIR) ] || exit 0; set -e; if [ -z "$(BOLTVERSION)" ]; then rm -rf $(LOCAL_BOLTDIR); ln -sf $(BOLTDIR) $(LOCAL_BOLTDIR); exit 0; fi; [ "$$(git -C $(LOCAL_BOLTDIR) rev-list --max-count=1 HEAD 2>/dev/null)" != "$(BOLTVERSION)" ] || exit 0; rm -rf $(LOCAL_BOLTDIR) && git clone -q $(BOLTDIR) $(LOCAL_BOLTDIR) && cd $(LOCAL_BOLTDIR) && git checkout -q $(BOLTVERSION)
 
-check-source-bolt: $(ALL_TEST_PROGRAMS:%=bolt-check/%.c)
+check-source-bolt: $(ALL_NONGEN_SRCFILES:%=bolt-check/%)
 
 check-whitespace/%: %
 	@if grep -Hn '[ 	]$$' $<; then echo Extraneous whitespace found >&2; exit 1; fi
 
-check-whitespace: check-whitespace/Makefile check-whitespace/tools/check-bolt.c $(ALL_TEST_PROGRAMS:%=check-whitespace/%.c)
+check-whitespace: check-whitespace/Makefile check-whitespace/tools/check-bolt.c $(ALL_NONGEN_SRCFILES:%=check-whitespace/%)
 
 check-markdown:
 	@tools/check-markdown.sh
@@ -429,9 +459,6 @@ gen_version.h: FORCE
 gen_header_versions.h: tools/headerversions
 	@tools/headerversions $@
 
-# Rebuild the world if this changes.
-ALL_GEN_HEADERS += gen_header_versions.h
-
 # All binaries require the external libs, ccan and system library versions.
 $(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS): $(EXTERNAL_LIBS) $(CCAN_OBJS)
 
@@ -451,8 +478,8 @@ $(CCAN_OBJS) $(CDUMP_OBJS): $(CCAN_HEADERS) Makefile
 # Except for CCAN, we treat everything else as dependent on external/ bitcoin/ common/ wire/ and all generated headers, and Makefile
 $(ALL_OBJS): $(BITCOIN_HEADERS) $(COMMON_HEADERS) $(CCAN_HEADERS) $(WIRE_HEADERS) $(ALL_GEN_HEADERS) $(EXTERNAL_HEADERS) Makefile
 
-# We generate headers in two ways, so regen when either changes (or Makefile)
-$(ALL_GEN_HEADERS): ccan/ccan/cdump/tools/cdump-enumstr $(WIRE_GEN) Makefile
+# Regen headers when Makefile changes
+$(ALL_GEN_HEADERS): Makefile
 
 update-ccan:
 	mv ccan ccan.old
@@ -475,12 +502,13 @@ distclean: clean
 maintainer-clean: distclean
 	@echo 'This command is intended for maintainers to use; it'
 	@echo 'deletes files that may need special tools to rebuild.'
+	$(RM) $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES)
 
 clean:
 	$(RM) $(CCAN_OBJS) $(CDUMP_OBJS) $(ALL_OBJS)
-	$(RM) $(ALL_PROGRAMS) $(ALL_PROGRAMS:=.o)
-	$(RM) $(ALL_TEST_PROGRAMS) $(ALL_TEST_PROGRAMS:=.o)
-	$(RM) gen_*.h ccan/tools/configurator/configurator
+	$(RM) $(ALL_PROGRAMS)
+	$(RM) $(ALL_TEST_PROGRAMS)
+	$(RM) gen_*.h */gen_* ccan/tools/configurator/configurator
 	$(RM) ccan/ccan/cdump/tools/cdump-enumstr.o
 	find . -name '*gcda' -delete
 	find . -name '*gcno' -delete

bitcoin/test/Makefile

@@ -8,7 +8,7 @@ $(BITCOIN_TEST_PROGRAMS): $(CCAN_OBJS) $(BITCOIN_TEST_COMMON_OBJS) bitcoin/chain
 $(BITCOIN_TEST_OBJS): $(CCAN_HEADERS) $(BITCOIN_HEADERS) $(BITCOIN_SRC)
 
 ALL_TEST_PROGRAMS += $(BITCOIN_TEST_PROGRAMS)
-ALL_OBJS += $(BITCOIN_TEST_PROGRAMS:=.o)
+ALL_C_SOURCES += $(BITCOIN_TEST_PROGRAMS:=.c)
 
 # This needs to know what level of optimization we're using.
 bitcoin/test/run-secret_eq_consttime.o: CFLAGS += -DCOPTFLAGS="\"${COPTFLAGS}\""

channeld/Makefile

@@ -1,38 +1,31 @@
 #! /usr/bin/make
 
-# Designed to be run one level up
-channeld-wrongdir:
-	$(MAKE) -C ../.. lightningd/channel-all
-
-default: channeld-all
-
-channeld-all: lightningd/lightning_channeld
-
-LIGHTNINGD_CHANNEL_HEADERS_GEN :=			\
-	channeld/gen_full_channel_error_names.h		\
-	channeld/channeld_wiregen.h
-
-LIGHTNINGD_CHANNEL_HEADERS_NOGEN :=			\
+CHANNELD_HEADERS :=			\
+	channeld/gen_full_channel_error_names.h	\
+	channeld/channeld_wiregen.h		\
 	channeld/channeld_htlc.h		\
 	channeld/commit_tx.h			\
 	channeld/full_channel.h			\
 	channeld/full_channel_error.h		\
 	channeld/watchtower.h
 
-LIGHTNINGD_CHANNEL_HEADERS := $(LIGHTNINGD_CHANNEL_HEADERS_GEN) $(LIGHTNINGD_CHANNEL_HEADERS_NOGEN)
-
-LIGHTNINGD_CHANNEL_SRC := channeld/channeld.c	\
+CHANNELD_SRC := channeld/channeld.c	\
 	channeld/commit_tx.c			\
 	channeld/full_channel.c			\
 	channeld/channeld_wiregen.c		\
 	channeld/watchtower.c
 
-LIGHTNINGD_CHANNEL_OBJS := $(LIGHTNINGD_CHANNEL_SRC:.c=.o)
+CHANNELD_OBJS := $(CHANNELD_SRC:.c=.o)
+$(CHANNELD_OBJS): $(CHANNELD_HEADERS)
 
 # Make sure these depend on everything.
-ALL_OBJS += $(LIGHTNINGD_CHANNEL_OBJS)
+ALL_C_SOURCES += $(CHANNELD_SRC)
+ALL_C_HEADERS += $(CHANNELD_HEADERS)
 ALL_PROGRAMS += lightningd/lightning_channeld
-ALL_GEN_HEADERS += $(LIGHTNINGD_CHANNEL_HEADERS_GEN)
+
+# Here's what lightningd depends on
+LIGHTNINGD_CONTROL_HEADERS += channeld/channeld_wiregen.h
+LIGHTNINGD_CONTROL_OBJS += channeld/channeld_wiregen.o
 
 # Common source we use.
 CHANNELD_COMMON_OBJS :=				\
@@ -96,37 +89,9 @@ CHANNELD_COMMON_OBJS :=				\
 	wire/fromwire.o				\
 	wire/towire.o
 
-LIGHTNINGD_CHANNEL_SRC_GEN := $(filter channeld/%_wiregen.c, $(LIGHTNINGD_CHANNEL_SRC))
-
-LIGHTNINGD_CHANNEL_SRC_NOGEN := $(filter-out channeld/%_wiregen.c, $(LIGHTNINGD_CHANNEL_SRC))
-
-# Control daemon uses this:
-LIGHTNINGD_CHANNEL_CONTROL_HEADERS := $(LIGHTNINGD_CHANNEL_HEADERS_GEN)
-LIGHTNINGD_CHANNEL_CONTROL_SRC := $(LIGHTNINGD_CHANNEL_SRC_GEN)
-LIGHTNINGD_CHANNEL_CONTROL_OBJS := $(LIGHTNINGD_CHANNEL_CONTROL_SRC:.c=.o)
-
-# Add to headers which any object might need.
-LIGHTNINGD_HEADERS_GEN += $(LIGHTNINGD_CHANNEL_HEADERS_GEN)
-LIGHTNINGD_HEADERS_NOGEN += $(LIGHTNINGD_CHANNEL_HEADERS_NOGEN)
-
 channeld/gen_full_channel_error_names.h: channeld/full_channel_error.h ccan/ccan/cdump/tools/cdump-enumstr
 	ccan/ccan/cdump/tools/cdump-enumstr channeld/full_channel_error.h > $@
 
+lightningd/lightning_channeld: $(CHANNELD_OBJS) $(WIRE_ONION_OBJS) $(CHANNELD_COMMON_OBJS) $(WIRE_OBJS) $(BITCOIN_OBJS) $(HSMD_CLIENT_OBJS)
 
-$(LIGHTNINGD_CHANNEL_OBJS): $(LIGHTNINGD_HEADERS) $(LIGHTNINGD_GOSSIP_CONTROL_HEADERS)
-
-LIGHTNINGD_CHANNEL_OBJS := $(LIGHTNINGD_CHANNEL_SRC:.c=.o)
-
-lightningd/lightning_channeld: $(LIGHTNINGD_CHANNEL_OBJS) $(WIRE_ONION_OBJS) $(CHANNELD_COMMON_OBJS) $(WIRE_OBJS) $(BITCOIN_OBJS) $(LIGHTNINGD_HSM_CLIENT_OBJS)
-
-check-source: $(LIGHTNINGD_CHANNEL_SRC_NOGEN:%=check-src-include-order/%)
-check-source-bolt: $(LIGHTNINGD_CHANNEL_SRC:%=bolt-check/%) $(LIGHTNINGD_CHANNEL_HEADERS:%=bolt-check/%)
-
-check-whitespace: $(LIGHTNINGD_CHANNEL_SRC_NOGEN:%=check-whitespace/%) $(LIGHTNINGD_CHANNEL_HEADERS_NOGEN:%=check-whitespace/%)
-
-clean: lightningd/channel-clean
-
-lightningd/channel-clean:
-	$(RM) $(LIGHTNINGD_CHANNEL_OBJS) channeld/gen_*
-
--include channeld/test/Makefile
+include channeld/test/Makefile