[build] Normal & Diagnostic Log Verbosity, at once!

Fixes: dotnet#1792

We have a bit of a conundrum: when things break, the diagnostic build
logs are *essential* to tracking down *why* something broke.

However, the sheer *verbosity* of those diagnostic logs makes it
difficult to determine *what* broke; it's very easy to get lost.
Additionally, the gigantic log files make the **Console Output**
Jenkins link nearly useless, as most web browsers will cease up when
attempting to download ~1GB log files.

Split the difference: introduce a new `$(call MSBUILD_BINLOG)` syntax
which creates an [MSBuild binary log][0] file, which is like
diagnostic output *but even moreso*, while *also* overriding the
console logging to `/v:normal`.

[0]: https://github.com/Microsoft/msbuild/wiki/Binary-Log

This should allow the **Console Output** link to be a bit more
reasoanble, and if something goes wrong we can download the separate
`*.binlog` files for further analysis.

MSBuild binary log files can be read with MSBuild itself:

	msbuild /v:diag bin/BuildDebug/msbuild-*.binlog

Note: above command will actually fail; you can only print one log
file at a time with that syntax.

The created log files follow the pattern `msbuild-EPOCH-NAME`, where
EPOCH is the `time_t`/Unix epoch value *at the time the command was
executed*.  This should allow filenames to be non-colliding, and allow
plausibly sane sorting behavior.  (Maybe.)

Makefile

@@ -34,10 +34,10 @@ MSBUILD_FLAGS += /p:AndroidApiLevel=$(API_LEVEL) /p:AndroidFrameworkVersion=$(wo
 endif
 
 all::
-	$(_SLN_BUILD) $(MSBUILD_FLAGS) $(SOLUTION)
+	$(call MSBUILD_BINLOG,all,$(_SLN_BUILD)) $(MSBUILD_FLAGS) $(SOLUTION)
 
 all-tests::
-	MSBUILD="$(MSBUILD)" tools/scripts/xabuild $(MSBUILD_FLAGS) Xamarin.Android-Tests.sln
+	MSBUILD="$(MSBUILD)" $(call MSBUILD_BINLOG,all-tests,tools/scripts/xabuild) $(MSBUILD_FLAGS) Xamarin.Android-Tests.sln
 
 install::
 	@if [ ! -d "bin/$(CONFIGURATION)" ]; then \
@@ -110,7 +110,7 @@ prepare-external:
 
 prepare-deps: prepare-external
 	./build-tools/scripts/generate-os-info Configuration.OperatingSystem.props
-	$(MSBUILD) $(MSBUILD_FLAGS) build-tools/dependencies/dependencies.csproj
+	$(call MSBUILD_BINLOG,prepare-deps) build-tools/dependencies/dependencies.csproj
 
 prepare-props: prepare-deps
 	cp $(call GetPath,JavaInterop)/external/Mono.Cecil* "$(call GetPath,MonoSource)/external"
@@ -121,12 +121,12 @@ prepare-props: prepare-deps
 prepare-msbuild: prepare-props
 ifeq ($(USE_MSBUILD),1)
 	for proj in $(MSBUILD_PREPARE_PROJS); do \
-		$(MSBUILD) $(MSBUILD_FLAGS) "$$proj" || exit 1; \
+		$(call MSBUILD_BINLOG,prepare-msbuild) "$$proj" || exit 1; \
 	done
 endif	# msbuild
 
 prepare-image-dependencies:
-	$(MSBUILD) $(MSBUILD_FLAGS) build-tools/scripts/PrepareImageDependencies.targets /t:PrepareImageDependencies \
+	$(call MSBUILD_BINLOG,prepare-image-deps) build-tools/scripts/PrepareImageDependencies.targets /t:PrepareImageDependencies \
 		/p:AndroidSupportedHostJitAbis=mxe-Win32:mxe-Win64
 	cat bin/Build$(CONFIGURATION)/prepare-image-dependencies.sh | tr -d '\r' > prepare-image-dependencies.sh
 
@@ -152,7 +152,7 @@ $(XA_BUILD_PATHS_OUT): bin/Test%/XABuildPaths.cs: build-tools/scripts/XABuildPat
 
 # Usage: $(call CALL_CREATE_THIRD_PARTY_NOTICES,configuration,path,licenseType,includeExternalDeps,includeBuildDeps)
 define CREATE_THIRD_PARTY_NOTICES
-	$(MSBUILD) $(MSBUILD_FLAGS) $(_MSBUILD_ARGS) \
+	$(call MSBUILD_BINLOG,create-tpn,$(MSBUILD),$(1)) $(_MSBUILD_ARGS) \
 		$(topdir)/build-tools/ThirdPartyNotices/ThirdPartyNotices.csproj \
 		/p:Configuration=$(1) \
 		/p:ThirdPartyNoticeFile=$(topdir)/$(2) \
@@ -183,11 +183,11 @@ $(eval $(call CREATE_THIRD_PARTY_NOTICES_RULE,$(CONFIGURATION),ThirdPartyNotices
 $(eval $(call CREATE_THIRD_PARTY_NOTICES_RULE,$(CONFIGURATION),bin/$(CONFIGURATION)/lib/xamarin.android/ThirdPartyNotices.txt,$(THIRD_PARTY_NOTICE_LICENSE_TYPE),True,False))
 
 run-all-tests:
-	$(MSBUILD) $(MSBUILD_FLAGS) $(TEST_TARGETS) /t:RunAllTests
+	$(call MSBUILD_BINLOG,run-all-tests) $(TEST_TARGETS) /t:RunAllTests
 	$(MAKE) run-api-compatibility-tests
 
 clean:
-	$(MSBUILD) $(MSBUILD_FLAGS) /t:Clean Xamarin.Android.sln
+	$(call MSBUILD_BINLOG,clean) /t:Clean Xamarin.Android.sln
 	tools/scripts/xabuild $(MSBUILD_FLAGS) /t:Clean Xamarin.Android-Tests.sln
 
 distclean:
@@ -198,16 +198,16 @@ distclean:
 
 run-nunit-tests:
 ifeq ($(SKIP_NUNIT_TESTS),)
-	$(MSBUILD) $(MSBUILD_FLAGS) $(TEST_TARGETS) /t:RunNUnitTests
+	$(call MSBUILD_BINLOG,run-nunit-tests) $(TEST_TARGETS) /t:RunNUnitTests
 endif # $(SKIP_NUNIT_TESTS) == ''
 
 run-ji-tests:
-	$(MSBUILD) $(MSBUILD_FLAGS) $(TEST_TARGETS) /t:RunJavaInteropTests
+	$(call MSBUILD_BINLOG,run-ji-tests) $(TEST_TARGETS) /t:RunJavaInteropTests
 
 run-apk-tests:
-	$(MSBUILD) $(MSBUILD_FLAGS) $(TEST_TARGETS) /t:RunApkTests
+	$(call MSBUILD_BINLOG,run-apk-tests) $(TEST_TARGETS) /t:RunApkTests
 
 run-performance-tests:
-	$(MSBUILD) $(MSBUILD_FLAGS) $(TEST_TARGETS) /t:RunPerformanceTests
+	$(call MSBUILD_BINLOG,run-performance-tests) $(TEST_TARGETS) /t:RunPerformanceTests
 
 include build-tools/scripts/runtime-helpers.mk

build-tools/scripts/BuildEverything.mk

@@ -95,7 +95,7 @@ leeroy: leeroy-all framework-assemblies opentk-jcw
 
 leeroy-all:
 	$(foreach conf, $(CONFIGURATIONS), \
-		$(_SLN_BUILD) $(MSBUILD_FLAGS) $(SOLUTION) /p:Configuration=$(conf) $(_MSBUILD_ARGS) && \
+		$(call MSBUILD_BINLOG,leeroy-all,$(_SLN_BUILD),$(conf)) $(SOLUTION) /p:Configuration=$(conf) $(_MSBUILD_ARGS) && \
 		$(call CREATE_THIRD_PARTY_NOTICES,$(conf),bin/$(conf)/lib/xamarin.android/ThirdPartyNotices.txt,$(THIRD_PARTY_NOTICE_LICENSE_TYPE),True,False) && ) \
 	true
 
@@ -109,30 +109,35 @@ framework-assemblies:
 			if [ $$? -ne 0 ] ; then \
 				rm -f bin/$(conf)/lib/xamarin.android/xbuild-frameworks/MonoAndroid/$${CUR_VERSION}/RedistList/FrameworkList.xml; \
 			fi; \
-			$(_SLN_BUILD) $(MSBUILD_FLAGS) src/Mono.Android/Mono.Android.csproj /p:Configuration=$(conf) $(_MSBUILD_ARGS) \
+			$(call MSBUILD_BINLOG,Mono.Android,$(_SLN_BUILD),$(conf)) src/Mono.Android/Mono.Android.csproj \
+				/p:Configuration=$(conf) $(_MSBUILD_ARGS) \
 				/p:AndroidApiLevel=$(a) /p:AndroidPlatformId=$(word $(a), $(ALL_PLATFORM_IDS)) /p:AndroidFrameworkVersion=$${CUR_VERSION} \
 				/p:AndroidPreviousFrameworkVersion=$${PREV_VERSION} || exit 1; ) \
 		PREV_VERSION=$${CUR_VERSION}; )
 	$(foreach conf, $(CONFIGURATIONS), \
 		rm -f bin/$(conf)/lib/xamarin.android/xbuild-frameworks/MonoAndroid/v1.0/Xamarin.Android.NUnitLite.dll; \
-		$(_SLN_BUILD) $(MSBUILD_FLAGS) src/Xamarin.Android.NUnitLite/Xamarin.Android.NUnitLite.csproj /p:Configuration=$(conf) $(_MSBUILD_ARGS) \
+		$(call MSBUILD_BINLOG,NUnitLite,$(_SLN_BUILD),$(conf))) $(MSBUILD_FLAGS) src/Xamarin.Android.NUnitLite/Xamarin.Android.NUnitLite.csproj \
+			/p:Configuration=$(conf) $(_MSBUILD_ARGS) \
 			/p:AndroidApiLevel=$(firstword $(API_LEVELS)) /p:AndroidPlatformId=$(word $(firstword $(API_LEVELS)), $(ALL_PLATFORM_IDS)) \
 			/p:AndroidFrameworkVersion=$(firstword $(FRAMEWORKS)) || exit 1; )
 	_latest_stable_framework=$$($(MSBUILD) /p:DoNotLoadOSProperties=True /nologo /v:minimal /t:GetAndroidLatestStableFrameworkVersion build-tools/scripts/Info.targets | tr -d '[[:space:]]') ; \
 	$(foreach conf, $(CONFIGURATIONS), \
 		rm -f "bin/$(conf)/lib/xamarin.android/xbuild-frameworks/MonoAndroid/$$_latest_stable_framework"/Mono.Android.Export.* ; \
-		$(_SLN_BUILD) $(MSBUILD_FLAGS) src/Mono.Android.Export/Mono.Android.Export.csproj /p:Configuration=$(conf) $(_MSBUILD_ARGS) \
+		$(call MSBUILD_BINLOG,Mono.Android.Export,$(_SLN_BUILD),$(conf)) $(MSBUILD_FLAGS) src/Mono.Android.Export/Mono.Android.Export.csproj \
+			/p:Configuration=$(conf) $(_MSBUILD_ARGS) \
 			/p:AndroidApiLevel=$(firstword $(API_LEVELS)) /p:AndroidPlatformId=$(word $(firstword $(API_LEVELS)), $(ALL_PLATFORM_IDS)) \
 			/p:AndroidFrameworkVersion=$(firstword $(FRAMEWORKS)) || exit 1; ) \
 	$(foreach conf, $(CONFIGURATIONS), \
 		rm -f "bin/$(conf)/lib/xamarin.android/xbuild-frameworks/MonoAndroid/$$_latest_stable_framework"/OpenTK-1.0.* ; \
-		$(_SLN_BUILD) $(MSBUILD_FLAGS) src/OpenTK-1.0/OpenTK.csproj /p:Configuration=$(conf) $(_MSBUILD_ARGS) \
+		$(call MSBUILD_BINLOG,OpenTK,$(_SLN_BUILD),$(conf)) $(MSBUILD_FLAGS) src/OpenTK-1.0/OpenTK.csproj \
+			/p:Configuration=$(conf) $(_MSBUILD_ARGS) \
 			/p:AndroidApiLevel=$(firstword $(API_LEVELS)) /p:AndroidPlatformId=$(word $(firstword $(API_LEVELS)), $(ALL_PLATFORM_IDS)) \
 			/p:AndroidFrameworkVersion=$(firstword $(FRAMEWORKS)) || exit 1; )
 
 opentk-jcw:
 	$(foreach a, $(API_LEVELS), \
 		$(foreach conf, $(CONFIGURATIONS), \
 			touch bin/$(conf)/lib/xamarin.android/xbuild-frameworks/MonoAndroid/*/OpenTK-1.0.dll; \
-			$(_SLN_BUILD) $(MSBUILD_FLAGS) src/OpenTK-1.0/OpenTK.csproj /t:GenerateJavaCallableWrappers /p:Configuration=$(conf) $(_MSBUILD_ARGS) \
+			$(call MSBUILD_BINLOG,OpenTK-JCW,$(_SLN_BUILD),$(conf)) $(MSBUILD_FLAGS) src/OpenTK-1.0/OpenTK.csproj \
+				/t:GenerateJavaCallableWrappers /p:Configuration=$(conf) $(_MSBUILD_ARGS) \
 				/p:AndroidApiLevel=$(a) /p:AndroidPlatformId=$(word $(a), $(ALL_PLATFORM_IDS)) /p:AndroidFrameworkVersion=$(word $(a), $(ALL_FRAMEWORKS)) || exit 1; ))

build-tools/scripts/Packaging.mk

@@ -8,7 +8,7 @@ _BUNDLE_ZIPS_EXCLUDE  = \
 
 create-vsix:
 	$(foreach conf, $(CONFIGURATIONS), \
-		MONO_IOMAP=all MONO_OPTIONS="$(MONO_OPTIONS)" msbuild $(MSBUILD_FLAGS) /p:Configuration=$(conf) /p:CreateVsixContainer=True \
+		MONO_IOMAP=all MONO_OPTIONS="$(MONO_OPTIONS)" $(call MSBUILD_BINLOG,create-vsix) /p:Configuration=$(conf) /p:CreateVsixContainer=True \
 			build-tools/create-vsix/create-vsix.csproj \
 			$(if $(VSIX),"/p:VsixPath=$(VSIX)") \
 			$(if $(EXPERIMENTAL),/p:IsExperimental="$(EXPERIMENTAL)") \

build-tools/scripts/msbuild.mk

@@ -40,11 +40,24 @@ export USE_MSBUILD  = 1
 endif   # $(MSBUILD) == msbuild
 
 ifeq ($(USE_MSBUILD),1)
+
+# $(call MSBUILD_BINLOG,name,msbuild=$(MSBUILD),conf=$(CONFIGURATION))
+define MSBUILD_BINLOG
+	$(if $(2),$(2),$(MSBUILD)) $(MSBUILD_FLAGS) /v:normal \
+		/binaryLogger:"$(dir $(realpath $(firstword $(MAKEFILE_LIST))))/bin/Build$(if $(3),$(3),$(CONFIGURATION))/msbuild-`date +%s`-$(1).binlog"
+endef
+
 else    # $(MSBUILD) != 1
 _CSC_EMITS_PDB  := $(shell if $(_PKG_CONFIG) --atleast-version=4.9 mono ; then echo Pdb; fi )
 ifeq ($(_CSC_EMITS_PDB),Pdb)
 MSBUILD_FLAGS += /p:_DebugFileExt=.pdb
 else    # $(_CSC_EMITS_PDB) == ''
 MSBUILD_FLAGS += /p:_DebugFileExt=.mdb
 endif   # $(_CSC_EMITS_PDB) == Pdb
+
+# $(call MSBUILD_BINLOG,name,msbuild=$(MSBUILD),conf=$(CONFIGURATION))
+define MSBUILD_BINLOG
+	$(if $(2),$(2),$(MSBUILD)) $(MSBUILD_FLAGS)
+endef
+
 endif   # $(USE_MSBUILD) == 1