Switch from test-ref-configs.pl to separate components

Makefile

@@ -28,6 +28,9 @@ no_test: programs
 programs: lib mbedtls_test
 	$(MAKE) -C programs
 
+ssl-opt: lib mbedtls_test
+	$(MAKE) -C programs ssl-opt
+
 lib:
 	$(MAKE) -C library
 

programs/CMakeLists.txt

@@ -1,3 +1,9 @@
+set(programs_target "${MBEDTLS_TARGET_PREFIX}programs")
+add_custom_target(${programs_target})
+
+set(ssl_opt_target "${MBEDTLS_TARGET_PREFIX}ssl-opt")
+add_custom_target(${ssl_opt_target})
+
 add_subdirectory(aes)
 add_subdirectory(cipher)
 if (NOT WIN32)

programs/Makefile

@@ -116,6 +116,12 @@ ifndef WINDOWS
 all: fuzz
 endif
 
+SSL_OPT_APPS = $(filter ssl/%,$(APPS))
+SSL_OPT_APPS += test/query_compile_time_config test/udp_proxy
+# Just the programs needed to run ssl-opt.sh (and compat.sh)
+ssl-opt: $(patsubst %,%$(EXEXT),$(SSL_OPT_APPS))
+.PHONY: ssl-opt
+
 fuzz: ${MBEDTLS_TEST_OBJS}
 	$(MAKE) -C fuzz
 

programs/aes/CMakeLists.txt

@@ -1,6 +1,7 @@
 set(executables
     crypt_and_hash
 )
+add_dependencies(${programs_target} ${executables})
 
 foreach(exe IN LISTS executables)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>)

programs/cipher/CMakeLists.txt

@@ -1,6 +1,7 @@
 set(executables
     cipher_aead_demo
 )
+add_dependencies(${programs_target} ${executables})
 
 foreach(exe IN LISTS executables)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>)

programs/fuzz/CMakeLists.txt

@@ -15,6 +15,7 @@ set(executables_no_common_c
     fuzz_x509csr
     fuzz_pkcs7
 )
+add_dependencies(${programs_target} ${executables_no_common_c})
 
 set(executables_with_common_c
     fuzz_privkey
@@ -23,6 +24,7 @@ set(executables_with_common_c
     fuzz_dtlsserver
     fuzz_server
 )
+add_dependencies(${programs_target} ${executables_with_common_c})
 
 foreach(exe IN LISTS executables_no_common_c executables_with_common_c)
 

programs/hash/CMakeLists.txt

@@ -3,6 +3,7 @@ set(executables
     hello
     md_hmac_demo
 )
+add_dependencies(${programs_target} ${executables})
 
 foreach(exe IN LISTS executables)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>)

programs/pkey/CMakeLists.txt

@@ -2,6 +2,7 @@ set(executables_mbedtls
     dh_client
     dh_server
 )
+add_dependencies(${programs_target} ${executables_mbedtls})
 
 foreach(exe IN LISTS executables_mbedtls)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>)
@@ -29,6 +30,7 @@ set(executables_mbedcrypto
     rsa_verify
     rsa_verify_pss
 )
+add_dependencies(${programs_target} ${executables_mbedcrypto})
 
 foreach(exe IN LISTS executables_mbedcrypto)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>)

programs/psa/CMakeLists.txt

@@ -6,6 +6,7 @@ set(executables
     psa_constant_names
     psa_hash
 )
+add_dependencies(${programs_target} ${executables})
 
 if(GEN_FILES)
     add_custom_command(

programs/random/CMakeLists.txt

@@ -2,6 +2,7 @@ set(executables
     gen_entropy
     gen_random_ctr_drbg
 )
+add_dependencies(${programs_target} ${executables})
 
 foreach(exe IN LISTS executables)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>)

programs/ssl/CMakeLists.txt

@@ -16,6 +16,8 @@ set(executables
     ssl_server
     ssl_server2
 )
+add_dependencies(${programs_target} ${executables})
+add_dependencies(${ssl_opt_target} ${executables})
 
 if(GEN_FILES)
     # Inform CMake that the following file will be generated as part of the build

programs/test/CMakeLists.txt

@@ -8,12 +8,16 @@ set(executables_libs
     selftest
     udp_proxy
 )
+add_dependencies(${programs_target} ${executables_libs})
+add_dependencies(${ssl_opt_target} udp_proxy)
 
 set(executables_mbedcrypto
     benchmark
     query_compile_time_config
     zeroize
 )
+add_dependencies(${programs_target} ${executables_mbedcrypto})
+add_dependencies(${ssl_opt_target} query_compile_time_config)
 
 if(TEST_CPP)
     set(cpp_dummy_build_cpp "${CMAKE_CURRENT_BINARY_DIR}/cpp_dummy_build.cpp")

programs/util/CMakeLists.txt

@@ -6,6 +6,7 @@ set(executables
     pem2der
     strerror
 )
+add_dependencies(${programs_target} ${executables})
 
 foreach(exe IN LISTS executables)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>)

programs/x509/CMakeLists.txt

@@ -10,6 +10,7 @@ set(executables
     load_roots
     req_app
 )
+add_dependencies(${programs_target} ${executables})
 
 foreach(exe IN LISTS executables)
     add_executable(${exe} ${exe}.c $<TARGET_OBJECTS:mbedtls_test>)

tests/scripts/analyze_outcomes.py

@@ -576,7 +576,7 @@ def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -
     'analyze_driver_vs_reference_tfm_config': {
         'test_function':  do_analyze_driver_vs_reference,
         'args': {
-            'component_ref': 'test_tfm_config',
+            'component_ref': 'test_tfm_config_no_p256m',
             'component_driver': 'test_tfm_config_p256m_driver_accel_ec',
             'ignored_suites': [
                 # Modules replaced by drivers

tests/scripts/basic-in-docker.sh

@@ -31,7 +31,6 @@ for compiler in clang gcc; do
     run_in_docker programs/test/selftest
     run_in_docker -e OSSL_NO_DTLS=1 tests/compat.sh
     run_in_docker tests/ssl-opt.sh -e '\(DTLS\|SCSV\).*openssl'
-    run_in_docker tests/scripts/test-ref-configs.pl
     run_in_docker tests/scripts/depends.py curves
     run_in_docker tests/scripts/depends.py kex
 done

tests/scripts/components-configuration-crypto.sh

@@ -408,6 +408,25 @@ component_test_full_no_ccm_star_no_tag () {
     make test
 }
 
+component_test_config_symmetric_only () {
+    msg "build: configs/config-symmetric-only.h"
+    cp configs/config-symmetric-only.h "$CONFIG_H"
+    # test-ref-configs works by overwriting mbedtls_config.h; this makes cmake
+    # want to re-generate generated files that depend on it, quite correctly.
+    # However this doesn't work as the generation script expects a specific
+    # format for mbedtls_config.h, which the other files don't follow. Also,
+    # cmake can't know this, but re-generation is actually not necessary as
+    # the generated files only depend on the list of available options, not
+    # whether they're on or off. So, disable cmake's (over-sensitive here)
+    # dependency resolution for generated files and just rely on them being
+    # present (thanks to pre_generate_files) by turning GEN_FILES off.
+    CC=$ASAN_CC cmake -D GEN_FILES=Off -D CMAKE_BUILD_TYPE:String=Asan .
+    make
+
+    msg "test: configs/config-symmetric-only.h - unit tests"
+    make test
+}
+
 component_test_everest () {
     msg "build: Everest ECDH context (ASan build)" # ~ 6 min
     scripts/config.py set MBEDTLS_ECDH_VARIANT_EVEREST_ENABLED
@@ -1413,9 +1432,19 @@ component_test_psa_crypto_config_reference_ecc_ffdh_no_bignum () {
     common_test_psa_crypto_config_reference_ecc_ffdh_no_bignum "ECC_DH"
 }
 
+component_test_tfm_config_as_is () {
+    msg "build: configs/config-tfm.h"
+    cp configs/config-tfm.h "$CONFIG_H"
+    CC=$ASAN_CC cmake -D CMAKE_BUILD_TYPE:String=Asan .
+    make
+
+    msg "test: configs/config-tfm.h - unit tests"
+    make test
+}
+
 # Helper for setting common configurations between:
 # - component_test_tfm_config_p256m_driver_accel_ec()
-# - component_test_tfm_config()
+# - component_test_tfm_config_no_p256m()
 common_tfm_config () {
     # Enable TF-M config
     cp configs/config-tfm.h "$CONFIG_H"
@@ -1469,14 +1498,14 @@ component_test_tfm_config_p256m_driver_accel_ec () {
 # Keep this in sync with component_test_tfm_config_p256m_driver_accel_ec() as
 # they are both meant to be used in analyze_outcomes.py for driver's coverage
 # analysis.
-component_test_tfm_config () {
+component_test_tfm_config_no_p256m () {
     common_tfm_config
 
     # Disable P256M driver, which is on by default, so that analyze_outcomes
     # can compare this test with test_tfm_config_p256m_driver_accel_ec
     echo "#undef MBEDTLS_PSA_P256M_DRIVER_ENABLED" >> "$CONFIG_H"
 
-    msg "build: TF-M config"
+    msg "build: TF-M config without p256m"
     make CFLAGS='-Werror -Wall -Wextra -I../tests/include/spe' tests
 
     # Check that p256m was not built
@@ -1486,7 +1515,7 @@ component_test_tfm_config () {
     # files, so we want to ensure that it has not be re-enabled accidentally.
     not grep mbedtls_cipher ${BUILTIN_SRC_PATH}/cipher.o
 
-    msg "test: TF-M config"
+    msg "test: TF-M config without p256m"
     make test
 }
 

tests/scripts/components-configuration-tls.sh

@@ -9,6 +9,38 @@
 #### Configuration Testing - TLS
 ################################################################
 
+component_test_config_suite_b () {
+    msg "build: configs/config-suite-b.h"
+    cp configs/config-suite-b.h "$CONFIG_H"
+    # test-ref-configs works by overwriting mbedtls_config.h; this makes cmake
+    # want to re-generate generated files that depend on it, quite correctly.
+    # However this doesn't work as the generation script expects a specific
+    # format for mbedtls_config.h, which the other files don't follow. Also,
+    # cmake can't know this, but re-generation is actually not necessary as
+    # the generated files only depend on the list of available options, not
+    # whether they're on or off. So, disable cmake's (over-sensitive here)
+    # dependency resolution for generated files and just rely on them being
+    # present (thanks to pre_generate_files) by turning GEN_FILES off.
+    CC=$ASAN_CC cmake -D GEN_FILES=Off -D CMAKE_BUILD_TYPE:String=Asan .
+    make
+
+    msg "test: configs/config-suite-b.h - unit tests"
+    make test
+
+    msg "test: configs/config-suite-b.h - compat.sh"
+    tests/compat.sh -m tls12 -f 'ECDHE_ECDSA.*AES.*GCM' -p mbedTLS
+
+    msg "build: configs/config-suite-b.h + DEBUG"
+    MBEDTLS_TEST_CONFIGURATION="$MBEDTLS_TEST_CONFIGURATION+DEBUG"
+    make clean
+    scripts/config.py set MBEDTLS_DEBUG_C
+    scripts/config.py set MBEDTLS_ERROR_C
+    make ssl-opt
+
+    msg "test: configs/config-suite-b.h + DEBUG - ssl-opt.sh"
+    tests/ssl-opt.sh
+}
+
 component_test_no_renegotiation () {
     msg "build: Default + !MBEDTLS_SSL_RENEGOTIATION (ASan build)" # ~ 6 min
     scripts/config.py unset MBEDTLS_SSL_RENEGOTIATION
@@ -130,6 +162,28 @@ component_test_tls1_2_default_cbc_legacy_cbc_etm_cipher_only () {
     tests/ssl-opt.sh -f "TLS 1.2"
 }
 
+component_test_config_thread () {
+    msg "build: configs/config-thread.h"
+    cp configs/config-thread.h "$CONFIG_H"
+    # test-ref-configs works by overwriting mbedtls_config.h; this makes cmake
+    # want to re-generate generated files that depend on it, quite correctly.
+    # However this doesn't work as the generation script expects a specific
+    # format for mbedtls_config.h, which the other files don't follow. Also,
+    # cmake can't know this, but re-generation is actually not necessary as
+    # the generated files only depend on the list of available options, not
+    # whether they're on or off. So, disable cmake's (over-sensitive here)
+    # dependency resolution for generated files and just rely on them being
+    # present (thanks to pre_generate_files) by turning GEN_FILES off.
+    CC=$ASAN_CC cmake -D GEN_FILES=Off -D CMAKE_BUILD_TYPE:String=Asan .
+    make
+
+    msg "test: configs/config-thread.h - unit tests"
+    make test
+
+    msg "test: configs/config-thread.h - ssl-opt.sh"
+    tests/ssl-opt.sh -f 'ECJPAKE.*nolog'
+}
+
 # We're not aware of any other (open source) implementation of EC J-PAKE in TLS
 # that we could use for interop testing. However, we now have sort of two
 # implementations ourselves: one using PSA, the other not. At least test that
@@ -163,6 +217,60 @@ component_test_tls1_2_ecjpake_compatibility () {
     rm s2_no_use_psa c2_no_use_psa
 }
 
+component_test_tls1_2_ccm_psk () {
+    msg "build: configs/config-ccm-psk-tls1_2.h"
+    cp configs/config-ccm-psk-tls1_2.h "$CONFIG_H"
+    # test-ref-configs works by overwriting mbedtls_config.h; this makes cmake
+    # want to re-generate generated files that depend on it, quite correctly.
+    # However this doesn't work as the generation script expects a specific
+    # format for mbedtls_config.h, which the other files don't follow. Also,
+    # cmake can't know this, but re-generation is actually not necessary as
+    # the generated files only depend on the list of available options, not
+    # whether they're on or off. So, disable cmake's (over-sensitive here)
+    # dependency resolution for generated files and just rely on them being
+    # present (thanks to pre_generate_files) by turning GEN_FILES off.
+    CC=$ASAN_CC cmake -D GEN_FILES=Off -D CMAKE_BUILD_TYPE:String=Asan .
+    make
+
+    msg "test: configs/config-ccm-psk-tls1_2.h - unit tests"
+    make test
+
+    msg "test: configs/config-ccm-psk-tls1_2.h - compat.sh"
+    tests/compat.sh -m tls12 -f '^TLS_PSK_WITH_AES_..._CCM_8'
+}
+
+component_test_tls1_2_ccm_psk_dtls () {
+    msg "build: configs/config-ccm-psk-dtls1_2.h"
+    cp configs/config-ccm-psk-dtls1_2.h "$CONFIG_H"
+    # test-ref-configs works by overwriting mbedtls_config.h; this makes cmake
+    # want to re-generate generated files that depend on it, quite correctly.
+    # However this doesn't work as the generation script expects a specific
+    # format for mbedtls_config.h, which the other files don't follow. Also,
+    # cmake can't know this, but re-generation is actually not necessary as
+    # the generated files only depend on the list of available options, not
+    # whether they're on or off. So, disable cmake's (over-sensitive here)
+    # dependency resolution for generated files and just rely on them being
+    # present (thanks to pre_generate_files) by turning GEN_FILES off.
+    CC=$ASAN_CC cmake -D GEN_FILES=Off -D CMAKE_BUILD_TYPE:String=Asan .
+    make
+
+    msg "test: configs/config-ccm-psk-dtls1_2.h - unit tests"
+    make test
+
+    msg "test: configs/config-ccm-psk-dtls1_2.h - compat.sh"
+    tests/compat.sh -m dtls12 -f '^TLS_PSK_WITH_AES_..._CCM_8'
+
+    msg "build: configs/config-ccm-psk-dtls1_2.h + DEBUG"
+    MBEDTLS_TEST_CONFIGURATION="$MBEDTLS_TEST_CONFIGURATION+DEBUG"
+    make clean
+    scripts/config.py set MBEDTLS_DEBUG_C
+    scripts/config.py set MBEDTLS_ERROR_C
+    make ssl-opt
+
+    msg "test: configs/config-ccm-psk-dtls1_2.h + DEBUG - ssl-opt.sh"
+    tests/ssl-opt.sh
+}
+
 component_test_small_ssl_out_content_len () {
     msg "build: small SSL_OUT_CONTENT_LEN (ASan build)"
     scripts/config.py set MBEDTLS_SSL_IN_CONTENT_LEN 16384

tests/scripts/components-configuration.sh

@@ -129,21 +129,6 @@ component_test_full_cmake_gcc_asan_new_bignum () {
     tests/context-info.sh
 }
 
-component_test_ref_configs () {
-    msg "test/build: ref-configs (ASan build)" # ~ 6 min 20s
-    # test-ref-configs works by overwriting mbedtls_config.h; this makes cmake
-    # want to re-generate generated files that depend on it, quite correctly.
-    # However this doesn't work as the generation script expects a specific
-    # format for mbedtls_config.h, which the other files don't follow. Also,
-    # cmake can't know this, but re-generation is actually not necessary as
-    # the generated files only depend on the list of available options, not
-    # whether they're on or off. So, disable cmake's (over-sensitive here)
-    # dependency resolution for generated files and just rely on them being
-    # present (thanks to pre_generate_files) by turning GEN_FILES off.
-    CC=$ASAN_CC cmake -D GEN_FILES=Off -D CMAKE_BUILD_TYPE:String=Asan .
-    tests/scripts/test-ref-configs.pl
-}
-
 component_test_full_cmake_clang () {
     msg "build: cmake, full config, clang" # ~ 50s
     scripts/config.py full