CMake build linkage error

[webanck]

Trying to compile on the IN2P3 environment with cmake-3.18.2 compiled from source, I get linkage errors:

[100%] Linking CXX executable pbrt_test
libpbrt_lib.a(cameras.cpp.o): In function `pbrt::RealisticCamera::RenderExitPupil(float, float, char const*) const':
cameras.cpp:(.text+0x40a3): undefined reference to `pstd::optional<pbrt::Bounds2<int> >::optional()'
cameras.cpp:(.text+0x40cb): undefined reference to `pstd::optional<pbrt::Point2<int> >::optional()'
cameras.cpp:(.text+0x40ef): undefined reference to `pstd::optional<int>::optional()'
cameras.cpp:(.text+0x4134): undefined reference to `pstd::optional<pbrt::RGBColorSpace const*>::optional()'
cameras.cpp:(.text+0x417d): undefined reference to `std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >::map()'
libpbrt_lib.a(stats.cpp.o): In function `pbrt::StatsAccumulator::WritePixelImages() const':
stats.cpp:(.text+0x1a27): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x1aac): undefined reference to `pstd::optional<pbrt::Bounds2<int> >::optional()'
stats.cpp:(.text+0x1ad0): undefined reference to `pstd::optional<pbrt::Point2<int> >::optional()'
stats.cpp:(.text+0x1af0): undefined reference to `pstd::optional<int>::optional()'
stats.cpp:(.text+0x1b10): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x1b30): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x1b4b): undefined reference to `pstd::optional<pbrt::RGBColorSpace const*>::optional()'
stats.cpp:(.text+0x1f02): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x1f8a): undefined reference to `pstd::optional<pbrt::Bounds2<int> >::optional()'
stats.cpp:(.text+0x1fae): undefined reference to `pstd::optional<pbrt::Point2<int> >::optional()'
stats.cpp:(.text+0x1fce): undefined reference to `pstd::optional<int>::optional()'
stats.cpp:(.text+0x1fee): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x200e): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x2029): undefined reference to `pstd::optional<pbrt::RGBColorSpace const*>::optional()'
stats.cpp:(.text+0x258f): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x2614): undefined reference to `pstd::optional<pbrt::Bounds2<int> >::optional()'
stats.cpp:(.text+0x2638): undefined reference to `pstd::optional<pbrt::Point2<int> >::optional()'
stats.cpp:(.text+0x2658): undefined reference to `pstd::optional<int>::optional()'
stats.cpp:(.text+0x2678): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x2698): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x26b3): undefined reference to `pstd::optional<pbrt::RGBColorSpace const*>::optional()'
collect2: error: ld returned 1 exit status

How to pass the linkage successfully ?

For more details, below is the cmake output:

-- The CXX compiler identification is GNU 7.3.0
-- The C compiler identification is GNU 7.3.0
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Check for working CXX compiler: /pbs/software/centos-7-x86_64/gcc/7.3.0/bin/c++ - skipped
-- Detecting CXX compile features
-- Detecting CXX compile features - done
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Check for working C compiler: /pbs/software/centos-7-x86_64/gcc/7.3.0/bin/gcc - skipped
-- Detecting C compile features
-- Detecting C compile features - done
-- Setting build type to 'Release' as none was specified.
-- Found Git: /usr/bin/git (found version "1.8.3.1") 
-- Looking for pthread.h
-- Looking for pthread.h - found
-- Performing Test CMAKE_HAVE_LIBC_PTHREAD
-- Performing Test CMAKE_HAVE_LIBC_PTHREAD - Failed
-- Looking for pthread_create in pthreads
-- Looking for pthread_create in pthreads - not found
-- Looking for pthread_create in pthread
-- Looking for pthread_create in pthread - found
-- Found Threads: TRUE  
-- Found ZLIB: /usr/lib64/libz.so (found version "1.2.7") 
-- Configure ILMBASE Version: 2.5.3 Lib API: 25.0.2
-- Looking for include file ucontext.h
-- Looking for include file ucontext.h - found
-- Performing Test ILMBASE_HAVE_CONTROL_REGISTER_SUPPORT
-- Performing Test ILMBASE_HAVE_CONTROL_REGISTER_SUPPORT - Success
-- Looking for include file semaphore.h
-- Looking for include file semaphore.h - found
-- Looking for sem_init in pthread
-- Looking for sem_init in pthread - found
-- Configure OpenEXR Version: 2.5.3 Lib API: 25.0.2
-- Performing Test OPENEXR_IMF_HAVE_SYSCONF_NPROCESSORS_ONLN
-- Performing Test OPENEXR_IMF_HAVE_SYSCONF_NPROCESSORS_ONLN - Success
-- Performing Test OPENEXR_IMF_HAVE_GCC_INLINE_ASM_AVX
-- Performing Test OPENEXR_IMF_HAVE_GCC_INLINE_ASM_AVX - Success
-- clang-format not found.
-- Found Doxygen: /usr/bin/doxygen (found version "1.8.5") found components: doxygen dot 
-- Unable to find -lprofiler
-- Looking for a CUDA compiler
-- Looking for a CUDA compiler - NOTFOUND
-- CUDA not found
-- Performing Test COMPILER_SUPPORTS_MARCH_NATIVE
-- Performing Test COMPILER_SUPPORTS_MARCH_NATIVE - Success
-- Performing Test HAVE_MMAP
-- Performing Test HAVE_MMAP - Success
-- Performing Test HAS_INTRIN_H
-- Performing Test HAS_INTRIN_H - Failed
-- Performing Test HAVE_DECLSPEC_NOINLINE
-- Performing Test HAVE_DECLSPEC_NOINLINE - Failed
-- Performing Test HAVE_ATTRIBUTE_NOINLINE
-- Performing Test HAVE_ATTRIBUTE_NOINLINE - Success
-- Performing Test HAVE__ALIGNED_MALLOC
-- Performing Test HAVE__ALIGNED_MALLOC - Failed
-- Performing Test HAVE_POSIX_MEMALIGN
-- Performing Test HAVE_POSIX_MEMALIGN - Success
-- Performing Test INT64_IS_OWN_TYPE
-- Performing Test INT64_IS_OWN_TYPE - Failed
-- Configuring done
-- Generating done
-- Build files have been written to: <...>/pbrt-v4/build

[pierremoreau]

I'll double-check what I get on Linux, but this seems like a compiler bug as the compiler is told to default-generate optional's default constructor (see the constructor's declaration).

[mmp]

That is very puzzling. Agreed with @pierremoreau. I'm also surprised that it happens in just those two files; I would imagine that default constructor is used in many more files, so if there was a weird failure / compiler issue, I'd expect it to be more widespread. If you do a make clean and then rebuild, does it still happen?

[webanck]

@mmp I am sorry but yes; I tried multiple times from scratch (removing the build directory and creating a new one), even restricting the compilation to a unique thread and it always produces the same error, but even earlier:

[...]
[ 60%] Linking CXX executable pbrt_test
libpbrt_lib.a(cameras.cpp.o): In function `pbrt::RealisticCamera::RenderExitPupil(float, float, char const*) const':
cameras.cpp:(.text+0x40a3): undefined reference to `pstd::optional<pbrt::Bounds2<int> >::optional()'
cameras.cpp:(.text+0x40cb): undefined reference to `pstd::optional<pbrt::Point2<int> >::optional()'
cameras.cpp:(.text+0x40ef): undefined reference to `pstd::optional<int>::optional()'
cameras.cpp:(.text+0x4134): undefined reference to `pstd::optional<pbrt::RGBColorSpace const*>::optional()'
cameras.cpp:(.text+0x417d): undefined reference to `std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >::map()'
libpbrt_lib.a(stats.cpp.o): In function `pbrt::StatsAccumulator::WritePixelImages() const':
stats.cpp:(.text+0x1a27): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x1aac): undefined reference to `pstd::optional<pbrt::Bounds2<int> >::optional()'
stats.cpp:(.text+0x1ad0): undefined reference to `pstd::optional<pbrt::Point2<int> >::optional()'
stats.cpp:(.text+0x1af0): undefined reference to `pstd::optional<int>::optional()'
stats.cpp:(.text+0x1b10): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x1b30): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x1b4b): undefined reference to `pstd::optional<pbrt::RGBColorSpace const*>::optional()'
stats.cpp:(.text+0x1f02): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x1f8a): undefined reference to `pstd::optional<pbrt::Bounds2<int> >::optional()'
stats.cpp:(.text+0x1fae): undefined reference to `pstd::optional<pbrt::Point2<int> >::optional()'
stats.cpp:(.text+0x1fce): undefined reference to `pstd::optional<int>::optional()'
stats.cpp:(.text+0x1fee): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x200e): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x2029): undefined reference to `pstd::optional<pbrt::RGBColorSpace const*>::optional()'
stats.cpp:(.text+0x258f): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x2614): undefined reference to `pstd::optional<pbrt::Bounds2<int> >::optional()'
stats.cpp:(.text+0x2638): undefined reference to `pstd::optional<pbrt::Point2<int> >::optional()'
stats.cpp:(.text+0x2658): undefined reference to `pstd::optional<int>::optional()'
stats.cpp:(.text+0x2678): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x2698): undefined reference to `pstd::optional<float>::optional()'
stats.cpp:(.text+0x26b3): undefined reference to `pstd::optional<pbrt::RGBColorSpace const*>::optional()'
collect2: error: ld returned 1 exit status
make[2]: *** [pbrt_test] Error 1
make[1]: *** [CMakeFiles/pbrt_test.dir/all] Error 2
make: *** [all] Error 2

[pierremoreau]

I'm also surprised that it happens in just those two files; I would imagine that default constructor is used in many more files, so if there was a weird failure / compiler issue, I'd expect it to be more widespread.

It could be that only the first 20 or so undefined references were returned and not all of them, so it could potentially be more widespread than reported.

I tried compiling the latest master with GCC 10.2.0 on Linux without any issues; I did use Ninja rather than make so I’ll try again tomorrow with make and see if I get different results.

@webanck Are you able to try a more recent version of GCC, or using clang instead?

[mmp]

...this might be a g++ bug. Here is an old bug report that fits the symptoms. It's not marked fixed yet, but it's pretty old...

Could you try changing the line optional() = default; to optional() { } in util/pstd.h and see if that happens to fix it?

[webanck]

@pierremoreau I don't have sudoers rights on the environment; that's the problem. So to try with a more recent version of gcc, I would have to compile it from source.
@mmp I followed your advice but got the same output when generating the makefiles and building again from scratch.

[pierremoreau]

Maybe you could force the compiler to instantiate the constructors by adding the following to util/pstd.cpp:

pstd::optional<pbrt::Bounds2<int>> _unused1;
pstd::optional<pbrt::Point2<int>> _unused2;
pstd::optional<pbrt::RGBColorSpace const*> _unused3;
pstd::optional<int> _unused4;
pstd::optional<float> _unused5;

I don't have sudoers rights on the environment; that's the problem. So to try with a more recent version of gcc, I would have to compile it from source.

That's indeed a problem… no clang already installed I guess otherwise you would have tried it.

It's not marked fixed yet, but it's pretty old...

It must have been fixed at some point since I did not ran in it with GCC 10.2.0.

[webanck]

Maybe you could force the compiler to instantiate the constructors by adding the following to util/pstd.cpp

Not all the required types such as Bound2 or Point2 are declared in the scope of util/pstd.cpp.
Forcing the required instanciations in cameras.cpp and util/stats.cpp, I got rid of the "optional" errors.
But I still got multiple occurences of the same error:

libpbrt_lib.a(cameras.cpp.o): In function `pbrt::RealisticCamera::RenderExitPupil(float, float, char const*) const':
cameras.cpp:(.text+0x40a5): undefined reference to `std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >::map()'

My suspected culprit is the line using the Image class.

That's indeed a problem… no clang already installed I guess otherwise you would have tried it.

Exactly.

[webanck]

I just tried to compile in Debug mode, and it worked like a charm!
So what could be wrong between Debug and Release modes?

[pierremoreau]

The optimisation level is different between the two; you could try -Og or -O0 in Release mode and see if you still hit the same issue.

[webanck]

After further tests, it appears to compile and link successfully only with the -g flag and without any optimization enabled, except if the -fkeep-inline-functions flag is used. With this new flag, I now compile in Release mode with the options -DNDEBUG -fkeep-inline-functions -O3.

By the way, adding set(CMAKE_EXPORT_COMPILE_COMMANDS ON) in the CMakeLists.txt, I found the -DNDEBUG flag two times in each compilation command line, so it may not be necessary to add it explicitly (see this line).

[pierremoreau]

Ah, good find!

By the way, adding set(CMAKE_EXPORT_COMPILE_COMMANDS ON) in the CMakeLists.txt, I found the -DNDEBUG flag two times in each compilation command line, so it may not be necessary to add it explicitly (see this line).

Yes, CMake adds -DNDEBUG by default for non-Debug builds; I am working on improving the current CMake configuration and that was one of the changes I have locally.

[mmp]

So as far as a fix (or "workaround", I suppose), pbrt's CMakeLists.txt file should add -fkeep-inline-functions for gcc versions before v8? (Admittedly we don't know when this was fixed, but can adjust the use of that flag as needed in the future...)

[mmp]

I've just (finally) pushed a fix that should in theory add that flag with versions of gcc before 8.0. If you have a chance to try it out and see if it builds out of the box on that system at some point, that'd be fantastic.

[shadeops]

Flagging that the change in 95b4c9b is now breaking with gcc 7.4.

/usr/lib64/gcc/x86_64-suse-linux/7/../../../../x86_64-suse-linux/bin/ld: libpbrt_lib.a(check.cpp.o): in function `__gnu_cxx::recursive_init_error::recursive_init_error()':
check.cpp:(.text._ZN9__gnu_cxx20recursive_init_errorC2Ev[_ZN9__gnu_cxx20recursive_init_errorC5Ev]+0x3): undefined reference to `vtable for __gnu_cxx::recursive_init_error'
collect2: error: ld returned 1 exit status

I was able to build pbrt-v4 up until this change without any issue.

This appears to have exposed another compiler bug that was fixed in gcc 7.5
https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51333

I suspect, but haven't tested/verified, that the original issue is limited to just gcc 7.2, 7.3 and 8.0.
https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81860

Edit, now that I've tested with gcc 7.3
Using -fkeep-inline-functions in gcc 7.3 causes you to run into the error above..

[webanck]

I just tried to compile on the IN2P3 environment, and I am getting the same kind of error as @shadeops:

[ 99%] Linking CXX executable imgtool
[ 99%] Linking CXX executable pbrt
libpbrt_lib.a(check.cpp.o): In function `__gnu_cxx::recursive_init_error::recursive_init_error()':
check.cpp:(.text._ZN9__gnu_cxx20recursive_init_errorC2Ev[_ZN9__gnu_cxx20recursive_init_errorC5Ev]+0x3): undefined reference to `vtable for __gnu_cxx::recursive_init_error'
collect2: error: ld returned 1 exit status
make[2]: *** [imgtool] Error 1
make[1]: *** [CMakeFiles/imgtool.dir/all] Error 2
make[1]: *** Waiting for unfinished jobs....
[100%] Linking CXX executable pbrt_test
libpbrt_lib.a(check.cpp.o): In function `__gnu_cxx::recursive_init_error::recursive_init_error()':
check.cpp:(.text._ZN9__gnu_cxx20recursive_init_errorC2Ev[_ZN9__gnu_cxx20recursive_init_errorC5Ev]+0x3): undefined reference to `vtable for __gnu_cxx::recursive_init_error'
collect2: error: ld returned 1 exit status
make[2]: *** [pbrt] Error 1
make[1]: *** [CMakeFiles/pbrt_exe.dir/all] Error 2
CMakeFiles/pbrt_test.dir/src/pbrt/cmd/pbrt_test.cpp.o: In function `__gnu_cxx::recursive_init_error::recursive_init_error()':
pbrt_test.cpp:(.text._ZN9__gnu_cxx20recursive_init_errorC2Ev[_ZN9__gnu_cxx20recursive_init_errorC5Ev]+0x3): undefined reference to `vtable for __gnu_cxx::recursive_init_error'
collect2: error: ld returned 1 exit status

[webanck]

I should add that my current modified version of the CMakeFiles.txt works but is configured for static build:

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++")
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)

[shadeops]

Came across this while Googling which appears to be the same issue with possible workaround -

google/googletest#2328

[mmp]

Ack--sorry. I've reverted that, then.

@webanck are those the only changes you need to build with gcc 7.3.0, or is it that plus -fkeep-inline-functions? Also, would you mind trying the workaround that @shadeops found, i.e. changing:

   optional() = default;

to

    __attribute__((__used__)) optional() = default;

in util/pstd.h and seeing if it builds with the top-of-tree CMakeLists.txt?

[webanck]

@mmp The complete set of necessary options to compile in the environment of the IN2P3 was:

-static-libgcc -static-libstdc++ -DNDEBUG -fkeep-inline-functions -O3 -march=native -fkeep-inline-functions

Sadly, following your advice with __attribute__((__used__)) optional() = default; still produces the same undefined reference to pstd::optional errors.