package 中的 linkgroups 丢失 -Wl,--whole-archive

[ivanallen]

Xmake 版本

xmake v2.8.5+dev.9efd45f

操作系统版本和架构

Linux pain 6.5.0-44-generic #44-Ubuntu SMP PREEMPT_DYNAMIC Fri Jun 7 15:10:09 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux

描述问题

我写了 2 个 package，分别是 dpdk 和 spdk。这两个包中的 lib 库都使用了 linkgroups 功能。

依赖关系是这样：

target("mydemo")
    add_files("*.c")
    add_package("spdk")

package("spdk")
    ...
    add_deps("dpdk") -- spdk 依赖了 dpdk
    add_links("s1", "s2", "s3")
    add_linkgroups("s1", "s2", "s3", {name = "spdk", whole = true, group = true})
    on_install(...)


package("dpdk")
   ...
    add_links("d1", "d2", "d3")
    add_linkgroups("d1", "d2", "d3", {name = "dpdk", whole = true, group = true})
    on_install(...)

spdk 和 dpdk 都使用了 linkgroups 这个功能。但是主程序最后 Link 的时候，dpdk 的 -Wl,--whole-archive 丢失。

它会像下面这样 link，d1 d2 d3 这几个文件没有被 --whole-archive 包起来。

g++ -o mydemo mydemo.o -L${spdk_lib_path} -L${dpdk_lib_path} -Wl,--whole-archive -Wl,--start-group -ls1 -ls2 -ls3 -Wl,--end-group -Wl,--no-whole-archive -ld1 -ld2 -ld3

于是报 Undefine 的错误。

---

如果我改成这样

target("mydemo")
    add_files("*.c")
    add_package("spdk")
+   add_package("dpdk") -- 这里增加一行

那么 xmake 会像下面这样去 link，出现重复 link d1 d2 d3，结果会报多重定义错误。

g++ -o mydemo mydemo.o -L${spdk_lib_path} -L${dpdk_lib_path} -Wl,--whole-archive -Wl,--start-group -ls1 -ls2 -ls3 -Wl,--end-group -Wl,--no-whole-archive -ld1 -ld2 -ld3 -Wl,--whole-archive -Wl,--start-group -ld1 -ld2 -ld3 -Wl,--end-group -Wl,--no-whole-archive 

期待的结果

主程序 link 最后 link 的时候，带上 dpdk 的 -Wl,--whole-archive

工程配置

target("nvmf-tgt")
    set_kind("binary")
    add_files("**.c")
    add_packages("spdk")
    -- add_packages("dpdk")

package spdk

package("spdk")
    set_description("The spdk package")

    add_urls("https://github.com/spdk/spdk.git")

    add_versions("v24.09", "19524ad45373e35d5fdc06f4e081444d87ceed80")

    -- sudo apt install libaio-dev
    -- sudo apt install uuid-dev
    -- sudo apt install nasm
    -- sudo apt install autoconf automake libtool
    -- sudo apt install libibverbs-dev
    -- sudo apt install librdmacm-dev
    -- sudo apt install patchelf

    add_deps("dpdk")
    add_deps("uuid", {system = true})
    add_deps("aio", {system = true})
    add_deps("ibverbs", {system = true})
    add_deps("rdmacm", {system = true})
    add_deps("openssl", {system = true})

    add_syslinks("pthread", "m", "dl", "rt")

    add_links("spdk_vhost","spdk_scsi","spdk_dma","spdk_bdev_aio","spdk_bdev_nvme","spdk_rdma_provider", "spdk_rdma_utils",
        "spdk_bdev","spdk_bdev_malloc","spdk_bdev_null","spdk_bdev_virtio","spdk_bdev_passthru",
        "spdk_virtio","spdk_vfio_user","spdk_lvol","spdk_blob","spdk_vmd","spdk_nvme","spdk_nvmf","spdk_sock","spdk_sock_posix",
        "spdk_thread","spdk_trace","spdk_rpc","spdk_jsonrpc","spdk_json","spdk_conf","spdk_util","spdk_log",
        "spdk_notify","spdk_accel","spdk_scheduler_dynamic",
        "spdk_env_dpdk","spdk_env_dpdk_rpc","spdk_event_iobuf",
        "spdk_event_accel","spdk_event_vmd","spdk_event_sock","spdk_event_scheduler",
        "spdk_event_vhost_scsi","spdk_event_vhost_blk",
        "spdk_event_scsi","spdk_event_bdev", "spdk_keyring", "spdk_event_keyring",
        "spdk_event","spdk_init", "isal", "isal_crypto")

    add_linkgroups("spdk_vhost","spdk_scsi","spdk_dma","spdk_bdev_aio","spdk_bdev_nvme","spdk_rdma_provider", "spdk_rdma_utils",
        "spdk_bdev","spdk_bdev_malloc","spdk_bdev_null","spdk_bdev_virtio","spdk_bdev_passthru",
        "spdk_virtio","spdk_vfio_user","spdk_lvol","spdk_blob","spdk_vmd","spdk_nvme","spdk_nvmf","spdk_sock","spdk_sock_posix",
        "spdk_thread","spdk_trace","spdk_rpc","spdk_jsonrpc","spdk_json","spdk_conf","spdk_util","spdk_log",
        "spdk_notify","spdk_accel","spdk_scheduler_dynamic",
        "spdk_env_dpdk","spdk_env_dpdk_rpc","spdk_event_iobuf",
        "spdk_event_accel","spdk_event_vmd","spdk_event_sock","spdk_event_scheduler",
        "spdk_event_vhost_scsi","spdk_event_vhost_blk",
        "spdk_event_scsi","spdk_event_bdev","spdk_keyring", "spdk_event_keyring",
        "spdk_event","spdk_init", "isal", "isal_crypto", {name = "spdk", whole = true, group = true})

    on_install(function (package)
        local configs = {}
        if package:debug() then
            table.insert(configs, "--enable-debug")
        end

        table.insert(configs, "--disable-tests")
        table.insert(configs, "--disable-unit-tests")
        table.insert(configs, "--without-nvme-cuse")
        table.insert(configs, "--with-crypto")
        table.insert(configs, "--with-rdma")
        -- table.insert(configs, "--max-numa-nodes=1")
        table.insert(configs, "--with-dpdk=" .. package:dep("dpdk"):installdir())

        import("package.tools.autoconf").install(package, configs)
        os.cp("include/spdk_internal/*.h", package:installdir("include/spdk_internal"))
        os.cp("isa-l/.libs/*.a", package:installdir("lib"))
        os.cp("isa-l-crypto/.libs/*.a", package:installdir("lib"))
        os.cp("isa-l/include/*.h", package:installdir("include/isa-l"))
        os.cp("isa-l-crypto/include/*.h", package:installdir("include/isa-l-crypto"))
        os.cp("scripts/*", package:installdir("scripts"))
        os.cp("python/*", package:installdir("python"))
        os.cp("$(buildir)/examples/*", package:installdir("examples"))
        os.cp("$(buildir)/fio/*", package:installdir("fio"))
    end)

    on_test(function (package)
        assert(package:has_cincludes("spdk/version.h"))
    end)

package("dpdk")
    set_description("The dpdk package")

    add_urls("https://fast.dpdk.org/rel/dpdk-$(version).tar.xz")
    add_versions("23.11.2", "fbe4f971326653a2b9dc203dbfa990731e590b2e684e038b0513f45abf1a6cda")

    add_deps("meson", "ninja")
    add_deps("zlib")
    add_deps("ibverbs", "crypto", "mana", "mlx4", "elf", {system = true})
    add_links("rte_node", "rte_graph", "rte_pipeline", "rte_table", "rte_pdump",
        "rte_port", "rte_fib", "rte_pdcp", "rte_ipsec", "rte_vhost", "rte_stack",
        "rte_security", "rte_sched", "rte_reorder", "rte_rib", "rte_mldev", "rte_regexdev",
        "rte_rawdev", "rte_power", "rte_pcapng", "rte_member", "rte_lpm", "rte_latencystats",
        "rte_jobstats", "rte_ip_frag", "rte_gso", "rte_gro", "rte_gpudev", "rte_dispatcher", "rte_eventdev",
        "rte_efd", "rte_dmadev", "rte_distributor", "rte_cryptodev", "rte_compressdev", "rte_cfgfile", "rte_bpf",
        "rte_bitratestats", "rte_bbdev", "rte_acl", "rte_timer", "rte_hash", "rte_metrics", "rte_cmdline", "rte_pci",
        "rte_ethdev", "rte_meter", "rte_net", "rte_mbuf", "rte_mempool", "rte_rcu", "rte_ring", "rte_eal", "rte_telemetry",
        "rte_kvargs", "rte_log")
    add_linkgroups("rte_node", "rte_graph", "rte_pipeline", "rte_table", "rte_pdump",
        "rte_port", "rte_fib", "rte_pdcp", "rte_ipsec", "rte_vhost", "rte_stack",
        "rte_security", "rte_sched", "rte_reorder", "rte_rib", "rte_mldev", "rte_regexdev",
        "rte_rawdev", "rte_power", "rte_pcapng", "rte_member", "rte_lpm", "rte_latencystats",
        "rte_jobstats", "rte_ip_frag", "rte_gso", "rte_gro", "rte_gpudev", "rte_dispatcher", "rte_eventdev",
        "rte_efd", "rte_dmadev", "rte_distributor", "rte_cryptodev", "rte_compressdev", "rte_cfgfile", "rte_bpf",
        "rte_bitratestats", "rte_bbdev", "rte_acl", "rte_timer", "rte_hash", "rte_metrics", "rte_cmdline", "rte_pci",
        "rte_ethdev", "rte_meter", "rte_net", "rte_mbuf", "rte_mempool", "rte_rcu", "rte_ring", "rte_eal", "rte_telemetry",
        "rte_kvargs", "rte_log", {name = "dpdk", whole = true})

    on_install(function (package)
        local configs = {}
        table.insert(configs, "--default-library=static")
        table.insert(configs, "-Dmax_numa_nodes=1")
        import("package.tools.meson").install(package, configs)
        -- rm *.so
        if not package:config("shared") then
            os.rm(package:installdir("lib/*.so*"))
        end
    end)

    on_test(function (package)
        assert(package:has_cincludes("rte_eal.h"))
    end)

附加信息和错误日志

部分报错日志

/usr/bin/ld: /home/allen/.xmake/packages/s/spdk/v24.09/8080a6cb324e41189f22895a3d261994/lib/libspdk_env_dpdk.a(pci_dpdk_2207.o): in function `pci_driver_register_2207':
/home/allen/.xmake/cache/packages/2411/s/spdk/v24.09/source/spdk/lib/env_dpdk/pci_dpdk_2207.c:143:(.text+0x25b): undefined reference to `rte_pci_register'
/usr/bin/ld: /home/allen/.xmake/packages/s/spdk/v24.09/8080a6cb324e41189f22895a3d261994/lib/libspdk_env_dpdk.a(pci_dpdk_2207.o): in function `pci_device_write_config_2207':
/home/allen/.xmake/cache/packages/2411/s/spdk/v24.09/source/spdk/lib/env_dpdk/pci_dpdk_2207.c:74:(.text+0x2ad): undefined reference to `rte_pci_write_config'
/usr/bin/ld: /home/allen/.xmake/packages/s/spdk/v24.09/8080a6cb324e41189f22895a3d261994/lib/libspdk_env_dpdk.a(pci_dpdk_2207.o): in function `pci_device_read_config_2207':
/home/allen/.xmake/cache/packages/2411/s/spdk/v24.09/source/spdk/lib/env_dpdk/pci_dpdk_2207.c:64:(.text+0x2dd): undefined reference to `rte_pci_read_config'
/usr/bin/ld: /home/allen/.xmake/packages/s/spdk/v24.09/8080a6cb324e41189f22895a3d261994/lib/libspdk_env_dpdk.a(pci_dpdk_2211.o): in function `pci_driver_register_2211':
/home/allen/.xmake/cache/packages/2411/s/spdk/v24.09/source/spdk/lib/env_dpdk/pci_dpdk_2211.c:151:(.text+0x25b): undefined reference to `rte_pci_register'
/usr/bin/ld: /home/allen/.xmake/packages/s/spdk/v24.09/8080a6cb324e41189f22895a3d261994/lib/libspdk_env_dpdk.a(pci_dpdk_2211.o): in function `pci_device_write_config_2211':
/home/allen/.xmake/cache/packages/2411/s/spdk/v24.09/source/spdk/lib/env_dpdk/pci_dpdk_2211.c:82:(.text+0x2ad): undefined reference to `rte_pci_write_config'
/usr/bin/ld: /home/allen/.xmake/packages/s/spdk/v24.09/8080a6cb324e41189f22895a3d261994/lib/libspdk_env_dpdk.a(pci_dpdk_2211.o): in function `pci_device_read_config_2211':
/home/allen/.xmake/cache/packages/2411/s/spdk/v24.09/source/spdk/lib/env_dpdk/pci_dpdk_2211.c:72:(.text+0x2dd): undefined reference to `rte_pci_read_config'
collect2: error: ld returned 1 exit status
error: execv(/usr/bin/g++ -o build/linux/x86_64/debug/nvmf-tgt build/.objs/nvmf-tgt/linux/x86_64/debug/src/examples/nvmf_tgt/main.c.o -m64 -L/home/allen/.xmake/packages/s/spdk/v24.09/8080a6cb324e41189f22895a3d261994/lib -L/home/allen/.xmake/packages/d/dpdk/23.11.2/6bd94595e356486f9f52aa4c532ee380/lib -L/home/allen/.xmake/packages/z/zlib/v1.3/45ea3fb9069b468faf0ef6782964c796/lib -L/usr/lib/x86_64-linux-gnu -Wl,--whole-archive -Wl,--start-group -lspdk_vhost -lspdk_scsi -lspdk_dma -lspdk_bdev_aio -lspdk_bdev_nvme -lspdk_rdma_provider -lspdk_rdma_utils -lspdk_bdev -lspdk_bdev_malloc -lspdk_bdev_null -lspdk_bdev_virtio -lspdk_bdev_passthru -lspdk_virtio -lspdk_vfio_user -lspdk_lvol -lspdk_blob -lspdk_vmd -lspdk_nvme -lspdk_nvmf -lspdk_sock -lspdk_sock_posix -lspdk_thread -lspdk_trace -lspdk_rpc -lspdk_jsonrpc -lspdk_json -lspdk_conf -lspdk_util -lspdk_log -lspdk_notify -lspdk_accel -lspdk_scheduler_dynamic -lspdk_env_dpdk -lspdk_env_dpdk_rpc -lspdk_event_iobuf -lspdk_event_accel -lspdk_event_vmd -lspdk_event_sock -lspdk_event_scheduler -lspdk_event_vhost_scsi -lspdk_event_vhost_blk -lspdk_event_scsi -lspdk_event_bdev -lspdk_keyring -lspdk_event_keyring -lspdk_event -lspdk_init -lisal -lisal_crypto -Wl,--end-group -Wl,--no-whole-archive -lrte_node -lrte_graph -lrte_pipeline -lrte_table -lrte_pdump -lrte_port -lrte_fib -lrte_pdcp -lrte_ipsec -lrte_vhost -lrte_stack -lrte_security -lrte_sched -lrte_reorder -lrte_rib -lrte_mldev -lrte_regexdev -lrte_rawdev -lrte_power -lrte_pcapng -lrte_member -lrte_lpm -lrte_latencystats -lrte_jobstats -lrte_ip_frag -lrte_gso -lrte_gro -lrte_gpudev -lrte_dispatcher -lrte_eventdev -lrte_efd -lrte_dmadev -lrte_distributor -lrte_cryptodev -lrte_compressdev -lrte_cfgfile -lrte_bpf -lrte_bitratestats -lrte_bbdev -lrte_acl -lrte_timer -lrte_hash -lrte_metrics -lrte_cmdline -lrte_pci -lrte_ethdev -lrte_meter -lrte_net -lrte_mbuf -lrte_mempool -lrte_rcu -lrte_ring -lrte_eal -lrte_telemetry -lrte_kvargs -lrte_log -Wl,--whole-archive -lrte_node -lrte_graph -lrte_pipeline -lrte_table -lrte_pdump -lrte_port -lrte_fib -lrte_pdcp -lrte_ipsec -lrte_vhost -lrte_stack -lrte_security -lrte_sched -lrte_reorder -lrte_rib -lrte_mldev -lrte_regexdev -lrte_rawdev -lrte_power -lrte_pcapng -lrte_member -lrte_lpm -lrte_latencystats -lrte_jobstats -lrte_ip_frag -lrte_gso -lrte_gro -lrte_gpudev -lrte_dispatcher -lrte_eventdev -lrte_efd -lrte_dmadev -lrte_distributor -lrte_cryptodev -lrte_compressdev -lrte_cfgfile -lrte_bpf -lrte_bitratestats -lrte_bbdev -lrte_acl -lrte_timer -lrte_hash -lrte_metrics -lrte_cmdline -lrte_pci -lrte_ethdev -lrte_meter -lrte_net -lrte_mbuf -lrte_mempool -lrte_rcu -lrte_ring -lrte_eal -lrte_telemetry -lrte_kvargs -lrte_log -Wl,--no-whole-archive -luuid -laio -lrdmacm -lssl -lz -libverbs -lcrypto -lmana -lmlx4 -lelf -lpthread -lm -ldl -lrt -fsanitize=address) failed(1)

[Issues-translate-bot]

Bot detected the issue body's language is not English, translate it automatically.

---

Title: package are missing

[ivanallen]

cicd log:
https://github.com/ivanallen/pain/actions/runs/11761774623/job/32763923014?pr=37
cicd 上的 log 看起来和我本地测试刚好相反， spdk 的 -Wl,--whole-archive 丢失了，dpdk 的还在。

pr: https://github.com/ivanallen/pain/pull/37/files#diff-be20a72e11b6d7af7fcdfcc642d605c2fa7379e5cf6407372df8f99a62550c58

[ivanallen]

@waruqi 请问下，这个问题可以帮忙看下吗，是我用的不对还是 bug？需要我补充额外信息吗？

[Issues-translate-bot]

Bot detected the issue body's language is not English, translate it automatically.

---

@waruqi Excuse me, can you help me look into this problem? Is it something I’m using incorrectly or is it a bug? Do you need me to add additional information?

[waruqi]

能给个完整的最小复现工程么？

[Issues-translate-bot]

Bot detected the issue body's language is not English, translate it automatically.

---

Can you give me a complete minimal reproduction project?

[ivanallen]

@waruqi

最小复现工程： https://github.com/ivanallen/test_linkgroups
可直接使用 xmake b -v 进行编译。

最后 link 阶段 spdk_mul 这个库缺少了 -Wl,--whole-archive

[ 75%]: linking.release test
/opt/rh/gcc-toolset-12/root/usr/bin/g++ -o build/linux/x86_64/release/test build/.objs/test/linux/x86_64/release/src/main.cc.o -m64 -L/work/build/.packages/s/spdk/latest/4e0143c97b65425b855ad5fd03038b6a/lib -L/work/build/.packages/d/dpdk/latest/4e0143c97b65425b855ad5fd03038b6a/lib -s -lspdk_mul -Wl,--whole-archive -Wl,--start-group -lrte_add -Wl,--end-group -Wl,--no-whole-archive

[Issues-translate-bot]

Bot detected the issue body's language is not English, translate it automatically.

---

@waruqi

Minimal reproduction project: https://github.com/ivanallen/test_linkgroups
You can use xmake b -v to compile directly.

In the final link stage, the spdk_mul library is missing -Wl,--whole-archive

[75%]: linking.release test
/opt/rh/gcc-toolset-12/root/usr/bin/g++ -o build/linux/x86_64/release/test build/.objs/test/linux/x86_64/release/src/main.cc.o - m64 -L/work/build/.packages/s/spdk/latest/4e0143c97b65425b855ad5fd03038b6a/lib -L/work/build/.packages/d/dpdk/latest/4e0143c97b65425b855ad5fd03038b6a/lib -s -lspdk_mul -Wl, --whole- archive -Wl,--start-group -lrte_add -Wl,--end-group -Wl,--no-whole-archive

[waruqi]

好的，等后两天我看下，最近有点忙

[Issues-translate-bot]

Bot detected the issue body's language is not English, translate it automatically.

---

Okay, I'll check it out in the next two days. I've been a little busy lately.

[waruqi]

试试这个 patch #5866