Merge pull request #4429 from mwichmann/feature/libliteral

Add LIBLITERALPREFIX to support gcc -l:filename

CHANGES.txt

@@ -210,7 +210,9 @@ RELEASE  VERSION/DATE TO BE FILLED IN LATER
     - Eliminate more http: references (mostly in comments/docstrings where
       they really weren't harmful). A few links labeled dead with no alt.
     - Add JDK 21 LTS support
-
+    - Add a LIBLITERALPREFIX variable which can be set to the linker's
+      prefix for considering a library argument unmodified (e.g. for the
+      GNU linker, the ':' in '-l:libfoo.a'). Fixes Github issue #3951.
 
   From Jonathon Reinhart:
     - Fix another instance of `int main()` in CheckLib() causing failures

RELEASE.txt

@@ -23,6 +23,9 @@ NEW FUNCTIONALITY
 - MSVC: If necessary, automatically define VSCMD_SKIP_SENDTELEMETRY for VS2019 and later
   on arm64 hosts when using an arm (32-bit) build of python to prevent a powershell
   error pop-up window (powershell dll not found).
+- Add a LIBLITERALPREFIX variable which can be set to the linker's
+  prefix for considering a library argument unmodified (e.g. for the
+  GNU linker, the ':' in '-l:libfoo.a'). Fixes Github issue #3951.
 
 DEPRECATED FUNCTIONALITY
 ------------------------

SCons/Defaults.py

@@ -36,7 +36,7 @@
 import stat
 import sys
 import time
-from typing import List
+from typing import List, Callable
 
 import SCons.Action
 import SCons.Builder
@@ -455,16 +455,35 @@ def _concat_ixes(prefix, items_iter, suffix, env):
     return result
 
 
-def _stripixes(prefix, itms, suffix, stripprefixes, stripsuffixes, env, c=None):
+def _stripixes(
+    prefix: str,
+    items,
+    suffix: str,
+    stripprefixes: List[str],
+    stripsuffixes: List[str],
+    env,
+    literal_prefix: str = "",
+    c: Callable[[list], list] = None,
+) -> list:
+    """Returns a list with text added to items after first stripping them.
+
+    A companion to :func:`_concat_ixes`, used by tools (like the GNU
+    linker) that need to turn something like ``libfoo.a`` into ``-lfoo``.
+    *stripprefixes* and *stripsuffixes* are stripped from *items*.
+    Calls function *c* to postprocess the result.
+
+    Args:
+        prefix: string to prepend to elements
+        items: string or iterable to transform
+        suffix: string to append to elements
+        stripprefixes: prefix string(s) to strip from elements
+        stripsuffixes: suffix string(s) to strip from elements
+        env: construction environment for variable interpolation
+        c: optional function to perform a transformation on the list.
+           The default is `None`, which will select :func:`_concat_ixes`.
     """
-    This is a wrapper around _concat()/_concat_ixes() that checks for
-    the existence of prefixes or suffixes on list items and strips them
-    where it finds them.  This is used by tools (like the GNU linker)
-    that need to turn something like 'libfoo.a' into '-lfoo'.
-    """
-
-    if not itms:
-        return itms
+    if not items:
+        return items
 
     if not callable(c):
         env_c = env['_concat']
@@ -480,15 +499,27 @@ def _stripixes(prefix, itms, suffix, stripprefixes, stripsuffixes, env, c=None):
     stripprefixes = list(map(env.subst, flatten(stripprefixes)))
     stripsuffixes = list(map(env.subst, flatten(stripsuffixes)))
 
+    # This is a little funky: if literal_prefix is the same as os.pathsep
+    # (e.g. both ':'), the normal conversion to a PathList will drop the
+    # literal_prefix prefix. Tell it not to split in that case, which *should*
+    # be okay because if we come through here, we're normally processing
+    # library names and won't have strings like "path:secondpath:thirdpath"
+    # which is why PathList() otherwise wants to split strings.
+    do_split = not literal_prefix == os.pathsep
+
     stripped = []
-    for l in SCons.PathList.PathList(itms).subst_path(env, None, None):
+    for l in SCons.PathList.PathList(items, do_split).subst_path(env, None, None):
         if isinstance(l, SCons.Node.FS.File):
             stripped.append(l)
             continue
 
         if not is_String(l):
             l = str(l)
 
+        if literal_prefix and l.startswith(literal_prefix):
+            stripped.append(l)
+            continue
+
         for stripprefix in stripprefixes:
             lsp = len(stripprefix)
             if l[:lsp] == stripprefix:

SCons/Defaults.xml

@@ -250,7 +250,7 @@ as the result will be non-portable
 and the directories will not be searched by the dependency scanner.
 &cv-CPPPATH; should be a list of path strings,
 or a single string, not a pathname list joined by
-Python's <systemitem>os.sep</systemitem>.
+Python's <systemitem>os.pathsep</systemitem>.
 </para>
 
 <para>
@@ -473,7 +473,7 @@ when the &cv-link-_LIBDIRFLAGS; variable is automatically generated.
 An automatically-generated construction variable
 containing the linker command-line options
 for specifying libraries to be linked with the resulting target.
-The value of &cv-link-_LIBFLAGS; is created
+The value of &cv-_LIBFLAGS; is created
 by respectively prepending and appending &cv-link-LIBLINKPREFIX;
 and &cv-link-LIBLINKSUFFIX;
 to each filename in &cv-link-LIBS;.
@@ -510,7 +510,7 @@ The list of directories that will be searched for libraries
 specified by the &cv-link-LIBS; &consvar;.
 &cv-LIBPATH; should be a list of path strings,
 or a single string, not a pathname list joined by
-Python's <systemitem>os.sep</systemitem>.
+Python's <systemitem>os.pathsep</systemitem>.
 <!-- XXX OLD
 The implicit dependency scanner will search these
 directories for include files. Don't explicitly put include directory
@@ -573,61 +573,130 @@ env = Environment(LINKCOM="my_linker $_LIBDIRFLAGS $_LIBFLAGS -o $TARGET $SOURCE
 <cvar name="LIBS">
 <summary>
 <para>
-A list of one or more libraries
+The list of libraries
 that will be added to the link line
 for linking with any executable program, shared library, or loadable module
 created by the &consenv; or override.
 </para>
 
 <para>
-String-valued library names should include
-only the library base names,
+For portability,
+a string-valued library name should include
+only the base library name,
 without prefixes such as <filename>lib</filename>
 or suffixes such as <filename>.so</filename> or <filename>.dll</filename>.
-The library list will be added to command lines
-through the automatically-generated
-&cv-_LIBFLAGS; &consvar;
+&SCons; <emphasis>will</emphasis> attempt to
+strip prefixes from the &cv-link-LIBPREFIXES; list
+and suffixes from the &cv-link-LIBSUFFIXES; list,
+but depending on that behavior will make the build
+less portable:
+for example, on a POSIX system,
+no attempt will be made to strip a suffix like
+<filename>.dll</filename>.
+Library name strings in &cv-LIBS; should not include a path component:
+instead use &cv-link-LIBPATH; to direct the compiler
+to look for libraries in those paths,
+plus any default paths the linker searches in.
+If &cv-link-LIBLITERALPREFIX; is set to a non-empty string,
+then a string-valued &cv-LIBS; entry
+that starts with &cv-link-LIBLITERALPREFIX;
+will cause the rest of the entry
+to be searched for for unmodified,
+but respecting normal library search paths
+(this is an exception to the guideline above
+about leaving off the prefix/suffix from the library name).
+</para>
+
+<para>
+If a &cv-LIBS; entry is a Node object
+(either as returned by a previous Builder call,
+or as the result of an explicit call to &f-link-File;),
+the pathname from that Node will be added to
+&cv-_LIBFLAGS;,
+and thus to the link line,
+unmodified - without adding
+&cv-LIBLINKPREFIX;
+or
+&cv-LIBLINKSUFFIX;.
+Such entries are searched for literally
+(including any path component);
+the library search paths are not used.
+For example:
+</para>
+
+<example_commands>
+env.Append(LIBS=File('/tmp/mylib.so'))
+</example_commands>
+
+<para>
+<!-- is this actually true? -->
+For each &Builder; call that causes linking with libraries,
+&SCons; will add the libraries in the setting of &cv-LIBS;
+in effect at that moment to the dependecy graph
+as dependencies of the target being generated.
+</para>
+
+<para>
+The library list will transformed to command line
+arguments through the automatically-generated
+&cv-link-_LIBFLAGS; &consvar;
 which is constructed by
 respectively prepending and appending the values of the
-&cv-LIBLINKPREFIX; and &cv-LIBLINKSUFFIX; &consvars;
-to each library name in &cv-LIBS;.
-Library name strings should not include a
-path component, instead the compiler will be
-directed to look for libraries in the paths
-specified by &cv-link-LIBPATH;.
+&cv-link-LIBLINKPREFIX; and &cv-link-LIBLINKSUFFIX; &consvars;
+to each library name.
 </para>
 
 <para>
-Any command lines you define that need
-the &cv-LIBS; library list should
-include &cv-_LIBFLAGS;:
+Any command lines you define yourself that need
+the libraries from &cv-LIBS; should include &cv-_LIBFLAGS;
+(as well as &cv-link-_LIBDIRFLAGS;)
+rather than &cv-LIBS;.
+For example:
 </para>
 
 <example_commands>
 env = Environment(LINKCOM="my_linker $_LIBDIRFLAGS $_LIBFLAGS -o $TARGET $SOURCE")
 </example_commands>
+</summary>
+</cvar>
 
+<cvar name="LIBLITERALPREFIX">
+<summary>
 <para>
-If you add a
-<classname>File</classname>
-object to the
-&cv-LIBS;
-list, the name of that file will be added to
-&cv-_LIBFLAGS;,
-and thus to the link line, as-is, without
-&cv-LIBLINKPREFIX;
-or
-&cv-LIBLINKSUFFIX;.
-For example:
+If the linker supports command line syntax directing
+that the argument specifying a library should be
+searched for literally (without modification),
+&cv-LIBLITERALPREFIX; can be set to that indicator.
+For example, the GNU linker follows this rule:
+<quote>
+<literal>-l:foo</literal> searches the library path
+for a filename called <literal>foo</literal>,
+without converting it to
+<literal>libfoo.so</literal> or
+<literal>libfoo.a</literal>.
+</quote>
+If &cv-LIBLITERALPREFIX; is set,
+&SCons; will not transform a string-valued entry in
+&cv-link-LIBS; that starts with that string.
+The entry will still be surrounded with
+&cv-link-LIBLINKPREFIX; and &cv-link-LIBLINKSUFFIX;
+on the command line.
+This is useful, for example,
+in directing that a static library
+be used when both a static and dynamic library are available
+and linker policy is to prefer dynamic libraries.
+Compared to the example in &cv-link-LIBS;,
 </para>
-
 <example_commands>
-env.Append(LIBS=File('/tmp/mylib.so'))
+env.Append(LIBS=":libmylib.a")
 </example_commands>
-
 <para>
-In all cases, scons will add dependencies from the executable program to
-all the libraries in this list.
+will let the linker select that specific (static)
+library name if found in the library search path.
+This differs from using a
+<classname>File</classname> object
+to specify the static library,
+as the latter bypasses the library search path entirely.
 </para>
 </summary>
 </cvar>

SCons/Environment.py

@@ -37,6 +37,7 @@
 import shlex
 from collections import UserDict, deque
 from subprocess import PIPE, DEVNULL
+from typing import Optional
 
 import SCons.Action
 import SCons.Builder
@@ -679,7 +680,7 @@ def gvars(self):
     def lvars(self):
         return {}
 
-    def subst(self, string, raw: int=0, target=None, source=None, conv=None, executor=None, overrides: bool=False):
+    def subst(self, string, raw: int=0, target=None, source=None, conv=None, executor=None, overrides: Optional[dict] = None):
         """Recursively interpolates construction variables from the
         Environment into the specified string, returning the expanded
         result.  Construction variables are specified by a $ prefix
@@ -705,9 +706,11 @@ def subst_kw(self, kw, raw: int=0, target=None, source=None):
             nkw[k] = v
         return nkw
 
-    def subst_list(self, string, raw: int=0, target=None, source=None, conv=None, executor=None, overrides: bool=False):
-        """Calls through to SCons.Subst.scons_subst_list().  See
-        the documentation for that function."""
+    def subst_list(self, string, raw: int=0, target=None, source=None, conv=None, executor=None, overrides: Optional[dict] = None):
+        """Calls through to SCons.Subst.scons_subst_list().
+
+        See the documentation for that function.
+        """
         gvars = self.gvars()
         lvars = self.lvars()
         lvars['__env__'] = self
@@ -716,9 +719,10 @@ def subst_list(self, string, raw: int=0, target=None, source=None, conv=None, ex
         return SCons.Subst.scons_subst_list(string, self, raw, target, source, gvars, lvars, conv, overrides=overrides)
 
     def subst_path(self, path, target=None, source=None):
-        """Substitute a path list, turning EntryProxies into Nodes
-        and leaving Nodes (and other objects) as-is."""
+        """Substitute a path list.
 
+        Turns EntryProxies into Nodes, leaving Nodes (and other objects) as-is.
+        """
         if not is_List(path):
             path = [path]