yacc tool updates

The yacc tool now recognizes the bison syntax of using --header, --defines
and --graph options in YACCFLAGS without supplying an option argument.
Note that byacc doesn't allow leaving off the option argument, and would
error out in such cases.

Fixes #4326

The yacc tool now recognizes -H as a partial synonym for --header.
It only works in the no-option-argument form - giving an option argument
will likely break things.

Fixes #4327

Tests are expanded to cover these.

The graph file name (-g) is now generated relative to the requested
target file name, not to the source name, to match actual current
behavior. This is rare case - only kicks in if target explicitly requested
with a different base name than source - in this case SCons would emit a
different name than bison produces.  Unlikely to affect any real usage -
you would have to set up the condition described, and additionally use
the graph file as a source for some other build target.

The default file suffix for graph files is set to .gv, which is current
for Bison as of 3.8.  The previous suffix for Bison, dating back to 2.4,
was .dot, which is still the suffix byacc uses.  Docs updated to better
describe when you need to set this. The former default, .vcg, has not
been used since 2006.

Documentation updated for the two header suffix construction variables
to better describe (I hope) what these mean and when to set them. Other
doc updates as well.

Signed-off-by: Mats Wichmann <[email protected]>

CHANGES.txt

@@ -23,6 +23,15 @@ RELEASE  VERSION/DATE TO BE FILLED IN LATER
       calls dunder method __call__. Invoke instance directly."
     - Python 3.9 dropped the alias base64.decodestring, deprecated since 3.1.
       Only used in msvs.py. Use base64.decodebytes instead.
+    - The yacc tool now understands the bison behavior of --header,
+      --defines and --graph being called without option-argument as being
+      synonyms for -d (first two) and -g. -H also recognized as a synonym
+      for -d.  Default value for $YACCVCGFILESUFFIX changed to '.gv'
+      to match current bison default (since bison 3.8).  The graph file name
+      (-g) is now generated relative to the requested target file name,
+      not to the source file name, to match actual current behavior (only
+      affects if target explicitly requested with a different base name
+      than source).  Docs updated.  Fixes #4326 and #4327.
 
 
 RELEASE 4.5.2 -  Sun, 21 Mar 2023 14:08:29 -0700

SCons/Tool/Tool.xml

@@ -37,10 +37,19 @@ Example:
 
 <example_commands>
 # builds foo.c
-env.CFile(target = 'foo.c', source = 'foo.l')
+env.CFile(target='foo.c', source='foo.l')
+
 # builds bar.c
-env.CFile(target = 'bar', source = 'bar.y')
+env.CFile(target='bar', source='bar.y')
 </example_commands>
+
+<para>
+Note that for yacc files,
+the output file name is derived from <parameter>target</parameter>,
+or from the source file name if a target is not specified;
+the traditional yacc default name
+<filename>y.tab.c</filename> is not used.
+</para>
 </summary>
 </builder>
 
@@ -59,10 +68,19 @@ Example:
 
 <example_commands>
 # builds foo.cc
-env.CXXFile(target = 'foo.cc', source = 'foo.ll')
+env.CXXFile(target='foo.cc', source='foo.ll')
+
 # builds bar.cc
-env.CXXFile(target = 'bar', source = 'bar.yy')
+env.CXXFile(target='bar', source='bar.yy')
 </example_commands>
+
+<para>
+Note that for yacc files,
+the output file name is derived from <parameter>target</parameter>,
+or from the source file name if a target is not specified;
+the traditional yacc default name
+<filename>y.tab.cc</filename> is not used.
+</para>
 </summary>
 </builder>
 

SCons/Tool/yacc.py

@@ -23,8 +23,11 @@
 
 """Tool-specific initialization for yacc.
 
-This tool should support multiple yacc implementations,
-but is in actuality biased towards GNU Bison.
+This tool should support multiple yacc implementations, but is in actuality
+biased towards GNU Bison. In particular, it forces the output file name (thus
+avoiding the default convention of y.tab.c or foo.tab.c), so the tool *must*
+support the -o option, which pure POSIX yacc does not.  byacc should be okay
+as an alternative to bison.
 
 There normally shouldn't be any need to import this module directly.
 It will usually be imported through the generic SCons.Tool.Tool()
@@ -64,15 +67,17 @@ def _yaccEmitter(target, source, env, ysuf, hsuf) -> tuple:
         target = [targetBase + ".m"]  # the extension is ".m".
 
     # If -d is specified on the command line, yacc will emit a .h
-    # or .hpp file with the same name as the .c or .cpp output file.
-    if '-d' in flags:
+    # or .hpp file with the same base name as the .c or .cpp output file.
+    # if '-d' in flags:  # add bison options -H, --header, --defines (obsolete)
+    if "-d" in flags or "-H" in flags or "--header" in flags or "--defines" in flags:
         target.append(targetBase + env.subst(hsuf, target=target, source=source))
 
-    # If -g is specified on the command line, yacc will emit a .vcg
-    # file with the same base name as the .y, .yacc, .ym or .yy file.
-    if "-g" in flags:
-        base, ext = os.path.splitext(to_String(source[0]))
-        target.append(base + env.subst("$YACCVCGFILESUFFIX"))
+    # If -g is specified on the command line, yacc will emit a graph
+    # file with the same base name as the .c or .cpp output file.
+    # TODO: should this be handled like -v? i.e. a side effect, not target
+    # if "-g" in flags:  # add bison option --graph
+    if "-g" in flags or "--graph" in flags:
+        target.append(targetBase + env.subst("$YACCVCGFILESUFFIX"))
 
     # If -v is specified yacc will create the output debug file
     # which is not really source for any process, but should
@@ -82,11 +87,13 @@ def _yaccEmitter(target, source, env, ysuf, hsuf) -> tuple:
         env.Clean(target[0], targetBase + '.output')
 
     # With --defines and --graph, the file to write is defined by the option
-    # argument. Extract this and include in the list of targets.
-    # NOTE: a filename passed to the command this way is not modified by SCons,
-    # and so will be interpreted relative to the project top directory at
-    # execution time, while the name added to the target list will be
+    # argument, if present (the no-option-argument cases were caught above).
+    # Extract this and include in the list of targets.
+    # NOTE: a filename passed to the command this way is not modified by
+    # SCons, and so will be interpreted relative to the project top directory
+    # at execution time, while the name added to the target list will be
     # interpreted relative to the SConscript directory - a possible mismatch.
+    # Better to use YACC_HEADER_FILE and YACC_GRAPH_FILE to pass these.
     #
     # These are GNU bison-only options.
     # Since bison 3.8, --header is the preferred name over --defines
@@ -185,7 +192,7 @@ def generate(env) -> None:
     env['YACCCOM'] = '$YACC $YACCFLAGS $_YACC_HEADER $_YACC_GRAPH -o $TARGET $SOURCES'
     env['YACCHFILESUFFIX'] = '.h'
     env['YACCHXXFILESUFFIX'] = '.hpp'
-    env['YACCVCGFILESUFFIX'] = '.vcg'
+    env['YACCVCGFILESUFFIX'] = '.gv'
     env['_YACC_HEADER'] = '${YACC_HEADER_FILE and "--header=" + str(YACC_HEADER_FILE)}'
     env['_YACC_GRAPH'] = '${YACC_GRAPH_FILE and "--graph=" + str(YACC_GRAPH_FILE)}'
 

SCons/Tool/yacc.xml

@@ -115,6 +115,7 @@ with the suffix defined by &cv-link-YACCHFILESUFFIX;
 if the yacc source file ends in a <filename>.y</filename> suffix,
 or a file with the suffix defined by &cv-link-YACCHXXFILESUFFIX;
 if the yacc source file ends in a <filename>.yy</filename> suffix.
+The header will have the same base name as the requested target.
 </para>
 
 <para>
@@ -131,22 +132,31 @@ with the suffix <filename>.output</filename>.
 
 <para>
 Also recognized are GNU &bison; options
-<option>--header=</option> and its deprecated synonym
-<option>--defines=</option>,
+<option>--header</option>
+(and its deprecated synonym <option>--defines</option>),
 which is similar to
 <option>-d</option>
-but the output filename is named by the option argument;
-and <option>--graph=</option>,
+but gives the option to explicitly name the output header file
+through an option argument;
+and <option>--graph</option>,
 which is similar to
 <option>-g</option>
-but the output filename is named by the option argument.
+but gives the option to explicitly name the output graph file
+through an option argument.
+The file suffixes described for
+<option>-d</option> and <option>-g</option> above
+are not applied if these are used in the option=argument form.
 </para>
 
 <para>
 Note that files specified by <option>--header=</option> and
 <option>--graph=</option> may not be properly handled
-by &SCons; in all situations. Consider using
-&cv-link-YACC_HEADER_FILE; and &cv-link-YACC_GRAPH_FILE; instead.
+by &SCons; in all situations, and using those in &cv-YACCFLAGS;
+should be considered legacy support only.
+Consider using &cv-link-YACC_HEADER_FILE;
+and &cv-link-YACC_GRAPH_FILE; instead
+if the files need to be explicitly named
+(<emphasis>new in version 4.4.0</emphasis>).
 </para>
 </summary>
 </cvar>
@@ -159,6 +169,7 @@ Will be emitted as a <option>--header=</option>
 command-line option. Use this in preference to including
 <option>--header=</option> in &cv-link-YACCFLAGS; directly.
 </para>
+<para><emphasis>New in version 4.4.0.</emphasis></para>
 </summary>
 </cvar>
 
@@ -171,6 +182,7 @@ Will be emitted as a <option>--graph=</option>
 command-line option. Use this in preference to including
 <option>--graph=</option> in &cv-link-YACCFLAGS; directly.
 </para>
+<para><emphasis>New in version 4.4.0.</emphasis></para>
 </summary>
 </cvar>
 
@@ -179,14 +191,13 @@ command-line option. Use this in preference to including
 <para>
 The suffix of the C
 header file generated by the parser generator
-when the
-<option>-d</option>
-option is used.
-Note that setting this variable does not cause
-the parser generator to generate a header
-file with the specified suffix,
-it exists to allow you to specify
-what suffix the parser generator will use of its own accord.
+when the <option>-d</option> option
+(or <option>--header</option> without an option-argument)
+is used in &cv-link-YACCFLAGS;.
+Note that setting this variable informs &SCons;
+how to construct the header filename for tracking purposes,
+it does not affect the actual generated filename.
+Set this to match what your parser generator produces.
 The default value is
 <filename>.h</filename>.
 </para>
@@ -198,22 +209,14 @@ The default value is
 <para>
 The suffix of the C++
 header file generated by the parser generator
-when the
-<option>-d</option>
-option is used.
-Note that setting this variable does not cause
-the parser generator to generate a header
-file with the specified suffix,
-it exists to allow you to specify
-what suffix the parser generator will use of its own accord.
-The default value is
-<filename>.hpp</filename>,
-except on Mac OS X,
-where the default is
-<filename>${TARGET.suffix}.h</filename>.
-because the default &bison; parser generator just
-appends <filename>.h</filename>
-to the name of the generated C++ file.
+when the <option>-d</option> option
+(or <option>--header</option> without an option-argument)
+is used in &cv-link-YACCFLAGS;.
+Note that setting this variable informs &SCons;
+how to construct the header filename for tracking purposes,
+it does not affect the actual generated filename.
+Set this to match what your parser generator produces.
+The default value is <filename>.hpp</filename>.
 </para>
 </summary>
 </cvar>
@@ -222,17 +225,22 @@ to the name of the generated C++ file.
 <summary>
 <para>
 The suffix of the file
-containing the VCG grammar automaton definition
-when the
-<option>--graph=</option>
-option is used.
-Note that setting this variable does not cause
-the parser generator to generate a VCG
-file with the specified suffix,
-it exists to allow you to specify
-what suffix the parser generator will use of its own accord.
-The default value is
-<filename>.vcg</filename>.
+containing a graph of the grammar automaton
+when the <option>-g</option> option
+(or <option>--graph=</option> without an option-argument)
+is used in &cv-link-YACCFLAGS;.
+Note that setting this variable informs &SCons;
+how to construct the graph filename for tracking purposes,
+it does not affect the actual generated filename.
+Various yacc tools have emitted various formats
+at different times.
+Set this to match what your parser generator produces.
+The default value is <filename>.gv</filename>.
+</para>
+<para>
+<emphasis>Changed in version 4.X.Y</emphasis>: the default value
+changed from <filename>.vcg</filename> (&bison; stopped generating
+<filename>.vcg</filename> output with version 2.4, in 2006).
 </para>
 </summary>
 </cvar>

test/YACC/BISONFLAGS.py

@@ -47,12 +47,13 @@
 
 test.subdir('sub1')
 test.subdir('sub2')
+test.subdir('sub3')
 
 test.dir_fixture('YACCFLAGS-fixture')
 
 test.write('SConstruct', """\
 DefaultEnvironment(tools=[])
-SConscript(dirs=['sub1', 'sub2'])
+SConscript(dirs=['sub1', 'sub2', 'sub3'])
 """)
 
 # this SConscript is for the options-in-flags version
@@ -90,8 +91,50 @@
 """ % locals())
 test.write(['sub2', 'aaa.y'], "aaa.y\nYACCFLAGS\n")
 
+# this SConscript is to try various other flag combos
+test.write(['sub3', 'SConscript'], """\
+import sys
+
+env = Environment(
+    YACC=r'%(_python_)s myyacc.py',
+    YACCFLAGS='-x --header=header.h --graph=graph.g',
+    tools=['yacc', '%(linker)s', '%(compiler)s'],
+)
+
+def check(targets, expected):
+    t = [str(target) for target in targets]
+    assert t == expected, t
+
+targs1 = env.CFile('trg1', source='aaa.y', YACCFLAGS='-d')
+check(targs1, ['trg1.c', 'trg1.h'])
+
+targs2 = env.CXXFile('trg2', source='aaa.yy', YACCFLAGS='-d')
+check(targs2, ['trg2.cc', 'trg2.hpp'])
+
+targs3 = env.CFile('trg3', source='aaa.y', YACCFLAGS='--defines=zot.q')
+check(targs3, ['trg3.c', 'zot.q'])
+
+targs4 = env.CFile('trg4', source='aaa.y', YACCFLAGS='--header')
+check(targs4, ['trg4.c', 'trg4.h'])
+
+targs5 = env.CFile('trg5', source='aaa.y', YACCFLAGS='-H')
+check(targs5, ['trg5.c', 'trg5.h'])
+
+targs6 = env.CFile('trg6', source='aaa.y', YACCFLAGS='-g')
+check(targs6, ['trg6.c', 'trg6.gv'])
+
+targs7 = env.CFile('trg7', source='aaa.y', YACCFLAGS='-g -H')
+check(targs7, ['trg7.c', 'trg7.h', 'trg7.gv'])
+
+targs8 = env.CFile('trg8', source='aaa.y', YACCFLAGS='--graph --header')
+check(targs8, ['trg8.c', 'trg8.h', 'trg8.gv'])
+""" % locals())
+
+test.write(['sub3', 'aaa.y'], "aaa.y\nYACCFLAGS\n")
+test.write(['sub3', 'aaa.yy'], "aaa.yy\nYACCFLAGS\n")
+
 test.run('.', stderr=None)
-test.must_match(['sub1', 'aaa.c'], "aaa.y\n -x --header=header.h --graph=graph.g\n")
+test.must_match(['sub1', 'aaa.c'], "aaa.y\n-x --header=header.h --graph=graph.g\n")
 
 # NOTE: this behavior is "wrong" but we're keeping it for compat:
 # the generated files should go into 'sub1', not the topdir.
@@ -109,7 +152,7 @@
 sub2 = Path('sub2')
 headerfile = sub2 / 'header.h'
 graphfile = sub2 / 'graph.g'
-yaccflags = f"aaa.y\n -x --header={headerfile} --graph={graphfile}\n"
+yaccflags = f"aaa.y\n-x --header={headerfile} --graph={graphfile}\n"
 test.must_match(['sub2', 'aaa.c'], yaccflags)
 test.must_match(['sub2', 'header.h'], 'yacc header\n')
 test.must_match(['sub2', 'graph.g'], 'yacc graph\n')