Tokenizer not applying tab replacement to heredoc/nowdoc closers

package.xml

@@ -143,6 +143,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
       <file baseinstalldir="" name="FinallyTest.php" role="test" />
       <file baseinstalldir="" name="GotoLabelTest.inc" role="test" />
       <file baseinstalldir="" name="GotoLabelTest.php" role="test" />
+      <file baseinstalldir="" name="HeredocNowdocCloserTest.inc" role="test" />
+      <file baseinstalldir="" name="HeredocNowdocCloserTest.php" role="test" />
       <file baseinstalldir="" name="NamedFunctionCallArgumentsTest.inc" role="test" />
       <file baseinstalldir="" name="NamedFunctionCallArgumentsTest.php" role="test" />
       <file baseinstalldir="" name="NullsafeObjectOperatorTest.inc" role="test" />
@@ -2116,6 +2118,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
    <install as="CodeSniffer/Core/Tokenizer/FinallyTest.inc" name="tests/Core/Tokenizer/FinallyTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.php" name="tests/Core/Tokenizer/GotoLabelTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.inc" name="tests/Core/Tokenizer/GotoLabelTest.inc" />
+   <install as="CodeSniffer/Core/Tokenizer/HeredocNowdocCloserTest.php" name="tests/Core/Tokenizer/HeredocNowdocCloserTest.php" />
+   <install as="CodeSniffer/Core/Tokenizer/HeredocNowdocCloserTest.inc" name="tests/Core/Tokenizer/HeredocNowdocCloserTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.inc" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />
@@ -2220,6 +2224,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
    <install as="CodeSniffer/Core/Tokenizer/FinallyTest.inc" name="tests/Core/Tokenizer/FinallyTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.php" name="tests/Core/Tokenizer/GotoLabelTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.inc" name="tests/Core/Tokenizer/GotoLabelTest.inc" />
+   <install as="CodeSniffer/Core/Tokenizer/HeredocNowdocCloserTest.php" name="tests/Core/Tokenizer/HeredocNowdocCloserTest.php" />
+   <install as="CodeSniffer/Core/Tokenizer/HeredocNowdocCloserTest.inc" name="tests/Core/Tokenizer/HeredocNowdocCloserTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.inc" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />

src/Tokenizers/Tokenizer.php

@@ -194,6 +194,8 @@ private function createPositionMap()
             T_DOUBLE_QUOTED_STRING     => true,
             T_HEREDOC                  => true,
             T_NOWDOC                   => true,
+            T_END_HEREDOC              => true,
+            T_END_NOWDOC               => true,
             T_INLINE_HTML              => true,
         ];
 

tests/Core/Tokenizer/HeredocNowdocCloserTest.inc

@@ -0,0 +1,43 @@
+<?php
+
+/* testHeredocCloserNoIndent */
+$heredoc = <<<EOD
+some text
+      some text
+  some text
+EOD;
+
+/* testNowdocCloserNoIndent */
+$nowdoc = <<<'EOD'
+some text
+      some text
+  some text
+EOD;
+
+/* testHeredocCloserSpaceIndent */
+$heredoc = <<<END
+      a
+     b
+    c
+    END;
+
+/* testNowdocCloserSpaceIndent */
+$nowdoc = <<<'END'
+          a
+        b
+      c
+     END;
+
+/* testHeredocCloserTabIndent */
+$heredoc = <<<"END"
+		  a
+		b
+	  c
+	 END;
+
+/* testNowdocCloserTabIndent */
+$nowdoc = <<<'END'
+	  a
+	 b
+	c
+	END;

tests/Core/Tokenizer/HeredocNowdocCloserTest.php

@@ -0,0 +1,150 @@
+<?php
+/**
+ * Tests the tokenization of goto declarations and statements.
+ *
+ * @author    Juliette Reinders Folmer <[email protected]>
+ * @copyright 2020 Squiz Pty Ltd (ABN 77 084 670 600)
+ * @license   https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
+ */
+
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+
+use PHP_CodeSniffer\Config;
+use PHP_CodeSniffer\Ruleset;
+use PHP_CodeSniffer\Files\DummyFile;
+use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest;
+
+/**
+ * Heredoc/nowdoc closer token test.
+ *
+ * @requires PHP 7.3
+ */
+class HeredocNowdocCloserTest extends AbstractMethodUnitTest
+{
+
+
+    /**
+     * Initialize & tokenize \PHP_CodeSniffer\Files\File with code from the test case file.
+     *
+     * {@internal This is a near duplicate of the original method. Only difference is that
+     * tab replacement is enabled for this test.}
+     *
+     * @return void
+     */
+    public static function setUpBeforeClass()
+    {
+        $config            = new Config();
+        $config->standards = ['PSR1'];
+        $config->tabWidth  = 4;
+
+        $ruleset = new Ruleset($config);
+
+        // Default to a file with the same name as the test class. Extension is property based.
+        $relativeCN     = str_replace(__NAMESPACE__, '', get_called_class());
+        $relativePath   = str_replace('\\', DIRECTORY_SEPARATOR, $relativeCN);
+        $pathToTestFile = realpath(__DIR__).$relativePath.'.'.static::$fileExtension;
+
+        // Make sure the file gets parsed correctly based on the file type.
+        $contents  = 'phpcs_input_file: '.$pathToTestFile.PHP_EOL;
+        $contents .= file_get_contents($pathToTestFile);
+
+        self::$phpcsFile = new DummyFile($contents, $ruleset, $config);
+        self::$phpcsFile->process();
+
+    }//end setUpBeforeClass()
+
+
+    /**
+     * Verify that leading (indent) whitespace in a heredoc/nowdoc closer token get the tab replacement treatment.
+     *
+     * @param string $testMarker The comment prefacing the target token.
+     * @param array  $expected   Expectations for the token array.
+     *
+     * @dataProvider dataHeredocNowdocCloserTabReplacement
+     * @covers       PHP_CodeSniffer\Tokenizers\Tokenizer::createPositionMap
+     *
+     * @return void
+     */
+    public function testHeredocNowdocCloserTabReplacement($testMarker, $expected)
+    {
+        $tokens = self::$phpcsFile->getTokens();
+
+        $closer = $this->getTargetToken($testMarker, [T_END_HEREDOC, T_END_NOWDOC]);
+
+        foreach ($expected as $key => $value) {
+            if ($key === 'orig_content' && $value === null) {
+                $this->assertArrayNotHasKey($key, $tokens[$closer], "Unexpected 'orig_content' key found in the token array.");
+                continue;
+            }
+
+            $this->assertArrayHasKey($key, $tokens[$closer], "Key $key not found in the token array.");
+            $this->assertSame($value, $tokens[$closer][$key], "Value for key $key does not match expectation.");
+        }
+
+    }//end testHeredocNowdocCloserTabReplacement()
+
+
+    /**
+     * Data provider.
+     *
+     * @see testHeredocNowdocCloserTabReplacement()
+     *
+     * @return array
+     */
+    public function dataHeredocNowdocCloserTabReplacement()
+    {
+        return [
+            [
+                'testMarker' => '/* testHeredocCloserNoIndent */',
+                'expected'   => [
+                    'length'       => 3,
+                    'content'      => 'EOD',
+                    'orig_content' => null,
+                ],
+            ],
+            [
+                'testMarker' => '/* testNowdocCloserNoIndent */',
+                'expected'   => [
+                    'length'       => 3,
+                    'content'      => 'EOD',
+                    'orig_content' => null,
+                ],
+            ],
+            [
+                'testMarker' => '/* testHeredocCloserSpaceIndent */',
+                'expected'   => [
+                    'length'       => 7,
+                    'content'      => '    END',
+                    'orig_content' => null,
+                ],
+            ],
+            [
+                'testMarker' => '/* testNowdocCloserSpaceIndent */',
+                'expected'   => [
+                    'length'       => 8,
+                    'content'      => '     END',
+                    'orig_content' => null,
+                ],
+            ],
+            [
+                'testMarker' => '/* testHeredocCloserTabIndent */',
+                'expected'   => [
+                    'length'       => 8,
+                    'content'      => '     END',
+                    'orig_content' => '	 END',
+                ],
+            ],
+            [
+                'testMarker' => '/* testNowdocCloserTabIndent */',
+                'expected'   => [
+                    'length'       => 7,
+                    'content'      => '    END',
+                    'orig_content' => '	END',
+                ],
+            ],
+        ];
+
+    }//end dataHeredocNowdocCloserTabReplacement()
+
+
+}//end class