Add basic support for response files

Cargo.toml

@@ -101,6 +101,7 @@ chrono = "0.4"
 escargot = "0.3"
 itertools = "0.7"
 predicates = "0.9.0"
+pretty_assertions = "0.5.1"
 selenium-rs = "0.1"
 
 [target.'cfg(unix)'.dependencies]

src/compiler/args.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::cmp::Ordering;
 use std::error::Error;
 use std::ffi::OsString;
@@ -630,6 +631,183 @@ macro_rules! take_arg {
     };
 }
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct ParseError(char);
+
+impl fmt::Display for ParseError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        use std::error::Error;
+        write!(f, "{}: `{}`", self.description(), self.0)
+    }
+}
+
+impl std::error::Error for ParseError {
+    fn description(&self) -> &str {
+        "missing closing quote"
+    }
+}
+
+enum State {
+    /// Within a delimiter.
+    Delimiter,
+    /// After backslash, but before starting word.
+    Backslash,
+    /// Within an unquoted word.
+    Unquoted,
+    /// After backslash in an unquoted word.
+    UnquotedBackslash,
+    /// Within a single quoted word.
+    SingleQuoted,
+    /// Within a double quoted word.
+    DoubleQuoted,
+    /// After backslash inside a double quoted word.
+    DoubleQuotedBackslash,
+}
+
+/// Given the contents of a response file, attempts to parse it as a shell would (mostly)
+/// to preserve the semantics of the arguments, namely, not splitting arguments on whitespace
+/// if that whitespace is encapsulated inside double or single quotes.
+pub fn split_response_contents(s: &str) -> Result<Vec<Cow<str>>, ParseError> {
+    use self::State::*;
+
+    let mut words = Vec::new();
+    let mut chars = s.char_indices();
+    let mut state = Delimiter;
+    let mut ranges = Vec::with_capacity(2);
+    let len = s.len();
+    let mut index = 0;
+
+    macro_rules! coalesce {
+        ($s:ident, $ind:expr, $w:ident, $r:ident) => {
+            if index < $ind {
+                ranges.push(index..$ind);
+            }
+
+            let len = $r.iter().map(|r| r.len()).sum();
+
+            let word = $r.iter().fold(String::with_capacity(len), |mut string, r| {
+                let part = &$s[r.clone()];
+                string.push_str(part);
+                string
+            });
+
+            $w.push(Cow::from(word));
+            $r.clear();
+        };
+    }
+
+    macro_rules! cut {
+        ($ind:expr, $r:ident) => {
+            if index < $ind {
+                ranges.push(index..$ind);
+            }
+
+            index = $ind + 1;
+        };
+    }
+
+    loop {
+        let (ind, c) = chars
+            .next()
+            .map(|(ind, c)| (ind, Some(c)))
+            .unwrap_or_else(|| (len, None));
+        state = match state {
+            Delimiter => match c {
+                None => break,
+                Some('\'') => {
+                    index = ind + 1;
+                    SingleQuoted
+                }
+                Some('\"') => {
+                    index = ind + 1;
+                    DoubleQuoted
+                }
+                Some('\\') => {
+                    index = ind;
+                    Backslash
+                }
+                Some(' ') | Some('\n') | Some('\r') | Some('\t') => Delimiter,
+                _ => {
+                    index = ind;
+                    Unquoted
+                }
+            },
+            Backslash => match c {
+                None => {
+                    coalesce!(s, ind, words, ranges);
+                    break;
+                }
+                Some('\n') => {
+                    coalesce!(s, ind, words, ranges);
+                    Delimiter
+                }
+                _ => Unquoted,
+            },
+            Unquoted => match c {
+                None => {
+                    coalesce!(s, ind, words, ranges);
+                    break;
+                }
+                Some('\'') => {
+                    cut!(ind, ranges);
+                    SingleQuoted
+                }
+                Some('\"') => {
+                    cut!(ind, ranges);
+                    DoubleQuoted
+                }
+                Some('\\') => {
+                    //cut!(ind, ranges);
+                    UnquotedBackslash
+                }
+                Some(' ') | Some('\n') | Some('\r') | Some('\t') => {
+                    coalesce!(s, ind, words, ranges);
+                    Delimiter
+                }
+                _ => Unquoted,
+            },
+            UnquotedBackslash => match c {
+                None => {
+                    coalesce!(s, ind, words, ranges);
+                    break;
+                }
+                Some('\n') => {
+                    cut!(ind, ranges);
+                    Unquoted
+                }
+                _ => Unquoted,
+            },
+            SingleQuoted => match c {
+                None => return Err(ParseError('\'')),
+                Some('\'') => {
+                    cut!(ind, ranges);
+                    Unquoted
+                }
+                _ => SingleQuoted,
+            },
+            DoubleQuoted => match c {
+                None => return Err(ParseError('\"')),
+                Some('\"') => {
+                    cut!(ind, ranges);
+                    Unquoted
+                }
+                Some('\\') => DoubleQuotedBackslash,
+                _ => DoubleQuoted,
+            },
+            DoubleQuotedBackslash => match c {
+                None => return Err(ParseError('\"')),
+                Some('\n') => {
+                    cut!(ind, ranges);
+                    DoubleQuoted
+                }
+                _ => DoubleQuoted,
+            },
+        }
+    }
+
+    Ok(words)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/compiler/clang.rs

@@ -110,6 +110,7 @@ mod test {
     use std::collections::HashMap;
     use std::path::PathBuf;
     use super::*;
+    use tempdir::TempDir;
     use test::utils::*;
 
     fn _parse_arguments(arguments: &[String]) -> CompilerArguments<ParsedArguments> {
@@ -204,4 +205,84 @@ mod test {
         assert_eq!(ovec!["plugin.so"], a.extra_hash_files);
     }
 
+    #[test]
+    fn handle_response_file_with_quotes() {
+        use pretty_assertions::assert_eq;
+
+        let td = TempDir::new("clang_response_file").unwrap();
+        let rsp_path = td.path().join("nix_response.rsp");
+
+        std::fs::copy("tests/ue_linux.rsp", &rsp_path).unwrap();
+
+        let parsed = parses!(format!("@{}", rsp_path.display()));
+
+        assert_eq!(parsed, ParsedArguments {
+            input: PathBuf::from("/home/jake/code/unreal/Engine/Intermediate/Build/Linux/B4D820EA/UnrealHeaderTool/Development/CoreUObject/Module.CoreUObject.5_of_6.cpp"),
+            language: Language::Cxx,
+            depfile: None,
+            outputs: vec![("obj", PathBuf::from("/home/jake/code/unreal/Engine/Intermediate/Build/Linux/B4D820EA/UnrealHeaderTool/Development/CoreUObject/Module.CoreUObject.5_of_6.cpp.o"))].into_iter().collect(),
+            preprocessor_args: ovec![
+                "-nostdinc++",
+                r#"-DFOO=\"Bar Value\""#,
+                "-IThirdParty/Linux/LibCxx/include/",
+                "-IThirdParty/Linux/LibCxx/include/c++/v1",
+                "-DPLATFORM_EXCEPTIONS_DISABLED=0",
+                "-D_LINUX64",
+                "-I/home/jake/code/unreal/Engine/Source",
+                "-I/home/jake/code/unreal/Engine/Source/Runtime/CoreUObject/Private",
+                "-I/home/jake/code/unreal/Engine/Source/Runtime",
+                "-I/home/jake/code/unreal/Engine/Source/Runtime/Projects/Public",
+                "-I/home/jake/code/unreal/Engine/Source/Runtime/Core/Public/Linux",
+                "-I/home/jake/code/unreal/Engine/Source/Runtime/Core/Public",
+                "-I/home/jake/code/unreal/Engine/Source/Runtime/Json/Public",
+                "-I/home/jake/code/unreal/Engine/Source/Developer",
+                "-I/home/jake/code/unreal/Engine/Source/Developer/TargetPlatform/Public",
+                "-I/home/jake/code/unreal/Engine/Intermediate/Build/Linux/B4D820EA/UnrealHeaderTool/Inc/CoreUObject",
+                "-I/home/jake/code/unreal/Engine/Source/Runtime/CoreUObject/Public",
+                "-I/home/path that shouldnt exist because spaces are the devil/but shit happens",
+                "-include",
+                "/home/jake/code/unreal/Engine/Intermediate/Build/Linux/B4D820EA/UnrealHeaderTool/Development/CoreUObject/PCH.CoreUObject.h"
+            ],
+            common_args: ovec![
+                "-pipe",
+                "-Wall",
+                "-Werror",
+                "-Wsequence-point",
+                "-Wdelete-non-virtual-dtor",
+                "-fno-math-errno",
+                "-fno-rtti",
+                "-fcolor-diagnostics",
+                "-Wno-unused-private-field",
+                "-Wno-tautological-compare",
+                "-Wno-undefined-bool-conversion",
+                "-Wno-unused-local-typedef",
+                "-Wno-inconsistent-missing-override",
+                "-Wno-undefined-var-template",
+                "-Wno-unused-lambda-capture",
+                "-Wno-unused-variable",
+                "-Wno-unused-function",
+                "-Wno-switch",
+                "-Wno-unknown-pragmas",
+                "-Wno-invalid-offsetof",
+                "-Wno-gnu-string-literal-operator-template",
+                "-Wshadow",
+                "-Wno-error=shadow",
+                "-Wundef",
+                "-gdwarf-4",
+                "-glldb",
+                "-fstandalone-debug",
+                "-O2",
+                "-fPIC",
+                "-ftls-model=local-dynamic",
+                "-fexceptions",
+                "-target",
+                "x86_64-unknown-linux-gnu",
+                "--sysroot=/home/jake/code/unreal/Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64/v12_clang-6.0.1-centos7/x86_64-unknown-linux-gnu",
+                "-std=c++14"
+            ],
+            extra_hash_files: Vec::new(),
+            msvc_show_includes: false,
+            profile_generate: false,
+        });
+    }
 }

src/compiler/gcc.rs

@@ -596,11 +596,24 @@ impl<'a> Iterator for ExpandIncludeFile<'a> {
                 debug!("failed to read @-file `{}`: {}", file.display(), e);
                 return Some(arg)
             }
+
+            // If any arguments are quoted we need to be a bit more involved than splitting
+            // on whitespace
             if contents.contains('"') || contents.contains('\'') {
-                return Some(arg)
-            }
-            let new_args = contents.split_whitespace().collect::<Vec<_>>();
-            self.stack.extend(new_args.iter().rev().map(|s| s.into()));
+                let contents = contents.trim();
+                match split_response_contents(&contents) {
+                    Ok(words) => {
+                        self.stack.extend(words.into_iter().rev().map(|s| String::from(s).into()));
+                    }
+                    Err(e) => {
+                        debug!("failed to parse @-file `{}`: {}", file.display(), e);
+                        return Some(arg)
+                    }
+                }
+            } else {
+                let new_args: Vec<_> = contents.split_whitespace().collect();
+                self.stack.extend(new_args.iter().rev().map(|s| s.into()));
+            };
         }
     }
 }