Merge pull request #281 from avast-tl/corruption_check

Corruption check is complete

deps/pelib/CMakeLists.txt

@@ -8,8 +8,8 @@ if(CMAKE_CXX_COMPILER)
 endif()
 
 ExternalProject_Add(pelib-project
-	URL https://github.com/avast-tl/pelib/archive/e93eaa7c150f4608a5d02a67f5edc9e54456fe24.zip
-	URL_HASH SHA256=2ffd7e89451c980a1af6d24d4f6dfbb69a660b06ad5de44c481f6431e21de394
+	URL https://github.com/avast-tl/pelib/archive/9ccd3943c1a0917ffdaff5ab983fa7cdbb6dc5a2.zip
+	URL_HASH SHA256=a502c9023a52b3f248b331ce3a12a04c1b668a1d077dc69e03ea86aa9b0543df
 	DOWNLOAD_NAME pelib.zip
 	CMAKE_ARGS
 		# This does not work on MSVC, but may be useful on Linux.

include/retdec/fileformat/file_format/file_format.h

@@ -21,6 +21,20 @@
 namespace retdec {
 namespace fileformat {
 
+/**
+* LoaderErrorInfo - common structure that contains loader error code, error message and user-friendly error message
+*/
+
+struct LoaderErrorInfo
+{
+	LoaderErrorInfo() : loaderErrorCode(0), loaderError(nullptr), loaderErrorUserFriendly(nullptr)
+	{}
+
+	std::uint32_t loaderErrorCode;               // Loader error code, cast to uint32_t
+	const char * loaderError;
+	const char * loaderErrorUserFriendly;
+};
+
 /**
  * FileFormat - abstract class for parsing files
  */
@@ -69,10 +83,11 @@ class FileFormat : public retdec::utils::ByteValueStorage, private retdec::utils
 		CertificateTable *certificateTable;                               ///< table of certificates
 		ElfCoreInfo *elfCoreInfo;                                         ///< information about core file structures
 		Format fileFormat;                                                ///< format of input file
+		LoaderErrorInfo _ldrErrInfo;                                      ///< loader error (e.g. Windows loader error for PE files)
 		bool stateIsValid;                                                ///< internal state of instance
 		std::vector<std::pair<std::size_t, std::size_t>> secHashInfo;     ///< information for calculation of section table hash
-		retdec::utils::Maybe<bool> signatureVerified;                    ///< indicates whether the signature is present and also verified
-		retdec::utils::RangeContainer<std::uint64_t> nonDecodableRanges; ///< Address ranges which should not be decoded for instructions.
+		retdec::utils::Maybe<bool> signatureVerified;                     ///< indicates whether the signature is present and also verified
+		retdec::utils::RangeContainer<std::uint64_t> nonDecodableRanges;  ///< Address ranges which should not be decoded for instructions.
 
 		/// @name Clear methods
 		/// @{
@@ -130,7 +145,9 @@ class FileFormat : public retdec::utils::ByteValueStorage, private retdec::utils
 		virtual std::size_t getByteLength() const override;
 		virtual std::size_t getWordLength() const override;
 		virtual std::size_t getNumberOfNibblesInByte() const override;
+
 		/// @}
+		const LoaderErrorInfo & getLoaderErrorInfo() const;
 
 		/// @name Detection methods
 		/// @{

include/retdec/fileformat/file_format/pe/pe_format.h

@@ -44,6 +44,7 @@ class PeFormat : public FileFormat
 
 		/// @name Initialization methods
 		/// @{
+		void initLoaderErrorInfo();
 		void initStructures();
 		/// @}
 
@@ -147,6 +148,7 @@ class PeFormat : public FileFormat
 		std::size_t getSizeOfHeapCommit() const;
 		std::size_t getNumberOfDataDirectories() const;
 		std::size_t getDeclaredNumberOfDataDirectories() const;
+
 		int getPeClass() const;
 		bool isDotNet() const;
 		bool isPackedDotNet() const;

include/retdec/fileformat/file_format/pe/pe_template.h

@@ -579,7 +579,7 @@ template<int bits> bool peDelayImport(const PeLib::PeHeaderT<bits> &peHeader,
 	import.setAddress(peImageBase(peHeader) + function->address.Value);
 	import.setLibraryIndex(fileIndex);
 	import.invalidateOrdinalNumber();
-	if(library->ordinalNumbersAreValid())
+	if(library->ordinalNumbersAreValid() && function->hint != 0)
 	{
 		import.setOrdinalNumber(function->hint);
 	}

include/retdec/loader/loader/image.h

@@ -74,6 +74,7 @@ class Image : public retdec::utils::ByteValueStorage
 	std::pair<const std::uint8_t*, std::uint64_t> getRawSegmentData(std::uint64_t address) const;
 
 	const std::string& getStatusMessage() const;
+	const retdec::fileformat::LoaderErrorInfo & getLoaderErrorInfo() const;
 
 protected:
 	Segment* insertSegment(std::unique_ptr<Segment> segment);

include/retdec/utils/conversion.h

@@ -205,16 +205,24 @@ template<typename N> void bytesToHexString(const N *data, std::size_t dataSize,
 		size = (size == 0 || offset + size > dataSize) ? dataSize - offset : size;
 	}
 
-	result.clear();
-	result.reserve(size * 2);
-	std::ostringstream oStr;
-
-	for(std::size_t i = 0; i < size; ++i)
+	// Sample: 4A2A008CF1AEE9BA49D8D1DAA22D8E868365ACE633823D464478239F27ED4F18
+	// Tool: redec-fileinfo.exe, Debug, x64, data = image, dataSize = 0xE1BC00
+	// Optimized: This code now takes 0.106 seconds to convert (measured in VS 2015 IDE)
+	// (down from about 40 seconds)
+	const char * intToHex = uppercase ? "0123456789ABCDEF" : "0123456789abcdef";
+	std::size_t hexIndex = 0;
+
+	// Reserve the necessary space for the hexa string
+	result.resize(size * 2);
+
+	// Convert to hexa byte-by-byte. No reallocations
+	for (std::size_t i = 0; i < size; ++i, hexIndex += 2)
 	{
-		byteToHexString(oStr, data[offset + i], uppercase);
-	}
+		std::uint8_t oneByte = data[offset + i];
 
-	result = oStr.str();
+		result[hexIndex + 0] = intToHex[(oneByte >> 0x04) & 0x0F];
+		result[hexIndex + 1] = intToHex[(oneByte >> 0x00) & 0x0F];
+	}
 }
 
 /**

src/fileformat/file_format/file_format.cpp

@@ -117,7 +117,7 @@ bool isAddressFromRegion(const SecSeg *actualRegion, const SecSeg *newRegion, st
  * @param loadFlags Load flags
  */
 FileFormat::FileFormat(std::istream &inputStream, LoadFlags loadFlags) : loadedBytes(&bytes),
-	loadFlags(loadFlags), fileStream(inputStream)
+	loadFlags(loadFlags), fileStream(inputStream), _ldrErrInfo()
 {
 	stateIsValid = !inputStream.fail();
 	init();
@@ -129,7 +129,7 @@ FileFormat::FileFormat(std::istream &inputStream, LoadFlags loadFlags) : loadedB
  * @param loadFlags Load flags
  */
 FileFormat::FileFormat(std::string pathToFile, LoadFlags loadFlags) : loadedBytes(&bytes),
-	loadFlags(loadFlags), filePath(pathToFile), fileStream(auxStream)
+	loadFlags(loadFlags), filePath(pathToFile), fileStream(auxStream), _ldrErrInfo()
 {
 	auxStream.open(filePath, std::ifstream::binary);
 	stateIsValid = auxStream.is_open();
@@ -702,6 +702,15 @@ std::size_t FileFormat::getNumberOfNibblesInByte() const
 	return !getNibbleLength() ? 0 : !(getByteLength() % getNibbleLength()) ? getByteLength() / getNibbleLength() : 0;
 }
 
+/**
+* Get reference to the loader error info
+* @return The LoaderErrorInfo structire
+*/
+const LoaderErrorInfo & FileFormat::getLoaderErrorInfo() const
+{
+	return _ldrErrInfo;
+}
+
 /**
  * Find out if target architecture is x86
  * @return @c true if target architecture is x86, @c false otherwise

src/fileformat/file_format/pe/pe_format.cpp

@@ -305,6 +305,18 @@ PeFormat::~PeFormat()
 	delete formatParser;
 }
 
+/**
+* Init information from PE loader
+*/
+void PeFormat::initLoaderErrorInfo()
+{
+	PeLib::LoaderError ldrError = file->loaderError();
+
+	_ldrErrInfo.loaderErrorCode = static_cast<std::uint32_t>(ldrError);
+	_ldrErrInfo.loaderError = getLoaderErrorString(ldrError, false);
+	_ldrErrInfo.loaderErrorUserFriendly = getLoaderErrorString(ldrError, true);
+}
+
 /**
  * Init internal structures
  */
@@ -332,6 +344,10 @@ void PeFormat::initStructures()
 			file->readResourceDirectory();
 			file->readSecurityDirectory();
 			file->readComHeaderDirectory();
+
+			// Fill-in the loader error info from PE file
+			initLoaderErrorInfo();
+
 			mzHeader = file->mzHeader();
 			switch((peClass = getFileType(filePath)))
 			{

src/fileinfo/file_detector/file_detector.cpp

@@ -245,8 +245,16 @@ void FileDetector::getCertificates()
 /**
  * Get loader information
  */
+
 void FileDetector::getLoaderInfo()
 {
+	// Propagate loader error no matter if the Image pointer will be created or not
+	auto ldrErrInfo = getFileParser()->getLoaderErrorInfo();
+	if (ldrErrInfo.loaderErrorCode != 0)
+	{
+		fileInfo.setLoaderErrorInfo(ldrErrInfo);
+	}
+
 	std::unique_ptr<retdec::loader::Image> image = retdec::loader::createImage(fileParser);
 	if(!image)
 	{

src/fileinfo/file_information/file_information.cpp

@@ -2734,6 +2734,15 @@ const std::string& FileInformation::getLoaderStatusMessage() const
 	return loaderInfo.getStatusMessage();
 }
 
+/**
+* Gets loader error message.
+* @return The error message of the loader.
+*/
+const retdec::fileformat::LoaderErrorInfo & FileInformation::getLoaderErrorInfo() const
+{
+	return loaderInfo.getLoaderErrorInfo();
+}
+
 /**
  * Checks whether .NET information are used.
  * @return @c true if it is used, otherwise @c false/
@@ -3501,6 +3510,15 @@ void FileInformation::setLoaderStatusMessage(const std::string& statusMessage)
 	loaderInfo.setStatusMessage(statusMessage);
 }
 
+/**
+* Sets loader error message.
+* @param statusMessage The loader error message.
+*/
+void FileInformation::setLoaderErrorInfo(const retdec::fileformat::LoaderErrorInfo & ldrErrInfo)
+{
+	loaderInfo.setLoaderErrorInfo(ldrErrInfo);
+}
+
 /**
  * Sets whether .NET information are used.
  * @param set @c true if used, otherwise @c false.

src/fileinfo/file_information/file_information.h

@@ -57,7 +57,7 @@ class FileInformation
 		std::vector<Pattern> malwarePatterns;          ///< detected malware patterns
 		std::vector<Pattern> otherPatterns;            ///< other detected patterns
 		Strings strings;                               ///< detected strings
-		retdec::utils::Maybe<bool> signatureVerified; ///< indicates whether the signature is present and if it is verified
+		retdec::utils::Maybe<bool> signatureVerified;  ///< indicates whether the signature is present and if it is verified
 		DotnetInfo dotnetInfo;                         ///< .NET information
 	public:
 		retdec::cpdetect::ToolInformation toolInfo; ///< detected tools
@@ -408,7 +408,8 @@ class FileInformation
 		std::string getNumberOfLoadedSegmentsStr(std::ios_base &(* format)(std::ios_base &)) const;
 		const LoadedSegment& getLoadedSegment(std::size_t index) const;
 		const std::string& getLoaderStatusMessage() const;
-		/// @}
+		const retdec::fileformat::LoaderErrorInfo & getLoaderErrorInfo() const;
+	    /// @}
 
 		/// @name Getters of @a dotnetInfo
 		/// @{
@@ -500,6 +501,7 @@ class FileInformation
 		void setSignatureVerified(bool verified);
 		void setLoadedBaseAddress(unsigned long long baseAddress);
 		void setLoaderStatusMessage(const std::string& statusMessage);
+		void setLoaderErrorInfo(const retdec::fileformat::LoaderErrorInfo & ldrErrInfo);
 		void setDotnetUsed(bool set);
 		void setDotnetRuntimeVersion(std::uint64_t majorVersion, std::uint64_t minorVersion);
 		void setDotnetMetadataHeaderAddress(std::uint64_t address);

src/fileinfo/file_information/file_information_types/loader_info.cpp

@@ -4,6 +4,7 @@
  * @copyright (c) 2017 Avast Software, licensed under the MIT license
  */
 
+#include "retdec/fileformat/file_format/file_format.h"
 #include "fileinfo/file_information/file_information_types/loader_info.h"
 #include "fileinfo/file_information/file_information_types/type_conversions.h"
 
@@ -85,6 +86,11 @@ const std::string& LoaderInfo::getStatusMessage() const
 	return _statusMessage;
 }
 
+const retdec::fileformat::LoaderErrorInfo & LoaderInfo::getLoaderErrorInfo() const
+{
+	return _ldrErrInfo;
+}
+
 void LoaderInfo::setBaseAddress(unsigned long long baseAddress)
 {
 	_baseAddress = baseAddress;
@@ -95,6 +101,11 @@ void LoaderInfo::setStatusMessage(const std::string& statusMessage)
 	_statusMessage = statusMessage;
 }
 
+void LoaderInfo::setLoaderErrorInfo(const retdec::fileformat::LoaderErrorInfo & ldrErrInfo)
+{
+	_ldrErrInfo = ldrErrInfo;
+}
+
 void LoaderInfo::addLoadedSegment(const LoadedSegment& segment)
 {
 	_loadedSegments.push_back(segment);

src/fileinfo/file_information/file_information_types/loader_info.h

@@ -37,6 +37,8 @@ class LoaderInfo
 		unsigned long long _baseAddress;
 		std::vector<LoadedSegment> _loadedSegments;
 		std::string _statusMessage;
+		retdec::fileformat::LoaderErrorInfo _ldrErrInfo;
+
 	public:
 		LoaderInfo();
 		~LoaderInfo();
@@ -48,12 +50,14 @@ class LoaderInfo
 		unsigned long long getNumberOfLoadedSegments() const;
 		const LoadedSegment& getLoadedSegment(unsigned long long index) const;
 		const std::string& getStatusMessage() const;
+		const retdec::fileformat::LoaderErrorInfo & getLoaderErrorInfo() const;
 		/// @}
 
 		/// @name Setters
 		/// @{
 		void setBaseAddress(unsigned long long baseAddress);
 		void setStatusMessage(const std::string& statusMessage);
+		void setLoaderErrorInfo(const retdec::fileformat::LoaderErrorInfo & ldrErrInfo);
 		/// @}
 
 		/// @name Other methods

src/fileinfo/file_presentation/getters/simple_getter/basic_plain_getter.cpp

@@ -32,6 +32,8 @@ BasicPlainGetter::~BasicPlainGetter()
 
 std::size_t BasicPlainGetter::loadInformation(std::vector<std::string> &desc, std::vector<std::string> &info) const
 {
+	const char * loaderErrorUserFriendly = fileinfo.getLoaderErrorInfo().loaderErrorUserFriendly;
+
 	desc.clear();
 	info.clear();
 
@@ -41,6 +43,11 @@ std::size_t BasicPlainGetter::loadInformation(std::vector<std::string> &desc, st
 	desc.push_back("File format              : ");
 	desc.push_back("File class               : ");
 	desc.push_back("File type                : ");
+
+	// Save the title for loader error (if there was a loader error detected)
+	if(loaderErrorUserFriendly != nullptr)
+		desc.push_back("Loader error             : ");
+
 	desc.push_back("Architecture             : ");
 	desc.push_back("Endianness               : ");
 	desc.push_back("Image base address       : ");
@@ -56,6 +63,11 @@ std::size_t BasicPlainGetter::loadInformation(std::vector<std::string> &desc, st
 	info.push_back(fileinfo.getFileFormat());
 	info.push_back(fileinfo.getFileClass());
 	info.push_back(fileinfo.getFileType());
+
+	// Save the text loader error
+	if(loaderErrorUserFriendly != nullptr)
+		info.push_back(loaderErrorUserFriendly);
+
 	info.push_back(fileinfo.getTargetArchitecture());
 	info.push_back(fileinfo.getEndianness());
 	info.push_back(fileinfo.getImageBaseStr(hexWithPrefix));

src/fileinfo/file_presentation/json_presentation.cpp

@@ -125,6 +125,26 @@ void JsonPresentation::presentErrors(Json::Value &root) const
 	}
 }
 
+/**
+* Present information about Windows PE loader error
+* @param root Parent node in output document
+*/
+void JsonPresentation::presentLoaderError(Json::Value &root) const
+{
+	auto ldrErrInfo = fileinfo.getLoaderErrorInfo();
+
+	if (ldrErrInfo.loaderErrorCode != 0)
+	{
+		Json::Value loaderErrorNode;
+
+		loaderErrorNode["code"] = ldrErrInfo.loaderErrorCode;
+		loaderErrorNode["code_text"] = ldrErrInfo.loaderError;
+		loaderErrorNode["description"] = ldrErrInfo.loaderErrorUserFriendly;
+
+		root["loaderError"] = loaderErrorNode;
+	}
+}
+
 /**
  * Present information about detected compilers and packers
  * @param root Parent node in output document
@@ -668,6 +688,7 @@ bool JsonPresentation::present()
 	Value root, jEp;
 	root["inputFile"] = fileinfo.getPathToFile();
 	presentErrors(root);
+	presentLoaderError(root);
 	presentSimple(BasicJsonGetter(fileinfo), root);
 	if(presentSimple(EntryPointJsonGetter(fileinfo), jEp))
 	{

src/fileinfo/file_presentation/json_presentation.h

@@ -23,6 +23,7 @@ class JsonPresentation : public FilePresentation
 		/// @name Auxiliary presentation methods
 		/// @{
 		void presentErrors(Json::Value &root) const;
+		void presentLoaderError(Json::Value &root) const;
 		void presentCompiler(Json::Value &root) const;
 		void presentLanguages(Json::Value &root) const;
 		void presentRichHeader(Json::Value &root) const;