Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fuzzy hashing: SSDEEP and CTPH #251

Merged
merged 1 commit into from
Mar 4, 2018
Merged

Fuzzy hashing: SSDEEP and CTPH #251

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@
"bootstrap-switch": "^3.3.4",
"crypto-api": "^0.8.0",
"crypto-js": "^3.1.9-1",
"ctph.js": "0.0.5",
"diff": "^3.4.0",
"escodegen": "^1.9.1",
"esmangle": "^1.0.1",
Expand All @@ -101,6 +102,7 @@
"sladex-blowfish": "^0.8.1",
"sortablejs": "^1.7.0",
"split.js": "^1.3.5",
"ssdeep.js": "0.0.2",
"ua-parser-js": "^0.7.17",
"utf8": "^3.0.0",
"vkbeautify": "^0.99.3",
Expand Down
4 changes: 4 additions & 0 deletions src/core/config/Categories.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,10 @@ const Categories = [
"HAS-160",
"Whirlpool",
"Snefru",
"SSDEEP",
"CTPH",
"Compare SSDEEP hashes",
"Compare CTPH hashes",
"HMAC",
"Fletcher-8 Checksum",
"Fletcher-16 Checksum",
Expand Down
40 changes: 40 additions & 0 deletions src/core/config/OperationConfig.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3174,6 +3174,46 @@ const OperationConfig = {
}
]
},
"SSDEEP": {
module: "Hashing",
description: "SSDEEP is a program for computing context triggered piecewise hashes (CTPH). Also called fuzzy hashes, CTPH can match inputs that have homologies. Such inputs have sequences of identical bytes in the same order, although bytes in between these sequences may be different in both content and length.<br><br>SSDEEP hashes are now widely used for simple identification purposes (e.g. the 'Basic Properties' section in VirusTotal). Although 'better' fuzzy hashes are available, SSDEEP is still one of the primary choices because of its speed and being a de facto standard.<br><br>This operation is fundamentally the same as the CTPH operation, however their outputs differ in format.",
inputType: "string",
outputType: "string",
args: []
},
"CTPH": {
module: "Hashing",
description: "Context Triggered Piecewise Hashing, also called Fuzzy Hashing, can match inputs that have homologies. Such inputs have sequences of identical bytes in the same order, although bytes in between these sequences may be different in both content and length.<br><br>CTPH was originally based on the work of Dr. Andrew Tridgell and a spam email detector called SpamSum. This method was adapted by Jesse Kornblum and published at the DFRWS conference in 2006 in a paper 'Identifying Almost Identical Files Using Context Triggered Piecewise Hashing'.",
inputType: "string",
outputType: "string",
args: []
},
"Compare SSDEEP hashes": {
module: "Hashing",
description: "Compares two SSDEEP fuzzy hashes to determine the similarity between them on a scale of 0 to 100.",
inputType: "string",
outputType: "Number",
args: [
{
name: "Delimiter",
type: "option",
value: Hash.DELIM_OPTIONS
}
]
},
"Compare CTPH hashes": {
module: "Hashing",
description: "Compares two Context Triggered Piecewise Hashing (CTPH) fuzzy hashes to determine the similarity between them on a scale of 0 to 100.",
inputType: "string",
outputType: "Number",
args: [
{
name: "Delimiter",
type: "option",
value: Hash.DELIM_OPTIONS
}
]
},
"HMAC": {
module: "Hashing",
description: "Keyed-Hash Message Authentication Codes (HMAC) are a mechanism for message authentication using cryptographic hash functions.",
Expand Down
54 changes: 29 additions & 25 deletions src/core/config/modules/Hashing.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,31 +18,35 @@ import Hash from "../../operations/Hash.js";
let OpModules = typeof self === "undefined" ? {} : self.OpModules || {};

OpModules.Hashing = {
"Analyse hash": Hash.runAnalyse,
"Generate all hashes": Hash.runAll,
"MD2": Hash.runMD2,
"MD4": Hash.runMD4,
"MD5": Hash.runMD5,
"MD6": Hash.runMD6,
"SHA0": Hash.runSHA0,
"SHA1": Hash.runSHA1,
"SHA2": Hash.runSHA2,
"SHA3": Hash.runSHA3,
"Keccak": Hash.runKeccak,
"Shake": Hash.runShake,
"RIPEMD": Hash.runRIPEMD,
"HAS-160": Hash.runHAS,
"Whirlpool": Hash.runWhirlpool,
"Snefru": Hash.runSnefru,
"HMAC": Hash.runHMAC,
"Fletcher-8 Checksum": Checksum.runFletcher8,
"Fletcher-16 Checksum": Checksum.runFletcher16,
"Fletcher-32 Checksum": Checksum.runFletcher32,
"Fletcher-64 Checksum": Checksum.runFletcher64,
"Adler-32 Checksum": Checksum.runAdler32,
"CRC-16 Checksum": Checksum.runCRC16,
"CRC-32 Checksum": Checksum.runCRC32,
"TCP/IP Checksum": Checksum.runTCPIP,
"Analyse hash": Hash.runAnalyse,
"Generate all hashes": Hash.runAll,
"MD2": Hash.runMD2,
"MD4": Hash.runMD4,
"MD5": Hash.runMD5,
"MD6": Hash.runMD6,
"SHA0": Hash.runSHA0,
"SHA1": Hash.runSHA1,
"SHA2": Hash.runSHA2,
"SHA3": Hash.runSHA3,
"Keccak": Hash.runKeccak,
"Shake": Hash.runShake,
"RIPEMD": Hash.runRIPEMD,
"HAS-160": Hash.runHAS,
"Whirlpool": Hash.runWhirlpool,
"Snefru": Hash.runSnefru,
"CTPH": Hash.runCTPH,
"SSDEEP": Hash.runSSDEEP,
"Compare CTPH hashes": Hash.runCompareCTPH,
"Compare SSDEEP hashes": Hash.runCompareSSDEEP,
"HMAC": Hash.runHMAC,
"Fletcher-8 Checksum": Checksum.runFletcher8,
"Fletcher-16 Checksum": Checksum.runFletcher16,
"Fletcher-32 Checksum": Checksum.runFletcher32,
"Fletcher-64 Checksum": Checksum.runFletcher64,
"Adler-32 Checksum": Checksum.runAdler32,
"CRC-16 Checksum": Checksum.runCRC16,
"CRC-32 Checksum": Checksum.runCRC32,
"TCP/IP Checksum": Checksum.runTCPIP,
};

export default OpModules;
62 changes: 62 additions & 0 deletions src/core/operations/Hash.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ import CryptoApi from "babel-loader!crypto-api";
import MD6 from "node-md6";
import * as SHA3 from "js-sha3";
import Checksum from "./Checksum.js";
import ctph from "ctph.js";
import ssdeep from "ssdeep.js";


/**
Expand Down Expand Up @@ -336,6 +338,64 @@ const Hash = {
},


/**
* CTPH operation.
*
* @param {string} input
* @param {Object[]} args
* @returns {string}
*/
runCTPH: function (input, args) {
return ctph.digest(input);
},


/**
* SSDEEP operation.
*
* @param {string} input
* @param {Object[]} args
* @returns {string}
*/
runSSDEEP: function (input, args) {
return ssdeep.digest(input);
},


/**
* @constant
* @default
*/
DELIM_OPTIONS: ["Line feed", "CRLF", "Space", "Comma", "Semi-colon", "Colon"],

/**
* Compare CTPH hashes operation.
*
* @param {string} input
* @param {Object[]} args
* @returns {Number}
*/
runCompareCTPH: function (input, args) {
const samples = input.split(Utils.charRep[args[0]]);
if (samples.length !== 2) throw "Incorrect number of samples.";
return ctph.similarity(samples[0], samples[1]);
},


/**
* Compare SSDEEP hashes operation.
*
* @param {string} input
* @param {Object[]} args
* @returns {Number}
*/
runCompareSSDEEP: function (input, args) {
const samples = input.split(Utils.charRep[args[0]]);
if (samples.length !== 2) throw "Incorrect number of samples.";
return ssdeep.similarity(samples[0], samples[1]);
},


/**
* @constant
* @default
Expand Down Expand Up @@ -428,6 +488,8 @@ const Hash = {
"\nWhirlpool-0: " + Hash.runWhirlpool(arrayBuffer, ["Whirlpool-0"]) +
"\nWhirlpool-T: " + Hash.runWhirlpool(arrayBuffer, ["Whirlpool-T"]) +
"\nWhirlpool: " + Hash.runWhirlpool(arrayBuffer, ["Whirlpool"]) +
"\nSSDEEP: " + Hash.runSSDEEP(str) +
"\nCTPH: " + Hash.runCTPH(str) +
"\n\nChecksums:" +
"\nFletcher-8: " + Checksum.runFletcher8(byteArray, []) +
"\nFletcher-16: " + Checksum.runFletcher16(byteArray, []) +
Expand Down