Variable Length Gap Pattern Matching

.gitmodules

@@ -1,3 +1,6 @@
+[submodule "benchmark/gapped-matching/external/easyloggingpp"]
+	path = benchmark/gapped-matching/external/easyloggingpp
+	url = https://github.com/easylogging/easyloggingpp.git
 [submodule "external/googletest"]
 	path = external/googletest
 	url = https://chromium.googlesource.com/external/googletest

benchmark/gapped-matching/.gitignore

@@ -0,0 +1,7 @@
+results*
+collections/*
+build/*
+*.sdsl
+*.csv
+*.swp
+*~

benchmark/gapped-matching/CMakeLists.txt

@@ -0,0 +1,81 @@
+cmake_minimum_required(VERSION 2.8)
+cmake_policy(SET CMP0015 NEW)
+set(CMAKE_MODULE_PATH "${CMAKE_HOME_DIRECTORY}/../../CMakeModules")
+include(AppendCompilerFlags)
+include(ExternalProject)
+
+project(gapped-matching CXX C)
+set(CMAKE_BUILD_TYPE "Release")
+
+# parse make helper (HACK  :( )
+file(STRINGS ${CMAKE_HOME_DIRECTORY}/../../Make.helper makehelper_contents)
+list (GET makehelper_contents 0 INC_DIR_STR)
+list (GET makehelper_contents 1 LIB_DIR_STR)
+STRING(REGEX REPLACE "INC_DIR = (.*)" "\\1" SDSL_INC_DIR "${LIB_DIR_STR}")
+STRING(REGEX REPLACE "LIB_DIR = (.*)" "\\1" SDSL_LIB_DIR "${INC_DIR_STR}")
+MESSAGE( STATUS "SDSL_INC_DIR: " ${SDSL_INC_DIR} )
+MESSAGE( STATUS "SDSL_LIB_DIR: " ${SDSL_LIB_DIR} )
+
+
+# set include and lib dirs
+INCLUDE_DIRECTORIES(${SDSL_INC_DIR} 
+					${CMAKE_HOME_DIRECTORY}/external/easyloggingpp/src/
+					${CMAKE_HOME_DIRECTORY}/include
+					)
+LINK_DIRECTORIES(${SDSL_LIB_DIR})
+
+
+# C++11 compiler Check
+if(NOT CMAKE_CXX_COMPILER_VERSION) # work around for cmake versions smaller than 2.8.10
+    execute_process(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE CMAKE_CXX_COMPILER_VERSION)
+endif()
+if(CMAKE_CXX_COMPILER MATCHES ".*clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+      set(CMAKE_COMPILER_IS_CLANGXX 1)
+endif()
+if( (CMAKE_COMPILER_IS_GNUCXX AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 4.7) OR
+    (CMAKE_COMPILER_IS_CLANGXX AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 3.2))
+  message(FATAL_ERROR "Your C++ compiler does not support C++11. Please install g++ 4.7 (or greater) or clang 3.2 (or greater)")
+else()
+  message(STATUS "Compiler is recent enough to support C++11.")
+endif()
+if( MEASURE_ENERGY )
+    message(STATUS "Measure energy.")
+    append_cxx_compiler_flags("-DLIKWID_PERFMON -L$ENV{LIKWID_LIB} -I$ENV{LIKWID_INCLUDE} -llikwid" "GCC" CMAKE_CXX_FLAGS)
+endif()
+if( CMAKE_COMPILER_IS_GNUCXX )
+    append_cxx_compiler_flags("-std=c++11 -Wall -Wextra " "GCC" CMAKE_CXX_FLAGS)
+    append_cxx_compiler_flags("-msse4.2 -O3 -ffast-math -funroll-loops" "GCC" CMAKE_CXX_FLAGS_RELEASE)
+else()
+    append_cxx_compiler_flags("-std=c++11" "CLANG" CMAKE_CXX_FLAGS)
+    append_cxx_compiler_flags("-stdlib=libc++" "CLANG" CMAKE_CXX_FLAGS)
+    append_cxx_compiler_flags("-msse4.2 -O3  -ffast-math -funroll-loops -DNDEBUG" "CLANG" CMAKE_CXX_FLAGS_RELEASE)
+endif()
+
+FIND_PACKAGE(Boost 1.45 COMPONENTS regex)
+
+# # read the index configs
+file(GLOB index_config_files RELATIVE ${CMAKE_HOME_DIRECTORY}/config/ "${CMAKE_HOME_DIRECTORY}/config/*.config")
+foreach(f ${index_config_files})
+	file(STRINGS ${CMAKE_HOME_DIRECTORY}/config/${f} config_contents)
+	set(compile_defs "")
+	foreach(keyvalue ${config_contents})
+		string(REGEX REPLACE "^[ ]+" "" keyvalue ${keyvalue})
+		string(REGEX MATCH "^[^=]+" key ${keyvalue})
+		string(REPLACE "${key}=" "" value ${keyvalue})
+		set(${key} "${value}")
+		list(APPEND compile_defs ${key}=${value})
+	endforeach(keyvalue)
+    INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS})
+	ADD_EXECUTABLE(gm_index-${NAME}.x src/gm_index.cpp)
+	TARGET_LINK_LIBRARIES(gm_index-${NAME}.x sdsl divsufsort divsufsort64 pthread ${Boost_LIBRARIES})
+	set_property(TARGET gm_index-${NAME}.x PROPERTY COMPILE_DEFINITIONS IDXNAME="${NAME}" ${compile_defs})
+
+    INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS})
+	ADD_EXECUTABLE(gm_search-${NAME}.x src/gm_search.cpp)
+	TARGET_LINK_LIBRARIES(gm_search-${NAME}.x sdsl divsufsort divsufsort64 pthread ${Boost_LIBRARIES})
+	set_property(TARGET gm_search-${NAME}.x PROPERTY COMPILE_DEFINITIONS IDXNAME="${NAME}" ${compile_defs})
+endforeach(f)
+
+ADD_EXECUTABLE(create_collection.x src/create_collection.cpp)
+TARGET_LINK_LIBRARIES(create_collection.x sdsl divsufsort divsufsort64 pthread)
+

benchmark/gapped-matching/Makefile

@@ -0,0 +1,130 @@
+include ../Make.helper
+include ../Make.download
+
+TIMEOUT = 2050s
+MEASURE_ENERGY = 0
+MEASURE_CMD = $(if $(findstring 1,$(MEASURE_ENERGY)), likwid-perfctr -C S0:0 -g ENERGY -m,)
+
+CXX_FLAGS = $(MY_CXX_FLAGS) # in compile_options.config
+LIBS = -lsdsl 
+SRC_DIR = src
+TMP_DIR = ../tmp
+
+.SECONDARY:
+
+NUM_SAMPLES:=20
+NUM_SPS:=$(call config_ids,num_subpatterns.config)
+TC_IDS:=$(call config_ids,test_case.config)
+TC_COLLECTIONS:=$(foreach TC_ID,$(TC_IDS),collections/$(TC_ID)/text.TEXT)
+UNIQUE_DATA_FILES:=$(shell echo $(call config_column,test_case.config,2) | tr " " "\n" | uniq | tr "\n" " ")
+
+PATTERN_IDS:=$(call config_ids,patterns.config)
+UNIQUE_SP_LENGTHS:=$(shell echo $(call config_column,patterns.config,2) | tr " " "\n" | uniq | tr "\n" " ")
+
+ALGOS:=$(call config_ids,algorithms.config)
+
+GM_WORDS = $(foreach TC_ID,$(TC_IDS),$(foreach SP_LENGTH,$(UNIQUE_SP_LENGTHS),collections/$(TC_ID)/patterns/words.$(TC_ID).$(SP_LENGTH).words.txt))
+GM_PATTERNS = $(foreach TC_ID,$(TC_IDS),$(foreach PATTERN_ID,$(PATTERN_IDS),$(foreach NUM_SP,$(NUM_SPS),collections/$(TC_ID)/patterns/regex.$(TC_ID).$(PATTERN_ID).$(NUM_SP).regex.txt)))
+GM_EXECS = $(foreach ALGO,$(ALGOS),build/gm_index-$(ALGO).x) \
+           $(foreach ALGO,$(ALGOS),build/gm_search-$(ALGO).x) \
+           build/create_collection.x
+INDEX_SENTINELS = $(foreach TC_ID,$(TC_IDS),$(foreach ALGO,$(ALGOS),collections/$(TC_ID).$(ALGO).sentinel))
+
+RES_FILES = $(foreach TC_ID,$(TC_IDS),$(foreach PATTERN_ID,$(PATTERN_IDS),$(foreach NUM_SP,$(NUM_SPS),$(foreach ALGO,$(ALGOS),results/$(ALGO).$(PATTERN_ID).$(NUM_SP).$(TC_ID)))))
+RES_FILE=results/all.txt
+
+
+debug:
+	@echo $(NUM_SPS)
+
+inputs: $(GM_PATTERNS)
+indices: $(INDEX_SENTINELS)
+
+# Format: collections/[TC_ID].[ALGO].sentinel
+collections/%.sentinel: $(GM_EXECS) $(TC_COLLECTIONS)
+	$(eval TC_ID:=$(call dim,1,$*))
+	$(eval ALGO:=$(call dim,2,$*))
+	test -e $@ || build/gm_index-$(ALGO).x -c collections/$(TC_ID)
+	touch $@
+
+execs: $(GM_EXECS)
+
+collections/%/text.TEXT: build/create_collection.x $(UNIQUE_DATA_FILES)
+	$(eval TC_ID:=$(call dim,1,$*))
+	$(eval SOURCE_FILE:=$(call config_select,test_case.config,$(TC_ID),2))
+	$(eval NUM_BYTES:=$(call config_select,test_case.config,$(TC_ID),5))
+	$(eval PREFIX_BYTES:=$(call config_select,test_case.config,$(TC_ID),6))
+	$(eval TEMP_DATA_FILE:=$(TC_ID).$(NUM_BYTES).$(PREFIX_BYTES).tmp)
+	@test -e $@ || echo "Creating collection $(TC_ID) from file $(SOURCE_FILE)"
+	$(if $(findstring -,$(PREFIX_BYTES)), , @test -e $@ || head -c $(PREFIX_BYTES) $(SOURCE_FILE) > $(TEMP_DATA_FILE))
+	$(if $(findstring -,$(PREFIX_BYTES)), , $(eval SOURCE_FILE:=$(TEMP_DATA_FILE)))
+	@test -e $@ || build/create_collection.x -i $(SOURCE_FILE) -n $(NUM_BYTES) -c collections/$(TC_ID)
+	$(if $(findstring -,$(PREFIX_BYTES)), , @rm -f $(TEMP_DATA_FILE))
+
+timing: $(RES_FILES)
+	tail -n +1 $(RES_FILES) > $(RES_FILE)
+	@cd visualize; make	
+
+# Format: results/[ALGO].[PATTERN_ID].[NUM_SP].[TC_ID]
+results/%: $(GM_EXECS) $(TC_COLLECTIONS) $(GM_PATTERNS) $(INDEX_SENTINELS)
+	@mkdir -p results
+	$(eval ALGO:=$(call dim,1,$*))
+	$(eval PATTERN_ID:=$(call dim,2,$*))
+	$(eval NUM_SP:=$(call dim,3,$*))
+	$(eval TC_ID:=$(call dim,4,$*))
+	$(eval IS_TEXT:=$(call config_select,test_case.config,$(TC_ID),7))
+	@test -e $@ || ( echo "# COLL_ID = $(TC_ID)" > $@ &&\
+	echo "# PATT_SAMPLE = $(PATTERN_ID)" >> $@ &&\
+	echo "# ALGO = $(ALGO)" >> $@ &&\
+	echo "gm_search-$(ALGO).x -c collections/$(TC_ID) -p collections/$(TC_ID)/patterns/regex.$(TC_ID).$(PATTERN_ID).$(NUM_SP).regex.txt -t $(IS_TEXT)" &&\
+	(timeout $(TIMEOUT) $(MEASURE_CMD) build/gm_search-$(ALGO).x -c collections/$(TC_ID) -p collections/$(TC_ID)/patterns/regex.$(TC_ID).$(PATTERN_ID).$(NUM_SP).regex.txt -t $(IS_TEXT) >> $@ || echo TIMEOUT) &&\
+	tail -n 1 mem-mon-out.csv >> $@ &&\
+	echo "" >> $@ &&\
+	mv mem-mon-out.csv [email protected])
+	#Rscript memvis.R [email protected] ;
+
+../../examples/generate-pattern.x:
+	@cd ../../examples; make generate-pattern.x
+
+# Format: collections/[TC_ID]/patterns/words.[TC_ID].[SP_LENGTH].words.txt
+collections/%.words.txt: $(TC_COLLECTIONS) ../../examples/generate-pattern.x
+	$(eval TC_ID:=$(call dim,2,$*))
+	$(eval SP_LENGTH:=$(call dim,3,$*))
+	$(eval IS_TEXT:=$(call config_select,test_case.config,$(TC_ID),7))
+	@echo "Extracting words of length $(SP_LENGTH) from $(TC_ID)..."
+	@../../examples/generate-pattern.x collections/$(TC_ID)/text.TEXT 200 $(SP_LENGTH) $(IS_TEXT) > $@
+
+# Format: collections/[TC_ID]/patterns/regex.[TC_ID].[PATTERN_ID].[NUM_SP].regex.txt
+collections/%.regex.txt: $(GM_WORDS)
+	$(eval TC_ID:=$(call dim,2,$*))
+	$(eval PATTERN_ID:=$(call dim,3,$*))
+	$(eval NUM_SP:=$(call dim,4,$*))
+	$(eval SP_LENGTH:=$(call config_select,patterns.config,$(PATTERN_ID),2))
+	$(eval GAP:=$(call config_select,patterns.config,$(PATTERN_ID),3))
+	@echo "$(PATTERN_ID)"
+	@echo "Creating regex patterns with $(NUM_SP) words of length $(SP_LENGTH) from $(TC_ID) and a gap of $(GAP)..."
+	@cat collections/$(TC_ID)/patterns/words.$(TC_ID).$(SP_LENGTH).words.txt | sed -e 's/ /___/g' | xargs -n$(NUM_SP) | sed -e 's/ /.{$(GAP)}/g' | head -n $(NUM_SAMPLES) | sed -e 's/___/ /g' > $@
+
+build/%:
+	@mkdir -p build
+	cd build && cmake -DMEASURE_ENERGY=$(MEASURE_ENERGY) .. && make -j 2
+
+clean-build:
+	@echo "Remove executables"
+	rm -rf build/*
+
+clean:
+	rm -rf build/*
+	rm -f $(GM_PATTERNS)
+	rm -f $(GM_WORDS)
+	rm -f $(INDEX_SENTINELS)
+	rm -f $(RES_FILES) 
+
+clean-results: 
+	rm -f $(RES_FILES) 
+
+clean-inputs:
+	rm -f $(GM_PATTERNS)
+	rm -f $(GM_WORDS)
+
+#cleanall: clean clean-results clean-build

benchmark/gapped-matching/README.md

@@ -0,0 +1,17 @@
+INSTALL
+
+1. git submodule update --init
+2. mkdir build
+3. cd build
+4. cmake .. && make
+
+CREATE COLLECTIONS
+
+1. cd build
+2. ./create-collection.x -c ../collection/NEWNAME -i NEWNAME.raw
+
+EXECUTE BENCHMARK
+
+1. cd build
+2. ./gm_index-YOUR_IDX.x -c ../collections/your_collection
+3. ./gm_search-YOUR_IDX.x -c ../collections/your_collection -p ../collections/your_collection/patterns/your_pattern.txt

benchmark/gapped-matching/algorithms.config

@@ -0,0 +1,21 @@
+# Configuration for test files
+# (1) Identifier of configuration to use
+# (2) LaTeX name
+# (3) point type
+# (4) line type
+# (5) color
+
+#REGEXP_ECMA;regexp-ecma;1;solid;black
+#REGEXP_ECMAOPT;regexp-ecma-opt;1;solid;blue
+QGRAM_REGEXP_ECMA;qgram-regexp-ecma;1;solid;blue
+REGEXP_ECMA_BOOST;regexp-ecma-boost;1;solid;red
+#WCSEARCH_DFS;wildcard-search-dfs;1;solid;green
+WCSEARCH_DFS3;wildcard-search-dfs3;1;solid;green
+SASEARCH;baseline-sa-search;2;solid;yellow
+#STREE;bs-tree;3;solid;blue
+
+
+#WCSEARCH_FULL_DFS;wildcard-search-dfs-full;2;solid;green
+#WCSEARCH_BFS;wildcard-search-bfs;3;solid;green
+#DBSEARCH;double-binary-search;2;solid;yellow
+

benchmark/gapped-matching/config/QGRAM_REGEXP_ECMA.config

@@ -0,0 +1,3 @@
+NAME=QGRAM_REGEXP_ECMA
+INDEX_TYPE=index_qgram_regexp<3>
+REGEXP_TYPE=std::regex::ECMAScript

benchmark/gapped-matching/config/REGEXP_ECMA_BOOST.config

@@ -0,0 +1,3 @@
+NAME=REGEXP_ECMA_BOOST
+INDEX_TYPE=index_regexp_boost
+REGEXP_TYPE=std::regex::ECMAScript

benchmark/gapped-matching/config/SASEARCH.config

@@ -0,0 +1,3 @@
+NAME=SASEARCH
+INDEX_TYPE=index_sasearch
+REGEXP_TYPE=std::regex::ECMAScript

benchmark/gapped-matching/config/WCSEARCH_DFS3.config

@@ -0,0 +1,3 @@
+NAME=WCSEARCH_DFS3
+INDEX_TYPE=index_wcsearch3
+REGEXP_TYPE=std::regex::ECMAScript