diff --git a/.travis.yml b/.travis.yml index 900dae0..e876d86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,12 @@ env: LUA_PATH="./?/init.lua;./?.lua" before_install: - sudo apt-get update -qq - sudo apt-get install -y screen lua5.2 liblua5.2-dev libboost-filesystem-dev libboost-filesystem-dev libboost-thread-dev libssl-dev - - wget https://github.com/mongodb/mongo-cxx-driver/archive/legacy-0.9.0.tar.gz - - tar zxvf legacy-0.9.0.tar.gz - - cd mongo-cxx-driver-legacy-0.9.0 && sudo scons --prefix=/usr install-mongoclient && cd - - - wget https://github.com/moai/luamongo/archive/v0.4.3.tar.gz - - tar zxvf v0.4.3.tar.gz - - cd luamongo-0.4.3 && make && sudo mkdir -p /usr/lib/lua/5.2 && sudo cp mongo.so /usr/lib/lua/5.2 && cd - + - wget https://github.com/mongodb/mongo-cxx-driver/archive/legacy-1.1.0.tar.gz + - tar zxvf legacy-1.1.0.tar.gz + - cd mongo-cxx-driver-legacy-1.1.0 && sudo scons --sharedclient=yes --prefix=/usr install && cd - + - wget https://github.com/moai/luamongo/archive/v0.5.0.tar.gz + - tar zxvf v0.5.0.tar.gz + - cd luamongo-0.5.0 && make && sudo mkdir -p /usr/lib/lua/5.2 && sudo cp mongo.so /usr/lib/lua/5.2 && cd - - ssh-keygen -b 2048 -f /home/travis/.ssh/id_rsa -t rsa -q -N "" - ssh-keyscan -t rsa localhost > ~/.ssh/known_hosts - ssh-keyscan -t rsa $(hostname) >> ~/.ssh/known_hosts diff --git a/mapreduce/cnn.lua b/mapreduce/cnn.lua index cb2b1de..2ddec9c 100644 --- a/mapreduce/cnn.lua +++ b/mapreduce/cnn.lua @@ -44,8 +44,15 @@ function cnn:gridfs() return gridfs end +-- this weak table allow to index gridfs objects using weak grid_file_builder +-- keys +local gridfs_instances = setmetatable({}, { __mode="k" }) + function cnn:grid_file_builder() - return mongo.GridFileBuilder.New(self:connect(), self.gridfs_dbname) + local gridfs = self:gridfs() + local grid_file_builder = mongo.GridFileBuilder.New(gridfs) + gridfs_instances[grid_file_builder] = gridfs + return grid_file_builder end function cnn:get_dbname() diff --git a/mapreduce/examples/APRIL-ANN/common.lua b/mapreduce/examples/APRIL-ANN/common.lua index fa3cc67..d583ba9 100644 --- a/mapreduce/examples/APRIL-ANN/common.lua +++ b/mapreduce/examples/APRIL-ANN/common.lua @@ -23,13 +23,13 @@ util.omp_set_num_threads(1) local serialize_to_gridfs = function(gridfs, filename, obj) gridfs:remove_file(filename) - local builder = mongo.GridFileBuilder.New(db, dbname) + local builder = mongo.GridFileBuilder.New(gridfs) builder:append(util.serialize(obj)) builder:build(filename) end local deserialize_from_gridfs = function(gridfs, filename) - local file = assert( gridfs:find_file(filename) ) + local file = assert( gridfs:find_file_by_name(filename) ) local str_tbl = {} for i=1,file:num_chunks() do local chunk = file:chunk(i-1) diff --git a/mapreduce/server.lua b/mapreduce/server.lua index 41b0cf6..df4274b 100644 --- a/mapreduce/server.lua +++ b/mapreduce/server.lua @@ -333,7 +333,9 @@ local function server_drop_collections(self) local dbname = self.cnn:get_dbname() -- drop all the collections for _,name in ipairs(db:get_collections(dbname)) do - db:drop_collection(name) + if name ~= "system_indexes" then + db:drop_collection(dbname .. "." .. name) + end end local gridfs = self.cnn:gridfs() local list = gridfs:list() diff --git a/mapreduce/utils.lua b/mapreduce/utils.lua index 386da63..3b59db2 100644 --- a/mapreduce/utils.lua +++ b/mapreduce/utils.lua @@ -131,7 +131,7 @@ end -- and last chunks where the line is contained, and the first and last position -- inside the corresponding chunks local function gridfs_lines_iterator(gridfs, filename) - local gridfile = gridfs:find_file(filename) + local gridfile = assert( gridfs:find_file_by_name(filename) ) local size = #gridfile local current_chunk = 0 local current_pos = 1 diff --git a/test.sh b/test.sh index 80b9364..c7863b9 100755 --- a/test.sh +++ b/test.sh @@ -6,7 +6,7 @@ if [[ $? -ne 0 ]]; then exit 1 fi for storage in gridfs shared sshfs; do - ## COMBINER + ASSOCIATIVE COMMUTATIVE IDEMPOTENT REDUCER + echo COMBINER + ASSOCIATIVE COMMUTATIVE IDEMPOTENT REDUCER screen -d -m ./execute_example_worker.sh diff <(./execute_example_server.sh $storage | awk '{ print $1,$2 }' | sort) \ <(cat mapreduce/server.lua \ @@ -17,7 +17,7 @@ for storage in gridfs shared sshfs; do echo "ERROR" exit 1 fi - ## NO COMBINER + ASSOCIATIVE COMMUTATIVE IDEMPOTENT REDUCER + echo NO COMBINER + ASSOCIATIVE COMMUTATIVE IDEMPOTENT REDUCER screen -d -m ./execute_example_worker.sh diff <(lua execute_server.lua localhost wordcount \ mapreduce.examples.WordCount.taskfn \ @@ -34,7 +34,7 @@ for storage in gridfs shared sshfs; do echo "ERROR" exit 1 fi - ## NO COMBINER + GENERAL REDUCER + echo NO COMBINER + GENERAL REDUCER screen -d -m ./execute_example_worker.sh diff <(lua execute_server.lua localhost wordcount \ mapreduce.examples.WordCount.taskfn \ @@ -51,7 +51,7 @@ for storage in gridfs shared sshfs; do echo "ERROR" exit 1 fi - ## INIT SCRIPT + echo INIT SCRIPT screen -d -m ./execute_example_worker.sh diff <(lua execute_server.lua localhost wordcount \ mapreduce.examples.WordCount \