Merge pull request #1 from Toutou98/master

Replace MD5 hashing with std::hash and add Dockerfile for fdb-kv deployment

Dockerfile

@@ -0,0 +1,104 @@
+# Stage 1: God created the heavens and the earth
+FROM ubuntu:22.04 AS builder
+
+ENV eckit_ROOT /root/local
+ENV CMAKE_PREFIX_PATH /usr/local
+ENV metkit_ROOT /root/local
+
+COPY hammer-fdb-kv.sh /hammer-fdb-kv.sh
+
+RUN apt-get update && apt-get upgrade -y
+
+RUN DEBIAN_FRONTEND=noninteractive TZ=Europe/Athens apt-get install -y wget git cmake g++ gfortran python3 python3-pip libnuma-dev libboost-all-dev
+RUN pip3 install eccodes
+
+# fdb dependecies
+RUN wget https://gitlab.dkrz.de/k202009/libaec/-/archive/v1.1.1/libaec-v1.1.1.tar.gz && \
+    tar -xzf libaec-v1.1.1.tar.gz && \
+    cd libaec-v1.1.1 && \
+    mkdir build && \
+    cd build && \
+    cmake .. # -CMAKE_INSTALL_PREFIX=~/local
+
+RUN cd libaec-v1.1.1/build && \
+    make install && \
+    cd /
+
+RUN wget https://confluence.ecmwf.int/download/attachments/45757960/eccodes-2.35.0-Source.tar.gz?api=v2 && \
+    tar -xzf eccodes-2.35.0-Source.tar.gz?api=v2 && \
+    cd eccodes-2.35.0-Source && \
+    mkdir build && \
+    cd build && \
+    cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local && \
+    make -j10 && \
+    ctest && \
+    make install && \
+    cd / 
+
+RUN git clone https://github.com/ecmwf/ecbuild && \
+    cd ecbuild && \
+    mkdir bootstrap && \
+    cd bootstrap && \
+    ../bin/ecbuild --prefix=/usr/local .. && \
+    ctest && \
+    make install && \
+    cd /
+
+RUN git clone https://github.com/ecmwf/eckit.git && \
+    cd eckit && \
+    mkdir build && \
+    cd build && \
+    ecbuild --prefix=$HOME/local -- ../$pwd && \
+    make -j10 && \
+    make install && \
+    cd /
+
+RUN git clone https://github.com/ecmwf/metkit.git && \
+    cd metkit && \
+    mkdir build && \
+    cd build && \
+    ecbuild --prefix=$HOME/local -- -DECKIT_PATH=/usr/local ../$pwd && \
+    make -j10 && \
+    make install && \
+    cd /
+# Parallax
+RUN git clone https://github.com/CARV-ICS-FORTH/parallax.git && \
+    cd parallax && \
+    git checkout extra_hdf5_functionality && \
+    mkdir build && \
+    cd build && \
+    cmake .. -DCMAKE_INSTALL_PREFIX=/root/local -DBUILD_SHARED_LIBS=ON && \
+    make -j10 && \
+    make install 
+
+# fdb-kv
+RUN git clone https://github.com/Toutou98/fdb-kv.git && \
+    cd fdb-kv && \
+    mkdir build && \
+    cd build && \
+    ecbuild --prefix=$HOME/local -- -DCMAKE_INSTALL_PREFIX=/root/local -DCMAKE_FIND_LIBRARY_PREFIXES="lib" -DCMAKE_FIND_LIBRARY_SUFFIXES=".so;.a" .. && \
+    make -j10
+
+RUN cd / && \
+    git clone https://github.com/Toutou98/grib.git
+
+RUN chmod +x /hammer-fdb-kv.sh
+# the great purge
+RUN apt purge -y wget git cmake g++ gfortran python3 python3-pip libnuma-dev libboost-all-dev && \
+    apt autoremove -y
+
+# Stage 2: The second coming
+FROM ubuntu:22.04
+
+ENV eckit_ROOT /root/local
+ENV CMAKE_PREFIX_PATH /usr/local
+ENV metkit_ROOT /root/local
+
+COPY --from=builder /usr /usr
+COPY --from=builder /root /root
+COPY --from=builder /fdb-kv /fdb-kv
+COPY --from=builder /hammer-fdb-kv.sh /hammer-fdb-kv.sh
+COPY --from=builder /grib /grib
+
+RUN chmod +x /hammer-fdb-kv.sh
+

hammer-fdb-kv.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Define variables
+EXPVER="xxxx"
+CLASS="rd"
+NLEVELS=201
+NSTEPS=1
+NPARAMS=5
+NENSEMBLES=1
+INPUT_FILE="/grib/o3_new.grib"
+
+# Command to run fdb-hammer
+./fdb-kv/build/bin/fdb-hammer \
+    --statistics \
+    --expver="$EXPVER" \
+    --class="$CLASS" \
+    --nlevels="$NLEVELS" \
+    --nsteps="$NSTEPS" \
+    --nparams="$NPARAMS" \
+    --nensembles="$NENSEMBLES" \
+    "$INPUT_FILE"

src/fdb5/toc/LSMIndex.cc

@@ -8,7 +8,6 @@
  * does it submit to any jurisdiction.
  */
 #include <assert.h>
-#include <openssl/md5.h>
 #include <parallax.h>
 #include <signal.h>
 #include <fstream>
@@ -142,7 +141,10 @@ class LSMIndex : public BTreeIndex {
         // create the dummy index file so fdb-hammer does not nag
         std::ofstream outfile(path.asString());
         // ... write to the file or do other things with it.
-        std::string dbName = md5(path.asString());
+        size_t hashValue = std::hash<std::string>{}(path.asString());
+        std::ostringstream oss;
+        oss << std::hex << std::setw(2) << std::setfill('0') << hashValue;
+        std::string dbName = oss.str();
         std::cout << "DB name is " << path.asString() << "hash  name: " << dbName << std::endl;
         const char* volume_name = ParallaxStore::getInstance().getVolumeName();
 
@@ -273,21 +275,6 @@ class LSMIndex : public BTreeIndex {
         LSM_DEBUG("Nothing to preload here we are PARALLAX");
     }
 
-private:
-    std::string md5(const std::string& data) {
-        unsigned char digest[MD5_DIGEST_LENGTH];
-
-        // Calculate the MD5 hash of the input data
-        MD5(reinterpret_cast<const unsigned char*>(data.c_str()), data.size(), digest);
-
-        // Convert the binary hash to a hexadecimal string
-        std::ostringstream oss;
-        for (int i = 0; i < MD5_DIGEST_LENGTH; i++) {
-            oss << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(digest[i]);
-        }
-
-        return oss.str();
-    }
 };