From 2702ef563810dd2d9ccc724781b8e5e1810eebc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E4=BA=91=E8=BE=89?= Date: Fri, 23 Aug 2019 11:12:25 +0800 Subject: [PATCH 01/79] Initial empty repository From daac6ea89314f77cf164d8bc6b620b73996fdaf8 Mon Sep 17 00:00:00 2001 From: majie1 Date: Fri, 20 Sep 2019 17:31:04 +0800 Subject: [PATCH 02/79] basic structure of project Change-Id: I321cafe23cfb0f39d7e4bce893db51ea404b6981 --- .gitignore | 8 +++ 3rdparty/CMakeLists.txt | 11 ++++ CMakeLists.txt | 39 +++++++++++ Dockerfile.develop | 34 ++++++++++ Dockerfile.package | 30 +++++++++ Dockerfile.unittest | 35 ++++++++++ README | 93 ++++++++++++++++++++++++++ cmake/modules/BuildLibconfig.cmake | 37 +++++++++++ debian/changelog | 5 ++ debian/compat | 1 + debian/control | 41 ++++++++++++ debian/nebd.dirs | 3 + debian/nebd.install | 14 ++++ debian/rules | 36 ++++++++++ debian/source/format | 1 + do_cmake | 16 +++++ do_package | 10 +++ docker_unittest.sh | 28 ++++++++ etc/nebd/nebd-client.conf | 16 +++++ etc/nebd/nebd-server.conf | 0 get_3rdparty | 13 ++++ install_deps | 8 +++ src/CMakeLists.txt | 41 ++++++++++++ src/common/CMakeLists.txt | 17 +++++ src/logrotate.conf | 12 ++++ src/part1/CMakeLists.txt | 37 +++++++++++ src/part2/CMakeLists.txt | 44 +++++++++++++ tests/CMakeLists.txt | 29 ++++++++ tests/common/CMakeLists.txt | 16 +++++ tests/part1/CMakeLists.txt | 53 +++++++++++++++ tests/part2/CMakeLists.txt | 102 +++++++++++++++++++++++++++++ 31 files changed, 830 insertions(+) create mode 100644 .gitignore create mode 100644 3rdparty/CMakeLists.txt create mode 100644 CMakeLists.txt create mode 100644 Dockerfile.develop create mode 100644 Dockerfile.package create mode 100644 Dockerfile.unittest create mode 100644 README create mode 100644 cmake/modules/BuildLibconfig.cmake create mode 100644 debian/changelog create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/nebd.dirs create mode 100644 debian/nebd.install create mode 100755 debian/rules create mode 100644 debian/source/format create mode 100755 do_cmake create mode 100755 do_package create mode 100755 docker_unittest.sh create mode 100644 etc/nebd/nebd-client.conf create mode 100644 etc/nebd/nebd-server.conf create mode 100755 get_3rdparty create mode 100755 install_deps create mode 100644 src/CMakeLists.txt create mode 100644 src/common/CMakeLists.txt create mode 100644 src/logrotate.conf create mode 100644 src/part1/CMakeLists.txt create mode 100644 src/part2/CMakeLists.txt create mode 100644 tests/CMakeLists.txt create mode 100644 tests/common/CMakeLists.txt create mode 100644 tests/part1/CMakeLists.txt create mode 100644 tests/part2/CMakeLists.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..dc6d70fc15 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.gitreview +3rdparty/boost +3rdparty/brpc +3rdparty/googletest +3rdparty/jsoncpp +3rdparty/libconfig +3rdparty/cmock +build diff --git a/3rdparty/CMakeLists.txt b/3rdparty/CMakeLists.txt new file mode 100644 index 0000000000..80dfcbb0b8 --- /dev/null +++ b/3rdparty/CMakeLists.txt @@ -0,0 +1,11 @@ +# we need shared library +option(BUILD_SHARED_LIBS "enable shared library" ON) + +include(BuildLibconfig) +build_libconfig() +add_subdirectory(brpc) +add_subdirectory(googletest) +add_subdirectory(jsoncpp) + +install(TARGETS brpc-shared DESTINATION lib) +install(TARGETS jsoncpp_lib DESTINATION lib) diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000000..5365e07caa --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,39 @@ +cmake_minimum_required(VERSION 3.15.0) + +project(nebd) +set(VERSION 1.0.0) +# LINUX ONLY + +include(GNUInstallDirs) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules/") + +# put all the libs and binaries in one place +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +set(CMAKE_REQUIRED_LIBRARIES pthread) +find_package(Threads REQUIRED) + +option(NEBD_DEBUG_ENABLE "enable/disable debug" OFF) +option(NEBD_COVERAGE_ENABLE "enable/disable coverage" OFF) + +# we use gcc-4.9-backport, so we need to set rpath for all projects +set(CMAKE_SKIP_BUILD_RPATH FALSE) +set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) +set(CMAKE_INSTALL_RPATH "/usr/lib/gcc-4.9-backport/lib/;\ + ${CMAKE_INSTALL_PREFIX}/lib;${CMAKE_BINARY_DIR}/lib") +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + +# google test headers +set(GMOCK_INCLUDE_DIRS + "${CMAKE_SOURCE_DIR}/3rdparty/googletest/googlemock/include/gmock") +set(GTEST_INCLUDE_DIRS + "${CMAKE_SOURCE_DIR}/3rdparty/googletest/googletest/include/gtest") +include_directories("${CMAKE_SOURCE_DIR}/3rdparty/googletest/googlemock/include") +include_directories("${CMAKE_SOURCE_DIR}/3rdparty/googletest/googletest/include") + +add_subdirectory(3rdparty) +add_subdirectory(src) +enable_testing() +add_subdirectory(tests) diff --git a/Dockerfile.develop b/Dockerfile.develop new file mode 100644 index 0000000000..a1bce5cdc7 --- /dev/null +++ b/Dockerfile.develop @@ -0,0 +1,34 @@ +FROM debian:wheezy +MAINTAINER majie1 + +WORKDIR /root/ +RUN echo "deb [trusted=yes] http://pkg.cloud.netease.com/debian wheezy main" >/etc/apt/sources.list +RUN echo "deb [trusted=yes] http://pkg.cloud.netease.com/debian wheezy-updates main" >>/etc/apt/sources.list +RUN echo "deb [trusted=yes] http://pkg.cloud.netease.com/debian wheezy-backports main" >>/etc/apt/sources.list +RUN echo "deb [trusted=yes] http://pkg.cloud.netease.com/debian-security wheezy-updates main" >>/etc/apt/sources.list +RUN echo "deb [trusted=yes] http://pkg.cloud.netease.com/openstack ceph-wheezy main" >>/etc/apt/sources.list +RUN apt-get update + +COPY ./ /root/nebd +WORKDIR /root/nebd +RUN ./install_deps + +WORKDIR /root/ +RUN rm -rf /root/nebd + +ENV CC /usr/lib/gcc-4.9-backport/bin/gcc +ENV CXX /usr/lib/gcc-4.9-backport/bin/g++ +ENV LD_LIBRARY_PATH /usr/lib/gcc-4.9-backport/lib/:/usr/lib:/usr/lib/x86_64-linux-gnu +ENV PATH /root/cmake-3.15.3/bin:$PATH +ADD http://10.187.0.105/nebd-build/cmake_3_15_3.tar.gz ./ +RUN tar -xf cmake_3_15_3.tar.gz +WORKDIR /root/cmake-3.15.3/ +RUN ./bootstrap --parallel=$(nproc) && make -j$(nproc) + +WORKDIR /root +RUN git clone https://github.com/linux-test-project/lcov.git +WORKDIR /root/lcov +RUN make install + +# you code dir -> /root/nebd in docker +WORKDIR /root/ diff --git a/Dockerfile.package b/Dockerfile.package new file mode 100644 index 0000000000..fbd8dc6997 --- /dev/null +++ b/Dockerfile.package @@ -0,0 +1,30 @@ +FROM debian:wheezy +MAINTAINER majie1 + +WORKDIR /root/ +RUN echo "deb [trusted=yes] http://pkg.cloud.netease.com/debian wheezy main" >/etc/apt/sources.list +RUN echo "deb [trusted=yes] http://pkg.cloud.netease.com/debian wheezy-updates main" >>/etc/apt/sources.list +RUN echo "deb [trusted=yes] http://pkg.cloud.netease.com/debian wheezy-backports main" >>/etc/apt/sources.list +RUN echo "deb [trusted=yes] http://pkg.cloud.netease.com/debian-security wheezy-updates main" >>/etc/apt/sources.list +RUN echo "deb [trusted=yes] http://pkg.cloud.netease.com/openstack ceph-wheezy main" >>/etc/apt/sources.list +RUN apt-get update + +COPY ./ /root/nebd +WORKDIR /root/nebd +RUN ./install_deps + +WORKDIR /root/ +RUN rm -rf /root/nebd + +ENV CC /usr/lib/gcc-4.9-backport/bin/gcc +ENV CXX /usr/lib/gcc-4.9-backport/bin/g++ +ENV LD_LIBRARY_PATH /usr/lib/gcc-4.9-backport/lib/:/usr/lib:/usr/lib/x86_64-linux-gnu +ENV PATH /root/cmake-3.15.3/bin:$PATH +ADD http://10.187.0.105/nebd-build/cmake_3_15_3.tar.gz ./ +RUN tar -xf cmake_3_15_3.tar.gz +WORKDIR /root/cmake-3.15.3/ +RUN ./bootstrap --parallel=$(nproc) && make -j$(nproc) + +# dir on jenkins machine -> /root/WORKDIR in docker, nebd path is /root/WORKDIR/nebd +WORKDIR /root/WORKDIR/nebd +CMD ["/bin/bash", "do_package"] diff --git a/Dockerfile.unittest b/Dockerfile.unittest new file mode 100644 index 0000000000..892b9fb732 --- /dev/null +++ b/Dockerfile.unittest @@ -0,0 +1,35 @@ +FROM debian:wheezy +MAINTAINER majie1 + +WORKDIR /root/ +RUN echo "deb [trusted=yes] http://pkg.cloud.netease.com/debian wheezy main" >/etc/apt/sources.list +RUN echo "deb [trusted=yes] http://pkg.cloud.netease.com/debian wheezy-updates main" >>/etc/apt/sources.list +RUN echo "deb [trusted=yes] http://pkg.cloud.netease.com/debian wheezy-backports main" >>/etc/apt/sources.list +RUN echo "deb [trusted=yes] http://pkg.cloud.netease.com/debian-security wheezy-updates main" >>/etc/apt/sources.list +RUN echo "deb [trusted=yes] http://pkg.cloud.netease.com/openstack ceph-wheezy main" >>/etc/apt/sources.list +RUN apt-get update + +COPY ./ /root/nebd +WORKDIR /root/nebd +RUN ./install_deps + +WORKDIR /root/ +RUN rm -rf /root/nebd + +ENV CC /usr/lib/gcc-4.9-backport/bin/gcc +ENV CXX /usr/lib/gcc-4.9-backport/bin/g++ +ENV LD_LIBRARY_PATH /usr/lib/gcc-4.9-backport/lib/:/usr/lib:/usr/lib/x86_64-linux-gnu +ENV PATH /root/cmake-3.15.3/bin:$PATH +ADD http://10.187.0.105/nebd-build/cmake_3_15_3.tar.gz ./ +RUN tar -xf cmake_3_15_3.tar.gz +WORKDIR /root/cmake-3.15.3/ +RUN ./bootstrap --parallel=$(nproc) && make -j$(nproc) + +WORKDIR /root +RUN git clone https://github.com/linux-test-project/lcov.git +WORKDIR /root/lcov +RUN make install + +# dir on jenkins machine -> /root/nebd in docker +WORKDIR /root/nebd +CMD ["/bin/bash", "docker_unittest.sh"] diff --git a/README b/README new file mode 100644 index 0000000000..73f1c2dbb2 --- /dev/null +++ b/README @@ -0,0 +1,93 @@ +# Guide + +## Need to know first +1. use cmake==3.15.3, need build first +2. 3rd party lib list + - brpc 0.9.6 + - boost 1.66.0 (use head only library in part2) + - googletest 1.8.1 + - jsoncpp 1.8.4 + - libconfig 1.7.2 + +## Develop +### manually +1. build cmake + - download source code + - `./bootstrap --parallel=24 && make -j24` +2. add env to .basrc or something + ``` bash + echo " + export CC=/usr/lib/gcc-4.9-backport/bin/gcc + export CXX=/usr/lib/gcc-4.9-backport/bin/g++ + export LD_LIBRARY_PATH=/usr/lib/gcc-4.9-backport/lib/:/usr/lib:/usr/lib/x86_64-linux-gnu:\$LD_LIBRARY_PATH + export PATH=/home/username/cmake-3.15.3/bin:\$PATH + " >> ~/.bashrc + source ~/.bashrc + ``` +3. install build deps +4. `bash do_cmake.sh` +5. go to build/, which contians bin, lib. + +### quick start using docker +1. build docker image +``` +cd nebd +sudo docker build -f Dockerfile.develop --tag something:latest . +``` +2. run a container, map your code path to container +``` +sudo docker run -it -v /your/path:/root/nebd something:latest bash +``` +3. in the docker +``` +# cd nebd +# ./do_cmake +# go to build dir +``` + +## UnitTest +### manually +1. we use ctest, run unittest after build +``` +cd build +ctest (-j4 for multijobs, -V for detail) +``` + +### quick start using docker +1. build docker image +``` +cd nebd +sudo docker build -f Dockerfile.unittest --tag somehing:latest . +``` +2. run unittest +``` +sudo docker run --rm -t -v /your/path:/root/nebd something:latest +``` + +## Package +### manually +1. build cmake +2. add env +3. get 3rd party source code, `./get_3rdparty.sh` +4. update debian/changelog +5. `dpkg-buildpackage -us -uc -j24` + +### quick start using docker +1. build docoker image +``` +cd nebd; +sudo docker build -f Dockerfile.unittest --tag something:latest . +``` +2. build nebd package +``` +mdkir WORKDIR +mv /your/codepath/ WORKDIR/ +sudo docker run --rm -t -v WORKDIR:/root/WORKDIR something:latest +``` +3. your deb package will under WORKDIR, nebd and nebd-dbg + +## How to build qemu +todo + +## Hints +todo diff --git a/cmake/modules/BuildLibconfig.cmake b/cmake/modules/BuildLibconfig.cmake new file mode 100644 index 0000000000..1d17dbe2d1 --- /dev/null +++ b/cmake/modules/BuildLibconfig.cmake @@ -0,0 +1,37 @@ +# This module builds libconfig library. Following hints are respected +# +# LIBCONFIG_J: integer (defanult 1) + +function(do_build_libconfig) + set(configure_command ./configure --prefix=/) + + set(build_command make) + if(LIBCONFIG_J) + message(STATUS "BUILDING libconfig Libraries at j ${LIBCONFIG_J}") + list(APPEND BUILD_COMMAND -j${LIBCONFIG_J}) + endif() + + set(install_command make DESTDIR=${CMAKE_BINARY_DIR} install) + + set(source_dir SOURCE_DIR "${PROJECT_SOURCE_DIR}/3rdparty/libconfig") + include(ExternalProject) + ExternalProject_Add(Libconfig + ${source_dir} + CONFIGURE_COMMAND ${configure_command} + BUILD_COMMAND ${build_command} + BUILD_IN_SOURCE 1 + INSTALL_COMMAND ${install_command} + PREFIX ${CMAKE_BINARY_DIR}) +endfunction() + +macro(build_libconfig) + do_build_libconfig() + ExternalProject_Get_Property(Libconfig install_dir) + add_library(config + SHARED + IMPORTED + GLOBAL) + set(LIBCONFIG_SHARED ${install_dir}/lib/libconfig.so.11.0.2) + set_property(TARGET config PROPERTY IMPORTED_LOCATION ${LIBCONFIG_SHARED}) + install(FILES ${LIBCONFIG_SHARED} DESTINATION lib RENAME libconfig.so.11) +endmacro() diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000000..075c68fd8f --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +nebd (1.0.0-1) stable; urgency=medium + + * New upstream release + + -- Ceph Team Fri, 20 Sep 2019 10:30:00 +0800 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000000..ec635144f6 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +9 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000000..a84ed081e8 --- /dev/null +++ b/debian/control @@ -0,0 +1,41 @@ +Source: nebd +Section: admin +Priority: optional +Maintainer: ceph-team +Uploaders: majie1 +Build-Depends: + dpkg-dev, + debhelper (>= 9), + dh-exec, + git, + gcc-4.9-backport, + librbd-dev (> 0.94.0), + librados-dev (> 0.94.0), + libgflags-dev, + libleveldb-dev, + libprotobuf-dev, + libprotoc-dev, + libsnappy-dev, + libssl-dev, + protobuf-compiler, + automake, +Standards-Version: 3.9.3 + +Package: nebd +Architecture: linux-any +Depends: gcc-4.9-backport, + librbd1 (> 0.94.0), + librados2 (> 0.94.0), + libgflags2, + libleveldb1, + libprotoc7, + libprotobuf7, + libsnappy1, + logrotate, +Description: nebd, hot update + +Package: nebd-dbg +Architecture: linux-any +Section: debug +Depends: nebd (= ${binary:Version}), +Description: debug symbols for nebd diff --git a/debian/nebd.dirs b/debian/nebd.dirs new file mode 100644 index 0000000000..0101b7ae1a --- /dev/null +++ b/debian/nebd.dirs @@ -0,0 +1,3 @@ +etc/nebd +var/log/nebd +run/nebd-server diff --git a/debian/nebd.install b/debian/nebd.install new file mode 100644 index 0000000000..c965e026f8 --- /dev/null +++ b/debian/nebd.install @@ -0,0 +1,14 @@ +#! /usr/bin/dh-exec --with=install + +etc/nebd/nebd-client.conf +etc/nebd/nebd-server.conf +usr/bin/nebd-server +usr/lib/libbrpc.so +usr/lib/libconfig.so.11 +usr/lib/libnebd.so +usr/lib/libnebd_common.so +usr/lib/libclient_proto.so +usr/lib/libjsoncpp.so +usr/lib/libjsoncpp.so.19 +usr/lib/libjsoncpp.so.1.8.4 +usr/lib/libgtest.so diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000000..397f38196e --- /dev/null +++ b/debian/rules @@ -0,0 +1,36 @@ +#!/usr/bin/make -f +# -*- makefile -*- +export DH_VERBOSE=1 + +extraopts += -DCMAKE_INSTALL_LIBDIR=/usr/lib +extraopts += -DCMAKE_INSTALL_LIBEXECDIR=/usr/lib +extraopts += -DCMAKE_INSTALL_SYSCONFDIR=/etc + +%: + dh $@ --buildsystem=cmake --parallel + +override_dh_auto_configure: + dh_auto_configure --buildsystem=cmake -- $(extraopts) + +override_dh_auto_clean: + dh_auto_clean --buildsystem=cmake + rm -f debian/nebd.logrotate + +override_dh_auto_install: + dh_auto_install --buildsystem=cmake --destdir=$(CURDIR)/debian/tmp + +override_dh_installlogrotate: + cp src/logrotate.conf debian/nebd.logrotate + dh_installlogrotate -pnebd + +override_dh_shlibdeps: + LD_LIBRARY_PATH=$(LD_LIBRARY_PATH):$(CURDIR)/debian/tmp + dh_shlibdeps --dpkg-shlibdeps-params=--ignore-missing-info + +override_dh_strip: + dh_strip --dbg-package=nebd-dbg + +# do not run tests +override_dh_auto_test: + +.PHONY: override_dh_autoreconf override_dh_auto_configure override_dh_auto_build override_dh_auto_clean override_dh_auto_install override_dh_installdocs override_dh_installlogrotate override_dh_installinit override_dh_systemd_start override_dh_strip override_dh_auto_test diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000000..d3827e75a5 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +1.0 diff --git a/do_cmake b/do_cmake new file mode 100755 index 0000000000..fdc87d4f96 --- /dev/null +++ b/do_cmake @@ -0,0 +1,16 @@ +#!/bin/bash -x + +if [[ -d build ]]; then + echo 'build dir already exists; rm -rf build and re-run' + exit 1 +fi + +./get_3rdparty + +export CC=/usr/lib/gcc-4.9-backport/bin/gcc +export CXX=/usr/lib/gcc-4.9-backport/bin/g++ +export LD_LIBRARY_PATH=/usr/lib/gcc-4.9-backport/lib/:/usr/lib:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH + +mkdir build && cd build +cmake -DNEBD_DEBUG_ENABLE=ON -DNEBD_COVERAGE_ENABLE=ON .. +make -j24 diff --git a/do_package b/do_package new file mode 100755 index 0000000000..f8fb067f89 --- /dev/null +++ b/do_package @@ -0,0 +1,10 @@ +#!/bin/bash + +if [[ -d build ]]; then + echo 'build dir already exists; rm it before build package' + exit 1 +fi + +./get_3rdparty + +dpkg-buildpackage -us -uc -j24 diff --git a/docker_unittest.sh b/docker_unittest.sh new file mode 100755 index 0000000000..0383dc26e0 --- /dev/null +++ b/docker_unittest.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -x +set -e + +rm -rf build coverage + +./do_cmake +cd build +ctest -V + +if [[ $? -ne 0 ]]; then + exit 1 +fi + +cd .. +mkdir coverage +gcda=$(find $(pwd)/build/src -name *.gcda) +for i in ${gcda}; do + j="$(echo $i | rev | cut -d . -f 2- | rev).gcno" + cp --parents $i coverage + cp --parents $j coverage +done + +lcov --directory coverage$(pwd)/build --capture --output-file post.info +lcov --directory coverage$(pwd)/build --capture --initial --output-file raw.info +lcov -a post.info -a raw.info -o cov.info --rc lcov_branch_coverage=1 +lcov --remove cov.info '/usr/*' '*3rdparty*' '*/build/*' -o result.info +genhtml result.info --branch-coverage --ignore-errors source -o result \ No newline at end of file diff --git a/etc/nebd/nebd-client.conf b/etc/nebd/nebd-client.conf new file mode 100644 index 0000000000..eba5ce4f5c --- /dev/null +++ b/etc/nebd/nebd-client.conf @@ -0,0 +1,16 @@ +part2ProcName=nebd-server +part2ProcPath=/usr/bin/nebd-server +part2Addr=127.0.0.1 +part2KillCheckRetryTimes=5 +part2KillCheckRetryIntervalUs=100000 +qemuProcName=qemu-system-x86_64 +lockFile=/tmp/nebd-server.port.file.lock +metadataPrefix=/var/run/nebd-server/ +part2StartRetryTimes=5 +part2StartRetryIntervalUs=100000 +connectibleCheckTimes=5 +portGetRetryTimes=5 +portGetRetryIntervalUs=100000 +heartbeatIntervalUs=1000000 +rpcRetryTimes=10 +rpcRetryIntervalUs=10000000 diff --git a/etc/nebd/nebd-server.conf b/etc/nebd/nebd-server.conf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/get_3rdparty b/get_3rdparty new file mode 100755 index 0000000000..2575792442 --- /dev/null +++ b/get_3rdparty @@ -0,0 +1,13 @@ +#!/bin/bash -x +mkdir 3rd_party_tarballs + +for pkg_ver in boost_1_66_0 brpc_0_9_6 googletest_1_8_1 libconfig_1_7_2 jsoncpp_1_8_4 cmock_20180514; do + wget -c --no-verbose http://10.187.0.105/nebd-build/$pkg_ver.tar.gz -P 3rd_party_tarballs/ + wget -c --no-verbose http://10.187.0.105/nebd-build/$pkg_ver.tar.gz.md5sum -P 3rd_party_tarballs/ + pkg=$(echo ${pkg_ver} | cut -d_ -f1) + rm -rf 3rdparty/$pkg + mkdir -p 3rdparty/$pkg + tar xf 3rd_party_tarballs/$pkg_ver.tar.gz -C 3rdparty/$pkg --strip-components=1 +done + +rm -rf 3rd_party_tarballs diff --git a/install_deps b/install_deps new file mode 100755 index 0000000000..20528af076 --- /dev/null +++ b/install_deps @@ -0,0 +1,8 @@ +#!/bin/bash +if [[ $(id -u) != 0 ]]; then + SUDO=sudo +fi +export LC_ALL=C +$SUDO apt-get install sudo devscripts equivs wget lcov -y --force-yes +$SUDO mk-build-deps --install --remove --tool="apt-get -y --no-install-recommends" debian/control +$SUDO apt-get remove nebd-build-deps -y diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000000..e9c42279a5 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,41 @@ +enable_language(CXX) + +# require c++11 +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +set(EXTRALIBS "") + +# coverage option +if(${NEBD_COVERAGE_ENABLE}) + find_program(HAVE_GCOV gcov) + if(NOT HAVE_GCOV) + message(FATAL_ERROR "Coverage Enabled but gcov Not Found") + endif(NOT HAVE_GCOV) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage -O0") + list(APPEND EXTRALIBS gcov) +endif(${NEBD_COVERAGE_ENABLE}) + +# debug option +if(${NEBD_DEBUG_ENABLE}) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g3 -gdwarf-4") +else(${NEBD_DEBUG_ENABLE}) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -g") +endif(${NEBD_DEBUG_ENABLE}) + +# set 3rd party headers +set(Boost_INCLUDE_DIRS "${CMAKE_SOURCE_DIR}/3rdparty/boost") +set(Brpc_INCLUDE_DIRS "${CMAKE_BINARY_DIR}/3rdparty/brpc/output/include") +set(Jsoncpp_INCLUDE_DIRS "${CMAKE_SOURCE_DIR}/3rdparty/jsoncpp/include") +set(Libconfig_INCLUDE_DIRS "${CMAKE_SOURCE_DIR}/3rdparty/libconfig/lib") + +# brpc +include_directories(${Brpc_INCLUDE_DIRS}) +# our header +include_directories(${CMAKE_SOURCE_DIR}) +include_directories(${CMAKE_BINARY_DIR}) + +add_subdirectory(common) +add_subdirectory(part1) +add_subdirectory(part2) diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt new file mode 100644 index 0000000000..2b42abacb5 --- /dev/null +++ b/src/common/CMakeLists.txt @@ -0,0 +1,17 @@ +# libnebd_common.so +file(GLOB COMMON_SRC + "*.h" + "*.cpp" +) + +add_library(nebd_common SHARED ${COMMON_SRC}) +install(TARGETS nebd_common DESTINATION lib) +target_link_libraries(nebd_common ${EXTRALIBS}) + +# libclient_proto.so +include(FindProtobuf) +FIND_PACKAGE(Protobuf REQUIRED) +include_directories(${PROTOBUF_INCLUDE_DIR}) +protobuf_generate_cpp(PROTO_SRC PROTO_HEADER client.proto) +add_library(client_proto ${PROTO_HEADER} ${PROTO_SRC}) +install(TARGETS client_proto DESTINATION lib) diff --git a/src/logrotate.conf b/src/logrotate.conf new file mode 100644 index 0000000000..0694a0e0db --- /dev/null +++ b/src/logrotate.conf @@ -0,0 +1,12 @@ +/var/log/nebd/*.log { + rotate 7 + daily + compress + sharedscripts + postrotate + killall -q -1 nebd-server || true + endscript + missingok + notifempty +} + diff --git a/src/part1/CMakeLists.txt b/src/part1/CMakeLists.txt new file mode 100644 index 0000000000..4cde106620 --- /dev/null +++ b/src/part1/CMakeLists.txt @@ -0,0 +1,37 @@ +# 3rd party headers +include_directories("${Jsoncpp_INCLUDE_DIRS}") + +set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -std=c++0x -pipe -W -Wall \ + -Wno-unused-parameter -fPIC -fno-omit-frame-pointer \ + -momit-leaf-frame-pointer -msse4.2 \ + -pthread -Wsign-compare -Wno-unused-parameter \ + -Wno-unused-variable -Wno-missing-field-initializers \ + -Woverloaded-virtual -Wnon-virtual-dtor") +add_definitions( + "-DGFLAGS=gflags -DOS_LINUX -DSNAPPY -DHAVE_ZLIB -DHAVE_SSE42 -DNDEBUG") + +# libnebd.so +set(LIBNEBD_SRC + libnebd.cpp + libnebd.h + libnebd_file.cpp + libnebd_file.h + nebd_client.cpp + nebd_client.h + nebd_lifecycle.cpp + nebd_lifecycle.h) +set(LIBNEBD_LINK + brpc-shared + rt + ssl + crypto + dl + z + pthread + protobuf + gflags + jsoncpp_lib) +add_library(nebd SHARED ${LIBNEBD_SRC} ${PROTO_SRCS} ${PROTO_HDRS}) +add_dependencies(nebd jsoncpp_lib brpc-shared) +target_link_libraries(nebd ${LIBNEBD_LINK} ${EXTRALIBS}) +install(TARGETS nebd DESTINATION lib) \ No newline at end of file diff --git a/src/part2/CMakeLists.txt b/src/part2/CMakeLists.txt new file mode 100644 index 0000000000..51b1513566 --- /dev/null +++ b/src/part2/CMakeLists.txt @@ -0,0 +1,44 @@ +# 3rd party header +include_directories(${Boost_INCLUDE_DIRS}) +include_directories(${Libconfig_INCLUDE_DIRS}) + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \ + -std=c++0x -pipe -W -Wall -Wno-unused-parameter -fPIC \ + -fno-omit-frame-pointer -momit-leaf-frame-pointer -msse4.2 \ + -pthread -Wsign-compare -Wno-unused-parameter \ + -Wno-unused-variable -Wno-missing-field-initializers \ + -Woverloaded-virtual -Wnon-virtual-dtor") +add_definitions( + "-DGFLAGS=gflags -DOS_LINUX -DSNAPPY -DHAVE_ZLIB -DHAVE_SSE42 -DNDEBUG") + +file(GLOB + PART2_SRC + "*.h" + "*.cpp") + +set(PART2_LINK + brpc-shared + config + crypto + dl + gflags + gtest + leveldb + client_proto + pthread + rados + rbd + rt + snappy + ssl + z) + + +add_executable(nebd-server ${PART2_SRC} ${PROTO_SRCS} ${PROTO_HDRS}) +target_link_libraries(nebd-server ${PART2_LINK} ${EXTRALIBS}) +install(TARGETS nebd-server DESTINATION bin) + +# libnebdserver.so, only for test +add_library(nebdserver SHARED ${PART2_SRC} ${PROTO_SRCS} ${PROTO_HDRS}) +target_link_libraries(nebdserver ${PART2_LINK} ${EXTRALIBS}) + diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000000..abe80bb048 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,29 @@ +enable_language(CXX) + +# require c++11 +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +if(NEBD_DEBUG_ENABLE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g3 -gdwarf-4") +else(NEBD_DEBUG_ENABLE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -g") +endif(NEBD_DEBUG_ENABLE) + +# set 3rd party headers +set(Boost_INCLUDE_DIRS "${CMAKE_SOURCE_DIR}/3rdparty/boost") +set(Brpc_INCLUDE_DIRS "${CMAKE_BINARY_DIR}/3rdparty/brpc/output/include") +set(Jsoncpp_INCLUDE_DIRS "${CMAKE_SOURCE_DIR}/3rdparty/jsoncpp/include") +set(Libconfig_INCLUDE_DIRS "${CMAKE_SOURCE_DIR}/3rdparty/libconfig/lib") +set(CMock_INCLUDE_DIRS "${CMAKE_SOURCE_DIR}/3rdparty/cmock/include/cmock") + +# brpc +include_directories(${Brpc_INCLUDE_DIRS}) +# our header +include_directories(${CMAKE_SOURCE_DIR}) +include_directories(${CMAKE_BINARY_DIR}) + +add_subdirectory(common) +add_subdirectory(part1) +add_subdirectory(part2) diff --git a/tests/common/CMakeLists.txt b/tests/common/CMakeLists.txt new file mode 100644 index 0000000000..3891efc126 --- /dev/null +++ b/tests/common/CMakeLists.txt @@ -0,0 +1,16 @@ +set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -std=c++0x -pipe -W -Wall \ + -Wno-unused-parameter -fPIC -fno-omit-frame-pointer \ + -momit-leaf-frame-pointer -msse4.2 \ + -pthread -Wsign-compare -Wno-unused-parameter \ + -Wno-unused-variable -Wno-missing-field-initializers \ + -Woverloaded-virtual -Wnon-virtual-dtor") +add_definitions( + "-DGFLAGS=gflags -DOS_LINUX -DSNAPPY -DHAVE_ZLIB -DHAVE_SSE42 -DNDEBUG") + +# test_configuration +set(TEST_CONF_SRC configuration_test.cpp) +set(TEST_CONF_LINK nebd_common gflags gtest gmock) +add_executable(test_configuration ${TEST_CONF_SRC}) +target_link_libraries(test_configuration ${TEST_CONF_LINK}) +install(TARGETS test_configuration DESTINATION bin) +add_test(NAME test_configuration COMMAND test_configuration) diff --git a/tests/part1/CMakeLists.txt b/tests/part1/CMakeLists.txt new file mode 100644 index 0000000000..ac3edec678 --- /dev/null +++ b/tests/part1/CMakeLists.txt @@ -0,0 +1,53 @@ +# 3rd party headers +include_directories("${Jsoncpp_INCLUDE_DIRS}") + +set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -std=c++0x -pipe -W -Wall \ + -Wno-unused-parameter -fPIC -fno-omit-frame-pointer \ + -momit-leaf-frame-pointer -msse4.2 \ + -pthread -Wsign-compare -Wno-unused-parameter \ + -Wno-unused-variable -Wno-missing-field-initializers \ + -Woverloaded-virtual -Wnon-virtual-dtor") +add_definitions( + "-DGFLAGS=gflags -DOS_LINUX -DSNAPPY -DHAVE_ZLIB -DHAVE_SSE42 -DNDEBUG") + +# libfake_client_service.so +set(FAKE_CLT_SRV_SRC fake_client_service.cpp fake_client_service.h) +set(FAKE_CLT_SRV_LINK + brpc-shared + client_proto + gflags + gtest + gmock) +add_library(fake_client_service SHARED ${FAKE_CLT_SRV_SRC}) +target_link_libraries(fake_client_service ${FAKE_CLT_SRV_LINK}) +install(TARGETS fake_client_service DESTINATION lib) + +# client_server +set(CLT_SERVER_SRC client_server.cpp) +set(CLT_SERVER_LINK fake_client_service jsoncpp_lib) +add_executable(client_server ${CLT_SERVER_SRC}) +target_link_libraries(client_server ${CLT_SERVER_LINK}) +install(TARGETS client_server DESTINATION bin) + +# test_part1 +set(TEST_PART1_SRC nebd_test.cpp mock_client_service.h) +set(TEST_PART1_LINK + rt + ssl + crypto + dl + z + pthread + gflags + gtest + gmock + nebd + nebd_common + client_proto) + +add_executable(test_part1 ${TEST_PART1_SRC}) +target_link_libraries(test_part1 ${TEST_PART1_LINK}) +install(TARGETS test_part1 DESTINATION bin) +add_test(NAME test_part1 + COMMAND test_part1 -uuid 1234 + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) diff --git a/tests/part2/CMakeLists.txt b/tests/part2/CMakeLists.txt new file mode 100644 index 0000000000..6f0e5a584b --- /dev/null +++ b/tests/part2/CMakeLists.txt @@ -0,0 +1,102 @@ +include_directories(${Boost_INCLUDE_DIRS}) +include_directories(${Libconfig_INCLUDE_DIRS}) +include_directories(${CMock_INCLUDE_DIRS}) + +set(CPP_FLAGS "-DBRPC_WITH_GLOG=0 -DGFLAGS_NS=google") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CPP_FLAGS} -std=c++0x -DNDEBUG -O2 \ + -D__const__= -pipe -W -Wall -Wno-unused-parameter -fPIC \ + -fno-omit-frame-pointer") + +set(RELOADUNIT_SRC test_reload.cpp test_reload.h TestReload.cpp) +add_executable(reload_unit ${RELOADUNIT_SRC}) +target_link_libraries(reload_unit + nebdserver + gtest + gmock) +install(TARGETS reload_unit DESTINATION bin) +add_test(NAME reload_unit COMMAND reload_unit) + +set(RELOAD2UNIT_SRC test_reload2.cpp test_reload2.h TestReload2.cpp) +add_executable(reload2_unit ${RELOAD2UNIT_SRC}) +target_link_libraries(reload2_unit + nebdserver + gtest + gmock) +install(TARGETS reload2_unit DESTINATION bin) +add_test(NAME reload2_unit COMMAND reload2_unit) + +set(COMMONUNIT_SRC test_common.cpp test_common.h TestCommon.cpp) +add_executable(common_unit ${COMMONUNIT_SRC}) +target_link_libraries(common_unit + nebdserver + gtest + gmock) +install(TARGETS common_unit DESTINATION bin) +add_test(NAME common_unit COMMAND common_unit) + +set(RADOSUNIT_SRC test_rados.cpp test_rados.h TestRados.cpp) +add_executable(rados_unit ${RADOSUNIT_SRC}) +target_link_libraries(rados_unit + nebdserver + gtest + gmock) +install(TARGETS rados_unit DESTINATION bin) +add_test(NAME rados_unit COMMAND rados_unit) + +set(RPCSERVERUNIT_SRC TestRpcServer.cpp) +add_executable(rpcserver_unit ${RPCSERVERUNIT_SRC}) +target_link_libraries(rpcserver_unit + nebdserver + gtest + gmock) +install(TARGETS rpcserver_unit DESTINATION bin) +add_test(NAME rpcserver_unit COMMAND rpcserver_unit) + +set(RPCCEPHUNIT_SRC TestRpcCeph.cpp test_rpcceph.cpp test_rpcceph.h) +add_executable(rpcceph_unit ${RPCCEPHUNIT_SRC}) +target_link_libraries(rpcceph_unit + nebdserver + gtest + gmock + rados + rbd) +install(TARGETS rpcceph_unit DESTINATION bin) +add_test(NAME rpcceph_unit COMMAND rpcceph_unit) + +set(CONFIGUNIT_SRC TestConfig.cpp) +add_executable(config_unit ${CONFIGUNIT_SRC}) +target_link_libraries(config_unit + nebdserver + gtest + gmock + rados + rbd) +install(TARGETS config_unit DESTINATION bin) +add_test(NAME config_unit COMMAND config_unit) + +set(HEARTBEATUNIT_SRC TestHeartbeat.cpp test_heartbeat.cpp test_heartbeat.h) +add_executable(heartbeat_unit ${HEARTBEATUNIT_SRC}) +target_link_libraries(heartbeat_unit + nebdserver + gtest + gmock) +install(TARGETS heartbeat_unit DESTINATION bin) +add_test(NAME heartbeat_unit COMMAND heartbeat_unit) + +set(HEARTBEATUNIT2_SRC TestHeartbeat2.cpp test_heartbeat2.cpp test_heartbeat2.h) +add_executable(heartbeat2_unit ${HEARTBEATUNIT2_SRC}) +target_link_libraries(heartbeat2_unit + nebdserver + gtest + gmock) +install(TARGETS heartbeat2_unit DESTINATION bin) +add_test(NAME heartbeat2_unit COMMAND heartbeat2_unit) + +set(HEARTBEAT3UNIT_SRC TestHeartbeat3.cpp test_heartbeat3.cpp test_heartbeat3.h) +add_executable(heartbeat3_unit ${HEARTBEAT3UNIT_SRC}) +target_link_libraries(heartbeat3_unit + nebdserver + gtest + gmock) +install(TARGETS heartbeat3_unit DESTINATION bin) +add_test(NAME heartbeat3_unit COMMAND heartbeat3_unit) From 12244d903e13d50c602f11e33751905f3db2fa30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=A8=81?= Date: Wed, 16 Oct 2019 10:33:41 +0800 Subject: [PATCH 03/79] add part1 code Change-Id: I145bf3f36da680683414cd6bf7e03642f7b3f4d8 --- etc/nebd/nebd-client.conf | 47 +- src/common/client.proto | 118 +++++ src/common/configuration.cpp | 214 ++++++++ src/common/configuration.h | 110 +++++ src/part1/libnebd.cpp | 92 ++++ src/part1/libnebd.h | 175 +++++++ src/part1/libnebd_file.cpp | 62 +++ src/part1/libnebd_file.h | 91 ++++ src/part1/nebd_client.cpp | 556 +++++++++++++++++++++ src/part1/nebd_client.h | 232 +++++++++ src/part1/nebd_lifecycle.cpp | 733 ++++++++++++++++++++++++++++ src/part1/nebd_lifecycle.h | 198 ++++++++ tests/common/configuration_test.cpp | 284 +++++++++++ tests/part1/client_server.cpp | 89 ++++ tests/part1/fake_client_service.cpp | 180 +++++++ tests/part1/fake_client_service.h | 77 +++ tests/part1/lock.file | 0 tests/part1/mock_client_service.h | 69 +++ tests/part1/nebd.conf | 45 ++ tests/part1/nebd_test.cpp | 601 +++++++++++++++++++++++ 20 files changed, 3964 insertions(+), 9 deletions(-) create mode 100644 src/common/client.proto create mode 100644 src/common/configuration.cpp create mode 100644 src/common/configuration.h create mode 100644 src/part1/libnebd.cpp create mode 100644 src/part1/libnebd.h create mode 100644 src/part1/libnebd_file.cpp create mode 100644 src/part1/libnebd_file.h create mode 100644 src/part1/nebd_client.cpp create mode 100644 src/part1/nebd_client.h create mode 100644 src/part1/nebd_lifecycle.cpp create mode 100644 src/part1/nebd_lifecycle.h create mode 100644 tests/common/configuration_test.cpp create mode 100644 tests/part1/client_server.cpp create mode 100644 tests/part1/fake_client_service.cpp create mode 100644 tests/part1/fake_client_service.h create mode 100644 tests/part1/lock.file create mode 100644 tests/part1/mock_client_service.h create mode 100644 tests/part1/nebd.conf create mode 100644 tests/part1/nebd_test.cpp diff --git a/etc/nebd/nebd-client.conf b/etc/nebd/nebd-client.conf index eba5ce4f5c..66ca076626 100644 --- a/etc/nebd/nebd-client.conf +++ b/etc/nebd/nebd-client.conf @@ -1,16 +1,45 @@ +# nebd part2的进程名字 part2ProcName=nebd-server +# nebd part2的程序的启动路径，从该路径拉起part2 part2ProcPath=/usr/bin/nebd-server +# nebd part2的ip地址 part2Addr=127.0.0.1 -part2KillCheckRetryTimes=5 -part2KillCheckRetryIntervalUs=100000 +# kill part2服务之后，检查part2服务是否存活的重试次数 +part2KillCheckRetryTimes=150 +# kill part2服务之后，检查part2服务是否存活的重试间隔，单位Us +part2KillCheckRetryIntervalUs=200000 +# nebd part1的qemu进程的名字 qemuProcName=qemu-system-x86_64 +# nebd part1和part2共用的文件锁文件的路径 lockFile=/tmp/nebd-server.port.file.lock +# nebd part1和part2共用的元数据文件的目录 metadataPrefix=/var/run/nebd-server/ -part2StartRetryTimes=5 -part2StartRetryIntervalUs=100000 -connectibleCheckTimes=5 -portGetRetryTimes=5 -portGetRetryIntervalUs=100000 +# 拉起part2服务的重试次数 +part2StartRetryTimes=150 +# 拉起part2服务的重试间隔，单位Us +part2StartRetryIntervalUs=200000 +# 检查part2的port是否联通的重试次数 +connectibleCheckTimes=150 +# 检查part2的port是否联通的重试间隔，单位Us +connectibleCheckIntervalUs=200000 +# 获取part2的port的重试次数 +portGetRetryTimes=150 +# 获取part2的port的重试间隔，单位Us +portGetRetryIntervalUs=200000 +# 心跳服务的检查间隔，单位Us heartbeatIntervalUs=1000000 -rpcRetryTimes=10 -rpcRetryIntervalUs=10000000 +# 非io的rpc请求的异常重试次数 +rpcRetryTimes=50 +# rpc请求的重试间隔，包括io请求和非io请求 +rpcRetryIntervalUs=2000000 +# rpc请求的重试间隔，当io请求遇到server服务未启动的时候使用这个重试间隔， +# 这个值一般比rpcRetryIntervalUs更小，为了更快恢复io服务 +rpcHostDownRetryIntervalUs=200000 +# rpc请求的最大重试间隔 +rpcRetryMaxIntervalUs=64000000 +# rpc请求的超时时间，单位Ms +rpcTimeoutMs=30000 +# brpc的健康检查周期时间，单位s +rpcHealthCheckIntervalS=1 +# 异步IO操作RPC失败日志打印间隔，每隔多少次打印一次 +aioRpcFailLogInterval=10 \ No newline at end of file diff --git a/src/common/client.proto b/src/common/client.proto new file mode 100644 index 0000000000..84202c711e --- /dev/null +++ b/src/common/client.proto @@ -0,0 +1,118 @@ +syntax="proto2"; +package nebd.client; +option cc_generic_services = true; +enum RetCode { + kNoOK = -1; + kOK = 0; +} + +message OpenFileRequest { + required string fileName = 1; +} +message OpenFileResponse { + required RetCode retCode = 1; + optional string retMsg = 2; + optional int32 fd = 3; + // ps 最好fd最为一个64位以内的随机数，发挥sessionid的作用 +} + +message CloseFileRequest { + required int32 fd = 1; +} +message CloseFileResponse { + required RetCode retCode = 1; + optional string retMsg = 2; +} + +message ReadRequest { + required int32 fd = 1; + required uint64 offset = 2; + required uint64 size = 3; +} +message ReadResponse { + required RetCode retCode = 1; + optional string retMsg = 2; +} +// readbody in attachment + +message WriteRequest { + required int32 fd = 1; + required uint64 offset = 2; + required uint64 size = 3; +} +// write content in attachment +message WriteResponse { + required RetCode RetCode = 1; + optional string RetMsg = 2; +} + +message DiscardRequest { + required int32 fd = 1; + required uint64 offset = 2; + required uint64 size = 3; +} +message DiscardResponse { + required RetCode RetCode = 1; + optional string RetMsg = 2; +} + +message StatFileRequest { + required int32 fd = 1; +} +message StatFileResponse { + required RetCode retCode = 1; + optional string retMsg = 2; + optional uint64 size = 3; +} + +message ResizeRequest { + required int32 fd = 1; + required uint64 newSize = 2; +} +message ResizeResponse { + required RetCode retCode = 1; + optional string retMsg = 2; +} + +// 以下proto定义，仅ceph使用 +message FlushRequest { + required int32 fd = 1; +} + +message FlushResponse { + required RetCode retCode = 1; + optional string retMsg = 2; +} + +message GetInfoRequest { + required int32 fd = 1; +} +message GetInfoResponse { + required RetCode retCode = 1; + optional string retMsg = 2; + optional uint64 objSize = 3; +} + +message InvalidateCacheRequest { + required int32 fd = 1; +} + +message InvalidateCacheResponse { + required RetCode retCode = 1; + optional string retMsg = 2; +} + +service QemuClientService { + // for ceph & curve + rpc OpenFile(OpenFileRequest) returns (OpenFileResponse); + rpc CloseFile(CloseFileRequest) returns (CloseFileResponse); + rpc Read(ReadRequest) returns (ReadResponse); + rpc Write(WriteRequest) returns (WriteResponse); + rpc Discard(DiscardRequest) returns (DiscardResponse); + rpc StatFile(StatFileRequest) returns (StatFileResponse); + rpc ResizeFile(ResizeRequest) returns (ResizeResponse); + // for ceph only + rpc Flush(FlushRequest) returns (FlushResponse); + rpc GetInfo(GetInfoRequest) returns (GetInfoResponse); + rpc InvalidateCache(InvalidateCacheRequest) returns (InvalidateCacheResponse); +}; diff --git a/src/common/configuration.cpp b/src/common/configuration.cpp new file mode 100644 index 0000000000..4efe53713a --- /dev/null +++ b/src/common/configuration.cpp @@ -0,0 +1,214 @@ +/* + * Copyright (C) 2018 NetEase Inc. All rights reserved. + * Project: Curve + * + * History: + * 2018/08/30 Wenyu Zhou Initial version + */ +/* + * Project: nebd + * File Created: 2019-08-07 + * Author: hzchenwei7 + * Copyright (c) 2018 NetEase + */ +#include "src/common/configuration.h" + +#include +#include +#include +#include + +namespace nebd { +namespace common { + +bool Configuration::LoadConfig() { + std::ifstream cFile(confFile_); + + if (cFile.is_open()) { + std::string line; + while (getline(cFile, line)) { + // FIXME: may not remove middle spaces + line.erase(std::remove_if(line.begin(), line.end(), isspace), + line.end()); + if (line[0] == '#' || line.empty()) + continue; + + int delimiterPos = line.find("="); + std::string key = line.substr(0, delimiterPos); + std::string value = line.substr(delimiterPos + 1); + config_[key] = value; + } + } else { + return false; + } + + return true; +} + +bool Configuration::SaveConfig() { + // TODO(wenyu): to implement + return false; +} + +std::string Configuration::DumpConfig() { + // TODO(wenyu): to implement + return ""; +} + + +std::map Configuration::ListConfig() const { + return config_; +} + +void Configuration::SetConfigPath(const std::string &path) { + confFile_ = path; +} + +std::string Configuration::GetConfigPath() { + return confFile_; +} + +std::string Configuration::GetStringValue(const std::string &key) { + return GetValue(key); +} + +bool Configuration::GetStringValue(const std::string &key, std::string *out) { + return GetValue(key, out); +} + +void Configuration::SetStringValue(const std::string &key, + const std::string &value) { + SetValue(key, value); +} + +int Configuration::GetIntValue(const std::string &key, uint64_t defaultvalue) { + std::string value = GetValue(key); + return (value == "") ? defaultvalue : std::stoi(value); +} + +bool Configuration::GetIntValue(const std::string &key, int *out) { + std::string res; + if (GetValue(key, &res)) { + *out = std::stoi(res); + return true; + } + return false; +} + +bool Configuration::GetUInt32Value(const std::string &key, uint32_t *out) { + std::string res; + if (GetValue(key, &res)) { + *out = std::stoul(res); + return true; + } + return false; +} + +bool Configuration::GetUInt64Value(const std::string &key, uint64_t *out) { + std::string res; + if (GetValue(key, &res)) { + *out = std::stoull(res); + return true; + } + return false; +} + + +void Configuration::SetIntValue(const std::string &key, const int value) { + SetValue(key, std::to_string(value)); +} + +double Configuration::GetDoubleValue( + const std::string &key, + double defaultvalue) { + std::string value = GetValue(key); + return (value == "") ? defaultvalue : std::stod(value); +} + +bool Configuration::GetDoubleValue(const std::string &key, double *out) { + std::string res; + if (GetValue(key, &res)) { + *out = std::stod(res); + return true; + } + return false; +} + +void Configuration::SetDoubleValue(const std::string &key, const double value) { + SetValue(key, std::to_string(value)); +} + + +double Configuration::GetFloatValue( + const std::string &key, float defaultvalue) { + std::string value = GetValue(key); + return (value == "") ? defaultvalue : std::stof(value); +} + +bool Configuration::GetFloatValue(const std::string &key, float *out) { + std::string res; + if (GetValue(key, &res)) { + *out = std::stof(res); + return true; + } + return false; +} + +void Configuration::SetFloatValue(const std::string &key, const float value) { + SetValue(key, std::to_string(value)); +} + +bool Configuration::GetBoolValue(const std::string &key, bool defaultvalue) { + std::string svalue = config_[key]; + transform(svalue.begin(), svalue.end(), svalue.begin(), ::tolower); + + bool istrue = (svalue == "true") || (svalue == "yes") || (svalue == "1"); + bool isfalse = (svalue == "false") || (svalue == "no") || (svalue == "0"); + bool ret = istrue ? true : isfalse ? false : defaultvalue; + return ret; +} + +bool Configuration::GetBoolValue(const std::string &key, bool *out) { + std::string res; + if (GetValue(key, &res)) { + transform(res.begin(), res.end(), res.begin(), ::tolower); + bool istrue = (res == "true") || (res == "yes") || (res == "1"); + bool isfalse = (res == "false") || (res == "no") || (res == "0"); + if (istrue) { + *out = true; + return true; + } + if (isfalse) { + *out = false; + return true; + } + return false; + } + + return false; +} + + +void Configuration::SetBoolValue(const std::string &key, const bool value) { + SetValue(key, std::to_string(value)); +} + +std::string Configuration::GetValue(const std::string &key) { + return config_[key]; +} + +bool Configuration::GetValue(const std::string &key, std::string *out) { + if (config_.find(key) != config_.end()) { + *out = config_[key]; + return true; + } + + return false; +} + +void Configuration::SetValue(const std::string &key, const std::string &value) { + config_[key] = value; +} + +} // namespace common +} // namespace nebd diff --git a/src/common/configuration.h b/src/common/configuration.h new file mode 100644 index 0000000000..aa9678c0eb --- /dev/null +++ b/src/common/configuration.h @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2018 NetEase Inc. All rights reserved. + * Project: Curve + * + * History: + * 2018/08/30 Wenyu Zhou Initial version + */ +/* + * Project: nebd + * File Created: 2019-08-07 + * Author: hzchenwei7 + * Copyright (c) 2018 NetEase + */ +#include +#include + +#ifndef SRC_COMMON_CONFIGURATION_H_ +#define SRC_COMMON_CONFIGURATION_H_ + +namespace nebd { +namespace common { + +class Configuration { + public: + Configuration() {} + ~Configuration() {} + + bool LoadConfig(); + bool SaveConfig(); + std::string DumpConfig(); + std::map ListConfig() const; + + void SetConfigPath(const std::string &path); + std::string GetConfigPath(); + + std::string GetStringValue(const std::string &key); + /* + * @brief GetStringValue 获取指定配置项的值 + * + * @param[in] key 配置项名称 + * @param[out] out 获取的值 + * + * @return false-未获取到 true-获取成功 + */ + bool GetStringValue(const std::string &key, std::string *out); + void SetStringValue(const std::string &key, const std::string &value); + + int GetIntValue(const std::string &key, uint64_t defaultvalue = 0); + /* + * @brief GetIntValue/GetUInt32Value/GetUInt64Value 获取指定配置项的值 //NOLINT + * + * @param[in] key 配置项名称 + * @param[out] out 获取的值 + * + * @return false-未获取到 true-获取成功 + */ + bool GetIntValue(const std::string &key, int *out); + bool GetUInt32Value(const std::string &key, uint32_t *out); + bool GetUInt64Value(const std::string &key, uint64_t *out); + void SetIntValue(const std::string &key, const int value); + + double GetDoubleValue(const std::string &key, double defaultvalue = 0.0); + /* + * @brief GetDoubleValue 获取指定配置项的值 + * + * @param[in] key 配置项名称 + * @param[out] out 获取的值 + * + * @return false-未获取到 true-获取成功 + */ + bool GetDoubleValue(const std::string &key, double *out); + void SetDoubleValue(const std::string &key, const double value); + + double GetFloatValue(const std::string &key, float defaultvalue = 0.0); + /* + * @brief GetFloatValue 获取指定配置项的值 + * + * @param[in] key 配置项名称 + * @param[out] out 获取的值 + * + * @return false-未获取到 true-获取成功 + */ + bool GetFloatValue(const std::string &key, float *out); + void SetFloatValue(const std::string &key, const float value); + + bool GetBoolValue(const std::string &key, bool defaultvalue = false); + /* + * @brief GetBoolValue 获取指定配置项的值 + * + * @param[in] key 配置项名称 + * @param[out] out 获取的值 + * + * @return false-未获取到 true-获取成功 + */ + bool GetBoolValue(const std::string &key, bool *out); + void SetBoolValue(const std::string &key, const bool value); + + std::string GetValue(const std::string &key); + bool GetValue(const std::string &key, std::string *out); + void SetValue(const std::string &key, const std::string &value); + + private: + std::string confFile_; + std::map config_; +}; + +} // namespace common +} // namespace nebd + +#endif // SRC_COMMON_CONFIGURATION_H_ diff --git a/src/part1/libnebd.cpp b/src/part1/libnebd.cpp new file mode 100644 index 0000000000..a30ae28391 --- /dev/null +++ b/src/part1/libnebd.cpp @@ -0,0 +1,92 @@ +/* + * Project: nebd + * File Created: 2019-08-07 + * Author: hzchenwei7 + * Copyright (c) 2018 NetEase + */ + +#include "src/part1/libnebd.h" +#include "src/part1/libnebd_file.h" + +extern "C" { +bool g_inited = false; +const char* confpath = "/etc/nebd/nebd-client.conf"; +int nebd_lib_init() { + if (g_inited) { + return 0; + } + + int ret = Init4Qemu(confpath); + if (ret != 0) { + return ret; + } + + g_inited = true; + + return ret; +} + +int nebd_lib_uninit() { + if (g_inited) { + Uninit4Qemu(); + g_inited = false; + } + + return 0; +} + +int nebd_lib_open(const char* filename) { + return Open4Qemu(filename); +} + +int nebd_lib_close(int fd) { + return Close4Qemu(fd); +} + +int nebd_lib_pread(int fd, void* buf, off_t offset, size_t length) { + // not support sync read + return -1; +} + +int nebd_lib_pwrite(int fd, const void* buf, off_t offset, size_t length) { + // not support sync write + return -1; +} + +int nebd_lib_discard(int fd, ClientAioContext* context) { + return Discard4Qemu(fd, context); +} + +int nebd_lib_aio_pread(int fd, ClientAioContext* context) { + return AioRead4Qemu(fd, context); +} + +int nebd_lib_aio_pwrite(int fd, ClientAioContext* context) { + return AioWrite4Qemu(fd, context); +} + +int nebd_lib_sync(int fd) { + return 0; +} + +int64_t nebd_lib_filesize(int fd) { + return StatFile4Qemu(fd); +} + +int nebd_lib_resize(int fd, int64_t size) { + return Extend4Qemu(fd, size); +} + +int nebd_lib_flush(int fd, ClientAioContext* context) { + return Flush4Qemu(fd, context); +} + +int64_t nebd_lib_getinfo(int fd) { + return GetInfo4Qemu(fd); +} + +int nebd_lib_invalidcache(int fd) { + return InvalidCache4Qemu(fd); +} + +} // extern "C" diff --git a/src/part1/libnebd.h b/src/part1/libnebd.h new file mode 100644 index 0000000000..1d3149b59e --- /dev/null +++ b/src/part1/libnebd.h @@ -0,0 +1,175 @@ +/* + * Project: nebd + * File Created: 2019-08-07 + * Author: hzchenwei7 + * Copyright (c) 2018 NetEase + */ + +#ifndef SRC_PART1_LIBNEBD_H_ +#define SRC_PART1_LIBNEBD_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +// 文件路径最大的长度，单位字节 +#define NEBD_MAX_FILE_PATH_LEN 1024 +// 请求的buf的最大长度，读写操作的buf长度不得超过该值，单位字节 +#define NEBD_MAX_BUF_LEN 1024 * 1024 * 32 + +// nebd异步请求的类型 +typedef enum LIBAIO_OP { + LIBAIO_OP_READ, + LIBAIO_OP_WRITE, + LIBAIO_OP_DISCARD, + LIBAIO_OP_FLUSH, +} LIBAIO_OP; + +struct ClientAioContext; + +// nebd回调函数的类型 +typedef void (*LibAioCallBack)(struct ClientAioContext* context); + +typedef struct ClientAioContext { + off_t offset; // 请求的offset + size_t length; // 请求的length + int ret; // 记录异步返回的返回值 + LIBAIO_OP op; // 异步请求的类型，详见定义 + LibAioCallBack cb; // 异步请求的回调函数 + void* buf; // 请求的buf + unsigned int retryCount; // 记录异步请求的重试次数 +} ClientAioContext; + +// int nebd_lib_fini(void); +// for ceph & curve +/** + * @brief 初始化nebd，仅在第一次调用的时候真正执行初始化逻辑 + * @param none + * @return 成功返回0，失败返回-1 + */ +int nebd_lib_init(void); + +/** + * @brief 反初始化nebd + * @param none + * @return 成功返回0，失败返回-1 + */ +int nebd_lib_uninit(void); + +/** + * @brief open文件 + * @param filename：文件名 + * @return 成功返回文件fd，失败返回错误码 + */ +int nebd_lib_open(const char* filename); + +/** + * @brief close文件 + * @param fd：文件的fd + * @return 成功返回0，失败返回错误码 + */ +int nebd_lib_close(int fd); + +/** + * @brief 同步读文件 + * @param fd：文件的fd + * buf：存放读取data的buf + * offset：读取的位置offset + * length：读取的长度 + * @return 成功返回0，失败返回错误码 + */ +int nebd_lib_pread(int fd, void* buf, off_t offset, size_t length); + +/** + * @brief 同步写文件 + * @param fd：文件的fd + * buf：存放写入data的buf + * offset：写入的位置offset + * length：写入的长度 + * @return 成功返回0，失败返回错误码 + */ +int nebd_lib_pwrite(int fd, const void* buf, off_t offset, size_t length); + +/** + * @brief discard文件，异步函数 + * @param fd：文件的fd + * context：异步请求的上下文，包含请求所需的信息以及回调 + * @return 成功返回0，失败返回错误码 + */ +int nebd_lib_discard(int fd, ClientAioContext* context); + +/** + * @brief 读文件，异步函数 + * @param fd：文件的fd + * context：异步请求的上下文，包含请求所需的信息以及回调 + * @return 成功返回0，失败返回错误码 + */ +int nebd_lib_aio_pread(int fd, ClientAioContext* context); + +/** + * @brief 写文件，异步函数 + * @param fd：文件的fd + * context：异步请求的上下文，包含请求所需的信息以及回调 + * @return 成功返回0，失败返回错误码 + */ +int nebd_lib_aio_pwrite(int fd, ClientAioContext* context); + +/** + * @brief sync文件 + * @param fd：文件的fd + * @return 成功返回0，失败返回错误码 + */ +int nebd_lib_sync(int fd); + +/** + * @brief 获取文件size + * @param fd：文件的fd + * @return 成功返回文件size，失败返回错误码 + */ +int64_t nebd_lib_filesize(int fd); + +/** + * @brief resize文件 + * @param fd：文件的fd + * size：调整后的文件size + * @return 成功返回0，失败返回错误码 + */ +int nebd_lib_resize(int fd, int64_t size); + +// for ceph only +/** + * @brief flush文件，异步函数 + * @param fd：文件的fd + * context：异步请求的上下文，包含请求所需的信息以及回调 + * @return 成功返回0，失败返回错误码 + */ +int nebd_lib_flush(int fd, ClientAioContext* context); + +/** + * @brief 获取文件info + * @param fd：文件的fd + * @return 成功返回文件对象size，失败返回错误码 + */ +int64_t nebd_lib_getinfo(int fd); + +/** + * @brief 刷新cache，等所有异步请求返回 + * @param fd：文件的fd + * @return 成功返回0，失败返回错误码 + */ +int nebd_lib_invalidcache(int fd); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // SRC_PART1_LIBNEBD_H_ diff --git a/src/part1/libnebd_file.cpp b/src/part1/libnebd_file.cpp new file mode 100644 index 0000000000..df273feabd --- /dev/null +++ b/src/part1/libnebd_file.cpp @@ -0,0 +1,62 @@ +/* + * Project: nebd + * File Created: 2019-08-07 + * Author: hzchenwei7 + * Copyright (c) 2018 NetEase + */ + +#include "src/part1/libnebd_file.h" +#include +#include "src/common/configuration.h" +#include "src/part1/nebd_lifecycle.h" +#include "src/part1/nebd_client.h" + + +int Init4Qemu(const char* confpath) { + return nebd::client::fileClient.Init(confpath); +} + +void Uninit4Qemu() { + nebd::client::fileClient.Uninit(); + return; +} + +int Open4Qemu(const char* filename) { + return nebd::client::fileClient.Open(filename); +} + +int Close4Qemu(int fd) { + return nebd::client::fileClient.Close(fd); +} + +int Extend4Qemu(int fd, int64_t newsize) { + return nebd::client::fileClient.Extend(fd, newsize); +} + +int64_t StatFile4Qemu(int fd) { + return nebd::client::fileClient.StatFile(fd); +} + +int Discard4Qemu(int fd, ClientAioContext* aioctx) { + return nebd::client::fileClient.Discard(fd, aioctx); +} + +int AioRead4Qemu(int fd, ClientAioContext* aioctx) { + return nebd::client::fileClient.AioRead(fd, aioctx); +} + +int AioWrite4Qemu(int fd, ClientAioContext* aioctx) { + return nebd::client::fileClient.AioWrite(fd, aioctx); +} + +int Flush4Qemu(int fd, ClientAioContext* aioctx) { + return nebd::client::fileClient.Flush(fd, aioctx); +} + +int64_t GetInfo4Qemu(int fd) { + return nebd::client::fileClient.GetInfo(fd); +} + +int InvalidCache4Qemu(int fd) { + return nebd::client::fileClient.InvalidCache(fd); +} diff --git a/src/part1/libnebd_file.h b/src/part1/libnebd_file.h new file mode 100644 index 0000000000..4caf875941 --- /dev/null +++ b/src/part1/libnebd_file.h @@ -0,0 +1,91 @@ +/* + * Project: nebd + * File Created: 2019-08-07 + * Author: hzchenwei7 + * Copyright (c) 2018 NetEase + */ + +#ifndef SRC_PART1_LIBNEBD_FILE_H_ +#define SRC_PART1_LIBNEBD_FILE_H_ + +#include "src/part1/libnebd.h" + +/** + * @brief 初始化nebd，仅在第一次调用的时候真正执行初始化逻辑 + * @param none + * @return 成功返回0，失败返回-1 + */ +int Init4Qemu(const char* confpath); +/** + * @brief 反初始化nebd + * @param none + * @return 成功返回0，失败返回-1 + */ +void Uninit4Qemu(); +/** + * @brief open文件 + * @param filename：文件名 + * @return 成功返回文件fd，失败返回错误码 + */ +int Open4Qemu(const char* filename); +/** + * @brief close文件 + * @param fd：文件的fd + * @return 成功返回0，失败返回错误码 + */ +int Close4Qemu(int fd); +/** + * @brief resize文件 + * @param fd：文件的fd + * size：调整后的文件size + * @return 成功返回0，失败返回错误码 + */ +int Extend4Qemu(int fd, int64_t newsize); +/** + * @brief 获取文件size + * @param fd：文件的fd + * @return 成功返回文件size，失败返回错误码 + */ +int64_t StatFile4Qemu(int fd); +/** + * @brief discard文件，异步函数 + * @param fd：文件的fd + * context：异步请求的上下文，包含请求所需的信息以及回调 + * @return 成功返回0，失败返回错误码 + */ +int Discard4Qemu(int fd, ClientAioContext* aioctx); +/** + * @brief 读文件，异步函数 + * @param fd：文件的fd + * context：异步请求的上下文，包含请求所需的信息以及回调 + * @return 成功返回0，失败返回错误码 + */ +int AioRead4Qemu(int fd, ClientAioContext* aioctx); +/** + * @brief 写文件，异步函数 + * @param fd：文件的fd + * context：异步请求的上下文，包含请求所需的信息以及回调 + * @return 成功返回0，失败返回错误码 + */ +int AioWrite4Qemu(int fd, ClientAioContext* aioctx); +/** + * @brief flush文件，异步函数 + * @param fd：文件的fd + * context：异步请求的上下文，包含请求所需的信息以及回调 + * @return 成功返回0，失败返回错误码 + */ +int Flush4Qemu(int fd, ClientAioContext* aioctx); +/** + * @brief 获取文件info + * @param fd：文件的fd + * @return 成功返回文件对象size，失败返回错误码 + */ +int64_t GetInfo4Qemu(int fd); +/** + * @brief 刷新cache，等所有异步请求返回 + * @param fd：文件的fd + * @return 成功返回0，失败返回错误码 + */ +int InvalidCache4Qemu(int fd); + +#endif // SRC_PART1_LIBNEBD_FILE_H_ diff --git a/src/part1/nebd_client.cpp b/src/part1/nebd_client.cpp new file mode 100644 index 0000000000..9eeb8c3fbe --- /dev/null +++ b/src/part1/nebd_client.cpp @@ -0,0 +1,556 @@ +/* + * Project: nebd + * File Created: 2019-10-08 + * Author: hzchenwei7 + * Copyright (c) 2018 NetEase + */ + +#include "src/part1/nebd_client.h" +#include +#include +#include +#include +#include +#include +#include +#include "src/common/configuration.h" + +// 修改brpc的health_check_interval参数，这个参数用来控制健康检查的周期 +// ## 健康检查 +// 连接断开的server会被暂时隔离而不会被负载均衡算法选中，brpc会定期连接被隔离的server，以检查他们是否恢复正常，间隔由参数-health_check_interval控制: // NOLINT +// | Name | Value | Description | Defined At | // NOLINT +// | ------------------------- | ----- | ---------------------------------------- | ----------------------- | // NOLINT +// | health_check_interval （R） | 3 | seconds between consecutive health-checkings | src/brpc/socket_map.cpp | // NOLINT +// 一旦server被连接上，它会恢复为可用状态。如果在隔离过程中，server从命名服务中删除了，brpc也会停止连接尝试。 // NOLINT +namespace brpc { + DECLARE_int32(health_check_interval); +} // namespace brpc + +namespace nebd { +namespace client { +FileClient &fileClient = FileClient::GetInstance(); + +int FileClient::Init(const char* confpath) { + // 从配置文件中获取 + using ::nebd::common::Configuration; + Configuration conf; + conf.SetConfigPath(confpath); + if (!conf.LoadConfig()) { + LOG(ERROR) << "load conf fail, conf path = " << confpath; + return -1; + } + + int ret = LoadConf(&conf); + if (ret != 0) { + LOG(ERROR) << "FileClient LoadConf fail."; + return -1; + } + + ret = lifeCycleManager_.Start(&conf); + if (ret != 0) { + LOG(ERROR) << "FileClient lifecycle manager start fail"; + return -1; + } + + std::string addr = lifeCycleManager_.GetPart2Addr(); + ret = InitChannel(addr); + if (ret != 0) { + LOG(ERROR) << "FileClient init channel fail, addr = " << addr; + return -1; + } + + lifeCycleManager_.StartHeartbeat(); + + LOG(INFO) << "FileClient init success."; + return 0; +} + +void FileClient::Uninit() { + lifeCycleManager_.Stop(); + LOG(INFO) << "FileClient uninit success."; +} + +int FileClient::Open(const char* filename) { + uint32_t retryCount = 0; + do { + nebd::client::QemuClientService_Stub stub(&channel_); + brpc::Controller cntl; + cntl.set_timeout_ms(rpcTimeoutMs_); + nebd::client::OpenFileRequest request; + nebd::client::OpenFileResponse response; + request.set_filename(filename); + stub.OpenFile(&cntl, &request, &response, NULL); + if (!cntl.Failed()) { + if (response.retcode() != nebd::client::RetCode::kOK) { + LOG(ERROR) << "Open fail, retCode = " + << response.retcode() + << ", filename = " << filename; + return -1; + } + std::string retMsg = response.retmsg(); + uint64_t fd = response.fd(); + LOG(INFO) << "Open ok." << "retMsg: " << retMsg + << ", filename = " << filename + << ", fd = " << fd; + return fd; + } else { + LOG(WARNING) << "Open fail, filename = " << filename + << ", cntl errorCode: " << cntl.ErrorCode() + << ", cntl error: " << cntl.ErrorText(); + } + retryCount++; + bthread_usleep(rpcRetryIntervalUs_); + } while (retryCount < rpcRetryTimes_); + + LOG(ERROR) << "Open fail, filename = " << filename + << ", retryCount = " << retryCount; + return -1; +} + +int FileClient::Close(int fd) { + uint32_t retryCount = 0; + do { + nebd::client::QemuClientService_Stub stub(&channel_); + brpc::Controller cntl; + cntl.set_timeout_ms(rpcTimeoutMs_); + nebd::client::CloseFileRequest request; + nebd::client::CloseFileResponse response; + request.set_fd(fd); + stub.CloseFile(&cntl, &request, &response, NULL); + if (!cntl.Failed()) { + if (response.retcode() != nebd::client::RetCode::kOK) { + LOG(ERROR) << "Close fail, retCode = " + << response.retcode() + << ", fd = " << fd; + return -1; + } + std::string retMsg = response.retmsg(); + LOG(INFO) << "Close ok." << "retMsg: " << retMsg + << ", fd = " << fd; + return 0; + } else { + LOG(ERROR) << "Close fail, fd = " << fd + << ", cntl errorCode: " << cntl.ErrorCode() + << ", cntl error: " << cntl.ErrorText(); + } + retryCount++; + bthread_usleep(rpcRetryIntervalUs_); + } while (retryCount < rpcRetryTimes_); + + LOG(ERROR) << "Close fail, fd = " << fd + << ", retryCount = " << retryCount; + return -1; +} + +int FileClient::Extend(int fd, int64_t newsize) { + uint32_t retryCount = 0; + do { + nebd::client::QemuClientService_Stub stub(&channel_); + brpc::Controller cntl; + cntl.set_timeout_ms(rpcTimeoutMs_); + nebd::client::ResizeRequest request; + nebd::client::ResizeResponse response; + request.set_fd(fd); + request.set_newsize(newsize); + stub.ResizeFile(&cntl, &request, &response, NULL); + if (!cntl.Failed()) { + if (response.retcode() != nebd::client::RetCode::kOK) { + LOG(ERROR) << "Extend fail, retCode = " + << response.retcode() + << ", fd = " << fd + << ", newsize = " << newsize; + return -1; + } + std::string retMsg = response.retmsg(); + LOG(INFO) << "Extend ok." << "retMsg: " << retMsg + << ", fd = " << fd + << ", newsize = " << newsize; + return 0; + } else { + LOG(ERROR) << "Extend fail, fd = " << fd + << ", newsize = " << newsize + << ", cntl errorCode: " << cntl.ErrorCode() + << ", cntl error: " << cntl.ErrorText(); + } + + retryCount++; + bthread_usleep(rpcRetryIntervalUs_); + } while (retryCount < rpcRetryTimes_); + + LOG(ERROR) << "Extend fail, fd = " << fd + << ", newsize = " << newsize + << ", retryCount = " << retryCount; + return -1; +} + +int64_t FileClient::StatFile(int fd) { + uint32_t retryCount = 0; + do { + nebd::client::QemuClientService_Stub stub(&channel_); + brpc::Controller cntl; + cntl.set_timeout_ms(rpcTimeoutMs_); + nebd::client::StatFileRequest request; + nebd::client::StatFileResponse response; + request.set_fd(fd); + stub.StatFile(&cntl, &request, &response, NULL); + if (!cntl.Failed()) { + if (response.retcode() != nebd::client::RetCode::kOK) { + LOG(ERROR) << "StatFile fail, retCode = " + << response.retcode() + << ", fd = " << fd; + return -1; + } + std::string retMsg = response.retmsg(); + uint64_t size = response.size(); + LOG(INFO) << "StatFile ok." << "retMsg: " << retMsg + << ", fd = " << fd + << ", size = " << size; + return size; + } else { + LOG(ERROR) << "StatFile fail, fd = " << fd + << ", cntl errorCode: " << cntl.ErrorCode() + << ", cntl error: " << cntl.ErrorText(); + } + + retryCount++; + bthread_usleep(rpcRetryIntervalUs_); + } while (retryCount < rpcRetryTimes_); + + LOG(ERROR) << "StatFile fail, fd = " << fd + << ", retryCount = " << retryCount; + return -1; +} + +int FileClient::Discard(int fd, ClientAioContext* aioctx) { + DVLOG(6) << "Discard start, fd = " << fd + << ", offset = " << aioctx->offset + << ", length = " << aioctx->length; + nebd::client::QemuClientService_Stub stub(&channel_); + nebd::client::DiscardRequest request; + request.set_fd(fd); + request.set_offset(aioctx->offset); + request.set_size(aioctx->length); + DiscardDone *discardDone = new DiscardDone(fd, aioctx, rpcTimeoutMs_, + rpcRetryIntervalUs_, rpcRetryMaxIntervalUs_, + rpcHostDownRetryIntervalUs_); + stub.Discard(&discardDone->cntl_, &request, + &discardDone->response_, discardDone); + return 0; +} + +int FileClient::AioRead(int fd, ClientAioContext* aioctx) { + DVLOG(6) << "AioRead start, fd = " << fd + << ", offset = " << aioctx->offset + << ", length = " << aioctx->length; + nebd::client::QemuClientService_Stub stub(&channel_); + nebd::client::ReadRequest request; + request.set_fd(fd); + request.set_offset(aioctx->offset); + request.set_size(aioctx->length); + ReadDone *readDone = new ReadDone(fd, aioctx, rpcTimeoutMs_, + rpcRetryIntervalUs_, rpcRetryMaxIntervalUs_, + rpcHostDownRetryIntervalUs_); + stub.Read(&readDone->cntl_, &request, &readDone->response_, readDone); + return 0; +} + +int FileClient::AioWrite(int fd, ClientAioContext* aioctx) { + DVLOG(6) << "AioWrite start, fd = " << fd + << ", offset = " << aioctx->offset + << ", length = " << aioctx->length; + nebd::client::QemuClientService_Stub stub(&channel_); + nebd::client::WriteRequest request; + request.set_fd(fd); + request.set_offset(aioctx->offset); + request.set_size(aioctx->length); + WriteDone *writeDone = new WriteDone(fd, aioctx, rpcTimeoutMs_, + rpcRetryIntervalUs_, rpcRetryMaxIntervalUs_, + rpcHostDownRetryIntervalUs_); + writeDone->cntl_.request_attachment().append(aioctx->buf, aioctx->length); + stub.Write(&writeDone->cntl_, &request, &writeDone->response_, writeDone); + return 0; +} + +int FileClient::Flush(int fd, ClientAioContext* aioctx) { + DVLOG(6) << "Flush start, fd = " << fd; + nebd::client::QemuClientService_Stub stub(&channel_); + nebd::client::FlushRequest request; + request.set_fd(fd); + FlushDone *flushDone = new FlushDone(fd, aioctx, rpcTimeoutMs_, + rpcRetryIntervalUs_, rpcRetryMaxIntervalUs_, + rpcHostDownRetryIntervalUs_); + stub.Flush(&flushDone->cntl_, &request, &flushDone->response_, flushDone); + return 0; +} + +int64_t FileClient::GetInfo(int fd) { + uint32_t retryCount = 0; + do { + nebd::client::QemuClientService_Stub stub(&channel_); + brpc::Controller cntl; + cntl.set_timeout_ms(rpcTimeoutMs_); + nebd::client::GetInfoRequest request; + nebd::client::GetInfoResponse response; + request.set_fd(fd); + stub.GetInfo(&cntl, &request, &response, NULL); + if (!cntl.Failed()) { + if (response.retcode() != nebd::client::RetCode::kOK) { + LOG(ERROR) << "GetInfo fail, retCode = " + << response.retcode() + << ", fd = " << fd; + return -1; + } + std::string retMsg = response.retmsg(); + uint64_t size = response.objsize(); + LOG(INFO) << "GetInfo ok." << "retMsg: " << retMsg + << ", fd = " << fd + << ", size = " << size; + return size; + } else { + LOG(ERROR) << "GetInfo fail, fd = " << fd + << ", cntl errorCode: " << cntl.ErrorCode() + << ", cntl error: " << cntl.ErrorText(); + } + + retryCount++; + bthread_usleep(rpcRetryIntervalUs_); + } while (retryCount < rpcRetryTimes_); + + LOG(ERROR) << "GetInfo fail, fd = " << fd + << ", retryCount = " << retryCount; + return -1; +} + +int FileClient::InvalidCache(int fd) { + uint32_t retryCount = 0; + do { + nebd::client::QemuClientService_Stub stub(&channel_); + brpc::Controller cntl; + cntl.set_timeout_ms(rpcTimeoutMs_); + nebd::client::InvalidateCacheRequest request; + nebd::client::InvalidateCacheResponse response; + request.set_fd(fd); + stub.InvalidateCache(&cntl, &request, &response, NULL); + if (!cntl.Failed()) { + if (response.retcode() != nebd::client::RetCode::kOK) { + LOG(ERROR) << "InvalidateCache fail, retCode = " + << response.retcode() + << ", fd = " << fd; + return -1; + } + std::string retMsg = response.retmsg(); + LOG(INFO) << "InvalidateCache ok." << "retMsg: " << retMsg + << ", fd = " << fd; + return 0; + } else { + LOG(ERROR) << "InvalidateCache fail, fd = " << fd + << ", cntl errorCode: " << cntl.ErrorCode() + << ", cntl error: " << cntl.ErrorText(); + } + + retryCount++; + bthread_usleep(rpcRetryIntervalUs_); + } while (retryCount < rpcRetryTimes_); + + LOG(ERROR) << "InvalidateCache fail, fd = " << fd + << ", retryCount = " << retryCount; + return -1; +} + +int FileClient::LoadConf(common::Configuration *conf) { + if (!conf->GetUInt32Value("rpcRetryTimes", &rpcRetryTimes_)) { + LOG(ERROR) << "get rpcRetryTimes fail."; + return -1; + } + + if (!conf->GetUInt32Value("rpcRetryIntervalUs", &rpcRetryIntervalUs_)) { + LOG(ERROR) << "get rpcRetryIntervalUs fail."; + return -1; + } + + if (!conf->GetUInt32Value("rpcHostDownRetryIntervalUs", + &rpcHostDownRetryIntervalUs_)) { + LOG(ERROR) << "get rpcHostDownRetryIntervalUs fail."; + return -1; + } + + if (!conf->GetUInt32Value("rpcRetryMaxIntervalUs", + &rpcRetryMaxIntervalUs_)) { + LOG(ERROR) << "get rpcRetryMaxIntervalUs fail."; + return -1; + } + + if (!conf->GetUInt32Value("rpcTimeoutMs", &rpcTimeoutMs_)) { + LOG(ERROR) << "get rpcTimeoutMs fail."; + return -1; + } + + if (!conf->GetUInt32Value("rpcHealthCheckIntervalS", + &rpcHealthCheckIntervalS_)) { + LOG(ERROR) << "get rpcHealthCheckIntervalS fail."; + return -1; + } + + if (!conf->GetUInt32Value("aioRpcFailLogInterval", + &aioRpcFailLogInterval_)) { + LOG(ERROR) << "get aioRpcFailLogInterval fail."; + return -1; + } + + LOG(INFO) << "FileClient load conf success."; + + return 0; +} + +int FileClient::InitChannel(const std::string& addr) { + brpc::FLAGS_health_check_interval = rpcHealthCheckIntervalS_; + if (channel_.Init(addr.c_str(), nullptr) != 0) { + LOG(ERROR) << "Init channel failed!"; + return -1; + } + + return 0; +} + +uint32_t AsyncRequestDone::GetRpcRetryIntervalUs(uint32_t retryCount) { + if (retryCount == 0) { + return rpcRetryIntervalUs_; + } + + // EHOSTDOWN: 找不到可用的server。 + // server可能停止服务了，也可能正在退出中(返回了ELOGOFF) + if (cntl_.ErrorCode() == EHOSTDOWN) { + return rpcHostDownRetryIntervalUs_; + } + + uint64_t tempRetryInteval = (uint64_t)rpcRetryIntervalUs_ * retryCount; + return tempRetryInteval > rpcRetryMaxIntervalUs_ ? rpcRetryMaxIntervalUs_ + : tempRetryInteval; +} + +void ReadDone::Run() { + std::unique_ptr self_guard(this); + + if (cntl_.Failed()) { + LOG_IF(WARNING, aioctx_->retryCount % aioRpcFailLogInterval_ == 0) + << "read rpc fail, fd = " << fd_ + << ", offset = " << aioctx_->offset + << ", length = " << aioctx_->length + << ", cntl errorCode: " << cntl_.ErrorCode() + << ", cntl error: " << cntl_.ErrorText() + << ", retryCount = " << aioctx_->retryCount; + aioctx_->retryCount++; + bthread_usleep(GetRpcRetryIntervalUs(aioctx_->retryCount)); + fileClient.AioRead(fd_, aioctx_); + } else { + if (nebd::client::RetCode::kOK == response_.retcode()) { + DVLOG(6) << "read success, fd = " << fd_ + << ", offset = " << aioctx_->offset + << ", length = " << aioctx_->length; + memcpy(aioctx_->buf, + cntl_.response_attachment().to_string().c_str(), + cntl_.response_attachment().size()); + aioctx_->ret = 0; + aioctx_->cb(aioctx_); + } else { + LOG(ERROR) << "read fail, fd = " << fd_ + << ", offset = " << aioctx_->offset + << ", length = " << aioctx_->length + << ", retCode = " << response_.retcode(); + aioctx_->ret = -1; + aioctx_->cb(aioctx_); + } + } +} + +void WriteDone::Run() { + std::unique_ptr self_guard(this); + + if (cntl_.Failed()) { + LOG_IF(WARNING, aioctx_->retryCount % aioRpcFailLogInterval_ == 0) + << "write rpc fail, fd = " << fd_ + << ", offset = " << aioctx_->offset + << ", length = " << aioctx_->length + << ", cntl errorCode: " << cntl_.ErrorCode() + << ", cntl error: " << cntl_.ErrorText() + << ", retryCount = " << aioctx_->retryCount; + aioctx_->retryCount++; + bthread_usleep(GetRpcRetryIntervalUs(aioctx_->retryCount)); + fileClient.AioWrite(fd_, aioctx_); + } else { + if (nebd::client::RetCode::kOK == response_.retcode()) { + DVLOG(6) << "write success, fd = " << fd_ + << ", offset = " << aioctx_->offset + << ", length = " << aioctx_->length; + aioctx_->ret = 0; + aioctx_->cb(aioctx_); + } else { + LOG(ERROR) << "write fail, fd = " << fd_ + << ", offset = " << aioctx_->offset + << ", length = " << aioctx_->length + << ", retCode = " << response_.retcode(); + aioctx_->ret = -1; + aioctx_->cb(aioctx_); + } + } +} + +void DiscardDone::Run() { + std::unique_ptr self_guard(this); + + if (cntl_.Failed()) { + LOG_IF(WARNING, aioctx_->retryCount % aioRpcFailLogInterval_ == 0) + << "discard rpc fail, fd = " << fd_ + << ", offset = " << aioctx_->offset + << ", length = " << aioctx_->length + << ", cntl errorCode: " << cntl_.ErrorCode() + << ", cntl error: " << cntl_.ErrorText() + << ", retryCount = " << aioctx_->retryCount; + aioctx_->retryCount++; + bthread_usleep(GetRpcRetryIntervalUs(aioctx_->retryCount)); + fileClient.Discard(fd_, aioctx_); + } else { + if (nebd::client::RetCode::kOK == response_.retcode()) { + DVLOG(6) << "discard success, fd = " << fd_ + << ", offset = " << aioctx_->offset + << ", length = " << aioctx_->length; + aioctx_->ret = 0; + aioctx_->cb(aioctx_); + } else { + LOG(ERROR) << "discard fail, fd = " << fd_ + << ", offset = " << aioctx_->offset + << ", length = " << aioctx_->length + << ", retCode = " << response_.retcode(); + aioctx_->ret = -1; + aioctx_->cb(aioctx_); + } + } +} + +void FlushDone::Run() { + std::unique_ptr self_guard(this); + + if (cntl_.Failed()) { + LOG_IF(WARNING, aioctx_->retryCount % aioRpcFailLogInterval_ == 0) + << "flush rpc fail, fd = " << fd_ + << ", cntl errorCode: " << cntl_.ErrorCode() + << ", cntl error: " << cntl_.ErrorText() + << ", retryCount = " << aioctx_->retryCount; + aioctx_->retryCount++; + bthread_usleep(GetRpcRetryIntervalUs(aioctx_->retryCount)); + fileClient.Flush(fd_, aioctx_); + } else { + if (nebd::client::RetCode::kOK == response_.retcode()) { + DVLOG(6) << "flush success, fd = " << fd_; + aioctx_->ret = 0; + aioctx_->cb(aioctx_); + } else { + LOG(ERROR) << "flush fail, fd = " << fd_ + << ", retCode = " << response_.retcode(); + aioctx_->ret = -1; + aioctx_->cb(aioctx_); + } + } +} +} // namespace client +} // namespace nebd diff --git a/src/part1/nebd_client.h b/src/part1/nebd_client.h new file mode 100644 index 0000000000..4d1ea1c3fe --- /dev/null +++ b/src/part1/nebd_client.h @@ -0,0 +1,232 @@ +/* + * Project: nebd + * File Created: 2019-10-08 + * Author: hzchenwei7 + * Copyright (c) 2018 NetEase + */ + +#ifndef SRC_PART1_NEBD_CLIENT_H_ +#define SRC_PART1_NEBD_CLIENT_H_ + +#include +#include +#include "src/part1/libnebd.h" +#include "src/common/configuration.h" +#include "src/part1/nebd_lifecycle.h" +#include "src/common/client.pb.h" + +namespace nebd { +namespace client { + +class FileClient { + public: + static FileClient &GetInstance() { + static FileClient fileClient; + return fileClient; + } + ~FileClient() = default; + /** + * @brief 初始化nebd，仅在第一次调用的时候真正执行初始化逻辑 + * @param none + * @return 成功返回0，失败返回-1 + */ + int Init(const char* confpath); + /** + * @brief 反初始化nebd + * @param none + * @return 成功返回0，失败返回-1 + */ + void Uninit(); + /** + * @brief open文件 + * @param filename：文件名 + * @return 成功返回文件fd，失败返回错误码 + */ + int Open(const char* filename); + /** + * @brief close文件 + * @param fd：文件的fd + * @return 成功返回0，失败返回错误码 + */ + int Close(int fd); + /** + * @brief resize文件 + * @param fd：文件的fd + * size：调整后的文件size + * @return 成功返回0，失败返回错误码 + */ + int Extend(int fd, int64_t newsize); + /** + * @brief 获取文件size + * @param fd：文件的fd + * @return 成功返回文件size，失败返回错误码 + */ + int64_t StatFile(int fd); + /** + * @brief discard文件，异步函数 + * @param fd：文件的fd + * context：异步请求的上下文，包含请求所需的信息以及回调 + * @return 成功返回0，失败返回错误码 + */ + int Discard(int fd, ClientAioContext* aioctx); + /** + * @brief 读文件，异步函数 + * @param fd：文件的fd + * context：异步请求的上下文，包含请求所需的信息以及回调 + * @return 成功返回0，失败返回错误码 + */ + int AioRead(int fd, ClientAioContext* aioctx); + /** + * @brief 写文件，异步函数 + * @param fd：文件的fd + * context：异步请求的上下文，包含请求所需的信息以及回调 + * @return 成功返回0，失败返回错误码 + */ + int AioWrite(int fd, ClientAioContext* aioctx); + /** + * @brief flush文件，异步函数 + * @param fd：文件的fd + * context：异步请求的上下文，包含请求所需的信息以及回调 + * @return 成功返回0，失败返回错误码 + */ + int Flush(int fd, ClientAioContext* aioctx); + /** + * @brief 获取文件info + * @param fd：文件的fd + * @return 成功返回文件对象size，失败返回错误码 + */ + int64_t GetInfo(int fd); + /** + * @brief 刷新cache，等所有异步请求返回 + * @param fd：文件的fd + * @return 成功返回0，失败返回错误码 + */ + int InvalidCache(int fd); + + // for test + int InitChannel(const std::string& addr); + int LoadConf(common::Configuration *conf); + + private: + // int LoadConf(common::Configuration *conf); + // int InitChannel(const std::string& addr); + + private: + brpc::Channel channel_; + // 同步rpc请求的最大重试次数 + uint32_t rpcRetryTimes_; + // 同步rpc请求的重试间隔； + // 也用来计算异步rpc请求的重试间隔，也是异步rpc请求重试的最小间隔 + uint32_t rpcRetryIntervalUs_; + // 当异步请求rpc返回EHOSTDOWN错误时的重试时间 + uint32_t rpcHostDownRetryIntervalUs_; + // 用来计算rpc请求的重试间隔，也是异步rpc请求重试的最大间隔 + uint32_t rpcRetryMaxIntervalUs_; + // rpc请求的超时时间 + uint32_t rpcTimeoutMs_; + // brpc的健康检查周期时间 + uint32_t rpcHealthCheckIntervalS_; + // 异步IO操作RPC失败日志打印间隔，每隔多少次打印一次 + uint32_t aioRpcFailLogInterval_; + // 管理生命周期的服务 + nebd::client::LifeCycleManager lifeCycleManager_; +}; + +extern FileClient &fileClient; + +class AsyncRequestDone: public google::protobuf::Closure { + public: + AsyncRequestDone(int fd, ClientAioContext* aioctx, + uint32_t rpcTimeoutMs, + uint32_t rpcRetryIntervalUs, + uint32_t rpcRetryMaxIntervalUs, + uint32_t rpcHostDownRetryIntervalUs): + fd_(fd), + aioctx_(aioctx), + rpcTimeoutMs_(rpcTimeoutMs), + rpcRetryIntervalUs_(rpcRetryIntervalUs), + rpcRetryMaxIntervalUs_(rpcRetryMaxIntervalUs), + rpcHostDownRetryIntervalUs_(rpcHostDownRetryIntervalUs) { + cntl_.set_timeout_ms(rpcTimeoutMs); + } + + /** + * @brief 计算rpc的重试间隔，重试时间根据重试次数线性增加， + * 最小不小于rpcRetryIntervalUs_，最大不大于rpcRetryMaxIntervalUs_ + * @param retryCount: 重试次数 + * @return 返回计算出来的重试间隔 + */ + uint32_t GetRpcRetryIntervalUs(uint32_t retryCount); + + virtual void Run() {} + + // 请求的fd + int fd_; + // 请求的上下文 + ClientAioContext* aioctx_; + // 用来设置rpc请求的超时时间 + uint32_t rpcTimeoutMs_; + // 用来计算rpc请求的重试间隔，也是异步rpc请求重试的最小间隔 + uint32_t rpcRetryIntervalUs_; + // 用来计算rpc请求的重试间隔，也是异步rpc请求重试的最大间隔 + uint32_t rpcRetryMaxIntervalUs_; + // 当异步请求rpc返回EHOSTDOWN错误时的重试时间 + uint32_t rpcHostDownRetryIntervalUs_; + // 异步IO操作RPC失败日志打印间隔，每隔多少次打印一次 + uint32_t aioRpcFailLogInterval_; + // brpc异步请求的controller，保存下来避免预期之外的析构 + brpc::Controller cntl_; +}; + +class ReadDone: public AsyncRequestDone { + public: + ReadDone(int fd, ClientAioContext* aioctx, uint32_t rpcTimeoutMs, + uint32_t rpcRetryIntervalUs, uint32_t rpcRetryMaxIntervalUs, + uint32_t rpcHostDownRetryIntervalUs): + AsyncRequestDone(fd, aioctx, rpcTimeoutMs, + rpcRetryIntervalUs, rpcRetryMaxIntervalUs, + rpcHostDownRetryIntervalUs) {} + void Run(); + nebd::client::ReadResponse response_; +}; + +class WriteDone: public AsyncRequestDone { + public: + WriteDone(int fd, ClientAioContext* aioctx, uint32_t rpcTimeoutMs, + uint32_t rpcRetryIntervalUs, uint32_t rpcRetryMaxIntervalUs, + uint32_t rpcHostDownRetryIntervalUs): + AsyncRequestDone(fd, aioctx, rpcTimeoutMs, + rpcRetryIntervalUs, rpcRetryMaxIntervalUs, + rpcHostDownRetryIntervalUs) {} + void Run(); + nebd::client::WriteResponse response_; +}; + +class DiscardDone: public AsyncRequestDone { + public: + DiscardDone(int fd, ClientAioContext* aioctx, uint32_t rpcTimeoutMs, + uint32_t rpcRetryIntervalUs, uint32_t rpcRetryMaxIntervalUs, + uint32_t rpcHostDownRetryIntervalUs): + AsyncRequestDone(fd, aioctx, rpcTimeoutMs, + rpcRetryIntervalUs, rpcRetryMaxIntervalUs, + rpcHostDownRetryIntervalUs) {} + void Run(); + nebd::client::DiscardResponse response_; +}; + +class FlushDone: public AsyncRequestDone { + public: + FlushDone(int fd, ClientAioContext* aioctx, uint32_t rpcTimeoutMs, + uint32_t rpcRetryIntervalUs, uint32_t rpcRetryMaxIntervalUs, + uint32_t rpcHostDownRetryIntervalUs): + AsyncRequestDone(fd, aioctx, rpcTimeoutMs, + rpcRetryIntervalUs, rpcRetryMaxIntervalUs, + rpcHostDownRetryIntervalUs) {} + void Run(); + nebd::client::FlushResponse response_; +}; + +} // namespace client +} // namespace nebd + +#endif // SRC_PART1_NEBD_CLIENT_H_ diff --git a/src/part1/nebd_lifecycle.cpp b/src/part1/nebd_lifecycle.cpp new file mode 100644 index 0000000000..1f870a12d0 --- /dev/null +++ b/src/part1/nebd_lifecycle.cpp @@ -0,0 +1,733 @@ +/* + * Project: nebd + * File Created: 2019-10-08 + * Author: hzchenwei7 + * Copyright (c) 2018 NetEase + */ + +#include "src/part1/nebd_lifecycle.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/common/configuration.h" +#include "src/part1/libnebd_file.h" + +namespace nebd { +namespace client { + +int LifeCycleManager::Start(common::Configuration *conf) { + int ret = LoadConf(conf); + if (ret != 0) { + LOG(ERROR) << "Init4Qemu, LoadConf fail"; + return -1; + } + + // 根据本进程pid，查询qemu的uuid + pidPart1_ = getpid(); + LOG(INFO) << "qemu pid = " << pidPart1_; + + char uuid[kUuidLength + 1] = {0}; + ret = getUuid(pidPart1_, lifeCycleOptions_.qemuProcName.c_str(), uuid); + if (ret != 0) { + LOG(ERROR) << "get qemu uuid fail"; + return -1; + } + + qemuUUID_ = uuid; + LOG(INFO) << "get qemu uuid = " << qemuUUID_; + + // open文件锁 + int fd = open(lifeCycleOptions_.lockFile.c_str(), O_RDONLY); + if (fd < 0) { + LOG(ERROR) << "open lock file fail, file = " + << lifeCycleOptions_.lockFile; + return -1; + } + + // 检查part2是否运行 + bool isPart2Alive = false; + + uint32_t retryCount = 0; + do { + // 加文件锁 + int lock = flock(fd, LOCK_EX | LOCK_NB); + if (lock < 0) { + LOG(ERROR) << "lock file fail, file = " + << lifeCycleOptions_.lockFile; + close(fd); + return -1; + } + + ret = CheckProcAlive(uuid, lifeCycleOptions_.part2ProcName.c_str(), + &isPart2Alive, &pidPart2_); + if (ret < 0) { + int release = flock(fd, LOCK_UN); + if (release < 0) { + LOG(ERROR) << "unlock file fail, file = " + << lifeCycleOptions_.lockFile; + } + LOG(ERROR) << "check part2 alive fail"; + close(fd); + return -1; + } + + if (isPart2Alive) { + // 如果part2存活，先不释放文件锁，继续持有锁去查询port信息 + break; + } + + // 清理初始化文件，拉起part2 + CleanMetadataFile(); + StartPart2(); + + // 释放文件锁 + int release = flock(fd, LOCK_UN); + if (release < 0) { + LOG(ERROR) << "unlock file fail, file = " + << lifeCycleOptions_.lockFile; + close(fd); + return -1; + } + + retryCount++; + + usleep(lifeCycleOptions_.part2StartRetryIntervalUs); + } while (!isPart2Alive + && retryCount < lifeCycleOptions_.part2StartRetryTimes); + + // 拉起part2多次失败 + if (!isPart2Alive) { + LOG(ERROR) << "start part2 fail."; + close(fd); + return -1; + } + + // 读取端口，端口读取成功后，启动心跳服务 + uint32_t port = 0; + retryCount = 0; + while (1) { + ret = GetPortFromPart2(&port); + + // 前面在查询part2是否存活时已经加锁，这里读取port之后释放文件锁 + int release = flock(fd, LOCK_UN); + if (release < 0) { + LOG(ERROR) << "unlock file fail, file = " + << lifeCycleOptions_.lockFile; + close(fd); + return -1; + } + + if (ret == 0) { + // 找到port + LOG(INFO) << "get port success, port = " << port; + break; + } + + retryCount++; + LOG(WARNING) << "get port fail, retry, retryCount = " << retryCount; + if (retryCount == lifeCycleOptions_.portGetRetryTimes) { + // 达到重试次数，未找到port + LOG(ERROR) << "get port fail reach portGetRetryTimes = " + << lifeCycleOptions_.portGetRetryTimes; + break; + } + + usleep(lifeCycleOptions_.portGetRetryIntervalUs); + + // 为下一次循环，加文件锁 + int lock = flock(fd, LOCK_EX | LOCK_NB); + if (lock < 0) { + LOG(ERROR) << "lock file fail, file = " + << lifeCycleOptions_.lockFile; + close(fd); + return -1; + } + } + + // 未成功读取port信息，kill part2 + if (port == 0) { + int lock = flock(fd, LOCK_EX | LOCK_NB); + if (lock < 0) { + LOG(ERROR) << "lock file fail, file = " + << lifeCycleOptions_.lockFile; + close(fd); + return -1; + } + + KillPart2(); + + int release = flock(fd, LOCK_UN); + if (release < 0) { + LOG(ERROR) << "unlock file fail, file = " + << lifeCycleOptions_.lockFile; + close(fd); + return -1; + } + + LOG(ERROR) << "get port fail, kill part2."; + close(fd); + return -1; + } + + part2Port_ = port; + + close(fd); + return 0; +} + +void LifeCycleManager::Stop() { + LOG(INFO) << "LifeCycleManager STOP start."; + shouldHeartbeatStop_ = true; + if (heartbeatThread_ != nullptr && heartbeatThread_->joinable()) { + heartbeatThread_->join(); + delete heartbeatThread_; + heartbeatThread_ = nullptr; + } + + isHeartbeatThreadStart_ = false; + + KillPart2(); + + pidPart2_ = -1; + pidPart1_ = -1; + part2Port_ = 0; + + LOG(INFO) << "LifeCycleManager STOPED."; + + return; +} + +int LifeCycleManager::LoadConf(common::Configuration *conf) { + if (!conf->GetStringValue("part2ProcName", + &lifeCycleOptions_.part2ProcName)) { + LOG(ERROR) << "get part2ProcName fail."; + return -1; + } + + if (!conf->GetStringValue("part2ProcPath", + &lifeCycleOptions_.part2ProcPath)) { + LOG(ERROR) << "get part2ProcPath fail."; + return -1; + } + + if (!conf->GetStringValue("part2Addr", &lifeCycleOptions_.part2Addr)) { + LOG(ERROR) << "get part2Addr fail."; + return -1; + } + + if (!conf->GetUInt32Value("part2KillCheckRetryTimes", + &lifeCycleOptions_.part2KillCheckRetryTimes)) { + LOG(ERROR) << "get part2KillCheckRetryTimes fail."; + return -1; + } + + if (!conf->GetUInt32Value("part2KillCheckRetryIntervalUs", + &lifeCycleOptions_.part2KillCheckRetryIntervalUs)) { + LOG(ERROR) << "get part2KillCheckRetryIntervalUs fail."; + return -1; + } + + if (!conf->GetStringValue("qemuProcName", + &lifeCycleOptions_.qemuProcName)) { + LOG(ERROR) << "get qemuProcName fail."; + return -1; + } + + if (!conf->GetStringValue("lockFile", &lifeCycleOptions_.lockFile)) { + LOG(ERROR) << "get lockFile fail."; + return -1; + } + + if (!conf->GetStringValue("metadataPrefix", + &lifeCycleOptions_.metadataPrefix)) { + LOG(ERROR) << "get metadataPrefix fail."; + return -1; + } + + if (!conf->GetUInt32Value("part2StartRetryTimes", + &lifeCycleOptions_.part2StartRetryTimes)) { + LOG(ERROR) << "get part2StartRetryTimes fail."; + return -1; + } + + if (!conf->GetUInt32Value("connectibleCheckTimes", + &lifeCycleOptions_.connectibleCheckTimes)) { + LOG(ERROR) << "get connectibleCheckTimes fail."; + return -1; + } + + if (!conf->GetUInt32Value("connectibleCheckIntervalUs", + &lifeCycleOptions_.connectibleCheckIntervalUs)) { + LOG(ERROR) << "get connectibleCheckIntervalUs fail."; + return -1; + } + + if (!conf->GetUInt32Value("portGetRetryTimes", + &lifeCycleOptions_.portGetRetryTimes)) { + LOG(ERROR) << "get portGetRetryTimes fail."; + return -1; + } + + if (!conf->GetUInt32Value("portGetRetryIntervalUs", + &lifeCycleOptions_.portGetRetryIntervalUs)) { + LOG(ERROR) << "get portGetRetryIntervalUs fail."; + return -1; + } + + if (!conf->GetUInt32Value("heartbeatIntervalUs", + &lifeCycleOptions_.heartbeatIntervalUs)) { + LOG(ERROR) << "get heartbeatIntervalUs fail."; + return -1; + } + + LOG(INFO) << "load conf success"; + return 0; +} + +char *LifeCycleManager::SkipSpace(char *in, int *offset, int len) { + while (*offset < len) { + if (isspace(in[*offset]) || in[*offset] == '\0') { + ++*offset; + continue; + } + break; + } + return in + *offset; +} + +char *LifeCycleManager::SkipNoSpace(char *in, int *offset, int len) { + while (*offset < len) { + if (!isspace(in[*offset]) && in[*offset] != '\0') { + ++*offset; + continue; + } + break; + } + return in + *offset; +} + +int LifeCycleManager::getUuid(pid_t pid, const char *procName, char *uuid) { + char path[kCmdLinePathBufLength]; + snprintf(path, sizeof(path), "/proc/%d/cmdline", pid); + int fd = open(path, O_RDONLY); + if (fd < 0) { + return -1; + } + + char line[kCmdLineBufLength]; + int nbytesread = read(fd, line, kCmdLineBufLength); + close(fd); + + if (strncmp(basename(line), procName, strlen(procName)) != 0) { + return -1; + } + + char *p; + int i = strlen(line); + bool find = false; + for (; i < nbytesread; ) { + p = SkipSpace(line, &i, kCmdLineBufLength); + if (strncmp(p, "-uuid", 5) == 0) { + i += strlen(p); + p = SkipSpace(line, &i, kCmdLineBufLength); + strncpy(uuid, p, kUuidLength); + find = true; + break; + } + p = SkipNoSpace(line, &i, kCmdLineBufLength); + } + + if (find) { + return 0; + } else { + return -1; + } +} + +// 检查uuid和procName对应的进程在不在 +// 返回值-1，查找失败；返回值0，查找成功，返回是否查到及pid +int LifeCycleManager::CheckProcAlive(const char* uuid, const char* procName, + bool *procExist, pid_t *pid) { + DIR* dir = opendir("/proc"); + if (dir == NULL) { + LOG(ERROR) << "opendir(/proc) fail."; + return -1; + } + + // 遍历/proc目录下的所有进程 + struct dirent* ent; + while ((ent = readdir(dir))) { + if (!isdigit(*ent->d_name)) { + continue; + } + + pid_t tempPid = strtol(ent->d_name, NULL, 10); + + char procUuid[kUuidLength + 1] = {0}; + int ret = getUuid(tempPid, procName, procUuid); + if (ret < 0) { + continue; + } + + if (strncmp(procUuid, uuid, strlen(uuid)) == 0) { + closedir(dir); + *pid = tempPid; + *procExist = true; + DVLOG(6) << "CheckProcAlive alive, uuid = " << uuid + << ", procName = " << procName + << ", pid = " << tempPid; + return 0; + } + } + + // 遍历 /proc目录下的所有进程，没有找到uuid和procName对应的进程 + closedir(dir); + *procExist = false; + LOG(INFO) << "CheckProcAlive not alive, uuid = " << uuid + << ", procName = " << procName; + return 0; +} + +int LifeCycleManager::GetPortFromPart2(uint32_t *port) { + // 读取json文件 + std::string metadataFile = lifeCycleOptions_.metadataPrefix + qemuUUID_; + char jsonStr[kMetaFileBufLength] = {}; + int fd = open(metadataFile.c_str(), O_RDONLY); + if (fd < 0) { + LOG(ERROR) << "meta file open failed, " << metadataFile + << ", errno = " << errno; + return -1; + } + int ret = read(fd, jsonStr, kMetaFileBufLength); + if (ret <= 0) { + LOG(ERROR) << "meta file read failed, " << metadataFile; + close(fd); + return -1; + } + close(fd); + + // 解析json文件中的port信息 + Json::Reader reader; + Json::Value value; + if (!reader.parse(jsonStr, value)) { + return false; + } + + if (!value["port"].isNull() && value["port"].isString()) { + *port = stoi(value["port"].asString()); + LOG(INFO) << "read port from file, port = " << *port; + return 0; + } else { + return -1; + } + + return 0; +} + +bool LifeCycleManager::IsPart2Connectible() { + butil::EndPoint ep; + std::string addr = GetPart2Addr(); + str2endpoint(addr.c_str(), &ep); + int fd = butil::tcp_connect(ep, NULL); + if (fd == -1) { + LOG(WARNING) << "part2 is NOT connectible, addr = " << addr; + return false; + } + close(fd); + return true; +} + +bool LifeCycleManager::IsPart2ConnectibleWithRetry(uint32_t retryTimes, + uint32_t retryIntervalUs) { + uint32_t retryCount = 0; + while (retryCount < retryTimes) { + if (IsPart2Connectible()) { + return true; + } + + retryCount++; + LOG(WARNING) << "part2 is not connectible, retry next time" + << ", unconnectCount = " << retryCount; + usleep(retryIntervalUs); + } + + return false; +} + +void LifeCycleManager::CleanMetadataFile() { + std::string metadataFile = lifeCycleOptions_.metadataPrefix + qemuUUID_; + if (access(metadataFile.c_str(), F_OK) == 0) { + LOG(INFO) << "CleanMetadataFile, meta file = " << metadataFile; + if (unlink(metadataFile.c_str()) != 0) { + LOG(WARNING) << "delete metadata file fail, file = " + << metadataFile; + } + } + + return; +} + +void LifeCycleManager::StartPart2() { + std::string cmd = lifeCycleOptions_.part2ProcPath + " -uuid " + qemuUUID_; + LOG(INFO) << "StartPart2, rum cmd: " << cmd; + system(cmd.c_str()); + + return; +} + +int LifeCycleManager::KillPart2() { + if (pidPart2_ == -1) { + return 0; + } + + std::string cmd = "kill " + std::to_string(pidPart2_); + LOG(INFO) << "KillPart2, run cmd = " << cmd; + system(cmd.c_str()); + + // 检查part2是否已经kill + uint32_t retryCount = 0; + bool procExist = true; + pid_t tempPid = 0; + while (retryCount < lifeCycleOptions_.part2KillCheckRetryTimes) { + int ret = CheckProcAlive(qemuUUID_.c_str(), + lifeCycleOptions_.part2ProcName.c_str(), + &procExist, &tempPid); + if (ret == 0 && (!procExist || tempPid != pidPart2_)) { + pidPart2_ = -1; + return 0; + } + + retryCount++; + LOG(INFO) << "kill part2, part2 still alive, retry" + << ", retryCount = " << retryCount; + usleep(lifeCycleOptions_.part2KillCheckRetryIntervalUs); + } + + // 多次检查失败，执行kill -9 + if (procExist) { + std::string cmd = "kill -9 " + std::to_string(pidPart2_); + LOG(INFO) << "KillPart2, run cmd = " << cmd; + system(cmd.c_str()); + } + + int ret = CheckProcAlive(qemuUUID_.c_str(), + lifeCycleOptions_.part2ProcName.c_str(), + &procExist, &tempPid); + if (ret != 0 || (procExist && tempPid == pidPart2_)) { + return -1; + } + + pidPart2_ = -1; + return 0; +} + +int LifeCycleManager::GetPortWithRetry(int fd, uint32_t *port) { + uint32_t retryCount = 0; + while (retryCount < lifeCycleOptions_.portGetRetryTimes) { + // 加文件锁 + int lock = flock(fd, LOCK_EX | LOCK_NB); + if (lock < 0) { + LOG(WARNING) << "lock file fail, file = " + << lifeCycleOptions_.lockFile; + retryCount++; + usleep(lifeCycleOptions_.portGetRetryIntervalUs); + continue; + } + + // 重新读取并更新port信息 + uint32_t tempPort = 0; + int ret = GetPortFromPart2(&tempPort); + + // 文件锁解锁 + int release = flock(fd, LOCK_UN); + LOG_IF(WARNING, release < 0) << "unlock file fail, file = " + << lifeCycleOptions_.lockFile; + + // 获取port信息失败，下次重试 + if (ret != 0 || tempPort == 0) { + retryCount++; + LOG(WARNING) << "update port from part2 fail, retry" + << "retryCount" << retryCount; + usleep(lifeCycleOptions_.portGetRetryIntervalUs); + continue; + } + + // 成功获取port信息 + LOG(INFO) << "get port with retry success, port = " << tempPort; + *port = tempPort; + return 0; + } + + // 尝试多次，未能成功获取port信息，返回失败 + return -1; +} + +int LifeCycleManager::StartPart2AndGetPortRetry(int fd, uint32_t *port) { + uint32_t retryCount = 0; + do { + StartPart2(); + bool isPart2Alive = false; + int ret = CheckProcAlive(qemuUUID_.c_str(), + lifeCycleOptions_.part2ProcName.c_str(), + &isPart2Alive, &pidPart2_); + if (ret < 0) { + LOG(ERROR) << "check part2 alive fail"; + return -1; + } + + if (isPart2Alive) { + uint32_t tempPort; + int ret = GetPortWithRetry(fd, &tempPort); + if (ret != 0) { + LOG(ERROR) << "get port fail"; + return -1; + } + *port = tempPort; + return 0; + } + + retryCount++; + LOG(INFO) << "start part2 but check alive fail, retry, retryCount = " + << retryCount; + + usleep(lifeCycleOptions_.part2StartRetryIntervalUs); + } while (retryCount < lifeCycleOptions_.part2StartRetryTimes); + + return -1; +} + +void LifeCycleManager::HeartbeatThreadFunc() { + LOG(INFO) << "heartbeat check thread start."; + + // open文件锁 + int fd = open(lifeCycleOptions_.lockFile.c_str(), O_RDONLY); + if (fd < 0) { + LOG(ERROR) << "open lock file fail, file = " + << lifeCycleOptions_.lockFile; + return; + } + + while (!shouldHeartbeatStop_) { + // 检查part2是否运行 + bool isPart2Alive = false; + pid_t tempPid = 0; + int ret = CheckProcAlive(qemuUUID_.c_str(), + lifeCycleOptions_.part2ProcName.c_str(), + &isPart2Alive, &tempPid); + if (ret < 0) { + LOG(ERROR) << "kill part1."; + close(fd); + exit(1); + return; + } + + // 检查part2是否存活，如果未存活，需要拉起。拉起多次失败之后，退出part1。 + if (!isPart2Alive) { + uint32_t port; + int ret = StartPart2AndGetPortRetry(fd, &port); + if (ret < 0) { + LOG(ERROR) << "kill part1."; + close(fd); + exit(1); + return; + } + + // 拉起成功之后，读取并更新端口信息。 + part2Port_ = port; + continue; + } + + // 检查端口的连通性，如果联通失败，kill part2并重新拉起 + if (!IsPart2ConnectibleWithRetry( + lifeCycleOptions_.connectibleCheckTimes, + lifeCycleOptions_.connectibleCheckIntervalUs)) { + LOG(WARNING) << "part2 is not connectible, " + << "reach connectibleCheckTimes, unconnectCount = " + << lifeCycleOptions_.connectibleCheckTimes; + + // 连续多次未联通，kill part2 + int ret = KillPart2(); + if (ret != 0) { + LOG(WARNING) << "part2 is not connectible, kill part2 fail"; + usleep(lifeCycleOptions_.heartbeatIntervalUs); + continue; + } + + uint32_t port; + ret = StartPart2AndGetPortRetry(fd, &port); + if (ret < 0) { + LOG(ERROR) << "kill part1."; + close(fd); + exit(1); + return; + } + + if (!IsPart2ConnectibleWithRetry( + lifeCycleOptions_.connectibleCheckTimes, + lifeCycleOptions_.connectibleCheckIntervalUs)) { + KillPart2(); + LOG(ERROR) << "kill part2, kill part1."; + close(fd); + exit(1); + return; + } + + part2Port_ = port; + continue; + } + + usleep(lifeCycleOptions_.heartbeatIntervalUs); + } + + LOG(INFO) << "stop heartbeat func."; + + close(fd); + return; +} + +void LifeCycleManager::StartHeartbeat() { + if (!isHeartbeatThreadStart_) { + shouldHeartbeatStop_ = false; + heartbeatThread_ = new std::thread( + &LifeCycleManager::HeartbeatThreadFunc, + this); + isHeartbeatThreadStart_ = true; + } + + return; +} + +std::string LifeCycleManager::GetPart2Addr() { + return lifeCycleOptions_.part2Addr + ":" + std::to_string(part2Port_); +} + +bool LifeCycleManager::IsPart2Alive() { + if (qemuUUID_ == "") { + return false; + } + + bool isPart2Alive = false; + pid_t pidPart2 = -1; + int ret = CheckProcAlive(qemuUUID_.c_str(), + lifeCycleOptions_.part2ProcName.c_str(), + &isPart2Alive, &pidPart2); + if (ret < 0) { + LOG(ERROR) << "check part2 alive fail"; + return false; + } + + return isPart2Alive; +} + +bool LifeCycleManager::IsHeartbeatThreadStart() { + return isHeartbeatThreadStart_; +} + +} // namespace client +} // namespace nebd diff --git a/src/part1/nebd_lifecycle.h b/src/part1/nebd_lifecycle.h new file mode 100644 index 0000000000..0109e0e2e1 --- /dev/null +++ b/src/part1/nebd_lifecycle.h @@ -0,0 +1,198 @@ +/* + * Project: nebd + * File Created: 2019-10-08 + * Author: hzchenwei7 + * Copyright (c) 2018 NetEase + */ + +#ifndef SRC_PART1_NEBD_LIFECYCLE_H_ +#define SRC_PART1_NEBD_LIFECYCLE_H_ + +#include +#include +#include // NOLINT +#include "src/common/configuration.h" + +const uint32_t kCmdLinePathBufLength = 32; +const uint32_t kCmdLineBufLength = 2048; +const uint32_t kUuidLength = 36; +const uint32_t kMetaFileBufLength = 2048; + +namespace nebd { +namespace client { +typedef struct LifeCycleOptions { + std::string part2ProcName; + std::string part2ProcPath; + std::string part2Addr; + uint32_t part2KillCheckRetryTimes; + uint32_t part2KillCheckRetryIntervalUs; + std::string qemuProcName; + std::string lockFile; + std::string metadataPrefix; + uint32_t part2StartRetryTimes; + uint32_t part2StartRetryIntervalUs; + uint32_t connectibleCheckTimes; + uint32_t connectibleCheckIntervalUs; + uint32_t portGetRetryTimes; + uint32_t portGetRetryIntervalUs; + uint32_t heartbeatIntervalUs; + uint32_t rpcRetryTimes; + uint32_t rpcRetryIntervalUs; + uint32_t rpcRetryMaxIntervalUs; + uint32_t rpcTimeoutMs; +} LifeCycleOptions; + +// 负责生命周期管理服务 +class LifeCycleManager { + public: + LifeCycleManager() { + part2Port_ = 0; + pidPart1_ = -1; + pidPart2_ = -1; + qemuUUID_ = ""; + isHeartbeatThreadStart_ = false; + heartbeatThread_ = nullptr; + shouldHeartbeatStop_ = true; + } + ~LifeCycleManager() {} + + /** + * @brief 启动生命周期管理服务，检测part2是否拉起，拉起part2并获取端口信息 + * @param conf: 配置信息 + * @return 执行成功返回0，失败返回-1 + */ + int Start(common::Configuration *conf); + /** + * @brief 停止声明周期管理服务，退出心跳线程，kill part2 + * @param void + * @return void + */ + void Stop(); + /** + * @brief 获取part2的地址 + * @param void + * @return 返回获取的part2的地址 + */ + std::string GetPart2Addr(); + /** + * @brief 启动心跳服务 + * @param void + * @return void + */ + void StartHeartbeat(); + // for test + /** + * @brief part2进程是否存活 + * @param void + * @return 如果存活返回true，如果不存活返回false + */ + bool IsPart2Alive(); + /** + * @brief 心跳服务线程是否启动 + * @param void + * @return 如果启动返回true，如果未启动返回false + */ + bool IsHeartbeatThreadStart(); + /** + * @brief kill part2服务 + * @param void + * @return 执行成功返回0，失败返回-1 + */ + int KillPart2(); + + private: + int LoadConf(common::Configuration *conf); + char *SkipSpace(char *in, int *offset, int len); + char *SkipNoSpace(char *in, int *offset, int len); + /** + * @brief 根据pid和进程名，从cmdline中解析uuid + * @param pid: 进程id + * @param procName: 进程名字 + * @param uuid: 返回从cmdline中获取的uuid + * @return 执行成功返回0，失败返回-1 + */ + int getUuid(pid_t pid, const char *procName, char *uuid); + /** + * @brief 检查指定uuid和进程名的进程是否活着，如果进程活着，pid返回进程id + * @param uuid: + * @param procName: 进程名字 + * @param[out] procExist: 进程是否存活 + * @param[out] pid：进程id + * @return 执行成功返回0，失败返回-1 + */ + int CheckProcAlive(const char* uuid, const char* procName, + bool *procExist, pid_t *pid); + /** + * @brief 从metafile中读取port信息 + * @param[out] port：返回读到的port信息 + * @return 执行成功返回0，失败返回-1 + */ + int GetPortFromPart2(uint32_t *port); + /** + * @brief 从metafile中读取port信息，如果读取失败多次读取 + * @param fd：读取port时需要加文件锁，文件锁的fd + * @param[out] port：返回读到的port信息 + * @return 执行成功返回0，失败返回-1 + */ + int GetPortWithRetry(int fd, uint32_t *port); + /** + * @brief 检测part2是否可联通 + * @param void + * @return 可联通返回true，否则返回false + */ + bool IsPart2Connectible(); + /** + * @brief 检测part2是否可联通，重复多次检查不可联通才返回失败 + * @param retryTimes: 重试次数 + * @param retryIntervalUs：重试间隔 + * @return 可联通返回true，否则返回false + */ + bool IsPart2ConnectibleWithRetry(uint32_t retryTimes, + uint32_t retryIntervalUs); + /** + * @brief 清理part1和part2共用的metadate文件 + * @param void + * @return void + */ + void CleanMetadataFile(); + /** + * @brief 拉起part2服务，只负责拉起，不负责检测是否拉起成功 + * @param void + * @return void + */ + void StartPart2(); + /** + * @brief 拉起part2服务，拉起成功后，读取port信息 + * @param fd：读取port时需要加文件锁，文件锁的fd + * @param[out] port: 返回读取到的port信息 + * @return 执行成功返回0，失败返回-1 + */ + int StartPart2AndGetPortRetry(int fd, uint32_t *port); + /** + * @brief 心跳线程的执行函数 + * @param void + * @return void + */ + void HeartbeatThreadFunc(); + + private: + // 从配置文件中读取到的声明周期管理所需要的配置字段 + LifeCycleOptions lifeCycleOptions_; + // part2的服务的port + uint32_t part2Port_; + // part1的pid + pid_t pidPart1_; + // part2的pid + pid_t pidPart2_; + // qemu的UUID + std::string qemuUUID_; + // heartbeat线程是否启动 + std::atomic isHeartbeatThreadStart_; + // heartbeat的线程 + std::thread *heartbeatThread_; + // 控制heartbeat线程退出的标志位 + std::atomic shouldHeartbeatStop_; +}; +} // namespace client +} // namespace nebd +#endif // SRC_PART1_NEBD_LIFECYCLE_H_ diff --git a/tests/common/configuration_test.cpp b/tests/common/configuration_test.cpp new file mode 100644 index 0000000000..83d158970f --- /dev/null +++ b/tests/common/configuration_test.cpp @@ -0,0 +1,284 @@ +/* + * Copyright (C) 2018 NetEase Inc. All rights reserved. + * Project: Curve + * + * History: + * 2018/11/23 Wenyu Zhou Initial version + */ + +#include +#include + +#include +#include +#include +#include + +#include "src/common/configuration.h" + +namespace nebd { +namespace common { + +class ConfigurationTest : public ::testing::Test { + public: + void SetUp() { + std::string confItem; + + confFile_ = "curve.conf.test"; + std::ofstream cFile(confFile_); + ASSERT_TRUE(cFile.is_open()); + + confItem = "test.str1=teststring\n"; + cFile << confItem; + + confItem = "test.int1=12345\n"; + cFile << confItem; + + confItem = "test.int2=-2345\n"; + cFile << confItem; + + confItem = "test.int3=0\n"; + cFile << confItem; + + confItem = "test.bool1=0\n"; + cFile << confItem; + + confItem = "test.bool2=1\n"; + cFile << confItem; + + confItem = "test.bool3=false\n"; + cFile << confItem; + + confItem = "test.bool4=true\n"; + cFile << confItem; + + confItem = "test.bool5=no\n"; + cFile << confItem; + + confItem = "test.bool6=yes\n"; + cFile << confItem; + + confItem = "test.double1=3.1415926\n"; + cFile << confItem; + + confItem = "test.double2=1\n"; + cFile << confItem; + + confItem = "test.double3=1.0\n"; + cFile << confItem; + + confItem = "test.double4=0.1\n"; + cFile << confItem; + } + + void TearDown() { + ASSERT_EQ(0, unlink(confFile_.c_str())); + } + + std::string confFile_; +}; + +TEST_F(ConfigurationTest, SetAndGetConfigPath) { + Configuration conf; + + conf.SetConfigPath(confFile_); + ASSERT_EQ(conf.GetConfigPath(), confFile_); +} + +TEST_F(ConfigurationTest, LoadNonExistConfigFile) { + bool ret; + std::string confFile = "curve.conf.test.nonexist"; + Configuration conf; + + conf.SetConfigPath(confFile); + ret = conf.LoadConfig(); + ASSERT_EQ(ret, false); +} + +TEST_F(ConfigurationTest, LoadNormalConfigFile) { + bool ret; + Configuration conf; + + conf.SetConfigPath(confFile_); + ret = conf.LoadConfig(); + ASSERT_EQ(ret, true); +} + +TEST_F(ConfigurationTest, DumpConfig) { + Configuration conf; + + conf.SetConfigPath(confFile_); + // not implemented yet, assert null returned + ASSERT_EQ(conf.DumpConfig(), ""); +} + +TEST_F(ConfigurationTest, ListConfig) { + Configuration conf; + + conf.SetConfigPath(confFile_); + int ret = conf.LoadConfig(); + ASSERT_EQ(ret, true); + std::map configs; + configs = conf.ListConfig(); + ASSERT_NE(0, configs.size()); + // 抽几个key来校验以下 + ASSERT_EQ(configs["test.int1"], "12345"); + ASSERT_EQ(configs["test.bool1"], "0"); + // 如果key不存在，返回为空 + ASSERT_EQ(configs["xxx"], ""); +} + +TEST_F(ConfigurationTest, SaveConfig) { + bool ret; + Configuration conf; + + conf.SetConfigPath(confFile_); + ret = conf.SaveConfig(); + // not implemented yet, assert false + ASSERT_EQ(ret, false); +} + +TEST_F(ConfigurationTest, GetSetValue) { + bool ret; + Configuration conf; + + conf.SetConfigPath(confFile_); + ret = conf.LoadConfig(); + ASSERT_EQ(ret, true); + + ASSERT_EQ(conf.GetValue("test.str1"), "teststring"); + ASSERT_EQ(conf.GetValue("test.int1"), "12345"); + ASSERT_EQ(conf.GetValue("test.bool1"), "0"); + ASSERT_EQ(conf.GetValue("test.str.nonexist"), ""); + + conf.SetValue("test.str1", "teststring2"); + ASSERT_EQ(conf.GetValue("test.str1"), "teststring2"); + std::string out; + ASSERT_FALSE(conf.GetValue("no.exist", &out)); + conf.SetValue("put.in", "put.in"); + ASSERT_TRUE(conf.GetValue("put.in", &out)); + ASSERT_EQ("put.in", out); +} + +TEST_F(ConfigurationTest, GetSetStringValue) { + bool ret; + Configuration conf; + + conf.SetConfigPath(confFile_); + ret = conf.LoadConfig(); + ASSERT_EQ(ret, true); + + ASSERT_EQ(conf.GetStringValue("test.str1"), "teststring"); + ASSERT_EQ(conf.GetStringValue("test.int1"), "12345"); + ASSERT_EQ(conf.GetStringValue("test.bool1"), "0"); + ASSERT_EQ(conf.GetStringValue("test.str.nonexist"), ""); + + conf.SetStringValue("test.str1", "teststring2"); + ASSERT_EQ(conf.GetStringValue("test.str1"), "teststring2"); + + std::string out; + ASSERT_FALSE(conf.GetStringValue("no.exist", &out)); + conf.SetStringValue("put.in", "put.in"); + ASSERT_TRUE(conf.GetStringValue("put.in", &out)); + ASSERT_EQ("put.in", out); +} + +TEST_F(ConfigurationTest, GetSetIntValue) { + bool ret; + Configuration conf; + + conf.SetConfigPath(confFile_); + ret = conf.LoadConfig(); + ASSERT_EQ(ret, true); + + ASSERT_EQ(conf.GetIntValue("test.int1"), 12345); + ASSERT_EQ(conf.GetIntValue("test.int2"), -2345); + ASSERT_EQ(conf.GetIntValue("test.int3"), 0); + ASSERT_EQ(conf.GetIntValue("test.int.nonexist"), 0); + + conf.SetIntValue("test.int1", 123); + ASSERT_EQ(conf.GetIntValue("test.int1"), 123); + + int out; + ASSERT_FALSE(conf.GetIntValue("no.exist", &out)); + conf.SetIntValue("no.exist", 1); + ASSERT_TRUE(conf.GetIntValue("no.exist", &out)); + ASSERT_EQ(1, out); + + uint32_t outu32; + ASSERT_FALSE(conf.GetUInt32Value("no.exist.u32", &outu32)); + conf.SetIntValue("no.exist.u32", 2); + ASSERT_TRUE(conf.GetUInt32Value("no.exist.u32", &outu32)); + ASSERT_EQ(2, outu32); + + uint64_t outu64; + ASSERT_FALSE(conf.GetUInt64Value("no.exist.u64", &outu64)); + conf.SetIntValue("no.exist.u64", 3); + ASSERT_TRUE(conf.GetUInt64Value("no.exist.u64", &outu64)); + ASSERT_EQ(3, outu64); +} + +TEST_F(ConfigurationTest, GetSetBoolValue) { + bool ret; + Configuration conf; + + conf.SetConfigPath(confFile_); + ret = conf.LoadConfig(); + ASSERT_EQ(ret, true); + + ASSERT_EQ(conf.GetBoolValue("test.bool1"), false); + ASSERT_EQ(conf.GetBoolValue("test.bool2"), true); + ASSERT_EQ(conf.GetBoolValue("test.bool3"), false); + ASSERT_EQ(conf.GetBoolValue("test.bool4"), true); + ASSERT_EQ(conf.GetBoolValue("test.bool5"), false); + ASSERT_EQ(conf.GetBoolValue("test.bool6"), true); + ASSERT_EQ(conf.GetBoolValue("test.bool.nonexist"), false); + + conf.SetBoolValue("test.bool1", true); + ASSERT_EQ(conf.GetBoolValue("test.bool1"), true); + + bool out; + ASSERT_FALSE(conf.GetBoolValue("no.exist", &out)); + conf.SetIntValue("no.exist", false); + ASSERT_TRUE(conf.GetBoolValue("no.exist", &out)); + ASSERT_FALSE(out); +} + +TEST_F(ConfigurationTest, GetSetDoubleAndFloatValue) { + bool ret; + Configuration conf; + + conf.SetConfigPath(confFile_); + ret = conf.LoadConfig(); + ASSERT_EQ(ret, true); + + ASSERT_EQ(conf.GetDoubleValue("test.double1"), 3.1415926); + ASSERT_EQ(conf.GetDoubleValue("test.double2"), 1); + ASSERT_EQ(conf.GetDoubleValue("test.double3"), 1.0); + ASSERT_EQ(conf.GetDoubleValue("test.double4"), 0.1); + ASSERT_EQ(conf.GetFloatValue("test.double4"), 0.1f); + + conf.SetDoubleValue("test.double1", 100.0); + ASSERT_EQ(conf.GetDoubleValue("test.double1"), 100.0); + + double out; + float outf; + ASSERT_FALSE(conf.GetDoubleValue("no.exist", &out)); + ASSERT_FALSE(conf.GetFloatValue("no.exist", &outf)); + conf.SetDoubleValue("no.exist", 0.009); + ASSERT_TRUE(conf.GetDoubleValue("no.exist", &out)); + ASSERT_TRUE(conf.GetFloatValue("no.exist", &outf)); + ASSERT_EQ(0.009, out); + ASSERT_EQ(0.009f, outf); +} + +} // namespace common +} // namespace nebd + +int main(int argc, char ** argv) { + ::testing::InitGoogleTest(&argc, argv); + ::testing::InitGoogleMock(&argc, argv); + int ret = RUN_ALL_TESTS(); + + return ret; +} diff --git a/tests/part1/client_server.cpp b/tests/part1/client_server.cpp new file mode 100644 index 0000000000..1650bbd56b --- /dev/null +++ b/tests/part1/client_server.cpp @@ -0,0 +1,89 @@ +/* + * Project: nebd + * Created Date: 2019-08-12 + * Author: hzchenwei7 + * Copyright (c) 2018 netease + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "tests/part1/fake_client_service.h" + +DEFINE_string(uuid, "12345678-1234-1234-1234-123456789012", "uuid"); +DEFINE_string(port, "6667", "port"); + +namespace nebd { +namespace client { +int client_main(int argc, char **argv) { + google::ParseCommandLineFlags(&argc, &argv, false); + + // add service + brpc::Server server; + ClientService clientService; + int ret = server.AddService(&clientService, + brpc::SERVER_DOESNT_OWN_SERVICE); + if (ret != 0) { + LOG(FATAL) << "add clientService error"; + return -1; + } + + // start rpc server + brpc::ServerOptions option; + option.idle_timeout_sec = -1; + std::string listenAddr = "127.0.0.1:" + FLAGS_port; + ret = server.Start(listenAddr.c_str(), &option); + if (ret != 0) { + LOG(FATAL) << "start brpc server error"; + return -1; + } + + // 写meta + std::string filePath = "file_" + FLAGS_uuid; + Json::Value root; + root["port"] = FLAGS_port; + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_SYNC, 0644); + if (fd < 0) { + LOG(ERROR) << "open metadata fail, filePath = " << filePath + << ", errno = " << errno; + return -1; + } + ret = write(fd, root.toStyledString().c_str(), + root.toStyledString().size()); + if (ret < root.toStyledString().size()) { + LOG(ERROR) << "write matadata fail, filePath = " << filePath; + close(fd); + return -1; + } + close(fd); + LOG(INFO) << "write metadata filePath = " << filePath + << ", content : " << root.toStyledString().c_str(); + + server.RunUntilAskedToQuit(); + return 0; +} +} // namespace client +} // namespace nebd + +int main(int argc, char **argv) { + // google::InitGoogleLogging(argv[0]); + // 使进程为守护进程 + if (daemon(1, 0) < 0) { + LOG(ERROR) << "create daemon error."; + return -1; + } + + //初始化日志模块 + logging::LoggingSettings log; + log.logging_dest = logging::LOG_TO_FILE; + log.log_file = "tests/part1/client_server.log"; + logging::InitLogging(log); + + LOG(INFO) << "start client server."; + return nebd::client::client_main(argc, argv); +} diff --git a/tests/part1/fake_client_service.cpp b/tests/part1/fake_client_service.cpp new file mode 100644 index 0000000000..ec58d63689 --- /dev/null +++ b/tests/part1/fake_client_service.cpp @@ -0,0 +1,180 @@ +/* + * Project: nebd + * Created Date: 2019-08-12 + * Author: hzchenwei7 + * Copyright (c) 2018 netease + */ +#include "tests/part1/fake_client_service.h" + +namespace nebd { +namespace client { + +uint64_t filesize = 50*1024*1024; // 50MB +char *buf = reinterpret_cast(malloc(filesize)); + +void ClientService::OpenFile(::google::protobuf::RpcController* controller, + const ::nebd::client::OpenFileRequest* request, + ::nebd::client::OpenFileResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + brpc::Controller* cntl = static_cast(controller); + + LOG(INFO) << "logid = " << cntl->log_id() << "OpenFile."; + if (buf == nullptr) { + response->set_retcode(RetCode::kNoOK); + response->set_retmsg("OpenFile FAIL"); + return; + } + + response->set_retcode(RetCode::kOK); + response->set_retmsg("OpenFile OK"); + response->set_fd(1); + + return; +} + +void ClientService::CloseFile(::google::protobuf::RpcController* controller, + const ::nebd::client::CloseFileRequest* request, + ::nebd::client::CloseFileResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + brpc::Controller* cntl = static_cast(controller); + + LOG(INFO) << "logid = " << cntl->log_id() << "CloseFile."; + + response->set_retcode(RetCode::kOK); + response->set_retmsg("CloseFile OK"); + + return; +} + +void ClientService::Read(::google::protobuf::RpcController* controller, + const ::nebd::client::ReadRequest* request, + ::nebd::client::ReadResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + brpc::Controller* cntl = static_cast(controller); + + LOG(INFO) << "logid = " << cntl->log_id() << "Read."; + + cntl->response_attachment().append(buf + request->offset(), + request->size()); + response->set_retcode(RetCode::kOK); + response->set_retmsg("Read OK"); + + return; +} + +void ClientService::Write(::google::protobuf::RpcController* controller, + const ::nebd::client::WriteRequest* request, + ::nebd::client::WriteResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + brpc::Controller* cntl = static_cast(controller); + + LOG(INFO) << "logid = " << cntl->log_id() << "Write."; + + memcpy(buf + request->offset(), + cntl->request_attachment().to_string().c_str(), request->size()); + response->set_retcode(RetCode::kOK); + response->set_retmsg("Write OK"); + + return; +} + +void ClientService::Discard(::google::protobuf::RpcController* controller, + const ::nebd::client::DiscardRequest* request, + ::nebd::client::DiscardResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + brpc::Controller* cntl = static_cast(controller); + + LOG(INFO) << "logid = " << cntl->log_id() << "Discard."; + + response->set_retcode(RetCode::kOK); + response->set_retmsg("Discard OK"); + + return; +} + +void ClientService::StatFile(::google::protobuf::RpcController* controller, + const ::nebd::client::StatFileRequest* request, + ::nebd::client::StatFileResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + brpc::Controller* cntl = static_cast(controller); + + LOG(INFO) << "logid = " << cntl->log_id() << "StatFile."; + + response->set_retcode(RetCode::kOK); + response->set_retmsg("StatFile OK"); + response->set_size(filesize); + + return; +} + +void ClientService::ResizeFile(::google::protobuf::RpcController* controller, + const ::nebd::client::ResizeRequest* request, + ::nebd::client::ResizeResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + brpc::Controller* cntl = static_cast(controller); + + LOG(INFO) << "logid = " << cntl->log_id() << "ResizeFile."; + + response->set_retcode(RetCode::kOK); + response->set_retmsg("ResizeFile OK"); + + return; +} + +void ClientService::Flush(::google::protobuf::RpcController* controller, + const ::nebd::client::FlushRequest* request, + ::nebd::client::FlushResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + brpc::Controller* cntl = static_cast(controller); + + LOG(INFO) << "logid = " << cntl->log_id() << "Flush."; + + response->set_retcode(RetCode::kOK); + response->set_retmsg("Flush OK"); + + return; +} + +void ClientService::GetInfo(::google::protobuf::RpcController* controller, + const ::nebd::client::GetInfoRequest* request, + ::nebd::client::GetInfoResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + brpc::Controller* cntl = static_cast(controller); + + LOG(INFO) << "logid = " << cntl->log_id() << "GetInfo."; + + response->set_retcode(RetCode::kOK); + response->set_retmsg("GetInfo OK"); + response->set_objsize(filesize); + + return; +} + +void ClientService::InvalidateCache( + ::google::protobuf::RpcController* controller, + const ::nebd::client::InvalidateCacheRequest* request, + ::nebd::client::InvalidateCacheResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + brpc::Controller* cntl = static_cast(controller); + + LOG(INFO) << "logid = " << cntl->log_id() << "InvalidateCache."; + + response->set_retcode(RetCode::kOK); + response->set_retmsg("InvalidateCache OK"); + + return; +} + +} // namespace client +} // namespace nebd + diff --git a/tests/part1/fake_client_service.h b/tests/part1/fake_client_service.h new file mode 100644 index 0000000000..bb6078407c --- /dev/null +++ b/tests/part1/fake_client_service.h @@ -0,0 +1,77 @@ +/* + * Project: nebd + * Created Date: 2019-08-12 + * Author: hzchenwei7 + * Copyright (c) 2018 netease + */ + +#ifndef TESTS_PART1_FAKE_CLIENT_SERVICE_H_ +#define TESTS_PART1_FAKE_CLIENT_SERVICE_H_ + +#include +#include +#include +#include "src/common/client.pb.h" + +namespace nebd { +namespace client { + +class ClientService: public QemuClientService { + public: + ClientService() {} + + virtual ~ClientService() {} + + void OpenFile(::google::protobuf::RpcController* controller, + const ::nebd::client::OpenFileRequest* request, + ::nebd::client::OpenFileResponse* response, + ::google::protobuf::Closure* done) override; + + void CloseFile(::google::protobuf::RpcController* controller, + const ::nebd::client::CloseFileRequest* request, + ::nebd::client::CloseFileResponse* response, + ::google::protobuf::Closure* done) override; + + void Read(::google::protobuf::RpcController* controller, + const ::nebd::client::ReadRequest* request, + ::nebd::client::ReadResponse* response, + ::google::protobuf::Closure* done) override; + + void Write(::google::protobuf::RpcController* controller, + const ::nebd::client::WriteRequest* request, + ::nebd::client::WriteResponse* response, + ::google::protobuf::Closure* done) override; + + void Discard(::google::protobuf::RpcController* controller, + const ::nebd::client::DiscardRequest* request, + ::nebd::client::DiscardResponse* response, + ::google::protobuf::Closure* done) override; + + void StatFile(::google::protobuf::RpcController* controller, + const ::nebd::client::StatFileRequest* request, + ::nebd::client::StatFileResponse* response, + ::google::protobuf::Closure* done) override; + + void ResizeFile(::google::protobuf::RpcController* controller, + const ::nebd::client::ResizeRequest* request, + ::nebd::client::ResizeResponse* response, + ::google::protobuf::Closure* done) override; + + void Flush(::google::protobuf::RpcController* controller, + const ::nebd::client::FlushRequest* request, + ::nebd::client::FlushResponse* response, + ::google::protobuf::Closure* done) override; + + void GetInfo(::google::protobuf::RpcController* controller, + const ::nebd::client::GetInfoRequest* request, + ::nebd::client::GetInfoResponse* response, + ::google::protobuf::Closure* done) override; + + void InvalidateCache(::google::protobuf::RpcController* controller, + const ::nebd::client::InvalidateCacheRequest* request, + ::nebd::client::InvalidateCacheResponse* response, + ::google::protobuf::Closure* done) override; +}; +} // namespace client +} // namespace nebd +#endif // TESTS_PART1_FAKE_CLIENT_SERVICE_H_ diff --git a/tests/part1/lock.file b/tests/part1/lock.file new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/part1/mock_client_service.h b/tests/part1/mock_client_service.h new file mode 100644 index 0000000000..28c0d18959 --- /dev/null +++ b/tests/part1/mock_client_service.h @@ -0,0 +1,69 @@ +/* + * Project: nebd + * Created Date: 2019-10-11 + * Author: hzchenwei7 + * Copyright (c) 2018 netease + */ + +#ifndef TESTS_PART1_MOCK_CLIENT_SERVICE_H_ +#define TESTS_PART1_MOCK_CLIENT_SERVICE_H_ + +#include +#include +#include +#include "src/common/client.pb.h" + +namespace nebd { +namespace client { + +class MockQemuClientService : public QemuClientService { + public: + MockQemuClientService() : QemuClientService() {} + ~MockQemuClientService() = default; + + MOCK_METHOD4(OpenFile, void(::google::protobuf::RpcController* controller, + const ::nebd::client::OpenFileRequest* request, + ::nebd::client::OpenFileResponse* response, + ::google::protobuf::Closure* done)); + MOCK_METHOD4(CloseFile, void(::google::protobuf::RpcController* controller, + const ::nebd::client::CloseFileRequest* request, + ::nebd::client::CloseFileResponse* response, + ::google::protobuf::Closure* done)); + MOCK_METHOD4(Read, void(::google::protobuf::RpcController* controller, + const ::nebd::client::ReadRequest* request, + ::nebd::client::ReadResponse* response, + ::google::protobuf::Closure* done)); + MOCK_METHOD4(Write, void(::google::protobuf::RpcController* controller, + const ::nebd::client::WriteRequest* request, + ::nebd::client::WriteResponse* response, + ::google::protobuf::Closure* done)); + MOCK_METHOD4(Discard, void(::google::protobuf::RpcController* controller, + const ::nebd::client::DiscardRequest* request, + ::nebd::client::DiscardResponse* response, + ::google::protobuf::Closure* done)); + MOCK_METHOD4(StatFile, void(::google::protobuf::RpcController* controller, + const ::nebd::client::StatFileRequest* request, + ::nebd::client::StatFileResponse* response, + ::google::protobuf::Closure* done)); + MOCK_METHOD4(ResizeFile, void(::google::protobuf::RpcController* controller, + const ::nebd::client::ResizeRequest* request, + ::nebd::client::ResizeResponse* response, + ::google::protobuf::Closure* done)); + MOCK_METHOD4(Flush, void(::google::protobuf::RpcController* controller, + const ::nebd::client::FlushRequest* request, + ::nebd::client::FlushResponse* response, + ::google::protobuf::Closure* done)); + MOCK_METHOD4(GetInfo, void(::google::protobuf::RpcController* controller, + const ::nebd::client::GetInfoRequest* request, + ::nebd::client::GetInfoResponse* response, + ::google::protobuf::Closure* done)); + MOCK_METHOD4(InvalidateCache, void( + ::google::protobuf::RpcController* controller, + const ::nebd::client::InvalidateCacheRequest* request, + ::nebd::client::InvalidateCacheResponse* response, + ::google::protobuf::Closure* done)); +}; +} // namespace client +} // namespace nebd + +#endif // TESTS_PART1_MOCK_CLIENT_SERVICE_H_ diff --git a/tests/part1/nebd.conf b/tests/part1/nebd.conf new file mode 100644 index 0000000000..3040353ce1 --- /dev/null +++ b/tests/part1/nebd.conf @@ -0,0 +1,45 @@ +# nebd part2的进程名字 +part2ProcName=client_server +# nebd part2的程序的启动路径，从该路径拉起part2 +part2ProcPath=build/bin/client_server +# nebd part2的ip地址 +part2Addr=127.0.0.1 +# kill part2服务之后，检查part2服务是否存活的重试次数 +part2KillCheckRetryTimes=5 +# kill part2服务之后，检查part2服务是否存活的重试间隔，单位Us +part2KillCheckRetryIntervalUs=100000 +# nebd part1的qemu进程的名字 +qemuProcName=test_part1 +# nebd part1和part2共用的文件锁文件的路径 +lockFile=tests/part1/lock.file +# nebd part1和part2共用的元数据文件的目录 +metadataPrefix=file_ +# 拉起part2服务的重试次数 +part2StartRetryTimes=5 +# 拉起part2服务的重试间隔，单位Us +part2StartRetryIntervalUs=100000 +# 检查part2的port是否联通的重试次数 +connectibleCheckTimes=5 +# 检查part2的port是否联通的重试间隔，单位Us +connectibleCheckIntervalUs=100000 +# 获取part2的port的重试次数 +portGetRetryTimes=5 +# 获取part2的port的重试间隔，单位Us +portGetRetryIntervalUs=100000 +# 心跳服务的检查间隔，单位Us +heartbeatIntervalUs=1000000 +# 非io的rpc请求的异常重试次数 +rpcRetryTimes=5 +# rpc请求的重试间隔，包括io请求和非io请求 +rpcRetryIntervalUs=200000 +# rpc请求的重试间隔，当io请求遇到server服务未启动的时候使用这个重试间隔， +# 这个值一般比rpcRetryIntervalUs更小，为了更快恢复io服务 +rpcHostDownRetryIntervalUs=200000 +# rpc请求的最大重试间隔 +rpcRetryMaxIntervalUs=6400000 +# rpc请求的超时时间，单位Ms +rpcTimeoutMs=5000 +# brpc的健康检查周期时间，单位s +rpcHealthCheckIntervalS=1 +# 异步IO操作RPC失败日志打印间隔，每隔多少次打印一次 +aioRpcFailLogInterval=10 \ No newline at end of file diff --git a/tests/part1/nebd_test.cpp b/tests/part1/nebd_test.cpp new file mode 100644 index 0000000000..e32ca63114 --- /dev/null +++ b/tests/part1/nebd_test.cpp @@ -0,0 +1,601 @@ +/* + * Project: nebd + * Created Date: 2019-08-12 + * Author: hzchenwei7 + * Copyright (c) 2018 netease + */ +#include +#include +#include +#include +#include +#include // NOLINT +#include // NOLINT +#include "tests/part1/mock_client_service.h" +#include "src/part1/libnebd.h" +#include "src/part1/libnebd_file.h" +#include "src/part1/nebd_client.h" +#include "src/part1/nebd_lifecycle.h" + +DEFINE_string(uuid, "12345678-1234-1234-1234-123456789012", "uuid"); + +#define confFilename "tests/part1/nebd.conf" +#define filename "test_file_name" +#define DEFAULT_FD 1 +#define BUFSIZE 512 +#define DEFAULT_FILEZIZE (50*1024*1024) +std::mutex mtx; +std::condition_variable condition; +bool callback; + +void LibAioCallBackFunc(struct ClientAioContext* context) { + ASSERT_EQ(context->ret, 0); + callback = true; + condition.notify_one(); + ASSERT_EQ(context->retryCount, 0); +} + +void LibAioFailCallBackFunc(struct ClientAioContext* context) { + ASSERT_EQ(context->ret, -1); + callback = true; + condition.notify_one(); + ASSERT_EQ(context->retryCount, 0); +} + + +class nebdClientTest: public ::testing::Test { + protected: + void SetUp() override { + callback = false; + system("sudo killall client_server"); + system("sudo mkdir -p /etc/nebd"); + system("sudo cp tests/part1/nebd.conf /etc/nebd/nebd-client.conf"); + ASSERT_EQ(0, nebd_lib_init()); + } + void TearDown() override { + nebd_lib_uninit(); + } +}; + +TEST_F(nebdClientTest, nebdCommonTest) { + // nebd_lib_open_test + int ret; + ret = nebd_lib_open(filename); + ASSERT_EQ(DEFAULT_FD, ret); + + // nebd_lib_pread_test + char buf[BUFSIZE] = {}; + ret = nebd_lib_pread(DEFAULT_FD, buf, 0, BUFSIZE); + ASSERT_EQ(-1, ret); + + // nebd_lib_pwrite_test + memset(buf, 'a', BUFSIZE); + ret = nebd_lib_pwrite(DEFAULT_FD, buf, 0, BUFSIZE); + ASSERT_EQ(-1, ret); + + // nebd_lib_aio_pread_pwrite_test + char writeBuf[BUFSIZE] = "test"; + char readBuf[BUFSIZE]; + ClientAioContext context; + context.buf = writeBuf; + context.offset = 0; + context.length = BUFSIZE; + context.ret = 0; + context.op = LIBAIO_OP_WRITE; + context.cb = LibAioCallBackFunc; + context.retryCount = 0; + + ret = nebd_lib_aio_pwrite(DEFAULT_FD, &context); + ASSERT_EQ(ret, 0); + + std::unique_lock lock{mtx}; + condition.wait(lock); + ASSERT_TRUE(callback); + + callback = false; + ret = nebd_lib_aio_pread(DEFAULT_FD, &context); + ASSERT_EQ(ret, 0); + + condition.wait(lock); + ASSERT_TRUE(callback); + + // nebd_lib_sync_test + ret = nebd_lib_sync(DEFAULT_FD); + ASSERT_EQ(0, ret); + + // nebd_lib_discard_test + context.buf = nullptr; + context.offset = 0; + context.length = BUFSIZE; + context.ret = 0; + context.op = LIBAIO_OP_DISCARD; + context.cb = LibAioCallBackFunc; + context.retryCount = 0; + + ret = nebd_lib_discard(DEFAULT_FD, &context); + ASSERT_EQ(ret, 0); + + condition.wait(lock); + ASSERT_TRUE(callback); + + // nebd_lib_filesize_test + int64_t filesize; + filesize = nebd_lib_filesize(DEFAULT_FD); + ASSERT_EQ(DEFAULT_FILEZIZE, filesize); + + // nebd_lib_resize_test + ret = nebd_lib_resize(DEFAULT_FD, DEFAULT_FILEZIZE); + ASSERT_EQ(0, ret); + + // nebd_lib_getinfo_test + ret = nebd_lib_getinfo(DEFAULT_FD); + ASSERT_EQ(DEFAULT_FILEZIZE, ret); + + // nebd_lib_flush_test + context.buf = nullptr; + context.offset = 0; + context.length = 0; + context.ret = 0; + context.op = LIBAIO_OP_FLUSH; + context.cb = LibAioCallBackFunc; + context.retryCount = 0; + + ret = nebd_lib_flush(DEFAULT_FD, &context); + ASSERT_EQ(ret, 0); + + condition.wait(lock); + ASSERT_TRUE(callback); + + // nebd_lib_getinfo_test + filesize = nebd_lib_getinfo(DEFAULT_FD); + ASSERT_EQ(DEFAULT_FILEZIZE, filesize); + + // nebd_lib_invalidcache_test + ret = nebd_lib_invalidcache(DEFAULT_FD); + ASSERT_EQ(0, ret); + + // nebd_lib_close_test + ret = nebd_lib_close(DEFAULT_FD); + ASSERT_EQ(0, ret); +} + +class nebdFileClientTest: public ::testing::Test { + protected: + void SetUp() override { + system("sudo killall client_server"); + system("sudo mkdir -p /etc/nebd"); + system("sudo cp tests/part1/nebd.conf /etc/nebd/nebd-client.conf"); + } + void TearDown() override { + } +}; + +TEST_F(nebdFileClientTest, common_test) { + nebd::client::FileClient fileClientTest; + ASSERT_EQ(fileClientTest.Init(confFilename), 0); + ASSERT_EQ(fileClientTest.Open(filename), DEFAULT_FD); + ASSERT_EQ(fileClientTest.Extend(DEFAULT_FD, DEFAULT_FILEZIZE), 0); + ASSERT_EQ(fileClientTest.StatFile(DEFAULT_FD), DEFAULT_FILEZIZE); + ASSERT_EQ(fileClientTest.GetInfo(DEFAULT_FD), DEFAULT_FILEZIZE); + ASSERT_EQ(fileClientTest.InvalidCache(DEFAULT_FD), 0); + + char writeBuf[BUFSIZE] = "test"; + char readBuf[BUFSIZE]; + ClientAioContext context; + context.buf = writeBuf; + context.offset = 0; + context.length = BUFSIZE; + context.ret = 0; + context.op = LIBAIO_OP_WRITE; + context.cb = LibAioCallBackFunc; + context.retryCount = 0; + + ASSERT_EQ(fileClientTest.AioRead(DEFAULT_FD, &context), 0); + std::unique_lock lock{mtx}; + condition.wait(lock); + + ASSERT_EQ(fileClientTest.AioWrite(DEFAULT_FD, &context), 0); + condition.wait(lock); + ASSERT_TRUE(callback); + + context.buf = nullptr; + context.offset = 0; + context.length = BUFSIZE; + context.ret = 0; + context.op = LIBAIO_OP_DISCARD; + context.cb = LibAioCallBackFunc; + context.retryCount = 0; + ASSERT_EQ(fileClientTest.Discard(DEFAULT_FD, &context), 0); + condition.wait(lock); + ASSERT_TRUE(callback); + + context.buf = nullptr; + context.offset = 0; + context.length = 0; + context.ret = 0; + context.op = LIBAIO_OP_FLUSH; + context.cb = LibAioCallBackFunc; + context.retryCount = 0; + ASSERT_EQ(fileClientTest.Flush(DEFAULT_FD, &context), 0); + condition.wait(lock); + ASSERT_TRUE(callback); + + ASSERT_EQ(fileClientTest.Close(DEFAULT_FD), 0); + fileClientTest.Uninit(); + return; +} + +TEST_F(nebdFileClientTest, no_rpc_server_test) { + nebd::client::FileClient fileClientTest; + ASSERT_EQ(fileClientTest.Init("wrongConfPath"), -1); + + nebd::common::Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_TRUE(conf.LoadConfig()); + + ASSERT_EQ(fileClientTest.LoadConf(&conf), 0); + + ASSERT_EQ(fileClientTest.InitChannel("127.0.0.1:6667"), 0); + ASSERT_EQ(fileClientTest.Open(filename), -1); + ASSERT_EQ(fileClientTest.Extend(DEFAULT_FD, DEFAULT_FILEZIZE), -1); + ASSERT_EQ(fileClientTest.StatFile(DEFAULT_FD), -1); + ASSERT_EQ(fileClientTest.GetInfo(DEFAULT_FD), -1); + ASSERT_EQ(fileClientTest.InvalidCache(DEFAULT_FD), -1); + + // char writeBuf[BUFSIZE] = "test"; + // char readBuf[BUFSIZE]; + // ClientAioContext context; + // context.buf = writeBuf; + // context.offset = 0; + // context.length = BUFSIZE; + // context.ret = 0; + // context.op = LIBAIO_OP_WRITE; + // context.cb = LibAioCallBackFunc; + // context.retryCount = 0; + + // ASSERT_EQ(fileClientTest.AioRead(DEFAULT_FD, &context), 0); + // std::unique_lock lock{mtx}; + // condition.wait(lock); + + // ASSERT_EQ(fileClientTest.AioWrite(DEFAULT_FD, &context), 0); + // condition.wait(lock); + // ASSERT_TRUE(callback); + + // context.buf = nullptr; + // context.offset = 0; + // context.length = BUFSIZE; + // context.ret = 0; + // context.op = LIBAIO_OP_DISCARD; + // context.cb = LibAioCallBackFunc; + // context.retryCount = 0; + // ASSERT_EQ(fileClientTest.Discard(DEFAULT_FD, &context), 0); + // condition.wait(lock); + // ASSERT_TRUE(callback); + + // context.buf = nullptr; + // context.offset = 0; + // context.length = 0; + // context.ret = 0; + // context.op = LIBAIO_OP_FLUSH; + // context.cb = LibAioCallBackFunc; + // context.retryCount = 0; + // ASSERT_EQ(fileClientTest.Flush(DEFAULT_FD, &context), 0); + // condition.wait(lock); + // ASSERT_TRUE(callback); + + ASSERT_EQ(fileClientTest.Close(DEFAULT_FD), -1); + fileClientTest.Uninit(); + return; +} + +static void OpenFileFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::OpenFileRequest* request, + ::nebd::client::OpenFileResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + // if (0 != gReadCntlFailedCode) { + // brpc::Controller *cntl = dynamic_cast(controller); + // cntl->SetFailed(-1, "open file controller error"); + // } +} + +static void CloseFileFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::CloseFileRequest* request, + ::nebd::client::CloseFileResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +static void ReadFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::ReadRequest* request, + ::nebd::client::ReadResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +static void WriteFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::WriteRequest* request, + ::nebd::client::WriteResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +static void DiscardFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::DiscardRequest* request, + ::nebd::client::DiscardResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +static void StatFileFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::StatFileRequest* request, + ::nebd::client::StatFileResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +static void ResizeFileFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::ResizeRequest* request, + ::nebd::client::ResizeResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +static void FlushFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::FlushRequest* request, + ::nebd::client::FlushResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +static void GetInfoFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::GetInfoRequest* request, + ::nebd::client::GetInfoResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +static void InvalidateCacheFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::InvalidateCacheRequest* request, + ::nebd::client::InvalidateCacheResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +using ::testing::_; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::AnyNumber; +using ::testing::DoAll; +using ::testing::SetArgPointee; +using ::testing::SetArgReferee; +using ::testing::InSequence; +using ::testing::AtLeast; +using ::testing::SaveArgPointee; + +TEST_F(nebdFileClientTest, rpc_fail_test) { + nebd::client::FileClient fileClientTest; + nebd::common::Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_TRUE(conf.LoadConfig()); + ASSERT_EQ(fileClientTest.LoadConf(&conf), 0); + + // add service + brpc::Server server; + nebd::client::MockQemuClientService clientService; + ASSERT_EQ(server.AddService(&clientService, + brpc::SERVER_DOESNT_OWN_SERVICE), 0); + + // start rpc server + brpc::ServerOptions option; + option.idle_timeout_sec = -1; + std::string listenAddr = "127.0.0.1:6667"; + ASSERT_EQ(server.Start(listenAddr.c_str(), &option), 0); + + ASSERT_EQ(fileClientTest.InitChannel("127.0.0.1:6667"), 0); + + ::nebd::client::OpenFileResponse openFileResponse; + openFileResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, OpenFile(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(openFileResponse), + Invoke(OpenFileFunc))); + ASSERT_EQ(fileClientTest.Open(filename), -1); + + ::nebd::client::ResizeResponse resizeResponse; + resizeResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, ResizeFile(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(resizeResponse), + Invoke(ResizeFileFunc))); + ASSERT_EQ(fileClientTest.Extend(DEFAULT_FD, DEFAULT_FILEZIZE), -1); + + ::nebd::client::StatFileResponse statFileResponse; + statFileResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, StatFile(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(statFileResponse), + Invoke(StatFileFunc))); + ASSERT_EQ(fileClientTest.StatFile(DEFAULT_FD), -1); + + ::nebd::client::GetInfoResponse getInfoResponse; + getInfoResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, GetInfo(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(getInfoResponse), + Invoke(GetInfoFunc))); + ASSERT_EQ(fileClientTest.GetInfo(DEFAULT_FD), -1); + + ::nebd::client::InvalidateCacheResponse invalidateCacheResponse; + invalidateCacheResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, InvalidateCache(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(invalidateCacheResponse), + Invoke(InvalidateCacheFunc))); + ASSERT_EQ(fileClientTest.InvalidCache(DEFAULT_FD), -1); + + char writeBuf[BUFSIZE] = "test"; + char readBuf[BUFSIZE]; + ClientAioContext context; + context.buf = writeBuf; + context.offset = 0; + context.length = BUFSIZE; + context.ret = 0; + context.op = LIBAIO_OP_WRITE; + context.cb = LibAioFailCallBackFunc; + context.retryCount = 0; + + ::nebd::client::ReadResponse readResponse; + readResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, Read(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(readResponse), + Invoke(ReadFunc))); + ASSERT_EQ(fileClientTest.AioRead(DEFAULT_FD, &context), 0); + std::unique_lock lock{mtx}; + condition.wait(lock); + ASSERT_TRUE(callback); + + + context.buf = readBuf; + context.offset = 0; + context.length = BUFSIZE; + context.ret = 0; + context.op = LIBAIO_OP_READ; + context.cb = LibAioFailCallBackFunc; + context.retryCount = 0; + ::nebd::client::WriteResponse writeResponse; + writeResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, Write(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(writeResponse), + Invoke(WriteFunc))); + ASSERT_EQ(fileClientTest.AioWrite(DEFAULT_FD, &context), 0); + condition.wait(lock); + ASSERT_TRUE(callback); + + context.buf = nullptr; + context.offset = 0; + context.length = BUFSIZE; + context.ret = 0; + context.op = LIBAIO_OP_DISCARD; + context.cb = LibAioFailCallBackFunc; + context.retryCount = 0; + ::nebd::client::DiscardResponse discardResponse; + discardResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, Discard(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(discardResponse), + Invoke(DiscardFunc))); + ASSERT_EQ(fileClientTest.Discard(DEFAULT_FD, &context), 0); + condition.wait(lock); + ASSERT_TRUE(callback); + + context.buf = nullptr; + context.offset = 0; + context.length = 0; + context.ret = 0; + context.op = LIBAIO_OP_FLUSH; + context.cb = LibAioFailCallBackFunc; + context.retryCount = 0; + ::nebd::client::FlushResponse flushResponse; + flushResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, Flush(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(flushResponse), + Invoke(FlushFunc))); + ASSERT_EQ(fileClientTest.Flush(DEFAULT_FD, &context), 0); + condition.wait(lock); + ASSERT_TRUE(callback); + + ::nebd::client::CloseFileResponse closeFileResponse; + closeFileResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, CloseFile(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(closeFileResponse), + Invoke(CloseFileFunc))); + ASSERT_EQ(fileClientTest.Close(DEFAULT_FD), -1); + fileClientTest.Uninit(); + return; +} + +class nebdClientLifeCycleTest: public ::testing::Test { + protected: + void SetUp() override {} + void TearDown() override {} +}; + +TEST_F(nebdClientLifeCycleTest, init_when_part2_not_alive) { + ASSERT_EQ(Init4Qemu(confFilename), 0); + Uninit4Qemu(); +} + +TEST_F(nebdClientLifeCycleTest, init_when_part2_alive) { + ASSERT_EQ(Init4Qemu(confFilename), 0); + ASSERT_EQ(Init4Qemu(confFilename), 0); + Uninit4Qemu(); +} + +TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_stop_without_hb_test) { + using ::nebd::common::Configuration; + Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_EQ(conf.LoadConfig(), true); + + nebd::client::LifeCycleManager manager; + ASSERT_EQ(manager.Start(&conf), 0); + ASSERT_EQ(manager.IsPart2Alive(), true); + ASSERT_EQ(manager.IsHeartbeatThreadStart(), false); + ASSERT_EQ(manager.Start(&conf), 0); + ASSERT_EQ(manager.IsPart2Alive(), true); + ASSERT_EQ(manager.IsHeartbeatThreadStart(), false); + manager.Stop(); + ASSERT_EQ(manager.IsPart2Alive(), false); +} + +TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_stop_with_hb_test) { + using ::nebd::common::Configuration; + Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_EQ(conf.LoadConfig(), true); + + nebd::client::LifeCycleManager manager; + ASSERT_EQ(manager.Start(&conf), 0); + ASSERT_EQ(manager.IsPart2Alive(), true); + ASSERT_EQ(manager.Start(&conf), 0); + ASSERT_EQ(manager.IsPart2Alive(), true); + ASSERT_EQ(manager.IsHeartbeatThreadStart(), false); + manager.StartHeartbeat(); + ASSERT_EQ(manager.IsHeartbeatThreadStart(), true); + ASSERT_EQ(manager.IsPart2Alive(), true); + manager.Stop(); + ASSERT_EQ(manager.IsPart2Alive(), false); +} + +TEST_F(nebdClientLifeCycleTest, lifecycle_kill_part2_test) { + using ::nebd::common::Configuration; + Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_EQ(conf.LoadConfig(), true); + + nebd::client::LifeCycleManager manager; + ASSERT_EQ(manager.Start(&conf), 0); + ASSERT_EQ(manager.IsPart2Alive(), true); + ASSERT_EQ(manager.IsHeartbeatThreadStart(), false); + manager.StartHeartbeat(); + ASSERT_EQ(manager.IsHeartbeatThreadStart(), true); + manager.KillPart2(); + sleep(2); + ASSERT_EQ(manager.IsPart2Alive(), true); + manager.Stop(); + ASSERT_EQ(manager.IsPart2Alive(), false); +} + +int main(int argc, char ** argv) { + ::testing::InitGoogleTest(&argc, argv); + ::testing::InitGoogleMock(&argc, argv); + google::ParseCommandLineFlags(&argc, &argv, false); + int ret = RUN_ALL_TESTS(); + + return ret; +} From af10aad887628f1a333f20f45e50b9a8e5714df7 Mon Sep 17 00:00:00 2001 From: hzwuhongsong Date: Tue, 15 Oct 2019 17:33:17 +0800 Subject: [PATCH 04/79] create part2 Change-Id: Ia767657203673376515d7bb1e8a89b1000cd832c --- src/part2/common_type.cpp | 627 ++++++++++++++++ src/part2/common_type.h | 64 ++ src/part2/config.cpp | 178 +++++ src/part2/config.h | 60 ++ src/part2/heartbeat.cpp | 348 +++++++++ src/part2/heartbeat.h | 43 ++ src/part2/interrupt_sleep.h | 39 + src/part2/main.cpp | 68 ++ src/part2/rados_interface.cpp | 173 +++++ src/part2/rados_interface.h | 23 + src/part2/reload.cpp | 121 +++ src/part2/reload.h | 23 + src/part2/rpc_ceph.cpp | 385 ++++++++++ src/part2/rpc_ceph.h | 54 ++ src/part2/rpc_request.h | 92 +++ src/part2/rpc_server.cpp | 457 +++++++++++ src/part2/rpc_server.h | 80 ++ tests/part2/TestCommon.cpp | 477 ++++++++++++ tests/part2/TestCommon2.cpp | 189 +++++ tests/part2/TestCommon3.cpp | 103 +++ tests/part2/TestConfig.cpp | 331 ++++++++ tests/part2/TestHeartbeat.cpp | 221 ++++++ tests/part2/TestHeartbeat2.cpp | 218 ++++++ tests/part2/TestHeartbeat3.cpp | 43 ++ tests/part2/TestRados.cpp | 334 +++++++++ tests/part2/TestReload.cpp | 107 +++ tests/part2/TestReload2.cpp | 104 +++ tests/part2/TestRpcCeph.cpp | 1252 +++++++++++++++++++++++++++++++ tests/part2/TestRpcServer.cpp | 1187 +++++++++++++++++++++++++++++ tests/part2/test_common.cpp | 18 + tests/part2/test_common.h | 30 + tests/part2/test_common2.cpp | 10 + tests/part2/test_common2.h | 20 + tests/part2/test_config.cpp | 13 + tests/part2/test_heartbeat.cpp | 17 + tests/part2/test_heartbeat.h | 31 + tests/part2/test_heartbeat2.cpp | 16 + tests/part2/test_heartbeat2.h | 31 + tests/part2/test_heartbeat3.cpp | 10 + tests/part2/test_heartbeat3.h | 24 + tests/part2/test_rados.cpp | 24 + tests/part2/test_rados.h | 31 + tests/part2/test_reload.cpp | 13 + tests/part2/test_reload.h | 25 + tests/part2/test_reload2.cpp | 11 + tests/part2/test_reload2.h | 24 + tests/part2/test_rpcceph.cpp | 41 + tests/part2/test_rpcceph.h | 46 ++ 48 files changed, 7836 insertions(+) create mode 100644 src/part2/common_type.cpp create mode 100644 src/part2/common_type.h create mode 100644 src/part2/config.cpp create mode 100644 src/part2/config.h create mode 100644 src/part2/heartbeat.cpp create mode 100644 src/part2/heartbeat.h create mode 100644 src/part2/interrupt_sleep.h create mode 100644 src/part2/main.cpp create mode 100644 src/part2/rados_interface.cpp create mode 100644 src/part2/rados_interface.h create mode 100644 src/part2/reload.cpp create mode 100644 src/part2/reload.h create mode 100644 src/part2/rpc_ceph.cpp create mode 100644 src/part2/rpc_ceph.h create mode 100644 src/part2/rpc_request.h create mode 100644 src/part2/rpc_server.cpp create mode 100644 src/part2/rpc_server.h create mode 100644 tests/part2/TestCommon.cpp create mode 100644 tests/part2/TestCommon2.cpp create mode 100644 tests/part2/TestCommon3.cpp create mode 100644 tests/part2/TestConfig.cpp create mode 100644 tests/part2/TestHeartbeat.cpp create mode 100644 tests/part2/TestHeartbeat2.cpp create mode 100644 tests/part2/TestHeartbeat3.cpp create mode 100644 tests/part2/TestRados.cpp create mode 100644 tests/part2/TestReload.cpp create mode 100644 tests/part2/TestReload2.cpp create mode 100644 tests/part2/TestRpcCeph.cpp create mode 100644 tests/part2/TestRpcServer.cpp create mode 100644 tests/part2/test_common.cpp create mode 100644 tests/part2/test_common.h create mode 100644 tests/part2/test_common2.cpp create mode 100644 tests/part2/test_common2.h create mode 100644 tests/part2/test_config.cpp create mode 100644 tests/part2/test_heartbeat.cpp create mode 100644 tests/part2/test_heartbeat.h create mode 100644 tests/part2/test_heartbeat2.cpp create mode 100644 tests/part2/test_heartbeat2.h create mode 100644 tests/part2/test_heartbeat3.cpp create mode 100644 tests/part2/test_heartbeat3.h create mode 100644 tests/part2/test_rados.cpp create mode 100644 tests/part2/test_rados.h create mode 100644 tests/part2/test_reload.cpp create mode 100644 tests/part2/test_reload.h create mode 100644 tests/part2/test_reload2.cpp create mode 100644 tests/part2/test_reload2.h create mode 100644 tests/part2/test_rpcceph.cpp create mode 100644 tests/part2/test_rpcceph.h diff --git a/src/part2/common_type.cpp b/src/part2/common_type.cpp new file mode 100644 index 0000000000..74908e30fe --- /dev/null +++ b/src/part2/common_type.cpp @@ -0,0 +1,627 @@ +/* + * Project: nebd + * File Created: 2019-09-30 + * Author: hzwuhongsong + * Copyright (c) 2019 NetEase + */ + +#include "src/part2/common_type.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/part2/config.h" +#include "src/part2/heartbeat.h" +#include "src/part2/reload.h" +#include "src/part2/rpc_server.h" + +// qemu进程的uuid +char* g_uuid; +// qemu进程的pid +int64_t g_oldPid = -1; +// 该nebd-server程序使用的端口号 +int g_port; +// 记录请求数量 +std::atomic_long requestForRpcWrite(0); +std::atomic_long requestForCephWrite(0); +std::atomic_long requestForRpcWriteCounts(0); +// uuid持久化文件存放路径 +const char* g_filePath; +// fd的内存映像 +std::map g_imageMap; + +// 保存云主机当前挂载的卷信息 +std::vector g_qemuPoolVolumes; + +// 文件锁函数 +int LockFile(const char* file) { + int lockfd = open(file, O_RDONLY | O_CREAT, 0644); + if (lockfd < 0) { + LOG(ERROR) << "open lock file failed, " << file; + return -1; + } + int ret = flock(lockfd, LOCK_EX); + if (ret < 0) { + LOG(ERROR) << "add lock failed, " << file; + close(lockfd); + return -1; + } + return lockfd; +} + +void UnlockFile(int lockfd) { + flock(lockfd, LOCK_UN); + close(lockfd); +} + +// 获取所有云主机当前挂载的卷信息 +int ReadQemuXmls() { + struct dirent* ent; + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return -1; + } + std::string qemu_xml_dir = ReadQemuXmlDir(cfg); + config_destroy(cfg); + DIR* dir = opendir(qemu_xml_dir.c_str()); + if (dir == NULL) { + LOG(ERROR) << "qemu xml dir open failed, " << qemu_xml_dir; + return -1; + } + while ((ent = readdir(dir))) { + if ((ent->d_name[0] == '.' && ent->d_name[1] == '\0') || + (ent->d_name[1] == '.' && ent->d_name[2] == '\0')) + continue; + + if (strstr(ent->d_name, ".xml") == NULL) continue; + std::string qemu_xml_file = qemu_xml_dir + "/" + ent->d_name; + ReadQemuXml(qemu_xml_file.c_str()); + } + return 0; +} + +// 获取一个云主机挂载的卷信息，存入qemu_xml_map +void ReadQemuXml(const char* xml_file) { + LOG(NOTICE) << "xml file, " << xml_file; + + boost::property_tree::ptree pt; + try { + boost::property_tree::xml_parser::read_xml(xml_file, pt); + } catch (boost::property_tree::ptree_error pt_error) { + LOG(ERROR) << "read xml throw an exception. " << pt_error.what(); + return; + } + + std::string protocol, pool_vol_name; + + auto item = pt.get_child_optional("domstatus.domain.uuid"); + if (!item) { + LOG(ERROR) << "domstatus.domain.uuid is not exist"; + return; + } + item = pt.get_child_optional("domstatus.domain.devices"); + if (!item) { + LOG(ERROR) << "domstatus.domain.devices is not exist"; + return; + } + + std::string uuid; + uuid = pt.get("domstatus.domain.uuid"); + if (uuid != g_uuid) { + LOG(ERROR) << "uuid is not equal domain.uuid, " << g_uuid << ", " + << uuid; + return; + } + + BOOST_AUTO(child, pt.get_child("domstatus.domain.devices")); + + for (auto iter = child.begin(); iter != child.end(); ++iter) { + if (iter->first != "disk") { + continue; + } + BOOST_AUTO(pos, iter->second.get_child("source")); + for (auto nextiter = pos.begin(); nextiter != pos.end(); ++nextiter) { + if ("" != nextiter->first) { + continue; + } + protocol = nextiter->second.get("protocol"); + pool_vol_name = nextiter->second.get("name"); + + if (protocol == "rbd") { + g_qemuPoolVolumes.push_back(pool_vol_name); + } else { + LOG(ERROR) << "protocol is: " << protocol + << ", pool_vol_name is: " << pool_vol_name; + } + } + } + for (auto name : g_qemuPoolVolumes) { + LOG(NOTICE) << "pool and vol is:" << name; + } +} + +// 生成port信息并存入持久化文件 +int GeneratePort(int port) { + std::string metadata_file = GetUuidFile(); + boost::property_tree::ptree root; + boost::property_tree::ptree items; + + try { + root.put("port", port); + boost::property_tree::write_json(metadata_file, root); + } catch (boost::property_tree::ptree_error pt) { + LOG(ERROR) << "write json throw an exception. " << pt.what(); + return -1; + } + + return 0; +} + +// 生成fd信息并存入持久化文件 +int GenerateFd(char* filename, int fd) { + std::string metadata_file = GetUuidFile(); + LOG(NOTICE) << "generate fd start. " << filename << ", " << fd; + boost::property_tree::ptree root, items; + + try { + boost::property_tree::read_json( + metadata_file, root); + } catch (boost::property_tree::ptree_error pt) { + LOG(ERROR) << "read json throw an exception. " << pt.what(); + return -1; + } + + // 第一次生成volumes + auto volumes_exist = root.get_child_optional("volumes"); + if (!volumes_exist) { + boost::property_tree::ptree item1; + item1.put("filename", filename); + item1.put("fd", fd); + items.push_back(std::make_pair("", item1)); + root.put("port", g_port); + root.put_child("volumes", items); + try { + boost::property_tree::write_json(metadata_file, root); + } catch (boost::property_tree::ptree_error pt) { + LOG(ERROR) << "write json throw an exception. " << pt.what(); + return -1; + } + return 0; + } + + items = root.get_child("volumes"); + int count = 0; + for (boost::property_tree::ptree::iterator it = items.begin(); + it != items.end(); ++it) { + count++; + } + + // boost::property_tree::ptree item[count + 1]; + std::vector item; + item.resize(count + 1); + int item_index = 0; + for (boost::property_tree::ptree::iterator it = items.begin(); + it != items.end(); ++it) { + std::string filename_json = it->second.get("filename"); + std::string fd_json = it->second.get("fd"); + item[item_index].put("filename", filename_json); + item[item_index].put("fd", fd_json); + item_index++; + } + + item[count].put("filename", filename); + item[count].put("fd", fd); + + boost::property_tree::ptree items_write; + for (int i = 0; i <= count; i++) { + items_write.push_back(std::make_pair("", item[i])); + } + + root.put("port", g_port); + root.put_child("volumes", items_write); + + try { + boost::property_tree::write_json(metadata_file, root); + } catch (boost::property_tree::ptree_error pt) { + LOG(ERROR) << "write json throw an exception. " << pt.what(); + return -1; + } + LOG(NOTICE) << "generate fd end. " << filename << ", " << fd; + return 0; +} + +// 从持久化文件删除fd相关信息 +int RmFd(int fd) { + std::string metadata_file = GetUuidFile(); + boost::property_tree::ptree root, items; + + try { + boost::property_tree::read_json( + metadata_file, root); + } catch (boost::property_tree::ptree_error pt) { + LOG(ERROR) << "read json throw an exception. " << pt.what(); + return -1; + } + + auto volumes_exist = root.get_child_optional("volumes"); + if (!volumes_exist) { + LOG(ERROR) << "rm fd failed, no volume is exist. fd is: " << fd; + return -1; + } + + items = root.get_child("volumes"); + + std::vector vector_items; + + bool FdExist = false; + for (boost::property_tree::ptree::iterator it = items.begin(); + it != items.end(); ++it) { + std::string filename = it->second.get("filename"); + int fd_tmp = atoi(it->second.get("fd").c_str()); + if (fd == fd_tmp) { + FdExist = true; + continue; + } + + boost::property_tree::ptree item_tmp; + item_tmp.put("filename", filename); + item_tmp.put("fd", fd_tmp); + + vector_items.push_back(item_tmp); + } + if (!FdExist) { + LOG(ERROR) << "rm fd failed, fd is not exist. fd is: " << fd; + return -1; + } + + boost::property_tree::ptree items_write; + for (std::vector::iterator iter = + vector_items.begin(); + iter != vector_items.end(); iter++) { + items_write.push_back(std::make_pair("", *iter)); + } + + root.put("port", g_port); + root.put_child("volumes", items_write); + + try { + boost::property_tree::write_json(metadata_file, root); + } catch (boost::property_tree::ptree_error pt) { + LOG(ERROR) << "write json throw an exception. " << pt.what(); + return -1; + } + + LOG(NOTICE) << "rm fd success. " << fd; + return 0; +} + +void Squeeze(char s[], char c) { + int i, j; + for (i = 0, j = 0; s[i] != '\0'; i++) { + if (s[i] != c) { + s[j++] = s[i]; + } + } + s[j] = '\0'; +} + +// 获得mon host +std::string GetMonHost(const std::string& str) { + char* mon_host_out = NULL; + std::vector result; + if (*str.begin() == ':' || *str.end() == ':') { + LOG(ERROR) << "wrong format. " << str; + std::string mon_host; + return mon_host; + } + + auto sub_begin = str.begin(); + auto sub_end = str.begin(); + for (auto it = str.begin(); it != str.end(); ++it, ++sub_end) { + if (*it == ':' && std::isalpha(*(it + 1))) { + result.push_back(std::string(sub_begin, sub_end)); + sub_begin = it + 1; + } + } + result.push_back(std::string(sub_begin, sub_end)); + + for (auto piece : result) { + if (strstr(piece.c_str(), "mon_host") != NULL) { + std::vector mon_host = split(piece, "="); + mon_host_out = const_cast(mon_host[1].c_str()); + Squeeze(mon_host_out, '\\'); + LOG(ERROR) << "mon host is. " << mon_host_out; + } + } + return mon_host_out; +} + +// 字符串拆分，pattern为分隔符 +std::vector split(const std::string& str, + const std::string& pattern) { + std::vector res; + if (str == "") return res; + std::string strs = str + pattern; + size_t pos = strs.find(pattern); + + while (pos != strs.npos) { + std::string temp = strs.substr(0, pos); + res.push_back(temp); + strs = strs.substr(pos + 1, strs.size()); + pos = strs.find(pattern); + } + + return res; +} + +// 找到可用的port +int FindPort(config_t* cfg, int port) { + LOG(NOTICE) << "find port start."; + + struct dirent* ent; + std::set port_inuse; + + DIR* proc = opendir(g_filePath); + if (proc == NULL) { + LOG(ERROR) << "opendir file_path is error"; + return -1; + } + + while ((ent = readdir(proc))) { + if ((ent->d_name[0] == '.' && ent->d_name[1] == '\0') || + (ent->d_name[1] == '.' && ent->d_name[2] == '\0')) + continue; + std::string tmp = "/tmp/"; + std::string ent_name = ent->d_name; + std::string filepath_tmp = g_filePath; + std::string uuid_file_tmp = filepath_tmp + "/" + ent_name; + std::string uuid_lockfile_tmp = tmp + ent_name; + char* uuid_lockfile = const_cast(uuid_lockfile_tmp.c_str()); + char* uuid_file = const_cast(uuid_file_tmp.c_str()); + int lockfd = LockFile(uuid_lockfile); + if (lockfd < 0) { + closedir(proc); + LOG(ERROR) << "lock file failed." << uuid_lockfile; + return -1; + } + + boost::property_tree::ptree root; + boost::property_tree::ptree items; + int port; + + try { + boost::property_tree::read_json( + uuid_file, root); + auto port_exist = root.get_child_optional("port"); + if (!port_exist) { + LOG(ERROR) << "port is not exist."; + UnlockFile(lockfd); + continue; + } + port = root.get("port", 0); + } catch (boost::property_tree::ptree_error pt) { + LOG(ERROR) << "read json throw an exception. " << pt.what(); + UnlockFile(lockfd); + continue; + } + + port_inuse.insert(port); + + UnlockFile(lockfd); + } + + closedir(proc); + + port_option_t* port_option = new port_option_t; + ReadPort(cfg, port_option); + int min_port, max_port; + min_port = port_option->min_port; + max_port = port_option->max_port; + delete port_option; + int start_port; + if (port == 0) { + start_port = min_port; + } else { + start_port = port; + } + + for (int i = start_port; i < max_port; i++) { + if (!(port_inuse.find(i) != port_inuse.end())) { + LOG(ERROR) << "port is not inuse, " << i; + return i; + } else { + LOG(NOTICE) << " port is inuse, " << i; + } + } + + LOG(NOTICE) << "find port end."; + return 0; +} + +std::string GetUuidFile() { + std::string uuid_file; + std::string file_path_tmp = g_filePath; + std::string g_uuid_tmp = g_uuid; + uuid_file = file_path_tmp + "/" + g_uuid_tmp; + return uuid_file; +} + +std::string GetUuidLockfile() { + std::string uuid_lockfile; + std::string g_uuid_tmp = g_uuid; + uuid_lockfile = "/tmp/" + g_uuid_tmp; + return uuid_lockfile; +} + +// 设置cpu亲和性 +int SetCpuAffinity(pid_t qemu) { + cpu_set_t mask; + + int r = sched_getaffinity(qemu, sizeof(cpu_set_t), &mask); + if (r < 0) { + LOG(NOTICE) << "getaffinity failed."; + return r; + } + r = sched_setaffinity(0, sizeof(cpu_set_t), &mask); + if (r < 0) { + LOG(NOTICE) << "setaffinity failed."; + return r; + } + LOG(NOTICE) << "setaffinity success. " << qemu; + return 0; +} + +// 初始化 +int Init() { + // 初始化配置 + config_t* cfg = InitConfig(); + if (cfg == NULL) { + char buffer[128]; + const char* err = "init config fialed."; + snprintf(buffer, sizeof(buffer), + "echo %s>>/var/log/nebd/nebd-server-initfailed.log", err); + ::system(buffer); + return -1; + } + std::string file_path_tmp = ReadUuidPath(cfg); + g_filePath = file_path_tmp.c_str(); + + std::string log_path = ReadLogPath(cfg); + std::string log_file = log_path + "/" + g_uuid + ".log"; + // 初始化日志模块 + logging::LoggingSettings log; + log.logging_dest = logging::LOG_TO_FILE; + log.log_file = log_file.c_str(); + logging::InitLogging(log); + int loglevel = ReadLogLevel(cfg); + logging::SetMinLogLevel(loglevel); + if (NULL == opendir(g_filePath)) { + if (mkdir(g_filePath, S_IRWXU | S_IRWXG) < 0) { + LOG(ERROR) << "mkdir failed."; + return -1; + } + } + int ret = CheckProc(g_uuid); + if (ret < 0) { + LOG(ERROR) << "check proc failed."; + return -1; + } + + if (SetCpuAffinity(g_oldPid) < 0) { + LOG(ERROR) << "set cpu affinity failed."; + return -1; + } + + // rpc option + rpc_option_t* rpc_option = new rpc_option_t; + ReadRpcOption(cfg, rpc_option); + brpc::ServerOptions options; + options.idle_timeout_sec = rpc_option->brpc_idle_timeout; + options.num_threads = rpc_option->brpc_num_threads; + delete rpc_option; + brpc::Server brpc_server; + QemuClientServiceImpl* qemuclient_service_impl = new QemuClientServiceImpl; + if (brpc_server.AddService(qemuclient_service_impl, + brpc::SERVER_OWNS_SERVICE) != 0) { + LOG(ERROR) << "Fail to add service"; + return -1; + } + + bool new_port = false; + std::string uuid_file = GetUuidFile(); + std::string uuid_lockfile = GetUuidLockfile(); + + int lock_port_fd; + int lockfd = LockFile(uuid_lockfile.c_str()); + if (lockfd < 0) { + LOG(ERROR) << "lock file failed."; + return -1; + } + + // 如果持久化文件存在，则reload + if (access(uuid_file.c_str(), F_OK) != -1) { + int ret = Reload(); + UnlockFile(lockfd); + if (ret <= 0) { + LOG(ERROR) << "file is exist, but port is not exist."; + return -1; + } else { + g_port = ret; + } + } else { + UnlockFile(lockfd); + std::string lock_port_file = ReadLockPortFile(cfg); + lock_port_fd = LockFile(const_cast(lock_port_file.c_str())); + if (lock_port_fd < 0) { + LOG(ERROR) << "add port file lock failed."; + return -1; + } + g_port = FindPort(cfg, 0); + new_port = true; + } + LOG(NOTICE) << "port is: " << g_port; + // 启动brpc server，重试retry_counts次 + int retry = 0; + int retry_counts = ReadRetryCounts(cfg); + std::string ip = "127.0.0.1:"; + while (++retry <= retry_counts) { + std::string ip_port = ip + std::to_string(g_port); + if (brpc_server.Start(ip_port.c_str(), &options) != 0) { + LOG(ERROR) << "Fail to start QemuClientServer, retry time is: " + << retry; + g_port = FindPort(cfg, g_port + 1); + } else { + LOG(ERROR) << "start QemuClientServer success."; + break; + } + } + if (retry >= retry_counts) { + LOG(ERROR) << "Fail to start QemuClientServer."; + return -1; + } + // 如果是新启动的，则保存进持久化文件 + if (new_port) { + lockfd = LockFile(uuid_lockfile.c_str()); + if (lockfd < 0) { + LOG(ERROR) << "add lock failed."; + return -1; + } + + int ret; + ret = GeneratePort(g_port); + if (ret < 0) { + LOG(ERROR) << "generate port failed."; + return -1; + } + UnlockFile(lockfd); + UnlockFile(lock_port_fd); + } + config_destroy(cfg); + // 启动心跳线程 + bthread_t th; + if (bthread_start_background(&th, NULL, HeartbeatProcess, NULL) != 0) { + LOG(ERROR) << "Fail to create bthread"; + return -1; + } + // 响应SIGINT以及SIGTERM + signal(SIGTERM, SigProcess); + signal(SIGINT, SigProcess); + signal(SIGHUP, SigLogReplace); + bthread_join(th, NULL); + brpc_server.Stop(0); + brpc_server.Join(); + return 0; +} diff --git a/src/part2/common_type.h b/src/part2/common_type.h new file mode 100644 index 0000000000..cfdbf8fc51 --- /dev/null +++ b/src/part2/common_type.h @@ -0,0 +1,64 @@ +/* + * Project: nebd + * File Created: 2019-09-30 + * Author: hzwuhongsong + * Copyright (c) 2019 NetEase + */ + +#ifndef SRC_PART2_COMMON_TYPE_H_ +#define SRC_PART2_COMMON_TYPE_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct FdImage { + rados_t* cluster; + rados_ioctx_t* io; + rbd_image_t* image; + char* filename; +} FdImage_t; +// 主键为fd +extern std::map g_imageMap; // fd的内存镜像 +// ceph的fd取值范围为[1，1000]，curve取值范围为[1001, 20000] +#define FD_CEPH_MAX 1000 +#define FD_CURVE_MAX 20000 + +int LockFile(const char* file); +void UnlockFile(int lockfd); +std::vector split(const std::string& str, + const std::string& pattern); +int ReadQemuXmls(); +void ReadQemuXml(const char*); +int GenerateFd(char* filename, int fd); +int GeneratePort(int port); +int FindPort(config_t* cfg, int port); +int RmFd(int fd); +std::string GetUuidFile(); +std::string GetUuidLockfile(); +int SetCpuAffinity(pid_t qemu); +std::string GetMonHost(const std::string &str); +int Init(); +void Squeeze(char s[], char c); +// qemu达[秾K潚~Duuid +extern char* g_uuid; +// qemu达[秾K潚~Dpid +extern int64_t g_oldPid; +// 该nebd-server秾K幾O使潔¨潚~D端住£住· +extern int g_port; +// 记弾U请氾B录°轇~O +extern std::atomic_long requestForRpcWrite; +extern std::atomic_long requestForCephWrite; +extern std::atomic_long requestForRpcWriteCounts; +// uuid彌~A举E佌~V彖~G件嬾X彔 +extern const char* g_filePath; + +// 侾]嬾X乾Q主彜º弾S佉~M彌~B载潚~D位·信彁¯ +extern std::vector g_qemuPoolVolumes; + +#endif // SRC_PART2_COMMON_TYPE_H_ diff --git a/src/part2/config.cpp b/src/part2/config.cpp new file mode 100644 index 0000000000..fd30103acb --- /dev/null +++ b/src/part2/config.cpp @@ -0,0 +1,178 @@ +/* + * Project: nebd + * File Created: 2019-09-30 + * Author: hzwuhongsong + * Copyright (c) 2019 NetEase + */ + +#include +#include +#include +#include +#include "src/part2/config.h" +#include "src/part2/common_type.h" + +#define NEBD_CONF_FILE_DEFAULT "/etc/nebd/nebd-server.conf" +#define LOG_CONF_FILE_DEFAULT "/var/log/nebd/" +#define UUID_CONF_FILE_DEFAULT "/var/run/nebd-server/" +#define QEMU_XML_DIR_DEFAULT "/var/run/libvirt/qemu/" +#define LOCKPORT_CONF_FILE_DEFAULT "/tmp/nebd-server.port.file.lock" +#define NEBD_CEPH_CONF "/etc/ceph/ceph.conf" +#define BRPC_NUM_THREADS 2 +#define BRPC_IDLE_TIMEOUT -1 +#define BRPC_MAX_CONCURRENCY 0 +#define BRPC_METHOD_MAX_CONCURRENCY 0 +#define LOGLEVEL 1 +#define MIN_PORT 6200 +#define MAX_PORT 6500 +#define RETRY_COUNTS 3 +#define HEARTBEAT_INTERVAL 1 +#define CHECK_ASSERT_TIMES 10 +#define CHECHK_DETACHED_INTERVAL 5 +#define CHECK_DETACHED_TIME 6 + +config_t* InitConfig() { + config_t* cfg = new config_t; + config_init(cfg); + + // Read the file. If there is an error, return NUll. + if (!config_read_file(cfg, NEBD_CONF_FILE_DEFAULT)) { + char buffer[128]; + const char *err = "read config file failed."; + snprintf(buffer, sizeof(buffer), + "echo %s >> /var/log/nebd/nebd-server-initfailed.log", err); + ::system(buffer); + config_destroy(cfg); + return NULL; + } + return cfg; +} + +void ReadRpcOption(config_t* cfg, rpc_option_t* rpc_option) { + if (!config_lookup_int(cfg, "brpc_num_threads", + &rpc_option->brpc_num_threads)) + rpc_option->brpc_num_threads = BRPC_NUM_THREADS; + + if (!config_lookup_int(cfg, "brpc_idle_timeout", + &rpc_option->brpc_idle_timeout)) + rpc_option->brpc_idle_timeout = BRPC_IDLE_TIMEOUT; + + if (!config_lookup_int(cfg, "brpc_max_concurrency", + &rpc_option->brpc_max_concurrency)) + rpc_option->brpc_max_concurrency = BRPC_MAX_CONCURRENCY; + + if (!config_lookup_int(cfg, "brpc_methodmax_concurrency", + &rpc_option->brpc_method_max_concurrency)) + rpc_option->brpc_method_max_concurrency = BRPC_METHOD_MAX_CONCURRENCY; + return; +} + +void ReadPort(config_t* cfg, port_option_t* port_option) { + if (!config_lookup_int(cfg, "min_port", &port_option->min_port)) + port_option->min_port = MIN_PORT; + + if (!config_lookup_int(cfg, "max_port", &port_option->max_port)) + port_option->max_port = MAX_PORT; + + return; +} + +int ReadLogLevel(config_t* cfg) { + int loglevel; + if (!config_lookup_int(cfg, "loglevel", &loglevel)) loglevel = LOGLEVEL; + + return loglevel; +} + +std::string ReadUuidPath(config_t* cfg) { + std::string uuid_path; + const char* uuid_path_tmp; + + if (!config_lookup_string(cfg, "uuid_file_path", &uuid_path_tmp)) + uuid_path_tmp = UUID_CONF_FILE_DEFAULT; + + uuid_path = uuid_path_tmp; + return uuid_path; +} + +int ReadDetachedTimes(config_t* cfg) { + int detached_times; + if (!config_lookup_int(cfg, "detached_times", &detached_times)) + detached_times = CHECK_DETACHED_TIME; + return detached_times; +} + +int ReadRetryCounts(config_t* cfg) { + int retry_counts; + if (!config_lookup_int(cfg, "retry_counts", &retry_counts)) + retry_counts = RETRY_COUNTS; + + return retry_counts; +} + +std::string ReadLockPortFile(config_t* cfg) { + std::string lock_port_file; + const char* lock_port_file_tmp; + + if (!config_lookup_string(cfg, "lock_port_file", &lock_port_file_tmp)) + lock_port_file_tmp = LOCKPORT_CONF_FILE_DEFAULT; + + lock_port_file = lock_port_file_tmp; + return lock_port_file; +} + +std::string ReadCephConf(config_t* cfg) { + std::string ceph_conf; + const char* ceph_conf_tmp; + + if (!config_lookup_string(cfg, "nebd_ceph_conf", &ceph_conf_tmp)) + ceph_conf_tmp = NEBD_CEPH_CONF; + + ceph_conf = ceph_conf_tmp; + return ceph_conf; +} + +std::string ReadLogPath(config_t* cfg) { + std::string log_path; + const char* log_path_tmp; + if (!config_lookup_string(cfg, "log_path", &log_path_tmp)) + log_path_tmp = LOG_CONF_FILE_DEFAULT; + + log_path = log_path_tmp; + return log_path; +} + +std::string ReadQemuXmlDir(config_t* cfg) { + std::string qemu_dir; + const char* qemu_dir_tmp; + if (!config_lookup_string(cfg, "qemu_xml_dir", &qemu_dir_tmp)) + qemu_dir_tmp = QEMU_XML_DIR_DEFAULT; + + qemu_dir = qemu_dir_tmp; + return qemu_dir; +} + +int ReadHeartbeatInterval(config_t* cfg) { + int interval; + if (!config_lookup_int(cfg, "heartbeat_interval", &interval)) + interval = HEARTBEAT_INTERVAL; + + return interval; +} + +int ReadAssertTimes(config_t* cfg) { + int check_assert_times; + if (!config_lookup_int(cfg, "check_assert_times", &check_assert_times)) + check_assert_times = CHECK_ASSERT_TIMES; + + return check_assert_times; +} + +int ReadCheckDetachedInterval(config_t* cfg) { + int interval; + if (!config_lookup_int(cfg, "check_detached_interval", &interval)) + interval = CHECHK_DETACHED_INTERVAL; + + return interval; +} + diff --git a/src/part2/config.h b/src/part2/config.h new file mode 100644 index 0000000000..63a442abc3 --- /dev/null +++ b/src/part2/config.h @@ -0,0 +1,60 @@ +/* + * Project: nebd + * File Created: 2019-09-30 + * Author: hzwuhongsong + * Copyright (c) 2019 NetEase + */ + +#ifndef SRC_PART2_CONFIG_H_ +#define SRC_PART2_CONFIG_H_ + +#include +#include + +/** + * @brief 初始化配置 + */ +config_t* InitConfig(); + +/** + * @brief rpc相关配置 + */ +typedef struct rpc_option { + int brpc_num_threads; // brpc默认线程数 + int brpc_idle_timeout; + int brpc_max_concurrency; + int brpc_method_max_concurrency; +} rpc_option_t; +void ReadRpcOption(config_t* cfg, rpc_option_t* rpc_option); + +/** + * @brief port相关配置 + */ +typedef struct port_option { + int min_port; + int max_port; +} port_option_t; +void ReadPort(config_t* cfg, port_option_t* port_option); +// 或取日志级别 +int ReadLogLevel(config_t* cfg); +// 获取持久化文件路径 +std::string ReadUuidPath(config_t* cfg); +// 获取启动brpc server的重试次数 +int ReadRetryCounts(config_t* cfg); +// 获取qemu xml文件的路径 +std::string ReadQemuXmlDir(config_t* cfg); +// 获取心跳间隔 +int ReadHeartbeatInterval(config_t* cfg); +// 获取心跳检查次数，超过该次数未检查到，则进程自杀 +int ReadAssertTimes(config_t* cfg); +// 获取日志文件路径 +std::string ReadLogPath(config_t* cfg); +// 获取检查detach卷的间隔 +int ReadCheckDetachedInterval(config_t* cfg); +// 获取lock port文件 +std::string ReadLockPortFile(config_t* cfg); +// 获取ceph配置文件路径 +std::string ReadCephConf(config_t* cfg); +// 获取detach卷检查次数 +int ReadDetachedTimes(config_t* cfg); +#endif // SRC_PART2_CONFIG_H_ diff --git a/src/part2/heartbeat.cpp b/src/part2/heartbeat.cpp new file mode 100644 index 0000000000..684a92d7d5 --- /dev/null +++ b/src/part2/heartbeat.cpp @@ -0,0 +1,348 @@ +/* + * Project: nebd + * File Created: 2019-09-30 + * Author: hzwuhongsong + * Copyright (c) 2019 NetEase + */ + +#include "src/part2/heartbeat.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/part2/common_type.h" +#include "src/part2/config.h" +#include "src/part2/interrupt_sleep.h" +#include "src/part2/rados_interface.h" + +static int g_check_times = 0; +static int g_check_detached_times = 0; +static const char* g_proc_name = "qemu-system-x86_64"; +static int64_t g_last_received(0); +static bool heartbeat_check_first = true; +static InterruptibleSleeper sleeper; +/** + *@brief 关闭已卸载卷线程 + *@detail + *若有已卸载卷，那么检查detacheedTimes次后，卸载 + */ +void* CloseDetachedVolumesProcess(void*) { + LOG(NOTICE) << "close detached volumes process start."; + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return NULL; + } + int checkdetached_interval = ReadCheckDetachedInterval(cfg); + int detachedTimes = ReadDetachedTimes(cfg); + config_destroy(cfg); + sleep(checkdetached_interval); + CloseQemuDetachedVolumes(detachedTimes); + LOG(NOTICE) << "close detached volumes process end."; + return NULL; +} +/** + * @brief 心跳线程 + * @detail + * 每隔heartbeat_interval秒检查一次qemu进程， + * 若assert_times检查后，qemu进程依然不在，那么自杀 + */ +void* HeartbeatProcess(void*) { + int ret = 0; + config_t* cfg = InitConfig(); + if (cfg == NULL) { + return NULL; + } + int heartbeat_interval = ReadHeartbeatInterval(cfg); + int check_assert_times = ReadAssertTimes(cfg); + config_destroy(cfg); + while (sleeper.wait_for( + std::chrono::milliseconds(1000 * heartbeat_interval))) { + ret = CheckProc(g_uuid); + if (ret < 0) { + if (++g_check_times < check_assert_times) { + LOG(ERROR) << "donnt stop, check_time is: " << g_check_times + << ", " + << "check_assert_times is: " << check_assert_times; + continue; + } + LOG(ERROR) << "qemu proc is stop, so stop myself"; + StopProc(false); + } else { + g_check_times = 0; + if (requestForRpcWriteCounts > 0) { + int64_t last_received; + last_received = requestForRpcWriteCounts - g_last_received; + g_last_received = requestForRpcWriteCounts; + LOG(NOTICE) << "qemu proc is running, " + << "rpc inflight: " << requestForRpcWrite + << ", ceph inflight: " << requestForCephWrite + << ", last received: " << last_received; + } else { + requestForRpcWriteCounts = 0; + g_last_received = 0; + LOG(NOTICE) << "rpc write counts is zero or has overflowed."; + } + } + } + LOG(ERROR) << "the heartbeat is over."; + return NULL; +} + +/** + * @brief 响应SIGINT以及SIGTERM信号，客户端升级使用 + */ +void SigProcess(int sig_no) { + LOG(ERROR) << "signal has been received, " << sig_no; + StopProc(true); +} + +/** + * @brief 响应SIGHUP信号，日志回滚使用 + */ +void SigLogReplace(int sig_no) { + LOG(WARNING) << "signal has been received, " << sig_no; + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + std::string log_path = ReadLogPath(cfg); + std::string log_file = log_path + "/" + g_uuid + ".log"; + logging::LoggingSettings log; + log.logging_dest = logging::LOG_TO_FILE; + log.log_file = log_file.c_str(); + logging::InitLogging(log); + int loglevel = ReadLogLevel(cfg); + logging::SetMinLogLevel(loglevel); + + LOG(WARNING) << "log has been logrotated."; + config_destroy(cfg); +} + +int StopProc(bool need_check_qemu_proc) { + bool remove_file = true; + std::string uuid_file = GetUuidFile(); + std::string uuid_lockfile = GetUuidLockfile(); + + int lockfd = LockFile(uuid_lockfile.c_str()); + if (lockfd < 0) { + LOG(ERROR) << "add lock failed."; + return -1; + } + + std::map::iterator iter; + for (iter = g_imageMap.begin(); iter != g_imageMap.end(); iter++) { + if (iter->first <= 0 || iter->first > FD_CEPH_MAX) { + LOG(ERROR) << "fd is abnormal. " << iter->first; + continue; + } + int ret = CloseImage(iter->first); + if (ret < 0) { + LOG(ERROR) << "close image failed."; + UnlockFile(lockfd); + return -1; + } + } + + if (need_check_qemu_proc) { + int ret = CheckProc(g_uuid); + if (ret >= 0) remove_file = false; + } + + if (remove_file) { + if (remove(uuid_file.c_str()) == 0) { + LOG(ERROR) << "remove uuid file suc. " << uuid_file; + } else { + LOG(ERROR) << "remove uuid file failed. " << uuid_file; + } + } + UnlockFile(lockfd); + + sleeper.interrupt(); + return 0; +} + +int CloseQemuDetachedVolumes(int detachedTimes) { + LOG(NOTICE) << "close detached volumes start."; + if (ReadQemuXmls() < 0) { + LOG(NOTICE) << "read qrmu xmls failed."; + return -1; + } + std::string uuid_lockfile = GetUuidLockfile(); + std::vector fd_pool_vols; + std::map>::iterator iter; + + // 遍历image_map，关闭所有qemu已经关闭的卷 + std::map::iterator iter_fd; + for (iter_fd = g_imageMap.begin(); iter_fd != g_imageMap.end(); iter_fd++) { + char* filename = iter_fd->second->filename; + std::vector split_firstly = split(filename, ":"); + if (split_firstly.empty()) continue; + std::string pool_vol; + pool_vol = split_firstly[1]; + if (pool_vol.empty()) continue; + + LOG(ERROR) << "change, pool and vol is: " << pool_vol; + fd_pool_vols.push_back(pool_vol); + std::vector::iterator ret; + ret = + find(g_qemuPoolVolumes.begin(), g_qemuPoolVolumes.end(), pool_vol); + if (ret == g_qemuPoolVolumes.end()) { + if (++g_check_detached_times < detachedTimes) { + LOG(ERROR) << "volume has not been found, " << pool_vol + << ", no close now, " << g_check_detached_times + << " less " << detachedTimes; + bthread_t th; + if (bthread_start_background( + &th, NULL, CloseDetachedVolumesProcess, NULL) != 0) { + LOG(ERROR) << "Fail to create bthread"; + return -1; + } + LOG(NOTICE) << "no volume has been detached."; + return 0; + } + LOG(ERROR) << "volume has not been found, detach now: " << pool_vol; + int lockfd = LockFile(uuid_lockfile.c_str()); + if (lockfd < 0) { + LOG(ERROR) << "add lock failed."; + return -1; + } + + if (CloseImage(iter_fd->first) < 0) { + LOG(ERROR) << "close image failed."; + UnlockFile(lockfd); + return -1; + } + if (RmFd(iter_fd->first) < 0) { + LOG(ERROR) << "rm fd failed."; + UnlockFile(lockfd); + return -1; + } + UnlockFile(lockfd); + } else { + LOG(NOTICE) << "volume has been found. " << pool_vol; + } + } + + // 本次检查结束，清空vector + g_qemuPoolVolumes.clear(); + LOG(NOTICE) << "close detached volumes end."; + return 0; +} + +char* SkipSpace(char* in, int* offset, int len) { + while (*offset < len) { + if (isspace(in[*offset]) || in[*offset] == '\0') { + ++*offset; + + continue; + } + break; + } + return in + *offset; +} + +int CheckCmdline(int64_t pid, const char* proc_name, char* uuid, int uuid_len) { + int i = 0; + char path[32], line[2048], *p; + + snprintf(path, sizeof(path), "/proc/%ld/cmdline", pid); + + int cmdline = open(path, O_RDONLY); + if (!cmdline) return -1; + + int nbytesread = read(cmdline, line, 2048); + close(cmdline); + + LOG(INFO) << "name is " << basename(line) << " " << proc_name << " " + << strlen(proc_name); + + if (strncmp(basename(line), proc_name, strlen(proc_name)) != 0) return -1; + + i += strlen(line); + + for (; i < nbytesread; i++) { + p = SkipSpace(line, &i, nbytesread); + + if (strncmp(p, "-uuid", 5) == 0) { + i += strlen(p); + p = SkipSpace(line, &i, nbytesread); + strncpy(uuid, p, uuid_len); + return 0; + } + i += strlen(p); + } + return -1; +} +/** + * @brief 检查qemu进程是否存在 + */ +int CheckProc(const char* uuid) { + int64_t pid; + char uuid_check[64]; + int r; + if (g_oldPid < 0) { + // get pid from proc name + DIR* proc = opendir("/proc"); + struct dirent* ent; + + if (proc == NULL) { + LOG(ERROR) << "open dir failed."; + return -1; + } + + while ((ent = readdir(proc))) { + if (!isdigit(*ent->d_name)) continue; + + pid = strtol(ent->d_name, NULL, 10); + + LOG(INFO) << "pid is " << pid; + + r = CheckCmdline(pid, g_proc_name, uuid_check, strlen(uuid)); + if (r >= 0) { + if (strncmp(uuid_check, uuid, strlen(uuid)) == 0) { + if (!heartbeat_check_first) { + LOG(NOTICE) << "pid change, close detached volumes."; + // 查看挂载的卷是否发生变化--卸载所有已变化的卷 + bthread_t th; + if (bthread_start_background( + &th, NULL, CloseDetachedVolumesProcess, NULL) != + 0) { + LOG(ERROR) << "Fail to create bthread"; + return -1; + } + // heartbeat_check_first = true; + } + LOG(NOTICE) << "qemu pid has changed, new pid is: " << pid; + g_oldPid = pid; + closedir(proc); + return 0; + } + } + } + + g_oldPid = -1; // reset old pid of checking proc + closedir(proc); + return -1; + } + // proc下面进程文件存在，则表示进程还存在 + char proc_file[32]; + snprintf(proc_file, sizeof(proc_file), "/proc/%ld", g_oldPid); + if ((access(proc_file, F_OK)) != -1) { + heartbeat_check_first = false; + return 0; + } + LOG(NOTICE) << "reset old pid."; + g_oldPid = -1; // reset old pid of checking proc + return -1; +} diff --git a/src/part2/heartbeat.h b/src/part2/heartbeat.h new file mode 100644 index 0000000000..bba0390ed4 --- /dev/null +++ b/src/part2/heartbeat.h @@ -0,0 +1,43 @@ +/* + * Project: nebd + * File Created: 2019-09-30 + * Author: hzwuhongsong + * Copyright (c) 2019 NetEase + */ + +#ifndef SRC_PART2_HEARTBEAT_H_ +#define SRC_PART2_HEARTBEAT_H_ +#include +/** + * @brief 心跳线程 + */ +void* HeartbeatProcess(void*); +/** + * @brief 响应SIGINT，SIGTERM信号 + */ +void SigProcess(int sig_no); +/** + * @brief 响应SIGHUP信号 + */ +void SigLogReplace(int sig_no); +/** + * @brief 关闭所有已经卸载的卷线程 + */ +void* CloseDetachedVolumesProcess(void*); +/** + * @brief 停止part2 + */ +int StopProc(bool need_check_qemu_proc); +/** + * @brief 关闭所有已经卸载的卷 + * @param detachedTimes 检查次数。检查次数超过其才会真的卸载卷 + */ +int CloseQemuDetachedVolumes(int detachedTimes); +/** + * @brief 检查qemu进程是否存在 + * @param uuid qemu进程对应的uuid + */ +int CheckProc(const char* uuid); +char* SkipSpace(char* in, int* offset, int len); +int CheckCmdline(int64_t pid, const char* proc_name, char* uuid, int uuid_len); +#endif // SRC_PART2_HEARTBEAT_H_ diff --git a/src/part2/interrupt_sleep.h b/src/part2/interrupt_sleep.h new file mode 100644 index 0000000000..f961bf5a19 --- /dev/null +++ b/src/part2/interrupt_sleep.h @@ -0,0 +1,39 @@ +/* + * Project: nebd + * File Created: 2019-09-30 + * Author: hzwuhongsong + * Copyright (c) 2019 NetEase + */ + +#ifndef SRC_PART2_INTERRUPT_SLEEP_H_ +#define SRC_PART2_INTERRUPT_SLEEP_H_ + +#include +#include +// #include +#include // NOLINT +// #include +// #include + +class InterruptibleSleeper { + public: + // returns false if killed: + template + bool wait_for(std::chrono::duration const& time ) { + std::unique_lock lock(m); + return !cv.wait_for(lock, time, [&]{return terminate;}); + } + + void interrupt() { + std::unique_lock lock(m); + terminate = true; + LOG(ERROR) << "interrupt sleep."; + cv.notify_all(); + } + private: + std::condition_variable cv; + std::mutex m; + bool terminate = false; +}; + +#endif // SRC_PART2_INTERRUPT_SLEEP_H_ diff --git a/src/part2/main.cpp b/src/part2/main.cpp new file mode 100644 index 0000000000..ff6ec82441 --- /dev/null +++ b/src/part2/main.cpp @@ -0,0 +1,68 @@ +/* + * Project: nebd + * File Created: 2019-09-30 + * Author: hzwuhongsong + * Copyright (c) 2019 NetEase + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/part2/heartbeat.h" +#include "src/part2/rados_interface.h" +#include "src/part2/reload.h" +#include "src/part2/rpc_server.h" +#include "src/part2/config.h" +#include "src/common/client.pb.h" +#define BOOST_SPIRIT_THREADSAFE + +int main(int argc, char* argv[]) { + // 使进程为守护进程 + if (daemon(0, 0) < 0) { + char buffer[128]; + const char *err = "create daemon failed."; + snprintf(buffer, sizeof(buffer), + "echo %s >> /var/log/nebd/nebd-server-initfailed.log", err); + ::system(buffer); + return -1; + } + // 解析参数 + if (argc != 3) { + char buffer[128]; + const char *err = "Incorrect parameter counts."; + snprintf(buffer, sizeof(buffer), + "echo %s >> /var/log/nebd/nebd-server-initfailed.log", err); + ::system(buffer); + return -1; + } + if (strcmp(argv[1], "-uuid") != 0) { + char buffer[128]; + const char *err = "uuid is not given."; + snprintf(buffer, sizeof(buffer), + "echo %s >> /var/log/nebd/nebd-server-initfailed.log", err); + ::system(buffer); + return -1; + } + g_uuid = argv[2]; + int ret = Init(); + if (ret < 0) { + char buffer[128]; + const char *err = "init fialed."; + snprintf(buffer, sizeof(buffer), + "echo %s >> /var/log/nebd/nebd-server-initfailed.log", err); + ::system(buffer); + return -1; + } + return 0; +} diff --git a/src/part2/rados_interface.cpp b/src/part2/rados_interface.cpp new file mode 100644 index 0000000000..259fa6a8df --- /dev/null +++ b/src/part2/rados_interface.cpp @@ -0,0 +1,173 @@ +/* + * Project: nebd + * File Created: 2019-09-30 + * Author: hzwuhongsong + * Copyright (c) 2019 NetEase + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/part2/rados_interface.h" +#include "src/part2/common_type.h" +#include "src/part2/config.h" + +int OpenImage(rados_t* cluster, const char* poolname, const char* volname, + int fd, char* filename) { + LOG(NOTICE) << "open image start. " << filename; + + rados_ioctx_t* io = new rados_ioctx_t; + int ret; + ret = rados_ioctx_create(*cluster, poolname, io); + if (ret < 0) { + LOG(ERROR) << "create ioctx failed. " << poolname << ", " << volname; + delete filename; + return ret; + } + rbd_image_t* image = new rbd_image_t; + + ret = rbd_open(*io, volname, image, NULL); + if (ret < 0) { + LOG(ERROR) << "open image failed."; + delete image; + delete filename; + rados_ioctx_destroy(*io); + return ret; + } + + if (fd <= 0) { + while (true) { + srand((unsigned) time(NULL)); + unsigned int seed; + fd = rand_r(&seed) % FD_CEPH_MAX; + if (fd == 0) fd++; + LOG(NOTICE) << "fd is: " << fd; + if (!g_imageMap.count(fd)) { + break; + } + } + } + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.insert(std::pair(fd, fd_image)); + + LOG(NOTICE) << "open image success."; + return fd; +} + +//关闭卷。该函数的调用需加文件锁保护 +int CloseImage(int fd) { + rbd_image_t* image; + std::map::iterator iter; + iter = g_imageMap.find(fd); + + if (iter == g_imageMap.end()) { + LOG(ERROR) << "close image failed, because fd is not exist, fd is: " + << fd; + return -1; + } + LOG(NOTICE) << "fd is exist, fd is: " << fd; + image = iter->second->image; + + rados_ioctx_t* io; + rados_t* cluster; + io = iter->second->io; + cluster = iter->second->cluster; + delete (iter->second->filename); + rbd_close(*image); + rados_ioctx_destroy(*io); + CloseRados(cluster); + // 删除fd内存镜像 + g_imageMap.erase(iter); + + LOG(NOTICE) << "close image success. fd is: " << fd; + + return 0; +} + +rados_t* ConnectRados(const char* mon_host) { + LOG(NOTICE) << "connect rados start. " << mon_host; + int ret; + rados_t* cluster = new rados_t; + ret = rados_create(cluster, "admin"); + if (ret < 0) { + delete cluster; + LOG(ERROR) << "create cluster failed."; + return NULL; + } + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return NULL; + } + std::string ceph_conf = ReadCephConf(cfg); + config_destroy(cfg); + ret = rados_conf_read_file(*cluster, ceph_conf.c_str()); + if (ret < 0) { + LOG(ERROR) << "read config file failed."; + } + ret = rados_conf_set(*cluster, "mon_host", mon_host); + if (ret < 0) { + LOG(ERROR) << "set conf failed."; + CloseRados(cluster); + return NULL; + } + ret = rados_connect(*cluster); + if (ret < 0) { + LOG(ERROR) << "connect rados failed."; + CloseRados(cluster); + return NULL; + } + + LOG(NOTICE) << "connect rados success."; + return cluster; +} + +void CloseRados(rados_t* cluster) { + rados_shutdown(*cluster); + delete cluster; +} + +int FilenameFdExist(char* filename) { + std::map::iterator iter; + + for (iter = g_imageMap.begin(); iter != g_imageMap.end(); iter++) { + LOG(NOTICE) << " fd " + << "filename is: " << iter->second->filename << " " + << filename; + if (strcmp(filename, iter->second->filename) == 0) { + LOG(NOTICE) << " fd exist." + << "filename is: " << iter->second->filename + << ", fd is: " << iter->first; + return iter->first; + } + } + LOG(NOTICE) << " fd is not exist. " + << "filename is: " << filename; + return 0; +} + +bool FdExist(int fd, rbd_image_t** image) { + std::map::iterator iter; + for (iter = g_imageMap.begin(); iter != g_imageMap.end(); iter++) { + if (iter->first == fd) { + LOG(INFO) << "fd is exist: " << fd; + *image = iter->second->image; + return true; + } + } + + LOG(NOTICE) << "fd is not exist: " << fd; + return false; +} diff --git a/src/part2/rados_interface.h b/src/part2/rados_interface.h new file mode 100644 index 0000000000..b9c9821dd3 --- /dev/null +++ b/src/part2/rados_interface.h @@ -0,0 +1,23 @@ +/* + * Project: nebd + * File Created: 2019-09-30 + * Author: hzwuhongsong + * Copyright (c) 2019 NetEase + */ + +#ifndef SRC_PART2_RADOS_INTERFACE_H_ +#define SRC_PART2_RADOS_INTERFACE_H_ + +#include +#include +#include "src/part2/common_type.h" + +int OpenImage(rados_t* cluster, const char* poolname, const char* volname, + int fd, char* filename); +rados_t* ConnectRados(const char* mon_host); +void CloseRados(rados_t* cluster); +int FilenameFdExist(char* filename); +int CloseImage(int fd); +bool FdExist(int fd, rbd_image_t** image); + +#endif // SRC_PART2_RADOS_INTERFACE_H_ diff --git a/src/part2/reload.cpp b/src/part2/reload.cpp new file mode 100644 index 0000000000..b944e791be --- /dev/null +++ b/src/part2/reload.cpp @@ -0,0 +1,121 @@ +/* + * Project: nebd + * File Created: 2019-09-30 + * Author: hzwuhongsong + * Copyright (c) 2019 NetEase + */ + +#include "src/part2/reload.h" +#include +#include +#include +#include +#include +#include +#include "src/part2/rados_interface.h" + +/** + * @brief 从持久化文件中加载ceph卷 + */ +int ReloadCephVolume(int fd, char* filename) { + LOG(NOTICE) << "reload ceph volume start, " << filename; + rados_t* cluster = NULL; + std::vector split_firstly = split(filename, "="); + if (split_firstly.size() <= 1) { + LOG(ERROR) << "filename format is incorrect."; + delete filename; + return -1; + } + + std::vector split_secondly = split(split_firstly[0], ":"); + if (split_secondly.size() <= 1) { + LOG(ERROR) << "filename format is incorrect."; + delete filename; + return -1; + } + + std::vector split_thirdly = split(split_secondly[1], "/"); + if (split_thirdly.size() <= 1) { + LOG(ERROR) << "filename format is incorrect."; + delete filename; + return -1; + } + char* imagename = const_cast(split_thirdly[1].c_str()); + char* poolname = const_cast(split_thirdly[0].c_str()); + std::string mon_host = GetMonHost(filename); + if (mon_host.empty()) { + LOG(ERROR) << "mon host is null."; + delete filename; + return -1; + } + + cluster = ConnectRados(mon_host.c_str()); + if (cluster == NULL) { + LOG(ERROR) << "connect rados failed. " << mon_host; + delete filename; + return -1; + } + int ret = OpenImage(cluster, poolname, imagename, fd, filename); + if (ret < 0) { + LOG(ERROR) << "open image failed. " << poolname << ", " << imagename; + CloseRados(cluster); + delete filename; + return ret; + } + + LOG(NOTICE) << "reload vol success, " << poolname << ", " << imagename; + return 0; +} + +/** + * @brief 从持久化文件中加载卷 + */ +int Reload() { + LOG(NOTICE) << "reload start."; + std::string metadata_file = GetUuidFile(); + boost::property_tree::ptree root, items; + try { + boost::property_tree::read_json< + boost::property_tree::ptree>(metadata_file, root); + } + catch (boost::property_tree::ptree_error pt) { + LOG(ERROR) << "read json throw an exception. " << pt.what(); + return -1; + } + + // 获取port + auto port_exist = root.get_child_optional("port"); + if (!port_exist) { + LOG(ERROR) << "port is not exist."; + return -1; + } + int port = root.get("port", 0); + + auto volumes_exist = root.get_child_optional("volumes"); + if (!volumes_exist) { + return port; + } + + // 遍历所有的volumes + items = root.get_child("volumes"); + for (boost::property_tree::ptree::iterator it = items.begin(); + it != items.end(); ++it) { + int fd = atoi(it->second.get("fd").c_str()); + char* filename_tmp = NULL; + filename_tmp = + const_cast(it->second.get("filename").c_str()); + if (filename_tmp == NULL) { + LOG(ERROR) << "filename is null."; + continue; + } + char* filename = new char[strlen(filename_tmp) + 1](); + snprintf(filename, strlen(filename_tmp) + 1, "%s", filename_tmp); + + if (fd <= FD_CEPH_MAX) { + ReloadCephVolume(fd, filename); + } + } + + LOG(NOTICE) << "reload end."; + return port; +} diff --git a/src/part2/reload.h b/src/part2/reload.h new file mode 100644 index 0000000000..189c97aab1 --- /dev/null +++ b/src/part2/reload.h @@ -0,0 +1,23 @@ +/* + * Project: nebd + * File Created: 2019-09-30 + * Author: hzwuhongsong + * Copyright (c) 2019 NetEase + */ + +#ifndef SRC_PART2_RELOAD_H_ +#define SRC_PART2_RELOAD_H_ + +#include +#include +#include + +int Reload(); +/** + * @brief 从持久化文件中加载ceph卷 + * @param fd 卷对应的fd + * @param filename 卷对应的filename，filename包含集群mon，pool，rbd等信息 + * @return 错误码 + */ +int ReloadCephVolume(int fd, char* filename); +#endif // SRC_PART2_RELOAD_H_ diff --git a/src/part2/rpc_ceph.cpp b/src/part2/rpc_ceph.cpp new file mode 100644 index 0000000000..d3e7016996 --- /dev/null +++ b/src/part2/rpc_ceph.cpp @@ -0,0 +1,385 @@ +/* + * Project: nebd + * File Created: 2019-09-30 + * Author: hzwuhongsong + * Copyright (c) 2019 NetEase + */ + +#include "src/part2/rpc_ceph.h" +#include +#include +#include +#include +#include +#include "src/common/client.pb.h" +#include "src/part2/rpc_request.h" +#include "src/part2/rados_interface.h" +#include "src/part2/common_type.h" + +void RbdFinishAioWrite(rbd_completion_t c, AsyncWrite* write) { + google::protobuf::Closure* done = write->done; + brpc::ClosureGuard done_guard(done); + uint64_t offset = write->request->offset(); + uint64_t size = write->request->size(); + int fd = write->request->fd(); + + int ret = rbd_aio_get_return_value(c); + rbd_aio_release(c); + + if (ret == 0) { + LOG(INFO) << "write success. fd is: " << fd + << ", offset is: " << offset << ", size is: " << size; + write->response->set_retcode(nebd::client::kOK); + } else { + LOG(ERROR) << "write failed. fd is: " << fd << ", offset is: " << offset + << ", size is: " << size; + } + requestForRpcWrite--; + requestForCephWrite--; + delete write; +} + +void RbdFinishAioRead(rbd_completion_t c, AsyncRead* read) { + google::protobuf::Closure* done = read->done; + brpc::ClosureGuard done_guard(done); + uint64_t offset = read->request->offset(); + uint64_t size = read->request->size(); + int fd = read->request->fd(); + + uint64_t ret = rbd_aio_get_return_value(c); + rbd_aio_release(c); + + if (ret > 0) { + LOG(INFO) << "read success. fd is: " << fd << ", offset is: " << offset + << ", request size is: " << size + << ", actual size is: " << ret; + if (ret != size) LOG(ERROR) << "read not equal."; + read->response->set_retcode(nebd::client::kOK); + read->cntl->response_attachment().append(read->buf, size); + } else { + LOG(ERROR) << "read failed. fd is: " << fd << ", offset is: " << offset + << ", size is: " << size; + } + + delete read->buf; + delete read; +} + +void RbdFinishAioDiscard(rbd_completion_t c, AsyncDiscard* discard) { + google::protobuf::Closure* done = discard->done; + brpc::ClosureGuard done_guard(done); + uint64_t offset = discard->request->offset(); + uint64_t size = discard->request->size(); + int fd = discard->request->fd(); + + int ret = rbd_aio_get_return_value(c); + rbd_aio_release(c); + + if (ret == 0) { + LOG(INFO) << "discard success. fd is: " << fd + << ", offset is: " << offset << ", size is: " << size; + discard->response->set_retcode(nebd::client::kOK); + } else { + LOG(ERROR) << "discard failed. fd is: " << fd + << ", offset is: " << offset << ", size is: " << size; + } + + delete discard; +} + +void RbdFinishAioFlush(rbd_completion_t c, AsyncFlush* flush) { + google::protobuf::Closure* done = flush->done; + brpc::ClosureGuard done_guard(done); + int fd = flush->request->fd(); + + int ret = rbd_aio_get_return_value(c); + rbd_aio_release(c); + + if (ret == 0) { + LOG(INFO) << "flush success. fd is: " << fd; + flush->response->set_retcode(nebd::client::kOK); + } else { + LOG(ERROR) << "flush failed. fd is: " << fd; + } + + delete flush; +} + +int RpcRequestCeph::OpenFile(char* rpc_filename) { + LOG(ERROR) << "ceph open file start. " << rpc_filename; + char* filename = new char[strlen(rpc_filename) + 1](); + snprintf(filename, strlen(rpc_filename) + 1, "%s", rpc_filename); + + std::vector split_firstly = split(filename, "="); + if (split_firstly.size() <= 1) { + LOG(ERROR) << "filename format is incorrect."; + delete filename; + return -1; + } + + std::vector split_secondly = split(split_firstly[0], ":"); + if (split_secondly.size() <= 1) { + LOG(ERROR) << "filename format is incorrect."; + delete filename; + return -1; + } + + std::vector split_thirdly = split(split_secondly[1], "/"); + if (split_thirdly.size() <= 1) { + LOG(ERROR) << "filename format is incorrect."; + delete filename; + return -1; + } + char* imagename = const_cast(split_thirdly[1].c_str()); + char* poolname = const_cast(split_thirdly[0].c_str()); + + std::string mon_host = GetMonHost(filename); + if (mon_host.empty()) { + LOG(ERROR) << "mon host is null."; + delete filename; + return -1; + } + + // 如果fd已经存在，则直接返回 + int fd; + fd = FilenameFdExist(filename); + if (fd > 0) { + LOG(ERROR) << "fd is already exist."; + delete filename; + return fd; + } + + rados_t* cluster = NULL; + cluster = ConnectRados(mon_host.c_str()); + if (cluster == NULL) { + LOG(ERROR) << "connect rados failed."; + delete filename; + return -1; + } + + fd = OpenImage(cluster, poolname, imagename, -1, filename); + if (fd < 0) { + LOG(ERROR) << "open image failed."; + CloseRados(cluster); + delete filename; + return -1; + } + std::string uuid_lockfile = GetUuidLockfile().c_str(); + int lockfd = LockFile(uuid_lockfile.c_str()); + if (lockfd < 0) { + LOG(ERROR) << "lock file failed."; + CloseImage(fd); + return -1; + } + int ret = GenerateFd(filename, fd); + if (ret < 0) { + LOG(ERROR) << "generate fd failed."; + CloseImage(fd); + UnlockFile(lockfd); + return -1; + } + UnlockFile(lockfd); + LOG(NOTICE) << "ceph open file end."; + return fd; +} + +int RpcRequestCeph::Write(AsyncWrite* writejob) { + rbd_completion_t c; + int ret; + rbd_image_t* image; + if (!FdExist(writejob->request->fd(), &image)) { + LOG(ERROR) << "fd is not exist."; + return -1; + } + + uint64_t offset = writejob->request->offset(); + uint64_t size = writejob->request->size(); + ret = rbd_aio_create_completion(writejob, + (rbd_callback_t)RbdFinishAioWrite, &c); + if (ret < 0) { + LOG(ERROR) << "create write completion failed"; + return -1; + } + + requestForCephWrite++; + ret = rbd_aio_write(*image, offset, size, writejob->buf.c_str(), c); + if (ret < 0) { + LOG(ERROR) << "write image failed"; + return -1; + } + return 0; +} + +int RpcRequestCeph::Read(AsyncRead* readjob) { + rbd_completion_t c; + uint64_t offset = readjob->request->offset(); + uint64_t size = readjob->request->size(); + + rbd_image_t* image; + if (!FdExist(readjob->request->fd(), &image)) { + LOG(ERROR) << "fd is not exist."; + return -1; + } + + char* buf = new char[size + 1](); + buf[size] = '\0'; + readjob->buf = buf; + + int ret = rbd_aio_create_completion( + readjob, (rbd_callback_t)RbdFinishAioRead, &c); + if (ret < 0) { + LOG(ERROR) << "create read completion failed"; + return -1; + } + ret = rbd_aio_read(*image, offset, size, buf, c); + if (ret < 0) { + LOG(ERROR) << "read failed."; + delete buf; + return -1; + } + return 0; +} + +int RpcRequestCeph::StatFile(int fd, nebd::client::StatFileResponse* response) { + rbd_image_t* image; + if (!FdExist(fd, &image)) { + LOG(ERROR) << "fd is not exist."; + return -1; + } + + rbd_image_info_t info; + int ret; + ret = rbd_stat(*image, &info, sizeof(info)); + if (ret < 0) { + return ret; + } + response->set_retcode(nebd::client::kOK); + response->set_size(info.size); + return 0; +} + +int RpcRequestCeph::GetInfo(int fd, nebd::client::GetInfoResponse* response) { + rbd_image_t* image; + if (!FdExist(fd, &image)) { + LOG(ERROR) << "fd is not exist."; + return -1; + } + + rbd_image_info_t info; + int ret; + ret = rbd_stat(*image, &info, sizeof(info)); + if (ret < 0) { + return ret; + } + response->set_retcode(nebd::client::kOK); + response->set_objsize(info.obj_size); + + return 0; +} + +int RpcRequestCeph::CloseFile(int fd) { + std::string uuid_lockfile = GetUuidLockfile(); + int lockfd = LockFile(uuid_lockfile.c_str()); + if (lockfd < 0) { + LOG(ERROR) << "add lock failed."; + return -1; + } + + int ret = CloseImage(fd); + if (ret < 0) { + UnlockFile(lockfd); + LOG(ERROR) << "close image failed."; + return -1; + } + + ret = RmFd(fd); + if (ret < 0) { + UnlockFile(lockfd); + LOG(ERROR) << "rm fd failed."; + return -1; + } + + UnlockFile(lockfd); + + return ret; +} + +int RpcRequestCeph::Discard(AsyncDiscard* discardjob) { + rbd_completion_t c; + int ret; + rbd_image_t* image; + if (!FdExist(discardjob->request->fd(), &image)) { + LOG(ERROR) << "fd is not exist."; + return -1; + } + + uint64_t offset = discardjob->request->offset(); + uint64_t size = discardjob->request->size(); + ret = rbd_aio_create_completion(discardjob, + (rbd_callback_t)RbdFinishAioDiscard, &c); + if (ret < 0) { + LOG(ERROR) << "create discard completion failed."; + return -1; + } + + ret = rbd_aio_discard(*image, offset, size, c); + if (ret < 0) { + LOG(ERROR) << "aio_discard failed."; + return -1; + } + return 0; +} + +int RpcRequestCeph::Flush(AsyncFlush* flushjob) { + rbd_completion_t c; + int ret; + rbd_image_t* image; + if (!FdExist(flushjob->request->fd(), &image)) { + LOG(ERROR) << "fd is not exist."; + return -1; + } + + ret = rbd_aio_create_completion(flushjob, + (rbd_callback_t)RbdFinishAioFlush, &c); + if (ret < 0) { + LOG(ERROR) << "create flush completion failed."; + return -1; + } + + ret = rbd_aio_flush(*image, c); + if (ret < 0) { + LOG(ERROR) << "aio flush failed."; + return -1; + } + return 0; +} + +int RpcRequestCeph::Resize(int fd, uint64_t size) { + int ret; + rbd_image_t* image; + if (!FdExist(fd, &image)) { + LOG(ERROR) << "fd is not exist."; + return -1; + } + + ret = rbd_resize(*image, size); + if (ret < 0) { + LOG(ERROR) << "rbd resize failed."; + } + return ret; +} + +int RpcRequestCeph::InvalidateCache(int fd) { + int ret; + rbd_image_t* image; + if (!FdExist(fd, &image)) { + LOG(ERROR) << "fd is not exist."; + return -1; + } + + ret = rbd_invalidate_cache(*image); + if (ret < 0) { + LOG(ERROR) << "rbd invalidate cache failed."; + } + return ret; +} diff --git a/src/part2/rpc_ceph.h b/src/part2/rpc_ceph.h new file mode 100644 index 0000000000..162615402f --- /dev/null +++ b/src/part2/rpc_ceph.h @@ -0,0 +1,54 @@ +/* + * Project: nebd + * File Created: 2019-09-30 + * Author: hzwuhongsong + * Copyright (c) 2019 NetEase + */ + +#ifndef SRC_PART2_RPC_CEPH_H_ +#define SRC_PART2_RPC_CEPH_H_ + +#include +#include +#include +#include +#include +#include "src/common/client.pb.h" +#include "src/part2/rpc_request.h" + +void RbdFinishAioWrite(rbd_completion_t c, AsyncWrite* write); +void RbdFinishAioRead(rbd_completion_t c, AsyncRead* read); +void RbdFinishAioDiscard(rbd_completion_t c, AsyncDiscard* discard); +void RbdFinishAioFlush(rbd_completion_t c, AsyncFlush* flush); + +/** + * @brief rpc请求对应的ceph处理类 + * @detail + * - Filename中包含了集群mon ip，逻辑池以及卷信息 + * - OpenFile函数根据Filename打开卷，然后就可以对卷进行读写等一系列操作了 + */ +class RpcRequestCeph : public RpcRequest { + public: + RpcRequestCeph() {} + virtual ~RpcRequestCeph() {} + virtual int OpenFile(char* filename); + + virtual int Write(AsyncWrite* writejob); + + virtual int Read(AsyncRead* writejob); + + virtual int StatFile(int fd, nebd::client::StatFileResponse* response); + + virtual int GetInfo(int fd, nebd::client::GetInfoResponse* response); + + virtual int CloseFile(int fd); + + virtual int Flush(AsyncFlush* flushjob); + + virtual int Discard(AsyncDiscard* discardjob); + + virtual int Resize(int fd, uint64_t size); + virtual int InvalidateCache(int fd); +}; + +#endif // SRC_PART2_RPC_CEPH_H_ diff --git a/src/part2/rpc_request.h b/src/part2/rpc_request.h new file mode 100644 index 0000000000..777d6da9b7 --- /dev/null +++ b/src/part2/rpc_request.h @@ -0,0 +1,92 @@ +/* + * Project: nebd + * File Created: 2019-09-30 + * Author: hzwuhongsong + * Copyright (c) 2019 NetEase + */ + +#ifndef SRC_PART2_RPC_REQUEST_H_ +#define SRC_PART2_RPC_REQUEST_H_ + +#include +#include +#include +#include +#include +#include +#include "src/common/client.pb.h" + +struct AsyncWrite { + google::protobuf::Closure* done; + const nebd::client::WriteRequest* request; + nebd::client::WriteResponse* response; + brpc::Controller* cntl; + std::string buf; +}; + +struct AsyncRead { + google::protobuf::Closure* done; + const nebd::client::ReadRequest* request; + nebd::client::ReadResponse* response; + brpc::Controller* cntl; + char* buf; +}; + +struct AsyncDiscard { + google::protobuf::Closure* done; + const nebd::client::DiscardRequest* request; + nebd::client::DiscardResponse* response; +}; + +struct AsyncFlush { + google::protobuf::Closure* done; + const nebd::client::FlushRequest* request; + nebd::client::FlushResponse* response; +}; + +class RpcRequest { + public: + RpcRequest() {} + virtual ~RpcRequest() {} + virtual int OpenFile(char* filename) { + return 0; + } + + virtual int Write(AsyncWrite* writejob) { + return 0; + } + + virtual int Read(AsyncRead* readjob) { + return 0; + } + + virtual int StatFile(int fd, nebd::client::StatFileResponse* response) { + return 0; + } + + virtual int GetInfo(int fd, nebd::client::GetInfoResponse* response) { + return 0; + } + + virtual int CloseFile(int fd) { + return 0; + } + + virtual int Flush(AsyncFlush* flushjob) { + return 0; + } + + virtual int Resize(int fd, uint64_t size) { + return 0; + } + + virtual int Discard(AsyncDiscard* discardjob) { + return 0; + } + + virtual int InvalidateCache(int fd) { + return 0; + } +}; + +#endif // SRC_PART2_RPC_REQUEST_H_ diff --git a/src/part2/rpc_server.cpp b/src/part2/rpc_server.cpp new file mode 100644 index 0000000000..f092e5b8f3 --- /dev/null +++ b/src/part2/rpc_server.cpp @@ -0,0 +1,457 @@ +/* + * Project: nebd + * File Created: 2019-09-30 + * Author: hzwuhongsong + * Copyright (c) 2019 NetEase + */ + +#include "src/part2/rpc_server.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/part2/common_type.h" +#include "src/part2/heartbeat.h" +#include "src/part2/rados_interface.h" +#include "src/part2/rpc_ceph.h" +std::mutex g_mutex; +void QemuClientServiceImpl::OpenFile( + google::protobuf::RpcController* cntl_base, + const nebd::client::OpenFileRequest* request, + nebd::client::OpenFileResponse* response, google::protobuf::Closure* done) { + brpc::ClosureGuard done_guard(done); + + brpc::Controller* cntl = static_cast(cntl_base); + + LOG(INFO) << "Received open request[log_id=" << cntl->log_id() << "] from " + << cntl->remote_side(); + + response->set_retcode(nebd::client::kNoOK); + + char* filename = const_cast(request->filename().c_str()); + + // 如果fd已经存在，则直接返回 + int fd; + fd = FilenameFdExist(filename); + if (fd > 0) { + response->set_fd(fd); + response->set_retcode(nebd::client::kOK); + return; + } + + g_mutex.lock(); + if (!opening_image.empty()) { + std::vector::iterator ret; + ret = std::find(opening_image.begin(), opening_image.end(), + request->filename()); + if (ret != opening_image.end()) { + LOG(WARNING) << "image is opening: " << filename; + g_mutex.unlock(); + while (1) { + bthread_usleep(1000000); + g_mutex.lock(); + std::vector::iterator ret; + ret = std::find(opening_image.begin(), opening_image.end(), + request->filename()); + if (ret != opening_image.end()) { + LOG(WARNING) << "image is opening: " << filename; + g_mutex.unlock(); + } else { + fd = FilenameFdExist(filename); + LOG(WARNING) + << "image is opening success by others: " << filename + << ", " << fd; + g_mutex.unlock(); + response->set_fd(fd); + response->set_retcode(nebd::client::kOK); + return; + } + } + } + } + opening_image.push_back(filename); + g_mutex.unlock(); + + std::vector split_firstly = split(filename, ":"); + std::string block_type = split_firstly[0]; + + RpcRequest* rpc_request = NULL; + if (block_type == "rbd") { + rpc_request = request_ceph; + } else { + LOG(ERROR) << "not recognized block_type: " << block_type; + return; + } + fd = rpc_request->OpenFile(filename); + LOG(WARNING) << "open file, fd is: " << fd; + if (fd > 0) { + response->set_fd(fd); + response->set_retcode(nebd::client::kOK); + } + + g_mutex.lock(); + std::vector::iterator it = opening_image.begin(); + for (; it != opening_image.end();) { + if (*it == request->filename()) { + LOG(WARNING) << "image open success, delete from opening_image, " + << *it; + it = opening_image.erase(it); + } else { + ++it; + } + } + g_mutex.unlock(); +} + +void QemuClientServiceImpl::Write(google::protobuf::RpcController* cntl_base, + const nebd::client::WriteRequest* request, + nebd::client::WriteResponse* response, + google::protobuf::Closure* done) { + requestForRpcWrite++; + requestForRpcWriteCounts++; + brpc::ClosureGuard done_guard(done); + + brpc::Controller* cntl = static_cast(cntl_base); + + LOG(INFO) << "Received write request[log_id=" << cntl->log_id() << "] from " + << cntl->remote_side() << ", offset is: " << request->offset() + << ", len is: " << request->size(); + + response->set_retcode(nebd::client::kNoOK); + + int fd = request->fd(); + rbd_image_t* image; + + if (!FdExist(fd, &image)) { + LOG(ERROR) << "fd is not exist. " << fd; + return; + } + + RpcRequest* rpc_request = NULL; + AsyncWrite* writejob = NULL; + if (fd <= FD_CEPH_MAX) { + writejob = new AsyncWrite; + writejob->done = done; + writejob->response = response; + writejob->request = request; + writejob->cntl = cntl; + writejob->buf = cntl->request_attachment().to_string(); + rpc_request = request_ceph; + } else { + LOG(ERROR) << "not recognized fd: " << fd; + return; + } + if (rpc_request->Write(writejob) < 0) { + LOG(ERROR) << "write failed. " << fd; + delete writejob; + return; + } + done_guard.release(); +} + +void QemuClientServiceImpl::Read(google::protobuf::RpcController* cntl_base, + const nebd::client::ReadRequest* request, + nebd::client::ReadResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard done_guard(done); + + brpc::Controller* cntl = static_cast(cntl_base); + + LOG(INFO) << "Received Read request[log_id=" << cntl->log_id() << "] from " + << cntl->remote_side(); + + response->set_retcode(nebd::client::kNoOK); + + int fd = request->fd(); + rbd_image_t* image; + if (!FdExist(fd, &image)) { + LOG(ERROR) << "fd is not exist. " << fd; + return; + } + + RpcRequest* rpc_request = NULL; + AsyncRead* readjob = NULL; + if (fd <= FD_CEPH_MAX) { + readjob = new AsyncRead; + readjob->done = done; + readjob->response = response; + readjob->request = request; + readjob->cntl = cntl; + rpc_request = request_ceph; + } else { + LOG(ERROR) << "not recognized fd: " << fd; + return; + } + if (rpc_request->Read(readjob) < 0) { + LOG(ERROR) << "read failed. " << fd; + delete readjob; + return; + } + done_guard.release(); +} + +void QemuClientServiceImpl::StatFile( + google::protobuf::RpcController* cntl_base, + const nebd::client::StatFileRequest* request, + nebd::client::StatFileResponse* response, google::protobuf::Closure* done) { + brpc::ClosureGuard done_guard(done); + + brpc::Controller* cntl = static_cast(cntl_base); + + LOG(INFO) << "Received Size request[log_id=" << cntl->log_id() << "] from " + << cntl->remote_side() + << " (attached=" << cntl->request_attachment() << ")"; + + response->set_retcode(nebd::client::kNoOK); + response->set_size(0); + + int fd = request->fd(); + rbd_image_t* image; + if (!FdExist(fd, &image)) { + LOG(ERROR) << "fd is not exist. " << fd; + return; + } + + RpcRequest* rpc_request = NULL; + if (fd <= FD_CEPH_MAX) { + rpc_request = request_ceph; + } else { + LOG(ERROR) << "not recognized fd: " << fd; + return; + } + rpc_request->StatFile(fd, response); +} + +void QemuClientServiceImpl::GetInfo(google::protobuf::RpcController* cntl_base, + const nebd::client::GetInfoRequest* request, + nebd::client::GetInfoResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard done_guard(done); + + brpc::Controller* cntl = static_cast(cntl_base); + + LOG(INFO) << "Received Size request[log_id=" << cntl->log_id() << "] from " + << cntl->remote_side() + << " (attached=" << cntl->request_attachment() << ")"; + + response->set_retcode(nebd::client::kNoOK); + response->set_objsize(0); + + int fd = request->fd(); + rbd_image_t* image; + if (!FdExist(fd, &image)) { + LOG(ERROR) << "fd is not exist. " << fd; + return; + } + + RpcRequest* rpc_request = NULL; + if (fd <= FD_CEPH_MAX) { + rpc_request = request_ceph; + } else { + LOG(ERROR) << "not recognized fd: " << fd; + return; + } + rpc_request->GetInfo(fd, response); + return; +} + +void QemuClientServiceImpl::Flush(google::protobuf::RpcController* cntl_base, + const nebd::client::FlushRequest* request, + nebd::client::FlushResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard done_guard(done); + + brpc::Controller* cntl = static_cast(cntl_base); + + LOG(INFO) << "Received Flush request[log_id=" << cntl->log_id() << "] from " + << cntl->remote_side() + << " (attached=" << cntl->request_attachment() << ")"; + + response->set_retcode(nebd::client::kNoOK); + + int fd = request->fd(); + rbd_image_t* image; + if (!FdExist(fd, &image)) { + LOG(ERROR) << "fd is not exist. " << fd; + return; + } + + RpcRequest* rpc_request = NULL; + AsyncFlush* flushjob = NULL; + if (fd <= FD_CEPH_MAX) { + flushjob = new AsyncFlush; + flushjob->done = done; + flushjob->response = response; + flushjob->request = request; + rpc_request = request_ceph; + } else { + LOG(ERROR) << "not recognized fd: " << fd; + return; + } + if (rpc_request->Flush(flushjob) < 0) { + LOG(ERROR) << "flush failed. " << fd; + delete flushjob; + return; + } + done_guard.release(); +} + +void QemuClientServiceImpl::CloseFile( + google::protobuf::RpcController* cntl_base, + const nebd::client::CloseFileRequest* request, + nebd::client::CloseFileResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard done_guard(done); + + brpc::Controller* cntl = static_cast(cntl_base); + + LOG(INFO) << "Received Close request[log_id=" << cntl->log_id() << "] from " + << cntl->remote_side(); + + response->set_retcode(nebd::client::kNoOK); + + int fd = request->fd(); + rbd_image_t* image; + if (!FdExist(fd, &image)) { + LOG(ERROR) << "fd is not exist. " << fd; + return; + } + + RpcRequest* rpc_request = NULL; + if (fd <= FD_CEPH_MAX) { + rpc_request = request_ceph; + } else { + LOG(ERROR) << "not recognized fd: " << fd; + return; + } + + if (rpc_request->CloseFile(fd) == 0) { + response->set_retcode(nebd::client::kOK); + } + return; +} + +void QemuClientServiceImpl::Discard(google::protobuf::RpcController* cntl_base, + const nebd::client::DiscardRequest* request, + nebd::client::DiscardResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard done_guard(done); + + brpc::Controller* cntl = static_cast(cntl_base); + + LOG(INFO) << "Received Discard request[log_id=" << cntl->log_id() + << "] from " << cntl->remote_side() + << " (attached=" << cntl->request_attachment() << ")"; + + response->set_retcode(nebd::client::kNoOK); + + int fd = request->fd(); + rbd_image_t* image; + if (!FdExist(fd, &image)) { + LOG(ERROR) << "fd is not exist. " << fd; + return; + } + + RpcRequest* rpc_request = NULL; + AsyncDiscard* discardjob = NULL; + if (fd <= FD_CEPH_MAX) { + discardjob = new AsyncDiscard; + discardjob->done = done; + discardjob->response = response; + discardjob->request = request; + rpc_request = request_ceph; + } else { + LOG(ERROR) << "not recognized fd: " << fd; + return; + } + if (rpc_request->Discard(discardjob) < 0) { + LOG(ERROR) << "discard failed. " << fd; + delete discardjob; + return; + } + done_guard.release(); +} + +void QemuClientServiceImpl::ResizeFile( + google::protobuf::RpcController* cntl_base, + const nebd::client::ResizeRequest* request, + nebd::client::ResizeResponse* response, google::protobuf::Closure* done) { + brpc::ClosureGuard done_guard(done); + + brpc::Controller* cntl = static_cast(cntl_base); + + LOG(INFO) << "Received ResizeFile request[log_id=" << cntl->log_id() + << "] from " << cntl->remote_side() + << " (attached=" << cntl->request_attachment() << ")"; + + response->set_retcode(nebd::client::kNoOK); + + int fd = request->fd(); + rbd_image_t* image; + if (!FdExist(fd, &image)) { + LOG(ERROR) << "fd is not exist. " << fd; + return; + } + + RpcRequest* rpc_request = NULL; + if (fd <= FD_CEPH_MAX) { + rpc_request = request_ceph; + } else { + LOG(ERROR) << "not recognized fd: " << fd; + return; + } + + if (rpc_request->Resize(fd, request->newsize()) == 0) { + response->set_retcode(nebd::client::kOK); + } + return; +} + +void QemuClientServiceImpl::InvalidateCache( + google::protobuf::RpcController* cntl_base, + const nebd::client::InvalidateCacheRequest* request, + nebd::client::InvalidateCacheResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard done_guard(done); + + brpc::Controller* cntl = static_cast(cntl_base); + + LOG(INFO) << "Received InvalidateCache request[log_id=" << cntl->log_id() + << "] from " << cntl->remote_side() + << " (attached=" << cntl->request_attachment() << ")"; + + response->set_retcode(nebd::client::kNoOK); + + int fd = request->fd(); + rbd_image_t* image; + if (!FdExist(fd, &image)) { + LOG(ERROR) << "fd is not exist. " << fd; + return; + } + + RpcRequest* rpc_request = NULL; + if (fd <= FD_CEPH_MAX) { // for ceph + rpc_request = request_ceph; + } else { + LOG(ERROR) << "not recognized fd: " << fd; + return; + } + + if (rpc_request->InvalidateCache(fd) >= 0) { + response->set_retcode(nebd::client::kOK); + } +} diff --git a/src/part2/rpc_server.h b/src/part2/rpc_server.h new file mode 100644 index 0000000000..87fd097312 --- /dev/null +++ b/src/part2/rpc_server.h @@ -0,0 +1,80 @@ +/* + * Project: nebd + * File Created: 2019-09-30 + * Author: hzwuhongsong + * Copyright (c) 2019 NetEase + */ + +#ifndef SRC_PART2_RPC_SERVER_H_ +#define SRC_PART2_RPC_SERVER_H_ + +#include +#include +#include +#include +#include +#include "src/common/client.pb.h" +#include "src/part2/rpc_ceph.h" + +class QemuClientServiceImpl : public nebd::client::QemuClientService { + public: + QemuClientServiceImpl() {request_ceph = new RpcRequestCeph;} + virtual ~QemuClientServiceImpl() {delete request_ceph;} + virtual void OpenFile(google::protobuf::RpcController* cntl_base, + const nebd::client::OpenFileRequest* request, + nebd::client::OpenFileResponse* response, + google::protobuf::Closure* done); + + virtual void Write(google::protobuf::RpcController* cntl_base, + const nebd::client::WriteRequest* request, + nebd::client::WriteResponse* response, + google::protobuf::Closure* done); + + virtual void Read(google::protobuf::RpcController* cntl_base, + const nebd::client::ReadRequest* request, + nebd::client::ReadResponse* response, + google::protobuf::Closure* done); + + virtual void StatFile(google::protobuf::RpcController* cntl_base, + const nebd::client::StatFileRequest* request, + nebd::client::StatFileResponse* response, + google::protobuf::Closure* done); + + virtual void GetInfo(google::protobuf::RpcController* cntl_base, + const nebd::client::GetInfoRequest* request, + nebd::client::GetInfoResponse* response, + google::protobuf::Closure* done); + + virtual void Flush(google::protobuf::RpcController* cntl_base, + const nebd::client::FlushRequest* request, + nebd::client::FlushResponse* response, + google::protobuf::Closure* done); + + virtual void CloseFile(google::protobuf::RpcController* cntl_base, + const nebd::client::CloseFileRequest* request, + nebd::client::CloseFileResponse* response, + google::protobuf::Closure* done); + + virtual void Discard(google::protobuf::RpcController* cntl_base, + const nebd::client::DiscardRequest* request, + nebd::client::DiscardResponse* response, + google::protobuf::Closure* done); + + virtual void ResizeFile(google::protobuf::RpcController* cntl_base, + const nebd::client::ResizeRequest* request, + nebd::client::ResizeResponse* response, + google::protobuf::Closure* done); + + virtual void InvalidateCache(google::protobuf::RpcController* cntl_base, + const nebd::client::InvalidateCacheRequest* request, + nebd::client::InvalidateCacheResponse* response, + google::protobuf::Closure* done); + + void SetRpcRequest(RpcRequestCeph *request) {request_ceph = request;} + + private: + RpcRequestCeph* request_ceph; + std::vector opening_image; +}; + +#endif // SRC_PART2_RPC_SERVER_H_ diff --git a/tests/part2/TestCommon.cpp b/tests/part2/TestCommon.cpp new file mode 100644 index 0000000000..f6a71ae14f --- /dev/null +++ b/tests/part2/TestCommon.cpp @@ -0,0 +1,477 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/part2/common_type.h" +#include "tests/part2/test_common.h" + +TEST(ReadQemuXmls, ReadQemuXmls_1) { + NebdServerMocker mock; + + // EXPECT_CALL(mock, InitConfig()).WillOnce(testing::ReturnNull()); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("rm -r /etc/nebd"); + EXPECT_EQ(-1, ReadQemuXmls()); +} + +TEST(ReadQemuXmls, ReadQemuXmls_2) { + NebdServerMocker mock; + ::system("mkdir -p /etc/nebd"); + ::system("touch /etc/nebd/nebd-server.conf"); + // std::string xml_dir = "/var/run/nebd/nebd-test"; + std::string xml_dir = "/111/222"; + EXPECT_CALL(mock, ReadQemuXmlDir(::testing::_)) + .WillOnce(testing::Return(xml_dir)); + // EXPECT_CALL(mock, ReadQemuXml(::testing::_)).WillOnce(testing::Return()); + + EXPECT_EQ(-1, ReadQemuXmls()); +} + +TEST(ReadQemuXmls, ReadQemuXmls_3) { + NebdServerMocker mock; + + ::system("rm -rf /etc/nebd"); + ::system("mkdir -p /etc/nebd"); + ::system("touch /etc/nebd/nebd-server.conf"); + std::string xml_dir = "/tmp/nebd-test"; + ::system("mkdir /tmp/nebd-test"); + ::system("touch /tmp/nebd-test/test"); + + boost::property_tree::ptree pt; + pt.add("name", "nebd"); + write_xml("/tmp/nebd-test/test.xml", pt); + + EXPECT_CALL(mock, ReadQemuXmlDir(::testing::_)) + .WillOnce(testing::Return(xml_dir)); + EXPECT_CALL(mock, ReadQemuXml(::testing::_)).WillOnce(testing::Return()); + + EXPECT_EQ(0, ReadQemuXmls()); +} + +TEST(GenerateFd, GenerateFd_1) { + ::system("rm /tmp/unit_test"); + NebdServerMocker mock; + std::string metadata_file = "/111/222/333"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + + std::string filename = "unittestnew"; + EXPECT_EQ(-1, GenerateFd(const_cast(filename.c_str()), 2)); +} +/* +TEST(GenerateFd, GenerateFd_2) { + NebdServerMocker mock; + ::system("rm /tmp/unit_test"); + std::string metadata_file = "/tmp/unit_test"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + + boost::property_tree::ptree root; + boost::property_tree::ptree items; + try { + root.put_child("volumes", items); + + boost::property_tree::write_json(metadata_file, root); + } catch (boost::property_tree::ptree_error pt) { + LOG(ERROR) << "write json throw an exception. " << pt.what(); + return; + } + + std::string filename_input = "unittestnew"; + EXPECT_EQ(0, GenerateFd(const_cast(filename_input.c_str()), 2)); +} + +TEST(GenerateFd, GenerateFd_3) +{ + LOG(ERROR) << "whs start."; + ::system("rm /tmp/unit_test"); + NebdServerMocker mock; + std::string metadata_file = "/tmp/unit_test"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + + boost::property_tree::ptree root; + boost::property_tree::ptree items; + const char* filename = "unittest"; + try { + ptree item1; + item1.put("filename", filename); + item1.put("fd", 1); + items.push_back(std::make_pair("", item1)); + + root.put_child("volumes", items); + + boost::property_tree::write_json(metadata_file, root); + } + catch (ptree_error pt) { + LOG(ERROR) << "write json throw an exception. " << pt.what(); + return; + } + + std::string filename_input = "unittestnew"; + EXPECT_EQ(0, GenerateFd(const_cast(filename_input.c_str()), 2)); +} +*/ + +TEST(RmFd, RmFd_1) { + NebdServerMocker mock; + std::string metadata_file = "/111/222/333"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + + EXPECT_EQ(-1, RmFd(2)); +} + +TEST(RmFd, RmFd_2) { + NebdServerMocker mock; + std::string metadata_file = "/tmp/unit_test"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + + boost::property_tree::ptree root; + boost::property_tree::ptree items; + try { + // ptree item1; + // item1.put("filename", filename); + // item1.put("fd", 1); + // items.push_back(std::make_pair("", item1)); + + root.put_child("port", items); + + boost::property_tree::write_json(metadata_file, root); + } catch (boost::property_tree::ptree_error pt) { + LOG(ERROR) << "write json throw an exception. " << pt.what(); + return; + } + + EXPECT_EQ(-1, RmFd(2)); +} + +/* +TEST(RmFd, RmFd_3) +{ + NebdServerMocker mock; + std::string metadata_file = "/tmp/unit_test2"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + + boost::property_tree::ptree root_tmp; + boost::property_tree::ptree items_tmp; + try { + ptree item1, item2; + item1.put("filename", "file1"); + item1.put("fd", 1); + items_tmp.push_back(std::make_pair("", item1)); + item2.put("filename", "file2"); + item2.put("fd", 2); + items_tmp.push_back(std::make_pair("", item2)); + + root_tmp.put_child("volumes", items_tmp); + + boost::property_tree::write_json(metadata_file, root_tmp); + } + catch (ptree_error pt) { + LOG(ERROR) << "write json throw an exception. " << pt.what(); + return; + } + + EXPECT_EQ(-1, RmFd(3)); + +} + +TEST(RmFd, RmFd_4) +{ + NebdServerMocker mock; + std::string metadata_file = "/tmp/unit_test"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + g_port = 6300; + + boost::property_tree::ptree root; + boost::property_tree::ptree items; + try { + ptree item1, item2; + item1.put("filename", "file1"); + item1.put("fd", 1); + items.push_back(std::make_pair("", item1)); + item2.put("filename", "file2"); + item2.put("fd", 2); + items.push_back(std::make_pair("", item2)); + + root.put_child("volumes", items); + + boost::property_tree::write_json(metadata_file, root); + } + catch (ptree_error pt) { + LOG(ERROR) << "write json throw an exception. " << pt.what(); + return; + } + + EXPECT_EQ(0, RmFd(1)); +} +*/ +/* +TEST(FindPort, FindPort_1) +{ + config_t* cfg = new config_t; + g_filePath = "/111/222"; + EXPECT_EQ(-1, FindPort(cfg, 5)); +} + +TEST(FindPort, FindPort_2) +{ + config_t* cfg = new config_t; + g_uuid = 666; + g_filePath = "/tmp/nebd/"; + ::system("mkdir /tmp/nebd"); + std::string metadata_file = "/tmp/nebd/666"; + + boost::property_tree::ptree root; + boost::property_tree::ptree items; + + try { + root.put("port", 10); + boost::property_tree::write_json(metadata_file, root); + } + catch (ptree_error pt) { + LOG(ERROR) << "write json throw an exception. " << pt.what(); + return; + } + + + EXPECT_EQ(-1, FindPort(cfg, 5)); +} + + +TEST(GetUuidFile, GetUuidFile_1) +{ + g_filePath = "/tmp"; + g_uuid = const_cast("666"); + + std::string uuid_file = "/tmp/666"; + EXPECT_STREQ(uuid_file.c_str(), GetUuidFile().c_str()); +} +*/ +TEST(SetCpuAffinity, SetCpuAffinity_1) { EXPECT_EQ(-1, SetCpuAffinity(-1)); } + +TEST(SetCpuAffinity, SetCpuAffinity_2) { EXPECT_EQ(0, SetCpuAffinity(1)); } + +TEST(Init, Init_1) { + ::system("rm /etc/nebd/nebd-server.conf"); + EXPECT_EQ(-1, Init()); +} + +TEST(Init, Init_2) { + ::system("mkdir -p /etc/nebd"); + ::system("touch /etc/nebd/nebd-server.conf"); + std::string uuid = "666"; + g_uuid = const_cast(uuid.c_str()); + NebdServerMocker mock; + EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, Init()); +} + +TEST(Init, Init_3) { + ::system("mkdir -p /etc/nebd"); + ::system("touch /etc/nebd/nebd-server.conf"); + std::string uuid = "666"; + g_uuid = const_cast(uuid.c_str()); + NebdServerMocker mock; + EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_EQ(-1, Init()); +} + +TEST(Init, Init_4) { + ::system("mkdir -p /etc/nebd"); + ::system("touch /etc/nebd/nebd-server.conf"); + std::string uuid = "666"; + g_oldPid = 1; + g_uuid = const_cast(uuid.c_str()); + NebdServerMocker mock; + std::string lockfile = "/tmp/unit_test"; + std::string metadata_file = "/tmp/unit_test2"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); + EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, Init()); +} + +TEST(Init, Init_5) { + ::system("mkdir -p /etc/nebd"); + ::system("touch /etc/nebd/nebd-server.conf"); + ::system("rm /tmp/unit_test"); + ::system("rm /tmp/unit_test2"); + ::system("touch /tmp/unit_test2"); + std::string uuid = "666"; + g_oldPid = 1; + g_uuid = const_cast(uuid.c_str()); + NebdServerMocker mock; + std::string lockfile = "/tmp/unit_test"; + std::string metadata_file = "/tmp/unit_test2"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); + EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, Reload()).WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, Init()); +} + +TEST(Init, Init_6) { + ::system("mkdir -p /etc/nebd"); + ::system("touch /etc/nebd/nebd-server.conf"); + ::system("rm /tmp/unit_test"); + ::system("rm /tmp/unit_test2"); + std::string uuid = "666"; + g_oldPid = 1; + g_uuid = const_cast(uuid.c_str()); + NebdServerMocker mock; + std::string lockfile = "/tmp/unit_test"; + std::string metadata_file = "/tmp/unit_test2"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); + EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, LockFile(::testing::_)) + .Times(2) + .WillOnce(testing::Return(0)) + .WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, Init()); +} + +TEST(Init, Init_7) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /tmp/unit_test"); + ::system("rm /tmp/unit_test2"); + ::system("touch /tmp/unit_test2"); + std::string uuid = "666"; + g_oldPid = 1; + g_uuid = const_cast(uuid.c_str()); + NebdServerMocker mock; + std::string lockfile = "/tmp/unit_test"; + std::string metadata_file = "/tmp/unit_test2"; + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + std::string file_path = "/etc/nebd/nebd-server.conf"; + int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); + std::string str = "retry_counts=0;\n"; + write(fd, str.c_str(), str.size()); + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); + EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, Reload()).WillOnce(testing::Return(6200)); + EXPECT_EQ(-1, Init()); +} +/* +TEST(Init, Init_8) +{ + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + ::system("rm /tmp/unit_test"); + ::system("rm /tmp/unit_test2"); + ::system("touch /tmp/unit_test2"); + std::string uuid = "666"; + g_oldPid = 1; + g_uuid = const_cast(uuid.c_str()); + NebdServerMocker mock; + std::string lockfile = "/tmp/unit_test"; + std::string metadata_file = "/tmp/unit_test2"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); + EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, Reload()).WillOnce(testing::Return(6200)); + EXPECT_EQ(-1, Init()); +} +*/ + +TEST(Init, Init_8) { + ::system("mkdir -p /etc/nebd"); + ::system("touch /etc/nebd/nebd-server.conf"); + ::system("rm /tmp/unit_test"); + ::system("rm /tmp/unit_test2"); + std::string uuid = "666"; + g_oldPid = 1; + g_uuid = const_cast(uuid.c_str()); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + std::string file_path = "/etc/nebd/nebd-server.conf"; + int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); + std::string str = "retry_counts=0;\n"; + write(fd, str.c_str(), str.size()); + NebdServerMocker mock; + std::string lockfile = "/tmp/unit_test"; + std::string metadata_file = "/tmp/unit_test2"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); + EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, FindPort(::testing::_, ::testing::_)) + .WillOnce(testing::Return(6200)); + EXPECT_CALL(mock, LockFile(::testing::_)) + .Times(2) + .WillOnce(testing::Return(0)) + .WillOnce(testing::Return(0)); + EXPECT_EQ(-1, Init()); +} + +TEST(Init, Init_9) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + ::system("rm /tmp/unit_test"); + ::system("rm /tmp/unit_test2"); + std::string uuid = "666"; + g_oldPid = 1; + g_uuid = const_cast(uuid.c_str()); + NebdServerMocker mock; + std::string lockfile = "/tmp/unit_test"; + std::string metadata_file = "/tmp/unit_test2"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); + EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, FindPort(::testing::_, ::testing::_)) + .WillOnce(testing::Return(6200)); + EXPECT_CALL(mock, LockFile(::testing::_)) + .Times(3) + .WillOnce(testing::Return(0)) + .WillOnce(testing::Return(0)) + .WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, Init()); +} + +TEST(Init, Init_10) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + ::system("rm /tmp/unit_test"); + ::system("rm /tmp/unit_test2"); + std::string uuid = "666"; + g_oldPid = 1; + g_uuid = const_cast(uuid.c_str()); + NebdServerMocker mock; + std::string lockfile = "/tmp/unit_test"; + std::string metadata_file = "/tmp/unit_test2"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); + EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, GeneratePort(::testing::_)).WillOnce(testing::Return(-1)); + EXPECT_CALL(mock, FindPort(::testing::_, ::testing::_)) + .WillOnce(testing::Return(6200)); + EXPECT_CALL(mock, LockFile(::testing::_)) + .WillRepeatedly(testing::Return(0)); + EXPECT_EQ(-1, Init()); +} + +int main(int argc, char *argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/part2/TestCommon2.cpp b/tests/part2/TestCommon2.cpp new file mode 100644 index 0000000000..ff6ddd4486 --- /dev/null +++ b/tests/part2/TestCommon2.cpp @@ -0,0 +1,189 @@ +/* + *** Project: nebd + *** File Created: 2019-09-30 + *** Author: hzwuhongsong + *** Copyright (c) 2019 NetEase + ***/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/part2/common_type.h" +#include "tests/part2/test_common2.h" + +TEST(FindPort, FindPort_1) { + config_t* cfg = new config_t; + g_filePath = "/111/222"; + EXPECT_EQ(-1, FindPort(cfg, 5)); +} + +TEST(FindPort, FindPort_2) { + NebdServerMocker mock; + config_t* cfg = new config_t; + g_uuid = const_cast("99999"); + g_filePath = "/tmp/nebd/"; + ::system("mkdir /tmp/nebd"); + ::system("touch /tmp/nebd/99999"); + + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(-1)); + + EXPECT_EQ(-1, FindPort(cfg, 6234)); +} +/* +TEST(FindPort, FindPort_3) +{ + NebdServerMocker mock; + config_t* cfg = new config_t; + //g_uuid = const_cast("99999"); + g_filePath = "/tmp/nebd_test2/"; + ::system("rm -r /tmp/nebd_test2"); + ::system("mkdir /tmp/nebd_test2"); + ::system("touch /tmp/nebd_test2/99999"); + + boost::property_tree::ptree root; + boost::property_tree::ptree items; + + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(1)); + //EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(1)); + + //port_option_t port_option; + //port_option.min_port = 6200; + // port_option.max_port = 6500; + // EXPECT_CALL(mock, read_config_port(::testing::_, ::testing::_)) + // .WillOnce(SetArgReferee<1>(port_option)) + // .WillOnce(testing::Return());; + + EXPECT_EQ(6234, FindPort(cfg, 6234)); +} +*/ +TEST(FindPort, FindPort_4) { + NebdServerMocker mock; + config_t* cfg = new config_t; + g_uuid = const_cast("666"); + g_filePath = "/tmp/nebd/"; + ::system("rm -r /tmp/nebd"); + ::system("mkdir /tmp/nebd"); + ::system("touch /tmp/nebd/666"); + std::string metadata_file = "/tmp/nebd/666"; + + boost::property_tree::ptree root; + boost::property_tree::ptree items; + + try { + root.put("port", 10); + boost::property_tree::write_json(metadata_file, root); + } catch (ptree_error pt) { + LOG(ERROR) << "write json throw an exception. " << pt.what(); + return; + } + + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(1000)); + // EXPECT_CALL(mock, unLockFile(1000)).WillOnce(testing::Return()); + EXPECT_EQ(6234, FindPort(cfg, 6234)); +} +/* +TEST(FindPort, FindPort_5) +{ + NebdServerMocker mock; + config_t* cfg = new config_t; + g_uuid = const_cast("666"); + g_filePath = "/tmp/nebd/"; + ::system("rm -r /tmp/nebd"); + ::system("mkdir /tmp/nebd"); + std::string metadata_file = "/tmp/nebd/666"; + + boost::property_tree::ptree root; + boost::property_tree::ptree items; + + try { + root.put("port", 6234); + boost::property_tree::write_json(metadata_file, root); + } + catch (ptree_error pt) { + LOG(ERROR) << "write json throw an exception. " << pt.what(); + return; + } + + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(1)); + EXPECT_EQ(6235, FindPort(cfg, 6234)); +} + +TEST(FindPort, FindPort_6) +{ + NebdServerMocker mock; + config_t* cfg = new config_t; + g_uuid = const_cast("666"); + g_filePath = "/tmp/nebd/"; + ::system("rm -r /tmp/nebd"); + ::system("mkdir /tmp/nebd"); + std::string metadata_file = "/tmp/nebd/666"; + + boost::property_tree::ptree root; + boost::property_tree::ptree items; + + try { + root.put("port", 7000); + boost::property_tree::write_json(metadata_file, root); + } + catch (ptree_error pt) { + LOG(ERROR) << "write json throw an exception. " << pt.what(); + return; + } + + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(1)); + EXPECT_CALL(mock, unLockFile(1)).WillOnce(testing::Return()); + EXPECT_EQ(0, FindPort(cfg, 7000)); +} +*/ + +TEST(Squeeze, Squeeze_1) { + std::string str = "10.182.30.27\\:6789"; + Squeeze(const_cast(str.c_str()), '\\'); + std::string str_out = "10.182.30.27:6789"; + EXPECT_STREQ(str_out.c_str(), str.c_str()); +} + +TEST(GetMonHost, GetMonHost_1) { + std::string str = ":test"; + std::string host; + std::string host_actual = GetMonHost(str); + EXPECT_STREQ(host.c_str(), host_actual.c_str()); +} + +TEST(GetMonHost, GetMonHost_2) { + std::string str = + "rbd:rbd/" + "ci_rbd_sys_disk:auth_supported=none:mon_host=10.182.30.27\\:6789"; + std::string host = "10.182.30.27:6789"; + std::string host_actual = GetMonHost(str); + EXPECT_STREQ(host.c_str(), host_actual.c_str()); +} + +TEST(GetUuidFile, GetUuidFile_1) { + ::system("rm -r /tmp/nebd"); + g_filePath = "/tmp"; + g_uuid = const_cast("666"); + + std::string uuid_file = "/tmp/666"; + std::string uuid_actual = GetUuidFile(); + EXPECT_STREQ(uuid_file.c_str(), uuid_actual.c_str()); +} +TEST(GetUuidLockfile, GetUuidLockfile_1) { + ::system("rm -r /tmp/nebd"); + g_uuid = const_cast("99988"); + + std::string LockFile = "/tmp/99988"; + std::string lock_actual = GetUuidLockfile(); + EXPECT_STREQ(LockFile.c_str(), lock_actual.c_str()); +} + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/part2/TestCommon3.cpp b/tests/part2/TestCommon3.cpp new file mode 100644 index 0000000000..3e35ff6477 --- /dev/null +++ b/tests/part2/TestCommon3.cpp @@ -0,0 +1,103 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/part2/common_type.h" +#include "tests/part2/test_common.h" + +TEST(LockFile, LockFile_1) { + const char* file = "/111/222/333"; + NebdServerMocker mock; + // EXPECT_CALL(mock, open(::testing::_, ::testing::_, + // ::testing::_)).WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, LockFile(file)); +} + +TEST(LockFile, LockFile_2) { + const char* file = "/tmp/unit_test2"; + + EXPECT_EQ(3, LockFile(file)); +} + +TEST(GeneratePort, GeneratePort_1) { + ::system("rm -rf /etc/nebd"); + ::system("rm -rf /tmp/nebd-test"); + NebdServerMocker mock; + std::string metadata_file = "/111/222.txt"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + + EXPECT_EQ(-1, GeneratePort(2)); +} + +TEST(GeneratePort, GeneratePort_2) { + NebdServerMocker mock; + std::string metadata_file = "/tmp/unit_test"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + + EXPECT_EQ(0, GeneratePort(2)); +} +/* +TEST(GetUuidFile, GetUuidFile_1) +{ + g_filePath = "/tmp"; + g_uuid = const_cast("666"); + + std::string uuid_file = "/tmp/666"; + EXPECT_STREQ(uuid_file.c_str(), GetUuidFile().c_str()); +} + +TEST(SetCpuAffinity, SetCpuAffinity_1) +{ + EXPECT_EQ(-1, SetCpuAffinity(-1)); +} + +TEST(SetCpuAffinity, SetCpuAffinity_2) +{ + EXPECT_EQ(0, SetCpuAffinity(1)); +} +*/ +TEST(GenerateFd, GenerateFd_3) { + LOG(ERROR) << "whs start."; + ::system("rm /tmp/unit_test"); + NebdServerMocker mock; + std::string metadata_file = "/tmp/unit_test"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + + boost::property_tree::ptree root; + boost::property_tree::ptree items; + const char* filename = "unittest"; + try { + ptree item1; + item1.put("filename", filename); + item1.put("fd", 1); + items.push_back(std::make_pair("", item1)); + + root.put_child("volumes", items); + + boost::property_tree::write_json(metadata_file, root); + } catch (ptree_error pt) { + LOG(ERROR) << "write json throw an exception. " << pt.what(); + return; + } + + std::string filename_input = "unittestnew"; + EXPECT_EQ(0, GenerateFd(const_cast(filename_input.c_str()), 2)); +} + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/part2/TestConfig.cpp b/tests/part2/TestConfig.cpp new file mode 100644 index 0000000000..14dd592ccb --- /dev/null +++ b/tests/part2/TestConfig.cpp @@ -0,0 +1,331 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/part2/rados_interface.h" +#include "tests/part2/test_rados.h" + +TEST(ReadRpcOption, ReadRpcOption_1) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + rpc_option_t option; + ReadRpcOption(cfg, &option); + EXPECT_EQ(2, option.brpc_num_threads); +} + +TEST(ReadRpcOption, ReadRpcOption_2) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + std::string file_path = "/etc/nebd/nebd-server.conf"; + int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); + std::string str = + "brpc_num_threads=2;\nbrpc_idle_timeout=2;\nbrpc_max_concurrency=2;" + "\nbrpc_methodmax_concurrency=2;\n"; + write(fd, str.c_str(), str.size()); + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + rpc_option_t option; + ReadRpcOption(cfg, &option); + EXPECT_EQ(2, option.brpc_idle_timeout); +} + +TEST(ReadPort, ReadPort_1) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + // NebdServerMocker mock; + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + port_option_t option; + ReadPort(cfg, &option); + EXPECT_EQ(6200, option.min_port); +} + +TEST(ReadPort, ReadPort_2) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + std::string file_path = "/etc/nebd/nebd-server.conf"; + int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); + std::string str = "min_port = 6203;\n"; + write(fd, str.c_str(), str.size()); + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + port_option_t option; + ReadPort(cfg, &option); + EXPECT_EQ(6203, option.min_port); +} + +TEST(ReadLogLevel, ReadLogLevel_1) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + // NebdServerMocker mock; + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + EXPECT_EQ(1, ReadLogLevel(cfg)); +} + +TEST(ReadLogLevel, ReadLogLevel_2) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + std::string file_path = "/etc/nebd/nebd-server.conf"; + int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); + std::string str = "loglevel = 2;\n"; + write(fd, str.c_str(), str.size()); + + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + EXPECT_EQ(2, ReadLogLevel(cfg)); +} + +TEST(ReadUuidPath, ReadUuidPath_1) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + std::string path = "/var/run/nebd-server/"; + EXPECT_STREQ(path.c_str(), ReadUuidPath(cfg).c_str()); +} + +TEST(ReadDetachedTimes, ReadDetachedTimes_1) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + EXPECT_EQ(6, ReadDetachedTimes(cfg)); +} + +TEST(ReadDetachedTimes, ReadDetachedTimes_2) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + std::string file_path = "/etc/nebd/nebd-server.conf"; + int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); + std::string str = "detached_times = 2;\n"; + write(fd, str.c_str(), str.size()); + + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + EXPECT_EQ(2, ReadDetachedTimes(cfg)); +} + +TEST(ReadRetryCounts, ReadRetryCounts_1) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + EXPECT_EQ(3, ReadRetryCounts(cfg)); +} + +TEST(ReadRetryCounts, ReadRetryCounts_2) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + std::string file_path = "/etc/nebd/nebd-server.conf"; + int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); + std::string str = "retry_counts = 2;\n"; + write(fd, str.c_str(), str.size()); + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + EXPECT_EQ(2, ReadRetryCounts(cfg)); +} + +TEST(ReadLockPortFile, ReadLockPortFile_1) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + + std::string path = "/tmp/nebd-server.port.file.lock"; + EXPECT_STREQ(path.c_str(), ReadLockPortFile(cfg).c_str()); +} + +TEST(ReadCephConf, ReadCephConf_1) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + + std::string path = "/etc/ceph/ceph.conf"; + EXPECT_STREQ(path.c_str(), ReadCephConf(cfg).c_str()); +} + +TEST(ReadLogPath, ReadLogPath_1) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + + std::string path = "/var/log/nebd/"; + EXPECT_STREQ(path.c_str(), ReadLogPath(cfg).c_str()); +} + +TEST(ReadQemuXmlDir, ReadQemuXmlDir_1) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + + std::string path = "/var/run/libvirt/qemu/"; + EXPECT_STREQ(path.c_str(), ReadQemuXmlDir(cfg).c_str()); +} + +TEST(ReadHeartbeatInterval, ReadHeartbeatInterval_1) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + EXPECT_EQ(1, ReadHeartbeatInterval(cfg)); +} + +TEST(ReadHeartbeatInterval, ReadHeartbeatInterval_2) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + std::string file_path = "/etc/nebd/nebd-server.conf"; + int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); + std::string str = "heartbeat_interval = 2;\n"; + write(fd, str.c_str(), str.size()); + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + EXPECT_EQ(2, ReadHeartbeatInterval(cfg)); +} + +TEST(ReadAssertTimes, ReadAssertTimes_1) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + EXPECT_EQ(10, ReadAssertTimes(cfg)); +} + +TEST(ReadAssertTimes, ReadAssertTimes_2) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + std::string file_path = "/etc/nebd/nebd-server.conf"; + int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); + std::string str = "check_assert_times = 2;\n"; + write(fd, str.c_str(), str.size()); + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + EXPECT_EQ(2, ReadAssertTimes(cfg)); +} + +TEST(ReadCheckDetachedInterval, ReadCheckDetachedInterval_1) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + EXPECT_EQ(5, ReadCheckDetachedInterval(cfg)); +} + +TEST(ReadCheckDetachedInterval, ReadCheckDetachedInterval_2) { + ::system("mkdir -p /etc/nebd"); + ::system("rm /etc/nebd/nebd-server.conf"); + ::system("touch /etc/nebd/nebd-server.conf"); + std::string file_path = "/etc/nebd/nebd-server.conf"; + int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); + std::string str = "check_detached_interval = 2;\n"; + write(fd, str.c_str(), str.size()); + config_t* cfg = InitConfig(); + if (cfg == NULL) { + LOG(ERROR) << "init config failed."; + return; + } + EXPECT_EQ(2, ReadCheckDetachedInterval(cfg)); +} + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/part2/TestHeartbeat.cpp b/tests/part2/TestHeartbeat.cpp new file mode 100644 index 0000000000..8b62090988 --- /dev/null +++ b/tests/part2/TestHeartbeat.cpp @@ -0,0 +1,221 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/part2/heartbeat.h" +#include "tests/part2/test_heartbeat.h" + +TEST(CloseDetachedVolumesProcess, CloseDetachedVolumesProcess_1) { + ::system("rm /etc/nebd/nebd-server.conf"); + NebdServerMocker mock; + EXPECT_EQ(NULL, CloseDetachedVolumesProcess(NULL)); +} + +TEST(CloseDetachedVolumesProcess, CloseDetachedVolumesProcess_2) { + ::system("mkdir /etc/nebd"); + ::system("touch /etc/nebd/nebd-server.conf"); + NebdServerMocker mock; + EXPECT_CALL(mock, CloseQemuDetachedVolumes(::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_EQ(NULL, CloseDetachedVolumesProcess(NULL)); +} + +TEST(HeartbeatProcess, HeartbeatProcess_1) { + ::system("rm /etc/nebd/nebd-server.conf"); + NebdServerMocker mock; + EXPECT_EQ(NULL, HeartbeatProcess(NULL)); +} + +TEST(SkipSpace, SkipSpace_1) { + ::system("rm /etc/nebd/nebd-server.conf"); + NebdServerMocker mock; + std::string str = "123456"; + std::string str_want = "23456"; + int i = 1; + char* str_out = SkipSpace(const_cast(str.c_str()), &i, 8); + EXPECT_STREQ(str_want.c_str(), str_out); +} + +TEST(StopProc, StopProc_1) { + ::system("rm /tmp/unit_test"); + ::system("rm /tmp/unit_test_lockfile"); + NebdServerMocker mock; + std::string metadata_file = "/tmp/unit_test"; + std::string lock_file = "/tmp/unit_test_lockfile"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, StopProc(0)); +} + +TEST(StopProc, StopProc_2) { + ::system("rm /tmp/unit_test"); + ::system("rm /tmp/unit_test_lockfile"); + NebdServerMocker mock; + std::string metadata_file = "/tmp/unit_test"; + std::string lock_file = "/tmp/unit_test_lockfile"; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 6; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(-1)); + + EXPECT_EQ(-1, StopProc(0)); +} + +TEST(StopProc, StopProc_3) { + ::system("rm /tmp/unit_test"); + ::system("rm /tmp/unit_test_lockfile"); + NebdServerMocker mock; + std::string metadata_file = "/tmp/unit_test"; + std::string lock_file = "/tmp/unit_test_lockfile"; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 6; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); + + EXPECT_EQ(0, StopProc(0)); +} + +TEST(StopProc, StopProc_4) { + ::system("rm /tmp/unit_test"); + ::system("rm /tmp/unit_test_lockfile"); + ::system("touch /tmp/unit_test"); + NebdServerMocker mock; + std::string metadata_file = "/tmp/unit_test"; + std::string lock_file = "/tmp/unit_test_lockfile"; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 6; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); + + EXPECT_EQ(0, StopProc(0)); +} + +TEST(StopProc, StopProc_5) { + ::system("rm /tmp/unit_test"); + ::system("rm /tmp/unit_test_lockfile"); + NebdServerMocker mock; + std::string metadata_file = "/tmp/unit_test"; + std::string lock_file = "/tmp/unit_test_lockfile"; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 6; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(-1)); + + EXPECT_EQ(0, StopProc(1)); +} + +TEST(StopProc, StopProc_6) { + ::system("rm /tmp/unit_test"); + ::system("rm /tmp/unit_test_lockfile"); + ::system("touch /tmp/unit_test"); + NebdServerMocker mock; + std::string metadata_file = "/tmp/unit_test"; + std::string lock_file = "/tmp/unit_test_lockfile"; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 6; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(-1)); + + EXPECT_EQ(0, StopProc(1)); +} + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/part2/TestHeartbeat2.cpp b/tests/part2/TestHeartbeat2.cpp new file mode 100644 index 0000000000..1cb4c62120 --- /dev/null +++ b/tests/part2/TestHeartbeat2.cpp @@ -0,0 +1,218 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/part2/heartbeat.h" +#include "tests/part2/test_heartbeat2.h" + +TEST(CloseQemuDetachedVolumes, CloseQemuDetachedVolumes_1) { + NebdServerMocker mock; + EXPECT_CALL(mock, ReadQemuXmls()).WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, CloseQemuDetachedVolumes(1)); +} + +TEST(CloseQemuDetachedVolumes, CloseQemuDetachedVolumes_2) { + int fd = 6; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + FdImage_t fd_image; + fd_image.filename = filename; + + int fd2 = 7; + char* filename2 = const_cast("mon_host=10.182.30.27"); + FdImage_t fd_image2; + fd_image2.filename = filename2; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, &fd_image)); + g_imageMap.insert(std::pair(fd2, &fd_image2)); + + g_qemuPoolVolumes.push_back("rbd/volume02"); + NebdServerMocker mock; + EXPECT_CALL(mock, ReadQemuXmls()).WillOnce(testing::Return(0)); + std::string lock_file = "/tmp/unit_test_lockfile"; + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); + + EXPECT_EQ(0, CloseQemuDetachedVolumes(3)); +} + +TEST(CloseQemuDetachedVolumes, CloseQemuDetachedVolumes_3) { + int fd = 6; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + FdImage_t fd_image; + fd_image.filename = filename; + + int fd2 = 7; + char* filename2 = const_cast("mon_host=10.182.30.27"); + FdImage_t fd_image2; + fd_image2.filename = filename2; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, &fd_image)); + g_imageMap.insert(std::pair(fd2, &fd_image2)); + + g_qemuPoolVolumes.push_back("rbd/volume02"); + NebdServerMocker mock; + EXPECT_CALL(mock, ReadQemuXmls()).WillOnce(testing::Return(0)); + std::string lock_file = "/tmp/unit_test_lockfile"; + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(-1)); + + EXPECT_EQ(-1, CloseQemuDetachedVolumes(0)); +} + +TEST(CloseQemuDetachedVolumes, CloseQemuDetachedVolumes_4) { + int fd = 6; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + FdImage_t fd_image; + fd_image.filename = filename; + + int fd2 = 7; + char* filename2 = const_cast("mon_host=10.182.30.27"); + FdImage_t fd_image2; + fd_image2.filename = filename2; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, &fd_image)); + g_imageMap.insert(std::pair(fd2, &fd_image2)); + + g_qemuPoolVolumes.push_back("rbd/volume02"); + NebdServerMocker mock; + EXPECT_CALL(mock, ReadQemuXmls()).WillOnce(testing::Return(0)); + std::string lock_file = "/tmp/unit_test_lockfile"; + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(-1)); + + EXPECT_EQ(-1, CloseQemuDetachedVolumes(0)); +} + +TEST(CloseQemuDetachedVolumes, CloseQemuDetachedVolumes_5) { + int fd = 6; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + FdImage_t fd_image; + fd_image.filename = filename; + + int fd2 = 7; + char* filename2 = const_cast("mon_host=10.182.30.27"); + FdImage_t fd_image2; + fd_image2.filename = filename2; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, &fd_image)); + g_imageMap.insert(std::pair(fd2, &fd_image2)); + + g_qemuPoolVolumes.push_back("rbd/volume02"); + NebdServerMocker mock; + EXPECT_CALL(mock, ReadQemuXmls()).WillOnce(testing::Return(0)); + std::string lock_file = "/tmp/unit_test_lockfile"; + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, RmFd(::testing::_)).WillOnce(testing::Return(-1)); + + EXPECT_EQ(-1, CloseQemuDetachedVolumes(0)); +} +/* +TEST(CloseQemuDetachedVolumes, CloseQemuDetachedVolumes_6) +{ + int fd = 6; + char* filename = +const_cast("rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + FdImage_t fd_image; + fd_image.filename = filename; + + int fd2 = 7; + char* filename2 = const_cast("mon_host=10.182.30.27"); + FdImage_t fd_image2; + fd_image2.filename = filename2; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, &fd_image)); + g_imageMap.insert(std::pair(fd2, &fd_image2)); + + g_qemuPoolVolumes.push_back("rbd/volume02"); + NebdServerMocker mock; + EXPECT_CALL(mock, ReadQemuXmls()).WillOnce(testing::Return(0)); + std::string lock_file = "/tmp/unit_test_lockfile"; + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, UnLockFile(::testing::_)).WillOnce(testing::Return()); + EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, RmFd(::testing::_)).WillOnce(testing::Return(0)); + + EXPECT_EQ(0, CloseQemuDetachedVolumes(0)); +} +*/ +TEST(CloseQemuDetachedVolumes, + CloseQemuDetachedVolumes_7) { + int fd = 6; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + FdImage_t fd_image; + fd_image.filename = filename; + + int fd2 = 7; + char* filename2 = const_cast("mon_host=10.182.30.27"); + FdImage_t fd_image2; + fd_image2.filename = filename2; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, &fd_image)); + g_imageMap.insert(std::pair(fd2, &fd_image2)); + + g_qemuPoolVolumes.push_back("rbd/volume01"); + NebdServerMocker mock; + EXPECT_CALL(mock, ReadQemuXmls()).WillOnce(testing::Return(0)); + std::string lock_file = "/tmp/unit_test_lockfile"; + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); + + EXPECT_EQ(0, CloseQemuDetachedVolumes(3)); +} + +TEST(CheckCmdline, CheckCmdline_1) { + char uuid[5]; + NebdServerMocker mock; + // EXPECT_CALL(mock, open(::testing::_, ::testing::_, + // ::testing::_)).WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, CheckCmdline(6666666, "test", uuid, 4)); +} + +TEST(CheckCmdline, CheckCmdline_2) { + char uuid[5]; + NebdServerMocker mock; + // EXPECT_CALL(mock, open(::testing::_, ::testing::_, + // ::testing::_)).WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, CheckCmdline(1, "test", uuid, 4)); +} + +TEST(CheckProc, CheckProc_1) { + g_oldPid = 66666; + EXPECT_EQ(-1, CheckProc("test")); +} + +TEST(CheckProc, CheckProc_2) { + g_oldPid = 1; + EXPECT_EQ(0, CheckProc("test")); +} + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/part2/TestHeartbeat3.cpp b/tests/part2/TestHeartbeat3.cpp new file mode 100644 index 0000000000..56e701808c --- /dev/null +++ b/tests/part2/TestHeartbeat3.cpp @@ -0,0 +1,43 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/part2/heartbeat.h" +#include "tests/part2/test_heartbeat3.h" + +TEST(CheckProc, CheckProc_1) { + g_oldPid = 66666; + EXPECT_EQ(-1, CheckProc("test")); +} + +TEST(CheckProc, CheckProc_2) { + g_oldPid = 1; + EXPECT_EQ(0, CheckProc("test")); +} + +TEST(CheckProc, CheckProc_3) { + g_oldPid = -1; + NebdServerMocker mock; + EXPECT_CALL(mock, CheckCmdline(::testing::_, ::testing::_, ::testing::_, + ::testing::_)) + .WillRepeatedly(testing::Return(-1)); + EXPECT_EQ(-1, CheckProc("test")); +} + +int main(int argc, char *argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/part2/TestRados.cpp b/tests/part2/TestRados.cpp new file mode 100644 index 0000000000..576e695196 --- /dev/null +++ b/tests/part2/TestRados.cpp @@ -0,0 +1,334 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/part2/rados_interface.h" +#include "tests/part2/test_rados.h" + +TEST(OpenImage, OpenImage_1) { + rados_t cluster; + const char* poolname = "rbd"; + const char* volname = "volume01"; + int fd = 4; + char* filename = new char[5](); + filename[0] = '1'; + NebdServerMocker mock; + EXPECT_CALL(mock, + rados_ioctx_create(::testing::_, ::testing::_, ::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, OpenImage(&cluster, poolname, volname, fd, filename)); +} + +TEST(OpenImage, OpenImage_2) { + rados_t cluster; + const char* poolname = "rbd"; + const char* volname = "volume01"; + int fd = 4; + char* filename = new char[5](); + filename[0] = '1'; + NebdServerMocker mock; + EXPECT_CALL(mock, + rados_ioctx_create(::testing::_, ::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL( + mock, rbd_open(::testing::_, ::testing::_, ::testing::_, ::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_CALL(mock, rados_ioctx_destroy(::testing::_)) + .WillOnce(testing::Return()); + EXPECT_EQ(-1, OpenImage(&cluster, poolname, volname, fd, filename)); +} + +TEST(OpenImage, OpenImage_3) { + rados_t cluster; + const char* poolname = "rbd"; + const char* volname = "volume01"; + int fd = 4; + char* filename = new char[5](); + filename[0] = '1'; + NebdServerMocker mock; + EXPECT_CALL(mock, + rados_ioctx_create(::testing::_, ::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL( + mock, rbd_open(::testing::_, ::testing::_, ::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_EQ(4, OpenImage(&cluster, poolname, volname, fd, filename)); +} + +TEST(CloseImage, CloseImage_1) { + g_imageMap.clear(); + EXPECT_EQ(-1, CloseImage(4)); +} + +TEST(CloseImage, CloseImage_2) { + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = new char[5](); + filename[0] = '1'; + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + NebdServerMocker mock; + EXPECT_CALL(mock, rados_ioctx_destroy(::testing::_)) + .WillOnce(testing::Return()); + EXPECT_CALL(mock, CloseRados(::testing::_)).WillOnce(testing::Return()); + EXPECT_CALL(mock, rbd_close(::testing::_)).WillOnce(testing::Return(0)); + + EXPECT_EQ(0, CloseImage(4)); +} + +TEST(ConnectRados, ConnectRados_1) { + NebdServerMocker mock; + EXPECT_CALL(mock, rados_create(::testing::_, ::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_EQ(NULL, ConnectRados("test")); +} + +TEST(ConnectRados, ConnectRados_2) { + ::system("rm /etc/nebd/nebd-server.conf"); + NebdServerMocker mock; + EXPECT_CALL(mock, rados_create(::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_EQ(NULL, ConnectRados("test")); +} + +TEST(ConnectRados, ConnectRados_3) { + ::system("mkdir -p /etc/nebd"); + ::system("touch /etc/nebd/nebd-server.conf"); + NebdServerMocker mock; + EXPECT_CALL(mock, rados_create(::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rados_conf_read_file(::testing::_, ::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_CALL(mock, rados_conf_set(::testing::_, ::testing::_, ::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_CALL(mock, CloseRados(::testing::_)).WillOnce(testing::Return()); + EXPECT_EQ(NULL, ConnectRados("test")); +} + +TEST(ConnectRados, ConnectRados_4) { + ::system("mkdir -p /etc/nebd"); + ::system("touch /etc/nebd/nebd-server.conf"); + NebdServerMocker mock; + EXPECT_CALL(mock, rados_create(::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rados_conf_read_file(::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rados_conf_set(::testing::_, ::testing::_, ::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_CALL(mock, CloseRados(::testing::_)).WillOnce(testing::Return()); + EXPECT_EQ(NULL, ConnectRados("test")); +} + +TEST(ConnectRados, ConnectRados_5) { + ::system("mkdir -p /etc/nebd"); + ::system("touch /etc/nebd/nebd-server.conf"); + NebdServerMocker mock; + EXPECT_CALL(mock, rados_create(::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rados_conf_read_file(::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rados_conf_set(::testing::_, ::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, CloseRados(::testing::_)).WillOnce(testing::Return()); + EXPECT_CALL(mock, rados_connect(::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_EQ(NULL, ConnectRados("test")); +} +/* +TEST(ConnectRados, ConnectRados_6) +{ + ::system("mkdir -p /etc/nebd"); + ::system("touch /etc/nebd/nebd-server.conf"); + NebdServerMocker mock; + EXPECT_CALL(mock, rados_create(::testing::_, +::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rados_conf_read_file(::testing::_, +::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rados_conf_set(::testing::_, ::testing::_, +::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, CloseRados(::testing::_)).WillOnce(testing::Return()); + EXPECT_CALL(mock, rados_connect(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_EQ(NULL, ConnectRados("test")); +} +*/ +/* +TEST(open_image, open_image_1) +{ + NebdServerMocker mock; + rados_t* cluster = new rados_t; + const char* poolname = "rbd"; + const char* volname = "volume01"; + int fd = 4; + char* filename = new char[5](); + filename[0] = '1'; + filename[2] = '1'; + + EXPECT_CALL(mock, rados_ioctx_create(::testing::_, ::testing::_, +::testing::_)).WillOnce(testing::Return(-1)); + + EXPECT_EQ(-1, open_image(cluster, poolname, volname, fd, filename)); +} +*/ +/* +TEST(ConnectRados, ConnectRados_1) +{ + NebdServerMocker mock; + rados_t* cluster = new rados_t; + const char* poolname = "rbd"; + const char* volname = "volume01"; + int fd = 4; + char* filename = new char[5](); + filename[0] = '1'; + filename[2] = '1'; + + rados_t* rados = NULL; + EXPECT_CALL(mock, rados_create(::testing::_, +::testing::_)).WillOnce(testing::Return(-1)); + const char* mon_host = "ttt"; + EXPECT_EQ(rados, ConnectRados(mon_host)); +} + +TEST(ConnectRados, ConnectRados_2) +{ + NebdServerMocker mock; + rados_t* cluster = new rados_t; + const char* poolname = "rbd"; + const char* volname = "volume01"; + int fd = 4; + char* filename = new char[5](); + filename[0] = '1'; + filename[2] = '1'; + + rados_t* rados = NULL; + //EXPECT_CALL(mock, rados_create(::testing::_, +::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rados_conf_read_file(::testing::_, +::testing::_)).WillOnce(testing::Return(-1)); + const char* mon_host = "ttt"; + EXPECT_EQ(rados, ConnectRados(mon_host)); +} +*/ +TEST(FilenameFdExist, FilenameFdExist_1) { + NebdServerMocker mock; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = new char[5](); + filename[0] = '1'; + filename[1] = '1'; + + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + const char* filename_input = "11"; + EXPECT_EQ(4, FilenameFdExist(const_cast(filename_input))); +} + +TEST(FilenameFdExist, FilenameFdExist_2) { + NebdServerMocker mock; + + g_imageMap.clear(); + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = new char[5](); + filename[0] = '1'; + + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.insert(std::pair(fd, fd_image)); + + const char* filename_input = "11"; + EXPECT_EQ(0, FilenameFdExist(const_cast(filename_input))); +} + +TEST(FdExist, FdExist_1) { + NebdServerMocker mock; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = new char[5](); + filename[0] = '1'; + filename[2] = '1'; + + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.insert(std::pair(fd, fd_image)); + + rbd_image_t* image_ret; + // EXPECT_CALL(mock, rados_ioctx_create(::testing::_, ::testing::_, + // ::testing::_)).WillOnce(testing::Return(-1)); + + EXPECT_TRUE(FdExist(4, &image_ret)); +} + +TEST(FdExist, FdExist_2) { + NebdServerMocker mock; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = new char[5](); + filename[0] = '1'; + filename[2] = '1'; + + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.insert(std::pair(fd, fd_image)); + + rbd_image_t* image_ret; + // EXPECT_CALL(mock, rados_ioctx_create(::testing::_, ::testing::_, + // ::testing::_)).WillOnce(testing::Return(-1)); + + EXPECT_FALSE(FdExist(8, &image_ret)); +} + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/part2/TestReload.cpp b/tests/part2/TestReload.cpp new file mode 100644 index 0000000000..dca5ca7c4c --- /dev/null +++ b/tests/part2/TestReload.cpp @@ -0,0 +1,107 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/part2/reload.h" +#include "tests/part2/test_reload.h" + +TEST(Reload, reload_1) { + NebdServerMocker mock; + std::string metadata_file = "/var/444"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + + EXPECT_EQ(-1, Reload()); +} + +TEST(Reload, reload_2) { + NebdServerMocker mock; + std::string metadata_file = "/tmp/unit_test"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + + boost::property_tree::ptree root; + boost::property_tree::ptree items; + + try { + root.put("volume", 5); + boost::property_tree::write_json(metadata_file, root); + } catch (boost::property_tree::ptree_error pt) { + LOG(ERROR) << "write json throw an exception. " << pt.what(); + return; + } + + EXPECT_EQ(-1, Reload()); +} + +TEST(Reload, reload_3) { + NebdServerMocker mock; + std::string metadata_file = "/tmp/unit_test"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + + boost::property_tree::ptree root; + boost::property_tree::ptree items; + + try { + root.put("port", 10); + boost::property_tree::write_json(metadata_file, root); + } catch (boost::property_tree::ptree_error pt) { + LOG(ERROR) << "write json throw an exception. " << pt.what(); + return; + } + + EXPECT_EQ(10, Reload()); +} + +TEST(Reload, reload_4) { + NebdServerMocker mock; + std::string metadata_file = "/tmp/unit_test"; + EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); + + boost::property_tree::ptree root; + boost::property_tree::ptree items; + + const char* filename1 = ""; + const char* filename2 = "unittest"; + try { + root.put("port", 10); + boost::property_tree::write_json(metadata_file, root); + + boost::property_tree::ptree item1, item2; + item1.put("filename", filename1); + item1.put("fd", 1); + items.push_back(std::make_pair("", item1)); + item2.put("filename", filename2); + item2.put("fd", 2); + items.push_back(std::make_pair("", item2)); + + root.put_child("volumes", items); + + boost::property_tree::write_json(metadata_file, root); + } catch (boost::property_tree::ptree_error pt) { + LOG(ERROR) << "write json throw an exception. " << pt.what(); + return; + } + + EXPECT_CALL(mock, ReloadCephVolume(1, ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, ReloadCephVolume(2, ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_EQ(10, Reload()); +} + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/part2/TestReload2.cpp b/tests/part2/TestReload2.cpp new file mode 100644 index 0000000000..79819f85d8 --- /dev/null +++ b/tests/part2/TestReload2.cpp @@ -0,0 +1,104 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/part2/reload.h" +#include "tests/part2/test_reload2.h" + +TEST(ReloadCephVolume, ReloadCephVolume_1) { + const char* tmp = "rbdrbdvolume03auth_supportednonemon_host=10.182.30.27"; + char* filename = new char[strlen(tmp) + 1](); + snprintf(filename, strlen(tmp) + 1, "%s", tmp); + EXPECT_EQ(-1, ReloadCephVolume(4, filename)); +} + +TEST(ReloadCephVolume, ReloadCephVolume_2) { + const char* tmp = "rbdrbdvolume03auth_supported=nonemon_host=10.182.30.27"; + char* filename = new char[strlen(tmp) + 1](); + snprintf(filename, strlen(tmp) + 1, "%s", tmp); + EXPECT_EQ(-1, ReloadCephVolume(4, filename)); +} + +TEST(ReloadCephVolume, ReloadCephVolume_3) { + const char* tmp = + "rbd:rbdvolume03:auth_supported=none:mon_host=10.182.30.27:6789"; + char* filename = new char[strlen(tmp) + 1](); + snprintf(filename, strlen(tmp) + 1, "%s", tmp); + + EXPECT_EQ(-1, ReloadCephVolume(4, filename)); +} + +TEST(ReloadCephVolume, ReloadCephVolume_4) { + const char* tmp = "rbd:rbd/volume03:auth_supported=none:mon_host"; + char* filename = new char[strlen(tmp) + 1](); + snprintf(filename, strlen(tmp) + 1, "%s", tmp); + + EXPECT_EQ(-1, ReloadCephVolume(4, filename)); +} + +TEST(ReloadCephVolume, ReloadCephVolume_8) { + const char* tmp = + "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; + char* filename = new char[strlen(tmp) + 1](); + snprintf(filename, strlen(tmp) + 1, "%s", tmp); + NebdServerMocker mock; + EXPECT_CALL(mock, ConnectRados(::testing::_)) + .WillOnce(testing::ReturnNull()); + EXPECT_EQ(-1, ReloadCephVolume(4, filename)); +} + +TEST(ReloadCephVolume, ReloadCephVolume_9) { + const char* tmp = + "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; + char* filename = new char[strlen(tmp) + 1](); + snprintf(filename, strlen(tmp) + 1, "%s", tmp); + + NebdServerMocker mock; + + rados_t* cluster = new rados_t; + int ret = rados_create(cluster, "admin"); + if (ret < 0) { + delete cluster; + return; + } + EXPECT_CALL(mock, ConnectRados(::testing::_)) + .WillOnce(testing::Return(cluster)); + EXPECT_CALL(mock, OpenImage(::testing::_, ::testing::_, ::testing::_, 4, + ::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, ReloadCephVolume(4, filename)); +} + +TEST(ReloadCephVolume, ReloadCephVolume_10) { + const char* tmp = + "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; + char* filename = new char[strlen(tmp) + 1](); + snprintf(filename, strlen(tmp) + 1, "%s", tmp); + + NebdServerMocker mock; + rados_t* cluster = new rados_t; + EXPECT_CALL(mock, ConnectRados(::testing::_)) + .WillOnce(testing::Return(cluster)); + EXPECT_CALL(mock, OpenImage(::testing::_, ::testing::_, ::testing::_, 4, + ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_EQ(0, ReloadCephVolume(4, filename)); +} + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/part2/TestRpcCeph.cpp b/tests/part2/TestRpcCeph.cpp new file mode 100644 index 0000000000..6b74fad2c6 --- /dev/null +++ b/tests/part2/TestRpcCeph.cpp @@ -0,0 +1,1252 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/common/client.pb.h" +#include "src/part2/rados_interface.h" +#include "src/part2/rpc_server.h" +#include "tests/part2/test_rpcceph.h" + +class RpcServiceTestClosure : public ::google::protobuf::Closure { + public: + explicit RpcServiceTestClosure(int sleepUs = 0) : sleep_(sleepUs) {} + virtual ~RpcServiceTestClosure() = default; + + void Run() override { + if (0 != sleep_) { + ::usleep(sleep_); + LOG(INFO) << "return rpc"; + } + } + + private: + int sleep_; +}; + +TEST(RpcRequestCephTest, RbdFinishAioWrite_1) { + nebd::client::WriteResponse response; + brpc::Controller cntl; + nebd::client::WriteRequest request; + request.set_fd(4); + request.set_offset(4); + request.set_size(4); + RpcServiceTestClosure done; + AsyncWrite* writejob = NULL; + writejob = new AsyncWrite; + writejob->done = &done; + writejob->response = &response; + writejob->request = &request; + writejob->cntl = &cntl; + + rbd_completion_t c; + rbd_aio_create_completion(NULL, NULL, &c); + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_get_return_value(::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rbd_aio_release(::testing::_)) + .WillOnce(testing::Return()); + response.set_retcode(nebd::client::kNoOK); + RbdFinishAioWrite(c, writejob); + EXPECT_EQ(nebd::client::kOK, response.retcode()); +} + +TEST(RpcRequestCephTest, RbdFinishAioWrite_2) { + nebd::client::WriteResponse response; + brpc::Controller cntl; + nebd::client::WriteRequest request; + request.set_fd(4); + request.set_offset(4); + request.set_size(4); + RpcServiceTestClosure done; + AsyncWrite* writejob = NULL; + writejob = new AsyncWrite; + writejob->done = &done; + writejob->response = &response; + writejob->request = &request; + writejob->cntl = &cntl; + + rbd_completion_t c; + rbd_aio_create_completion(NULL, NULL, &c); + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_get_return_value(::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_CALL(mock, rbd_aio_release(::testing::_)) + .WillOnce(testing::Return()); + response.set_retcode(nebd::client::kNoOK); + RbdFinishAioWrite(c, writejob); + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(RpcRequestCephTest, RbdFinishAioRead_1) { + nebd::client::ReadResponse response; + brpc::Controller cntl; + nebd::client::ReadRequest request; + request.set_fd(4); + request.set_offset(4); + request.set_size(4); + RpcServiceTestClosure done; + AsyncRead* readjob = NULL; + readjob = new AsyncRead; + readjob->done = &done; + readjob->response = &response; + readjob->request = &request; + readjob->cntl = &cntl; + readjob->buf = new char[5](); + rbd_completion_t c; + rbd_aio_create_completion(NULL, NULL, &c); + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_get_return_value(::testing::_)) + .WillOnce(testing::Return(1)); + EXPECT_CALL(mock, rbd_aio_release(::testing::_)) + .WillOnce(testing::Return()); + response.set_retcode(nebd::client::kNoOK); + RbdFinishAioRead(c, readjob); + EXPECT_EQ(nebd::client::kOK, response.retcode()); +} + +TEST(RpcRequestCephTest, RbdFinishAioRead_2) { + nebd::client::ReadResponse response; + brpc::Controller cntl; + nebd::client::ReadRequest request; + request.set_fd(4); + request.set_offset(4); + request.set_size(4); + RpcServiceTestClosure done; + AsyncRead* readjob = NULL; + readjob = new AsyncRead; + readjob->done = &done; + readjob->response = &response; + readjob->request = &request; + readjob->cntl = &cntl; + readjob->buf = new char[5](); + rbd_completion_t c; + rbd_aio_create_completion(NULL, NULL, &c); + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_get_return_value(::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rbd_aio_release(::testing::_)) + .WillOnce(testing::Return()); + response.set_retcode(nebd::client::kNoOK); + RbdFinishAioRead(c, readjob); + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(RpcRequestCephTest, RbdFinishAioDiscard_1) { + nebd::client::DiscardResponse response; + brpc::Controller cntl; + nebd::client::DiscardRequest request; + request.set_fd(4); + request.set_offset(1); + request.set_size(1); + RpcServiceTestClosure done; + AsyncDiscard* discardjob = NULL; + discardjob = new AsyncDiscard; + discardjob->done = &done; + discardjob->response = &response; + discardjob->request = &request; + + rbd_completion_t c; + rbd_aio_create_completion(NULL, NULL, &c); + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_get_return_value(::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rbd_aio_release(::testing::_)) + .WillOnce(testing::Return()); + response.set_retcode(nebd::client::kNoOK); + RbdFinishAioDiscard(c, discardjob); + EXPECT_EQ(nebd::client::kOK, response.retcode()); +} + +TEST(RpcRequestCephTest, RbdFinishAioDiscard_2) { + nebd::client::DiscardResponse response; + brpc::Controller cntl; + nebd::client::DiscardRequest request; + request.set_fd(4); + request.set_offset(1); + request.set_size(1); + RpcServiceTestClosure done; + AsyncDiscard* discardjob = NULL; + discardjob = new AsyncDiscard; + discardjob->done = &done; + discardjob->response = &response; + discardjob->request = &request; + + rbd_completion_t c; + rbd_aio_create_completion(NULL, NULL, &c); + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_get_return_value(::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_CALL(mock, rbd_aio_release(::testing::_)) + .WillOnce(testing::Return()); + response.set_retcode(nebd::client::kNoOK); + RbdFinishAioDiscard(c, discardjob); + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(RpcRequestCephTest, RbdFinishAioFlush_1) { + nebd::client::FlushResponse response; + nebd::client::FlushRequest request; + request.set_fd(3); + RpcServiceTestClosure done; + AsyncFlush* flushjob = NULL; + flushjob = new AsyncFlush; + flushjob->done = &done; + flushjob->response = &response; + flushjob->request = &request; + rbd_completion_t c; + rbd_aio_create_completion(NULL, NULL, &c); + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_get_return_value(::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rbd_aio_release(::testing::_)) + .WillOnce(testing::Return()); + response.set_retcode(nebd::client::kNoOK); + RbdFinishAioFlush(c, flushjob); + EXPECT_EQ(nebd::client::kOK, response.retcode()); +} + +TEST(RpcRequestCephTest, RbdFinishAioFlush_2) { + nebd::client::FlushResponse response; + nebd::client::FlushRequest request; + request.set_fd(3); + RpcServiceTestClosure done; + AsyncFlush* flushjob = NULL; + flushjob = new AsyncFlush; + flushjob->done = &done; + flushjob->response = &response; + flushjob->request = &request; + rbd_completion_t c; + rbd_aio_create_completion(NULL, NULL, &c); + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_get_return_value(::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_CALL(mock, rbd_aio_release(::testing::_)) + .WillOnce(testing::Return()); + response.set_retcode(nebd::client::kNoOK); + RbdFinishAioFlush(c, flushjob); + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(RpcRequestCephTest, InvalidateCache_1) { + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestCeph cephImpl; + + EXPECT_EQ(-1, cephImpl.InvalidateCache(3)); +} + +TEST(RpcRequestCephTest, InvalidateCache_2) { + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_invalidate_cache(::testing::_)) + .WillOnce(testing::Return(-1)); + + RpcRequestCeph cephImpl; + EXPECT_EQ(-1, cephImpl.InvalidateCache(4)); +} + +TEST(RpcRequestCephTest, InvalidateCache_3) { + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_invalidate_cache(::testing::_)) + .WillOnce(testing::Return(0)); + + RpcRequestCeph cephImpl; + EXPECT_EQ(0, cephImpl.InvalidateCache(4)); +} + +TEST(RpcRequestCephTest, Resize_1) { + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestCeph cephImpl; + + EXPECT_EQ(-1, cephImpl.Resize(3, 1024)); +} + +TEST(RpcRequestCephTest, Resize_2) { + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_resize(::testing::_, ::testing::_)) + .WillOnce(testing::Return(-1)); + + RpcRequestCeph cephImpl; + EXPECT_EQ(-1, cephImpl.Resize(4, 1024)); +} + +TEST(RpcRequestCephTest, Resize_3) { + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_resize(::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + + RpcRequestCeph cephImpl; + EXPECT_EQ(0, cephImpl.Resize(4, 1024)); +} + +TEST(RpcRequestCephTest, GetInfo_1) { + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestCeph cephImpl; + nebd::client::GetInfoResponse response; + EXPECT_EQ(-1, cephImpl.GetInfo(3, &response)); +} + +TEST(RpcRequestCephTest, GetInfo_2) { + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_stat(::testing::_, ::testing::_, ::testing::_)) + .WillOnce(testing::Return(-1)); + + RpcRequestCeph cephImpl; + nebd::client::GetInfoResponse response; + EXPECT_EQ(-1, cephImpl.GetInfo(4, &response)); +} + +TEST(RpcRequestCephTest, GetInfo_3) { + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_stat(::testing::_, ::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + + RpcRequestCeph cephImpl; + nebd::client::GetInfoResponse response; + EXPECT_EQ(0, cephImpl.GetInfo(4, &response)); +} + +TEST(RpcRequestCephTest, StatFile_1) { + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestCeph cephImpl; + + nebd::client::StatFileResponse response; + EXPECT_EQ(-1, cephImpl.StatFile(3, &response)); +} + +TEST(RpcRequestCephTest, StatFile_2) { + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_stat(::testing::_, ::testing::_, ::testing::_)) + .WillOnce(testing::Return(-1)); + RpcRequestCeph cephImpl; + nebd::client::StatFileResponse response; + EXPECT_EQ(-1, cephImpl.StatFile(4, &response)); +} + +TEST(RpcRequestCephTest, StatFile_3) { + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_stat(::testing::_, ::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + RpcRequestCeph cephImpl; + nebd::client::StatFileResponse response; + EXPECT_EQ(0, cephImpl.StatFile(4, &response)); +} + +TEST(RpcRequestCephTest, CloseFile_1) { + NebdServerMocker mock; + std::string lockfile = "/tmp/unit_test"; + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(-1)); + + RpcRequestCeph cephImpl; + nebd::client::StatFileResponse response; + EXPECT_EQ(-1, cephImpl.CloseFile(4)); +} + +TEST(RpcRequestCephTest, CloseFile_2) { + NebdServerMocker mock; + std::string lockfile = "/tmp/unit_test"; + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(-1)); + + RpcRequestCeph cephImpl; + nebd::client::StatFileResponse response; + EXPECT_EQ(-1, cephImpl.CloseFile(4)); +} + +TEST(RpcRequestCephTest, CloseFile_3) { + NebdServerMocker mock; + std::string lockfile = "/tmp/unit_test"; + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, RmFd(::testing::_)).WillOnce(testing::Return(-1)); + + RpcRequestCeph cephImpl; + nebd::client::StatFileResponse response; + EXPECT_EQ(-1, cephImpl.CloseFile(4)); +} + +TEST(RpcRequestCephTest, CloseFile_4) { + NebdServerMocker mock; + std::string lockfile = "/tmp/unit_test"; + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, RmFd(::testing::_)).WillOnce(testing::Return(0)); + + RpcRequestCeph cephImpl; + nebd::client::StatFileResponse response; + EXPECT_EQ(0, cephImpl.CloseFile(4)); +} + +TEST(RpcRequestCephTest, Write_1) { + nebd::client::WriteResponse response; + brpc::Controller cntl; + nebd::client::WriteRequest request; + request.set_fd(3); + request.set_offset(1); + request.set_size(1); + RpcServiceTestClosure done; + AsyncWrite* writejob = NULL; + writejob = new AsyncWrite; + writejob->done = &done; + writejob->response = &response; + writejob->request = &request; + writejob->cntl = &cntl; + + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + FdImage_t fd_image; + fd_image.filename = filename; + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, &fd_image)); + + RpcRequestCeph cephImpl; + + NebdServerMocker mock; + // EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, + // ::testing::_)).WillOnce(testing::Return(-1)); + // EXPECT_CALL(mock, rbd_aio_write(::testing::_, 1, 1, ::testing::_, + // ::testing::_)).WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, cephImpl.Write(writejob)); +} + +TEST(RpcRequestCephTest, Write_2) { + nebd::client::WriteResponse response; + brpc::Controller cntl; + nebd::client::WriteRequest request; + request.set_fd(4); + request.set_offset(4); + request.set_size(4); + RpcServiceTestClosure done; + AsyncWrite* writejob = NULL; + writejob = new AsyncWrite; + writejob->done = &done; + writejob->response = &response; + writejob->request = &request; + writejob->cntl = &cntl; + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestCeph cephImpl; + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, + ::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, cephImpl.Write(writejob)); +} + +TEST(RpcRequestCephTest, Write_3) { + nebd::client::WriteResponse response; + brpc::Controller cntl; + nebd::client::WriteRequest request; + request.set_fd(4); + request.set_offset(4); + request.set_size(4); + RpcServiceTestClosure done; + AsyncWrite* writejob = NULL; + writejob = new AsyncWrite; + writejob->done = &done; + writejob->response = &response; + writejob->request = &request; + writejob->cntl = &cntl; + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestCeph cephImpl; + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, + ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rbd_aio_write(::testing::_, ::testing::_, ::testing::_, + ::testing::_, ::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, cephImpl.Write(writejob)); +} + +TEST(RpcRequestCephTest, Write_4) { + nebd::client::WriteResponse response; + brpc::Controller cntl; + nebd::client::WriteRequest request; + request.set_fd(4); + request.set_offset(4); + request.set_size(4); + RpcServiceTestClosure done; + AsyncWrite* writejob = NULL; + writejob = new AsyncWrite; + writejob->done = &done; + writejob->response = &response; + writejob->request = &request; + writejob->cntl = &cntl; + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestCeph cephImpl; + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, + ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rbd_aio_write(::testing::_, ::testing::_, ::testing::_, + ::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_EQ(0, cephImpl.Write(writejob)); +} + +TEST(RpcRequestCephTest, Read_1) { + nebd::client::ReadResponse response; + brpc::Controller cntl; + nebd::client::ReadRequest request; + request.set_fd(3); + request.set_offset(1); + request.set_size(1); + RpcServiceTestClosure done; + AsyncRead* readjob = NULL; + readjob = new AsyncRead; + readjob->done = &done; + readjob->response = &response; + readjob->request = &request; + readjob->cntl = &cntl; + + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + FdImage_t fd_image; + fd_image.filename = filename; + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, &fd_image)); + + RpcRequestCeph cephImpl; + + NebdServerMocker mock; + // EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, + // ::testing::_)).WillOnce(testing::Return(-1)); + // EXPECT_CALL(mock, rbd_aio_write(::testing::_, 1, 1, ::testing::_, + // ::testing::_)).WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, cephImpl.Read(readjob)); +} + +TEST(RpcRequestCephTest, Read_2) { + nebd::client::ReadResponse response; + brpc::Controller cntl; + nebd::client::ReadRequest request; + request.set_fd(4); + request.set_offset(4); + request.set_size(1); + RpcServiceTestClosure done; + AsyncRead* readjob = NULL; + readjob = new AsyncRead; + readjob->done = &done; + readjob->response = &response; + readjob->request = &request; + readjob->cntl = &cntl; + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestCeph cephImpl; + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, + ::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, cephImpl.Read(readjob)); +} + +TEST(RpcRequestCephTest, Read_3) { + nebd::client::ReadResponse response; + brpc::Controller cntl; + nebd::client::ReadRequest request; + request.set_fd(4); + request.set_offset(4); + request.set_size(1); + RpcServiceTestClosure done; + AsyncRead* readjob = NULL; + readjob = new AsyncRead; + readjob->done = &done; + readjob->response = &response; + readjob->request = &request; + readjob->cntl = &cntl; + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestCeph cephImpl; + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, + ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rbd_aio_read(::testing::_, ::testing::_, ::testing::_, + ::testing::_, ::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, cephImpl.Read(readjob)); +} + +TEST(RpcRequestCephTest, Read_4) { + nebd::client::ReadResponse response; + brpc::Controller cntl; + nebd::client::ReadRequest request; + request.set_fd(4); + request.set_offset(4); + request.set_size(1); + RpcServiceTestClosure done; + AsyncRead* readjob = NULL; + readjob = new AsyncRead; + readjob->done = &done; + readjob->response = &response; + readjob->request = &request; + readjob->cntl = &cntl; + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestCeph cephImpl; + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, + ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rbd_aio_read(::testing::_, ::testing::_, ::testing::_, + ::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_EQ(0, cephImpl.Read(readjob)); +} + +TEST(RpcRequestCephTest, Discard_1) { + nebd::client::DiscardResponse response; + brpc::Controller cntl; + nebd::client::DiscardRequest request; + request.set_fd(3); + request.set_offset(1); + request.set_size(1); + RpcServiceTestClosure done; + AsyncDiscard* discardjob = NULL; + discardjob = new AsyncDiscard; + discardjob->done = &done; + discardjob->response = &response; + discardjob->request = &request; + + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + FdImage_t fd_image; + fd_image.filename = filename; + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, &fd_image)); + + RpcRequestCeph cephImpl; + + NebdServerMocker mock; + EXPECT_EQ(-1, cephImpl.Discard(discardjob)); +} + +TEST(RpcRequestCephTest, Discard_2) { + nebd::client::DiscardResponse response; + brpc::Controller cntl; + nebd::client::DiscardRequest request; + request.set_fd(4); + request.set_offset(1); + request.set_size(1); + RpcServiceTestClosure done; + AsyncDiscard* discardjob = NULL; + discardjob = new AsyncDiscard; + discardjob->done = &done; + discardjob->response = &response; + discardjob->request = &request; + + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + FdImage_t fd_image; + fd_image.filename = filename; + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, &fd_image)); + + RpcRequestCeph cephImpl; + + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, + ::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, cephImpl.Discard(discardjob)); +} + +TEST(RpcRequestCephTest, Discard_3) { + nebd::client::DiscardResponse response; + brpc::Controller cntl; + nebd::client::DiscardRequest request; + request.set_fd(4); + request.set_offset(1); + request.set_size(1); + RpcServiceTestClosure done; + AsyncDiscard* discardjob = NULL; + discardjob = new AsyncDiscard; + discardjob->done = &done; + discardjob->response = &response; + discardjob->request = &request; + + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + FdImage_t fd_image; + fd_image.filename = filename; + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, &fd_image)); + + RpcRequestCeph cephImpl; + + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, + ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rbd_aio_discard(::testing::_, ::testing::_, ::testing::_, + ::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, cephImpl.Discard(discardjob)); +} + +TEST(RpcRequestCephTest, Discard_4) { + nebd::client::DiscardResponse response; + brpc::Controller cntl; + nebd::client::DiscardRequest request; + request.set_fd(4); + request.set_offset(1); + request.set_size(1); + RpcServiceTestClosure done; + AsyncDiscard* discardjob = NULL; + discardjob = new AsyncDiscard; + discardjob->done = &done; + discardjob->response = &response; + discardjob->request = &request; + + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + FdImage_t fd_image; + fd_image.filename = filename; + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, &fd_image)); + + RpcRequestCeph cephImpl; + + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, + ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rbd_aio_discard(::testing::_, ::testing::_, ::testing::_, + ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_EQ(0, cephImpl.Discard(discardjob)); +} + +TEST(RpcRequestCephTest, Flush_1) { + nebd::client::FlushResponse response; + nebd::client::FlushRequest request; + request.set_fd(3); + RpcServiceTestClosure done; + AsyncFlush* flushjob = NULL; + flushjob = new AsyncFlush; + flushjob->done = &done; + flushjob->response = &response; + flushjob->request = &request; + + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + FdImage_t fd_image; + fd_image.filename = filename; + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, &fd_image)); + + RpcRequestCeph cephImpl; + + NebdServerMocker mock; + EXPECT_EQ(-1, cephImpl.Flush(flushjob)); +} + +TEST(RpcRequestCephTest, Flush_2) { + nebd::client::FlushResponse response; + nebd::client::FlushRequest request; + request.set_fd(4); + RpcServiceTestClosure done; + AsyncFlush* flushjob = NULL; + flushjob = new AsyncFlush; + flushjob->done = &done; + flushjob->response = &response; + flushjob->request = &request; + + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + FdImage_t fd_image; + fd_image.filename = filename; + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, &fd_image)); + + RpcRequestCeph cephImpl; + + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, + ::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, cephImpl.Flush(flushjob)); +} + +TEST(RpcRequestCephTest, Flush_3) { + nebd::client::FlushResponse response; + nebd::client::FlushRequest request; + request.set_fd(4); + RpcServiceTestClosure done; + AsyncFlush* flushjob = NULL; + flushjob = new AsyncFlush; + flushjob->done = &done; + flushjob->response = &response; + flushjob->request = &request; + + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + FdImage_t fd_image; + fd_image.filename = filename; + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, &fd_image)); + + RpcRequestCeph cephImpl; + + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, + ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rbd_aio_flush(::testing::_, ::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, cephImpl.Flush(flushjob)); +} + +TEST(RpcRequestCephTest, Flush_4) { + nebd::client::FlushResponse response; + nebd::client::FlushRequest request; + request.set_fd(4); + RpcServiceTestClosure done; + AsyncFlush* flushjob = NULL; + flushjob = new AsyncFlush; + flushjob->done = &done; + flushjob->response = &response; + flushjob->request = &request; + + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + FdImage_t fd_image; + fd_image.filename = filename; + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, &fd_image)); + + RpcRequestCeph cephImpl; + + NebdServerMocker mock; + EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, + ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_CALL(mock, rbd_aio_flush(::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_EQ(0, cephImpl.Flush(flushjob)); +} + +TEST(OpenFile, OpenFile_1) { + const char* tmp = "rbdrbdvolume03auth_supportednonemon_host"; + char* filename = new char[strlen(tmp) + 1](); + snprintf(filename, strlen(tmp) + 1, "%s", tmp); + RpcRequestCeph cephImpl; + EXPECT_EQ(-1, cephImpl.OpenFile(filename)); +} + +TEST(OpenFile, OpenFile_2) { + const char* tmp = "rbdrbdvolume03auth_supported=nonemon_host=10.182.30.27"; + char* filename = new char[strlen(tmp) + 1](); + snprintf(filename, strlen(tmp) + 1, "%s", tmp); + RpcRequestCeph cephImpl; + EXPECT_EQ(-1, cephImpl.OpenFile(filename)); +} + +TEST(OpenFile, OpenFile_3) { + const char* tmp = + "rbd:rbdvolume03:auth_supported=none:mon_host=10.182.30.27:6789"; + char* filename = new char[strlen(tmp) + 1](); + snprintf(filename, strlen(tmp) + 1, "%s", tmp); + RpcRequestCeph cephImpl; + EXPECT_EQ(-1, cephImpl.OpenFile(filename)); +} + +TEST(OpenFile, OpenFile_4) { + const char* tmp = "rbd:rbd/volume03:auth_supported=none:mon_host"; + char* filename = new char[strlen(tmp) + 1](); + snprintf(filename, strlen(tmp) + 1, "%s", tmp); + RpcRequestCeph cephImpl; + EXPECT_EQ(-1, cephImpl.OpenFile(filename)); +} + +TEST(OpenFile, OpenFile_5) { + const char* tmp = + "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; + char* filename = new char[strlen(tmp) + 1](); + snprintf(filename, strlen(tmp) + 1, "%s", tmp); + RpcRequestCeph cephImpl; + NebdServerMocker mock; + EXPECT_CALL(mock, FilenameFdExist(::testing::_)) + .WillOnce(testing::Return(3)); + EXPECT_EQ(3, cephImpl.OpenFile(filename)); +} + +TEST(OpenFile, OpenFile_6) { + const char* filename = + "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; + RpcRequestCeph cephImpl; + NebdServerMocker mock; + EXPECT_CALL(mock, FilenameFdExist(::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_CALL(mock, ConnectRados(::testing::_)) + .WillOnce(testing::ReturnNull()); + EXPECT_EQ(-1, cephImpl.OpenFile(const_cast(filename))); +} + +TEST(OpenFile, OpenFile_7) { + const char* filename = + "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; + RpcRequestCeph cephImpl; + rados_t cluster; + rados_create(&cluster, "admin"); + NebdServerMocker mock; + EXPECT_CALL(mock, FilenameFdExist(::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_CALL(mock, ConnectRados(::testing::_)) + .WillOnce(testing::Return(&cluster)); + EXPECT_CALL(mock, CloseRados(::testing::_)).WillOnce(testing::Return()); + EXPECT_CALL(mock, OpenImage(::testing::_, ::testing::_, ::testing::_, + ::testing::_, ::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, cephImpl.OpenFile(const_cast(filename))); +} + +TEST(OpenFile, OpenFile_8) { + const char* filename = + "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; + RpcRequestCeph cephImpl; + rados_t cluster; + rados_create(&cluster, "admin"); + NebdServerMocker mock; + EXPECT_CALL(mock, FilenameFdExist(::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_CALL(mock, ConnectRados(::testing::_)) + .WillOnce(testing::Return(&cluster)); + EXPECT_CALL(mock, OpenImage(::testing::_, ::testing::_, ::testing::_, + ::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + std::string lockfile = "/tmp/unittest"; + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); + EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, cephImpl.OpenFile(const_cast(filename))); +} + +TEST(OpenFile, OpenFile_9) { + const char* filename = + "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; + RpcRequestCeph cephImpl; + rados_t cluster; + rados_create(&cluster, "admin"); + NebdServerMocker mock; + EXPECT_CALL(mock, FilenameFdExist(::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_CALL(mock, ConnectRados(::testing::_)) + .WillOnce(testing::Return(&cluster)); + EXPECT_CALL(mock, OpenImage(::testing::_, ::testing::_, ::testing::_, + ::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + std::string lockfile = "/tmp/unittest"; + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); + EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, GenerateFd(::testing::_, ::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_EQ(-1, cephImpl.OpenFile(const_cast(filename))); +} + +TEST(OpenFile, OpenFile_10) { + const char* filename = + "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; + RpcRequestCeph cephImpl; + rados_t cluster; + rados_create(&cluster, "admin"); + NebdServerMocker mock; + EXPECT_CALL(mock, FilenameFdExist(::testing::_)) + .WillOnce(testing::Return(-1)); + EXPECT_CALL(mock, ConnectRados(::testing::_)) + .WillOnce(testing::Return(&cluster)); + EXPECT_CALL(mock, OpenImage(::testing::_, ::testing::_, ::testing::_, + ::testing::_, ::testing::_)) + .WillOnce(testing::Return(5)); + std::string lockfile = "/tmp/unittest"; + EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); + EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); + EXPECT_CALL(mock, GenerateFd(::testing::_, ::testing::_)) + .WillOnce(testing::Return(0)); + EXPECT_EQ(5, cephImpl.OpenFile(const_cast(filename))); +} + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/part2/TestRpcServer.cpp b/tests/part2/TestRpcServer.cpp new file mode 100644 index 0000000000..a36f15b2da --- /dev/null +++ b/tests/part2/TestRpcServer.cpp @@ -0,0 +1,1187 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/common/client.pb.h" +#include "src/part2/rados_interface.h" +#include "src/part2/rpc_server.h" + +class RpcRequestTestOk : public RpcRequestCeph { + public: + RpcRequestTestOk() {} + virtual ~RpcRequestTestOk() {} + int OpenFile(char* filename) { return 100; } + int StatFile(int fd, nebd::client::StatFileResponse* response) { return 0; } + int Write(AsyncWrite* writejob) { return 0; } + int Read(AsyncRead* writejob) { return 0; } + int GetInfo(int fd, nebd::client::GetInfoResponse* response) { return 0; } + int CloseFile(int fd) { return 0; } + int Flush(AsyncFlush* flushjob) { return 0; } + int Discard(AsyncDiscard* discardjob) { return 0; } + int Resize(int fd, uint64_t size) { return 0; } + int InvalidateCache(int fd) { return 0; } +}; + +class RpcRequestTestNoOk : public RpcRequestCeph { + public: + RpcRequestTestNoOk() {} + virtual ~RpcRequestTestNoOk() {} + int OpenFile(char* filename) { return -1; } + int StatFile(int fd, nebd::client::StatFileResponse* response) { + return -1; + } + int Write(AsyncWrite* writejob) { return -1; } + int Read(AsyncRead* writejob) { return -1; } + int GetInfo(int fd, nebd::client::GetInfoResponse* response) { return -1; } + int CloseFile(int fd) { return -1; } + int Flush(AsyncFlush* flushjob) { return -1; } + int Discard(AsyncDiscard* discardjob) { return -1; } + int Resize(int fd, uint64_t size) { return -1; } + int InvalidateCache(int fd) { return -1; } +}; + +class RpcServiceTestClosure : public ::google::protobuf::Closure { + public: + explicit RpcServiceTestClosure(int sleepUs = 0) : sleep_(sleepUs) {} + virtual ~RpcServiceTestClosure() = default; + + void Run() override { + if (0 != sleep_) { + ::usleep(sleep_); + LOG(INFO) << "return rpc"; + } + } + + private: + int sleep_; +}; + +TEST(QemuClientServiceImplTest, OpenFile_1) { + nebd::client::OpenFileResponse response; + brpc::Controller cntl; + nebd::client::OpenFileRequest request; + request.set_filename( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.OpenFile(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, OpenFile_2) { + nebd::client::OpenFileResponse response; + brpc::Controller cntl; + nebd::client::OpenFileRequest request; + request.set_filename( + "nbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + RpcServiceTestClosure done; + + g_imageMap.clear(); + + QemuClientServiceImpl qemuImpl; + qemuImpl.OpenFile(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, OpenFile_3) { + nebd::client::OpenFileResponse response; + brpc::Controller cntl; + nebd::client::OpenFileRequest request; + request.set_filename( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + RpcServiceTestClosure done; + + g_imageMap.clear(); + RpcRequestTestNoOk* req = new RpcRequestTestNoOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.OpenFile(&cntl, &request, &response, &done); + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, OpenFile_4) { + nebd::client::OpenFileResponse response; + brpc::Controller cntl; + nebd::client::OpenFileRequest request; + request.set_filename( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + RpcServiceTestClosure done; + + g_imageMap.clear(); + RpcRequestTestOk* req = new RpcRequestTestOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.OpenFile(&cntl, &request, &response, &done); + EXPECT_EQ(nebd::client::kOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, Write_1) { + nebd::client::WriteResponse response; + brpc::Controller cntl; + nebd::client::WriteRequest request; + request.set_fd(3); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.Write(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, Write_2) { + nebd::client::WriteResponse response; + brpc::Controller cntl; + nebd::client::WriteRequest request; + request.set_fd(10000); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 10000; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.Write(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, Write_3) { + nebd::client::WriteResponse response; + brpc::Controller cntl; + nebd::client::WriteRequest request; + request.set_fd(300); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 300; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestTestOk* req = new RpcRequestTestOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.Write(&cntl, &request, &response, &done); + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, Write_4) { + nebd::client::WriteResponse response; + brpc::Controller cntl; + nebd::client::WriteRequest request; + request.set_fd(300); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 300; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestTestNoOk* req = new RpcRequestTestNoOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.Write(&cntl, &request, &response, &done); + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, read_1) { + nebd::client::ReadResponse response; + brpc::Controller cntl; + nebd::client::ReadRequest request; + request.set_fd(3); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.Read(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, Read_2) { + nebd::client::ReadResponse response; + brpc::Controller cntl; + nebd::client::ReadRequest request; + request.set_fd(10000); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 10000; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.Read(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, Read_3) { + nebd::client::ReadResponse response; + brpc::Controller cntl; + nebd::client::ReadRequest request; + request.set_fd(300); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 300; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestTestNoOk* req = new RpcRequestTestNoOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.Read(&cntl, &request, &response, &done); + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, Read_4) { + nebd::client::ReadResponse response; + brpc::Controller cntl; + nebd::client::ReadRequest request; + request.set_fd(300); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 300; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestTestOk* req = new RpcRequestTestOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.Read(&cntl, &request, &response, &done); + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, StatFile_1) { + nebd::client::StatFileResponse response; + brpc::Controller cntl; + nebd::client::StatFileRequest request; + request.set_fd(3); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.StatFile(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, StatFile_2) { + nebd::client::StatFileResponse response; + brpc::Controller cntl; + nebd::client::StatFileRequest request; + request.set_fd(10000); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 10000; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestTestOk* req = new RpcRequestTestOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.StatFile(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, StatFile_3) { + nebd::client::StatFileResponse response; + brpc::Controller cntl; + nebd::client::StatFileRequest request; + request.set_fd(100); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 100; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestTestOk* req = new RpcRequestTestOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.StatFile(&cntl, &request, &response, &done); + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, StatFile_4) { + nebd::client::StatFileResponse response; + brpc::Controller cntl; + nebd::client::StatFileRequest request; + request.set_fd(100); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 100; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestTestNoOk* req = new RpcRequestTestNoOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.StatFile(&cntl, &request, &response, &done); + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, GetInfo_1) { + nebd::client::GetInfoResponse response; + brpc::Controller cntl; + nebd::client::GetInfoRequest request; + request.set_fd(3); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.GetInfo(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, GetInfo_2) { + nebd::client::GetInfoResponse response; + brpc::Controller cntl; + nebd::client::GetInfoRequest request; + request.set_fd(10000); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 10000; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.GetInfo(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, GetInfo_3) { + nebd::client::GetInfoResponse response; + brpc::Controller cntl; + nebd::client::GetInfoRequest request; + request.set_fd(300); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 300; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestTestNoOk* req = new RpcRequestTestNoOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.GetInfo(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, GetInfo_4) { + nebd::client::GetInfoResponse response; + brpc::Controller cntl; + nebd::client::GetInfoRequest request; + request.set_fd(300); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 300; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestTestOk* req = new RpcRequestTestOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.GetInfo(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, Flush_1) { + nebd::client::FlushResponse response; + brpc::Controller cntl; + nebd::client::FlushRequest request; + request.set_fd(3); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.Flush(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, Flush_2) { + nebd::client::FlushResponse response; + brpc::Controller cntl; + nebd::client::FlushRequest request; + request.set_fd(10000); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 10000; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.Flush(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, Flush_3) { + nebd::client::FlushResponse response; + brpc::Controller cntl; + nebd::client::FlushRequest request; + request.set_fd(300); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 300; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestTestOk* req = new RpcRequestTestOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.Flush(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, Flush_4) { + nebd::client::FlushResponse response; + brpc::Controller cntl; + nebd::client::FlushRequest request; + request.set_fd(300); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 300; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + RpcRequestTestNoOk* req = new RpcRequestTestNoOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.Flush(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, CloseFile_1) { + nebd::client::CloseFileResponse response; + brpc::Controller cntl; + nebd::client::CloseFileRequest request; + request.set_fd(3); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.CloseFile(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, CloseFile_2) { + nebd::client::CloseFileResponse response; + brpc::Controller cntl; + nebd::client::CloseFileRequest request; + request.set_fd(10000); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 10000; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.CloseFile(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, CloseFile_3) { + nebd::client::CloseFileResponse response; + brpc::Controller cntl; + nebd::client::CloseFileRequest request; + request.set_fd(300); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 300; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestTestNoOk* req = new RpcRequestTestNoOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.CloseFile(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, CloseFile_4) { + nebd::client::CloseFileResponse response; + brpc::Controller cntl; + nebd::client::CloseFileRequest request; + request.set_fd(300); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 300; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestTestOk* req = new RpcRequestTestOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.CloseFile(&cntl, &request, &response, &done); + EXPECT_EQ(nebd::client::kOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, Discard_1) { + nebd::client::DiscardResponse response; + brpc::Controller cntl; + nebd::client::DiscardRequest request; + request.set_fd(3); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.Discard(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, Discard_2) { + nebd::client::DiscardResponse response; + brpc::Controller cntl; + nebd::client::DiscardRequest request; + request.set_fd(10000); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 10000; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.Discard(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, Discard_3) { + nebd::client::DiscardResponse response; + brpc::Controller cntl; + nebd::client::DiscardRequest request; + request.set_fd(300); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 300; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + RpcRequestTestOk* req = new RpcRequestTestOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.Discard(&cntl, &request, &response, &done); + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, Discard_4) { + nebd::client::DiscardResponse response; + brpc::Controller cntl; + nebd::client::DiscardRequest request; + request.set_fd(300); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 300; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + RpcRequestTestNoOk* req = new RpcRequestTestNoOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.Discard(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, ResizeFile_1) { + nebd::client::ResizeResponse response; + brpc::Controller cntl; + nebd::client::ResizeRequest request; + request.set_fd(3); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.ResizeFile(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, ResizeFile_2) { + nebd::client::ResizeResponse response; + brpc::Controller cntl; + nebd::client::ResizeRequest request; + request.set_fd(10000); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 10000; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.ResizeFile(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, ResizeFile_3) { + nebd::client::ResizeResponse response; + brpc::Controller cntl; + nebd::client::ResizeRequest request; + request.set_fd(300); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 300; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestTestNoOk* req = new RpcRequestTestNoOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.ResizeFile(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, ResizeFile_4) { + nebd::client::ResizeResponse response; + brpc::Controller cntl; + nebd::client::ResizeRequest request; + request.set_fd(300); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 300; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestTestOk* req = new RpcRequestTestOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.ResizeFile(&cntl, &request, &response, &done); + EXPECT_EQ(nebd::client::kOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, InvalidateCache_1) { + nebd::client::InvalidateCacheResponse response; + brpc::Controller cntl; + nebd::client::InvalidateCacheRequest request; + request.set_fd(3); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 4; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.InvalidateCache(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, InvalidateCache_2) { + nebd::client::InvalidateCacheResponse response; + brpc::Controller cntl; + nebd::client::InvalidateCacheRequest request; + request.set_fd(10000); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 10000; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + QemuClientServiceImpl qemuImpl; + qemuImpl.InvalidateCache(&cntl, &request, &response, &done); + + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, InvalidateCache_3) { + nebd::client::InvalidateCacheResponse response; + brpc::Controller cntl; + nebd::client::InvalidateCacheRequest request; + request.set_fd(300); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 300; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestTestNoOk* req = new RpcRequestTestNoOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.InvalidateCache(&cntl, &request, &response, &done); + EXPECT_EQ(nebd::client::kNoOK, response.retcode()); +} + +TEST(QemuClientServiceImplTest, InvalidateCache_4) { + nebd::client::InvalidateCacheResponse response; + brpc::Controller cntl; + nebd::client::InvalidateCacheRequest request; + request.set_fd(300); + RpcServiceTestClosure done; + + rados_ioctx_t* io = new rados_ioctx_t; + rados_t* cluster = new rados_t; + int fd = 300; + char* filename = const_cast( + "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); + rbd_image_t* image = new rbd_image_t; + FdImage_t* fd_image = new FdImage_t; + fd_image->cluster = cluster; + fd_image->io = io; + fd_image->image = image; + fd_image->filename = filename; + + g_imageMap.clear(); + g_imageMap.insert(std::pair(fd, fd_image)); + + RpcRequestTestOk* req = new RpcRequestTestOk; + QemuClientServiceImpl qemuImpl; + qemuImpl.SetRpcRequest(req); + qemuImpl.InvalidateCache(&cntl, &request, &response, &done); + EXPECT_EQ(nebd::client::kOK, response.retcode()); +} + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/part2/test_common.cpp b/tests/part2/test_common.cpp new file mode 100644 index 0000000000..4e195f298f --- /dev/null +++ b/tests/part2/test_common.cpp @@ -0,0 +1,18 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include "tests/part2/test_common.h" +CMOCK_MOCK_FUNCTION1(NebdServerMocker, GeneratePort, int(int)); +CMOCK_MOCK_FUNCTION2(NebdServerMocker, FindPort, int(config_t*, int)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, LockFile, int(const char*)); +CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidLockfile, std::string()); +CMOCK_MOCK_FUNCTION0(NebdServerMocker, Reload, int()); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, CheckProc, int(const char*)); +CMOCK_MOCK_FUNCTION3(NebdServerMocker, open, int(const char*, int, mode_t)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, ReadQemuXmlDir, std::string(config_t*)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, ReadQemuXml, void(const char*)); +CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidFile, std::string()); diff --git a/tests/part2/test_common.h b/tests/part2/test_common.h new file mode 100644 index 0000000000..812a4fbc4f --- /dev/null +++ b/tests/part2/test_common.h @@ -0,0 +1,30 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#ifndef TESTS_PART2_TEST_COMMON_H_ +#define TESTS_PART2_TEST_COMMON_H_ + +#include +#include +#include "src/part2/config.h" + +class NebdServerMocker : public CMockMocker { + public: + // MOCK_METHOD0(InitConfig, config_t*()); + MOCK_METHOD1(ReadQemuXmlDir, std::string(config_t *)); + MOCK_METHOD1(ReadQemuXml, void(const char *)); + MOCK_METHOD0(GetUuidFile, std::string()); + MOCK_METHOD0(GetUuidLockfile, std::string()); + MOCK_METHOD3(open, int(const char *, int, mode_t)); + MOCK_METHOD1(CheckProc, int(const char *)); + MOCK_METHOD1(LockFile, int(const char *)); + MOCK_METHOD0(Reload, int()); + MOCK_METHOD2(FindPort, int(config_t *, int)); + MOCK_METHOD1(GeneratePort, int(int)); +}; + +#endif // TESTS_PART2_TEST_COMMON_H_ diff --git a/tests/part2/test_common2.cpp b/tests/part2/test_common2.cpp new file mode 100644 index 0000000000..1695eeadef --- /dev/null +++ b/tests/part2/test_common2.cpp @@ -0,0 +1,10 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include "tests/part2/test_common2.h" +CMOCK_MOCK_FUNCTION1(NebdServerMocker, LockFile, int(const char*)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, unLockFile, void(int)); diff --git a/tests/part2/test_common2.h b/tests/part2/test_common2.h new file mode 100644 index 0000000000..ac3899a796 --- /dev/null +++ b/tests/part2/test_common2.h @@ -0,0 +1,20 @@ +/* + *** Project: nebd + *** File Created: 2019-09-30 + *** Author: hzwuhongsong + *** Copyright (c) 2019 NetEase + ***/ + +#ifndef TESTS_PART2_TEST_COMMON2_H_ +#define TESTS_PART2_TEST_COMMON2_H_ + +#include +#include "src/part2/config.h" + +class NebdServerMocker : public CMockMocker { + public: + MOCK_METHOD1(LockFile, int(const char*)); + MOCK_METHOD1(unLockFile, void(int)); +}; + +#endif // TESTS_PART2_TEST_COMMON2_H_ diff --git a/tests/part2/test_config.cpp b/tests/part2/test_config.cpp new file mode 100644 index 0000000000..389913c276 --- /dev/null +++ b/tests/part2/test_config.cpp @@ -0,0 +1,13 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include "tests/part2/test_reload.h" +/* +CMOCK_MOCK_FUNCTION1(NebdServerMocker, ConnectRados, rados_t*(const char*)); +CMOCK_MOCK_FUNCTION2(NebdServerMocker, ReloadCephVolume, int(int, char*)); +CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidFile, std::string()); +*/ diff --git a/tests/part2/test_heartbeat.cpp b/tests/part2/test_heartbeat.cpp new file mode 100644 index 0000000000..479123b045 --- /dev/null +++ b/tests/part2/test_heartbeat.cpp @@ -0,0 +1,17 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include "tests/part2/test_heartbeat.h" +CMOCK_MOCK_FUNCTION1(NebdServerMocker, LockFile, int(const char*)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, UnLockFile, void(int)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, CloseImage, int(int)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, CheckProc, int(const char*)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, RmFd, int(int)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, CloseQemuDetachedVolumes, int(int)); +CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidFile, std::string()); +CMOCK_MOCK_FUNCTION0(NebdServerMocker, ReadQemuXmls, int()); +CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidLockfile, std::string()); diff --git a/tests/part2/test_heartbeat.h b/tests/part2/test_heartbeat.h new file mode 100644 index 0000000000..fc7833caf1 --- /dev/null +++ b/tests/part2/test_heartbeat.h @@ -0,0 +1,31 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#ifndef TESTS_PART2_TEST_HEARTBEAT_H_ +#define TESTS_PART2_TEST_HEARTBEAT_H_ + +#include +#include +#include "src/part2/common_type.h" +#include "src/part2/config.h" +#include "src/part2/heartbeat.h" +#include "src/part2/rados_interface.h" + +class NebdServerMocker : public CMockMocker { + public: + MOCK_METHOD1(LockFile, int(const char*)); + MOCK_METHOD1(UnLockFile, void(int)); + MOCK_METHOD1(CloseImage, int(int)); + MOCK_METHOD0(GetUuidFile, std::string()); + MOCK_METHOD0(GetUuidLockfile, std::string()); + MOCK_METHOD1(CheckProc, int(const char*)); + MOCK_METHOD1(RmFd, int(int)); + MOCK_METHOD0(ReadQemuXmls, int()); + MOCK_METHOD1(CloseQemuDetachedVolumes, int(int)); +}; + +#endif // TESTS_PART2_TEST_HEARTBEAT_H_ diff --git a/tests/part2/test_heartbeat2.cpp b/tests/part2/test_heartbeat2.cpp new file mode 100644 index 0000000000..b25581c3e5 --- /dev/null +++ b/tests/part2/test_heartbeat2.cpp @@ -0,0 +1,16 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include "tests/part2/test_heartbeat2.h" +CMOCK_MOCK_FUNCTION3(NebdServerMocker, open, int(const char*, int, mode_t)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, LockFile, int(const char*)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, UnLockFile, void(int)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, CloseImage, int(int)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, RmFd, int(int)); +CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidFile, std::string()); +CMOCK_MOCK_FUNCTION0(NebdServerMocker, ReadQemuXmls, int()); +CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidLockfile, std::string()); diff --git a/tests/part2/test_heartbeat2.h b/tests/part2/test_heartbeat2.h new file mode 100644 index 0000000000..e22e2a7631 --- /dev/null +++ b/tests/part2/test_heartbeat2.h @@ -0,0 +1,31 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#ifndef TESTS_PART2_TEST_HEARTBEAT2_H_ +#define TESTS_PART2_TEST_HEARTBEAT2_H_ + +#include +#include +#include +#include +#include "src/part2/common_type.h" +#include "src/part2/config.h" +#include "src/part2/heartbeat.h" +#include "src/part2/rados_interface.h" +class NebdServerMocker : public CMockMocker { + public: + MOCK_METHOD1(LockFile, int(const char *)); + MOCK_METHOD1(UnLockFile, void(int)); + MOCK_METHOD1(CloseImage, int(int)); + MOCK_METHOD0(GetUuidFile, std::string()); + MOCK_METHOD0(GetUuidLockfile, std::string()); + MOCK_METHOD1(RmFd, int(int)); + MOCK_METHOD0(ReadQemuXmls, int()); + MOCK_METHOD3(open, int(const char *, int, mode_t)); +}; + +#endif // TESTS_PART2_TEST_HEARTBEAT2_H_ diff --git a/tests/part2/test_heartbeat3.cpp b/tests/part2/test_heartbeat3.cpp new file mode 100644 index 0000000000..38f6fb3cd9 --- /dev/null +++ b/tests/part2/test_heartbeat3.cpp @@ -0,0 +1,10 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include "tests/part2/test_heartbeat3.h" +CMOCK_MOCK_FUNCTION4(NebdServerMocker, CheckCmdline, + int(int64_t, const char*, char*, int)); diff --git a/tests/part2/test_heartbeat3.h b/tests/part2/test_heartbeat3.h new file mode 100644 index 0000000000..9d17ada779 --- /dev/null +++ b/tests/part2/test_heartbeat3.h @@ -0,0 +1,24 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#ifndef TESTS_PART2_TEST_HEARTBEAT3_H_ +#define TESTS_PART2_TEST_HEARTBEAT3_H_ + +#include +#include +#include +#include +#include "src/part2/common_type.h" +#include "src/part2/config.h" +#include "src/part2/heartbeat.h" +#include "src/part2/rados_interface.h" +class NebdServerMocker : public CMockMocker { + public: + MOCK_METHOD4(CheckCmdline, int(int64_t, const char*, char*, int)); +}; + +#endif // TESTS_PART2_TEST_HEARTBEAT3_H_ diff --git a/tests/part2/test_rados.cpp b/tests/part2/test_rados.cpp new file mode 100644 index 0000000000..7ddddb8418 --- /dev/null +++ b/tests/part2/test_rados.cpp @@ -0,0 +1,24 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include "tests/part2/test_rados.h" +CMOCK_MOCK_FUNCTION1(NebdServerMocker, CloseRados, void(rados_t *)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, rados_ioctx_destroy, + void(rados_ioctx_t)); +CMOCK_MOCK_FUNCTION4(NebdServerMocker, rbd_open, + int(rados_ioctx_t, const char *, rbd_image_t *, + const char *)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, rbd_close, int(rbd_image_t)); +CMOCK_MOCK_FUNCTION3(NebdServerMocker, rados_ioctx_create, + int(rados_t, const char *, rados_ioctx_t *)); +CMOCK_MOCK_FUNCTION2(NebdServerMocker, rados_create, + int(rados_t *, const char *)); +CMOCK_MOCK_FUNCTION2(NebdServerMocker, rados_conf_read_file, + int(rados_t, const char *)); +CMOCK_MOCK_FUNCTION3(NebdServerMocker, rados_conf_set, + int(rados_t, const char *, const char *)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, rados_connect, int(rados_t)); diff --git a/tests/part2/test_rados.h b/tests/part2/test_rados.h new file mode 100644 index 0000000000..9021d20225 --- /dev/null +++ b/tests/part2/test_rados.h @@ -0,0 +1,31 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#ifndef TESTS_PART2_TEST_RADOS_H_ +#define TESTS_PART2_TEST_RADOS_H_ + +#include +#include +#include +#include "src/part2/config.h" + +class NebdServerMocker : public CMockMocker { + public: + MOCK_METHOD1(CloseRados, void(rados_t *)); + MOCK_METHOD1(rados_ioctx_destroy, void(rados_ioctx_t)); + MOCK_METHOD3(rados_ioctx_create, + int(rados_t, const char *, rados_ioctx_t *)); + MOCK_METHOD4(rbd_open, + int(rados_ioctx_t, const char *, rbd_image_t *, const char *)); + MOCK_METHOD1(rbd_close, int(rbd_image_t)); + MOCK_METHOD2(rados_create, int(rados_t *, const char *)); + MOCK_METHOD2(rados_conf_read_file, int(rados_t, const char *)); + MOCK_METHOD3(rados_conf_set, int(rados_t, const char *, const char *)); + MOCK_METHOD1(rados_connect, int(rados_t)); +}; + +#endif // TESTS_PART2_TEST_RADOS_H_ diff --git a/tests/part2/test_reload.cpp b/tests/part2/test_reload.cpp new file mode 100644 index 0000000000..88a99d15ea --- /dev/null +++ b/tests/part2/test_reload.cpp @@ -0,0 +1,13 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include "tests/part2/test_reload.h" +#include + +CMOCK_MOCK_FUNCTION1(NebdServerMocker, ConnectRados, rados_t*(const char*)); +CMOCK_MOCK_FUNCTION2(NebdServerMocker, ReloadCephVolume, int(int, char*)); +CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidFile, std::string()); diff --git a/tests/part2/test_reload.h b/tests/part2/test_reload.h new file mode 100644 index 0000000000..0cee5329a5 --- /dev/null +++ b/tests/part2/test_reload.h @@ -0,0 +1,25 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#ifndef TESTS_PART2_TEST_RELOAD_H_ +#define TESTS_PART2_TEST_RELOAD_H_ + +#include +#include +#include +#include +#include "src/part2/rados_interface.h" +#include "src/part2/reload.h" + +class NebdServerMocker : public CMockMocker { + public: + MOCK_METHOD1(ConnectRados, rados_t*(const char*)); + MOCK_METHOD2(ReloadCephVolume, int(int, char*)); + MOCK_METHOD0(GetUuidFile, std::string()); +}; + +#endif // TESTS_PART2_TEST_RELOAD_H_ diff --git a/tests/part2/test_reload2.cpp b/tests/part2/test_reload2.cpp new file mode 100644 index 0000000000..c4155fd592 --- /dev/null +++ b/tests/part2/test_reload2.cpp @@ -0,0 +1,11 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include "tests/part2/test_reload2.h" +CMOCK_MOCK_FUNCTION1(NebdServerMocker, ConnectRados, rados_t*(const char*)); +CMOCK_MOCK_FUNCTION5(NebdServerMocker, OpenImage, + int(rados_t*, const char*, const char*, int, char*)); diff --git a/tests/part2/test_reload2.h b/tests/part2/test_reload2.h new file mode 100644 index 0000000000..fb81cc4c7f --- /dev/null +++ b/tests/part2/test_reload2.h @@ -0,0 +1,24 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#ifndef TESTS_PART2_TEST_RELOAD2_H_ +#define TESTS_PART2_TEST_RELOAD2_H_ + +#include +#include +#include +#include "src/part2/rados_interface.h" +#include "src/part2/reload.h" + +class NebdServerMocker : public CMockMocker { + public: + MOCK_METHOD1(ConnectRados, rados_t*(const char*)); + MOCK_METHOD5(OpenImage, + int(rados_t*, const char*, const char*, int, char*)); +}; + +#endif // TESTS_PART2_TEST_RELOAD2_H_ diff --git a/tests/part2/test_rpcceph.cpp b/tests/part2/test_rpcceph.cpp new file mode 100644 index 0000000000..20cc57b295 --- /dev/null +++ b/tests/part2/test_rpcceph.cpp @@ -0,0 +1,41 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#include +#include "tests/part2/test_rpcceph.h" + +CMOCK_MOCK_FUNCTION2(NebdServerMocker, GenerateFd, int(char *, int)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, CloseRados, void(rados_t *)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, FilenameFdExist, int(char *)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, ConnectRados, rados_t *(const char *)); +CMOCK_MOCK_FUNCTION5(NebdServerMocker, OpenImage, + int(rados_t *, const char *, const char *, int, char *)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, rbd_aio_release, void(rbd_completion_t)); +CMOCK_MOCK_FUNCTION5(NebdServerMocker, rbd_aio_read, + int(rbd_image_t, uint64_t, size_t, char *, + rbd_completion_t)); +CMOCK_MOCK_FUNCTION4(NebdServerMocker, rbd_aio_discard, + int(rbd_image_t, uint64_t, uint64_t, rbd_completion_t)); +CMOCK_MOCK_FUNCTION2(NebdServerMocker, rbd_aio_flush, + int(rbd_image_t, rbd_completion_t)); +CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidLockfile, std::string()); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, LockFile, int(const char *)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, rbd_aio_get_return_value, + ssize_t(rbd_completion_t)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, UnLockFile, void(int)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, CloseImage, int(int)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, RmFd, int(int)); +CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidFile, std::string()); +CMOCK_MOCK_FUNCTION5(NebdServerMocker, rbd_aio_write, + int(rbd_image_t, uint64_t, size_t, const char *, + rbd_completion_t)); +CMOCK_MOCK_FUNCTION3(NebdServerMocker, rbd_aio_create_completion, + int(void *, rbd_callback_t, rbd_completion_t *)); +CMOCK_MOCK_FUNCTION3(NebdServerMocker, rbd_stat, + int(rbd_image_t, rbd_image_info_t *, size_t)); +CMOCK_MOCK_FUNCTION2(NebdServerMocker, rbd_resize, int(rbd_image_t, uint64_t)); +CMOCK_MOCK_FUNCTION1(NebdServerMocker, rbd_invalidate_cache, int(rbd_image_t)); diff --git a/tests/part2/test_rpcceph.h b/tests/part2/test_rpcceph.h new file mode 100644 index 0000000000..7d77e56d02 --- /dev/null +++ b/tests/part2/test_rpcceph.h @@ -0,0 +1,46 @@ +/* + ** Project: nebd + ** File Created: 2019-09-30 + ** Author: hzwuhongsong + ** Copyright (c) 2019 NetEase + **/ + +#ifndef TESTS_PART2_TEST_RPCCEPH_H_ +#define TESTS_PART2_TEST_RPCCEPH_H_ + +#include +#include +#include +#include +#include "src/part2/config.h" + +class NebdServerMocker : public CMockMocker { + public: + MOCK_METHOD5(rbd_aio_write, int(rbd_image_t, uint64_t, size_t, const char *, + rbd_completion_t)); + MOCK_METHOD5(rbd_aio_read, + int(rbd_image_t, uint64_t, size_t, char *, rbd_completion_t)); + MOCK_METHOD4(rbd_aio_discard, + int(rbd_image_t, uint64_t, uint64_t, rbd_completion_t)); + MOCK_METHOD2(rbd_aio_flush, int(rbd_image_t, rbd_completion_t)); + MOCK_METHOD3(rbd_aio_create_completion, + int(void *, rbd_callback_t, rbd_completion_t *)); + MOCK_METHOD1(rbd_invalidate_cache, int(rbd_image_t)); + MOCK_METHOD2(rbd_resize, int(rbd_image_t, uint64_t)); + MOCK_METHOD3(rbd_stat, int(rbd_image_t, rbd_image_info_t *, size_t)); + MOCK_METHOD1(LockFile, int(const char *)); + MOCK_METHOD1(UnLockFile, void(int)); + MOCK_METHOD1(CloseImage, int(int)); + MOCK_METHOD0(GetUuidFile, std::string()); + MOCK_METHOD1(RmFd, int(int)); + MOCK_METHOD1(rbd_aio_get_return_value, ssize_t(rbd_completion_t)); + MOCK_METHOD0(GetUuidLockfile, std::string()); + MOCK_METHOD1(rbd_aio_release, void(rbd_completion_t)); + MOCK_METHOD1(FilenameFdExist, int(char *)); + MOCK_METHOD1(ConnectRados, rados_t *(const char *)); + MOCK_METHOD5(OpenImage, + int(rados_t *, const char *, const char *, int, char *)); + MOCK_METHOD1(CloseRados, void(rados_t *)); + MOCK_METHOD2(GenerateFd, int(char *, int)); +}; +#endif // TESTS_PART2_TEST_RPCCEPH_H_ From 6f054cc5b7369b3a9edf43434f3b75c67df3ac1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E7=9B=BC?= Date: Wed, 11 Sep 2019 10:45:01 +0800 Subject: [PATCH 05/79] add test files CLDNBS-1381 Change-Id: I323aba0e043a49dd4c954fe823cac3c6d9e4e7e7 --- .gitreview | 5 + tests/test-suites/create_100_vms.sh | 62 + tests/test-suites/id_rsa | 27 + tests/test-suites/kill_nebd.sh | 26 + tests/test-suites/templates/disk_xml.j2 | 16 + tests/test-suites/templates/domain_xml.j2 | 52 + tests/test-suites/test_10_disks.sh | 82 ++ tests/test-suites/test_discard.sh | 60 + tests/test-suites/test_nebd.py | 1407 +++++++++++++++++++++ 9 files changed, 1737 insertions(+) create mode 100644 .gitreview create mode 100644 tests/test-suites/create_100_vms.sh create mode 100644 tests/test-suites/id_rsa create mode 100644 tests/test-suites/kill_nebd.sh create mode 100644 tests/test-suites/templates/disk_xml.j2 create mode 100644 tests/test-suites/templates/domain_xml.j2 create mode 100644 tests/test-suites/test_10_disks.sh create mode 100644 tests/test-suites/test_discard.sh create mode 100755 tests/test-suites/test_nebd.py diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000000..eab3c762de --- /dev/null +++ b/.gitreview @@ -0,0 +1,5 @@ +[gerrit] +host=gerrit.storage.netease.com +port=29418 +project=nebd +defaultbranch=feature diff --git a/tests/test-suites/create_100_vms.sh b/tests/test-suites/create_100_vms.sh new file mode 100644 index 0000000000..fa6b8eb6f6 --- /dev/null +++ b/tests/test-suites/create_100_vms.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +POOL=rbd +MON_HOST=10.182.30.27 + +for i in {1..90}; do +echo ====$i==== +# 从snap创建系统盘 +rbd rm $POOL/vm-$i +rbd clone $POOL/rbd_sys_disk@snap $POOL/vm-$i + +# 从系统盘创建1个vm +echo ''' + + #vm-# + 2097152 + 2 + + hvm + + + + + + + + + /usr/bin/qemu-system-x86_64 + + + + + + + + +
+ + + + + + + + + + + + + +''' > vm-$i.xml + +sed -i "s/#POOL#/$POOL/g" vm-$i.xml +sed -i "s/#MON_HOST#/$MON_HOST/g" vm-$i.xml +sed -i "s/#vm-#/vm-$i/g" vm-$i.xml + +sudo virsh undefine vm-$i +sudo virsh define vm-$i.xml +sudo virsh start vm-$i +sleep 1 + +done diff --git a/tests/test-suites/id_rsa b/tests/test-suites/id_rsa new file mode 100644 index 0000000000..2115ed75ae --- /dev/null +++ b/tests/test-suites/id_rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA2A5lNh09w5O1hWotsEsWx65PNMmHS+a7xtXLgs+Bq/qtWMkD +ouGmxQGIgpHf6omvy+u2diqUu79CM4mBWGObDFCIOwjQ9x0+uTEu3drLGc5oYNB5 +jwTIGAGnlkNmooRF1HfgDtj4PeEk3F6Q75k08MnylY+j1BCd/O+p3JLJTRXybwK1 +w0ejUuPX7sTDZOacP/khpcQwSaBi5RcdWjsyzzqC0NMbpKSsKrOKWuAhQqE/2x10 +26VKU5hBBK4lLJf6RiCcZXBKq/PYTtlZUzEbwoX1jIGuHDWiFAnTpWhTf3J255Dp +CUQhzT2i/3teKEJQGBy40aNBvyEm/TI+h35rrwIDAQABAoIBAEg+eAViPbbZ+0aD +PIfIc/ONC5shEAaHPp67+nLhJAD3r2MSxA6A2ZziaF0Qngj0xT2wsbBNiLJGv8Iv +5npBxCSYARWMWyNmOgT+tycj8+nLJQfKbfsupCGy2/DY1Mbe5+Klutv8HPXEBOlT +Da+wUQ162Yy9HRDaBZoAIZJJ2tnVMXzolCYC6CgYpPzZ2aIRewgO9gkHvvlIjlk6 +lVEx8XwuTFnMAu+1yBnHiatn5UonpWL2CAGdCgYDUF7ehoOV2NOL+Qc6zHXUsVEo +GBnJuAcPJ57jacRp4FEY0bgXBRXltb84S+unoYzcrDbZDCZfOYIaGn5txsrIjSa+ +DkoFqVECgYEA85OaIFEDvYs9huwKfSMsYCa3RVQ+SA0gKCvn7WnxYmnn7ctFMfgt +io1QZIAnz6f3Y7yKGDbwJq4kBdxiiEwMIz4YT+WGN+r/ERcIZFvStqq0kSKK620m +1NQH4qZmJGwtCHB/O6/EoeJqfZPZXG/1+9DUXRCrO+dpd9ltJt4XgSUCgYEA4xN1 +SA7Ey42+gkqV8AtYp1Jb7w54hP9E6Y8kOR+P/eJYoI/7b1e9J9Wy61D+cxWSJpLX +jN73H6OFxeQbyzTu2h9dIIOzi4/a1Kd040f0FzAsP0O6bURPxMfnLhUgK2t2FGLD +4obaWTKAzHSo9SIrA3JY3q4EdiumHXkazZjEc0MCgYBJty/7+8jAAnXacXEzMgPq +DY2Fo9s9fwtuXOtSBLAS8q5IXy5P0yYlOmNMJyTlSZYrTxIUBfxkGGu6glonmaxK +ti4xb+tQ3QPYrjnR9aAtvDftfKDCY0RPQOQqCeX7u+ldFn7LF4TQ4W8NFkPAH/o3 +/ZlnmPXZqU1YAitwieJfKQKBgQCNqfgDKmOP8jAmLM354bo/ONxt+YMqxe6CH3Ej +yhMU+v77vJfKUNyLW4Cx2efEeHKjafzbv8ZAkM3GNZc5YR6aKfL5c/CxYt/Mx2Pe +32biiIUyOti1jmaNh4EseLWHIRWbhjpaunLCLwW3iBhYRia7kbzrv3i9GGIcBQ+5 +sFLu7QKBgFE+vTWpJcq2HuHcqlcBtSj2pbrWWskGb4YQBmaErTJz3Mdaezdl7XUo +0RAxcOeNumoJEXT22E52rZvLNA1FiJ8hQWWHmb4g2PwARbFEz9OF6pnZtXdUlr/s +W4RsnvkS93R8jOZNF7Ifx+3tetzlmGUKkHXKj06TjgPnFEYt9scX +-----END RSA PRIVATE KEY----- diff --git a/tests/test-suites/kill_nebd.sh b/tests/test-suites/kill_nebd.sh new file mode 100644 index 0000000000..f479ed62a3 --- /dev/null +++ b/tests/test-suites/kill_nebd.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +i=1 +while true; do + # 每10s随机kill一个nebd-server进程 + t1=$(($i % 35)) + if [ $t1 = 0 ]; then + pids=`pidof nebd-server` + len=`echo $pids | wc -w` + if [ $len -gt 0 ]; then + pid=`echo $pids | awk '{print $1}'` + echo going to kill -15 $pid + sudo kill -15 $pid + fi + fi + + # 每33s kill掉所有nebd-server进程 + ta=$(($i % 127)) + if [ $ta = 0 ]; then + echo going to killall -15 nebd-server + sudo killall -15 nebd-server + fi + + ((i++)) + sleep 1 +done diff --git a/tests/test-suites/templates/disk_xml.j2 b/tests/test-suites/templates/disk_xml.j2 new file mode 100644 index 0000000000..6a75958b64 --- /dev/null +++ b/tests/test-suites/templates/disk_xml.j2 @@ -0,0 +1,16 @@ + + {%- if target_dev == 'sdc' %} + + + {% else %} + + + {%- endif %} + + {%- if protocol == 'rbd' %} + {% for mon_host in mon_hosts -%} + + {% endfor -%} + {% endif -%} + + \ No newline at end of file diff --git a/tests/test-suites/templates/domain_xml.j2 b/tests/test-suites/templates/domain_xml.j2 new file mode 100644 index 0000000000..b86dca0996 --- /dev/null +++ b/tests/test-suites/templates/domain_xml.j2 @@ -0,0 +1,52 @@ + + {{ uuid }} + {{ name }} + 2097152 + 2 + + hvm + + + + + + + + + /usr/bin/qemu-system-x86_64 + + {%- if disk_type == "file" %} + + + {%- elif disk_type == "network" %} + + + {%- if protocol == 'rbd' %} + {% for mon_host in mon_hosts -%} + + {% endfor -%} + {% endif -%} + + {% endif -%} + + + +
+ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/test-suites/test_10_disks.sh b/tests/test-suites/test_10_disks.sh new file mode 100644 index 0000000000..e805104450 --- /dev/null +++ b/tests/test-suites/test_10_disks.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +# 创建10个卷 +POOL=rbd +MON_HOST=10.182.30.27 + +for i in {1..10}; do + rbd create -p $POOL vol-$i --image-format 2 --size 1024 +done + + +rbd snap create $POOL/rbd_sys_disk@snap +rbd snap protect $POOL/rbd_sys_disk@snap + +# 创建1个vm +rbd clone $POOL/rbd_sys_disk@snap $POOL/vm-with-10-disk + +echo ''' + + vm-with-10-disk + 8388608 + 4 + + hvm + + + + + + + + + /usr/bin/qemu-system-x86_64 + + + + + + + + +
+ + + + + + + + + + + + + +''' > vm-with-10-disk.xml +sed -i "s/#POOL#/$POOL/g" vm-with-10-disk.xml +sed -i "s/#MON_HOST#/$MON_HOST/g" vm-with-10-disk.xml + +sudo virsh define vm-with-10-disk.xml +sudo virsh start vm-with-10-disk +sleep 10 + +# 挂载10个卷到vm +target_devs=(vdc vdd vde vdf vdg vdh vdi vdj vdk vdl) +for i in {0..9}; do +echo ''' + + + + + + +''' > ./disk.xml +sed -i "s/#target_dev#/${target_devs[$i]}/" ./disk.xml +sed -i "s/#disk_name#/$POOL\/vol-$((i+1))/" ./disk.xml +sed -i "s/#MON_HOST#/$MON_HOST/g" ./disk.xml +sudo virsh attach-device vm-with-10-disk disk.xml +sleep 1 +done + +# vm里面对挂载的卷跑fio diff --git a/tests/test-suites/test_discard.sh b/tests/test-suites/test_discard.sh new file mode 100644 index 0000000000..5084ee7e3e --- /dev/null +++ b/tests/test-suites/test_discard.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +POOL=rbd +MON_HOST=10.182.30.27 + +# 从snap创建系统盘 +rbd clone $POOL/image-centos_7.5@snap $POOL/vm-discard +rbd create --image-format 2 --size 8388608 $POOL/disk-8T + +# 从系统盘创建1个vm +echo ''' + + vm-discard + 2097152 + 2 + + hvm + + + + + + + + + /usr/bin/qemu-system-x86_64 + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + +''' > vm-discard.xml +sed -i "s/#POOL#/$POOL/g" vm-discard.xml +sed -i "s/#MON_HOST#/$MON_HOST/g" vm-discard.xml +sudo virsh define vm-discard.xml +sudo virsh start vm-discard diff --git a/tests/test-suites/test_nebd.py b/tests/test-suites/test_nebd.py new file mode 100755 index 0000000000..e1e4e58ccd --- /dev/null +++ b/tests/test-suites/test_nebd.py @@ -0,0 +1,1407 @@ +#!/usr/bin/env python +# -*- coding:UTF-8 + +from jinja2 import Environment, FileSystemLoader +import json +from multiprocessing import cpu_count +import os +import shlex +from subprocess import Popen, PIPE +from threading import Timer +import time +import unittest + + +import libvirt +import paramiko +import rados +import rbd + + +CEPH_CONF = "/etc/ceph/ceph.conf" +MON_HOSTS = ["10.182.30.27", "10.182.30.28", "10.182.30.29"] +REUSE_RBD_DISK = False # True表示测试前不创建rbd卷（使用已有的），跑完不清理rbd卷，False则相反 +RBD_POOL = "rbd" +RBD_VM_IMG = "rbd_sys_disk" # 系统盘 +RBD_VM_IMG_SIZE = 4<<30 # GB，需要比文件镜像大 +RBD_VOL1 = "rbd_logic_disk" # 云盘 +RBD_VOL1_SIZE = 1<<30 # GB，至少1G +LOCAL_IMG = "/mnt/centos_7.5.raw" # 必须是raw格式镜像，不能qcow2 +QEMU_PROC_NAME = "qemu-system-x86_64" +NEBD_SERVER_EXEC_FILE = "/usr/bin/nebd-server" +METADATA_DIR = "/var/run/nebd-server/" +CEPH_CLIENT_OLD_PACKAGE = "0.94.6+netease.1.2-0" +CEPH_CLIENT_NEW_PACKAGE = "0.94.6+netease.1.5-2" +CEPH_CLIENT_REAL_VERSION = { # 通过admin socket version命令获取的版本号 + CEPH_CLIENT_OLD_PACKAGE: "0.94.6-11-g61fb955", + CEPH_CLIENT_NEW_PACKAGE: "0.94.6+netease.1.5-2" +} +CEPH_CLIENT_ASOK_DIR = "/var/run/ceph/guests/" + + +VM_SSH_KEY = "/home/hzwangpan/nebd/test/id_rsa" # 免密ssh到vm私钥（root用户600权限） +VM_HOST_IP = "192.168.10.10" # vm的ip地址，最好配置为静态ip +UUID1 = "5d1289be-50e1-47b7-86de-1de0ff16a9d4" +UUID2 = "b5fdf3de-320c-41cf-80b2-acb794078012" +MAX_CPUS = cpu_count() + +LIBVIRT_CONN = None +JJ_ENV = None + + + +def run(cmd, timeout=5, ignore=''): + print 'going to run cmd: %s' % cmd + proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) + timer = Timer(timeout, proc.kill) + try: + timer.start() + stdout, stderr = proc.communicate() + if stderr: + if ignore == '' or ignore not in stderr: + raise OSError(stderr) + return stdout + except Exception as ex: + print ex + raise ex + finally: + timer.cancel() + + +def create_ssh_connect(host, port=22, timeout=3): + key = paramiko.RSAKey.from_private_key_file(VM_SSH_KEY) + ssh = paramiko.SSHClient() + ssh.load_system_host_keys() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + while timeout > 0: + timeout -= 1 + try: + ssh.connect(host, username='root', port=port, pkey=key, timeout=1) + return ssh + except paramiko.ssh_exception.NoValidConnectionsError as exc: + print "ssh to %s failed: %s" % (host, str(exc)) + time.sleep(1) + except Exception as exc: + print "ssh to %s failed: %s" % (host, str(exc)) + return None + + +def ssh_exec(conn, cmd, timeout=5): + print "going to ssh_exec cmd %s" % cmd + stdin, stdout, stderr = conn.exec_command(cmd, timeout=timeout) + return (stdin, stdout.readlines(), stderr.readlines()) + + +class VMIntegTest(unittest.TestCase): + '''热升级功能集成测试类''' + + def _clear_all_vms(self): + print 'going to clear all vms...' + doms = LIBVIRT_CONN.listAllDomains() + for dom in doms: + if dom.isActive(): + dom.destroy() + doms = LIBVIRT_CONN.listAllDomains() + for dom in doms: + if dom.isPersistent(): + dom.undefine() + + def setUp(self): + self._clear_all_vms() + + def _clear_all_metadatafiles(self): + print 'going to clear all metadata files...' + fs = os.listdir(METADATA_DIR) + for f in fs: + if len(f) == len(UUID1): + os.remove(os.path.join(METADATA_DIR, f)) + + def _killall_nebd_servers(self): + cmd_killall = 'killall -9 %s' % os.path.basename(NEBD_SERVER_EXEC_FILE) + try: + run(cmd_killall) + except OSError: + pass + + def tearDown(self): + self._clear_all_vms() + self._killall_nebd_servers() + self._clear_all_metadatafiles() + + def test_create_destroy_vm(self): + cpuset = '0-%d' % (MAX_CPUS - 1) + if MAX_CPUS >= 2: + cpuset = '0-%d' % (MAX_CPUS / 2 - 1) + dominfo = {"uuid": UUID1, "cpuset": cpuset, + "name": "test_create_destroy_vm", + "disk_type": "network", + "protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), + "mon_hosts": MON_HOSTS + } + dom = self._create_vm(dominfo) + + if dom.isActive(): + # 检查nebd-server进程的cpu亲和性是否与qemu进程一致 + time.sleep(3) + nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + self.assertTrue(nebd_server_pid > 0) + ns_cpu_affinity = run('taskset -pc %d' % nebd_server_pid) + qemu_pid = self._get_pid_by_cmdline(dominfo['uuid'], QEMU_PROC_NAME) + self.assertTrue(qemu_pid > 0) + qemu_cpu_affinity = run('taskset -pc %d' % qemu_pid) + self.assertEqual(ns_cpu_affinity.split(':')[-1].strip(), + qemu_cpu_affinity.split(':')[-1].strip()) + self._destroy_vm(dominfo) + else: + self.assertFalse("Domain creates failed") + + def test_shutdown_vm(self): + rbd_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_shutdown_rbd_vm", + "disk_type": "network", + "protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), + "mon_hosts": MON_HOSTS + } + local_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_shutdown_local_vm", + "disk_type": "file", + "disk_file": LOCAL_IMG + } + dominfos = [rbd_vm, local_vm] + + for dominfo in dominfos: + dom = self._create_vm(dominfo) + if dom.isActive(): + self._destroy_vm(dominfo, shutdown=True, timeout=120) + else: + self.assertFalse("Domain creates failed") + + def test_reboot_vm(self): + rbd_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_reboot_rbd_vm", + "disk_type": "network", + "protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), + "mon_hosts": MON_HOSTS + } + local_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_reboot_local_vm", + "disk_type": "file", + "disk_file": LOCAL_IMG + } + dominfos = [rbd_vm, local_vm] + + for dominfo in dominfos: + dom = self._create_vm(dominfo) + if not dom.isActive(): + self.assertFalse("Domain creates failed") + # 等待虚拟机操作系统启动 + ssh = create_ssh_connect(VM_HOST_IP, timeout=120) + self.assertTrue(ssh is not None) + # 关机 + self._destroy_vm(dominfo, shutdown=True, timeout=120) + # 再启动（模拟重启） + dom = self._create_vm(dominfo) + if not dom.isActive(): + self.assertFalse("Domain creates failed") + # 等待虚拟机操作系统启动 + ssh = create_ssh_connect(VM_HOST_IP, timeout=120) + self.assertTrue(ssh is not None) + # 关机 + self._destroy_vm(dominfo) + + def test_poweroff_vm_inside(self): + rbd_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_poweroff_rbd_vm_inside", + "disk_type": "network", + "protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), + "mon_hosts": MON_HOSTS + } + local_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_poweroff_local_vm_inside", + "disk_type": "file", + "disk_file": LOCAL_IMG + } + dominfos = [rbd_vm, local_vm] + + for dominfo in dominfos: + dom = self._create_vm(dominfo) + if not dom.isActive(): + self.assertFalse("Domain creates failed") + + ssh = create_ssh_connect(VM_HOST_IP, timeout=120) + self.assertTrue(ssh is not None) + ssh_exec(ssh, 'poweroff') + + is_poweroff = False + poweroff_timeout = 30 + while poweroff_timeout > 0: + try: + dom = LIBVIRT_CONN.lookupByUUIDString(dominfo['uuid']) + if dom.info()[0] == libvirt.VIR_DOMAIN_SHUTOFF: + is_poweroff = True + break + poweroff_timeout -= 1 + print 'sleep 1s' + time.sleep(1) + except libvirt.libvirtError: + is_poweroff = True + break + self.assertTrue(is_poweroff) + + def test_reboot_vm_inside(self): + rbd_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_reboot_rbd_vm_inside", + "disk_type": "network", + "protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), + "mon_hosts": MON_HOSTS + } + local_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_reboot_local_vm_inside", + "disk_type": "file", + "disk_file": LOCAL_IMG + } + dominfos = [rbd_vm, local_vm] + + for dominfo in dominfos: + dom = self._create_vm(dominfo) + if not dom.isActive(): + self.assertFalse("Domain creates failed") + + ssh = create_ssh_connect(VM_HOST_IP, timeout=120) + self.assertTrue(ssh is not None) + ssh_exec(ssh, '>/var/log/wtmp') + ssh_exec(ssh, 'reboot') + + is_reboot = False + reboot_timeout = 30 + while reboot_timeout > 0: + ssh = create_ssh_connect(VM_HOST_IP, timeout=120) + if ssh is not None: + _, out, err = ssh_exec(ssh, 'last reboot|head -1') + print out, err + if 'reboot' in out[0]: + is_reboot = True + break + reboot_timeout -= 1 + print 'sleep 1s' + time.sleep(1) + self.assertTrue(is_reboot) + self._destroy_vm(dominfo) + + def test_pause_unpause_vm(self): + dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_pause_unpause_vm", + "disk_type": "network", + "protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), + "mon_hosts": MON_HOSTS + } + dom = self._create_vm(dominfo) + + if dom.isActive(): + ret = dom.suspend() + self.assertTrue(ret >= 0) + else: + self.assertFalse("Domain creates failed") + + self.assertEqual(3, dom.info()[0]) + ret = dom.resume() + self.assertTrue(ret >= 0) + self.assertEqual(1, dom.info()[0]) + ret = dom.destroy() + self.assertTrue(ret >= 0) + + def test_save_restore_vm(self): + rbd_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_save_restore_rbd_vm_kill_part2", + "disk_type": "network", + "protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), + "mon_hosts": MON_HOSTS + } + rbd_vm1 = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_save_restore_rbd_vm", + "disk_type": "network", + "protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), + "mon_hosts": MON_HOSTS + } + local_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_save_restore_local_vm", + "disk_type": "file", + "disk_file": LOCAL_IMG + } + dominfos = [rbd_vm, rbd_vm1, local_vm] + + # 依次测试rbd、rbd、local系统盘 + for dominfo in dominfos: + dom = self._create_vm(dominfo, transient=False) + part2_pid = -1 + if dom.isActive(): + part2_pid = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + self.assertTrue(part2_pid > 0) + ret = dom.save('/tmp/' + dominfo['uuid']) + self.assertTrue(ret >= 0) + else: + self.assertFalse("Domain creates failed") + + self.assertEqual(libvirt.VIR_DOMAIN_SHUTOFF, dom.info()[0]) + # rbd系统盘第一个用例测试kill part2场景下restore vm + if dominfos.index(dominfo) == 0: + try: + os.kill(part2_pid, 9) + except OSError as exc: + if "No such process" in str(exc): + pass + time.sleep(1) + self.assertFalse(self._check_proc_running(part2_pid, + dominfo['uuid'], NEBD_SERVER_EXEC_FILE)) + ret = LIBVIRT_CONN.restore('/tmp/' + dominfo['uuid']) + self.assertTrue(ret >= 0) + self.assertEqual(1, dom.info()[0]) + ret = dom.undefine() + self.assertTrue(ret >= 0) + ret = dom.destroy() + self.assertTrue(ret >= 0) + os.remove('/tmp/' + dominfo['uuid']) + + # 创建vm，并挂卸载卷，设置卷QoS + def _attach_detach_disk_on_vm(self, dominfo, diskinfo): + template = JJ_ENV.get_template('domain_xml.j2') + domxml = template.render(dominfo) + ret = LIBVIRT_CONN.defineXML(domxml) + self.assertTrue(ret >= 0) + dom = LIBVIRT_CONN.lookupByUUIDString(dominfo['uuid']) + + # 离线挂载卷测试 + template = JJ_ENV.get_template('disk_xml.j2') + diskxml = template.render(diskinfo) + ret = dom.attachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_CONFIG) + self.assertTrue(ret >= 0) + xml = dom.XMLDesc(0) + self.assertTrue(diskinfo['disk_name'] in xml) + + # 离线设置卷QoS + params = {'total_bytes_sec': 1024000, 'total_iops_sec': 100} + flag = libvirt.VIR_DOMAIN_AFFECT_CONFIG + ret = dom.setBlockIoTune(diskinfo['target_dev'], params, flag) + self.assertTrue(ret >= 0) + xml = dom.XMLDesc(0) + self.assertTrue(params.keys()[0] in xml + and str(params.values()[0]) in xml) + + # 离线卸载卷测试 + ret = dom.detachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_CONFIG) + self.assertTrue(ret >= 0) + xml = dom.XMLDesc(0) + self.assertFalse(diskinfo['disk_name'] in xml) + + # 在线挂载卷测试 + ret = dom.create() + self.assertTrue(ret >= 0) + + timeout = 30 + is_active = False + while timeout > 0: + try: + dom = LIBVIRT_CONN.lookupByUUIDString(dominfo['uuid']) + except: + self.assertFalse("Look up domain by uuid failed") + if dom.isActive(): + is_active = True + break + timeout -= 1 + print 'sleep 1s' + time.sleep(1) + + if not is_active: + self.assertFalse("Domain creates failed") + time.sleep(3) + ret = dom.attachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_LIVE) + self.assertTrue(ret >= 0) + xml = dom.XMLDesc(0) + self.assertTrue(diskinfo['disk_name'] in xml) + + # 在线设置卷QoS + params = {'write_bytes_sec': 1024000, 'read_bytes_sec': 2048000} + flag = libvirt.VIR_DOMAIN_AFFECT_LIVE + ret = dom.setBlockIoTune(diskinfo['target_dev'], params, flag) + self.assertTrue(ret >= 0) + xml = dom.XMLDesc(0) + self.assertTrue(params.keys()[0] in xml + and str(params.values()[0]) in xml) + + # 在线卸载卷测试 + time.sleep(3) + ret = dom.detachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_LIVE) + self.assertTrue(ret >= 0) + xml = dom.XMLDesc(0) + self.assertFalse(diskinfo['disk_name'] in xml) + + ret = dom.undefine() + self.assertTrue(ret >= 0) + ret = dom.destroy() + self.assertTrue(ret >= 0) + + # vm系统盘使用rbd卷，挂卸载rbd卷测试 + def test_attach_detach_rbd_disk_on_rbd_vm(self): + dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_attach_detach_rbd_disk_on_rbd_vm", + "disk_type": "network", + "protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), + "mon_hosts": MON_HOSTS + } + diskinfo = {"protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VOL1), + "mon_hosts": MON_HOSTS, + "target_dev": "vdc" + } + self._attach_detach_disk_on_vm(dominfo, diskinfo) + + # vm系统盘使用本地qcow2 file，挂卸载rbd卷测试 + def test_attach_detach_rbd_disk_on_local_vm(self): + dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_attach_detach_rbd_disk_on_local_vm", + "disk_type": "file", + "disk_file": LOCAL_IMG + } + diskinfo = {"protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VOL1), + "mon_hosts": MON_HOSTS, + "target_dev": "vdc" + } + self._attach_detach_disk_on_vm(dominfo, diskinfo) + + # vm系统盘使用rbd卷，挂卸载cbd卷测试 + def test_attach_detach_cbd_disk_on_rbd_vm(self): + # TODO + pass + + # vm系统盘使用本地qcow2 file，挂卸载rbd卷测试 + def test_attach_detach_cbd_disk_on_local_vm(self): + # TODO + pass + + def _destroy_vm(self, dominfo, shutdown=False, transient=True, timeout=5): + dom = LIBVIRT_CONN.lookupByUUIDString(dominfo['uuid']) + shutdown_timeout = timeout + if shutdown: + # 等待vm完全启动，通过尝试ssh来判断 + ssh_timeout = timeout * 2 / 3 + shutdown_timeout = timeout - ssh_timeout + ssh = create_ssh_connect(VM_HOST_IP, timeout=ssh_timeout) + ret = dom.shutdown() + else: + ret = dom.destroy() + self.assertTrue(ret >= 0) + + is_destroy = False + while shutdown_timeout > 0: + try: + dom = LIBVIRT_CONN.lookupByUUIDString(dominfo['uuid']) + if dom.info()[0] == libvirt.VIR_DOMAIN_SHUTOFF: + is_destroy = True + break + shutdown_timeout -= 1 + print 'sleep 1s' + time.sleep(1) + except libvirt.libvirtError: + is_destroy = True + break + self.assertTrue(is_destroy) + + if not transient: + ret = dom.undefine() + self.assertTrue(ret >= 0) + + # 创建虚拟机 + def _create_vm(self, dominfo, transient=True, + xml_j2='domain_xml.j2', timeout=30): + template = JJ_ENV.get_template(xml_j2) + domxml = template.render(dominfo) + if transient: # 非持久化vm + ret = LIBVIRT_CONN.createXML(domxml) + else: # 持久化vm + ret = LIBVIRT_CONN.defineXML(domxml) + self.assertTrue(ret >= 0) + + dom = LIBVIRT_CONN.lookupByUUIDString(dominfo['uuid']) + if not transient: + ret = dom.create() + self.assertTrue(ret >= 0) + + is_active = False + while timeout > 0: + try: + dom = LIBVIRT_CONN.lookupByUUIDString(dominfo['uuid']) + except libvirt.libvirtError: + self.assertFalse("Look up domain by uuid failed") + if dom.isActive(): + is_active = True + break + timeout -= 1 + print 'sleep 1s' + time.sleep(1) + + if not is_active: + self.assertFalse("Domain creates failed") + return dom + + def test_freeze_thaw_vm_rbd_disk(self): + dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_freeze_thaw_vm_rbd_disk", + "disk_type": "network", + "protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), + "mon_hosts": MON_HOSTS + } + dom = self._create_vm(dominfo) + + diskinfo = {"protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VOL1), + "mon_hosts": MON_HOSTS, + "target_dev": "vdc" + } + template = JJ_ENV.get_template('disk_xml.j2') + diskxml = template.render(diskinfo) + # 等待vm内部操作系统启动 + ssh = create_ssh_connect(VM_HOST_IP, timeout=120) + self.assertTrue(ssh is not None) + ret = dom.attachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_LIVE) + self.assertTrue(ret >= 0) + time.sleep(1) + xml = dom.XMLDesc(0) + self.assertTrue(diskinfo['disk_name'] in xml) + + try: + ret = dom.fsFreeze(diskinfo['target_dev']) + self.assertTrue(ret >= 0) + except libvirt.libvirtError as exc: + self.assertFalse(str(exc)) + try: + ret = dom.fsInfo() + self.assertTrue(ret >= 0) + except libvirt.libvirtError as exc: + self.assertTrue("guest-get-fsinfo has been disabled" in str(exc)) + except AttributeError as exc: + print 'Warning: %s' % str(exc) + try: + ret = dom.fsThaw(diskinfo['target_dev']) + self.assertTrue(ret >= 0) + except libvirt.libvirtError as exc: + self.assertFalse(str(exc)) + + ret = dom.destroy() + self.assertTrue(ret >= 0) + + # vm内部IO随机读写测试 + def _fio_randrw_in_vm(self, cmd, kill_nebd_server=False): + rbd_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_fio_randrw_in_rbd_vm", + "disk_type": "network", + "protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), + "mon_hosts": MON_HOSTS + } + local_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_fio_randrw_in_local_vm", + "disk_type": "file", + "disk_file": LOCAL_IMG + } + dominfos = [rbd_vm, local_vm] + + for dominfo in dominfos: + dom = self._create_vm(dominfo) + if not dom.isActive(): + self.assertFalse("Domain creates failed") + + ssh = create_ssh_connect(VM_HOST_IP, timeout=120) + self.assertTrue(ssh is not None) + nebd_server_pid0 = -1 + meta_dict0 = None + if kill_nebd_server and dominfo['disk_type'] == 'network': + with open(os.path.join(METADATA_DIR, dominfo['uuid'])) as mf: + metadata = mf.read() + meta_dict0 = json.loads(metadata) + nebd_server_pid0 = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + self.assertTrue(nebd_server_pid0 > 0) + # 设置定时器，5s之后杀掉nebd-server进程 + timer = Timer(5, os.kill, (nebd_server_pid0, 15)) + timer.start() + + _, out, err = ssh_exec(ssh, cmd, timeout=120) + self.assertTrue(len(out)) + fio_ok = 0 + for line in out: + if 'read' in line and 'IOPS' in line: + fio_ok += 1 + if 'write' in line and 'IOPS' in line: + fio_ok += 1 + if fio_ok == 2: + break + self.assertEqual(2, fio_ok) + + if kill_nebd_server and dominfo['disk_type'] == 'network': + # 检查是否重启，以及重启是否成功（pid发生变化且大于0） + nebd_server_pid1 = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + if dominfo == rbd_vm: # rbd镜像vm会重新拉起nebd-server + self.assertTrue(nebd_server_pid1 > 0) + elif dominfo == local_vm: # 本地镜像vm不会拉起nebd-server + self.assertTrue(nebd_server_pid1 < 0) + self.assertTrue(nebd_server_pid1 != nebd_server_pid0) + # 检查持久化文件，端口号及fd等信息不能发生变化 + with open(os.path.join(METADATA_DIR, dominfo['uuid'])) as mf: + metadata = mf.read() + meta_dict1 = json.loads(metadata) + self.assertTrue(meta_dict1 == meta_dict0) + + self._destroy_vm(dominfo) + + # vm内部fio读写测试(randrw 4k, seqrw 128k) + def test_fio_in_vm(self): + cmd_4k = ('/usr/bin/fio -name=/fio.data -direct=1 -iodepth=8 ' + + '-rw=randrw -ioengine=libaio -bs=4k -size=100M -numjobs=1') + self._fio_randrw_in_vm(cmd_4k, kill_nebd_server=False) + + cmd_128k = ('/usr/bin/fio -name=/fio.data -direct=1 -iodepth=8 ' + + '-rw=rw -ioengine=libaio -bs=128k -size=200M -numjobs=1') + self._fio_randrw_in_vm(cmd_128k, kill_nebd_server=False) + + # vm内部fio读写测试(randrw 4k, seqrw 128k)过程中kill nebd-server + def test_fio_in_vm_kill_nebd_server(self): + # 注意fio测试时长，要大于5s（定时器设置5s后kill nebd-server进程） + cmd_4k = ('/usr/bin/fio -name=/fio.data -direct=1 -iodepth=8 ' + + '-rw=randrw -ioengine=libaio -bs=4k -size=100M ' + + '-numjobs=1 -rate_iops=1000') + self._fio_randrw_in_vm(cmd_4k, kill_nebd_server=True) + + cmd_128k = ('/usr/bin/fio -name=/fio.data -direct=1 -iodepth=8 ' + + '-rw=rw -ioengine=libaio -bs=128k -size=200M ' + + '-numjobs=1 -rate_iops=60') + self._fio_randrw_in_vm(cmd_128k, kill_nebd_server=True) + + def test_start_vm_part2_running_with_rbd_disk(self): + # 1. 创建vm，rbd卷做系统盘 + dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_start_vm_part2_running_with_rbd_disk", + "disk_type": "network", + "protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), + "mon_hosts": MON_HOSTS + } + dom = self._create_vm(dominfo) + if not dom.isActive(): + self.assertFalse("Domain creates failed") + # 等待vm内部操作系统启动（保证rpc open卷成功） + ssh = create_ssh_connect(VM_HOST_IP, timeout=120) + self.assertTrue(ssh is not None) + nebd_server_pid0 = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + self.assertTrue(nebd_server_pid0 > 0) + metadata = None + with open(os.path.join(METADATA_DIR, dominfo['uuid'])) as mf: + metadata = mf.read() + meta_dict = json.loads(metadata) + # 检查fd大小是否在正确范围内 + for vol in meta_dict['volumes']: + self.assertTrue(1000 >= int(vol['fd']) > 0) + + # 检查rados admin socket是否正常 + asoks = os.listdir(CEPH_CLIENT_ASOK_DIR) + for asok in asoks: + if asok.startswith('ceph-client.admin.%d' % nebd_server_pid0): + asok_path = os.path.join(CEPH_CLIENT_ASOK_DIR, asok) + cmd_asok = 'ceph daemon %s version' % asok_path + out = run(cmd_asok) + self.assertTrue("version" in out) + + # 2. destroy vm + self._destroy_vm(dominfo) + + # 3. 在part2超时退出之前再次启动vm（模拟重启vm操作） + nebd_server_pid1 = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + self.assertTrue(nebd_server_pid1 > 0) + dom = self._create_vm(dominfo) + if not dom.isActive(): + self.assertFalse("Domain creates failed") + nebd_server_pid2 = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + self.assertTrue(nebd_server_pid2 > 0) + # 等待vm内部操作系统启动（保证rpc open卷成功） + ssh = create_ssh_connect(VM_HOST_IP, timeout=120) + self.assertTrue(ssh is not None) + nebd_server_pid3 = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + self.assertTrue(nebd_server_pid3 > 0) + self.assertTrue(nebd_server_pid0 == nebd_server_pid1 + == nebd_server_pid2 == nebd_server_pid3) + + def test_start_vm_without_part2_with_residual_metadatafile(self): + # 准备持久化信息文件（模拟残留） + port = 6300 + fake_uuid = '42d57731-19f6-478f-a145-b77a6d590274' + for uuid in [fake_uuid, UUID1, UUID2]: + meta_file = os.path.join(METADATA_DIR, uuid) + with open(meta_file, 'w') as mf: + if uuid == fake_uuid: + mf.write(json.dumps({"port": 6200})) # 模拟端口被占用 + else: + mf.write(json.dumps({"port": port})) + port += 1 + nebd_server_pid = self._get_pid_by_cmdline(uuid, + NEBD_SERVER_EXEC_FILE) + self.assertTrue(nebd_server_pid < 0) + # 1. 创建vm，rbd卷做系统盘 + dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_start_vm_without_part2_with_residual_metafile", + "disk_type": "network", + "protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), + "mon_hosts": MON_HOSTS + } + dom = self._create_vm(dominfo) + if not dom.isActive(): + self.assertFalse("Domain creates failed") + # 等待vm内部操作系统启动（保证rpc open卷成功） + ssh = create_ssh_connect(VM_HOST_IP, timeout=120) + self.assertTrue(ssh is not None) + nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + self.assertTrue(nebd_server_pid > 0) + # 检查父进程id是否为1（是否为daemon模式） + status = [] + with open(os.path.join('/proc/', str(nebd_server_pid), 'status')) as f: + status = f.readlines() + for line in status: + if line.startswith('PPid:'): + # 0 is for docker + self.assertTrue(int(line.split(':')[-1]) in [0, 1]) + break + # 检查端口号是否符合预期（检查part1清理持久化文件逻辑） + meta_file = os.path.join(METADATA_DIR, dominfo['uuid']) + metadata = None + with open(meta_file, 'r') as mf: + metadata = mf.read() + self.assertTrue(metadata) + meta_dict = json.loads(metadata) + self.assertTrue(int(meta_dict['port']) == 6201) + # 2. destroy vm + self._destroy_vm(dominfo) + # 等待nebd server进程退出，退出后检查持久化文件是否清理 + timeout = 30 + while timeout > 0: + timeout -= 1 + nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + if nebd_server_pid < 0: + self.assertFalse(os.path.exists(meta_file)) + self.assertTrue(os.path.exists( + os.path.join(METADATA_DIR, fake_uuid))) + self.assertTrue(os.path.exists( + os.path.join(METADATA_DIR, UUID2))) + break + else: + time.sleep(1) + + def test_nebd_server_boot_args(self): + # 创建一个使用本地文件做系统盘的虚拟机，为nebd-server心跳做准备 + dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_nebd_server_boot_args", + "disk_type": "file", + "disk_file": LOCAL_IMG + } + dom = self._create_vm(dominfo) + if not dom.isActive(): + self.assertFalse("Domain creates failed") + + # 等待vm内部操作系统启动（保证rpc open卷成功，如果有） + ssh = create_ssh_connect(VM_HOST_IP, timeout=120) + self.assertTrue(ssh is not None) + nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + self.assertTrue(nebd_server_pid < 0) + + # 正确参数 + cmd_ok = ' '.join([NEBD_SERVER_EXEC_FILE, '-uuid', dominfo['uuid']]) + out = run(cmd_ok) + self.assertFalse(out) + nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + self.assertTrue(nebd_server_pid > 0) + os.kill(nebd_server_pid, 9) + nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + self.assertTrue(nebd_server_pid < 0) + + # 错误参数1：uuid对应的qemu进程不存在 + fake_uuid = 'e35a4a69-853b-441e-a5ec-8b8a545ce20f' + cmd_ok = ' '.join([NEBD_SERVER_EXEC_FILE, '-uuid', fake_uuid]) + out = run(cmd_ok) + self.assertFalse(out) + time.sleep(1) + nebd_server_pid = self._get_pid_by_cmdline(fake_uuid, + NEBD_SERVER_EXEC_FILE) + self.assertTrue(nebd_server_pid < 0) + # 错误参数2：--uuid + cmd_ok = ' '.join([NEBD_SERVER_EXEC_FILE, '--uuid', dominfo['uuid']]) + out = run(cmd_ok) + self.assertFalse(out) + time.sleep(1) + nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + self.assertTrue(nebd_server_pid < 0) + + self._destroy_vm(dominfo) + + def test_nebd_server_kill_signal(self): + # 创建一个使用本地文件做系统盘的虚拟机，为nebd-server心跳做准备 + dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_nebd_server_kill_signal", + "disk_type": "file", + "disk_file": LOCAL_IMG + } + dom = self._create_vm(dominfo) + if not dom.isActive(): + self.assertFalse("Domain creates failed") + + # 等待vm内部操作系统启动（保证rpc open卷成功，如果有） + ssh = create_ssh_connect(VM_HOST_IP, timeout=120) + self.assertTrue(ssh is not None) + nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + self.assertTrue(nebd_server_pid < 0) + + # SIGINT(2), SIGTERM(15) + for sig in [2, 15]: + port = 6300 + meta_file = os.path.join(METADATA_DIR, dominfo['uuid']) + with open(meta_file, 'w') as mf: + mf.write(json.dumps({"port": port})) + cmd_ok = ' '.join([NEBD_SERVER_EXEC_FILE, '-uuid', dominfo['uuid']]) + out = run(cmd_ok) + self.assertFalse(out) + nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + self.assertTrue(nebd_server_pid > 0) + # os.kill(nebd_server_pid, sig) + # docker里执行os.kill貌似进程会收不到信号，改成kill命令就好了。。。 + cmd_kill = "/bin/kill -%d %d" % (sig, nebd_server_pid) + out = run(cmd_kill) + self.assertFalse(out) + timeout = 5 + is_exit = False + while timeout > 0: + timeout -= 1 + nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + if nebd_server_pid < 0: + is_exit = True + break + else: + time.sleep(1) + self.assertTrue(is_exit) + self.assertTrue(os.path.exists(meta_file)) + self._clear_all_metadatafiles() + + self._destroy_vm(dominfo) + + def _check_proc_running(self, pid, uuid, name): + '''根据/proc/pid/cmdline文件检查进程名是否与给定的名称匹配，进程参数是否包含给定的uuid''' + cmdline = "" + try: + with open(os.path.join("/proc", str(pid), "cmdline")) as f: + cmdline = f.read() + except IOError: + return False + + cmdline = cmdline.split('\0') + proc_name = os.path.basename(cmdline[0]) + if proc_name == os.path.basename(name): + if uuid: + if uuid in cmdline: + return True + else: + return False + return True + return False + + def _get_pid_by_cmdline(self, uuid, name): + pids = os.listdir("/proc") + for pid in pids: + if pid.isdigit() and self._check_proc_running(pid, uuid, name): + return int(pid) + return -1 + + def test_force_kill_vm_qemu_proc(self): + dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_force_kill_vm_qemu_proc", + "disk_type": "network", + "protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), + "mon_hosts": MON_HOSTS + } + dom = self._create_vm(dominfo) + time.sleep(5) + qemu_pid = self._get_pid_by_cmdline(dominfo['uuid'], QEMU_PROC_NAME) + self.assertTrue(qemu_pid > 0) + part2_pid = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + self.assertTrue(part2_pid > 0) + os.kill(qemu_pid, 9) + time.sleep(1) + self.assertFalse(self._check_proc_running(qemu_pid, dominfo['uuid'], + QEMU_PROC_NAME)) + + timeout = 30 + part2_exit = False + while timeout > 0: + if not self._check_proc_running(part2_pid, dominfo['uuid'], + NEBD_SERVER_EXEC_FILE): + part2_exit = True + break + timeout -= 1 + print 'sleep 1s' + time.sleep(1) + + self.assertTrue(part2_exit) + + # TODO: 检查后端卷被close + + # 检查持久化metadata file被清理 + self.assertFalse(os.path.exists( + os.path.join(METADATA_DIR, dominfo['uuid']))) + + def test_create_destroy_local_vm_without_part2(self): + dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_create_destroy_local_vm_without_part2", + "disk_type": "file", + "disk_file": LOCAL_IMG + } + dom = self._create_vm(dominfo) + time.sleep(5) + # 保证part2进程不会启动 + part2_pid = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + self.assertTrue(part2_pid < 0) + # 检查持久化metadata file被清理 + self.assertFalse( + os.path.exists(os.path.join(METADATA_DIR, dominfo['uuid']))) + + # 测试destroy vm + self._destroy_vm(dominfo) + + def test_disk_operations_in_vm(self): + dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_disk_operations_in_vm", + "disk_type": "file", + "disk_file": LOCAL_IMG + } + dom = self._create_vm(dominfo) + if not dom.isActive(): + self.assertFalse("Domain creates failed") + + # attach disk + diskinfo = {"protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VOL1), + "mon_hosts": MON_HOSTS, + "target_dev": "vdc" + } + template = JJ_ENV.get_template('disk_xml.j2') + diskxml = template.render(diskinfo) + ret = dom.attachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_LIVE) + self.assertTrue(ret >= 0) + xml = dom.XMLDesc(0) + self.assertTrue(diskinfo['disk_name'] in xml) + + # ssh到vm内部执行如下操作： + # 1. lsblk + # 2. parted(mkpart) + # 3. mkfs.(ext3, ext4, xfs) + # 3.1 mount + # 3.2 df + # 3.3 dd + # 3.4 sync + # 3.5 umount + # 4. parted(rm) + ssh = create_ssh_connect(VM_HOST_IP, timeout=120) + self.assertTrue(ssh is not None) + cmd_lsblk = '/usr/bin/lsblk -b|grep vd|grep disk|tail -1' + _, out, err = ssh_exec(ssh, cmd_lsblk, timeout=3) + self.assertTrue(len(out)) + vd = out[0].split()[0] + self.assertTrue(vd.startswith('vd')) + disk_size = int(out[0].split()[3]) + self.assertTrue(disk_size > (RBD_VOL1_SIZE * 0.8) + and disk_size < RBD_VOL1_SIZE * 1.1) + disk = os.path.join('/dev', vd) + cmd_parted_mkpart1 = ('/usr/sbin/parted -s %s ' % disk + + 'mklabel gpt mkpart primary 1 200M') + _, out, err = ssh_exec(ssh, cmd_parted_mkpart1, timeout=5) + self.assertTrue(out == [] and err == []) + cmd_parted_mkpart2 = ('/usr/sbin/parted -s %s ' % disk + + 'mkpart primary 200M 500M') + _, out, err = ssh_exec(ssh, cmd_parted_mkpart2, timeout=5) + self.assertTrue(out == [] and err == []) + + for fs in ['ext3', 'ext4', 'xfs']: + cmd_mkfs = '/sbin/mkfs.%s -F %s1' % (fs, disk) + _, out, err = ssh_exec(ssh, cmd_mkfs, timeout=30) + cmd_mount = '/usr/bin/mount %s1 /mnt' % disk + _, out, err = ssh_exec(ssh, cmd_mount, timeout=5) + cmd_df = '/bin/df | grep /mnt' + _, out, err = ssh_exec(ssh, cmd_df, timeout=5) + self.assertTrue(len(out) and disk in out[0]) + cmd_dd = ('/bin/dd if=/dev/zero of=/mnt/dd.zero ' + + 'bs=1M count=100 oflag=direct') + _, out, err = ssh_exec(ssh, cmd_dd, timeout=30) + _, out, err = ssh_exec(ssh, 'sync', timeout=30) + cmd_umount = '/usr/bin/umount /mnt' + _, out, err = ssh_exec(ssh, cmd_umount, timeout=5) + + cmd_parted_rmpart1 = '/usr/sbin/parted -s %s ' % disk + 'rm 1' + _, out, err = ssh_exec(ssh, cmd_parted_rmpart1, timeout=5) + self.assertTrue(out == [] and err == []) + cmd_parted_rmpart2 = '/usr/sbin/parted -s %s ' % disk + 'rm 2' + _, out, err = ssh_exec(ssh, cmd_parted_rmpart2, timeout=5) + self.assertTrue(out == [] and err == []) + cmd_lsblk = '/usr/bin/lsblk -l|grep %s|grep part' % vd + _, out, err = ssh_exec(ssh, cmd_lsblk, timeout=3) + self.assertTrue(out == []) + + # 卸载卷 + ret = dom.detachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_LIVE) + self.assertTrue(ret >= 0) + xml = dom.XMLDesc(0) + self.assertTrue(diskinfo['disk_name'] not in xml) + + # 挂载scsi disk，测试fstrim操作 + diskinfo = {"protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VOL1), + "mon_hosts": MON_HOSTS, + "target_dev": "sdc" + } + template = JJ_ENV.get_template('disk_xml.j2') + diskxml = template.render(diskinfo) + ret = dom.attachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_LIVE) + self.assertTrue(ret >= 0) + xml = dom.XMLDesc(0) + self.assertTrue(diskinfo['disk_name'] in xml and 'scsi' in xml) + ssh = create_ssh_connect(VM_HOST_IP, timeout=120) + self.assertTrue(ssh is not None) + cmd_lsblk = '/usr/bin/lsblk -b|grep sd|grep disk|tail -1' + _, out, err = ssh_exec(ssh, cmd_lsblk, timeout=3) + self.assertTrue(len(out)) + sd = out[0].split()[0] + self.assertTrue(sd.startswith('sd')) + disk = os.path.join('/dev', sd) + cmd_mkfs = '/sbin/mkfs.ext4 -F %s' % disk + _, out, err = ssh_exec(ssh, cmd_mkfs, timeout=30) + cmd_mount = '/usr/bin/mount %s /mnt' % disk + _, out, err = ssh_exec(ssh, cmd_mount, timeout=5) + cmd_df = '/bin/df | grep /mnt' + _, out, err = ssh_exec(ssh, cmd_df, timeout=5) + self.assertTrue(len(out) and disk in out[0]) + cmd_dd = ('/bin/dd if=/dev/zero of=/mnt/dd.zero ' + + 'bs=1M count=100 oflag=direct') + _, out, err = ssh_exec(ssh, cmd_dd, timeout=30) + cmd_rm = '/bin/rm /mnt/dd.zero' + _, out, err = ssh_exec(ssh, cmd_rm, timeout=5) + cmd_trim = '/sbin/fstrim /mnt' + _, out, err = ssh_exec(ssh, cmd_trim, timeout=30) + self.assertTrue(out == [] and err == []) + + self._destroy_vm(dominfo) + + def test_resize_rbd_disk(self): + dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_resize_rbd_disk", + "disk_type": "network", + "protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), + "mon_hosts": MON_HOSTS + } + dom = self._create_vm(dominfo) + time.sleep(5) + + if dom.isActive(): + capacity = dom.blockInfo('vda')[0] # Byte + new_size = (capacity + RBD_VM_IMG_SIZE) / 1024 # KB + ret = dom.blockResize('vda', new_size) + self.assertTrue(ret >= 0) + self.assertTrue(dom.blockInfo('vda')[0] > + (capacity + RBD_VM_IMG_SIZE * 0.8)) + else: + self.assertFalse("Domain creates failed") + self._destroy_vm(dominfo) + + def test_live_upgrade_ceph_client(self): + old_version = CEPH_CLIENT_REAL_VERSION[CEPH_CLIENT_OLD_PACKAGE] + new_version = CEPH_CLIENT_REAL_VERSION[CEPH_CLIENT_NEW_PACKAGE] + packages = ['ceph', 'ceph-common', 'librbd1', 'python-rados', + 'python-cephfs', 'python-rbd', 'librados2', + 'libcephfs1', 'python-ceph'] + + rbd_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_live_upgrade_ceph_client", + "disk_type": "network", + "protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), + "mon_hosts": MON_HOSTS + } + local_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_live_upgrade_ceph_client", + "disk_type": "file", + "disk_file": LOCAL_IMG + } + dominfos = [rbd_vm, local_vm] + + for dominfo in dominfos: + # 安装ceph client旧包 + cmd_install_old_package = 'apt-get install' + for p in packages: + cmd_install_old_package = ' '.join([cmd_install_old_package, p]) + cmd_install_old_package += ('=%s' % CEPH_CLIENT_OLD_PACKAGE) + cmd_install_old_package += ' -y --force-yes' + run(cmd_install_old_package, timeout=30) + out = run('/usr/bin/dpkg -l librbd1') + + self.assertTrue(out and CEPH_CLIENT_OLD_PACKAGE in out) + dom = self._create_vm(dominfo) + if not dom.isActive(): + self.assertFalse("Domain creates failed") + # attach disk + diskinfo = {"protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VOL1), + "mon_hosts": MON_HOSTS, + "target_dev": "vdc" + } + template = JJ_ENV.get_template('disk_xml.j2') + diskxml = template.render(diskinfo) + ret = dom.attachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_LIVE) + self.assertTrue(ret >= 0) + xml = dom.XMLDesc(0) + self.assertTrue(diskinfo['disk_name'] in xml) + + ssh = create_ssh_connect(VM_HOST_IP, timeout=120) + self.assertTrue(ssh is not None) + cmd_lsblk = '/usr/bin/lsblk -b|grep vd|grep disk|tail -1' + _, out, err = ssh_exec(ssh, cmd_lsblk, timeout=3) + self.assertTrue(len(out)) + vd = out[0].split()[0] + self.assertTrue(vd.startswith('vd')) + disk = os.path.join('/dev', vd) + cmd_dd = ('/bin/dd if=/dev/zero of=%s ' % disk + + 'bs=1M count=100 oflag=direct') + _, out, err = ssh_exec(ssh, cmd_dd, timeout=30) + self.assertEqual([], out) + self.assertTrue(len(err) and 'records' in err[0]) + + # 安装ceph client新包 + cmd_install_new_package = 'apt-get install' + for p in packages: + cmd_install_new_package = ' '.join([cmd_install_new_package, p]) + cmd_install_new_package += ('=%s' % CEPH_CLIENT_NEW_PACKAGE) + cmd_install_new_package += ' -y --force-yes' + run(cmd_install_new_package, timeout=30) + out = run('/usr/bin/dpkg -l librbd1') + self.assertTrue(out and CEPH_CLIENT_NEW_PACKAGE in out) + + nebd_server_pid0 = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + self.assertTrue(nebd_server_pid0 > 0) + # 检查rados admin socket是否正常 + asoks = os.listdir(CEPH_CLIENT_ASOK_DIR) + for asok in asoks: + if asok.startswith('ceph-client.admin.%d' % nebd_server_pid0): + asok_path = os.path.join(CEPH_CLIENT_ASOK_DIR, asok) + cmd_asok = 'ceph daemon %s version' % asok_path + out = run(cmd_asok) + self.assertTrue(out and "version" in out and + old_version in out) + os.kill(nebd_server_pid0, 15) + timeout = 10 + is_reboot = False + while timeout > 0: + timeout -= 1 + nebd_server_pid1 = self._get_pid_by_cmdline(dominfo['uuid'], + NEBD_SERVER_EXEC_FILE) + if (nebd_server_pid1 > 0 and + nebd_server_pid1 != nebd_server_pid0): + is_reboot = True + asoks = os.listdir(CEPH_CLIENT_ASOK_DIR) + for asok in asoks: + if asok.startswith('ceph-client.admin.%d' % + nebd_server_pid1): + asok_path = os.path.join(CEPH_CLIENT_ASOK_DIR, asok) + cmd_asok = 'ceph daemon %s version' % asok_path + out = run(cmd_asok) + self.assertTrue(out and "version" in out and + new_version in out) + break + else: + time.sleep(1) + self.assertTrue(is_reboot) + _, out, err = ssh_exec(ssh, cmd_dd, timeout=30) + self.assertEqual([], out) + self.assertTrue(len(err) and 'records' in err[0]) + self._destroy_vm(dominfo) + self._killall_nebd_servers() + + def test_live_upgrade_curve_client(self): + # TODO + pass + + def test_ceph_rbd_rados_cli(self): + try: + ceph_cmd_base = ('ceph -c %s ' % CEPH_CONF + + '--admin_socket /tmp/ceph-client.asok ') + out = run(ceph_cmd_base + '-s') + self.assertTrue('cluster' in out) + out = run(ceph_cmd_base + 'osd tree') + self.assertTrue('ID' in out and 'WEIGHT' in out) + out = run(ceph_cmd_base + 'osd pool ls detail') + self.assertTrue('pool' in out and RBD_POOL in out) + + rbd_cmd_base = ('rbd -c %s ' % CEPH_CONF + + '--admin_socket /tmp/ceph-client.asok ') + out = run(rbd_cmd_base + 'ls -p %s' % RBD_POOL) + self.assertTrue(RBD_VM_IMG in out) + out = run(rbd_cmd_base + + 'info %s' % os.path.join(RBD_POOL, RBD_VM_IMG)) + self.assertTrue(RBD_VM_IMG in out) + out = run(rbd_cmd_base + + 'create --image-format 2 --size 128 %s' + % os.path.join(RBD_POOL, 'rbd-cli-test')) + self.assertFalse(out) + out = run(rbd_cmd_base + + 'snap create %s' + % os.path.join(RBD_POOL, 'rbd-cli-test@snap-test')) + self.assertFalse(out) + out = run(rbd_cmd_base + + 'snap ls %s' % os.path.join(RBD_POOL, 'rbd-cli-test')) + self.assertTrue('snap-test' in out) + out = run(rbd_cmd_base + + 'snap rm %s' + % os.path.join(RBD_POOL, 'rbd-cli-test@snap-test')) + self.assertFalse(out) + out = run(rbd_cmd_base + + 'rm %s' % os.path.join(RBD_POOL, 'rbd-cli-test'), + ignore='Removing image') + self.assertFalse(out) + + rados_cmd_base = ('rados -c %s ' % CEPH_CONF + + '--admin_socket /tmp/ceph-client.asok ') + out = run(rados_cmd_base + 'lspools') + self.assertTrue(RBD_POOL in out) + out = run(rados_cmd_base + 'df') + self.assertTrue(RBD_POOL in out) + except Exception as exc: + self.assertFalse(str(exc)) + + +def _get_rbd_ctx(): + cluster = rados.Rados(conffile=CEPH_CONF) + cluster.connect() + ioctx = cluster.open_ioctx(RBD_POOL) + rbd_inst = rbd.RBD() + return (ioctx, rbd_inst, cluster) + + +def prepare_rbd_disk(): + if REUSE_RBD_DISK: + return + clear_rbd_disk() + + print 'going to create all rbd disks...' + ioctx, rbd_inst, cluster = _get_rbd_ctx() + rbd_inst.create(ioctx, RBD_VOL1, RBD_VOL1_SIZE, + 22, old_format=False, features=1) + rbd_inst.create(ioctx, RBD_VM_IMG, RBD_VM_IMG_SIZE, + 22, old_format=False, features=1) + image = rbd.Image(ioctx, RBD_VM_IMG) + # LOCAL_IMG需要为raw格式镜像，虚拟机xml里面指定了raw + # 并且导入到rbd卷也需要是raw格式 + f = open(LOCAL_IMG, "rb") + chunk_size = 4*1024*1024 + try: + offset = 0 + data = f.read(chunk_size) + while data != "": + print "Writing data at offset " + str(offset) + \ + "(" + str(offset / 1024 / 1024) + "MB)" + offset += image.write(data, offset) + data = f.read(chunk_size) + finally: + f.close() + image.close() + ioctx.close() + cluster.shutdown() + + + +def clear_rbd_disk(): + if REUSE_RBD_DISK: + return + print 'going to rm all rbd disks...' + ioctx, rbd_inst, cluster = _get_rbd_ctx() + for img in [RBD_VM_IMG, RBD_VOL1]: + try: + rbd_inst.remove(ioctx, img) + except rbd.ImageNotFound: + pass + except Exception as exc: + ioctx.close() + cluster.shutdown() + raise exc + ioctx.close() + cluster.shutdown() + + +def clear_log_dirs(): + print 'going to clear all log files...' + for d in ['/var/log/nebd', '/var/log/libvirt/qemu']: + fs = os.listdir(d) + for f in fs: + os.remove(os.path.join(d, f)) + + +def install_rbd_client(): + # 安装ceph client新包 + packages = ['ceph', 'ceph-common', 'librbd1', 'python-rados', + 'python-cephfs', 'python-rbd', 'librados2', + 'libcephfs1', 'python-ceph'] + cmd_install_new_package = 'apt-get install' + for p in packages: + cmd_install_new_package = ' '.join([cmd_install_new_package, p]) + cmd_install_new_package += ('=%s' % CEPH_CLIENT_NEW_PACKAGE) + cmd_install_new_package += ' -y --force-yes' + run(cmd_install_new_package, timeout=30) + out = run('/usr/bin/dpkg -l grep librbd1') + assert(out and CEPH_CLIENT_NEW_PACKAGE in out) + + +if __name__ == '__main__': + clear_log_dirs() + install_rbd_client() + prepare_rbd_disk() + path = '{}/templates/'.format(os.path.dirname(os.path.abspath(__file__))) + if JJ_ENV is None: + JJ_ENV = Environment(loader=FileSystemLoader(path)) + if LIBVIRT_CONN is None: + LIBVIRT_CONN = libvirt.open(None) + + unittest.main() + + LIBVIRT_CONN.close() + clear_rbd_disk() From 580aa00b171277d4933af4c45221cb6bc8ca0feb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=A8=81?= Date: Tue, 22 Oct 2019 13:59:38 +0800 Subject: [PATCH 06/79] fix part1 aioRpcFailLogInterval not asign bug Change-Id: I25d4bc6439cdd24c744c18bc639e3f05495a6511 --- src/part1/nebd_client.cpp | 12 ++++++++---- src/part1/nebd_client.h | 30 ++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/part1/nebd_client.cpp b/src/part1/nebd_client.cpp index 9eeb8c3fbe..957c3758f0 100644 --- a/src/part1/nebd_client.cpp +++ b/src/part1/nebd_client.cpp @@ -232,7 +232,8 @@ int FileClient::Discard(int fd, ClientAioContext* aioctx) { request.set_size(aioctx->length); DiscardDone *discardDone = new DiscardDone(fd, aioctx, rpcTimeoutMs_, rpcRetryIntervalUs_, rpcRetryMaxIntervalUs_, - rpcHostDownRetryIntervalUs_); + rpcHostDownRetryIntervalUs_, + aioRpcFailLogInterval_); stub.Discard(&discardDone->cntl_, &request, &discardDone->response_, discardDone); return 0; @@ -249,7 +250,8 @@ int FileClient::AioRead(int fd, ClientAioContext* aioctx) { request.set_size(aioctx->length); ReadDone *readDone = new ReadDone(fd, aioctx, rpcTimeoutMs_, rpcRetryIntervalUs_, rpcRetryMaxIntervalUs_, - rpcHostDownRetryIntervalUs_); + rpcHostDownRetryIntervalUs_, + aioRpcFailLogInterval_); stub.Read(&readDone->cntl_, &request, &readDone->response_, readDone); return 0; } @@ -265,7 +267,8 @@ int FileClient::AioWrite(int fd, ClientAioContext* aioctx) { request.set_size(aioctx->length); WriteDone *writeDone = new WriteDone(fd, aioctx, rpcTimeoutMs_, rpcRetryIntervalUs_, rpcRetryMaxIntervalUs_, - rpcHostDownRetryIntervalUs_); + rpcHostDownRetryIntervalUs_, + aioRpcFailLogInterval_); writeDone->cntl_.request_attachment().append(aioctx->buf, aioctx->length); stub.Write(&writeDone->cntl_, &request, &writeDone->response_, writeDone); return 0; @@ -278,7 +281,8 @@ int FileClient::Flush(int fd, ClientAioContext* aioctx) { request.set_fd(fd); FlushDone *flushDone = new FlushDone(fd, aioctx, rpcTimeoutMs_, rpcRetryIntervalUs_, rpcRetryMaxIntervalUs_, - rpcHostDownRetryIntervalUs_); + rpcHostDownRetryIntervalUs_, + aioRpcFailLogInterval_); stub.Flush(&flushDone->cntl_, &request, &flushDone->response_, flushDone); return 0; } diff --git a/src/part1/nebd_client.h b/src/part1/nebd_client.h index 4d1ea1c3fe..b3ed05de53 100644 --- a/src/part1/nebd_client.h +++ b/src/part1/nebd_client.h @@ -140,13 +140,15 @@ class AsyncRequestDone: public google::protobuf::Closure { uint32_t rpcTimeoutMs, uint32_t rpcRetryIntervalUs, uint32_t rpcRetryMaxIntervalUs, - uint32_t rpcHostDownRetryIntervalUs): + uint32_t rpcHostDownRetryIntervalUs, + uint32_t aioRpcFailLogInterval): fd_(fd), aioctx_(aioctx), rpcTimeoutMs_(rpcTimeoutMs), rpcRetryIntervalUs_(rpcRetryIntervalUs), rpcRetryMaxIntervalUs_(rpcRetryMaxIntervalUs), - rpcHostDownRetryIntervalUs_(rpcHostDownRetryIntervalUs) { + rpcHostDownRetryIntervalUs_(rpcHostDownRetryIntervalUs), + aioRpcFailLogInterval_(aioRpcFailLogInterval) { cntl_.set_timeout_ms(rpcTimeoutMs); } @@ -182,10 +184,12 @@ class ReadDone: public AsyncRequestDone { public: ReadDone(int fd, ClientAioContext* aioctx, uint32_t rpcTimeoutMs, uint32_t rpcRetryIntervalUs, uint32_t rpcRetryMaxIntervalUs, - uint32_t rpcHostDownRetryIntervalUs): + uint32_t rpcHostDownRetryIntervalUs, + uint32_t aioRpcFailLogInterval): AsyncRequestDone(fd, aioctx, rpcTimeoutMs, rpcRetryIntervalUs, rpcRetryMaxIntervalUs, - rpcHostDownRetryIntervalUs) {} + rpcHostDownRetryIntervalUs, + aioRpcFailLogInterval) {} void Run(); nebd::client::ReadResponse response_; }; @@ -194,10 +198,12 @@ class WriteDone: public AsyncRequestDone { public: WriteDone(int fd, ClientAioContext* aioctx, uint32_t rpcTimeoutMs, uint32_t rpcRetryIntervalUs, uint32_t rpcRetryMaxIntervalUs, - uint32_t rpcHostDownRetryIntervalUs): + uint32_t rpcHostDownRetryIntervalUs, + uint32_t aioRpcFailLogInterval): AsyncRequestDone(fd, aioctx, rpcTimeoutMs, rpcRetryIntervalUs, rpcRetryMaxIntervalUs, - rpcHostDownRetryIntervalUs) {} + rpcHostDownRetryIntervalUs, + aioRpcFailLogInterval) {} void Run(); nebd::client::WriteResponse response_; }; @@ -206,10 +212,12 @@ class DiscardDone: public AsyncRequestDone { public: DiscardDone(int fd, ClientAioContext* aioctx, uint32_t rpcTimeoutMs, uint32_t rpcRetryIntervalUs, uint32_t rpcRetryMaxIntervalUs, - uint32_t rpcHostDownRetryIntervalUs): + uint32_t rpcHostDownRetryIntervalUs, + uint32_t aioRpcFailLogInterval): AsyncRequestDone(fd, aioctx, rpcTimeoutMs, rpcRetryIntervalUs, rpcRetryMaxIntervalUs, - rpcHostDownRetryIntervalUs) {} + rpcHostDownRetryIntervalUs, + aioRpcFailLogInterval) {} void Run(); nebd::client::DiscardResponse response_; }; @@ -218,10 +226,12 @@ class FlushDone: public AsyncRequestDone { public: FlushDone(int fd, ClientAioContext* aioctx, uint32_t rpcTimeoutMs, uint32_t rpcRetryIntervalUs, uint32_t rpcRetryMaxIntervalUs, - uint32_t rpcHostDownRetryIntervalUs): + uint32_t rpcHostDownRetryIntervalUs, + uint32_t aioRpcFailLogInterval): AsyncRequestDone(fd, aioctx, rpcTimeoutMs, rpcRetryIntervalUs, rpcRetryMaxIntervalUs, - rpcHostDownRetryIntervalUs) {} + rpcHostDownRetryIntervalUs, + aioRpcFailLogInterval) {} void Run(); nebd::client::FlushResponse response_; }; From 0544df692c69b02f09c2dfbc351a1d23d2278e93 Mon Sep 17 00:00:00 2001 From: hzwuhongsong Date: Tue, 22 Oct 2019 06:38:38 +0000 Subject: [PATCH 07/79] modify tc coredump Change-Id: Ia5076200640d0464770b800eecd84aeae828d4de --- tests/part2/TestHeartbeat2.cpp | 38 +--------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/tests/part2/TestHeartbeat2.cpp b/tests/part2/TestHeartbeat2.cpp index 1cb4c62120..c024e17ae8 100644 --- a/tests/part2/TestHeartbeat2.cpp +++ b/tests/part2/TestHeartbeat2.cpp @@ -129,53 +129,17 @@ TEST(CloseQemuDetachedVolumes, CloseQemuDetachedVolumes_5) { EXPECT_EQ(-1, CloseQemuDetachedVolumes(0)); } -/* -TEST(CloseQemuDetachedVolumes, CloseQemuDetachedVolumes_6) -{ - int fd = 6; - char* filename = -const_cast("rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - FdImage_t fd_image; - fd_image.filename = filename; - - int fd2 = 7; - char* filename2 = const_cast("mon_host=10.182.30.27"); - FdImage_t fd_image2; - fd_image2.filename = filename2; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, &fd_image)); - g_imageMap.insert(std::pair(fd2, &fd_image2)); - - g_qemuPoolVolumes.push_back("rbd/volume02"); - NebdServerMocker mock; - EXPECT_CALL(mock, ReadQemuXmls()).WillOnce(testing::Return(0)); - std::string lock_file = "/tmp/unit_test_lockfile"; - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, UnLockFile(::testing::_)).WillOnce(testing::Return()); - EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, RmFd(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_EQ(0, CloseQemuDetachedVolumes(0)); -} -*/ TEST(CloseQemuDetachedVolumes, - CloseQemuDetachedVolumes_7) { + CloseQemuDetachedVolumes_6) { int fd = 6; char* filename = const_cast( "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); FdImage_t fd_image; fd_image.filename = filename; - int fd2 = 7; - char* filename2 = const_cast("mon_host=10.182.30.27"); - FdImage_t fd_image2; - fd_image2.filename = filename2; - g_imageMap.clear(); g_imageMap.insert(std::pair(fd, &fd_image)); - g_imageMap.insert(std::pair(fd2, &fd_image2)); g_qemuPoolVolumes.push_back("rbd/volume01"); NebdServerMocker mock; From a2d149764f9916c4811bdd2d6b3e800f5f0235c8 Mon Sep 17 00:00:00 2001 From: hzwuhongsong Date: Wed, 23 Oct 2019 15:20:45 +0800 Subject: [PATCH 08/79] fix bug Change-Id: Icfaa02020b0e6467f2cbbdbc4bb1792dbe161b55 CLDNBS-1473 --- src/part2/common_type.cpp | 10 +++++----- src/part2/common_type.h | 3 +++ src/part2/heartbeat.cpp | 2 ++ tests/part2/TestCommon.cpp | 18 ++++++++++++++++++ 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/part2/common_type.cpp b/src/part2/common_type.cpp index 74908e30fe..6a50f58f45 100644 --- a/src/part2/common_type.cpp +++ b/src/part2/common_type.cpp @@ -45,6 +45,9 @@ std::map g_imageMap; // 保存云主机当前挂载的卷信息 std::vector g_qemuPoolVolumes; +// brpc server +brpc::Server g_server; + // 文件锁函数 int LockFile(const char* file) { int lockfd = open(file, O_RDONLY | O_CREAT, 0644); @@ -532,9 +535,8 @@ int Init() { options.idle_timeout_sec = rpc_option->brpc_idle_timeout; options.num_threads = rpc_option->brpc_num_threads; delete rpc_option; - brpc::Server brpc_server; QemuClientServiceImpl* qemuclient_service_impl = new QemuClientServiceImpl; - if (brpc_server.AddService(qemuclient_service_impl, + if (g_server.AddService(qemuclient_service_impl, brpc::SERVER_OWNS_SERVICE) != 0) { LOG(ERROR) << "Fail to add service"; return -1; @@ -579,7 +581,7 @@ int Init() { std::string ip = "127.0.0.1:"; while (++retry <= retry_counts) { std::string ip_port = ip + std::to_string(g_port); - if (brpc_server.Start(ip_port.c_str(), &options) != 0) { + if (g_server.Start(ip_port.c_str(), &options) != 0) { LOG(ERROR) << "Fail to start QemuClientServer, retry time is: " << retry; g_port = FindPort(cfg, g_port + 1); @@ -621,7 +623,5 @@ int Init() { signal(SIGINT, SigProcess); signal(SIGHUP, SigLogReplace); bthread_join(th, NULL); - brpc_server.Stop(0); - brpc_server.Join(); return 0; } diff --git a/src/part2/common_type.h b/src/part2/common_type.h index cfdbf8fc51..9c5ff65ccd 100644 --- a/src/part2/common_type.h +++ b/src/part2/common_type.h @@ -17,6 +17,9 @@ #include #include +// brpc server +extern brpc::Server g_server; + typedef struct FdImage { rados_t* cluster; rados_ioctx_t* io; diff --git a/src/part2/heartbeat.cpp b/src/part2/heartbeat.cpp index 684a92d7d5..8c530ee991 100644 --- a/src/part2/heartbeat.cpp +++ b/src/part2/heartbeat.cpp @@ -134,6 +134,8 @@ int StopProc(bool need_check_qemu_proc) { std::string uuid_file = GetUuidFile(); std::string uuid_lockfile = GetUuidLockfile(); + g_server.Stop(0); + g_server.Join(); int lockfd = LockFile(uuid_lockfile.c_str()); if (lockfd < 0) { LOG(ERROR) << "add lock failed."; diff --git a/tests/part2/TestCommon.cpp b/tests/part2/TestCommon.cpp index f6a71ae14f..f2216509c5 100644 --- a/tests/part2/TestCommon.cpp +++ b/tests/part2/TestCommon.cpp @@ -316,6 +316,9 @@ TEST(Init, Init_5) { NebdServerMocker mock; std::string lockfile = "/tmp/unit_test"; std::string metadata_file = "/tmp/unit_test2"; + g_server.Stop(0); + g_server.Join(); + g_server.ClearServices(); EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); @@ -335,6 +338,9 @@ TEST(Init, Init_6) { NebdServerMocker mock; std::string lockfile = "/tmp/unit_test"; std::string metadata_file = "/tmp/unit_test2"; + g_server.Stop(0); + g_server.Join(); + g_server.ClearServices(); EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); @@ -363,6 +369,9 @@ TEST(Init, Init_7) { int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); std::string str = "retry_counts=0;\n"; write(fd, str.c_str(), str.size()); + g_server.Stop(0); + g_server.Join(); + g_server.ClearServices(); EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); @@ -411,6 +420,9 @@ TEST(Init, Init_8) { NebdServerMocker mock; std::string lockfile = "/tmp/unit_test"; std::string metadata_file = "/tmp/unit_test2"; + g_server.Stop(0); + g_server.Join(); + g_server.ClearServices(); EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); @@ -435,6 +447,9 @@ TEST(Init, Init_9) { NebdServerMocker mock; std::string lockfile = "/tmp/unit_test"; std::string metadata_file = "/tmp/unit_test2"; + g_server.Stop(0); + g_server.Join(); + g_server.ClearServices(); EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); @@ -460,6 +475,9 @@ TEST(Init, Init_10) { NebdServerMocker mock; std::string lockfile = "/tmp/unit_test"; std::string metadata_file = "/tmp/unit_test2"; + g_server.Stop(0); + g_server.Join(); + g_server.ClearServices(); EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); From 9287d10f59d42c66b599591c19196190aaf57384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=A8=81?= Date: Thu, 24 Oct 2019 15:57:05 +0800 Subject: [PATCH 09/79] fix part1 part2 connect check logic Change-Id: I2273098ba26f67d200e02b39bcda5b132cadfb62 --- src/part1/nebd_lifecycle.cpp | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/part1/nebd_lifecycle.cpp b/src/part1/nebd_lifecycle.cpp index 1f870a12d0..69e31b9b2b 100644 --- a/src/part1/nebd_lifecycle.cpp +++ b/src/part1/nebd_lifecycle.cpp @@ -22,6 +22,8 @@ namespace nebd { namespace client { int LifeCycleManager::Start(common::Configuration *conf) { + LOG(INFO) << "start LifeCycleManager begin."; + int ret = LoadConf(conf); if (ret != 0) { LOG(ERROR) << "Init4Qemu, LoadConf fail"; @@ -125,7 +127,7 @@ int LifeCycleManager::Start(common::Configuration *conf) { if (ret == 0) { // 找到port - LOG(INFO) << "get port success, port = " << port; + LOG(INFO) << "get port from part2 success, port = " << port; break; } @@ -178,6 +180,7 @@ int LifeCycleManager::Start(common::Configuration *conf) { part2Port_ = port; close(fd); + LOG(INFO) << "start LifeCycleManager success."; return 0; } @@ -286,7 +289,7 @@ int LifeCycleManager::LoadConf(common::Configuration *conf) { return -1; } - LOG(INFO) << "load conf success"; + LOG(INFO) << "LifeCycleManager load conf success"; return 0; } @@ -389,7 +392,7 @@ int LifeCycleManager::CheckProcAlive(const char* uuid, const char* procName, // 遍历 /proc目录下的所有进程，没有找到uuid和procName对应的进程 closedir(dir); *procExist = false; - LOG(INFO) << "CheckProcAlive not alive, uuid = " << uuid + LOG(INFO) << "CheckProcAlive proc not alive, uuid = " << uuid << ", procName = " << procName; return 0; } @@ -421,7 +424,8 @@ int LifeCycleManager::GetPortFromPart2(uint32_t *port) { if (!value["port"].isNull() && value["port"].isString()) { *port = stoi(value["port"].asString()); - LOG(INFO) << "read port from file, port = " << *port; + LOG(INFO) << "read port from file = " << metadataFile + << ", port = " << *port; return 0; } else { return -1; @@ -451,6 +455,12 @@ bool LifeCycleManager::IsPart2ConnectibleWithRetry(uint32_t retryTimes, return true; } + // 如果检查part2不连通，再看下是否part2已经退出了 + if (!IsPart2Alive()) { + LOG(WARNING) << "part2 not connectible and part2 not alive."; + return false; + } + retryCount++; LOG(WARNING) << "part2 is not connectible, retry next time" << ", unconnectCount = " << retryCount; @@ -487,7 +497,7 @@ int LifeCycleManager::KillPart2() { } std::string cmd = "kill " + std::to_string(pidPart2_); - LOG(INFO) << "KillPart2, run cmd = " << cmd; + LOG(INFO) << "KillPart2, run cmd: " << cmd; system(cmd.c_str()); // 检查part2是否已经kill @@ -648,8 +658,8 @@ void LifeCycleManager::HeartbeatThreadFunc() { lifeCycleOptions_.connectibleCheckTimes, lifeCycleOptions_.connectibleCheckIntervalUs)) { LOG(WARNING) << "part2 is not connectible, " - << "reach connectibleCheckTimes, unconnectCount = " - << lifeCycleOptions_.connectibleCheckTimes; + << "reach connectibleCheckTimes " << lifeCycleOptions_.connectibleCheckTimes // NOLINT + << " or part2 not alive"; // 连续多次未联通，kill part2 int ret = KillPart2(); From 63a4645af735b2d64c0983ca22cb2174ed8e0445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=A8=81?= Date: Fri, 25 Oct 2019 16:40:47 +0800 Subject: [PATCH 10/79] fix open file lock bug Change-Id: I9370bf078ae92fdba2f9dc258230982dd31577ff --- src/part1/nebd_lifecycle.cpp | 6 +++--- tests/part1/lock.file | 0 tests/part1/nebd_test.cpp | 16 ++++++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) delete mode 100644 tests/part1/lock.file diff --git a/src/part1/nebd_lifecycle.cpp b/src/part1/nebd_lifecycle.cpp index 69e31b9b2b..fcf9b6e82c 100644 --- a/src/part1/nebd_lifecycle.cpp +++ b/src/part1/nebd_lifecycle.cpp @@ -26,7 +26,7 @@ int LifeCycleManager::Start(common::Configuration *conf) { int ret = LoadConf(conf); if (ret != 0) { - LOG(ERROR) << "Init4Qemu, LoadConf fail"; + LOG(ERROR) << "LifeCycleManager LoadConf fail"; return -1; } @@ -45,7 +45,7 @@ int LifeCycleManager::Start(common::Configuration *conf) { LOG(INFO) << "get qemu uuid = " << qemuUUID_; // open文件锁 - int fd = open(lifeCycleOptions_.lockFile.c_str(), O_RDONLY); + int fd = open(lifeCycleOptions_.lockFile.c_str(), O_RDONLY | O_CREAT, 0644); if (fd < 0) { LOG(ERROR) << "open lock file fail, file = " << lifeCycleOptions_.lockFile; @@ -616,7 +616,7 @@ void LifeCycleManager::HeartbeatThreadFunc() { LOG(INFO) << "heartbeat check thread start."; // open文件锁 - int fd = open(lifeCycleOptions_.lockFile.c_str(), O_RDONLY); + int fd = open(lifeCycleOptions_.lockFile.c_str(), O_RDONLY | O_CREAT, 0644); if (fd < 0) { LOG(ERROR) << "open lock file fail, file = " << lifeCycleOptions_.lockFile; diff --git a/tests/part1/lock.file b/tests/part1/lock.file deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/part1/nebd_test.cpp b/tests/part1/nebd_test.cpp index e32ca63114..7ac4e045d6 100644 --- a/tests/part1/nebd_test.cpp +++ b/tests/part1/nebd_test.cpp @@ -591,6 +591,22 @@ TEST_F(nebdClientLifeCycleTest, lifecycle_kill_part2_test) { ASSERT_EQ(manager.IsPart2Alive(), false); } +TEST_F(nebdClientLifeCycleTest, load_conf_fail_test) { + ::nebd::common::Configuration conf; + nebd::client::LifeCycleManager manager; + ASSERT_EQ(manager.Start(&conf), -1); +} + +TEST_F(nebdClientLifeCycleTest, get_uuid_fail_test) { + ::nebd::common::Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_EQ(conf.LoadConfig(), true); + conf.SetStringValue("qemuProcName", "wrong_qemuProcName"); + + nebd::client::LifeCycleManager manager; + ASSERT_EQ(manager.Start(&conf), -1); +} + int main(int argc, char ** argv) { ::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleMock(&argc, argv); From ce1de7f8f8118190a3a09fd6c6e5b0a63dedf8c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=A8=81?= Date: Fri, 25 Oct 2019 17:10:04 +0800 Subject: [PATCH 11/79] add part1 lifecycle test code Change-Id: Ia1b6d24e14061d484b89414012cd4ab167a03dde --- tests/part1/nebd_test.cpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/part1/nebd_test.cpp b/tests/part1/nebd_test.cpp index 7ac4e045d6..438d048b2e 100644 --- a/tests/part1/nebd_test.cpp +++ b/tests/part1/nebd_test.cpp @@ -591,13 +591,13 @@ TEST_F(nebdClientLifeCycleTest, lifecycle_kill_part2_test) { ASSERT_EQ(manager.IsPart2Alive(), false); } -TEST_F(nebdClientLifeCycleTest, load_conf_fail_test) { +TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_load_conf_fail_test) { ::nebd::common::Configuration conf; nebd::client::LifeCycleManager manager; ASSERT_EQ(manager.Start(&conf), -1); } -TEST_F(nebdClientLifeCycleTest, get_uuid_fail_test) { +TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_get_uuid_fail_test) { ::nebd::common::Configuration conf; conf.SetConfigPath(confFilename); ASSERT_EQ(conf.LoadConfig(), true); @@ -607,6 +607,26 @@ TEST_F(nebdClientLifeCycleTest, get_uuid_fail_test) { ASSERT_EQ(manager.Start(&conf), -1); } +TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_start_part2_fail) { + ::nebd::common::Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_EQ(conf.LoadConfig(), true); + conf.SetStringValue("part2ProcPath", "wrong_part2ProcPath"); + + nebd::client::LifeCycleManager manager; + ASSERT_EQ(manager.Start(&conf), -1); +} + +TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_read_port_fail) { + ::nebd::common::Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_EQ(conf.LoadConfig(), true); + conf.SetStringValue("metadataPrefix", "wrong_metadataPrefix"); + + nebd::client::LifeCycleManager manager; + ASSERT_EQ(manager.Start(&conf), -1); +} + int main(int argc, char ** argv) { ::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleMock(&argc, argv); From 6a5e533714be2e07c2902cb50291a9797564d805 Mon Sep 17 00:00:00 2001 From: Wangpan Date: Wed, 30 Oct 2019 11:30:50 +0800 Subject: [PATCH 12/79] add test_reboot_vm_with_exist_vnet Change-Id: Ib137168171eb6430e1e1ad1f558d178b48b48b68 --- tests/test-suites/templates/domain_xml.j2 | 3 + tests/test-suites/test_nebd.py | 91 ++++++++++++++++++----- 2 files changed, 77 insertions(+), 17 deletions(-) diff --git a/tests/test-suites/templates/domain_xml.j2 b/tests/test-suites/templates/domain_xml.j2 index b86dca0996..bcbdfeacfa 100644 --- a/tests/test-suites/templates/domain_xml.j2 +++ b/tests/test-suites/templates/domain_xml.j2 @@ -37,6 +37,9 @@ + {%- if tap == "tap0" %} + + {% endif -%} diff --git a/tests/test-suites/test_nebd.py b/tests/test-suites/test_nebd.py index e1e4e58ccd..058fbef7ef 100755 --- a/tests/test-suites/test_nebd.py +++ b/tests/test-suites/test_nebd.py @@ -17,7 +17,7 @@ import rados import rbd - +# 需要先在物理机上配置好网桥，名称bri，配置dhcp服务，并修改如下相关配置项 CEPH_CONF = "/etc/ceph/ceph.conf" MON_HOSTS = ["10.182.30.27", "10.182.30.28", "10.182.30.29"] REUSE_RBD_DISK = False # True表示测试前不创建rbd卷（使用已有的），跑完不清理rbd卷，False则相反 @@ -39,7 +39,7 @@ CEPH_CLIENT_ASOK_DIR = "/var/run/ceph/guests/" -VM_SSH_KEY = "/home/hzwangpan/nebd/test/id_rsa" # 免密ssh到vm私钥（root用户600权限） +VM_SSH_KEY = "./id_rsa" # 免密ssh到vm私钥（root用户600权限） VM_HOST_IP = "192.168.10.10" # vm的ip地址，最好配置为静态ip UUID1 = "5d1289be-50e1-47b7-86de-1de0ff16a9d4" UUID2 = "b5fdf3de-320c-41cf-80b2-acb794078012" @@ -95,19 +95,8 @@ def ssh_exec(conn, cmd, timeout=5): class VMIntegTest(unittest.TestCase): '''热升级功能集成测试类''' - def _clear_all_vms(self): - print 'going to clear all vms...' - doms = LIBVIRT_CONN.listAllDomains() - for dom in doms: - if dom.isActive(): - dom.destroy() - doms = LIBVIRT_CONN.listAllDomains() - for dom in doms: - if dom.isPersistent(): - dom.undefine() - def setUp(self): - self._clear_all_vms() + clear_all_vms() def _clear_all_metadatafiles(self): print 'going to clear all metadata files...' @@ -124,7 +113,7 @@ def _killall_nebd_servers(self): pass def tearDown(self): - self._clear_all_vms() + clear_all_vms() self._killall_nebd_servers() self._clear_all_metadatafiles() @@ -213,6 +202,62 @@ def test_reboot_vm(self): # 关机 self._destroy_vm(dominfo) + def test_reboot_vm_with_exist_vnet(self): + tap_name = "tap0" + rbd_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_reboot_rbd_vm_with_exist_vnet", + "disk_type": "network", + "protocol": "rbd", + "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), + "mon_hosts": MON_HOSTS, + "tap": tap_name + } + local_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), + "name": "test_reboot_local_vm_with_exist_vnet", + "disk_type": "file", + "disk_file": LOCAL_IMG, + "tap": tap_name + } + dominfos = [rbd_vm, local_vm] + + for dominfo in dominfos: + # 删除tap设备 + cmd_tap = ' '.join(['/usr/sbin/tunctl', '-d', tap_name]) + out = run(cmd_tap) + # 创建tap设备 + cmd_tap = ' '.join(['/usr/sbin/tunctl', '-t', tap_name]) + out = run(cmd_tap) + self.assertTrue(tap_name in out and 'persistent' in out) + dom = self._create_vm(dominfo) + if not dom.isActive(): + self.assertFalse("Domain creates failed") + # 等待虚拟机操作系统启动 + ssh = create_ssh_connect(VM_HOST_IP, timeout=120) + self.assertTrue(ssh is not None) + # 关机 + self._destroy_vm(dominfo) + # 从网桥上移除tap设备 + cmd_brctl = ' '.join(['/sbin/brctl', 'delif', 'bri', tap_name]) + out = run(cmd_brctl) + self.assertFalse(out) + # 再启动（模拟重启） + dom = self._create_vm(dominfo) + if not dom.isActive(): + self.assertFalse("Domain creates failed") + # 等待虚拟机操作系统启动 + ssh = create_ssh_connect(VM_HOST_IP, timeout=120) + self.assertTrue(ssh is not None) + # 关机 + self._destroy_vm(dominfo) + # 从网桥上移除tap设备 + cmd_brctl = ' '.join(['/sbin/brctl', 'delif', 'bri', tap_name]) + out = run(cmd_brctl) + self.assertFalse(out) + # 删除tap设备 + cmd_tap = ' '.join(['/usr/sbin/tunctl', '-d', tap_name]) + out = run(cmd_tap) + self.assertTrue(tap_name in out and 'nonpersistent' in out) + def test_poweroff_vm_inside(self): rbd_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), "name": "test_poweroff_rbd_vm_inside", @@ -1349,6 +1394,17 @@ def prepare_rbd_disk(): cluster.shutdown() +def clear_all_vms(): + print 'going to clear all vms...' + doms = LIBVIRT_CONN.listAllDomains() + for dom in doms: + if dom.isActive(): + dom.destroy() + doms = LIBVIRT_CONN.listAllDomains() + for dom in doms: + if dom.isPersistent(): + dom.undefine() + def clear_rbd_disk(): if REUSE_RBD_DISK: @@ -1392,14 +1448,15 @@ def install_rbd_client(): if __name__ == '__main__': + if LIBVIRT_CONN is None: + LIBVIRT_CONN = libvirt.open(None) + clear_all_vms() clear_log_dirs() install_rbd_client() prepare_rbd_disk() path = '{}/templates/'.format(os.path.dirname(os.path.abspath(__file__))) if JJ_ENV is None: JJ_ENV = Environment(loader=FileSystemLoader(path)) - if LIBVIRT_CONN is None: - LIBVIRT_CONN = libvirt.open(None) unittest.main() From 418ab10a0f95bd0627153d6411dbe4d1db0d8de2 Mon Sep 17 00:00:00 2001 From: hzwuhongsong Date: Wed, 30 Oct 2019 16:09:17 +0800 Subject: [PATCH 13/79] close parent fd CLDNBS-1499 Change-Id: If912f0986c77de4b67cbec3630a8c7a32af70ddc --- src/part2/common_type.cpp | 28 ++++++++++++++++++++++++++++ src/part2/common_type.h | 1 + src/part2/main.cpp | 3 +++ tests/part2/TestCommon.cpp | 4 ++++ 4 files changed, 36 insertions(+) diff --git a/src/part2/common_type.cpp b/src/part2/common_type.cpp index 6a50f58f45..edcfad27c2 100644 --- a/src/part2/common_type.cpp +++ b/src/part2/common_type.cpp @@ -487,6 +487,34 @@ int SetCpuAffinity(pid_t qemu) { return 0; } +int CloseParentFd() { + pid_t pid = getpid(); + std::string pid_str = std::to_string(pid); + std::string proc_str = "/proc/"; + std::string fd_str = "/fd"; + std::string pid_proc = proc_str + pid_str + fd_str; + + DIR* dir = opendir(pid_proc.c_str()); + struct dirent* ent; + if (dir == NULL) { + return -1; + } + while ((ent = readdir(dir))) { + if ((ent->d_name[0] == '.' && ent->d_name[1] == '\0') || + (ent->d_name[1] == '.' && ent->d_name[2] == '\0')) + continue; + int fd = atoi(ent->d_name); + if (fd < 3) + continue; + int ret = -1; + ret = close(fd); + if (ret < 0) { + return -1; + } + } + return 0; +} + // 初始化 int Init() { // 初始化配置 diff --git a/src/part2/common_type.h b/src/part2/common_type.h index 9c5ff65ccd..f8f365479c 100644 --- a/src/part2/common_type.h +++ b/src/part2/common_type.h @@ -32,6 +32,7 @@ extern std::map g_imageMap; // fd的内存镜像 #define FD_CEPH_MAX 1000 #define FD_CURVE_MAX 20000 +int CloseParentFd(); int LockFile(const char* file); void UnlockFile(int lockfd); std::vector split(const std::string& str, diff --git a/src/part2/main.cpp b/src/part2/main.cpp index ff6ec82441..8d17fd79f7 100644 --- a/src/part2/main.cpp +++ b/src/part2/main.cpp @@ -28,6 +28,9 @@ #define BOOST_SPIRIT_THREADSAFE int main(int argc, char* argv[]) { + if (CloseParentFd() < 0) { + return -1; + } // 使进程为守护进程 if (daemon(0, 0) < 0) { char buffer[128]; diff --git a/tests/part2/TestCommon.cpp b/tests/part2/TestCommon.cpp index f2216509c5..6315dd45d8 100644 --- a/tests/part2/TestCommon.cpp +++ b/tests/part2/TestCommon.cpp @@ -19,6 +19,10 @@ #include "src/part2/common_type.h" #include "tests/part2/test_common.h" +TEST(CloseParentFd, CloseParentFd_1) { + EXPECT_EQ(0, CloseParentFd()); +} + TEST(ReadQemuXmls, ReadQemuXmls_1) { NebdServerMocker mock; From e09d22ffa48d82560fdb850f16d38e9594c07ac5 Mon Sep 17 00:00:00 2001 From: Wangpan Date: Wed, 30 Oct 2019 17:23:34 +0800 Subject: [PATCH 14/79] fix test_reboot_vm_inside Change-Id: I2e92b341066e5c11819e9a1aa0249028dce142b3 --- tests/test-suites/test_nebd.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/test-suites/test_nebd.py b/tests/test-suites/test_nebd.py index 058fbef7ef..943c4980da 100755 --- a/tests/test-suites/test_nebd.py +++ b/tests/test-suites/test_nebd.py @@ -325,14 +325,19 @@ def test_reboot_vm_inside(self): is_reboot = False reboot_timeout = 30 + print 'sleep 1s' + time.sleep(1) while reboot_timeout > 0: ssh = create_ssh_connect(VM_HOST_IP, timeout=120) if ssh is not None: - _, out, err = ssh_exec(ssh, 'last reboot|head -1') - print out, err - if 'reboot' in out[0]: - is_reboot = True - break + try: + _, out, err = ssh_exec(ssh, 'last reboot|head -1') + print out, err + if 'reboot' in out[0]: + is_reboot = True + break + except Exception: + pass reboot_timeout -= 1 print 'sleep 1s' time.sleep(1) From b198a3a79e2c0ab623fd11a7e7bf992dd6772eca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=A8=81?= Date: Wed, 13 Nov 2019 17:21:24 +0800 Subject: [PATCH 15/79] lock file with retry Change-Id: Ic01a44c15a6311242169de42917986059082b628 --- etc/nebd/nebd-client.conf | 6 +++- src/part1/nebd_lifecycle.cpp | 59 ++++++++++++++++++++++++++++++------ src/part1/nebd_lifecycle.h | 11 +++++++ tests/part1/nebd.conf | 6 +++- tests/part1/nebd_test.cpp | 22 ++++++++++++++ 5 files changed, 93 insertions(+), 11 deletions(-) diff --git a/etc/nebd/nebd-client.conf b/etc/nebd/nebd-client.conf index 66ca076626..2708e328df 100644 --- a/etc/nebd/nebd-client.conf +++ b/etc/nebd/nebd-client.conf @@ -42,4 +42,8 @@ rpcTimeoutMs=30000 # brpc的健康检查周期时间，单位s rpcHealthCheckIntervalS=1 # 异步IO操作RPC失败日志打印间隔，每隔多少次打印一次 -aioRpcFailLogInterval=10 \ No newline at end of file +aioRpcFailLogInterval=10 +# 加文件锁的重试次数 +fileLockRetryTimes=150 +# 加文件锁的重试间隔，单位Us +fileLockRetryIntervalUs=200000 \ No newline at end of file diff --git a/src/part1/nebd_lifecycle.cpp b/src/part1/nebd_lifecycle.cpp index fcf9b6e82c..651d3bd30f 100644 --- a/src/part1/nebd_lifecycle.cpp +++ b/src/part1/nebd_lifecycle.cpp @@ -58,9 +58,12 @@ int LifeCycleManager::Start(common::Configuration *conf) { uint32_t retryCount = 0; do { // 加文件锁 - int lock = flock(fd, LOCK_EX | LOCK_NB); - if (lock < 0) { - LOG(ERROR) << "lock file fail, file = " + bool boolRet = LockFileWithRetry(fd, + lifeCycleOptions_.fileLockRetryTimes, + lifeCycleOptions_.fileLockRetryIntervalUs); + + if (!boolRet) { + LOG(ERROR) << "lock file with retry fail, file = " << lifeCycleOptions_.lockFile; close(fd); return -1; @@ -143,9 +146,12 @@ int LifeCycleManager::Start(common::Configuration *conf) { usleep(lifeCycleOptions_.portGetRetryIntervalUs); // 为下一次循环，加文件锁 - int lock = flock(fd, LOCK_EX | LOCK_NB); - if (lock < 0) { - LOG(ERROR) << "lock file fail, file = " + bool boolRet = LockFileWithRetry(fd, + lifeCycleOptions_.fileLockRetryTimes, + lifeCycleOptions_.fileLockRetryIntervalUs); + + if (!boolRet) { + LOG(ERROR) << "lock file with retry fail, file = " << lifeCycleOptions_.lockFile; close(fd); return -1; @@ -154,9 +160,12 @@ int LifeCycleManager::Start(common::Configuration *conf) { // 未成功读取port信息，kill part2 if (port == 0) { - int lock = flock(fd, LOCK_EX | LOCK_NB); - if (lock < 0) { - LOG(ERROR) << "lock file fail, file = " + bool boolRet = LockFileWithRetry(fd, + lifeCycleOptions_.fileLockRetryTimes, + lifeCycleOptions_.fileLockRetryIntervalUs); + + if (!boolRet) { + LOG(ERROR) << "lock file with retry fail, file = " << lifeCycleOptions_.lockFile; close(fd); return -1; @@ -289,6 +298,18 @@ int LifeCycleManager::LoadConf(common::Configuration *conf) { return -1; } + if (!conf->GetUInt32Value("fileLockRetryTimes", + &lifeCycleOptions_.fileLockRetryTimes)) { + LOG(ERROR) << "get fileLockRetryTimes fail."; + return -1; + } + + if (!conf->GetUInt32Value("fileLockRetryIntervalUs", + &lifeCycleOptions_.fileLockRetryIntervalUs)) { + LOG(ERROR) << "get fileLockRetryIntervalUs fail."; + return -1; + } + LOG(INFO) << "LifeCycleManager load conf success"; return 0; } @@ -739,5 +760,25 @@ bool LifeCycleManager::IsHeartbeatThreadStart() { return isHeartbeatThreadStart_; } +bool LifeCycleManager::LockFileWithRetry(int fd, uint32_t retryTimes, + uint32_t retryIntervalUs) { + uint32_t retryCount = 0; + while (retryCount < retryTimes) { + // 加文件锁 + int lock = flock(fd, LOCK_EX | LOCK_NB); + if (lock < 0) { + LOG(WARNING) << "lock file fail, file = " + << lifeCycleOptions_.lockFile + << ", retryCount = " << retryCount; + retryCount++; + usleep(retryIntervalUs); + } else { + return true; + } + } + + return false; +} + } // namespace client } // namespace nebd diff --git a/src/part1/nebd_lifecycle.h b/src/part1/nebd_lifecycle.h index 0109e0e2e1..67bb17f371 100644 --- a/src/part1/nebd_lifecycle.h +++ b/src/part1/nebd_lifecycle.h @@ -40,6 +40,8 @@ typedef struct LifeCycleOptions { uint32_t rpcRetryIntervalUs; uint32_t rpcRetryMaxIntervalUs; uint32_t rpcTimeoutMs; + uint32_t fileLockRetryTimes; + uint32_t fileLockRetryIntervalUs; } LifeCycleOptions; // 负责生命周期管理服务 @@ -174,6 +176,15 @@ class LifeCycleManager { * @return void */ void HeartbeatThreadFunc(); + /** + * @brief 对文件加锁 + * @param fd: 加文件的文件的fd + * @param retryTimes: 如果加锁失败最大的重试次数 + * @param retryIntervalUs: 如果加锁失败进行重试的重试间隔 + * @return 执行成功返回true，失败返回false + */ + bool LockFileWithRetry(int fd, uint32_t retryTimes, + uint32_t retryIntervalUs); private: // 从配置文件中读取到的声明周期管理所需要的配置字段 diff --git a/tests/part1/nebd.conf b/tests/part1/nebd.conf index 3040353ce1..45d27855c5 100644 --- a/tests/part1/nebd.conf +++ b/tests/part1/nebd.conf @@ -42,4 +42,8 @@ rpcTimeoutMs=5000 # brpc的健康检查周期时间，单位s rpcHealthCheckIntervalS=1 # 异步IO操作RPC失败日志打印间隔，每隔多少次打印一次 -aioRpcFailLogInterval=10 \ No newline at end of file +aioRpcFailLogInterval=10 +# 加文件锁的重试次数 +fileLockRetryTimes=5 +# 加文件锁的重试间隔，单位Us +fileLockRetryIntervalUs=100000 \ No newline at end of file diff --git a/tests/part1/nebd_test.cpp b/tests/part1/nebd_test.cpp index 438d048b2e..c877c84686 100644 --- a/tests/part1/nebd_test.cpp +++ b/tests/part1/nebd_test.cpp @@ -9,6 +9,7 @@ #include #include #include +#include // flock #include // NOLINT #include // NOLINT #include "tests/part1/mock_client_service.h" @@ -627,6 +628,27 @@ TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_read_port_fail) { ASSERT_EQ(manager.Start(&conf), -1); } +TEST_F(nebdClientLifeCycleTest, lifecycle_manager_lock_file_fail) { + ::nebd::common::Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_TRUE(conf.LoadConfig()); + + // 先占用文件锁 + std::string lockFile; + ASSERT_TRUE(conf.GetStringValue("lockFile", &lockFile)); + int fd = open(lockFile.c_str(), O_RDONLY | O_CREAT, 0644); + ASSERT_GT(fd, 0); + ASSERT_EQ(flock(fd, LOCK_EX | LOCK_NB), 0); + + // 再启lifecyclemanager，会因为文件锁被占用而失败 + nebd::client::LifeCycleManager manager; + ASSERT_EQ(manager.Start(&conf), -1); + + // 再释放文件锁 + ASSERT_EQ(flock(fd, LOCK_UN), 0); + close(fd); +} + int main(int argc, char ** argv) { ::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleMock(&argc, argv); From 9a70f2200c685db294f3393d1484de68412d5101 Mon Sep 17 00:00:00 2001 From: majie1 Date: Wed, 20 Nov 2019 11:52:16 +0800 Subject: [PATCH 16/79] update coverage statistics Change-Id: I1bbf5e960ac1b4f5f3299cab6e8ac84e6388bce6 --- docker_unittest.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docker_unittest.sh b/docker_unittest.sh index 0383dc26e0..b70138d439 100755 --- a/docker_unittest.sh +++ b/docker_unittest.sh @@ -2,7 +2,14 @@ set -x set -e -rm -rf build coverage +rm -rf build coverage lcov + +git clone https://github.com/linux-test-project/lcov.git +cd lcov && make install && cd .. + +update-alternatives --install /usr/bin/gcc gcc /usr/lib/gcc-4.9-backport/bin/gcc 90\ + --slave /usr/bin/g++ g++ /usr/lib/gcc-4.9-backport/bin/g++ \ + --slave /usr/bin/gcov gcov /usr/lib/gcc-4.9-backport/bin/gcov ./do_cmake cd build @@ -21,8 +28,6 @@ for i in ${gcda}; do cp --parents $j coverage done -lcov --directory coverage$(pwd)/build --capture --output-file post.info -lcov --directory coverage$(pwd)/build --capture --initial --output-file raw.info -lcov -a post.info -a raw.info -o cov.info --rc lcov_branch_coverage=1 -lcov --remove cov.info '/usr/*' '*3rdparty*' '*/build/*' -o result.info +lcov --directory coverage$(pwd)/build --capture --output-file cov.info --rc lcov_branch_coverage=1 +lcov --remove cov.info '/usr/*' '*3rdparty*' '*/build/*' -o result.info --rc lcov_branch_coverage=1 genhtml result.info --branch-coverage --ignore-errors source -o result \ No newline at end of file From c34b5a4346ad65299d805d9737284187fcecb6d0 Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Thu, 16 Jan 2020 20:37:52 +0800 Subject: [PATCH 17/79] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E6=A1=86=E6=9E=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I22ef329f295e22b0a5be548d037378e8f3feced8 --- src/common/CMakeLists.txt | 6 +- src/common/client.proto | 9 +- src/common/common.proto | 8 + src/common/heartbeat.proto | 22 + src/{part2 => common}/interrupt_sleep.h | 15 +- src/common/rw_lock.h | 87 + src/common/uncopyable.h | 27 + src/part1/CMakeLists.txt | 4 +- src/part1/connection_manager.cpp | 32 + src/part1/connection_manager.h | 33 + src/part1/libnebd.cpp | 34 +- src/part1/libnebd.h | 18 +- src/part1/libnebd_file.cpp | 49 +- src/part1/libnebd_file.h | 24 +- src/part1/nebd_client.cpp | 517 +----- src/part1/nebd_client.h | 154 +- src/part1/nebd_lifecycle.cpp | 784 --------- src/part1/nebd_lifecycle.h | 209 --- src/part1/nebd_metacache.cpp | 24 + src/part1/nebd_metacache.h | 33 + src/part2/common_type.cpp | 655 -------- src/part2/common_type.h | 68 - src/part2/config.cpp | 178 -- src/part2/config.h | 60 - src/part2/define.h | 67 + src/part2/file_manager.cpp | 87 + src/part2/file_manager.h | 54 + src/part2/file_service.cpp | 79 + src/part2/{rpc_server.h => file_service.h} | 38 +- src/part2/heartbeat.cpp | 350 ---- src/part2/heartbeat.h | 43 - src/part2/heartbeat_service.cpp | 16 + src/part2/heartbeat_service.h | 30 + src/part2/main.cpp | 57 - src/part2/metafile_manager.cpp | 29 + src/part2/metafile_manager.h | 28 + src/part2/nebd_server.cpp | 23 + src/part2/nebd_server.h | 19 + src/part2/rados_interface.cpp | 173 -- src/part2/rados_interface.h | 23 - src/part2/reload.cpp | 121 -- src/part2/reload.h | 23 - src/part2/request_executor.h | 34 + src/part2/request_executor_ceph.cpp | 60 + src/part2/request_executor_ceph.h | 39 + src/part2/request_executor_curve.cpp | 60 + src/part2/request_executor_curve.h | 39 + src/part2/rpc_ceph.cpp | 385 ----- src/part2/rpc_ceph.h | 54 - src/part2/rpc_request.h | 92 -- src/part2/rpc_server.cpp | 457 ----- tests/part1/CMakeLists.txt | 40 +- tests/part1/client_server.cpp | 89 - ...ient_service.cpp => fake_file_service.cpp} | 22 +- ...e_client_service.h => fake_file_service.h} | 12 +- ...k_client_service.h => mock_file_service.h} | 6 +- tests/part1/nebd.conf | 49 - tests/part1/nebd_test.cpp | 659 -------- tests/part2/CMakeLists.txt | 94 -- tests/part2/TestCommon.cpp | 499 ------ tests/part2/TestCommon2.cpp | 189 --- tests/part2/TestCommon3.cpp | 103 -- tests/part2/TestConfig.cpp | 331 ---- tests/part2/TestHeartbeat.cpp | 221 --- tests/part2/TestHeartbeat2.cpp | 182 -- tests/part2/TestHeartbeat3.cpp | 43 - tests/part2/TestRados.cpp | 334 ---- tests/part2/TestReload.cpp | 107 -- tests/part2/TestReload2.cpp | 104 -- tests/part2/TestRpcCeph.cpp | 1252 -------------- tests/part2/TestRpcServer.cpp | 1187 ------------- tests/part2/test_common.cpp | 18 - tests/part2/test_common.h | 30 - tests/part2/test_common2.cpp | 10 - tests/part2/test_common2.h | 20 - tests/part2/test_config.cpp | 13 - tests/part2/test_heartbeat.cpp | 17 - tests/part2/test_heartbeat.h | 31 - tests/part2/test_heartbeat2.cpp | 16 - tests/part2/test_heartbeat2.h | 31 - tests/part2/test_heartbeat3.cpp | 10 - tests/part2/test_heartbeat3.h | 24 - tests/part2/test_rados.cpp | 24 - tests/part2/test_rados.h | 31 - tests/part2/test_reload.cpp | 13 - tests/part2/test_reload.h | 25 - tests/part2/test_reload2.cpp | 11 - tests/part2/test_reload2.h | 24 - tests/part2/test_rpcceph.cpp | 41 - tests/part2/test_rpcceph.h | 46 - tests/test-suites/create_100_vms.sh | 62 - tests/test-suites/id_rsa | 27 - tests/test-suites/kill_nebd.sh | 26 - tests/test-suites/templates/disk_xml.j2 | 16 - tests/test-suites/templates/domain_xml.j2 | 55 - tests/test-suites/test_10_disks.sh | 82 - tests/test-suites/test_discard.sh | 60 - tests/test-suites/test_nebd.py | 1469 ----------------- 98 files changed, 1099 insertions(+), 12186 deletions(-) create mode 100644 src/common/common.proto create mode 100644 src/common/heartbeat.proto rename src/{part2 => common}/interrupt_sleep.h (78%) create mode 100644 src/common/rw_lock.h create mode 100644 src/common/uncopyable.h create mode 100644 src/part1/connection_manager.cpp create mode 100644 src/part1/connection_manager.h delete mode 100644 src/part1/nebd_lifecycle.cpp delete mode 100644 src/part1/nebd_lifecycle.h create mode 100644 src/part1/nebd_metacache.cpp create mode 100644 src/part1/nebd_metacache.h delete mode 100644 src/part2/common_type.cpp delete mode 100644 src/part2/common_type.h delete mode 100644 src/part2/config.cpp delete mode 100644 src/part2/config.h create mode 100644 src/part2/define.h create mode 100644 src/part2/file_manager.cpp create mode 100644 src/part2/file_manager.h create mode 100644 src/part2/file_service.cpp rename src/part2/{rpc_server.h => file_service.h} (82%) delete mode 100644 src/part2/heartbeat.cpp delete mode 100644 src/part2/heartbeat.h create mode 100644 src/part2/heartbeat_service.cpp create mode 100644 src/part2/heartbeat_service.h create mode 100644 src/part2/metafile_manager.cpp create mode 100644 src/part2/metafile_manager.h create mode 100644 src/part2/nebd_server.cpp create mode 100644 src/part2/nebd_server.h delete mode 100644 src/part2/rados_interface.cpp delete mode 100644 src/part2/rados_interface.h delete mode 100644 src/part2/reload.cpp delete mode 100644 src/part2/reload.h create mode 100644 src/part2/request_executor.h create mode 100644 src/part2/request_executor_ceph.cpp create mode 100644 src/part2/request_executor_ceph.h create mode 100644 src/part2/request_executor_curve.cpp create mode 100644 src/part2/request_executor_curve.h delete mode 100644 src/part2/rpc_ceph.cpp delete mode 100644 src/part2/rpc_ceph.h delete mode 100644 src/part2/rpc_request.h delete mode 100644 src/part2/rpc_server.cpp delete mode 100644 tests/part1/client_server.cpp rename tests/part1/{fake_client_service.cpp => fake_file_service.cpp} (88%) rename tests/part1/{fake_client_service.h => fake_file_service.h} (92%) rename tests/part1/{mock_client_service.h => mock_file_service.h} (95%) delete mode 100644 tests/part1/nebd.conf delete mode 100644 tests/part1/nebd_test.cpp delete mode 100644 tests/part2/TestCommon.cpp delete mode 100644 tests/part2/TestCommon2.cpp delete mode 100644 tests/part2/TestCommon3.cpp delete mode 100644 tests/part2/TestConfig.cpp delete mode 100644 tests/part2/TestHeartbeat.cpp delete mode 100644 tests/part2/TestHeartbeat2.cpp delete mode 100644 tests/part2/TestHeartbeat3.cpp delete mode 100644 tests/part2/TestRados.cpp delete mode 100644 tests/part2/TestReload.cpp delete mode 100644 tests/part2/TestReload2.cpp delete mode 100644 tests/part2/TestRpcCeph.cpp delete mode 100644 tests/part2/TestRpcServer.cpp delete mode 100644 tests/part2/test_common.cpp delete mode 100644 tests/part2/test_common.h delete mode 100644 tests/part2/test_common2.cpp delete mode 100644 tests/part2/test_common2.h delete mode 100644 tests/part2/test_config.cpp delete mode 100644 tests/part2/test_heartbeat.cpp delete mode 100644 tests/part2/test_heartbeat.h delete mode 100644 tests/part2/test_heartbeat2.cpp delete mode 100644 tests/part2/test_heartbeat2.h delete mode 100644 tests/part2/test_heartbeat3.cpp delete mode 100644 tests/part2/test_heartbeat3.h delete mode 100644 tests/part2/test_rados.cpp delete mode 100644 tests/part2/test_rados.h delete mode 100644 tests/part2/test_reload.cpp delete mode 100644 tests/part2/test_reload.h delete mode 100644 tests/part2/test_reload2.cpp delete mode 100644 tests/part2/test_reload2.h delete mode 100644 tests/part2/test_rpcceph.cpp delete mode 100644 tests/part2/test_rpcceph.h delete mode 100644 tests/test-suites/create_100_vms.sh delete mode 100644 tests/test-suites/id_rsa delete mode 100644 tests/test-suites/kill_nebd.sh delete mode 100644 tests/test-suites/templates/disk_xml.j2 delete mode 100644 tests/test-suites/templates/domain_xml.j2 delete mode 100644 tests/test-suites/test_10_disks.sh delete mode 100644 tests/test-suites/test_discard.sh delete mode 100755 tests/test-suites/test_nebd.py diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 2b42abacb5..6f8982e3a3 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -4,6 +4,10 @@ file(GLOB COMMON_SRC "*.cpp" ) +file (GLOB PROTO_FILES + "*.proto" +) + add_library(nebd_common SHARED ${COMMON_SRC}) install(TARGETS nebd_common DESTINATION lib) target_link_libraries(nebd_common ${EXTRALIBS}) @@ -12,6 +16,6 @@ target_link_libraries(nebd_common ${EXTRALIBS}) include(FindProtobuf) FIND_PACKAGE(Protobuf REQUIRED) include_directories(${PROTOBUF_INCLUDE_DIR}) -protobuf_generate_cpp(PROTO_SRC PROTO_HEADER client.proto) +protobuf_generate_cpp(PROTO_SRC PROTO_HEADER ${PROTO_FILES}) add_library(client_proto ${PROTO_HEADER} ${PROTO_SRC}) install(TARGETS client_proto DESTINATION lib) diff --git a/src/common/client.proto b/src/common/client.proto index 84202c711e..c970f677fa 100644 --- a/src/common/client.proto +++ b/src/common/client.proto @@ -1,10 +1,9 @@ syntax="proto2"; +import "common.proto"; + package nebd.client; + option cc_generic_services = true; -enum RetCode { - kNoOK = -1; - kOK = 0; -} message OpenFileRequest { required string fileName = 1; @@ -102,7 +101,7 @@ message InvalidateCacheResponse { optional string retMsg = 2; } -service QemuClientService { +service NebdFileService { // for ceph & curve rpc OpenFile(OpenFileRequest) returns (OpenFileResponse); rpc CloseFile(CloseFileRequest) returns (CloseFileResponse); diff --git a/src/common/common.proto b/src/common/common.proto new file mode 100644 index 0000000000..74828b6ebd --- /dev/null +++ b/src/common/common.proto @@ -0,0 +1,8 @@ +syntax="proto2"; +package nebd.client; +option cc_generic_services = true; + +enum RetCode { + kNoOK = -1; + kOK = 0; +}; \ No newline at end of file diff --git a/src/common/heartbeat.proto b/src/common/heartbeat.proto new file mode 100644 index 0000000000..eebee91a05 --- /dev/null +++ b/src/common/heartbeat.proto @@ -0,0 +1,22 @@ +syntax="proto2"; +import "common.proto"; + +package nebd.client; + +option cc_generic_services = true; + +message FileInfo { + required int32 fd = 1; + required string name = 2; +} + +message HeartbeatRequest { + repeated FileInfo info = 1; +} +message HeartbeatResponse { + required RetCode retCode = 1; +} + +service NebdHeartbeatService { + rpc KeepAlive(HeartbeatRequest) returns (HeartbeatResponse); +}; \ No newline at end of file diff --git a/src/part2/interrupt_sleep.h b/src/common/interrupt_sleep.h similarity index 78% rename from src/part2/interrupt_sleep.h rename to src/common/interrupt_sleep.h index f961bf5a19..e0d7792151 100644 --- a/src/part2/interrupt_sleep.h +++ b/src/common/interrupt_sleep.h @@ -5,15 +5,15 @@ * Copyright (c) 2019 NetEase */ -#ifndef SRC_PART2_INTERRUPT_SLEEP_H_ -#define SRC_PART2_INTERRUPT_SLEEP_H_ +#ifndef SRC_COMMON_INTERRUPT_SLEEP_H_ +#define SRC_COMMON_INTERRUPT_SLEEP_H_ #include #include -// #include #include // NOLINT -// #include -// #include + +namespace nebd { +namespace common { class InterruptibleSleeper { public: @@ -36,4 +36,7 @@ class InterruptibleSleeper { bool terminate = false; }; -#endif // SRC_PART2_INTERRUPT_SLEEP_H_ +} // namespace common +} // namespace nebd + +#endif // SRC_COMMON_INTERRUPT_SLEEP_H_ diff --git a/src/common/rw_lock.h b/src/common/rw_lock.h new file mode 100644 index 0000000000..8e372999ae --- /dev/null +++ b/src/common/rw_lock.h @@ -0,0 +1,87 @@ +/* + * Project: nebd + * Created Date: 18-10-11 + * Author: wudemiao + * Copyright (c) 2018 netease + */ + +#ifndef SRC_COMMON_RW_LOCK_H_ +#define SRC_COMMON_RW_LOCK_H_ + +#include +#include +#include + +#include "src/common/uncopyable.h" + +namespace nebd { +namespace common { + +class RWLock : public Uncopyable { + public: + RWLock() { + pthread_rwlock_init(&rwlock_, nullptr); + } + + ~RWLock() { + pthread_rwlock_destroy(&rwlock_); + } + + void WRLock() { + int ret = pthread_rwlock_wrlock(&rwlock_); + CHECK(0 == ret) << "wlock failed: " << ret << ", " << strerror(ret); + } + + int TryWRLock() { + return pthread_rwlock_trywrlock(&rwlock_); + } + + void RDLock() { + int ret = pthread_rwlock_rdlock(&rwlock_); + CHECK(0 == ret) << "rlock failed: " << ret << ", " << strerror(ret); + } + + int TryRDLock() { + return pthread_rwlock_tryrdlock(&rwlock_); + } + + void Unlock() { + pthread_rwlock_unlock(&rwlock_); + } + + private: + pthread_rwlock_t rwlock_; +}; // RWLock class + +class ReadLockGuard : public Uncopyable { + public: + explicit ReadLockGuard(RWLock &rwlock) : rwlock_(rwlock) { + rwlock_.RDLock(); + } + + ~ReadLockGuard() { + rwlock_.Unlock(); + } + + private: + RWLock &rwlock_; +}; // ReadLockGuard class + +class WriteLockGuard : public Uncopyable { + public: + explicit WriteLockGuard(RWLock &rwlock) : rwlock_(rwlock) { + rwlock_.WRLock(); + } + + ~WriteLockGuard() { + rwlock_.Unlock(); + } + + private: + RWLock &rwlock_; +}; // WriteLockGuard class + +} // namespace common +} // namespace nebd + +#endif // SRC_COMMON_RW_LOCK_H_ diff --git a/src/common/uncopyable.h b/src/common/uncopyable.h new file mode 100644 index 0000000000..2218e3d6fc --- /dev/null +++ b/src/common/uncopyable.h @@ -0,0 +1,27 @@ +/* + * Project: nebd + * Created Date: 18-8-31 + * Author: wudemiao + * Copyright (c) 2018 netease + */ + +#ifndef SRC_COMMON_UNCOPYABLE_H_ +#define SRC_COMMON_UNCOPYABLE_H_ + +namespace nebd { +namespace common { + +class Uncopyable { + protected: + Uncopyable() = default; + virtual ~Uncopyable() = default; + + private: + Uncopyable(const Uncopyable &) = delete; + Uncopyable &operator=(const Uncopyable &) = delete; +}; + +} // namespace common +} // namespace nebd + +#endif // SRC_COMMON_UNCOPYABLE_H_ diff --git a/src/part1/CMakeLists.txt b/src/part1/CMakeLists.txt index 4cde106620..cef353fd67 100644 --- a/src/part1/CMakeLists.txt +++ b/src/part1/CMakeLists.txt @@ -18,8 +18,8 @@ set(LIBNEBD_SRC libnebd_file.h nebd_client.cpp nebd_client.h - nebd_lifecycle.cpp - nebd_lifecycle.h) + connection_manager.cpp + connection_manager.h) set(LIBNEBD_LINK brpc-shared rt diff --git a/src/part1/connection_manager.cpp b/src/part1/connection_manager.cpp new file mode 100644 index 0000000000..c1b2d4a426 --- /dev/null +++ b/src/part1/connection_manager.cpp @@ -0,0 +1,32 @@ + +#include "src/part1/connection_manager.h" + +namespace nebd { +namespace client { + +ConnectionManager::ConnectionManager( + uint32_t heartbeatIntervalS, + std::shared_ptr metaCache) + : heartbeatIntervalS_(heartbeatIntervalS) + , metaCache_(metaCache) { + // TODO +} + +ConnectionManager::~ConnectionManager() { + // TODO +} + +void ConnectionManager::Run() { + // TODO +} + +brpc::Channel* ConnectionManager::GetChannel() { + return channel_; +} + +void ConnectionManager::SendHeartBeat() { + // TODO +} + +} // namespace client +} // namespace nebd \ No newline at end of file diff --git a/src/part1/connection_manager.h b/src/part1/connection_manager.h new file mode 100644 index 0000000000..eb99b9d020 --- /dev/null +++ b/src/part1/connection_manager.h @@ -0,0 +1,33 @@ +#ifndef SRC_PART1_CONNECTION_MANAGER_H_ +#define SRC_PART1_CONNECTION_MANAGER_H_ + +#include +#include // NOLINT +#include +#include "src/part1/nebd_metacache.h" + +namespace nebd { +namespace client { + +class ConnectionManager { + public: + ConnectionManager(uint32_t heartbeatIntervalS, + std::shared_ptr metaCache); + ~ConnectionManager(); + void Run(); + brpc::Channel* GetChannel(); + + private: + void SendHeartBeat(); + + private: + brpc::Channel* channel_; + uint32_t heartbeatIntervalS_; + std::shared_ptr metaCache_; + std::thread heartbeatThread_; +}; + +} // namespace client +} // namespace nebd + +#endif // SRC_PART1_CONNECTION_MANAGER_H_ diff --git a/src/part1/libnebd.cpp b/src/part1/libnebd.cpp index a30ae28391..28c9a432e1 100644 --- a/src/part1/libnebd.cpp +++ b/src/part1/libnebd.cpp @@ -10,13 +10,15 @@ extern "C" { bool g_inited = false; + +// Note: 配置文件路径是否有上层传下来比较合适，评估是否要修改 const char* confpath = "/etc/nebd/nebd-client.conf"; int nebd_lib_init() { if (g_inited) { return 0; } - int ret = Init4Qemu(confpath); + int ret = Init4Nebd(confpath); if (ret != 0) { return ret; } @@ -28,7 +30,7 @@ int nebd_lib_init() { int nebd_lib_uninit() { if (g_inited) { - Uninit4Qemu(); + Uninit4Nebd(); g_inited = false; } @@ -36,11 +38,11 @@ int nebd_lib_uninit() { } int nebd_lib_open(const char* filename) { - return Open4Qemu(filename); + return Open4Nebd(filename); } int nebd_lib_close(int fd) { - return Close4Qemu(fd); + return Close4Nebd(fd); } int nebd_lib_pread(int fd, void* buf, off_t offset, size_t length) { @@ -53,16 +55,16 @@ int nebd_lib_pwrite(int fd, const void* buf, off_t offset, size_t length) { return -1; } -int nebd_lib_discard(int fd, ClientAioContext* context) { - return Discard4Qemu(fd, context); +int nebd_lib_discard(int fd, NebdClientAioContext* context) { + return Discard4Nebd(fd, context); } -int nebd_lib_aio_pread(int fd, ClientAioContext* context) { - return AioRead4Qemu(fd, context); +int nebd_lib_aio_pread(int fd, NebdClientAioContext* context) { + return AioRead4Nebd(fd, context); } -int nebd_lib_aio_pwrite(int fd, ClientAioContext* context) { - return AioWrite4Qemu(fd, context); +int nebd_lib_aio_pwrite(int fd, NebdClientAioContext* context) { + return AioWrite4Nebd(fd, context); } int nebd_lib_sync(int fd) { @@ -70,23 +72,23 @@ int nebd_lib_sync(int fd) { } int64_t nebd_lib_filesize(int fd) { - return StatFile4Qemu(fd); + return StatFile4Nebd(fd); } int nebd_lib_resize(int fd, int64_t size) { - return Extend4Qemu(fd, size); + return Extend4Nebd(fd, size); } -int nebd_lib_flush(int fd, ClientAioContext* context) { - return Flush4Qemu(fd, context); +int nebd_lib_flush(int fd, NebdClientAioContext* context) { + return Flush4Nebd(fd, context); } int64_t nebd_lib_getinfo(int fd) { - return GetInfo4Qemu(fd); + return GetInfo4Nebd(fd); } int nebd_lib_invalidcache(int fd) { - return InvalidCache4Qemu(fd); + return InvalidCache4Nebd(fd); } } // extern "C" diff --git a/src/part1/libnebd.h b/src/part1/libnebd.h index 1d3149b59e..c0d94dd014 100644 --- a/src/part1/libnebd.h +++ b/src/part1/libnebd.h @@ -23,6 +23,8 @@ extern "C" { // 文件路径最大的长度，单位字节 #define NEBD_MAX_FILE_PATH_LEN 1024 + +// Note: 这个需要评估是否要去掉，之前出现过上层IO超过32MB的情况 // 请求的buf的最大长度，读写操作的buf长度不得超过该值，单位字节 #define NEBD_MAX_BUF_LEN 1024 * 1024 * 32 @@ -34,12 +36,12 @@ typedef enum LIBAIO_OP { LIBAIO_OP_FLUSH, } LIBAIO_OP; -struct ClientAioContext; +struct NebdClientAioContext; // nebd回调函数的类型 -typedef void (*LibAioCallBack)(struct ClientAioContext* context); +typedef void (*LibAioCallBack)(struct NebdClientAioContext* context); -typedef struct ClientAioContext { +struct NebdClientAioContext { off_t offset; // 请求的offset size_t length; // 请求的length int ret; // 记录异步返回的返回值 @@ -47,7 +49,7 @@ typedef struct ClientAioContext { LibAioCallBack cb; // 异步请求的回调函数 void* buf; // 请求的buf unsigned int retryCount; // 记录异步请求的重试次数 -} ClientAioContext; +}; // int nebd_lib_fini(void); // for ceph & curve @@ -105,7 +107,7 @@ int nebd_lib_pwrite(int fd, const void* buf, off_t offset, size_t length); * context：异步请求的上下文，包含请求所需的信息以及回调 * @return 成功返回0，失败返回错误码 */ -int nebd_lib_discard(int fd, ClientAioContext* context); +int nebd_lib_discard(int fd, NebdClientAioContext* context); /** * @brief 读文件，异步函数 @@ -113,7 +115,7 @@ int nebd_lib_discard(int fd, ClientAioContext* context); * context：异步请求的上下文，包含请求所需的信息以及回调 * @return 成功返回0，失败返回错误码 */ -int nebd_lib_aio_pread(int fd, ClientAioContext* context); +int nebd_lib_aio_pread(int fd, NebdClientAioContext* context); /** * @brief 写文件，异步函数 @@ -121,7 +123,7 @@ int nebd_lib_aio_pread(int fd, ClientAioContext* context); * context：异步请求的上下文，包含请求所需的信息以及回调 * @return 成功返回0，失败返回错误码 */ -int nebd_lib_aio_pwrite(int fd, ClientAioContext* context); +int nebd_lib_aio_pwrite(int fd, NebdClientAioContext* context); /** * @brief sync文件 @@ -152,7 +154,7 @@ int nebd_lib_resize(int fd, int64_t size); * context：异步请求的上下文，包含请求所需的信息以及回调 * @return 成功返回0，失败返回错误码 */ -int nebd_lib_flush(int fd, ClientAioContext* context); +int nebd_lib_flush(int fd, NebdClientAioContext* context); /** * @brief 获取文件info diff --git a/src/part1/libnebd_file.cpp b/src/part1/libnebd_file.cpp index df273feabd..11e27d52c5 100644 --- a/src/part1/libnebd_file.cpp +++ b/src/part1/libnebd_file.cpp @@ -8,55 +8,54 @@ #include "src/part1/libnebd_file.h" #include #include "src/common/configuration.h" -#include "src/part1/nebd_lifecycle.h" #include "src/part1/nebd_client.h" -int Init4Qemu(const char* confpath) { - return nebd::client::fileClient.Init(confpath); +int Init4Nebd(const char* confpath) { + return nebd::client::nebdClient.Init(confpath); } -void Uninit4Qemu() { - nebd::client::fileClient.Uninit(); +void Uninit4Nebd() { + nebd::client::nebdClient.Uninit(); return; } -int Open4Qemu(const char* filename) { - return nebd::client::fileClient.Open(filename); +int Open4Nebd(const char* filename) { + return nebd::client::nebdClient.Open(filename); } -int Close4Qemu(int fd) { - return nebd::client::fileClient.Close(fd); +int Close4Nebd(int fd) { + return nebd::client::nebdClient.Close(fd); } -int Extend4Qemu(int fd, int64_t newsize) { - return nebd::client::fileClient.Extend(fd, newsize); +int Extend4Nebd(int fd, int64_t newsize) { + return nebd::client::nebdClient.Extend(fd, newsize); } -int64_t StatFile4Qemu(int fd) { - return nebd::client::fileClient.StatFile(fd); +int64_t StatFile4Nebd(int fd) { + return nebd::client::nebdClient.StatFile(fd); } -int Discard4Qemu(int fd, ClientAioContext* aioctx) { - return nebd::client::fileClient.Discard(fd, aioctx); +int Discard4Nebd(int fd, NebdClientAioContext* aioctx) { + return nebd::client::nebdClient.Discard(fd, aioctx); } -int AioRead4Qemu(int fd, ClientAioContext* aioctx) { - return nebd::client::fileClient.AioRead(fd, aioctx); +int AioRead4Nebd(int fd, NebdClientAioContext* aioctx) { + return nebd::client::nebdClient.AioRead(fd, aioctx); } -int AioWrite4Qemu(int fd, ClientAioContext* aioctx) { - return nebd::client::fileClient.AioWrite(fd, aioctx); +int AioWrite4Nebd(int fd, NebdClientAioContext* aioctx) { + return nebd::client::nebdClient.AioWrite(fd, aioctx); } -int Flush4Qemu(int fd, ClientAioContext* aioctx) { - return nebd::client::fileClient.Flush(fd, aioctx); +int Flush4Nebd(int fd, NebdClientAioContext* aioctx) { + return nebd::client::nebdClient.Flush(fd, aioctx); } -int64_t GetInfo4Qemu(int fd) { - return nebd::client::fileClient.GetInfo(fd); +int64_t GetInfo4Nebd(int fd) { + return nebd::client::nebdClient.GetInfo(fd); } -int InvalidCache4Qemu(int fd) { - return nebd::client::fileClient.InvalidCache(fd); +int InvalidCache4Nebd(int fd) { + return nebd::client::nebdClient.InvalidCache(fd); } diff --git a/src/part1/libnebd_file.h b/src/part1/libnebd_file.h index 4caf875941..02da6ff7dc 100644 --- a/src/part1/libnebd_file.h +++ b/src/part1/libnebd_file.h @@ -15,77 +15,77 @@ * @param none * @return 成功返回0，失败返回-1 */ -int Init4Qemu(const char* confpath); +int Init4Nebd(const char* confpath); /** * @brief 反初始化nebd * @param none * @return 成功返回0，失败返回-1 */ -void Uninit4Qemu(); +void Uninit4Nebd(); /** * @brief open文件 * @param filename：文件名 * @return 成功返回文件fd，失败返回错误码 */ -int Open4Qemu(const char* filename); +int Open4Nebd(const char* filename); /** * @brief close文件 * @param fd：文件的fd * @return 成功返回0，失败返回错误码 */ -int Close4Qemu(int fd); +int Close4Nebd(int fd); /** * @brief resize文件 * @param fd：文件的fd * size：调整后的文件size * @return 成功返回0，失败返回错误码 */ -int Extend4Qemu(int fd, int64_t newsize); +int Extend4Nebd(int fd, int64_t newsize); /** * @brief 获取文件size * @param fd：文件的fd * @return 成功返回文件size，失败返回错误码 */ -int64_t StatFile4Qemu(int fd); +int64_t StatFile4Nebd(int fd); /** * @brief discard文件，异步函数 * @param fd：文件的fd * context：异步请求的上下文，包含请求所需的信息以及回调 * @return 成功返回0，失败返回错误码 */ -int Discard4Qemu(int fd, ClientAioContext* aioctx); +int Discard4Nebd(int fd, NebdClientAioContext* aioctx); /** * @brief 读文件，异步函数 * @param fd：文件的fd * context：异步请求的上下文，包含请求所需的信息以及回调 * @return 成功返回0，失败返回错误码 */ -int AioRead4Qemu(int fd, ClientAioContext* aioctx); +int AioRead4Nebd(int fd, NebdClientAioContext* aioctx); /** * @brief 写文件，异步函数 * @param fd：文件的fd * context：异步请求的上下文，包含请求所需的信息以及回调 * @return 成功返回0，失败返回错误码 */ -int AioWrite4Qemu(int fd, ClientAioContext* aioctx); +int AioWrite4Nebd(int fd, NebdClientAioContext* aioctx); /** * @brief flush文件，异步函数 * @param fd：文件的fd * context：异步请求的上下文，包含请求所需的信息以及回调 * @return 成功返回0，失败返回错误码 */ -int Flush4Qemu(int fd, ClientAioContext* aioctx); +int Flush4Nebd(int fd, NebdClientAioContext* aioctx); /** * @brief 获取文件info * @param fd：文件的fd * @return 成功返回文件对象size，失败返回错误码 */ -int64_t GetInfo4Qemu(int fd); +int64_t GetInfo4Nebd(int fd); /** * @brief 刷新cache，等所有异步请求返回 * @param fd：文件的fd * @return 成功返回0，失败返回错误码 */ -int InvalidCache4Qemu(int fd); +int InvalidCache4Nebd(int fd); #endif // SRC_PART1_LIBNEBD_FILE_H_ diff --git a/src/part1/nebd_client.cpp b/src/part1/nebd_client.cpp index 957c3758f0..3ea455fe6e 100644 --- a/src/part1/nebd_client.cpp +++ b/src/part1/nebd_client.cpp @@ -28,533 +28,68 @@ namespace brpc { namespace nebd { namespace client { -FileClient &fileClient = FileClient::GetInstance(); +NebdClient &nebdClient = NebdClient::GetInstance(); -int FileClient::Init(const char* confpath) { - // 从配置文件中获取 - using ::nebd::common::Configuration; - Configuration conf; - conf.SetConfigPath(confpath); - if (!conf.LoadConfig()) { - LOG(ERROR) << "load conf fail, conf path = " << confpath; - return -1; - } - - int ret = LoadConf(&conf); - if (ret != 0) { - LOG(ERROR) << "FileClient LoadConf fail."; - return -1; - } - - ret = lifeCycleManager_.Start(&conf); - if (ret != 0) { - LOG(ERROR) << "FileClient lifecycle manager start fail"; - return -1; - } - - std::string addr = lifeCycleManager_.GetPart2Addr(); - ret = InitChannel(addr); - if (ret != 0) { - LOG(ERROR) << "FileClient init channel fail, addr = " << addr; - return -1; - } - - lifeCycleManager_.StartHeartbeat(); - - LOG(INFO) << "FileClient init success."; +int NebdClient::Init(const char* confpath) { + // TODO return 0; } -void FileClient::Uninit() { - lifeCycleManager_.Stop(); - LOG(INFO) << "FileClient uninit success."; +void NebdClient::Uninit() { + // TODO + LOG(INFO) << "NebdClient uninit success."; } -int FileClient::Open(const char* filename) { - uint32_t retryCount = 0; - do { - nebd::client::QemuClientService_Stub stub(&channel_); - brpc::Controller cntl; - cntl.set_timeout_ms(rpcTimeoutMs_); - nebd::client::OpenFileRequest request; - nebd::client::OpenFileResponse response; - request.set_filename(filename); - stub.OpenFile(&cntl, &request, &response, NULL); - if (!cntl.Failed()) { - if (response.retcode() != nebd::client::RetCode::kOK) { - LOG(ERROR) << "Open fail, retCode = " - << response.retcode() - << ", filename = " << filename; - return -1; - } - std::string retMsg = response.retmsg(); - uint64_t fd = response.fd(); - LOG(INFO) << "Open ok." << "retMsg: " << retMsg - << ", filename = " << filename - << ", fd = " << fd; - return fd; - } else { - LOG(WARNING) << "Open fail, filename = " << filename - << ", cntl errorCode: " << cntl.ErrorCode() - << ", cntl error: " << cntl.ErrorText(); - } - retryCount++; - bthread_usleep(rpcRetryIntervalUs_); - } while (retryCount < rpcRetryTimes_); - - LOG(ERROR) << "Open fail, filename = " << filename - << ", retryCount = " << retryCount; +int NebdClient::Open(const char* filename) { + // TODO return -1; } -int FileClient::Close(int fd) { - uint32_t retryCount = 0; - do { - nebd::client::QemuClientService_Stub stub(&channel_); - brpc::Controller cntl; - cntl.set_timeout_ms(rpcTimeoutMs_); - nebd::client::CloseFileRequest request; - nebd::client::CloseFileResponse response; - request.set_fd(fd); - stub.CloseFile(&cntl, &request, &response, NULL); - if (!cntl.Failed()) { - if (response.retcode() != nebd::client::RetCode::kOK) { - LOG(ERROR) << "Close fail, retCode = " - << response.retcode() - << ", fd = " << fd; - return -1; - } - std::string retMsg = response.retmsg(); - LOG(INFO) << "Close ok." << "retMsg: " << retMsg - << ", fd = " << fd; - return 0; - } else { - LOG(ERROR) << "Close fail, fd = " << fd - << ", cntl errorCode: " << cntl.ErrorCode() - << ", cntl error: " << cntl.ErrorText(); - } - retryCount++; - bthread_usleep(rpcRetryIntervalUs_); - } while (retryCount < rpcRetryTimes_); - - LOG(ERROR) << "Close fail, fd = " << fd - << ", retryCount = " << retryCount; +int NebdClient::Close(int fd) { + // TODO return -1; } -int FileClient::Extend(int fd, int64_t newsize) { - uint32_t retryCount = 0; - do { - nebd::client::QemuClientService_Stub stub(&channel_); - brpc::Controller cntl; - cntl.set_timeout_ms(rpcTimeoutMs_); - nebd::client::ResizeRequest request; - nebd::client::ResizeResponse response; - request.set_fd(fd); - request.set_newsize(newsize); - stub.ResizeFile(&cntl, &request, &response, NULL); - if (!cntl.Failed()) { - if (response.retcode() != nebd::client::RetCode::kOK) { - LOG(ERROR) << "Extend fail, retCode = " - << response.retcode() - << ", fd = " << fd - << ", newsize = " << newsize; - return -1; - } - std::string retMsg = response.retmsg(); - LOG(INFO) << "Extend ok." << "retMsg: " << retMsg - << ", fd = " << fd - << ", newsize = " << newsize; - return 0; - } else { - LOG(ERROR) << "Extend fail, fd = " << fd - << ", newsize = " << newsize - << ", cntl errorCode: " << cntl.ErrorCode() - << ", cntl error: " << cntl.ErrorText(); - } - - retryCount++; - bthread_usleep(rpcRetryIntervalUs_); - } while (retryCount < rpcRetryTimes_); - - LOG(ERROR) << "Extend fail, fd = " << fd - << ", newsize = " << newsize - << ", retryCount = " << retryCount; +int NebdClient::Extend(int fd, int64_t newsize) { + // TODO return -1; } -int64_t FileClient::StatFile(int fd) { - uint32_t retryCount = 0; - do { - nebd::client::QemuClientService_Stub stub(&channel_); - brpc::Controller cntl; - cntl.set_timeout_ms(rpcTimeoutMs_); - nebd::client::StatFileRequest request; - nebd::client::StatFileResponse response; - request.set_fd(fd); - stub.StatFile(&cntl, &request, &response, NULL); - if (!cntl.Failed()) { - if (response.retcode() != nebd::client::RetCode::kOK) { - LOG(ERROR) << "StatFile fail, retCode = " - << response.retcode() - << ", fd = " << fd; - return -1; - } - std::string retMsg = response.retmsg(); - uint64_t size = response.size(); - LOG(INFO) << "StatFile ok." << "retMsg: " << retMsg - << ", fd = " << fd - << ", size = " << size; - return size; - } else { - LOG(ERROR) << "StatFile fail, fd = " << fd - << ", cntl errorCode: " << cntl.ErrorCode() - << ", cntl error: " << cntl.ErrorText(); - } - - retryCount++; - bthread_usleep(rpcRetryIntervalUs_); - } while (retryCount < rpcRetryTimes_); - - LOG(ERROR) << "StatFile fail, fd = " << fd - << ", retryCount = " << retryCount; +int64_t NebdClient::StatFile(int fd) { + // TODO return -1; } -int FileClient::Discard(int fd, ClientAioContext* aioctx) { - DVLOG(6) << "Discard start, fd = " << fd - << ", offset = " << aioctx->offset - << ", length = " << aioctx->length; - nebd::client::QemuClientService_Stub stub(&channel_); - nebd::client::DiscardRequest request; - request.set_fd(fd); - request.set_offset(aioctx->offset); - request.set_size(aioctx->length); - DiscardDone *discardDone = new DiscardDone(fd, aioctx, rpcTimeoutMs_, - rpcRetryIntervalUs_, rpcRetryMaxIntervalUs_, - rpcHostDownRetryIntervalUs_, - aioRpcFailLogInterval_); - stub.Discard(&discardDone->cntl_, &request, - &discardDone->response_, discardDone); +int NebdClient::Discard(int fd, NebdClientAioContext* aioctx) { + // TODO return 0; } -int FileClient::AioRead(int fd, ClientAioContext* aioctx) { - DVLOG(6) << "AioRead start, fd = " << fd - << ", offset = " << aioctx->offset - << ", length = " << aioctx->length; - nebd::client::QemuClientService_Stub stub(&channel_); - nebd::client::ReadRequest request; - request.set_fd(fd); - request.set_offset(aioctx->offset); - request.set_size(aioctx->length); - ReadDone *readDone = new ReadDone(fd, aioctx, rpcTimeoutMs_, - rpcRetryIntervalUs_, rpcRetryMaxIntervalUs_, - rpcHostDownRetryIntervalUs_, - aioRpcFailLogInterval_); - stub.Read(&readDone->cntl_, &request, &readDone->response_, readDone); +int NebdClient::AioRead(int fd, NebdClientAioContext* aioctx) { + // TODO return 0; } -int FileClient::AioWrite(int fd, ClientAioContext* aioctx) { - DVLOG(6) << "AioWrite start, fd = " << fd - << ", offset = " << aioctx->offset - << ", length = " << aioctx->length; - nebd::client::QemuClientService_Stub stub(&channel_); - nebd::client::WriteRequest request; - request.set_fd(fd); - request.set_offset(aioctx->offset); - request.set_size(aioctx->length); - WriteDone *writeDone = new WriteDone(fd, aioctx, rpcTimeoutMs_, - rpcRetryIntervalUs_, rpcRetryMaxIntervalUs_, - rpcHostDownRetryIntervalUs_, - aioRpcFailLogInterval_); - writeDone->cntl_.request_attachment().append(aioctx->buf, aioctx->length); - stub.Write(&writeDone->cntl_, &request, &writeDone->response_, writeDone); +int NebdClient::AioWrite(int fd, NebdClientAioContext* aioctx) { + // TODO return 0; } -int FileClient::Flush(int fd, ClientAioContext* aioctx) { - DVLOG(6) << "Flush start, fd = " << fd; - nebd::client::QemuClientService_Stub stub(&channel_); - nebd::client::FlushRequest request; - request.set_fd(fd); - FlushDone *flushDone = new FlushDone(fd, aioctx, rpcTimeoutMs_, - rpcRetryIntervalUs_, rpcRetryMaxIntervalUs_, - rpcHostDownRetryIntervalUs_, - aioRpcFailLogInterval_); - stub.Flush(&flushDone->cntl_, &request, &flushDone->response_, flushDone); +int NebdClient::Flush(int fd, NebdClientAioContext* aioctx) { + // TODO return 0; } -int64_t FileClient::GetInfo(int fd) { - uint32_t retryCount = 0; - do { - nebd::client::QemuClientService_Stub stub(&channel_); - brpc::Controller cntl; - cntl.set_timeout_ms(rpcTimeoutMs_); - nebd::client::GetInfoRequest request; - nebd::client::GetInfoResponse response; - request.set_fd(fd); - stub.GetInfo(&cntl, &request, &response, NULL); - if (!cntl.Failed()) { - if (response.retcode() != nebd::client::RetCode::kOK) { - LOG(ERROR) << "GetInfo fail, retCode = " - << response.retcode() - << ", fd = " << fd; - return -1; - } - std::string retMsg = response.retmsg(); - uint64_t size = response.objsize(); - LOG(INFO) << "GetInfo ok." << "retMsg: " << retMsg - << ", fd = " << fd - << ", size = " << size; - return size; - } else { - LOG(ERROR) << "GetInfo fail, fd = " << fd - << ", cntl errorCode: " << cntl.ErrorCode() - << ", cntl error: " << cntl.ErrorText(); - } - - retryCount++; - bthread_usleep(rpcRetryIntervalUs_); - } while (retryCount < rpcRetryTimes_); - - LOG(ERROR) << "GetInfo fail, fd = " << fd - << ", retryCount = " << retryCount; +int64_t NebdClient::GetInfo(int fd) { + // TODO return -1; } -int FileClient::InvalidCache(int fd) { - uint32_t retryCount = 0; - do { - nebd::client::QemuClientService_Stub stub(&channel_); - brpc::Controller cntl; - cntl.set_timeout_ms(rpcTimeoutMs_); - nebd::client::InvalidateCacheRequest request; - nebd::client::InvalidateCacheResponse response; - request.set_fd(fd); - stub.InvalidateCache(&cntl, &request, &response, NULL); - if (!cntl.Failed()) { - if (response.retcode() != nebd::client::RetCode::kOK) { - LOG(ERROR) << "InvalidateCache fail, retCode = " - << response.retcode() - << ", fd = " << fd; - return -1; - } - std::string retMsg = response.retmsg(); - LOG(INFO) << "InvalidateCache ok." << "retMsg: " << retMsg - << ", fd = " << fd; - return 0; - } else { - LOG(ERROR) << "InvalidateCache fail, fd = " << fd - << ", cntl errorCode: " << cntl.ErrorCode() - << ", cntl error: " << cntl.ErrorText(); - } - - retryCount++; - bthread_usleep(rpcRetryIntervalUs_); - } while (retryCount < rpcRetryTimes_); - - LOG(ERROR) << "InvalidateCache fail, fd = " << fd - << ", retryCount = " << retryCount; +int NebdClient::InvalidCache(int fd) { + // TODO return -1; } -int FileClient::LoadConf(common::Configuration *conf) { - if (!conf->GetUInt32Value("rpcRetryTimes", &rpcRetryTimes_)) { - LOG(ERROR) << "get rpcRetryTimes fail."; - return -1; - } - - if (!conf->GetUInt32Value("rpcRetryIntervalUs", &rpcRetryIntervalUs_)) { - LOG(ERROR) << "get rpcRetryIntervalUs fail."; - return -1; - } - - if (!conf->GetUInt32Value("rpcHostDownRetryIntervalUs", - &rpcHostDownRetryIntervalUs_)) { - LOG(ERROR) << "get rpcHostDownRetryIntervalUs fail."; - return -1; - } - - if (!conf->GetUInt32Value("rpcRetryMaxIntervalUs", - &rpcRetryMaxIntervalUs_)) { - LOG(ERROR) << "get rpcRetryMaxIntervalUs fail."; - return -1; - } - - if (!conf->GetUInt32Value("rpcTimeoutMs", &rpcTimeoutMs_)) { - LOG(ERROR) << "get rpcTimeoutMs fail."; - return -1; - } - - if (!conf->GetUInt32Value("rpcHealthCheckIntervalS", - &rpcHealthCheckIntervalS_)) { - LOG(ERROR) << "get rpcHealthCheckIntervalS fail."; - return -1; - } - - if (!conf->GetUInt32Value("aioRpcFailLogInterval", - &aioRpcFailLogInterval_)) { - LOG(ERROR) << "get aioRpcFailLogInterval fail."; - return -1; - } - - LOG(INFO) << "FileClient load conf success."; - - return 0; -} - -int FileClient::InitChannel(const std::string& addr) { - brpc::FLAGS_health_check_interval = rpcHealthCheckIntervalS_; - if (channel_.Init(addr.c_str(), nullptr) != 0) { - LOG(ERROR) << "Init channel failed!"; - return -1; - } - - return 0; -} - -uint32_t AsyncRequestDone::GetRpcRetryIntervalUs(uint32_t retryCount) { - if (retryCount == 0) { - return rpcRetryIntervalUs_; - } - - // EHOSTDOWN: 找不到可用的server。 - // server可能停止服务了，也可能正在退出中(返回了ELOGOFF) - if (cntl_.ErrorCode() == EHOSTDOWN) { - return rpcHostDownRetryIntervalUs_; - } - - uint64_t tempRetryInteval = (uint64_t)rpcRetryIntervalUs_ * retryCount; - return tempRetryInteval > rpcRetryMaxIntervalUs_ ? rpcRetryMaxIntervalUs_ - : tempRetryInteval; -} - -void ReadDone::Run() { - std::unique_ptr self_guard(this); - - if (cntl_.Failed()) { - LOG_IF(WARNING, aioctx_->retryCount % aioRpcFailLogInterval_ == 0) - << "read rpc fail, fd = " << fd_ - << ", offset = " << aioctx_->offset - << ", length = " << aioctx_->length - << ", cntl errorCode: " << cntl_.ErrorCode() - << ", cntl error: " << cntl_.ErrorText() - << ", retryCount = " << aioctx_->retryCount; - aioctx_->retryCount++; - bthread_usleep(GetRpcRetryIntervalUs(aioctx_->retryCount)); - fileClient.AioRead(fd_, aioctx_); - } else { - if (nebd::client::RetCode::kOK == response_.retcode()) { - DVLOG(6) << "read success, fd = " << fd_ - << ", offset = " << aioctx_->offset - << ", length = " << aioctx_->length; - memcpy(aioctx_->buf, - cntl_.response_attachment().to_string().c_str(), - cntl_.response_attachment().size()); - aioctx_->ret = 0; - aioctx_->cb(aioctx_); - } else { - LOG(ERROR) << "read fail, fd = " << fd_ - << ", offset = " << aioctx_->offset - << ", length = " << aioctx_->length - << ", retCode = " << response_.retcode(); - aioctx_->ret = -1; - aioctx_->cb(aioctx_); - } - } -} - -void WriteDone::Run() { - std::unique_ptr self_guard(this); - - if (cntl_.Failed()) { - LOG_IF(WARNING, aioctx_->retryCount % aioRpcFailLogInterval_ == 0) - << "write rpc fail, fd = " << fd_ - << ", offset = " << aioctx_->offset - << ", length = " << aioctx_->length - << ", cntl errorCode: " << cntl_.ErrorCode() - << ", cntl error: " << cntl_.ErrorText() - << ", retryCount = " << aioctx_->retryCount; - aioctx_->retryCount++; - bthread_usleep(GetRpcRetryIntervalUs(aioctx_->retryCount)); - fileClient.AioWrite(fd_, aioctx_); - } else { - if (nebd::client::RetCode::kOK == response_.retcode()) { - DVLOG(6) << "write success, fd = " << fd_ - << ", offset = " << aioctx_->offset - << ", length = " << aioctx_->length; - aioctx_->ret = 0; - aioctx_->cb(aioctx_); - } else { - LOG(ERROR) << "write fail, fd = " << fd_ - << ", offset = " << aioctx_->offset - << ", length = " << aioctx_->length - << ", retCode = " << response_.retcode(); - aioctx_->ret = -1; - aioctx_->cb(aioctx_); - } - } -} -void DiscardDone::Run() { - std::unique_ptr self_guard(this); - - if (cntl_.Failed()) { - LOG_IF(WARNING, aioctx_->retryCount % aioRpcFailLogInterval_ == 0) - << "discard rpc fail, fd = " << fd_ - << ", offset = " << aioctx_->offset - << ", length = " << aioctx_->length - << ", cntl errorCode: " << cntl_.ErrorCode() - << ", cntl error: " << cntl_.ErrorText() - << ", retryCount = " << aioctx_->retryCount; - aioctx_->retryCount++; - bthread_usleep(GetRpcRetryIntervalUs(aioctx_->retryCount)); - fileClient.Discard(fd_, aioctx_); - } else { - if (nebd::client::RetCode::kOK == response_.retcode()) { - DVLOG(6) << "discard success, fd = " << fd_ - << ", offset = " << aioctx_->offset - << ", length = " << aioctx_->length; - aioctx_->ret = 0; - aioctx_->cb(aioctx_); - } else { - LOG(ERROR) << "discard fail, fd = " << fd_ - << ", offset = " << aioctx_->offset - << ", length = " << aioctx_->length - << ", retCode = " << response_.retcode(); - aioctx_->ret = -1; - aioctx_->cb(aioctx_); - } - } -} - -void FlushDone::Run() { - std::unique_ptr self_guard(this); - - if (cntl_.Failed()) { - LOG_IF(WARNING, aioctx_->retryCount % aioRpcFailLogInterval_ == 0) - << "flush rpc fail, fd = " << fd_ - << ", cntl errorCode: " << cntl_.ErrorCode() - << ", cntl error: " << cntl_.ErrorText() - << ", retryCount = " << aioctx_->retryCount; - aioctx_->retryCount++; - bthread_usleep(GetRpcRetryIntervalUs(aioctx_->retryCount)); - fileClient.Flush(fd_, aioctx_); - } else { - if (nebd::client::RetCode::kOK == response_.retcode()) { - DVLOG(6) << "flush success, fd = " << fd_; - aioctx_->ret = 0; - aioctx_->cb(aioctx_); - } else { - LOG(ERROR) << "flush fail, fd = " << fd_ - << ", retCode = " << response_.retcode(); - aioctx_->ret = -1; - aioctx_->cb(aioctx_); - } - } -} } // namespace client } // namespace nebd diff --git a/src/part1/nebd_client.h b/src/part1/nebd_client.h index b3ed05de53..26383afcfa 100644 --- a/src/part1/nebd_client.h +++ b/src/part1/nebd_client.h @@ -10,21 +10,22 @@ #include #include -#include "src/part1/libnebd.h" #include "src/common/configuration.h" -#include "src/part1/nebd_lifecycle.h" #include "src/common/client.pb.h" +#include "src/part1/libnebd.h" +#include "src/part1/connection_manager.h" +#include "src/part1/nebd_metacache.h" namespace nebd { namespace client { -class FileClient { +class NebdClient { public: - static FileClient &GetInstance() { - static FileClient fileClient; - return fileClient; + static NebdClient &GetInstance() { + static NebdClient client; + return client; } - ~FileClient() = default; + ~NebdClient() = default; /** * @brief 初始化nebd，仅在第一次调用的时候真正执行初始化逻辑 * @param none @@ -68,28 +69,28 @@ class FileClient { * context：异步请求的上下文，包含请求所需的信息以及回调 * @return 成功返回0，失败返回错误码 */ - int Discard(int fd, ClientAioContext* aioctx); + int Discard(int fd, NebdClientAioContext* aioctx); /** * @brief 读文件，异步函数 * @param fd：文件的fd * context：异步请求的上下文，包含请求所需的信息以及回调 * @return 成功返回0，失败返回错误码 */ - int AioRead(int fd, ClientAioContext* aioctx); + int AioRead(int fd, NebdClientAioContext* aioctx); /** * @brief 写文件，异步函数 * @param fd：文件的fd * context：异步请求的上下文，包含请求所需的信息以及回调 * @return 成功返回0，失败返回错误码 */ - int AioWrite(int fd, ClientAioContext* aioctx); + int AioWrite(int fd, NebdClientAioContext* aioctx); /** * @brief flush文件，异步函数 * @param fd：文件的fd * context：异步请求的上下文，包含请求所需的信息以及回调 * @return 成功返回0，失败返回错误码 */ - int Flush(int fd, ClientAioContext* aioctx); + int Flush(int fd, NebdClientAioContext* aioctx); /** * @brief 获取文件info * @param fd：文件的fd @@ -103,138 +104,13 @@ class FileClient { */ int InvalidCache(int fd); - // for test - int InitChannel(const std::string& addr); - int LoadConf(common::Configuration *conf); - private: - // int LoadConf(common::Configuration *conf); - // int InitChannel(const std::string& addr); - - private: - brpc::Channel channel_; - // 同步rpc请求的最大重试次数 - uint32_t rpcRetryTimes_; - // 同步rpc请求的重试间隔； - // 也用来计算异步rpc请求的重试间隔，也是异步rpc请求重试的最小间隔 + std::shared_ptr connectionMgr_; + std::shared_ptr metaCache_; uint32_t rpcRetryIntervalUs_; - // 当异步请求rpc返回EHOSTDOWN错误时的重试时间 - uint32_t rpcHostDownRetryIntervalUs_; - // 用来计算rpc请求的重试间隔，也是异步rpc请求重试的最大间隔 - uint32_t rpcRetryMaxIntervalUs_; - // rpc请求的超时时间 - uint32_t rpcTimeoutMs_; - // brpc的健康检查周期时间 - uint32_t rpcHealthCheckIntervalS_; - // 异步IO操作RPC失败日志打印间隔，每隔多少次打印一次 - uint32_t aioRpcFailLogInterval_; - // 管理生命周期的服务 - nebd::client::LifeCycleManager lifeCycleManager_; }; -extern FileClient &fileClient; - -class AsyncRequestDone: public google::protobuf::Closure { - public: - AsyncRequestDone(int fd, ClientAioContext* aioctx, - uint32_t rpcTimeoutMs, - uint32_t rpcRetryIntervalUs, - uint32_t rpcRetryMaxIntervalUs, - uint32_t rpcHostDownRetryIntervalUs, - uint32_t aioRpcFailLogInterval): - fd_(fd), - aioctx_(aioctx), - rpcTimeoutMs_(rpcTimeoutMs), - rpcRetryIntervalUs_(rpcRetryIntervalUs), - rpcRetryMaxIntervalUs_(rpcRetryMaxIntervalUs), - rpcHostDownRetryIntervalUs_(rpcHostDownRetryIntervalUs), - aioRpcFailLogInterval_(aioRpcFailLogInterval) { - cntl_.set_timeout_ms(rpcTimeoutMs); - } - - /** - * @brief 计算rpc的重试间隔，重试时间根据重试次数线性增加， - * 最小不小于rpcRetryIntervalUs_，最大不大于rpcRetryMaxIntervalUs_ - * @param retryCount: 重试次数 - * @return 返回计算出来的重试间隔 - */ - uint32_t GetRpcRetryIntervalUs(uint32_t retryCount); - - virtual void Run() {} - - // 请求的fd - int fd_; - // 请求的上下文 - ClientAioContext* aioctx_; - // 用来设置rpc请求的超时时间 - uint32_t rpcTimeoutMs_; - // 用来计算rpc请求的重试间隔，也是异步rpc请求重试的最小间隔 - uint32_t rpcRetryIntervalUs_; - // 用来计算rpc请求的重试间隔，也是异步rpc请求重试的最大间隔 - uint32_t rpcRetryMaxIntervalUs_; - // 当异步请求rpc返回EHOSTDOWN错误时的重试时间 - uint32_t rpcHostDownRetryIntervalUs_; - // 异步IO操作RPC失败日志打印间隔，每隔多少次打印一次 - uint32_t aioRpcFailLogInterval_; - // brpc异步请求的controller，保存下来避免预期之外的析构 - brpc::Controller cntl_; -}; - -class ReadDone: public AsyncRequestDone { - public: - ReadDone(int fd, ClientAioContext* aioctx, uint32_t rpcTimeoutMs, - uint32_t rpcRetryIntervalUs, uint32_t rpcRetryMaxIntervalUs, - uint32_t rpcHostDownRetryIntervalUs, - uint32_t aioRpcFailLogInterval): - AsyncRequestDone(fd, aioctx, rpcTimeoutMs, - rpcRetryIntervalUs, rpcRetryMaxIntervalUs, - rpcHostDownRetryIntervalUs, - aioRpcFailLogInterval) {} - void Run(); - nebd::client::ReadResponse response_; -}; - -class WriteDone: public AsyncRequestDone { - public: - WriteDone(int fd, ClientAioContext* aioctx, uint32_t rpcTimeoutMs, - uint32_t rpcRetryIntervalUs, uint32_t rpcRetryMaxIntervalUs, - uint32_t rpcHostDownRetryIntervalUs, - uint32_t aioRpcFailLogInterval): - AsyncRequestDone(fd, aioctx, rpcTimeoutMs, - rpcRetryIntervalUs, rpcRetryMaxIntervalUs, - rpcHostDownRetryIntervalUs, - aioRpcFailLogInterval) {} - void Run(); - nebd::client::WriteResponse response_; -}; - -class DiscardDone: public AsyncRequestDone { - public: - DiscardDone(int fd, ClientAioContext* aioctx, uint32_t rpcTimeoutMs, - uint32_t rpcRetryIntervalUs, uint32_t rpcRetryMaxIntervalUs, - uint32_t rpcHostDownRetryIntervalUs, - uint32_t aioRpcFailLogInterval): - AsyncRequestDone(fd, aioctx, rpcTimeoutMs, - rpcRetryIntervalUs, rpcRetryMaxIntervalUs, - rpcHostDownRetryIntervalUs, - aioRpcFailLogInterval) {} - void Run(); - nebd::client::DiscardResponse response_; -}; - -class FlushDone: public AsyncRequestDone { - public: - FlushDone(int fd, ClientAioContext* aioctx, uint32_t rpcTimeoutMs, - uint32_t rpcRetryIntervalUs, uint32_t rpcRetryMaxIntervalUs, - uint32_t rpcHostDownRetryIntervalUs, - uint32_t aioRpcFailLogInterval): - AsyncRequestDone(fd, aioctx, rpcTimeoutMs, - rpcRetryIntervalUs, rpcRetryMaxIntervalUs, - rpcHostDownRetryIntervalUs, - aioRpcFailLogInterval) {} - void Run(); - nebd::client::FlushResponse response_; -}; +extern NebdClient &nebdClient; } // namespace client } // namespace nebd diff --git a/src/part1/nebd_lifecycle.cpp b/src/part1/nebd_lifecycle.cpp deleted file mode 100644 index 651d3bd30f..0000000000 --- a/src/part1/nebd_lifecycle.cpp +++ /dev/null @@ -1,784 +0,0 @@ -/* - * Project: nebd - * File Created: 2019-10-08 - * Author: hzchenwei7 - * Copyright (c) 2018 NetEase - */ - -#include "src/part1/nebd_lifecycle.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/common/configuration.h" -#include "src/part1/libnebd_file.h" - -namespace nebd { -namespace client { - -int LifeCycleManager::Start(common::Configuration *conf) { - LOG(INFO) << "start LifeCycleManager begin."; - - int ret = LoadConf(conf); - if (ret != 0) { - LOG(ERROR) << "LifeCycleManager LoadConf fail"; - return -1; - } - - // 根据本进程pid，查询qemu的uuid - pidPart1_ = getpid(); - LOG(INFO) << "qemu pid = " << pidPart1_; - - char uuid[kUuidLength + 1] = {0}; - ret = getUuid(pidPart1_, lifeCycleOptions_.qemuProcName.c_str(), uuid); - if (ret != 0) { - LOG(ERROR) << "get qemu uuid fail"; - return -1; - } - - qemuUUID_ = uuid; - LOG(INFO) << "get qemu uuid = " << qemuUUID_; - - // open文件锁 - int fd = open(lifeCycleOptions_.lockFile.c_str(), O_RDONLY | O_CREAT, 0644); - if (fd < 0) { - LOG(ERROR) << "open lock file fail, file = " - << lifeCycleOptions_.lockFile; - return -1; - } - - // 检查part2是否运行 - bool isPart2Alive = false; - - uint32_t retryCount = 0; - do { - // 加文件锁 - bool boolRet = LockFileWithRetry(fd, - lifeCycleOptions_.fileLockRetryTimes, - lifeCycleOptions_.fileLockRetryIntervalUs); - - if (!boolRet) { - LOG(ERROR) << "lock file with retry fail, file = " - << lifeCycleOptions_.lockFile; - close(fd); - return -1; - } - - ret = CheckProcAlive(uuid, lifeCycleOptions_.part2ProcName.c_str(), - &isPart2Alive, &pidPart2_); - if (ret < 0) { - int release = flock(fd, LOCK_UN); - if (release < 0) { - LOG(ERROR) << "unlock file fail, file = " - << lifeCycleOptions_.lockFile; - } - LOG(ERROR) << "check part2 alive fail"; - close(fd); - return -1; - } - - if (isPart2Alive) { - // 如果part2存活，先不释放文件锁，继续持有锁去查询port信息 - break; - } - - // 清理初始化文件，拉起part2 - CleanMetadataFile(); - StartPart2(); - - // 释放文件锁 - int release = flock(fd, LOCK_UN); - if (release < 0) { - LOG(ERROR) << "unlock file fail, file = " - << lifeCycleOptions_.lockFile; - close(fd); - return -1; - } - - retryCount++; - - usleep(lifeCycleOptions_.part2StartRetryIntervalUs); - } while (!isPart2Alive - && retryCount < lifeCycleOptions_.part2StartRetryTimes); - - // 拉起part2多次失败 - if (!isPart2Alive) { - LOG(ERROR) << "start part2 fail."; - close(fd); - return -1; - } - - // 读取端口，端口读取成功后，启动心跳服务 - uint32_t port = 0; - retryCount = 0; - while (1) { - ret = GetPortFromPart2(&port); - - // 前面在查询part2是否存活时已经加锁，这里读取port之后释放文件锁 - int release = flock(fd, LOCK_UN); - if (release < 0) { - LOG(ERROR) << "unlock file fail, file = " - << lifeCycleOptions_.lockFile; - close(fd); - return -1; - } - - if (ret == 0) { - // 找到port - LOG(INFO) << "get port from part2 success, port = " << port; - break; - } - - retryCount++; - LOG(WARNING) << "get port fail, retry, retryCount = " << retryCount; - if (retryCount == lifeCycleOptions_.portGetRetryTimes) { - // 达到重试次数，未找到port - LOG(ERROR) << "get port fail reach portGetRetryTimes = " - << lifeCycleOptions_.portGetRetryTimes; - break; - } - - usleep(lifeCycleOptions_.portGetRetryIntervalUs); - - // 为下一次循环，加文件锁 - bool boolRet = LockFileWithRetry(fd, - lifeCycleOptions_.fileLockRetryTimes, - lifeCycleOptions_.fileLockRetryIntervalUs); - - if (!boolRet) { - LOG(ERROR) << "lock file with retry fail, file = " - << lifeCycleOptions_.lockFile; - close(fd); - return -1; - } - } - - // 未成功读取port信息，kill part2 - if (port == 0) { - bool boolRet = LockFileWithRetry(fd, - lifeCycleOptions_.fileLockRetryTimes, - lifeCycleOptions_.fileLockRetryIntervalUs); - - if (!boolRet) { - LOG(ERROR) << "lock file with retry fail, file = " - << lifeCycleOptions_.lockFile; - close(fd); - return -1; - } - - KillPart2(); - - int release = flock(fd, LOCK_UN); - if (release < 0) { - LOG(ERROR) << "unlock file fail, file = " - << lifeCycleOptions_.lockFile; - close(fd); - return -1; - } - - LOG(ERROR) << "get port fail, kill part2."; - close(fd); - return -1; - } - - part2Port_ = port; - - close(fd); - LOG(INFO) << "start LifeCycleManager success."; - return 0; -} - -void LifeCycleManager::Stop() { - LOG(INFO) << "LifeCycleManager STOP start."; - shouldHeartbeatStop_ = true; - if (heartbeatThread_ != nullptr && heartbeatThread_->joinable()) { - heartbeatThread_->join(); - delete heartbeatThread_; - heartbeatThread_ = nullptr; - } - - isHeartbeatThreadStart_ = false; - - KillPart2(); - - pidPart2_ = -1; - pidPart1_ = -1; - part2Port_ = 0; - - LOG(INFO) << "LifeCycleManager STOPED."; - - return; -} - -int LifeCycleManager::LoadConf(common::Configuration *conf) { - if (!conf->GetStringValue("part2ProcName", - &lifeCycleOptions_.part2ProcName)) { - LOG(ERROR) << "get part2ProcName fail."; - return -1; - } - - if (!conf->GetStringValue("part2ProcPath", - &lifeCycleOptions_.part2ProcPath)) { - LOG(ERROR) << "get part2ProcPath fail."; - return -1; - } - - if (!conf->GetStringValue("part2Addr", &lifeCycleOptions_.part2Addr)) { - LOG(ERROR) << "get part2Addr fail."; - return -1; - } - - if (!conf->GetUInt32Value("part2KillCheckRetryTimes", - &lifeCycleOptions_.part2KillCheckRetryTimes)) { - LOG(ERROR) << "get part2KillCheckRetryTimes fail."; - return -1; - } - - if (!conf->GetUInt32Value("part2KillCheckRetryIntervalUs", - &lifeCycleOptions_.part2KillCheckRetryIntervalUs)) { - LOG(ERROR) << "get part2KillCheckRetryIntervalUs fail."; - return -1; - } - - if (!conf->GetStringValue("qemuProcName", - &lifeCycleOptions_.qemuProcName)) { - LOG(ERROR) << "get qemuProcName fail."; - return -1; - } - - if (!conf->GetStringValue("lockFile", &lifeCycleOptions_.lockFile)) { - LOG(ERROR) << "get lockFile fail."; - return -1; - } - - if (!conf->GetStringValue("metadataPrefix", - &lifeCycleOptions_.metadataPrefix)) { - LOG(ERROR) << "get metadataPrefix fail."; - return -1; - } - - if (!conf->GetUInt32Value("part2StartRetryTimes", - &lifeCycleOptions_.part2StartRetryTimes)) { - LOG(ERROR) << "get part2StartRetryTimes fail."; - return -1; - } - - if (!conf->GetUInt32Value("connectibleCheckTimes", - &lifeCycleOptions_.connectibleCheckTimes)) { - LOG(ERROR) << "get connectibleCheckTimes fail."; - return -1; - } - - if (!conf->GetUInt32Value("connectibleCheckIntervalUs", - &lifeCycleOptions_.connectibleCheckIntervalUs)) { - LOG(ERROR) << "get connectibleCheckIntervalUs fail."; - return -1; - } - - if (!conf->GetUInt32Value("portGetRetryTimes", - &lifeCycleOptions_.portGetRetryTimes)) { - LOG(ERROR) << "get portGetRetryTimes fail."; - return -1; - } - - if (!conf->GetUInt32Value("portGetRetryIntervalUs", - &lifeCycleOptions_.portGetRetryIntervalUs)) { - LOG(ERROR) << "get portGetRetryIntervalUs fail."; - return -1; - } - - if (!conf->GetUInt32Value("heartbeatIntervalUs", - &lifeCycleOptions_.heartbeatIntervalUs)) { - LOG(ERROR) << "get heartbeatIntervalUs fail."; - return -1; - } - - if (!conf->GetUInt32Value("fileLockRetryTimes", - &lifeCycleOptions_.fileLockRetryTimes)) { - LOG(ERROR) << "get fileLockRetryTimes fail."; - return -1; - } - - if (!conf->GetUInt32Value("fileLockRetryIntervalUs", - &lifeCycleOptions_.fileLockRetryIntervalUs)) { - LOG(ERROR) << "get fileLockRetryIntervalUs fail."; - return -1; - } - - LOG(INFO) << "LifeCycleManager load conf success"; - return 0; -} - -char *LifeCycleManager::SkipSpace(char *in, int *offset, int len) { - while (*offset < len) { - if (isspace(in[*offset]) || in[*offset] == '\0') { - ++*offset; - continue; - } - break; - } - return in + *offset; -} - -char *LifeCycleManager::SkipNoSpace(char *in, int *offset, int len) { - while (*offset < len) { - if (!isspace(in[*offset]) && in[*offset] != '\0') { - ++*offset; - continue; - } - break; - } - return in + *offset; -} - -int LifeCycleManager::getUuid(pid_t pid, const char *procName, char *uuid) { - char path[kCmdLinePathBufLength]; - snprintf(path, sizeof(path), "/proc/%d/cmdline", pid); - int fd = open(path, O_RDONLY); - if (fd < 0) { - return -1; - } - - char line[kCmdLineBufLength]; - int nbytesread = read(fd, line, kCmdLineBufLength); - close(fd); - - if (strncmp(basename(line), procName, strlen(procName)) != 0) { - return -1; - } - - char *p; - int i = strlen(line); - bool find = false; - for (; i < nbytesread; ) { - p = SkipSpace(line, &i, kCmdLineBufLength); - if (strncmp(p, "-uuid", 5) == 0) { - i += strlen(p); - p = SkipSpace(line, &i, kCmdLineBufLength); - strncpy(uuid, p, kUuidLength); - find = true; - break; - } - p = SkipNoSpace(line, &i, kCmdLineBufLength); - } - - if (find) { - return 0; - } else { - return -1; - } -} - -// 检查uuid和procName对应的进程在不在 -// 返回值-1，查找失败；返回值0，查找成功，返回是否查到及pid -int LifeCycleManager::CheckProcAlive(const char* uuid, const char* procName, - bool *procExist, pid_t *pid) { - DIR* dir = opendir("/proc"); - if (dir == NULL) { - LOG(ERROR) << "opendir(/proc) fail."; - return -1; - } - - // 遍历/proc目录下的所有进程 - struct dirent* ent; - while ((ent = readdir(dir))) { - if (!isdigit(*ent->d_name)) { - continue; - } - - pid_t tempPid = strtol(ent->d_name, NULL, 10); - - char procUuid[kUuidLength + 1] = {0}; - int ret = getUuid(tempPid, procName, procUuid); - if (ret < 0) { - continue; - } - - if (strncmp(procUuid, uuid, strlen(uuid)) == 0) { - closedir(dir); - *pid = tempPid; - *procExist = true; - DVLOG(6) << "CheckProcAlive alive, uuid = " << uuid - << ", procName = " << procName - << ", pid = " << tempPid; - return 0; - } - } - - // 遍历 /proc目录下的所有进程，没有找到uuid和procName对应的进程 - closedir(dir); - *procExist = false; - LOG(INFO) << "CheckProcAlive proc not alive, uuid = " << uuid - << ", procName = " << procName; - return 0; -} - -int LifeCycleManager::GetPortFromPart2(uint32_t *port) { - // 读取json文件 - std::string metadataFile = lifeCycleOptions_.metadataPrefix + qemuUUID_; - char jsonStr[kMetaFileBufLength] = {}; - int fd = open(metadataFile.c_str(), O_RDONLY); - if (fd < 0) { - LOG(ERROR) << "meta file open failed, " << metadataFile - << ", errno = " << errno; - return -1; - } - int ret = read(fd, jsonStr, kMetaFileBufLength); - if (ret <= 0) { - LOG(ERROR) << "meta file read failed, " << metadataFile; - close(fd); - return -1; - } - close(fd); - - // 解析json文件中的port信息 - Json::Reader reader; - Json::Value value; - if (!reader.parse(jsonStr, value)) { - return false; - } - - if (!value["port"].isNull() && value["port"].isString()) { - *port = stoi(value["port"].asString()); - LOG(INFO) << "read port from file = " << metadataFile - << ", port = " << *port; - return 0; - } else { - return -1; - } - - return 0; -} - -bool LifeCycleManager::IsPart2Connectible() { - butil::EndPoint ep; - std::string addr = GetPart2Addr(); - str2endpoint(addr.c_str(), &ep); - int fd = butil::tcp_connect(ep, NULL); - if (fd == -1) { - LOG(WARNING) << "part2 is NOT connectible, addr = " << addr; - return false; - } - close(fd); - return true; -} - -bool LifeCycleManager::IsPart2ConnectibleWithRetry(uint32_t retryTimes, - uint32_t retryIntervalUs) { - uint32_t retryCount = 0; - while (retryCount < retryTimes) { - if (IsPart2Connectible()) { - return true; - } - - // 如果检查part2不连通，再看下是否part2已经退出了 - if (!IsPart2Alive()) { - LOG(WARNING) << "part2 not connectible and part2 not alive."; - return false; - } - - retryCount++; - LOG(WARNING) << "part2 is not connectible, retry next time" - << ", unconnectCount = " << retryCount; - usleep(retryIntervalUs); - } - - return false; -} - -void LifeCycleManager::CleanMetadataFile() { - std::string metadataFile = lifeCycleOptions_.metadataPrefix + qemuUUID_; - if (access(metadataFile.c_str(), F_OK) == 0) { - LOG(INFO) << "CleanMetadataFile, meta file = " << metadataFile; - if (unlink(metadataFile.c_str()) != 0) { - LOG(WARNING) << "delete metadata file fail, file = " - << metadataFile; - } - } - - return; -} - -void LifeCycleManager::StartPart2() { - std::string cmd = lifeCycleOptions_.part2ProcPath + " -uuid " + qemuUUID_; - LOG(INFO) << "StartPart2, rum cmd: " << cmd; - system(cmd.c_str()); - - return; -} - -int LifeCycleManager::KillPart2() { - if (pidPart2_ == -1) { - return 0; - } - - std::string cmd = "kill " + std::to_string(pidPart2_); - LOG(INFO) << "KillPart2, run cmd: " << cmd; - system(cmd.c_str()); - - // 检查part2是否已经kill - uint32_t retryCount = 0; - bool procExist = true; - pid_t tempPid = 0; - while (retryCount < lifeCycleOptions_.part2KillCheckRetryTimes) { - int ret = CheckProcAlive(qemuUUID_.c_str(), - lifeCycleOptions_.part2ProcName.c_str(), - &procExist, &tempPid); - if (ret == 0 && (!procExist || tempPid != pidPart2_)) { - pidPart2_ = -1; - return 0; - } - - retryCount++; - LOG(INFO) << "kill part2, part2 still alive, retry" - << ", retryCount = " << retryCount; - usleep(lifeCycleOptions_.part2KillCheckRetryIntervalUs); - } - - // 多次检查失败，执行kill -9 - if (procExist) { - std::string cmd = "kill -9 " + std::to_string(pidPart2_); - LOG(INFO) << "KillPart2, run cmd = " << cmd; - system(cmd.c_str()); - } - - int ret = CheckProcAlive(qemuUUID_.c_str(), - lifeCycleOptions_.part2ProcName.c_str(), - &procExist, &tempPid); - if (ret != 0 || (procExist && tempPid == pidPart2_)) { - return -1; - } - - pidPart2_ = -1; - return 0; -} - -int LifeCycleManager::GetPortWithRetry(int fd, uint32_t *port) { - uint32_t retryCount = 0; - while (retryCount < lifeCycleOptions_.portGetRetryTimes) { - // 加文件锁 - int lock = flock(fd, LOCK_EX | LOCK_NB); - if (lock < 0) { - LOG(WARNING) << "lock file fail, file = " - << lifeCycleOptions_.lockFile; - retryCount++; - usleep(lifeCycleOptions_.portGetRetryIntervalUs); - continue; - } - - // 重新读取并更新port信息 - uint32_t tempPort = 0; - int ret = GetPortFromPart2(&tempPort); - - // 文件锁解锁 - int release = flock(fd, LOCK_UN); - LOG_IF(WARNING, release < 0) << "unlock file fail, file = " - << lifeCycleOptions_.lockFile; - - // 获取port信息失败，下次重试 - if (ret != 0 || tempPort == 0) { - retryCount++; - LOG(WARNING) << "update port from part2 fail, retry" - << "retryCount" << retryCount; - usleep(lifeCycleOptions_.portGetRetryIntervalUs); - continue; - } - - // 成功获取port信息 - LOG(INFO) << "get port with retry success, port = " << tempPort; - *port = tempPort; - return 0; - } - - // 尝试多次，未能成功获取port信息，返回失败 - return -1; -} - -int LifeCycleManager::StartPart2AndGetPortRetry(int fd, uint32_t *port) { - uint32_t retryCount = 0; - do { - StartPart2(); - bool isPart2Alive = false; - int ret = CheckProcAlive(qemuUUID_.c_str(), - lifeCycleOptions_.part2ProcName.c_str(), - &isPart2Alive, &pidPart2_); - if (ret < 0) { - LOG(ERROR) << "check part2 alive fail"; - return -1; - } - - if (isPart2Alive) { - uint32_t tempPort; - int ret = GetPortWithRetry(fd, &tempPort); - if (ret != 0) { - LOG(ERROR) << "get port fail"; - return -1; - } - *port = tempPort; - return 0; - } - - retryCount++; - LOG(INFO) << "start part2 but check alive fail, retry, retryCount = " - << retryCount; - - usleep(lifeCycleOptions_.part2StartRetryIntervalUs); - } while (retryCount < lifeCycleOptions_.part2StartRetryTimes); - - return -1; -} - -void LifeCycleManager::HeartbeatThreadFunc() { - LOG(INFO) << "heartbeat check thread start."; - - // open文件锁 - int fd = open(lifeCycleOptions_.lockFile.c_str(), O_RDONLY | O_CREAT, 0644); - if (fd < 0) { - LOG(ERROR) << "open lock file fail, file = " - << lifeCycleOptions_.lockFile; - return; - } - - while (!shouldHeartbeatStop_) { - // 检查part2是否运行 - bool isPart2Alive = false; - pid_t tempPid = 0; - int ret = CheckProcAlive(qemuUUID_.c_str(), - lifeCycleOptions_.part2ProcName.c_str(), - &isPart2Alive, &tempPid); - if (ret < 0) { - LOG(ERROR) << "kill part1."; - close(fd); - exit(1); - return; - } - - // 检查part2是否存活，如果未存活，需要拉起。拉起多次失败之后，退出part1。 - if (!isPart2Alive) { - uint32_t port; - int ret = StartPart2AndGetPortRetry(fd, &port); - if (ret < 0) { - LOG(ERROR) << "kill part1."; - close(fd); - exit(1); - return; - } - - // 拉起成功之后，读取并更新端口信息。 - part2Port_ = port; - continue; - } - - // 检查端口的连通性，如果联通失败，kill part2并重新拉起 - if (!IsPart2ConnectibleWithRetry( - lifeCycleOptions_.connectibleCheckTimes, - lifeCycleOptions_.connectibleCheckIntervalUs)) { - LOG(WARNING) << "part2 is not connectible, " - << "reach connectibleCheckTimes " << lifeCycleOptions_.connectibleCheckTimes // NOLINT - << " or part2 not alive"; - - // 连续多次未联通，kill part2 - int ret = KillPart2(); - if (ret != 0) { - LOG(WARNING) << "part2 is not connectible, kill part2 fail"; - usleep(lifeCycleOptions_.heartbeatIntervalUs); - continue; - } - - uint32_t port; - ret = StartPart2AndGetPortRetry(fd, &port); - if (ret < 0) { - LOG(ERROR) << "kill part1."; - close(fd); - exit(1); - return; - } - - if (!IsPart2ConnectibleWithRetry( - lifeCycleOptions_.connectibleCheckTimes, - lifeCycleOptions_.connectibleCheckIntervalUs)) { - KillPart2(); - LOG(ERROR) << "kill part2, kill part1."; - close(fd); - exit(1); - return; - } - - part2Port_ = port; - continue; - } - - usleep(lifeCycleOptions_.heartbeatIntervalUs); - } - - LOG(INFO) << "stop heartbeat func."; - - close(fd); - return; -} - -void LifeCycleManager::StartHeartbeat() { - if (!isHeartbeatThreadStart_) { - shouldHeartbeatStop_ = false; - heartbeatThread_ = new std::thread( - &LifeCycleManager::HeartbeatThreadFunc, - this); - isHeartbeatThreadStart_ = true; - } - - return; -} - -std::string LifeCycleManager::GetPart2Addr() { - return lifeCycleOptions_.part2Addr + ":" + std::to_string(part2Port_); -} - -bool LifeCycleManager::IsPart2Alive() { - if (qemuUUID_ == "") { - return false; - } - - bool isPart2Alive = false; - pid_t pidPart2 = -1; - int ret = CheckProcAlive(qemuUUID_.c_str(), - lifeCycleOptions_.part2ProcName.c_str(), - &isPart2Alive, &pidPart2); - if (ret < 0) { - LOG(ERROR) << "check part2 alive fail"; - return false; - } - - return isPart2Alive; -} - -bool LifeCycleManager::IsHeartbeatThreadStart() { - return isHeartbeatThreadStart_; -} - -bool LifeCycleManager::LockFileWithRetry(int fd, uint32_t retryTimes, - uint32_t retryIntervalUs) { - uint32_t retryCount = 0; - while (retryCount < retryTimes) { - // 加文件锁 - int lock = flock(fd, LOCK_EX | LOCK_NB); - if (lock < 0) { - LOG(WARNING) << "lock file fail, file = " - << lifeCycleOptions_.lockFile - << ", retryCount = " << retryCount; - retryCount++; - usleep(retryIntervalUs); - } else { - return true; - } - } - - return false; -} - -} // namespace client -} // namespace nebd diff --git a/src/part1/nebd_lifecycle.h b/src/part1/nebd_lifecycle.h deleted file mode 100644 index 67bb17f371..0000000000 --- a/src/part1/nebd_lifecycle.h +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Project: nebd - * File Created: 2019-10-08 - * Author: hzchenwei7 - * Copyright (c) 2018 NetEase - */ - -#ifndef SRC_PART1_NEBD_LIFECYCLE_H_ -#define SRC_PART1_NEBD_LIFECYCLE_H_ - -#include -#include -#include // NOLINT -#include "src/common/configuration.h" - -const uint32_t kCmdLinePathBufLength = 32; -const uint32_t kCmdLineBufLength = 2048; -const uint32_t kUuidLength = 36; -const uint32_t kMetaFileBufLength = 2048; - -namespace nebd { -namespace client { -typedef struct LifeCycleOptions { - std::string part2ProcName; - std::string part2ProcPath; - std::string part2Addr; - uint32_t part2KillCheckRetryTimes; - uint32_t part2KillCheckRetryIntervalUs; - std::string qemuProcName; - std::string lockFile; - std::string metadataPrefix; - uint32_t part2StartRetryTimes; - uint32_t part2StartRetryIntervalUs; - uint32_t connectibleCheckTimes; - uint32_t connectibleCheckIntervalUs; - uint32_t portGetRetryTimes; - uint32_t portGetRetryIntervalUs; - uint32_t heartbeatIntervalUs; - uint32_t rpcRetryTimes; - uint32_t rpcRetryIntervalUs; - uint32_t rpcRetryMaxIntervalUs; - uint32_t rpcTimeoutMs; - uint32_t fileLockRetryTimes; - uint32_t fileLockRetryIntervalUs; -} LifeCycleOptions; - -// 负责生命周期管理服务 -class LifeCycleManager { - public: - LifeCycleManager() { - part2Port_ = 0; - pidPart1_ = -1; - pidPart2_ = -1; - qemuUUID_ = ""; - isHeartbeatThreadStart_ = false; - heartbeatThread_ = nullptr; - shouldHeartbeatStop_ = true; - } - ~LifeCycleManager() {} - - /** - * @brief 启动生命周期管理服务，检测part2是否拉起，拉起part2并获取端口信息 - * @param conf: 配置信息 - * @return 执行成功返回0，失败返回-1 - */ - int Start(common::Configuration *conf); - /** - * @brief 停止声明周期管理服务，退出心跳线程，kill part2 - * @param void - * @return void - */ - void Stop(); - /** - * @brief 获取part2的地址 - * @param void - * @return 返回获取的part2的地址 - */ - std::string GetPart2Addr(); - /** - * @brief 启动心跳服务 - * @param void - * @return void - */ - void StartHeartbeat(); - // for test - /** - * @brief part2进程是否存活 - * @param void - * @return 如果存活返回true，如果不存活返回false - */ - bool IsPart2Alive(); - /** - * @brief 心跳服务线程是否启动 - * @param void - * @return 如果启动返回true，如果未启动返回false - */ - bool IsHeartbeatThreadStart(); - /** - * @brief kill part2服务 - * @param void - * @return 执行成功返回0，失败返回-1 - */ - int KillPart2(); - - private: - int LoadConf(common::Configuration *conf); - char *SkipSpace(char *in, int *offset, int len); - char *SkipNoSpace(char *in, int *offset, int len); - /** - * @brief 根据pid和进程名，从cmdline中解析uuid - * @param pid: 进程id - * @param procName: 进程名字 - * @param uuid: 返回从cmdline中获取的uuid - * @return 执行成功返回0，失败返回-1 - */ - int getUuid(pid_t pid, const char *procName, char *uuid); - /** - * @brief 检查指定uuid和进程名的进程是否活着，如果进程活着，pid返回进程id - * @param uuid: - * @param procName: 进程名字 - * @param[out] procExist: 进程是否存活 - * @param[out] pid：进程id - * @return 执行成功返回0，失败返回-1 - */ - int CheckProcAlive(const char* uuid, const char* procName, - bool *procExist, pid_t *pid); - /** - * @brief 从metafile中读取port信息 - * @param[out] port：返回读到的port信息 - * @return 执行成功返回0，失败返回-1 - */ - int GetPortFromPart2(uint32_t *port); - /** - * @brief 从metafile中读取port信息，如果读取失败多次读取 - * @param fd：读取port时需要加文件锁，文件锁的fd - * @param[out] port：返回读到的port信息 - * @return 执行成功返回0，失败返回-1 - */ - int GetPortWithRetry(int fd, uint32_t *port); - /** - * @brief 检测part2是否可联通 - * @param void - * @return 可联通返回true，否则返回false - */ - bool IsPart2Connectible(); - /** - * @brief 检测part2是否可联通，重复多次检查不可联通才返回失败 - * @param retryTimes: 重试次数 - * @param retryIntervalUs：重试间隔 - * @return 可联通返回true，否则返回false - */ - bool IsPart2ConnectibleWithRetry(uint32_t retryTimes, - uint32_t retryIntervalUs); - /** - * @brief 清理part1和part2共用的metadate文件 - * @param void - * @return void - */ - void CleanMetadataFile(); - /** - * @brief 拉起part2服务，只负责拉起，不负责检测是否拉起成功 - * @param void - * @return void - */ - void StartPart2(); - /** - * @brief 拉起part2服务，拉起成功后，读取port信息 - * @param fd：读取port时需要加文件锁，文件锁的fd - * @param[out] port: 返回读取到的port信息 - * @return 执行成功返回0，失败返回-1 - */ - int StartPart2AndGetPortRetry(int fd, uint32_t *port); - /** - * @brief 心跳线程的执行函数 - * @param void - * @return void - */ - void HeartbeatThreadFunc(); - /** - * @brief 对文件加锁 - * @param fd: 加文件的文件的fd - * @param retryTimes: 如果加锁失败最大的重试次数 - * @param retryIntervalUs: 如果加锁失败进行重试的重试间隔 - * @return 执行成功返回true，失败返回false - */ - bool LockFileWithRetry(int fd, uint32_t retryTimes, - uint32_t retryIntervalUs); - - private: - // 从配置文件中读取到的声明周期管理所需要的配置字段 - LifeCycleOptions lifeCycleOptions_; - // part2的服务的port - uint32_t part2Port_; - // part1的pid - pid_t pidPart1_; - // part2的pid - pid_t pidPart2_; - // qemu的UUID - std::string qemuUUID_; - // heartbeat线程是否启动 - std::atomic isHeartbeatThreadStart_; - // heartbeat的线程 - std::thread *heartbeatThread_; - // 控制heartbeat线程退出的标志位 - std::atomic shouldHeartbeatStop_; -}; -} // namespace client -} // namespace nebd -#endif // SRC_PART1_NEBD_LIFECYCLE_H_ diff --git a/src/part1/nebd_metacache.cpp b/src/part1/nebd_metacache.cpp new file mode 100644 index 0000000000..62d7ac87ff --- /dev/null +++ b/src/part1/nebd_metacache.cpp @@ -0,0 +1,24 @@ +#include "src/part1/nebd_metacache.h" + +namespace nebd { +namespace client { + +NebdClientMetaCache::NebdClientMetaCache() {} +NebdClientMetaCache::~NebdClientMetaCache() {} + +void NebdClientMetaCache::AddFileInfo(NebdClientFileInfo fileInfo) { + // TODO +} + +void NebdClientMetaCache::RemoveFileInfo(NebdClientFileInfo fileInfo) { + // TODO +} + +std::vector NebdClientMetaCache::GetAllFileInfo() { + // TODO + std::vector result; + return result; +} + +} // namespace client +} // namespace nebd diff --git a/src/part1/nebd_metacache.h b/src/part1/nebd_metacache.h new file mode 100644 index 0000000000..92be69432f --- /dev/null +++ b/src/part1/nebd_metacache.h @@ -0,0 +1,33 @@ +#ifndef SRC_PART1_NEBD_METACACHE_H_ +#define SRC_PART1_NEBD_METACACHE_H_ + +#include +#include +#include +#include "src/common/rw_lock.h" + +namespace nebd { +namespace client { + +struct NebdClientFileInfo { + int fd; + std::string fileName; +}; + +class NebdClientMetaCache { + public: + NebdClientMetaCache(); + ~NebdClientMetaCache(); + void AddFileInfo(NebdClientFileInfo fileInfo); + void RemoveFileInfo(NebdClientFileInfo fileInfo); + std::vector GetAllFileInfo(); + + private: + std::unordered_map fileinfos_; + common::RWLock rwLock_; +}; + +} // namespace client +} // namespace nebd + +#endif // SRC_PART1_NEBD_METACACHE_H_ diff --git a/src/part2/common_type.cpp b/src/part2/common_type.cpp deleted file mode 100644 index edcfad27c2..0000000000 --- a/src/part2/common_type.cpp +++ /dev/null @@ -1,655 +0,0 @@ -/* - * Project: nebd - * File Created: 2019-09-30 - * Author: hzwuhongsong - * Copyright (c) 2019 NetEase - */ - -#include "src/part2/common_type.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/part2/config.h" -#include "src/part2/heartbeat.h" -#include "src/part2/reload.h" -#include "src/part2/rpc_server.h" - -// qemu进程的uuid -char* g_uuid; -// qemu进程的pid -int64_t g_oldPid = -1; -// 该nebd-server程序使用的端口号 -int g_port; -// 记录请求数量 -std::atomic_long requestForRpcWrite(0); -std::atomic_long requestForCephWrite(0); -std::atomic_long requestForRpcWriteCounts(0); -// uuid持久化文件存放路径 -const char* g_filePath; -// fd的内存映像 -std::map g_imageMap; - -// 保存云主机当前挂载的卷信息 -std::vector g_qemuPoolVolumes; - -// brpc server -brpc::Server g_server; - -// 文件锁函数 -int LockFile(const char* file) { - int lockfd = open(file, O_RDONLY | O_CREAT, 0644); - if (lockfd < 0) { - LOG(ERROR) << "open lock file failed, " << file; - return -1; - } - int ret = flock(lockfd, LOCK_EX); - if (ret < 0) { - LOG(ERROR) << "add lock failed, " << file; - close(lockfd); - return -1; - } - return lockfd; -} - -void UnlockFile(int lockfd) { - flock(lockfd, LOCK_UN); - close(lockfd); -} - -// 获取所有云主机当前挂载的卷信息 -int ReadQemuXmls() { - struct dirent* ent; - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return -1; - } - std::string qemu_xml_dir = ReadQemuXmlDir(cfg); - config_destroy(cfg); - DIR* dir = opendir(qemu_xml_dir.c_str()); - if (dir == NULL) { - LOG(ERROR) << "qemu xml dir open failed, " << qemu_xml_dir; - return -1; - } - while ((ent = readdir(dir))) { - if ((ent->d_name[0] == '.' && ent->d_name[1] == '\0') || - (ent->d_name[1] == '.' && ent->d_name[2] == '\0')) - continue; - - if (strstr(ent->d_name, ".xml") == NULL) continue; - std::string qemu_xml_file = qemu_xml_dir + "/" + ent->d_name; - ReadQemuXml(qemu_xml_file.c_str()); - } - return 0; -} - -// 获取一个云主机挂载的卷信息，存入qemu_xml_map -void ReadQemuXml(const char* xml_file) { - LOG(NOTICE) << "xml file, " << xml_file; - - boost::property_tree::ptree pt; - try { - boost::property_tree::xml_parser::read_xml(xml_file, pt); - } catch (boost::property_tree::ptree_error pt_error) { - LOG(ERROR) << "read xml throw an exception. " << pt_error.what(); - return; - } - - std::string protocol, pool_vol_name; - - auto item = pt.get_child_optional("domstatus.domain.uuid"); - if (!item) { - LOG(ERROR) << "domstatus.domain.uuid is not exist"; - return; - } - item = pt.get_child_optional("domstatus.domain.devices"); - if (!item) { - LOG(ERROR) << "domstatus.domain.devices is not exist"; - return; - } - - std::string uuid; - uuid = pt.get("domstatus.domain.uuid"); - if (uuid != g_uuid) { - LOG(ERROR) << "uuid is not equal domain.uuid, " << g_uuid << ", " - << uuid; - return; - } - - BOOST_AUTO(child, pt.get_child("domstatus.domain.devices")); - - for (auto iter = child.begin(); iter != child.end(); ++iter) { - if (iter->first != "disk") { - continue; - } - BOOST_AUTO(pos, iter->second.get_child("source")); - for (auto nextiter = pos.begin(); nextiter != pos.end(); ++nextiter) { - if ("" != nextiter->first) { - continue; - } - protocol = nextiter->second.get("protocol"); - pool_vol_name = nextiter->second.get("name"); - - if (protocol == "rbd") { - g_qemuPoolVolumes.push_back(pool_vol_name); - } else { - LOG(ERROR) << "protocol is: " << protocol - << ", pool_vol_name is: " << pool_vol_name; - } - } - } - for (auto name : g_qemuPoolVolumes) { - LOG(NOTICE) << "pool and vol is:" << name; - } -} - -// 生成port信息并存入持久化文件 -int GeneratePort(int port) { - std::string metadata_file = GetUuidFile(); - boost::property_tree::ptree root; - boost::property_tree::ptree items; - - try { - root.put("port", port); - boost::property_tree::write_json(metadata_file, root); - } catch (boost::property_tree::ptree_error pt) { - LOG(ERROR) << "write json throw an exception. " << pt.what(); - return -1; - } - - return 0; -} - -// 生成fd信息并存入持久化文件 -int GenerateFd(char* filename, int fd) { - std::string metadata_file = GetUuidFile(); - LOG(NOTICE) << "generate fd start. " << filename << ", " << fd; - boost::property_tree::ptree root, items; - - try { - boost::property_tree::read_json( - metadata_file, root); - } catch (boost::property_tree::ptree_error pt) { - LOG(ERROR) << "read json throw an exception. " << pt.what(); - return -1; - } - - // 第一次生成volumes - auto volumes_exist = root.get_child_optional("volumes"); - if (!volumes_exist) { - boost::property_tree::ptree item1; - item1.put("filename", filename); - item1.put("fd", fd); - items.push_back(std::make_pair("", item1)); - root.put("port", g_port); - root.put_child("volumes", items); - try { - boost::property_tree::write_json(metadata_file, root); - } catch (boost::property_tree::ptree_error pt) { - LOG(ERROR) << "write json throw an exception. " << pt.what(); - return -1; - } - return 0; - } - - items = root.get_child("volumes"); - int count = 0; - for (boost::property_tree::ptree::iterator it = items.begin(); - it != items.end(); ++it) { - count++; - } - - // boost::property_tree::ptree item[count + 1]; - std::vector item; - item.resize(count + 1); - int item_index = 0; - for (boost::property_tree::ptree::iterator it = items.begin(); - it != items.end(); ++it) { - std::string filename_json = it->second.get("filename"); - std::string fd_json = it->second.get("fd"); - item[item_index].put("filename", filename_json); - item[item_index].put("fd", fd_json); - item_index++; - } - - item[count].put("filename", filename); - item[count].put("fd", fd); - - boost::property_tree::ptree items_write; - for (int i = 0; i <= count; i++) { - items_write.push_back(std::make_pair("", item[i])); - } - - root.put("port", g_port); - root.put_child("volumes", items_write); - - try { - boost::property_tree::write_json(metadata_file, root); - } catch (boost::property_tree::ptree_error pt) { - LOG(ERROR) << "write json throw an exception. " << pt.what(); - return -1; - } - LOG(NOTICE) << "generate fd end. " << filename << ", " << fd; - return 0; -} - -// 从持久化文件删除fd相关信息 -int RmFd(int fd) { - std::string metadata_file = GetUuidFile(); - boost::property_tree::ptree root, items; - - try { - boost::property_tree::read_json( - metadata_file, root); - } catch (boost::property_tree::ptree_error pt) { - LOG(ERROR) << "read json throw an exception. " << pt.what(); - return -1; - } - - auto volumes_exist = root.get_child_optional("volumes"); - if (!volumes_exist) { - LOG(ERROR) << "rm fd failed, no volume is exist. fd is: " << fd; - return -1; - } - - items = root.get_child("volumes"); - - std::vector vector_items; - - bool FdExist = false; - for (boost::property_tree::ptree::iterator it = items.begin(); - it != items.end(); ++it) { - std::string filename = it->second.get("filename"); - int fd_tmp = atoi(it->second.get("fd").c_str()); - if (fd == fd_tmp) { - FdExist = true; - continue; - } - - boost::property_tree::ptree item_tmp; - item_tmp.put("filename", filename); - item_tmp.put("fd", fd_tmp); - - vector_items.push_back(item_tmp); - } - if (!FdExist) { - LOG(ERROR) << "rm fd failed, fd is not exist. fd is: " << fd; - return -1; - } - - boost::property_tree::ptree items_write; - for (std::vector::iterator iter = - vector_items.begin(); - iter != vector_items.end(); iter++) { - items_write.push_back(std::make_pair("", *iter)); - } - - root.put("port", g_port); - root.put_child("volumes", items_write); - - try { - boost::property_tree::write_json(metadata_file, root); - } catch (boost::property_tree::ptree_error pt) { - LOG(ERROR) << "write json throw an exception. " << pt.what(); - return -1; - } - - LOG(NOTICE) << "rm fd success. " << fd; - return 0; -} - -void Squeeze(char s[], char c) { - int i, j; - for (i = 0, j = 0; s[i] != '\0'; i++) { - if (s[i] != c) { - s[j++] = s[i]; - } - } - s[j] = '\0'; -} - -// 获得mon host -std::string GetMonHost(const std::string& str) { - char* mon_host_out = NULL; - std::vector result; - if (*str.begin() == ':' || *str.end() == ':') { - LOG(ERROR) << "wrong format. " << str; - std::string mon_host; - return mon_host; - } - - auto sub_begin = str.begin(); - auto sub_end = str.begin(); - for (auto it = str.begin(); it != str.end(); ++it, ++sub_end) { - if (*it == ':' && std::isalpha(*(it + 1))) { - result.push_back(std::string(sub_begin, sub_end)); - sub_begin = it + 1; - } - } - result.push_back(std::string(sub_begin, sub_end)); - - for (auto piece : result) { - if (strstr(piece.c_str(), "mon_host") != NULL) { - std::vector mon_host = split(piece, "="); - mon_host_out = const_cast(mon_host[1].c_str()); - Squeeze(mon_host_out, '\\'); - LOG(ERROR) << "mon host is. " << mon_host_out; - } - } - return mon_host_out; -} - -// 字符串拆分，pattern为分隔符 -std::vector split(const std::string& str, - const std::string& pattern) { - std::vector res; - if (str == "") return res; - std::string strs = str + pattern; - size_t pos = strs.find(pattern); - - while (pos != strs.npos) { - std::string temp = strs.substr(0, pos); - res.push_back(temp); - strs = strs.substr(pos + 1, strs.size()); - pos = strs.find(pattern); - } - - return res; -} - -// 找到可用的port -int FindPort(config_t* cfg, int port) { - LOG(NOTICE) << "find port start."; - - struct dirent* ent; - std::set port_inuse; - - DIR* proc = opendir(g_filePath); - if (proc == NULL) { - LOG(ERROR) << "opendir file_path is error"; - return -1; - } - - while ((ent = readdir(proc))) { - if ((ent->d_name[0] == '.' && ent->d_name[1] == '\0') || - (ent->d_name[1] == '.' && ent->d_name[2] == '\0')) - continue; - std::string tmp = "/tmp/"; - std::string ent_name = ent->d_name; - std::string filepath_tmp = g_filePath; - std::string uuid_file_tmp = filepath_tmp + "/" + ent_name; - std::string uuid_lockfile_tmp = tmp + ent_name; - char* uuid_lockfile = const_cast(uuid_lockfile_tmp.c_str()); - char* uuid_file = const_cast(uuid_file_tmp.c_str()); - int lockfd = LockFile(uuid_lockfile); - if (lockfd < 0) { - closedir(proc); - LOG(ERROR) << "lock file failed." << uuid_lockfile; - return -1; - } - - boost::property_tree::ptree root; - boost::property_tree::ptree items; - int port; - - try { - boost::property_tree::read_json( - uuid_file, root); - auto port_exist = root.get_child_optional("port"); - if (!port_exist) { - LOG(ERROR) << "port is not exist."; - UnlockFile(lockfd); - continue; - } - port = root.get("port", 0); - } catch (boost::property_tree::ptree_error pt) { - LOG(ERROR) << "read json throw an exception. " << pt.what(); - UnlockFile(lockfd); - continue; - } - - port_inuse.insert(port); - - UnlockFile(lockfd); - } - - closedir(proc); - - port_option_t* port_option = new port_option_t; - ReadPort(cfg, port_option); - int min_port, max_port; - min_port = port_option->min_port; - max_port = port_option->max_port; - delete port_option; - int start_port; - if (port == 0) { - start_port = min_port; - } else { - start_port = port; - } - - for (int i = start_port; i < max_port; i++) { - if (!(port_inuse.find(i) != port_inuse.end())) { - LOG(ERROR) << "port is not inuse, " << i; - return i; - } else { - LOG(NOTICE) << " port is inuse, " << i; - } - } - - LOG(NOTICE) << "find port end."; - return 0; -} - -std::string GetUuidFile() { - std::string uuid_file; - std::string file_path_tmp = g_filePath; - std::string g_uuid_tmp = g_uuid; - uuid_file = file_path_tmp + "/" + g_uuid_tmp; - return uuid_file; -} - -std::string GetUuidLockfile() { - std::string uuid_lockfile; - std::string g_uuid_tmp = g_uuid; - uuid_lockfile = "/tmp/" + g_uuid_tmp; - return uuid_lockfile; -} - -// 设置cpu亲和性 -int SetCpuAffinity(pid_t qemu) { - cpu_set_t mask; - - int r = sched_getaffinity(qemu, sizeof(cpu_set_t), &mask); - if (r < 0) { - LOG(NOTICE) << "getaffinity failed."; - return r; - } - r = sched_setaffinity(0, sizeof(cpu_set_t), &mask); - if (r < 0) { - LOG(NOTICE) << "setaffinity failed."; - return r; - } - LOG(NOTICE) << "setaffinity success. " << qemu; - return 0; -} - -int CloseParentFd() { - pid_t pid = getpid(); - std::string pid_str = std::to_string(pid); - std::string proc_str = "/proc/"; - std::string fd_str = "/fd"; - std::string pid_proc = proc_str + pid_str + fd_str; - - DIR* dir = opendir(pid_proc.c_str()); - struct dirent* ent; - if (dir == NULL) { - return -1; - } - while ((ent = readdir(dir))) { - if ((ent->d_name[0] == '.' && ent->d_name[1] == '\0') || - (ent->d_name[1] == '.' && ent->d_name[2] == '\0')) - continue; - int fd = atoi(ent->d_name); - if (fd < 3) - continue; - int ret = -1; - ret = close(fd); - if (ret < 0) { - return -1; - } - } - return 0; -} - -// 初始化 -int Init() { - // 初始化配置 - config_t* cfg = InitConfig(); - if (cfg == NULL) { - char buffer[128]; - const char* err = "init config fialed."; - snprintf(buffer, sizeof(buffer), - "echo %s>>/var/log/nebd/nebd-server-initfailed.log", err); - ::system(buffer); - return -1; - } - std::string file_path_tmp = ReadUuidPath(cfg); - g_filePath = file_path_tmp.c_str(); - - std::string log_path = ReadLogPath(cfg); - std::string log_file = log_path + "/" + g_uuid + ".log"; - // 初始化日志模块 - logging::LoggingSettings log; - log.logging_dest = logging::LOG_TO_FILE; - log.log_file = log_file.c_str(); - logging::InitLogging(log); - int loglevel = ReadLogLevel(cfg); - logging::SetMinLogLevel(loglevel); - if (NULL == opendir(g_filePath)) { - if (mkdir(g_filePath, S_IRWXU | S_IRWXG) < 0) { - LOG(ERROR) << "mkdir failed."; - return -1; - } - } - int ret = CheckProc(g_uuid); - if (ret < 0) { - LOG(ERROR) << "check proc failed."; - return -1; - } - - if (SetCpuAffinity(g_oldPid) < 0) { - LOG(ERROR) << "set cpu affinity failed."; - return -1; - } - - // rpc option - rpc_option_t* rpc_option = new rpc_option_t; - ReadRpcOption(cfg, rpc_option); - brpc::ServerOptions options; - options.idle_timeout_sec = rpc_option->brpc_idle_timeout; - options.num_threads = rpc_option->brpc_num_threads; - delete rpc_option; - QemuClientServiceImpl* qemuclient_service_impl = new QemuClientServiceImpl; - if (g_server.AddService(qemuclient_service_impl, - brpc::SERVER_OWNS_SERVICE) != 0) { - LOG(ERROR) << "Fail to add service"; - return -1; - } - - bool new_port = false; - std::string uuid_file = GetUuidFile(); - std::string uuid_lockfile = GetUuidLockfile(); - - int lock_port_fd; - int lockfd = LockFile(uuid_lockfile.c_str()); - if (lockfd < 0) { - LOG(ERROR) << "lock file failed."; - return -1; - } - - // 如果持久化文件存在，则reload - if (access(uuid_file.c_str(), F_OK) != -1) { - int ret = Reload(); - UnlockFile(lockfd); - if (ret <= 0) { - LOG(ERROR) << "file is exist, but port is not exist."; - return -1; - } else { - g_port = ret; - } - } else { - UnlockFile(lockfd); - std::string lock_port_file = ReadLockPortFile(cfg); - lock_port_fd = LockFile(const_cast(lock_port_file.c_str())); - if (lock_port_fd < 0) { - LOG(ERROR) << "add port file lock failed."; - return -1; - } - g_port = FindPort(cfg, 0); - new_port = true; - } - LOG(NOTICE) << "port is: " << g_port; - // 启动brpc server，重试retry_counts次 - int retry = 0; - int retry_counts = ReadRetryCounts(cfg); - std::string ip = "127.0.0.1:"; - while (++retry <= retry_counts) { - std::string ip_port = ip + std::to_string(g_port); - if (g_server.Start(ip_port.c_str(), &options) != 0) { - LOG(ERROR) << "Fail to start QemuClientServer, retry time is: " - << retry; - g_port = FindPort(cfg, g_port + 1); - } else { - LOG(ERROR) << "start QemuClientServer success."; - break; - } - } - if (retry >= retry_counts) { - LOG(ERROR) << "Fail to start QemuClientServer."; - return -1; - } - // 如果是新启动的，则保存进持久化文件 - if (new_port) { - lockfd = LockFile(uuid_lockfile.c_str()); - if (lockfd < 0) { - LOG(ERROR) << "add lock failed."; - return -1; - } - - int ret; - ret = GeneratePort(g_port); - if (ret < 0) { - LOG(ERROR) << "generate port failed."; - return -1; - } - UnlockFile(lockfd); - UnlockFile(lock_port_fd); - } - config_destroy(cfg); - // 启动心跳线程 - bthread_t th; - if (bthread_start_background(&th, NULL, HeartbeatProcess, NULL) != 0) { - LOG(ERROR) << "Fail to create bthread"; - return -1; - } - // 响应SIGINT以及SIGTERM - signal(SIGTERM, SigProcess); - signal(SIGINT, SigProcess); - signal(SIGHUP, SigLogReplace); - bthread_join(th, NULL); - return 0; -} diff --git a/src/part2/common_type.h b/src/part2/common_type.h deleted file mode 100644 index f8f365479c..0000000000 --- a/src/part2/common_type.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Project: nebd - * File Created: 2019-09-30 - * Author: hzwuhongsong - * Copyright (c) 2019 NetEase - */ - -#ifndef SRC_PART2_COMMON_TYPE_H_ -#define SRC_PART2_COMMON_TYPE_H_ - -#include -#include -#include -#include -#include -#include -#include -#include - -// brpc server -extern brpc::Server g_server; - -typedef struct FdImage { - rados_t* cluster; - rados_ioctx_t* io; - rbd_image_t* image; - char* filename; -} FdImage_t; -// 主键为fd -extern std::map g_imageMap; // fd的内存镜像 -// ceph的fd取值范围为[1，1000]，curve取值范围为[1001, 20000] -#define FD_CEPH_MAX 1000 -#define FD_CURVE_MAX 20000 - -int CloseParentFd(); -int LockFile(const char* file); -void UnlockFile(int lockfd); -std::vector split(const std::string& str, - const std::string& pattern); -int ReadQemuXmls(); -void ReadQemuXml(const char*); -int GenerateFd(char* filename, int fd); -int GeneratePort(int port); -int FindPort(config_t* cfg, int port); -int RmFd(int fd); -std::string GetUuidFile(); -std::string GetUuidLockfile(); -int SetCpuAffinity(pid_t qemu); -std::string GetMonHost(const std::string &str); -int Init(); -void Squeeze(char s[], char c); -// qemu达[秾K潚~Duuid -extern char* g_uuid; -// qemu达[秾K潚~Dpid -extern int64_t g_oldPid; -// 该nebd-server秾K幾O使潔¨潚~D端住£住· -extern int g_port; -// 记弾U请氾B录°轇~O -extern std::atomic_long requestForRpcWrite; -extern std::atomic_long requestForCephWrite; -extern std::atomic_long requestForRpcWriteCounts; -// uuid彌~A举E佌~V彖~G件嬾X彔 -extern const char* g_filePath; - -// 侾]嬾X乾Q主彜º弾S佉~M彌~B载潚~D位·信彁¯ -extern std::vector g_qemuPoolVolumes; - -#endif // SRC_PART2_COMMON_TYPE_H_ diff --git a/src/part2/config.cpp b/src/part2/config.cpp deleted file mode 100644 index fd30103acb..0000000000 --- a/src/part2/config.cpp +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Project: nebd - * File Created: 2019-09-30 - * Author: hzwuhongsong - * Copyright (c) 2019 NetEase - */ - -#include -#include -#include -#include -#include "src/part2/config.h" -#include "src/part2/common_type.h" - -#define NEBD_CONF_FILE_DEFAULT "/etc/nebd/nebd-server.conf" -#define LOG_CONF_FILE_DEFAULT "/var/log/nebd/" -#define UUID_CONF_FILE_DEFAULT "/var/run/nebd-server/" -#define QEMU_XML_DIR_DEFAULT "/var/run/libvirt/qemu/" -#define LOCKPORT_CONF_FILE_DEFAULT "/tmp/nebd-server.port.file.lock" -#define NEBD_CEPH_CONF "/etc/ceph/ceph.conf" -#define BRPC_NUM_THREADS 2 -#define BRPC_IDLE_TIMEOUT -1 -#define BRPC_MAX_CONCURRENCY 0 -#define BRPC_METHOD_MAX_CONCURRENCY 0 -#define LOGLEVEL 1 -#define MIN_PORT 6200 -#define MAX_PORT 6500 -#define RETRY_COUNTS 3 -#define HEARTBEAT_INTERVAL 1 -#define CHECK_ASSERT_TIMES 10 -#define CHECHK_DETACHED_INTERVAL 5 -#define CHECK_DETACHED_TIME 6 - -config_t* InitConfig() { - config_t* cfg = new config_t; - config_init(cfg); - - // Read the file. If there is an error, return NUll. - if (!config_read_file(cfg, NEBD_CONF_FILE_DEFAULT)) { - char buffer[128]; - const char *err = "read config file failed."; - snprintf(buffer, sizeof(buffer), - "echo %s >> /var/log/nebd/nebd-server-initfailed.log", err); - ::system(buffer); - config_destroy(cfg); - return NULL; - } - return cfg; -} - -void ReadRpcOption(config_t* cfg, rpc_option_t* rpc_option) { - if (!config_lookup_int(cfg, "brpc_num_threads", - &rpc_option->brpc_num_threads)) - rpc_option->brpc_num_threads = BRPC_NUM_THREADS; - - if (!config_lookup_int(cfg, "brpc_idle_timeout", - &rpc_option->brpc_idle_timeout)) - rpc_option->brpc_idle_timeout = BRPC_IDLE_TIMEOUT; - - if (!config_lookup_int(cfg, "brpc_max_concurrency", - &rpc_option->brpc_max_concurrency)) - rpc_option->brpc_max_concurrency = BRPC_MAX_CONCURRENCY; - - if (!config_lookup_int(cfg, "brpc_methodmax_concurrency", - &rpc_option->brpc_method_max_concurrency)) - rpc_option->brpc_method_max_concurrency = BRPC_METHOD_MAX_CONCURRENCY; - return; -} - -void ReadPort(config_t* cfg, port_option_t* port_option) { - if (!config_lookup_int(cfg, "min_port", &port_option->min_port)) - port_option->min_port = MIN_PORT; - - if (!config_lookup_int(cfg, "max_port", &port_option->max_port)) - port_option->max_port = MAX_PORT; - - return; -} - -int ReadLogLevel(config_t* cfg) { - int loglevel; - if (!config_lookup_int(cfg, "loglevel", &loglevel)) loglevel = LOGLEVEL; - - return loglevel; -} - -std::string ReadUuidPath(config_t* cfg) { - std::string uuid_path; - const char* uuid_path_tmp; - - if (!config_lookup_string(cfg, "uuid_file_path", &uuid_path_tmp)) - uuid_path_tmp = UUID_CONF_FILE_DEFAULT; - - uuid_path = uuid_path_tmp; - return uuid_path; -} - -int ReadDetachedTimes(config_t* cfg) { - int detached_times; - if (!config_lookup_int(cfg, "detached_times", &detached_times)) - detached_times = CHECK_DETACHED_TIME; - return detached_times; -} - -int ReadRetryCounts(config_t* cfg) { - int retry_counts; - if (!config_lookup_int(cfg, "retry_counts", &retry_counts)) - retry_counts = RETRY_COUNTS; - - return retry_counts; -} - -std::string ReadLockPortFile(config_t* cfg) { - std::string lock_port_file; - const char* lock_port_file_tmp; - - if (!config_lookup_string(cfg, "lock_port_file", &lock_port_file_tmp)) - lock_port_file_tmp = LOCKPORT_CONF_FILE_DEFAULT; - - lock_port_file = lock_port_file_tmp; - return lock_port_file; -} - -std::string ReadCephConf(config_t* cfg) { - std::string ceph_conf; - const char* ceph_conf_tmp; - - if (!config_lookup_string(cfg, "nebd_ceph_conf", &ceph_conf_tmp)) - ceph_conf_tmp = NEBD_CEPH_CONF; - - ceph_conf = ceph_conf_tmp; - return ceph_conf; -} - -std::string ReadLogPath(config_t* cfg) { - std::string log_path; - const char* log_path_tmp; - if (!config_lookup_string(cfg, "log_path", &log_path_tmp)) - log_path_tmp = LOG_CONF_FILE_DEFAULT; - - log_path = log_path_tmp; - return log_path; -} - -std::string ReadQemuXmlDir(config_t* cfg) { - std::string qemu_dir; - const char* qemu_dir_tmp; - if (!config_lookup_string(cfg, "qemu_xml_dir", &qemu_dir_tmp)) - qemu_dir_tmp = QEMU_XML_DIR_DEFAULT; - - qemu_dir = qemu_dir_tmp; - return qemu_dir; -} - -int ReadHeartbeatInterval(config_t* cfg) { - int interval; - if (!config_lookup_int(cfg, "heartbeat_interval", &interval)) - interval = HEARTBEAT_INTERVAL; - - return interval; -} - -int ReadAssertTimes(config_t* cfg) { - int check_assert_times; - if (!config_lookup_int(cfg, "check_assert_times", &check_assert_times)) - check_assert_times = CHECK_ASSERT_TIMES; - - return check_assert_times; -} - -int ReadCheckDetachedInterval(config_t* cfg) { - int interval; - if (!config_lookup_int(cfg, "check_detached_interval", &interval)) - interval = CHECHK_DETACHED_INTERVAL; - - return interval; -} - diff --git a/src/part2/config.h b/src/part2/config.h deleted file mode 100644 index 63a442abc3..0000000000 --- a/src/part2/config.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Project: nebd - * File Created: 2019-09-30 - * Author: hzwuhongsong - * Copyright (c) 2019 NetEase - */ - -#ifndef SRC_PART2_CONFIG_H_ -#define SRC_PART2_CONFIG_H_ - -#include -#include - -/** - * @brief 初始化配置 - */ -config_t* InitConfig(); - -/** - * @brief rpc相关配置 - */ -typedef struct rpc_option { - int brpc_num_threads; // brpc默认线程数 - int brpc_idle_timeout; - int brpc_max_concurrency; - int brpc_method_max_concurrency; -} rpc_option_t; -void ReadRpcOption(config_t* cfg, rpc_option_t* rpc_option); - -/** - * @brief port相关配置 - */ -typedef struct port_option { - int min_port; - int max_port; -} port_option_t; -void ReadPort(config_t* cfg, port_option_t* port_option); -// 或取日志级别 -int ReadLogLevel(config_t* cfg); -// 获取持久化文件路径 -std::string ReadUuidPath(config_t* cfg); -// 获取启动brpc server的重试次数 -int ReadRetryCounts(config_t* cfg); -// 获取qemu xml文件的路径 -std::string ReadQemuXmlDir(config_t* cfg); -// 获取心跳间隔 -int ReadHeartbeatInterval(config_t* cfg); -// 获取心跳检查次数，超过该次数未检查到，则进程自杀 -int ReadAssertTimes(config_t* cfg); -// 获取日志文件路径 -std::string ReadLogPath(config_t* cfg); -// 获取检查detach卷的间隔 -int ReadCheckDetachedInterval(config_t* cfg); -// 获取lock port文件 -std::string ReadLockPortFile(config_t* cfg); -// 获取ceph配置文件路径 -std::string ReadCephConf(config_t* cfg); -// 获取detach卷检查次数 -int ReadDetachedTimes(config_t* cfg); -#endif // SRC_PART2_CONFIG_H_ diff --git a/src/part2/define.h b/src/part2/define.h new file mode 100644 index 0000000000..c8fad264cd --- /dev/null +++ b/src/part2/define.h @@ -0,0 +1,67 @@ +#ifndef SRC_PART2_DEFINE_H_ +#define SRC_PART2_DEFINE_H_ + +#include +#include + +#include "src/common/rw_lock.h" + +using nebd::common::RWLock; +using ::google::protobuf::Message; +using ::google::protobuf::Closure; + +namespace nebd { +namespace server { + +// nebd异步请求的类型 +enum class LIBAIO_OP { + LIBAIO_OP_READ, + LIBAIO_OP_WRITE, + LIBAIO_OP_DISCARD, + LIBAIO_OP_FLUSH, +}; + +enum class NebdFileStatus { + OPENED = 0, + CLOSED = 1, +}; + +enum class NebdFileType { + CEPH = 0, + CURVE = 1, +}; + +class NebdFileInstance; +class NebdRequestExecutor; + +struct NebdFileInfo { + int fd; + std::string fileName; + NebdFileType type; + NebdFileStatus status; + uint64_t timeStamp; + RWLock rwLock; + NebdFileInstance* fileInstance; + NebdRequestExecutor* executor; +}; + +struct NebdServerAioContext; + +// nebd回调函数的类型 +typedef void (*NebdAioCallBack)(struct NebdServerAioContext* context); + +typedef struct NebdServerAioContext { + off_t offset; // 请求的offset + size_t length; // 请求的length + int ret; // 记录异步返回的返回值 + LIBAIO_OP op; // 异步请求的类型，详见定义 + NebdAioCallBack cb; // 异步请求的回调函数 + void* buf; // 请求的buf + Message* response; // 请求返回内容 + Closure *done; // 请求回调函数 +} ClientAioContext; + +} // namespace server +} // namespace nebd + +#endif // SRC_PART2_DEFINE_H_ diff --git a/src/part2/file_manager.cpp b/src/part2/file_manager.cpp new file mode 100644 index 0000000000..d5bd0d159e --- /dev/null +++ b/src/part2/file_manager.cpp @@ -0,0 +1,87 @@ +#include "src/part2/file_manager.h" + +namespace nebd { +namespace server { + +NebdFileManager::NebdFileManager() + : metaFileManager_(nullptr) + , heartbeatTimeoutS_(10) { + +} + +NebdFileManager::~NebdFileManager() { + // TODO +} + +int NebdFileManager::Init(NebdFileManagerOption option) { + // TODO + return 0; +} + +int NebdFileManager::Load() { + // TODO + return 0; +} + +int NebdFileManager::UpdateFileTimestamp(int fd) { + // TODO + return 0; +} + +int NebdFileManager::Open(const std::string& filename) { + // TODO + return 0; +} + +int NebdFileManager::Close(int fd) { + // TODO + return 0; +} + +int NebdFileManager::Extend(int fd, int64_t newsize) { + // TODO + return 0; +} + +int NebdFileManager::StatFile(int fd) { + // TODO + return 0; +} + +int NebdFileManager::Discard(int fd, NebdServerAioContext* aioctx) { + // TODO + return 0; +} + +int NebdFileManager::AioRead(int fd, NebdServerAioContext* aioctx) { + // TODO + return 0; +} + +int NebdFileManager::AioWrite(int fd, NebdServerAioContext* aioctx) { + // TODO + return 0; +} + +int NebdFileManager::Flush(int fd, NebdServerAioContext* aioctx) { + // TODO + return 0; +} + +int NebdFileManager::GetInfo(int fd) { + // TODO + return 0; +} + +int NebdFileManager::InvalidCache(int fd) { + // TODO + return 0; +} + +void NebdFileManager::CheckTimeoutFunc() { + // TODO +} + +} // namespace server +} // namespace nebd + diff --git a/src/part2/file_manager.h b/src/part2/file_manager.h new file mode 100644 index 0000000000..22f52199e7 --- /dev/null +++ b/src/part2/file_manager.h @@ -0,0 +1,54 @@ +#ifndef SRC_PART2_FILE_MANAGER_H_ +#define SRC_PART2_FILE_MANAGER_H_ + +#include +#include +#include // NOLINT +#include + +#include "src/part2/define.h" +#include "src/part2/metafile_manager.h" + +namespace nebd { +namespace server { + +using MetaFileManagerPtr = std::shared_ptr; + +struct NebdFileManagerOption { + uint32_t heartbeatTimeoutS; + MetaFileManagerPtr metaFileManager; +}; + +class NebdFileManager { + public: + NebdFileManager(); + virtual ~NebdFileManager(); + virtual int Init(NebdFileManagerOption option); + virtual int Load(); + virtual int UpdateFileTimestamp(int fd); + virtual int Open(const std::string& filename); + virtual int Close(int fd); + virtual int Extend(int fd, int64_t newsize); + virtual int StatFile(int fd); + virtual int Discard(int fd, NebdServerAioContext* aioctx); + virtual int AioRead(int fd, NebdServerAioContext* aioctx); + virtual int AioWrite(int fd, NebdServerAioContext* aioctx); + virtual int Flush(int fd, NebdServerAioContext* aioctx); + virtual int GetInfo(int fd); + virtual int InvalidCache(int fd); + + private: + void CheckTimeoutFunc(); + + private: + using FileInfoMap = std::unordered_map>; + FileInfoMap fileInfoMap_; + MetaFileManagerPtr metaFileManager_; + uint32_t heartbeatTimeoutS_; + std::thread checkTimeoutThread_; +}; + +} // namespace server +} // namespace nebd + +#endif // SRC_PART2_FILE_MANAGER_H_ diff --git a/src/part2/file_service.cpp b/src/part2/file_service.cpp new file mode 100644 index 0000000000..accab6a15f --- /dev/null +++ b/src/part2/file_service.cpp @@ -0,0 +1,79 @@ +#include "src/part2/file_service.h" + +namespace nebd { +namespace server { + +void NebdFileServiceImpl::OpenFile( + google::protobuf::RpcController* cntl_base, + const nebd::client::OpenFileRequest* request, + nebd::client::OpenFileResponse* response, google::protobuf::Closure* done) { + // TODO +} + +void NebdFileServiceImpl::Write(google::protobuf::RpcController* cntl_base, + const nebd::client::WriteRequest* request, + nebd::client::WriteResponse* response, + google::protobuf::Closure* done) { + // TODO +} + +void NebdFileServiceImpl::Read(google::protobuf::RpcController* cntl_base, + const nebd::client::ReadRequest* request, + nebd::client::ReadResponse* response, + google::protobuf::Closure* done) { + // TODO +} + +void NebdFileServiceImpl::StatFile( + google::protobuf::RpcController* cntl_base, + const nebd::client::StatFileRequest* request, + nebd::client::StatFileResponse* response, google::protobuf::Closure* done) { + // TODO +} + +void NebdFileServiceImpl::GetInfo(google::protobuf::RpcController* cntl_base, + const nebd::client::GetInfoRequest* request, + nebd::client::GetInfoResponse* response, + google::protobuf::Closure* done) { + // TODO +} + +void NebdFileServiceImpl::Flush(google::protobuf::RpcController* cntl_base, + const nebd::client::FlushRequest* request, + nebd::client::FlushResponse* response, + google::protobuf::Closure* done) { + // TODO +} + +void NebdFileServiceImpl::CloseFile( + google::protobuf::RpcController* cntl_base, + const nebd::client::CloseFileRequest* request, + nebd::client::CloseFileResponse* response, + google::protobuf::Closure* done) { + // TODO +} + +void NebdFileServiceImpl::Discard(google::protobuf::RpcController* cntl_base, + const nebd::client::DiscardRequest* request, + nebd::client::DiscardResponse* response, + google::protobuf::Closure* done) { + // TODO +} + +void NebdFileServiceImpl::ResizeFile( + google::protobuf::RpcController* cntl_base, + const nebd::client::ResizeRequest* request, + nebd::client::ResizeResponse* response, google::protobuf::Closure* done) { + // TODO +} + +void NebdFileServiceImpl::InvalidateCache( + google::protobuf::RpcController* cntl_base, + const nebd::client::InvalidateCacheRequest* request, + nebd::client::InvalidateCacheResponse* response, + google::protobuf::Closure* done) { + // TODO +} + +} // namespace server +} // namespace nebd diff --git a/src/part2/rpc_server.h b/src/part2/file_service.h similarity index 82% rename from src/part2/rpc_server.h rename to src/part2/file_service.h index 87fd097312..be65748377 100644 --- a/src/part2/rpc_server.h +++ b/src/part2/file_service.h @@ -1,25 +1,25 @@ -/* - * Project: nebd - * File Created: 2019-09-30 - * Author: hzwuhongsong - * Copyright (c) 2019 NetEase - */ - -#ifndef SRC_PART2_RPC_SERVER_H_ -#define SRC_PART2_RPC_SERVER_H_ +#ifndef SRC_PART2_FILE_SERVICE_H_ +#define SRC_PART2_FILE_SERVICE_H_ #include #include -#include #include #include +#include + #include "src/common/client.pb.h" -#include "src/part2/rpc_ceph.h" +#include "src/part2/file_manager.h" -class QemuClientServiceImpl : public nebd::client::QemuClientService { +namespace nebd { +namespace server { + +class NebdFileServiceImpl : public nebd::client::NebdFileService { public: - QemuClientServiceImpl() {request_ceph = new RpcRequestCeph;} - virtual ~QemuClientServiceImpl() {delete request_ceph;} + explicit NebdFileServiceImpl(std::shared_ptr fileManager) + : fileManager_(fileManager) {} + + virtual ~NebdFileServiceImpl() {} + virtual void OpenFile(google::protobuf::RpcController* cntl_base, const nebd::client::OpenFileRequest* request, nebd::client::OpenFileResponse* response, @@ -70,11 +70,11 @@ class QemuClientServiceImpl : public nebd::client::QemuClientService { nebd::client::InvalidateCacheResponse* response, google::protobuf::Closure* done); - void SetRpcRequest(RpcRequestCeph *request) {request_ceph = request;} - private: - RpcRequestCeph* request_ceph; - std::vector opening_image; + std::shared_ptr fileManager_; }; -#endif // SRC_PART2_RPC_SERVER_H_ +} // namespace server +} // namespace nebd + +#endif // SRC_PART2_FILE_SERVICE_H_ diff --git a/src/part2/heartbeat.cpp b/src/part2/heartbeat.cpp deleted file mode 100644 index 8c530ee991..0000000000 --- a/src/part2/heartbeat.cpp +++ /dev/null @@ -1,350 +0,0 @@ -/* - * Project: nebd - * File Created: 2019-09-30 - * Author: hzwuhongsong - * Copyright (c) 2019 NetEase - */ - -#include "src/part2/heartbeat.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/part2/common_type.h" -#include "src/part2/config.h" -#include "src/part2/interrupt_sleep.h" -#include "src/part2/rados_interface.h" - -static int g_check_times = 0; -static int g_check_detached_times = 0; -static const char* g_proc_name = "qemu-system-x86_64"; -static int64_t g_last_received(0); -static bool heartbeat_check_first = true; -static InterruptibleSleeper sleeper; -/** - *@brief 关闭已卸载卷线程 - *@detail - *若有已卸载卷，那么检查detacheedTimes次后，卸载 - */ -void* CloseDetachedVolumesProcess(void*) { - LOG(NOTICE) << "close detached volumes process start."; - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return NULL; - } - int checkdetached_interval = ReadCheckDetachedInterval(cfg); - int detachedTimes = ReadDetachedTimes(cfg); - config_destroy(cfg); - sleep(checkdetached_interval); - CloseQemuDetachedVolumes(detachedTimes); - LOG(NOTICE) << "close detached volumes process end."; - return NULL; -} -/** - * @brief 心跳线程 - * @detail - * 每隔heartbeat_interval秒检查一次qemu进程， - * 若assert_times检查后，qemu进程依然不在，那么自杀 - */ -void* HeartbeatProcess(void*) { - int ret = 0; - config_t* cfg = InitConfig(); - if (cfg == NULL) { - return NULL; - } - int heartbeat_interval = ReadHeartbeatInterval(cfg); - int check_assert_times = ReadAssertTimes(cfg); - config_destroy(cfg); - while (sleeper.wait_for( - std::chrono::milliseconds(1000 * heartbeat_interval))) { - ret = CheckProc(g_uuid); - if (ret < 0) { - if (++g_check_times < check_assert_times) { - LOG(ERROR) << "donnt stop, check_time is: " << g_check_times - << ", " - << "check_assert_times is: " << check_assert_times; - continue; - } - LOG(ERROR) << "qemu proc is stop, so stop myself"; - StopProc(false); - } else { - g_check_times = 0; - if (requestForRpcWriteCounts > 0) { - int64_t last_received; - last_received = requestForRpcWriteCounts - g_last_received; - g_last_received = requestForRpcWriteCounts; - LOG(NOTICE) << "qemu proc is running, " - << "rpc inflight: " << requestForRpcWrite - << ", ceph inflight: " << requestForCephWrite - << ", last received: " << last_received; - } else { - requestForRpcWriteCounts = 0; - g_last_received = 0; - LOG(NOTICE) << "rpc write counts is zero or has overflowed."; - } - } - } - LOG(ERROR) << "the heartbeat is over."; - return NULL; -} - -/** - * @brief 响应SIGINT以及SIGTERM信号，客户端升级使用 - */ -void SigProcess(int sig_no) { - LOG(ERROR) << "signal has been received, " << sig_no; - StopProc(true); -} - -/** - * @brief 响应SIGHUP信号，日志回滚使用 - */ -void SigLogReplace(int sig_no) { - LOG(WARNING) << "signal has been received, " << sig_no; - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - std::string log_path = ReadLogPath(cfg); - std::string log_file = log_path + "/" + g_uuid + ".log"; - logging::LoggingSettings log; - log.logging_dest = logging::LOG_TO_FILE; - log.log_file = log_file.c_str(); - logging::InitLogging(log); - int loglevel = ReadLogLevel(cfg); - logging::SetMinLogLevel(loglevel); - - LOG(WARNING) << "log has been logrotated."; - config_destroy(cfg); -} - -int StopProc(bool need_check_qemu_proc) { - bool remove_file = true; - std::string uuid_file = GetUuidFile(); - std::string uuid_lockfile = GetUuidLockfile(); - - g_server.Stop(0); - g_server.Join(); - int lockfd = LockFile(uuid_lockfile.c_str()); - if (lockfd < 0) { - LOG(ERROR) << "add lock failed."; - return -1; - } - - std::map::iterator iter; - for (iter = g_imageMap.begin(); iter != g_imageMap.end(); iter++) { - if (iter->first <= 0 || iter->first > FD_CEPH_MAX) { - LOG(ERROR) << "fd is abnormal. " << iter->first; - continue; - } - int ret = CloseImage(iter->first); - if (ret < 0) { - LOG(ERROR) << "close image failed."; - UnlockFile(lockfd); - return -1; - } - } - - if (need_check_qemu_proc) { - int ret = CheckProc(g_uuid); - if (ret >= 0) remove_file = false; - } - - if (remove_file) { - if (remove(uuid_file.c_str()) == 0) { - LOG(ERROR) << "remove uuid file suc. " << uuid_file; - } else { - LOG(ERROR) << "remove uuid file failed. " << uuid_file; - } - } - UnlockFile(lockfd); - - sleeper.interrupt(); - return 0; -} - -int CloseQemuDetachedVolumes(int detachedTimes) { - LOG(NOTICE) << "close detached volumes start."; - if (ReadQemuXmls() < 0) { - LOG(NOTICE) << "read qrmu xmls failed."; - return -1; - } - std::string uuid_lockfile = GetUuidLockfile(); - std::vector fd_pool_vols; - std::map>::iterator iter; - - // 遍历image_map，关闭所有qemu已经关闭的卷 - std::map::iterator iter_fd; - for (iter_fd = g_imageMap.begin(); iter_fd != g_imageMap.end(); iter_fd++) { - char* filename = iter_fd->second->filename; - std::vector split_firstly = split(filename, ":"); - if (split_firstly.empty()) continue; - std::string pool_vol; - pool_vol = split_firstly[1]; - if (pool_vol.empty()) continue; - - LOG(ERROR) << "change, pool and vol is: " << pool_vol; - fd_pool_vols.push_back(pool_vol); - std::vector::iterator ret; - ret = - find(g_qemuPoolVolumes.begin(), g_qemuPoolVolumes.end(), pool_vol); - if (ret == g_qemuPoolVolumes.end()) { - if (++g_check_detached_times < detachedTimes) { - LOG(ERROR) << "volume has not been found, " << pool_vol - << ", no close now, " << g_check_detached_times - << " less " << detachedTimes; - bthread_t th; - if (bthread_start_background( - &th, NULL, CloseDetachedVolumesProcess, NULL) != 0) { - LOG(ERROR) << "Fail to create bthread"; - return -1; - } - LOG(NOTICE) << "no volume has been detached."; - return 0; - } - LOG(ERROR) << "volume has not been found, detach now: " << pool_vol; - int lockfd = LockFile(uuid_lockfile.c_str()); - if (lockfd < 0) { - LOG(ERROR) << "add lock failed."; - return -1; - } - - if (CloseImage(iter_fd->first) < 0) { - LOG(ERROR) << "close image failed."; - UnlockFile(lockfd); - return -1; - } - if (RmFd(iter_fd->first) < 0) { - LOG(ERROR) << "rm fd failed."; - UnlockFile(lockfd); - return -1; - } - UnlockFile(lockfd); - } else { - LOG(NOTICE) << "volume has been found. " << pool_vol; - } - } - - // 本次检查结束，清空vector - g_qemuPoolVolumes.clear(); - LOG(NOTICE) << "close detached volumes end."; - return 0; -} - -char* SkipSpace(char* in, int* offset, int len) { - while (*offset < len) { - if (isspace(in[*offset]) || in[*offset] == '\0') { - ++*offset; - - continue; - } - break; - } - return in + *offset; -} - -int CheckCmdline(int64_t pid, const char* proc_name, char* uuid, int uuid_len) { - int i = 0; - char path[32], line[2048], *p; - - snprintf(path, sizeof(path), "/proc/%ld/cmdline", pid); - - int cmdline = open(path, O_RDONLY); - if (!cmdline) return -1; - - int nbytesread = read(cmdline, line, 2048); - close(cmdline); - - LOG(INFO) << "name is " << basename(line) << " " << proc_name << " " - << strlen(proc_name); - - if (strncmp(basename(line), proc_name, strlen(proc_name)) != 0) return -1; - - i += strlen(line); - - for (; i < nbytesread; i++) { - p = SkipSpace(line, &i, nbytesread); - - if (strncmp(p, "-uuid", 5) == 0) { - i += strlen(p); - p = SkipSpace(line, &i, nbytesread); - strncpy(uuid, p, uuid_len); - return 0; - } - i += strlen(p); - } - return -1; -} -/** - * @brief 检查qemu进程是否存在 - */ -int CheckProc(const char* uuid) { - int64_t pid; - char uuid_check[64]; - int r; - if (g_oldPid < 0) { - // get pid from proc name - DIR* proc = opendir("/proc"); - struct dirent* ent; - - if (proc == NULL) { - LOG(ERROR) << "open dir failed."; - return -1; - } - - while ((ent = readdir(proc))) { - if (!isdigit(*ent->d_name)) continue; - - pid = strtol(ent->d_name, NULL, 10); - - LOG(INFO) << "pid is " << pid; - - r = CheckCmdline(pid, g_proc_name, uuid_check, strlen(uuid)); - if (r >= 0) { - if (strncmp(uuid_check, uuid, strlen(uuid)) == 0) { - if (!heartbeat_check_first) { - LOG(NOTICE) << "pid change, close detached volumes."; - // 查看挂载的卷是否发生变化--卸载所有已变化的卷 - bthread_t th; - if (bthread_start_background( - &th, NULL, CloseDetachedVolumesProcess, NULL) != - 0) { - LOG(ERROR) << "Fail to create bthread"; - return -1; - } - // heartbeat_check_first = true; - } - LOG(NOTICE) << "qemu pid has changed, new pid is: " << pid; - g_oldPid = pid; - closedir(proc); - return 0; - } - } - } - - g_oldPid = -1; // reset old pid of checking proc - closedir(proc); - return -1; - } - // proc下面进程文件存在，则表示进程还存在 - char proc_file[32]; - snprintf(proc_file, sizeof(proc_file), "/proc/%ld", g_oldPid); - if ((access(proc_file, F_OK)) != -1) { - heartbeat_check_first = false; - return 0; - } - LOG(NOTICE) << "reset old pid."; - g_oldPid = -1; // reset old pid of checking proc - return -1; -} diff --git a/src/part2/heartbeat.h b/src/part2/heartbeat.h deleted file mode 100644 index bba0390ed4..0000000000 --- a/src/part2/heartbeat.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Project: nebd - * File Created: 2019-09-30 - * Author: hzwuhongsong - * Copyright (c) 2019 NetEase - */ - -#ifndef SRC_PART2_HEARTBEAT_H_ -#define SRC_PART2_HEARTBEAT_H_ -#include -/** - * @brief 心跳线程 - */ -void* HeartbeatProcess(void*); -/** - * @brief 响应SIGINT，SIGTERM信号 - */ -void SigProcess(int sig_no); -/** - * @brief 响应SIGHUP信号 - */ -void SigLogReplace(int sig_no); -/** - * @brief 关闭所有已经卸载的卷线程 - */ -void* CloseDetachedVolumesProcess(void*); -/** - * @brief 停止part2 - */ -int StopProc(bool need_check_qemu_proc); -/** - * @brief 关闭所有已经卸载的卷 - * @param detachedTimes 检查次数。检查次数超过其才会真的卸载卷 - */ -int CloseQemuDetachedVolumes(int detachedTimes); -/** - * @brief 检查qemu进程是否存在 - * @param uuid qemu进程对应的uuid - */ -int CheckProc(const char* uuid); -char* SkipSpace(char* in, int* offset, int len); -int CheckCmdline(int64_t pid, const char* proc_name, char* uuid, int uuid_len); -#endif // SRC_PART2_HEARTBEAT_H_ diff --git a/src/part2/heartbeat_service.cpp b/src/part2/heartbeat_service.cpp new file mode 100644 index 0000000000..bb05257512 --- /dev/null +++ b/src/part2/heartbeat_service.cpp @@ -0,0 +1,16 @@ +#include "src/part2/heartbeat_service.h" + +namespace nebd { +namespace server { + +void NebdHeartbeatServiceImpl::KeepAlive( + google::protobuf::RpcController* cntl_base, + const nebd::client::HeartbeatRequest* request, + nebd::client::HeartbeatResponse* response, + google::protobuf::Closure* done) { + // TODO +} + +} // namespace server +} // namespace nebd + diff --git a/src/part2/heartbeat_service.h b/src/part2/heartbeat_service.h new file mode 100644 index 0000000000..89d1af095f --- /dev/null +++ b/src/part2/heartbeat_service.h @@ -0,0 +1,30 @@ +#ifndef SRC_PART2_HEARTBEAT_SERVICE_H_ +#define SRC_PART2_HEARTBEAT_SERVICE_H_ + +#include + +#include "src/common/heartbeat.pb.h" +#include "src/part2/file_manager.h" + +namespace nebd { +namespace server { + +class NebdHeartbeatServiceImpl : public nebd::client::NebdHeartbeatService { + public: + explicit NebdHeartbeatServiceImpl( + std::shared_ptr fileManager) + : fileManager_(fileManager) {} + virtual ~NebdHeartbeatServiceImpl() {} + virtual void KeepAlive(google::protobuf::RpcController* cntl_base, + const nebd::client::HeartbeatRequest* request, + nebd::client::HeartbeatResponse* response, + google::protobuf::Closure* done); + + private: + std::shared_ptr fileManager_; +}; + +} // namespace server +} // namespace nebd + +#endif // SRC_PART2_HEARTBEAT_SERVICE_H_ diff --git a/src/part2/main.cpp b/src/part2/main.cpp index 8d17fd79f7..89d4b1c688 100644 --- a/src/part2/main.cpp +++ b/src/part2/main.cpp @@ -7,65 +7,8 @@ #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/part2/heartbeat.h" -#include "src/part2/rados_interface.h" -#include "src/part2/reload.h" -#include "src/part2/rpc_server.h" -#include "src/part2/config.h" -#include "src/common/client.pb.h" #define BOOST_SPIRIT_THREADSAFE int main(int argc, char* argv[]) { - if (CloseParentFd() < 0) { - return -1; - } - // 使进程为守护进程 - if (daemon(0, 0) < 0) { - char buffer[128]; - const char *err = "create daemon failed."; - snprintf(buffer, sizeof(buffer), - "echo %s >> /var/log/nebd/nebd-server-initfailed.log", err); - ::system(buffer); - return -1; - } - // 解析参数 - if (argc != 3) { - char buffer[128]; - const char *err = "Incorrect parameter counts."; - snprintf(buffer, sizeof(buffer), - "echo %s >> /var/log/nebd/nebd-server-initfailed.log", err); - ::system(buffer); - return -1; - } - if (strcmp(argv[1], "-uuid") != 0) { - char buffer[128]; - const char *err = "uuid is not given."; - snprintf(buffer, sizeof(buffer), - "echo %s >> /var/log/nebd/nebd-server-initfailed.log", err); - ::system(buffer); - return -1; - } - g_uuid = argv[2]; - int ret = Init(); - if (ret < 0) { - char buffer[128]; - const char *err = "init fialed."; - snprintf(buffer, sizeof(buffer), - "echo %s >> /var/log/nebd/nebd-server-initfailed.log", err); - ::system(buffer); - return -1; - } return 0; } diff --git a/src/part2/metafile_manager.cpp b/src/part2/metafile_manager.cpp new file mode 100644 index 0000000000..888978a748 --- /dev/null +++ b/src/part2/metafile_manager.cpp @@ -0,0 +1,29 @@ +#include "src/part2/metafile_manager.h" + +namespace nebd { +namespace server { + +NebdMetaFileManager::NebdMetaFileManager(const std::string& metaFilePath) + : metaFilePath_(metaFilePath) { + // TODO +} + +NebdMetaFileManager::~NebdMetaFileManager() {} + +int NebdMetaFileManager::RemoveFileInfo(const std::string& fileName) { + // TODO + return 0; +} + +int NebdMetaFileManager::UpdateFileInfo(const NebdFileInfo& fileInfo) { + // TODO + return 0; +} + +int NebdMetaFileManager::ListFileInfo(std::vector* fileInfos) { + // TODO + return 0; +} + +} // namespace server +} // namespace nebd \ No newline at end of file diff --git a/src/part2/metafile_manager.h b/src/part2/metafile_manager.h new file mode 100644 index 0000000000..faeb7d0d31 --- /dev/null +++ b/src/part2/metafile_manager.h @@ -0,0 +1,28 @@ +#ifndef SRC_PART2_METAFILE_MANAGER_H_ +#define SRC_PART2_METAFILE_MANAGER_H_ + +#include +#include + +#include "src/part2/define.h" + +namespace nebd { +namespace server { + +class NebdMetaFileManager { + public: + explicit NebdMetaFileManager(const std::string& metaFilePath); + virtual ~NebdMetaFileManager(); + + virtual int RemoveFileInfo(const std::string& fileName); + virtual int UpdateFileInfo(const NebdFileInfo& fileInfo); + virtual int ListFileInfo(std::vector* fileInfos); + + private: + std::string metaFilePath_; +}; + +} // namespace server +} // namespace nebd + +#endif // SRC_PART2_METAFILE_MANAGER_H_ diff --git a/src/part2/nebd_server.cpp b/src/part2/nebd_server.cpp new file mode 100644 index 0000000000..775acfcbff --- /dev/null +++ b/src/part2/nebd_server.cpp @@ -0,0 +1,23 @@ +#include "src/part2/nebd_server.h" + +namespace nebd { +namespace server { + +int NebdServer::Init() { + // TODO + return 0; +} + +int Run() { + // TODO + return 0; +} + +int Fini() { + // TODO + return 0; +} + +} // namespace server +} // namespace nebd + diff --git a/src/part2/nebd_server.h b/src/part2/nebd_server.h new file mode 100644 index 0000000000..060da66b09 --- /dev/null +++ b/src/part2/nebd_server.h @@ -0,0 +1,19 @@ +#ifndef SRC_PART2_NEBD_SERVER_H_ +#define SRC_PART2_NEBD_SERVER_H_ + +namespace nebd { +namespace server { + +class NebdServer { + public: + NebdServer() {} + virtual ~NebdServer() {} + int Init(); + int Run(); + int Fini(); +}; + +} // namespace server +} // namespace nebd + +#endif // SRC_PART2_NEBD_SERVER_H_ diff --git a/src/part2/rados_interface.cpp b/src/part2/rados_interface.cpp deleted file mode 100644 index 259fa6a8df..0000000000 --- a/src/part2/rados_interface.cpp +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Project: nebd - * File Created: 2019-09-30 - * Author: hzwuhongsong - * Copyright (c) 2019 NetEase - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/part2/rados_interface.h" -#include "src/part2/common_type.h" -#include "src/part2/config.h" - -int OpenImage(rados_t* cluster, const char* poolname, const char* volname, - int fd, char* filename) { - LOG(NOTICE) << "open image start. " << filename; - - rados_ioctx_t* io = new rados_ioctx_t; - int ret; - ret = rados_ioctx_create(*cluster, poolname, io); - if (ret < 0) { - LOG(ERROR) << "create ioctx failed. " << poolname << ", " << volname; - delete filename; - return ret; - } - rbd_image_t* image = new rbd_image_t; - - ret = rbd_open(*io, volname, image, NULL); - if (ret < 0) { - LOG(ERROR) << "open image failed."; - delete image; - delete filename; - rados_ioctx_destroy(*io); - return ret; - } - - if (fd <= 0) { - while (true) { - srand((unsigned) time(NULL)); - unsigned int seed; - fd = rand_r(&seed) % FD_CEPH_MAX; - if (fd == 0) fd++; - LOG(NOTICE) << "fd is: " << fd; - if (!g_imageMap.count(fd)) { - break; - } - } - } - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.insert(std::pair(fd, fd_image)); - - LOG(NOTICE) << "open image success."; - return fd; -} - -//关闭卷。该函数的调用需加文件锁保护 -int CloseImage(int fd) { - rbd_image_t* image; - std::map::iterator iter; - iter = g_imageMap.find(fd); - - if (iter == g_imageMap.end()) { - LOG(ERROR) << "close image failed, because fd is not exist, fd is: " - << fd; - return -1; - } - LOG(NOTICE) << "fd is exist, fd is: " << fd; - image = iter->second->image; - - rados_ioctx_t* io; - rados_t* cluster; - io = iter->second->io; - cluster = iter->second->cluster; - delete (iter->second->filename); - rbd_close(*image); - rados_ioctx_destroy(*io); - CloseRados(cluster); - // 删除fd内存镜像 - g_imageMap.erase(iter); - - LOG(NOTICE) << "close image success. fd is: " << fd; - - return 0; -} - -rados_t* ConnectRados(const char* mon_host) { - LOG(NOTICE) << "connect rados start. " << mon_host; - int ret; - rados_t* cluster = new rados_t; - ret = rados_create(cluster, "admin"); - if (ret < 0) { - delete cluster; - LOG(ERROR) << "create cluster failed."; - return NULL; - } - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return NULL; - } - std::string ceph_conf = ReadCephConf(cfg); - config_destroy(cfg); - ret = rados_conf_read_file(*cluster, ceph_conf.c_str()); - if (ret < 0) { - LOG(ERROR) << "read config file failed."; - } - ret = rados_conf_set(*cluster, "mon_host", mon_host); - if (ret < 0) { - LOG(ERROR) << "set conf failed."; - CloseRados(cluster); - return NULL; - } - ret = rados_connect(*cluster); - if (ret < 0) { - LOG(ERROR) << "connect rados failed."; - CloseRados(cluster); - return NULL; - } - - LOG(NOTICE) << "connect rados success."; - return cluster; -} - -void CloseRados(rados_t* cluster) { - rados_shutdown(*cluster); - delete cluster; -} - -int FilenameFdExist(char* filename) { - std::map::iterator iter; - - for (iter = g_imageMap.begin(); iter != g_imageMap.end(); iter++) { - LOG(NOTICE) << " fd " - << "filename is: " << iter->second->filename << " " - << filename; - if (strcmp(filename, iter->second->filename) == 0) { - LOG(NOTICE) << " fd exist." - << "filename is: " << iter->second->filename - << ", fd is: " << iter->first; - return iter->first; - } - } - LOG(NOTICE) << " fd is not exist. " - << "filename is: " << filename; - return 0; -} - -bool FdExist(int fd, rbd_image_t** image) { - std::map::iterator iter; - for (iter = g_imageMap.begin(); iter != g_imageMap.end(); iter++) { - if (iter->first == fd) { - LOG(INFO) << "fd is exist: " << fd; - *image = iter->second->image; - return true; - } - } - - LOG(NOTICE) << "fd is not exist: " << fd; - return false; -} diff --git a/src/part2/rados_interface.h b/src/part2/rados_interface.h deleted file mode 100644 index b9c9821dd3..0000000000 --- a/src/part2/rados_interface.h +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Project: nebd - * File Created: 2019-09-30 - * Author: hzwuhongsong - * Copyright (c) 2019 NetEase - */ - -#ifndef SRC_PART2_RADOS_INTERFACE_H_ -#define SRC_PART2_RADOS_INTERFACE_H_ - -#include -#include -#include "src/part2/common_type.h" - -int OpenImage(rados_t* cluster, const char* poolname, const char* volname, - int fd, char* filename); -rados_t* ConnectRados(const char* mon_host); -void CloseRados(rados_t* cluster); -int FilenameFdExist(char* filename); -int CloseImage(int fd); -bool FdExist(int fd, rbd_image_t** image); - -#endif // SRC_PART2_RADOS_INTERFACE_H_ diff --git a/src/part2/reload.cpp b/src/part2/reload.cpp deleted file mode 100644 index b944e791be..0000000000 --- a/src/part2/reload.cpp +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Project: nebd - * File Created: 2019-09-30 - * Author: hzwuhongsong - * Copyright (c) 2019 NetEase - */ - -#include "src/part2/reload.h" -#include -#include -#include -#include -#include -#include -#include "src/part2/rados_interface.h" - -/** - * @brief 从持久化文件中加载ceph卷 - */ -int ReloadCephVolume(int fd, char* filename) { - LOG(NOTICE) << "reload ceph volume start, " << filename; - rados_t* cluster = NULL; - std::vector split_firstly = split(filename, "="); - if (split_firstly.size() <= 1) { - LOG(ERROR) << "filename format is incorrect."; - delete filename; - return -1; - } - - std::vector split_secondly = split(split_firstly[0], ":"); - if (split_secondly.size() <= 1) { - LOG(ERROR) << "filename format is incorrect."; - delete filename; - return -1; - } - - std::vector split_thirdly = split(split_secondly[1], "/"); - if (split_thirdly.size() <= 1) { - LOG(ERROR) << "filename format is incorrect."; - delete filename; - return -1; - } - char* imagename = const_cast(split_thirdly[1].c_str()); - char* poolname = const_cast(split_thirdly[0].c_str()); - std::string mon_host = GetMonHost(filename); - if (mon_host.empty()) { - LOG(ERROR) << "mon host is null."; - delete filename; - return -1; - } - - cluster = ConnectRados(mon_host.c_str()); - if (cluster == NULL) { - LOG(ERROR) << "connect rados failed. " << mon_host; - delete filename; - return -1; - } - int ret = OpenImage(cluster, poolname, imagename, fd, filename); - if (ret < 0) { - LOG(ERROR) << "open image failed. " << poolname << ", " << imagename; - CloseRados(cluster); - delete filename; - return ret; - } - - LOG(NOTICE) << "reload vol success, " << poolname << ", " << imagename; - return 0; -} - -/** - * @brief 从持久化文件中加载卷 - */ -int Reload() { - LOG(NOTICE) << "reload start."; - std::string metadata_file = GetUuidFile(); - boost::property_tree::ptree root, items; - try { - boost::property_tree::read_json< - boost::property_tree::ptree>(metadata_file, root); - } - catch (boost::property_tree::ptree_error pt) { - LOG(ERROR) << "read json throw an exception. " << pt.what(); - return -1; - } - - // 获取port - auto port_exist = root.get_child_optional("port"); - if (!port_exist) { - LOG(ERROR) << "port is not exist."; - return -1; - } - int port = root.get("port", 0); - - auto volumes_exist = root.get_child_optional("volumes"); - if (!volumes_exist) { - return port; - } - - // 遍历所有的volumes - items = root.get_child("volumes"); - for (boost::property_tree::ptree::iterator it = items.begin(); - it != items.end(); ++it) { - int fd = atoi(it->second.get("fd").c_str()); - char* filename_tmp = NULL; - filename_tmp = - const_cast(it->second.get("filename").c_str()); - if (filename_tmp == NULL) { - LOG(ERROR) << "filename is null."; - continue; - } - char* filename = new char[strlen(filename_tmp) + 1](); - snprintf(filename, strlen(filename_tmp) + 1, "%s", filename_tmp); - - if (fd <= FD_CEPH_MAX) { - ReloadCephVolume(fd, filename); - } - } - - LOG(NOTICE) << "reload end."; - return port; -} diff --git a/src/part2/reload.h b/src/part2/reload.h deleted file mode 100644 index 189c97aab1..0000000000 --- a/src/part2/reload.h +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Project: nebd - * File Created: 2019-09-30 - * Author: hzwuhongsong - * Copyright (c) 2019 NetEase - */ - -#ifndef SRC_PART2_RELOAD_H_ -#define SRC_PART2_RELOAD_H_ - -#include -#include -#include - -int Reload(); -/** - * @brief 从持久化文件中加载ceph卷 - * @param fd 卷对应的fd - * @param filename 卷对应的filename，filename包含集群mon，pool，rbd等信息 - * @return 错误码 - */ -int ReloadCephVolume(int fd, char* filename); -#endif // SRC_PART2_RELOAD_H_ diff --git a/src/part2/request_executor.h b/src/part2/request_executor.h new file mode 100644 index 0000000000..0fc3452e9b --- /dev/null +++ b/src/part2/request_executor.h @@ -0,0 +1,34 @@ +#ifndef SRC_PART2_REQUEST_EXECUTOR_H_ +#define SRC_PART2_REQUEST_EXECUTOR_H_ + +#include "src/part2/define.h" + +namespace nebd { +namespace server { + +class NebdFileInstance { + public: + NebdFileInstance() {} + virtual ~NebdFileInstance() {} +}; + +class NebdRequestExcutor { + public: + NebdRequestExcutor() {} + virtual ~NebdRequestExcutor() {} + virtual std::shared_ptr Open(const std::string& filename) = 0; // NOLINT + virtual int Close(NebdFileInstance* fd) = 0; + virtual int Extend(NebdFileInstance* fd, int64_t newsize) = 0; + virtual int StatFile(NebdFileInstance* fd) = 0; + virtual int Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) = 0; + virtual int AioRead(NebdFileInstance* fd, NebdServerAioContext* aioctx) = 0; + virtual int AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) = 0; + virtual int Flush(NebdFileInstance* fd, NebdServerAioContext* aioctx) = 0; + virtual int GetInfo(NebdFileInstance* fd) = 0; + virtual int InvalidCache(NebdFileInstance* fd) = 0; +}; + +} // namespace server +} // namespace nebd + +#endif // SRC_PART2_REQUEST_EXECUTOR_H_ diff --git a/src/part2/request_executor_ceph.cpp b/src/part2/request_executor_ceph.cpp new file mode 100644 index 0000000000..1d0bd91789 --- /dev/null +++ b/src/part2/request_executor_ceph.cpp @@ -0,0 +1,60 @@ +#include "src/part2/request_executor_ceph.h" + +namespace nebd { +namespace server { + +std::shared_ptr +CephRequestExcutor::Open(const std::string& filename) { + // TODO + return nullptr; +} + +int CephRequestExcutor::Close(NebdFileInstance* fd) { + // TODO + return 0; +} + +int CephRequestExcutor::Extend(NebdFileInstance* fd, int64_t newsize) { + // TODO + return 0; +} + +int CephRequestExcutor::StatFile(NebdFileInstance* fd) { + // TODO + return 0; +} + +int CephRequestExcutor::Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) { + // TODO + return 0; +} + +int CephRequestExcutor::AioRead(NebdFileInstance* fd, NebdServerAioContext* aioctx) { + // TODO + return 0; +} + +int CephRequestExcutor::AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) { + // TODO + return 0; +} + +int CephRequestExcutor::Flush(NebdFileInstance* fd, NebdServerAioContext* aioctx) { + // TODO + return 0; +} + +int CephRequestExcutor::GetInfo(NebdFileInstance* fd) { + // TODO + return 0; +} + +int CephRequestExcutor::InvalidCache(NebdFileInstance* fd) { + // TODO + return 0; +} + + +} // namespace server +} // namespace nebd + diff --git a/src/part2/request_executor_ceph.h b/src/part2/request_executor_ceph.h new file mode 100644 index 0000000000..206fda0ae1 --- /dev/null +++ b/src/part2/request_executor_ceph.h @@ -0,0 +1,39 @@ +#ifndef SRC_PART2_REQUEST_EXECUTO_CEPH_H_ +#define SRC_PART2_REQUEST_EXECUTO_CEPH_H_ + +#include +#include +#include "src/part2/request_executor.h" + +namespace nebd { +namespace server { + +class CephFileInstance : public NebdFileInstance { + public: + CephFileInstance() {} + ~CephFileInstance() {} + + int fd; + std::string sessionid; +}; + +class CephRequestExcutor : public NebdRequestExcutor { + public: + CephRequestExcutor() {} + ~CephRequestExcutor() {} + std::shared_ptr Open(const std::string& filename) override; // NOLINT + int Close(NebdFileInstance* fd) override; + int Extend(NebdFileInstance* fd, int64_t newsize) override; + int StatFile(NebdFileInstance* fd) override; + int Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; + int AioRead(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; + int AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; + int Flush(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; + int GetInfo(NebdFileInstance* fd) override; + int InvalidCache(NebdFileInstance* fd) override; +}; + +} // namespace server +} // namespace nebd + +#endif // SRC_PART2_REQUEST_EXECUTO_CEPH_H_ diff --git a/src/part2/request_executor_curve.cpp b/src/part2/request_executor_curve.cpp new file mode 100644 index 0000000000..3be98208e9 --- /dev/null +++ b/src/part2/request_executor_curve.cpp @@ -0,0 +1,60 @@ +#include "src/part2/request_executor_curve.h" + +namespace nebd { +namespace server { + +std::shared_ptr +CurveRequestExcutor::Open(const std::string& filename) { + // TODO + return nullptr; +} + +int CurveRequestExcutor::Close(NebdFileInstance* fd) { + // TODO + return 0; +} + +int CurveRequestExcutor::Extend(NebdFileInstance* fd, int64_t newsize) { + // TODO + return 0; +} + +int CurveRequestExcutor::StatFile(NebdFileInstance* fd) { + // TODO + return 0; +} + +int CurveRequestExcutor::Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) { + // TODO + return 0; +} + +int CurveRequestExcutor::AioRead(NebdFileInstance* fd, NebdServerAioContext* aioctx) { + // TODO + return 0; +} + +int CurveRequestExcutor::AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) { + // TODO + return 0; +} + +int CurveRequestExcutor::Flush(NebdFileInstance* fd, NebdServerAioContext* aioctx) { + // TODO + return 0; +} + +int CurveRequestExcutor::GetInfo(NebdFileInstance* fd) { + // TODO + return 0; +} + +int CurveRequestExcutor::InvalidCache(NebdFileInstance* fd) { + // TODO + return 0; +} + + +} // namespace server +} // namespace nebd + diff --git a/src/part2/request_executor_curve.h b/src/part2/request_executor_curve.h new file mode 100644 index 0000000000..eb51c74307 --- /dev/null +++ b/src/part2/request_executor_curve.h @@ -0,0 +1,39 @@ +#ifndef SRC_PART2_REQUEST_EXECUTO_CURVE_H_ +#define SRC_PART2_REQUEST_EXECUTO_CURVE_H_ + +#include +#include +#include "src/part2/request_executor.h" + +namespace nebd { +namespace server { + +class CurveFileInstance : public NebdFileInstance { + public: + CurveFileInstance() {} + ~CurveFileInstance() {} + + int fd; + std::string sessionid; +}; + +class CurveRequestExcutor : public NebdRequestExcutor { + public: + CurveRequestExcutor() {} + ~CurveRequestExcutor() {} + std::shared_ptr Open(const std::string& filename) override; // NOLINT + int Close(NebdFileInstance* fd) override; + int Extend(NebdFileInstance* fd, int64_t newsize) override; + int StatFile(NebdFileInstance* fd) override; + int Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; + int AioRead(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; + int AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; + int Flush(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; + int GetInfo(NebdFileInstance* fd) override; + int InvalidCache(NebdFileInstance* fd) override; +}; + +} // namespace server +} // namespace nebd + +#endif // SRC_PART2_REQUEST_EXECUTO_CURVE_H_ diff --git a/src/part2/rpc_ceph.cpp b/src/part2/rpc_ceph.cpp deleted file mode 100644 index d3e7016996..0000000000 --- a/src/part2/rpc_ceph.cpp +++ /dev/null @@ -1,385 +0,0 @@ -/* - * Project: nebd - * File Created: 2019-09-30 - * Author: hzwuhongsong - * Copyright (c) 2019 NetEase - */ - -#include "src/part2/rpc_ceph.h" -#include -#include -#include -#include -#include -#include "src/common/client.pb.h" -#include "src/part2/rpc_request.h" -#include "src/part2/rados_interface.h" -#include "src/part2/common_type.h" - -void RbdFinishAioWrite(rbd_completion_t c, AsyncWrite* write) { - google::protobuf::Closure* done = write->done; - brpc::ClosureGuard done_guard(done); - uint64_t offset = write->request->offset(); - uint64_t size = write->request->size(); - int fd = write->request->fd(); - - int ret = rbd_aio_get_return_value(c); - rbd_aio_release(c); - - if (ret == 0) { - LOG(INFO) << "write success. fd is: " << fd - << ", offset is: " << offset << ", size is: " << size; - write->response->set_retcode(nebd::client::kOK); - } else { - LOG(ERROR) << "write failed. fd is: " << fd << ", offset is: " << offset - << ", size is: " << size; - } - requestForRpcWrite--; - requestForCephWrite--; - delete write; -} - -void RbdFinishAioRead(rbd_completion_t c, AsyncRead* read) { - google::protobuf::Closure* done = read->done; - brpc::ClosureGuard done_guard(done); - uint64_t offset = read->request->offset(); - uint64_t size = read->request->size(); - int fd = read->request->fd(); - - uint64_t ret = rbd_aio_get_return_value(c); - rbd_aio_release(c); - - if (ret > 0) { - LOG(INFO) << "read success. fd is: " << fd << ", offset is: " << offset - << ", request size is: " << size - << ", actual size is: " << ret; - if (ret != size) LOG(ERROR) << "read not equal."; - read->response->set_retcode(nebd::client::kOK); - read->cntl->response_attachment().append(read->buf, size); - } else { - LOG(ERROR) << "read failed. fd is: " << fd << ", offset is: " << offset - << ", size is: " << size; - } - - delete read->buf; - delete read; -} - -void RbdFinishAioDiscard(rbd_completion_t c, AsyncDiscard* discard) { - google::protobuf::Closure* done = discard->done; - brpc::ClosureGuard done_guard(done); - uint64_t offset = discard->request->offset(); - uint64_t size = discard->request->size(); - int fd = discard->request->fd(); - - int ret = rbd_aio_get_return_value(c); - rbd_aio_release(c); - - if (ret == 0) { - LOG(INFO) << "discard success. fd is: " << fd - << ", offset is: " << offset << ", size is: " << size; - discard->response->set_retcode(nebd::client::kOK); - } else { - LOG(ERROR) << "discard failed. fd is: " << fd - << ", offset is: " << offset << ", size is: " << size; - } - - delete discard; -} - -void RbdFinishAioFlush(rbd_completion_t c, AsyncFlush* flush) { - google::protobuf::Closure* done = flush->done; - brpc::ClosureGuard done_guard(done); - int fd = flush->request->fd(); - - int ret = rbd_aio_get_return_value(c); - rbd_aio_release(c); - - if (ret == 0) { - LOG(INFO) << "flush success. fd is: " << fd; - flush->response->set_retcode(nebd::client::kOK); - } else { - LOG(ERROR) << "flush failed. fd is: " << fd; - } - - delete flush; -} - -int RpcRequestCeph::OpenFile(char* rpc_filename) { - LOG(ERROR) << "ceph open file start. " << rpc_filename; - char* filename = new char[strlen(rpc_filename) + 1](); - snprintf(filename, strlen(rpc_filename) + 1, "%s", rpc_filename); - - std::vector split_firstly = split(filename, "="); - if (split_firstly.size() <= 1) { - LOG(ERROR) << "filename format is incorrect."; - delete filename; - return -1; - } - - std::vector split_secondly = split(split_firstly[0], ":"); - if (split_secondly.size() <= 1) { - LOG(ERROR) << "filename format is incorrect."; - delete filename; - return -1; - } - - std::vector split_thirdly = split(split_secondly[1], "/"); - if (split_thirdly.size() <= 1) { - LOG(ERROR) << "filename format is incorrect."; - delete filename; - return -1; - } - char* imagename = const_cast(split_thirdly[1].c_str()); - char* poolname = const_cast(split_thirdly[0].c_str()); - - std::string mon_host = GetMonHost(filename); - if (mon_host.empty()) { - LOG(ERROR) << "mon host is null."; - delete filename; - return -1; - } - - // 如果fd已经存在，则直接返回 - int fd; - fd = FilenameFdExist(filename); - if (fd > 0) { - LOG(ERROR) << "fd is already exist."; - delete filename; - return fd; - } - - rados_t* cluster = NULL; - cluster = ConnectRados(mon_host.c_str()); - if (cluster == NULL) { - LOG(ERROR) << "connect rados failed."; - delete filename; - return -1; - } - - fd = OpenImage(cluster, poolname, imagename, -1, filename); - if (fd < 0) { - LOG(ERROR) << "open image failed."; - CloseRados(cluster); - delete filename; - return -1; - } - std::string uuid_lockfile = GetUuidLockfile().c_str(); - int lockfd = LockFile(uuid_lockfile.c_str()); - if (lockfd < 0) { - LOG(ERROR) << "lock file failed."; - CloseImage(fd); - return -1; - } - int ret = GenerateFd(filename, fd); - if (ret < 0) { - LOG(ERROR) << "generate fd failed."; - CloseImage(fd); - UnlockFile(lockfd); - return -1; - } - UnlockFile(lockfd); - LOG(NOTICE) << "ceph open file end."; - return fd; -} - -int RpcRequestCeph::Write(AsyncWrite* writejob) { - rbd_completion_t c; - int ret; - rbd_image_t* image; - if (!FdExist(writejob->request->fd(), &image)) { - LOG(ERROR) << "fd is not exist."; - return -1; - } - - uint64_t offset = writejob->request->offset(); - uint64_t size = writejob->request->size(); - ret = rbd_aio_create_completion(writejob, - (rbd_callback_t)RbdFinishAioWrite, &c); - if (ret < 0) { - LOG(ERROR) << "create write completion failed"; - return -1; - } - - requestForCephWrite++; - ret = rbd_aio_write(*image, offset, size, writejob->buf.c_str(), c); - if (ret < 0) { - LOG(ERROR) << "write image failed"; - return -1; - } - return 0; -} - -int RpcRequestCeph::Read(AsyncRead* readjob) { - rbd_completion_t c; - uint64_t offset = readjob->request->offset(); - uint64_t size = readjob->request->size(); - - rbd_image_t* image; - if (!FdExist(readjob->request->fd(), &image)) { - LOG(ERROR) << "fd is not exist."; - return -1; - } - - char* buf = new char[size + 1](); - buf[size] = '\0'; - readjob->buf = buf; - - int ret = rbd_aio_create_completion( - readjob, (rbd_callback_t)RbdFinishAioRead, &c); - if (ret < 0) { - LOG(ERROR) << "create read completion failed"; - return -1; - } - ret = rbd_aio_read(*image, offset, size, buf, c); - if (ret < 0) { - LOG(ERROR) << "read failed."; - delete buf; - return -1; - } - return 0; -} - -int RpcRequestCeph::StatFile(int fd, nebd::client::StatFileResponse* response) { - rbd_image_t* image; - if (!FdExist(fd, &image)) { - LOG(ERROR) << "fd is not exist."; - return -1; - } - - rbd_image_info_t info; - int ret; - ret = rbd_stat(*image, &info, sizeof(info)); - if (ret < 0) { - return ret; - } - response->set_retcode(nebd::client::kOK); - response->set_size(info.size); - return 0; -} - -int RpcRequestCeph::GetInfo(int fd, nebd::client::GetInfoResponse* response) { - rbd_image_t* image; - if (!FdExist(fd, &image)) { - LOG(ERROR) << "fd is not exist."; - return -1; - } - - rbd_image_info_t info; - int ret; - ret = rbd_stat(*image, &info, sizeof(info)); - if (ret < 0) { - return ret; - } - response->set_retcode(nebd::client::kOK); - response->set_objsize(info.obj_size); - - return 0; -} - -int RpcRequestCeph::CloseFile(int fd) { - std::string uuid_lockfile = GetUuidLockfile(); - int lockfd = LockFile(uuid_lockfile.c_str()); - if (lockfd < 0) { - LOG(ERROR) << "add lock failed."; - return -1; - } - - int ret = CloseImage(fd); - if (ret < 0) { - UnlockFile(lockfd); - LOG(ERROR) << "close image failed."; - return -1; - } - - ret = RmFd(fd); - if (ret < 0) { - UnlockFile(lockfd); - LOG(ERROR) << "rm fd failed."; - return -1; - } - - UnlockFile(lockfd); - - return ret; -} - -int RpcRequestCeph::Discard(AsyncDiscard* discardjob) { - rbd_completion_t c; - int ret; - rbd_image_t* image; - if (!FdExist(discardjob->request->fd(), &image)) { - LOG(ERROR) << "fd is not exist."; - return -1; - } - - uint64_t offset = discardjob->request->offset(); - uint64_t size = discardjob->request->size(); - ret = rbd_aio_create_completion(discardjob, - (rbd_callback_t)RbdFinishAioDiscard, &c); - if (ret < 0) { - LOG(ERROR) << "create discard completion failed."; - return -1; - } - - ret = rbd_aio_discard(*image, offset, size, c); - if (ret < 0) { - LOG(ERROR) << "aio_discard failed."; - return -1; - } - return 0; -} - -int RpcRequestCeph::Flush(AsyncFlush* flushjob) { - rbd_completion_t c; - int ret; - rbd_image_t* image; - if (!FdExist(flushjob->request->fd(), &image)) { - LOG(ERROR) << "fd is not exist."; - return -1; - } - - ret = rbd_aio_create_completion(flushjob, - (rbd_callback_t)RbdFinishAioFlush, &c); - if (ret < 0) { - LOG(ERROR) << "create flush completion failed."; - return -1; - } - - ret = rbd_aio_flush(*image, c); - if (ret < 0) { - LOG(ERROR) << "aio flush failed."; - return -1; - } - return 0; -} - -int RpcRequestCeph::Resize(int fd, uint64_t size) { - int ret; - rbd_image_t* image; - if (!FdExist(fd, &image)) { - LOG(ERROR) << "fd is not exist."; - return -1; - } - - ret = rbd_resize(*image, size); - if (ret < 0) { - LOG(ERROR) << "rbd resize failed."; - } - return ret; -} - -int RpcRequestCeph::InvalidateCache(int fd) { - int ret; - rbd_image_t* image; - if (!FdExist(fd, &image)) { - LOG(ERROR) << "fd is not exist."; - return -1; - } - - ret = rbd_invalidate_cache(*image); - if (ret < 0) { - LOG(ERROR) << "rbd invalidate cache failed."; - } - return ret; -} diff --git a/src/part2/rpc_ceph.h b/src/part2/rpc_ceph.h deleted file mode 100644 index 162615402f..0000000000 --- a/src/part2/rpc_ceph.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Project: nebd - * File Created: 2019-09-30 - * Author: hzwuhongsong - * Copyright (c) 2019 NetEase - */ - -#ifndef SRC_PART2_RPC_CEPH_H_ -#define SRC_PART2_RPC_CEPH_H_ - -#include -#include -#include -#include -#include -#include "src/common/client.pb.h" -#include "src/part2/rpc_request.h" - -void RbdFinishAioWrite(rbd_completion_t c, AsyncWrite* write); -void RbdFinishAioRead(rbd_completion_t c, AsyncRead* read); -void RbdFinishAioDiscard(rbd_completion_t c, AsyncDiscard* discard); -void RbdFinishAioFlush(rbd_completion_t c, AsyncFlush* flush); - -/** - * @brief rpc请求对应的ceph处理类 - * @detail - * - Filename中包含了集群mon ip，逻辑池以及卷信息 - * - OpenFile函数根据Filename打开卷，然后就可以对卷进行读写等一系列操作了 - */ -class RpcRequestCeph : public RpcRequest { - public: - RpcRequestCeph() {} - virtual ~RpcRequestCeph() {} - virtual int OpenFile(char* filename); - - virtual int Write(AsyncWrite* writejob); - - virtual int Read(AsyncRead* writejob); - - virtual int StatFile(int fd, nebd::client::StatFileResponse* response); - - virtual int GetInfo(int fd, nebd::client::GetInfoResponse* response); - - virtual int CloseFile(int fd); - - virtual int Flush(AsyncFlush* flushjob); - - virtual int Discard(AsyncDiscard* discardjob); - - virtual int Resize(int fd, uint64_t size); - virtual int InvalidateCache(int fd); -}; - -#endif // SRC_PART2_RPC_CEPH_H_ diff --git a/src/part2/rpc_request.h b/src/part2/rpc_request.h deleted file mode 100644 index 777d6da9b7..0000000000 --- a/src/part2/rpc_request.h +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Project: nebd - * File Created: 2019-09-30 - * Author: hzwuhongsong - * Copyright (c) 2019 NetEase - */ - -#ifndef SRC_PART2_RPC_REQUEST_H_ -#define SRC_PART2_RPC_REQUEST_H_ - -#include -#include -#include -#include -#include -#include -#include "src/common/client.pb.h" - -struct AsyncWrite { - google::protobuf::Closure* done; - const nebd::client::WriteRequest* request; - nebd::client::WriteResponse* response; - brpc::Controller* cntl; - std::string buf; -}; - -struct AsyncRead { - google::protobuf::Closure* done; - const nebd::client::ReadRequest* request; - nebd::client::ReadResponse* response; - brpc::Controller* cntl; - char* buf; -}; - -struct AsyncDiscard { - google::protobuf::Closure* done; - const nebd::client::DiscardRequest* request; - nebd::client::DiscardResponse* response; -}; - -struct AsyncFlush { - google::protobuf::Closure* done; - const nebd::client::FlushRequest* request; - nebd::client::FlushResponse* response; -}; - -class RpcRequest { - public: - RpcRequest() {} - virtual ~RpcRequest() {} - virtual int OpenFile(char* filename) { - return 0; - } - - virtual int Write(AsyncWrite* writejob) { - return 0; - } - - virtual int Read(AsyncRead* readjob) { - return 0; - } - - virtual int StatFile(int fd, nebd::client::StatFileResponse* response) { - return 0; - } - - virtual int GetInfo(int fd, nebd::client::GetInfoResponse* response) { - return 0; - } - - virtual int CloseFile(int fd) { - return 0; - } - - virtual int Flush(AsyncFlush* flushjob) { - return 0; - } - - virtual int Resize(int fd, uint64_t size) { - return 0; - } - - virtual int Discard(AsyncDiscard* discardjob) { - return 0; - } - - virtual int InvalidateCache(int fd) { - return 0; - } -}; - -#endif // SRC_PART2_RPC_REQUEST_H_ diff --git a/src/part2/rpc_server.cpp b/src/part2/rpc_server.cpp deleted file mode 100644 index f092e5b8f3..0000000000 --- a/src/part2/rpc_server.cpp +++ /dev/null @@ -1,457 +0,0 @@ -/* - * Project: nebd - * File Created: 2019-09-30 - * Author: hzwuhongsong - * Copyright (c) 2019 NetEase - */ - -#include "src/part2/rpc_server.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/part2/common_type.h" -#include "src/part2/heartbeat.h" -#include "src/part2/rados_interface.h" -#include "src/part2/rpc_ceph.h" -std::mutex g_mutex; -void QemuClientServiceImpl::OpenFile( - google::protobuf::RpcController* cntl_base, - const nebd::client::OpenFileRequest* request, - nebd::client::OpenFileResponse* response, google::protobuf::Closure* done) { - brpc::ClosureGuard done_guard(done); - - brpc::Controller* cntl = static_cast(cntl_base); - - LOG(INFO) << "Received open request[log_id=" << cntl->log_id() << "] from " - << cntl->remote_side(); - - response->set_retcode(nebd::client::kNoOK); - - char* filename = const_cast(request->filename().c_str()); - - // 如果fd已经存在，则直接返回 - int fd; - fd = FilenameFdExist(filename); - if (fd > 0) { - response->set_fd(fd); - response->set_retcode(nebd::client::kOK); - return; - } - - g_mutex.lock(); - if (!opening_image.empty()) { - std::vector::iterator ret; - ret = std::find(opening_image.begin(), opening_image.end(), - request->filename()); - if (ret != opening_image.end()) { - LOG(WARNING) << "image is opening: " << filename; - g_mutex.unlock(); - while (1) { - bthread_usleep(1000000); - g_mutex.lock(); - std::vector::iterator ret; - ret = std::find(opening_image.begin(), opening_image.end(), - request->filename()); - if (ret != opening_image.end()) { - LOG(WARNING) << "image is opening: " << filename; - g_mutex.unlock(); - } else { - fd = FilenameFdExist(filename); - LOG(WARNING) - << "image is opening success by others: " << filename - << ", " << fd; - g_mutex.unlock(); - response->set_fd(fd); - response->set_retcode(nebd::client::kOK); - return; - } - } - } - } - opening_image.push_back(filename); - g_mutex.unlock(); - - std::vector split_firstly = split(filename, ":"); - std::string block_type = split_firstly[0]; - - RpcRequest* rpc_request = NULL; - if (block_type == "rbd") { - rpc_request = request_ceph; - } else { - LOG(ERROR) << "not recognized block_type: " << block_type; - return; - } - fd = rpc_request->OpenFile(filename); - LOG(WARNING) << "open file, fd is: " << fd; - if (fd > 0) { - response->set_fd(fd); - response->set_retcode(nebd::client::kOK); - } - - g_mutex.lock(); - std::vector::iterator it = opening_image.begin(); - for (; it != opening_image.end();) { - if (*it == request->filename()) { - LOG(WARNING) << "image open success, delete from opening_image, " - << *it; - it = opening_image.erase(it); - } else { - ++it; - } - } - g_mutex.unlock(); -} - -void QemuClientServiceImpl::Write(google::protobuf::RpcController* cntl_base, - const nebd::client::WriteRequest* request, - nebd::client::WriteResponse* response, - google::protobuf::Closure* done) { - requestForRpcWrite++; - requestForRpcWriteCounts++; - brpc::ClosureGuard done_guard(done); - - brpc::Controller* cntl = static_cast(cntl_base); - - LOG(INFO) << "Received write request[log_id=" << cntl->log_id() << "] from " - << cntl->remote_side() << ", offset is: " << request->offset() - << ", len is: " << request->size(); - - response->set_retcode(nebd::client::kNoOK); - - int fd = request->fd(); - rbd_image_t* image; - - if (!FdExist(fd, &image)) { - LOG(ERROR) << "fd is not exist. " << fd; - return; - } - - RpcRequest* rpc_request = NULL; - AsyncWrite* writejob = NULL; - if (fd <= FD_CEPH_MAX) { - writejob = new AsyncWrite; - writejob->done = done; - writejob->response = response; - writejob->request = request; - writejob->cntl = cntl; - writejob->buf = cntl->request_attachment().to_string(); - rpc_request = request_ceph; - } else { - LOG(ERROR) << "not recognized fd: " << fd; - return; - } - if (rpc_request->Write(writejob) < 0) { - LOG(ERROR) << "write failed. " << fd; - delete writejob; - return; - } - done_guard.release(); -} - -void QemuClientServiceImpl::Read(google::protobuf::RpcController* cntl_base, - const nebd::client::ReadRequest* request, - nebd::client::ReadResponse* response, - google::protobuf::Closure* done) { - brpc::ClosureGuard done_guard(done); - - brpc::Controller* cntl = static_cast(cntl_base); - - LOG(INFO) << "Received Read request[log_id=" << cntl->log_id() << "] from " - << cntl->remote_side(); - - response->set_retcode(nebd::client::kNoOK); - - int fd = request->fd(); - rbd_image_t* image; - if (!FdExist(fd, &image)) { - LOG(ERROR) << "fd is not exist. " << fd; - return; - } - - RpcRequest* rpc_request = NULL; - AsyncRead* readjob = NULL; - if (fd <= FD_CEPH_MAX) { - readjob = new AsyncRead; - readjob->done = done; - readjob->response = response; - readjob->request = request; - readjob->cntl = cntl; - rpc_request = request_ceph; - } else { - LOG(ERROR) << "not recognized fd: " << fd; - return; - } - if (rpc_request->Read(readjob) < 0) { - LOG(ERROR) << "read failed. " << fd; - delete readjob; - return; - } - done_guard.release(); -} - -void QemuClientServiceImpl::StatFile( - google::protobuf::RpcController* cntl_base, - const nebd::client::StatFileRequest* request, - nebd::client::StatFileResponse* response, google::protobuf::Closure* done) { - brpc::ClosureGuard done_guard(done); - - brpc::Controller* cntl = static_cast(cntl_base); - - LOG(INFO) << "Received Size request[log_id=" << cntl->log_id() << "] from " - << cntl->remote_side() - << " (attached=" << cntl->request_attachment() << ")"; - - response->set_retcode(nebd::client::kNoOK); - response->set_size(0); - - int fd = request->fd(); - rbd_image_t* image; - if (!FdExist(fd, &image)) { - LOG(ERROR) << "fd is not exist. " << fd; - return; - } - - RpcRequest* rpc_request = NULL; - if (fd <= FD_CEPH_MAX) { - rpc_request = request_ceph; - } else { - LOG(ERROR) << "not recognized fd: " << fd; - return; - } - rpc_request->StatFile(fd, response); -} - -void QemuClientServiceImpl::GetInfo(google::protobuf::RpcController* cntl_base, - const nebd::client::GetInfoRequest* request, - nebd::client::GetInfoResponse* response, - google::protobuf::Closure* done) { - brpc::ClosureGuard done_guard(done); - - brpc::Controller* cntl = static_cast(cntl_base); - - LOG(INFO) << "Received Size request[log_id=" << cntl->log_id() << "] from " - << cntl->remote_side() - << " (attached=" << cntl->request_attachment() << ")"; - - response->set_retcode(nebd::client::kNoOK); - response->set_objsize(0); - - int fd = request->fd(); - rbd_image_t* image; - if (!FdExist(fd, &image)) { - LOG(ERROR) << "fd is not exist. " << fd; - return; - } - - RpcRequest* rpc_request = NULL; - if (fd <= FD_CEPH_MAX) { - rpc_request = request_ceph; - } else { - LOG(ERROR) << "not recognized fd: " << fd; - return; - } - rpc_request->GetInfo(fd, response); - return; -} - -void QemuClientServiceImpl::Flush(google::protobuf::RpcController* cntl_base, - const nebd::client::FlushRequest* request, - nebd::client::FlushResponse* response, - google::protobuf::Closure* done) { - brpc::ClosureGuard done_guard(done); - - brpc::Controller* cntl = static_cast(cntl_base); - - LOG(INFO) << "Received Flush request[log_id=" << cntl->log_id() << "] from " - << cntl->remote_side() - << " (attached=" << cntl->request_attachment() << ")"; - - response->set_retcode(nebd::client::kNoOK); - - int fd = request->fd(); - rbd_image_t* image; - if (!FdExist(fd, &image)) { - LOG(ERROR) << "fd is not exist. " << fd; - return; - } - - RpcRequest* rpc_request = NULL; - AsyncFlush* flushjob = NULL; - if (fd <= FD_CEPH_MAX) { - flushjob = new AsyncFlush; - flushjob->done = done; - flushjob->response = response; - flushjob->request = request; - rpc_request = request_ceph; - } else { - LOG(ERROR) << "not recognized fd: " << fd; - return; - } - if (rpc_request->Flush(flushjob) < 0) { - LOG(ERROR) << "flush failed. " << fd; - delete flushjob; - return; - } - done_guard.release(); -} - -void QemuClientServiceImpl::CloseFile( - google::protobuf::RpcController* cntl_base, - const nebd::client::CloseFileRequest* request, - nebd::client::CloseFileResponse* response, - google::protobuf::Closure* done) { - brpc::ClosureGuard done_guard(done); - - brpc::Controller* cntl = static_cast(cntl_base); - - LOG(INFO) << "Received Close request[log_id=" << cntl->log_id() << "] from " - << cntl->remote_side(); - - response->set_retcode(nebd::client::kNoOK); - - int fd = request->fd(); - rbd_image_t* image; - if (!FdExist(fd, &image)) { - LOG(ERROR) << "fd is not exist. " << fd; - return; - } - - RpcRequest* rpc_request = NULL; - if (fd <= FD_CEPH_MAX) { - rpc_request = request_ceph; - } else { - LOG(ERROR) << "not recognized fd: " << fd; - return; - } - - if (rpc_request->CloseFile(fd) == 0) { - response->set_retcode(nebd::client::kOK); - } - return; -} - -void QemuClientServiceImpl::Discard(google::protobuf::RpcController* cntl_base, - const nebd::client::DiscardRequest* request, - nebd::client::DiscardResponse* response, - google::protobuf::Closure* done) { - brpc::ClosureGuard done_guard(done); - - brpc::Controller* cntl = static_cast(cntl_base); - - LOG(INFO) << "Received Discard request[log_id=" << cntl->log_id() - << "] from " << cntl->remote_side() - << " (attached=" << cntl->request_attachment() << ")"; - - response->set_retcode(nebd::client::kNoOK); - - int fd = request->fd(); - rbd_image_t* image; - if (!FdExist(fd, &image)) { - LOG(ERROR) << "fd is not exist. " << fd; - return; - } - - RpcRequest* rpc_request = NULL; - AsyncDiscard* discardjob = NULL; - if (fd <= FD_CEPH_MAX) { - discardjob = new AsyncDiscard; - discardjob->done = done; - discardjob->response = response; - discardjob->request = request; - rpc_request = request_ceph; - } else { - LOG(ERROR) << "not recognized fd: " << fd; - return; - } - if (rpc_request->Discard(discardjob) < 0) { - LOG(ERROR) << "discard failed. " << fd; - delete discardjob; - return; - } - done_guard.release(); -} - -void QemuClientServiceImpl::ResizeFile( - google::protobuf::RpcController* cntl_base, - const nebd::client::ResizeRequest* request, - nebd::client::ResizeResponse* response, google::protobuf::Closure* done) { - brpc::ClosureGuard done_guard(done); - - brpc::Controller* cntl = static_cast(cntl_base); - - LOG(INFO) << "Received ResizeFile request[log_id=" << cntl->log_id() - << "] from " << cntl->remote_side() - << " (attached=" << cntl->request_attachment() << ")"; - - response->set_retcode(nebd::client::kNoOK); - - int fd = request->fd(); - rbd_image_t* image; - if (!FdExist(fd, &image)) { - LOG(ERROR) << "fd is not exist. " << fd; - return; - } - - RpcRequest* rpc_request = NULL; - if (fd <= FD_CEPH_MAX) { - rpc_request = request_ceph; - } else { - LOG(ERROR) << "not recognized fd: " << fd; - return; - } - - if (rpc_request->Resize(fd, request->newsize()) == 0) { - response->set_retcode(nebd::client::kOK); - } - return; -} - -void QemuClientServiceImpl::InvalidateCache( - google::protobuf::RpcController* cntl_base, - const nebd::client::InvalidateCacheRequest* request, - nebd::client::InvalidateCacheResponse* response, - google::protobuf::Closure* done) { - brpc::ClosureGuard done_guard(done); - - brpc::Controller* cntl = static_cast(cntl_base); - - LOG(INFO) << "Received InvalidateCache request[log_id=" << cntl->log_id() - << "] from " << cntl->remote_side() - << " (attached=" << cntl->request_attachment() << ")"; - - response->set_retcode(nebd::client::kNoOK); - - int fd = request->fd(); - rbd_image_t* image; - if (!FdExist(fd, &image)) { - LOG(ERROR) << "fd is not exist. " << fd; - return; - } - - RpcRequest* rpc_request = NULL; - if (fd <= FD_CEPH_MAX) { // for ceph - rpc_request = request_ceph; - } else { - LOG(ERROR) << "not recognized fd: " << fd; - return; - } - - if (rpc_request->InvalidateCache(fd) >= 0) { - response->set_retcode(nebd::client::kOK); - } -} diff --git a/tests/part1/CMakeLists.txt b/tests/part1/CMakeLists.txt index ac3edec678..41e17834ca 100644 --- a/tests/part1/CMakeLists.txt +++ b/tests/part1/CMakeLists.txt @@ -10,44 +10,14 @@ set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -std=c++0x -pipe -W -Wall \ add_definitions( "-DGFLAGS=gflags -DOS_LINUX -DSNAPPY -DHAVE_ZLIB -DHAVE_SSE42 -DNDEBUG") -# libfake_client_service.so -set(FAKE_CLT_SRV_SRC fake_client_service.cpp fake_client_service.h) +# libfake_file_service.so +set(FAKE_CLT_SRV_SRC fake_file_service.cpp fake_file_service.h) set(FAKE_CLT_SRV_LINK brpc-shared client_proto gflags gtest gmock) -add_library(fake_client_service SHARED ${FAKE_CLT_SRV_SRC}) -target_link_libraries(fake_client_service ${FAKE_CLT_SRV_LINK}) -install(TARGETS fake_client_service DESTINATION lib) - -# client_server -set(CLT_SERVER_SRC client_server.cpp) -set(CLT_SERVER_LINK fake_client_service jsoncpp_lib) -add_executable(client_server ${CLT_SERVER_SRC}) -target_link_libraries(client_server ${CLT_SERVER_LINK}) -install(TARGETS client_server DESTINATION bin) - -# test_part1 -set(TEST_PART1_SRC nebd_test.cpp mock_client_service.h) -set(TEST_PART1_LINK - rt - ssl - crypto - dl - z - pthread - gflags - gtest - gmock - nebd - nebd_common - client_proto) - -add_executable(test_part1 ${TEST_PART1_SRC}) -target_link_libraries(test_part1 ${TEST_PART1_LINK}) -install(TARGETS test_part1 DESTINATION bin) -add_test(NAME test_part1 - COMMAND test_part1 -uuid 1234 - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +add_library(fake_file_service SHARED ${FAKE_CLT_SRV_SRC}) +target_link_libraries(fake_file_service ${FAKE_CLT_SRV_LINK}) +install(TARGETS fake_file_service DESTINATION lib) diff --git a/tests/part1/client_server.cpp b/tests/part1/client_server.cpp deleted file mode 100644 index 1650bbd56b..0000000000 --- a/tests/part1/client_server.cpp +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Project: nebd - * Created Date: 2019-08-12 - * Author: hzchenwei7 - * Copyright (c) 2018 netease - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include "tests/part1/fake_client_service.h" - -DEFINE_string(uuid, "12345678-1234-1234-1234-123456789012", "uuid"); -DEFINE_string(port, "6667", "port"); - -namespace nebd { -namespace client { -int client_main(int argc, char **argv) { - google::ParseCommandLineFlags(&argc, &argv, false); - - // add service - brpc::Server server; - ClientService clientService; - int ret = server.AddService(&clientService, - brpc::SERVER_DOESNT_OWN_SERVICE); - if (ret != 0) { - LOG(FATAL) << "add clientService error"; - return -1; - } - - // start rpc server - brpc::ServerOptions option; - option.idle_timeout_sec = -1; - std::string listenAddr = "127.0.0.1:" + FLAGS_port; - ret = server.Start(listenAddr.c_str(), &option); - if (ret != 0) { - LOG(FATAL) << "start brpc server error"; - return -1; - } - - // 写meta - std::string filePath = "file_" + FLAGS_uuid; - Json::Value root; - root["port"] = FLAGS_port; - int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_SYNC, 0644); - if (fd < 0) { - LOG(ERROR) << "open metadata fail, filePath = " << filePath - << ", errno = " << errno; - return -1; - } - ret = write(fd, root.toStyledString().c_str(), - root.toStyledString().size()); - if (ret < root.toStyledString().size()) { - LOG(ERROR) << "write matadata fail, filePath = " << filePath; - close(fd); - return -1; - } - close(fd); - LOG(INFO) << "write metadata filePath = " << filePath - << ", content : " << root.toStyledString().c_str(); - - server.RunUntilAskedToQuit(); - return 0; -} -} // namespace client -} // namespace nebd - -int main(int argc, char **argv) { - // google::InitGoogleLogging(argv[0]); - // 使进程为守护进程 - if (daemon(1, 0) < 0) { - LOG(ERROR) << "create daemon error."; - return -1; - } - - //初始化日志模块 - logging::LoggingSettings log; - log.logging_dest = logging::LOG_TO_FILE; - log.log_file = "tests/part1/client_server.log"; - logging::InitLogging(log); - - LOG(INFO) << "start client server."; - return nebd::client::client_main(argc, argv); -} diff --git a/tests/part1/fake_client_service.cpp b/tests/part1/fake_file_service.cpp similarity index 88% rename from tests/part1/fake_client_service.cpp rename to tests/part1/fake_file_service.cpp index ec58d63689..d6c8b72ab9 100644 --- a/tests/part1/fake_client_service.cpp +++ b/tests/part1/fake_file_service.cpp @@ -4,7 +4,7 @@ * Author: hzchenwei7 * Copyright (c) 2018 netease */ -#include "tests/part1/fake_client_service.h" +#include "tests/part1/fake_file_service.h" namespace nebd { namespace client { @@ -12,7 +12,7 @@ namespace client { uint64_t filesize = 50*1024*1024; // 50MB char *buf = reinterpret_cast(malloc(filesize)); -void ClientService::OpenFile(::google::protobuf::RpcController* controller, +void FileService::OpenFile(::google::protobuf::RpcController* controller, const ::nebd::client::OpenFileRequest* request, ::nebd::client::OpenFileResponse* response, ::google::protobuf::Closure* done) { @@ -33,7 +33,7 @@ void ClientService::OpenFile(::google::protobuf::RpcController* controller, return; } -void ClientService::CloseFile(::google::protobuf::RpcController* controller, +void FileService::CloseFile(::google::protobuf::RpcController* controller, const ::nebd::client::CloseFileRequest* request, ::nebd::client::CloseFileResponse* response, ::google::protobuf::Closure* done) { @@ -48,7 +48,7 @@ void ClientService::CloseFile(::google::protobuf::RpcController* controller, return; } -void ClientService::Read(::google::protobuf::RpcController* controller, +void FileService::Read(::google::protobuf::RpcController* controller, const ::nebd::client::ReadRequest* request, ::nebd::client::ReadResponse* response, ::google::protobuf::Closure* done) { @@ -65,7 +65,7 @@ void ClientService::Read(::google::protobuf::RpcController* controller, return; } -void ClientService::Write(::google::protobuf::RpcController* controller, +void FileService::Write(::google::protobuf::RpcController* controller, const ::nebd::client::WriteRequest* request, ::nebd::client::WriteResponse* response, ::google::protobuf::Closure* done) { @@ -82,7 +82,7 @@ void ClientService::Write(::google::protobuf::RpcController* controller, return; } -void ClientService::Discard(::google::protobuf::RpcController* controller, +void FileService::Discard(::google::protobuf::RpcController* controller, const ::nebd::client::DiscardRequest* request, ::nebd::client::DiscardResponse* response, ::google::protobuf::Closure* done) { @@ -97,7 +97,7 @@ void ClientService::Discard(::google::protobuf::RpcController* controller, return; } -void ClientService::StatFile(::google::protobuf::RpcController* controller, +void FileService::StatFile(::google::protobuf::RpcController* controller, const ::nebd::client::StatFileRequest* request, ::nebd::client::StatFileResponse* response, ::google::protobuf::Closure* done) { @@ -113,7 +113,7 @@ void ClientService::StatFile(::google::protobuf::RpcController* controller, return; } -void ClientService::ResizeFile(::google::protobuf::RpcController* controller, +void FileService::ResizeFile(::google::protobuf::RpcController* controller, const ::nebd::client::ResizeRequest* request, ::nebd::client::ResizeResponse* response, ::google::protobuf::Closure* done) { @@ -128,7 +128,7 @@ void ClientService::ResizeFile(::google::protobuf::RpcController* controller, return; } -void ClientService::Flush(::google::protobuf::RpcController* controller, +void FileService::Flush(::google::protobuf::RpcController* controller, const ::nebd::client::FlushRequest* request, ::nebd::client::FlushResponse* response, ::google::protobuf::Closure* done) { @@ -143,7 +143,7 @@ void ClientService::Flush(::google::protobuf::RpcController* controller, return; } -void ClientService::GetInfo(::google::protobuf::RpcController* controller, +void FileService::GetInfo(::google::protobuf::RpcController* controller, const ::nebd::client::GetInfoRequest* request, ::nebd::client::GetInfoResponse* response, ::google::protobuf::Closure* done) { @@ -159,7 +159,7 @@ void ClientService::GetInfo(::google::protobuf::RpcController* controller, return; } -void ClientService::InvalidateCache( +void FileService::InvalidateCache( ::google::protobuf::RpcController* controller, const ::nebd::client::InvalidateCacheRequest* request, ::nebd::client::InvalidateCacheResponse* response, diff --git a/tests/part1/fake_client_service.h b/tests/part1/fake_file_service.h similarity index 92% rename from tests/part1/fake_client_service.h rename to tests/part1/fake_file_service.h index bb6078407c..127f06a731 100644 --- a/tests/part1/fake_client_service.h +++ b/tests/part1/fake_file_service.h @@ -5,8 +5,8 @@ * Copyright (c) 2018 netease */ -#ifndef TESTS_PART1_FAKE_CLIENT_SERVICE_H_ -#define TESTS_PART1_FAKE_CLIENT_SERVICE_H_ +#ifndef TESTS_PART1_FAKE_FILE_SERVICE_H_ +#define TESTS_PART1_FAKE_FILE_SERVICE_H_ #include #include @@ -16,11 +16,11 @@ namespace nebd { namespace client { -class ClientService: public QemuClientService { +class FileService: public NebdFileService { public: - ClientService() {} + FileService() {} - virtual ~ClientService() {} + virtual ~FileService() {} void OpenFile(::google::protobuf::RpcController* controller, const ::nebd::client::OpenFileRequest* request, @@ -74,4 +74,4 @@ class ClientService: public QemuClientService { }; } // namespace client } // namespace nebd -#endif // TESTS_PART1_FAKE_CLIENT_SERVICE_H_ +#endif // TESTS_PART1_FAKE_FILE_SERVICE_H_ diff --git a/tests/part1/mock_client_service.h b/tests/part1/mock_file_service.h similarity index 95% rename from tests/part1/mock_client_service.h rename to tests/part1/mock_file_service.h index 28c0d18959..86032a12b2 100644 --- a/tests/part1/mock_client_service.h +++ b/tests/part1/mock_file_service.h @@ -16,10 +16,10 @@ namespace nebd { namespace client { -class MockQemuClientService : public QemuClientService { +class MockNebdFileService : public NebdFileService { public: - MockQemuClientService() : QemuClientService() {} - ~MockQemuClientService() = default; + MockNebdFileService() : NebdFileService() {} + ~MockNebdFileService() = default; MOCK_METHOD4(OpenFile, void(::google::protobuf::RpcController* controller, const ::nebd::client::OpenFileRequest* request, diff --git a/tests/part1/nebd.conf b/tests/part1/nebd.conf deleted file mode 100644 index 45d27855c5..0000000000 --- a/tests/part1/nebd.conf +++ /dev/null @@ -1,49 +0,0 @@ -# nebd part2的进程名字 -part2ProcName=client_server -# nebd part2的程序的启动路径，从该路径拉起part2 -part2ProcPath=build/bin/client_server -# nebd part2的ip地址 -part2Addr=127.0.0.1 -# kill part2服务之后，检查part2服务是否存活的重试次数 -part2KillCheckRetryTimes=5 -# kill part2服务之后，检查part2服务是否存活的重试间隔，单位Us -part2KillCheckRetryIntervalUs=100000 -# nebd part1的qemu进程的名字 -qemuProcName=test_part1 -# nebd part1和part2共用的文件锁文件的路径 -lockFile=tests/part1/lock.file -# nebd part1和part2共用的元数据文件的目录 -metadataPrefix=file_ -# 拉起part2服务的重试次数 -part2StartRetryTimes=5 -# 拉起part2服务的重试间隔，单位Us -part2StartRetryIntervalUs=100000 -# 检查part2的port是否联通的重试次数 -connectibleCheckTimes=5 -# 检查part2的port是否联通的重试间隔，单位Us -connectibleCheckIntervalUs=100000 -# 获取part2的port的重试次数 -portGetRetryTimes=5 -# 获取part2的port的重试间隔，单位Us -portGetRetryIntervalUs=100000 -# 心跳服务的检查间隔，单位Us -heartbeatIntervalUs=1000000 -# 非io的rpc请求的异常重试次数 -rpcRetryTimes=5 -# rpc请求的重试间隔，包括io请求和非io请求 -rpcRetryIntervalUs=200000 -# rpc请求的重试间隔，当io请求遇到server服务未启动的时候使用这个重试间隔， -# 这个值一般比rpcRetryIntervalUs更小，为了更快恢复io服务 -rpcHostDownRetryIntervalUs=200000 -# rpc请求的最大重试间隔 -rpcRetryMaxIntervalUs=6400000 -# rpc请求的超时时间，单位Ms -rpcTimeoutMs=5000 -# brpc的健康检查周期时间，单位s -rpcHealthCheckIntervalS=1 -# 异步IO操作RPC失败日志打印间隔，每隔多少次打印一次 -aioRpcFailLogInterval=10 -# 加文件锁的重试次数 -fileLockRetryTimes=5 -# 加文件锁的重试间隔，单位Us -fileLockRetryIntervalUs=100000 \ No newline at end of file diff --git a/tests/part1/nebd_test.cpp b/tests/part1/nebd_test.cpp deleted file mode 100644 index c877c84686..0000000000 --- a/tests/part1/nebd_test.cpp +++ /dev/null @@ -1,659 +0,0 @@ -/* - * Project: nebd - * Created Date: 2019-08-12 - * Author: hzchenwei7 - * Copyright (c) 2018 netease - */ -#include -#include -#include -#include -#include -#include // flock -#include // NOLINT -#include // NOLINT -#include "tests/part1/mock_client_service.h" -#include "src/part1/libnebd.h" -#include "src/part1/libnebd_file.h" -#include "src/part1/nebd_client.h" -#include "src/part1/nebd_lifecycle.h" - -DEFINE_string(uuid, "12345678-1234-1234-1234-123456789012", "uuid"); - -#define confFilename "tests/part1/nebd.conf" -#define filename "test_file_name" -#define DEFAULT_FD 1 -#define BUFSIZE 512 -#define DEFAULT_FILEZIZE (50*1024*1024) -std::mutex mtx; -std::condition_variable condition; -bool callback; - -void LibAioCallBackFunc(struct ClientAioContext* context) { - ASSERT_EQ(context->ret, 0); - callback = true; - condition.notify_one(); - ASSERT_EQ(context->retryCount, 0); -} - -void LibAioFailCallBackFunc(struct ClientAioContext* context) { - ASSERT_EQ(context->ret, -1); - callback = true; - condition.notify_one(); - ASSERT_EQ(context->retryCount, 0); -} - - -class nebdClientTest: public ::testing::Test { - protected: - void SetUp() override { - callback = false; - system("sudo killall client_server"); - system("sudo mkdir -p /etc/nebd"); - system("sudo cp tests/part1/nebd.conf /etc/nebd/nebd-client.conf"); - ASSERT_EQ(0, nebd_lib_init()); - } - void TearDown() override { - nebd_lib_uninit(); - } -}; - -TEST_F(nebdClientTest, nebdCommonTest) { - // nebd_lib_open_test - int ret; - ret = nebd_lib_open(filename); - ASSERT_EQ(DEFAULT_FD, ret); - - // nebd_lib_pread_test - char buf[BUFSIZE] = {}; - ret = nebd_lib_pread(DEFAULT_FD, buf, 0, BUFSIZE); - ASSERT_EQ(-1, ret); - - // nebd_lib_pwrite_test - memset(buf, 'a', BUFSIZE); - ret = nebd_lib_pwrite(DEFAULT_FD, buf, 0, BUFSIZE); - ASSERT_EQ(-1, ret); - - // nebd_lib_aio_pread_pwrite_test - char writeBuf[BUFSIZE] = "test"; - char readBuf[BUFSIZE]; - ClientAioContext context; - context.buf = writeBuf; - context.offset = 0; - context.length = BUFSIZE; - context.ret = 0; - context.op = LIBAIO_OP_WRITE; - context.cb = LibAioCallBackFunc; - context.retryCount = 0; - - ret = nebd_lib_aio_pwrite(DEFAULT_FD, &context); - ASSERT_EQ(ret, 0); - - std::unique_lock lock{mtx}; - condition.wait(lock); - ASSERT_TRUE(callback); - - callback = false; - ret = nebd_lib_aio_pread(DEFAULT_FD, &context); - ASSERT_EQ(ret, 0); - - condition.wait(lock); - ASSERT_TRUE(callback); - - // nebd_lib_sync_test - ret = nebd_lib_sync(DEFAULT_FD); - ASSERT_EQ(0, ret); - - // nebd_lib_discard_test - context.buf = nullptr; - context.offset = 0; - context.length = BUFSIZE; - context.ret = 0; - context.op = LIBAIO_OP_DISCARD; - context.cb = LibAioCallBackFunc; - context.retryCount = 0; - - ret = nebd_lib_discard(DEFAULT_FD, &context); - ASSERT_EQ(ret, 0); - - condition.wait(lock); - ASSERT_TRUE(callback); - - // nebd_lib_filesize_test - int64_t filesize; - filesize = nebd_lib_filesize(DEFAULT_FD); - ASSERT_EQ(DEFAULT_FILEZIZE, filesize); - - // nebd_lib_resize_test - ret = nebd_lib_resize(DEFAULT_FD, DEFAULT_FILEZIZE); - ASSERT_EQ(0, ret); - - // nebd_lib_getinfo_test - ret = nebd_lib_getinfo(DEFAULT_FD); - ASSERT_EQ(DEFAULT_FILEZIZE, ret); - - // nebd_lib_flush_test - context.buf = nullptr; - context.offset = 0; - context.length = 0; - context.ret = 0; - context.op = LIBAIO_OP_FLUSH; - context.cb = LibAioCallBackFunc; - context.retryCount = 0; - - ret = nebd_lib_flush(DEFAULT_FD, &context); - ASSERT_EQ(ret, 0); - - condition.wait(lock); - ASSERT_TRUE(callback); - - // nebd_lib_getinfo_test - filesize = nebd_lib_getinfo(DEFAULT_FD); - ASSERT_EQ(DEFAULT_FILEZIZE, filesize); - - // nebd_lib_invalidcache_test - ret = nebd_lib_invalidcache(DEFAULT_FD); - ASSERT_EQ(0, ret); - - // nebd_lib_close_test - ret = nebd_lib_close(DEFAULT_FD); - ASSERT_EQ(0, ret); -} - -class nebdFileClientTest: public ::testing::Test { - protected: - void SetUp() override { - system("sudo killall client_server"); - system("sudo mkdir -p /etc/nebd"); - system("sudo cp tests/part1/nebd.conf /etc/nebd/nebd-client.conf"); - } - void TearDown() override { - } -}; - -TEST_F(nebdFileClientTest, common_test) { - nebd::client::FileClient fileClientTest; - ASSERT_EQ(fileClientTest.Init(confFilename), 0); - ASSERT_EQ(fileClientTest.Open(filename), DEFAULT_FD); - ASSERT_EQ(fileClientTest.Extend(DEFAULT_FD, DEFAULT_FILEZIZE), 0); - ASSERT_EQ(fileClientTest.StatFile(DEFAULT_FD), DEFAULT_FILEZIZE); - ASSERT_EQ(fileClientTest.GetInfo(DEFAULT_FD), DEFAULT_FILEZIZE); - ASSERT_EQ(fileClientTest.InvalidCache(DEFAULT_FD), 0); - - char writeBuf[BUFSIZE] = "test"; - char readBuf[BUFSIZE]; - ClientAioContext context; - context.buf = writeBuf; - context.offset = 0; - context.length = BUFSIZE; - context.ret = 0; - context.op = LIBAIO_OP_WRITE; - context.cb = LibAioCallBackFunc; - context.retryCount = 0; - - ASSERT_EQ(fileClientTest.AioRead(DEFAULT_FD, &context), 0); - std::unique_lock lock{mtx}; - condition.wait(lock); - - ASSERT_EQ(fileClientTest.AioWrite(DEFAULT_FD, &context), 0); - condition.wait(lock); - ASSERT_TRUE(callback); - - context.buf = nullptr; - context.offset = 0; - context.length = BUFSIZE; - context.ret = 0; - context.op = LIBAIO_OP_DISCARD; - context.cb = LibAioCallBackFunc; - context.retryCount = 0; - ASSERT_EQ(fileClientTest.Discard(DEFAULT_FD, &context), 0); - condition.wait(lock); - ASSERT_TRUE(callback); - - context.buf = nullptr; - context.offset = 0; - context.length = 0; - context.ret = 0; - context.op = LIBAIO_OP_FLUSH; - context.cb = LibAioCallBackFunc; - context.retryCount = 0; - ASSERT_EQ(fileClientTest.Flush(DEFAULT_FD, &context), 0); - condition.wait(lock); - ASSERT_TRUE(callback); - - ASSERT_EQ(fileClientTest.Close(DEFAULT_FD), 0); - fileClientTest.Uninit(); - return; -} - -TEST_F(nebdFileClientTest, no_rpc_server_test) { - nebd::client::FileClient fileClientTest; - ASSERT_EQ(fileClientTest.Init("wrongConfPath"), -1); - - nebd::common::Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_TRUE(conf.LoadConfig()); - - ASSERT_EQ(fileClientTest.LoadConf(&conf), 0); - - ASSERT_EQ(fileClientTest.InitChannel("127.0.0.1:6667"), 0); - ASSERT_EQ(fileClientTest.Open(filename), -1); - ASSERT_EQ(fileClientTest.Extend(DEFAULT_FD, DEFAULT_FILEZIZE), -1); - ASSERT_EQ(fileClientTest.StatFile(DEFAULT_FD), -1); - ASSERT_EQ(fileClientTest.GetInfo(DEFAULT_FD), -1); - ASSERT_EQ(fileClientTest.InvalidCache(DEFAULT_FD), -1); - - // char writeBuf[BUFSIZE] = "test"; - // char readBuf[BUFSIZE]; - // ClientAioContext context; - // context.buf = writeBuf; - // context.offset = 0; - // context.length = BUFSIZE; - // context.ret = 0; - // context.op = LIBAIO_OP_WRITE; - // context.cb = LibAioCallBackFunc; - // context.retryCount = 0; - - // ASSERT_EQ(fileClientTest.AioRead(DEFAULT_FD, &context), 0); - // std::unique_lock lock{mtx}; - // condition.wait(lock); - - // ASSERT_EQ(fileClientTest.AioWrite(DEFAULT_FD, &context), 0); - // condition.wait(lock); - // ASSERT_TRUE(callback); - - // context.buf = nullptr; - // context.offset = 0; - // context.length = BUFSIZE; - // context.ret = 0; - // context.op = LIBAIO_OP_DISCARD; - // context.cb = LibAioCallBackFunc; - // context.retryCount = 0; - // ASSERT_EQ(fileClientTest.Discard(DEFAULT_FD, &context), 0); - // condition.wait(lock); - // ASSERT_TRUE(callback); - - // context.buf = nullptr; - // context.offset = 0; - // context.length = 0; - // context.ret = 0; - // context.op = LIBAIO_OP_FLUSH; - // context.cb = LibAioCallBackFunc; - // context.retryCount = 0; - // ASSERT_EQ(fileClientTest.Flush(DEFAULT_FD, &context), 0); - // condition.wait(lock); - // ASSERT_TRUE(callback); - - ASSERT_EQ(fileClientTest.Close(DEFAULT_FD), -1); - fileClientTest.Uninit(); - return; -} - -static void OpenFileFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::OpenFileRequest* request, - ::nebd::client::OpenFileResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); - // if (0 != gReadCntlFailedCode) { - // brpc::Controller *cntl = dynamic_cast(controller); - // cntl->SetFailed(-1, "open file controller error"); - // } -} - -static void CloseFileFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::CloseFileRequest* request, - ::nebd::client::CloseFileResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -static void ReadFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::ReadRequest* request, - ::nebd::client::ReadResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -static void WriteFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::WriteRequest* request, - ::nebd::client::WriteResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -static void DiscardFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::DiscardRequest* request, - ::nebd::client::DiscardResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -static void StatFileFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::StatFileRequest* request, - ::nebd::client::StatFileResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -static void ResizeFileFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::ResizeRequest* request, - ::nebd::client::ResizeResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -static void FlushFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::FlushRequest* request, - ::nebd::client::FlushResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -static void GetInfoFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::GetInfoRequest* request, - ::nebd::client::GetInfoResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -static void InvalidateCacheFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::InvalidateCacheRequest* request, - ::nebd::client::InvalidateCacheResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -using ::testing::_; -using ::testing::Invoke; -using ::testing::Return; -using ::testing::AnyNumber; -using ::testing::DoAll; -using ::testing::SetArgPointee; -using ::testing::SetArgReferee; -using ::testing::InSequence; -using ::testing::AtLeast; -using ::testing::SaveArgPointee; - -TEST_F(nebdFileClientTest, rpc_fail_test) { - nebd::client::FileClient fileClientTest; - nebd::common::Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_TRUE(conf.LoadConfig()); - ASSERT_EQ(fileClientTest.LoadConf(&conf), 0); - - // add service - brpc::Server server; - nebd::client::MockQemuClientService clientService; - ASSERT_EQ(server.AddService(&clientService, - brpc::SERVER_DOESNT_OWN_SERVICE), 0); - - // start rpc server - brpc::ServerOptions option; - option.idle_timeout_sec = -1; - std::string listenAddr = "127.0.0.1:6667"; - ASSERT_EQ(server.Start(listenAddr.c_str(), &option), 0); - - ASSERT_EQ(fileClientTest.InitChannel("127.0.0.1:6667"), 0); - - ::nebd::client::OpenFileResponse openFileResponse; - openFileResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, OpenFile(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(openFileResponse), - Invoke(OpenFileFunc))); - ASSERT_EQ(fileClientTest.Open(filename), -1); - - ::nebd::client::ResizeResponse resizeResponse; - resizeResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, ResizeFile(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(resizeResponse), - Invoke(ResizeFileFunc))); - ASSERT_EQ(fileClientTest.Extend(DEFAULT_FD, DEFAULT_FILEZIZE), -1); - - ::nebd::client::StatFileResponse statFileResponse; - statFileResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, StatFile(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(statFileResponse), - Invoke(StatFileFunc))); - ASSERT_EQ(fileClientTest.StatFile(DEFAULT_FD), -1); - - ::nebd::client::GetInfoResponse getInfoResponse; - getInfoResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, GetInfo(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(getInfoResponse), - Invoke(GetInfoFunc))); - ASSERT_EQ(fileClientTest.GetInfo(DEFAULT_FD), -1); - - ::nebd::client::InvalidateCacheResponse invalidateCacheResponse; - invalidateCacheResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, InvalidateCache(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(invalidateCacheResponse), - Invoke(InvalidateCacheFunc))); - ASSERT_EQ(fileClientTest.InvalidCache(DEFAULT_FD), -1); - - char writeBuf[BUFSIZE] = "test"; - char readBuf[BUFSIZE]; - ClientAioContext context; - context.buf = writeBuf; - context.offset = 0; - context.length = BUFSIZE; - context.ret = 0; - context.op = LIBAIO_OP_WRITE; - context.cb = LibAioFailCallBackFunc; - context.retryCount = 0; - - ::nebd::client::ReadResponse readResponse; - readResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, Read(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(readResponse), - Invoke(ReadFunc))); - ASSERT_EQ(fileClientTest.AioRead(DEFAULT_FD, &context), 0); - std::unique_lock lock{mtx}; - condition.wait(lock); - ASSERT_TRUE(callback); - - - context.buf = readBuf; - context.offset = 0; - context.length = BUFSIZE; - context.ret = 0; - context.op = LIBAIO_OP_READ; - context.cb = LibAioFailCallBackFunc; - context.retryCount = 0; - ::nebd::client::WriteResponse writeResponse; - writeResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, Write(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(writeResponse), - Invoke(WriteFunc))); - ASSERT_EQ(fileClientTest.AioWrite(DEFAULT_FD, &context), 0); - condition.wait(lock); - ASSERT_TRUE(callback); - - context.buf = nullptr; - context.offset = 0; - context.length = BUFSIZE; - context.ret = 0; - context.op = LIBAIO_OP_DISCARD; - context.cb = LibAioFailCallBackFunc; - context.retryCount = 0; - ::nebd::client::DiscardResponse discardResponse; - discardResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, Discard(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(discardResponse), - Invoke(DiscardFunc))); - ASSERT_EQ(fileClientTest.Discard(DEFAULT_FD, &context), 0); - condition.wait(lock); - ASSERT_TRUE(callback); - - context.buf = nullptr; - context.offset = 0; - context.length = 0; - context.ret = 0; - context.op = LIBAIO_OP_FLUSH; - context.cb = LibAioFailCallBackFunc; - context.retryCount = 0; - ::nebd::client::FlushResponse flushResponse; - flushResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, Flush(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(flushResponse), - Invoke(FlushFunc))); - ASSERT_EQ(fileClientTest.Flush(DEFAULT_FD, &context), 0); - condition.wait(lock); - ASSERT_TRUE(callback); - - ::nebd::client::CloseFileResponse closeFileResponse; - closeFileResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, CloseFile(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(closeFileResponse), - Invoke(CloseFileFunc))); - ASSERT_EQ(fileClientTest.Close(DEFAULT_FD), -1); - fileClientTest.Uninit(); - return; -} - -class nebdClientLifeCycleTest: public ::testing::Test { - protected: - void SetUp() override {} - void TearDown() override {} -}; - -TEST_F(nebdClientLifeCycleTest, init_when_part2_not_alive) { - ASSERT_EQ(Init4Qemu(confFilename), 0); - Uninit4Qemu(); -} - -TEST_F(nebdClientLifeCycleTest, init_when_part2_alive) { - ASSERT_EQ(Init4Qemu(confFilename), 0); - ASSERT_EQ(Init4Qemu(confFilename), 0); - Uninit4Qemu(); -} - -TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_stop_without_hb_test) { - using ::nebd::common::Configuration; - Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_EQ(conf.LoadConfig(), true); - - nebd::client::LifeCycleManager manager; - ASSERT_EQ(manager.Start(&conf), 0); - ASSERT_EQ(manager.IsPart2Alive(), true); - ASSERT_EQ(manager.IsHeartbeatThreadStart(), false); - ASSERT_EQ(manager.Start(&conf), 0); - ASSERT_EQ(manager.IsPart2Alive(), true); - ASSERT_EQ(manager.IsHeartbeatThreadStart(), false); - manager.Stop(); - ASSERT_EQ(manager.IsPart2Alive(), false); -} - -TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_stop_with_hb_test) { - using ::nebd::common::Configuration; - Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_EQ(conf.LoadConfig(), true); - - nebd::client::LifeCycleManager manager; - ASSERT_EQ(manager.Start(&conf), 0); - ASSERT_EQ(manager.IsPart2Alive(), true); - ASSERT_EQ(manager.Start(&conf), 0); - ASSERT_EQ(manager.IsPart2Alive(), true); - ASSERT_EQ(manager.IsHeartbeatThreadStart(), false); - manager.StartHeartbeat(); - ASSERT_EQ(manager.IsHeartbeatThreadStart(), true); - ASSERT_EQ(manager.IsPart2Alive(), true); - manager.Stop(); - ASSERT_EQ(manager.IsPart2Alive(), false); -} - -TEST_F(nebdClientLifeCycleTest, lifecycle_kill_part2_test) { - using ::nebd::common::Configuration; - Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_EQ(conf.LoadConfig(), true); - - nebd::client::LifeCycleManager manager; - ASSERT_EQ(manager.Start(&conf), 0); - ASSERT_EQ(manager.IsPart2Alive(), true); - ASSERT_EQ(manager.IsHeartbeatThreadStart(), false); - manager.StartHeartbeat(); - ASSERT_EQ(manager.IsHeartbeatThreadStart(), true); - manager.KillPart2(); - sleep(2); - ASSERT_EQ(manager.IsPart2Alive(), true); - manager.Stop(); - ASSERT_EQ(manager.IsPart2Alive(), false); -} - -TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_load_conf_fail_test) { - ::nebd::common::Configuration conf; - nebd::client::LifeCycleManager manager; - ASSERT_EQ(manager.Start(&conf), -1); -} - -TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_get_uuid_fail_test) { - ::nebd::common::Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_EQ(conf.LoadConfig(), true); - conf.SetStringValue("qemuProcName", "wrong_qemuProcName"); - - nebd::client::LifeCycleManager manager; - ASSERT_EQ(manager.Start(&conf), -1); -} - -TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_start_part2_fail) { - ::nebd::common::Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_EQ(conf.LoadConfig(), true); - conf.SetStringValue("part2ProcPath", "wrong_part2ProcPath"); - - nebd::client::LifeCycleManager manager; - ASSERT_EQ(manager.Start(&conf), -1); -} - -TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_read_port_fail) { - ::nebd::common::Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_EQ(conf.LoadConfig(), true); - conf.SetStringValue("metadataPrefix", "wrong_metadataPrefix"); - - nebd::client::LifeCycleManager manager; - ASSERT_EQ(manager.Start(&conf), -1); -} - -TEST_F(nebdClientLifeCycleTest, lifecycle_manager_lock_file_fail) { - ::nebd::common::Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_TRUE(conf.LoadConfig()); - - // 先占用文件锁 - std::string lockFile; - ASSERT_TRUE(conf.GetStringValue("lockFile", &lockFile)); - int fd = open(lockFile.c_str(), O_RDONLY | O_CREAT, 0644); - ASSERT_GT(fd, 0); - ASSERT_EQ(flock(fd, LOCK_EX | LOCK_NB), 0); - - // 再启lifecyclemanager，会因为文件锁被占用而失败 - nebd::client::LifeCycleManager manager; - ASSERT_EQ(manager.Start(&conf), -1); - - // 再释放文件锁 - ASSERT_EQ(flock(fd, LOCK_UN), 0); - close(fd); -} - -int main(int argc, char ** argv) { - ::testing::InitGoogleTest(&argc, argv); - ::testing::InitGoogleMock(&argc, argv); - google::ParseCommandLineFlags(&argc, &argv, false); - int ret = RUN_ALL_TESTS(); - - return ret; -} diff --git a/tests/part2/CMakeLists.txt b/tests/part2/CMakeLists.txt index 6f0e5a584b..1f210e75b4 100644 --- a/tests/part2/CMakeLists.txt +++ b/tests/part2/CMakeLists.txt @@ -6,97 +6,3 @@ set(CPP_FLAGS "-DBRPC_WITH_GLOG=0 -DGFLAGS_NS=google") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CPP_FLAGS} -std=c++0x -DNDEBUG -O2 \ -D__const__= -pipe -W -Wall -Wno-unused-parameter -fPIC \ -fno-omit-frame-pointer") - -set(RELOADUNIT_SRC test_reload.cpp test_reload.h TestReload.cpp) -add_executable(reload_unit ${RELOADUNIT_SRC}) -target_link_libraries(reload_unit - nebdserver - gtest - gmock) -install(TARGETS reload_unit DESTINATION bin) -add_test(NAME reload_unit COMMAND reload_unit) - -set(RELOAD2UNIT_SRC test_reload2.cpp test_reload2.h TestReload2.cpp) -add_executable(reload2_unit ${RELOAD2UNIT_SRC}) -target_link_libraries(reload2_unit - nebdserver - gtest - gmock) -install(TARGETS reload2_unit DESTINATION bin) -add_test(NAME reload2_unit COMMAND reload2_unit) - -set(COMMONUNIT_SRC test_common.cpp test_common.h TestCommon.cpp) -add_executable(common_unit ${COMMONUNIT_SRC}) -target_link_libraries(common_unit - nebdserver - gtest - gmock) -install(TARGETS common_unit DESTINATION bin) -add_test(NAME common_unit COMMAND common_unit) - -set(RADOSUNIT_SRC test_rados.cpp test_rados.h TestRados.cpp) -add_executable(rados_unit ${RADOSUNIT_SRC}) -target_link_libraries(rados_unit - nebdserver - gtest - gmock) -install(TARGETS rados_unit DESTINATION bin) -add_test(NAME rados_unit COMMAND rados_unit) - -set(RPCSERVERUNIT_SRC TestRpcServer.cpp) -add_executable(rpcserver_unit ${RPCSERVERUNIT_SRC}) -target_link_libraries(rpcserver_unit - nebdserver - gtest - gmock) -install(TARGETS rpcserver_unit DESTINATION bin) -add_test(NAME rpcserver_unit COMMAND rpcserver_unit) - -set(RPCCEPHUNIT_SRC TestRpcCeph.cpp test_rpcceph.cpp test_rpcceph.h) -add_executable(rpcceph_unit ${RPCCEPHUNIT_SRC}) -target_link_libraries(rpcceph_unit - nebdserver - gtest - gmock - rados - rbd) -install(TARGETS rpcceph_unit DESTINATION bin) -add_test(NAME rpcceph_unit COMMAND rpcceph_unit) - -set(CONFIGUNIT_SRC TestConfig.cpp) -add_executable(config_unit ${CONFIGUNIT_SRC}) -target_link_libraries(config_unit - nebdserver - gtest - gmock - rados - rbd) -install(TARGETS config_unit DESTINATION bin) -add_test(NAME config_unit COMMAND config_unit) - -set(HEARTBEATUNIT_SRC TestHeartbeat.cpp test_heartbeat.cpp test_heartbeat.h) -add_executable(heartbeat_unit ${HEARTBEATUNIT_SRC}) -target_link_libraries(heartbeat_unit - nebdserver - gtest - gmock) -install(TARGETS heartbeat_unit DESTINATION bin) -add_test(NAME heartbeat_unit COMMAND heartbeat_unit) - -set(HEARTBEATUNIT2_SRC TestHeartbeat2.cpp test_heartbeat2.cpp test_heartbeat2.h) -add_executable(heartbeat2_unit ${HEARTBEATUNIT2_SRC}) -target_link_libraries(heartbeat2_unit - nebdserver - gtest - gmock) -install(TARGETS heartbeat2_unit DESTINATION bin) -add_test(NAME heartbeat2_unit COMMAND heartbeat2_unit) - -set(HEARTBEAT3UNIT_SRC TestHeartbeat3.cpp test_heartbeat3.cpp test_heartbeat3.h) -add_executable(heartbeat3_unit ${HEARTBEAT3UNIT_SRC}) -target_link_libraries(heartbeat3_unit - nebdserver - gtest - gmock) -install(TARGETS heartbeat3_unit DESTINATION bin) -add_test(NAME heartbeat3_unit COMMAND heartbeat3_unit) diff --git a/tests/part2/TestCommon.cpp b/tests/part2/TestCommon.cpp deleted file mode 100644 index 6315dd45d8..0000000000 --- a/tests/part2/TestCommon.cpp +++ /dev/null @@ -1,499 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/part2/common_type.h" -#include "tests/part2/test_common.h" - -TEST(CloseParentFd, CloseParentFd_1) { - EXPECT_EQ(0, CloseParentFd()); -} - -TEST(ReadQemuXmls, ReadQemuXmls_1) { - NebdServerMocker mock; - - // EXPECT_CALL(mock, InitConfig()).WillOnce(testing::ReturnNull()); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("rm -r /etc/nebd"); - EXPECT_EQ(-1, ReadQemuXmls()); -} - -TEST(ReadQemuXmls, ReadQemuXmls_2) { - NebdServerMocker mock; - ::system("mkdir -p /etc/nebd"); - ::system("touch /etc/nebd/nebd-server.conf"); - // std::string xml_dir = "/var/run/nebd/nebd-test"; - std::string xml_dir = "/111/222"; - EXPECT_CALL(mock, ReadQemuXmlDir(::testing::_)) - .WillOnce(testing::Return(xml_dir)); - // EXPECT_CALL(mock, ReadQemuXml(::testing::_)).WillOnce(testing::Return()); - - EXPECT_EQ(-1, ReadQemuXmls()); -} - -TEST(ReadQemuXmls, ReadQemuXmls_3) { - NebdServerMocker mock; - - ::system("rm -rf /etc/nebd"); - ::system("mkdir -p /etc/nebd"); - ::system("touch /etc/nebd/nebd-server.conf"); - std::string xml_dir = "/tmp/nebd-test"; - ::system("mkdir /tmp/nebd-test"); - ::system("touch /tmp/nebd-test/test"); - - boost::property_tree::ptree pt; - pt.add("name", "nebd"); - write_xml("/tmp/nebd-test/test.xml", pt); - - EXPECT_CALL(mock, ReadQemuXmlDir(::testing::_)) - .WillOnce(testing::Return(xml_dir)); - EXPECT_CALL(mock, ReadQemuXml(::testing::_)).WillOnce(testing::Return()); - - EXPECT_EQ(0, ReadQemuXmls()); -} - -TEST(GenerateFd, GenerateFd_1) { - ::system("rm /tmp/unit_test"); - NebdServerMocker mock; - std::string metadata_file = "/111/222/333"; - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - - std::string filename = "unittestnew"; - EXPECT_EQ(-1, GenerateFd(const_cast(filename.c_str()), 2)); -} -/* -TEST(GenerateFd, GenerateFd_2) { - NebdServerMocker mock; - ::system("rm /tmp/unit_test"); - std::string metadata_file = "/tmp/unit_test"; - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - - boost::property_tree::ptree root; - boost::property_tree::ptree items; - try { - root.put_child("volumes", items); - - boost::property_tree::write_json(metadata_file, root); - } catch (boost::property_tree::ptree_error pt) { - LOG(ERROR) << "write json throw an exception. " << pt.what(); - return; - } - - std::string filename_input = "unittestnew"; - EXPECT_EQ(0, GenerateFd(const_cast(filename_input.c_str()), 2)); -} - -TEST(GenerateFd, GenerateFd_3) -{ - LOG(ERROR) << "whs start."; - ::system("rm /tmp/unit_test"); - NebdServerMocker mock; - std::string metadata_file = "/tmp/unit_test"; - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - - boost::property_tree::ptree root; - boost::property_tree::ptree items; - const char* filename = "unittest"; - try { - ptree item1; - item1.put("filename", filename); - item1.put("fd", 1); - items.push_back(std::make_pair("", item1)); - - root.put_child("volumes", items); - - boost::property_tree::write_json(metadata_file, root); - } - catch (ptree_error pt) { - LOG(ERROR) << "write json throw an exception. " << pt.what(); - return; - } - - std::string filename_input = "unittestnew"; - EXPECT_EQ(0, GenerateFd(const_cast(filename_input.c_str()), 2)); -} -*/ - -TEST(RmFd, RmFd_1) { - NebdServerMocker mock; - std::string metadata_file = "/111/222/333"; - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - - EXPECT_EQ(-1, RmFd(2)); -} - -TEST(RmFd, RmFd_2) { - NebdServerMocker mock; - std::string metadata_file = "/tmp/unit_test"; - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - - boost::property_tree::ptree root; - boost::property_tree::ptree items; - try { - // ptree item1; - // item1.put("filename", filename); - // item1.put("fd", 1); - // items.push_back(std::make_pair("", item1)); - - root.put_child("port", items); - - boost::property_tree::write_json(metadata_file, root); - } catch (boost::property_tree::ptree_error pt) { - LOG(ERROR) << "write json throw an exception. " << pt.what(); - return; - } - - EXPECT_EQ(-1, RmFd(2)); -} - -/* -TEST(RmFd, RmFd_3) -{ - NebdServerMocker mock; - std::string metadata_file = "/tmp/unit_test2"; - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - - boost::property_tree::ptree root_tmp; - boost::property_tree::ptree items_tmp; - try { - ptree item1, item2; - item1.put("filename", "file1"); - item1.put("fd", 1); - items_tmp.push_back(std::make_pair("", item1)); - item2.put("filename", "file2"); - item2.put("fd", 2); - items_tmp.push_back(std::make_pair("", item2)); - - root_tmp.put_child("volumes", items_tmp); - - boost::property_tree::write_json(metadata_file, root_tmp); - } - catch (ptree_error pt) { - LOG(ERROR) << "write json throw an exception. " << pt.what(); - return; - } - - EXPECT_EQ(-1, RmFd(3)); - -} - -TEST(RmFd, RmFd_4) -{ - NebdServerMocker mock; - std::string metadata_file = "/tmp/unit_test"; - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - g_port = 6300; - - boost::property_tree::ptree root; - boost::property_tree::ptree items; - try { - ptree item1, item2; - item1.put("filename", "file1"); - item1.put("fd", 1); - items.push_back(std::make_pair("", item1)); - item2.put("filename", "file2"); - item2.put("fd", 2); - items.push_back(std::make_pair("", item2)); - - root.put_child("volumes", items); - - boost::property_tree::write_json(metadata_file, root); - } - catch (ptree_error pt) { - LOG(ERROR) << "write json throw an exception. " << pt.what(); - return; - } - - EXPECT_EQ(0, RmFd(1)); -} -*/ -/* -TEST(FindPort, FindPort_1) -{ - config_t* cfg = new config_t; - g_filePath = "/111/222"; - EXPECT_EQ(-1, FindPort(cfg, 5)); -} - -TEST(FindPort, FindPort_2) -{ - config_t* cfg = new config_t; - g_uuid = 666; - g_filePath = "/tmp/nebd/"; - ::system("mkdir /tmp/nebd"); - std::string metadata_file = "/tmp/nebd/666"; - - boost::property_tree::ptree root; - boost::property_tree::ptree items; - - try { - root.put("port", 10); - boost::property_tree::write_json(metadata_file, root); - } - catch (ptree_error pt) { - LOG(ERROR) << "write json throw an exception. " << pt.what(); - return; - } - - - EXPECT_EQ(-1, FindPort(cfg, 5)); -} - - -TEST(GetUuidFile, GetUuidFile_1) -{ - g_filePath = "/tmp"; - g_uuid = const_cast("666"); - - std::string uuid_file = "/tmp/666"; - EXPECT_STREQ(uuid_file.c_str(), GetUuidFile().c_str()); -} -*/ -TEST(SetCpuAffinity, SetCpuAffinity_1) { EXPECT_EQ(-1, SetCpuAffinity(-1)); } - -TEST(SetCpuAffinity, SetCpuAffinity_2) { EXPECT_EQ(0, SetCpuAffinity(1)); } - -TEST(Init, Init_1) { - ::system("rm /etc/nebd/nebd-server.conf"); - EXPECT_EQ(-1, Init()); -} - -TEST(Init, Init_2) { - ::system("mkdir -p /etc/nebd"); - ::system("touch /etc/nebd/nebd-server.conf"); - std::string uuid = "666"; - g_uuid = const_cast(uuid.c_str()); - NebdServerMocker mock; - EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, Init()); -} - -TEST(Init, Init_3) { - ::system("mkdir -p /etc/nebd"); - ::system("touch /etc/nebd/nebd-server.conf"); - std::string uuid = "666"; - g_uuid = const_cast(uuid.c_str()); - NebdServerMocker mock; - EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_EQ(-1, Init()); -} - -TEST(Init, Init_4) { - ::system("mkdir -p /etc/nebd"); - ::system("touch /etc/nebd/nebd-server.conf"); - std::string uuid = "666"; - g_oldPid = 1; - g_uuid = const_cast(uuid.c_str()); - NebdServerMocker mock; - std::string lockfile = "/tmp/unit_test"; - std::string metadata_file = "/tmp/unit_test2"; - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); - EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, Init()); -} - -TEST(Init, Init_5) { - ::system("mkdir -p /etc/nebd"); - ::system("touch /etc/nebd/nebd-server.conf"); - ::system("rm /tmp/unit_test"); - ::system("rm /tmp/unit_test2"); - ::system("touch /tmp/unit_test2"); - std::string uuid = "666"; - g_oldPid = 1; - g_uuid = const_cast(uuid.c_str()); - NebdServerMocker mock; - std::string lockfile = "/tmp/unit_test"; - std::string metadata_file = "/tmp/unit_test2"; - g_server.Stop(0); - g_server.Join(); - g_server.ClearServices(); - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); - EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, Reload()).WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, Init()); -} - -TEST(Init, Init_6) { - ::system("mkdir -p /etc/nebd"); - ::system("touch /etc/nebd/nebd-server.conf"); - ::system("rm /tmp/unit_test"); - ::system("rm /tmp/unit_test2"); - std::string uuid = "666"; - g_oldPid = 1; - g_uuid = const_cast(uuid.c_str()); - NebdServerMocker mock; - std::string lockfile = "/tmp/unit_test"; - std::string metadata_file = "/tmp/unit_test2"; - g_server.Stop(0); - g_server.Join(); - g_server.ClearServices(); - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); - EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, LockFile(::testing::_)) - .Times(2) - .WillOnce(testing::Return(0)) - .WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, Init()); -} - -TEST(Init, Init_7) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /tmp/unit_test"); - ::system("rm /tmp/unit_test2"); - ::system("touch /tmp/unit_test2"); - std::string uuid = "666"; - g_oldPid = 1; - g_uuid = const_cast(uuid.c_str()); - NebdServerMocker mock; - std::string lockfile = "/tmp/unit_test"; - std::string metadata_file = "/tmp/unit_test2"; - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - std::string file_path = "/etc/nebd/nebd-server.conf"; - int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); - std::string str = "retry_counts=0;\n"; - write(fd, str.c_str(), str.size()); - g_server.Stop(0); - g_server.Join(); - g_server.ClearServices(); - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); - EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, Reload()).WillOnce(testing::Return(6200)); - EXPECT_EQ(-1, Init()); -} -/* -TEST(Init, Init_8) -{ - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - ::system("rm /tmp/unit_test"); - ::system("rm /tmp/unit_test2"); - ::system("touch /tmp/unit_test2"); - std::string uuid = "666"; - g_oldPid = 1; - g_uuid = const_cast(uuid.c_str()); - NebdServerMocker mock; - std::string lockfile = "/tmp/unit_test"; - std::string metadata_file = "/tmp/unit_test2"; - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); - EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, Reload()).WillOnce(testing::Return(6200)); - EXPECT_EQ(-1, Init()); -} -*/ - -TEST(Init, Init_8) { - ::system("mkdir -p /etc/nebd"); - ::system("touch /etc/nebd/nebd-server.conf"); - ::system("rm /tmp/unit_test"); - ::system("rm /tmp/unit_test2"); - std::string uuid = "666"; - g_oldPid = 1; - g_uuid = const_cast(uuid.c_str()); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - std::string file_path = "/etc/nebd/nebd-server.conf"; - int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); - std::string str = "retry_counts=0;\n"; - write(fd, str.c_str(), str.size()); - NebdServerMocker mock; - std::string lockfile = "/tmp/unit_test"; - std::string metadata_file = "/tmp/unit_test2"; - g_server.Stop(0); - g_server.Join(); - g_server.ClearServices(); - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); - EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, FindPort(::testing::_, ::testing::_)) - .WillOnce(testing::Return(6200)); - EXPECT_CALL(mock, LockFile(::testing::_)) - .Times(2) - .WillOnce(testing::Return(0)) - .WillOnce(testing::Return(0)); - EXPECT_EQ(-1, Init()); -} - -TEST(Init, Init_9) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - ::system("rm /tmp/unit_test"); - ::system("rm /tmp/unit_test2"); - std::string uuid = "666"; - g_oldPid = 1; - g_uuid = const_cast(uuid.c_str()); - NebdServerMocker mock; - std::string lockfile = "/tmp/unit_test"; - std::string metadata_file = "/tmp/unit_test2"; - g_server.Stop(0); - g_server.Join(); - g_server.ClearServices(); - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); - EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, FindPort(::testing::_, ::testing::_)) - .WillOnce(testing::Return(6200)); - EXPECT_CALL(mock, LockFile(::testing::_)) - .Times(3) - .WillOnce(testing::Return(0)) - .WillOnce(testing::Return(0)) - .WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, Init()); -} - -TEST(Init, Init_10) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - ::system("rm /tmp/unit_test"); - ::system("rm /tmp/unit_test2"); - std::string uuid = "666"; - g_oldPid = 1; - g_uuid = const_cast(uuid.c_str()); - NebdServerMocker mock; - std::string lockfile = "/tmp/unit_test"; - std::string metadata_file = "/tmp/unit_test2"; - g_server.Stop(0); - g_server.Join(); - g_server.ClearServices(); - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); - EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, GeneratePort(::testing::_)).WillOnce(testing::Return(-1)); - EXPECT_CALL(mock, FindPort(::testing::_, ::testing::_)) - .WillOnce(testing::Return(6200)); - EXPECT_CALL(mock, LockFile(::testing::_)) - .WillRepeatedly(testing::Return(0)); - EXPECT_EQ(-1, Init()); -} - -int main(int argc, char *argv[]) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tests/part2/TestCommon2.cpp b/tests/part2/TestCommon2.cpp deleted file mode 100644 index ff6ddd4486..0000000000 --- a/tests/part2/TestCommon2.cpp +++ /dev/null @@ -1,189 +0,0 @@ -/* - *** Project: nebd - *** File Created: 2019-09-30 - *** Author: hzwuhongsong - *** Copyright (c) 2019 NetEase - ***/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/part2/common_type.h" -#include "tests/part2/test_common2.h" - -TEST(FindPort, FindPort_1) { - config_t* cfg = new config_t; - g_filePath = "/111/222"; - EXPECT_EQ(-1, FindPort(cfg, 5)); -} - -TEST(FindPort, FindPort_2) { - NebdServerMocker mock; - config_t* cfg = new config_t; - g_uuid = const_cast("99999"); - g_filePath = "/tmp/nebd/"; - ::system("mkdir /tmp/nebd"); - ::system("touch /tmp/nebd/99999"); - - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(-1)); - - EXPECT_EQ(-1, FindPort(cfg, 6234)); -} -/* -TEST(FindPort, FindPort_3) -{ - NebdServerMocker mock; - config_t* cfg = new config_t; - //g_uuid = const_cast("99999"); - g_filePath = "/tmp/nebd_test2/"; - ::system("rm -r /tmp/nebd_test2"); - ::system("mkdir /tmp/nebd_test2"); - ::system("touch /tmp/nebd_test2/99999"); - - boost::property_tree::ptree root; - boost::property_tree::ptree items; - - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(1)); - //EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(1)); - - //port_option_t port_option; - //port_option.min_port = 6200; - // port_option.max_port = 6500; - // EXPECT_CALL(mock, read_config_port(::testing::_, ::testing::_)) - // .WillOnce(SetArgReferee<1>(port_option)) - // .WillOnce(testing::Return());; - - EXPECT_EQ(6234, FindPort(cfg, 6234)); -} -*/ -TEST(FindPort, FindPort_4) { - NebdServerMocker mock; - config_t* cfg = new config_t; - g_uuid = const_cast("666"); - g_filePath = "/tmp/nebd/"; - ::system("rm -r /tmp/nebd"); - ::system("mkdir /tmp/nebd"); - ::system("touch /tmp/nebd/666"); - std::string metadata_file = "/tmp/nebd/666"; - - boost::property_tree::ptree root; - boost::property_tree::ptree items; - - try { - root.put("port", 10); - boost::property_tree::write_json(metadata_file, root); - } catch (ptree_error pt) { - LOG(ERROR) << "write json throw an exception. " << pt.what(); - return; - } - - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(1000)); - // EXPECT_CALL(mock, unLockFile(1000)).WillOnce(testing::Return()); - EXPECT_EQ(6234, FindPort(cfg, 6234)); -} -/* -TEST(FindPort, FindPort_5) -{ - NebdServerMocker mock; - config_t* cfg = new config_t; - g_uuid = const_cast("666"); - g_filePath = "/tmp/nebd/"; - ::system("rm -r /tmp/nebd"); - ::system("mkdir /tmp/nebd"); - std::string metadata_file = "/tmp/nebd/666"; - - boost::property_tree::ptree root; - boost::property_tree::ptree items; - - try { - root.put("port", 6234); - boost::property_tree::write_json(metadata_file, root); - } - catch (ptree_error pt) { - LOG(ERROR) << "write json throw an exception. " << pt.what(); - return; - } - - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(1)); - EXPECT_EQ(6235, FindPort(cfg, 6234)); -} - -TEST(FindPort, FindPort_6) -{ - NebdServerMocker mock; - config_t* cfg = new config_t; - g_uuid = const_cast("666"); - g_filePath = "/tmp/nebd/"; - ::system("rm -r /tmp/nebd"); - ::system("mkdir /tmp/nebd"); - std::string metadata_file = "/tmp/nebd/666"; - - boost::property_tree::ptree root; - boost::property_tree::ptree items; - - try { - root.put("port", 7000); - boost::property_tree::write_json(metadata_file, root); - } - catch (ptree_error pt) { - LOG(ERROR) << "write json throw an exception. " << pt.what(); - return; - } - - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(1)); - EXPECT_CALL(mock, unLockFile(1)).WillOnce(testing::Return()); - EXPECT_EQ(0, FindPort(cfg, 7000)); -} -*/ - -TEST(Squeeze, Squeeze_1) { - std::string str = "10.182.30.27\\:6789"; - Squeeze(const_cast(str.c_str()), '\\'); - std::string str_out = "10.182.30.27:6789"; - EXPECT_STREQ(str_out.c_str(), str.c_str()); -} - -TEST(GetMonHost, GetMonHost_1) { - std::string str = ":test"; - std::string host; - std::string host_actual = GetMonHost(str); - EXPECT_STREQ(host.c_str(), host_actual.c_str()); -} - -TEST(GetMonHost, GetMonHost_2) { - std::string str = - "rbd:rbd/" - "ci_rbd_sys_disk:auth_supported=none:mon_host=10.182.30.27\\:6789"; - std::string host = "10.182.30.27:6789"; - std::string host_actual = GetMonHost(str); - EXPECT_STREQ(host.c_str(), host_actual.c_str()); -} - -TEST(GetUuidFile, GetUuidFile_1) { - ::system("rm -r /tmp/nebd"); - g_filePath = "/tmp"; - g_uuid = const_cast("666"); - - std::string uuid_file = "/tmp/666"; - std::string uuid_actual = GetUuidFile(); - EXPECT_STREQ(uuid_file.c_str(), uuid_actual.c_str()); -} -TEST(GetUuidLockfile, GetUuidLockfile_1) { - ::system("rm -r /tmp/nebd"); - g_uuid = const_cast("99988"); - - std::string LockFile = "/tmp/99988"; - std::string lock_actual = GetUuidLockfile(); - EXPECT_STREQ(LockFile.c_str(), lock_actual.c_str()); -} - -int main(int argc, char* argv[]) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tests/part2/TestCommon3.cpp b/tests/part2/TestCommon3.cpp deleted file mode 100644 index 3e35ff6477..0000000000 --- a/tests/part2/TestCommon3.cpp +++ /dev/null @@ -1,103 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/part2/common_type.h" -#include "tests/part2/test_common.h" - -TEST(LockFile, LockFile_1) { - const char* file = "/111/222/333"; - NebdServerMocker mock; - // EXPECT_CALL(mock, open(::testing::_, ::testing::_, - // ::testing::_)).WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, LockFile(file)); -} - -TEST(LockFile, LockFile_2) { - const char* file = "/tmp/unit_test2"; - - EXPECT_EQ(3, LockFile(file)); -} - -TEST(GeneratePort, GeneratePort_1) { - ::system("rm -rf /etc/nebd"); - ::system("rm -rf /tmp/nebd-test"); - NebdServerMocker mock; - std::string metadata_file = "/111/222.txt"; - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - - EXPECT_EQ(-1, GeneratePort(2)); -} - -TEST(GeneratePort, GeneratePort_2) { - NebdServerMocker mock; - std::string metadata_file = "/tmp/unit_test"; - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - - EXPECT_EQ(0, GeneratePort(2)); -} -/* -TEST(GetUuidFile, GetUuidFile_1) -{ - g_filePath = "/tmp"; - g_uuid = const_cast("666"); - - std::string uuid_file = "/tmp/666"; - EXPECT_STREQ(uuid_file.c_str(), GetUuidFile().c_str()); -} - -TEST(SetCpuAffinity, SetCpuAffinity_1) -{ - EXPECT_EQ(-1, SetCpuAffinity(-1)); -} - -TEST(SetCpuAffinity, SetCpuAffinity_2) -{ - EXPECT_EQ(0, SetCpuAffinity(1)); -} -*/ -TEST(GenerateFd, GenerateFd_3) { - LOG(ERROR) << "whs start."; - ::system("rm /tmp/unit_test"); - NebdServerMocker mock; - std::string metadata_file = "/tmp/unit_test"; - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - - boost::property_tree::ptree root; - boost::property_tree::ptree items; - const char* filename = "unittest"; - try { - ptree item1; - item1.put("filename", filename); - item1.put("fd", 1); - items.push_back(std::make_pair("", item1)); - - root.put_child("volumes", items); - - boost::property_tree::write_json(metadata_file, root); - } catch (ptree_error pt) { - LOG(ERROR) << "write json throw an exception. " << pt.what(); - return; - } - - std::string filename_input = "unittestnew"; - EXPECT_EQ(0, GenerateFd(const_cast(filename_input.c_str()), 2)); -} - -int main(int argc, char* argv[]) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tests/part2/TestConfig.cpp b/tests/part2/TestConfig.cpp deleted file mode 100644 index 14dd592ccb..0000000000 --- a/tests/part2/TestConfig.cpp +++ /dev/null @@ -1,331 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/part2/rados_interface.h" -#include "tests/part2/test_rados.h" - -TEST(ReadRpcOption, ReadRpcOption_1) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - rpc_option_t option; - ReadRpcOption(cfg, &option); - EXPECT_EQ(2, option.brpc_num_threads); -} - -TEST(ReadRpcOption, ReadRpcOption_2) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - std::string file_path = "/etc/nebd/nebd-server.conf"; - int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); - std::string str = - "brpc_num_threads=2;\nbrpc_idle_timeout=2;\nbrpc_max_concurrency=2;" - "\nbrpc_methodmax_concurrency=2;\n"; - write(fd, str.c_str(), str.size()); - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - rpc_option_t option; - ReadRpcOption(cfg, &option); - EXPECT_EQ(2, option.brpc_idle_timeout); -} - -TEST(ReadPort, ReadPort_1) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - // NebdServerMocker mock; - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - port_option_t option; - ReadPort(cfg, &option); - EXPECT_EQ(6200, option.min_port); -} - -TEST(ReadPort, ReadPort_2) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - std::string file_path = "/etc/nebd/nebd-server.conf"; - int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); - std::string str = "min_port = 6203;\n"; - write(fd, str.c_str(), str.size()); - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - port_option_t option; - ReadPort(cfg, &option); - EXPECT_EQ(6203, option.min_port); -} - -TEST(ReadLogLevel, ReadLogLevel_1) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - // NebdServerMocker mock; - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - EXPECT_EQ(1, ReadLogLevel(cfg)); -} - -TEST(ReadLogLevel, ReadLogLevel_2) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - std::string file_path = "/etc/nebd/nebd-server.conf"; - int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); - std::string str = "loglevel = 2;\n"; - write(fd, str.c_str(), str.size()); - - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - EXPECT_EQ(2, ReadLogLevel(cfg)); -} - -TEST(ReadUuidPath, ReadUuidPath_1) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - std::string path = "/var/run/nebd-server/"; - EXPECT_STREQ(path.c_str(), ReadUuidPath(cfg).c_str()); -} - -TEST(ReadDetachedTimes, ReadDetachedTimes_1) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - EXPECT_EQ(6, ReadDetachedTimes(cfg)); -} - -TEST(ReadDetachedTimes, ReadDetachedTimes_2) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - std::string file_path = "/etc/nebd/nebd-server.conf"; - int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); - std::string str = "detached_times = 2;\n"; - write(fd, str.c_str(), str.size()); - - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - EXPECT_EQ(2, ReadDetachedTimes(cfg)); -} - -TEST(ReadRetryCounts, ReadRetryCounts_1) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - EXPECT_EQ(3, ReadRetryCounts(cfg)); -} - -TEST(ReadRetryCounts, ReadRetryCounts_2) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - std::string file_path = "/etc/nebd/nebd-server.conf"; - int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); - std::string str = "retry_counts = 2;\n"; - write(fd, str.c_str(), str.size()); - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - EXPECT_EQ(2, ReadRetryCounts(cfg)); -} - -TEST(ReadLockPortFile, ReadLockPortFile_1) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - - std::string path = "/tmp/nebd-server.port.file.lock"; - EXPECT_STREQ(path.c_str(), ReadLockPortFile(cfg).c_str()); -} - -TEST(ReadCephConf, ReadCephConf_1) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - - std::string path = "/etc/ceph/ceph.conf"; - EXPECT_STREQ(path.c_str(), ReadCephConf(cfg).c_str()); -} - -TEST(ReadLogPath, ReadLogPath_1) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - - std::string path = "/var/log/nebd/"; - EXPECT_STREQ(path.c_str(), ReadLogPath(cfg).c_str()); -} - -TEST(ReadQemuXmlDir, ReadQemuXmlDir_1) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - - std::string path = "/var/run/libvirt/qemu/"; - EXPECT_STREQ(path.c_str(), ReadQemuXmlDir(cfg).c_str()); -} - -TEST(ReadHeartbeatInterval, ReadHeartbeatInterval_1) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - EXPECT_EQ(1, ReadHeartbeatInterval(cfg)); -} - -TEST(ReadHeartbeatInterval, ReadHeartbeatInterval_2) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - std::string file_path = "/etc/nebd/nebd-server.conf"; - int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); - std::string str = "heartbeat_interval = 2;\n"; - write(fd, str.c_str(), str.size()); - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - EXPECT_EQ(2, ReadHeartbeatInterval(cfg)); -} - -TEST(ReadAssertTimes, ReadAssertTimes_1) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - EXPECT_EQ(10, ReadAssertTimes(cfg)); -} - -TEST(ReadAssertTimes, ReadAssertTimes_2) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - std::string file_path = "/etc/nebd/nebd-server.conf"; - int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); - std::string str = "check_assert_times = 2;\n"; - write(fd, str.c_str(), str.size()); - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - EXPECT_EQ(2, ReadAssertTimes(cfg)); -} - -TEST(ReadCheckDetachedInterval, ReadCheckDetachedInterval_1) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - EXPECT_EQ(5, ReadCheckDetachedInterval(cfg)); -} - -TEST(ReadCheckDetachedInterval, ReadCheckDetachedInterval_2) { - ::system("mkdir -p /etc/nebd"); - ::system("rm /etc/nebd/nebd-server.conf"); - ::system("touch /etc/nebd/nebd-server.conf"); - std::string file_path = "/etc/nebd/nebd-server.conf"; - int fd = open(file_path.c_str(), O_RDWR | O_CREAT, 0644); - std::string str = "check_detached_interval = 2;\n"; - write(fd, str.c_str(), str.size()); - config_t* cfg = InitConfig(); - if (cfg == NULL) { - LOG(ERROR) << "init config failed."; - return; - } - EXPECT_EQ(2, ReadCheckDetachedInterval(cfg)); -} - -int main(int argc, char* argv[]) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tests/part2/TestHeartbeat.cpp b/tests/part2/TestHeartbeat.cpp deleted file mode 100644 index 8b62090988..0000000000 --- a/tests/part2/TestHeartbeat.cpp +++ /dev/null @@ -1,221 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/part2/heartbeat.h" -#include "tests/part2/test_heartbeat.h" - -TEST(CloseDetachedVolumesProcess, CloseDetachedVolumesProcess_1) { - ::system("rm /etc/nebd/nebd-server.conf"); - NebdServerMocker mock; - EXPECT_EQ(NULL, CloseDetachedVolumesProcess(NULL)); -} - -TEST(CloseDetachedVolumesProcess, CloseDetachedVolumesProcess_2) { - ::system("mkdir /etc/nebd"); - ::system("touch /etc/nebd/nebd-server.conf"); - NebdServerMocker mock; - EXPECT_CALL(mock, CloseQemuDetachedVolumes(::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_EQ(NULL, CloseDetachedVolumesProcess(NULL)); -} - -TEST(HeartbeatProcess, HeartbeatProcess_1) { - ::system("rm /etc/nebd/nebd-server.conf"); - NebdServerMocker mock; - EXPECT_EQ(NULL, HeartbeatProcess(NULL)); -} - -TEST(SkipSpace, SkipSpace_1) { - ::system("rm /etc/nebd/nebd-server.conf"); - NebdServerMocker mock; - std::string str = "123456"; - std::string str_want = "23456"; - int i = 1; - char* str_out = SkipSpace(const_cast(str.c_str()), &i, 8); - EXPECT_STREQ(str_want.c_str(), str_out); -} - -TEST(StopProc, StopProc_1) { - ::system("rm /tmp/unit_test"); - ::system("rm /tmp/unit_test_lockfile"); - NebdServerMocker mock; - std::string metadata_file = "/tmp/unit_test"; - std::string lock_file = "/tmp/unit_test_lockfile"; - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, StopProc(0)); -} - -TEST(StopProc, StopProc_2) { - ::system("rm /tmp/unit_test"); - ::system("rm /tmp/unit_test_lockfile"); - NebdServerMocker mock; - std::string metadata_file = "/tmp/unit_test"; - std::string lock_file = "/tmp/unit_test_lockfile"; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 6; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(-1)); - - EXPECT_EQ(-1, StopProc(0)); -} - -TEST(StopProc, StopProc_3) { - ::system("rm /tmp/unit_test"); - ::system("rm /tmp/unit_test_lockfile"); - NebdServerMocker mock; - std::string metadata_file = "/tmp/unit_test"; - std::string lock_file = "/tmp/unit_test_lockfile"; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 6; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); - - EXPECT_EQ(0, StopProc(0)); -} - -TEST(StopProc, StopProc_4) { - ::system("rm /tmp/unit_test"); - ::system("rm /tmp/unit_test_lockfile"); - ::system("touch /tmp/unit_test"); - NebdServerMocker mock; - std::string metadata_file = "/tmp/unit_test"; - std::string lock_file = "/tmp/unit_test_lockfile"; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 6; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); - - EXPECT_EQ(0, StopProc(0)); -} - -TEST(StopProc, StopProc_5) { - ::system("rm /tmp/unit_test"); - ::system("rm /tmp/unit_test_lockfile"); - NebdServerMocker mock; - std::string metadata_file = "/tmp/unit_test"; - std::string lock_file = "/tmp/unit_test_lockfile"; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 6; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(-1)); - - EXPECT_EQ(0, StopProc(1)); -} - -TEST(StopProc, StopProc_6) { - ::system("rm /tmp/unit_test"); - ::system("rm /tmp/unit_test_lockfile"); - ::system("touch /tmp/unit_test"); - NebdServerMocker mock; - std::string metadata_file = "/tmp/unit_test"; - std::string lock_file = "/tmp/unit_test_lockfile"; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 6; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, CheckProc(::testing::_)).WillOnce(testing::Return(-1)); - - EXPECT_EQ(0, StopProc(1)); -} - -int main(int argc, char* argv[]) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tests/part2/TestHeartbeat2.cpp b/tests/part2/TestHeartbeat2.cpp deleted file mode 100644 index c024e17ae8..0000000000 --- a/tests/part2/TestHeartbeat2.cpp +++ /dev/null @@ -1,182 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/part2/heartbeat.h" -#include "tests/part2/test_heartbeat2.h" - -TEST(CloseQemuDetachedVolumes, CloseQemuDetachedVolumes_1) { - NebdServerMocker mock; - EXPECT_CALL(mock, ReadQemuXmls()).WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, CloseQemuDetachedVolumes(1)); -} - -TEST(CloseQemuDetachedVolumes, CloseQemuDetachedVolumes_2) { - int fd = 6; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - FdImage_t fd_image; - fd_image.filename = filename; - - int fd2 = 7; - char* filename2 = const_cast("mon_host=10.182.30.27"); - FdImage_t fd_image2; - fd_image2.filename = filename2; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, &fd_image)); - g_imageMap.insert(std::pair(fd2, &fd_image2)); - - g_qemuPoolVolumes.push_back("rbd/volume02"); - NebdServerMocker mock; - EXPECT_CALL(mock, ReadQemuXmls()).WillOnce(testing::Return(0)); - std::string lock_file = "/tmp/unit_test_lockfile"; - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); - - EXPECT_EQ(0, CloseQemuDetachedVolumes(3)); -} - -TEST(CloseQemuDetachedVolumes, CloseQemuDetachedVolumes_3) { - int fd = 6; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - FdImage_t fd_image; - fd_image.filename = filename; - - int fd2 = 7; - char* filename2 = const_cast("mon_host=10.182.30.27"); - FdImage_t fd_image2; - fd_image2.filename = filename2; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, &fd_image)); - g_imageMap.insert(std::pair(fd2, &fd_image2)); - - g_qemuPoolVolumes.push_back("rbd/volume02"); - NebdServerMocker mock; - EXPECT_CALL(mock, ReadQemuXmls()).WillOnce(testing::Return(0)); - std::string lock_file = "/tmp/unit_test_lockfile"; - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(-1)); - - EXPECT_EQ(-1, CloseQemuDetachedVolumes(0)); -} - -TEST(CloseQemuDetachedVolumes, CloseQemuDetachedVolumes_4) { - int fd = 6; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - FdImage_t fd_image; - fd_image.filename = filename; - - int fd2 = 7; - char* filename2 = const_cast("mon_host=10.182.30.27"); - FdImage_t fd_image2; - fd_image2.filename = filename2; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, &fd_image)); - g_imageMap.insert(std::pair(fd2, &fd_image2)); - - g_qemuPoolVolumes.push_back("rbd/volume02"); - NebdServerMocker mock; - EXPECT_CALL(mock, ReadQemuXmls()).WillOnce(testing::Return(0)); - std::string lock_file = "/tmp/unit_test_lockfile"; - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(-1)); - - EXPECT_EQ(-1, CloseQemuDetachedVolumes(0)); -} - -TEST(CloseQemuDetachedVolumes, CloseQemuDetachedVolumes_5) { - int fd = 6; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - FdImage_t fd_image; - fd_image.filename = filename; - - int fd2 = 7; - char* filename2 = const_cast("mon_host=10.182.30.27"); - FdImage_t fd_image2; - fd_image2.filename = filename2; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, &fd_image)); - g_imageMap.insert(std::pair(fd2, &fd_image2)); - - g_qemuPoolVolumes.push_back("rbd/volume02"); - NebdServerMocker mock; - EXPECT_CALL(mock, ReadQemuXmls()).WillOnce(testing::Return(0)); - std::string lock_file = "/tmp/unit_test_lockfile"; - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, RmFd(::testing::_)).WillOnce(testing::Return(-1)); - - EXPECT_EQ(-1, CloseQemuDetachedVolumes(0)); -} - -TEST(CloseQemuDetachedVolumes, - CloseQemuDetachedVolumes_6) { - int fd = 6; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - FdImage_t fd_image; - fd_image.filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, &fd_image)); - - g_qemuPoolVolumes.push_back("rbd/volume01"); - NebdServerMocker mock; - EXPECT_CALL(mock, ReadQemuXmls()).WillOnce(testing::Return(0)); - std::string lock_file = "/tmp/unit_test_lockfile"; - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lock_file)); - - EXPECT_EQ(0, CloseQemuDetachedVolumes(3)); -} - -TEST(CheckCmdline, CheckCmdline_1) { - char uuid[5]; - NebdServerMocker mock; - // EXPECT_CALL(mock, open(::testing::_, ::testing::_, - // ::testing::_)).WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, CheckCmdline(6666666, "test", uuid, 4)); -} - -TEST(CheckCmdline, CheckCmdline_2) { - char uuid[5]; - NebdServerMocker mock; - // EXPECT_CALL(mock, open(::testing::_, ::testing::_, - // ::testing::_)).WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, CheckCmdline(1, "test", uuid, 4)); -} - -TEST(CheckProc, CheckProc_1) { - g_oldPid = 66666; - EXPECT_EQ(-1, CheckProc("test")); -} - -TEST(CheckProc, CheckProc_2) { - g_oldPid = 1; - EXPECT_EQ(0, CheckProc("test")); -} - -int main(int argc, char* argv[]) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tests/part2/TestHeartbeat3.cpp b/tests/part2/TestHeartbeat3.cpp deleted file mode 100644 index 56e701808c..0000000000 --- a/tests/part2/TestHeartbeat3.cpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/part2/heartbeat.h" -#include "tests/part2/test_heartbeat3.h" - -TEST(CheckProc, CheckProc_1) { - g_oldPid = 66666; - EXPECT_EQ(-1, CheckProc("test")); -} - -TEST(CheckProc, CheckProc_2) { - g_oldPid = 1; - EXPECT_EQ(0, CheckProc("test")); -} - -TEST(CheckProc, CheckProc_3) { - g_oldPid = -1; - NebdServerMocker mock; - EXPECT_CALL(mock, CheckCmdline(::testing::_, ::testing::_, ::testing::_, - ::testing::_)) - .WillRepeatedly(testing::Return(-1)); - EXPECT_EQ(-1, CheckProc("test")); -} - -int main(int argc, char *argv[]) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tests/part2/TestRados.cpp b/tests/part2/TestRados.cpp deleted file mode 100644 index 576e695196..0000000000 --- a/tests/part2/TestRados.cpp +++ /dev/null @@ -1,334 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/part2/rados_interface.h" -#include "tests/part2/test_rados.h" - -TEST(OpenImage, OpenImage_1) { - rados_t cluster; - const char* poolname = "rbd"; - const char* volname = "volume01"; - int fd = 4; - char* filename = new char[5](); - filename[0] = '1'; - NebdServerMocker mock; - EXPECT_CALL(mock, - rados_ioctx_create(::testing::_, ::testing::_, ::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, OpenImage(&cluster, poolname, volname, fd, filename)); -} - -TEST(OpenImage, OpenImage_2) { - rados_t cluster; - const char* poolname = "rbd"; - const char* volname = "volume01"; - int fd = 4; - char* filename = new char[5](); - filename[0] = '1'; - NebdServerMocker mock; - EXPECT_CALL(mock, - rados_ioctx_create(::testing::_, ::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL( - mock, rbd_open(::testing::_, ::testing::_, ::testing::_, ::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_CALL(mock, rados_ioctx_destroy(::testing::_)) - .WillOnce(testing::Return()); - EXPECT_EQ(-1, OpenImage(&cluster, poolname, volname, fd, filename)); -} - -TEST(OpenImage, OpenImage_3) { - rados_t cluster; - const char* poolname = "rbd"; - const char* volname = "volume01"; - int fd = 4; - char* filename = new char[5](); - filename[0] = '1'; - NebdServerMocker mock; - EXPECT_CALL(mock, - rados_ioctx_create(::testing::_, ::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL( - mock, rbd_open(::testing::_, ::testing::_, ::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_EQ(4, OpenImage(&cluster, poolname, volname, fd, filename)); -} - -TEST(CloseImage, CloseImage_1) { - g_imageMap.clear(); - EXPECT_EQ(-1, CloseImage(4)); -} - -TEST(CloseImage, CloseImage_2) { - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = new char[5](); - filename[0] = '1'; - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - NebdServerMocker mock; - EXPECT_CALL(mock, rados_ioctx_destroy(::testing::_)) - .WillOnce(testing::Return()); - EXPECT_CALL(mock, CloseRados(::testing::_)).WillOnce(testing::Return()); - EXPECT_CALL(mock, rbd_close(::testing::_)).WillOnce(testing::Return(0)); - - EXPECT_EQ(0, CloseImage(4)); -} - -TEST(ConnectRados, ConnectRados_1) { - NebdServerMocker mock; - EXPECT_CALL(mock, rados_create(::testing::_, ::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_EQ(NULL, ConnectRados("test")); -} - -TEST(ConnectRados, ConnectRados_2) { - ::system("rm /etc/nebd/nebd-server.conf"); - NebdServerMocker mock; - EXPECT_CALL(mock, rados_create(::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_EQ(NULL, ConnectRados("test")); -} - -TEST(ConnectRados, ConnectRados_3) { - ::system("mkdir -p /etc/nebd"); - ::system("touch /etc/nebd/nebd-server.conf"); - NebdServerMocker mock; - EXPECT_CALL(mock, rados_create(::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rados_conf_read_file(::testing::_, ::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_CALL(mock, rados_conf_set(::testing::_, ::testing::_, ::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_CALL(mock, CloseRados(::testing::_)).WillOnce(testing::Return()); - EXPECT_EQ(NULL, ConnectRados("test")); -} - -TEST(ConnectRados, ConnectRados_4) { - ::system("mkdir -p /etc/nebd"); - ::system("touch /etc/nebd/nebd-server.conf"); - NebdServerMocker mock; - EXPECT_CALL(mock, rados_create(::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rados_conf_read_file(::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rados_conf_set(::testing::_, ::testing::_, ::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_CALL(mock, CloseRados(::testing::_)).WillOnce(testing::Return()); - EXPECT_EQ(NULL, ConnectRados("test")); -} - -TEST(ConnectRados, ConnectRados_5) { - ::system("mkdir -p /etc/nebd"); - ::system("touch /etc/nebd/nebd-server.conf"); - NebdServerMocker mock; - EXPECT_CALL(mock, rados_create(::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rados_conf_read_file(::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rados_conf_set(::testing::_, ::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, CloseRados(::testing::_)).WillOnce(testing::Return()); - EXPECT_CALL(mock, rados_connect(::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_EQ(NULL, ConnectRados("test")); -} -/* -TEST(ConnectRados, ConnectRados_6) -{ - ::system("mkdir -p /etc/nebd"); - ::system("touch /etc/nebd/nebd-server.conf"); - NebdServerMocker mock; - EXPECT_CALL(mock, rados_create(::testing::_, -::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rados_conf_read_file(::testing::_, -::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rados_conf_set(::testing::_, ::testing::_, -::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, CloseRados(::testing::_)).WillOnce(testing::Return()); - EXPECT_CALL(mock, rados_connect(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_EQ(NULL, ConnectRados("test")); -} -*/ -/* -TEST(open_image, open_image_1) -{ - NebdServerMocker mock; - rados_t* cluster = new rados_t; - const char* poolname = "rbd"; - const char* volname = "volume01"; - int fd = 4; - char* filename = new char[5](); - filename[0] = '1'; - filename[2] = '1'; - - EXPECT_CALL(mock, rados_ioctx_create(::testing::_, ::testing::_, -::testing::_)).WillOnce(testing::Return(-1)); - - EXPECT_EQ(-1, open_image(cluster, poolname, volname, fd, filename)); -} -*/ -/* -TEST(ConnectRados, ConnectRados_1) -{ - NebdServerMocker mock; - rados_t* cluster = new rados_t; - const char* poolname = "rbd"; - const char* volname = "volume01"; - int fd = 4; - char* filename = new char[5](); - filename[0] = '1'; - filename[2] = '1'; - - rados_t* rados = NULL; - EXPECT_CALL(mock, rados_create(::testing::_, -::testing::_)).WillOnce(testing::Return(-1)); - const char* mon_host = "ttt"; - EXPECT_EQ(rados, ConnectRados(mon_host)); -} - -TEST(ConnectRados, ConnectRados_2) -{ - NebdServerMocker mock; - rados_t* cluster = new rados_t; - const char* poolname = "rbd"; - const char* volname = "volume01"; - int fd = 4; - char* filename = new char[5](); - filename[0] = '1'; - filename[2] = '1'; - - rados_t* rados = NULL; - //EXPECT_CALL(mock, rados_create(::testing::_, -::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rados_conf_read_file(::testing::_, -::testing::_)).WillOnce(testing::Return(-1)); - const char* mon_host = "ttt"; - EXPECT_EQ(rados, ConnectRados(mon_host)); -} -*/ -TEST(FilenameFdExist, FilenameFdExist_1) { - NebdServerMocker mock; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = new char[5](); - filename[0] = '1'; - filename[1] = '1'; - - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - const char* filename_input = "11"; - EXPECT_EQ(4, FilenameFdExist(const_cast(filename_input))); -} - -TEST(FilenameFdExist, FilenameFdExist_2) { - NebdServerMocker mock; - - g_imageMap.clear(); - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = new char[5](); - filename[0] = '1'; - - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.insert(std::pair(fd, fd_image)); - - const char* filename_input = "11"; - EXPECT_EQ(0, FilenameFdExist(const_cast(filename_input))); -} - -TEST(FdExist, FdExist_1) { - NebdServerMocker mock; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = new char[5](); - filename[0] = '1'; - filename[2] = '1'; - - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.insert(std::pair(fd, fd_image)); - - rbd_image_t* image_ret; - // EXPECT_CALL(mock, rados_ioctx_create(::testing::_, ::testing::_, - // ::testing::_)).WillOnce(testing::Return(-1)); - - EXPECT_TRUE(FdExist(4, &image_ret)); -} - -TEST(FdExist, FdExist_2) { - NebdServerMocker mock; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = new char[5](); - filename[0] = '1'; - filename[2] = '1'; - - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.insert(std::pair(fd, fd_image)); - - rbd_image_t* image_ret; - // EXPECT_CALL(mock, rados_ioctx_create(::testing::_, ::testing::_, - // ::testing::_)).WillOnce(testing::Return(-1)); - - EXPECT_FALSE(FdExist(8, &image_ret)); -} - -int main(int argc, char* argv[]) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tests/part2/TestReload.cpp b/tests/part2/TestReload.cpp deleted file mode 100644 index dca5ca7c4c..0000000000 --- a/tests/part2/TestReload.cpp +++ /dev/null @@ -1,107 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/part2/reload.h" -#include "tests/part2/test_reload.h" - -TEST(Reload, reload_1) { - NebdServerMocker mock; - std::string metadata_file = "/var/444"; - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - - EXPECT_EQ(-1, Reload()); -} - -TEST(Reload, reload_2) { - NebdServerMocker mock; - std::string metadata_file = "/tmp/unit_test"; - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - - boost::property_tree::ptree root; - boost::property_tree::ptree items; - - try { - root.put("volume", 5); - boost::property_tree::write_json(metadata_file, root); - } catch (boost::property_tree::ptree_error pt) { - LOG(ERROR) << "write json throw an exception. " << pt.what(); - return; - } - - EXPECT_EQ(-1, Reload()); -} - -TEST(Reload, reload_3) { - NebdServerMocker mock; - std::string metadata_file = "/tmp/unit_test"; - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - - boost::property_tree::ptree root; - boost::property_tree::ptree items; - - try { - root.put("port", 10); - boost::property_tree::write_json(metadata_file, root); - } catch (boost::property_tree::ptree_error pt) { - LOG(ERROR) << "write json throw an exception. " << pt.what(); - return; - } - - EXPECT_EQ(10, Reload()); -} - -TEST(Reload, reload_4) { - NebdServerMocker mock; - std::string metadata_file = "/tmp/unit_test"; - EXPECT_CALL(mock, GetUuidFile()).WillOnce(testing::Return(metadata_file)); - - boost::property_tree::ptree root; - boost::property_tree::ptree items; - - const char* filename1 = ""; - const char* filename2 = "unittest"; - try { - root.put("port", 10); - boost::property_tree::write_json(metadata_file, root); - - boost::property_tree::ptree item1, item2; - item1.put("filename", filename1); - item1.put("fd", 1); - items.push_back(std::make_pair("", item1)); - item2.put("filename", filename2); - item2.put("fd", 2); - items.push_back(std::make_pair("", item2)); - - root.put_child("volumes", items); - - boost::property_tree::write_json(metadata_file, root); - } catch (boost::property_tree::ptree_error pt) { - LOG(ERROR) << "write json throw an exception. " << pt.what(); - return; - } - - EXPECT_CALL(mock, ReloadCephVolume(1, ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, ReloadCephVolume(2, ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_EQ(10, Reload()); -} - -int main(int argc, char* argv[]) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tests/part2/TestReload2.cpp b/tests/part2/TestReload2.cpp deleted file mode 100644 index 79819f85d8..0000000000 --- a/tests/part2/TestReload2.cpp +++ /dev/null @@ -1,104 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/part2/reload.h" -#include "tests/part2/test_reload2.h" - -TEST(ReloadCephVolume, ReloadCephVolume_1) { - const char* tmp = "rbdrbdvolume03auth_supportednonemon_host=10.182.30.27"; - char* filename = new char[strlen(tmp) + 1](); - snprintf(filename, strlen(tmp) + 1, "%s", tmp); - EXPECT_EQ(-1, ReloadCephVolume(4, filename)); -} - -TEST(ReloadCephVolume, ReloadCephVolume_2) { - const char* tmp = "rbdrbdvolume03auth_supported=nonemon_host=10.182.30.27"; - char* filename = new char[strlen(tmp) + 1](); - snprintf(filename, strlen(tmp) + 1, "%s", tmp); - EXPECT_EQ(-1, ReloadCephVolume(4, filename)); -} - -TEST(ReloadCephVolume, ReloadCephVolume_3) { - const char* tmp = - "rbd:rbdvolume03:auth_supported=none:mon_host=10.182.30.27:6789"; - char* filename = new char[strlen(tmp) + 1](); - snprintf(filename, strlen(tmp) + 1, "%s", tmp); - - EXPECT_EQ(-1, ReloadCephVolume(4, filename)); -} - -TEST(ReloadCephVolume, ReloadCephVolume_4) { - const char* tmp = "rbd:rbd/volume03:auth_supported=none:mon_host"; - char* filename = new char[strlen(tmp) + 1](); - snprintf(filename, strlen(tmp) + 1, "%s", tmp); - - EXPECT_EQ(-1, ReloadCephVolume(4, filename)); -} - -TEST(ReloadCephVolume, ReloadCephVolume_8) { - const char* tmp = - "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; - char* filename = new char[strlen(tmp) + 1](); - snprintf(filename, strlen(tmp) + 1, "%s", tmp); - NebdServerMocker mock; - EXPECT_CALL(mock, ConnectRados(::testing::_)) - .WillOnce(testing::ReturnNull()); - EXPECT_EQ(-1, ReloadCephVolume(4, filename)); -} - -TEST(ReloadCephVolume, ReloadCephVolume_9) { - const char* tmp = - "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; - char* filename = new char[strlen(tmp) + 1](); - snprintf(filename, strlen(tmp) + 1, "%s", tmp); - - NebdServerMocker mock; - - rados_t* cluster = new rados_t; - int ret = rados_create(cluster, "admin"); - if (ret < 0) { - delete cluster; - return; - } - EXPECT_CALL(mock, ConnectRados(::testing::_)) - .WillOnce(testing::Return(cluster)); - EXPECT_CALL(mock, OpenImage(::testing::_, ::testing::_, ::testing::_, 4, - ::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, ReloadCephVolume(4, filename)); -} - -TEST(ReloadCephVolume, ReloadCephVolume_10) { - const char* tmp = - "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; - char* filename = new char[strlen(tmp) + 1](); - snprintf(filename, strlen(tmp) + 1, "%s", tmp); - - NebdServerMocker mock; - rados_t* cluster = new rados_t; - EXPECT_CALL(mock, ConnectRados(::testing::_)) - .WillOnce(testing::Return(cluster)); - EXPECT_CALL(mock, OpenImage(::testing::_, ::testing::_, ::testing::_, 4, - ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_EQ(0, ReloadCephVolume(4, filename)); -} - -int main(int argc, char* argv[]) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tests/part2/TestRpcCeph.cpp b/tests/part2/TestRpcCeph.cpp deleted file mode 100644 index 6b74fad2c6..0000000000 --- a/tests/part2/TestRpcCeph.cpp +++ /dev/null @@ -1,1252 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/common/client.pb.h" -#include "src/part2/rados_interface.h" -#include "src/part2/rpc_server.h" -#include "tests/part2/test_rpcceph.h" - -class RpcServiceTestClosure : public ::google::protobuf::Closure { - public: - explicit RpcServiceTestClosure(int sleepUs = 0) : sleep_(sleepUs) {} - virtual ~RpcServiceTestClosure() = default; - - void Run() override { - if (0 != sleep_) { - ::usleep(sleep_); - LOG(INFO) << "return rpc"; - } - } - - private: - int sleep_; -}; - -TEST(RpcRequestCephTest, RbdFinishAioWrite_1) { - nebd::client::WriteResponse response; - brpc::Controller cntl; - nebd::client::WriteRequest request; - request.set_fd(4); - request.set_offset(4); - request.set_size(4); - RpcServiceTestClosure done; - AsyncWrite* writejob = NULL; - writejob = new AsyncWrite; - writejob->done = &done; - writejob->response = &response; - writejob->request = &request; - writejob->cntl = &cntl; - - rbd_completion_t c; - rbd_aio_create_completion(NULL, NULL, &c); - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_get_return_value(::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rbd_aio_release(::testing::_)) - .WillOnce(testing::Return()); - response.set_retcode(nebd::client::kNoOK); - RbdFinishAioWrite(c, writejob); - EXPECT_EQ(nebd::client::kOK, response.retcode()); -} - -TEST(RpcRequestCephTest, RbdFinishAioWrite_2) { - nebd::client::WriteResponse response; - brpc::Controller cntl; - nebd::client::WriteRequest request; - request.set_fd(4); - request.set_offset(4); - request.set_size(4); - RpcServiceTestClosure done; - AsyncWrite* writejob = NULL; - writejob = new AsyncWrite; - writejob->done = &done; - writejob->response = &response; - writejob->request = &request; - writejob->cntl = &cntl; - - rbd_completion_t c; - rbd_aio_create_completion(NULL, NULL, &c); - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_get_return_value(::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_CALL(mock, rbd_aio_release(::testing::_)) - .WillOnce(testing::Return()); - response.set_retcode(nebd::client::kNoOK); - RbdFinishAioWrite(c, writejob); - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(RpcRequestCephTest, RbdFinishAioRead_1) { - nebd::client::ReadResponse response; - brpc::Controller cntl; - nebd::client::ReadRequest request; - request.set_fd(4); - request.set_offset(4); - request.set_size(4); - RpcServiceTestClosure done; - AsyncRead* readjob = NULL; - readjob = new AsyncRead; - readjob->done = &done; - readjob->response = &response; - readjob->request = &request; - readjob->cntl = &cntl; - readjob->buf = new char[5](); - rbd_completion_t c; - rbd_aio_create_completion(NULL, NULL, &c); - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_get_return_value(::testing::_)) - .WillOnce(testing::Return(1)); - EXPECT_CALL(mock, rbd_aio_release(::testing::_)) - .WillOnce(testing::Return()); - response.set_retcode(nebd::client::kNoOK); - RbdFinishAioRead(c, readjob); - EXPECT_EQ(nebd::client::kOK, response.retcode()); -} - -TEST(RpcRequestCephTest, RbdFinishAioRead_2) { - nebd::client::ReadResponse response; - brpc::Controller cntl; - nebd::client::ReadRequest request; - request.set_fd(4); - request.set_offset(4); - request.set_size(4); - RpcServiceTestClosure done; - AsyncRead* readjob = NULL; - readjob = new AsyncRead; - readjob->done = &done; - readjob->response = &response; - readjob->request = &request; - readjob->cntl = &cntl; - readjob->buf = new char[5](); - rbd_completion_t c; - rbd_aio_create_completion(NULL, NULL, &c); - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_get_return_value(::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rbd_aio_release(::testing::_)) - .WillOnce(testing::Return()); - response.set_retcode(nebd::client::kNoOK); - RbdFinishAioRead(c, readjob); - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(RpcRequestCephTest, RbdFinishAioDiscard_1) { - nebd::client::DiscardResponse response; - brpc::Controller cntl; - nebd::client::DiscardRequest request; - request.set_fd(4); - request.set_offset(1); - request.set_size(1); - RpcServiceTestClosure done; - AsyncDiscard* discardjob = NULL; - discardjob = new AsyncDiscard; - discardjob->done = &done; - discardjob->response = &response; - discardjob->request = &request; - - rbd_completion_t c; - rbd_aio_create_completion(NULL, NULL, &c); - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_get_return_value(::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rbd_aio_release(::testing::_)) - .WillOnce(testing::Return()); - response.set_retcode(nebd::client::kNoOK); - RbdFinishAioDiscard(c, discardjob); - EXPECT_EQ(nebd::client::kOK, response.retcode()); -} - -TEST(RpcRequestCephTest, RbdFinishAioDiscard_2) { - nebd::client::DiscardResponse response; - brpc::Controller cntl; - nebd::client::DiscardRequest request; - request.set_fd(4); - request.set_offset(1); - request.set_size(1); - RpcServiceTestClosure done; - AsyncDiscard* discardjob = NULL; - discardjob = new AsyncDiscard; - discardjob->done = &done; - discardjob->response = &response; - discardjob->request = &request; - - rbd_completion_t c; - rbd_aio_create_completion(NULL, NULL, &c); - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_get_return_value(::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_CALL(mock, rbd_aio_release(::testing::_)) - .WillOnce(testing::Return()); - response.set_retcode(nebd::client::kNoOK); - RbdFinishAioDiscard(c, discardjob); - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(RpcRequestCephTest, RbdFinishAioFlush_1) { - nebd::client::FlushResponse response; - nebd::client::FlushRequest request; - request.set_fd(3); - RpcServiceTestClosure done; - AsyncFlush* flushjob = NULL; - flushjob = new AsyncFlush; - flushjob->done = &done; - flushjob->response = &response; - flushjob->request = &request; - rbd_completion_t c; - rbd_aio_create_completion(NULL, NULL, &c); - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_get_return_value(::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rbd_aio_release(::testing::_)) - .WillOnce(testing::Return()); - response.set_retcode(nebd::client::kNoOK); - RbdFinishAioFlush(c, flushjob); - EXPECT_EQ(nebd::client::kOK, response.retcode()); -} - -TEST(RpcRequestCephTest, RbdFinishAioFlush_2) { - nebd::client::FlushResponse response; - nebd::client::FlushRequest request; - request.set_fd(3); - RpcServiceTestClosure done; - AsyncFlush* flushjob = NULL; - flushjob = new AsyncFlush; - flushjob->done = &done; - flushjob->response = &response; - flushjob->request = &request; - rbd_completion_t c; - rbd_aio_create_completion(NULL, NULL, &c); - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_get_return_value(::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_CALL(mock, rbd_aio_release(::testing::_)) - .WillOnce(testing::Return()); - response.set_retcode(nebd::client::kNoOK); - RbdFinishAioFlush(c, flushjob); - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(RpcRequestCephTest, InvalidateCache_1) { - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestCeph cephImpl; - - EXPECT_EQ(-1, cephImpl.InvalidateCache(3)); -} - -TEST(RpcRequestCephTest, InvalidateCache_2) { - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_invalidate_cache(::testing::_)) - .WillOnce(testing::Return(-1)); - - RpcRequestCeph cephImpl; - EXPECT_EQ(-1, cephImpl.InvalidateCache(4)); -} - -TEST(RpcRequestCephTest, InvalidateCache_3) { - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_invalidate_cache(::testing::_)) - .WillOnce(testing::Return(0)); - - RpcRequestCeph cephImpl; - EXPECT_EQ(0, cephImpl.InvalidateCache(4)); -} - -TEST(RpcRequestCephTest, Resize_1) { - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestCeph cephImpl; - - EXPECT_EQ(-1, cephImpl.Resize(3, 1024)); -} - -TEST(RpcRequestCephTest, Resize_2) { - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_resize(::testing::_, ::testing::_)) - .WillOnce(testing::Return(-1)); - - RpcRequestCeph cephImpl; - EXPECT_EQ(-1, cephImpl.Resize(4, 1024)); -} - -TEST(RpcRequestCephTest, Resize_3) { - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_resize(::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - - RpcRequestCeph cephImpl; - EXPECT_EQ(0, cephImpl.Resize(4, 1024)); -} - -TEST(RpcRequestCephTest, GetInfo_1) { - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestCeph cephImpl; - nebd::client::GetInfoResponse response; - EXPECT_EQ(-1, cephImpl.GetInfo(3, &response)); -} - -TEST(RpcRequestCephTest, GetInfo_2) { - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_stat(::testing::_, ::testing::_, ::testing::_)) - .WillOnce(testing::Return(-1)); - - RpcRequestCeph cephImpl; - nebd::client::GetInfoResponse response; - EXPECT_EQ(-1, cephImpl.GetInfo(4, &response)); -} - -TEST(RpcRequestCephTest, GetInfo_3) { - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_stat(::testing::_, ::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - - RpcRequestCeph cephImpl; - nebd::client::GetInfoResponse response; - EXPECT_EQ(0, cephImpl.GetInfo(4, &response)); -} - -TEST(RpcRequestCephTest, StatFile_1) { - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestCeph cephImpl; - - nebd::client::StatFileResponse response; - EXPECT_EQ(-1, cephImpl.StatFile(3, &response)); -} - -TEST(RpcRequestCephTest, StatFile_2) { - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_stat(::testing::_, ::testing::_, ::testing::_)) - .WillOnce(testing::Return(-1)); - RpcRequestCeph cephImpl; - nebd::client::StatFileResponse response; - EXPECT_EQ(-1, cephImpl.StatFile(4, &response)); -} - -TEST(RpcRequestCephTest, StatFile_3) { - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_stat(::testing::_, ::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - RpcRequestCeph cephImpl; - nebd::client::StatFileResponse response; - EXPECT_EQ(0, cephImpl.StatFile(4, &response)); -} - -TEST(RpcRequestCephTest, CloseFile_1) { - NebdServerMocker mock; - std::string lockfile = "/tmp/unit_test"; - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(-1)); - - RpcRequestCeph cephImpl; - nebd::client::StatFileResponse response; - EXPECT_EQ(-1, cephImpl.CloseFile(4)); -} - -TEST(RpcRequestCephTest, CloseFile_2) { - NebdServerMocker mock; - std::string lockfile = "/tmp/unit_test"; - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(-1)); - - RpcRequestCeph cephImpl; - nebd::client::StatFileResponse response; - EXPECT_EQ(-1, cephImpl.CloseFile(4)); -} - -TEST(RpcRequestCephTest, CloseFile_3) { - NebdServerMocker mock; - std::string lockfile = "/tmp/unit_test"; - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, RmFd(::testing::_)).WillOnce(testing::Return(-1)); - - RpcRequestCeph cephImpl; - nebd::client::StatFileResponse response; - EXPECT_EQ(-1, cephImpl.CloseFile(4)); -} - -TEST(RpcRequestCephTest, CloseFile_4) { - NebdServerMocker mock; - std::string lockfile = "/tmp/unit_test"; - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, RmFd(::testing::_)).WillOnce(testing::Return(0)); - - RpcRequestCeph cephImpl; - nebd::client::StatFileResponse response; - EXPECT_EQ(0, cephImpl.CloseFile(4)); -} - -TEST(RpcRequestCephTest, Write_1) { - nebd::client::WriteResponse response; - brpc::Controller cntl; - nebd::client::WriteRequest request; - request.set_fd(3); - request.set_offset(1); - request.set_size(1); - RpcServiceTestClosure done; - AsyncWrite* writejob = NULL; - writejob = new AsyncWrite; - writejob->done = &done; - writejob->response = &response; - writejob->request = &request; - writejob->cntl = &cntl; - - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - FdImage_t fd_image; - fd_image.filename = filename; - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, &fd_image)); - - RpcRequestCeph cephImpl; - - NebdServerMocker mock; - // EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, - // ::testing::_)).WillOnce(testing::Return(-1)); - // EXPECT_CALL(mock, rbd_aio_write(::testing::_, 1, 1, ::testing::_, - // ::testing::_)).WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, cephImpl.Write(writejob)); -} - -TEST(RpcRequestCephTest, Write_2) { - nebd::client::WriteResponse response; - brpc::Controller cntl; - nebd::client::WriteRequest request; - request.set_fd(4); - request.set_offset(4); - request.set_size(4); - RpcServiceTestClosure done; - AsyncWrite* writejob = NULL; - writejob = new AsyncWrite; - writejob->done = &done; - writejob->response = &response; - writejob->request = &request; - writejob->cntl = &cntl; - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestCeph cephImpl; - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, - ::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, cephImpl.Write(writejob)); -} - -TEST(RpcRequestCephTest, Write_3) { - nebd::client::WriteResponse response; - brpc::Controller cntl; - nebd::client::WriteRequest request; - request.set_fd(4); - request.set_offset(4); - request.set_size(4); - RpcServiceTestClosure done; - AsyncWrite* writejob = NULL; - writejob = new AsyncWrite; - writejob->done = &done; - writejob->response = &response; - writejob->request = &request; - writejob->cntl = &cntl; - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestCeph cephImpl; - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, - ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rbd_aio_write(::testing::_, ::testing::_, ::testing::_, - ::testing::_, ::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, cephImpl.Write(writejob)); -} - -TEST(RpcRequestCephTest, Write_4) { - nebd::client::WriteResponse response; - brpc::Controller cntl; - nebd::client::WriteRequest request; - request.set_fd(4); - request.set_offset(4); - request.set_size(4); - RpcServiceTestClosure done; - AsyncWrite* writejob = NULL; - writejob = new AsyncWrite; - writejob->done = &done; - writejob->response = &response; - writejob->request = &request; - writejob->cntl = &cntl; - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestCeph cephImpl; - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, - ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rbd_aio_write(::testing::_, ::testing::_, ::testing::_, - ::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_EQ(0, cephImpl.Write(writejob)); -} - -TEST(RpcRequestCephTest, Read_1) { - nebd::client::ReadResponse response; - brpc::Controller cntl; - nebd::client::ReadRequest request; - request.set_fd(3); - request.set_offset(1); - request.set_size(1); - RpcServiceTestClosure done; - AsyncRead* readjob = NULL; - readjob = new AsyncRead; - readjob->done = &done; - readjob->response = &response; - readjob->request = &request; - readjob->cntl = &cntl; - - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - FdImage_t fd_image; - fd_image.filename = filename; - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, &fd_image)); - - RpcRequestCeph cephImpl; - - NebdServerMocker mock; - // EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, - // ::testing::_)).WillOnce(testing::Return(-1)); - // EXPECT_CALL(mock, rbd_aio_write(::testing::_, 1, 1, ::testing::_, - // ::testing::_)).WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, cephImpl.Read(readjob)); -} - -TEST(RpcRequestCephTest, Read_2) { - nebd::client::ReadResponse response; - brpc::Controller cntl; - nebd::client::ReadRequest request; - request.set_fd(4); - request.set_offset(4); - request.set_size(1); - RpcServiceTestClosure done; - AsyncRead* readjob = NULL; - readjob = new AsyncRead; - readjob->done = &done; - readjob->response = &response; - readjob->request = &request; - readjob->cntl = &cntl; - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestCeph cephImpl; - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, - ::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, cephImpl.Read(readjob)); -} - -TEST(RpcRequestCephTest, Read_3) { - nebd::client::ReadResponse response; - brpc::Controller cntl; - nebd::client::ReadRequest request; - request.set_fd(4); - request.set_offset(4); - request.set_size(1); - RpcServiceTestClosure done; - AsyncRead* readjob = NULL; - readjob = new AsyncRead; - readjob->done = &done; - readjob->response = &response; - readjob->request = &request; - readjob->cntl = &cntl; - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestCeph cephImpl; - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, - ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rbd_aio_read(::testing::_, ::testing::_, ::testing::_, - ::testing::_, ::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, cephImpl.Read(readjob)); -} - -TEST(RpcRequestCephTest, Read_4) { - nebd::client::ReadResponse response; - brpc::Controller cntl; - nebd::client::ReadRequest request; - request.set_fd(4); - request.set_offset(4); - request.set_size(1); - RpcServiceTestClosure done; - AsyncRead* readjob = NULL; - readjob = new AsyncRead; - readjob->done = &done; - readjob->response = &response; - readjob->request = &request; - readjob->cntl = &cntl; - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestCeph cephImpl; - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, - ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rbd_aio_read(::testing::_, ::testing::_, ::testing::_, - ::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_EQ(0, cephImpl.Read(readjob)); -} - -TEST(RpcRequestCephTest, Discard_1) { - nebd::client::DiscardResponse response; - brpc::Controller cntl; - nebd::client::DiscardRequest request; - request.set_fd(3); - request.set_offset(1); - request.set_size(1); - RpcServiceTestClosure done; - AsyncDiscard* discardjob = NULL; - discardjob = new AsyncDiscard; - discardjob->done = &done; - discardjob->response = &response; - discardjob->request = &request; - - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - FdImage_t fd_image; - fd_image.filename = filename; - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, &fd_image)); - - RpcRequestCeph cephImpl; - - NebdServerMocker mock; - EXPECT_EQ(-1, cephImpl.Discard(discardjob)); -} - -TEST(RpcRequestCephTest, Discard_2) { - nebd::client::DiscardResponse response; - brpc::Controller cntl; - nebd::client::DiscardRequest request; - request.set_fd(4); - request.set_offset(1); - request.set_size(1); - RpcServiceTestClosure done; - AsyncDiscard* discardjob = NULL; - discardjob = new AsyncDiscard; - discardjob->done = &done; - discardjob->response = &response; - discardjob->request = &request; - - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - FdImage_t fd_image; - fd_image.filename = filename; - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, &fd_image)); - - RpcRequestCeph cephImpl; - - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, - ::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, cephImpl.Discard(discardjob)); -} - -TEST(RpcRequestCephTest, Discard_3) { - nebd::client::DiscardResponse response; - brpc::Controller cntl; - nebd::client::DiscardRequest request; - request.set_fd(4); - request.set_offset(1); - request.set_size(1); - RpcServiceTestClosure done; - AsyncDiscard* discardjob = NULL; - discardjob = new AsyncDiscard; - discardjob->done = &done; - discardjob->response = &response; - discardjob->request = &request; - - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - FdImage_t fd_image; - fd_image.filename = filename; - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, &fd_image)); - - RpcRequestCeph cephImpl; - - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, - ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rbd_aio_discard(::testing::_, ::testing::_, ::testing::_, - ::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, cephImpl.Discard(discardjob)); -} - -TEST(RpcRequestCephTest, Discard_4) { - nebd::client::DiscardResponse response; - brpc::Controller cntl; - nebd::client::DiscardRequest request; - request.set_fd(4); - request.set_offset(1); - request.set_size(1); - RpcServiceTestClosure done; - AsyncDiscard* discardjob = NULL; - discardjob = new AsyncDiscard; - discardjob->done = &done; - discardjob->response = &response; - discardjob->request = &request; - - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - FdImage_t fd_image; - fd_image.filename = filename; - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, &fd_image)); - - RpcRequestCeph cephImpl; - - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, - ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rbd_aio_discard(::testing::_, ::testing::_, ::testing::_, - ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_EQ(0, cephImpl.Discard(discardjob)); -} - -TEST(RpcRequestCephTest, Flush_1) { - nebd::client::FlushResponse response; - nebd::client::FlushRequest request; - request.set_fd(3); - RpcServiceTestClosure done; - AsyncFlush* flushjob = NULL; - flushjob = new AsyncFlush; - flushjob->done = &done; - flushjob->response = &response; - flushjob->request = &request; - - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - FdImage_t fd_image; - fd_image.filename = filename; - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, &fd_image)); - - RpcRequestCeph cephImpl; - - NebdServerMocker mock; - EXPECT_EQ(-1, cephImpl.Flush(flushjob)); -} - -TEST(RpcRequestCephTest, Flush_2) { - nebd::client::FlushResponse response; - nebd::client::FlushRequest request; - request.set_fd(4); - RpcServiceTestClosure done; - AsyncFlush* flushjob = NULL; - flushjob = new AsyncFlush; - flushjob->done = &done; - flushjob->response = &response; - flushjob->request = &request; - - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - FdImage_t fd_image; - fd_image.filename = filename; - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, &fd_image)); - - RpcRequestCeph cephImpl; - - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, - ::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, cephImpl.Flush(flushjob)); -} - -TEST(RpcRequestCephTest, Flush_3) { - nebd::client::FlushResponse response; - nebd::client::FlushRequest request; - request.set_fd(4); - RpcServiceTestClosure done; - AsyncFlush* flushjob = NULL; - flushjob = new AsyncFlush; - flushjob->done = &done; - flushjob->response = &response; - flushjob->request = &request; - - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - FdImage_t fd_image; - fd_image.filename = filename; - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, &fd_image)); - - RpcRequestCeph cephImpl; - - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, - ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rbd_aio_flush(::testing::_, ::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, cephImpl.Flush(flushjob)); -} - -TEST(RpcRequestCephTest, Flush_4) { - nebd::client::FlushResponse response; - nebd::client::FlushRequest request; - request.set_fd(4); - RpcServiceTestClosure done; - AsyncFlush* flushjob = NULL; - flushjob = new AsyncFlush; - flushjob->done = &done; - flushjob->response = &response; - flushjob->request = &request; - - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - FdImage_t fd_image; - fd_image.filename = filename; - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, &fd_image)); - - RpcRequestCeph cephImpl; - - NebdServerMocker mock; - EXPECT_CALL(mock, rbd_aio_create_completion(::testing::_, ::testing::_, - ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_CALL(mock, rbd_aio_flush(::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_EQ(0, cephImpl.Flush(flushjob)); -} - -TEST(OpenFile, OpenFile_1) { - const char* tmp = "rbdrbdvolume03auth_supportednonemon_host"; - char* filename = new char[strlen(tmp) + 1](); - snprintf(filename, strlen(tmp) + 1, "%s", tmp); - RpcRequestCeph cephImpl; - EXPECT_EQ(-1, cephImpl.OpenFile(filename)); -} - -TEST(OpenFile, OpenFile_2) { - const char* tmp = "rbdrbdvolume03auth_supported=nonemon_host=10.182.30.27"; - char* filename = new char[strlen(tmp) + 1](); - snprintf(filename, strlen(tmp) + 1, "%s", tmp); - RpcRequestCeph cephImpl; - EXPECT_EQ(-1, cephImpl.OpenFile(filename)); -} - -TEST(OpenFile, OpenFile_3) { - const char* tmp = - "rbd:rbdvolume03:auth_supported=none:mon_host=10.182.30.27:6789"; - char* filename = new char[strlen(tmp) + 1](); - snprintf(filename, strlen(tmp) + 1, "%s", tmp); - RpcRequestCeph cephImpl; - EXPECT_EQ(-1, cephImpl.OpenFile(filename)); -} - -TEST(OpenFile, OpenFile_4) { - const char* tmp = "rbd:rbd/volume03:auth_supported=none:mon_host"; - char* filename = new char[strlen(tmp) + 1](); - snprintf(filename, strlen(tmp) + 1, "%s", tmp); - RpcRequestCeph cephImpl; - EXPECT_EQ(-1, cephImpl.OpenFile(filename)); -} - -TEST(OpenFile, OpenFile_5) { - const char* tmp = - "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; - char* filename = new char[strlen(tmp) + 1](); - snprintf(filename, strlen(tmp) + 1, "%s", tmp); - RpcRequestCeph cephImpl; - NebdServerMocker mock; - EXPECT_CALL(mock, FilenameFdExist(::testing::_)) - .WillOnce(testing::Return(3)); - EXPECT_EQ(3, cephImpl.OpenFile(filename)); -} - -TEST(OpenFile, OpenFile_6) { - const char* filename = - "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; - RpcRequestCeph cephImpl; - NebdServerMocker mock; - EXPECT_CALL(mock, FilenameFdExist(::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_CALL(mock, ConnectRados(::testing::_)) - .WillOnce(testing::ReturnNull()); - EXPECT_EQ(-1, cephImpl.OpenFile(const_cast(filename))); -} - -TEST(OpenFile, OpenFile_7) { - const char* filename = - "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; - RpcRequestCeph cephImpl; - rados_t cluster; - rados_create(&cluster, "admin"); - NebdServerMocker mock; - EXPECT_CALL(mock, FilenameFdExist(::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_CALL(mock, ConnectRados(::testing::_)) - .WillOnce(testing::Return(&cluster)); - EXPECT_CALL(mock, CloseRados(::testing::_)).WillOnce(testing::Return()); - EXPECT_CALL(mock, OpenImage(::testing::_, ::testing::_, ::testing::_, - ::testing::_, ::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, cephImpl.OpenFile(const_cast(filename))); -} - -TEST(OpenFile, OpenFile_8) { - const char* filename = - "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; - RpcRequestCeph cephImpl; - rados_t cluster; - rados_create(&cluster, "admin"); - NebdServerMocker mock; - EXPECT_CALL(mock, FilenameFdExist(::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_CALL(mock, ConnectRados(::testing::_)) - .WillOnce(testing::Return(&cluster)); - EXPECT_CALL(mock, OpenImage(::testing::_, ::testing::_, ::testing::_, - ::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - std::string lockfile = "/tmp/unittest"; - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); - EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, cephImpl.OpenFile(const_cast(filename))); -} - -TEST(OpenFile, OpenFile_9) { - const char* filename = - "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; - RpcRequestCeph cephImpl; - rados_t cluster; - rados_create(&cluster, "admin"); - NebdServerMocker mock; - EXPECT_CALL(mock, FilenameFdExist(::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_CALL(mock, ConnectRados(::testing::_)) - .WillOnce(testing::Return(&cluster)); - EXPECT_CALL(mock, OpenImage(::testing::_, ::testing::_, ::testing::_, - ::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - std::string lockfile = "/tmp/unittest"; - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); - EXPECT_CALL(mock, CloseImage(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, GenerateFd(::testing::_, ::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_EQ(-1, cephImpl.OpenFile(const_cast(filename))); -} - -TEST(OpenFile, OpenFile_10) { - const char* filename = - "rbd:rbd/volume03:auth_supported=none:mon_host=10.182.30.27\\:6789"; - RpcRequestCeph cephImpl; - rados_t cluster; - rados_create(&cluster, "admin"); - NebdServerMocker mock; - EXPECT_CALL(mock, FilenameFdExist(::testing::_)) - .WillOnce(testing::Return(-1)); - EXPECT_CALL(mock, ConnectRados(::testing::_)) - .WillOnce(testing::Return(&cluster)); - EXPECT_CALL(mock, OpenImage(::testing::_, ::testing::_, ::testing::_, - ::testing::_, ::testing::_)) - .WillOnce(testing::Return(5)); - std::string lockfile = "/tmp/unittest"; - EXPECT_CALL(mock, GetUuidLockfile()).WillOnce(testing::Return(lockfile)); - EXPECT_CALL(mock, LockFile(::testing::_)).WillOnce(testing::Return(0)); - EXPECT_CALL(mock, GenerateFd(::testing::_, ::testing::_)) - .WillOnce(testing::Return(0)); - EXPECT_EQ(5, cephImpl.OpenFile(const_cast(filename))); -} - -int main(int argc, char* argv[]) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tests/part2/TestRpcServer.cpp b/tests/part2/TestRpcServer.cpp deleted file mode 100644 index a36f15b2da..0000000000 --- a/tests/part2/TestRpcServer.cpp +++ /dev/null @@ -1,1187 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "src/common/client.pb.h" -#include "src/part2/rados_interface.h" -#include "src/part2/rpc_server.h" - -class RpcRequestTestOk : public RpcRequestCeph { - public: - RpcRequestTestOk() {} - virtual ~RpcRequestTestOk() {} - int OpenFile(char* filename) { return 100; } - int StatFile(int fd, nebd::client::StatFileResponse* response) { return 0; } - int Write(AsyncWrite* writejob) { return 0; } - int Read(AsyncRead* writejob) { return 0; } - int GetInfo(int fd, nebd::client::GetInfoResponse* response) { return 0; } - int CloseFile(int fd) { return 0; } - int Flush(AsyncFlush* flushjob) { return 0; } - int Discard(AsyncDiscard* discardjob) { return 0; } - int Resize(int fd, uint64_t size) { return 0; } - int InvalidateCache(int fd) { return 0; } -}; - -class RpcRequestTestNoOk : public RpcRequestCeph { - public: - RpcRequestTestNoOk() {} - virtual ~RpcRequestTestNoOk() {} - int OpenFile(char* filename) { return -1; } - int StatFile(int fd, nebd::client::StatFileResponse* response) { - return -1; - } - int Write(AsyncWrite* writejob) { return -1; } - int Read(AsyncRead* writejob) { return -1; } - int GetInfo(int fd, nebd::client::GetInfoResponse* response) { return -1; } - int CloseFile(int fd) { return -1; } - int Flush(AsyncFlush* flushjob) { return -1; } - int Discard(AsyncDiscard* discardjob) { return -1; } - int Resize(int fd, uint64_t size) { return -1; } - int InvalidateCache(int fd) { return -1; } -}; - -class RpcServiceTestClosure : public ::google::protobuf::Closure { - public: - explicit RpcServiceTestClosure(int sleepUs = 0) : sleep_(sleepUs) {} - virtual ~RpcServiceTestClosure() = default; - - void Run() override { - if (0 != sleep_) { - ::usleep(sleep_); - LOG(INFO) << "return rpc"; - } - } - - private: - int sleep_; -}; - -TEST(QemuClientServiceImplTest, OpenFile_1) { - nebd::client::OpenFileResponse response; - brpc::Controller cntl; - nebd::client::OpenFileRequest request; - request.set_filename( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.OpenFile(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, OpenFile_2) { - nebd::client::OpenFileResponse response; - brpc::Controller cntl; - nebd::client::OpenFileRequest request; - request.set_filename( - "nbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - RpcServiceTestClosure done; - - g_imageMap.clear(); - - QemuClientServiceImpl qemuImpl; - qemuImpl.OpenFile(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, OpenFile_3) { - nebd::client::OpenFileResponse response; - brpc::Controller cntl; - nebd::client::OpenFileRequest request; - request.set_filename( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - RpcServiceTestClosure done; - - g_imageMap.clear(); - RpcRequestTestNoOk* req = new RpcRequestTestNoOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.OpenFile(&cntl, &request, &response, &done); - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, OpenFile_4) { - nebd::client::OpenFileResponse response; - brpc::Controller cntl; - nebd::client::OpenFileRequest request; - request.set_filename( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - RpcServiceTestClosure done; - - g_imageMap.clear(); - RpcRequestTestOk* req = new RpcRequestTestOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.OpenFile(&cntl, &request, &response, &done); - EXPECT_EQ(nebd::client::kOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, Write_1) { - nebd::client::WriteResponse response; - brpc::Controller cntl; - nebd::client::WriteRequest request; - request.set_fd(3); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.Write(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, Write_2) { - nebd::client::WriteResponse response; - brpc::Controller cntl; - nebd::client::WriteRequest request; - request.set_fd(10000); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 10000; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.Write(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, Write_3) { - nebd::client::WriteResponse response; - brpc::Controller cntl; - nebd::client::WriteRequest request; - request.set_fd(300); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 300; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestTestOk* req = new RpcRequestTestOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.Write(&cntl, &request, &response, &done); - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, Write_4) { - nebd::client::WriteResponse response; - brpc::Controller cntl; - nebd::client::WriteRequest request; - request.set_fd(300); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 300; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestTestNoOk* req = new RpcRequestTestNoOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.Write(&cntl, &request, &response, &done); - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, read_1) { - nebd::client::ReadResponse response; - brpc::Controller cntl; - nebd::client::ReadRequest request; - request.set_fd(3); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.Read(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, Read_2) { - nebd::client::ReadResponse response; - brpc::Controller cntl; - nebd::client::ReadRequest request; - request.set_fd(10000); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 10000; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.Read(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, Read_3) { - nebd::client::ReadResponse response; - brpc::Controller cntl; - nebd::client::ReadRequest request; - request.set_fd(300); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 300; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestTestNoOk* req = new RpcRequestTestNoOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.Read(&cntl, &request, &response, &done); - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, Read_4) { - nebd::client::ReadResponse response; - brpc::Controller cntl; - nebd::client::ReadRequest request; - request.set_fd(300); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 300; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestTestOk* req = new RpcRequestTestOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.Read(&cntl, &request, &response, &done); - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, StatFile_1) { - nebd::client::StatFileResponse response; - brpc::Controller cntl; - nebd::client::StatFileRequest request; - request.set_fd(3); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.StatFile(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, StatFile_2) { - nebd::client::StatFileResponse response; - brpc::Controller cntl; - nebd::client::StatFileRequest request; - request.set_fd(10000); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 10000; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestTestOk* req = new RpcRequestTestOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.StatFile(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, StatFile_3) { - nebd::client::StatFileResponse response; - brpc::Controller cntl; - nebd::client::StatFileRequest request; - request.set_fd(100); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 100; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestTestOk* req = new RpcRequestTestOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.StatFile(&cntl, &request, &response, &done); - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, StatFile_4) { - nebd::client::StatFileResponse response; - brpc::Controller cntl; - nebd::client::StatFileRequest request; - request.set_fd(100); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 100; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestTestNoOk* req = new RpcRequestTestNoOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.StatFile(&cntl, &request, &response, &done); - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, GetInfo_1) { - nebd::client::GetInfoResponse response; - brpc::Controller cntl; - nebd::client::GetInfoRequest request; - request.set_fd(3); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.GetInfo(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, GetInfo_2) { - nebd::client::GetInfoResponse response; - brpc::Controller cntl; - nebd::client::GetInfoRequest request; - request.set_fd(10000); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 10000; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.GetInfo(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, GetInfo_3) { - nebd::client::GetInfoResponse response; - brpc::Controller cntl; - nebd::client::GetInfoRequest request; - request.set_fd(300); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 300; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestTestNoOk* req = new RpcRequestTestNoOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.GetInfo(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, GetInfo_4) { - nebd::client::GetInfoResponse response; - brpc::Controller cntl; - nebd::client::GetInfoRequest request; - request.set_fd(300); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 300; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestTestOk* req = new RpcRequestTestOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.GetInfo(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, Flush_1) { - nebd::client::FlushResponse response; - brpc::Controller cntl; - nebd::client::FlushRequest request; - request.set_fd(3); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.Flush(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, Flush_2) { - nebd::client::FlushResponse response; - brpc::Controller cntl; - nebd::client::FlushRequest request; - request.set_fd(10000); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 10000; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.Flush(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, Flush_3) { - nebd::client::FlushResponse response; - brpc::Controller cntl; - nebd::client::FlushRequest request; - request.set_fd(300); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 300; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestTestOk* req = new RpcRequestTestOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.Flush(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, Flush_4) { - nebd::client::FlushResponse response; - brpc::Controller cntl; - nebd::client::FlushRequest request; - request.set_fd(300); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 300; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - RpcRequestTestNoOk* req = new RpcRequestTestNoOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.Flush(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, CloseFile_1) { - nebd::client::CloseFileResponse response; - brpc::Controller cntl; - nebd::client::CloseFileRequest request; - request.set_fd(3); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.CloseFile(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, CloseFile_2) { - nebd::client::CloseFileResponse response; - brpc::Controller cntl; - nebd::client::CloseFileRequest request; - request.set_fd(10000); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 10000; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.CloseFile(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, CloseFile_3) { - nebd::client::CloseFileResponse response; - brpc::Controller cntl; - nebd::client::CloseFileRequest request; - request.set_fd(300); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 300; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestTestNoOk* req = new RpcRequestTestNoOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.CloseFile(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, CloseFile_4) { - nebd::client::CloseFileResponse response; - brpc::Controller cntl; - nebd::client::CloseFileRequest request; - request.set_fd(300); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 300; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestTestOk* req = new RpcRequestTestOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.CloseFile(&cntl, &request, &response, &done); - EXPECT_EQ(nebd::client::kOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, Discard_1) { - nebd::client::DiscardResponse response; - brpc::Controller cntl; - nebd::client::DiscardRequest request; - request.set_fd(3); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.Discard(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, Discard_2) { - nebd::client::DiscardResponse response; - brpc::Controller cntl; - nebd::client::DiscardRequest request; - request.set_fd(10000); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 10000; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.Discard(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, Discard_3) { - nebd::client::DiscardResponse response; - brpc::Controller cntl; - nebd::client::DiscardRequest request; - request.set_fd(300); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 300; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - RpcRequestTestOk* req = new RpcRequestTestOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.Discard(&cntl, &request, &response, &done); - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, Discard_4) { - nebd::client::DiscardResponse response; - brpc::Controller cntl; - nebd::client::DiscardRequest request; - request.set_fd(300); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 300; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - RpcRequestTestNoOk* req = new RpcRequestTestNoOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.Discard(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, ResizeFile_1) { - nebd::client::ResizeResponse response; - brpc::Controller cntl; - nebd::client::ResizeRequest request; - request.set_fd(3); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.ResizeFile(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, ResizeFile_2) { - nebd::client::ResizeResponse response; - brpc::Controller cntl; - nebd::client::ResizeRequest request; - request.set_fd(10000); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 10000; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.ResizeFile(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, ResizeFile_3) { - nebd::client::ResizeResponse response; - brpc::Controller cntl; - nebd::client::ResizeRequest request; - request.set_fd(300); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 300; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestTestNoOk* req = new RpcRequestTestNoOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.ResizeFile(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, ResizeFile_4) { - nebd::client::ResizeResponse response; - brpc::Controller cntl; - nebd::client::ResizeRequest request; - request.set_fd(300); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 300; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestTestOk* req = new RpcRequestTestOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.ResizeFile(&cntl, &request, &response, &done); - EXPECT_EQ(nebd::client::kOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, InvalidateCache_1) { - nebd::client::InvalidateCacheResponse response; - brpc::Controller cntl; - nebd::client::InvalidateCacheRequest request; - request.set_fd(3); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 4; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.InvalidateCache(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, InvalidateCache_2) { - nebd::client::InvalidateCacheResponse response; - brpc::Controller cntl; - nebd::client::InvalidateCacheRequest request; - request.set_fd(10000); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 10000; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - QemuClientServiceImpl qemuImpl; - qemuImpl.InvalidateCache(&cntl, &request, &response, &done); - - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, InvalidateCache_3) { - nebd::client::InvalidateCacheResponse response; - brpc::Controller cntl; - nebd::client::InvalidateCacheRequest request; - request.set_fd(300); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 300; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestTestNoOk* req = new RpcRequestTestNoOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.InvalidateCache(&cntl, &request, &response, &done); - EXPECT_EQ(nebd::client::kNoOK, response.retcode()); -} - -TEST(QemuClientServiceImplTest, InvalidateCache_4) { - nebd::client::InvalidateCacheResponse response; - brpc::Controller cntl; - nebd::client::InvalidateCacheRequest request; - request.set_fd(300); - RpcServiceTestClosure done; - - rados_ioctx_t* io = new rados_ioctx_t; - rados_t* cluster = new rados_t; - int fd = 300; - char* filename = const_cast( - "rbd:rbd/volume01:auth_supported=none:mon_host=10.182.30.27:6789"); - rbd_image_t* image = new rbd_image_t; - FdImage_t* fd_image = new FdImage_t; - fd_image->cluster = cluster; - fd_image->io = io; - fd_image->image = image; - fd_image->filename = filename; - - g_imageMap.clear(); - g_imageMap.insert(std::pair(fd, fd_image)); - - RpcRequestTestOk* req = new RpcRequestTestOk; - QemuClientServiceImpl qemuImpl; - qemuImpl.SetRpcRequest(req); - qemuImpl.InvalidateCache(&cntl, &request, &response, &done); - EXPECT_EQ(nebd::client::kOK, response.retcode()); -} - -int main(int argc, char* argv[]) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tests/part2/test_common.cpp b/tests/part2/test_common.cpp deleted file mode 100644 index 4e195f298f..0000000000 --- a/tests/part2/test_common.cpp +++ /dev/null @@ -1,18 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include "tests/part2/test_common.h" -CMOCK_MOCK_FUNCTION1(NebdServerMocker, GeneratePort, int(int)); -CMOCK_MOCK_FUNCTION2(NebdServerMocker, FindPort, int(config_t*, int)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, LockFile, int(const char*)); -CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidLockfile, std::string()); -CMOCK_MOCK_FUNCTION0(NebdServerMocker, Reload, int()); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, CheckProc, int(const char*)); -CMOCK_MOCK_FUNCTION3(NebdServerMocker, open, int(const char*, int, mode_t)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, ReadQemuXmlDir, std::string(config_t*)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, ReadQemuXml, void(const char*)); -CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidFile, std::string()); diff --git a/tests/part2/test_common.h b/tests/part2/test_common.h deleted file mode 100644 index 812a4fbc4f..0000000000 --- a/tests/part2/test_common.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#ifndef TESTS_PART2_TEST_COMMON_H_ -#define TESTS_PART2_TEST_COMMON_H_ - -#include -#include -#include "src/part2/config.h" - -class NebdServerMocker : public CMockMocker { - public: - // MOCK_METHOD0(InitConfig, config_t*()); - MOCK_METHOD1(ReadQemuXmlDir, std::string(config_t *)); - MOCK_METHOD1(ReadQemuXml, void(const char *)); - MOCK_METHOD0(GetUuidFile, std::string()); - MOCK_METHOD0(GetUuidLockfile, std::string()); - MOCK_METHOD3(open, int(const char *, int, mode_t)); - MOCK_METHOD1(CheckProc, int(const char *)); - MOCK_METHOD1(LockFile, int(const char *)); - MOCK_METHOD0(Reload, int()); - MOCK_METHOD2(FindPort, int(config_t *, int)); - MOCK_METHOD1(GeneratePort, int(int)); -}; - -#endif // TESTS_PART2_TEST_COMMON_H_ diff --git a/tests/part2/test_common2.cpp b/tests/part2/test_common2.cpp deleted file mode 100644 index 1695eeadef..0000000000 --- a/tests/part2/test_common2.cpp +++ /dev/null @@ -1,10 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include "tests/part2/test_common2.h" -CMOCK_MOCK_FUNCTION1(NebdServerMocker, LockFile, int(const char*)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, unLockFile, void(int)); diff --git a/tests/part2/test_common2.h b/tests/part2/test_common2.h deleted file mode 100644 index ac3899a796..0000000000 --- a/tests/part2/test_common2.h +++ /dev/null @@ -1,20 +0,0 @@ -/* - *** Project: nebd - *** File Created: 2019-09-30 - *** Author: hzwuhongsong - *** Copyright (c) 2019 NetEase - ***/ - -#ifndef TESTS_PART2_TEST_COMMON2_H_ -#define TESTS_PART2_TEST_COMMON2_H_ - -#include -#include "src/part2/config.h" - -class NebdServerMocker : public CMockMocker { - public: - MOCK_METHOD1(LockFile, int(const char*)); - MOCK_METHOD1(unLockFile, void(int)); -}; - -#endif // TESTS_PART2_TEST_COMMON2_H_ diff --git a/tests/part2/test_config.cpp b/tests/part2/test_config.cpp deleted file mode 100644 index 389913c276..0000000000 --- a/tests/part2/test_config.cpp +++ /dev/null @@ -1,13 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include "tests/part2/test_reload.h" -/* -CMOCK_MOCK_FUNCTION1(NebdServerMocker, ConnectRados, rados_t*(const char*)); -CMOCK_MOCK_FUNCTION2(NebdServerMocker, ReloadCephVolume, int(int, char*)); -CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidFile, std::string()); -*/ diff --git a/tests/part2/test_heartbeat.cpp b/tests/part2/test_heartbeat.cpp deleted file mode 100644 index 479123b045..0000000000 --- a/tests/part2/test_heartbeat.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include "tests/part2/test_heartbeat.h" -CMOCK_MOCK_FUNCTION1(NebdServerMocker, LockFile, int(const char*)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, UnLockFile, void(int)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, CloseImage, int(int)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, CheckProc, int(const char*)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, RmFd, int(int)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, CloseQemuDetachedVolumes, int(int)); -CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidFile, std::string()); -CMOCK_MOCK_FUNCTION0(NebdServerMocker, ReadQemuXmls, int()); -CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidLockfile, std::string()); diff --git a/tests/part2/test_heartbeat.h b/tests/part2/test_heartbeat.h deleted file mode 100644 index fc7833caf1..0000000000 --- a/tests/part2/test_heartbeat.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#ifndef TESTS_PART2_TEST_HEARTBEAT_H_ -#define TESTS_PART2_TEST_HEARTBEAT_H_ - -#include -#include -#include "src/part2/common_type.h" -#include "src/part2/config.h" -#include "src/part2/heartbeat.h" -#include "src/part2/rados_interface.h" - -class NebdServerMocker : public CMockMocker { - public: - MOCK_METHOD1(LockFile, int(const char*)); - MOCK_METHOD1(UnLockFile, void(int)); - MOCK_METHOD1(CloseImage, int(int)); - MOCK_METHOD0(GetUuidFile, std::string()); - MOCK_METHOD0(GetUuidLockfile, std::string()); - MOCK_METHOD1(CheckProc, int(const char*)); - MOCK_METHOD1(RmFd, int(int)); - MOCK_METHOD0(ReadQemuXmls, int()); - MOCK_METHOD1(CloseQemuDetachedVolumes, int(int)); -}; - -#endif // TESTS_PART2_TEST_HEARTBEAT_H_ diff --git a/tests/part2/test_heartbeat2.cpp b/tests/part2/test_heartbeat2.cpp deleted file mode 100644 index b25581c3e5..0000000000 --- a/tests/part2/test_heartbeat2.cpp +++ /dev/null @@ -1,16 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include "tests/part2/test_heartbeat2.h" -CMOCK_MOCK_FUNCTION3(NebdServerMocker, open, int(const char*, int, mode_t)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, LockFile, int(const char*)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, UnLockFile, void(int)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, CloseImage, int(int)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, RmFd, int(int)); -CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidFile, std::string()); -CMOCK_MOCK_FUNCTION0(NebdServerMocker, ReadQemuXmls, int()); -CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidLockfile, std::string()); diff --git a/tests/part2/test_heartbeat2.h b/tests/part2/test_heartbeat2.h deleted file mode 100644 index e22e2a7631..0000000000 --- a/tests/part2/test_heartbeat2.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#ifndef TESTS_PART2_TEST_HEARTBEAT2_H_ -#define TESTS_PART2_TEST_HEARTBEAT2_H_ - -#include -#include -#include -#include -#include "src/part2/common_type.h" -#include "src/part2/config.h" -#include "src/part2/heartbeat.h" -#include "src/part2/rados_interface.h" -class NebdServerMocker : public CMockMocker { - public: - MOCK_METHOD1(LockFile, int(const char *)); - MOCK_METHOD1(UnLockFile, void(int)); - MOCK_METHOD1(CloseImage, int(int)); - MOCK_METHOD0(GetUuidFile, std::string()); - MOCK_METHOD0(GetUuidLockfile, std::string()); - MOCK_METHOD1(RmFd, int(int)); - MOCK_METHOD0(ReadQemuXmls, int()); - MOCK_METHOD3(open, int(const char *, int, mode_t)); -}; - -#endif // TESTS_PART2_TEST_HEARTBEAT2_H_ diff --git a/tests/part2/test_heartbeat3.cpp b/tests/part2/test_heartbeat3.cpp deleted file mode 100644 index 38f6fb3cd9..0000000000 --- a/tests/part2/test_heartbeat3.cpp +++ /dev/null @@ -1,10 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include "tests/part2/test_heartbeat3.h" -CMOCK_MOCK_FUNCTION4(NebdServerMocker, CheckCmdline, - int(int64_t, const char*, char*, int)); diff --git a/tests/part2/test_heartbeat3.h b/tests/part2/test_heartbeat3.h deleted file mode 100644 index 9d17ada779..0000000000 --- a/tests/part2/test_heartbeat3.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#ifndef TESTS_PART2_TEST_HEARTBEAT3_H_ -#define TESTS_PART2_TEST_HEARTBEAT3_H_ - -#include -#include -#include -#include -#include "src/part2/common_type.h" -#include "src/part2/config.h" -#include "src/part2/heartbeat.h" -#include "src/part2/rados_interface.h" -class NebdServerMocker : public CMockMocker { - public: - MOCK_METHOD4(CheckCmdline, int(int64_t, const char*, char*, int)); -}; - -#endif // TESTS_PART2_TEST_HEARTBEAT3_H_ diff --git a/tests/part2/test_rados.cpp b/tests/part2/test_rados.cpp deleted file mode 100644 index 7ddddb8418..0000000000 --- a/tests/part2/test_rados.cpp +++ /dev/null @@ -1,24 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include "tests/part2/test_rados.h" -CMOCK_MOCK_FUNCTION1(NebdServerMocker, CloseRados, void(rados_t *)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, rados_ioctx_destroy, - void(rados_ioctx_t)); -CMOCK_MOCK_FUNCTION4(NebdServerMocker, rbd_open, - int(rados_ioctx_t, const char *, rbd_image_t *, - const char *)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, rbd_close, int(rbd_image_t)); -CMOCK_MOCK_FUNCTION3(NebdServerMocker, rados_ioctx_create, - int(rados_t, const char *, rados_ioctx_t *)); -CMOCK_MOCK_FUNCTION2(NebdServerMocker, rados_create, - int(rados_t *, const char *)); -CMOCK_MOCK_FUNCTION2(NebdServerMocker, rados_conf_read_file, - int(rados_t, const char *)); -CMOCK_MOCK_FUNCTION3(NebdServerMocker, rados_conf_set, - int(rados_t, const char *, const char *)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, rados_connect, int(rados_t)); diff --git a/tests/part2/test_rados.h b/tests/part2/test_rados.h deleted file mode 100644 index 9021d20225..0000000000 --- a/tests/part2/test_rados.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#ifndef TESTS_PART2_TEST_RADOS_H_ -#define TESTS_PART2_TEST_RADOS_H_ - -#include -#include -#include -#include "src/part2/config.h" - -class NebdServerMocker : public CMockMocker { - public: - MOCK_METHOD1(CloseRados, void(rados_t *)); - MOCK_METHOD1(rados_ioctx_destroy, void(rados_ioctx_t)); - MOCK_METHOD3(rados_ioctx_create, - int(rados_t, const char *, rados_ioctx_t *)); - MOCK_METHOD4(rbd_open, - int(rados_ioctx_t, const char *, rbd_image_t *, const char *)); - MOCK_METHOD1(rbd_close, int(rbd_image_t)); - MOCK_METHOD2(rados_create, int(rados_t *, const char *)); - MOCK_METHOD2(rados_conf_read_file, int(rados_t, const char *)); - MOCK_METHOD3(rados_conf_set, int(rados_t, const char *, const char *)); - MOCK_METHOD1(rados_connect, int(rados_t)); -}; - -#endif // TESTS_PART2_TEST_RADOS_H_ diff --git a/tests/part2/test_reload.cpp b/tests/part2/test_reload.cpp deleted file mode 100644 index 88a99d15ea..0000000000 --- a/tests/part2/test_reload.cpp +++ /dev/null @@ -1,13 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include "tests/part2/test_reload.h" -#include - -CMOCK_MOCK_FUNCTION1(NebdServerMocker, ConnectRados, rados_t*(const char*)); -CMOCK_MOCK_FUNCTION2(NebdServerMocker, ReloadCephVolume, int(int, char*)); -CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidFile, std::string()); diff --git a/tests/part2/test_reload.h b/tests/part2/test_reload.h deleted file mode 100644 index 0cee5329a5..0000000000 --- a/tests/part2/test_reload.h +++ /dev/null @@ -1,25 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#ifndef TESTS_PART2_TEST_RELOAD_H_ -#define TESTS_PART2_TEST_RELOAD_H_ - -#include -#include -#include -#include -#include "src/part2/rados_interface.h" -#include "src/part2/reload.h" - -class NebdServerMocker : public CMockMocker { - public: - MOCK_METHOD1(ConnectRados, rados_t*(const char*)); - MOCK_METHOD2(ReloadCephVolume, int(int, char*)); - MOCK_METHOD0(GetUuidFile, std::string()); -}; - -#endif // TESTS_PART2_TEST_RELOAD_H_ diff --git a/tests/part2/test_reload2.cpp b/tests/part2/test_reload2.cpp deleted file mode 100644 index c4155fd592..0000000000 --- a/tests/part2/test_reload2.cpp +++ /dev/null @@ -1,11 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include "tests/part2/test_reload2.h" -CMOCK_MOCK_FUNCTION1(NebdServerMocker, ConnectRados, rados_t*(const char*)); -CMOCK_MOCK_FUNCTION5(NebdServerMocker, OpenImage, - int(rados_t*, const char*, const char*, int, char*)); diff --git a/tests/part2/test_reload2.h b/tests/part2/test_reload2.h deleted file mode 100644 index fb81cc4c7f..0000000000 --- a/tests/part2/test_reload2.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#ifndef TESTS_PART2_TEST_RELOAD2_H_ -#define TESTS_PART2_TEST_RELOAD2_H_ - -#include -#include -#include -#include "src/part2/rados_interface.h" -#include "src/part2/reload.h" - -class NebdServerMocker : public CMockMocker { - public: - MOCK_METHOD1(ConnectRados, rados_t*(const char*)); - MOCK_METHOD5(OpenImage, - int(rados_t*, const char*, const char*, int, char*)); -}; - -#endif // TESTS_PART2_TEST_RELOAD2_H_ diff --git a/tests/part2/test_rpcceph.cpp b/tests/part2/test_rpcceph.cpp deleted file mode 100644 index 20cc57b295..0000000000 --- a/tests/part2/test_rpcceph.cpp +++ /dev/null @@ -1,41 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#include -#include "tests/part2/test_rpcceph.h" - -CMOCK_MOCK_FUNCTION2(NebdServerMocker, GenerateFd, int(char *, int)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, CloseRados, void(rados_t *)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, FilenameFdExist, int(char *)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, ConnectRados, rados_t *(const char *)); -CMOCK_MOCK_FUNCTION5(NebdServerMocker, OpenImage, - int(rados_t *, const char *, const char *, int, char *)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, rbd_aio_release, void(rbd_completion_t)); -CMOCK_MOCK_FUNCTION5(NebdServerMocker, rbd_aio_read, - int(rbd_image_t, uint64_t, size_t, char *, - rbd_completion_t)); -CMOCK_MOCK_FUNCTION4(NebdServerMocker, rbd_aio_discard, - int(rbd_image_t, uint64_t, uint64_t, rbd_completion_t)); -CMOCK_MOCK_FUNCTION2(NebdServerMocker, rbd_aio_flush, - int(rbd_image_t, rbd_completion_t)); -CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidLockfile, std::string()); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, LockFile, int(const char *)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, rbd_aio_get_return_value, - ssize_t(rbd_completion_t)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, UnLockFile, void(int)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, CloseImage, int(int)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, RmFd, int(int)); -CMOCK_MOCK_FUNCTION0(NebdServerMocker, GetUuidFile, std::string()); -CMOCK_MOCK_FUNCTION5(NebdServerMocker, rbd_aio_write, - int(rbd_image_t, uint64_t, size_t, const char *, - rbd_completion_t)); -CMOCK_MOCK_FUNCTION3(NebdServerMocker, rbd_aio_create_completion, - int(void *, rbd_callback_t, rbd_completion_t *)); -CMOCK_MOCK_FUNCTION3(NebdServerMocker, rbd_stat, - int(rbd_image_t, rbd_image_info_t *, size_t)); -CMOCK_MOCK_FUNCTION2(NebdServerMocker, rbd_resize, int(rbd_image_t, uint64_t)); -CMOCK_MOCK_FUNCTION1(NebdServerMocker, rbd_invalidate_cache, int(rbd_image_t)); diff --git a/tests/part2/test_rpcceph.h b/tests/part2/test_rpcceph.h deleted file mode 100644 index 7d77e56d02..0000000000 --- a/tests/part2/test_rpcceph.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - ** Project: nebd - ** File Created: 2019-09-30 - ** Author: hzwuhongsong - ** Copyright (c) 2019 NetEase - **/ - -#ifndef TESTS_PART2_TEST_RPCCEPH_H_ -#define TESTS_PART2_TEST_RPCCEPH_H_ - -#include -#include -#include -#include -#include "src/part2/config.h" - -class NebdServerMocker : public CMockMocker { - public: - MOCK_METHOD5(rbd_aio_write, int(rbd_image_t, uint64_t, size_t, const char *, - rbd_completion_t)); - MOCK_METHOD5(rbd_aio_read, - int(rbd_image_t, uint64_t, size_t, char *, rbd_completion_t)); - MOCK_METHOD4(rbd_aio_discard, - int(rbd_image_t, uint64_t, uint64_t, rbd_completion_t)); - MOCK_METHOD2(rbd_aio_flush, int(rbd_image_t, rbd_completion_t)); - MOCK_METHOD3(rbd_aio_create_completion, - int(void *, rbd_callback_t, rbd_completion_t *)); - MOCK_METHOD1(rbd_invalidate_cache, int(rbd_image_t)); - MOCK_METHOD2(rbd_resize, int(rbd_image_t, uint64_t)); - MOCK_METHOD3(rbd_stat, int(rbd_image_t, rbd_image_info_t *, size_t)); - MOCK_METHOD1(LockFile, int(const char *)); - MOCK_METHOD1(UnLockFile, void(int)); - MOCK_METHOD1(CloseImage, int(int)); - MOCK_METHOD0(GetUuidFile, std::string()); - MOCK_METHOD1(RmFd, int(int)); - MOCK_METHOD1(rbd_aio_get_return_value, ssize_t(rbd_completion_t)); - MOCK_METHOD0(GetUuidLockfile, std::string()); - MOCK_METHOD1(rbd_aio_release, void(rbd_completion_t)); - MOCK_METHOD1(FilenameFdExist, int(char *)); - MOCK_METHOD1(ConnectRados, rados_t *(const char *)); - MOCK_METHOD5(OpenImage, - int(rados_t *, const char *, const char *, int, char *)); - MOCK_METHOD1(CloseRados, void(rados_t *)); - MOCK_METHOD2(GenerateFd, int(char *, int)); -}; -#endif // TESTS_PART2_TEST_RPCCEPH_H_ diff --git a/tests/test-suites/create_100_vms.sh b/tests/test-suites/create_100_vms.sh deleted file mode 100644 index fa6b8eb6f6..0000000000 --- a/tests/test-suites/create_100_vms.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -POOL=rbd -MON_HOST=10.182.30.27 - -for i in {1..90}; do -echo ====$i==== -# 从snap创建系统盘 -rbd rm $POOL/vm-$i -rbd clone $POOL/rbd_sys_disk@snap $POOL/vm-$i - -# 从系统盘创建1个vm -echo ''' - - #vm-# - 2097152 - 2 - - hvm - - - - - - - - - /usr/bin/qemu-system-x86_64 - - - - - - - - -
- - - - - - - - - - - - - -''' > vm-$i.xml - -sed -i "s/#POOL#/$POOL/g" vm-$i.xml -sed -i "s/#MON_HOST#/$MON_HOST/g" vm-$i.xml -sed -i "s/#vm-#/vm-$i/g" vm-$i.xml - -sudo virsh undefine vm-$i -sudo virsh define vm-$i.xml -sudo virsh start vm-$i -sleep 1 - -done diff --git a/tests/test-suites/id_rsa b/tests/test-suites/id_rsa deleted file mode 100644 index 2115ed75ae..0000000000 --- a/tests/test-suites/id_rsa +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEA2A5lNh09w5O1hWotsEsWx65PNMmHS+a7xtXLgs+Bq/qtWMkD -ouGmxQGIgpHf6omvy+u2diqUu79CM4mBWGObDFCIOwjQ9x0+uTEu3drLGc5oYNB5 -jwTIGAGnlkNmooRF1HfgDtj4PeEk3F6Q75k08MnylY+j1BCd/O+p3JLJTRXybwK1 -w0ejUuPX7sTDZOacP/khpcQwSaBi5RcdWjsyzzqC0NMbpKSsKrOKWuAhQqE/2x10 -26VKU5hBBK4lLJf6RiCcZXBKq/PYTtlZUzEbwoX1jIGuHDWiFAnTpWhTf3J255Dp -CUQhzT2i/3teKEJQGBy40aNBvyEm/TI+h35rrwIDAQABAoIBAEg+eAViPbbZ+0aD -PIfIc/ONC5shEAaHPp67+nLhJAD3r2MSxA6A2ZziaF0Qngj0xT2wsbBNiLJGv8Iv -5npBxCSYARWMWyNmOgT+tycj8+nLJQfKbfsupCGy2/DY1Mbe5+Klutv8HPXEBOlT -Da+wUQ162Yy9HRDaBZoAIZJJ2tnVMXzolCYC6CgYpPzZ2aIRewgO9gkHvvlIjlk6 -lVEx8XwuTFnMAu+1yBnHiatn5UonpWL2CAGdCgYDUF7ehoOV2NOL+Qc6zHXUsVEo -GBnJuAcPJ57jacRp4FEY0bgXBRXltb84S+unoYzcrDbZDCZfOYIaGn5txsrIjSa+ -DkoFqVECgYEA85OaIFEDvYs9huwKfSMsYCa3RVQ+SA0gKCvn7WnxYmnn7ctFMfgt -io1QZIAnz6f3Y7yKGDbwJq4kBdxiiEwMIz4YT+WGN+r/ERcIZFvStqq0kSKK620m -1NQH4qZmJGwtCHB/O6/EoeJqfZPZXG/1+9DUXRCrO+dpd9ltJt4XgSUCgYEA4xN1 -SA7Ey42+gkqV8AtYp1Jb7w54hP9E6Y8kOR+P/eJYoI/7b1e9J9Wy61D+cxWSJpLX -jN73H6OFxeQbyzTu2h9dIIOzi4/a1Kd040f0FzAsP0O6bURPxMfnLhUgK2t2FGLD -4obaWTKAzHSo9SIrA3JY3q4EdiumHXkazZjEc0MCgYBJty/7+8jAAnXacXEzMgPq -DY2Fo9s9fwtuXOtSBLAS8q5IXy5P0yYlOmNMJyTlSZYrTxIUBfxkGGu6glonmaxK -ti4xb+tQ3QPYrjnR9aAtvDftfKDCY0RPQOQqCeX7u+ldFn7LF4TQ4W8NFkPAH/o3 -/ZlnmPXZqU1YAitwieJfKQKBgQCNqfgDKmOP8jAmLM354bo/ONxt+YMqxe6CH3Ej -yhMU+v77vJfKUNyLW4Cx2efEeHKjafzbv8ZAkM3GNZc5YR6aKfL5c/CxYt/Mx2Pe -32biiIUyOti1jmaNh4EseLWHIRWbhjpaunLCLwW3iBhYRia7kbzrv3i9GGIcBQ+5 -sFLu7QKBgFE+vTWpJcq2HuHcqlcBtSj2pbrWWskGb4YQBmaErTJz3Mdaezdl7XUo -0RAxcOeNumoJEXT22E52rZvLNA1FiJ8hQWWHmb4g2PwARbFEz9OF6pnZtXdUlr/s -W4RsnvkS93R8jOZNF7Ifx+3tetzlmGUKkHXKj06TjgPnFEYt9scX ------END RSA PRIVATE KEY----- diff --git a/tests/test-suites/kill_nebd.sh b/tests/test-suites/kill_nebd.sh deleted file mode 100644 index f479ed62a3..0000000000 --- a/tests/test-suites/kill_nebd.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -i=1 -while true; do - # 每10s随机kill一个nebd-server进程 - t1=$(($i % 35)) - if [ $t1 = 0 ]; then - pids=`pidof nebd-server` - len=`echo $pids | wc -w` - if [ $len -gt 0 ]; then - pid=`echo $pids | awk '{print $1}'` - echo going to kill -15 $pid - sudo kill -15 $pid - fi - fi - - # 每33s kill掉所有nebd-server进程 - ta=$(($i % 127)) - if [ $ta = 0 ]; then - echo going to killall -15 nebd-server - sudo killall -15 nebd-server - fi - - ((i++)) - sleep 1 -done diff --git a/tests/test-suites/templates/disk_xml.j2 b/tests/test-suites/templates/disk_xml.j2 deleted file mode 100644 index 6a75958b64..0000000000 --- a/tests/test-suites/templates/disk_xml.j2 +++ /dev/null @@ -1,16 +0,0 @@ - - {%- if target_dev == 'sdc' %} - - - {% else %} - - - {%- endif %} - - {%- if protocol == 'rbd' %} - {% for mon_host in mon_hosts -%} - - {% endfor -%} - {% endif -%} - - \ No newline at end of file diff --git a/tests/test-suites/templates/domain_xml.j2 b/tests/test-suites/templates/domain_xml.j2 deleted file mode 100644 index bcbdfeacfa..0000000000 --- a/tests/test-suites/templates/domain_xml.j2 +++ /dev/null @@ -1,55 +0,0 @@ - - {{ uuid }} - {{ name }} - 2097152 - 2 - - hvm - - - - - - - - - /usr/bin/qemu-system-x86_64 - - {%- if disk_type == "file" %} - - - {%- elif disk_type == "network" %} - - - {%- if protocol == 'rbd' %} - {% for mon_host in mon_hosts -%} - - {% endfor -%} - {% endif -%} - - {% endif -%} - - - -
- - - - - - {%- if tap == "tap0" %} - - {% endif -%} - - - - - - - - - - - - - \ No newline at end of file diff --git a/tests/test-suites/test_10_disks.sh b/tests/test-suites/test_10_disks.sh deleted file mode 100644 index e805104450..0000000000 --- a/tests/test-suites/test_10_disks.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -# 创建10个卷 -POOL=rbd -MON_HOST=10.182.30.27 - -for i in {1..10}; do - rbd create -p $POOL vol-$i --image-format 2 --size 1024 -done - - -rbd snap create $POOL/rbd_sys_disk@snap -rbd snap protect $POOL/rbd_sys_disk@snap - -# 创建1个vm -rbd clone $POOL/rbd_sys_disk@snap $POOL/vm-with-10-disk - -echo ''' - - vm-with-10-disk - 8388608 - 4 - - hvm - - - - - - - - - /usr/bin/qemu-system-x86_64 - - - - - - - - -
- - - - - - - - - - - - - -''' > vm-with-10-disk.xml -sed -i "s/#POOL#/$POOL/g" vm-with-10-disk.xml -sed -i "s/#MON_HOST#/$MON_HOST/g" vm-with-10-disk.xml - -sudo virsh define vm-with-10-disk.xml -sudo virsh start vm-with-10-disk -sleep 10 - -# 挂载10个卷到vm -target_devs=(vdc vdd vde vdf vdg vdh vdi vdj vdk vdl) -for i in {0..9}; do -echo ''' - - - - - - -''' > ./disk.xml -sed -i "s/#target_dev#/${target_devs[$i]}/" ./disk.xml -sed -i "s/#disk_name#/$POOL\/vol-$((i+1))/" ./disk.xml -sed -i "s/#MON_HOST#/$MON_HOST/g" ./disk.xml -sudo virsh attach-device vm-with-10-disk disk.xml -sleep 1 -done - -# vm里面对挂载的卷跑fio diff --git a/tests/test-suites/test_discard.sh b/tests/test-suites/test_discard.sh deleted file mode 100644 index 5084ee7e3e..0000000000 --- a/tests/test-suites/test_discard.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -POOL=rbd -MON_HOST=10.182.30.27 - -# 从snap创建系统盘 -rbd clone $POOL/image-centos_7.5@snap $POOL/vm-discard -rbd create --image-format 2 --size 8388608 $POOL/disk-8T - -# 从系统盘创建1个vm -echo ''' - - vm-discard - 2097152 - 2 - - hvm - - - - - - - - - /usr/bin/qemu-system-x86_64 - - - - - - - - - - - - - - - -
- - - - - - - - - - - - - -''' > vm-discard.xml -sed -i "s/#POOL#/$POOL/g" vm-discard.xml -sed -i "s/#MON_HOST#/$MON_HOST/g" vm-discard.xml -sudo virsh define vm-discard.xml -sudo virsh start vm-discard diff --git a/tests/test-suites/test_nebd.py b/tests/test-suites/test_nebd.py deleted file mode 100755 index 943c4980da..0000000000 --- a/tests/test-suites/test_nebd.py +++ /dev/null @@ -1,1469 +0,0 @@ -#!/usr/bin/env python -# -*- coding:UTF-8 - -from jinja2 import Environment, FileSystemLoader -import json -from multiprocessing import cpu_count -import os -import shlex -from subprocess import Popen, PIPE -from threading import Timer -import time -import unittest - - -import libvirt -import paramiko -import rados -import rbd - -# 需要先在物理机上配置好网桥，名称bri，配置dhcp服务，并修改如下相关配置项 -CEPH_CONF = "/etc/ceph/ceph.conf" -MON_HOSTS = ["10.182.30.27", "10.182.30.28", "10.182.30.29"] -REUSE_RBD_DISK = False # True表示测试前不创建rbd卷（使用已有的），跑完不清理rbd卷，False则相反 -RBD_POOL = "rbd" -RBD_VM_IMG = "rbd_sys_disk" # 系统盘 -RBD_VM_IMG_SIZE = 4<<30 # GB，需要比文件镜像大 -RBD_VOL1 = "rbd_logic_disk" # 云盘 -RBD_VOL1_SIZE = 1<<30 # GB，至少1G -LOCAL_IMG = "/mnt/centos_7.5.raw" # 必须是raw格式镜像，不能qcow2 -QEMU_PROC_NAME = "qemu-system-x86_64" -NEBD_SERVER_EXEC_FILE = "/usr/bin/nebd-server" -METADATA_DIR = "/var/run/nebd-server/" -CEPH_CLIENT_OLD_PACKAGE = "0.94.6+netease.1.2-0" -CEPH_CLIENT_NEW_PACKAGE = "0.94.6+netease.1.5-2" -CEPH_CLIENT_REAL_VERSION = { # 通过admin socket version命令获取的版本号 - CEPH_CLIENT_OLD_PACKAGE: "0.94.6-11-g61fb955", - CEPH_CLIENT_NEW_PACKAGE: "0.94.6+netease.1.5-2" -} -CEPH_CLIENT_ASOK_DIR = "/var/run/ceph/guests/" - - -VM_SSH_KEY = "./id_rsa" # 免密ssh到vm私钥（root用户600权限） -VM_HOST_IP = "192.168.10.10" # vm的ip地址，最好配置为静态ip -UUID1 = "5d1289be-50e1-47b7-86de-1de0ff16a9d4" -UUID2 = "b5fdf3de-320c-41cf-80b2-acb794078012" -MAX_CPUS = cpu_count() - -LIBVIRT_CONN = None -JJ_ENV = None - - - -def run(cmd, timeout=5, ignore=''): - print 'going to run cmd: %s' % cmd - proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) - timer = Timer(timeout, proc.kill) - try: - timer.start() - stdout, stderr = proc.communicate() - if stderr: - if ignore == '' or ignore not in stderr: - raise OSError(stderr) - return stdout - except Exception as ex: - print ex - raise ex - finally: - timer.cancel() - - -def create_ssh_connect(host, port=22, timeout=3): - key = paramiko.RSAKey.from_private_key_file(VM_SSH_KEY) - ssh = paramiko.SSHClient() - ssh.load_system_host_keys() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - while timeout > 0: - timeout -= 1 - try: - ssh.connect(host, username='root', port=port, pkey=key, timeout=1) - return ssh - except paramiko.ssh_exception.NoValidConnectionsError as exc: - print "ssh to %s failed: %s" % (host, str(exc)) - time.sleep(1) - except Exception as exc: - print "ssh to %s failed: %s" % (host, str(exc)) - return None - - -def ssh_exec(conn, cmd, timeout=5): - print "going to ssh_exec cmd %s" % cmd - stdin, stdout, stderr = conn.exec_command(cmd, timeout=timeout) - return (stdin, stdout.readlines(), stderr.readlines()) - - -class VMIntegTest(unittest.TestCase): - '''热升级功能集成测试类''' - - def setUp(self): - clear_all_vms() - - def _clear_all_metadatafiles(self): - print 'going to clear all metadata files...' - fs = os.listdir(METADATA_DIR) - for f in fs: - if len(f) == len(UUID1): - os.remove(os.path.join(METADATA_DIR, f)) - - def _killall_nebd_servers(self): - cmd_killall = 'killall -9 %s' % os.path.basename(NEBD_SERVER_EXEC_FILE) - try: - run(cmd_killall) - except OSError: - pass - - def tearDown(self): - clear_all_vms() - self._killall_nebd_servers() - self._clear_all_metadatafiles() - - def test_create_destroy_vm(self): - cpuset = '0-%d' % (MAX_CPUS - 1) - if MAX_CPUS >= 2: - cpuset = '0-%d' % (MAX_CPUS / 2 - 1) - dominfo = {"uuid": UUID1, "cpuset": cpuset, - "name": "test_create_destroy_vm", - "disk_type": "network", - "protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), - "mon_hosts": MON_HOSTS - } - dom = self._create_vm(dominfo) - - if dom.isActive(): - # 检查nebd-server进程的cpu亲和性是否与qemu进程一致 - time.sleep(3) - nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - self.assertTrue(nebd_server_pid > 0) - ns_cpu_affinity = run('taskset -pc %d' % nebd_server_pid) - qemu_pid = self._get_pid_by_cmdline(dominfo['uuid'], QEMU_PROC_NAME) - self.assertTrue(qemu_pid > 0) - qemu_cpu_affinity = run('taskset -pc %d' % qemu_pid) - self.assertEqual(ns_cpu_affinity.split(':')[-1].strip(), - qemu_cpu_affinity.split(':')[-1].strip()) - self._destroy_vm(dominfo) - else: - self.assertFalse("Domain creates failed") - - def test_shutdown_vm(self): - rbd_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_shutdown_rbd_vm", - "disk_type": "network", - "protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), - "mon_hosts": MON_HOSTS - } - local_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_shutdown_local_vm", - "disk_type": "file", - "disk_file": LOCAL_IMG - } - dominfos = [rbd_vm, local_vm] - - for dominfo in dominfos: - dom = self._create_vm(dominfo) - if dom.isActive(): - self._destroy_vm(dominfo, shutdown=True, timeout=120) - else: - self.assertFalse("Domain creates failed") - - def test_reboot_vm(self): - rbd_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_reboot_rbd_vm", - "disk_type": "network", - "protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), - "mon_hosts": MON_HOSTS - } - local_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_reboot_local_vm", - "disk_type": "file", - "disk_file": LOCAL_IMG - } - dominfos = [rbd_vm, local_vm] - - for dominfo in dominfos: - dom = self._create_vm(dominfo) - if not dom.isActive(): - self.assertFalse("Domain creates failed") - # 等待虚拟机操作系统启动 - ssh = create_ssh_connect(VM_HOST_IP, timeout=120) - self.assertTrue(ssh is not None) - # 关机 - self._destroy_vm(dominfo, shutdown=True, timeout=120) - # 再启动（模拟重启） - dom = self._create_vm(dominfo) - if not dom.isActive(): - self.assertFalse("Domain creates failed") - # 等待虚拟机操作系统启动 - ssh = create_ssh_connect(VM_HOST_IP, timeout=120) - self.assertTrue(ssh is not None) - # 关机 - self._destroy_vm(dominfo) - - def test_reboot_vm_with_exist_vnet(self): - tap_name = "tap0" - rbd_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_reboot_rbd_vm_with_exist_vnet", - "disk_type": "network", - "protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), - "mon_hosts": MON_HOSTS, - "tap": tap_name - } - local_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_reboot_local_vm_with_exist_vnet", - "disk_type": "file", - "disk_file": LOCAL_IMG, - "tap": tap_name - } - dominfos = [rbd_vm, local_vm] - - for dominfo in dominfos: - # 删除tap设备 - cmd_tap = ' '.join(['/usr/sbin/tunctl', '-d', tap_name]) - out = run(cmd_tap) - # 创建tap设备 - cmd_tap = ' '.join(['/usr/sbin/tunctl', '-t', tap_name]) - out = run(cmd_tap) - self.assertTrue(tap_name in out and 'persistent' in out) - dom = self._create_vm(dominfo) - if not dom.isActive(): - self.assertFalse("Domain creates failed") - # 等待虚拟机操作系统启动 - ssh = create_ssh_connect(VM_HOST_IP, timeout=120) - self.assertTrue(ssh is not None) - # 关机 - self._destroy_vm(dominfo) - # 从网桥上移除tap设备 - cmd_brctl = ' '.join(['/sbin/brctl', 'delif', 'bri', tap_name]) - out = run(cmd_brctl) - self.assertFalse(out) - # 再启动（模拟重启） - dom = self._create_vm(dominfo) - if not dom.isActive(): - self.assertFalse("Domain creates failed") - # 等待虚拟机操作系统启动 - ssh = create_ssh_connect(VM_HOST_IP, timeout=120) - self.assertTrue(ssh is not None) - # 关机 - self._destroy_vm(dominfo) - # 从网桥上移除tap设备 - cmd_brctl = ' '.join(['/sbin/brctl', 'delif', 'bri', tap_name]) - out = run(cmd_brctl) - self.assertFalse(out) - # 删除tap设备 - cmd_tap = ' '.join(['/usr/sbin/tunctl', '-d', tap_name]) - out = run(cmd_tap) - self.assertTrue(tap_name in out and 'nonpersistent' in out) - - def test_poweroff_vm_inside(self): - rbd_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_poweroff_rbd_vm_inside", - "disk_type": "network", - "protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), - "mon_hosts": MON_HOSTS - } - local_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_poweroff_local_vm_inside", - "disk_type": "file", - "disk_file": LOCAL_IMG - } - dominfos = [rbd_vm, local_vm] - - for dominfo in dominfos: - dom = self._create_vm(dominfo) - if not dom.isActive(): - self.assertFalse("Domain creates failed") - - ssh = create_ssh_connect(VM_HOST_IP, timeout=120) - self.assertTrue(ssh is not None) - ssh_exec(ssh, 'poweroff') - - is_poweroff = False - poweroff_timeout = 30 - while poweroff_timeout > 0: - try: - dom = LIBVIRT_CONN.lookupByUUIDString(dominfo['uuid']) - if dom.info()[0] == libvirt.VIR_DOMAIN_SHUTOFF: - is_poweroff = True - break - poweroff_timeout -= 1 - print 'sleep 1s' - time.sleep(1) - except libvirt.libvirtError: - is_poweroff = True - break - self.assertTrue(is_poweroff) - - def test_reboot_vm_inside(self): - rbd_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_reboot_rbd_vm_inside", - "disk_type": "network", - "protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), - "mon_hosts": MON_HOSTS - } - local_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_reboot_local_vm_inside", - "disk_type": "file", - "disk_file": LOCAL_IMG - } - dominfos = [rbd_vm, local_vm] - - for dominfo in dominfos: - dom = self._create_vm(dominfo) - if not dom.isActive(): - self.assertFalse("Domain creates failed") - - ssh = create_ssh_connect(VM_HOST_IP, timeout=120) - self.assertTrue(ssh is not None) - ssh_exec(ssh, '>/var/log/wtmp') - ssh_exec(ssh, 'reboot') - - is_reboot = False - reboot_timeout = 30 - print 'sleep 1s' - time.sleep(1) - while reboot_timeout > 0: - ssh = create_ssh_connect(VM_HOST_IP, timeout=120) - if ssh is not None: - try: - _, out, err = ssh_exec(ssh, 'last reboot|head -1') - print out, err - if 'reboot' in out[0]: - is_reboot = True - break - except Exception: - pass - reboot_timeout -= 1 - print 'sleep 1s' - time.sleep(1) - self.assertTrue(is_reboot) - self._destroy_vm(dominfo) - - def test_pause_unpause_vm(self): - dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_pause_unpause_vm", - "disk_type": "network", - "protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), - "mon_hosts": MON_HOSTS - } - dom = self._create_vm(dominfo) - - if dom.isActive(): - ret = dom.suspend() - self.assertTrue(ret >= 0) - else: - self.assertFalse("Domain creates failed") - - self.assertEqual(3, dom.info()[0]) - ret = dom.resume() - self.assertTrue(ret >= 0) - self.assertEqual(1, dom.info()[0]) - ret = dom.destroy() - self.assertTrue(ret >= 0) - - def test_save_restore_vm(self): - rbd_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_save_restore_rbd_vm_kill_part2", - "disk_type": "network", - "protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), - "mon_hosts": MON_HOSTS - } - rbd_vm1 = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_save_restore_rbd_vm", - "disk_type": "network", - "protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), - "mon_hosts": MON_HOSTS - } - local_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_save_restore_local_vm", - "disk_type": "file", - "disk_file": LOCAL_IMG - } - dominfos = [rbd_vm, rbd_vm1, local_vm] - - # 依次测试rbd、rbd、local系统盘 - for dominfo in dominfos: - dom = self._create_vm(dominfo, transient=False) - part2_pid = -1 - if dom.isActive(): - part2_pid = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - self.assertTrue(part2_pid > 0) - ret = dom.save('/tmp/' + dominfo['uuid']) - self.assertTrue(ret >= 0) - else: - self.assertFalse("Domain creates failed") - - self.assertEqual(libvirt.VIR_DOMAIN_SHUTOFF, dom.info()[0]) - # rbd系统盘第一个用例测试kill part2场景下restore vm - if dominfos.index(dominfo) == 0: - try: - os.kill(part2_pid, 9) - except OSError as exc: - if "No such process" in str(exc): - pass - time.sleep(1) - self.assertFalse(self._check_proc_running(part2_pid, - dominfo['uuid'], NEBD_SERVER_EXEC_FILE)) - ret = LIBVIRT_CONN.restore('/tmp/' + dominfo['uuid']) - self.assertTrue(ret >= 0) - self.assertEqual(1, dom.info()[0]) - ret = dom.undefine() - self.assertTrue(ret >= 0) - ret = dom.destroy() - self.assertTrue(ret >= 0) - os.remove('/tmp/' + dominfo['uuid']) - - # 创建vm，并挂卸载卷，设置卷QoS - def _attach_detach_disk_on_vm(self, dominfo, diskinfo): - template = JJ_ENV.get_template('domain_xml.j2') - domxml = template.render(dominfo) - ret = LIBVIRT_CONN.defineXML(domxml) - self.assertTrue(ret >= 0) - dom = LIBVIRT_CONN.lookupByUUIDString(dominfo['uuid']) - - # 离线挂载卷测试 - template = JJ_ENV.get_template('disk_xml.j2') - diskxml = template.render(diskinfo) - ret = dom.attachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_CONFIG) - self.assertTrue(ret >= 0) - xml = dom.XMLDesc(0) - self.assertTrue(diskinfo['disk_name'] in xml) - - # 离线设置卷QoS - params = {'total_bytes_sec': 1024000, 'total_iops_sec': 100} - flag = libvirt.VIR_DOMAIN_AFFECT_CONFIG - ret = dom.setBlockIoTune(diskinfo['target_dev'], params, flag) - self.assertTrue(ret >= 0) - xml = dom.XMLDesc(0) - self.assertTrue(params.keys()[0] in xml - and str(params.values()[0]) in xml) - - # 离线卸载卷测试 - ret = dom.detachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_CONFIG) - self.assertTrue(ret >= 0) - xml = dom.XMLDesc(0) - self.assertFalse(diskinfo['disk_name'] in xml) - - # 在线挂载卷测试 - ret = dom.create() - self.assertTrue(ret >= 0) - - timeout = 30 - is_active = False - while timeout > 0: - try: - dom = LIBVIRT_CONN.lookupByUUIDString(dominfo['uuid']) - except: - self.assertFalse("Look up domain by uuid failed") - if dom.isActive(): - is_active = True - break - timeout -= 1 - print 'sleep 1s' - time.sleep(1) - - if not is_active: - self.assertFalse("Domain creates failed") - time.sleep(3) - ret = dom.attachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_LIVE) - self.assertTrue(ret >= 0) - xml = dom.XMLDesc(0) - self.assertTrue(diskinfo['disk_name'] in xml) - - # 在线设置卷QoS - params = {'write_bytes_sec': 1024000, 'read_bytes_sec': 2048000} - flag = libvirt.VIR_DOMAIN_AFFECT_LIVE - ret = dom.setBlockIoTune(diskinfo['target_dev'], params, flag) - self.assertTrue(ret >= 0) - xml = dom.XMLDesc(0) - self.assertTrue(params.keys()[0] in xml - and str(params.values()[0]) in xml) - - # 在线卸载卷测试 - time.sleep(3) - ret = dom.detachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_LIVE) - self.assertTrue(ret >= 0) - xml = dom.XMLDesc(0) - self.assertFalse(diskinfo['disk_name'] in xml) - - ret = dom.undefine() - self.assertTrue(ret >= 0) - ret = dom.destroy() - self.assertTrue(ret >= 0) - - # vm系统盘使用rbd卷，挂卸载rbd卷测试 - def test_attach_detach_rbd_disk_on_rbd_vm(self): - dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_attach_detach_rbd_disk_on_rbd_vm", - "disk_type": "network", - "protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), - "mon_hosts": MON_HOSTS - } - diskinfo = {"protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VOL1), - "mon_hosts": MON_HOSTS, - "target_dev": "vdc" - } - self._attach_detach_disk_on_vm(dominfo, diskinfo) - - # vm系统盘使用本地qcow2 file，挂卸载rbd卷测试 - def test_attach_detach_rbd_disk_on_local_vm(self): - dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_attach_detach_rbd_disk_on_local_vm", - "disk_type": "file", - "disk_file": LOCAL_IMG - } - diskinfo = {"protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VOL1), - "mon_hosts": MON_HOSTS, - "target_dev": "vdc" - } - self._attach_detach_disk_on_vm(dominfo, diskinfo) - - # vm系统盘使用rbd卷，挂卸载cbd卷测试 - def test_attach_detach_cbd_disk_on_rbd_vm(self): - # TODO - pass - - # vm系统盘使用本地qcow2 file，挂卸载rbd卷测试 - def test_attach_detach_cbd_disk_on_local_vm(self): - # TODO - pass - - def _destroy_vm(self, dominfo, shutdown=False, transient=True, timeout=5): - dom = LIBVIRT_CONN.lookupByUUIDString(dominfo['uuid']) - shutdown_timeout = timeout - if shutdown: - # 等待vm完全启动，通过尝试ssh来判断 - ssh_timeout = timeout * 2 / 3 - shutdown_timeout = timeout - ssh_timeout - ssh = create_ssh_connect(VM_HOST_IP, timeout=ssh_timeout) - ret = dom.shutdown() - else: - ret = dom.destroy() - self.assertTrue(ret >= 0) - - is_destroy = False - while shutdown_timeout > 0: - try: - dom = LIBVIRT_CONN.lookupByUUIDString(dominfo['uuid']) - if dom.info()[0] == libvirt.VIR_DOMAIN_SHUTOFF: - is_destroy = True - break - shutdown_timeout -= 1 - print 'sleep 1s' - time.sleep(1) - except libvirt.libvirtError: - is_destroy = True - break - self.assertTrue(is_destroy) - - if not transient: - ret = dom.undefine() - self.assertTrue(ret >= 0) - - # 创建虚拟机 - def _create_vm(self, dominfo, transient=True, - xml_j2='domain_xml.j2', timeout=30): - template = JJ_ENV.get_template(xml_j2) - domxml = template.render(dominfo) - if transient: # 非持久化vm - ret = LIBVIRT_CONN.createXML(domxml) - else: # 持久化vm - ret = LIBVIRT_CONN.defineXML(domxml) - self.assertTrue(ret >= 0) - - dom = LIBVIRT_CONN.lookupByUUIDString(dominfo['uuid']) - if not transient: - ret = dom.create() - self.assertTrue(ret >= 0) - - is_active = False - while timeout > 0: - try: - dom = LIBVIRT_CONN.lookupByUUIDString(dominfo['uuid']) - except libvirt.libvirtError: - self.assertFalse("Look up domain by uuid failed") - if dom.isActive(): - is_active = True - break - timeout -= 1 - print 'sleep 1s' - time.sleep(1) - - if not is_active: - self.assertFalse("Domain creates failed") - return dom - - def test_freeze_thaw_vm_rbd_disk(self): - dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_freeze_thaw_vm_rbd_disk", - "disk_type": "network", - "protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), - "mon_hosts": MON_HOSTS - } - dom = self._create_vm(dominfo) - - diskinfo = {"protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VOL1), - "mon_hosts": MON_HOSTS, - "target_dev": "vdc" - } - template = JJ_ENV.get_template('disk_xml.j2') - diskxml = template.render(diskinfo) - # 等待vm内部操作系统启动 - ssh = create_ssh_connect(VM_HOST_IP, timeout=120) - self.assertTrue(ssh is not None) - ret = dom.attachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_LIVE) - self.assertTrue(ret >= 0) - time.sleep(1) - xml = dom.XMLDesc(0) - self.assertTrue(diskinfo['disk_name'] in xml) - - try: - ret = dom.fsFreeze(diskinfo['target_dev']) - self.assertTrue(ret >= 0) - except libvirt.libvirtError as exc: - self.assertFalse(str(exc)) - try: - ret = dom.fsInfo() - self.assertTrue(ret >= 0) - except libvirt.libvirtError as exc: - self.assertTrue("guest-get-fsinfo has been disabled" in str(exc)) - except AttributeError as exc: - print 'Warning: %s' % str(exc) - try: - ret = dom.fsThaw(diskinfo['target_dev']) - self.assertTrue(ret >= 0) - except libvirt.libvirtError as exc: - self.assertFalse(str(exc)) - - ret = dom.destroy() - self.assertTrue(ret >= 0) - - # vm内部IO随机读写测试 - def _fio_randrw_in_vm(self, cmd, kill_nebd_server=False): - rbd_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_fio_randrw_in_rbd_vm", - "disk_type": "network", - "protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), - "mon_hosts": MON_HOSTS - } - local_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_fio_randrw_in_local_vm", - "disk_type": "file", - "disk_file": LOCAL_IMG - } - dominfos = [rbd_vm, local_vm] - - for dominfo in dominfos: - dom = self._create_vm(dominfo) - if not dom.isActive(): - self.assertFalse("Domain creates failed") - - ssh = create_ssh_connect(VM_HOST_IP, timeout=120) - self.assertTrue(ssh is not None) - nebd_server_pid0 = -1 - meta_dict0 = None - if kill_nebd_server and dominfo['disk_type'] == 'network': - with open(os.path.join(METADATA_DIR, dominfo['uuid'])) as mf: - metadata = mf.read() - meta_dict0 = json.loads(metadata) - nebd_server_pid0 = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - self.assertTrue(nebd_server_pid0 > 0) - # 设置定时器，5s之后杀掉nebd-server进程 - timer = Timer(5, os.kill, (nebd_server_pid0, 15)) - timer.start() - - _, out, err = ssh_exec(ssh, cmd, timeout=120) - self.assertTrue(len(out)) - fio_ok = 0 - for line in out: - if 'read' in line and 'IOPS' in line: - fio_ok += 1 - if 'write' in line and 'IOPS' in line: - fio_ok += 1 - if fio_ok == 2: - break - self.assertEqual(2, fio_ok) - - if kill_nebd_server and dominfo['disk_type'] == 'network': - # 检查是否重启，以及重启是否成功（pid发生变化且大于0） - nebd_server_pid1 = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - if dominfo == rbd_vm: # rbd镜像vm会重新拉起nebd-server - self.assertTrue(nebd_server_pid1 > 0) - elif dominfo == local_vm: # 本地镜像vm不会拉起nebd-server - self.assertTrue(nebd_server_pid1 < 0) - self.assertTrue(nebd_server_pid1 != nebd_server_pid0) - # 检查持久化文件，端口号及fd等信息不能发生变化 - with open(os.path.join(METADATA_DIR, dominfo['uuid'])) as mf: - metadata = mf.read() - meta_dict1 = json.loads(metadata) - self.assertTrue(meta_dict1 == meta_dict0) - - self._destroy_vm(dominfo) - - # vm内部fio读写测试(randrw 4k, seqrw 128k) - def test_fio_in_vm(self): - cmd_4k = ('/usr/bin/fio -name=/fio.data -direct=1 -iodepth=8 ' + - '-rw=randrw -ioengine=libaio -bs=4k -size=100M -numjobs=1') - self._fio_randrw_in_vm(cmd_4k, kill_nebd_server=False) - - cmd_128k = ('/usr/bin/fio -name=/fio.data -direct=1 -iodepth=8 ' + - '-rw=rw -ioengine=libaio -bs=128k -size=200M -numjobs=1') - self._fio_randrw_in_vm(cmd_128k, kill_nebd_server=False) - - # vm内部fio读写测试(randrw 4k, seqrw 128k)过程中kill nebd-server - def test_fio_in_vm_kill_nebd_server(self): - # 注意fio测试时长，要大于5s（定时器设置5s后kill nebd-server进程） - cmd_4k = ('/usr/bin/fio -name=/fio.data -direct=1 -iodepth=8 ' + - '-rw=randrw -ioengine=libaio -bs=4k -size=100M ' + - '-numjobs=1 -rate_iops=1000') - self._fio_randrw_in_vm(cmd_4k, kill_nebd_server=True) - - cmd_128k = ('/usr/bin/fio -name=/fio.data -direct=1 -iodepth=8 ' + - '-rw=rw -ioengine=libaio -bs=128k -size=200M ' + - '-numjobs=1 -rate_iops=60') - self._fio_randrw_in_vm(cmd_128k, kill_nebd_server=True) - - def test_start_vm_part2_running_with_rbd_disk(self): - # 1. 创建vm，rbd卷做系统盘 - dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_start_vm_part2_running_with_rbd_disk", - "disk_type": "network", - "protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), - "mon_hosts": MON_HOSTS - } - dom = self._create_vm(dominfo) - if not dom.isActive(): - self.assertFalse("Domain creates failed") - # 等待vm内部操作系统启动（保证rpc open卷成功） - ssh = create_ssh_connect(VM_HOST_IP, timeout=120) - self.assertTrue(ssh is not None) - nebd_server_pid0 = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - self.assertTrue(nebd_server_pid0 > 0) - metadata = None - with open(os.path.join(METADATA_DIR, dominfo['uuid'])) as mf: - metadata = mf.read() - meta_dict = json.loads(metadata) - # 检查fd大小是否在正确范围内 - for vol in meta_dict['volumes']: - self.assertTrue(1000 >= int(vol['fd']) > 0) - - # 检查rados admin socket是否正常 - asoks = os.listdir(CEPH_CLIENT_ASOK_DIR) - for asok in asoks: - if asok.startswith('ceph-client.admin.%d' % nebd_server_pid0): - asok_path = os.path.join(CEPH_CLIENT_ASOK_DIR, asok) - cmd_asok = 'ceph daemon %s version' % asok_path - out = run(cmd_asok) - self.assertTrue("version" in out) - - # 2. destroy vm - self._destroy_vm(dominfo) - - # 3. 在part2超时退出之前再次启动vm（模拟重启vm操作） - nebd_server_pid1 = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - self.assertTrue(nebd_server_pid1 > 0) - dom = self._create_vm(dominfo) - if not dom.isActive(): - self.assertFalse("Domain creates failed") - nebd_server_pid2 = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - self.assertTrue(nebd_server_pid2 > 0) - # 等待vm内部操作系统启动（保证rpc open卷成功） - ssh = create_ssh_connect(VM_HOST_IP, timeout=120) - self.assertTrue(ssh is not None) - nebd_server_pid3 = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - self.assertTrue(nebd_server_pid3 > 0) - self.assertTrue(nebd_server_pid0 == nebd_server_pid1 - == nebd_server_pid2 == nebd_server_pid3) - - def test_start_vm_without_part2_with_residual_metadatafile(self): - # 准备持久化信息文件（模拟残留） - port = 6300 - fake_uuid = '42d57731-19f6-478f-a145-b77a6d590274' - for uuid in [fake_uuid, UUID1, UUID2]: - meta_file = os.path.join(METADATA_DIR, uuid) - with open(meta_file, 'w') as mf: - if uuid == fake_uuid: - mf.write(json.dumps({"port": 6200})) # 模拟端口被占用 - else: - mf.write(json.dumps({"port": port})) - port += 1 - nebd_server_pid = self._get_pid_by_cmdline(uuid, - NEBD_SERVER_EXEC_FILE) - self.assertTrue(nebd_server_pid < 0) - # 1. 创建vm，rbd卷做系统盘 - dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_start_vm_without_part2_with_residual_metafile", - "disk_type": "network", - "protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), - "mon_hosts": MON_HOSTS - } - dom = self._create_vm(dominfo) - if not dom.isActive(): - self.assertFalse("Domain creates failed") - # 等待vm内部操作系统启动（保证rpc open卷成功） - ssh = create_ssh_connect(VM_HOST_IP, timeout=120) - self.assertTrue(ssh is not None) - nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - self.assertTrue(nebd_server_pid > 0) - # 检查父进程id是否为1（是否为daemon模式） - status = [] - with open(os.path.join('/proc/', str(nebd_server_pid), 'status')) as f: - status = f.readlines() - for line in status: - if line.startswith('PPid:'): - # 0 is for docker - self.assertTrue(int(line.split(':')[-1]) in [0, 1]) - break - # 检查端口号是否符合预期（检查part1清理持久化文件逻辑） - meta_file = os.path.join(METADATA_DIR, dominfo['uuid']) - metadata = None - with open(meta_file, 'r') as mf: - metadata = mf.read() - self.assertTrue(metadata) - meta_dict = json.loads(metadata) - self.assertTrue(int(meta_dict['port']) == 6201) - # 2. destroy vm - self._destroy_vm(dominfo) - # 等待nebd server进程退出，退出后检查持久化文件是否清理 - timeout = 30 - while timeout > 0: - timeout -= 1 - nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - if nebd_server_pid < 0: - self.assertFalse(os.path.exists(meta_file)) - self.assertTrue(os.path.exists( - os.path.join(METADATA_DIR, fake_uuid))) - self.assertTrue(os.path.exists( - os.path.join(METADATA_DIR, UUID2))) - break - else: - time.sleep(1) - - def test_nebd_server_boot_args(self): - # 创建一个使用本地文件做系统盘的虚拟机，为nebd-server心跳做准备 - dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_nebd_server_boot_args", - "disk_type": "file", - "disk_file": LOCAL_IMG - } - dom = self._create_vm(dominfo) - if not dom.isActive(): - self.assertFalse("Domain creates failed") - - # 等待vm内部操作系统启动（保证rpc open卷成功，如果有） - ssh = create_ssh_connect(VM_HOST_IP, timeout=120) - self.assertTrue(ssh is not None) - nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - self.assertTrue(nebd_server_pid < 0) - - # 正确参数 - cmd_ok = ' '.join([NEBD_SERVER_EXEC_FILE, '-uuid', dominfo['uuid']]) - out = run(cmd_ok) - self.assertFalse(out) - nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - self.assertTrue(nebd_server_pid > 0) - os.kill(nebd_server_pid, 9) - nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - self.assertTrue(nebd_server_pid < 0) - - # 错误参数1：uuid对应的qemu进程不存在 - fake_uuid = 'e35a4a69-853b-441e-a5ec-8b8a545ce20f' - cmd_ok = ' '.join([NEBD_SERVER_EXEC_FILE, '-uuid', fake_uuid]) - out = run(cmd_ok) - self.assertFalse(out) - time.sleep(1) - nebd_server_pid = self._get_pid_by_cmdline(fake_uuid, - NEBD_SERVER_EXEC_FILE) - self.assertTrue(nebd_server_pid < 0) - # 错误参数2：--uuid - cmd_ok = ' '.join([NEBD_SERVER_EXEC_FILE, '--uuid', dominfo['uuid']]) - out = run(cmd_ok) - self.assertFalse(out) - time.sleep(1) - nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - self.assertTrue(nebd_server_pid < 0) - - self._destroy_vm(dominfo) - - def test_nebd_server_kill_signal(self): - # 创建一个使用本地文件做系统盘的虚拟机，为nebd-server心跳做准备 - dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_nebd_server_kill_signal", - "disk_type": "file", - "disk_file": LOCAL_IMG - } - dom = self._create_vm(dominfo) - if not dom.isActive(): - self.assertFalse("Domain creates failed") - - # 等待vm内部操作系统启动（保证rpc open卷成功，如果有） - ssh = create_ssh_connect(VM_HOST_IP, timeout=120) - self.assertTrue(ssh is not None) - nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - self.assertTrue(nebd_server_pid < 0) - - # SIGINT(2), SIGTERM(15) - for sig in [2, 15]: - port = 6300 - meta_file = os.path.join(METADATA_DIR, dominfo['uuid']) - with open(meta_file, 'w') as mf: - mf.write(json.dumps({"port": port})) - cmd_ok = ' '.join([NEBD_SERVER_EXEC_FILE, '-uuid', dominfo['uuid']]) - out = run(cmd_ok) - self.assertFalse(out) - nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - self.assertTrue(nebd_server_pid > 0) - # os.kill(nebd_server_pid, sig) - # docker里执行os.kill貌似进程会收不到信号，改成kill命令就好了。。。 - cmd_kill = "/bin/kill -%d %d" % (sig, nebd_server_pid) - out = run(cmd_kill) - self.assertFalse(out) - timeout = 5 - is_exit = False - while timeout > 0: - timeout -= 1 - nebd_server_pid = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - if nebd_server_pid < 0: - is_exit = True - break - else: - time.sleep(1) - self.assertTrue(is_exit) - self.assertTrue(os.path.exists(meta_file)) - self._clear_all_metadatafiles() - - self._destroy_vm(dominfo) - - def _check_proc_running(self, pid, uuid, name): - '''根据/proc/pid/cmdline文件检查进程名是否与给定的名称匹配，进程参数是否包含给定的uuid''' - cmdline = "" - try: - with open(os.path.join("/proc", str(pid), "cmdline")) as f: - cmdline = f.read() - except IOError: - return False - - cmdline = cmdline.split('\0') - proc_name = os.path.basename(cmdline[0]) - if proc_name == os.path.basename(name): - if uuid: - if uuid in cmdline: - return True - else: - return False - return True - return False - - def _get_pid_by_cmdline(self, uuid, name): - pids = os.listdir("/proc") - for pid in pids: - if pid.isdigit() and self._check_proc_running(pid, uuid, name): - return int(pid) - return -1 - - def test_force_kill_vm_qemu_proc(self): - dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_force_kill_vm_qemu_proc", - "disk_type": "network", - "protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), - "mon_hosts": MON_HOSTS - } - dom = self._create_vm(dominfo) - time.sleep(5) - qemu_pid = self._get_pid_by_cmdline(dominfo['uuid'], QEMU_PROC_NAME) - self.assertTrue(qemu_pid > 0) - part2_pid = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - self.assertTrue(part2_pid > 0) - os.kill(qemu_pid, 9) - time.sleep(1) - self.assertFalse(self._check_proc_running(qemu_pid, dominfo['uuid'], - QEMU_PROC_NAME)) - - timeout = 30 - part2_exit = False - while timeout > 0: - if not self._check_proc_running(part2_pid, dominfo['uuid'], - NEBD_SERVER_EXEC_FILE): - part2_exit = True - break - timeout -= 1 - print 'sleep 1s' - time.sleep(1) - - self.assertTrue(part2_exit) - - # TODO: 检查后端卷被close - - # 检查持久化metadata file被清理 - self.assertFalse(os.path.exists( - os.path.join(METADATA_DIR, dominfo['uuid']))) - - def test_create_destroy_local_vm_without_part2(self): - dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_create_destroy_local_vm_without_part2", - "disk_type": "file", - "disk_file": LOCAL_IMG - } - dom = self._create_vm(dominfo) - time.sleep(5) - # 保证part2进程不会启动 - part2_pid = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - self.assertTrue(part2_pid < 0) - # 检查持久化metadata file被清理 - self.assertFalse( - os.path.exists(os.path.join(METADATA_DIR, dominfo['uuid']))) - - # 测试destroy vm - self._destroy_vm(dominfo) - - def test_disk_operations_in_vm(self): - dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_disk_operations_in_vm", - "disk_type": "file", - "disk_file": LOCAL_IMG - } - dom = self._create_vm(dominfo) - if not dom.isActive(): - self.assertFalse("Domain creates failed") - - # attach disk - diskinfo = {"protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VOL1), - "mon_hosts": MON_HOSTS, - "target_dev": "vdc" - } - template = JJ_ENV.get_template('disk_xml.j2') - diskxml = template.render(diskinfo) - ret = dom.attachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_LIVE) - self.assertTrue(ret >= 0) - xml = dom.XMLDesc(0) - self.assertTrue(diskinfo['disk_name'] in xml) - - # ssh到vm内部执行如下操作： - # 1. lsblk - # 2. parted(mkpart) - # 3. mkfs.(ext3, ext4, xfs) - # 3.1 mount - # 3.2 df - # 3.3 dd - # 3.4 sync - # 3.5 umount - # 4. parted(rm) - ssh = create_ssh_connect(VM_HOST_IP, timeout=120) - self.assertTrue(ssh is not None) - cmd_lsblk = '/usr/bin/lsblk -b|grep vd|grep disk|tail -1' - _, out, err = ssh_exec(ssh, cmd_lsblk, timeout=3) - self.assertTrue(len(out)) - vd = out[0].split()[0] - self.assertTrue(vd.startswith('vd')) - disk_size = int(out[0].split()[3]) - self.assertTrue(disk_size > (RBD_VOL1_SIZE * 0.8) - and disk_size < RBD_VOL1_SIZE * 1.1) - disk = os.path.join('/dev', vd) - cmd_parted_mkpart1 = ('/usr/sbin/parted -s %s ' % disk - + 'mklabel gpt mkpart primary 1 200M') - _, out, err = ssh_exec(ssh, cmd_parted_mkpart1, timeout=5) - self.assertTrue(out == [] and err == []) - cmd_parted_mkpart2 = ('/usr/sbin/parted -s %s ' % disk - + 'mkpart primary 200M 500M') - _, out, err = ssh_exec(ssh, cmd_parted_mkpart2, timeout=5) - self.assertTrue(out == [] and err == []) - - for fs in ['ext3', 'ext4', 'xfs']: - cmd_mkfs = '/sbin/mkfs.%s -F %s1' % (fs, disk) - _, out, err = ssh_exec(ssh, cmd_mkfs, timeout=30) - cmd_mount = '/usr/bin/mount %s1 /mnt' % disk - _, out, err = ssh_exec(ssh, cmd_mount, timeout=5) - cmd_df = '/bin/df | grep /mnt' - _, out, err = ssh_exec(ssh, cmd_df, timeout=5) - self.assertTrue(len(out) and disk in out[0]) - cmd_dd = ('/bin/dd if=/dev/zero of=/mnt/dd.zero ' - + 'bs=1M count=100 oflag=direct') - _, out, err = ssh_exec(ssh, cmd_dd, timeout=30) - _, out, err = ssh_exec(ssh, 'sync', timeout=30) - cmd_umount = '/usr/bin/umount /mnt' - _, out, err = ssh_exec(ssh, cmd_umount, timeout=5) - - cmd_parted_rmpart1 = '/usr/sbin/parted -s %s ' % disk + 'rm 1' - _, out, err = ssh_exec(ssh, cmd_parted_rmpart1, timeout=5) - self.assertTrue(out == [] and err == []) - cmd_parted_rmpart2 = '/usr/sbin/parted -s %s ' % disk + 'rm 2' - _, out, err = ssh_exec(ssh, cmd_parted_rmpart2, timeout=5) - self.assertTrue(out == [] and err == []) - cmd_lsblk = '/usr/bin/lsblk -l|grep %s|grep part' % vd - _, out, err = ssh_exec(ssh, cmd_lsblk, timeout=3) - self.assertTrue(out == []) - - # 卸载卷 - ret = dom.detachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_LIVE) - self.assertTrue(ret >= 0) - xml = dom.XMLDesc(0) - self.assertTrue(diskinfo['disk_name'] not in xml) - - # 挂载scsi disk，测试fstrim操作 - diskinfo = {"protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VOL1), - "mon_hosts": MON_HOSTS, - "target_dev": "sdc" - } - template = JJ_ENV.get_template('disk_xml.j2') - diskxml = template.render(diskinfo) - ret = dom.attachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_LIVE) - self.assertTrue(ret >= 0) - xml = dom.XMLDesc(0) - self.assertTrue(diskinfo['disk_name'] in xml and 'scsi' in xml) - ssh = create_ssh_connect(VM_HOST_IP, timeout=120) - self.assertTrue(ssh is not None) - cmd_lsblk = '/usr/bin/lsblk -b|grep sd|grep disk|tail -1' - _, out, err = ssh_exec(ssh, cmd_lsblk, timeout=3) - self.assertTrue(len(out)) - sd = out[0].split()[0] - self.assertTrue(sd.startswith('sd')) - disk = os.path.join('/dev', sd) - cmd_mkfs = '/sbin/mkfs.ext4 -F %s' % disk - _, out, err = ssh_exec(ssh, cmd_mkfs, timeout=30) - cmd_mount = '/usr/bin/mount %s /mnt' % disk - _, out, err = ssh_exec(ssh, cmd_mount, timeout=5) - cmd_df = '/bin/df | grep /mnt' - _, out, err = ssh_exec(ssh, cmd_df, timeout=5) - self.assertTrue(len(out) and disk in out[0]) - cmd_dd = ('/bin/dd if=/dev/zero of=/mnt/dd.zero ' - + 'bs=1M count=100 oflag=direct') - _, out, err = ssh_exec(ssh, cmd_dd, timeout=30) - cmd_rm = '/bin/rm /mnt/dd.zero' - _, out, err = ssh_exec(ssh, cmd_rm, timeout=5) - cmd_trim = '/sbin/fstrim /mnt' - _, out, err = ssh_exec(ssh, cmd_trim, timeout=30) - self.assertTrue(out == [] and err == []) - - self._destroy_vm(dominfo) - - def test_resize_rbd_disk(self): - dominfo = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_resize_rbd_disk", - "disk_type": "network", - "protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), - "mon_hosts": MON_HOSTS - } - dom = self._create_vm(dominfo) - time.sleep(5) - - if dom.isActive(): - capacity = dom.blockInfo('vda')[0] # Byte - new_size = (capacity + RBD_VM_IMG_SIZE) / 1024 # KB - ret = dom.blockResize('vda', new_size) - self.assertTrue(ret >= 0) - self.assertTrue(dom.blockInfo('vda')[0] > - (capacity + RBD_VM_IMG_SIZE * 0.8)) - else: - self.assertFalse("Domain creates failed") - self._destroy_vm(dominfo) - - def test_live_upgrade_ceph_client(self): - old_version = CEPH_CLIENT_REAL_VERSION[CEPH_CLIENT_OLD_PACKAGE] - new_version = CEPH_CLIENT_REAL_VERSION[CEPH_CLIENT_NEW_PACKAGE] - packages = ['ceph', 'ceph-common', 'librbd1', 'python-rados', - 'python-cephfs', 'python-rbd', 'librados2', - 'libcephfs1', 'python-ceph'] - - rbd_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_live_upgrade_ceph_client", - "disk_type": "network", - "protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VM_IMG), - "mon_hosts": MON_HOSTS - } - local_vm = {"uuid": UUID1, "cpuset": '0-%d' % (MAX_CPUS - 1), - "name": "test_live_upgrade_ceph_client", - "disk_type": "file", - "disk_file": LOCAL_IMG - } - dominfos = [rbd_vm, local_vm] - - for dominfo in dominfos: - # 安装ceph client旧包 - cmd_install_old_package = 'apt-get install' - for p in packages: - cmd_install_old_package = ' '.join([cmd_install_old_package, p]) - cmd_install_old_package += ('=%s' % CEPH_CLIENT_OLD_PACKAGE) - cmd_install_old_package += ' -y --force-yes' - run(cmd_install_old_package, timeout=30) - out = run('/usr/bin/dpkg -l librbd1') - - self.assertTrue(out and CEPH_CLIENT_OLD_PACKAGE in out) - dom = self._create_vm(dominfo) - if not dom.isActive(): - self.assertFalse("Domain creates failed") - # attach disk - diskinfo = {"protocol": "rbd", - "disk_name": os.path.join(RBD_POOL, RBD_VOL1), - "mon_hosts": MON_HOSTS, - "target_dev": "vdc" - } - template = JJ_ENV.get_template('disk_xml.j2') - diskxml = template.render(diskinfo) - ret = dom.attachDeviceFlags(diskxml, libvirt.VIR_DOMAIN_AFFECT_LIVE) - self.assertTrue(ret >= 0) - xml = dom.XMLDesc(0) - self.assertTrue(diskinfo['disk_name'] in xml) - - ssh = create_ssh_connect(VM_HOST_IP, timeout=120) - self.assertTrue(ssh is not None) - cmd_lsblk = '/usr/bin/lsblk -b|grep vd|grep disk|tail -1' - _, out, err = ssh_exec(ssh, cmd_lsblk, timeout=3) - self.assertTrue(len(out)) - vd = out[0].split()[0] - self.assertTrue(vd.startswith('vd')) - disk = os.path.join('/dev', vd) - cmd_dd = ('/bin/dd if=/dev/zero of=%s ' % disk - + 'bs=1M count=100 oflag=direct') - _, out, err = ssh_exec(ssh, cmd_dd, timeout=30) - self.assertEqual([], out) - self.assertTrue(len(err) and 'records' in err[0]) - - # 安装ceph client新包 - cmd_install_new_package = 'apt-get install' - for p in packages: - cmd_install_new_package = ' '.join([cmd_install_new_package, p]) - cmd_install_new_package += ('=%s' % CEPH_CLIENT_NEW_PACKAGE) - cmd_install_new_package += ' -y --force-yes' - run(cmd_install_new_package, timeout=30) - out = run('/usr/bin/dpkg -l librbd1') - self.assertTrue(out and CEPH_CLIENT_NEW_PACKAGE in out) - - nebd_server_pid0 = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - self.assertTrue(nebd_server_pid0 > 0) - # 检查rados admin socket是否正常 - asoks = os.listdir(CEPH_CLIENT_ASOK_DIR) - for asok in asoks: - if asok.startswith('ceph-client.admin.%d' % nebd_server_pid0): - asok_path = os.path.join(CEPH_CLIENT_ASOK_DIR, asok) - cmd_asok = 'ceph daemon %s version' % asok_path - out = run(cmd_asok) - self.assertTrue(out and "version" in out and - old_version in out) - os.kill(nebd_server_pid0, 15) - timeout = 10 - is_reboot = False - while timeout > 0: - timeout -= 1 - nebd_server_pid1 = self._get_pid_by_cmdline(dominfo['uuid'], - NEBD_SERVER_EXEC_FILE) - if (nebd_server_pid1 > 0 and - nebd_server_pid1 != nebd_server_pid0): - is_reboot = True - asoks = os.listdir(CEPH_CLIENT_ASOK_DIR) - for asok in asoks: - if asok.startswith('ceph-client.admin.%d' % - nebd_server_pid1): - asok_path = os.path.join(CEPH_CLIENT_ASOK_DIR, asok) - cmd_asok = 'ceph daemon %s version' % asok_path - out = run(cmd_asok) - self.assertTrue(out and "version" in out and - new_version in out) - break - else: - time.sleep(1) - self.assertTrue(is_reboot) - _, out, err = ssh_exec(ssh, cmd_dd, timeout=30) - self.assertEqual([], out) - self.assertTrue(len(err) and 'records' in err[0]) - self._destroy_vm(dominfo) - self._killall_nebd_servers() - - def test_live_upgrade_curve_client(self): - # TODO - pass - - def test_ceph_rbd_rados_cli(self): - try: - ceph_cmd_base = ('ceph -c %s ' % CEPH_CONF - + '--admin_socket /tmp/ceph-client.asok ') - out = run(ceph_cmd_base + '-s') - self.assertTrue('cluster' in out) - out = run(ceph_cmd_base + 'osd tree') - self.assertTrue('ID' in out and 'WEIGHT' in out) - out = run(ceph_cmd_base + 'osd pool ls detail') - self.assertTrue('pool' in out and RBD_POOL in out) - - rbd_cmd_base = ('rbd -c %s ' % CEPH_CONF - + '--admin_socket /tmp/ceph-client.asok ') - out = run(rbd_cmd_base + 'ls -p %s' % RBD_POOL) - self.assertTrue(RBD_VM_IMG in out) - out = run(rbd_cmd_base - + 'info %s' % os.path.join(RBD_POOL, RBD_VM_IMG)) - self.assertTrue(RBD_VM_IMG in out) - out = run(rbd_cmd_base - + 'create --image-format 2 --size 128 %s' - % os.path.join(RBD_POOL, 'rbd-cli-test')) - self.assertFalse(out) - out = run(rbd_cmd_base - + 'snap create %s' - % os.path.join(RBD_POOL, 'rbd-cli-test@snap-test')) - self.assertFalse(out) - out = run(rbd_cmd_base - + 'snap ls %s' % os.path.join(RBD_POOL, 'rbd-cli-test')) - self.assertTrue('snap-test' in out) - out = run(rbd_cmd_base - + 'snap rm %s' - % os.path.join(RBD_POOL, 'rbd-cli-test@snap-test')) - self.assertFalse(out) - out = run(rbd_cmd_base - + 'rm %s' % os.path.join(RBD_POOL, 'rbd-cli-test'), - ignore='Removing image') - self.assertFalse(out) - - rados_cmd_base = ('rados -c %s ' % CEPH_CONF - + '--admin_socket /tmp/ceph-client.asok ') - out = run(rados_cmd_base + 'lspools') - self.assertTrue(RBD_POOL in out) - out = run(rados_cmd_base + 'df') - self.assertTrue(RBD_POOL in out) - except Exception as exc: - self.assertFalse(str(exc)) - - -def _get_rbd_ctx(): - cluster = rados.Rados(conffile=CEPH_CONF) - cluster.connect() - ioctx = cluster.open_ioctx(RBD_POOL) - rbd_inst = rbd.RBD() - return (ioctx, rbd_inst, cluster) - - -def prepare_rbd_disk(): - if REUSE_RBD_DISK: - return - clear_rbd_disk() - - print 'going to create all rbd disks...' - ioctx, rbd_inst, cluster = _get_rbd_ctx() - rbd_inst.create(ioctx, RBD_VOL1, RBD_VOL1_SIZE, - 22, old_format=False, features=1) - rbd_inst.create(ioctx, RBD_VM_IMG, RBD_VM_IMG_SIZE, - 22, old_format=False, features=1) - image = rbd.Image(ioctx, RBD_VM_IMG) - # LOCAL_IMG需要为raw格式镜像，虚拟机xml里面指定了raw - # 并且导入到rbd卷也需要是raw格式 - f = open(LOCAL_IMG, "rb") - chunk_size = 4*1024*1024 - try: - offset = 0 - data = f.read(chunk_size) - while data != "": - print "Writing data at offset " + str(offset) + \ - "(" + str(offset / 1024 / 1024) + "MB)" - offset += image.write(data, offset) - data = f.read(chunk_size) - finally: - f.close() - image.close() - ioctx.close() - cluster.shutdown() - - -def clear_all_vms(): - print 'going to clear all vms...' - doms = LIBVIRT_CONN.listAllDomains() - for dom in doms: - if dom.isActive(): - dom.destroy() - doms = LIBVIRT_CONN.listAllDomains() - for dom in doms: - if dom.isPersistent(): - dom.undefine() - - -def clear_rbd_disk(): - if REUSE_RBD_DISK: - return - print 'going to rm all rbd disks...' - ioctx, rbd_inst, cluster = _get_rbd_ctx() - for img in [RBD_VM_IMG, RBD_VOL1]: - try: - rbd_inst.remove(ioctx, img) - except rbd.ImageNotFound: - pass - except Exception as exc: - ioctx.close() - cluster.shutdown() - raise exc - ioctx.close() - cluster.shutdown() - - -def clear_log_dirs(): - print 'going to clear all log files...' - for d in ['/var/log/nebd', '/var/log/libvirt/qemu']: - fs = os.listdir(d) - for f in fs: - os.remove(os.path.join(d, f)) - - -def install_rbd_client(): - # 安装ceph client新包 - packages = ['ceph', 'ceph-common', 'librbd1', 'python-rados', - 'python-cephfs', 'python-rbd', 'librados2', - 'libcephfs1', 'python-ceph'] - cmd_install_new_package = 'apt-get install' - for p in packages: - cmd_install_new_package = ' '.join([cmd_install_new_package, p]) - cmd_install_new_package += ('=%s' % CEPH_CLIENT_NEW_PACKAGE) - cmd_install_new_package += ' -y --force-yes' - run(cmd_install_new_package, timeout=30) - out = run('/usr/bin/dpkg -l grep librbd1') - assert(out and CEPH_CLIENT_NEW_PACKAGE in out) - - -if __name__ == '__main__': - if LIBVIRT_CONN is None: - LIBVIRT_CONN = libvirt.open(None) - clear_all_vms() - clear_log_dirs() - install_rbd_client() - prepare_rbd_disk() - path = '{}/templates/'.format(os.path.dirname(os.path.abspath(__file__))) - if JJ_ENV is None: - JJ_ENV = Environment(loader=FileSystemLoader(path)) - - unittest.main() - - LIBVIRT_CONN.close() - clear_rbd_disk() From 566f4e00eaf655ca94e52f42232189f186d1bde4 Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Fri, 17 Jan 2020 10:50:44 +0800 Subject: [PATCH 18/79] update arch Change-Id: I4bea6749eaa26bef532b330b356b0906279428df --- src/part2/define.h | 12 +++++++++--- src/part2/file_manager.cpp | 11 +++++++++-- src/part2/file_manager.h | 16 ++++++++++++---- src/part2/metafile_manager.cpp | 6 +++--- src/part2/metafile_manager.h | 6 +++--- src/part2/request_executor.h | 6 +++--- src/part2/request_executor_ceph.cpp | 4 ++-- src/part2/request_executor_ceph.h | 4 ++-- src/part2/request_executor_curve.cpp | 4 ++-- src/part2/request_executor_curve.h | 4 ++-- 10 files changed, 47 insertions(+), 26 deletions(-) diff --git a/src/part2/define.h b/src/part2/define.h index c8fad264cd..71c1e86afc 100644 --- a/src/part2/define.h +++ b/src/part2/define.h @@ -34,7 +34,7 @@ enum class NebdFileType { class NebdFileInstance; class NebdRequestExecutor; -struct NebdFileInfo { +struct NebdFileRecord { int fd; std::string fileName; NebdFileType type; @@ -50,7 +50,7 @@ struct NebdServerAioContext; // nebd回调函数的类型 typedef void (*NebdAioCallBack)(struct NebdServerAioContext* context); -typedef struct NebdServerAioContext { +struct NebdServerAioContext { off_t offset; // 请求的offset size_t length; // 请求的length int ret; // 记录异步返回的返回值 @@ -59,7 +59,13 @@ typedef struct NebdServerAioContext { void* buf; // 请求的buf Message* response; // 请求返回内容 Closure *done; // 请求回调函数 -} ClientAioContext; +}; + +struct NebdFileInfo { + uint64_t size; + uint64_t obj_size; + uint64_t num_objs; +}; } // namespace server } // namespace nebd diff --git a/src/part2/file_manager.cpp b/src/part2/file_manager.cpp index d5bd0d159e..3a952f4c10 100644 --- a/src/part2/file_manager.cpp +++ b/src/part2/file_manager.cpp @@ -1,3 +1,10 @@ +/* + * Project: nebd + * Created Date: Thursday January 16th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + #include "src/part2/file_manager.h" namespace nebd { @@ -43,7 +50,7 @@ int NebdFileManager::Extend(int fd, int64_t newsize) { return 0; } -int NebdFileManager::StatFile(int fd) { +int NebdFileManager::StatFile(int fd, NebdFileInfo* fileInfo) { // TODO return 0; } @@ -68,7 +75,7 @@ int NebdFileManager::Flush(int fd, NebdServerAioContext* aioctx) { return 0; } -int NebdFileManager::GetInfo(int fd) { +int NebdFileManager::GetInfo(int fd, NebdFileInfo* fileInfo) { // TODO return 0; } diff --git a/src/part2/file_manager.h b/src/part2/file_manager.h index 22f52199e7..a281bffb43 100644 --- a/src/part2/file_manager.h +++ b/src/part2/file_manager.h @@ -1,3 +1,10 @@ +/* + * Project: nebd + * Created Date: Thursday January 16th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + #ifndef SRC_PART2_FILE_MANAGER_H_ #define SRC_PART2_FILE_MANAGER_H_ @@ -29,20 +36,21 @@ class NebdFileManager { virtual int Open(const std::string& filename); virtual int Close(int fd); virtual int Extend(int fd, int64_t newsize); - virtual int StatFile(int fd); + virtual int GetInfo(int fd, NebdFileInfo* fileInfo); + virtual int StatFile(int fd, NebdFileInfo* fileInfo); virtual int Discard(int fd, NebdServerAioContext* aioctx); virtual int AioRead(int fd, NebdServerAioContext* aioctx); virtual int AioWrite(int fd, NebdServerAioContext* aioctx); virtual int Flush(int fd, NebdServerAioContext* aioctx); - virtual int GetInfo(int fd); virtual int InvalidCache(int fd); private: void CheckTimeoutFunc(); private: - using FileInfoMap = std::unordered_map>; - FileInfoMap fileInfoMap_; + using FileRecordMap = + std::unordered_map>; + FileRecordMap fileRecordMap_; MetaFileManagerPtr metaFileManager_; uint32_t heartbeatTimeoutS_; std::thread checkTimeoutThread_; diff --git a/src/part2/metafile_manager.cpp b/src/part2/metafile_manager.cpp index 888978a748..a7b483a5cf 100644 --- a/src/part2/metafile_manager.cpp +++ b/src/part2/metafile_manager.cpp @@ -10,17 +10,17 @@ NebdMetaFileManager::NebdMetaFileManager(const std::string& metaFilePath) NebdMetaFileManager::~NebdMetaFileManager() {} -int NebdMetaFileManager::RemoveFileInfo(const std::string& fileName) { +int NebdMetaFileManager::RemoveFileRecord(const std::string& fileName) { // TODO return 0; } -int NebdMetaFileManager::UpdateFileInfo(const NebdFileInfo& fileInfo) { +int NebdMetaFileManager::UpdateFileRecord(const NebdFileRecord& fileRecord) { // TODO return 0; } -int NebdMetaFileManager::ListFileInfo(std::vector* fileInfos) { +int NebdMetaFileManager::ListFileRecord(std::vector* fileRecords) { // TODO return 0; } diff --git a/src/part2/metafile_manager.h b/src/part2/metafile_manager.h index faeb7d0d31..5db3b7a114 100644 --- a/src/part2/metafile_manager.h +++ b/src/part2/metafile_manager.h @@ -14,9 +14,9 @@ class NebdMetaFileManager { explicit NebdMetaFileManager(const std::string& metaFilePath); virtual ~NebdMetaFileManager(); - virtual int RemoveFileInfo(const std::string& fileName); - virtual int UpdateFileInfo(const NebdFileInfo& fileInfo); - virtual int ListFileInfo(std::vector* fileInfos); + virtual int RemoveFileRecord(const std::string& fileName); + virtual int UpdateFileRecord(const NebdFileRecord& fileRecord); + virtual int ListFileRecord(std::vector* fileRecords); private: std::string metaFilePath_; diff --git a/src/part2/request_executor.h b/src/part2/request_executor.h index 0fc3452e9b..084f0e130e 100644 --- a/src/part2/request_executor.h +++ b/src/part2/request_executor.h @@ -19,12 +19,12 @@ class NebdRequestExcutor { virtual std::shared_ptr Open(const std::string& filename) = 0; // NOLINT virtual int Close(NebdFileInstance* fd) = 0; virtual int Extend(NebdFileInstance* fd, int64_t newsize) = 0; - virtual int StatFile(NebdFileInstance* fd) = 0; + virtual int GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) = 0; + virtual int StatFile(NebdFileInstance* fd, NebdFileInfo* fileInfo) = 0; virtual int Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) = 0; virtual int AioRead(NebdFileInstance* fd, NebdServerAioContext* aioctx) = 0; - virtual int AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) = 0; + virtual int AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) = 0; // NOLINT virtual int Flush(NebdFileInstance* fd, NebdServerAioContext* aioctx) = 0; - virtual int GetInfo(NebdFileInstance* fd) = 0; virtual int InvalidCache(NebdFileInstance* fd) = 0; }; diff --git a/src/part2/request_executor_ceph.cpp b/src/part2/request_executor_ceph.cpp index 1d0bd91789..16e319652f 100644 --- a/src/part2/request_executor_ceph.cpp +++ b/src/part2/request_executor_ceph.cpp @@ -19,7 +19,7 @@ int CephRequestExcutor::Extend(NebdFileInstance* fd, int64_t newsize) { return 0; } -int CephRequestExcutor::StatFile(NebdFileInstance* fd) { +int CephRequestExcutor::StatFile(NebdFileInstance* fd, NebdFileInfo* fileInfo) { // TODO return 0; } @@ -44,7 +44,7 @@ int CephRequestExcutor::Flush(NebdFileInstance* fd, NebdServerAioContext* aioctx return 0; } -int CephRequestExcutor::GetInfo(NebdFileInstance* fd) { +int CephRequestExcutor::GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) { // TODO return 0; } diff --git a/src/part2/request_executor_ceph.h b/src/part2/request_executor_ceph.h index 206fda0ae1..65e9776f59 100644 --- a/src/part2/request_executor_ceph.h +++ b/src/part2/request_executor_ceph.h @@ -24,12 +24,12 @@ class CephRequestExcutor : public NebdRequestExcutor { std::shared_ptr Open(const std::string& filename) override; // NOLINT int Close(NebdFileInstance* fd) override; int Extend(NebdFileInstance* fd, int64_t newsize) override; - int StatFile(NebdFileInstance* fd) override; + int GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) override; + int StatFile(NebdFileInstance* fd, NebdFileInfo* fileInfo) override; int Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; int AioRead(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; int AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; int Flush(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; - int GetInfo(NebdFileInstance* fd) override; int InvalidCache(NebdFileInstance* fd) override; }; diff --git a/src/part2/request_executor_curve.cpp b/src/part2/request_executor_curve.cpp index 3be98208e9..3d5351bc60 100644 --- a/src/part2/request_executor_curve.cpp +++ b/src/part2/request_executor_curve.cpp @@ -19,7 +19,7 @@ int CurveRequestExcutor::Extend(NebdFileInstance* fd, int64_t newsize) { return 0; } -int CurveRequestExcutor::StatFile(NebdFileInstance* fd) { +int CurveRequestExcutor::StatFile(NebdFileInstance* fd, NebdFileInfo* fileInfo) { // TODO return 0; } @@ -44,7 +44,7 @@ int CurveRequestExcutor::Flush(NebdFileInstance* fd, NebdServerAioContext* aioct return 0; } -int CurveRequestExcutor::GetInfo(NebdFileInstance* fd) { +int CurveRequestExcutor::GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) { // TODO return 0; } diff --git a/src/part2/request_executor_curve.h b/src/part2/request_executor_curve.h index eb51c74307..3f573b95b2 100644 --- a/src/part2/request_executor_curve.h +++ b/src/part2/request_executor_curve.h @@ -24,12 +24,12 @@ class CurveRequestExcutor : public NebdRequestExcutor { std::shared_ptr Open(const std::string& filename) override; // NOLINT int Close(NebdFileInstance* fd) override; int Extend(NebdFileInstance* fd, int64_t newsize) override; - int StatFile(NebdFileInstance* fd) override; + int GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) override; + int StatFile(NebdFileInstance* fd, NebdFileInfo* fileInfo) override; int Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; int AioRead(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; int AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; int Flush(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; - int GetInfo(NebdFileInstance* fd) override; int InvalidCache(NebdFileInstance* fd) override; }; From 36f4a95bcf183f59a4e8aec6814feb0ec5c49991 Mon Sep 17 00:00:00 2001 From: majie1 Date: Mon, 3 Feb 2020 17:41:03 +0800 Subject: [PATCH 19/79] add filterbr to fix coverage branch statistics Change-Id: Id78a6a86f0f12b2993e73102a811fbe1756976f0 --- docker_unittest.sh | 5 +- filterbr.py | 113 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 1 deletion(-) create mode 100755 filterbr.py diff --git a/docker_unittest.sh b/docker_unittest.sh index b70138d439..b9555d1163 100755 --- a/docker_unittest.sh +++ b/docker_unittest.sh @@ -28,6 +28,9 @@ for i in ${gcda}; do cp --parents $j coverage done +export LC_ALL=C.UTF-8 lcov --directory coverage$(pwd)/build --capture --output-file cov.info --rc lcov_branch_coverage=1 lcov --remove cov.info '/usr/*' '*3rdparty*' '*/build/*' -o result.info --rc lcov_branch_coverage=1 -genhtml result.info --branch-coverage --ignore-errors source -o result \ No newline at end of file +python3 filterbr.py result.info > filterbr.info +genhtml filterbr.info --branch-coverage --ignore-errors source -o result + diff --git a/filterbr.py b/filterbr.py new file mode 100755 index 0000000000..16ea991562 --- /dev/null +++ b/filterbr.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 + +# 2017, Georg Sauthoff , GPLv3 + +import sys + +def skip_comments(lines): + state = 0 + for line in lines: + n = len(line) + l = '' + p = 0 + while p < n: + if state == 0: + a = line.find('//', p) + b = line.find('/*', p) + if a > -1 and (a < b or b == -1): + l += line[p:a] + p = n + elif b > -1 and (b < a or a == -1): + l += line[p:b] + p = b+2 + state = 1 + else: + l += line[p:] + p = n + elif state == 1: + a = line.rfind('*/', p) + if a == -1: + p = n + else: + p = a + 2 + state = 0 + yield l + +def cond_lines(lines): + state = 0 + pcnt = 0 + for nr, line in enumerate(lines, 1): + if not line: + continue + n = len(line) + p = 0 + do_yield = False + while p < n: + if state == 0: +# p = line.strip().startswith('if', p) + p = line.find('if', p) + if p == -1: + p = n + continue + if (p == 0 or not line[p-1].isalpha()) \ + and (p+2 == len(line) or not line[p+2].isalpha()): + do_yield = True + state = 1 + p += 2 + elif state == 1: + do_yield = True + p = line.find('(', p) + if p == -1: + p = n + else: + p += 1 + state = 2 + pcnt = 1 + elif state == 2: + do_yield = True + for p in range(p, n): + if line[p] == '(': + pcnt += 1 + elif line[p] == ')': + pcnt -= 1 + if not pcnt: + state = 0 + break + p += 1 + if do_yield: + yield nr + +def cond_lines_from_file(filename): + with open(filename) as f: + for line in cond_lines(skip_comments(f)): + yield line + # yield from cond_lines(skip_comments(f)) + +def filter_lcov_trace(lines): + nrs = set() + for line in lines: + if line.startswith('SF:'): + nrs = set(cond_lines_from_file(line[3:-1])) + elif line.startswith('BRDA:'): + xs = line[5:].split(',') + nr = int(xs[0]) if xs else 0 + if nr not in nrs: + continue + yield line + +def filter_lcov_trace_file(s_filename, d_file): + with open(s_filename) as f: + for l in filter_lcov_trace(f): + print(l, end='', file=d_file) + +if __name__ == '__main__': + #for l in cond_lines_from_file(sys.argv[1]): + # print(l) + + filter_lcov_trace_file(sys.argv[1], sys.stdout) + + #with open(sys.argv[1]) as f: + # for l in skip_comments(f): + # print(l) + + From b98b8d9c8d8c5662b58c7f91fe79e62c733c788e Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Sun, 19 Jan 2020 16:15:17 +0800 Subject: [PATCH 20/79] implement filemanager Change-Id: I2709c54088d7bdbdac306f704dde34092acc6ee4 --- src/common/CMakeLists.txt | 5 +- src/common/interrupt_sleep.h | 2 +- src/common/name_lock.cpp | 95 +++ src/common/name_lock.h | 93 +++ src/common/timeutility.h | 53 ++ src/part1/CMakeLists.txt | 13 +- src/part2/CMakeLists.txt | 1 + src/part2/define.h | 62 +- src/part2/file_manager.cpp | 439 ++++++++++++- src/part2/file_manager.h | 191 +++++- src/part2/file_record_map.cpp | 96 +++ src/part2/file_record_map.h | 88 +++ src/part2/metafile_manager.cpp | 8 +- src/part2/metafile_manager.h | 4 +- src/part2/request_executor.cpp | 38 ++ src/part2/request_executor.h | 38 +- src/part2/request_executor_ceph.cpp | 27 +- src/part2/request_executor_ceph.h | 17 +- src/part2/request_executor_curve.cpp | 27 +- src/part2/request_executor_curve.h | 17 +- src/part2/util.cpp | 68 ++ src/part2/util.h | 44 ++ tests/common/CMakeLists.txt | 20 +- tests/common/interruptible_sleeper_test.cpp | 44 ++ tests/common/rw_lock_test.cpp | 88 +++ tests/common/test_name_lock.cpp | 91 +++ tests/part2/CMakeLists.txt | 31 + tests/part2/file_manager_unittest.cpp | 677 ++++++++++++++++++++ tests/part2/file_record_map_unittest.cpp | 101 +++ tests/part2/mock_metafile_manager.h | 33 + tests/part2/mock_request_executor.h | 48 ++ 31 files changed, 2444 insertions(+), 115 deletions(-) create mode 100644 src/common/name_lock.cpp create mode 100644 src/common/name_lock.h create mode 100644 src/common/timeutility.h create mode 100644 src/part2/file_record_map.cpp create mode 100644 src/part2/file_record_map.h create mode 100644 src/part2/request_executor.cpp create mode 100644 src/part2/util.cpp create mode 100644 src/part2/util.h create mode 100644 tests/common/interruptible_sleeper_test.cpp create mode 100644 tests/common/rw_lock_test.cpp create mode 100644 tests/common/test_name_lock.cpp create mode 100644 tests/part2/file_manager_unittest.cpp create mode 100644 tests/part2/file_record_map_unittest.cpp create mode 100644 tests/part2/mock_metafile_manager.h create mode 100644 tests/part2/mock_request_executor.h diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 6f8982e3a3..354476b7cf 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -8,9 +8,12 @@ file (GLOB PROTO_FILES "*.proto" ) +set(COMMON_LINK + brpc-shared) + add_library(nebd_common SHARED ${COMMON_SRC}) install(TARGETS nebd_common DESTINATION lib) -target_link_libraries(nebd_common ${EXTRALIBS}) +target_link_libraries(nebd_common ${COMMON_LINK} ${EXTRALIBS}) # libclient_proto.so include(FindProtobuf) diff --git a/src/common/interrupt_sleep.h b/src/common/interrupt_sleep.h index e0d7792151..fc3deb6d59 100644 --- a/src/common/interrupt_sleep.h +++ b/src/common/interrupt_sleep.h @@ -27,7 +27,7 @@ class InterruptibleSleeper { void interrupt() { std::unique_lock lock(m); terminate = true; - LOG(ERROR) << "interrupt sleep."; + LOG(WARNING) << "interrupt sleeper."; cv.notify_all(); } private: diff --git a/src/common/name_lock.cpp b/src/common/name_lock.cpp new file mode 100644 index 0000000000..97a7215c15 --- /dev/null +++ b/src/common/name_lock.cpp @@ -0,0 +1,95 @@ + +/* + * Project: nebd + * Created Date: Thu Aug 08 2019 + * Author: xuchaojie + * Copyright (c) 2019 netease + */ + +#include "src/common/name_lock.h" + +namespace nebd { +namespace common { + +using LockGuard = std::lock_guard; + +NameLock::NameLock(int bucketNum) { + locks_.reserve(bucketNum); + for (int i = 0; i < bucketNum; i++) { + locks_.push_back(std::make_shared()); + } +} + +void NameLock::Lock(const std::string &lockStr) { + LockEntryPtr entry = NULL; + + int bucketOffset = GetBucketOffset(lockStr); + LockBucketPtr lockBucket = locks_[bucketOffset]; + + { + LockGuard guard(lockBucket->mu); + auto it = lockBucket->lockMap.find(lockStr); + if (it == lockBucket->lockMap.end()) { + entry = std::make_shared(); + entry->ref_.store(1); + lockBucket->lockMap.emplace(lockStr, entry); + } else { + entry = it->second; + entry->ref_.fetch_add(1); + } + } + entry->lock_.lock(); +} + +bool NameLock::TryLock(const std::string &lockStr) { + LockEntryPtr entry = NULL; + + int bucketOffset = GetBucketOffset(lockStr); + LockBucketPtr lockBucket = locks_[bucketOffset]; + + { + LockGuard guard(lockBucket->mu); + auto it = lockBucket->lockMap.find(lockStr); + if (it == lockBucket->lockMap.end()) { + entry = std::make_shared(); + entry->ref_.store(1); + lockBucket->lockMap.emplace(lockStr, entry); + } else { + entry = it->second; + entry->ref_.fetch_add(1); + } + } + if (entry->lock_.try_lock()) { + return true; + } else { + LockGuard guard(lockBucket->mu); + if (entry->ref_.fetch_sub(1) == 1) { + lockBucket->lockMap.erase(lockStr); + } + return false; + } +} + +void NameLock::Unlock(const std::string &lockStr) { + int bucketOffset = GetBucketOffset(lockStr); + LockBucketPtr lockBucket = locks_[bucketOffset]; + + LockGuard guard(lockBucket->mu); + auto it = lockBucket->lockMap.find(lockStr); + if (it != lockBucket->lockMap.end()) { + LockEntryPtr entry = it->second; + entry->lock_.unlock(); + if (entry->ref_.fetch_sub(1) == 1) { + lockBucket->lockMap.erase(it); + } + } +} + +int NameLock::GetBucketOffset(const std::string &lockStr) { + std::hash hash_fn; + return hash_fn(lockStr) % locks_.size(); +} + + +} // namespace common +} // namespace nebd diff --git a/src/common/name_lock.h b/src/common/name_lock.h new file mode 100644 index 0000000000..3beab33868 --- /dev/null +++ b/src/common/name_lock.h @@ -0,0 +1,93 @@ + +/* + * Project: nebd + * Created Date: Thu Aug 08 2019 + * Author: xuchaojie + * Copyright (c) 2019 netease + */ + +#ifndef SRC_COMMON_NEMA_LOCK_H_ +#define SRC_COMMON_NEMA_LOCK_H_ + +#include +#include +#include +#include +#include +#include // NOLINT +#include "src/common/uncopyable.h" + +namespace nebd { +namespace common { + +class NameLock : public Uncopyable { + public: + explicit NameLock(int bucketNum = 256); + + /** + * @brief 对指定string加锁 + * + * @param lockStr 被加锁的string + */ + void Lock(const std::string &lockStr); + + /** + * @brief 尝试指定sting加锁 + * + * @param lockStr 被加锁的string + * + * @retval 成功 + * @retval 失败 + */ + bool TryLock(const std::string &lockStr); + + /** + * @brief 对指定string解锁 + * + * @param lockStr 被加锁的string + */ + void Unlock(const std::string &lockStr); + + + private: + struct LockEntry { + std::atomic ref_; + std::mutex lock_; + }; + using LockEntryPtr = std::shared_ptr; + + struct LockBucket { + std::mutex mu; + std::unordered_map lockMap; + }; + using LockBucketPtr = std::shared_ptr; + + int GetBucketOffset(const std::string &lockStr); + + private: + std::vector locks_; +}; + +class NameLockGuard : public Uncopyable { + public: + NameLockGuard(NameLock &lock, const std::string &lockStr) : //NOLINT + lock_(lock), + lockStr_(lockStr) { + lock_.Lock(lockStr_); + } + + ~NameLockGuard() { + lock_.Unlock(lockStr_); + } + + private: + NameLock &lock_; + std::string lockStr_; +}; + + +} // namespace common +} // namespace nebd + + +#endif // SRC_COMMON_NEMA_LOCK_H_ diff --git a/src/common/timeutility.h b/src/common/timeutility.h new file mode 100644 index 0000000000..6da3624db9 --- /dev/null +++ b/src/common/timeutility.h @@ -0,0 +1,53 @@ +/* + * Project: curve + * Created Date: Wednesday August 22nd 2018 + * Author: hzsunjianliang + * Copyright (c) 2018 netease + */ +#ifndef SRC_COMMON_TIMEUTILITY_H_ +#define SRC_COMMON_TIMEUTILITY_H_ + +#include +#include +#include +#include +#include + +namespace nebd { +namespace common { + +class TimeUtility { + public: + static inline uint64_t GetTimeofDayUs() { + timeval now; + gettimeofday(&now, NULL); + return now.tv_sec * 1000000L + now.tv_usec; + } + + static inline uint64_t GetTimeofDayMs() { + timeval now; + gettimeofday(&now, NULL); + return now.tv_sec * 1000L + now.tv_usec / 1000; + } + + static inline uint64_t GetTimeofDaySec() { + timeval tm; + gettimeofday(&tm, NULL); + return tm.tv_sec; + } + + // 时间戳转成标准时间输出在standard里面,时间戳单位为秒 + static inline void TimeStampToStandard(time_t timeStamp, + std::string* standard) { + char now[64]; + struct tm p; + p = *localtime_r(&timeStamp, &p); + strftime(now, 64, "%Y-%m-%d %H:%M:%S", &p); + *standard = std::string(now); + } +}; + +} // namespace common +} // namespace nebd + +#endif // SRC_COMMON_TIMEUTILITY_H_ diff --git a/src/part1/CMakeLists.txt b/src/part1/CMakeLists.txt index cef353fd67..363ba5c899 100644 --- a/src/part1/CMakeLists.txt +++ b/src/part1/CMakeLists.txt @@ -11,15 +11,10 @@ add_definitions( "-DGFLAGS=gflags -DOS_LINUX -DSNAPPY -DHAVE_ZLIB -DHAVE_SSE42 -DNDEBUG") # libnebd.so -set(LIBNEBD_SRC - libnebd.cpp - libnebd.h - libnebd_file.cpp - libnebd_file.h - nebd_client.cpp - nebd_client.h - connection_manager.cpp - connection_manager.h) +file(GLOB + LIBNEBD_SRC + "*.cpp" + "*.h") set(LIBNEBD_LINK brpc-shared rt diff --git a/src/part2/CMakeLists.txt b/src/part2/CMakeLists.txt index 51b1513566..a3b7a33c66 100644 --- a/src/part2/CMakeLists.txt +++ b/src/part2/CMakeLists.txt @@ -17,6 +17,7 @@ file(GLOB "*.cpp") set(PART2_LINK + nebd_common brpc-shared config crypto diff --git a/src/part2/define.h b/src/part2/define.h index 71c1e86afc..bb9d173d0c 100644 --- a/src/part2/define.h +++ b/src/part2/define.h @@ -3,6 +3,7 @@ #include #include +#include #include "src/common/rw_lock.h" @@ -13,6 +14,10 @@ using ::google::protobuf::Closure; namespace nebd { namespace server { +const char CURVE_PREFIX[] = "cbd"; +const char CEPH_PREFIX[] = "rbd"; +const char TEST_PREFIX[] = "test"; + // nebd异步请求的类型 enum class LIBAIO_OP { LIBAIO_OP_READ, @@ -29,41 +34,68 @@ enum class NebdFileStatus { enum class NebdFileType { CEPH = 0, CURVE = 1, + TEST = 2, + UNKWOWN = 3, }; class NebdFileInstance; class NebdRequestExecutor; +using NebdFileInstancePtr = std::shared_ptr; +// nebd server记录的文件信息内存结构 struct NebdFileRecord { - int fd; - std::string fileName; - NebdFileType type; - NebdFileStatus status; - uint64_t timeStamp; + // 文件读写锁，处理请求前加读锁，close文件的时候加写锁 + // 避免close时还有请求未处理完 RWLock rwLock; - NebdFileInstance* fileInstance; - NebdRequestExecutor* executor; + // nebd server为该文件分配的唯一标识符 + int fd = 0; + // 文件名称 + std::string fileName = ""; + // 文件类型：ceph文件、curve文件或测试文件 + NebdFileType type = NebdFileType::UNKWOWN; + // 文件当前状态，opened表示文件已打开，closed表示文件已关闭 + NebdFileStatus status = NebdFileStatus::CLOSED; + // 该文件上一次收到心跳时的时间戳 + uint64_t timeStamp = 0; + // 文件在executor open时返回上下文信息，用于后续文件的请求处理 + NebdFileInstancePtr fileInstance = nullptr; + // 文件实际的执行处理对象 + NebdRequestExecutor* executor = nullptr; }; +using NebdFileRecordPtr = std::shared_ptr; struct NebdServerAioContext; // nebd回调函数的类型 typedef void (*NebdAioCallBack)(struct NebdServerAioContext* context); +// nebd server端异步请求的上下文 +// 记录请求的类型、参数、返回信息、rpc信息 struct NebdServerAioContext { - off_t offset; // 请求的offset - size_t length; // 请求的length - int ret; // 记录异步返回的返回值 - LIBAIO_OP op; // 异步请求的类型，详见定义 - NebdAioCallBack cb; // 异步请求的回调函数 - void* buf; // 请求的buf - Message* response; // 请求返回内容 - Closure *done; // 请求回调函数 + // 请求的offset + off_t offset; + // 请求的length + size_t length; + // 记录异步返回的返回值 + int ret; + // 异步请求的类型，详见定义 + LIBAIO_OP op; + // 异步请求结束时调用的回调函数 + NebdAioCallBack cb; + // 请求的buf + void* buf; + // rpc请求的相应内容 + Message* response; + // rpc请求的回调函数 + Closure *done; }; struct NebdFileInfo { + // 文件大小 uint64_t size; + // object大小（ceph为object，curve为chunk） uint64_t obj_size; + // object数量 uint64_t num_objs; }; diff --git a/src/part2/file_manager.cpp b/src/part2/file_manager.cpp index 3a952f4c10..83ff6d3c30 100644 --- a/src/part2/file_manager.cpp +++ b/src/part2/file_manager.cpp @@ -5,88 +5,459 @@ * Copyright (c) 2020 netease */ +#include +#include // NOLINT +#include + #include "src/part2/file_manager.h" +#include "src/part2/util.h" namespace nebd { namespace server { -NebdFileManager::NebdFileManager() - : metaFileManager_(nullptr) - , heartbeatTimeoutS_(10) { +NebdFileManager::NebdFileManager() + : heartbeatTimeoutS_(0) + , checkTimeoutIntervalMs_(0) + , isRunning_(false) + , metaFileManager_(nullptr) {} + +NebdFileManager::~NebdFileManager() {} +int NebdFileManager::Init(NebdFileManagerOption option) { + if (isRunning_.load()) { + LOG(WARNING) << "Init failed, file manager is on running. "; + return -1; + } + metaFileManager_ = option.metaFileManager; + heartbeatTimeoutS_ = option.heartbeatTimeoutS; + checkTimeoutIntervalMs_ = option.checkTimeoutIntervalMs; + return 0; } -NebdFileManager::~NebdFileManager() { - // TODO +int NebdFileManager::Run() { + if (isRunning_.exchange(true)) { + LOG(WARNING) << "file manager is on running."; + return -1; + } + + CHECK(metaFileManager_ != nullptr) << "meta file manager is null."; + int ret = Load(); + if (ret < 0) { + LOG(ERROR) << "Run file manager failed."; + isRunning_.store(false); + return -1; + } + + checkTimeoutThread_ = + std::thread(&NebdFileManager::CheckTimeoutFunc, this); + LOG(INFO) << "Run file manager success."; + return 0; } -int NebdFileManager::Init(NebdFileManagerOption option) { - // TODO +int NebdFileManager::Fini() { + if (isRunning_.exchange(false)) { + LOG(INFO) << "Stop file manager..."; + sleeper_.interrupt(); + checkTimeoutThread_.join(); + } + LOG(INFO) << "Stop file manager ok."; return 0; } int NebdFileManager::Load() { - // TODO + std::vector fileRecords; + // 从元数据文件中读取持久化的文件信息 + int ret = metaFileManager_->ListFileRecord(&fileRecords); + if (ret < 0) { + LOG(ERROR) << "List file record failed."; + return ret; + } + // 根据持久化的信息重新open文件 + int maxFd = 0; + for (auto& fileRecord : fileRecords) { + maxFd = std::max(maxFd, fileRecord->fd); + Reopen(fileRecord); + bool updateSuccess = fileRecordMap_.UpdateRecord(fileRecord); + if (!updateSuccess) { + LOG(ERROR) << "Update file record failed. " + << "filename: " << fileRecord->fileName; + return -1; + } + } + fdAlloc_.InitFd(maxFd); + LOG(INFO) << "Load file record finished."; return 0; } int NebdFileManager::UpdateFileTimestamp(int fd) { - // TODO + NebdFileRecordPtr fileRecord = fileRecordMap_.GetRecord(fd); + if (fileRecord != nullptr) { + fileRecord->timeStamp = TimeUtility::GetTimeofDayMs(); + } return 0; } int NebdFileManager::Open(const std::string& filename) { - // TODO - return 0; + int ret = OpenInternal(filename, true); + if (ret < 0) { + LOG(ERROR) << "open file failed. " + << "filename: " << filename + << ", ret: " << ret; + return -1; + } + return ret; } int NebdFileManager::Close(int fd) { - // TODO + NebdFileRecordPtr fileRecord = fileRecordMap_.GetRecord(fd); + if (fileRecord == nullptr) { + LOG(WARNING) << "File record not exist, fd: " << fd; + return 0; + } + int ret = CloseInternal(fileRecord); + if (ret < 0) { + LOG(ERROR) << "Close file failed. " + << "fd: " << fd + << ", filename: " << fileRecord->fileName; + return -1; + } + ret = metaFileManager_->RemoveFileRecord(fileRecord->fileName); + if (ret < 0) { + LOG(ERROR) << "Close file failed. " + << "fd: " << fd + << ", filename: " << fileRecord->fileName; + return -1; + } + fileRecordMap_.RemoveRecord(fd); + LOG(INFO) << "Close file success. " + << "fd: " << fd + << ", filename: " << fileRecord->fileName; return 0; } +int NebdFileManager::Discard(int fd, NebdServerAioContext* aioctx) { + auto task = [&](NebdProcessClosure* done) { + NebdFileRecordPtr fileRecord = done->GetFileRecord(); + done->SetClosure(aioctx->done); + aioctx->done = done; + int ret = fileRecord->executor->Discard( + fileRecord->fileInstance.get(), aioctx); + if (ret < 0) { + aioctx->done = done->GetClosure(); + done->SetClosure(nullptr); + done->Run(); + LOG(ERROR) << "Discard file failed. " + << "fd: " << fd + << ", fileName: " << fileRecord->fileName + << ", context: " << *aioctx; + return -1; + } + return 0; + }; + return ProcessRequest(fd, task); +} + +int NebdFileManager::AioRead(int fd, NebdServerAioContext* aioctx) { + auto task = [&](NebdProcessClosure* done) { + NebdFileRecordPtr fileRecord = done->GetFileRecord(); + done->SetClosure(aioctx->done); + aioctx->done = done; + int ret = fileRecord->executor->AioRead( + fileRecord->fileInstance.get(), aioctx); + if (ret < 0) { + aioctx->done = done->GetClosure(); + done->SetClosure(nullptr); + done->Run(); + LOG(ERROR) << "AioRead file failed. " + << "fd: " << fd + << ", fileName: " << fileRecord->fileName + << ", context: " << *aioctx; + return -1; + } + return 0; + }; + return ProcessRequest(fd, task); +} + +int NebdFileManager::AioWrite(int fd, NebdServerAioContext* aioctx) { + auto task = [&](NebdProcessClosure* done) { + NebdFileRecordPtr fileRecord = done->GetFileRecord(); + done->SetClosure(aioctx->done); + aioctx->done = done; + int ret = fileRecord->executor->AioWrite( + fileRecord->fileInstance.get(), aioctx); + if (ret < 0) { + aioctx->done = done->GetClosure(); + done->SetClosure(nullptr); + done->Run(); + LOG(ERROR) << "AioWrite file failed. " + << "fd: " << fd + << ", fileName: " << fileRecord->fileName + << ", context: " << *aioctx; + return -1; + } + return 0; + }; + return ProcessRequest(fd, task); +} + +int NebdFileManager::Flush(int fd, NebdServerAioContext* aioctx) { + auto task = [&](NebdProcessClosure* done) { + NebdFileRecordPtr fileRecord = done->GetFileRecord(); + done->SetClosure(aioctx->done); + aioctx->done = done; + int ret = fileRecord->executor->Flush( + fileRecord->fileInstance.get(), aioctx); + if (ret < 0) { + aioctx->done = done->GetClosure(); + done->SetClosure(nullptr); + done->Run(); + LOG(ERROR) << "Flush file failed. " + << "fd: " << fd + << ", fileName: " << fileRecord->fileName + << ", context: " << *aioctx; + return -1; + } + return 0; + }; + return ProcessRequest(fd, task); +} + int NebdFileManager::Extend(int fd, int64_t newsize) { - // TODO - return 0; + auto task = [&](NebdProcessClosure* done) { + brpc::ClosureGuard doneGuard(done); + NebdFileRecordPtr fileRecord = done->GetFileRecord(); + int ret = fileRecord->executor->Extend( + fileRecord->fileInstance.get(), newsize); + if (ret < 0) { + LOG(ERROR) << "Extend file failed. " + << "fd: " << fd + << ", newsize: " << newsize + << ", fileName" << fileRecord->fileName; + return -1; + } + return 0; + }; + return ProcessRequest(fd, task); } -int NebdFileManager::StatFile(int fd, NebdFileInfo* fileInfo) { - // TODO - return 0; +int NebdFileManager::GetInfo(int fd, NebdFileInfo* fileInfo) { + auto task = [&](NebdProcessClosure* done) { + brpc::ClosureGuard doneGuard(done); + NebdFileRecordPtr fileRecord = done->GetFileRecord(); + int ret = fileRecord->executor->GetInfo( + fileRecord->fileInstance.get(), fileInfo); + if (ret < 0) { + LOG(ERROR) << "Get file info failed. " + << "fd: " << fd + << ", fileName" << fileRecord->fileName; + return -1; + } + return 0; + }; + return ProcessRequest(fd, task); } -int NebdFileManager::Discard(int fd, NebdServerAioContext* aioctx) { - // TODO - return 0; +int NebdFileManager::InvalidCache(int fd) { + auto task = [&](NebdProcessClosure* done) { + brpc::ClosureGuard doneGuard(done); + NebdFileRecordPtr fileRecord = done->GetFileRecord(); + int ret = fileRecord->executor->InvalidCache( + fileRecord->fileInstance.get()); + if (ret < 0) { + LOG(ERROR) << "Invalid cache failed. " + << "fd: " << fd + << ", fileName" << fileRecord->fileName; + return -1; + } + return 0; + }; + return ProcessRequest(fd, task); } -int NebdFileManager::AioRead(int fd, NebdServerAioContext* aioctx) { - // TODO - return 0; +void NebdFileManager::CheckTimeoutFunc() { + while (sleeper_.wait_for( + std::chrono::milliseconds(checkTimeoutIntervalMs_))) { + std::unordered_map fileRecords + = fileRecordMap_.ListRecords(); + for (auto& fileRecord : fileRecords) { + uint64_t interval = + TimeUtility::GetTimeofDayMs() - fileRecord.second->timeStamp; + if (interval < (uint64_t)1000 * heartbeatTimeoutS_) { + continue; + } + std::string standardTime; + TimeUtility::TimeStampToStandard( + fileRecord.second->timeStamp / 1000, &standardTime); + LOG(INFO) << "Close file which has timed out. " + << "Last time received heartbeat: " << standardTime; + CloseInternal(fileRecord.second); + } + } } -int NebdFileManager::AioWrite(int fd, NebdServerAioContext* aioctx) { - // TODO - return 0; +int NebdFileManager::OpenInternal(const std::string& fileName, + bool create) { + // 同名文件open需要互斥 + NameLockGuard(nameLock_, fileName); + + NebdFileRecordPtr fileRecord = fileRecordMap_.GetRecord(fileName); + if (fileRecord != nullptr && fileRecord->status == NebdFileStatus::OPENED) { + return fileRecord->fd; + } + + if (create) { + fileRecord = std::make_shared(); + fileRecord->fileName = fileName; + int newFd = GetValidFd(); + fileRecord->fd = newFd; + fileRecord->type = GetFileType(fileName); + } else { + if (fileRecord == nullptr) { + LOG(ERROR) << "open file failed: no record. " + << "filename: " << fileName; + return -1; + } + } + + NebdRequestExecutor* executor = + NebdRequestExecutorFactory::GetExecutor(fileRecord->type); + if (executor == nullptr) { + LOG(ERROR) << "open file failed, invalid filename. " + << "filename: " << fileRecord->fileName; + return -1; + } + + NebdFileInstancePtr fileInstance = executor->Open(fileRecord->fileName); + if (fileInstance == nullptr) { + LOG(ERROR) << "open file failed. " + << "filename: " << fileRecord->fileName; + return -1; + } + + // 先持久化元数据信息到文件，再去更新内存 + int ret = metaFileManager_->UpdateFileRecord(fileRecord); + if (ret < 0) { + LOG(ERROR) << "persist file record failed. filename: " + << fileRecord->fileName; + return -1; + } + + fileRecord->executor = executor; + fileRecord->fileInstance = fileInstance; + fileRecord->status = NebdFileStatus::OPENED; + fileRecord->timeStamp = TimeUtility::GetTimeofDayMs(); + + bool updateSuccess = fileRecordMap_.UpdateRecord(fileRecord); + if (!updateSuccess) { + metaFileManager_->RemoveFileRecord(fileRecord->fileName); + LOG(ERROR) << "Update file record failed. " + << "filename: " << fileRecord->fileName; + return -1; + } + LOG(INFO) << "Open file success. " + << "fd: " << fileRecord->fd + << ", filename: " << fileRecord->fileName; + return fileRecord->fd; } -int NebdFileManager::Flush(int fd, NebdServerAioContext* aioctx) { - // TODO +int NebdFileManager::Reopen(NebdFileRecordPtr fileRecord) { + fileRecord->type = GetFileType(fileRecord->fileName); + NebdRequestExecutor* executor = + NebdRequestExecutorFactory::GetExecutor(fileRecord->type); + if (executor == nullptr) { + LOG(ERROR) << "open file failed, invalid filename. " + << "filename: " << fileRecord->fileName; + return -1; + } + NebdFileInstancePtr fileInstance = executor->Reopen( + fileRecord->fileName, fileRecord->fileInstance->addition); + if (fileInstance == nullptr) { + LOG(ERROR) << "Reopen file failed. " + << "filename: " << fileRecord->fileName; + return -1; + } + + fileRecord->executor = executor; + fileRecord->fileInstance = fileInstance; + fileRecord->status = NebdFileStatus::OPENED; + fileRecord->timeStamp = TimeUtility::GetTimeofDayMs(); + + LOG(INFO) << "Reopen file success. " + << "fd: " << fileRecord->fd + << ", filename: " << fileRecord->fileName; return 0; } -int NebdFileManager::GetInfo(int fd, NebdFileInfo* fileInfo) { - // TODO +int NebdFileManager::CloseInternal(NebdFileRecordPtr fileRecord) { + if (fileRecord->status != NebdFileStatus::OPENED) { + LOG(INFO) << "File has been closed. " + << "fd: " << fileRecord->fd + << ", filename: " << fileRecord->fileName; + return 0; + } + + // 用于和其他用户请求互斥，避免文件被close后，请求发到后端导致返回失败 + WriteLockGuard writeLock(fileRecord->rwLock); + int ret = fileRecord->executor->Close(fileRecord->fileInstance.get()); + if (ret < 0) { + LOG(ERROR) << "Close file failed. " + << "fd: " << fileRecord->fd + << ", filename: " << fileRecord->fileName; + return -1; + } + fileRecord->status = NebdFileStatus::CLOSED; + LOG(INFO) << "Close file success. " + << "fd: " << fileRecord->fd + << ", filename: " << fileRecord->fileName; return 0; } -int NebdFileManager::InvalidCache(int fd) { - // TODO +int NebdFileManager::ProcessRequest(int fd, ProcessTask task) { + NebdFileRecordPtr fileRecord = fileRecordMap_.GetRecord(fd); + if (fileRecord == nullptr) { + LOG(WARNING) << "File record not exist, fd: " << fd; + return -1; + } + + NebdProcessClosure* done = + new (std::nothrow) NebdProcessClosure(fileRecord); + brpc::ClosureGuard doneGuard(done); + + if (fileRecord->status != NebdFileStatus::OPENED) { + int ret = OpenInternal(fileRecord->fileName); + if (ret != fd) { + LOG(WARNING) << "Get opened file failed. " + << "filename: " << fileRecord->fileName + << ", fd: " << fd + << ", ret: " << ret; + return -1; + } + } + + int ret = task(dynamic_cast(doneGuard.release())); + if (ret < 0) { + LOG(ERROR) << "Process request failed. " + << "fd: " << fd + << ", fileName" << fileRecord->fileName; + return -1; + } return 0; } -void NebdFileManager::CheckTimeoutFunc() { - // TODO +std::unordered_map NebdFileManager::GetRecordMap() { + return fileRecordMap_.ListRecords(); +} + +int NebdFileManager::GetValidFd() { + int fd = 0; + while (true) { + fd = fdAlloc_.GetNext(); + if (!fileRecordMap_.Exist(fd)) { + break; + } + } + return fd; } } // namespace server diff --git a/src/part2/file_manager.h b/src/part2/file_manager.h index a281bffb43..7cb1762c99 100644 --- a/src/part2/file_manager.h +++ b/src/part2/file_manager.h @@ -8,52 +8,227 @@ #ifndef SRC_PART2_FILE_MANAGER_H_ #define SRC_PART2_FILE_MANAGER_H_ -#include +#include +#include #include #include // NOLINT #include +#include // NOLINT +#include "src/common/rw_lock.h" +#include "src/common/name_lock.h" +#include "src/common/timeutility.h" +#include "src/common/interrupt_sleep.h" #include "src/part2/define.h" +#include "src/part2/util.h" #include "src/part2/metafile_manager.h" +#include "src/part2/file_record_map.h" +#include "src/part2/request_executor_ceph.h" +#include "src/part2/request_executor_curve.h" namespace nebd { namespace server { +using nebd::common::NameLock; +using nebd::common::NameLockGuard; +using nebd::common::WriteLockGuard; +using nebd::common::ReadLockGuard; +using nebd::common::TimeUtility; +using nebd::common::InterruptibleSleeper; using MetaFileManagerPtr = std::shared_ptr; struct NebdFileManagerOption { + // 文件心跳超时时间（单位：秒） uint32_t heartbeatTimeoutS; + // 心跳超时检测线程的检测间隔（时长：毫秒） + uint32_t checkTimeoutIntervalMs; + // metafilemanager 对象指针 MetaFileManagerPtr metaFileManager; }; +// 处理用户请求时需要加读写锁，避免close时仍有用户IO未处理完成 +// 对于异步IO来说，只有返回时才能释放读锁，所以封装成Closure +// 在发送异步请求前，将closure赋值给NebdServerAioContext +class NebdProcessClosure : public Closure { + public: + explicit NebdProcessClosure(NebdFileRecordPtr record) + : record_(record) + , done_(nullptr) { + record_->rwLock.RDLock(); + } + NebdProcessClosure() {} + + void Run() { + std::unique_ptr selfGuard(this); + brpc::ClosureGuard doneGuard(done_); + record_->rwLock.Unlock(); + } + + void SetClosure(Closure* done) { + done_ = done; + } + + Closure* GetClosure() { + return done_; + } + + NebdFileRecordPtr GetFileRecord() { + return record_; + } + + private: + NebdFileRecordPtr record_; + Closure* done_; +}; + class NebdFileManager { public: NebdFileManager(); virtual ~NebdFileManager(); + /** + * 初始化FileManager各成员 + * @param option: 初始化参数 + * @return 成功返回0，失败返回-1 + */ virtual int Init(NebdFileManagerOption option); - virtual int Load(); + /** + * 停止FileManager并释放FileManager资源 + * @return 成功返回0，失败返回-1 + */ + virtual int Fini(); + /** + * 启动FileManager + * @return 成功返回0，失败返回-1 + */ + virtual int Run(); + /** + * part2收到心跳后，会通过该接口更新心跳中包含的文件在内存中记录的时间戳 + * 心跳检测线程会根据该时间戳判断是否需要关闭文件 + * @param fd: 指定文件的fd + * @return 成功返回0，失败返回-1 + */ virtual int UpdateFileTimestamp(int fd); + /** + * 打开文件 + * @param filename: 文件的filename + * @return 成功返回fd，失败返回-1 + */ virtual int Open(const std::string& filename); + /** + * 关闭文件 + * @param fd: 文件的fd + * @return 成功返回0，失败返回-1 + */ virtual int Close(int fd); + /** + * 给文件扩容 + * @param fd: 文件的fd + * @param newsize: 新的文件大小 + * @return 成功返回0，失败返回-1 + */ virtual int Extend(int fd, int64_t newsize); + /** + * 获取文件信息 + * @param fd: 文件的fd + * @param fileInfo[out]: 文件信息 + * @return 成功返回0，失败返回-1 + */ virtual int GetInfo(int fd, NebdFileInfo* fileInfo); - virtual int StatFile(int fd, NebdFileInfo* fileInfo); + /** + * 异步请求，回收指定区域空间 + * @param fd: 文件的fd + * @param aioctx: 异步请求上下文 + * @return 成功返回0，失败返回-1 + */ virtual int Discard(int fd, NebdServerAioContext* aioctx); + /** + * 异步请求，读取指定区域内容 + * @param fd: 文件的fd + * @param aioctx: 异步请求上下文 + * @return 成功返回0，失败返回-1 + */ virtual int AioRead(int fd, NebdServerAioContext* aioctx); + /** + * 异步请求，写数据到指定区域 + * @param fd: 文件的fd + * @param aioctx: 异步请求上下文 + * @return 成功返回0，失败返回-1 + */ virtual int AioWrite(int fd, NebdServerAioContext* aioctx); + /** + * 异步请求，flush文件缓存 + * @param fd: 文件的fd + * @param aioctx: 异步请求上下文 + * @return 成功返回0，失败返回-1 + */ virtual int Flush(int fd, NebdServerAioContext* aioctx); + /** + * 使指定文件缓存失效 + * @param fd: 文件的fd + * @return 成功返回0，失败返回-1 + */ virtual int InvalidCache(int fd); - private: + // set public for test + int Load(); void CheckTimeoutFunc(); + std::unordered_map GetRecordMap(); private: - using FileRecordMap = - std::unordered_map>; - FileRecordMap fileRecordMap_; - MetaFileManagerPtr metaFileManager_; + /** + * 打开指定文件并更新元数据信息 + * @param filename: 文件名 + * @param create: 为true将产生新的filerecord + * @return: 成功返回0，失败返回-1 + */ + int OpenInternal(const std::string& fileName, bool create = false); + /** + * 根据文件内存记录信息关闭指定文件 + * @param fileRecord: 文件的内存记录 + * @return: 成功返回0，失败返回-1 + */ + int CloseInternal(NebdFileRecordPtr fileRecord); + /** + * 根据文件内存记录信息重新open文件 + * @param fileRecord: 文件的内存记录 + * @return: 成功返回0，失败返回-1 + */ + int Reopen(NebdFileRecordPtr fileRecord); + /** + * 分配新的可用的fd + * @return: 成功返回有效的fd，失败返回-1 + */ + int GetValidFd(); + /** + * 请求统一处理函数 + * @param fd: 请求对应文件的fd + * @param task: 实际请求执行的函数体 + * @return: 成功返回0，失败返回-1 + */ + using ProcessTask = std::function; + int ProcessRequest(int fd, ProcessTask task); + + private: + // TODO(YYK) 后续封装HeartbeatManager + // 文件心跳超时时长 uint32_t heartbeatTimeoutS_; + // 心跳超时检测线程的检测时间间隔 + uint32_t checkTimeoutIntervalMs_; + // 心跳检测线程 std::thread checkTimeoutThread_; + // 心跳检测线程的sleeper + InterruptibleSleeper sleeper_; + // 当前filemanager的运行状态，true表示正在运行，false标为未运行 + std::atomic isRunning_; + // 文件名锁，对同名文件加锁 + NameLock nameLock_; + // fd分配器 + FdAllocator fdAlloc_; + // TODO(YYK) 与metafilemanager整合成FileRecordManager + // 文件信息内存记录映射表 + FileRecordMap fileRecordMap_; + // 元数据文件持久化管理 + MetaFileManagerPtr metaFileManager_; }; } // namespace server diff --git a/src/part2/file_record_map.cpp b/src/part2/file_record_map.cpp new file mode 100644 index 0000000000..83d6f9817a --- /dev/null +++ b/src/part2/file_record_map.cpp @@ -0,0 +1,96 @@ +/* + * Project: nebd + * Created Date: Thursday January 16th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#include "src/part2/file_record_map.h" + +namespace nebd { +namespace server { + +NebdFileRecordPtr FileRecordMap::GetRecord(int fd) { + ReadLockGuard readGuard(rwLock_); + if (map_.find(fd) == map_.end()) { + return nullptr; + } + return map_[fd]; +} + +NebdFileRecordPtr FileRecordMap::GetRecord(const std::string& fileName) { + ReadLockGuard readGuard(rwLock_); + int fd = FindRecordUnlocked(fileName); + if (fd > 0) { + return map_[fd]; + } + return nullptr; +} + +bool FileRecordMap::UpdateRecord(NebdFileRecordPtr fileRecord) { + WriteLockGuard writeGuard(rwLock_); + int fd = fileRecord->fd; + + // 如果fd已存在，判断文件名是否相同，不同则返回错误，相同则直接返回成功 + if (map_.find(fd) != map_.end()) { + if (map_[fd]->fileName == fileRecord->fileName) { + return true; + } else { + LOG(ERROR) << "Conflict record. " + << "Old fileName: " << map_[fd]->fileName + << ", new fileName: " << fileRecord->fileName + << ", fd: " << fd; + return false; + } + } + + // 如果该文件记录已存在，则先删除旧的记录 + int oldFd = FindRecordUnlocked(fileRecord->fileName); + if (oldFd > 0) { + LOG(WARNING) << "Remove old record, filename: " + << fileRecord->fileName + << ", old fd: " << oldFd + << ", newFd: " << fd; + map_.erase(oldFd); + } + + map_[fd] = fileRecord; + return true; +} + +void FileRecordMap::RemoveRecord(int fd) { + WriteLockGuard writeGuard(rwLock_); + if (map_.find(fd) != map_.end()) { + map_.erase(fd); + } +} + +bool FileRecordMap::Exist(int fd) { + ReadLockGuard readGuard(rwLock_); + return map_.find(fd) != map_.end(); +} + +void FileRecordMap::Clear() { + WriteLockGuard writeGuard(rwLock_); + map_.clear(); +} + +std::unordered_map FileRecordMap::ListRecords() { + ReadLockGuard readGuard(rwLock_); + return map_; +} + +int FileRecordMap::FindRecordUnlocked(const std::string& fileName) { + int fd = 0; + for (const auto& pair : map_) { + if (pair.second->fileName == fileName) { + fd = pair.first; + break; + } + } + return fd; +} + +} // namespace server +} // namespace nebd + diff --git a/src/part2/file_record_map.h b/src/part2/file_record_map.h new file mode 100644 index 0000000000..bde2a5ad5c --- /dev/null +++ b/src/part2/file_record_map.h @@ -0,0 +1,88 @@ +/* + * Project: nebd + * Created Date: Thursday January 16th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#ifndef SRC_PART2_FILE_RECORD_MAP_H_ +#define SRC_PART2_FILE_RECORD_MAP_H_ + +#include +#include +#include +#include +#include + +#include "src/common/rw_lock.h" +#include "src/part2/define.h" + +namespace nebd { +namespace server { + +using nebd::common::RWLock; +using nebd::common::WriteLockGuard; +using nebd::common::ReadLockGuard; + +// nebd server 文件信息的内存记录 +class FileRecordMap { + public: + FileRecordMap() {} + virtual ~FileRecordMap() {} + /** + * 根据fd获取指定文件的内存记录 + * @param fd: 文件的fd + * @return: 文件的内存记录 + */ + NebdFileRecordPtr GetRecord(int fd); + /** + * 根据文件名获取指定文件的内存记录 + * @param fileName: 文件名称 + * @return: 文件的内存记录 + */ + NebdFileRecordPtr GetRecord(const std::string& fileName); + /** + * 更新文件内存记录 + * 如果记录不存在，插入一条新的记录 + * 如果相同文件名的记录已存在，但是fd不同，则删除旧的记录，再插入当前记录 + * 如果相同fd的记录已存在，但是文件名不同，则返回失败 + * @param fileRecord: 文件记录 + * @return: 成功返回true，失败返回false + */ + bool UpdateRecord(NebdFileRecordPtr fileRecord); + /** + * 根据指定fd对应的文件记录 + * @param fd: 文件的fd + */ + void RemoveRecord(int fd); + /** + * 判断指定fd对应的文件记录是否存在 + * @param fd: 文件的fd + * @return: 如果存在返回true，不存在返回false + */ + bool Exist(int fd); + /** + * 清楚所有的记录 + */ + void Clear(); + /** + * 获取所有文件记录的映射表 + * @return: 返回所有文件记录的映射表 + */ + std::unordered_map ListRecords(); + + private: + // 获取指定文件名的文件记录（非线程安全） + int FindRecordUnlocked(const std::string& fileName); + + private: + // 保护文件记录映射表的读写锁 + RWLock rwLock_; + // 文件记录映射表 + std::unordered_map map_; +}; + +} // namespace server +} // namespace nebd + +#endif // SRC_PART2_FILE_RECORD_MAP_H_ diff --git a/src/part2/metafile_manager.cpp b/src/part2/metafile_manager.cpp index a7b483a5cf..b18aa283a1 100644 --- a/src/part2/metafile_manager.cpp +++ b/src/part2/metafile_manager.cpp @@ -2,7 +2,7 @@ namespace nebd { namespace server { - + NebdMetaFileManager::NebdMetaFileManager(const std::string& metaFilePath) : metaFilePath_(metaFilePath) { // TODO @@ -11,16 +11,16 @@ NebdMetaFileManager::NebdMetaFileManager(const std::string& metaFilePath) NebdMetaFileManager::~NebdMetaFileManager() {} int NebdMetaFileManager::RemoveFileRecord(const std::string& fileName) { - // TODO + // TODO return 0; } -int NebdMetaFileManager::UpdateFileRecord(const NebdFileRecord& fileRecord) { +int NebdMetaFileManager::UpdateFileRecord(const NebdFileRecordPtr& fileRecord) { // TODO return 0; } -int NebdMetaFileManager::ListFileRecord(std::vector* fileRecords) { +int NebdMetaFileManager::ListFileRecord(std::vector* fileRecords) { // TODO return 0; } diff --git a/src/part2/metafile_manager.h b/src/part2/metafile_manager.h index 5db3b7a114..3dbbcbd93f 100644 --- a/src/part2/metafile_manager.h +++ b/src/part2/metafile_manager.h @@ -15,8 +15,8 @@ class NebdMetaFileManager { virtual ~NebdMetaFileManager(); virtual int RemoveFileRecord(const std::string& fileName); - virtual int UpdateFileRecord(const NebdFileRecord& fileRecord); - virtual int ListFileRecord(std::vector* fileRecords); + virtual int UpdateFileRecord(const NebdFileRecordPtr& fileRecord); + virtual int ListFileRecord(std::vector* fileRecords); private: std::string metaFilePath_; diff --git a/src/part2/request_executor.cpp b/src/part2/request_executor.cpp new file mode 100644 index 0000000000..f9c8d13561 --- /dev/null +++ b/src/part2/request_executor.cpp @@ -0,0 +1,38 @@ +/* + * Project: nebd + * Created Date: Thursday January 16th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#include "src/part2/request_executor.h" +#include "src/part2/request_executor_ceph.h" +#include "src/part2/request_executor_curve.h" + +namespace nebd { +namespace server { + +NebdRequestExecutor* g_test_executor = nullptr; + +NebdRequestExecutor* +NebdRequestExecutorFactory::GetExecutor(NebdFileType type) { + NebdRequestExecutor* executor = nullptr; + switch (type) { + case NebdFileType::CEPH: + executor = &CephRequestExecutor::GetInstance(); + break; + case NebdFileType::CURVE: + executor = &CurveRequestExecutor::GetInstance(); + break; + case NebdFileType::TEST: + executor = g_test_executor; + break; + default: + break; + } + return executor; +} + +} // namespace server +} // namespace nebd + diff --git a/src/part2/request_executor.h b/src/part2/request_executor.h index 084f0e130e..e59fe54560 100644 --- a/src/part2/request_executor.h +++ b/src/part2/request_executor.h @@ -1,22 +1,43 @@ +/* + * Project: nebd + * Created Date: Thursday January 16th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + #ifndef SRC_PART2_REQUEST_EXECUTOR_H_ #define SRC_PART2_REQUEST_EXECUTOR_H_ +#include +#include #include "src/part2/define.h" namespace nebd { namespace server { +class CurveRequestExecutor; +class CephRequestExecutor; + +using AdditionType = std::map; +// 具体RequestExecutor中会用到的文件实例上下文信息 +// 该类为抽象结构，ceph或curve需要继承定义自己的FileInstance +// RequestExecutor需要用到的文件上下文信息都记录到FileInstance内 class NebdFileInstance { public: NebdFileInstance() {} virtual ~NebdFileInstance() {} + // 需要持久化到文件的内容，以kv形式返回，例如curve open时返回的sessionid + // 文件reopen的时候也会用到该内容 + AdditionType addition; }; -class NebdRequestExcutor { +class NebdRequestExecutor { public: - NebdRequestExcutor() {} - virtual ~NebdRequestExcutor() {} + NebdRequestExecutor() {} + virtual ~NebdRequestExecutor() {} virtual std::shared_ptr Open(const std::string& filename) = 0; // NOLINT + virtual std::shared_ptr Reopen( + const std::string& filename, AdditionType addtion) = 0; virtual int Close(NebdFileInstance* fd) = 0; virtual int Extend(NebdFileInstance* fd, int64_t newsize) = 0; virtual int GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) = 0; @@ -28,6 +49,17 @@ class NebdRequestExcutor { virtual int InvalidCache(NebdFileInstance* fd) = 0; }; +class NebdRequestExecutorFactory { + public: + NebdRequestExecutorFactory() = default; + ~NebdRequestExecutorFactory() = default; + + static NebdRequestExecutor* GetExecutor(NebdFileType type); +}; + +// for test +extern NebdRequestExecutor* g_test_executor; + } // namespace server } // namespace nebd diff --git a/src/part2/request_executor_ceph.cpp b/src/part2/request_executor_ceph.cpp index 16e319652f..99093dd706 100644 --- a/src/part2/request_executor_ceph.cpp +++ b/src/part2/request_executor_ceph.cpp @@ -4,52 +4,59 @@ namespace nebd { namespace server { std::shared_ptr -CephRequestExcutor::Open(const std::string& filename) { +CephRequestExecutor::Open(const std::string& filename) { // TODO return nullptr; } -int CephRequestExcutor::Close(NebdFileInstance* fd) { +std::shared_ptr +CephRequestExecutor::Reopen(const std::string& filename, + AdditionType addtion) { + // TODO + return nullptr; +} + +int CephRequestExecutor::Close(NebdFileInstance* fd) { // TODO return 0; } -int CephRequestExcutor::Extend(NebdFileInstance* fd, int64_t newsize) { +int CephRequestExecutor::Extend(NebdFileInstance* fd, int64_t newsize) { // TODO return 0; } -int CephRequestExcutor::StatFile(NebdFileInstance* fd, NebdFileInfo* fileInfo) { +int CephRequestExecutor::StatFile(NebdFileInstance* fd, NebdFileInfo* fileInfo) { // TODO return 0; } -int CephRequestExcutor::Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) { +int CephRequestExecutor::Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) { // TODO return 0; } -int CephRequestExcutor::AioRead(NebdFileInstance* fd, NebdServerAioContext* aioctx) { +int CephRequestExecutor::AioRead(NebdFileInstance* fd, NebdServerAioContext* aioctx) { // TODO return 0; } -int CephRequestExcutor::AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) { +int CephRequestExecutor::AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) { // TODO return 0; } -int CephRequestExcutor::Flush(NebdFileInstance* fd, NebdServerAioContext* aioctx) { +int CephRequestExecutor::Flush(NebdFileInstance* fd, NebdServerAioContext* aioctx) { // TODO return 0; } -int CephRequestExcutor::GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) { +int CephRequestExecutor::GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) { // TODO return 0; } -int CephRequestExcutor::InvalidCache(NebdFileInstance* fd) { +int CephRequestExecutor::InvalidCache(NebdFileInstance* fd) { // TODO return 0; } diff --git a/src/part2/request_executor_ceph.h b/src/part2/request_executor_ceph.h index 65e9776f59..564a7f773c 100644 --- a/src/part2/request_executor_ceph.h +++ b/src/part2/request_executor_ceph.h @@ -12,16 +12,18 @@ class CephFileInstance : public NebdFileInstance { public: CephFileInstance() {} ~CephFileInstance() {} - - int fd; - std::string sessionid; }; -class CephRequestExcutor : public NebdRequestExcutor { +class CephRequestExecutor : public NebdRequestExecutor { public: - CephRequestExcutor() {} - ~CephRequestExcutor() {} + static CephRequestExecutor& GetInstance() { + static CephRequestExecutor executor; + return executor; + } + ~CephRequestExecutor() {} std::shared_ptr Open(const std::string& filename) override; // NOLINT + std::shared_ptr Reopen( + const std::string& filename, AdditionType addtion) override; int Close(NebdFileInstance* fd) override; int Extend(NebdFileInstance* fd, int64_t newsize) override; int GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) override; @@ -31,6 +33,9 @@ class CephRequestExcutor : public NebdRequestExcutor { int AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; int Flush(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; int InvalidCache(NebdFileInstance* fd) override; + + private: + CephRequestExecutor() {} }; } // namespace server diff --git a/src/part2/request_executor_curve.cpp b/src/part2/request_executor_curve.cpp index 3d5351bc60..7eb205ea07 100644 --- a/src/part2/request_executor_curve.cpp +++ b/src/part2/request_executor_curve.cpp @@ -4,52 +4,59 @@ namespace nebd { namespace server { std::shared_ptr -CurveRequestExcutor::Open(const std::string& filename) { +CurveRequestExecutor::Open(const std::string& filename) { // TODO return nullptr; } -int CurveRequestExcutor::Close(NebdFileInstance* fd) { +std::shared_ptr +CurveRequestExecutor::Reopen(const std::string& filename, + AdditionType addtion) { + // TODO + return nullptr; +} + +int CurveRequestExecutor::Close(NebdFileInstance* fd) { // TODO return 0; } -int CurveRequestExcutor::Extend(NebdFileInstance* fd, int64_t newsize) { +int CurveRequestExecutor::Extend(NebdFileInstance* fd, int64_t newsize) { // TODO return 0; } -int CurveRequestExcutor::StatFile(NebdFileInstance* fd, NebdFileInfo* fileInfo) { +int CurveRequestExecutor::StatFile(NebdFileInstance* fd, NebdFileInfo* fileInfo) { // TODO return 0; } -int CurveRequestExcutor::Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) { +int CurveRequestExecutor::Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) { // TODO return 0; } -int CurveRequestExcutor::AioRead(NebdFileInstance* fd, NebdServerAioContext* aioctx) { +int CurveRequestExecutor::AioRead(NebdFileInstance* fd, NebdServerAioContext* aioctx) { // TODO return 0; } -int CurveRequestExcutor::AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) { +int CurveRequestExecutor::AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) { // TODO return 0; } -int CurveRequestExcutor::Flush(NebdFileInstance* fd, NebdServerAioContext* aioctx) { +int CurveRequestExecutor::Flush(NebdFileInstance* fd, NebdServerAioContext* aioctx) { // TODO return 0; } -int CurveRequestExcutor::GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) { +int CurveRequestExecutor::GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) { // TODO return 0; } -int CurveRequestExcutor::InvalidCache(NebdFileInstance* fd) { +int CurveRequestExecutor::InvalidCache(NebdFileInstance* fd) { // TODO return 0; } diff --git a/src/part2/request_executor_curve.h b/src/part2/request_executor_curve.h index 3f573b95b2..be0275b58f 100644 --- a/src/part2/request_executor_curve.h +++ b/src/part2/request_executor_curve.h @@ -13,15 +13,19 @@ class CurveFileInstance : public NebdFileInstance { CurveFileInstance() {} ~CurveFileInstance() {} - int fd; - std::string sessionid; + int fd = 0; }; -class CurveRequestExcutor : public NebdRequestExcutor { +class CurveRequestExecutor : public NebdRequestExecutor { public: - CurveRequestExcutor() {} - ~CurveRequestExcutor() {} + static CurveRequestExecutor& GetInstance() { + static CurveRequestExecutor executor; + return executor; + } + ~CurveRequestExecutor() {} std::shared_ptr Open(const std::string& filename) override; // NOLINT + std::shared_ptr Reopen( + const std::string& filename, AdditionType addtion) override; int Close(NebdFileInstance* fd) override; int Extend(NebdFileInstance* fd, int64_t newsize) override; int GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) override; @@ -31,6 +35,9 @@ class CurveRequestExcutor : public NebdRequestExcutor { int AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; int Flush(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; int InvalidCache(NebdFileInstance* fd) override; + + private: + CurveRequestExecutor() {} }; } // namespace server diff --git a/src/part2/util.cpp b/src/part2/util.cpp new file mode 100644 index 0000000000..6ef6847291 --- /dev/null +++ b/src/part2/util.cpp @@ -0,0 +1,68 @@ +/* + * Project: nebd + * Created Date: Sunday January 19th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#include "src/part2/util.h" + +namespace nebd { +namespace server { + +NebdFileType GetFileType(const std::string& fileName) { + size_t pos = fileName.find_first_of(':'); + if (pos == std::string::npos) { + return NebdFileType::UNKWOWN; + } + std::string type = fileName.substr(0, pos); + if (type == CURVE_PREFIX) { + return NebdFileType::CURVE; + } else if (type == CEPH_PREFIX) { + return NebdFileType::CEPH; + } else if (type == TEST_PREFIX) { + return NebdFileType::TEST; + } else { + return NebdFileType::UNKWOWN; + } +} + +std::string Op2Str(LIBAIO_OP op) { + switch (op) { + case LIBAIO_OP::LIBAIO_OP_READ: + return "READ"; + case LIBAIO_OP::LIBAIO_OP_WRITE: + return "WRITE"; + case LIBAIO_OP::LIBAIO_OP_DISCARD: + return "DISCARD"; + case LIBAIO_OP::LIBAIO_OP_FLUSH: + return "FLUSH"; + default: + return "UNKWOWN"; + } +} + +std::ostream& operator<<(std::ostream& os, const NebdServerAioContext& c) { + os << "[type: " << Op2Str(c.op) + << ", offset: " << c.offset + << ", length: " << c.length + << ", ret: " << c.ret + << "]"; + return os; +} + +int FdAllocator::GetNext() { + std::unique_lock lock(mtx_); + if (fd_ == INT_MAX || fd_ < 0) { + fd_ = 0; + } + return ++fd_; +} + +void FdAllocator::InitFd(int fd) { + fd_ = fd; +} + +} // namespace server +} // namespace nebd + diff --git a/src/part2/util.h b/src/part2/util.h new file mode 100644 index 0000000000..2ed30c5140 --- /dev/null +++ b/src/part2/util.h @@ -0,0 +1,44 @@ +/* + * Project: nebd + * Created Date: Sunday January 19th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#ifndef SRC_PART2_UTIL_H_ +#define SRC_PART2_UTIL_H_ + +#include +#include // NOLINT +#include + +#include "src/part2/define.h" + +namespace nebd { +namespace server { + +NebdFileType GetFileType(const std::string& fileName); + +std::string NebdFileType2Str(NebdFileType type); + +std::ostream& operator<<(std::ostream& os, const NebdServerAioContext& c); + +class FdAllocator { + public: + FdAllocator() : fd_(0) {} + ~FdAllocator() {} + + // fd的有效值范围为[1, INT_MAX] + int GetNext(); + // 初始化fd的值 + void InitFd(int fd); + + private: + std::mutex mtx_; + int fd_; +}; + +} // namespace server +} // namespace nebd + +#endif // SRC_PART2_UTIL_H_ diff --git a/tests/common/CMakeLists.txt b/tests/common/CMakeLists.txt index 3891efc126..1e40dcfec9 100644 --- a/tests/common/CMakeLists.txt +++ b/tests/common/CMakeLists.txt @@ -7,10 +7,16 @@ set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -std=c++0x -pipe -W -Wall \ add_definitions( "-DGFLAGS=gflags -DOS_LINUX -DSNAPPY -DHAVE_ZLIB -DHAVE_SSE42 -DNDEBUG") -# test_configuration -set(TEST_CONF_SRC configuration_test.cpp) -set(TEST_CONF_LINK nebd_common gflags gtest gmock) -add_executable(test_configuration ${TEST_CONF_SRC}) -target_link_libraries(test_configuration ${TEST_CONF_LINK}) -install(TARGETS test_configuration DESTINATION bin) -add_test(NAME test_configuration COMMAND test_configuration) +# COMMON_TEST_SRC +file(GLOB COMMON_TEST_SRC + "*.h" + "*.cpp" +) + +# common_test +set(COMMON_TEST_LINK nebd_common gflags gtest gmock) +add_executable(common_test ${COMMON_TEST_SRC}) +target_link_libraries(common_test ${COMMON_TEST_LINK}) +install(TARGETS common_test DESTINATION bin) +add_test(NAME common_test COMMAND common_test) + diff --git a/tests/common/interruptible_sleeper_test.cpp b/tests/common/interruptible_sleeper_test.cpp new file mode 100644 index 0000000000..e0592b081e --- /dev/null +++ b/tests/common/interruptible_sleeper_test.cpp @@ -0,0 +1,44 @@ +/* + * Project: nebd + * Created Date: 2019-11-25 + * Author: lixiaocui + * Copyright (c) 2019 netease + */ + +#include +#include "src/common/interrupt_sleep.h" +#include "src/common/timeutility.h" + +namespace nebd { +namespace common { +InterruptibleSleeper sleeper; +bool is_stop = false; + +void handler(int sig) { + sleeper.interrupt(); +} + +TEST(InterruptibleSleeperTest, test_interruptible_sleeper) { + pid_t pid = ::fork(); + if (0 > pid) { + ASSERT_TRUE(false); + } else if (0 == pid) { + struct sigaction action; + action.sa_handler = handler; + sigemptyset(&action.sa_mask); + action.sa_flags = 0; + sigaction(SIGTERM, &action, NULL); + + while (sleeper.wait_for(std::chrono::seconds(10))) {} + return; + } + + usleep(50 * 1000); + uint64_t startKill = TimeUtility::GetTimeofDayMs(); + int waitstatus; + kill(pid, SIGTERM); + waitpid(pid, &waitstatus, 0); + ASSERT_GT(8000, TimeUtility::GetTimeofDayMs() - startKill); +} +} // namespace common +} // namespace nebd diff --git a/tests/common/rw_lock_test.cpp b/tests/common/rw_lock_test.cpp new file mode 100644 index 0000000000..b72e1ac2e5 --- /dev/null +++ b/tests/common/rw_lock_test.cpp @@ -0,0 +1,88 @@ +/* + * Project: nebd + * Created Date: 18-10-12 + * Author: wudemiao + * Copyright (c) 2018 netease + */ + +#include +#include + +#include // NOLINT + +#include "src/common/rw_lock.h" + +namespace nebd { +namespace common { + +TEST(RWLockTest, basic_test) { + RWLock rwlock; + { + ReadLockGuard readLockGuard(rwlock); + ASSERT_TRUE(true); + } + { + WriteLockGuard writeLockGuard(rwlock); + ASSERT_TRUE(true); + } + { + WriteLockGuard writeLockGuard(rwlock); + ASSERT_TRUE(true); + } + { + ReadLockGuard readLockGuard1(rwlock); + ReadLockGuard readLockGuard2(rwlock); + ASSERT_TRUE(true); + } + { + ReadLockGuard readLockGuard1(rwlock); + ReadLockGuard readLockGuard2(rwlock); + ReadLockGuard readLockGuard3(rwlock); + ReadLockGuard readLockGuard4(rwlock); + ASSERT_TRUE(true); + } + { + ReadLockGuard readLockGuard(rwlock); + ASSERT_EQ(0, rwlock.TryRDLock()); + ASSERT_EQ(EBUSY, rwlock.TryWRLock()); + /* be careful */ + rwlock.Unlock(); + } + { + WriteLockGuard writeLockGuard(rwlock); + ASSERT_EQ(EBUSY, rwlock.TryRDLock()); + ASSERT_EQ(EBUSY, rwlock.TryWRLock()); + } + uint64_t writeCnt = 0; + auto writeFunc = [&] { + for (uint64_t i = 0; i < 10000; ++i) { + WriteLockGuard writeLockGuard(rwlock); + ++writeCnt; + } + }; + auto readFunc = [&] { + for (uint64_t i = 0; i < 10000; ++i) { + ReadLockGuard readLockGuard(rwlock); + auto j = writeCnt + i; + } + }; + { + std::thread t1(writeFunc); + std::thread t2(readFunc); + std::thread t3(writeFunc); + std::thread t4(readFunc); + std::thread t5(writeFunc); + std::thread t6(writeFunc); + t1.join(); + t2.join(); + t3.join(); + t4.join(); + t5.join(); + t6.join(); + + ASSERT_EQ(4 * 10000, writeCnt); + } +} + +} // namespace common +} // namespace nebd diff --git a/tests/common/test_name_lock.cpp b/tests/common/test_name_lock.cpp new file mode 100644 index 0000000000..b068366dee --- /dev/null +++ b/tests/common/test_name_lock.cpp @@ -0,0 +1,91 @@ +/* + * Project: nebd + * Created Date: Fri Aug 09 2019 + * Author: xuchaojie + * Copyright (c) 2019 netease + */ + +#include +#include +#include + +#include "src/common/name_lock.h" + +namespace nebd { +namespace common { + +TEST(TestNameLock, TestNameLockBasic) { + NameLock lock1, lock2, lock3; + + // lock测试 + lock1.Lock("str1"); + // 同锁不同str可lock不死锁 + lock1.Lock("str2"); + // 不同锁同str可lock不死锁 + lock2.Lock("str1"); + + + + // 同锁同str TryLock失败 + ASSERT_FALSE(lock1.TryLock("str1")); + // 同锁不同str TryLock成功 + ASSERT_TRUE(lock1.TryLock("str3")); + // 不同锁同str TryLock成功 + ASSERT_TRUE(lock3.TryLock("str1")); + + // unlock测试 + lock1.Unlock("str1"); + lock1.Unlock("str2"); + lock1.Unlock("str3"); + lock2.Unlock("str1"); + lock3.Unlock("str1"); + // 未锁unlock ok + lock2.Unlock("str2"); +} + +TEST(TestNameLock, TestNameLockGuardBasic) { + NameLock lock1, lock2; + { + NameLockGuard guard1(lock1, "str1"); + NameLockGuard guard2(lock1, "str2"); + NameLockGuard guard3(lock2, "str1"); + // 作用域内加锁成功，不可再加锁 + ASSERT_FALSE(lock1.TryLock("str1")); + ASSERT_FALSE(lock1.TryLock("str2")); + ASSERT_FALSE(lock2.TryLock("str1")); + } + // 作用域外自动解锁，可再加锁 + ASSERT_TRUE(lock1.TryLock("str1")); + ASSERT_TRUE(lock1.TryLock("str2")); + ASSERT_TRUE(lock2.TryLock("str1")); + lock1.Unlock("str1"); + lock1.Unlock("str2"); + lock2.Unlock("str1"); +} + +TEST(TestNameLock, TestNameLockConcurrent) { + NameLock lock1; + auto worker = [&] (const std::string &str) { + for (int i = 0; i < 10000; i++) { + NameLockGuard guard(lock1, str); + } + }; + + std::vector threadpool; + for (auto &t : threadpool) { + std::string str1 = "aaaa"; + std::string str2 = "bbbb"; + std::srand(std::time(nullptr)); + std::string rstr = (std::rand() / 2) ? str1 : str2; + t = std::thread(worker, rstr); + } + + for (auto &t : threadpool) { + t.join(); + } +} + + + +} // namespace common +} // namespace nebd diff --git a/tests/part2/CMakeLists.txt b/tests/part2/CMakeLists.txt index 1f210e75b4..73a876cb85 100644 --- a/tests/part2/CMakeLists.txt +++ b/tests/part2/CMakeLists.txt @@ -6,3 +6,34 @@ set(CPP_FLAGS "-DBRPC_WITH_GLOG=0 -DGFLAGS_NS=google") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CPP_FLAGS} -std=c++0x -DNDEBUG -O2 \ -D__const__= -pipe -W -Wall -Wno-unused-parameter -fPIC \ -fno-omit-frame-pointer") + +set(PART2_TEST_LINK + nebdserver + gtest + gmock) + +# libpart2mock.so +set(PART2_MOCK_SRC + mock_metafile_manager.h + mock_request_executor.h) +add_library(part2mock ${PART2_MOCK_SRC}) +set_target_properties(part2mock PROPERTIES LINKER_LANGUAGE CXX) +target_link_libraries(part2mock + ${PART2_TEST_LINK}) + +# filerecordmap_test +set(RECORD_MAP_TEST_SRC file_record_map_unittest.cpp) +add_executable(filerecordmap_test ${RECORD_MAP_TEST_SRC}) +target_link_libraries(filerecordmap_test + ${PART2_TEST_LINK}) +install(TARGETS filerecordmap_test DESTINATION bin) +add_test(NAME filerecordmap_test COMMAND filerecordmap_test) + +# filemanager_test +set(FILE_MANAGER_TEST_SRC file_manager_unittest.cpp) +add_executable(filemanager_test ${FILE_MANAGER_TEST_SRC}) +target_link_libraries(filemanager_test + part2mock + ${PART2_TEST_LINK}) +install(TARGETS filemanager_test DESTINATION bin) +add_test(NAME filemanager_test COMMAND filemanager_test) \ No newline at end of file diff --git a/tests/part2/file_manager_unittest.cpp b/tests/part2/file_manager_unittest.cpp new file mode 100644 index 0000000000..6231b4ebfe --- /dev/null +++ b/tests/part2/file_manager_unittest.cpp @@ -0,0 +1,677 @@ +/* + * Project: nebd + * Created Date: Monday January 20th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#include +#include +#include +#include + +#include "src/part2/file_manager.h" +#include "tests/part2/mock_metafile_manager.h" +#include "tests/part2/mock_request_executor.h" + +namespace nebd { +namespace server { + +const char testFile1[] = "test:/cinder/111"; +const char testFile2[] = "test:/cinder/222"; +const char unknownFile[] = "un:/cinder/666"; + +using ::testing::_; +using ::testing::Return; +using ::testing::NotNull; +using ::testing::DoAll; +using ::testing::ReturnArg; +using ::testing::ElementsAre; +using ::testing::SetArgPointee; +using ::testing::SetArrayArgument; + +class FileManagerTest : public ::testing::Test { + public: + void SetUp() { + aioContext_ = new NebdServerAioContext(); + mockInstance_ = std::make_shared(); + metaFileManager_ = std::make_shared(); + executor_ = std::make_shared(); + g_test_executor = executor_.get(); + option_.heartbeatTimeoutS = 5; + option_.checkTimeoutIntervalMs = 1000; + option_.metaFileManager = metaFileManager_; + ASSERT_EQ(fileManager_.Init(option_), 0); + } + void TearDown() { + delete aioContext_; + } + + // 构造初始环境，open两个文件 + void InitEnv() { + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(0)); + int fd = fileManager_.Open(testFile1); + ASSERT_EQ(fd, 1); + + EXPECT_CALL(*executor_, Open(testFile2)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(0)); + fd = fileManager_.Open(testFile2); + ASSERT_EQ(fd, 2); + } + + protected: + NebdFileManager fileManager_; + std::shared_ptr metaFileManager_; + std::shared_ptr executor_; + std::shared_ptr mockInstance_; + NebdServerAioContext* aioContext_; + NebdFileManagerOption option_; +}; + +TEST_F(FileManagerTest, OpenTest) { + // open一个不存在的文件 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(0)); + int fd = fileManager_.Open(testFile1); + ASSERT_EQ(fd, 1); + + // 重复open + EXPECT_CALL(*executor_, Open(_)) + .Times(0); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .Times(0); + fd = fileManager_.Open(testFile1); + ASSERT_EQ(fd, 1); + + // 再次open一个新的文件 + EXPECT_CALL(*executor_, Open(testFile2)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(0)); + fd = fileManager_.Open(testFile2); + ASSERT_EQ(fd, 2); + + // open 已经close的文件 + std::unordered_map recordMap + = fileManager_.GetRecordMap(); + ASSERT_NE(recordMap[1], nullptr); + recordMap[1]->status = NebdFileStatus::CLOSED; + // 重新open该文件 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(0)); + fd = fileManager_.Open(testFile1); + // 校验结果，生成了新的记录并删除了旧的记录 + ASSERT_EQ(fd, 3); + recordMap = fileManager_.GetRecordMap(); + ASSERT_NE(recordMap[3], nullptr); + ASSERT_EQ(recordMap[1], nullptr); + ASSERT_EQ(3, recordMap[3]->fd); + ASSERT_EQ(testFile1, recordMap[3]->fileName); +} + +TEST_F(FileManagerTest, OpenFailTest) { + // 调用后端open接口时出错 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(nullptr)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .Times(0); + int fd = fileManager_.Open(testFile1); + ASSERT_EQ(fd, -1); + + // 持久化元数据信息失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(-1)); + fd = fileManager_.Open(testFile1); + ASSERT_EQ(fd, -1); + + // Open一个非法的filename + EXPECT_CALL(*executor_, Open(_)) + .Times(0); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .Times(0); + fd = fileManager_.Open(unknownFile); + ASSERT_EQ(fd, -1); +} + +TEST_F(FileManagerTest, LoadTest) { + // 初始化从metafile返回的元数据 + std::vector fileRecords; + NebdFileRecordPtr record1 = std::make_shared(); + NebdFileRecordPtr record2 = std::make_shared(); + NebdFileRecordPtr record3 = std::make_shared(); + record1->fileName = testFile1; + record1->fd = 1; + record1->fileInstance = mockInstance_; + record2->fileName = testFile2; + record2->fd = 2; + record2->fileInstance = mockInstance_; + record3->fileName = unknownFile; + record3->fd = 3; + record3->fileInstance = mockInstance_; + fileRecords.push_back(record1); + fileRecords.push_back(record2); + fileRecords.push_back(record3); + + EXPECT_CALL(*metaFileManager_, ListFileRecord(_)) + .WillOnce(DoAll(SetArgPointee<0>(fileRecords), + Return(0))); + EXPECT_CALL(*executor_, Reopen(testFile1, _)) + .WillOnce(Return(mockInstance_)); + // reopen失败，不会导致load失败 + EXPECT_CALL(*executor_, Reopen(testFile2, _)) + .WillOnce(Return(nullptr)); + // 无法识别的文件不会被打开，但不会导致load失败 + EXPECT_CALL(*executor_, Reopen(unknownFile, _)) + .Times(0); + ASSERT_EQ(0, fileManager_.Load()); +} + +TEST_F(FileManagerTest, LoadFailTest) { + // 初始化 + std::vector fileRecords; + NebdFileRecordPtr record1 = std::make_shared(); + NebdFileRecordPtr record2 = std::make_shared(); + record1->fileName = testFile1; + record1->fd = 1; + record1->fileInstance = mockInstance_; + // fd相同，文件名不同会出现冲突（这种情况理论也不会发生） + record2->fileName = testFile2; + record2->fd = 1; + record2->fileInstance = mockInstance_; + fileRecords.push_back(record1); + fileRecords.push_back(record2); + + EXPECT_CALL(*metaFileManager_, ListFileRecord(_)) + .WillOnce(DoAll(SetArgPointee<0>(fileRecords), + Return(0))); + EXPECT_CALL(*executor_, Reopen(testFile1, _)) + .WillOnce(Return(mockInstance_)); + // reopen失败，不会导致load失败 + EXPECT_CALL(*executor_, Reopen(testFile2, _)) + .WillOnce(Return(mockInstance_)); + ASSERT_EQ(-1, fileManager_.Load()); +} + +TEST_F(FileManagerTest, CloseTest) { + InitEnv(); + // 指定的fd不存在,直接返回成功 + ASSERT_EQ(0, fileManager_.Close(3)); + + // fd存在 + EXPECT_CALL(*executor_, Close(NotNull())) + .WillOnce(Return(0)); + EXPECT_CALL(*metaFileManager_, RemoveFileRecord(testFile1)) + .WillOnce(Return(0)); + ASSERT_EQ(0, fileManager_.Close(1)); + + EXPECT_CALL(*executor_, Close(NotNull())) + .WillOnce(Return(0)); + EXPECT_CALL(*metaFileManager_, RemoveFileRecord(testFile2)) + .WillOnce(Return(0)); + ASSERT_EQ(0, fileManager_.Close(2)); +} + +TEST_F(FileManagerTest, CloseFailTest) { + InitEnv(); + // 指定的fd不存在,直接返回成功 + ASSERT_EQ(0, fileManager_.Close(3)); + + // 调用后端存储close时失败 + EXPECT_CALL(*executor_, Close(NotNull())) + .WillOnce(Return(-1)); + EXPECT_CALL(*metaFileManager_, RemoveFileRecord(testFile1)) + .Times(0); + ASSERT_EQ(-1, fileManager_.Close(1)); + + // 从元数据文件中移除记录时失败 + EXPECT_CALL(*executor_, Close(NotNull())) + .WillOnce(Return(0)); + EXPECT_CALL(*metaFileManager_, RemoveFileRecord(testFile1)) + .WillOnce(Return(-1)); + ASSERT_EQ(-1, fileManager_.Close(1)); +} + +TEST_F(FileManagerTest, ExtendTest) { + InitEnv(); + + // 文件是opened状态 + EXPECT_CALL(*executor_, Extend(NotNull(), 4096)) + .WillOnce(Return(0)); + ASSERT_EQ(0, fileManager_.Extend(1, 4096)); + + // 文件状态为closed，会重新open文件 + std::unordered_map recordMap + = fileManager_.GetRecordMap(); + recordMap[1]->status = NebdFileStatus::CLOSED; + // 重新open该文件，fd不变 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(0)); + EXPECT_CALL(*executor_, Extend(NotNull(), 4096)) + .WillOnce(Return(0)); + ASSERT_EQ(0, fileManager_.Extend(1, 4096)); + // 校验结果，fd不变 + recordMap = fileManager_.GetRecordMap(); + ASSERT_NE(recordMap[1], nullptr); + ASSERT_EQ(testFile1, recordMap[1]->fileName); +} + +TEST_F(FileManagerTest, ExtendFaileTest) { + InitEnv(); + + // 文件是opened状态 + EXPECT_CALL(*executor_, Extend(NotNull(), 4096)) + .WillOnce(Return(-1)); + ASSERT_EQ(-1, fileManager_.Extend(1, 4096)); + + // 文件状态为closed，会重新open文件 + std::unordered_map recordMap + = fileManager_.GetRecordMap(); + recordMap[1]->status = NebdFileStatus::CLOSED; + // 重新open文件时，open失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(nullptr)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .Times(0); + EXPECT_CALL(*executor_, Extend(NotNull(), 4096)) + .Times(0); + ASSERT_EQ(-1, fileManager_.Extend(1, 4096)); + // 重新open文件时，更新meta file失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(-1)); + EXPECT_CALL(*executor_, Extend(NotNull(), 4096)) + .Times(0); + ASSERT_EQ(-1, fileManager_.Extend(1, 4096)); +} + +TEST_F(FileManagerTest, GetInfoTest) { + InitEnv(); + + NebdFileInfo fileInfo; + // 文件是opened状态 + EXPECT_CALL(*executor_, GetInfo(NotNull(), &fileInfo)) + .WillOnce(Return(0)); + ASSERT_EQ(0, fileManager_.GetInfo(1, &fileInfo)); + + // 文件状态为closed，会重新open文件 + std::unordered_map recordMap + = fileManager_.GetRecordMap(); + recordMap[1]->status = NebdFileStatus::CLOSED; + // 重新open该文件，fd不变 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(0)); + EXPECT_CALL(*executor_, GetInfo(NotNull(), &fileInfo)) + .WillOnce(Return(0)); + ASSERT_EQ(0, fileManager_.GetInfo(1, &fileInfo)); + // 校验结果，fd不变 + recordMap = fileManager_.GetRecordMap(); + ASSERT_NE(recordMap[1], nullptr); + ASSERT_EQ(testFile1, recordMap[1]->fileName); +} + +TEST_F(FileManagerTest, GetInfoFaileTest) { + InitEnv(); + + NebdFileInfo fileInfo; + // 文件是opened状态 + EXPECT_CALL(*executor_, GetInfo(NotNull(), &fileInfo)) + .WillOnce(Return(-1)); + ASSERT_EQ(-1, fileManager_.GetInfo(1, &fileInfo)); + + // 文件状态为closed，会重新open文件 + std::unordered_map recordMap + = fileManager_.GetRecordMap(); + recordMap[1]->status = NebdFileStatus::CLOSED; + // 重新open文件时，open失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(nullptr)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .Times(0); + EXPECT_CALL(*executor_, GetInfo(NotNull(), _)) + .Times(0); + ASSERT_EQ(-1, fileManager_.GetInfo(1, &fileInfo)); + // 重新open文件时，更新meta file失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(-1)); + EXPECT_CALL(*executor_, GetInfo(NotNull(), _)) + .Times(0); + ASSERT_EQ(-1, fileManager_.GetInfo(1, &fileInfo)); +} + +TEST_F(FileManagerTest, InvalidCacheTest) { + InitEnv(); + + // 文件是opened状态 + EXPECT_CALL(*executor_, InvalidCache(NotNull())) + .WillOnce(Return(0)); + ASSERT_EQ(0, fileManager_.InvalidCache(1)); + + // 文件状态为closed，会重新open文件 + std::unordered_map recordMap + = fileManager_.GetRecordMap(); + recordMap[1]->status = NebdFileStatus::CLOSED; + // 重新open该文件，fd不变 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(0)); + EXPECT_CALL(*executor_, InvalidCache(NotNull())) + .WillOnce(Return(0)); + ASSERT_EQ(0, fileManager_.InvalidCache(1)); + // 校验结果，fd不变 + recordMap = fileManager_.GetRecordMap(); + ASSERT_NE(recordMap[1], nullptr); + ASSERT_EQ(testFile1, recordMap[1]->fileName); +} + +TEST_F(FileManagerTest, InvalidCacheFaileTest) { + InitEnv(); + + // 文件是opened状态 + EXPECT_CALL(*executor_, InvalidCache(NotNull())) + .WillOnce(Return(-1)); + ASSERT_EQ(-1, fileManager_.InvalidCache(1)); + + // 文件状态为closed，会重新open文件 + std::unordered_map recordMap + = fileManager_.GetRecordMap(); + recordMap[1]->status = NebdFileStatus::CLOSED; + // 重新open文件时，open失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(nullptr)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .Times(0); + EXPECT_CALL(*executor_, InvalidCache(NotNull())) + .Times(0); + ASSERT_EQ(-1, fileManager_.InvalidCache(1)); + // 重新open文件时，更新meta file失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(-1)); + EXPECT_CALL(*executor_, InvalidCache(NotNull())) + .Times(0); + ASSERT_EQ(-1, fileManager_.InvalidCache(1)); +} + +TEST_F(FileManagerTest, AioReadTest) { + InitEnv(); + + // 文件是opened状态 + EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) + .WillOnce(Return(0)); + ASSERT_EQ(0, fileManager_.AioRead(1, aioContext_)); + + // 文件状态为closed，会重新open文件 + std::unordered_map recordMap + = fileManager_.GetRecordMap(); + recordMap[1]->status = NebdFileStatus::CLOSED; + // 重新open该文件，fd不变 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(0)); + EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) + .WillOnce(Return(0)); + ASSERT_EQ(0, fileManager_.AioRead(1, aioContext_)); + // 校验结果，fd不变 + recordMap = fileManager_.GetRecordMap(); + ASSERT_NE(recordMap[1], nullptr); + ASSERT_EQ(testFile1, recordMap[1]->fileName); +} + +TEST_F(FileManagerTest, AioReadFaileTest) { + InitEnv(); + + // 文件是opened状态 + EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) + .WillOnce(Return(-1)); + ASSERT_EQ(-1, fileManager_.AioRead(1, aioContext_)); + + // 文件状态为closed，会重新open文件 + std::unordered_map recordMap + = fileManager_.GetRecordMap(); + recordMap[1]->status = NebdFileStatus::CLOSED; + // 重新open文件时，open失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(nullptr)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .Times(0); + EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) + .Times(0); + ASSERT_EQ(-1, fileManager_.AioRead(1, aioContext_)); + // 重新open文件时，更新meta file失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(-1)); + EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) + .Times(0); + ASSERT_EQ(-1, fileManager_.AioRead(1, aioContext_)); +} + +TEST_F(FileManagerTest, AioWriteTest) { + InitEnv(); + + // 文件是opened状态 + EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) + .WillOnce(Return(0)); + ASSERT_EQ(0, fileManager_.AioWrite(1, aioContext_)); + + // 文件状态为closed，会重新open文件 + std::unordered_map recordMap + = fileManager_.GetRecordMap(); + recordMap[1]->status = NebdFileStatus::CLOSED; + // 重新open该文件，fd不变 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(0)); + EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) + .WillOnce(Return(0)); + ASSERT_EQ(0, fileManager_.AioWrite(1, aioContext_)); + // 校验结果，fd不变 + recordMap = fileManager_.GetRecordMap(); + ASSERT_NE(recordMap[1], nullptr); + ASSERT_EQ(testFile1, recordMap[1]->fileName); +} + +TEST_F(FileManagerTest, AioWriteFaileTest) { + InitEnv(); + + // 文件是opened状态 + EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) + .WillOnce(Return(-1)); + ASSERT_EQ(-1, fileManager_.AioWrite(1, aioContext_)); + + // 文件状态为closed，会重新open文件 + std::unordered_map recordMap + = fileManager_.GetRecordMap(); + recordMap[1]->status = NebdFileStatus::CLOSED; + // 重新open文件时，open失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(nullptr)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .Times(0); + EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) + .Times(0); + ASSERT_EQ(-1, fileManager_.AioWrite(1, aioContext_)); + // 重新open文件时，更新meta file失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(-1)); + EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) + .Times(0); + ASSERT_EQ(-1, fileManager_.AioWrite(1, aioContext_)); +} + +TEST_F(FileManagerTest, DiscardTest) { + InitEnv(); + + // 文件是opened状态 + EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) + .WillOnce(Return(0)); + ASSERT_EQ(0, fileManager_.Discard(1, aioContext_)); + + // 文件状态为closed，会重新open文件 + std::unordered_map recordMap + = fileManager_.GetRecordMap(); + recordMap[1]->status = NebdFileStatus::CLOSED; + // 重新open该文件，fd不变 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(0)); + EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) + .WillOnce(Return(0)); + ASSERT_EQ(0, fileManager_.Discard(1, aioContext_)); + // 校验结果，fd不变 + recordMap = fileManager_.GetRecordMap(); + ASSERT_NE(recordMap[1], nullptr); + ASSERT_EQ(testFile1, recordMap[1]->fileName); +} + +TEST_F(FileManagerTest, DiscardFaileTest) { + InitEnv(); + + // 文件是opened状态 + EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) + .WillOnce(Return(-1)); + ASSERT_EQ(-1, fileManager_.Discard(1, aioContext_)); + + // 文件状态为closed，会重新open文件 + std::unordered_map recordMap + = fileManager_.GetRecordMap(); + recordMap[1]->status = NebdFileStatus::CLOSED; + // 重新open文件时，open失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(nullptr)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .Times(0); + EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) + .Times(0); + ASSERT_EQ(-1, fileManager_.Discard(1, aioContext_)); + // 重新open文件时，更新meta file失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(-1)); + EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) + .Times(0); + ASSERT_EQ(-1, fileManager_.Discard(1, aioContext_)); +} + +TEST_F(FileManagerTest, FlushTest) { + InitEnv(); + + // 文件是opened状态 + EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) + .WillOnce(Return(0)); + ASSERT_EQ(0, fileManager_.Flush(1, aioContext_)); + + // 文件状态为closed，会重新open文件 + std::unordered_map recordMap + = fileManager_.GetRecordMap(); + recordMap[1]->status = NebdFileStatus::CLOSED; + // 重新open该文件，fd不变 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(0)); + EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) + .WillOnce(Return(0)); + ASSERT_EQ(0, fileManager_.Flush(1, aioContext_)); + // 校验结果，fd不变 + recordMap = fileManager_.GetRecordMap(); + ASSERT_NE(recordMap[1], nullptr); + ASSERT_EQ(testFile1, recordMap[1]->fileName); +} + +TEST_F(FileManagerTest, FlushFaileTest) { + InitEnv(); + + // 文件是opened状态 + EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) + .WillOnce(Return(-1)); + ASSERT_EQ(-1, fileManager_.Flush(1, aioContext_)); + + // 文件状态为closed，会重新open文件 + std::unordered_map recordMap + = fileManager_.GetRecordMap(); + recordMap[1]->status = NebdFileStatus::CLOSED; + // 重新open文件时，open失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(nullptr)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .Times(0); + EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) + .Times(0); + ASSERT_EQ(-1, fileManager_.Flush(1, aioContext_)); + // 重新open文件时，更新meta file失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + .WillOnce(Return(-1)); + EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) + .Times(0); + ASSERT_EQ(-1, fileManager_.Flush(1, aioContext_)); +} + +TEST_F(FileManagerTest, CheckTimeoutTest) { + ASSERT_EQ(fileManager_.Run(), 0); + // 已经在run了不允许重复Run + ASSERT_EQ(fileManager_.Run(), -1); + + // 校验是否在检查超时 + InitEnv(); + std::unordered_map recordMap + = fileManager_.GetRecordMap(); + ASSERT_EQ(recordMap[1]->status, NebdFileStatus::OPENED); + ASSERT_EQ(recordMap[2]->status, NebdFileStatus::OPENED); + // 等待一段时间（未超过超时时间），模拟一个文件收到心跳，另一个没收到 + ::sleep(option_.heartbeatTimeoutS - 2); + ASSERT_EQ(0, fileManager_.UpdateFileTimestamp(1)); + // 在等待一段时间，其中一个文件超时，另一个文件未超时 + EXPECT_CALL(*executor_, Close(NotNull())) + .WillOnce(Return(0)); + ::sleep(option_.checkTimeoutIntervalMs / 1000 + 2); + recordMap = fileManager_.GetRecordMap(); + ASSERT_EQ(recordMap[1]->status, NebdFileStatus::OPENED); + ASSERT_EQ(recordMap[2]->status, NebdFileStatus::CLOSED); + // 继续等待一段时间，两个文件都超时 + EXPECT_CALL(*executor_, Close(NotNull())) + .WillOnce(Return(0)); + ::sleep(option_.heartbeatTimeoutS - 2); + ASSERT_EQ(recordMap[1]->status, NebdFileStatus::CLOSED); + ASSERT_EQ(recordMap[2]->status, NebdFileStatus::CLOSED); + + ASSERT_EQ(fileManager_.Fini(), 0); +} + +} // namespace server +} // namespace nebd + +int main(int argc, char ** argv) { + ::testing::InitGoogleTest(&argc, argv); + ::testing::InitGoogleMock(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/part2/file_record_map_unittest.cpp b/tests/part2/file_record_map_unittest.cpp new file mode 100644 index 0000000000..229d568892 --- /dev/null +++ b/tests/part2/file_record_map_unittest.cpp @@ -0,0 +1,101 @@ +/* + * Project: nebd + * Created Date: Monday January 20th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#include +#include +#include +#include + +#include "src/part2/file_record_map.h" + +namespace nebd { +namespace server { + +class FileRecordMapTest : public ::testing::Test { + public: + void SetUp() {} + void TearDown() {} + + protected: + FileRecordMap recordMap_; +}; + +TEST_F(FileRecordMapTest, BasicTest) { + int fd1 = 1; + std::string fileName1 = "file1"; + + // 初始测试 + ASSERT_FALSE(recordMap_.Exist(fd1)); + ASSERT_EQ(recordMap_.GetRecord(fd1), nullptr); + ASSERT_EQ(recordMap_.GetRecord(fileName1), nullptr); + + // UpdateRecord + NebdFileRecordPtr fileRecord1 = std::make_shared(); + fileRecord1->fileName = fileName1; + fileRecord1->fd = fd1; + ASSERT_TRUE(recordMap_.UpdateRecord(fileRecord1)); + ASSERT_TRUE(recordMap_.Exist(fd1)); + // 通过fd获取record + NebdFileRecordPtr tempRecord = recordMap_.GetRecord(fd1); + ASSERT_NE(tempRecord, nullptr); + ASSERT_EQ(tempRecord->fd, fd1); + ASSERT_EQ(tempRecord->fileName, fileName1); + // 通过filename获取record + tempRecord = recordMap_.GetRecord(fileName1); + ASSERT_NE(tempRecord, nullptr); + ASSERT_EQ(tempRecord->fd, fd1); + ASSERT_EQ(tempRecord->fileName, fileName1); + + // 插入一条filename相同，fd不同的记录 + fileRecord1->fd = ++fd1; + ASSERT_TRUE(recordMap_.UpdateRecord(fileRecord1)); + ASSERT_FALSE(recordMap_.Exist(fd1-1)); + ASSERT_TRUE(recordMap_.Exist(fd1)); + + // 插入不同记录 + NebdFileRecordPtr fileRecord2 = std::make_shared(); + std::string fileName2 = "file2"; + int fd2 = 3; + fileRecord2->fileName = fileName2; + fileRecord2->fd = fd2; + ASSERT_TRUE(recordMap_.UpdateRecord(fileRecord2)); + ASSERT_TRUE(recordMap_.Exist(fd1)); + ASSERT_TRUE(recordMap_.Exist(fd2)); + + // 插入fd相同，名字不同的记录 + NebdFileRecordPtr fileRecord3 = std::make_shared(); + fileRecord3->fileName = fileName1; + fileRecord3->fd = fd2; + ASSERT_FALSE(recordMap_.UpdateRecord(fileRecord3)); + + // 校验最终结果 + std::unordered_map list = recordMap_.ListRecords(); + ASSERT_EQ(list.size(), 2); + // 通过fd获取record并检查结果 + NebdFileRecordPtr tempRecord1 = recordMap_.GetRecord(fd1); + ASSERT_NE(tempRecord1, nullptr); + ASSERT_EQ(tempRecord1->fd, fd1); + ASSERT_EQ(tempRecord1->fileName, fileName1); + NebdFileRecordPtr tempRecord2 = recordMap_.GetRecord(fd2); + ASSERT_NE(tempRecord2, nullptr); + ASSERT_EQ(tempRecord2->fd, fd2); + ASSERT_EQ(tempRecord2->fileName, fileName2); + + // clear + recordMap_.Clear(); + list = recordMap_.ListRecords(); + ASSERT_EQ(list.size(), 0); +} + +} // namespace server +} // namespace nebd + +int main(int argc, char ** argv) { + ::testing::InitGoogleTest(&argc, argv); + ::testing::InitGoogleMock(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/part2/mock_metafile_manager.h b/tests/part2/mock_metafile_manager.h new file mode 100644 index 0000000000..69d4ae9596 --- /dev/null +++ b/tests/part2/mock_metafile_manager.h @@ -0,0 +1,33 @@ +/* + * Project: nebd + * Created Date: Tuesday January 21st 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#ifndef TESTS_PART2_MOCK_METAFILE_MANAGER_H_ +#define TESTS_PART2_MOCK_METAFILE_MANAGER_H_ + +#include +#include +#include + +#include "src/part2/metafile_manager.h" + +namespace nebd { +namespace server { + +class MockMetaFileManager : public NebdMetaFileManager { + public: + MockMetaFileManager() : NebdMetaFileManager("") {} + ~MockMetaFileManager() {} + + MOCK_METHOD1(RemoveFileRecord, int(const std::string&)); + MOCK_METHOD1(UpdateFileRecord, int(const NebdFileRecordPtr&)); + MOCK_METHOD1(ListFileRecord, int(std::vector*)); +}; + +} // namespace server +} // namespace nebd + +#endif // TESTS_PART2_MOCK_METAFILE_MANAGER_H_ diff --git a/tests/part2/mock_request_executor.h b/tests/part2/mock_request_executor.h new file mode 100644 index 0000000000..48fecbb4e4 --- /dev/null +++ b/tests/part2/mock_request_executor.h @@ -0,0 +1,48 @@ +/* + * Project: nebd + * Created Date: Tuesday January 21st 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#ifndef TESTS_PART2_MOCK_REQUEST_EXECUTOR_H_ +#define TESTS_PART2_MOCK_REQUEST_EXECUTOR_H_ + +#include +#include +#include + +#include "src/part2/request_executor.h" + +namespace nebd { +namespace server { + +class MockFileInstance : public NebdFileInstance { + public: + MockFileInstance() {} + ~MockFileInstance() {} +}; + +class MockRequestExecutor : public NebdRequestExecutor { + public: + MockRequestExecutor() {} + ~MockRequestExecutor() {} + + MOCK_METHOD1(Open, std::shared_ptr(const std::string&)); + MOCK_METHOD2(Reopen, std::shared_ptr(const std::string&, + AdditionType)); + MOCK_METHOD1(Close, int(NebdFileInstance*)); + MOCK_METHOD2(Extend, int(NebdFileInstance*, int64_t)); + MOCK_METHOD2(GetInfo, int(NebdFileInstance*, NebdFileInfo*)); + MOCK_METHOD2(StatFile, int(NebdFileInstance*, NebdFileInfo*)); + MOCK_METHOD2(Discard, int(NebdFileInstance*, NebdServerAioContext*)); + MOCK_METHOD2(AioRead, int(NebdFileInstance*, NebdServerAioContext*)); + MOCK_METHOD2(AioWrite, int(NebdFileInstance*, NebdServerAioContext*)); + MOCK_METHOD2(Flush, int(NebdFileInstance*, NebdServerAioContext*)); + MOCK_METHOD1(InvalidCache, int(NebdFileInstance*)); +}; + +} // namespace server +} // namespace nebd + +#endif // TESTS_PART2_MOCK_REQUEST_EXECUTOR_H_ From 2071000db981c128f2de75253b9d79f8f15f9582 Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Wed, 5 Feb 2020 10:13:42 +0800 Subject: [PATCH 21/79] implement file service Change-Id: I0d7a6d5f146d04746f4b439f598bde405ac5b984 --- src/common/client.proto | 1 - src/part2/define.h | 15 +- src/part2/file_manager.cpp | 15 +- src/part2/file_service.cpp | 307 +++++++++++++++-- src/part2/file_service.h | 10 + tests/part2/CMakeLists.txt | 14 +- tests/part2/file_manager_unittest.cpp | 32 +- tests/part2/file_service_unittest.cpp | 466 ++++++++++++++++++++++++++ tests/part2/mock_file_manager.h | 44 +++ 9 files changed, 856 insertions(+), 48 deletions(-) create mode 100644 tests/part2/file_service_unittest.cpp create mode 100644 tests/part2/mock_file_manager.h diff --git a/src/common/client.proto b/src/common/client.proto index c970f677fa..4da202612f 100644 --- a/src/common/client.proto +++ b/src/common/client.proto @@ -12,7 +12,6 @@ message OpenFileResponse { required RetCode retCode = 1; optional string retMsg = 2; optional int32 fd = 3; - // ps 最好fd最为一个64位以内的随机数，发挥sessionid的作用 } message CloseFileRequest { diff --git a/src/part2/define.h b/src/part2/define.h index bb9d173d0c..b4724e060b 100644 --- a/src/part2/define.h +++ b/src/part2/define.h @@ -1,18 +1,27 @@ +/* + * Project: nebd + * Created Date: Tuesday February 11th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + #ifndef SRC_PART2_DEFINE_H_ #define SRC_PART2_DEFINE_H_ #include +#include #include #include #include "src/common/rw_lock.h" +namespace nebd { +namespace server { + using nebd::common::RWLock; using ::google::protobuf::Message; using ::google::protobuf::Closure; - -namespace nebd { -namespace server { +using ::google::protobuf::RpcController; const char CURVE_PREFIX[] = "cbd"; const char CEPH_PREFIX[] = "rbd"; diff --git a/src/part2/file_manager.cpp b/src/part2/file_manager.cpp index 83ff6d3c30..f9cda70278 100644 --- a/src/part2/file_manager.cpp +++ b/src/part2/file_manager.cpp @@ -91,9 +91,12 @@ int NebdFileManager::Load() { int NebdFileManager::UpdateFileTimestamp(int fd) { NebdFileRecordPtr fileRecord = fileRecordMap_.GetRecord(fd); - if (fileRecord != nullptr) { - fileRecord->timeStamp = TimeUtility::GetTimeofDayMs(); + if (fileRecord == nullptr) { + LOG(WARNING) << "Update file timestamp failed, no record. " + << "fd: " << fd; + return -1; } + fileRecord->timeStamp = TimeUtility::GetTimeofDayMs(); return 0; } @@ -143,9 +146,9 @@ int NebdFileManager::Discard(int fd, NebdServerAioContext* aioctx) { int ret = fileRecord->executor->Discard( fileRecord->fileInstance.get(), aioctx); if (ret < 0) { + brpc::ClosureGuard doneGuard(done); aioctx->done = done->GetClosure(); done->SetClosure(nullptr); - done->Run(); LOG(ERROR) << "Discard file failed. " << "fd: " << fd << ", fileName: " << fileRecord->fileName @@ -165,9 +168,9 @@ int NebdFileManager::AioRead(int fd, NebdServerAioContext* aioctx) { int ret = fileRecord->executor->AioRead( fileRecord->fileInstance.get(), aioctx); if (ret < 0) { + brpc::ClosureGuard doneGuard(done); aioctx->done = done->GetClosure(); done->SetClosure(nullptr); - done->Run(); LOG(ERROR) << "AioRead file failed. " << "fd: " << fd << ", fileName: " << fileRecord->fileName @@ -187,9 +190,9 @@ int NebdFileManager::AioWrite(int fd, NebdServerAioContext* aioctx) { int ret = fileRecord->executor->AioWrite( fileRecord->fileInstance.get(), aioctx); if (ret < 0) { + brpc::ClosureGuard doneGuard(done); aioctx->done = done->GetClosure(); done->SetClosure(nullptr); - done->Run(); LOG(ERROR) << "AioWrite file failed. " << "fd: " << fd << ", fileName: " << fileRecord->fileName @@ -209,9 +212,9 @@ int NebdFileManager::Flush(int fd, NebdServerAioContext* aioctx) { int ret = fileRecord->executor->Flush( fileRecord->fileInstance.get(), aioctx); if (ret < 0) { + brpc::ClosureGuard doneGuard(done); aioctx->done = done->GetClosure(); done->SetClosure(nullptr); - done->Run(); LOG(ERROR) << "Flush file failed. " << "fd: " << fd << ", fileName: " << fileRecord->fileName diff --git a/src/part2/file_service.cpp b/src/part2/file_service.cpp index accab6a15f..e80e54de60 100644 --- a/src/part2/file_service.cpp +++ b/src/part2/file_service.cpp @@ -1,48 +1,260 @@ +/* + * Project: nebd + * Created Date: Thursday January 16th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#include +#include + #include "src/part2/file_service.h" namespace nebd { namespace server { +using nebd::client::RetCode; + +void NebdFileServiceCallback(NebdServerAioContext* context) { + CHECK(context != nullptr); + std::unique_ptr contextGuard(context); + brpc::ClosureGuard doneGuard(context->done); + switch (context->op) { + case LIBAIO_OP::LIBAIO_OP_READ: + { + nebd::client::ReadResponse* response = + dynamic_cast(context->response); + if (context->ret < 0) { + response->set_retcode(RetCode::kNoOK); + LOG(ERROR) << "Read file failed. " + << "return code: " << context->ret; + } else { + brpc::Controller* cntl = + dynamic_cast(context->cntl); + cntl->response_attachment().append(context->buf, + context->size); + response->set_retcode(RetCode::kOK); + } + delete[] reinterpret_cast(context->buf); + break; + } + case LIBAIO_OP::LIBAIO_OP_WRITE: + { + nebd::client::WriteResponse* response = + dynamic_cast(context->response); + if (context->ret < 0) { + response->set_retcode(RetCode::kNoOK); + LOG(ERROR) << "Write file failed. " + << "return code: " << context->ret; + } else { + response->set_retcode(RetCode::kOK); + } + break; + } + case LIBAIO_OP::LIBAIO_OP_FLUSH: + { + nebd::client::FlushResponse* response = + dynamic_cast(context->response); + if (context->ret < 0) { + response->set_retcode(RetCode::kNoOK); + LOG(ERROR) << "Flush file failed. " + << "return code: " << context->ret; + } else { + response->set_retcode(RetCode::kOK); + } + break; + } + case LIBAIO_OP::LIBAIO_OP_DISCARD: + { + nebd::client::DiscardResponse* response = + dynamic_cast(context->response); + if (context->ret < 0) { + response->set_retcode(RetCode::kNoOK); + LOG(ERROR) << "Discard file failed. " + << "return code: " << context->ret; + } else { + response->set_retcode(RetCode::kOK); + } + break; + } + default: + break; + } +} + void NebdFileServiceImpl::OpenFile( google::protobuf::RpcController* cntl_base, const nebd::client::OpenFileRequest* request, - nebd::client::OpenFileResponse* response, google::protobuf::Closure* done) { - // TODO + nebd::client::OpenFileResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + response->set_retcode(RetCode::kNoOK); + + int fd = fileManager_->Open(request->filename()); + if (fd > 0) { + response->set_retcode(RetCode::kOK); + response->set_fd(fd); + } else { + LOG(ERROR) << "Open file failed. " + << "filename: " << request->filename() + << ", return code: " << fd; + } +} + +void NebdFileServiceImpl::Write( + google::protobuf::RpcController* cntl_base, + const nebd::client::WriteRequest* request, + nebd::client::WriteResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + response->set_retcode(RetCode::kNoOK); + + NebdServerAioContext* aioContext + = new (std::nothrow) NebdServerAioContext(); + aioContext->offset = request->offset(); + aioContext->size = request->size(); + aioContext->op = LIBAIO_OP::LIBAIO_OP_WRITE; + aioContext->cb = NebdFileServiceCallback; + brpc::Controller* cntl = dynamic_cast(cntl_base); + aioContext->buf = + const_cast(cntl->request_attachment().to_string().c_str()); + aioContext->response = response; + aioContext->done = done; + aioContext->cntl = cntl_base; + int rc = fileManager_->AioWrite(request->fd(), aioContext); + if (rc < 0) { + LOG(ERROR) << "Write file failed. " + << "fd: " << request->fd() + << ", offset: " << request->offset() + << ", size: " << request->size() + << ", return code: " << rc; + } else { + doneGuard.release(); + } +} + +void NebdFileServiceImpl::Read( + google::protobuf::RpcController* cntl_base, + const nebd::client::ReadRequest* request, + nebd::client::ReadResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + response->set_retcode(RetCode::kNoOK); + + NebdServerAioContext* aioContext + = new (std::nothrow) NebdServerAioContext(); + aioContext->offset = request->offset(); + aioContext->size = request->size(); + aioContext->op = LIBAIO_OP::LIBAIO_OP_READ; + aioContext->cb = NebdFileServiceCallback; + aioContext->buf = new char[request->size()]; + aioContext->response = response; + aioContext->done = done; + aioContext->cntl = cntl_base; + int rc = fileManager_->AioRead(request->fd(), aioContext); + if (rc < 0) { + LOG(ERROR) << "Read file failed. " + << "fd: " << request->fd() + << ", offset: " << request->offset() + << ", size: " << request->size() + << ", return code: " << rc; + } else { + doneGuard.release(); + } } -void NebdFileServiceImpl::Write(google::protobuf::RpcController* cntl_base, - const nebd::client::WriteRequest* request, - nebd::client::WriteResponse* response, - google::protobuf::Closure* done) { - // TODO +void NebdFileServiceImpl::Flush( + google::protobuf::RpcController* cntl_base, + const nebd::client::FlushRequest* request, + nebd::client::FlushResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + response->set_retcode(RetCode::kNoOK); + + NebdServerAioContext* aioContext + = new (std::nothrow) NebdServerAioContext(); + aioContext->op = LIBAIO_OP::LIBAIO_OP_FLUSH; + aioContext->cb = NebdFileServiceCallback; + aioContext->response = response; + aioContext->done = done; + aioContext->cntl = cntl_base; + int rc = fileManager_->Flush(request->fd(), aioContext); + if (rc < 0) { + LOG(ERROR) << "Flush file failed. " + << "fd: " << request->fd() + << ", return code: " << rc; + } else { + doneGuard.release(); + } } -void NebdFileServiceImpl::Read(google::protobuf::RpcController* cntl_base, - const nebd::client::ReadRequest* request, - nebd::client::ReadResponse* response, - google::protobuf::Closure* done) { - // TODO +void NebdFileServiceImpl::Discard( + google::protobuf::RpcController* cntl_base, + const nebd::client::DiscardRequest* request, + nebd::client::DiscardResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + response->set_retcode(RetCode::kNoOK); + + NebdServerAioContext* aioContext + = new (std::nothrow) NebdServerAioContext(); + aioContext->offset = request->offset(); + aioContext->size = request->size(); + aioContext->op = LIBAIO_OP::LIBAIO_OP_DISCARD; + aioContext->cb = NebdFileServiceCallback; + aioContext->response = response; + aioContext->done = done; + aioContext->cntl = cntl_base; + int rc = fileManager_->Discard(request->fd(), aioContext); + if (rc < 0) { + LOG(ERROR) << "Flush file failed. " + << "fd: " << request->fd() + << ", offset: " << request->offset() + << ", size: " << request->size() + << ", return code: " << rc; + } else { + doneGuard.release(); + } } void NebdFileServiceImpl::StatFile( google::protobuf::RpcController* cntl_base, const nebd::client::StatFileRequest* request, - nebd::client::StatFileResponse* response, google::protobuf::Closure* done) { - // TODO -} + nebd::client::StatFileResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + response->set_retcode(RetCode::kNoOK); -void NebdFileServiceImpl::GetInfo(google::protobuf::RpcController* cntl_base, - const nebd::client::GetInfoRequest* request, - nebd::client::GetInfoResponse* response, - google::protobuf::Closure* done) { - // TODO + NebdFileInfo fileInfo; + int rc = fileManager_->StatFile(request->fd(), &fileInfo); + if (rc < 0) { + LOG(ERROR) << "Stat file failed. " + << "fd: " << request->fd() + << ", return code: " << rc; + } else { + response->set_retcode(RetCode::kOK); + response->set_size(fileInfo.size); + } } -void NebdFileServiceImpl::Flush(google::protobuf::RpcController* cntl_base, - const nebd::client::FlushRequest* request, - nebd::client::FlushResponse* response, - google::protobuf::Closure* done) { - // TODO +void NebdFileServiceImpl::GetInfo( + google::protobuf::RpcController* cntl_base, + const nebd::client::GetInfoRequest* request, + nebd::client::GetInfoResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + response->set_retcode(RetCode::kNoOK); + + NebdFileInfo fileInfo; + int rc = fileManager_->GetInfo(request->fd(), &fileInfo); + if (rc < 0) { + LOG(ERROR) << "Get file info failed. " + << "fd: " << request->fd() + << ", return code: " << rc; + } else { + response->set_retcode(RetCode::kOK); + response->set_objsize(fileInfo.obj_size); + } } void NebdFileServiceImpl::CloseFile( @@ -50,21 +262,36 @@ void NebdFileServiceImpl::CloseFile( const nebd::client::CloseFileRequest* request, nebd::client::CloseFileResponse* response, google::protobuf::Closure* done) { - // TODO -} + brpc::ClosureGuard doneGuard(done); + response->set_retcode(RetCode::kNoOK); -void NebdFileServiceImpl::Discard(google::protobuf::RpcController* cntl_base, - const nebd::client::DiscardRequest* request, - nebd::client::DiscardResponse* response, - google::protobuf::Closure* done) { - // TODO + int rc = fileManager_->Close(request->fd()); + if (rc < 0) { + LOG(ERROR) << "Close file failed. " + << "fd: " << request->fd() + << ", return code: " << rc; + } else { + response->set_retcode(RetCode::kOK); + } } void NebdFileServiceImpl::ResizeFile( google::protobuf::RpcController* cntl_base, const nebd::client::ResizeRequest* request, - nebd::client::ResizeResponse* response, google::protobuf::Closure* done) { - // TODO + nebd::client::ResizeResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + response->set_retcode(RetCode::kNoOK); + + int rc = fileManager_->Extend(request->fd(), request->newsize()); + if (rc < 0) { + LOG(ERROR) << "Resize file failed. " + << "fd: " << request->fd() + << ", newsize: " << request->newsize() + << ", return code: " << rc; + } else { + response->set_retcode(RetCode::kOK); + } } void NebdFileServiceImpl::InvalidateCache( @@ -72,7 +299,17 @@ void NebdFileServiceImpl::InvalidateCache( const nebd::client::InvalidateCacheRequest* request, nebd::client::InvalidateCacheResponse* response, google::protobuf::Closure* done) { - // TODO + brpc::ClosureGuard doneGuard(done); + response->set_retcode(RetCode::kNoOK); + + int rc = fileManager_->InvalidCache(request->fd()); + if (rc < 0) { + LOG(ERROR) << "Invalid file cache failed. " + << "fd: " << request->fd() + << ", return code: " << rc; + } else { + response->set_retcode(RetCode::kOK); + } } } // namespace server diff --git a/src/part2/file_service.h b/src/part2/file_service.h index be65748377..8e44a14b3f 100644 --- a/src/part2/file_service.h +++ b/src/part2/file_service.h @@ -1,3 +1,10 @@ +/* + * Project: nebd + * Created Date: Thursday January 16th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + #ifndef SRC_PART2_FILE_SERVICE_H_ #define SRC_PART2_FILE_SERVICE_H_ @@ -7,12 +14,15 @@ #include #include +#include "src/common/common.pb.h" #include "src/common/client.pb.h" #include "src/part2/file_manager.h" namespace nebd { namespace server { +void NebdFileServiceCallback(NebdServerAioContext* context); + class NebdFileServiceImpl : public nebd::client::NebdFileService { public: explicit NebdFileServiceImpl(std::shared_ptr fileManager) diff --git a/tests/part2/CMakeLists.txt b/tests/part2/CMakeLists.txt index 73a876cb85..6d35470496 100644 --- a/tests/part2/CMakeLists.txt +++ b/tests/part2/CMakeLists.txt @@ -15,7 +15,8 @@ set(PART2_TEST_LINK # libpart2mock.so set(PART2_MOCK_SRC mock_metafile_manager.h - mock_request_executor.h) + mock_request_executor.h + mock_file_manager.h) add_library(part2mock ${PART2_MOCK_SRC}) set_target_properties(part2mock PROPERTIES LINKER_LANGUAGE CXX) target_link_libraries(part2mock @@ -36,4 +37,13 @@ target_link_libraries(filemanager_test part2mock ${PART2_TEST_LINK}) install(TARGETS filemanager_test DESTINATION bin) -add_test(NAME filemanager_test COMMAND filemanager_test) \ No newline at end of file +add_test(NAME filemanager_test COMMAND filemanager_test) + +# fileservice_test +set(FILE_SERVICE_TEST_SRC file_service_unittest.cpp) +add_executable(fileservice_test ${FILE_SERVICE_TEST_SRC}) +target_link_libraries(fileservice_test + part2mock + ${PART2_TEST_LINK}) +install(TARGETS fileservice_test DESTINATION bin) +add_test(NAME fileservice_test COMMAND fileservice_test) \ No newline at end of file diff --git a/tests/part2/file_manager_unittest.cpp b/tests/part2/file_manager_unittest.cpp index 6231b4ebfe..b513b919ab 100644 --- a/tests/part2/file_manager_unittest.cpp +++ b/tests/part2/file_manager_unittest.cpp @@ -192,6 +192,11 @@ TEST_F(FileManagerTest, LoadFailTest) { fileRecords.push_back(record1); fileRecords.push_back(record2); + // ListFileRecord 失败，返回失败 + EXPECT_CALL(*metaFileManager_, ListFileRecord(_)) + .WillOnce(Return(-1)); + ASSERT_EQ(-1, fileManager_.Load()); + EXPECT_CALL(*metaFileManager_, ListFileRecord(_)) .WillOnce(DoAll(SetArgPointee<0>(fileRecords), Return(0))); @@ -441,6 +446,11 @@ TEST_F(FileManagerTest, AioReadTest) { TEST_F(FileManagerTest, AioReadFaileTest) { InitEnv(); + // 文件不存在 + EXPECT_CALL(*executor_, AioRead(_, _)) + .Times(0); + ASSERT_EQ(-1, fileManager_.AioRead(10, aioContext_)); + // 文件是opened状态 EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) .WillOnce(Return(-1)); @@ -497,6 +507,11 @@ TEST_F(FileManagerTest, AioWriteTest) { TEST_F(FileManagerTest, AioWriteFaileTest) { InitEnv(); + // 文件不存在 + EXPECT_CALL(*executor_, AioWrite(_, _)) + .Times(0); + ASSERT_EQ(-1, fileManager_.AioWrite(10, aioContext_)); + // 文件是opened状态 EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) .WillOnce(Return(-1)); @@ -553,6 +568,11 @@ TEST_F(FileManagerTest, DiscardTest) { TEST_F(FileManagerTest, DiscardFaileTest) { InitEnv(); + // 文件不存在 + EXPECT_CALL(*executor_, Discard(_, _)) + .Times(0); + ASSERT_EQ(-1, fileManager_.Discard(10, aioContext_)); + // 文件是opened状态 EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) .WillOnce(Return(-1)); @@ -609,6 +629,11 @@ TEST_F(FileManagerTest, FlushTest) { TEST_F(FileManagerTest, FlushFaileTest) { InitEnv(); + // 文件不存在 + EXPECT_CALL(*executor_, Flush(_, _)) + .Times(0); + ASSERT_EQ(-1, fileManager_.Flush(10, aioContext_)); + // 文件是opened状态 EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) .WillOnce(Return(-1)); @@ -638,8 +663,9 @@ TEST_F(FileManagerTest, FlushFaileTest) { TEST_F(FileManagerTest, CheckTimeoutTest) { ASSERT_EQ(fileManager_.Run(), 0); - // 已经在run了不允许重复Run + // 已经在run了不允许重复Run或者Init ASSERT_EQ(fileManager_.Run(), -1); + ASSERT_EQ(fileManager_.Init(option_), -1); // 校验是否在检查超时 InitEnv(); @@ -650,6 +676,8 @@ TEST_F(FileManagerTest, CheckTimeoutTest) { // 等待一段时间（未超过超时时间），模拟一个文件收到心跳，另一个没收到 ::sleep(option_.heartbeatTimeoutS - 2); ASSERT_EQ(0, fileManager_.UpdateFileTimestamp(1)); + // 更新不存在的fd，返回失败 + ASSERT_EQ(-1, fileManager_.UpdateFileTimestamp(10)); // 在等待一段时间，其中一个文件超时，另一个文件未超时 EXPECT_CALL(*executor_, Close(NotNull())) .WillOnce(Return(0)); @@ -665,6 +693,8 @@ TEST_F(FileManagerTest, CheckTimeoutTest) { ASSERT_EQ(recordMap[2]->status, NebdFileStatus::CLOSED); ASSERT_EQ(fileManager_.Fini(), 0); + // 重复Fini，也返回成功 + ASSERT_EQ(fileManager_.Fini(), 0); } } // namespace server diff --git a/tests/part2/file_service_unittest.cpp b/tests/part2/file_service_unittest.cpp new file mode 100644 index 0000000000..f0797cbaee --- /dev/null +++ b/tests/part2/file_service_unittest.cpp @@ -0,0 +1,466 @@ +/* + * Project: nebd + * Created Date: Tuesday February 4th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + + +#include +#include +#include +#include + +#include "src/part2/file_service.h" +#include "tests/part2/mock_file_manager.h" + +namespace nebd { +namespace server { + +const char testFile1[] = "test:/cinder/111"; + +using ::testing::_; +using ::testing::Return; +using ::testing::NotNull; +using ::testing::DoAll; +using ::testing::ReturnArg; +using ::testing::ElementsAre; +using ::testing::SetArgPointee; +using ::testing::SetArrayArgument; + +using google::protobuf::RpcController; +using google::protobuf::Closure; + +using nebd::client::RetCode; + +class FileServiceTestClosure : public Closure { + public: + FileServiceTestClosure() : runned_(false) {} + ~FileServiceTestClosure() {} + void Run() { + runned_ = true; + } + bool IsRunned() { + return runned_; + } + void Reset() { + runned_ = false; + } + + private: + bool runned_; +}; + +class FileServiceTest : public ::testing::Test { + public: + void SetUp() { + fileManager_ = std::make_shared(); + fileService_ = std::make_shared(fileManager_); + } + void TearDown() {} + protected: + std::shared_ptr fileManager_; + std::shared_ptr fileService_; +}; + +TEST_F(FileServiceTest, OpenTest) { + brpc::Controller cntl; + nebd::client::OpenFileRequest request; + request.set_filename(testFile1); + nebd::client::OpenFileResponse response; + FileServiceTestClosure done; + + // open success + EXPECT_CALL(*fileManager_, Open(testFile1)) + .WillOnce(Return(1)); + fileService_->OpenFile(&cntl, &request, &response, &done); + ASSERT_EQ(response.retcode(), RetCode::kOK); + ASSERT_EQ(response.fd(), 1); + ASSERT_TRUE(done.IsRunned()); + + // open failed + done.Reset(); + EXPECT_CALL(*fileManager_, Open(testFile1)) + .WillOnce(Return(-1)); + fileService_->OpenFile(&cntl, &request, &response, &done); + ASSERT_EQ(response.retcode(), RetCode::kNoOK); + ASSERT_TRUE(done.IsRunned()); +} + +TEST_F(FileServiceTest, WriteTest) { + int fd = 1; + uint64_t offset = 0; + uint64_t size = 4096; + brpc::Controller cntl; + nebd::client::WriteRequest request; + request.set_fd(fd); + request.set_offset(offset); + request.set_size(size); + nebd::client::WriteResponse response; + FileServiceTestClosure done; + + // write success + EXPECT_CALL(*fileManager_, AioWrite(fd, NotNull())) + .WillOnce(Return(0)); + fileService_->Write(&cntl, &request, &response, &done); + ASSERT_FALSE(done.IsRunned()); + + // write failed + done.Reset(); + EXPECT_CALL(*fileManager_, AioWrite(fd, NotNull())) + .WillOnce(Return(-1)); + fileService_->Write(&cntl, &request, &response, &done); + ASSERT_EQ(response.retcode(), RetCode::kNoOK); + ASSERT_TRUE(done.IsRunned()); +} + +TEST_F(FileServiceTest, ReadTest) { + int fd = 1; + uint64_t offset = 0; + uint64_t size = 4096; + brpc::Controller cntl; + nebd::client::ReadRequest request; + request.set_fd(fd); + request.set_offset(offset); + request.set_size(size); + nebd::client::ReadResponse response; + FileServiceTestClosure done; + + // read success + EXPECT_CALL(*fileManager_, AioRead(fd, NotNull())) + .WillOnce(Return(0)); + fileService_->Read(&cntl, &request, &response, &done); + ASSERT_FALSE(done.IsRunned()); + + // read failed + done.Reset(); + EXPECT_CALL(*fileManager_, AioRead(fd, NotNull())) + .WillOnce(Return(-1)); + fileService_->Read(&cntl, &request, &response, &done); + ASSERT_EQ(response.retcode(), RetCode::kNoOK); + ASSERT_TRUE(done.IsRunned()); +} + +TEST_F(FileServiceTest, FlushTest) { + int fd = 1; + brpc::Controller cntl; + nebd::client::FlushRequest request; + request.set_fd(fd); + nebd::client::FlushResponse response; + FileServiceTestClosure done; + + // flush success + EXPECT_CALL(*fileManager_, Flush(fd, NotNull())) + .WillOnce(Return(0)); + fileService_->Flush(&cntl, &request, &response, &done); + ASSERT_FALSE(done.IsRunned()); + + // flush failed + done.Reset(); + EXPECT_CALL(*fileManager_, Flush(fd, NotNull())) + .WillOnce(Return(-1)); + fileService_->Flush(&cntl, &request, &response, &done); + ASSERT_EQ(response.retcode(), RetCode::kNoOK); + ASSERT_TRUE(done.IsRunned()); +} + +TEST_F(FileServiceTest, DiscardTest) { + int fd = 1; + uint64_t offset = 0; + uint64_t size = 4096; + brpc::Controller cntl; + nebd::client::DiscardRequest request; + request.set_fd(fd); + request.set_offset(offset); + request.set_size(size); + nebd::client::DiscardResponse response; + FileServiceTestClosure done; + + // discard success + EXPECT_CALL(*fileManager_, Discard(fd, NotNull())) + .WillOnce(Return(0)); + fileService_->Discard(&cntl, &request, &response, &done); + ASSERT_FALSE(done.IsRunned()); + + // discard failed + done.Reset(); + EXPECT_CALL(*fileManager_, Discard(fd, NotNull())) + .WillOnce(Return(-1)); + fileService_->Discard(&cntl, &request, &response, &done); + ASSERT_EQ(response.retcode(), RetCode::kNoOK); + ASSERT_TRUE(done.IsRunned()); +} + +TEST_F(FileServiceTest, StatFileTest) { + int fd = 1; + brpc::Controller cntl; + nebd::client::StatFileRequest request; + request.set_fd(fd); + nebd::client::StatFileResponse response; + FileServiceTestClosure done; + + // stat file success + NebdFileInfo fileInfo; + fileInfo.size = 4096; + EXPECT_CALL(*fileManager_, StatFile(fd, NotNull())) + .WillOnce(DoAll(SetArgPointee<1>(fileInfo), + Return(0))); + fileService_->StatFile(&cntl, &request, &response, &done); + ASSERT_EQ(response.retcode(), RetCode::kOK); + ASSERT_EQ(response.size(), fileInfo.size); + ASSERT_TRUE(done.IsRunned()); + + // stat file failed + done.Reset(); + EXPECT_CALL(*fileManager_, StatFile(fd, NotNull())) + .WillOnce(Return(-1)); + fileService_->StatFile(&cntl, &request, &response, &done); + ASSERT_EQ(response.retcode(), RetCode::kNoOK); + ASSERT_TRUE(done.IsRunned()); +} + +TEST_F(FileServiceTest, GetInfoTest) { + int fd = 1; + brpc::Controller cntl; + nebd::client::GetInfoRequest request; + request.set_fd(fd); + nebd::client::GetInfoResponse response; + FileServiceTestClosure done; + + // stat file success + NebdFileInfo fileInfo; + fileInfo.obj_size = 4096; + EXPECT_CALL(*fileManager_, GetInfo(fd, NotNull())) + .WillOnce(DoAll(SetArgPointee<1>(fileInfo), + Return(0))); + fileService_->GetInfo(&cntl, &request, &response, &done); + ASSERT_EQ(response.retcode(), RetCode::kOK); + ASSERT_EQ(response.objsize(), fileInfo.obj_size); + ASSERT_TRUE(done.IsRunned()); + + // stat file failed + done.Reset(); + EXPECT_CALL(*fileManager_, GetInfo(fd, NotNull())) + .WillOnce(Return(-1)); + fileService_->GetInfo(&cntl, &request, &response, &done); + ASSERT_EQ(response.retcode(), RetCode::kNoOK); + ASSERT_TRUE(done.IsRunned()); +} + +TEST_F(FileServiceTest, CloseTest) { + int fd = 1; + brpc::Controller cntl; + nebd::client::CloseFileRequest request; + request.set_fd(fd); + nebd::client::CloseFileResponse response; + FileServiceTestClosure done; + + // close success + EXPECT_CALL(*fileManager_, Close(fd)) + .WillOnce(Return(0)); + fileService_->CloseFile(&cntl, &request, &response, &done); + ASSERT_EQ(response.retcode(), RetCode::kOK); + ASSERT_TRUE(done.IsRunned()); + + // close failed + done.Reset(); + EXPECT_CALL(*fileManager_, Close(fd)) + .WillOnce(Return(-1)); + fileService_->CloseFile(&cntl, &request, &response, &done); + ASSERT_EQ(response.retcode(), RetCode::kNoOK); + ASSERT_TRUE(done.IsRunned()); +} + +TEST_F(FileServiceTest, ResizeTest) { + int fd = 1; + uint64_t size = 4096; + brpc::Controller cntl; + nebd::client::ResizeRequest request; + request.set_fd(fd); + request.set_newsize(size); + nebd::client::ResizeResponse response; + FileServiceTestClosure done; + + // resize success + EXPECT_CALL(*fileManager_, Extend(fd, size)) + .WillOnce(Return(0)); + fileService_->ResizeFile(&cntl, &request, &response, &done); + ASSERT_EQ(response.retcode(), RetCode::kOK); + ASSERT_TRUE(done.IsRunned()); + + // resize failed + done.Reset(); + EXPECT_CALL(*fileManager_, Extend(fd, size)) + .WillOnce(Return(-1)); + fileService_->ResizeFile(&cntl, &request, &response, &done); + ASSERT_EQ(response.retcode(), RetCode::kNoOK); + ASSERT_TRUE(done.IsRunned()); +} + +TEST_F(FileServiceTest, InvalidCacheTest) { + int fd = 1; + brpc::Controller cntl; + nebd::client::InvalidateCacheRequest request; + request.set_fd(fd); + nebd::client::InvalidateCacheResponse response; + FileServiceTestClosure done; + + // invalid cache success + EXPECT_CALL(*fileManager_, InvalidCache(fd)) + .WillOnce(Return(0)); + fileService_->InvalidateCache(&cntl, &request, &response, &done); + ASSERT_EQ(response.retcode(), RetCode::kOK); + ASSERT_TRUE(done.IsRunned()); + + // invalid cache failed + done.Reset(); + EXPECT_CALL(*fileManager_, InvalidCache(fd)) + .WillOnce(Return(-1)); + fileService_->InvalidateCache(&cntl, &request, &response, &done); + ASSERT_EQ(response.retcode(), RetCode::kNoOK); + ASSERT_TRUE(done.IsRunned()); +} + +TEST_F(FileServiceTest, CallbackTest) { + // read success + { + brpc::Controller cntl; + nebd::client::ReadResponse response; + FileServiceTestClosure done; + NebdServerAioContext* context = new NebdServerAioContext; + context->op = LIBAIO_OP::LIBAIO_OP_READ; + context->cntl = &cntl; + context->response = &response; + context->offset = 0; + context->size = 4096; + context->done = &done; + context->buf = new char[4096]; + context->ret = 0; + NebdFileServiceCallback(context); + ASSERT_TRUE(done.IsRunned()); + ASSERT_EQ(response.retcode(), RetCode::kOK); + } + // read failed + { + brpc::Controller cntl; + nebd::client::ReadResponse response; + FileServiceTestClosure done; + NebdServerAioContext* context = new NebdServerAioContext; + context->op = LIBAIO_OP::LIBAIO_OP_READ; + context->cntl = &cntl; + context->response = &response; + context->offset = 0; + context->size = 4096; + context->done = &done; + context->buf = new char[4096]; + context->ret = -1; + NebdFileServiceCallback(context); + ASSERT_TRUE(done.IsRunned()); + ASSERT_EQ(response.retcode(), RetCode::kNoOK); + } + // write success + { + brpc::Controller cntl; + nebd::client::WriteResponse response; + FileServiceTestClosure done; + NebdServerAioContext* context = new NebdServerAioContext; + context->op = LIBAIO_OP::LIBAIO_OP_WRITE; + context->cntl = &cntl; + context->response = &response; + context->offset = 0; + context->size = 4096; + context->done = &done; + context->buf = new char[4096]; + context->ret = 0; + NebdFileServiceCallback(context); + ASSERT_TRUE(done.IsRunned()); + ASSERT_EQ(response.retcode(), RetCode::kOK); + } + // write failed + { + brpc::Controller cntl; + nebd::client::WriteResponse response; + FileServiceTestClosure done; + NebdServerAioContext* context = new NebdServerAioContext; + context->op = LIBAIO_OP::LIBAIO_OP_WRITE; + context->cntl = &cntl; + context->response = &response; + context->offset = 0; + context->size = 4096; + context->done = &done; + context->buf = new char[4096]; + context->ret = -1; + NebdFileServiceCallback(context); + ASSERT_TRUE(done.IsRunned()); + ASSERT_EQ(response.retcode(), RetCode::kNoOK); + } + // flush success + { + brpc::Controller cntl; + nebd::client::FlushResponse response; + FileServiceTestClosure done; + NebdServerAioContext* context = new NebdServerAioContext; + context->op = LIBAIO_OP::LIBAIO_OP_FLUSH; + context->cntl = &cntl; + context->response = &response; + context->done = &done; + context->ret = 0; + NebdFileServiceCallback(context); + ASSERT_TRUE(done.IsRunned()); + ASSERT_EQ(response.retcode(), RetCode::kOK); + } + // flush failed + { + brpc::Controller cntl; + nebd::client::FlushResponse response; + FileServiceTestClosure done; + NebdServerAioContext* context = new NebdServerAioContext; + context->op = LIBAIO_OP::LIBAIO_OP_FLUSH; + context->cntl = &cntl; + context->response = &response; + context->done = &done; + context->ret = -1; + NebdFileServiceCallback(context); + ASSERT_TRUE(done.IsRunned()); + ASSERT_EQ(response.retcode(), RetCode::kNoOK); + } + // discard success + { + brpc::Controller cntl; + nebd::client::DiscardResponse response; + FileServiceTestClosure done; + NebdServerAioContext* context = new NebdServerAioContext; + context->op = LIBAIO_OP::LIBAIO_OP_DISCARD; + context->cntl = &cntl; + context->response = &response; + context->done = &done; + context->ret = 0; + NebdFileServiceCallback(context); + ASSERT_TRUE(done.IsRunned()); + ASSERT_EQ(response.retcode(), RetCode::kOK); + } + // discard failed + { + brpc::Controller cntl; + nebd::client::DiscardResponse response; + FileServiceTestClosure done; + NebdServerAioContext* context = new NebdServerAioContext; + context->op = LIBAIO_OP::LIBAIO_OP_DISCARD; + context->cntl = &cntl; + context->response = &response; + context->done = &done; + context->ret = -1; + NebdFileServiceCallback(context); + ASSERT_TRUE(done.IsRunned()); + ASSERT_EQ(response.retcode(), RetCode::kNoOK); + } +} + +} // namespace server +} // namespace nebd + +int main(int argc, char ** argv) { + ::testing::InitGoogleTest(&argc, argv); + ::testing::InitGoogleMock(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/part2/mock_file_manager.h b/tests/part2/mock_file_manager.h new file mode 100644 index 0000000000..242c15ed10 --- /dev/null +++ b/tests/part2/mock_file_manager.h @@ -0,0 +1,44 @@ +/* + * Project: nebd + * Created Date: Tuesday February 4th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#ifndef TESTS_PART2_MOCK_FILE_MANAGER_H_ +#define TESTS_PART2_MOCK_FILE_MANAGER_H_ + +#include +#include +#include + +#include "src/part2/file_manager.h" + +namespace nebd { +namespace server { + +class MockFileManager : public NebdFileManager { + public: + MockFileManager() {} + ~MockFileManager() {} + + MOCK_METHOD1(Init, int(NebdFileManagerOption)); + MOCK_METHOD0(Fini, int()); + MOCK_METHOD0(Run, int()); + MOCK_METHOD1(UpdateFileTimestamp, int(int)); + MOCK_METHOD1(Open, int(const std::string&)); + MOCK_METHOD1(Close, int(int)); + MOCK_METHOD2(Extend, int(int, uint64_t)); + MOCK_METHOD2(GetInfo, int(int, NebdFileInfo*)); + MOCK_METHOD2(StatFile, int(int, NebdFileInfo*)); + MOCK_METHOD2(Discard, int(int, NebdServerAioContext*)); + MOCK_METHOD2(AioRead, int(int, NebdServerAioContext*)); + MOCK_METHOD2(AioWrite, int(int, NebdServerAioContext*)); + MOCK_METHOD2(Flush, int(int, NebdServerAioContext*)); + MOCK_METHOD1(InvalidCache, int(int)); +}; + +} // namespace server +} // namespace nebd + +#endif // TESTS_PART2_MOCK_FILE_MANAGER_H_ From f25230b1e28692c8c716f8cf4a239836472e82fe Mon Sep 17 00:00:00 2001 From: charisu Date: Mon, 3 Feb 2020 17:06:01 +0800 Subject: [PATCH 22/79] implement heartbeat service Change-Id: I54ced32da83168933fda677b79df91aefc2d0a03 --- get_3rdparty | 8 ++- src/part2/heartbeat_service.cpp | 33 ++++++++-- src/part2/heartbeat_service.h | 9 +++ tests/part2/CMakeLists.txt | 11 +++- tests/part2/heartbeat_service_test.cpp | 83 ++++++++++++++++++++++++++ 5 files changed, 137 insertions(+), 7 deletions(-) create mode 100644 tests/part2/heartbeat_service_test.cpp diff --git a/get_3rdparty b/get_3rdparty index 2575792442..651d621d31 100755 --- a/get_3rdparty +++ b/get_3rdparty @@ -1,7 +1,7 @@ #!/bin/bash -x mkdir 3rd_party_tarballs -for pkg_ver in boost_1_66_0 brpc_0_9_6 googletest_1_8_1 libconfig_1_7_2 jsoncpp_1_8_4 cmock_20180514; do +for pkg_ver in boost_1_66_0 googletest_1_8_1 libconfig_1_7_2 jsoncpp_1_8_4 cmock_20180514; do wget -c --no-verbose http://10.187.0.105/nebd-build/$pkg_ver.tar.gz -P 3rd_party_tarballs/ wget -c --no-verbose http://10.187.0.105/nebd-build/$pkg_ver.tar.gz.md5sum -P 3rd_party_tarballs/ pkg=$(echo ${pkg_ver} | cut -d_ -f1) @@ -10,4 +10,10 @@ for pkg_ver in boost_1_66_0 brpc_0_9_6 googletest_1_8_1 libconfig_1_7_2 jsoncpp_ tar xf 3rd_party_tarballs/$pkg_ver.tar.gz -C 3rdparty/$pkg --strip-components=1 done +rm -rf 3rdparty/brpc +cd 3rdparty +git clone -b feature http://gerrit.storage.netease.com/curve/curve-brpc +mv curve-brpc brpc +cd .. + rm -rf 3rd_party_tarballs diff --git a/src/part2/heartbeat_service.cpp b/src/part2/heartbeat_service.cpp index bb05257512..178fe2b4cd 100644 --- a/src/part2/heartbeat_service.cpp +++ b/src/part2/heartbeat_service.cpp @@ -1,14 +1,37 @@ +/* + * Project: nebd + * Created Date: 2020-02-03 + * Author: charisu + * Copyright (c) 2020 netease + */ + #include "src/part2/heartbeat_service.h" namespace nebd { namespace server { void NebdHeartbeatServiceImpl::KeepAlive( - google::protobuf::RpcController* cntl_base, - const nebd::client::HeartbeatRequest* request, - nebd::client::HeartbeatResponse* response, - google::protobuf::Closure* done) { - // TODO + google::protobuf::RpcController* cntl_base, + const nebd::client::HeartbeatRequest* request, + nebd::client::HeartbeatResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + bool ok = true; + for (int i = 0; i < request->info_size(); ++i) { + const auto& info = request->info(i); + int res = fileManager_->UpdateFileTimestamp(info.fd()); + if (res != 0) { + LOG(WARNING) << "Update file timestamp fail, fd: " + << info.fd() << ", name: " << info.name(); + ok = false; + } + } + + if (ok) { + response->set_retcode(nebd::client::RetCode::kOK); + } else { + response->set_retcode(nebd::client::RetCode::kNoOK); + } } } // namespace server diff --git a/src/part2/heartbeat_service.h b/src/part2/heartbeat_service.h index 89d1af095f..a21a479f99 100644 --- a/src/part2/heartbeat_service.h +++ b/src/part2/heartbeat_service.h @@ -1,6 +1,15 @@ +/* + * Project: nebd + * Created Date: 2020-02-03 + * Author: charisu + * Copyright (c) 2020 netease + */ + #ifndef SRC_PART2_HEARTBEAT_SERVICE_H_ #define SRC_PART2_HEARTBEAT_SERVICE_H_ +#include +#include #include #include "src/common/heartbeat.pb.h" diff --git a/tests/part2/CMakeLists.txt b/tests/part2/CMakeLists.txt index 6d35470496..90ecefcff2 100644 --- a/tests/part2/CMakeLists.txt +++ b/tests/part2/CMakeLists.txt @@ -46,4 +46,13 @@ target_link_libraries(fileservice_test part2mock ${PART2_TEST_LINK}) install(TARGETS fileservice_test DESTINATION bin) -add_test(NAME fileservice_test COMMAND fileservice_test) \ No newline at end of file +add_test(NAME fileservice_test COMMAND fileservice_test) + +# heartbeat_service_test +set(HEARTBEAT_SERVICE_TEST_SRC heartbeat_service_test.cpp) +add_executable(heartbeat_service_test ${HEARTBEAT_SERVICE_TEST_SRC}) +target_link_libraries(heartbeat_service_test + part2mock + ${PART2_TEST_LINK}) +install(TARGETS heartbeat_service_test DESTINATION bin) +add_test(NAME heartbeat_service_test COMMAND heartbeat_service_test) diff --git a/tests/part2/heartbeat_service_test.cpp b/tests/part2/heartbeat_service_test.cpp new file mode 100644 index 0000000000..6e212da5cf --- /dev/null +++ b/tests/part2/heartbeat_service_test.cpp @@ -0,0 +1,83 @@ +/* + * Project: nebd + * Created Date: 2020-02-03 + * Author: charisu + * Copyright (c) 2020 netease + */ + +#include +#include +#include +#include + +#include "src/common/heartbeat.pb.h" +#include "src/part2/heartbeat_service.h" +#include "tests/part2/mock_file_manager.h" + +using ::testing::_; +using ::testing::Return; + +namespace nebd { +namespace server { + +class HeartbeatServiceTest : public ::testing::Test { + protected: + void SetUp() override { + fileManager_ = std::make_shared(); + } + std::shared_ptr fileManager_; + const std::string kSockFile_ = "/tmp/heartbeat_service_test.sock"; +}; + +TEST_F(HeartbeatServiceTest, KeepAlive) { + // 启动server + brpc::Server server; + NebdHeartbeatServiceImpl heartbeatService(fileManager_); + ASSERT_EQ(0, server.AddService(&heartbeatService, + brpc::SERVER_DOESNT_OWN_SERVICE)); + brpc::ServerOptions option; + option.idle_timeout_sec = -1; + ASSERT_EQ(0, server.StartAtSockFile(kSockFile_.c_str(), &option)); + + nebd::client::HeartbeatRequest request; + nebd::client::HeartbeatResponse response; + for (int i = 0; i < 3; ++i) { + auto* info = request.add_info(); + info->set_fd(i); + info->set_name("/test/file" + std::to_string(i)); + } + brpc::Channel channel; + ASSERT_EQ(0, channel.InitWithSockFile(kSockFile_.c_str(), nullptr)); + nebd::client::NebdHeartbeatService_Stub stub(&channel); + brpc::Controller cntl; + + // 正常情况 + EXPECT_CALL(*fileManager_, UpdateFileTimestamp(_)) + .Times(3) + .WillRepeatedly(Return(0)); + stub.KeepAlive(&cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(nebd::client::RetCode::kOK, response.retcode()); + + // 有文件更新时间戳失败 + EXPECT_CALL(*fileManager_, UpdateFileTimestamp(_)) + .Times(3) + .WillOnce(Return(-1)) + .WillRepeatedly(Return(0)); + cntl.Reset(); + stub.KeepAlive(&cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(nebd::client::RetCode::kNoOK, response.retcode()); + + // 停止server + server.Stop(0); + server.Join(); +} +} // namespace server +} // namespace nebd + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + From 45c68565cb1001cb3950cb6e94fc9c6f93120ea0 Mon Sep 17 00:00:00 2001 From: charisu Date: Mon, 20 Jan 2020 15:57:47 +0800 Subject: [PATCH 23/79] implement metafilemanager Change-Id: Icca22f12e43d477e21f813416af212a197c0a5b5 --- src/common/posix_wrapper.cpp | 37 ++++ src/common/posix_wrapper.h | 34 ++++ src/part2/CMakeLists.txt | 4 +- src/part2/metafile_manager.cpp | 241 ++++++++++++++++++++++++-- src/part2/metafile_manager.h | 85 ++++++++- src/part2/request_executor.cpp | 16 ++ src/part2/request_executor.h | 8 + tests/common/CMakeLists.txt | 1 + tests/common/posix_wrapper_test.cpp | 29 ++++ tests/part2/CMakeLists.txt | 14 +- tests/part2/metafile_manager_test.cpp | 189 ++++++++++++++++++++ tests/part2/mock_posix_wrapper.h | 30 ++++ 12 files changed, 675 insertions(+), 13 deletions(-) create mode 100644 src/common/posix_wrapper.cpp create mode 100644 src/common/posix_wrapper.h create mode 100644 tests/common/posix_wrapper_test.cpp create mode 100644 tests/part2/metafile_manager_test.cpp create mode 100644 tests/part2/mock_posix_wrapper.h diff --git a/src/common/posix_wrapper.cpp b/src/common/posix_wrapper.cpp new file mode 100644 index 0000000000..c32657969e --- /dev/null +++ b/src/common/posix_wrapper.cpp @@ -0,0 +1,37 @@ +/* + * Project: nebd + * Created Date: 2020-01-21 + * Author: charisu + * Copyright (c) 2020 netease + */ + +#include "src/common/posix_wrapper.h" + +namespace nebd { +namespace common { + +int PosixWrapper::open(const char *pathname, int flags, mode_t mode) { + return ::open(pathname, flags, mode); +} + +int PosixWrapper::close(int fd) { + return ::close(fd); +} + +int PosixWrapper::remove(const char *pathname) { + return ::remove(pathname); +} + +int PosixWrapper::rename(const char *oldPath, const char *newPath) { + return ::rename(oldPath, newPath); +} + +ssize_t PosixWrapper::pwrite(int fd, + const void *buf, + size_t count, + off_t offset) { + return ::pwrite(fd, buf, count, offset); +} + +} // namespace common +} // namespace nebd diff --git a/src/common/posix_wrapper.h b/src/common/posix_wrapper.h new file mode 100644 index 0000000000..dbc1d7f8c4 --- /dev/null +++ b/src/common/posix_wrapper.h @@ -0,0 +1,34 @@ +/* + * Project: nebd + * Created Date: 2020-01-21 + * Author: charisu + * Copyright (c) 2020 netease + */ + +#ifndef SRC_COMMON_POSIX_WRAPPER_H_ +#define SRC_COMMON_POSIX_WRAPPER_H_ + +#include +#include +#include +#include +#include + +namespace nebd { +namespace common { +class PosixWrapper { + public: + PosixWrapper() {} + virtual ~PosixWrapper() {} + virtual int open(const char *pathname, int flags, mode_t mode); + virtual int close(int fd); + virtual int remove(const char *pathname); + virtual int rename(const char *oldpath, const char *newpath); + virtual ssize_t pwrite(int fd, + const void *buf, + size_t count, + off_t offset); +}; +} // namespace common +} // namespace nebd +#endif // SRC_COMMON_POSIX_WRAPPER_H_ diff --git a/src/part2/CMakeLists.txt b/src/part2/CMakeLists.txt index a3b7a33c66..bba3f5471a 100644 --- a/src/part2/CMakeLists.txt +++ b/src/part2/CMakeLists.txt @@ -1,6 +1,7 @@ # 3rd party header include_directories(${Boost_INCLUDE_DIRS}) include_directories(${Libconfig_INCLUDE_DIRS}) +include_directories("${Jsoncpp_INCLUDE_DIRS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \ -std=c++0x -pipe -W -Wall -Wno-unused-parameter -fPIC \ @@ -32,7 +33,8 @@ set(PART2_LINK rt snappy ssl - z) + z + jsoncpp_lib) add_executable(nebd-server ${PART2_SRC} ${PROTO_SRCS} ${PROTO_HDRS}) diff --git a/src/part2/metafile_manager.cpp b/src/part2/metafile_manager.cpp index b18aa283a1..1481527083 100644 --- a/src/part2/metafile_manager.cpp +++ b/src/part2/metafile_manager.cpp @@ -1,29 +1,250 @@ +/* + * Project: nebd + * Created Date: 2020-01-19 + * Author: charisu + * Copyright (c) 2020 netease + */ + +#include + #include "src/part2/metafile_manager.h" +#include "src/part2/util.h" +#include "src/part2/request_executor.h" namespace nebd { namespace server { -NebdMetaFileManager::NebdMetaFileManager(const std::string& metaFilePath) - : metaFilePath_(metaFilePath) { - // TODO -} +NebdMetaFileManager::NebdMetaFileManager(const std::string& metaFilePath, + std::shared_ptr wrapper, + std::shared_ptr parser) + : metaFilePath_(metaFilePath), wrapper_(wrapper), + parser_(parser), loaded_(false) {} NebdMetaFileManager::~NebdMetaFileManager() {} int NebdMetaFileManager::RemoveFileRecord(const std::string& fileName) { - // TODO - return 0; + std::unique_lock lock(mtx_); + std::vector fileRecords; + if (ListFileRecordUnlocked(&fileRecords) != 0) { + LOG(ERROR) << "List file record fail"; + return -1; + } + + // 删除fileName对应的元素 + bool exist = false; + for (auto it = fileRecords.begin(); it != fileRecords.end();) { + if ((*it)->fileName == fileName) { + exist = true; + it = fileRecords.erase(it); + break; + } else { + ++it; + } + } + + // 卷不存在返回0，不更新文件 + if (!exist) { + LOG(INFO) << "File " << fileName << " not found"; + return 0; + } + return UpdateMetaFile(fileRecords); } int NebdMetaFileManager::UpdateFileRecord(const NebdFileRecordPtr& fileRecord) { - // TODO + std::unique_lock lock(mtx_); + if (!fileRecord) { + LOG(ERROR) << "The argument is a null pointer!"; + return -1; + } + + std::vector fileRecords; + if (ListFileRecordUnlocked(&fileRecords) != 0) { + LOG(ERROR) << "List file record fail"; + return -1; + } + + bool exist = false; + for (const auto& record : fileRecords) { + // 如果文件已存在，则更新fd和addtion + if (record->fileName == fileRecord->fileName) { + // 如果fd和addition都没有变化的话直接return + if (RecordsEqual(record, fileRecord)) { + return 0; + } + record->fd = fileRecord->fd; + record->fileInstance = fileRecord->fileInstance; + exist = true; + break; + } + } + + // 不存在的话push一个进去 + if (!exist) { + fileRecords.emplace_back(fileRecord); + } + return UpdateMetaFile(fileRecords); +} + +int NebdMetaFileManager::UpdateMetaFile( + const std::vector& fileRecords) { + // 构建json + Json::Value volumes; + for (const auto& record : fileRecords) { + Json::Value volume; + volume[kFileName] = record->fileName; + volume[kFd] = record->fd; + if (record->fileInstance) { + for (const auto item : record->fileInstance->addition) { + volume[item.first] = item.second; + } + } + volumes.append(volume); + } + Json::Value root; + root[kVolumes] = volumes; + + int res = AtomicWriteFile(root); + if (res != 0) { + LOG(ERROR) << "AtomicWriteFile fail"; + return -1; + } + metaDataJson_ = root; + return 0; +} + +int NebdMetaFileManager::AtomicWriteFile(const Json::Value& root) { + // 写入tmp文件 + std::string tmpFilePath = metaFilePath_ + ".tmp"; + int fd = wrapper_->open(tmpFilePath.c_str(), O_CREAT|O_RDWR, 0644); + // open文件失败 + if (fd <= 0) { + LOG(ERROR) << "Open tmp file " << tmpFilePath << " fail"; + return -1; + } + // 写入 + std::string jsonString = root.toStyledString(); + int writeSize = wrapper_->pwrite(fd, jsonString.c_str(), + jsonString.size(), 0); + wrapper_->close(fd); + if (writeSize != jsonString.size()) { + LOG(ERROR) << "Write tmp file " << tmpFilePath << " fail"; + return -1; + } + + // 重命名 + int res = wrapper_->rename(tmpFilePath.c_str(), metaFilePath_.c_str()); + if (res != 0) { + LOG(ERROR) << "rename file " << tmpFilePath << " to " + << metaFilePath_ << " fail"; + return -1; + } return 0; } -int NebdMetaFileManager::ListFileRecord(std::vector* fileRecords) { - // TODO +int NebdMetaFileManager::ListFileRecord( + std::vector* fileRecords) { + fileRecords->clear(); + std::unique_lock lock(mtx_); + return ListFileRecordUnlocked(fileRecords); +} + +int NebdMetaFileManager::ListFileRecordUnlocked( + std::vector* fileRecords) { + // 判断是否从文件加载过 + if (loaded_) { + return parser_->Parse(metaDataJson_, fileRecords); + } + return ListFileRecordFromFile(fileRecords); +} + +int NebdMetaFileManager::ListFileRecordFromFile( + std::vector* fileRecords) { + std::ifstream in(metaFilePath_, std::ios::binary); + if (!in) { + // 这里不应该返回错误，第一次初始化的时候文件可能还未创建 + LOG(WARNING) << "File not exist: " << metaFilePath_; + return 0; + } + + Json::CharReaderBuilder reader; + Json::Value root; + JSONCPP_STRING errs; + bool ok = Json::parseFromStream(reader, in, &root, &errs); + in.close(); + if (!ok) { + LOG(ERROR) << "Parse meta file " << metaFilePath_ + << " fail: " << errs; + return -1; + } + + int res = parser_->Parse(root, fileRecords); + if (res != 0) { + LOG(ERROR) << "ConvertJsonToFileRecord fail"; + return -1; + } + metaDataJson_ = root; + loaded_ = true; + return 0; +} + +bool NebdMetaFileManager::RecordsEqual(const NebdFileRecordPtr& record1, + const NebdFileRecordPtr& record2) { + return record1->fileName == record2->fileName + && record1->fd == record2->fd + && record1->fileInstance->addition == record2->fileInstance->addition; +} + +int NebdMetaFileParser::Parse(const Json::Value& root, + std::vector* fileRecords) { + // 没有volume字段 + const auto& volumes = root[kVolumes]; + if (volumes.isNull()) { + LOG(ERROR) << "No volumes in json: " << root; + return -1; + } + + for (const auto& volume : volumes) { + std::string fileName; + int fd; + auto record = std::make_shared(); + + if (volume[kFileName].isNull()) { + LOG(ERROR) << "Parse json: " << root + << " fail, no filename"; + return -1; + } else { + record->fileName = volume[kFileName].asString(); + } + + if (volume[kFd].isNull()) { + LOG(ERROR) << "Parse json: " << root + << " fail, no fd"; + return -1; + } else { + record->fd = volume[kFd].asInt(); + } + + auto fileType = GetFileType(record->fileName); + record->fileInstance = NebdFileInstanceFactory::GetInstance(fileType); + if (!record->fileInstance) { + LOG(ERROR) << "Unknown file type, filename: " << record->fileName; + return -1; + } + + // 除了filename和fd的部分统一放到fileinstance的addition里面 + Json::Value::Members mem = volume.getMemberNames(); + AdditionType addition; + for (auto iter = mem.begin(); iter != mem.end(); iter++) { + if (*iter == kFileName || *iter == kFd) { + continue; + } + record->fileInstance->addition.emplace(*iter, + volume[*iter].asString()); + } + fileRecords->emplace_back(record); + } return 0; } } // namespace server -} // namespace nebd \ No newline at end of file +} // namespace nebd diff --git a/src/part2/metafile_manager.h b/src/part2/metafile_manager.h index 3dbbcbd93f..3ef222cf14 100644 --- a/src/part2/metafile_manager.h +++ b/src/part2/metafile_manager.h @@ -1,25 +1,108 @@ +/* + * Project: nebd + * Created Date: 2020-01-19 + * Author: charisu + * Copyright (c) 2020 netease + */ + #ifndef SRC_PART2_METAFILE_MANAGER_H_ #define SRC_PART2_METAFILE_MANAGER_H_ +#include #include #include +#include +#include // NOLINT +#include // NOLINT +#include "src/common/posix_wrapper.h" #include "src/part2/define.h" namespace nebd { namespace server { +using nebd::common::PosixWrapper; + +const char kVolumes[] = "volumes"; +const char kFileName[] = "filename"; +const char kFd[] = "fd"; + +class NebdMetaFileParser { + public: + int Parse(const Json::Value& root, + std::vector* fileRecords); +}; + class NebdMetaFileManager { public: - explicit NebdMetaFileManager(const std::string& metaFilePath); + NebdMetaFileManager(const std::string& metaFilePath, + std::shared_ptr wrapper = + std::make_shared(), + std::shared_ptr parser = + std::make_shared()); virtual ~NebdMetaFileManager(); + /** + * @brief 删除文件记录 + * + * @param fileName 要删除的文件名 + * + * @return 成功返回0，失败返回-1 + * + */ virtual int RemoveFileRecord(const std::string& fileName); + + /** + * @brief 更新文件记录 + * + * @param fileRecord 要更新的文件记录 + * + * @return 成功返回0，失败返回-1 + * + */ virtual int UpdateFileRecord(const NebdFileRecordPtr& fileRecord); + + /** + * @brief 列出文件记录 + * + * @param fileRecord 要更新的文件记录 + * + * @return 成功返回0，失败返回-1 + * + */ virtual int ListFileRecord(std::vector* fileRecords); private: + // 更新元数据文件 + int UpdateMetaFile( + const std::vector& fileRecords); + + // 原子写文件 + int AtomicWriteFile(const Json::Value& root); + + // 判断两条记录是否相等 + bool RecordsEqual(const NebdFileRecordPtr& record1, + const NebdFileRecordPtr& record2); + + // 内部List，无锁版本 + int ListFileRecordUnlocked(std::vector* fileRecords); + + // 从文件list fileRecord + int ListFileRecordFromFile(std::vector* fileRecords); + + private: + // 元数据文件路径 std::string metaFilePath_; + // 文件系统操作封装 + std::shared_ptr wrapper_; + // 用于解析Json格式的元数据 + std::shared_ptr parser_; + // 读写锁，保护json和文件 + std::mutex mtx_; + // 元数据的json + Json::Value metaDataJson_; + // 是否从文件加载过 + bool loaded_; }; } // namespace server diff --git a/src/part2/request_executor.cpp b/src/part2/request_executor.cpp index f9c8d13561..0b0ef7948b 100644 --- a/src/part2/request_executor.cpp +++ b/src/part2/request_executor.cpp @@ -33,6 +33,22 @@ NebdRequestExecutorFactory::GetExecutor(NebdFileType type) { return executor; } +NebdFileInstancePtr +NebdFileInstanceFactory::GetInstance(NebdFileType type) { + NebdFileInstancePtr instance = nullptr; + switch (type) { + case NebdFileType::CEPH: + instance = std::make_shared(); + break; + case NebdFileType::CURVE: + instance = std::make_shared(); + break; + default: + break; + } + return instance; +} + } // namespace server } // namespace nebd diff --git a/src/part2/request_executor.h b/src/part2/request_executor.h index e59fe54560..1acd13c614 100644 --- a/src/part2/request_executor.h +++ b/src/part2/request_executor.h @@ -57,6 +57,14 @@ class NebdRequestExecutorFactory { static NebdRequestExecutor* GetExecutor(NebdFileType type); }; +class NebdFileInstanceFactory { + public: + NebdFileInstanceFactory() = default; + ~NebdFileInstanceFactory() = default; + + static NebdFileInstancePtr GetInstance(NebdFileType type); +}; + // for test extern NebdRequestExecutor* g_test_executor; diff --git a/tests/common/CMakeLists.txt b/tests/common/CMakeLists.txt index 1e40dcfec9..eedb3912b2 100644 --- a/tests/common/CMakeLists.txt +++ b/tests/common/CMakeLists.txt @@ -20,3 +20,4 @@ target_link_libraries(common_test ${COMMON_TEST_LINK}) install(TARGETS common_test DESTINATION bin) add_test(NAME common_test COMMAND common_test) + diff --git a/tests/common/posix_wrapper_test.cpp b/tests/common/posix_wrapper_test.cpp new file mode 100644 index 0000000000..5f34ec8751 --- /dev/null +++ b/tests/common/posix_wrapper_test.cpp @@ -0,0 +1,29 @@ +/* + * Project: nebd + * Created Date: 2020-01-21 + * Author: charisu + * Copyright (c) 2020 netease + */ + +#include +#include "src/common/posix_wrapper.h" + +#define FILE_PATH1 "/tmp/wraptest1" +#define FILE_PATH2 "/tmp/wraptest2" + +namespace nebd { +namespace common { + +TEST(PosixWrapperTest, BasicTest) { + char buf[4096] = {0}; + PosixWrapper wrapper; + int fd = wrapper.open(FILE_PATH1, O_CREAT|O_RDWR, 0644); + ASSERT_GE(fd, 0); + ASSERT_EQ(4096, wrapper.pwrite(fd, buf, 4096, 0)); + ASSERT_EQ(0, wrapper.close(fd)); + ASSERT_EQ(0, wrapper.rename(FILE_PATH1, FILE_PATH2)); + ASSERT_EQ(0, wrapper.remove(FILE_PATH2)); +} + +} // namespace common +} // namespace nebd diff --git a/tests/part2/CMakeLists.txt b/tests/part2/CMakeLists.txt index 90ecefcff2..8169743ebd 100644 --- a/tests/part2/CMakeLists.txt +++ b/tests/part2/CMakeLists.txt @@ -1,6 +1,7 @@ include_directories(${Boost_INCLUDE_DIRS}) include_directories(${Libconfig_INCLUDE_DIRS}) include_directories(${CMock_INCLUDE_DIRS}) +include_directories("${Jsoncpp_INCLUDE_DIRS}") set(CPP_FLAGS "-DBRPC_WITH_GLOG=0 -DGFLAGS_NS=google") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CPP_FLAGS} -std=c++0x -DNDEBUG -O2 \ @@ -16,7 +17,8 @@ set(PART2_TEST_LINK set(PART2_MOCK_SRC mock_metafile_manager.h mock_request_executor.h - mock_file_manager.h) + mock_file_manager.h + mock_posix_wrapper.h) add_library(part2mock ${PART2_MOCK_SRC}) set_target_properties(part2mock PROPERTIES LINKER_LANGUAGE CXX) target_link_libraries(part2mock @@ -56,3 +58,13 @@ target_link_libraries(heartbeat_service_test ${PART2_TEST_LINK}) install(TARGETS heartbeat_service_test DESTINATION bin) add_test(NAME heartbeat_service_test COMMAND heartbeat_service_test) + +# metafile_manager_test +set(METAFILE_MANAGER_TEST_SRC metafile_manager_test.cpp) +add_executable(metafile_manager_test ${METAFILE_MANAGER_TEST_SRC}) +target_link_libraries(metafile_manager_test + part2mock + ${PART2_TEST_LINK}) +install(TARGETS metafile_manager_test DESTINATION bin) +add_test(NAME metafile_manager_test COMMAND metafile_manager_test) + diff --git a/tests/part2/metafile_manager_test.cpp b/tests/part2/metafile_manager_test.cpp new file mode 100644 index 0000000000..6559fb8daa --- /dev/null +++ b/tests/part2/metafile_manager_test.cpp @@ -0,0 +1,189 @@ +/* + * Project: nebd + * Created Date: 2020-01-19 + * Author: charisu + * Copyright (c) 2020 netease + */ + +#include +#include +#include + +#include "src/part2/metafile_manager.h" +#include "src/part2/request_executor_curve.h" +#include "tests/part2/mock_posix_wrapper.h" + +using ::testing::_; +using ::testing::Return; + +namespace nebd { +namespace server { + +const char metaPath[] = "/tmp/nebd-test-metafilemanager.meta"; + +class MetaFileManagerTest : public ::testing::Test { + protected: + void SetUp() override { + wrapper_ = std::make_shared(); + } + void TearDown() override { + unlink(metaPath); + std::string tmpPath = std::string(metaPath) + ".tmp"; + unlink(tmpPath.c_str()); + } + std::shared_ptr wrapper_; +}; + +TEST_F(MetaFileManagerTest, common) { + NebdMetaFileManager metaFileManager(metaPath); + std::vector records; + // 文件不存在 + ASSERT_EQ(0, metaFileManager.ListFileRecord(&records)); + ASSERT_TRUE(records.empty()); + + // 添加两条记录，ceph和curve各一个 + NebdFileRecordPtr fileRecord1 = std::make_shared(); + fileRecord1->fileName = "rbd:volume1"; + fileRecord1->fd = 111; + ASSERT_EQ(0, metaFileManager.UpdateFileRecord(fileRecord1)); + NebdFileRecordPtr fileRecord2 = std::make_shared(); + fileRecord2->fileName = "cbd:volume2"; + fileRecord2->fd = 222; + fileRecord2->fileInstance = std::make_shared(); + fileRecord2->fileInstance->addition["session"] = "test-session"; + ASSERT_EQ(0, metaFileManager.UpdateFileRecord(fileRecord2)); + fileRecord2->fd = 3333; + ASSERT_EQ(0, metaFileManager.UpdateFileRecord(fileRecord2)); + fileRecord2->fileInstance->addition["session"] = "test-session-2"; + ASSERT_EQ(0, metaFileManager.UpdateFileRecord(fileRecord2)); + // 重复更新 + ASSERT_EQ(0, metaFileManager.UpdateFileRecord(fileRecord2)); + + // listFileRecord + ASSERT_EQ(0, metaFileManager.ListFileRecord(&records)); + ASSERT_EQ(2, records.size()); + ASSERT_EQ(fileRecord1->fileName, records[0]->fileName); + ASSERT_EQ(fileRecord1->fd, records[0]->fd); + ASSERT_EQ(fileRecord2->fileName, records[1]->fileName); + ASSERT_EQ(fileRecord2->fd, records[1]->fd); + ASSERT_EQ(fileRecord2->fileInstance->addition, + records[1]->fileInstance->addition); + + // RemoveFileRecord + ASSERT_EQ(0, metaFileManager.RemoveFileRecord("cbd:volume2")); + ASSERT_EQ(0, metaFileManager.ListFileRecord(&records)); + ASSERT_EQ(1, records.size()); + ASSERT_EQ(fileRecord1->fileName, records[0]->fileName); + ASSERT_EQ(fileRecord1->fd, records[0]->fd); + // volume not exist + ASSERT_EQ(0, metaFileManager.RemoveFileRecord("cbd:volume123")); + ASSERT_EQ(0, metaFileManager.ListFileRecord(&records)); + ASSERT_EQ(1, records.size()); +} + +TEST_F(MetaFileManagerTest, error) { + NebdMetaFileManager metaFileManager(metaPath, wrapper_); + std::vector records; + NebdFileRecordPtr fileRecord = std::make_shared(); + fileRecord->fileName = "rbd:volume1"; + fileRecord->fd = 111; + + // open临时文件失败 + EXPECT_CALL(*wrapper_, open(_, _, _)) + .Times(1) + .WillOnce(Return(-1)); + ASSERT_EQ(-1, metaFileManager.UpdateFileRecord(fileRecord)); + + // 写入临时文件失败 + EXPECT_CALL(*wrapper_, open(_, _, _)) + .Times(1) + .WillOnce(Return(1)); + EXPECT_CALL(*wrapper_, pwrite(_, _, _, _)) + .Times(1) + .WillOnce(Return(0)); + ASSERT_EQ(-1, metaFileManager.UpdateFileRecord(fileRecord)); + + // rename失败 + EXPECT_CALL(*wrapper_, open(_, _, _)) + .Times(1) + .WillOnce(Return(1)); + EXPECT_CALL(*wrapper_, pwrite(_, _, _, _)) + .Times(1) + .WillOnce(Return(77)); + EXPECT_CALL(*wrapper_, rename(_, _)) + .Times(1) + .WillOnce(Return(-1)); + ASSERT_EQ(-1, metaFileManager.UpdateFileRecord(fileRecord)); + + // 解析失败 + std::ofstream out(metaPath); + out.close(); + ASSERT_EQ(-1, metaFileManager.ListFileRecord(&records)); + ASSERT_EQ(-1, metaFileManager.RemoveFileRecord("cbd:volume2")); + ASSERT_EQ(-1, metaFileManager.UpdateFileRecord(fileRecord)); + + // UpdataFileRecord的参数为空 + ASSERT_EQ(-1, metaFileManager.UpdateFileRecord(nullptr)); +} + +TEST(MetaFileParserTest, Parse) { + NebdMetaFileParser parser; + Json::Value root; + Json::Value volume; + Json::Value volumes; + std::vector records; + + // 正常情况 + volume[kFileName] = "rbd:volume1"; + volume[kFd] = 1; + volumes.append(volume); + volume[kFileName] = "cbd:volume2"; + volume[kFd] = 2; + root[kVolumes] = volumes; + ASSERT_EQ(0, parser.Parse(root, &records)); + + // 文件格式不正确 + root.clear(); + ASSERT_EQ(-1, parser.Parse(root, &records)); + + // 没有volume字段 + root["key"] = "value"; + ASSERT_EQ(-1, parser.Parse(root, &records)); + + // 记录中没有filename + volume.clear(); + volumes.clear(); + root.clear(); + volume[kFd] = 1234; + volumes.append(volume); + root[kVolumes] = volumes; + ASSERT_EQ(-1, parser.Parse(root, &records)); + + // 记录中没有fd + volume.clear(); + volumes.clear(); + root.clear(); + volume[kFileName] = "cbd:volume2"; + volumes.append(volume); + root[kVolumes] = volumes; + ASSERT_EQ(-1, parser.Parse(root, &records)); + + // 文件名格式不对 + volume.clear(); + volumes.clear(); + root.clear(); + volume[kFileName] = "volume2"; + volume[kFd] = 1234; + volumes.append(volume); + root[kVolumes] = volumes; + ASSERT_EQ(-1, parser.Parse(root, &records)); +} + +} // namespace server +} // namespace nebd + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + RUN_ALL_TESTS(); + return 0; +} diff --git a/tests/part2/mock_posix_wrapper.h b/tests/part2/mock_posix_wrapper.h new file mode 100644 index 0000000000..06c3f5070f --- /dev/null +++ b/tests/part2/mock_posix_wrapper.h @@ -0,0 +1,30 @@ +/* + * Project: nebd + * Created Date: 2020-01-21 + * Author: charisu + * Copyright (c) 2020 netease + */ + +#ifndef TESTS_PART2_MOCK_POSIX_WRAPPER_H_ +#define TESTS_PART2_MOCK_POSIX_WRAPPER_H_ + +#include +#include "src/common/posix_wrapper.h" + +namespace nebd { +namespace common { + +class MockPosixWrapper : public PosixWrapper { + public: + MOCK_METHOD3(open, int(const char *, int, mode_t)); + MOCK_METHOD1(close, int(int)); + MOCK_METHOD1(remove, int(const char *)); + MOCK_METHOD2(rename, int(const char *, const char *)); + MOCK_METHOD4(pwrite, ssize_t(int fd, const void *buf, + size_t count, off_t offset)); +}; + +} // namespace common +} // namespace nebd + +#endif // TESTS_PART2_MOCK_POSIX_WRAPPER_H_ From ba02a8ccf363b8fb02210bf6418a13335f0b36e2 Mon Sep 17 00:00:00 2001 From: lixiaocui1 Date: Fri, 17 Jan 2020 15:05:52 +0800 Subject: [PATCH 24/79] =?UTF-8?q?[part2]=20nebdserver=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: If38da9d4d680a147bcb018854097696874f023d7 --- etc/nebd/nebd-server.conf | 8 ++ src/part2/define.h | 5 ++ src/part2/file_manager.cpp | 10 +++ src/part2/file_manager.h | 1 + src/part2/main.cpp | 21 +++++ src/part2/nebd_server.cpp | 147 +++++++++++++++++++++++++++++-- src/part2/nebd_server.h | 68 +++++++++++++- tests/part2/CMakeLists.txt | 11 ++- tests/part2/nebd-server.err.conf | 5 ++ tests/part2/test_nebd_server.cpp | 49 +++++++++++ 10 files changed, 316 insertions(+), 9 deletions(-) create mode 100644 tests/part2/nebd-server.err.conf create mode 100644 tests/part2/test_nebd_server.cpp diff --git a/etc/nebd/nebd-server.conf b/etc/nebd/nebd-server.conf index e69de29bb2..079f3f7fa2 100644 --- a/etc/nebd/nebd-server.conf +++ b/etc/nebd/nebd-server.conf @@ -0,0 +1,8 @@ +#brpc server监听端口 +listen.address=/var/run/nebd.sock + +#元数据文件地址,包含文件名 +meta.file.path = ./nebdserver.meta + +#心跳超时时间 +heartbeat.timeout.sec=30 \ No newline at end of file diff --git a/src/part2/define.h b/src/part2/define.h index b4724e060b..1c375e3e86 100644 --- a/src/part2/define.h +++ b/src/part2/define.h @@ -108,6 +108,11 @@ struct NebdFileInfo { uint64_t num_objs; }; +// part2配置项 +const char LISTENADDRESS[] = "listen.address"; +const char METAFILEPATH[] = "meta.file.path"; +const char HEARTBEATTIMEOUTSEC[] = "heartbeat.timeout.sec"; + } // namespace server } // namespace nebd diff --git a/src/part2/file_manager.cpp b/src/part2/file_manager.cpp index f9cda70278..7df80ae6f3 100644 --- a/src/part2/file_manager.cpp +++ b/src/part2/file_manager.cpp @@ -89,6 +89,16 @@ int NebdFileManager::Load() { return 0; } +int NebdFileManager::Run() { + // TODO + return 0; +} + +int NebdFileManager::Fini() { + // TODO + return 0; +} + int NebdFileManager::UpdateFileTimestamp(int fd) { NebdFileRecordPtr fileRecord = fileRecordMap_.GetRecord(fd); if (fileRecord == nullptr) { diff --git a/src/part2/file_manager.h b/src/part2/file_manager.h index 7cb1762c99..75200e2d3f 100644 --- a/src/part2/file_manager.h +++ b/src/part2/file_manager.h @@ -25,6 +25,7 @@ #include "src/part2/file_record_map.h" #include "src/part2/request_executor_ceph.h" #include "src/part2/request_executor_curve.h" +#include "src/common/client.pb.h" namespace nebd { namespace server { diff --git a/src/part2/main.cpp b/src/part2/main.cpp index 89d4b1c688..21e1a24266 100644 --- a/src/part2/main.cpp +++ b/src/part2/main.cpp @@ -7,8 +7,29 @@ #include #include +#include +#include "src/part2/nebd_server.h" + #define BOOST_SPIRIT_THREADSAFE +DEFINE_string(confPath, "/etc/nebd/nebd.conf", "nebd conf path"); + int main(int argc, char* argv[]) { + // 解析参数 + google::ParseCommandLineFlags(&argc, &argv, false); + std::string confPath = FLAGS_confPath.c_str(); + + // 启动nebd server + auto server = std::make_shared<::nebd::server::NebdServer>(); + int initRes = server->Init(confPath); + if (initRes < 0) { + LOG(ERROR) << "init nebd server fail"; + return -1; + } + server->RunUntilAskedToQuit(); + + // 停止nebd server + server->Fini(); + return 0; } diff --git a/src/part2/nebd_server.cpp b/src/part2/nebd_server.cpp index 775acfcbff..e0bd22924b 100644 --- a/src/part2/nebd_server.cpp +++ b/src/part2/nebd_server.cpp @@ -1,23 +1,158 @@ +/* + * Project: nebd + * Created Date: 2020-01-16 + * Author: lixiaocui + * Copyright (c) 2020 netease + */ + +#include +#include #include "src/part2/nebd_server.h" +#include "src/part2/file_service.h" +#include "src/part2/heartbeat_service.h" namespace nebd { namespace server { +int NebdServer::Init(const std::string &confPath) { + if (isRunning_) { + LOG(WARNING) << "NebdServer is inited"; + return -1; + } + + bool loadConf = LoadConfFromFile(confPath); + if (false == loadConf) { + LOG(ERROR) << "NebdServer load config from file fail"; + return -1; + } + + bool initAddressOk = conf_.GetStringValue(LISTENADDRESS, &listenAddress_); + if (false == initAddressOk) { + LOG(ERROR) << "NebdServer init socket file address fail"; + return -1; + } + + bool initFileManagerOk = InitFileManager(); + if (false == initFileManagerOk) { + LOG(ERROR) << "NebdServer init fileManager fail"; + return -1; + } -int NebdServer::Init() { - // TODO + LOG(INFO) << "NebdServer init ok"; return 0; } -int Run() { - // TODO +int NebdServer::RunUntilAskedToQuit() { + if (false == StartServer()) { + LOG(INFO) << "start server fail"; + return -1; + } + return 0; } -int Fini() { - // TODO +int NebdServer::Fini() { + if (isRunning_) { + brpc::AskToQuit(); + } + + if (fileManager_ != nullptr) { + fileManager_->Fini(); + } return 0; } +bool NebdServer::LoadConfFromFile(const std::string &confPath) { + conf_.SetConfigPath(confPath); + return conf_.LoadConfig(); +} + +bool NebdServer::InitFileManager() { + fileManager_ = std::make_shared(); + + NebdFileManagerOption opt; + bool initOptionOk = InitNebdFileManagerOption(&opt); + if (false == initOptionOk) { + LOG(ERROR) << "NebdServer init NebdFileManagerOption fail"; + return false; + } + + int initRes = fileManager_->Init(opt); + if (0 != initRes) { + LOG(ERROR) << "Init nebd file server fail"; + return false; + } + + int runRes = fileManager_->Run(); + if (0 != runRes) { + LOG(ERROR) << "nebd file manager run fail"; + return false; + } + + return true; +} + +bool NebdServer::InitNebdFileManagerOption(NebdFileManagerOption *opt) { + bool getOk = conf_.GetUInt32Value( + HEARTBEATTIMEOUTSEC, &opt->heartbeatTimeoutS); + if (false == getOk) { + LOG(ERROR) << "NebdServer get heartbeat.timeout.sec fail"; + return false; + } + + opt->metaFileManager = InitMetaFileManager(); + if (nullptr == opt->metaFileManager) { + LOG(ERROR) << "NebdServer init meta file manager fail"; + return false; + } + + return true; +} + +MetaFileManagerPtr NebdServer::InitMetaFileManager() { + std::string metaFilePath; + bool getOk = conf_.GetStringValue(METAFILEPATH, &metaFilePath); + if (false == getOk) { + return nullptr; + } + + return std::make_shared(metaFilePath); +} + +bool NebdServer::StartServer() { + // add service + NebdFileServiceImpl fileService(fileManager_); + int addFileServiceRes = server_.AddService( + &fileService, brpc::SERVER_DOESNT_OWN_SERVICE); + if (0 != addFileServiceRes) { + LOG(ERROR) << "NebdServer add file service fail"; + return false; + } + + NebdHeartbeatServiceImpl heartbeatService(fileManager_); + addFileServiceRes = server_.AddService( + &heartbeatService, brpc::SERVER_DOESNT_OWN_SERVICE); + if (0 != addFileServiceRes) { + LOG(ERROR) << "NebdServer add heartbeat service fail"; + return false; + } + + // start brcp server + brpc::ServerOptions option; + option.idle_timeout_sec = -1; + int startBrpcServerRes = server_.Start(listenAddress_.c_str(), &option); + if (0 != startBrpcServerRes) { + LOG(ERROR) << "NebdServer start brpc server fail, res=" + << startBrpcServerRes; + return false; + } + + isRunning_ = true; + server_.RunUntilAskedToQuit(); + + isRunning_ = false; + return true; +} + } // namespace server } // namespace nebd diff --git a/src/part2/nebd_server.h b/src/part2/nebd_server.h index 060da66b09..b5a1fde887 100644 --- a/src/part2/nebd_server.h +++ b/src/part2/nebd_server.h @@ -1,16 +1,80 @@ +/* + * Project: nebd + * Created Date: 2020-01-16 + * Author: lixiaocui + * Copyright (c) 2020 netease + */ + #ifndef SRC_PART2_NEBD_SERVER_H_ #define SRC_PART2_NEBD_SERVER_H_ +#include +#include +#include +#include "src/common/configuration.h" +#include "src/part2/file_manager.h" + namespace nebd { namespace server { +using ::nebd::common::Configuration; + class NebdServer { public: NebdServer() {} virtual ~NebdServer() {} - int Init(); - int Run(); + + int Init(const std::string &confPath); + + int RunUntilAskedToQuit(); + int Fini(); + + private: + /** + * @brief 从配置文件加载配置项 + * @param[in] confPath 配置文件路径 + * @return false-加载配置文件失败 true-加载配置文件成功 + */ + bool LoadConfFromFile(const std::string &confPath); + + /** + * @brief 初始化NebdFileManager + * @return false-初始化失败 true-初始化成功 + */ + bool InitFileManager(); + + /** + * @brief 初始化NebdFileManagerOption + * @param[out] opt + * @return false-初始化失败 true-初始化成功 + */ + bool InitNebdFileManagerOption(NebdFileManagerOption *opt); + + /** + * @brief 初始化NebdMetaFileManager + * @return nullptr-初始化不成功 否则表示初始化成功 + */ + MetaFileManagerPtr InitMetaFileManager(); + + /** + * @brief 启动brpc service + * @return false-启动service失败 true-启动service成功 + */ + bool StartServer(); + + private: + // 配置项 + Configuration conf_; + // NebdServer监听地址 + std::string listenAddress_; + // NebdServer是否处于running状态 + bool isRunning_ = false; + + // brpc server + brpc::Server server_; + // 用于接受和处理client端的各种请求 + std::shared_ptr fileManager_; }; } // namespace server diff --git a/tests/part2/CMakeLists.txt b/tests/part2/CMakeLists.txt index 8169743ebd..4757a1e345 100644 --- a/tests/part2/CMakeLists.txt +++ b/tests/part2/CMakeLists.txt @@ -11,7 +11,8 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CPP_FLAGS} -std=c++0x -DNDEBUG -O2 \ set(PART2_TEST_LINK nebdserver gtest - gmock) + gmock + gflags) # libpart2mock.so set(PART2_MOCK_SRC @@ -68,3 +69,11 @@ target_link_libraries(metafile_manager_test install(TARGETS metafile_manager_test DESTINATION bin) add_test(NAME metafile_manager_test COMMAND metafile_manager_test) +# test_nebd_server +set(TEST_NEBDSERVER_SRC test_nebd_server.cpp) +add_executable(test_nebd_server ${TEST_NEBDSERVER_SRC}) +target_link_libraries(test_nebd_server + part2mock + ${PART2_TEST_LINK}) +install(TARGETS test_nebd_server DESTINATION bin) +add_test(NAME test_nebd_server COMMAND test_nebd_server) diff --git a/tests/part2/nebd-server.err.conf b/tests/part2/nebd-server.err.conf new file mode 100644 index 0000000000..ec992a33b5 --- /dev/null +++ b/tests/part2/nebd-server.err.conf @@ -0,0 +1,5 @@ +#元数据文件地址 +meta.file.path = ./ + +#心跳超时时间 +heartbeat.timeout.sec=30 \ No newline at end of file diff --git a/tests/part2/test_nebd_server.cpp b/tests/part2/test_nebd_server.cpp new file mode 100644 index 0000000000..0c6d4a18bd --- /dev/null +++ b/tests/part2/test_nebd_server.cpp @@ -0,0 +1,49 @@ +/* + * Project: nebd + * Created Date: 2020-01-16 + * Author: lixiaocui + * Copyright (c) 2020 netease + */ + +#include +#include "src/part2/nebd_server.h" + +namespace nebd { +namespace server { + +TEST(TestNebdServer, test_Init_Run_Fini) { + NebdServer server; + + std::string confPath; + // 1. 配置文件不存在, init失败 + confPath = "./nebd.conf"; + ASSERT_EQ(-1, server.Init(confPath)); + + // 2. 配置文件存在, 监听端口未设置 + confPath = "./nebd-server.err.conf"; + ASSERT_EQ(-1, server.Init(confPath)); + + // 3. 初始化成功 + confPath = "./etc/nebd/nebd-server.conf"; + ASSERT_EQ(0, server.Init(confPath)); + + // 4. run成功 + std::thread nebdServerThread(&NebdServer::RunUntilAskedToQuit, &server); + sleep(1); + + // 5. stop成功 + ASSERT_EQ(0, server.Fini()); + + // 6. 再次stop不会重复释放资源 + ASSERT_EQ(0, server.Fini()); + nebdServerThread.join(); +} + +} // namespace server +} // namespace nebd + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + RUN_ALL_TESTS(); + return 0; +} From 6d1ef198ebe9da069eb718b738ebee6bcb399299 Mon Sep 17 00:00:00 2001 From: lixiaocui1 Date: Mon, 20 Jan 2020 14:03:42 +0800 Subject: [PATCH 25/79] =?UTF-8?q?[part2]=20ioexcutor=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I2f470079ab8735183b844d9951dd95a418dd9cd1 --- src/common/interrupt_sleep.h | 2 +- src/common/rw_lock.h | 2 +- src/part1/nebd_client.cpp | 2 +- src/part2/CMakeLists.txt | 5 +- src/part2/define.h | 3 + src/part2/file_service.h | 2 +- src/part2/nebd_server.cpp | 2 +- src/part2/request_executor.h | 2 +- src/part2/request_executor_ceph.cpp | 5 - src/part2/request_executor_ceph.h | 1 - src/part2/request_executor_curve.cpp | 241 ++++++- src/part2/request_executor_curve.h | 77 ++- tests/part1/nebd_test.cpp | 659 ++++++++++++++++++++ tests/part2/CMakeLists.txt | 13 +- tests/part2/mock_curve_client.h | 36 ++ tests/part2/test_request_executor_curve.cpp | 368 +++++++++++ 16 files changed, 1382 insertions(+), 38 deletions(-) create mode 100644 tests/part1/nebd_test.cpp create mode 100644 tests/part2/mock_curve_client.h create mode 100644 tests/part2/test_request_executor_curve.cpp diff --git a/src/common/interrupt_sleep.h b/src/common/interrupt_sleep.h index fc3deb6d59..8493825b60 100644 --- a/src/common/interrupt_sleep.h +++ b/src/common/interrupt_sleep.h @@ -9,7 +9,7 @@ #define SRC_COMMON_INTERRUPT_SLEEP_H_ #include -#include +#include #include // NOLINT namespace nebd { diff --git a/src/common/rw_lock.h b/src/common/rw_lock.h index 8e372999ae..21ea645bee 100644 --- a/src/common/rw_lock.h +++ b/src/common/rw_lock.h @@ -10,7 +10,7 @@ #include #include -#include +#include #include "src/common/uncopyable.h" diff --git a/src/part1/nebd_client.cpp b/src/part1/nebd_client.cpp index 3ea455fe6e..636092e030 100644 --- a/src/part1/nebd_client.cpp +++ b/src/part1/nebd_client.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/part2/CMakeLists.txt b/src/part2/CMakeLists.txt index bba3f5471a..aacb864149 100644 --- a/src/part2/CMakeLists.txt +++ b/src/part2/CMakeLists.txt @@ -35,7 +35,10 @@ set(PART2_LINK ssl z jsoncpp_lib) - + curve + bthread + glog + ) add_executable(nebd-server ${PART2_SRC} ${PROTO_SRCS} ${PROTO_HDRS}) target_link_libraries(nebd-server ${PART2_LINK} ${EXTRALIBS}) diff --git a/src/part2/define.h b/src/part2/define.h index 1c375e3e86..c0b4ea5dee 100644 --- a/src/part2/define.h +++ b/src/part2/define.h @@ -27,6 +27,9 @@ const char CURVE_PREFIX[] = "cbd"; const char CEPH_PREFIX[] = "rbd"; const char TEST_PREFIX[] = "test"; +const char CURVE_PREFIX[] = "cbd"; +const char CEPH_PREFIX[] = "rbd"; + // nebd异步请求的类型 enum class LIBAIO_OP { LIBAIO_OP_READ, diff --git a/src/part2/file_service.h b/src/part2/file_service.h index 8e44a14b3f..4203b63b23 100644 --- a/src/part2/file_service.h +++ b/src/part2/file_service.h @@ -9,7 +9,7 @@ #define SRC_PART2_FILE_SERVICE_H_ #include -#include +#include #include #include #include diff --git a/src/part2/nebd_server.cpp b/src/part2/nebd_server.cpp index e0bd22924b..6960a75524 100644 --- a/src/part2/nebd_server.cpp +++ b/src/part2/nebd_server.cpp @@ -5,7 +5,7 @@ * Copyright (c) 2020 netease */ -#include +#include #include #include "src/part2/nebd_server.h" #include "src/part2/file_service.h" diff --git a/src/part2/request_executor.h b/src/part2/request_executor.h index 1acd13c614..dee936486d 100644 --- a/src/part2/request_executor.h +++ b/src/part2/request_executor.h @@ -10,6 +10,7 @@ #include #include +#include #include "src/part2/define.h" namespace nebd { @@ -41,7 +42,6 @@ class NebdRequestExecutor { virtual int Close(NebdFileInstance* fd) = 0; virtual int Extend(NebdFileInstance* fd, int64_t newsize) = 0; virtual int GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) = 0; - virtual int StatFile(NebdFileInstance* fd, NebdFileInfo* fileInfo) = 0; virtual int Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) = 0; virtual int AioRead(NebdFileInstance* fd, NebdServerAioContext* aioctx) = 0; virtual int AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) = 0; // NOLINT diff --git a/src/part2/request_executor_ceph.cpp b/src/part2/request_executor_ceph.cpp index 99093dd706..23880f953c 100644 --- a/src/part2/request_executor_ceph.cpp +++ b/src/part2/request_executor_ceph.cpp @@ -26,11 +26,6 @@ int CephRequestExecutor::Extend(NebdFileInstance* fd, int64_t newsize) { return 0; } -int CephRequestExecutor::StatFile(NebdFileInstance* fd, NebdFileInfo* fileInfo) { - // TODO - return 0; -} - int CephRequestExecutor::Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) { // TODO return 0; diff --git a/src/part2/request_executor_ceph.h b/src/part2/request_executor_ceph.h index 564a7f773c..5e70871f76 100644 --- a/src/part2/request_executor_ceph.h +++ b/src/part2/request_executor_ceph.h @@ -27,7 +27,6 @@ class CephRequestExecutor : public NebdRequestExecutor { int Close(NebdFileInstance* fd) override; int Extend(NebdFileInstance* fd, int64_t newsize) override; int GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) override; - int StatFile(NebdFileInstance* fd, NebdFileInfo* fileInfo) override; int Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; int AioRead(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; int AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; diff --git a/src/part2/request_executor_curve.cpp b/src/part2/request_executor_curve.cpp index 7eb205ea07..1cca0fb2a9 100644 --- a/src/part2/request_executor_curve.cpp +++ b/src/part2/request_executor_curve.cpp @@ -1,67 +1,270 @@ +/* + * Project: nebd + * Created Date: 2020-02-03 + * Author: lixiaocui + * Copyright (c) 2020 netease + */ + +#include #include "src/part2/request_executor_curve.h" namespace nebd { namespace server { +using ::curve::client::UserInfo_t; + +std::string FileNameParser::Parse(const std::string& fileName) { + int beginPos = fileName.find_first_of("/"); + if (beginPos == std::string::npos) { + LOG(ERROR) << "error format fileName: " << fileName; + return ""; + } + beginPos += 1; + + int endPos = fileName.find_last_of(":"); + if (endPos == std::string::npos) { + LOG(ERROR) << "error format fileName: " << fileName; + return ""; + } + + if (beginPos >= endPos) { + LOG(ERROR) << "error format fileName: " << fileName; + return ""; + } + + int length = endPos - beginPos; + if (length <= 2) { + LOG(ERROR) << "error format fileName: " << fileName; + return ""; + } + + return fileName.substr(beginPos, length); +} + +void CurveRequestExecutor::Init(const std::shared_ptr &client) { + client_ = client; +} + std::shared_ptr CurveRequestExecutor::Open(const std::string& filename) { - // TODO + std::string curveFileName = FileNameParser::Parse(filename); + if (curveFileName.empty()) { + return nullptr; + } + + std::string sessionId; + int fd = client_->Open(curveFileName, &sessionId); + + if (fd >= 0) { + auto curveFileInstance = std::make_shared(); + curveFileInstance->fd = fd; + curveFileInstance->fileName = curveFileName; + curveFileInstance->addition["session"] = sessionId; + return curveFileInstance; + } + return nullptr; } std::shared_ptr CurveRequestExecutor::Reopen(const std::string& filename, AdditionType addtion) { - // TODO + std::string curveFileName = FileNameParser::Parse(filename); + if (curveFileName.empty()) { + return nullptr; + } + + std::string newSessionId; + std::string oldSessionId = addtion["session"]; + int fd = client_->ReOpen(curveFileName, oldSessionId, &newSessionId); + if (fd >= 0) { + auto curveFileInstance = std::make_shared(); + curveFileInstance->fd = fd; + curveFileInstance->fileName = curveFileName; + curveFileInstance->addition["session"] = newSessionId; + return curveFileInstance; + } + return nullptr; } int CurveRequestExecutor::Close(NebdFileInstance* fd) { - // TODO + int curveFd = GetCurveFdFromNebdFileInstance(fd); + if (curveFd < 0) { + return -1; + } + + int res = client_->Close(curveFd); + if (res != LIBCURVE_ERROR::OK) { + return -1; + } + return 0; } int CurveRequestExecutor::Extend(NebdFileInstance* fd, int64_t newsize) { - // TODO + std::string fileName = GetFileNameFromNebdFileInstance(fd); + if (fileName.empty()) { + return -1; + } + + int res = client_->Extend(fileName, newsize); + if (res != LIBCURVE_ERROR::OK) { + return -1; + } + return 0; } -int CurveRequestExecutor::StatFile(NebdFileInstance* fd, NebdFileInfo* fileInfo) { - // TODO +int CurveRequestExecutor::GetInfo( + NebdFileInstance* fd, NebdFileInfo* fileInfo) { + std::string fileName = GetFileNameFromNebdFileInstance(fd); + if (fileName.empty()) { + return -1; + } + + int64_t size = client_->StatFile(fileName); + if (size < 0) { + return -1; + } + + fileInfo->size = size; return 0; } -int CurveRequestExecutor::Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) { - // TODO +int CurveRequestExecutor::Discard( + NebdFileInstance* fd, NebdServerAioContext* aioctx) { + return 0; } -int CurveRequestExecutor::AioRead(NebdFileInstance* fd, NebdServerAioContext* aioctx) { - // TODO +int CurveRequestExecutor::AioRead( + NebdFileInstance* fd, NebdServerAioContext* aioctx) { + int curveFd = GetCurveFdFromNebdFileInstance(fd); + if (curveFd < 0) { + return -1; + } + + CurveAioCombineContext *curveCombineCtx = new CurveAioCombineContext(); + curveCombineCtx->nebdCtx = aioctx; + curveCombineCtx->curveCtx = new CurveAioContext(); + int ret = FromNebdCtxToCurveCtx(aioctx, curveCombineCtx->curveCtx); + if (ret < 0) { + delete curveCombineCtx->curveCtx; + delete curveCombineCtx; + return -1; + } + + ret = client_->AioRead(curveFd, curveCombineCtx->curveCtx); + if (ret != LIBCURVE_ERROR::OK) { + delete curveCombineCtx->curveCtx; + delete curveCombineCtx; + return -1; + } + return 0; } -int CurveRequestExecutor::AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) { - // TODO +int CurveRequestExecutor::AioWrite( + NebdFileInstance* fd, NebdServerAioContext* aioctx) { + int curveFd = GetCurveFdFromNebdFileInstance(fd); + if (curveFd < 0) { + return -1; + } + + CurveAioCombineContext *curveCombineCtx = new CurveAioCombineContext(); + curveCombineCtx->nebdCtx = aioctx; + curveCombineCtx->curveCtx = new CurveAioContext(); + int ret = FromNebdCtxToCurveCtx(aioctx, curveCombineCtx->curveCtx); + if (ret < 0) { + delete curveCombineCtx->curveCtx; + delete curveCombineCtx; + return -1; + } + + ret = client_->AioWrite(curveFd, curveCombineCtx->curveCtx); + if (ret != LIBCURVE_ERROR::OK) { + delete curveCombineCtx->curveCtx; + delete curveCombineCtx; + return -1; + } + return 0; } -int CurveRequestExecutor::Flush(NebdFileInstance* fd, NebdServerAioContext* aioctx) { - // TODO +int CurveRequestExecutor::Flush( + NebdFileInstance* fd, NebdServerAioContext* aioctx) { + // TODO(lxc) return 0; } -int CurveRequestExecutor::GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) { - // TODO +int CurveRequestExecutor::InvalidCache(NebdFileInstance* fd) { + auto curveFileInstance = dynamic_cast(fd); + if (curveFileInstance == nullptr || + curveFileInstance->fd < 0 || + curveFileInstance->fileName.empty()) { + return -1; + } + return 0; } -int CurveRequestExecutor::InvalidCache(NebdFileInstance* fd) { - // TODO +int CurveRequestExecutor::GetCurveFdFromNebdFileInstance(NebdFileInstance* fd) { + auto curveFileInstance = dynamic_cast(fd); + if (curveFileInstance == nullptr) { + return -1; + } + + return curveFileInstance->fd; +} + +std::string CurveRequestExecutor::GetFileNameFromNebdFileInstance( + NebdFileInstance* fd) { + auto curveFileInstance = dynamic_cast(fd); + if (curveFileInstance == nullptr) { + return ""; + } + + return curveFileInstance->fileName; +} + +int CurveRequestExecutor::FromNebdCtxToCurveCtx( + NebdServerAioContext *nebdCtx, CurveAioContext *curveCtx) { + curveCtx->offset = nebdCtx->offset; + curveCtx->length = nebdCtx->length; + int ret = FromNebdOpToCurveOp(nebdCtx->op, &curveCtx->op); + if (ret < 0) { + return -1; + } + curveCtx->buf = nebdCtx->buf; + curveCtx->cb = CurveAioCallback; return 0; } +int CurveRequestExecutor::FromNebdOpToCurveOp(LIBAIO_OP op, LIBCURVE_OP *out) { + switch (op) { + case LIBAIO_OP::LIBAIO_OP_READ: + *out = LIBCURVE_OP::LIBCURVE_OP_READ; + return 0; + case LIBAIO_OP::LIBAIO_OP_WRITE: + *out = LIBCURVE_OP_WRITE; + return 0; + + default: + return -1; + } +} + +void CurveAioCallback(struct CurveAioContext* curveCtx) { + auto curveCombineCtx = reinterpret_cast( + reinterpret_cast(curveCtx) - + offsetof(CurveAioCombineContext, curveCtx)); + + curveCombineCtx->nebdCtx->ret = curveCtx->ret; + curveCombineCtx->nebdCtx->cb(curveCombineCtx->nebdCtx); + delete curveCtx; + delete curveCombineCtx; +} } // namespace server } // namespace nebd - diff --git a/src/part2/request_executor_curve.h b/src/part2/request_executor_curve.h index be0275b58f..6be3d225f2 100644 --- a/src/part2/request_executor_curve.h +++ b/src/part2/request_executor_curve.h @@ -1,19 +1,49 @@ -#ifndef SRC_PART2_REQUEST_EXECUTO_CURVE_H_ -#define SRC_PART2_REQUEST_EXECUTO_CURVE_H_ +/* + * Project: nebd + * Created Date: 2020-02-03 + * Author: lixiaocui + * Copyright (c) 2020 netease + */ +#ifndef SRC_PART2_REQUEST_EXECUTOR_CURVE_H_ +#define SRC_PART2_REQUEST_EXECUTOR_CURVE_H_ + +#include #include #include #include "src/part2/request_executor.h" +#include "src/part2/define.h" namespace nebd { namespace server { +using ::curve::client::CurveClient; + class CurveFileInstance : public NebdFileInstance { public: CurveFileInstance() {} ~CurveFileInstance() {} - int fd = 0; + int fd = -1; + std::string fileName; +}; + +class CurveAioCombineContext { + public: + NebdServerAioContext* nebdCtx; + CurveAioContext* curveCtx; +}; +static void CurveAioCallback(struct CurveAioContext* curveCtx); + +class FileNameParser { + public: + /** + * @brief 解析fileName + * 一般格式: "cbd:pool1//cinder/volume-6f30d296-07f7-452e-a983-513191f8cd95_cinder_:/etc/curve/client.conf" //NOLINT + * @param[in] fileName + * @return 解析出的字符串: "/cinder/volume-6f30d296-07f7-452e-a983-513191f8cd95_cinder_" //NOLINT + */ + static std::string Parse(const std::string& fileName); }; class CurveRequestExecutor : public NebdRequestExecutor { @@ -23,13 +53,13 @@ class CurveRequestExecutor : public NebdRequestExecutor { return executor; } ~CurveRequestExecutor() {} + void Init(const std::shared_ptr &client); std::shared_ptr Open(const std::string& filename) override; // NOLINT std::shared_ptr Reopen( const std::string& filename, AdditionType addtion) override; int Close(NebdFileInstance* fd) override; int Extend(NebdFileInstance* fd, int64_t newsize) override; int GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) override; - int StatFile(NebdFileInstance* fd, NebdFileInfo* fileInfo) override; int Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; int AioRead(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; int AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) override; @@ -37,10 +67,47 @@ class CurveRequestExecutor : public NebdRequestExecutor { int InvalidCache(NebdFileInstance* fd) override; private: + /** + * @brief 构造函数 + */ CurveRequestExecutor() {} + + /** + * @brief 从NebdFileInstance中解析出curve_client需要的fd + * @param[in] fd NebdFileInstance类型 + * @return 返回curve_client中文件的fd, 如果小于0，表示解析结果错误 + */ + int GetCurveFdFromNebdFileInstance(NebdFileInstance* fd); + + /** + * @brief 从NebdFileInstance中解析出curbe_client需要的filename + * @param[in] fd NebdFileInstance类型 + * @return 返回curve_client中的filename, 如果为空，表示解析出错 + */ + std::string GetFileNameFromNebdFileInstance(NebdFileInstance* fd); + + /** + * @brief 将NebdServerAioContext类型转换为CurveAioContext类型 + * @param[in] nebdCtx NebdServerAioContext类型 + * @param[out] curveCtx CurveAioContext类型 + * @return -1转换失败，0转换成功 + */ + int FromNebdCtxToCurveCtx( + NebdServerAioContext *nebdCtx, CurveAioContext *curveCtx); + + /** + * @brief 将LIBAIO_OP类型转换为curve_client中LIBCURVE_OP类型 + * @param[in] op LIBAIO_OP类型 + * @param[out] out LIBCURVE_OP类型 + * @return -1转换失败，0转换成功 + */ + int FromNebdOpToCurveOp(LIBAIO_OP op, LIBCURVE_OP *out); + + private: + std::shared_ptr<::curve::client::CurveClient> client_; }; } // namespace server } // namespace nebd -#endif // SRC_PART2_REQUEST_EXECUTO_CURVE_H_ +#endif // SRC_PART2_REQUEST_EXECUTOR_CURVE_H_ diff --git a/tests/part1/nebd_test.cpp b/tests/part1/nebd_test.cpp new file mode 100644 index 0000000000..07b75956b8 --- /dev/null +++ b/tests/part1/nebd_test.cpp @@ -0,0 +1,659 @@ +/* + * Project: nebd + * Created Date: 2019-08-12 + * Author: hzchenwei7 + * Copyright (c) 2018 netease + */ +#include +#include +#include +#include +#include +#include // flock +#include // NOLINT +#include // NOLINT +#include "tests/part1/mock_client_service.h" +#include "src/part1/libnebd.h" +#include "src/part1/libnebd_file.h" +#include "src/part1/nebd_client.h" +#include "src/part1/nebd_lifecycle.h" + +DEFINE_string(uuid, "12345678-1234-1234-1234-123456789012", "uuid"); + +#define confFilename "tests/part1/nebd.conf" +#define filename "test_file_name" +#define DEFAULT_FD 1 +#define BUFSIZE 512 +#define DEFAULT_FILEZIZE (50*1024*1024) +std::mutex mtx; +std::condition_variable condition; +bool callback; + +void LibAioCallBackFunc(struct ClientAioContext* context) { + ASSERT_EQ(context->ret, 0); + callback = true; + condition.notify_one(); + ASSERT_EQ(context->retryCount, 0); +} + +void LibAioFailCallBackFunc(struct ClientAioContext* context) { + ASSERT_EQ(context->ret, -1); + callback = true; + condition.notify_one(); + ASSERT_EQ(context->retryCount, 0); +} + + +class nebdClientTest: public ::testing::Test { + protected: + void SetUp() override { + callback = false; + system("sudo killall client_server"); + system("sudo mkdir -p /etc/nebd"); + system("sudo cp tests/part1/nebd.conf /etc/nebd/nebd-client.conf"); + ASSERT_EQ(0, nebd_lib_init()); + } + void TearDown() override { + nebd_lib_uninit(); + } +}; + +TEST_F(nebdClientTest, nebdCommonTest) { + // nebd_lib_open_test + int ret; + ret = nebd_lib_open(filename); + ASSERT_EQ(DEFAULT_FD, ret); + + // nebd_lib_pread_test + char buf[BUFSIZE] = {}; + ret = nebd_lib_pread(DEFAULT_FD, buf, 0, BUFSIZE); + ASSERT_EQ(-1, ret); + + // nebd_lib_pwrite_test + memset(buf, 'a', BUFSIZE); + ret = nebd_lib_pwrite(DEFAULT_FD, buf, 0, BUFSIZE); + ASSERT_EQ(-1, ret); + + // nebd_lib_aio_pread_pwrite_test + char writeBuf[BUFSIZE] = "test"; + char readBuf[BUFSIZE]; + ClientAioContext context; + context.buf = writeBuf; + context.offset = 0; + context.length = BUFSIZE; + context.ret = 0; + context.op = LIBAIO_OP_WRITE; + context.cb = LibAioCallBackFunc; + context.retryCount = 0; + + ret = nebd_lib_aio_pwrite(DEFAULT_FD, &context); + ASSERT_EQ(ret, 0); + + std::unique_lock lock{mtx}; + condition.wait(lock); + ASSERT_TRUE(callback); + + callback = false; + ret = nebd_lib_aio_pread(DEFAULT_FD, &context); + ASSERT_EQ(ret, 0); + + condition.wait(lock); + ASSERT_TRUE(callback); + + // nebd_lib_sync_test + ret = nebd_lib_sync(DEFAULT_FD); + ASSERT_EQ(0, ret); + + // nebd_lib_discard_test + context.buf = nullptr; + context.offset = 0; + context.length = BUFSIZE; + context.ret = 0; + context.op = LIBAIO_OP_DISCARD; + context.cb = LibAioCallBackFunc; + context.retryCount = 0; + + ret = nebd_lib_discard(DEFAULT_FD, &context); + ASSERT_EQ(ret, 0); + + condition.wait(lock); + ASSERT_TRUE(callback); + + // nebd_lib_filesize_test + int64_t filesize; + filesize = nebd_lib_filesize(DEFAULT_FD); + ASSERT_EQ(DEFAULT_FILEZIZE, filesize); + + // nebd_lib_resize_test + ret = nebd_lib_resize(DEFAULT_FD, DEFAULT_FILEZIZE); + ASSERT_EQ(0, ret); + + // nebd_lib_getinfo_test + ret = nebd_lib_getinfo(DEFAULT_FD); + ASSERT_EQ(DEFAULT_FILEZIZE, ret); + + // nebd_lib_flush_test + context.buf = nullptr; + context.offset = 0; + context.length = 0; + context.ret = 0; + context.op = LIBAIO_OP_FLUSH; + context.cb = LibAioCallBackFunc; + context.retryCount = 0; + + ret = nebd_lib_flush(DEFAULT_FD, &context); + ASSERT_EQ(ret, 0); + + condition.wait(lock); + ASSERT_TRUE(callback); + + // nebd_lib_getinfo_test + filesize = nebd_lib_getinfo(DEFAULT_FD); + ASSERT_EQ(DEFAULT_FILEZIZE, filesize); + + // nebd_lib_invalidcache_test + ret = nebd_lib_invalidcache(DEFAULT_FD); + ASSERT_EQ(0, ret); + + // nebd_lib_close_test + ret = nebd_lib_close(DEFAULT_FD); + ASSERT_EQ(0, ret); +} + +class nebdFileClientTest: public ::testing::Test { + protected: + void SetUp() override { + system("sudo killall client_server"); + system("sudo mkdir -p /etc/nebd"); + system("sudo cp tests/part1/nebd.conf /etc/nebd/nebd-client.conf"); + } + void TearDown() override { + } +}; + +TEST_F(nebdFileClientTest, common_test) { + nebd::client::FileClient fileClientTest; + ASSERT_EQ(fileClientTest.Init(confFilename), 0); + ASSERT_EQ(fileClientTest.Open(filename), DEFAULT_FD); + ASSERT_EQ(fileClientTest.Extend(DEFAULT_FD, DEFAULT_FILEZIZE), 0); + ASSERT_EQ(fileClientTest.StatFile(DEFAULT_FD), DEFAULT_FILEZIZE); + ASSERT_EQ(fileClientTest.GetInfo(DEFAULT_FD), DEFAULT_FILEZIZE); + ASSERT_EQ(fileClientTest.InvalidCache(DEFAULT_FD), 0); + + char writeBuf[BUFSIZE] = "test"; + char readBuf[BUFSIZE]; + ClientAioContext context; + context.buf = writeBuf; + context.offset = 0; + context.length = BUFSIZE; + context.ret = 0; + context.op = LIBAIO_OP_WRITE; + context.cb = LibAioCallBackFunc; + context.retryCount = 0; + + ASSERT_EQ(fileClientTest.AioRead(DEFAULT_FD, &context), 0); + std::unique_lock lock{mtx}; + condition.wait(lock); + + ASSERT_EQ(fileClientTest.AioWrite(DEFAULT_FD, &context), 0); + condition.wait(lock); + ASSERT_TRUE(callback); + + context.buf = nullptr; + context.offset = 0; + context.length = BUFSIZE; + context.ret = 0; + context.op = LIBAIO_OP_DISCARD; + context.cb = LibAioCallBackFunc; + context.retryCount = 0; + ASSERT_EQ(fileClientTest.Discard(DEFAULT_FD, &context), 0); + condition.wait(lock); + ASSERT_TRUE(callback); + + context.buf = nullptr; + context.offset = 0; + context.length = 0; + context.ret = 0; + context.op = LIBAIO_OP_FLUSH; + context.cb = LibAioCallBackFunc; + context.retryCount = 0; + ASSERT_EQ(fileClientTest.Flush(DEFAULT_FD, &context), 0); + condition.wait(lock); + ASSERT_TRUE(callback); + + ASSERT_EQ(fileClientTest.Close(DEFAULT_FD), 0); + fileClientTest.Uninit(); + return; +} + +TEST_F(nebdFileClientTest, no_rpc_server_test) { + nebd::client::FileClient fileClientTest; + ASSERT_EQ(fileClientTest.Init("wrongConfPath"), -1); + + nebd::common::Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_TRUE(conf.LoadConfig()); + + ASSERT_EQ(fileClientTest.LoadConf(&conf), 0); + + ASSERT_EQ(fileClientTest.InitChannel("127.0.0.1:6667"), 0); + ASSERT_EQ(fileClientTest.Open(filename), -1); + ASSERT_EQ(fileClientTest.Extend(DEFAULT_FD, DEFAULT_FILEZIZE), -1); + ASSERT_EQ(fileClientTest.StatFile(DEFAULT_FD), -1); + ASSERT_EQ(fileClientTest.GetInfo(DEFAULT_FD), -1); + ASSERT_EQ(fileClientTest.InvalidCache(DEFAULT_FD), -1); + + // char writeBuf[BUFSIZE] = "test"; + // char readBuf[BUFSIZE]; + // ClientAioContext context; + // context.buf = writeBuf; + // context.offset = 0; + // context.length = BUFSIZE; + // context.ret = 0; + // context.op = LIBAIO_OP_WRITE; + // context.cb = LibAioCallBackFunc; + // context.retryCount = 0; + + // ASSERT_EQ(fileClientTest.AioRead(DEFAULT_FD, &context), 0); + // std::unique_lock lock{mtx}; + // condition.wait(lock); + + // ASSERT_EQ(fileClientTest.AioWrite(DEFAULT_FD, &context), 0); + // condition.wait(lock); + // ASSERT_TRUE(callback); + + // context.buf = nullptr; + // context.offset = 0; + // context.length = BUFSIZE; + // context.ret = 0; + // context.op = LIBAIO_OP_DISCARD; + // context.cb = LibAioCallBackFunc; + // context.retryCount = 0; + // ASSERT_EQ(fileClientTest.Discard(DEFAULT_FD, &context), 0); + // condition.wait(lock); + // ASSERT_TRUE(callback); + + // context.buf = nullptr; + // context.offset = 0; + // context.length = 0; + // context.ret = 0; + // context.op = LIBAIO_OP_FLUSH; + // context.cb = LibAioCallBackFunc; + // context.retryCount = 0; + // ASSERT_EQ(fileClientTest.Flush(DEFAULT_FD, &context), 0); + // condition.wait(lock); + // ASSERT_TRUE(callback); + + ASSERT_EQ(fileClientTest.Close(DEFAULT_FD), -1); + fileClientTest.Uninit(); + return; +} + +static void OpenFileFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::OpenFileRequest* request, + ::nebd::client::OpenFileResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + // if (0 != gReadCntlFailedCode) { + // brpc::Controller *cntl = dynamic_cast(controller); + // cntl->SetFailed(-1, "open file controller error"); + // } +} + +static void CloseFileFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::CloseFileRequest* request, + ::nebd::client::CloseFileResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +static void ReadFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::ReadRequest* request, + ::nebd::client::ReadResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +static void WriteFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::WriteRequest* request, + ::nebd::client::WriteResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +static void DiscardFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::DiscardRequest* request, + ::nebd::client::DiscardResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +static void StatFileFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::StatFileRequest* request, + ::nebd::client::StatFileResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +static void ResizeFileFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::ResizeRequest* request, + ::nebd::client::ResizeResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +static void FlushFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::FlushRequest* request, + ::nebd::client::FlushResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +static void GetInfoFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::GetInfoRequest* request, + ::nebd::client::GetInfoResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +static void InvalidateCacheFunc(::google::protobuf::RpcController* controller, + const ::nebd::client::InvalidateCacheRequest* request, + ::nebd::client::InvalidateCacheResponse* response, + ::google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +using ::testing::_; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::AnyNumber; +using ::testing::DoAll; +using ::testing::SetArgPointee; +using ::testing::SetArgReferee; +using ::testing::InSequence; +using ::testing::AtLeast; +using ::testing::SaveArgPointee; + +TEST_F(nebdFileClientTest, rpc_fail_test) { + nebd::client::FileClient fileClientTest; + nebd::common::Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_TRUE(conf.LoadConfig()); + ASSERT_EQ(fileClientTest.LoadConf(&conf), 0); + + // add service + brpc::Server server; + nebd::client::MockQemuClientService clientService; + ASSERT_EQ(server.AddService(&clientService, + brpc::SERVER_DOESNT_OWN_SERVICE), 0); + + // start rpc server + brpc::ServerOptions option; + option.idle_timeout_sec = -1; + std::string listenAddr = "127.0.0.1:6667"; + ASSERT_EQ(server.Start(listenAddr.c_str(), &option), 0); + + ASSERT_EQ(fileClientTest.InitChannel("127.0.0.1:6667"), 0); + + ::nebd::client::OpenFileResponse openFileResponse; + openFileResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, OpenFile(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(openFileResponse), + Invoke(OpenFileFunc))); + ASSERT_EQ(fileClientTest.Open(filename), -1); + + ::nebd::client::ResizeResponse resizeResponse; + resizeResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, ResizeFile(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(resizeResponse), + Invoke(ResizeFileFunc))); + ASSERT_EQ(fileClientTest.Extend(DEFAULT_FD, DEFAULT_FILEZIZE), -1); + + ::nebd::client::StatFileResponse statFileResponse; + statFileResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, StatFile(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(statFileResponse), + Invoke(StatFileFunc))); + ASSERT_EQ(fileClientTest.StatFile(DEFAULT_FD), -1); + + ::nebd::client::GetInfoResponse getInfoResponse; + getInfoResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, GetInfo(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(getInfoResponse), + Invoke(GetInfoFunc))); + ASSERT_EQ(fileClientTest.GetInfo(DEFAULT_FD), -1); + + ::nebd::client::InvalidateCacheResponse invalidateCacheResponse; + invalidateCacheResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, InvalidateCache(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(invalidateCacheResponse), + Invoke(InvalidateCacheFunc))); + ASSERT_EQ(fileClientTest.InvalidCache(DEFAULT_FD), -1); + + char writeBuf[BUFSIZE] = "test"; + char readBuf[BUFSIZE]; + ClientAioContext context; + context.buf = writeBuf; + context.offset = 0; + context.length = BUFSIZE; + context.ret = 0; + context.op = LIBAIO_OP_WRITE; + context.cb = LibAioFailCallBackFunc; + context.retryCount = 0; + + ::nebd::client::ReadResponse readResponse; + readResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, Read(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(readResponse), + Invoke(ReadFunc))); + ASSERT_EQ(fileClientTest.AioRead(DEFAULT_FD, &context), 0); + std::unique_lock lock{mtx}; + condition.wait(lock); + ASSERT_TRUE(callback); + + + context.buf = readBuf; + context.offset = 0; + context.length = BUFSIZE; + context.ret = 0; + context.op = LIBAIO_OP_READ; + context.cb = LibAioFailCallBackFunc; + context.retryCount = 0; + ::nebd::client::WriteResponse writeResponse; + writeResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, Write(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(writeResponse), + Invoke(WriteFunc))); + ASSERT_EQ(fileClientTest.AioWrite(DEFAULT_FD, &context), 0); + condition.wait(lock); + ASSERT_TRUE(callback); + + context.buf = nullptr; + context.offset = 0; + context.length = BUFSIZE; + context.ret = 0; + context.op = LIBAIO_OP_DISCARD; + context.cb = LibAioFailCallBackFunc; + context.retryCount = 0; + ::nebd::client::DiscardResponse discardResponse; + discardResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, Discard(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(discardResponse), + Invoke(DiscardFunc))); + ASSERT_EQ(fileClientTest.Discard(DEFAULT_FD, &context), 0); + condition.wait(lock); + ASSERT_TRUE(callback); + + context.buf = nullptr; + context.offset = 0; + context.length = 0; + context.ret = 0; + context.op = LIBAIO_OP_FLUSH; + context.cb = LibAioFailCallBackFunc; + context.retryCount = 0; + ::nebd::client::FlushResponse flushResponse; + flushResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, Flush(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(flushResponse), + Invoke(FlushFunc))); + ASSERT_EQ(fileClientTest.Flush(DEFAULT_FD, &context), 0); + condition.wait(lock); + ASSERT_TRUE(callback); + + ::nebd::client::CloseFileResponse closeFileResponse; + closeFileResponse.set_retcode(nebd::client::RetCode::kNoOK); + EXPECT_CALL(clientService, CloseFile(_, _, _, _)) + .Times(1) + .WillOnce(DoAll(SetArgPointee<2>(closeFileResponse), + Invoke(CloseFileFunc))); + ASSERT_EQ(fileClientTest.Close(DEFAULT_FD), -1); + fileClientTest.Uninit(); + return; +} + +class nebdClientLifeCycleTest: public ::testing::Test { + protected: + void SetUp() override {} + void TearDown() override {} +}; + +TEST_F(nebdClientLifeCycleTest, init_when_part2_not_alive) { + ASSERT_EQ(Init4Qemu(confFilename), 0); + Uninit4Qemu(); +} + +TEST_F(nebdClientLifeCycleTest, init_when_part2_alive) { + ASSERT_EQ(Init4Qemu(confFilename), 0); + ASSERT_EQ(Init4Qemu(confFilename), 0); + Uninit4Qemu(); +} + +TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_stop_without_hb_test) { + using ::nebd::common::Configuration; + Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_EQ(conf.LoadConfig(), true); + + nebd::client::LifeCycleManager manager; + ASSERT_EQ(manager.Start(&conf), 0); + ASSERT_EQ(manager.IsPart2Alive(), true); + ASSERT_EQ(manager.IsHeartbeatThreadStart(), false); + ASSERT_EQ(manager.Start(&conf), 0); + ASSERT_EQ(manager.IsPart2Alive(), true); + ASSERT_EQ(manager.IsHeartbeatThreadStart(), false); + manager.Stop(); + ASSERT_EQ(manager.IsPart2Alive(), false); +} + +TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_stop_with_hb_test) { + using ::nebd::common::Configuration; + Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_EQ(conf.LoadConfig(), true); + + nebd::client::LifeCycleManager manager; + ASSERT_EQ(manager.Start(&conf), 0); + ASSERT_EQ(manager.IsPart2Alive(), true); + ASSERT_EQ(manager.Start(&conf), 0); + ASSERT_EQ(manager.IsPart2Alive(), true); + ASSERT_EQ(manager.IsHeartbeatThreadStart(), false); + manager.StartHeartbeat(); + ASSERT_EQ(manager.IsHeartbeatThreadStart(), true); + ASSERT_EQ(manager.IsPart2Alive(), true); + manager.Stop(); + ASSERT_EQ(manager.IsPart2Alive(), false); +} + +TEST_F(nebdClientLifeCycleTest, lifecycle_kill_part2_test) { + using ::nebd::common::Configuration; + Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_EQ(conf.LoadConfig(), true); + + nebd::client::LifeCycleManager manager; + ASSERT_EQ(manager.Start(&conf), 0); + ASSERT_EQ(manager.IsPart2Alive(), true); + ASSERT_EQ(manager.IsHeartbeatThreadStart(), false); + manager.StartHeartbeat(); + ASSERT_EQ(manager.IsHeartbeatThreadStart(), true); + manager.KillPart2(); + sleep(2); + ASSERT_EQ(manager.IsPart2Alive(), true); + manager.Stop(); + ASSERT_EQ(manager.IsPart2Alive(), false); +} + +TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_load_conf_fail_test) { + ::nebd::common::Configuration conf; + nebd::client::LifeCycleManager manager; + ASSERT_EQ(manager.Start(&conf), -1); +} + +TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_get_uuid_fail_test) { + ::nebd::common::Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_EQ(conf.LoadConfig(), true); + conf.SetStringValue("qemuProcName", "wrong_qemuProcName"); + + nebd::client::LifeCycleManager manager; + ASSERT_EQ(manager.Start(&conf), -1); +} + +TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_start_part2_fail) { + ::nebd::common::Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_EQ(conf.LoadConfig(), true); + conf.SetStringValue("part2ProcPath", "wrong_part2ProcPath"); + + nebd::client::LifeCycleManager manager; + ASSERT_EQ(manager.Start(&conf), -1); +} + +TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_read_port_fail) { + ::nebd::common::Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_EQ(conf.LoadConfig(), true); + conf.SetStringValue("metadataPrefix", "wrong_metadataPrefix"); + + nebd::client::LifeCycleManager manager; + ASSERT_EQ(manager.Start(&conf), -1); +} + +TEST_F(nebdClientLifeCycleTest, lifecycle_manager_lock_file_fail) { + ::nebd::common::Configuration conf; + conf.SetConfigPath(confFilename); + ASSERT_TRUE(conf.LoadConfig()); + + // 先占用文件锁 + std::string lockFile; + ASSERT_TRUE(conf.GetStringValue("lockFile", &lockFile)); + int fd = open(lockFile.c_str(), O_RDONLY | O_CREAT, 0644); + ASSERT_GT(fd, 0); + ASSERT_EQ(flock(fd, LOCK_EX | LOCK_NB), 0); + + // 再启lifecyclemanager，会因为文件锁被占用而失败 + nebd::client::LifeCycleManager manager; + ASSERT_EQ(manager.Start(&conf), -1); + + // 再释放文件锁 + ASSERT_EQ(flock(fd, LOCK_UN), 0); + close(fd); +} + +int main(int argc, char ** argv) { + ::testing::InitGoogleTest(&argc, argv); + ::testing::InitGoogleMock(&argc, argv); + google::ParseCommandLineFlags(&argc, &argv, false); + int ret = RUN_ALL_TESTS(); + + return ret; +} diff --git a/tests/part2/CMakeLists.txt b/tests/part2/CMakeLists.txt index 4757a1e345..ece5580748 100644 --- a/tests/part2/CMakeLists.txt +++ b/tests/part2/CMakeLists.txt @@ -10,6 +10,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CPP_FLAGS} -std=c++0x -DNDEBUG -O2 \ set(PART2_TEST_LINK nebdserver + pthread gtest gmock gflags) @@ -19,7 +20,8 @@ set(PART2_MOCK_SRC mock_metafile_manager.h mock_request_executor.h mock_file_manager.h - mock_posix_wrapper.h) + mock_posix_wrapper.h + mock_curve_client.h) add_library(part2mock ${PART2_MOCK_SRC}) set_target_properties(part2mock PROPERTIES LINKER_LANGUAGE CXX) target_link_libraries(part2mock @@ -77,3 +79,12 @@ target_link_libraries(test_nebd_server ${PART2_TEST_LINK}) install(TARGETS test_nebd_server DESTINATION bin) add_test(NAME test_nebd_server COMMAND test_nebd_server) + +# test_request_executor_curve +set(TEST_REQUESTEXECUTORCURVE_SRC test_request_executor_curve.cpp) +add_executable(test_request_executor_curve ${TEST_REQUESTEXECUTORCURVE_SRC}) +target_link_libraries(test_request_executor_curve + part2mock + ${PART2_TEST_LINK}) +install(TARGETS test_request_executor_curve DESTINATION bin) +add_test(NAME test_request_executor_curve COMMAND test_request_executor_curve) diff --git a/tests/part2/mock_curve_client.h b/tests/part2/mock_curve_client.h new file mode 100644 index 0000000000..459753d283 --- /dev/null +++ b/tests/part2/mock_curve_client.h @@ -0,0 +1,36 @@ +/* + * Project: nebd + * Created Date: 2020-02-05 + * Author: lixiaocui + * Copyright (c) 2020 netease + */ + +#ifndef TEST_MOCK_CURVE_CLIENT_H_ +#define TEST_MOCK_CURVE_CLIENT_H_ + +#include +#include +#include + +namespace nebd { +namespace server { +class MockCurveClient : public ::curve::client::CurveClient { + public: + MockCurveClient() {} + ~MockCurveClient() {} + MOCK_METHOD1(Init, int(const std::string&)); + MOCK_METHOD0(UnInit, void()); + MOCK_METHOD2(Open, int(const std::string&, std::string*)); + MOCK_METHOD3(ReOpen, int( + const std::string&, const std::string&, std::string*)); + MOCK_METHOD1(Close, int(int)); + MOCK_METHOD2(Extend, int(const std::string&, int64_t)); + MOCK_METHOD1(StatFile, int64_t(const std::string&)); + MOCK_METHOD2(AioRead, int(int, CurveAioContext*)); + MOCK_METHOD2(AioWrite, int(int, CurveAioContext*)); +}; + +} // namespace server +} // namespace nebd + +#endif // TEST_MOCK_CURVE_CLIENT_H_ diff --git a/tests/part2/test_request_executor_curve.cpp b/tests/part2/test_request_executor_curve.cpp new file mode 100644 index 0000000000..156a602045 --- /dev/null +++ b/tests/part2/test_request_executor_curve.cpp @@ -0,0 +1,368 @@ +/* + * Project: nebd + * Created Date: 2020-02-05 + * Author: lixiaocui + * Copyright (c) 2020 netease + */ + +#include +#include "src/part2/request_executor_curve.h" +#include "tests/part2/mock_curve_client.h" + +namespace nebd { +namespace server { + +using ::testing::Return; +using ::testing::_; +using ::testing::SetArgPointee; +using ::testing::DoAll; + +class TestReuqestExecutorCurve : public ::testing::Test { + protected: + void SetUp() { + curveClient_ = std::make_shared(); + CurveRequestExecutor::GetInstance().Init(curveClient_); + } + + protected: + std::shared_ptr curveClient_; +}; + +TEST_F(TestReuqestExecutorCurve, test_Open) { + auto executor = CurveRequestExecutor::GetInstance(); + + std::string fileName("cbd:pool1//cinder/volume-1234_cinder_:/client.conf"); + std::string curveFileName("/cinder/volume-1234_cinder_"); + + // 1. 传入的fileName解析失败 + { + std::string errFileName("cbd:pool1/:"); + EXPECT_CALL(*curveClient_, Open(fileName, _)).Times(0); + std::shared_ptr ret = executor.Open(errFileName); + ASSERT_TRUE(nullptr == ret); + } + + // 2. curveclient open失败 + { + EXPECT_CALL(*curveClient_, Open(curveFileName, _)) + .WillOnce(DoAll(SetArgPointee<1>(""), Return(-1))); + std::shared_ptr ret = executor.Open(fileName); + ASSERT_TRUE(nullptr == ret); + } + + // 3. open成功 + { + EXPECT_CALL(*curveClient_, Open(curveFileName, _)) + .WillOnce(DoAll(SetArgPointee<1>("abc"), Return(1))); + std::shared_ptr ret = executor.Open(fileName); + ASSERT_TRUE(nullptr != ret); + auto *curveIns = dynamic_cast(ret.get()); + ASSERT_TRUE(nullptr != curveIns); + ASSERT_EQ(curveFileName, curveIns->fileName); + ASSERT_EQ(1, curveIns->fd); + ASSERT_EQ("abc", curveIns->addition["session"]); + } +} + +TEST_F(TestReuqestExecutorCurve, test_ReOpen) { + auto executor = CurveRequestExecutor::GetInstance(); + AdditionType addtion; + addtion["session"] = "abc"; + std::string fileName("cbd:pool1//cinder/volume-1234_cinder_:/client.conf"); + std::string curveFileName("/cinder/volume-1234_cinder_"); + + // 1. 传入的fileName解析失败 + { + std::string errFileName("cbd:pool1/:"); + EXPECT_CALL(*curveClient_, Open(_, _)).Times(0); + std::shared_ptr ret = executor.Reopen( + errFileName, addtion); + ASSERT_TRUE(nullptr == ret); + } + + // 2. repoen失败 + { + EXPECT_CALL(*curveClient_, ReOpen(curveFileName, addtion["session"], _)) + .WillOnce(DoAll(SetArgPointee<2>(""), Return(-1))); + std::shared_ptr ret = + executor.Reopen(fileName, addtion); + ASSERT_TRUE(nullptr == ret); + } + + // 3. reopen成功 + { + EXPECT_CALL(*curveClient_, ReOpen(curveFileName, addtion["session"], _)) + .WillOnce(DoAll(SetArgPointee<2>("bcd"), Return(1))); + std::shared_ptr ret = + executor.Reopen(fileName, addtion); + ASSERT_TRUE(nullptr != ret); + auto *curveIns = dynamic_cast(ret.get()); + ASSERT_TRUE(nullptr != curveIns); + ASSERT_EQ(curveFileName, curveIns->fileName); + ASSERT_EQ(1, curveIns->fd); + ASSERT_EQ("bcd", curveIns->addition["session"]); + } +} + +TEST_F(TestReuqestExecutorCurve, test_Close) { + auto executor = CurveRequestExecutor::GetInstance(); + + // 1. nebdFileIns不是CurveFileInstance类型, close失败 + { + auto nebdFileIns = new NebdFileInstance(); + EXPECT_CALL(*curveClient_, Close(_)).Times(0); + ASSERT_EQ(-1, executor.Close(nebdFileIns)); + + } + + // 2. nebdFileIns中的fd<0, close失败 + { + auto curveFileIns = new CurveFileInstance(); + curveFileIns->fd = -1; + EXPECT_CALL(*curveClient_, Close(_)).Times(0); + ASSERT_EQ(-1, executor.Close(curveFileIns)); + } + + // 3. 调用curveclient的close接口失败， close失败 + { + auto curveFileIns = new CurveFileInstance(); + curveFileIns->fd = 1; + EXPECT_CALL(*curveClient_, Close(1)) + .WillOnce(Return(LIBCURVE_ERROR::FAILED)); + ASSERT_EQ(-1, executor.Close(curveFileIns)); + } + + // 4. close成功 + { + auto curveFileIns = new CurveFileInstance(); + curveFileIns->fd = 1; + EXPECT_CALL(*curveClient_, Close(1)) + .WillOnce(Return(LIBCURVE_ERROR::OK)); + ASSERT_EQ(0, executor.Close(curveFileIns)); + } +} + +TEST_F(TestReuqestExecutorCurve, test_Extend) { + auto executor = CurveRequestExecutor::GetInstance(); + std::string curveFilename("/cinder/volume-1234_cinder_"); + + // 1. nebdFileIns不是CurveFileInstance类型, extend失败 + { + auto nebdFileIns = new NebdFileInstance(); + EXPECT_CALL(*curveClient_, Extend(_, _)).Times(0); + ASSERT_EQ(-1, executor.Extend(nebdFileIns, 1)); + } + + // 2. nebdFileIns中的fileName为空, extend失败 + { + auto curveFileIns = new CurveFileInstance(); + EXPECT_CALL(*curveClient_, Extend(_, _)).Times(0); + ASSERT_EQ(-1, executor.Extend(curveFileIns, 1)); + } + + // 3. 调用curveclient的extend接口失败， extend失败 + { + auto curveFileIns = new CurveFileInstance(); + curveFileIns->fileName = curveFilename; + EXPECT_CALL(*curveClient_, Extend(curveFilename, 1)) + .WillOnce(Return(LIBCURVE_ERROR::FAILED)); + ASSERT_EQ(-1, executor.Extend(curveFileIns, 1)); + } + + // 4. extend成功 + { + auto curveFileIns = new CurveFileInstance(); + curveFileIns->fileName = curveFilename; + EXPECT_CALL(*curveClient_, Extend(curveFilename, 1)) + .WillOnce(Return(LIBCURVE_ERROR::OK)); + ASSERT_EQ(0, executor.Extend(curveFileIns, 1)); + } +} + +TEST_F(TestReuqestExecutorCurve, test_GetInfo) { + auto executor = CurveRequestExecutor::GetInstance(); + NebdFileInfo fileInfo; + std::string curveFilename("/cinder/volume-1234_cinder_"); + + // 1. nebdFileIns不是CurveFileInstance类型, stat失败 + { + auto nebdFileIns = new NebdFileInstance(); + EXPECT_CALL(*curveClient_, StatFile(_)).Times(0); + ASSERT_EQ(-1, executor.GetInfo(nebdFileIns, &fileInfo)); + } + + // 2. nebdFileIns中的fileName为空, extend失败 + { + auto curveFileIns = new CurveFileInstance(); + EXPECT_CALL(*curveClient_, StatFile(_)).Times(0); + ASSERT_EQ(-1, executor.GetInfo(curveFileIns, &fileInfo)); + } + + // 3. 调用curveclient的extend接口失败， extend失败 + { + auto curveFileIns = new CurveFileInstance(); + curveFileIns->fileName = curveFilename; + EXPECT_CALL(*curveClient_, StatFile(curveFilename)) + .WillOnce(Return(-1)); + ASSERT_EQ(-1, executor.GetInfo(curveFileIns, &fileInfo)); + } + + // 4. extend成功 + { + auto curveFileIns = new CurveFileInstance(); + curveFileIns->fileName = curveFilename; + EXPECT_CALL(*curveClient_, StatFile(curveFilename)).WillOnce(Return(1)); + ASSERT_EQ(0, executor.GetInfo(curveFileIns, &fileInfo)); + ASSERT_EQ(1, fileInfo.size); + } +} + +TEST_F(TestReuqestExecutorCurve, test_AioRead) { + auto executor = CurveRequestExecutor::GetInstance(); + NebdServerAioContext aiotcx; + std::string curveFilename("/cinder/volume-1234_cinder_"); + + // 1. nebdFileIns不是CurveFileInstance类型, 异步读失败 + { + auto nebdFileIns = new NebdFileInstance(); + EXPECT_CALL(*curveClient_, AioRead(_, _)).Times(0); + ASSERT_EQ(-1, executor.AioRead(nebdFileIns, &aiotcx)); + } + + // 2. nebdFileIns中的fd<0, 异步读失败 + { + auto curveFileIns = new CurveFileInstance(); + curveFileIns->fd = -1; + EXPECT_CALL(*curveClient_, AioRead(_, _)).Times(0); + ASSERT_EQ(-1, executor.AioRead(curveFileIns, &aiotcx)); + } + + // 3. 调用curveclient的AioRead接口失败， 异步读失败 + { + auto curveFileIns = new CurveFileInstance(); + aiotcx.length = 1; + aiotcx.offset = 0; + aiotcx.buf = new char[10]; + aiotcx.op = LIBAIO_OP::LIBAIO_OP_READ; + curveFileIns->fd = 1; + curveFileIns->fileName = curveFilename; + EXPECT_CALL(*curveClient_, AioRead(1, _)) + .WillOnce(Return(LIBCURVE_ERROR::FAILED)); + ASSERT_EQ(-1, executor.AioRead(curveFileIns, &aiotcx)); + } + + // 4. 异步读取成功 + { + auto curveFileIns = new CurveFileInstance(); + curveFileIns->fd = 1; + curveFileIns->fileName = curveFilename; + EXPECT_CALL(*curveClient_, AioRead(1, _)) + .WillOnce(Return(LIBCURVE_ERROR::OK)); + ASSERT_EQ(0, executor.AioRead(curveFileIns, &aiotcx)); + } +} + +TEST_F(TestReuqestExecutorCurve, test_AioWrite) { + auto executor = CurveRequestExecutor::GetInstance(); + NebdServerAioContext aiotcx; + std::string curveFilename("/cinder/volume-1234_cinder_"); + + // 1. nebdFileIns不是CurveFileInstance类型, 异步写失败 + { + auto nebdFileIns = new NebdFileInstance(); + EXPECT_CALL(*curveClient_, AioWrite(_, _)).Times(0); + ASSERT_EQ(-1, executor.AioWrite(nebdFileIns, &aiotcx)); + } + + // 2. nebdFileIns中的fd<0, 异步写失败 + { + auto curveFileIns = new CurveFileInstance(); + curveFileIns->fd = -1; + EXPECT_CALL(*curveClient_, AioWrite(_, _)).Times(0); + ASSERT_EQ(-1, executor.AioWrite(curveFileIns, &aiotcx)); + } + + // 3. 调用curveclient的AioWrite接口失败， 异步写失败 + { + auto curveFileIns = new CurveFileInstance(); + aiotcx.length = 1; + aiotcx.offset = 0; + aiotcx.buf = new char[10]; + aiotcx.op = LIBAIO_OP::LIBAIO_OP_READ; + curveFileIns->fd = 1; + curveFileIns->fileName = curveFilename; + EXPECT_CALL(*curveClient_, AioWrite(1, _)) + .WillOnce(Return(LIBCURVE_ERROR::FAILED)); + ASSERT_EQ(-1, executor.AioWrite(curveFileIns, &aiotcx)); + } + + // 4. 异步写入成功 + { + auto curveFileIns = new CurveFileInstance(); + curveFileIns->fd = 1; + curveFileIns->fileName = curveFilename; + EXPECT_CALL(*curveClient_, AioWrite(1, _)) + .WillOnce(Return(LIBCURVE_ERROR::OK)); + ASSERT_EQ(0, executor.AioWrite(curveFileIns, &aiotcx)); + } +} + +TEST_F(TestReuqestExecutorCurve, test_InvalidCache) { + auto executor = CurveRequestExecutor::GetInstance(); + std::string curveFilename("/cinder/volume-1234_cinder_"); + + // 1. nebdFileIns不是CurveFileInstance类型, 不合法 + { + auto nebdFileIns = new NebdFileInstance(); + ASSERT_EQ(-1, executor.InvalidCache(nebdFileIns)); + } + + // 2. fd<0, 不合法 + { + auto curveFileIns = new CurveFileInstance(); + curveFileIns->fileName = curveFilename; + curveFileIns->fd = -1; + ASSERT_EQ(-1, executor.InvalidCache(curveFileIns)); + } + + // 3. filename为空，不合法 + { + auto curveFileIns = new CurveFileInstance(); + curveFileIns->fd = 1; + ASSERT_EQ(-1, executor.InvalidCache(curveFileIns)); + } + + // 4. 合法 + { + auto curveFileIns = new CurveFileInstance(); + curveFileIns->fd = 1; + curveFileIns->fileName = curveFilename; + ASSERT_EQ(0, executor.InvalidCache(curveFileIns)); + } +} + +TEST(TestFileNameParser, test_Parse) { + std::string fileName("cbd:pool1//cinder/volume-1234_cinder_:/client.conf"); + std::string res("/cinder/volume-1234_cinder_"); + ASSERT_EQ(res, FileNameParser::Parse(fileName)); + + fileName = "cbd:pool1"; + ASSERT_EQ("", FileNameParser::Parse(fileName)); + + fileName = "cbd:pool1//cinder/volume-1234_cinder_"; + ASSERT_EQ("", FileNameParser::Parse(fileName)); + + fileName = "cbd:pool1//:"; + ASSERT_EQ("", FileNameParser::Parse(fileName)); +} + +} // namespace server +} // namespace nebd + +int main(int argc, char ** argv) { + ::testing::InitGoogleTest(&argc, argv); + ::testing::InitGoogleMock(&argc, argv); + return RUN_ALL_TESTS(); +} + + From 03c31ad5773c29059baf5167518b9838cdd96ba8 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Tue, 21 Jan 2020 12:55:51 +0800 Subject: [PATCH 26/79] nebd part1 Change-Id: I6db1ad4ab108bf7a7b3b35ab3e47d9a89c2d0470 --- .gitignore | 2 + do_cmake | 2 +- etc/nebd/nebd-client.conf | 66 +-- get_3rdparty | 5 +- src/common/configuration.cpp | 9 + src/common/configuration.h | 1 + src/common/file_lock.cpp | 54 ++ src/common/file_lock.h | 46 ++ src/common/heartbeat.proto | 7 +- src/part1/CMakeLists.txt | 6 +- src/part1/async_request_closure.cpp | 99 ++++ src/part1/async_request_closure.h | 129 +++++ src/part1/connection_manager.cpp | 32 -- src/part1/connection_manager.h | 33 -- src/part1/heartbeat_manager.cpp | 102 ++++ src/part1/heartbeat_manager.h | 77 +++ src/part1/nebd_client.cpp | 467 +++++++++++++++++- src/part1/nebd_client.h | 46 +- src/part1/nebd_common.h | 48 ++ src/part1/nebd_metacache.cpp | 40 +- src/part1/nebd_metacache.h | 60 ++- src/part2/CMakeLists.txt | 2 - tests/part1/CMakeLists.txt | 36 +- tests/part1/fake_file_service.cpp | 67 +-- tests/part1/fake_file_service.h | 21 +- tests/part1/fake_heartbeat_service.h | 61 +++ tests/part1/heartbeat_manager_unittest.cpp | 158 ++++++ tests/part1/mock_file_service.h | 6 +- tests/part1/mock_heartbeat_service.h | 30 ++ tests/part1/nebd-client-aio-test.conf | 19 + tests/part1/nebd-client.conf | 19 + tests/part1/nebd_client_unittest.cpp | 547 +++++++++++++++++++++ tests/part1/nebd_metacache_unittest.cpp | 81 +++ 33 files changed, 2173 insertions(+), 205 deletions(-) create mode 100644 src/common/file_lock.cpp create mode 100644 src/common/file_lock.h create mode 100644 src/part1/async_request_closure.cpp create mode 100644 src/part1/async_request_closure.h delete mode 100644 src/part1/connection_manager.cpp delete mode 100644 src/part1/connection_manager.h create mode 100644 src/part1/heartbeat_manager.cpp create mode 100644 src/part1/heartbeat_manager.h create mode 100644 src/part1/nebd_common.h create mode 100644 tests/part1/fake_heartbeat_service.h create mode 100644 tests/part1/heartbeat_manager_unittest.cpp create mode 100644 tests/part1/mock_heartbeat_service.h create mode 100644 tests/part1/nebd-client-aio-test.conf create mode 100644 tests/part1/nebd-client.conf create mode 100644 tests/part1/nebd_client_unittest.cpp create mode 100644 tests/part1/nebd_metacache_unittest.cpp diff --git a/.gitignore b/.gitignore index dc6d70fc15..28beb4fab0 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ 3rdparty/libconfig 3rdparty/cmock build + +.vscode diff --git a/do_cmake b/do_cmake index fdc87d4f96..03df9f22e0 100755 --- a/do_cmake +++ b/do_cmake @@ -13,4 +13,4 @@ export LD_LIBRARY_PATH=/usr/lib/gcc-4.9-backport/lib/:/usr/lib:/usr/lib/x86_64-l mkdir build && cd build cmake -DNEBD_DEBUG_ENABLE=ON -DNEBD_COVERAGE_ENABLE=ON .. -make -j24 +make -j24 \ No newline at end of file diff --git a/etc/nebd/nebd-client.conf b/etc/nebd/nebd-client.conf index 2708e328df..07b434389a 100644 --- a/etc/nebd/nebd-client.conf +++ b/etc/nebd/nebd-client.conf @@ -1,49 +1,21 @@ -# nebd part2的进程名字 -part2ProcName=nebd-server -# nebd part2的程序的启动路径，从该路径拉起part2 -part2ProcPath=/usr/bin/nebd-server -# nebd part2的ip地址 -part2Addr=127.0.0.1 -# kill part2服务之后，检查part2服务是否存活的重试次数 -part2KillCheckRetryTimes=150 -# kill part2服务之后，检查part2服务是否存活的重试间隔，单位Us -part2KillCheckRetryIntervalUs=200000 -# nebd part1的qemu进程的名字 -qemuProcName=qemu-system-x86_64 -# nebd part1和part2共用的文件锁文件的路径 -lockFile=/tmp/nebd-server.port.file.lock -# nebd part1和part2共用的元数据文件的目录 -metadataPrefix=/var/run/nebd-server/ -# 拉起part2服务的重试次数 -part2StartRetryTimes=150 -# 拉起part2服务的重试间隔，单位Us -part2StartRetryIntervalUs=200000 -# 检查part2的port是否联通的重试次数 -connectibleCheckTimes=150 -# 检查part2的port是否联通的重试间隔，单位Us -connectibleCheckIntervalUs=200000 -# 获取part2的port的重试次数 -portGetRetryTimes=150 -# 获取part2的port的重试间隔，单位Us -portGetRetryIntervalUs=200000 -# 心跳服务的检查间隔，单位Us -heartbeatIntervalUs=1000000 -# 非io的rpc请求的异常重试次数 -rpcRetryTimes=50 -# rpc请求的重试间隔，包括io请求和非io请求 -rpcRetryIntervalUs=2000000 -# rpc请求的重试间隔，当io请求遇到server服务未启动的时候使用这个重试间隔， -# 这个值一般比rpcRetryIntervalUs更小，为了更快恢复io服务 -rpcHostDownRetryIntervalUs=200000 +# part2 socket file address +nebdserver.serverAddress=/var/run/nebd.sock + +# 文件锁路径 +metacache.fileLockPath=/var/run/nebd-client + +# 同步rpc的最大重试次数 +request.syncRpcMaxRetryTimes=50 +# rpc请求的重试间隔 +request.rpcRetryIntervalUs=100000 # rpc请求的最大重试间隔 -rpcRetryMaxIntervalUs=64000000 -# rpc请求的超时时间，单位Ms -rpcTimeoutMs=30000 +request.rpcRetryMaxIntervalUs=64000000 +# rpc hostdown情况下的重试时间 +request.rpcHostDownRetryIntervalUs=10000 # brpc的健康检查周期时间，单位s -rpcHealthCheckIntervalS=1 -# 异步IO操作RPC失败日志打印间隔，每隔多少次打印一次 -aioRpcFailLogInterval=10 -# 加文件锁的重试次数 -fileLockRetryTimes=150 -# 加文件锁的重试间隔，单位Us -fileLockRetryIntervalUs=200000 \ No newline at end of file +request.rpcHealthCheckIntervalS=1 + +# heartbeat间隔 +heartbeat.intervalS=5 +# heartbeat rpc超时时间 +heartbeat.rpcTimeoutMs=500 \ No newline at end of file diff --git a/get_3rdparty b/get_3rdparty index 651d621d31..165d3b6774 100755 --- a/get_3rdparty +++ b/get_3rdparty @@ -11,9 +11,6 @@ for pkg_ver in boost_1_66_0 googletest_1_8_1 libconfig_1_7_2 jsoncpp_1_8_4 cmock done rm -rf 3rdparty/brpc -cd 3rdparty -git clone -b feature http://gerrit.storage.netease.com/curve/curve-brpc -mv curve-brpc brpc -cd .. +git clone -b feature http://gerrit.storage.netease.com/curve/curve-brpc ./3rdparty/brpc rm -rf 3rd_party_tarballs diff --git a/src/common/configuration.cpp b/src/common/configuration.cpp index 4efe53713a..7434a336c0 100644 --- a/src/common/configuration.cpp +++ b/src/common/configuration.cpp @@ -113,6 +113,15 @@ bool Configuration::GetUInt64Value(const std::string &key, uint64_t *out) { return false; } +bool Configuration::GetInt64Value(const std::string& key, int64_t* out) { + std::string res; + if (GetValue(key, &res)) { + *out = std::stoll(res); + return true; + } + + return false; +} void Configuration::SetIntValue(const std::string &key, const int value) { SetValue(key, std::to_string(value)); diff --git a/src/common/configuration.h b/src/common/configuration.h index aa9678c0eb..b2881210aa 100644 --- a/src/common/configuration.h +++ b/src/common/configuration.h @@ -57,6 +57,7 @@ class Configuration { bool GetIntValue(const std::string &key, int *out); bool GetUInt32Value(const std::string &key, uint32_t *out); bool GetUInt64Value(const std::string &key, uint64_t *out); + bool GetInt64Value(const std::string& key, int64_t* out); void SetIntValue(const std::string &key, const int value); double GetDoubleValue(const std::string &key, double defaultvalue = 0.0); diff --git a/src/common/file_lock.cpp b/src/common/file_lock.cpp new file mode 100644 index 0000000000..25d7c4cc71 --- /dev/null +++ b/src/common/file_lock.cpp @@ -0,0 +1,54 @@ +/** + * Project: nebd + * Create Date: 2020-01-20 + * Author: wuhanqing + * Copyright (c) 2020 netease + */ + +#include "src/common/file_lock.h" + +#include +#include + +namespace nebd { +namespace common { + +const int kBufSize = 128; + +int FileLock::AcquireFileLock() { + char buffer[kBufSize]; + + fd_ = open(fileName_.c_str(), O_CREAT | O_RDONLY, 0644); + if (fd_ < 0) { + LOG(ERROR) << "open file failed, error = " + << strerror_r(errno, buffer, kBufSize) + << ", filename = " << fileName_; + return -1; + } + + int ret = flock(fd_, LOCK_EX | LOCK_NB); + if (ret != 0) { + LOG(ERROR) << "flock failed, error = " + << strerror_r(errno, buffer, kBufSize) + << ", filename = " << fileName_; + close(fd_); + return -1; + } + + return 0; +} + +void FileLock::ReleaseFileLock() { + char buffer[kBufSize]; + int ret = flock(fd_, LOCK_UN); + close(fd_); + if (ret != 0) { + LOG(ERROR) << "release file lock failed, error = " + << strerror_r(errno, buffer, kBufSize) + << ", fd = " << fd_; + } + unlink(fileName_.c_str()); +} + +} // namespace common +} // namespace nebd diff --git a/src/common/file_lock.h b/src/common/file_lock.h new file mode 100644 index 0000000000..0a48862ca2 --- /dev/null +++ b/src/common/file_lock.h @@ -0,0 +1,46 @@ +/** + * Project: nebd + * Create Date: 2020-01-20 + * Author: wuhanqing + * Copyright (c) 2020 netease + */ + +#ifndef SRC_COMMON_FILE_LOCK_H_ +#define SRC_COMMON_FILE_LOCK_H_ + +#include + +namespace nebd { +namespace common { + +// 文件锁 +class FileLock { + public: + explicit FileLock(const std::string fileName) : fileName_(fileName), + fd_(-1) {} + FileLock() : fileName_(""), fd_(-1) {} + ~FileLock() = default; + + /** + * @brief 获取文件锁 + * @return 成功返回0，失败返回-1 + */ + int AcquireFileLock(); + + + /** + * @brief 释放文件锁 + */ + void ReleaseFileLock(); + + private: + // 锁文件的文件名 + std::string fileName_; + // 锁文件的fd + int fd_; +}; + +} // namespace common +} // namespace nebd + +#endif // SRC_COMMON_FILE_LOCK_H_ diff --git a/src/common/heartbeat.proto b/src/common/heartbeat.proto index eebee91a05..5ae0172520 100644 --- a/src/common/heartbeat.proto +++ b/src/common/heartbeat.proto @@ -4,19 +4,20 @@ import "common.proto"; package nebd.client; option cc_generic_services = true; - + message FileInfo { required int32 fd = 1; required string name = 2; } - + message HeartbeatRequest { repeated FileInfo info = 1; } + message HeartbeatResponse { required RetCode retCode = 1; } - + service NebdHeartbeatService { rpc KeepAlive(HeartbeatRequest) returns (HeartbeatResponse); }; \ No newline at end of file diff --git a/src/part1/CMakeLists.txt b/src/part1/CMakeLists.txt index 363ba5c899..66bc793eff 100644 --- a/src/part1/CMakeLists.txt +++ b/src/part1/CMakeLists.txt @@ -1,7 +1,7 @@ # 3rd party headers include_directories("${Jsoncpp_INCLUDE_DIRS}") -set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -std=c++0x -pipe -W -Wall \ +set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -std=c++11 -pipe -W -Wall \ -Wno-unused-parameter -fPIC -fno-omit-frame-pointer \ -momit-leaf-frame-pointer -msse4.2 \ -pthread -Wsign-compare -Wno-unused-parameter \ @@ -25,7 +25,9 @@ set(LIBNEBD_LINK pthread protobuf gflags - jsoncpp_lib) + jsoncpp_lib + client_proto + nebd_common) add_library(nebd SHARED ${LIBNEBD_SRC} ${PROTO_SRCS} ${PROTO_HDRS}) add_dependencies(nebd jsoncpp_lib brpc-shared) target_link_libraries(nebd ${LIBNEBD_LINK} ${EXTRALIBS}) diff --git a/src/part1/async_request_closure.cpp b/src/part1/async_request_closure.cpp new file mode 100644 index 0000000000..ca4b17b4bb --- /dev/null +++ b/src/part1/async_request_closure.cpp @@ -0,0 +1,99 @@ +/** + * Project: nebd + * Create Date: 2020-01-20 + * Author: wuhanqing + * Copyright (c) 2020 netease + */ + +#include "src/part1/async_request_closure.h" + +#include +#include + +#include +#include + +namespace nebd { +namespace client { + +void AsyncRequestClosure::Run() { + std::unique_ptr selfGuard(this); + + if (cntl.Failed()) { + ++aioCtx->retryCount; + int64_t sleepUs = GetRpcRetryIntervalUs(aioCtx->retryCount); + LOG(WARNING) << OpTypeToString(aioCtx->op) << " rpc failed" + << ", error = " << cntl.ErrorText() + << ", fd = " << fd + << ", log id = " << cntl.log_id() + << ", retryCount = " << aioCtx->retryCount + << ", sleep " << (sleepUs / 1000) << " ms"; + bthread_usleep(sleepUs); + Retry(); + } else { + auto retCode = GetResponseRetCode(); + if (nebd::client::RetCode::kOK == retCode) { + DVLOG(6) << OpTypeToString(aioCtx->op) << " success, fd = " << fd; + + // 读请求复制数据 + if (aioCtx->op == LIBAIO_OP::LIBAIO_OP_READ) { + memcpy(aioCtx->buf, + cntl.response_attachment().to_string().c_str(), + cntl.response_attachment().size()); + } + + aioCtx->ret = 0; + aioCtx->cb(aioCtx); + } else { + LOG(ERROR) << OpTypeToString(aioCtx->op) << " failed, fd = " << fd + << ", offset = " << aioCtx->offset + << ", length = " << aioCtx->length + << ", retCode = " << GetResponseRetCode() + << ", log id = " << cntl.log_id(); + aioCtx->ret = -1; + aioCtx->cb(aioCtx); + } + } +} + +int64_t AsyncRequestClosure::GetRpcRetryIntervalUs(int64_t retryCount) const { + if (retryCount == 0) { + return requestOption_.rpcRetryIntervalUs; + } + + // EHOSTDOWN: 找不到可用的server。 + // server可能停止服务了，也可能正在退出中(返回了ELOGOFF) + if (cntl.ErrorCode() == EHOSTDOWN) { + return requestOption_.rpcHostDownRetryIntervalUs; + } + + return std::max( + requestOption_.rpcRetryIntervalUs, + std::min(requestOption_.rpcRetryIntervalUs * retryCount, + requestOption_.rpcRetryMaxIntervalUs)); +} + +void AsyncRequestClosure::Retry() const { + switch (aioCtx->op) { + case LIBAIO_OP::LIBAIO_OP_WRITE: + nebdClient.AioWrite(fd, aioCtx); + break; + case LIBAIO_OP::LIBAIO_OP_READ: + nebdClient.AioRead(fd, aioCtx); + break; + case LIBAIO_OP::LIBAIO_OP_FLUSH: + nebdClient.Flush(fd, aioCtx); + break; + case LIBAIO_OP::LIBAIO_OP_DISCARD: + nebdClient.Discard(fd, aioCtx); + break; + default: + LOG(ERROR) << "Aio Operation Type error, op = " << aioCtx->op + << ", fd = " << fd; + aioCtx->ret = -1; + aioCtx->cb(aioCtx); + } +} + +} // namespace client +} // namespace nebd diff --git a/src/part1/async_request_closure.h b/src/part1/async_request_closure.h new file mode 100644 index 0000000000..9b270365b6 --- /dev/null +++ b/src/part1/async_request_closure.h @@ -0,0 +1,129 @@ +/** + * Project: nebd + * Create Date: 2020-01-20 + * Author: wuhanqing + * Copyright (c) 2020 netease + */ + +#ifndef SRC_PART1_ASYNC_REQUEST_CLOSURE_H_ +#define SRC_PART1_ASYNC_REQUEST_CLOSURE_H_ + +#include + +#include "src/part1/nebd_client.h" +#include "src/part1/nebd_common.h" + +namespace nebd { +namespace client { + +struct AsyncRequestClosure : public google::protobuf::Closure { + AsyncRequestClosure(int fd, + NebdClientAioContext* ctx, + const RequestOption& option) + : fd(fd), + aioCtx(ctx), + requestOption_(option) {} + + void Run() override; + + virtual RetCode GetResponseRetCode() const = 0; + + int64_t GetRpcRetryIntervalUs(int64_t retryCount) const; + + void Retry() const; + + // 请求fd + int fd; + + // 请求上下文信息 + NebdClientAioContext* aioCtx; + + // brpc请求的controller + brpc::Controller cntl; + + RequestOption requestOption_; +}; + +struct AioWriteClosure : public AsyncRequestClosure { + AioWriteClosure(int fd, + NebdClientAioContext* ctx, + const RequestOption& option) + : AsyncRequestClosure( + fd, + ctx, + option) {} + + WriteResponse response; + + RetCode GetResponseRetCode() const override { + return response.retcode(); + } +}; + +struct AioReadClosure : public AsyncRequestClosure { + AioReadClosure(int fd, + NebdClientAioContext* ctx, + const RequestOption& option) + : AsyncRequestClosure( + fd, + ctx, + option) {} + + ReadResponse response; + + RetCode GetResponseRetCode() const override { + return response.retcode(); + } +}; + +struct AioDiscardClosure : public AsyncRequestClosure { + AioDiscardClosure(int fd, + NebdClientAioContext* ctx, + const RequestOption& option) + : AsyncRequestClosure( + fd, + ctx, + option) {} + + DiscardResponse response; + + RetCode GetResponseRetCode() const override { + return response.retcode(); + } +}; + +struct AioFlushClosure : public AsyncRequestClosure { + AioFlushClosure(int fd, + NebdClientAioContext* ctx, + const RequestOption& option) + : AsyncRequestClosure( + fd, + ctx, + option) {} + + FlushResponse response; + + RetCode GetResponseRetCode() const override { + return response.retcode(); + } +}; + +inline const char* OpTypeToString(LIBAIO_OP opType) { + switch (opType) { + case LIBAIO_OP::LIBAIO_OP_READ: + return "Read"; + case LIBAIO_OP::LIBAIO_OP_WRITE: + return "Write"; + case LIBAIO_OP::LIBAIO_OP_DISCARD: + return "Discard"; + case LIBAIO_OP::LIBAIO_OP_FLUSH: + return "Flush"; + default: + return "Unknown"; + } +} + +} // namespace client +} // namespace nebd + +#endif // SRC_PART1_ASYNC_REQUEST_CLOSURE_H_ diff --git a/src/part1/connection_manager.cpp b/src/part1/connection_manager.cpp deleted file mode 100644 index c1b2d4a426..0000000000 --- a/src/part1/connection_manager.cpp +++ /dev/null @@ -1,32 +0,0 @@ - -#include "src/part1/connection_manager.h" - -namespace nebd { -namespace client { - -ConnectionManager::ConnectionManager( - uint32_t heartbeatIntervalS, - std::shared_ptr metaCache) - : heartbeatIntervalS_(heartbeatIntervalS) - , metaCache_(metaCache) { - // TODO -} - -ConnectionManager::~ConnectionManager() { - // TODO -} - -void ConnectionManager::Run() { - // TODO -} - -brpc::Channel* ConnectionManager::GetChannel() { - return channel_; -} - -void ConnectionManager::SendHeartBeat() { - // TODO -} - -} // namespace client -} // namespace nebd \ No newline at end of file diff --git a/src/part1/connection_manager.h b/src/part1/connection_manager.h deleted file mode 100644 index eb99b9d020..0000000000 --- a/src/part1/connection_manager.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef SRC_PART1_CONNECTION_MANAGER_H_ -#define SRC_PART1_CONNECTION_MANAGER_H_ - -#include -#include // NOLINT -#include -#include "src/part1/nebd_metacache.h" - -namespace nebd { -namespace client { - -class ConnectionManager { - public: - ConnectionManager(uint32_t heartbeatIntervalS, - std::shared_ptr metaCache); - ~ConnectionManager(); - void Run(); - brpc::Channel* GetChannel(); - - private: - void SendHeartBeat(); - - private: - brpc::Channel* channel_; - uint32_t heartbeatIntervalS_; - std::shared_ptr metaCache_; - std::thread heartbeatThread_; -}; - -} // namespace client -} // namespace nebd - -#endif // SRC_PART1_CONNECTION_MANAGER_H_ diff --git a/src/part1/heartbeat_manager.cpp b/src/part1/heartbeat_manager.cpp new file mode 100644 index 0000000000..eec637f7de --- /dev/null +++ b/src/part1/heartbeat_manager.cpp @@ -0,0 +1,102 @@ +/** + * Project: nebd + * Create Date: 2020-01-20 + * Author: wuhanqing + * Copyright (c) 2020 netease + */ + +#include "src/part1/heartbeat_manager.h" + +#include +#include +#include + +#include "src/common/heartbeat.pb.h" + +namespace nebd { +namespace client { + +HeartbeatManager::HeartbeatManager( + std::shared_ptr metaCache) + : metaCache_(metaCache) + , running_(false) + , logId_(1) { +} + +int HeartbeatManager::Init(const HeartbeatOption& option) { + heartbeatOption_ = option; + + int ret = channel_.InitWithSockFile( + option.serverAddress.c_str(), nullptr); + if (ret != 0) { + LOG(ERROR) << "Connection Manager channel init failed"; + return -1; + } + + return 0; +} + +void HeartbeatManager::HeartBetaThreadFunc() { + LOG(INFO) << "Heartbeat thread started"; + + while (running_) { + SendHeartBeat(); + sleeper_.wait_for(std::chrono::seconds( + heartbeatOption_.intervalS)); + } + + LOG(INFO) << "Heartbeat thread stopped"; +} + +void HeartbeatManager::Run() { + running_ = true; + heartbeatThread_ = std::thread( + &HeartbeatManager::HeartBetaThreadFunc, this); +} + +void HeartbeatManager::Stop() { + running_ = false; + sleeper_.interrupt(); + heartbeatThread_.join(); + + LOG(INFO) << "Connection Manager stopped success"; +} + +void HeartbeatManager::SendHeartBeat() { + std::vector fileInfos(metaCache_->GetAllFileInfo()); + if (fileInfos.empty()) { + return; + } + + HeartbeatRequest request; + HeartbeatResponse response; + + brpc::Controller cntl; + cntl.set_log_id(logId_.fetch_add(1, std::memory_order_relaxed)); + cntl.set_timeout_ms(heartbeatOption_.rpcTimeoutMs); + + NebdHeartbeatService_Stub stub(&channel_); + + std::ostringstream oss; + for (const auto& fileInfo : fileInfos) { + nebd::client::FileInfo* info = request.add_info(); + info->set_fd(fileInfo.fd); + info->set_name(fileInfo.fileName); + + oss << fileInfo.fd << " : " << fileInfo.fileName << ", "; + } + + LOG(INFO) << "Send Heartbeat request, log id = " << cntl.log_id() + << ", files [" << oss.str() << ']'; + + stub.KeepAlive(&cntl, &request, &response, nullptr); + + if (cntl.Failed()) { + LOG(WARNING) << "Heartbeat request failed, error = " + << cntl.ErrorText() + << ", log id = " << cntl.log_id(); + } +} + +} // namespace client +} // namespace nebd diff --git a/src/part1/heartbeat_manager.h b/src/part1/heartbeat_manager.h new file mode 100644 index 0000000000..3efd3f672a --- /dev/null +++ b/src/part1/heartbeat_manager.h @@ -0,0 +1,77 @@ +/** + * Project: nebd + * Create Date: 2020-01-20 + * Author: wuhanqing + * Copyright (c) 2020 netease + */ + +#ifndef SRC_PART1_HEARTBEAT_MANAGER_H_ +#define SRC_PART1_HEARTBEAT_MANAGER_H_ + +#include + +#include // NOLINT +#include +#include + +#include "src/part1/nebd_common.h" +#include "src/part1/nebd_metacache.h" +#include "src/common/interrupt_sleep.h" + +namespace nebd { +namespace client { + +// Heartbeat 管理类 +// 定期向nebd-server发送已打开文件的心跳信息 +class HeartbeatManager { + public: + explicit HeartbeatManager(std::shared_ptr metaCache); + + ~HeartbeatManager() = default; + + /** + * @brief: 启动心跳线程 + */ + void Run(); + + /** + * @brief: 停止心跳线程 + */ + void Stop(); + + /** + * @brief 初始化 + * @param heartbeatOption heartbeat 配置项 + * @return 0 初始化成功 / -1 初始化失败 + */ + int Init(const HeartbeatOption& option); + + private: + /** + * @brief: 心跳线程执行函数，定期发送心跳消息 + */ + void HeartBetaThreadFunc(); + + /** + * @brief: 向part2发送心跳消息，包括当前已打开的卷信息 + */ + void SendHeartBeat(); + + private: + brpc::Channel channel_; + + HeartbeatOption heartbeatOption_; + + std::shared_ptr metaCache_; + + std::thread heartbeatThread_; + common::InterruptibleSleeper sleeper_; + + std::atomic running_; + std::atomic logId_; +}; + +} // namespace client +} // namespace nebd + +#endif // SRC_PART1_HEARTBEAT_MANAGER_H_ diff --git a/src/part1/nebd_client.cpp b/src/part1/nebd_client.cpp index 636092e030..071a02a9b5 100644 --- a/src/part1/nebd_client.cpp +++ b/src/part1/nebd_client.cpp @@ -6,13 +6,17 @@ */ #include "src/part1/nebd_client.h" + #include +#include #include #include #include #include #include #include + +#include "src/part1/async_request_closure.h" #include "src/common/configuration.h" // 修改brpc的health_check_interval参数，这个参数用来控制健康检查的周期 @@ -28,68 +32,495 @@ namespace brpc { namespace nebd { namespace client { + +using nebd::common::FileLock; + NebdClient &nebdClient = NebdClient::GetInstance(); +constexpr int32_t kBufSize = 128; + int NebdClient::Init(const char* confpath) { - // TODO + nebd::common::Configuration conf; + conf.SetConfigPath(confpath); + + if (!conf.LoadConfig()) { + LOG(ERROR) << "Load config failed, conf path = " << confpath; + return -1; + } + + int ret = InitNebdClientOption(&conf); + if (ret != 0) { + LOG(ERROR) << "InitNebdClientOption failed"; + return -1; + } + + HeartbeatOption heartbeatOption; + ret = InitHeartBeatOption(&conf, &heartbeatOption); + if (ret != 0) { + LOG(ERROR) << "InitHeartBeatOption failed"; + return -1; + } + + LOG(INFO) << "Load config success!"; + + ret = InitChannel(); + if (ret != 0) { + LOG(ERROR) << "InitChannel failed"; + return -1; + } + + metaCache_ = std::make_shared(); + heartbeatMgr_ = std::make_shared( + metaCache_); + + ret = heartbeatMgr_->Init(heartbeatOption); + if (ret != 0) { + LOG(ERROR) << "Heartbeat Manager InitChannel failed"; + return -1; + } + + heartbeatMgr_->Run(); + return 0; } void NebdClient::Uninit() { - // TODO + heartbeatMgr_->Stop(); LOG(INFO) << "NebdClient uninit success."; } int NebdClient::Open(const char* filename) { - // TODO - return -1; + // 加文件锁 + std::string fileLockName = option_.fileLockPath + "/" + filename; + FileLock fileLock(fileLockName); + int res = fileLock.AcquireFileLock(); + if (res < 0) { + LOG(ERROR) << "Open file failed when AcquireFileLock, filename = " + << filename; + return -1; + } + + auto task = [&](brpc::Controller* cntl, + brpc::Channel* channel, + bool* rpcFailed) -> int64_t { + NebdFileService_Stub stub(channel); + OpenFileRequest request; + OpenFileResponse response; + + request.set_filename(filename); + stub.OpenFile(cntl, &request, &response, nullptr); + + *rpcFailed = cntl->Failed(); + if (*rpcFailed) { + LOG(WARNING) << "OpenFile rpc failed, error = " + << cntl->ErrorText() + << ", filename = " << filename + << ", log id = " << cntl->log_id(); + return -1; + } else { + if (response.retcode() != RetCode::kOK) { + LOG(ERROR) << "OpenFile failed, " + << "retcode = " << response.retcode() + <<", retmsg = " << response.retmsg() + << ", filename = " << filename + << ", log id = " << cntl->log_id(); + return -1; + } + + return response.fd(); + } + }; + + int fd = ExecuteSyncRpc(task); + if (fd < 0) { + LOG(ERROR) << "Open file failed, filename = " << filename; + fileLock.ReleaseFileLock(); + return -1; + } + + metaCache_->AddFileInfo({fd, filename, fileLock}); + return fd; } int NebdClient::Close(int fd) { - // TODO - return -1; + auto task = [&](brpc::Controller* cntl, + brpc::Channel* channel, + bool* rpcFailed) -> int64_t { + NebdFileService_Stub stub(channel); + CloseFileRequest request; + CloseFileResponse response; + + request.set_fd(fd); + stub.CloseFile(cntl, &request, &response, nullptr); + + *rpcFailed = cntl->Failed(); + if (*rpcFailed) { + LOG(WARNING) << "CloseFile rpc failed, error = " + << cntl->ErrorText() + << ", log id = " << cntl->log_id(); + return cntl->ErrorCode(); + } else { + if (response.retcode() != RetCode::kOK) { + LOG(ERROR) << "CloseFile failed, " + << "retcode = " << response.retcode() + <<", retmsg = " << response.retmsg() + << ", fd = " << fd + << ", log id = " << cntl->log_id(); + } + + return 0; + } + }; + + int rpcRet = ExecuteSyncRpc(task); + NebdClientFileInfo fileInfo; + int ret = metaCache_->GetFileInfo(fd, &fileInfo); + if (ret == 0) { + fileInfo.fileLock.ReleaseFileLock(); + metaCache_->RemoveFileInfo(fd); + } + + return rpcRet; } int NebdClient::Extend(int fd, int64_t newsize) { - // TODO - return -1; + auto task = [&](brpc::Controller* cntl, + brpc::Channel* channel, + bool* rpcFailed) -> int64_t { + nebd::client::NebdFileService_Stub stub(&channel_); + nebd::client::ResizeRequest request; + nebd::client::ResizeResponse response; + + request.set_fd(fd); + request.set_newsize(newsize); + + stub.ResizeFile(cntl, &request, &response, nullptr); + + *rpcFailed = cntl->Failed(); + if (*rpcFailed) { + LOG(WARNING) << "Resize RPC failed, error = " + << cntl->ErrorText() + << ", log id = " << cntl->log_id(); + return cntl->ErrorCode(); + } else { + if (response.retcode() != nebd::client::RetCode::kOK) { + LOG(ERROR) << "ExtendFile failed, " + << "retcode = " << response.retcode() + <<", retmsg = " << response.retmsg() + << ", fd = " << fd + << ", newsize = " << newsize + << ", log id = " << cntl->log_id(); + return -1; + } else { + return 0; + } + } + }; + + int64_t ret = ExecuteSyncRpc(task); + if (ret < 0) { + LOG(ERROR) << "Extend failed, fd = " << fd + << ", newsize = " << newsize; + } + return ret; } int64_t NebdClient::StatFile(int fd) { - // TODO - return -1; + auto task = [&](brpc::Controller* cntl, + brpc::Channel* channel, + bool* rpcFailed) -> int64_t { + nebd::client::NebdFileService_Stub stub(channel); + nebd::client::StatFileRequest request; + nebd::client::StatFileResponse response; + + request.set_fd(fd); + stub.StatFile(cntl, &request, &response, nullptr); + + *rpcFailed = cntl->Failed(); + if (*rpcFailed) { + LOG(WARNING) << "StateFile rpc faield, error = " + << cntl->ErrorText() + << ", log id = " << cntl->log_id(); + return cntl->ErrorCode(); + } else { + if (response.retcode() != nebd::client::RetCode::kOK) { + LOG(ERROR) << "StatFile failed, " + << "retcode = " << response.retcode() + <<", retmsg = " << response.retmsg() + << ", fd = " << fd + << ", log id = " << cntl->log_id(); + return -1; + } else { + return response.size(); + } + } + }; + + int64_t ret = ExecuteSyncRpc(task); + if (ret < 0) { + LOG(ERROR) << "StatFile failed, fd = " << fd; + } + return ret; } int NebdClient::Discard(int fd, NebdClientAioContext* aioctx) { - // TODO + nebd::client::NebdFileService_Stub stub(&channel_); + nebd::client::DiscardRequest request; + request.set_fd(fd); + request.set_offset(aioctx->offset); + request.set_size(aioctx->length); + + AioDiscardClosure* done = new(std::nothrow) AioDiscardClosure( + fd, aioctx, option_.requestOption); + done->cntl.set_timeout_ms(-1); + done->cntl.set_log_id(logId_.fetch_add(1, std::memory_order_relaxed)); + stub.Discard(&done->cntl, &request, &done->response, done); + return 0; } int NebdClient::AioRead(int fd, NebdClientAioContext* aioctx) { - // TODO + nebd::client::NebdFileService_Stub stub(&channel_); + nebd::client::ReadRequest request; + request.set_fd(fd); + request.set_offset(aioctx->offset); + request.set_size(aioctx->length); + + AioReadClosure* done = new(std::nothrow) AioReadClosure( + fd, aioctx, option_.requestOption); + done->cntl.set_timeout_ms(-1); + done->cntl.set_log_id(logId_.fetch_add(1, std::memory_order_relaxed)); + stub.Read(&done->cntl, &request, &done->response, done); return 0; } int NebdClient::AioWrite(int fd, NebdClientAioContext* aioctx) { - // TODO + nebd::client::NebdFileService_Stub stub(&channel_); + nebd::client::WriteRequest request; + request.set_fd(fd); + request.set_offset(aioctx->offset); + request.set_size(aioctx->length); + + AioWriteClosure* done = new(std::nothrow) AioWriteClosure( + fd, aioctx, option_.requestOption); + + done->cntl.set_timeout_ms(-1); + done->cntl.set_log_id(logId_.fetch_add(1, std::memory_order_relaxed)); + done->cntl.request_attachment().append(aioctx->buf, aioctx->length); + stub.Write(&done->cntl, &request, &done->response, done); + return 0; } int NebdClient::Flush(int fd, NebdClientAioContext* aioctx) { - // TODO + nebd::client::NebdFileService_Stub stub(&channel_); + nebd::client::FlushRequest request; + request.set_fd(fd); + + AioFlushClosure* done = new(std::nothrow) AioFlushClosure( + fd, aioctx, option_.requestOption); + done->cntl.set_timeout_ms(-1); + done->cntl.set_log_id(logId_.fetch_add(1, std::memory_order_relaxed)); + stub.Flush(&done->cntl, &request, &done->response, done); return 0; } int64_t NebdClient::GetInfo(int fd) { - // TODO - return -1; + auto task = [&](brpc::Controller* cntl, + brpc::Channel* channel, + bool* rpcFailed) -> int64_t { + nebd::client::NebdFileService_Stub stub(channel); + nebd::client::GetInfoRequest request; + nebd::client::GetInfoResponse response; + + request.set_fd(fd); + stub.GetInfo(cntl, &request, &response, nullptr); + + *rpcFailed = cntl->Failed(); + if (*rpcFailed) { + LOG(WARNING) << "GetInfo rpc failed, error = " + << cntl->ErrorText() + << ", log id = " << cntl->log_id(); + return cntl->ErrorCode(); + } else { + if (response.retcode() != nebd::client::RetCode::kOK) { + LOG(ERROR) << "GetInfo failed, " + << "retcode = " << response.retcode() + <<", retmsg = " << response.retmsg() + << ", fd = " << fd + << ", log id = " << cntl->log_id(); + return -1; + } else { + return response.objsize(); + } + } + }; + + int64_t ret = ExecuteSyncRpc(task); + if (ret < 0) { + LOG(ERROR) << "GetInfo failed, fd = " << fd; + } + return ret; } int NebdClient::InvalidCache(int fd) { - // TODO - return -1; + auto task = [&](brpc::Controller* cntl, + brpc::Channel* channel, + bool* rpcFailed) -> int64_t { + nebd::client::NebdFileService_Stub stub(channel); + nebd::client::InvalidateCacheRequest request; + nebd::client::InvalidateCacheResponse response; + + request.set_fd(fd); + stub.InvalidateCache(cntl, &request, &response, nullptr); + + *rpcFailed = cntl->Failed(); + if (*rpcFailed) { + LOG(WARNING) << "InvalidCache rpc failed, error = " + << cntl->ErrorText() + << ", log id = " << cntl->log_id(); + return cntl->ErrorCode(); + } else { + if (response.retcode() != nebd::client::RetCode::kOK) { + LOG(ERROR) << "InvalidCache failed, " + << "retcode = " << response.retcode() + <<", retmsg = " << response.retmsg() + << ", fd = " << fd + << ", log id = " << cntl->log_id(); + return -1; + } else { + return 0; + } + } + }; + + int64_t ret = ExecuteSyncRpc(task); + if (ret < 0) { + LOG(ERROR) << "InvalidCache failed, fd = " << fd; + } + return ret; +} + +int NebdClient::InitNebdClientOption(Configuration* conf) { + bool ret = false; + ret = conf->GetStringValue("nebdserver.serverAddress", + &option_.serverAddress); + if (!ret) { + LOG(ERROR) << "Load nebdserver.serverAddress failed"; + return -1; + } + + ret = conf->GetStringValue("metacache.fileLockPath", + &option_.fileLockPath); + if (!ret) { + LOG(ERROR) << "Load metacache.fileLockPath failed"; + return -1; + } + + RequestOption requestOption; + + ret = conf->GetInt64Value("request.syncRpcMaxRetryTimes", + &requestOption.syncRpcMaxRetryTimes); + if (!ret) { + LOG(ERROR) << "Load request.syncRpcMaxRetryTimes failed"; + return -1; + } + + ret = conf->GetInt64Value("request.rpcRetryIntervalUs", + &requestOption.rpcRetryIntervalUs); + if (!ret) { + LOG(ERROR) << "Load request.rpcRetryIntervalUs failed"; + return -1; + } + + ret = conf->GetInt64Value("request.rpcRetryMaxIntervalUs", + &requestOption.rpcRetryMaxIntervalUs); + if (!ret) { + LOG(ERROR) << "Load request.rpcRetryMaxIntervalUs failed"; + return -1; + } + + ret = conf->GetInt64Value("request.rpcHostDownRetryIntervalUs", + &requestOption.rpcHostDownRetryIntervalUs); + if (!ret) { + LOG(ERROR) << "Load request.rpcHostDownRetryIntervalUs failed"; + return -1; + } + + ret = conf->GetInt64Value("request.rpcHealthCheckIntervalS", + &requestOption.rpcHealthCheckIntervalS); + if (!ret) { + LOG(ERROR) << "Load request.rpcHealthCheckIntervalS failed"; + return -1; + } + option_.requestOption = requestOption; + return 0; +} + +int NebdClient::InitHeartBeatOption(Configuration* conf, + HeartbeatOption* heartbeatOption) { + bool ret = conf->GetInt64Value("heartbeat.intervalS", + &heartbeatOption->intervalS); + if (!ret) { + LOG(ERROR) << "Load heartbeat.intervalS failed"; + return -1; + } + + ret = conf->GetInt64Value("heartbeat.rpcTimeoutMs", + &heartbeatOption->rpcTimeoutMs); + if (!ret) { + LOG(ERROR) << "Load heartbeat.rpcTimeoutMs failed"; + return -1; + } + + ret = conf->GetStringValue("nebdserver.serverAddress", + &heartbeatOption->serverAddress); + if (!ret) { + LOG(ERROR) << "Load nebdserver.serverAddress failed"; + return -1; + } + return 0; } +int NebdClient::InitChannel() { + brpc::FLAGS_health_check_interval = + option_.requestOption.rpcHealthCheckIntervalS; + int ret = channel_.InitWithSockFile( + option_.serverAddress.c_str(), nullptr); + if (ret != 0) { + LOG(ERROR) << "Init Channel failed, socket addr = " + << option_.serverAddress; + return -1; + } + + return 0; +} + +int64_t NebdClient::ExecuteSyncRpc(RpcTask task) { + int64_t retryTimes = 0; + int64_t ret = 0; + + while (retryTimes++ < option_.requestOption.syncRpcMaxRetryTimes) { + brpc::Controller cntl; + + cntl.set_timeout_ms(-1); + cntl.set_log_id(logId_.fetch_add(1, std::memory_order_relaxed)); + + bool rpcFailed = false; + ret = task(&cntl, &channel_, &rpcFailed); + if (rpcFailed) { + bthread_usleep(option_.requestOption.rpcRetryIntervalUs); + continue; + } else { + return ret; + } + } + + LOG(ERROR) << "retried " << retryTimes << " times, max retry times " + << option_.requestOption.syncRpcMaxRetryTimes; + + return -1; +} } // namespace client } // namespace nebd diff --git a/src/part1/nebd_client.h b/src/part1/nebd_client.h index 26383afcfa..b82c231a2b 100644 --- a/src/part1/nebd_client.h +++ b/src/part1/nebd_client.h @@ -9,47 +9,63 @@ #define SRC_PART1_NEBD_CLIENT_H_ #include + +#include #include +#include + +#include "src/part1/nebd_common.h" #include "src/common/configuration.h" #include "src/common/client.pb.h" #include "src/part1/libnebd.h" -#include "src/part1/connection_manager.h" +#include "src/part1/heartbeat_manager.h" #include "src/part1/nebd_metacache.h" namespace nebd { namespace client { +using RpcTask = std::function; +using nebd::common::Configuration; + class NebdClient { public: static NebdClient &GetInstance() { static NebdClient client; return client; } + ~NebdClient() = default; + /** * @brief 初始化nebd，仅在第一次调用的时候真正执行初始化逻辑 * @param none * @return 成功返回0，失败返回-1 */ int Init(const char* confpath); + /** * @brief 反初始化nebd * @param none * @return 成功返回0，失败返回-1 */ void Uninit(); + /** * @brief open文件 * @param filename：文件名 * @return 成功返回文件fd，失败返回错误码 */ int Open(const char* filename); + /** * @brief close文件 * @param fd：文件的fd * @return 成功返回0，失败返回错误码 */ int Close(int fd); + /** * @brief resize文件 * @param fd：文件的fd @@ -57,12 +73,14 @@ class NebdClient { * @return 成功返回0，失败返回错误码 */ int Extend(int fd, int64_t newsize); + /** * @brief 获取文件size * @param fd：文件的fd * @return 成功返回文件size，失败返回错误码 */ int64_t StatFile(int fd); + /** * @brief discard文件，异步函数 * @param fd：文件的fd @@ -70,6 +88,7 @@ class NebdClient { * @return 成功返回0，失败返回错误码 */ int Discard(int fd, NebdClientAioContext* aioctx); + /** * @brief 读文件，异步函数 * @param fd：文件的fd @@ -77,6 +96,7 @@ class NebdClient { * @return 成功返回0，失败返回错误码 */ int AioRead(int fd, NebdClientAioContext* aioctx); + /** * @brief 写文件，异步函数 * @param fd：文件的fd @@ -84,6 +104,7 @@ class NebdClient { * @return 成功返回0，失败返回错误码 */ int AioWrite(int fd, NebdClientAioContext* aioctx); + /** * @brief flush文件，异步函数 * @param fd：文件的fd @@ -91,12 +112,14 @@ class NebdClient { * @return 成功返回0，失败返回错误码 */ int Flush(int fd, NebdClientAioContext* aioctx); + /** * @brief 获取文件info * @param fd：文件的fd * @return 成功返回文件对象size，失败返回错误码 */ int64_t GetInfo(int fd); + /** * @brief 刷新cache，等所有异步请求返回 * @param fd：文件的fd @@ -105,9 +128,24 @@ class NebdClient { int InvalidCache(int fd); private: - std::shared_ptr connectionMgr_; - std::shared_ptr metaCache_; - uint32_t rpcRetryIntervalUs_; + int InitNebdClientOption(Configuration* conf); + + int InitHeartBeatOption(Configuration* conf, + HeartbeatOption* hearbeatOption); + + int InitChannel(); + + int64_t ExecuteSyncRpc(RpcTask task); + // 心跳管理模块 + std::shared_ptr heartbeatMgr_; + // 缓存模块 + std::shared_ptr metaCache_; + + NebdClientOption option_; + + brpc::Channel channel_; + + std::atomic logId_{1}; }; extern NebdClient &nebdClient; diff --git a/src/part1/nebd_common.h b/src/part1/nebd_common.h new file mode 100644 index 0000000000..57a2cde51f --- /dev/null +++ b/src/part1/nebd_common.h @@ -0,0 +1,48 @@ +/* + * Project: nebd + * File Created: 2020-02-07 + * Author: wuhanqing + * Copyright (c) 2020 NetEase + */ + +#ifndef SRC_PART1_NEBD_COMMON_H_ +#define SRC_PART1_NEBD_COMMON_H_ + +#include + +// rpc request配置项 +struct RequestOption { + // 同步rpc的最大重试次数 + int64_t syncRpcMaxRetryTimes; + // rpc请求的重试间隔 + int64_t rpcRetryIntervalUs; + // rpc请求的最大重试间隔 + int64_t rpcRetryMaxIntervalUs; + // rpc hostdown情况下的重试时间 + int64_t rpcHostDownRetryIntervalUs; + // brpc的健康检查周期时间 + int64_t rpcHealthCheckIntervalS; +}; + +// nebd client配置项 +struct NebdClientOption { + // part2 socket file address + std::string serverAddress; + // 文件锁路径 + std::string fileLockPath; + // rpc request配置项 + RequestOption requestOption; +}; + + +// heartbeat配置项 +struct HeartbeatOption { + // part2 socket file address + std::string serverAddress; + // heartbeat间隔 + int64_t intervalS; + // heartbeat rpc超时时间 + int64_t rpcTimeoutMs; +}; + +#endif // SRC_PART1_NEBD_COMMON_H_ diff --git a/src/part1/nebd_metacache.cpp b/src/part1/nebd_metacache.cpp index 62d7ac87ff..6c249b9cff 100644 --- a/src/part1/nebd_metacache.cpp +++ b/src/part1/nebd_metacache.cpp @@ -1,22 +1,46 @@ +/** + * Project: nebd + * Create Date: 2020-01-20 + * Author: wuhanqing + * Copyright (c) 2020 netease + */ + #include "src/part1/nebd_metacache.h" namespace nebd { namespace client { -NebdClientMetaCache::NebdClientMetaCache() {} -NebdClientMetaCache::~NebdClientMetaCache() {} +void NebdClientMetaCache::AddFileInfo(const NebdClientFileInfo& fileInfo) { + common::WriteLockGuard guard(rwLock_); + fileinfos_.emplace(fileInfo.fd, fileInfo); +} -void NebdClientMetaCache::AddFileInfo(NebdClientFileInfo fileInfo) { - // TODO +void NebdClientMetaCache::RemoveFileInfo(int fd) { + common::WriteLockGuard guard(rwLock_); + fileinfos_.erase(fd); } -void NebdClientMetaCache::RemoveFileInfo(NebdClientFileInfo fileInfo) { - // TODO +int NebdClientMetaCache::GetFileInfo( + int fd, NebdClientFileInfo* fileInfo) const { + common::ReadLockGuard guard(rwLock_); + auto iter = fileinfos_.find(fd); + if (iter != fileinfos_.end()) { + *fileInfo = iter->second; + return 0; + } + + return -1; } -std::vector NebdClientMetaCache::GetAllFileInfo() { - // TODO +std::vector NebdClientMetaCache::GetAllFileInfo() const { + common::ReadLockGuard guard(rwLock_); std::vector result; + + result.reserve(fileinfos_.size()); + for (const auto& kv : fileinfos_) { + result.push_back(kv.second); + } + return result; } diff --git a/src/part1/nebd_metacache.h b/src/part1/nebd_metacache.h index 92be69432f..b47815c4d2 100644 --- a/src/part1/nebd_metacache.h +++ b/src/part1/nebd_metacache.h @@ -1,30 +1,78 @@ +/** + * Project: nebd + * Create Date: 2020-01-20 + * Author: wuhanqing + * Copyright (c) 2020 netease + */ + #ifndef SRC_PART1_NEBD_METACACHE_H_ #define SRC_PART1_NEBD_METACACHE_H_ #include #include #include + #include "src/common/rw_lock.h" +#include "src/common/file_lock.h" namespace nebd { namespace client { +using nebd::common::FileLock; + struct NebdClientFileInfo { int fd; std::string fileName; + FileLock fileLock; + + NebdClientFileInfo() = default; + + NebdClientFileInfo( + int fd, const std::string& fileName, + const FileLock& fileLock) + : fd(fd), + fileName(fileName), + fileLock(fileLock) {} }; +/** + * @brief: 保存当前已打开文件信息 + */ class NebdClientMetaCache { public: - NebdClientMetaCache(); - ~NebdClientMetaCache(); - void AddFileInfo(NebdClientFileInfo fileInfo); - void RemoveFileInfo(NebdClientFileInfo fileInfo); - std::vector GetAllFileInfo(); + NebdClientMetaCache() = default; + ~NebdClientMetaCache() = default; + + /** + * @brief: 添加文件信息 + * @param: fileInfo 文件信息 + */ + void AddFileInfo(const NebdClientFileInfo& fileInfo); + + /** + * @brief: 删除文件信息 + * @param: fd 文件描述符 + */ + void RemoveFileInfo(int fd); + + /** + * @brief: 获取对应fd的文件信息 + * @param: fd 文件fd + * @param[out]: fileInfo + * @return: 0 成功 / -1 返回 + */ + int GetFileInfo(int fd, NebdClientFileInfo* fileInfo) const; + + /** + * @brief: 获取当前已打开文件信息 + * @return: 当前已打开文件信息 + */ + std::vector GetAllFileInfo() const; private: + // 当前已打开文件信息 std::unordered_map fileinfos_; - common::RWLock rwLock_; + mutable common::RWLock rwLock_; }; } // namespace client diff --git a/src/part2/CMakeLists.txt b/src/part2/CMakeLists.txt index aacb864149..319996c766 100644 --- a/src/part2/CMakeLists.txt +++ b/src/part2/CMakeLists.txt @@ -28,8 +28,6 @@ set(PART2_LINK leveldb client_proto pthread - rados - rbd rt snappy ssl diff --git a/tests/part1/CMakeLists.txt b/tests/part1/CMakeLists.txt index 41e17834ca..34caa1d907 100644 --- a/tests/part1/CMakeLists.txt +++ b/tests/part1/CMakeLists.txt @@ -1,7 +1,7 @@ # 3rd party headers include_directories("${Jsoncpp_INCLUDE_DIRS}") -set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -std=c++0x -pipe -W -Wall \ +set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -std=c++11 -pipe -W -Wall \ -Wno-unused-parameter -fPIC -fno-omit-frame-pointer \ -momit-leaf-frame-pointer -msse4.2 \ -pthread -Wsign-compare -Wno-unused-parameter \ @@ -21,3 +21,37 @@ set(FAKE_CLT_SRV_LINK add_library(fake_file_service SHARED ${FAKE_CLT_SRV_SRC}) target_link_libraries(fake_file_service ${FAKE_CLT_SRV_LINK}) install(TARGETS fake_file_service DESTINATION lib) + +# libfake_heartbeat_service.so +#set(FAKE_HEARTBEAT_SRV_SRC fake_heartbeat_service.h) +#set(FAKE_HEARTBEAT_SRV_LINK +# brpc-shared +# client_proto +# nebd_common +# gflags +# gtest +# gmock) +#add_library(fake_heartbeat_service SHARED ${FAKE_HEARTBEAT_SRV_SRC}) +#target_link_libraries(fake_heartbeat_service ${FAKE_HEARTBEAT_SRV_LINK}) +#install(TARGETS fake_heartbeat_service DESTINATION lib) + +set(MetaCacheTestSrc nebd_metacache_unittest.cpp) +set(MetaCacheTestLink gflags gtest nebd) +add_executable(nebd_metacache_unittest ${MetaCacheTestSrc}) +target_link_libraries(nebd_metacache_unittest ${MetaCacheTestLink}) +install(TARGETS nebd_metacache_unittest DESTINATION bin) +add_test(NAME nebd_metacache_unittest COMMAND nebd_metacache_unittest) + +set(HeartbeatManagerTestSrc heartbeat_manager_unittest.cpp) +set(HeartbeatManagerTestLink gflags gtest nebd) +add_executable(heartbeat_manager_unittest ${HeartbeatManagerTestSrc}) +target_link_libraries(heartbeat_manager_unittest ${HeartbeatManagerTestLink}) +install(TARGETS heartbeat_manager_unittest DESTINATION bin) +add_test(NAME heartbeat_manager_unittest COMMAND heartbeat_manager_unittest) + +set(NebdClientTestSrc nebd_client_unittest.cpp) +set(NebdClientTestLink gflags gtest nebd fake_file_service) +add_executable(nebd_client_unittest ${NebdClientTestSrc}) +target_link_libraries(nebd_client_unittest ${NebdClientTestLink}) +install(TARGETS nebd_client_unittest DESTINATION bin) +add_test(NAME nebd_client_unittest COMMAND nebd_client_unittest) \ No newline at end of file diff --git a/tests/part1/fake_file_service.cpp b/tests/part1/fake_file_service.cpp index d6c8b72ab9..be70655f4d 100644 --- a/tests/part1/fake_file_service.cpp +++ b/tests/part1/fake_file_service.cpp @@ -9,18 +9,18 @@ namespace nebd { namespace client { -uint64_t filesize = 50*1024*1024; // 50MB -char *buf = reinterpret_cast(malloc(filesize)); +const int64_t kBufferSize = 1024; +char buffer[kBufferSize]; -void FileService::OpenFile(::google::protobuf::RpcController* controller, - const ::nebd::client::OpenFileRequest* request, - ::nebd::client::OpenFileResponse* response, - ::google::protobuf::Closure* done) { +void FakeNebdFileService::OpenFile(::google::protobuf::RpcController* controller, // NOLINT + const ::nebd::client::OpenFileRequest* request, + ::nebd::client::OpenFileResponse* response, + ::google::protobuf::Closure* done) { brpc::ClosureGuard doneGuard(done); brpc::Controller* cntl = static_cast(controller); - LOG(INFO) << "logid = " << cntl->log_id() << "OpenFile."; - if (buf == nullptr) { + LOG(INFO) << "logid = " << cntl->log_id() << ", OpenFile."; + if (buffer == nullptr) { response->set_retcode(RetCode::kNoOK); response->set_retmsg("OpenFile FAIL"); return; @@ -33,14 +33,14 @@ void FileService::OpenFile(::google::protobuf::RpcController* controller, return; } -void FileService::CloseFile(::google::protobuf::RpcController* controller, +void FakeNebdFileService::CloseFile(::google::protobuf::RpcController* controller, // NOLINT const ::nebd::client::CloseFileRequest* request, ::nebd::client::CloseFileResponse* response, ::google::protobuf::Closure* done) { brpc::ClosureGuard doneGuard(done); brpc::Controller* cntl = static_cast(controller); - LOG(INFO) << "logid = " << cntl->log_id() << "CloseFile."; + LOG(INFO) << "logid = " << cntl->log_id() << ", CloseFile."; response->set_retcode(RetCode::kOK); response->set_retmsg("CloseFile OK"); @@ -48,48 +48,49 @@ void FileService::CloseFile(::google::protobuf::RpcController* controller, return; } -void FileService::Read(::google::protobuf::RpcController* controller, +void FakeNebdFileService::Read(::google::protobuf::RpcController* controller, const ::nebd::client::ReadRequest* request, ::nebd::client::ReadResponse* response, ::google::protobuf::Closure* done) { brpc::ClosureGuard doneGuard(done); brpc::Controller* cntl = static_cast(controller); - LOG(INFO) << "logid = " << cntl->log_id() << "Read."; + LOG(INFO) << "logid = " << cntl->log_id() << ", Read."; - cntl->response_attachment().append(buf + request->offset(), - request->size()); + cntl->response_attachment().append(buffer + request->offset(), + request->size()); response->set_retcode(RetCode::kOK); response->set_retmsg("Read OK"); return; } -void FileService::Write(::google::protobuf::RpcController* controller, +void FakeNebdFileService::Write(::google::protobuf::RpcController* controller, const ::nebd::client::WriteRequest* request, ::nebd::client::WriteResponse* response, ::google::protobuf::Closure* done) { brpc::ClosureGuard doneGuard(done); brpc::Controller* cntl = static_cast(controller); - LOG(INFO) << "logid = " << cntl->log_id() << "Write."; + LOG(INFO) << "logid = " << cntl->log_id() << ", Write."; - memcpy(buf + request->offset(), - cntl->request_attachment().to_string().c_str(), request->size()); + // memcpy(buffer + request->offset(), + // cntl->request_attachment().to_string().c_str(), + // request->size()); response->set_retcode(RetCode::kOK); response->set_retmsg("Write OK"); return; } -void FileService::Discard(::google::protobuf::RpcController* controller, +void FakeNebdFileService::Discard(::google::protobuf::RpcController* controller, const ::nebd::client::DiscardRequest* request, ::nebd::client::DiscardResponse* response, ::google::protobuf::Closure* done) { brpc::ClosureGuard doneGuard(done); brpc::Controller* cntl = static_cast(controller); - LOG(INFO) << "logid = " << cntl->log_id() << "Discard."; + LOG(INFO) << "logid = " << cntl->log_id() << ", Discard."; response->set_retcode(RetCode::kOK); response->set_retmsg("Discard OK"); @@ -97,45 +98,47 @@ void FileService::Discard(::google::protobuf::RpcController* controller, return; } -void FileService::StatFile(::google::protobuf::RpcController* controller, +void FakeNebdFileService::StatFile(::google::protobuf::RpcController* controller, // NOLINT const ::nebd::client::StatFileRequest* request, ::nebd::client::StatFileResponse* response, ::google::protobuf::Closure* done) { brpc::ClosureGuard doneGuard(done); brpc::Controller* cntl = static_cast(controller); - LOG(INFO) << "logid = " << cntl->log_id() << "StatFile."; + LOG(INFO) << "logid = " << cntl->log_id() << ", StatFile."; response->set_retcode(RetCode::kOK); response->set_retmsg("StatFile OK"); - response->set_size(filesize); + response->set_size(fileSize_); return; } -void FileService::ResizeFile(::google::protobuf::RpcController* controller, +void FakeNebdFileService::ResizeFile(::google::protobuf::RpcController* controller, // NOLINT const ::nebd::client::ResizeRequest* request, ::nebd::client::ResizeResponse* response, ::google::protobuf::Closure* done) { brpc::ClosureGuard doneGuard(done); brpc::Controller* cntl = static_cast(controller); - LOG(INFO) << "logid = " << cntl->log_id() << "ResizeFile."; + LOG(INFO) << "logid = " << cntl->log_id() << ", ResizeFile."; response->set_retcode(RetCode::kOK); response->set_retmsg("ResizeFile OK"); + fileSize_ = request->newsize(); + return; } -void FileService::Flush(::google::protobuf::RpcController* controller, +void FakeNebdFileService::Flush(::google::protobuf::RpcController* controller, const ::nebd::client::FlushRequest* request, ::nebd::client::FlushResponse* response, ::google::protobuf::Closure* done) { brpc::ClosureGuard doneGuard(done); brpc::Controller* cntl = static_cast(controller); - LOG(INFO) << "logid = " << cntl->log_id() << "Flush."; + LOG(INFO) << "logid = " << cntl->log_id() << ", Flush."; response->set_retcode(RetCode::kOK); response->set_retmsg("Flush OK"); @@ -143,23 +146,23 @@ void FileService::Flush(::google::protobuf::RpcController* controller, return; } -void FileService::GetInfo(::google::protobuf::RpcController* controller, +void FakeNebdFileService::GetInfo(::google::protobuf::RpcController* controller, const ::nebd::client::GetInfoRequest* request, ::nebd::client::GetInfoResponse* response, ::google::protobuf::Closure* done) { brpc::ClosureGuard doneGuard(done); brpc::Controller* cntl = static_cast(controller); - LOG(INFO) << "logid = " << cntl->log_id() << "GetInfo."; + LOG(INFO) << "logid = " << cntl->log_id() << ", GetInfo."; response->set_retcode(RetCode::kOK); response->set_retmsg("GetInfo OK"); - response->set_objsize(filesize); + response->set_objsize(fileSize_); return; } -void FileService::InvalidateCache( +void FakeNebdFileService::InvalidateCache( ::google::protobuf::RpcController* controller, const ::nebd::client::InvalidateCacheRequest* request, ::nebd::client::InvalidateCacheResponse* response, @@ -167,7 +170,7 @@ void FileService::InvalidateCache( brpc::ClosureGuard doneGuard(done); brpc::Controller* cntl = static_cast(controller); - LOG(INFO) << "logid = " << cntl->log_id() << "InvalidateCache."; + LOG(INFO) << "logid = " << cntl->log_id() << ", InvalidateCache."; response->set_retcode(RetCode::kOK); response->set_retmsg("InvalidateCache OK"); diff --git a/tests/part1/fake_file_service.h b/tests/part1/fake_file_service.h index 127f06a731..47df681587 100644 --- a/tests/part1/fake_file_service.h +++ b/tests/part1/fake_file_service.h @@ -16,21 +16,21 @@ namespace nebd { namespace client { -class FileService: public NebdFileService { +class FakeNebdFileService: public NebdFileService { public: - FileService() {} + FakeNebdFileService() {} - virtual ~FileService() {} + virtual ~FakeNebdFileService() {} void OpenFile(::google::protobuf::RpcController* controller, - const ::nebd::client::OpenFileRequest* request, - ::nebd::client::OpenFileResponse* response, - ::google::protobuf::Closure* done) override; + const ::nebd::client::OpenFileRequest* request, + ::nebd::client::OpenFileResponse* response, + ::google::protobuf::Closure* done) override; void CloseFile(::google::protobuf::RpcController* controller, - const ::nebd::client::CloseFileRequest* request, - ::nebd::client::CloseFileResponse* response, - ::google::protobuf::Closure* done) override; + const ::nebd::client::CloseFileRequest* request, + ::nebd::client::CloseFileResponse* response, + ::google::protobuf::Closure* done) override; void Read(::google::protobuf::RpcController* controller, const ::nebd::client::ReadRequest* request, @@ -71,6 +71,9 @@ class FileService: public NebdFileService { const ::nebd::client::InvalidateCacheRequest* request, ::nebd::client::InvalidateCacheResponse* response, ::google::protobuf::Closure* done) override; + + private: + int64_t fileSize_; }; } // namespace client } // namespace nebd diff --git a/tests/part1/fake_heartbeat_service.h b/tests/part1/fake_heartbeat_service.h new file mode 100644 index 0000000000..5caa87f3bf --- /dev/null +++ b/tests/part1/fake_heartbeat_service.h @@ -0,0 +1,61 @@ +/** + * Project: nebd + * Create Date: 2020-01-20 + * Author: wuhanqing + * Copyright (c) 2020 netease + */ + +#ifndef TESTS_PART1_FAKE_HEARTBEAT_SERVICE_H_ +#define TESTS_PART1_FAKE_HEARTBEAT_SERVICE_H_ + +#include +#include +#include +#include + +#include "src/common/heartbeat.pb.h" + +namespace nebd { +namespace client { + +class FakeHeartbeatService : public NebdHeartbeatService { + public: + FakeHeartbeatService() = default; + virtual ~FakeHeartbeatService() = default; + + void KeepAlive(::google::protobuf::RpcController* controller, + const ::nebd::client::HeartbeatRequest* request, + ::nebd::client::HeartbeatResponse* response, + ::google::protobuf::Closure* done) override { + brpc::ClosureGuard doneGuard(done); + ++invokeTimes_; + latestFileInfos_.clear(); + + response->set_retcode(RetCode::kOK); + + for (int i = 0; i < request->info_size(); ++i) { + latestFileInfos_.push_back(request->info(i)); + } + } + + std::vector GetLatestRequestFileInfos() const { + return latestFileInfos_; + } + + void ClearInvokeTimes() { + invokeTimes_ = 0; + } + + int GetInvokeTimes() const { + return invokeTimes_; + } + + private: + int invokeTimes_{0}; + std::vector latestFileInfos_; +}; + +} // namespace client +} // namespace nebd + +#endif // TESTS_PART1_FAKE_HEARTBEAT_SERVICE_H_ diff --git a/tests/part1/heartbeat_manager_unittest.cpp b/tests/part1/heartbeat_manager_unittest.cpp new file mode 100644 index 0000000000..7f183aa8d7 --- /dev/null +++ b/tests/part1/heartbeat_manager_unittest.cpp @@ -0,0 +1,158 @@ +/* + * Project: nebd + * Created Date: 2019-08-12 + * Author: hzchenwei7 + * Copyright (c) 2018 netease + */ + +#include +#include +#include + +#include +#include // NOLINT + +#include "src/part1/heartbeat_manager.h" +#include "src/part1/nebd_metacache.h" +#include "tests/part1/fake_heartbeat_service.h" + +namespace nebd { +namespace client { + +const char* kHeartBeatSockFile = "./heartbeat.test.sock"; + +class HeartbeatManagerTest : public testing::Test { + public: + void SetUp() override { + metaCache = std::make_shared(); + manager.reset(new HeartbeatManager(metaCache)); + + int ret = server.AddService(&fakeHeartBeatService, + brpc::SERVER_DOESNT_OWN_SERVICE); + ASSERT_EQ(0, ret) << "AddService heartbeat falied"; + + ret = server.StartAtSockFile(kHeartBeatSockFile, nullptr); + ASSERT_EQ(0, ret) << "Start Server failed"; + + option.intervalS = 1; + option.rpcTimeoutMs = 100; + option.serverAddress = kHeartBeatSockFile; + } + + void TearDown() override { + server.Stop(0); + server.Join(); + } + + std::unique_ptr manager; + std::shared_ptr metaCache; + brpc::Server server; + FakeHeartbeatService fakeHeartBeatService; + HeartbeatOption option; +}; + +TEST_F(HeartbeatManagerTest, InitTest) { + ASSERT_EQ(0, manager->Init( + option)); +} + +TEST_F(HeartbeatManagerTest, InvokeTimesTest) { + ASSERT_EQ(0, manager->Init( + option)); + + manager->Run(); + + // metaCache中数据为空，不发送心跳消息 + for (int i = 0; i < 10; ++i) { + ASSERT_EQ(0, fakeHeartBeatService.GetInvokeTimes()); + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + + // 添加数据 + NebdClientFileInfo fileInfo(1, "/test1", FileLock("/test1.lock")); + metaCache->AddFileInfo(fileInfo); + + std::this_thread::sleep_for(std::chrono::seconds(10)); + int times = fakeHeartBeatService.GetInvokeTimes(); + ASSERT_TRUE(times >= 9 && times <= 11); + + // 清空metaCache数据 + metaCache->RemoveFileInfo(1); + std::this_thread::sleep_for(std::chrono::seconds(2)); + + fakeHeartBeatService.ClearInvokeTimes(); + std::this_thread::sleep_for(std::chrono::seconds(5)); + ASSERT_EQ(0, fakeHeartBeatService.GetInvokeTimes()); + + manager->Stop(); +} + +TEST_F(HeartbeatManagerTest, RequestValidTest) { + ASSERT_EQ(0, manager->Init( + option)); + manager->Run(); + + std::vector currentFileInfos; + + // 添加一个文件 + NebdClientFileInfo fileInfo(1, "/test1", FileLock("/test1.lock")); + metaCache->AddFileInfo(fileInfo); + FileInfo info; + info.set_fd(1); + info.set_name("/test1"); + currentFileInfos.push_back(info); + + std::this_thread::sleep_for(std::chrono::seconds(2)); + auto latestFileInfos = fakeHeartBeatService.GetLatestRequestFileInfos(); + ASSERT_EQ(1, latestFileInfos.size()); + + for (int i = 0; i < currentFileInfos.size(); ++i) { + ASSERT_EQ(currentFileInfos[i].fd(), latestFileInfos[i].fd()); + ASSERT_EQ(currentFileInfos[i].name(), latestFileInfos[i].name()); + } + + // 添加第二个文件 + fileInfo = NebdClientFileInfo(2, "/test2", FileLock("/test2.lock")); + metaCache->AddFileInfo(fileInfo); + info.set_fd(2); + info.set_name("/test2"); + currentFileInfos.push_back(info); + + std::this_thread::sleep_for(std::chrono::seconds(2)); + latestFileInfos = fakeHeartBeatService.GetLatestRequestFileInfos(); + ASSERT_EQ(2, latestFileInfos.size()); + + std::sort(latestFileInfos.begin(), latestFileInfos.end(), + [](const FileInfo& lhs, const FileInfo& rhs) { + return lhs.fd() < rhs.fd(); + }); + + for (int i = 0; i < currentFileInfos.size(); ++i) { + ASSERT_EQ(currentFileInfos[i].fd(), latestFileInfos[i].fd()); + ASSERT_EQ(currentFileInfos[i].name(), latestFileInfos[i].name()); + } + + // 删除第一个文件 + metaCache->RemoveFileInfo(1); + currentFileInfos.erase(currentFileInfos.begin()); + + std::this_thread::sleep_for(std::chrono::seconds(2)); + latestFileInfos = fakeHeartBeatService.GetLatestRequestFileInfos(); + ASSERT_EQ(1, latestFileInfos.size()); + + for (int i = 0; i < currentFileInfos.size(); ++i) { + ASSERT_EQ(currentFileInfos[i].fd(), latestFileInfos[i].fd()); + ASSERT_EQ(currentFileInfos[i].name(), latestFileInfos[i].name()); + } + + manager->Stop(); +} + +} // namespace client +} // namespace nebd + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/tests/part1/mock_file_service.h b/tests/part1/mock_file_service.h index 86032a12b2..4723ec1efc 100644 --- a/tests/part1/mock_file_service.h +++ b/tests/part1/mock_file_service.h @@ -5,8 +5,8 @@ * Copyright (c) 2018 netease */ -#ifndef TESTS_PART1_MOCK_CLIENT_SERVICE_H_ -#define TESTS_PART1_MOCK_CLIENT_SERVICE_H_ +#ifndef TESTS_PART1_MOCK_FILE_SERVICE_H_ +#define TESTS_PART1_MOCK_FILE_SERVICE_H_ #include #include @@ -66,4 +66,4 @@ class MockNebdFileService : public NebdFileService { } // namespace client } // namespace nebd -#endif // TESTS_PART1_MOCK_CLIENT_SERVICE_H_ +#endif // TESTS_PART1_MOCK_FILE_SERVICE_H_ diff --git a/tests/part1/mock_heartbeat_service.h b/tests/part1/mock_heartbeat_service.h new file mode 100644 index 0000000000..49d1946dd8 --- /dev/null +++ b/tests/part1/mock_heartbeat_service.h @@ -0,0 +1,30 @@ +/** + * Project: nebd + * Created Date: 2020-01-20 + * Author: wuhanqing + * Copyright (c) 2019 netease + */ + +#ifndef TESTS_PART1_MOCK_HEARTBEAT_SERVICE_H_ +#define TESTS_PART1_MOCK_HEARTBEAT_SERVICE_H_ + +#include +#include +#include +#include "src/common/heartbeat.pb.h" + +namespace nebd { +namespace client { + +class MockHeartBeatService : public NebdHeartbeatService { + public: + MOCK_METHOD4(KeepAlive, void(::google::protobuf::RpcController* cntl, + const HeartbeatRequest* request, + HeartbeatResponse* response, + ::google::protobuf::Closure* done)); +}; + +} // namespace client +} // namespace nebd + +#endif // TESTS_PART1_MOCK_HEARTBEAT_SERVICE_H_ diff --git a/tests/part1/nebd-client-aio-test.conf b/tests/part1/nebd-client-aio-test.conf new file mode 100644 index 0000000000..a06a25f651 --- /dev/null +++ b/tests/part1/nebd-client-aio-test.conf @@ -0,0 +1,19 @@ +# part2 socket file address +nebdServerAddress=./nebd.aiotest.sock +# 文件锁路径 +fileLockPath=./tests +# 同步rpc的最大重试次数 +syncRpcMaxRetryTimes=10 +# rpc请求的重试间隔 +rpcRetryIntervalUs=100000 +# rpc请求的最大重试间隔 +rpcRetryMaxIntervalUs=64000000 +# rpc hostdown情况下的重试时间 +rpcHostDownRetryIntervalUs=10000 +# brpc的健康检查周期时间，单位s +rpcHealthCheckIntervalS=1 + +# heartbeat间隔 +heartbeatIntervalS=5 +# heartbeat rpc超时时间 +heartbeatRpcTimeoutMs=500 \ No newline at end of file diff --git a/tests/part1/nebd-client.conf b/tests/part1/nebd-client.conf new file mode 100644 index 0000000000..163e6bddf7 --- /dev/null +++ b/tests/part1/nebd-client.conf @@ -0,0 +1,19 @@ +# part2 socket file address +nebdserver.serverAddress=/root/nebd/nebd.test.sock +# 文件锁路径 +metacache.fileLockPath=/root/nebd/tests +# 同步rpc的最大重试次数 +request.syncRpcMaxRetryTimes=10 +# rpc请求的重试间隔 +request.rpcRetryIntervalUs=100000 +# rpc请求的最大重试间隔 +request.rpcRetryMaxIntervalUs=64000000 +# rpc hostdown情况下的重试时间 +request.rpcHostDownRetryIntervalUs=10000 +# brpc的健康检查周期时间，单位s +request.rpcHealthCheckIntervalS=1 + +# heartbeat间隔 +heartbeat.intervalS=5 +# heartbeat rpc超时时间 +heartbeat.rpcTimeoutMs=500 \ No newline at end of file diff --git a/tests/part1/nebd_client_unittest.cpp b/tests/part1/nebd_client_unittest.cpp new file mode 100644 index 0000000000..bcb844cc7a --- /dev/null +++ b/tests/part1/nebd_client_unittest.cpp @@ -0,0 +1,547 @@ +/** + * Project: nebd + * Create Date: 2020-01-20 + * Author: wuhanqing + * Copyright (c) 2020 netease + */ + +#include +#include +#include + +#include // NOLINT +#include // NOLINT +#include + +#include "src/part1/nebd_client.h" +#include "src/part1/libnebd.h" +#include "src/part1/libnebd_file.h" + +#include "tests/part1/fake_file_service.h" +#include "tests/part1/mock_file_service.h" + +namespace nebd { +namespace client { + +const char* kFileName = "nebd-test-filename"; +const char* kNebdServerTestAddress = "/root/nebd/nebd.test.sock"; +const char* kNebdClientConf = "/root/nebd/tests/part1/nebd-client.conf"; +const int64_t kFileSize = 10LL * 1024 * 1024 * 1024; +const int64_t kBufSize = 1024; + +std::mutex mtx; +std::condition_variable cond; +std::atomic aioOpReturn{false}; + +void AioCallBack(NebdClientAioContext* ctx) { + ASSERT_EQ(0, ctx->ret); + ASSERT_EQ(0, ctx->retryCount); + std::lock_guard lk(mtx); + aioOpReturn = true; + cond.notify_one(); + delete ctx; +} + +void AioResponseFailCallBack(NebdClientAioContext* ctx) { + ASSERT_EQ(-1, ctx->ret); + ASSERT_EQ(0, ctx->retryCount); + std::lock_guard lk(mtx); + aioOpReturn = true; + cond.notify_one(); + delete ctx; +} + +void AioRpcFailCallBack(NebdClientAioContext* ctx) { + ASSERT_EQ(0, ctx->ret); + + std::lock_guard lk(mtx); + aioOpReturn = true; + cond.notify_one(); + delete ctx; +} + +template +void MockClientFunc(google::protobuf::RpcController* cntl_base, + const Request* request, + Response* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); +} + +template +void MockClientRpcFailedFunc(google::protobuf::RpcController* cntl_base, + const Request* request, + Response* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard doneGuard(done); + static int invokeTimes = 0; + ++invokeTimes; + + if (invokeTimes < 10) { + brpc::Controller* cntl = static_cast(cntl_base); + cntl->SetFailed(RpcErrCode, "failed"); + } else { + LOG(INFO) << "invoke 10 times, set response"; + response->set_retcode(RetCode::kOK); + } +} + +class NebdFileClientTest : public ::testing::Test { + public: + void SetUp() override {} + + void TearDown() override {} + + void AddFakeService() { + ASSERT_EQ(0, server.AddService( + &fakeService, + brpc::SERVER_DOESNT_OWN_SERVICE)) << "Add service failed"; + } + + void AddMockService() { + ASSERT_EQ(0, server.AddService( + &mockService, + brpc::SERVER_DOESNT_OWN_SERVICE)) << "Add service failed"; + } + + void StartServer(const std::string& address = kNebdServerTestAddress) { + ASSERT_EQ(0, server.StartAtSockFile( + address.c_str(), nullptr)) << "Start server failed"; + } + + void StopServer() { + server.Stop(0); + server.Join(); + } + + brpc::Server server; + FakeNebdFileService fakeService; + MockNebdFileService mockService; +}; + +using ::testing::_; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::AnyNumber; +using ::testing::DoAll; +using ::testing::SetArgPointee; +using ::testing::SetArgReferee; +using ::testing::InSequence; +using ::testing::AtLeast; +using ::testing::SaveArgPointee; + +TEST_F(NebdFileClientTest, AioRpcFailTest) { + AddMockService(); + StartServer(); + ASSERT_EQ(0, Init4Nebd(kNebdClientConf)); + + char buffer[kBufSize]; + + { + NebdClientAioContext* ctx = new NebdClientAioContext(); + ctx->buf = buffer; + ctx->offset = 0; + ctx->length = kBufSize; + ctx->ret = 0; + ctx->op = LIBAIO_OP_WRITE; + ctx->cb = AioRpcFailCallBack; + ctx->retryCount = 0; + + EXPECT_CALL(mockService, Write(_, _, _, _)) + .Times(10) + .WillRepeatedly( + Invoke(MockClientRpcFailedFunc)); // NOLINT + + aioOpReturn = false; + auto start = std::chrono::system_clock::now(); + ASSERT_EQ(0, AioWrite4Nebd(1, ctx)); + + std::unique_lock ulk(mtx); + cond.wait(ulk, []() { return aioOpReturn.load(); }); + ASSERT_TRUE(aioOpReturn.load()); + auto end = std::chrono::system_clock::now(); + auto elpased = std::chrono::duration_cast(end - start).count(); // NOLINT + + // 重试睡眠时间: 100ms + 200ms + ... + 900ms = 4500ms + ASSERT_TRUE(elpased >= 4000 && elpased <= 5000); + } + + { + NebdClientAioContext* ctx = new NebdClientAioContext(); + ctx->buf = buffer; + ctx->offset = 0; + ctx->length = kBufSize; + ctx->ret = 0; + ctx->op = LIBAIO_OP_READ; + ctx->cb = AioRpcFailCallBack; + ctx->retryCount = 0; + + EXPECT_CALL(mockService, Read(_, _, _, _)) + .Times(10) + .WillRepeatedly( + Invoke(MockClientRpcFailedFunc)); // NOLINT + aioOpReturn = false; + ASSERT_EQ(0, AioRead4Nebd(1, ctx)); + + std::unique_lock ulk(mtx); + cond.wait(ulk, []() { return aioOpReturn.load(); }); + ASSERT_TRUE(aioOpReturn.load()); + } + + { + NebdClientAioContext* ctx = new NebdClientAioContext(); + ctx->buf = 0; + ctx->offset = 0; + ctx->length = kBufSize; + ctx->ret = 0; + ctx->op = LIBAIO_OP_DISCARD; + ctx->cb = AioRpcFailCallBack; + ctx->retryCount = 0; + + EXPECT_CALL(mockService, Discard(_, _, _, _)) + .Times(10) + .WillRepeatedly( + Invoke(MockClientRpcFailedFunc)); // NOLINT + aioOpReturn = false; + ASSERT_EQ(0, Discard4Nebd(1, ctx)); + + std::unique_lock ulk(mtx); + cond.wait(ulk, []() { return aioOpReturn.load(); }); + ASSERT_TRUE(aioOpReturn.load()); + } + + { + NebdClientAioContext* ctx = new NebdClientAioContext(); + ctx->buf = 0; + ctx->offset = 0; + ctx->length = kBufSize; + ctx->ret = 0; + ctx->op = LIBAIO_OP_FLUSH; + ctx->cb = AioRpcFailCallBack; + ctx->retryCount = 0; + + EXPECT_CALL(mockService, Flush(_, _, _, _)) + .Times(10) + .WillRepeatedly( + Invoke(MockClientRpcFailedFunc)); // NOLINT + aioOpReturn = false; + ASSERT_EQ(0, Flush4Nebd(1, ctx)); + + std::unique_lock ulk(mtx); + cond.wait(ulk, []() { return aioOpReturn.load(); }); + ASSERT_TRUE(aioOpReturn.load()); + } + + ASSERT_NO_THROW(Uninit4Nebd()); + StopServer(); +} + +TEST_F(NebdFileClientTest, NoNebdServerTest) { + ASSERT_EQ(-1, Init4Nebd("/xxx/nebd-client.conf")); + ASSERT_EQ(0, Init4Nebd(kNebdClientConf)); + + { + auto start = std::chrono::system_clock::now(); + ASSERT_EQ(-1, Open4Nebd(kFileName)); + auto end = std::chrono::system_clock::now(); + auto elapsed = std::chrono::duration_cast( + end - start).count(); + + // rpc failed的清空下，睡眠100ms后继续重试，共重试10次 + ASSERT_TRUE(elapsed >= 900 && elapsed <= 1100); + } + ASSERT_EQ(-1, Extend4Nebd(1, kFileSize)); + ASSERT_EQ(-1, StatFile4Nebd(1)); + ASSERT_EQ(-1, GetInfo4Nebd(1)); + ASSERT_EQ(-1, InvalidCache4Nebd(1)); + ASSERT_EQ(-1, Close4Nebd(1)); + + ASSERT_NO_THROW(Uninit4Nebd()); +} + +TEST_F(NebdFileClientTest, CommonTest) { + AddFakeService(); + StartServer(); + + ASSERT_EQ(0, Init4Nebd(kNebdClientConf)); + + int fd = Open4Nebd(kFileName); + ASSERT_GE(fd, 0); + + ASSERT_EQ(0, Extend4Nebd(fd, kFileSize)); + ASSERT_EQ(kFileSize, StatFile4Nebd(fd)); + ASSERT_EQ(kFileSize, GetInfo4Nebd(fd)); + ASSERT_EQ(0, InvalidCache4Nebd(fd)); + + char buffer[kBufSize]; + + { + NebdClientAioContext* ctx = new NebdClientAioContext(); + ctx->buf = buffer; + ctx->offset = 0; + ctx->length = kBufSize; + ctx->ret = 0; + ctx->op = LIBAIO_OP_WRITE; + ctx->cb = AioCallBack; + ctx->retryCount = 0; + + aioOpReturn = false; + ASSERT_EQ(0, AioWrite4Nebd(fd, ctx)); + std::unique_lock ulk(mtx); + cond.wait(ulk, []() { return aioOpReturn.load(); }); + ASSERT_TRUE(aioOpReturn.load()); + } + + { + NebdClientAioContext* ctx = new NebdClientAioContext(); + ctx->buf = buffer; + ctx->offset = 0; + ctx->length = kBufSize; + ctx->ret = 0; + ctx->op = LIBAIO_OP_READ; + ctx->cb = AioCallBack; + ctx->retryCount = 0; + + aioOpReturn = false; + ASSERT_EQ(0, AioRead4Nebd(fd, ctx)); + std::unique_lock ulk2(mtx); + cond.wait(ulk2, []() { return aioOpReturn.load(); }); + ASSERT_TRUE(aioOpReturn.load()); + } + + { + NebdClientAioContext* ctx = new NebdClientAioContext(); + ctx->buf = 0; + ctx->offset = 0; + ctx->length = kBufSize; + ctx->ret = 0; + ctx->op = LIBAIO_OP_DISCARD; + ctx->cb = AioCallBack; + ctx->retryCount = 0; + + aioOpReturn = false; + ASSERT_EQ(0, Discard4Nebd(fd, ctx)); + std::unique_lock ulk2(mtx); + cond.wait(ulk2, []() { return aioOpReturn.load(); }); + ASSERT_TRUE(aioOpReturn.load()); + } + + { + NebdClientAioContext* ctx = new NebdClientAioContext(); + ctx->buf = 0; + ctx->offset = 0; + ctx->length = kBufSize; + ctx->ret = 0; + ctx->op = LIBAIO_OP_FLUSH; + ctx->cb = AioCallBack; + ctx->retryCount = 0; + + aioOpReturn = false; + ASSERT_EQ(0, Flush4Nebd(fd, ctx)); + std::unique_lock ulk2(mtx); + cond.wait(ulk2, []() { return aioOpReturn.load(); }); + ASSERT_TRUE(aioOpReturn.load()); + } + + ASSERT_EQ(0, Close4Nebd(fd)); + ASSERT_NO_THROW(Uninit4Nebd()); + StopServer(); +} + +TEST_F(NebdFileClientTest, ReOpenTest) { + AddFakeService(); + StartServer(); + + ASSERT_EQ(0, Init4Nebd(kNebdClientConf)); + + int fd = Open4Nebd(kFileName); + ASSERT_GT(fd, 0); + + // 文件已经被打开，并占用文件锁 + // 再次打开时，获取文件锁失败，直接返回 + ASSERT_EQ(-1, Open4Nebd(kFileName)); + + ASSERT_EQ(0, Close4Nebd(fd)); + + fd = Open4Nebd(kFileName); + ASSERT_GT(fd, 0); + ASSERT_EQ(0, Close4Nebd(fd)); + + ASSERT_NO_THROW(Uninit4Nebd()); + + StopServer(); +} + +TEST_F(NebdFileClientTest, ResponseFailTest) { + AddMockService(); + StartServer(); + + ASSERT_EQ(0, Init4Nebd(kNebdClientConf)); + + { + OpenFileResponse response; + response.set_retcode(RetCode::kNoOK); + EXPECT_CALL(mockService, OpenFile(_, _, _, _)) + .Times(1) + .WillOnce(DoAll( + SetArgPointee<2>(response), + Invoke(MockClientFunc))); // NOLINT + ASSERT_EQ(-1, Open4Nebd(kFileName)); + } + + { + ResizeResponse response; + response.set_retcode(RetCode::kNoOK); + EXPECT_CALL(mockService, ResizeFile(_, _, _, _)) + .Times(1) + .WillOnce(DoAll( + SetArgPointee<2>(response), + Invoke(MockClientFunc))); + ASSERT_EQ(-1, Extend4Nebd(1, kFileSize)); + } + + { + StatFileResponse response; + response.set_retcode(RetCode::kNoOK); + EXPECT_CALL(mockService, StatFile(_, _, _, _)) + .Times(1) + .WillOnce(DoAll( + SetArgPointee<2>(response), + Invoke(MockClientFunc))); // NOLINT + ASSERT_EQ(-1, StatFile4Nebd(1)); + } + + { + GetInfoResponse response; + response.set_retcode(RetCode::kNoOK); + EXPECT_CALL(mockService, GetInfo(_, _, _, _)) + .Times(1) + .WillOnce(DoAll( + SetArgPointee<2>(response), + Invoke(MockClientFunc))); // NOLINT + ASSERT_EQ(-1, GetInfo4Nebd(1)); + } + + { + InvalidateCacheResponse response; + response.set_retcode(RetCode::kNoOK); + EXPECT_CALL(mockService, InvalidateCache(_, _, _, _)) + .Times(1) + .WillOnce(DoAll( + SetArgPointee<2>(response), + Invoke(MockClientFunc))); // NOLINT + ASSERT_EQ(-1, InvalidCache4Nebd(1)); + } + + char buffer[kBufSize]; + + { + NebdClientAioContext* ctx = new NebdClientAioContext(); + ctx->buf = buffer; + ctx->offset = 0; + ctx->length = kBufSize; + ctx->ret = 0; + ctx->op = LIBAIO_OP_WRITE; + ctx->cb = AioResponseFailCallBack; + ctx->retryCount = 0; + + WriteResponse response; + response.set_retcode(RetCode::kNoOK); + EXPECT_CALL(mockService, Write(_, _, _, _)) + .Times(1) + .WillOnce(DoAll( + SetArgPointee<2>(response), + Invoke(MockClientFunc))); // NOLINT + aioOpReturn = false; + ASSERT_EQ(0, AioWrite4Nebd(1, ctx)); + std::unique_lock ulk(mtx); + cond.wait(ulk, []() { return aioOpReturn.load(); }); + ASSERT_TRUE(aioOpReturn.load()); + } + + { + NebdClientAioContext* ctx = new NebdClientAioContext(); + ctx->buf = buffer; + ctx->offset = 0; + ctx->length = kBufSize; + ctx->ret = 0; + ctx->op = LIBAIO_OP_READ; + ctx->cb = AioResponseFailCallBack; + ctx->retryCount = 0; + + ReadResponse response; + response.set_retcode(RetCode::kNoOK); + EXPECT_CALL(mockService, Read(_, _, _, _)) + .Times(1) + .WillOnce(DoAll( + SetArgPointee<2>(response), + Invoke(MockClientFunc))); + aioOpReturn = false; + ASSERT_EQ(0, AioRead4Nebd(1, ctx)); + std::unique_lock ulk(mtx); + cond.wait(ulk, []() { return aioOpReturn.load(); }); + ASSERT_TRUE(aioOpReturn.load()); + } + + { + NebdClientAioContext* ctx = new NebdClientAioContext(); + ctx->buf = 0; + ctx->offset = 0; + ctx->length = kBufSize; + ctx->ret = 0; + ctx->op = LIBAIO_OP_DISCARD; + ctx->cb = AioResponseFailCallBack; + ctx->retryCount = 0; + + DiscardResponse response; + response.set_retcode(RetCode::kNoOK); + EXPECT_CALL(mockService, Discard(_, _, _, _)) + .Times(1) + .WillOnce(DoAll( + SetArgPointee<2>(response), + Invoke(MockClientFunc))); // NOLINT + aioOpReturn = false; + ASSERT_EQ(0, Discard4Nebd(1, ctx)); + std::unique_lock ulk(mtx); + cond.wait(ulk, []() { return aioOpReturn.load(); }); + ASSERT_TRUE(aioOpReturn.load()); + } + + { + NebdClientAioContext* ctx = new NebdClientAioContext(); + ctx->buf = 0; + ctx->offset = 0; + ctx->length = kBufSize; + ctx->ret = 0; + ctx->op = LIBAIO_OP_FLUSH; + ctx->cb = AioResponseFailCallBack; + ctx->retryCount = 0; + + FlushResponse response; + response.set_retcode(RetCode::kNoOK); + EXPECT_CALL(mockService, Flush(_, _, _, _)) + .Times(1) + .WillOnce(DoAll( + SetArgPointee<2>(response), + Invoke(MockClientFunc))); + aioOpReturn = false; + ASSERT_EQ(0, Flush4Nebd(1, ctx)); + std::unique_lock ulk(mtx); + cond.wait(ulk, []() { return aioOpReturn.load(); }); + ASSERT_TRUE(aioOpReturn.load()); + } + + ASSERT_NO_THROW(Uninit4Nebd()); + StopServer(); +} + + + +} // namespace client +} // namespace nebd + + +int main(int argc, char* argv[]) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/part1/nebd_metacache_unittest.cpp b/tests/part1/nebd_metacache_unittest.cpp new file mode 100644 index 0000000000..e586e09c7e --- /dev/null +++ b/tests/part1/nebd_metacache_unittest.cpp @@ -0,0 +1,81 @@ +/** + * Project: nebd + * Create Date: 2020-01-20 + * Author: wuhanqing + * Copyright (c) 2020 netease + */ + +#include "src/part1/nebd_metacache.h" + +#include +#include + +namespace nebd { +namespace client { + +TEST(MetaCacheTest, AddTest) { + NebdClientMetaCache metaCache; + ASSERT_NO_THROW(metaCache.AddFileInfo({1, "/file1", + FileLock("/file1.lock")})); + ASSERT_NO_THROW(metaCache.AddFileInfo({2, "/file2", + FileLock("/file2.lock")})); + ASSERT_NO_THROW(metaCache.AddFileInfo({1, "/flie3", + FileLock("/file4.lock")})); +} + +TEST(MetaCacheTest, RemoveTest) { + NebdClientMetaCache metaCache; + ASSERT_NO_THROW(metaCache.AddFileInfo({1, "/file1", + FileLock("/file1.lock")})); + ASSERT_NO_THROW(metaCache.RemoveFileInfo(1)); + ASSERT_NO_THROW(metaCache.RemoveFileInfo(2)); +} + +TEST(MetaCacheTest, GetTest) { + NebdClientMetaCache metaCache; + ASSERT_NO_THROW(metaCache.AddFileInfo({1, "/file1", + FileLock("/file1.lock")})); + + NebdClientFileInfo fileInfo; + ASSERT_EQ(0, metaCache.GetFileInfo(1, &fileInfo)); + ASSERT_EQ(1, fileInfo.fd); + ASSERT_STREQ("/file1", fileInfo.fileName.c_str()); + + ASSERT_EQ(-1, metaCache.GetFileInfo(2, &fileInfo)); +} + +TEST(MetaCacheTest, GetAllTest) { + NebdClientMetaCache metaCache; + + ASSERT_EQ(0, metaCache.GetAllFileInfo().size()); + + ASSERT_NO_THROW(metaCache.AddFileInfo({1, "/file1", + FileLock("/file1.lock")})); + ASSERT_EQ(1, metaCache.GetAllFileInfo().size()); + + ASSERT_NO_THROW(metaCache.AddFileInfo({1, "/file1", + FileLock("/file1.lock")})); + ASSERT_EQ(1, metaCache.GetAllFileInfo().size()); + + ASSERT_NO_THROW(metaCache.AddFileInfo({2, "/file2", + FileLock("/file2.lock")})); + ASSERT_EQ(2, metaCache.GetAllFileInfo().size()); + + NebdClientFileInfo fileInfo; + ASSERT_NO_THROW(metaCache.RemoveFileInfo(2)); + ASSERT_EQ(1, metaCache.GetAllFileInfo().size()); + + ASSERT_NO_THROW(metaCache.RemoveFileInfo(2)); + ASSERT_EQ(1, metaCache.GetAllFileInfo().size()); + + ASSERT_NO_THROW(metaCache.RemoveFileInfo(1)); + ASSERT_EQ(0, metaCache.GetAllFileInfo().size()); +} + +} // namespace client +} // namespace nebd + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 41af48547628e9b83a8763779edd589991855ce3 Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Wed, 12 Feb 2020 15:53:32 +0800 Subject: [PATCH 27/79] fix compile error Change-Id: Ib3f48352e557dffe24870e4daa797aa2393d2a3f --- src/common/file_lock.cpp | 2 +- src/part1/async_request_closure.cpp | 2 +- src/part2/CMakeLists.txt | 2 +- src/part2/define.h | 9 ++++----- src/part2/file_manager.cpp | 10 ---------- src/part2/file_service.cpp | 2 +- src/part2/request_executor_curve.cpp | 2 +- src/part2/request_executor_curve.h | 2 +- src/part2/util.cpp | 2 +- tests/part2/test_request_executor_curve.cpp | 4 ++-- 10 files changed, 13 insertions(+), 24 deletions(-) diff --git a/src/common/file_lock.cpp b/src/common/file_lock.cpp index 25d7c4cc71..56b0568784 100644 --- a/src/common/file_lock.cpp +++ b/src/common/file_lock.cpp @@ -8,7 +8,7 @@ #include "src/common/file_lock.h" #include -#include +#include namespace nebd { namespace common { diff --git a/src/part1/async_request_closure.cpp b/src/part1/async_request_closure.cpp index ca4b17b4bb..6f8821cf79 100644 --- a/src/part1/async_request_closure.cpp +++ b/src/part1/async_request_closure.cpp @@ -7,7 +7,7 @@ #include "src/part1/async_request_closure.h" -#include +#include #include #include diff --git a/src/part2/CMakeLists.txt b/src/part2/CMakeLists.txt index 319996c766..31dfdd6298 100644 --- a/src/part2/CMakeLists.txt +++ b/src/part2/CMakeLists.txt @@ -32,7 +32,7 @@ set(PART2_LINK snappy ssl z - jsoncpp_lib) + jsoncpp_lib curve bthread glog diff --git a/src/part2/define.h b/src/part2/define.h index c0b4ea5dee..ab1a4207cd 100644 --- a/src/part2/define.h +++ b/src/part2/define.h @@ -27,9 +27,6 @@ const char CURVE_PREFIX[] = "cbd"; const char CEPH_PREFIX[] = "rbd"; const char TEST_PREFIX[] = "test"; -const char CURVE_PREFIX[] = "cbd"; -const char CEPH_PREFIX[] = "rbd"; - // nebd异步请求的类型 enum class LIBAIO_OP { LIBAIO_OP_READ, @@ -86,8 +83,8 @@ typedef void (*NebdAioCallBack)(struct NebdServerAioContext* context); struct NebdServerAioContext { // 请求的offset off_t offset; - // 请求的length - size_t length; + // 请求的size + size_t size; // 记录异步返回的返回值 int ret; // 异步请求的类型，详见定义 @@ -100,6 +97,8 @@ struct NebdServerAioContext { Message* response; // rpc请求的回调函数 Closure *done; + // rpc请求的controller + RpcController* cntl; }; struct NebdFileInfo { diff --git a/src/part2/file_manager.cpp b/src/part2/file_manager.cpp index 7df80ae6f3..f9cda70278 100644 --- a/src/part2/file_manager.cpp +++ b/src/part2/file_manager.cpp @@ -89,16 +89,6 @@ int NebdFileManager::Load() { return 0; } -int NebdFileManager::Run() { - // TODO - return 0; -} - -int NebdFileManager::Fini() { - // TODO - return 0; -} - int NebdFileManager::UpdateFileTimestamp(int fd) { NebdFileRecordPtr fileRecord = fileRecordMap_.GetRecord(fd); if (fileRecord == nullptr) { diff --git a/src/part2/file_service.cpp b/src/part2/file_service.cpp index e80e54de60..ae8ff6f86f 100644 --- a/src/part2/file_service.cpp +++ b/src/part2/file_service.cpp @@ -226,7 +226,7 @@ void NebdFileServiceImpl::StatFile( response->set_retcode(RetCode::kNoOK); NebdFileInfo fileInfo; - int rc = fileManager_->StatFile(request->fd(), &fileInfo); + int rc = fileManager_->GetInfo(request->fd(), &fileInfo); if (rc < 0) { LOG(ERROR) << "Stat file failed. " << "fd: " << request->fd() diff --git a/src/part2/request_executor_curve.cpp b/src/part2/request_executor_curve.cpp index 1cca0fb2a9..4fe3f20a46 100644 --- a/src/part2/request_executor_curve.cpp +++ b/src/part2/request_executor_curve.cpp @@ -231,7 +231,7 @@ std::string CurveRequestExecutor::GetFileNameFromNebdFileInstance( int CurveRequestExecutor::FromNebdCtxToCurveCtx( NebdServerAioContext *nebdCtx, CurveAioContext *curveCtx) { curveCtx->offset = nebdCtx->offset; - curveCtx->length = nebdCtx->length; + curveCtx->length = nebdCtx->size; int ret = FromNebdOpToCurveOp(nebdCtx->op, &curveCtx->op); if (ret < 0) { return -1; diff --git a/src/part2/request_executor_curve.h b/src/part2/request_executor_curve.h index 6be3d225f2..ef597ef8c9 100644 --- a/src/part2/request_executor_curve.h +++ b/src/part2/request_executor_curve.h @@ -33,7 +33,7 @@ class CurveAioCombineContext { NebdServerAioContext* nebdCtx; CurveAioContext* curveCtx; }; -static void CurveAioCallback(struct CurveAioContext* curveCtx); +void CurveAioCallback(struct CurveAioContext* curveCtx); class FileNameParser { public: diff --git a/src/part2/util.cpp b/src/part2/util.cpp index 6ef6847291..e0bcb4d47a 100644 --- a/src/part2/util.cpp +++ b/src/part2/util.cpp @@ -45,7 +45,7 @@ std::string Op2Str(LIBAIO_OP op) { std::ostream& operator<<(std::ostream& os, const NebdServerAioContext& c) { os << "[type: " << Op2Str(c.op) << ", offset: " << c.offset - << ", length: " << c.length + << ", size: " << c.size << ", ret: " << c.ret << "]"; return os; diff --git a/tests/part2/test_request_executor_curve.cpp b/tests/part2/test_request_executor_curve.cpp index 156a602045..8b60e77751 100644 --- a/tests/part2/test_request_executor_curve.cpp +++ b/tests/part2/test_request_executor_curve.cpp @@ -240,7 +240,7 @@ TEST_F(TestReuqestExecutorCurve, test_AioRead) { // 3. 调用curveclient的AioRead接口失败， 异步读失败 { auto curveFileIns = new CurveFileInstance(); - aiotcx.length = 1; + aiotcx.size = 1; aiotcx.offset = 0; aiotcx.buf = new char[10]; aiotcx.op = LIBAIO_OP::LIBAIO_OP_READ; @@ -285,7 +285,7 @@ TEST_F(TestReuqestExecutorCurve, test_AioWrite) { // 3. 调用curveclient的AioWrite接口失败， 异步写失败 { auto curveFileIns = new CurveFileInstance(); - aiotcx.length = 1; + aiotcx.size = 1; aiotcx.offset = 0; aiotcx.buf = new char[10]; aiotcx.op = LIBAIO_OP::LIBAIO_OP_READ; From 8acaaa0546d6017bd4feb9ead1d488c791ff9275 Mon Sep 17 00:00:00 2001 From: charisu Date: Tue, 11 Feb 2020 15:07:44 +0800 Subject: [PATCH 28/79] support bazel Change-Id: I34abbbdcb7fb44db5247dc82c420c9e32f5d5ed8 --- .gitignore | 74 +++++++++++ WORKSPACE | 132 ++++++++++++++++++++ bazel/gmock.BUILD | 28 +++++ bazel/jsoncpp.BUILD | 14 +++ bazel/leveldb.BUILD | 82 ++++++++++++ mk-deb.sh | 57 +++++++++ proto/BUILD | 15 +++ {src/common => proto}/client.proto | 2 +- {src/common => proto}/common.proto | 0 {src/common => proto}/heartbeat.proto | 4 +- src/common/BUILD | 13 ++ src/common/name_lock.h | 6 +- src/part1/BUILD | 43 +++++++ src/part1/heartbeat_manager.cpp | 2 +- src/part1/nebd_client.h | 2 +- src/part2/BUILD | 57 +++++++++ src/part2/file_manager.h | 3 +- src/part2/file_service.h | 4 +- src/part2/heartbeat_service.h | 2 +- src/part2/nebd_server.cpp | 11 +- src/part2/request_executor_ceph.cpp | 35 +++--- src/part2/request_executor_ceph.h | 13 +- src/part2/request_executor_curve.h | 2 +- tests/common/test_name_lock.cpp | 2 +- tests/part1/BUILD | 52 ++++++++ tests/part1/fake_file_service.h | 2 +- tests/part1/fake_heartbeat_service.h | 2 +- tests/part1/mock_file_service.h | 2 +- tests/part1/nebd-client.conf | 6 +- tests/part1/nebd_client_unittest.cpp | 4 +- tests/part2/BUILD | 102 +++++++++++++++ tests/part2/file_service_unittest.cpp | 6 +- tests/part2/heartbeat_service_test.cpp | 2 +- tests/part2/mock_curve_client.h | 8 +- tests/part2/mock_file_manager.h | 2 +- tests/part2/mock_request_executor.h | 1 + tests/part2/test_nebd_server.cpp | 7 +- tests/part2/test_request_executor_curve.cpp | 1 - 38 files changed, 746 insertions(+), 54 deletions(-) create mode 100644 WORKSPACE create mode 100644 bazel/gmock.BUILD create mode 100644 bazel/jsoncpp.BUILD create mode 100644 bazel/leveldb.BUILD create mode 100755 mk-deb.sh create mode 100644 proto/BUILD rename {src/common => proto}/client.proto (98%) rename {src/common => proto}/common.proto (100%) rename {src/common => proto}/heartbeat.proto (92%) create mode 100644 src/common/BUILD create mode 100644 src/part1/BUILD create mode 100644 src/part2/BUILD create mode 100644 tests/part1/BUILD create mode 100644 tests/part2/BUILD diff --git a/.gitignore b/.gitignore index 28beb4fab0..376fc5f567 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,78 @@ 3rdparty/cmock build +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +#*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# clion +.idea +cmake-build-debug +CMakeLists.txt +bin +lib +*.temp +.clwb + +# cscope +cscope.out +cscope.in.out +cscope.po.out +cscope.files +tags + +.sconsign.dblite +build.err +build.log + +# python +*.pyc + + +# protobuf +*.pb.h +*.pb.cc + +# bazel +bazel-* +bazel-bin +bazel-curve +bazel-genfiles +bazel-out +bazel-testlogs + +# vscode .vscode + +# vimprj +.vimprj/ + +#log +*.log + diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000000..23098eabea --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,132 @@ +workspace(name = "nebd") + +# proto_library, cc_proto_library, and java_proto_library rules implicitly +# depend on @com_google_protobuf for protoc and proto runtimes. +# This statement defines the @com_google_protobuf repo. +http_archive( + name = "com_google_protobuf", + sha256 = "cef7f1b5a7c5fba672bec2a319246e8feba471f04dcebfe362d55930ee7c1c30", + strip_prefix = "protobuf-3.5.0", + urls = ["https://github.com/google/protobuf/archive/v3.5.0.zip"], +) + +bind( + name = "protobuf", + actual = "@com_google_protobuf//:protobuf", +) + +#import the gtest files. +new_git_repository( + name = "com_google_googletest", + build_file = "bazel/gmock.BUILD", + remote = "https://github.com/google/googletest", + tag = "release-1.8.0", +) + +bind( + name = "gtest", + actual = "@com_google_googletest//:gtest", +) + +#Import the glog files. +# brpc内BUILD文件在依赖glog时, 直接指定的依赖是"@com_github_google_glog//:glog" +git_repository( + name = "com_github_google_glog", + remote = "http://gerrit.storage.netease.com/curve/curve-glog", + commit = "fa4439bd6a4aae0486fe1b4e9ba7f2b761976ac2", +) + +bind( + name = "glog", + actual = "@com_github_google_glog//:glog" +) + +# glog depends on gflags-2.2.2 +http_archive( + name = "com_github_gflags_gflags", + strip_prefix = "gflags-2.2.2", + urls = [ + "https://mirror.bazel.build/github.com/gflags/gflags/archive/v2.2.2.tar.gz", + "https://github.com/gflags/gflags/archive/v2.2.2.tar.gz", + ], +) + +bind( + name = "gflags", + actual = "@com_github_gflags_gflags//:gflags", +) + +new_http_archive( + name = "com_github_google_leveldb", + build_file = "bazel/leveldb.BUILD", + strip_prefix = "leveldb-a53934a3ae1244679f812d998a4f16f2c7f309a6", + url = "https://github.com/google/leveldb/archive/a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz", +) + +bind( + name = "leveldb", + actual = "@com_github_google_leveldb//:leveldb", +) + +git_repository( + name = "com_netease_storage_gerrit_curve_curve_brpc", + remote = "http://gerrit.storage.netease.com/curve/curve-brpc", + commit = "8ed04de5b807b50e0691575916ec96621caad734", +) + +bind( + name = "brpc", + actual = "@com_netease_storage_gerrit_curve_curve_brpc//:brpc", +) + +bind( + name = "butil", + actual = "@com_netease_storage_gerrit_curve_curve_brpc//:butil", +) + +bind( + name = "bthread", + actual = "@com_netease_storage_gerrit_curve_curve_brpc//:bthread", +) + +bind( + name = "bvar", + actual = "@com_netease_storage_gerrit_curve_curve_brpc//:bvar", +) + +# curve +git_repository( + name = "com_netease_storage_gerrit_curve_curve", + remote = "http://gerrit.storage.netease.com/curve", + commit = "3f3a65750164060bc372b10a3ce5a43515dcaa3d", +) + +bind( + name = "curve", + actual = "@com_netease_storage_gerrit_curve_curve//src/client:curve", +) + +git_repository( + name = "com_netease_storage_gerrit_curve_curve_braft", + remote = "http://gerrit.storage.netease.com/curve/curve-braft", + commit = "27dedf9e2e0d26f99af4e7de35ced5c2106cb711", +) + +bind( + name = "braft", + actual = "@com_netease_storage_gerrit_curve_curve_braft//:braft", +) + +# jsoncpp +new_git_repository( + name = "jsoncpp", + build_file = "bazel/jsoncpp.BUILD", + remote = "https://github.com/open-source-parsers/jsoncpp.git", + tag = "1.8.4", +) + +bind( + name = "json", + actual = "@jsoncpp//:json", +) + diff --git a/bazel/gmock.BUILD b/bazel/gmock.BUILD new file mode 100644 index 0000000000..87bd62815b --- /dev/null +++ b/bazel/gmock.BUILD @@ -0,0 +1,28 @@ +cc_library( + name = "gtest", + srcs = [ + "googletest/src/gtest-all.cc", + "googlemock/src/gmock-all.cc", + ], + hdrs = glob([ + "**/*.h", + "googletest/src/*.cc", + "googlemock/src/*.cc", + ]), + includes = [ + "googlemock", + "googletest", + "googletest/include", + "googlemock/include", + ], + linkopts = ["-pthread"], + visibility = ["//visibility:public"], + ) + +cc_library( + name = "gtest_main", + srcs = ["googlemock/src/gmock_main.cc"], + linkopts = ["-pthread"], + visibility = ["//visibility:public"], + deps = [":gtest"], +) diff --git a/bazel/jsoncpp.BUILD b/bazel/jsoncpp.BUILD new file mode 100644 index 0000000000..5d7fcea07a --- /dev/null +++ b/bazel/jsoncpp.BUILD @@ -0,0 +1,14 @@ +cc_library( + name = "json", + srcs = glob(["src/lib_json/*.cpp"]), + hdrs = glob([ + "src/lib_json/*.h", + "include/json/*.h", + "src/lib_json/*.inl", + "src/lib_json/*.in", + ]), + includes = [ + "include", + ], + visibility = ["//visibility:public"], +) diff --git a/bazel/leveldb.BUILD b/bazel/leveldb.BUILD new file mode 100644 index 0000000000..1ec8cc3d49 --- /dev/null +++ b/bazel/leveldb.BUILD @@ -0,0 +1,82 @@ +package(default_visibility = ["//visibility:public"]) + + +config_setting( + name = "darwin", + values = {"cpu": "darwin"}, + visibility = ["//visibility:public"], +) + +SOURCES = ["db/builder.cc", + "db/c.cc", + "db/dbformat.cc", + "db/db_impl.cc", + "db/db_iter.cc", + "db/dumpfile.cc", + "db/filename.cc", + "db/log_reader.cc", + "db/log_writer.cc", + "db/memtable.cc", + "db/repair.cc", + "db/table_cache.cc", + "db/version_edit.cc", + "db/version_set.cc", + "db/write_batch.cc", + "table/block_builder.cc", + "table/block.cc", + "table/filter_block.cc", + "table/format.cc", + "table/iterator.cc", + "table/merger.cc", + "table/table_builder.cc", + "table/table.cc", + "table/two_level_iterator.cc", + "util/arena.cc", + "util/bloom.cc", + "util/cache.cc", + "util/coding.cc", + "util/comparator.cc", + "util/crc32c.cc", + "util/env.cc", + "util/env_posix.cc", + "util/filter_policy.cc", + "util/hash.cc", + "util/histogram.cc", + "util/logging.cc", + "util/options.cc", + "util/status.cc", + "port/port_posix.cc", + "port/port_posix_sse.cc", + "helpers/memenv/memenv.cc", + ] + +cc_library( + name = "leveldb", + srcs = SOURCES, + hdrs = glob([ + "helpers/memenv/*.h", + "util/*.h", + "port/*.h", + "port/win/*.h", + "table/*.h", + "db/*.h", + "include/leveldb/*.h" + ], + exclude = [ + "**/*test.*", + ]), + includes = [ + "include/", + ], + copts = [ + "-fno-builtin-memcmp", + "-DLEVELDB_PLATFORM_POSIX=1", + "-DLEVELDB_ATOMIC_PRESENT", + ], + defines = [ + "LEVELDB_PLATFORM_POSIX", + ] + select({ + ":darwin": ["OS_MACOSX"], + "//conditions:default": [], + }), +) diff --git a/mk-deb.sh b/mk-deb.sh new file mode 100755 index 0000000000..489f2e4406 --- /dev/null +++ b/mk-deb.sh @@ -0,0 +1,57 @@ +#!/bin/bash +dir=`pwd` +set -x +# step1 清楚生成的目录和文件 +bazel clean +rm -rf *deb +rm -rf build + +# step2 编译 +if [ "$1" = "debug" ] +then + bazel build ... --copt -DHAVE_ZLIB=1 --compilation_mode=dbg -s --define=with_glog=true \ + --define=libunwind=true --copt -DGFLAGS_NS=google --copt \ + -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX + if [ $? -ne 0 ] + then + echo "build phase1 failed" + exit + fi +else + bazel build ... --copt -DHAVE_ZLIB=1 --copt -O2 -s --define=with_glog=true \ + --define=libunwind=true --copt -DGFLAGS_NS=google --copt \ + -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX + if [ $? -ne 0 ] + then + echo "build phase1 failed" + exit + fi +fi + +# step3 创建临时目录，拷贝二进制、lib库和配置模板 +mkdir build +cp -r nebd-package build/ +mkdir -p build/nebd-package/usr/bin +mkdir -p build/nebd-package/usr/lib + +for i in `find bazel-bin/|grep -w so|grep -v solib|grep -v params| grep -v test` + do + cp -f $i build/nebd-package/usr/lib + done + +cp bazel-bin/src/part2/nebd-server build/nebd-package/usr/bin +cp -r etc build/nebd-package/ + +# step4 获取git提交版本信息，记录到debian包的配置文件 +commit_id=`git show --abbrev-commit HEAD|head -n 1|awk '{print $2}'` +if [ "$1" = "debug" ] +then + debug="+debug" +else + debug="" +fi +version="Version: 0.0.1+${commit_id}${debug}" +echo $version >> build/nebd-package/DEBIAN/control + +# step5 打包debian包 +dpkg-deb -b build/nebd-package . diff --git a/proto/BUILD b/proto/BUILD new file mode 100644 index 0000000000..a45f04983a --- /dev/null +++ b/proto/BUILD @@ -0,0 +1,15 @@ +cc_proto_library( + name = "client_cc_proto", + visibility = ["//visibility:public"], + deps = [":client_proto"], +) + +proto_library( + name = "client_proto", + srcs = glob([ + "common.proto", + "client.proto", + "heartbeat.proto" + ] + ), +) diff --git a/src/common/client.proto b/proto/client.proto similarity index 98% rename from src/common/client.proto rename to proto/client.proto index 4da202612f..d8f611f298 100644 --- a/src/common/client.proto +++ b/proto/client.proto @@ -1,5 +1,5 @@ syntax="proto2"; -import "common.proto"; +import "proto/common.proto"; package nebd.client; diff --git a/src/common/common.proto b/proto/common.proto similarity index 100% rename from src/common/common.proto rename to proto/common.proto diff --git a/src/common/heartbeat.proto b/proto/heartbeat.proto similarity index 92% rename from src/common/heartbeat.proto rename to proto/heartbeat.proto index 5ae0172520..3c2aea0cb4 100644 --- a/src/common/heartbeat.proto +++ b/proto/heartbeat.proto @@ -1,5 +1,5 @@ syntax="proto2"; -import "common.proto"; +import "proto/common.proto"; package nebd.client; @@ -20,4 +20,4 @@ message HeartbeatResponse { service NebdHeartbeatService { rpc KeepAlive(HeartbeatRequest) returns (HeartbeatResponse); -}; \ No newline at end of file +}; diff --git a/src/common/BUILD b/src/common/BUILD new file mode 100644 index 0000000000..9794ec3e36 --- /dev/null +++ b/src/common/BUILD @@ -0,0 +1,13 @@ +cc_library( + name = "nebd_common", + srcs = glob([ + "*.h", + "*.cpp", + ] + ), + visibility = ["//visibility:public"], + deps = [ + "//external:glog", + ], +) + diff --git a/src/common/name_lock.h b/src/common/name_lock.h index 3beab33868..4b0506435d 100644 --- a/src/common/name_lock.h +++ b/src/common/name_lock.h @@ -6,8 +6,8 @@ * Copyright (c) 2019 netease */ -#ifndef SRC_COMMON_NEMA_LOCK_H_ -#define SRC_COMMON_NEMA_LOCK_H_ +#ifndef SRC_COMMON_NAME_LOCK_H_ +#define SRC_COMMON_NAME_LOCK_H_ #include #include @@ -90,4 +90,4 @@ class NameLockGuard : public Uncopyable { } // namespace nebd -#endif // SRC_COMMON_NEMA_LOCK_H_ +#endif // SRC_COMMON_NAME_LOCK_H_ diff --git a/src/part1/BUILD b/src/part1/BUILD new file mode 100644 index 0000000000..269a100e19 --- /dev/null +++ b/src/part1/BUILD @@ -0,0 +1,43 @@ +COPTS = [ + "-DGFLAGS=gflags", + "-DOS_LINUX", + "-DSNAPPY", + "-DHAVE_ZLIB", + "-DHAVE_SSE42", + "-DNDEBUG", + "-std=c++0x", + "-pipe", + "-W", + "-Wall", + "-Wno-unused-parameter", + "-fPIC", + "-fno-omit-frame-pointer", + "-momit-leaf-frame-pointer", + "-msse4.2", + "-pthread", + "-Wsign-compare", + "-Wno-unused-variable", + "-Wno-missing-field-initializers", + "-Woverloaded-virtual", + "-Wnon-virtual-dtor", +] + +cc_library( + name = "nebdclient", + srcs = glob([ + "*.h", + "*.cpp", + ] + ), + visibility = ["//visibility:public"], + deps = [ + "//external:glog", + "//external:brpc", + "//src/common:nebd_common", + "//proto:client_cc_proto" + ], + copts = COPTS, +) + + + diff --git a/src/part1/heartbeat_manager.cpp b/src/part1/heartbeat_manager.cpp index eec637f7de..a1380e557a 100644 --- a/src/part1/heartbeat_manager.cpp +++ b/src/part1/heartbeat_manager.cpp @@ -11,7 +11,7 @@ #include #include -#include "src/common/heartbeat.pb.h" +#include "proto/heartbeat.pb.h" namespace nebd { namespace client { diff --git a/src/part1/nebd_client.h b/src/part1/nebd_client.h index b82c231a2b..43fcc29b96 100644 --- a/src/part1/nebd_client.h +++ b/src/part1/nebd_client.h @@ -16,7 +16,7 @@ #include "src/part1/nebd_common.h" #include "src/common/configuration.h" -#include "src/common/client.pb.h" +#include "proto/client.pb.h" #include "src/part1/libnebd.h" #include "src/part1/heartbeat_manager.h" #include "src/part1/nebd_metacache.h" diff --git a/src/part2/BUILD b/src/part2/BUILD new file mode 100644 index 0000000000..67febb01f8 --- /dev/null +++ b/src/part2/BUILD @@ -0,0 +1,57 @@ +COPTS = [ + "-DGFLAGS=gflags", + "-DOS_LINUX", + "-DSNAPPY", + "-DHAVE_ZLIB", + "-DHAVE_SSE42", + "-DNDEBUG", + "-std=c++0x", + "-pipe", + "-W", + "-Wall", + "-Wno-unused-parameter", + "-fPIC", + "-fno-omit-frame-pointer", + "-momit-leaf-frame-pointer", + "-msse4.2", + "-pthread", + "-Wsign-compare", + "-Wno-unused-variable", + "-Wno-missing-field-initializers", + "-Woverloaded-virtual", + "-Wnon-virtual-dtor" +] + +cc_library( + name = "nebdserver", + srcs = glob([ + "*.h", + "*.cpp", + ],exclude = ["main.cpp"] + ), + visibility = ["//visibility:public"], + deps = [ + "//external:glog", + "//external:brpc", + "//external:json", + "//external:gflags", + "//external:curve", + "//src/common:nebd_common", + "//proto:client_cc_proto" + ], + copts = COPTS, +) + +cc_binary( + name = "nebd-server", + srcs = glob([ + "main.cpp", + ] + ), + deps = [ + "//src/part2:nebdserver", + ], + copts = COPTS, +) + + diff --git a/src/part2/file_manager.h b/src/part2/file_manager.h index 75200e2d3f..b97fe541dc 100644 --- a/src/part2/file_manager.h +++ b/src/part2/file_manager.h @@ -14,6 +14,7 @@ #include // NOLINT #include #include // NOLINT +#include #include "src/common/rw_lock.h" #include "src/common/name_lock.h" @@ -25,7 +26,7 @@ #include "src/part2/file_record_map.h" #include "src/part2/request_executor_ceph.h" #include "src/part2/request_executor_curve.h" -#include "src/common/client.pb.h" +#include "proto/client.pb.h" namespace nebd { namespace server { diff --git a/src/part2/file_service.h b/src/part2/file_service.h index 4203b63b23..1b7066e843 100644 --- a/src/part2/file_service.h +++ b/src/part2/file_service.h @@ -14,8 +14,8 @@ #include #include -#include "src/common/common.pb.h" -#include "src/common/client.pb.h" +#include "proto/common.pb.h" +#include "proto/client.pb.h" #include "src/part2/file_manager.h" namespace nebd { diff --git a/src/part2/heartbeat_service.h b/src/part2/heartbeat_service.h index a21a479f99..24830aa48e 100644 --- a/src/part2/heartbeat_service.h +++ b/src/part2/heartbeat_service.h @@ -12,7 +12,7 @@ #include #include -#include "src/common/heartbeat.pb.h" +#include "proto/heartbeat.pb.h" #include "src/part2/file_manager.h" namespace nebd { diff --git a/src/part2/nebd_server.cpp b/src/part2/nebd_server.cpp index 6960a75524..c25fd18f9b 100644 --- a/src/part2/nebd_server.cpp +++ b/src/part2/nebd_server.cpp @@ -7,6 +7,7 @@ #include #include +#include "src/common/file_lock.h" #include "src/part2/nebd_server.h" #include "src/part2/file_service.h" #include "src/part2/heartbeat_service.h" @@ -139,7 +140,14 @@ bool NebdServer::StartServer() { // start brcp server brpc::ServerOptions option; option.idle_timeout_sec = -1; - int startBrpcServerRes = server_.Start(listenAddress_.c_str(), &option); + // 获取文件锁 + common::FileLock fileLock(listenAddress_); + if (fileLock.AcquireFileLock() != 0) { + LOG(ERROR) << "Address already in use"; + return -1; + } + int startBrpcServerRes = server_.StartAtSockFile( + listenAddress_.c_str(), &option); if (0 != startBrpcServerRes) { LOG(ERROR) << "NebdServer start brpc server fail, res=" << startBrpcServerRes; @@ -150,6 +158,7 @@ bool NebdServer::StartServer() { server_.RunUntilAskedToQuit(); isRunning_ = false; + fileLock.ReleaseFileLock(); return true; } diff --git a/src/part2/request_executor_ceph.cpp b/src/part2/request_executor_ceph.cpp index 23880f953c..1c7eb8bd97 100644 --- a/src/part2/request_executor_ceph.cpp +++ b/src/part2/request_executor_ceph.cpp @@ -1,3 +1,10 @@ +/* + * Project: nebd + * Created Date: 2020-02-14 + * Author: charisu + * Copyright (c) 2020 netease + */ + #include "src/part2/request_executor_ceph.h" namespace nebd { @@ -5,54 +12,50 @@ namespace server { std::shared_ptr CephRequestExecutor::Open(const std::string& filename) { - // TODO return nullptr; } std::shared_ptr CephRequestExecutor::Reopen(const std::string& filename, AdditionType addtion) { - // TODO return nullptr; } int CephRequestExecutor::Close(NebdFileInstance* fd) { - // TODO return 0; } -int CephRequestExecutor::Extend(NebdFileInstance* fd, int64_t newsize) { - // TODO +int CephRequestExecutor::Extend(NebdFileInstance* fd, + int64_t newsize) { return 0; } -int CephRequestExecutor::Discard(NebdFileInstance* fd, NebdServerAioContext* aioctx) { - // TODO +int CephRequestExecutor::Discard(NebdFileInstance* fd, + NebdServerAioContext* aioctx) { return 0; } -int CephRequestExecutor::AioRead(NebdFileInstance* fd, NebdServerAioContext* aioctx) { - // TODO +int CephRequestExecutor::AioRead(NebdFileInstance* fd, + NebdServerAioContext* aioctx) { return 0; } -int CephRequestExecutor::AioWrite(NebdFileInstance* fd, NebdServerAioContext* aioctx) { - // TODO +int CephRequestExecutor::AioWrite(NebdFileInstance* fd, + NebdServerAioContext* aioctx) { return 0; } -int CephRequestExecutor::Flush(NebdFileInstance* fd, NebdServerAioContext* aioctx) { - // TODO +int CephRequestExecutor::Flush(NebdFileInstance* fd, + NebdServerAioContext* aioctx) { return 0; } -int CephRequestExecutor::GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) { - // TODO +int CephRequestExecutor::GetInfo(NebdFileInstance* fd, + NebdFileInfo* fileInfo) { return 0; } int CephRequestExecutor::InvalidCache(NebdFileInstance* fd) { - // TODO return 0; } diff --git a/src/part2/request_executor_ceph.h b/src/part2/request_executor_ceph.h index 5e70871f76..d9333fe99e 100644 --- a/src/part2/request_executor_ceph.h +++ b/src/part2/request_executor_ceph.h @@ -1,5 +1,12 @@ -#ifndef SRC_PART2_REQUEST_EXECUTO_CEPH_H_ -#define SRC_PART2_REQUEST_EXECUTO_CEPH_H_ +/* + * Project: nebd + * Created Date: 2020-02-14 + * Author: charisu + * Copyright (c) 2020 netease + */ + +#ifndef SRC_PART2_REQUEST_EXECUTOR_CEPH_H_ +#define SRC_PART2_REQUEST_EXECUTOR_CEPH_H_ #include #include @@ -40,4 +47,4 @@ class CephRequestExecutor : public NebdRequestExecutor { } // namespace server } // namespace nebd -#endif // SRC_PART2_REQUEST_EXECUTO_CEPH_H_ +#endif // SRC_PART2_REQUEST_EXECUTOR_CEPH_H_ diff --git a/src/part2/request_executor_curve.h b/src/part2/request_executor_curve.h index ef597ef8c9..d986e87a44 100644 --- a/src/part2/request_executor_curve.h +++ b/src/part2/request_executor_curve.h @@ -8,11 +8,11 @@ #ifndef SRC_PART2_REQUEST_EXECUTOR_CURVE_H_ #define SRC_PART2_REQUEST_EXECUTOR_CURVE_H_ -#include #include #include #include "src/part2/request_executor.h" #include "src/part2/define.h" +#include "include/client/libcurve.h" namespace nebd { namespace server { diff --git a/tests/common/test_name_lock.cpp b/tests/common/test_name_lock.cpp index b068366dee..31bbb09604 100644 --- a/tests/common/test_name_lock.cpp +++ b/tests/common/test_name_lock.cpp @@ -7,7 +7,7 @@ #include #include -#include +#include // NOLINT #include "src/common/name_lock.h" diff --git a/tests/part1/BUILD b/tests/part1/BUILD new file mode 100644 index 0000000000..a650cf50dc --- /dev/null +++ b/tests/part1/BUILD @@ -0,0 +1,52 @@ +cc_binary( + name = "nebd_metacache_unittest", + srcs = glob([ + "nebd_metacache_unittest.cpp", + ]), + deps = [ + "//external:gflags", + "//src/part1:nebdclient", + "@com_google_googletest//:gtest_main", + ], +) + +cc_binary( + name = "heartbeat_manager_unittest", + srcs = glob([ + "heartbeat_manager_unittest.cpp", + ]), + deps = [ + "//external:gflags", + "//src/part1:nebdclient", + "//tests/part1:fake_lib", + "@com_google_googletest//:gtest_main", + ], +) + +cc_binary( + name = "nebd_client_unittest", + srcs = glob([ + "nebd_client_unittest.cpp", + ]), + deps = [ + "//external:gflags", + "//src/part1:nebdclient", + "//tests/part1:fake_lib", + "@com_google_googletest//:gtest_main", + ], + linkstatic = False, +) + +cc_library( + name = "fake_lib", + srcs = glob([ + "fake_*", + "mock_*", + ]), + hdrs = glob(["*.h"]), + deps = [ + "@com_google_googletest//:gtest", + "//src/part1:nebdclient", + ], + visibility = ["//visibility:public"], +) diff --git a/tests/part1/fake_file_service.h b/tests/part1/fake_file_service.h index 47df681587..b5ca013ec8 100644 --- a/tests/part1/fake_file_service.h +++ b/tests/part1/fake_file_service.h @@ -11,7 +11,7 @@ #include #include #include -#include "src/common/client.pb.h" +#include "proto/client.pb.h" namespace nebd { namespace client { diff --git a/tests/part1/fake_heartbeat_service.h b/tests/part1/fake_heartbeat_service.h index 5caa87f3bf..e63d740700 100644 --- a/tests/part1/fake_heartbeat_service.h +++ b/tests/part1/fake_heartbeat_service.h @@ -13,7 +13,7 @@ #include #include -#include "src/common/heartbeat.pb.h" +#include "proto/heartbeat.pb.h" namespace nebd { namespace client { diff --git a/tests/part1/mock_file_service.h b/tests/part1/mock_file_service.h index 4723ec1efc..865601afc7 100644 --- a/tests/part1/mock_file_service.h +++ b/tests/part1/mock_file_service.h @@ -11,7 +11,7 @@ #include #include #include -#include "src/common/client.pb.h" +#include "proto/client.pb.h" namespace nebd { namespace client { diff --git a/tests/part1/nebd-client.conf b/tests/part1/nebd-client.conf index 163e6bddf7..2c2203af55 100644 --- a/tests/part1/nebd-client.conf +++ b/tests/part1/nebd-client.conf @@ -1,7 +1,7 @@ # part2 socket file address -nebdserver.serverAddress=/root/nebd/nebd.test.sock +nebdserver.serverAddress=/tmp/nebd-client-test.sock # 文件锁路径 -metacache.fileLockPath=/root/nebd/tests +metacache.fileLockPath=/tmp # 同步rpc的最大重试次数 request.syncRpcMaxRetryTimes=10 # rpc请求的重试间隔 @@ -16,4 +16,4 @@ request.rpcHealthCheckIntervalS=1 # heartbeat间隔 heartbeat.intervalS=5 # heartbeat rpc超时时间 -heartbeat.rpcTimeoutMs=500 \ No newline at end of file +heartbeat.rpcTimeoutMs=500 diff --git a/tests/part1/nebd_client_unittest.cpp b/tests/part1/nebd_client_unittest.cpp index bcb844cc7a..10484ad852 100644 --- a/tests/part1/nebd_client_unittest.cpp +++ b/tests/part1/nebd_client_unittest.cpp @@ -24,8 +24,8 @@ namespace nebd { namespace client { const char* kFileName = "nebd-test-filename"; -const char* kNebdServerTestAddress = "/root/nebd/nebd.test.sock"; -const char* kNebdClientConf = "/root/nebd/tests/part1/nebd-client.conf"; +const char* kNebdServerTestAddress = "/tmp/nebd-client-test.sock"; +const char* kNebdClientConf = "tests/part1/nebd-client.conf"; const int64_t kFileSize = 10LL * 1024 * 1024 * 1024; const int64_t kBufSize = 1024; diff --git a/tests/part2/BUILD b/tests/part2/BUILD new file mode 100644 index 0000000000..50707c44e4 --- /dev/null +++ b/tests/part2/BUILD @@ -0,0 +1,102 @@ +cc_binary( + name = "filerecordmap_test", + srcs = glob([ + "file_record_map_unittest.cpp", + ]), + deps = [ + "//external:gflags", + "//src/part2:nebdserver", + "@com_google_googletest//:gtest_main", + ], +) + +cc_binary( + name = "filemanager_test", + srcs = glob([ + "file_manager_unittest.cpp", + ]), + deps = [ + "//external:gflags", + "//src/part2:nebdserver", + "//tests/part2:mock_lib", + "@com_google_googletest//:gtest_main", + ], +) + +cc_binary( + name = "fileservice_test", + srcs = glob([ + "file_service_unittest.cpp", + ]), + deps = [ + "//external:gflags", + "//src/part2:nebdserver", + "//tests/part2:mock_lib", + "@com_google_googletest//:gtest_main", + ], +) + +cc_binary( + name = "heartbeat_service_test", + srcs = glob([ + "heartbeat_service_test.cpp", + ]), + deps = [ + "//external:gflags", + "//src/part2:nebdserver", + "//tests/part2:mock_lib", + "@com_google_googletest//:gtest_main", + ], +) + +cc_binary( + name = "metafile_manager_test", + srcs = glob([ + "metafile_manager_test.cpp", + ]), + deps = [ + "//external:gflags", + "//src/part2:nebdserver", + "//tests/part2:mock_lib", + "@com_google_googletest//:gtest_main", + ], +) + +cc_binary( + name = "test_nebd_server", + srcs = glob([ + "test_nebd_server.cpp", + ]), + deps = [ + "//external:gflags", + "//src/part2:nebdserver", + "@com_google_googletest//:gtest_main", + ], +) + +cc_binary( + name = "test_request_executor_curve", + srcs = glob([ + "test_request_executor_curve.cpp", + ]), + deps = [ + "//external:gflags", + "//src/part2:nebdserver", + "//tests/part2:mock_lib", + "@com_google_googletest//:gtest_main", + ], + linkopts = ["-lpthread"], +) + +cc_library( + name = "mock_lib", + srcs = glob([ + "mock_*", + ]), + hdrs = glob(["*.h"]), + deps = [ + "@com_google_googletest//:gtest", + "//src/part2:nebdserver", + ], + visibility = ["//visibility:public"], +) diff --git a/tests/part2/file_service_unittest.cpp b/tests/part2/file_service_unittest.cpp index f0797cbaee..f47029a342 100644 --- a/tests/part2/file_service_unittest.cpp +++ b/tests/part2/file_service_unittest.cpp @@ -8,8 +8,8 @@ #include #include -#include #include +#include #include "src/part2/file_service.h" #include "tests/part2/mock_file_manager.h" @@ -202,7 +202,7 @@ TEST_F(FileServiceTest, StatFileTest) { // stat file success NebdFileInfo fileInfo; fileInfo.size = 4096; - EXPECT_CALL(*fileManager_, StatFile(fd, NotNull())) + EXPECT_CALL(*fileManager_, GetInfo(fd, NotNull())) .WillOnce(DoAll(SetArgPointee<1>(fileInfo), Return(0))); fileService_->StatFile(&cntl, &request, &response, &done); @@ -212,7 +212,7 @@ TEST_F(FileServiceTest, StatFileTest) { // stat file failed done.Reset(); - EXPECT_CALL(*fileManager_, StatFile(fd, NotNull())) + EXPECT_CALL(*fileManager_, GetInfo(fd, NotNull())) .WillOnce(Return(-1)); fileService_->StatFile(&cntl, &request, &response, &done); ASSERT_EQ(response.retcode(), RetCode::kNoOK); diff --git a/tests/part2/heartbeat_service_test.cpp b/tests/part2/heartbeat_service_test.cpp index 6e212da5cf..0d932cf4f1 100644 --- a/tests/part2/heartbeat_service_test.cpp +++ b/tests/part2/heartbeat_service_test.cpp @@ -10,7 +10,7 @@ #include #include -#include "src/common/heartbeat.pb.h" +#include "proto/heartbeat.pb.h" #include "src/part2/heartbeat_service.h" #include "tests/part2/mock_file_manager.h" diff --git a/tests/part2/mock_curve_client.h b/tests/part2/mock_curve_client.h index 459753d283..91da8d4ec8 100644 --- a/tests/part2/mock_curve_client.h +++ b/tests/part2/mock_curve_client.h @@ -5,12 +5,12 @@ * Copyright (c) 2020 netease */ -#ifndef TEST_MOCK_CURVE_CLIENT_H_ -#define TEST_MOCK_CURVE_CLIENT_H_ +#ifndef TESTS_PART2_MOCK_CURVE_CLIENT_H_ +#define TESTS_PART2_MOCK_CURVE_CLIENT_H_ -#include #include #include +#include "include/client/libcurve.h" namespace nebd { namespace server { @@ -33,4 +33,4 @@ class MockCurveClient : public ::curve::client::CurveClient { } // namespace server } // namespace nebd -#endif // TEST_MOCK_CURVE_CLIENT_H_ +#endif // TESTS_PART2_MOCK_CURVE_CLIENT_H_ diff --git a/tests/part2/mock_file_manager.h b/tests/part2/mock_file_manager.h index 242c15ed10..49a728d2c5 100644 --- a/tests/part2/mock_file_manager.h +++ b/tests/part2/mock_file_manager.h @@ -28,7 +28,7 @@ class MockFileManager : public NebdFileManager { MOCK_METHOD1(UpdateFileTimestamp, int(int)); MOCK_METHOD1(Open, int(const std::string&)); MOCK_METHOD1(Close, int(int)); - MOCK_METHOD2(Extend, int(int, uint64_t)); + MOCK_METHOD2(Extend, int(int, int64_t)); MOCK_METHOD2(GetInfo, int(int, NebdFileInfo*)); MOCK_METHOD2(StatFile, int(int, NebdFileInfo*)); MOCK_METHOD2(Discard, int(int, NebdServerAioContext*)); diff --git a/tests/part2/mock_request_executor.h b/tests/part2/mock_request_executor.h index 48fecbb4e4..f1726eb92e 100644 --- a/tests/part2/mock_request_executor.h +++ b/tests/part2/mock_request_executor.h @@ -11,6 +11,7 @@ #include #include #include +#include #include "src/part2/request_executor.h" diff --git a/tests/part2/test_nebd_server.cpp b/tests/part2/test_nebd_server.cpp index 0c6d4a18bd..889b11b035 100644 --- a/tests/part2/test_nebd_server.cpp +++ b/tests/part2/test_nebd_server.cpp @@ -31,10 +31,13 @@ TEST(TestNebdServer, test_Init_Run_Fini) { std::thread nebdServerThread(&NebdServer::RunUntilAskedToQuit, &server); sleep(1); - // 5. stop成功 + // 5、再次Run会失败 + ASSERT_EQ(-1, server.RunUntilAskedToQuit()); + + // 6. stop成功 ASSERT_EQ(0, server.Fini()); - // 6. 再次stop不会重复释放资源 + // 7. 再次stop不会重复释放资源 ASSERT_EQ(0, server.Fini()); nebdServerThread.join(); } diff --git a/tests/part2/test_request_executor_curve.cpp b/tests/part2/test_request_executor_curve.cpp index 8b60e77751..572b7c5a31 100644 --- a/tests/part2/test_request_executor_curve.cpp +++ b/tests/part2/test_request_executor_curve.cpp @@ -112,7 +112,6 @@ TEST_F(TestReuqestExecutorCurve, test_Close) { auto nebdFileIns = new NebdFileInstance(); EXPECT_CALL(*curveClient_, Close(_)).Times(0); ASSERT_EQ(-1, executor.Close(nebdFileIns)); - } // 2. nebdFileIns中的fd<0, close失败 From ae44c5d6915038806279294545a1c794e2afba93 Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Wed, 12 Feb 2020 17:45:11 +0800 Subject: [PATCH 29/79] refactor code Change-Id: Ia508a3932099cbc4065253b56d43a06805374936 --- etc/nebd/nebd-server.conf | 7 +- proto/client.proto | 19 +- proto/heartbeat.proto | 4 +- src/part1/heartbeat_manager.cpp | 2 +- src/part1/libnebd.cpp | 2 +- src/part1/libnebd.h | 4 - src/part1/libnebd_file.cpp | 4 +- src/part1/libnebd_file.h | 2 +- src/part1/nebd_client.cpp | 28 +- src/part1/nebd_client.h | 2 +- src/part2/define.h | 8 +- src/part2/file_manager.cpp | 307 +++---- src/part2/file_manager.h | 84 +- src/part2/file_record_map.cpp | 96 --- src/part2/file_record_map.h | 88 -- src/part2/file_service.cpp | 28 +- src/part2/file_service.h | 5 - src/part2/filerecord_manager.cpp | 168 ++++ src/part2/filerecord_manager.h | 130 +++ src/part2/heartbeat_manager.cpp | 85 ++ src/part2/heartbeat_manager.h | 74 ++ src/part2/heartbeat_service.cpp | 6 +- src/part2/heartbeat_service.h | 8 +- src/part2/metafile_manager.cpp | 137 +-- src/part2/metafile_manager.h | 50 +- src/part2/nebd_server.cpp | 72 +- src/part2/nebd_server.h | 19 +- tests/common/BUILD | 12 + tests/part1/BUILD | 8 +- tests/part1/fake_file_service.cpp | 23 +- tests/part1/fake_file_service.h | 5 - tests/part1/fake_heartbeat_service.h | 4 +- tests/part1/heartbeat_manager_unittest.cpp | 6 +- tests/part1/mock_file_service.h | 4 - tests/part1/nebd_client_unittest.cpp | 12 +- tests/part1/nebd_test.cpp | 659 --------------- tests/part2/BUILD | 28 +- tests/part2/CMakeLists.txt | 23 +- tests/part2/file_manager_unittest.cpp | 831 +++++++++++-------- tests/part2/file_record_manager_unittest.cpp | 193 +++++ tests/part2/file_record_map_unittest.cpp | 101 --- tests/part2/file_service_unittest.cpp | 40 +- tests/part2/heartbeat_manager_unittest.cpp | 98 +++ tests/part2/heartbeat_service_test.cpp | 16 +- tests/part2/metafile_manager_test.cpp | 95 +-- tests/part2/mock_file_manager.h | 8 +- tests/part2/mock_filerecord_manager.h | 42 + tests/part2/mock_heartbeat_manager.h | 28 + tests/part2/mock_metafile_manager.h | 5 +- tests/part2/mock_request_executor.h | 1 - tests/part2/nebd-server.err.conf | 5 +- 51 files changed, 1745 insertions(+), 1941 deletions(-) delete mode 100644 src/part2/file_record_map.cpp delete mode 100644 src/part2/file_record_map.h create mode 100644 src/part2/filerecord_manager.cpp create mode 100644 src/part2/filerecord_manager.h create mode 100644 src/part2/heartbeat_manager.cpp create mode 100644 src/part2/heartbeat_manager.h create mode 100644 tests/common/BUILD delete mode 100644 tests/part1/nebd_test.cpp create mode 100644 tests/part2/file_record_manager_unittest.cpp delete mode 100644 tests/part2/file_record_map_unittest.cpp create mode 100644 tests/part2/heartbeat_manager_unittest.cpp create mode 100644 tests/part2/mock_filerecord_manager.h create mode 100644 tests/part2/mock_heartbeat_manager.h diff --git a/etc/nebd/nebd-server.conf b/etc/nebd/nebd-server.conf index 079f3f7fa2..84b4da1111 100644 --- a/etc/nebd/nebd-server.conf +++ b/etc/nebd/nebd-server.conf @@ -2,7 +2,10 @@ listen.address=/var/run/nebd.sock #元数据文件地址,包含文件名 -meta.file.path = ./nebdserver.meta +meta.file.path=./nebdserver.meta #心跳超时时间 -heartbeat.timeout.sec=30 \ No newline at end of file +heartbeat.timeout.sec=30 + +#文件超时检测时间间隔 +heartbeat.check.interval.ms=3000 diff --git a/proto/client.proto b/proto/client.proto index d8f611f298..56e653fd86 100644 --- a/proto/client.proto +++ b/proto/client.proto @@ -54,15 +54,6 @@ message DiscardResponse { optional string RetMsg = 2; } -message StatFileRequest { - required int32 fd = 1; -} -message StatFileResponse { - required RetCode retCode = 1; - optional string retMsg = 2; - optional uint64 size = 3; -} - message ResizeRequest { required int32 fd = 1; required uint64 newSize = 2; @@ -82,13 +73,20 @@ message FlushResponse { optional string retMsg = 2; } +message FileInfo { + required uint64 size = 1; + required uint64 objSize = 2; + required uint64 objNums = 3; +} + message GetInfoRequest { required int32 fd = 1; } + message GetInfoResponse { required RetCode retCode = 1; optional string retMsg = 2; - optional uint64 objSize = 3; + optional FileInfo info = 3; } message InvalidateCacheRequest { @@ -107,7 +105,6 @@ service NebdFileService { rpc Read(ReadRequest) returns (ReadResponse); rpc Write(WriteRequest) returns (WriteResponse); rpc Discard(DiscardRequest) returns (DiscardResponse); - rpc StatFile(StatFileRequest) returns (StatFileResponse); rpc ResizeFile(ResizeRequest) returns (ResizeResponse); // for ceph only rpc Flush(FlushRequest) returns (FlushResponse); diff --git a/proto/heartbeat.proto b/proto/heartbeat.proto index 3c2aea0cb4..79ba28938a 100644 --- a/proto/heartbeat.proto +++ b/proto/heartbeat.proto @@ -5,13 +5,13 @@ package nebd.client; option cc_generic_services = true; -message FileInfo { +message HeartbeatFileInfo { required int32 fd = 1; required string name = 2; } message HeartbeatRequest { - repeated FileInfo info = 1; + repeated HeartbeatFileInfo info = 1; } message HeartbeatResponse { diff --git a/src/part1/heartbeat_manager.cpp b/src/part1/heartbeat_manager.cpp index a1380e557a..9e7f7e6115 100644 --- a/src/part1/heartbeat_manager.cpp +++ b/src/part1/heartbeat_manager.cpp @@ -79,7 +79,7 @@ void HeartbeatManager::SendHeartBeat() { std::ostringstream oss; for (const auto& fileInfo : fileInfos) { - nebd::client::FileInfo* info = request.add_info(); + nebd::client::HeartbeatFileInfo* info = request.add_info(); info->set_fd(fileInfo.fd); info->set_name(fileInfo.fileName); diff --git a/src/part1/libnebd.cpp b/src/part1/libnebd.cpp index 28c9a432e1..29a791d889 100644 --- a/src/part1/libnebd.cpp +++ b/src/part1/libnebd.cpp @@ -72,7 +72,7 @@ int nebd_lib_sync(int fd) { } int64_t nebd_lib_filesize(int fd) { - return StatFile4Nebd(fd); + return GetFileSize4Nebd(fd); } int nebd_lib_resize(int fd, int64_t size) { diff --git a/src/part1/libnebd.h b/src/part1/libnebd.h index c0d94dd014..7e4ec64aed 100644 --- a/src/part1/libnebd.h +++ b/src/part1/libnebd.h @@ -24,10 +24,6 @@ extern "C" { // 文件路径最大的长度，单位字节 #define NEBD_MAX_FILE_PATH_LEN 1024 -// Note: 这个需要评估是否要去掉，之前出现过上层IO超过32MB的情况 -// 请求的buf的最大长度，读写操作的buf长度不得超过该值，单位字节 -#define NEBD_MAX_BUF_LEN 1024 * 1024 * 32 - // nebd异步请求的类型 typedef enum LIBAIO_OP { LIBAIO_OP_READ, diff --git a/src/part1/libnebd_file.cpp b/src/part1/libnebd_file.cpp index 11e27d52c5..90241d510d 100644 --- a/src/part1/libnebd_file.cpp +++ b/src/part1/libnebd_file.cpp @@ -32,8 +32,8 @@ int Extend4Nebd(int fd, int64_t newsize) { return nebd::client::nebdClient.Extend(fd, newsize); } -int64_t StatFile4Nebd(int fd) { - return nebd::client::nebdClient.StatFile(fd); +int64_t GetFileSize4Nebd(int fd) { + return nebd::client::nebdClient.GetFileSize(fd); } int Discard4Nebd(int fd, NebdClientAioContext* aioctx) { diff --git a/src/part1/libnebd_file.h b/src/part1/libnebd_file.h index 02da6ff7dc..dddbe23852 100644 --- a/src/part1/libnebd_file.h +++ b/src/part1/libnebd_file.h @@ -46,7 +46,7 @@ int Extend4Nebd(int fd, int64_t newsize); * @param fd：文件的fd * @return 成功返回文件size，失败返回错误码 */ -int64_t StatFile4Nebd(int fd); +int64_t GetFileSize4Nebd(int fd); /** * @brief discard文件，异步函数 * @param fd：文件的fd diff --git a/src/part1/nebd_client.cpp b/src/part1/nebd_client.cpp index 071a02a9b5..e555496aa8 100644 --- a/src/part1/nebd_client.cpp +++ b/src/part1/nebd_client.cpp @@ -158,7 +158,7 @@ int NebdClient::Close(int fd) { LOG(WARNING) << "CloseFile rpc failed, error = " << cntl->ErrorText() << ", log id = " << cntl->log_id(); - return cntl->ErrorCode(); + return -1; } else { if (response.retcode() != RetCode::kOK) { LOG(ERROR) << "CloseFile failed, " @@ -201,7 +201,7 @@ int NebdClient::Extend(int fd, int64_t newsize) { LOG(WARNING) << "Resize RPC failed, error = " << cntl->ErrorText() << ", log id = " << cntl->log_id(); - return cntl->ErrorCode(); + return -1; } else { if (response.retcode() != nebd::client::RetCode::kOK) { LOG(ERROR) << "ExtendFile failed, " @@ -225,40 +225,40 @@ int NebdClient::Extend(int fd, int64_t newsize) { return ret; } -int64_t NebdClient::StatFile(int fd) { +int64_t NebdClient::GetFileSize(int fd) { auto task = [&](brpc::Controller* cntl, brpc::Channel* channel, bool* rpcFailed) -> int64_t { nebd::client::NebdFileService_Stub stub(channel); - nebd::client::StatFileRequest request; - nebd::client::StatFileResponse response; + nebd::client::GetInfoRequest request; + nebd::client::GetInfoResponse response; request.set_fd(fd); - stub.StatFile(cntl, &request, &response, nullptr); + stub.GetInfo(cntl, &request, &response, nullptr); *rpcFailed = cntl->Failed(); if (*rpcFailed) { - LOG(WARNING) << "StateFile rpc faield, error = " + LOG(WARNING) << "GetFileSize faield, error = " << cntl->ErrorText() << ", log id = " << cntl->log_id(); - return cntl->ErrorCode(); + return -1; } else { if (response.retcode() != nebd::client::RetCode::kOK) { - LOG(ERROR) << "StatFile failed, " + LOG(ERROR) << "GetFileSize failed, " << "retcode = " << response.retcode() <<", retmsg = " << response.retmsg() << ", fd = " << fd << ", log id = " << cntl->log_id(); return -1; } else { - return response.size(); + return response.info().size(); } } }; int64_t ret = ExecuteSyncRpc(task); if (ret < 0) { - LOG(ERROR) << "StatFile failed, fd = " << fd; + LOG(ERROR) << "GetFileSize failed, fd = " << fd; } return ret; } @@ -341,7 +341,7 @@ int64_t NebdClient::GetInfo(int fd) { LOG(WARNING) << "GetInfo rpc failed, error = " << cntl->ErrorText() << ", log id = " << cntl->log_id(); - return cntl->ErrorCode(); + return -1; } else { if (response.retcode() != nebd::client::RetCode::kOK) { LOG(ERROR) << "GetInfo failed, " @@ -351,7 +351,7 @@ int64_t NebdClient::GetInfo(int fd) { << ", log id = " << cntl->log_id(); return -1; } else { - return response.objsize(); + return response.info().objsize(); } } }; @@ -379,7 +379,7 @@ int NebdClient::InvalidCache(int fd) { LOG(WARNING) << "InvalidCache rpc failed, error = " << cntl->ErrorText() << ", log id = " << cntl->log_id(); - return cntl->ErrorCode(); + return -1; } else { if (response.retcode() != nebd::client::RetCode::kOK) { LOG(ERROR) << "InvalidCache failed, " diff --git a/src/part1/nebd_client.h b/src/part1/nebd_client.h index 43fcc29b96..f493e202c6 100644 --- a/src/part1/nebd_client.h +++ b/src/part1/nebd_client.h @@ -79,7 +79,7 @@ class NebdClient { * @param fd：文件的fd * @return 成功返回文件size，失败返回错误码 */ - int64_t StatFile(int fd); + int64_t GetFileSize(int fd); /** * @brief discard文件，异步函数 diff --git a/src/part2/define.h b/src/part2/define.h index ab1a4207cd..c08a03cf87 100644 --- a/src/part2/define.h +++ b/src/part2/define.h @@ -50,25 +50,24 @@ enum class NebdFileType { class NebdFileInstance; class NebdRequestExecutor; using NebdFileInstancePtr = std::shared_ptr; +using RWLockPtr = std::shared_ptr; // nebd server记录的文件信息内存结构 struct NebdFileRecord { // 文件读写锁，处理请求前加读锁，close文件的时候加写锁 // 避免close时还有请求未处理完 - RWLock rwLock; + RWLockPtr rwLock = std::make_shared(); // nebd server为该文件分配的唯一标识符 int fd = 0; // 文件名称 std::string fileName = ""; - // 文件类型：ceph文件、curve文件或测试文件 - NebdFileType type = NebdFileType::UNKWOWN; // 文件当前状态，opened表示文件已打开，closed表示文件已关闭 NebdFileStatus status = NebdFileStatus::CLOSED; // 该文件上一次收到心跳时的时间戳 uint64_t timeStamp = 0; // 文件在executor open时返回上下文信息，用于后续文件的请求处理 NebdFileInstancePtr fileInstance = nullptr; - // 文件实际的执行处理对象 + // 文件对应的executor的指针 NebdRequestExecutor* executor = nullptr; }; using NebdFileRecordPtr = std::shared_ptr; @@ -114,6 +113,7 @@ struct NebdFileInfo { const char LISTENADDRESS[] = "listen.address"; const char METAFILEPATH[] = "meta.file.path"; const char HEARTBEATTIMEOUTSEC[] = "heartbeat.timeout.sec"; +const char HEARTBEATCHECKINTERVALMS[] = "heartbeat.check.interval.ms"; } // namespace server } // namespace nebd diff --git a/src/part2/file_manager.cpp b/src/part2/file_manager.cpp index f9cda70278..d65ed10fdf 100644 --- a/src/part2/file_manager.cpp +++ b/src/part2/file_manager.cpp @@ -15,73 +15,51 @@ namespace nebd { namespace server { -NebdFileManager::NebdFileManager() - : heartbeatTimeoutS_(0) - , checkTimeoutIntervalMs_(0) - , isRunning_(false) - , metaFileManager_(nullptr) {} +NebdFileManager::NebdFileManager(FileRecordManagerPtr recordManager) + : isRunning_(false) + , fileRecordManager_(recordManager) {} NebdFileManager::~NebdFileManager() {} -int NebdFileManager::Init(NebdFileManagerOption option) { - if (isRunning_.load()) { - LOG(WARNING) << "Init failed, file manager is on running. "; - return -1; - } - metaFileManager_ = option.metaFileManager; - heartbeatTimeoutS_ = option.heartbeatTimeoutS; - checkTimeoutIntervalMs_ = option.checkTimeoutIntervalMs; - return 0; -} - int NebdFileManager::Run() { if (isRunning_.exchange(true)) { LOG(WARNING) << "file manager is on running."; return -1; } - CHECK(metaFileManager_ != nullptr) << "meta file manager is null."; int ret = Load(); if (ret < 0) { LOG(ERROR) << "Run file manager failed."; isRunning_.store(false); return -1; } - - checkTimeoutThread_ = - std::thread(&NebdFileManager::CheckTimeoutFunc, this); - LOG(INFO) << "Run file manager success."; return 0; } int NebdFileManager::Fini() { - if (isRunning_.exchange(false)) { - LOG(INFO) << "Stop file manager..."; - sleeper_.interrupt(); - checkTimeoutThread_.join(); - } - LOG(INFO) << "Stop file manager ok."; + isRunning_.store(false); + LOG(INFO) << "Stop file manager success."; return 0; } int NebdFileManager::Load() { - std::vector fileRecords; // 从元数据文件中读取持久化的文件信息 - int ret = metaFileManager_->ListFileRecord(&fileRecords); + int ret = fileRecordManager_->Load(); if (ret < 0) { - LOG(ERROR) << "List file record failed."; + LOG(ERROR) << "Load file record failed."; return ret; } + FileRecordMap fileRecords = fileRecordManager_->ListRecords(); // 根据持久化的信息重新open文件 int maxFd = 0; for (auto& fileRecord : fileRecords) { - maxFd = std::max(maxFd, fileRecord->fd); - Reopen(fileRecord); - bool updateSuccess = fileRecordMap_.UpdateRecord(fileRecord); - if (!updateSuccess) { - LOG(ERROR) << "Update file record failed. " - << "filename: " << fileRecord->fileName; - return -1; + maxFd = std::max(maxFd, fileRecord.first); + // reopen失败忽略，此时文件状态为closed，下次访问仍然会去open + // 这么考虑是为了防止个别文件有问题导致整个part2不可用 + int ret = Reopen(fileRecord.second); + if (ret < 0) { + LOG(WARNING) << "Reopen file failed. " + << "filenam: " << fileRecord.second.fileName; } } fdAlloc_.InitFd(maxFd); @@ -89,17 +67,6 @@ int NebdFileManager::Load() { return 0; } -int NebdFileManager::UpdateFileTimestamp(int fd) { - NebdFileRecordPtr fileRecord = fileRecordMap_.GetRecord(fd); - if (fileRecord == nullptr) { - LOG(WARNING) << "Update file timestamp failed, no record. " - << "fd: " << fd; - return -1; - } - fileRecord->timeStamp = TimeUtility::GetTimeofDayMs(); - return 0; -} - int NebdFileManager::Open(const std::string& filename) { int ret = OpenInternal(filename, true); if (ret < 0) { @@ -111,47 +78,52 @@ int NebdFileManager::Open(const std::string& filename) { return ret; } -int NebdFileManager::Close(int fd) { - NebdFileRecordPtr fileRecord = fileRecordMap_.GetRecord(fd); - if (fileRecord == nullptr) { +int NebdFileManager::Close(int fd, bool removeRecord) { + NebdFileRecord fileRecord; + bool getSuccess = fileRecordManager_->GetRecord(fd, &fileRecord); + if (!getSuccess) { LOG(WARNING) << "File record not exist, fd: " << fd; return 0; } - int ret = CloseInternal(fileRecord); + + // 用于和其他用户请求互斥，避免文件被close后，请求发到后端导致返回失败 + WriteLockGuard writeLock(*fileRecord.rwLock); + int ret = CloseInternal(fd); if (ret < 0) { LOG(ERROR) << "Close file failed. " << "fd: " << fd - << ", filename: " << fileRecord->fileName; + << ", filename: " << fileRecord.fileName; return -1; } - ret = metaFileManager_->RemoveFileRecord(fileRecord->fileName); - if (ret < 0) { - LOG(ERROR) << "Close file failed. " - << "fd: " << fd - << ", filename: " << fileRecord->fileName; - return -1; + if (removeRecord) { + bool removeSuccess = fileRecordManager_->RemoveRecord(fd); + if (!removeSuccess) { + LOG(ERROR) << "Remove file record failed. " + << "fd: " << fd + << ", filename: " << fileRecord.fileName; + return -1; + } } - fileRecordMap_.RemoveRecord(fd); LOG(INFO) << "Close file success. " << "fd: " << fd - << ", filename: " << fileRecord->fileName; + << ", filename: " << fileRecord.fileName; return 0; } int NebdFileManager::Discard(int fd, NebdServerAioContext* aioctx) { auto task = [&](NebdProcessClosure* done) { - NebdFileRecordPtr fileRecord = done->GetFileRecord(); + const NebdFileRecord& fileRecord = done->GetFileRecord(); done->SetClosure(aioctx->done); aioctx->done = done; - int ret = fileRecord->executor->Discard( - fileRecord->fileInstance.get(), aioctx); + int ret = fileRecord.executor->Discard( + fileRecord.fileInstance.get(), aioctx); if (ret < 0) { brpc::ClosureGuard doneGuard(done); aioctx->done = done->GetClosure(); done->SetClosure(nullptr); LOG(ERROR) << "Discard file failed. " << "fd: " << fd - << ", fileName: " << fileRecord->fileName + << ", fileName: " << fileRecord.fileName << ", context: " << *aioctx; return -1; } @@ -162,18 +134,18 @@ int NebdFileManager::Discard(int fd, NebdServerAioContext* aioctx) { int NebdFileManager::AioRead(int fd, NebdServerAioContext* aioctx) { auto task = [&](NebdProcessClosure* done) { - NebdFileRecordPtr fileRecord = done->GetFileRecord(); + const NebdFileRecord& fileRecord = done->GetFileRecord(); done->SetClosure(aioctx->done); aioctx->done = done; - int ret = fileRecord->executor->AioRead( - fileRecord->fileInstance.get(), aioctx); + int ret = fileRecord.executor->AioRead( + fileRecord.fileInstance.get(), aioctx); if (ret < 0) { brpc::ClosureGuard doneGuard(done); aioctx->done = done->GetClosure(); done->SetClosure(nullptr); LOG(ERROR) << "AioRead file failed. " << "fd: " << fd - << ", fileName: " << fileRecord->fileName + << ", fileName: " << fileRecord.fileName << ", context: " << *aioctx; return -1; } @@ -184,18 +156,18 @@ int NebdFileManager::AioRead(int fd, NebdServerAioContext* aioctx) { int NebdFileManager::AioWrite(int fd, NebdServerAioContext* aioctx) { auto task = [&](NebdProcessClosure* done) { - NebdFileRecordPtr fileRecord = done->GetFileRecord(); + const NebdFileRecord& fileRecord = done->GetFileRecord(); done->SetClosure(aioctx->done); aioctx->done = done; - int ret = fileRecord->executor->AioWrite( - fileRecord->fileInstance.get(), aioctx); + int ret = fileRecord.executor->AioWrite( + fileRecord.fileInstance.get(), aioctx); if (ret < 0) { brpc::ClosureGuard doneGuard(done); aioctx->done = done->GetClosure(); done->SetClosure(nullptr); LOG(ERROR) << "AioWrite file failed. " << "fd: " << fd - << ", fileName: " << fileRecord->fileName + << ", fileName: " << fileRecord.fileName << ", context: " << *aioctx; return -1; } @@ -206,18 +178,18 @@ int NebdFileManager::AioWrite(int fd, NebdServerAioContext* aioctx) { int NebdFileManager::Flush(int fd, NebdServerAioContext* aioctx) { auto task = [&](NebdProcessClosure* done) { - NebdFileRecordPtr fileRecord = done->GetFileRecord(); + const NebdFileRecord& fileRecord = done->GetFileRecord(); done->SetClosure(aioctx->done); aioctx->done = done; - int ret = fileRecord->executor->Flush( - fileRecord->fileInstance.get(), aioctx); + int ret = fileRecord.executor->Flush( + fileRecord.fileInstance.get(), aioctx); if (ret < 0) { brpc::ClosureGuard doneGuard(done); aioctx->done = done->GetClosure(); done->SetClosure(nullptr); LOG(ERROR) << "Flush file failed. " << "fd: " << fd - << ", fileName: " << fileRecord->fileName + << ", fileName: " << fileRecord.fileName << ", context: " << *aioctx; return -1; } @@ -229,14 +201,14 @@ int NebdFileManager::Flush(int fd, NebdServerAioContext* aioctx) { int NebdFileManager::Extend(int fd, int64_t newsize) { auto task = [&](NebdProcessClosure* done) { brpc::ClosureGuard doneGuard(done); - NebdFileRecordPtr fileRecord = done->GetFileRecord(); - int ret = fileRecord->executor->Extend( - fileRecord->fileInstance.get(), newsize); + const NebdFileRecord& fileRecord = done->GetFileRecord(); + int ret = fileRecord.executor->Extend( + fileRecord.fileInstance.get(), newsize); if (ret < 0) { LOG(ERROR) << "Extend file failed. " << "fd: " << fd << ", newsize: " << newsize - << ", fileName" << fileRecord->fileName; + << ", fileName" << fileRecord.fileName; return -1; } return 0; @@ -247,13 +219,13 @@ int NebdFileManager::Extend(int fd, int64_t newsize) { int NebdFileManager::GetInfo(int fd, NebdFileInfo* fileInfo) { auto task = [&](NebdProcessClosure* done) { brpc::ClosureGuard doneGuard(done); - NebdFileRecordPtr fileRecord = done->GetFileRecord(); - int ret = fileRecord->executor->GetInfo( - fileRecord->fileInstance.get(), fileInfo); + const NebdFileRecord& fileRecord = done->GetFileRecord(); + int ret = fileRecord.executor->GetInfo( + fileRecord.fileInstance.get(), fileInfo); if (ret < 0) { LOG(ERROR) << "Get file info failed. " << "fd: " << fd - << ", fileName" << fileRecord->fileName; + << ", fileName" << fileRecord.fileName; return -1; } return 0; @@ -264,13 +236,13 @@ int NebdFileManager::GetInfo(int fd, NebdFileInfo* fileInfo) { int NebdFileManager::InvalidCache(int fd) { auto task = [&](NebdProcessClosure* done) { brpc::ClosureGuard doneGuard(done); - NebdFileRecordPtr fileRecord = done->GetFileRecord(); - int ret = fileRecord->executor->InvalidCache( - fileRecord->fileInstance.get()); + const NebdFileRecord& fileRecord = done->GetFileRecord(); + int ret = fileRecord.executor->InvalidCache( + fileRecord.fileInstance.get()); if (ret < 0) { LOG(ERROR) << "Invalid cache failed. " << "fd: " << fd - << ", fileName" << fileRecord->fileName; + << ", fileName" << fileRecord.fileName; return -1; } return 0; @@ -278,147 +250,130 @@ int NebdFileManager::InvalidCache(int fd) { return ProcessRequest(fd, task); } -void NebdFileManager::CheckTimeoutFunc() { - while (sleeper_.wait_for( - std::chrono::milliseconds(checkTimeoutIntervalMs_))) { - std::unordered_map fileRecords - = fileRecordMap_.ListRecords(); - for (auto& fileRecord : fileRecords) { - uint64_t interval = - TimeUtility::GetTimeofDayMs() - fileRecord.second->timeStamp; - if (interval < (uint64_t)1000 * heartbeatTimeoutS_) { - continue; - } - std::string standardTime; - TimeUtility::TimeStampToStandard( - fileRecord.second->timeStamp / 1000, &standardTime); - LOG(INFO) << "Close file which has timed out. " - << "Last time received heartbeat: " << standardTime; - CloseInternal(fileRecord.second); - } - } -} - int NebdFileManager::OpenInternal(const std::string& fileName, bool create) { // 同名文件open需要互斥 NameLockGuard(nameLock_, fileName); - NebdFileRecordPtr fileRecord = fileRecordMap_.GetRecord(fileName); - if (fileRecord != nullptr && fileRecord->status == NebdFileStatus::OPENED) { - return fileRecord->fd; + NebdFileRecord fileRecord; + bool getSuccess = fileRecordManager_->GetRecord(fileName, &fileRecord); + if (getSuccess && fileRecord.status == NebdFileStatus::OPENED) { + return fileRecord.fd; } if (create) { - fileRecord = std::make_shared(); - fileRecord->fileName = fileName; - int newFd = GetValidFd(); - fileRecord->fd = newFd; - fileRecord->type = GetFileType(fileName); + fileRecord.fileName = fileName; + fileRecord.fd = GetValidFd(); } else { - if (fileRecord == nullptr) { + // create为false的情况下，如果record不存在，需要返回错误 + if (!getSuccess) { LOG(ERROR) << "open file failed: no record. " << "filename: " << fileName; return -1; } } + NebdFileType type = GetFileType(fileName); NebdRequestExecutor* executor = - NebdRequestExecutorFactory::GetExecutor(fileRecord->type); + NebdRequestExecutorFactory::GetExecutor(type); if (executor == nullptr) { LOG(ERROR) << "open file failed, invalid filename. " - << "filename: " << fileRecord->fileName; + << "filename: " << fileName; return -1; } - NebdFileInstancePtr fileInstance = executor->Open(fileRecord->fileName); + NebdFileInstancePtr fileInstance = executor->Open(fileName); if (fileInstance == nullptr) { LOG(ERROR) << "open file failed. " - << "filename: " << fileRecord->fileName; + << "filename: " << fileName; return -1; } - // 先持久化元数据信息到文件，再去更新内存 - int ret = metaFileManager_->UpdateFileRecord(fileRecord); - if (ret < 0) { - LOG(ERROR) << "persist file record failed. filename: " - << fileRecord->fileName; - return -1; - } - - fileRecord->executor = executor; - fileRecord->fileInstance = fileInstance; - fileRecord->status = NebdFileStatus::OPENED; - fileRecord->timeStamp = TimeUtility::GetTimeofDayMs(); + fileRecord.executor = executor; + fileRecord.fileInstance = fileInstance; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.timeStamp = TimeUtility::GetTimeofDayMs(); - bool updateSuccess = fileRecordMap_.UpdateRecord(fileRecord); + bool updateSuccess = fileRecordManager_->UpdateRecord(fileRecord); if (!updateSuccess) { - metaFileManager_->RemoveFileRecord(fileRecord->fileName); + executor->Close(fileInstance.get()); LOG(ERROR) << "Update file record failed. " - << "filename: " << fileRecord->fileName; + << "filename: " << fileName; return -1; } LOG(INFO) << "Open file success. " - << "fd: " << fileRecord->fd - << ", filename: " << fileRecord->fileName; - return fileRecord->fd; + << "fd: " << fileRecord.fd + << ", filename: " << fileRecord.fileName; + return fileRecord.fd; } -int NebdFileManager::Reopen(NebdFileRecordPtr fileRecord) { - fileRecord->type = GetFileType(fileRecord->fileName); +int NebdFileManager::Reopen(NebdFileRecord fileRecord) { + NebdFileType type = GetFileType(fileRecord.fileName); NebdRequestExecutor* executor = - NebdRequestExecutorFactory::GetExecutor(fileRecord->type); + NebdRequestExecutorFactory::GetExecutor(type); if (executor == nullptr) { - LOG(ERROR) << "open file failed, invalid filename. " - << "filename: " << fileRecord->fileName; + LOG(ERROR) << "Reopen file failed, invalid filename. " + << "filename: " << fileRecord.fileName; return -1; } NebdFileInstancePtr fileInstance = executor->Reopen( - fileRecord->fileName, fileRecord->fileInstance->addition); + fileRecord.fileName, fileRecord.fileInstance->addition); if (fileInstance == nullptr) { LOG(ERROR) << "Reopen file failed. " - << "filename: " << fileRecord->fileName; + << "filename: " << fileRecord.fileName; return -1; } - fileRecord->executor = executor; - fileRecord->fileInstance = fileInstance; - fileRecord->status = NebdFileStatus::OPENED; - fileRecord->timeStamp = TimeUtility::GetTimeofDayMs(); + fileRecord.executor = executor; + fileRecord.fileInstance = fileInstance; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.timeStamp = TimeUtility::GetTimeofDayMs(); + + bool updateSuccess = fileRecordManager_->UpdateRecord(fileRecord); + if (!updateSuccess) { + executor->Close(fileInstance.get()); + LOG(ERROR) << "Update file record failed. " + << "filename: " << fileRecord.fileName; + return -1; + } LOG(INFO) << "Reopen file success. " - << "fd: " << fileRecord->fd - << ", filename: " << fileRecord->fileName; + << "fd: " << fileRecord.fd + << ", filename: " << fileRecord.fileName; return 0; } -int NebdFileManager::CloseInternal(NebdFileRecordPtr fileRecord) { - if (fileRecord->status != NebdFileStatus::OPENED) { +int NebdFileManager::CloseInternal(int fd) { + NebdFileRecord fileRecord; + bool getSuccess = fileRecordManager_->GetRecord(fd, &fileRecord); + if (!getSuccess) { + LOG(WARNING) << "File record not exist, fd: " << fd; + return 0; + } + + if (fileRecord.status != NebdFileStatus::OPENED) { LOG(INFO) << "File has been closed. " - << "fd: " << fileRecord->fd - << ", filename: " << fileRecord->fileName; + << "fd: " << fileRecord.fd + << ", filename: " << fileRecord.fileName; return 0; } - // 用于和其他用户请求互斥，避免文件被close后，请求发到后端导致返回失败 - WriteLockGuard writeLock(fileRecord->rwLock); - int ret = fileRecord->executor->Close(fileRecord->fileInstance.get()); + int ret = fileRecord.executor->Close(fileRecord.fileInstance.get()); if (ret < 0) { LOG(ERROR) << "Close file failed. " - << "fd: " << fileRecord->fd - << ", filename: " << fileRecord->fileName; + << "fd: " << fileRecord.fd + << ", filename: " << fileRecord.fileName; return -1; } - fileRecord->status = NebdFileStatus::CLOSED; - LOG(INFO) << "Close file success. " - << "fd: " << fileRecord->fd - << ", filename: " << fileRecord->fileName; + + fileRecordManager_->UpdateFileStatus(fileRecord.fd, NebdFileStatus::CLOSED); return 0; } int NebdFileManager::ProcessRequest(int fd, ProcessTask task) { - NebdFileRecordPtr fileRecord = fileRecordMap_.GetRecord(fd); - if (fileRecord == nullptr) { + NebdFileRecord fileRecord; + bool getSuccess = fileRecordManager_->GetRecord(fd, &fileRecord); + if (!getSuccess) { LOG(WARNING) << "File record not exist, fd: " << fd; return -1; } @@ -427,11 +382,11 @@ int NebdFileManager::ProcessRequest(int fd, ProcessTask task) { new (std::nothrow) NebdProcessClosure(fileRecord); brpc::ClosureGuard doneGuard(done); - if (fileRecord->status != NebdFileStatus::OPENED) { - int ret = OpenInternal(fileRecord->fileName); + if (fileRecord.status != NebdFileStatus::OPENED) { + int ret = OpenInternal(fileRecord.fileName); if (ret != fd) { LOG(WARNING) << "Get opened file failed. " - << "filename: " << fileRecord->fileName + << "filename: " << fileRecord.fileName << ", fd: " << fd << ", ret: " << ret; return -1; @@ -442,21 +397,21 @@ int NebdFileManager::ProcessRequest(int fd, ProcessTask task) { if (ret < 0) { LOG(ERROR) << "Process request failed. " << "fd: " << fd - << ", fileName" << fileRecord->fileName; + << ", fileName" << fileRecord.fileName; return -1; } return 0; } -std::unordered_map NebdFileManager::GetRecordMap() { - return fileRecordMap_.ListRecords(); +FileRecordManagerPtr NebdFileManager::GetRecordManager() { + return fileRecordManager_; } int NebdFileManager::GetValidFd() { int fd = 0; while (true) { fd = fdAlloc_.GetNext(); - if (!fileRecordMap_.Exist(fd)) { + if (!fileRecordManager_->Exist(fd)) { break; } } diff --git a/src/part2/file_manager.h b/src/part2/file_manager.h index b97fe541dc..911ae633ab 100644 --- a/src/part2/file_manager.h +++ b/src/part2/file_manager.h @@ -11,7 +11,6 @@ #include #include #include -#include // NOLINT #include #include // NOLINT #include @@ -19,13 +18,10 @@ #include "src/common/rw_lock.h" #include "src/common/name_lock.h" #include "src/common/timeutility.h" -#include "src/common/interrupt_sleep.h" #include "src/part2/define.h" #include "src/part2/util.h" -#include "src/part2/metafile_manager.h" -#include "src/part2/file_record_map.h" -#include "src/part2/request_executor_ceph.h" -#include "src/part2/request_executor_curve.h" +#include "src/part2/filerecord_manager.h" +#include "src/part2/request_executor.h" #include "proto/client.pb.h" namespace nebd { @@ -36,34 +32,23 @@ using nebd::common::NameLockGuard; using nebd::common::WriteLockGuard; using nebd::common::ReadLockGuard; using nebd::common::TimeUtility; -using nebd::common::InterruptibleSleeper; -using MetaFileManagerPtr = std::shared_ptr; - -struct NebdFileManagerOption { - // 文件心跳超时时间（单位：秒） - uint32_t heartbeatTimeoutS; - // 心跳超时检测线程的检测间隔（时长：毫秒） - uint32_t checkTimeoutIntervalMs; - // metafilemanager 对象指针 - MetaFileManagerPtr metaFileManager; -}; // 处理用户请求时需要加读写锁，避免close时仍有用户IO未处理完成 // 对于异步IO来说，只有返回时才能释放读锁，所以封装成Closure // 在发送异步请求前，将closure赋值给NebdServerAioContext class NebdProcessClosure : public Closure { public: - explicit NebdProcessClosure(NebdFileRecordPtr record) - : record_(record) + explicit NebdProcessClosure(const NebdFileRecord& fileRecord) + : fileRecord_(fileRecord) , done_(nullptr) { - record_->rwLock.RDLock(); + fileRecord_.rwLock->RDLock(); } NebdProcessClosure() {} void Run() { std::unique_ptr selfGuard(this); brpc::ClosureGuard doneGuard(done_); - record_->rwLock.Unlock(); + fileRecord_.rwLock->Unlock(); } void SetClosure(Closure* done) { @@ -74,25 +59,19 @@ class NebdProcessClosure : public Closure { return done_; } - NebdFileRecordPtr GetFileRecord() { - return record_; + NebdFileRecord GetFileRecord() { + return fileRecord_; } private: - NebdFileRecordPtr record_; + NebdFileRecord fileRecord_; Closure* done_; }; class NebdFileManager { public: - NebdFileManager(); + explicit NebdFileManager(FileRecordManagerPtr recordManager); virtual ~NebdFileManager(); - /** - * 初始化FileManager各成员 - * @param option: 初始化参数 - * @return 成功返回0，失败返回-1 - */ - virtual int Init(NebdFileManagerOption option); /** * 停止FileManager并释放FileManager资源 * @return 成功返回0，失败返回-1 @@ -103,13 +82,6 @@ class NebdFileManager { * @return 成功返回0，失败返回-1 */ virtual int Run(); - /** - * part2收到心跳后，会通过该接口更新心跳中包含的文件在内存中记录的时间戳 - * 心跳检测线程会根据该时间戳判断是否需要关闭文件 - * @param fd: 指定文件的fd - * @return 成功返回0，失败返回-1 - */ - virtual int UpdateFileTimestamp(int fd); /** * 打开文件 * @param filename: 文件的filename @@ -119,9 +91,12 @@ class NebdFileManager { /** * 关闭文件 * @param fd: 文件的fd + * @param removeRecord: 是否要移除文件记录，true表示移除，false表示不移除 + * 如果是part1传过来的close请求，此参数为true + * 如果是heartbeat manager发起的close请求，此参数为false * @return 成功返回0，失败返回-1 */ - virtual int Close(int fd); + virtual int Close(int fd, bool removeRecord); /** * 给文件扩容 * @param fd: 文件的fd @@ -171,10 +146,12 @@ class NebdFileManager { */ virtual int InvalidCache(int fd); + // set public for test + // 启动时从metafile加载文件记录，并reopen文件 int Load(); - void CheckTimeoutFunc(); - std::unordered_map GetRecordMap(); + // 返回recordmanager + virtual FileRecordManagerPtr GetRecordManager(); private: /** @@ -185,17 +162,17 @@ class NebdFileManager { */ int OpenInternal(const std::string& fileName, bool create = false); /** - * 根据文件内存记录信息关闭指定文件 - * @param fileRecord: 文件的内存记录 + * 关闭指定文件，并将文件状态变更为CLOSED + * @param fd: 文件的内存记录 * @return: 成功返回0，失败返回-1 */ - int CloseInternal(NebdFileRecordPtr fileRecord); + int CloseInternal(int fd); /** * 根据文件内存记录信息重新open文件 * @param fileRecord: 文件的内存记录 * @return: 成功返回0，失败返回-1 */ - int Reopen(NebdFileRecordPtr fileRecord); + int Reopen(NebdFileRecord fileRecord); /** * 分配新的可用的fd * @return: 成功返回有效的fd，失败返回-1 @@ -211,27 +188,16 @@ class NebdFileManager { int ProcessRequest(int fd, ProcessTask task); private: - // TODO(YYK) 后续封装HeartbeatManager - // 文件心跳超时时长 - uint32_t heartbeatTimeoutS_; - // 心跳超时检测线程的检测时间间隔 - uint32_t checkTimeoutIntervalMs_; - // 心跳检测线程 - std::thread checkTimeoutThread_; - // 心跳检测线程的sleeper - InterruptibleSleeper sleeper_; // 当前filemanager的运行状态，true表示正在运行，false标为未运行 std::atomic isRunning_; // 文件名锁，对同名文件加锁 NameLock nameLock_; // fd分配器 FdAllocator fdAlloc_; - // TODO(YYK) 与metafilemanager整合成FileRecordManager - // 文件信息内存记录映射表 - FileRecordMap fileRecordMap_; - // 元数据文件持久化管理 - MetaFileManagerPtr metaFileManager_; + // nebd server 文件记录管理 + FileRecordManagerPtr fileRecordManager_; }; +using NebdFileManagerPtr = std::shared_ptr; } // namespace server } // namespace nebd diff --git a/src/part2/file_record_map.cpp b/src/part2/file_record_map.cpp deleted file mode 100644 index 83d6f9817a..0000000000 --- a/src/part2/file_record_map.cpp +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Project: nebd - * Created Date: Thursday January 16th 2020 - * Author: yangyaokai - * Copyright (c) 2020 netease - */ - -#include "src/part2/file_record_map.h" - -namespace nebd { -namespace server { - -NebdFileRecordPtr FileRecordMap::GetRecord(int fd) { - ReadLockGuard readGuard(rwLock_); - if (map_.find(fd) == map_.end()) { - return nullptr; - } - return map_[fd]; -} - -NebdFileRecordPtr FileRecordMap::GetRecord(const std::string& fileName) { - ReadLockGuard readGuard(rwLock_); - int fd = FindRecordUnlocked(fileName); - if (fd > 0) { - return map_[fd]; - } - return nullptr; -} - -bool FileRecordMap::UpdateRecord(NebdFileRecordPtr fileRecord) { - WriteLockGuard writeGuard(rwLock_); - int fd = fileRecord->fd; - - // 如果fd已存在，判断文件名是否相同，不同则返回错误，相同则直接返回成功 - if (map_.find(fd) != map_.end()) { - if (map_[fd]->fileName == fileRecord->fileName) { - return true; - } else { - LOG(ERROR) << "Conflict record. " - << "Old fileName: " << map_[fd]->fileName - << ", new fileName: " << fileRecord->fileName - << ", fd: " << fd; - return false; - } - } - - // 如果该文件记录已存在，则先删除旧的记录 - int oldFd = FindRecordUnlocked(fileRecord->fileName); - if (oldFd > 0) { - LOG(WARNING) << "Remove old record, filename: " - << fileRecord->fileName - << ", old fd: " << oldFd - << ", newFd: " << fd; - map_.erase(oldFd); - } - - map_[fd] = fileRecord; - return true; -} - -void FileRecordMap::RemoveRecord(int fd) { - WriteLockGuard writeGuard(rwLock_); - if (map_.find(fd) != map_.end()) { - map_.erase(fd); - } -} - -bool FileRecordMap::Exist(int fd) { - ReadLockGuard readGuard(rwLock_); - return map_.find(fd) != map_.end(); -} - -void FileRecordMap::Clear() { - WriteLockGuard writeGuard(rwLock_); - map_.clear(); -} - -std::unordered_map FileRecordMap::ListRecords() { - ReadLockGuard readGuard(rwLock_); - return map_; -} - -int FileRecordMap::FindRecordUnlocked(const std::string& fileName) { - int fd = 0; - for (const auto& pair : map_) { - if (pair.second->fileName == fileName) { - fd = pair.first; - break; - } - } - return fd; -} - -} // namespace server -} // namespace nebd - diff --git a/src/part2/file_record_map.h b/src/part2/file_record_map.h deleted file mode 100644 index bde2a5ad5c..0000000000 --- a/src/part2/file_record_map.h +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Project: nebd - * Created Date: Thursday January 16th 2020 - * Author: yangyaokai - * Copyright (c) 2020 netease - */ - -#ifndef SRC_PART2_FILE_RECORD_MAP_H_ -#define SRC_PART2_FILE_RECORD_MAP_H_ - -#include -#include -#include -#include -#include - -#include "src/common/rw_lock.h" -#include "src/part2/define.h" - -namespace nebd { -namespace server { - -using nebd::common::RWLock; -using nebd::common::WriteLockGuard; -using nebd::common::ReadLockGuard; - -// nebd server 文件信息的内存记录 -class FileRecordMap { - public: - FileRecordMap() {} - virtual ~FileRecordMap() {} - /** - * 根据fd获取指定文件的内存记录 - * @param fd: 文件的fd - * @return: 文件的内存记录 - */ - NebdFileRecordPtr GetRecord(int fd); - /** - * 根据文件名获取指定文件的内存记录 - * @param fileName: 文件名称 - * @return: 文件的内存记录 - */ - NebdFileRecordPtr GetRecord(const std::string& fileName); - /** - * 更新文件内存记录 - * 如果记录不存在，插入一条新的记录 - * 如果相同文件名的记录已存在，但是fd不同，则删除旧的记录，再插入当前记录 - * 如果相同fd的记录已存在，但是文件名不同，则返回失败 - * @param fileRecord: 文件记录 - * @return: 成功返回true，失败返回false - */ - bool UpdateRecord(NebdFileRecordPtr fileRecord); - /** - * 根据指定fd对应的文件记录 - * @param fd: 文件的fd - */ - void RemoveRecord(int fd); - /** - * 判断指定fd对应的文件记录是否存在 - * @param fd: 文件的fd - * @return: 如果存在返回true，不存在返回false - */ - bool Exist(int fd); - /** - * 清楚所有的记录 - */ - void Clear(); - /** - * 获取所有文件记录的映射表 - * @return: 返回所有文件记录的映射表 - */ - std::unordered_map ListRecords(); - - private: - // 获取指定文件名的文件记录（非线程安全） - int FindRecordUnlocked(const std::string& fileName); - - private: - // 保护文件记录映射表的读写锁 - RWLock rwLock_; - // 文件记录映射表 - std::unordered_map map_; -}; - -} // namespace server -} // namespace nebd - -#endif // SRC_PART2_FILE_RECORD_MAP_H_ diff --git a/src/part2/file_service.cpp b/src/part2/file_service.cpp index ae8ff6f86f..31a9605964 100644 --- a/src/part2/file_service.cpp +++ b/src/part2/file_service.cpp @@ -217,26 +217,6 @@ void NebdFileServiceImpl::Discard( } } -void NebdFileServiceImpl::StatFile( - google::protobuf::RpcController* cntl_base, - const nebd::client::StatFileRequest* request, - nebd::client::StatFileResponse* response, - google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); - response->set_retcode(RetCode::kNoOK); - - NebdFileInfo fileInfo; - int rc = fileManager_->GetInfo(request->fd(), &fileInfo); - if (rc < 0) { - LOG(ERROR) << "Stat file failed. " - << "fd: " << request->fd() - << ", return code: " << rc; - } else { - response->set_retcode(RetCode::kOK); - response->set_size(fileInfo.size); - } -} - void NebdFileServiceImpl::GetInfo( google::protobuf::RpcController* cntl_base, const nebd::client::GetInfoRequest* request, @@ -252,8 +232,12 @@ void NebdFileServiceImpl::GetInfo( << "fd: " << request->fd() << ", return code: " << rc; } else { + nebd::client::FileInfo* info = new nebd::client::FileInfo(); + info->set_size(fileInfo.size); + info->set_objsize(fileInfo.obj_size); + info->set_objnums(fileInfo.num_objs); response->set_retcode(RetCode::kOK); - response->set_objsize(fileInfo.obj_size); + response->set_allocated_info(info); } } @@ -265,7 +249,7 @@ void NebdFileServiceImpl::CloseFile( brpc::ClosureGuard doneGuard(done); response->set_retcode(RetCode::kNoOK); - int rc = fileManager_->Close(request->fd()); + int rc = fileManager_->Close(request->fd(), true); if (rc < 0) { LOG(ERROR) << "Close file failed. " << "fd: " << request->fd() diff --git a/src/part2/file_service.h b/src/part2/file_service.h index 1b7066e843..91fcb7e1f2 100644 --- a/src/part2/file_service.h +++ b/src/part2/file_service.h @@ -45,11 +45,6 @@ class NebdFileServiceImpl : public nebd::client::NebdFileService { nebd::client::ReadResponse* response, google::protobuf::Closure* done); - virtual void StatFile(google::protobuf::RpcController* cntl_base, - const nebd::client::StatFileRequest* request, - nebd::client::StatFileResponse* response, - google::protobuf::Closure* done); - virtual void GetInfo(google::protobuf::RpcController* cntl_base, const nebd::client::GetInfoRequest* request, nebd::client::GetInfoResponse* response, diff --git a/src/part2/filerecord_manager.cpp b/src/part2/filerecord_manager.cpp new file mode 100644 index 0000000000..fd71481d1a --- /dev/null +++ b/src/part2/filerecord_manager.cpp @@ -0,0 +1,168 @@ +/* + * Project: nebd + * Created Date: Thursday February 13th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#include + +#include "src/part2/filerecord_manager.h" + +namespace nebd { +namespace server { + +int FileRecordManager::Load() { + int ret = metaFileManager_->ListFileRecord(&map_); + if (ret < 0) { + LOG(ERROR) << "List file records from meta file failed. "; + return -1; + } + return 0; +} + +bool FileRecordManager::GetRecord(int fd, NebdFileRecord* fileRecord) { + ReadLockGuard readGuard(rwLock_); + if (map_.find(fd) == map_.end()) { + return false; + } + *fileRecord = map_[fd]; + return true; +} + +bool FileRecordManager::GetRecord(const std::string& fileName, + NebdFileRecord* fileRecord) { + ReadLockGuard readGuard(rwLock_); + int fd = FindRecordUnlocked(fileName); + if (fd > 0) { + *fileRecord = map_[fd]; + return true; + } + return false; +} + +bool FileRecordManager::UpdateRecord(const NebdFileRecord& fileRecord) { + WriteLockGuard writeGuard(rwLock_); + int fd = fileRecord.fd; + + // 如果fd已存在，判断文件名是否相同，不同则返回错误 + bool isConflict = map_.find(fd) != map_.end() && + map_[fd].fileName != fileRecord.fileName; + if (isConflict) { + LOG(ERROR) << "Conflict record. " + << "Old fileName: " << map_[fd].fileName + << ", new fileName: " << fileRecord.fileName + << ", fd: " << fd; + return false; + } + + FileRecordMap tempMap = map_; + // 如果该文件记录已存在，且fd不同，则先删除旧的记录 + int oldFd = FindRecordUnlocked(fileRecord.fileName); + if (oldFd != fileRecord.fd) { + LOG(WARNING) << "Remove old record, filename: " + << fileRecord.fileName + << ", old fd: " << oldFd + << ", newFd: " << fileRecord.fd; + tempMap.erase(oldFd); + } + tempMap[fd] = fileRecord; + + // 先更新元数据文件，如果更新成功再修改内存记录，否则返回失败 + int ret = metaFileManager_->UpdateMetaFile(tempMap); + if (ret < 0) { + LOG(ERROR) << "Update meta file failed. " + << "file name: " << fileRecord.fileName + << ", fd: " << fd; + return false; + } + map_ = std::move(tempMap); + return true; +} + +bool FileRecordManager::RemoveRecord(int fd) { + WriteLockGuard writeGuard(rwLock_); + if (map_.find(fd) == map_.end()) { + return true; + } + + FileRecordMap tempMap = map_; + tempMap.erase(fd); + + // 先更新元数据文件，如果更新成功再修改内存记录，否则返回失败 + int ret = metaFileManager_->UpdateMetaFile(tempMap); + if (ret < 0) { + LOG(ERROR) << "Update meta file failed. " + << "file name: " << map_[fd].fileName + << ", fd: " << fd; + return false; + } + map_ = std::move(tempMap); + return true; +} + +bool FileRecordManager::Exist(int fd) { + ReadLockGuard readGuard(rwLock_); + return map_.find(fd) != map_.end(); +} + +void FileRecordManager::Clear() { + WriteLockGuard writeGuard(rwLock_); + map_.clear(); +} + +FileRecordMap FileRecordManager::ListRecords() { + ReadLockGuard readGuard(rwLock_); + return map_; +} + +int FileRecordManager::FindRecordUnlocked(const std::string& fileName) { + int fd = -1; + for (const auto& pair : map_) { + if (pair.second.fileName == fileName) { + fd = pair.first; + break; + } + } + return fd; +} + +bool FileRecordManager::UpdateFileTimestamp(int fd, uint64_t timestamp) { + WriteLockGuard writeGuard(rwLock_); + if (map_.find(fd) == map_.end()) { + return false; + } + map_[fd].timeStamp = timestamp; + return true; +} + +bool FileRecordManager::GetFileTimestamp(int fd, uint64_t* timestamp) { + ReadLockGuard readGuard(rwLock_); + if (map_.find(fd) == map_.end()) { + return false; + } + *timestamp = map_[fd].timeStamp; + return true; +} + +bool FileRecordManager::UpdateFileStatus(int fd, NebdFileStatus status) { + WriteLockGuard writeGuard(rwLock_); + if (map_.find(fd) == map_.end()) { + return false; + } + map_[fd].status = status; + return true; +} + +bool FileRecordManager::GetFileStatus(int fd, NebdFileStatus* status) { + ReadLockGuard readGuard(rwLock_); + if (map_.find(fd) == map_.end()) { + return false; + } + *status = map_[fd].status; + return true; +} + +} // namespace server +} // namespace nebd + diff --git a/src/part2/filerecord_manager.h b/src/part2/filerecord_manager.h new file mode 100644 index 0000000000..90cc84f7a4 --- /dev/null +++ b/src/part2/filerecord_manager.h @@ -0,0 +1,130 @@ +/* + * Project: nebd + * Created Date: Thursday February 13th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#ifndef SRC_PART2_FILERECORD_MANAGER_H_ +#define SRC_PART2_FILERECORD_MANAGER_H_ + +#include +#include +#include +#include + +#include "src/common/rw_lock.h" +#include "src/part2/define.h" +#include "src/part2/metafile_manager.h" + +namespace nebd { +namespace server { + +using nebd::common::RWLock; +using nebd::common::WriteLockGuard; +using nebd::common::ReadLockGuard; +using FileRecordMap = std::unordered_map; + +// nebd server 文件记录管理（包括内存状态和持久化状态） +class FileRecordManager { + public: + explicit FileRecordManager(MetaFileManagerPtr metaFileManager) + : metaFileManager_(metaFileManager) {} + virtual ~FileRecordManager() {} + /** + * 从持久化文件中加载文件记录到内存 + * @return: 成功返回0，失败返回-1 + */ + virtual int Load(); + /** + * 根据fd获取指定文件的内存记录 + * @param fd: 文件的fd + * @param fileRecord[out]: 文件记录 + * @return: 成功返回true，失败返回false + */ + virtual bool GetRecord(int fd, NebdFileRecord* fileRecord); + /** + * 根据文件名获取指定文件的内存记录 + * @param fileName: 文件名称 + * @param fileRecord[out]: 文件记录 + * @return: 成功返回true，失败返回false + */ + virtual bool GetRecord(const std::string& fileName, + NebdFileRecord* fileRecord); + /** + * 修改文件持久化内容，并更新文件内存记录 + * 如果记录不存在，插入一条新的记录 + * 如果相同文件名的记录已存在，但是fd不同，则删除旧的记录，再插入当前记录 + * 如果相同fd的记录已存在，但是文件名不同，则返回失败 + * @param fileRecord: 文件记录 + * @return: 成功返回true，失败返回false + */ + virtual bool UpdateRecord(const NebdFileRecord& fileRecord); + /** + * 删除指定fd对应的文件记录，先更新元数据文件，然后跟新内存记录 + * @param fd: 文件的fd + * @return: 成功返回true，失败返回false + */ + virtual bool RemoveRecord(int fd); + /** + * 判断指定fd对应的文件记录是否存在 + * @param fd: 文件的fd + * @return: 如果存在返回true，不存在返回false + */ + virtual bool Exist(int fd); + /** + * 清楚所有的记录 + */ + virtual void Clear(); + /** + * 获取所有文件记录的映射表 + * @return: 返回所有文件记录的映射表 + */ + virtual FileRecordMap ListRecords(); + /** + * 更新文件内存记录的时间戳 + * @param fd: 指定的文件fd + * @param timestamp: 要变更为的时间戳值 + * @return: 成功返回true，指定fd不存在返回false + */ + virtual bool UpdateFileTimestamp(int fd, uint64_t timestamp); + /** + * 获取文件内存记录的时间戳 + * @param fd: 指定的文件fd + * @param timestamp[out]: 获取到的时间戳值 + * @return: 成功返回true，指定fd不存在返回false + */ + virtual bool GetFileTimestamp(int fd, uint64_t* timestamp); + /** + * 更新文件内存记录的状态 + * @param fd: 指定的文件fd + * @param status: 要变更为的文件状态 + * @return: 成功返回true，指定fd不存在返回false + */ + virtual bool UpdateFileStatus(int fd, NebdFileStatus status); + /** + * 获取文件内存记录的状态 + * @param fd: 指定的文件fd + * @param status[out]: 获取到的文件状态 + * @return: 成功返回true，指定fd不存在返回false + */ + virtual bool GetFileStatus(int fd, NebdFileStatus* status); + + private: + // 获取指定文件名的文件记录（非线程安全） + int FindRecordUnlocked(const std::string& fileName); + + private: + // 保护文件记录映射表的读写锁 + RWLock rwLock_; + // 文件记录映射表 + FileRecordMap map_; + // 持久化元数据文件管理 + MetaFileManagerPtr metaFileManager_; +}; +using FileRecordManagerPtr = std::shared_ptr; + +} // namespace server +} // namespace nebd + +#endif // SRC_PART2_FILERECORD_MANAGER_H_ diff --git a/src/part2/heartbeat_manager.cpp b/src/part2/heartbeat_manager.cpp new file mode 100644 index 0000000000..74b3d49bb0 --- /dev/null +++ b/src/part2/heartbeat_manager.cpp @@ -0,0 +1,85 @@ +/* + * Project: nebd + * Created Date: Friday February 14th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#include +#include + +#include "src/common/timeutility.h" +#include "src/part2/heartbeat_manager.h" + +namespace nebd { +namespace server { + +using common::TimeUtility; + +int HeartbeatManager::Run() { + if (isRunning_.exchange(true)) { + LOG(WARNING) << "heartbeat manager is on running."; + return -1; + } + + checkTimeoutThread_ = + std::thread(&HeartbeatManager::CheckTimeoutFunc, this); + LOG(INFO) << "Run heartbeat manager success."; + return 0; +} + +int HeartbeatManager::Fini() { + if (isRunning_.exchange(false)) { + LOG(INFO) << "Stopping heartbeat manager..."; + sleeper_.interrupt(); + checkTimeoutThread_.join(); + } + LOG(INFO) << "Stop heartbeat manager success."; + return 0; +} + +int HeartbeatManager::UpdateFileTimestamp(int fd, uint64_t timestamp) { + return fileManager_->GetRecordManager()->UpdateFileTimestamp(fd, timestamp); +} + +void HeartbeatManager::CheckTimeoutFunc() { + while (sleeper_.wait_for( + std::chrono::milliseconds(checkTimeoutIntervalMs_))) { + FileRecordManagerPtr recordManager = fileManager_->GetRecordManager(); + FileRecordMap fileRecords = recordManager->ListRecords(); + LOG_EVERY_N(INFO, 60 * 1000 / checkTimeoutIntervalMs_) + << "Checking timeout, file records num: " << fileRecords.size(); + for (auto& fileRecord : fileRecords) { + bool needClose = CheckNeedClosed(fileRecord.first); + if (!needClose) { + continue; + } + std::string standardTime; + TimeUtility::TimeStampToStandard( + fileRecord.second.timeStamp / 1000, &standardTime); + LOG(INFO) << "Close file which has timed out. " + << "Last time received heartbeat or request: " + << standardTime; + fileManager_->Close(fileRecord.first, false); + } + } +} + +bool HeartbeatManager::CheckNeedClosed(int fd) { + FileRecordManagerPtr recordManager = fileManager_->GetRecordManager(); + NebdFileRecord record; + bool getTimeSuccess = recordManager->GetRecord(fd, &record); + if (!getTimeSuccess) { + return false; + } + + uint64_t curTime = TimeUtility::GetTimeofDayMs(); + uint64_t interval = curTime - record.timeStamp; + // 文件如果是opened状态，并且已经超时，则需要调用close + bool needClose = record.status == NebdFileStatus::OPENED + && interval > (uint64_t)1000 * heartbeatTimeoutS_; + return needClose; +} + +} // namespace server +} // namespace nebd diff --git a/src/part2/heartbeat_manager.h b/src/part2/heartbeat_manager.h new file mode 100644 index 0000000000..0bfef6e231 --- /dev/null +++ b/src/part2/heartbeat_manager.h @@ -0,0 +1,74 @@ +/* + * Project: nebd + * Created Date: Friday February 14th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#ifndef SRC_PART2_HEARTBEAT_MANAGER_H_ +#define SRC_PART2_HEARTBEAT_MANAGER_H_ + +#include // NOLINT +#include +#include + +#include "src/common/interrupt_sleep.h" +#include "src/part2/file_manager.h" + +namespace nebd { +namespace server { + +using nebd::common::InterruptibleSleeper; + +struct HeartbeatManagerOption { + // 文件心跳超时时间（单位：秒） + uint32_t heartbeatTimeoutS; + // 心跳超时检测线程的检测间隔（时长：毫秒） + uint32_t checkTimeoutIntervalMs; + // filemanager 对象指针 + NebdFileManagerPtr fileManager; +}; + +// 负责文件心跳超时管理 +class HeartbeatManager { + public: + explicit HeartbeatManager(HeartbeatManagerOption option) + : isRunning_(false) + , heartbeatTimeoutS_(option.heartbeatTimeoutS) + , checkTimeoutIntervalMs_(option.checkTimeoutIntervalMs) + , fileManager_(option.fileManager) {} + virtual ~HeartbeatManager() {} + + // 启动心跳检测线程 + virtual int Run(); + // 停止心跳检测线程 + virtual int Fini(); + // part2收到心跳后，会通过该接口更新心跳中包含的文件在内存中记录的时间戳 + // 心跳检测线程会根据该时间戳判断是否需要关闭文件 + virtual int UpdateFileTimestamp(int fd, uint64_t timestamp); + + private: + // 心跳检测线程的函数执行体 + void CheckTimeoutFunc(); + // 判断文件是否需要close + bool CheckNeedClosed(int fd); + + private: + // 当前heartbeatmanager的运行状态，true表示正在运行，false标为未运行 + std::atomic isRunning_; + // 文件心跳超时时长 + uint32_t heartbeatTimeoutS_; + // 心跳超时检测线程的检测时间间隔 + uint32_t checkTimeoutIntervalMs_; + // 心跳检测线程 + std::thread checkTimeoutThread_; + // 心跳检测线程的sleeper + InterruptibleSleeper sleeper_; + // filemanager 对象指针 + NebdFileManagerPtr fileManager_; +}; + +} // namespace server +} // namespace nebd + +#endif // SRC_PART2_HEARTBEAT_MANAGER_H_ diff --git a/src/part2/heartbeat_service.cpp b/src/part2/heartbeat_service.cpp index 178fe2b4cd..60718f8081 100644 --- a/src/part2/heartbeat_service.cpp +++ b/src/part2/heartbeat_service.cpp @@ -5,11 +5,14 @@ * Copyright (c) 2020 netease */ +#include "src/common/timeutility.h" #include "src/part2/heartbeat_service.h" namespace nebd { namespace server { +using common::TimeUtility; + void NebdHeartbeatServiceImpl::KeepAlive( google::protobuf::RpcController* cntl_base, const nebd::client::HeartbeatRequest* request, @@ -17,9 +20,10 @@ void NebdHeartbeatServiceImpl::KeepAlive( google::protobuf::Closure* done) { brpc::ClosureGuard doneGuard(done); bool ok = true; + uint64_t curTime = TimeUtility::GetTimeofDayMs(); for (int i = 0; i < request->info_size(); ++i) { const auto& info = request->info(i); - int res = fileManager_->UpdateFileTimestamp(info.fd()); + int res = heartbeatManager_->UpdateFileTimestamp(info.fd(), curTime); if (res != 0) { LOG(WARNING) << "Update file timestamp fail, fd: " << info.fd() << ", name: " << info.name(); diff --git a/src/part2/heartbeat_service.h b/src/part2/heartbeat_service.h index 24830aa48e..b6286997f1 100644 --- a/src/part2/heartbeat_service.h +++ b/src/part2/heartbeat_service.h @@ -13,7 +13,7 @@ #include #include "proto/heartbeat.pb.h" -#include "src/part2/file_manager.h" +#include "src/part2/heartbeat_manager.h" namespace nebd { namespace server { @@ -21,8 +21,8 @@ namespace server { class NebdHeartbeatServiceImpl : public nebd::client::NebdHeartbeatService { public: explicit NebdHeartbeatServiceImpl( - std::shared_ptr fileManager) - : fileManager_(fileManager) {} + std::shared_ptr heartbeatManager) + : heartbeatManager_(heartbeatManager) {} virtual ~NebdHeartbeatServiceImpl() {} virtual void KeepAlive(google::protobuf::RpcController* cntl_base, const nebd::client::HeartbeatRequest* request, @@ -30,7 +30,7 @@ class NebdHeartbeatServiceImpl : public nebd::client::NebdHeartbeatService { google::protobuf::Closure* done); private: - std::shared_ptr fileManager_; + std::shared_ptr heartbeatManager_; }; } // namespace server diff --git a/src/part2/metafile_manager.cpp b/src/part2/metafile_manager.cpp index 1481527083..8b991f2ec7 100644 --- a/src/part2/metafile_manager.cpp +++ b/src/part2/metafile_manager.cpp @@ -14,87 +14,25 @@ namespace nebd { namespace server { -NebdMetaFileManager::NebdMetaFileManager(const std::string& metaFilePath, - std::shared_ptr wrapper, - std::shared_ptr parser) - : metaFilePath_(metaFilePath), wrapper_(wrapper), - parser_(parser), loaded_(false) {} +NebdMetaFileManager::NebdMetaFileManager( + const std::string& metaFilePath, + std::shared_ptr wrapper, + std::shared_ptr parser) + : metaFilePath_(metaFilePath) + , wrapper_(wrapper) + , parser_(parser) {} NebdMetaFileManager::~NebdMetaFileManager() {} -int NebdMetaFileManager::RemoveFileRecord(const std::string& fileName) { - std::unique_lock lock(mtx_); - std::vector fileRecords; - if (ListFileRecordUnlocked(&fileRecords) != 0) { - LOG(ERROR) << "List file record fail"; - return -1; - } - - // 删除fileName对应的元素 - bool exist = false; - for (auto it = fileRecords.begin(); it != fileRecords.end();) { - if ((*it)->fileName == fileName) { - exist = true; - it = fileRecords.erase(it); - break; - } else { - ++it; - } - } - - // 卷不存在返回0，不更新文件 - if (!exist) { - LOG(INFO) << "File " << fileName << " not found"; - return 0; - } - return UpdateMetaFile(fileRecords); -} - -int NebdMetaFileManager::UpdateFileRecord(const NebdFileRecordPtr& fileRecord) { - std::unique_lock lock(mtx_); - if (!fileRecord) { - LOG(ERROR) << "The argument is a null pointer!"; - return -1; - } - - std::vector fileRecords; - if (ListFileRecordUnlocked(&fileRecords) != 0) { - LOG(ERROR) << "List file record fail"; - return -1; - } - - bool exist = false; - for (const auto& record : fileRecords) { - // 如果文件已存在，则更新fd和addtion - if (record->fileName == fileRecord->fileName) { - // 如果fd和addition都没有变化的话直接return - if (RecordsEqual(record, fileRecord)) { - return 0; - } - record->fd = fileRecord->fd; - record->fileInstance = fileRecord->fileInstance; - exist = true; - break; - } - } - - // 不存在的话push一个进去 - if (!exist) { - fileRecords.emplace_back(fileRecord); - } - return UpdateMetaFile(fileRecords); -} - -int NebdMetaFileManager::UpdateMetaFile( - const std::vector& fileRecords) { +int NebdMetaFileManager::UpdateMetaFile(const FileRecordMap& fileRecords) { // 构建json Json::Value volumes; for (const auto& record : fileRecords) { Json::Value volume; - volume[kFileName] = record->fileName; - volume[kFd] = record->fd; - if (record->fileInstance) { - for (const auto item : record->fileInstance->addition) { + volume[kFileName] = record.second.fileName; + volume[kFd] = record.second.fd; + if (record.second.fileInstance) { + for (const auto item : record.second.fileInstance->addition) { volume[item.first] = item.second; } } @@ -108,7 +46,6 @@ int NebdMetaFileManager::UpdateMetaFile( LOG(ERROR) << "AtomicWriteFile fail"; return -1; } - metaDataJson_ = root; return 0; } @@ -141,24 +78,9 @@ int NebdMetaFileManager::AtomicWriteFile(const Json::Value& root) { return 0; } -int NebdMetaFileManager::ListFileRecord( - std::vector* fileRecords) { +int NebdMetaFileManager::ListFileRecord(FileRecordMap* fileRecords) { + CHECK(fileRecords != nullptr) << "fileRecords is null."; fileRecords->clear(); - std::unique_lock lock(mtx_); - return ListFileRecordUnlocked(fileRecords); -} - -int NebdMetaFileManager::ListFileRecordUnlocked( - std::vector* fileRecords) { - // 判断是否从文件加载过 - if (loaded_) { - return parser_->Parse(metaDataJson_, fileRecords); - } - return ListFileRecordFromFile(fileRecords); -} - -int NebdMetaFileManager::ListFileRecordFromFile( - std::vector* fileRecords) { std::ifstream in(metaFilePath_, std::ios::binary); if (!in) { // 这里不应该返回错误，第一次初始化的时候文件可能还未创建 @@ -182,20 +104,11 @@ int NebdMetaFileManager::ListFileRecordFromFile( LOG(ERROR) << "ConvertJsonToFileRecord fail"; return -1; } - metaDataJson_ = root; - loaded_ = true; return 0; } -bool NebdMetaFileManager::RecordsEqual(const NebdFileRecordPtr& record1, - const NebdFileRecordPtr& record2) { - return record1->fileName == record2->fileName - && record1->fd == record2->fd - && record1->fileInstance->addition == record2->fileInstance->addition; -} - int NebdMetaFileParser::Parse(const Json::Value& root, - std::vector* fileRecords) { + FileRecordMap* fileRecords) { // 没有volume字段 const auto& volumes = root[kVolumes]; if (volumes.isNull()) { @@ -206,14 +119,14 @@ int NebdMetaFileParser::Parse(const Json::Value& root, for (const auto& volume : volumes) { std::string fileName; int fd; - auto record = std::make_shared(); + NebdFileRecord record; if (volume[kFileName].isNull()) { LOG(ERROR) << "Parse json: " << root << " fail, no filename"; return -1; } else { - record->fileName = volume[kFileName].asString(); + record.fileName = volume[kFileName].asString(); } if (volume[kFd].isNull()) { @@ -221,13 +134,13 @@ int NebdMetaFileParser::Parse(const Json::Value& root, << " fail, no fd"; return -1; } else { - record->fd = volume[kFd].asInt(); + record.fd = volume[kFd].asInt(); } - auto fileType = GetFileType(record->fileName); - record->fileInstance = NebdFileInstanceFactory::GetInstance(fileType); - if (!record->fileInstance) { - LOG(ERROR) << "Unknown file type, filename: " << record->fileName; + auto fileType = GetFileType(record.fileName); + record.fileInstance = NebdFileInstanceFactory::GetInstance(fileType); + if (!record.fileInstance) { + LOG(ERROR) << "Unknown file type, filename: " << record.fileName; return -1; } @@ -238,10 +151,10 @@ int NebdMetaFileParser::Parse(const Json::Value& root, if (*iter == kFileName || *iter == kFd) { continue; } - record->fileInstance->addition.emplace(*iter, - volume[*iter].asString()); + record.fileInstance->addition.emplace( + *iter, volume[*iter].asString()); } - fileRecords->emplace_back(record); + fileRecords->emplace(record.fd, record); } return 0; } diff --git a/src/part2/metafile_manager.h b/src/part2/metafile_manager.h index 3ef222cf14..e89051583c 100644 --- a/src/part2/metafile_manager.h +++ b/src/part2/metafile_manager.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include // NOLINT #include // NOLINT @@ -22,6 +23,7 @@ namespace nebd { namespace server { using nebd::common::PosixWrapper; +using FileRecordMap = std::unordered_map; const char kVolumes[] = "volumes"; const char kFileName[] = "filename"; @@ -30,7 +32,7 @@ const char kFd[] = "fd"; class NebdMetaFileParser { public: int Parse(const Json::Value& root, - std::vector* fileRecords); + FileRecordMap* fileRecords); }; class NebdMetaFileManager { @@ -42,54 +44,23 @@ class NebdMetaFileManager { std::make_shared()); virtual ~NebdMetaFileManager(); - /** - * @brief 删除文件记录 - * - * @param fileName 要删除的文件名 - * - * @return 成功返回0，失败返回-1 - * - */ - virtual int RemoveFileRecord(const std::string& fileName); - - /** - * @brief 更新文件记录 - * - * @param fileRecord 要更新的文件记录 - * - * @return 成功返回0，失败返回-1 - * - */ - virtual int UpdateFileRecord(const NebdFileRecordPtr& fileRecord); - /** * @brief 列出文件记录 * - * @param fileRecord 要更新的文件记录 + * @param fileRecord 持久化的文件记录 * * @return 成功返回0，失败返回-1 * */ - virtual int ListFileRecord(std::vector* fileRecords); + virtual int ListFileRecord(FileRecordMap* fileRecords); - private: // 更新元数据文件 - int UpdateMetaFile( - const std::vector& fileRecords); + virtual int UpdateMetaFile(const FileRecordMap& fileRecords); + private: // 原子写文件 int AtomicWriteFile(const Json::Value& root); - // 判断两条记录是否相等 - bool RecordsEqual(const NebdFileRecordPtr& record1, - const NebdFileRecordPtr& record2); - - // 内部List，无锁版本 - int ListFileRecordUnlocked(std::vector* fileRecords); - - // 从文件list fileRecord - int ListFileRecordFromFile(std::vector* fileRecords); - private: // 元数据文件路径 std::string metaFilePath_; @@ -97,13 +68,8 @@ class NebdMetaFileManager { std::shared_ptr wrapper_; // 用于解析Json格式的元数据 std::shared_ptr parser_; - // 读写锁，保护json和文件 - std::mutex mtx_; - // 元数据的json - Json::Value metaDataJson_; - // 是否从文件加载过 - bool loaded_; }; +using MetaFileManagerPtr = std::shared_ptr; } // namespace server } // namespace nebd diff --git a/src/part2/nebd_server.cpp b/src/part2/nebd_server.cpp index c25fd18f9b..4661914f6f 100644 --- a/src/part2/nebd_server.cpp +++ b/src/part2/nebd_server.cpp @@ -38,6 +38,12 @@ int NebdServer::Init(const std::string &confPath) { return -1; } + bool initHeartbeatManagerOk = InitHeartbeatManager(); + if (false == initHeartbeatManagerOk) { + LOG(ERROR) << "NebdServer init heartbeatManager fail"; + return -1; + } + LOG(INFO) << "NebdServer init ok"; return 0; } @@ -59,6 +65,10 @@ int NebdServer::Fini() { if (fileManager_ != nullptr) { fileManager_->Fini(); } + + if (heartbeatManager_ != nullptr) { + heartbeatManager_->Fini(); + } return 0; } @@ -68,20 +78,18 @@ bool NebdServer::LoadConfFromFile(const std::string &confPath) { } bool NebdServer::InitFileManager() { - fileManager_ = std::make_shared(); - - NebdFileManagerOption opt; - bool initOptionOk = InitNebdFileManagerOption(&opt); - if (false == initOptionOk) { - LOG(ERROR) << "NebdServer init NebdFileManagerOption fail"; + MetaFileManagerPtr metaFileManager = InitMetaFileManager(); + if (nullptr == metaFileManager) { + LOG(ERROR) << "NebdServer init meta file manager fail"; return false; } - int initRes = fileManager_->Init(opt); - if (0 != initRes) { - LOG(ERROR) << "Init nebd file server fail"; - return false; - } + FileRecordManagerPtr recordmanager = + std::make_shared(metaFileManager); + CHECK(recordmanager != nullptr) << "Init record manager failed."; + + fileManager_ = std::make_shared(recordmanager); + CHECK(fileManager_ != nullptr) << "Init file manager failed."; int runRes = fileManager_->Run(); if (0 != runRes) { @@ -92,7 +100,17 @@ bool NebdServer::InitFileManager() { return true; } -bool NebdServer::InitNebdFileManagerOption(NebdFileManagerOption *opt) { +MetaFileManagerPtr NebdServer::InitMetaFileManager() { + std::string metaFilePath; + bool getOk = conf_.GetStringValue(METAFILEPATH, &metaFilePath); + if (false == getOk) { + return nullptr; + } + + return std::make_shared(metaFilePath); +} + +bool NebdServer::InitHeartbeatManagerOption(HeartbeatManagerOption *opt) { bool getOk = conf_.GetUInt32Value( HEARTBEATTIMEOUTSEC, &opt->heartbeatTimeoutS); if (false == getOk) { @@ -100,23 +118,33 @@ bool NebdServer::InitNebdFileManagerOption(NebdFileManagerOption *opt) { return false; } - opt->metaFileManager = InitMetaFileManager(); - if (nullptr == opt->metaFileManager) { - LOG(ERROR) << "NebdServer init meta file manager fail"; + getOk = conf_.GetUInt32Value( + HEARTBEATCHECKINTERVALMS, &opt->checkTimeoutIntervalMs); + if (false == getOk) { + LOG(ERROR) << "NebdServer get heartbeat.check.interval.ms fail"; return false; } + opt->fileManager = fileManager_; return true; } -MetaFileManagerPtr NebdServer::InitMetaFileManager() { - std::string metaFilePath; - bool getOk = conf_.GetStringValue(METAFILEPATH, &metaFilePath); - if (false == getOk) { - return nullptr; +bool NebdServer::InitHeartbeatManager() { + HeartbeatManagerOption option; + bool initOptionSuccess = InitHeartbeatManagerOption(&option); + if (!initOptionSuccess) { + LOG(ERROR) << "NebdServer init heartbeat manager option fail"; + return false; } + heartbeatManager_ = std::make_shared(option); + CHECK(heartbeatManager_ != nullptr) << "Init heartbeat manager failed."; - return std::make_shared(metaFilePath); + int runRes = heartbeatManager_->Run(); + if (0 != runRes) { + LOG(ERROR) << "nebd heartbeat manager run fail"; + return false; + } + return true; } bool NebdServer::StartServer() { @@ -129,7 +157,7 @@ bool NebdServer::StartServer() { return false; } - NebdHeartbeatServiceImpl heartbeatService(fileManager_); + NebdHeartbeatServiceImpl heartbeatService(heartbeatManager_); addFileServiceRes = server_.AddService( &heartbeatService, brpc::SERVER_DOESNT_OWN_SERVICE); if (0 != addFileServiceRes) { diff --git a/src/part2/nebd_server.h b/src/part2/nebd_server.h index b5a1fde887..d6b3bc7979 100644 --- a/src/part2/nebd_server.h +++ b/src/part2/nebd_server.h @@ -13,6 +13,7 @@ #include #include "src/common/configuration.h" #include "src/part2/file_manager.h" +#include "src/part2/heartbeat_manager.h" namespace nebd { namespace server { @@ -45,17 +46,23 @@ class NebdServer { bool InitFileManager(); /** - * @brief 初始化NebdFileManagerOption + * @brief 初始化NebdMetaFileManager + * @return nullptr-初始化不成功 否则表示初始化成功 + */ + MetaFileManagerPtr InitMetaFileManager(); + + /** + * @brief 初始化HeartbeatManagerOption * @param[out] opt * @return false-初始化失败 true-初始化成功 */ - bool InitNebdFileManagerOption(NebdFileManagerOption *opt); + bool InitHeartbeatManagerOption(HeartbeatManagerOption *opt); /** - * @brief 初始化NebdMetaFileManager - * @return nullptr-初始化不成功 否则表示初始化成功 + * @brief 初始化HeartbeatManager + * @return false-初始化失败 true-初始化成功 */ - MetaFileManagerPtr InitMetaFileManager(); + bool InitHeartbeatManager(); /** * @brief 启动brpc service @@ -75,6 +82,8 @@ class NebdServer { brpc::Server server_; // 用于接受和处理client端的各种请求 std::shared_ptr fileManager_; + // 负责文件心跳超时处理 + std::shared_ptr heartbeatManager_; }; } // namespace server diff --git a/tests/common/BUILD b/tests/common/BUILD new file mode 100644 index 0000000000..694c3dd2f2 --- /dev/null +++ b/tests/common/BUILD @@ -0,0 +1,12 @@ +cc_binary( + name = "nebd_common_unittest", + srcs = glob([ + "*.cpp", + ]), + deps = [ + "//external:gflags", + "//src/common:nebd_common", + "@com_google_googletest//:gtest_main", + ], +) + diff --git a/tests/part1/BUILD b/tests/part1/BUILD index a650cf50dc..8bfa8a96b8 100644 --- a/tests/part1/BUILD +++ b/tests/part1/BUILD @@ -18,7 +18,7 @@ cc_binary( deps = [ "//external:gflags", "//src/part1:nebdclient", - "//tests/part1:fake_lib", + "//tests/part1:fake_lib", "@com_google_googletest//:gtest_main", ], ) @@ -31,7 +31,7 @@ cc_binary( deps = [ "//external:gflags", "//src/part1:nebdclient", - "//tests/part1:fake_lib", + "//tests/part1:fake_lib", "@com_google_googletest//:gtest_main", ], linkstatic = False, @@ -40,8 +40,8 @@ cc_binary( cc_library( name = "fake_lib", srcs = glob([ - "fake_*", - "mock_*", + "fake_*", + "mock_*", ]), hdrs = glob(["*.h"]), deps = [ diff --git a/tests/part1/fake_file_service.cpp b/tests/part1/fake_file_service.cpp index be70655f4d..8562a6685b 100644 --- a/tests/part1/fake_file_service.cpp +++ b/tests/part1/fake_file_service.cpp @@ -98,22 +98,6 @@ void FakeNebdFileService::Discard(::google::protobuf::RpcController* controller, return; } -void FakeNebdFileService::StatFile(::google::protobuf::RpcController* controller, // NOLINT - const ::nebd::client::StatFileRequest* request, - ::nebd::client::StatFileResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); - brpc::Controller* cntl = static_cast(controller); - - LOG(INFO) << "logid = " << cntl->log_id() << ", StatFile."; - - response->set_retcode(RetCode::kOK); - response->set_retmsg("StatFile OK"); - response->set_size(fileSize_); - - return; -} - void FakeNebdFileService::ResizeFile(::google::protobuf::RpcController* controller, // NOLINT const ::nebd::client::ResizeRequest* request, ::nebd::client::ResizeResponse* response, @@ -154,10 +138,13 @@ void FakeNebdFileService::GetInfo(::google::protobuf::RpcController* controller, brpc::Controller* cntl = static_cast(controller); LOG(INFO) << "logid = " << cntl->log_id() << ", GetInfo."; - + nebd::client::FileInfo* info = new nebd::client::FileInfo(); + info->set_size(fileSize_); + info->set_objsize(fileSize_); + info->set_objnums(1); response->set_retcode(RetCode::kOK); response->set_retmsg("GetInfo OK"); - response->set_objsize(fileSize_); + response->set_allocated_info(info); return; } diff --git a/tests/part1/fake_file_service.h b/tests/part1/fake_file_service.h index b5ca013ec8..06d0e119b2 100644 --- a/tests/part1/fake_file_service.h +++ b/tests/part1/fake_file_service.h @@ -47,11 +47,6 @@ class FakeNebdFileService: public NebdFileService { ::nebd::client::DiscardResponse* response, ::google::protobuf::Closure* done) override; - void StatFile(::google::protobuf::RpcController* controller, - const ::nebd::client::StatFileRequest* request, - ::nebd::client::StatFileResponse* response, - ::google::protobuf::Closure* done) override; - void ResizeFile(::google::protobuf::RpcController* controller, const ::nebd::client::ResizeRequest* request, ::nebd::client::ResizeResponse* response, diff --git a/tests/part1/fake_heartbeat_service.h b/tests/part1/fake_heartbeat_service.h index e63d740700..ab119f42f4 100644 --- a/tests/part1/fake_heartbeat_service.h +++ b/tests/part1/fake_heartbeat_service.h @@ -38,7 +38,7 @@ class FakeHeartbeatService : public NebdHeartbeatService { } } - std::vector GetLatestRequestFileInfos() const { + std::vector GetLatestRequestFileInfos() const { return latestFileInfos_; } @@ -52,7 +52,7 @@ class FakeHeartbeatService : public NebdHeartbeatService { private: int invokeTimes_{0}; - std::vector latestFileInfos_; + std::vector latestFileInfos_; }; } // namespace client diff --git a/tests/part1/heartbeat_manager_unittest.cpp b/tests/part1/heartbeat_manager_unittest.cpp index 7f183aa8d7..9dd3278498 100644 --- a/tests/part1/heartbeat_manager_unittest.cpp +++ b/tests/part1/heartbeat_manager_unittest.cpp @@ -92,12 +92,12 @@ TEST_F(HeartbeatManagerTest, RequestValidTest) { option)); manager->Run(); - std::vector currentFileInfos; + std::vector currentFileInfos; // 添加一个文件 NebdClientFileInfo fileInfo(1, "/test1", FileLock("/test1.lock")); metaCache->AddFileInfo(fileInfo); - FileInfo info; + HeartbeatFileInfo info; info.set_fd(1); info.set_name("/test1"); currentFileInfos.push_back(info); @@ -123,7 +123,7 @@ TEST_F(HeartbeatManagerTest, RequestValidTest) { ASSERT_EQ(2, latestFileInfos.size()); std::sort(latestFileInfos.begin(), latestFileInfos.end(), - [](const FileInfo& lhs, const FileInfo& rhs) { + [](const HeartbeatFileInfo& lhs, const HeartbeatFileInfo& rhs) { return lhs.fd() < rhs.fd(); }); diff --git a/tests/part1/mock_file_service.h b/tests/part1/mock_file_service.h index 865601afc7..eeddae7750 100644 --- a/tests/part1/mock_file_service.h +++ b/tests/part1/mock_file_service.h @@ -41,10 +41,6 @@ class MockNebdFileService : public NebdFileService { const ::nebd::client::DiscardRequest* request, ::nebd::client::DiscardResponse* response, ::google::protobuf::Closure* done)); - MOCK_METHOD4(StatFile, void(::google::protobuf::RpcController* controller, - const ::nebd::client::StatFileRequest* request, - ::nebd::client::StatFileResponse* response, - ::google::protobuf::Closure* done)); MOCK_METHOD4(ResizeFile, void(::google::protobuf::RpcController* controller, const ::nebd::client::ResizeRequest* request, ::nebd::client::ResizeResponse* response, diff --git a/tests/part1/nebd_client_unittest.cpp b/tests/part1/nebd_client_unittest.cpp index 10484ad852..9e89dd3691 100644 --- a/tests/part1/nebd_client_unittest.cpp +++ b/tests/part1/nebd_client_unittest.cpp @@ -251,7 +251,7 @@ TEST_F(NebdFileClientTest, NoNebdServerTest) { ASSERT_TRUE(elapsed >= 900 && elapsed <= 1100); } ASSERT_EQ(-1, Extend4Nebd(1, kFileSize)); - ASSERT_EQ(-1, StatFile4Nebd(1)); + ASSERT_EQ(-1, GetFileSize4Nebd(1)); ASSERT_EQ(-1, GetInfo4Nebd(1)); ASSERT_EQ(-1, InvalidCache4Nebd(1)); ASSERT_EQ(-1, Close4Nebd(1)); @@ -269,7 +269,7 @@ TEST_F(NebdFileClientTest, CommonTest) { ASSERT_GE(fd, 0); ASSERT_EQ(0, Extend4Nebd(fd, kFileSize)); - ASSERT_EQ(kFileSize, StatFile4Nebd(fd)); + ASSERT_EQ(kFileSize, GetFileSize4Nebd(fd)); ASSERT_EQ(kFileSize, GetInfo4Nebd(fd)); ASSERT_EQ(0, InvalidCache4Nebd(fd)); @@ -401,14 +401,14 @@ TEST_F(NebdFileClientTest, ResponseFailTest) { } { - StatFileResponse response; + GetInfoResponse response; response.set_retcode(RetCode::kNoOK); - EXPECT_CALL(mockService, StatFile(_, _, _, _)) + EXPECT_CALL(mockService, GetInfo(_, _, _, _)) .Times(1) .WillOnce(DoAll( SetArgPointee<2>(response), - Invoke(MockClientFunc))); // NOLINT - ASSERT_EQ(-1, StatFile4Nebd(1)); + Invoke(MockClientFunc))); // NOLINT + ASSERT_EQ(-1, GetFileSize4Nebd(1)); } { diff --git a/tests/part1/nebd_test.cpp b/tests/part1/nebd_test.cpp deleted file mode 100644 index 07b75956b8..0000000000 --- a/tests/part1/nebd_test.cpp +++ /dev/null @@ -1,659 +0,0 @@ -/* - * Project: nebd - * Created Date: 2019-08-12 - * Author: hzchenwei7 - * Copyright (c) 2018 netease - */ -#include -#include -#include -#include -#include -#include // flock -#include // NOLINT -#include // NOLINT -#include "tests/part1/mock_client_service.h" -#include "src/part1/libnebd.h" -#include "src/part1/libnebd_file.h" -#include "src/part1/nebd_client.h" -#include "src/part1/nebd_lifecycle.h" - -DEFINE_string(uuid, "12345678-1234-1234-1234-123456789012", "uuid"); - -#define confFilename "tests/part1/nebd.conf" -#define filename "test_file_name" -#define DEFAULT_FD 1 -#define BUFSIZE 512 -#define DEFAULT_FILEZIZE (50*1024*1024) -std::mutex mtx; -std::condition_variable condition; -bool callback; - -void LibAioCallBackFunc(struct ClientAioContext* context) { - ASSERT_EQ(context->ret, 0); - callback = true; - condition.notify_one(); - ASSERT_EQ(context->retryCount, 0); -} - -void LibAioFailCallBackFunc(struct ClientAioContext* context) { - ASSERT_EQ(context->ret, -1); - callback = true; - condition.notify_one(); - ASSERT_EQ(context->retryCount, 0); -} - - -class nebdClientTest: public ::testing::Test { - protected: - void SetUp() override { - callback = false; - system("sudo killall client_server"); - system("sudo mkdir -p /etc/nebd"); - system("sudo cp tests/part1/nebd.conf /etc/nebd/nebd-client.conf"); - ASSERT_EQ(0, nebd_lib_init()); - } - void TearDown() override { - nebd_lib_uninit(); - } -}; - -TEST_F(nebdClientTest, nebdCommonTest) { - // nebd_lib_open_test - int ret; - ret = nebd_lib_open(filename); - ASSERT_EQ(DEFAULT_FD, ret); - - // nebd_lib_pread_test - char buf[BUFSIZE] = {}; - ret = nebd_lib_pread(DEFAULT_FD, buf, 0, BUFSIZE); - ASSERT_EQ(-1, ret); - - // nebd_lib_pwrite_test - memset(buf, 'a', BUFSIZE); - ret = nebd_lib_pwrite(DEFAULT_FD, buf, 0, BUFSIZE); - ASSERT_EQ(-1, ret); - - // nebd_lib_aio_pread_pwrite_test - char writeBuf[BUFSIZE] = "test"; - char readBuf[BUFSIZE]; - ClientAioContext context; - context.buf = writeBuf; - context.offset = 0; - context.length = BUFSIZE; - context.ret = 0; - context.op = LIBAIO_OP_WRITE; - context.cb = LibAioCallBackFunc; - context.retryCount = 0; - - ret = nebd_lib_aio_pwrite(DEFAULT_FD, &context); - ASSERT_EQ(ret, 0); - - std::unique_lock lock{mtx}; - condition.wait(lock); - ASSERT_TRUE(callback); - - callback = false; - ret = nebd_lib_aio_pread(DEFAULT_FD, &context); - ASSERT_EQ(ret, 0); - - condition.wait(lock); - ASSERT_TRUE(callback); - - // nebd_lib_sync_test - ret = nebd_lib_sync(DEFAULT_FD); - ASSERT_EQ(0, ret); - - // nebd_lib_discard_test - context.buf = nullptr; - context.offset = 0; - context.length = BUFSIZE; - context.ret = 0; - context.op = LIBAIO_OP_DISCARD; - context.cb = LibAioCallBackFunc; - context.retryCount = 0; - - ret = nebd_lib_discard(DEFAULT_FD, &context); - ASSERT_EQ(ret, 0); - - condition.wait(lock); - ASSERT_TRUE(callback); - - // nebd_lib_filesize_test - int64_t filesize; - filesize = nebd_lib_filesize(DEFAULT_FD); - ASSERT_EQ(DEFAULT_FILEZIZE, filesize); - - // nebd_lib_resize_test - ret = nebd_lib_resize(DEFAULT_FD, DEFAULT_FILEZIZE); - ASSERT_EQ(0, ret); - - // nebd_lib_getinfo_test - ret = nebd_lib_getinfo(DEFAULT_FD); - ASSERT_EQ(DEFAULT_FILEZIZE, ret); - - // nebd_lib_flush_test - context.buf = nullptr; - context.offset = 0; - context.length = 0; - context.ret = 0; - context.op = LIBAIO_OP_FLUSH; - context.cb = LibAioCallBackFunc; - context.retryCount = 0; - - ret = nebd_lib_flush(DEFAULT_FD, &context); - ASSERT_EQ(ret, 0); - - condition.wait(lock); - ASSERT_TRUE(callback); - - // nebd_lib_getinfo_test - filesize = nebd_lib_getinfo(DEFAULT_FD); - ASSERT_EQ(DEFAULT_FILEZIZE, filesize); - - // nebd_lib_invalidcache_test - ret = nebd_lib_invalidcache(DEFAULT_FD); - ASSERT_EQ(0, ret); - - // nebd_lib_close_test - ret = nebd_lib_close(DEFAULT_FD); - ASSERT_EQ(0, ret); -} - -class nebdFileClientTest: public ::testing::Test { - protected: - void SetUp() override { - system("sudo killall client_server"); - system("sudo mkdir -p /etc/nebd"); - system("sudo cp tests/part1/nebd.conf /etc/nebd/nebd-client.conf"); - } - void TearDown() override { - } -}; - -TEST_F(nebdFileClientTest, common_test) { - nebd::client::FileClient fileClientTest; - ASSERT_EQ(fileClientTest.Init(confFilename), 0); - ASSERT_EQ(fileClientTest.Open(filename), DEFAULT_FD); - ASSERT_EQ(fileClientTest.Extend(DEFAULT_FD, DEFAULT_FILEZIZE), 0); - ASSERT_EQ(fileClientTest.StatFile(DEFAULT_FD), DEFAULT_FILEZIZE); - ASSERT_EQ(fileClientTest.GetInfo(DEFAULT_FD), DEFAULT_FILEZIZE); - ASSERT_EQ(fileClientTest.InvalidCache(DEFAULT_FD), 0); - - char writeBuf[BUFSIZE] = "test"; - char readBuf[BUFSIZE]; - ClientAioContext context; - context.buf = writeBuf; - context.offset = 0; - context.length = BUFSIZE; - context.ret = 0; - context.op = LIBAIO_OP_WRITE; - context.cb = LibAioCallBackFunc; - context.retryCount = 0; - - ASSERT_EQ(fileClientTest.AioRead(DEFAULT_FD, &context), 0); - std::unique_lock lock{mtx}; - condition.wait(lock); - - ASSERT_EQ(fileClientTest.AioWrite(DEFAULT_FD, &context), 0); - condition.wait(lock); - ASSERT_TRUE(callback); - - context.buf = nullptr; - context.offset = 0; - context.length = BUFSIZE; - context.ret = 0; - context.op = LIBAIO_OP_DISCARD; - context.cb = LibAioCallBackFunc; - context.retryCount = 0; - ASSERT_EQ(fileClientTest.Discard(DEFAULT_FD, &context), 0); - condition.wait(lock); - ASSERT_TRUE(callback); - - context.buf = nullptr; - context.offset = 0; - context.length = 0; - context.ret = 0; - context.op = LIBAIO_OP_FLUSH; - context.cb = LibAioCallBackFunc; - context.retryCount = 0; - ASSERT_EQ(fileClientTest.Flush(DEFAULT_FD, &context), 0); - condition.wait(lock); - ASSERT_TRUE(callback); - - ASSERT_EQ(fileClientTest.Close(DEFAULT_FD), 0); - fileClientTest.Uninit(); - return; -} - -TEST_F(nebdFileClientTest, no_rpc_server_test) { - nebd::client::FileClient fileClientTest; - ASSERT_EQ(fileClientTest.Init("wrongConfPath"), -1); - - nebd::common::Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_TRUE(conf.LoadConfig()); - - ASSERT_EQ(fileClientTest.LoadConf(&conf), 0); - - ASSERT_EQ(fileClientTest.InitChannel("127.0.0.1:6667"), 0); - ASSERT_EQ(fileClientTest.Open(filename), -1); - ASSERT_EQ(fileClientTest.Extend(DEFAULT_FD, DEFAULT_FILEZIZE), -1); - ASSERT_EQ(fileClientTest.StatFile(DEFAULT_FD), -1); - ASSERT_EQ(fileClientTest.GetInfo(DEFAULT_FD), -1); - ASSERT_EQ(fileClientTest.InvalidCache(DEFAULT_FD), -1); - - // char writeBuf[BUFSIZE] = "test"; - // char readBuf[BUFSIZE]; - // ClientAioContext context; - // context.buf = writeBuf; - // context.offset = 0; - // context.length = BUFSIZE; - // context.ret = 0; - // context.op = LIBAIO_OP_WRITE; - // context.cb = LibAioCallBackFunc; - // context.retryCount = 0; - - // ASSERT_EQ(fileClientTest.AioRead(DEFAULT_FD, &context), 0); - // std::unique_lock lock{mtx}; - // condition.wait(lock); - - // ASSERT_EQ(fileClientTest.AioWrite(DEFAULT_FD, &context), 0); - // condition.wait(lock); - // ASSERT_TRUE(callback); - - // context.buf = nullptr; - // context.offset = 0; - // context.length = BUFSIZE; - // context.ret = 0; - // context.op = LIBAIO_OP_DISCARD; - // context.cb = LibAioCallBackFunc; - // context.retryCount = 0; - // ASSERT_EQ(fileClientTest.Discard(DEFAULT_FD, &context), 0); - // condition.wait(lock); - // ASSERT_TRUE(callback); - - // context.buf = nullptr; - // context.offset = 0; - // context.length = 0; - // context.ret = 0; - // context.op = LIBAIO_OP_FLUSH; - // context.cb = LibAioCallBackFunc; - // context.retryCount = 0; - // ASSERT_EQ(fileClientTest.Flush(DEFAULT_FD, &context), 0); - // condition.wait(lock); - // ASSERT_TRUE(callback); - - ASSERT_EQ(fileClientTest.Close(DEFAULT_FD), -1); - fileClientTest.Uninit(); - return; -} - -static void OpenFileFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::OpenFileRequest* request, - ::nebd::client::OpenFileResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); - // if (0 != gReadCntlFailedCode) { - // brpc::Controller *cntl = dynamic_cast(controller); - // cntl->SetFailed(-1, "open file controller error"); - // } -} - -static void CloseFileFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::CloseFileRequest* request, - ::nebd::client::CloseFileResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -static void ReadFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::ReadRequest* request, - ::nebd::client::ReadResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -static void WriteFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::WriteRequest* request, - ::nebd::client::WriteResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -static void DiscardFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::DiscardRequest* request, - ::nebd::client::DiscardResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -static void StatFileFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::StatFileRequest* request, - ::nebd::client::StatFileResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -static void ResizeFileFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::ResizeRequest* request, - ::nebd::client::ResizeResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -static void FlushFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::FlushRequest* request, - ::nebd::client::FlushResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -static void GetInfoFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::GetInfoRequest* request, - ::nebd::client::GetInfoResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -static void InvalidateCacheFunc(::google::protobuf::RpcController* controller, - const ::nebd::client::InvalidateCacheRequest* request, - ::nebd::client::InvalidateCacheResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); -} - -using ::testing::_; -using ::testing::Invoke; -using ::testing::Return; -using ::testing::AnyNumber; -using ::testing::DoAll; -using ::testing::SetArgPointee; -using ::testing::SetArgReferee; -using ::testing::InSequence; -using ::testing::AtLeast; -using ::testing::SaveArgPointee; - -TEST_F(nebdFileClientTest, rpc_fail_test) { - nebd::client::FileClient fileClientTest; - nebd::common::Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_TRUE(conf.LoadConfig()); - ASSERT_EQ(fileClientTest.LoadConf(&conf), 0); - - // add service - brpc::Server server; - nebd::client::MockQemuClientService clientService; - ASSERT_EQ(server.AddService(&clientService, - brpc::SERVER_DOESNT_OWN_SERVICE), 0); - - // start rpc server - brpc::ServerOptions option; - option.idle_timeout_sec = -1; - std::string listenAddr = "127.0.0.1:6667"; - ASSERT_EQ(server.Start(listenAddr.c_str(), &option), 0); - - ASSERT_EQ(fileClientTest.InitChannel("127.0.0.1:6667"), 0); - - ::nebd::client::OpenFileResponse openFileResponse; - openFileResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, OpenFile(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(openFileResponse), - Invoke(OpenFileFunc))); - ASSERT_EQ(fileClientTest.Open(filename), -1); - - ::nebd::client::ResizeResponse resizeResponse; - resizeResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, ResizeFile(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(resizeResponse), - Invoke(ResizeFileFunc))); - ASSERT_EQ(fileClientTest.Extend(DEFAULT_FD, DEFAULT_FILEZIZE), -1); - - ::nebd::client::StatFileResponse statFileResponse; - statFileResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, StatFile(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(statFileResponse), - Invoke(StatFileFunc))); - ASSERT_EQ(fileClientTest.StatFile(DEFAULT_FD), -1); - - ::nebd::client::GetInfoResponse getInfoResponse; - getInfoResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, GetInfo(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(getInfoResponse), - Invoke(GetInfoFunc))); - ASSERT_EQ(fileClientTest.GetInfo(DEFAULT_FD), -1); - - ::nebd::client::InvalidateCacheResponse invalidateCacheResponse; - invalidateCacheResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, InvalidateCache(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(invalidateCacheResponse), - Invoke(InvalidateCacheFunc))); - ASSERT_EQ(fileClientTest.InvalidCache(DEFAULT_FD), -1); - - char writeBuf[BUFSIZE] = "test"; - char readBuf[BUFSIZE]; - ClientAioContext context; - context.buf = writeBuf; - context.offset = 0; - context.length = BUFSIZE; - context.ret = 0; - context.op = LIBAIO_OP_WRITE; - context.cb = LibAioFailCallBackFunc; - context.retryCount = 0; - - ::nebd::client::ReadResponse readResponse; - readResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, Read(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(readResponse), - Invoke(ReadFunc))); - ASSERT_EQ(fileClientTest.AioRead(DEFAULT_FD, &context), 0); - std::unique_lock lock{mtx}; - condition.wait(lock); - ASSERT_TRUE(callback); - - - context.buf = readBuf; - context.offset = 0; - context.length = BUFSIZE; - context.ret = 0; - context.op = LIBAIO_OP_READ; - context.cb = LibAioFailCallBackFunc; - context.retryCount = 0; - ::nebd::client::WriteResponse writeResponse; - writeResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, Write(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(writeResponse), - Invoke(WriteFunc))); - ASSERT_EQ(fileClientTest.AioWrite(DEFAULT_FD, &context), 0); - condition.wait(lock); - ASSERT_TRUE(callback); - - context.buf = nullptr; - context.offset = 0; - context.length = BUFSIZE; - context.ret = 0; - context.op = LIBAIO_OP_DISCARD; - context.cb = LibAioFailCallBackFunc; - context.retryCount = 0; - ::nebd::client::DiscardResponse discardResponse; - discardResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, Discard(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(discardResponse), - Invoke(DiscardFunc))); - ASSERT_EQ(fileClientTest.Discard(DEFAULT_FD, &context), 0); - condition.wait(lock); - ASSERT_TRUE(callback); - - context.buf = nullptr; - context.offset = 0; - context.length = 0; - context.ret = 0; - context.op = LIBAIO_OP_FLUSH; - context.cb = LibAioFailCallBackFunc; - context.retryCount = 0; - ::nebd::client::FlushResponse flushResponse; - flushResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, Flush(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(flushResponse), - Invoke(FlushFunc))); - ASSERT_EQ(fileClientTest.Flush(DEFAULT_FD, &context), 0); - condition.wait(lock); - ASSERT_TRUE(callback); - - ::nebd::client::CloseFileResponse closeFileResponse; - closeFileResponse.set_retcode(nebd::client::RetCode::kNoOK); - EXPECT_CALL(clientService, CloseFile(_, _, _, _)) - .Times(1) - .WillOnce(DoAll(SetArgPointee<2>(closeFileResponse), - Invoke(CloseFileFunc))); - ASSERT_EQ(fileClientTest.Close(DEFAULT_FD), -1); - fileClientTest.Uninit(); - return; -} - -class nebdClientLifeCycleTest: public ::testing::Test { - protected: - void SetUp() override {} - void TearDown() override {} -}; - -TEST_F(nebdClientLifeCycleTest, init_when_part2_not_alive) { - ASSERT_EQ(Init4Qemu(confFilename), 0); - Uninit4Qemu(); -} - -TEST_F(nebdClientLifeCycleTest, init_when_part2_alive) { - ASSERT_EQ(Init4Qemu(confFilename), 0); - ASSERT_EQ(Init4Qemu(confFilename), 0); - Uninit4Qemu(); -} - -TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_stop_without_hb_test) { - using ::nebd::common::Configuration; - Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_EQ(conf.LoadConfig(), true); - - nebd::client::LifeCycleManager manager; - ASSERT_EQ(manager.Start(&conf), 0); - ASSERT_EQ(manager.IsPart2Alive(), true); - ASSERT_EQ(manager.IsHeartbeatThreadStart(), false); - ASSERT_EQ(manager.Start(&conf), 0); - ASSERT_EQ(manager.IsPart2Alive(), true); - ASSERT_EQ(manager.IsHeartbeatThreadStart(), false); - manager.Stop(); - ASSERT_EQ(manager.IsPart2Alive(), false); -} - -TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_stop_with_hb_test) { - using ::nebd::common::Configuration; - Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_EQ(conf.LoadConfig(), true); - - nebd::client::LifeCycleManager manager; - ASSERT_EQ(manager.Start(&conf), 0); - ASSERT_EQ(manager.IsPart2Alive(), true); - ASSERT_EQ(manager.Start(&conf), 0); - ASSERT_EQ(manager.IsPart2Alive(), true); - ASSERT_EQ(manager.IsHeartbeatThreadStart(), false); - manager.StartHeartbeat(); - ASSERT_EQ(manager.IsHeartbeatThreadStart(), true); - ASSERT_EQ(manager.IsPart2Alive(), true); - manager.Stop(); - ASSERT_EQ(manager.IsPart2Alive(), false); -} - -TEST_F(nebdClientLifeCycleTest, lifecycle_kill_part2_test) { - using ::nebd::common::Configuration; - Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_EQ(conf.LoadConfig(), true); - - nebd::client::LifeCycleManager manager; - ASSERT_EQ(manager.Start(&conf), 0); - ASSERT_EQ(manager.IsPart2Alive(), true); - ASSERT_EQ(manager.IsHeartbeatThreadStart(), false); - manager.StartHeartbeat(); - ASSERT_EQ(manager.IsHeartbeatThreadStart(), true); - manager.KillPart2(); - sleep(2); - ASSERT_EQ(manager.IsPart2Alive(), true); - manager.Stop(); - ASSERT_EQ(manager.IsPart2Alive(), false); -} - -TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_load_conf_fail_test) { - ::nebd::common::Configuration conf; - nebd::client::LifeCycleManager manager; - ASSERT_EQ(manager.Start(&conf), -1); -} - -TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_get_uuid_fail_test) { - ::nebd::common::Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_EQ(conf.LoadConfig(), true); - conf.SetStringValue("qemuProcName", "wrong_qemuProcName"); - - nebd::client::LifeCycleManager manager; - ASSERT_EQ(manager.Start(&conf), -1); -} - -TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_start_part2_fail) { - ::nebd::common::Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_EQ(conf.LoadConfig(), true); - conf.SetStringValue("part2ProcPath", "wrong_part2ProcPath"); - - nebd::client::LifeCycleManager manager; - ASSERT_EQ(manager.Start(&conf), -1); -} - -TEST_F(nebdClientLifeCycleTest, lifecycle_manager_start_read_port_fail) { - ::nebd::common::Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_EQ(conf.LoadConfig(), true); - conf.SetStringValue("metadataPrefix", "wrong_metadataPrefix"); - - nebd::client::LifeCycleManager manager; - ASSERT_EQ(manager.Start(&conf), -1); -} - -TEST_F(nebdClientLifeCycleTest, lifecycle_manager_lock_file_fail) { - ::nebd::common::Configuration conf; - conf.SetConfigPath(confFilename); - ASSERT_TRUE(conf.LoadConfig()); - - // 先占用文件锁 - std::string lockFile; - ASSERT_TRUE(conf.GetStringValue("lockFile", &lockFile)); - int fd = open(lockFile.c_str(), O_RDONLY | O_CREAT, 0644); - ASSERT_GT(fd, 0); - ASSERT_EQ(flock(fd, LOCK_EX | LOCK_NB), 0); - - // 再启lifecyclemanager，会因为文件锁被占用而失败 - nebd::client::LifeCycleManager manager; - ASSERT_EQ(manager.Start(&conf), -1); - - // 再释放文件锁 - ASSERT_EQ(flock(fd, LOCK_UN), 0); - close(fd); -} - -int main(int argc, char ** argv) { - ::testing::InitGoogleTest(&argc, argv); - ::testing::InitGoogleMock(&argc, argv); - google::ParseCommandLineFlags(&argc, &argv, false); - int ret = RUN_ALL_TESTS(); - - return ret; -} diff --git a/tests/part2/BUILD b/tests/part2/BUILD index 50707c44e4..55d806d89a 100644 --- a/tests/part2/BUILD +++ b/tests/part2/BUILD @@ -1,11 +1,12 @@ cc_binary( - name = "filerecordmap_test", + name = "filerecordmanager_test", srcs = glob([ - "file_record_map_unittest.cpp", + "file_record_manager_unittest.cpp", ]), deps = [ "//external:gflags", "//src/part2:nebdserver", + "//tests/part2:mock_lib", "@com_google_googletest//:gtest_main", ], ) @@ -18,7 +19,7 @@ cc_binary( deps = [ "//external:gflags", "//src/part2:nebdserver", - "//tests/part2:mock_lib", + "//tests/part2:mock_lib", "@com_google_googletest//:gtest_main", ], ) @@ -31,7 +32,7 @@ cc_binary( deps = [ "//external:gflags", "//src/part2:nebdserver", - "//tests/part2:mock_lib", + "//tests/part2:mock_lib", "@com_google_googletest//:gtest_main", ], ) @@ -44,7 +45,20 @@ cc_binary( deps = [ "//external:gflags", "//src/part2:nebdserver", - "//tests/part2:mock_lib", + "//tests/part2:mock_lib", + "@com_google_googletest//:gtest_main", + ], +) + +cc_binary( + name = "heartbeat_manager_test", + srcs = glob([ + "heartbeat_manager_unittest.cpp", + ]), + deps = [ + "//external:gflags", + "//src/part2:nebdserver", + "//tests/part2:mock_lib", "@com_google_googletest//:gtest_main", ], ) @@ -57,7 +71,7 @@ cc_binary( deps = [ "//external:gflags", "//src/part2:nebdserver", - "//tests/part2:mock_lib", + "//tests/part2:mock_lib", "@com_google_googletest//:gtest_main", ], ) @@ -82,7 +96,7 @@ cc_binary( deps = [ "//external:gflags", "//src/part2:nebdserver", - "//tests/part2:mock_lib", + "//tests/part2:mock_lib", "@com_google_googletest//:gtest_main", ], linkopts = ["-lpthread"], diff --git a/tests/part2/CMakeLists.txt b/tests/part2/CMakeLists.txt index ece5580748..781b59ee41 100644 --- a/tests/part2/CMakeLists.txt +++ b/tests/part2/CMakeLists.txt @@ -17,7 +17,9 @@ set(PART2_TEST_LINK # libpart2mock.so set(PART2_MOCK_SRC + mock_heartbeat_manager.h mock_metafile_manager.h + mock_filerecord_manager.h mock_request_executor.h mock_file_manager.h mock_posix_wrapper.h @@ -27,13 +29,13 @@ set_target_properties(part2mock PROPERTIES LINKER_LANGUAGE CXX) target_link_libraries(part2mock ${PART2_TEST_LINK}) -# filerecordmap_test -set(RECORD_MAP_TEST_SRC file_record_map_unittest.cpp) -add_executable(filerecordmap_test ${RECORD_MAP_TEST_SRC}) -target_link_libraries(filerecordmap_test +# filerecordmanager_test +set(RECORD_MANAGER_TEST_SRC file_record_manager_unittest.cpp) +add_executable(filerecordmanager_test ${RECORD_MANAGER_TEST_SRC}) +target_link_libraries(filerecordmanager_test ${PART2_TEST_LINK}) -install(TARGETS filerecordmap_test DESTINATION bin) -add_test(NAME filerecordmap_test COMMAND filerecordmap_test) +install(TARGETS filerecordmanager_test DESTINATION bin) +add_test(NAME filerecordmanager_test COMMAND filerecordmanager_test) # filemanager_test set(FILE_MANAGER_TEST_SRC file_manager_unittest.cpp) @@ -62,6 +64,15 @@ target_link_libraries(heartbeat_service_test install(TARGETS heartbeat_service_test DESTINATION bin) add_test(NAME heartbeat_service_test COMMAND heartbeat_service_test) +# heartbeat_manager_test +set(HEARTBEAT_MANAGER_TEST_SRC heartbeat_manager_unittest.cpp) +add_executable(heartbeat_manager_test ${HEARTBEAT_MANAGER_TEST_SRC}) +target_link_libraries(heartbeat_manager_test + part2mock + ${PART2_TEST_LINK}) +install(TARGETS heartbeat_manager_test DESTINATION bin) +add_test(NAME heartbeat_manager_test COMMAND heartbeat_manager_test) + # metafile_manager_test set(METAFILE_MANAGER_TEST_SRC metafile_manager_test.cpp) add_executable(metafile_manager_test ${METAFILE_MANAGER_TEST_SRC}) diff --git a/tests/part2/file_manager_unittest.cpp b/tests/part2/file_manager_unittest.cpp index b513b919ab..c61369eaf6 100644 --- a/tests/part2/file_manager_unittest.cpp +++ b/tests/part2/file_manager_unittest.cpp @@ -11,7 +11,7 @@ #include #include "src/part2/file_manager.h" -#include "tests/part2/mock_metafile_manager.h" +#include "tests/part2/mock_filerecord_manager.h" #include "tests/part2/mock_request_executor.h" namespace nebd { @@ -35,666 +35,813 @@ class FileManagerTest : public ::testing::Test { void SetUp() { aioContext_ = new NebdServerAioContext(); mockInstance_ = std::make_shared(); - metaFileManager_ = std::make_shared(); executor_ = std::make_shared(); g_test_executor = executor_.get(); - option_.heartbeatTimeoutS = 5; - option_.checkTimeoutIntervalMs = 1000; - option_.metaFileManager = metaFileManager_; - ASSERT_EQ(fileManager_.Init(option_), 0); + filerecordManager_ = std::make_shared(); + fileManager_ = std::make_shared(filerecordManager_); + ON_CALL(*filerecordManager_, Exist(_)) + .WillByDefault(Return(false)); } void TearDown() { delete aioContext_; } - // 构造初始环境，open两个文件 - void InitEnv() { - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(0)); - int fd = fileManager_.Open(testFile1); - ASSERT_EQ(fd, 1); - - EXPECT_CALL(*executor_, Open(testFile2)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(0)); - fd = fileManager_.Open(testFile2); - ASSERT_EQ(fd, 2); - } - protected: - NebdFileManager fileManager_; - std::shared_ptr metaFileManager_; + std::shared_ptr fileManager_; + std::shared_ptr filerecordManager_; std::shared_ptr executor_; std::shared_ptr mockInstance_; NebdServerAioContext* aioContext_; - NebdFileManagerOption option_; }; TEST_F(FileManagerTest, OpenTest) { // open一个不存在的文件 + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillOnce(Return(false)); EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(0)); - int fd = fileManager_.Open(testFile1); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(true)); + int fd = fileManager_->Open(testFile1); ASSERT_EQ(fd, 1); // 重复open + NebdFileRecord fileRecord; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.fd = 1; + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillOnce(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, Open(_)) .Times(0); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .Times(0); - fd = fileManager_.Open(testFile1); + fd = fileManager_->Open(testFile1); ASSERT_EQ(fd, 1); - // 再次open一个新的文件 - EXPECT_CALL(*executor_, Open(testFile2)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(0)); - fd = fileManager_.Open(testFile2); - ASSERT_EQ(fd, 2); - // open 已经close的文件 - std::unordered_map recordMap - = fileManager_.GetRecordMap(); - ASSERT_NE(recordMap[1], nullptr); - recordMap[1]->status = NebdFileStatus::CLOSED; - // 重新open该文件 + fileRecord.status = NebdFileStatus::CLOSED; + fileRecord.fd = 1; + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillOnce(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(0)); - fd = fileManager_.Open(testFile1); - // 校验结果，生成了新的记录并删除了旧的记录 - ASSERT_EQ(fd, 3); - recordMap = fileManager_.GetRecordMap(); - ASSERT_NE(recordMap[3], nullptr); - ASSERT_EQ(recordMap[1], nullptr); - ASSERT_EQ(3, recordMap[3]->fd); - ASSERT_EQ(testFile1, recordMap[3]->fileName); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(true)); + fd = fileManager_->Open(testFile1); + ASSERT_EQ(fd, 2); } TEST_F(FileManagerTest, OpenFailTest) { // 调用后端open接口时出错 + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillOnce(Return(false)); EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(nullptr)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) .Times(0); - int fd = fileManager_.Open(testFile1); + int fd = fileManager_->Open(testFile1); ASSERT_EQ(fd, -1); // 持久化元数据信息失败 + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillOnce(Return(false)); EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(-1)); - fd = fileManager_.Open(testFile1); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(false)); + EXPECT_CALL(*executor_, Close(mockInstance_.get())) + .WillOnce(Return(0)); + fd = fileManager_->Open(testFile1); ASSERT_EQ(fd, -1); // Open一个非法的filename + EXPECT_CALL(*filerecordManager_, GetRecord(unknownFile, _)) + .WillOnce(Return(false)); EXPECT_CALL(*executor_, Open(_)) .Times(0); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) .Times(0); - fd = fileManager_.Open(unknownFile); + fd = fileManager_->Open(unknownFile); ASSERT_EQ(fd, -1); } TEST_F(FileManagerTest, LoadTest) { // 初始化从metafile返回的元数据 - std::vector fileRecords; - NebdFileRecordPtr record1 = std::make_shared(); - NebdFileRecordPtr record2 = std::make_shared(); - NebdFileRecordPtr record3 = std::make_shared(); - record1->fileName = testFile1; - record1->fd = 1; - record1->fileInstance = mockInstance_; - record2->fileName = testFile2; - record2->fd = 2; - record2->fileInstance = mockInstance_; - record3->fileName = unknownFile; - record3->fd = 3; - record3->fileInstance = mockInstance_; - fileRecords.push_back(record1); - fileRecords.push_back(record2); - fileRecords.push_back(record3); - - EXPECT_CALL(*metaFileManager_, ListFileRecord(_)) - .WillOnce(DoAll(SetArgPointee<0>(fileRecords), - Return(0))); + FileRecordMap fileRecords; + NebdFileRecord record1; + record1.fileName = testFile1; + record1.fd = 1; + record1.fileInstance = mockInstance_; + fileRecords.emplace(1, record1); + + EXPECT_CALL(*filerecordManager_, Load()) + .WillOnce(Return(0)); + EXPECT_CALL(*filerecordManager_, ListRecords()) + .WillOnce(Return(fileRecords)); EXPECT_CALL(*executor_, Reopen(testFile1, _)) .WillOnce(Return(mockInstance_)); - // reopen失败，不会导致load失败 - EXPECT_CALL(*executor_, Reopen(testFile2, _)) - .WillOnce(Return(nullptr)); - // 无法识别的文件不会被打开，但不会导致load失败 - EXPECT_CALL(*executor_, Reopen(unknownFile, _)) - .Times(0); - ASSERT_EQ(0, fileManager_.Load()); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(true)); + ASSERT_EQ(0, fileManager_->Load()); } TEST_F(FileManagerTest, LoadFailTest) { - // 初始化 - std::vector fileRecords; - NebdFileRecordPtr record1 = std::make_shared(); - NebdFileRecordPtr record2 = std::make_shared(); - record1->fileName = testFile1; - record1->fd = 1; - record1->fileInstance = mockInstance_; - // fd相同，文件名不同会出现冲突（这种情况理论也不会发生） - record2->fileName = testFile2; - record2->fd = 1; - record2->fileInstance = mockInstance_; - fileRecords.push_back(record1); - fileRecords.push_back(record2); - - // ListFileRecord 失败，返回失败 - EXPECT_CALL(*metaFileManager_, ListFileRecord(_)) + // 初始化从metafile返回的元数据 + FileRecordMap fileRecords; + NebdFileRecord record1; + NebdFileRecord record2; + NebdFileRecord record3; + record1.fileName = testFile1; + record1.fd = 1; + record1.fileInstance = mockInstance_; + record2.fileName = testFile2; + record2.fd = 2; + record2.fileInstance = mockInstance_; + record3.fileName = unknownFile; + record3.fd = 3; + record3.fileInstance = mockInstance_; + fileRecords.emplace(1, record1); + fileRecords.emplace(2, record2); + fileRecords.emplace(3, record3); + + // load 失败 + EXPECT_CALL(*filerecordManager_, Load()) .WillOnce(Return(-1)); - ASSERT_EQ(-1, fileManager_.Load()); + ASSERT_EQ(-1, fileManager_->Load()); - EXPECT_CALL(*metaFileManager_, ListFileRecord(_)) - .WillOnce(DoAll(SetArgPointee<0>(fileRecords), - Return(0))); + // 其他失败都会被忽略，不会导致load失败 + EXPECT_CALL(*filerecordManager_, Load()) + .WillOnce(Return(0)); + EXPECT_CALL(*filerecordManager_, ListRecords()) + .WillOnce(Return(fileRecords)); + // uodate record 失败，不会导致load失败 EXPECT_CALL(*executor_, Reopen(testFile1, _)) .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(false)); // reopen失败，不会导致load失败 EXPECT_CALL(*executor_, Reopen(testFile2, _)) - .WillOnce(Return(mockInstance_)); - ASSERT_EQ(-1, fileManager_.Load()); + .WillOnce(Return(nullptr)); + // 无法识别的文件不会被打开，但不会导致load失败 + EXPECT_CALL(*executor_, Reopen(unknownFile, _)) + .Times(0); + ASSERT_EQ(0, fileManager_->Load()); } TEST_F(FileManagerTest, CloseTest) { - InitEnv(); // 指定的fd不存在,直接返回成功 - ASSERT_EQ(0, fileManager_.Close(3)); - - // fd存在 + EXPECT_CALL(*filerecordManager_, GetRecord(3, _)) + .WillOnce(Return(false)); + ASSERT_EQ(0, fileManager_->Close(3, true)); + + // fd存在,文件状态为opened,removerecord为true + NebdFileRecord fileRecord; + fileRecord.fd = 1; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.executor = executor_.get(); + fileRecord.fileInstance = mockInstance_; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .Times(2) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, Close(NotNull())) .WillOnce(Return(0)); - EXPECT_CALL(*metaFileManager_, RemoveFileRecord(testFile1)) - .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_.Close(1)); - + EXPECT_CALL(*filerecordManager_, UpdateFileStatus(1, _)) + .WillOnce(Return(true)); + EXPECT_CALL(*filerecordManager_, RemoveRecord(1)) + .WillOnce(Return(true)); + ASSERT_EQ(0, fileManager_->Close(1, true)); + + // fd存在,文件状态为opened,removerecord为false + fileRecord.fd = 1; + fileRecord.status = NebdFileStatus::OPENED; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .Times(2) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, Close(NotNull())) .WillOnce(Return(0)); - EXPECT_CALL(*metaFileManager_, RemoveFileRecord(testFile2)) - .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_.Close(2)); + EXPECT_CALL(*filerecordManager_, UpdateFileStatus(1, _)) + .WillOnce(Return(true)); + EXPECT_CALL(*filerecordManager_, RemoveRecord(1)) + .Times(0); + ASSERT_EQ(0, fileManager_->Close(1, false)); + + // 文件状态为closed + fileRecord.fd = 1; + fileRecord.status = NebdFileStatus::CLOSED; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .Times(2) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); + EXPECT_CALL(*executor_, Close(NotNull())) + .Times(0); + EXPECT_CALL(*filerecordManager_, UpdateFileStatus(_, _)) + .Times(0); + EXPECT_CALL(*filerecordManager_, RemoveRecord(1)) + .WillOnce(Return(true)); + ASSERT_EQ(0, fileManager_->Close(1, true)); } TEST_F(FileManagerTest, CloseFailTest) { - InitEnv(); - // 指定的fd不存在,直接返回成功 - ASSERT_EQ(0, fileManager_.Close(3)); - - // 调用后端存储close时失败 + NebdFileRecord fileRecord; + fileRecord.fd = 1; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.executor = executor_.get(); + fileRecord.fileInstance = mockInstance_; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); + + // executor close 失败 EXPECT_CALL(*executor_, Close(NotNull())) .WillOnce(Return(-1)); - EXPECT_CALL(*metaFileManager_, RemoveFileRecord(testFile1)) - .Times(0); - ASSERT_EQ(-1, fileManager_.Close(1)); + ASSERT_EQ(-1, fileManager_->Close(1, true)); - // 从元数据文件中移除记录时失败 + // remove record 失败 EXPECT_CALL(*executor_, Close(NotNull())) .WillOnce(Return(0)); - EXPECT_CALL(*metaFileManager_, RemoveFileRecord(testFile1)) - .WillOnce(Return(-1)); - ASSERT_EQ(-1, fileManager_.Close(1)); + EXPECT_CALL(*filerecordManager_, UpdateFileStatus(1, _)) + .WillOnce(Return(true)); + EXPECT_CALL(*filerecordManager_, RemoveRecord(1)) + .WillOnce(Return(false)); + ASSERT_EQ(-1, fileManager_->Close(1, true)); } TEST_F(FileManagerTest, ExtendTest) { - InitEnv(); - // 文件是opened状态 + NebdFileRecord fileRecord; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.executor = executor_.get(); + fileRecord.fileInstance = mockInstance_; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillOnce(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, Extend(NotNull(), 4096)) .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_.Extend(1, 4096)); + ASSERT_EQ(0, fileManager_->Extend(1, 4096)); // 文件状态为closed，会重新open文件 - std::unordered_map recordMap - = fileManager_.GetRecordMap(); - recordMap[1]->status = NebdFileStatus::CLOSED; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::CLOSED; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); // 重新open该文件，fd不变 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(0)); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(true)); EXPECT_CALL(*executor_, Extend(NotNull(), 4096)) .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_.Extend(1, 4096)); - // 校验结果，fd不变 - recordMap = fileManager_.GetRecordMap(); - ASSERT_NE(recordMap[1], nullptr); - ASSERT_EQ(testFile1, recordMap[1]->fileName); + ASSERT_EQ(0, fileManager_->Extend(1, 4096)); } TEST_F(FileManagerTest, ExtendFaileTest) { - InitEnv(); - // 文件是opened状态 + NebdFileRecord fileRecord; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.executor = executor_.get(); + fileRecord.fileInstance = mockInstance_; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillOnce(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, Extend(NotNull(), 4096)) .WillOnce(Return(-1)); - ASSERT_EQ(-1, fileManager_.Extend(1, 4096)); + ASSERT_EQ(-1, fileManager_->Extend(1, 4096)); // 文件状态为closed，会重新open文件 - std::unordered_map recordMap - = fileManager_.GetRecordMap(); - recordMap[1]->status = NebdFileStatus::CLOSED; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::CLOSED; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); // 重新open文件时，open失败 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(nullptr)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) .Times(0); EXPECT_CALL(*executor_, Extend(NotNull(), 4096)) .Times(0); - ASSERT_EQ(-1, fileManager_.Extend(1, 4096)); + ASSERT_EQ(-1, fileManager_->Extend(1, 4096)); // 重新open文件时，更新meta file失败 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(-1)); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(false)); EXPECT_CALL(*executor_, Extend(NotNull(), 4096)) .Times(0); - ASSERT_EQ(-1, fileManager_.Extend(1, 4096)); + ASSERT_EQ(-1, fileManager_->Extend(1, 4096)); } TEST_F(FileManagerTest, GetInfoTest) { - InitEnv(); - NebdFileInfo fileInfo; // 文件是opened状态 + NebdFileRecord fileRecord; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.executor = executor_.get(); + fileRecord.fileInstance = mockInstance_; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillOnce(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, GetInfo(NotNull(), &fileInfo)) .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_.GetInfo(1, &fileInfo)); + ASSERT_EQ(0, fileManager_->GetInfo(1, &fileInfo)); // 文件状态为closed，会重新open文件 - std::unordered_map recordMap - = fileManager_.GetRecordMap(); - recordMap[1]->status = NebdFileStatus::CLOSED; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::CLOSED; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); // 重新open该文件，fd不变 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(0)); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(true)); EXPECT_CALL(*executor_, GetInfo(NotNull(), &fileInfo)) .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_.GetInfo(1, &fileInfo)); - // 校验结果，fd不变 - recordMap = fileManager_.GetRecordMap(); - ASSERT_NE(recordMap[1], nullptr); - ASSERT_EQ(testFile1, recordMap[1]->fileName); + ASSERT_EQ(0, fileManager_->GetInfo(1, &fileInfo)); } TEST_F(FileManagerTest, GetInfoFaileTest) { - InitEnv(); - NebdFileInfo fileInfo; // 文件是opened状态 + NebdFileRecord fileRecord; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.executor = executor_.get(); + fileRecord.fileInstance = mockInstance_; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillOnce(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, GetInfo(NotNull(), &fileInfo)) .WillOnce(Return(-1)); - ASSERT_EQ(-1, fileManager_.GetInfo(1, &fileInfo)); + ASSERT_EQ(-1, fileManager_->GetInfo(1, &fileInfo)); // 文件状态为closed，会重新open文件 - std::unordered_map recordMap - = fileManager_.GetRecordMap(); - recordMap[1]->status = NebdFileStatus::CLOSED; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::CLOSED; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); // 重新open文件时，open失败 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(nullptr)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) .Times(0); EXPECT_CALL(*executor_, GetInfo(NotNull(), _)) .Times(0); - ASSERT_EQ(-1, fileManager_.GetInfo(1, &fileInfo)); + ASSERT_EQ(-1, fileManager_->GetInfo(1, &fileInfo)); // 重新open文件时，更新meta file失败 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(-1)); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(false)); EXPECT_CALL(*executor_, GetInfo(NotNull(), _)) .Times(0); - ASSERT_EQ(-1, fileManager_.GetInfo(1, &fileInfo)); + ASSERT_EQ(-1, fileManager_->GetInfo(1, &fileInfo)); } TEST_F(FileManagerTest, InvalidCacheTest) { - InitEnv(); - // 文件是opened状态 + NebdFileRecord fileRecord; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.executor = executor_.get(); + fileRecord.fileInstance = mockInstance_; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillOnce(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, InvalidCache(NotNull())) .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_.InvalidCache(1)); + ASSERT_EQ(0, fileManager_->InvalidCache(1)); // 文件状态为closed，会重新open文件 - std::unordered_map recordMap - = fileManager_.GetRecordMap(); - recordMap[1]->status = NebdFileStatus::CLOSED; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::CLOSED; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); // 重新open该文件，fd不变 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(0)); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(true)); EXPECT_CALL(*executor_, InvalidCache(NotNull())) .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_.InvalidCache(1)); - // 校验结果，fd不变 - recordMap = fileManager_.GetRecordMap(); - ASSERT_NE(recordMap[1], nullptr); - ASSERT_EQ(testFile1, recordMap[1]->fileName); + ASSERT_EQ(0, fileManager_->InvalidCache(1)); } TEST_F(FileManagerTest, InvalidCacheFaileTest) { - InitEnv(); - // 文件是opened状态 + NebdFileRecord fileRecord; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.executor = executor_.get(); + fileRecord.fileInstance = mockInstance_; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillOnce(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, InvalidCache(NotNull())) .WillOnce(Return(-1)); - ASSERT_EQ(-1, fileManager_.InvalidCache(1)); + ASSERT_EQ(-1, fileManager_->InvalidCache(1)); // 文件状态为closed，会重新open文件 - std::unordered_map recordMap - = fileManager_.GetRecordMap(); - recordMap[1]->status = NebdFileStatus::CLOSED; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::CLOSED; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); // 重新open文件时，open失败 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(nullptr)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) .Times(0); EXPECT_CALL(*executor_, InvalidCache(NotNull())) .Times(0); - ASSERT_EQ(-1, fileManager_.InvalidCache(1)); + ASSERT_EQ(-1, fileManager_->InvalidCache(1)); // 重新open文件时，更新meta file失败 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(-1)); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(false)); EXPECT_CALL(*executor_, InvalidCache(NotNull())) .Times(0); - ASSERT_EQ(-1, fileManager_.InvalidCache(1)); + ASSERT_EQ(-1, fileManager_->InvalidCache(1)); } TEST_F(FileManagerTest, AioReadTest) { - InitEnv(); - // 文件是opened状态 + NebdFileRecord fileRecord; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.executor = executor_.get(); + fileRecord.fileInstance = mockInstance_; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillOnce(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_.AioRead(1, aioContext_)); + ASSERT_EQ(0, fileManager_->AioRead(1, aioContext_)); // 文件状态为closed，会重新open文件 - std::unordered_map recordMap - = fileManager_.GetRecordMap(); - recordMap[1]->status = NebdFileStatus::CLOSED; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::CLOSED; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); // 重新open该文件，fd不变 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(0)); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(true)); EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_.AioRead(1, aioContext_)); - // 校验结果，fd不变 - recordMap = fileManager_.GetRecordMap(); - ASSERT_NE(recordMap[1], nullptr); - ASSERT_EQ(testFile1, recordMap[1]->fileName); + ASSERT_EQ(0, fileManager_->AioRead(1, aioContext_)); } TEST_F(FileManagerTest, AioReadFaileTest) { - InitEnv(); - // 文件不存在 + EXPECT_CALL(*filerecordManager_, GetRecord(10, _)) + .WillOnce(Return(false)); EXPECT_CALL(*executor_, AioRead(_, _)) .Times(0); - ASSERT_EQ(-1, fileManager_.AioRead(10, aioContext_)); + ASSERT_EQ(-1, fileManager_->AioRead(10, aioContext_)); // 文件是opened状态 + NebdFileRecord fileRecord; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.executor = executor_.get(); + fileRecord.fileInstance = mockInstance_; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillOnce(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) .WillOnce(Return(-1)); - ASSERT_EQ(-1, fileManager_.AioRead(1, aioContext_)); + ASSERT_EQ(-1, fileManager_->AioRead(1, aioContext_)); // 文件状态为closed，会重新open文件 - std::unordered_map recordMap - = fileManager_.GetRecordMap(); - recordMap[1]->status = NebdFileStatus::CLOSED; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::CLOSED; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); // 重新open文件时，open失败 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(nullptr)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) .Times(0); EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) .Times(0); - ASSERT_EQ(-1, fileManager_.AioRead(1, aioContext_)); + ASSERT_EQ(-1, fileManager_->AioRead(1, aioContext_)); // 重新open文件时，更新meta file失败 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(-1)); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(false)); EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) .Times(0); - ASSERT_EQ(-1, fileManager_.AioRead(1, aioContext_)); + ASSERT_EQ(-1, fileManager_->AioRead(1, aioContext_)); } TEST_F(FileManagerTest, AioWriteTest) { - InitEnv(); - // 文件是opened状态 + NebdFileRecord fileRecord; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.executor = executor_.get(); + fileRecord.fileInstance = mockInstance_; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillOnce(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_.AioWrite(1, aioContext_)); + ASSERT_EQ(0, fileManager_->AioWrite(1, aioContext_)); // 文件状态为closed，会重新open文件 - std::unordered_map recordMap - = fileManager_.GetRecordMap(); - recordMap[1]->status = NebdFileStatus::CLOSED; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::CLOSED; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); // 重新open该文件，fd不变 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(0)); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(true)); EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_.AioWrite(1, aioContext_)); - // 校验结果，fd不变 - recordMap = fileManager_.GetRecordMap(); - ASSERT_NE(recordMap[1], nullptr); - ASSERT_EQ(testFile1, recordMap[1]->fileName); + ASSERT_EQ(0, fileManager_->AioWrite(1, aioContext_)); } TEST_F(FileManagerTest, AioWriteFaileTest) { - InitEnv(); - // 文件不存在 + EXPECT_CALL(*filerecordManager_, GetRecord(10, _)) + .WillOnce(Return(false)); EXPECT_CALL(*executor_, AioWrite(_, _)) .Times(0); - ASSERT_EQ(-1, fileManager_.AioWrite(10, aioContext_)); + ASSERT_EQ(-1, fileManager_->AioWrite(10, aioContext_)); // 文件是opened状态 + NebdFileRecord fileRecord; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.executor = executor_.get(); + fileRecord.fileInstance = mockInstance_; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillOnce(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) .WillOnce(Return(-1)); - ASSERT_EQ(-1, fileManager_.AioWrite(1, aioContext_)); + ASSERT_EQ(-1, fileManager_->AioWrite(1, aioContext_)); // 文件状态为closed，会重新open文件 - std::unordered_map recordMap - = fileManager_.GetRecordMap(); - recordMap[1]->status = NebdFileStatus::CLOSED; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::CLOSED; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); // 重新open文件时，open失败 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(nullptr)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) .Times(0); EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) .Times(0); - ASSERT_EQ(-1, fileManager_.AioWrite(1, aioContext_)); + ASSERT_EQ(-1, fileManager_->AioWrite(1, aioContext_)); // 重新open文件时，更新meta file失败 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(-1)); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(false)); EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) .Times(0); - ASSERT_EQ(-1, fileManager_.AioWrite(1, aioContext_)); + ASSERT_EQ(-1, fileManager_->AioWrite(1, aioContext_)); } TEST_F(FileManagerTest, DiscardTest) { - InitEnv(); - // 文件是opened状态 + NebdFileRecord fileRecord; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.executor = executor_.get(); + fileRecord.fileInstance = mockInstance_; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillOnce(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_.Discard(1, aioContext_)); + ASSERT_EQ(0, fileManager_->Discard(1, aioContext_)); // 文件状态为closed，会重新open文件 - std::unordered_map recordMap - = fileManager_.GetRecordMap(); - recordMap[1]->status = NebdFileStatus::CLOSED; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::CLOSED; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); // 重新open该文件，fd不变 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(0)); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(true)); EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_.Discard(1, aioContext_)); - // 校验结果，fd不变 - recordMap = fileManager_.GetRecordMap(); - ASSERT_NE(recordMap[1], nullptr); - ASSERT_EQ(testFile1, recordMap[1]->fileName); + ASSERT_EQ(0, fileManager_->Discard(1, aioContext_)); } TEST_F(FileManagerTest, DiscardFaileTest) { - InitEnv(); - // 文件不存在 + EXPECT_CALL(*filerecordManager_, GetRecord(10, _)) + .WillOnce(Return(false)); EXPECT_CALL(*executor_, Discard(_, _)) .Times(0); - ASSERT_EQ(-1, fileManager_.Discard(10, aioContext_)); + ASSERT_EQ(-1, fileManager_->Discard(10, aioContext_)); // 文件是opened状态 + NebdFileRecord fileRecord; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.executor = executor_.get(); + fileRecord.fileInstance = mockInstance_; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillOnce(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) .WillOnce(Return(-1)); - ASSERT_EQ(-1, fileManager_.Discard(1, aioContext_)); + ASSERT_EQ(-1, fileManager_->Discard(1, aioContext_)); // 文件状态为closed，会重新open文件 - std::unordered_map recordMap - = fileManager_.GetRecordMap(); - recordMap[1]->status = NebdFileStatus::CLOSED; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::CLOSED; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); // 重新open文件时，open失败 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(nullptr)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) .Times(0); EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) .Times(0); - ASSERT_EQ(-1, fileManager_.Discard(1, aioContext_)); + ASSERT_EQ(-1, fileManager_->Discard(1, aioContext_)); // 重新open文件时，更新meta file失败 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(-1)); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(false)); EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) .Times(0); - ASSERT_EQ(-1, fileManager_.Discard(1, aioContext_)); + ASSERT_EQ(-1, fileManager_->Discard(1, aioContext_)); } TEST_F(FileManagerTest, FlushTest) { - InitEnv(); - // 文件是opened状态 + NebdFileRecord fileRecord; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.executor = executor_.get(); + fileRecord.fileInstance = mockInstance_; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillOnce(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_.Flush(1, aioContext_)); + ASSERT_EQ(0, fileManager_->Flush(1, aioContext_)); // 文件状态为closed，会重新open文件 - std::unordered_map recordMap - = fileManager_.GetRecordMap(); - recordMap[1]->status = NebdFileStatus::CLOSED; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::CLOSED; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); // 重新open该文件，fd不变 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(0)); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(true)); EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_.Flush(1, aioContext_)); - // 校验结果，fd不变 - recordMap = fileManager_.GetRecordMap(); - ASSERT_NE(recordMap[1], nullptr); - ASSERT_EQ(testFile1, recordMap[1]->fileName); + ASSERT_EQ(0, fileManager_->Flush(1, aioContext_)); } TEST_F(FileManagerTest, FlushFaileTest) { - InitEnv(); - // 文件不存在 + EXPECT_CALL(*filerecordManager_, GetRecord(10, _)) + .WillOnce(Return(false)); EXPECT_CALL(*executor_, Flush(_, _)) .Times(0); - ASSERT_EQ(-1, fileManager_.Flush(10, aioContext_)); + ASSERT_EQ(-1, fileManager_->Flush(10, aioContext_)); // 文件是opened状态 + NebdFileRecord fileRecord; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::OPENED; + fileRecord.executor = executor_.get(); + fileRecord.fileInstance = mockInstance_; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillOnce(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) .WillOnce(Return(-1)); - ASSERT_EQ(-1, fileManager_.Flush(1, aioContext_)); + ASSERT_EQ(-1, fileManager_->Flush(1, aioContext_)); // 文件状态为closed，会重新open文件 - std::unordered_map recordMap - = fileManager_.GetRecordMap(); - recordMap[1]->status = NebdFileStatus::CLOSED; + fileRecord.fd = 1; + fileRecord.fileName = testFile1; + fileRecord.status = NebdFileStatus::CLOSED; + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); + EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); // 重新open文件时，open失败 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(nullptr)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) .Times(0); EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) .Times(0); - ASSERT_EQ(-1, fileManager_.Flush(1, aioContext_)); + ASSERT_EQ(-1, fileManager_->Flush(1, aioContext_)); // 重新open文件时，更新meta file失败 EXPECT_CALL(*executor_, Open(testFile1)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*metaFileManager_, UpdateFileRecord(_)) - .WillOnce(Return(-1)); + EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + .WillOnce(Return(false)); EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) .Times(0); - ASSERT_EQ(-1, fileManager_.Flush(1, aioContext_)); -} - -TEST_F(FileManagerTest, CheckTimeoutTest) { - ASSERT_EQ(fileManager_.Run(), 0); - // 已经在run了不允许重复Run或者Init - ASSERT_EQ(fileManager_.Run(), -1); - ASSERT_EQ(fileManager_.Init(option_), -1); - - // 校验是否在检查超时 - InitEnv(); - std::unordered_map recordMap - = fileManager_.GetRecordMap(); - ASSERT_EQ(recordMap[1]->status, NebdFileStatus::OPENED); - ASSERT_EQ(recordMap[2]->status, NebdFileStatus::OPENED); - // 等待一段时间（未超过超时时间），模拟一个文件收到心跳，另一个没收到 - ::sleep(option_.heartbeatTimeoutS - 2); - ASSERT_EQ(0, fileManager_.UpdateFileTimestamp(1)); - // 更新不存在的fd，返回失败 - ASSERT_EQ(-1, fileManager_.UpdateFileTimestamp(10)); - // 在等待一段时间，其中一个文件超时，另一个文件未超时 - EXPECT_CALL(*executor_, Close(NotNull())) - .WillOnce(Return(0)); - ::sleep(option_.checkTimeoutIntervalMs / 1000 + 2); - recordMap = fileManager_.GetRecordMap(); - ASSERT_EQ(recordMap[1]->status, NebdFileStatus::OPENED); - ASSERT_EQ(recordMap[2]->status, NebdFileStatus::CLOSED); - // 继续等待一段时间，两个文件都超时 - EXPECT_CALL(*executor_, Close(NotNull())) - .WillOnce(Return(0)); - ::sleep(option_.heartbeatTimeoutS - 2); - ASSERT_EQ(recordMap[1]->status, NebdFileStatus::CLOSED); - ASSERT_EQ(recordMap[2]->status, NebdFileStatus::CLOSED); - - ASSERT_EQ(fileManager_.Fini(), 0); - // 重复Fini，也返回成功 - ASSERT_EQ(fileManager_.Fini(), 0); + ASSERT_EQ(-1, fileManager_->Flush(1, aioContext_)); } } // namespace server diff --git a/tests/part2/file_record_manager_unittest.cpp b/tests/part2/file_record_manager_unittest.cpp new file mode 100644 index 0000000000..83a1803e94 --- /dev/null +++ b/tests/part2/file_record_manager_unittest.cpp @@ -0,0 +1,193 @@ +/* + * Project: nebd + * Created Date: Monday January 20th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#include +#include +#include +#include + +#include "src/part2/filerecord_manager.h" +#include "tests/part2/mock_metafile_manager.h" + +namespace nebd { +namespace server { + +using ::testing::_; +using ::testing::Return; +using ::testing::NotNull; +using ::testing::DoAll; +using ::testing::ReturnArg; +using ::testing::ElementsAre; +using ::testing::SetArgPointee; +using ::testing::SetArrayArgument; + +class FileRecordManagerTest : public ::testing::Test { + public: + void SetUp() { + metaFileManager_ = std::make_shared(); + recordManager_ = std::make_shared(metaFileManager_); + } + void TearDown() {} + + protected: + std::shared_ptr recordManager_; + std::shared_ptr metaFileManager_; +}; + +TEST_F(FileRecordManagerTest, BasicTest) { + int fd1 = 1; + std::string fileName1 = "file1"; + NebdFileRecord fileRecord1; + + // 初始测试 + ASSERT_FALSE(recordManager_->Exist(fd1)); + ASSERT_FALSE(recordManager_->GetRecord(fd1, &fileRecord1)); + ASSERT_FALSE(recordManager_->GetRecord(fileName1, &fileRecord1)); + + uint64_t time1 = 1000; + uint64_t time2 = 2000; + // UpdateRecord + fileRecord1.fileName = fileName1; + fileRecord1.fd = fd1; + fileRecord1.status = NebdFileStatus::OPENED; + fileRecord1.timeStamp = time1; + EXPECT_CALL(*metaFileManager_, UpdateMetaFile(_)) + .WillOnce(Return(0)); + ASSERT_TRUE(recordManager_->UpdateRecord(fileRecord1)); + ASSERT_TRUE(recordManager_->Exist(fd1)); + // 通过fd获取record + NebdFileRecord tempRecord; + ASSERT_TRUE(recordManager_->GetRecord(fd1, &tempRecord)); + ASSERT_EQ(tempRecord.fd, fd1); + ASSERT_EQ(tempRecord.fileName, fileName1); + // 通过filename获取record + ASSERT_TRUE(recordManager_->GetRecord(fileName1, &tempRecord)); + ASSERT_EQ(tempRecord.fd, fd1); + ASSERT_EQ(tempRecord.fileName, fileName1); + + // 插入一条filename相同，fd不同的记录 + fileRecord1.fd = ++fd1; + EXPECT_CALL(*metaFileManager_, UpdateMetaFile(_)) + .WillOnce(Return(0)); + ASSERT_TRUE(recordManager_->UpdateRecord(fileRecord1)); + ASSERT_FALSE(recordManager_->Exist(fd1-1)); + ASSERT_TRUE(recordManager_->Exist(fd1)); + + // get/update timestamp 测试 + uint64_t record1TimeStamp; + ASSERT_TRUE(recordManager_->GetFileTimestamp(fd1, &record1TimeStamp)); + ASSERT_EQ(record1TimeStamp, time1); + ASSERT_TRUE(recordManager_->UpdateFileTimestamp(fd1, time2)); + ASSERT_TRUE(recordManager_->GetFileTimestamp(fd1, &record1TimeStamp)); + ASSERT_EQ(record1TimeStamp, time2); + // 指定fd记录不存在 + ASSERT_FALSE(recordManager_->GetFileTimestamp(10, &record1TimeStamp)); + ASSERT_FALSE(recordManager_->UpdateFileTimestamp(10, time2)); + + // get/update status 测试 + NebdFileStatus record1Status; + ASSERT_TRUE(recordManager_->GetFileStatus(fd1, &record1Status)); + ASSERT_EQ(record1Status, NebdFileStatus::OPENED); + ASSERT_TRUE(recordManager_->UpdateFileStatus(fd1, NebdFileStatus::CLOSED)); + ASSERT_TRUE(recordManager_->GetFileStatus(fd1, &record1Status)); + ASSERT_EQ(record1Status, NebdFileStatus::CLOSED); + // 指定fd记录不存在 + ASSERT_FALSE(recordManager_->GetFileStatus(10, &record1Status)); + ASSERT_FALSE(recordManager_->UpdateFileStatus(10, NebdFileStatus::CLOSED)); + + + // 插入不同记录 + NebdFileRecord fileRecord2; + std::string fileName2 = "file2"; + int fd2 = 3; + fileRecord2.fileName = fileName2; + fileRecord2.fd = fd2; + EXPECT_CALL(*metaFileManager_, UpdateMetaFile(_)) + .WillOnce(Return(0)); + ASSERT_TRUE(recordManager_->UpdateRecord(fileRecord2)); + ASSERT_TRUE(recordManager_->Exist(fd1)); + ASSERT_TRUE(recordManager_->Exist(fd2)); + + // 插入fd相同，名字不同的记录,返回失败 + NebdFileRecord fileRecord3; + fileRecord3.fileName = "file3"; + fileRecord3.fd = fd2; + EXPECT_CALL(*metaFileManager_, UpdateMetaFile(_)) + .Times(0); + ASSERT_FALSE(recordManager_->UpdateRecord(fileRecord3)); + + // update metafile 失败，返回失败 + fileRecord3.fileName = "file3"; + fileRecord3.fd = 4; + EXPECT_CALL(*metaFileManager_, UpdateMetaFile(_)) + .WillOnce(Return(-1)); + ASSERT_FALSE(recordManager_->UpdateRecord(fileRecord3)); + + // remove record + // 请求删除的fd不存在，直接返回成功 + ASSERT_TRUE(recordManager_->RemoveRecord(10)); + // 请求删除的fd存在，更新metafile，并删除记录 + EXPECT_CALL(*metaFileManager_, UpdateMetaFile(_)) + .WillOnce(Return(0)); + ASSERT_TRUE(recordManager_->RemoveRecord(fd2)); + // 如果更新metafile失败，则不删除记录 + EXPECT_CALL(*metaFileManager_, UpdateMetaFile(_)) + .WillOnce(Return(-1)); + ASSERT_FALSE(recordManager_->RemoveRecord(fd1)); + + // 校验最终结果 + FileRecordMap list = recordManager_->ListRecords(); + ASSERT_EQ(list.size(), 1); + // 通过fd获取record并检查结果 + auto recordIter = list.begin(); + ASSERT_EQ(recordIter->first, fd1); + ASSERT_EQ(recordIter->second.fd, fd1); + ASSERT_EQ(recordIter->second.fileName, fileName1); + + // clear + recordManager_->Clear(); + list = recordManager_->ListRecords(); + ASSERT_EQ(list.size(), 0); +} + +TEST_F(FileRecordManagerTest, LoadTest) { + FileRecordMap list = recordManager_->ListRecords(); + ASSERT_EQ(list.size(), 0); + + // list 失败 + EXPECT_CALL(*metaFileManager_, ListFileRecord(_)) + .WillOnce(Return(-1)); + ASSERT_EQ(-1, recordManager_->Load()); + + // list 成功 + FileRecordMap recordsFromFile; + NebdFileRecord fileRecord1; + fileRecord1.fileName = "111"; + fileRecord1.fd = 1; + recordsFromFile.emplace(1, fileRecord1); + EXPECT_CALL(*metaFileManager_, ListFileRecord(_)) + .WillOnce(DoAll(SetArgPointee<0>(recordsFromFile), + Return(0))); + ASSERT_EQ(0, recordManager_->Load()); + + // 校验结果 + list = recordManager_->ListRecords(); + ASSERT_EQ(list.size(), 1); + auto recordIter = list.begin(); + ASSERT_EQ(recordIter->first, 1); + ASSERT_EQ(recordIter->second.fd, 1); + ASSERT_EQ(recordIter->second.fileName, "111"); +} + +} // namespace server +} // namespace nebd + +int main(int argc, char ** argv) { + ::testing::InitGoogleTest(&argc, argv); + ::testing::InitGoogleMock(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/part2/file_record_map_unittest.cpp b/tests/part2/file_record_map_unittest.cpp deleted file mode 100644 index 229d568892..0000000000 --- a/tests/part2/file_record_map_unittest.cpp +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Project: nebd - * Created Date: Monday January 20th 2020 - * Author: yangyaokai - * Copyright (c) 2020 netease - */ - -#include -#include -#include -#include - -#include "src/part2/file_record_map.h" - -namespace nebd { -namespace server { - -class FileRecordMapTest : public ::testing::Test { - public: - void SetUp() {} - void TearDown() {} - - protected: - FileRecordMap recordMap_; -}; - -TEST_F(FileRecordMapTest, BasicTest) { - int fd1 = 1; - std::string fileName1 = "file1"; - - // 初始测试 - ASSERT_FALSE(recordMap_.Exist(fd1)); - ASSERT_EQ(recordMap_.GetRecord(fd1), nullptr); - ASSERT_EQ(recordMap_.GetRecord(fileName1), nullptr); - - // UpdateRecord - NebdFileRecordPtr fileRecord1 = std::make_shared(); - fileRecord1->fileName = fileName1; - fileRecord1->fd = fd1; - ASSERT_TRUE(recordMap_.UpdateRecord(fileRecord1)); - ASSERT_TRUE(recordMap_.Exist(fd1)); - // 通过fd获取record - NebdFileRecordPtr tempRecord = recordMap_.GetRecord(fd1); - ASSERT_NE(tempRecord, nullptr); - ASSERT_EQ(tempRecord->fd, fd1); - ASSERT_EQ(tempRecord->fileName, fileName1); - // 通过filename获取record - tempRecord = recordMap_.GetRecord(fileName1); - ASSERT_NE(tempRecord, nullptr); - ASSERT_EQ(tempRecord->fd, fd1); - ASSERT_EQ(tempRecord->fileName, fileName1); - - // 插入一条filename相同，fd不同的记录 - fileRecord1->fd = ++fd1; - ASSERT_TRUE(recordMap_.UpdateRecord(fileRecord1)); - ASSERT_FALSE(recordMap_.Exist(fd1-1)); - ASSERT_TRUE(recordMap_.Exist(fd1)); - - // 插入不同记录 - NebdFileRecordPtr fileRecord2 = std::make_shared(); - std::string fileName2 = "file2"; - int fd2 = 3; - fileRecord2->fileName = fileName2; - fileRecord2->fd = fd2; - ASSERT_TRUE(recordMap_.UpdateRecord(fileRecord2)); - ASSERT_TRUE(recordMap_.Exist(fd1)); - ASSERT_TRUE(recordMap_.Exist(fd2)); - - // 插入fd相同，名字不同的记录 - NebdFileRecordPtr fileRecord3 = std::make_shared(); - fileRecord3->fileName = fileName1; - fileRecord3->fd = fd2; - ASSERT_FALSE(recordMap_.UpdateRecord(fileRecord3)); - - // 校验最终结果 - std::unordered_map list = recordMap_.ListRecords(); - ASSERT_EQ(list.size(), 2); - // 通过fd获取record并检查结果 - NebdFileRecordPtr tempRecord1 = recordMap_.GetRecord(fd1); - ASSERT_NE(tempRecord1, nullptr); - ASSERT_EQ(tempRecord1->fd, fd1); - ASSERT_EQ(tempRecord1->fileName, fileName1); - NebdFileRecordPtr tempRecord2 = recordMap_.GetRecord(fd2); - ASSERT_NE(tempRecord2, nullptr); - ASSERT_EQ(tempRecord2->fd, fd2); - ASSERT_EQ(tempRecord2->fileName, fileName2); - - // clear - recordMap_.Clear(); - list = recordMap_.ListRecords(); - ASSERT_EQ(list.size(), 0); -} - -} // namespace server -} // namespace nebd - -int main(int argc, char ** argv) { - ::testing::InitGoogleTest(&argc, argv); - ::testing::InitGoogleMock(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tests/part2/file_service_unittest.cpp b/tests/part2/file_service_unittest.cpp index f47029a342..771f15da75 100644 --- a/tests/part2/file_service_unittest.cpp +++ b/tests/part2/file_service_unittest.cpp @@ -92,6 +92,8 @@ TEST_F(FileServiceTest, WriteTest) { uint64_t offset = 0; uint64_t size = 4096; brpc::Controller cntl; + char buf[4096]; + cntl.request_attachment().append(buf, 4096); nebd::client::WriteRequest request; request.set_fd(fd); request.set_offset(offset); @@ -191,34 +193,6 @@ TEST_F(FileServiceTest, DiscardTest) { ASSERT_TRUE(done.IsRunned()); } -TEST_F(FileServiceTest, StatFileTest) { - int fd = 1; - brpc::Controller cntl; - nebd::client::StatFileRequest request; - request.set_fd(fd); - nebd::client::StatFileResponse response; - FileServiceTestClosure done; - - // stat file success - NebdFileInfo fileInfo; - fileInfo.size = 4096; - EXPECT_CALL(*fileManager_, GetInfo(fd, NotNull())) - .WillOnce(DoAll(SetArgPointee<1>(fileInfo), - Return(0))); - fileService_->StatFile(&cntl, &request, &response, &done); - ASSERT_EQ(response.retcode(), RetCode::kOK); - ASSERT_EQ(response.size(), fileInfo.size); - ASSERT_TRUE(done.IsRunned()); - - // stat file failed - done.Reset(); - EXPECT_CALL(*fileManager_, GetInfo(fd, NotNull())) - .WillOnce(Return(-1)); - fileService_->StatFile(&cntl, &request, &response, &done); - ASSERT_EQ(response.retcode(), RetCode::kNoOK); - ASSERT_TRUE(done.IsRunned()); -} - TEST_F(FileServiceTest, GetInfoTest) { int fd = 1; brpc::Controller cntl; @@ -229,13 +203,17 @@ TEST_F(FileServiceTest, GetInfoTest) { // stat file success NebdFileInfo fileInfo; + fileInfo.size = 4096; fileInfo.obj_size = 4096; + fileInfo.num_objs = 1; EXPECT_CALL(*fileManager_, GetInfo(fd, NotNull())) .WillOnce(DoAll(SetArgPointee<1>(fileInfo), Return(0))); fileService_->GetInfo(&cntl, &request, &response, &done); ASSERT_EQ(response.retcode(), RetCode::kOK); - ASSERT_EQ(response.objsize(), fileInfo.obj_size); + ASSERT_EQ(response.info().size(), fileInfo.size); + ASSERT_EQ(response.info().objsize(), fileInfo.obj_size); + ASSERT_EQ(response.info().objnums(), fileInfo.num_objs); ASSERT_TRUE(done.IsRunned()); // stat file failed @@ -256,7 +234,7 @@ TEST_F(FileServiceTest, CloseTest) { FileServiceTestClosure done; // close success - EXPECT_CALL(*fileManager_, Close(fd)) + EXPECT_CALL(*fileManager_, Close(fd, true)) .WillOnce(Return(0)); fileService_->CloseFile(&cntl, &request, &response, &done); ASSERT_EQ(response.retcode(), RetCode::kOK); @@ -264,7 +242,7 @@ TEST_F(FileServiceTest, CloseTest) { // close failed done.Reset(); - EXPECT_CALL(*fileManager_, Close(fd)) + EXPECT_CALL(*fileManager_, Close(fd, true)) .WillOnce(Return(-1)); fileService_->CloseFile(&cntl, &request, &response, &done); ASSERT_EQ(response.retcode(), RetCode::kNoOK); diff --git a/tests/part2/heartbeat_manager_unittest.cpp b/tests/part2/heartbeat_manager_unittest.cpp new file mode 100644 index 0000000000..d7124ae09e --- /dev/null +++ b/tests/part2/heartbeat_manager_unittest.cpp @@ -0,0 +1,98 @@ +/* + * Project: nebd + * Created Date: Friday February 14th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#include +#include + +#include "src/part2/heartbeat_manager.h" +#include "src/part2/filerecord_manager.h" +#include "tests/part2/mock_filerecord_manager.h" +#include "tests/part2/mock_file_manager.h" + +using ::testing::_; +using ::testing::Return; + +namespace nebd { +namespace server { + +using ::testing::_; +using ::testing::AtLeast; +using ::testing::Return; +using ::testing::NotNull; +using ::testing::DoAll; +using ::testing::ReturnArg; +using ::testing::ElementsAre; +using ::testing::SetArgPointee; +using ::testing::SetArrayArgument; + +class HeartbeatManagerTest : public ::testing::Test { + protected: + void SetUp() override { + fileRecordManager_ = std::make_shared(); + fileManager_ = std::make_shared(); + HeartbeatManagerOption option; + option.heartbeatTimeoutS = 10; + option.checkTimeoutIntervalMs = 1000; + option.fileManager = fileManager_; + heartbeatManager_ = std::make_shared(option); + EXPECT_CALL(*fileManager_, GetRecordManager()) + .WillRepeatedly(Return(fileRecordManager_)); + } + std::shared_ptr fileManager_; + std::shared_ptr fileRecordManager_; + std::shared_ptr heartbeatManager_; +}; + +TEST_F(HeartbeatManagerTest, CheckTimeoutTest) { + ASSERT_EQ(heartbeatManager_->Run(), 0); + // 已经在run了不允许重复Run或者Init + ASSERT_EQ(heartbeatManager_->Run(), -1); + + // 校验是否在检查超时 + uint64_t curTime = TimeUtility::GetTimeofDayMs(); + FileRecordMap fileRecords; + NebdFileRecord record1; + NebdFileRecord record2; + NebdFileRecord record3; + record1.fd = 1; + record1.timeStamp = curTime - 2 * 10 * 1000; + record1.status = NebdFileStatus::OPENED; + record2.fd = 2; + record2.timeStamp = curTime - 2 * 10 * 1000; + record2.status = NebdFileStatus::CLOSED; + record3.fd = 3; + record3.timeStamp = curTime; + record3.status = NebdFileStatus::OPENED; + fileRecords.emplace(1, record1); + fileRecords.emplace(2, record2); + fileRecords.emplace(3, record3); + + EXPECT_CALL(*fileRecordManager_, ListRecords()) + .WillRepeatedly(Return(fileRecords)); + + EXPECT_CALL(*fileManager_, Close(1, false)) + .Times(AtLeast(1)); + EXPECT_CALL(*fileManager_, Close(2, false)) + .Times(0); + EXPECT_CALL(*fileManager_, Close(3, false)) + .Times(0); + + ::sleep(2); + ASSERT_EQ(heartbeatManager_->Fini(), 0); + // 重复Fini，也返回成功 + ASSERT_EQ(heartbeatManager_->Fini(), 0); +} + + +} // namespace server +} // namespace nebd + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/tests/part2/heartbeat_service_test.cpp b/tests/part2/heartbeat_service_test.cpp index 0d932cf4f1..d3898bcfda 100644 --- a/tests/part2/heartbeat_service_test.cpp +++ b/tests/part2/heartbeat_service_test.cpp @@ -12,27 +12,27 @@ #include "proto/heartbeat.pb.h" #include "src/part2/heartbeat_service.h" -#include "tests/part2/mock_file_manager.h" - +#include "tests/part2/mock_heartbeat_manager.h" using ::testing::_; using ::testing::Return; namespace nebd { namespace server { +const std::string kSockFile_ = "/tmp/heartbeat_service_test.sock"; // NOLINT + class HeartbeatServiceTest : public ::testing::Test { protected: void SetUp() override { - fileManager_ = std::make_shared(); + heartbeatManager_ = std::make_shared(); } - std::shared_ptr fileManager_; - const std::string kSockFile_ = "/tmp/heartbeat_service_test.sock"; + std::shared_ptr heartbeatManager_; }; TEST_F(HeartbeatServiceTest, KeepAlive) { // 启动server brpc::Server server; - NebdHeartbeatServiceImpl heartbeatService(fileManager_); + NebdHeartbeatServiceImpl heartbeatService(heartbeatManager_); ASSERT_EQ(0, server.AddService(&heartbeatService, brpc::SERVER_DOESNT_OWN_SERVICE)); brpc::ServerOptions option; @@ -52,7 +52,7 @@ TEST_F(HeartbeatServiceTest, KeepAlive) { brpc::Controller cntl; // 正常情况 - EXPECT_CALL(*fileManager_, UpdateFileTimestamp(_)) + EXPECT_CALL(*heartbeatManager_, UpdateFileTimestamp(_, _)) .Times(3) .WillRepeatedly(Return(0)); stub.KeepAlive(&cntl, &request, &response, nullptr); @@ -60,7 +60,7 @@ TEST_F(HeartbeatServiceTest, KeepAlive) { ASSERT_EQ(nebd::client::RetCode::kOK, response.retcode()); // 有文件更新时间戳失败 - EXPECT_CALL(*fileManager_, UpdateFileTimestamp(_)) + EXPECT_CALL(*heartbeatManager_, UpdateFileTimestamp(_, _)) .Times(3) .WillOnce(Return(-1)) .WillRepeatedly(Return(0)); diff --git a/tests/part2/metafile_manager_test.cpp b/tests/part2/metafile_manager_test.cpp index 6559fb8daa..bb072bf704 100644 --- a/tests/part2/metafile_manager_test.cpp +++ b/tests/part2/metafile_manager_test.cpp @@ -34,96 +34,71 @@ class MetaFileManagerTest : public ::testing::Test { std::shared_ptr wrapper_; }; -TEST_F(MetaFileManagerTest, common) { +TEST_F(MetaFileManagerTest, nomaltest) { NebdMetaFileManager metaFileManager(metaPath); - std::vector records; + FileRecordMap records; // 文件不存在 ASSERT_EQ(0, metaFileManager.ListFileRecord(&records)); ASSERT_TRUE(records.empty()); - // 添加两条记录，ceph和curve各一个 - NebdFileRecordPtr fileRecord1 = std::make_shared(); - fileRecord1->fileName = "rbd:volume1"; - fileRecord1->fd = 111; - ASSERT_EQ(0, metaFileManager.UpdateFileRecord(fileRecord1)); - NebdFileRecordPtr fileRecord2 = std::make_shared(); - fileRecord2->fileName = "cbd:volume2"; - fileRecord2->fd = 222; - fileRecord2->fileInstance = std::make_shared(); - fileRecord2->fileInstance->addition["session"] = "test-session"; - ASSERT_EQ(0, metaFileManager.UpdateFileRecord(fileRecord2)); - fileRecord2->fd = 3333; - ASSERT_EQ(0, metaFileManager.UpdateFileRecord(fileRecord2)); - fileRecord2->fileInstance->addition["session"] = "test-session-2"; - ASSERT_EQ(0, metaFileManager.UpdateFileRecord(fileRecord2)); - // 重复更新 - ASSERT_EQ(0, metaFileManager.UpdateFileRecord(fileRecord2)); + // 添加两条记录，ceph和curve各一 + NebdFileRecord fileRecord1; + fileRecord1.fileName = "rbd:volume1"; + fileRecord1.fd = 1; + records.emplace(1, fileRecord1); + ASSERT_EQ(0, metaFileManager.UpdateMetaFile(records)); + NebdFileRecord fileRecord2; + fileRecord2.fileName = "cbd:volume2"; + fileRecord2.fd = 2; + fileRecord2.fileInstance = std::make_shared(); + fileRecord2.fileInstance->addition["session"] = "test-session"; + records.emplace(2, fileRecord2); + ASSERT_EQ(0, metaFileManager.UpdateMetaFile(records)); // listFileRecord + records.clear(); ASSERT_EQ(0, metaFileManager.ListFileRecord(&records)); ASSERT_EQ(2, records.size()); - ASSERT_EQ(fileRecord1->fileName, records[0]->fileName); - ASSERT_EQ(fileRecord1->fd, records[0]->fd); - ASSERT_EQ(fileRecord2->fileName, records[1]->fileName); - ASSERT_EQ(fileRecord2->fd, records[1]->fd); - ASSERT_EQ(fileRecord2->fileInstance->addition, - records[1]->fileInstance->addition); - - // RemoveFileRecord - ASSERT_EQ(0, metaFileManager.RemoveFileRecord("cbd:volume2")); - ASSERT_EQ(0, metaFileManager.ListFileRecord(&records)); - ASSERT_EQ(1, records.size()); - ASSERT_EQ(fileRecord1->fileName, records[0]->fileName); - ASSERT_EQ(fileRecord1->fd, records[0]->fd); - // volume not exist - ASSERT_EQ(0, metaFileManager.RemoveFileRecord("cbd:volume123")); - ASSERT_EQ(0, metaFileManager.ListFileRecord(&records)); - ASSERT_EQ(1, records.size()); + ASSERT_EQ(fileRecord1.fileName, records[1].fileName); + ASSERT_EQ(fileRecord1.fd, records[1].fd); + ASSERT_EQ(fileRecord2.fileName, records[2].fileName); + ASSERT_EQ(fileRecord2.fd, records[2].fd); + ASSERT_EQ(fileRecord2.fileInstance->addition, + records[2].fileInstance->addition); } TEST_F(MetaFileManagerTest, error) { NebdMetaFileManager metaFileManager(metaPath, wrapper_); - std::vector records; - NebdFileRecordPtr fileRecord = std::make_shared(); - fileRecord->fileName = "rbd:volume1"; - fileRecord->fd = 111; + FileRecordMap records; + NebdFileRecord fileRecord; + fileRecord.fileName = "rbd:volume1"; + fileRecord.fd = 111; + records.emplace(111, fileRecord); // open临时文件失败 EXPECT_CALL(*wrapper_, open(_, _, _)) - .Times(1) .WillOnce(Return(-1)); - ASSERT_EQ(-1, metaFileManager.UpdateFileRecord(fileRecord)); + ASSERT_EQ(-1, metaFileManager.UpdateMetaFile(records)); // 写入临时文件失败 EXPECT_CALL(*wrapper_, open(_, _, _)) - .Times(1) .WillOnce(Return(1)); EXPECT_CALL(*wrapper_, pwrite(_, _, _, _)) - .Times(1) .WillOnce(Return(0)); - ASSERT_EQ(-1, metaFileManager.UpdateFileRecord(fileRecord)); + EXPECT_CALL(*wrapper_, close(_)) + .Times(1); + ASSERT_EQ(-1, metaFileManager.UpdateMetaFile(records)); // rename失败 EXPECT_CALL(*wrapper_, open(_, _, _)) - .Times(1) .WillOnce(Return(1)); EXPECT_CALL(*wrapper_, pwrite(_, _, _, _)) - .Times(1) .WillOnce(Return(77)); + EXPECT_CALL(*wrapper_, close(_)) + .Times(1); EXPECT_CALL(*wrapper_, rename(_, _)) - .Times(1) .WillOnce(Return(-1)); - ASSERT_EQ(-1, metaFileManager.UpdateFileRecord(fileRecord)); - - // 解析失败 - std::ofstream out(metaPath); - out.close(); - ASSERT_EQ(-1, metaFileManager.ListFileRecord(&records)); - ASSERT_EQ(-1, metaFileManager.RemoveFileRecord("cbd:volume2")); - ASSERT_EQ(-1, metaFileManager.UpdateFileRecord(fileRecord)); - - // UpdataFileRecord的参数为空 - ASSERT_EQ(-1, metaFileManager.UpdateFileRecord(nullptr)); + ASSERT_EQ(-1, metaFileManager.UpdateMetaFile(records)); } TEST(MetaFileParserTest, Parse) { @@ -131,7 +106,7 @@ TEST(MetaFileParserTest, Parse) { Json::Value root; Json::Value volume; Json::Value volumes; - std::vector records; + FileRecordMap records; // 正常情况 volume[kFileName] = "rbd:volume1"; diff --git a/tests/part2/mock_file_manager.h b/tests/part2/mock_file_manager.h index 49a728d2c5..af67d17c3f 100644 --- a/tests/part2/mock_file_manager.h +++ b/tests/part2/mock_file_manager.h @@ -19,23 +19,21 @@ namespace server { class MockFileManager : public NebdFileManager { public: - MockFileManager() {} + MockFileManager() : NebdFileManager(nullptr) {} ~MockFileManager() {} - MOCK_METHOD1(Init, int(NebdFileManagerOption)); MOCK_METHOD0(Fini, int()); MOCK_METHOD0(Run, int()); - MOCK_METHOD1(UpdateFileTimestamp, int(int)); MOCK_METHOD1(Open, int(const std::string&)); - MOCK_METHOD1(Close, int(int)); + MOCK_METHOD2(Close, int(int, bool)); MOCK_METHOD2(Extend, int(int, int64_t)); MOCK_METHOD2(GetInfo, int(int, NebdFileInfo*)); - MOCK_METHOD2(StatFile, int(int, NebdFileInfo*)); MOCK_METHOD2(Discard, int(int, NebdServerAioContext*)); MOCK_METHOD2(AioRead, int(int, NebdServerAioContext*)); MOCK_METHOD2(AioWrite, int(int, NebdServerAioContext*)); MOCK_METHOD2(Flush, int(int, NebdServerAioContext*)); MOCK_METHOD1(InvalidCache, int(int)); + MOCK_METHOD0(GetRecordManager, FileRecordManagerPtr()); }; } // namespace server diff --git a/tests/part2/mock_filerecord_manager.h b/tests/part2/mock_filerecord_manager.h new file mode 100644 index 0000000000..15f0456dd1 --- /dev/null +++ b/tests/part2/mock_filerecord_manager.h @@ -0,0 +1,42 @@ +/* + * Project: nebd + * Created Date: Sunday February 16th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#ifndef TESTS_PART2_MOCK_FILERECORD_MANAGER_H_ +#define TESTS_PART2_MOCK_FILERECORD_MANAGER_H_ + +#include +#include +#include + +#include "src/part2/filerecord_manager.h" + +namespace nebd { +namespace server { + +class MockFileRecordManager : public FileRecordManager { + public: + MockFileRecordManager() : FileRecordManager(nullptr) {} + ~MockFileRecordManager() {} + + MOCK_METHOD0(Load, int()); + MOCK_METHOD2(GetRecord, bool(int, NebdFileRecord*)); + MOCK_METHOD2(GetRecord, bool(const std::string&, NebdFileRecord*)); + MOCK_METHOD1(UpdateRecord, bool(const NebdFileRecord&)); + MOCK_METHOD1(RemoveRecord, bool(int)); + MOCK_METHOD1(Exist, bool(int)); + MOCK_METHOD0(Clear, void(void)); + MOCK_METHOD0(ListRecords, FileRecordMap(void)); + MOCK_METHOD2(UpdateFileTimestamp, bool(int, uint64_t)); + MOCK_METHOD2(GetFileTimestamp, bool(int, uint64_t*)); + MOCK_METHOD2(UpdateFileStatus, bool(int, NebdFileStatus)); + MOCK_METHOD2(GetFileStatus, bool(int, NebdFileStatus*)); +}; + +} // namespace server +} // namespace nebd + +#endif // TESTS_PART2_MOCK_FILERECORD_MANAGER_H_ diff --git a/tests/part2/mock_heartbeat_manager.h b/tests/part2/mock_heartbeat_manager.h new file mode 100644 index 0000000000..8ba405edd2 --- /dev/null +++ b/tests/part2/mock_heartbeat_manager.h @@ -0,0 +1,28 @@ +/* + * Project: nebd + * Created Date: Friday February 14th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#ifndef TESTS_PART2_MOCK_HEARTBEAT_MANAGER_H_ +#define TESTS_PART2_MOCK_HEARTBEAT_MANAGER_H_ + +#include +#include "src/part2/heartbeat_manager.h" + +namespace nebd { +namespace server { +class MockHeartbeatManager : public HeartbeatManager { + public: + MockHeartbeatManager() : HeartbeatManager({0, 0, nullptr}) {} + ~MockHeartbeatManager() {} + MOCK_METHOD0(Init, int()); + MOCK_METHOD0(Fini, int()); + MOCK_METHOD2(UpdateFileTimestamp, int(int, uint64_t)); +}; + +} // namespace server +} // namespace nebd + +#endif // TESTS_PART2_MOCK_HEARTBEAT_MANAGER_H_ diff --git a/tests/part2/mock_metafile_manager.h b/tests/part2/mock_metafile_manager.h index 69d4ae9596..866ca60992 100644 --- a/tests/part2/mock_metafile_manager.h +++ b/tests/part2/mock_metafile_manager.h @@ -22,9 +22,8 @@ class MockMetaFileManager : public NebdMetaFileManager { MockMetaFileManager() : NebdMetaFileManager("") {} ~MockMetaFileManager() {} - MOCK_METHOD1(RemoveFileRecord, int(const std::string&)); - MOCK_METHOD1(UpdateFileRecord, int(const NebdFileRecordPtr&)); - MOCK_METHOD1(ListFileRecord, int(std::vector*)); + MOCK_METHOD1(UpdateMetaFile, int(const FileRecordMap&)); + MOCK_METHOD1(ListFileRecord, int(FileRecordMap*)); }; } // namespace server diff --git a/tests/part2/mock_request_executor.h b/tests/part2/mock_request_executor.h index f1726eb92e..c32c9889d3 100644 --- a/tests/part2/mock_request_executor.h +++ b/tests/part2/mock_request_executor.h @@ -35,7 +35,6 @@ class MockRequestExecutor : public NebdRequestExecutor { MOCK_METHOD1(Close, int(NebdFileInstance*)); MOCK_METHOD2(Extend, int(NebdFileInstance*, int64_t)); MOCK_METHOD2(GetInfo, int(NebdFileInstance*, NebdFileInfo*)); - MOCK_METHOD2(StatFile, int(NebdFileInstance*, NebdFileInfo*)); MOCK_METHOD2(Discard, int(NebdFileInstance*, NebdServerAioContext*)); MOCK_METHOD2(AioRead, int(NebdFileInstance*, NebdServerAioContext*)); MOCK_METHOD2(AioWrite, int(NebdFileInstance*, NebdServerAioContext*)); diff --git a/tests/part2/nebd-server.err.conf b/tests/part2/nebd-server.err.conf index ec992a33b5..9dbfa9e956 100644 --- a/tests/part2/nebd-server.err.conf +++ b/tests/part2/nebd-server.err.conf @@ -2,4 +2,7 @@ meta.file.path = ./ #心跳超时时间 -heartbeat.timeout.sec=30 \ No newline at end of file +heartbeat.timeout.sec=30 + +#文件超时检测时间间隔 +heartbeat.check.interval.ms=3000 \ No newline at end of file From b1538b173b865d0ba4756c109400cdaed3128179 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Wed, 19 Feb 2020 15:03:32 +0800 Subject: [PATCH 30/79] replace '/' with '+' Change-Id: I4cffd80938613d92d66d91d2a016e063cda2ad23 --- src/common/file_lock.h | 5 +++-- src/part1/nebd_client.cpp | 14 +++++++++++++- src/part1/nebd_client.h | 8 ++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/common/file_lock.h b/src/common/file_lock.h index 0a48862ca2..db2ea2a0ab 100644 --- a/src/common/file_lock.h +++ b/src/common/file_lock.h @@ -16,8 +16,9 @@ namespace common { // 文件锁 class FileLock { public: - explicit FileLock(const std::string fileName) : fileName_(fileName), - fd_(-1) {} + explicit FileLock(const std::string& fileName) + : fileName_(fileName), fd_(-1) {} + FileLock() : fileName_(""), fd_(-1) {} ~FileLock() = default; diff --git a/src/part1/nebd_client.cpp b/src/part1/nebd_client.cpp index e555496aa8..62b81ea61a 100644 --- a/src/part1/nebd_client.cpp +++ b/src/part1/nebd_client.cpp @@ -91,7 +91,8 @@ void NebdClient::Uninit() { int NebdClient::Open(const char* filename) { // 加文件锁 - std::string fileLockName = option_.fileLockPath + "/" + filename; + std::string fileLockName = + option_.fileLockPath + "/" + ReplaceSlash(filename); FileLock fileLock(fileLockName); int res = fileLock.AcquireFileLock(); if (res < 0) { @@ -522,5 +523,16 @@ int64_t NebdClient::ExecuteSyncRpc(RpcTask task) { return -1; } +std::string NebdClient::ReplaceSlash(const std::string& str) { + std::string ret(str); + for (auto& ch : ret) { + if (ch == '/') { + ch = '+'; + } + } + + return ret; +} + } // namespace client } // namespace nebd diff --git a/src/part1/nebd_client.h b/src/part1/nebd_client.h index f493e202c6..d9d1fc39d1 100644 --- a/src/part1/nebd_client.h +++ b/src/part1/nebd_client.h @@ -135,6 +135,14 @@ class NebdClient { int InitChannel(); + /** + * @brief 替换字符串中的 '/' 为 '+' + * + * @param str 需要替换的字符串 + * @return 替换后的字符串 + */ + std::string ReplaceSlash(const std::string& str); + int64_t ExecuteSyncRpc(RpcTask task); // 心跳管理模块 std::shared_ptr heartbeatMgr_; From a7ed9795988c4b44914a657e878e77d2953fee4c Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Wed, 19 Feb 2020 16:26:17 +0800 Subject: [PATCH 31/79] fix ut failed Change-Id: Ia9a89d43695e5052fe18fe7a1676d35674102009 --- tests/part2/heartbeat_manager_unittest.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/part2/heartbeat_manager_unittest.cpp b/tests/part2/heartbeat_manager_unittest.cpp index d7124ae09e..1db21e3858 100644 --- a/tests/part2/heartbeat_manager_unittest.cpp +++ b/tests/part2/heartbeat_manager_unittest.cpp @@ -74,6 +74,17 @@ TEST_F(HeartbeatManagerTest, CheckTimeoutTest) { EXPECT_CALL(*fileRecordManager_, ListRecords()) .WillRepeatedly(Return(fileRecords)); + EXPECT_CALL(*fileRecordManager_, GetRecord(1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(record1), + Return(true))); + + EXPECT_CALL(*fileRecordManager_, GetRecord(2, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(record2), + Return(true))); + + EXPECT_CALL(*fileRecordManager_, GetRecord(3, _)) + .WillRepeatedly(Return(false)); + EXPECT_CALL(*fileManager_, Close(1, false)) .Times(AtLeast(1)); EXPECT_CALL(*fileManager_, Close(2, false)) From 0d41b702ab90f791072933f55bd6141248572a8d Mon Sep 17 00:00:00 2001 From: charisu Date: Wed, 19 Feb 2020 14:34:44 +0800 Subject: [PATCH 32/79] add nebd-daemon Change-Id: I8ad9284caf7ca02cc882e2c2c642dd7fb57a3e3c --- .gitignore | 1 - etc/nebd/nebd-server.conf | 2 +- nebd-package/DEBIAN/control | 10 ++ nebd-package/usr/bin/nebd-daemon | 197 +++++++++++++++++++++++++++++++ src/part2/main.cpp | 2 + src/part2/nebd_server.cpp | 2 +- 6 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 nebd-package/DEBIAN/control create mode 100755 nebd-package/usr/bin/nebd-daemon diff --git a/.gitignore b/.gitignore index 376fc5f567..cb424c04dc 100644 --- a/.gitignore +++ b/.gitignore @@ -41,7 +41,6 @@ build .idea cmake-build-debug CMakeLists.txt -bin lib *.temp .clwb diff --git a/etc/nebd/nebd-server.conf b/etc/nebd/nebd-server.conf index 84b4da1111..97a0ece75e 100644 --- a/etc/nebd/nebd-server.conf +++ b/etc/nebd/nebd-server.conf @@ -2,7 +2,7 @@ listen.address=/var/run/nebd.sock #元数据文件地址,包含文件名 -meta.file.path=./nebdserver.meta +meta.file.path=/etc/nebd/nebdserver.meta #心跳超时时间 heartbeat.timeout.sec=30 diff --git a/nebd-package/DEBIAN/control b/nebd-package/DEBIAN/control new file mode 100644 index 0000000000..5dde3f4f01 --- /dev/null +++ b/nebd-package/DEBIAN/control @@ -0,0 +1,10 @@ +Package: nebd +Section: +Priority: optional +Depends: libunwind8 +Suggests: +Architecture:amd64 +Installed-Size: +Maintainer: nebd-dev +Provides: +Description: diff --git a/nebd-package/usr/bin/nebd-daemon b/nebd-package/usr/bin/nebd-daemon new file mode 100755 index 0000000000..0bc8ffa807 --- /dev/null +++ b/nebd-package/usr/bin/nebd-daemon @@ -0,0 +1,197 @@ +#!/bin/bash + +# nebd-server路径 +bin=/usr/bin/nebd-server + +# 默认配置文件 +confPath=/etc/nebd/nebd-server.conf + +# 日志文件路径 +logPath=${HOME} + +# pidfile +pidFile=${HOME}/nebd-server.pid + +# daemon log +daemonLog=${HOME}/nebd-server-daemon.log + +# console output +consoleLog=${HOME}/nebd-server-console.log + +# 启动nebd-server +function start() { + # 检查daemon + if ! type daemon &> /dev/null + then + echo "No daemon installed" + exit 1 + fi + + # 检查nebd-server + if [ ! -f ${bin} ] + then + echo "No nebd-server installed" + exit 1 + fi + + # 检查配置文件 + if [ ! -f ${confPath} ] + then + echo "Not found nebd-server.conf, Path is ${confPath}" + exit 1 + fi + + # 判断是否已经通过daemon启动了nebd-server + daemon --name nebd-server --pidfile ${pidFile} --running + if [ $? -eq 0 ] + then + echo "Already started nebd-server by daemon" + exit 1 + fi + + # 创建logPath + mkdir -p ${logPath} > /dev/null 2>&1 + if [ $? -ne 0 ] + then + echo "Create log dir failed: ${logPath}" + exit 1 + fi + + # 检查logPath是否有写权限 + if [ ! -w ${logPath} ] + then + echo "Write permission denied: ${logPath}" + exit 1 + fi + + # 检查consoleLog是否可写或者是否能够创建 + touch ${consoleLog} > /dev/null 2>&1 + if [ $? -ne 0 ] + then + echo "Can't Write or Create console Log: ${consoleLog}" + exit 1 + fi + + # 检查daemonLog是否可写或者是否能够创建 + touch ${daemonLog} > /dev/null 2>&1 + if [ $? -ne 0 ] + then + echo "Can't Write or Create daemon logfile: ${daemonLog}" + exit 1 + fi + + daemon --name nebd-server --core --inherit \ + --respawn --attempts 100 --delay 10 \ + --pidfile ${pidFile} \ + --errlog ${daemonLog} \ + --output ${consoleLog} \ + -- ${bin} -confPath=${confPath} -log_dir=${logPath} -graceful_quit_on_sigterm=true -stderrthreshold=3 +} + +# 停止daemon进程，且停止nebd-server +function stop() { + # 判断是否已经通过daemon启动了nebd-server + daemon --name nebd-server --pidfile ${pidFile} --running + if [ $? -ne 0 ] + then + echo "Didn't start nebd-server by daemon" + exit 1 + fi + + daemon --name nebd-server --pidfile ${pidFile} --stop + if [ $? -ne 0 ] + then + echo "stop may not success!" + else + echo "nebd-server exit success!" + echo "daemon exit success!" + fi +} + +# restart +function restart() { + # 判断是否已经通过daemon启动了nebd-server + daemon --name nebd-server --pidfile ${pidFile} --running + if [ $? -ne 0 ] + then + echo "Didn't start nebd-server by daemon" + exit 1 + fi + + daemon --name nebd-server --pidfile ${pidFile} --restart +} + +# status +function status() { + daemon --name nebd-server --pidfile ${pidFile} --running + if [ $? -ne 0 ] + then + echo "Didn't start nebd-server by daemon" + else + echo "nebd-server is running by daemon" + fi +} + +# 使用方式 +function usage() { + echo "Usage:" + echo " nebd-daemon start -- start deamon process and watch on nebd-server process" + echo " [-c|--confPath path] conf path" + echo " [-l|--logPath path] log path" + echo " nebd-daemon stop -- stop daemon process and nebd-server" + echo " nebd-daemon restart -- restart nebd-server" + echo " nebd-daemon status -- show if the nebd-server is running by daemon" + echo "Examples:" + echo " nebd-daemon start -c /etc/nebd/nebd-server.conf -l ${HOME}/" +} + +# 检查参数启动参数，最少1个 +if [ $# -lt 1 ] +then + usage + exit +fi + +case $1 in +"start") + shift # pass first argument + + # 解析参数 + while [[ $# -gt 1 ]] + do + key=$1 + + case $key in + -c|--confPath) + confPath=`realpath $2` + shift # pass key + shift # pass value + ;; + -l|--logPath) + logPath=`realpath $2` + shift # pass key + shift # pass value + ;; + *) + usage + exit + ;; + esac + done + + start + ;; +"stop") + stop + ;; +"restart") + restart + ;; +"status") + status + ;; +*) + usage + ;; +esac + diff --git a/src/part2/main.cpp b/src/part2/main.cpp index 21e1a24266..afdd0e7fba 100644 --- a/src/part2/main.cpp +++ b/src/part2/main.cpp @@ -17,6 +17,7 @@ DEFINE_string(confPath, "/etc/nebd/nebd.conf", "nebd conf path"); int main(int argc, char* argv[]) { // 解析参数 google::ParseCommandLineFlags(&argc, &argv, false); + google::InitGoogleLogging(argv[0]); std::string confPath = FLAGS_confPath.c_str(); // 启动nebd server @@ -31,5 +32,6 @@ int main(int argc, char* argv[]) { // 停止nebd server server->Fini(); + google::ShutdownGoogleLogging(); return 0; } diff --git a/src/part2/nebd_server.cpp b/src/part2/nebd_server.cpp index 4661914f6f..a30b6776ea 100644 --- a/src/part2/nebd_server.cpp +++ b/src/part2/nebd_server.cpp @@ -169,7 +169,7 @@ bool NebdServer::StartServer() { brpc::ServerOptions option; option.idle_timeout_sec = -1; // 获取文件锁 - common::FileLock fileLock(listenAddress_); + common::FileLock fileLock(listenAddress_ + ".lock"); if (fileLock.AcquireFileLock() != 0) { LOG(ERROR) << "Address already in use"; return -1; From d1e3cef44f8bb0770d9df02d29533f6c11c8f01e Mon Sep 17 00:00:00 2001 From: lixiaocui1 Date: Wed, 19 Feb 2020 15:00:19 +0800 Subject: [PATCH 33/79] =?UTF-8?q?[fix]=E5=88=9D=E5=A7=8B=E5=8C=96request?= =?UTF-8?q?=20executor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I5712a4b20fd005264d6db90abc359772b29a3eed --- etc/nebd/nebd-server.conf | 5 ++- proto/BUILD | 2 +- proto/client.proto | 2 +- proto/heartbeat.proto | 2 +- proto/{common.proto => nebd-common.proto} | 2 +- src/part2/define.h | 1 + src/part2/nebd_server.cpp | 39 ++++++++++++++++++++++- src/part2/nebd_server.h | 14 +++++++- tests/part2/BUILD | 1 + tests/part2/test_nebd_server.cpp | 31 +++++++++++++----- 10 files changed, 84 insertions(+), 15 deletions(-) rename proto/{common.proto => nebd-common.proto} (97%) diff --git a/etc/nebd/nebd-server.conf b/etc/nebd/nebd-server.conf index 97a0ece75e..928927ee83 100644 --- a/etc/nebd/nebd-server.conf +++ b/etc/nebd/nebd-server.conf @@ -1,8 +1,11 @@ +# curve-client配置文件地址 +curveclient.confPath=/etc/curve/client.conf + #brpc server监听端口 listen.address=/var/run/nebd.sock #元数据文件地址,包含文件名 -meta.file.path=/etc/nebd/nebdserver.meta +meta.file.path=/etc/nebd/nebdserver.meta #心跳超时时间 heartbeat.timeout.sec=30 diff --git a/proto/BUILD b/proto/BUILD index a45f04983a..d98db221e8 100644 --- a/proto/BUILD +++ b/proto/BUILD @@ -7,7 +7,7 @@ cc_proto_library( proto_library( name = "client_proto", srcs = glob([ - "common.proto", + "nebd-common.proto", "client.proto", "heartbeat.proto" ] diff --git a/proto/client.proto b/proto/client.proto index 56e653fd86..144402605b 100644 --- a/proto/client.proto +++ b/proto/client.proto @@ -1,5 +1,5 @@ syntax="proto2"; -import "proto/common.proto"; +import "proto/nebd-common.proto"; package nebd.client; diff --git a/proto/heartbeat.proto b/proto/heartbeat.proto index 79ba28938a..5d2e7a8068 100644 --- a/proto/heartbeat.proto +++ b/proto/heartbeat.proto @@ -1,5 +1,5 @@ syntax="proto2"; -import "proto/common.proto"; +import "proto/nebd-common.proto"; package nebd.client; diff --git a/proto/common.proto b/proto/nebd-common.proto similarity index 97% rename from proto/common.proto rename to proto/nebd-common.proto index 74828b6ebd..55746a6bd3 100644 --- a/proto/common.proto +++ b/proto/nebd-common.proto @@ -5,4 +5,4 @@ option cc_generic_services = true; enum RetCode { kNoOK = -1; kOK = 0; -}; \ No newline at end of file +}; diff --git a/src/part2/define.h b/src/part2/define.h index c08a03cf87..3fd76c7234 100644 --- a/src/part2/define.h +++ b/src/part2/define.h @@ -114,6 +114,7 @@ const char LISTENADDRESS[] = "listen.address"; const char METAFILEPATH[] = "meta.file.path"; const char HEARTBEATTIMEOUTSEC[] = "heartbeat.timeout.sec"; const char HEARTBEATCHECKINTERVALMS[] = "heartbeat.check.interval.ms"; +const char CURVECLIENTCONFPATH[] = "curveclient.confPath"; } // namespace server } // namespace nebd diff --git a/src/part2/nebd_server.cpp b/src/part2/nebd_server.cpp index a30b6776ea..df59095aa1 100644 --- a/src/part2/nebd_server.cpp +++ b/src/part2/nebd_server.cpp @@ -14,7 +14,8 @@ namespace nebd { namespace server { -int NebdServer::Init(const std::string &confPath) { +int NebdServer::Init(const std::string &confPath, + std::shared_ptr curveClient) { if (isRunning_) { LOG(WARNING) << "NebdServer is inited"; return -1; @@ -25,24 +26,37 @@ int NebdServer::Init(const std::string &confPath) { LOG(ERROR) << "NebdServer load config from file fail"; return -1; } + LOG(INFO) << "NebdServer load config from file ok"; bool initAddressOk = conf_.GetStringValue(LISTENADDRESS, &listenAddress_); if (false == initAddressOk) { LOG(ERROR) << "NebdServer init socket file address fail"; return -1; } + LOG(INFO) << "NebdServer init socket file address ok"; + bool initFileManagerOk = InitFileManager(); if (false == initFileManagerOk) { LOG(ERROR) << "NebdServer init fileManager fail"; return -1; } + LOG(INFO) << "NebdServer init fileManager ok"; + + curveClient_ = curveClient; + bool initExecutorOk = InitCurveRequestExecutor(); + if (false == initExecutorOk) { + LOG(ERROR) << "NebdServer init curveRequestExecutor fail"; + return -1; + } + LOG(INFO) << "NebdServer init curveRequestExecutor ok"; bool initHeartbeatManagerOk = InitHeartbeatManager(); if (false == initHeartbeatManagerOk) { LOG(ERROR) << "NebdServer init heartbeatManager fail"; return -1; } + LOG(INFO) << "NebdServer init heartbeatManager ok"; LOG(INFO) << "NebdServer init ok"; return 0; @@ -62,6 +76,10 @@ int NebdServer::Fini() { brpc::AskToQuit(); } + if (curveClient_ != nullptr) { + curveClient_ ->UnInit(); + } + if (fileManager_ != nullptr) { fileManager_->Fini(); } @@ -69,6 +87,7 @@ int NebdServer::Fini() { if (heartbeatManager_ != nullptr) { heartbeatManager_->Fini(); } + return 0; } @@ -100,6 +119,24 @@ bool NebdServer::InitFileManager() { return true; } +bool NebdServer::InitCurveRequestExecutor() { + std::string confPath; + bool getOk = conf_.GetStringValue(CURVECLIENTCONFPATH, &confPath); + if (!getOk) { + LOG(ERROR) << "get " << CURVECLIENTCONFPATH << " fail"; + return false; + } + + int initRes = curveClient_->Init(confPath); + if (initRes < 0) { + LOG(ERROR) << "Init curve client fail"; + return false; + } + + CurveRequestExecutor::GetInstance().Init(curveClient_); + return true; +} + MetaFileManagerPtr NebdServer::InitMetaFileManager() { std::string metaFilePath; bool getOk = conf_.GetStringValue(METAFILEPATH, &metaFilePath); diff --git a/src/part2/nebd_server.h b/src/part2/nebd_server.h index d6b3bc7979..b6eb9c764d 100644 --- a/src/part2/nebd_server.h +++ b/src/part2/nebd_server.h @@ -14,18 +14,22 @@ #include "src/common/configuration.h" #include "src/part2/file_manager.h" #include "src/part2/heartbeat_manager.h" +#include "src/part2/request_executor_curve.h" namespace nebd { namespace server { using ::nebd::common::Configuration; +using ::curve::client::CurveClient; class NebdServer { public: NebdServer() {} virtual ~NebdServer() {} - int Init(const std::string &confPath); + int Init(const std::string &confPath, + std::shared_ptr curveClient = + std::make_shared()); int RunUntilAskedToQuit(); @@ -45,6 +49,12 @@ class NebdServer { */ bool InitFileManager(); + /** + * @brief 初始化request_executor_curve + * @return false-初始化失败 true-初始化成功 + */ + bool InitCurveRequestExecutor(); + /** * @brief 初始化NebdMetaFileManager * @return nullptr-初始化不成功 否则表示初始化成功 @@ -84,6 +94,8 @@ class NebdServer { std::shared_ptr fileManager_; // 负责文件心跳超时处理 std::shared_ptr heartbeatManager_; + // curveclient + std::shared_ptr curveClient_; }; } // namespace server diff --git a/tests/part2/BUILD b/tests/part2/BUILD index 55d806d89a..da40fd927c 100644 --- a/tests/part2/BUILD +++ b/tests/part2/BUILD @@ -84,6 +84,7 @@ cc_binary( deps = [ "//external:gflags", "//src/part2:nebdserver", + "//tests/part2:mock_lib", "@com_google_googletest//:gtest_main", ], ) diff --git a/tests/part2/test_nebd_server.cpp b/tests/part2/test_nebd_server.cpp index 889b11b035..f18fb5d285 100644 --- a/tests/part2/test_nebd_server.cpp +++ b/tests/part2/test_nebd_server.cpp @@ -7,37 +7,52 @@ #include #include "src/part2/nebd_server.h" +#include "tests/part2/mock_curve_client.h" namespace nebd { namespace server { +using ::testing::Return; +using ::testing::_; +using ::testing::SetArgPointee; +using ::testing::DoAll; + TEST(TestNebdServer, test_Init_Run_Fini) { NebdServer server; + auto curveClient = std::make_shared(); std::string confPath; // 1. 配置文件不存在, init失败 + EXPECT_CALL(*curveClient, Init(_)).Times(0); confPath = "./nebd.conf"; - ASSERT_EQ(-1, server.Init(confPath)); + ASSERT_EQ(-1, server.Init(confPath, curveClient)); // 2. 配置文件存在, 监听端口未设置 + EXPECT_CALL(*curveClient, Init(_)).Times(0); confPath = "./nebd-server.err.conf"; - ASSERT_EQ(-1, server.Init(confPath)); + ASSERT_EQ(-1, server.Init(confPath, curveClient)); - // 3. 初始化成功 + // 3. curveclient init失败 + EXPECT_CALL(*curveClient, Init(_)).WillOnce(Return(-1)); confPath = "./etc/nebd/nebd-server.conf"; - ASSERT_EQ(0, server.Init(confPath)); + ASSERT_EQ(-1, server.Init(confPath, curveClient)); + + // 4. 初始化成功 + EXPECT_CALL(*curveClient, Init(_)).WillOnce(Return(0)); + ASSERT_EQ(0, server.Init(confPath, curveClient)); - // 4. run成功 + // 5. run成功 + EXPECT_CALL(*curveClient, UnInit()).Times(2); std::thread nebdServerThread(&NebdServer::RunUntilAskedToQuit, &server); sleep(1); - // 5、再次Run会失败 + // 6、再次Run会失败 ASSERT_EQ(-1, server.RunUntilAskedToQuit()); - // 6. stop成功 + // 7. stop成功 ASSERT_EQ(0, server.Fini()); - // 7. 再次stop不会重复释放资源 + // 8. 再次stop不会重复释放资源 ASSERT_EQ(0, server.Fini()); nebdServerThread.join(); } From 6c17227ca9536b6195e0d2ba2d749eee619cc24b Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Wed, 19 Feb 2020 21:03:13 +0800 Subject: [PATCH 34/79] fix aiocallback core Change-Id: I9eb1e36b36ad7eeed5a07fdb657727e2fadb445d --- src/part2/request_executor_curve.cpp | 16 ++++------------ src/part2/request_executor_curve.h | 2 +- tests/part2/test_request_executor_curve.cpp | 19 +++++++++++++++++-- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/part2/request_executor_curve.cpp b/src/part2/request_executor_curve.cpp index 4fe3f20a46..78e0243e79 100644 --- a/src/part2/request_executor_curve.cpp +++ b/src/part2/request_executor_curve.cpp @@ -147,17 +147,14 @@ int CurveRequestExecutor::AioRead( CurveAioCombineContext *curveCombineCtx = new CurveAioCombineContext(); curveCombineCtx->nebdCtx = aioctx; - curveCombineCtx->curveCtx = new CurveAioContext(); - int ret = FromNebdCtxToCurveCtx(aioctx, curveCombineCtx->curveCtx); + int ret = FromNebdCtxToCurveCtx(aioctx, &curveCombineCtx->curveCtx); if (ret < 0) { - delete curveCombineCtx->curveCtx; delete curveCombineCtx; return -1; } - ret = client_->AioRead(curveFd, curveCombineCtx->curveCtx); + ret = client_->AioRead(curveFd, &curveCombineCtx->curveCtx); if (ret != LIBCURVE_ERROR::OK) { - delete curveCombineCtx->curveCtx; delete curveCombineCtx; return -1; } @@ -174,17 +171,14 @@ int CurveRequestExecutor::AioWrite( CurveAioCombineContext *curveCombineCtx = new CurveAioCombineContext(); curveCombineCtx->nebdCtx = aioctx; - curveCombineCtx->curveCtx = new CurveAioContext(); - int ret = FromNebdCtxToCurveCtx(aioctx, curveCombineCtx->curveCtx); + int ret = FromNebdCtxToCurveCtx(aioctx, &curveCombineCtx->curveCtx); if (ret < 0) { - delete curveCombineCtx->curveCtx; delete curveCombineCtx; return -1; } - ret = client_->AioWrite(curveFd, curveCombineCtx->curveCtx); + ret = client_->AioWrite(curveFd, &curveCombineCtx->curveCtx); if (ret != LIBCURVE_ERROR::OK) { - delete curveCombineCtx->curveCtx; delete curveCombineCtx; return -1; } @@ -259,10 +253,8 @@ void CurveAioCallback(struct CurveAioContext* curveCtx) { auto curveCombineCtx = reinterpret_cast( reinterpret_cast(curveCtx) - offsetof(CurveAioCombineContext, curveCtx)); - curveCombineCtx->nebdCtx->ret = curveCtx->ret; curveCombineCtx->nebdCtx->cb(curveCombineCtx->nebdCtx); - delete curveCtx; delete curveCombineCtx; } diff --git a/src/part2/request_executor_curve.h b/src/part2/request_executor_curve.h index d986e87a44..029fa1cb04 100644 --- a/src/part2/request_executor_curve.h +++ b/src/part2/request_executor_curve.h @@ -31,7 +31,7 @@ class CurveFileInstance : public NebdFileInstance { class CurveAioCombineContext { public: NebdServerAioContext* nebdCtx; - CurveAioContext* curveCtx; + CurveAioContext curveCtx; }; void CurveAioCallback(struct CurveAioContext* curveCtx); diff --git a/tests/part2/test_request_executor_curve.cpp b/tests/part2/test_request_executor_curve.cpp index 572b7c5a31..a7fb79df07 100644 --- a/tests/part2/test_request_executor_curve.cpp +++ b/tests/part2/test_request_executor_curve.cpp @@ -16,6 +16,11 @@ using ::testing::Return; using ::testing::_; using ::testing::SetArgPointee; using ::testing::DoAll; +using ::testing::SaveArg; + +void NebdUnitTestCallback(NebdServerAioContext* context) { + std::cout << "callback" << std::endl; +} class TestReuqestExecutorCurve : public ::testing::Test { protected: @@ -219,6 +224,7 @@ TEST_F(TestReuqestExecutorCurve, test_GetInfo) { TEST_F(TestReuqestExecutorCurve, test_AioRead) { auto executor = CurveRequestExecutor::GetInstance(); NebdServerAioContext aiotcx; + aiotcx.cb = NebdUnitTestCallback; std::string curveFilename("/cinder/volume-1234_cinder_"); // 1. nebdFileIns不是CurveFileInstance类型, 异步读失败 @@ -255,15 +261,19 @@ TEST_F(TestReuqestExecutorCurve, test_AioRead) { auto curveFileIns = new CurveFileInstance(); curveFileIns->fd = 1; curveFileIns->fileName = curveFilename; + CurveAioContext* curveCtx; EXPECT_CALL(*curveClient_, AioRead(1, _)) - .WillOnce(Return(LIBCURVE_ERROR::OK)); + .WillOnce(DoAll(SaveArg<1>(&curveCtx), + Return(LIBCURVE_ERROR::OK))); ASSERT_EQ(0, executor.AioRead(curveFileIns, &aiotcx)); + curveCtx->cb(curveCtx); } } TEST_F(TestReuqestExecutorCurve, test_AioWrite) { auto executor = CurveRequestExecutor::GetInstance(); NebdServerAioContext aiotcx; + aiotcx.cb = NebdUnitTestCallback; std::string curveFilename("/cinder/volume-1234_cinder_"); // 1. nebdFileIns不是CurveFileInstance类型, 异步写失败 @@ -300,9 +310,12 @@ TEST_F(TestReuqestExecutorCurve, test_AioWrite) { auto curveFileIns = new CurveFileInstance(); curveFileIns->fd = 1; curveFileIns->fileName = curveFilename; + CurveAioContext* curveCtx; EXPECT_CALL(*curveClient_, AioWrite(1, _)) - .WillOnce(Return(LIBCURVE_ERROR::OK)); + .WillOnce(DoAll(SaveArg<1>(&curveCtx), + Return(LIBCURVE_ERROR::OK))); ASSERT_EQ(0, executor.AioWrite(curveFileIns, &aiotcx)); + curveCtx->cb(curveCtx); } } @@ -340,6 +353,7 @@ TEST_F(TestReuqestExecutorCurve, test_InvalidCache) { } } + TEST(TestFileNameParser, test_Parse) { std::string fileName("cbd:pool1//cinder/volume-1234_cinder_:/client.conf"); std::string res("/cinder/volume-1234_cinder_"); @@ -355,6 +369,7 @@ TEST(TestFileNameParser, test_Parse) { ASSERT_EQ("", FileNameParser::Parse(fileName)); } + } // namespace server } // namespace nebd From d4b76efa5761ef77f10cc25a7d84e4b11df30bac Mon Sep 17 00:00:00 2001 From: charisu Date: Wed, 19 Feb 2020 20:47:36 +0800 Subject: [PATCH 35/79] fix:nebd-server init bug Change-Id: I03af83ccae9defd1b59686f262390de0e96a990c --- WORKSPACE | 2 +- src/common/configuration.cpp | 14 +++++++-- src/part2/main.cpp | 2 +- src/part2/nebd_server.cpp | 23 +++++++------- tests/common/configuration_test.cpp | 17 ++++++++-- tests/part2/nebd-server.err.conf | 8 ----- tests/part2/test_nebd_server.cpp | 49 +++++++++++++++++++++++------ 7 files changed, 78 insertions(+), 37 deletions(-) delete mode 100644 tests/part2/nebd-server.err.conf diff --git a/WORKSPACE b/WORKSPACE index 23098eabea..ea0f0d7067 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -98,7 +98,7 @@ bind( git_repository( name = "com_netease_storage_gerrit_curve_curve", remote = "http://gerrit.storage.netease.com/curve", - commit = "3f3a65750164060bc372b10a3ce5a43515dcaa3d", + commit = "45e8f8887580d367644d3624d7ebd033ebc69503", ) bind( diff --git a/src/common/configuration.cpp b/src/common/configuration.cpp index 7434a336c0..f457238fa5 100644 --- a/src/common/configuration.cpp +++ b/src/common/configuration.cpp @@ -46,8 +46,18 @@ bool Configuration::LoadConfig() { } bool Configuration::SaveConfig() { - // TODO(wenyu): to implement - return false; + // 当前先只保存配置，原文件的注释等内容先忽略 + // TODO(yyk): 后续考虑改成原文件格式不变，只修改配置值 + std::ofstream wStream(confFile_); + if (wStream.is_open()) { + for (auto& pair : config_) { + wStream << pair.first << "=" << pair.second << std::endl; + } + wStream.close(); + } else { + return false; + } + return true; } std::string Configuration::DumpConfig() { diff --git a/src/part2/main.cpp b/src/part2/main.cpp index afdd0e7fba..2cf92604ad 100644 --- a/src/part2/main.cpp +++ b/src/part2/main.cpp @@ -12,7 +12,7 @@ #define BOOST_SPIRIT_THREADSAFE -DEFINE_string(confPath, "/etc/nebd/nebd.conf", "nebd conf path"); +DEFINE_string(confPath, "/etc/nebd/nebd-server.conf", "nebd server conf path"); int main(int argc, char* argv[]) { // 解析参数 diff --git a/src/part2/nebd_server.cpp b/src/part2/nebd_server.cpp index df59095aa1..1a430b7dcf 100644 --- a/src/part2/nebd_server.cpp +++ b/src/part2/nebd_server.cpp @@ -35,14 +35,6 @@ int NebdServer::Init(const std::string &confPath, } LOG(INFO) << "NebdServer init socket file address ok"; - - bool initFileManagerOk = InitFileManager(); - if (false == initFileManagerOk) { - LOG(ERROR) << "NebdServer init fileManager fail"; - return -1; - } - LOG(INFO) << "NebdServer init fileManager ok"; - curveClient_ = curveClient; bool initExecutorOk = InitCurveRequestExecutor(); if (false == initExecutorOk) { @@ -51,6 +43,13 @@ int NebdServer::Init(const std::string &confPath, } LOG(INFO) << "NebdServer init curveRequestExecutor ok"; + bool initFileManagerOk = InitFileManager(); + if (false == initFileManagerOk) { + LOG(ERROR) << "NebdServer init fileManager fail"; + return -1; + } + LOG(INFO) << "NebdServer init fileManager ok"; + bool initHeartbeatManagerOk = InitHeartbeatManager(); if (false == initHeartbeatManagerOk) { LOG(ERROR) << "NebdServer init heartbeatManager fail"; @@ -76,14 +75,14 @@ int NebdServer::Fini() { brpc::AskToQuit(); } - if (curveClient_ != nullptr) { - curveClient_ ->UnInit(); - } - if (fileManager_ != nullptr) { fileManager_->Fini(); } + if (curveClient_ != nullptr) { + curveClient_ ->UnInit(); + } + if (heartbeatManager_ != nullptr) { heartbeatManager_->Fini(); } diff --git a/tests/common/configuration_test.cpp b/tests/common/configuration_test.cpp index 83d158970f..ce9c5705e5 100644 --- a/tests/common/configuration_test.cpp +++ b/tests/common/configuration_test.cpp @@ -131,11 +131,22 @@ TEST_F(ConfigurationTest, ListConfig) { TEST_F(ConfigurationTest, SaveConfig) { bool ret; Configuration conf; - conf.SetConfigPath(confFile_); + + // 自定义配置项并保存 + conf.SetStringValue("test.str1", "new"); ret = conf.SaveConfig(); - // not implemented yet, assert false - ASSERT_EQ(ret, false); + ASSERT_EQ(ret, true); + + // 重新加载配置项 + Configuration conf2; + conf2.SetConfigPath(confFile_); + ret = conf2.LoadConfig(); + ASSERT_EQ(ret, true); + + // 可以读取自定义配置项，原有配置项被覆盖，读取不到 + ASSERT_EQ(conf2.GetValue("test.str1"), "new"); + ASSERT_EQ(conf2.GetValue("test.int1"), ""); } TEST_F(ConfigurationTest, GetSetValue) { diff --git a/tests/part2/nebd-server.err.conf b/tests/part2/nebd-server.err.conf deleted file mode 100644 index 9dbfa9e956..0000000000 --- a/tests/part2/nebd-server.err.conf +++ /dev/null @@ -1,8 +0,0 @@ -#元数据文件地址 -meta.file.path = ./ - -#心跳超时时间 -heartbeat.timeout.sec=30 - -#文件超时检测时间间隔 -heartbeat.check.interval.ms=3000 \ No newline at end of file diff --git a/tests/part2/test_nebd_server.cpp b/tests/part2/test_nebd_server.cpp index f18fb5d285..6a390ca150 100644 --- a/tests/part2/test_nebd_server.cpp +++ b/tests/part2/test_nebd_server.cpp @@ -23,32 +23,61 @@ TEST(TestNebdServer, test_Init_Run_Fini) { std::string confPath; // 1. 配置文件不存在, init失败 - EXPECT_CALL(*curveClient, Init(_)).Times(0); confPath = "./nebd.conf"; - ASSERT_EQ(-1, server.Init(confPath, curveClient)); + ASSERT_EQ(-1, server.Init(confPath)); // 2. 配置文件存在, 监听端口未设置 - EXPECT_CALL(*curveClient, Init(_)).Times(0); - confPath = "./nebd-server.err.conf"; - ASSERT_EQ(-1, server.Init(confPath, curveClient)); + confPath = "./tests/part2/nebd-server-err.conf"; + Configuration conf; + conf.SetConfigPath(confPath); + conf.SaveConfig(); + ASSERT_EQ(-1, server.Init(confPath)); - // 3. curveclient init失败 + // 3、配置文件中没有client配置 + conf.SetStringValue("listen.address", "/tmp/nebd-server.sock"); + conf.SaveConfig(); + ASSERT_EQ(-1, server.Init(confPath)); + + // 4. curveclient init失败 + conf.SetStringValue("curveclient.confPath", "/etc/curve/client.conf"); + conf.SaveConfig(); EXPECT_CALL(*curveClient, Init(_)).WillOnce(Return(-1)); - confPath = "./etc/nebd/nebd-server.conf"; ASSERT_EQ(-1, server.Init(confPath, curveClient)); - // 4. 初始化成功 + // 5、初始化fileManager失败 + EXPECT_CALL(*curveClient, Init(_)).WillOnce(Return(0)); + ASSERT_EQ(-1, server.Init(confPath, curveClient)); + + // 6、没有heartbeat.timeout字段 EXPECT_CALL(*curveClient, Init(_)).WillOnce(Return(0)); + conf.SetStringValue("meta.file.path", "./nebd-server-test.meta"); + conf.SaveConfig(); + ASSERT_EQ(-1, server.Init(confPath, curveClient)); + + // 7、没有heartbeat.check.interval.ms字段 + EXPECT_CALL(*curveClient, Init(_)).WillOnce(Return(0)); + conf.SetIntValue("heartbeat.timeout.sec", 30); + conf.SaveConfig(); + ASSERT_EQ(-1, server.Init(confPath, curveClient)); + + + // 8. 初始化成功 + EXPECT_CALL(*curveClient, Init(_)).WillOnce(Return(0)); + conf.SetIntValue("heartbeat.check.interval.ms", 3000); + conf.SaveConfig(); ASSERT_EQ(0, server.Init(confPath, curveClient)); - // 5. run成功 + // 9. run成功 EXPECT_CALL(*curveClient, UnInit()).Times(2); std::thread nebdServerThread(&NebdServer::RunUntilAskedToQuit, &server); sleep(1); - // 6、再次Run会失败 + // 10、再次Run会失败 ASSERT_EQ(-1, server.RunUntilAskedToQuit()); + // 11、Run之后Init会失败 + ASSERT_EQ(-1, server.Init(confPath, curveClient)); + // 7. stop成功 ASSERT_EQ(0, server.Fini()); From 02d81661819a1877f677dfcbf63e8d33eb23ccb6 Mon Sep 17 00:00:00 2001 From: charisu Date: Thu, 20 Feb 2020 15:23:40 +0800 Subject: [PATCH 36/79] metafilemanager check crc Change-Id: I4610812203fe6c00443db59aa7149c59b69096d5 --- src/common/BUILD | 1 + src/common/crc32.h | 45 +++++++++++++++ src/part2/metafile_manager.cpp | 69 +++++++++++++++++------ src/part2/metafile_manager.h | 5 +- tests/common/crc32_test.cpp | 81 +++++++++++++++++++++++++++ tests/part2/metafile_manager_test.cpp | 39 +++++++++++-- 6 files changed, 216 insertions(+), 24 deletions(-) create mode 100644 src/common/crc32.h create mode 100644 tests/common/crc32_test.cpp diff --git a/src/common/BUILD b/src/common/BUILD index 9794ec3e36..88bce27388 100644 --- a/src/common/BUILD +++ b/src/common/BUILD @@ -8,6 +8,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ "//external:glog", + "//external:butil", ], ) diff --git a/src/common/crc32.h b/src/common/crc32.h new file mode 100644 index 0000000000..8b4c54ca8b --- /dev/null +++ b/src/common/crc32.h @@ -0,0 +1,45 @@ +/* + * Project: curve + * Created Date: Saturday December 29th 2018 + * Author: yangyaokai + * Copyright (c) 2018 netease + */ + +#ifndef SRC_COMMON_CRC32_H_ +#define SRC_COMMON_CRC32_H_ + +#include +#include + +#include + +namespace nebd { +namespace common { + +/** + * 计算数据的CRC32校验码(CRC32C)，基于brpc的crc32库进行封装 + * @param pData 待计算的数据 + * @param iLen 待计算的数据长度 + * @return 32位的数据CRC32校验码 + */ +inline uint32_t CRC32(const char *pData, size_t iLen) { + return butil::crc32c::Value(pData, iLen); +} + +/** + * 计算数据的CRC32校验码(CRC32C)，基于brpc的crc32库进行封装. 此函数支持继承式 + * 计算，以支持对SGL类型的数据计算单个CRC校验码。满足如下约束: + * CRC32("hello world", 11) == CRC32(CRC32("hello ", 6), "world", 5) + * @param crc 起始的crc校验码 + * @param pData 待计算的数据 + * @param iLen 待计算的数据长度 + * @return 32位的数据CRC32校验码 + */ +inline uint32_t CRC32(uint32_t crc, const char *pData, size_t iLen) { + return butil::crc32c::Extend(crc, pData, iLen); +} + +} // namespace common +} // namespace nebd + +#endif // SRC_COMMON_CRC32_H_ diff --git a/src/part2/metafile_manager.cpp b/src/part2/metafile_manager.cpp index 8b991f2ec7..7d0c4aae52 100644 --- a/src/part2/metafile_manager.cpp +++ b/src/part2/metafile_manager.cpp @@ -25,21 +25,7 @@ NebdMetaFileManager::NebdMetaFileManager( NebdMetaFileManager::~NebdMetaFileManager() {} int NebdMetaFileManager::UpdateMetaFile(const FileRecordMap& fileRecords) { - // 构建json - Json::Value volumes; - for (const auto& record : fileRecords) { - Json::Value volume; - volume[kFileName] = record.second.fileName; - volume[kFd] = record.second.fd; - if (record.second.fileInstance) { - for (const auto item : record.second.fileInstance->addition) { - volume[item.first] = item.second; - } - } - volumes.append(volume); - } - Json::Value root; - root[kVolumes] = volumes; + Json::Value root = parser_->ConvertFileRecordsToJson(fileRecords); int res = AtomicWriteFile(root); if (res != 0) { @@ -107,13 +93,36 @@ int NebdMetaFileManager::ListFileRecord(FileRecordMap* fileRecords) { return 0; } -int NebdMetaFileParser::Parse(const Json::Value& root, +int NebdMetaFileParser::Parse(Json::Value root, FileRecordMap* fileRecords) { + if (!fileRecords) { + LOG(ERROR) << "the argument fileRecords is null pointer"; + return -1; + } + fileRecords->clear(); + // 检验crc + if (root[kCRC].isNull()) { + LOG(ERROR) << "Parse json: " << root + << " fail, no crc"; + return -1; + } + uint32_t crcValue = root[kCRC].asUInt(); + root.removeMember(kCRC); + std::string jsonString = root.toStyledString(); + uint32_t crcCalc = nebd::common::CRC32(jsonString.c_str(), + jsonString.size()); + if (crcValue != crcCalc) { + LOG(ERROR) << "Parse json: " << root + << " fail, crc not match"; + return -1; + } + + // 没有volume字段 const auto& volumes = root[kVolumes]; if (volumes.isNull()) { - LOG(ERROR) << "No volumes in json: " << root; - return -1; + LOG(WARNING) << "No volumes in json: " << root; + return 0; } for (const auto& volume : volumes) { @@ -159,5 +168,29 @@ int NebdMetaFileParser::Parse(const Json::Value& root, return 0; } +Json::Value NebdMetaFileParser::ConvertFileRecordsToJson( + const FileRecordMap& fileRecords) { + Json::Value volumes; + for (const auto& record : fileRecords) { + Json::Value volume; + volume[kFileName] = record.second.fileName; + volume[kFd] = record.second.fd; + if (record.second.fileInstance) { + for (const auto item : record.second.fileInstance->addition) { + volume[item.first] = item.second; + } + } + volumes.append(volume); + } + Json::Value root; + root[kVolumes] = volumes; + + // 计算crc + std::string jsonString = root.toStyledString(); + uint32_t crc = nebd::common::CRC32(jsonString.c_str(), jsonString.size()); + root[kCRC] = crc; + return root; +} + } // namespace server } // namespace nebd diff --git a/src/part2/metafile_manager.h b/src/part2/metafile_manager.h index e89051583c..ae06b758c1 100644 --- a/src/part2/metafile_manager.h +++ b/src/part2/metafile_manager.h @@ -17,6 +17,7 @@ #include // NOLINT #include "src/common/posix_wrapper.h" +#include "src/common/crc32.h" #include "src/part2/define.h" namespace nebd { @@ -28,11 +29,13 @@ using FileRecordMap = std::unordered_map; const char kVolumes[] = "volumes"; const char kFileName[] = "filename"; const char kFd[] = "fd"; +const char kCRC[] = "crc"; class NebdMetaFileParser { public: - int Parse(const Json::Value& root, + int Parse(Json::Value root, FileRecordMap* fileRecords); + Json::Value ConvertFileRecordsToJson(const FileRecordMap& fileRecords); }; class NebdMetaFileManager { diff --git a/tests/common/crc32_test.cpp b/tests/common/crc32_test.cpp new file mode 100644 index 0000000000..2a4a4a93b5 --- /dev/null +++ b/tests/common/crc32_test.cpp @@ -0,0 +1,81 @@ +/* + * Project: curve + * Created Date: Saturday December 29th 2018 + * Author: yangyaokai + * Copyright (c) 2018 netease + */ + +#include + +#include "src/common/crc32.h" + +namespace nebd { +namespace common { + +TEST(Crc32TEST, BasicTest) { + char buf1[10]; + ::memset(buf1, 0, 10); + char buf2[10]; + ::memset(buf2, 1, 10); + char buf3[20]; + ::memset(buf3, 0, 20); + char buf4[10]; + ::memset(buf4, 0, 10); + uint32_t crc1 = CRC32(buf1, sizeof(buf1)); + uint32_t crc2 = CRC32(buf2, sizeof(buf2)); + uint32_t crc3 = CRC32(buf3, sizeof(buf3)); + uint32_t crc4 = CRC32(buf4, sizeof(buf4)); + ASSERT_EQ(crc1, crc4); + ASSERT_NE(crc1, crc2); + ASSERT_NE(crc1, crc3); + ASSERT_NE(crc2, crc3); +} + +TEST(Crc32TEST, StandardResults) { + // From rfc3720 section B.4. + char buf[32]; + + memset(buf, 0, sizeof(buf)); + ASSERT_EQ(0x8a9136aaU, CRC32(buf, sizeof(buf))); + + memset(buf, 0xff, sizeof(buf)); + ASSERT_EQ(0x62a8ab43U, CRC32(buf, sizeof(buf))); + + for (int i = 0; i < 32; i++) { + buf[i] = i; + } + ASSERT_EQ(0x46dd794eU, CRC32(buf, sizeof(buf))); + + for (int i = 0; i < 32; i++) { + buf[i] = 31 - i; + } + ASSERT_EQ(0x113fdb5cU, CRC32(buf, sizeof(buf))); + + unsigned char data[48] = { + 0x01, 0xc0, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x14, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x14, + 0x00, 0x00, 0x00, 0x18, + 0x28, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }; + ASSERT_EQ(0xd9963a56, CRC32(reinterpret_cast(data), sizeof(data))); +} + +TEST(Crc32TEST, Values) { + ASSERT_NE(CRC32("a", 1), CRC32("foo", 3)); +} + +TEST(Crc32TEST, Extend) { + ASSERT_EQ(CRC32("hello world", 11), + CRC32(CRC32("hello ", 6), "world", 5)); +} + +} // namespace common +} // namespace nebd diff --git a/tests/part2/metafile_manager_test.cpp b/tests/part2/metafile_manager_test.cpp index bb072bf704..1a49360eaa 100644 --- a/tests/part2/metafile_manager_test.cpp +++ b/tests/part2/metafile_manager_test.cpp @@ -21,6 +21,13 @@ namespace server { const char metaPath[] = "/tmp/nebd-test-metafilemanager.meta"; +void FillCrc(Json::Value* root) { + std::string jsonString = root->toStyledString(); + uint32_t crc = nebd::common::CRC32(jsonString.c_str(), + jsonString.size()); + (*root)[kCRC] = crc; +} + class MetaFileManagerTest : public ::testing::Test { protected: void SetUp() override { @@ -90,10 +97,12 @@ TEST_F(MetaFileManagerTest, error) { ASSERT_EQ(-1, metaFileManager.UpdateMetaFile(records)); // rename失败 + NebdMetaFileParser parser; + Json::Value root = parser.ConvertFileRecordsToJson(records); EXPECT_CALL(*wrapper_, open(_, _, _)) .WillOnce(Return(1)); EXPECT_CALL(*wrapper_, pwrite(_, _, _, _)) - .WillOnce(Return(77)); + .WillOnce(Return(root.toStyledString().size())); EXPECT_CALL(*wrapper_, close(_)) .Times(1); EXPECT_CALL(*wrapper_, rename(_, _)) @@ -115,16 +124,33 @@ TEST(MetaFileParserTest, Parse) { volume[kFileName] = "cbd:volume2"; volume[kFd] = 2; root[kVolumes] = volumes; + FillCrc(&root); ASSERT_EQ(0, parser.Parse(root, &records)); - // 文件格式不正确 - root.clear(); + // 空指针 + ASSERT_EQ(-1, parser.Parse(root, nullptr)); + + // crc校验不正确 + root[kCRC] = root[kCRC].asUInt() + 1; ASSERT_EQ(-1, parser.Parse(root, &records)); - // 没有volume字段 - root["key"] = "value"; + // 没有crc字段 + root.removeMember(kCRC); ASSERT_EQ(-1, parser.Parse(root, &records)); + // 没有volumes字段或volumes字段是null,不应该报错 + root.clear(); + root["key"] = "value"; + FillCrc(&root); + ASSERT_EQ(0, parser.Parse(root, &records)); + ASSERT_TRUE(records.empty()); + root.clear(); + Json::Value value; + root[kVolumes] = value; + FillCrc(&root); + ASSERT_EQ(0, parser.Parse(root, &records)); + ASSERT_TRUE(records.empty()); + // 记录中没有filename volume.clear(); volumes.clear(); @@ -132,6 +158,7 @@ TEST(MetaFileParserTest, Parse) { volume[kFd] = 1234; volumes.append(volume); root[kVolumes] = volumes; + FillCrc(&root); ASSERT_EQ(-1, parser.Parse(root, &records)); // 记录中没有fd @@ -141,6 +168,7 @@ TEST(MetaFileParserTest, Parse) { volume[kFileName] = "cbd:volume2"; volumes.append(volume); root[kVolumes] = volumes; + FillCrc(&root); ASSERT_EQ(-1, parser.Parse(root, &records)); // 文件名格式不对 @@ -151,6 +179,7 @@ TEST(MetaFileParserTest, Parse) { volume[kFd] = 1234; volumes.append(volume); root[kVolumes] = volumes; + FillCrc(&root); ASSERT_EQ(-1, parser.Parse(root, &records)); } From 13978b89a7a34a2e992cd8593ccab5b6d67e80d1 Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Thu, 20 Feb 2020 15:11:17 +0800 Subject: [PATCH 37/79] fix update timestamp Change-Id: I95930d33e3dd255e1ea2b86bbd8c3794be5b180e --- src/part2/heartbeat_manager.cpp | 2 +- src/part2/heartbeat_manager.h | 2 +- src/part2/heartbeat_service.cpp | 4 ++-- tests/part2/heartbeat_manager_unittest.cpp | 9 +++++++++ tests/part2/heartbeat_service_test.cpp | 6 +++--- tests/part2/mock_heartbeat_manager.h | 2 +- 6 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/part2/heartbeat_manager.cpp b/src/part2/heartbeat_manager.cpp index 74b3d49bb0..6e02f7af60 100644 --- a/src/part2/heartbeat_manager.cpp +++ b/src/part2/heartbeat_manager.cpp @@ -38,7 +38,7 @@ int HeartbeatManager::Fini() { return 0; } -int HeartbeatManager::UpdateFileTimestamp(int fd, uint64_t timestamp) { +bool HeartbeatManager::UpdateFileTimestamp(int fd, uint64_t timestamp) { return fileManager_->GetRecordManager()->UpdateFileTimestamp(fd, timestamp); } diff --git a/src/part2/heartbeat_manager.h b/src/part2/heartbeat_manager.h index 0bfef6e231..ce79e7fdfc 100644 --- a/src/part2/heartbeat_manager.h +++ b/src/part2/heartbeat_manager.h @@ -45,7 +45,7 @@ class HeartbeatManager { virtual int Fini(); // part2收到心跳后，会通过该接口更新心跳中包含的文件在内存中记录的时间戳 // 心跳检测线程会根据该时间戳判断是否需要关闭文件 - virtual int UpdateFileTimestamp(int fd, uint64_t timestamp); + virtual bool UpdateFileTimestamp(int fd, uint64_t timestamp); private: // 心跳检测线程的函数执行体 diff --git a/src/part2/heartbeat_service.cpp b/src/part2/heartbeat_service.cpp index 60718f8081..ece6969b6f 100644 --- a/src/part2/heartbeat_service.cpp +++ b/src/part2/heartbeat_service.cpp @@ -23,8 +23,8 @@ void NebdHeartbeatServiceImpl::KeepAlive( uint64_t curTime = TimeUtility::GetTimeofDayMs(); for (int i = 0; i < request->info_size(); ++i) { const auto& info = request->info(i); - int res = heartbeatManager_->UpdateFileTimestamp(info.fd(), curTime); - if (res != 0) { + bool res = heartbeatManager_->UpdateFileTimestamp(info.fd(), curTime); + if (!res) { LOG(WARNING) << "Update file timestamp fail, fd: " << info.fd() << ", name: " << info.name(); ok = false; diff --git a/tests/part2/heartbeat_manager_unittest.cpp b/tests/part2/heartbeat_manager_unittest.cpp index 1db21e3858..83db89227d 100644 --- a/tests/part2/heartbeat_manager_unittest.cpp +++ b/tests/part2/heartbeat_manager_unittest.cpp @@ -98,6 +98,15 @@ TEST_F(HeartbeatManagerTest, CheckTimeoutTest) { ASSERT_EQ(heartbeatManager_->Fini(), 0); } +TEST_F(HeartbeatManagerTest, UpdateTimeStampTest) { + EXPECT_CALL(*fileRecordManager_, UpdateFileTimestamp(_, _)) + .WillOnce(Return(true)); + ASSERT_TRUE(heartbeatManager_->UpdateFileTimestamp(1, 100)); + + EXPECT_CALL(*fileRecordManager_, UpdateFileTimestamp(_, _)) + .WillOnce(Return(false)); + ASSERT_FALSE(heartbeatManager_->UpdateFileTimestamp(1, 100)); +} } // namespace server } // namespace nebd diff --git a/tests/part2/heartbeat_service_test.cpp b/tests/part2/heartbeat_service_test.cpp index d3898bcfda..474d4f01d3 100644 --- a/tests/part2/heartbeat_service_test.cpp +++ b/tests/part2/heartbeat_service_test.cpp @@ -54,7 +54,7 @@ TEST_F(HeartbeatServiceTest, KeepAlive) { // 正常情况 EXPECT_CALL(*heartbeatManager_, UpdateFileTimestamp(_, _)) .Times(3) - .WillRepeatedly(Return(0)); + .WillRepeatedly(Return(true)); stub.KeepAlive(&cntl, &request, &response, nullptr); ASSERT_FALSE(cntl.Failed()); ASSERT_EQ(nebd::client::RetCode::kOK, response.retcode()); @@ -62,8 +62,8 @@ TEST_F(HeartbeatServiceTest, KeepAlive) { // 有文件更新时间戳失败 EXPECT_CALL(*heartbeatManager_, UpdateFileTimestamp(_, _)) .Times(3) - .WillOnce(Return(-1)) - .WillRepeatedly(Return(0)); + .WillOnce(Return(false)) + .WillRepeatedly(Return(true)); cntl.Reset(); stub.KeepAlive(&cntl, &request, &response, nullptr); ASSERT_FALSE(cntl.Failed()); diff --git a/tests/part2/mock_heartbeat_manager.h b/tests/part2/mock_heartbeat_manager.h index 8ba405edd2..748cd21f93 100644 --- a/tests/part2/mock_heartbeat_manager.h +++ b/tests/part2/mock_heartbeat_manager.h @@ -19,7 +19,7 @@ class MockHeartbeatManager : public HeartbeatManager { ~MockHeartbeatManager() {} MOCK_METHOD0(Init, int()); MOCK_METHOD0(Fini, int()); - MOCK_METHOD2(UpdateFileTimestamp, int(int, uint64_t)); + MOCK_METHOD2(UpdateFileTimestamp, bool(int, uint64_t)); }; } // namespace server From 52a19943cdbd0c616ce90adf00afcc19270cbde9 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Thu, 20 Feb 2020 09:59:27 +0800 Subject: [PATCH 38/79] =?UTF-8?q?part1=20=E4=BF=9D=E5=AD=98=E6=97=A5?= =?UTF-8?q?=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I1754c7acfa296c900caf1f0dd56a68c24b1cbc0a --- WORKSPACE | 2 +- etc/nebd/nebd-client.conf | 5 ++++- nebd-package/var/log/nebd/.gitkeep | 0 nebd-package/var/run/nebd-client/.gitkeep | 0 src/part1/async_request_closure.cpp | 11 +++++----- src/part1/heartbeat_manager.cpp | 3 ++- src/part1/nebd_client.cpp | 25 +++++++++++++++++++++++ src/part1/nebd_client.h | 9 ++++++++ src/part1/nebd_common.h | 9 +++++++- tests/part1/nebd-client.conf | 5 ++++- tests/part1/nebd_client_unittest.cpp | 18 +++++++++++++++- 11 files changed, 76 insertions(+), 11 deletions(-) create mode 100644 nebd-package/var/log/nebd/.gitkeep create mode 100644 nebd-package/var/run/nebd-client/.gitkeep diff --git a/WORKSPACE b/WORKSPACE index ea0f0d7067..1a66fc44d0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -98,7 +98,7 @@ bind( git_repository( name = "com_netease_storage_gerrit_curve_curve", remote = "http://gerrit.storage.netease.com/curve", - commit = "45e8f8887580d367644d3624d7ebd033ebc69503", + commit = "3fe29a07d2d14e848559c0b918568d73eb7048bd", ) bind( diff --git a/etc/nebd/nebd-client.conf b/etc/nebd/nebd-client.conf index 07b434389a..60ee5c8677 100644 --- a/etc/nebd/nebd-client.conf +++ b/etc/nebd/nebd-client.conf @@ -18,4 +18,7 @@ request.rpcHealthCheckIntervalS=1 # heartbeat间隔 heartbeat.intervalS=5 # heartbeat rpc超时时间 -heartbeat.rpcTimeoutMs=500 \ No newline at end of file +heartbeat.rpcTimeoutMs=500 + +# 日志路径 +log.path=/var/log/nebd diff --git a/nebd-package/var/log/nebd/.gitkeep b/nebd-package/var/log/nebd/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nebd-package/var/run/nebd-client/.gitkeep b/nebd-package/var/run/nebd-client/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/part1/async_request_closure.cpp b/src/part1/async_request_closure.cpp index 6f8821cf79..c340f37b2a 100644 --- a/src/part1/async_request_closure.cpp +++ b/src/part1/async_request_closure.cpp @@ -19,7 +19,8 @@ namespace client { void AsyncRequestClosure::Run() { std::unique_ptr selfGuard(this); - if (cntl.Failed()) { + bool isCntlFailed = cntl.Failed(); + if (isCntlFailed) { ++aioCtx->retryCount; int64_t sleepUs = GetRpcRetryIntervalUs(aioCtx->retryCount); LOG(WARNING) << OpTypeToString(aioCtx->op) << " rpc failed" @@ -57,16 +58,16 @@ void AsyncRequestClosure::Run() { } int64_t AsyncRequestClosure::GetRpcRetryIntervalUs(int64_t retryCount) const { - if (retryCount == 0) { - return requestOption_.rpcRetryIntervalUs; - } - // EHOSTDOWN: 找不到可用的server。 // server可能停止服务了，也可能正在退出中(返回了ELOGOFF) if (cntl.ErrorCode() == EHOSTDOWN) { return requestOption_.rpcHostDownRetryIntervalUs; } + if (retryCount <= 1) { + return requestOption_.rpcRetryIntervalUs; + } + return std::max( requestOption_.rpcRetryIntervalUs, std::min(requestOption_.rpcRetryIntervalUs * retryCount, diff --git a/src/part1/heartbeat_manager.cpp b/src/part1/heartbeat_manager.cpp index 9e7f7e6115..66e8dcc034 100644 --- a/src/part1/heartbeat_manager.cpp +++ b/src/part1/heartbeat_manager.cpp @@ -91,7 +91,8 @@ void HeartbeatManager::SendHeartBeat() { stub.KeepAlive(&cntl, &request, &response, nullptr); - if (cntl.Failed()) { + bool isCntlFailed = cntl.Failed(); + if (isCntlFailed) { LOG(WARNING) << "Heartbeat request failed, error = " << cntl.ErrorText() << ", log id = " << cntl.log_id(); diff --git a/src/part1/nebd_client.cpp b/src/part1/nebd_client.cpp index 62b81ea61a..11af01b1e7 100644 --- a/src/part1/nebd_client.cpp +++ b/src/part1/nebd_client.cpp @@ -54,6 +54,9 @@ int NebdClient::Init(const char* confpath) { return -1; } + // init glog + static LoggerGuard logger(option_.logOption); + HeartbeatOption heartbeatOption; ret = InitHeartBeatOption(&conf, &heartbeatOption); if (ret != 0) { @@ -455,6 +458,13 @@ int NebdClient::InitNebdClientOption(Configuration* conf) { return -1; } option_.requestOption = requestOption; + + ret = conf->GetStringValue("log.path", &option_.logOption.logPath); + if (!ret) { + LOG(ERROR) << "Load log.path failed"; + return -1; + } + return 0; } @@ -534,5 +544,20 @@ std::string NebdClient::ReplaceSlash(const std::string& str) { return ret; } +LoggerGuard::LoggerGuard(const LogOption& logOption) { + InitLogger(logOption); +} + +LoggerGuard::~LoggerGuard() { + google::ShutdownGoogleLogging(); +} + +void LoggerGuard::InitLogger(const LogOption& logOption) { + static const char* kProcessName = "nebd-client"; + + FLAGS_log_dir = logOption.logPath; + google::InitGoogleLogging(kProcessName); +} + } // namespace client } // namespace nebd diff --git a/src/part1/nebd_client.h b/src/part1/nebd_client.h index d9d1fc39d1..a1fc1ae14b 100644 --- a/src/part1/nebd_client.h +++ b/src/part1/nebd_client.h @@ -29,6 +29,15 @@ using RpcTask = std::function; using nebd::common::Configuration; +class LoggerGuard { + public: + explicit LoggerGuard(const LogOption& logOption); + ~LoggerGuard(); + + private: + void InitLogger(const LogOption& logOption); +}; + class NebdClient { public: static NebdClient &GetInstance() { diff --git a/src/part1/nebd_common.h b/src/part1/nebd_common.h index 57a2cde51f..dda087bb6d 100644 --- a/src/part1/nebd_common.h +++ b/src/part1/nebd_common.h @@ -24,6 +24,12 @@ struct RequestOption { int64_t rpcHealthCheckIntervalS; }; +// 日志配置项 +struct LogOption { + // 日志存放目录 + std::string logPath; +}; + // nebd client配置项 struct NebdClientOption { // part2 socket file address @@ -32,9 +38,10 @@ struct NebdClientOption { std::string fileLockPath; // rpc request配置项 RequestOption requestOption; + // 日志配置项 + LogOption logOption; }; - // heartbeat配置项 struct HeartbeatOption { // part2 socket file address diff --git a/tests/part1/nebd-client.conf b/tests/part1/nebd-client.conf index 2c2203af55..31eaa10998 100644 --- a/tests/part1/nebd-client.conf +++ b/tests/part1/nebd-client.conf @@ -1,5 +1,5 @@ # part2 socket file address -nebdserver.serverAddress=/tmp/nebd-client-test.sock +nebdserver.serverAddress=./nebd-client-test.sock # 文件锁路径 metacache.fileLockPath=/tmp # 同步rpc的最大重试次数 @@ -17,3 +17,6 @@ request.rpcHealthCheckIntervalS=1 heartbeat.intervalS=5 # heartbeat rpc超时时间 heartbeat.rpcTimeoutMs=500 + +# 日志路径 +log.path=. diff --git a/tests/part1/nebd_client_unittest.cpp b/tests/part1/nebd_client_unittest.cpp index 9e89dd3691..f8edbf90f2 100644 --- a/tests/part1/nebd_client_unittest.cpp +++ b/tests/part1/nebd_client_unittest.cpp @@ -24,7 +24,8 @@ namespace nebd { namespace client { const char* kFileName = "nebd-test-filename"; -const char* kNebdServerTestAddress = "/tmp/nebd-client-test.sock"; +const char* kFileNameWithSlash = "nebd-test-filenae//filename"; +const char* kNebdServerTestAddress = "./nebd-client-test.sock"; const char* kNebdClientConf = "tests/part1/nebd-client.conf"; const int64_t kFileSize = 10LL * 1024 * 1024 * 1024; const int64_t kBufSize = 1024; @@ -268,6 +269,10 @@ TEST_F(NebdFileClientTest, CommonTest) { int fd = Open4Nebd(kFileName); ASSERT_GE(fd, 0); + int fd2 = Open4Nebd(kFileNameWithSlash); + ASSERT_GE(fd2, 0); + ASSERT_EQ(0, Close4Nebd(fd2)); + ASSERT_EQ(0, Extend4Nebd(fd, kFileSize)); ASSERT_EQ(kFileSize, GetFileSize4Nebd(fd)); ASSERT_EQ(kFileSize, GetInfo4Nebd(fd)); @@ -389,6 +394,17 @@ TEST_F(NebdFileClientTest, ResponseFailTest) { ASSERT_EQ(-1, Open4Nebd(kFileName)); } + { + CloseFileResponse response; + response.set_retcode(RetCode::kNoOK); + EXPECT_CALL(mockService, CloseFile(_, _, _, _)) + .Times(1) + .WillOnce(DoAll( + SetArgPointee<2>(response), + Invoke(MockClientFunc))); // NOLINT + ASSERT_EQ(0, Close4Nebd(0)); + } + { ResizeResponse response; response.set_retcode(RetCode::kNoOK); From dea5f7038786cd47edc64ebb30f7b68a8d2d10a4 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Thu, 20 Feb 2020 22:03:14 +0800 Subject: [PATCH 39/79] fix qemu exit bug Change-Id: I43ac3e18813ac45bc5862edbcc2a59587bbea91c --- src/part1/heartbeat_manager.cpp | 5 ++++- src/part1/nebd_client.cpp | 4 ++++ src/part1/nebd_client.h | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/part1/heartbeat_manager.cpp b/src/part1/heartbeat_manager.cpp index 66e8dcc034..69f80c5ad2 100644 --- a/src/part1/heartbeat_manager.cpp +++ b/src/part1/heartbeat_manager.cpp @@ -57,7 +57,10 @@ void HeartbeatManager::Run() { void HeartbeatManager::Stop() { running_ = false; sleeper_.interrupt(); - heartbeatThread_.join(); + + if (heartbeatThread_.joinable()) { + heartbeatThread_.join(); + } LOG(INFO) << "Connection Manager stopped success"; } diff --git a/src/part1/nebd_client.cpp b/src/part1/nebd_client.cpp index 11af01b1e7..adc2957869 100644 --- a/src/part1/nebd_client.cpp +++ b/src/part1/nebd_client.cpp @@ -92,6 +92,10 @@ void NebdClient::Uninit() { LOG(INFO) << "NebdClient uninit success."; } +NebdClient::~NebdClient() { + Uninit(); +} + int NebdClient::Open(const char* filename) { // 加文件锁 std::string fileLockName = diff --git a/src/part1/nebd_client.h b/src/part1/nebd_client.h index a1fc1ae14b..9237d54e96 100644 --- a/src/part1/nebd_client.h +++ b/src/part1/nebd_client.h @@ -45,7 +45,7 @@ class NebdClient { return client; } - ~NebdClient() = default; + ~NebdClient(); /** * @brief 初始化nebd，仅在第一次调用的时候真正执行初始化逻辑 From 511996fd174449cff226dff4a1cbf6d1ede205cf Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Fri, 21 Feb 2020 14:20:13 +0800 Subject: [PATCH 40/79] fix write caused core Change-Id: If7050c6fd2e7f2bf88c4eff89f7ad7dce88b532e --- src/part2/file_service.cpp | 19 +++++++++++++++++-- tests/part2/file_service_unittest.cpp | 25 ++++++++++++++++++++----- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/part2/file_service.cpp b/src/part2/file_service.cpp index 31a9605964..7860e9f3dd 100644 --- a/src/part2/file_service.cpp +++ b/src/part2/file_service.cpp @@ -49,6 +49,7 @@ void NebdFileServiceCallback(NebdServerAioContext* context) { } else { response->set_retcode(RetCode::kOK); } + delete[] reinterpret_cast(context->buf); break; } case LIBAIO_OP::LIBAIO_OP_FLUSH: @@ -115,9 +116,21 @@ void NebdFileServiceImpl::Write( aioContext->size = request->size(); aioContext->op = LIBAIO_OP::LIBAIO_OP_WRITE; aioContext->cb = NebdFileServiceCallback; + brpc::Controller* cntl = dynamic_cast(cntl_base); - aioContext->buf = - const_cast(cntl->request_attachment().to_string().c_str()); + aioContext->buf = new char[aioContext->size]; + size_t copySize = + cntl->request_attachment().copy_to(aioContext->buf, aioContext->size); + if (copySize != aioContext->size) { + LOG(ERROR) << "Copy attachment failed. " + << "fd: " << request->fd() + << ", offset: " << request->offset() + << ", size: " << request->size() + << ", copy size: " << copySize; + delete[] reinterpret_cast(aioContext->buf); + return; + } + aioContext->response = response; aioContext->done = done; aioContext->cntl = cntl_base; @@ -128,6 +141,7 @@ void NebdFileServiceImpl::Write( << ", offset: " << request->offset() << ", size: " << request->size() << ", return code: " << rc; + delete[] reinterpret_cast(aioContext->buf); } else { doneGuard.release(); } @@ -158,6 +172,7 @@ void NebdFileServiceImpl::Read( << ", offset: " << request->offset() << ", size: " << request->size() << ", return code: " << rc; + delete[] reinterpret_cast(aioContext->buf); } else { doneGuard.release(); } diff --git a/tests/part2/file_service_unittest.cpp b/tests/part2/file_service_unittest.cpp index 771f15da75..86ef4d7b84 100644 --- a/tests/part2/file_service_unittest.cpp +++ b/tests/part2/file_service_unittest.cpp @@ -27,6 +27,8 @@ using ::testing::ReturnArg; using ::testing::ElementsAre; using ::testing::SetArgPointee; using ::testing::SetArrayArgument; +using ::testing::SaveArgPointee; +using ::testing::SaveArg; using google::protobuf::RpcController; using google::protobuf::Closure; @@ -90,21 +92,24 @@ TEST_F(FileServiceTest, OpenTest) { TEST_F(FileServiceTest, WriteTest) { int fd = 1; uint64_t offset = 0; - uint64_t size = 4096; + const uint64_t kSize = 2 * 4096; brpc::Controller cntl; - char buf[4096]; - cntl.request_attachment().append(buf, 4096); + char buf[kSize]; + memset(buf, 1, kSize); + cntl.request_attachment().append(buf, kSize); nebd::client::WriteRequest request; request.set_fd(fd); request.set_offset(offset); - request.set_size(size); + request.set_size(kSize); nebd::client::WriteResponse response; FileServiceTestClosure done; + NebdServerAioContext* aioCtx; // write success EXPECT_CALL(*fileManager_, AioWrite(fd, NotNull())) - .WillOnce(Return(0)); + .WillOnce(DoAll(SaveArg<1>(&aioCtx), Return(0))); fileService_->Write(&cntl, &request, &response, &done); + ASSERT_EQ(0, strncmp((char*)aioCtx->buf, buf, kSize)); ASSERT_FALSE(done.IsRunned()); // write failed @@ -114,6 +119,16 @@ TEST_F(FileServiceTest, WriteTest) { fileService_->Write(&cntl, &request, &response, &done); ASSERT_EQ(response.retcode(), RetCode::kNoOK); ASSERT_TRUE(done.IsRunned()); + + // attachment size not equal request size + done.Reset(); + cntl.request_attachment().clear(); + cntl.request_attachment().append(buf, 4096); + EXPECT_CALL(*fileManager_, AioWrite(_, _)) + .Times(0); + fileService_->Write(&cntl, &request, &response, &done); + ASSERT_EQ(response.retcode(), RetCode::kNoOK); + ASSERT_TRUE(done.IsRunned()); } TEST_F(FileServiceTest, ReadTest) { From bb76aff8992655a1be8b234275c7f8b0d7097e1c Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Fri, 21 Feb 2020 15:21:30 +0800 Subject: [PATCH 41/79] fix write failed after open again Change-Id: I2cd86c6e45f22a6f98fdb4702c0d965d276de6db --- src/part2/file_manager.cpp | 11 +++++++++-- src/part2/file_manager.h | 4 ++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/part2/file_manager.cpp b/src/part2/file_manager.cpp index d65ed10fdf..35bf6fbff3 100644 --- a/src/part2/file_manager.cpp +++ b/src/part2/file_manager.cpp @@ -374,7 +374,7 @@ int NebdFileManager::ProcessRequest(int fd, ProcessTask task) { NebdFileRecord fileRecord; bool getSuccess = fileRecordManager_->GetRecord(fd, &fileRecord); if (!getSuccess) { - LOG(WARNING) << "File record not exist, fd: " << fd; + LOG(ERROR) << "File record not exist, fd: " << fd; return -1; } @@ -385,12 +385,19 @@ int NebdFileManager::ProcessRequest(int fd, ProcessTask task) { if (fileRecord.status != NebdFileStatus::OPENED) { int ret = OpenInternal(fileRecord.fileName); if (ret != fd) { - LOG(WARNING) << "Get opened file failed. " + LOG(ERROR) << "Get opened file failed. " << "filename: " << fileRecord.fileName << ", fd: " << fd << ", ret: " << ret; return -1; } + // 重新open后要获取新的record + getSuccess = fileRecordManager_->GetRecord(fd, &fileRecord); + if (!getSuccess) { + LOG(ERROR) << "File record not exist, fd: " << fd; + return -1; + } + done->SetFileRecord(fileRecord); } int ret = task(dynamic_cast(doneGuard.release())); diff --git a/src/part2/file_manager.h b/src/part2/file_manager.h index 911ae633ab..e6792ccd60 100644 --- a/src/part2/file_manager.h +++ b/src/part2/file_manager.h @@ -63,6 +63,10 @@ class NebdProcessClosure : public Closure { return fileRecord_; } + void SetFileRecord(const NebdFileRecord& fileRecord) { + fileRecord_ = fileRecord; + } + private: NebdFileRecord fileRecord_; Closure* done_; From d53c4ebc4d9fc4158a44c2c7141b2ca730d0cbe4 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Fri, 21 Feb 2020 21:54:30 +0800 Subject: [PATCH 42/79] update curve commit id Change-Id: I6119e5e20d77743ad694cd9a5044f30eeec500aa --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 1a66fc44d0..1de2a79005 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -98,7 +98,7 @@ bind( git_repository( name = "com_netease_storage_gerrit_curve_curve", remote = "http://gerrit.storage.netease.com/curve", - commit = "3fe29a07d2d14e848559c0b918568d73eb7048bd", + commit = "d18bedeccee47944eb9ff06c25090c898952a58f", ) bind( From c37d7d4afa6fedd56fb56561df0f55e448f7d3d8 Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Sat, 22 Feb 2020 13:59:20 +0800 Subject: [PATCH 43/79] optimize nebd Change-Id: Ibf7825d32881e3e7ca8d5fe01c07bae697c875fd --- mk-deb.sh | 4 ++-- src/part1/async_request_closure.cpp | 18 +++++++++--------- src/part1/nebd_client.cpp | 12 +++--------- src/part1/nebd_client.h | 11 ++--------- src/part2/file_manager.cpp | 5 ++++- 5 files changed, 20 insertions(+), 30 deletions(-) diff --git a/mk-deb.sh b/mk-deb.sh index 489f2e4406..5aeb5d581b 100755 --- a/mk-deb.sh +++ b/mk-deb.sh @@ -1,6 +1,5 @@ #!/bin/bash dir=`pwd` -set -x # step1 清楚生成的目录和文件 bazel clean rm -rf *deb @@ -20,13 +19,14 @@ then else bazel build ... --copt -DHAVE_ZLIB=1 --copt -O2 -s --define=with_glog=true \ --define=libunwind=true --copt -DGFLAGS_NS=google --copt \ - -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX + -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX if [ $? -ne 0 ] then echo "build phase1 failed" exit fi fi +bazel shutdown # step3 创建临时目录，拷贝二进制、lib库和配置模板 mkdir build diff --git a/src/part1/async_request_closure.cpp b/src/part1/async_request_closure.cpp index c340f37b2a..97043064da 100644 --- a/src/part1/async_request_closure.cpp +++ b/src/part1/async_request_closure.cpp @@ -23,12 +23,13 @@ void AsyncRequestClosure::Run() { if (isCntlFailed) { ++aioCtx->retryCount; int64_t sleepUs = GetRpcRetryIntervalUs(aioCtx->retryCount); - LOG(WARNING) << OpTypeToString(aioCtx->op) << " rpc failed" - << ", error = " << cntl.ErrorText() - << ", fd = " << fd - << ", log id = " << cntl.log_id() - << ", retryCount = " << aioCtx->retryCount - << ", sleep " << (sleepUs / 1000) << " ms"; + LOG_EVERY_SECOND(WARNING) + << OpTypeToString(aioCtx->op) << " rpc failed" + << ", error = " << cntl.ErrorText() + << ", fd = " << fd + << ", log id = " << cntl.log_id() + << ", retryCount = " << aioCtx->retryCount + << ", sleep " << (sleepUs / 1000) << " ms"; bthread_usleep(sleepUs); Retry(); } else { @@ -38,9 +39,8 @@ void AsyncRequestClosure::Run() { // 读请求复制数据 if (aioCtx->op == LIBAIO_OP::LIBAIO_OP_READ) { - memcpy(aioCtx->buf, - cntl.response_attachment().to_string().c_str(), - cntl.response_attachment().size()); + cntl.response_attachment().copy_to( + aioCtx->buf, cntl.response_attachment().size()); } aioCtx->ret = 0; diff --git a/src/part1/nebd_client.cpp b/src/part1/nebd_client.cpp index adc2957869..e4ae86b78c 100644 --- a/src/part1/nebd_client.cpp +++ b/src/part1/nebd_client.cpp @@ -55,7 +55,7 @@ int NebdClient::Init(const char* confpath) { } // init glog - static LoggerGuard logger(option_.logOption); + InitLogger(option_.logOption); HeartbeatOption heartbeatOption; ret = InitHeartBeatOption(&conf, &heartbeatOption); @@ -90,6 +90,7 @@ int NebdClient::Init(const char* confpath) { void NebdClient::Uninit() { heartbeatMgr_->Stop(); LOG(INFO) << "NebdClient uninit success."; + google::ShutdownGoogleLogging(); } NebdClient::~NebdClient() { @@ -548,15 +549,8 @@ std::string NebdClient::ReplaceSlash(const std::string& str) { return ret; } -LoggerGuard::LoggerGuard(const LogOption& logOption) { - InitLogger(logOption); -} - -LoggerGuard::~LoggerGuard() { - google::ShutdownGoogleLogging(); -} -void LoggerGuard::InitLogger(const LogOption& logOption) { +void NebdClient::InitLogger(const LogOption& logOption) { static const char* kProcessName = "nebd-client"; FLAGS_log_dir = logOption.logPath; diff --git a/src/part1/nebd_client.h b/src/part1/nebd_client.h index 9237d54e96..7db1cbae95 100644 --- a/src/part1/nebd_client.h +++ b/src/part1/nebd_client.h @@ -29,15 +29,6 @@ using RpcTask = std::function; using nebd::common::Configuration; -class LoggerGuard { - public: - explicit LoggerGuard(const LogOption& logOption); - ~LoggerGuard(); - - private: - void InitLogger(const LogOption& logOption); -}; - class NebdClient { public: static NebdClient &GetInstance() { @@ -144,6 +135,8 @@ class NebdClient { int InitChannel(); + void InitLogger(const LogOption& logOption); + /** * @brief 替换字符串中的 '/' 为 '+' * diff --git a/src/part2/file_manager.cpp b/src/part2/file_manager.cpp index 35bf6fbff3..9cb36acdf6 100644 --- a/src/part2/file_manager.cpp +++ b/src/part2/file_manager.cpp @@ -253,11 +253,14 @@ int NebdFileManager::InvalidCache(int fd) { int NebdFileManager::OpenInternal(const std::string& fileName, bool create) { // 同名文件open需要互斥 - NameLockGuard(nameLock_, fileName); + NameLockGuard openGuard(nameLock_, fileName); NebdFileRecord fileRecord; bool getSuccess = fileRecordManager_->GetRecord(fileName, &fileRecord); if (getSuccess && fileRecord.status == NebdFileStatus::OPENED) { + LOG(WARNING) << "File is already opened. " + << "filename: " << fileName + << "fd: " << fileRecord.fd; return fileRecord.fd; } From 18b5f817642f5bc770ae04e4f22c54abebb881fd Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Mon, 24 Feb 2020 15:39:01 +0800 Subject: [PATCH 44/79] add request_attachment zero copy Change-Id: I889d4daaab1b0cad695b36cb9b156955d6222464 --- src/part1/nebd_client.cpp | 5 ++++- src/part2/file_service.cpp | 11 ++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/part1/nebd_client.cpp b/src/part1/nebd_client.cpp index e4ae86b78c..11cb012800 100644 --- a/src/part1/nebd_client.cpp +++ b/src/part1/nebd_client.cpp @@ -303,6 +303,8 @@ int NebdClient::AioRead(int fd, NebdClientAioContext* aioctx) { return 0; } +static void EmptyDeleter(void* m) {} + int NebdClient::AioWrite(int fd, NebdClientAioContext* aioctx) { nebd::client::NebdFileService_Stub stub(&channel_); nebd::client::WriteRequest request; @@ -315,7 +317,8 @@ int NebdClient::AioWrite(int fd, NebdClientAioContext* aioctx) { done->cntl.set_timeout_ms(-1); done->cntl.set_log_id(logId_.fetch_add(1, std::memory_order_relaxed)); - done->cntl.request_attachment().append(aioctx->buf, aioctx->length); + done->cntl.request_attachment().append_user_data( + aioctx->buf, aioctx->length, EmptyDeleter); stub.Write(&done->cntl, &request, &done->response, done); return 0; diff --git a/src/part2/file_service.cpp b/src/part2/file_service.cpp index 7860e9f3dd..5eac07d8bc 100644 --- a/src/part2/file_service.cpp +++ b/src/part2/file_service.cpp @@ -15,6 +15,10 @@ namespace server { using nebd::client::RetCode; +static void AioReadDeleter(void* m) { + delete[] reinterpret_cast(m); +} + void NebdFileServiceCallback(NebdServerAioContext* context) { CHECK(context != nullptr); std::unique_ptr contextGuard(context); @@ -24,6 +28,9 @@ void NebdFileServiceCallback(NebdServerAioContext* context) { { nebd::client::ReadResponse* response = dynamic_cast(context->response); + butil::IOBuf readBuf; + readBuf.append_user_data( + context->buf, context->size, AioReadDeleter); if (context->ret < 0) { response->set_retcode(RetCode::kNoOK); LOG(ERROR) << "Read file failed. " @@ -31,11 +38,9 @@ void NebdFileServiceCallback(NebdServerAioContext* context) { } else { brpc::Controller* cntl = dynamic_cast(context->cntl); - cntl->response_attachment().append(context->buf, - context->size); + cntl->response_attachment().append(readBuf); response->set_retcode(RetCode::kOK); } - delete[] reinterpret_cast(context->buf); break; } case LIBAIO_OP::LIBAIO_OP_WRITE: From 2cc80fe5850167494f3dd34aacf175b9f38c60aa Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Tue, 25 Feb 2020 10:14:02 +0800 Subject: [PATCH 45/79] update curve commit id Change-Id: I1d3d5a3003f254c7d72096d2f3e5c8018ac5df17 --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 1de2a79005..4d77e01a04 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -98,7 +98,7 @@ bind( git_repository( name = "com_netease_storage_gerrit_curve_curve", remote = "http://gerrit.storage.netease.com/curve", - commit = "d18bedeccee47944eb9ff06c25090c898952a58f", + commit = "6faf02d716a13c4353603ca1c81d83887ef470ed", ) bind( From 19ef804f0202aef3f56e2a0e40c5e1b5af73b489 Mon Sep 17 00:00:00 2001 From: charisu Date: Tue, 25 Feb 2020 16:00:45 +0800 Subject: [PATCH 46/79] update curve commit id Change-Id: I24aef393704efe61868f3406c6dff4f6f6c4c4b7 --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 4d77e01a04..a3249cfa11 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -98,7 +98,7 @@ bind( git_repository( name = "com_netease_storage_gerrit_curve_curve", remote = "http://gerrit.storage.netease.com/curve", - commit = "6faf02d716a13c4353603ca1c81d83887ef470ed", + commit = "ecccc5a6028dccca4a0194fe489dd5817a95b9fa", ) bind( From 8d5800d0a6e730c648947c4c566c6fab6b929cc6 Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Wed, 26 Feb 2020 11:22:26 +0800 Subject: [PATCH 47/79] support jemalloc Change-Id: Ib6c76bf8b932ea638778a7b8195852a37495ce6b --- nebd-package/DEBIAN/control | 2 +- nebd-package/usr/bin/nebd-daemon | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/nebd-package/DEBIAN/control b/nebd-package/DEBIAN/control index 5dde3f4f01..87c41ee368 100644 --- a/nebd-package/DEBIAN/control +++ b/nebd-package/DEBIAN/control @@ -1,7 +1,7 @@ Package: nebd Section: Priority: optional -Depends: libunwind8 +Depends: libunwind8, libjemalloc1(= 3.6.0-9.1) Suggests: Architecture:amd64 Installed-Size: diff --git a/nebd-package/usr/bin/nebd-daemon b/nebd-package/usr/bin/nebd-daemon index 0bc8ffa807..ae0c067286 100755 --- a/nebd-package/usr/bin/nebd-daemon +++ b/nebd-package/usr/bin/nebd-daemon @@ -80,7 +80,15 @@ function start() { exit 1 fi - daemon --name nebd-server --core --inherit \ + jemallocpath=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 + # 检查jemalloc库文件 + if [ ! -f ${jemallocpath} ] + then + echo "Not found jemalloc library, Path is ${jemallocpath}" + exit 1 + fi + + LD_PRELOAD=${jemallocpath} daemon --name nebd-server --core --inherit \ --respawn --attempts 100 --delay 10 \ --pidfile ${pidFile} \ --errlog ${daemonLog} \ From ffb271e0ce0344dca385e6270c203960d9ebf4bb Mon Sep 17 00:00:00 2001 From: charisu Date: Wed, 26 Feb 2020 16:33:54 +0800 Subject: [PATCH 48/79] =?UTF-8?q?=E8=B0=83=E6=95=B4deamon=E9=81=87?= =?UTF-8?q?=E5=88=B0=E9=94=99=E8=AF=AF=E9=87=8D=E8=AF=95=E6=AC=A1=E6=95=B0?= =?UTF-8?q?=EF=BC=8C=E6=A3=80=E6=B5=8Bdaemon=E8=BF=90=E8=A1=8C=E6=98=AF?= =?UTF-8?q?=E5=90=A6=E9=81=87=E5=88=B0=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I5b09ae47575576121e62db823088efef60c68e93 --- nebd-package/usr/bin/nebd-daemon | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/nebd-package/usr/bin/nebd-daemon b/nebd-package/usr/bin/nebd-daemon index ae0c067286..60df75230e 100755 --- a/nebd-package/usr/bin/nebd-daemon +++ b/nebd-package/usr/bin/nebd-daemon @@ -88,12 +88,24 @@ function start() { exit 1 fi + # 记录运行daemon前的daemonLog的行数 + line1=`cat ${daemonLog} | wc -l` + LD_PRELOAD=${jemallocpath} daemon --name nebd-server --core --inherit \ - --respawn --attempts 100 --delay 10 \ + --respawn --attempts 10 --delay 10 \ --pidfile ${pidFile} \ --errlog ${daemonLog} \ --output ${consoleLog} \ -- ${bin} -confPath=${confPath} -log_dir=${logPath} -graceful_quit_on_sigterm=true -stderrthreshold=3 + + # sleep 1秒检测daemonLog是否有变化，如果有变化，说明启动遇到问题 + sleep 1 + line2=`cat ${daemonLog} | wc -l` + if [ $line1 != $line2 ] + then + echo "start nebd-server met error!" + stop + fi } # 停止daemon进程，且停止nebd-server From f298e4e0c44d8fa1894a64148eb0c1ea768c5883 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Thu, 27 Feb 2020 14:50:18 +0800 Subject: [PATCH 49/79] chang metadata and log folder Change-Id: I4039550b962e2850ea6b31829a060d7f3b372b85 --- etc/nebd/nebd-client.conf | 6 +++--- etc/nebd/nebd-server.conf | 4 ++-- nebd-package/DEBIAN/control | 2 +- nebd-package/DEBIAN/postinst | 11 +++++++++++ .../{var/log/nebd => data/log/nebd/client}/.gitkeep | 0 .../run/nebd-client => data/log/nebd/server}/.gitkeep | 0 nebd-package/data/nebd/lock/.gitkeep | 0 nebd-package/usr/bin/nebd-daemon | 9 +++++---- 8 files changed, 22 insertions(+), 10 deletions(-) create mode 100755 nebd-package/DEBIAN/postinst rename nebd-package/{var/log/nebd => data/log/nebd/client}/.gitkeep (100%) rename nebd-package/{var/run/nebd-client => data/log/nebd/server}/.gitkeep (100%) create mode 100644 nebd-package/data/nebd/lock/.gitkeep diff --git a/etc/nebd/nebd-client.conf b/etc/nebd/nebd-client.conf index 60ee5c8677..fe4b5a4052 100644 --- a/etc/nebd/nebd-client.conf +++ b/etc/nebd/nebd-client.conf @@ -1,8 +1,8 @@ # part2 socket file address -nebdserver.serverAddress=/var/run/nebd.sock +nebdserver.serverAddress=/data/nebd/nebd.sock # 文件锁路径 -metacache.fileLockPath=/var/run/nebd-client +metacache.fileLockPath=/data/nebd/lock # 同步rpc的最大重试次数 request.syncRpcMaxRetryTimes=50 @@ -21,4 +21,4 @@ heartbeat.intervalS=5 heartbeat.rpcTimeoutMs=500 # 日志路径 -log.path=/var/log/nebd +log.path=/data/log/nebd/client diff --git a/etc/nebd/nebd-server.conf b/etc/nebd/nebd-server.conf index 928927ee83..a3e7d4b40d 100644 --- a/etc/nebd/nebd-server.conf +++ b/etc/nebd/nebd-server.conf @@ -2,10 +2,10 @@ curveclient.confPath=/etc/curve/client.conf #brpc server监听端口 -listen.address=/var/run/nebd.sock +listen.address=/data/nebd/nebd.sock #元数据文件地址,包含文件名 -meta.file.path=/etc/nebd/nebdserver.meta +meta.file.path=/data/nebd/nebdserver.meta #心跳超时时间 heartbeat.timeout.sec=30 diff --git a/nebd-package/DEBIAN/control b/nebd-package/DEBIAN/control index 87c41ee368..6b82eae00c 100644 --- a/nebd-package/DEBIAN/control +++ b/nebd-package/DEBIAN/control @@ -7,4 +7,4 @@ Architecture:amd64 Installed-Size: Maintainer: nebd-dev Provides: -Description: +Description: nebd is a proxy between qemu and cbd/rbd diff --git a/nebd-package/DEBIAN/postinst b/nebd-package/DEBIAN/postinst new file mode 100755 index 0000000000..b9d046e8d3 --- /dev/null +++ b/nebd-package/DEBIAN/postinst @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +chown -R root:root /data/nebd +chown -R root:root /data/log/nebd + +chmod -R 777 /data/nebd +chmod -R 777 /data/log/nebd + +exit 0 diff --git a/nebd-package/var/log/nebd/.gitkeep b/nebd-package/data/log/nebd/client/.gitkeep similarity index 100% rename from nebd-package/var/log/nebd/.gitkeep rename to nebd-package/data/log/nebd/client/.gitkeep diff --git a/nebd-package/var/run/nebd-client/.gitkeep b/nebd-package/data/log/nebd/server/.gitkeep similarity index 100% rename from nebd-package/var/run/nebd-client/.gitkeep rename to nebd-package/data/log/nebd/server/.gitkeep diff --git a/nebd-package/data/nebd/lock/.gitkeep b/nebd-package/data/nebd/lock/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nebd-package/usr/bin/nebd-daemon b/nebd-package/usr/bin/nebd-daemon index 60df75230e..33abce0cc6 100755 --- a/nebd-package/usr/bin/nebd-daemon +++ b/nebd-package/usr/bin/nebd-daemon @@ -7,16 +7,17 @@ bin=/usr/bin/nebd-server confPath=/etc/nebd/nebd-server.conf # 日志文件路径 -logPath=${HOME} +baseLogPath=/data/log/nebd +logPath=${baseLogPath}/server # pidfile -pidFile=${HOME}/nebd-server.pid +pidFile=${baseLogPath}/nebd-server.pid # daemon log -daemonLog=${HOME}/nebd-server-daemon.log +daemonLog=${baseLogPath}/nebd-server-daemon.log # console output -consoleLog=${HOME}/nebd-server-console.log +consoleLog=${baseLogPath}/nebd-server-console.log # 启动nebd-server function start() { From 8a47f088d148182104d138ba9804e433706a0f8c Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Fri, 28 Feb 2020 13:43:14 +0800 Subject: [PATCH 50/79] move library to nebd path Change-Id: I0dfe06b5e71b1a709f6e6f8f4a611d62354b745d --- mk-deb.sh | 4 ++-- src/part1/BUILD | 7 ++++--- src/part2/BUILD | 13 +++++++------ 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/mk-deb.sh b/mk-deb.sh index 5aeb5d581b..a4cd517f2f 100755 --- a/mk-deb.sh +++ b/mk-deb.sh @@ -32,11 +32,11 @@ bazel shutdown mkdir build cp -r nebd-package build/ mkdir -p build/nebd-package/usr/bin -mkdir -p build/nebd-package/usr/lib +mkdir -p build/nebd-package/usr/lib/nebd for i in `find bazel-bin/|grep -w so|grep -v solib|grep -v params| grep -v test` do - cp -f $i build/nebd-package/usr/lib + cp -f $i build/nebd-package/usr/lib/nebd done cp bazel-bin/src/part2/nebd-server build/nebd-package/usr/bin diff --git a/src/part1/BUILD b/src/part1/BUILD index 269a100e19..6d09a32027 100644 --- a/src/part1/BUILD +++ b/src/part1/BUILD @@ -32,11 +32,12 @@ cc_library( visibility = ["//visibility:public"], deps = [ "//external:glog", - "//external:brpc", - "//src/common:nebd_common", - "//proto:client_cc_proto" + "//external:brpc", + "//src/common:nebd_common", + "//proto:client_cc_proto" ], copts = COPTS, + linkopts = ["-Wl,-rpath=/usr/lib/nebd"], ) diff --git a/src/part2/BUILD b/src/part2/BUILD index 67febb01f8..159b55c7ec 100644 --- a/src/part2/BUILD +++ b/src/part2/BUILD @@ -32,12 +32,12 @@ cc_library( visibility = ["//visibility:public"], deps = [ "//external:glog", - "//external:brpc", - "//external:json", - "//external:gflags", - "//external:curve", - "//src/common:nebd_common", - "//proto:client_cc_proto" + "//external:brpc", + "//external:json", + "//external:gflags", + "//external:curve", + "//src/common:nebd_common", + "//proto:client_cc_proto" ], copts = COPTS, ) @@ -52,6 +52,7 @@ cc_binary( "//src/part2:nebdserver", ], copts = COPTS, + linkopts = ["-Wl,-rpath=/usr/lib/nebd"], ) From e2a71f49b1376db300a74e74f0dcf69db777ad03 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Mon, 2 Mar 2020 09:48:47 +0800 Subject: [PATCH 51/79] update curve commit id Change-Id: Id37887f38cbba2efef9ea009257f479763635570 --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index a3249cfa11..e22ed65189 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -98,7 +98,7 @@ bind( git_repository( name = "com_netease_storage_gerrit_curve_curve", remote = "http://gerrit.storage.netease.com/curve", - commit = "ecccc5a6028dccca4a0194fe489dd5817a95b9fa", + commit = "b5568f122664be95bb44a3813a0a08160ec11968", ) bind( From e423eedf01e19d70f4b3a46e18daed179f3e28f4 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Mon, 2 Mar 2020 14:50:58 +0800 Subject: [PATCH 52/79] fix segv fault Change-Id: Ic6f728771658d91687c7f37999891e61f349d150 --- src/part1/heartbeat_manager.cpp | 11 +++++------ src/part1/heartbeat_manager.h | 4 +++- src/part1/nebd_client.cpp | 8 +++----- src/part1/nebd_client.h | 2 +- tests/part1/nebd_client_unittest.cpp | 11 +++++++++++ 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/part1/heartbeat_manager.cpp b/src/part1/heartbeat_manager.cpp index 69f80c5ad2..6ed523ce7b 100644 --- a/src/part1/heartbeat_manager.cpp +++ b/src/part1/heartbeat_manager.cpp @@ -55,14 +55,13 @@ void HeartbeatManager::Run() { } void HeartbeatManager::Stop() { - running_ = false; - sleeper_.interrupt(); - - if (heartbeatThread_.joinable()) { + if (running_ == true) { + running_ = false; + sleeper_.interrupt(); heartbeatThread_.join(); - } - LOG(INFO) << "Connection Manager stopped success"; + LOG(INFO) << "Connection Manager stopped success"; + } } void HeartbeatManager::SendHeartBeat() { diff --git a/src/part1/heartbeat_manager.h b/src/part1/heartbeat_manager.h index 3efd3f672a..b0deab5f13 100644 --- a/src/part1/heartbeat_manager.h +++ b/src/part1/heartbeat_manager.h @@ -27,7 +27,9 @@ class HeartbeatManager { public: explicit HeartbeatManager(std::shared_ptr metaCache); - ~HeartbeatManager() = default; + ~HeartbeatManager() { + Stop(); + } /** * @brief: 启动心跳线程 diff --git a/src/part1/nebd_client.cpp b/src/part1/nebd_client.cpp index 11cb012800..33fc75784f 100644 --- a/src/part1/nebd_client.cpp +++ b/src/part1/nebd_client.cpp @@ -88,15 +88,13 @@ int NebdClient::Init(const char* confpath) { } void NebdClient::Uninit() { - heartbeatMgr_->Stop(); + if (heartbeatMgr_ != nullptr) { + heartbeatMgr_->Stop(); + } LOG(INFO) << "NebdClient uninit success."; google::ShutdownGoogleLogging(); } -NebdClient::~NebdClient() { - Uninit(); -} - int NebdClient::Open(const char* filename) { // 加文件锁 std::string fileLockName = diff --git a/src/part1/nebd_client.h b/src/part1/nebd_client.h index 7db1cbae95..5a07ae1ad8 100644 --- a/src/part1/nebd_client.h +++ b/src/part1/nebd_client.h @@ -36,7 +36,7 @@ class NebdClient { return client; } - ~NebdClient(); + ~NebdClient() = default; /** * @brief 初始化nebd，仅在第一次调用的时候真正执行初始化逻辑 diff --git a/tests/part1/nebd_client_unittest.cpp b/tests/part1/nebd_client_unittest.cpp index f8edbf90f2..fd83c7264b 100644 --- a/tests/part1/nebd_client_unittest.cpp +++ b/tests/part1/nebd_client_unittest.cpp @@ -551,7 +551,18 @@ TEST_F(NebdFileClientTest, ResponseFailTest) { StopServer(); } +TEST_F(NebdFileClientTest, InitAndUninitTest) { + ASSERT_NO_FATAL_FAILURE(nebdClient.Uninit()); + AddFakeService(); + StartServer(); + + ASSERT_NO_FATAL_FAILURE(nebdClient.Init(kNebdClientConf)); + ASSERT_NO_FATAL_FAILURE(nebdClient.Uninit()); + ASSERT_NO_FATAL_FAILURE(nebdClient.Uninit()); + + StopServer(); +} } // namespace client } // namespace nebd From 88127d8fac819bf6207cd0b5a7630856881c27ad Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Mon, 2 Mar 2020 21:18:31 +0800 Subject: [PATCH 53/79] fix inconsistent file status Change-Id: I68b84ace1a3f06209cae8215bfebc143093d1b11 --- src/part2/file_manager.cpp | 9 ++++ tests/part2/file_manager_unittest.cpp | 63 +++++++++++++++------------ 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/src/part2/file_manager.cpp b/src/part2/file_manager.cpp index 9cb36acdf6..a6cf10b7c2 100644 --- a/src/part2/file_manager.cpp +++ b/src/part2/file_manager.cpp @@ -385,6 +385,15 @@ int NebdFileManager::ProcessRequest(int fd, ProcessTask task) { new (std::nothrow) NebdProcessClosure(fileRecord); brpc::ClosureGuard doneGuard(done); + // 此时文件可能被close了，与之前获取到的record不一致，因此需要重新获取 + // TODO(yyk) 这不是一种优雅的实现方法，后续重构 + getSuccess = fileRecordManager_->GetRecord(fd, &fileRecord); + if (!getSuccess) { + LOG(ERROR) << "File record not exist, fd: " << fd; + return -1; + } + done->SetFileRecord(fileRecord); + if (fileRecord.status != NebdFileStatus::OPENED) { int ret = OpenInternal(fileRecord.fileName); if (ret != fd) { diff --git a/tests/part2/file_manager_unittest.cpp b/tests/part2/file_manager_unittest.cpp index c61369eaf6..c0c4857600 100644 --- a/tests/part2/file_manager_unittest.cpp +++ b/tests/part2/file_manager_unittest.cpp @@ -54,6 +54,8 @@ class FileManagerTest : public ::testing::Test { NebdServerAioContext* aioContext_; }; +// TODO(yyk): 后续重构单测，消除重复代码 + TEST_F(FileManagerTest, OpenTest) { // open一个不存在的文件 EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) @@ -277,8 +279,8 @@ TEST_F(FileManagerTest, ExtendTest) { fileRecord.executor = executor_.get(); fileRecord.fileInstance = mockInstance_; EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillOnce(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, Extend(NotNull(), 4096)) .WillOnce(Return(0)); ASSERT_EQ(0, fileManager_->Extend(1, 4096)); @@ -312,8 +314,8 @@ TEST_F(FileManagerTest, ExtendFaileTest) { fileRecord.executor = executor_.get(); fileRecord.fileInstance = mockInstance_; EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillOnce(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, Extend(NotNull(), 4096)) .WillOnce(Return(-1)); ASSERT_EQ(-1, fileManager_->Extend(1, 4096)); @@ -356,8 +358,8 @@ TEST_F(FileManagerTest, GetInfoTest) { fileRecord.executor = executor_.get(); fileRecord.fileInstance = mockInstance_; EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillOnce(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, GetInfo(NotNull(), &fileInfo)) .WillOnce(Return(0)); ASSERT_EQ(0, fileManager_->GetInfo(1, &fileInfo)); @@ -392,8 +394,8 @@ TEST_F(FileManagerTest, GetInfoFaileTest) { fileRecord.executor = executor_.get(); fileRecord.fileInstance = mockInstance_; EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillOnce(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, GetInfo(NotNull(), &fileInfo)) .WillOnce(Return(-1)); ASSERT_EQ(-1, fileManager_->GetInfo(1, &fileInfo)); @@ -435,8 +437,8 @@ TEST_F(FileManagerTest, InvalidCacheTest) { fileRecord.executor = executor_.get(); fileRecord.fileInstance = mockInstance_; EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillOnce(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, InvalidCache(NotNull())) .WillOnce(Return(0)); ASSERT_EQ(0, fileManager_->InvalidCache(1)); @@ -470,8 +472,8 @@ TEST_F(FileManagerTest, InvalidCacheFaileTest) { fileRecord.executor = executor_.get(); fileRecord.fileInstance = mockInstance_; EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillOnce(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, InvalidCache(NotNull())) .WillOnce(Return(-1)); ASSERT_EQ(-1, fileManager_->InvalidCache(1)); @@ -513,8 +515,8 @@ TEST_F(FileManagerTest, AioReadTest) { fileRecord.executor = executor_.get(); fileRecord.fileInstance = mockInstance_; EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillOnce(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) .WillOnce(Return(0)); ASSERT_EQ(0, fileManager_->AioRead(1, aioContext_)); @@ -555,8 +557,13 @@ TEST_F(FileManagerTest, AioReadFaileTest) { fileRecord.executor = executor_.get(); fileRecord.fileInstance = mockInstance_; EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillOnce(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); + .WillOnce(DoAll(SetArgPointee<1>(fileRecord), Return(true))) + .WillOnce(Return(false)); + ASSERT_EQ(-1, fileManager_->AioRead(1, aioContext_)); + + EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) .WillOnce(Return(-1)); ASSERT_EQ(-1, fileManager_->AioRead(1, aioContext_)); @@ -598,8 +605,8 @@ TEST_F(FileManagerTest, AioWriteTest) { fileRecord.executor = executor_.get(); fileRecord.fileInstance = mockInstance_; EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillOnce(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) .WillOnce(Return(0)); ASSERT_EQ(0, fileManager_->AioWrite(1, aioContext_)); @@ -640,8 +647,8 @@ TEST_F(FileManagerTest, AioWriteFaileTest) { fileRecord.executor = executor_.get(); fileRecord.fileInstance = mockInstance_; EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillOnce(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) .WillOnce(Return(-1)); ASSERT_EQ(-1, fileManager_->AioWrite(1, aioContext_)); @@ -683,8 +690,8 @@ TEST_F(FileManagerTest, DiscardTest) { fileRecord.executor = executor_.get(); fileRecord.fileInstance = mockInstance_; EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillOnce(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) .WillOnce(Return(0)); ASSERT_EQ(0, fileManager_->Discard(1, aioContext_)); @@ -725,8 +732,8 @@ TEST_F(FileManagerTest, DiscardFaileTest) { fileRecord.executor = executor_.get(); fileRecord.fileInstance = mockInstance_; EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillOnce(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) .WillOnce(Return(-1)); ASSERT_EQ(-1, fileManager_->Discard(1, aioContext_)); @@ -768,8 +775,8 @@ TEST_F(FileManagerTest, FlushTest) { fileRecord.executor = executor_.get(); fileRecord.fileInstance = mockInstance_; EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillOnce(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) .WillOnce(Return(0)); ASSERT_EQ(0, fileManager_->Flush(1, aioContext_)); @@ -810,8 +817,8 @@ TEST_F(FileManagerTest, FlushFaileTest) { fileRecord.executor = executor_.get(); fileRecord.fileInstance = mockInstance_; EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillOnce(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); + .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), + Return(true))); EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) .WillOnce(Return(-1)); ASSERT_EQ(-1, fileManager_->Flush(1, aioContext_)); From 0a255325d99d57e7fd01dc92bd3c74bbe244d1fd Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Tue, 3 Mar 2020 11:28:39 +0800 Subject: [PATCH 54/79] =?UTF-8?q?nebd-daemon=E5=88=A4=E6=96=ADroot?= =?UTF-8?q?=E6=9D=83=E9=99=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I5d3f2e8b2a415564ac879c506fc834524f61506e --- nebd-package/usr/bin/nebd-daemon | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nebd-package/usr/bin/nebd-daemon b/nebd-package/usr/bin/nebd-daemon index 33abce0cc6..7375c6145f 100755 --- a/nebd-package/usr/bin/nebd-daemon +++ b/nebd-package/usr/bin/nebd-daemon @@ -1,5 +1,12 @@ #!/bin/bash +# 检查脚本执行是否具有root权限 +if [[ $(id -u) -ne 0 ]] +then + echo "Please run as root" + exit 1 +fi + # nebd-server路径 bin=/usr/bin/nebd-server From 022c6db32c53f12acc5b2a3cf4542fae85b32ae4 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Tue, 3 Mar 2020 18:01:44 +0800 Subject: [PATCH 55/79] add rpcMaxDelayHealthCheckIntervalMs Change-Id: Ib6ba8234be92f1a97bce890cea1f77f473b87604 --- etc/nebd/nebd-client.conf | 2 + src/part1/nebd_client.cpp | 80 +++++------ src/part1/nebd_common.h | 2 + tests/part1/BUILD | 16 +++ tests/part1/nebd-client-aio-test.conf | 19 --- tests/part1/nebd-client.conf | 22 --- tests/part1/nebd_client_unittest.cpp | 21 ++- tests/part1/nebd_lib_unittest.cpp | 194 ++++++++++++++++++++++++++ tests/utils/BUILD | 12 ++ tests/utils/config_generator.h | 67 +++++++++ 10 files changed, 345 insertions(+), 90 deletions(-) delete mode 100644 tests/part1/nebd-client-aio-test.conf delete mode 100644 tests/part1/nebd-client.conf create mode 100644 tests/part1/nebd_lib_unittest.cpp create mode 100644 tests/utils/BUILD create mode 100644 tests/utils/config_generator.h diff --git a/etc/nebd/nebd-client.conf b/etc/nebd/nebd-client.conf index fe4b5a4052..c0c02c37b8 100644 --- a/etc/nebd/nebd-client.conf +++ b/etc/nebd/nebd-client.conf @@ -14,6 +14,8 @@ request.rpcRetryMaxIntervalUs=64000000 request.rpcHostDownRetryIntervalUs=10000 # brpc的健康检查周期时间，单位s request.rpcHealthCheckIntervalS=1 +# brpc从rpc失败到进行健康检查的最大时间间隔，单位ms +request.rpcMaxDelayHealthCheckIntervalMs=100 # heartbeat间隔 heartbeat.intervalS=5 diff --git a/src/part1/nebd_client.cpp b/src/part1/nebd_client.cpp index 33fc75784f..3737d27144 100644 --- a/src/part1/nebd_client.cpp +++ b/src/part1/nebd_client.cpp @@ -19,6 +19,8 @@ #include "src/part1/async_request_closure.h" #include "src/common/configuration.h" +#define RETURN_IF_FALSE(val) if (val == false) { return -1; } + // 修改brpc的health_check_interval参数，这个参数用来控制健康检查的周期 // ## 健康检查 // 连接断开的server会被暂时隔离而不会被负载均衡算法选中，brpc会定期连接被隔离的server，以检查他们是否恢复正常，间隔由参数-health_check_interval控制: // NOLINT @@ -28,6 +30,7 @@ // 一旦server被连接上，它会恢复为可用状态。如果在隔离过程中，server从命名服务中删除了，brpc也会停止连接尝试。 // NOLINT namespace brpc { DECLARE_int32(health_check_interval); + DECLARE_int32(circuit_breaker_max_isolation_duration_ms); } // namespace brpc namespace nebd { @@ -415,61 +418,51 @@ int NebdClient::InitNebdClientOption(Configuration* conf) { bool ret = false; ret = conf->GetStringValue("nebdserver.serverAddress", &option_.serverAddress); - if (!ret) { - LOG(ERROR) << "Load nebdserver.serverAddress failed"; - return -1; - } + LOG_IF(ERROR, ret != true) << "Load nebdserver.serverAddress failed"; + RETURN_IF_FALSE(ret); ret = conf->GetStringValue("metacache.fileLockPath", &option_.fileLockPath); - if (!ret) { - LOG(ERROR) << "Load metacache.fileLockPath failed"; - return -1; - } + LOG_IF(ERROR, ret != true) << "Load metacache.fileLockPath failed"; + RETURN_IF_FALSE(ret); RequestOption requestOption; ret = conf->GetInt64Value("request.syncRpcMaxRetryTimes", &requestOption.syncRpcMaxRetryTimes); - if (!ret) { - LOG(ERROR) << "Load request.syncRpcMaxRetryTimes failed"; - return -1; - } + LOG_IF(ERROR, ret != true) << "Load request.syncRpcMaxRetryTimes failed"; + RETURN_IF_FALSE(ret); ret = conf->GetInt64Value("request.rpcRetryIntervalUs", &requestOption.rpcRetryIntervalUs); - if (!ret) { - LOG(ERROR) << "Load request.rpcRetryIntervalUs failed"; - return -1; - } + LOG_IF(ERROR, ret != true) << "Load request.rpcRetryIntervalUs failed"; + RETURN_IF_FALSE(ret); ret = conf->GetInt64Value("request.rpcRetryMaxIntervalUs", &requestOption.rpcRetryMaxIntervalUs); - if (!ret) { - LOG(ERROR) << "Load request.rpcRetryMaxIntervalUs failed"; - return -1; - } + LOG_IF(ERROR, ret != true) << "Load request.rpcRetryMaxIntervalUs failed"; + RETURN_IF_FALSE(ret); ret = conf->GetInt64Value("request.rpcHostDownRetryIntervalUs", &requestOption.rpcHostDownRetryIntervalUs); - if (!ret) { - LOG(ERROR) << "Load request.rpcHostDownRetryIntervalUs failed"; - return -1; - } + LOG_IF(ERROR, ret != true) << "Load request.rpcHostDownRetryIntervalUs failed"; // NOLINT + RETURN_IF_FALSE(ret); ret = conf->GetInt64Value("request.rpcHealthCheckIntervalS", &requestOption.rpcHealthCheckIntervalS); - if (!ret) { - LOG(ERROR) << "Load request.rpcHealthCheckIntervalS failed"; - return -1; - } + LOG_IF(ERROR, ret != true) << "Load request.rpcHealthCheckIntervalS failed"; + RETURN_IF_FALSE(ret); + + ret = conf->GetInt64Value("request.rpcMaxDelayHealthCheckIntervalMs", + &requestOption.rpcMaxDelayHealthCheckIntervalMs); + LOG_IF(ERROR, ret != true) << "Load request.rpcMaxDelayHealthCheckIntervalMs failed"; // NOLINT + RETURN_IF_FALSE(ret); + option_.requestOption = requestOption; ret = conf->GetStringValue("log.path", &option_.logOption.logPath); - if (!ret) { - LOG(ERROR) << "Load log.path failed"; - return -1; - } + LOG_IF(ERROR, ret != true) << "Load log.path failed"; + RETURN_IF_FALSE(ret); return 0; } @@ -478,30 +471,27 @@ int NebdClient::InitHeartBeatOption(Configuration* conf, HeartbeatOption* heartbeatOption) { bool ret = conf->GetInt64Value("heartbeat.intervalS", &heartbeatOption->intervalS); - if (!ret) { - LOG(ERROR) << "Load heartbeat.intervalS failed"; - return -1; - } + LOG_IF(ERROR, ret != true) << "Load heartbeat.intervalS failed"; + RETURN_IF_FALSE(ret); ret = conf->GetInt64Value("heartbeat.rpcTimeoutMs", &heartbeatOption->rpcTimeoutMs); - if (!ret) { - LOG(ERROR) << "Load heartbeat.rpcTimeoutMs failed"; - return -1; - } + LOG_IF(ERROR, ret != true) << "Load heartbeat.rpcTimeoutMs failed"; + RETURN_IF_FALSE(ret); ret = conf->GetStringValue("nebdserver.serverAddress", &heartbeatOption->serverAddress); - if (!ret) { - LOG(ERROR) << "Load nebdserver.serverAddress failed"; - return -1; - } + LOG_IF(ERROR, ret != true) << "Load nebdserver.serverAddress failed"; + RETURN_IF_FALSE(ret); + return 0; } int NebdClient::InitChannel() { brpc::FLAGS_health_check_interval = - option_.requestOption.rpcHealthCheckIntervalS; + option_.requestOption.rpcHealthCheckIntervalS; + brpc::FLAGS_circuit_breaker_max_isolation_duration_ms = + option_.requestOption.rpcMaxDelayHealthCheckIntervalMs; int ret = channel_.InitWithSockFile( option_.serverAddress.c_str(), nullptr); if (ret != 0) { diff --git a/src/part1/nebd_common.h b/src/part1/nebd_common.h index dda087bb6d..14eb18273d 100644 --- a/src/part1/nebd_common.h +++ b/src/part1/nebd_common.h @@ -22,6 +22,8 @@ struct RequestOption { int64_t rpcHostDownRetryIntervalUs; // brpc的健康检查周期时间 int64_t rpcHealthCheckIntervalS; + // brpc从rpc失败到进行健康检查的最大时间间隔 + int64_t rpcMaxDelayHealthCheckIntervalMs; }; // 日志配置项 diff --git a/tests/part1/BUILD b/tests/part1/BUILD index 8bfa8a96b8..66b76be340 100644 --- a/tests/part1/BUILD +++ b/tests/part1/BUILD @@ -32,6 +32,22 @@ cc_binary( "//external:gflags", "//src/part1:nebdclient", "//tests/part1:fake_lib", + "//tests/utils:test_utils_lib", + "@com_google_googletest//:gtest_main", + ], + linkstatic = False, +) + +cc_binary( + name = "nebd_lib_unittest", + srcs = glob([ + "nebd_lib_unittest.cpp", + ]), + deps = [ + "//external:gflags", + "//src/part1:nebdclient", + "//tests/part1:fake_lib", + "//tests/utils:test_utils_lib", "@com_google_googletest//:gtest_main", ], linkstatic = False, diff --git a/tests/part1/nebd-client-aio-test.conf b/tests/part1/nebd-client-aio-test.conf deleted file mode 100644 index a06a25f651..0000000000 --- a/tests/part1/nebd-client-aio-test.conf +++ /dev/null @@ -1,19 +0,0 @@ -# part2 socket file address -nebdServerAddress=./nebd.aiotest.sock -# 文件锁路径 -fileLockPath=./tests -# 同步rpc的最大重试次数 -syncRpcMaxRetryTimes=10 -# rpc请求的重试间隔 -rpcRetryIntervalUs=100000 -# rpc请求的最大重试间隔 -rpcRetryMaxIntervalUs=64000000 -# rpc hostdown情况下的重试时间 -rpcHostDownRetryIntervalUs=10000 -# brpc的健康检查周期时间，单位s -rpcHealthCheckIntervalS=1 - -# heartbeat间隔 -heartbeatIntervalS=5 -# heartbeat rpc超时时间 -heartbeatRpcTimeoutMs=500 \ No newline at end of file diff --git a/tests/part1/nebd-client.conf b/tests/part1/nebd-client.conf deleted file mode 100644 index 31eaa10998..0000000000 --- a/tests/part1/nebd-client.conf +++ /dev/null @@ -1,22 +0,0 @@ -# part2 socket file address -nebdserver.serverAddress=./nebd-client-test.sock -# 文件锁路径 -metacache.fileLockPath=/tmp -# 同步rpc的最大重试次数 -request.syncRpcMaxRetryTimes=10 -# rpc请求的重试间隔 -request.rpcRetryIntervalUs=100000 -# rpc请求的最大重试间隔 -request.rpcRetryMaxIntervalUs=64000000 -# rpc hostdown情况下的重试时间 -request.rpcHostDownRetryIntervalUs=10000 -# brpc的健康检查周期时间，单位s -request.rpcHealthCheckIntervalS=1 - -# heartbeat间隔 -heartbeat.intervalS=5 -# heartbeat rpc超时时间 -heartbeat.rpcTimeoutMs=500 - -# 日志路径 -log.path=. diff --git a/tests/part1/nebd_client_unittest.cpp b/tests/part1/nebd_client_unittest.cpp index fd83c7264b..b4d69fef0a 100644 --- a/tests/part1/nebd_client_unittest.cpp +++ b/tests/part1/nebd_client_unittest.cpp @@ -19,17 +19,18 @@ #include "tests/part1/fake_file_service.h" #include "tests/part1/mock_file_service.h" - -namespace nebd { -namespace client { +#include "tests/utils/config_generator.h" const char* kFileName = "nebd-test-filename"; const char* kFileNameWithSlash = "nebd-test-filenae//filename"; const char* kNebdServerTestAddress = "./nebd-client-test.sock"; -const char* kNebdClientConf = "tests/part1/nebd-client.conf"; +const char* kNebdClientConf = "tests/part1/nebd-client-test.conf"; const int64_t kFileSize = 10LL * 1024 * 1024 * 1024; const int64_t kBufSize = 1024; +namespace nebd { +namespace client { + std::mutex mtx; std::condition_variable cond; std::atomic aioOpReturn{false}; @@ -569,6 +570,18 @@ TEST_F(NebdFileClientTest, InitAndUninitTest) { int main(int argc, char* argv[]) { + std::vector nebdConfig { + std::string("nebdserver.serverAddress=") + kNebdServerTestAddress, + std::string("metacache.fileLockPath=/tmp"), + std::string("request.syncRpcMaxRetryTimes=10"), + std::string("log.path=.") + }; + + nebd::common::NebdClientConfigGenerator generator; + generator.SetConfigPath(kNebdClientConf); + generator.SetConfigOptions(nebdConfig); + generator.Generate(); + ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/tests/part1/nebd_lib_unittest.cpp b/tests/part1/nebd_lib_unittest.cpp new file mode 100644 index 0000000000..867b6118aa --- /dev/null +++ b/tests/part1/nebd_lib_unittest.cpp @@ -0,0 +1,194 @@ +/** + * Project: nebd + * Create Date: 2020-03-03 + * Author: wuhanqing + * Copyright (c) 2020 netease + */ + +#include +#include +#include + +#include // NOLINT +#include // NOLINT +#include + +#include "src/part1/nebd_client.h" +#include "src/part1/libnebd.h" +#include "src/part1/libnebd_file.h" + +#include "tests/part1/fake_file_service.h" +#include "tests/part1/mock_file_service.h" + +#include "tests/utils/config_generator.h" + +const char* kFileName = "nebd-lib-test-filename"; +const char* kNebdServerTestAddress = "./nebd-lib-unittest.sock"; +const char* kNebdClientConf = "./tests/part1/nebd-lib-test.conf"; +const int64_t kFileSize = 10LL * 1024 * 1024 * 1024; +const int64_t kBufSize = 1024; + +namespace nebd { +namespace client { + +std::mutex mtx; +std::condition_variable cond; +std::atomic aioOpReturn{false}; + +void AioCallBack(NebdClientAioContext* ctx) { + ASSERT_EQ(0, ctx->ret); + ASSERT_EQ(0, ctx->retryCount); + std::lock_guard lk(mtx); + aioOpReturn = true; + cond.notify_one(); + delete ctx; +} + +class NebdLibTest : public ::testing::Test { + public: + void SetUp() override { + ::system("mkdir -p /etc/nebd"); + std::string copyCmd = "cp " + std::string(kNebdClientConf) + + " /etc/nebd/nebd-client.conf"; + LOG(INFO) << copyCmd; + int ret = ::system(copyCmd.c_str()); + ASSERT_EQ(0, ret) << "copy config file failed"; + } + + void TearDown() override {} + + void AddFakeService() { + ASSERT_EQ(0, server.AddService( + &fakeService, + brpc::SERVER_DOESNT_OWN_SERVICE)) << "Add service failed"; + } + + void StartServer(const std::string& address = kNebdServerTestAddress) { + ASSERT_EQ(0, server.StartAtSockFile( + address.c_str(), nullptr)) << "Start server failed"; + } + + void StopServer() { + server.Stop(0); + server.Join(); + } + + brpc::Server server; + FakeNebdFileService fakeService; + MockNebdFileService mockService; +}; + +TEST_F(NebdLibTest, CommonTest) { + AddFakeService(); + StartServer(); + + ASSERT_EQ(0, nebd_lib_init()); + ASSERT_EQ(0, nebd_lib_init()); + + int fd = nebd_lib_open(kFileName); + ASSERT_GE(fd, 0); + + ASSERT_EQ(0, nebd_lib_resize(fd, kFileSize)); + ASSERT_EQ(kFileSize, nebd_lib_filesize(fd)); + ASSERT_EQ(kFileSize, nebd_lib_getinfo(fd)); + ASSERT_EQ(0, nebd_lib_invalidcache(fd)); + + ASSERT_EQ(-1, nebd_lib_pread(fd, 0, 0, 0)); + ASSERT_EQ(-1, nebd_lib_pwrite(fd, 0, 0, 0)); + + char buffer[kBufSize]; + + { + NebdClientAioContext* ctx = new NebdClientAioContext(); + ctx->buf = buffer; + ctx->offset = 0; + ctx->length = kBufSize; + ctx->ret = 0; + ctx->op = LIBAIO_OP_WRITE; + ctx->cb = AioCallBack; + ctx->retryCount = 0; + + aioOpReturn = false; + ASSERT_EQ(0, AioWrite4Nebd(fd, ctx)); + std::unique_lock ulk(mtx); + cond.wait(ulk, []() { return aioOpReturn.load(); }); + ASSERT_TRUE(aioOpReturn.load()); + } + + { + NebdClientAioContext* ctx = new NebdClientAioContext(); + ctx->buf = buffer; + ctx->offset = 0; + ctx->length = kBufSize; + ctx->ret = 0; + ctx->op = LIBAIO_OP_READ; + ctx->cb = AioCallBack; + ctx->retryCount = 0; + + aioOpReturn = false; + ASSERT_EQ(0, AioRead4Nebd(fd, ctx)); + std::unique_lock ulk2(mtx); + cond.wait(ulk2, []() { return aioOpReturn.load(); }); + ASSERT_TRUE(aioOpReturn.load()); + } + + { + NebdClientAioContext* ctx = new NebdClientAioContext(); + ctx->buf = 0; + ctx->offset = 0; + ctx->length = kBufSize; + ctx->ret = 0; + ctx->op = LIBAIO_OP_DISCARD; + ctx->cb = AioCallBack; + ctx->retryCount = 0; + + aioOpReturn = false; + ASSERT_EQ(0, Discard4Nebd(fd, ctx)); + std::unique_lock ulk2(mtx); + cond.wait(ulk2, []() { return aioOpReturn.load(); }); + ASSERT_TRUE(aioOpReturn.load()); + } + + { + NebdClientAioContext* ctx = new NebdClientAioContext(); + ctx->buf = 0; + ctx->offset = 0; + ctx->length = kBufSize; + ctx->ret = 0; + ctx->op = LIBAIO_OP_FLUSH; + ctx->cb = AioCallBack; + ctx->retryCount = 0; + + aioOpReturn = false; + ASSERT_EQ(0, Flush4Nebd(fd, ctx)); + std::unique_lock ulk2(mtx); + cond.wait(ulk2, []() { return aioOpReturn.load(); }); + ASSERT_TRUE(aioOpReturn.load()); + } + + ASSERT_EQ(0, nebd_lib_close(fd)); + ASSERT_EQ(0, nebd_lib_uninit()); + ASSERT_EQ(0, nebd_lib_uninit()); + StopServer(); +} + +} // namespace client +} // namespace nebd + +int main(int argc, char* argv[]) { + std::vector nebdConfig { + std::string("nebdserver.serverAddress=") + kNebdServerTestAddress, + std::string("metacache.fileLockPath=/tmp"), + std::string("request.syncRpcMaxRetryTimes=10"), + std::string("log.path=.") + }; + + nebd::common::NebdClientConfigGenerator generator; + generator.SetConfigPath(kNebdClientConf); + generator.SetConfigOptions(nebdConfig); + generator.Generate(); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/tests/utils/BUILD b/tests/utils/BUILD new file mode 100644 index 0000000000..4a7ef2b16f --- /dev/null +++ b/tests/utils/BUILD @@ -0,0 +1,12 @@ +cc_library( + name = "test_utils_lib", + srcs = glob([ + "*.h", + "*.cpp" + ]), + hdrs = glob(["*.h"]), + deps = [ + "//src/common:nebd_common", + ], + visibility = ["//visibility:public"], +) \ No newline at end of file diff --git a/tests/utils/config_generator.h b/tests/utils/config_generator.h new file mode 100644 index 0000000000..38e34e1b26 --- /dev/null +++ b/tests/utils/config_generator.h @@ -0,0 +1,67 @@ +/** + * Project: nebd + * Create Date: 2020/03/04 + * Author: wuhanqing + * Copyright (c) 2020 netease + */ + +#ifndef TESTS_UTILS_CONFIG_GENERATOR_H_ +#define TESTS_UTILS_CONFIG_GENERATOR_H_ + +#include +#include + +#include "src/common/configuration.h" + +namespace nebd { +namespace common { + +static const char* kNebdClientConfigPath = "etc/nebd/nebd-client.conf"; +static const char* kNebdServerConfigPath = "etc/nebd/nebd-server.conf"; + +class NebdClientConfigGenerator { + public: + NebdClientConfigGenerator() { + LoadDefaultConfig(); + } + + void SetConfigPath(const std::string& configPath) { + configPath_ = configPath; + } + + void SetConfigOptions(const std::vector& options) { + for (const auto& opt : options) { + auto pos = opt.find("="); + std::string key = opt.substr(0, pos); + std::string value = opt.substr(pos + 1); + SetKV(key, value); + } + } + + bool Generate() { + if (configPath_.empty()) { + return false; + } + + conf_.SetConfigPath(configPath_); + return conf_.SaveConfig(); + } + + private: + void LoadDefaultConfig() { + conf_.SetConfigPath(kNebdClientConfigPath); + conf_.LoadConfig(); + } + + void SetKV(const std::string& key, const std::string& value) { + conf_.SetValue(key, value); + } + + std::string configPath_; + Configuration conf_; +}; + +} // namespace common +} // namespace nebd + +#endif // TESTS_UTILS_CONFIG_GENERATOR_H_ From 2a34475058e55ff2f4df93c0ba63450740514a80 Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Tue, 3 Mar 2020 23:25:16 +0800 Subject: [PATCH 56/79] part2 refactor Change-Id: I375dd8b12c7bd3b67d30ab69d32d058ee59bc662 --- WORKSPACE | 2 +- src/common/BUILD | 1 + src/common/rw_lock.h | 70 +- src/part2/define.h | 47 +- src/part2/file_entity.cpp | 343 ++++++ src/part2/file_entity.h | 216 ++++ src/part2/file_manager.cpp | 464 +++----- src/part2/file_manager.h | 110 +- src/part2/file_service.cpp | 5 + src/part2/filerecord_manager.cpp | 168 --- src/part2/filerecord_manager.h | 130 -- src/part2/heartbeat_manager.cpp | 37 +- src/part2/heartbeat_manager.h | 2 +- src/part2/metafile_manager.cpp | 142 ++- src/part2/metafile_manager.h | 55 +- src/part2/nebd_server.cpp | 22 +- src/part2/request_executor.h | 5 +- src/part2/request_executor_ceph.cpp | 2 +- src/part2/request_executor_ceph.h | 2 +- src/part2/request_executor_curve.cpp | 8 +- src/part2/request_executor_curve.h | 2 +- src/part2/util.cpp | 32 + src/part2/util.h | 6 + tests/common/rw_lock_test.cpp | 69 ++ tests/part2/BUILD | 13 - tests/part2/file_manager_unittest.cpp | 1113 ++++++------------ tests/part2/file_record_manager_unittest.cpp | 193 --- tests/part2/heartbeat_manager_unittest.cpp | 86 +- tests/part2/metafile_manager_test.cpp | 188 ++- tests/part2/mock_file_entity.h | 45 + tests/part2/mock_file_manager.h | 3 +- tests/part2/mock_filerecord_manager.h | 42 - tests/part2/mock_metafile_manager.h | 8 +- tests/part2/mock_request_executor.h | 4 +- tests/part2/test_request_executor_curve.cpp | 18 +- 35 files changed, 1718 insertions(+), 1935 deletions(-) create mode 100644 src/part2/file_entity.cpp create mode 100644 src/part2/file_entity.h delete mode 100644 src/part2/filerecord_manager.cpp delete mode 100644 src/part2/filerecord_manager.h delete mode 100644 tests/part2/file_record_manager_unittest.cpp create mode 100644 tests/part2/mock_file_entity.h delete mode 100644 tests/part2/mock_filerecord_manager.h diff --git a/WORKSPACE b/WORKSPACE index e22ed65189..4c89651957 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -71,7 +71,7 @@ bind( git_repository( name = "com_netease_storage_gerrit_curve_curve_brpc", remote = "http://gerrit.storage.netease.com/curve/curve-brpc", - commit = "8ed04de5b807b50e0691575916ec96621caad734", + commit = "8d5e3c085b38598b6c436999029ebdefa9301450", ) bind( diff --git a/src/common/BUILD b/src/common/BUILD index 88bce27388..fc82e087d8 100644 --- a/src/common/BUILD +++ b/src/common/BUILD @@ -9,6 +9,7 @@ cc_library( deps = [ "//external:glog", "//external:butil", + "//external:bthread", ], ) diff --git a/src/common/rw_lock.h b/src/common/rw_lock.h index 21ea645bee..0dadd4be4a 100644 --- a/src/common/rw_lock.h +++ b/src/common/rw_lock.h @@ -11,13 +11,25 @@ #include #include #include +#include #include "src/common/uncopyable.h" namespace nebd { namespace common { -class RWLock : public Uncopyable { +class RWLockBase : public Uncopyable { + public: + RWLockBase() {} + virtual ~RWLockBase() {} + virtual void WRLock() = 0; + virtual int TryWRLock() = 0; + virtual void RDLock() = 0; + virtual int TryRDLock() = 0; + virtual void Unlock() = 0; +}; + +class RWLock : public RWLockBase { public: RWLock() { pthread_rwlock_init(&rwlock_, nullptr); @@ -27,25 +39,25 @@ class RWLock : public Uncopyable { pthread_rwlock_destroy(&rwlock_); } - void WRLock() { + void WRLock() override { int ret = pthread_rwlock_wrlock(&rwlock_); CHECK(0 == ret) << "wlock failed: " << ret << ", " << strerror(ret); } - int TryWRLock() { + int TryWRLock() override { return pthread_rwlock_trywrlock(&rwlock_); } - void RDLock() { + void RDLock() override { int ret = pthread_rwlock_rdlock(&rwlock_); CHECK(0 == ret) << "rlock failed: " << ret << ", " << strerror(ret); } - int TryRDLock() { + int TryRDLock() override { return pthread_rwlock_tryrdlock(&rwlock_); } - void Unlock() { + void Unlock() override { pthread_rwlock_unlock(&rwlock_); } @@ -53,9 +65,47 @@ class RWLock : public Uncopyable { pthread_rwlock_t rwlock_; }; // RWLock class +class BthreadRWLock : public RWLockBase { + public: + BthreadRWLock() { + bthread_rwlock_init(&rwlock_, nullptr); + } + + ~BthreadRWLock() { + bthread_rwlock_destroy(&rwlock_); + } + + void WRLock() override { + int ret = bthread_rwlock_wrlock(&rwlock_); + CHECK(0 == ret) << "wlock failed: " << ret << ", " << strerror(ret); + } + + int TryWRLock() override { + // not support yet + return EINVAL; + } + + void RDLock() override { + int ret = bthread_rwlock_rdlock(&rwlock_); + CHECK(0 == ret) << "rlock failed: " << ret << ", " << strerror(ret); + } + + int TryRDLock() override { + // not support yet + return EINVAL; + } + + void Unlock() override { + bthread_rwlock_unlock(&rwlock_); + } + + private: + bthread_rwlock_t rwlock_; +}; // RWLock class + class ReadLockGuard : public Uncopyable { public: - explicit ReadLockGuard(RWLock &rwlock) : rwlock_(rwlock) { + explicit ReadLockGuard(RWLockBase &rwlock) : rwlock_(rwlock) { rwlock_.RDLock(); } @@ -64,12 +114,12 @@ class ReadLockGuard : public Uncopyable { } private: - RWLock &rwlock_; + RWLockBase &rwlock_; }; // ReadLockGuard class class WriteLockGuard : public Uncopyable { public: - explicit WriteLockGuard(RWLock &rwlock) : rwlock_(rwlock) { + explicit WriteLockGuard(RWLockBase &rwlock) : rwlock_(rwlock) { rwlock_.WRLock(); } @@ -78,7 +128,7 @@ class WriteLockGuard : public Uncopyable { } private: - RWLock &rwlock_; + RWLockBase &rwlock_; }; // WriteLockGuard class } // namespace common diff --git a/src/part2/define.h b/src/part2/define.h index 3fd76c7234..9dd039e748 100644 --- a/src/part2/define.h +++ b/src/part2/define.h @@ -12,6 +12,7 @@ #include #include #include +#include #include "src/common/rw_lock.h" @@ -33,11 +34,13 @@ enum class LIBAIO_OP { LIBAIO_OP_WRITE, LIBAIO_OP_DISCARD, LIBAIO_OP_FLUSH, + LIBAIO_OP_UNKNOWN, }; enum class NebdFileStatus { OPENED = 0, CLOSED = 1, + DESTROYED = 2, }; enum class NebdFileType { @@ -52,26 +55,6 @@ class NebdRequestExecutor; using NebdFileInstancePtr = std::shared_ptr; using RWLockPtr = std::shared_ptr; -// nebd server记录的文件信息内存结构 -struct NebdFileRecord { - // 文件读写锁，处理请求前加读锁，close文件的时候加写锁 - // 避免close时还有请求未处理完 - RWLockPtr rwLock = std::make_shared(); - // nebd server为该文件分配的唯一标识符 - int fd = 0; - // 文件名称 - std::string fileName = ""; - // 文件当前状态，opened表示文件已打开，closed表示文件已关闭 - NebdFileStatus status = NebdFileStatus::CLOSED; - // 该文件上一次收到心跳时的时间戳 - uint64_t timeStamp = 0; - // 文件在executor open时返回上下文信息，用于后续文件的请求处理 - NebdFileInstancePtr fileInstance = nullptr; - // 文件对应的executor的指针 - NebdRequestExecutor* executor = nullptr; -}; -using NebdFileRecordPtr = std::shared_ptr; - struct NebdServerAioContext; // nebd回调函数的类型 @@ -81,23 +64,23 @@ typedef void (*NebdAioCallBack)(struct NebdServerAioContext* context); // 记录请求的类型、参数、返回信息、rpc信息 struct NebdServerAioContext { // 请求的offset - off_t offset; + off_t offset = 0; // 请求的size - size_t size; + size_t size = 0; // 记录异步返回的返回值 - int ret; + int ret = -1; // 异步请求的类型，详见定义 - LIBAIO_OP op; + LIBAIO_OP op = LIBAIO_OP::LIBAIO_OP_UNKNOWN; // 异步请求结束时调用的回调函数 NebdAioCallBack cb; // 请求的buf - void* buf; + void* buf = nullptr; // rpc请求的相应内容 - Message* response; + Message* response = nullptr; // rpc请求的回调函数 - Closure *done; + Closure *done = nullptr; // rpc请求的controller - RpcController* cntl; + RpcController* cntl = nullptr; }; struct NebdFileInfo { @@ -109,6 +92,14 @@ struct NebdFileInfo { uint64_t num_objs; }; +using ExtendAttribute = std::map; +// nebd server 端文件持久化的元数据信息 +struct NebdFileMeta { + int fd; + std::string fileName; + ExtendAttribute xattr; +}; + // part2配置项 const char LISTENADDRESS[] = "listen.address"; const char METAFILEPATH[] = "meta.file.path"; diff --git a/src/part2/file_entity.cpp b/src/part2/file_entity.cpp new file mode 100644 index 0000000000..ce525cf2eb --- /dev/null +++ b/src/part2/file_entity.cpp @@ -0,0 +1,343 @@ +/* + * Project: nebd + * Created Date: Tuesday March 3rd 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#include +#include // NOLINT +#include + +#include "src/part2/file_entity.h" +#include "src/part2/util.h" + +namespace nebd { +namespace server { + +NebdFileEntity::NebdFileEntity() + : fd_(0) + , fileName_("") + , status_(NebdFileStatus::CLOSED) + , timeStamp_(0) + , fileInstance_(nullptr) + , executor_(nullptr) + , metaFileManager_(nullptr) {} + +NebdFileEntity::~NebdFileEntity() {} + +int NebdFileEntity::Init(const NebdFileEntityOption& option) { + fd_ = option.fd; + fileName_ = option.fileName; + metaFileManager_ = option.metaFileManager_; + + NebdFileType type = GetFileType(fileName_); + executor_ = NebdRequestExecutorFactory::GetExecutor(type); + if (executor_ == nullptr) { + LOG(ERROR) << "Init file failed, invalid filename. " + << "filename: " << fileName_; + return -1; + } + + return 0; +} + +int NebdFileEntity::Open() { + CHECK(executor_ != nullptr) << "file entity is not inited. " + << "filename: " << fileName_; + std::unique_lock lock(fileStatusMtx_); + if (status_ == NebdFileStatus::OPENED) { + LOG(WARNING) << "File is already opened. " + << "filename: " << fileName_ + << "fd: " << fd_; + return fd_; + } + + NebdFileInstancePtr fileInstance = executor_->Open(fileName_); + if (fileInstance == nullptr) { + LOG(ERROR) << "open file failed. " + << "filename: " << fileName_; + return -1; + } + + int ret = UpdateFileStatus(fileInstance); + if (ret != 0) { + executor_->Close(fileInstance.get()); + LOG(ERROR) << "Open file failed. " + << "filename: " << fileName_; + return -1; + } + LOG(INFO) << "Open file success. " + << "fd: " << fd_ + << ", filename: " << fileName_; + return fd_; +} + +int NebdFileEntity::Reopen(const ExtendAttribute& xattr) { + CHECK(executor_ != nullptr) << "file entity is not inited. " + << "filename: " << fileName_; + std::unique_lock lock(fileStatusMtx_); + NebdFileInstancePtr fileInstance = executor_->Reopen(fileName_, xattr); + if (fileInstance == nullptr) { + LOG(ERROR) << "Reopen file failed. " + << "filename: " << fileName_; + return -1; + } + + int ret = UpdateFileStatus(fileInstance); + if (ret != 0) { + executor_->Close(fileInstance.get()); + LOG(ERROR) << "Reopen file failed. " + << "filename: " << fileName_; + return -1; + } + LOG(INFO) << "Reopen file success. " + << "fd: " << fd_ + << ", filename: " << fileName_; + return fd_; +} + +int NebdFileEntity::Close(bool removeMeta) { + CHECK(executor_ != nullptr) << "file entity is not inited. " + << "filename: " << fileName_; + // 用于和其他用户请求互斥，避免文件被close后，请求发到后端导致返回失败 + WriteLockGuard writeLock(rwLock_); + // 这里的互斥锁是为了跟open请求互斥，以下情况可能导致close和open并发 + // part2重启，导致文件被reopen，然后由于超时，文件准备被close + // 此时用户发送了挂载卷请求对文件进行open + std::unique_lock lock(fileStatusMtx_); + if (status_ == NebdFileStatus::OPENED) { + int ret = executor_->Close(fileInstance_.get()); + if (ret < 0) { + LOG(ERROR) << "Close file failed. " + << "fd: " << fd_ + << ", filename: " << fileName_; + return -1; + } + status_ = NebdFileStatus::CLOSED; + } + + if (removeMeta && status_ != NebdFileStatus::DESTROYED) { + int ret = metaFileManager_->RemoveFileMeta(fileName_); + if (ret != 0) { + LOG(ERROR) << "Remove file record failed. " + << "fd: " << fd_ + << ", filename: " << fileName_; + return -1; + } + status_ = NebdFileStatus::DESTROYED; + } + LOG(INFO) << "Close file success. " + << "fd: " << fd_ + << ", filename: " << fileName_ + << ", meta removed? " << (removeMeta ? "yes" : "no"); + return 0; +} + +int NebdFileEntity::Discard(NebdServerAioContext* aioctx) { + auto task = [&]() { + int ret = executor_->Discard(fileInstance_.get(), aioctx); + if (ret < 0) { + LOG(ERROR) << "Discard file failed. " + << "fd: " << fd_ + << ", fileName: " << fileName_ + << ", context: " << *aioctx; + return -1; + } + return 0; + }; + return ProcessAsyncRequest(task, aioctx); +} + +int NebdFileEntity::AioRead(NebdServerAioContext* aioctx) { + auto task = [&]() { + int ret = executor_->AioRead(fileInstance_.get(), aioctx); + if (ret < 0) { + LOG(ERROR) << "AioRead file failed. " + << "fd: " << fd_ + << ", fileName: " << fileName_ + << ", context: " << *aioctx; + return -1; + } + return 0; + }; + return ProcessAsyncRequest(task, aioctx); +} + +int NebdFileEntity::AioWrite(NebdServerAioContext* aioctx) { + auto task = [&]() { + int ret = executor_->AioWrite(fileInstance_.get(), aioctx); + if (ret < 0) { + LOG(ERROR) << "AioWrite file failed. " + << "fd: " << fd_ + << ", fileName: " << fileName_ + << ", context: " << *aioctx; + return -1; + } + return 0; + }; + return ProcessAsyncRequest(task, aioctx); +} + +int NebdFileEntity::Flush(NebdServerAioContext* aioctx) { + auto task = [&]() { + int ret = executor_->Flush(fileInstance_.get(), aioctx); + if (ret < 0) { + LOG(ERROR) << "Flush file failed. " + << "fd: " << fd_ + << ", fileName: " << fileName_ + << ", context: " << *aioctx; + return -1; + } + return 0; + }; + return ProcessAsyncRequest(task, aioctx); +} + +int NebdFileEntity::Extend(int64_t newsize) { + auto task = [&]() { + int ret = executor_->Extend(fileInstance_.get(), newsize); + if (ret < 0) { + LOG(ERROR) << "Extend file failed. " + << "fd: " << fd_ + << ", newsize: " << newsize + << ", fileName" << fileName_; + return -1; + } + return 0; + }; + return ProcessSyncRequest(task); +} + +int NebdFileEntity::GetInfo(NebdFileInfo* fileInfo) { + auto task = [&]() { + int ret = executor_->GetInfo(fileInstance_.get(), fileInfo); + if (ret < 0) { + LOG(ERROR) << "Get file info failed. " + << "fd: " << fd_ + << ", fileName" << fileName_; + return -1; + } + return 0; + }; + return ProcessSyncRequest(task); +} + +int NebdFileEntity::InvalidCache() { + auto task = [&]() { + int ret = executor_->InvalidCache(fileInstance_.get()); + if (ret < 0) { + LOG(ERROR) << "Invalid cache failed. " + << "fd: " << fd_ + << ", fileName" << fileName_; + return -1; + } + return 0; + }; + return ProcessSyncRequest(task); +} + +int NebdFileEntity::ProcessSyncRequest(ProcessTask task) { + CHECK(executor_ != nullptr) << "file entity is not inited. " + << "filename: " << fileName_; + + NebdRequestReadLockClosure* done = + new (std::nothrow) NebdRequestReadLockClosure(rwLock_); + brpc::ClosureGuard doneGuard(done); + + bool isFileOpened = GuaranteeFileOpened(); + if (!isFileOpened) { + return -1; + } + + int ret = task(); + if (ret < 0) { + LOG(ERROR) << "Process sync request failed. " + << "fd: " << fd_ + << ", fileName" << fileName_; + return -1; + } + return 0; +} + +int NebdFileEntity::ProcessAsyncRequest(ProcessTask task, + NebdServerAioContext* aioctx) { + CHECK(executor_ != nullptr) << "file entity is not inited. " + << "filename: " << fileName_; + CHECK(aioctx != nullptr) << "AioContext should not be null."; + + NebdRequestReadLockClosure* done = + new (std::nothrow) NebdRequestReadLockClosure(rwLock_); + brpc::ClosureGuard doneGuard(done); + + bool isFileOpened = GuaranteeFileOpened(); + if (!isFileOpened) { + return -1; + } + + // 对于异步请求，将此closure传给aiocontext，从而在请求返回时释放读锁 + done->SetClosure(aioctx->done); + aioctx->done = doneGuard.release(); + int ret = task(); + if (ret < 0) { + // 如果请求失败,这里要主动释放锁,并将aiocontext还原回去 + brpc::ClosureGuard doneGuard(done); + aioctx->done = done->GetClosure(); + done->SetClosure(nullptr); + LOG(ERROR) << "Process async request failed. " + << "fd: " << fd_ + << ", fileName" << fileName_; + return -1; + } + return 0; +} + +int NebdFileEntity::UpdateFileStatus(NebdFileInstancePtr fileInstance) { + NebdFileMeta fileMeta; + fileMeta.fd = fd_; + fileMeta.fileName = fileName_; + fileMeta.xattr = fileInstance->xattr; + int ret = metaFileManager_->UpdateFileMeta(fileName_, fileMeta); + if (ret != 0) { + LOG(ERROR) << "Update file meta failed. " + << "filename: " << fileName_; + return -1; + } + + fileInstance_ = fileInstance; + status_ = NebdFileStatus::OPENED; + timeStamp_ = TimeUtility::GetTimeofDayMs(); + return 0; +} + +bool NebdFileEntity::GuaranteeFileOpened() { + // 文件如果已经被用户close了，就不允许后面请求再自动打开进行操作了 + if (status_ == NebdFileStatus::DESTROYED) { + LOG(ERROR) << "File has been destroyed. " + << "filename: " << fileName_ + << ", fd: " << fd_; + return false; + } + + if (status_ == NebdFileStatus::CLOSED) { + int ret = Open(); + if (ret != fd_) { + LOG(ERROR) << "Get opened file failed. " + << "filename: " << fileName_ + << ", fd: " << fd_ + << ", ret: " << ret; + return false; + } + } + return true; +} + +std::ostream& operator<<(std::ostream& os, const NebdFileEntity& entity) { + os << "[filename: " << entity.GetFileName() << ", fd: " << entity.GetFd() + << ", status: " << NebdFileStatus2Str(entity.GetFileStatus()) << "]"; + return os; +} + +} // namespace server +} // namespace nebd + diff --git a/src/part2/file_entity.h b/src/part2/file_entity.h new file mode 100644 index 0000000000..dafe62731a --- /dev/null +++ b/src/part2/file_entity.h @@ -0,0 +1,216 @@ +/* + * Project: nebd + * Created Date: Tuesday March 3rd 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#ifndef SRC_PART2_FILE_ENTITY_H_ +#define SRC_PART2_FILE_ENTITY_H_ + +#include +#include +#include +#include +#include +#include // NOLINT +#include + +#include "src/common/rw_lock.h" +#include "src/common/timeutility.h" +#include "src/part2/define.h" +#include "src/part2/util.h" +#include "src/part2/request_executor.h" +#include "src/part2/metafile_manager.h" + +namespace nebd { +namespace server { + +using nebd::common::BthreadRWLock; +using nebd::common::WriteLockGuard; +using nebd::common::ReadLockGuard; +using nebd::common::TimeUtility; + +class NebdFileInstance; +class NebdRequestExecutor; +using NebdFileInstancePtr = std::shared_ptr; + +// 处理用户请求时需要加读写锁，避免close时仍有用户IO未处理完成 +// 对于异步IO来说，只有返回时才能释放读锁，所以封装成Closure +// 在发送异步请求前，将closure赋值给NebdServerAioContext +class NebdRequestReadLockClosure : public Closure { + public: + explicit NebdRequestReadLockClosure(BthreadRWLock& rwLock) // NOLINT + : rwLock_(rwLock) + , done_(nullptr) { + rwLock_.RDLock(); + } + ~NebdRequestReadLockClosure() {} + + void Run() { + std::unique_ptr selfGuard(this); + brpc::ClosureGuard doneGuard(done_); + rwLock_.Unlock(); + } + + void SetClosure(Closure* done) { + done_ = done; + } + + Closure* GetClosure() { + return done_; + } + + private: + BthreadRWLock& rwLock_; + Closure* done_; +}; + +struct NebdFileEntityOption { + int fd; + std::string fileName; + MetaFileManagerPtr metaFileManager_; +}; + +class NebdFileEntity : public std::enable_shared_from_this { + public: + NebdFileEntity(); + virtual ~NebdFileEntity(); + + /** + * 初始化文件实体 + * @param option: 初始化参数 + * @return 成功返回0， 失败返回-1 + */ + virtual int Init(const NebdFileEntityOption& option); + /** + * 打开文件 + * @return 成功返回fd，失败返回-1 + */ + virtual int Open(); + /** + * 重新open文件，如果之前的后端存储的连接还存在则复用之前的连接 + * 否则与后端存储建立新的连接 + * @param xattr: 文件reopen需要的信息 + * @return 成功返回fd，失败返回-1 + */ + virtual int Reopen(const ExtendAttribute& xattr); + /** + * 关闭文件 + * @param removeMeta: 是否要移除文件元数据记录，true表示移除，false表示不移除 + * 如果是part1传过来的close请求，此参数为true + * 如果是heartbeat manager发起的close请求，此参数为false + * @return 成功返回0，失败返回-1 + */ + virtual int Close(bool removeMeta); + /** + * 给文件扩容 + * @param newsize: 新的文件大小 + * @return 成功返回0，失败返回-1 + */ + virtual int Extend(int64_t newsize); + /** + * 获取文件信息 + * @param fileInfo[out]: 文件信息 + * @return 成功返回0，失败返回-1 + */ + virtual int GetInfo(NebdFileInfo* fileInfo); + /** + * 异步请求，回收指定区域空间 + * @param aioctx: 异步请求上下文 + * @return 成功返回0，失败返回-1 + */ + virtual int Discard(NebdServerAioContext* aioctx); + /** + * 异步请求，读取指定区域内容 + * @param aioctx: 异步请求上下文 + * @return 成功返回0，失败返回-1 + */ + virtual int AioRead(NebdServerAioContext* aioctx); + /** + * 异步请求，写数据到指定区域 + * @param aioctx: 异步请求上下文 + * @return 成功返回0，失败返回-1 + */ + virtual int AioWrite(NebdServerAioContext* aioctx); + /** + * 异步请求，flush文件缓存 + * @param aioctx: 异步请求上下文 + * @return 成功返回0，失败返回-1 + */ + virtual int Flush(NebdServerAioContext* aioctx); + /** + * 使指定文件缓存失效 + * @return 成功返回0，失败返回-1 + */ + virtual int InvalidCache(); + + virtual const std::string GetFileName() const { + return fileName_; + } + + virtual const int GetFd() const { + return fd_; + } + + virtual void UpdateFileTimeStamp(uint64_t timestamp) { + timeStamp_.store(timeStamp_); + } + + virtual const uint64_t GetFileTimeStamp() const { + return timeStamp_.load(); + } + + virtual const NebdFileStatus GetFileStatus() const { + return status_.load(); + } + + private: + /** + * 更新文件状态，包括元信息文件和内存状态 + * @param fileInstancea: open或reopen返回的文件上下文信息 + * @return: 成功返回0，失败返回-1 + */ + int UpdateFileStatus(NebdFileInstancePtr fileInstance); + /** + * 请求统一处理函数 + * @param task: 实际请求执行的函数体 + * @return: 成功返回0，失败返回-1 + */ + using ProcessTask = std::function; + int ProcessSyncRequest(ProcessTask task); + int ProcessAsyncRequest(ProcessTask task, NebdServerAioContext* aioctx); + + // 确保文件处于opened状态，如果不是则尝试进行open + // 无法open或者open失败，则返回false， + // 如果文件处于open状态，则返回true + bool GuaranteeFileOpened(); + + private: + // 文件读写锁，处理请求前加读锁，close文件的时候加写锁 + // 避免close时还有请求未处理完 + BthreadRWLock rwLock_; + // 互斥锁，用于open、close之间的互斥 + butil::Mutex fileStatusMtx_; + // nebd server为该文件分配的唯一标识符 + int fd_; + // 文件名称 + std::string fileName_; + // 文件当前状态，opened表示文件已打开，closed表示文件已关闭 + std::atomic status_; + // 该文件上一次收到心跳时的时间戳 + std::atomic timeStamp_; + // 文件在executor open时返回上下文信息，用于后续文件的请求处理 + NebdFileInstancePtr fileInstance_; + // 文件对应的executor的指针 + NebdRequestExecutor* executor_; + // 元数据持久化管理 + MetaFileManagerPtr metaFileManager_; +}; +using NebdFileEntityPtr = std::shared_ptr; +std::ostream& operator<<(std::ostream& os, const NebdFileEntity& entity); + +} // namespace server +} // namespace nebd + +#endif // SRC_PART2_FILE_ENTITY_H_ diff --git a/src/part2/file_manager.cpp b/src/part2/file_manager.cpp index a6cf10b7c2..5b299ad747 100644 --- a/src/part2/file_manager.cpp +++ b/src/part2/file_manager.cpp @@ -15,9 +15,9 @@ namespace nebd { namespace server { -NebdFileManager::NebdFileManager(FileRecordManagerPtr recordManager) +NebdFileManager::NebdFileManager(MetaFileManagerPtr metaFileManager) : isRunning_(false) - , fileRecordManager_(recordManager) {} + , metaFileManager_(metaFileManager) {} NebdFileManager::~NebdFileManager() {} @@ -38,29 +38,32 @@ int NebdFileManager::Run() { int NebdFileManager::Fini() { isRunning_.store(false); + fileMap_.clear(); LOG(INFO) << "Stop file manager success."; return 0; } int NebdFileManager::Load() { // 从元数据文件中读取持久化的文件信息 - int ret = fileRecordManager_->Load(); + std::vector fileMetas; + int ret = metaFileManager_->ListFileMeta(&fileMetas); if (ret < 0) { - LOG(ERROR) << "Load file record failed."; + LOG(ERROR) << "Load file metas failed."; return ret; } - FileRecordMap fileRecords = fileRecordManager_->ListRecords(); // 根据持久化的信息重新open文件 int maxFd = 0; - for (auto& fileRecord : fileRecords) { - maxFd = std::max(maxFd, fileRecord.first); - // reopen失败忽略，此时文件状态为closed，下次访问仍然会去open - // 这么考虑是为了防止个别文件有问题导致整个part2不可用 - int ret = Reopen(fileRecord.second); + for (auto& fileMeta : fileMetas) { + NebdFileEntityPtr entity = + GenerateFileEntity(fileMeta.fd, fileMeta.fileName); + CHECK(entity != nullptr) << "file entity is null."; + int ret = entity->Reopen(fileMeta.xattr); if (ret < 0) { LOG(WARNING) << "Reopen file failed. " - << "filenam: " << fileRecord.second.fileName; + << "filename: " << fileMeta.fileName + << ", fd: " << fileMeta.fd; } + maxFd = std::max(maxFd, fileMeta.fd); } fdAlloc_.InitFd(maxFd); LOG(INFO) << "Load file record finished."; @@ -68,373 +71,184 @@ int NebdFileManager::Load() { } int NebdFileManager::Open(const std::string& filename) { - int ret = OpenInternal(filename, true); - if (ret < 0) { - LOG(ERROR) << "open file failed. " - << "filename: " << filename - << ", ret: " << ret; + NebdFileEntityPtr entity = GetOrCreateFileEntity(filename); + if (entity == nullptr) { + LOG(ERROR) << "Open file failed. filename: " << filename; return -1; } - return ret; + return entity->Open(); } int NebdFileManager::Close(int fd, bool removeRecord) { - NebdFileRecord fileRecord; - bool getSuccess = fileRecordManager_->GetRecord(fd, &fileRecord); - if (!getSuccess) { - LOG(WARNING) << "File record not exist, fd: " << fd; + NebdFileEntityPtr entity = GetFileEntity(fd); + if (entity == nullptr) { + LOG(WARNING) << "Close file failed. fd: " << fd; return 0; } - - // 用于和其他用户请求互斥，避免文件被close后，请求发到后端导致返回失败 - WriteLockGuard writeLock(*fileRecord.rwLock); - int ret = CloseInternal(fd); - if (ret < 0) { - LOG(ERROR) << "Close file failed. " - << "fd: " << fd - << ", filename: " << fileRecord.fileName; - return -1; - } - if (removeRecord) { - bool removeSuccess = fileRecordManager_->RemoveRecord(fd); - if (!removeSuccess) { - LOG(ERROR) << "Remove file record failed. " - << "fd: " << fd - << ", filename: " << fileRecord.fileName; - return -1; - } + int ret = entity->Close(removeRecord); + if (ret == 0 && removeRecord) { + RemoveEntity(fd); + LOG(INFO) << "file entity is removed. fd: " << fd; } - LOG(INFO) << "Close file success. " - << "fd: " << fd - << ", filename: " << fileRecord.fileName; - return 0; + return ret; } int NebdFileManager::Discard(int fd, NebdServerAioContext* aioctx) { - auto task = [&](NebdProcessClosure* done) { - const NebdFileRecord& fileRecord = done->GetFileRecord(); - done->SetClosure(aioctx->done); - aioctx->done = done; - int ret = fileRecord.executor->Discard( - fileRecord.fileInstance.get(), aioctx); - if (ret < 0) { - brpc::ClosureGuard doneGuard(done); - aioctx->done = done->GetClosure(); - done->SetClosure(nullptr); - LOG(ERROR) << "Discard file failed. " - << "fd: " << fd - << ", fileName: " << fileRecord.fileName - << ", context: " << *aioctx; - return -1; - } - return 0; - }; - return ProcessRequest(fd, task); + NebdFileEntityPtr entity = GetFileEntity(fd); + if (entity == nullptr) { + LOG(ERROR) << "Discard file failed. fd: " << fd; + return -1; + } + return entity->Discard(aioctx); } int NebdFileManager::AioRead(int fd, NebdServerAioContext* aioctx) { - auto task = [&](NebdProcessClosure* done) { - const NebdFileRecord& fileRecord = done->GetFileRecord(); - done->SetClosure(aioctx->done); - aioctx->done = done; - int ret = fileRecord.executor->AioRead( - fileRecord.fileInstance.get(), aioctx); - if (ret < 0) { - brpc::ClosureGuard doneGuard(done); - aioctx->done = done->GetClosure(); - done->SetClosure(nullptr); - LOG(ERROR) << "AioRead file failed. " - << "fd: " << fd - << ", fileName: " << fileRecord.fileName - << ", context: " << *aioctx; - return -1; - } - return 0; - }; - return ProcessRequest(fd, task); + NebdFileEntityPtr entity = GetFileEntity(fd); + if (entity == nullptr) { + LOG(ERROR) << "AioRead file failed. fd: " << fd; + return -1; + } + return entity->AioRead(aioctx); } int NebdFileManager::AioWrite(int fd, NebdServerAioContext* aioctx) { - auto task = [&](NebdProcessClosure* done) { - const NebdFileRecord& fileRecord = done->GetFileRecord(); - done->SetClosure(aioctx->done); - aioctx->done = done; - int ret = fileRecord.executor->AioWrite( - fileRecord.fileInstance.get(), aioctx); - if (ret < 0) { - brpc::ClosureGuard doneGuard(done); - aioctx->done = done->GetClosure(); - done->SetClosure(nullptr); - LOG(ERROR) << "AioWrite file failed. " - << "fd: " << fd - << ", fileName: " << fileRecord.fileName - << ", context: " << *aioctx; - return -1; - } - return 0; - }; - return ProcessRequest(fd, task); -} - -int NebdFileManager::Flush(int fd, NebdServerAioContext* aioctx) { - auto task = [&](NebdProcessClosure* done) { - const NebdFileRecord& fileRecord = done->GetFileRecord(); - done->SetClosure(aioctx->done); - aioctx->done = done; - int ret = fileRecord.executor->Flush( - fileRecord.fileInstance.get(), aioctx); - if (ret < 0) { - brpc::ClosureGuard doneGuard(done); - aioctx->done = done->GetClosure(); - done->SetClosure(nullptr); - LOG(ERROR) << "Flush file failed. " - << "fd: " << fd - << ", fileName: " << fileRecord.fileName - << ", context: " << *aioctx; - return -1; - } - return 0; - }; - return ProcessRequest(fd, task); -} - -int NebdFileManager::Extend(int fd, int64_t newsize) { - auto task = [&](NebdProcessClosure* done) { - brpc::ClosureGuard doneGuard(done); - const NebdFileRecord& fileRecord = done->GetFileRecord(); - int ret = fileRecord.executor->Extend( - fileRecord.fileInstance.get(), newsize); - if (ret < 0) { - LOG(ERROR) << "Extend file failed. " - << "fd: " << fd - << ", newsize: " << newsize - << ", fileName" << fileRecord.fileName; - return -1; - } - return 0; - }; - return ProcessRequest(fd, task); -} - -int NebdFileManager::GetInfo(int fd, NebdFileInfo* fileInfo) { - auto task = [&](NebdProcessClosure* done) { - brpc::ClosureGuard doneGuard(done); - const NebdFileRecord& fileRecord = done->GetFileRecord(); - int ret = fileRecord.executor->GetInfo( - fileRecord.fileInstance.get(), fileInfo); - if (ret < 0) { - LOG(ERROR) << "Get file info failed. " - << "fd: " << fd - << ", fileName" << fileRecord.fileName; - return -1; - } - return 0; - }; - return ProcessRequest(fd, task); -} - -int NebdFileManager::InvalidCache(int fd) { - auto task = [&](NebdProcessClosure* done) { - brpc::ClosureGuard doneGuard(done); - const NebdFileRecord& fileRecord = done->GetFileRecord(); - int ret = fileRecord.executor->InvalidCache( - fileRecord.fileInstance.get()); - if (ret < 0) { - LOG(ERROR) << "Invalid cache failed. " - << "fd: " << fd - << ", fileName" << fileRecord.fileName; - return -1; - } - return 0; - }; - return ProcessRequest(fd, task); -} - -int NebdFileManager::OpenInternal(const std::string& fileName, - bool create) { - // 同名文件open需要互斥 - NameLockGuard openGuard(nameLock_, fileName); - - NebdFileRecord fileRecord; - bool getSuccess = fileRecordManager_->GetRecord(fileName, &fileRecord); - if (getSuccess && fileRecord.status == NebdFileStatus::OPENED) { - LOG(WARNING) << "File is already opened. " - << "filename: " << fileName - << "fd: " << fileRecord.fd; - return fileRecord.fd; - } - - if (create) { - fileRecord.fileName = fileName; - fileRecord.fd = GetValidFd(); - } else { - // create为false的情况下，如果record不存在，需要返回错误 - if (!getSuccess) { - LOG(ERROR) << "open file failed: no record. " - << "filename: " << fileName; - return -1; - } - } - - NebdFileType type = GetFileType(fileName); - NebdRequestExecutor* executor = - NebdRequestExecutorFactory::GetExecutor(type); - if (executor == nullptr) { - LOG(ERROR) << "open file failed, invalid filename. " - << "filename: " << fileName; + NebdFileEntityPtr entity = GetFileEntity(fd); + if (entity == nullptr) { + LOG(ERROR) << "AioWrite file failed. fd: " << fd; return -1; } + return entity->AioWrite(aioctx); +} - NebdFileInstancePtr fileInstance = executor->Open(fileName); - if (fileInstance == nullptr) { - LOG(ERROR) << "open file failed. " - << "filename: " << fileName; +int NebdFileManager::Flush(int fd, NebdServerAioContext* aioctx) { + NebdFileEntityPtr entity = GetFileEntity(fd); + if (entity == nullptr) { + LOG(ERROR) << "Flush file failed. fd: " << fd; return -1; } + return entity->Flush(aioctx); +} - fileRecord.executor = executor; - fileRecord.fileInstance = fileInstance; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.timeStamp = TimeUtility::GetTimeofDayMs(); - - bool updateSuccess = fileRecordManager_->UpdateRecord(fileRecord); - if (!updateSuccess) { - executor->Close(fileInstance.get()); - LOG(ERROR) << "Update file record failed. " - << "filename: " << fileName; +int NebdFileManager::Extend(int fd, int64_t newsize) { + NebdFileEntityPtr entity = GetFileEntity(fd); + if (entity == nullptr) { + LOG(ERROR) << "Extend file failed. fd: " << fd; return -1; } - LOG(INFO) << "Open file success. " - << "fd: " << fileRecord.fd - << ", filename: " << fileRecord.fileName; - return fileRecord.fd; + return entity->Extend(newsize); } -int NebdFileManager::Reopen(NebdFileRecord fileRecord) { - NebdFileType type = GetFileType(fileRecord.fileName); - NebdRequestExecutor* executor = - NebdRequestExecutorFactory::GetExecutor(type); - if (executor == nullptr) { - LOG(ERROR) << "Reopen file failed, invalid filename. " - << "filename: " << fileRecord.fileName; - return -1; - } - NebdFileInstancePtr fileInstance = executor->Reopen( - fileRecord.fileName, fileRecord.fileInstance->addition); - if (fileInstance == nullptr) { - LOG(ERROR) << "Reopen file failed. " - << "filename: " << fileRecord.fileName; +int NebdFileManager::GetInfo(int fd, NebdFileInfo* fileInfo) { + NebdFileEntityPtr entity = GetFileEntity(fd); + if (entity == nullptr) { + LOG(ERROR) << "Get file info failed. fd: " << fd; return -1; } + return entity->GetInfo(fileInfo); +} - fileRecord.executor = executor; - fileRecord.fileInstance = fileInstance; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.timeStamp = TimeUtility::GetTimeofDayMs(); - - bool updateSuccess = fileRecordManager_->UpdateRecord(fileRecord); - if (!updateSuccess) { - executor->Close(fileInstance.get()); - LOG(ERROR) << "Update file record failed. " - << "filename: " << fileRecord.fileName; +int NebdFileManager::InvalidCache(int fd) { + NebdFileEntityPtr entity = GetFileEntity(fd); + if (entity == nullptr) { + LOG(ERROR) << "Invalid file cache failed. fd: " << fd; return -1; } - - LOG(INFO) << "Reopen file success. " - << "fd: " << fileRecord.fd - << ", filename: " << fileRecord.fileName; - return 0; + return entity->InvalidCache(); } -int NebdFileManager::CloseInternal(int fd) { - NebdFileRecord fileRecord; - bool getSuccess = fileRecordManager_->GetRecord(fd, &fileRecord); - if (!getSuccess) { - LOG(WARNING) << "File record not exist, fd: " << fd; - return 0; +NebdFileEntityPtr +NebdFileManager::GetFileEntity(int fd) { + ReadLockGuard readLock(rwLock_); + auto iter = fileMap_.find(fd); + if (iter == fileMap_.end()) { + return nullptr; } + return iter->second; +} - if (fileRecord.status != NebdFileStatus::OPENED) { - LOG(INFO) << "File has been closed. " - << "fd: " << fileRecord.fd - << ", filename: " << fileRecord.fileName; - return 0; +int NebdFileManager::GenerateValidFd() { + int fd = 0; + while (true) { + fd = fdAlloc_.GetNext(); + if (fileMap_.find(fd) == fileMap_.end()) { + break; + } } + return fd; +} - int ret = fileRecord.executor->Close(fileRecord.fileInstance.get()); - if (ret < 0) { - LOG(ERROR) << "Close file failed. " - << "fd: " << fileRecord.fd - << ", filename: " << fileRecord.fileName; - return -1; +NebdFileEntityPtr +NebdFileManager::GetOrCreateFileEntity(const std::string& fileName) { + { + ReadLockGuard readLock(rwLock_); + for (const auto& pair : fileMap_) { + std::string curFile = pair.second->GetFileName(); + if (fileName == curFile) { + return pair.second; + } + } } - - fileRecordManager_->UpdateFileStatus(fileRecord.fd, NebdFileStatus::CLOSED); - return 0; + int fd = GenerateValidFd(); + return GenerateFileEntity(fd, fileName); } -int NebdFileManager::ProcessRequest(int fd, ProcessTask task) { - NebdFileRecord fileRecord; - bool getSuccess = fileRecordManager_->GetRecord(fd, &fileRecord); - if (!getSuccess) { - LOG(ERROR) << "File record not exist, fd: " << fd; - return -1; +NebdFileEntityPtr +NebdFileManager::GenerateFileEntity(int fd, const std::string& fileName) { + WriteLockGuard writeLock(rwLock_); + for (const auto& pair : fileMap_) { + std::string curFile = pair.second->GetFileName(); + if (fileName == curFile) { + return pair.second; + } } - NebdProcessClosure* done = - new (std::nothrow) NebdProcessClosure(fileRecord); - brpc::ClosureGuard doneGuard(done); - - // 此时文件可能被close了，与之前获取到的record不一致，因此需要重新获取 - // TODO(yyk) 这不是一种优雅的实现方法，后续重构 - getSuccess = fileRecordManager_->GetRecord(fd, &fileRecord); - if (!getSuccess) { - LOG(ERROR) << "File record not exist, fd: " << fd; - return -1; + // 检测是否存在冲突的文件记录 + auto iter = fileMap_.find(fd); + if (iter != fileMap_.end()) { + LOG(ERROR) << "File entity conflict. " + << "Exist filename: " << iter->second->GetFileName() + << ", Exist fd: " << iter->first + << ", Create filename: " << fileName + << ", Create fd: " << fd; + return nullptr; } - done->SetFileRecord(fileRecord); - if (fileRecord.status != NebdFileStatus::OPENED) { - int ret = OpenInternal(fileRecord.fileName); - if (ret != fd) { - LOG(ERROR) << "Get opened file failed. " - << "filename: " << fileRecord.fileName - << ", fd: " << fd - << ", ret: " << ret; - return -1; - } - // 重新open后要获取新的record - getSuccess = fileRecordManager_->GetRecord(fd, &fileRecord); - if (!getSuccess) { - LOG(ERROR) << "File record not exist, fd: " << fd; - return -1; - } - done->SetFileRecord(fileRecord); + NebdFileEntityOption option; + option.fd = fd; + option.fileName = fileName; + option.metaFileManager_ = metaFileManager_; + NebdFileEntityPtr entity = std::make_shared(); + int ret = entity->Init(option); + if (ret != 0) { + LOG(ERROR) << "Generate file entity failed."; + return nullptr; } + fileMap_.emplace(fd, entity); + return entity; +} - int ret = task(dynamic_cast(doneGuard.release())); - if (ret < 0) { - LOG(ERROR) << "Process request failed. " - << "fd: " << fd - << ", fileName" << fileRecord.fileName; - return -1; +void NebdFileManager::RemoveEntity(int fd) { + WriteLockGuard writeLock(rwLock_); + auto iter = fileMap_.find(fd); + if (iter != fileMap_.end()) { + fileMap_.erase(iter); } - return 0; } -FileRecordManagerPtr NebdFileManager::GetRecordManager() { - return fileRecordManager_; +FileEntityMap NebdFileManager::GetFileEntityMap() { + ReadLockGuard readLock(rwLock_); + return fileMap_; } -int NebdFileManager::GetValidFd() { - int fd = 0; - while (true) { - fd = fdAlloc_.GetNext(); - if (!fileRecordManager_->Exist(fd)) { - break; - } +std::string NebdFileManager::DumpAllFileStatus() { + ReadLockGuard readLock(rwLock_); + std::ostringstream os; + os << "{"; + for (const auto& pair : fileMap_) { + os << *(pair.second); } - return fd; + os << "}"; + return os.str(); } } // namespace server diff --git a/src/part2/file_manager.h b/src/part2/file_manager.h index e6792ccd60..498ac42ce0 100644 --- a/src/part2/file_manager.h +++ b/src/part2/file_manager.h @@ -11,17 +11,17 @@ #include #include #include +#include #include #include // NOLINT #include #include "src/common/rw_lock.h" #include "src/common/name_lock.h" -#include "src/common/timeutility.h" #include "src/part2/define.h" #include "src/part2/util.h" -#include "src/part2/filerecord_manager.h" -#include "src/part2/request_executor.h" +#include "src/part2/file_entity.h" +#include "src/part2/metafile_manager.h" #include "proto/client.pb.h" namespace nebd { @@ -31,50 +31,11 @@ using nebd::common::NameLock; using nebd::common::NameLockGuard; using nebd::common::WriteLockGuard; using nebd::common::ReadLockGuard; -using nebd::common::TimeUtility; - -// 处理用户请求时需要加读写锁，避免close时仍有用户IO未处理完成 -// 对于异步IO来说，只有返回时才能释放读锁，所以封装成Closure -// 在发送异步请求前，将closure赋值给NebdServerAioContext -class NebdProcessClosure : public Closure { - public: - explicit NebdProcessClosure(const NebdFileRecord& fileRecord) - : fileRecord_(fileRecord) - , done_(nullptr) { - fileRecord_.rwLock->RDLock(); - } - NebdProcessClosure() {} - - void Run() { - std::unique_ptr selfGuard(this); - brpc::ClosureGuard doneGuard(done_); - fileRecord_.rwLock->Unlock(); - } - - void SetClosure(Closure* done) { - done_ = done; - } - - Closure* GetClosure() { - return done_; - } - - NebdFileRecord GetFileRecord() { - return fileRecord_; - } - - void SetFileRecord(const NebdFileRecord& fileRecord) { - fileRecord_ = fileRecord; - } - - private: - NebdFileRecord fileRecord_; - Closure* done_; -}; +using FileEntityMap = std::unordered_map; class NebdFileManager { public: - explicit NebdFileManager(FileRecordManagerPtr recordManager); + explicit NebdFileManager(MetaFileManagerPtr metaFileManager); virtual ~NebdFileManager(); /** * 停止FileManager并释放FileManager资源 @@ -150,46 +111,33 @@ class NebdFileManager { */ virtual int InvalidCache(int fd); + // 根据fd从map中获取指定的entity + // 如果entity已存在，返回entity指针，否则返回nullptr + virtual NebdFileEntityPtr GetFileEntity(int fd); + + virtual FileEntityMap GetFileEntityMap(); + + // 将所有文件状态输出到字符串 + std::string DumpAllFileStatus(); // set public for test // 启动时从metafile加载文件记录，并reopen文件 int Load(); - // 返回recordmanager - virtual FileRecordManagerPtr GetRecordManager(); private: - /** - * 打开指定文件并更新元数据信息 - * @param filename: 文件名 - * @param create: 为true将产生新的filerecord - * @return: 成功返回0，失败返回-1 - */ - int OpenInternal(const std::string& fileName, bool create = false); - /** - * 关闭指定文件，并将文件状态变更为CLOSED - * @param fd: 文件的内存记录 - * @return: 成功返回0，失败返回-1 - */ - int CloseInternal(int fd); - /** - * 根据文件内存记录信息重新open文件 - * @param fileRecord: 文件的内存记录 - * @return: 成功返回0，失败返回-1 - */ - int Reopen(NebdFileRecord fileRecord); - /** - * 分配新的可用的fd - * @return: 成功返回有效的fd，失败返回-1 - */ - int GetValidFd(); - /** - * 请求统一处理函数 - * @param fd: 请求对应文件的fd - * @param task: 实际请求执行的函数体 - * @return: 成功返回0，失败返回-1 - */ - using ProcessTask = std::function; - int ProcessRequest(int fd, ProcessTask task); + // 分配新的可用的fd，fd不允许和已经存在的重复 + // 成功返回的可用fd，失败返回-1 + int GenerateValidFd(); + // 根据文件名获取file entity + // 如果entity存在，直接返回entity指针 + // 如果entity不存在，则创建新的entity，并插入map，然后返回 + NebdFileEntityPtr GetOrCreateFileEntity(const std::string& fileName); + // 根据fd和文件名生成file entity， + // 如果fd对于的entity已存在,直接返回entity指针 + // 如果entity不存在，则生成新的entity，并插入map，然后返回 + NebdFileEntityPtr GenerateFileEntity(int fd, const std::string& fileName); + // 删除指定fd对应的entity + void RemoveEntity(int fd); private: // 当前filemanager的运行状态，true表示正在运行，false标为未运行 @@ -199,7 +147,11 @@ class NebdFileManager { // fd分配器 FdAllocator fdAlloc_; // nebd server 文件记录管理 - FileRecordManagerPtr fileRecordManager_; + MetaFileManagerPtr metaFileManager_; + // file map 读写保护锁 + RWLock rwLock_; + // 文件fd和文件实体的映射 + FileEntityMap fileMap_; }; using NebdFileManagerPtr = std::shared_ptr; diff --git a/src/part2/file_service.cpp b/src/part2/file_service.cpp index 5eac07d8bc..643d5f7b42 100644 --- a/src/part2/file_service.cpp +++ b/src/part2/file_service.cpp @@ -100,6 +100,9 @@ void NebdFileServiceImpl::OpenFile( if (fd > 0) { response->set_retcode(RetCode::kOK); response->set_fd(fd); + LOG(INFO) << "Open file success. " + << "filename: " << request->filename() + << ", fd: " << fd; } else { LOG(ERROR) << "Open file failed. " << "filename: " << request->filename() @@ -276,6 +279,8 @@ void NebdFileServiceImpl::CloseFile( << ", return code: " << rc; } else { response->set_retcode(RetCode::kOK); + LOG(INFO) << "Close file success. " + << "fd: " << request->fd(); } } diff --git a/src/part2/filerecord_manager.cpp b/src/part2/filerecord_manager.cpp deleted file mode 100644 index fd71481d1a..0000000000 --- a/src/part2/filerecord_manager.cpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Project: nebd - * Created Date: Thursday February 13th 2020 - * Author: yangyaokai - * Copyright (c) 2020 netease - */ - -#include - -#include "src/part2/filerecord_manager.h" - -namespace nebd { -namespace server { - -int FileRecordManager::Load() { - int ret = metaFileManager_->ListFileRecord(&map_); - if (ret < 0) { - LOG(ERROR) << "List file records from meta file failed. "; - return -1; - } - return 0; -} - -bool FileRecordManager::GetRecord(int fd, NebdFileRecord* fileRecord) { - ReadLockGuard readGuard(rwLock_); - if (map_.find(fd) == map_.end()) { - return false; - } - *fileRecord = map_[fd]; - return true; -} - -bool FileRecordManager::GetRecord(const std::string& fileName, - NebdFileRecord* fileRecord) { - ReadLockGuard readGuard(rwLock_); - int fd = FindRecordUnlocked(fileName); - if (fd > 0) { - *fileRecord = map_[fd]; - return true; - } - return false; -} - -bool FileRecordManager::UpdateRecord(const NebdFileRecord& fileRecord) { - WriteLockGuard writeGuard(rwLock_); - int fd = fileRecord.fd; - - // 如果fd已存在，判断文件名是否相同，不同则返回错误 - bool isConflict = map_.find(fd) != map_.end() && - map_[fd].fileName != fileRecord.fileName; - if (isConflict) { - LOG(ERROR) << "Conflict record. " - << "Old fileName: " << map_[fd].fileName - << ", new fileName: " << fileRecord.fileName - << ", fd: " << fd; - return false; - } - - FileRecordMap tempMap = map_; - // 如果该文件记录已存在，且fd不同，则先删除旧的记录 - int oldFd = FindRecordUnlocked(fileRecord.fileName); - if (oldFd != fileRecord.fd) { - LOG(WARNING) << "Remove old record, filename: " - << fileRecord.fileName - << ", old fd: " << oldFd - << ", newFd: " << fileRecord.fd; - tempMap.erase(oldFd); - } - tempMap[fd] = fileRecord; - - // 先更新元数据文件，如果更新成功再修改内存记录，否则返回失败 - int ret = metaFileManager_->UpdateMetaFile(tempMap); - if (ret < 0) { - LOG(ERROR) << "Update meta file failed. " - << "file name: " << fileRecord.fileName - << ", fd: " << fd; - return false; - } - map_ = std::move(tempMap); - return true; -} - -bool FileRecordManager::RemoveRecord(int fd) { - WriteLockGuard writeGuard(rwLock_); - if (map_.find(fd) == map_.end()) { - return true; - } - - FileRecordMap tempMap = map_; - tempMap.erase(fd); - - // 先更新元数据文件，如果更新成功再修改内存记录，否则返回失败 - int ret = metaFileManager_->UpdateMetaFile(tempMap); - if (ret < 0) { - LOG(ERROR) << "Update meta file failed. " - << "file name: " << map_[fd].fileName - << ", fd: " << fd; - return false; - } - map_ = std::move(tempMap); - return true; -} - -bool FileRecordManager::Exist(int fd) { - ReadLockGuard readGuard(rwLock_); - return map_.find(fd) != map_.end(); -} - -void FileRecordManager::Clear() { - WriteLockGuard writeGuard(rwLock_); - map_.clear(); -} - -FileRecordMap FileRecordManager::ListRecords() { - ReadLockGuard readGuard(rwLock_); - return map_; -} - -int FileRecordManager::FindRecordUnlocked(const std::string& fileName) { - int fd = -1; - for (const auto& pair : map_) { - if (pair.second.fileName == fileName) { - fd = pair.first; - break; - } - } - return fd; -} - -bool FileRecordManager::UpdateFileTimestamp(int fd, uint64_t timestamp) { - WriteLockGuard writeGuard(rwLock_); - if (map_.find(fd) == map_.end()) { - return false; - } - map_[fd].timeStamp = timestamp; - return true; -} - -bool FileRecordManager::GetFileTimestamp(int fd, uint64_t* timestamp) { - ReadLockGuard readGuard(rwLock_); - if (map_.find(fd) == map_.end()) { - return false; - } - *timestamp = map_[fd].timeStamp; - return true; -} - -bool FileRecordManager::UpdateFileStatus(int fd, NebdFileStatus status) { - WriteLockGuard writeGuard(rwLock_); - if (map_.find(fd) == map_.end()) { - return false; - } - map_[fd].status = status; - return true; -} - -bool FileRecordManager::GetFileStatus(int fd, NebdFileStatus* status) { - ReadLockGuard readGuard(rwLock_); - if (map_.find(fd) == map_.end()) { - return false; - } - *status = map_[fd].status; - return true; -} - -} // namespace server -} // namespace nebd - diff --git a/src/part2/filerecord_manager.h b/src/part2/filerecord_manager.h deleted file mode 100644 index 90cc84f7a4..0000000000 --- a/src/part2/filerecord_manager.h +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Project: nebd - * Created Date: Thursday February 13th 2020 - * Author: yangyaokai - * Copyright (c) 2020 netease - */ - -#ifndef SRC_PART2_FILERECORD_MANAGER_H_ -#define SRC_PART2_FILERECORD_MANAGER_H_ - -#include -#include -#include -#include - -#include "src/common/rw_lock.h" -#include "src/part2/define.h" -#include "src/part2/metafile_manager.h" - -namespace nebd { -namespace server { - -using nebd::common::RWLock; -using nebd::common::WriteLockGuard; -using nebd::common::ReadLockGuard; -using FileRecordMap = std::unordered_map; - -// nebd server 文件记录管理（包括内存状态和持久化状态） -class FileRecordManager { - public: - explicit FileRecordManager(MetaFileManagerPtr metaFileManager) - : metaFileManager_(metaFileManager) {} - virtual ~FileRecordManager() {} - /** - * 从持久化文件中加载文件记录到内存 - * @return: 成功返回0，失败返回-1 - */ - virtual int Load(); - /** - * 根据fd获取指定文件的内存记录 - * @param fd: 文件的fd - * @param fileRecord[out]: 文件记录 - * @return: 成功返回true，失败返回false - */ - virtual bool GetRecord(int fd, NebdFileRecord* fileRecord); - /** - * 根据文件名获取指定文件的内存记录 - * @param fileName: 文件名称 - * @param fileRecord[out]: 文件记录 - * @return: 成功返回true，失败返回false - */ - virtual bool GetRecord(const std::string& fileName, - NebdFileRecord* fileRecord); - /** - * 修改文件持久化内容，并更新文件内存记录 - * 如果记录不存在，插入一条新的记录 - * 如果相同文件名的记录已存在，但是fd不同，则删除旧的记录，再插入当前记录 - * 如果相同fd的记录已存在，但是文件名不同，则返回失败 - * @param fileRecord: 文件记录 - * @return: 成功返回true，失败返回false - */ - virtual bool UpdateRecord(const NebdFileRecord& fileRecord); - /** - * 删除指定fd对应的文件记录，先更新元数据文件，然后跟新内存记录 - * @param fd: 文件的fd - * @return: 成功返回true，失败返回false - */ - virtual bool RemoveRecord(int fd); - /** - * 判断指定fd对应的文件记录是否存在 - * @param fd: 文件的fd - * @return: 如果存在返回true，不存在返回false - */ - virtual bool Exist(int fd); - /** - * 清楚所有的记录 - */ - virtual void Clear(); - /** - * 获取所有文件记录的映射表 - * @return: 返回所有文件记录的映射表 - */ - virtual FileRecordMap ListRecords(); - /** - * 更新文件内存记录的时间戳 - * @param fd: 指定的文件fd - * @param timestamp: 要变更为的时间戳值 - * @return: 成功返回true，指定fd不存在返回false - */ - virtual bool UpdateFileTimestamp(int fd, uint64_t timestamp); - /** - * 获取文件内存记录的时间戳 - * @param fd: 指定的文件fd - * @param timestamp[out]: 获取到的时间戳值 - * @return: 成功返回true，指定fd不存在返回false - */ - virtual bool GetFileTimestamp(int fd, uint64_t* timestamp); - /** - * 更新文件内存记录的状态 - * @param fd: 指定的文件fd - * @param status: 要变更为的文件状态 - * @return: 成功返回true，指定fd不存在返回false - */ - virtual bool UpdateFileStatus(int fd, NebdFileStatus status); - /** - * 获取文件内存记录的状态 - * @param fd: 指定的文件fd - * @param status[out]: 获取到的文件状态 - * @return: 成功返回true，指定fd不存在返回false - */ - virtual bool GetFileStatus(int fd, NebdFileStatus* status); - - private: - // 获取指定文件名的文件记录（非线程安全） - int FindRecordUnlocked(const std::string& fileName); - - private: - // 保护文件记录映射表的读写锁 - RWLock rwLock_; - // 文件记录映射表 - FileRecordMap map_; - // 持久化元数据文件管理 - MetaFileManagerPtr metaFileManager_; -}; -using FileRecordManagerPtr = std::shared_ptr; - -} // namespace server -} // namespace nebd - -#endif // SRC_PART2_FILERECORD_MANAGER_H_ diff --git a/src/part2/heartbeat_manager.cpp b/src/part2/heartbeat_manager.cpp index 6e02f7af60..65919a2a25 100644 --- a/src/part2/heartbeat_manager.cpp +++ b/src/part2/heartbeat_manager.cpp @@ -39,44 +39,45 @@ int HeartbeatManager::Fini() { } bool HeartbeatManager::UpdateFileTimestamp(int fd, uint64_t timestamp) { - return fileManager_->GetRecordManager()->UpdateFileTimestamp(fd, timestamp); + NebdFileEntityPtr entity = fileManager_->GetFileEntity(fd); + if (entity == nullptr) { + LOG(ERROR) << "File not exist, fd: " << fd; + return false; + } + entity->UpdateFileTimeStamp(timestamp); + return true; } void HeartbeatManager::CheckTimeoutFunc() { while (sleeper_.wait_for( std::chrono::milliseconds(checkTimeoutIntervalMs_))) { - FileRecordManagerPtr recordManager = fileManager_->GetRecordManager(); - FileRecordMap fileRecords = recordManager->ListRecords(); LOG_EVERY_N(INFO, 60 * 1000 / checkTimeoutIntervalMs_) - << "Checking timeout, file records num: " << fileRecords.size(); - for (auto& fileRecord : fileRecords) { - bool needClose = CheckNeedClosed(fileRecord.first); + << "Checking timeout, file status: " + << fileManager_->DumpAllFileStatus(); + FileEntityMap fileEntityMap = fileManager_->GetFileEntityMap(); + NebdFileEntityPtr curEntity; + for (const auto& entityPair : fileEntityMap) { + curEntity = entityPair.second; + bool needClose = CheckNeedClosed(curEntity); if (!needClose) { continue; } std::string standardTime; TimeUtility::TimeStampToStandard( - fileRecord.second.timeStamp / 1000, &standardTime); + curEntity->GetFileTimeStamp() / 1000, &standardTime); LOG(INFO) << "Close file which has timed out. " << "Last time received heartbeat or request: " << standardTime; - fileManager_->Close(fileRecord.first, false); + curEntity->Close(false); } } } -bool HeartbeatManager::CheckNeedClosed(int fd) { - FileRecordManagerPtr recordManager = fileManager_->GetRecordManager(); - NebdFileRecord record; - bool getTimeSuccess = recordManager->GetRecord(fd, &record); - if (!getTimeSuccess) { - return false; - } - +bool HeartbeatManager::CheckNeedClosed(NebdFileEntityPtr entity) { uint64_t curTime = TimeUtility::GetTimeofDayMs(); - uint64_t interval = curTime - record.timeStamp; + uint64_t interval = curTime - entity->GetFileTimeStamp(); // 文件如果是opened状态，并且已经超时，则需要调用close - bool needClose = record.status == NebdFileStatus::OPENED + bool needClose = entity->GetFileStatus() == NebdFileStatus::OPENED && interval > (uint64_t)1000 * heartbeatTimeoutS_; return needClose; } diff --git a/src/part2/heartbeat_manager.h b/src/part2/heartbeat_manager.h index ce79e7fdfc..8e931f093f 100644 --- a/src/part2/heartbeat_manager.h +++ b/src/part2/heartbeat_manager.h @@ -51,7 +51,7 @@ class HeartbeatManager { // 心跳检测线程的函数执行体 void CheckTimeoutFunc(); // 判断文件是否需要close - bool CheckNeedClosed(int fd); + bool CheckNeedClosed(NebdFileEntityPtr entity); private: // 当前heartbeatmanager的运行状态，true表示正在运行，false标为未运行 diff --git a/src/part2/metafile_manager.cpp b/src/part2/metafile_manager.cpp index 7d0c4aae52..3573d9665e 100644 --- a/src/part2/metafile_manager.cpp +++ b/src/part2/metafile_manager.cpp @@ -6,30 +6,84 @@ */ #include +#include #include "src/part2/metafile_manager.h" -#include "src/part2/util.h" #include "src/part2/request_executor.h" namespace nebd { namespace server { -NebdMetaFileManager::NebdMetaFileManager( - const std::string& metaFilePath, - std::shared_ptr wrapper, - std::shared_ptr parser) - : metaFilePath_(metaFilePath) - , wrapper_(wrapper) - , parser_(parser) {} +NebdMetaFileManager::NebdMetaFileManager() + : metaFilePath_("") + , wrapper_(nullptr) + , parser_(nullptr) {} NebdMetaFileManager::~NebdMetaFileManager() {} -int NebdMetaFileManager::UpdateMetaFile(const FileRecordMap& fileRecords) { - Json::Value root = parser_->ConvertFileRecordsToJson(fileRecords); +int NebdMetaFileManager::Init(const NebdMetaFileManagerOption& option) { + metaFilePath_ = option.metaFilePath; + wrapper_ = option.wrapper; + parser_ = option.parser; + int ret = LoadFileMeta(); + if (ret < 0) { + LOG(ERROR) << "Load file meta from " << metaFilePath_ << " failed."; + return -1; + } + LOG(INFO) << "Init metafilemanager success."; + return 0; +} + +int NebdMetaFileManager::UpdateFileMeta(const std::string& fileName, + const NebdFileMeta& fileMeta) { + WriteLockGuard writeLock(rwLock_); + bool needUpdate = metaCache_.find(fileName) == metaCache_.end() + || fileMeta != metaCache_[fileName]; + // 如果元数据信息没发生变更，则不需要写文件 + if (!needUpdate) { + return 0; + } + + FileMetaMap tempMap = metaCache_; + tempMap[fileName] = fileMeta; + + int res = UpdateMetaFile(tempMap); + if (res != 0) { + LOG(ERROR) << "Update file meta failed, fileName: " << fileName; + return -1; + } + metaCache_ = std::move(tempMap); + LOG(INFO) << "Update file meta success. " + << "file meta: " << fileMeta; + return 0; +} + +int NebdMetaFileManager::RemoveFileMeta(const std::string& fileName) { + WriteLockGuard writeLock(rwLock_); + bool isExist = metaCache_.find(fileName) != metaCache_.end(); + if (!isExist) { + return 0; + } + FileMetaMap tempMap = metaCache_; + tempMap.erase(fileName); + + int res = UpdateMetaFile(tempMap); + if (res != 0) { + LOG(ERROR) << "Remove file meta failed, fileName: " << fileName; + return -1; + } + metaCache_ = std::move(tempMap); + LOG(INFO) << "Remove file meta success. " + << "file name: " << fileName; + return 0; +} + +int NebdMetaFileManager::UpdateMetaFile(const FileMetaMap& fileMetas) { + Json::Value root = parser_->ConvertFileMetasToJson(fileMetas); int res = AtomicWriteFile(root); if (res != 0) { - LOG(ERROR) << "AtomicWriteFile fail"; + LOG(ERROR) << "AtomicWriteFile fail."; return -1; } return 0; @@ -64,9 +118,9 @@ int NebdMetaFileManager::AtomicWriteFile(const Json::Value& root) { return 0; } -int NebdMetaFileManager::ListFileRecord(FileRecordMap* fileRecords) { - CHECK(fileRecords != nullptr) << "fileRecords is null."; - fileRecords->clear(); +int NebdMetaFileManager::LoadFileMeta() { + ReadLockGuard readLock(rwLock_); + FileMetaMap tempMetas; std::ifstream in(metaFilePath_, std::ios::binary); if (!in) { // 这里不应该返回错误，第一次初始化的时候文件可能还未创建 @@ -85,21 +139,32 @@ int NebdMetaFileManager::ListFileRecord(FileRecordMap* fileRecords) { return -1; } - int res = parser_->Parse(root, fileRecords); + int res = parser_->Parse(root, &tempMetas); if (res != 0) { LOG(ERROR) << "ConvertJsonToFileRecord fail"; return -1; } + metaCache_ = std::move(tempMetas); + return 0; +} + +int NebdMetaFileManager::ListFileMeta(std::vector* fileMetas) { + CHECK(fileMetas != nullptr) << "fileMetas is nullptr."; + ReadLockGuard readLock(rwLock_); + fileMetas->clear(); + for (const auto& metaPair : metaCache_) { + fileMetas->emplace_back(metaPair.second); + } return 0; } int NebdMetaFileParser::Parse(Json::Value root, - FileRecordMap* fileRecords) { - if (!fileRecords) { - LOG(ERROR) << "the argument fileRecords is null pointer"; + FileMetaMap* fileMetas) { + if (!fileMetas) { + LOG(ERROR) << "the argument fileMetas is null pointer"; return -1; } - fileRecords->clear(); + fileMetas->clear(); // 检验crc if (root[kCRC].isNull()) { LOG(ERROR) << "Parse json: " << root @@ -117,7 +182,6 @@ int NebdMetaFileParser::Parse(Json::Value root, return -1; } - // 没有volume字段 const auto& volumes = root[kVolumes]; if (volumes.isNull()) { @@ -128,14 +192,14 @@ int NebdMetaFileParser::Parse(Json::Value root, for (const auto& volume : volumes) { std::string fileName; int fd; - NebdFileRecord record; + NebdFileMeta meta; if (volume[kFileName].isNull()) { LOG(ERROR) << "Parse json: " << root << " fail, no filename"; return -1; } else { - record.fileName = volume[kFileName].asString(); + meta.fileName = volume[kFileName].asString(); } if (volume[kFd].isNull()) { @@ -143,42 +207,32 @@ int NebdMetaFileParser::Parse(Json::Value root, << " fail, no fd"; return -1; } else { - record.fd = volume[kFd].asInt(); + meta.fd = volume[kFd].asInt(); } - auto fileType = GetFileType(record.fileName); - record.fileInstance = NebdFileInstanceFactory::GetInstance(fileType); - if (!record.fileInstance) { - LOG(ERROR) << "Unknown file type, filename: " << record.fileName; - return -1; - } - - // 除了filename和fd的部分统一放到fileinstance的addition里面 + // 除了filename和fd的部分统一放到xattr里面 Json::Value::Members mem = volume.getMemberNames(); - AdditionType addition; + ExtendAttribute xattr; for (auto iter = mem.begin(); iter != mem.end(); iter++) { if (*iter == kFileName || *iter == kFd) { continue; } - record.fileInstance->addition.emplace( - *iter, volume[*iter].asString()); + meta.xattr.emplace(*iter, volume[*iter].asString()); } - fileRecords->emplace(record.fd, record); + fileMetas->emplace(meta.fileName, meta); } return 0; } -Json::Value NebdMetaFileParser::ConvertFileRecordsToJson( - const FileRecordMap& fileRecords) { +Json::Value NebdMetaFileParser::ConvertFileMetasToJson( + const FileMetaMap& fileMetas) { Json::Value volumes; - for (const auto& record : fileRecords) { + for (const auto& meta : fileMetas) { Json::Value volume; - volume[kFileName] = record.second.fileName; - volume[kFd] = record.second.fd; - if (record.second.fileInstance) { - for (const auto item : record.second.fileInstance->addition) { - volume[item.first] = item.second; - } + volume[kFileName] = meta.second.fileName; + volume[kFd] = meta.second.fd; + for (const auto item : meta.second.xattr) { + volume[item.first] = item.second; } volumes.append(volume); } diff --git a/src/part2/metafile_manager.h b/src/part2/metafile_manager.h index ae06b758c1..704435df1f 100644 --- a/src/part2/metafile_manager.h +++ b/src/part2/metafile_manager.h @@ -16,15 +16,20 @@ #include // NOLINT #include // NOLINT +#include "src/common/rw_lock.h" #include "src/common/posix_wrapper.h" #include "src/common/crc32.h" #include "src/part2/define.h" +#include "src/part2/util.h" namespace nebd { namespace server { using nebd::common::PosixWrapper; -using FileRecordMap = std::unordered_map; +using nebd::common::RWLock; +using nebd::common::WriteLockGuard; +using nebd::common::ReadLockGuard; +using FileMetaMap = std::unordered_map; const char kVolumes[] = "volumes"; const char kFileName[] = "filename"; @@ -34,35 +39,43 @@ const char kCRC[] = "crc"; class NebdMetaFileParser { public: int Parse(Json::Value root, - FileRecordMap* fileRecords); - Json::Value ConvertFileRecordsToJson(const FileRecordMap& fileRecords); + FileMetaMap* fileMetas); + Json::Value ConvertFileMetasToJson(const FileMetaMap& fileMetas); +}; + +struct NebdMetaFileManagerOption { + std::string metaFilePath = ""; + std::shared_ptr wrapper + = std::make_shared(); + std::shared_ptr parser + = std::make_shared(); }; class NebdMetaFileManager { public: - NebdMetaFileManager(const std::string& metaFilePath, - std::shared_ptr wrapper = - std::make_shared(), - std::shared_ptr parser = - std::make_shared()); + NebdMetaFileManager(); virtual ~NebdMetaFileManager(); - /** - * @brief 列出文件记录 - * - * @param fileRecord 持久化的文件记录 - * - * @return 成功返回0，失败返回-1 - * - */ - virtual int ListFileRecord(FileRecordMap* fileRecords); + // 初始化，主要从文件读取元数据信息并加载到内存 + virtual int Init(const NebdMetaFileManagerOption& option); + + // 列出文件记录 + virtual int ListFileMeta(std::vector* fileMetas); + + // 更新文件元数据 + virtual int UpdateFileMeta(const std::string& fileName, + const NebdFileMeta& fileMeta); - // 更新元数据文件 - virtual int UpdateMetaFile(const FileRecordMap& fileRecords); + // 删除文件元数据 + virtual int RemoveFileMeta(const std::string& fileName); private: // 原子写文件 int AtomicWriteFile(const Json::Value& root); + // 更新元数据文件并更新内存缓存 + int UpdateMetaFile(const FileMetaMap& fileMetas); + // 初始化从持久化文件读取到内存 + int LoadFileMeta(); private: // 元数据文件路径 @@ -71,6 +84,10 @@ class NebdMetaFileManager { std::shared_ptr wrapper_; // 用于解析Json格式的元数据 std::shared_ptr parser_; + // MetaFileManager 线程安全读写锁 + RWLock rwLock_; + // meta文件内存缓存 + FileMetaMap metaCache_; }; using MetaFileManagerPtr = std::shared_ptr; diff --git a/src/part2/nebd_server.cpp b/src/part2/nebd_server.cpp index 1a430b7dcf..78e5632c48 100644 --- a/src/part2/nebd_server.cpp +++ b/src/part2/nebd_server.cpp @@ -102,11 +102,7 @@ bool NebdServer::InitFileManager() { return false; } - FileRecordManagerPtr recordmanager = - std::make_shared(metaFileManager); - CHECK(recordmanager != nullptr) << "Init record manager failed."; - - fileManager_ = std::make_shared(recordmanager); + fileManager_ = std::make_shared(metaFileManager); CHECK(fileManager_ != nullptr) << "Init file manager failed."; int runRes = fileManager_->Run(); @@ -137,13 +133,23 @@ bool NebdServer::InitCurveRequestExecutor() { } MetaFileManagerPtr NebdServer::InitMetaFileManager() { - std::string metaFilePath; - bool getOk = conf_.GetStringValue(METAFILEPATH, &metaFilePath); + NebdMetaFileManagerOption option; + option.wrapper = std::make_shared(); + option.parser = std::make_shared(); + bool getOk = conf_.GetStringValue(METAFILEPATH, &option.metaFilePath); if (false == getOk) { return nullptr; } - return std::make_shared(metaFilePath); + MetaFileManagerPtr metaFileManager = + std::make_shared(); + CHECK(metaFileManager != nullptr) << "meta file manager is nullptr"; + int ret = metaFileManager->Init(option); + if (ret != 0) { + LOG(ERROR) << "Init meta file manager failed."; + return nullptr; + } + return metaFileManager; } bool NebdServer::InitHeartbeatManagerOption(HeartbeatManagerOption *opt) { diff --git a/src/part2/request_executor.h b/src/part2/request_executor.h index dee936486d..e3731cf7bf 100644 --- a/src/part2/request_executor.h +++ b/src/part2/request_executor.h @@ -19,7 +19,6 @@ namespace server { class CurveRequestExecutor; class CephRequestExecutor; -using AdditionType = std::map; // 具体RequestExecutor中会用到的文件实例上下文信息 // 该类为抽象结构，ceph或curve需要继承定义自己的FileInstance // RequestExecutor需要用到的文件上下文信息都记录到FileInstance内 @@ -29,7 +28,7 @@ class NebdFileInstance { virtual ~NebdFileInstance() {} // 需要持久化到文件的内容，以kv形式返回，例如curve open时返回的sessionid // 文件reopen的时候也会用到该内容 - AdditionType addition; + ExtendAttribute xattr; }; class NebdRequestExecutor { @@ -38,7 +37,7 @@ class NebdRequestExecutor { virtual ~NebdRequestExecutor() {} virtual std::shared_ptr Open(const std::string& filename) = 0; // NOLINT virtual std::shared_ptr Reopen( - const std::string& filename, AdditionType addtion) = 0; + const std::string& filename, const ExtendAttribute& xattr) = 0; virtual int Close(NebdFileInstance* fd) = 0; virtual int Extend(NebdFileInstance* fd, int64_t newsize) = 0; virtual int GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) = 0; diff --git a/src/part2/request_executor_ceph.cpp b/src/part2/request_executor_ceph.cpp index 1c7eb8bd97..9b839a0414 100644 --- a/src/part2/request_executor_ceph.cpp +++ b/src/part2/request_executor_ceph.cpp @@ -17,7 +17,7 @@ CephRequestExecutor::Open(const std::string& filename) { std::shared_ptr CephRequestExecutor::Reopen(const std::string& filename, - AdditionType addtion) { + const ExtendAttribute& xattr) { return nullptr; } diff --git a/src/part2/request_executor_ceph.h b/src/part2/request_executor_ceph.h index d9333fe99e..eeec382972 100644 --- a/src/part2/request_executor_ceph.h +++ b/src/part2/request_executor_ceph.h @@ -30,7 +30,7 @@ class CephRequestExecutor : public NebdRequestExecutor { ~CephRequestExecutor() {} std::shared_ptr Open(const std::string& filename) override; // NOLINT std::shared_ptr Reopen( - const std::string& filename, AdditionType addtion) override; + const std::string& filename, const ExtendAttribute& xattr) override; int Close(NebdFileInstance* fd) override; int Extend(NebdFileInstance* fd, int64_t newsize) override; int GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) override; diff --git a/src/part2/request_executor_curve.cpp b/src/part2/request_executor_curve.cpp index 78e0243e79..04b00c6410 100644 --- a/src/part2/request_executor_curve.cpp +++ b/src/part2/request_executor_curve.cpp @@ -59,7 +59,7 @@ CurveRequestExecutor::Open(const std::string& filename) { auto curveFileInstance = std::make_shared(); curveFileInstance->fd = fd; curveFileInstance->fileName = curveFileName; - curveFileInstance->addition["session"] = sessionId; + curveFileInstance->xattr["session"] = sessionId; return curveFileInstance; } @@ -68,20 +68,20 @@ CurveRequestExecutor::Open(const std::string& filename) { std::shared_ptr CurveRequestExecutor::Reopen(const std::string& filename, - AdditionType addtion) { + const ExtendAttribute& xattr) { std::string curveFileName = FileNameParser::Parse(filename); if (curveFileName.empty()) { return nullptr; } std::string newSessionId; - std::string oldSessionId = addtion["session"]; + std::string oldSessionId = xattr.at("session"); int fd = client_->ReOpen(curveFileName, oldSessionId, &newSessionId); if (fd >= 0) { auto curveFileInstance = std::make_shared(); curveFileInstance->fd = fd; curveFileInstance->fileName = curveFileName; - curveFileInstance->addition["session"] = newSessionId; + curveFileInstance->xattr["session"] = newSessionId; return curveFileInstance; } diff --git a/src/part2/request_executor_curve.h b/src/part2/request_executor_curve.h index 029fa1cb04..b84b9131e8 100644 --- a/src/part2/request_executor_curve.h +++ b/src/part2/request_executor_curve.h @@ -56,7 +56,7 @@ class CurveRequestExecutor : public NebdRequestExecutor { void Init(const std::shared_ptr &client); std::shared_ptr Open(const std::string& filename) override; // NOLINT std::shared_ptr Reopen( - const std::string& filename, AdditionType addtion) override; + const std::string& filename, const ExtendAttribute& xattr) override; int Close(NebdFileInstance* fd) override; int Extend(NebdFileInstance* fd, int64_t newsize) override; int GetInfo(NebdFileInstance* fd, NebdFileInfo* fileInfo) override; diff --git a/src/part2/util.cpp b/src/part2/util.cpp index e0bcb4d47a..cb6e36fb56 100644 --- a/src/part2/util.cpp +++ b/src/part2/util.cpp @@ -27,6 +27,19 @@ NebdFileType GetFileType(const std::string& fileName) { } } +std::string NebdFileStatus2Str(NebdFileStatus status) { + switch (status) { + case NebdFileStatus::CLOSED: + return "CLOSED"; + case NebdFileStatus::OPENED: + return "OPENED"; + case NebdFileStatus::DESTROYED: + return "DESTROYED"; + default: + return "UNKWOWN"; + } +} + std::string Op2Str(LIBAIO_OP op) { switch (op) { case LIBAIO_OP::LIBAIO_OP_READ: @@ -51,6 +64,25 @@ std::ostream& operator<<(std::ostream& os, const NebdServerAioContext& c) { return os; } +std::ostream& operator<<(std::ostream& os, const NebdFileMeta& meta) { + os << "[filename: " << meta.fileName << ", fd: " << meta.fd; + for (const auto& pair : meta.xattr) { + os << ", " << pair.first << ": " << pair.second; + } + os << "]"; + return os; +} + +bool operator==(const NebdFileMeta& lMeta, const NebdFileMeta& rMeta) { + return lMeta.fd == rMeta.fd && + lMeta.fileName == rMeta.fileName && + lMeta.xattr == rMeta.xattr; +} + +bool operator!=(const NebdFileMeta& lMeta, const NebdFileMeta& rMeta) { + return !(lMeta == rMeta); +} + int FdAllocator::GetNext() { std::unique_lock lock(mtx_); if (fd_ == INT_MAX || fd_ < 0) { diff --git a/src/part2/util.h b/src/part2/util.h index 2ed30c5140..1503fd755e 100644 --- a/src/part2/util.h +++ b/src/part2/util.h @@ -21,7 +21,13 @@ NebdFileType GetFileType(const std::string& fileName); std::string NebdFileType2Str(NebdFileType type); +std::string NebdFileStatus2Str(NebdFileStatus status); + std::ostream& operator<<(std::ostream& os, const NebdServerAioContext& c); +std::ostream& operator<<(std::ostream& os, const NebdFileMeta& meta); + +bool operator==(const NebdFileMeta& lMeta, const NebdFileMeta& rMeta); +bool operator!=(const NebdFileMeta& lMeta, const NebdFileMeta& rMeta); class FdAllocator { public: diff --git a/tests/common/rw_lock_test.cpp b/tests/common/rw_lock_test.cpp index b72e1ac2e5..316b7aebe6 100644 --- a/tests/common/rw_lock_test.cpp +++ b/tests/common/rw_lock_test.cpp @@ -84,5 +84,74 @@ TEST(RWLockTest, basic_test) { } } +TEST(BthreadRWLockTest, basic_test) { + BthreadRWLock rwlock; + { + ReadLockGuard readLockGuard(rwlock); + ASSERT_TRUE(true); + } + { + WriteLockGuard writeLockGuard(rwlock); + ASSERT_TRUE(true); + } + { + WriteLockGuard writeLockGuard(rwlock); + ASSERT_TRUE(true); + } + { + ReadLockGuard readLockGuard1(rwlock); + ReadLockGuard readLockGuard2(rwlock); + ASSERT_TRUE(true); + } + { + ReadLockGuard readLockGuard1(rwlock); + ReadLockGuard readLockGuard2(rwlock); + ReadLockGuard readLockGuard3(rwlock); + ReadLockGuard readLockGuard4(rwlock); + ASSERT_TRUE(true); + } + { + ReadLockGuard readLockGuard(rwlock); + ASSERT_EQ(EINVAL, rwlock.TryRDLock()); + ASSERT_EQ(EINVAL, rwlock.TryWRLock()); + /* be careful */ + rwlock.Unlock(); + } + { + WriteLockGuard writeLockGuard(rwlock); + ASSERT_EQ(EINVAL, rwlock.TryRDLock()); + ASSERT_EQ(EINVAL, rwlock.TryWRLock()); + } + uint64_t writeCnt = 0; + auto writeFunc = [&] { + for (uint64_t i = 0; i < 10000; ++i) { + WriteLockGuard writeLockGuard(rwlock); + ++writeCnt; + } + }; + auto readFunc = [&] { + for (uint64_t i = 0; i < 10000; ++i) { + ReadLockGuard readLockGuard(rwlock); + auto j = writeCnt + i; + } + }; + { + std::thread t1(writeFunc); + std::thread t2(readFunc); + std::thread t3(writeFunc); + std::thread t4(readFunc); + std::thread t5(writeFunc); + std::thread t6(writeFunc); + t1.join(); + t2.join(); + t3.join(); + t4.join(); + t5.join(); + t6.join(); + + ASSERT_EQ(4 * 10000, writeCnt); + } +} + } // namespace common } // namespace nebd diff --git a/tests/part2/BUILD b/tests/part2/BUILD index da40fd927c..ab52a9fb41 100644 --- a/tests/part2/BUILD +++ b/tests/part2/BUILD @@ -1,16 +1,3 @@ -cc_binary( - name = "filerecordmanager_test", - srcs = glob([ - "file_record_manager_unittest.cpp", - ]), - deps = [ - "//external:gflags", - "//src/part2:nebdserver", - "//tests/part2:mock_lib", - "@com_google_googletest//:gtest_main", - ], -) - cc_binary( name = "filemanager_test", srcs = glob([ diff --git a/tests/part2/file_manager_unittest.cpp b/tests/part2/file_manager_unittest.cpp index c0c4857600..81198beda2 100644 --- a/tests/part2/file_manager_unittest.cpp +++ b/tests/part2/file_manager_unittest.cpp @@ -11,8 +11,9 @@ #include #include "src/part2/file_manager.h" -#include "tests/part2/mock_filerecord_manager.h" +#include "src/part2/file_entity.h" #include "tests/part2/mock_request_executor.h" +#include "tests/part2/mock_metafile_manager.h" namespace nebd { namespace server { @@ -30,6 +31,16 @@ using ::testing::ElementsAre; using ::testing::SetArgPointee; using ::testing::SetArrayArgument; +enum class RequestType { + EXTEND = 0, + GETINFO = 1, + DISCARD = 2, + AIOREAD = 3, + AIOWRITE = 4, + FLUSH = 5, + INVALIDCACHE = 6, +}; + class FileManagerTest : public ::testing::Test { public: void SetUp() { @@ -37,818 +48,464 @@ class FileManagerTest : public ::testing::Test { mockInstance_ = std::make_shared(); executor_ = std::make_shared(); g_test_executor = executor_.get(); - filerecordManager_ = std::make_shared(); - fileManager_ = std::make_shared(filerecordManager_); - ON_CALL(*filerecordManager_, Exist(_)) - .WillByDefault(Return(false)); + metaFileManager_ = std::make_shared(); + fileManager_ = std::make_shared(metaFileManager_); } void TearDown() { delete aioContext_; } + using TestTask = std::function; + // 构造初始环境 + void InitEnv() { + NebdFileMeta meta; + meta.fd = 1; + meta.fileName = testFile1; + std::vector fileMetas; + fileMetas.emplace_back(meta); + + EXPECT_CALL(*metaFileManager_, ListFileMeta(_)) + .WillOnce(DoAll(SetArgPointee<0>(fileMetas), + Return(0))); + EXPECT_CALL(*executor_, Reopen(_, _)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileMeta(_, _)) + .WillOnce(Return(0)); + ASSERT_EQ(fileManager_->Run(), 0); + } + + void UnInitEnv() { + ASSERT_EQ(fileManager_->Fini(), 0); + } + + void ExpectCallRequest(RequestType type, int ret) { + switch (type) { + case RequestType::EXTEND: + EXPECT_CALL(*executor_, Extend(_, _)).WillOnce(Return(ret)); + break; + case RequestType::GETINFO: + EXPECT_CALL(*executor_, GetInfo(_, _)).WillOnce(Return(ret)); + break; + case RequestType::DISCARD: + EXPECT_CALL(*executor_, Discard(_, _)).WillOnce(Return(ret)); + break; + case RequestType::AIOREAD: + EXPECT_CALL(*executor_, AioRead(_, _)).WillOnce(Return(ret)); + break; + case RequestType::AIOWRITE: + EXPECT_CALL(*executor_, AioWrite(_, _)).WillOnce(Return(ret)); + break; + case RequestType::FLUSH: + EXPECT_CALL(*executor_, Flush(_, _)).WillOnce(Return(ret)); + break; + case RequestType::INVALIDCACHE: + EXPECT_CALL(*executor_, InvalidCache(_)).WillOnce(Return(ret)); + break; + } + } + + void RequestSuccssTest(RequestType type, TestTask task) { + InitEnv(); + NebdFileEntityPtr entity1 = fileManager_->GetFileEntity(1); + ASSERT_NE(nullptr, entity1); + ASSERT_EQ(entity1->GetFileStatus(), NebdFileStatus::OPENED); + + // 文件状态为OPENED + ExpectCallRequest(type, 0); + ASSERT_EQ(0, task(1)); + ASSERT_EQ(entity1->GetFileStatus(), NebdFileStatus::OPENED); + + EXPECT_CALL(*executor_, Close(NotNull())) + .WillOnce(Return(0)); + ASSERT_EQ(entity1->Close(false), 0); + ASSERT_EQ(entity1->GetFileStatus(), NebdFileStatus::CLOSED); + // 文件状态为CLOSED + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileMeta(testFile1, _)) + .WillOnce(Return(0)); + ExpectCallRequest(type, 0); + ASSERT_EQ(0, task(1)); + ASSERT_EQ(entity1->GetFileStatus(), NebdFileStatus::OPENED); + UnInitEnv(); + } + + void RequestFailTest(RequestType type, TestTask task) { + InitEnv(); + // 将文件close + NebdFileEntityPtr entity1 = fileManager_->GetFileEntity(1); + ASSERT_NE(nullptr, entity1); + EXPECT_CALL(*executor_, Close(NotNull())) + .WillOnce(Return(0)); + ASSERT_EQ(entity1->Close(false), 0); + ASSERT_EQ(entity1->GetFileStatus(), NebdFileStatus::CLOSED); + + // open文件失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(nullptr)); + EXPECT_CALL(*metaFileManager_, UpdateFileMeta(testFile1, _)) + .Times(0); + ASSERT_EQ(-1, task(1)); + ASSERT_EQ(entity1->GetFileStatus(), NebdFileStatus::CLOSED); + + // 更新元数据文件失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileMeta(testFile1, _)) + .WillOnce(Return(-1)); + EXPECT_CALL(*executor_, Close(NotNull())) + .WillOnce(Return(0)); + ASSERT_EQ(-1, task(1)); + ASSERT_EQ(entity1->GetFileStatus(), NebdFileStatus::CLOSED); + + // 执行处理函数失败 + EXPECT_CALL(*executor_, Open(testFile1)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileMeta(testFile1, _)) + .WillOnce(Return(0)); + ExpectCallRequest(type, -1); + ASSERT_EQ(-1, task(1)); + ASSERT_EQ(entity1->GetFileStatus(), NebdFileStatus::OPENED); + + // 将文件状态置为DESTROYED + EXPECT_CALL(*executor_, Close(NotNull())) + .WillOnce(Return(0)); + EXPECT_CALL(*metaFileManager_, RemoveFileMeta(testFile1)) + .WillOnce(Return(0)); + ASSERT_EQ(entity1->Close(true), 0); + ASSERT_EQ(entity1->GetFileStatus(), NebdFileStatus::DESTROYED); + EXPECT_CALL(*executor_, Open(testFile1)) + .Times(0); + ASSERT_EQ(-1, task(1)); + + // 直接将文件删除 + ASSERT_EQ(0, fileManager_->Close(1, true)); + ASSERT_EQ(nullptr, fileManager_->GetFileEntity(1)); + ASSERT_EQ(-1, task(1)); + UnInitEnv(); + } + protected: std::shared_ptr fileManager_; - std::shared_ptr filerecordManager_; + std::shared_ptr metaFileManager_; std::shared_ptr executor_; std::shared_ptr mockInstance_; NebdServerAioContext* aioContext_; }; -// TODO(yyk): 后续重构单测，消除重复代码 +TEST_F(FileManagerTest, RunTest) { + NebdFileMeta meta; + meta.fd = 1; + meta.fileName = testFile1; + std::vector fileMetas; + fileMetas.emplace_back(meta); + + EXPECT_CALL(*metaFileManager_, ListFileMeta(_)) + .WillOnce(DoAll(SetArgPointee<0>(fileMetas), + Return(0))); + EXPECT_CALL(*executor_, Reopen(_, _)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileMeta(_, _)) + .WillOnce(Return(0)); + ASSERT_EQ(fileManager_->Run(), 0); + // 重复run返回失败 + ASSERT_EQ(fileManager_->Run(), -1); + + // 校验结果 + FileEntityMap entityMap = fileManager_->GetFileEntityMap(); + ASSERT_EQ(1, entityMap.size()); + ASSERT_NE(nullptr, entityMap[meta.fd]); +} + +TEST_F(FileManagerTest, RunFailTest) { + NebdFileMeta meta; + meta.fd = 1; + meta.fileName = testFile1; + std::vector fileMetas; + fileMetas.emplace_back(meta); + + // list file meta失败 + EXPECT_CALL(*metaFileManager_, ListFileMeta(_)) + .WillOnce(Return(-1)); + ASSERT_EQ(fileManager_->Run(), -1); + + // reopen失败不影响Run成功 + EXPECT_CALL(*metaFileManager_, ListFileMeta(_)) + .WillOnce(DoAll(SetArgPointee<0>(fileMetas), + Return(0))); + EXPECT_CALL(*executor_, Reopen(_, _)) + .WillOnce(Return(nullptr)); + ASSERT_EQ(fileManager_->Run(), 0); + ASSERT_EQ(fileManager_->Fini(), 0); + + // 更新metafile失败不影响Run成功 + EXPECT_CALL(*metaFileManager_, ListFileMeta(_)) + .WillOnce(DoAll(SetArgPointee<0>(fileMetas), + Return(0))); + EXPECT_CALL(*executor_, Reopen(_, _)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileMeta(_, _)) + .WillOnce(Return(-1)); + EXPECT_CALL(*executor_, Close(NotNull())) + .Times(1); + ASSERT_EQ(fileManager_->Run(), 0); +} TEST_F(FileManagerTest, OpenTest) { + InitEnv(); // open一个不存在的文件 - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillOnce(Return(false)); - EXPECT_CALL(*executor_, Open(testFile1)) + EXPECT_CALL(*executor_, Open(testFile2)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(true)); - int fd = fileManager_->Open(testFile1); - ASSERT_EQ(fd, 1); + EXPECT_CALL(*metaFileManager_, UpdateFileMeta(testFile2, _)) + .WillOnce(Return(0)); + int fd = fileManager_->Open(testFile2); + ASSERT_EQ(fd, 2); // 重复open - NebdFileRecord fileRecord; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.fd = 1; - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillOnce(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*executor_, Open(_)) - .Times(0); - fd = fileManager_->Open(testFile1); - ASSERT_EQ(fd, 1); - - // open 已经close的文件 - fileRecord.status = NebdFileStatus::CLOSED; - fileRecord.fd = 1; - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillOnce(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*executor_, Open(testFile1)) + fd = fileManager_->Open(testFile2); + ASSERT_EQ(fd, 2); + + NebdFileEntityPtr entity2 = fileManager_->GetFileEntity(2); + ASSERT_NE(entity2, nullptr); + ASSERT_EQ(entity2->GetFileStatus(), NebdFileStatus::OPENED); + + EXPECT_CALL(*executor_, Close(_)) + .WillOnce(Return(0)); + ASSERT_EQ(entity2->Close(false), 0); + ASSERT_EQ(entity2->GetFileStatus(), NebdFileStatus::CLOSED); + // open 已经close的文件, fd不变 + EXPECT_CALL(*executor_, Open(testFile2)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(true)); - fd = fileManager_->Open(testFile1); + EXPECT_CALL(*metaFileManager_, UpdateFileMeta(testFile2, _)) + .WillOnce(Return(0)); + fd = fileManager_->Open(testFile2); ASSERT_EQ(fd, 2); + ASSERT_EQ(entity2->GetFileStatus(), NebdFileStatus::OPENED); } TEST_F(FileManagerTest, OpenFailTest) { + InitEnv(); // 调用后端open接口时出错 - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillOnce(Return(false)); - EXPECT_CALL(*executor_, Open(testFile1)) + EXPECT_CALL(*executor_, Open(testFile2)) .WillOnce(Return(nullptr)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) + EXPECT_CALL(*metaFileManager_, UpdateFileMeta(testFile2, _)) .Times(0); - int fd = fileManager_->Open(testFile1); + int fd = fileManager_->Open(testFile2); ASSERT_EQ(fd, -1); // 持久化元数据信息失败 - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillOnce(Return(false)); - EXPECT_CALL(*executor_, Open(testFile1)) + EXPECT_CALL(*executor_, Open(testFile2)) .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(false)); - EXPECT_CALL(*executor_, Close(mockInstance_.get())) - .WillOnce(Return(0)); - fd = fileManager_->Open(testFile1); + EXPECT_CALL(*metaFileManager_, UpdateFileMeta(testFile2, _)) + .WillOnce(Return(-1)); + EXPECT_CALL(*executor_, Close(_)) + .Times(1); + fd = fileManager_->Open(testFile2); ASSERT_EQ(fd, -1); // Open一个非法的filename - EXPECT_CALL(*filerecordManager_, GetRecord(unknownFile, _)) - .WillOnce(Return(false)); EXPECT_CALL(*executor_, Open(_)) .Times(0); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .Times(0); fd = fileManager_->Open(unknownFile); ASSERT_EQ(fd, -1); } -TEST_F(FileManagerTest, LoadTest) { - // 初始化从metafile返回的元数据 - FileRecordMap fileRecords; - NebdFileRecord record1; - record1.fileName = testFile1; - record1.fd = 1; - record1.fileInstance = mockInstance_; - fileRecords.emplace(1, record1); - - EXPECT_CALL(*filerecordManager_, Load()) - .WillOnce(Return(0)); - EXPECT_CALL(*filerecordManager_, ListRecords()) - .WillOnce(Return(fileRecords)); - EXPECT_CALL(*executor_, Reopen(testFile1, _)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(true)); - ASSERT_EQ(0, fileManager_->Load()); -} - -TEST_F(FileManagerTest, LoadFailTest) { - // 初始化从metafile返回的元数据 - FileRecordMap fileRecords; - NebdFileRecord record1; - NebdFileRecord record2; - NebdFileRecord record3; - record1.fileName = testFile1; - record1.fd = 1; - record1.fileInstance = mockInstance_; - record2.fileName = testFile2; - record2.fd = 2; - record2.fileInstance = mockInstance_; - record3.fileName = unknownFile; - record3.fd = 3; - record3.fileInstance = mockInstance_; - fileRecords.emplace(1, record1); - fileRecords.emplace(2, record2); - fileRecords.emplace(3, record3); - - // load 失败 - EXPECT_CALL(*filerecordManager_, Load()) - .WillOnce(Return(-1)); - ASSERT_EQ(-1, fileManager_->Load()); - - // 其他失败都会被忽略，不会导致load失败 - EXPECT_CALL(*filerecordManager_, Load()) - .WillOnce(Return(0)); - EXPECT_CALL(*filerecordManager_, ListRecords()) - .WillOnce(Return(fileRecords)); - // uodate record 失败，不会导致load失败 - EXPECT_CALL(*executor_, Reopen(testFile1, _)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(false)); - // reopen失败，不会导致load失败 - EXPECT_CALL(*executor_, Reopen(testFile2, _)) - .WillOnce(Return(nullptr)); - // 无法识别的文件不会被打开，但不会导致load失败 - EXPECT_CALL(*executor_, Reopen(unknownFile, _)) - .Times(0); - ASSERT_EQ(0, fileManager_->Load()); -} - TEST_F(FileManagerTest, CloseTest) { + InitEnv(); // 指定的fd不存在,直接返回成功 - EXPECT_CALL(*filerecordManager_, GetRecord(3, _)) - .WillOnce(Return(false)); - ASSERT_EQ(0, fileManager_->Close(3, true)); - - // fd存在,文件状态为opened,removerecord为true - NebdFileRecord fileRecord; - fileRecord.fd = 1; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.executor = executor_.get(); - fileRecord.fileInstance = mockInstance_; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .Times(2) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*executor_, Close(NotNull())) - .WillOnce(Return(0)); - EXPECT_CALL(*filerecordManager_, UpdateFileStatus(1, _)) - .WillOnce(Return(true)); - EXPECT_CALL(*filerecordManager_, RemoveRecord(1)) - .WillOnce(Return(true)); - ASSERT_EQ(0, fileManager_->Close(1, true)); + ASSERT_EQ(nullptr, fileManager_->GetFileEntity(2)); + ASSERT_EQ(0, fileManager_->Close(2, true)); - // fd存在,文件状态为opened,removerecord为false - fileRecord.fd = 1; - fileRecord.status = NebdFileStatus::OPENED; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .Times(2) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); + NebdFileEntityPtr entity1 = fileManager_->GetFileEntity(1); + ASSERT_NE(nullptr, entity1); + ASSERT_EQ(entity1->GetFileStatus(), NebdFileStatus::OPENED); + // 文件存在，且文件状态为OPENED，removeRecord为false EXPECT_CALL(*executor_, Close(NotNull())) .WillOnce(Return(0)); - EXPECT_CALL(*filerecordManager_, UpdateFileStatus(1, _)) - .WillOnce(Return(true)); - EXPECT_CALL(*filerecordManager_, RemoveRecord(1)) + EXPECT_CALL(*metaFileManager_, RemoveFileMeta(testFile1)) .Times(0); ASSERT_EQ(0, fileManager_->Close(1, false)); + ASSERT_EQ(entity1->GetFileStatus(), NebdFileStatus::CLOSED); - // 文件状态为closed - fileRecord.fd = 1; - fileRecord.status = NebdFileStatus::CLOSED; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .Times(2) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); + // 文件存在，文件状态为CLOSED，removeRecord为false EXPECT_CALL(*executor_, Close(NotNull())) .Times(0); - EXPECT_CALL(*filerecordManager_, UpdateFileStatus(_, _)) + EXPECT_CALL(*metaFileManager_, RemoveFileMeta(testFile1)) + .Times(0); + ASSERT_EQ(0, fileManager_->Close(1, false)); + ASSERT_EQ(entity1->GetFileStatus(), NebdFileStatus::CLOSED); + + // 文件存在，文件状态为CLOSED，removeRecord为true + EXPECT_CALL(*executor_, Close(NotNull())) .Times(0); - EXPECT_CALL(*filerecordManager_, RemoveRecord(1)) - .WillOnce(Return(true)); + EXPECT_CALL(*metaFileManager_, RemoveFileMeta(testFile1)) + .WillOnce(Return(0)); ASSERT_EQ(0, fileManager_->Close(1, true)); + ASSERT_EQ(entity1->GetFileStatus(), NebdFileStatus::DESTROYED); + ASSERT_EQ(nullptr, fileManager_->GetFileEntity(1)); + + EXPECT_CALL(*executor_, Open(testFile2)) + .WillOnce(Return(mockInstance_)); + EXPECT_CALL(*metaFileManager_, UpdateFileMeta(testFile2, _)) + .WillOnce(Return(0)); + int fd = fileManager_->Open(testFile2); + ASSERT_EQ(fd, 2); + NebdFileEntityPtr entity2 = fileManager_->GetFileEntity(2); + ASSERT_NE(entity2, nullptr); + ASSERT_EQ(entity2->GetFileStatus(), NebdFileStatus::OPENED); + // 文件存在，文件状态为OPENED，removeRecord为true + EXPECT_CALL(*executor_, Close(NotNull())) + .WillOnce(Return(0)); + EXPECT_CALL(*metaFileManager_, RemoveFileMeta(testFile2)) + .WillOnce(Return(0)); + ASSERT_EQ(0, fileManager_->Close(fd, true)); + ASSERT_EQ(nullptr, fileManager_->GetFileEntity(1)); } TEST_F(FileManagerTest, CloseFailTest) { - NebdFileRecord fileRecord; - fileRecord.fd = 1; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.executor = executor_.get(); - fileRecord.fileInstance = mockInstance_; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); + InitEnv(); + NebdFileEntityPtr entity1 = fileManager_->GetFileEntity(1); + ASSERT_NE(nullptr, entity1); + ASSERT_EQ(entity1->GetFileStatus(), NebdFileStatus::OPENED); // executor close 失败 EXPECT_CALL(*executor_, Close(NotNull())) .WillOnce(Return(-1)); + EXPECT_CALL(*metaFileManager_, RemoveFileMeta(testFile1)) + .Times(0); ASSERT_EQ(-1, fileManager_->Close(1, true)); + ASSERT_NE(nullptr, fileManager_->GetFileEntity(1)); + ASSERT_EQ(entity1->GetFileStatus(), NebdFileStatus::OPENED); - // remove record 失败 + // remove file meta 失败 EXPECT_CALL(*executor_, Close(NotNull())) .WillOnce(Return(0)); - EXPECT_CALL(*filerecordManager_, UpdateFileStatus(1, _)) - .WillOnce(Return(true)); - EXPECT_CALL(*filerecordManager_, RemoveRecord(1)) - .WillOnce(Return(false)); + EXPECT_CALL(*metaFileManager_, RemoveFileMeta(testFile1)) + .WillOnce(Return(-1)); ASSERT_EQ(-1, fileManager_->Close(1, true)); + ASSERT_NE(nullptr, fileManager_->GetFileEntity(1)); + ASSERT_EQ(entity1->GetFileStatus(), NebdFileStatus::CLOSED); } TEST_F(FileManagerTest, ExtendTest) { - // 文件是opened状态 - NebdFileRecord fileRecord; - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.executor = executor_.get(); - fileRecord.fileInstance = mockInstance_; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*executor_, Extend(NotNull(), 4096)) - .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_->Extend(1, 4096)); - - // 文件状态为closed，会重新open文件 - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::CLOSED; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - // 重新open该文件，fd不变 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(true)); - EXPECT_CALL(*executor_, Extend(NotNull(), 4096)) - .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_->Extend(1, 4096)); -} - -TEST_F(FileManagerTest, ExtendFaileTest) { - // 文件是opened状态 - NebdFileRecord fileRecord; - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.executor = executor_.get(); - fileRecord.fileInstance = mockInstance_; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*executor_, Extend(NotNull(), 4096)) - .WillOnce(Return(-1)); - ASSERT_EQ(-1, fileManager_->Extend(1, 4096)); - - // 文件状态为closed，会重新open文件 - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::CLOSED; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - // 重新open文件时，open失败 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(nullptr)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .Times(0); - EXPECT_CALL(*executor_, Extend(NotNull(), 4096)) - .Times(0); - ASSERT_EQ(-1, fileManager_->Extend(1, 4096)); - // 重新open文件时，更新meta file失败 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(false)); - EXPECT_CALL(*executor_, Extend(NotNull(), 4096)) - .Times(0); - ASSERT_EQ(-1, fileManager_->Extend(1, 4096)); + auto task = [&](int fd)->int { + return fileManager_->Extend(fd, 4096); + }; + RequestSuccssTest(RequestType::EXTEND, task); + RequestFailTest(RequestType::EXTEND, task); } TEST_F(FileManagerTest, GetInfoTest) { NebdFileInfo fileInfo; - // 文件是opened状态 - NebdFileRecord fileRecord; - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.executor = executor_.get(); - fileRecord.fileInstance = mockInstance_; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*executor_, GetInfo(NotNull(), &fileInfo)) - .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_->GetInfo(1, &fileInfo)); - - // 文件状态为closed，会重新open文件 - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::CLOSED; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - // 重新open该文件，fd不变 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(true)); - EXPECT_CALL(*executor_, GetInfo(NotNull(), &fileInfo)) - .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_->GetInfo(1, &fileInfo)); -} - -TEST_F(FileManagerTest, GetInfoFaileTest) { - NebdFileInfo fileInfo; - // 文件是opened状态 - NebdFileRecord fileRecord; - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.executor = executor_.get(); - fileRecord.fileInstance = mockInstance_; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*executor_, GetInfo(NotNull(), &fileInfo)) - .WillOnce(Return(-1)); - ASSERT_EQ(-1, fileManager_->GetInfo(1, &fileInfo)); - - // 文件状态为closed，会重新open文件 - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::CLOSED; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - // 重新open文件时，open失败 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(nullptr)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .Times(0); - EXPECT_CALL(*executor_, GetInfo(NotNull(), _)) - .Times(0); - ASSERT_EQ(-1, fileManager_->GetInfo(1, &fileInfo)); - // 重新open文件时，更新meta file失败 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(false)); - EXPECT_CALL(*executor_, GetInfo(NotNull(), _)) - .Times(0); - ASSERT_EQ(-1, fileManager_->GetInfo(1, &fileInfo)); + auto task = [&](int fd)->int { + return fileManager_->GetInfo(fd, &fileInfo); + }; + RequestSuccssTest(RequestType::GETINFO, task); + RequestFailTest(RequestType::GETINFO, task); } TEST_F(FileManagerTest, InvalidCacheTest) { - // 文件是opened状态 - NebdFileRecord fileRecord; - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.executor = executor_.get(); - fileRecord.fileInstance = mockInstance_; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*executor_, InvalidCache(NotNull())) - .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_->InvalidCache(1)); - - // 文件状态为closed，会重新open文件 - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::CLOSED; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - // 重新open该文件，fd不变 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(true)); - EXPECT_CALL(*executor_, InvalidCache(NotNull())) - .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_->InvalidCache(1)); -} - -TEST_F(FileManagerTest, InvalidCacheFaileTest) { - // 文件是opened状态 - NebdFileRecord fileRecord; - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.executor = executor_.get(); - fileRecord.fileInstance = mockInstance_; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*executor_, InvalidCache(NotNull())) - .WillOnce(Return(-1)); - ASSERT_EQ(-1, fileManager_->InvalidCache(1)); - - // 文件状态为closed，会重新open文件 - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::CLOSED; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - // 重新open文件时，open失败 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(nullptr)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .Times(0); - EXPECT_CALL(*executor_, InvalidCache(NotNull())) - .Times(0); - ASSERT_EQ(-1, fileManager_->InvalidCache(1)); - // 重新open文件时，更新meta file失败 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(false)); - EXPECT_CALL(*executor_, InvalidCache(NotNull())) - .Times(0); - ASSERT_EQ(-1, fileManager_->InvalidCache(1)); + auto task = [&](int fd)->int { + return fileManager_->InvalidCache(fd); + }; + RequestSuccssTest(RequestType::INVALIDCACHE, task); + RequestFailTest(RequestType::INVALIDCACHE, task); } TEST_F(FileManagerTest, AioReadTest) { - // 文件是opened状态 - NebdFileRecord fileRecord; - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.executor = executor_.get(); - fileRecord.fileInstance = mockInstance_; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) - .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_->AioRead(1, aioContext_)); - - // 文件状态为closed，会重新open文件 - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::CLOSED; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - // 重新open该文件，fd不变 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(true)); - EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) - .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_->AioRead(1, aioContext_)); -} - -TEST_F(FileManagerTest, AioReadFaileTest) { - // 文件不存在 - EXPECT_CALL(*filerecordManager_, GetRecord(10, _)) - .WillOnce(Return(false)); - EXPECT_CALL(*executor_, AioRead(_, _)) - .Times(0); - ASSERT_EQ(-1, fileManager_->AioRead(10, aioContext_)); - - // 文件是opened状态 - NebdFileRecord fileRecord; - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.executor = executor_.get(); - fileRecord.fileInstance = mockInstance_; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillOnce(DoAll(SetArgPointee<1>(fileRecord), Return(true))) - .WillOnce(Return(false)); - ASSERT_EQ(-1, fileManager_->AioRead(1, aioContext_)); - - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) - .WillOnce(Return(-1)); - ASSERT_EQ(-1, fileManager_->AioRead(1, aioContext_)); - - // 文件状态为closed，会重新open文件 - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::CLOSED; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - // 重新open文件时，open失败 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(nullptr)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .Times(0); - EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) - .Times(0); - ASSERT_EQ(-1, fileManager_->AioRead(1, aioContext_)); - // 重新open文件时，更新meta file失败 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(false)); - EXPECT_CALL(*executor_, AioRead(NotNull(), aioContext_)) - .Times(0); - ASSERT_EQ(-1, fileManager_->AioRead(1, aioContext_)); + NebdServerAioContext aioContext; + auto task = [&](int fd)->int { + int ret = fileManager_->AioRead(fd, &aioContext); + if (ret < 0) { + if (aioContext.done != nullptr) { + --ret; + brpc::ClosureGuard doneGuard(aioContext.done); + aioContext.done = nullptr; + } + } else { + if (aioContext.done == nullptr) { + --ret; + } else { + brpc::ClosureGuard doneGuard(aioContext.done); + aioContext.done = nullptr; + } + } + return ret; + }; + RequestSuccssTest(RequestType::AIOREAD, task); + RequestFailTest(RequestType::AIOREAD, task); } TEST_F(FileManagerTest, AioWriteTest) { - // 文件是opened状态 - NebdFileRecord fileRecord; - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.executor = executor_.get(); - fileRecord.fileInstance = mockInstance_; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) - .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_->AioWrite(1, aioContext_)); - - // 文件状态为closed，会重新open文件 - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::CLOSED; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - // 重新open该文件，fd不变 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(true)); - EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) - .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_->AioWrite(1, aioContext_)); -} - -TEST_F(FileManagerTest, AioWriteFaileTest) { - // 文件不存在 - EXPECT_CALL(*filerecordManager_, GetRecord(10, _)) - .WillOnce(Return(false)); - EXPECT_CALL(*executor_, AioWrite(_, _)) - .Times(0); - ASSERT_EQ(-1, fileManager_->AioWrite(10, aioContext_)); - - // 文件是opened状态 - NebdFileRecord fileRecord; - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.executor = executor_.get(); - fileRecord.fileInstance = mockInstance_; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) - .WillOnce(Return(-1)); - ASSERT_EQ(-1, fileManager_->AioWrite(1, aioContext_)); - - // 文件状态为closed，会重新open文件 - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::CLOSED; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - // 重新open文件时，open失败 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(nullptr)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .Times(0); - EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) - .Times(0); - ASSERT_EQ(-1, fileManager_->AioWrite(1, aioContext_)); - // 重新open文件时，更新meta file失败 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(false)); - EXPECT_CALL(*executor_, AioWrite(NotNull(), aioContext_)) - .Times(0); - ASSERT_EQ(-1, fileManager_->AioWrite(1, aioContext_)); + NebdServerAioContext aioContext; + auto task = [&](int fd)->int { + int ret = fileManager_->AioWrite(fd, &aioContext); + if (ret < 0) { + if (aioContext.done != nullptr) { + --ret; + brpc::ClosureGuard doneGuard(aioContext.done); + aioContext.done = nullptr; + } + } else { + if (aioContext.done == nullptr) { + --ret; + } else { + brpc::ClosureGuard doneGuard(aioContext.done); + aioContext.done = nullptr; + } + } + return ret; + }; + RequestSuccssTest(RequestType::AIOWRITE, task); + RequestFailTest(RequestType::AIOWRITE, task); } TEST_F(FileManagerTest, DiscardTest) { - // 文件是opened状态 - NebdFileRecord fileRecord; - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.executor = executor_.get(); - fileRecord.fileInstance = mockInstance_; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) - .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_->Discard(1, aioContext_)); - - // 文件状态为closed，会重新open文件 - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::CLOSED; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - // 重新open该文件，fd不变 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(true)); - EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) - .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_->Discard(1, aioContext_)); -} - -TEST_F(FileManagerTest, DiscardFaileTest) { - // 文件不存在 - EXPECT_CALL(*filerecordManager_, GetRecord(10, _)) - .WillOnce(Return(false)); - EXPECT_CALL(*executor_, Discard(_, _)) - .Times(0); - ASSERT_EQ(-1, fileManager_->Discard(10, aioContext_)); - - // 文件是opened状态 - NebdFileRecord fileRecord; - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.executor = executor_.get(); - fileRecord.fileInstance = mockInstance_; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) - .WillOnce(Return(-1)); - ASSERT_EQ(-1, fileManager_->Discard(1, aioContext_)); - - // 文件状态为closed，会重新open文件 - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::CLOSED; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - // 重新open文件时，open失败 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(nullptr)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .Times(0); - EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) - .Times(0); - ASSERT_EQ(-1, fileManager_->Discard(1, aioContext_)); - // 重新open文件时，更新meta file失败 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(false)); - EXPECT_CALL(*executor_, Discard(NotNull(), aioContext_)) - .Times(0); - ASSERT_EQ(-1, fileManager_->Discard(1, aioContext_)); + NebdServerAioContext aioContext; + auto task = [&](int fd)->int { + int ret = fileManager_->Discard(fd, &aioContext); + if (ret < 0) { + if (aioContext.done != nullptr) { + --ret; + brpc::ClosureGuard doneGuard(aioContext.done); + aioContext.done = nullptr; + } + } else { + if (aioContext.done == nullptr) { + --ret; + } else { + brpc::ClosureGuard doneGuard(aioContext.done); + aioContext.done = nullptr; + } + } + return ret; + }; + RequestSuccssTest(RequestType::DISCARD, task); + RequestFailTest(RequestType::DISCARD, task); } TEST_F(FileManagerTest, FlushTest) { - // 文件是opened状态 - NebdFileRecord fileRecord; - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.executor = executor_.get(); - fileRecord.fileInstance = mockInstance_; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) - .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_->Flush(1, aioContext_)); - - // 文件状态为closed，会重新open文件 - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::CLOSED; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - // 重新open该文件，fd不变 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(true)); - EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) - .WillOnce(Return(0)); - ASSERT_EQ(0, fileManager_->Flush(1, aioContext_)); -} - -TEST_F(FileManagerTest, FlushFaileTest) { - // 文件不存在 - EXPECT_CALL(*filerecordManager_, GetRecord(10, _)) - .WillOnce(Return(false)); - EXPECT_CALL(*executor_, Flush(_, _)) - .Times(0); - ASSERT_EQ(-1, fileManager_->Flush(10, aioContext_)); - - // 文件是opened状态 - NebdFileRecord fileRecord; - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::OPENED; - fileRecord.executor = executor_.get(); - fileRecord.fileInstance = mockInstance_; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) - .WillOnce(Return(-1)); - ASSERT_EQ(-1, fileManager_->Flush(1, aioContext_)); - - // 文件状态为closed，会重新open文件 - fileRecord.fd = 1; - fileRecord.fileName = testFile1; - fileRecord.status = NebdFileStatus::CLOSED; - EXPECT_CALL(*filerecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - EXPECT_CALL(*filerecordManager_, GetRecord(testFile1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(fileRecord), - Return(true))); - // 重新open文件时，open失败 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(nullptr)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .Times(0); - EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) - .Times(0); - ASSERT_EQ(-1, fileManager_->Flush(1, aioContext_)); - // 重新open文件时，更新meta file失败 - EXPECT_CALL(*executor_, Open(testFile1)) - .WillOnce(Return(mockInstance_)); - EXPECT_CALL(*filerecordManager_, UpdateRecord(_)) - .WillOnce(Return(false)); - EXPECT_CALL(*executor_, Flush(NotNull(), aioContext_)) - .Times(0); - ASSERT_EQ(-1, fileManager_->Flush(1, aioContext_)); + NebdServerAioContext aioContext; + auto task = [&](int fd)->int { + int ret = fileManager_->Flush(fd, &aioContext); + if (ret < 0) { + if (aioContext.done != nullptr) { + --ret; + brpc::ClosureGuard doneGuard(aioContext.done); + aioContext.done = nullptr; + } + } else { + if (aioContext.done == nullptr) { + --ret; + } else { + brpc::ClosureGuard doneGuard(aioContext.done); + aioContext.done = nullptr; + } + } + return ret; + }; + RequestSuccssTest(RequestType::FLUSH, task); + RequestFailTest(RequestType::FLUSH, task); } } // namespace server diff --git a/tests/part2/file_record_manager_unittest.cpp b/tests/part2/file_record_manager_unittest.cpp deleted file mode 100644 index 83a1803e94..0000000000 --- a/tests/part2/file_record_manager_unittest.cpp +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Project: nebd - * Created Date: Monday January 20th 2020 - * Author: yangyaokai - * Copyright (c) 2020 netease - */ - -#include -#include -#include -#include - -#include "src/part2/filerecord_manager.h" -#include "tests/part2/mock_metafile_manager.h" - -namespace nebd { -namespace server { - -using ::testing::_; -using ::testing::Return; -using ::testing::NotNull; -using ::testing::DoAll; -using ::testing::ReturnArg; -using ::testing::ElementsAre; -using ::testing::SetArgPointee; -using ::testing::SetArrayArgument; - -class FileRecordManagerTest : public ::testing::Test { - public: - void SetUp() { - metaFileManager_ = std::make_shared(); - recordManager_ = std::make_shared(metaFileManager_); - } - void TearDown() {} - - protected: - std::shared_ptr recordManager_; - std::shared_ptr metaFileManager_; -}; - -TEST_F(FileRecordManagerTest, BasicTest) { - int fd1 = 1; - std::string fileName1 = "file1"; - NebdFileRecord fileRecord1; - - // 初始测试 - ASSERT_FALSE(recordManager_->Exist(fd1)); - ASSERT_FALSE(recordManager_->GetRecord(fd1, &fileRecord1)); - ASSERT_FALSE(recordManager_->GetRecord(fileName1, &fileRecord1)); - - uint64_t time1 = 1000; - uint64_t time2 = 2000; - // UpdateRecord - fileRecord1.fileName = fileName1; - fileRecord1.fd = fd1; - fileRecord1.status = NebdFileStatus::OPENED; - fileRecord1.timeStamp = time1; - EXPECT_CALL(*metaFileManager_, UpdateMetaFile(_)) - .WillOnce(Return(0)); - ASSERT_TRUE(recordManager_->UpdateRecord(fileRecord1)); - ASSERT_TRUE(recordManager_->Exist(fd1)); - // 通过fd获取record - NebdFileRecord tempRecord; - ASSERT_TRUE(recordManager_->GetRecord(fd1, &tempRecord)); - ASSERT_EQ(tempRecord.fd, fd1); - ASSERT_EQ(tempRecord.fileName, fileName1); - // 通过filename获取record - ASSERT_TRUE(recordManager_->GetRecord(fileName1, &tempRecord)); - ASSERT_EQ(tempRecord.fd, fd1); - ASSERT_EQ(tempRecord.fileName, fileName1); - - // 插入一条filename相同，fd不同的记录 - fileRecord1.fd = ++fd1; - EXPECT_CALL(*metaFileManager_, UpdateMetaFile(_)) - .WillOnce(Return(0)); - ASSERT_TRUE(recordManager_->UpdateRecord(fileRecord1)); - ASSERT_FALSE(recordManager_->Exist(fd1-1)); - ASSERT_TRUE(recordManager_->Exist(fd1)); - - // get/update timestamp 测试 - uint64_t record1TimeStamp; - ASSERT_TRUE(recordManager_->GetFileTimestamp(fd1, &record1TimeStamp)); - ASSERT_EQ(record1TimeStamp, time1); - ASSERT_TRUE(recordManager_->UpdateFileTimestamp(fd1, time2)); - ASSERT_TRUE(recordManager_->GetFileTimestamp(fd1, &record1TimeStamp)); - ASSERT_EQ(record1TimeStamp, time2); - // 指定fd记录不存在 - ASSERT_FALSE(recordManager_->GetFileTimestamp(10, &record1TimeStamp)); - ASSERT_FALSE(recordManager_->UpdateFileTimestamp(10, time2)); - - // get/update status 测试 - NebdFileStatus record1Status; - ASSERT_TRUE(recordManager_->GetFileStatus(fd1, &record1Status)); - ASSERT_EQ(record1Status, NebdFileStatus::OPENED); - ASSERT_TRUE(recordManager_->UpdateFileStatus(fd1, NebdFileStatus::CLOSED)); - ASSERT_TRUE(recordManager_->GetFileStatus(fd1, &record1Status)); - ASSERT_EQ(record1Status, NebdFileStatus::CLOSED); - // 指定fd记录不存在 - ASSERT_FALSE(recordManager_->GetFileStatus(10, &record1Status)); - ASSERT_FALSE(recordManager_->UpdateFileStatus(10, NebdFileStatus::CLOSED)); - - - // 插入不同记录 - NebdFileRecord fileRecord2; - std::string fileName2 = "file2"; - int fd2 = 3; - fileRecord2.fileName = fileName2; - fileRecord2.fd = fd2; - EXPECT_CALL(*metaFileManager_, UpdateMetaFile(_)) - .WillOnce(Return(0)); - ASSERT_TRUE(recordManager_->UpdateRecord(fileRecord2)); - ASSERT_TRUE(recordManager_->Exist(fd1)); - ASSERT_TRUE(recordManager_->Exist(fd2)); - - // 插入fd相同，名字不同的记录,返回失败 - NebdFileRecord fileRecord3; - fileRecord3.fileName = "file3"; - fileRecord3.fd = fd2; - EXPECT_CALL(*metaFileManager_, UpdateMetaFile(_)) - .Times(0); - ASSERT_FALSE(recordManager_->UpdateRecord(fileRecord3)); - - // update metafile 失败，返回失败 - fileRecord3.fileName = "file3"; - fileRecord3.fd = 4; - EXPECT_CALL(*metaFileManager_, UpdateMetaFile(_)) - .WillOnce(Return(-1)); - ASSERT_FALSE(recordManager_->UpdateRecord(fileRecord3)); - - // remove record - // 请求删除的fd不存在，直接返回成功 - ASSERT_TRUE(recordManager_->RemoveRecord(10)); - // 请求删除的fd存在，更新metafile，并删除记录 - EXPECT_CALL(*metaFileManager_, UpdateMetaFile(_)) - .WillOnce(Return(0)); - ASSERT_TRUE(recordManager_->RemoveRecord(fd2)); - // 如果更新metafile失败，则不删除记录 - EXPECT_CALL(*metaFileManager_, UpdateMetaFile(_)) - .WillOnce(Return(-1)); - ASSERT_FALSE(recordManager_->RemoveRecord(fd1)); - - // 校验最终结果 - FileRecordMap list = recordManager_->ListRecords(); - ASSERT_EQ(list.size(), 1); - // 通过fd获取record并检查结果 - auto recordIter = list.begin(); - ASSERT_EQ(recordIter->first, fd1); - ASSERT_EQ(recordIter->second.fd, fd1); - ASSERT_EQ(recordIter->second.fileName, fileName1); - - // clear - recordManager_->Clear(); - list = recordManager_->ListRecords(); - ASSERT_EQ(list.size(), 0); -} - -TEST_F(FileRecordManagerTest, LoadTest) { - FileRecordMap list = recordManager_->ListRecords(); - ASSERT_EQ(list.size(), 0); - - // list 失败 - EXPECT_CALL(*metaFileManager_, ListFileRecord(_)) - .WillOnce(Return(-1)); - ASSERT_EQ(-1, recordManager_->Load()); - - // list 成功 - FileRecordMap recordsFromFile; - NebdFileRecord fileRecord1; - fileRecord1.fileName = "111"; - fileRecord1.fd = 1; - recordsFromFile.emplace(1, fileRecord1); - EXPECT_CALL(*metaFileManager_, ListFileRecord(_)) - .WillOnce(DoAll(SetArgPointee<0>(recordsFromFile), - Return(0))); - ASSERT_EQ(0, recordManager_->Load()); - - // 校验结果 - list = recordManager_->ListRecords(); - ASSERT_EQ(list.size(), 1); - auto recordIter = list.begin(); - ASSERT_EQ(recordIter->first, 1); - ASSERT_EQ(recordIter->second.fd, 1); - ASSERT_EQ(recordIter->second.fileName, "111"); -} - -} // namespace server -} // namespace nebd - -int main(int argc, char ** argv) { - ::testing::InitGoogleTest(&argc, argv); - ::testing::InitGoogleMock(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tests/part2/heartbeat_manager_unittest.cpp b/tests/part2/heartbeat_manager_unittest.cpp index 83db89227d..89a5ecd938 100644 --- a/tests/part2/heartbeat_manager_unittest.cpp +++ b/tests/part2/heartbeat_manager_unittest.cpp @@ -9,8 +9,7 @@ #include #include "src/part2/heartbeat_manager.h" -#include "src/part2/filerecord_manager.h" -#include "tests/part2/mock_filerecord_manager.h" +#include "tests/part2/mock_file_entity.h" #include "tests/part2/mock_file_manager.h" using ::testing::_; @@ -32,18 +31,14 @@ using ::testing::SetArrayArgument; class HeartbeatManagerTest : public ::testing::Test { protected: void SetUp() override { - fileRecordManager_ = std::make_shared(); fileManager_ = std::make_shared(); HeartbeatManagerOption option; option.heartbeatTimeoutS = 10; option.checkTimeoutIntervalMs = 1000; option.fileManager = fileManager_; heartbeatManager_ = std::make_shared(option); - EXPECT_CALL(*fileManager_, GetRecordManager()) - .WillRepeatedly(Return(fileRecordManager_)); } std::shared_ptr fileManager_; - std::shared_ptr fileRecordManager_; std::shared_ptr heartbeatManager_; }; @@ -52,44 +47,41 @@ TEST_F(HeartbeatManagerTest, CheckTimeoutTest) { // 已经在run了不允许重复Run或者Init ASSERT_EQ(heartbeatManager_->Run(), -1); - // 校验是否在检查超时 + // 构造file entity uint64_t curTime = TimeUtility::GetTimeofDayMs(); - FileRecordMap fileRecords; - NebdFileRecord record1; - NebdFileRecord record2; - NebdFileRecord record3; - record1.fd = 1; - record1.timeStamp = curTime - 2 * 10 * 1000; - record1.status = NebdFileStatus::OPENED; - record2.fd = 2; - record2.timeStamp = curTime - 2 * 10 * 1000; - record2.status = NebdFileStatus::CLOSED; - record3.fd = 3; - record3.timeStamp = curTime; - record3.status = NebdFileStatus::OPENED; - fileRecords.emplace(1, record1); - fileRecords.emplace(2, record2); - fileRecords.emplace(3, record3); - - EXPECT_CALL(*fileRecordManager_, ListRecords()) - .WillRepeatedly(Return(fileRecords)); - - EXPECT_CALL(*fileRecordManager_, GetRecord(1, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(record1), - Return(true))); - - EXPECT_CALL(*fileRecordManager_, GetRecord(2, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(record2), - Return(true))); - - EXPECT_CALL(*fileRecordManager_, GetRecord(3, _)) - .WillRepeatedly(Return(false)); - - EXPECT_CALL(*fileManager_, Close(1, false)) + std::shared_ptr entity1 = + std::make_shared(); + std::shared_ptr entity2 = + std::make_shared(); + std::shared_ptr entity3 = + std::make_shared(); + EXPECT_CALL(*entity1, GetFileTimeStamp()) + .WillRepeatedly(Return(curTime - 2 * 10 * 1000)); + EXPECT_CALL(*entity1, GetFileStatus()) + .WillRepeatedly(Return(NebdFileStatus::OPENED)); + EXPECT_CALL(*entity2, GetFileTimeStamp()) + .WillRepeatedly(Return(curTime - 2 * 10 * 1000)); + EXPECT_CALL(*entity2, GetFileStatus()) + .WillRepeatedly(Return(NebdFileStatus::CLOSED)); + EXPECT_CALL(*entity3, GetFileTimeStamp()) + .WillRepeatedly(Return(curTime)); + EXPECT_CALL(*entity3, GetFileStatus()) + .WillRepeatedly(Return(NebdFileStatus::OPENED)); + + // 构造file map + FileEntityMap entityMap; + entityMap.emplace(1, entity1); + entityMap.emplace(2, entity2); + entityMap.emplace(3, entity3); + EXPECT_CALL(*fileManager_, GetFileEntityMap()) + .WillRepeatedly(Return(entityMap)); + + // 预期结果 + EXPECT_CALL(*entity1, Close(false)) .Times(AtLeast(1)); - EXPECT_CALL(*fileManager_, Close(2, false)) + EXPECT_CALL(*entity2, Close(false)) .Times(0); - EXPECT_CALL(*fileManager_, Close(3, false)) + EXPECT_CALL(*entity3, Close(false)) .Times(0); ::sleep(2); @@ -99,12 +91,16 @@ TEST_F(HeartbeatManagerTest, CheckTimeoutTest) { } TEST_F(HeartbeatManagerTest, UpdateTimeStampTest) { - EXPECT_CALL(*fileRecordManager_, UpdateFileTimestamp(_, _)) - .WillOnce(Return(true)); + std::shared_ptr entity = std::make_shared(); + + EXPECT_CALL(*fileManager_, GetFileEntity(1)) + .WillOnce(Return(entity)); + EXPECT_CALL(*entity, UpdateFileTimeStamp(100)) + .Times(1); ASSERT_TRUE(heartbeatManager_->UpdateFileTimestamp(1, 100)); - EXPECT_CALL(*fileRecordManager_, UpdateFileTimestamp(_, _)) - .WillOnce(Return(false)); + EXPECT_CALL(*fileManager_, GetFileEntity(1)) + .WillOnce(Return(nullptr)); ASSERT_FALSE(heartbeatManager_->UpdateFileTimestamp(1, 100)); } diff --git a/tests/part2/metafile_manager_test.cpp b/tests/part2/metafile_manager_test.cpp index 1a49360eaa..45e15a95e8 100644 --- a/tests/part2/metafile_manager_test.cpp +++ b/tests/part2/metafile_manager_test.cpp @@ -10,7 +10,6 @@ #include #include "src/part2/metafile_manager.h" -#include "src/part2/request_executor_curve.h" #include "tests/part2/mock_posix_wrapper.h" using ::testing::_; @@ -42,50 +41,67 @@ class MetaFileManagerTest : public ::testing::Test { }; TEST_F(MetaFileManagerTest, nomaltest) { - NebdMetaFileManager metaFileManager(metaPath); - FileRecordMap records; + NebdMetaFileManagerOption option; + option.metaFilePath = metaPath; + NebdMetaFileManager metaFileManager; + ASSERT_EQ(metaFileManager.Init(option), 0); + std::vector fileMetas; // 文件不存在 - ASSERT_EQ(0, metaFileManager.ListFileRecord(&records)); - ASSERT_TRUE(records.empty()); + ASSERT_EQ(0, metaFileManager.ListFileMeta(&fileMetas)); + ASSERT_TRUE(fileMetas.empty()); // 添加两条记录，ceph和curve各一 - NebdFileRecord fileRecord1; - fileRecord1.fileName = "rbd:volume1"; - fileRecord1.fd = 1; - records.emplace(1, fileRecord1); - ASSERT_EQ(0, metaFileManager.UpdateMetaFile(records)); - NebdFileRecord fileRecord2; - fileRecord2.fileName = "cbd:volume2"; - fileRecord2.fd = 2; - fileRecord2.fileInstance = std::make_shared(); - fileRecord2.fileInstance->addition["session"] = "test-session"; - records.emplace(2, fileRecord2); - ASSERT_EQ(0, metaFileManager.UpdateMetaFile(records)); + NebdFileMeta fileMeta1; + fileMeta1.fileName = "rbd:volume1"; + fileMeta1.fd = 1; + ASSERT_EQ(0, metaFileManager.UpdateFileMeta(fileMeta1.fileName, fileMeta1)); + // 使用相同的内容Update + ASSERT_EQ(0, metaFileManager.UpdateFileMeta(fileMeta1.fileName, fileMeta1)); + + // 插入不同的meta + NebdFileMeta fileMeta2; + fileMeta2.fileName = "cbd:volume2"; + fileMeta2.fd = 2; + fileMeta2.xattr["session"] = "test-session"; + ASSERT_EQ(0, metaFileManager.UpdateFileMeta(fileMeta2.fileName, fileMeta2)); // listFileRecord - records.clear(); - ASSERT_EQ(0, metaFileManager.ListFileRecord(&records)); - ASSERT_EQ(2, records.size()); - ASSERT_EQ(fileRecord1.fileName, records[1].fileName); - ASSERT_EQ(fileRecord1.fd, records[1].fd); - ASSERT_EQ(fileRecord2.fileName, records[2].fileName); - ASSERT_EQ(fileRecord2.fd, records[2].fd); - ASSERT_EQ(fileRecord2.fileInstance->addition, - records[2].fileInstance->addition); + fileMetas.clear(); + ASSERT_EQ(0, metaFileManager.ListFileMeta(&fileMetas)); + ASSERT_EQ(2, fileMetas.size()); + ASSERT_EQ(fileMeta1, fileMetas[1]); + ASSERT_EQ(fileMeta2, fileMetas[0]); + + // remove meta + ASSERT_EQ(0, metaFileManager.RemoveFileMeta(fileMeta2.fileName)); + // remove 不存在的meta + ASSERT_EQ(0, metaFileManager.RemoveFileMeta("unknown")); + // 校验结果 + fileMetas.clear(); + ASSERT_EQ(0, metaFileManager.ListFileMeta(&fileMetas)); + ASSERT_EQ(1, fileMetas.size()); + ASSERT_EQ(fileMeta1, fileMetas[0]); } -TEST_F(MetaFileManagerTest, error) { - NebdMetaFileManager metaFileManager(metaPath, wrapper_); - FileRecordMap records; - NebdFileRecord fileRecord; - fileRecord.fileName = "rbd:volume1"; - fileRecord.fd = 111; - records.emplace(111, fileRecord); +TEST_F(MetaFileManagerTest, UpdateMetaFailTest) { + NebdMetaFileManagerOption option; + option.metaFilePath = metaPath; + option.wrapper = wrapper_; + NebdMetaFileManager metaFileManager; + ASSERT_EQ(metaFileManager.Init(option), 0); + NebdFileMeta fileMeta; + fileMeta.fileName = "rbd:volume1"; + fileMeta.fd = 111; + FileMetaMap fileMetaMap; + fileMetaMap.emplace(fileMeta.fileName, fileMeta); + std::vector fileMetas; // open临时文件失败 EXPECT_CALL(*wrapper_, open(_, _, _)) .WillOnce(Return(-1)); - ASSERT_EQ(-1, metaFileManager.UpdateMetaFile(records)); + ASSERT_EQ(-1, metaFileManager.UpdateFileMeta(fileMeta.fileName, fileMeta)); + ASSERT_EQ(0, metaFileManager.ListFileMeta(&fileMetas)); + ASSERT_EQ(0, fileMetas.size()); // 写入临时文件失败 EXPECT_CALL(*wrapper_, open(_, _, _)) @@ -94,11 +110,13 @@ TEST_F(MetaFileManagerTest, error) { .WillOnce(Return(0)); EXPECT_CALL(*wrapper_, close(_)) .Times(1); - ASSERT_EQ(-1, metaFileManager.UpdateMetaFile(records)); + ASSERT_EQ(-1, metaFileManager.UpdateFileMeta(fileMeta.fileName, fileMeta)); + ASSERT_EQ(0, metaFileManager.ListFileMeta(&fileMetas)); + ASSERT_EQ(0, fileMetas.size()); // rename失败 NebdMetaFileParser parser; - Json::Value root = parser.ConvertFileRecordsToJson(records); + Json::Value root = parser.ConvertFileMetasToJson(fileMetaMap); EXPECT_CALL(*wrapper_, open(_, _, _)) .WillOnce(Return(1)); EXPECT_CALL(*wrapper_, pwrite(_, _, _, _)) @@ -107,7 +125,72 @@ TEST_F(MetaFileManagerTest, error) { .Times(1); EXPECT_CALL(*wrapper_, rename(_, _)) .WillOnce(Return(-1)); - ASSERT_EQ(-1, metaFileManager.UpdateMetaFile(records)); + ASSERT_EQ(-1, metaFileManager.UpdateFileMeta(fileMeta.fileName, fileMeta)); + ASSERT_EQ(0, metaFileManager.ListFileMeta(&fileMetas)); + ASSERT_EQ(0, fileMetas.size()); +} + +TEST_F(MetaFileManagerTest, RemoveMetaFailTest) { + NebdMetaFileManagerOption option; + option.metaFilePath = metaPath; + option.wrapper = wrapper_; + NebdMetaFileManager metaFileManager; + ASSERT_EQ(metaFileManager.Init(option), 0); + NebdFileMeta fileMeta; + fileMeta.fileName = "rbd:volume1"; + fileMeta.fd = 111; + FileMetaMap fileMetaMap; + fileMetaMap.emplace(fileMeta.fileName, fileMeta); + std::vector fileMetas; + NebdMetaFileParser parser; + Json::Value root = parser.ConvertFileMetasToJson(fileMetaMap); + + // 先插入一条数据 + EXPECT_CALL(*wrapper_, open(_, _, _)) + .WillOnce(Return(1)); + EXPECT_CALL(*wrapper_, pwrite(_, _, _, _)) + .WillOnce(Return(root.toStyledString().size())); + EXPECT_CALL(*wrapper_, close(_)) + .Times(1); + EXPECT_CALL(*wrapper_, rename(_, _)) + .WillOnce(Return(0)); + ASSERT_EQ(0, metaFileManager.UpdateFileMeta(fileMeta.fileName, fileMeta)); + ASSERT_EQ(0, metaFileManager.ListFileMeta(&fileMetas)); + ASSERT_EQ(1, fileMetas.size()); + + fileMetaMap.erase(fileMeta.fileName); + root = parser.ConvertFileMetasToJson(fileMetaMap); + + // open临时文件失败 + EXPECT_CALL(*wrapper_, open(_, _, _)) + .WillOnce(Return(-1)); + ASSERT_EQ(-1, metaFileManager.RemoveFileMeta(fileMeta.fileName)); + ASSERT_EQ(0, metaFileManager.ListFileMeta(&fileMetas)); + ASSERT_EQ(1, fileMetas.size()); + + // 写入临时文件失败 + EXPECT_CALL(*wrapper_, open(_, _, _)) + .WillOnce(Return(1)); + EXPECT_CALL(*wrapper_, pwrite(_, _, _, _)) + .WillOnce(Return(0)); + EXPECT_CALL(*wrapper_, close(_)) + .Times(1); + ASSERT_EQ(-1, metaFileManager.RemoveFileMeta(fileMeta.fileName)); + ASSERT_EQ(0, metaFileManager.ListFileMeta(&fileMetas)); + ASSERT_EQ(1, fileMetas.size()); + + // rename失败 + EXPECT_CALL(*wrapper_, open(_, _, _)) + .WillOnce(Return(1)); + EXPECT_CALL(*wrapper_, pwrite(_, _, _, _)) + .WillOnce(Return(root.toStyledString().size())); + EXPECT_CALL(*wrapper_, close(_)) + .Times(1); + EXPECT_CALL(*wrapper_, rename(_, _)) + .WillOnce(Return(-1)); + ASSERT_EQ(-1, metaFileManager.RemoveFileMeta(fileMeta.fileName)); + ASSERT_EQ(0, metaFileManager.ListFileMeta(&fileMetas)); + ASSERT_EQ(1, fileMetas.size()); } TEST(MetaFileParserTest, Parse) { @@ -115,7 +198,7 @@ TEST(MetaFileParserTest, Parse) { Json::Value root; Json::Value volume; Json::Value volumes; - FileRecordMap records; + FileMetaMap fileMetas; // 正常情况 volume[kFileName] = "rbd:volume1"; @@ -125,31 +208,31 @@ TEST(MetaFileParserTest, Parse) { volume[kFd] = 2; root[kVolumes] = volumes; FillCrc(&root); - ASSERT_EQ(0, parser.Parse(root, &records)); + ASSERT_EQ(0, parser.Parse(root, &fileMetas)); // 空指针 ASSERT_EQ(-1, parser.Parse(root, nullptr)); // crc校验不正确 root[kCRC] = root[kCRC].asUInt() + 1; - ASSERT_EQ(-1, parser.Parse(root, &records)); + ASSERT_EQ(-1, parser.Parse(root, &fileMetas)); // 没有crc字段 root.removeMember(kCRC); - ASSERT_EQ(-1, parser.Parse(root, &records)); + ASSERT_EQ(-1, parser.Parse(root, &fileMetas)); // 没有volumes字段或volumes字段是null,不应该报错 root.clear(); root["key"] = "value"; FillCrc(&root); - ASSERT_EQ(0, parser.Parse(root, &records)); - ASSERT_TRUE(records.empty()); + ASSERT_EQ(0, parser.Parse(root, &fileMetas)); + ASSERT_TRUE(fileMetas.empty()); root.clear(); Json::Value value; root[kVolumes] = value; FillCrc(&root); - ASSERT_EQ(0, parser.Parse(root, &records)); - ASSERT_TRUE(records.empty()); + ASSERT_EQ(0, parser.Parse(root, &fileMetas)); + ASSERT_TRUE(fileMetas.empty()); // 记录中没有filename volume.clear(); @@ -159,7 +242,7 @@ TEST(MetaFileParserTest, Parse) { volumes.append(volume); root[kVolumes] = volumes; FillCrc(&root); - ASSERT_EQ(-1, parser.Parse(root, &records)); + ASSERT_EQ(-1, parser.Parse(root, &fileMetas)); // 记录中没有fd volume.clear(); @@ -169,18 +252,7 @@ TEST(MetaFileParserTest, Parse) { volumes.append(volume); root[kVolumes] = volumes; FillCrc(&root); - ASSERT_EQ(-1, parser.Parse(root, &records)); - - // 文件名格式不对 - volume.clear(); - volumes.clear(); - root.clear(); - volume[kFileName] = "volume2"; - volume[kFd] = 1234; - volumes.append(volume); - root[kVolumes] = volumes; - FillCrc(&root); - ASSERT_EQ(-1, parser.Parse(root, &records)); + ASSERT_EQ(-1, parser.Parse(root, &fileMetas)); } } // namespace server diff --git a/tests/part2/mock_file_entity.h b/tests/part2/mock_file_entity.h new file mode 100644 index 0000000000..da555374c9 --- /dev/null +++ b/tests/part2/mock_file_entity.h @@ -0,0 +1,45 @@ +/* + * Project: nebd + * Created Date: Thursday March 5th 2020 + * Author: yangyaokai + * Copyright (c) 2020 netease + */ + +#ifndef TESTS_PART2_MOCK_FILE_ENTITY_H_ +#define TESTS_PART2_MOCK_FILE_ENTITY_H_ + +#include +#include +#include + +#include "src/part2/file_entity.h" + +namespace nebd { +namespace server { + +class MockFileEntity : public NebdFileEntity { + public: + MockFileEntity() : NebdFileEntity() {} + ~MockFileEntity() {} + + MOCK_METHOD1(Init, int(const NebdFileEntityOption&)); + MOCK_METHOD0(Open, int()); + MOCK_METHOD1(Close, int(bool)); + MOCK_METHOD1(Extend, int(int64_t)); + MOCK_METHOD1(GetInfo, int(NebdFileInfo*)); + MOCK_METHOD1(Discard, int(NebdServerAioContext*)); + MOCK_METHOD1(AioRead, int(NebdServerAioContext*)); + MOCK_METHOD1(AioWrite, int(NebdServerAioContext*)); + MOCK_METHOD1(Flush, int(NebdServerAioContext*)); + MOCK_METHOD0(InvalidCache, int()); + MOCK_CONST_METHOD0(GetFileName, const std::string()); + MOCK_CONST_METHOD0(GetFd, const int()); + MOCK_METHOD1(UpdateFileTimeStamp, void(uint64_t)); + MOCK_CONST_METHOD0(GetFileTimeStamp, const uint64_t()); + MOCK_CONST_METHOD0(GetFileStatus, const NebdFileStatus()); +}; + +} // namespace server +} // namespace nebd + +#endif // TESTS_PART2_MOCK_FILE_ENTITY_H_ diff --git a/tests/part2/mock_file_manager.h b/tests/part2/mock_file_manager.h index af67d17c3f..ed5424e5b6 100644 --- a/tests/part2/mock_file_manager.h +++ b/tests/part2/mock_file_manager.h @@ -33,7 +33,8 @@ class MockFileManager : public NebdFileManager { MOCK_METHOD2(AioWrite, int(int, NebdServerAioContext*)); MOCK_METHOD2(Flush, int(int, NebdServerAioContext*)); MOCK_METHOD1(InvalidCache, int(int)); - MOCK_METHOD0(GetRecordManager, FileRecordManagerPtr()); + MOCK_METHOD1(GetFileEntity, NebdFileEntityPtr(int)); + MOCK_METHOD0(GetFileEntityMap, FileEntityMap()); }; } // namespace server diff --git a/tests/part2/mock_filerecord_manager.h b/tests/part2/mock_filerecord_manager.h deleted file mode 100644 index 15f0456dd1..0000000000 --- a/tests/part2/mock_filerecord_manager.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Project: nebd - * Created Date: Sunday February 16th 2020 - * Author: yangyaokai - * Copyright (c) 2020 netease - */ - -#ifndef TESTS_PART2_MOCK_FILERECORD_MANAGER_H_ -#define TESTS_PART2_MOCK_FILERECORD_MANAGER_H_ - -#include -#include -#include - -#include "src/part2/filerecord_manager.h" - -namespace nebd { -namespace server { - -class MockFileRecordManager : public FileRecordManager { - public: - MockFileRecordManager() : FileRecordManager(nullptr) {} - ~MockFileRecordManager() {} - - MOCK_METHOD0(Load, int()); - MOCK_METHOD2(GetRecord, bool(int, NebdFileRecord*)); - MOCK_METHOD2(GetRecord, bool(const std::string&, NebdFileRecord*)); - MOCK_METHOD1(UpdateRecord, bool(const NebdFileRecord&)); - MOCK_METHOD1(RemoveRecord, bool(int)); - MOCK_METHOD1(Exist, bool(int)); - MOCK_METHOD0(Clear, void(void)); - MOCK_METHOD0(ListRecords, FileRecordMap(void)); - MOCK_METHOD2(UpdateFileTimestamp, bool(int, uint64_t)); - MOCK_METHOD2(GetFileTimestamp, bool(int, uint64_t*)); - MOCK_METHOD2(UpdateFileStatus, bool(int, NebdFileStatus)); - MOCK_METHOD2(GetFileStatus, bool(int, NebdFileStatus*)); -}; - -} // namespace server -} // namespace nebd - -#endif // TESTS_PART2_MOCK_FILERECORD_MANAGER_H_ diff --git a/tests/part2/mock_metafile_manager.h b/tests/part2/mock_metafile_manager.h index 866ca60992..aa142e2c33 100644 --- a/tests/part2/mock_metafile_manager.h +++ b/tests/part2/mock_metafile_manager.h @@ -19,11 +19,13 @@ namespace server { class MockMetaFileManager : public NebdMetaFileManager { public: - MockMetaFileManager() : NebdMetaFileManager("") {} + MockMetaFileManager() {} ~MockMetaFileManager() {} - MOCK_METHOD1(UpdateMetaFile, int(const FileRecordMap&)); - MOCK_METHOD1(ListFileRecord, int(FileRecordMap*)); + MOCK_METHOD1(Init, int(const NebdMetaFileManagerOption&)); + MOCK_METHOD1(ListFileMeta, int(std::vector*)); + MOCK_METHOD2(UpdateFileMeta, int(const std::string&, const NebdFileMeta&)); + MOCK_METHOD1(RemoveFileMeta, int(const std::string&)); }; } // namespace server diff --git a/tests/part2/mock_request_executor.h b/tests/part2/mock_request_executor.h index c32c9889d3..657b0e99d2 100644 --- a/tests/part2/mock_request_executor.h +++ b/tests/part2/mock_request_executor.h @@ -30,8 +30,8 @@ class MockRequestExecutor : public NebdRequestExecutor { ~MockRequestExecutor() {} MOCK_METHOD1(Open, std::shared_ptr(const std::string&)); - MOCK_METHOD2(Reopen, std::shared_ptr(const std::string&, - AdditionType)); + MOCK_METHOD2(Reopen, std::shared_ptr( + const std::string&, const ExtendAttribute&)); MOCK_METHOD1(Close, int(NebdFileInstance*)); MOCK_METHOD2(Extend, int(NebdFileInstance*, int64_t)); MOCK_METHOD2(GetInfo, int(NebdFileInstance*, NebdFileInfo*)); diff --git a/tests/part2/test_request_executor_curve.cpp b/tests/part2/test_request_executor_curve.cpp index a7fb79df07..70bf8a12cf 100644 --- a/tests/part2/test_request_executor_curve.cpp +++ b/tests/part2/test_request_executor_curve.cpp @@ -65,14 +65,14 @@ TEST_F(TestReuqestExecutorCurve, test_Open) { ASSERT_TRUE(nullptr != curveIns); ASSERT_EQ(curveFileName, curveIns->fileName); ASSERT_EQ(1, curveIns->fd); - ASSERT_EQ("abc", curveIns->addition["session"]); + ASSERT_EQ("abc", curveIns->xattr["session"]); } } TEST_F(TestReuqestExecutorCurve, test_ReOpen) { auto executor = CurveRequestExecutor::GetInstance(); - AdditionType addtion; - addtion["session"] = "abc"; + ExtendAttribute xattr; + xattr["session"] = "abc"; std::string fileName("cbd:pool1//cinder/volume-1234_cinder_:/client.conf"); std::string curveFileName("/cinder/volume-1234_cinder_"); @@ -81,31 +81,31 @@ TEST_F(TestReuqestExecutorCurve, test_ReOpen) { std::string errFileName("cbd:pool1/:"); EXPECT_CALL(*curveClient_, Open(_, _)).Times(0); std::shared_ptr ret = executor.Reopen( - errFileName, addtion); + errFileName, xattr); ASSERT_TRUE(nullptr == ret); } // 2. repoen失败 { - EXPECT_CALL(*curveClient_, ReOpen(curveFileName, addtion["session"], _)) + EXPECT_CALL(*curveClient_, ReOpen(curveFileName, xattr["session"], _)) .WillOnce(DoAll(SetArgPointee<2>(""), Return(-1))); std::shared_ptr ret = - executor.Reopen(fileName, addtion); + executor.Reopen(fileName, xattr); ASSERT_TRUE(nullptr == ret); } // 3. reopen成功 { - EXPECT_CALL(*curveClient_, ReOpen(curveFileName, addtion["session"], _)) + EXPECT_CALL(*curveClient_, ReOpen(curveFileName, xattr["session"], _)) .WillOnce(DoAll(SetArgPointee<2>("bcd"), Return(1))); std::shared_ptr ret = - executor.Reopen(fileName, addtion); + executor.Reopen(fileName, xattr); ASSERT_TRUE(nullptr != ret); auto *curveIns = dynamic_cast(ret.get()); ASSERT_TRUE(nullptr != curveIns); ASSERT_EQ(curveFileName, curveIns->fileName); ASSERT_EQ(1, curveIns->fd); - ASSERT_EQ("bcd", curveIns->addition["session"]); + ASSERT_EQ("bcd", curveIns->xattr["session"]); } } From a1d908407e3120848bbb226a71dd38cc1900e475 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Mon, 9 Mar 2020 13:58:18 +0800 Subject: [PATCH 57/79] update curve commit id Change-Id: I8252bd67a8a3257639bac9fe4b577688d6d25f76 --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 4c89651957..8fee15280b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -98,7 +98,7 @@ bind( git_repository( name = "com_netease_storage_gerrit_curve_curve", remote = "http://gerrit.storage.netease.com/curve", - commit = "b5568f122664be95bb44a3813a0a08160ec11968", + commit = "0f9f1132d51e12490e3dee5cbaaa36e6ce1d1e56", ) bind( From 99d11b6e71760d4d01ed4233462e1c990ab96126 Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Mon, 9 Mar 2020 16:08:08 +0800 Subject: [PATCH 58/79] fix update timestamp error Change-Id: I1332ec0b8b0cdb1508cad729373eda8b86208ac3 --- src/part2/file_entity.cpp | 6 +++++- src/part2/file_entity.h | 2 +- tests/part2/file_manager_unittest.cpp | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/part2/file_entity.cpp b/src/part2/file_entity.cpp index ce525cf2eb..19348f419f 100644 --- a/src/part2/file_entity.cpp +++ b/src/part2/file_entity.cpp @@ -333,8 +333,12 @@ bool NebdFileEntity::GuaranteeFileOpened() { } std::ostream& operator<<(std::ostream& os, const NebdFileEntity& entity) { + std::string standardTime; + TimeUtility::TimeStampToStandard( + entity.GetFileTimeStamp() / 1000, &standardTime); os << "[filename: " << entity.GetFileName() << ", fd: " << entity.GetFd() - << ", status: " << NebdFileStatus2Str(entity.GetFileStatus()) << "]"; + << ", status: " << NebdFileStatus2Str(entity.GetFileStatus()) + << ", timestamp: " << standardTime << "]"; return os; } diff --git a/src/part2/file_entity.h b/src/part2/file_entity.h index dafe62731a..d1568c3407 100644 --- a/src/part2/file_entity.h +++ b/src/part2/file_entity.h @@ -154,7 +154,7 @@ class NebdFileEntity : public std::enable_shared_from_this { } virtual void UpdateFileTimeStamp(uint64_t timestamp) { - timeStamp_.store(timeStamp_); + timeStamp_.store(timestamp); } virtual const uint64_t GetFileTimeStamp() const { diff --git a/tests/part2/file_manager_unittest.cpp b/tests/part2/file_manager_unittest.cpp index 81198beda2..8841a08f4d 100644 --- a/tests/part2/file_manager_unittest.cpp +++ b/tests/part2/file_manager_unittest.cpp @@ -508,6 +508,20 @@ TEST_F(FileManagerTest, FlushTest) { RequestFailTest(RequestType::FLUSH, task); } +TEST_F(FileManagerTest, UpdateTimestampTest) { + InitEnv(); + NebdFileEntityPtr entity = fileManager_->GetFileEntity(1); + ASSERT_NE(nullptr, entity); + ASSERT_EQ(1, entity->GetFd()); + + ::usleep(1000); + uint64_t curTime = TimeUtility::GetTimeofDayMs(); + ASSERT_NE(curTime, entity->GetFileTimeStamp()); + entity->UpdateFileTimeStamp(curTime); + ASSERT_EQ(curTime, entity->GetFileTimeStamp()); + std::cout << fileManager_->DumpAllFileStatus(); +} + } // namespace server } // namespace nebd From d1705108e8a76c78f497221722bc2eae0a2aec4e Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Tue, 10 Mar 2020 10:59:18 +0800 Subject: [PATCH 59/79] add daemon acceptable Change-Id: Ib5b51d840f2f14a32ef5206b318c6c01af5fecd0 --- nebd-package/usr/bin/nebd-daemon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nebd-package/usr/bin/nebd-daemon b/nebd-package/usr/bin/nebd-daemon index 7375c6145f..fdf64b2a5a 100755 --- a/nebd-package/usr/bin/nebd-daemon +++ b/nebd-package/usr/bin/nebd-daemon @@ -100,7 +100,7 @@ function start() { line1=`cat ${daemonLog} | wc -l` LD_PRELOAD=${jemallocpath} daemon --name nebd-server --core --inherit \ - --respawn --attempts 10 --delay 10 \ + --respawn --attempts 10 --delay 10 --acceptable 5\ --pidfile ${pidFile} \ --errlog ${daemonLog} \ --output ${consoleLog} \ From 4c4cd54fb602c4fd1b075dc86dab2348eba1a3a1 Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Tue, 10 Mar 2020 14:38:45 +0800 Subject: [PATCH 60/79] support auto run after reboot Change-Id: Ib52453f499fdda6e51469059ca64be53f485a0e9 --- nebd-package/DEBIAN/postinst | 3 +++ nebd-package/usr/bin/nebd-daemon | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/nebd-package/DEBIAN/postinst b/nebd-package/DEBIAN/postinst index b9d046e8d3..c83b23eda3 100755 --- a/nebd-package/DEBIAN/postinst +++ b/nebd-package/DEBIAN/postinst @@ -8,4 +8,7 @@ chown -R root:root /data/log/nebd chmod -R 777 /data/nebd chmod -R 777 /data/log/nebd +cp /usr/bin/nebd-daemon /etc/init.d +update-rc.d nebd-daemon defaults + exit 0 diff --git a/nebd-package/usr/bin/nebd-daemon b/nebd-package/usr/bin/nebd-daemon index fdf64b2a5a..01dc7b4258 100755 --- a/nebd-package/usr/bin/nebd-daemon +++ b/nebd-package/usr/bin/nebd-daemon @@ -1,5 +1,15 @@ #!/bin/bash +### BEGIN INIT INFO +# Provides: nebd-daemon +# Required-Start: $local_fs $remote_fs $network $syslog $named +# Required-Stop: $local_fs $remote_fs $network $syslog $named +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: nebd-daemon service +# Description: Start the nebd-daemon service and associated helpers +### END INIT INFO + # 检查脚本执行是否具有root权限 if [[ $(id -u) -ne 0 ]] then @@ -100,7 +110,7 @@ function start() { line1=`cat ${daemonLog} | wc -l` LD_PRELOAD=${jemallocpath} daemon --name nebd-server --core --inherit \ - --respawn --attempts 10 --delay 10 --acceptable 5\ + --respawn --attempts 10 --delay 10 --acceptable 10\ --pidfile ${pidFile} \ --errlog ${daemonLog} \ --output ${consoleLog} \ From 45a746c5eee34a716e0756b50f1e752fb185bf33 Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Tue, 10 Mar 2020 16:45:07 +0800 Subject: [PATCH 61/79] fix boot order Change-Id: I0c07ca53155f072af3d70b768a72609e64e63039 --- nebd-package/usr/bin/nebd-daemon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nebd-package/usr/bin/nebd-daemon b/nebd-package/usr/bin/nebd-daemon index 01dc7b4258..2f4a7ec564 100755 --- a/nebd-package/usr/bin/nebd-daemon +++ b/nebd-package/usr/bin/nebd-daemon @@ -2,8 +2,8 @@ ### BEGIN INIT INFO # Provides: nebd-daemon -# Required-Start: $local_fs $remote_fs $network $syslog $named -# Required-Stop: $local_fs $remote_fs $network $syslog $named +# Required-Start: $local_fs $remote_fs $network +# Required-Stop: $local_fs $remote_fs $network # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: nebd-daemon service From 6dd1a4f141e0c4f53b7ced2c495feee34d79ae8a Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Mon, 9 Mar 2020 16:56:45 +0800 Subject: [PATCH 62/79] add curve version check Change-Id: Ifdafffc0802ec6de92c82b95ed479bbd5e7431b0 --- mk-deb.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mk-deb.sh b/mk-deb.sh index a4cd517f2f..66d362beb6 100755 --- a/mk-deb.sh +++ b/mk-deb.sh @@ -5,12 +5,20 @@ bazel clean rm -rf *deb rm -rf build +#获取curve tag版本 +curve_tag_version=`cat WORKSPACE | grep -m 1 -A 10 "http://gerrit.storage.netease.com/curve\"" | grep "tag" | awk -F "[\"\"]" '{print $2}' | awk -F"v" '{print $2}'` +if [ -z ${curve_tag_version} ] +then + echo "not found curve version info, set curve tag version to 9.9.9" + curve_tag_version=9.9.9 +fi + # step2 编译 if [ "$1" = "debug" ] then bazel build ... --copt -DHAVE_ZLIB=1 --compilation_mode=dbg -s --define=with_glog=true \ --define=libunwind=true --copt -DGFLAGS_NS=google --copt \ - -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX + -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_tag_version} if [ $? -ne 0 ] then echo "build phase1 failed" @@ -19,7 +27,7 @@ then else bazel build ... --copt -DHAVE_ZLIB=1 --copt -O2 -s --define=with_glog=true \ --define=libunwind=true --copt -DGFLAGS_NS=google --copt \ - -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX + -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_tag_version} if [ $? -ne 0 ] then echo "build phase1 failed" From b7322b3394237e84223dbb49996cc21edacab8f1 Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Wed, 18 Mar 2020 23:21:27 +0800 Subject: [PATCH 63/79] add tcp check script Change-Id: Id3129d29ea230ca639c323afdfc291c182a21e7a --- nebd-package/DEBIAN/postinst | 3 + nebd-package/usr/bin/nebd-check-tcp.sh | 95 ++++++++++++ nebd-package/usr/bin/nebd-daemon | 4 +- nebd-package/usr/bin/nebd-tcpcheck-daemon | 174 ++++++++++++++++++++++ 4 files changed, 274 insertions(+), 2 deletions(-) create mode 100644 nebd-package/usr/bin/nebd-check-tcp.sh create mode 100755 nebd-package/usr/bin/nebd-tcpcheck-daemon diff --git a/nebd-package/DEBIAN/postinst b/nebd-package/DEBIAN/postinst index c83b23eda3..98e0f9abc1 100755 --- a/nebd-package/DEBIAN/postinst +++ b/nebd-package/DEBIAN/postinst @@ -11,4 +11,7 @@ chmod -R 777 /data/log/nebd cp /usr/bin/nebd-daemon /etc/init.d update-rc.d nebd-daemon defaults +cp /usr/bin/nebd-tcpcheck-daemon /etc/init.d +update-rc.d nebd-tcpcheck-daemon defaults + exit 0 diff --git a/nebd-package/usr/bin/nebd-check-tcp.sh b/nebd-package/usr/bin/nebd-check-tcp.sh new file mode 100644 index 0000000000..2170139162 --- /dev/null +++ b/nebd-package/usr/bin/nebd-check-tcp.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +nebd_pid="" +nebd_port="" +# rtt超过该值，则认为是不符合预期的 +slowrtt=10 +# rtt在连续时间内超过上面的值，则认为连接不正常，将进程kill +looptimes=5 +# 超时连接的计数器 +declare -A slow_times + +# 获取nebd server进程号及metric的端口号 +function get_nebdport() +{ + nebd_process_info=`sudo ps -ef |grep /usr/bin/nebd-server|grep -v daemon|grep -v grep` + pid=`echo $nebd_process_info|awk '{print $2}'` + if [[ $pid -eq $nebd_pid ]];then + return 0 + fi + nebd_pid=$pid + nebd_port_info=`sudo lsof -i|grep $nebd_pid|grep LISTEN` + nebd_port=`echo $nebd_port_info|awk '{print $9}'|cut -b 3-6` + echo "check nebd connection healthy status. pid : $nebd_pid, client port : $nebd_port" +} + +# 增加超时连接的计数 +function incref() +{ + if [[ "${!slow_times[*]}" =~ "$1" ]];then + slow_times[$1]=$((${slow_times[$1]}+1)) + else + slow_times[$1]=1 + fi + echo "slowtimes[$1]=${slow_times[$1]}" +} + +# 检查连接的rtt是否不符合预期，如果是，增加计数 +function check_rtt() +{ + connection_rtt_str=`sudo curl -i -s 127.0.0.1:$nebd_port/connections |grep 'baidu_std'` + oldifs="$IFS" + IFS=$'\n' + + for rttline in $connection_rtt_str;do + rtt=`echo $rttline|awk '{print $(NF-1)}' |sed 's/|\(.*\)\/.*/\1/g'` + localport=`echo $rttline|awk '{print $2}' |sed 's/|\(.*\)|.*/\1/g'` + isslowtcp=`echo "$rtt $slowrtt"|awk '{print ($1 > $2)}'` + if [[ $isslowtcp -eq 1 ]];then + incref $localport + echo "connection_port $localport connection rtt is slow than $slowrtt, rtt is : $rtt" + fi + done + IFS="$oldifs" +} + +# 连续时间内出现超时，则将进程强制kill +function kill_nebd_ifunhealthy() +{ + for connection in ${!slow_times[*]};do + if [[ ${slow_times[$connection]} -eq $looptimes ]];then + echo "connection_port $connection is unhealthy, nebd server will be killed. pid : $nebd_pid" + sudo kill -9 $nebd_pid + fi + echo "connection_port $connection slow times: ${slow_times[$connection]}" + done +} + +loop=1 +while [[ $loop -le $looptimes ]];do + echo "check ${loop}th times. " + + get_nebdport + if [ $? -ne 0 ];then + echo "get nebd port failed" + exit 1 + fi + + check_rtt + if [ $? -ne 0 ];then + echo "get nebd rtt failed" + exit 1 + fi + + if [[ $loop -eq $looptimes ]];then + kill_nebd_ifunhealthy + loop=1 + # 清空计数器 + unset slow_times + declare -A slow_times + else + let loop++ + fi + + sleep 1 +done \ No newline at end of file diff --git a/nebd-package/usr/bin/nebd-daemon b/nebd-package/usr/bin/nebd-daemon index 2f4a7ec564..b8412795a7 100755 --- a/nebd-package/usr/bin/nebd-daemon +++ b/nebd-package/usr/bin/nebd-daemon @@ -10,10 +10,10 @@ # Description: Start the nebd-daemon service and associated helpers ### END INIT INFO -# 检查脚本执行是否具有root权限 +# 检查脚本执行是否通过root权限执行 if [[ $(id -u) -ne 0 ]] then - echo "Please run as root" + echo "Please run with sudo" exit 1 fi diff --git a/nebd-package/usr/bin/nebd-tcpcheck-daemon b/nebd-package/usr/bin/nebd-tcpcheck-daemon new file mode 100755 index 0000000000..4f3976bab2 --- /dev/null +++ b/nebd-package/usr/bin/nebd-tcpcheck-daemon @@ -0,0 +1,174 @@ +#!/bin/bash + +### BEGIN INIT INFO +# Provides: nebd-tcpcheck-daemon +# Required-Start: $all +# Required-Stop: $all +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: nebd-tcpcheck-daemon service +# Description: Start the nebd-tcpcheck-daemon service and associated helpers +### END INIT INFO + +# 检查脚本执行是否通过root权限执行 +if [[ $(id -u) -ne 0 ]] +then + echo "Please run with sudo" + exit 1 +fi + +# nebd-tcpcheck路径 +bin=/usr/bin/nebd-check-tcp.sh + +# 日志文件路径 +baseLogPath=/data/log/nebd + +# pidfile +pidFile=${baseLogPath}/nebd-tcpcheck.pid + +# daemon log +daemonLog=${baseLogPath}/nebd-tcpcheck-daemon.log + +# console output +consoleLog=${baseLogPath}/nebd-tcpcheck-console.log + +# 启动nebd-tcpcheck +function start() { + # 检查daemon + if ! type daemon &> /dev/null + then + echo "No daemon installed" + exit 1 + fi + + # 检查nebd-tcpcheck + if [ ! -f ${bin} ] + then + echo "No nebd-tcpcheck installed" + exit 1 + fi + + # 判断是否已经通过daemon启动了nebd-tcpcheck + daemon --name nebd-tcpcheck --pidfile ${pidFile} --running + if [ $? -eq 0 ] + then + echo "Already started nebd-tcpcheck by daemon" + exit 1 + fi + + # 检查consoleLog是否可写或者是否能够创建 + touch ${consoleLog} > /dev/null 2>&1 + if [ $? -ne 0 ] + then + echo "Can't Write or Create console Log: ${consoleLog}" + exit 1 + fi + + # 检查daemonLog是否可写或者是否能够创建 + touch ${daemonLog} > /dev/null 2>&1 + if [ $? -ne 0 ] + then + echo "Can't Write or Create daemon logfile: ${daemonLog}" + exit 1 + fi + + # 记录运行daemon前的daemonLog的行数 + line1=`cat ${daemonLog} | wc -l` + + daemon --name nebd-tcpcheck --core --inherit \ + --respawn --attempts 10 --delay 10 --acceptable 10\ + --pidfile ${pidFile} \ + --errlog ${daemonLog} \ + --output ${consoleLog} \ + -- sh ${bin} + + # sleep 1秒检测daemonLog是否有变化，如果有变化，说明启动遇到问题 + sleep 1 + line2=`cat ${daemonLog} | wc -l` + if [ $line1 != $line2 ] + then + echo "start nebd-tcpcheck met error!" + stop + fi +} + +# 停止daemon进程，且停止nebd-tcpcheck +function stop() { + # 判断是否已经通过daemon启动了nebd-tcpcheck + daemon --name nebd-tcpcheck --pidfile ${pidFile} --running + if [ $? -ne 0 ] + then + echo "Didn't start nebd-tcpcheck by daemon" + exit 1 + fi + + daemon --name nebd-tcpcheck --pidfile ${pidFile} --stop + if [ $? -ne 0 ] + then + echo "stop may not success!" + else + echo "nebd-tcpcheck exit success!" + echo "daemon exit success!" + fi +} + +# restart +function restart() { + # 判断是否已经通过daemon启动了nebd-tcpcheck + daemon --name nebd-tcpcheck --pidfile ${pidFile} --running + if [ $? -ne 0 ] + then + echo "Didn't start nebd-tcpcheck by daemon" + exit 1 + fi + + daemon --name nebd-tcpcheck --pidfile ${pidFile} --restart +} + +# status +function status() { + daemon --name nebd-tcpcheck --pidfile ${pidFile} --running + if [ $? -ne 0 ] + then + echo "Didn't start nebd-tcpcheck by daemon" + else + echo "nebd-tcpcheck is running by daemon" + fi +} + +# 使用方式 +function usage() { + echo "Usage:" + echo " nebd-tcpcheck-daemon start -- start deamon process and watch on nebd-tcpcheck process" + echo " nebd-tcpcheck-daemon stop -- stop daemon process and nebd-tcpcheck" + echo " nebd-tcpcheck-daemon restart -- restart nebd-tcpcheck" + echo " nebd-tcpcheck-daemon status -- show if the nebd-tcpcheck is running by daemon" + echo "Examples:" + echo " nebd-tcpcheck-daemon start" +} + +# 检查参数启动参数，最少1个 +if [ $# -lt 1 ] +then + usage + exit +fi + +case $1 in +"start") + start + ;; +"stop") + stop + ;; +"restart") + restart + ;; +"status") + status + ;; +*) + usage + ;; +esac + From abff4b7f90102b1edb7af040079f2b32315a2603 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Mon, 23 Mar 2020 11:24:58 +0800 Subject: [PATCH 64/79] =?UTF-8?q?=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E7=94=B1puppet=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: Id17db6d4abcdf63e7d1b880f4f6989d6b5ba3fd3 --- mk-deb.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mk-deb.sh b/mk-deb.sh index 66d362beb6..64cd0e5805 100755 --- a/mk-deb.sh +++ b/mk-deb.sh @@ -1,6 +1,6 @@ #!/bin/bash dir=`pwd` -# step1 清楚生成的目录和文件 +# step1 清除生成的目录和文件 bazel clean rm -rf *deb rm -rf build @@ -48,7 +48,6 @@ for i in `find bazel-bin/|grep -w so|grep -v solib|grep -v params| grep -v test` done cp bazel-bin/src/part2/nebd-server build/nebd-package/usr/bin -cp -r etc build/nebd-package/ # step4 获取git提交版本信息，记录到debian包的配置文件 commit_id=`git show --abbrev-commit HEAD|head -n 1|awk '{print $2}'` From 2287639eda15d16690db31e373a93ab1a620fa8a Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Mon, 23 Mar 2020 17:19:22 +0800 Subject: [PATCH 65/79] use bthread mutex Change-Id: Id8c506c173c44f7b3325719f854809b88026f512 --- src/part2/file_entity.cpp | 6 +++--- src/part2/file_entity.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/part2/file_entity.cpp b/src/part2/file_entity.cpp index 19348f419f..6befe5db3b 100644 --- a/src/part2/file_entity.cpp +++ b/src/part2/file_entity.cpp @@ -45,7 +45,7 @@ int NebdFileEntity::Init(const NebdFileEntityOption& option) { int NebdFileEntity::Open() { CHECK(executor_ != nullptr) << "file entity is not inited. " << "filename: " << fileName_; - std::unique_lock lock(fileStatusMtx_); + std::unique_lock lock(fileStatusMtx_); if (status_ == NebdFileStatus::OPENED) { LOG(WARNING) << "File is already opened. " << "filename: " << fileName_ @@ -76,7 +76,7 @@ int NebdFileEntity::Open() { int NebdFileEntity::Reopen(const ExtendAttribute& xattr) { CHECK(executor_ != nullptr) << "file entity is not inited. " << "filename: " << fileName_; - std::unique_lock lock(fileStatusMtx_); + std::unique_lock lock(fileStatusMtx_); NebdFileInstancePtr fileInstance = executor_->Reopen(fileName_, xattr); if (fileInstance == nullptr) { LOG(ERROR) << "Reopen file failed. " @@ -105,7 +105,7 @@ int NebdFileEntity::Close(bool removeMeta) { // 这里的互斥锁是为了跟open请求互斥，以下情况可能导致close和open并发 // part2重启，导致文件被reopen，然后由于超时，文件准备被close // 此时用户发送了挂载卷请求对文件进行open - std::unique_lock lock(fileStatusMtx_); + std::unique_lock lock(fileStatusMtx_); if (status_ == NebdFileStatus::OPENED) { int ret = executor_->Close(fileInstance_.get()); if (ret < 0) { diff --git a/src/part2/file_entity.h b/src/part2/file_entity.h index d1568c3407..8380fc8fb7 100644 --- a/src/part2/file_entity.h +++ b/src/part2/file_entity.h @@ -191,7 +191,7 @@ class NebdFileEntity : public std::enable_shared_from_this { // 避免close时还有请求未处理完 BthreadRWLock rwLock_; // 互斥锁，用于open、close之间的互斥 - butil::Mutex fileStatusMtx_; + bthread::Mutex fileStatusMtx_; // nebd server为该文件分配的唯一标识符 int fd_; // 文件名称 From 54a2ac57d713bb1d0280a81774597eb33b8baaa1 Mon Sep 17 00:00:00 2001 From: charisu Date: Mon, 23 Mar 2020 17:30:57 +0800 Subject: [PATCH 66/79] add nebd version Change-Id: Iace90bd0642da30e76d03f46c8461998732e3e32 --- WORKSPACE | 2 +- mk-deb.sh | 16 ++++-- proto/heartbeat.proto | 4 +- src/common/BUILD | 1 + src/common/nebd_version.cpp | 38 +++++++++++++ src/common/nebd_version.h | 22 ++++++++ src/common/stringstatus.cpp | 51 ++++++++++++++++++ src/common/stringstatus.h | 63 ++++++++++++++++++++++ src/part1/heartbeat_manager.cpp | 10 ++++ src/part1/heartbeat_manager.h | 4 ++ src/part2/heartbeat_manager.cpp | 40 ++++++++++++++ src/part2/heartbeat_manager.h | 42 +++++++++++++++ src/part2/heartbeat_service.cpp | 3 ++ src/part2/nebd_server.cpp | 4 ++ tests/part2/heartbeat_manager_unittest.cpp | 14 +++++ tests/part2/heartbeat_service_test.cpp | 2 + tests/stringstatus_test.cpp | 32 +++++++++++ 17 files changed, 343 insertions(+), 5 deletions(-) create mode 100644 src/common/nebd_version.cpp create mode 100644 src/common/nebd_version.h create mode 100644 src/common/stringstatus.cpp create mode 100644 src/common/stringstatus.h create mode 100644 tests/stringstatus_test.cpp diff --git a/WORKSPACE b/WORKSPACE index 8fee15280b..df2f7b2a81 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -98,7 +98,7 @@ bind( git_repository( name = "com_netease_storage_gerrit_curve_curve", remote = "http://gerrit.storage.netease.com/curve", - commit = "0f9f1132d51e12490e3dee5cbaaa36e6ce1d1e56", + commit = "9d8c574da20e53ef0c4cd57c3d742f41135aee99", ) bind( diff --git a/mk-deb.sh b/mk-deb.sh index 64cd0e5805..4ef3cb4fca 100755 --- a/mk-deb.sh +++ b/mk-deb.sh @@ -13,12 +13,21 @@ then curve_tag_version=9.9.9 fi +#获取tag版本 +tag_version=`git status | grep -w "HEAD detached at" | awk '{print $NF}' | awk -F"v" '{print $2}'` +if [ -z ${tag_version} ] +then + echo "not found version info, set version to 9.9.9" + tag_version=9.9.9 +fi + # step2 编译 if [ "$1" = "debug" ] then bazel build ... --copt -DHAVE_ZLIB=1 --compilation_mode=dbg -s --define=with_glog=true \ --define=libunwind=true --copt -DGFLAGS_NS=google --copt \ - -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_tag_version} + -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_tag_version} \ + --copt -DNEBDVERSION=${tag_version} if [ $? -ne 0 ] then echo "build phase1 failed" @@ -27,7 +36,8 @@ then else bazel build ... --copt -DHAVE_ZLIB=1 --copt -O2 -s --define=with_glog=true \ --define=libunwind=true --copt -DGFLAGS_NS=google --copt \ - -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_tag_version} + -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_tag_version} \ + --copt -DNEBDVERSION=${tag_version} if [ $? -ne 0 ] then echo "build phase1 failed" @@ -57,7 +67,7 @@ then else debug="" fi -version="Version: 0.0.1+${commit_id}${debug}" +version="Version: ${tag_version}+${commit_id}${debug}" echo $version >> build/nebd-package/DEBIAN/control # step5 打包debian包 diff --git a/proto/heartbeat.proto b/proto/heartbeat.proto index 5d2e7a8068..471fae2bdb 100644 --- a/proto/heartbeat.proto +++ b/proto/heartbeat.proto @@ -11,7 +11,9 @@ message HeartbeatFileInfo { } message HeartbeatRequest { - repeated HeartbeatFileInfo info = 1; + required int32 pid = 1; + required string nebdVersion = 2; + repeated HeartbeatFileInfo info = 3; } message HeartbeatResponse { diff --git a/src/common/BUILD b/src/common/BUILD index fc82e087d8..a008fd1823 100644 --- a/src/common/BUILD +++ b/src/common/BUILD @@ -10,6 +10,7 @@ cc_library( "//external:glog", "//external:butil", "//external:bthread", + "//external:bvar", ], ) diff --git a/src/common/nebd_version.cpp b/src/common/nebd_version.cpp new file mode 100644 index 0000000000..713525892f --- /dev/null +++ b/src/common/nebd_version.cpp @@ -0,0 +1,38 @@ +/* + * Project: nebd + * Created Date: 2020-03-23 + * Author: charsiu + * Copyright (c) 2020 netease + */ + +#include "src/common/nebd_version.h" +#include "src/common/stringstatus.h" + +namespace nebd { +namespace common { + +// https://gcc.gnu.org/onlinedocs/gcc-4.8.5/cpp/Stringification.html +std::string NebdVersion() { + static const std::string version = +#ifdef NEBDVERSION +# define STR(val) #val +# define XSTR(val) STR(val) + std::string(XSTR(NEBDVERSION)); +#else + std::string("unknown"); +#endif + return version; +} + +const char kNebdMetricPrefix[] = "nebd"; +const char kVersion[] = "version"; + +void ExposeNebdVersion() { + static StringStatus version; + version.ExposeAs(kNebdMetricPrefix, kVersion); + version.Set(kVersion, NebdVersion()); + version.Update(); +} + +} // namespace common +} // namespace nebd diff --git a/src/common/nebd_version.h b/src/common/nebd_version.h new file mode 100644 index 0000000000..8dc05e1728 --- /dev/null +++ b/src/common/nebd_version.h @@ -0,0 +1,22 @@ +/* + * Project: nebd + * Created Date: 2020-03-23 + * Author: charsiu + * Copyright (c) 2020 netease + */ + +#ifndef SRC_COMMON_NEBD_VERSION_H_ +#define SRC_COMMON_NEBD_VERSION_H_ + +#include + +namespace nebd { +namespace common { + +std::string NebdVersion(); +void ExposeNebdVersion(); + +} // namespace common +} // namespace nebd + +#endif // SRC_COMMON_NEBD_VERSION_H_ diff --git a/src/common/stringstatus.cpp b/src/common/stringstatus.cpp new file mode 100644 index 0000000000..b9a227f964 --- /dev/null +++ b/src/common/stringstatus.cpp @@ -0,0 +1,51 @@ +/* + * Project: nebd + * Created Date: 20190819 + * Author: lixiaocui + * Copyright (c) 2019 netease + */ + +#include "src/common/stringstatus.h" + +namespace nebd { +namespace common { +void StringStatus::ExposeAs( + const std::string &prefix, const std::string &name) { + status_.expose_as(prefix, name); +} + +void StringStatus::Set(const std::string& key, const std::string& value) { + kvs_[key] = value; +} + +void StringStatus::Update() { + if (kvs_.empty()) { + return; + } + + std::string jsonStr = "{"; + int count = 0; + for (auto &item : kvs_) { + count += 1; + if (count == kvs_.size()) { + jsonStr += + "\"" + item.first + "\"" + ":" + "\"" + item.second + "\""; + } else { + jsonStr += "\"" + item.first + "\"" + ":" + "\"" + item.second + + "\"" + ","; + } + } + + jsonStr += "}"; + status_.set_value(jsonStr); +} + +std::string StringStatus::JsonBody() { + return status_.get_value(); +} + +std::string StringStatus::GetValueByKey(const std::string &key) { + return kvs_[key]; +} +} // namespace common +} // namespace nebd diff --git a/src/common/stringstatus.h b/src/common/stringstatus.h new file mode 100644 index 0000000000..a0ff1a8ab5 --- /dev/null +++ b/src/common/stringstatus.h @@ -0,0 +1,63 @@ +/* + * Project: nebd + * Created Date: 20190819 + * Author: lixiaocui + * Copyright (c) 2019 netease + */ + + +#ifndef SRC_COMMON_STRINGSTATUS_H_ +#define SRC_COMMON_STRINGSTATUS_H_ + +#include +#include +#include + +namespace nebd { +namespace common { +class StringStatus { + public: + /** + * @brief ExposeAs 用于初始化bvar + * + * @param[in] prefix, 前缀 + * @param[in] name, 名字 + */ + void ExposeAs(const std::string &prefix, const std::string &name); + + /** + * @brief Set 设置每项key-value信息 + * + * @param[in] key + * @param[in] value + */ + void Set(const std::string& key, const std::string& value); + + /** + * @brief Update 把当前key-value map中的键值对以json string的形式设置到status中 //NOLINT + */ + void Update(); + + /** + * @brief GetValueByKey 获取指定key对应的value + * + * @param[in] key 指定key + */ + std::string GetValueByKey(const std::string &key); + + /** + * @brief JsonBody 获取当前key-value map对应的json形式字符串 + */ + std::string JsonBody(); + + private: + // 需要导出的结构体的key-value map + std::map kvs_; + // 该导出项对应的status + bvar::Status status_; +}; +} // namespace common +} // namespace nebd + +#endif // SRC_COMMON_STRINGSTATUS_H_ + diff --git a/src/part1/heartbeat_manager.cpp b/src/part1/heartbeat_manager.cpp index 6ed523ce7b..db7b5722b6 100644 --- a/src/part1/heartbeat_manager.cpp +++ b/src/part1/heartbeat_manager.cpp @@ -7,11 +7,13 @@ #include "src/part1/heartbeat_manager.h" +#include #include #include #include #include "proto/heartbeat.pb.h" +#include "src/common/nebd_version.h" namespace nebd { namespace client { @@ -33,6 +35,9 @@ int HeartbeatManager::Init(const HeartbeatOption& option) { return -1; } + pid_ = getpid(); + nebdVersion_ = nebd::common::NebdVersion(); + return 0; } @@ -79,6 +84,9 @@ void HeartbeatManager::SendHeartBeat() { NebdHeartbeatService_Stub stub(&channel_); + request.set_pid(pid_); + request.set_nebdversion(nebdVersion_); + std::ostringstream oss; for (const auto& fileInfo : fileInfos) { nebd::client::HeartbeatFileInfo* info = request.add_info(); @@ -89,6 +97,8 @@ void HeartbeatManager::SendHeartBeat() { } LOG(INFO) << "Send Heartbeat request, log id = " << cntl.log_id() + << ", pid = " << request.pid() + << ", nebd version = " << request.nebdversion() << ", files [" << oss.str() << ']'; stub.KeepAlive(&cntl, &request, &response, nullptr); diff --git a/src/part1/heartbeat_manager.h b/src/part1/heartbeat_manager.h index b0deab5f13..d5ab40cada 100644 --- a/src/part1/heartbeat_manager.h +++ b/src/part1/heartbeat_manager.h @@ -71,6 +71,10 @@ class HeartbeatManager { std::atomic running_; std::atomic logId_; + // nebd version + std::string nebdVersion_; + // process id + int pid_; }; } // namespace client diff --git a/src/part2/heartbeat_manager.cpp b/src/part2/heartbeat_manager.cpp index 65919a2a25..83b5b7f8b2 100644 --- a/src/part2/heartbeat_manager.cpp +++ b/src/part2/heartbeat_manager.cpp @@ -48,6 +48,20 @@ bool HeartbeatManager::UpdateFileTimestamp(int fd, uint64_t timestamp) { return true; } +void HeartbeatManager::UpdateNebdClientInfo(int pid, const std::string& version, + uint64_t timestamp) { + WriteLockGuard writeLock(rwLock_); + const auto& iter = nebdClients_.find(pid); + if (iter == nebdClients_.end()) { + nebdClients_[pid] = + std::make_shared(pid, version, timestamp); + } else { + nebdClients_[pid]->timeStamp = timestamp; + nebdClients_[pid]->version.Set(kVersion, version); + nebdClients_[pid]->version.Update(); + } +} + void HeartbeatManager::CheckTimeoutFunc() { while (sleeper_.wait_for( std::chrono::milliseconds(checkTimeoutIntervalMs_))) { @@ -70,6 +84,7 @@ void HeartbeatManager::CheckTimeoutFunc() { << standardTime; curEntity->Close(false); } + RemoveTimeoutNebdClient(); } } @@ -82,5 +97,30 @@ bool HeartbeatManager::CheckNeedClosed(NebdFileEntityPtr entity) { return needClose; } +std::ostream& operator<<(std::ostream& os, NebdClientInfo* info) { + std::string standardTime; + TimeUtility::TimeStampToStandard(info->timeStamp / 1000, &standardTime); + os << "pid: " << info->pid << ", version: " + << info->version.GetValueByKey(kVersion) + << ", last time received heartbeat: " << standardTime; + return os; +} + +void HeartbeatManager::RemoveTimeoutNebdClient() { + WriteLockGuard writeLock(rwLock_); + auto iter = nebdClients_.begin(); + while (iter != nebdClients_.end()) { + uint64_t curTime = TimeUtility::GetTimeofDayMs(); + uint64_t interval = curTime - iter->second->timeStamp; + if (interval > (uint64_t)1000 * heartbeatTimeoutS_) { + LOG(INFO) << "Delete nebd client info which has timed out. " + << "client info: " << iter->second; + iter = nebdClients_.erase(iter); + } else { + iter++; + } + } +} + } // namespace server } // namespace nebd diff --git a/src/part2/heartbeat_manager.h b/src/part2/heartbeat_manager.h index 8e931f093f..b5a17c476b 100644 --- a/src/part2/heartbeat_manager.h +++ b/src/part2/heartbeat_manager.h @@ -8,17 +8,26 @@ #ifndef SRC_PART2_HEARTBEAT_MANAGER_H_ #define SRC_PART2_HEARTBEAT_MANAGER_H_ +#include #include // NOLINT #include #include +#include +#include #include "src/common/interrupt_sleep.h" +#include "src/common/rw_lock.h" +#include "src/common/stringstatus.h" #include "src/part2/file_manager.h" +#include "src/part2/define.h" namespace nebd { namespace server { using nebd::common::InterruptibleSleeper; +using nebd::common::RWLock; +using nebd::common::WriteLockGuard; +using nebd::common::ReadLockGuard; struct HeartbeatManagerOption { // 文件心跳超时时间（单位：秒） @@ -29,6 +38,26 @@ struct HeartbeatManagerOption { NebdFileManagerPtr fileManager; }; +const char kNebdClientMetricPrefix[] = "nebd_client_pid_"; +const char kVersion[] = "version"; + +struct NebdClientInfo { + NebdClientInfo(int pid2, const std::string& version2, + uint64_t timeStamp2) : + pid(pid2), timeStamp(timeStamp2) { + version.ExposeAs(kNebdClientMetricPrefix, + std::to_string(pid2) + "_version"); + version.Set(kVersion, version2); + version.Update(); + } + // nebd client的进程号 + int pid; + // nebd version的metric + nebd::common::StringStatus version; + // 上次心跳的时间戳 + uint64_t timeStamp; +}; + // 负责文件心跳超时管理 class HeartbeatManager { public: @@ -46,12 +75,21 @@ class HeartbeatManager { // part2收到心跳后，会通过该接口更新心跳中包含的文件在内存中记录的时间戳 // 心跳检测线程会根据该时间戳判断是否需要关闭文件 virtual bool UpdateFileTimestamp(int fd, uint64_t timestamp); + // part2收到心跳后，会通过该接口更新part1的时间戳 + virtual void UpdateNebdClientInfo(int pid, const std::string& version, + uint64_t timestamp); + std::map> GetNebdClients() { + ReadLockGuard readLock(rwLock_); + return nebdClients_; + } private: // 心跳检测线程的函数执行体 void CheckTimeoutFunc(); // 判断文件是否需要close bool CheckNeedClosed(NebdFileEntityPtr entity); + // 从内存中删除已经超时的nebdClientInfo + void RemoveTimeoutNebdClient(); private: // 当前heartbeatmanager的运行状态，true表示正在运行，false标为未运行 @@ -66,6 +104,10 @@ class HeartbeatManager { InterruptibleSleeper sleeper_; // filemanager 对象指针 NebdFileManagerPtr fileManager_; + // nebd client的信息 + std::map> nebdClients_; + // file map 读写保护锁 + RWLock rwLock_; }; } // namespace server diff --git a/src/part2/heartbeat_service.cpp b/src/part2/heartbeat_service.cpp index ece6969b6f..288fe39be2 100644 --- a/src/part2/heartbeat_service.cpp +++ b/src/part2/heartbeat_service.cpp @@ -7,6 +7,7 @@ #include "src/common/timeutility.h" #include "src/part2/heartbeat_service.h" +#include "src/part2/define.h" namespace nebd { namespace server { @@ -21,6 +22,8 @@ void NebdHeartbeatServiceImpl::KeepAlive( brpc::ClosureGuard doneGuard(done); bool ok = true; uint64_t curTime = TimeUtility::GetTimeofDayMs(); + heartbeatManager_->UpdateNebdClientInfo(request->pid(), + request->nebdversion(), curTime); for (int i = 0; i < request->info_size(); ++i) { const auto& info = request->info(i); bool res = heartbeatManager_->UpdateFileTimestamp(info.fd(), curTime); diff --git a/src/part2/nebd_server.cpp b/src/part2/nebd_server.cpp index 78e5632c48..d6c3391944 100644 --- a/src/part2/nebd_server.cpp +++ b/src/part2/nebd_server.cpp @@ -8,6 +8,7 @@ #include #include #include "src/common/file_lock.h" +#include "src/common/nebd_version.h" #include "src/part2/nebd_server.h" #include "src/part2/file_service.h" #include "src/part2/heartbeat_service.h" @@ -58,6 +59,9 @@ int NebdServer::Init(const std::string &confPath, LOG(INFO) << "NebdServer init heartbeatManager ok"; LOG(INFO) << "NebdServer init ok"; + // 暴露版本信息 + LOG(INFO) << "nebd version: " << nebd::common::NebdVersion(); + nebd::common::ExposeNebdVersion(); return 0; } diff --git a/tests/part2/heartbeat_manager_unittest.cpp b/tests/part2/heartbeat_manager_unittest.cpp index 89a5ecd938..929ca6ca14 100644 --- a/tests/part2/heartbeat_manager_unittest.cpp +++ b/tests/part2/heartbeat_manager_unittest.cpp @@ -104,6 +104,20 @@ TEST_F(HeartbeatManagerTest, UpdateTimeStampTest) { ASSERT_FALSE(heartbeatManager_->UpdateFileTimestamp(1, 100)); } +TEST_F(HeartbeatManagerTest, UpdateNebdClientInfo) { + int pid = 12345; + uint64_t timeStamp = TimeUtility::GetTimeofDayMs() - 2 * 10 * 1000; + heartbeatManager_->UpdateNebdClientInfo(pid, "0.0.1", timeStamp); + const auto& clients = heartbeatManager_->GetNebdClients(); + ASSERT_EQ(1, clients.size()); + ASSERT_NE(clients.end(), clients.find(pid)); + + ASSERT_EQ(heartbeatManager_->Run(), 0); + ::sleep(2); + ASSERT_TRUE(heartbeatManager_->GetNebdClients().empty()); + ASSERT_EQ(heartbeatManager_->Fini(), 0); +} + } // namespace server } // namespace nebd diff --git a/tests/part2/heartbeat_service_test.cpp b/tests/part2/heartbeat_service_test.cpp index 474d4f01d3..d237c09e0d 100644 --- a/tests/part2/heartbeat_service_test.cpp +++ b/tests/part2/heartbeat_service_test.cpp @@ -40,6 +40,8 @@ TEST_F(HeartbeatServiceTest, KeepAlive) { ASSERT_EQ(0, server.StartAtSockFile(kSockFile_.c_str(), &option)); nebd::client::HeartbeatRequest request; + request.set_pid(12345); + request.set_nebdversion("0.0.1"); nebd::client::HeartbeatResponse response; for (int i = 0; i < 3; ++i) { auto* info = request.add_info(); diff --git a/tests/stringstatus_test.cpp b/tests/stringstatus_test.cpp new file mode 100644 index 0000000000..098321adad --- /dev/null +++ b/tests/stringstatus_test.cpp @@ -0,0 +1,32 @@ +/* + * Project: nebd + * Created Date: 20190819 + * Author: lixiaocui + * Copyright (c) 2019 netease + */ + +#include +#include "src/common/stringstatus.h" + +namespace nebd { +namespace common { + +TEST(Common, string_status_test) { + StringStatus status; + status.ExposeAs("test1_", "1"); + status.Update(); + ASSERT_TRUE(status.JsonBody().empty()); + + status.Set("hello", "world"); + status.Update(); + ASSERT_EQ("{\"hello\":\"world\"}", status.JsonBody()); + ASSERT_EQ("world", status.GetValueByKey("hello")); + + status.Set("code", "smart"); + status.Update(); + ASSERT_EQ("{\"code\":\"smart\",\"hello\":\"world\"}", status.JsonBody()); + ASSERT_EQ("smart", status.GetValueByKey("code")); +} + +} // namespace common +} // namespace nebd From c16099ab053d0d4f9b220357a9002141a15f2226 Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Tue, 24 Mar 2020 11:01:02 +0800 Subject: [PATCH 67/79] fix tcp check Change-Id: I04993353c0cd076ee8fae06191adc92a9afcf9b5 --- nebd-package/usr/bin/nebd-check-tcp.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nebd-package/usr/bin/nebd-check-tcp.sh b/nebd-package/usr/bin/nebd-check-tcp.sh index 2170139162..955dd9b0ef 100644 --- a/nebd-package/usr/bin/nebd-check-tcp.sh +++ b/nebd-package/usr/bin/nebd-check-tcp.sh @@ -43,6 +43,10 @@ function check_rtt() for rttline in $connection_rtt_str;do rtt=`echo $rttline|awk '{print $(NF-1)}' |sed 's/|\(.*\)\/.*/\1/g'` + if [[ $rtt -eq "|-" ]];then + echo "skip this line: $rttline" + continue + fi localport=`echo $rttline|awk '{print $2}' |sed 's/|\(.*\)|.*/\1/g'` isslowtcp=`echo "$rtt $slowrtt"|awk '{print ($1 > $2)}'` if [[ $isslowtcp -eq 1 ]];then From 83d446a366d2c35c7a59fa470e3bce1889bfd129 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Thu, 26 Mar 2020 18:54:28 +0800 Subject: [PATCH 68/79] update curve tag Change-Id: I44dbb05cb1eddae0213aa8afd29bd396ba2d09b2 --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index df2f7b2a81..2e9e8d291d 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -98,7 +98,7 @@ bind( git_repository( name = "com_netease_storage_gerrit_curve_curve", remote = "http://gerrit.storage.netease.com/curve", - commit = "9d8c574da20e53ef0c4cd57c3d742f41135aee99", + tag = "v0.0.5.3", ) bind( From ac4277b8cfee59a60f96a883e1bf0c96a1a58f5c Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Mon, 30 Mar 2020 11:42:12 +0800 Subject: [PATCH 69/79] fix tcp check script Change-Id: Ia8558de45d1133c2db2c761a4769d0e1a868130f --- nebd-package/usr/bin/nebd-check-tcp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nebd-package/usr/bin/nebd-check-tcp.sh b/nebd-package/usr/bin/nebd-check-tcp.sh index 955dd9b0ef..dde035a784 100644 --- a/nebd-package/usr/bin/nebd-check-tcp.sh +++ b/nebd-package/usr/bin/nebd-check-tcp.sh @@ -43,7 +43,7 @@ function check_rtt() for rttline in $connection_rtt_str;do rtt=`echo $rttline|awk '{print $(NF-1)}' |sed 's/|\(.*\)\/.*/\1/g'` - if [[ $rtt -eq "|-" ]];then + if [[ $rtt == "|-" ]];then echo "skip this line: $rttline" continue fi From 4f6eb11171e61985e1c006815886fe43aced1396 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Wed, 15 Apr 2020 13:47:19 +0800 Subject: [PATCH 70/79] update curve client to v0.0.6.1 Change-Id: Icb98b78022ab05247e645a8e5b202a330946eb5c --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 2e9e8d291d..9dc65998c7 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -98,7 +98,7 @@ bind( git_repository( name = "com_netease_storage_gerrit_curve_curve", remote = "http://gerrit.storage.netease.com/curve", - tag = "v0.0.5.3", + tag = "v0.0.6.1", ) bind( From 88d3bd91f179adbff80cf1b41c5443037942de4b Mon Sep 17 00:00:00 2001 From: charisu Date: Mon, 20 Apr 2020 19:18:35 +0800 Subject: [PATCH 71/79] add nebd client num to metric Change-Id: Ib3f89364127d2134d6c4545cafc365a326c66098 --- src/part2/heartbeat_manager.cpp | 2 ++ src/part2/heartbeat_manager.h | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/part2/heartbeat_manager.cpp b/src/part2/heartbeat_manager.cpp index 83b5b7f8b2..0539d2fc50 100644 --- a/src/part2/heartbeat_manager.cpp +++ b/src/part2/heartbeat_manager.cpp @@ -55,6 +55,7 @@ void HeartbeatManager::UpdateNebdClientInfo(int pid, const std::string& version, if (iter == nebdClients_.end()) { nebdClients_[pid] = std::make_shared(pid, version, timestamp); + nebdClientNum_ << 1; } else { nebdClients_[pid]->timeStamp = timestamp; nebdClients_[pid]->version.Set(kVersion, version); @@ -116,6 +117,7 @@ void HeartbeatManager::RemoveTimeoutNebdClient() { LOG(INFO) << "Delete nebd client info which has timed out. " << "client info: " << iter->second; iter = nebdClients_.erase(iter); + nebdClientNum_ << -1; } else { iter++; } diff --git a/src/part2/heartbeat_manager.h b/src/part2/heartbeat_manager.h index b5a17c476b..312e48db1e 100644 --- a/src/part2/heartbeat_manager.h +++ b/src/part2/heartbeat_manager.h @@ -65,7 +65,9 @@ class HeartbeatManager { : isRunning_(false) , heartbeatTimeoutS_(option.heartbeatTimeoutS) , checkTimeoutIntervalMs_(option.checkTimeoutIntervalMs) - , fileManager_(option.fileManager) {} + , fileManager_(option.fileManager) { + nebdClientNum_.expose("nebd_client_num"); + } virtual ~HeartbeatManager() {} // 启动心跳检测线程 @@ -106,6 +108,8 @@ class HeartbeatManager { NebdFileManagerPtr fileManager_; // nebd client的信息 std::map> nebdClients_; + // nebdClient的计数器 + bvar::Adder nebdClientNum_; // file map 读写保护锁 RWLock rwLock_; }; From d559ffbc4e5cbe0624bfd7ef41d26fd9246e59bb Mon Sep 17 00:00:00 2001 From: yangyaokai Date: Fri, 24 Apr 2020 11:18:25 +0800 Subject: [PATCH 72/79] rm tcp auto restart Change-Id: I84065134d2bccf8d56e0cc662200ac61877f7d98 --- nebd-package/DEBIAN/postinst | 3 --- nebd-package/usr/bin/nebd-check-tcp.sh | 15 +++++++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/nebd-package/DEBIAN/postinst b/nebd-package/DEBIAN/postinst index 98e0f9abc1..c83b23eda3 100755 --- a/nebd-package/DEBIAN/postinst +++ b/nebd-package/DEBIAN/postinst @@ -11,7 +11,4 @@ chmod -R 777 /data/log/nebd cp /usr/bin/nebd-daemon /etc/init.d update-rc.d nebd-daemon defaults -cp /usr/bin/nebd-tcpcheck-daemon /etc/init.d -update-rc.d nebd-tcpcheck-daemon defaults - exit 0 diff --git a/nebd-package/usr/bin/nebd-check-tcp.sh b/nebd-package/usr/bin/nebd-check-tcp.sh index dde035a784..f3a0842dc0 100644 --- a/nebd-package/usr/bin/nebd-check-tcp.sh +++ b/nebd-package/usr/bin/nebd-check-tcp.sh @@ -9,6 +9,13 @@ looptimes=5 # 超时连接的计数器 declare -A slow_times +# 打印当前时间 +function echo_time() +{ + time=$(date "+%Y-%m-%d %H:%M:%S") + echo "${time}" +} + # 获取nebd server进程号及metric的端口号 function get_nebdport() { @@ -48,8 +55,13 @@ function check_rtt() continue fi localport=`echo $rttline|awk '{print $2}' |sed 's/|\(.*\)|.*/\1/g'` + if [[ $rtt == "|0" ]];then + echo "skip this line: $rttline" + continue + fi isslowtcp=`echo "$rtt $slowrtt"|awk '{print ($1 > $2)}'` if [[ $isslowtcp -eq 1 ]];then + echo_time incref $localport echo "connection_port $localport connection rtt is slow than $slowrtt, rtt is : $rtt" fi @@ -62,6 +74,7 @@ function kill_nebd_ifunhealthy() { for connection in ${!slow_times[*]};do if [[ ${slow_times[$connection]} -eq $looptimes ]];then + echo_time echo "connection_port $connection is unhealthy, nebd server will be killed. pid : $nebd_pid" sudo kill -9 $nebd_pid fi @@ -71,8 +84,6 @@ function kill_nebd_ifunhealthy() loop=1 while [[ $loop -le $looptimes ]];do - echo "check ${loop}th times. " - get_nebdport if [ $? -ne 0 ];then echo "get nebd port failed" From 53bd5c0083cdf205379109c10fc92b014ff5a916 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Thu, 30 Apr 2020 15:46:27 +0800 Subject: [PATCH 73/79] update curve-brpc commit id Change-Id: I2008db43bb8fe8f6c1c9b66329efe3c2e2f9e55d --- WORKSPACE | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 9dc65998c7..04855184a1 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -71,7 +71,7 @@ bind( git_repository( name = "com_netease_storage_gerrit_curve_curve_brpc", remote = "http://gerrit.storage.netease.com/curve/curve-brpc", - commit = "8d5e3c085b38598b6c436999029ebdefa9301450", + commit = "0f0287df3f232b8b12a30b4d99392d716de1a16b", ) bind( @@ -129,4 +129,3 @@ bind( name = "json", actual = "@jsoncpp//:json", ) - From 4739be74d7d5ba6f40cc9da926fa0530ff0a83f4 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Mon, 13 Apr 2020 17:14:09 +0800 Subject: [PATCH 74/79] check nebd pid when restart Change-Id: I2a2f862bf7e314b5f35da3daae7b622cd811f169 --- nebd-package/usr/bin/nebd-daemon | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/nebd-package/usr/bin/nebd-daemon b/nebd-package/usr/bin/nebd-daemon index b8412795a7..36037b1554 100755 --- a/nebd-package/usr/bin/nebd-daemon +++ b/nebd-package/usr/bin/nebd-daemon @@ -36,6 +36,11 @@ daemonLog=${baseLogPath}/nebd-server-daemon.log # console output consoleLog=${baseLogPath}/nebd-server-console.log +function get_nebd_server_pid() { + local tmp=`ps -ef | grep nebd-server | grep -v grep | grep -v daemon | awk '{print $2}'` + echo $tmp +} + # 启动nebd-server function start() { # 检查daemon @@ -156,7 +161,18 @@ function restart() { exit 1 fi + old_pid=$(get_nebd_server_pid) + daemon --name nebd-server --pidfile ${pidFile} --restart + + sleep 1 + new_pid=$(get_nebd_server_pid) + + if [ $old_pid == $new_pid ] + then + echo "restart nebd-server may failed, try kill -9 nebd-server" + kill -9 $old_pid + fi } # status @@ -232,4 +248,3 @@ case $1 in usage ;; esac - From 119f849d113a7af5a1a7b316da83aad76f5142a5 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Thu, 7 May 2020 17:05:01 +0800 Subject: [PATCH 75/79] fix: Discard & Flush run done Change-Id: Id41030aae7751fc57bd91e855188f58b6c259514 --- src/part2/request_executor_curve.cpp | 11 +++- tests/part2/test_request_executor_curve.cpp | 60 ++++++++++++++++++++- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/src/part2/request_executor_curve.cpp b/src/part2/request_executor_curve.cpp index 04b00c6410..dc83092aac 100644 --- a/src/part2/request_executor_curve.cpp +++ b/src/part2/request_executor_curve.cpp @@ -5,9 +5,10 @@ * Copyright (c) 2020 netease */ -#include #include "src/part2/request_executor_curve.h" +#include + namespace nebd { namespace server { @@ -135,6 +136,9 @@ int CurveRequestExecutor::GetInfo( int CurveRequestExecutor::Discard( NebdFileInstance* fd, NebdServerAioContext* aioctx) { + aioctx->ret = 0; + aioctx->cb(aioctx); + return 0; } @@ -188,7 +192,10 @@ int CurveRequestExecutor::AioWrite( int CurveRequestExecutor::Flush( NebdFileInstance* fd, NebdServerAioContext* aioctx) { - // TODO(lxc) + + aioctx->ret = 0; + aioctx->cb(aioctx); + return 0; } diff --git a/tests/part2/test_request_executor_curve.cpp b/tests/part2/test_request_executor_curve.cpp index 70bf8a12cf..1ae92f3a42 100644 --- a/tests/part2/test_request_executor_curve.cpp +++ b/tests/part2/test_request_executor_curve.cpp @@ -9,6 +9,10 @@ #include "src/part2/request_executor_curve.h" #include "tests/part2/mock_curve_client.h" +#include "proto/client.pb.h" +#include "proto/heartbeat.pb.h" +#include "src/part2/file_service.h" + namespace nebd { namespace server { @@ -18,6 +22,24 @@ using ::testing::SetArgPointee; using ::testing::DoAll; using ::testing::SaveArg; +class TestReuqestExecutorCurveClosure : public google::protobuf::Closure { + public: + TestReuqestExecutorCurveClosure() : runned_(false) {} + ~TestReuqestExecutorCurveClosure() {} + void Run() { + runned_ = true; + } + bool IsRunned() { + return runned_; + } + void Reset() { + runned_ = false; + } + + private: + bool runned_; +}; + void NebdUnitTestCallback(NebdServerAioContext* context) { std::cout << "callback" << std::endl; } @@ -319,6 +341,42 @@ TEST_F(TestReuqestExecutorCurve, test_AioWrite) { } } +TEST_F(TestReuqestExecutorCurve, test_Discard) { + auto executor = CurveRequestExecutor::GetInstance(); + std::string curveFilename("/cinder/volume-1234_cinder_"); + std::unique_ptr curveFileIns(new CurveFileInstance()); + NebdServerAioContext* aioctx = new NebdServerAioContext(); + nebd::client::DiscardResponse response; + TestReuqestExecutorCurveClosure done; + + aioctx->op = LIBAIO_OP::LIBAIO_OP_DISCARD; + aioctx->cb = NebdFileServiceCallback; + aioctx->response = &response; + aioctx->done = &done; + + ASSERT_EQ(0, executor.Discard(curveFileIns.get(), aioctx)); + ASSERT_TRUE(done.IsRunned()); + ASSERT_EQ(response.retcode(), nebd::client::RetCode::kOK); +} + +TEST_F(TestReuqestExecutorCurve, test_Flush) { + auto executor = CurveRequestExecutor::GetInstance(); + std::string curveFilename("/cinder/volume-1234_cinder_"); + std::unique_ptr curveFileIns(new CurveFileInstance()); + NebdServerAioContext* aioctx = new NebdServerAioContext(); + nebd::client::FlushResponse response; + TestReuqestExecutorCurveClosure done; + + aioctx->op = LIBAIO_OP::LIBAIO_OP_FLUSH; + aioctx->cb = NebdFileServiceCallback; + aioctx->response = &response; + aioctx->done = &done; + + ASSERT_EQ(0, executor.Flush(curveFileIns.get(), aioctx)); + ASSERT_TRUE(done.IsRunned()); + ASSERT_EQ(response.retcode(), nebd::client::RetCode::kOK); +} + TEST_F(TestReuqestExecutorCurve, test_InvalidCache) { auto executor = CurveRequestExecutor::GetInstance(); std::string curveFilename("/cinder/volume-1234_cinder_"); @@ -378,5 +436,3 @@ int main(int argc, char ** argv) { ::testing::InitGoogleMock(&argc, argv); return RUN_ALL_TESTS(); } - - From a53f2cdf0411e39440028f352b9a4e3d4a6417c5 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Tue, 9 Jun 2020 19:34:41 +0800 Subject: [PATCH 76/79] modify curve filename parser & disable error log to console Change-Id: Ie6053dc0f9bf560e4d870480c963c28ed3f91121 --- WORKSPACE | 2 +- src/part1/nebd_client.cpp | 1 + src/part2/request_executor_curve.cpp | 10 +++++++--- src/part2/request_executor_curve.h | 4 +++- tests/part2/test_request_executor_curve.cpp | 5 ++++- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 04855184a1..1c4e995575 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -98,7 +98,7 @@ bind( git_repository( name = "com_netease_storage_gerrit_curve_curve", remote = "http://gerrit.storage.netease.com/curve", - tag = "v0.0.6.1", + commit = "ff44014a2738b1ac6e297bac38600aa9257e402e", ) bind( diff --git a/src/part1/nebd_client.cpp b/src/part1/nebd_client.cpp index 3737d27144..60a6eba2a3 100644 --- a/src/part1/nebd_client.cpp +++ b/src/part1/nebd_client.cpp @@ -545,6 +545,7 @@ void NebdClient::InitLogger(const LogOption& logOption) { static const char* kProcessName = "nebd-client"; FLAGS_log_dir = logOption.logPath; + FLAGS_stderrthreshold = 3; google::InitGoogleLogging(kProcessName); } diff --git a/src/part2/request_executor_curve.cpp b/src/part2/request_executor_curve.cpp index dc83092aac..2c12424182 100644 --- a/src/part2/request_executor_curve.cpp +++ b/src/part2/request_executor_curve.cpp @@ -15,25 +15,29 @@ namespace server { using ::curve::client::UserInfo_t; std::string FileNameParser::Parse(const std::string& fileName) { - int beginPos = fileName.find_first_of("/"); + auto beginPos = fileName.find_first_of("/"); if (beginPos == std::string::npos) { LOG(ERROR) << "error format fileName: " << fileName; return ""; } beginPos += 1; - int endPos = fileName.find_last_of(":"); + auto endPos = fileName.find_last_of(":"); if (endPos == std::string::npos) { LOG(ERROR) << "error format fileName: " << fileName; return ""; } + if (endPos < beginPos) { + endPos = fileName.length(); + } + if (beginPos >= endPos) { LOG(ERROR) << "error format fileName: " << fileName; return ""; } - int length = endPos - beginPos; + auto length = endPos - beginPos; if (length <= 2) { LOG(ERROR) << "error format fileName: " << fileName; return ""; diff --git a/src/part2/request_executor_curve.h b/src/part2/request_executor_curve.h index b84b9131e8..441820a9a4 100644 --- a/src/part2/request_executor_curve.h +++ b/src/part2/request_executor_curve.h @@ -39,7 +39,9 @@ class FileNameParser { public: /** * @brief 解析fileName - * 一般格式: "cbd:pool1//cinder/volume-6f30d296-07f7-452e-a983-513191f8cd95_cinder_:/etc/curve/client.conf" //NOLINT + * 一般格式: + * qemu "cbd:pool1//cinder/volume-6f30d296-07f7-452e-a983-513191f8cd95_cinder_:/etc/curve/client.conf" //NOLINT + * nbd "cbd:pool1//cinder/volume-6f30d296-07f7-452e-a983-513191f8cd95_cinder_" // NOLINT * @param[in] fileName * @return 解析出的字符串: "/cinder/volume-6f30d296-07f7-452e-a983-513191f8cd95_cinder_" //NOLINT */ diff --git a/tests/part2/test_request_executor_curve.cpp b/tests/part2/test_request_executor_curve.cpp index 1ae92f3a42..479a2af60d 100644 --- a/tests/part2/test_request_executor_curve.cpp +++ b/tests/part2/test_request_executor_curve.cpp @@ -421,10 +421,13 @@ TEST(TestFileNameParser, test_Parse) { ASSERT_EQ("", FileNameParser::Parse(fileName)); fileName = "cbd:pool1//cinder/volume-1234_cinder_"; - ASSERT_EQ("", FileNameParser::Parse(fileName)); + ASSERT_EQ(res, FileNameParser::Parse(fileName)); fileName = "cbd:pool1//:"; ASSERT_EQ("", FileNameParser::Parse(fileName)); + + fileName = "cbd:pool1//"; + ASSERT_EQ("", FileNameParser::Parse(fileName)); } From 1ddad04b6e4b40cbdb6fe2d4eefcc62bea17e129 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Thu, 2 Jul 2020 14:02:08 +0800 Subject: [PATCH 77/79] Moving nebd into its own subdirectory Change-Id: I98d5c2795ac3fe59c32f21286c1059446c577ea6 --- {3rdparty => nebd/3rdparty}/CMakeLists.txt | 0 CMakeLists.txt => nebd/CMakeLists.txt | 0 Dockerfile.develop => nebd/Dockerfile.develop | 0 Dockerfile.package => nebd/Dockerfile.package | 0 Dockerfile.unittest => nebd/Dockerfile.unittest | 0 README => nebd/README | 0 WORKSPACE => nebd/WORKSPACE | 0 {bazel => nebd/bazel}/gmock.BUILD | 0 {bazel => nebd/bazel}/jsoncpp.BUILD | 0 {bazel => nebd/bazel}/leveldb.BUILD | 0 {cmake => nebd/cmake}/modules/BuildLibconfig.cmake | 0 {debian => nebd/debian}/changelog | 0 {debian => nebd/debian}/compat | 0 {debian => nebd/debian}/control | 0 {debian => nebd/debian}/nebd.dirs | 0 {debian => nebd/debian}/nebd.install | 0 {debian => nebd/debian}/rules | 0 {debian => nebd/debian}/source/format | 0 do_cmake => nebd/do_cmake | 0 do_package => nebd/do_package | 0 docker_unittest.sh => nebd/docker_unittest.sh | 0 {etc => nebd/etc}/nebd/nebd-client.conf | 0 {etc => nebd/etc}/nebd/nebd-server.conf | 0 filterbr.py => nebd/filterbr.py | 0 get_3rdparty => nebd/get_3rdparty | 0 install_deps => nebd/install_deps | 0 mk-deb.sh => nebd/mk-deb.sh | 0 {nebd-package => nebd/nebd-package}/DEBIAN/control | 0 {nebd-package => nebd/nebd-package}/DEBIAN/postinst | 0 {nebd-package => nebd/nebd-package}/data/log/nebd/client/.gitkeep | 0 {nebd-package => nebd/nebd-package}/data/log/nebd/server/.gitkeep | 0 {nebd-package => nebd/nebd-package}/data/nebd/lock/.gitkeep | 0 {nebd-package => nebd/nebd-package}/usr/bin/nebd-check-tcp.sh | 0 {nebd-package => nebd/nebd-package}/usr/bin/nebd-daemon | 0 {nebd-package => nebd/nebd-package}/usr/bin/nebd-tcpcheck-daemon | 0 {proto => nebd/proto}/BUILD | 0 {proto => nebd/proto}/client.proto | 0 {proto => nebd/proto}/heartbeat.proto | 0 {proto => nebd/proto}/nebd-common.proto | 0 {src => nebd/src}/CMakeLists.txt | 0 {src => nebd/src}/common/BUILD | 0 {src => nebd/src}/common/CMakeLists.txt | 0 {src => nebd/src}/common/configuration.cpp | 0 {src => nebd/src}/common/configuration.h | 0 {src => nebd/src}/common/crc32.h | 0 {src => nebd/src}/common/file_lock.cpp | 0 {src => nebd/src}/common/file_lock.h | 0 {src => nebd/src}/common/interrupt_sleep.h | 0 {src => nebd/src}/common/name_lock.cpp | 0 {src => nebd/src}/common/name_lock.h | 0 {src => nebd/src}/common/nebd_version.cpp | 0 {src => nebd/src}/common/nebd_version.h | 0 {src => nebd/src}/common/posix_wrapper.cpp | 0 {src => nebd/src}/common/posix_wrapper.h | 0 {src => nebd/src}/common/rw_lock.h | 0 {src => nebd/src}/common/stringstatus.cpp | 0 {src => nebd/src}/common/stringstatus.h | 0 {src => nebd/src}/common/timeutility.h | 0 {src => nebd/src}/common/uncopyable.h | 0 {src => nebd/src}/logrotate.conf | 0 {src => nebd/src}/part1/BUILD | 0 {src => nebd/src}/part1/CMakeLists.txt | 0 {src => nebd/src}/part1/async_request_closure.cpp | 0 {src => nebd/src}/part1/async_request_closure.h | 0 {src => nebd/src}/part1/heartbeat_manager.cpp | 0 {src => nebd/src}/part1/heartbeat_manager.h | 0 {src => nebd/src}/part1/libnebd.cpp | 0 {src => nebd/src}/part1/libnebd.h | 0 {src => nebd/src}/part1/libnebd_file.cpp | 0 {src => nebd/src}/part1/libnebd_file.h | 0 {src => nebd/src}/part1/nebd_client.cpp | 0 {src => nebd/src}/part1/nebd_client.h | 0 {src => nebd/src}/part1/nebd_common.h | 0 {src => nebd/src}/part1/nebd_metacache.cpp | 0 {src => nebd/src}/part1/nebd_metacache.h | 0 {src => nebd/src}/part2/BUILD | 0 {src => nebd/src}/part2/CMakeLists.txt | 0 {src => nebd/src}/part2/define.h | 0 {src => nebd/src}/part2/file_entity.cpp | 0 {src => nebd/src}/part2/file_entity.h | 0 {src => nebd/src}/part2/file_manager.cpp | 0 {src => nebd/src}/part2/file_manager.h | 0 {src => nebd/src}/part2/file_service.cpp | 0 {src => nebd/src}/part2/file_service.h | 0 {src => nebd/src}/part2/heartbeat_manager.cpp | 0 {src => nebd/src}/part2/heartbeat_manager.h | 0 {src => nebd/src}/part2/heartbeat_service.cpp | 0 {src => nebd/src}/part2/heartbeat_service.h | 0 {src => nebd/src}/part2/main.cpp | 0 {src => nebd/src}/part2/metafile_manager.cpp | 0 {src => nebd/src}/part2/metafile_manager.h | 0 {src => nebd/src}/part2/nebd_server.cpp | 0 {src => nebd/src}/part2/nebd_server.h | 0 {src => nebd/src}/part2/request_executor.cpp | 0 {src => nebd/src}/part2/request_executor.h | 0 {src => nebd/src}/part2/request_executor_ceph.cpp | 0 {src => nebd/src}/part2/request_executor_ceph.h | 0 {src => nebd/src}/part2/request_executor_curve.cpp | 0 {src => nebd/src}/part2/request_executor_curve.h | 0 {src => nebd/src}/part2/util.cpp | 0 {src => nebd/src}/part2/util.h | 0 {tests => nebd/tests}/CMakeLists.txt | 0 {tests => nebd/tests}/common/BUILD | 0 {tests => nebd/tests}/common/CMakeLists.txt | 0 {tests => nebd/tests}/common/configuration_test.cpp | 0 {tests => nebd/tests}/common/crc32_test.cpp | 0 {tests => nebd/tests}/common/interruptible_sleeper_test.cpp | 0 {tests => nebd/tests}/common/posix_wrapper_test.cpp | 0 {tests => nebd/tests}/common/rw_lock_test.cpp | 0 {tests => nebd/tests}/common/test_name_lock.cpp | 0 {tests => nebd/tests}/part1/BUILD | 0 {tests => nebd/tests}/part1/CMakeLists.txt | 0 {tests => nebd/tests}/part1/fake_file_service.cpp | 0 {tests => nebd/tests}/part1/fake_file_service.h | 0 {tests => nebd/tests}/part1/fake_heartbeat_service.h | 0 {tests => nebd/tests}/part1/heartbeat_manager_unittest.cpp | 0 {tests => nebd/tests}/part1/mock_file_service.h | 0 {tests => nebd/tests}/part1/mock_heartbeat_service.h | 0 {tests => nebd/tests}/part1/nebd_client_unittest.cpp | 0 {tests => nebd/tests}/part1/nebd_lib_unittest.cpp | 0 {tests => nebd/tests}/part1/nebd_metacache_unittest.cpp | 0 {tests => nebd/tests}/part2/BUILD | 0 {tests => nebd/tests}/part2/CMakeLists.txt | 0 {tests => nebd/tests}/part2/file_manager_unittest.cpp | 0 {tests => nebd/tests}/part2/file_service_unittest.cpp | 0 {tests => nebd/tests}/part2/heartbeat_manager_unittest.cpp | 0 {tests => nebd/tests}/part2/heartbeat_service_test.cpp | 0 {tests => nebd/tests}/part2/metafile_manager_test.cpp | 0 {tests => nebd/tests}/part2/mock_curve_client.h | 0 {tests => nebd/tests}/part2/mock_file_entity.h | 0 {tests => nebd/tests}/part2/mock_file_manager.h | 0 {tests => nebd/tests}/part2/mock_heartbeat_manager.h | 0 {tests => nebd/tests}/part2/mock_metafile_manager.h | 0 {tests => nebd/tests}/part2/mock_posix_wrapper.h | 0 {tests => nebd/tests}/part2/mock_request_executor.h | 0 {tests => nebd/tests}/part2/test_nebd_server.cpp | 0 {tests => nebd/tests}/part2/test_request_executor_curve.cpp | 0 {tests => nebd/tests}/stringstatus_test.cpp | 0 {tests => nebd/tests}/utils/BUILD | 0 {tests => nebd/tests}/utils/config_generator.h | 0 140 files changed, 0 insertions(+), 0 deletions(-) rename {3rdparty => nebd/3rdparty}/CMakeLists.txt (100%) rename CMakeLists.txt => nebd/CMakeLists.txt (100%) rename Dockerfile.develop => nebd/Dockerfile.develop (100%) rename Dockerfile.package => nebd/Dockerfile.package (100%) rename Dockerfile.unittest => nebd/Dockerfile.unittest (100%) rename README => nebd/README (100%) rename WORKSPACE => nebd/WORKSPACE (100%) rename {bazel => nebd/bazel}/gmock.BUILD (100%) rename {bazel => nebd/bazel}/jsoncpp.BUILD (100%) rename {bazel => nebd/bazel}/leveldb.BUILD (100%) rename {cmake => nebd/cmake}/modules/BuildLibconfig.cmake (100%) rename {debian => nebd/debian}/changelog (100%) rename {debian => nebd/debian}/compat (100%) rename {debian => nebd/debian}/control (100%) rename {debian => nebd/debian}/nebd.dirs (100%) rename {debian => nebd/debian}/nebd.install (100%) rename {debian => nebd/debian}/rules (100%) rename {debian => nebd/debian}/source/format (100%) rename do_cmake => nebd/do_cmake (100%) rename do_package => nebd/do_package (100%) rename docker_unittest.sh => nebd/docker_unittest.sh (100%) rename {etc => nebd/etc}/nebd/nebd-client.conf (100%) rename {etc => nebd/etc}/nebd/nebd-server.conf (100%) rename filterbr.py => nebd/filterbr.py (100%) rename get_3rdparty => nebd/get_3rdparty (100%) rename install_deps => nebd/install_deps (100%) rename mk-deb.sh => nebd/mk-deb.sh (100%) rename {nebd-package => nebd/nebd-package}/DEBIAN/control (100%) rename {nebd-package => nebd/nebd-package}/DEBIAN/postinst (100%) rename {nebd-package => nebd/nebd-package}/data/log/nebd/client/.gitkeep (100%) rename {nebd-package => nebd/nebd-package}/data/log/nebd/server/.gitkeep (100%) rename {nebd-package => nebd/nebd-package}/data/nebd/lock/.gitkeep (100%) rename {nebd-package => nebd/nebd-package}/usr/bin/nebd-check-tcp.sh (100%) rename {nebd-package => nebd/nebd-package}/usr/bin/nebd-daemon (100%) rename {nebd-package => nebd/nebd-package}/usr/bin/nebd-tcpcheck-daemon (100%) rename {proto => nebd/proto}/BUILD (100%) rename {proto => nebd/proto}/client.proto (100%) rename {proto => nebd/proto}/heartbeat.proto (100%) rename {proto => nebd/proto}/nebd-common.proto (100%) rename {src => nebd/src}/CMakeLists.txt (100%) rename {src => nebd/src}/common/BUILD (100%) rename {src => nebd/src}/common/CMakeLists.txt (100%) rename {src => nebd/src}/common/configuration.cpp (100%) rename {src => nebd/src}/common/configuration.h (100%) rename {src => nebd/src}/common/crc32.h (100%) rename {src => nebd/src}/common/file_lock.cpp (100%) rename {src => nebd/src}/common/file_lock.h (100%) rename {src => nebd/src}/common/interrupt_sleep.h (100%) rename {src => nebd/src}/common/name_lock.cpp (100%) rename {src => nebd/src}/common/name_lock.h (100%) rename {src => nebd/src}/common/nebd_version.cpp (100%) rename {src => nebd/src}/common/nebd_version.h (100%) rename {src => nebd/src}/common/posix_wrapper.cpp (100%) rename {src => nebd/src}/common/posix_wrapper.h (100%) rename {src => nebd/src}/common/rw_lock.h (100%) rename {src => nebd/src}/common/stringstatus.cpp (100%) rename {src => nebd/src}/common/stringstatus.h (100%) rename {src => nebd/src}/common/timeutility.h (100%) rename {src => nebd/src}/common/uncopyable.h (100%) rename {src => nebd/src}/logrotate.conf (100%) rename {src => nebd/src}/part1/BUILD (100%) rename {src => nebd/src}/part1/CMakeLists.txt (100%) rename {src => nebd/src}/part1/async_request_closure.cpp (100%) rename {src => nebd/src}/part1/async_request_closure.h (100%) rename {src => nebd/src}/part1/heartbeat_manager.cpp (100%) rename {src => nebd/src}/part1/heartbeat_manager.h (100%) rename {src => nebd/src}/part1/libnebd.cpp (100%) rename {src => nebd/src}/part1/libnebd.h (100%) rename {src => nebd/src}/part1/libnebd_file.cpp (100%) rename {src => nebd/src}/part1/libnebd_file.h (100%) rename {src => nebd/src}/part1/nebd_client.cpp (100%) rename {src => nebd/src}/part1/nebd_client.h (100%) rename {src => nebd/src}/part1/nebd_common.h (100%) rename {src => nebd/src}/part1/nebd_metacache.cpp (100%) rename {src => nebd/src}/part1/nebd_metacache.h (100%) rename {src => nebd/src}/part2/BUILD (100%) rename {src => nebd/src}/part2/CMakeLists.txt (100%) rename {src => nebd/src}/part2/define.h (100%) rename {src => nebd/src}/part2/file_entity.cpp (100%) rename {src => nebd/src}/part2/file_entity.h (100%) rename {src => nebd/src}/part2/file_manager.cpp (100%) rename {src => nebd/src}/part2/file_manager.h (100%) rename {src => nebd/src}/part2/file_service.cpp (100%) rename {src => nebd/src}/part2/file_service.h (100%) rename {src => nebd/src}/part2/heartbeat_manager.cpp (100%) rename {src => nebd/src}/part2/heartbeat_manager.h (100%) rename {src => nebd/src}/part2/heartbeat_service.cpp (100%) rename {src => nebd/src}/part2/heartbeat_service.h (100%) rename {src => nebd/src}/part2/main.cpp (100%) rename {src => nebd/src}/part2/metafile_manager.cpp (100%) rename {src => nebd/src}/part2/metafile_manager.h (100%) rename {src => nebd/src}/part2/nebd_server.cpp (100%) rename {src => nebd/src}/part2/nebd_server.h (100%) rename {src => nebd/src}/part2/request_executor.cpp (100%) rename {src => nebd/src}/part2/request_executor.h (100%) rename {src => nebd/src}/part2/request_executor_ceph.cpp (100%) rename {src => nebd/src}/part2/request_executor_ceph.h (100%) rename {src => nebd/src}/part2/request_executor_curve.cpp (100%) rename {src => nebd/src}/part2/request_executor_curve.h (100%) rename {src => nebd/src}/part2/util.cpp (100%) rename {src => nebd/src}/part2/util.h (100%) rename {tests => nebd/tests}/CMakeLists.txt (100%) rename {tests => nebd/tests}/common/BUILD (100%) rename {tests => nebd/tests}/common/CMakeLists.txt (100%) rename {tests => nebd/tests}/common/configuration_test.cpp (100%) rename {tests => nebd/tests}/common/crc32_test.cpp (100%) rename {tests => nebd/tests}/common/interruptible_sleeper_test.cpp (100%) rename {tests => nebd/tests}/common/posix_wrapper_test.cpp (100%) rename {tests => nebd/tests}/common/rw_lock_test.cpp (100%) rename {tests => nebd/tests}/common/test_name_lock.cpp (100%) rename {tests => nebd/tests}/part1/BUILD (100%) rename {tests => nebd/tests}/part1/CMakeLists.txt (100%) rename {tests => nebd/tests}/part1/fake_file_service.cpp (100%) rename {tests => nebd/tests}/part1/fake_file_service.h (100%) rename {tests => nebd/tests}/part1/fake_heartbeat_service.h (100%) rename {tests => nebd/tests}/part1/heartbeat_manager_unittest.cpp (100%) rename {tests => nebd/tests}/part1/mock_file_service.h (100%) rename {tests => nebd/tests}/part1/mock_heartbeat_service.h (100%) rename {tests => nebd/tests}/part1/nebd_client_unittest.cpp (100%) rename {tests => nebd/tests}/part1/nebd_lib_unittest.cpp (100%) rename {tests => nebd/tests}/part1/nebd_metacache_unittest.cpp (100%) rename {tests => nebd/tests}/part2/BUILD (100%) rename {tests => nebd/tests}/part2/CMakeLists.txt (100%) rename {tests => nebd/tests}/part2/file_manager_unittest.cpp (100%) rename {tests => nebd/tests}/part2/file_service_unittest.cpp (100%) rename {tests => nebd/tests}/part2/heartbeat_manager_unittest.cpp (100%) rename {tests => nebd/tests}/part2/heartbeat_service_test.cpp (100%) rename {tests => nebd/tests}/part2/metafile_manager_test.cpp (100%) rename {tests => nebd/tests}/part2/mock_curve_client.h (100%) rename {tests => nebd/tests}/part2/mock_file_entity.h (100%) rename {tests => nebd/tests}/part2/mock_file_manager.h (100%) rename {tests => nebd/tests}/part2/mock_heartbeat_manager.h (100%) rename {tests => nebd/tests}/part2/mock_metafile_manager.h (100%) rename {tests => nebd/tests}/part2/mock_posix_wrapper.h (100%) rename {tests => nebd/tests}/part2/mock_request_executor.h (100%) rename {tests => nebd/tests}/part2/test_nebd_server.cpp (100%) rename {tests => nebd/tests}/part2/test_request_executor_curve.cpp (100%) rename {tests => nebd/tests}/stringstatus_test.cpp (100%) rename {tests => nebd/tests}/utils/BUILD (100%) rename {tests => nebd/tests}/utils/config_generator.h (100%) diff --git a/3rdparty/CMakeLists.txt b/nebd/3rdparty/CMakeLists.txt similarity index 100% rename from 3rdparty/CMakeLists.txt rename to nebd/3rdparty/CMakeLists.txt diff --git a/CMakeLists.txt b/nebd/CMakeLists.txt similarity index 100% rename from CMakeLists.txt rename to nebd/CMakeLists.txt diff --git a/Dockerfile.develop b/nebd/Dockerfile.develop similarity index 100% rename from Dockerfile.develop rename to nebd/Dockerfile.develop diff --git a/Dockerfile.package b/nebd/Dockerfile.package similarity index 100% rename from Dockerfile.package rename to nebd/Dockerfile.package diff --git a/Dockerfile.unittest b/nebd/Dockerfile.unittest similarity index 100% rename from Dockerfile.unittest rename to nebd/Dockerfile.unittest diff --git a/README b/nebd/README similarity index 100% rename from README rename to nebd/README diff --git a/WORKSPACE b/nebd/WORKSPACE similarity index 100% rename from WORKSPACE rename to nebd/WORKSPACE diff --git a/bazel/gmock.BUILD b/nebd/bazel/gmock.BUILD similarity index 100% rename from bazel/gmock.BUILD rename to nebd/bazel/gmock.BUILD diff --git a/bazel/jsoncpp.BUILD b/nebd/bazel/jsoncpp.BUILD similarity index 100% rename from bazel/jsoncpp.BUILD rename to nebd/bazel/jsoncpp.BUILD diff --git a/bazel/leveldb.BUILD b/nebd/bazel/leveldb.BUILD similarity index 100% rename from bazel/leveldb.BUILD rename to nebd/bazel/leveldb.BUILD diff --git a/cmake/modules/BuildLibconfig.cmake b/nebd/cmake/modules/BuildLibconfig.cmake similarity index 100% rename from cmake/modules/BuildLibconfig.cmake rename to nebd/cmake/modules/BuildLibconfig.cmake diff --git a/debian/changelog b/nebd/debian/changelog similarity index 100% rename from debian/changelog rename to nebd/debian/changelog diff --git a/debian/compat b/nebd/debian/compat similarity index 100% rename from debian/compat rename to nebd/debian/compat diff --git a/debian/control b/nebd/debian/control similarity index 100% rename from debian/control rename to nebd/debian/control diff --git a/debian/nebd.dirs b/nebd/debian/nebd.dirs similarity index 100% rename from debian/nebd.dirs rename to nebd/debian/nebd.dirs diff --git a/debian/nebd.install b/nebd/debian/nebd.install similarity index 100% rename from debian/nebd.install rename to nebd/debian/nebd.install diff --git a/debian/rules b/nebd/debian/rules similarity index 100% rename from debian/rules rename to nebd/debian/rules diff --git a/debian/source/format b/nebd/debian/source/format similarity index 100% rename from debian/source/format rename to nebd/debian/source/format diff --git a/do_cmake b/nebd/do_cmake similarity index 100% rename from do_cmake rename to nebd/do_cmake diff --git a/do_package b/nebd/do_package similarity index 100% rename from do_package rename to nebd/do_package diff --git a/docker_unittest.sh b/nebd/docker_unittest.sh similarity index 100% rename from docker_unittest.sh rename to nebd/docker_unittest.sh diff --git a/etc/nebd/nebd-client.conf b/nebd/etc/nebd/nebd-client.conf similarity index 100% rename from etc/nebd/nebd-client.conf rename to nebd/etc/nebd/nebd-client.conf diff --git a/etc/nebd/nebd-server.conf b/nebd/etc/nebd/nebd-server.conf similarity index 100% rename from etc/nebd/nebd-server.conf rename to nebd/etc/nebd/nebd-server.conf diff --git a/filterbr.py b/nebd/filterbr.py similarity index 100% rename from filterbr.py rename to nebd/filterbr.py diff --git a/get_3rdparty b/nebd/get_3rdparty similarity index 100% rename from get_3rdparty rename to nebd/get_3rdparty diff --git a/install_deps b/nebd/install_deps similarity index 100% rename from install_deps rename to nebd/install_deps diff --git a/mk-deb.sh b/nebd/mk-deb.sh similarity index 100% rename from mk-deb.sh rename to nebd/mk-deb.sh diff --git a/nebd-package/DEBIAN/control b/nebd/nebd-package/DEBIAN/control similarity index 100% rename from nebd-package/DEBIAN/control rename to nebd/nebd-package/DEBIAN/control diff --git a/nebd-package/DEBIAN/postinst b/nebd/nebd-package/DEBIAN/postinst similarity index 100% rename from nebd-package/DEBIAN/postinst rename to nebd/nebd-package/DEBIAN/postinst diff --git a/nebd-package/data/log/nebd/client/.gitkeep b/nebd/nebd-package/data/log/nebd/client/.gitkeep similarity index 100% rename from nebd-package/data/log/nebd/client/.gitkeep rename to nebd/nebd-package/data/log/nebd/client/.gitkeep diff --git a/nebd-package/data/log/nebd/server/.gitkeep b/nebd/nebd-package/data/log/nebd/server/.gitkeep similarity index 100% rename from nebd-package/data/log/nebd/server/.gitkeep rename to nebd/nebd-package/data/log/nebd/server/.gitkeep diff --git a/nebd-package/data/nebd/lock/.gitkeep b/nebd/nebd-package/data/nebd/lock/.gitkeep similarity index 100% rename from nebd-package/data/nebd/lock/.gitkeep rename to nebd/nebd-package/data/nebd/lock/.gitkeep diff --git a/nebd-package/usr/bin/nebd-check-tcp.sh b/nebd/nebd-package/usr/bin/nebd-check-tcp.sh similarity index 100% rename from nebd-package/usr/bin/nebd-check-tcp.sh rename to nebd/nebd-package/usr/bin/nebd-check-tcp.sh diff --git a/nebd-package/usr/bin/nebd-daemon b/nebd/nebd-package/usr/bin/nebd-daemon similarity index 100% rename from nebd-package/usr/bin/nebd-daemon rename to nebd/nebd-package/usr/bin/nebd-daemon diff --git a/nebd-package/usr/bin/nebd-tcpcheck-daemon b/nebd/nebd-package/usr/bin/nebd-tcpcheck-daemon similarity index 100% rename from nebd-package/usr/bin/nebd-tcpcheck-daemon rename to nebd/nebd-package/usr/bin/nebd-tcpcheck-daemon diff --git a/proto/BUILD b/nebd/proto/BUILD similarity index 100% rename from proto/BUILD rename to nebd/proto/BUILD diff --git a/proto/client.proto b/nebd/proto/client.proto similarity index 100% rename from proto/client.proto rename to nebd/proto/client.proto diff --git a/proto/heartbeat.proto b/nebd/proto/heartbeat.proto similarity index 100% rename from proto/heartbeat.proto rename to nebd/proto/heartbeat.proto diff --git a/proto/nebd-common.proto b/nebd/proto/nebd-common.proto similarity index 100% rename from proto/nebd-common.proto rename to nebd/proto/nebd-common.proto diff --git a/src/CMakeLists.txt b/nebd/src/CMakeLists.txt similarity index 100% rename from src/CMakeLists.txt rename to nebd/src/CMakeLists.txt diff --git a/src/common/BUILD b/nebd/src/common/BUILD similarity index 100% rename from src/common/BUILD rename to nebd/src/common/BUILD diff --git a/src/common/CMakeLists.txt b/nebd/src/common/CMakeLists.txt similarity index 100% rename from src/common/CMakeLists.txt rename to nebd/src/common/CMakeLists.txt diff --git a/src/common/configuration.cpp b/nebd/src/common/configuration.cpp similarity index 100% rename from src/common/configuration.cpp rename to nebd/src/common/configuration.cpp diff --git a/src/common/configuration.h b/nebd/src/common/configuration.h similarity index 100% rename from src/common/configuration.h rename to nebd/src/common/configuration.h diff --git a/src/common/crc32.h b/nebd/src/common/crc32.h similarity index 100% rename from src/common/crc32.h rename to nebd/src/common/crc32.h diff --git a/src/common/file_lock.cpp b/nebd/src/common/file_lock.cpp similarity index 100% rename from src/common/file_lock.cpp rename to nebd/src/common/file_lock.cpp diff --git a/src/common/file_lock.h b/nebd/src/common/file_lock.h similarity index 100% rename from src/common/file_lock.h rename to nebd/src/common/file_lock.h diff --git a/src/common/interrupt_sleep.h b/nebd/src/common/interrupt_sleep.h similarity index 100% rename from src/common/interrupt_sleep.h rename to nebd/src/common/interrupt_sleep.h diff --git a/src/common/name_lock.cpp b/nebd/src/common/name_lock.cpp similarity index 100% rename from src/common/name_lock.cpp rename to nebd/src/common/name_lock.cpp diff --git a/src/common/name_lock.h b/nebd/src/common/name_lock.h similarity index 100% rename from src/common/name_lock.h rename to nebd/src/common/name_lock.h diff --git a/src/common/nebd_version.cpp b/nebd/src/common/nebd_version.cpp similarity index 100% rename from src/common/nebd_version.cpp rename to nebd/src/common/nebd_version.cpp diff --git a/src/common/nebd_version.h b/nebd/src/common/nebd_version.h similarity index 100% rename from src/common/nebd_version.h rename to nebd/src/common/nebd_version.h diff --git a/src/common/posix_wrapper.cpp b/nebd/src/common/posix_wrapper.cpp similarity index 100% rename from src/common/posix_wrapper.cpp rename to nebd/src/common/posix_wrapper.cpp diff --git a/src/common/posix_wrapper.h b/nebd/src/common/posix_wrapper.h similarity index 100% rename from src/common/posix_wrapper.h rename to nebd/src/common/posix_wrapper.h diff --git a/src/common/rw_lock.h b/nebd/src/common/rw_lock.h similarity index 100% rename from src/common/rw_lock.h rename to nebd/src/common/rw_lock.h diff --git a/src/common/stringstatus.cpp b/nebd/src/common/stringstatus.cpp similarity index 100% rename from src/common/stringstatus.cpp rename to nebd/src/common/stringstatus.cpp diff --git a/src/common/stringstatus.h b/nebd/src/common/stringstatus.h similarity index 100% rename from src/common/stringstatus.h rename to nebd/src/common/stringstatus.h diff --git a/src/common/timeutility.h b/nebd/src/common/timeutility.h similarity index 100% rename from src/common/timeutility.h rename to nebd/src/common/timeutility.h diff --git a/src/common/uncopyable.h b/nebd/src/common/uncopyable.h similarity index 100% rename from src/common/uncopyable.h rename to nebd/src/common/uncopyable.h diff --git a/src/logrotate.conf b/nebd/src/logrotate.conf similarity index 100% rename from src/logrotate.conf rename to nebd/src/logrotate.conf diff --git a/src/part1/BUILD b/nebd/src/part1/BUILD similarity index 100% rename from src/part1/BUILD rename to nebd/src/part1/BUILD diff --git a/src/part1/CMakeLists.txt b/nebd/src/part1/CMakeLists.txt similarity index 100% rename from src/part1/CMakeLists.txt rename to nebd/src/part1/CMakeLists.txt diff --git a/src/part1/async_request_closure.cpp b/nebd/src/part1/async_request_closure.cpp similarity index 100% rename from src/part1/async_request_closure.cpp rename to nebd/src/part1/async_request_closure.cpp diff --git a/src/part1/async_request_closure.h b/nebd/src/part1/async_request_closure.h similarity index 100% rename from src/part1/async_request_closure.h rename to nebd/src/part1/async_request_closure.h diff --git a/src/part1/heartbeat_manager.cpp b/nebd/src/part1/heartbeat_manager.cpp similarity index 100% rename from src/part1/heartbeat_manager.cpp rename to nebd/src/part1/heartbeat_manager.cpp diff --git a/src/part1/heartbeat_manager.h b/nebd/src/part1/heartbeat_manager.h similarity index 100% rename from src/part1/heartbeat_manager.h rename to nebd/src/part1/heartbeat_manager.h diff --git a/src/part1/libnebd.cpp b/nebd/src/part1/libnebd.cpp similarity index 100% rename from src/part1/libnebd.cpp rename to nebd/src/part1/libnebd.cpp diff --git a/src/part1/libnebd.h b/nebd/src/part1/libnebd.h similarity index 100% rename from src/part1/libnebd.h rename to nebd/src/part1/libnebd.h diff --git a/src/part1/libnebd_file.cpp b/nebd/src/part1/libnebd_file.cpp similarity index 100% rename from src/part1/libnebd_file.cpp rename to nebd/src/part1/libnebd_file.cpp diff --git a/src/part1/libnebd_file.h b/nebd/src/part1/libnebd_file.h similarity index 100% rename from src/part1/libnebd_file.h rename to nebd/src/part1/libnebd_file.h diff --git a/src/part1/nebd_client.cpp b/nebd/src/part1/nebd_client.cpp similarity index 100% rename from src/part1/nebd_client.cpp rename to nebd/src/part1/nebd_client.cpp diff --git a/src/part1/nebd_client.h b/nebd/src/part1/nebd_client.h similarity index 100% rename from src/part1/nebd_client.h rename to nebd/src/part1/nebd_client.h diff --git a/src/part1/nebd_common.h b/nebd/src/part1/nebd_common.h similarity index 100% rename from src/part1/nebd_common.h rename to nebd/src/part1/nebd_common.h diff --git a/src/part1/nebd_metacache.cpp b/nebd/src/part1/nebd_metacache.cpp similarity index 100% rename from src/part1/nebd_metacache.cpp rename to nebd/src/part1/nebd_metacache.cpp diff --git a/src/part1/nebd_metacache.h b/nebd/src/part1/nebd_metacache.h similarity index 100% rename from src/part1/nebd_metacache.h rename to nebd/src/part1/nebd_metacache.h diff --git a/src/part2/BUILD b/nebd/src/part2/BUILD similarity index 100% rename from src/part2/BUILD rename to nebd/src/part2/BUILD diff --git a/src/part2/CMakeLists.txt b/nebd/src/part2/CMakeLists.txt similarity index 100% rename from src/part2/CMakeLists.txt rename to nebd/src/part2/CMakeLists.txt diff --git a/src/part2/define.h b/nebd/src/part2/define.h similarity index 100% rename from src/part2/define.h rename to nebd/src/part2/define.h diff --git a/src/part2/file_entity.cpp b/nebd/src/part2/file_entity.cpp similarity index 100% rename from src/part2/file_entity.cpp rename to nebd/src/part2/file_entity.cpp diff --git a/src/part2/file_entity.h b/nebd/src/part2/file_entity.h similarity index 100% rename from src/part2/file_entity.h rename to nebd/src/part2/file_entity.h diff --git a/src/part2/file_manager.cpp b/nebd/src/part2/file_manager.cpp similarity index 100% rename from src/part2/file_manager.cpp rename to nebd/src/part2/file_manager.cpp diff --git a/src/part2/file_manager.h b/nebd/src/part2/file_manager.h similarity index 100% rename from src/part2/file_manager.h rename to nebd/src/part2/file_manager.h diff --git a/src/part2/file_service.cpp b/nebd/src/part2/file_service.cpp similarity index 100% rename from src/part2/file_service.cpp rename to nebd/src/part2/file_service.cpp diff --git a/src/part2/file_service.h b/nebd/src/part2/file_service.h similarity index 100% rename from src/part2/file_service.h rename to nebd/src/part2/file_service.h diff --git a/src/part2/heartbeat_manager.cpp b/nebd/src/part2/heartbeat_manager.cpp similarity index 100% rename from src/part2/heartbeat_manager.cpp rename to nebd/src/part2/heartbeat_manager.cpp diff --git a/src/part2/heartbeat_manager.h b/nebd/src/part2/heartbeat_manager.h similarity index 100% rename from src/part2/heartbeat_manager.h rename to nebd/src/part2/heartbeat_manager.h diff --git a/src/part2/heartbeat_service.cpp b/nebd/src/part2/heartbeat_service.cpp similarity index 100% rename from src/part2/heartbeat_service.cpp rename to nebd/src/part2/heartbeat_service.cpp diff --git a/src/part2/heartbeat_service.h b/nebd/src/part2/heartbeat_service.h similarity index 100% rename from src/part2/heartbeat_service.h rename to nebd/src/part2/heartbeat_service.h diff --git a/src/part2/main.cpp b/nebd/src/part2/main.cpp similarity index 100% rename from src/part2/main.cpp rename to nebd/src/part2/main.cpp diff --git a/src/part2/metafile_manager.cpp b/nebd/src/part2/metafile_manager.cpp similarity index 100% rename from src/part2/metafile_manager.cpp rename to nebd/src/part2/metafile_manager.cpp diff --git a/src/part2/metafile_manager.h b/nebd/src/part2/metafile_manager.h similarity index 100% rename from src/part2/metafile_manager.h rename to nebd/src/part2/metafile_manager.h diff --git a/src/part2/nebd_server.cpp b/nebd/src/part2/nebd_server.cpp similarity index 100% rename from src/part2/nebd_server.cpp rename to nebd/src/part2/nebd_server.cpp diff --git a/src/part2/nebd_server.h b/nebd/src/part2/nebd_server.h similarity index 100% rename from src/part2/nebd_server.h rename to nebd/src/part2/nebd_server.h diff --git a/src/part2/request_executor.cpp b/nebd/src/part2/request_executor.cpp similarity index 100% rename from src/part2/request_executor.cpp rename to nebd/src/part2/request_executor.cpp diff --git a/src/part2/request_executor.h b/nebd/src/part2/request_executor.h similarity index 100% rename from src/part2/request_executor.h rename to nebd/src/part2/request_executor.h diff --git a/src/part2/request_executor_ceph.cpp b/nebd/src/part2/request_executor_ceph.cpp similarity index 100% rename from src/part2/request_executor_ceph.cpp rename to nebd/src/part2/request_executor_ceph.cpp diff --git a/src/part2/request_executor_ceph.h b/nebd/src/part2/request_executor_ceph.h similarity index 100% rename from src/part2/request_executor_ceph.h rename to nebd/src/part2/request_executor_ceph.h diff --git a/src/part2/request_executor_curve.cpp b/nebd/src/part2/request_executor_curve.cpp similarity index 100% rename from src/part2/request_executor_curve.cpp rename to nebd/src/part2/request_executor_curve.cpp diff --git a/src/part2/request_executor_curve.h b/nebd/src/part2/request_executor_curve.h similarity index 100% rename from src/part2/request_executor_curve.h rename to nebd/src/part2/request_executor_curve.h diff --git a/src/part2/util.cpp b/nebd/src/part2/util.cpp similarity index 100% rename from src/part2/util.cpp rename to nebd/src/part2/util.cpp diff --git a/src/part2/util.h b/nebd/src/part2/util.h similarity index 100% rename from src/part2/util.h rename to nebd/src/part2/util.h diff --git a/tests/CMakeLists.txt b/nebd/tests/CMakeLists.txt similarity index 100% rename from tests/CMakeLists.txt rename to nebd/tests/CMakeLists.txt diff --git a/tests/common/BUILD b/nebd/tests/common/BUILD similarity index 100% rename from tests/common/BUILD rename to nebd/tests/common/BUILD diff --git a/tests/common/CMakeLists.txt b/nebd/tests/common/CMakeLists.txt similarity index 100% rename from tests/common/CMakeLists.txt rename to nebd/tests/common/CMakeLists.txt diff --git a/tests/common/configuration_test.cpp b/nebd/tests/common/configuration_test.cpp similarity index 100% rename from tests/common/configuration_test.cpp rename to nebd/tests/common/configuration_test.cpp diff --git a/tests/common/crc32_test.cpp b/nebd/tests/common/crc32_test.cpp similarity index 100% rename from tests/common/crc32_test.cpp rename to nebd/tests/common/crc32_test.cpp diff --git a/tests/common/interruptible_sleeper_test.cpp b/nebd/tests/common/interruptible_sleeper_test.cpp similarity index 100% rename from tests/common/interruptible_sleeper_test.cpp rename to nebd/tests/common/interruptible_sleeper_test.cpp diff --git a/tests/common/posix_wrapper_test.cpp b/nebd/tests/common/posix_wrapper_test.cpp similarity index 100% rename from tests/common/posix_wrapper_test.cpp rename to nebd/tests/common/posix_wrapper_test.cpp diff --git a/tests/common/rw_lock_test.cpp b/nebd/tests/common/rw_lock_test.cpp similarity index 100% rename from tests/common/rw_lock_test.cpp rename to nebd/tests/common/rw_lock_test.cpp diff --git a/tests/common/test_name_lock.cpp b/nebd/tests/common/test_name_lock.cpp similarity index 100% rename from tests/common/test_name_lock.cpp rename to nebd/tests/common/test_name_lock.cpp diff --git a/tests/part1/BUILD b/nebd/tests/part1/BUILD similarity index 100% rename from tests/part1/BUILD rename to nebd/tests/part1/BUILD diff --git a/tests/part1/CMakeLists.txt b/nebd/tests/part1/CMakeLists.txt similarity index 100% rename from tests/part1/CMakeLists.txt rename to nebd/tests/part1/CMakeLists.txt diff --git a/tests/part1/fake_file_service.cpp b/nebd/tests/part1/fake_file_service.cpp similarity index 100% rename from tests/part1/fake_file_service.cpp rename to nebd/tests/part1/fake_file_service.cpp diff --git a/tests/part1/fake_file_service.h b/nebd/tests/part1/fake_file_service.h similarity index 100% rename from tests/part1/fake_file_service.h rename to nebd/tests/part1/fake_file_service.h diff --git a/tests/part1/fake_heartbeat_service.h b/nebd/tests/part1/fake_heartbeat_service.h similarity index 100% rename from tests/part1/fake_heartbeat_service.h rename to nebd/tests/part1/fake_heartbeat_service.h diff --git a/tests/part1/heartbeat_manager_unittest.cpp b/nebd/tests/part1/heartbeat_manager_unittest.cpp similarity index 100% rename from tests/part1/heartbeat_manager_unittest.cpp rename to nebd/tests/part1/heartbeat_manager_unittest.cpp diff --git a/tests/part1/mock_file_service.h b/nebd/tests/part1/mock_file_service.h similarity index 100% rename from tests/part1/mock_file_service.h rename to nebd/tests/part1/mock_file_service.h diff --git a/tests/part1/mock_heartbeat_service.h b/nebd/tests/part1/mock_heartbeat_service.h similarity index 100% rename from tests/part1/mock_heartbeat_service.h rename to nebd/tests/part1/mock_heartbeat_service.h diff --git a/tests/part1/nebd_client_unittest.cpp b/nebd/tests/part1/nebd_client_unittest.cpp similarity index 100% rename from tests/part1/nebd_client_unittest.cpp rename to nebd/tests/part1/nebd_client_unittest.cpp diff --git a/tests/part1/nebd_lib_unittest.cpp b/nebd/tests/part1/nebd_lib_unittest.cpp similarity index 100% rename from tests/part1/nebd_lib_unittest.cpp rename to nebd/tests/part1/nebd_lib_unittest.cpp diff --git a/tests/part1/nebd_metacache_unittest.cpp b/nebd/tests/part1/nebd_metacache_unittest.cpp similarity index 100% rename from tests/part1/nebd_metacache_unittest.cpp rename to nebd/tests/part1/nebd_metacache_unittest.cpp diff --git a/tests/part2/BUILD b/nebd/tests/part2/BUILD similarity index 100% rename from tests/part2/BUILD rename to nebd/tests/part2/BUILD diff --git a/tests/part2/CMakeLists.txt b/nebd/tests/part2/CMakeLists.txt similarity index 100% rename from tests/part2/CMakeLists.txt rename to nebd/tests/part2/CMakeLists.txt diff --git a/tests/part2/file_manager_unittest.cpp b/nebd/tests/part2/file_manager_unittest.cpp similarity index 100% rename from tests/part2/file_manager_unittest.cpp rename to nebd/tests/part2/file_manager_unittest.cpp diff --git a/tests/part2/file_service_unittest.cpp b/nebd/tests/part2/file_service_unittest.cpp similarity index 100% rename from tests/part2/file_service_unittest.cpp rename to nebd/tests/part2/file_service_unittest.cpp diff --git a/tests/part2/heartbeat_manager_unittest.cpp b/nebd/tests/part2/heartbeat_manager_unittest.cpp similarity index 100% rename from tests/part2/heartbeat_manager_unittest.cpp rename to nebd/tests/part2/heartbeat_manager_unittest.cpp diff --git a/tests/part2/heartbeat_service_test.cpp b/nebd/tests/part2/heartbeat_service_test.cpp similarity index 100% rename from tests/part2/heartbeat_service_test.cpp rename to nebd/tests/part2/heartbeat_service_test.cpp diff --git a/tests/part2/metafile_manager_test.cpp b/nebd/tests/part2/metafile_manager_test.cpp similarity index 100% rename from tests/part2/metafile_manager_test.cpp rename to nebd/tests/part2/metafile_manager_test.cpp diff --git a/tests/part2/mock_curve_client.h b/nebd/tests/part2/mock_curve_client.h similarity index 100% rename from tests/part2/mock_curve_client.h rename to nebd/tests/part2/mock_curve_client.h diff --git a/tests/part2/mock_file_entity.h b/nebd/tests/part2/mock_file_entity.h similarity index 100% rename from tests/part2/mock_file_entity.h rename to nebd/tests/part2/mock_file_entity.h diff --git a/tests/part2/mock_file_manager.h b/nebd/tests/part2/mock_file_manager.h similarity index 100% rename from tests/part2/mock_file_manager.h rename to nebd/tests/part2/mock_file_manager.h diff --git a/tests/part2/mock_heartbeat_manager.h b/nebd/tests/part2/mock_heartbeat_manager.h similarity index 100% rename from tests/part2/mock_heartbeat_manager.h rename to nebd/tests/part2/mock_heartbeat_manager.h diff --git a/tests/part2/mock_metafile_manager.h b/nebd/tests/part2/mock_metafile_manager.h similarity index 100% rename from tests/part2/mock_metafile_manager.h rename to nebd/tests/part2/mock_metafile_manager.h diff --git a/tests/part2/mock_posix_wrapper.h b/nebd/tests/part2/mock_posix_wrapper.h similarity index 100% rename from tests/part2/mock_posix_wrapper.h rename to nebd/tests/part2/mock_posix_wrapper.h diff --git a/tests/part2/mock_request_executor.h b/nebd/tests/part2/mock_request_executor.h similarity index 100% rename from tests/part2/mock_request_executor.h rename to nebd/tests/part2/mock_request_executor.h diff --git a/tests/part2/test_nebd_server.cpp b/nebd/tests/part2/test_nebd_server.cpp similarity index 100% rename from tests/part2/test_nebd_server.cpp rename to nebd/tests/part2/test_nebd_server.cpp diff --git a/tests/part2/test_request_executor_curve.cpp b/nebd/tests/part2/test_request_executor_curve.cpp similarity index 100% rename from tests/part2/test_request_executor_curve.cpp rename to nebd/tests/part2/test_request_executor_curve.cpp diff --git a/tests/stringstatus_test.cpp b/nebd/tests/stringstatus_test.cpp similarity index 100% rename from tests/stringstatus_test.cpp rename to nebd/tests/stringstatus_test.cpp diff --git a/tests/utils/BUILD b/nebd/tests/utils/BUILD similarity index 100% rename from tests/utils/BUILD rename to nebd/tests/utils/BUILD diff --git a/tests/utils/config_generator.h b/nebd/tests/utils/config_generator.h similarity index 100% rename from tests/utils/config_generator.h rename to nebd/tests/utils/config_generator.h From 5fbc69e487efc755cac0321ae518be00d1807e67 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Thu, 2 Jul 2020 15:02:10 +0800 Subject: [PATCH 78/79] split nbd and separate nebd/nbd debian package --- curvefs_python/BUILD_bak | 3 -- curvefs_python/configure.sh | 12 ++++-- mk-deb.sh | 39 ++++++++++++++----- {src/tools/nbd => nbd}/README.md | 0 nbd/nbd-package/DEBIAN/control | 10 +++++ {src/tools/nbd => nbd/src}/BUILD | 34 +--------------- {src/tools/nbd => nbd/src}/ImageInstance.cpp | 2 +- {src/tools/nbd => nbd/src}/ImageInstance.h | 2 +- {src/tools/nbd => nbd/src}/NBDController.cpp | 2 +- {src/tools/nbd => nbd/src}/NBDController.h | 6 +-- {src/tools/nbd => nbd/src}/NBDServer.cpp | 4 +- {src/tools/nbd => nbd/src}/NBDServer.h | 8 ++-- {src/tools/nbd => nbd/src}/NBDTool.cpp | 6 +-- {src/tools/nbd => nbd/src}/NBDTool.h | 10 ++--- .../tools/nbd => nbd/src}/NBDWatchContext.cpp | 2 +- {src/tools/nbd => nbd/src}/NBDWatchContext.h | 6 +-- {src/tools/nbd => nbd/src}/SafeIO.cpp | 4 +- {src/tools/nbd => nbd/src}/SafeIO.h | 0 {src/tools/nbd => nbd/src}/argparse.cpp | 2 +- {src/tools/nbd => nbd/src}/argparse.h | 0 {src/tools/nbd => nbd/src}/define.h | 0 .../nbd => nbd/src}/interruptible_sleeper.h | 0 {src/tools/nbd => nbd/src}/libnebd.h | 0 {src/tools/nbd => nbd/src}/main.cpp | 16 ++++---- {src/tools/nbd => nbd/src}/nbd-netlink.h | 0 {src/tools/nbd => nbd/src}/texttable.cpp | 2 +- {src/tools/nbd => nbd/src}/texttable.h | 0 {src/tools/nbd => nbd/src}/util.cpp | 6 +-- {src/tools/nbd => nbd/src}/util.h | 2 +- {test/tools/nbd => nbd/test}/BUILD | 9 ++--- {test/tools/nbd => nbd/test}/fake_safe_io.h | 2 +- {test/tools/nbd => nbd/test}/main.cpp | 0 .../nbd => nbd/test}/mock_image_instance.h | 2 +- .../nbd => nbd/test}/mock_nbd_controller.h | 2 +- {test/tools/nbd => nbd/test}/mock_safe_io.h | 2 +- .../nbd => nbd/test}/nbd_server_test.cpp | 8 ++-- .../tools/nbd => nbd/test}/nbd_tool_test.cpp | 6 +-- .../test}/nbd_watch_context_test.cpp | 6 +-- 38 files changed, 107 insertions(+), 108 deletions(-) mode change 100644 => 100755 mk-deb.sh rename {src/tools/nbd => nbd}/README.md (100%) create mode 100644 nbd/nbd-package/DEBIAN/control rename {src/tools/nbd => nbd/src}/BUILD (57%) rename {src/tools/nbd => nbd/src}/ImageInstance.cpp (96%) rename {src/tools/nbd => nbd/src}/ImageInstance.h (97%) rename {src/tools/nbd => nbd/src}/NBDController.cpp (99%) rename {src/tools/nbd => nbd/src}/NBDController.h (97%) rename {src/tools/nbd => nbd/src}/NBDServer.cpp (99%) rename {src/tools/nbd => nbd/src}/NBDServer.h (95%) rename {src/tools/nbd => nbd/src}/NBDTool.cpp (97%) rename {src/tools/nbd => nbd/src}/NBDTool.h (92%) rename {src/tools/nbd => nbd/src}/NBDWatchContext.cpp (95%) rename {src/tools/nbd => nbd/src}/NBDWatchContext.h (91%) rename {src/tools/nbd => nbd/src}/SafeIO.cpp (88%) rename {src/tools/nbd => nbd/src}/SafeIO.h (100%) rename {src/tools/nbd => nbd/src}/argparse.cpp (99%) rename {src/tools/nbd => nbd/src}/argparse.h (100%) rename {src/tools/nbd => nbd/src}/define.h (100%) rename {src/tools/nbd => nbd/src}/interruptible_sleeper.h (100%) rename {src/tools/nbd => nbd/src}/libnebd.h (100%) rename {src/tools/nbd => nbd/src}/main.cpp (94%) rename {src/tools/nbd => nbd/src}/nbd-netlink.h (100%) rename {src/tools/nbd => nbd/src}/texttable.cpp (98%) rename {src/tools/nbd => nbd/src}/texttable.h (100%) rename {src/tools/nbd => nbd/src}/util.cpp (99%) rename {src/tools/nbd => nbd/src}/util.h (98%) rename {test/tools/nbd => nbd/test}/BUILD (82%) rename {test/tools/nbd => nbd/test}/fake_safe_io.h (97%) rename {test/tools/nbd => nbd/test}/main.cpp (100%) rename {test/tools/nbd => nbd/test}/mock_image_instance.h (95%) rename {test/tools/nbd => nbd/test}/mock_nbd_controller.h (94%) rename {test/tools/nbd => nbd/test}/mock_safe_io.h (94%) rename {test/tools/nbd => nbd/test}/nbd_server_test.cpp (97%) rename {test/tools/nbd => nbd/test}/nbd_tool_test.cpp (97%) rename {test/tools/nbd => nbd/test}/nbd_watch_context_test.cpp (93%) diff --git a/curvefs_python/BUILD_bak b/curvefs_python/BUILD_bak index ede877fa6e..5581451f4d 100644 --- a/curvefs_python/BUILD_bak +++ b/curvefs_python/BUILD_bak @@ -41,7 +41,6 @@ cc_library( ], copts = COPTS, linkopts = [ - "-lbraft", "-lcbd", "-lcurve_client", "-lbrpc", @@ -54,7 +53,6 @@ cc_library( "-lcc_brpc_internal_proto", "-lcc_brpc_idl_options_proto", "-lleveldb", - "-lcc_braft_internal_proto", "-lcurve_common", "-lcurve_auth", "-lcurve_concurrent", @@ -67,7 +65,6 @@ cc_library( "-lprotobuf_lite", "-lssl", "-lrt", - "-lssl", "-lcrypto", "-ldl", "-lz", diff --git a/curvefs_python/configure.sh b/curvefs_python/configure.sh index 5519ed5b65..618d44c6cb 100755 --- a/curvefs_python/configure.sh +++ b/curvefs_python/configure.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash echo "********************************************************************" echo "***********Prepare for curvefs python API build env.****************" echo "********************************************************************" @@ -15,11 +15,17 @@ cp BUILD_bak BUILD -f echo "copy libs to tmplib directory" +libs=`cat BUILD | tr -d "[:blank:]" | grep "^\"-l" | sed 's/[",]//g' | awk '{ print substr($0, 3) }'` mkdir tmplib for i in `find $curve_path/bazel-bin/|grep -w so|grep -v solib|grep -v params` do - echo $i - cp -f $i ./tmplib/ + basename=$(basename $i) + linkname=`echo $basename | awk -F'.' '{ print $1 }' | awk '{ print substr($0, 4) }'` + if [[ $libs =~ $linkname ]] + then + echo $i + cp -f $i ./tmplib/ + fi done echo "Prepare env done, you can build curvefs now." diff --git a/mk-deb.sh b/mk-deb.sh old mode 100644 new mode 100755 index cbcd13694a..8d3b3b8d2c --- a/mk-deb.sh +++ b/mk-deb.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash dir=`pwd` #step1 清除生成的目录和文件 bazel clean @@ -43,13 +43,14 @@ if [ "$1" = "debug" ] then bazel build ... --copt -DHAVE_ZLIB=1 --compilation_mode=dbg -s --define=with_glog=true \ --define=libunwind=true --copt -DGFLAGS_NS=google --copt \ --Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${tag_version} +-Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${tag_version} \ +--linkopt -L${dir}/thirdparties/etcdclient if [ $? -ne 0 ] then echo "build phase1 failed" exit fi -sh ./curvefs_python/configure.sh +bash ./curvefs_python/configure.sh if [ $? -ne 0 ] then echo "configure failed" @@ -68,13 +69,14 @@ fi else bazel build ... --copt -DHAVE_ZLIB=1 --copt -O2 -s --define=with_glog=true \ --define=libunwind=true --copt -DGFLAGS_NS=google --copt \ --Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${tag_version} +-Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${tag_version} \ +--linkopt -L${dir}/thirdparties/etcdclient if [ $? -ne 0 ] then echo "build phase1 failed" exit fi -sh ./curvefs_python/configure.sh +bash ./curvefs_python/configure.sh if [ $? -ne 0 ] then echo "configure failed" @@ -303,11 +305,7 @@ if [ $? -ne 0 ] then exit fi -cp ./bazel-bin/src/tools/nbd/curve-nbd build/curve-sdk/usr/bin/curve-nbd -if [ $? -ne 0 ] -then - exit -fi + #cp ./conf/client.conf build/curve-sdk/etc/curve/client.conf #if [ $? -ne 0 ] #then @@ -380,6 +378,23 @@ then exit fi +# step 3.1 prepare for nebd package +cp -r nebd/nebd-package build/ +mkdir -p build/nebd-package/usr/bin +mkdir -p build/nebd-package/usr/lib/nebd + +for i in `find bazel-bin/|grep -w so|grep -v solib|grep -v params|grep -v test|grep -v fake` +do + cp -f $i build/nebd-package/usr/lib/nebd +done + +cp bazel-bin/nebd/src/part2/nebd-server build/nebd-package/usr/bin + +# step 3.2 prepare for curve-nbd package +cp -r nbd/nbd-package build +mkdir -p build/nbd-package/usr/bin +cp bazel-bin/nbd/src/curve-nbd build/nbd-package/usr/bin + #step4 获取git提交版本信息，记录到debian包的配置文件 commit_id=`git show --abbrev-commit HEAD|head -n 1|awk '{print $2}'` if [ "$1" = "debug" ] @@ -396,6 +411,8 @@ echo ${version} >> build/curve-tools/DEBIAN/control echo ${version} >> build/curve-monitor/DEBIAN/control echo ${version} >> build/curve-snapshotcloneserver/DEBIAN/control echo ${version} >> build/curve-nginx/DEBIAN/control +echo ${version} >> build/nebd-package/DEBIAN/control +echo ${version} >> build/nbd-package/DEBIAN/control #step5 打包debian包 dpkg-deb -b build/curve-mds . @@ -405,6 +422,8 @@ dpkg-deb -b build/curve-tools . dpkg-deb -b build/curve-monitor . dpkg-deb -b build/curve-snapshotcloneserver . dpkg-deb -b build/curve-nginx . +dpkg-deb -b build/nebd-package . +dpkg-deb -b build/nbd-package . #aws-c-common(commit=0302570a3cbabd98293ee03971e0867f28355086) #aws-checksums(commit=78be31b81a2b0445597e60ecb2412bc44e762a99) #aws-c-event-stream(commit=ad9a8b2a42d6c6ef07ccf251b5038b89487eacb3) diff --git a/src/tools/nbd/README.md b/nbd/README.md similarity index 100% rename from src/tools/nbd/README.md rename to nbd/README.md diff --git a/nbd/nbd-package/DEBIAN/control b/nbd/nbd-package/DEBIAN/control new file mode 100644 index 0000000000..8bb9fc7917 --- /dev/null +++ b/nbd/nbd-package/DEBIAN/control @@ -0,0 +1,10 @@ +Package: curve-nbd +Section: +Priority: optional +Depends: libunwind8 +Suggests: +Architecture: amd64 +Installed-Size: +Maintainer: curve-dev +Provides: +Description: NBD-based client for curve diff --git a/src/tools/nbd/BUILD b/nbd/src/BUILD similarity index 57% rename from src/tools/nbd/BUILD rename to nbd/src/BUILD index 3637d7e78a..9993ac61f8 100644 --- a/src/tools/nbd/BUILD +++ b/nbd/src/BUILD @@ -28,35 +28,10 @@ cc_library( deps = [ "//external:gflags", "//external:glog", + "//nebd/src/part1:nebdclient", ], copts = COPTS, linkopts = [ - "-L/usr/lib/nebd", - "-lnebdclient", - "-lclient_proto", - "-lnebd_common", - "-lbrpc", - "-lcc_brpc_idl_options_proto", - "-lcc_brpc_internal_proto", - "-lbthread", - "-lbvar", - "-ljson2pb", - "-lmcpack2pb", - "-lbutil", - "-lprotoc_lib", - "-lleveldb", - "-lprotobuf", - "-lprotobuf_lite", - "-lssl", - "-lrt", - "-lssl", - "-lcrypto", - "-ldl", - "-lz", - "-lpthread", - "-lunwind", - "-lstdc++", - "-lm", "-lnl-3", "-lnl-genl-3", ], @@ -70,12 +45,7 @@ cc_binary( ] ), deps = [ - "//src/tools/nbd:curvenbd" + "//nbd/src:curvenbd" ], copts = COPTS, - linkopts = [ - "-Wl,-rpath=/usr/lib/nebd" - ], ) - - diff --git a/src/tools/nbd/ImageInstance.cpp b/nbd/src/ImageInstance.cpp similarity index 96% rename from src/tools/nbd/ImageInstance.cpp rename to nbd/src/ImageInstance.cpp index 9a3d354de2..e209e0ccff 100644 --- a/src/tools/nbd/ImageInstance.cpp +++ b/nbd/src/ImageInstance.cpp @@ -5,7 +5,7 @@ * Copyright (c) 2020 Netease */ -#include "src/tools/nbd/ImageInstance.h" +#include "nbd/src/ImageInstance.h" #include namespace curve { diff --git a/src/tools/nbd/ImageInstance.h b/nbd/src/ImageInstance.h similarity index 97% rename from src/tools/nbd/ImageInstance.h rename to nbd/src/ImageInstance.h index 2c93892aae..55456f987f 100644 --- a/src/tools/nbd/ImageInstance.h +++ b/nbd/src/ImageInstance.h @@ -10,7 +10,7 @@ #include #include -#include "src/tools/nbd/libnebd.h" +#include "nbd/src/libnebd.h" namespace curve { namespace nbd { diff --git a/src/tools/nbd/NBDController.cpp b/nbd/src/NBDController.cpp similarity index 99% rename from src/tools/nbd/NBDController.cpp rename to nbd/src/NBDController.cpp index 2cd30b70b3..0594dc805a 100644 --- a/src/tools/nbd/NBDController.cpp +++ b/nbd/src/NBDController.cpp @@ -5,7 +5,7 @@ * Copyright (c) 2020 netease */ -#include "src/tools/nbd/NBDController.h" +#include "nbd/src/NBDController.h" namespace curve { namespace nbd { diff --git a/src/tools/nbd/NBDController.h b/nbd/src/NBDController.h similarity index 97% rename from src/tools/nbd/NBDController.h rename to nbd/src/NBDController.h index 57e7368986..f4c4fb7318 100644 --- a/src/tools/nbd/NBDController.h +++ b/nbd/src/NBDController.h @@ -19,9 +19,9 @@ #include #include -#include "src/tools/nbd/nbd-netlink.h" -#include "src/tools/nbd/define.h" -#include "src/tools/nbd/util.h" +#include "nbd/src/nbd-netlink.h" +#include "nbd/src/define.h" +#include "nbd/src/util.h" namespace curve { namespace nbd { diff --git a/src/tools/nbd/NBDServer.cpp b/nbd/src/NBDServer.cpp similarity index 99% rename from src/tools/nbd/NBDServer.cpp rename to nbd/src/NBDServer.cpp index ff537d8c29..271309acc4 100644 --- a/src/tools/nbd/NBDServer.cpp +++ b/nbd/src/NBDServer.cpp @@ -5,14 +5,14 @@ * Copyright (c) 2020 Netease */ -#include "src/tools/nbd/NBDServer.h" +#include "nbd/src/NBDServer.h" #include #include #include #include -#include "src/tools/nbd/util.h" +#include "nbd/src/util.h" namespace curve { namespace nbd { diff --git a/src/tools/nbd/NBDServer.h b/nbd/src/NBDServer.h similarity index 95% rename from src/tools/nbd/NBDServer.h rename to nbd/src/NBDServer.h index db8bf65713..1b3a05e1bf 100644 --- a/src/tools/nbd/NBDServer.h +++ b/nbd/src/NBDServer.h @@ -17,10 +17,10 @@ #include #include // NOLINT -#include "src/tools/nbd/ImageInstance.h" -#include "src/tools/nbd/NBDController.h" -#include "src/tools/nbd/SafeIO.h" -#include "src/tools/nbd/libnebd.h" +#include "nbd/src/ImageInstance.h" +#include "nbd/src/NBDController.h" +#include "nbd/src/SafeIO.h" +#include "nbd/src/libnebd.h" namespace curve { namespace nbd { diff --git a/src/tools/nbd/NBDTool.cpp b/nbd/src/NBDTool.cpp similarity index 97% rename from src/tools/nbd/NBDTool.cpp rename to nbd/src/NBDTool.cpp index 5560fc648f..d576e0d2db 100644 --- a/src/tools/nbd/NBDTool.cpp +++ b/nbd/src/NBDTool.cpp @@ -9,9 +9,9 @@ #include #include #include -#include "src/tools/nbd/NBDTool.h" -#include "src/tools/nbd/argparse.h" -#include "src/tools/nbd/texttable.h" +#include "nbd/src/NBDTool.h" +#include "nbd/src/argparse.h" +#include "nbd/src/texttable.h" namespace curve { namespace nbd { diff --git a/src/tools/nbd/NBDTool.h b/nbd/src/NBDTool.h similarity index 92% rename from src/tools/nbd/NBDTool.h rename to nbd/src/NBDTool.h index e65857357e..9fec3a7eb1 100644 --- a/src/tools/nbd/NBDTool.h +++ b/nbd/src/NBDTool.h @@ -11,11 +11,11 @@ #include #include #include -#include "src/tools/nbd/define.h" -#include "src/tools/nbd/NBDController.h" -#include "src/tools/nbd/NBDServer.h" -#include "src/tools/nbd/ImageInstance.h" -#include "src/tools/nbd/NBDWatchContext.h" +#include "nbd/src/define.h" +#include "nbd/src/NBDController.h" +#include "nbd/src/NBDServer.h" +#include "nbd/src/ImageInstance.h" +#include "nbd/src/NBDWatchContext.h" namespace curve { namespace nbd { diff --git a/src/tools/nbd/NBDWatchContext.cpp b/nbd/src/NBDWatchContext.cpp similarity index 95% rename from src/tools/nbd/NBDWatchContext.cpp rename to nbd/src/NBDWatchContext.cpp index 261cdae324..1ea137e546 100644 --- a/src/tools/nbd/NBDWatchContext.cpp +++ b/nbd/src/NBDWatchContext.cpp @@ -5,7 +5,7 @@ * Copyright (c) 2020 NetEase */ -#include "src/tools/nbd/NBDWatchContext.h" +#include "nbd/src/NBDWatchContext.h" #include namespace curve { diff --git a/src/tools/nbd/NBDWatchContext.h b/nbd/src/NBDWatchContext.h similarity index 91% rename from src/tools/nbd/NBDWatchContext.h rename to nbd/src/NBDWatchContext.h index b76db0dc9d..b59b9b0bb8 100644 --- a/src/tools/nbd/NBDWatchContext.h +++ b/nbd/src/NBDWatchContext.h @@ -11,9 +11,9 @@ #include #include // NOLINT #include -#include "src/tools/nbd/ImageInstance.h" -#include "src/tools/nbd/NBDController.h" -#include "src/tools/nbd/interruptible_sleeper.h" +#include "nbd/src/ImageInstance.h" +#include "nbd/src/NBDController.h" +#include "nbd/src/interruptible_sleeper.h" namespace curve { namespace nbd { diff --git a/src/tools/nbd/SafeIO.cpp b/nbd/src/SafeIO.cpp similarity index 88% rename from src/tools/nbd/SafeIO.cpp rename to nbd/src/SafeIO.cpp index 9c337cd544..8a403646eb 100644 --- a/src/tools/nbd/SafeIO.cpp +++ b/nbd/src/SafeIO.cpp @@ -5,10 +5,10 @@ * Copyright (c) 2020 Netease */ -#include "src/tools/nbd/SafeIO.h" +#include "nbd/src/SafeIO.h" #include -#include "src/tools/nbd/util.h" +#include "nbd/src/util.h" namespace curve { namespace nbd { diff --git a/src/tools/nbd/SafeIO.h b/nbd/src/SafeIO.h similarity index 100% rename from src/tools/nbd/SafeIO.h rename to nbd/src/SafeIO.h diff --git a/src/tools/nbd/argparse.cpp b/nbd/src/argparse.cpp similarity index 99% rename from src/tools/nbd/argparse.cpp rename to nbd/src/argparse.cpp index b6eef7ce30..187c27d5d0 100644 --- a/src/tools/nbd/argparse.cpp +++ b/nbd/src/argparse.cpp @@ -13,7 +13,7 @@ #include #include -#include "src/tools/nbd/argparse.h" +#include "nbd/src/argparse.h" namespace curve { namespace nbd { diff --git a/src/tools/nbd/argparse.h b/nbd/src/argparse.h similarity index 100% rename from src/tools/nbd/argparse.h rename to nbd/src/argparse.h diff --git a/src/tools/nbd/define.h b/nbd/src/define.h similarity index 100% rename from src/tools/nbd/define.h rename to nbd/src/define.h diff --git a/src/tools/nbd/interruptible_sleeper.h b/nbd/src/interruptible_sleeper.h similarity index 100% rename from src/tools/nbd/interruptible_sleeper.h rename to nbd/src/interruptible_sleeper.h diff --git a/src/tools/nbd/libnebd.h b/nbd/src/libnebd.h similarity index 100% rename from src/tools/nbd/libnebd.h rename to nbd/src/libnebd.h diff --git a/src/tools/nbd/main.cpp b/nbd/src/main.cpp similarity index 94% rename from src/tools/nbd/main.cpp rename to nbd/src/main.cpp index 6d38ff57a0..961bd98157 100644 --- a/src/tools/nbd/main.cpp +++ b/nbd/src/main.cpp @@ -11,14 +11,14 @@ #include #include -#include "src/tools/nbd/ImageInstance.h" -#include "src/tools/nbd/NBDController.h" -#include "src/tools/nbd/NBDServer.h" -#include "src/tools/nbd/NBDTool.h" -#include "src/tools/nbd/NBDWatchContext.h" -#include "src/tools/nbd/argparse.h" -#include "src/tools/nbd/define.h" -#include "src/tools/nbd/util.h" +#include "nbd/src/ImageInstance.h" +#include "nbd/src/NBDController.h" +#include "nbd/src/NBDServer.h" +#include "nbd/src/NBDTool.h" +#include "nbd/src/NBDWatchContext.h" +#include "nbd/src/argparse.h" +#include "nbd/src/define.h" +#include "nbd/src/util.h" namespace curve { namespace nbd { diff --git a/src/tools/nbd/nbd-netlink.h b/nbd/src/nbd-netlink.h similarity index 100% rename from src/tools/nbd/nbd-netlink.h rename to nbd/src/nbd-netlink.h diff --git a/src/tools/nbd/texttable.cpp b/nbd/src/texttable.cpp similarity index 98% rename from src/tools/nbd/texttable.cpp rename to nbd/src/texttable.cpp index 8791377630..2df4769380 100644 --- a/src/tools/nbd/texttable.cpp +++ b/nbd/src/texttable.cpp @@ -11,7 +11,7 @@ * Foundation. See file COPYING. * */ -#include "src/tools/nbd/texttable.h" +#include "nbd/src/texttable.h" namespace curve { namespace nbd { diff --git a/src/tools/nbd/texttable.h b/nbd/src/texttable.h similarity index 100% rename from src/tools/nbd/texttable.h rename to nbd/src/texttable.h diff --git a/src/tools/nbd/util.cpp b/nbd/src/util.cpp similarity index 99% rename from src/tools/nbd/util.cpp rename to nbd/src/util.cpp index 4082dad080..c3e6034cba 100644 --- a/src/tools/nbd/util.cpp +++ b/nbd/src/util.cpp @@ -11,9 +11,9 @@ #include #include #include -#include "src/tools/nbd/define.h" -#include "src/tools/nbd/util.h" -#include "src/tools/nbd/argparse.h" +#include "nbd/src/define.h" +#include "nbd/src/util.h" +#include "nbd/src/argparse.h" namespace curve { namespace nbd { diff --git a/src/tools/nbd/util.h b/nbd/src/util.h similarity index 98% rename from src/tools/nbd/util.h rename to nbd/src/util.h index 822a55e0e7..0e1f7be8e5 100644 --- a/src/tools/nbd/util.h +++ b/nbd/src/util.h @@ -10,7 +10,7 @@ #include #include -#include "src/tools/nbd/define.h" +#include "nbd/src/define.h" namespace curve { namespace nbd { diff --git a/test/tools/nbd/BUILD b/nbd/test/BUILD similarity index 82% rename from test/tools/nbd/BUILD rename to nbd/test/BUILD index 145933338f..81045ca74b 100644 --- a/test/tools/nbd/BUILD +++ b/nbd/test/BUILD @@ -27,7 +27,7 @@ cc_library( hdrs = glob(["*.h"]), deps = [ "//external:gtest", - "//src/tools/nbd:curvenbd", + "//nbd/src:curvenbd", ], copts = COPTS, visibility = ["//visibility:public"], @@ -39,11 +39,8 @@ cc_binary( "*.cpp", ]), deps = [ - "//src/tools/nbd:curvenbd", - "//test/tools/nbd:mock_lib" + "//nbd/src:curvenbd", + "//nbd/test:mock_lib" ], copts = COPTS, - linkopts = [ - "-Wl,-rpath=/usr/lib/nebd" - ], ) diff --git a/test/tools/nbd/fake_safe_io.h b/nbd/test/fake_safe_io.h similarity index 97% rename from test/tools/nbd/fake_safe_io.h rename to nbd/test/fake_safe_io.h index 05062a6afd..fc27759ea9 100644 --- a/test/tools/nbd/fake_safe_io.h +++ b/nbd/test/fake_safe_io.h @@ -9,7 +9,7 @@ #define TEST_TOOLS_NBD_FAKE_SAFE_IO_H_ #include -#include "src/tools/nbd/SafeIO.h" +#include "nbd/src/SafeIO.h" namespace curve { namespace nbd { diff --git a/test/tools/nbd/main.cpp b/nbd/test/main.cpp similarity index 100% rename from test/tools/nbd/main.cpp rename to nbd/test/main.cpp diff --git a/test/tools/nbd/mock_image_instance.h b/nbd/test/mock_image_instance.h similarity index 95% rename from test/tools/nbd/mock_image_instance.h rename to nbd/test/mock_image_instance.h index ba4081028f..6f957d20c4 100644 --- a/test/tools/nbd/mock_image_instance.h +++ b/nbd/test/mock_image_instance.h @@ -9,7 +9,7 @@ #define TEST_TOOLS_NBD_MOCK_IMAGE_INSTANCE_H_ #include -#include "src/tools/nbd/ImageInstance.h" +#include "nbd/src/ImageInstance.h" namespace curve { namespace nbd { diff --git a/test/tools/nbd/mock_nbd_controller.h b/nbd/test/mock_nbd_controller.h similarity index 94% rename from test/tools/nbd/mock_nbd_controller.h rename to nbd/test/mock_nbd_controller.h index 4275b9f8b3..9f1e95b1d2 100644 --- a/test/tools/nbd/mock_nbd_controller.h +++ b/nbd/test/mock_nbd_controller.h @@ -10,7 +10,7 @@ #include #include -#include "src/tools/nbd/NBDController.h" +#include "nbd/src/NBDController.h" namespace curve { namespace nbd { diff --git a/test/tools/nbd/mock_safe_io.h b/nbd/test/mock_safe_io.h similarity index 94% rename from test/tools/nbd/mock_safe_io.h rename to nbd/test/mock_safe_io.h index d13e7643c0..972c29d0ca 100644 --- a/test/tools/nbd/mock_safe_io.h +++ b/nbd/test/mock_safe_io.h @@ -10,7 +10,7 @@ #include -#include "src/tools/nbd/SafeIO.h" +#include "nbd/src/SafeIO.h" namespace curve { namespace nbd { diff --git a/test/tools/nbd/nbd_server_test.cpp b/nbd/test/nbd_server_test.cpp similarity index 97% rename from test/tools/nbd/nbd_server_test.cpp rename to nbd/test/nbd_server_test.cpp index 3dec6c217c..96bcd002a4 100644 --- a/test/tools/nbd/nbd_server_test.cpp +++ b/nbd/test/nbd_server_test.cpp @@ -13,10 +13,10 @@ #include #include #include -#include "src/tools/nbd/NBDServer.h" -#include "test/tools/nbd/fake_safe_io.h" -#include "test/tools/nbd/mock_image_instance.h" -#include "test/tools/nbd/mock_safe_io.h" +#include "nbd/src/NBDServer.h" +#include "nbd/test/fake_safe_io.h" +#include "nbd/test/mock_image_instance.h" +#include "nbd/test/mock_safe_io.h" namespace curve { namespace nbd { diff --git a/test/tools/nbd/nbd_tool_test.cpp b/nbd/test/nbd_tool_test.cpp similarity index 97% rename from test/tools/nbd/nbd_tool_test.cpp rename to nbd/test/nbd_tool_test.cpp index 9164c104d1..8b31f71404 100644 --- a/test/tools/nbd/nbd_tool_test.cpp +++ b/nbd/test/nbd_tool_test.cpp @@ -12,9 +12,9 @@ #include #include #include -#include "test/tools/nbd/mock_image_instance.h" -#include "src/tools/nbd/NBDTool.h" -#include "src/tools/nbd/util.h" +#include "nbd/test/mock_image_instance.h" +#include "nbd/src/NBDTool.h" +#include "nbd/src/util.h" namespace curve { namespace nbd { diff --git a/test/tools/nbd/nbd_watch_context_test.cpp b/nbd/test/nbd_watch_context_test.cpp similarity index 93% rename from test/tools/nbd/nbd_watch_context_test.cpp rename to nbd/test/nbd_watch_context_test.cpp index 96aac8cb6a..765eb6cd60 100644 --- a/test/tools/nbd/nbd_watch_context_test.cpp +++ b/nbd/test/nbd_watch_context_test.cpp @@ -5,9 +5,9 @@ * Copyright (c) 2020 Netease */ -#include "src/tools/nbd/NBDWatchContext.h" -#include "test/tools/nbd/mock_image_instance.h" -#include "test/tools/nbd/mock_nbd_controller.h" +#include "nbd/src/NBDWatchContext.h" +#include "nbd/test/mock_image_instance.h" +#include "nbd/test/mock_nbd_controller.h" namespace curve { namespace nbd { From a1e7624bfa6f0ba576509d4706fb4d9f27a8a7fd Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Mon, 6 Jul 2020 16:47:50 +0800 Subject: [PATCH 79/79] remove common code from nebd/nbd --- nbd/src/ImageInstance.h | 2 +- nbd/src/NBDServer.h | 2 +- nbd/src/NBDWatchContext.h | 4 +- nbd/src/interruptible_sleeper.h | 54 ---- nbd/src/libnebd.h | 70 ----- nebd/src/common/BUILD | 3 +- nebd/src/common/configuration.cpp | 233 -------------- nebd/src/common/configuration.h | 111 ------- nebd/src/common/crc32.h | 45 --- nebd/src/common/interrupt_sleep.h | 42 --- nebd/src/common/name_lock.h | 4 +- nebd/src/common/nebd_version.cpp | 38 --- nebd/src/common/nebd_version.h | 22 -- nebd/src/common/rw_lock.h | 137 -------- nebd/src/common/stringstatus.cpp | 51 --- nebd/src/common/stringstatus.h | 63 ---- nebd/src/common/timeutility.h | 53 ---- nebd/src/common/uncopyable.h | 27 -- nebd/src/logrotate.conf | 12 - nebd/src/part1/BUILD | 5 +- nebd/src/part1/heartbeat_manager.cpp | 4 +- nebd/src/part1/heartbeat_manager.h | 4 +- nebd/src/part1/libnebd_file.cpp | 2 +- nebd/src/part1/nebd_client.cpp | 4 +- nebd/src/part1/nebd_client.h | 4 +- nebd/src/part1/nebd_metacache.cpp | 11 +- nebd/src/part1/nebd_metacache.h | 4 +- nebd/src/part2/BUILD | 4 +- nebd/src/part2/define.h | 4 +- nebd/src/part2/file_entity.h | 13 +- nebd/src/part2/file_manager.h | 6 +- nebd/src/part2/heartbeat_manager.cpp | 5 +- nebd/src/part2/heartbeat_manager.h | 16 +- nebd/src/part2/heartbeat_service.cpp | 4 +- nebd/src/part2/metafile_manager.cpp | 4 +- nebd/src/part2/metafile_manager.h | 11 +- nebd/src/part2/nebd_server.cpp | 6 +- nebd/src/part2/nebd_server.h | 4 +- nebd/test/common/BUILD | 1 + nebd/test/common/configuration_test.cpp | 295 ------------------ nebd/test/common/crc32_test.cpp | 81 ----- .../common/interruptible_sleeper_test.cpp | 44 --- nebd/test/common/rw_lock_test.cpp | 157 ---------- nebd/test/part2/BUILD | 1 + nebd/test/part2/metafile_manager_test.cpp | 2 +- nebd/test/stringstatus_test.cpp | 32 -- nebd/test/utils/config_generator.h | 4 +- src/common/configuration.cpp | 13 + src/common/configuration.h | 4 +- test/common/configuration_test.cpp | 9 + 50 files changed, 102 insertions(+), 1629 deletions(-) delete mode 100644 nbd/src/interruptible_sleeper.h delete mode 100644 nbd/src/libnebd.h delete mode 100644 nebd/src/common/configuration.cpp delete mode 100644 nebd/src/common/configuration.h delete mode 100644 nebd/src/common/crc32.h delete mode 100644 nebd/src/common/interrupt_sleep.h delete mode 100644 nebd/src/common/nebd_version.cpp delete mode 100644 nebd/src/common/nebd_version.h delete mode 100644 nebd/src/common/rw_lock.h delete mode 100644 nebd/src/common/stringstatus.cpp delete mode 100644 nebd/src/common/stringstatus.h delete mode 100644 nebd/src/common/timeutility.h delete mode 100644 nebd/src/common/uncopyable.h delete mode 100644 nebd/src/logrotate.conf delete mode 100644 nebd/test/common/configuration_test.cpp delete mode 100644 nebd/test/common/crc32_test.cpp delete mode 100644 nebd/test/common/interruptible_sleeper_test.cpp delete mode 100644 nebd/test/common/rw_lock_test.cpp delete mode 100644 nebd/test/stringstatus_test.cpp diff --git a/nbd/src/ImageInstance.h b/nbd/src/ImageInstance.h index 55456f987f..f5515e7378 100644 --- a/nbd/src/ImageInstance.h +++ b/nbd/src/ImageInstance.h @@ -10,7 +10,7 @@ #include #include -#include "nbd/src/libnebd.h" +#include "nebd/src/part1/libnebd.h" namespace curve { namespace nbd { diff --git a/nbd/src/NBDServer.h b/nbd/src/NBDServer.h index 1b3a05e1bf..405fd27745 100644 --- a/nbd/src/NBDServer.h +++ b/nbd/src/NBDServer.h @@ -20,7 +20,7 @@ #include "nbd/src/ImageInstance.h" #include "nbd/src/NBDController.h" #include "nbd/src/SafeIO.h" -#include "nbd/src/libnebd.h" +#include "nebd/src/part1/libnebd.h" namespace curve { namespace nbd { diff --git a/nbd/src/NBDWatchContext.h b/nbd/src/NBDWatchContext.h index b59b9b0bb8..341eab2ff7 100644 --- a/nbd/src/NBDWatchContext.h +++ b/nbd/src/NBDWatchContext.h @@ -13,7 +13,7 @@ #include #include "nbd/src/ImageInstance.h" #include "nbd/src/NBDController.h" -#include "nbd/src/interruptible_sleeper.h" +#include "src/common/interruptible_sleeper.h" namespace curve { namespace nbd { @@ -62,7 +62,7 @@ class NBDWatchContext { // 任务线程 std::thread watchThread_; - InterruptibleSleeper sleeper_; + curve::common::InterruptibleSleeper sleeper_; }; } // namespace nbd diff --git a/nbd/src/interruptible_sleeper.h b/nbd/src/interruptible_sleeper.h deleted file mode 100644 index e544f6c7e5..0000000000 --- a/nbd/src/interruptible_sleeper.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Project: curve - * Created Date: 2019-11-25 - * Author: lixiaocui - * Copyright (c) 2018 netease - */ - -#ifndef SRC_TOOLS_NBD_INTERRUPTIBLE_SLEEPER_H_ -#define SRC_TOOLS_NBD_INTERRUPTIBLE_SLEEPER_H_ - -#include // NOLINT -#include // NOLINT -#include -#include // NOLINT - -namespace curve { -namespace nbd { -/** - * InterruptibleSleeper 实现可 interruptible 的 sleep 功能. - * 正常情况下 wait_for 超时, 接收到退出信号之后, 程序会立即被唤醒, - * 退出 while 循环, 并执行 cleanup 代码. - */ -class InterruptibleSleeper { - public: - /** - * @brief wait_for 等待指定时间，如果接受到退出信号立刻返回 - * @param[in] time 指定wait时长 - * @return false-收到退出信号 true-超时后退出 - */ - template - bool wait_for(std::chrono::duration const& time) { - std::unique_lock lock(m); - return !cv.wait_for(lock, time, [&] { return terminate; }); - } - - /** - * @brief interrupt 给当前wait发送退出信号 - */ - void interrupt() { - std::lock_guard lock(m); - terminate = true; - cv.notify_all(); - } - - private: - std::condition_variable cv; - std::mutex m; - bool terminate = false; -}; - -} // namespace nbd -} // namespace curve - -#endif // SRC_TOOLS_NBD_INTERRUPTIBLE_SLEEPER_H_ diff --git a/nbd/src/libnebd.h b/nbd/src/libnebd.h deleted file mode 100644 index 1e860181e6..0000000000 --- a/nbd/src/libnebd.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Project: curve - * File Created: 2019-08-07 - * Author: hzchenwei7 - * Copyright (c) 2018 NetEase - */ - -#ifndef SRC_TOOLS_NBD_LIBNEBD_H_ -#define SRC_TOOLS_NBD_LIBNEBD_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include -#include -#include -#include -#include - -#define NEBD_MAX_FILE_PATH_LEN 1024 -#define NEBD_MAX_BUF_LEN 1024 * 1024 * 32 - -typedef enum LIBAIO_OP { - LIBAIO_OP_READ, - LIBAIO_OP_WRITE, - LIBAIO_OP_DISCARD, - LIBAIO_OP_FLUSH, -} LIBAIO_OP; - -struct NebdClientAioContext; - -typedef void (*LibAioCallBack)(struct NebdClientAioContext* context); - -typedef struct NebdClientAioContext { - off_t offset; - size_t length; - int ret; - LIBAIO_OP op; - LibAioCallBack cb; - void* buf; - unsigned int retryCount; -} NebdClientAioContext; - -// int nebd_lib_fini(void); -// for ceph & curve -int nebd_lib_init(void); -int nebd_lib_open(const char* filename); -int nebd_lib_close(int fd); -int nebd_lib_pread(int fd, void* buf, off_t offset, size_t length); -int nebd_lib_pwrite(int fd, const void* buf, off_t offset, size_t length); -int nebd_lib_discard(int fd, NebdClientAioContext* context); -int nebd_lib_aio_pread(int fd, NebdClientAioContext* context); -int nebd_lib_aio_pwrite(int fd, NebdClientAioContext* context); -int nebd_lib_sync(int fd); -int64_t nebd_lib_filesize(int fd); -int nebd_lib_resize(int fd, int64_t size); -// for ceph only -int nebd_lib_flush(int fd, NebdClientAioContext* context); -int64_t nebd_lib_getinfo(int fd); -int nebd_lib_invalidcache(int fd); - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif // SRC_TOOLS_NBD_LIBNEBD_H_ diff --git a/nebd/src/common/BUILD b/nebd/src/common/BUILD index a008fd1823..90c92bc136 100644 --- a/nebd/src/common/BUILD +++ b/nebd/src/common/BUILD @@ -11,6 +11,7 @@ cc_library( "//external:butil", "//external:bthread", "//external:bvar", + "//src/common:curve_uncopy", + "//src/common:curve_common", ], ) - diff --git a/nebd/src/common/configuration.cpp b/nebd/src/common/configuration.cpp deleted file mode 100644 index 1938402b39..0000000000 --- a/nebd/src/common/configuration.cpp +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Copyright (C) 2018 NetEase Inc. All rights reserved. - * Project: Curve - * - * History: - * 2018/08/30 Wenyu Zhou Initial version - */ -/* - * Project: nebd - * File Created: 2019-08-07 - * Author: hzchenwei7 - * Copyright (c) 2018 NetEase - */ -#include "nebd/src/common/configuration.h" - -#include -#include -#include -#include - -namespace nebd { -namespace common { - -bool Configuration::LoadConfig() { - std::ifstream cFile(confFile_); - - if (cFile.is_open()) { - std::string line; - while (getline(cFile, line)) { - // FIXME: may not remove middle spaces - line.erase(std::remove_if(line.begin(), line.end(), isspace), - line.end()); - if (line[0] == '#' || line.empty()) - continue; - - int delimiterPos = line.find("="); - std::string key = line.substr(0, delimiterPos); - std::string value = line.substr(delimiterPos + 1); - config_[key] = value; - } - } else { - return false; - } - - return true; -} - -bool Configuration::SaveConfig() { - // 当前先只保存配置，原文件的注释等内容先忽略 - // TODO(yyk): 后续考虑改成原文件格式不变，只修改配置值 - std::ofstream wStream(confFile_); - if (wStream.is_open()) { - for (auto& pair : config_) { - wStream << pair.first << "=" << pair.second << std::endl; - } - wStream.close(); - } else { - return false; - } - return true; -} - -std::string Configuration::DumpConfig() { - // TODO(wenyu): to implement - return ""; -} - - -std::map Configuration::ListConfig() const { - return config_; -} - -void Configuration::SetConfigPath(const std::string &path) { - confFile_ = path; -} - -std::string Configuration::GetConfigPath() { - return confFile_; -} - -std::string Configuration::GetStringValue(const std::string &key) { - return GetValue(key); -} - -bool Configuration::GetStringValue(const std::string &key, std::string *out) { - return GetValue(key, out); -} - -void Configuration::SetStringValue(const std::string &key, - const std::string &value) { - SetValue(key, value); -} - -int Configuration::GetIntValue(const std::string &key, uint64_t defaultvalue) { - std::string value = GetValue(key); - return (value == "") ? defaultvalue : std::stoi(value); -} - -bool Configuration::GetIntValue(const std::string &key, int *out) { - std::string res; - if (GetValue(key, &res)) { - *out = std::stoi(res); - return true; - } - return false; -} - -bool Configuration::GetUInt32Value(const std::string &key, uint32_t *out) { - std::string res; - if (GetValue(key, &res)) { - *out = std::stoul(res); - return true; - } - return false; -} - -bool Configuration::GetUInt64Value(const std::string &key, uint64_t *out) { - std::string res; - if (GetValue(key, &res)) { - *out = std::stoull(res); - return true; - } - return false; -} - -bool Configuration::GetInt64Value(const std::string& key, int64_t* out) { - std::string res; - if (GetValue(key, &res)) { - *out = std::stoll(res); - return true; - } - - return false; -} - -void Configuration::SetIntValue(const std::string &key, const int value) { - SetValue(key, std::to_string(value)); -} - -double Configuration::GetDoubleValue( - const std::string &key, - double defaultvalue) { - std::string value = GetValue(key); - return (value == "") ? defaultvalue : std::stod(value); -} - -bool Configuration::GetDoubleValue(const std::string &key, double *out) { - std::string res; - if (GetValue(key, &res)) { - *out = std::stod(res); - return true; - } - return false; -} - -void Configuration::SetDoubleValue(const std::string &key, const double value) { - SetValue(key, std::to_string(value)); -} - - -double Configuration::GetFloatValue( - const std::string &key, float defaultvalue) { - std::string value = GetValue(key); - return (value == "") ? defaultvalue : std::stof(value); -} - -bool Configuration::GetFloatValue(const std::string &key, float *out) { - std::string res; - if (GetValue(key, &res)) { - *out = std::stof(res); - return true; - } - return false; -} - -void Configuration::SetFloatValue(const std::string &key, const float value) { - SetValue(key, std::to_string(value)); -} - -bool Configuration::GetBoolValue(const std::string &key, bool defaultvalue) { - std::string svalue = config_[key]; - transform(svalue.begin(), svalue.end(), svalue.begin(), ::tolower); - - bool istrue = (svalue == "true") || (svalue == "yes") || (svalue == "1"); - bool isfalse = (svalue == "false") || (svalue == "no") || (svalue == "0"); - bool ret = istrue ? true : isfalse ? false : defaultvalue; - return ret; -} - -bool Configuration::GetBoolValue(const std::string &key, bool *out) { - std::string res; - if (GetValue(key, &res)) { - transform(res.begin(), res.end(), res.begin(), ::tolower); - bool istrue = (res == "true") || (res == "yes") || (res == "1"); - bool isfalse = (res == "false") || (res == "no") || (res == "0"); - if (istrue) { - *out = true; - return true; - } - if (isfalse) { - *out = false; - return true; - } - return false; - } - - return false; -} - - -void Configuration::SetBoolValue(const std::string &key, const bool value) { - SetValue(key, std::to_string(value)); -} - -std::string Configuration::GetValue(const std::string &key) { - return config_[key]; -} - -bool Configuration::GetValue(const std::string &key, std::string *out) { - if (config_.find(key) != config_.end()) { - *out = config_[key]; - return true; - } - - return false; -} - -void Configuration::SetValue(const std::string &key, const std::string &value) { - config_[key] = value; -} - -} // namespace common -} // namespace nebd diff --git a/nebd/src/common/configuration.h b/nebd/src/common/configuration.h deleted file mode 100644 index b2881210aa..0000000000 --- a/nebd/src/common/configuration.h +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (C) 2018 NetEase Inc. All rights reserved. - * Project: Curve - * - * History: - * 2018/08/30 Wenyu Zhou Initial version - */ -/* - * Project: nebd - * File Created: 2019-08-07 - * Author: hzchenwei7 - * Copyright (c) 2018 NetEase - */ -#include -#include - -#ifndef SRC_COMMON_CONFIGURATION_H_ -#define SRC_COMMON_CONFIGURATION_H_ - -namespace nebd { -namespace common { - -class Configuration { - public: - Configuration() {} - ~Configuration() {} - - bool LoadConfig(); - bool SaveConfig(); - std::string DumpConfig(); - std::map ListConfig() const; - - void SetConfigPath(const std::string &path); - std::string GetConfigPath(); - - std::string GetStringValue(const std::string &key); - /* - * @brief GetStringValue 获取指定配置项的值 - * - * @param[in] key 配置项名称 - * @param[out] out 获取的值 - * - * @return false-未获取到 true-获取成功 - */ - bool GetStringValue(const std::string &key, std::string *out); - void SetStringValue(const std::string &key, const std::string &value); - - int GetIntValue(const std::string &key, uint64_t defaultvalue = 0); - /* - * @brief GetIntValue/GetUInt32Value/GetUInt64Value 获取指定配置项的值 //NOLINT - * - * @param[in] key 配置项名称 - * @param[out] out 获取的值 - * - * @return false-未获取到 true-获取成功 - */ - bool GetIntValue(const std::string &key, int *out); - bool GetUInt32Value(const std::string &key, uint32_t *out); - bool GetUInt64Value(const std::string &key, uint64_t *out); - bool GetInt64Value(const std::string& key, int64_t* out); - void SetIntValue(const std::string &key, const int value); - - double GetDoubleValue(const std::string &key, double defaultvalue = 0.0); - /* - * @brief GetDoubleValue 获取指定配置项的值 - * - * @param[in] key 配置项名称 - * @param[out] out 获取的值 - * - * @return false-未获取到 true-获取成功 - */ - bool GetDoubleValue(const std::string &key, double *out); - void SetDoubleValue(const std::string &key, const double value); - - double GetFloatValue(const std::string &key, float defaultvalue = 0.0); - /* - * @brief GetFloatValue 获取指定配置项的值 - * - * @param[in] key 配置项名称 - * @param[out] out 获取的值 - * - * @return false-未获取到 true-获取成功 - */ - bool GetFloatValue(const std::string &key, float *out); - void SetFloatValue(const std::string &key, const float value); - - bool GetBoolValue(const std::string &key, bool defaultvalue = false); - /* - * @brief GetBoolValue 获取指定配置项的值 - * - * @param[in] key 配置项名称 - * @param[out] out 获取的值 - * - * @return false-未获取到 true-获取成功 - */ - bool GetBoolValue(const std::string &key, bool *out); - void SetBoolValue(const std::string &key, const bool value); - - std::string GetValue(const std::string &key); - bool GetValue(const std::string &key, std::string *out); - void SetValue(const std::string &key, const std::string &value); - - private: - std::string confFile_; - std::map config_; -}; - -} // namespace common -} // namespace nebd - -#endif // SRC_COMMON_CONFIGURATION_H_ diff --git a/nebd/src/common/crc32.h b/nebd/src/common/crc32.h deleted file mode 100644 index 8b4c54ca8b..0000000000 --- a/nebd/src/common/crc32.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Project: curve - * Created Date: Saturday December 29th 2018 - * Author: yangyaokai - * Copyright (c) 2018 netease - */ - -#ifndef SRC_COMMON_CRC32_H_ -#define SRC_COMMON_CRC32_H_ - -#include -#include - -#include - -namespace nebd { -namespace common { - -/** - * 计算数据的CRC32校验码(CRC32C)，基于brpc的crc32库进行封装 - * @param pData 待计算的数据 - * @param iLen 待计算的数据长度 - * @return 32位的数据CRC32校验码 - */ -inline uint32_t CRC32(const char *pData, size_t iLen) { - return butil::crc32c::Value(pData, iLen); -} - -/** - * 计算数据的CRC32校验码(CRC32C)，基于brpc的crc32库进行封装. 此函数支持继承式 - * 计算，以支持对SGL类型的数据计算单个CRC校验码。满足如下约束: - * CRC32("hello world", 11) == CRC32(CRC32("hello ", 6), "world", 5) - * @param crc 起始的crc校验码 - * @param pData 待计算的数据 - * @param iLen 待计算的数据长度 - * @return 32位的数据CRC32校验码 - */ -inline uint32_t CRC32(uint32_t crc, const char *pData, size_t iLen) { - return butil::crc32c::Extend(crc, pData, iLen); -} - -} // namespace common -} // namespace nebd - -#endif // SRC_COMMON_CRC32_H_ diff --git a/nebd/src/common/interrupt_sleep.h b/nebd/src/common/interrupt_sleep.h deleted file mode 100644 index 8493825b60..0000000000 --- a/nebd/src/common/interrupt_sleep.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Project: nebd - * File Created: 2019-09-30 - * Author: hzwuhongsong - * Copyright (c) 2019 NetEase - */ - -#ifndef SRC_COMMON_INTERRUPT_SLEEP_H_ -#define SRC_COMMON_INTERRUPT_SLEEP_H_ - -#include -#include -#include // NOLINT - -namespace nebd { -namespace common { - -class InterruptibleSleeper { - public: - // returns false if killed: - template - bool wait_for(std::chrono::duration const& time ) { - std::unique_lock lock(m); - return !cv.wait_for(lock, time, [&]{return terminate;}); - } - - void interrupt() { - std::unique_lock lock(m); - terminate = true; - LOG(WARNING) << "interrupt sleeper."; - cv.notify_all(); - } - private: - std::condition_variable cv; - std::mutex m; - bool terminate = false; -}; - -} // namespace common -} // namespace nebd - -#endif // SRC_COMMON_INTERRUPT_SLEEP_H_ diff --git a/nebd/src/common/name_lock.h b/nebd/src/common/name_lock.h index 4c2c9f75cd..7e2ef6c00c 100644 --- a/nebd/src/common/name_lock.h +++ b/nebd/src/common/name_lock.h @@ -15,11 +15,13 @@ #include #include #include // NOLINT -#include "nebd/src/common/uncopyable.h" +#include "src/common/uncopyable.h" namespace nebd { namespace common { +using curve::common::Uncopyable; + class NameLock : public Uncopyable { public: explicit NameLock(int bucketNum = 256); diff --git a/nebd/src/common/nebd_version.cpp b/nebd/src/common/nebd_version.cpp deleted file mode 100644 index 10c4c2120b..0000000000 --- a/nebd/src/common/nebd_version.cpp +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Project: nebd - * Created Date: 2020-03-23 - * Author: charsiu - * Copyright (c) 2020 netease - */ - -#include "nebd/src/common/nebd_version.h" -#include "nebd/src/common/stringstatus.h" - -namespace nebd { -namespace common { - -// https://gcc.gnu.org/onlinedocs/gcc-4.8.5/cpp/Stringification.html -std::string NebdVersion() { - static const std::string version = -#ifdef NEBDVERSION -# define STR(val) #val -# define XSTR(val) STR(val) - std::string(XSTR(NEBDVERSION)); -#else - std::string("unknown"); -#endif - return version; -} - -const char kNebdMetricPrefix[] = "nebd"; -const char kVersion[] = "version"; - -void ExposeNebdVersion() { - static StringStatus version; - version.ExposeAs(kNebdMetricPrefix, kVersion); - version.Set(kVersion, NebdVersion()); - version.Update(); -} - -} // namespace common -} // namespace nebd diff --git a/nebd/src/common/nebd_version.h b/nebd/src/common/nebd_version.h deleted file mode 100644 index 8dc05e1728..0000000000 --- a/nebd/src/common/nebd_version.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Project: nebd - * Created Date: 2020-03-23 - * Author: charsiu - * Copyright (c) 2020 netease - */ - -#ifndef SRC_COMMON_NEBD_VERSION_H_ -#define SRC_COMMON_NEBD_VERSION_H_ - -#include - -namespace nebd { -namespace common { - -std::string NebdVersion(); -void ExposeNebdVersion(); - -} // namespace common -} // namespace nebd - -#endif // SRC_COMMON_NEBD_VERSION_H_ diff --git a/nebd/src/common/rw_lock.h b/nebd/src/common/rw_lock.h deleted file mode 100644 index 827b3090b6..0000000000 --- a/nebd/src/common/rw_lock.h +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Project: nebd - * Created Date: 18-10-11 - * Author: wudemiao - * Copyright (c) 2018 netease - */ - -#ifndef SRC_COMMON_RW_LOCK_H_ -#define SRC_COMMON_RW_LOCK_H_ - -#include -#include -#include -#include - -#include "nebd/src/common/uncopyable.h" - -namespace nebd { -namespace common { - -class RWLockBase : public Uncopyable { - public: - RWLockBase() {} - virtual ~RWLockBase() {} - virtual void WRLock() = 0; - virtual int TryWRLock() = 0; - virtual void RDLock() = 0; - virtual int TryRDLock() = 0; - virtual void Unlock() = 0; -}; - -class RWLock : public RWLockBase { - public: - RWLock() { - pthread_rwlock_init(&rwlock_, nullptr); - } - - ~RWLock() { - pthread_rwlock_destroy(&rwlock_); - } - - void WRLock() override { - int ret = pthread_rwlock_wrlock(&rwlock_); - CHECK(0 == ret) << "wlock failed: " << ret << ", " << strerror(ret); - } - - int TryWRLock() override { - return pthread_rwlock_trywrlock(&rwlock_); - } - - void RDLock() override { - int ret = pthread_rwlock_rdlock(&rwlock_); - CHECK(0 == ret) << "rlock failed: " << ret << ", " << strerror(ret); - } - - int TryRDLock() override { - return pthread_rwlock_tryrdlock(&rwlock_); - } - - void Unlock() override { - pthread_rwlock_unlock(&rwlock_); - } - - private: - pthread_rwlock_t rwlock_; -}; // RWLock class - -class BthreadRWLock : public RWLockBase { - public: - BthreadRWLock() { - bthread_rwlock_init(&rwlock_, nullptr); - } - - ~BthreadRWLock() { - bthread_rwlock_destroy(&rwlock_); - } - - void WRLock() override { - int ret = bthread_rwlock_wrlock(&rwlock_); - CHECK(0 == ret) << "wlock failed: " << ret << ", " << strerror(ret); - } - - int TryWRLock() override { - // not support yet - return EINVAL; - } - - void RDLock() override { - int ret = bthread_rwlock_rdlock(&rwlock_); - CHECK(0 == ret) << "rlock failed: " << ret << ", " << strerror(ret); - } - - int TryRDLock() override { - // not support yet - return EINVAL; - } - - void Unlock() override { - bthread_rwlock_unlock(&rwlock_); - } - - private: - bthread_rwlock_t rwlock_; -}; // RWLock class - -class ReadLockGuard : public Uncopyable { - public: - explicit ReadLockGuard(RWLockBase &rwlock) : rwlock_(rwlock) { - rwlock_.RDLock(); - } - - ~ReadLockGuard() { - rwlock_.Unlock(); - } - - private: - RWLockBase &rwlock_; -}; // ReadLockGuard class - -class WriteLockGuard : public Uncopyable { - public: - explicit WriteLockGuard(RWLockBase &rwlock) : rwlock_(rwlock) { - rwlock_.WRLock(); - } - - ~WriteLockGuard() { - rwlock_.Unlock(); - } - - private: - RWLockBase &rwlock_; -}; // WriteLockGuard class - -} // namespace common -} // namespace nebd - -#endif // SRC_COMMON_RW_LOCK_H_ diff --git a/nebd/src/common/stringstatus.cpp b/nebd/src/common/stringstatus.cpp deleted file mode 100644 index a03f2825f0..0000000000 --- a/nebd/src/common/stringstatus.cpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Project: nebd - * Created Date: 20190819 - * Author: lixiaocui - * Copyright (c) 2019 netease - */ - -#include "nebd/src/common/stringstatus.h" - -namespace nebd { -namespace common { -void StringStatus::ExposeAs( - const std::string &prefix, const std::string &name) { - status_.expose_as(prefix, name); -} - -void StringStatus::Set(const std::string& key, const std::string& value) { - kvs_[key] = value; -} - -void StringStatus::Update() { - if (kvs_.empty()) { - return; - } - - std::string jsonStr = "{"; - int count = 0; - for (auto &item : kvs_) { - count += 1; - if (count == kvs_.size()) { - jsonStr += - "\"" + item.first + "\"" + ":" + "\"" + item.second + "\""; - } else { - jsonStr += "\"" + item.first + "\"" + ":" + "\"" + item.second + - "\"" + ","; - } - } - - jsonStr += "}"; - status_.set_value(jsonStr); -} - -std::string StringStatus::JsonBody() { - return status_.get_value(); -} - -std::string StringStatus::GetValueByKey(const std::string &key) { - return kvs_[key]; -} -} // namespace common -} // namespace nebd diff --git a/nebd/src/common/stringstatus.h b/nebd/src/common/stringstatus.h deleted file mode 100644 index a0ff1a8ab5..0000000000 --- a/nebd/src/common/stringstatus.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Project: nebd - * Created Date: 20190819 - * Author: lixiaocui - * Copyright (c) 2019 netease - */ - - -#ifndef SRC_COMMON_STRINGSTATUS_H_ -#define SRC_COMMON_STRINGSTATUS_H_ - -#include -#include -#include - -namespace nebd { -namespace common { -class StringStatus { - public: - /** - * @brief ExposeAs 用于初始化bvar - * - * @param[in] prefix, 前缀 - * @param[in] name, 名字 - */ - void ExposeAs(const std::string &prefix, const std::string &name); - - /** - * @brief Set 设置每项key-value信息 - * - * @param[in] key - * @param[in] value - */ - void Set(const std::string& key, const std::string& value); - - /** - * @brief Update 把当前key-value map中的键值对以json string的形式设置到status中 //NOLINT - */ - void Update(); - - /** - * @brief GetValueByKey 获取指定key对应的value - * - * @param[in] key 指定key - */ - std::string GetValueByKey(const std::string &key); - - /** - * @brief JsonBody 获取当前key-value map对应的json形式字符串 - */ - std::string JsonBody(); - - private: - // 需要导出的结构体的key-value map - std::map kvs_; - // 该导出项对应的status - bvar::Status status_; -}; -} // namespace common -} // namespace nebd - -#endif // SRC_COMMON_STRINGSTATUS_H_ - diff --git a/nebd/src/common/timeutility.h b/nebd/src/common/timeutility.h deleted file mode 100644 index 6da3624db9..0000000000 --- a/nebd/src/common/timeutility.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Project: curve - * Created Date: Wednesday August 22nd 2018 - * Author: hzsunjianliang - * Copyright (c) 2018 netease - */ -#ifndef SRC_COMMON_TIMEUTILITY_H_ -#define SRC_COMMON_TIMEUTILITY_H_ - -#include -#include -#include -#include -#include - -namespace nebd { -namespace common { - -class TimeUtility { - public: - static inline uint64_t GetTimeofDayUs() { - timeval now; - gettimeofday(&now, NULL); - return now.tv_sec * 1000000L + now.tv_usec; - } - - static inline uint64_t GetTimeofDayMs() { - timeval now; - gettimeofday(&now, NULL); - return now.tv_sec * 1000L + now.tv_usec / 1000; - } - - static inline uint64_t GetTimeofDaySec() { - timeval tm; - gettimeofday(&tm, NULL); - return tm.tv_sec; - } - - // 时间戳转成标准时间输出在standard里面,时间戳单位为秒 - static inline void TimeStampToStandard(time_t timeStamp, - std::string* standard) { - char now[64]; - struct tm p; - p = *localtime_r(&timeStamp, &p); - strftime(now, 64, "%Y-%m-%d %H:%M:%S", &p); - *standard = std::string(now); - } -}; - -} // namespace common -} // namespace nebd - -#endif // SRC_COMMON_TIMEUTILITY_H_ diff --git a/nebd/src/common/uncopyable.h b/nebd/src/common/uncopyable.h deleted file mode 100644 index 2218e3d6fc..0000000000 --- a/nebd/src/common/uncopyable.h +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Project: nebd - * Created Date: 18-8-31 - * Author: wudemiao - * Copyright (c) 2018 netease - */ - -#ifndef SRC_COMMON_UNCOPYABLE_H_ -#define SRC_COMMON_UNCOPYABLE_H_ - -namespace nebd { -namespace common { - -class Uncopyable { - protected: - Uncopyable() = default; - virtual ~Uncopyable() = default; - - private: - Uncopyable(const Uncopyable &) = delete; - Uncopyable &operator=(const Uncopyable &) = delete; -}; - -} // namespace common -} // namespace nebd - -#endif // SRC_COMMON_UNCOPYABLE_H_ diff --git a/nebd/src/logrotate.conf b/nebd/src/logrotate.conf deleted file mode 100644 index 0694a0e0db..0000000000 --- a/nebd/src/logrotate.conf +++ /dev/null @@ -1,12 +0,0 @@ -/var/log/nebd/*.log { - rotate 7 - daily - compress - sharedscripts - postrotate - killall -q -1 nebd-server || true - endscript - missingok - notifempty -} - diff --git a/nebd/src/part1/BUILD b/nebd/src/part1/BUILD index f929f8e593..0bd3f20fb8 100644 --- a/nebd/src/part1/BUILD +++ b/nebd/src/part1/BUILD @@ -34,7 +34,10 @@ cc_library( "//external:glog", "//external:brpc", "//nebd/src/common:nebd_common", - "//nebd/proto:client_cc_proto" + "//nebd/proto:client_cc_proto", + "//src/common:curve_common", + "//src/common:curve_uncopy", + "//src/common/concurrent:curve_concurrent", ], copts = COPTS, linkopts = ["-Wl,-rpath=/usr/lib/nebd"], diff --git a/nebd/src/part1/heartbeat_manager.cpp b/nebd/src/part1/heartbeat_manager.cpp index 96354bab9c..89aaec6aa5 100644 --- a/nebd/src/part1/heartbeat_manager.cpp +++ b/nebd/src/part1/heartbeat_manager.cpp @@ -13,7 +13,7 @@ #include #include "nebd/proto/heartbeat.pb.h" -#include "nebd/src/common/nebd_version.h" +#include "src/common/curve_version.h" namespace nebd { namespace client { @@ -36,7 +36,7 @@ int HeartbeatManager::Init(const HeartbeatOption& option) { } pid_ = getpid(); - nebdVersion_ = nebd::common::NebdVersion(); + nebdVersion_ = curve::common::CurveVersion(); return 0; } diff --git a/nebd/src/part1/heartbeat_manager.h b/nebd/src/part1/heartbeat_manager.h index a58b0ec4d0..d406046368 100644 --- a/nebd/src/part1/heartbeat_manager.h +++ b/nebd/src/part1/heartbeat_manager.h @@ -16,7 +16,7 @@ #include "nebd/src/part1/nebd_common.h" #include "nebd/src/part1/nebd_metacache.h" -#include "nebd/src/common/interrupt_sleep.h" +#include "src/common/interruptible_sleeper.h" namespace nebd { namespace client { @@ -67,7 +67,7 @@ class HeartbeatManager { std::shared_ptr metaCache_; std::thread heartbeatThread_; - common::InterruptibleSleeper sleeper_; + curve::common::InterruptibleSleeper sleeper_; std::atomic running_; std::atomic logId_; diff --git a/nebd/src/part1/libnebd_file.cpp b/nebd/src/part1/libnebd_file.cpp index c0bc322925..0b20819f96 100644 --- a/nebd/src/part1/libnebd_file.cpp +++ b/nebd/src/part1/libnebd_file.cpp @@ -7,7 +7,7 @@ #include "nebd/src/part1/libnebd_file.h" #include -#include "nebd/src/common/configuration.h" +#include "src/common/configuration.h" #include "nebd/src/part1/nebd_client.h" diff --git a/nebd/src/part1/nebd_client.cpp b/nebd/src/part1/nebd_client.cpp index f58e6b4a8a..8716b1259a 100644 --- a/nebd/src/part1/nebd_client.cpp +++ b/nebd/src/part1/nebd_client.cpp @@ -17,7 +17,7 @@ #include #include "nebd/src/part1/async_request_closure.h" -#include "nebd/src/common/configuration.h" +#include "src/common/configuration.h" #define RETURN_IF_FALSE(val) if (val == false) { return -1; } @@ -43,7 +43,7 @@ NebdClient &nebdClient = NebdClient::GetInstance(); constexpr int32_t kBufSize = 128; int NebdClient::Init(const char* confpath) { - nebd::common::Configuration conf; + curve::common::Configuration conf; conf.SetConfigPath(confpath); if (!conf.LoadConfig()) { diff --git a/nebd/src/part1/nebd_client.h b/nebd/src/part1/nebd_client.h index 055d91e34d..e17504613b 100644 --- a/nebd/src/part1/nebd_client.h +++ b/nebd/src/part1/nebd_client.h @@ -15,7 +15,7 @@ #include #include "nebd/src/part1/nebd_common.h" -#include "nebd/src/common/configuration.h" +#include "src/common/configuration.h" #include "nebd/proto/client.pb.h" #include "nebd/src/part1/libnebd.h" #include "nebd/src/part1/heartbeat_manager.h" @@ -27,7 +27,7 @@ namespace client { using RpcTask = std::function; -using nebd::common::Configuration; +using curve::common::Configuration; class NebdClient { public: diff --git a/nebd/src/part1/nebd_metacache.cpp b/nebd/src/part1/nebd_metacache.cpp index 902253a779..1a851fa4af 100644 --- a/nebd/src/part1/nebd_metacache.cpp +++ b/nebd/src/part1/nebd_metacache.cpp @@ -10,19 +10,22 @@ namespace nebd { namespace client { +using curve::common::WriteLockGuard; +using curve::common::ReadLockGuard; + void NebdClientMetaCache::AddFileInfo(const NebdClientFileInfo& fileInfo) { - common::WriteLockGuard guard(rwLock_); + WriteLockGuard guard(rwLock_); fileinfos_.emplace(fileInfo.fd, fileInfo); } void NebdClientMetaCache::RemoveFileInfo(int fd) { - common::WriteLockGuard guard(rwLock_); + WriteLockGuard guard(rwLock_); fileinfos_.erase(fd); } int NebdClientMetaCache::GetFileInfo( int fd, NebdClientFileInfo* fileInfo) const { - common::ReadLockGuard guard(rwLock_); + ReadLockGuard guard(rwLock_); auto iter = fileinfos_.find(fd); if (iter != fileinfos_.end()) { *fileInfo = iter->second; @@ -33,7 +36,7 @@ int NebdClientMetaCache::GetFileInfo( } std::vector NebdClientMetaCache::GetAllFileInfo() const { - common::ReadLockGuard guard(rwLock_); + ReadLockGuard guard(rwLock_); std::vector result; result.reserve(fileinfos_.size()); diff --git a/nebd/src/part1/nebd_metacache.h b/nebd/src/part1/nebd_metacache.h index 0619e7e187..b746d038ae 100644 --- a/nebd/src/part1/nebd_metacache.h +++ b/nebd/src/part1/nebd_metacache.h @@ -12,7 +12,7 @@ #include #include -#include "nebd/src/common/rw_lock.h" +#include "src/common/concurrent/rw_lock.h" #include "nebd/src/common/file_lock.h" namespace nebd { @@ -72,7 +72,7 @@ class NebdClientMetaCache { private: // 当前已打开文件信息 std::unordered_map fileinfos_; - mutable common::RWLock rwLock_; + mutable curve::common::RWLock rwLock_; }; } // namespace client diff --git a/nebd/src/part2/BUILD b/nebd/src/part2/BUILD index f3e61604f2..ce257c68f1 100644 --- a/nebd/src/part2/BUILD +++ b/nebd/src/part2/BUILD @@ -36,8 +36,10 @@ cc_library( "//external:json", "//external:gflags", "//src/client:curve", + "//src/common:curve_common", + "//src/common/concurrent:curve_concurrent", "//nebd/src/common:nebd_common", - "//nebd/proto:client_cc_proto" + "//nebd/proto:client_cc_proto", ], copts = COPTS, ) diff --git a/nebd/src/part2/define.h b/nebd/src/part2/define.h index 72303bcd0a..6dbed7412c 100644 --- a/nebd/src/part2/define.h +++ b/nebd/src/part2/define.h @@ -14,12 +14,12 @@ #include #include -#include "nebd/src/common/rw_lock.h" +#include "src/common/concurrent/rw_lock.h" namespace nebd { namespace server { -using nebd::common::RWLock; +using curve::common::RWLock; using ::google::protobuf::Message; using ::google::protobuf::Closure; using ::google::protobuf::RpcController; diff --git a/nebd/src/part2/file_entity.h b/nebd/src/part2/file_entity.h index 118a36245a..38cd5025c8 100644 --- a/nebd/src/part2/file_entity.h +++ b/nebd/src/part2/file_entity.h @@ -16,20 +16,21 @@ #include // NOLINT #include -#include "nebd/src/common/rw_lock.h" -#include "nebd/src/common/timeutility.h" +#include "src/common/concurrent/rw_lock.h" #include "nebd/src/part2/define.h" #include "nebd/src/part2/util.h" #include "nebd/src/part2/request_executor.h" #include "nebd/src/part2/metafile_manager.h" +#include "src/common/timeutility.h" + namespace nebd { namespace server { -using nebd::common::BthreadRWLock; -using nebd::common::WriteLockGuard; -using nebd::common::ReadLockGuard; -using nebd::common::TimeUtility; +using curve::common::BthreadRWLock; +using curve::common::WriteLockGuard; +using curve::common::ReadLockGuard; +using curve::common::TimeUtility; class NebdFileInstance; class NebdRequestExecutor; diff --git a/nebd/src/part2/file_manager.h b/nebd/src/part2/file_manager.h index 113b4eed85..a0804f3681 100644 --- a/nebd/src/part2/file_manager.h +++ b/nebd/src/part2/file_manager.h @@ -16,7 +16,7 @@ #include // NOLINT #include -#include "nebd/src/common/rw_lock.h" +#include "src/common/concurrent/rw_lock.h" #include "nebd/src/common/name_lock.h" #include "nebd/src/part2/define.h" #include "nebd/src/part2/util.h" @@ -29,8 +29,8 @@ namespace server { using nebd::common::NameLock; using nebd::common::NameLockGuard; -using nebd::common::WriteLockGuard; -using nebd::common::ReadLockGuard; +using curve::common::WriteLockGuard; +using curve::common::ReadLockGuard; using FileEntityMap = std::unordered_map; class NebdFileManager { diff --git a/nebd/src/part2/heartbeat_manager.cpp b/nebd/src/part2/heartbeat_manager.cpp index 79c91e3a12..75e4557ea4 100644 --- a/nebd/src/part2/heartbeat_manager.cpp +++ b/nebd/src/part2/heartbeat_manager.cpp @@ -8,13 +8,14 @@ #include #include -#include "nebd/src/common/timeutility.h" #include "nebd/src/part2/heartbeat_manager.h" +#include "src/common/timeutility.h" + namespace nebd { namespace server { -using common::TimeUtility; +using curve::common::TimeUtility; int HeartbeatManager::Run() { if (isRunning_.exchange(true)) { diff --git a/nebd/src/part2/heartbeat_manager.h b/nebd/src/part2/heartbeat_manager.h index c4dc8e5ac1..d59d12df06 100644 --- a/nebd/src/part2/heartbeat_manager.h +++ b/nebd/src/part2/heartbeat_manager.h @@ -15,19 +15,19 @@ #include #include -#include "nebd/src/common/interrupt_sleep.h" -#include "nebd/src/common/rw_lock.h" -#include "nebd/src/common/stringstatus.h" +#include "src/common/interruptible_sleeper.h" +#include "src/common/concurrent/rw_lock.h" +#include "src/common/stringstatus.h" #include "nebd/src/part2/file_manager.h" #include "nebd/src/part2/define.h" namespace nebd { namespace server { -using nebd::common::InterruptibleSleeper; -using nebd::common::RWLock; -using nebd::common::WriteLockGuard; -using nebd::common::ReadLockGuard; +using curve::common::InterruptibleSleeper; +using curve::common::RWLock; +using curve::common::WriteLockGuard; +using curve::common::ReadLockGuard; struct HeartbeatManagerOption { // 文件心跳超时时间（单位：秒） @@ -53,7 +53,7 @@ struct NebdClientInfo { // nebd client的进程号 int pid; // nebd version的metric - nebd::common::StringStatus version; + curve::common::StringStatus version; // 上次心跳的时间戳 uint64_t timeStamp; }; diff --git a/nebd/src/part2/heartbeat_service.cpp b/nebd/src/part2/heartbeat_service.cpp index 83b09590ed..0ca75b98c7 100644 --- a/nebd/src/part2/heartbeat_service.cpp +++ b/nebd/src/part2/heartbeat_service.cpp @@ -5,14 +5,14 @@ * Copyright (c) 2020 netease */ -#include "nebd/src/common/timeutility.h" #include "nebd/src/part2/heartbeat_service.h" #include "nebd/src/part2/define.h" +#include "src/common/timeutility.h" namespace nebd { namespace server { -using common::TimeUtility; +using curve::common::TimeUtility; void NebdHeartbeatServiceImpl::KeepAlive( google::protobuf::RpcController* cntl_base, diff --git a/nebd/src/part2/metafile_manager.cpp b/nebd/src/part2/metafile_manager.cpp index fee2f1e7e5..3f7f83eba1 100644 --- a/nebd/src/part2/metafile_manager.cpp +++ b/nebd/src/part2/metafile_manager.cpp @@ -174,7 +174,7 @@ int NebdMetaFileParser::Parse(Json::Value root, uint32_t crcValue = root[kCRC].asUInt(); root.removeMember(kCRC); std::string jsonString = root.toStyledString(); - uint32_t crcCalc = nebd::common::CRC32(jsonString.c_str(), + uint32_t crcCalc = curve::common::CRC32(jsonString.c_str(), jsonString.size()); if (crcValue != crcCalc) { LOG(ERROR) << "Parse json: " << root @@ -241,7 +241,7 @@ Json::Value NebdMetaFileParser::ConvertFileMetasToJson( // 计算crc std::string jsonString = root.toStyledString(); - uint32_t crc = nebd::common::CRC32(jsonString.c_str(), jsonString.size()); + uint32_t crc = curve::common::CRC32(jsonString.c_str(), jsonString.size()); root[kCRC] = crc; return root; } diff --git a/nebd/src/part2/metafile_manager.h b/nebd/src/part2/metafile_manager.h index 109f97a935..285d965f2c 100644 --- a/nebd/src/part2/metafile_manager.h +++ b/nebd/src/part2/metafile_manager.h @@ -16,19 +16,20 @@ #include // NOLINT #include // NOLINT -#include "nebd/src/common/rw_lock.h" +#include "src/common/concurrent/rw_lock.h" #include "nebd/src/common/posix_wrapper.h" -#include "nebd/src/common/crc32.h" #include "nebd/src/part2/define.h" #include "nebd/src/part2/util.h" +#include "src/common/crc32.h" + namespace nebd { namespace server { using nebd::common::PosixWrapper; -using nebd::common::RWLock; -using nebd::common::WriteLockGuard; -using nebd::common::ReadLockGuard; +using curve::common::RWLock; +using curve::common::WriteLockGuard; +using curve::common::ReadLockGuard; using FileMetaMap = std::unordered_map; const char kVolumes[] = "volumes"; diff --git a/nebd/src/part2/nebd_server.cpp b/nebd/src/part2/nebd_server.cpp index 4cd19ea147..2d2f8aaf70 100644 --- a/nebd/src/part2/nebd_server.cpp +++ b/nebd/src/part2/nebd_server.cpp @@ -8,10 +8,10 @@ #include #include #include "nebd/src/common/file_lock.h" -#include "nebd/src/common/nebd_version.h" #include "nebd/src/part2/nebd_server.h" #include "nebd/src/part2/file_service.h" #include "nebd/src/part2/heartbeat_service.h" +#include "src/common/curve_version.h" namespace nebd { namespace server { @@ -60,8 +60,8 @@ int NebdServer::Init(const std::string &confPath, LOG(INFO) << "NebdServer init ok"; // 暴露版本信息 - LOG(INFO) << "nebd version: " << nebd::common::NebdVersion(); - nebd::common::ExposeNebdVersion(); + LOG(INFO) << "nebd version: " << curve::common::CurveVersion(); + curve::common::ExposeCurveVersion(); return 0; } diff --git a/nebd/src/part2/nebd_server.h b/nebd/src/part2/nebd_server.h index 2ae3fcda40..bf8fc5e72c 100644 --- a/nebd/src/part2/nebd_server.h +++ b/nebd/src/part2/nebd_server.h @@ -11,7 +11,7 @@ #include #include #include -#include "nebd/src/common/configuration.h" +#include "src/common/configuration.h" #include "nebd/src/part2/file_manager.h" #include "nebd/src/part2/heartbeat_manager.h" #include "nebd/src/part2/request_executor_curve.h" @@ -19,7 +19,7 @@ namespace nebd { namespace server { -using ::nebd::common::Configuration; +using ::curve::common::Configuration; using ::curve::client::CurveClient; class NebdServer { diff --git a/nebd/test/common/BUILD b/nebd/test/common/BUILD index dcf0c31a2e..7db2a474d0 100644 --- a/nebd/test/common/BUILD +++ b/nebd/test/common/BUILD @@ -6,6 +6,7 @@ cc_binary( deps = [ "//external:gflags", "//nebd/src/common:nebd_common", + "//src/common:curve_common", "@com_google_googletest//:gtest_main", ], ) diff --git a/nebd/test/common/configuration_test.cpp b/nebd/test/common/configuration_test.cpp deleted file mode 100644 index fd871fef9e..0000000000 --- a/nebd/test/common/configuration_test.cpp +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Copyright (C) 2018 NetEase Inc. All rights reserved. - * Project: Curve - * - * History: - * 2018/11/23 Wenyu Zhou Initial version - */ - -#include -#include - -#include -#include -#include -#include - -#include "nebd/src/common/configuration.h" - -namespace nebd { -namespace common { - -class ConfigurationTest : public ::testing::Test { - public: - void SetUp() { - std::string confItem; - - confFile_ = "curve.conf.test"; - std::ofstream cFile(confFile_); - ASSERT_TRUE(cFile.is_open()); - - confItem = "test.str1=teststring\n"; - cFile << confItem; - - confItem = "test.int1=12345\n"; - cFile << confItem; - - confItem = "test.int2=-2345\n"; - cFile << confItem; - - confItem = "test.int3=0\n"; - cFile << confItem; - - confItem = "test.bool1=0\n"; - cFile << confItem; - - confItem = "test.bool2=1\n"; - cFile << confItem; - - confItem = "test.bool3=false\n"; - cFile << confItem; - - confItem = "test.bool4=true\n"; - cFile << confItem; - - confItem = "test.bool5=no\n"; - cFile << confItem; - - confItem = "test.bool6=yes\n"; - cFile << confItem; - - confItem = "test.double1=3.1415926\n"; - cFile << confItem; - - confItem = "test.double2=1\n"; - cFile << confItem; - - confItem = "test.double3=1.0\n"; - cFile << confItem; - - confItem = "test.double4=0.1\n"; - cFile << confItem; - } - - void TearDown() { - ASSERT_EQ(0, unlink(confFile_.c_str())); - } - - std::string confFile_; -}; - -TEST_F(ConfigurationTest, SetAndGetConfigPath) { - Configuration conf; - - conf.SetConfigPath(confFile_); - ASSERT_EQ(conf.GetConfigPath(), confFile_); -} - -TEST_F(ConfigurationTest, LoadNonExistConfigFile) { - bool ret; - std::string confFile = "curve.conf.test.nonexist"; - Configuration conf; - - conf.SetConfigPath(confFile); - ret = conf.LoadConfig(); - ASSERT_EQ(ret, false); -} - -TEST_F(ConfigurationTest, LoadNormalConfigFile) { - bool ret; - Configuration conf; - - conf.SetConfigPath(confFile_); - ret = conf.LoadConfig(); - ASSERT_EQ(ret, true); -} - -TEST_F(ConfigurationTest, DumpConfig) { - Configuration conf; - - conf.SetConfigPath(confFile_); - // not implemented yet, assert null returned - ASSERT_EQ(conf.DumpConfig(), ""); -} - -TEST_F(ConfigurationTest, ListConfig) { - Configuration conf; - - conf.SetConfigPath(confFile_); - int ret = conf.LoadConfig(); - ASSERT_EQ(ret, true); - std::map configs; - configs = conf.ListConfig(); - ASSERT_NE(0, configs.size()); - // 抽几个key来校验以下 - ASSERT_EQ(configs["test.int1"], "12345"); - ASSERT_EQ(configs["test.bool1"], "0"); - // 如果key不存在，返回为空 - ASSERT_EQ(configs["xxx"], ""); -} - -TEST_F(ConfigurationTest, SaveConfig) { - bool ret; - Configuration conf; - conf.SetConfigPath(confFile_); - - // 自定义配置项并保存 - conf.SetStringValue("test.str1", "new"); - ret = conf.SaveConfig(); - ASSERT_EQ(ret, true); - - // 重新加载配置项 - Configuration conf2; - conf2.SetConfigPath(confFile_); - ret = conf2.LoadConfig(); - ASSERT_EQ(ret, true); - - // 可以读取自定义配置项，原有配置项被覆盖，读取不到 - ASSERT_EQ(conf2.GetValue("test.str1"), "new"); - ASSERT_EQ(conf2.GetValue("test.int1"), ""); -} - -TEST_F(ConfigurationTest, GetSetValue) { - bool ret; - Configuration conf; - - conf.SetConfigPath(confFile_); - ret = conf.LoadConfig(); - ASSERT_EQ(ret, true); - - ASSERT_EQ(conf.GetValue("test.str1"), "teststring"); - ASSERT_EQ(conf.GetValue("test.int1"), "12345"); - ASSERT_EQ(conf.GetValue("test.bool1"), "0"); - ASSERT_EQ(conf.GetValue("test.str.nonexist"), ""); - - conf.SetValue("test.str1", "teststring2"); - ASSERT_EQ(conf.GetValue("test.str1"), "teststring2"); - std::string out; - ASSERT_FALSE(conf.GetValue("no.exist", &out)); - conf.SetValue("put.in", "put.in"); - ASSERT_TRUE(conf.GetValue("put.in", &out)); - ASSERT_EQ("put.in", out); -} - -TEST_F(ConfigurationTest, GetSetStringValue) { - bool ret; - Configuration conf; - - conf.SetConfigPath(confFile_); - ret = conf.LoadConfig(); - ASSERT_EQ(ret, true); - - ASSERT_EQ(conf.GetStringValue("test.str1"), "teststring"); - ASSERT_EQ(conf.GetStringValue("test.int1"), "12345"); - ASSERT_EQ(conf.GetStringValue("test.bool1"), "0"); - ASSERT_EQ(conf.GetStringValue("test.str.nonexist"), ""); - - conf.SetStringValue("test.str1", "teststring2"); - ASSERT_EQ(conf.GetStringValue("test.str1"), "teststring2"); - - std::string out; - ASSERT_FALSE(conf.GetStringValue("no.exist", &out)); - conf.SetStringValue("put.in", "put.in"); - ASSERT_TRUE(conf.GetStringValue("put.in", &out)); - ASSERT_EQ("put.in", out); -} - -TEST_F(ConfigurationTest, GetSetIntValue) { - bool ret; - Configuration conf; - - conf.SetConfigPath(confFile_); - ret = conf.LoadConfig(); - ASSERT_EQ(ret, true); - - ASSERT_EQ(conf.GetIntValue("test.int1"), 12345); - ASSERT_EQ(conf.GetIntValue("test.int2"), -2345); - ASSERT_EQ(conf.GetIntValue("test.int3"), 0); - ASSERT_EQ(conf.GetIntValue("test.int.nonexist"), 0); - - conf.SetIntValue("test.int1", 123); - ASSERT_EQ(conf.GetIntValue("test.int1"), 123); - - int out; - ASSERT_FALSE(conf.GetIntValue("no.exist", &out)); - conf.SetIntValue("no.exist", 1); - ASSERT_TRUE(conf.GetIntValue("no.exist", &out)); - ASSERT_EQ(1, out); - - uint32_t outu32; - ASSERT_FALSE(conf.GetUInt32Value("no.exist.u32", &outu32)); - conf.SetIntValue("no.exist.u32", 2); - ASSERT_TRUE(conf.GetUInt32Value("no.exist.u32", &outu32)); - ASSERT_EQ(2, outu32); - - uint64_t outu64; - ASSERT_FALSE(conf.GetUInt64Value("no.exist.u64", &outu64)); - conf.SetIntValue("no.exist.u64", 3); - ASSERT_TRUE(conf.GetUInt64Value("no.exist.u64", &outu64)); - ASSERT_EQ(3, outu64); -} - -TEST_F(ConfigurationTest, GetSetBoolValue) { - bool ret; - Configuration conf; - - conf.SetConfigPath(confFile_); - ret = conf.LoadConfig(); - ASSERT_EQ(ret, true); - - ASSERT_EQ(conf.GetBoolValue("test.bool1"), false); - ASSERT_EQ(conf.GetBoolValue("test.bool2"), true); - ASSERT_EQ(conf.GetBoolValue("test.bool3"), false); - ASSERT_EQ(conf.GetBoolValue("test.bool4"), true); - ASSERT_EQ(conf.GetBoolValue("test.bool5"), false); - ASSERT_EQ(conf.GetBoolValue("test.bool6"), true); - ASSERT_EQ(conf.GetBoolValue("test.bool.nonexist"), false); - - conf.SetBoolValue("test.bool1", true); - ASSERT_EQ(conf.GetBoolValue("test.bool1"), true); - - bool out; - ASSERT_FALSE(conf.GetBoolValue("no.exist", &out)); - conf.SetIntValue("no.exist", false); - ASSERT_TRUE(conf.GetBoolValue("no.exist", &out)); - ASSERT_FALSE(out); -} - -TEST_F(ConfigurationTest, GetSetDoubleAndFloatValue) { - bool ret; - Configuration conf; - - conf.SetConfigPath(confFile_); - ret = conf.LoadConfig(); - ASSERT_EQ(ret, true); - - ASSERT_EQ(conf.GetDoubleValue("test.double1"), 3.1415926); - ASSERT_EQ(conf.GetDoubleValue("test.double2"), 1); - ASSERT_EQ(conf.GetDoubleValue("test.double3"), 1.0); - ASSERT_EQ(conf.GetDoubleValue("test.double4"), 0.1); - ASSERT_EQ(conf.GetFloatValue("test.double4"), 0.1f); - - conf.SetDoubleValue("test.double1", 100.0); - ASSERT_EQ(conf.GetDoubleValue("test.double1"), 100.0); - - double out; - float outf; - ASSERT_FALSE(conf.GetDoubleValue("no.exist", &out)); - ASSERT_FALSE(conf.GetFloatValue("no.exist", &outf)); - conf.SetDoubleValue("no.exist", 0.009); - ASSERT_TRUE(conf.GetDoubleValue("no.exist", &out)); - ASSERT_TRUE(conf.GetFloatValue("no.exist", &outf)); - ASSERT_EQ(0.009, out); - ASSERT_EQ(0.009f, outf); -} - -} // namespace common -} // namespace nebd - -int main(int argc, char ** argv) { - ::testing::InitGoogleTest(&argc, argv); - ::testing::InitGoogleMock(&argc, argv); - int ret = RUN_ALL_TESTS(); - - return ret; -} diff --git a/nebd/test/common/crc32_test.cpp b/nebd/test/common/crc32_test.cpp deleted file mode 100644 index c697432c5e..0000000000 --- a/nebd/test/common/crc32_test.cpp +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Project: curve - * Created Date: Saturday December 29th 2018 - * Author: yangyaokai - * Copyright (c) 2018 netease - */ - -#include - -#include "nebd/src/common/crc32.h" - -namespace nebd { -namespace common { - -TEST(Crc32TEST, BasicTest) { - char buf1[10]; - ::memset(buf1, 0, 10); - char buf2[10]; - ::memset(buf2, 1, 10); - char buf3[20]; - ::memset(buf3, 0, 20); - char buf4[10]; - ::memset(buf4, 0, 10); - uint32_t crc1 = CRC32(buf1, sizeof(buf1)); - uint32_t crc2 = CRC32(buf2, sizeof(buf2)); - uint32_t crc3 = CRC32(buf3, sizeof(buf3)); - uint32_t crc4 = CRC32(buf4, sizeof(buf4)); - ASSERT_EQ(crc1, crc4); - ASSERT_NE(crc1, crc2); - ASSERT_NE(crc1, crc3); - ASSERT_NE(crc2, crc3); -} - -TEST(Crc32TEST, StandardResults) { - // From rfc3720 section B.4. - char buf[32]; - - memset(buf, 0, sizeof(buf)); - ASSERT_EQ(0x8a9136aaU, CRC32(buf, sizeof(buf))); - - memset(buf, 0xff, sizeof(buf)); - ASSERT_EQ(0x62a8ab43U, CRC32(buf, sizeof(buf))); - - for (int i = 0; i < 32; i++) { - buf[i] = i; - } - ASSERT_EQ(0x46dd794eU, CRC32(buf, sizeof(buf))); - - for (int i = 0; i < 32; i++) { - buf[i] = 31 - i; - } - ASSERT_EQ(0x113fdb5cU, CRC32(buf, sizeof(buf))); - - unsigned char data[48] = { - 0x01, 0xc0, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x14, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x04, 0x00, - 0x00, 0x00, 0x00, 0x14, - 0x00, 0x00, 0x00, 0x18, - 0x28, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - }; - ASSERT_EQ(0xd9963a56, CRC32(reinterpret_cast(data), sizeof(data))); -} - -TEST(Crc32TEST, Values) { - ASSERT_NE(CRC32("a", 1), CRC32("foo", 3)); -} - -TEST(Crc32TEST, Extend) { - ASSERT_EQ(CRC32("hello world", 11), - CRC32(CRC32("hello ", 6), "world", 5)); -} - -} // namespace common -} // namespace nebd diff --git a/nebd/test/common/interruptible_sleeper_test.cpp b/nebd/test/common/interruptible_sleeper_test.cpp deleted file mode 100644 index f01c8e3883..0000000000 --- a/nebd/test/common/interruptible_sleeper_test.cpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Project: nebd - * Created Date: 2019-11-25 - * Author: lixiaocui - * Copyright (c) 2019 netease - */ - -#include -#include "nebd/src/common/interrupt_sleep.h" -#include "nebd/src/common/timeutility.h" - -namespace nebd { -namespace common { -InterruptibleSleeper sleeper; -bool is_stop = false; - -void handler(int sig) { - sleeper.interrupt(); -} - -TEST(InterruptibleSleeperTest, test_interruptible_sleeper) { - pid_t pid = ::fork(); - if (0 > pid) { - ASSERT_TRUE(false); - } else if (0 == pid) { - struct sigaction action; - action.sa_handler = handler; - sigemptyset(&action.sa_mask); - action.sa_flags = 0; - sigaction(SIGTERM, &action, NULL); - - while (sleeper.wait_for(std::chrono::seconds(10))) {} - return; - } - - usleep(50 * 1000); - uint64_t startKill = TimeUtility::GetTimeofDayMs(); - int waitstatus; - kill(pid, SIGTERM); - waitpid(pid, &waitstatus, 0); - ASSERT_GT(8000, TimeUtility::GetTimeofDayMs() - startKill); -} -} // namespace common -} // namespace nebd diff --git a/nebd/test/common/rw_lock_test.cpp b/nebd/test/common/rw_lock_test.cpp deleted file mode 100644 index 51bbf3fab1..0000000000 --- a/nebd/test/common/rw_lock_test.cpp +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Project: nebd - * Created Date: 18-10-12 - * Author: wudemiao - * Copyright (c) 2018 netease - */ - -#include -#include - -#include // NOLINT - -#include "nebd/src/common/rw_lock.h" - -namespace nebd { -namespace common { - -TEST(RWLockTest, basic_test) { - RWLock rwlock; - { - ReadLockGuard readLockGuard(rwlock); - ASSERT_TRUE(true); - } - { - WriteLockGuard writeLockGuard(rwlock); - ASSERT_TRUE(true); - } - { - WriteLockGuard writeLockGuard(rwlock); - ASSERT_TRUE(true); - } - { - ReadLockGuard readLockGuard1(rwlock); - ReadLockGuard readLockGuard2(rwlock); - ASSERT_TRUE(true); - } - { - ReadLockGuard readLockGuard1(rwlock); - ReadLockGuard readLockGuard2(rwlock); - ReadLockGuard readLockGuard3(rwlock); - ReadLockGuard readLockGuard4(rwlock); - ASSERT_TRUE(true); - } - { - ReadLockGuard readLockGuard(rwlock); - ASSERT_EQ(0, rwlock.TryRDLock()); - ASSERT_EQ(EBUSY, rwlock.TryWRLock()); - /* be careful */ - rwlock.Unlock(); - } - { - WriteLockGuard writeLockGuard(rwlock); - ASSERT_EQ(EBUSY, rwlock.TryRDLock()); - ASSERT_EQ(EBUSY, rwlock.TryWRLock()); - } - uint64_t writeCnt = 0; - auto writeFunc = [&] { - for (uint64_t i = 0; i < 10000; ++i) { - WriteLockGuard writeLockGuard(rwlock); - ++writeCnt; - } - }; - auto readFunc = [&] { - for (uint64_t i = 0; i < 10000; ++i) { - ReadLockGuard readLockGuard(rwlock); - auto j = writeCnt + i; - } - }; - { - std::thread t1(writeFunc); - std::thread t2(readFunc); - std::thread t3(writeFunc); - std::thread t4(readFunc); - std::thread t5(writeFunc); - std::thread t6(writeFunc); - t1.join(); - t2.join(); - t3.join(); - t4.join(); - t5.join(); - t6.join(); - - ASSERT_EQ(4 * 10000, writeCnt); - } -} - -TEST(BthreadRWLockTest, basic_test) { - BthreadRWLock rwlock; - { - ReadLockGuard readLockGuard(rwlock); - ASSERT_TRUE(true); - } - { - WriteLockGuard writeLockGuard(rwlock); - ASSERT_TRUE(true); - } - { - WriteLockGuard writeLockGuard(rwlock); - ASSERT_TRUE(true); - } - { - ReadLockGuard readLockGuard1(rwlock); - ReadLockGuard readLockGuard2(rwlock); - ASSERT_TRUE(true); - } - { - ReadLockGuard readLockGuard1(rwlock); - ReadLockGuard readLockGuard2(rwlock); - ReadLockGuard readLockGuard3(rwlock); - ReadLockGuard readLockGuard4(rwlock); - ASSERT_TRUE(true); - } - { - ReadLockGuard readLockGuard(rwlock); - ASSERT_EQ(EINVAL, rwlock.TryRDLock()); - ASSERT_EQ(EINVAL, rwlock.TryWRLock()); - /* be careful */ - rwlock.Unlock(); - } - { - WriteLockGuard writeLockGuard(rwlock); - ASSERT_EQ(EINVAL, rwlock.TryRDLock()); - ASSERT_EQ(EINVAL, rwlock.TryWRLock()); - } - uint64_t writeCnt = 0; - auto writeFunc = [&] { - for (uint64_t i = 0; i < 10000; ++i) { - WriteLockGuard writeLockGuard(rwlock); - ++writeCnt; - } - }; - auto readFunc = [&] { - for (uint64_t i = 0; i < 10000; ++i) { - ReadLockGuard readLockGuard(rwlock); - auto j = writeCnt + i; - } - }; - { - std::thread t1(writeFunc); - std::thread t2(readFunc); - std::thread t3(writeFunc); - std::thread t4(readFunc); - std::thread t5(writeFunc); - std::thread t6(writeFunc); - t1.join(); - t2.join(); - t3.join(); - t4.join(); - t5.join(); - t6.join(); - - ASSERT_EQ(4 * 10000, writeCnt); - } -} - -} // namespace common -} // namespace nebd diff --git a/nebd/test/part2/BUILD b/nebd/test/part2/BUILD index a71a13ca68..8c1bf338a9 100644 --- a/nebd/test/part2/BUILD +++ b/nebd/test/part2/BUILD @@ -33,6 +33,7 @@ cc_binary( "//external:gflags", "//nebd/src/part2:nebdserver", "//nebd/test/part2:mock_lib", + "//src/common:curve_common", "@com_google_googletest//:gtest_main", ], ) diff --git a/nebd/test/part2/metafile_manager_test.cpp b/nebd/test/part2/metafile_manager_test.cpp index 086099e29e..f133907d81 100644 --- a/nebd/test/part2/metafile_manager_test.cpp +++ b/nebd/test/part2/metafile_manager_test.cpp @@ -22,7 +22,7 @@ const char metaPath[] = "/tmp/nebd-test-metafilemanager.meta"; void FillCrc(Json::Value* root) { std::string jsonString = root->toStyledString(); - uint32_t crc = nebd::common::CRC32(jsonString.c_str(), + uint32_t crc = curve::common::CRC32(jsonString.c_str(), jsonString.size()); (*root)[kCRC] = crc; } diff --git a/nebd/test/stringstatus_test.cpp b/nebd/test/stringstatus_test.cpp deleted file mode 100644 index 27648a3c7f..0000000000 --- a/nebd/test/stringstatus_test.cpp +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Project: nebd - * Created Date: 20190819 - * Author: lixiaocui - * Copyright (c) 2019 netease - */ - -#include -#include "nebd/src/common/stringstatus.h" - -namespace nebd { -namespace common { - -TEST(Common, string_status_test) { - StringStatus status; - status.ExposeAs("test1_", "1"); - status.Update(); - ASSERT_TRUE(status.JsonBody().empty()); - - status.Set("hello", "world"); - status.Update(); - ASSERT_EQ("{\"hello\":\"world\"}", status.JsonBody()); - ASSERT_EQ("world", status.GetValueByKey("hello")); - - status.Set("code", "smart"); - status.Update(); - ASSERT_EQ("{\"code\":\"smart\",\"hello\":\"world\"}", status.JsonBody()); - ASSERT_EQ("smart", status.GetValueByKey("code")); -} - -} // namespace common -} // namespace nebd diff --git a/nebd/test/utils/config_generator.h b/nebd/test/utils/config_generator.h index 02155bf2c2..7262978b15 100644 --- a/nebd/test/utils/config_generator.h +++ b/nebd/test/utils/config_generator.h @@ -11,7 +11,7 @@ #include #include -#include "nebd/src/common/configuration.h" +#include "src/common/configuration.h" namespace nebd { namespace common { @@ -58,7 +58,7 @@ class NebdClientConfigGenerator { } std::string configPath_; - Configuration conf_; + curve::common::Configuration conf_; }; } // namespace common diff --git a/src/common/configuration.cpp b/src/common/configuration.cpp index 3fdc9a9d46..71586b66dd 100644 --- a/src/common/configuration.cpp +++ b/src/common/configuration.cpp @@ -155,6 +155,19 @@ bool Configuration::GetUInt64Value(const std::string &key, uint64_t *out) { return false; } +bool Configuration::GetInt64Value(const std::string& key, int64_t* out) { + std::string res; + if (GetValue(key, &res)) { + *out = std::stoll(res); + return true; + } + + return false; +} + +void Configuration::SetInt64Value(const std::string& key, const int64_t value) { + SetValue(key, std::to_string(value)); +} void Configuration::SetIntValue(const std::string &key, const int value) { SetValue(key, std::to_string(value)); diff --git a/src/common/configuration.h b/src/common/configuration.h index 6561b97446..46cd2419f6 100644 --- a/src/common/configuration.h +++ b/src/common/configuration.h @@ -56,7 +56,7 @@ class Configuration { int GetIntValue(const std::string &key, uint64_t defaultvalue = 0); /* - * @brief GetIntValue/GetUInt32Value/GetUInt64Value 获取指定配置项的值 //NOLINT + * @brief GetIntValue/GetUInt32Value/GetUInt64Value/GetInt64Value 获取指定配置项的值 //NOLINT * * @param[in] key 配置项名称 * @param[out] out 获取的值 @@ -68,6 +68,8 @@ class Configuration { bool GetUInt64Value(const std::string &key, uint64_t *out); void SetIntValue(const std::string &key, const int value); void SetUInt64Value(const std::string &key, const uint64_t value); + bool GetInt64Value(const std::string& key, int64_t* out); + void SetInt64Value(const std::string& key, const int64_t value); double GetDoubleValue(const std::string &key, double defaultvalue = 0.0); /* diff --git a/test/common/configuration_test.cpp b/test/common/configuration_test.cpp index de4391a029..ce9b000e84 100644 --- a/test/common/configuration_test.cpp +++ b/test/common/configuration_test.cpp @@ -247,6 +247,15 @@ TEST_F(ConfigurationTest, GetSetIntValue) { conf.SetIntValue("no.exist.u64", 3); ASSERT_TRUE(conf.GetUInt64Value("no.exist.u64", &outu64)); ASSERT_EQ(3, outu64); + + int64_t outi64; + ASSERT_FALSE(conf.GetInt64Value("no.exist.i64", &outi64)); + conf.SetInt64Value("no.exist.i64", std::numeric_limits::min()); + ASSERT_TRUE(conf.GetInt64Value("no.exist.i64", &outi64)); + ASSERT_EQ(std::numeric_limits::min(), outi64); + conf.SetInt64Value("no.exist.i64", std::numeric_limits::max()); + ASSERT_TRUE(conf.GetInt64Value("no.exist.i64", &outi64)); + ASSERT_EQ(std::numeric_limits::max(), outi64); } TEST_F(ConfigurationTest, GetSetBoolValue) {