From cac7efa4d6e4909ff2bb8cdd021a1d6e6fb91ba5 Mon Sep 17 00:00:00 2001 From: Lu Yufei Date: Tue, 28 Nov 2023 19:18:38 +0800 Subject: [PATCH] repo-sync-2023-11-28T19:05:47+0800 (#107) --- CHANGELOGS.md | 43 +++----- README.md | 95 +++++++++------- README_cn.md | 77 +++++++++++++ heu/library/algorithms/ou/decryptor.cc | 7 ++ heu/library/algorithms/ou/decryptor.h | 3 +- .../algorithms/paillier_zahlen/decryptor.cc | 7 ++ .../algorithms/paillier_zahlen/decryptor.h | 3 +- heu/library/numpy/eigen_traits.h | 8 +- heu/library/numpy/test/numpy_test.cc | 4 +- heu/library/phe/BUILD.bazel | 10 -- heu/library/phe/base/plaintext.cc | 10 +- heu/library/phe/phe.cc | 31 +++++- heu/library/phe/phe.h | 2 + heu/library/phe/phe_test/BUILD.bazel | 19 ++++ .../batch_eval_test.cc} | 72 ++---------- heu/library/phe/phe_test/ser_test.cc | 104 ++++++++++++++++++ .../numpy_binding/extension_functions.cc | 15 +-- heu/pylib/numpy_binding/py_slicer.cc | 3 +- heu/pylib/numpy_binding/slice_tool.cc | 4 +- heu/pylib/numpy_binding/slice_tool.h | 2 +- heu/pylib/phe_binding/bind_phe.cc | 11 +- heu/pylib/phe_test.py | 7 +- heu/pylib/version.py | 2 +- third_party/bazel_cpp/eigen.BUILD | 7 ++ third_party/bazel_cpp/repositories.bzl | 8 +- 25 files changed, 382 insertions(+), 172 deletions(-) create mode 100644 README_cn.md create mode 100644 heu/library/phe/phe_test/BUILD.bazel rename heu/library/phe/{phe_test.cc => phe_test/batch_eval_test.cc} (64%) create mode 100644 heu/library/phe/phe_test/ser_test.cc diff --git a/CHANGELOGS.md b/CHANGELOGS.md index b1d1483e..dd3f4cf5 100644 --- a/CHANGELOGS.md +++ b/CHANGELOGS.md @@ -15,6 +15,7 @@ - [Feature] Add ic-paillier that meets CAICT interconnection standards - [Optimize] Optimize vectorized spi in mat mul - [Add] Add test case Mul in NpBenchmarks +- [Add] New api: Create HeKit by pre-generated pk ans sk ## [0.4.4] @@ -22,10 +23,8 @@ ## [0.4.3] -- [Feature] New api: TreePredictWithIndices support prediction with non-complete - trees. -- [Feature] New api: Add range check for OU on decryption to block plaintext - overflow attack +- [Feature] New api: TreePredictWithIndices support prediction with non-complete trees. +- [Feature] New api: Add range check for OU on decryption to block plaintext overflow attack - [Bugfix] Code improve: Make tree pred with indices safer and faster. - [other] Update libtommath to head version @@ -36,27 +35,24 @@ ## [0.4.1] -- [Feature] New api: PMatrix/CMatrix add FeatureWiseBucketSum api for better - performance +- [Feature] New api: PMatrix/CMatrix add FeatureWiseBucketSum api for better performance - [Feature] New api: pylib extension add TreePredict api for performance - [Optimize] Optimize CI. ## [0.4.0] - [Feature] Add Okamoto–Uchiyama cryptosystem -- [Feature] New api: PMatrix/CMatrix add BatchSelectSum api for better - performance +- [Feature] New api: PMatrix/CMatrix add BatchSelectSum api for better performance - [Docs] Add docs to help users choose algorithms -- [Break change] Split BatchEncoder into BatchIntegerEncoder and - BatchFloatEncoder. Please see upgrade guide doc for details. +- [Break change] Split BatchEncoder into BatchIntegerEncoder and BatchFloatEncoder. Please see + upgrade guide doc for details. ## [0.3.2] - [Optimize] MPInt serialize is 81x faster and deserialize is 53x faster. - [Feature] Add benchmark for numpy api -- [Feature/experimental] Add a new PHE algorithm implementation - IPCL. IPCL has - very good performance on Intel AVX512-IFMA cpu instruction set and/or Intel - QAT accelerator +- [Feature/experimental] Add a new PHE algorithm implementation - IPCL. IPCL has very good + performance on Intel AVX512-IFMA cpu instruction set and/or Intel QAT accelerator ## [0.3.1] @@ -68,9 +64,9 @@ - HEU supports a variety of big integer arithmetic libraries now - PHE algorithms: Add vectorized SPI support - add phe.parse_schema_type() to parse string to phe.SchemaType -- [Break change] When creating an Encoder instance, you need to pass in schema - information, because different schemas may be based on completely different - integer operation libraries. Please see upgrade guide doc for details. +- [Break change] When creating an Encoder instance, you need to pass in schema information, because + different schemas may be based on completely different integer operation libraries. Please see + upgrade guide doc for details. ## [0.2.0] @@ -84,11 +80,9 @@ ## [0.1.1] -- [Break change] The encoder was divided into two types: IntegerEncoder and - FloatEncoder +- [Break change] The encoder was divided into two types: IntegerEncoder and FloatEncoder - Add two new encoder type: phe.BigintEncoder and phe.BatchEncoder -- Python lib: add numpy-like APIs, most of which have been implemented in a - parallelized way +- Python lib: add numpy-like APIs, most of which have been implemented in a parallelized way - C++ lib: add support for matrix operations ## [0.1.0] @@ -99,10 +93,8 @@ ## [0.0.6] -- phe.encryptor.encrypt_raw() and phe.decryptor.decrypt_raw() support high - precision integers -- phe.Plaintext supports conversion to and from arbitrary precision python - integers +- phe.encryptor.encrypt_raw() and phe.decryptor.decrypt_raw() support high precision integers +- phe.Plaintext supports conversion to and from arbitrary precision python integers - Improve the security of paillier ## [0.0.5] @@ -119,5 +111,4 @@ ## [0.0.3] -- Implement an efficient PHE library and provide easy-to-use C++, Python - interfaces +- Implement an efficient PHE library and provide easy-to-use C++, Python interfaces diff --git a/README.md b/README.md index 558f0676..870f8fda 100644 --- a/README.md +++ b/README.md @@ -3,64 +3,83 @@ [![CircleCI](https://dl.circleci.com/status-badge/img/gh/secretflow/heu/tree/main.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/secretflow/heu/tree/main) ![PyPI version](https://img.shields.io/pypi/v/sf-heu) -Homomorphic Encryption processing Unit (HEU) is a subproject of Secretflow that -implements high-performance homomorphic encryption algorithms. +[中文](README_cn.md) -The purpose of HEU is to lower the threshold for the use of homomorphic -encryption, so that users can use homomorphic encryption algorithms to build -privacy-preserving applications without professional cryptography knowledge. +HEU (Homomorphic Encryption processing Unit) is a user-friendly and high-performance homomorphic +encryption library that supports multiple types and scalable hardware acceleration. -## Documentation +## document https://www.secretflow.org.cn/docs/heu/ -## Background +## Repo status -This project consists of two parts: +Homomorphic encryption algorithms are mainly divided into two categories: partially homomorphic +encryption (PHE) and fully homomorphic encryption (FHE). Currently, HEU supports most PHE +algorithms, while FHE is still under development and will take some time. -HE Library (currently implemented): This project can be used as a -high-performance and complete homomorphic encryption library, which integrates -almost all homomorphic encryption algorithms in the industry. At the same time, -HEU encapsulates each algorithm and provides a uniform interface. You can switch -between different HE algorithms at any time without modifying business code. +Supported algorithms: -HE device (in work): As a component of Secretflow, HEU abstracts the homomorphic -encryption algorithms into a programmable device, making it easy for users to -flexibly build applications using the homomorphic encryption technology without -professional knowledge. HEU (device) aims to build a complete computing solution -through HE, that is, based on HE, any type of computing can be completed. -Compared with PPU, HEU's computation is purely local without any network -communication, so HEU and PPU are complementary +- Additive homomorphic encryption + - Paillier (recommended) + - Okamoto–Uchiyama (recommended) + - EC ElGamal + - Damgard-Jurik + - Damgard-Geisler-Krøigaard (DGK) +- Fully homomorphic encryption + - Under development and is the current focus of work. -Depending on the computing power, HEU has 4 working modes: +Each algorithm includes a variety of different implementations, and some implementations support +hardware accelerators. For more details, please refer to +the [Document](https://www.secretflow.org.cn/docs/heu/latest/zh-Hans/getting_started/algo_choice ) -| working mode | Supported calculation types | Number of calculations | HE algorithms | Calculating speed | Ciphertext size | -|-----------------------|-------------------------------------------|------------------------|----------------|--------------------|---------------------| -| PHEU | addition | Unlimited | Paillier, OU | Fast | Small | -| LHEU | addition, multiplication | Limited | BGV, CKKS | Fast (packed mode) | Least (packed mode) | -| FHEU (low precision) | addition, mux, LUT | Unlimited | TFHE (Torus) | Fastest | Large | -| FHEU (high precision) | addition, multiplication, comparison, mux | Unlimited | TFHE (Bitwise) | Very slow | Largest | +## Compile and install -## Repo status +### Environmental requirements + +- CPU + - x86_64: minimum required AVX instruction set + - AArch64: ARMv8 +- OS + - Ubuntu 18.04+ + - Centos 7 + - macOS 11.1+ (macOS Big Sur+) +- Python + - Python 3.8+ -PHE has been basically developed, LHE and FHE are under development +### Install via Pip -## Build & UnitTest +```shell +pip install sf-heu +``` -Build all +### Install from source +The following command will automatically compile and install HEU into the default Python +environment: ```shell -# build all -bazel build heu/... +git clone git@github.com:secretflow/heu.git +cd heu +sh build_wheel_entrypoint.sh -# test all (optional) -bazel test heu/... ``` -Build python lib +### Run unit tests (optional) + + + + ```shell -# build and install python library -sh build_wheel_entrypoint.sh +# just compile, do not run any UT (optional) +bazel build heu/... + +# compile and run all UTs +bazel test heu/... ``` + +## Contribution Guidelines + +SecretFlow is an open and inclusive community, and we welcome any kind of contribution. If you want +to improve HEU, please refer to [Contribution Guide](CONTRIBUTING.md) diff --git a/README_cn.md b/README_cn.md new file mode 100644 index 00000000..1e26dc49 --- /dev/null +++ b/README_cn.md @@ -0,0 +1,77 @@ +# HEU + +[![CircleCI](https://dl.circleci.com/status-badge/img/gh/secretflow/heu/tree/main.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/secretflow/heu/tree/main) +![PyPI version](https://img.shields.io/pypi/v/sf-heu) + +HEU（Homomorphic Encryption processing Unit）是一个低门槛、高性能的同态加密库，支持多类型、可扩展的硬件加速生态。 + +## 文档 + +https://www.secretflow.org.cn/docs/heu/ + +## 开发状态 + +同态加密算法主要分为半同态（PHE）和全同态（FHE）两大类，目前 HEU 已支持大部分 PHE 算法，而 FHE 仍需要一段时间的开发 + +支持的算法： + +- 加法同态加密 + - Paillier （推荐） + - Okamoto–Uchiyama （推荐） + - EC ElGamal + - Damgard-Jurik + - Damgard-Geisler-Krøigaard (DGK) +- 全同态加密 + - 正在开发中，并且是当前 HEU 的工作重心 + +其中每一类算法又包含多种不同的实现，部分实现支持硬件加速器，详见[文档](https://www.secretflow.org.cn/docs/heu/latest/zh-Hans/getting_started/algo_choice) + +## 编译和安装 + +### 环境要求 + +- CPU + - x86_64: 至少支持 AVX 指令集 + - AArch64: ARMv8 +- OS + - Ubuntu 18.04+ + - Centos 7 + - macOS 11.1+ (macOS Big Sur+) +- Python + - Python 3.8+ + +### 通过 Pip 安装 + +```shell +pip install sf-heu +``` + +### 从源码安装 + +以下命令将自动编译并安装 HEU 到默认 Python 环境： + +```shell +git clone git@github.com:secretflow/heu.git +cd heu +sh build_wheel_entrypoint.sh + +``` + +### 运行单元测试（可选） + + + + + +```shell +# just compile, do not run any UT (optional) +bazel build heu/... + +# compile and run all UTs +bazel test heu/... +``` + +## 贡献指南 + +隐语是一个非常包容和开放的社区，我们欢迎任何形式的贡献，如果您想要改进 +HEU，请参考[贡献指南](CONTRIBUTING.md) diff --git a/heu/library/algorithms/ou/decryptor.cc b/heu/library/algorithms/ou/decryptor.cc index f0546845..e89f17df 100644 --- a/heu/library/algorithms/ou/decryptor.cc +++ b/heu/library/algorithms/ou/decryptor.cc @@ -22,6 +22,13 @@ namespace heu::lib::algorithms::ou { HE_ASSERT(!(ct).c_.IsNegative() && (ct).c_ < pk_.n_, \ "Decryptor: Invalid ciphertext") +Decryptor::Decryptor(PublicKey pk, SecretKey sk) + : pk_(std::move(pk)), sk_(std::move(sk)) { + YACL_ENFORCE(sk_.p2_ * sk_.q_ == pk_.n_, + "pk and sk are not paired, {}^2 * {} != {}", sk_.p_, sk_.q_, + pk_.n_); +} + void Decryptor::Decrypt(const Ciphertext& ct, MPInt* out) const { VALIDATE(ct); diff --git a/heu/library/algorithms/ou/decryptor.h b/heu/library/algorithms/ou/decryptor.h index 8c1c519f..836f45f2 100644 --- a/heu/library/algorithms/ou/decryptor.h +++ b/heu/library/algorithms/ou/decryptor.h @@ -24,8 +24,7 @@ namespace heu::lib::algorithms::ou { class Decryptor { public: - explicit Decryptor(PublicKey pk, SecretKey sk) - : pk_(std::move(pk)), sk_(std::move(sk)) {} + explicit Decryptor(PublicKey pk, SecretKey sk); void Decrypt(const Ciphertext& ct, MPInt* out) const; [[nodiscard]] MPInt Decrypt(const Ciphertext& ct) const; diff --git a/heu/library/algorithms/paillier_zahlen/decryptor.cc b/heu/library/algorithms/paillier_zahlen/decryptor.cc index cec4fb32..6ec80cb3 100644 --- a/heu/library/algorithms/paillier_zahlen/decryptor.cc +++ b/heu/library/algorithms/paillier_zahlen/decryptor.cc @@ -22,6 +22,13 @@ namespace heu::lib::algorithms::paillier_z { HE_ASSERT(!(ct).c_.IsNegative() && (ct).c_ < pk_.n_square_, \ "Decryptor: Invalid ciphertext") +Decryptor::Decryptor(PublicKey pk, SecretKey sk) + : pk_(std::move(pk)), sk_(std::move(sk)) { + YACL_ENFORCE(sk_.p_ * sk_.q_ == pk_.n_, + "pk and sk are not paired, {} * {} != {}", sk_.p_, sk_.q_, + pk_.n_); +} + void Decryptor::Decrypt(const Ciphertext& ct, MPInt* out) const { VALIDATE(ct); diff --git a/heu/library/algorithms/paillier_zahlen/decryptor.h b/heu/library/algorithms/paillier_zahlen/decryptor.h index 0e8dd67a..f2111e87 100644 --- a/heu/library/algorithms/paillier_zahlen/decryptor.h +++ b/heu/library/algorithms/paillier_zahlen/decryptor.h @@ -24,8 +24,7 @@ namespace heu::lib::algorithms::paillier_z { class Decryptor { public: - explicit Decryptor(PublicKey pk, SecretKey sk) - : pk_(std::move(pk)), sk_(std::move(sk)) {} + explicit Decryptor(PublicKey pk, SecretKey sk); void Decrypt(const Ciphertext& ct, MPInt* out) const; MPInt Decrypt(const Ciphertext& ct) const; diff --git a/heu/library/numpy/eigen_traits.h b/heu/library/numpy/eigen_traits.h index bfb9ae23..a6e0e251 100644 --- a/heu/library/numpy/eigen_traits.h +++ b/heu/library/numpy/eigen_traits.h @@ -34,6 +34,8 @@ struct NumTraits : GenericNumTraits<Plaintext> { // static inline Real epsilon() { return Real(0); } // static inline Real dummy_precision() { return Real(0); } + static inline int digits() { return 0; } + static inline int digits10() { return 0; } enum { @@ -53,10 +55,12 @@ struct NumTraits<Ciphertext> : GenericNumTraits<Ciphertext> { typedef Ciphertext NonInteger; typedef Ciphertext Nested; - static inline int digits10() { return 0; } + static inline int digits() { return 32; } + + static inline int digits10() { return 32; } enum { - IsInteger = 0, + IsInteger = 1, // 0 means floating point IsSigned = 0, IsComplex = 0, RequireInitialization = 1, diff --git a/heu/library/numpy/test/numpy_test.cc b/heu/library/numpy/test/numpy_test.cc index a6e3b751..ae1e0ac9 100644 --- a/heu/library/numpy/test/numpy_test.cc +++ b/heu/library/numpy/test/numpy_test.cc @@ -175,8 +175,8 @@ TEST_F(NumpyTest, BinSumWorks) { {0, 0, 2, 2, 3, 3, 0, 3, 2, 2}}); auto bucket_num = 5; auto sum = he_kit_.GetEvaluator()->FeatureWiseBucketSum( - m.GetItem(subgroup_indices, Eigen::all), - order_map(subgroup_indices, Eigen::all), bucket_num)(2, 0); + m.GetItem(subgroup_indices, Eigen::placeholders::all), + order_map(subgroup_indices, Eigen::placeholders::all), bucket_num)(2, 0); EXPECT_EQ(sum.GetValue<int64_t>(), 4); } diff --git a/heu/library/phe/BUILD.bazel b/heu/library/phe/BUILD.bazel index 140b7d75..38bd6537 100644 --- a/heu/library/phe/BUILD.bazel +++ b/heu/library/phe/BUILD.bazel @@ -41,10 +41,6 @@ yacl_cc_library( ], ) -test_suite( - name = "phe_tests", -) - yacl_cc_test( name = "encryptor_test", srcs = ["encryptor_test.cc"], @@ -62,9 +58,3 @@ yacl_cc_test( srcs = ["evaluator_test.cc"], deps = [":phe"], ) - -yacl_cc_test( - name = "phe_test", - srcs = ["phe_test.cc"], - deps = [":phe"], -) diff --git a/heu/library/phe/base/plaintext.cc b/heu/library/phe/base/plaintext.cc index e0104bba..202691ba 100644 --- a/heu/library/phe/base/plaintext.cc +++ b/heu/library/phe/base/plaintext.cc @@ -50,13 +50,13 @@ void Plaintext::ToBytes(unsigned char* buf, size_t buf_len, } #define OPERATOR_RET_IMPL(op, opb) \ - Plaintext Plaintext::operator op(const Plaintext& operand2) const { \ + Plaintext Plaintext::operator op(const Plaintext & operand2) const { \ return Visit([&](const auto& pt) -> Plaintext { \ FOR_EACH_TYPE(pt) return Plaintext(pt op operand2.AsTypeLike(pt)); \ }); \ } \ \ - Plaintext Plaintext::operator opb(const Plaintext& operand2) { \ + Plaintext Plaintext::operator opb(const Plaintext & operand2) { \ Visit([&](auto& pt) { \ FOR_EACH_TYPE(pt) pt opb operand2.AsTypeLike(pt); \ }); \ @@ -104,9 +104,9 @@ Plaintext Plaintext::operator>>=(size_t operand2) { return *this; } -#define OPERATOR_BOOL_IMPL(op) \ - bool Plaintext::operator op(const Plaintext& other) const { \ - return var_ op other.var_; \ +#define OPERATOR_BOOL_IMPL(op) \ + bool Plaintext::operator op(const Plaintext & other) const { \ + return var_ op other.var_; \ } OPERATOR_BOOL_IMPL(>) diff --git a/heu/library/phe/phe.cc b/heu/library/phe/phe.cc index 5402be46..ee252ade 100644 --- a/heu/library/phe/phe.cc +++ b/heu/library/phe/phe.cc @@ -28,7 +28,8 @@ void HeKitPublicBase::Setup(std::shared_ptr<PublicKey> pk) { ++hit; } } - YACL_ENFORCE("Cannot detect the schema type of public key {}, hit={}", + YACL_ENFORCE(hit == 1, + "Cannot detect the schema type of public key {}, hit={}", public_key_->ToString(), hit); } @@ -36,6 +37,10 @@ void HeKitSecretBase::Setup(std::shared_ptr<PublicKey> pk, std::shared_ptr<SecretKey> sk) { HeKitPublicBase::Setup(std::move(pk)); secret_key_ = std::move(sk); + YACL_ENFORCE(secret_key_->IsCompatible(schema_type_), + "The public key and secret key do not belong to the same " + "algorithm, pk={}", + schema_type_); } #define GEN_KEY_AND_INIT(ns) \ @@ -84,6 +89,30 @@ HeKit::HeKit(SchemaType schema_type) { std::make_shared<Encryptor>(schema_type_, ns::Encryptor(pk1)); \ } +#define HE_SPECIAL_SETUP_BY_SK(ns) \ + [&](const ns::SecretKey& sk1) { \ + decryptor_ = std::make_shared<Decryptor>( \ + schema_type_, ns::Decryptor(public_key_->As<ns::PublicKey>(), sk1)); \ + } + +HeKit::HeKit(std::shared_ptr<PublicKey> pk, std::shared_ptr<SecretKey> sk) { + Setup(std::move(pk), std::move(sk)); + public_key_->Visit(HE_DISPATCH(HE_SPECIAL_SETUP_BY_PK)); + secret_key_->Visit(HE_DISPATCH(HE_SPECIAL_SETUP_BY_SK)); +} + +HeKit::HeKit(yacl::ByteContainerView pk_buffer, + yacl::ByteContainerView sk_buffer) { + auto pk = std::make_shared<PublicKey>(); + pk->Deserialize(pk_buffer); + auto sk = std::make_shared<SecretKey>(); + sk->Deserialize(sk_buffer); + + Setup(std::move(pk), std::move(sk)); + public_key_->Visit(HE_DISPATCH(HE_SPECIAL_SETUP_BY_PK)); + secret_key_->Visit(HE_DISPATCH(HE_SPECIAL_SETUP_BY_SK)); +} + DestinationHeKit::DestinationHeKit(std::shared_ptr<PublicKey> pk) { Setup(std::move(pk)); public_key_->Visit(HE_DISPATCH(HE_SPECIAL_SETUP_BY_PK)); diff --git a/heu/library/phe/phe.h b/heu/library/phe/phe.h index 2d37fdd1..f5db42a7 100644 --- a/heu/library/phe/phe.h +++ b/heu/library/phe/phe.h @@ -61,6 +61,8 @@ class HeKit : public HeKitSecretBase { public: HeKit(SchemaType schema_type, size_t key_size); explicit HeKit(SchemaType schema_type); + HeKit(std::shared_ptr<PublicKey> pk, std::shared_ptr<SecretKey> sk); + HeKit(yacl::ByteContainerView pk_buffer, yacl::ByteContainerView sk_buffer); [[nodiscard]] const std::shared_ptr<Encryptor>& GetEncryptor() const { return encryptor_; diff --git a/heu/library/phe/phe_test/BUILD.bazel b/heu/library/phe/phe_test/BUILD.bazel new file mode 100644 index 00000000..88d195f3 --- /dev/null +++ b/heu/library/phe/phe_test/BUILD.bazel @@ -0,0 +1,19 @@ +load("@yacl//bazel:yacl.bzl", "yacl_cc_library", "yacl_cc_test") + +package(default_visibility = ["//visibility:public"]) + +test_suite( + name = "phe_tests", +) + +yacl_cc_test( + name = "batch_eval_test", + srcs = ["batch_eval_test.cc"], + deps = ["//heu/library/phe"], +) + +yacl_cc_test( + name = "ser_test", + srcs = ["ser_test.cc"], + deps = ["//heu/library/phe"], +) diff --git a/heu/library/phe/phe_test.cc b/heu/library/phe/phe_test/batch_eval_test.cc similarity index 64% rename from heu/library/phe/phe_test.cc rename to heu/library/phe/phe_test/batch_eval_test.cc index 757d76f3..17586983 100644 --- a/heu/library/phe/phe_test.cc +++ b/heu/library/phe/phe_test/batch_eval_test.cc @@ -1,4 +1,4 @@ -// Copyright 2022 Ant Group Co., Ltd. +// Copyright 2023 Ant Group Co., Ltd. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,16 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "heu/library/phe/phe.h" - #include "fmt/format.h" #include "gtest/gtest.h" #include "heu/library/phe/encoding/encoding.h" +#include "heu/library/phe/phe.h" namespace heu::lib::phe::test { -class PheTest : public ::testing::TestWithParam<SchemaType> { +class BatchEvalTest : public ::testing::TestWithParam<SchemaType> { protected: void SetUp() override { auto sk_str = he_kit_.GetSecretKey()->ToString(); @@ -32,70 +31,13 @@ class PheTest : public ::testing::TestWithParam<SchemaType> { protected: HeKit he_kit_ = HeKit(GetParam()); - PlainEncoder edr = he_kit_.GetEncoder<PlainEncoder>(1); }; -INSTANTIATE_TEST_SUITE_P(Schema, PheTest, ::testing::ValuesIn(GetAllSchema())); - -TEST_P(PheTest, KeySerialize) { - // test pk - auto buffer_pk = he_kit_.GetPublicKey()->Serialize(); - EXPECT_NE(*he_kit_.GetPublicKey(), PublicKey()); - EXPECT_NE(*he_kit_.GetPublicKey(), PublicKey(he_kit_.GetSchemaType())); - PublicKey pk; - pk.Deserialize(buffer_pk); - EXPECT_EQ(*he_kit_.GetPublicKey(), pk); - - // test sk - auto buffer_sk = he_kit_.GetSecretKey()->Serialize(); - EXPECT_NE(*he_kit_.GetSecretKey(), SecretKey()); - EXPECT_NE(*he_kit_.GetSecretKey(), SecretKey(he_kit_.GetSchemaType())); - SecretKey sk; - sk.Deserialize(buffer_sk); - EXPECT_EQ(*he_kit_.GetSecretKey(), sk); -} - -TEST_P(PheTest, VarSerialize) { - // test serialize plaintext - auto clear = he_kit_.GetSchemaType() == SchemaType::DGK ? -9632 : -963258741; - auto plain = Plaintext(he_kit_.GetSchemaType(), clear); - EXPECT_NE(plain, Plaintext()); - EXPECT_NE(plain, Plaintext(he_kit_.GetSchemaType())); - EXPECT_NE(plain, Plaintext(he_kit_.GetSchemaType(), -clear)); - EXPECT_EQ(plain, Plaintext(he_kit_.GetSchemaType(), clear)); - auto buffer = plain.Serialize(); - Plaintext pt2; - pt2.Deserialize(buffer); - EXPECT_EQ(plain, pt2); - - // test serialize ciphertext - auto ct0 = he_kit_.GetEncryptor()->Encrypt(plain); - EXPECT_GE(ct0.ToString().length(), 10) << ct0.ToString(); - EXPECT_NE(ct0, Ciphertext()); - EXPECT_NE(ct0, Ciphertext(he_kit_.GetSchemaType())); - buffer = ct0.Serialize(); - EXPECT_GT(buffer.size(), sizeof(size_t)) << buffer; - - Ciphertext ct1; - ct1.Deserialize(buffer); - EXPECT_EQ(ct0, ct1); - EXPECT_EQ(he_kit_.GetDecryptor()->Decrypt(ct1), plain); - - // test serialize public key - auto buffer_pk = he_kit_.GetPublicKey()->Serialize(); - DestinationHeKit server(buffer_pk); - server.GetEvaluator()->AddInplace(&ct1, edr.Encode(666)); - server.GetEvaluator()->Randomize(&ct1); - buffer = ct1.Serialize(); - - // send back to client - Ciphertext ct2; - ct2.Deserialize(buffer); - EXPECT_EQ(he_kit_.GetDecryptor()->Decrypt(ct1), plain + edr.Encode(666)); -} +INSTANTIATE_TEST_SUITE_P(Schema, BatchEvalTest, + ::testing::ValuesIn(GetAllSchema())); // test batch encoding -TEST_P(PheTest, BatchEncoding) { +TEST_P(BatchEvalTest, BatchEncoding) { if (GetParam() == SchemaType::ElGamal) { GTEST_SKIP() << "Plaintext range is not enough, Skip ElGamal"; } @@ -148,7 +90,7 @@ TEST_P(PheTest, BatchEncoding) { std::numeric_limits<int64_t>::lowest()); } -TEST_P(PheTest, BatchAdd) { +TEST_P(BatchEvalTest, BatchAdd) { if (GetParam() == SchemaType::ElGamal) { GTEST_SKIP() << "Plaintext range is not enough, Skip ElGamal"; } diff --git a/heu/library/phe/phe_test/ser_test.cc b/heu/library/phe/phe_test/ser_test.cc new file mode 100644 index 00000000..20e64995 --- /dev/null +++ b/heu/library/phe/phe_test/ser_test.cc @@ -0,0 +1,104 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fmt/format.h" +#include "gtest/gtest.h" + +#include "heu/library/phe/encoding/encoding.h" +#include "heu/library/phe/phe.h" + +namespace heu::lib::phe::test { + +class SerTest : public ::testing::TestWithParam<SchemaType> { + protected: + HeKit he_kit_ = HeKit(GetParam()); + PlainEncoder edr_ = he_kit_.GetEncoder<PlainEncoder>(1); +}; + +INSTANTIATE_TEST_SUITE_P(Schema, SerTest, ::testing::ValuesIn(GetAllSchema())); + +TEST_P(SerTest, KeySerialize) { + // make sure key has content + auto sk_str = he_kit_.GetSecretKey()->ToString(); + EXPECT_GT(sk_str.length(), 30) << sk_str; + auto pk_str = he_kit_.GetPublicKey()->ToString(); + EXPECT_GT(pk_str.length(), 30) << pk_str; + + // test pk + auto buffer_pk = he_kit_.GetPublicKey()->Serialize(); + EXPECT_NE(*he_kit_.GetPublicKey(), PublicKey()); + EXPECT_NE(*he_kit_.GetPublicKey(), PublicKey(he_kit_.GetSchemaType())); + PublicKey pk; + pk.Deserialize(buffer_pk); + EXPECT_EQ(*he_kit_.GetPublicKey(), pk); + + // test sk + auto buffer_sk = he_kit_.GetSecretKey()->Serialize(); + EXPECT_NE(*he_kit_.GetSecretKey(), SecretKey()); + EXPECT_NE(*he_kit_.GetSecretKey(), SecretKey(he_kit_.GetSchemaType())); + SecretKey sk; + sk.Deserialize(buffer_sk); + EXPECT_EQ(*he_kit_.GetSecretKey(), sk); + + // recover HeKit from pk/sk + auto kit2 = HeKit(buffer_pk, buffer_sk); + auto kit3 = HeKit(he_kit_.GetPublicKey(), he_kit_.GetSecretKey()); + ASSERT_EQ(*kit2.GetPublicKey(), *kit3.GetPublicKey()); + ASSERT_EQ(*kit2.GetSecretKey(), *kit3.GetSecretKey()); + auto pt = edr_.Encode(4321); + auto ct = he_kit_.GetEncryptor()->Encrypt(pt); + EXPECT_EQ(kit2.GetDecryptor()->Decrypt(ct), pt); + EXPECT_EQ(kit3.GetDecryptor()->Decrypt(ct), pt); +} + +TEST_P(SerTest, VarSerialize) { + // test serialize plaintext + auto clear = he_kit_.GetSchemaType() == SchemaType::DGK ? -9632 : -963258741; + auto plain = Plaintext(he_kit_.GetSchemaType(), clear); + EXPECT_NE(plain, Plaintext()); + EXPECT_NE(plain, Plaintext(he_kit_.GetSchemaType())); + EXPECT_NE(plain, Plaintext(he_kit_.GetSchemaType(), -clear)); + EXPECT_EQ(plain, Plaintext(he_kit_.GetSchemaType(), clear)); + auto buffer = plain.Serialize(); + Plaintext pt2; + pt2.Deserialize(buffer); + EXPECT_EQ(plain, pt2); + + // test serialize ciphertext + auto ct0 = he_kit_.GetEncryptor()->Encrypt(plain); + EXPECT_GE(ct0.ToString().length(), 10) << ct0.ToString(); + EXPECT_NE(ct0, Ciphertext()); + EXPECT_NE(ct0, Ciphertext(he_kit_.GetSchemaType())); + buffer = ct0.Serialize(); + EXPECT_GT(buffer.size(), sizeof(size_t)) << buffer; + + Ciphertext ct1; + ct1.Deserialize(buffer); + EXPECT_EQ(ct0, ct1); + EXPECT_EQ(he_kit_.GetDecryptor()->Decrypt(ct1), plain); + + // test serialize public key + auto buffer_pk = he_kit_.GetPublicKey()->Serialize(); + DestinationHeKit server(buffer_pk); + server.GetEvaluator()->AddInplace(&ct1, edr_.Encode(666)); + server.GetEvaluator()->Randomize(&ct1); + buffer = ct1.Serialize(); + + // send back to client + Ciphertext ct2; + ct2.Deserialize(buffer); + EXPECT_EQ(he_kit_.GetDecryptor()->Decrypt(ct1), plain + edr_.Encode(666)); +} + +} // namespace heu::lib::phe::test diff --git a/heu/pylib/numpy_binding/extension_functions.cc b/heu/pylib/numpy_binding/extension_functions.cc index ad27b179..01d67fac 100644 --- a/heu/pylib/numpy_binding/extension_functions.cc +++ b/heu/pylib/numpy_binding/extension_functions.cc @@ -55,7 +55,7 @@ T ExtensionFunctions<T>::SelectSum(const hnp::Evaluator& evaluator, // key dimension is less than tensor dimension bool sq_row; auto s_row = slice_tool::Parse(key, p_matrix.rows(), &sq_row); - return evaluator.SelectSum(p_matrix, s_row.indices, Eigen::all); + return evaluator.SelectSum(p_matrix, s_row.indices, Eigen::placeholders::all); } namespace { @@ -99,9 +99,10 @@ lib::numpy::DenseMatrix<T> ExtensionFunctions<T>::FeatureWiseBucketSum( subgroup_indices.push_back(j); } } - e.FeatureWiseBucketSumInplace(x.GetItem(subgroup_indices, Eigen::all), - order_map(subgroup_indices, Eigen::all), - bucket_num, res, cumsum); + e.FeatureWiseBucketSumInplace( + x.GetItem(subgroup_indices, Eigen::placeholders::all), + order_map(subgroup_indices, Eigen::placeholders::all), bucket_num, res, + cumsum); return res; } @@ -133,9 +134,9 @@ ExtensionFunctions<T>::BatchFeatureWiseBucketSum( ++subgroup_index) { if (subgroup_indices[subgroup_index].size() > 0) { e.FeatureWiseBucketSumInplace( - x.GetItem(subgroup_indices[subgroup_index], Eigen::all), - order_map(subgroup_indices[subgroup_index], Eigen::all), bucket_num, - res[subgroup_index], cumsum); + x.GetItem(subgroup_indices[subgroup_index], Eigen::placeholders::all), + order_map(subgroup_indices[subgroup_index], Eigen::placeholders::all), + bucket_num, res[subgroup_index], cumsum); } else { auto buf = res[subgroup_index].data(); yacl::parallel_for(0, total_bucket_num * x.cols(), 1, diff --git a/heu/pylib/numpy_binding/py_slicer.cc b/heu/pylib/numpy_binding/py_slicer.cc index 1ea97835..35fdb875 100644 --- a/heu/pylib/numpy_binding/py_slicer.cc +++ b/heu/pylib/numpy_binding/py_slicer.cc @@ -85,7 +85,8 @@ py::object PySlicer<T>::GetItem(const hnp::DenseMatrix<T>& p_matrix, // key dimension is less than tensor dimension bool sq_row; auto s_row = slice_tool::Parse(key, p_matrix.rows(), &sq_row); - return CastMatrix(p_matrix.GetItem(s_row.indices, Eigen::all, sq_row)); + return CastMatrix( + p_matrix.GetItem(s_row.indices, Eigen::placeholders::all, sq_row)); } template <typename T> diff --git a/heu/pylib/numpy_binding/slice_tool.cc b/heu/pylib/numpy_binding/slice_tool.cc index a1725434..9317ad38 100644 --- a/heu/pylib/numpy_binding/slice_tool.cc +++ b/heu/pylib/numpy_binding/slice_tool.cc @@ -85,8 +85,8 @@ PySlice<std::vector<int64_t>> Parse(const pybind11::object& src, return res; } -auto All(ssize_t dim_len) -> PySlice<decltype(Eigen::all)> { - return {dim_len, Eigen::all}; +auto All(ssize_t dim_len) -> PySlice<decltype(Eigen::placeholders::all)> { + return {dim_len, Eigen::placeholders::all}; }; } // namespace slice_tool diff --git a/heu/pylib/numpy_binding/slice_tool.h b/heu/pylib/numpy_binding/slice_tool.h index 458b54a8..af3d26fe 100644 --- a/heu/pylib/numpy_binding/slice_tool.h +++ b/heu/pylib/numpy_binding/slice_tool.h @@ -39,7 +39,7 @@ PySlice<std::vector<int64_t>> Parse(const pybind11::object& src, bool* should_squeeze = nullptr); // express python slice [:] in PySlice struct. -auto All(ssize_t dim_len) -> PySlice<decltype(Eigen::all)>; +auto All(ssize_t dim_len) -> PySlice<decltype(Eigen::placeholders::all)>; } // namespace slice_tool diff --git a/heu/pylib/phe_binding/bind_phe.cc b/heu/pylib/phe_binding/bind_phe.cc index 58c967bd..eb42ebea 100644 --- a/heu/pylib/phe_binding/bind_phe.cc +++ b/heu/pylib/phe_binding/bind_phe.cc @@ -235,6 +235,14 @@ void PyBindPhe(pybind11::module& m) { py::arg("schema_string") = "z-paillier", py::return_value_policy::move, "Setup phe environment by schema string"); + m.def( + "setup", + [](std::shared_ptr<phe::PublicKey> pk, std::shared_ptr<phe::SecretKey> sk) + -> phe::HeKit { return phe::HeKit(std::move(pk), std::move(sk)); }, + py::arg("public_key"), py::arg("secret_key"), + py::return_value_policy::move, + "Setup phe environment by pre-generated pk and sk"); + // api for evaluator party py::class_<phe::DestinationHeKit, phe::HeKitPublicBase>(m, "DestinationHeKit") .def("encryptor", &phe::DestinationHeKit::GetEncryptor, "Get encryptor") @@ -243,8 +251,7 @@ void PyBindPhe(pybind11::module& m) { m.def( "setup", [](std::shared_ptr<phe::PublicKey> pk) { - phe::DestinationHeKit ahe(std::move(pk)); - return ahe; + return phe::DestinationHeKit(std::move(pk)); }, py::arg("public_key"), py::return_value_policy::move, "Setup phe environment by an already generated public key"); diff --git a/heu/pylib/phe_test.py b/heu/pylib/phe_test.py index 439ce088..6804e8a5 100644 --- a/heu/pylib/phe_test.py +++ b/heu/pylib/phe_test.py @@ -12,8 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest import random +import unittest + from heu import phe @@ -263,8 +264,10 @@ def test_pickle(self): ct3_buffer = pickle.dumps(ct3) # client: decrypt + sk_buffer = pickle.dumps(client_he.secret_key()) + client_he2 = phe.setup(pickle.loads(pk_buffer), pickle.loads(sk_buffer)) ct_x = pickle.loads(ct3_buffer) - self.assertEqual(client_he.decryptor().decrypt_raw(ct_x), 123 - 456) + self.assertEqual(client_he2.decryptor().decrypt_raw(ct_x), 123 - 456) if __name__ == "__main__": diff --git a/heu/pylib/version.py b/heu/pylib/version.py index c88037de..1adff92e 100644 --- a/heu/pylib/version.py +++ b/heu/pylib/version.py @@ -13,4 +13,4 @@ # limitations under the License. -__version__ = "0.5.0.dev20231118" +__version__ = "0.5.0.dev20231128" diff --git a/third_party/bazel_cpp/eigen.BUILD b/third_party/bazel_cpp/eigen.BUILD index b345de79..2791f38c 100644 --- a/third_party/bazel_cpp/eigen.BUILD +++ b/third_party/bazel_cpp/eigen.BUILD @@ -43,6 +43,13 @@ EIGEN_MPL2_HEADER_FILES = glob( cc_library( name = "eigen3", hdrs = EIGEN_MPL2_HEADER_FILES, + # If your program crashes, you can try to add this flag globally + copts = select({ + "@platforms//cpu:x86_64": [ + "-mavx", + ], + "//conditions:default": [], + }), defines = [ # This define (mostly) guarantees we don't link any problematic # code. We use it, but we do not rely on it, as evidenced above. diff --git a/third_party/bazel_cpp/repositories.bzl b/third_party/bazel_cpp/repositories.bzl index b50df525..a212e922 100644 --- a/third_party/bazel_cpp/repositories.bzl +++ b/third_party/bazel_cpp/repositories.bzl @@ -9,14 +9,16 @@ def heu_cpp_deps(): _com_github_nvlabs_cgbn() def _com_github_eigenteam_eigen(): + EIGEN_COMMIT = "66e8f38891841bf88ee976a316c0c78a52f0cee5" + EIGEN_SHA256 = "01fcd68409c038bbcfd16394274c2bf71e2bb6dda89a2319e23fc59a2da17210" maybe( http_archive, name = "com_github_eigenteam_eigen", - sha256 = "606c404bdcbdb3782b4df3e9e32d76e1d940aa6704af5be0e75f6249bc9a6730", + sha256 = EIGEN_SHA256, build_file = "@com_alipay_sf_heu//third_party/bazel_cpp:eigen.BUILD", - strip_prefix = "eigen-3.4", + strip_prefix = "eigen-{commit}".format(commit = EIGEN_COMMIT), urls = [ - "https://gitlab.com/libeigen/eigen/-/archive/3.4/eigen-3.4.tar.gz", + "https://gitlab.com/libeigen/eigen/-/archive/{commit}/eigen-{commit}.tar.gz".format(commit = EIGEN_COMMIT), ], )