Reworked proof of work #288

Default parameter for grinding is bits, not mask #288

Replaced masks with bits #288

Simplified multiexp test to make CI runs faster #288

merge-in-progress

wip

Added missing includes for boost::hash_combine #288

Moved multiexp to benchmarks #288

Renamed int_be to to_bytes_array #288

Author: vo-nil

libs/algebra/include/nil/crypto3/algebra/fields/detail/element/fp2.hpp

@@ -27,6 +27,7 @@
 #define CRYPTO3_ALGEBRA_FIELDS_ELEMENT_FP2_HPP
 
 #include <type_traits>
+#include <boost/functional/hash.hpp>
 
 #include <nil/crypto3/algebra/fields/detail/exponentiation.hpp>
 #include <nil/crypto3/algebra/fields/detail/element/operations.hpp>

libs/algebra/test/CMakeLists.txt

@@ -56,7 +56,6 @@ set(RUNTIME_TESTS_NAMES
         "fields"
         "fields_static"
         "pairing"
-        "multiexp"
 )
 
 set(COMPILE_TIME_TESTS_NAMES

libs/algebra/test/bench_test/CMakeLists.txt

@@ -37,6 +37,7 @@ endmacro()
 set(RUNTIME_TESTS_NAMES
     "bench_curves"
     "bench_fields"
+    "bench_multiexp"
     )
 
 foreach(TEST_NAME ${RUNTIME_TESTS_NAMES})

libs/algebra/test/multiexp.cpp libs/algebra/test/bench_test/bench_multiexp.cpp

@@ -35,6 +35,8 @@
 #include <iterator>
 #include <unordered_map>
 
+#include <boost/functional/hash.hpp>
+
 #include <nil/crypto3/math/algorithms/make_evaluation_domain.hpp>
 #include <nil/crypto3/math/domains/evaluation_domain.hpp>
 #include <nil/crypto3/math/polynomial/basic_operations.hpp>

libs/math/include/nil/crypto3/math/polynomial/polynomial_dfs.hpp

@@ -13,6 +13,7 @@
 #include <boost/multiprecision/detail/constexpr.hpp>
 #include <boost/multiprecision/detail/bitscan.hpp>    // lsb etc
 #include <boost/functional/hash_fwd.hpp>
+#include <boost/functional/hash.hpp>
 
 #ifdef BOOST_MSVC
 #pragma warning(push)

libs/multiprecision/include/nil/crypto3/multiprecision/cpp_int_modular/misc.hpp

@@ -72,6 +72,25 @@ namespace nil {
                     std::map<std::size_t, bool> _locked; // _locked[batch] is true after it is commited
                     std::map<std::size_t, std::vector<std::vector<typename field_type::value_type>>> _points;
 
+                    // We frequently search over the this->_points structure, and it's better to keep a hashmap that maps point to
+                    // it's index in vector for faster search. We need to duplicate this data for now, because the order of points matters.
+                    std::map<std::size_t, std::vector<std::unordered_map<typename field_type::value_type, std::size_t>>> _points_map;
+
+                    // Creates '_points_map'. We need to think about re-designing this class later. Currently this is used from LPC.
+                    void build_points_map() {
+                        for (const auto& [i, V]: this->_points) {
+                            _points_map[i].resize(V.size());
+                            for (std::size_t j = 0; j < V.size(); ++j) {
+                                const auto& batch = V[j];
+                                for (std::size_t k = 0; k < batch.size(); ++k) {
+                                    // We need to store the index of the first occurance of each point.
+                                    if (_points_map[i][j].find(batch[k]) == _points_map[i][j].end())
+                                        _points_map[i][j][batch[k]] = k;
+                                }
+                            }
+                        }
+                    }
+
                 protected:
                     math::polynomial<typename field_type::value_type> get_V(
                         const std::vector<typename field_type::value_type> &points) const {
@@ -108,16 +127,18 @@ namespace nil {
                     }
 
                     // We call them singles in recursive verifier
+                    // We keep the order of points, not sure if that was required.
                     std::vector<typename field_type::value_type> get_unique_points() const {
+
                         std::vector<typename field_type::value_type> result;
-//                        std::unordered_set<typename field_type::value_type> result_set;
+                        std::unordered_set<typename field_type::value_type> result_set;
 
-                        for( auto const &[k, point_batch]:_points ){
+                        for( auto const &[k, point_batch]: _points ){
                             for( auto const &point_set: point_batch ){
-                                for( auto const &point:point_set ){
-                                    if( std::find(result.begin(), result.end(), point) == result.end() ){
+                                for( auto const &point: point_set ){
+                                    if (result_set.find(point) == result_set.end()) {
                                         result.push_back(point);
-  //                                      result_set.insert(point);
+                                        result_set.insert(point);
                                     }
                                 }
                             }
@@ -180,7 +201,7 @@ namespace nil {
 
                             BOOST_ASSERT(poly.size() == point.size() || point.size() == 1);
 
-                            for (std::size_t i = 0; i < poly.size(); i++) {
+                            for (std::size_t i = 0; i < poly.size(); ++i) {
                                 _z.set_poly_points_number(k, i, point[i].size());
                                 for (std::size_t j = 0; j < point[i].size(); j++) {
                                     _z.set(k, i, j, poly[i].evaluate(point[i][j]));

libs/zk/include/nil/crypto3/zk/commitments/batched_commitment.hpp

@@ -154,7 +154,7 @@ namespace nil {
                                 std::size_t lambda,
                                 std::size_t expand_factor,
                                 bool use_grinding = false,
-                                std::size_t grinding_parameter = 0xFFFF
+                                std::size_t grinding_parameter = 16
                             ): lambda(lambda)
                               , use_grinding(use_grinding)
                               , grinding_parameter(grinding_parameter)
@@ -171,7 +171,7 @@ namespace nil {
                                 std::size_t lambda,
                                 std::size_t expand_factor,
                                 bool use_grinding = false,
-                                std::size_t grinding_parameter = 0xFFFF
+                                std::size_t grinding_parameter = 16
                             ): lambda(lambda)
                               , use_grinding(use_grinding)
                               , grinding_parameter(grinding_parameter)
@@ -189,7 +189,7 @@ namespace nil {
                                 std::size_t expand_factor,
                                 std::size_t lambda,
                                 bool use_grinding = false,
-                                std::size_t grinding_parameter = 0xFFFF
+                                std::size_t grinding_parameter = 16
                             ) : lambda(lambda)
                               , use_grinding(use_grinding)
                               , grinding_parameter(grinding_parameter)

libs/zk/include/nil/crypto3/zk/commitments/detail/polynomial/basic_fri.hpp

@@ -1,5 +1,6 @@
 //---------------------------------------------------------------------------//
 // Copyright (c) 2023 Elena Tatuzova <[email protected]>
+// Copyright (c) 2024 Vasiliy Olekhov <[email protected]>
 //
 // MIT License
 //
@@ -44,37 +45,40 @@ namespace nil {
                     using transcript_type = transcript::fiat_shamir_heuristic_sequential<transcript_hash_type>;
                     using output_type = OutType;
 
-                    static inline OutType generate(transcript_type &transcript, OutType mask=0xFFFF) {
+                    static inline std::array<std::uint8_t, sizeof(OutType)>
+                        to_byte_array(OutType v) {
+                            std::array<std::uint8_t, sizeof(OutType)> bytes;
+                            for(int i = sizeof(v)-1; i>=0; --i) {
+                                bytes[i] = v & 0xFF;
+                                v >>= 8;
+                            }
+                            return bytes;
+                        }
+
+                    static inline OutType generate(transcript_type &transcript, std::size_t GrindingBits = 16) {
+                        BOOST_ASSERT_MSG(GrindingBits < 64, "Grinding parameter should be bits, not mask");
+                        output_type mask = GrindingBits > 0 ? ( 1ULL << GrindingBits ) - 1 : 0;
                         output_type proof_of_work = std::rand();
                         output_type result;
-                        std::vector<std::uint8_t> bytes(4);
 
                         while( true ) {
                             transcript_type tmp_transcript = transcript;
-                            bytes[0] = std::uint8_t((proof_of_work&0xFF000000)>>24);
-                            bytes[1] = std::uint8_t((proof_of_work&0x00FF0000)>>16);
-                            bytes[2] = std::uint8_t((proof_of_work&0x0000FF00)>>8);
-                            bytes[3] = std::uint8_t(proof_of_work&0x000000FF);
-
-                            tmp_transcript(bytes);
+                            tmp_transcript(to_byte_array(proof_of_work));
                             result = tmp_transcript.template int_challenge<output_type>();
                             if ((result & mask) == 0)
                                 break;
                             proof_of_work++;
                         }
-                        transcript(bytes);
+                        transcript(to_byte_array(proof_of_work));
                         result = transcript.template int_challenge<output_type>();
                         return proof_of_work;
                     }
 
-                    static inline bool verify(transcript_type &transcript, output_type proof_of_work, OutType mask=0xFFFF) {
-                        std::vector<std::uint8_t> bytes(4);
-                        bytes[0] = std::uint8_t((proof_of_work&0xFF000000)>>24);
-                        bytes[1] = std::uint8_t((proof_of_work&0x00FF0000)>>16);
-                        bytes[2] = std::uint8_t((proof_of_work&0x0000FF00)>>8);
-                        bytes[3] = std::uint8_t(proof_of_work&0x000000FF);
-                        transcript(bytes);
+                    static inline bool verify(transcript_type &transcript, output_type proof_of_work, std::size_t GrindingBits = 16) {
+                        BOOST_ASSERT_MSG(GrindingBits < 64, "Grinding parameter should be bits, not mask");
+                        transcript(to_byte_array(proof_of_work));
                         output_type result = transcript.template int_challenge<output_type>();
+                        output_type mask = GrindingBits > 0 ? ( 1ULL << GrindingBits ) - 1 : 0;
                         return ((result & mask) == 0);
                     }
                 };
@@ -91,7 +95,7 @@ namespace nil {
                     using value_type = typename FieldType::value_type;
                     using integral_type = typename FieldType::integral_type;
 
-                    static inline value_type generate(transcript_type &transcript, std::size_t GrindingBits=16) {
+                    static inline value_type generate(transcript_type &transcript, std::size_t GrindingBits = 16) {
                         static boost::random::random_device dev;
                         static nil::crypto3::random::algebraic_engine<FieldType> random_engine(dev);
                         value_type proof_of_work = random_engine();
@@ -115,7 +119,7 @@ namespace nil {
                         return proof_of_work;
                     }
 
-                    static inline bool verify(transcript_type &transcript, value_type proof_of_work, std::size_t GrindingBits=16) {
+                    static inline bool verify(transcript_type &transcript, value_type proof_of_work, std::size_t GrindingBits = 16) {
                         transcript(proof_of_work);
                         integral_type mask =
                             (GrindingBits > 0 ?

libs/zk/include/nil/crypto3/zk/commitments/detail/polynomial/proof_of_work.hpp

@@ -703,6 +703,7 @@ namespace nil {
                                            typename CommitmentSchemeType::transcript_type &transcript) {
                         /* The procedure of updating the transcript is subject to review and change
                          * #295 */
+
                         // Push commitments to transcript
 
                         transcript(_commitments[batch_ind]);

libs/zk/include/nil/crypto3/zk/commitments/polynomial/kzg.hpp

@@ -190,6 +190,7 @@ namespace nil {
                     // Differs from static, because we pack the result into byte blob.
                     commitment_type commit(std::size_t index) {
                         this->_ind_commitments[index] = {};
+                        this->_ind_commitments[index].resize(this->_polys[index].size());
                         this->state_commited(index);
 
                         std::vector<std::uint8_t> result = {};

libs/zk/include/nil/crypto3/zk/commitments/polynomial/kzg_v2.hpp

@@ -177,7 +177,7 @@ namespace nil {
                         if constexpr (std::is_same<math::polynomial_dfs<value_type>, PolynomialType>::value ) {
                             combined_Q.from_coefficients(combined_Q_normal);
                         } else {
-                            combined_Q = combined_Q_normal;
+                            combined_Q = std::move(combined_Q_normal);
                         }
 
                         precommitment_type combined_Q_precommitment = nil::crypto3::zk::algorithms::precommit<fri_type>(

libs/zk/include/nil/crypto3/zk/commitments/polynomial/lpc.hpp

@@ -88,7 +88,9 @@ namespace nil {
                         });
                         std::shared_ptr<math::evaluation_domain<FieldType>> extended_domain =
                             math::make_evaluation_domain<FieldType>(extended_domain_size);
+
                         visitor.visit(expr);
+
                         for (const auto& [var, count]: variable_counts) {
                             // We may have variable values in required sizes in some cases.
                             if (variable_values_out.find(var) != variable_values_out.end())

libs/zk/include/nil/crypto3/zk/snark/systems/plonk/placeholder/gates_argument.hpp

@@ -546,7 +546,7 @@ namespace nil {
                     }
 
                     // Each lookup table should fill full rectangle inside assignment table
-                    // Lookup tables may contain repeated values, but they shoul be placed into one
+                    // Lookup tables may contain repeated values, but they should be placed into one
                     // option one under another.
                     // Because of theta randomness compressed lookup tables' vectors for different table may contain
                     // similar values only with negligible probability.

libs/zk/include/nil/crypto3/zk/snark/systems/plonk/placeholder/lookup_argument.hpp

@@ -56,7 +56,7 @@ set(TESTS_NAMES
 #    "commitment/r1cs_gg_ppzksnark_mpc"
 #    "commitment/type_traits"
 #    "commitment/kimchi_pedersen"
-#    "commitment/proof_of_work"
+    "commitment/proof_of_work"
 
     "math/expression"
 

libs/zk/test/CMakeLists.txt

@@ -116,7 +116,7 @@ BOOST_AUTO_TEST_SUITE(fri_test_suite)
                 2, //expand_factor
                 lambda,
                 true,
-                0xFFFFF
+                20
         );
 
         BOOST_CHECK(D[1]->m == D[0]->m / 2);

libs/zk/test/commitment/fri.cpp

@@ -210,7 +210,7 @@ BOOST_AUTO_TEST_SUITE(lpc_math_polynomial_suite);
                 2, //expand_factor
                 lambda,
                 true,
-                0xFFF
+                12
         );
 
         using lpc_scheme_type = nil::crypto3::zk::commitments::lpc_commitment_scheme<lpc_type, math::polynomial<typename FieldType::value_type>>;
@@ -510,7 +510,7 @@ BOOST_AUTO_TEST_SUITE(lpc_params_test_suite)
                 2,  //expand_factor
                 lambda,
                 true,
-                0xFF
+                8
         );
 
         using lpc_scheme_type = nil::crypto3::zk::commitments::lpc_commitment_scheme<lpc_type, math::polynomial<typename FieldType::value_type>>;

libs/zk/test/commitment/lpc.cpp

@@ -51,16 +51,18 @@ BOOST_AUTO_TEST_SUITE(proof_of_knowledge_test_suite)
         using poseidon = nil::crypto3::hashes::poseidon<policy>;
         using pow_type = nil::crypto3::zk::commitments::field_proof_of_work<poseidon, field_type>;
 
-        const integral_type expected_mask = integral_type(0xFF80000000000000) << (field_type::modulus_bits - 64);
+        std::size_t grinding_bits = 9;
         nil::crypto3::zk::transcript::fiat_shamir_heuristic_sequential<poseidon> transcript;
         auto old_transcript_1 = transcript, old_transcript_2 = transcript;
 
-        auto result = pow_type::generate(transcript, 9);
-        BOOST_ASSERT(pow_type::verify(old_transcript_1, result, 9));
+        auto result = pow_type::generate(transcript, grinding_bits);
+        BOOST_ASSERT(pow_type::verify(old_transcript_1, result, grinding_bits));
 
         // manually reimplement verify to ensure that changes in implementation didn't break it
         old_transcript_2(result);
         auto chal = old_transcript_2.template challenge<field_type>();
+        const integral_type expected_mask = integral_type( (1 << grinding_bits) - 1 ) << (field_type::modulus_bits - grinding_bits);
+
         BOOST_ASSERT((integral_type(chal.data) & expected_mask) == 0);
 
         using hard_pow_type = nil::crypto3::zk::commitments::field_proof_of_work<poseidon, field_type>;
@@ -70,28 +72,26 @@ BOOST_AUTO_TEST_SUITE(proof_of_knowledge_test_suite)
 
     BOOST_AUTO_TEST_CASE(pow_basic_test) {
         using keccak = nil::crypto3::hashes::keccak_1600<512>;
-        const std::uint32_t mask = 0xFFFFF000;
+
+        const std::uint32_t grinding_bits = 20;
+        const uint64_t expected_mask = (1ULL << grinding_bits) - 1;
+
         using pow_type = nil::crypto3::zk::commitments::proof_of_work<keccak, std::uint32_t>;
 
         nil::crypto3::zk::transcript::fiat_shamir_heuristic_sequential<keccak> transcript;
         auto old_transcript_1 = transcript, old_transcript_2 = transcript;
 
-        auto result = pow_type::generate(transcript, mask);
-        BOOST_ASSERT(pow_type::verify(old_transcript_1, result, mask));
+        auto result = pow_type::generate(transcript, grinding_bits);
+        BOOST_ASSERT(pow_type::verify(old_transcript_1, result, grinding_bits));
 
         // manually reimplement verify to ensure that changes in implementation didn't break it
-        std::array<std::uint8_t, 4> bytes;
-        bytes[0] = std::uint8_t((result & 0xFF000000) >> 24);
-        bytes[1] = std::uint8_t((result & 0x00FF0000) >> 16);
-        bytes[2] = std::uint8_t((result & 0x0000FF00) >> 8);
-        bytes[3] = std::uint8_t(result & 0x000000FF);
-        old_transcript_2(bytes);
+        old_transcript_2(pow_type::to_byte_array(result));
         auto chal = old_transcript_2.template int_challenge<std::uint32_t>();
-        BOOST_ASSERT((chal & mask) == 0);
+        BOOST_ASSERT( (chal & expected_mask) == 0);
 
         // check that random stuff doesn't pass verify
         using hard_pow_type = nil::crypto3::zk::commitments::proof_of_work<keccak, std::uint32_t>;
-        BOOST_ASSERT(!hard_pow_type::verify(old_transcript_1, result, mask));
+        BOOST_ASSERT(!hard_pow_type::verify(old_transcript_1, result, grinding_bits));
     }
 
 BOOST_AUTO_TEST_SUITE_END()