From 414a1f792cfcb8729e378e8afbf4ea0b30812e3b Mon Sep 17 00:00:00 2001 From: Yavor Litchev Date: Fri, 17 Apr 2026 05:09:52 -0400 Subject: [PATCH 01/10] GPU paillier client code added, execution context modified to include GPU code, additional tests for GPU clients have been added, deemed fully functional --- src/xtrace_sdk/x_vec/crypto/_gpu_loader.py | 40 + .../x_vec/crypto/paillier-GPU-client/Makefile | 60 + .../paillier_GPU_client.cu | 1077 +++++++ .../paillier-GPU-lookup-client/Makefile | 61 + .../paillier_GPU_lookup_client.cu | 2655 +++++++++++++++++ .../x_vec/crypto/paillier_gpu_client.py | 173 ++ .../crypto/paillier_lookup_gpu_client.py | 82 + .../x_vec/utils/execution_context.py | 52 +- src/xtrace_sdk/x_vec/utils/settings.py | 9 +- tests/x_vec/test_GPU_paillier_clients.py | 130 + 10 files changed, 4326 insertions(+), 13 deletions(-) create mode 100644 src/xtrace_sdk/x_vec/crypto/_gpu_loader.py create mode 100644 src/xtrace_sdk/x_vec/crypto/paillier-GPU-client/Makefile create mode 100644 src/xtrace_sdk/x_vec/crypto/paillier-GPU-client/paillier_GPU_client.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/paillier-GPU-lookup-client/Makefile create mode 100644 src/xtrace_sdk/x_vec/crypto/paillier-GPU-lookup-client/paillier_GPU_lookup_client.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/paillier_gpu_client.py create mode 100644 src/xtrace_sdk/x_vec/crypto/paillier_lookup_gpu_client.py create mode 100644 tests/x_vec/test_GPU_paillier_clients.py diff --git a/src/xtrace_sdk/x_vec/crypto/_gpu_loader.py b/src/xtrace_sdk/x_vec/crypto/_gpu_loader.py new file mode 100644 index 0000000..c9539da --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/_gpu_loader.py @@ -0,0 +1,40 @@ +import importlib.util +import sys +from functools import lru_cache +from pathlib import Path +from types import ModuleType + + +_CRYPTO_DIR = Path(__file__).resolve().parent +_EXTENSION_GLOBS = ("*.so", "*.pyd", "*.dylib") + + +@lru_cache(maxsize=None) +def load_gpu_extension(module_name: str, relative_dir: str) -> ModuleType: + """Load a pybind extension from one of the in-tree GPU build directories.""" + search_dir = _CRYPTO_DIR / relative_dir + candidates: list[Path] = [] + for pattern in _EXTENSION_GLOBS: + candidates.extend(sorted(search_dir.glob(f"{module_name}{pattern[1:]}"))) + candidates.extend(sorted(search_dir.glob(f"{module_name}*{pattern[1:]}"))) + + if not candidates: + raise ImportError(f"Unable to locate GPU extension {module_name!r} under {search_dir}.") + + module_path = candidates[0] + spec = importlib.util.spec_from_file_location(module_name, module_path) + if spec is None or spec.loader is None: + raise ImportError(f"Unable to create an import spec for {module_path}.") + + existing = sys.modules.get(module_name) + if existing is not None: + return existing + + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop(module_name, None) + raise + return module diff --git a/src/xtrace_sdk/x_vec/crypto/paillier-GPU-client/Makefile b/src/xtrace_sdk/x_vec/crypto/paillier-GPU-client/Makefile new file mode 100644 index 0000000..385953f --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/paillier-GPU-client/Makefile @@ -0,0 +1,60 @@ +# GPU_paillier_client/Makefile +# +# Build a Python extension module paillier_GPU_client.* for different GPU archs and key sizes. +# +# Usage examples: +# make # defaults: KEY_BITS=1024, SMS="70 80 90", TPI=32 +# make KEY_BITS=2048 SMS="80 90" # compile for A100/H100 with 8192-bit n^2 +# make KEY_BITS=1024 TPI=32 SMS="86" # compile for RTX 30-series (sm_86) +# +# Directory layout (as you described): +# ./GPU_paillier_client/paillier_GPU_client.cu +# ./include/pybind11/... +# ./include/cgbn/... + +# -------- Parameters you can tweak on the command line -------- +KEY_BITS ?= 1024 +TPI ?= 32 +# List of SM versions to build for (space separated) +SMS ?= 70 80 86 89 90 + +# Compiler/paths +NVCC ?= nvcc +CUDA_PATH?= /usr/local/cuda + +# Python extension suffix (e.g., .cpython-311-x86_64-linux-gnu.so) +PY_INC := $(shell python3 -c 'import sysconfig, sys; print(sysconfig.get_paths()["include"])') +EXT := $(shell python3 -c "import sysconfig;print(sysconfig.get_config_var('EXT_SUFFIX'))") +TARGET := paillier_GPU_client$(EXT) + +SRC := paillier_GPU_client.cu +INCDIR := ../include +CXXSTD := -std=c++17 + +# Construct gencode flags for each SM +GENCODE := $(foreach sm,$(SMS),-gencode arch=compute_$(sm),code=sm_$(sm)) + +# Include paths +INCLUDES := -I$(INCDIR) -I$(INCDIR)/pybind11 -I$(INCDIR)/cgbn -I$(CUDA_PATH)/include -I$(PY_INC) + +# Build flags +NVCCFLAGS := $(CXXSTD) -O3 -Xcompiler -fPIC -shared \ + -DKEY_BITS=$(KEY_BITS) -DCGBN_TPI=$(TPI) -DCGBN_USE_GMP \ + -Wno-deprecated-gpu-targets \ + $(GENCODE) $(INCLUDES) + +# Link GMP for the CPU fallback (header demands GMP present) +LDLIBS := -lgmp + +# Note: pybind11 is header-only; normally you don't need to link libpython explicitly. +# If your system requires it, uncomment the following: +# PY_LDFLAGS := $(shell python3-config --ldflags) +# LDFLAGS := $(PY_LDFLAGS) + +all: $(TARGET) + +$(TARGET): $(SRC) + $(NVCC) $(NVCCFLAGS) -o $@ $< $(LDLIBS) + +clean: + rm -f $(TARGET) diff --git a/src/xtrace_sdk/x_vec/crypto/paillier-GPU-client/paillier_GPU_client.cu b/src/xtrace_sdk/x_vec/crypto/paillier-GPU-client/paillier_GPU_client.cu new file mode 100644 index 0000000..faa70f2 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/paillier-GPU-client/paillier_GPU_client.cu @@ -0,0 +1,1077 @@ +// production_GPU_paillier_client/paillier_GPU_client.cu +// +// This file implements a CUDA-accelerated Paillier client whose +// public API mirrors xtrace_sdk.crypto.paillier_client.PaillierClient: +// +// class PaillierGPUClient: +// def __init__(self, embed_len: int = 512, key_len: int = 1024, skip_key_gen: bool = False) +// def encrypt(self, embd: list[int]) -> list[int] +// def encode_hamming_client(self, ct1: list[int], ct2: list[int]) -> list[int] +// @staticmethod +// def encode_hamming_server(ct1: list[int], ct2: list[int], pk: dict) -> list[int] +// def decode_hamming_client(self, cipher: list[int]) -> int +// +// Keys are generated once in the constructor (unless skip_key_gen=True) +// using GMP big integers on the CPU, and stored inside the C++ class. +// GPU kernels use NVlabs CGBN for modular arithmetic on ciphertexts. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" + +#include +#include "cgbn/cgbn.h" + +namespace py = pybind11; + +// ---------------------- Compile-time configuration ---------------------- +#ifndef KEY_BITS +#define KEY_BITS 1024 +#endif + +#ifndef CGBN_TPI +#define CGBN_TPI 32 +#endif + +// In the Python client, key_len is the prime bit-length. +// n is roughly 2*key_len bits, and we pad plaintext chunks to 2*key_len bits. +#define PAILLIER_N_SQUARED_BITS (KEY_BITS*4) +static constexpr int LIMBS = (PAILLIER_N_SQUARED_BITS + 31) / 32; + +// ---------------------- Error handling helpers ---------------------- +#define CUDA_CHECK(call) do { \ + cudaError_t _e = (call); \ + if (_e != cudaSuccess) { \ + std::ostringstream _oss; \ + _oss << "CUDA error " << cudaGetErrorName(_e) \ + << " (" << cudaGetErrorString(_e) << ") at " << __FILE__ \ + << ":" << __LINE__; \ + throw std::runtime_error(_oss.str()); \ + } \ +} while(0) + +// ---------------------- Minimal GMP big-int wrapper ---------------------- +namespace bigint { + +class big_int_t { +public: + mpz_t v; + + big_int_t() { mpz_init(v); } + explicit big_int_t(unsigned long long x) { mpz_init(v); mpz_set_ui(v, static_cast(x)); } + explicit big_int_t(const std::string &s, int base = 10) { + mpz_init(v); + if (mpz_set_str(v, s.c_str(), base) != 0) { + throw std::invalid_argument("big_int_t: invalid string"); + } + } + big_int_t(const big_int_t &o) { mpz_init_set(v, o.v); } + big_int_t(big_int_t &&o) noexcept { mpz_init(v); mpz_swap(v, o.v); } + ~big_int_t() { mpz_clear(v); } + + big_int_t &operator=(const big_int_t &o) { + if (this != &o) mpz_set(v, o.v); + return *this; + } + big_int_t &operator=(big_int_t &&o) noexcept { + if (this != &o) mpz_swap(v, o.v); + return *this; + } + + std::string to_string(int base = 10) const { + char *raw = mpz_get_str(nullptr, base, v); + if (!raw) return {}; + std::string s(raw); + void (*freefunc)(void*, size_t) = nullptr; + mp_get_memory_functions(nullptr, nullptr, &freefunc); + if (freefunc) freefunc(raw, std::strlen(raw) + 1); + return s; + } +}; + +inline bigint::big_int_t powm(const bigint::big_int_t &base, + const bigint::big_int_t &exp, + const bigint::big_int_t &mod) { + bigint::big_int_t r; + mpz_powm(r.v, base.v, exp.v, mod.v); + return r; +} + +inline bigint::big_int_t gcd(const bigint::big_int_t &a, const bigint::big_int_t &b) { + bigint::big_int_t r; + mpz_gcd(r.v, a.v, b.v); + return r; +} + +// RAII GMP RNG +struct gmp_rand_ctx { + gmp_randstate_t st; + gmp_rand_ctx() { + gmp_randinit_default(st); + // Simple seeding based on current time; no std::random_device / chrono, + // to avoid extra dependencies in NVCC builds. + unsigned long seed = static_cast(std::time(nullptr)); + mpz_t mseed; + mpz_init_set_ui(mseed, seed); + gmp_randseed(st, mseed); + mpz_clear(mseed); + } + ~gmp_rand_ctx() { gmp_randclear(st); } + gmp_rand_ctx(const gmp_rand_ctx &) = delete; + gmp_rand_ctx &operator=(const gmp_rand_ctx &) = delete; +}; + +inline bigint::big_int_t random_prime_bits(unsigned bits, gmp_rand_ctx &rng) { + if (bits < 2) throw std::invalid_argument("bits must be >= 2"); + bigint::big_int_t x; + mpz_urandomb(x.v, rng.st, bits); + mpz_setbit(x.v, bits - 1); + if (mpz_even_p(x.v)) mpz_add_ui(x.v, x.v, 1); + bigint::big_int_t p; + mpz_nextprime(p.v, x.v); + while (static_cast(mpz_sizeinbase(p.v, 2)) != bits) { + mpz_urandomb(x.v, rng.st, bits); + mpz_setbit(x.v, bits - 1); + if (mpz_even_p(x.v)) mpz_add_ui(x.v, x.v, 1); + mpz_nextprime(p.v, x.v); + } + return p; +} + +} // namespace bigint + +// ---------------------- CPU Paillier key structures ---------------------- +struct PaillierPublicKeyCPU { + bigint::big_int_t g; + bigint::big_int_t n; + bigint::big_int_t n_squared; +}; + +struct PaillierSecretKeyCPU { + bigint::big_int_t phi; + bigint::big_int_t inv; +}; + +struct PaillierKeyPairCPU { + PaillierPublicKeyCPU pk; + PaillierSecretKeyCPU sk; +}; + +class PaillierCPU { +public: + static PaillierKeyPairCPU key_gen(unsigned prime_bits) { + using namespace bigint; + gmp_rand_ctx rng; + + big_int_t p = random_prime_bits(prime_bits, rng); + big_int_t q = random_prime_bits(prime_bits, rng); + + big_int_t n; + mpz_mul(n.v, p.v, q.v); + + big_int_t p_minus_1, q_minus_1; + mpz_sub_ui(p_minus_1.v, p.v, 1); + mpz_sub_ui(q_minus_1.v, q.v, 1); + + big_int_t phi; + mpz_mul(phi.v, p_minus_1.v, q_minus_1.v); + + big_int_t g; + mpz_add_ui(g.v, n.v, 1); + + big_int_t n2; + mpz_mul(n2.v, n.v, n.v); + + big_int_t inv; + if (!mpz_invert(inv.v, phi.v, n.v)) { + throw std::runtime_error("Paillier key_gen: phi^{-1} mod n does not exist"); + } + + PaillierPublicKeyCPU pk{g, n, n2}; + PaillierSecretKeyCPU sk{phi, inv}; + return PaillierKeyPairCPU{pk, sk}; + } +}; + +// ---------------------- Hex <-> limb conversion ---------------------- +static inline uint32_t hex_nibble(char c) { + if (c >= '0' && c <= '9') return (uint32_t)(c - '0'); + if (c >= 'a' && c <= 'f') return (uint32_t)(10 + c - 'a'); + if (c >= 'A' && c <= 'F') return (uint32_t)(10 + c - 'A'); + throw std::runtime_error("Invalid hex digit"); +} + +static std::string normalize_hex(const std::string &s) { + std::string h = s; + if (h.size() >= 2 && h[0] == '0' && (h[1] == 'x' || h[1] == 'X')) h = h.substr(2); + size_t p = 0; + while (p < h.size() && h[p] == '0') ++p; + if (p == h.size()) return "0"; + return h.substr(p); +} + +static std::vector hex_to_le_words(const std::string &hex_in, int words) { + std::string h = normalize_hex(hex_in); + if (h == "0") return std::vector(words, 0u); + if (h.size() & 1) h = "0" + h; + std::vector bytes(h.size() / 2); + for (size_t i = 0, j = 0; i < h.size(); i += 2, ++j) { + uint8_t hi = (uint8_t)hex_nibble(h[i]); + uint8_t lo = (uint8_t)hex_nibble(h[i + 1]); + bytes[j] = (uint8_t)((hi << 4) | lo); + } + std::vector words_le(words, 0u); + const size_t num_bytes = bytes.size(); + for (size_t i = 0; i < num_bytes; ++i) { + size_t limb_index = i / 4; + size_t byte_index = i % 4; + if (limb_index >= (size_t)words) break; + words_le[limb_index] |= ((uint32_t)bytes[num_bytes - 1 - i]) << (8 * byte_index); + } + return words_le; +} + +static std::string le_words_to_hex(const std::vector &words) { + std::vector bytes; + bytes.reserve(words.size() * 4); + for (uint32_t w : words) { + bytes.push_back((uint8_t)(w & 0xFF)); + bytes.push_back((uint8_t)((w >> 8) & 0xFF)); + bytes.push_back((uint8_t)((w >> 16) & 0xFF)); + bytes.push_back((uint8_t)((w >> 24) & 0xFF)); + } + while (!bytes.empty() && bytes.back() == 0) bytes.pop_back(); + if (bytes.empty()) return "0"; + static const char *HEX = "0123456789abcdef"; + std::string hex; + hex.resize(bytes.size() * 2); + size_t idx = 0; + for (int i = (int)bytes.size() - 1; i >= 0; --i) { + uint8_t b = bytes[i]; + hex[idx++] = HEX[(b >> 4) & 0xF]; + hex[idx++] = HEX[b & 0xF]; + } + size_t p = 0; + while (p + 1 < hex.size() && hex[p] == '0') ++p; + return hex.substr(p); +} + +// hex -> binary string (MSB-first), no fixed width; "0" for zero +static inline std::string hex_to_bin_nopad(const std::string &hex) { + auto hn = [](char c) -> int { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return 10 + (c - 'a'); + if (c >= 'A' && c <= 'F') return 10 + (c - 'A'); + return 0; + }; + static const char *LUT[16] = { + "0000", "0001", "0010", "0011", "0100", "0101", "0110", "0111", + "1000", "1001", "1010", "1011", "1100", "1101", "1110", "1111"}; + std::string out; + out.reserve(hex.size() * 4); + for (char c : hex) out.append(LUT[hn(c)]); + size_t p = out.find_first_not_of('0'); + if (p == std::string::npos) return "0"; + return out.substr(p); +} + +// ---------------------- CGBN types ---------------------- +typedef cgbn_context_t context_t; +typedef cgbn_env_t env_t; +typedef cgbn_error_report_t error_report_t; +typedef cgbn_mem_t mem_t; + + + +// ------------------ Arithmetic helpers ------------------ + +// --- RNG: xorshift64* --- +static __device__ __forceinline__ uint64_t xorshift64star(uint64_t &s) { + s ^= s >> 12; s ^= s << 25; s ^= s >> 27; + return s * 0x2545F4914F6CDD1DULL; +} + +static __device__ __forceinline__ uint32_t rand32(uint64_t &s) { + return (uint32_t)(xorshift64star(s) >> 32); +} + +// Extract pointer to 32-bit limbs from mem_t (little-endian) +static __device__ __forceinline__ const uint32_t* limbs_of(const mem_t *m) { + return reinterpret_cast(m); +} + +// Find MSB index (bit position) of exponent in mem_t (return -1 if zero) +static __device__ __forceinline__ int msb_index_mem(const mem_t *exp_mem) { + const uint32_t *w = limbs_of(exp_mem); + for (int i=LIMBS-1; i>=0; --i) { + uint32_t v = w[i]; + if (v != 0) { + // position of highest set bit within word + int lz = __clz(v); // 0..32 + int bit = 31 - lz; // 0..31 + return i*32 + bit; + } + } + return -1; +} + +// Square-and-multiply: result = base^exp_mem (mod mod_n2) +// exp_mem is provided as mem_t (n's limbs) +static __device__ void pow_mod_mem(env_t &env, + typename env_t::cgbn_t &result, + const typename env_t::cgbn_t &base, + const mem_t *exp_mem, + const typename env_t::cgbn_t &mod_n2) { + typename env_t::cgbn_t res, b; + typename env_t::cgbn_wide_t wide; + + cgbn_set_ui32(env, res, 1); + cgbn_set(env, b, base); + + const uint32_t *EW = reinterpret_cast(exp_mem); + + // Iterate all LIMBS*32 bits, LSB to MSB + for (int w = 0; w < LIMBS; ++w) { + uint32_t x = EW[w]; + for (int k = 0; k < 32; ++k) { + if (x & 1u) { + cgbn_mul_wide(env, wide, res, b); + cgbn_rem_wide(env, res, wide, mod_n2); + } + // b = b*b mod mod_n2 + cgbn_mul_wide(env, wide, b, b); + cgbn_rem_wide(env, b, wide, mod_n2); + x >>= 1; + } + } + cgbn_set(env, result, res); +} + +// Euclidean GCD (returns gcd(a,b) in 'a'); destructive to inputs +static __device__ void gcd_big(env_t &env, + typename env_t::cgbn_t &a, + typename env_t::cgbn_t &b) { + typename env_t::cgbn_t t; + while (cgbn_compare_ui32(env, b, 0) != 0) { + cgbn_rem(env, t, a, b); + cgbn_set(env, a, b); + cgbn_set(env, b, t); + } +} + +// Sample r in [1, n-1] and (optionally) ensure gcd(r, n) = 1. +// Very unlikely to need retries. +static __device__ void sample_r(env_t &env, + typename env_t::cgbn_t &r, + const typename env_t::cgbn_t &n, + uint64_t seed, + int instance_id, + bool ensure_coprime=true) { + typename env_t::cgbn_t tmp, a, b; + typename env_t::cgbn_t n_minus_1; + cgbn_sub_ui32(env, n_minus_1, n, 1); + + // seed per instance + uint64_t s = seed ^ (0x9E3779B97F4A7C15ULL * (uint64_t)(instance_id + 1)) ^ (uint64_t)clock64(); + + for (int tries=0; tries<16; ++tries) { + // Fill a mem_t with random words + mem_t rmem; + uint32_t *rw = reinterpret_cast(&rmem); + for (int i=0; i ensures 1..n-1 + cgbn_load(env, tmp, &rmem); + cgbn_rem(env, tmp, tmp, n_minus_1); + cgbn_add_ui32(env, r, tmp, 1); + + if (!ensure_coprime) return; + + // Check gcd(r, n) == 1 + cgbn_set(env, a, n); + cgbn_set(env, b, r); + gcd_big(env, a, b); // gcd in 'a' + if (cgbn_compare_ui32(env, a, 1) == 0) + return; // good + } + + // Fallback: r=1 (extremely unlikely path) + cgbn_set_ui32(env, r, 1); +} + + + +// Build a mask that selects bits at MSB-first positions 1,3,5,... within width=chunk_len +// mapped to little-endian limb storage used by cgbn_mem_t. +static inline mem_t make_odd_mask_host(const int chunk_len) { + mem_t mask{}; + uint32_t *w = reinterpret_cast(&mask); + // zero init + for (int i=0;i> 5; + const int pos = lsb_index & 31; + w[idx] |= (1u << pos); + } + return mask; +} + + + + +// ---------------------- CUDA kernel ---------------------- +__global__ void modmul_pointwise_kernel(error_report_t *report, + mem_t *A, + mem_t *B, + mem_t *MOD, // single modulus at index 0 + mem_t *R, + int count) { + int thread = blockIdx.x * blockDim.x + threadIdx.x; + int instance = thread / CGBN_TPI; + if (instance >= count) return; + + context_t context(cgbn_report_monitor, report, instance); + env_t env(context); + + typename env_t::cgbn_t a, b, m, r; + typename env_t::cgbn_wide_t wide; // <<< wide 2*BITS + cgbn_load(env, a, &A[instance]); + cgbn_load(env, b, &B[instance]); + cgbn_load(env, m, &MOD[0]); + + // Use CGBN modular multiply: r = (a*b) mod m + cgbn_mul_wide(env, wide, a, b); // wide = a*b + cgbn_rem_wide(env, r, wide, m); // r = wide % m + + cgbn_store(env, &R[instance], r); +} + + +// Each instance handles one (vector, chunk) +__global__ void encrypt_kernel(error_report_t *report, + const uint8_t *all_bits, // [batch * embed_len], 0/1 + int batch, + int embed_len, + int chunk_len, + int chunk_num, + mem_t *N, // n + mem_t *N2, // n^2 + mem_t *out_ct, // [batch * chunk_num] + uint64_t seed) { + int thread = blockIdx.x * blockDim.x + threadIdx.x; + int instance = thread / CGBN_TPI; + int total = batch * chunk_num; + if (instance >= total) return; + + int vec_idx = instance / chunk_num; + int chunk_idx = instance % chunk_num; + + context_t context(cgbn_report_monitor, report, instance); + env_t env(context); + + // Load n and n^2 into bigints + typename env_t::cgbn_t n, n2; + cgbn_load(env, n, &N[0]); + cgbn_load(env, n2, &N2[0]); + + // --- Build plaintext m from padded bits within this chunk --- + // Padded scheme: for original bit x[j], padded has at positions: 2*j -> 0, 2*j+1 -> x[j]. + typename env_t::cgbn_t m; + cgbn_set_ui32(env, m, 0); + + const int padded_total = 2 * embed_len; // total padded bits for this vector + const int chunk_base = chunk_idx * chunk_len; // start bit in padded stream + const uint8_t *vec_bits = all_bits + vec_idx * embed_len; + + // Only consume the bits that actually exist in this chunk, do NOT extend to chunk_len + int avail = 0; + if (chunk_base < padded_total) { + int max_take = padded_total - chunk_base; + avail = (max_take > chunk_len) ? chunk_len : max_take; + } + + // Build m by scanning MSB->LSB *for the avail bits only* + for (int t=0; t carries original bit + int j = p >> 1; // source index in the embedding + bit = (uint32_t)(vec_bits[j] & 1); + } + if (bit) { + cgbn_add_ui32(env, m, m, 1); + } + } + + // Make sure 0 <= m < n (same invariant as Python version) + // If m >= n, reduce m modulo n (more permissive than Python assert) + if (cgbn_compare(env, m, n) >= 0) { + cgbn_rem(env, m, m, n); + } + + // --- Compute a = 1 + m*n (mod n^2) using (1+n)^m trick --- + typename env_t::cgbn_t a, prod; + typename env_t::cgbn_wide_t wide; + cgbn_mul_wide(env, wide, m, n); + cgbn_rem_wide(env, prod, wide, n2); + cgbn_add_ui32(env, a, prod, 1); + if (cgbn_compare(env, a, n2) >= 0) { + cgbn_sub(env, a, a, n2); + } + + // --- Sample r and compute b = r^n mod n^2 --- + typename env_t::cgbn_t r, b; + // DEBUG: force r=1 to remove the random factor during debugging + // cgbn_set_ui32(env, r, 1); + sample_r(env, r, n, seed, instance, true); + pow_mod_mem(env, b, r, N, n2); // exponent is n, supplied as mem_t N + + // --- c = (a * b) mod n^2 --- + typename env_t::cgbn_t c; + cgbn_mul_wide(env, wide, a, b); + cgbn_rem_wide(env, c, wide, n2); + + // Store result + cgbn_store(env, &out_ct[instance], c); +} + +// ---------- GPU: decrypt-only kernel (per-chunk) ---------- +__global__ void decrypt_only_kernel(error_report_t *report, + mem_t *CT, // [total] ciphertexts + int total, // batch * chunk_num + mem_t *N, + mem_t *N2, + mem_t *PHI, + mem_t *INV, + mem_t *OUT_M) { // [total] plaintext m + const int thread = blockIdx.x*blockDim.x + threadIdx.x; + const int instance = thread / CGBN_TPI; + if (instance >= total) return; + + typedef cgbn_context_t context_t; + context_t context(cgbn_report_monitor, report, instance); + env_t env(context); + + typename env_t::cgbn_t n, n2, phi, inv, c, t, x, q, rrem, m; + typename env_t::cgbn_wide_t wide; + + cgbn_load(env, c, &CT[instance]); + cgbn_load(env, n, &N[0]); + cgbn_load(env, n2, &N2[0]); + cgbn_load(env, phi, &PHI[0]); + cgbn_load(env, inv, &INV[0]); + + // t = c^phi mod n^2 (LSB-first exp) + pow_mod_mem(env, t, c, PHI, n2); + + // x = t - 1, then exact div by n: x = q*n + rrem + cgbn_sub_ui32(env, x, t, 1); + cgbn_div(env, q, x, n); + cgbn_rem(env, rrem, x, n); // for debug you can check rrem==0 + + // m = (q * inv) mod n + cgbn_mul_wide(env, wide, q, inv); + cgbn_rem_wide(env, m, wide, n); + + cgbn_store(env, &OUT_M[instance], m); +} + + + +// Each instance handles one (vector, chunk) ciphertext -> produces count of ones at odd bit positions +__global__ void decode_hamming_kernel(error_report_t *report, + mem_t *CT, // [total] ciphertexts + int total, // batch * chunk_num + int chunk_len, // = 2*key_len + mem_t *N, mem_t *N2, + mem_t *PHI, mem_t *INV, + uint32_t *out_odd, // <-- NEW name + uint32_t *out_even, // <-- NEW + uint32_t *out_rem_nz, // <-- NEW + mem_t *out_m_dbg, // <-- OPTIONAL (can pass nullptr) + /* Additional parameters for GPU mask inputs for bitcounting*/ + const mem_t *ODD_MASK, + int words_used) +{ + + int thread = blockIdx.x * blockDim.x + threadIdx.x; + int instance = thread / CGBN_TPI; + if (instance >= total) return; + + context_t context(cgbn_report_monitor, report, instance); + env_t env(context); + + // Load inputs + typename env_t::cgbn_t n, n2, phi, inv, c; + cgbn_load(env, n, &N[0]); + cgbn_load(env, n2, &N2[0]); + cgbn_load(env, phi, &PHI[0]); + cgbn_load(env, inv, &INV[0]); + cgbn_load(env, c, &CT[instance]); + + // Decrypt: t = c^phi mod n^2 + typename env_t::cgbn_t t, x, q, rrem, m; +typename env_t::cgbn_wide_t wide; + +pow_mod_mem(env, t, c, PHI, n2); // t = c^phi mod n^2 +// x = t - 1 (t is in [1, n^2-1], so this is >= 0) +cgbn_sub_ui32(env, x, t, 1); + +// Divide exactly: x = q*n + rrem (rrem must be 0) +cgbn_div(env, q, x, n); +cgbn_rem(env, rrem, x, n); +uint32_t rem_nz = (cgbn_compare_ui32(env, rrem, 0) != 0); + +// m = (q * inv) mod n +cgbn_mul_wide(env, wide, q, inv); +cgbn_rem_wide(env, m, wide, n); + +// Store m for host-side inspection if requested +if (out_m_dbg != nullptr) { + cgbn_store(env, &out_m_dbg[instance], m); +} + +// Count odd/even bit positions MSB-first over chunk_len +// ---- PURE GPU COUNTING via mask+popcount ---- + mem_t mmem; + cgbn_store(env, &mmem, m); + + const uint32_t *mw = reinterpret_cast(&mmem); + const uint32_t *ow = reinterpret_cast(&ODD_MASK[0]); + + uint32_t odd_cnt = 0; + #pragma unroll + for (int w=0; w gpu_pointwise_mulmod_hex(const std::vector &ct1_hex, + const std::vector &ct2_hex, + const std::string &n_squared_hex) { + if (ct1_hex.size() != ct2_hex.size()) { + throw std::runtime_error("encode_hamming_*: ct1 and ct2 must have the same length"); + } + const int count = static_cast(ct1_hex.size()); + if (count == 0) return {}; + + std::vector h_A(count), h_B(count); + std::vector mod_words = hex_to_le_words(n_squared_hex, LIMBS); + mem_t h_MOD_one{}; + std::memcpy(&h_MOD_one, mod_words.data(), LIMBS * sizeof(uint32_t)); + + for (int i = 0; i < count; ++i) { + auto a = hex_to_le_words(ct1_hex[i], LIMBS); + auto b = hex_to_le_words(ct2_hex[i], LIMBS); + std::memcpy(&h_A[i], a.data(), LIMBS * sizeof(uint32_t)); + std::memcpy(&h_B[i], b.data(), LIMBS * sizeof(uint32_t)); + } + + error_report_t *d_report = nullptr; + mem_t *d_A = nullptr, *d_B = nullptr, *d_MOD = nullptr, *d_R = nullptr; + + CUDA_CHECK(cudaMalloc((void **)&d_report, sizeof(error_report_t))); + CUDA_CHECK(cudaMemset(d_report, 0, sizeof(error_report_t))); + + CUDA_CHECK(cudaMalloc((void **)&d_A, count * sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_B, count * sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_R, count * sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_MOD, sizeof(mem_t))); + + CUDA_CHECK(cudaMemcpy(d_A, h_A.data(), count * sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMemcpy(d_B, h_B.data(), count * sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMemcpy(d_MOD, &h_MOD_one, sizeof(mem_t), cudaMemcpyHostToDevice)); + + const int INSTS_PER_BLK = 8; + const int THREADS = INSTS_PER_BLK * CGBN_TPI; + const int blocks = (count + INSTS_PER_BLK - 1) / INSTS_PER_BLK; + + modmul_pointwise_kernel<<>>(d_report, d_A, d_B, d_MOD, d_R, count); + CUDA_CHECK(cudaPeekAtLastError()); + CUDA_CHECK(cudaDeviceSynchronize()); + + std::vector h_R(count); + CUDA_CHECK(cudaMemcpy(h_R.data(), d_R, count * sizeof(mem_t), cudaMemcpyDeviceToHost)); + + cudaFree(d_A); + cudaFree(d_B); + cudaFree(d_R); + cudaFree(d_MOD); + cudaFree(d_report); + + std::vector out(count); + for (int i = 0; i < count; ++i) { + std::vector words(LIMBS); + std::memcpy(words.data(), &h_R[i], LIMBS * sizeof(uint32_t)); + out[i] = le_words_to_hex(words); + } + return out; +} + +// ---------------------- PaillierGPUClient class ---------------------- +class PaillierGPUClient { +public: + PaillierGPUClient(int embed_len = 512, int key_len = KEY_BITS, bool skip_key_gen = false) + : embed_len_(embed_len), + key_len_(key_len), + chunk_len_(key_len * 2), + chunk_num_(embed_len > key_len ? ((2 * embed_len) / (key_len * 2) + + (((2 * embed_len) % (key_len * 2)) != 0)) + : 1), + have_keys_(!skip_key_gen) { + if (!skip_key_gen) { + keys_ = PaillierCPU::key_gen(static_cast(key_len_)); + pk_n_hex_ = keys_.pk.n.to_string(16); + pk_n2_hex_ = keys_.pk.n_squared.to_string(16); + sk_phi_hex_ = keys_.sk.phi.to_string(16); + sk_inv_hex_ = keys_.sk.inv.to_string(16); + // Light debug: print generated keys (hex) + // std::cerr << "[PaillierGPUClient] Generated keys (hex):\n" + // << " n = " << pk_n_hex_ << "\n" + // << " n_squared= " << pk_n2_hex_ << "\n" + // << " phi = " << sk_phi_hex_ << "\n" + // << " inv = " << sk_inv_hex_ << std::endl; + } + } + + int embed_len() const { return embed_len_; } + int key_len() const { return key_len_; } + int chunk_len() const { return chunk_len_; } + int chunk_num() const { return chunk_num_; } + + // Expose public key components as hex for Python-side checks + std::pair get_pk_hex() const { + if (!have_keys_) { + throw std::runtime_error("get_pk_hex: keys not initialized"); + } + return {pk_n_hex_, pk_n2_hex_}; + } + + // Expose full key components as hex: (n, n_squared, phi, inv) + std::tuple get_keys_hex() const { + if (!have_keys_) { + throw std::runtime_error("get_keys_hex: keys not initialized"); + } + return {pk_n_hex_, pk_n2_hex_, sk_phi_hex_, sk_inv_hex_}; + } + + // Batched encrypt: embeddings is [batch][embed_len] of 0/1, returns [batch][chunk_num] hex ciphers + std::vector> + encrypt(const std::vector> &embeddings, std::uint64_t seed = 0) const { + if (!have_keys_) { + throw std::runtime_error("encrypt: keys not initialized"); + } + const int batch = static_cast(embeddings.size()); + const int embed_len = embed_len_; + const int chunk_len = chunk_len_; + const int chunk_num = chunk_num_; + + if (batch == 0) return {}; + + // Validate shapes and binary values + for (int i = 0; i < batch; ++i) { + if ((int)embeddings[i].size() != embed_len) { + throw std::runtime_error("encrypt: each embedding must have length == embed_len"); + } + for (int v : embeddings[i]) { + if (v != 0 && v != 1) { + throw std::runtime_error("encrypt: embeddings must be binary 0/1"); + } + } + } + + // Flatten bits into [batch * embed_len] + std::vector h_bits; + h_bits.reserve(static_cast(batch) * static_cast(embed_len)); + for (int i = 0; i < batch; ++i) { + for (int j = 0; j < embed_len; ++j) { + h_bits.push_back(static_cast(embeddings[i][j])); + } + } + + std::vector n_words = hex_to_le_words(pk_n_hex_, LIMBS); + std::vector n2_words = hex_to_le_words(pk_n2_hex_, LIMBS); + mem_t h_N{}, h_N2{}; + std::memcpy(&h_N, n_words.data(), LIMBS * sizeof(uint32_t)); + std::memcpy(&h_N2, n2_words.data(), LIMBS * sizeof(uint32_t)); + + error_report_t *d_report = nullptr; + uint8_t *d_bits = nullptr; + mem_t *d_N = nullptr, *d_N2 = nullptr, *d_out = nullptr; + + const int total = batch * chunk_num; + + CUDA_CHECK(cudaMalloc((void **)&d_report, sizeof(error_report_t))); + CUDA_CHECK(cudaMemset(d_report, 0, sizeof(error_report_t))); + CUDA_CHECK(cudaMalloc((void **)&d_bits, + static_cast(batch) * static_cast(embed_len) * + sizeof(uint8_t))); + CUDA_CHECK(cudaMemcpy(d_bits, + h_bits.data(), + static_cast(batch) * static_cast(embed_len) * + sizeof(uint8_t), + cudaMemcpyHostToDevice)); + + CUDA_CHECK(cudaMalloc((void **)&d_N, sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_N2, sizeof(mem_t))); + CUDA_CHECK(cudaMemcpy(d_N, &h_N, sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMemcpy(d_N2, &h_N2, sizeof(mem_t), cudaMemcpyHostToDevice)); + + CUDA_CHECK(cudaMalloc((void **)&d_out, total * sizeof(mem_t))); + + // If seed == 0, derive a pseudo-random seed from time and simple bit-mixing + uint64_t actual_seed = seed; + if (actual_seed == 0) { + actual_seed = static_cast(std::time(nullptr)); + // simple scrambling to make it less predictable + actual_seed ^= (actual_seed << 13); + actual_seed ^= (actual_seed >> 7); + actual_seed ^= 0x9E3779B97F4A7C15ULL; + } + + const int INSTS_PER_BLK = 8; + const int THREADS = INSTS_PER_BLK * CGBN_TPI; + const int blocks = (total + INSTS_PER_BLK - 1) / INSTS_PER_BLK; + + encrypt_kernel<<>>(d_report, d_bits, batch, embed_len, chunk_len, + chunk_num, d_N, d_N2, d_out, actual_seed); + CUDA_CHECK(cudaPeekAtLastError()); + CUDA_CHECK(cudaDeviceSynchronize()); + + std::vector h_out(total); + CUDA_CHECK(cudaMemcpy(h_out.data(), d_out, total * sizeof(mem_t), cudaMemcpyDeviceToHost)); + + cudaFree(d_bits); + cudaFree(d_N); + cudaFree(d_N2); + cudaFree(d_out); + cudaFree(d_report); + + // Reshape back to [batch][chunk_num] hex strings + std::vector> out(batch, std::vector(chunk_num)); + for (int i = 0; i < batch; ++i) { + for (int c = 0; c < chunk_num; ++c) { + int idx = i * chunk_num + c; + std::vector words(LIMBS); + std::memcpy(words.data(), &h_out[idx], LIMBS * sizeof(uint32_t)); + out[i][c] = le_words_to_hex(words); + } + } + return out; + } + + // Client-side homomorphic addition: ct1[i]*ct2[i] mod n^2 + std::vector encode_hamming_client(const std::vector &ct1, + const std::vector &ct2) const { + if (!have_keys_) { + throw std::runtime_error("encode_hamming_client: keys not initialized"); + } + return gpu_pointwise_mulmod_hex(ct1, ct2, pk_n2_hex_); + } + + // Server-side version: takes pk dict with "n_squared" + static std::vector encode_hamming_server(const std::vector &ct1, + const std::vector &ct2, + const py::dict &pk) { + auto n2_obj = pk["n_squared"]; + std::string n2_hex = py::cast(py::str(n2_obj)); + return gpu_pointwise_mulmod_hex(ct1, ct2, n2_hex); + } + + // GPU decrypt chunks to plaintext m (hex), then CPU bit-padding and odd-bit counting + std::vector + decode_hamming_client(const std::vector> &ciphers_hex_batch) const { + if (!have_keys_) { + throw std::runtime_error("decode_hamming_client: keys not initialized"); + } + const int batch = static_cast(ciphers_hex_batch.size()); + const int chunk_num = chunk_num_; + const int chunk_len = chunk_len_; + if (batch == 0) return {}; + + for (int i = 0; i < batch; ++i) { + if ((int)ciphers_hex_batch[i].size() != chunk_num) { + throw std::runtime_error("decode_hamming_client: each inner list must have length == chunk_num"); + } + } + + // 1) GPU decrypt per-chunk to get plaintext m as hex strings + auto m_hex_batch = decrypt_chunks(ciphers_hex_batch); + + // 2) CPU: mirror Python's padding and odd-bit counting + std::vector out(batch, 0); + for (int i = 0; i < batch; ++i) { + long long ham = 0; + std::string bin_concat; + bin_concat.reserve(static_cast(chunk_len) * static_cast(chunk_num)); + for (int c = 0; c < chunk_num; ++c) { + const std::string &mh = m_hex_batch[i][c]; + std::string b = hex_to_bin_nopad(mh); // MSB-first, minimal width ("0" for zero) + if ((int)b.size() != chunk_len && !b.empty()) { + if ((int)b.size() < chunk_len) { + b.insert(0, chunk_len - (int)b.size(), '0'); + } + } else if (b == "0") { + b.assign(static_cast(chunk_len), '0'); + } + bin_concat += b; + } + for (size_t k = 1; k < bin_concat.size(); k += 2) { + if (bin_concat[k] == '1') ++ham; + } + out[i] = ham; + } + return out; + } + +private: + int embed_len_; + int key_len_; + int chunk_len_; + int chunk_num_; + bool have_keys_; + + PaillierKeyPairCPU keys_; + std::string pk_n_hex_; + std::string pk_n2_hex_; + std::string sk_phi_hex_; + std::string sk_inv_hex_; + + // Helper: GPU decrypt all chunks to plaintext m (hex), mirroring original GPU client + std::vector> + decrypt_chunks(const std::vector> &ciphers_hex_batch) const { + const int batch = static_cast(ciphers_hex_batch.size()); + const int chunk_num = this->chunk_num_; + if (batch == 0) return {}; + + for (int i = 0; i < batch; ++i) { + if ((int)ciphers_hex_batch[i].size() != chunk_num) { + throw std::runtime_error("decrypt_chunks: each inner list must have length == chunk_num"); + } + } + + const int total = batch * chunk_num; + + // pack CT + std::vector h_CT(total); + for (int i = 0; i < batch; ++i) { + for (int c = 0; c < chunk_num; ++c) { + const int idx = i * chunk_num + c; + auto w = hex_to_le_words(ciphers_hex_batch[i][c], LIMBS); + std::memcpy(&h_CT[idx], w.data(), LIMBS * sizeof(uint32_t)); + } + } + + // keys + mem_t h_N{}, h_N2{}, h_PHI{}, h_INV{}; + std::memcpy(&h_N, hex_to_le_words(pk_n_hex_, LIMBS).data(), LIMBS * sizeof(uint32_t)); + std::memcpy(&h_N2, hex_to_le_words(pk_n2_hex_, LIMBS).data(), LIMBS * sizeof(uint32_t)); + std::memcpy(&h_PHI, hex_to_le_words(sk_phi_hex_, LIMBS).data(), LIMBS * sizeof(uint32_t)); + std::memcpy(&h_INV, hex_to_le_words(sk_inv_hex_, LIMBS).data(), LIMBS * sizeof(uint32_t)); + + // device + error_report_t *d_report = nullptr; + mem_t *d_CT = nullptr, *d_N = nullptr, *d_N2 = nullptr, *d_PHI = nullptr, *d_INV = nullptr, + *d_M = nullptr; + + CUDA_CHECK(cudaMalloc((void **)&d_report, sizeof(error_report_t))); + CUDA_CHECK(cudaMemset(d_report, 0, sizeof(error_report_t))); + CUDA_CHECK(cudaMalloc((void **)&d_CT, total * sizeof(mem_t))); + CUDA_CHECK(cudaMemcpy(d_CT, h_CT.data(), total * sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMalloc((void **)&d_N, sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_N2, sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_PHI, sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_INV, sizeof(mem_t))); + CUDA_CHECK(cudaMemcpy(d_N, &h_N, sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMemcpy(d_N2, &h_N2, sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMemcpy(d_PHI, &h_PHI, sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMemcpy(d_INV, &h_INV, sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMalloc((void **)&d_M, total * sizeof(mem_t))); + + const int INSTS_PER_BLK = 8; + const int THREADS = INSTS_PER_BLK * CGBN_TPI; + const int blocks = (total + INSTS_PER_BLK - 1) / INSTS_PER_BLK; + decrypt_only_kernel<<>>(d_report, d_CT, total, d_N, d_N2, d_PHI, d_INV, d_M); + CUDA_CHECK(cudaPeekAtLastError()); + CUDA_CHECK(cudaDeviceSynchronize()); + + std::vector h_M(total); + CUDA_CHECK(cudaMemcpy(h_M.data(), d_M, total * sizeof(mem_t), cudaMemcpyDeviceToHost)); + + cudaFree(d_CT); + cudaFree(d_N); + cudaFree(d_N2); + cudaFree(d_PHI); + cudaFree(d_INV); + cudaFree(d_M); + cudaFree(d_report); + + std::vector> out(batch, std::vector(chunk_num)); + for (int i = 0; i < batch; ++i) { + for (int c = 0; c < chunk_num; ++c) { + const int idx = i * chunk_num + c; + std::vector w(LIMBS); + std::memcpy(w.data(), &h_M[idx], LIMBS * sizeof(uint32_t)); + out[i][c] = le_words_to_hex(w); + } + } + return out; + } +}; + +// ---------------------- Pybind module ---------------------- +PYBIND11_MODULE(paillier_GPU_client, m) { + py::class_(m, "PaillierGPUClient") + .def(py::init(), + py::arg("embed_len") = 512, + py::arg("key_len") = KEY_BITS, + py::arg("skip_key_gen") = false) + .def("encrypt", &PaillierGPUClient::encrypt, + py::arg("embeddings"), + py::arg("seed") = 0ULL) + .def("encode_hamming_client", &PaillierGPUClient::encode_hamming_client, + py::arg("ct1"), + py::arg("ct2")) + .def_static("encode_hamming_server", &PaillierGPUClient::encode_hamming_server, + py::arg("ct1"), + py::arg("ct2"), + py::arg("pk")) + .def("decode_hamming_client", &PaillierGPUClient::decode_hamming_client, + py::arg("ciphers_hex_batch")) + .def("get_pk_hex", &PaillierGPUClient::get_pk_hex) + .def("get_keys_hex", &PaillierGPUClient::get_keys_hex) + .def_property_readonly("embed_len", &PaillierGPUClient::embed_len) + .def_property_readonly("key_len", &PaillierGPUClient::key_len) + .def_property_readonly("chunk_len", &PaillierGPUClient::chunk_len) + .def_property_readonly("chunk_num", &PaillierGPUClient::chunk_num); +} diff --git a/src/xtrace_sdk/x_vec/crypto/paillier-GPU-lookup-client/Makefile b/src/xtrace_sdk/x_vec/crypto/paillier-GPU-lookup-client/Makefile new file mode 100644 index 0000000..3a0e67a --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/paillier-GPU-lookup-client/Makefile @@ -0,0 +1,61 @@ +# GPU_paillier_client/Makefile +# +# Build a Python extension module paillier_GPU_lookup_client.* for different GPU archs and key sizes. +# +# Usage examples: +# make # defaults: KEY_BITS=1024, SMS="70 80 90", TPI=32 +# make KEY_BITS=2048 SMS="80 90" # compile for A100/H100 with 8192-bit n^2 +# make KEY_BITS=1024 TPI=32 SMS="86" # compile for RTX 30-series (sm_86) +# +# Directory layout (as you described): +# ./lookup_GPU_paillier_client/paillier_GPU_lookup_client.cu +# ./include/pybind11/... +# ./include/cgbn/... + +# -------- Parameters you can tweak on the command line -------- +KEY_BITS ?= 1024 +TPI ?= 32 +ALPHA_LEN ?= 280 +# List of SM versions to build for (space separated) +SMS ?= 70 80 86 89 90 + +# Compiler/paths +NVCC ?= nvcc +CUDA_PATH?= /usr/local/cuda + +# Python extension suffix (e.g., .cpython-311-x86_64-linux-gnu.so) +PY_INC := $(shell python3 -c 'import sysconfig, sys; print(sysconfig.get_paths()["include"])') +EXT := $(shell python3 -c "import sysconfig;print(sysconfig.get_config_var('EXT_SUFFIX'))") +TARGET := paillier_GPU_lookup_client$(EXT) + +SRC := paillier_GPU_lookup_client.cu +INCDIR := ../include +CXXSTD := -std=c++17 + +# Construct gencode flags for each SM +GENCODE := $(foreach sm,$(SMS),-gencode arch=compute_$(sm),code=sm_$(sm)) + +# Include paths +INCLUDES := -I$(INCDIR) -I$(INCDIR)/pybind11 -I$(INCDIR)/cgbn -I$(CUDA_PATH)/include -I$(PY_INC) + +# Build flags +NVCCFLAGS := $(CXXSTD) -O3 -Xcompiler -fPIC -shared \ + -DKEY_BITS=$(KEY_BITS) -DALPHA_LEN=$(ALPHA_LEN) -DCGBN_TPI=$(TPI) -DCGBN_USE_GMP \ + -Wno-deprecated-gpu-targets \ + $(GENCODE) $(INCLUDES) + +# Link GMP for the CPU fallback (header demands GMP present) +LDLIBS := -lgmp + +# Note: pybind11 is header-only; normally you don't need to link libpython explicitly. +# If your system requires it, uncomment the following: +# PY_LDFLAGS := $(shell python3-config --ldflags) +# LDFLAGS := $(PY_LDFLAGS) + +all: $(TARGET) + +$(TARGET): $(SRC) + $(NVCC) $(NVCCFLAGS) -o $@ $< $(LDLIBS) + +clean: + rm -f $(TARGET) diff --git a/src/xtrace_sdk/x_vec/crypto/paillier-GPU-lookup-client/paillier_GPU_lookup_client.cu b/src/xtrace_sdk/x_vec/crypto/paillier-GPU-lookup-client/paillier_GPU_lookup_client.cu new file mode 100644 index 0000000..21cbf0d --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/paillier-GPU-lookup-client/paillier_GPU_lookup_client.cu @@ -0,0 +1,2655 @@ +// production_GPU_paillier_client/paillier_GPU_client.cu +// +// This file implements a CUDA-accelerated Paillier client whose +// public API mirrors xtrace_sdk.crypto.paillier_client.PaillierClient: +// +// class PaillierGPUClient: +// def __init__(self, embed_len: int = 512, key_len: int = 1024, skip_key_gen: bool = False) +// def encrypt(self, embd: list[int]) -> list[int] +// def encode_hamming_client(self, ct1: list[int], ct2: list[int]) -> list[int] +// @staticmethod +// def encode_hamming_server(ct1: list[int], ct2: list[int], pk: dict) -> list[int] +// def decode_hamming_client(self, cipher: list[int]) -> int +// +// Keys are generated once in the constructor (unless skip_key_gen=True) +// using GMP big integers on the CPU, and stored inside the C++ class. +// GPU kernels use NVlabs CGBN for modular arithmetic on ciphertexts. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" + +#include +#include "cgbn/cgbn.h" + +namespace py = pybind11; + +// ---------------------- Compile-time configuration ---------------------- +#ifndef KEY_BITS +#define KEY_BITS 1024 +#endif + +#ifndef ALPHA_LEN +#define ALPHA_LEN 280 +#endif + +#ifndef CGBN_TPI +#define CGBN_TPI 32 +#endif + +// In the Python client, key_len is the prime bit-length. +// n is roughly 2*key_len bits, and we pad plaintext chunks to 2*key_len bits. +#define PAILLIER_N_SQUARED_BITS (KEY_BITS*4) +static constexpr int LIMBS = (PAILLIER_N_SQUARED_BITS + 31) / 32; + +// Lookup-table parameters (mirror Python paillier_lookup.py defaults) +#define PAILLIER_MSG_BITS 8 +static constexpr int PAILLIER_MSG_TABLE_SIZE = (1 << PAILLIER_MSG_BITS); + +// Precomputed noise table size and how many random entries to multiply +static constexpr int PAILLIER_NOISE_TABLE_SIZE = 1 << 8; // 2**8 +static constexpr int PAILLIER_NOISE_MULTIPLES = 14; + +// ---------------------- Error handling helpers ---------------------- +#define CUDA_CHECK(call) do { \ + cudaError_t _e = (call); \ + if (_e != cudaSuccess) { \ + std::ostringstream _oss; \ + _oss << "CUDA error " << cudaGetErrorName(_e) \ + << " (" << cudaGetErrorString(_e) << ") at " << __FILE__ \ + << ":" << __LINE__; \ + throw std::runtime_error(_oss.str()); \ + } \ +} while(0) + +// ---------------------- Minimal GMP big-int wrapper ---------------------- +namespace bigint { + +class big_int_t { +public: + mpz_t v; + + big_int_t() { mpz_init(v); } + explicit big_int_t(unsigned long long x) { mpz_init(v); mpz_set_ui(v, static_cast(x)); } + explicit big_int_t(const std::string &s, int base = 10) { + mpz_init(v); + if (mpz_set_str(v, s.c_str(), base) != 0) { + throw std::invalid_argument("big_int_t: invalid string"); + } + } + big_int_t(const big_int_t &o) { mpz_init_set(v, o.v); } + big_int_t(big_int_t &&o) noexcept { mpz_init(v); mpz_swap(v, o.v); } + ~big_int_t() { mpz_clear(v); } + + big_int_t &operator=(const big_int_t &o) { + if (this != &o) mpz_set(v, o.v); + return *this; + } + big_int_t &operator=(big_int_t &&o) noexcept { + if (this != &o) mpz_swap(v, o.v); + return *this; + } + + std::string to_string(int base = 10) const { + char *raw = mpz_get_str(nullptr, base, v); + if (!raw) return {}; + std::string s(raw); + void (*freefunc)(void*, size_t) = nullptr; + mp_get_memory_functions(nullptr, nullptr, &freefunc); + if (freefunc) freefunc(raw, std::strlen(raw) + 1); + return s; + } +}; + +inline bigint::big_int_t powm(const bigint::big_int_t &base, + const bigint::big_int_t &exp, + const bigint::big_int_t &mod) { + bigint::big_int_t r; + mpz_powm(r.v, base.v, exp.v, mod.v); + return r; +} + +inline bigint::big_int_t gcd(const bigint::big_int_t &a, const bigint::big_int_t &b) { + bigint::big_int_t r; + mpz_gcd(r.v, a.v, b.v); + return r; +} + +// RAII GMP RNG +struct gmp_rand_ctx { + gmp_randstate_t st; + gmp_rand_ctx() { + gmp_randinit_default(st); + // Simple seeding based on current time; no std::random_device / chrono, + // to avoid extra dependencies in NVCC builds. + unsigned long seed = static_cast(std::time(nullptr)); + mpz_t mseed; + mpz_init_set_ui(mseed, seed); + gmp_randseed(st, mseed); + mpz_clear(mseed); + } + ~gmp_rand_ctx() { gmp_randclear(st); } + gmp_rand_ctx(const gmp_rand_ctx &) = delete; + gmp_rand_ctx &operator=(const gmp_rand_ctx &) = delete; +}; + +inline bigint::big_int_t random_prime_bits(unsigned bits, gmp_rand_ctx &rng) { + if (bits < 2) throw std::invalid_argument("bits must be >= 2"); + bigint::big_int_t x; + mpz_urandomb(x.v, rng.st, bits); + mpz_setbit(x.v, bits - 1); + if (mpz_even_p(x.v)) mpz_add_ui(x.v, x.v, 1); + bigint::big_int_t p; + mpz_nextprime(p.v, x.v); + while (static_cast(mpz_sizeinbase(p.v, 2)) != bits) { + mpz_urandomb(x.v, rng.st, bits); + mpz_setbit(x.v, bits - 1); + if (mpz_even_p(x.v)) mpz_add_ui(x.v, x.v, 1); + mpz_nextprime(p.v, x.v); + } + return p; +} + +} // namespace bigint + +// ---------------------- Lookup-style helpers (CPU, GMP) ---------------------- +// Bit length of a big_int_t in base 2. +static inline unsigned bitlen_mpz(const bigint::big_int_t &x) { + if (mpz_sgn(x.v) <= 0) return 0; + return static_cast(mpz_sizeinbase(x.v, 2)); +} + +// L(x) = (x-1)/n, assuming x ≡ 1 (mod n) +static inline void L_div(bigint::big_int_t &out, + const bigint::big_int_t &x, + const bigint::big_int_t &n) { + mpz_sub_ui(out.v, x.v, 1); + mpz_tdiv_q(out.v, out.v, n.v); +} + +// CRT: find r such that r ≡ a1 (mod m1), r ≡ a2 (mod m2). +static void crt_pair(const bigint::big_int_t &a1, const bigint::big_int_t &m1, + const bigint::big_int_t &a2, const bigint::big_int_t &m2, + bigint::big_int_t &r_out, bigint::big_int_t &M_out) { + using namespace bigint; + mpz_mul(M_out.v, m1.v, m2.v); + + big_int_t m1_mod_m2; + mpz_mod(m1_mod_m2.v, m1.v, m2.v); + + big_int_t inv; + if (!mpz_invert(inv.v, m1_mod_m2.v, m2.v)) { + throw std::runtime_error("crt_pair: inverse does not exist"); + } + + big_int_t t, diff; + mpz_sub(diff.v, a2.v, a1.v); + mpz_mod(diff.v, diff.v, m2.v); + mpz_mul(t.v, diff.v, inv.v); + mpz_mod(t.v, t.v, m2.v); + + big_int_t tmp; + mpz_mul(tmp.v, m1.v, t.v); + mpz_add(r_out.v, a1.v, tmp.v); + mpz_mod(r_out.v, r_out.v, M_out.v); +} + +// Multiply by (1+p) to inject an order-p component in Z_{p^2}*. +static bigint::big_int_t lift_to_p2_with_p_component(const bigint::big_int_t &g_mod_p, + const bigint::big_int_t &p) { + using namespace bigint; + big_int_t p2, one_plus_p, r; + mpz_mul(p2.v, p.v, p.v); + mpz_add_ui(one_plus_p.v, p.v, 1); + mpz_mul(r.v, g_mod_p.v, one_plus_p.v); + mpz_mod(r.v, r.v, p2.v); + return r; +} + +// gen_dsa_params_custom: generate (p,q,g) with p_bits for p and q_bits for q, and g of order q mod p. +static void gen_dsa_params_custom(unsigned p_bits, unsigned q_bits, + bigint::gmp_rand_ctx &rng, + bigint::big_int_t &p_out, + bigint::big_int_t &q_out, + bigint::big_int_t &g_out) { + using namespace bigint; + if (q_bits >= p_bits) { + throw std::invalid_argument("alpha_len too large: q_bits must be < p_bits"); + } + + // q: exact q_bits prime + big_int_t q = random_prime_bits(q_bits, rng); + + // choose k so that p = k*q + 1 has exactly p_bits bits + big_int_t two_pow_p_1, two_pow_p; + mpz_set_ui(two_pow_p_1.v, 1); + mpz_mul_2exp(two_pow_p_1.v, two_pow_p_1.v, p_bits - 1); + + mpz_set_ui(two_pow_p.v, 1); + mpz_mul_2exp(two_pow_p.v, two_pow_p.v, p_bits); + + big_int_t k_lo, k_hi, tmp; + mpz_tdiv_q(k_lo.v, two_pow_p_1.v, q.v); + + mpz_sub_ui(tmp.v, two_pow_p.v, 1); + mpz_tdiv_q(k_hi.v, tmp.v, q.v); + + if (mpz_odd_p(k_lo.v)) { + mpz_add_ui(k_lo.v, k_lo.v, 1); + } + + big_int_t p, span, t, k; + while (true) { + // span = (k_hi - k_lo) / 2 + 1 + mpz_sub(span.v, k_hi.v, k_lo.v); + mpz_tdiv_q_2exp(span.v, span.v, 1); + mpz_add_ui(span.v, span.v, 1); + + mpz_urandomm(t.v, rng.st, span.v); // t in [0, span-1] + mpz_mul_2exp(k.v, t.v, 1); // 2*t + mpz_add(k.v, k.v, k_lo.v); // k = k_lo + 2*t + + mpz_mul(p.v, k.v, q.v); + mpz_add_ui(p.v, p.v, 1); + + if (bitlen_mpz(p) != p_bits) { + continue; + } + if (mpz_probab_prime_p(p.v, 40) > 0) { + break; + } + } + + // generator of order q + big_int_t e; + mpz_sub_ui(e.v, p.v, 1); + mpz_tdiv_q(e.v, e.v, q.v); + + big_int_t h, g; + big_int_t p_minus2; + mpz_sub_ui(p_minus2.v, p.v, 2); + + while (true) { + // pick h in [2, p-1] + mpz_urandomm(h.v, rng.st, p_minus2.v); // 0..p-3 + mpz_add_ui(h.v, h.v, 2); // 2..p-1 + mpz_powm(g.v, h.v, e.v, p.v); + if (mpz_cmp_ui(g.v, 1) != 0) { + break; + } + } + + p_out = p; + q_out = q; + g_out = g; +} + +// ---------------------- CPU Paillier key structures ---------------------- +struct PaillierPublicKeyCPU { + bigint::big_int_t g; + bigint::big_int_t n; + bigint::big_int_t n_squared; +}; + +struct PaillierSecretKeyCPU { + bigint::big_int_t phi; + bigint::big_int_t inv; + // Lookup-optimized decryption exponent and inverse factor + bigint::big_int_t a; + bigint::big_int_t g_a_inv; +}; + +struct PaillierKeyPairCPU { + PaillierPublicKeyCPU pk; + PaillierSecretKeyCPU sk; +}; + +class PaillierCPU { +public: + static PaillierKeyPairCPU key_gen(unsigned prime_bits, unsigned alpha_bits) { + using namespace bigint; + gmp_rand_ctx rng; + while (true) { + // --- pick q-bit split close to requested alpha_len --- + unsigned q_bits_bound = alpha_bits; + unsigned q_bits_1 = q_bits_bound / 2; + unsigned q_bits_2 = q_bits_bound - q_bits_1; + + big_int_t p1, q1, g1; + big_int_t p2, q2, g2; + gen_dsa_params_custom(prime_bits, q_bits_1, rng, p1, q1, g1); + gen_dsa_params_custom(prime_bits, q_bits_2, rng, p2, q2, g2); + + big_int_t p = p1; + big_int_t q = p2; + + big_int_t n; + mpz_mul(n.v, p.v, q.v); + + big_int_t n2; + mpz_mul(n2.v, n.v, n.v); + + // Lift g1, g2 to p^2, q^2 so their orders gain factors p and q + big_int_t g1_p2 = lift_to_p2_with_p_component(g1, p); + big_int_t g2_q2 = lift_to_p2_with_p_component(g2, q); + + big_int_t p2_mod, q2_mod; + mpz_mul(p2_mod.v, p.v, p.v); + mpz_mul(q2_mod.v, q.v, q.v); + + big_int_t g, M_dummy; + crt_pair(g1_p2, p2_mod, g2_q2, q2_mod, g, M_dummy); + + // alpha = lcm(q1, q2) + big_int_t a; + mpz_lcm(a.v, q1.v, q2.v); + + // (sanity) ensure ord(g) | n*a and n-part is present + big_int_t n_mul_a; + mpz_mul(n_mul_a.v, n.v, a.v); + + big_int_t check1, check2; + mpz_powm(check1.v, g.v, n_mul_a.v, n2.v); + mpz_powm(check2.v, g.v, n.v, n2.v); + + if (mpz_cmp_ui(check1.v, 1) != 0 || mpz_cmp_ui(check2.v, 1) == 0) { + // extremely rare; try again + continue; + } + + big_int_t p_minus_1, q_minus_1; + mpz_sub_ui(p_minus_1.v, p.v, 1); + mpz_sub_ui(q_minus_1.v, q.v, 1); + + big_int_t phi; + mpz_mul(phi.v, p_minus_1.v, q_minus_1.v); + + big_int_t g_n; + mpz_powm(g_n.v, g.v, n.v, n2.v); + + big_int_t g_pow_a; + mpz_powm(g_pow_a.v, g.v, a.v, n2.v); + + big_int_t L_ga; + L_div(L_ga, g_pow_a, n); + + big_int_t g_a_inv; + if (!mpz_invert(g_a_inv.v, L_ga.v, n.v)) { + // extremely unlikely; regenerate keys + continue; + } + + big_int_t inv; + if (!mpz_invert(inv.v, phi.v, n.v)) { + // extremely unlikely; regenerate keys + continue; + } + + PaillierPublicKeyCPU pk{g, n, n2}; + PaillierSecretKeyCPU sk{phi, inv, a, g_a_inv}; + return PaillierKeyPairCPU{pk, sk}; + } + } +}; + +// ---------------------- Hex <-> limb conversion ---------------------- +static inline uint32_t hex_nibble(char c) { + if (c >= '0' && c <= '9') return (uint32_t)(c - '0'); + if (c >= 'a' && c <= 'f') return (uint32_t)(10 + c - 'a'); + if (c >= 'A' && c <= 'F') return (uint32_t)(10 + c - 'A'); + throw std::runtime_error("Invalid hex digit"); +} + +static std::string normalize_hex(const std::string &s) { + std::string h = s; + if (h.size() >= 2 && h[0] == '0' && (h[1] == 'x' || h[1] == 'X')) h = h.substr(2); + size_t p = 0; + while (p < h.size() && h[p] == '0') ++p; + if (p == h.size()) return "0"; + return h.substr(p); +} + +static std::vector hex_to_le_words(const std::string &hex_in, int words) { + std::string h = normalize_hex(hex_in); + if (h == "0") return std::vector(words, 0u); + if (h.size() & 1) h = "0" + h; + std::vector bytes(h.size() / 2); + for (size_t i = 0, j = 0; i < h.size(); i += 2, ++j) { + uint8_t hi = (uint8_t)hex_nibble(h[i]); + uint8_t lo = (uint8_t)hex_nibble(h[i + 1]); + bytes[j] = (uint8_t)((hi << 4) | lo); + } + std::vector words_le(words, 0u); + const size_t num_bytes = bytes.size(); + for (size_t i = 0; i < num_bytes; ++i) { + size_t limb_index = i / 4; + size_t byte_index = i % 4; + if (limb_index >= (size_t)words) break; + words_le[limb_index] |= ((uint32_t)bytes[num_bytes - 1 - i]) << (8 * byte_index); + } + return words_le; +} + +static std::string le_words_to_hex(const std::vector &words) { + std::vector bytes; + bytes.reserve(words.size() * 4); + for (uint32_t w : words) { + bytes.push_back((uint8_t)(w & 0xFF)); + bytes.push_back((uint8_t)((w >> 8) & 0xFF)); + bytes.push_back((uint8_t)((w >> 16) & 0xFF)); + bytes.push_back((uint8_t)((w >> 24) & 0xFF)); + } + while (!bytes.empty() && bytes.back() == 0) bytes.pop_back(); + if (bytes.empty()) return "0"; + static const char *HEX = "0123456789abcdef"; + std::string hex; + hex.resize(bytes.size() * 2); + size_t idx = 0; + for (int i = (int)bytes.size() - 1; i >= 0; --i) { + uint8_t b = bytes[i]; + hex[idx++] = HEX[(b >> 4) & 0xF]; + hex[idx++] = HEX[b & 0xF]; + } + size_t p = 0; + while (p + 1 < hex.size() && hex[p] == '0') ++p; + return hex.substr(p); +} + +// hex -> binary string (MSB-first), no fixed width; "0" for zero +static inline std::string hex_to_bin_nopad(const std::string &hex) { + auto hn = [](char c) -> int { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return 10 + (c - 'a'); + if (c >= 'A' && c <= 'F') return 10 + (c - 'A'); + return 0; + }; + static const char *LUT[16] = { + "0000", "0001", "0010", "0011", "0100", "0101", "0110", "0111", + "1000", "1001", "1010", "1011", "1100", "1101", "1110", "1111"}; + std::string out; + out.reserve(hex.size() * 4); + for (char c : hex) out.append(LUT[hn(c)]); + size_t p = out.find_first_not_of('0'); + if (p == std::string::npos) return "0"; + return out.substr(p); +} + +// ---------------------- CGBN types ---------------------- +typedef cgbn_context_t context_t; +typedef cgbn_env_t env_t; +typedef cgbn_error_report_t error_report_t; +typedef cgbn_mem_t mem_t; + +// Small set of global constants for decryption, shared across all threads +__device__ __constant__ mem_t CONST_N; +__device__ __constant__ mem_t CONST_N2; +__device__ __constant__ mem_t CONST_A; +__device__ __constant__ mem_t CONST_G_A_INV; + +// Convert a GMP big integer to our fixed-size mem_t (little-endian limbs) +static inline mem_t bigint_to_mem(const bigint::big_int_t &x) { + std::array words{}; + size_t count = 0; + mpz_export(words.data(), &count, -1, sizeof(uint32_t), 0, 0, x.v); + if (count > LIMBS) { + throw std::runtime_error("bigint_to_mem: value does not fit in mem_t"); + } + mem_t out{}; + std::memcpy(&out, words.data(), LIMBS * sizeof(uint32_t)); + return out; +} + +// Convert mem_t (little-endian limbs) to a GMP big integer +static inline bigint::big_int_t mem_to_bigint(const mem_t &mem) { + bigint::big_int_t out; + std::array words{}; + std::memcpy(words.data(), &mem, LIMBS * sizeof(uint32_t)); + mpz_import(out.v, LIMBS, -1, sizeof(uint32_t), 0, 0, words.data()); + return out; +} + +// ---------------------- Lookup-table precomputation (CPU) ---------------------- +// Build g_table[i][j] = g^(j * 2^(MSG_BITS * i)) mod n^2, flattened as +// [message_chunks * PAILLIER_MSG_TABLE_SIZE] of mem_t in little-endian limbs. +static std::vector +precompute_g_table(const PaillierKeyPairCPU &keys, int key_len_bits) { + using namespace bigint; + + const big_int_t &g = keys.pk.g; + const big_int_t &n = keys.pk.n; + const big_int_t &n2 = keys.pk.n_squared; + + const int msg_bits = PAILLIER_MSG_BITS; + const int message_chunks = (key_len_bits * 2 + msg_bits - 1) / msg_bits; + const int table_size = PAILLIER_MSG_TABLE_SIZE; + + std::vector table(static_cast(message_chunks) * + static_cast(table_size)); + + // For each chunk i, base = g^(2^(msg_bits * i)) mod n^2, then table[i][j] = base^j. + for (int i = 0; i < message_chunks; ++i) { + big_int_t exp; + mpz_set_ui(exp.v, 1); + mpz_mul_2exp(exp.v, exp.v, msg_bits * i); + + big_int_t base; + mpz_powm(base.v, g.v, exp.v, n2.v); + + big_int_t cur; + mpz_set_ui(cur.v, 1); // base^0 + + const size_t row_offset = static_cast(i) * static_cast(table_size); + for (int j = 0; j < table_size; ++j) { + table[row_offset + static_cast(j)] = bigint_to_mem(cur); + if (j + 1 < table_size) { + mpz_mul(cur.v, cur.v, base.v); + mpz_mod(cur.v, cur.v, n2.v); + } + } + } + + return table; +} + +// Build noise_table[k] = g_n^r mod n^2 for random r in [1, n-1] with gcd(r,n)=1, +// as mem_t entries. This mirrors Paillier_Lookup.precompute_noise_table. +static std::vector +precompute_noise_table(const PaillierKeyPairCPU &keys) { + using namespace bigint; + + const big_int_t &g = keys.pk.g; + const big_int_t &n = keys.pk.n; + const big_int_t &n2 = keys.pk.n_squared; + + gmp_rand_ctx rng; + + big_int_t g_n; + mpz_powm(g_n.v, g.v, n.v, n2.v); + + std::vector table(PAILLIER_NOISE_TABLE_SIZE); + + for (int k = 0; k < PAILLIER_NOISE_TABLE_SIZE; ++k) { + big_int_t r; + // Sample r uniformly in [1, n-1] with gcd(r, n) = 1. + while (true) { + mpz_urandomm(r.v, rng.st, n.v); // 0 <= r < n + if (mpz_cmp_ui(r.v, 0) == 0) { + continue; + } + big_int_t g_r; + mpz_gcd(g_r.v, r.v, n.v); + if (mpz_cmp_ui(g_r.v, 1) == 0) { + break; + } + } + + big_int_t noise; + mpz_powm(noise.v, g_n.v, r.v, n2.v); + table[static_cast(k)] = bigint_to_mem(noise); + } + + return table; +} + +// ---------------------- Host-side g_table cache ---------------------- +struct GTableCacheEntry { + std::shared_ptr> table; + std::list::iterator lru_it; +}; + +static std::mutex g_table_cache_mutex; +static std::unordered_map g_table_cache; +static std::list g_table_lru; +static constexpr size_t G_TABLE_CACHE_MAX = 4; + +static std::shared_ptr> +get_cached_g_table(const std::string &cache_key, const PaillierKeyPairCPU &keys, int key_len_bits) { + { + std::lock_guard lock(g_table_cache_mutex); + auto it = g_table_cache.find(cache_key); + if (it != g_table_cache.end()) { + g_table_lru.erase(it->second.lru_it); + g_table_lru.push_back(cache_key); + it->second.lru_it = std::prev(g_table_lru.end()); + return it->second.table; + } + } + + auto computed = std::make_shared>(precompute_g_table(keys, key_len_bits)); + + std::lock_guard lock(g_table_cache_mutex); + auto it = g_table_cache.find(cache_key); + if (it != g_table_cache.end()) { + return it->second.table; + } + g_table_lru.push_back(cache_key); + auto lru_it = std::prev(g_table_lru.end()); + g_table_cache.emplace(cache_key, GTableCacheEntry{computed, lru_it}); + if (g_table_cache.size() > G_TABLE_CACHE_MAX) { + const std::string &old_key = g_table_lru.front(); + g_table_cache.erase(old_key); + g_table_lru.pop_front(); + } + return computed; +} + + + +// ------------------ Arithmetic helpers ------------------ + +// --- RNG: xorshift64* --- +static __device__ __forceinline__ uint64_t xorshift64star(uint64_t &s) { + s ^= s >> 12; s ^= s << 25; s ^= s >> 27; + return s * 0x2545F4914F6CDD1DULL; +} + +static __device__ __forceinline__ uint32_t rand32(uint64_t &s) { + return (uint32_t)(xorshift64star(s) >> 32); +} + +// Extract pointer to 32-bit limbs from mem_t (little-endian) +static __device__ __forceinline__ const uint32_t* limbs_of(const mem_t *m) { + return reinterpret_cast(m); +} + +// Find MSB index (bit position) of exponent in mem_t (return -1 if zero) +static __device__ __forceinline__ int msb_index_mem(const mem_t *exp_mem) { + const uint32_t *w = limbs_of(exp_mem); + for (int i=LIMBS-1; i>=0; --i) { + uint32_t v = w[i]; + if (v != 0) { + // position of highest set bit within word + int lz = __clz(v); // 0..32 + int bit = 31 - lz; // 0..31 + return i*32 + bit; + } + } + return -1; +} + +// Square-and-multiply: result = base^exp_mem (mod mod_n2) +// exp_mem is provided as mem_t (n's limbs) +static __device__ void pow_mod_mem(env_t &env, + typename env_t::cgbn_t &result, + const typename env_t::cgbn_t &base, + const mem_t *exp_mem, + const typename env_t::cgbn_t &mod_n2) { + typename env_t::cgbn_t res, b; + typename env_t::cgbn_wide_t wide; + + cgbn_set_ui32(env, res, 1); + cgbn_set(env, b, base); + + const uint32_t *EW = reinterpret_cast(exp_mem); + + // Iterate all LIMBS*32 bits, LSB to MSB + for (int w = 0; w < LIMBS; ++w) { + uint32_t x = EW[w]; + for (int k = 0; k < 32; ++k) { + if (x & 1u) { + cgbn_mul_wide(env, wide, res, b); + cgbn_rem_wide(env, res, wide, mod_n2); + } + // b = b*b mod mod_n2 + cgbn_mul_wide(env, wide, b, b); + cgbn_rem_wide(env, b, wide, mod_n2); + x >>= 1; + } + } + cgbn_set(env, result, res); +} + +// Square-and-multiply specialized for a small exponent with dynamic bit length. +// Iterates only over exp_bits bits of exp_mem instead of the full LIMBS*32. +static __device__ void pow_mod_small_exp(env_t &env, + typename env_t::cgbn_t &result, + const typename env_t::cgbn_t &base, + const mem_t *exp_mem, + const typename env_t::cgbn_t &mod_n2, + int exp_bits) { + typename env_t::cgbn_t res, b; + typename env_t::cgbn_wide_t wide; + + cgbn_set_ui32(env, res, 1); + cgbn_set(env, b, base); + + const uint32_t *EW = reinterpret_cast(exp_mem); + + const int total_bits = exp_bits; + const int full_words = total_bits / 32; + const int rem_bits = total_bits % 32; + + // Process full 32-bit words + for (int w = 0; w < full_words; ++w) { + uint32_t x = EW[w]; + for (int k = 0; k < 32; ++k) { + if (x & 1u) { + cgbn_mul_wide(env, wide, res, b); + cgbn_rem_wide(env, res, wide, mod_n2); + } + cgbn_mul_wide(env, wide, b, b); + cgbn_rem_wide(env, b, wide, mod_n2); + x >>= 1; + } + } + + // Process remaining bits in the last word, if any + if (rem_bits > 0) { + uint32_t x = EW[full_words]; + for (int k = 0; k < rem_bits; ++k) { + if (x & 1u) { + cgbn_mul_wide(env, wide, res, b); + cgbn_rem_wide(env, res, wide, mod_n2); + } + cgbn_mul_wide(env, wide, b, b); + cgbn_rem_wide(env, b, wide, mod_n2); + x >>= 1; + } + } + + cgbn_set(env, result, res); +} + +// Euclidean GCD (returns gcd(a,b) in 'a'); destructive to inputs +static __device__ void gcd_big(env_t &env, + typename env_t::cgbn_t &a, + typename env_t::cgbn_t &b) { + typename env_t::cgbn_t t; + while (cgbn_compare_ui32(env, b, 0) != 0) { + cgbn_rem(env, t, a, b); + cgbn_set(env, a, b); + cgbn_set(env, b, t); + } +} + +// Sample r in [1, n-1] and (optionally) ensure gcd(r, n) = 1. +// Very unlikely to need retries. +static __device__ void sample_r(env_t &env, + typename env_t::cgbn_t &r, + const typename env_t::cgbn_t &n, + uint64_t seed, + int instance_id, + bool ensure_coprime=true) { + typename env_t::cgbn_t tmp, a, b; + typename env_t::cgbn_t n_minus_1; + cgbn_sub_ui32(env, n_minus_1, n, 1); + + // seed per instance + uint64_t s = seed ^ (0x9E3779B97F4A7C15ULL * (uint64_t)(instance_id + 1)) ^ (uint64_t)clock64(); + + for (int tries=0; tries<16; ++tries) { + // Fill a mem_t with random words + mem_t rmem; + uint32_t *rw = reinterpret_cast(&rmem); + for (int i=0; i ensures 1..n-1 + cgbn_load(env, tmp, &rmem); + cgbn_rem(env, tmp, tmp, n_minus_1); + cgbn_add_ui32(env, r, tmp, 1); + + if (!ensure_coprime) return; + + // Check gcd(r, n) == 1 + cgbn_set(env, a, n); + cgbn_set(env, b, r); + gcd_big(env, a, b); // gcd in 'a' + if (cgbn_compare_ui32(env, a, 1) == 0) + return; // good + } + + // Fallback: r=1 (extremely unlikely path) + cgbn_set_ui32(env, r, 1); +} + + + +// Build a mask that selects bits at MSB-first positions 1,3,5,... within width=chunk_len +// mapped to little-endian limb storage used by cgbn_mem_t. +static inline mem_t make_odd_mask_host(const int chunk_len) { + mem_t mask{}; + uint32_t *w = reinterpret_cast(&mask); + // zero init + for (int i=0;i> 5; + const int pos = lsb_index & 31; + w[idx] |= (1u << pos); + } + return mask; +} + + + + +// ---------------------- CUDA kernel ---------------------- +__global__ void modmul_pointwise_kernel(error_report_t *report, + mem_t *A, + mem_t *B, + mem_t *MOD, // single modulus at index 0 + mem_t *R, + int count) { + int thread = blockIdx.x * blockDim.x + threadIdx.x; + int instance = thread / CGBN_TPI; + if (instance >= count) return; + + context_t context(cgbn_report_monitor, report, instance); + env_t env(context); + + typename env_t::cgbn_t a, b, m, r; + typename env_t::cgbn_wide_t wide; // <<< wide 2*BITS + cgbn_load(env, a, &A[instance]); + cgbn_load(env, b, &B[instance]); + cgbn_load(env, m, &MOD[0]); + + // Use CGBN modular multiply: r = (a*b) mod m + cgbn_mul_wide(env, wide, a, b); // wide = a*b + cgbn_rem_wide(env, r, wide, m); // r = wide % m + + cgbn_store(env, &R[instance], r); +} + + +// Each instance handles one (vector, chunk) +__global__ void encrypt_kernel(error_report_t *report, + const uint8_t *all_bits, // [batch * embed_len], 0/1 + int batch, + int embed_len, + int chunk_len, + int chunk_num, + mem_t *N, // n + mem_t *N2, // n^2 + mem_t *G_TABLE, // [message_chunks * table_size] + mem_t *NOISE_TABLE, // [noise_table_size] + int message_chunks, + int noise_table_size, + mem_t *out_ct, // [batch * chunk_num] + uint64_t seed) { + int thread = blockIdx.x * blockDim.x + threadIdx.x; + int instance = thread / CGBN_TPI; + int total = batch * chunk_num; + if (instance >= total) return; + + int vec_idx = instance / chunk_num; + int chunk_idx = instance % chunk_num; + + context_t context(cgbn_report_monitor, report, instance); + env_t env(context); + + // Load n and n^2 into bigints + typename env_t::cgbn_t n, n2; + cgbn_load(env, n, &N[0]); + cgbn_load(env, n2, &N2[0]); + + // --- Build per-chunk base-2^MSG_BITS digits from padded bits --- + // Padded scheme: for original bit x[j], padded has at positions: 2*j -> 0, 2*j+1 -> x[j]. + const int padded_total = 2 * embed_len; // total padded bits for this vector + const int chunk_base = chunk_idx * chunk_len; // start bit in padded stream + const uint8_t *vec_bits = all_bits + vec_idx * embed_len; + + // Only consume the bits that actually exist in this chunk, do NOT extend to chunk_len + int avail = 0; + if (chunk_base < padded_total) { + int max_take = padded_total - chunk_base; + avail = (max_take > chunk_len) ? chunk_len : max_take; + } + + // Split the (at most avail)-bit chunk into MSG_BITS-wide digits, matching + // Python's behavior: for the local substring s = padded_embd[chunk_base:chunk_base+avail], + // interpret m_local = int(s, 2), then m_split[i] = (m_local >> (MSG_BITS*i)) & ((1< carries original bit + int j = p >> 1; // source index in the embedding + bit = (uint32_t)(vec_bits[j] & 1); + } + if (!bit) continue; + + // position within this chunk's substring, MSB-first + int local_e = (local_len - 1) - t; // exponent in 2^local_e + int digit_idx = local_e / msg_bits; + int offset = local_e % msg_bits; + if (digit_idx >= 0 && digit_idx < message_chunks) { + m_split_local[digit_idx] |= (uint8_t)(1u << offset); + } + } + + // --- Compute g^m via lookup table: g_m = prod_i g_table[i][m_split_local[i]] mod n^2 --- + typename env_t::cgbn_t g_m; + typename env_t::cgbn_wide_t wide; + cgbn_set_ui32(env, g_m, 1); + + for (int i = 0; i < message_chunks; ++i) { + uint8_t d = m_split_local[i]; + if (d == 0) continue; // g_table[i][0] == 1 + int idx = i * table_size + (int)d; + typename env_t::cgbn_t entry; + cgbn_load(env, entry, &G_TABLE[idx]); + cgbn_mul_wide(env, wide, g_m, entry); + cgbn_rem_wide(env, g_m, wide, n2); + } + + // --- Sample noise using precomputed NOISE_TABLE entries --- + typename env_t::cgbn_t noise; + cgbn_set_ui32(env, noise, 1); + + // Simple per-instance RNG seed, reusing xorshift helper. + uint64_t s = seed ^ (0x9E3779B97F4A7C15ULL * (uint64_t)(instance + 1)) ^ (uint64_t)clock64(); + for (int k = 0; k < PAILLIER_NOISE_MULTIPLES; ++k) { + uint32_t r32 = rand32(s); + int idx = (int)(r32 % (uint32_t)noise_table_size); + typename env_t::cgbn_t entry; + cgbn_load(env, entry, &NOISE_TABLE[idx]); + cgbn_mul_wide(env, wide, noise, entry); + cgbn_rem_wide(env, noise, wide, n2); + } + + // --- c = g_m * noise mod n^2 --- + typename env_t::cgbn_t c; + cgbn_mul_wide(env, wide, g_m, noise); + cgbn_rem_wide(env, c, wide, n2); + + // Store result + cgbn_store(env, &out_ct[instance], c); +} + +// Each instance handles one plaintext integer (already split into MSG_BITS digits) +__global__ void encrypt_cts_kernel(error_report_t *report, + const uint8_t *all_digits, // [batch * message_chunks] + int batch, + int message_chunks, + mem_t *N, // n + mem_t *N2, // n^2 + mem_t *G_TABLE, // [message_chunks * table_size] + mem_t *NOISE_TABLE, // [noise_table_size] + int noise_table_size, + mem_t *out_ct, // [batch] + uint64_t seed) { + int thread = blockIdx.x * blockDim.x + threadIdx.x; + int instance = thread / CGBN_TPI; + if (instance >= batch) return; + + context_t context(cgbn_report_monitor, report, instance); + env_t env(context); + + // Load n and n^2 into bigints + typename env_t::cgbn_t n, n2; + cgbn_load(env, n, &N[0]); + cgbn_load(env, n2, &N2[0]); + + const int table_size = PAILLIER_MSG_TABLE_SIZE; + + // --- Compute g^m via lookup table: g_m = prod_i g_table[i][digit_i] mod n^2 --- + typename env_t::cgbn_t g_m; + typename env_t::cgbn_wide_t wide; + cgbn_set_ui32(env, g_m, 1); + + const uint8_t *digits = all_digits + instance * message_chunks; + for (int i = 0; i < message_chunks; ++i) { + uint8_t d = digits[i]; + if (d == 0) continue; // g_table[i][0] == 1 + int idx = i * table_size + (int)d; + typename env_t::cgbn_t entry; + cgbn_load(env, entry, &G_TABLE[idx]); + cgbn_mul_wide(env, wide, g_m, entry); + cgbn_rem_wide(env, g_m, wide, n2); + } + + // --- Sample noise using precomputed NOISE_TABLE entries --- + typename env_t::cgbn_t noise; + cgbn_set_ui32(env, noise, 1); + + uint64_t s = seed ^ (0x9E3779B97F4A7C15ULL * (uint64_t)(instance + 1)) ^ (uint64_t)clock64(); + for (int k = 0; k < PAILLIER_NOISE_MULTIPLES; ++k) { + uint32_t r32 = rand32(s); + int idx = (int)(r32 % (uint32_t)noise_table_size); + typename env_t::cgbn_t entry; + cgbn_load(env, entry, &NOISE_TABLE[idx]); + cgbn_mul_wide(env, wide, noise, entry); + cgbn_rem_wide(env, noise, wide, n2); + } + + // --- c = g_m * noise mod n^2 --- + typename env_t::cgbn_t c; + cgbn_mul_wide(env, wide, g_m, noise); + cgbn_rem_wide(env, c, wide, n2); + + cgbn_store(env, &out_ct[instance], c); +} + +// ---------- GPU: decrypt-only kernel (per-chunk) ---------- +__global__ void decrypt_only_kernel(error_report_t *report, + mem_t *CT, // [total] ciphertexts + int total, // batch * chunk_num + mem_t *OUT_M, // [total] plaintext m + int exp_bits) { + const int thread = blockIdx.x*blockDim.x + threadIdx.x; + const int instance = thread / CGBN_TPI; + if (instance >= total) return; + + typedef cgbn_context_t context_t; + context_t context(cgbn_report_monitor, report, instance); + env_t env(context); + + typename env_t::cgbn_t n, n2, a, g_a_inv, c, t, x, q, rrem, m; + typename env_t::cgbn_wide_t wide; + + cgbn_load(env, c, &CT[instance]); + cgbn_load(env, n, &CONST_N); + cgbn_load(env, n2, &CONST_N2); + cgbn_load(env, a, &CONST_A); + cgbn_load(env, g_a_inv, &CONST_G_A_INV); + + // t = c^a mod n^2 (LSB-first exp), using small exponent loop + pow_mod_small_exp(env, t, c, &CONST_A, n2, exp_bits); + + // x = t - 1, then exact div by n: x = q*n + rrem + cgbn_sub_ui32(env, x, t, 1); + cgbn_div(env, q, x, n); + cgbn_rem(env, rrem, x, n); // for debug you can check rrem==0 + // q = L(t) = (t - 1) / n + + // m = (q * g_a_inv) mod n + cgbn_mul_wide(env, wide, q, g_a_inv); + cgbn_rem_wide(env, m, wide, n); + + cgbn_store(env, &OUT_M[instance], m); +} + +// Fused decrypt-then-reencrypt kernel (one ciphertext per instance) +__global__ void decrypt_then_reencrypt_kernel(error_report_t *report, + mem_t *CT, // [batch] ciphertexts under key_a + int batch, + mem_t *N_a, // key_a n + mem_t *N2_a, // key_a n^2 + mem_t *A_a, // key_a a + mem_t *G_A_INV_a, // key_a g_a_inv + int exp_bits_a, + mem_t *N2_b, // key_b n^2 + mem_t *G_TABLE_b, // [message_chunks * table_size] + mem_t *NOISE_TABLE_b, // [noise_table_size] + int message_chunks, + int noise_table_size, + mem_t *OUT_CT, // [batch] ciphertexts under key_b + uint64_t seed) { + int thread = blockIdx.x * blockDim.x + threadIdx.x; + int instance = thread / CGBN_TPI; + if (instance >= batch) return; + + context_t context(cgbn_report_monitor, report, instance); + env_t env(context); + + typename env_t::cgbn_t n_a, n2_a, a_a, g_a_inv_a; + typename env_t::cgbn_t c_in, t, x, q, rrem, m; + typename env_t::cgbn_wide_t wide; + + cgbn_load(env, c_in, &CT[instance]); + cgbn_load(env, n_a, &N_a[0]); + cgbn_load(env, n2_a, &N2_a[0]); + cgbn_load(env, a_a, &A_a[0]); + cgbn_load(env, g_a_inv_a, &G_A_INV_a[0]); + + // Decrypt: t = c^a mod n^2 + pow_mod_small_exp(env, t, c_in, A_a, n2_a, exp_bits_a); + + // x = t - 1, then x = q*n + rrem + cgbn_sub_ui32(env, x, t, 1); + cgbn_div(env, q, x, n_a); + cgbn_rem(env, rrem, x, n_a); + + // m = (q * g_a_inv) mod n + cgbn_mul_wide(env, wide, q, g_a_inv_a); + cgbn_rem_wide(env, m, wide, n_a); + + // Extract plaintext digits (base 2^MSG_BITS) from m + const int table_size = PAILLIER_MSG_TABLE_SIZE; + + // Load n2_b for encryption + typename env_t::cgbn_t n2_b; + cgbn_load(env, n2_b, &N2_b[0]); + + // Compute g^m via lookup table + typename env_t::cgbn_t g_m; + cgbn_set_ui32(env, g_m, 1); + for (int i = 0; i < message_chunks; ++i) { + uint32_t d = env.extract_bits_ui32(m, i * PAILLIER_MSG_BITS, PAILLIER_MSG_BITS); + if (d == 0) continue; + int idx = i * table_size + (int)d; + typename env_t::cgbn_t entry; + cgbn_load(env, entry, &G_TABLE_b[idx]); + cgbn_mul_wide(env, wide, g_m, entry); + cgbn_rem_wide(env, g_m, wide, n2_b); + } + + // Sample noise using precomputed table + typename env_t::cgbn_t noise; + cgbn_set_ui32(env, noise, 1); + uint64_t s = seed ^ (0x9E3779B97F4A7C15ULL * (uint64_t)(instance + 1)) ^ (uint64_t)clock64(); + for (int k = 0; k < PAILLIER_NOISE_MULTIPLES; ++k) { + uint32_t r32 = rand32(s); + int idx = (int)(r32 % (uint32_t)noise_table_size); + typename env_t::cgbn_t entry; + cgbn_load(env, entry, &NOISE_TABLE_b[idx]); + cgbn_mul_wide(env, wide, noise, entry); + cgbn_rem_wide(env, noise, wide, n2_b); + } + + // c = g_m * noise mod n^2 + typename env_t::cgbn_t c_out; + cgbn_mul_wide(env, wide, g_m, noise); + cgbn_rem_wide(env, c_out, wide, n2_b); + + cgbn_store(env, &OUT_CT[instance], c_out); +} + + + +// Fused decrypt + odd-bit popcount kernel +__global__ void decode_hamming_fused_kernel(error_report_t *report, + mem_t *CT, // [total] ciphertexts + int total, // batch * chunk_num + int chunk_len, // = 2*key_len + const mem_t *ODD_MASK, + int words_used, + int exp_bits, + uint32_t *out_odd) { + const int thread = blockIdx.x * blockDim.x + threadIdx.x; + const int instance = thread / CGBN_TPI; + if (instance >= total) return; + + typedef cgbn_context_t context_t; + context_t context(cgbn_report_monitor, report, instance); + env_t env(context); + + // Load keys from constant memory + typename env_t::cgbn_t n, n2, a, g_a_inv, c, t, x, q, rrem, m; + typename env_t::cgbn_wide_t wide; + + cgbn_load(env, c, &CT[instance]); + cgbn_load(env, n, &CONST_N); + cgbn_load(env, n2, &CONST_N2); + cgbn_load(env, a, &CONST_A); + cgbn_load(env, g_a_inv, &CONST_G_A_INV); + + // Decrypt: t = c^a mod n^2 using small exponent loop + pow_mod_small_exp(env, t, c, &CONST_A, n2, exp_bits); + + // x = t - 1, then exact div by n: x = q*n + rrem + cgbn_sub_ui32(env, x, t, 1); + cgbn_div(env, q, x, n); + cgbn_rem(env, rrem, x, n); // rrem should be 0; q = L(t) + + // m = (q * g_a_inv) mod n + cgbn_mul_wide(env, wide, q, g_a_inv); + cgbn_rem_wide(env, m, wide, n); + + // Count odd bit positions via mask + popcount + mem_t mmem; + cgbn_store(env, &mmem, m); + + const uint32_t *mw = reinterpret_cast(&mmem); + const uint32_t *ow = reinterpret_cast(&ODD_MASK[0]); + + uint32_t odd_cnt = 0; + #pragma unroll + for (int w = 0; w < words_used; ++w) { + odd_cnt += __popc(mw[w] & ow[w]); + } + + out_odd[instance] = odd_cnt; +} + + +// Each instance handles one (vector, chunk) ciphertext -> produces count of ones at odd bit positions +__global__ void decode_hamming_kernel(error_report_t *report, + mem_t *CT, // [total] ciphertexts + int total, // batch * chunk_num + int chunk_len, // = 2*key_len + mem_t *N, mem_t *N2, + mem_t *A, mem_t *G_A_INV, + uint32_t *out_odd, // <-- NEW name + uint32_t *out_even, // <-- NEW + uint32_t *out_rem_nz, // <-- NEW + mem_t *out_m_dbg, // <-- OPTIONAL (can pass nullptr) + /* Additional parameters for GPU mask inputs for bitcounting*/ + const mem_t *ODD_MASK, + int words_used, + int exp_bits) +{ + + int thread = blockIdx.x * blockDim.x + threadIdx.x; + int instance = thread / CGBN_TPI; + if (instance >= total) return; + + context_t context(cgbn_report_monitor, report, instance); + env_t env(context); + + // Load inputs + typename env_t::cgbn_t n, n2, a, g_a_inv, c; + cgbn_load(env, n, &N[0]); + cgbn_load(env, n2, &N2[0]); + cgbn_load(env, a, &A[0]); + cgbn_load(env, g_a_inv, &G_A_INV[0]); + cgbn_load(env, c, &CT[instance]); + + // Decrypt: t = c^a mod n^2 + typename env_t::cgbn_t t, x, q, rrem, m; +typename env_t::cgbn_wide_t wide; + +pow_mod_small_exp(env, t, c, A, n2, exp_bits); // t = c^a mod n^2 +pow_mod_small_exp(env, t, c, A, n2, exp_bits); // t = c^a mod n^2 +// x = t - 1 (t is in [1, n^2-1], so this is >= 0) +cgbn_sub_ui32(env, x, t, 1); + +// Divide exactly: x = q*n + rrem (rrem must be 0) +cgbn_div(env, q, x, n); +cgbn_rem(env, rrem, x, n); +uint32_t rem_nz = (cgbn_compare_ui32(env, rrem, 0) != 0); + +// q = L(t) = (t - 1) / n + +// m = (q * g_a_inv) mod n +cgbn_mul_wide(env, wide, q, g_a_inv); +cgbn_rem_wide(env, m, wide, n); + +// Store m for host-side inspection if requested +if (out_m_dbg != nullptr) { + cgbn_store(env, &out_m_dbg[instance], m); +} + +// Count odd/even bit positions MSB-first over chunk_len +// ---- PURE GPU COUNTING via mask+popcount ---- + mem_t mmem; + cgbn_store(env, &mmem, m); + + const uint32_t *mw = reinterpret_cast(&mmem); + const uint32_t *ow = reinterpret_cast(&ODD_MASK[0]); + + uint32_t odd_cnt = 0; + #pragma unroll + for (int w=0; w= total) return; + + mem_t mmem = M[instance]; + const uint32_t *mw = reinterpret_cast(&mmem); + const uint32_t *ow = reinterpret_cast(&ODD_MASK[0]); + + uint32_t odd_cnt = 0; + #pragma unroll + for (int w = 0; w < words_used; ++w) { + odd_cnt += __popc(mw[w] & ow[w]); + } + out_odd[instance] = odd_cnt; +} + +// ---------------------- Python <-> GMP helpers ---------------------- +static bigint::big_int_t big_from_py(py::handle obj) { + std::string s = py::cast(py::str(obj)); + size_t l = s.find_first_not_of(" \t\n\r"); + size_t r = s.find_last_not_of(" \t\n\r"); + if (l == std::string::npos) { + s = "0"; + } else { + s = s.substr(l, r - l + 1); + } + int base = 10; + if (s.size() > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) { + base = 16; + s = s.substr(2); + } else if (s.size() > 3 && s[0] == '-' && s[1] == '0' && + (s[2] == 'x' || s[2] == 'X')) { + base = 16; + s = "-" + s.substr(3); + } + return bigint::big_int_t(s, base); +} + +static py::int_ py_int_from_big(const bigint::big_int_t &x) { + std::string dec = x.to_string(10); + PyObject *pobj = PyLong_FromString(const_cast(dec.c_str()), nullptr, 10); + if (!pobj) { + throw py::error_already_set(); + } + return py::reinterpret_steal(pobj); +} + +static py::object py_mpz_from_big(const bigint::big_int_t &x) { + static py::object mpz_ctor = py::module_::import("gmpy2").attr("mpz"); + return mpz_ctor(py::str(x.to_string(10))); +} + +static py::dict extract_dict_field(const py::dict &d, const char *field) { + if (d.contains(field)) { + return py::cast(d[field]); + } + return d; +} + +static PaillierPublicKeyCPU pk_from_py(const py::dict &d_in) { + py::dict pk = extract_dict_field(d_in, "pk"); + if (!pk.contains("g") || !pk.contains("n") || !pk.contains("n_squared")) { + throw std::runtime_error("key pk must contain g, n, n_squared"); + } + PaillierPublicKeyCPU pk_out; + pk_out.g = big_from_py(pk["g"]); + pk_out.n = big_from_py(pk["n"]); + pk_out.n_squared = big_from_py(pk["n_squared"]); + return pk_out; +} + +static PaillierSecretKeyCPU sk_from_py(const py::dict &d_in) { + py::dict sk = extract_dict_field(d_in, "sk"); + if (!sk.contains("a") || !sk.contains("g_a_inv")) { + throw std::runtime_error("key sk must contain a and g_a_inv"); + } + PaillierSecretKeyCPU sk_out; + sk_out.a = big_from_py(sk["a"]); + sk_out.g_a_inv = big_from_py(sk["g_a_inv"]); + if (sk.contains("phi")) sk_out.phi = big_from_py(sk["phi"]); + if (sk.contains("inv")) sk_out.inv = big_from_py(sk["inv"]); + return sk_out; +} + +static int message_chunks_from_key(const py::dict &kb, const bigint::big_int_t &n) { + if (kb.contains("message_chunks")) { + return py::cast(kb["message_chunks"]); + } + if (kb.contains("key_len")) { + int key_len_bits = py::cast(kb["key_len"]); + return (key_len_bits * 2 + PAILLIER_MSG_BITS - 1) / PAILLIER_MSG_BITS; + } + unsigned n_bits = static_cast(mpz_sizeinbase(n.v, 2)); + return (static_cast(n_bits) + PAILLIER_MSG_BITS - 1) / PAILLIER_MSG_BITS; +} + +static std::vector g_table_from_py(const py::handle &obj, int message_chunks) { + const int table_size = PAILLIER_MSG_TABLE_SIZE; + std::vector table(static_cast(message_chunks) * static_cast(table_size)); + if (obj.is_none()) return {}; + + auto load_row = [&](int i, const py::handle &row_obj) { + py::sequence row = py::cast(row_obj); + if (static_cast(py::len(row)) != table_size) { + throw std::runtime_error("g_table row length mismatch"); + } + size_t base = static_cast(i) * static_cast(table_size); + for (int j = 0; j < table_size; ++j) { + table[base + static_cast(j)] = bigint_to_mem(big_from_py(row[j])); + } + }; + + if (py::isinstance(obj)) { + py::dict d = py::reinterpret_borrow(obj); + for (int i = 0; i < message_chunks; ++i) { + py::object row_obj; + py::int_ key_i(i); + if (d.contains(key_i)) { + row_obj = d[key_i]; + } else { + py::str key_s(std::to_string(i)); + if (!d.contains(key_s)) { + throw std::runtime_error("g_table missing row"); + } + row_obj = d[key_s]; + } + load_row(i, row_obj); + } + return table; + } + + if (py::isinstance(obj)) { + py::sequence outer = py::cast(obj); + if (static_cast(py::len(outer)) != message_chunks) { + throw std::runtime_error("g_table outer length mismatch"); + } + for (int i = 0; i < message_chunks; ++i) { + load_row(i, outer[i]); + } + return table; + } + + throw std::runtime_error("g_table must be dict or sequence"); +} + +static std::vector noise_table_from_py(const py::handle &obj) { + if (obj.is_none()) return {}; + py::sequence seq = py::cast(obj); + const int size = static_cast(py::len(seq)); + if (size <= 0) return {}; + std::vector table(static_cast(size)); + for (int i = 0; i < size; ++i) { + table[static_cast(i)] = bigint_to_mem(big_from_py(seq[i])); + } + return table; +} + + +// ---------------------- Host glue helpers ---------------------- +static std::vector gpu_pointwise_mulmod_hex(const std::vector &ct1_hex, + const std::vector &ct2_hex, + const std::string &n_squared_hex) { + if (ct1_hex.size() != ct2_hex.size()) { + throw std::runtime_error("encode_hamming_*: ct1 and ct2 must have the same length"); + } + const int count = static_cast(ct1_hex.size()); + if (count == 0) return {}; + + std::vector h_A(count), h_B(count); + std::vector mod_words = hex_to_le_words(n_squared_hex, LIMBS); + mem_t h_MOD_one{}; + std::memcpy(&h_MOD_one, mod_words.data(), LIMBS * sizeof(uint32_t)); + + for (int i = 0; i < count; ++i) { + auto a = hex_to_le_words(ct1_hex[i], LIMBS); + auto b = hex_to_le_words(ct2_hex[i], LIMBS); + std::memcpy(&h_A[i], a.data(), LIMBS * sizeof(uint32_t)); + std::memcpy(&h_B[i], b.data(), LIMBS * sizeof(uint32_t)); + } + + error_report_t *d_report = nullptr; + mem_t *d_A = nullptr, *d_B = nullptr, *d_MOD = nullptr, *d_R = nullptr; + + CUDA_CHECK(cudaMalloc((void **)&d_report, sizeof(error_report_t))); + CUDA_CHECK(cudaMemset(d_report, 0, sizeof(error_report_t))); + + CUDA_CHECK(cudaMalloc((void **)&d_A, count * sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_B, count * sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_R, count * sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_MOD, sizeof(mem_t))); + + CUDA_CHECK(cudaMemcpy(d_A, h_A.data(), count * sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMemcpy(d_B, h_B.data(), count * sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMemcpy(d_MOD, &h_MOD_one, sizeof(mem_t), cudaMemcpyHostToDevice)); + + const int INSTS_PER_BLK = 8; + const int THREADS = INSTS_PER_BLK * CGBN_TPI; + const int blocks = (count + INSTS_PER_BLK - 1) / INSTS_PER_BLK; + + modmul_pointwise_kernel<<>>(d_report, d_A, d_B, d_MOD, d_R, count); + CUDA_CHECK(cudaPeekAtLastError()); + CUDA_CHECK(cudaDeviceSynchronize()); + + std::vector h_R(count); + CUDA_CHECK(cudaMemcpy(h_R.data(), d_R, count * sizeof(mem_t), cudaMemcpyDeviceToHost)); + + cudaFree(d_A); + cudaFree(d_B); + cudaFree(d_R); + cudaFree(d_MOD); + cudaFree(d_report); + + std::vector out(count); + for (int i = 0; i < count; ++i) { + std::vector words(LIMBS); + std::memcpy(words.data(), &h_R[i], LIMBS * sizeof(uint32_t)); + out[i] = le_words_to_hex(words); + } + return out; +} + +static std::vector +gpu_pointwise_mulmod_bigint(const std::vector &ct1, + const std::vector &ct2, + const bigint::big_int_t &n_squared) { + if (ct1.size() != ct2.size()) { + throw std::runtime_error("encode_hamming_*: ct1 and ct2 must have the same length"); + } + const int count = static_cast(ct1.size()); + if (count == 0) return {}; + + std::vector h_A(count), h_B(count); + mem_t h_MOD_one = bigint_to_mem(n_squared); + + for (int i = 0; i < count; ++i) { + h_A[i] = bigint_to_mem(ct1[i]); + h_B[i] = bigint_to_mem(ct2[i]); + } + + error_report_t *d_report = nullptr; + mem_t *d_A = nullptr, *d_B = nullptr, *d_MOD = nullptr, *d_R = nullptr; + + CUDA_CHECK(cudaMalloc((void **)&d_report, sizeof(error_report_t))); + CUDA_CHECK(cudaMemset(d_report, 0, sizeof(error_report_t))); + + CUDA_CHECK(cudaMalloc((void **)&d_A, count * sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_B, count * sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_R, count * sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_MOD, sizeof(mem_t))); + + CUDA_CHECK(cudaMemcpy(d_A, h_A.data(), count * sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMemcpy(d_B, h_B.data(), count * sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMemcpy(d_MOD, &h_MOD_one, sizeof(mem_t), cudaMemcpyHostToDevice)); + + const int INSTS_PER_BLK = 8; + const int THREADS = INSTS_PER_BLK * CGBN_TPI; + const int blocks = (count + INSTS_PER_BLK - 1) / INSTS_PER_BLK; + + modmul_pointwise_kernel<<>>(d_report, d_A, d_B, d_MOD, d_R, count); + CUDA_CHECK(cudaPeekAtLastError()); + CUDA_CHECK(cudaDeviceSynchronize()); + + std::vector h_R(count); + CUDA_CHECK(cudaMemcpy(h_R.data(), d_R, count * sizeof(mem_t), cudaMemcpyDeviceToHost)); + + cudaFree(d_A); + cudaFree(d_B); + cudaFree(d_R); + cudaFree(d_MOD); + cudaFree(d_report); + + std::vector out; + out.reserve(static_cast(count)); + for (int i = 0; i < count; ++i) { + out.emplace_back(mem_to_bigint(h_R[i])); + } + return out; +} + +// ---------------------- PaillierGPUClient class ---------------------- +class PaillierGPULookupClient { +public: + PaillierGPULookupClient(int embed_len = 512, + int key_len = KEY_BITS, + int alpha_len = ALPHA_LEN, + bool skip_key_gen = false) + : embed_len_(embed_len), + key_len_(key_len), + chunk_len_(key_len * 2), + chunk_num_(embed_len > key_len ? ((2 * embed_len) / (key_len * 2) + + (((2 * embed_len) % (key_len * 2)) != 0)) + : 1), + alpha_len_(alpha_len), + a_bits_(alpha_len), + have_keys_(!skip_key_gen), + message_chunks_(0), + tables_ready_(!skip_key_gen), + device_tables_ready_(false), + d_g_table_(nullptr), + d_noise_table_(nullptr), + d_g_table_entries_(0), + d_noise_table_entries_(0) { + if (!skip_key_gen) { + keys_ = PaillierCPU::key_gen(static_cast(key_len_), + static_cast(alpha_len_)); + pk_g_hex_ = keys_.pk.g.to_string(16); + pk_n_hex_ = keys_.pk.n.to_string(16); + pk_n2_hex_ = keys_.pk.n_squared.to_string(16); + sk_phi_hex_ = keys_.sk.phi.to_string(16); + sk_inv_hex_ = keys_.sk.inv.to_string(16); + sk_a_hex_ = keys_.sk.a.to_string(16); + sk_g_a_inv_hex_ = keys_.sk.g_a_inv.to_string(16); + a_bits_ = static_cast(mpz_sizeinbase(keys_.sk.a.v, 2)); + message_chunks_ = (key_len_ * 2 + PAILLIER_MSG_BITS - 1) / PAILLIER_MSG_BITS; + g_table_ = std::make_shared>(precompute_g_table(keys_, key_len_)); + noise_table_ = precompute_noise_table(keys_); + // Light debug: print generated keys (hex) + // std::cerr << "[PaillierGPUClient] Generated keys (hex):\n" + // << " n = " << pk_n_hex_ << "\n" + // << " n_squared= " << pk_n2_hex_ << "\n" + // << " phi = " << sk_phi_hex_ << "\n" + // << " inv = " << sk_inv_hex_ << std::endl; + } + } + + ~PaillierGPULookupClient() { + free_device_tables_(); + } + + int embed_len() const { return embed_len_; } + int key_len() const { return key_len_; } + int chunk_len() const { return chunk_len_; } + int chunk_num() const { return chunk_num_; } + + // Expose public key components as hex for Python-side checks + std::pair get_pk_hex() const { + if (!have_keys_) { + throw std::runtime_error("get_pk_hex: keys not initialized"); + } + return {pk_n_hex_, pk_n2_hex_}; + } + + // Expose full key components as hex: (n, n_squared, phi, inv) + std::tuple get_keys_hex() const { + if (!have_keys_) { + throw std::runtime_error("get_keys_hex: keys not initialized"); + } + return {pk_n_hex_, pk_n2_hex_, sk_phi_hex_, sk_inv_hex_}; + } + + // JSON helpers for interoperability with the Python lookup client + std::string stringify_pk() const { + if (!have_keys_) { + throw std::runtime_error("stringify_pk: keys not initialized"); + } + using namespace bigint; + py::object json = py::module_::import("json"); + + // Compute g_n = g^n mod n^2 + big_int_t g_n; + mpz_powm(g_n.v, keys_.pk.g.v, keys_.pk.n.v, keys_.pk.n_squared.v); + + py::dict pk_dict; + auto to_py_int = [](const big_int_t &x) { + std::string dec = x.to_string(10); + py::str s_dec(dec); + return py::int_(s_dec); + }; + + pk_dict["g"] = to_py_int(keys_.pk.g); + pk_dict["n"] = to_py_int(keys_.pk.n); + pk_dict["n_squared"] = to_py_int(keys_.pk.n_squared); + pk_dict["g_n"] = to_py_int(g_n); + + py::object s = json.attr("dumps")(pk_dict); + return py::cast(s); + } + + std::string stringify_sk() const { + if (!have_keys_) { + throw std::runtime_error("stringify_sk: keys not initialized"); + } + using namespace bigint; + py::object json = py::module_::import("json"); + + py::dict sk_dict; + auto to_py_int = [](const big_int_t &x) { + std::string dec = x.to_string(10); + py::str s_dec(dec); + return py::int_(s_dec); + }; + + sk_dict["phi"] = to_py_int(keys_.sk.phi); + sk_dict["a"] = to_py_int(keys_.sk.a); + sk_dict["g_a_inv"] = to_py_int(keys_.sk.g_a_inv); + + py::object s = json.attr("dumps")(sk_dict); + return py::cast(s); + } + + std::string stringify_config() const { + py::object json = py::module_::import("json"); + py::dict cfg; + cfg["message_chunks"] = py::int_(message_chunks_); + cfg["alpha_len"] = py::int_(alpha_len_); + cfg["embed_len"] = py::int_(embed_len_); + cfg["key_len"] = py::int_(key_len_); + py::object s = json.attr("dumps")(cfg); + return py::cast(s); + } + + void load_stringified_keys(const std::string &pk_json, + const std::string &sk_json) { + using namespace bigint; + py::object json = py::module_::import("json"); + + // Parse pk + py::dict pk_dict = json.attr("loads")(pk_json).cast(); + + auto get_big_int = [](const py::handle &v) -> big_int_t { + py::str s_dec = py::str(v); + std::string s = py::cast(s_dec); + return big_int_t(s, 10); + }; + + big_int_t g = get_big_int(pk_dict["g"]); + big_int_t n = get_big_int(pk_dict["n"]); + big_int_t n2 = get_big_int(pk_dict["n_squared"]); + + // Parse sk + py::dict sk_dict = json.attr("loads")(sk_json).cast(); + big_int_t phi = get_big_int(sk_dict["phi"]); + big_int_t a = get_big_int(sk_dict["a"]); + big_int_t g_a_inv = get_big_int(sk_dict["g_a_inv"]); + + // Recompute inv = phi^{-1} mod n + big_int_t inv; + if (!mpz_invert(inv.v, phi.v, n.v)) { + throw std::runtime_error("load_stringified_keys: phi^{-1} mod n does not exist"); + } + + // Install keys + keys_.pk.g = g; + keys_.pk.n = n; + keys_.pk.n_squared = n2; + keys_.sk.phi = phi; + keys_.sk.inv = inv; + keys_.sk.a = a; + keys_.sk.g_a_inv = g_a_inv; + + pk_g_hex_ = keys_.pk.g.to_string(16); + pk_n_hex_ = keys_.pk.n.to_string(16); + pk_n2_hex_ = keys_.pk.n_squared.to_string(16); + sk_phi_hex_ = keys_.sk.phi.to_string(16); + sk_inv_hex_ = keys_.sk.inv.to_string(16); + sk_a_hex_ = keys_.sk.a.to_string(16); + sk_g_a_inv_hex_ = keys_.sk.g_a_inv.to_string(16); + a_bits_ = static_cast(mpz_sizeinbase(keys_.sk.a.v, 2)); + alpha_len_ = a_bits_; + + g_table_.reset(); + noise_table_.clear(); + tables_ready_ = false; + free_device_tables_(); + + have_keys_ = true; + } + + void load_config(const py::dict &config) { + if (!have_keys_) { + throw std::runtime_error("load_config: keys must be loaded first"); + } + + embed_len_ = py::cast(config["embed_len"]); + key_len_ = py::cast(config["key_len"]); + alpha_len_ = py::cast(config["alpha_len"]); + chunk_len_ = key_len_ * 2; + if (embed_len_ > key_len_) { + chunk_num_ = 2 * embed_len_ / chunk_len_ + + int((2 * embed_len_) % chunk_len_ != 0); + } else { + chunk_num_ = 1; + } + + message_chunks_ = py::cast(config["message_chunks"]); + + // Lazy init: defer expensive table computation to first use + g_table_.reset(); + noise_table_.clear(); + tables_ready_ = false; + free_device_tables_(); + a_bits_ = static_cast(mpz_sizeinbase(keys_.sk.a.v, 2)); + } + + // Batched encrypt: embeddings is [batch][embed_len] of 0/1, returns [batch][chunk_num] ciphers + py::list + encrypt(const std::vector> &embeddings, std::uint64_t seed = 0) const { + if (!have_keys_) { + throw std::runtime_error("encrypt: keys not initialized"); + } + ensure_tables_ready_(); + ensure_device_tables_(); + const int batch = static_cast(embeddings.size()); + const int embed_len = embed_len_; + const int chunk_len = chunk_len_; + const int chunk_num = chunk_num_; + + if (batch == 0) return py::list(); + + // Validate shapes and binary values + for (int i = 0; i < batch; ++i) { + if ((int)embeddings[i].size() != embed_len) { + throw std::runtime_error("encrypt: each embedding must have length == embed_len"); + } + for (int v : embeddings[i]) { + if (v != 0 && v != 1) { + throw std::runtime_error("encrypt: embeddings must be binary 0/1"); + } + } + } + + // Flatten bits into [batch * embed_len] + std::vector h_bits; + h_bits.reserve(static_cast(batch) * static_cast(embed_len)); + for (int i = 0; i < batch; ++i) { + for (int j = 0; j < embed_len; ++j) { + h_bits.push_back(static_cast(embeddings[i][j])); + } + } + + std::vector n_words = hex_to_le_words(pk_n_hex_, LIMBS); + std::vector n2_words = hex_to_le_words(pk_n2_hex_, LIMBS); + mem_t h_N{}, h_N2{}; + std::memcpy(&h_N, n_words.data(), LIMBS * sizeof(uint32_t)); + std::memcpy(&h_N2, n2_words.data(), LIMBS * sizeof(uint32_t)); + + error_report_t *d_report = nullptr; + uint8_t *d_bits = nullptr; + mem_t *d_N = nullptr, *d_N2 = nullptr, *d_out = nullptr; + + const int total = batch * chunk_num; + + CUDA_CHECK(cudaMalloc((void **)&d_report, sizeof(error_report_t))); + CUDA_CHECK(cudaMemset(d_report, 0, sizeof(error_report_t))); + CUDA_CHECK(cudaMalloc((void **)&d_bits, + static_cast(batch) * static_cast(embed_len) * + sizeof(uint8_t))); + CUDA_CHECK(cudaMemcpy(d_bits, + h_bits.data(), + static_cast(batch) * static_cast(embed_len) * + sizeof(uint8_t), + cudaMemcpyHostToDevice)); + + CUDA_CHECK(cudaMalloc((void **)&d_N, sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_N2, sizeof(mem_t))); + CUDA_CHECK(cudaMemcpy(d_N, &h_N, sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMemcpy(d_N2, &h_N2, sizeof(mem_t), cudaMemcpyHostToDevice)); + + CUDA_CHECK(cudaMalloc((void **)&d_out, total * sizeof(mem_t))); + + const int msg_bits = PAILLIER_MSG_BITS; + const int table_size = PAILLIER_MSG_TABLE_SIZE; + const int message_chunks = message_chunks_; + const int noise_table_size = PAILLIER_NOISE_TABLE_SIZE; + + if (!g_table_ || (int)g_table_->size() != message_chunks * table_size) { + throw std::runtime_error("encrypt: g_table size mismatch"); + } + if ((int)noise_table_.size() != noise_table_size) { + throw std::runtime_error("encrypt: noise_table size mismatch"); + } + + // If seed == 0, derive a pseudo-random seed from time and simple bit-mixing + uint64_t actual_seed = seed; + if (actual_seed == 0) { + actual_seed = static_cast(std::time(nullptr)); + // simple scrambling to make it less predictable + actual_seed ^= (actual_seed << 13); + actual_seed ^= (actual_seed >> 7); + actual_seed ^= 0x9E3779B97F4A7C15ULL; + } + + const int INSTS_PER_BLK = 8; + const int THREADS = INSTS_PER_BLK * CGBN_TPI; + const int blocks = (total + INSTS_PER_BLK - 1) / INSTS_PER_BLK; + + encrypt_kernel<<>>(d_report, + d_bits, + batch, + embed_len, + chunk_len, + chunk_num, + d_N, + d_N2, + d_g_table_, + d_noise_table_, + message_chunks, + noise_table_size, + d_out, + actual_seed); + CUDA_CHECK(cudaPeekAtLastError()); + CUDA_CHECK(cudaDeviceSynchronize()); + + std::vector h_out(total); + CUDA_CHECK(cudaMemcpy(h_out.data(), d_out, total * sizeof(mem_t), cudaMemcpyDeviceToHost)); + + cudaFree(d_bits); + cudaFree(d_N); + cudaFree(d_N2); + cudaFree(d_out); + cudaFree(d_report); + + // Reshape back to [batch][chunk_num] GMP ints + py::list out(batch); + for (int i = 0; i < batch; ++i) { + py::list row(chunk_num); + for (int c = 0; c < chunk_num; ++c) { + int idx = i * chunk_num + c; + row[c] = py_mpz_from_big(mem_to_bigint(h_out[idx])); + } + out[i] = row; + } + return out; + } + + // Batched Paillier encryption of arbitrary plaintext integers m < n, using lookup tables. + // Input: plaintexts as Python ints (converted via string), Output: ciphertexts as Python ints. + py::list + encrypt_cts(const std::vector &plaintexts, std::uint64_t seed = 0) const { + if (!have_keys_) { + throw std::runtime_error("encrypt_cts: keys not initialized"); + } + ensure_tables_ready_(); + ensure_device_tables_(); + const int batch = static_cast(plaintexts.size()); + if (batch == 0) return py::list(); + + // Prepare n for range checks + const bigint::big_int_t &n_big = keys_.pk.n; + + const int msg_bits = PAILLIER_MSG_BITS; + const int message_chunks = message_chunks_; + const uint64_t mask = (1u << msg_bits) - 1u; + + // Flatten digits: [batch * message_chunks] + std::vector h_digits(static_cast(batch) * + static_cast(message_chunks)); + + for (int i = 0; i < batch; ++i) { + // Convert Python int to decimal string, then to GMP big_int_t + py::object obj = plaintexts[i]; + py::str s_dec = py::str(obj); + std::string s = py::cast(s_dec); + bigint::big_int_t m_big(s, 10); + + if (mpz_sgn(m_big.v) < 0 || mpz_cmp(m_big.v, n_big.v) >= 0) { + throw std::runtime_error("encrypt_cts: plaintext out of range [0, n)"); + } + + for (int j = 0; j < message_chunks; ++j) { + bigint::big_int_t shifted; + mpz_tdiv_q_2exp(shifted.v, m_big.v, msg_bits * j); + unsigned long d = mpz_get_ui(shifted.v) & mask; + h_digits[static_cast(i) * static_cast(message_chunks) + + static_cast(j)] = static_cast(d); + } + } + + // Prepare n and n^2 for device + std::vector n_words = hex_to_le_words(pk_n_hex_, LIMBS); + std::vector n2_words = hex_to_le_words(pk_n2_hex_, LIMBS); + mem_t h_N{}, h_N2{}; + std::memcpy(&h_N, n_words.data(), LIMBS * sizeof(uint32_t)); + std::memcpy(&h_N2, n2_words.data(), LIMBS * sizeof(uint32_t)); + + error_report_t *d_report = nullptr; + uint8_t *d_digits = nullptr; + mem_t *d_N = nullptr, *d_N2 = nullptr, *d_out = nullptr; + + CUDA_CHECK(cudaMalloc((void **)&d_report, sizeof(error_report_t))); + CUDA_CHECK(cudaMemset(d_report, 0, sizeof(error_report_t))); + + CUDA_CHECK(cudaMalloc((void **)&d_digits, + static_cast(batch) * + static_cast(message_chunks) * sizeof(uint8_t))); + CUDA_CHECK(cudaMemcpy(d_digits, + h_digits.data(), + static_cast(batch) * + static_cast(message_chunks) * sizeof(uint8_t), + cudaMemcpyHostToDevice)); + + CUDA_CHECK(cudaMalloc((void **)&d_N, sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_N2, sizeof(mem_t))); + CUDA_CHECK(cudaMemcpy(d_N, &h_N, sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMemcpy(d_N2, &h_N2, sizeof(mem_t), cudaMemcpyHostToDevice)); + + CUDA_CHECK(cudaMalloc((void **)&d_out, batch * sizeof(mem_t))); + + const int table_size = PAILLIER_MSG_TABLE_SIZE; + const int noise_table_size = PAILLIER_NOISE_TABLE_SIZE; + + if (!g_table_ || (int)g_table_->size() != message_chunks * table_size) { + throw std::runtime_error("encrypt_cts: g_table size mismatch"); + } + if ((int)noise_table_.size() != noise_table_size) { + throw std::runtime_error("encrypt_cts: noise_table size mismatch"); + } + + // Seed + uint64_t actual_seed = seed; + if (actual_seed == 0) { + actual_seed = static_cast(std::time(nullptr)); + actual_seed ^= (actual_seed << 13); + actual_seed ^= (actual_seed >> 7); + actual_seed ^= 0x9E3779B97F4A7C15ULL; + } + + const int INSTS_PER_BLK = 8; + const int THREADS = INSTS_PER_BLK * CGBN_TPI; + const int blocks = (batch + INSTS_PER_BLK - 1) / INSTS_PER_BLK; + + encrypt_cts_kernel<<>>(d_report, + d_digits, + batch, + message_chunks, + d_N, + d_N2, + d_g_table_, + d_noise_table_, + noise_table_size, + d_out, + actual_seed); + CUDA_CHECK(cudaPeekAtLastError()); + CUDA_CHECK(cudaDeviceSynchronize()); + + std::vector h_out(batch); + CUDA_CHECK(cudaMemcpy(h_out.data(), d_out, batch * sizeof(mem_t), cudaMemcpyDeviceToHost)); + + cudaFree(d_digits); + cudaFree(d_N); + cudaFree(d_N2); + cudaFree(d_out); + cudaFree(d_report); + + // Convert to GMP ints + py::list out(batch); + for (int i = 0; i < batch; ++i) { + out[i] = py_mpz_from_big(mem_to_bigint(h_out[i])); + } + return out; + } + + // Decrypt ciphertexts under key_a, then re-encrypt under key_b (fused GPU kernel). + py::list decrypt_then_reencrypt(const py::sequence &ciphers, + const py::dict &key_a, + const py::dict &key_b, + std::uint64_t seed = 0) const { + const int batch = static_cast(py::len(ciphers)); + if (batch == 0) return py::list(); + + PaillierPublicKeyCPU pk_a = pk_from_py(key_a); + PaillierSecretKeyCPU sk_a = sk_from_py(key_a); + PaillierPublicKeyCPU pk_b = pk_from_py(key_b); + + const int message_chunks = message_chunks_from_key(key_b, pk_b.n); + const int table_size = PAILLIER_MSG_TABLE_SIZE; + + std::vector g_table; + std::vector noise_table; + + if (key_b.contains("g_table")) { + g_table = g_table_from_py(key_b["g_table"], message_chunks); + } + if (key_b.contains("noise_table")) { + noise_table = noise_table_from_py(key_b["noise_table"]); + } + + if (g_table.empty() || noise_table.empty()) { + PaillierKeyPairCPU keys_b{}; + keys_b.pk = pk_b; + if (g_table.empty()) { + int key_len_bits = (message_chunks * PAILLIER_MSG_BITS) / 2; + g_table = precompute_g_table(keys_b, key_len_bits); + } + if (noise_table.empty()) { + noise_table = precompute_noise_table(keys_b); + } + } + + if ((int)g_table.size() != message_chunks * table_size) { + throw std::runtime_error("decrypt_then_reencrypt: g_table size mismatch"); + } + const int noise_table_size = static_cast(noise_table.size()); + if (noise_table_size <= 0) { + throw std::runtime_error("decrypt_then_reencrypt: noise_table is empty"); + } + + // Pack ciphertexts + std::vector h_CT(batch); + for (int i = 0; i < batch; ++i) { + h_CT[i] = bigint_to_mem(big_from_py(ciphers[i])); + } + + mem_t h_Na = bigint_to_mem(pk_a.n); + mem_t h_N2a = bigint_to_mem(pk_a.n_squared); + mem_t h_Aa = bigint_to_mem(sk_a.a); + mem_t h_GaInv = bigint_to_mem(sk_a.g_a_inv); + mem_t h_N2b = bigint_to_mem(pk_b.n_squared); + + error_report_t *d_report = nullptr; + mem_t *d_CT = nullptr, *d_OUT = nullptr; + mem_t *d_Na = nullptr, *d_N2a = nullptr, *d_Aa = nullptr, *d_GaInv = nullptr; + mem_t *d_N2b = nullptr, *d_g_table = nullptr, *d_noise_table = nullptr; + + CUDA_CHECK(cudaMalloc((void **)&d_report, sizeof(error_report_t))); + CUDA_CHECK(cudaMemset(d_report, 0, sizeof(error_report_t))); + + CUDA_CHECK(cudaMalloc((void **)&d_CT, batch * sizeof(mem_t))); + CUDA_CHECK(cudaMemcpy(d_CT, h_CT.data(), batch * sizeof(mem_t), cudaMemcpyHostToDevice)); + + CUDA_CHECK(cudaMalloc((void **)&d_OUT, batch * sizeof(mem_t))); + + CUDA_CHECK(cudaMalloc((void **)&d_Na, sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_N2a, sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_Aa, sizeof(mem_t))); + CUDA_CHECK(cudaMalloc((void **)&d_GaInv, sizeof(mem_t))); + CUDA_CHECK(cudaMemcpy(d_Na, &h_Na, sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMemcpy(d_N2a, &h_N2a, sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMemcpy(d_Aa, &h_Aa, sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMemcpy(d_GaInv, &h_GaInv, sizeof(mem_t), cudaMemcpyHostToDevice)); + + CUDA_CHECK(cudaMalloc((void **)&d_N2b, sizeof(mem_t))); + CUDA_CHECK(cudaMemcpy(d_N2b, &h_N2b, sizeof(mem_t), cudaMemcpyHostToDevice)); + + CUDA_CHECK(cudaMalloc((void **)&d_g_table, + static_cast(message_chunks) * + static_cast(table_size) * sizeof(mem_t))); + CUDA_CHECK(cudaMemcpy(d_g_table, + g_table.data(), + static_cast(message_chunks) * + static_cast(table_size) * sizeof(mem_t), + cudaMemcpyHostToDevice)); + + CUDA_CHECK(cudaMalloc((void **)&d_noise_table, + static_cast(noise_table_size) * sizeof(mem_t))); + CUDA_CHECK(cudaMemcpy(d_noise_table, + noise_table.data(), + static_cast(noise_table_size) * sizeof(mem_t), + cudaMemcpyHostToDevice)); + + int exp_bits_a = static_cast(mpz_sizeinbase(sk_a.a.v, 2)); + if (exp_bits_a <= 0) exp_bits_a = key_len_ * 2; + + uint64_t actual_seed = seed; + if (actual_seed == 0) { + actual_seed = static_cast(std::time(nullptr)); + actual_seed ^= (actual_seed << 13); + actual_seed ^= (actual_seed >> 7); + actual_seed ^= 0x9E3779B97F4A7C15ULL; + } + + const int INSTS_PER_BLK = 8; + const int THREADS = INSTS_PER_BLK * CGBN_TPI; + const int blocks = (batch + INSTS_PER_BLK - 1) / INSTS_PER_BLK; + + decrypt_then_reencrypt_kernel<<>>(d_report, + d_CT, + batch, + d_Na, + d_N2a, + d_Aa, + d_GaInv, + exp_bits_a, + d_N2b, + d_g_table, + d_noise_table, + message_chunks, + noise_table_size, + d_OUT, + actual_seed); + CUDA_CHECK(cudaPeekAtLastError()); + CUDA_CHECK(cudaDeviceSynchronize()); + + std::vector h_OUT(batch); + CUDA_CHECK(cudaMemcpy(h_OUT.data(), d_OUT, batch * sizeof(mem_t), cudaMemcpyDeviceToHost)); + + cudaFree(d_CT); + cudaFree(d_OUT); + cudaFree(d_Na); + cudaFree(d_N2a); + cudaFree(d_Aa); + cudaFree(d_GaInv); + cudaFree(d_N2b); + cudaFree(d_g_table); + cudaFree(d_noise_table); + cudaFree(d_report); + + py::list out(batch); + for (int i = 0; i < batch; ++i) { + out[i] = py_mpz_from_big(mem_to_bigint(h_OUT[i])); + } + return out; + } + + // Client-side homomorphic addition: ct1[i]*ct2[i] mod n^2 + py::list encode_hamming_client(const py::sequence &ct1, + const py::sequence &ct2) const { + if (!have_keys_) { + throw std::runtime_error("encode_hamming_client: keys not initialized"); + } + if (py::len(ct1) != py::len(ct2)) { + throw std::runtime_error("encode_hamming_client: ct1 and ct2 must have the same length"); + } + std::vector a; + std::vector b; + a.reserve(py::len(ct1)); + b.reserve(py::len(ct2)); + for (auto &&o : ct1) a.emplace_back(big_from_py(o)); + for (auto &&o : ct2) b.emplace_back(big_from_py(o)); + + auto res = gpu_pointwise_mulmod_bigint(a, b, keys_.pk.n_squared); + py::list out(res.size()); + for (size_t i = 0; i < res.size(); ++i) { + out[i] = py_mpz_from_big(res[i]); + } + return out; + } + + // Server-side version: takes pk dict with "n_squared" + static py::list encode_hamming_server(const py::sequence &ct1, + const py::sequence &ct2, + const py::dict &pk) { + if (py::len(ct1) != py::len(ct2)) { + throw std::runtime_error("encode_hamming_server: ct1 and ct2 must have the same length"); + } + if (!pk.contains("n_squared")) { + throw std::runtime_error("encode_hamming_server: pk must contain n_squared"); + } + std::vector a; + std::vector b; + a.reserve(py::len(ct1)); + b.reserve(py::len(ct2)); + for (auto &&o : ct1) a.emplace_back(big_from_py(o)); + for (auto &&o : ct2) b.emplace_back(big_from_py(o)); + + bigint::big_int_t n2_big = big_from_py(pk["n_squared"]); + auto res = gpu_pointwise_mulmod_bigint(a, b, n2_big); + py::list out(res.size()); + for (size_t i = 0; i < res.size(); ++i) { + out[i] = py_mpz_from_big(res[i]); + } + return out; + } + + // GPU decrypt chunks to plaintext m, then CPU bit-padding and odd-bit counting + std::vector + decode_hamming_client(const py::sequence &ciphers_batch) const { + if (!have_keys_) { + throw std::runtime_error("decode_hamming_client: keys not initialized"); + } + const int batch = static_cast(py::len(ciphers_batch)); + const int chunk_num = chunk_num_; + const int chunk_len = chunk_len_; + if (batch == 0) return {}; + + const int total = batch * chunk_num; + + // Pack ciphertexts into mem_t array + std::vector h_CT(total); + for (int i = 0; i < batch; ++i) { + py::sequence row = py::cast(ciphers_batch[i]); + if (static_cast(py::len(row)) != chunk_num) { + throw std::runtime_error("decode_hamming_client: each inner list must have length == chunk_num"); + } + for (int c = 0; c < chunk_num; ++c) { + const int idx = i * chunk_num + c; + h_CT[idx] = bigint_to_mem(big_from_py(row[c])); + } + } + + // Keys -> constant memory (for decrypt kernels) + mem_t h_N{}, h_N2{}, h_A{}, h_G_A_INV{}; + std::memcpy(&h_N, hex_to_le_words(pk_n_hex_, LIMBS).data(), LIMBS * sizeof(uint32_t)); + std::memcpy(&h_N2, hex_to_le_words(pk_n2_hex_, LIMBS).data(), LIMBS * sizeof(uint32_t)); + std::memcpy(&h_A, hex_to_le_words(sk_a_hex_, LIMBS).data(), LIMBS * sizeof(uint32_t)); + std::memcpy(&h_G_A_INV, hex_to_le_words(sk_g_a_inv_hex_, LIMBS).data(), LIMBS * sizeof(uint32_t)); + + CUDA_CHECK(cudaMemcpyToSymbol(CONST_N, &h_N, sizeof(mem_t))); + CUDA_CHECK(cudaMemcpyToSymbol(CONST_N2, &h_N2, sizeof(mem_t))); + CUDA_CHECK(cudaMemcpyToSymbol(CONST_A, &h_A, sizeof(mem_t))); + CUDA_CHECK(cudaMemcpyToSymbol(CONST_G_A_INV, &h_G_A_INV, sizeof(mem_t))); + + // Build odd-position mask and compute words used + mem_t h_MASK = make_odd_mask_host(chunk_len); + const int words_used = (chunk_len + 31) / 32; + + // Device buffers + error_report_t *d_report = nullptr; + mem_t *d_CT = nullptr, *d_M = nullptr, *d_MASK = nullptr; + uint32_t *d_out_odd = nullptr; + + CUDA_CHECK(cudaMalloc((void **)&d_report, sizeof(error_report_t))); + CUDA_CHECK(cudaMemset(d_report, 0, sizeof(error_report_t))); + + CUDA_CHECK(cudaMalloc((void **)&d_CT, total * sizeof(mem_t))); + CUDA_CHECK(cudaMemcpy(d_CT, h_CT.data(), total * sizeof(mem_t), cudaMemcpyHostToDevice)); + + CUDA_CHECK(cudaMalloc((void **)&d_M, total * sizeof(mem_t))); + + CUDA_CHECK(cudaMalloc((void **)&d_MASK, sizeof(mem_t))); + CUDA_CHECK(cudaMemcpy(d_MASK, &h_MASK, sizeof(mem_t), cudaMemcpyHostToDevice)); + + CUDA_CHECK(cudaMalloc((void **)&d_out_odd, total * sizeof(uint32_t))); + + const int INSTS_PER_BLK = 8; + const int THREADS = INSTS_PER_BLK * CGBN_TPI; + const int blocks = (total + INSTS_PER_BLK - 1) / INSTS_PER_BLK; + int exp_bits = (a_bits_ > 0) ? a_bits_ : (key_len_ * 2); + + // First kernel: decrypt-only to plaintext m in d_M + decrypt_only_kernel<<>>(d_report, d_CT, total, d_M, exp_bits); + CUDA_CHECK(cudaPeekAtLastError()); + CUDA_CHECK(cudaDeviceSynchronize()); + + // Second kernel: GPU popcount on m with odd-bit mask + odd_popcount_kernel<<>>(d_M, + total, + d_MASK, + words_used, + d_out_odd); + CUDA_CHECK(cudaPeekAtLastError()); + CUDA_CHECK(cudaDeviceSynchronize()); + + std::vector h_out_odd(total); + CUDA_CHECK(cudaMemcpy(h_out_odd.data(), d_out_odd, total * sizeof(uint32_t), + cudaMemcpyDeviceToHost)); + + cudaFree(d_CT); + cudaFree(d_M); + cudaFree(d_MASK); + cudaFree(d_out_odd); + cudaFree(d_report); + + // Sum odd counts per embedding (across chunks) + std::vector out(batch, 0); + for (int i = 0; i < batch; ++i) { + long long ham = 0; + for (int c = 0; c < chunk_num; ++c) { + const int idx = i * chunk_num + c; + ham += static_cast(h_out_odd[idx]); + } + out[i] = ham; + } + return out; + } + + // Batched Paillier decryption: ciphers is [batch] ciphertext ints, returns Python ints. + std::vector + decrypt_cts(const py::sequence &ciphers) const { + if (!have_keys_) { + throw std::runtime_error("decrypt_cts: keys not initialized"); + } + const int batch = static_cast(py::len(ciphers)); + if (batch == 0) return {}; + + // Pack CT + std::vector h_CT(batch); + for (int i = 0; i < batch; ++i) { + h_CT[i] = bigint_to_mem(big_from_py(ciphers[i])); + } + + // Keys -> constant memory + mem_t h_N{}, h_N2{}, h_A{}, h_G_A_INV{}; + std::memcpy(&h_N, hex_to_le_words(pk_n_hex_, LIMBS).data(), LIMBS * sizeof(uint32_t)); + std::memcpy(&h_N2, hex_to_le_words(pk_n2_hex_, LIMBS).data(), LIMBS * sizeof(uint32_t)); + std::memcpy(&h_A, hex_to_le_words(sk_a_hex_, LIMBS).data(), LIMBS * sizeof(uint32_t)); + std::memcpy(&h_G_A_INV, hex_to_le_words(sk_g_a_inv_hex_, LIMBS).data(), LIMBS * sizeof(uint32_t)); + + CUDA_CHECK(cudaMemcpyToSymbol(CONST_N, &h_N, sizeof(mem_t))); + CUDA_CHECK(cudaMemcpyToSymbol(CONST_N2, &h_N2, sizeof(mem_t))); + CUDA_CHECK(cudaMemcpyToSymbol(CONST_A, &h_A, sizeof(mem_t))); + CUDA_CHECK(cudaMemcpyToSymbol(CONST_G_A_INV, &h_G_A_INV, sizeof(mem_t))); + + // Device + error_report_t *d_report = nullptr; + mem_t *d_CT = nullptr, *d_M = nullptr; + + CUDA_CHECK(cudaMalloc((void **)&d_report, sizeof(error_report_t))); + CUDA_CHECK(cudaMemset(d_report, 0, sizeof(error_report_t))); + + CUDA_CHECK(cudaMalloc((void **)&d_CT, batch * sizeof(mem_t))); + CUDA_CHECK(cudaMemcpy(d_CT, h_CT.data(), batch * sizeof(mem_t), cudaMemcpyHostToDevice)); + + CUDA_CHECK(cudaMalloc((void **)&d_M, batch * sizeof(mem_t))); + + const int INSTS_PER_BLK = 8; + const int THREADS = INSTS_PER_BLK * CGBN_TPI; + const int blocks = (batch + INSTS_PER_BLK - 1) / INSTS_PER_BLK; + int exp_bits = (a_bits_ > 0) ? a_bits_ : (key_len_ * 2); + decrypt_only_kernel<<>>(d_report, d_CT, batch, d_M, exp_bits); + CUDA_CHECK(cudaPeekAtLastError()); + CUDA_CHECK(cudaDeviceSynchronize()); + + std::vector h_M(batch); + CUDA_CHECK(cudaMemcpy(h_M.data(), d_M, batch * sizeof(mem_t), cudaMemcpyDeviceToHost)); + + cudaFree(d_CT); + cudaFree(d_M); + cudaFree(d_report); + + // Convert plaintexts to Python ints via decimal strings (avoid base issues) + std::vector out(batch); + for (int i = 0; i < batch; ++i) { + out[i] = py_int_from_big(mem_to_bigint(h_M[i])); + } + return out; + } + +private: + int embed_len_; + int key_len_; + int chunk_len_; + int chunk_num_; + int alpha_len_; + int a_bits_; + bool have_keys_; + + PaillierKeyPairCPU keys_; + std::string pk_g_hex_; + std::string pk_n_hex_; + std::string pk_n2_hex_; + std::string sk_phi_hex_; + std::string sk_inv_hex_; + std::string sk_a_hex_; + std::string sk_g_a_inv_hex_; + + mutable int message_chunks_; + mutable std::shared_ptr> g_table_; // [message_chunks_ * PAILLIER_MSG_TABLE_SIZE] + mutable std::vector noise_table_; // [PAILLIER_NOISE_TABLE_SIZE] + mutable bool tables_ready_; + mutable bool device_tables_ready_; + mutable mem_t *d_g_table_; + mutable mem_t *d_noise_table_; + mutable int d_g_table_entries_; + mutable int d_noise_table_entries_; + + // Helper: GPU decrypt all chunks to plaintext m (hex), mirroring original GPU client + std::vector> + decrypt_chunks(const std::vector> &ciphers_hex_batch) const { + const int batch = static_cast(ciphers_hex_batch.size()); + const int chunk_num = this->chunk_num_; + if (batch == 0) return {}; + + for (int i = 0; i < batch; ++i) { + if ((int)ciphers_hex_batch[i].size() != chunk_num) { + throw std::runtime_error("decrypt_chunks: each inner list must have length == chunk_num"); + } + } + + const int total = batch * chunk_num; + + // pack CT + std::vector h_CT(total); + for (int i = 0; i < batch; ++i) { + for (int c = 0; c < chunk_num; ++c) { + const int idx = i * chunk_num + c; + auto w = hex_to_le_words(ciphers_hex_batch[i][c], LIMBS); + std::memcpy(&h_CT[idx], w.data(), LIMBS * sizeof(uint32_t)); + } + } + + // keys -> constant memory + mem_t h_N{}, h_N2{}, h_A{}, h_G_A_INV{}; + std::memcpy(&h_N, hex_to_le_words(pk_n_hex_, LIMBS).data(), LIMBS * sizeof(uint32_t)); + std::memcpy(&h_N2, hex_to_le_words(pk_n2_hex_, LIMBS).data(), LIMBS * sizeof(uint32_t)); + std::memcpy(&h_A, hex_to_le_words(sk_a_hex_, LIMBS).data(), LIMBS * sizeof(uint32_t)); + std::memcpy(&h_G_A_INV, hex_to_le_words(sk_g_a_inv_hex_, LIMBS).data(), LIMBS * sizeof(uint32_t)); + + CUDA_CHECK(cudaMemcpyToSymbol(CONST_N, &h_N, sizeof(mem_t))); + CUDA_CHECK(cudaMemcpyToSymbol(CONST_N2, &h_N2, sizeof(mem_t))); + CUDA_CHECK(cudaMemcpyToSymbol(CONST_A, &h_A, sizeof(mem_t))); + CUDA_CHECK(cudaMemcpyToSymbol(CONST_G_A_INV, &h_G_A_INV, sizeof(mem_t))); + + // device + error_report_t *d_report = nullptr; + mem_t *d_CT = nullptr, *d_M = nullptr; + + CUDA_CHECK(cudaMalloc((void **)&d_report, sizeof(error_report_t))); + CUDA_CHECK(cudaMemset(d_report, 0, sizeof(error_report_t))); + CUDA_CHECK(cudaMalloc((void **)&d_CT, total * sizeof(mem_t))); + CUDA_CHECK(cudaMemcpy(d_CT, h_CT.data(), total * sizeof(mem_t), cudaMemcpyHostToDevice)); + CUDA_CHECK(cudaMalloc((void **)&d_M, total * sizeof(mem_t))); + + const int INSTS_PER_BLK = 8; + const int THREADS = INSTS_PER_BLK * CGBN_TPI; + const int blocks = (total + INSTS_PER_BLK - 1) / INSTS_PER_BLK; + int exp_bits = (a_bits_ > 0) ? a_bits_ : (key_len_ * 2); + decrypt_only_kernel<<>>(d_report, d_CT, total, d_M, exp_bits); + CUDA_CHECK(cudaPeekAtLastError()); + CUDA_CHECK(cudaDeviceSynchronize()); + + std::vector h_M(total); + CUDA_CHECK(cudaMemcpy(h_M.data(), d_M, total * sizeof(mem_t), cudaMemcpyDeviceToHost)); + + cudaFree(d_CT); + cudaFree(d_M); + cudaFree(d_report); + + std::vector> out(batch, std::vector(chunk_num)); + for (int i = 0; i < batch; ++i) { + for (int c = 0; c < chunk_num; ++c) { + const int idx = i * chunk_num + c; + std::vector w(LIMBS); + std::memcpy(w.data(), &h_M[idx], LIMBS * sizeof(uint32_t)); + out[i][c] = le_words_to_hex(w); + } + } + return out; + } + + std::string g_table_cache_key_() const { + return pk_g_hex_ + "|" + pk_n_hex_ + "|" + std::to_string(key_len_); + } + + void ensure_tables_ready_() const { + if (tables_ready_) return; + if (!have_keys_) { + throw std::runtime_error("lookup tables: keys not initialized"); + } + if (message_chunks_ <= 0) { + message_chunks_ = (key_len_ * 2 + PAILLIER_MSG_BITS - 1) / PAILLIER_MSG_BITS; + } + if (!g_table_) { + const std::string cache_key = g_table_cache_key_(); + g_table_ = get_cached_g_table(cache_key, keys_, key_len_); + } + if (noise_table_.empty()) { + noise_table_ = precompute_noise_table(keys_); + } + tables_ready_ = true; + } + + void ensure_device_tables_() const { + if (device_tables_ready_) return; + ensure_tables_ready_(); + if (!g_table_ || g_table_->empty()) { + throw std::runtime_error("lookup tables: g_table not initialized"); + } + if (noise_table_.empty()) { + throw std::runtime_error("lookup tables: noise_table not initialized"); + } + d_g_table_entries_ = static_cast(g_table_->size()); + d_noise_table_entries_ = static_cast(noise_table_.size()); + + CUDA_CHECK(cudaMalloc((void **)&d_g_table_, + static_cast(d_g_table_entries_) * sizeof(mem_t))); + CUDA_CHECK(cudaMemcpy(d_g_table_, + g_table_->data(), + static_cast(d_g_table_entries_) * sizeof(mem_t), + cudaMemcpyHostToDevice)); + + CUDA_CHECK(cudaMalloc((void **)&d_noise_table_, + static_cast(d_noise_table_entries_) * sizeof(mem_t))); + CUDA_CHECK(cudaMemcpy(d_noise_table_, + noise_table_.data(), + static_cast(d_noise_table_entries_) * sizeof(mem_t), + cudaMemcpyHostToDevice)); + + device_tables_ready_ = true; + } + + void free_device_tables_() const { + if (d_g_table_ != nullptr) { + cudaFree(d_g_table_); + d_g_table_ = nullptr; + } + if (d_noise_table_ != nullptr) { + cudaFree(d_noise_table_); + d_noise_table_ = nullptr; + } + device_tables_ready_ = false; + d_g_table_entries_ = 0; + d_noise_table_entries_ = 0; + } +}; + +// Backwards-compatible alias +using PaillierGPUClient = PaillierGPULookupClient; + +// ---------------------- Pybind module ---------------------- +PYBIND11_MODULE(paillier_GPU_lookup_client, m) { + py::class_ lookup_cls(m, "PaillierGPULookupClient"); + lookup_cls + .def(py::init(), + py::arg("embed_len") = 512, + py::arg("key_len") = KEY_BITS, + py::arg("alpha_len") = ALPHA_LEN, + py::arg("skip_key_gen") = false) + .def("encrypt", &PaillierGPULookupClient::encrypt, + py::arg("embeddings"), + py::arg("seed") = 0ULL) + .def("encrypt_cts", &PaillierGPULookupClient::encrypt_cts, + py::arg("plaintexts"), + py::arg("seed") = 0ULL) + .def("decrypt_then_reencrypt", &PaillierGPULookupClient::decrypt_then_reencrypt, + py::arg("ciphers"), + py::arg("key_a"), + py::arg("key_b"), + py::arg("seed") = 0ULL) + .def("encode_hamming_client", &PaillierGPULookupClient::encode_hamming_client, + py::arg("ct1"), + py::arg("ct2")) + .def_static("encode_hamming_server", &PaillierGPULookupClient::encode_hamming_server, + py::arg("ct1"), + py::arg("ct2"), + py::arg("pk")) + .def("decode_hamming_client", &PaillierGPULookupClient::decode_hamming_client, + py::arg("ciphers_batch")) + .def("decrypt_cts", &PaillierGPULookupClient::decrypt_cts, + py::arg("ciphers")) + .def("get_pk_hex", &PaillierGPULookupClient::get_pk_hex) + .def("get_keys_hex", &PaillierGPULookupClient::get_keys_hex) + .def("stringify_pk", &PaillierGPULookupClient::stringify_pk) + .def("stringify_sk", &PaillierGPULookupClient::stringify_sk) + .def("stringify_config", &PaillierGPULookupClient::stringify_config) + .def("load_stringified_keys", &PaillierGPULookupClient::load_stringified_keys) + .def("load_config", &PaillierGPULookupClient::load_config) + .def_property_readonly("embed_len", &PaillierGPULookupClient::embed_len) + .def_property_readonly("key_len", &PaillierGPULookupClient::key_len) + .def_property_readonly("chunk_len", &PaillierGPULookupClient::chunk_len) + .def_property_readonly("chunk_num", &PaillierGPULookupClient::chunk_num); + // Alias for compatibility with existing Python code + m.attr("PaillierGPUClient") = lookup_cls; +} diff --git a/src/xtrace_sdk/x_vec/crypto/paillier_gpu_client.py b/src/xtrace_sdk/x_vec/crypto/paillier_gpu_client.py new file mode 100644 index 0000000..e55ddb5 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/paillier_gpu_client.py @@ -0,0 +1,173 @@ +import json + +import gmpy2 + +from xtrace_sdk.x_vec.crypto._gpu_loader import load_gpu_extension +from xtrace_sdk.x_vec.crypto.encryption.paillier import Paillier +from xtrace_sdk.x_vec.crypto.hamming_client_base import HammingClientBase +from xtrace_sdk.x_vec.utils.xtrace_types import EncryptedVector, PaillierKeyPair + + +def _build_plain_paillier_keys(pk: str, sk: str) -> PaillierKeyPair: + pk_data = json.loads(pk) + sk_data = json.loads(sk) + return { + "pk": { + "g": gmpy2.mpz(pk_data["g"]), + "n": gmpy2.mpz(pk_data["n"]), + "n_squared": gmpy2.mpz(pk_data["n_squared"]), + }, + "sk": { + "phi": gmpy2.mpz(sk_data["phi"]), + "inv": gmpy2.mpz(sk_data["inv"]), + }, + } + + +class PaillierGPUClient(HammingClientBase): + """Standalone GPU Paillier wrapper for ExecutionContext integration and tests.""" + + def __init__(self, embed_len: int = 512, key_len: int = 1024, skip_key_gen: bool = False) -> None: + module = load_gpu_extension("paillier_GPU_client", "paillier-GPU-client") + self._gpu_cls = module.PaillierGPUClient + self._gpu_client = self._gpu_cls(embed_len=embed_len, key_len=key_len, skip_key_gen=skip_key_gen) + self.embed_len = embed_len + self.key_len = key_len + self.chunk_len = key_len * 2 + if embed_len > key_len: + self.chunk_num = 2 * embed_len // self.chunk_len + int((2 * embed_len) % self.chunk_len != 0) + else: + self.chunk_num = 1 + self.device = "gpu" + self._backend_has_keys = not skip_key_gen + self._cpu_keys: PaillierKeyPair | None = None + self._pk_string: str | None = None + self._sk_string: str | None = None + if not skip_key_gen: + self._sync_serialized_keys_from_gpu() + + @staticmethod + def is_available() -> bool: + try: + PaillierGPUClient(skip_key_gen=True) + return True + except Exception: + return False + + def _sync_serialized_keys_from_gpu(self) -> None: + n_hex, n_squared_hex, phi_hex, inv_hex = self._gpu_client.get_keys_hex() + n = int(n_hex, 16) + n_squared = int(n_squared_hex, 16) + self._pk_string = json.dumps( + { + "g": str(n + 1), + "n": str(n), + "n_squared": str(n_squared), + } + ) + self._sk_string = json.dumps( + { + "phi": str(int(phi_hex, 16)), + "inv": str(int(inv_hex, 16)), + } + ) + self._cpu_keys = _build_plain_paillier_keys(self._pk_string, self._sk_string) + + def _require_cpu_keys(self) -> PaillierKeyPair: + if self._cpu_keys is None: + raise RuntimeError("Keys not initialized") + return self._cpu_keys + + def _encrypt_with_cpu(self, embd: list[int]) -> EncryptedVector: + keys = self._require_cpu_keys() + if len(embd) != self.embed_len: + raise ValueError(f"Embedding length {len(embd)} does not match expected {self.embed_len}") + padded_embd: list[str] = [] + for i, bit in enumerate(embd): + if bit not in (0, 1): + raise ValueError(f"Embedding vector must be binary, got {bit} at index {i}") + padded_embd += ["0", str(bit)] + int_repr = [ + gmpy2.mpz(int("".join(padded_embd[i * self.chunk_len : (i + 1) * self.chunk_len]), 2)) + for i in range(self.chunk_num) + ] + return [int(Paillier.encrypt(value, keys["pk"])) for value in int_repr] + + def _decode_with_cpu(self, cipher: list[int | bytes]) -> int: + keys = self._require_cpu_keys() + decrypted = [ + Paillier.decrypt(int.from_bytes(c, byteorder="little") if isinstance(c, bytes) else int(c), keys) + for c in cipher + ] + bin_chunks = [f"{value:b}" for value in decrypted] + bin_string = "" + for chunk in bin_chunks: + if len(chunk) != self.chunk_len and len(chunk) != 0: + chunk = "0" * (self.chunk_len - len(chunk)) + chunk + bin_string += chunk + return sum(int(bin_string[i]) for i in range(1, len(bin_string), 2)) + + def encrypt_vec_one(self, embd: list[int]) -> EncryptedVector: + if not self._backend_has_keys: + return self._encrypt_with_cpu(embd) + return [int(cipher, 16) for cipher in self._gpu_client.encrypt([embd])[0]] + + def encrypt_vec_batch(self, embds: list[list[int]]) -> list[EncryptedVector]: + if not self._backend_has_keys: + return [self._encrypt_with_cpu(embd) for embd in embds] + return [[int(cipher, 16) for cipher in row] for row in self._gpu_client.encrypt(embds)] + + def decode_hamming_client_one(self, cipher: list[int | bytes]) -> int: + if not self._backend_has_keys: + return self._decode_with_cpu(cipher) + hex_cipher = [ + format( + int.from_bytes(value, byteorder="little") if isinstance(value, bytes) else int(value), + "x", + ) + for value in cipher + ] + return int(self._gpu_client.decode_hamming_client([hex_cipher])[0]) + + def decode_hamming_client_batch(self, ciphers: list[list[int | bytes]]) -> list[int]: + return [self.decode_hamming_client_one(cipher) for cipher in ciphers] + + def stringify_sk(self) -> str: + if self._sk_string is None: + raise RuntimeError("Keys not initialized") + return self._sk_string + + def stringify_pk(self) -> str: + if self._pk_string is None: + raise RuntimeError("Keys not initialized") + return self._pk_string + + def stringify_config(self) -> str: + return json.dumps({"embed_len": self.embed_len, "key_len": self.key_len}) + + def load_stringified_keys(self, pk: str, sk: str) -> None: + self._pk_string = pk + self._sk_string = sk + self._cpu_keys = _build_plain_paillier_keys(pk, sk) + self._backend_has_keys = False + + def load_config(self, config: dict) -> None: + self.embed_len = config["embed_len"] + self.key_len = config["key_len"] + self.chunk_len = self.key_len * 2 + if self.embed_len > self.key_len: + self.chunk_num = 2 * self.embed_len // self.chunk_len + int((2 * self.embed_len) % self.chunk_len != 0) + else: + self.chunk_num = 1 + + def encode_hamming_server(self, ct1: list[int], ct2: list[int]) -> EncryptedVector: + if not self._backend_has_keys: + keys = self._require_cpu_keys() + return [int(Paillier.add(int(a), int(b), keys["pk"])) for a, b in zip(ct1, ct2, strict=True)] + pk_data = json.loads(self.stringify_pk()) + result = self._gpu_cls.encode_hamming_server( + [format(int(cipher), "x") for cipher in ct1], + [format(int(cipher), "x") for cipher in ct2], + {"n_squared": format(int(pk_data["n_squared"]), "x")}, + ) + return [int(cipher, 16) for cipher in result] diff --git a/src/xtrace_sdk/x_vec/crypto/paillier_lookup_gpu_client.py b/src/xtrace_sdk/x_vec/crypto/paillier_lookup_gpu_client.py new file mode 100644 index 0000000..10aaee5 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/paillier_lookup_gpu_client.py @@ -0,0 +1,82 @@ +import json + +from xtrace_sdk.x_vec.crypto._gpu_loader import load_gpu_extension +from xtrace_sdk.x_vec.crypto.hamming_client_base import HammingClientBase +from xtrace_sdk.x_vec.utils.xtrace_types import EncryptedVector + + +class PaillierLookupGPUClient(HammingClientBase): + """Standalone GPU Paillier-Lookup wrapper for ExecutionContext integration and tests.""" + + def __init__( + self, + embed_len: int = 512, + key_len: int = 1024, + alpha_len: int = 50, + skip_key_gen: bool = False, + ) -> None: + module = load_gpu_extension("paillier_GPU_lookup_client", "paillier-GPU-lookup-client") + self._gpu_cls = module.PaillierGPULookupClient + self._gpu_client = self._gpu_cls( + embed_len=embed_len, + key_len=key_len, + alpha_len=alpha_len, + skip_key_gen=skip_key_gen, + ) + self.embed_len = embed_len + self.key_len = key_len + self.alpha_len = alpha_len + self.device = "gpu" + + @staticmethod + def is_available() -> bool: + try: + PaillierLookupGPUClient(skip_key_gen=True) + return True + except Exception: + return False + + def encrypt_vec_one(self, embd: list[int]) -> EncryptedVector: + return [int(cipher) for cipher in self._gpu_client.encrypt([embd])[0]] + + def encrypt_vec_batch(self, embds: list[list[int]]) -> list[EncryptedVector]: + return [[int(cipher) for cipher in row] for row in self._gpu_client.encrypt(embds)] + + def decode_hamming_client_one(self, cipher: list[int | bytes]) -> int: + normalized = [ + int.from_bytes(value, byteorder="little") if isinstance(value, bytes) else int(value) + for value in cipher + ] + return int(self._gpu_client.decode_hamming_client([normalized])[0]) + + def decode_hamming_client_batch(self, ciphers: list[list[int | bytes]]) -> list[int]: + normalized = [ + [ + int.from_bytes(value, byteorder="little") if isinstance(value, bytes) else int(value) + for value in cipher + ] + for cipher in ciphers + ] + return [int(value) for value in self._gpu_client.decode_hamming_client(normalized)] + + def stringify_sk(self) -> str: + return self._gpu_client.stringify_sk() + + def stringify_pk(self) -> str: + return self._gpu_client.stringify_pk() + + def stringify_config(self) -> str: + return self._gpu_client.stringify_config() + + def load_stringified_keys(self, pk: str, sk: str) -> None: + self._gpu_client.load_stringified_keys(pk, sk) + + def load_config(self, config: dict, precomputed_tables: dict | None = None) -> None: + self._gpu_client.load_config({k: v for k, v in config.items() if k != "device"}) + self.embed_len = config["embed_len"] + self.key_len = config["key_len"] + self.alpha_len = config.get("alpha_len", self.alpha_len) + + def encode_hamming_server(self, ct1: list[int], ct2: list[int]) -> EncryptedVector: + result = self._gpu_cls.encode_hamming_server(ct1, ct2, json.loads(self.stringify_pk())) + return [int(cipher) for cipher in result] diff --git a/src/xtrace_sdk/x_vec/utils/execution_context.py b/src/xtrace_sdk/x_vec/utils/execution_context.py index 6fd5b2a..ff79812 100644 --- a/src/xtrace_sdk/x_vec/utils/execution_context.py +++ b/src/xtrace_sdk/x_vec/utils/execution_context.py @@ -14,6 +14,8 @@ from xtrace_sdk.x_vec.utils.xtrace_types import EncryptedVector # noqa: E402 from xtrace_sdk.x_vec.crypto.paillier_client import PaillierClient # noqa: E402 from xtrace_sdk.x_vec.crypto.paillier_lookup_client import PaillierLookupClient # noqa: E402 +from xtrace_sdk.x_vec.crypto.paillier_gpu_client import PaillierGPUClient # noqa: E402 +from xtrace_sdk.x_vec.crypto.paillier_lookup_gpu_client import PaillierLookupGPUClient # noqa: E402 from xtrace_sdk.x_vec.utils.settings import SUPPORTED_HOMOMORPHIC_CLIENTS # noqa: E402 if TYPE_CHECKING: @@ -47,19 +49,28 @@ def _resolve_key_provider( raise ValueError("Either key_provider or passphrase must be supplied.") +def _canonical_homomorphic_type(client_name: str) -> str: + if client_name in {"PaillierClient", "PaillierGPUClient"}: + return "PaillierClient" + if client_name in {"PaillierLookupClient", "PaillierLookupGPUClient"}: + return "PaillierLookupClient" + return client_name + + class ExecutionContext: """Bundles a homomorphic encryption client and an AES key under a single key-provider-protected object. An ``ExecutionContext`` is the root secret for a XTrace deployment. It holds: - - A homomorphic client (``PaillierClient`` or ``PaillierLookupClient``) whose secret key is - used to decrypt Hamming distances returned by the XTrace server. + - A homomorphic client (``PaillierClient``, ``PaillierLookupClient``, + ``PaillierGPUClient``, or ``PaillierLookupGPUClient``) whose secret key is used to + decrypt Hamming distances returned by the XTrace server. - An AES key supplied by a :class:`KeyProvider`, used to encrypt chunk content before upload. The secret key is **never transmitted in plaintext** — it is AES-encrypted with the key provider's key before any remote storage. - :param homomorphic_client: An initialised ``PaillierClient`` or ``PaillierLookupClient``. + :param homomorphic_client: An initialised supported homomorphic client. :param key_provider: A :class:`KeyProvider` that supplies the AES encryption key. :param context_id: Optional deterministic ID. If omitted, one is derived from a SHA-256 hash of the public key and configuration. @@ -92,7 +103,8 @@ def create( :param passphrase: Secret passphrase used to derive the AES encryption key and protect the homomorphic secret key at rest. - :param homomorphic_client_type: ``"paillier"`` or ``"paillier_lookup"``. + :param homomorphic_client_type: ``"paillier"``, ``"paillier_lookup"``, + ``"paillier_gpu"``, or ``"paillier_lookup_gpu"``. :param embedding_length: Dimension of the binary embedding vectors (must match the model). :param key_len: RSA modulus size in bits (minimum ``1024``). :param salt: Optional salt bytes for passphrase-based key derivation. @@ -104,17 +116,26 @@ def create( """ provider = _resolve_key_provider(key_provider, passphrase, salt) - if homomorphic_client_type.lower() in ("paillier", "paillier_lookup") and embedding_length >= key_len: + if homomorphic_client_type.lower() in ( + "paillier", + "paillier_lookup", + "paillier_gpu", + "paillier_lookup_gpu", + ) and embedding_length >= key_len: raise ValueError( f"embedding_length ({embedding_length}) must be strictly less than key_len ({key_len}). " f"The Paillier-Lookup scheme requires embed_len < key_len to guarantee the padded " f"plaintext fits within the RSA modulus." ) - homomorphic_client: PaillierClient | PaillierLookupClient + homomorphic_client: HomomorphicClient if homomorphic_client_type.lower() == "paillier": homomorphic_client = PaillierClient(embed_len=embedding_length, key_len=key_len) elif homomorphic_client_type.lower() == "paillier_lookup": homomorphic_client = PaillierLookupClient(embed_len=embedding_length, key_len=key_len) + elif homomorphic_client_type.lower() == "paillier_gpu": + homomorphic_client = PaillierGPUClient(embed_len=embedding_length, key_len=key_len) + elif homomorphic_client_type.lower() == "paillier_lookup_gpu": + homomorphic_client = PaillierLookupGPUClient(embed_len=embedding_length, key_len=key_len) else: raise ValueError(f"Unsupported homomorphic client type: {homomorphic_client_type}") ctx = cls(homomorphic_client, provider) @@ -167,14 +188,15 @@ def __str__(self) -> str: def hash(self) -> str: """Compute a deterministic SHA-256 fingerprint of this context's cryptographic identity. - The ``device`` field is excluded so that CPU and GPU contexts sharing the same keys compare - as equal. + The ``device`` field and GPU wrapper type are excluded so that CPU and GPU contexts + sharing the same keys compare as equal. :return: Hex-encoded SHA-256 digest. :rtype: str """ _log.debug("Computing execution context hash...") data = self.to_dict_plain() + data["type"] = _canonical_homomorphic_type(data["type"]) data['config'] = self.homomorphic.stringify_config() str_data = json.dumps(data) hash_obj = hashlib.sha256(str_data.encode('utf-8')) @@ -247,21 +269,29 @@ def _from_serialized_exec_context( if json_obj["type"] not in SUPPORTED_HOMOMORPHIC_CLIENTS: raise ValueError(f"Unsupported homomorphic client type: {json_obj['type']}") - concrete_client: PaillierClient | PaillierLookupClient + concrete_client: HomomorphicClient if json_obj["type"] == "PaillierLookupClient": concrete_client = PaillierLookupClient(embed_len=config["embed_len"], key_len=config["key_len"], alpha_len=config["alpha_len"], skip_key_gen=True) + elif json_obj["type"] == "PaillierLookupGPUClient": + concrete_client = PaillierLookupGPUClient(embed_len=config["embed_len"], key_len=config["key_len"], alpha_len=config["alpha_len"], skip_key_gen=True) elif json_obj["type"] == "PaillierClient": concrete_client = PaillierClient(embed_len=config["embed_len"], key_len=config["key_len"], skip_key_gen=True) + elif json_obj["type"] == "PaillierGPUClient": + concrete_client = PaillierGPUClient(embed_len=config["embed_len"], key_len=config["key_len"], skip_key_gen=True) + else: + raise ValueError(f"Unsupported homomorphic client type: {json_obj['type']}") concrete_client.load_stringified_keys(json_obj['pk'], sk) # Inject tables if available to skip recomputation precomputed_tables = json_obj.get("tables") - if isinstance(concrete_client, PaillierLookupClient): + if isinstance(concrete_client, (PaillierLookupClient, PaillierLookupGPUClient)): concrete_client.load_config(config, precomputed_tables=precomputed_tables) else: - concrete_client.load_config(config) + load_config = getattr(concrete_client, "load_config", None) + if load_config is not None: + load_config(config) return cls(concrete_client, provider, context_id=context_id) diff --git a/src/xtrace_sdk/x_vec/utils/settings.py b/src/xtrace_sdk/x_vec/utils/settings.py index 1666848..360fc6d 100644 --- a/src/xtrace_sdk/x_vec/utils/settings.py +++ b/src/xtrace_sdk/x_vec/utils/settings.py @@ -1,3 +1,8 @@ -SUPPORTED_HOMOMORPHIC_CLIENTS = ["PaillierClient", "PaillierLookupClient"] +SUPPORTED_HOMOMORPHIC_CLIENTS = [ + "PaillierClient", + "PaillierLookupClient", + "PaillierGPUClient", + "PaillierLookupGPUClient", +] -REQUIRED_METADATA_TAGS = ["tag1", "tag2", "tag3", "tag4", "tag5", "facets"] \ No newline at end of file +REQUIRED_METADATA_TAGS = ["tag1", "tag2", "tag3", "tag4", "tag5", "facets"] diff --git a/tests/x_vec/test_GPU_paillier_clients.py b/tests/x_vec/test_GPU_paillier_clients.py new file mode 100644 index 0000000..d778504 --- /dev/null +++ b/tests/x_vec/test_GPU_paillier_clients.py @@ -0,0 +1,130 @@ +import json + +import pytest + +from xtrace_sdk.x_vec.crypto.paillier_gpu_client import PaillierGPUClient +from xtrace_sdk.x_vec.crypto.paillier_lookup_gpu_client import PaillierLookupGPUClient +from xtrace_sdk.x_vec.utils.execution_context import ExecutionContext + + +_PASSPHRASE = "test-gpu-exec-ctx-passphrase" +_EMBED_LEN = 8 +_KEY_LEN = 64 +_VECTORS = [ + [0, 1, 0, 1, 1, 0, 1, 0], + [1, 1, 0, 0, 1, 0, 0, 1], + [1, 0, 1, 0, 1, 1, 0, 0], +] + + +def _hamming(lhs: list[int], rhs: list[int]) -> int: + return sum(int(a != b) for a, b in zip(lhs, rhs, strict=True)) + + +def _clone_client(client_type: str) -> PaillierGPUClient | PaillierLookupGPUClient: + if client_type == "paillier_gpu": + return PaillierGPUClient(embed_len=_EMBED_LEN, key_len=_KEY_LEN, skip_key_gen=True) + return PaillierLookupGPUClient(embed_len=_EMBED_LEN, key_len=_KEY_LEN, skip_key_gen=True) + + +@pytest.mark.parametrize( + ("client_type", "client_cls"), + [ + pytest.param("paillier_gpu", PaillierGPUClient, id="paillier_gpu"), + pytest.param("paillier_lookup_gpu", PaillierLookupGPUClient, id="paillier_lookup_gpu"), + ], +) +def test_gpu_clients_expose_execution_context_protocol( + client_type: str, + client_cls: type[PaillierGPUClient] | type[PaillierLookupGPUClient], +) -> None: + if not client_cls.is_available(): + pytest.skip(f"{client_type} backend is unavailable on this machine") + + ctx = ExecutionContext.create( + passphrase=_PASSPHRASE, + homomorphic_client_type=client_type, + embedding_length=_EMBED_LEN, + key_len=_KEY_LEN, + ) + client = ctx.homomorphic + + assert ctx.device == "gpu" + assert ctx.embed_len() == _EMBED_LEN + assert ctx.key_len() == _KEY_LEN + + pk = client.stringify_pk() + sk = client.stringify_sk() + config = client.stringify_config() + config_dict = json.loads(config) + + assert json.loads(pk) + assert json.loads(sk) + assert config_dict["embed_len"] == _EMBED_LEN + assert config_dict["key_len"] == _KEY_LEN + + cipher_one = client.encrypt_vec_one(_VECTORS[0]) + batch_ciphers = client.encrypt_vec_batch(_VECTORS) + + assert len(batch_ciphers) == len(_VECTORS) + assert len(cipher_one) == len(batch_ciphers[0]) > 0 + + encoded_one = client.encode_hamming_server(cipher_one, batch_ciphers[1]) + assert client.decode_hamming_client_one(encoded_one) == _hamming(_VECTORS[0], _VECTORS[1]) + + encoded_batch = [ + client.encode_hamming_server(batch_ciphers[0], batch_ciphers[1]), + client.encode_hamming_server(batch_ciphers[1], batch_ciphers[2]), + ] + assert client.decode_hamming_client_batch(encoded_batch) == [ + _hamming(_VECTORS[0], _VECTORS[1]), + _hamming(_VECTORS[1], _VECTORS[2]), + ] + + clone = _clone_client(client_type) + clone.load_stringified_keys(pk, sk) + clone.load_config(config_dict) + + clone_cipher = clone.encrypt_vec_one(_VECTORS[0]) + clone_encoded = clone.encode_hamming_server(clone_cipher, clone.encrypt_vec_one(_VECTORS[1])) + + assert json.loads(clone.stringify_pk()) == json.loads(pk) + assert json.loads(clone.stringify_sk()) == json.loads(sk) + assert json.loads(clone.stringify_config()) == config_dict + assert clone.decode_hamming_client_one(clone_encoded) == _hamming(_VECTORS[0], _VECTORS[1]) + + +@pytest.mark.parametrize( + ("client_type", "client_cls"), + [ + pytest.param("paillier_gpu", PaillierGPUClient, id="paillier_gpu"), + pytest.param("paillier_lookup_gpu", PaillierLookupGPUClient, id="paillier_lookup_gpu"), + ], +) +def test_gpu_execution_context_serialization_roundtrip( + client_type: str, + client_cls: type[PaillierGPUClient] | type[PaillierLookupGPUClient], +) -> None: + if not client_cls.is_available(): + pytest.skip(f"{client_type} backend is unavailable on this machine") + + ctx = ExecutionContext.create( + passphrase=_PASSPHRASE, + homomorphic_client_type=client_type, + embedding_length=_EMBED_LEN, + key_len=_KEY_LEN, + ) + restored = ExecutionContext._from_serialized_exec_context( + json.loads(ctx.serialize_exec_context()), + passphrase=_PASSPHRASE, + ) + + assert restored.hash() == ctx.hash() + assert restored.device == "gpu" + assert type(restored.homomorphic).__name__ == type(ctx.homomorphic).__name__ + + encoded = restored.homomorphic.encode_hamming_server( + restored.homomorphic.encrypt_vec_one(_VECTORS[0]), + restored.homomorphic.encrypt_vec_one(_VECTORS[2]), + ) + assert restored.homomorphic.decode_hamming_client_one(encoded) == _hamming(_VECTORS[0], _VECTORS[2]) From fa0f8b63a030a71b18e99d860c3904651f79fbf2 Mon Sep 17 00:00:00 2001 From: "Liwen.O" Date: Sat, 18 Apr 2026 01:01:55 +0000 Subject: [PATCH 02/10] Add GPU build tooling and docs; route lookup client through shared GPU loader - build_gpu_binaries.sh: Docker-based build for the paillier GPU .so files - Route PaillierLookupGPU through load_gpu_extension so its missing-.so error matches the non-lookup path and points at the build script - Simplify encrypt/decode paths (drop the bytes-packing shim since the extension now returns ints directly) - Fix Repository/Issues URLs in pyproject.toml (xtrace-vec-sdk -> xtrace-sdk) - Document GPU setup and update CHANGELOG/README --- CHANGELOG.md | 14 +++ README.md | 29 +++++ build_gpu_binaries.sh | 118 ++++++++++++++++++ docs/manual_rst/configuration.rst | 27 ++-- docs/manual_rst/install.rst | 100 ++++++++++++++- pyproject.toml | 4 +- src/xtrace_sdk/x_vec/crypto/_gpu_loader.py | 6 +- .../x_vec/crypto/paillier_lookup_client.py | 54 +++----- 8 files changed, 304 insertions(+), 48 deletions(-) create mode 100755 build_gpu_binaries.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 552cf8f..45a7180 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ All notable changes to this project will be documented in this file. +## [0.2.0] - 2026-04-18 + +### Added + +- **CUDA backend for Paillier and Paillier-Lookup homomorphic encryption.** The `.cu` sources are now in-tree under `src/xtrace_sdk/x_vec/crypto/paillier-GPU-{,lookup-}client/`, and the new `build_gpu_binaries.sh` script compiles both pybind11 extensions inside a pinned `nvidia/cuda:12.4.0-devel-ubuntu22.04` Docker container. No local CUDA toolkit is required on the build host — only Docker and `nvcc` (provided by the image). Runtime requires NVIDIA driver ≥ 550. Opt-in via `DEVICE=gpu`; CPU remains the default. +- `PaillierGPUClient` and `PaillierLookupGPUClient` — standalone GPU-backed Hamming clients for `ExecutionContext`. Type dispatch in `ExecutionContext.create` now accepts these as first-class alternatives to their CPU counterparts. +- Documented `DEVICE` environment variable in the configuration reference. +- "GPU acceleration" section in the installation docs covering build script usage, runtime requirements, and per-build knobs (`PYTHON_VERSION`, `SMS`, `KEY_BITS`, `ALPHA_LEN`, `CUDA_IMAGE`). + +### Changed + +- `DEVICE` is now resolved at client instantiation time rather than at module import time. Previously, setting `os.environ["DEVICE"]` after importing a client (common in notebooks) had no effect; the module-level constant was already frozen to `cpu`. The env var is now read fresh inside `PaillierClient.__init__`, `PaillierLookupClient.__init__`, and `PaillierLookupClient.__setstate__`, so CPU and GPU clients can be instantiated in the same process and ordering of imports no longer matters. +- Error messages raised when the GPU extension cannot be loaded now point users at `./build_gpu_binaries.sh` and document runtime driver requirements, replacing the previous stale reference to `src/crypto/` and non-existent documentation. + ## [0.1.1] - 2026-03-31 ### Fixed diff --git a/README.md b/README.md index 6c8ddfb..c484f54 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,35 @@ uv pip install "xtrace-ai-sdk[embedding]" Requires Python 3.11+. +## GPU acceleration (optional) + +The Paillier homomorphic encryption path has an optional CUDA backend that is significantly faster than the CPU implementation for large batches. It's opt-in: by default the SDK runs on CPU and requires no extra setup. + +**Prebuilt binaries are not currently shipped on PyPI.** To use the GPU path, compile the pybind11 extensions yourself: + +```bash +# From the repo root. Requires Docker only — no local CUDA toolkit needed. +./build_gpu_binaries.sh +``` + +This produces two `.so` files in place under `src/xtrace_sdk/x_vec/crypto/paillier-GPU-{,lookup-}client/`. The runtime loader picks them up automatically once they exist. + +Enable the GPU path at runtime by setting `DEVICE=gpu` **before** importing any Paillier client: + +```python +import os +os.environ["DEVICE"] = "gpu" +from xtrace_sdk.x_vec.crypto.paillier_client import PaillierClient +``` + +**Requirements:** + +- Docker (to run the build; ~10 GB for the `nvidia/cuda` devel image) +- NVIDIA driver ≥ 550 (i.e. CUDA 12.x runtime) on the host that will execute the GPU code +- An NVIDIA GPU with compute capability 7.0–9.0 (V100, T4, A100, RTX 30/40, H100, etc.) + +**Knobs (env vars on the build script):** `PYTHON_VERSION` (default `3.11`), `SMS` (GPU arch list), `KEY_BITS`, `ALPHA_LEN` (lookup variant), `CUDA_IMAGE`. See the header of [`build_gpu_binaries.sh`](./build_gpu_binaries.sh) for defaults. + ## Documentation Full documentation at [docs.xtrace.ai](https://docs.xtrace.ai), or build locally: diff --git a/build_gpu_binaries.sh b/build_gpu_binaries.sh new file mode 100755 index 0000000..52b8b10 --- /dev/null +++ b/build_gpu_binaries.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# +# Build the Paillier GPU and Paillier-Lookup GPU pybind11 extensions inside a +# pinned nvidia/cuda Docker image. Produces the two .so files that +# xtrace_sdk.x_vec.crypto._gpu_loader expects to find in-tree: +# +# src/xtrace_sdk/x_vec/crypto/paillier-GPU-client/paillier_GPU_client*.so +# src/xtrace_sdk/x_vec/crypto/paillier-GPU-lookup-client/paillier_GPU_lookup_client*.so +# +# Overridable via env vars: +# PYTHON_VERSION Python minor version (default: 3.11) +# SMS Space-separated GPU SM list (default: "70 75 80 86 89 90 90a") +# KEY_BITS Paillier key size (default: 1024) +# ALPHA_LEN Lookup variant alpha length (default: 280) +# CUDA_IMAGE nvidia/cuda devel image (default: 12.4.0-devel-ubuntu22.04) +# CGBN_REPO_URL CGBN header-only lib source (default: NVlabs/CGBN on GitHub) +# +# Only nvcc is needed to compile — no GPU is required, so this runs on a +# standard CI runner (or Apple Silicon via --platform linux/amd64). +# +set -euo pipefail + +PYTHON_VERSION="${PYTHON_VERSION:-3.11}" +SMS="${SMS:-70 75 80 86 89 90 90a}" +KEY_BITS="${KEY_BITS:-1024}" +ALPHA_LEN="${ALPHA_LEN:-280}" +CUDA_IMAGE="${CUDA_IMAGE:-nvidia/cuda:12.4.0-devel-ubuntu22.04}" +CGBN_REPO_URL="${CGBN_REPO_URL:-https://github.com/NVlabs/CGBN.git}" + +REPO_ROOT="$(cd "$(dirname "$0")" && pwd)" +CRYPTO_DIR="$REPO_ROOT/src/xtrace_sdk/x_vec/crypto" + +if [ ! -d "$CRYPTO_DIR/paillier-GPU-client" ] || \ + [ ! -d "$CRYPTO_DIR/paillier-GPU-lookup-client" ]; then + echo "Error: expected GPU source dirs under $CRYPTO_DIR" >&2 + exit 1 +fi + +# Fall back through SUDO_UID/GID so `sudo ./build_gpu_binaries.sh` still chowns +# artifacts back to the invoking user, not root. +HOST_UID="${SUDO_UID:-$(id -u)}" +HOST_GID="${SUDO_GID:-$(id -g)}" + +echo "Building Paillier GPU extensions" +echo " Python: $PYTHON_VERSION" +echo " CUDA img: $CUDA_IMAGE" +echo " SMs: $SMS" +echo " KEY_BITS: $KEY_BITS" +echo " ALPHA_LEN: $ALPHA_LEN (lookup variant only)" +echo " Output: $CRYPTO_DIR/paillier-GPU-{,lookup-}client/*.so" +echo + +docker run --rm \ + --platform linux/amd64 \ + --env DEBIAN_FRONTEND=noninteractive \ + --env PYTHON_VERSION="$PYTHON_VERSION" \ + --env SMS="$SMS" \ + --env KEY_BITS="$KEY_BITS" \ + --env ALPHA_LEN="$ALPHA_LEN" \ + --env CGBN_REPO_URL="$CGBN_REPO_URL" \ + --env HOST_UID="$HOST_UID" \ + --env HOST_GID="$HOST_GID" \ + -v "$REPO_ROOT:/build/repo" \ + -w /build/repo \ + "$CUDA_IMAGE" \ + /bin/bash -c ' + set -euo pipefail + + apt-get update + apt-get install -y --no-install-recommends \ + software-properties-common gnupg2 ca-certificates \ + git build-essential libgmp-dev curl + + add-apt-repository -y ppa:deadsnakes/ppa + apt-get update + apt-get install -y --no-install-recommends \ + "python${PYTHON_VERSION}-dev" \ + "python${PYTHON_VERSION}-venv" + + update-alternatives --install /usr/bin/python3 python3 "/usr/bin/python${PYTHON_VERSION}" 1 + + "python${PYTHON_VERSION}" -m ensurepip --upgrade + "python${PYTHON_VERSION}" -m pip install --no-cache-dir pybind11 + + # Materialise the ../include/{pybind11,cgbn} layout the Makefiles expect. + INCDIR=/build/include + mkdir -p "$INCDIR" + PYBIND_INC="$("python${PYTHON_VERSION}" -c "import pybind11; print(pybind11.get_include())")" + ln -sfn "$PYBIND_INC/pybind11" "$INCDIR/pybind11" + + git clone --depth=1 "$CGBN_REPO_URL" /opt/CGBN + ln -sfn /opt/CGBN/include/cgbn "$INCDIR/cgbn" + + build_one() { + local dir="$1" + shift + echo "=== Building $dir ===" + cd "/build/repo/src/xtrace_sdk/x_vec/crypto/$dir" + make clean || true + make INCDIR="$INCDIR" SMS="$SMS" KEY_BITS="$KEY_BITS" "$@" + ls -lh ./*.so + } + + build_one paillier-GPU-client + build_one paillier-GPU-lookup-client ALPHA_LEN="$ALPHA_LEN" + + chown -R "$HOST_UID:$HOST_GID" \ + /build/repo/src/xtrace_sdk/x_vec/crypto/paillier-GPU-client \ + /build/repo/src/xtrace_sdk/x_vec/crypto/paillier-GPU-lookup-client + + echo "Done." + ' + +echo +echo "Artifacts:" +ls -lh "$CRYPTO_DIR/paillier-GPU-client/"*.so \ + "$CRYPTO_DIR/paillier-GPU-lookup-client/"*.so 2>/dev/null || \ + echo " (no .so produced — check output above)" diff --git a/docs/manual_rst/configuration.rst b/docs/manual_rst/configuration.rst index 59bfc18..ca959e0 100644 --- a/docs/manual_rst/configuration.rst +++ b/docs/manual_rst/configuration.rst @@ -13,15 +13,24 @@ Both ``PaillierClient`` and ``PaillierLookupClient`` run on CPU by default. .. note:: - **GPU backend (internal testing phase).** XTrace maintains a GPU-accelerated implementation of the - homomorphic encryption layer that is approximately **20× faster** than the CPU path for large - embedding collections. It is available as a compiled extension that slots into the same - ``DEVICE=gpu`` switch — no application code changes required. + **GPU backend (optional).** A CUDA-accelerated implementation of the Paillier + and Paillier-Lookup clients is included in the source tree (``~20× faster`` + than the CPU path on large embedding collections). It is **not** shipped + prebuilt on PyPI — compile it yourself with the included Docker-based build + script. See :doc:`install` → *GPU acceleration* for build instructions and + runtime requirements (NVIDIA driver ≥ 550). - The GPU implementation is not open-sourced at this time as it is under internal testing. Contact us at - `liwen@xtrace.ai `_ if you are interested in access. +Once the compiled extension is in place, activate the GPU backend by setting +``DEVICE=gpu`` before instantiating a client. The value is read at each +instantiation, so CPU and GPU clients can coexist in the same process: -Set ``DEVICE=gpu`` to activate the GPU backend once the compiled extension is in place. +.. code-block:: python + + import os + os.environ["DEVICE"] = "gpu" + + from xtrace_sdk.x_vec.crypto.paillier_client import PaillierClient + client = PaillierClient(embed_len=512, key_len=1024) **Paillier** — standard Paillier encryption: @@ -231,3 +240,7 @@ The following environment variables are read automatically: - Default path to a saved execution context. * - ``INFERENCE_API_KEY`` - API key for your inference provider (OpenAI, Redpill, etc.). + * - ``DEVICE`` + - Backend for ``PaillierClient`` / ``PaillierLookupClient``. Set to + ``gpu`` to use the CUDA extension (must be compiled — see :doc:`install`). + Defaults to ``cpu``. Read at each client instantiation. diff --git a/docs/manual_rst/install.rst b/docs/manual_rst/install.rst index 362db11..535e413 100644 --- a/docs/manual_rst/install.rst +++ b/docs/manual_rst/install.rst @@ -61,8 +61,8 @@ Install from source .. code-block:: bash - git clone https://github.com/XTraceAI/xtrace-vec-sdk.git - cd xtrace-vec-sdk + git clone https://github.com/XTraceAI/xtrace-sdk.git + cd xtrace-sdk pip install -e . To include extras: @@ -70,3 +70,99 @@ To include extras: .. code-block:: bash pip install -e ".[embedding,cli]" + + +GPU acceleration (optional) +--------------------------- + +The Paillier and Paillier-Lookup homomorphic encryption clients ship with an +optional CUDA backend that is roughly **20× faster** than the CPU path on large +embedding collections. It is opt-in — the base install runs fully on CPU and +requires no GPU, no CUDA toolkit, and no Docker. + +Prebuilt binaries are **not** currently published on PyPI. To enable the GPU +path, compile the pybind11 extensions from source using the included Docker-based +build script: + +.. code-block:: bash + + git clone https://github.com/XTraceAI/xtrace-sdk.git + cd xtrace-sdk + ./build_gpu_binaries.sh + +The script spins up an ``nvidia/cuda`` devel container, fetches ``pybind11`` and +``CGBN`` headers, and compiles two ``.so`` files that land in place under +``src/xtrace_sdk/x_vec/crypto/paillier-GPU-{,lookup-}client/``. The SDK's runtime +loader picks them up automatically on import. No NVIDIA GPU is required on the +*build* host — only ``nvcc``, which is provided by the Docker image. + +Enable the GPU backend at runtime by setting the ``DEVICE`` environment variable: + +.. code-block:: python + + import os + os.environ["DEVICE"] = "gpu" + + from xtrace_sdk.x_vec.crypto.paillier_client import PaillierClient + client = PaillierClient(embed_len=512, key_len=1024) # instantiates the GPU backend + +``DEVICE`` is read every time a client is instantiated, so it is safe to change +at runtime (for example, to run CPU and GPU clients side by side in the same +process). + +Requirements +^^^^^^^^^^^^ + +.. list-table:: + :header-rows: 1 + :widths: 25 75 + + * - Component + - Version + * - Build host + - Docker (any recent version). ~10 GB free disk for the CUDA devel image. + * - Runtime host + - NVIDIA driver **≥ 550** (CUDA 12.x runtime), with an NVIDIA GPU of + compute capability 7.0–9.0 (V100, T4, A100, RTX 30/40, H100, etc.). + +Build-time configuration +^^^^^^^^^^^^^^^^^^^^^^^^ + +All parameters are environment variables on the build script: + +.. list-table:: + :header-rows: 1 + + * - Variable + - Default + - Description + * - ``PYTHON_VERSION`` + - ``3.11`` + - CPython minor version. Must match the Python you will import the SDK from + (the ``.so`` is ABI-tagged per minor version). + * - ``SMS`` + - ``70 75 80 86 89 90 90a`` + - Space-separated GPU SM targets. Default fat-binary covers V100, T4, + A100, RTX 30/40, and H100 in a single file. + * - ``KEY_BITS`` + - ``1024`` + - Paillier modulus size (bits). Must match the ``key_len`` passed to the + client at runtime. + * - ``ALPHA_LEN`` + - ``280`` + - Paillier-Lookup alpha length. Lookup variant only. + * - ``CUDA_IMAGE`` + - ``nvidia/cuda:12.4.0-devel-ubuntu22.04`` + - Docker image used to compile. Swap for ``nvidia/cuda:11.x-devel-*`` if + you need a CUDA 11 runtime. + * - ``CGBN_REPO_URL`` + - NVlabs/CGBN on GitHub + - Override if you are building behind an air-gapped mirror. + +Example — build a CUDA 11 / Python 3.12 binary: + +.. code-block:: bash + + PYTHON_VERSION=3.12 \ + CUDA_IMAGE=nvidia/cuda:11.8.0-devel-ubuntu22.04 \ + ./build_gpu_binaries.sh diff --git a/pyproject.toml b/pyproject.toml index a417af1..1c89998 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,8 +24,8 @@ classifiers = ["Programming Language :: Python :: 3"] [project.urls] Homepage = "https://xtrace.ai" Documentation = "https://docs.xtrace.ai" -Repository = "https://github.com/XTraceAI/xtrace-vec-sdk" -Issues = "https://github.com/XTraceAI/xtrace-vec-sdk/issues" +Repository = "https://github.com/XTraceAI/xtrace-sdk" +Issues = "https://github.com/XTraceAI/xtrace-sdk/issues" [project.scripts] xtrace = "xtrace_sdk.cli.main:app" diff --git a/src/xtrace_sdk/x_vec/crypto/_gpu_loader.py b/src/xtrace_sdk/x_vec/crypto/_gpu_loader.py index c9539da..5324c36 100644 --- a/src/xtrace_sdk/x_vec/crypto/_gpu_loader.py +++ b/src/xtrace_sdk/x_vec/crypto/_gpu_loader.py @@ -19,7 +19,11 @@ def load_gpu_extension(module_name: str, relative_dir: str) -> ModuleType: candidates.extend(sorted(search_dir.glob(f"{module_name}*{pattern[1:]}"))) if not candidates: - raise ImportError(f"Unable to locate GPU extension {module_name!r} under {search_dir}.") + raise ImportError( + f"Unable to locate GPU extension {module_name!r} under {search_dir}. " + "To build it, run `./build_gpu_binaries.sh` from the repo root " + "(requires Docker). Runtime requires NVIDIA driver >= 550 (CUDA 12.x)." + ) module_path = candidates[0] spec = importlib.util.spec_from_file_location(module_name, module_path) diff --git a/src/xtrace_sdk/x_vec/crypto/paillier_lookup_client.py b/src/xtrace_sdk/x_vec/crypto/paillier_lookup_client.py index 4286527..3ec3ec0 100644 --- a/src/xtrace_sdk/x_vec/crypto/paillier_lookup_client.py +++ b/src/xtrace_sdk/x_vec/crypto/paillier_lookup_client.py @@ -9,7 +9,8 @@ from xtrace_sdk.x_vec.crypto.hamming_client_base import HammingClientBase from xtrace_sdk.x_vec.utils.xtrace_types import PaillierEncryptedNumber, PaillierLookupKeyPair -DEVICE = os.getenv("DEVICE", "cpu") +def _resolve_device() -> str: + return os.getenv("DEVICE", "cpu") @@ -21,15 +22,11 @@ class PaillierLookupGPU: """ def __init__(self,embed_len:int=512, key_len:int=1024, alpha_len:int=50, skip_key_gen:bool=False) -> None: - try: - #lazy import here to avoid import error when GPU backend is not available - from .paillier_GPU_lookup_client import PaillierGPULookupClient - except Exception as e: - raise ImportError( - "GPU backend unavailable. Expected Paillier GPU .so under src/crypto/ " - "Please refer to the documentation for compilation instructions." - ) from e - self.gpu_client = PaillierGPULookupClient(embed_len=embed_len, key_len=key_len, alpha_len=alpha_len ,skip_key_gen=skip_key_gen) + from xtrace_sdk.x_vec.crypto._gpu_loader import load_gpu_extension + module = load_gpu_extension("paillier_GPU_lookup_client", "paillier-GPU-lookup-client") + self.gpu_client = module.PaillierGPULookupClient( + embed_len=embed_len, key_len=key_len, alpha_len=alpha_len, skip_key_gen=skip_key_gen, + ) def encrypt(self, embds: list[list[int]]) -> list[PaillierEncryptedNumber]: """Encrypt a batch of binary embedding vectors on GPU. @@ -39,12 +36,7 @@ def encrypt(self, embds: list[list[int]]) -> list[PaillierEncryptedNumber]: :return: List of encrypted vectors. :rtype: list[PaillierEncryptedNumber] """ - # Fast path using raw bytes integration. Eliminates heavy python string operations. - bytes_batch = self.gpu_client.encrypt_bytes(embds) - res = [] - for bytes_list in bytes_batch: - res.append([int.from_bytes(b, byteorder='little') for b in bytes_list]) - return res + return [[int(cipher) for cipher in row] for row in self.gpu_client.encrypt(embds)] def decode_hamming_client(self, cipher: list[list[int | bytes]]) -> list[int]: """Decrypt a batch of encrypted Hamming distances on GPU. @@ -54,24 +46,14 @@ def decode_hamming_client(self, cipher: list[list[int | bytes]]) -> list[int]: :return: List of plain-text Hamming distances. :rtype: list[int] """ - chunk_byte_len = (self.gpu_client.key_len * 4) // 8 - - bytes_batch = [] - for c in cipher: - # c is a list of chunks (either int or bytes) - record_bytes = [] - for h in c: - if isinstance(h, bytes): - # Ensure bytes are exactly chunk_byte_len. - # Server might send variable length if it's using compact integer representation. - if len(h) != chunk_byte_len: - h = int.from_bytes(h, byteorder='little').to_bytes(chunk_byte_len, byteorder='little') - record_bytes.append(h) - else: - record_bytes.append(int(h).to_bytes(chunk_byte_len, byteorder='little')) - bytes_batch.append(record_bytes) - - return self.gpu_client.decode_hamming_client_bytes(bytes_batch) + normalized = [ + [ + int.from_bytes(h, byteorder='little') if isinstance(h, bytes) else int(h) + for h in record + ] + for record in cipher + ] + return [int(v) for v in self.gpu_client.decode_hamming_client(normalized)] def stringify_pk(self) -> str: """Return a JSON string representation of the public key.""" @@ -329,7 +311,7 @@ def __init__(self,embed_len:int=512, key_len:int=1024, alpha_len:int=50, skip_ke else: self.chunk_num = 1 - self.device = DEVICE + self.device = _resolve_device() self.client: PaillierLookupGPU | PaillierLookupCPU if self.device == "gpu": if not self.has_gpu(): @@ -487,7 +469,7 @@ def __getstate__(self) -> dict: def __setstate__(self, state: dict) -> None: """Restore from pickle""" - self.device = DEVICE + self.device = _resolve_device() self.embed_len = state["embed_len"] self.key_len = state["key_len"] self.alpha_len = state["alpha_len"] From 1fadb697b8fca6e0e301c81c55e65b2eb42b7e10 Mon Sep 17 00:00:00 2001 From: Yavor Litchev Date: Mon, 20 Apr 2026 21:09:44 -0400 Subject: [PATCH 03/10] Added include dependency to allow for proper comilation of CUDA scripts --- .../crypto/GPU_include/cgbn/arith/arith.h | 144 + .../crypto/GPU_include/cgbn/arith/asm.cu | 441 ++ .../crypto/GPU_include/cgbn/arith/chain.cu | 165 + .../crypto/GPU_include/cgbn/arith/dmp.cu | 373 ++ .../crypto/GPU_include/cgbn/arith/math.cu | 359 ++ .../x_vec/crypto/GPU_include/cgbn/arith/mp.cu | 297 ++ .../crypto/GPU_include/cgbn/arith/shifter.cu | 180 + .../GPU_include/cgbn/arith/static_divide.cu | 42 + .../x_vec/crypto/GPU_include/cgbn/cgbn.cu | 91 + .../x_vec/crypto/GPU_include/cgbn/cgbn.h | 512 +++ .../x_vec/crypto/GPU_include/cgbn/cgbn_cpu.h | 25 + .../x_vec/crypto/GPU_include/cgbn/cgbn_cuda.h | 258 ++ .../x_vec/crypto/GPU_include/cgbn/cgbn_mpz.h | 261 ++ .../crypto/GPU_include/cgbn/core/core.cu | 327 ++ .../GPU_include/cgbn/core/core_add_sub.cu | 57 + .../cgbn/core/core_binary_inverse.cu | 79 + .../GPU_include/cgbn/core/core_compare.cu | 37 + .../GPU_include/cgbn/core/core_counting.cu | 96 + .../cgbn/core/core_divide_multi.cu | 252 ++ .../cgbn/core/core_divide_single.cu | 197 + .../crypto/GPU_include/cgbn/core/core_gcd.cu | 266 ++ .../cgbn/core/core_insert_extract.cu | 130 + .../GPU_include/cgbn/core/core_logical.cu | 72 + .../cgbn/core/core_modular_inverse.cu | 392 ++ .../crypto/GPU_include/cgbn/core/core_mont.cu | 117 + .../GPU_include/cgbn/core/core_mont_imad.cu | 157 + .../GPU_include/cgbn/core/core_mont_wmad.cu | 466 +++ .../GPU_include/cgbn/core/core_mont_xmad.cu | 396 ++ .../crypto/GPU_include/cgbn/core/core_mul.cu | 89 + .../GPU_include/cgbn/core/core_mul_imad.cu | 158 + .../GPU_include/cgbn/core/core_mul_wmad.cu | 322 ++ .../GPU_include/cgbn/core/core_mul_xmad.cu | 228 ++ .../GPU_include/cgbn/core/core_short_math.cu | 294 ++ .../GPU_include/cgbn/core/core_singleton.cu | 169 + .../GPU_include/cgbn/core/core_sqrt_multi.cu | 454 +++ .../GPU_include/cgbn/core/core_sqrt_single.cu | 326 ++ .../GPU_include/cgbn/core/dispatch_dlimbs.cu | 472 +++ .../GPU_include/cgbn/core/dispatch_masking.cu | 253 ++ .../GPU_include/cgbn/core/dispatch_padding.cu | 91 + .../cgbn/core/dispatch_resolver.cu | 27 + .../cgbn/core/dispatch_shift_rotate.cu | 136 + .../GPU_include/cgbn/core/padded_resolver.cu | 288 ++ .../GPU_include/cgbn/core/subwarp_resolver.cu | 281 ++ .../crypto/GPU_include/cgbn/core/unpadded.cu | 41 + .../GPU_include/cgbn/core/warp_resolver.cu | 257 ++ .../crypto/GPU_include/cgbn/gpu_support.h | 66 + .../crypto/GPU_include/cgbn/impl_cuda.cu | 1431 +++++++ .../x_vec/crypto/GPU_include/cgbn/impl_mpz.cc | 1362 +++++++ .../x_vec/crypto/GPU_include/pybind11/attr.h | 722 ++++ .../crypto/GPU_include/pybind11/buffer_info.h | 208 + .../x_vec/crypto/GPU_include/pybind11/cast.h | 2356 +++++++++++ .../crypto/GPU_include/pybind11/chrono.h | 228 ++ .../crypto/GPU_include/pybind11/common.h | 2 + .../crypto/GPU_include/pybind11/complex.h | 74 + .../GPU_include/pybind11/conduit/README.txt | 15 + .../pybind11/conduit/pybind11_conduit_v1.h | 116 + .../conduit/pybind11_platform_abi_id.h | 87 + .../pybind11/conduit/wrap_include_python_h.h | 72 + .../GPU_include/pybind11/critical_section.h | 56 + .../GPU_include/pybind11/detail/class.h | 823 ++++ .../GPU_include/pybind11/detail/common.h | 1310 ++++++ .../GPU_include/pybind11/detail/cpp_conduit.h | 75 + .../GPU_include/pybind11/detail/descr.h | 226 ++ .../detail/dynamic_raw_ptr_cast_if_possible.h | 39 + .../pybind11/detail/exception_translation.h | 71 + .../detail/function_record_pyobject.h | 208 + .../crypto/GPU_include/pybind11/detail/init.h | 538 +++ .../GPU_include/pybind11/detail/internals.h | 786 ++++ .../pybind11/detail/native_enum_data.h | 209 + .../detail/pybind11_namespace_macros.h | 82 + .../pybind11/detail/struct_smart_holder.h | 378 ++ .../pybind11/detail/type_caster_base.h | 1591 ++++++++ .../GPU_include/pybind11/detail/typeid.h | 65 + .../pybind11/detail/using_smart_holder.h | 22 + .../pybind11/detail/value_and_holder.h | 90 + .../x_vec/crypto/GPU_include/pybind11/eigen.h | 12 + .../GPU_include/pybind11/eigen/common.h | 9 + .../GPU_include/pybind11/eigen/matrix.h | 723 ++++ .../GPU_include/pybind11/eigen/tensor.h | 521 +++ .../x_vec/crypto/GPU_include/pybind11/embed.h | 320 ++ .../x_vec/crypto/GPU_include/pybind11/eval.h | 161 + .../crypto/GPU_include/pybind11/functional.h | 147 + .../x_vec/crypto/GPU_include/pybind11/gil.h | 199 + .../GPU_include/pybind11/gil_safe_call_once.h | 102 + .../crypto/GPU_include/pybind11/gil_simple.h | 37 + .../crypto/GPU_include/pybind11/iostream.h | 265 ++ .../crypto/GPU_include/pybind11/native_enum.h | 67 + .../x_vec/crypto/GPU_include/pybind11/numpy.h | 2312 +++++++++++ .../crypto/GPU_include/pybind11/operators.h | 202 + .../crypto/GPU_include/pybind11/options.h | 92 + .../crypto/GPU_include/pybind11/pybind11.h | 3584 +++++++++++++++++ .../crypto/GPU_include/pybind11/pytypes.h | 2680 ++++++++++++ .../x_vec/crypto/GPU_include/pybind11/stl.h | 666 +++ .../GPU_include/pybind11/stl/filesystem.h | 114 + .../crypto/GPU_include/pybind11/stl_bind.h | 858 ++++ .../GPU_include/pybind11/subinterpreter.h | 305 ++ .../pybind11/trampoline_self_life_support.h | 65 + .../pybind11/type_caster_pyobject_ptr.h | 61 + .../crypto/GPU_include/pybind11/typing.h | 298 ++ .../crypto/GPU_include/pybind11/warnings.h | 75 + src/xtrace_sdk/x_vec/crypto/_gpu_loader.py | 4 +- .../x_vec/crypto/paillier-GPU-client/Makefile | 6 +- .../paillier-GPU-lookup-client/Makefile | 6 +- .../paillier_GPU_lookup_client.cu | 54 +- .../crypto/paillier_lookup_gpu_client.py | 6 +- tests/x_vec/test_GPU_paillier_clients.py | 42 + 106 files changed, 37358 insertions(+), 18 deletions(-) create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/arith.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/asm.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/chain.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/dmp.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/math.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/mp.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/shifter.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/static_divide.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn_cpu.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn_cuda.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn_mpz.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_add_sub.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_binary_inverse.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_compare.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_counting.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_divide_multi.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_divide_single.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_gcd.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_insert_extract.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_logical.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_modular_inverse.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mont.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mont_imad.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mont_wmad.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mont_xmad.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mul.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mul_imad.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mul_wmad.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mul_xmad.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_short_math.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_singleton.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_sqrt_multi.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_sqrt_single.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_dlimbs.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_masking.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_padding.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_resolver.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_shift_rotate.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/padded_resolver.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/subwarp_resolver.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/unpadded.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/warp_resolver.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/gpu_support.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/impl_cuda.cu create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/impl_mpz.cc create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/attr.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/buffer_info.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/cast.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/chrono.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/common.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/complex.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/conduit/README.txt create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/conduit/pybind11_conduit_v1.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/conduit/pybind11_platform_abi_id.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/conduit/wrap_include_python_h.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/critical_section.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/detail/class.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/detail/common.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/detail/cpp_conduit.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/detail/descr.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/detail/exception_translation.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/detail/function_record_pyobject.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/detail/init.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/detail/internals.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/detail/native_enum_data.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/detail/pybind11_namespace_macros.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/detail/struct_smart_holder.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/detail/type_caster_base.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/detail/typeid.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/detail/using_smart_holder.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/detail/value_and_holder.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/eigen.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/eigen/common.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/eigen/matrix.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/eigen/tensor.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/embed.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/eval.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/functional.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/gil.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/gil_safe_call_once.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/gil_simple.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/iostream.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/native_enum.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/numpy.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/operators.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/options.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/pybind11.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/pytypes.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/stl.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/stl/filesystem.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/stl_bind.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/subinterpreter.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/trampoline_self_life_support.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/type_caster_pyobject_ptr.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/typing.h create mode 100644 src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/warnings.h diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/arith.h b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/arith.h new file mode 100644 index 0000000..7248af9 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/arith.h @@ -0,0 +1,144 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +/* defines */ +#ifdef NDEBUG +#define ASM_ERROR(message) asm volatile("ASM_ERROR " message) +#else +#define ASM_ERROR(message) +#endif + +/* static math */ +template __device__ __forceinline__ uint32_t static_divide_small(uint32_t numerator); +template __device__ __forceinline__ uint32_t static_remainder_small(uint32_t numerator); + +/* asm routines */ +__device__ __forceinline__ uint32_t add_cc(uint32_t a, uint32_t b); +__device__ __forceinline__ uint32_t addc_cc(uint32_t a, uint32_t b); +__device__ __forceinline__ uint32_t addc(uint32_t a, uint32_t b); +__device__ __forceinline__ uint32_t sub_cc(uint32_t a, uint32_t b); +__device__ __forceinline__ uint32_t subc_cc(uint32_t a, uint32_t b); +__device__ __forceinline__ uint32_t subc(uint32_t a, uint32_t b); +__device__ __forceinline__ uint32_t madlo(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t madlo_cc(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t madloc_cc(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t madloc(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t madhi(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t madhi_cc(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t madhic_cc(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t madhic(uint32_t a, uint32_t b, uint32_t c); + +/* xmad routines */ +__device__ __forceinline__ uint32_t xmadll(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t xmadll_cc(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t xmadllc_cc(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t xmadllc(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t xmadlh(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t xmadlh_cc(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t xmadlhc_cc(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t xmadlhc(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t xmadhl(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t xmadhl_cc(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t xmadhlc_cc(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t xmadhlc(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t xmadhh(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t xmadhh_cc(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t xmadhhc_cc(uint32_t a, uint32_t b, uint32_t c); +__device__ __forceinline__ uint32_t xmadhhc(uint32_t a, uint32_t b, uint32_t c); + +/* funnel shifts */ +__device__ __forceinline__ uint32_t uleft_clamp(uint32_t lo, uint32_t hi, uint32_t amt); +__device__ __forceinline__ uint32_t uright_clamp(uint32_t lo, uint32_t hi, uint32_t amt); + +__device__ __forceinline__ uint32_t uleft_wrap(uint32_t lo, uint32_t hi, uint32_t amt); +__device__ __forceinline__ uint32_t uright_wrap(uint32_t lo, uint32_t hi, uint32_t amt); + +__device__ __forceinline__ uint32_t uabs(int32_t x); + +#define CGBN_INF_CHAIN 0xFFFFFFFF + +/* classes */ +template +class chain_t { + public: + uint32_t _position; + + __device__ __forceinline__ chain_t(); + __device__ __forceinline__ ~chain_t(); + __device__ __forceinline__ uint32_t add(uint32_t a, uint32_t b); + __device__ __forceinline__ uint32_t sub(uint32_t a, uint32_t b); + __device__ __forceinline__ uint32_t madlo(uint32_t a, uint32_t b, uint32_t c); + __device__ __forceinline__ uint32_t madhi(uint32_t a, uint32_t b, uint32_t c); + __device__ __forceinline__ uint32_t xmadll(uint32_t a, uint32_t b, uint32_t c); + __device__ __forceinline__ uint32_t xmadlh(uint32_t a, uint32_t b, uint32_t c); + __device__ __forceinline__ uint32_t xmadhl(uint32_t a, uint32_t b, uint32_t c); + __device__ __forceinline__ uint32_t xmadhh(uint32_t a, uint32_t b, uint32_t c); +}; + + +/* uint32 math routines */ +__device__ __forceinline__ uint32_t uclz(uint32_t x); +__device__ __forceinline__ uint32_t uctz(uint32_t x); +__device__ __forceinline__ uint32_t ubinary_inverse(uint32_t x); +__device__ __forceinline__ uint32_t ugcd(uint32_t a, uint32_t b); +__device__ __forceinline__ uint32_t uapprox(const uint32_t d); +__device__ __forceinline__ uint32_t udiv(const uint32_t lo, const uint32_t hi, const uint32_t d, const uint32_t approx); +__device__ __forceinline__ uint32_t udiv(const uint32_t a0, const uint32_t a1, const uint32_t a2, const uint32_t d0, const uint32_t d1, const uint32_t approx); + + +/* mp math routines */ +template __device__ __forceinline__ uint32_t mplor(const uint32_t a[limbs]); +template __device__ __forceinline__ uint32_t mpland(const uint32_t a[limbs]); +template __device__ __forceinline__ bool mpzeros(const uint32_t a[limbs]); +template __device__ __forceinline__ bool mpones(const uint32_t a[limbs]); + +template __device__ __forceinline__ void mpadd32_cc(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b); +template __device__ __forceinline__ uint32_t mpadd32(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b); +template __device__ __forceinline__ void mpadd_cc(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b[limbs]); +template __device__ __forceinline__ uint32_t mpadd(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b[limbs]); + +template __device__ __forceinline__ void mpsub32_cc(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b); +template __device__ __forceinline__ uint32_t mpsub32(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b); +template __device__ __forceinline__ void mpsub_cc(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b[limbs]); +template __device__ __forceinline__ uint32_t mpsub(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b[limbs]); + +template __device__ __forceinline__ uint32_t mpmul32(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b); +template __device__ __forceinline__ uint32_t mprem32(const uint32_t a[limbs], const uint32_t d, const uint32_t approx); + +template __device__ __forceinline__ void mpmul(uint32_t lo[limbs], uint32_t hi[limbs], const uint32_t a[limbs], const uint32_t b[limbs]); + +template __device__ __forceinline__ void mprotate_left(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t numlimbs); +template __device__ __forceinline__ void mprotate_right(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t numlimbs); + +} /* CGBN namespace */ + +#include "static_divide.cu" +#include "asm.cu" +#include "chain.cu" +#include "math.cu" +#include "shifter.cu" +#include "mp.cu" +#include "dmp.cu" \ No newline at end of file diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/asm.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/asm.cu new file mode 100644 index 0000000..2df2e77 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/asm.cu @@ -0,0 +1,441 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { +__device__ __forceinline__ uint32_t add_cc(uint32_t a, uint32_t b) { + uint32_t r; + + asm volatile ("add.cc.u32 %0, %1, %2;" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +__device__ __forceinline__ uint32_t addc_cc(uint32_t a, uint32_t b) { + uint32_t r; + + asm volatile ("addc.cc.u32 %0, %1, %2;" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +__device__ __forceinline__ uint32_t addc(uint32_t a, uint32_t b) { + uint32_t r; + + asm volatile ("addc.u32 %0, %1, %2;" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +__device__ __forceinline__ uint32_t sub_cc(uint32_t a, uint32_t b) { + uint32_t r; + + asm volatile ("sub.cc.u32 %0, %1, %2;" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +__device__ __forceinline__ uint32_t subc_cc(uint32_t a, uint32_t b) { + uint32_t r; + + asm volatile ("subc.cc.u32 %0, %1, %2;" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +__device__ __forceinline__ uint32_t subc(uint32_t a, uint32_t b) { + uint32_t r; + + asm volatile ("subc.u32 %0, %1, %2;" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +__device__ __forceinline__ uint32_t madlo(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("mad.lo.u32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t madlo_cc(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("mad.lo.cc.u32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t madloc_cc(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("madc.lo.cc.u32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t madloc(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("madc.lo.u32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t madhi(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("mad.hi.u32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t madhi_cc(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("mad.hi.cc.u32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t madhic_cc(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("madc.hi.cc.u32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t madhic(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("madc.hi.u32 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint64_t mad_wide(uint32_t a, uint32_t b, uint64_t c) { + uint64_t r; + + asm volatile ("mad.wide.u32 %0, %1, %2, %3;" : "=l"(r) : "r"(a), "r"(b), "l"(c)); + return r; +} + +__device__ __forceinline__ uint32_t xmadll(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u16 %al, %ah, %bl, %bh;\n\t" + "mov.b32 {%al,%ah},%1;\n\t" + "mov.b32 {%bl,%bh},%2;\n\t" + "mul.wide.u16 %0, %al, %bl;\n\t" + "add.u32 %0, %0, %3;\n\t" + "}" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t xmadll_cc(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u16 %al, %ah, %bl, %bh;\n\t" + "mov.b32 {%al,%ah},%1;\n\t" + "mov.b32 {%bl,%bh},%2;\n\t" + "mul.wide.u16 %0, %al, %bl;\n\t" + "add.cc.u32 %0, %0, %3;\n\t" + "}" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t xmadllc_cc(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u16 %al, %ah, %bl, %bh;\n\t" + "mov.b32 {%al,%ah},%1;\n\t" + "mov.b32 {%bl,%bh},%2;\n\t" + "mul.wide.u16 %0, %al, %bl;\n\t" + "addc.cc.u32 %0, %0, %3;\n\t" + "}" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t xmadllc(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u16 %al, %ah, %bl, %bh;\n\t" + "mov.b32 {%al,%ah},%1;\n\t" + "mov.b32 {%bl,%bh},%2;\n\t" + "mul.wide.u16 %0, %al, %bl;\n\t" + "addc.u32 %0, %0, %3;\n\t" + "}" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t xmadlh(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u16 %al, %ah, %bl, %bh;\n\t" + "mov.b32 {%al,%ah},%1;\n\t" + "mov.b32 {%bl,%bh},%2;\n\t" + "mul.wide.u16 %0, %al, %bh;\n\t" + "add.u32 %0, %0, %3;\n\t" + "}" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t xmadlh_cc(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u16 %al, %ah, %bl, %bh;\n\t" + "mov.b32 {%al,%ah},%1;\n\t" + "mov.b32 {%bl,%bh},%2;\n\t" + "mul.wide.u16 %0, %al, %bh;\n\t" + "add.cc.u32 %0, %0, %3;\n\t" + "}" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t xmadlhc_cc(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u16 %al, %ah, %bl, %bh;\n\t" + "mov.b32 {%al,%ah},%1;\n\t" + "mov.b32 {%bl,%bh},%2;\n\t" + "mul.wide.u16 %0, %al, %bh;\n\t" + "addc.cc.u32 %0, %0, %3;\n\t" + "}" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t xmadlhc(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u16 %al, %ah, %bl, %bh;\n\t" + "mov.b32 {%al,%ah},%1;\n\t" + "mov.b32 {%bl,%bh},%2;\n\t" + "mul.wide.u16 %0, %al, %bh;\n\t" + "addc.u32 %0, %0, %3;\n\t" + "}" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t xmadhl(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u16 %al, %ah, %bl, %bh;\n\t" + "mov.b32 {%al,%ah},%1;\n\t" + "mov.b32 {%bl,%bh},%2;\n\t" + "mul.wide.u16 %0, %ah, %bl;\n\t" + "add.u32 %0, %0, %3;\n\t" + "}" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t xmadhl_cc(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u16 %al, %ah, %bl, %bh;\n\t" + "mov.b32 {%al,%ah},%1;\n\t" + "mov.b32 {%bl,%bh},%2;\n\t" + "mul.wide.u16 %0, %ah, %bl;\n\t" + "add.cc.u32 %0, %0, %3;\n\t" + "}" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t xmadhlc_cc(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u16 %al, %ah, %bl, %bh;\n\t" + "mov.b32 {%al,%ah},%1;\n\t" + "mov.b32 {%bl,%bh},%2;\n\t" + "mul.wide.u16 %0, %ah, %bl;\n\t" + "addc.cc.u32 %0, %0, %3;\n\t" + "}" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t xmadhlc(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u16 %al, %ah, %bl, %bh;\n\t" + "mov.b32 {%al,%ah},%1;\n\t" + "mov.b32 {%bl,%bh},%2;\n\t" + "mul.wide.u16 %0, %ah, %bl;\n\t" + "addc.u32 %0, %0, %3;\n\t" + "}" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t xmadhh(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u16 %al, %ah, %bl, %bh;\n\t" + "mov.b32 {%al,%ah},%1;\n\t" + "mov.b32 {%bl,%bh},%2;\n\t" + "mul.wide.u16 %0, %ah, %bh;\n\t" + "add.u32 %0, %0, %3;\n\t" + "}" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t xmadhh_cc(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u16 %al, %ah, %bl, %bh;\n\t" + "mov.b32 {%al,%ah},%1;\n\t" + "mov.b32 {%bl,%bh},%2;\n\t" + "mul.wide.u16 %0, %ah, %bh;\n\t" + "add.cc.u32 %0, %0, %3;\n\t" + "}" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t xmadhhc_cc(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u16 %al, %ah, %bl, %bh;\n\t" + "mov.b32 {%al,%ah},%1;\n\t" + "mov.b32 {%bl,%bh},%2;\n\t" + "mul.wide.u16 %0, %ah, %bh;\n\t" + "addc.cc.u32 %0, %0, %3;\n\t" + "}" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t xmadhhc(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u16 %al, %ah, %bl, %bh;\n\t" + "mov.b32 {%al,%ah},%1;\n\t" + "mov.b32 {%bl,%bh},%2;\n\t" + "mul.wide.u16 %0, %ah, %bh;\n\t" + "addc.u32 %0, %0, %3;\n\t" + "}" : "=r"(r) : "r"(a), "r"(b), "r"(c)); + return r; +} + +__device__ __forceinline__ uint32_t umin(uint32_t a, uint32_t b) { + uint32_t r; + + asm volatile ("min.u32 %0,%1,%2;" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +__device__ __forceinline__ uint32_t umax(uint32_t a, uint32_t b) { + uint32_t r; + + asm volatile ("max.u32 %0,%1,%2;" : "=r"(r) : "r"(a), "r"(b)); + return r; +} + +__device__ __forceinline__ uint32_t uleft_clamp(uint32_t lo, uint32_t hi, uint32_t amt) { + uint32_t r; + + #if __CUDA_ARCH__>=320 + asm volatile ("shf.l.clamp.b32 %0,%1,%2,%3;" : "=r"(r) : "r"(lo), "r"(hi), "r"(amt)); + #else + amt=umin(amt, 32); + r=hi<>32-amt); + #endif + return r; +} + +__device__ __forceinline__ uint32_t uright_clamp(uint32_t lo, uint32_t hi, uint32_t amt) { + uint32_t r; + + #if __CUDA_ARCH__>=320 + asm volatile ("shf.r.clamp.b32 %0,%1,%2,%3;" : "=r"(r) : "r"(lo), "r"(hi), "r"(amt)); + #else + amt=umin(amt, 32); + r=lo>>amt; + r=r | (hi<<32-amt); + #endif + return r; +} + +__device__ __forceinline__ uint32_t uleft_wrap(uint32_t lo, uint32_t hi, uint32_t amt) { + uint32_t r; + + #if __CUDA_ARCH__>=320 + asm volatile ("shf.l.wrap.b32 %0,%1,%2,%3;" : "=r"(r) : "r"(lo), "r"(hi), "r"(amt)); + #else + amt=amt & 0x1F; + r=hi<>32-amt); + #endif + return r; +} + +__device__ __forceinline__ uint32_t uright_wrap(uint32_t lo, uint32_t hi, uint32_t amt) { + uint32_t r; + + #if __CUDA_ARCH__>=320 + asm volatile ("shf.r.wrap.b32 %0,%1,%2,%3;" : "=r"(r) : "r"(lo), "r"(hi), "r"(amt)); + #else + amt=amt & 0x1F; + r=lo>>amt; + r=r | (hi<<32-amt); + #endif + return r; +} + +__device__ __forceinline__ uint32_t uabs(int32_t x) { + uint32_t r; + + asm volatile ("abs.s32 %0,%1;" : "=r"(r) : "r"(x)); + return r; +} + +__device__ __forceinline__ uint32_t uhigh(uint64_t wide) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u32 %ignore;\n\t" + "mov.b64 {%ignore,%0},%1;\n\t" + "}" : "=r"(r) : "l"(wide)); + return r; +} + +__device__ __forceinline__ uint32_t ulow(uint64_t wide) { + uint32_t r; + + asm volatile ("{\n\t" + ".reg .u32 %ignore;\n\t" + "mov.b64 {%0,%ignore},%1;\n\t" + "}" : "=r"(r) : "l"(wide)); + return r; +} + +__device__ __forceinline__ uint64_t make_wide(uint32_t lo, uint32_t hi) { + uint64_t r; + + asm volatile ("mov.b64 %0,{%1,%2};" : "=l"(r) : "r"(lo), "r"(hi)); + return r; +} + +} /* namespace cgbn */ + + diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/chain.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/chain.cu new file mode 100644 index 0000000..a7fadc4 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/chain.cu @@ -0,0 +1,165 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +__device__ __forceinline__ chain_t::chain_t() : _position(0) { +} + +template +__device__ __forceinline__ chain_t::~chain_t() { + if(_positionlength) ASM_ERROR("chain destructor: chain too long"); +} + +template +__device__ __forceinline__ uint32_t chain_t::add(uint32_t a, uint32_t b) { + uint32_t r; + + _position++; + if(length==1 && _position==1 && !carry_in && !carry_out) + r=a+b; + else if(_position==1 && !carry_in) + r=add_cc(a, b); + else if(_position +__device__ __forceinline__ uint32_t chain_t::sub(uint32_t a, uint32_t b) { + uint32_t r; + + _position++; + if(length==1 && _position==1 && !carry_in && !carry_out) + r=a-b; + else if(_position==1 && !carry_in) + r=sub_cc(a, b); + else if(_position +__device__ __forceinline__ uint32_t chain_t::madlo(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + _position++; + if(length==1 && _position==1 && !carry_in && !carry_out) + r=madlo(a, b, c); + else if(_position==1 && !carry_in) + r=madlo_cc(a, b, c); + else if(_position +__device__ __forceinline__ uint32_t chain_t::madhi(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + _position++; + if(length==1 && _position==1 && !carry_in && !carry_out) + r=madhi(a, b, c); + else if(_position==1 && !carry_in) + r=madhi_cc(a, b, c); + else if(_position +__device__ __forceinline__ uint32_t chain_t::xmadll(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + _position++; + if(length==1 && _position==1 && !carry_in && !carry_out) + r=xmadll(a, b, c); + else if(_position==1 && !carry_in) + r=xmadll_cc(a, b, c); + else if(_position +__device__ __forceinline__ uint32_t chain_t::xmadlh(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + _position++; + if(length==1 && _position==1 && !carry_in && !carry_out) + r=xmadlh(a, b, c); + else if(_position==1 && !carry_in) + r=xmadlh_cc(a, b, c); + else if(_position +__device__ __forceinline__ uint32_t chain_t::xmadhl(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + _position++; + if(length==1 && _position==1 && !carry_in && !carry_out) + r=xmadhl(a, b, c); + else if(_position==1 && !carry_in) + r=xmadhl_cc(a, b, c); + else if(_position +__device__ __forceinline__ uint32_t chain_t::xmadhh(uint32_t a, uint32_t b, uint32_t c) { + uint32_t r; + + _position++; + if(length==1 && _position==1 && !carry_in && !carry_out) + r=xmadhh(a, b, c); + else if(_position==1 && !carry_in) + r=xmadhh_cc(a, b, c); + else if(_position +__device__ __forceinline__ bool dequals(const uint32_t sync, const uint32_t a[limbs], const uint32_t b[limbs]) { + static const uint32_t TPI_ONES=(1ull<>(group_thread ^ warp_thread); + return mask==TPI_ONES; +} + +template +__device__ __forceinline__ int32_t dcompare(const uint32_t sync, const uint32_t a[limbs], const uint32_t b[limbs]) { + static const uint32_t TPI_ONES=(1ull<=b[0]); + b_ballot=__ballot_sync(sync, a[0]<=b[0]); + } + else { + chain_t<> chain1; + #pragma unroll + for(int32_t index=0;index chain2; + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ void dmask_set(uint32_t r[limbs], const int32_t numbits) { + int32_t group_thread=threadIdx.x & tpi-1, group_base=group_thread*limbs; + int32_t bits=tpi*limbs*32; + + if(numbits>=bits || numbits<=-bits) { + #pragma unroll + for(int32_t index=0;index=0) { + int32_t limb=(numbits>>5)-group_base; + int32_t straddle=uleft_wrap(0xFFFFFFFF, 0, numbits); + + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=0xFFFFFFFF; + else + r[index]=straddle; + } + } + else { + int32_t limb=(numbits+bits>>5)-group_base; + int32_t straddle=uleft_wrap(0, 0xFFFFFFFF, numbits); + + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=0; + else + r[index]=straddle; + } + } +} + +template +__device__ __forceinline__ void dmask_and(uint32_t r[limbs], const uint32_t a[limbs], const int32_t numbits) { + int32_t group_thread=threadIdx.x & tpi-1, group_base=group_thread*limbs; + int32_t bits=tpi*limbs*32; + + if(numbits>=bits || numbits<=-bits) { + #pragma unroll + for(int32_t index=0;index=0) { + int32_t limb=(numbits>>5)-group_base; + int32_t straddle=uleft_wrap(0xFFFFFFFF, 0, numbits); + + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=a[index]; + else + r[index]=a[index] & straddle; + } + } + else { + int32_t limb=(numbits+bits>>5)-group_base; + int32_t straddle=uleft_wrap(0, 0xFFFFFFFF, numbits); + + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=0; + else + r[index]=a[index] & straddle; + } + } +} + +template +__device__ __forceinline__ void dmask_ior(uint32_t r[limbs], const uint32_t a[limbs], const int32_t numbits) { + int32_t group_thread=threadIdx.x & tpi-1, group_base=group_thread*limbs; + int32_t bits=tpi*limbs*32; + + if(numbits>=bits || numbits<=-bits) { + #pragma unroll + for(int32_t index=0;index=0) { + int32_t limb=(numbits>>5)-group_base; + int32_t straddle=uleft_wrap(0xFFFFFFFF, 0, numbits); + + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=0xFFFFFFFF; + else + r[index]=a[index] | straddle; + } + } + else { + int32_t limb=(numbits+bits>>5)-group_base; + int32_t straddle=uleft_wrap(0, 0xFFFFFFFF, numbits); + + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=a[index]; + else + r[index]=a[index] | straddle; + } + } +} + +template +__device__ __forceinline__ void dmask_xor(uint32_t r[limbs], const uint32_t a[limbs], const int32_t numbits) { + int32_t group_thread=threadIdx.x & tpi-1, group_base=group_thread*limbs; + int32_t bits=tpi*limbs*32; + + if(numbits>=bits || numbits<=-bits) { + #pragma unroll + for(int32_t index=0;index=0) { + int32_t limb=(numbits>>5)-group_base; + int32_t straddle=uleft_wrap(0xFFFFFFFF, 0, numbits); + + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=a[index] ^ 0xFFFFFFFF; + else + r[index]=a[index] ^ straddle; + } + } + else { + int32_t limb=(numbits+bits>>5)-group_base; + int32_t straddle=uleft_wrap(0, 0xFFFFFFFF, numbits); + + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=a[index]; + else + r[index]=a[index] ^ straddle; + } + } +} + +template +__device__ __forceinline__ void dmask_select(uint32_t r[limbs], const uint32_t clear[limbs], const uint32_t set[limbs], int32_t numbits) { + int32_t group_thread=threadIdx.x & tpi-1, group_base=group_thread*limbs; + int32_t bits=tpi*limbs*32; + + if(numbits>=bits || numbits<=-bits) { + #pragma unroll + for(int32_t index=0;index=0) { + int32_t limb=(numbits>>5)-group_base; + int32_t straddle=uleft_wrap(0xFFFFFFFF, 0, numbits); + + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=set[index]; + else + r[index]=(set[index] & straddle) | (clear[index] & ~straddle); + } + } + else { + int32_t limb=(bits+numbits>>5)-group_base; + int32_t straddle=uleft_wrap(0, 0xFFFFFFFF, numbits); + + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=clear[index]; + else + r[index]=(set[index] & straddle) | (clear[index] & ~straddle); + } + } +} + +template +__device__ __forceinline__ void drotate_left(const uint32_t sync, uint32_t r[limbs], const uint32_t x[limbs], const uint32_t numbits) { + uint32_t rotate_bits=numbits & 0x1F, numlimbs=numbits>>5, threads=static_divide_small(numlimbs); + + numlimbs=numlimbs-threads*limbs; + if(numlimbs==0) { + #pragma unroll + for(int32_t index=0;index(r, x, numlimbs); + #pragma unroll + for(int32_t index=0;index0) { + uint32_t fill=__shfl_sync(sync, r[limbs-1], threadIdx.x-1, tpi); + + mpleft(r, r, rotate_bits, fill); + } +} + +template +__device__ __forceinline__ void drotate_right(const uint32_t sync, uint32_t r[limbs], const uint32_t x[limbs], const uint32_t numbits) { + uint32_t rotate_bits=numbits & 0x1F, numlimbs=numbits>>5, threads=static_divide_small(numlimbs); + + numlimbs=numlimbs-threads*limbs; + if(numlimbs==0) { + #pragma unroll + for(int32_t index=0;index(r, x, numlimbs); + #pragma unroll + for(int32_t index=0;index0) { + uint32_t fill=__shfl_sync(sync, r[0], threadIdx.x+1, tpi); + + mpright(r, r, rotate_bits, fill); + } +} + +template +__device__ __forceinline__ void dscatter(const uint32_t sync, uint32_t &dest, const uint32_t source[limbs], const uint32_t source_thread=31) { + uint32_t group_thread=threadIdx.x & tpi-1; + uint32_t t; + + if(zero) + dest=0; + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ void dall_gather(const uint32_t sync, uint32_t dest[limbs], const uint32_t source) { + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ void fwgather(const uint32_t sync, uint32_t dest[limbs], const uint32_t source, const uint32_t destination_thread=31) { + uint32_t group_thread=threadIdx.x & warpSize-1; + uint32_t t; + + if(zero) { + #pragma unroll + for(int32_t index=0;indexb) ? 1 : 0; + compare=(a>acnt; + b=b>>bcnt; + gcdcnt=umin(acnt, bcnt); + + while(a!=b) { + t=__usad(a, b, 0); + b=umin(a, b); + a=t>>1; + while((a & 0x01)==0) + a=a>>1; + } + return a<=0x40000000) ? 0x8000 : 0; + #pragma unroll + for(int32_t bit=0x4000;bit>=0x1;bit=bit/2) { + t=s+bit; + r=t*t+neg; + s=(0>(int32_t)r) ? t : s; + } + return s; +} +*/ + +/* +__device__ __forceinline__ uint32_t uapprox(const uint32_t d) { + uint64_t a=d; + + if(d==0x80000000) + return 0xFFFFFFFF; + + // computes ceil(2^64/d) - 2^32 + a=-a / d; + return ((uint32_t)a)+2; +} +*/ + +/**************************************************************** + * d must be normalized (i.e., d>=0x80000000) + * computes ceil(2^64/d)-2^32 if d>0x80000000 + * ceil(2^64/d)-(2^32+1) if d=0x80000000 + ****************************************************************/ +__device__ __forceinline__ uint32_t uapprox(uint32_t d) { + float f; + uint32_t a, t0, t1; + int32_t s; + + // special case d=0x80000000 + if(d==0x80000000) + return 0xFFFFFFFF; + + // get a first estimate using float 1/x + f=__uint_as_float((d>>8) + 0x3F000000); + asm volatile("rcp.approx.f32 %0,%1;" : "=f"(f) : "f"(f)); + a=__float_as_uint(f); + a=madlo(a, 512, 0xFFFFFE00); + + // use Newton-Raphson to improve estimate + s=madhi(d, a, d); + t0=abs(s); + t0=madhi(t0, a, t0); + a=(s>=0) ? a-t0 : a+t0; + + // two corrections steps give exact result + a=a-madhi(d, a, d); // first correction + + t0=madlo_cc(d, a, 0); // second correction + t1=madhic(d, a, d); + t1=(t1!=0) ? t1 : (t0>=d); + a=a-t1; + return a; +} + +__device__ __forceinline__ uint32_t udiv(const uint32_t lo, const uint32_t hi, const uint32_t d, const uint32_t approx) { + uint32_t q, add, ylo, yhi; + + // q=MIN(0xFFFFFFFF, HI(approx * hi) + hi + ((lo>31; + + // first correction + y0=add_cc(x0, d0); + y1=addc_cc(x1, d1); + y2=addc_cc(x2, 0); + add=addc(add, 0); + q=q-add; + + // second correction + y0=add_cc(y0, d0); + y1=addc_cc(y1, d1); + y2=addc_cc(y2, 0); + add=addc(add, 0); + q=q-add; + + // third correction + y0=add_cc(y0, d0); + y1=addc_cc(y1, d1); + y2=addc_cc(y2, 0); + add=addc(add, 0); + q=q-add; + + // fourth correction + y0=add_cc(y0, d0); + y1=addc_cc(y1, d1); + y2=addc_cc(y2, 0); + add=addc(add, 0); + q=q-add; + + return q; +} + +/**************************************************************** + * x must be normalized, i.e., x>=0x40000000 + * computes sqrt(x) + ****************************************************************/ +__device__ __forceinline__ uint32_t usqrt(const uint32_t x) { + float f; + uint32_t a; + + if((x & 0x80000000)!=0) + f=__uint_as_float((x>>8) + 0x4e800000); + else + f=__uint_as_float((x>>7) + 0x4e000000); + + asm volatile("sqrt.approx.f32 %0,%1;" : "=f"(f) : "f"(f)); + + // round the approximation up + a=__float_as_uint(f)-0x467FFF80>>8; + + // single correction step + if(0>(int32_t)(x-a*a)) + a--; + return a; +} + +/**************************************************************** + * hi must be normalized, i.e., hi>=0x40000000 + * computes sqrt(x) + ****************************************************************/ +__device__ __forceinline__ uint32_t usqrt(const uint32_t lo, const uint32_t hi) { + uint32_t s=usqrt(hi), shifted=s<<16; + uint32_t rhi, rlo; + + // compute shifted*shifted - 2^16 + rlo=madlo_cc(shifted, shifted, 0xFFFF0000); + rhi=madhic(shifted, shifted, 0xFFFFFFFF); + rlo=sub_cc(lo, rlo); + rhi=subc(hi, rhi); + + // if rhi>=0x1FFFD, we will overflow + if(rhi>=0x1FFFD) + s=shifted+0xFFFF; + else { + rhi=uleft_wrap(rlo, rhi, 15); + s=shifted+rhi/s; + } + + // first correction + rlo=madlo(s, s, 0); + rhi=madhi(s, s, 0); + rlo=sub_cc(lo, rlo); + rhi=subc(hi, rhi); + s+=(0>(int32_t)rhi) ? 0xFFFFFFFF : 0; + + // second correction + rlo=madlo(s, s, 0); + rhi=madhi(s, s, 0); + rlo=sub_cc(lo, rlo); + rhi=subc(hi, rhi); + s+=(0>(int32_t)rhi) ? 0xFFFFFFFF : 0; + + return s; +} + +__device__ __forceinline__ uint32_t usqrt_div(const uint32_t lo, const uint32_t hi, const uint32_t d, const uint32_t approx) { + uint32_t x0, x1; + + x0=add_cc(lo, 1); + x1=addc(hi, 0); + + if(x1>=2) + return 0xFFFFFFFF; + + x1=uleft_wrap(x0, x1, 31); + x0=x0<<31; + return udiv(x0, x1, d, approx); +} + +} /* namespace cgbn */ \ No newline at end of file diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/mp.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/mp.cu new file mode 100644 index 0000000..3a52b5c --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/mp.cu @@ -0,0 +1,297 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +__device__ __forceinline__ void mpzero(uint32_t r[limbs]) { + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ void mpset(uint32_t r[limbs], const uint32_t a[limbs]) { + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ void mpswap(uint32_t x[limbs], uint32_t y[limbs]) { + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ uint32_t mplor(const uint32_t a[limbs]) { + uint32_t r=a[0]; + + #pragma unroll + for(int32_t index=1;index +__device__ __forceinline__ uint32_t mpland(const uint32_t a[limbs]) { + uint32_t r=a[0]; + + #pragma unroll + for(int32_t index=1;index +__device__ __forceinline__ bool mpzeros(const uint32_t a[limbs]) { + return mplor(a)==0; +} + +template +__device__ __forceinline__ bool mpones(const uint32_t a[limbs]) { + return mpland(a)==0xFFFFFFFF; +} + +template +__device__ __forceinline__ void mpadd32_cc(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b) { + chain_t chain; + r[0]=chain.add(a[0], b); + #pragma unroll + for(int32_t index=1;index +__device__ __forceinline__ uint32_t mpadd32(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b) { + mpadd32_cc(r, a, b); + return addc(0, 0); +} + + +template +__device__ __forceinline__ void mpadd_cc(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b[limbs]) { + chain_t chain; + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ uint32_t mpadd(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b[limbs]) { + mpadd_cc(r, a, b); + return addc(0, 0); +} + +template +__device__ __forceinline__ void mpsub32_cc(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b) { + chain_t chain; + r[0]=chain.sub(a[0], b); + #pragma unroll + for(int32_t index=1;index +__device__ __forceinline__ uint32_t mpsub32(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b) { + mpsub32_cc(r, a, b); + return subc(0, 0); +} + +template +__device__ __forceinline__ void mpsub_cc(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b[limbs]) { + chain_t chain; + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ uint32_t mpsub(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b[limbs]) { + mpsub_cc(r, a, b); + return subc(0, 0); +} + +template +__device__ __forceinline__ uint32_t mpmul32(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b) { + uint32_t carry=0; + + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ uint32_t mprem32(const uint32_t a[limbs], const uint32_t d, const uint32_t approx) { + uint32_t c, x0, x1, x2; + + // fast and avoids division + if(limbs==1) + return (a[0]>=d) ? a[0]-d : a[0]; + + // set c to 2^64 % d + c=-d*approx+d; + + x1=a[limbs-2]; + x2=a[limbs-1]; + #pragma unroll + for(int32_t index=limbs-1;index>=2;index--) { + x0=madlo_cc(x2, c, a[index-2]); + x1=madhic_cc(x2, c, x1); + x2=addc(0, 0); + + x0=madlo_cc(x2, c, x0); + x1=madhic(x2, c, x1); + + x2=x1; + x1=x0; + } + x2=(x2>=d) ? x2-d : x2; + return urem(x1, x2, d, approx); +} + +template +__device__ __forceinline__ void mpleft(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t numbits, const uint32_t fill=0) { + #pragma unroll + for(int32_t index=limbs-1;index>=1;index--) + r[index]=uleft_clamp(a[index-1], a[index], numbits); + r[0]=uleft_clamp(fill, a[0], numbits); +} + +template +__device__ __forceinline__ void mpright(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t numbits, const uint32_t fill=0) { + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ uint32_t mpclz(const uint32_t a[limbs]) { + uint32_t word=0, count=0; + + #pragma unroll + for(int32_t index=limbs-1;index>=0;index--) { + word=(word!=0) ? word : a[index]; + count=(word!=0) ? count : (limbs-index)*32; + } + if(word!=0) + count=count+uclz(word); + return count; +} + +template +__device__ __forceinline__ uint32_t mpctz(const uint32_t a[limbs]) { + uint32_t word=0, count=0; + + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ void mpmul(uint32_t lo[limbs], uint32_t hi[limbs], const uint32_t a[limbs], const uint32_t b[limbs]) { + uint32_t c; + + #pragma unroll + for(int32_t index=0;index chain1; + #pragma unroll + for(int32_t j=0;j chain2; + #pragma unroll + for(int32_t j=0;j +__device__ __forceinline__ void mprotate_left(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t numlimbs) { + #pragma unroll + for(int32_t index=0;indexbit_set::high_bit*2) + shifter_t::high_bit, true>::mprotate_left(r, numlimbs); + else if((limbs-1&limbs)==0) + shifter_t::mprotate_left(r, numlimbs); + else + shifter_t::high_bit, false>::mprotate_left(r, numlimbs); +} + +template +__device__ __forceinline__ void mprotate_right(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t numlimbs) { + #pragma unroll + for(int32_t index=0;indexbit_set::high_bit*2) + shifter_t::high_bit, true>::mprotate_right(r, numlimbs); + else if((limbs-1&limbs)==0) + shifter_t::mprotate_right(r, numlimbs); + else + shifter_t::high_bit, false>::mprotate_right(r, numlimbs); +} + +} /* cgbn namespace */ \ No newline at end of file diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/shifter.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/shifter.cu new file mode 100644 index 0000000..aa62699 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/shifter.cu @@ -0,0 +1,180 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +class bit_set { + public: + static const uint32_t high_bit=bit_set::high_bit<<1; + static const uint32_t low_bit=(value-1 & ~value)+1; +}; + +template<> +class bit_set<1> { + public: + static const uint32_t high_bit=1; + static const uint32_t low_bit=1; +}; + +/* mp rotate */ +template +class shifter_t { + public: + __device__ __forceinline__ static void mprotate_left(uint32_t r[limbs], uint32_t numlimbs); + __device__ __forceinline__ static void mprotate_right(uint32_t r[limbs], uint32_t numlimbs); +}; + +template +class shifter_t { + public: + __device__ __forceinline__ static void mprotate_left(uint32_t r[limbs], uint32_t numlimbs); + __device__ __forceinline__ static void mprotate_right(uint32_t r[limbs], uint32_t numlimbs); +}; + + +/* rotate right/left primitives */ + +template +__device__ __forceinline__ static void mprotate_left_static(uint32_t r[limbs]) { + int32_t from, to, offset, pow2gcd=min(bit_set::low_bit, bit); + uint32_t swap; + + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ static void mprotate_left_bitcheck(uint32_t r[limbs], const uint32_t numlimbs) { + int32_t from, to, offset, pow2gcd=min(bit_set::low_bit, bit); + uint32_t swap; + + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ static void mprotate_right_static(uint32_t r[limbs]) { + int32_t from, to, offset, pow2gcd=min(bit_set::low_bit, bit); + uint32_t swap; + + #pragma unroll + for(int32_t index=limbs-1;index>=0;index--) { + offset=pow2gcd*index/limbs; + to=(bit*index+offset)%limbs; + from=(to+limbs-bit)%limbs; + + if((index+1)*bit%limbs==0) + swap=r[to]; + if(index*bit%limbs!=0) + r[to]=r[from]; + else + r[to]=swap; + } +} + +template +__device__ __forceinline__ static void mprotate_right_bitcheck(uint32_t r[limbs], const uint32_t numlimbs) { + int32_t from, to, offset, pow2gcd=min(bit_set::low_bit, bit); + uint32_t swap; + + #pragma unroll + for(int32_t index=limbs-1;index>=0;index--) { + offset=pow2gcd*index/limbs; + to=(bit*index+offset)%limbs; + from=(to+limbs-bit)%limbs; + + if((index+1)*bit%limbs==0) + swap=r[to]; + if(index*bit%limbs!=0) + r[to]=((bit & numlimbs)!=0) ? r[from] : r[to]; + else + r[to]=((bit & numlimbs)!=0) ? swap : r[to]; + } +} + + +/* shifter implementation */ + +template +__device__ __forceinline__ void shifter_t::mprotate_left(uint32_t r[limbs], const uint32_t numlimbs) { + if(loop) { + uint32_t count=numlimbs; + + while(count>=bit) { + mprotate_right_static(r); + count-=bit; + } + } + else + mprotate_right_bitcheck(r, numlimbs); + shifter_t::mprotate_left(r, numlimbs); +} + +template +__device__ __forceinline__ void shifter_t::mprotate_left(uint32_t r[limbs], const uint32_t numlimbs) { +} + +template +__device__ __forceinline__ void shifter_t::mprotate_right(uint32_t r[limbs], const uint32_t numlimbs) { + if(loop) { + uint32_t count=numlimbs; + + while(count>=bit) { + mprotate_left_static(r); + count-=bit; + } + } + else + mprotate_left_bitcheck(r, numlimbs); + shifter_t::mprotate_right(r, numlimbs); +} + +template +__device__ __forceinline__ void shifter_t::mprotate_right(uint32_t r[limbs], const uint32_t numlimbs) { +} + +} /* namespace cgbn */ \ No newline at end of file diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/static_divide.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/static_divide.cu new file mode 100644 index 0000000..f7c0937 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/arith/static_divide.cu @@ -0,0 +1,42 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +__device__ __forceinline__ uint32_t static_divide_small(uint32_t numerator) { + uint32_t est=0xFFFFFFFF/denominator; + + // not exact, but ok for den<2^10 and num<2^20 + return __umulhi((uint32_t)est, numerator+1); +} + +template +__device__ __forceinline__ uint32_t static_remainder_small(uint32_t numerator) { + + // not exact, but ok for den<1024 and num<2^20 + return numerator-static_divide_small(numerator)*denominator; +} + +} \ No newline at end of file diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn.cu new file mode 100644 index 0000000..80f0b3e --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn.cu @@ -0,0 +1,91 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +cudaError_t cgbn_error_report_alloc(cgbn_error_report_t **report) { + cudaError_t status; + + status=cudaMallocManaged((void **)report, sizeof(cgbn_error_report_t)); + if(status!=0) + return status; + (*report)->_error=cgbn_no_error; + (*report)->_instance=0xFFFFFFFFu; + (*report)->_threadIdx.x=0xFFFFFFFFu; + (*report)->_threadIdx.y=0xFFFFFFFFu; + (*report)->_threadIdx.z=0xFFFFFFFFu; + (*report)->_blockIdx.x=0xFFFFFFFFu; + (*report)->_blockIdx.y=0xFFFFFFFFu; + (*report)->_blockIdx.z=0xFFFFFFFFu; + return status; +} + +cudaError_t cgbn_error_report_free(cgbn_error_report_t *report) { + return cudaFree(report); +} + +bool cgbn_error_report_check(cgbn_error_report_t *report) { + return report->_error!=cgbn_no_error; +} + +void cgbn_error_report_reset(cgbn_error_report_t *report) { + report->_error=cgbn_no_error; + report->_instance=0xFFFFFFFFu; + report->_threadIdx.x=0xFFFFFFFFu; + report->_threadIdx.y=0xFFFFFFFFu; + report->_threadIdx.z=0xFFFFFFFFu; + report->_blockIdx.x=0xFFFFFFFFu; + report->_blockIdx.y=0xFFFFFFFFu; + report->_blockIdx.z=0xFFFFFFFFu; +} + +const char *cgbn_error_string(cgbn_error_report_t *report) { + if(report->_error==cgbn_no_error) + return NULL; + switch(report->_error) { + case cgbn_unsupported_threads_per_instance: + return "unsupported threads per instance"; + case cgbn_unsupported_size: + return "unsupported size"; + case cgbn_unsupported_limbs_per_thread: + return "unsupported limbs per thread"; + case cgbn_unsupported_operation: + return "unsupported operation"; + case cgbn_threads_per_block_mismatch: + return "TPB does not match blockDim.x"; + case cgbn_threads_per_instance_mismatch: + return "TPI does not match env_t::TPI"; + case cgbn_division_by_zero_error: + return "division by zero"; + case cgbn_division_overflow_error: + return "division overflow"; + case cgbn_invalid_montgomery_modulus_error: + return "invalid montgomery modulus"; + case cgbn_modulus_not_odd_error: + return "invalid modulus (it must be odd)"; + case cgbn_inverse_does_not_exist_error: + return "inverse does not exist"; + case cgbn_no_error: + return NULL; + } + return NULL; +} diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn.h b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn.h new file mode 100644 index 0000000..5e103f6 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn.h @@ -0,0 +1,512 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +#include +#include +#include + +/* basic types */ +typedef enum { + cgbn_no_error=0, + cgbn_unsupported_threads_per_instance=1, + cgbn_unsupported_size=2, + cgbn_unsupported_limbs_per_thread=3, + cgbn_unsupported_operation=4, + cgbn_threads_per_block_mismatch=5, + cgbn_threads_per_instance_mismatch=6, + cgbn_division_by_zero_error=7, + cgbn_division_overflow_error=8, + cgbn_invalid_montgomery_modulus_error=9, + cgbn_modulus_not_odd_error=10, + cgbn_inverse_does_not_exist_error=11, +} cgbn_error_t; + +typedef struct { + volatile cgbn_error_t _error; + uint32_t _instance; + dim3 _threadIdx; + dim3 _blockIdx; +} cgbn_error_report_t; + +typedef enum { + cgbn_no_checks, /* disable error checking - improves performance */ + cgbn_report_monitor, /* writes errors to the reporter object, no other actions */ + cgbn_print_monitor, /* writes errors to the reporter and prints the error to stdout */ + cgbn_halt_monitor, /* writes errors to the reporter and halts */ +} cgbn_monitor_t; + +cudaError_t cgbn_error_report_alloc(cgbn_error_report_t **report); +cudaError_t cgbn_error_report_free(cgbn_error_report_t *report); +bool cgbn_error_report_check(cgbn_error_report_t *report); +void cgbn_error_report_reset(cgbn_error_report_t *report); +const char *cgbn_error_string(cgbn_error_report_t *report); + +#include "cgbn.cu" + +#if defined(__CUDA_ARCH__) + #if !defined(XMP_IMAD) && !defined(XMP_XMAD) && !defined(XMP_WMAD) + #if __CUDA_ARCH__<500 + #define XMP_IMAD + #elif __CUDA_ARCH__<700 + #define XMP_XMAD + #else + #define XMP_WMAD + #endif + #endif + #include "cgbn_cuda.h" +#elif defined(__GMP_H__) + #include "cgbn_mpz.h" +#else + #include "cgbn_cpu.h" +#endif + + +template +__host__ __device__ __forceinline__ void cgbn_set(env_t env, typename env_t::cgbn_t &r, const source_cgbn_t &a) { + env.set(r, a); +} + +template +__host__ __device__ __forceinline__ void cgbn_swap(env_t env, typename env_t::cgbn_t &r, typename env_t::cgbn_t &a) { + env.swap(r, a); +} + +template +__host__ __device__ __forceinline__ int32_t cgbn_add(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const typename env_t::cgbn_t &b) { + return env.add(r, a, b); +} + +template +__host__ __device__ __forceinline__ int32_t cgbn_sub(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const typename env_t::cgbn_t &b) { + return env.sub(r, a, b); +} + +template +__host__ __device__ __forceinline__ int32_t cgbn_negate(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a) { + return env.negate(r, a); +} + +template +__host__ __device__ __forceinline__ void cgbn_mul(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const typename env_t::cgbn_t &b) { + env.mul(r, a, b); +} + +template +__host__ __device__ __forceinline__ void cgbn_mul_high(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const typename env_t::cgbn_t &b) { + env.mul_high(r, a, b); +} + +template +__host__ __device__ __forceinline__ void cgbn_sqr(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a) { + env.sqr(r, a); +} + +template +__host__ __device__ __forceinline__ void cgbn_sqr_high(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a) { + env.sqr_high(r, a); +} + +template +__host__ __device__ __forceinline__ void cgbn_div(env_t env, typename env_t::cgbn_t &q, const typename env_t::cgbn_t &num, const typename env_t::cgbn_t &denom) { + env.div(q, num, denom); +} + +template +__host__ __device__ __forceinline__ void cgbn_rem(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &num, const typename env_t::cgbn_t &denom) { + env.rem(r, num, denom); +} + +template +__host__ __device__ __forceinline__ void cgbn_div_rem(env_t env, typename env_t::cgbn_t &q, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &num, const typename env_t::cgbn_t &denom) { + env.div_rem(q, r, num, denom); +} + +template +__host__ __device__ __forceinline__ void cgbn_sqrt(env_t env, typename env_t::cgbn_t &s, const typename env_t::cgbn_t &a) { + env.sqrt(s, a); +} + +template +__host__ __device__ __forceinline__ void cgbn_sqrt_rem(env_t env, typename env_t::cgbn_t &s, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a) { + env.sqrt_rem(s, r, a); +} + +template +__host__ __device__ __forceinline__ bool cgbn_equals(env_t env, const typename env_t::cgbn_t &a, const typename env_t::cgbn_t &b) { + return env.equals(a, b); +} + +template +__host__ __device__ __forceinline__ int32_t cgbn_compare(env_t env, const typename env_t::cgbn_t &a, const typename env_t::cgbn_t &b) { + return env.compare(a, b); +} + +template +__host__ __device__ __forceinline__ void cgbn_extract_bits(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const uint32_t start, const uint32_t len) { + env.extract_bits(r, a, start, len); +} + +template +__host__ __device__ __forceinline__ void cgbn_insert_bits(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const uint32_t start, const uint32_t len, const typename env_t::cgbn_t &value) { + env.insert_bits(r, a, start, len, value); +} + + +/* ui32 arithmetic routines*/ +template +__host__ __device__ __forceinline__ uint32_t cgbn_get_ui32(env_t env, const typename env_t::cgbn_t &a) { + return env.get_ui32(a); +} + +template +__host__ __device__ __forceinline__ void cgbn_set_ui32(env_t env, typename env_t::cgbn_t &r, const uint32_t value) { + env.set_ui32(r, value); +} + +template +__host__ __device__ __forceinline__ int32_t cgbn_add_ui32(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const uint32_t add) { + return env.add_ui32(r, a, add); +} + +template +__host__ __device__ __forceinline__ int32_t cgbn_sub_ui32(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const uint32_t sub) { + return env.sub_ui32(r, a, sub); +} + +template +__host__ __device__ __forceinline__ uint32_t cgbn_mul_ui32(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const uint32_t mul) { + return env.mul_ui32(r, a, mul); +} + +template +__host__ __device__ __forceinline__ uint32_t cgbn_div_ui32(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const uint32_t div) { + return env.div_ui32(r, a, div); +} + +template +__host__ __device__ __forceinline__ uint32_t cgbn_rem_ui32(env_t env, const typename env_t::cgbn_t &a, const uint32_t div) { + return env.rem_ui32(a, div); +} + +template +__host__ __device__ __forceinline__ bool cgbn_equals_ui32(env_t env, const typename env_t::cgbn_t &a, const uint32_t value) { + return env.equals_ui32(a, value); +} + +template +__host__ __device__ __forceinline__ int32_t cgbn_compare_ui32(env_t env, const typename env_t::cgbn_t &a, const uint32_t value) { + return env.compare_ui32(a, value); +} + +template +__host__ __device__ __forceinline__ uint32_t cgbn_extract_bits_ui32(env_t env, const typename env_t::cgbn_t &a, const uint32_t start, const uint32_t len) { + return env.extract_bits_ui32(a, start, len); +} + +template +__host__ __device__ __forceinline__ void cgbn_insert_bits_ui32(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const uint32_t start, const uint32_t len, const uint32_t value) { + env.insert_bits_ui32(r, a, start, len, value); +} + +template +__host__ __device__ __forceinline__ uint32_t cgbn_binary_inverse_ui32(env_t env, const uint32_t n0) { + return env.binary_inverse_ui32(n0); +} + +template +__host__ __device__ __forceinline__ uint32_t cgbn_gcd_ui32(env_t env, const typename env_t::cgbn_t &a, const uint32_t value) { + return env.gcd_ui32(a, value); +} + + +/* wide arithmetic routines */ +template +__host__ __device__ __forceinline__ void cgbn_mul_wide(env_t env, typename env_t::cgbn_wide_t &r, const typename env_t::cgbn_t &a, const typename env_t::cgbn_t &b) { + env.mul_wide(r, a, b); +} + +template +__host__ __device__ __forceinline__ void cgbn_sqr_wide(env_t env, typename env_t::cgbn_wide_t &r, const typename env_t::cgbn_t &a) { + env.sqr_wide(r, a); +} + +template +__host__ __device__ __forceinline__ void cgbn_div_wide(env_t env, typename env_t::cgbn_t &q, const typename env_t::cgbn_wide_t &num, const typename env_t::cgbn_t &denom) { + env.div_wide(q, num, denom); +} + +template +__host__ __device__ __forceinline__ void cgbn_rem_wide(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_wide_t &num, const typename env_t::cgbn_t &denom) { + env.rem_wide(r, num, denom); +} + +template +__host__ __device__ __forceinline__ void cgbn_div_rem_wide(env_t env, typename env_t::cgbn_t &q, typename env_t::cgbn_t &r, const typename env_t::cgbn_wide_t &num, const typename env_t::cgbn_t &denom) { + env.div_rem_wide(q, r, num, denom); +} + +template +__host__ __device__ __forceinline__ void cgbn_sqrt_wide(env_t env, typename env_t::cgbn_t &s, const typename env_t::cgbn_wide_t &a) { + env.sqrt_wide(s, a); +} + +template +__host__ __device__ __forceinline__ void cgbn_sqrt_rem_wide(env_t env, typename env_t::cgbn_t &s, typename env_t::cgbn_wide_t &r, const typename env_t::cgbn_wide_t &a) { + env.sqrt_rem_wide(s, r, a); +} + + +/* logical, shifting, masking */ +template +__host__ __device__ __forceinline__ void cgbn_bitwise_and(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const typename env_t::cgbn_t &b) { + env.bitwise_and(r, a, b); +} + +template +__host__ __device__ __forceinline__ void cgbn_bitwise_ior(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const typename env_t::cgbn_t &b) { + env.bitwise_ior(r, a, b); +} + +template +__host__ __device__ __forceinline__ void cgbn_bitwise_xor(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const typename env_t::cgbn_t &b) { + env.bitwise_xor(r, a, b); +} + +template +__host__ __device__ __forceinline__ void cgbn_bitwise_complement(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a) { + env.bitwise_complement(r, a); +} + +template +__host__ __device__ __forceinline__ void cgbn_bitwise_select(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &clear, const typename env_t::cgbn_t &set, const typename env_t::cgbn_t &select) { + env.bitwise_select(r, clear, set, select); +} + +template +__host__ __device__ __forceinline__ void cgbn_bitwise_mask_copy(env_t env, typename env_t::cgbn_t &r, const int32_t numbits) { + env.bitwise_mask_copy(r, numbits); +} + +template +__host__ __device__ __forceinline__ void cgbn_bitwise_mask_and(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const int32_t numbits) { + env.bitwise_mask_and(r, a, numbits); +} + +template +__host__ __device__ __forceinline__ void cgbn_bitwise_mask_ior(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const int32_t numbits) { + env.bitwise_mask_ior(r, a, numbits); +} + +template +__host__ __device__ __forceinline__ void cgbn_bitwise_mask_xor(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const int32_t numbits) { + env.bitwise_mask_xor(r, a, numbits); +} + +template +__host__ __device__ __forceinline__ void cgbn_bitwise_mask_select(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &clear, const typename env_t::cgbn_t &set, int32_t numbits) { + env.bitwise_mask_select(r, clear, set, numbits); +} + +template +__host__ __device__ __forceinline__ void cgbn_shift_left(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const uint32_t numbits) { + env.shift_left(r, a, numbits); +} + +template +__host__ __device__ __forceinline__ void cgbn_shift_right(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const uint32_t numbits) { + env.shift_right(r, a, numbits); +} + +template +__host__ __device__ __forceinline__ void cgbn_rotate_left(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const uint32_t numbits) { + env.rotate_left(r, a, numbits); +} + +template +__host__ __device__ __forceinline__ void cgbn_rotate_right(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const uint32_t numbits) { + env.rotate_right(r, a, numbits); +} + + +/* bit counting */ +template +__host__ __device__ __forceinline__ uint32_t cgbn_pop_count(env_t env, const typename env_t::cgbn_t &a) { + return env.pop_count(a); +} + +template +__host__ __device__ __forceinline__ uint32_t cgbn_clz(env_t env, const typename env_t::cgbn_t &a) { + return env.clz(a); +} + +template +__host__ __device__ __forceinline__ uint32_t cgbn_ctz(env_t env, const typename env_t::cgbn_t &a) { + return env.ctz(a); +} + + +/* accumulator APIs */ +template +__host__ __device__ __forceinline__ int32_t cgbn_resolve(env_t env, typename env_t::cgbn_t &sum, const typename env_t::cgbn_accumulator_t &accumulator) { + return env.resolve(sum, accumulator); +} + +template +__host__ __device__ __forceinline__ void cgbn_set(env_t env, typename env_t::cgbn_accumulator_t &accumulator, const typename env_t::cgbn_t &value) { + env.set(accumulator, value); +} + +template +__host__ __device__ __forceinline__ void cgbn_add(env_t env, typename env_t::cgbn_accumulator_t &accumulator, const typename env_t::cgbn_t &value) { + env.add(accumulator, value); +} + +template +__host__ __device__ __forceinline__ void cgbn_sub(env_t env, typename env_t::cgbn_accumulator_t &accumulator, const typename env_t::cgbn_t &value) { + env.sub(accumulator, value); +} + +template +__host__ __device__ __forceinline__ void cgbn_set_ui32(env_t env, typename env_t::cgbn_accumulator_t &accumulator, const uint32_t value) { + env.set_ui32(accumulator, value); +} + +template +__host__ __device__ __forceinline__ void cgbn_add_ui32(env_t env, typename env_t::cgbn_accumulator_t &accumulator, const uint32_t value) { + env.add_ui32(accumulator, value); +} + +template +__host__ __device__ __forceinline__ void cgbn_sub_ui32(env_t env, typename env_t::cgbn_accumulator_t &accumulator, const uint32_t value) { + env.sub_ui32(accumulator, value); +} + + +/* math */ +template +__host__ __device__ __forceinline__ void cgbn_binary_inverse(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &x) { + env.binary_inverse(r, x); +} + +template +__host__ __device__ __forceinline__ void cgbn_gcd(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const typename env_t::cgbn_t &b) { + env.gcd(r, a, b); +} + +template +__host__ __device__ __forceinline__ bool cgbn_modular_inverse(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &x, const typename env_t::cgbn_t &modulus) { + return env.modular_inverse(r, x, modulus); +} + +template +__host__ __device__ __forceinline__ void cgbn_modular_power(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &x, const typename env_t::cgbn_t &exponent, const typename env_t::cgbn_t &modulus) { + env.modular_power(r, x, exponent, modulus); +} + + +/* fast division: common divisor / modulus */ +template +__host__ __device__ __forceinline__ uint32_t cgbn_bn2mont(env_t env, typename env_t::cgbn_t &mont, const typename env_t::cgbn_t &bn, const typename env_t::cgbn_t &n) { + return env.bn2mont(mont, bn, n); +} + +template +__host__ __device__ __forceinline__ void cgbn_mont2bn(env_t env, typename env_t::cgbn_t &bn, const typename env_t::cgbn_t &mont, const typename env_t::cgbn_t &n, const uint32_t np0) { + env.mont2bn(bn, mont, n, np0); +} + +template +__host__ __device__ __forceinline__ void cgbn_mont_mul(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const typename env_t::cgbn_t &b, const typename env_t::cgbn_t &n, const uint32_t np0) { + env.mont_mul(r, a, b, n, np0); +} + +template +__host__ __device__ __forceinline__ void cgbn_mont_sqr(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &a, const typename env_t::cgbn_t &n, const uint32_t np0) { + env.mont_sqr(r, a, n, np0); +} + +template +__host__ __device__ __forceinline__ void cgbn_mont_reduce_wide(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_wide_t &a, const typename env_t::cgbn_t &n, const uint32_t np0) { + env.mont_reduce_wide(r, a, n, np0); +} + + +template +__host__ __device__ __forceinline__ uint32_t cgbn_barrett_approximation(env_t env, typename env_t::cgbn_t &approx, const typename env_t::cgbn_t &denom) { + return env.barrett_approximation(approx, denom); +} + +template +__host__ __device__ __forceinline__ void cgbn_barrett_div(env_t env, typename env_t::cgbn_t &q, const typename env_t::cgbn_t &num, const typename env_t::cgbn_t &denom, const typename env_t::cgbn_t &approx, const uint32_t denom_clz) { + env.barrett_div(q, num, denom, approx, denom_clz); +} + +template +__host__ __device__ __forceinline__ void cgbn_barrett_rem(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &num, const typename env_t::cgbn_t &denom, const typename env_t::cgbn_t &approx, const uint32_t denom_clz) { + env.barrett_rem(r, num, denom, approx, denom_clz); +} + +template +__host__ __device__ __forceinline__ void cgbn_barrett_div_rem(env_t env, typename env_t::cgbn_t &q, typename env_t::cgbn_t &r, const typename env_t::cgbn_t &num, const typename env_t::cgbn_t &denom, const typename env_t::cgbn_t &approx, const uint32_t denom_clz) { + env.barrett_div_rem(q, r, num, denom, approx, denom_clz); +} + +template +__host__ __device__ __forceinline__ void cgbn_barrett_div_wide(env_t env, typename env_t::cgbn_t &q, const typename env_t::cgbn_wide_t &num, const typename env_t::cgbn_t &denom, const typename env_t::cgbn_t &approx, const uint32_t denom_clz) { + env.barrett_div_wide(q, num, denom, approx, denom_clz); +} + +template +__host__ __device__ __forceinline__ void cgbn_barrett_rem_wide(env_t env, typename env_t::cgbn_t &r, const typename env_t::cgbn_wide_t &num, const typename env_t::cgbn_t &denom, const typename env_t::cgbn_t &approx, const uint32_t denom_clz) { + env.barrett_rem_wide(r, num, denom, approx, denom_clz); +} + +template +__host__ __device__ __forceinline__ void cgbn_barrett_div_rem_wide(env_t env, typename env_t::cgbn_t &q, typename env_t::cgbn_t &r, const typename env_t::cgbn_wide_t &num, const typename env_t::cgbn_t &denom, const typename env_t::cgbn_t &approx, const uint32_t denom_clz) { + env.barrett_div_rem_wide(q, r, num, denom, approx, denom_clz); +} + + +/* load/store to global or shared memory */ +template +__host__ __device__ __forceinline__ void cgbn_load(env_t env, typename env_t::cgbn_t &r, cgbn_mem_t *const address) { + env.load(r, address); +} + +template +__host__ __device__ __forceinline__ void cgbn_store(env_t env, cgbn_mem_t *address, const typename env_t::cgbn_t &a) { + env.store(address, a); +} + + +/* load/store to local memory */ +template +__host__ __device__ __forceinline__ void cgbn_load(env_t env, typename env_t::cgbn_t &r, typename env_t::cgbn_local_t *const address) { + env.load(r, address); +} + +template +__host__ __device__ __forceinline__ void cgbn_store(env_t env, typename env_t::cgbn_local_t *address, const typename env_t::cgbn_t &a) { + env.store(address, a); +} diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn_cpu.h b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn_cpu.h new file mode 100644 index 0000000..1a103de --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn_cpu.h @@ -0,0 +1,25 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +#error You must use GMP for now diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn_cuda.h b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn_cuda.h new file mode 100644 index 0000000..e01a881 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn_cuda.h @@ -0,0 +1,258 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +typedef enum { + cgbn_instance_syncable, + cgbn_warp_syncable, + cgbn_block_syncable, + cgbn_grid_syncable +} cgbn_syncable_t; + +class cgbn_default_parameters_t { + public: + + static const uint32_t TPB=0; + static const uint32_t MAX_ROTATION=4; + static const uint32_t SHM_LIMIT=0; + static const bool CONSTANT_TIME=false; +}; + +/* forward declarations */ +template +class cgbn_context_t; + +template +class cgbn_env_t; + +template +struct cgbn_mem_t { + public: + uint32_t _limbs[(bits+31)/32]; +}; + +/* main classes */ +template +class cgbn_context_t { + public: + static const uint32_t TPB=params::TPB; + static const uint32_t TPI=tpi; + static const uint32_t MAX_ROTATION=params::MAX_ROTATION; + static const uint32_t SHM_LIMIT=params::SHM_LIMIT; + static const bool CONSTANT_TIME=params::CONSTANT_TIME; + + const cgbn_monitor_t _monitor; + cgbn_error_report_t *const _report; + const int32_t _instance; + uint32_t *_scratch; + + public: + __device__ __forceinline__ cgbn_context_t(); + __device__ __forceinline__ cgbn_context_t(cgbn_monitor_t type); + __device__ __forceinline__ cgbn_context_t(cgbn_monitor_t type, cgbn_error_report_t *report); + __device__ __forceinline__ cgbn_context_t(cgbn_monitor_t type, cgbn_error_report_t *report, uint32_t instance); + + __device__ __forceinline__ uint32_t *scratch() const; + __device__ __forceinline__ bool check_errors() const; + __device__ __noinline__ void report_error(cgbn_error_t error) const; + + template + __device__ __forceinline__ env_t env() { + env_t env(*this); + + return env; + } + + template + __device__ __forceinline__ cgbn_env_t env() { + cgbn_env_t env(*this); + + return env; + } +}; + +template +class cgbn_env_t { + public: + + // bits must be divisible by 32 + static const uint32_t BITS=bits; + static const uint32_t TPB=context_t::TPB; + static const uint32_t TPI=context_t::TPI; + static const uint32_t MAX_ROTATION=context_t::MAX_ROTATION; + static const uint32_t SHM_LIMIT=context_t::SHM_LIMIT; + static const bool CONSTANT_TIME=context_t::CONSTANT_TIME; + static const cgbn_syncable_t SYNCABLE=syncable; + + static const uint32_t LIMBS=(bits/32+TPI-1)/TPI; + static const uint32_t LOCAL_LIMBS=((bits+32)/64+TPI-1)/TPI*TPI; + static const uint32_t UNPADDED_BITS=TPI*LIMBS*32; + static const uint32_t PADDING=bits/32%TPI; + static const uint32_t PAD_THREAD=(BITS/32)/LIMBS; + static const uint32_t PAD_LIMB=(BITS/32)%LIMBS; + + struct cgbn_t { + public: + typedef cgbn_env_t parent_env_t; + + uint32_t _limbs[LIMBS]; + }; + struct cgbn_wide_t { + public: + cgbn_t _low, _high; + }; + struct cgbn_local_t { + public: + uint64_t _limbs[LOCAL_LIMBS]; + }; + struct cgbn_accumulator_t { + public: + uint32_t _carry; + uint32_t _limbs[LIMBS]; + __device__ __forceinline__ cgbn_accumulator_t(); + }; + + const context_t &_context; + + __device__ __forceinline__ cgbn_env_t(const context_t &context); + + /* size conversion */ + template + __device__ __forceinline__ void set(cgbn_t &r, const source_cgbn_t &source) const; + + /* bn arithmetic routines */ + __device__ __forceinline__ void set(cgbn_t &r, const cgbn_t &a) const; + __device__ __forceinline__ void swap(cgbn_t &r, cgbn_t &a) const; + __device__ __forceinline__ int32_t add(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const; + __device__ __forceinline__ int32_t sub(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const; + __device__ __forceinline__ int32_t negate(cgbn_t &r, const cgbn_t &a) const; + __device__ __forceinline__ void mul(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const; + __device__ __forceinline__ void mul_high(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const; + __device__ __forceinline__ void sqr(cgbn_t &r, const cgbn_t &a) const; + __device__ __forceinline__ void sqr_high(cgbn_t &r, const cgbn_t &a) const; + __device__ __forceinline__ void div(cgbn_t &q, const cgbn_t &num, const cgbn_t &denom) const; + __device__ __forceinline__ void rem(cgbn_t &r, const cgbn_t &num, const cgbn_t &denom) const; + __device__ __forceinline__ void div_rem(cgbn_t &q, cgbn_t &r, const cgbn_t &num, const cgbn_t &denom) const; + __device__ __forceinline__ void sqrt(cgbn_t &s, const cgbn_t &a) const; + __device__ __forceinline__ void sqrt_rem(cgbn_t &s, cgbn_t &r, const cgbn_t &a) const; + __device__ __forceinline__ bool equals(const cgbn_t &a, const cgbn_t &b) const; + __device__ __forceinline__ int32_t compare(const cgbn_t &a, const cgbn_t &b) const; + __device__ __forceinline__ void extract_bits(cgbn_t &r, const cgbn_t &a, const uint32_t start, const uint32_t len) const; + __device__ __forceinline__ void insert_bits(cgbn_t &r, const cgbn_t &a, const uint32_t start, const uint32_t len, const cgbn_t &value) const; + + /* ui32 arithmetic routines*/ + __device__ __forceinline__ uint32_t get_ui32(const cgbn_t &a) const; + __device__ __forceinline__ void set_ui32(cgbn_t &r, const uint32_t value) const; + __device__ __forceinline__ int32_t add_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t add) const; + __device__ __forceinline__ int32_t sub_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t sub) const; + __device__ __forceinline__ uint32_t mul_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t mul) const; + __device__ __forceinline__ uint32_t div_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t div) const; + __device__ __forceinline__ uint32_t rem_ui32(const cgbn_t &a, const uint32_t div) const; + __device__ __forceinline__ bool equals_ui32(const cgbn_t &a, const uint32_t value) const; + __device__ __forceinline__ int32_t compare_ui32(const cgbn_t &a, const uint32_t value) const; + __device__ __forceinline__ uint32_t extract_bits_ui32(const cgbn_t &a, const uint32_t start, const uint32_t len) const; + __device__ __forceinline__ void insert_bits_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t start, const uint32_t len, const uint32_t value) const; + __device__ __forceinline__ uint32_t binary_inverse_ui32(const uint32_t n0) const; + __device__ __forceinline__ uint32_t gcd_ui32(const cgbn_t &a, const uint32_t value) const; + + /* wide arithmetic routines */ + __device__ __forceinline__ void mul_wide(cgbn_wide_t &r, const cgbn_t &a, const cgbn_t &b) const; + __device__ __forceinline__ void sqr_wide(cgbn_wide_t &r, const cgbn_t &a) const; + __device__ __forceinline__ void div_wide(cgbn_t &q, const cgbn_wide_t &num, const cgbn_t &denom) const; + __device__ __forceinline__ void rem_wide(cgbn_t &r, const cgbn_wide_t &num, const cgbn_t &denom) const; + __device__ __forceinline__ void div_rem_wide(cgbn_t &q, cgbn_t &r, const cgbn_wide_t &num, const cgbn_t &denom) const; + __device__ __forceinline__ void sqrt_wide(cgbn_t &s, const cgbn_wide_t &a) const; + __device__ __forceinline__ void sqrt_rem_wide(cgbn_t &s, cgbn_wide_t &r, const cgbn_wide_t &a) const; + + /* logical, shifting, masking */ + __device__ __forceinline__ void bitwise_and(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const; + __device__ __forceinline__ void bitwise_ior(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const; + __device__ __forceinline__ void bitwise_xor(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const; + __device__ __forceinline__ void bitwise_complement(cgbn_t &r, const cgbn_t &a) const; + __device__ __forceinline__ void bitwise_select(cgbn_t &r, const cgbn_t &clear, const cgbn_t &set, const cgbn_t &select) const; + __device__ __forceinline__ void bitwise_mask_copy(cgbn_t &r, const int32_t numbits) const; + __device__ __forceinline__ void bitwise_mask_and(cgbn_t &r, const cgbn_t &a, const int32_t numbits) const; + __device__ __forceinline__ void bitwise_mask_ior(cgbn_t &r, const cgbn_t &a, const int32_t numbits) const; + __device__ __forceinline__ void bitwise_mask_xor(cgbn_t &r, const cgbn_t &a, const int32_t numbits) const; + __device__ __forceinline__ void bitwise_mask_select(cgbn_t &r, const cgbn_t &clear, const cgbn_t &set, int32_t numbits) const; + __device__ __forceinline__ void shift_left(cgbn_t &r, const cgbn_t &a, const uint32_t numbits) const; + __device__ __forceinline__ void shift_right(cgbn_t &r, const cgbn_t &a, const uint32_t numbits) const; + __device__ __forceinline__ void rotate_left(cgbn_t &r, const cgbn_t &a, const uint32_t numbits) const; + __device__ __forceinline__ void rotate_right(cgbn_t &r, const cgbn_t &a, const uint32_t numbits) const; + + /* bit counting */ + __device__ __forceinline__ uint32_t pop_count(const cgbn_t &a) const; + __device__ __forceinline__ uint32_t clz(const cgbn_t &a) const; + __device__ __forceinline__ uint32_t ctz(const cgbn_t &a) const; + + /* accumulator APIs */ + __device__ __forceinline__ int32_t resolve(cgbn_t &sum, const cgbn_accumulator_t &accumulator) const; + __device__ __forceinline__ void set(cgbn_accumulator_t &accumulator, const cgbn_t &value) const; + __device__ __forceinline__ void add(cgbn_accumulator_t &accumulator, const cgbn_t &value) const; + __device__ __forceinline__ void sub(cgbn_accumulator_t &accumulator, const cgbn_t &value) const; + __device__ __forceinline__ void set_ui32(cgbn_accumulator_t &accumulator, const uint32_t value) const; + __device__ __forceinline__ void add_ui32(cgbn_accumulator_t &accumulator, const uint32_t value) const; + __device__ __forceinline__ void sub_ui32(cgbn_accumulator_t &accumulator, const uint32_t value) const; + + /* math */ + __device__ __forceinline__ void binary_inverse(cgbn_t &r, const cgbn_t &x) const; + __device__ __forceinline__ void gcd(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const; + __device__ __forceinline__ bool modular_inverse(cgbn_t &r, const cgbn_t &x, const cgbn_t &modulus) const; + __device__ __forceinline__ void modular_power(cgbn_t &r, const cgbn_t &x, const cgbn_t &exponent, const cgbn_t &modulus) const; + + /* fast division: common divisor / modulus */ + __device__ __forceinline__ uint32_t bn2mont(cgbn_t &mont, const cgbn_t &bn, const cgbn_t &n) const; + __device__ __forceinline__ void mont2bn(cgbn_t &bn, const cgbn_t &mont, const cgbn_t &n, const uint32_t np0) const; + __device__ __forceinline__ void mont_mul(cgbn_t &r, const cgbn_t &a, const cgbn_t &b, const cgbn_t &n, const uint32_t np0) const; + __device__ __forceinline__ void mont_sqr(cgbn_t &r, const cgbn_t &a, const cgbn_t &n, const uint32_t np0) const; + __device__ __forceinline__ void mont_reduce_wide(cgbn_t &r, const cgbn_wide_t &a, const cgbn_t &n, const uint32_t np0) const; + + __device__ __forceinline__ uint32_t barrett_approximation(cgbn_t &approx, const cgbn_t &denom) const; + __device__ __forceinline__ void barrett_div(cgbn_t &q, const cgbn_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const; + __device__ __forceinline__ void barrett_rem(cgbn_t &r, const cgbn_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const; + __device__ __forceinline__ void barrett_div_rem(cgbn_t &q, cgbn_t &r, const cgbn_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const; + __device__ __forceinline__ void barrett_div_wide(cgbn_t &q, const cgbn_wide_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const; + __device__ __forceinline__ void barrett_rem_wide(cgbn_t &r, const cgbn_wide_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const; + __device__ __forceinline__ void barrett_div_rem_wide(cgbn_t &q, cgbn_t &r, const cgbn_wide_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const; + + /* load/store to global or shared memory */ + __device__ __forceinline__ void load(cgbn_t &r, cgbn_mem_t *const address) const; + __device__ __forceinline__ void store(cgbn_mem_t *address, const cgbn_t &a) const; + + /* load/store to local memory */ + __device__ __forceinline__ void load(cgbn_t &r, cgbn_local_t *const address) const; + __device__ __forceinline__ void store(cgbn_local_t *address, const cgbn_t &a) const; +}; + +#include "impl_cuda.cu" + +/* +experimental: + + // faster shift and rotate by a constant number of bits + template __device__ __forceinline__ void shift_left(cgbn_t &r, const cgbn_t &a); + template __device__ __forceinline__ void shift_right(cgbn_t &r, const cgbn_t &a); + template __device__ __forceinline__ void rotate_left(cgbn_t &r, const cgbn_t &a); + template __device__ __forceinline__ void rotate_right(cgbn_t &r, const cgbn_t &a); + +*/ diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn_mpz.h b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn_mpz.h new file mode 100644 index 0000000..01e46fd --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/cgbn_mpz.h @@ -0,0 +1,261 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +#include +#include + +#if !defined(__CUDACC__) + typedef struct {uint32_t x; uint32_t y; uint32_t z;} dim3; + #define __host__ +#endif + +typedef enum { + cgbn_instance_converged, + cgbn_warp_converged, + cgbn_block_converged, + cgbn_grid_converged, +} cgbn_convergence_t; + +class cgbn_default_parameters_t { + public: + static const uint32_t TPB=0; +}; + +/* forward declarations */ +template +class cgbn_context_t; + +template +class cgbn_env_t; + +template +struct cgbn_mem_t { + public: + uint32_t _limbs[(bits+31)/32]; +}; + +template +class cgbn_context_t { + public: + static const uint32_t TPI=tpi; + + const cgbn_monitor_t _monitor; + cgbn_error_report_t *const _report; + int32_t _instance; + + public: + __host__ cgbn_context_t(); + __host__ cgbn_context_t(cgbn_monitor_t monitor); + __host__ cgbn_context_t(cgbn_monitor_t monitor, cgbn_error_report_t *report); + __host__ cgbn_context_t(cgbn_monitor_t monitor, cgbn_error_report_t *report, uint32_t instance); + __host__ bool check_errors() const; + __host__ void report_error(cgbn_error_t error) const; + + template + env_t env() { + env_t env(*this); + + return env; + } + + template + cgbn_env_t env() { + cgbn_env_t env(*this); + + return env; + } +}; + +template +class cgbn_env_t { + public: + static const uint32_t TPI=context_t::TPI; + static const uint32_t BITS=bits; + static const uint32_t LIMBS=(bits/32+TPI-1)/TPI; + static const uint32_t LOCAL_LIMBS=((bits+32)/64+TPI-1)/TPI*TPI; + static const uint32_t UNPADDED_BITS=TPI*LIMBS*32; + + struct cgbn_t { + public: + typedef cgbn_env_t parent_env_t; + mpz_t _z; + + __host__ cgbn_t() { + mpz_init(_z); + } + + __host__ ~cgbn_t() { + mpz_clear(_z); + } + }; + struct cgbn_wide_t { + public: + cgbn_t _low, _high; + }; + struct cgbn_local_t { + public: + mpz_t _z; + + __host__ cgbn_local_t() { + mpz_init(_z); + } + + __host__ ~cgbn_local_t() { + mpz_clear(_z); + } + }; + struct cgbn_accumulator_t { + public: + mpz_t _z; + + __host__ cgbn_accumulator_t() { + mpz_init(_z); + } + + __host__ ~cgbn_accumulator_t() { + mpz_clear(_z); + } + }; + + const context_t &_context; + + __host__ cgbn_env_t(const context_t &context); + + /* size conversion */ + template + __host__ void set(cgbn_t &r, const source_cgbn_t &source) const; + + /* set/get routines */ + __host__ void set(cgbn_t &r, const cgbn_t &a) const; + __host__ void swap(cgbn_t &r, cgbn_t &a) const; + __host__ void extract_bits(cgbn_t &r, const cgbn_t &a, uint32_t start, uint32_t len) const; + __host__ void insert_bits(cgbn_t &r, const cgbn_t &a, uint32_t start, uint32_t len, const cgbn_t &value) const; + + /* ui32 arithmetic routines*/ + __host__ uint32_t get_ui32(const cgbn_t &a) const; + __host__ void set_ui32(cgbn_t &r, const uint32_t value) const; + __host__ int32_t add_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t add) const; + __host__ int32_t sub_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t sub) const; + __host__ uint32_t mul_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t mul) const; + __host__ uint32_t div_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t div) const; + __host__ uint32_t rem_ui32(const cgbn_t &a, const uint32_t div) const; + __host__ bool equals_ui32(const cgbn_t &a, const uint32_t value) const; + __host__ int32_t compare_ui32(const cgbn_t &a, const uint32_t value) const; + __host__ uint32_t extract_bits_ui32(const cgbn_t &a, const uint32_t start, const uint32_t len) const; + __host__ void insert_bits_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t start, const uint32_t len, const uint32_t value) const; + __host__ uint32_t binary_inverse_ui32(const uint32_t n0) const; + __host__ uint32_t gcd_ui32(const cgbn_t &a, const uint32_t value) const; + + /* bn arithmetic routines */ + __host__ int32_t add(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const; + __host__ int32_t sub(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const; + __host__ int32_t negate(cgbn_t &r, const cgbn_t &a) const; + __host__ void mul(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const; + __host__ void mul_high(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const; + __host__ void sqr(cgbn_t &r, const cgbn_t &a) const; + __host__ void sqr_high(cgbn_t &r, const cgbn_t &a) const; + __host__ void div(cgbn_t &q, const cgbn_t &num, const cgbn_t &denom) const; + __host__ void rem(cgbn_t &r, const cgbn_t &num, const cgbn_t &denom) const; + __host__ void div_rem(cgbn_t &q, cgbn_t &r, const cgbn_t &num, const cgbn_t &denom) const; + __host__ void sqrt(cgbn_t &s, const cgbn_t &a) const; + __host__ void sqrt_rem(cgbn_t &s, cgbn_t &r, const cgbn_t &a) const; + __host__ bool equals(const cgbn_t &a, const cgbn_t &b) const; + __host__ int32_t compare(const cgbn_t &a, const cgbn_t &b) const; + + /* wide math routines */ + __host__ void mul_wide(cgbn_wide_t &r, const cgbn_t &a, const cgbn_t &b) const; + __host__ void sqr_wide(cgbn_wide_t &r, const cgbn_t &a) const; + __host__ void div_wide(cgbn_t &q, const cgbn_wide_t &num, const cgbn_t &denom) const; + __host__ void rem_wide(cgbn_t &r, const cgbn_wide_t &num, const cgbn_t &denom) const; + __host__ void div_rem_wide(cgbn_t &q, cgbn_t &r, const cgbn_wide_t &num, const cgbn_t &denom) const; + __host__ void sqrt_wide(cgbn_t &s, const cgbn_wide_t &a) const; + __host__ void sqrt_rem_wide(cgbn_t &s, cgbn_wide_t &r, const cgbn_wide_t &a) const; + + /* logical, shifting, masking */ + __host__ void bitwise_complement(cgbn_t &r, const cgbn_t &a) const; + __host__ void bitwise_and(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const; + __host__ void bitwise_ior(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const; + __host__ void bitwise_xor(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const; + __host__ void bitwise_select(cgbn_t &r, const cgbn_t &clear, const cgbn_t &set, const cgbn_t &select) const; + __host__ void bitwise_mask_copy(cgbn_t &r, const int32_t numbits) const; + __host__ void bitwise_mask_and(cgbn_t &r, const cgbn_t &a, const int32_t numbits) const; + __host__ void bitwise_mask_ior(cgbn_t &r, const cgbn_t &a, const int32_t numbits) const; + __host__ void bitwise_mask_xor(cgbn_t &r, const cgbn_t &a, const int32_t numbits) const; + __host__ void bitwise_mask_select(cgbn_t &r, const cgbn_t &clear, const cgbn_t &set, const int32_t numbits) const; + __host__ void shift_left(cgbn_t &r, const cgbn_t &a, const uint32_t numbits) const; + __host__ void shift_right(cgbn_t &r, const cgbn_t &a, const uint32_t numbits) const; + __host__ void rotate_left(cgbn_t &r, const cgbn_t &a, const uint32_t numbits) const; + __host__ void rotate_right(cgbn_t &r, const cgbn_t &a, const uint32_t numbits) const; + + /* faster shift and rotate by a constant number of bits */ + template __host__ void shift_left(cgbn_t &r, const cgbn_t &a) const; + template __host__ void shift_right(cgbn_t &r, const cgbn_t &a) const; + template __host__ void rotate_left(cgbn_t &r, const cgbn_t &a) const; + template __host__ void rotate_right(cgbn_t &r, const cgbn_t &a) const; + + /* bit counting */ + __host__ uint32_t pop_count(const cgbn_t &a) const; + __host__ uint32_t clz(const cgbn_t &a) const; + __host__ uint32_t ctz(const cgbn_t &a) const; + + /* accumulator APIs */ + __host__ int32_t resolve(cgbn_t &sum, const cgbn_accumulator_t &accumulator) const; + __host__ void set_ui32(cgbn_accumulator_t &accumulator, const uint32_t value) const; + __host__ void add_ui32(cgbn_accumulator_t &accumulator, const uint32_t value) const; + __host__ void sub_ui32(cgbn_accumulator_t &accumulator, const uint32_t value) const; + __host__ void set(cgbn_accumulator_t &accumulator, const cgbn_t &value) const; + __host__ void add(cgbn_accumulator_t &accumulator, const cgbn_t &value) const; + __host__ void sub(cgbn_accumulator_t &accumulator, const cgbn_t &value) const; + + /* math */ + __host__ void binary_inverse(cgbn_t &r, const cgbn_t &m) const; + __host__ bool modular_inverse(cgbn_t &r, const cgbn_t &x, const cgbn_t &modulus) const; + __host__ void modular_power(cgbn_t &r, const cgbn_t &a, const cgbn_t &k, const cgbn_t &m) const; + __host__ void gcd(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const; + + /* fast division: common divisor / modulus */ + __host__ uint32_t bn2mont(cgbn_t &mont, const cgbn_t &bn, const cgbn_t &n) const; + __host__ void mont2bn(cgbn_t &bn, const cgbn_t &mont, const cgbn_t &n, const uint32_t np0) const; + __host__ void mont_mul(cgbn_t &r, const cgbn_t &a, const cgbn_t &b, const cgbn_t &n, const uint32_t np0) const; + __host__ void mont_sqr(cgbn_t &r, const cgbn_t &a, const cgbn_t &n, const uint32_t np0) const; + __host__ void mont_reduce_wide(cgbn_t &r, const cgbn_wide_t &a, const cgbn_t &n, const uint32_t np0) const; + + __host__ uint32_t barrett_approximation(cgbn_t &approx, const cgbn_t &denom) const; + __host__ void barrett_div(cgbn_t &q, const cgbn_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const; + __host__ void barrett_rem(cgbn_t &r, const cgbn_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const; + __host__ void barrett_div_rem(cgbn_t &q, cgbn_t &r, const cgbn_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const; + __host__ void barrett_div_wide(cgbn_t &q, const cgbn_wide_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const; + __host__ void barrett_rem_wide(cgbn_t &r, const cgbn_wide_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const; + __host__ void barrett_div_rem_wide(cgbn_t &q, cgbn_t &r, const cgbn_wide_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const; + + /* load/store to global or shared memory */ + __host__ void load(cgbn_t &r, cgbn_mem_t *const address) const; + __host__ void store(cgbn_mem_t *address, const cgbn_t &a) const; + + /* load/store to local memory */ + __host__ void load(cgbn_t &r, cgbn_local_t *const address) const; + __host__ void store(cgbn_local_t *address, const cgbn_t &a) const; +}; + +#include "impl_mpz.cc" diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core.cu new file mode 100644 index 0000000..b355bd6 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core.cu @@ -0,0 +1,327 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +#include "dispatch_padding.cu" +#include "dispatch_resolver.cu" +#include "dispatch_masking.cu" +#include "dispatch_shift_rotate.cu" +#include "dispatch_dlimbs.cu" + +namespace cgbn { + +typedef struct { + // used in gcd + int32_t alpha_a; + int32_t alpha_b; + int32_t beta_a; + int32_t beta_b; +} signed_coeff_t; + +typedef struct { + // used in modinv + uint32_t alpha_a; + uint32_t alpha_b; + uint32_t beta_a; + uint32_t beta_b; +} unsigned_coeff_t; + +template +class core_t { + public: + static const uint32_t TPB=env::TPB; + static const uint32_t TPI=env::TPI; + static const uint32_t BITS=env::BITS; + static const uint32_t LIMBS=env::LIMBS; + static const uint32_t PADDING=env::PADDING; + static const uint32_t MAX_ROTATION=env::MAX_ROTATION; + static const uint32_t SHM_LIMIT=env::SHM_LIMIT; + static const bool CONSTANT_TIME=env::CONSTANT_TIME; + static const cgbn_syncable_t SYNCABLE=env::SYNCABLE; + + static const uint32_t TPI_ONES=(1ull<::clear_carry(x); + } + + __device__ __forceinline__ static uint32_t clear_carry(uint32_t x[LIMBS]) { + return dispatch_padding_t::clear_carry(x); + } + + __device__ __forceinline__ static void clear_padding(uint32_t &x) { + dispatch_padding_t::clear_padding(x); + } + + __device__ __forceinline__ static void clear_padding(uint32_t x[LIMBS]) { + dispatch_padding_t::clear_padding(x); + } + + /* CORE RESOLVER ROUTINES */ + __device__ __forceinline__ static int32_t fast_negate(uint32_t &x) { + return dispatch_resolver_t::fast_negate(x); + } + + __device__ __forceinline__ static int32_t fast_negate(uint32_t x[LIMBS]) { + return dispatch_resolver_t::fast_negate(x); + } + + __device__ __forceinline__ static int32_t fast_propagate_add(const int32_t carry, uint32_t &x) { + return dispatch_resolver_t::fast_propagate_add(carry, x); + } + + __device__ __forceinline__ static int32_t fast_propagate_add(const uint32_t carry, uint32_t x[LIMBS]) { + return dispatch_resolver_t::fast_propagate_add(carry, x); + } + + __device__ __forceinline__ static int32_t fast_propagate_sub(const uint32_t carry, uint32_t &x) { + return dispatch_resolver_t::fast_propagate_sub(carry, x); + } + + __device__ __forceinline__ static int32_t fast_propagate_sub(const uint32_t carry, uint32_t x[LIMBS]) { + return dispatch_resolver_t::fast_propagate_sub(carry, x); + } + + __device__ __forceinline__ static int32_t resolve_add(const uint32_t carry, uint32_t &x) { + return dispatch_resolver_t::resolve_add(carry, x); + } + + __device__ __forceinline__ static int32_t resolve_add(const uint32_t carry, uint32_t x[LIMBS]) { + return dispatch_resolver_t::resolve_add(carry, x); + } + + __device__ __forceinline__ static int32_t resolve_sub(const uint32_t carry, uint32_t &x) { + return dispatch_resolver_t::resolve_sub(carry, x); + } + + __device__ __forceinline__ static int32_t resolve_sub(const uint32_t carry, uint32_t x[LIMBS]) { + return dispatch_resolver_t::resolve_sub(carry, x); + } + + + // HELPER FUNCTIONS FOR GCD AND MOD INV + __device__ __forceinline__ static void gcd_product(const uint32_t sync, uint32_t r[LIMBS], const int32_t sa, const uint32_t a[LIMBS], const int32_t sb, const uint32_t b[LIMBS]); + __device__ __forceinline__ static void modinv_update_uw_qs(const uint32_t sync, uint32_t r[LIMBS], const uint32_t q, const int32_t s, const uint32_t x[LIMBS]); + __device__ __forceinline__ static void modinv_update_ab_sq(const uint32_t sync, uint32_t r[LIMBS], const int32_t shift, const uint32_t q, const uint32_t x[LIMBS]); + __device__ __forceinline__ static void modinv_update_uw(const uint32_t sync, uint32_t u[LIMBS], uint32_t w[LIMBS], const unsigned_coeff_t &coeffs); + __device__ __forceinline__ static bool modinv_small_delta(const uint32_t sync, uint32_t a[LIMBS], uint32_t b[LIMBS], uint32_t u[LIMBS], uint32_t w[LIMBS], int32_t &delta); + + // DISTRIBUTED LIMB FUNCTIONS FOR DIVISION AND SQUARE ROOT + __device__ __forceinline__ static void dlimbs_scatter(uint32_t r[DLIMBS], const uint32_t x[LIMBS], const uint32_t source_thread) { + dispatch_dlimbs_t::dlimbs_scatter(r, x, source_thread); + } + + __device__ __forceinline__ static void dlimbs_gather(uint32_t r[LIMBS], const uint32_t x[DLIMBS], const uint32_t destination_thread) { + dispatch_dlimbs_t::dlimbs_gather(r, x, destination_thread); + } + + __device__ __forceinline__ static void dlimbs_all_gather(uint32_t r[LIMBS], const uint32_t x[DLIMBS]) { + dispatch_dlimbs_t::dlimbs_all_gather(r, x); + } + + __device__ __forceinline__ static uint32_t dlimbs_sqrt_rem_wide(uint32_t s[DLIMBS], uint32_t r[DLIMBS], const uint32_t lo[DLIMBS], const uint32_t hi[DLIMBS]) { + return dispatch_dlimbs_t::dlimbs_sqrt_rem_wide(s, r, lo, hi); + } + + __device__ __forceinline__ static void dlimbs_approximate(uint32_t approx[DLIMBS], const uint32_t denom[DLIMBS]) { + dispatch_dlimbs_t::dlimbs_approximate(approx, denom); + } + + __device__ __forceinline__ static void dlimbs_div_estimate(uint32_t q[DLIMBS], const uint32_t num[DLIMBS], const uint32_t approx[DLIMBS]) { + dispatch_dlimbs_t::dlimbs_div_estimate(q, num, approx); + } + + __device__ __forceinline__ static void dlimbs_sqrt_estimate(uint32_t q[DLIMBS], const uint32_t top, const uint32_t x[DLIMBS], const uint32_t approx[DLIMBS]) { + dispatch_dlimbs_t::dlimbs_sqrt_estimate(q, top, x, approx); + } + + + public: + __device__ __forceinline__ static uint32_t instance_sync_mask() { + uint32_t group_thread=threadIdx.x & TPI-1, warp_thread=threadIdx.x & warpSize-1; + + return TPI_ONES<<(group_thread ^ warp_thread); + } + + __device__ __forceinline__ static uint32_t sync_mask() { + // the following is sure to blow up on gcd and modinv and possibly others + // return (SYNCABLE==cgbn_instance_converged) ? instance_sync_mask() : 0xFFFFFFFF; + + // instead, for now, always use + return instance_sync_mask(); + } + + + /* BN ROUTINES */ + __device__ __forceinline__ static void set(uint32_t r[LIMBS], const uint32_t a[LIMBS]) { + mpset(r, a); + } + __device__ __forceinline__ static void swap(uint32_t r[LIMBS], uint32_t a[LIMBS]) { + mpswap(r, a); + } + + /* BASIC ARITHMETIC */ + __device__ __forceinline__ static int32_t add(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS]); + __device__ __forceinline__ static int32_t sub(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS]); + __device__ __forceinline__ static int32_t negate(uint32_t r[LIMBS], const uint32_t a[LIMBS]); + __device__ __forceinline__ static void mul(uint32_t &r, const uint32_t a, const uint32_t b, const uint32_t add); + __device__ __forceinline__ static void mul(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS], const uint32_t add[LIMBS]); + __device__ __forceinline__ static void sqrt_resolve_rem(uint32_t &rem, const uint32_t s, const uint32_t top, const uint32_t r, const uint32_t shift); + __device__ __forceinline__ static void sqrt_resolve_rem(uint32_t rem[LIMBS], const uint32_t s[LIMBS], const uint32_t top, const uint32_t r[LIMBS], const uint32_t shift); + __device__ __forceinline__ static void sqrt(uint32_t &s, const uint32_t x, const uint32_t numthreads); + __device__ __forceinline__ static void sqrt(uint32_t s[LIMBS], const uint32_t x[LIMBS], const uint32_t numthreads); + __device__ __forceinline__ static void sqrt_rem(uint32_t &s, uint32_t &r, const uint32_t x, const uint32_t numthreads); + __device__ __forceinline__ static void sqrt_rem(uint32_t s[LIMBS], uint32_t r[LIMBS], const uint32_t x[LIMBS], const uint32_t numthreads); + __device__ __forceinline__ static bool equals(const uint32_t a[LIMBS], const uint32_t b[LIMBS]); + __device__ __forceinline__ static int32_t compare(const uint32_t a[LIMBS], const uint32_t b[LIMBS]); + + /* WIDE ARITHMETIC */ + __device__ __forceinline__ static void mul_wide(uint32_t &lo, uint32_t &hi, const uint32_t a, const uint32_t b, const uint32_t add); + __device__ __forceinline__ static void mul_wide(uint32_t lo[LIMBS], uint32_t hi[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS], const uint32_t add[LIMBS]); + __device__ __forceinline__ static void div_wide(uint32_t &q, const uint32_t lo, const uint32_t hi, const uint32_t denom, const uint32_t numthreads); + __device__ __forceinline__ static void div_wide(uint32_t q[LIMBS], const uint32_t lo[LIMBS], const uint32_t hi[LIMBS], const uint32_t denom[LIMBS], const uint32_t numthreads); + __device__ __forceinline__ static void rem_wide(uint32_t &r, const uint32_t lo, const uint32_t hi, const uint32_t denom, const uint32_t numthreads); + __device__ __forceinline__ static void rem_wide(uint32_t r[LIMBS], const uint32_t lo[LIMBS], const uint32_t hi[LIMBS], const uint32_t denom[LIMBS], const uint32_t numthreads); + __device__ __forceinline__ static void div_rem_wide(uint32_t &q, uint32_t &r, const uint32_t lo, const uint32_t hi, const uint32_t denom, const uint32_t numthreads); + __device__ __forceinline__ static void div_rem_wide(uint32_t q[LIMBS], uint32_t r[LIMBS], const uint32_t lo[LIMBS], const uint32_t hi[LIMBS], const uint32_t denom[LIMBS], const uint32_t numthreads); + __device__ __forceinline__ static void sqrt_wide(uint32_t &s, const uint32_t lo, const uint32_t hi, const uint32_t numthreads); + __device__ __forceinline__ static void sqrt_wide(uint32_t s[LIMBS], const uint32_t lo[LIMBS], const uint32_t hi[LIMBS], const uint32_t numthreads); + __device__ __forceinline__ static uint32_t sqrt_rem_wide(uint32_t &s, uint32_t &r, const uint32_t lo, const uint32_t hi, const uint32_t numthreads); + __device__ __forceinline__ static uint32_t sqrt_rem_wide(uint32_t s[LIMBS], uint32_t r[LIMBS], const uint32_t lo[LIMBS], const uint32_t hi[LIMBS], const uint32_t numthreads); + + /* UI32 ROUTINES */ + __device__ __forceinline__ static uint32_t get_ui32(const uint32_t a[LIMBS]); + __device__ __forceinline__ static void set_ui32(uint32_t r[LIMBS], const uint32_t value); + __device__ __forceinline__ static int32_t add_ui32(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t add); + __device__ __forceinline__ static int32_t sub_ui32(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t sub); + __device__ __forceinline__ static uint32_t mul_ui32(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t mul); + __device__ __forceinline__ static uint32_t div_ui32(uint32_t &r, const uint32_t a, const uint32_t div); + __device__ __forceinline__ static uint32_t div_ui32(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t div); + __device__ __forceinline__ static uint32_t rem_ui32(const uint32_t a, const uint32_t div); + __device__ __forceinline__ static uint32_t rem_ui32(const uint32_t a[LIMBS], const uint32_t div); + __device__ __forceinline__ static uint32_t extract_bits_ui32(const uint32_t a[LIMBS], const uint32_t start, const uint32_t len); + __device__ __forceinline__ static void insert_bits_ui32(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t start, const uint32_t len, uint32_t value); + __device__ __forceinline__ static bool equals_ui32(const uint32_t a[LIMBS], const uint32_t value); + __device__ __forceinline__ static int32_t compare_ui32(const uint32_t a[LIMBS], const uint32_t value); + + /* LOGICAL */ + __device__ __forceinline__ static void bitwise_and(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS]); + __device__ __forceinline__ static void bitwise_ior(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS]); + __device__ __forceinline__ static void bitwise_xor(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS]); + __device__ __forceinline__ static void bitwise_complement(uint32_t r[LIMBS], const uint32_t a[LIMBS]); + __device__ __forceinline__ static void bitwise_select(uint32_t r[LIMBS], const uint32_t clear[LIMBS], const uint32_t set[LIMBS], const uint32_t select[LIMBS]); + + /* MASKING */ + __device__ __forceinline__ static void bitwise_mask_copy(uint32_t r[LIMBS], const int32_t numbits) { + dispatch_masking_t::bitwise_mask_copy(r, numbits); + } + + __device__ __forceinline__ static void bitwise_mask_and(uint32_t r[LIMBS], const uint32_t a[LIMBS], const int32_t numbits) { + dispatch_masking_t::bitwise_mask_and(r, a, numbits); + } + + __device__ __forceinline__ static void bitwise_mask_ior(uint32_t r[LIMBS], const uint32_t a[LIMBS], const int32_t numbits) { + dispatch_masking_t::bitwise_mask_ior(r, a, numbits); + } + + __device__ __forceinline__ static void bitwise_mask_xor(uint32_t r[LIMBS], const uint32_t a[LIMBS], const int32_t numbits) { + dispatch_masking_t::bitwise_mask_xor(r, a, numbits); + } + + __device__ __forceinline__ static void bitwise_mask_select(uint32_t r[LIMBS], const uint32_t clear[LIMBS], const uint32_t set[LIMBS], const int32_t numbits) { + dispatch_masking_t::bitwise_mask_select(r, clear, set, numbits); + } + + __device__ __forceinline__ static void shift_left(uint32_t r[LIMBS], const uint32_t a[LIMBS], const int32_t numbits) { + dispatch_shift_rotate_t::shift_left(r, a, numbits); + } + + __device__ __forceinline__ static void shift_right(uint32_t r[LIMBS], const uint32_t a[LIMBS], const int32_t numbits) { + dispatch_shift_rotate_t::shift_right(r, a, numbits); + } + + __device__ __forceinline__ static void rotate_left(uint32_t r[LIMBS], const uint32_t a[LIMBS], const int32_t numbits) { + dispatch_shift_rotate_t::rotate_left(r, a, numbits); + } + + __device__ __forceinline__ static void rotate_right(uint32_t r[LIMBS], const uint32_t a[LIMBS], const int32_t numbits) { + dispatch_shift_rotate_t::rotate_right(r, a, numbits); + } + + /* BIT COUNTING AND THREAD COUNTING ROUTINES */ + __device__ __forceinline__ static uint32_t pop_count(const uint32_t a[LIMBS]); + __device__ __forceinline__ static uint32_t clz(const uint32_t a[LIMBS]); + __device__ __forceinline__ static uint32_t ctz(const uint32_t a[LIMBS]); + __device__ __forceinline__ static uint32_t clzt(const uint32_t a[LIMBS]); + __device__ __forceinline__ static uint32_t ctzt(const uint32_t a[LIMBS]); + + /* NUMBER THEORETIC ROUTINES */ + __device__ __forceinline__ static void gcd(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS]); + __device__ __forceinline__ static void binary_inverse(uint32_t inv[LIMBS], const uint32_t x[LIMBS]); + __device__ __forceinline__ static bool modular_inverse(uint32_t inv[LIMBS], const uint32_t x[LIMBS], const uint32_t y[LIMBS]); + + /* MONTGOMERY ROUTINES */ + __device__ __forceinline__ static void mont_mul(uint32_t &r, const uint32_t a, const uint32_t b, const uint32_t n, const uint32_t np0); + __device__ __forceinline__ static void mont_mul(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS], const uint32_t n[LIMBS], const uint32_t np0); + __device__ __forceinline__ static void mont_reduce_wide(uint32_t &r, const uint32_t lo, const uint32_t hi, const uint32_t n, const uint32_t np0, const bool zero); + __device__ __forceinline__ static void mont_reduce_wide(uint32_t r[LIMBS], const uint32_t lo[LIMBS], const uint32_t hi[LIMBS], const uint32_t n[LIMBS], const uint32_t np0, const bool zero); +}; + +} /* namespace cgbn */ + +#include "core_add_sub.cu" +#include "core_short_math.cu" +#include "core_compare.cu" +#include "core_counting.cu" +#include "core_insert_extract.cu" +#include "core_logical.cu" +#include "core_mul.cu" +#include "core_divide_single.cu" +#include "core_divide_multi.cu" +#include "core_sqrt_single.cu" +#include "core_sqrt_multi.cu" +#include "core_gcd.cu" +#include "core_binary_inverse.cu" +#include "core_modular_inverse.cu" +#include "core_mont.cu" + +#if defined(XMP_IMAD) + #include "core_mul_imad.cu" + #include "core_mont_imad.cu" +#elif defined(XMP_XMAD) + #include "core_mul_xmad.cu" + #include "core_mont_xmad.cu" +#elif defined(XMP_WMAD) + #include "core_mul_wmad.cu" + #include "core_mont_wmad.cu" +#else + #warning One of XMP_IMAD, XMP_XMAD, XMP_WMAD must be defined +#endif + diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_add_sub.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_add_sub.cu new file mode 100644 index 0000000..532916a --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_add_sub.cu @@ -0,0 +1,57 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +__device__ __forceinline__ int32_t core_t::add(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS]) { + uint32_t carry; + + chain_t<> chain; + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ int32_t core_t::sub(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS]) { + uint32_t carry; + + chain_t<> chain; + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ int32_t core_t::negate(uint32_t r[LIMBS], const uint32_t a[LIMBS]) { + mpset(r, a); + return fast_negate(r); +} + +} /* namespace cgbn */ diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_binary_inverse.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_binary_inverse.cu new file mode 100644 index 0000000..82ff373 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_binary_inverse.cu @@ -0,0 +1,79 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +__device__ __forceinline__ void core_t::binary_inverse(uint32_t r[LIMBS], const uint32_t x[LIMBS]) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t inverse[LIMBS], c1, current[LIMBS], inv, t, low; + uint32_t stop_thread; + + inv=-ubinary_inverse(__shfl_sync(sync, x[0], 0, TPI)); + + #pragma unroll + for(int32_t index=0;index chain1; + #pragma unroll + for(int32_t index=0;index chain2; + #pragma unroll + for(int32_t index=0;index(r, inverse); + clear_padding(r); +} + +} /* namespace cgbn */ \ No newline at end of file diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_compare.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_compare.cu new file mode 100644 index 0000000..05019ae --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_compare.cu @@ -0,0 +1,37 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +__device__ __forceinline__ bool core_t::equals(const uint32_t a[LIMBS], const uint32_t b[LIMBS]) { + return dequals(sync_mask(), a, b); +} + +template +__device__ __forceinline__ int32_t core_t::compare(const uint32_t a[LIMBS], const uint32_t b[LIMBS]) { + return dcompare(sync_mask(), a, b); +} + +} /* namespace cgbn */ \ No newline at end of file diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_counting.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_counting.cu new file mode 100644 index 0000000..c23615f --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_counting.cu @@ -0,0 +1,96 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +__device__ __forceinline__ uint32_t core_t::pop_count(const uint32_t a[LIMBS]) { + uint32_t sync=sync_mask(), total=0; + + #pragma unroll + for(int32_t index=0;index0;index=index>>1) + total+=__shfl_xor_sync(sync, total, index, TPI); + return total; +} + +template +__device__ __forceinline__ uint32_t core_t::clz(const uint32_t a[LIMBS]) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1, warp_thread=threadIdx.x & warpSize-1; + uint32_t clz, topclz; + + clz=mpclz(a); + topclz=__ballot_sync(sync, clz!=32*LIMBS); + if(TPI=TPI) + return BITS; + return __shfl_sync(sync, (TPI-1-group_thread)*32*LIMBS + clz, 31-topclz, TPI)-LIMBS*TPI*32+BITS; +} + +template +__device__ __forceinline__ uint32_t core_t::ctz(const uint32_t a[LIMBS]) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1, warp_thread=threadIdx.x & warpSize-1; + uint32_t ctz, bottomctz; + + ctz=mpctz(a); + bottomctz=__ballot_sync(sync, ctz!=32*LIMBS); + if(TPI>(warp_thread^group_thread); + bottomctz=uctz(bottomctz); + if(bottomctz>=TPI) + return BITS; + return __shfl_sync(sync, group_thread*32*LIMBS + ctz, bottomctz, TPI); +} + +template +__device__ __forceinline__ uint32_t core_t::clzt(const uint32_t a[LIMBS]) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1, warp_thread=threadIdx.x & warpSize-1; + uint32_t lor, topclz; + + lor=mplor(a); + topclz=__ballot_sync(sync, lor!=0); + if(TPI +__device__ __forceinline__ uint32_t core_t::ctzt(const uint32_t a[LIMBS]) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1, warp_thread=threadIdx.x & warpSize-1; + uint32_t lor, bottomctz; + + lor=mplor(a); + bottomctz=__ballot_sync(sync, lor!=0); + if(TPI>(warp_thread^group_thread); + bottomctz=uctz(bottomctz); + return umin(topctz, TPI); +} + +} /* namespace cgbn */ \ No newline at end of file diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_divide_multi.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_divide_multi.cu new file mode 100644 index 0000000..96da0dd --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_divide_multi.cu @@ -0,0 +1,252 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +__device__ __forceinline__ void core_t::div_wide(uint32_t q[LIMBS], const uint32_t lo[LIMBS], const uint32_t hi[LIMBS], const uint32_t denom[LIMBS], const uint32_t numthreads) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + int32_t x2; + uint32_t dtemp[DLIMBS], approx[DLIMBS], estimate[DLIMBS], t, c, x0, x1, d0, d1, correction; + uint32_t x[LIMBS], y[LIMBS], plo[LIMBS], phi[LIMBS], quotient[LIMBS]; + + mpzero(quotient); + if(numthreads chain; + #pragma unroll + for(int32_t index=0;index(x, y); + quotient[0]=(group_thread==numthreads) ? 1 : quotient[0]; + } + } + else + mpset(x, hi); + + d1=__shfl_sync(sync, denom[LIMBS-1], TPI-1, TPI); + d0=__shfl_sync(sync, denom[LIMBS-2], TPI-1, TPI); + + dlimbs_scatter(dtemp, denom, TPI-1); + dlimbs_approximate(approx, dtemp); + + // main loop that discovers the quotient + #pragma nounroll + for(int32_t thread=numthreads-1;thread>=0;thread--) { + dlimbs_scatter(dtemp, x, TPI-1); + dlimbs_div_estimate(estimate, dtemp, approx); + dlimbs_all_gather(y, estimate); + + mpmul(plo, phi, y, denom); + c=mpsub(x, x, phi); + x2=__shfl_sync(sync, x[0], TPI-1, TPI); + + #pragma unroll + for(int32_t index=0;index(x, x, plo); + + x2=x2+resolve_sub(c, x); + x1=__shfl_sync(sync, x[LIMBS-1], TPI-1, TPI); + x0=__shfl_sync(sync, x[LIMBS-2], TPI-1, TPI); + + correction=ucorrect(x0, x1, x2, d0, d1); + if(correction!=0) { + c=mpmul32(plo, denom, correction); + t=resolve_add(c, plo); + c=mpadd(x, x, plo); + x2=x2+t+fast_propagate_add(c, x); + } + if(x2<0) { + // usually the case + c=mpadd(x, x, denom); + fast_propagate_add(c, x); + correction++; + } + if(group_thread==thread) + mpsub32(quotient, y, correction); + } + mpset(q, quotient); +} + +template +__device__ __forceinline__ void core_t::rem_wide(uint32_t r[LIMBS], const uint32_t lo[LIMBS], const uint32_t hi[LIMBS], const uint32_t denom[LIMBS], const uint32_t numthreads) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + int32_t x2; + uint32_t dtemp[DLIMBS], approx[DLIMBS], estimate[DLIMBS], t, c, x0, x1, d0, d1, correction; + uint32_t x[LIMBS], y[LIMBS], plo[LIMBS], phi[LIMBS]; + + if(numthreads chain; + #pragma unroll + for(int32_t index=0;index(x, y); + } + else + mpset(x, hi); + + d1=__shfl_sync(sync, denom[LIMBS-1], TPI-1, TPI); + d0=__shfl_sync(sync, denom[LIMBS-2], TPI-1, TPI); + + dlimbs_scatter(dtemp, denom, TPI-1); + dlimbs_approximate(approx, dtemp); + + // main loop that discovers the quotient + #pragma nounroll + for(int32_t thread=numthreads-1;thread>=0;thread--) { + dlimbs_scatter(dtemp, x, TPI-1); + dlimbs_div_estimate(estimate, dtemp, approx); + dlimbs_all_gather(y, estimate); + + mpmul(plo, phi, y, denom); + c=mpsub(x, x, phi); + x2=__shfl_sync(sync, x[0], TPI-1, TPI); + + #pragma unroll + for(int32_t index=0;index(x, x, plo); + + x2=x2+resolve_sub(c, x); + x1=__shfl_sync(sync, x[LIMBS-1], TPI-1, TPI); + x0=__shfl_sync(sync, x[LIMBS-2], TPI-1, TPI); + + correction=ucorrect(x0, x1, x2, d0, d1); + if(correction!=0) { + c=mpmul32(plo, denom, correction); + t=resolve_add(c, plo); + c=mpadd(x, x, plo); + x2=x2+t+fast_propagate_add(c, x); + } + if(x2<0) { + // usually the case + c=mpadd(x, x, denom); + fast_propagate_add(c, x); + } + } + mpset(r, x); +} + +template +__device__ __forceinline__ void core_t::div_rem_wide(uint32_t q[LIMBS], uint32_t r[LIMBS], const uint32_t lo[LIMBS], const uint32_t hi[LIMBS], const uint32_t denom[LIMBS], const uint32_t numthreads) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + int32_t x2; + uint32_t dtemp[DLIMBS], approx[DLIMBS], estimate[DLIMBS], t, c, x0, x1, d0, d1, correction; + uint32_t x[LIMBS], y[LIMBS], plo[LIMBS], phi[LIMBS], quotient[LIMBS]; + + mpzero(quotient); + if(numthreads chain; + #pragma unroll + for(int32_t index=0;index(x, y); + quotient[0]=(group_thread==numthreads) ? 1 : quotient[0]; + } + } + else + mpset(x, hi); + + d1=__shfl_sync(sync, denom[LIMBS-1], TPI-1, TPI); + d0=__shfl_sync(sync, denom[LIMBS-2], TPI-1, TPI); + + dlimbs_scatter(dtemp, denom, TPI-1); + dlimbs_approximate(approx, dtemp); + + // main loop that discovers the quotient + #pragma nounroll + for(int32_t thread=numthreads-1;thread>=0;thread--) { + dlimbs_scatter(dtemp, x, TPI-1); + dlimbs_div_estimate(estimate, dtemp, approx); + dlimbs_all_gather(y, estimate); + + mpmul(plo, phi, y, denom); + c=mpsub(x, x, phi); + x2=__shfl_sync(sync, x[0], TPI-1, TPI); + + #pragma unroll + for(int32_t index=0;index(x, x, plo); + + x2=x2+resolve_sub(c, x); + x1=__shfl_sync(sync, x[LIMBS-1], TPI-1, TPI); + x0=__shfl_sync(sync, x[LIMBS-2], TPI-1, TPI); + + correction=ucorrect(x0, x1, x2, d0, d1); + if(correction!=0) { + c=mpmul32(plo, denom, correction); + t=resolve_add(c, plo); + c=mpadd(x, x, plo); + x2=x2+t+fast_propagate_add(c, x); + } + if(x2<0) { + // usually the case + c=mpadd(x, x, denom); + fast_propagate_add(c, x); + correction++; + } + if(group_thread==thread) + mpsub32(quotient, y, correction); + } + mpset(q, quotient); + mpset(r, x); +} + +} /* namespace cgbn */ diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_divide_single.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_divide_single.cu new file mode 100644 index 0000000..20d2ba1 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_divide_single.cu @@ -0,0 +1,197 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +__device__ __forceinline__ void core_t::div_wide(uint32_t &q, const uint32_t lo, const uint32_t hi, const uint32_t denom, const uint32_t numthreads) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t x, d0, d1, x0, x1, x2, est, t, a, h, l, quotient; + int32_t c, top; + + quotient=0; + x=hi; + if(numthreads=0;thread--) { + x0=__shfl_sync(sync, x, TPI-3, TPI); + x1=__shfl_sync(sync, x, TPI-2, TPI); + x2=__shfl_sync(sync, x, TPI-1, TPI); + est=udiv(x0, x1, x2, d0, d1, a); + + t=__shfl_sync(sync, lo, thread, TPI); + l=madlo(est, denom, 0); + h=madhi(est, denom, 0); + + x=sub_cc(x, h); + c=subc(0, 0); // thread TPI-1 is zero + + top=__shfl_sync(sync, x, TPI-1, TPI); + x=__shfl_sync(sync, x, threadIdx.x-1, TPI); + c=__shfl_sync(sync, c, threadIdx.x-1, TPI); + x=(group_thread==0) ? t : x; + + x=sub_cc(x, l); + c=subc(c, 0); + + if(top+resolve_sub(c, x)<0) { + // means a correction is required, should be very rare + x=add_cc(x, denom); + c=addc(0, 0); + fast_propagate_add(c, x); + est--; + } + quotient=(group_thread==thread) ? est : quotient; + } + q=quotient; +} + +template +__device__ __forceinline__ void core_t::rem_wide(uint32_t &r, const uint32_t lo, const uint32_t hi, const uint32_t denom, const uint32_t numthreads) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t x, d0, d1, x0, x1, x2, est, t, a, h, l; + int32_t c, top; + + x=hi; + if(numthreads=0;thread--) { + x0=__shfl_sync(sync, x, TPI-3, TPI); + x1=__shfl_sync(sync, x, TPI-2, TPI); + x2=__shfl_sync(sync, x, TPI-1, TPI); + est=udiv(x0, x1, x2, d0, d1, a); + + t=__shfl_sync(sync, lo, thread, TPI); + l=madlo(est, denom, 0); + h=madhi(est, denom, 0); + + x=sub_cc(x, h); + c=subc(0, 0); // thread TPI-1 is zero + + top=__shfl_sync(sync, x, TPI-1, TPI); + x=__shfl_sync(sync, x, threadIdx.x-1, TPI); + c=__shfl_sync(sync, c, threadIdx.x-1, TPI); + x=(group_thread==0) ? t : x; + + x=sub_cc(x, l); + c=subc(c, 0); + + if(top+resolve_sub(c, x)<0) { + // means a correction is required, should be very rare + x=add_cc(x, denom); + c=addc(0, 0); + fast_propagate_add(c, x); + } + } + r=x; +} + +template +__device__ __forceinline__ void core_t::div_rem_wide(uint32_t &q, uint32_t &r, const uint32_t lo, const uint32_t hi, const uint32_t denom, const uint32_t numthreads) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t x, d0, d1, x0, x1, x2, est, t, a, h, l, quotient; + int32_t c, top; + + quotient=0; + x=hi; + if(numthreads=0;thread--) { + x0=__shfl_sync(sync, x, TPI-3, TPI); + x1=__shfl_sync(sync, x, TPI-2, TPI); + x2=__shfl_sync(sync, x, TPI-1, TPI); + est=udiv(x0, x1, x2, d0, d1, a); + + t=__shfl_sync(sync, lo, thread, TPI); + l=madlo(est, denom, 0); + h=madhi(est, denom, 0); + + x=sub_cc(x, h); + c=subc(0, 0); // thread TPI-1 is zero + + top=__shfl_sync(sync, x, TPI-1, TPI); + x=__shfl_sync(sync, x, threadIdx.x-1, TPI); + c=__shfl_sync(sync, c, threadIdx.x-1, TPI); + x=(group_thread==0) ? t : x; + + x=sub_cc(x, l); + c=subc(c, 0); + + if(top+resolve_sub(c, x)<0) { + // means a correction is required, should be very rare + x=add_cc(x, denom); + c=addc(0, 0); + fast_propagate_add(c, x); + est--; + } + quotient=(group_thread==thread) ? est : quotient; + } + q=quotient; + r=x; +} + +} /* namespace cgbn */ \ No newline at end of file diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_gcd.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_gcd.cu new file mode 100644 index 0000000..ab59f0e --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_gcd.cu @@ -0,0 +1,266 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +__device__ __forceinline__ static void gcd_reduce(signed_coeff_t &coeffs, int32_t a, int32_t b) { + int32_t c0, c1, t, q, i, n=b-a, count=15; + + coeffs.alpha_a=1; + coeffs.alpha_b=0; + coeffs.beta_a=-1; + coeffs.beta_b=1; + + // a and b must both be odd + c1=0; + while(1) { + t=n^n-1; + c0=31-ushiftamt(t); + count=count-c0; + if(count<0) { + coeffs.alpha_a=coeffs.alpha_a<>c0; + i=b*(b*b+62); // 6 bit inverse + if(t>0x3F) { + i=i*(i*b+2); // 12 bit inverse + i=i*(i*b+2); // 24 bit inverse + } + + q=a*i & t; + t=q & n; + q=q-t-t; + n=b*q+a>>c0; + + coeffs.alpha_a=(coeffs.alpha_a<>c1; + i=a*(a*a+62); // 6 bit inverse + if(t>0x3F) { + i=i*(i*a+2); // 12 bit inverse + i=i*(i*a+2); // 24 bit inverse + } + + q=b*i & t; + t=q & n; + q=q-t-t; + n=a*q+b>>c1; + + coeffs.beta_a=(coeffs.beta_a< +__device__ __forceinline__ void core_t::gcd_product(const uint32_t sync, uint32_t r[LIMBS], const int32_t sa, const uint32_t a[LIMBS], const int32_t sb, const uint32_t b[LIMBS]) { + uint32_t group_thread=threadIdx.x & TPI-1; + uint32_t c, t, ua, ub, a_high, b_high; + int32_t sign; + + a_high=__shfl_sync(sync, a[LIMBS-1], group_thread-1, TPI); + b_high=__shfl_sync(sync, b[LIMBS-1], group_thread-1, TPI); + + sign=sa ^ sb; + ua=uabs(sa); + ub=uabs(sb); + + if(sign>=0) { + #pragma unroll + for(int32_t index=0;index chain1; + r[0]=chain1.madhi(ua, a_high, r[0]); + #pragma unroll + for(int32_t index=1;index chain2; + #pragma unroll + for(int32_t index=0;index chain3; + r[0]=chain3.madhi(ub, b_high, r[0]); + #pragma unroll + for(int32_t index=1;index chain4; + r[0]=chain4.madhi(ua, a_high, r[0]); + #pragma unroll + for(int32_t index=1;index chain5; + chain5.sub(0, group_thread); + #pragma unroll + for(int32_t index=0;index chain6; + chain6.sub(0, group_thread); + t=madhi(ub, b_high, 0); + r[0]=chain6.sub(r[0], t); + #pragma unroll + for(int32_t index=1;index +__device__ __forceinline__ void core_t::gcd(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS]) { + uint32_t sync=sync_mask(); + uint32_t a_current[LIMBS], b_current[LIMBS], a_temp[LIMBS], b_temp[LIMBS]; + uint32_t t, a_low, a_high, b_low, b_high; + int32_t delta, count, gcd_count; + signed_coeff_t reducer; + + // NOTE: this algorithm is for unpadded values only + + gcd_count=ctz(a); + if(gcd_count==BITS) { + mpset(r, b); + return; + } + count=ctz(b); + if(count==BITS) { + mpset(r, a); + return; + } + shift_right(a_current, a, gcd_count); + shift_right(b_current, b, count); + gcd_count=umin(gcd_count, count); + + // done when top 26 bits of A and B are zero + while(true) { + a_high=__shfl_sync(sync, a_current[LIMBS-1], 31, TPI); + b_high=__shfl_sync(sync, b_current[LIMBS-1], 31, TPI); + if((a_high | b_high)<=0x3F) + break; + if(a_high>b_high) + delta=1; + else if(a_high(a_temp, a_current, b_current); + fast_propagate_sub(t, a_temp); + count=ctz(a_temp); + rotate_right(a_current, a_temp, count); + } + else if(delta==-1) { + t=mpsub(b_temp, b_current, a_current); + fast_propagate_sub(t, b_temp); + count=ctz(b_temp); + rotate_right(b_current, b_temp, count); + } + } + + // require that top 26 bits of A and B are 0 + while(true) { + a_low=__shfl_sync(sync, a_current[0], 0, TPI); + b_low=__shfl_sync(sync, b_current[0], 0, TPI); + + gcd_reduce(reducer, a_low, b_low); + gcd_product(sync, a_temp, reducer.alpha_a, a_current, reducer.alpha_b, b_current); + gcd_product(sync, b_temp, reducer.beta_a, a_current, reducer.beta_b, b_current); + + a_low=__shfl_sync(sync, a_temp[0], 0, TPI); + b_low=__shfl_sync(sync, b_temp[0], 0, TPI); + + if(a_low!=0) { + t=__shfl_sync(sync, a_temp[0], threadIdx.x+1, TPI); + mpright(a_current, a_temp, uctz(a_low), t); + } + else { + count=ctz(a_temp); + if(count==BITS) + break; + rotate_right(a_current, a_temp, count); + } + + if(b_low!=0) { + t=__shfl_sync(sync, b_temp[0], threadIdx.x+1, TPI); + mpright(b_current, b_temp, uctz(b_low), t); + } + else { + count=ctz(b_temp); + if(count==BITS) + break; + rotate_right(b_current, b_temp, count); + } + + if(reducer.beta_a==-1 && reducer.beta_b==1) + mpswap(a_current, b_current); + } + + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ uint32_t core_t::extract_bits_ui32(const uint32_t a[LIMBS], const uint32_t start, const uint32_t len) { + uint32_t sync=sync_mask(); + uint32_t offset, limb, lane, low, high, mask; + + if(start>=BITS) + return 0; + + offset=start & 0x1F; + limb=start>>5; + mask=uleft_clamp(0xFFFFFFFF, 0, len); + + if(LIMBS==1) { + low=__shfl_sync(sync, a[0], limb, TPI); + if(BITS==32) + high=0; + else if(offset+len<=32 || (int32_t)start>=(int32_t)(BITS-32)) + high=0; + else + high=__shfl_sync(sync, a[0], limb+1, TPI); + } + else { + lane=static_divide_small(limb); + limb=limb-lane*LIMBS; + + low=0; + high=0; + #pragma unroll + for(int index=0;index=(int32_t)(BITS-32)) + high=0; + else + high=__shfl_sync(sync, high, lane+(limb==LIMBS-1), TPI); + } + return uright_wrap(low, high, offset) & mask; +} + +template +__device__ __forceinline__ void core_t::insert_bits_ui32(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t start, const uint32_t len, const uint32_t value) { + int32_t group_thread=threadIdx.x & TPI-1; + uint32_t offset, limb, mask, data, shifted_mask_l, shifted_data_l, shifted_mask_h, shifted_data_h; + + if(PADDING!=0) + if(start>=BITS) + return; + + offset=start & 0x1F; + limb=start>>5; + mask=uleft_clamp(0xFFFFFFFF, 0, len); + data=value & mask; + + if(LIMBS==1) { + r[0]=a[0]; + if(group_thread==limb) { + shifted_mask_l=~(mask<32 && group_thread==limb+1) { + shifted_mask_h=~uleft_wrap(mask, 0, offset); + shifted_data_h=uleft_wrap(data, 0, offset); + r[0] = (r[0] & shifted_mask_h) | shifted_data_h; + } + } + else { + limb=limb-group_thread*LIMBS; + + shifted_mask_l=~(mask< +__device__ __forceinline__ void core_t::bitwise_and(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS]) { + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ void core_t::bitwise_ior(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS]) { + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ void core_t::bitwise_xor(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS]) { + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ void core_t::bitwise_complement(uint32_t r[LIMBS], const uint32_t a[LIMBS]) { + if(PADDING==0) { + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ void core_t::bitwise_select(uint32_t r[LIMBS], const uint32_t clear[LIMBS], const uint32_t set[LIMBS], const uint32_t select[LIMBS]) { + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ uint32_t top32(const uint32_t sync, uint32_t x[limbs]) { + return __shfl_sync(sync, x[limbs-1], tpi-1, tpi); +} + +template +__device__ __forceinline__ uint64_t top64(const uint32_t sync, uint32_t x[limbs]) { + uint32_t lo, hi; + + if(limbs==1) + lo=__shfl_sync(sync, x[limbs-1], tpi-2, tpi); + else + lo=__shfl_sync(sync, x[limbs-2], tpi-1, tpi); + hi=__shfl_sync(sync, x[limbs-1], tpi-1, tpi); + return make_wide(lo, hi); +} + +template +__device__ __forceinline__ void core_t::modinv_update_uw_qs(const uint32_t sync, uint32_t r[LIMBS], const uint32_t q, const int32_t s, const uint32_t x[LIMBS]) { + uint32_t group_thread=threadIdx.x & TPI-1; + uint32_t temp[LIMBS], lo, hi, top, c; + + top=madhi(q, x[LIMBS-1], 0); + hi=__shfl_sync(sync, top, threadIdx.x-1, TPI); + if(group_thread==0) + hi=0; + + #pragma unroll + for(int32_t index=0;index(r, r, temp); + fast_propagate_add(c, r); +} + +template +__device__ __forceinline__ void core_t::modinv_update_ab_sq(const uint32_t sync, uint32_t r[LIMBS], const int32_t shift, const uint32_t q, const uint32_t x[LIMBS]) { + uint32_t group_thread=threadIdx.x & TPI-1; + uint32_t temp[LIMBS], lo, hi, t, c; + + t=__shfl_sync(sync, x[0], threadIdx.x+1, TPI); + mpright(temp, x, shift, t); + + c=madhi(q, temp[LIMBS-1], 0); + hi=__shfl_sync(sync, c, threadIdx.x-1, TPI); + if(group_thread==0) + hi=0; + + #pragma unroll + for(int32_t index=0;index(r, r, temp); + fast_propagate_sub(c, r); +} + +template +__device__ __forceinline__ void core_t::modinv_update_uw(uint32_t sync, uint32_t u[LIMBS], uint32_t w[LIMBS], const unsigned_coeff_t &coeffs) { + uint32_t group_thread=threadIdx.x & TPI-1; + uint32_t temp1[LIMBS], temp2[LIMBS], hi, lo, c, top; + + // u * alpha_a + top=madhi(coeffs.alpha_a, u[LIMBS-1], 0); + hi=__shfl_sync(sync, top, threadIdx.x-1, TPI); + if(group_thread==0) + hi=0; + + #pragma unroll + for(int32_t index=0;index(temp1, temp1, temp2); + resolve_add(c, temp1); + + // u * beta_a + top=madhi(coeffs.beta_a, u[LIMBS-1], 0); + hi=__shfl_sync(sync, top, threadIdx.x-1, TPI); + if(group_thread==0) + hi=0; + + #pragma unroll + for(int32_t index=0;index(w, u, w); + resolve_add(c, w); + + mpset(u, temp1); +} + +template +__device__ __forceinline__ bool core_t::modinv_small_delta(const uint32_t sync, uint32_t a[LIMBS], uint32_t b[LIMBS], uint32_t u[LIMBS], uint32_t w[LIMBS], int32_t &delta) { + uint32_t group_thread=threadIdx.x & TPI-1; + uint32_t t; + int32_t c, cmp; + unsigned_coeff_t coeffs; + + while(true) { + coeffs.alpha_a=1; coeffs.alpha_b=0; coeffs.beta_a=0; coeffs.beta_b=1; + while(true) { + cmp=ucmp(a[LIMBS-1], b[LIMBS-1]); + cmp=__shfl_sync(sync, cmp, TPI-1, TPI); + if(cmp==0) { + cmp=compare(a, b); + if(cmp==0) { + if(delta>0) { + coeffs.alpha_a=coeffs.alpha_a + ((1<0 || (delta==0 && cmp==1)) { + if(cmp==-1) { + c=mpadd(a, a, a); + c=__shfl_sync(sync, c, threadIdx.x-1, TPI); + if(group_thread!=0) + a[0]=a[0]+c; + delta--; + } + coeffs.alpha_a=coeffs.alpha_a + (coeffs.beta_a<(a, a, b); + fast_propagate_sub(c, a); + t=__shfl_sync(sync, a[LIMBS-1], TPI-1, TPI); + if(t!=0) { + c=ushiftamt(t); + delta-=c; + t=__shfl_sync(sync, a[LIMBS-1], threadIdx.x-1, TPI); + mpleft(a, a, c, t); + } + else { + c=clz(a); + delta-=c; + rotate_left(a, a, c); + } + } + else { + if(cmp==1) { + c=mpadd(b, b, b); + c=__shfl_sync(sync, c, threadIdx.x-1, TPI); + if(group_thread!=0) + b[0]=b[0]+c; + delta++; + } + coeffs.beta_a=coeffs.beta_a + (coeffs.alpha_a<<-delta); + coeffs.beta_b=coeffs.beta_b + (coeffs.alpha_b<<-delta); + c=mpsub(b, b, a); + fast_propagate_sub(c, b); + t=__shfl_sync(sync, b[LIMBS-1], TPI-1, TPI); + if(t!=0) { + c=ushiftamt(t); + delta+=c; + t=__shfl_sync(sync, b[LIMBS-1], threadIdx.x-1, TPI); + mpleft(b, b, c, t); + } + else { + c=clz(b); + delta+=c; + rotate_left(b, b, c); + } + } + if(uabs(delta)>=8) { + // faster to do a divide + break; + } + if((coeffs.alpha_a | coeffs.alpha_b | coeffs.beta_a | coeffs.beta_b)>0xFFFFFF) { + // poor mans max + break; + } + } + + // we have accumulated a bunch of operations, apply them to u and w + modinv_update_uw(sync, u, w, coeffs); + if(cmp==0) + return true; + if(uabs(delta)>=8) + return false; + } +} + +template +__device__ __forceinline__ bool core_t::modular_inverse(uint32_t inv[LIMBS], const uint32_t x[LIMBS], const uint32_t y[LIMBS]) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1, warp_thread=threadIdx.x & warpSize-1; + uint32_t a[LIMBS], b[LIMBS], u[LIMBS], w[LIMBS], d_top, q, ballot; + uint64_t n_top; + int32_t delta, ad, c; + + if(__shfl_sync(sync, (x[0] | y[0]) & 0x01, 0, TPI)==0) { + mpzero(inv); + return false; + } + + mpset(a, x); + mpset(b, y); + mpzero(u); + mpzero(w); + if(group_thread==0) + u[0]=1; + + c=clz(a); + delta=clz(b); + if(c==BITS || delta>=BITS-1) { + mpzero(inv); + return false; + } + rotate_left(a, a, c); + rotate_left(b, b, delta); + delta=delta-c; + + while(true) { + ad=uabs(delta); +/* +if(blockIdx.x==0 && threadIdx.x==0) printf("delta=%d\n", ad); +if(blockIdx.x==0 && threadIdx.x<32) { + for(int32_t index=31;index>=0;index--) { + uint32_t tt=__shfl_sync(sync, a[0], index, TPI); + if(threadIdx.x==0) printf("%08X", tt); + } + if(threadIdx.x==0) printf("\n"); +} +if(blockIdx.x==0 && threadIdx.x<32) { + for(int32_t index=31;index>=0;index--) { + uint32_t tt=__shfl_sync(sync, b[0], index, TPI); + if(threadIdx.x==0) printf("%08X", tt); + } + if(threadIdx.x==0) printf("\n"); +} +*/ + if(ad<8) { + if(modinv_small_delta(sync, a, b, u, w, delta)) + break; + } + else { + if(delta>0) { + n_top=top64(sync, a); + d_top=top32(sync, b); + } + else { + n_top=top64(sync, b); + d_top=top32(sync, a); + } + c=32; + if(ad<32) { + c=ad; + n_top=n_top>>32-c; + } + else if(uhigh(n_top)>=d_top) { + n_top=n_top>>1; + c--; + } + q=n_top/d_top-2; +//if(blockIdx.x==0 && threadIdx.x==0) printf("%016lX %08X q=%08X\n", n_top, d_top, q); + + if(delta>0) { + modinv_update_uw_qs(sync, u, q, delta-c, w); + modinv_update_ab_sq(sync, a, c, q, b); + c=clz(a); + if(c==BITS) { + mpset(a, b); + break; + } + rotate_left(a, a, c); + delta-=c; + } + else { + modinv_update_uw_qs(sync, w, q, -delta-c, u); + modinv_update_ab_sq(sync, b, c, q, a); + c=clz(b); + if(c==BITS) + break; + rotate_left(b, b, c); + delta+=c; + } + } + } + + if(group_thread==TPI-1) + a[LIMBS-1]=a[LIMBS-1] ^ 0x80000000; + ballot=__ballot_sync(sync, mplor(a)==0); + if(TPI>(warp_thread^group_thread)) & TPI_ONES; + if(ballot!=TPI_ONES) { + // gcd is not 1 + mpzero(inv); + return false; + } + + mpset(inv, u); + return true; +} + + + +/* +if(blockIdx.x==0 && threadIdx.x<32) { + for(int32_t index=31;index>=0;index--) { + uint32_t tt=__shfl_sync(sync, u[0], index, TPI); + if(threadIdx.x==0) printf("%08X", tt); + } + if(threadIdx.x==0) printf("\n"); +} +if(blockIdx.x==0 && threadIdx.x<32) { + for(int32_t index=31;index>=0;index--) { + uint32_t tt=__shfl_sync(sync, w[0], index, TPI); + if(threadIdx.x==0) printf("%08X", tt); + } + if(threadIdx.x==0) printf("\n"); +} +*/ + +/* +if(blockIdx.x==0 && threadIdx.x>=32 && threadIdx.x<64) { + for(int32_t index=31;index>=0;index--) { + uint32_t tt=__shfl_sync(sync, u[0], index, TPI); + if(group_thread==0) printf("%08X", tt); + } + if(threadIdx.x==0) printf("\n"); +} +*/ + + +} /* namespace cgbn */ \ No newline at end of file diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mont.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mont.cu new file mode 100644 index 0000000..6f9d753 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mont.cu @@ -0,0 +1,117 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +/* single limb version */ +template +__device__ __forceinline__ void core_t::mont_mul(uint32_t &r, const uint32_t a, const uint32_t b, const uint32_t n, const uint32_t np0) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t r0=0, r1=0, r2, t, c; + + #pragma unroll + for(int32_t thread=0;thread +__device__ __forceinline__ void core_t::mont_reduce_wide(uint32_t &r, const uint32_t lo, const uint32_t hi, const uint32_t n, const uint32_t np0, const bool zero) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t r0=lo, r1=0, t, top; + + #pragma unroll + for(int32_t thread=0;thread +__device__ __forceinline__ void core_t::mont_mul(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS], const uint32_t n[LIMBS], const uint32_t np0) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t x[LIMBS], x1=0, x2, t, q, c; + + mpzero(x); + + #pragma nounroll + for(int32_t thread=0;thread chain1; + #pragma unroll + for(int32_t index=0;index chain2; + for(int32_t index=0;index chain3; + #pragma unroll + for(int32_t index=0;index chain4; + for(int32_t index=0;index chain5; + r[0]=chain5.add(x[0], t); + #pragma unroll + for(int32_t index=1;index chain6; + r[0]=chain6.add(r[0], ~t & c); + #pragma unroll + for(int32_t index=1;index +__device__ __forceinline__ void core_t::mont_reduce_wide(uint32_t r[LIMBS], const uint32_t lo[LIMBS], const uint32_t hi[LIMBS], const uint32_t n[LIMBS], const uint32_t np0, const bool zero) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t c, x[LIMBS], t, q, top; + + mpset(x, lo); + + c=0; + #pragma nounroll + for(int32_t thread=0;thread chain1; + #pragma unroll + for(int32_t index=0;index chain2; + for(int32_t index=0;index chain3; + x[0]=chain3.add(x[0], ~t); + #pragma unroll + for(int32_t index=1;index(r, x); +} + +} /* namespace cgbn */ diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mont_wmad.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mont_wmad.cu new file mode 100644 index 0000000..07c7aef --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mont_wmad.cu @@ -0,0 +1,466 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +#if 1 +template +__device__ __forceinline__ void core_t::mont_mul(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS], const uint32_t n[LIMBS], const uint32_t np0) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t t, t0, t1, q, r1, ra[LIMBS+2], ru[LIMBS+1], c=0; + + #pragma unroll + for(int32_t index=0;index<=LIMBS;index++) { + ra[index]=0; + ru[index]=0; + } + ra[LIMBS+1]=0; + + #pragma nounroll + for(int32_t thread=0;thread chain1; // unaligned: T0 * A_odd + ra[0]=chain1.add(ra[0], c); + #pragma unroll + for(int32_t index=0;index chain2; // aligned: T0 * A_even + #pragma unroll + for(int32_t index=0;index chain3; // aligned: Q0 * N_even + q=__shfl_sync(sync, ra[0], 0, TPI)*np0; + #pragma unroll + for(int32_t index=0;index chain4; // unaligned: Q0 * N_odd + #pragma unroll + for(int32_t index=0;index chain5; // unaigned: T1 * A_even + #pragma unroll + for(int32_t index=0;index chain6; // aligned: T1 * A_odd + #pragma unroll + for(int32_t index=0;index chain7; // aligned: Q1 * N_odd + ru[0]=chain7.add(ru[0], ra[1]); + q=__shfl_sync(sync, ru[0], 0, TPI)*np0; + #pragma unroll + for(int32_t index=0;index<(int32_t)LIMBS-3;index+=2) { + ra[index]=chain7.madlo(n[index+1], q, ra[index+2]); + ra[index+1]=chain7.madhi(n[index+1], q, ra[index+3]); + } + ra[LIMBS-2-LIMBS%2]=chain3.madlo(n[LIMBS-1-LIMBS%2], q, ra[LIMBS-LIMBS%2]); + ra[LIMBS-1-LIMBS%2]=chain3.madhi(n[LIMBS-1-LIMBS%2], q, ra[LIMBS+1-LIMBS%2]); + if(LIMBS%2==1) { + ra[LIMBS-1]=chain3.add(ra[LIMBS+1], 0); + ra[LIMBS]=0; + } + else + ra[LIMBS-LIMBS%2]=chain3.add(0, 0); + + chain_t<> chain8; // unaigned: Q1 * N_even + t1=chain8.madlo(n[0], q, ru[0]); + c=chain8.madhi(n[0], q, ru[1]); + #pragma unroll + for(int32_t index=0;index chain9; + r[0]=chain9.add(ra[0], c); + #pragma unroll + for(int32_t index=1;index chain10; + r[0]=chain10.add(r[0], t); + #pragma unroll + for(int32_t index=1;index chain11; + r[0]=chain11.add(r[0], ~t & c); + #pragma unroll + for(int32_t index=1;index +__device__ __forceinline__ void core_t::mont_reduce_wide(uint32_t r[LIMBS], const uint32_t lo[LIMBS], const uint32_t hi[LIMBS], const uint32_t n[LIMBS], const uint32_t np0, const bool zero) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t t0, t1, q, ra[LIMBS+2], ru[LIMBS+1], c=0, top; + + #pragma unroll + for(int32_t index=0;index chain1; // unaligned: Q0 * N_odd + ra[0]=chain1.add(ra[0], c); + q=__shfl_sync(sync, ra[0], 0, TPI)*np0; + #pragma unroll + for(int32_t index=0;index chain2; // aligned: Q0 * N_even + #pragma unroll + for(int32_t index=0;index chain3; // aligned: Q1 * N_odd + ru[0]=chain3.add(ru[0], ra[1]); + q=__shfl_sync(sync, ru[0], 0, TPI)*np0; + #pragma unroll + for(int32_t index=0;index<(int32_t)LIMBS-3;index+=2) { + ra[index]=chain3.madlo(n[index+1], q, ra[index+2]); + ra[index+1]=chain3.madhi(n[index+1], q, ra[index+3]); + } + ra[LIMBS-2-LIMBS%2]=chain3.madlo(n[LIMBS-1-LIMBS%2], q, ra[LIMBS-LIMBS%2]); + ra[LIMBS-1-LIMBS%2]=chain3.madhi(n[LIMBS-1-LIMBS%2], q, ra[LIMBS+1-LIMBS%2]); + if(LIMBS%2==1) + ra[LIMBS-1]=chain3.add(0, 0); + + chain_t<> chain4; // unaligned: Q1 * N_even + t1=chain4.madlo(n[0], q, ru[0]); + c=chain4.madhi(n[0], q, ru[1]); + #pragma unroll + for(int32_t index=0;index chain5; + ra[0]=chain5.add(ra[0], c); + #pragma unroll + for(int32_t index=1;index chain3; + ra[0]=chain3.add(ra[0], ~t0); + #pragma unroll + for(int32_t index=1;index(r, ra); +} + +#if 0 +template +__device__ __forceinline__ void fwmont_mul(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS], const uint32_t n[LIMBS], const uint32_t np0) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t t, t0, t1, q, r1, ra[LIMBS+2], ru[LIMBS+1], c=0; + + #pragma unroll + for(int32_t index=0;index<=LIMBS;index++) { + ra[index]=0; + ru[index]=0; + } + ra[LIMBS+1]=0; + + #pragma nounroll + for(int32_t thread=0;thread chain1; // aligned: T0 * A_even + #pragma unroll + for(int32_t index=0;index chain2; // unaligned: T0 * A_odd + ra[0]=chain2.add(ra[0], c); + #pragma unroll + for(int32_t index=0;index chain3; // aligned: Q0 * N_even + #pragma unroll + for(int32_t index=0;index chain4; // unaligned: Q0 * N_odd + #pragma unroll + for(int32_t index=0;index chain5; // unaigned: T1 * A_even + #pragma unroll + for(int32_t index=0;index chain6; // aligned: T1 * A_odd + ru[0]=chain6.add(ru[0], ra[1]); + #pragma unroll + for(int32_t index=0;index chain7; // unaigned: Q1 * N_even + t=chain7.madlo(n[0], q, ru[0]); + c=chain7.madhi(n[0], q, ru[1]); + #pragma unroll + for(int32_t index=0;index chain8; // aligned: Q1 * N_odd + #pragma unroll + for(int32_t index=0;index chain9; + r[0]=chain9.add(ra[0], c); + #pragma unroll + for(int32_t index=1;index chain10; + r[0]=chain10.add(r[0], t); + #pragma unroll + for(int32_t index=1;index chain11; + r[0]=chain11.add(r[0], ~t & c); + #pragma unroll + for(int32_t index=1;index +__device__ __forceinline__ void core_t::mont_mul(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS], const uint32_t n[LIMBS], const uint32_t np0) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t u1, ru[LIMBS], r1=0, ra[LIMBS], c=0, t, q; + uint64_t sum; + + mpzero(ra); + mpzero(ru); + + #pragma nounroll + for(int32_t thread=0;thread chain1; + #pragma unroll + for(int32_t index=0;index chain2; + #pragma unroll + for(int32_t index=0;index chain3; + #pragma unroll + for(int32_t index=0;index chain4; + #pragma unroll + for(int32_t index=0;index>16; + c=addc(c, t); + + q=__shfl_sync(sync, ra[0], 0, TPI)*np0; + + // skip u[0] + chain_t chain5; + #pragma unroll + for(int32_t index=1;index chain6; + #pragma unroll + for(int32_t index=1;index chain7; + #pragma unroll + for(int32_t index=1;index chain8; // should be limbs-1 + #pragma unroll + for(int32_t index=1;index>32; + } + } + + #pragma unroll + for(int32_t index=LIMBS-1;index>0;index--) + ru[index]=__byte_perm(ru[index-1], ru[index], 0x5432); + ru[0]=__byte_perm(0, ru[0], 0x5432); + + chain_t chain10; + for(int32_t index=0;index chain11; + r[0]=chain11.add(r[0], t); + r[1]=chain11.add(r[1], c); + #pragma unroll + for(int32_t index=2;index chain12; + r[0]=chain12.add(r[0], ~t & c); + #pragma unroll + for(int32_t index=1;index +__device__ __forceinline__ void core_t::mont_reduce_wide(uint32_t r[LIMBS], const uint32_t lo[LIMBS], const uint32_t hi[LIMBS], const uint32_t n[LIMBS], const uint32_t np0, const bool zero) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t t0, t1, t, q, carry0, carry1, ru[LIMBS], ra[LIMBS], top; + uint64_t sum; + + #pragma unroll + for(int32_t index=0;index chain1; + #pragma unroll + for(int32_t index=0;index chain2; + #pragma unroll + for(int32_t index=0;index chain3; + t1=chain3.xmadlh(n[0], q, ru[0]); + #pragma unroll + for(int32_t index=1;index chain4; + t0=ra[0]; + #pragma unroll + for(int32_t index=0;index>16; + carry0=addc(carry0, t); + + // shift right by 32 bits (top thread gets zero) + t0=__shfl_sync(sync, t0, threadIdx.x+1, TPI); + if(!zero) { + top=__shfl_sync(sync, hi[word], thread, TPI); + t0=(group_thread==TPI-1) ? top : t0; + } + + sum=((uint64_t)ra[LIMBS-1]) + ((uint64_t)t0) + ((uint64_t)carry1); + ra[LIMBS-1]=sum; + carry1=sum>>32; + } + } + + #pragma unroll + for(int32_t index=LIMBS-1;index>=1;index--) + ru[index]=uleft_wrap(ru[index-1], ru[index], 16); + ru[0]=ru[0]<<16; + + mpadd32(ru, ru, carry0); + + chain_t chain5; + #pragma unroll + for(int32_t index=0;index chain6; + ra[0]=chain6.add(ra[0], ~t); + #pragma unroll + for(int32_t index=1;index(r, ra); +} + + +/****************************************************************************************************] + * FIX FIX FIX - figure out why this code doesn't work + * + * This code doesn't work, but it's more elegant than the other implementation. + * Keep it around for now. Note -- Q values are correct, wrap arounds are 0 + ****************************************************************************************************/ +#if 0 + +template +__device__ __forceinline__ void core_t::mont_mul(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS], const uint32_t n[LIMBS], const uint32_t np0) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t u1, ru[LIMBS], r1=0, ra[LIMBS], c=0, t, q, t0, t1; + uint64_t sum; + + #pragma unroll + for(int32_t index=0;index chain1; + #pragma unroll + for(int32_t index=0;index chain2; + #pragma unroll + for(int32_t index=0;index chain3; + #pragma unroll + for(int32_t index=0;index chain4; + #pragma unroll + for(int32_t index=0;index chain01; + #pragma unroll + for(int32_t index=0;index chain02; + #pragma unroll + for(int32_t index=0;index chain03; + t1=chain03.xmadlh(n[0], q, ru[0]); + #pragma unroll + for(int32_t index=1;index>16; + c=addc(c, t); + + chain_t<> chain04; + #pragma unroll + for(int32_t index=0;index>32; + } + } + + #pragma unroll + for(int32_t index=LIMBS-1;index>0;index--) + ru[index]=__byte_perm(ru[index-1], ru[index], 0x5432); + ru[0]=__byte_perm(0, ru[0], 0x5432); + + mpadd32(ru, ru, c); + + chain_t chain10; + for(int32_t index=0;index chain11; + r[0]=chain11.add(r[0], t); + r[1]=chain11.add(r[1], c); + #pragma unroll + for(int32_t index=2;index chain12; + r[0]=chain12.add(r[0], ~t & c); + #pragma unroll + for(int32_t index=1;index +__device__ __forceinline__ void core_t::mul(uint32_t &r, const uint32_t a, const uint32_t b, const uint32_t add) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t rl, p0=add, p1=0, t; + int32_t threads=(PADDING!=0) ? PADDING : TPI; + + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ void core_t::mul_wide(uint32_t &lo, uint32_t &hi, const uint32_t a, const uint32_t b, const uint32_t add) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t p0=add, p1=0, rl, t; + int32_t threads=(PADDING!=0) ? PADDING : TPI; + + if(PADDING!=0) + rl=0; + + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ void core_t::mul(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS], const uint32_t add[LIMBS]) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t t, rh[LIMBS+1], rl[LIMBS]; + int32_t threads=(PADDING!=0) ? (BITS/32+LIMBS-1)/LIMBS : TPI; + + if(PADDING!=0) + mpzero(rl); + mpset(rh, add); + rh[LIMBS]=0; + + #pragma nounroll + for(int32_t row=0;row chain1; + #pragma unroll + for(int index=0;index chain2; + #pragma unroll + for(int index=0;index(r, rl); + else { + #pragma unroll + for(int index=0;index +__device__ __forceinline__ void core_t::mul_wide(uint32_t lo[LIMBS], uint32_t hi[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS], const uint32_t add[LIMBS]) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t carry=0, low, t, rl[LIMBS], rh[LIMBS]; + int32_t threads=(PADDING!=0) ? (BITS/32)/LIMBS : TPI; + + if(PADDING!=0) + mpzero(rl); + + mpset(rh, add); + + #pragma nounroll + for(int32_t r=0;r chain1; + #pragma unroll + for(int32_t index=0;index chain2; + #pragma unroll + for(int32_t index=0;index chain3; + #pragma unroll + for(int32_t index=0;index chain4; + #pragma unroll + for(int32_t index=0;index(lo, rl); + mpset(hi, rh); +} + +} /* namespace cgbn */ \ No newline at end of file diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mul_wmad.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mul_wmad.cu new file mode 100644 index 0000000..594a457 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mul_wmad.cu @@ -0,0 +1,322 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +__device__ __forceinline__ void core_t::mul(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS], const uint32_t add[LIMBS]) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t t0, t1, term0, term1, carry, rl[LIMBS], ra[LIMBS+2], ru[LIMBS+1]; + int32_t threads=(PADDING!=0) ? (BITS/32+LIMBS-1)/LIMBS : TPI; + + if(PADDING!=0) + mpzero(rl); + mpset(ra, add); + ra[LIMBS]=0; + ra[LIMBS+1]=0; + mpzero(ru); + ru[LIMBS]=0; + + carry=0; + #pragma nounroll + for(int32_t row=0;row chain1; // aligned: T0 * A_even + #pragma unroll + for(int32_t index=0;index chain2; // unaligned: T0 * A_odd + t0=chain2.add(ra[0], carry); + #pragma unroll + for(int32_t index=0;index chain3; // unaligned: T1 * A_even + t1=chain3.madlo(a[0], term1, ru[0]); + carry=chain3.madhi(a[0], term1, ru[1]); + #pragma unroll + for(int32_t index=0;index chain4; // aligned: T1 * A_odd + t1=chain4.add(t1, ra[1]); + #pragma unroll + for(int32_t index=0;index<(int32_t)(LIMBS-3);index+=2) { + ra[index]=chain4.madlo(a[index+1], term1, ra[index+2]); + ra[index+1]=chain4.madhi(a[index+1], term1, ra[index+3]); + } + ra[LIMBS-2-LIMBS%2]=chain4.madlo(a[LIMBS-1-LIMBS%2], term1, ra[LIMBS-LIMBS%2]); + ra[LIMBS-1-LIMBS%2]=chain4.madhi(a[LIMBS-1-LIMBS%2], term1, ra[LIMBS+1-LIMBS%2]); + if(LIMBS%2==1) + ra[LIMBS-1]=chain4.add(0, 0); + + if(l(r, rl); + else { + #pragma unroll + for(int index=0;index +__device__ __forceinline__ void core_t::mul_wide(uint32_t lo[LIMBS], uint32_t hi[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS], const uint32_t add[LIMBS]) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t t, t0, t1, term0, term1, carry, rl[LIMBS], ra[LIMBS+2], ru[LIMBS+1]; + int32_t threads=(PADDING!=0) ? (BITS/32)/LIMBS & 0xFFFE : TPI; + + if(PADDING!=0) + mpzero(rl); + mpset(ra, add); + ra[LIMBS]=0; + ra[LIMBS+1]=0; + mpzero(ru); + ru[LIMBS]=0; + + carry=0; + #pragma nounroll + for(int32_t r=0;r chain1; // aligned: T0 * A_even + #pragma unroll + for(int32_t index=0;index chain2; // unaligned: T0 * A_odd + t0=chain2.add(ra[0], carry); + #pragma unroll + for(int32_t index=0;index chain3; // unaligned: T1 * A_even + t1=chain3.madlo(a[0], term1, ru[0]); + carry=chain3.madhi(a[0], term1, ru[1]); + #pragma unroll + for(int32_t index=0;index chain4; // aligned: T1 * A_odd + t1=chain4.add(t1, ra[1]); + #pragma unroll + for(int32_t index=0;index<(int32_t)(LIMBS-3);index+=2) { + ra[index]=chain4.madlo(a[index+1], term1, ra[index+2]); + ra[index+1]=chain4.madhi(a[index+1], term1, ra[index+3]); + } + ra[LIMBS-2-LIMBS%2]=chain4.madlo(a[LIMBS-1-LIMBS%2], term1, ra[LIMBS-LIMBS%2]); + ra[LIMBS-1-LIMBS%2]=chain4.madhi(a[LIMBS-1-LIMBS%2], term1, ra[LIMBS+1-LIMBS%2]); + if(LIMBS%2==1) + ra[LIMBS-1]=chain4.add(0, 0); + + if(l chainXX; + ra[0]=chainXX.add(ra[0], carry); + #pragma unroll + for(int32_t index=1;index=threads*LIMBS+LIMBS) { + #pragma unroll + for(int32_t l=0;l chain3; + #pragma unroll + for(int32_t index=0;index chain4; + #pragma unroll + for(int32_t index=0;index=threads*LIMBS+LIMBS); + + #pragma unroll + for(int32_t l=0;l<(BITS/32)%LIMBS;l++) { + t=__shfl_sync(sync, b[l], r, TPI); + + chain_t<> chain3; + #pragma unroll + for(int32_t index=0;index chain4; + #pragma unroll + for(int32_t index=0;index(lo, rl); + mpset(hi, ra); + + fast_propagate_add(carry, hi); +} + +} /* namespace cgbn */ \ No newline at end of file diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mul_xmad.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mul_xmad.cu new file mode 100644 index 0000000..796bb56 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_mul_xmad.cu @@ -0,0 +1,228 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +__device__ __forceinline__ void core_t::mul(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS], const uint32_t add[LIMBS]) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t t0, t1, t, carry0, carry1, ru[LIMBS], ra[LIMBS], rl[LIMBS]; + uint64_t sum; + int32_t threads=(PADDING!=0) ? (BITS/32+LIMBS-1)/LIMBS : TPI; + + if(PADDING!=0) + mpzero(rl); + mpset(ra, add); + mpzero(ru); + + carry0=0; + carry1=0; + #pragma nounroll + for(int32_t row=0;row chain1; + for(int index=0;index chain2; + for(int index=0;index chain3; + t0=chain3.xmadlh(a[0], t, ru[0]); + for(int32_t index=1;index>16, carry0); + + if(group_thread chain4; + for(int index=0;index>32; + } + } + + if(PADDING==0) + mpset(r, rl); + else { + #pragma unroll + for(int index=0;index +__device__ __forceinline__ void core_t::mul_wide(uint32_t lo[LIMBS], uint32_t hi[LIMBS], const uint32_t a[LIMBS], const uint32_t b[LIMBS], const uint32_t add[LIMBS]) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t t0, t1, term, carry0, carry1, rl[LIMBS], ra[LIMBS], ru[LIMBS]; + uint64_t sum; + int32_t threads=(PADDING!=0) ? (BITS/32)/LIMBS : TPI; + + if(PADDING!=0) + mpzero(rl); + + mpset(ra, add); + mpzero(ru); + + carry0=0; + carry1=0; + #pragma nounroll + for(int32_t r=0;r chain1; + for(int32_t index=0;index chain2; + for(int32_t index=0;index chain3; + t0=chain3.xmadlh(a[0], term, ru[0]); + for(int32_t index=1;index>16, carry0); + + t0=__shfl_sync(sync, t1, 0, TPI); + if(group_thread==r) + rl[l]=t0; + t1=__shfl_down_sync(sync, t1, 1, TPI); + t1=(group_thread==TPI-1) ? 0 : t1; + + chain_t<> chain4; + #pragma unroll + for(int32_t index=0;index>32; + } + } + + if(BITS/32!=threads*LIMBS) { + #pragma unroll + for(int32_t l=0;l chain5; + for(int32_t index=0;index chain6; + for(int32_t index=0;index chain7; + t0=chain7.xmadlh(a[0], term, ru[0]); + for(int32_t index=1;index>16, carry0); + + t0=__shfl_sync(sync, t1, 0, TPI); + if(group_thread==threads) + rl[l]=t0; + t1=__shfl_down_sync(sync, t1, 1, TPI); + t1=(group_thread==TPI-1) ? 0 : t1; + + chain_t<> chain8; + #pragma unroll + for(int32_t index=0;index>32; + } + } + + #pragma unroll + for(int32_t index=LIMBS-1;index>=1;index--) + ru[index]=uleft_wrap(ru[index-1], ru[index], 16); + ru[0]=ru[0]<<16; + + mpadd32(ru, ru, carry0); + + chain_t<> chain9; + #pragma unroll + for(int32_t index=0;index(lo, rl); + mpset(hi, ra); +} + +} /* namespace cgbn */ \ No newline at end of file diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_short_math.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_short_math.cu new file mode 100644 index 0000000..945ea13 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_short_math.cu @@ -0,0 +1,294 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +__device__ __forceinline__ uint32_t core_t::get_ui32(const uint32_t a[LIMBS]) { + uint32_t sync=sync_mask(); + + return __shfl_sync(sync, a[0], 0, TPI); +} + +template +__device__ __forceinline__ void core_t::set_ui32(uint32_t r[LIMBS], const uint32_t value) { + uint32_t group_thread=threadIdx.x & TPI-1; + + r[0]=(group_thread==0) ? value : 0; + #pragma unroll + for(int32_t index=1;index +__device__ __forceinline__ int32_t core_t::add_ui32(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t add) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1, carry; + int32_t result; + + // FIX FIX FIX -- use custom algorithm for each TPI/PADDING + + carry=mpadd32(r, a, (group_thread==0) ? add : 0); + result=fast_propagate_add(carry, r); + clear_carry(r); + return result; +} + +template +__device__ __forceinline__ int32_t core_t::sub_ui32(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t sub) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1, carry; + int32_t result; + + // FIX FIX FIX -- use custom algorithm for each TPI/PADDING + + carry=mpsub32(r, a, (group_thread==0) ? sub : 0); + result=-fast_propagate_sub(carry, r); + clear_carry(r); + return result; +} + +template +__device__ __forceinline__ uint32_t core_t::mul_ui32(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t mul) { + uint32_t carry; + + carry=mpmul32(r, a, mul); + carry=resolve_add(carry, r); + clear_carry(r); + return carry; +} + +template +__device__ __forceinline__ uint32_t core_t::div_ui32(uint32_t &r, const uint32_t a, const uint32_t d) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t x, bits, normalized, inv, approx, beta_k, t, rem, t0, t1; + + // back + bits=uctz(d); + normalized=d>>bits; + inv=ubinary_inverse(normalized); + t=__shfl_down_sync(sync, a, 1, TPI); + t=(group_thread==TPI-1) ? 0 : t; + x=uright_clamp(a, t, bits); + + // and forth + bits=uclz(normalized); + normalized=normalized<>bits; + + // Jebelean exact division + r=(x-rem)*inv; + + // distribute final remainder + return __shfl_sync(sync, a-r*d, 0, TPI); +} + +template +__device__ __forceinline__ uint32_t core_t::div_ui32(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t d) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t mod[LIMBS+1], x[LIMBS+1], bits, left_bits, right_bits, normalized, inv, approx, beta_k, t, rem, t0, t1; + + left_bits=uclz(d); + normalized=d<(mod, normalized, approx); + + // back + right_bits=uctz(d); + inv=ubinary_inverse(d>>right_bits); + t=__shfl_down_sync(sync, a[0], 1, TPI); + t=(group_thread==TPI-1) ? 0 : t; + mpright(x, a, right_bits, t); + + // and forth + bits=left_bits+right_bits; + mod[LIMBS]=uleft_clamp(x[LIMBS-1], 0, bits); + mpleft(mod, x, bits); + + // compute local remainder + rem=mprem32(mod, normalized, approx); + + // integrate remainder from other threads + #pragma unroll + for(int32_t index=1;index>bits; + + // Jebelean exact division + normalized=d>>right_bits; + #pragma unroll + for(int32_t index=0;indexx[index]) ? 1 : 0; + r[index]=(x[index]-rem)*inv; + rem=madhi(r[index], normalized, t); + } + + // distribute final remainder + return __shfl_sync(sync, a[0]-r[0]*d, 0, TPI); +} + +template +__device__ __forceinline__ uint32_t core_t::rem_ui32(const uint32_t a, const uint32_t d) { + uint32_t sync=sync_mask(); + uint32_t bits, normalized, approx, beta_k, t, rem, t0, t1; + + bits=uclz(d); + normalized=d<>bits; + + // distribute remainder + return __shfl_sync(sync, rem, 0, TPI); +} + +template +__device__ __forceinline__ uint32_t core_t::rem_ui32(const uint32_t a[LIMBS], const uint32_t d) { + uint32_t sync=sync_mask(); + uint32_t bits, normalized, approx, beta_k, t, rem, t0, t1; + uint32_t x[LIMBS+1]; + + bits=uclz(d); + normalized=d<(x, normalized, approx); + + x[LIMBS]=uleft_clamp(a[LIMBS-1], 0, bits); + mpleft(x, a, bits); + + // compute local remainder + rem=mprem32(x, normalized, approx); + + // integrate remainder from other threads + #pragma unroll + for(int32_t index=1;index>bits; + + // distribute remainder + return __shfl_sync(sync, rem, 0, TPI); +} + +template +__device__ __forceinline__ bool core_t::equals_ui32(const uint32_t a[LIMBS], const uint32_t value) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1, lor, mask; + + lor=a[0] ^ ((group_thread==0) ? value : 0); + #pragma unroll + for(int32_t index=1;index +__device__ __forceinline__ int32_t core_t::compare_ui32(const uint32_t a[LIMBS], const uint32_t value) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1, lor, mask; + int32_t result=1; + + // check for zero, ignoring least significant limb + lor=(group_thread==0) ? 0 : a[0]; + #pragma unroll + for(int32_t index=1;index +class core_singleton_t; + +/* NOTE: This class name is probably not well named. + Singleton connotes a single object instance (similar to a factory object in Java) but that's not what's intended here. + It's called singleton because it dispatches single limb vs multi-limb cgbn_t's to different APIs in the core_t object. */ + +template +class core_singleton_t { + public: + static const uint32_t limbs=1; + + __device__ __forceinline__ static void mul(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b[limbs], const uint32_t add[limbs]) { + core_t::mul(r[0], a[0], b[0], add[0]); + } + + __device__ __forceinline__ static void mul_high(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b[limbs], const uint32_t add[limbs]) { + uint32_t ignore; + + core_t::mul_wide(ignore, r[0], a[0], b[0], add[0]); + } + + __device__ __forceinline__ static void sqrt_resolve_rem(uint32_t rem[limbs], const uint32_t s[limbs], const uint32_t top, const uint32_t r[limbs], const uint32_t shift) { + core_t::sqrt_resolve_rem(rem[0], s[0], top, r[0], shift); + } + + __device__ __forceinline__ static void sqrt(uint32_t s[limbs], const uint32_t a[limbs], const uint32_t numthreads) { + core_t::sqrt(s[0], a[0], numthreads); + } + + __device__ __forceinline__ static void sqrt_rem(uint32_t s[limbs], uint32_t r[limbs], const uint32_t a[limbs], const uint32_t numthreads) { + core_t::sqrt_rem(s[0], r[0], a[0], numthreads); + } + + __device__ __forceinline__ static void mul_wide(uint32_t lo[limbs], uint32_t hi[limbs], const uint32_t a[limbs], const uint32_t b[limbs], const uint32_t add[limbs]) { + core_t::mul_wide(lo[0], hi[0], a[0], b[0], add[0]); + } + + __device__ __forceinline__ static void div_wide(uint32_t q[limbs], const uint32_t lo[limbs], const uint32_t hi[limbs], const uint32_t denom[limbs], const uint32_t numthreads) { + core_t::div_wide(q[0], lo[0], hi[0], denom[0], numthreads); + } + + __device__ __forceinline__ static void rem_wide(uint32_t r[limbs], const uint32_t lo[limbs], const uint32_t hi[limbs], const uint32_t denom[limbs], const uint32_t numthreads) { + core_t::rem_wide(r[0], lo[0], hi[0], denom[0], numthreads); + } + + __device__ __forceinline__ static void div_rem_wide(uint32_t q[limbs], uint32_t r[limbs], const uint32_t lo[limbs], const uint32_t hi[limbs], const uint32_t denom[limbs], const uint32_t numthreads) { + core_t::div_rem_wide(q[0], r[0], lo[0], hi[0], denom[0], numthreads); + } + + __device__ __forceinline__ static void sqrt_wide(uint32_t s[limbs], const uint32_t lo[limbs], const uint32_t hi[limbs], const uint32_t numthreads) { + core_t::sqrt_wide(s[0], lo[0], hi[0], numthreads); + } + + __device__ __forceinline__ static uint32_t sqrt_rem_wide(uint32_t s[limbs], uint32_t r[limbs], const uint32_t lo[limbs], const uint32_t hi[limbs], const uint32_t numthreads) { + return core_t::sqrt_rem_wide(s[0], r[0], lo[0], hi[0], numthreads); + } + + __device__ __forceinline__ static uint32_t div_ui32(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t div) { + return core_t::div_ui32(r[0], a[0], div); + } + + __device__ __forceinline__ static uint32_t rem_ui32(const uint32_t a[limbs], const uint32_t div) { + return core_t::rem_ui32(a[0], div); + } + + __device__ __forceinline__ static void mont_mul(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b[limbs], const uint32_t n[limbs], const uint32_t np0) { + core_t::mont_mul(r[0], a[0], b[0], n[0], np0); + } + + __device__ __forceinline__ static void mont_reduce_wide(uint32_t r[limbs], const uint32_t lo[limbs], const uint32_t hi[limbs], const uint32_t n[limbs], const uint32_t np0, const bool zero) { + core_t::mont_reduce_wide(r[0], lo[0], hi[0], n[0], np0, zero); + } +}; + +template +class core_singleton_t { + public: + + __device__ __forceinline__ static void mul(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b[limbs], const uint32_t add[limbs]) { + core_t::mul(r, a, b, add); + } + + __device__ __forceinline__ static void mul_high(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b[limbs], const uint32_t add[limbs]) { + uint32_t ignore[limbs]; + + core_t::mul_wide(ignore, r, a, b, add); + } + + __device__ __forceinline__ static void sqrt_resolve_rem(uint32_t rem[limbs], const uint32_t s[limbs], const uint32_t top, const uint32_t r[limbs], const uint32_t shift) { + core_t::sqrt_resolve_rem(rem, s, top, r, shift); + } + + __device__ __forceinline__ static void sqrt(uint32_t s[limbs], const uint32_t a[limbs], const uint32_t numthreads) { + core_t::sqrt(s, a, numthreads); + } + + __device__ __forceinline__ static void sqrt_rem(uint32_t s[limbs], uint32_t r[limbs], const uint32_t a[limbs], const uint32_t numthreads) { + core_t::sqrt_rem(s, r, a, numthreads); + } + + __device__ __forceinline__ static void mul_wide(uint32_t lo[limbs], uint32_t hi[limbs], const uint32_t a[limbs], const uint32_t b[limbs], const uint32_t add[limbs]) { + core_t::mul_wide(lo, hi, a, b, add); + } + + __device__ __forceinline__ static void div_wide(uint32_t q[limbs], const uint32_t lo[limbs], const uint32_t hi[limbs], const uint32_t denom[limbs], const uint32_t numthreads) { + core_t::div_wide(q, lo, hi, denom, numthreads); + } + + __device__ __forceinline__ static void rem_wide(uint32_t r[limbs], const uint32_t lo[limbs], const uint32_t hi[limbs], const uint32_t denom[limbs], const uint32_t numthreads) { + core_t::rem_wide(r, lo, hi, denom, numthreads); + } + + __device__ __forceinline__ static void div_rem_wide(uint32_t q[limbs], uint32_t r[limbs], const uint32_t lo[limbs], const uint32_t hi[limbs], const uint32_t denom[limbs], const uint32_t numthreads) { + core_t::div_rem_wide(q, r, lo, hi, denom, numthreads); + } + + __device__ __forceinline__ static void sqrt_wide(uint32_t s[limbs], const uint32_t lo[limbs], const uint32_t hi[limbs], const uint32_t numthreads) { + core_t::sqrt_wide(s, lo, hi, numthreads); + } + + __device__ __forceinline__ static uint32_t sqrt_rem_wide(uint32_t s[limbs], uint32_t r[limbs], const uint32_t lo[limbs], const uint32_t hi[limbs], const uint32_t numthreads) { + return core_t::sqrt_rem_wide(s, r, lo, hi, numthreads); + } + + __device__ __forceinline__ static uint32_t div_ui32(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t div) { + return core_t::div_ui32(r, a, div); + } + + __device__ __forceinline__ static uint32_t rem_ui32(const uint32_t a[limbs], const uint32_t div) { + return core_t::rem_ui32(a, div); + } + + __device__ __forceinline__ static void mont_mul(uint32_t r[limbs], const uint32_t a[limbs], const uint32_t b[limbs], const uint32_t n[limbs], const uint32_t np0) { + core_t::mont_mul(r, a, b, n, np0); + } + + __device__ __forceinline__ static void mont_reduce_wide(uint32_t r[limbs], const uint32_t lo[limbs], const uint32_t hi[limbs], const uint32_t n[limbs], const uint32_t np0, const bool zero) { + core_t::mont_reduce_wide(r, lo, hi, n, np0, zero); + } +}; + +} /* namespace cgbn */ diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_sqrt_multi.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_sqrt_multi.cu new file mode 100644 index 0000000..3eb232d --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_sqrt_multi.cu @@ -0,0 +1,454 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +__device__ __forceinline__ void core_t::sqrt_resolve_rem(uint32_t rem[LIMBS], const uint32_t s[LIMBS], const uint32_t top, const uint32_t r[LIMBS], const uint32_t shift) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t mask, phi[LIMBS], plo[LIMBS], t, s0[LIMBS]; + + // remainder computation: r'=(2*s*(s mod b)+r)/b^2 + // where s=square root, r=remainder, b=2^shift. + + mask=(1<<(shift & 0x1F))-1; + #pragma unroll + for(int32_t index=0;indexshift) + s0[index]=0; + else + s0[index]=__shfl_sync(sync, s[index], 0, TPI) & mask; + + mpadd(s0, s0, s0); + mpmul(plo, phi, s, s0); + + chain_t<> chain; + #pragma unroll + for(int32_t index=0;index(plo, plo, phi); + mpzero(phi); + } + t=fast_propagate_add(t, plo); + + if(group_thread==0) + mpadd32(phi, phi, t); + bitwise_mask_select(plo, plo, phi, shift*2); + rotate_right(rem, plo, shift*2); +} + +template +__device__ __forceinline__ void core_t::sqrt(uint32_t s[LIMBS], const uint32_t x[LIMBS], const uint32_t numthreads) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t dlo[DLIMBS], dhi[DLIMBS], rem[DLIMBS], dtemp[DLIMBS], approx[DLIMBS], t, c; + uint32_t remainder[LIMBS], q[LIMBS], plo[LIMBS], phi[LIMBS]; + int32_t top; + + dlimbs_scatter(dlo, x, TPI-2); + dlimbs_scatter(dhi, x, TPI-1); + + top=dlimbs_sqrt_rem_wide(dtemp, rem, dlo, dhi); + dlimbs_approximate(approx, dtemp); + + // set up remainder + #pragma unroll + for(int32_t index=0;index(dtemp, dtemp, 1, t); + mpzero(s); + dlimbs_gather(s, dtemp, TPI-1); + + // silent 1 at top of s + #pragma nounroll + for(int32_t index=TPI-2;index>=(int32_t)(TPI-numthreads);index--) { + dlimbs_scatter(dtemp, remainder, TPI-1); + dlimbs_sqrt_estimate(dtemp, top, dtemp, approx); + dlimbs_all_gather(q, dtemp); + + if(group_thread==index) + mpset(s, q); + + // compute low/high + mpmul(plo, phi, s, q); + + // double q in s + c=0; + if(group_thread==index) + c=mpadd(s, s, q); + c=__shfl_sync(sync, c, threadIdx.x-1, TPI); + s[0]=s[0]+c; + + c=mpsub(remainder, remainder, phi); + top=__shfl_sync(sync, remainder[0], TPI-1, TPI) - q[0]; // we subtract q[0] because of the silent 1 in s + + #pragma unroll + for(int32_t limb=0;limb(remainder, remainder, plo); + + top=top+resolve_sub(c, remainder); + + while(top<0) { + c=0; + if(group_thread==index) { + // decrement s by 2, if we borrow, need to resolve in next thread + c=mpsub32(s, s, 2); + } + c=__shfl_sync(sync, c, threadIdx.x-1, TPI); + s[0]=s[0]+c; + + add_cc(group_thread==index, 0xFFFFFFFF); + #pragma unroll + for(int32_t limb=0;limb(s, s, 1, t); + + #pragma unroll + for(int32_t limb=0;limb +__device__ __forceinline__ void core_t::sqrt_rem(uint32_t s[LIMBS], uint32_t r[LIMBS], const uint32_t x[LIMBS], const uint32_t numthreads) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t dlo[DLIMBS], dhi[DLIMBS], rem[DLIMBS], dtemp[DLIMBS], approx[DLIMBS], t, c; + uint32_t remainder[LIMBS], q[LIMBS], plo[LIMBS], phi[LIMBS]; + int32_t top; + + dlimbs_scatter(dlo, x, TPI-2); + dlimbs_scatter(dhi, x, TPI-1); + + top=dlimbs_sqrt_rem_wide(dtemp, rem, dlo, dhi); + dlimbs_approximate(approx, dtemp); + + // set up remainder + #pragma unroll + for(int32_t index=0;index(dtemp, dtemp, 1, t); + mpzero(s); + dlimbs_gather(s, dtemp, TPI-1); + + // silent 1 at top of s + #pragma nounroll + for(int32_t index=TPI-2;index>=(int32_t)(TPI-numthreads);index--) { + dlimbs_scatter(dtemp, remainder, TPI-1); + dlimbs_sqrt_estimate(dtemp, top, dtemp, approx); + dlimbs_all_gather(q, dtemp); + + if(group_thread==index) + mpset(s, q); + + // compute low/high + mpmul(plo, phi, s, q); + + // double q in s + c=0; + if(group_thread==index) + c=mpadd(s, s, q); + c=__shfl_sync(sync, c, threadIdx.x-1, TPI); + s[0]=s[0]+c; + + c=mpsub(remainder, remainder, phi); + top=__shfl_sync(sync, remainder[0], TPI-1, TPI) - q[0]; // we subtract q[0] because of the silent 1 in s + + #pragma unroll + for(int32_t limb=0;limb(remainder, remainder, plo); + + top=top+resolve_sub(c, remainder); + + while(top<0) { + c=0; + if(group_thread==index) { + // decrement s by 2, if we borrow, need to resolve in next thread + c=mpsub32(s, s, 2); + } + c=__shfl_sync(sync, c, threadIdx.x-1, TPI); + s[0]=s[0]+c; + + add_cc(group_thread==index, 0xFFFFFFFF); + #pragma unroll + for(int32_t limb=0;limb(s, s, 1, t); + + #pragma unroll + for(int32_t limb=0;limb=numthreads) + mpzero(r); + if(group_thread==numthreads) + r[0]=top; +} + +template +__device__ __forceinline__ void core_t::sqrt_wide(uint32_t s[LIMBS], const uint32_t lo[LIMBS], const uint32_t hi[LIMBS], const uint32_t numthreads) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t dlo[DLIMBS], dhi[DLIMBS], rem[DLIMBS], dtemp[DLIMBS], approx[DLIMBS], t, c; + uint32_t remainder[LIMBS], q[LIMBS], plo[LIMBS], phi[LIMBS]; + int32_t top; + + dlimbs_scatter(dlo, hi, TPI-2); + dlimbs_scatter(dhi, hi, TPI-1); + + top=dlimbs_sqrt_rem_wide(dtemp, rem, dlo, dhi); + dlimbs_approximate(approx, dtemp); + + // set up remainder + #pragma unroll + for(int32_t index=0;index(dtemp, dtemp, 1, t); + mpzero(s); + dlimbs_gather(s, dtemp, TPI-1); + + // silent 1 at top of s + #pragma nounroll + for(int32_t index=TPI-2;index>=(int32_t)(TPI-numthreads);index--) { + dlimbs_scatter(dtemp, remainder, TPI-1); + dlimbs_sqrt_estimate(dtemp, top, dtemp, approx); + dlimbs_all_gather(q, dtemp); + + if(group_thread==index) + mpset(s, q); + + // compute low/high + mpmul(plo, phi, s, q); + + // double q in s + c=0; + if(group_thread==index) + c=mpadd(s, s, q); + c=__shfl_sync(sync, c, threadIdx.x-1, TPI); + s[0]=s[0]+c; + + c=mpsub(remainder, remainder, phi); + top=__shfl_sync(sync, remainder[0], TPI-1, TPI) - q[0]; // we subtract q[0] because of the silent 1 in s + + #pragma unroll + for(int32_t limb=0;limb(remainder, remainder, plo); + + top=top+resolve_sub(c, remainder); + + while(top<0) { + c=0; + if(group_thread==index) { + // decrement s by 2, if we borrow, need to resolve in next thread + c=mpsub32(s, s, 2); + } + c=__shfl_sync(sync, c, threadIdx.x-1, TPI); + s[0]=s[0]+c; + + add_cc(group_thread==index, 0xFFFFFFFF); + #pragma unroll + for(int32_t limb=0;limb(s, s, 1, t); + + #pragma unroll + for(int32_t limb=0;limb +__device__ __forceinline__ uint32_t core_t::sqrt_rem_wide(uint32_t s[LIMBS], uint32_t r[LIMBS], const uint32_t lo[LIMBS], const uint32_t hi[LIMBS], const uint32_t numthreads) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t dlo[DLIMBS], dhi[DLIMBS], rem[DLIMBS], dtemp[DLIMBS], approx[DLIMBS], t, c; + uint32_t remainder[LIMBS], q[LIMBS], plo[LIMBS], phi[LIMBS]; + int32_t top; + + dlimbs_scatter(dlo, hi, TPI-2); + dlimbs_scatter(dhi, hi, TPI-1); + + top=dlimbs_sqrt_rem_wide(dtemp, rem, dlo, dhi); + dlimbs_approximate(approx, dtemp); + + // set up remainder + #pragma unroll + for(int32_t index=0;index(dtemp, dtemp, 1, t); + mpzero(s); + dlimbs_gather(s, dtemp, TPI-1); + + // silent 1 at top of s + #pragma nounroll + for(int32_t index=TPI-2;index>=(int32_t)(TPI-numthreads);index--) { + dlimbs_scatter(dtemp, remainder, TPI-1); + dlimbs_sqrt_estimate(dtemp, top, dtemp, approx); + dlimbs_all_gather(q, dtemp); + + if(group_thread==index) + mpset(s, q); + + // compute low/high + mpmul(plo, phi, s, q); + + // double q in s + c=0; + if(group_thread==index) + c=mpadd(s, s, q); + c=__shfl_sync(sync, c, threadIdx.x-1, TPI); + s[0]=s[0]+c; + + c=mpsub(remainder, remainder, phi); + top=__shfl_sync(sync, remainder[0], TPI-1, TPI) - q[0]; // we subtract q[0] because of the silent 1 in s + + #pragma unroll + for(int32_t limb=0;limb(remainder, remainder, plo); + + top=top+resolve_sub(c, remainder); + + while(top<0) { + c=0; + if(group_thread==index) { + // decrement s by 2, if we borrow, need to resolve in next thread + c=mpsub32(s, s, 2); + } + c=__shfl_sync(sync, c, threadIdx.x-1, TPI); + s[0]=s[0]+c; + + add_cc(group_thread==index, 0xFFFFFFFF); + #pragma unroll + for(int32_t limb=0;limb(s, s, 1, t); + + #pragma unroll + for(int32_t limb=0;limb=numthreads) + mpzero(r); + if(group_thread==numthreads) + r[0]=top; + return (numthreads==TPI) ? top : 0; +} + +} /* namespace cgbn */ diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_sqrt_single.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_sqrt_single.cu new file mode 100644 index 0000000..5418530 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/core_sqrt_single.cu @@ -0,0 +1,326 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +__device__ __forceinline__ void core_t::sqrt_resolve_rem(uint32_t &rem, const uint32_t s, const uint32_t top, const uint32_t r, const uint32_t shift) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t mask, hi, lo, t, s0; + + // remainder computation: r'=(2*s*(s mod b)+r)/b^2 + // where s=square root, r=remainder, b=2^shift. + + hi=(group_thread==TPI-1) ? top : 0; + + mask=(1<=16) { + t=__shfl_down_sync(sync, lo, 1, TPI); + lo=(group_thread==TPI-1) ? hi : t; + hi=0; + } + t=__shfl_down_sync(sync, lo, 1, TPI); + hi=(group_thread==TPI-1) ? hi : t; + rem=uright_wrap(lo, hi, shift+shift); +} + +template +__device__ __forceinline__ void core_t::sqrt(uint32_t &s, const uint32_t a, const uint32_t numthreads) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t x, x0, x1, t0, t1, divisor, approx, p, q, c, sqrt; + + x=a; + x0=__shfl_sync(sync, x, TPI-2, TPI); + x1=__shfl_sync(sync, x, TPI-1, TPI); + + divisor=usqrt(x0, x1); + approx=uapprox(divisor); + + t0=madlo(divisor, divisor, 0); + t1=madhi(divisor, divisor, 0); + x0=sub_cc(x0, t0); + x1=subc(x1, t1); + + x=__shfl_up_sync(sync, x, 1, TPI); + x=(group_thread==TPI-1) ? x0 : x; + sqrt=(group_thread==TPI-1) ? divisor+divisor : 0; // silent 1 at the top of sqrt + + #pragma nounroll + for(int32_t index=TPI-2;index>=(int32_t)(TPI-numthreads);index--) { + x0=__shfl_sync(sync, x, TPI-1, TPI); + q=usqrt_div(x0, x1, divisor, approx); + sqrt=(group_thread==index) ? q : sqrt; + + p=madhi(q, sqrt, 0); + x=sub_cc(x, p); + c=subc(0, 0); + fast_propagate_sub(c, x); + + x1=__shfl_sync(sync, x, TPI-1, TPI)-q; // we subtract q because of the silent 1 at the top of sqrt + x=__shfl_up_sync(sync, x, 1, TPI); + p=madlo(q, sqrt, 0); + x=sub_cc(x, p); + c=subc(0, 0); + x1-=fast_propagate_sub(c, x); + + while(0>(int32_t)x1) { + x1++; + q--; + + // correction step: add q and s + x=add_cc(x, (group_thread==index) ? q : 0); + c=addc(0, 0); + x=add_cc(x, sqrt); + c=addc(c, 0); + + x1+=resolve_add(c, x); + + // update s + sqrt=(group_thread==index) ? q : sqrt; + } + sqrt=(group_thread==index+1) ? sqrt+(q>>31) : sqrt; + sqrt=(group_thread==index) ? q+q : sqrt; + } + c=__shfl_down_sync(sync, sqrt, 1, TPI); + c=(group_thread==TPI-1) ? 1 : c; + sqrt=uright_wrap(sqrt, c, 1); + s=__shfl_sync(sync, sqrt, threadIdx.x-numthreads, TPI); +} + +template +__device__ __forceinline__ void core_t::sqrt_rem(uint32_t &s, uint32_t &r, const uint32_t a, const uint32_t numthreads) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t x0, x1, lo, hi, divisor, approx, p, q, c; + + r=a; + x1=__shfl_sync(sync, r, TPI-1, TPI); + x0=__shfl_sync(sync, r, TPI-2, TPI); + + divisor=usqrt(x0, x1); + approx=uapprox(divisor); + + lo=madlo(divisor, divisor, 0); + hi=madhi(divisor, divisor, 0); + lo=sub_cc(x0, lo); + hi=subc(x1, hi); + + r=__shfl_up_sync(sync, r, 1, TPI); + r=(group_thread==TPI-1) ? lo : r; + s=(group_thread==TPI-1) ? divisor+divisor : 0; // silent 1 at the top of s + + #pragma nounroll + for(int32_t index=TPI-2;index>=(int32_t)(TPI-numthreads);index--) { + lo=__shfl_sync(sync, r, TPI-1, TPI); + q=usqrt_div(lo, hi, divisor, approx); + s=(group_thread==index) ? q : s; + + p=madhi(q, s, 0); + r=sub_cc(r, p); + c=subc(0, 0); + fast_propagate_sub(c, r); + + hi=__shfl_sync(sync, r, TPI-1, TPI)-q; // we subtract q because of the silent 1 at the top of s + r=__shfl_up_sync(sync, r, 1, TPI); + p=madlo(q, s, 0); + r=sub_cc(r, p); + c=subc(0, 0); + hi-=fast_propagate_sub(c, r); + + while(0>(int32_t)hi) { + hi++; + q--; + + // correction step: add q and s + r=add_cc(r, (group_thread==index) ? q : 0); + c=addc(0, 0); + r=add_cc(r, s); + c=addc(c, 0); + + hi+=resolve_add(c, r); + + // update s + s=(group_thread==index) ? q : s; + } + + s=(group_thread==index+1) ? s + (q>>31) : s; + s=(group_thread==index) ? q+q : s; + } + c=__shfl_down_sync(sync, s, 1, TPI); + c=(group_thread==TPI-1) ? 1 : c; + s=uright_wrap(s, c, 1); + s=__shfl_sync(sync, s, threadIdx.x-numthreads, TPI); + + r=__shfl_sync(sync, r, threadIdx.x-numthreads, TPI); + r=(group_thread==numthreads) ? hi : r; + r=(group_thread>numthreads) ? 0 : r; +} + +template +__device__ __forceinline__ void core_t::sqrt_wide(uint32_t &s, const uint32_t lo, const uint32_t hi, const uint32_t numthreads) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t x, x0, x1, t0, t1, divisor, approx, p, q, c, sqrt; + + x=hi; + x0=__shfl_sync(sync, x, TPI-2, TPI); + x1=__shfl_sync(sync, x, TPI-1, TPI); + + divisor=usqrt(x0, x1); + approx=uapprox(divisor); + + t0=madlo_cc(divisor, divisor, 0); + t1=madhic(divisor, divisor, 0); + x0=sub_cc(x0, t0); + x1=subc(x1, t1); + + x=(group_thread==TPI-1) ? lo : x; + x=__shfl_sync(sync, x, threadIdx.x-1, TPI); + x=(group_thread==TPI-1) ? x0 : x; + sqrt=(group_thread==TPI-1) ? divisor+divisor : 0; // silent 1 at the top of s + + #pragma nounroll + for(int32_t index=TPI-2;index>=(int32_t)(TPI-numthreads);index--) { + x0=__shfl_sync(sync, x, TPI-1, TPI); + q=usqrt_div(x0, x1, divisor, approx); + sqrt=(group_thread==index) ? q : sqrt; + + p=madhi(q, sqrt, 0); + x=sub_cc(x, p); + c=subc(0, 0); + fast_propagate_sub(c, x); + + x1=__shfl_sync(sync, x, TPI-1, TPI)-q; // we subtract q because of the silent 1 at the top of s + t0=__shfl_sync(sync, lo, index, TPI); + x=__shfl_up_sync(sync, x, 1, TPI); + x=(group_thread==0) ? t0 : x; + + p=madlo(q, sqrt, 0); + x=sub_cc(x, p); + c=subc(0, 0); + x1-=fast_propagate_sub(c, x); + + while(0>(int32_t)x1) { + x1++; + q--; + // correction step: add q and sqrt + x=add_cc(x, (group_thread==index) ? q : 0); + c=addc(0, 0); + x=add_cc(x, sqrt); + c=addc(c, 0); + + x1+=resolve_add(c, x); + + // update sqrt + sqrt=(group_thread==index) ? q : sqrt; + } + + sqrt=(group_thread==index+1) ? sqrt+(q>>31) : sqrt; + sqrt=(group_thread==index) ? q+q : sqrt; + } + c=__shfl_down_sync(sync, sqrt, 1, TPI); + c=(group_thread==TPI-1) ? 1 : c; + sqrt=uright_wrap(sqrt, c, 1); + s=__shfl_sync(sync, sqrt, threadIdx.x-numthreads, TPI); +} + +template +__device__ __forceinline__ uint32_t core_t::sqrt_rem_wide(uint32_t &s, uint32_t &r, const uint32_t lo, const uint32_t hi, const uint32_t numthreads) { + uint32_t sync=sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t x0, x1, t0, t1, divisor, approx, p, q, c, low; + + low=lo; + r=hi; + x0=__shfl_sync(sync, r, TPI-2, TPI); + x1=__shfl_sync(sync, r, TPI-1, TPI); + + divisor=usqrt(x0, x1); + approx=uapprox(divisor); + + t0=madlo(divisor, divisor, 0); + t1=madhi(divisor, divisor, 0); + x0=sub_cc(x0, t0); + x1=subc(x1, t1); + + r=(group_thread==TPI-1) ? low : r; + r=__shfl_sync(sync, r, threadIdx.x-1, TPI); + r=(group_thread==TPI-1) ? x0 : r; + s=(group_thread==TPI-1) ? divisor+divisor : 0; // silent 1 at the top of s + + #pragma nounroll + for(int32_t index=TPI-2;index>=(int32_t)(TPI-numthreads);index--) { + x0=__shfl_sync(sync, r, TPI-1, TPI); + q=usqrt_div(x0, x1, divisor, approx); + s=(group_thread==index) ? q : s; + + p=madhi(q, s, 0); + r=sub_cc(r, p); + c=subc(0, 0); + fast_propagate_sub(c, r); + + x1=__shfl_sync(sync, r, TPI-1, TPI)-q; // we subtract q because of the silent 1 at the top of s + t0=__shfl_sync(sync, low, index, TPI); + r=__shfl_up_sync(sync, r, 1, TPI); + r=(group_thread==0) ? t0 : r; + + p=madlo(q, s, 0); + r=sub_cc(r, p); + c=subc(0, 0); + x1-=fast_propagate_sub(c, r); + + while(0>(int32_t)x1) { + x1++; + q--; + + // correction step: add q and s + r=add_cc(r, (group_thread==index) ? q : 0); + c=addc(0, 0); + r=add_cc(r, s); + c=addc(c, 0); + + x1+=resolve_add(c, r); + + // update s + s=(group_thread==index) ? q : s; + } + + s=(group_thread==index+1) ? s+(q>>31) : s; + s=(group_thread==index) ? q+q : s; + } + c=__shfl_down_sync(sync, s, 1, TPI); + c=(group_thread==TPI-1) ? 1 : c; + s=uright_wrap(s, c, 1); + s=__shfl_sync(sync, s, threadIdx.x-numthreads, TPI); + + r=__shfl_sync(sync, r, threadIdx.x-numthreads, TPI); + r=(group_thread==numthreads) ? x1 : r; + r=(group_thread>numthreads) ? 0 : r; + return numthreads==TPI ? x1 : 0; +} + +} /* namespace cgbn */ \ No newline at end of file diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_dlimbs.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_dlimbs.cu new file mode 100644 index 0000000..a075183 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_dlimbs.cu @@ -0,0 +1,472 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +typedef enum { + dlimbs_algs_common, + dlimbs_algs_half, + dlimbs_algs_full, + dlimbs_algs_multi +} dlimbs_algs_t; + +template +class dispatch_dlimbs_t; + +template +class dispatch_dlimbs_t { + public: + static const uint32_t TPI=core::TPI; + static const uint32_t LIMBS=core::LIMBS; + static const uint32_t DLIMBS=core::DLIMBS; + static const uint32_t LIMB_OFFSET=DLIMBS*TPI-LIMBS; + + __device__ __forceinline__ static void dlimbs_scatter(uint32_t r[DLIMBS], const uint32_t x[LIMBS], const uint32_t source_thread) { + uint32_t sync=core::sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t t; + + mpzero(r); + #pragma unroll + for(int32_t index=0;index +class dispatch_dlimbs_t { + public: + static const uint32_t TPI=core::TPI; + static const uint32_t LIMBS=core::LIMBS; + static const uint32_t DLIMBS=core::DLIMBS; + static const uint32_t LIMB_OFFSET=DLIMBS*TPI-LIMBS; + + // these algorithms require that LIMBS<=TPI/2 + + __device__ __forceinline__ static void dlimbs_approximate(uint32_t approx[DLIMBS], const uint32_t denom[DLIMBS]) { + uint32_t sync=core::sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t x, d0, d1, x0, x1, x2, est, a, h, l; + int32_t c, top; + + // computes (beta^2 - 1) / denom - beta, where beta=1<<32*LIMBS + + x=0xFFFFFFFF-denom[0]; + + d1=__shfl_sync(sync, denom[0], TPI-1, TPI); + d0=__shfl_sync(sync, denom[0], TPI-2, TPI); + + approx[0]=0; + a=uapprox(d1); + + #pragma nounroll + for(int32_t thread=LIMBS-1;thread>=0;thread--) { + x0=__shfl_sync(sync, x, TPI-3, TPI); + x1=__shfl_sync(sync, x, TPI-2, TPI); + x2=__shfl_sync(sync, x, TPI-1, TPI); + est=udiv(x0, x1, x2, d0, d1, a); + + l=madlo_cc(est, denom[0], 0); + h=madhic(est, denom[0], 0); + + x=sub_cc(x, h); + c=subc(0, 0); // thread TPI-1 is zero + + top=__shfl_sync(sync, x, TPI-1, TPI); + x=__shfl_sync(sync, x, threadIdx.x-1, TPI); + c=__shfl_sync(sync, c, threadIdx.x-1, TPI); + + x=sub_cc(x, l); + c=subc(c, 0); + + if(top+core::resolve_sub(c, x)<0) { + // means a correction is required, should be very rare + x=add_cc(x, denom[0]); + c=addc(0, 0); + core::fast_propagate_add(c, x); + est--; + } + approx[0]=(group_thread==thread+TPI-LIMBS) ? est : approx[0]; + } + } + + __device__ __forceinline__ static uint32_t dlimbs_sqrt_rem_wide(uint32_t s[DLIMBS], uint32_t r[DLIMBS], const uint32_t lo[DLIMBS], const uint32_t hi[DLIMBS]) { + uint32_t sync=core::sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t x, x0, x1, t0, t1, divisor, approx, p, q, c; + + // computes s=sqrt(x), r=x-s^2, where x=(hi<<32*LIMBS) + lo + + t0=__shfl_sync(sync, lo[0], threadIdx.x+LIMBS, TPI); + x=hi[0] | t0; + x0=__shfl_sync(sync, x, TPI-2, TPI); + x1=__shfl_sync(sync, x, TPI-1, TPI); + + divisor=usqrt(x0, x1); + approx=uapprox(divisor); + + t0=madlo(divisor, divisor, 0); + t1=madhi(divisor, divisor, 0); + x0=sub_cc(x0, t0); + x1=subc(x1, t1); + + x=__shfl_up_sync(sync, x, 1, TPI); + x=(group_thread==TPI-1) ? x0 : x; + s[0]=(group_thread==TPI-1) ? divisor+divisor : 0; // silent 1 at the top of s + + #pragma nounroll + for(int32_t index=TPI-2;index>=(int32_t)(TPI-LIMBS);index--) { + x0=__shfl_sync(sync, x, TPI-1, TPI); + q=usqrt_div(x0, x1, divisor, approx); + s[0]=(group_thread==index) ? q : s[0]; + + p=madhi(q, s[0], 0); + x=sub_cc(x, p); + c=subc(0, 0); + core::fast_propagate_sub(c, x); + + x1=__shfl_sync(sync, x, TPI-1, TPI)-q; // we subtract q because of the silent 1 at the top of s + x=__shfl_up_sync(sync, x, 1, TPI); + p=madlo(q, s[0], 0); + x=sub_cc(x, p); + c=subc(0, 0); + x1-=core::fast_propagate_sub(c, x); + + while(0>(int32_t)x1) { + x1++; + q--; + + // correction step: add q and s + x=add_cc(x, (group_thread==index) ? q : 0); + c=addc(0, 0); + x=add_cc(x, s[0]); + c=addc(c, 0); + + x1+=core::resolve_add(c, x); + + // update s + s[0]=(group_thread==index) ? q : s[0]; + } + s[0]=(group_thread==index+1) ? s[0]+(q>>31) : s[0]; + s[0]=(group_thread==index) ? q+q : s[0]; + } + t0=__shfl_down_sync(sync, s[0], 1, TPI); + t0=(group_thread==TPI-1) ? 1 : t0; + s[0]=uright_wrap(s[0], t0, 1); + r[0]=x; + return x1; + } + + __device__ __forceinline__ static void dlimbs_div_estimate(uint32_t q[DLIMBS], const uint32_t x[DLIMBS], const uint32_t approx[DLIMBS]) { + uint32_t sync=core::sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t t, c; + uint64_t w; + + // computes q=(x*approx>>32*LIMBS) + x + 3 + // q=min(q, (1<<32*LIMBS)-1); + // + // Notes: leaves junk in lower words of q + + w=0; + #pragma unroll + for(int32_t index=0;index>32)+t; + } + + // increase the estimate by 3 + t=(group_thread==TPI-LIMBS) ? 3 : 0; + w=w + t + x[0]; + + q[0]=ulow(w); + c=uhigh(w); + if(core::resolve_add(c, q[0])!=0) + q[0]=0xFFFFFFFF; + } + + __device__ __forceinline__ static void dlimbs_sqrt_estimate(uint32_t q[DLIMBS], uint32_t top, const uint32_t x[DLIMBS], const uint32_t approx[DLIMBS]) { + uint32_t sync=core::sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t t, high, low; + uint64_t w; + + // computes: + // 1. num=((top<<32*LIMBS) + x) / 2 + // 2. q=(num*approx>>32*LIMBS) + num + 4 + // 3. q=min(q, (1<<32*LIMBS)-1); + // + // Note: Leaves junk in lower words of q + + // shift x right by 1 bit. Fill high bit with top. + t=__shfl_down_sync(sync, x[0], 1, TPI); + t=(group_thread==TPI-1) ? top : t; + low=uright_wrap(x[0], t, 1); + + // if we're exactly half the size, need to clear out low limb + if(TPI==2*LIMBS) + low=(group_thread>=LIMBS) ? low : 0; + + // estimate is in low + w=0; + #pragma unroll + for(int32_t index=0;index>32)+t; + } + + // increase the estimate by 4 -- because we might have cleared low bit, estimate can be off by 4 + t=(group_thread==TPI-LIMBS) ? 4 : 0; + w=w + t + low; + + low=ulow(w); + high=uhigh(w); + if(core::resolve_add(high, low)!=0) + low=0xFFFFFFFF; + q[0]=low; + } +}; + +template +class dispatch_dlimbs_t { + public: + static const uint32_t TPI=core::TPI; + static const uint32_t LIMBS=core::LIMBS; + static const uint32_t DLIMBS=core::DLIMBS; + static const uint32_t LIMB_OFFSET=DLIMBS*TPI-LIMBS; + + // These algorithms are used then LIMBS<=TPI. Almost the same as the half size ones, few tweaks here and there. + + __device__ __forceinline__ static void dlimbs_approximate(uint32_t approx[DLIMBS], const uint32_t denom[DLIMBS]) { + uint32_t sync=core::sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t x, d0, d1, x0, x1, x2, est, a, h, l; + int32_t c, top; + + // computes (beta^2 - 1) / denom - beta, where beta=1<<32*LIMBS + + x=0xFFFFFFFF-denom[0]; + + d1=__shfl_sync(sync, denom[0], TPI-1, TPI); + d0=__shfl_sync(sync, denom[0], TPI-2, TPI); + + approx[0]=0; + a=uapprox(d1); + + #pragma nounroll + for(int32_t thread=LIMBS-1;thread>=0;thread--) { + x0=__shfl_sync(sync, x, TPI-3, TPI); + x1=__shfl_sync(sync, x, TPI-2, TPI); + x2=__shfl_sync(sync, x, TPI-1, TPI); + est=udiv(x0, x1, x2, d0, d1, a); + + l=madlo_cc(est, denom[0], 0); + h=madhic(est, denom[0], 0); + + x=sub_cc(x, h); + c=subc(0, 0); // thread TPI-1 is zero + + top=__shfl_sync(sync, x, TPI-1, TPI); + x=__shfl_up_sync(sync, x, 1, TPI); + c=__shfl_sync(sync, c, threadIdx.x-1, TPI); + x=(group_thread==0) ? 0xFFFFFFFF : x; + + x=sub_cc(x, l); + c=subc(c, 0); + + if(top+core::resolve_sub(c, x)<0) { + // means a correction is required, should be very rare + x=add_cc(x, denom[0]); + c=addc(0, 0); + core::fast_propagate_add(c, x); + est--; + } + approx[0]=(group_thread==thread+TPI-LIMBS) ? est : approx[0]; + } + } + + __device__ __forceinline__ static uint32_t dlimbs_sqrt_rem_wide(uint32_t s[DLIMBS], uint32_t r[DLIMBS], const uint32_t lo[DLIMBS], const uint32_t hi[DLIMBS]) { + uint32_t sync=core::sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t x, x0, x1, t0, t1, divisor, approx, p, q, c, low; + + // computes s=sqrt(x), r=x-s^2, where x=(hi<<32*LIMBS) + lo + + low=lo[0]; + x=hi[0]; + if(TPI!=LIMBS) { + low=__shfl_sync(sync, low, threadIdx.x-TPI+LIMBS, TPI); + x=((int32_t)group_thread>=(int32_t)(TPI-LIMBS)) ? x : low; // use casts to silence warning + } + x0=__shfl_sync(sync, x, TPI-2, TPI); + x1=__shfl_sync(sync, x, TPI-1, TPI); + + divisor=usqrt(x0, x1); + approx=uapprox(divisor); + + t0=madlo_cc(divisor, divisor, 0); + t1=madhic(divisor, divisor, 0); + x0=sub_cc(x0, t0); + x1=subc(x1, t1); + + x=(group_thread==TPI-1) ? low : x; + x=__shfl_sync(sync, x, threadIdx.x-1, TPI); + x=(group_thread==TPI-1) ? x0 : x; + s[0]=(group_thread==TPI-1) ? divisor+divisor : 0; // silent 1 at the top of s + + #pragma nounroll + for(int32_t index=TPI-2;index>=(int32_t)(TPI-LIMBS);index--) { + x0=__shfl_sync(sync, x, TPI-1, TPI); + q=usqrt_div(x0, x1, divisor, approx); + s[0]=(group_thread==index) ? q : s[0]; + + p=madhi(q, s[0], 0); + x=sub_cc(x, p); + c=subc(0, 0); + core::fast_propagate_sub(c, x); + + x1=__shfl_sync(sync, x, TPI-1, TPI)-q; // we subtract q because of the silent 1 at the top of s + t0=__shfl_sync(sync, low, index, TPI); + x=__shfl_up_sync(sync, x, 1, TPI); + x=(group_thread==0) ? t0 : x; + + p=madlo(q, s[0], 0); + x=sub_cc(x, p); + c=subc(0, 0); + x1-=core::fast_propagate_sub(c, x); + + while(0>(int32_t)x1) { + x1++; + q--; + + // correction step: add q and s + x=add_cc(x, (group_thread==index) ? q : 0); + c=addc(0, 0); + x=add_cc(x, s[0]); + c=addc(c, 0); + + x1+=core::resolve_add(c, x); + + // update s + s[0]=(group_thread==index) ? q : s[0]; + } + s[0]=(group_thread==index+1) ? s[0]+(q>>31) : s[0]; + s[0]=(group_thread==index) ? q+q : s[0]; + } + t0=__shfl_down_sync(sync, s[0], 1, TPI); + t0=(group_thread==TPI-1) ? 1 : t0; + s[0]=uright_wrap(s[0], t0, 1); + r[0]=x; + return x1; + } + + __device__ __forceinline__ static void dlimbs_div_estimate(uint32_t q[DLIMBS], const uint32_t x[DLIMBS], const uint32_t approx[DLIMBS]) { + uint32_t sync=core::sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t t, c; + uint64_t w; + + // computes q=(x*approx>>32*LIMBS) + x + 3 + // q=min(q, (1<<32*LIMBS)-1); + // + // Notes: leaves junk in lower words of q + + w=0; + #pragma unroll + for(int32_t index=0;index>32)+t; + } + + // increase the estimate by 3 + t=(group_thread==TPI-LIMBS) ? 3 : 0; + w=w + t + x[0]; + + q[0]=ulow(w); + c=uhigh(w); + if(core::resolve_add(c, q[0])!=0) + q[0]=0xFFFFFFFF; + } + + __device__ __forceinline__ static void dlimbs_sqrt_estimate(uint32_t q[DLIMBS], uint32_t top, const uint32_t x[DLIMBS], const uint32_t approx[DLIMBS]) { + uint32_t sync=core::sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t t, high, low; + uint64_t w; + + // computes: + // 1. num=((top<<32*LIMBS) + x) / 2 + // 2. q=(num*approx>>32*LIMBS) + num + 4 + // 3. q=min(q, (1<<32*LIMBS)-1); + // + // Note: Leaves junk in lower words of q + + // shift x right by 1 bit. Fill high bit with top. + t=__shfl_down_sync(sync, x[0], 1, TPI); + t=(group_thread==TPI-1) ? top : t; + low=uright_wrap(x[0], t, 1); + + // estimate is in low + w=0; + #pragma unroll + for(int32_t index=0;index>32)+t; + } + + // increase the estimate by 4 -- because we might have cleared low bit, estimate can be off by 4 + t=(group_thread==TPI-LIMBS) ? 4 : 0; + w=w + t + low; + + low=ulow(w); + high=uhigh(w); + if(core::resolve_add(high, low)!=0) + low=0xFFFFFFFF; + q[0]=low; + } +}; + +} /* namespace cgbn */ \ No newline at end of file diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_masking.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_masking.cu new file mode 100644 index 0000000..7337957 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_masking.cu @@ -0,0 +1,253 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +class dispatch_masking_t { + public: + static const uint32_t LIMBS=core::LIMBS; + static const uint32_t TPI=core::TPI; + + static const int32_t bits=(int32_t)core::BITS; + + __device__ __forceinline__ static void bitwise_mask_copy(uint32_t r[LIMBS], const int32_t numbits) { + int32_t group_thread=threadIdx.x & TPI-1, group_base=group_thread*LIMBS; + + if(numbits>=bits || numbits<=-bits) { + #pragma unroll + for(int32_t index=0;index=0) { + int32_t limb=(numbits>>5)-group_base; + int32_t straddle=uleft_wrap(0xFFFFFFFF, 0, numbits); + + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=0xFFFFFFFF; + else + r[index]=straddle; + } + } + else { + int32_t limb=(numbits+bits>>5)-group_base; + int32_t straddle=uleft_wrap(0, 0xFFFFFFFF, numbits); + + #pragma unroll + for(int32_t index=0;indexindex || group_base>=bits/32-index) + r[index]=0; + else if(limb=bits || numbits<=-bits) { + #pragma unroll + for(int32_t index=0;index=0) { + int32_t limb=(numbits>>5)-group_base; + int32_t straddle=uleft_wrap(0xFFFFFFFF, 0, numbits); + + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=a[index]; + else + r[index]=a[index] & straddle; + } + } + else { + int32_t limb=(numbits+bits>>5)-group_base; + int32_t straddle=uleft_wrap(0, 0xFFFFFFFF, numbits); + + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=0; + else if(limb=bits || numbits<=-bits) { + #pragma unroll + for(int32_t index=0;index=0) { + int32_t limb=(numbits>>5)-group_base; + int32_t straddle=uleft_wrap(0xFFFFFFFF, 0, numbits); + + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=0xFFFFFFFF; + else + r[index]=a[index] | straddle; + } + } + else { + int32_t limb=(numbits+bits>>5)-group_base; + int32_t straddle=uleft_wrap(0, 0xFFFFFFFF, numbits); + + #pragma unroll + for(int32_t index=0;indexindex || group_base>=bits/32-index) + r[index]=a[index]; + else if(limb=bits || numbits<=-bits) { + #pragma unroll + for(int32_t index=0;index=0) { + int32_t limb=(numbits>>5)-group_base; + int32_t straddle=uleft_wrap(0xFFFFFFFF, 0, numbits); + + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=a[index] ^ 0xFFFFFFFF; + else + r[index]=a[index] ^ straddle; + } + } + else { + int32_t limb=(numbits+bits>>5)-group_base; + int32_t straddle=uleft_wrap(0, 0xFFFFFFFF, numbits); + + #pragma unroll + for(int32_t index=0;indexindex || group_base>=bits/32-index) + r[index]=a[index]; + else if(limb=bits || numbits<=-bits) { + #pragma unroll + for(int32_t index=0;index=0) { + int32_t limb=(numbits>>5)-group_base; + int32_t straddle=uleft_wrap(0xFFFFFFFF, 0, numbits); + + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=set[index]; + else + r[index]=(set[index] & straddle) | (clear[index] & ~straddle); + } + } + else { + int32_t limb=(bits+numbits>>5)-group_base; + int32_t straddle=uleft_wrap(0, 0xFFFFFFFF, numbits); + + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=clear[index]; + else + r[index]=(set[index] & straddle) | (clear[index] & ~straddle); + } + } + } +}; + +template +class dispatch_masking_t { + public: + static const uint32_t LIMBS=core::LIMBS; + static const uint32_t TPI=core::TPI; + + __device__ __forceinline__ static void bitwise_mask_copy(uint32_t r[LIMBS], const int32_t numbits) { + dmask_set(r, numbits); + } + + __device__ __forceinline__ static void bitwise_mask_and(uint32_t r[LIMBS], const uint32_t a[LIMBS], const int32_t numbits) { + dmask_and(r, a, numbits); + } + + __device__ __forceinline__ static void bitwise_mask_ior(uint32_t r[LIMBS], const uint32_t a[LIMBS], const int32_t numbits) { + dmask_ior(r, a, numbits); + } + + __device__ __forceinline__ static void bitwise_mask_xor(uint32_t r[LIMBS], const uint32_t a[LIMBS], const int32_t numbits) { + dmask_xor(r, a, numbits); + } + + __device__ __forceinline__ static void bitwise_mask_select(uint32_t r[LIMBS], const uint32_t clear[LIMBS], const uint32_t set[LIMBS], const int32_t numbits) { + dmask_select(r, clear, set, numbits); + } +}; + +} /* namespace cgbn */ diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_padding.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_padding.cu new file mode 100644 index 0000000..cd55e06 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_padding.cu @@ -0,0 +1,91 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +struct dispatch_padding_t { + public: + static const uint32_t TPI=core::TPI; + static const uint32_t LIMBS=core::LIMBS; + static const uint32_t BITS=core::BITS; + + static const uint32_t PAD_THREAD=core::PAD_THREAD; + static const uint32_t PAD_LIMB=core::PAD_LIMB; + + __device__ __forceinline__ static uint32_t clear_carry(uint32_t &x) { + uint32_t sync=core::sync_mask(), group_thread=threadIdx.x & TPI-1; + uint32_t result; + + result=__shfl_sync(sync, x, PAD_THREAD, TPI); + x=(group_thread +struct dispatch_padding_t { + public: + static const uint32_t LIMBS=core::LIMBS; + + __device__ __forceinline__ static uint32_t clear_carry(uint32_t &x) { + return 0; + } + + __device__ __forceinline__ static uint32_t clear_carry(uint32_t x[LIMBS]) { + return 0; + } + + __device__ __forceinline__ static void clear_padding(uint32_t &x) { + } + + __device__ __forceinline__ static void clear_padding(uint32_t x[LIMBS]) { + } +}; + +} /* namespace cgbn */ diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_resolver.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_resolver.cu new file mode 100644 index 0000000..4218e11 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_resolver.cu @@ -0,0 +1,27 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +#include "padded_resolver.cu" +#include "warp_resolver.cu" +#include "subwarp_resolver.cu" diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_shift_rotate.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_shift_rotate.cu new file mode 100644 index 0000000..a0ad9e6 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/dispatch_shift_rotate.cu @@ -0,0 +1,136 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +class dispatch_shift_rotate_t { + public: + static const uint32_t TPI=core::TPI; + static const uint32_t LIMBS=core::LIMBS; + static const uint32_t BITS=core::BITS; + static const uint32_t MAX_ROTATION=core::MAX_ROTATION; + + __device__ __forceinline__ static void shift_left(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t numbits) { + uint32_t group_thread=threadIdx.x & TPI-1, group_base=group_thread*LIMBS; + int32_t delta; + + if(numbits(core::sync_mask(), r, a, numbits); + + delta=(numbits>>5)-group_base; + #pragma unroll + for(int32_t index=0;index=BITS/32-index || delta>index) + r[index]=0; + } + else + mpzero(r); + } + + __device__ __forceinline__ static void shift_right(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t numbits) { + uint32_t group_thread=threadIdx.x & TPI-1, group_base=group_thread*LIMBS; + int32_t delta; + + if(numbits(core::sync_mask(), r, a, numbits); + + delta=group_base-(BITS-numbits>>5); + #pragma unroll + for(int32_t index=0;indexindex) + r[index]=0; + } + else + mpzero(r); + } + + __device__ __forceinline__ static void rotate_left(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t numbits) { + uint32_t group_thread=threadIdx.x & TPI-1, group_base=group_thread*LIMBS; + uint32_t amount=numbits%BITS, temp[LIMBS]; + + drotate_left(core::sync_mask(), temp, a, amount+TPI*LIMBS*32-BITS); + drotate_left(core::sync_mask(), r, a, amount); + + dmask_select(r, r, temp, amount); + + // clear padding + #pragma unroll + for(int32_t index=0;index=BITS/32-index) + r[index]=0; + } + + __device__ __forceinline__ static void rotate_right(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t numbits) { + uint32_t group_thread=threadIdx.x & TPI-1, group_base=group_thread*LIMBS; + uint32_t amount=numbits%BITS, temp[LIMBS]; + + drotate_right(core::sync_mask(), temp, a, amount); + drotate_right(core::sync_mask(), r, a, amount+TPI*LIMBS*32-BITS); + + dmask_select(r, r, temp, BITS-amount); + + #pragma unroll + for(int32_t index=0;index=BITS/32-index) + r[index]=0; + } +}; + +template +class dispatch_shift_rotate_t { + public: + static const uint32_t TPI=core::TPI; + static const uint32_t LIMBS=core::LIMBS; + static const uint32_t BITS=core::BITS; + static const uint32_t MAX_ROTATION=core::MAX_ROTATION; + + __device__ __forceinline__ static void shift_left(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t numbits) { + if(numbits(core::sync_mask(), r, a, numbits); + dmask_and(r, r, (int32_t)(numbits-BITS)); + } + else + mpzero(r); + } + + __device__ __forceinline__ static void shift_right(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t numbits) { + if(numbits(core::sync_mask(), r, a, numbits); + dmask_and(r, r, BITS-numbits); + } + else + mpzero(r); + } + + __device__ __forceinline__ static void rotate_left(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t numbits) { + drotate_left(core::sync_mask(), r, a, numbits); + } + + __device__ __forceinline__ static void rotate_right(uint32_t r[LIMBS], const uint32_t a[LIMBS], const uint32_t numbits) { + drotate_right(core::sync_mask(), r, a, numbits); + } +}; + +} /* namespace cgbn */ \ No newline at end of file diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/padded_resolver.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/padded_resolver.cu new file mode 100644 index 0000000..9b26a29 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/padded_resolver.cu @@ -0,0 +1,288 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +class dispatch_resolver_t { + public: + static const uint32_t BITS=core::BITS; + static const uint32_t LIMBS=core::LIMBS; + + static const uint32_t PAD_THREAD=core::PAD_THREAD; + static const uint32_t PAD_LIMB=core::PAD_LIMB; + + static const uint32_t tpi_ONES=(1ull<(x); + p=__ballot_sync(sync, lor==0); + c=(p+GROUP_ONES^p)&lane; + + chain_t<> chain; + chain.add(c, 0xFFFFFFFF); + #pragma unroll + for(int32_t index=0;index(x); + g=__ballot_sync(sync, carry==1); + p=__ballot_sync(sync, land==0xFFFFFFFF); + + c=lane & (g+g+p ^ p); + + x[0]=add_cc(x[0], c!=0); + #pragma unroll + for(int32_t index=1;index(x); + g=__ballot_sync(sync, carry==0xFFFFFFFF); + p=__ballot_sync(sync, lor==0); + + c=lane & (g+g+p ^ p); + c=(c==0) ? 0 : 0xFFFFFFFF; + + x[0]=add_cc(x[0], c); + #pragma unroll + for(int32_t index=1;index(x); + g=__ballot_sync(sync, c==1); + p=__ballot_sync(sync, land==0xFFFFFFFF); + + c=lane & (g+g+p ^ p); + + x[0]=add_cc(x[0], c!=0); + #pragma unroll + for(int32_t index=1;index>31); + + g=__ballot_sync(sync, c==0xFFFFFFFF); + p=__ballot_sync(sync, x==0); + + c=lane & (g+g+p ^ p); + x=x-(c!=0); + + result=__shfl_sync(sync, x, padding, tpi); + x=(group_thread>31; + #pragma unroll + for(int32_t index=1;index(x); + g=__ballot_sync(sync, c==0xFFFFFFFF); + p=__ballot_sync(sync, lor==0); + + c=lane & (g+g+p ^ p); + + c=(c==0) ? 0 : 0xFFFFFFFF; + x[0]=add_cc(x[0], c); + #pragma unroll + for(int32_t index=1;index +class dispatch_resolver_t { + public: + static const uint32_t LIMBS=core::LIMBS; + static const uint32_t TPI_ONES=(1ull<(x); + p=__ballot_sync(sync, lor==0 && group_thread!=tpi-1); + c=(p+GROUP_ONES^p)&lane; + + chain_t<> chain; + chain.add(c, 0xFFFFFFFF); + #pragma unroll + for(int32_t index=0;index>32, 0, threadIdx.x - group_thread + tpi) & 0x01; + } + + __device__ __forceinline__ static int32_t fast_propagate_add(const uint32_t carry, uint32_t x[LIMBS]) { + uint32_t sync=core::sync_mask(), group_thread=threadIdx.x & tpi-1, warp_thread=threadIdx.x & warpSize-1; + uint32_t lane=(group_thread==0) ? 0 : 1<(x); + g=__ballot_sync(sync, carry==1); + p=__ballot_sync(sync, land==0xFFFFFFFF && group_thread!=0); + + // wrap the carry around + sum=make_wide(g, g) + make_wide(g, g) + make_wide(p, p); + c=lane & (p ^ sum); + + x[0]=add_cc(x[0], c!=0); + #pragma unroll + for(int32_t index=1;index>32, 0, threadIdx.x - group_thread + tpi) & 0x01; + } + + + /**************************************************************** + * returns 1 if borrows out + * returns 0 otherwise + ****************************************************************/ + __device__ __forceinline__ static int32_t fast_propagate_sub(const uint32_t carry, uint32_t &x) { + uint32_t sync=core::sync_mask(), group_thread=threadIdx.x & tpi-1; + uint32_t lane=(group_thread==0) ? 0 : 1<<(threadIdx.x & warpSize-1); + uint32_t g, p, c; + uint64_t sum; + + g=__ballot_sync(sync, carry==0xFFFFFFFF); + p=__ballot_sync(sync, x==0 && group_thread!=0); + + // wrap the carry around + sum=make_wide(g, g) + make_wide(g, g) + make_wide(p, p); + c=lane & (p ^ sum); + + x=x-(c!=0); + + return uright_wrap(sum>>32, 0, threadIdx.x - group_thread + tpi) & 0x01; + } + + __device__ __forceinline__ static int32_t fast_propagate_sub(const uint32_t carry, uint32_t x[LIMBS]) { + uint32_t sync=core::sync_mask(), group_thread=threadIdx.x & tpi-1; + uint32_t lane=(group_thread==0) ? 0 : 1<<(threadIdx.x & warpSize-1); + uint32_t lor, g, p, c; + uint64_t sum; + + lor=mplor(x); + g=__ballot_sync(sync, carry==0xFFFFFFFF); + p=__ballot_sync(sync, lor==0 && group_thread!=0); + + // wrap the carry around + sum=make_wide(g, g) + make_wide(g, g) + make_wide(p, p); + c=lane & (p ^ sum); + c=(c==0) ? 0 : 0xFFFFFFFF; + + x[0]=add_cc(x[0], c); + #pragma unroll + for(int32_t index=1;index>32, 0, threadIdx.x - group_thread + tpi) & 0x01; + } + + + /**************************************************************** + * returns the high word + ****************************************************************/ + __device__ __forceinline__ static int32_t resolve_add(const int32_t carry, uint32_t &x) { + uint32_t sync=core::sync_mask(), group_thread=threadIdx.x & tpi-1; + uint32_t lane=(group_thread==0) ? 0 : 1<<(threadIdx.x & warpSize-1); + uint32_t g, p, c; + uint64_t sum; + + c=__shfl_up_sync(sync, carry, 1, tpi); + c=(group_thread==0) ? 0 : c; + x=add_cc(x, c); + c=addc(0, 0); + + g=__ballot_sync(sync, c==1); + p=__ballot_sync(sync, x==0xFFFFFFFF && group_thread!=0); + + // wrap the carry around + sum=make_wide(g, g) + make_wide(g, g) + make_wide(p, p); + c=lane&(p^sum); + + x=x+(c!=0); + c=uright_wrap(sum>>32, 0, threadIdx.x - group_thread + tpi) & 0x01; + return __shfl_sync(sync, carry+c, tpi-1, tpi); + } + + __device__ __forceinline__ static int32_t resolve_add(const int32_t carry, uint32_t x[LIMBS]) { + uint32_t sync=core::sync_mask(), group_thread=threadIdx.x & tpi-1; + uint32_t lane=(group_thread==0) ? 0 : 1<<(threadIdx.x & warpSize-1); + uint32_t g, p, c, land; + uint64_t sum; + + c=__shfl_up_sync(sync, carry, 1, tpi); + c=(group_thread==0) ? 0 : c; + x[0]=add_cc(x[0], c); + #pragma unroll + for(int32_t index=1;index(x); + g=__ballot_sync(sync, c==1); + p=__ballot_sync(sync, land==0xFFFFFFFF && group_thread!=0); + + sum=make_wide(g, g) + make_wide(g, g) + make_wide(p, p); + c=lane&(p^sum); + + x[0]=add_cc(x[0], c!=0); + #pragma unroll + for(int32_t index=1;index>32, 0, threadIdx.x - group_thread + tpi) & 0x01; + return __shfl_sync(sync, carry+c, tpi-1, tpi); + } + + + /**************************************************************** + * returns the high word + ****************************************************************/ + __device__ __forceinline__ static int32_t resolve_sub(const int32_t carry, uint32_t &x) { + uint32_t sync=core::sync_mask(), group_thread=threadIdx.x & tpi-1; + uint32_t lane=(group_thread==0) ? 0 : 1<<(threadIdx.x & warpSize-1); + uint32_t g, p; + int32_t c; + uint64_t sum; + + c=__shfl_up_sync(sync, carry, 1, tpi); + c=(group_thread==0) ? 0 : c; + x=add_cc(x, c); + c=addc(0, c>>31); + + g=__ballot_sync(sync, c==0xFFFFFFFF); + p=__ballot_sync(sync, x==0 && group_thread!=0); + + // wrap the carry around + sum=make_wide(g, g) + make_wide(g, g) + make_wide(p, p); + c=lane&(p^sum); + + x=x-(c!=0); + c=uright_wrap(sum>>32, 0, threadIdx.x - group_thread + tpi) & 0x01; + return __shfl_sync(sync, carry-c, tpi-1, tpi); + } + + __device__ __forceinline__ static int32_t resolve_sub(const int32_t carry, uint32_t x[LIMBS]) { + uint32_t sync=core::sync_mask(), group_thread=threadIdx.x & tpi-1; + uint32_t lane=(group_thread==0) ? 0 : 1<<(threadIdx.x & warpSize-1); + uint32_t g, p, lor; + int32_t c; + uint64_t sum; + + c=__shfl_up_sync(sync, carry, 1, tpi); + c=(group_thread==0) ? 0 : c; + x[0]=add_cc(x[0], c); + c=c>>31; + #pragma unroll + for(int32_t index=1;index(x); + g=__ballot_sync(sync, c==0xFFFFFFFF); + p=__ballot_sync(sync, lor==0 && group_thread!=0); + + // wrap the carry around + sum=make_wide(g, g) + make_wide(g, g) + make_wide(p, p); + c=lane&(p^sum); + c=(c==0) ? 0 : 0xFFFFFFFF; + x[0]=add_cc(x[0], c); + #pragma unroll + for(int32_t index=1;index>32, 0, threadIdx.x - group_thread + tpi) & 0x01; + return __shfl_sync(sync, carry-c, tpi-1, tpi); + } + +}; + +} /* namespace cgbn */ + diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/unpadded.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/unpadded.cu new file mode 100644 index 0000000..5132ec5 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/unpadded.cu @@ -0,0 +1,41 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +class unpadded_t { + public: + static const uint32_t TPB=params::TPB; + static const uint32_t TPI=params::TPI; + static const uint32_t LIMBS=params::LIMBS; + static const uint32_t BITS=TPI*LIMBS*32; + static const uint32_t PADDING=0; + static const uint32_t MAX_ROTATION=params::MAX_ROTATION; + static const uint32_t SHM_LIMIT=params::SHM_LIMIT; + static const bool CONSTANT_TIME=params::CONSTANT_TIME; + static const cgbn_syncable_t SYNCABLE=params::SYNCABLE; +}; + +} /* namespace cgbn */ \ No newline at end of file diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/warp_resolver.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/warp_resolver.cu new file mode 100644 index 0000000..09cb511 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/core/warp_resolver.cu @@ -0,0 +1,257 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +namespace cgbn { + +template +class dispatch_resolver_t { + public: + static const uint32_t LIMBS=core::LIMBS; + + /**************************************************************** + * returns 0 if all bits are zero + * returns -1 otherwise + ****************************************************************/ + __device__ __forceinline__ static int32_t fast_negate(uint32_t &x) { + uint32_t sync=0xFFFFFFFF, warp_thread=threadIdx.x & warpSize-1, lane=1<(x); + p=__ballot_sync(sync, lor==0); + c=(p+1^p)&lane; + + chain_t<> chain; + chain.add(c, 0xFFFFFFFF); + #pragma unroll + for(int32_t index=0;index>32; // -(p==0xFFFFFFFF); + } + + __device__ __forceinline__ static int32_t fast_propagate_add(const uint32_t carry, uint32_t x[LIMBS]) { + uint32_t sync=0xFFFFFFFF, warp_thread=threadIdx.x & warpSize-1, lane=1<(x); + g=__ballot_sync(sync, carry==1); + p=__ballot_sync(sync, land==0xFFFFFFFF); + + sum=(uint64_t)g+(uint64_t)g+(uint64_t)p; + c=lane & (p ^ sum); + + x[0]=add_cc(x[0], c!=0); + #pragma unroll + for(int32_t index=1;index>32; // -(p==0xFFFFFFFF); + } + + + /**************************************************************** + * returns 1 if borrows out + * returns 0 otherwise + ****************************************************************/ + + __device__ __forceinline__ static int32_t fast_propagate_sub(const uint32_t carry, uint32_t &x) { + uint32_t sync=0xFFFFFFFF, warp_thread=threadIdx.x & warpSize-1, lane=1<>32); // -(p==0xFFFFFFFF); + } + + __device__ __forceinline__ static int32_t fast_propagate_sub(const uint32_t carry, uint32_t x[LIMBS]) { + uint32_t sync=0xFFFFFFFF, warp_thread=threadIdx.x & warpSize-1, lane=1<(x); + g=__ballot_sync(sync, carry==0xFFFFFFFF); + p=__ballot_sync(sync, lor==0); + + sum=(uint64_t)g+(uint64_t)g+(uint64_t)p; + c=lane & (p ^ sum); + c=(c==0) ? 0 : 0xFFFFFFFF; + + x[0]=add_cc(x[0], c); + #pragma unroll + for(int32_t index=1;index>32; // -(p==0xFFFFFFFF); + } + + + /**************************************************************** + * returns the high word + ****************************************************************/ + __device__ __forceinline__ static int32_t resolve_add(const int32_t carry, uint32_t &x) { + uint32_t sync=0xFFFFFFFF, warp_thread=threadIdx.x & warpSize-1, lane=1<>32); + return __shfl_sync(sync, c, 31); + } + + __device__ __forceinline__ static int32_t resolve_add(const int32_t carry, uint32_t x[LIMBS]) { + uint32_t sync=0xFFFFFFFF, warp_thread=threadIdx.x & warpSize-1, lane=1<(x); + g=__ballot_sync(sync, c==1); + p=__ballot_sync(sync, land==0xFFFFFFFF); + + sum=(uint64_t)g+(uint64_t)g+(uint64_t)p; + c=lane&(p^sum); + x[0]=add_cc(x[0], c!=0); + #pragma unroll + for(int32_t index=1;index>32); + return __shfl_sync(sync, c, 31); + } + + /**************************************************************** + * returns the high word + ****************************************************************/ + __device__ __forceinline__ static int32_t resolve_sub(const int32_t carry, uint32_t &x) { + uint32_t sync=0xFFFFFFFF, warp_thread=threadIdx.x & warpSize-1, lane=1<>31); + + g=__ballot_sync(sync, c==0xFFFFFFFF); + p=__ballot_sync(sync, x==0); + + // wrap the carry around + sum=(uint64_t)g+(uint64_t)g+(uint64_t)p; + c=lane&(p^sum); + + x=x-(c!=0); + c=carry-(sum>>32); + return __shfl_sync(sync, c, 31); + } + + __device__ __forceinline__ static int32_t resolve_sub(const int32_t carry, uint32_t x[LIMBS]) { + uint32_t sync=0xFFFFFFFF, warp_thread=threadIdx.x & warpSize-1, lane=1<>31; + #pragma unroll + for(int32_t index=1;index(x); + g=__ballot_sync(sync, c==0xFFFFFFFF); + p=__ballot_sync(sync, lor==0); + + // wrap the carry around + sum=(uint64_t)g+(uint64_t)g+(uint64_t)p; + c=lane&(p^sum); + c=(c==0) ? 0 : 0xFFFFFFFF; + x[0]=add_cc(x[0], c); + #pragma unroll + for(int32_t index=1;index>32); + return __shfl_sync(sync, c, 31); + } +}; + +} /* namespace cgbn */ diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/gpu_support.h b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/gpu_support.h new file mode 100644 index 0000000..58eda0e --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/gpu_support.h @@ -0,0 +1,66 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +// support routines +void cuda_check(cudaError_t status, const char *action=NULL, const char *file=NULL, int32_t line=0) { + // check for cuda errors + + if(status!=cudaSuccess) { + fprintf(stderr, "CUDA error occurred: %s\n", cudaGetErrorString(status)); + if(action!=NULL) + fprintf(stderr, "While running %s (file %s, line %d)\n", action, file, line); + // exit(1); + throw std::runtime_error(cudaGetErrorString(status)); + } +} + +void cgbn_check(cgbn_error_report_t *report, const char *file=NULL, int32_t line=0) { + // check for cgbn errors + + if(cgbn_error_report_check(report)) { + printf("\n"); + printf("CGBN error occurred: %s\n", cgbn_error_string(report)); + + if(report->_instance!=0xFFFFFFFF) { + printf("Error reported by instance %d", report->_instance); + if(report->_blockIdx.x!=0xFFFFFFFF || report->_threadIdx.x!=0xFFFFFFFF) + printf(", "); + if(report->_blockIdx.x!=0xFFFFFFFF) + printf("blockIdx=(%d, %d, %d) ", report->_blockIdx.x, report->_blockIdx.y, report->_blockIdx.z); + if(report->_threadIdx.x!=0xFFFFFFFF) + printf("threadIdx=(%d, %d, %d)", report->_threadIdx.x, report->_threadIdx.y, report->_threadIdx.z); + printf("\n"); + } + else { + printf("Error reported by blockIdx=(%d %d %d)", report->_blockIdx.x, report->_blockIdx.y, report->_blockIdx.z); + printf("threadIdx=(%d %d %d)\n", report->_threadIdx.x, report->_threadIdx.y, report->_threadIdx.z); + } + if(file!=NULL) + printf("file %s, line %d\n", file, line); + exit(1); + } +} + +#define CUDA_CHECK(action) cuda_check(action, #action, __FILE__, __LINE__) +#define CGBN_CHECK(report) cgbn_check(report, __FILE__, __LINE__) diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/impl_cuda.cu b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/impl_cuda.cu new file mode 100644 index 0000000..cd9ebca --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/impl_cuda.cu @@ -0,0 +1,1431 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +#include "arith/arith.h" +#include "core/unpadded.cu" +#include "core/core.cu" +#include "core/core_singleton.cu" + +#if(__CUDACC_VER_MAJOR__<9 || (__CUDACC_VER_MAJOR__==9 && __CUDACC_VER_MINOR__<2)) + #if __CUDA_ARCH__>=700 + #error CGBN requires CUDA version 9.2 or above on Volta + #endif +#endif + +/**************************************************************************************************************** + * cgbn_context_t implementation for CUDA + ****************************************************************************************************************/ +template +__device__ __forceinline__ cgbn_context_t::cgbn_context_t() : _monitor(cgbn_no_checks), _report(NULL), _instance(0xFFFFFFFF) { +} + +template +__device__ __forceinline__ cgbn_context_t::cgbn_context_t(cgbn_monitor_t monitor) : _monitor(monitor), _report(NULL), _instance(0xFFFFFFFF) { + if(monitor!=cgbn_no_checks) { + if(tpi!=32 && tpi!=16 && tpi!=8 && tpi!=4) + report_error(cgbn_unsupported_threads_per_instance); + if(params::TPB!=0 && params::TPB!=blockDim.x) + report_error(cgbn_threads_per_block_mismatch); + if(params::CONSTANT_TIME) + report_error(cgbn_unsupported_operation); + } +} + +template +__device__ __forceinline__ cgbn_context_t::cgbn_context_t(cgbn_monitor_t monitor, cgbn_error_report_t *report) : _monitor(monitor), _report(report), _instance(0xFFFFFFFF) { + if(monitor!=cgbn_no_checks) { + if(tpi!=32 && tpi!=16 && tpi!=8 && tpi!=4) + report_error(cgbn_unsupported_threads_per_instance); + if(params::TPB!=0 && params::TPB!=blockDim.x) + report_error(cgbn_threads_per_block_mismatch); + if(params::CONSTANT_TIME) + report_error(cgbn_unsupported_operation); + } +} + +template +__device__ __forceinline__ cgbn_context_t::cgbn_context_t(cgbn_monitor_t monitor, cgbn_error_report_t *report, uint32_t instance) : _monitor(monitor), _report(report), _instance(instance) { + if(monitor!=cgbn_no_checks) { + if(tpi!=32 && tpi!=16 && tpi!=8 && tpi!=4) + report_error(cgbn_unsupported_threads_per_instance); + if(params::TPB!=0 && params::TPB!=blockDim.x) + report_error(cgbn_threads_per_block_mismatch); + if(params::CONSTANT_TIME) + report_error(cgbn_unsupported_operation); + } +} + +template +__device__ __forceinline__ bool cgbn_context_t::check_errors() const { + return _monitor!=cgbn_no_checks; +} + +template +__device__ __noinline__ void cgbn_context_t::report_error(cgbn_error_t error) const { + if((threadIdx.x & tpi-1)==0) { + if(_report!=NULL) { + if(atomicCAS((uint32_t *)&(_report->_error), (uint32_t)cgbn_no_error, (uint32_t)error)==cgbn_no_error) { + _report->_instance=_instance; + _report->_threadIdx=threadIdx; + _report->_blockIdx=blockIdx; + } + } + + if(_monitor==cgbn_print_monitor) { + switch(_report->_error) { + case cgbn_unsupported_threads_per_instance: + printf("cgbn error: unsupported threads per instance\n"); + break; + case cgbn_unsupported_size: + printf("cgbn error: unsupported size\n"); + break; + case cgbn_unsupported_limbs_per_thread: + printf("cgbn error: unsupported limbs per thread\n"); + break; + case cgbn_unsupported_operation: + printf("cgbn error: unsupported operation\n"); + break; + case cgbn_threads_per_block_mismatch: + printf("cgbn error: TPB does not match blockDim.x\n"); + break; + case cgbn_threads_per_instance_mismatch: + printf("cgbn errpr: TPI does not match env_t::TPI\n"); + break; + case cgbn_division_by_zero_error: + printf("cgbn error: division by zero on instance\n"); + break; + case cgbn_division_overflow_error: + printf("cgbn error: division overflow on instance\n"); + break; + case cgbn_invalid_montgomery_modulus_error: + printf("cgbn error: division invalid montgomery modulus\n"); + break; + case cgbn_modulus_not_odd_error: + printf("cgbn error: invalid modulus (it must be odd)\n"); + break; + case cgbn_inverse_does_not_exist_error: + printf("cgbn error: inverse does not exist\n"); + break; + default: + printf("cgbn error: unknown error reported by instance\n"); + break; + } + } + else if(_monitor==cgbn_halt_monitor) { + __trap(); + } + } +} + +/* +template template +__device__ __forceinline__ cgbn_env_t cgbn_context_t::env() { + cgbn_env_t env(this); + + return env; +} + +template template + __device__ __forceinline__ cgbn_env_t cgbn_context_t::env() { + return env(); +} +*/ + +/**************************************************************************************************************** + * cgbn_env_t implementation for CUDA + ****************************************************************************************************************/ + +/* constructor */ +template +__device__ __forceinline__ cgbn_env_t::cgbn_env_t(const context_t &context) : _context(context) { + if(_context.check_errors()) { + if(bits==0 || (bits & 0x1F)!=0) + _context.report_error(cgbn_unsupported_size); + } +} + +template +__device__ __forceinline__ void cgbn_env_t::set(cgbn_t &r, const cgbn_t &a) const { + cgbn::core_t::set(r._limbs, a._limbs); +} + +template +__device__ __forceinline__ void cgbn_env_t::swap(cgbn_t &r, cgbn_t &a) const { + cgbn::core_t::swap(r._limbs, a._limbs); +} + +template template +__device__ __forceinline__ void cgbn_env_t::set(cgbn_t &r, const source_cgbn_t &source) const { + uint32_t sync, group_thread=threadIdx.x & TPI-1; + uint32_t source_thread=0, source_limb=0, value; + + // TPI and TPB must match. TPB matches automatically + if(_context.check_errors()) { + if(TPI!=source_cgbn_t::parent_env_t::TPI) { + _context.report_error(cgbn_threads_per_instance_mismatch); + return; + } + } + + sync=cgbn::core_t::sync_mask(); + cgbn::mpzero(r._limbs); + #pragma nounroll + for(int32_t index=0;index +__device__ __forceinline__ void cgbn_env_t::extract_bits(cgbn_t &r, const cgbn_t &a, const uint32_t start, const uint32_t len) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core; + + uint32_t local_len=len; + + if(start>=BITS) { + cgbn::mpzero(r._limbs); + return; + } + + local_len=cgbn::umin(local_len, BITS-start); + + core::rotate_right(r._limbs, a._limbs, start); + core::bitwise_mask_and(r._limbs, r._limbs, local_len); +} + +template +__device__ __forceinline__ void cgbn_env_t::insert_bits(cgbn_t &r, const cgbn_t &a, const uint32_t start, const uint32_t len, const cgbn_t &value) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core; + + uint32_t local_len=len; + uint32_t mask[LIMBS], temp[LIMBS]; + + if(start>=BITS) { + cgbn::mpset(r._limbs, a._limbs); + return; + } + + local_len=cgbn::umin(local_len, BITS-start); + + core::rotate_left(temp, value._limbs, start); + core::bitwise_mask_copy(mask, start+local_len); + core::bitwise_mask_xor(mask, mask, start); + core::bitwise_select(r._limbs, a._limbs, temp, mask); +} + +/* ui32 routines */ +template +__device__ __forceinline__ uint32_t cgbn_env_t::get_ui32(const cgbn_t &a) const { + return cgbn::core_t::get_ui32(a._limbs); +} + +template +__device__ __forceinline__ void cgbn_env_t::set_ui32(cgbn_t &r, const uint32_t value) const { + cgbn::core_t::set_ui32(r._limbs, value); +} + +template +__device__ __forceinline__ int32_t cgbn_env_t::add_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t add) const { + return cgbn::core_t::add_ui32(r._limbs, a._limbs, add); +} + +template +__device__ __forceinline__ int32_t cgbn_env_t::sub_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t sub) const { + return cgbn::core_t::sub_ui32(r._limbs, a._limbs, sub); +} + +template +__device__ __forceinline__ uint32_t cgbn_env_t::mul_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t mul) const { + return cgbn::core_t::mul_ui32(r._limbs, a._limbs, mul); +} + +template +__device__ __forceinline__ uint32_t cgbn_env_t::div_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t div) const { + if(div==0) { + if(_context.check_errors()) + _context.report_error(cgbn_division_by_zero_error); + return 0; + } + return cgbn::core_singleton_t::div_ui32(r._limbs, a._limbs, div); +} + +template +__device__ __forceinline__ uint32_t cgbn_env_t::rem_ui32(const cgbn_t &a, const uint32_t div) const { + if(div==0) { + if(_context.check_errors()) + _context.report_error(cgbn_division_by_zero_error); + return 0; + } + return cgbn::core_singleton_t::rem_ui32(a._limbs, div); +} + +template +__device__ __forceinline__ bool cgbn_env_t::equals_ui32(const cgbn_t &a, const uint32_t value) const { + return cgbn::core_t::equals_ui32(a._limbs, value); +} + +template +__device__ __forceinline__ int32_t cgbn_env_t::compare_ui32(const cgbn_t &a, const uint32_t value) const { + return cgbn::core_t::compare_ui32(a._limbs, value); +} + +template +__device__ __forceinline__ uint32_t cgbn_env_t::extract_bits_ui32(const cgbn_t &a, const uint32_t start, const uint32_t len) const { + return cgbn::core_t::extract_bits_ui32(a._limbs, start, len); +} + +template +__device__ __forceinline__ void cgbn_env_t::insert_bits_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t start, const uint32_t len, const uint32_t value) const { + cgbn::core_t::insert_bits_ui32(r._limbs, a._limbs, start, len, value); +} + +template +__device__ __forceinline__ uint32_t cgbn_env_t::binary_inverse_ui32(const uint32_t x) const { + if(_context.check_errors()) { + if((x & 0x01)==0) { + _context.report_error(cgbn_inverse_does_not_exist_error); + return 0; + } + } + return cgbn::ubinary_inverse(x); +} + +template +__device__ __forceinline__ uint32_t cgbn_env_t::gcd_ui32(const cgbn_t &a, const uint32_t value) const { + if(value==0) + return 0; + return cgbn::ugcd(value, rem_ui32(a, value)); +} + + +/* bn arithmetic routines */ +template +__device__ __forceinline__ int32_t cgbn_env_t::add(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const { + return cgbn::core_t::add(r._limbs, a._limbs, b._limbs); +} + +template +__device__ __forceinline__ int32_t cgbn_env_t::sub(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const { + return cgbn::core_t::sub(r._limbs, a._limbs, b._limbs); +} + +template +__device__ __forceinline__ int32_t cgbn_env_t::negate(cgbn_t &r, const cgbn_t &a) const { + return cgbn::core_t::negate(r._limbs, a._limbs); +} + +template +__device__ __forceinline__ void cgbn_env_t::mul(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const { + uint32_t add[LIMBS]; + + cgbn::mpzero(add); + cgbn::core_singleton_t::mul(r._limbs, a._limbs, b._limbs, add); +} + +template +__device__ __forceinline__ void cgbn_env_t::mul_high(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const { + uint32_t add[LIMBS]; + + cgbn::mpzero(add); + cgbn::core_singleton_t::mul_high(r._limbs, a._limbs, b._limbs, add); +} + +template +__device__ __forceinline__ void cgbn_env_t::sqr(cgbn_t &r, const cgbn_t &a) const { + uint32_t add[LIMBS]; + + cgbn::mpzero(add); + cgbn::core_singleton_t::mul(r._limbs, a._limbs, a._limbs, add); +} + +template +__device__ __forceinline__ void cgbn_env_t::sqr_high(cgbn_t &r, const cgbn_t &a) const { + uint32_t add[LIMBS]; + + cgbn::mpzero(add); + cgbn::core_singleton_t::mul_high(r._limbs, a._limbs, a._limbs, add); +} + +template +__device__ __forceinline__ void cgbn_env_t::div(cgbn_t &q, const cgbn_t &num, const cgbn_t &denom) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core; + typedef cgbn::core_singleton_t singleton; + + uint32_t num_low[LIMBS], num_high[LIMBS], denom_local[LIMBS]; + uint32_t shift, numthreads; + + if(_context.check_errors()) { + if(equals_ui32(denom, 0)) { + _context.report_error(cgbn_division_by_zero_error); + return; + } + } + + // division of padded values is the same as division of unpadded valuess + shift=core::clz(denom._limbs); + core::rotate_left(denom_local, denom._limbs, shift); + core::rotate_left(num_low, num._limbs, shift); + core::bitwise_mask_and(num_high, num_low, shift); + numthreads=TPI-core::clzt(num_high); + singleton::div_wide(q._limbs, num_low, num_high, denom_local, numthreads); +} + +template +__device__ __forceinline__ void cgbn_env_t::rem(cgbn_t &r, const cgbn_t &num, const cgbn_t &denom) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core; + typedef cgbn::core_singleton_t singleton; + + uint32_t num_low[LIMBS], num_high[LIMBS], denom_local[LIMBS]; + uint32_t shift, numthreads; + + if(_context.check_errors()) { + if(equals_ui32(denom, 0)) { + _context.report_error(cgbn_division_by_zero_error); + return; + } + } + // division of padded values is the same as division of unpadded valuess + shift=core::clz(denom._limbs); + core::rotate_left(denom_local, denom._limbs, shift); + core::rotate_left(num_low, num._limbs, shift); + core::bitwise_mask_and(num_high, num_low, shift); + core::bitwise_xor(num_low, num_low, num_high); + numthreads=TPI-core::clzt(num_high); + singleton::rem_wide(r._limbs, num_low, num_high, denom_local, numthreads); + core::rotate_right(r._limbs, r._limbs, shift); +} + +template +__device__ __forceinline__ void cgbn_env_t::div_rem(cgbn_t &q, cgbn_t &r, const cgbn_t &num, const cgbn_t &denom) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core; + typedef cgbn::core_singleton_t singleton; + + uint32_t num_low[LIMBS], num_high[LIMBS], denom_local[LIMBS]; + uint32_t shift, numthreads; + + if(_context.check_errors()) { + if(equals_ui32(denom, 0)) { + _context.report_error(cgbn_division_by_zero_error); + return; + } + } + // division of padded values is the same as division of unpadded valuess + shift=core::clz(denom._limbs); + core::rotate_left(denom_local, denom._limbs, shift); + core::rotate_left(num_low, num._limbs, shift); + core::bitwise_mask_and(num_high, num_low, shift); + core::bitwise_xor(num_low, num_low, num_high); + numthreads=TPI-core::clzt(num_high); + singleton::div_rem_wide(q._limbs, r._limbs, num_low, num_high, denom_local, numthreads); + core::rotate_right(r._limbs, r._limbs, shift); +} + +template +__device__ __forceinline__ void cgbn_env_t::sqrt(cgbn_t &s, const cgbn_t &a) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core; + typedef cgbn::core_singleton_t singleton; + + uint32_t shift, numthreads; + uint32_t shifted[LIMBS]; + + shift=core::clz(a._limbs); + if(shift==UNPADDED_BITS) { + cgbn::mpzero(s._limbs); + return; + } + numthreads=(UNPADDED_BITS+LIMBS*64-1-shift) / (LIMBS*64); + core::rotate_left(shifted, a._limbs, shift & 0xFFFFFFFE); + singleton::sqrt(s._limbs, shifted, numthreads); + shift=(shift>>1) % (LIMBS*32); + core::shift_right(s._limbs, s._limbs, shift); +} + +template +__device__ __forceinline__ void cgbn_env_t::sqrt_rem(cgbn_t &s, cgbn_t &r, const cgbn_t &a) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core; + typedef cgbn::core_singleton_t singleton; + + uint32_t shift, numthreads; + uint32_t remainder[LIMBS], temp[LIMBS]; + + shift=core::clz(a._limbs); + if(shift==UNPADDED_BITS) { + cgbn::mpzero(s._limbs); + cgbn::mpzero(r._limbs); + return; + } + numthreads=(UNPADDED_BITS+LIMBS*64-1-shift) / (LIMBS*64); + core::rotate_left(temp, a._limbs, shift & 0xFFFFFFFE); + singleton::sqrt_rem(s._limbs, remainder, temp, numthreads); + shift=(shift>>1) % (LIMBS*32); + singleton::sqrt_resolve_rem(r._limbs, s._limbs, 0, remainder, shift); + core::shift_right(s._limbs, s._limbs, shift); +} + +template +__device__ __forceinline__ bool cgbn_env_t::equals(const cgbn_t &a, const cgbn_t &b) const { + return cgbn::core_t::equals(a._limbs, b._limbs); +} + +template +__device__ __forceinline__ int32_t cgbn_env_t::compare(const cgbn_t &a, const cgbn_t &b) const { + return cgbn::core_t::compare(a._limbs, b._limbs); +} + +/* wide arithmetic routines */ +template +__device__ __forceinline__ void cgbn_env_t::mul_wide(cgbn_wide_t &r, const cgbn_t &a, const cgbn_t &b) const { + uint32_t add[LIMBS]; + + cgbn::mpzero(add); + cgbn::core_singleton_t::mul_wide(r._low._limbs, r._high._limbs, a._limbs, b._limbs, add); +} + +template +__device__ __forceinline__ void cgbn_env_t::sqr_wide(cgbn_wide_t &r, const cgbn_t &a) const { + uint32_t add[LIMBS]; + + cgbn::mpzero(add); + cgbn::core_singleton_t::mul_wide(r._low._limbs, r._high._limbs, a._limbs, a._limbs, add); +} + +template +__device__ __forceinline__ void cgbn_env_t::div_wide(cgbn_t &q, const cgbn_wide_t &num, const cgbn_t &denom) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core; + typedef cgbn::core_singleton_t singleton; + + uint32_t num_low[LIMBS], num_high[LIMBS], denom_local[LIMBS]; + uint32_t shift, numthreads; + + if(_context.check_errors()) { + if(core::compare(num._high._limbs, denom._limbs)>=0) { + _context.report_error(cgbn_division_overflow_error); + return; + } + } + + shift=core::clz(denom._limbs); + core::rotate_left(denom_local, denom._limbs, shift); + core::rotate_left(num_high, num._high._limbs, shift-(UNPADDED_BITS-BITS)); + core::rotate_left(num_low, num._low._limbs, shift); + core::bitwise_mask_select(num_high, num_high, num_low, shift-(UNPADDED_BITS-BITS)); + numthreads=TPI-core::clzt(num_high); + singleton::div_wide(q._limbs, num_low, num_high, denom_local, numthreads); +} + +template +__device__ __forceinline__ void cgbn_env_t::rem_wide(cgbn_t &r, const cgbn_wide_t &num, const cgbn_t &denom) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core; + typedef cgbn::core_singleton_t singleton; + + uint32_t num_low[LIMBS], num_high[LIMBS], denom_local[LIMBS]; + uint32_t shift, numthreads; + + if(_context.check_errors()) { + if(core::compare(num._high._limbs, denom._limbs)>=0) { + _context.report_error(cgbn_division_overflow_error); + return; + } + } + + shift=core::clz(denom._limbs); + core::rotate_left(denom_local, denom._limbs, shift); + core::rotate_left(num_high, num._high._limbs, shift-(UNPADDED_BITS-BITS)); + core::rotate_left(num_low, num._low._limbs, shift); + core::bitwise_mask_select(num_high, num_high, num_low, shift-(UNPADDED_BITS-BITS)); + core::bitwise_mask_and(num_low, num_low, shift-UNPADDED_BITS); + numthreads=TPI-core::clzt(num_high); + singleton::rem_wide(r._limbs, num_low, num_high, denom_local, numthreads); + core::rotate_right(r._limbs, r._limbs, shift); +} + +template +__device__ __forceinline__ void cgbn_env_t::div_rem_wide(cgbn_t &q, cgbn_t &r, const cgbn_wide_t &num, const cgbn_t &denom) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core; + typedef cgbn::core_singleton_t singleton; + + uint32_t num_low[LIMBS], num_high[LIMBS], denom_local[LIMBS]; + uint32_t shift, numthreads; + + if(_context.check_errors()) { + if(core::compare(num._high._limbs, denom._limbs)>=0) { + _context.report_error(cgbn_division_overflow_error); + return; + } + } + shift=core::clz(denom._limbs); + core::rotate_left(denom_local, denom._limbs, shift); + core::rotate_left(num_high, num._high._limbs, shift-(UNPADDED_BITS-BITS)); + core::rotate_left(num_low, num._low._limbs, shift); + core::bitwise_mask_select(num_high, num_high, num_low, shift-(UNPADDED_BITS-BITS)); + core::bitwise_mask_and(num_low, num_low, shift-UNPADDED_BITS); + numthreads=TPI-core::clzt(num_high); + singleton::div_rem_wide(q._limbs, r._limbs, num_low, num_high, denom_local, numthreads); + core::rotate_right(r._limbs, r._limbs, shift); +} + +template +__device__ __forceinline__ void cgbn_env_t::sqrt_wide(cgbn_t &s, const cgbn_wide_t &a) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core; + typedef cgbn::core_singleton_t singleton; + + uint32_t clz_shift, shift, numthreads; + uint32_t high_shifted[LIMBS], low_shifted[LIMBS]; + + clz_shift=core::clz(a._high._limbs); + if(clz_shift==UNPADDED_BITS) { + clz_shift=core::clz(a._low._limbs); + if(clz_shift==UNPADDED_BITS) { + cgbn::mpzero(s._limbs); + return; + } + clz_shift=clz_shift & 0xFFFFFFFE; + cgbn::mpset(high_shifted, a._low._limbs); + cgbn::mpzero(low_shifted); + shift=clz_shift + UNPADDED_BITS; + } + else { + clz_shift=clz_shift & 0xFFFFFFFE; + cgbn::mpset(high_shifted, a._high._limbs); + core::rotate_left(low_shifted, a._low._limbs, clz_shift+(UNPADDED_BITS-BITS)); + shift=clz_shift+UNPADDED_BITS-BITS; + } + numthreads=(2*UNPADDED_BITS+LIMBS*64-1-shift) / (LIMBS*64); + + core::rotate_left(high_shifted, high_shifted, clz_shift); + if(shift<2*UNPADDED_BITS-BITS) { + core::bitwise_mask_select(high_shifted, high_shifted, low_shifted, clz_shift); + core::bitwise_mask_and(low_shifted, low_shifted, (int32_t)(shift-UNPADDED_BITS)); + } + + singleton::sqrt_wide(s._limbs, low_shifted, high_shifted, numthreads); + + shift=(shift>>1) % (LIMBS*32); + core::shift_right(s._limbs, s._limbs, shift); +} + +template +__device__ __forceinline__ void cgbn_env_t::sqrt_rem_wide(cgbn_t &s, cgbn_wide_t &r, const cgbn_wide_t &a) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core_unpadded; + typedef cgbn::core_t core_padded; + typedef cgbn::core_singleton_t singleton; + + uint32_t group_thread=threadIdx.x & TPI-1; + uint32_t clz_shift, shift, numthreads, c; + uint32_t remainder[LIMBS], high_shifted[LIMBS], low_shifted[LIMBS]; + + clz_shift=core_unpadded::clz(a._high._limbs); + if(clz_shift==UNPADDED_BITS) { + clz_shift=core_unpadded::clz(a._low._limbs); + if(clz_shift==UNPADDED_BITS) { + cgbn::mpzero(s._limbs); + cgbn::mpzero(r._low._limbs); + cgbn::mpzero(r._high._limbs); + return; + } + clz_shift=clz_shift & 0xFFFFFFFE; + cgbn::mpset(high_shifted, a._low._limbs); + cgbn::mpzero(low_shifted); + shift=clz_shift + UNPADDED_BITS; + } + else { + clz_shift=clz_shift & 0xFFFFFFFE; + cgbn::mpset(high_shifted, a._high._limbs); + core_unpadded::rotate_left(low_shifted, a._low._limbs, clz_shift+(UNPADDED_BITS-BITS)); + shift=clz_shift+UNPADDED_BITS-BITS; + } + numthreads=(2*UNPADDED_BITS+LIMBS*64-1-shift) / (LIMBS*64); + + core_unpadded::rotate_left(high_shifted, high_shifted, clz_shift); + if(shift<2*UNPADDED_BITS-BITS) { + core_unpadded::bitwise_mask_select(high_shifted, high_shifted, low_shifted, clz_shift); + core_unpadded::bitwise_mask_and(low_shifted, low_shifted, (int32_t)(shift-UNPADDED_BITS)); + } + + c=singleton::sqrt_rem_wide(s._limbs, remainder, low_shifted, high_shifted, numthreads); + + shift=(shift>>1) % (LIMBS*32); + if(shift==0) { + if(UNPADDED_BITS!=BITS) + c=core_padded::clear_carry(remainder); + cgbn::mpset(r._low._limbs, remainder); + cgbn::mpzero(r._high._limbs); + r._high._limbs[0]=(group_thread==0) ? c : 0; + } + else { + singleton::sqrt_resolve_rem(r._low._limbs, s._limbs, c, remainder, shift); + cgbn::mpzero(r._high._limbs); + if(UNPADDED_BITS!=BITS) { + c=core_padded::clear_carry(r._low._limbs); + r._high._limbs[0]=(group_thread==0) ? c : 0; + } + core_unpadded::shift_right(s._limbs, s._limbs, shift); + } +} + +/* bit counting */ +template +__device__ __forceinline__ uint32_t cgbn_env_t::pop_count(const cgbn_t &a) const { + return cgbn::core_t::pop_count(a._limbs); +} + +template +__device__ __forceinline__ uint32_t cgbn_env_t::clz(const cgbn_t &a) const { + return cgbn::core_t::clz(a._limbs); +} + +template +__device__ __forceinline__ uint32_t cgbn_env_t::ctz(const cgbn_t &a) const { + return cgbn::core_t::ctz(a._limbs); +} + + +/* logical, shifting, masking */ +template +__device__ __forceinline__ void cgbn_env_t::bitwise_complement(cgbn_t &r, const cgbn_t &a) const { + cgbn::core_t::bitwise_complement(r._limbs, a._limbs); +} + +template +__device__ __forceinline__ void cgbn_env_t::bitwise_and(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const { + cgbn::core_t::bitwise_and(r._limbs, a._limbs, b._limbs); +} + +template +__device__ __forceinline__ void cgbn_env_t::bitwise_ior(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const { + cgbn::core_t::bitwise_ior(r._limbs, a._limbs, b._limbs); +} + +template +__device__ __forceinline__ void cgbn_env_t::bitwise_xor(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const { + cgbn::core_t::bitwise_xor(r._limbs, a._limbs, b._limbs); +} + +template +__device__ __forceinline__ void cgbn_env_t::bitwise_select(cgbn_t &r, const cgbn_t &clear, const cgbn_t &set, const cgbn_t &select) const { + cgbn::core_t::bitwise_select(r._limbs, clear._limbs, set._limbs, select._limbs); +} + +template +__device__ __forceinline__ void cgbn_env_t::bitwise_mask_copy(cgbn_t &r, const int32_t numbits) const { + cgbn::core_t::bitwise_mask_copy(r._limbs, numbits); +} + +template +__device__ __forceinline__ void cgbn_env_t::bitwise_mask_and(cgbn_t &r, const cgbn_t &a, const int32_t numbits) const { + cgbn::core_t::bitwise_mask_and(r._limbs, a._limbs, numbits); +} + +template +__device__ __forceinline__ void cgbn_env_t::bitwise_mask_ior(cgbn_t &r, const cgbn_t &a, const int32_t numbits) const { + cgbn::core_t::bitwise_mask_ior(r._limbs, a._limbs, numbits); +} + +template +__device__ __forceinline__ void cgbn_env_t::bitwise_mask_xor(cgbn_t &r, const cgbn_t &a, const int32_t numbits) const { + cgbn::core_t::bitwise_mask_xor(r._limbs, a._limbs, numbits); +} + +template +__device__ __forceinline__ void cgbn_env_t::bitwise_mask_select(cgbn_t &r, const cgbn_t &clear, const cgbn_t &set, const int32_t numbits) const { + cgbn::core_t::bitwise_mask_select(r._limbs, clear._limbs, set._limbs, numbits); +} + +template +__device__ __forceinline__ void cgbn_env_t::shift_left(cgbn_t &r, const cgbn_t &a, const uint32_t numbits) const { + cgbn::core_t::shift_left(r._limbs, a._limbs, numbits); +} + +template +__device__ __forceinline__ void cgbn_env_t::shift_right(cgbn_t &r, const cgbn_t &a, const uint32_t numbits) const { + cgbn::core_t::shift_right(r._limbs, a._limbs, numbits); +} + +template +__device__ __forceinline__ void cgbn_env_t::rotate_left(cgbn_t &r, const cgbn_t &a, const uint32_t numbits) const { + cgbn::core_t::rotate_left(r._limbs, a._limbs, numbits); +} + +template +__device__ __forceinline__ void cgbn_env_t::rotate_right(cgbn_t &r, const cgbn_t &a, const uint32_t numbits) const { + cgbn::core_t::rotate_right(r._limbs, a._limbs, numbits); +} + +#if 0 +template template +__device__ __forceinline__ void cgbn_env_t::shift_left(cgbn_t &r, const cgbn_t &a) const { + fwshift_left_constant(r._limbs, a._limbs); +} + +template template +__device__ __forceinline__ void cgbn_env_t::shift_right(cgbn_t &r, const cgbn_t &a) const { + fwshift_right_constant(r._limbs, a._limbs); +} + +template template +__device__ __forceinline__ void cgbn_env_t::rotate_left(cgbn_t &r, const cgbn_t &a) const { + fwrotate_left_constant(r._limbs, a._limbs); +} + +template template +__device__ __forceinline__ void cgbn_env_t::rotate_right(cgbn_t &r, const cgbn_t &a) const { + fwrotate_right_constant(r._limbs, a._limbs); +} +#endif + +/* accumulator APIs */ + +template +__device__ __forceinline__ cgbn_env_t::cgbn_accumulator_t::cgbn_accumulator_t() { + _carry=0; + cgbn::mpzero(_limbs); +} + +template +__device__ __forceinline__ int32_t cgbn_env_t::resolve(cgbn_t &sum, const cgbn_accumulator_t &accumulator) const { + typedef cgbn::core_t core; + + uint32_t carry=accumulator._carry; + int32_t result; + + cgbn::mpset(sum._limbs, accumulator._limbs); + result=core::resolve_add(carry, sum._limbs); + core::clear_padding(sum._limbs); + return result; +} + +template +__device__ __forceinline__ void cgbn_env_t::set_ui32(cgbn_accumulator_t &accumulator, const uint32_t value) const { + uint32_t group_thread=threadIdx.x & TPI-1; + + accumulator._carry=0; + accumulator._limbs[0]=(group_thread==0) ? value : 0; + #pragma unroll + for(int32_t index=1;index +__device__ __forceinline__ void cgbn_env_t::add_ui32(cgbn_accumulator_t &accumulator, const uint32_t value) const { + uint32_t group_thread=threadIdx.x & TPI-1; + + cgbn::chain_t<> chain; + accumulator._limbs[0]=chain.add(accumulator._limbs[0], (group_thread==0) ? value : 0); + #pragma unroll + for(int32_t index=1;index +__device__ __forceinline__ void cgbn_env_t::sub_ui32(cgbn_accumulator_t &accumulator, const uint32_t value) const { + uint32_t group_thread=threadIdx.x & TPI-1; + + cgbn::chain_t<> chain; + chain.sub(0, group_thread); + accumulator._limbs[0]=chain.sub(accumulator._limbs[0], (group_thread==0) ? value : 0); + #pragma unroll + for(int32_t index=1;index +__device__ __forceinline__ void cgbn_env_t::set(cgbn_accumulator_t &accumulator, const cgbn_t &value) const { + accumulator._carry=0; + cgbn::mpset(accumulator._limbs, value._limbs); +} + +template +__device__ __forceinline__ void cgbn_env_t::add(cgbn_accumulator_t &accumulator, const cgbn_t &value) const { + cgbn::chain_t<> chain; + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ void cgbn_env_t::sub(cgbn_accumulator_t &accumulator, const cgbn_t &value) const { + uint32_t group_thread=threadIdx.x & TPI-1; + + cgbn::chain_t<> chain; + chain.sub(0, group_thread); + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ void cgbn_env_t::binary_inverse(cgbn_t &r, const cgbn_t &x) const { + uint32_t low; + + if(_context.check_errors()) { + low=cgbn::core_t::get_ui32(x._limbs); + if((low & 0x01)==0) { + _context.report_error(cgbn_inverse_does_not_exist_error); + return; + } + } + + cgbn::core_t::binary_inverse(r._limbs, x._limbs); +} + +template +__device__ __forceinline__ bool cgbn_env_t::modular_inverse(cgbn_t &r, const cgbn_t &x, const cgbn_t &m) const { + typedef cgbn::unpadded_t unpadded; + + return cgbn::core_t::modular_inverse(r._limbs, x._limbs, m._limbs); +} + +template +__device__ __forceinline__ void cgbn_env_t::modular_power(cgbn_t &r, const cgbn_t &a, const cgbn_t &k, const cgbn_t &m) const { + cgbn_wide_t wide; + cgbn_t current, square, approx; + int32_t bit, m_clz, last; + + // FIX FIX FIX -- errors get checked again and again + + if(_context.check_errors()) { + if(compare(a, m)>=0) { + _context.report_error(cgbn_division_overflow_error); + return; + } + } + + set_ui32(current, 1); + set(square, a); + m_clz=barrett_approximation(approx, m); + last=bits-1-clz(k); + if(last==-1) { + set_ui32(r, 1); + return; + } + for(bit=0;bit +__device__ __forceinline__ void cgbn_env_t::gcd(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const { + typedef cgbn::unpadded_t unpadded; + + cgbn::core_t::gcd(r._limbs, a._limbs, b._limbs); +} + +/* fast division: common divisor / modulus */ +template +__device__ __forceinline__ uint32_t cgbn_env_t::bn2mont(cgbn_t &mont, const cgbn_t &bn, const cgbn_t &n) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core; + typedef cgbn::core_singleton_t singleton; + + uint32_t num_low[LIMBS], num_high[LIMBS], n_local[LIMBS]; + uint32_t shift, low; + + low=core::get_ui32(n._limbs); + + if(_context.check_errors()) { + if((low & 0x01)==0) { + _context.report_error(cgbn_modulus_not_odd_error); + return 0; + } + if(compare(bn, n)>=0) { + _context.report_error(cgbn_division_overflow_error); + return 0; + } + } + + // for padded values, we use a larger R + cgbn::mpzero(num_low); + shift=core::clz(n._limbs); + core::rotate_left(n_local, n._limbs, shift); + core::rotate_left(num_high, bn._limbs, shift); + singleton::rem_wide(mont._limbs, num_low, num_high, n_local, TPI); + core::shift_right(mont._limbs, mont._limbs, shift); + return -cgbn::ubinary_inverse(low); +} + +template +__device__ __forceinline__ void cgbn_env_t::mont2bn(cgbn_t &bn, const cgbn_t &mont, const cgbn_t &n, const uint32_t np0) const { + uint32_t zeros[LIMBS]; + + cgbn::mpzero(zeros); + + // mont_reduce_wide returns 0<=res<=n + cgbn::core_singleton_t::mont_reduce_wide(bn._limbs, mont._limbs, zeros, n._limbs, np0, true); + + // handle the case of res==n + if(cgbn::core_t::equals(bn._limbs, n._limbs)) + cgbn::mpzero(bn._limbs); +} + +template +__device__ __forceinline__ void cgbn_env_t::mont_mul(cgbn_t &r, const cgbn_t &a, const cgbn_t &b, const cgbn_t &n, const uint32_t np0) const { + cgbn::core_singleton_t::mont_mul(r._limbs, a._limbs, b._limbs, n._limbs, np0); +} + +template +__device__ __forceinline__ void cgbn_env_t::mont_sqr(cgbn_t &r, const cgbn_t &a, const cgbn_t &n, const uint32_t np0) const { + cgbn::core_singleton_t::mont_mul(r._limbs, a._limbs, a._limbs, n._limbs, np0); +} + +template +__device__ __forceinline__ void cgbn_env_t::mont_reduce_wide(cgbn_t &r, const cgbn_wide_t &a, const cgbn_t &n, const uint32_t np0) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core; + typedef cgbn::core_singleton_t singleton; + + uint32_t low[LIMBS], high[LIMBS]; + + cgbn::mpset(low, a._low._limbs); + cgbn::mpset(high, a._high._limbs); + + if(PADDING!=0) { + core::rotate_right(high, high, UNPADDED_BITS-BITS); + core::bitwise_mask_select(low, high, low, BITS); + core::bitwise_mask_and(high, high, BITS); + } + + // mont_reduce_wide returns 0<=res<=n + singleton::mont_reduce_wide(r._limbs, low, high, n._limbs, np0, false); + + // handle the case of res==n + if(core::equals(r._limbs, n._limbs)) + cgbn::mpzero(r._limbs); +} + +template +__device__ __forceinline__ uint32_t cgbn_env_t::barrett_approximation(cgbn_t &approx, const cgbn_t &denom) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core; + typedef cgbn::core_singleton_t singleton; + + uint32_t shift, shifted[LIMBS], low[LIMBS], high[LIMBS]; + + shift=core::clz(denom._limbs); + if(_context.check_errors()) { + if(shift==UNPADDED_BITS) { + _context.report_error(cgbn_division_by_zero_error); + return 0xFFFFFFFF; + } + } + + if(shift==UNPADDED_BITS) + return 0xFFFFFFFF; + + core::rotate_left(shifted, denom._limbs, shift); + + #pragma unroll + for(int32_t index=0;index +__device__ __forceinline__ void cgbn_env_t::barrett_div(cgbn_t &q, const cgbn_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core; + typedef cgbn::core_singleton_t singleton; + + uint32_t sync, group_thread=threadIdx.x & TPI-1; + uint32_t low[LIMBS], high[LIMBS], quotient[LIMBS], zero[LIMBS]; + uint32_t word, c, sub=0; + + sync=core::sync_mask(); + core::shift_right(high, num._limbs, UNPADDED_BITS-denom_clz); + cgbn::mpzero(zero); + singleton::mul_high(quotient, high, approx._limbs, zero); + + c=cgbn::mpadd(quotient, quotient, high); + c+=cgbn::mpadd32(quotient, quotient, group_thread==0 ? 3 : 0); + c=core::resolve_add(c, quotient); + + if(c!=0) { + #pragma unroll + for(int32_t index=0;index(low, num._limbs, low); + word-=core::fast_propagate_sub(c, low); + while(word!=0) { + sub++; + c=cgbn::mpadd(low, low, denom._limbs); + word+=core::fast_propagate_add(c, low); + } + core::sub_ui32(q._limbs, quotient, sub); +} + +template +__device__ __forceinline__ void cgbn_env_t::barrett_rem(cgbn_t &r, const cgbn_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core; + typedef cgbn::core_singleton_t singleton; + + uint32_t sync, group_thread=threadIdx.x & TPI-1; + uint32_t low[LIMBS], high[LIMBS], quotient[LIMBS], zero[LIMBS]; + uint32_t word, c; + + sync=core::sync_mask(); + core::shift_right(high, num._limbs, UNPADDED_BITS-denom_clz); + cgbn::mpzero(zero); + singleton::mul_high(quotient, high, approx._limbs, zero); + + c=cgbn::mpadd(quotient, quotient, high); + c+=cgbn::mpadd32(quotient, quotient, group_thread==0 ? 3 : 0); + c=core::resolve_add(c, quotient); + + if(c!=0) { + #pragma unroll + for(int32_t index=0;index(low, num._limbs, low); + word-=core::fast_propagate_sub(c, low); + while(word!=0) { + c=cgbn::mpadd(low, low, denom._limbs); + word+=core::fast_propagate_add(c, low); + } + cgbn::mpset(r._limbs, low); +} + +template +__device__ __forceinline__ void cgbn_env_t::barrett_div_rem(cgbn_t &q, cgbn_t &r, const cgbn_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core; + typedef cgbn::core_singleton_t singleton; + + uint32_t sync, group_thread=threadIdx.x & TPI-1; + uint32_t low[LIMBS], high[LIMBS], quotient[LIMBS], zero[LIMBS]; + uint32_t word, c, sub=0; + + sync=core::sync_mask(); + core::shift_right(high, num._limbs, UNPADDED_BITS-denom_clz); + cgbn::mpzero(zero); + singleton::mul_high(quotient, high, approx._limbs, zero); + + c=cgbn::mpadd(quotient, quotient, high); + c+=cgbn::mpadd32(quotient, quotient, group_thread==0 ? 3 : 0); + c=core::resolve_add(c, quotient); + + if(c!=0) { + #pragma unroll + for(int32_t index=0;index(low, num._limbs, low); + word-=core::fast_propagate_sub(c, low); + while(word!=0) { + sub++; + c=cgbn::mpadd(low, low, denom._limbs); + word+=core::fast_propagate_add(c, low); + } + core::sub_ui32(q._limbs, quotient, sub); + cgbn::mpset(r._limbs, low); +} + +template +__device__ __forceinline__ void cgbn_env_t::barrett_div_wide(cgbn_t &q, const cgbn_wide_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core_unpadded; + typedef cgbn::core_t core_padded; + typedef cgbn::core_singleton_t singleton; + + uint32_t sync, group_thread=threadIdx.x & TPI-1, group_base=group_thread*LIMBS; + uint32_t low[LIMBS], high[LIMBS], quotient[LIMBS], zero[LIMBS]; + uint32_t word, c, sub=0; + + if(_context.check_errors()) { + if(core_unpadded::compare(num._high._limbs, denom._limbs)>=0) { + _context.report_error(cgbn_division_overflow_error); + return; + } + } + + sync=core_unpadded::sync_mask(); + word=__shfl_sync(sync, num._high._limbs[0], 0, TPI); + core_unpadded::rotate_left(low, num._low._limbs, denom_clz); + core_unpadded::rotate_left(high, num._high._limbs, denom_clz-(UNPADDED_BITS-BITS)); + core_unpadded::bitwise_mask_select(high, high, low, denom_clz-(UNPADDED_BITS-BITS)); + cgbn::mpzero(zero); + singleton::mul_high(quotient, high, approx._limbs, zero); + + c=cgbn::mpadd(quotient, quotient, high); + c+=cgbn::mpadd32(quotient, quotient, group_thread==0 ? 3 : 0); + c=core_padded::resolve_add(c, quotient); + + if(c!=0) { + #pragma unroll + for(int32_t index=0;index(low, num._low._limbs, low); + word-=core_padded::fast_propagate_sub(c, low); + while(word!=0) { + sub++; + c=cgbn::mpadd(low, low, denom._limbs); + word+=core_padded::fast_propagate_add(c, low); + } + core_unpadded::sub_ui32(q._limbs, quotient, sub); +} + +template +__device__ __forceinline__ void cgbn_env_t::barrett_rem_wide(cgbn_t &r, const cgbn_wide_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core_unpadded; + typedef cgbn::core_t core_padded; + typedef cgbn::core_singleton_t singleton; + + uint32_t sync, group_thread=threadIdx.x & TPI-1, group_base=group_thread*LIMBS; + uint32_t low[LIMBS], high[LIMBS], quotient[LIMBS], zero[LIMBS]; + uint32_t word, c; + + if(_context.check_errors()) { + if(core_unpadded::compare(num._high._limbs, denom._limbs)>=0) { + _context.report_error(cgbn_division_overflow_error); + return; + } + } + + sync=core_unpadded::sync_mask(); + word=__shfl_sync(sync, num._high._limbs[0], 0, TPI); + core_unpadded::rotate_left(low, num._low._limbs, denom_clz); + core_unpadded::rotate_left(high, num._high._limbs, denom_clz-(UNPADDED_BITS-BITS)); + core_unpadded::bitwise_mask_select(high, high, low, denom_clz-(UNPADDED_BITS-BITS)); + cgbn::mpzero(zero); + singleton::mul_high(quotient, high, approx._limbs, zero); + + c=cgbn::mpadd(quotient, quotient, high); + c+=cgbn::mpadd32(quotient, quotient, group_thread==0 ? 3 : 0); + c=core_padded::resolve_add(c, quotient); + + if(c!=0) { + #pragma unroll + for(int32_t index=0;index(low, num._low._limbs, low); + word-=core_padded::fast_propagate_sub(c, low); + while(word!=0) { + c=cgbn::mpadd(low, low, denom._limbs); + word+=core_padded::fast_propagate_add(c, low); + } + cgbn::mpset(r._limbs, low); +} + +template +__device__ __forceinline__ void cgbn_env_t::barrett_div_rem_wide(cgbn_t &q, cgbn_t &r, const cgbn_wide_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const { + typedef cgbn::unpadded_t unpadded; + typedef cgbn::core_t core_unpadded; + typedef cgbn::core_t core_padded; + typedef cgbn::core_singleton_t singleton; + + uint32_t sync, group_thread=threadIdx.x & TPI-1, group_base=group_thread*LIMBS; + uint32_t low[LIMBS], high[LIMBS], quotient[LIMBS], zero[LIMBS]; + uint32_t word, c, sub=0; + + if(_context.check_errors()) { + if(core_unpadded::compare(num._high._limbs, denom._limbs)>=0) { + _context.report_error(cgbn_division_overflow_error); + return; + } + } + + sync=core_unpadded::sync_mask(); + word=__shfl_sync(sync, num._high._limbs[0], 0, TPI); + core_unpadded::rotate_left(low, num._low._limbs, denom_clz); + core_unpadded::rotate_left(high, num._high._limbs, denom_clz-(UNPADDED_BITS-BITS)); + core_unpadded::bitwise_mask_select(high, high, low, denom_clz-(UNPADDED_BITS-BITS)); + cgbn::mpzero(zero); + singleton::mul_high(quotient, high, approx._limbs, zero); + + c=cgbn::mpadd(quotient, quotient, high); + c+=cgbn::mpadd32(quotient, quotient, group_thread==0 ? 3 : 0); + c=core_padded::resolve_add(c, quotient); + + if(c!=0) { + #pragma unroll + for(int32_t index=0;index(low, num._low._limbs, low); + word-=core_padded::fast_propagate_sub(c, low); + while(word!=0) { + sub++; + c=cgbn::mpadd(low, low, denom._limbs); + word+=core_padded::fast_propagate_add(c, low); + } + core_unpadded::sub_ui32(q._limbs, quotient, sub); + cgbn::mpset(r._limbs, low); +} + +/* load/store routines */ +template +__device__ __forceinline__ void cgbn_env_t::load(cgbn_t &r, cgbn_mem_t *const address) const { + int32_t group_thread=threadIdx.x & TPI-1; + int32_t limb; + + #pragma unroll + for(limb=0;limb_limbs[group_thread*LIMBS + limb]; + } + else + r._limbs[limb]=address->_limbs[group_thread*LIMBS + limb]; + } +} + +template +__device__ __forceinline__ void cgbn_env_t::store(cgbn_mem_t *address, const cgbn_t &a) const { + int32_t group_thread=threadIdx.x & TPI-1; + int32_t limb; + + #pragma unroll + for(limb=0;limb_limbs[group_thread*LIMBS + limb]=a._limbs[limb]; +#if 1 + else + if(a._limbs[limb]!=0) { + printf("BAD LIMB: %d %d %d\n", blockIdx.x, threadIdx.x, limb); + __trap(); + } +#endif + } + else + address->_limbs[group_thread*LIMBS + limb]=a._limbs[limb]; + } +} + +template +__device__ __forceinline__ void cgbn_env_t::load(cgbn_t &r, cgbn_local_t *const address) const { + int32_t limb; + + #pragma unroll + for(limb=0;limb_limbs[limb]; +} + +template +__device__ __forceinline__ void cgbn_env_t::store(cgbn_local_t *address, const cgbn_t &a) const { + int32_t limb; + + #pragma unroll + for(limb=0;limb_limbs[limb]=a._limbs[limb]; +} diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/impl_mpz.cc b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/impl_mpz.cc new file mode 100644 index 0000000..9042e95 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/cgbn/impl_mpz.cc @@ -0,0 +1,1362 @@ +/*** + +Copyright (c) 2018-2019, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +***/ + +/**************************************************************************************************************** + * cgbn_context_t implementation using GMP + ****************************************************************************************************************/ +template +cgbn_context_t::cgbn_context_t() : _monitor(cgbn_no_checks), _report(NULL), _instance(0xFFFFFFFF) { +} + +template +cgbn_context_t::cgbn_context_t(cgbn_monitor_t monitor) : _monitor(monitor), _report(NULL), _instance(0xFFFFFFFF) { +} + +template +cgbn_context_t::cgbn_context_t(cgbn_monitor_t monitor, cgbn_error_report_t *report) : _monitor(monitor), _report(report), _instance(0xFFFFFFFF) { +} + +template +cgbn_context_t::cgbn_context_t(cgbn_monitor_t monitor, cgbn_error_report_t *report, uint32_t instance) : _monitor(monitor), _report(report), _instance(instance) { +} + +template +bool cgbn_context_t::check_errors() const { + return _monitor!=cgbn_no_checks; +} + +template +void cgbn_context_t::report_error(cgbn_error_t error) const { + if(_report!=NULL) { + dim3 maxdim; + + maxdim.x=0xFFFFFFFFu; maxdim.y=0xFFFFFFFFu; maxdim.z=0xFFFFFFFFu; + if(__sync_bool_compare_and_swap(&(_report->_error), cgbn_no_error, error)) { + _report->_instance=_instance; + _report->_threadIdx=maxdim; + _report->_blockIdx=maxdim; + } + } + if(_monitor==cgbn_print_monitor) { + switch(_report->_error) { + case cgbn_unsupported_threads_per_instance: + printf("cgbn error: unsupported threads per instance\n"); + break; + case cgbn_unsupported_size: + printf("cgbn error: unsupported size\n"); + break; + case cgbn_unsupported_limbs_per_thread: + printf("cgbn error: unsupported limbs per thread\n"); + break; + case cgbn_unsupported_operation: + printf("cgbn error: unsupported operation\n"); + break; + case cgbn_threads_per_block_mismatch: + printf("cgbn error: TPB does not match blockDim.x\n"); + break; + case cgbn_threads_per_instance_mismatch: + printf("cgbn errpr: TPI does not match env_t::TPI\n"); + break; + case cgbn_division_by_zero_error: + printf("cgbn error: division by zero on instance\n"); + break; + case cgbn_division_overflow_error: + printf("cgbn error: division overflow on instance\n"); + break; + case cgbn_invalid_montgomery_modulus_error: + printf("cgbn error: division invalid montgomery modulus\n"); + break; + case cgbn_modulus_not_odd_error: + printf("cgbn error: invalid modulus (it must be odd)\n"); + break; + case cgbn_inverse_does_not_exist_error: + printf("cgbn error: inverse does not exist\n"); + break; + default: + printf("cgbn error: unknown error reported by instance\n"); + break; + } + } + else if(_monitor==cgbn_halt_monitor) { + exit(1); + } +} + +/* forward declarations aren't working for this right now +template template +env_t cgbn_context_t::env() { + env_t env(this); + + return env; +} + +template template +cgbn_env_t cgbn_context_t::env() { + cgbn_env_t env(this); + + return env; +} +*/ + +/**************************************************************************************************************** + * cgbn_env_t implementation using gmp + ****************************************************************************************************************/ + +/* constructor */ +template +cgbn_env_t::cgbn_env_t(const context_t &context) : _context(context) { +} + + +/* size conversion */ +template template +void cgbn_env_t::set(cgbn_t &r, const source_cgbn_t &source) const { + mpz_fdiv_r_2exp(r._z, source._z, bits); +} + + +/* set get routines */ +template +void cgbn_env_t::set(cgbn_t &r, const cgbn_t &a) const { + mpz_fdiv_r_2exp(r._z, a._z, bits); +} + +template +void cgbn_env_t::swap(cgbn_t &r, cgbn_t &a) const { + mpz_swap(r._z, a._z); +} + +template +void cgbn_env_t::extract_bits(cgbn_t &r, const cgbn_t &a, const uint32_t start, const uint32_t len) const { + mpz_fdiv_q_2exp(r._z, a._z, start); + mpz_fdiv_r_2exp(r._z, r._z, len); +} + +template +void cgbn_env_t::insert_bits(cgbn_t &r, const cgbn_t &a, const uint32_t start, const uint32_t len, const cgbn_t &value) const { + mpz_t temp; + int32_t index; + + if(start>=bits) { + mpz_set(r._z, a._z); + return; + } + + mpz_init(temp); + mpz_fdiv_r_2exp(temp, value._z, len); + mpz_mul_2exp(temp, temp, start); + mpz_set(r._z, a._z); + for(index=start;index +uint32_t cgbn_env_t::get_ui32(const cgbn_t &a) const { + return mpz_get_ui(a._z); +} + +template +void cgbn_env_t::set_ui32(cgbn_t &r, const uint32_t value) const { + mpz_set_ui(r._z, value); +} + +template +int32_t cgbn_env_t::add_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t add) const { + uint32_t result; + mpz_t top; + + mpz_init(top); + mpz_add_ui(r._z, a._z, add); + mpz_fdiv_q_2exp(top, r._z, bits); + mpz_fdiv_r_2exp(r._z, r._z, bits); + result=mpz_get_ui(top); + mpz_clear(top); + return result; +} + +template +int32_t cgbn_env_t::sub_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t sub) const { + uint32_t result; + mpz_t top; + + mpz_init(top); + mpz_sub_ui(r._z, a._z, sub); + mpz_fdiv_q_2exp(top, r._z, bits); + mpz_fdiv_r_2exp(r._z, r._z, bits); + result=mpz_get_si(top); + mpz_clear(top); + return result; +} + +template +uint32_t cgbn_env_t::mul_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t mul) const { + uint32_t result; + mpz_t top; + + mpz_init(top); + mpz_mul_ui(r._z, a._z, mul); + mpz_fdiv_q_2exp(top, r._z, bits); + mpz_fdiv_r_2exp(r._z, r._z, bits); + result=mpz_get_ui(top); + mpz_clear(top); + return result; +} + +template +uint32_t cgbn_env_t::div_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t div) const { + uint32_t result; + mpz_t mod; + + if(_context.check_errors() && div==0) { + _context.report_error(cgbn_division_by_zero_error); + return 0; + } + + mpz_init(mod); + mpz_fdiv_r_ui(mod, a._z, div); + mpz_fdiv_q_ui(r._z, a._z, div); + result=mpz_get_ui(mod); + mpz_clear(mod); + return result; +} + +template +uint32_t cgbn_env_t::rem_ui32(const cgbn_t &a, const uint32_t div) const { + uint32_t result; + mpz_t mod; + + if(_context.check_errors() && div==0) { + _context.report_error(cgbn_division_by_zero_error); + return 0; + } + + mpz_init(mod); + mpz_fdiv_r_ui(mod, a._z, div); + result=mpz_get_ui(mod); + mpz_clear(mod); + return result; +} + +template +bool cgbn_env_t::equals_ui32(const cgbn_t &a, const uint32_t value) const { + return mpz_cmp_ui(a._z, value)==0; +} + +template +int32_t cgbn_env_t::compare_ui32(const cgbn_t &a, const uint32_t value) const { + int32_t cmp=mpz_cmp_ui(a._z, value); + + if(cmp>0) return 1; + if(cmp<0) return -1; + return 0; +} + +template +uint32_t cgbn_env_t::extract_bits_ui32(const cgbn_t &a, const uint32_t start, const uint32_t len) const { + uint32_t result; + mpz_t chunk; + + mpz_init(chunk); + mpz_fdiv_q_2exp(chunk, a._z, start); + if(len<32) + mpz_fdiv_r_2exp(chunk, chunk, len); + result=mpz_get_ui(chunk); + mpz_clear(chunk); + return result; +} + +template +void cgbn_env_t::insert_bits_ui32(cgbn_t &r, const cgbn_t &a, const uint32_t start, const uint32_t len, const uint32_t value) const { + int32_t index, local=len; + mpz_t chunk; + + if(start>bits) { + mpz_set(r._z, a._z); + return; + } + if(local>32) + local=32; + + mpz_init(chunk); + mpz_set_ui(chunk, value); + mpz_fdiv_r_2exp(chunk, chunk, local); + mpz_mul_2exp(chunk, chunk, start); + + mpz_set(r._z, a._z); + for(index=start;index +uint32_t cgbn_env_t::binary_inverse_ui32(const uint32_t n0) const { + uint32_t inv=n0; + + inv=inv*(inv*n0+14); + inv=inv*(inv*n0+2); + inv=inv*(inv*n0+2); + inv=inv*(inv*n0+2); + return -inv; +} + +template +uint32_t cgbn_env_t::gcd_ui32(const cgbn_t &a, const uint32_t value) const { + uint32_t result; + mpz_t temp; + + if(value==0) + return 0; + + mpz_init(temp); + mpz_set_ui(temp, value); + mpz_gcd(temp, temp, a._z); + result=mpz_get_ui(temp); + mpz_clear(temp); + return result; +} + + +/* bn arithmetic routines */ +template +int32_t cgbn_env_t::add(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const { + int32_t carry; + + mpz_add(r._z, a._z, b._z); + carry=mpz_tstbit(r._z, bits); + mpz_fdiv_r_2exp(r._z, r._z, bits); + return carry; +} + +template +int32_t cgbn_env_t::sub(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const { + int32_t carry; + + mpz_sub(r._z, a._z, b._z); + carry=(mpz_sgn(r._z)==-1) ? -1 : 0; + mpz_fdiv_r_2exp(r._z, r._z, bits); + return carry; +} + +template +int32_t cgbn_env_t::negate(cgbn_t &r, const cgbn_t &a) const { + mpz_ui_sub(r._z, 0, a._z); + mpz_fdiv_r_2exp(r._z, r._z, bits); + return (mpz_sgn(r._z)==0) ? 0 : -1; +} + +template +void cgbn_env_t::mul(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const { + mpz_mul(r._z, a._z, b._z); + mpz_fdiv_r_2exp(r._z, r._z, bits); +} + +template +void cgbn_env_t::mul_high(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const { + mpz_mul(r._z, a._z, b._z); + mpz_fdiv_q_2exp(r._z, r._z, bits); +} + +template +void cgbn_env_t::sqr(cgbn_t &r, const cgbn_t &a) const { + mpz_mul(r._z, a._z, a._z); + mpz_fdiv_r_2exp(r._z, r._z, bits); +} + +template +void cgbn_env_t::sqr_high(cgbn_t &r, const cgbn_t &a) const { + mpz_mul(r._z, a._z, a._z); + mpz_fdiv_q_2exp(r._z, r._z, bits); +} + +template +void cgbn_env_t::div(cgbn_t &q, const cgbn_t &num, const cgbn_t &denom) const { + mpz_fdiv_q(q._z, num._z, denom._z); +} + +template +void cgbn_env_t::rem(cgbn_t &r, const cgbn_t &num, const cgbn_t &denom) const { + mpz_fdiv_r(r._z, num._z, denom._z); +} + +template +void cgbn_env_t::div_rem(cgbn_t &q, cgbn_t &r, const cgbn_t &num, const cgbn_t &denom) const { + mpz_fdiv_qr(q._z, r._z, num._z, denom._z); +} + +template +void cgbn_env_t::sqrt(cgbn_t &s, const cgbn_t &a) const { + mpz_sqrt(s._z, a._z); +} + +template +void cgbn_env_t::sqrt_rem(cgbn_t &s, cgbn_t &r, const cgbn_t &a) const { + mpz_t sqrt, rem; + + mpz_init(sqrt); + mpz_init(rem); + mpz_sqrt(sqrt, a._z); + mpz_mul(rem, sqrt, sqrt); + mpz_sub(r._z, a._z, rem); + mpz_set(s._z, sqrt); + mpz_clear(sqrt); + mpz_clear(rem); +} + +template +bool cgbn_env_t::equals(const cgbn_t &a, const cgbn_t &b) const { + return mpz_cmp(a._z, b._z)==0; +} + +template +int32_t cgbn_env_t::compare(const cgbn_t &a, const cgbn_t &b) const { + int32_t cmp=mpz_cmp(a._z, b._z); + + if(cmp>0) return 1; + if(cmp<0) return -1; + return 0; +} + + +/* logical, shifting, masking */ +template +void cgbn_env_t::bitwise_and(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const { + mpz_and(r._z, a._z, b._z); +} + +template +void cgbn_env_t::bitwise_ior(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const { + mpz_ior(r._z, a._z, b._z); +} + +template +void cgbn_env_t::bitwise_xor(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const { + mpz_xor(r._z, a._z, b._z); +} + +template +void cgbn_env_t::bitwise_complement(cgbn_t &r, const cgbn_t &a) const { + mpz_t temp; + + mpz_init(temp); + mpz_set_ui(temp, 1); + mpz_mul_2exp(temp, temp, bits); + mpz_sub_ui(temp, temp, 1); + mpz_xor(r._z, a._z, temp); + mpz_clear(temp); +} + +template +void cgbn_env_t::bitwise_select(cgbn_t &r, const cgbn_t &clear, const cgbn_t &set, const cgbn_t &select) const { + mpz_t mask, temp; + + mpz_init(temp); + mpz_init(mask); + mpz_and(temp, set._z, select._z); + mpz_set_ui(mask, 1); + mpz_mul_2exp(mask, mask, bits); + mpz_sub_ui(mask, mask, 1); + mpz_xor(mask, mask, select._z); + mpz_and(mask, mask, clear._z); + mpz_ior(r._z, temp, mask); + mpz_clear(mask); + mpz_clear(temp); +} + +template +void make_mask(mpz_t mask, int32_t numbits) { + if(numbits>=0 && numbits-bits) { + mpz_set_ui(mask, 0); + mpz_setbit(mask, -numbits); + mpz_sub_ui(mask, mask, 1); + mpz_mul_2exp(mask, mask, bits+numbits); + } + else { + mpz_set_si(mask, -1); + mpz_fdiv_r_2exp(mask, mask, bits); + } +} + +template +void cgbn_env_t::bitwise_mask_copy(cgbn_t &r, const int32_t numbits) const { + make_mask(r._z, numbits); +} + +template +void cgbn_env_t::bitwise_mask_and(cgbn_t &r, const cgbn_t &a, const int32_t numbits) const { + mpz_t temp; + + mpz_init(temp); + make_mask(temp, numbits); + mpz_and(r._z, a._z, temp); + mpz_clear(temp); +} + +template +void cgbn_env_t::bitwise_mask_ior(cgbn_t &r, const cgbn_t &a, const int32_t numbits) const { + mpz_t temp; + + mpz_init(temp); + make_mask(temp, numbits); + mpz_ior(r._z, a._z, temp); + mpz_clear(temp); +} + +template +void cgbn_env_t::bitwise_mask_xor(cgbn_t &r, const cgbn_t &a, const int32_t numbits) const { + mpz_t temp; + + mpz_init(temp); + make_mask(temp, numbits); + mpz_xor(r._z, a._z, temp); + mpz_clear(temp); +} + +template +void cgbn_env_t::bitwise_mask_select(cgbn_t &r, const cgbn_t &clear, const cgbn_t &set, const int32_t numbits) const { + mpz_t temp, mask, select; + + mpz_init(temp); + mpz_init(mask); + mpz_init(select); + make_mask(select, numbits); + + mpz_and(temp, set._z, select); + mpz_set_ui(mask, 1); + mpz_mul_2exp(mask, mask, bits); + mpz_sub_ui(mask, mask, 1); + mpz_xor(select, select, mask); + mpz_and(select, select, clear._z); + mpz_ior(r._z, temp, select); + + mpz_clear(temp); + mpz_clear(mask); + mpz_clear(select); +} + + +template +void cgbn_env_t::shift_left(cgbn_t &r, const cgbn_t &a, const uint32_t numbits) const { + mpz_mul_2exp(r._z, a._z, numbits); + mpz_fdiv_r_2exp(r._z, r._z, bits); +} + +template +void cgbn_env_t::shift_right(cgbn_t &r, const cgbn_t &a, const uint32_t numbits) const { + mpz_fdiv_q_2exp(r._z, a._z, numbits); +} + +template +void cgbn_env_t::rotate_left(cgbn_t &r, const cgbn_t &a, const uint32_t numbits) const { + mpz_t left, right; + uint32_t amount; + + amount=numbits % bits; + mpz_init(left); + mpz_init(right); + mpz_mul_2exp(left, a._z, amount); + mpz_fdiv_r_2exp(left, left, bits); + mpz_fdiv_q_2exp(right, a._z, bits-amount); + mpz_ior(r._z, left, right); + mpz_clear(left); + mpz_clear(right); +} + +template +void cgbn_env_t::rotate_right(cgbn_t &r, const cgbn_t &a, const uint32_t numbits) const { + mpz_t left, right; + uint32_t amount; + + amount=bits-numbits % bits; + if(amount==bits) { + mpz_set(r._z, a._z); + return; + } + mpz_init(left); + mpz_init(right); + mpz_mul_2exp(left, a._z, amount); + mpz_fdiv_r_2exp(left, left, bits); + mpz_fdiv_q_2exp(right, a._z, bits-amount); + mpz_ior(r._z, left, right); + mpz_clear(left); + mpz_clear(right); +} + +template template +void cgbn_env_t::shift_left(cgbn_t &r, const cgbn_t &a) const { + shift_left(r, a, numbits); +} + +template template +void cgbn_env_t::shift_right(cgbn_t &r, const cgbn_t &a) const { + shift_right(r, a, numbits); +} + +template template +void cgbn_env_t::rotate_left(cgbn_t &r, const cgbn_t &a) const { + rotate_left(r, a, numbits); +} + +template template +void cgbn_env_t::rotate_right(cgbn_t &r, const cgbn_t &a) const { + rotate_right(r, a, numbits); +} + +/* bit counting */ +template +uint32_t cgbn_env_t::pop_count(const cgbn_t &a) const { + return mpz_popcount(a._z); +} + +template +uint32_t cgbn_env_t::clz(const cgbn_t &a) const { + if(mpz_sgn(a._z)==0) + return bits; + return bits-mpz_sizeinbase(a._z, 2); +} + +template +uint32_t cgbn_env_t::ctz(const cgbn_t &a) const { + if(mpz_sgn(a._z)==0) + return bits; + return mpz_scan1(a._z, 0); +} + + + +/* wide math routines */ +template +void cgbn_env_t::mul_wide(cgbn_wide_t &r, const cgbn_t &a, const cgbn_t &b) const { + mpz_t temp; + + mpz_init(temp); + mpz_mul(temp, a._z, b._z); + mpz_fdiv_r_2exp(r._low._z, temp, bits); + mpz_fdiv_q_2exp(r._high._z, temp, bits); + mpz_clear(temp); +} + +template +void cgbn_env_t::sqr_wide(cgbn_wide_t &r, const cgbn_t &a) const { + mpz_t temp; + + mpz_init(temp); + mpz_mul(temp, a._z, a._z); + mpz_fdiv_r_2exp(r._low._z, temp, bits); + mpz_fdiv_q_2exp(r._high._z, temp, bits); + mpz_clear(temp); +} + +template +void cgbn_env_t::div_wide(cgbn_t &q, const cgbn_wide_t &num, const cgbn_t &denom) const { + mpz_t temp; + + if(_context.check_errors()) { + if(mpz_sgn(denom._z)==0) { + _context.report_error(cgbn_division_by_zero_error); + return; + } + if(mpz_cmp(num._high._z, denom._z)>=0) { + _context.report_error(cgbn_division_overflow_error); + return; + } + } + mpz_init(temp); + mpz_set(temp, num._high._z); + mpz_mul_2exp(temp, temp, bits); + mpz_add(temp, temp, num._low._z); + mpz_fdiv_q(q._z, temp, denom._z); + mpz_clear(temp); +} + +template +void cgbn_env_t::rem_wide(cgbn_t &r, const cgbn_wide_t &num, const cgbn_t &denom) const { + mpz_t temp; + + if(_context.check_errors()) { + if(mpz_sgn(denom._z)==0) { + _context.report_error(cgbn_division_by_zero_error); + return; + } + } + mpz_init(temp); + mpz_set(temp, num._high._z); + mpz_mul_2exp(temp, temp, bits); + mpz_add(temp, temp, num._low._z); + mpz_fdiv_r(r._z, temp, denom._z); + mpz_clear(temp); +} + +template +void cgbn_env_t::div_rem_wide(cgbn_t &q, cgbn_t &r, const cgbn_wide_t &num, const cgbn_t &denom) const { + mpz_t temp; + + if(_context.check_errors()) { + if(mpz_sgn(denom._z)==0) { + _context.report_error(cgbn_division_by_zero_error); + return; + } + if(mpz_cmp(num._high._z, denom._z)>=0) { + _context.report_error(cgbn_division_overflow_error); + return; + } + } + mpz_init(temp); + mpz_set(temp, num._high._z); + mpz_mul_2exp(temp, temp, bits); + mpz_add(temp, temp, num._low._z); + mpz_fdiv_q(q._z, temp, denom._z); + mpz_fdiv_r(r._z, temp, denom._z); + mpz_clear(temp); +} + +template +void cgbn_env_t::sqrt_wide(cgbn_t &s, const cgbn_wide_t &a) const { + mpz_t temp; + + mpz_init(temp); + mpz_set(temp, a._high._z); + mpz_mul_2exp(temp, temp, bits); + mpz_add(temp, temp, a._low._z); + mpz_sqrt(s._z, temp); + mpz_clear(temp); +} + +template +void cgbn_env_t::sqrt_rem_wide(cgbn_t &s, cgbn_wide_t &r, const cgbn_wide_t &a) const { + mpz_t temp, sqrt; + + mpz_init(temp); + mpz_init(sqrt); + mpz_set(temp, a._high._z); + mpz_mul_2exp(temp, temp, bits); + mpz_add(temp, temp, a._low._z); + mpz_sqrt(sqrt, temp); + mpz_submul(temp, sqrt, sqrt); + mpz_set(s._z, sqrt); + mpz_fdiv_q_2exp(r._high._z, temp, bits); + mpz_fdiv_r_2exp(r._low._z, temp, bits); + mpz_clear(temp); + mpz_clear(sqrt); +} + +/* accumulator APIs */ +template +__host__ int32_t cgbn_env_t::resolve(cgbn_t &sum, const cgbn_accumulator_t &accumulator) const { + mpz_t temp; + uint32_t r; + + mpz_init(temp); + mpz_fdiv_r_2exp(sum._z, accumulator._z, bits); + mpz_fdiv_r_2exp(temp, accumulator._z, bits+32); + mpz_fdiv_q_2exp(temp, temp, bits); + r=mpz_get_ui(temp); + mpz_clear(temp); + return (int32_t)r; +} + +template +__host__ void cgbn_env_t::set_ui32(cgbn_accumulator_t &accumulator, const uint32_t value) const { + mpz_set_ui(accumulator._z, value); +} + +template +__host__ void cgbn_env_t::add_ui32(cgbn_accumulator_t &accumulator, const uint32_t value) const { + mpz_add_ui(accumulator._z, accumulator._z, value); +} + +template +__host__ void cgbn_env_t::sub_ui32(cgbn_accumulator_t &accumulator, const uint32_t value) const { + mpz_sub_ui(accumulator._z, accumulator._z, value); +} + +template +__host__ void cgbn_env_t::set(cgbn_accumulator_t &accumulator, const cgbn_t &value) const { + mpz_set(accumulator._z, value._z); +} + +template +__host__ void cgbn_env_t::add(cgbn_accumulator_t &accumulator, const cgbn_t &value) const { + mpz_add(accumulator._z, accumulator._z, value._z); +} + +template +__host__ void cgbn_env_t::sub(cgbn_accumulator_t &accumulator, const cgbn_t &value) const { + mpz_sub(accumulator._z, accumulator._z, value._z); +} + + +/* math */ +template +void cgbn_env_t::binary_inverse(cgbn_t &r, const cgbn_t &m) const { + mpz_t temp; + + if(_context.check_errors()) { + if(mpz_tstbit(m._z, 0)==0) { + _context.report_error(cgbn_modulus_not_odd_error); + return; + } + } + mpz_init(temp); + mpz_setbit(temp, bits); + mpz_invert(r._z, m._z, temp); + mpz_clear(temp); +} + +template +bool cgbn_env_t::modular_inverse(cgbn_t &r, const cgbn_t &x, const cgbn_t &modulus) const { + uint32_t inverted; + + inverted=0; + if(mpz_cmp_ui(modulus._z, 1)>0) + inverted=mpz_invert(r._z, x._z, modulus._z); + if(inverted==0) + mpz_set_ui(r._z, 0); + return inverted!=0; +} + +template +void cgbn_env_t::modular_power(cgbn_t &r, const cgbn_t &a, const cgbn_t &k, const cgbn_t &m) const { + if(_context.check_errors()) { + if(mpz_cmp(a._z, m._z)>=0) { + _context.report_error(cgbn_division_overflow_error); + return; + } + } + mpz_powm(r._z, a._z, k._z, m._z); +} + +template +void cgbn_env_t::gcd(cgbn_t &r, const cgbn_t &a, const cgbn_t &b) const { + mpz_gcd(r._z, a._z, b._z); +} + + + +/* fast division: common divisor / modulus */ +template +uint32_t cgbn_env_t::bn2mont(cgbn_t &mont, const cgbn_t &bn, const cgbn_t &n) const { + mpz_t temp; + uint32_t n0, inv; + + if(_context.check_errors()) { + if(mpz_tstbit(n._z, 0)==0) { + _context.report_error(cgbn_modulus_not_odd_error); + return 0; + } + if(mpz_cmp(bn._z, n._z)>=0) { + _context.report_error(cgbn_division_overflow_error); + return 0; + } + } + mpz_init(temp); + mpz_mul_2exp(temp, bn._z, UNPADDED_BITS); + mpz_fdiv_r(mont._z, temp, n._z); + mpz_clear(temp); + + n0=mpz_get_ui(n._z); + inv=n0*(n0*n0+14); + inv=inv*(inv*n0+2); + inv=inv*(inv*n0+2); + inv=inv*(inv*n0+2); + return inv; +} + +template +void cgbn_env_t::mont2bn(cgbn_t &bn, const cgbn_t &mont, const cgbn_t &n, uint32_t np0) const { + mpz_t prod, add; + int32_t index; + uint32_t low; + + if(_context.check_errors()) { + if(np0*(uint32_t)mpz_get_ui(n._z)!=0xFFFFFFFF) { + _context.report_error(cgbn_invalid_montgomery_modulus_error); + return; + } + } + mpz_init(prod); + mpz_init(add); + mpz_set(prod, mont._z); + for(index=0;index +void cgbn_env_t::mont_mul(cgbn_t &r, const cgbn_t &a, const cgbn_t &b, const cgbn_t &n, uint32_t np0) const { + mpz_t prod, add; + int32_t index; + uint32_t low; + + if(_context.check_errors()) { + if(np0*(uint32_t)mpz_get_ui(n._z)!=0xFFFFFFFF) { + _context.report_error(cgbn_invalid_montgomery_modulus_error); + return; + } + } + mpz_init(prod); + mpz_init(add); + mpz_mul(prod, a._z, b._z); + for(index=0;index +void cgbn_env_t::mont_sqr(cgbn_t &r, const cgbn_t &a, const cgbn_t &n, uint32_t np0) const { + mpz_t prod, add; + int32_t index; + uint32_t low; + + if(_context.check_errors()) { + if(np0*(uint32_t)mpz_get_ui(n._z)!=0xFFFFFFFF) { + _context.report_error(cgbn_invalid_montgomery_modulus_error); + return; + } + } + mpz_init(prod); + mpz_init(add); + mpz_mul(prod, a._z, a._z); + for(index=0;index +void cgbn_env_t::mont_reduce_wide(cgbn_t &r, const cgbn_wide_t &a, const cgbn_t &n, uint32_t np0) const { + mpz_t prod, add; + int32_t index; + uint32_t low; + + if(_context.check_errors()) { + if(np0*(uint32_t)mpz_get_ui(n._z)!=0xFFFFFFFF) { + _context.report_error(cgbn_invalid_montgomery_modulus_error); + return; + } + } + mpz_init(prod); + mpz_init(add); + mpz_set(prod, a._high._z); + mpz_mul_2exp(prod, prod, bits); + mpz_add(prod, prod, a._low._z); + for(index=0;index +uint32_t cgbn_env_t::barrett_approximation(cgbn_t &approx, const cgbn_t &denom) const { + mpz_t temp, shifted; + uint32_t clz; + + if(_context.check_errors()) { + if(mpz_sgn(denom._z)==0) { + _context.report_error(cgbn_division_by_zero_error); + return 0xFFFFFFFF; + } + } + + if(mpz_sgn(denom._z)==0) + return 0xFFFFFFFF; + clz=bits-mpz_sizeinbase(denom._z, 2); + + mpz_init(temp); + mpz_init(shifted); + mpz_mul_2exp(shifted, denom._z, clz); + mpz_setbit(temp, bits-1); + if(mpz_cmp(temp, shifted)==0) { + mpz_clrbit(temp, bits-1); + mpz_setbit(temp, bits); + mpz_sub_ui(approx._z, temp, 1); + } + else { + mpz_set_ui(temp, 0); + mpz_setbit(temp, 2*bits); + mpz_fdiv_q(approx._z, temp, shifted); + mpz_clrbit(approx._z, bits); + } + mpz_clear(temp); + mpz_clear(shifted); + return clz; +} + +template +void cgbn_env_t::barrett_div(cgbn_t &q, const cgbn_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const { + mpz_t temp, quot, rem; + int32_t count; + + mpz_init(temp); + mpz_init(quot); + mpz_init(rem); + + // compute top + mpz_fdiv_q_2exp(temp, num._z, bits-denom_clz); + + // q=(top * approx + top)>>bits + mpz_mul(quot, temp, approx._z); + mpz_fdiv_q_2exp(quot, quot, bits); + mpz_add(quot, quot, temp); + + // q=min(q+3, beta-1), min might be unnecessary + mpz_add_ui(quot, quot, 3); + if(mpz_tstbit(quot, bits)==1) { + mpz_set_ui(quot, 1); + mpz_mul_2exp(quot, quot, bits); + mpz_sub_ui(quot, quot, 1); + } + + // r=num-q*denom + mpz_mul(rem, quot, denom._z); + mpz_sub(rem, num._z, rem); + + // correction steps + count=0; + while(count<3 && mpz_sgn(rem)==-1) { + mpz_sub_ui(quot, quot, 1); + mpz_add(rem, rem, denom._z); + count++; + } + + // check it + mpz_fdiv_q(temp, num._z, denom._z); + if(mpz_cmp(quot, temp)!=0) { + printf(" num: "); + mpz_out_str(stdout, 16, num._z); + printf("\n"); + printf(" denom: "); + mpz_out_str(stdout, 16, denom._z); + printf("\n"); + printf(" approx: "); + mpz_out_str(stdout, 16, approx._z); + printf("\n"); + printf(" rem: "); + mpz_out_str(stdout, 16, rem); + printf("\n"); + printf(" q: "); + mpz_out_str(stdout, 16, quot); + printf("\n"); + printf("correct: "); + mpz_out_str(stdout, 16, temp); + printf("\n"); + fprintf(stderr, "barrett failed\n"); + exit(1); + } + + mpz_set(q._z, quot); + + mpz_clear(temp); + mpz_clear(quot); + mpz_clear(rem); +} + +template +void cgbn_env_t::barrett_rem(cgbn_t &q, const cgbn_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const { + mpz_t temp, quot, rem; + int32_t count; + + mpz_init(temp); + mpz_init(quot); + mpz_init(rem); + + // compute top + mpz_fdiv_q_2exp(temp, num._z, bits-denom_clz); + + // q=(top * approx + top)>>bits + mpz_mul(quot, temp, approx._z); + mpz_fdiv_q_2exp(quot, quot, bits); + mpz_add(quot, quot, temp); + + // q=min(q+3, beta-1), min might be unnecessary + mpz_add_ui(quot, quot, 3); + if(mpz_tstbit(quot, bits)==1) { + mpz_set_ui(quot, 1); + mpz_mul_2exp(quot, quot, bits); + mpz_sub_ui(quot, quot, 1); + } + + // r=num-q*denom + mpz_mul(rem, quot, denom._z); + mpz_sub(rem, num._z, rem); + + // correction steps + count=0; + while(count<3 && mpz_sgn(rem)==-1) { + mpz_add(rem, rem, denom._z); + count++; + } + + // check it + mpz_fdiv_r(temp, num._z, denom._z); + if(mpz_cmp(rem, temp)!=0) { + printf(" num: "); + mpz_out_str(stdout, 16, num._z); + printf("\n"); + printf(" denom: "); + mpz_out_str(stdout, 16, denom._z); + printf("\n"); + printf(" approx: "); + mpz_out_str(stdout, 16, approx._z); + printf("\n"); + printf(" rem: "); + mpz_out_str(stdout, 16, rem); + printf("\n"); + printf(" q: "); + mpz_out_str(stdout, 16, quot); + printf("\n"); + printf("correct: "); + mpz_out_str(stdout, 16, temp); + printf("\n"); + fprintf(stderr, "barrett failed\n"); + exit(1); + } + + mpz_set(q._z, rem); + + mpz_clear(temp); + mpz_clear(quot); + mpz_clear(rem); +} + +template +void cgbn_env_t::barrett_div_rem(cgbn_t &q, cgbn_t &r, const cgbn_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const { + mpz_t temp, quot, rem; + int32_t count; + + mpz_init(temp); + mpz_init(quot); + mpz_init(rem); + + // compute top + mpz_fdiv_q_2exp(temp, num._z, bits-denom_clz); + + // q=(top * approx + top)>>bits + mpz_mul(quot, temp, approx._z); + mpz_fdiv_q_2exp(quot, quot, bits); + mpz_add(quot, quot, temp); + + // q=min(q+3, beta-1), min might be unnecessary + mpz_add_ui(quot, quot, 3); + if(mpz_tstbit(quot, bits)==1) { + mpz_set_ui(quot, 1); + mpz_mul_2exp(quot, quot, bits); + mpz_sub_ui(quot, quot, 1); + } + + // r=num-q*denom + mpz_mul(rem, quot, denom._z); + mpz_sub(rem, num._z, rem); + + // correction steps + count=0; + while(count<3 && mpz_sgn(rem)==-1) { + mpz_sub_ui(quot, quot, 1); + mpz_add(rem, rem, denom._z); + count++; + } + + // check it + mpz_fdiv_q(temp, num._z, denom._z); + if(mpz_cmp(quot, temp)!=0) { + printf(" num: "); + mpz_out_str(stdout, 16, num._z); + printf("\n"); + printf(" denom: "); + mpz_out_str(stdout, 16, denom._z); + printf("\n"); + printf(" approx: "); + mpz_out_str(stdout, 16, approx._z); + printf("\n"); + printf(" rem: "); + mpz_out_str(stdout, 16, rem); + printf("\n"); + printf(" q: "); + mpz_out_str(stdout, 16, quot); + printf("\n"); + printf("correct: "); + mpz_out_str(stdout, 16, temp); + printf("\n"); + fprintf(stderr, "barrett failed\n"); + exit(1); + } + + mpz_set(q._z, quot); + mpz_set(r._z, rem); + + mpz_clear(temp); + mpz_clear(quot); + mpz_clear(rem); +} + +template +void cgbn_env_t::barrett_div_wide(cgbn_t &q, const cgbn_wide_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const { + mpz_t temp; + + if(_context.check_errors()) { + if(mpz_cmp(num._high._z, denom._z)>=0) { + _context.report_error(cgbn_division_overflow_error); + return; + } + } + mpz_init(temp); + mpz_set(temp, num._high._z); + mpz_mul_2exp(temp, temp, bits); + mpz_add(temp, temp, num._low._z); + mpz_fdiv_q(q._z, temp, denom._z); + mpz_clear(temp); +} + +template +void cgbn_env_t::barrett_rem_wide(cgbn_t &r, const cgbn_wide_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const { + mpz_t temp; + + if(_context.check_errors()) { + if(mpz_cmp(num._high._z, denom._z)>=0) { + _context.report_error(cgbn_division_overflow_error); + return; + } + } + mpz_init(temp); + mpz_set(temp, num._high._z); + mpz_mul_2exp(temp, temp, bits); + mpz_add(temp, temp, num._low._z); + mpz_fdiv_r(r._z, temp, denom._z); + mpz_clear(temp); +} + +template +void cgbn_env_t::barrett_div_rem_wide(cgbn_t &q, cgbn_t &r, const cgbn_wide_t &num, const cgbn_t &denom, const cgbn_t &approx, const uint32_t denom_clz) const { + mpz_t temp; + + if(_context.check_errors()) { + if(mpz_cmp(num._high._z, denom._z)>=0) { + _context.report_error(cgbn_division_overflow_error); + return; + } + } + mpz_init(temp); + mpz_set(temp, num._high._z); + mpz_mul_2exp(temp, temp, bits); + mpz_add(temp, temp, num._low._z); + mpz_fdiv_q(q._z, temp, denom._z); + mpz_fdiv_r(r._z, temp, denom._z); + mpz_clear(temp); +} + + +/* +template +void cgbn_env_t::barrett_div_wide(cgbn_t &q, const cgbn_wide_t &num, const cgbn_t &denom, const cgbn_t &approx) const { + mpz_t temp; + + if(_context.check_errors()) { + if(mpz_cmp(num._high._z, denom._z)>=0) { + _context.report_error(cgbn_division_overflow_error); + return; + } + } + mpz_init(temp); + mpz_set(temp, num._high._z); + mpz_mul_2exp(temp, temp, bits); + mpz_add(temp, temp, num._low._z); + mpz_fdiv_q(q._z, temp, denom._z); + mpz_clear(temp); +} +*/ + +/* load and store routines */ +template +void cgbn_env_t::load(cgbn_t &r, cgbn_mem_t *const address) const { + mpz_import(r._z, (bits+31)/32, -1, sizeof(uint32_t), 0, 0, (uint32_t *)address); +} + +template +void cgbn_env_t::store(cgbn_mem_t *address, const cgbn_t &a) const { + size_t words; + + if(mpz_sizeinbase(a._z, 2)>bits) { + fprintf(stderr, "from_mpz failed -- result does not fit"); + exit(1); + } + + mpz_export((uint32_t *)address, &words, -1, sizeof(uint32_t), 0, 0, a._z); + while(words<(bits+31)/32) + ((uint32_t *)address)[words++]=0; +} + +template +void cgbn_env_t::load(cgbn_t &r, cgbn_local_t *const address) const { + mpz_set(r._z, address->_z); +} + +template +void cgbn_env_t::store(cgbn_local_t *address, const cgbn_t &a) const { + mpz_set(address->_z, a._z); +} + diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/attr.h b/src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/attr.h new file mode 100644 index 0000000..9b631fa --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/attr.h @@ -0,0 +1,722 @@ +/* + pybind11/attr.h: Infrastructure for processing custom + type and function attributes + + Copyright (c) 2016 Wenzel Jakob + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "detail/common.h" +#include "cast.h" +#include "trampoline_self_life_support.h" + +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +/// \addtogroup annotations +/// @{ + +/// Annotation for methods +struct is_method { + handle class_; + explicit is_method(const handle &c) : class_(c) {} +}; + +/// Annotation for setters +struct is_setter {}; + +/// Annotation for operators +struct is_operator {}; + +/// Annotation for classes that cannot be subclassed +struct is_final {}; + +/// Annotation for parent scope +struct scope { + handle value; + explicit scope(const handle &s) : value(s) {} +}; + +/// Annotation for documentation +struct doc { + const char *value; + explicit doc(const char *value) : value(value) {} +}; + +/// Annotation for function names +struct name { + const char *value; + explicit name(const char *value) : value(value) {} +}; + +/// Annotation indicating that a function is an overload associated with a given "sibling" +struct sibling { + handle value; + explicit sibling(const handle &value) : value(value.ptr()) {} +}; + +/// Annotation indicating that a class derives from another given type +template +struct base { + + PYBIND11_DEPRECATED( + "base() was deprecated in favor of specifying 'T' as a template argument to class_") + base() = default; +}; + +/// Keep patient alive while nurse lives +template +struct keep_alive {}; + +/// Annotation indicating that a class is involved in a multiple inheritance relationship +struct multiple_inheritance {}; + +/// Annotation which enables dynamic attributes, i.e. adds `__dict__` to a class +struct dynamic_attr {}; + +/// Annotation which enables the buffer protocol for a type +struct buffer_protocol {}; + +/// Annotation which enables releasing the GIL before calling the C++ destructor of wrapped +/// instances (pybind/pybind11#1446). +struct release_gil_before_calling_cpp_dtor {}; + +/// Annotation which requests that a special metaclass is created for a type +struct metaclass { + handle value; + + PYBIND11_DEPRECATED("py::metaclass() is no longer required. It's turned on by default now.") + metaclass() = default; + + /// Override pybind11's default metaclass + explicit metaclass(handle value) : value(value) {} +}; + +/// Specifies a custom callback with signature `void (PyHeapTypeObject*)` that +/// may be used to customize the Python type. +/// +/// The callback is invoked immediately before `PyType_Ready`. +/// +/// Note: This is an advanced interface, and uses of it may require changes to +/// work with later versions of pybind11. You may wish to consult the +/// implementation of `make_new_python_type` in `detail/classes.h` to understand +/// the context in which the callback will be run. +struct custom_type_setup { + using callback = std::function; + + explicit custom_type_setup(callback value) : value(std::move(value)) {} + + callback value; +}; + +/// Annotation that marks a class as local to the module: +struct module_local { + const bool value; + constexpr explicit module_local(bool v = true) : value(v) {} +}; + +/// Annotation to mark enums as an arithmetic type +struct arithmetic {}; + +/// Mark a function for addition at the beginning of the existing overload chain instead of the end +struct prepend {}; + +/** \rst + A call policy which places one or more guard variables (``Ts...``) around the function call. + + For example, this definition: + + .. code-block:: cpp + + m.def("foo", foo, py::call_guard()); + + is equivalent to the following pseudocode: + + .. code-block:: cpp + + m.def("foo", [](args...) { + T scope_guard; + return foo(args...); // forwarded arguments + }); + \endrst */ +template +struct call_guard; + +template <> +struct call_guard<> { + using type = detail::void_type; +}; + +template +struct call_guard { + static_assert(std::is_default_constructible::value, + "The guard type must be default constructible"); + + using type = T; +}; + +template +struct call_guard { + struct type { + T guard{}; // Compose multiple guard types with left-to-right default-constructor order + typename call_guard::type next{}; + }; +}; + +/// @} annotations + +PYBIND11_NAMESPACE_BEGIN(detail) +/* Forward declarations */ +enum op_id : int; +enum op_type : int; +struct undefined_t; +template +struct op_; +void keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret); + +/// Internal data structure which holds metadata about a keyword argument +struct argument_record { + const char *name; ///< Argument name + const char *descr; ///< Human-readable version of the argument value + handle value; ///< Associated Python object + bool convert : 1; ///< True if the argument is allowed to convert when loading + bool none : 1; ///< True if None is allowed when loading + + argument_record(const char *name, const char *descr, handle value, bool convert, bool none) + : name(name), descr(descr), value(value), convert(convert), none(none) {} +}; + +/// Internal data structure which holds metadata about a bound function (signature, overloads, +/// etc.) +#define PYBIND11_DETAIL_FUNCTION_RECORD_ABI_ID "v1" // PLEASE UPDATE if the struct is changed. +struct function_record { + function_record() + : is_constructor(false), is_new_style_constructor(false), is_stateless(false), + is_operator(false), is_method(false), is_setter(false), has_args(false), + has_kwargs(false), prepend(false) {} + + /// Function name + char *name = nullptr; /* why no C++ strings? They generate heavier code.. */ + + // User-specified documentation string + char *doc = nullptr; + + /// Human-readable version of the function signature + char *signature = nullptr; + + /// List of registered keyword arguments + std::vector args; + + /// Pointer to lambda function which converts arguments and performs the actual call + handle (*impl)(function_call &) = nullptr; + + /// Storage for the wrapped function pointer and captured data, if any + void *data[3] = {}; + + /// Pointer to custom destructor for 'data' (if needed) + void (*free_data)(function_record *ptr) = nullptr; + + /// Return value policy associated with this function + return_value_policy policy = return_value_policy::automatic; + + /// True if name == '__init__' + bool is_constructor : 1; + + /// True if this is a new-style `__init__` defined in `detail/init.h` + bool is_new_style_constructor : 1; + + /// True if this is a stateless function pointer + bool is_stateless : 1; + + /// True if this is an operator (__add__), etc. + bool is_operator : 1; + + /// True if this is a method + bool is_method : 1; + + /// True if this is a setter + bool is_setter : 1; + + /// True if the function has a '*args' argument + bool has_args : 1; + + /// True if the function has a '**kwargs' argument + bool has_kwargs : 1; + + /// True if this function is to be inserted at the beginning of the overload resolution chain + bool prepend : 1; + + /// Number of arguments (including py::args and/or py::kwargs, if present) + std::uint16_t nargs; + + /// Number of leading positional arguments, which are terminated by a py::args or py::kwargs + /// argument or by a py::kw_only annotation. + std::uint16_t nargs_pos = 0; + + /// Number of leading arguments (counted in `nargs`) that are positional-only + std::uint16_t nargs_pos_only = 0; + + /// Python method object + PyMethodDef *def = nullptr; + + /// Python handle to the parent scope (a class or a module) + handle scope; + + /// Python handle to the sibling function representing an overload chain + handle sibling; + + /// Pointer to next overload + function_record *next = nullptr; +}; +// The main purpose of this macro is to make it easy to pin-point the critically related code +// sections. +#define PYBIND11_ENSURE_PRECONDITION_FOR_FUNCTIONAL_H_PERFORMANCE_OPTIMIZATIONS(...) \ + static_assert( \ + __VA_ARGS__, \ + "Violation of precondition for pybind11/functional.h performance optimizations!") + +/// Special data structure which (temporarily) holds metadata about a bound class +struct type_record { + PYBIND11_NOINLINE type_record() + : multiple_inheritance(false), dynamic_attr(false), buffer_protocol(false), + module_local(false), is_final(false), release_gil_before_calling_cpp_dtor(false) {} + + /// Handle to the parent scope + handle scope; + + /// Name of the class + const char *name = nullptr; + + // Pointer to RTTI type_info data structure + const std::type_info *type = nullptr; + + /// How large is the underlying C++ type? + size_t type_size = 0; + + /// What is the alignment of the underlying C++ type? + size_t type_align = 0; + + /// How large is the type's holder? + size_t holder_size = 0; + + /// The global operator new can be overridden with a class-specific variant + void *(*operator_new)(size_t) = nullptr; + + /// Function pointer to class_<..>::init_instance + void (*init_instance)(instance *, const void *) = nullptr; + + /// Function pointer to class_<..>::dealloc + void (*dealloc)(detail::value_and_holder &) = nullptr; + + /// Function pointer for casting alias class (aka trampoline) pointer to + /// trampoline_self_life_support pointer. Sidesteps cross-DSO RTTI issues + /// on platforms like macOS (see PR #5728 for details). + get_trampoline_self_life_support_fn get_trampoline_self_life_support + = [](void *) -> trampoline_self_life_support * { return nullptr; }; + + /// List of base classes of the newly created type + list bases; + + /// Optional docstring + const char *doc = nullptr; + + /// Custom metaclass (optional) + handle metaclass; + + /// Custom type setup. + custom_type_setup::callback custom_type_setup_callback; + + /// Multiple inheritance marker + bool multiple_inheritance : 1; + + /// Does the class manage a __dict__? + bool dynamic_attr : 1; + + /// Does the class implement the buffer protocol? + bool buffer_protocol : 1; + + /// Is the class definition local to the module shared object? + bool module_local : 1; + + /// Is the class inheritable from python classes? + bool is_final : 1; + + /// Solves pybind/pybind11#1446 + bool release_gil_before_calling_cpp_dtor : 1; + + holder_enum_t holder_enum_v = holder_enum_t::undefined; + + PYBIND11_NOINLINE void add_base(const std::type_info &base, void *(*caster)(void *) ) { + auto *base_info = detail::get_type_info(base, false); + if (!base_info) { + std::string tname(base.name()); + detail::clean_type_id(tname); + pybind11_fail("generic_type: type \"" + std::string(name) + + "\" referenced unknown base type \"" + tname + "\""); + } + + // SMART_HOLDER_BAKEIN_FOLLOW_ON: Refine holder compatibility checks. + bool this_has_unique_ptr_holder = (holder_enum_v == holder_enum_t::std_unique_ptr); + bool base_has_unique_ptr_holder + = (base_info->holder_enum_v == holder_enum_t::std_unique_ptr); + if (this_has_unique_ptr_holder != base_has_unique_ptr_holder) { + std::string tname(base.name()); + detail::clean_type_id(tname); + pybind11_fail("generic_type: type \"" + std::string(name) + "\" " + + (this_has_unique_ptr_holder ? "does not have" : "has") + + " a non-default holder type while its base \"" + tname + "\" " + + (base_has_unique_ptr_holder ? "does not" : "does")); + } + + bases.append((PyObject *) base_info->type); + +#ifdef PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET + dynamic_attr |= base_info->type->tp_dictoffset != 0; +#else + dynamic_attr |= (base_info->type->tp_flags & Py_TPFLAGS_MANAGED_DICT) != 0; +#endif + + if (caster) { + base_info->implicit_casts.emplace_back(type, caster); + } + } +}; + +inline function_call::function_call(const function_record &f, handle p) : func(f), parent(p) { + args.reserve(f.nargs); + args_convert.reserve(f.nargs); +} + +/// Tag for a new-style `__init__` defined in `detail/init.h` +struct is_new_style_constructor {}; + +/** + * Partial template specializations to process custom attributes provided to + * cpp_function_ and class_. These are either used to initialize the respective + * fields in the type_record and function_record data structures or executed at + * runtime to deal with custom call policies (e.g. keep_alive). + */ +template +struct process_attribute; + +template +struct process_attribute_default { + /// Default implementation: do nothing + static void init(const T &, function_record *) {} + static void init(const T &, type_record *) {} + static void precall(function_call &) {} + static void postcall(function_call &, handle) {} +}; + +/// Process an attribute specifying the function's name +template <> +struct process_attribute : process_attribute_default { + static void init(const name &n, function_record *r) { r->name = const_cast(n.value); } +}; + +/// Process an attribute specifying the function's docstring +template <> +struct process_attribute : process_attribute_default { + static void init(const doc &n, function_record *r) { r->doc = const_cast(n.value); } +}; + +/// Process an attribute specifying the function's docstring (provided as a C-style string) +template <> +struct process_attribute : process_attribute_default { + static void init(const char *d, function_record *r) { r->doc = const_cast(d); } + static void init(const char *d, type_record *r) { r->doc = d; } +}; +template <> +struct process_attribute : process_attribute {}; + +/// Process an attribute indicating the function's return value policy +template <> +struct process_attribute : process_attribute_default { + static void init(const return_value_policy &p, function_record *r) { r->policy = p; } +}; + +/// Process an attribute which indicates that this is an overloaded function associated with a +/// given sibling +template <> +struct process_attribute : process_attribute_default { + static void init(const sibling &s, function_record *r) { r->sibling = s.value; } +}; + +/// Process an attribute which indicates that this function is a method +template <> +struct process_attribute : process_attribute_default { + static void init(const is_method &s, function_record *r) { + r->is_method = true; + r->scope = s.class_; + } +}; + +/// Process an attribute which indicates that this function is a setter +template <> +struct process_attribute : process_attribute_default { + static void init(const is_setter &, function_record *r) { r->is_setter = true; } +}; + +/// Process an attribute which indicates the parent scope of a method +template <> +struct process_attribute : process_attribute_default { + static void init(const scope &s, function_record *r) { r->scope = s.value; } +}; + +/// Process an attribute which indicates that this function is an operator +template <> +struct process_attribute : process_attribute_default { + static void init(const is_operator &, function_record *r) { r->is_operator = true; } +}; + +template <> +struct process_attribute + : process_attribute_default { + static void init(const is_new_style_constructor &, function_record *r) { + r->is_new_style_constructor = true; + } +}; + +inline void check_kw_only_arg(const arg &a, function_record *r) { + if (r->args.size() > r->nargs_pos && (!a.name || a.name[0] == '\0')) { + pybind11_fail("arg(): cannot specify an unnamed argument after a kw_only() annotation or " + "args() argument"); + } +} + +inline void append_self_arg_if_needed(function_record *r) { + if (r->is_method && r->args.empty()) { + r->args.emplace_back("self", nullptr, handle(), /*convert=*/true, /*none=*/false); + } +} + +/// Process a keyword argument attribute (*without* a default value) +template <> +struct process_attribute : process_attribute_default { + static void init(const arg &a, function_record *r) { + append_self_arg_if_needed(r); + r->args.emplace_back(a.name, nullptr, handle(), !a.flag_noconvert, a.flag_none); + + check_kw_only_arg(a, r); + } +}; + +/// Process a keyword argument attribute (*with* a default value) +template <> +struct process_attribute : process_attribute_default { + static void init(const arg_v &a, function_record *r) { + if (r->is_method && r->args.empty()) { + r->args.emplace_back( + "self", /*descr=*/nullptr, /*parent=*/handle(), /*convert=*/true, /*none=*/false); + } + + if (!a.value) { +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + std::string descr("'"); + if (a.name) { + descr += std::string(a.name) + ": "; + } + descr += a.type + "'"; + if (r->is_method) { + if (r->name) { + descr += " in method '" + (std::string) str(r->scope) + "." + + (std::string) r->name + "'"; + } else { + descr += " in method of '" + (std::string) str(r->scope) + "'"; + } + } else if (r->name) { + descr += " in function '" + (std::string) r->name + "'"; + } + pybind11_fail("arg(): could not convert default argument " + descr + + " into a Python object (type not registered yet?)"); +#else + pybind11_fail("arg(): could not convert default argument " + "into a Python object (type not registered yet?). " + "#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for " + "more information."); +#endif + } + r->args.emplace_back(a.name, a.descr, a.value.inc_ref(), !a.flag_noconvert, a.flag_none); + + check_kw_only_arg(a, r); + } +}; + +/// Process a keyword-only-arguments-follow pseudo argument +template <> +struct process_attribute : process_attribute_default { + static void init(const kw_only &, function_record *r) { + append_self_arg_if_needed(r); + if (r->has_args && r->nargs_pos != static_cast(r->args.size())) { + pybind11_fail("Mismatched args() and kw_only(): they must occur at the same relative " + "argument location (or omit kw_only() entirely)"); + } + r->nargs_pos = static_cast(r->args.size()); + } +}; + +/// Process a positional-only-argument maker +template <> +struct process_attribute : process_attribute_default { + static void init(const pos_only &, function_record *r) { + append_self_arg_if_needed(r); + r->nargs_pos_only = static_cast(r->args.size()); + if (r->nargs_pos_only > r->nargs_pos) { + pybind11_fail("pos_only(): cannot follow a py::args() argument"); + } + // It also can't follow a kw_only, but a static_assert in pybind11.h checks that + } +}; + +/// Process a parent class attribute. Single inheritance only (class_ itself already guarantees +/// that) +template +struct process_attribute::value>> + : process_attribute_default { + static void init(const handle &h, type_record *r) { r->bases.append(h); } +}; + +/// Process a parent class attribute (deprecated, does not support multiple inheritance) +template +struct process_attribute> : process_attribute_default> { + static void init(const base &, type_record *r) { r->add_base(typeid(T), nullptr); } +}; + +/// Process a multiple inheritance attribute +template <> +struct process_attribute : process_attribute_default { + static void init(const multiple_inheritance &, type_record *r) { + r->multiple_inheritance = true; + } +}; + +template <> +struct process_attribute : process_attribute_default { + static void init(const dynamic_attr &, type_record *r) { r->dynamic_attr = true; } +}; + +template <> +struct process_attribute { + static void init(const custom_type_setup &value, type_record *r) { + r->custom_type_setup_callback = value.value; + } +}; + +template <> +struct process_attribute : process_attribute_default { + static void init(const is_final &, type_record *r) { r->is_final = true; } +}; + +template <> +struct process_attribute : process_attribute_default { + static void init(const buffer_protocol &, type_record *r) { r->buffer_protocol = true; } +}; + +template <> +struct process_attribute : process_attribute_default { + static void init(const metaclass &m, type_record *r) { r->metaclass = m.value; } +}; + +template <> +struct process_attribute : process_attribute_default { + static void init(const module_local &l, type_record *r) { r->module_local = l.value; } +}; + +template <> +struct process_attribute + : process_attribute_default { + static void init(const release_gil_before_calling_cpp_dtor &, type_record *r) { + r->release_gil_before_calling_cpp_dtor = true; + } +}; + +/// Process a 'prepend' attribute, putting this at the beginning of the overload chain +template <> +struct process_attribute : process_attribute_default { + static void init(const prepend &, function_record *r) { r->prepend = true; } +}; + +/// Process an 'arithmetic' attribute for enums (does nothing here) +template <> +struct process_attribute : process_attribute_default {}; + +template +struct process_attribute> : process_attribute_default> {}; + +/** + * Process a keep_alive call policy -- invokes keep_alive_impl during the + * pre-call handler if both Nurse, Patient != 0 and use the post-call handler + * otherwise + */ +template +struct process_attribute> + : public process_attribute_default> { + template = 0> + static void precall(function_call &call) { + keep_alive_impl(Nurse, Patient, call, handle()); + } + template = 0> + static void postcall(function_call &, handle) {} + template = 0> + static void precall(function_call &) {} + template = 0> + static void postcall(function_call &call, handle ret) { + keep_alive_impl(Nurse, Patient, call, ret); + } +}; + +/// Recursively iterate over variadic template arguments +template +struct process_attributes { + static void init(const Args &...args, function_record *r) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(r); + PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(r); + using expander = int[]; + (void) expander{ + 0, ((void) process_attribute::type>::init(args, r), 0)...}; + } + static void init(const Args &...args, type_record *r) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(r); + PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(r); + using expander = int[]; + (void) expander{0, + (process_attribute::type>::init(args, r), 0)...}; + } + static void precall(function_call &call) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(call); + using expander = int[]; + (void) expander{0, + (process_attribute::type>::precall(call), 0)...}; + } + static void postcall(function_call &call, handle fn_ret) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(call, fn_ret); + PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(fn_ret); + using expander = int[]; + (void) expander{ + 0, (process_attribute::type>::postcall(call, fn_ret), 0)...}; + } +}; + +template +using is_call_guard = is_instantiation; + +/// Extract the ``type`` from the first `call_guard` in `Extras...` (or `void_type` if none found) +template +using extract_guard_t = typename exactly_one_t, Extra...>::type; + +/// Check the number of named arguments at compile time +template ::value...), + size_t self = constexpr_sum(std::is_same::value...)> +constexpr bool expected_num_args(size_t nargs, bool has_args, bool has_kwargs) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(nargs, has_args, has_kwargs); + return named == 0 || (self + named + size_t(has_args) + size_t(has_kwargs)) == nargs; +} + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/buffer_info.h b/src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/buffer_info.h new file mode 100644 index 0000000..75aec0b --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/buffer_info.h @@ -0,0 +1,208 @@ +/* + pybind11/buffer_info.h: Python buffer object interface + + Copyright (c) 2016 Wenzel Jakob + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "detail/common.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_NAMESPACE_BEGIN(detail) + +// Default, C-style strides +inline std::vector c_strides(const std::vector &shape, ssize_t itemsize) { + auto ndim = shape.size(); + std::vector strides(ndim, itemsize); + if (ndim > 0) { + for (size_t i = ndim - 1; i > 0; --i) { + strides[i - 1] = strides[i] * shape[i]; + } + } + return strides; +} + +// F-style strides; default when constructing an array_t with `ExtraFlags & f_style` +inline std::vector f_strides(const std::vector &shape, ssize_t itemsize) { + auto ndim = shape.size(); + std::vector strides(ndim, itemsize); + for (size_t i = 1; i < ndim; ++i) { + strides[i] = strides[i - 1] * shape[i - 1]; + } + return strides; +} + +template +struct compare_buffer_info; + +PYBIND11_NAMESPACE_END(detail) + +/// Information record describing a Python buffer object +struct buffer_info { + void *ptr = nullptr; // Pointer to the underlying storage + ssize_t itemsize = 0; // Size of individual items in bytes + ssize_t size = 0; // Total number of entries + std::string format; // For homogeneous buffers, this should be set to + // format_descriptor::format() + ssize_t ndim = 0; // Number of dimensions + std::vector shape; // Shape of the tensor (1 entry per dimension) + std::vector strides; // Number of bytes between adjacent entries + // (for each per dimension) + bool readonly = false; // flag to indicate if the underlying storage may be written to + + buffer_info() = default; + + buffer_info(void *ptr, + ssize_t itemsize, + const std::string &format, + ssize_t ndim, + detail::any_container shape_in, + detail::any_container strides_in, + bool readonly = false) + : ptr(ptr), itemsize(itemsize), size(1), format(format), ndim(ndim), + shape(std::move(shape_in)), strides(std::move(strides_in)), readonly(readonly) { + if (ndim != (ssize_t) shape.size() || ndim != (ssize_t) strides.size()) { + pybind11_fail("buffer_info: ndim doesn't match shape and/or strides length"); + } + for (size_t i = 0; i < (size_t) ndim; ++i) { + size *= shape[i]; + } + } + + template + buffer_info(T *ptr, + detail::any_container shape_in, + detail::any_container strides_in, + bool readonly = false) + : buffer_info(private_ctr_tag(), + ptr, + sizeof(T), + format_descriptor::format(), + static_cast(shape_in->size()), + std::move(shape_in), + std::move(strides_in), + readonly) {} + + buffer_info(void *ptr, + ssize_t itemsize, + const std::string &format, + ssize_t size, + bool readonly = false) + : buffer_info(ptr, itemsize, format, 1, {size}, {itemsize}, readonly) {} + + template + buffer_info(T *ptr, ssize_t size, bool readonly = false) + : buffer_info(ptr, sizeof(T), format_descriptor::format(), size, readonly) {} + + template + buffer_info(const T *ptr, ssize_t size, bool readonly = true) + : buffer_info( + const_cast(ptr), sizeof(T), format_descriptor::format(), size, readonly) {} + + explicit buffer_info(Py_buffer *view, bool ownview = true) + : buffer_info( + view->buf, + view->itemsize, + view->format, + view->ndim, + {view->shape, view->shape + view->ndim}, + /* Though buffer::request() requests PyBUF_STRIDES, ctypes objects + * ignore this flag and return a view with NULL strides. + * When strides are NULL, build them manually. */ + view->strides + ? std::vector(view->strides, view->strides + view->ndim) + : detail::c_strides({view->shape, view->shape + view->ndim}, view->itemsize), + (view->readonly != 0)) { + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + this->m_view = view; + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + this->ownview = ownview; + } + + buffer_info(const buffer_info &) = delete; + buffer_info &operator=(const buffer_info &) = delete; + + buffer_info(buffer_info &&other) noexcept { (*this) = std::move(other); } + + buffer_info &operator=(buffer_info &&rhs) noexcept { + ptr = rhs.ptr; + itemsize = rhs.itemsize; + size = rhs.size; + format = std::move(rhs.format); + ndim = rhs.ndim; + shape = std::move(rhs.shape); + strides = std::move(rhs.strides); + std::swap(m_view, rhs.m_view); + std::swap(ownview, rhs.ownview); + readonly = rhs.readonly; + return *this; + } + + ~buffer_info() { + if (m_view && ownview) { + PyBuffer_Release(m_view); + delete m_view; + } + } + + Py_buffer *view() const { return m_view; } + Py_buffer *&view() { return m_view; } + + /* True if the buffer item type is equivalent to `T`. */ + // To define "equivalent" by example: + // `buffer_info::item_type_is_equivalent_to(b)` and + // `buffer_info::item_type_is_equivalent_to(b)` may both be true + // on some platforms, but `int` and `unsigned` will never be equivalent. + // For the ground truth, please inspect `detail::compare_buffer_info<>`. + template + bool item_type_is_equivalent_to() const { + return detail::compare_buffer_info::compare(*this); + } + +private: + struct private_ctr_tag {}; + + buffer_info(private_ctr_tag, + void *ptr, + ssize_t itemsize, + const std::string &format, + ssize_t ndim, + detail::any_container &&shape_in, + detail::any_container &&strides_in, + bool readonly) + : buffer_info( + ptr, itemsize, format, ndim, std::move(shape_in), std::move(strides_in), readonly) {} + + Py_buffer *m_view = nullptr; + bool ownview = false; +}; + +PYBIND11_NAMESPACE_BEGIN(detail) + +template +struct compare_buffer_info { + static bool compare(const buffer_info &b) { + // NOLINTNEXTLINE(bugprone-sizeof-expression) Needed for `PyObject *` + return b.format == format_descriptor::format() && b.itemsize == (ssize_t) sizeof(T); + } +}; + +template +struct compare_buffer_info::value>> { + static bool compare(const buffer_info &b) { + return (size_t) b.itemsize == sizeof(T) + && (b.format == format_descriptor::value + || ((sizeof(T) == sizeof(long)) + && b.format == (std::is_unsigned::value ? "L" : "l")) + || ((sizeof(T) == sizeof(size_t)) + && b.format == (std::is_unsigned::value ? "N" : "n"))); + } +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/cast.h b/src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/cast.h new file mode 100644 index 0000000..60dfea5 --- /dev/null +++ b/src/xtrace_sdk/x_vec/crypto/GPU_include/pybind11/cast.h @@ -0,0 +1,2356 @@ +/* + pybind11/cast.h: Partial template specializations to cast between + C++ and Python types + + Copyright (c) 2016 Wenzel Jakob + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "detail/common.h" +#include "detail/descr.h" +#include "detail/native_enum_data.h" +#include "detail/type_caster_base.h" +#include "detail/typeid.h" +#include "pytypes.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) + +PYBIND11_NAMESPACE_BEGIN(detail) + +template +class type_caster : public type_caster_base {}; +template +using make_caster = type_caster>; + +// Shortcut for calling a caster's `cast_op_type` cast operator for casting a type_caster to a T +template +typename make_caster::template cast_op_type cast_op(make_caster &caster) { + using result_t = typename make_caster::template cast_op_type; // See PR #4893 + return caster.operator result_t(); +} +template +typename make_caster::template cast_op_type::type> +cast_op(make_caster &&caster) { + using result_t = typename make_caster::template cast_op_type< + typename std::add_rvalue_reference::type>; // See PR #4893 + return std::move(caster).operator result_t(); +} + +template +class type_caster_enum_type { +private: + using Underlying = typename std::underlying_type::type; + +public: + static constexpr auto name = const_name(); + + template + static handle cast(SrcType &&src, return_value_policy, handle parent) { + handle native_enum + = global_internals_native_enum_type_map_get_item(std::type_index(typeid(EnumType))); + if (native_enum) { + return native_enum(static_cast(src)).release(); + } + return type_caster_base::cast( + std::forward(src), + // Fixes https://github.com/pybind/pybind11/pull/3643#issuecomment-1022987818: + return_value_policy::copy, + parent); + } + + bool load(handle src, bool convert) { + handle native_enum + = global_internals_native_enum_type_map_get_item(std::type_index(typeid(EnumType))); + if (native_enum) { + if (!isinstance(src, native_enum)) { + return false; + } + type_caster underlying_caster; + if (!underlying_caster.load(src.attr("value"), convert)) { + pybind11_fail("native_enum internal consistency failure."); + } + value = static_cast(static_cast(underlying_caster)); + return true; + } + if (!pybind11_enum_) { + pybind11_enum_.reset(new type_caster_base()); + } + return pybind11_enum_->load(src, convert); + } + + template + using cast_op_type = detail::cast_op_type; + + // NOLINTNEXTLINE(google-explicit-constructor) + operator EnumType *() { + if (!pybind11_enum_) { + return &value; + } + return pybind11_enum_->operator EnumType *(); + } + + // NOLINTNEXTLINE(google-explicit-constructor) + operator EnumType &() { + if (!pybind11_enum_) { + return value; + } + return pybind11_enum_->operator EnumType &(); + } + +private: + std::unique_ptr> pybind11_enum_; + EnumType value; +}; + +template +struct type_caster_enum_type_enabled : std::true_type {}; + +template +struct type_uses_type_caster_enum_type { + static constexpr bool value + = std::is_enum::value && type_caster_enum_type_enabled::value; +}; + +template +class type_caster::value>> + : public type_caster_enum_type {}; + +template ::value, int> = 0> +bool isinstance_native_enum_impl(handle obj, const std::type_info &tp) { + handle native_enum = global_internals_native_enum_type_map_get_item(tp); + if (!native_enum) { + return false; + } + return isinstance(obj, native_enum); +} + +template ::value, int> = 0> +bool isinstance_native_enum_impl(handle, const std::type_info &) { + return false; +} + +template +bool isinstance_native_enum(handle obj, const std::type_info &tp) { + return isinstance_native_enum_impl>(obj, tp); +} + +template +class type_caster> { +private: + using caster_t = make_caster; + caster_t subcaster; + using reference_t = type &; + using subcaster_cast_op_type = typename caster_t::template cast_op_type; + + static_assert( + std::is_same::type &, subcaster_cast_op_type>::value + || std::is_same::value, + "std::reference_wrapper caster requires T to have a caster with an " + "`operator T &()` or `operator const T &()`"); + +public: + bool load(handle src, bool convert) { return subcaster.load(src, convert); } + static constexpr auto name = caster_t::name; + static handle + cast(const std::reference_wrapper &src, return_value_policy policy, handle parent) { + // It is definitely wrong to take ownership of this pointer, so mask that rvp + if (policy == return_value_policy::take_ownership + || policy == return_value_policy::automatic) { + policy = return_value_policy::automatic_reference; + } + return caster_t::cast(&src.get(), policy, parent); + } + template + using cast_op_type = std::reference_wrapper; + explicit operator std::reference_wrapper() { return cast_op(subcaster); } +}; + +#define PYBIND11_TYPE_CASTER(type, py_name) \ +protected: \ + type value; \ + \ +public: \ + static constexpr auto name = py_name; \ + template >::value, \ + int> \ + = 0> \ + static ::pybind11::handle cast( \ + T_ *src, ::pybind11::return_value_policy policy, ::pybind11::handle parent) { \ + if (!src) \ + return ::pybind11::none().release(); \ + if (policy == ::pybind11::return_value_policy::take_ownership) { \ + auto h = cast(std::move(*src), policy, parent); \ + delete src; \ + return h; \ + } \ + return cast(*src, policy, parent); \ + } \ + operator type *() { return &value; } /* NOLINT(bugprone-macro-parentheses) */ \ + operator type &() { return value; } /* NOLINT(bugprone-macro-parentheses) */ \ + operator type &&() && { return std::move(value); } /* NOLINT(bugprone-macro-parentheses) */ \ + template \ + using cast_op_type = ::pybind11::detail::movable_cast_op_type + +template +using is_std_char_type = any_of, /* std::string */ +#if defined(PYBIND11_HAS_U8STRING) + std::is_same, /* std::u8string */ +#endif + std::is_same, /* std::u16string */ + std::is_same, /* std::u32string */ + std::is_same /* std::wstring */ + >; + +template +struct type_caster::value && !is_std_char_type::value>> { + using _py_type_0 = conditional_t; + using _py_type_1 = conditional_t::value, + _py_type_0, + typename std::make_unsigned<_py_type_0>::type>; + using py_type = conditional_t::value, double, _py_type_1>; + +public: + bool load(handle src, bool convert) { + py_type py_value; + + if (!src) { + return false; + } + +#if !defined(PYPY_VERSION) + auto index_check = [](PyObject *o) { return PyIndex_Check(o); }; +#else + // In PyPy 7.3.3, `PyIndex_Check` is implemented by calling `__index__`, + // while CPython only considers the existence of `nb_index`/`__index__`. + auto index_check = [](PyObject *o) { return hasattr(o, "__index__"); }; +#endif + + if (std::is_floating_point::value) { + if (convert || PyFloat_Check(src.ptr())) { + py_value = (py_type) PyFloat_AsDouble(src.ptr()); + } else { + return false; + } + } else if (PyFloat_Check(src.ptr()) + || (!convert && !PYBIND11_LONG_CHECK(src.ptr()) && !index_check(src.ptr()))) { + return false; + } else { + handle src_or_index = src; + // PyPy: 7.3.7's 3.8 does not implement PyLong_*'s __index__ calls. +#if defined(PYPY_VERSION) + object index; + if (!PYBIND11_LONG_CHECK(src.ptr())) { // So: index_check(src.ptr()) + index = reinterpret_steal(PyNumber_Index(src.ptr())); + if (!index) { + PyErr_Clear(); + if (!convert) + return false; + } else { + src_or_index = index; + } + } +#endif + if (std::is_unsigned::value) { + py_value = as_unsigned(src_or_index.ptr()); + } else { // signed integer: + py_value = sizeof(T) <= sizeof(long) + ? (py_type) PyLong_AsLong(src_or_index.ptr()) + : (py_type) PYBIND11_LONG_AS_LONGLONG(src_or_index.ptr()); + } + } + + // Python API reported an error + bool py_err = py_value == (py_type) -1 && PyErr_Occurred(); + + // Check to see if the conversion is valid (integers should match exactly) + // Signed/unsigned checks happen elsewhere + if (py_err + || (std::is_integral::value && sizeof(py_type) != sizeof(T) + && py_value != (py_type) (T) py_value)) { + PyErr_Clear(); + if (py_err && convert && (PyNumber_Check(src.ptr()) != 0)) { + auto tmp = reinterpret_steal(std::is_floating_point::value + ? PyNumber_Float(src.ptr()) + : PyNumber_Long(src.ptr())); + PyErr_Clear(); + return load(tmp, false); + } + return false; + } + + value = (T) py_value; + return true; + } + + template + static typename std::enable_if::value, handle>::type + cast(U src, return_value_policy /* policy */, handle /* parent */) { + return PyFloat_FromDouble((double) src); + } + + template + static typename std::enable_if::value && std::is_signed::value + && (sizeof(U) <= sizeof(long)), + handle>::type + cast(U src, return_value_policy /* policy */, handle /* parent */) { + return PYBIND11_LONG_FROM_SIGNED((long) src); + } + + template + static typename std::enable_if::value && std::is_unsigned::value + && (sizeof(U) <= sizeof(unsigned long)), + handle>::type + cast(U src, return_value_policy /* policy */, handle /* parent */) { + return PYBIND11_LONG_FROM_UNSIGNED((unsigned long) src); + } + + template + static typename std::enable_if::value && std::is_signed::value + && (sizeof(U) > sizeof(long)), + handle>::type + cast(U src, return_value_policy /* policy */, handle /* parent */) { + return PyLong_FromLongLong((long long) src); + } + + template + static typename std::enable_if::value && std::is_unsigned::value + && (sizeof(U) > sizeof(unsigned long)), + handle>::type + cast(U src, return_value_policy /* policy */, handle /* parent */) { + return PyLong_FromUnsignedLongLong((unsigned long long) src); + } + + PYBIND11_TYPE_CASTER(T, + io_name::value>( + "typing.SupportsInt", "int", "typing.SupportsFloat", "float")); +}; + +template +struct void_caster { +public: + bool load(handle src, bool) { + if (src && src.is_none()) { + return true; + } + return false; + } + static handle cast(T, return_value_policy /* policy */, handle /* parent */) { + return none().release(); + } + PYBIND11_TYPE_CASTER(T, const_name("None")); +}; + +template <> +class type_caster : public void_caster {}; + +template <> +class type_caster : public type_caster { +public: + using type_caster::cast; + + bool load(handle h, bool) { + if (!h) { + return false; + } + if (h.is_none()) { + value = nullptr; + return true; + } + + /* Check if this is a capsule */ + if (isinstance(h)) { + value = reinterpret_borrow(h); + return true; + } + + /* Check if this is a C++ type */ + const auto &bases = all_type_info((PyTypeObject *) type::handle_of(h).ptr()); + if (bases.size() == 1) { // Only allowing loading from a single-value type + value = values_and_holders(reinterpret_cast(h.ptr())).begin()->value_ptr(); + return true; + } + + /* Fail */ + return false; + } + + static handle cast(const void *ptr, return_value_policy /* policy */, handle /* parent */) { + if (ptr) { + return capsule(ptr).release(); + } + return none().release(); + } + + template + using cast_op_type = void *&; + explicit operator void *&() { return value; } + static constexpr auto name = const_name(PYBIND11_CAPSULE_TYPE_TYPE_HINT); + +private: + void *value = nullptr; +}; + +template <> +class type_caster : public void_caster {}; + +template <> +class type_caster { +public: + bool load(handle src, bool convert) { + if (!src) { + return false; + } + if (src.ptr() == Py_True) { + value = true; + return true; + } + if (src.ptr() == Py_False) { + value = false; + return true; + } + if (convert || is_numpy_bool(src)) { + // (allow non-implicit conversion for numpy booleans), use strncmp + // since NumPy 1.x had an additional trailing underscore. + + Py_ssize_t res = -1; + if (src.is_none()) { + res = 0; // None is implicitly converted to False + } +#if defined(PYPY_VERSION) + // On PyPy, check that "__bool__" attr exists + else if (hasattr(src, PYBIND11_BOOL_ATTR)) { + res = PyObject_IsTrue(src.ptr()); + } +#else + // Alternate approach for CPython: this does the same as the above, but optimized + // using the CPython API so as to avoid an unneeded attribute lookup. + else if (auto *tp_as_number = Py_TYPE(src.ptr())->tp_as_number) { + if (PYBIND11_NB_BOOL(tp_as_number)) { + res = (*PYBIND11_NB_BOOL(tp_as_number))(src.ptr()); + } + } +#endif + if (res == 0 || res == 1) { + value = (res != 0); + return true; + } + PyErr_Clear(); + } + return false; + } + static handle cast(bool src, return_value_policy /* policy */, handle /* parent */) { + return handle(src ? Py_True : Py_False).inc_ref(); + } + PYBIND11_TYPE_CASTER(bool, const_name("bool")); + +private: + // Test if an object is a NumPy boolean (without fetching the type). + static inline bool is_numpy_bool(handle object) { + const char *type_name = Py_TYPE(object.ptr())->tp_name; + // Name changed to `numpy.bool` in NumPy 2, `numpy.bool_` is needed for 1.x support + return std::strcmp("numpy.bool", type_name) == 0 + || std::strcmp("numpy.bool_", type_name) == 0; + } +}; + +// Helper class for UTF-{8,16,32} C++ stl strings: +template +struct string_caster { + using CharT = typename StringType::value_type; + + // Simplify life by being able to assume standard char sizes (the standard only guarantees + // minimums, but Python requires exact sizes) + static_assert(!std::is_same::value || sizeof(CharT) == 1, + "Unsupported char size != 1"); +#if defined(PYBIND11_HAS_U8STRING) + static_assert(!std::is_same::value || sizeof(CharT) == 1, + "Unsupported char8_t size != 1"); +#endif + static_assert(!std::is_same::value || sizeof(CharT) == 2, + "Unsupported char16_t size != 2"); + static_assert(!std::is_same::value || sizeof(CharT) == 4, + "Unsupported char32_t size != 4"); + // wchar_t can be either 16 bits (Windows) or 32 (everywhere else) + static_assert(!std::is_same::value || sizeof(CharT) == 2 || sizeof(CharT) == 4, + "Unsupported wchar_t size != 2/4"); + static constexpr size_t UTF_N = 8 * sizeof(CharT); + + bool load(handle src, bool) { + handle load_src = src; + if (!src) { + return false; + } + if (!PyUnicode_Check(load_src.ptr())) { + return load_raw(load_src); + } + + // For UTF-8 we avoid the need for a temporary `bytes` object by using + // `PyUnicode_AsUTF8AndSize`. + if (UTF_N == 8) { + Py_ssize_t size = -1; + const auto *buffer + = reinterpret_cast(PyUnicode_AsUTF8AndSize(load_src.ptr(), &size)); + if (!buffer) { + PyErr_Clear(); + return false; + } + value = StringType(buffer, static_cast(size)); + return true; + } + + auto utfNbytes + = reinterpret_steal(PyUnicode_AsEncodedString(load_src.ptr(), + UTF_N == 8 ? "utf-8" + : UTF_N == 16 ? "utf-16" + : "utf-32", + nullptr)); + if (!utfNbytes) { + PyErr_Clear(); + return false; + } + + const auto *buffer + = reinterpret_cast(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr())); + size_t length = (size_t) PYBIND11_BYTES_SIZE(utfNbytes.ptr()) / sizeof(CharT); + // Skip BOM for UTF-16/32 + if (UTF_N > 8) { + buffer++; + length--; + } + value = StringType(buffer, length); + + // If we're loading a string_view we need to keep the encoded Python object alive: + if (IsView) { + loader_life_support::add_patient(utfNbytes); + } + + return true; + } + + static handle + cast(const StringType &src, return_value_policy /* policy */, handle /* parent */) { + const char *buffer = reinterpret_cast(src.data()); + auto nbytes = ssize_t(src.size() * sizeof(CharT)); + handle s = decode_utfN(buffer, nbytes); + if (!s) { + throw error_already_set(); + } + return s; + } + + PYBIND11_TYPE_CASTER(StringType, const_name(PYBIND11_STRING_NAME)); + +private: + static handle decode_utfN(const char *buffer, ssize_t nbytes) { +#if !defined(PYPY_VERSION) + return UTF_N == 8 ? PyUnicode_DecodeUTF8(buffer, nbytes, nullptr) + : UTF_N == 16 ? PyUnicode_DecodeUTF16(buffer, nbytes, nullptr, nullptr) + : PyUnicode_DecodeUTF32(buffer, nbytes, nullptr, nullptr); +#else + // PyPy segfaults when on PyUnicode_DecodeUTF16 (and possibly on PyUnicode_DecodeUTF32 as + // well), so bypass the whole thing by just passing the encoding as a string value, which + // works properly: + return PyUnicode_Decode(buffer, + nbytes, + UTF_N == 8 ? "utf-8" + : UTF_N == 16 ? "utf-16" + : "utf-32", + nullptr); +#endif + } + + // When loading into a std::string or char*, accept a bytes/bytearray object as-is (i.e. + // without any encoding/decoding attempt). For other C++ char sizes this is a no-op. + // which supports loading a unicode from a str, doesn't take this path. + template + bool load_raw(enable_if_t::value, handle> src) { + if (PYBIND11_BYTES_CHECK(src.ptr())) { + // We were passed raw bytes; accept it into a std::string or char* + // without any encoding attempt. + const char *bytes = PYBIND11_BYTES_AS_STRING(src.ptr()); + if (!bytes) { + pybind11_fail("Unexpected PYBIND11_BYTES_AS_STRING() failure."); + } + value = StringType(bytes, (size_t) PYBIND11_BYTES_SIZE(src.ptr())); + return true; + } + if (PyByteArray_Check(src.ptr())) { + // We were passed a bytearray; accept it into a std::string or char* + // without any encoding attempt. + const char *bytearray = PyByteArray_AsString(src.ptr()); + if (!bytearray) { + pybind11_fail("Unexpected PyByteArray_AsString() failure."); + } + value = StringType(bytearray, (size_t) PyByteArray_Size(src.ptr())); + return true; + } + + return false; + } + + template + bool load_raw(enable_if_t::value, handle>) { + return false; + } +}; + +template +struct type_caster, + enable_if_t::value>> + : string_caster> {}; + +#ifdef PYBIND11_HAS_STRING_VIEW +template +struct type_caster, + enable_if_t::value>> + : string_caster, true> {}; +#endif + +// Type caster for C-style strings. We basically use a std::string type caster, but also add the +// ability to use None as a nullptr char* (which the string caster doesn't allow). +template +struct type_caster::value>> { + using StringType = std::basic_string; + using StringCaster = make_caster; + StringCaster str_caster; + bool none = false; + CharT one_char = 0; + +public: + bool load(handle src, bool convert) { + if (!src) { + return false; + } + if (src.is_none()) { + // Defer accepting None to other overloads (if we aren't in convert mode): + if (!convert) { + return false; + } + none = true; + return true; + } + return str_caster.load(src, convert); + } + + static handle cast(const CharT *src, return_value_policy policy, handle parent) { + if (src == nullptr) { + return pybind11::none().release(); + } + return StringCaster::cast(StringType(src), policy, parent); + } + + static handle cast(CharT src, return_value_policy policy, handle parent) { + if (std::is_same::value) { + handle s = PyUnicode_DecodeLatin1((const char *) &src, 1, nullptr); + if (!s) { + throw error_already_set(); + } + return s; + } + return StringCaster::cast(StringType(1, src), policy, parent); + } + + explicit operator CharT *() { + return none ? nullptr : const_cast(static_cast(str_caster).c_str()); + } + explicit operator CharT &() { + if (none) { + throw value_error("Cannot convert None to a character"); + } + + auto &value = static_cast(str_caster); + size_t str_len = value.size(); + if (str_len == 0) { + throw value_error("Cannot convert empty string to a character"); + } + + // If we're in UTF-8 mode, we have two possible failures: one for a unicode character that + // is too high, and one for multiple unicode characters (caught later), so we need to + // figure out how long the first encoded character is in bytes to distinguish between these + // two errors. We also allow want to allow unicode characters U+0080 through U+00FF, as + // those can fit into a single char value. + if (StringCaster::UTF_N == 8 && str_len > 1 && str_len <= 4) { + auto v0 = static_cast(value[0]); + // low bits only: 0-127 + // 0b110xxxxx - start of 2-byte sequence + // 0b1110xxxx - start of 3-byte sequence + // 0b11110xxx - start of 4-byte sequence + size_t char0_bytes = (v0 & 0x80) == 0 ? 1 + : (v0 & 0xE0) == 0xC0 ? 2 + : (v0 & 0xF0) == 0xE0 ? 3 + : 4; + + if (char0_bytes == str_len) { + // If we have a 128-255 value, we can decode it into a single char: + if (char0_bytes == 2 && (v0 & 0xFC) == 0xC0) { // 0x110000xx 0x10xxxxxx + one_char = static_cast(((v0 & 3) << 6) + + (static_cast(value[1]) & 0x3F)); + return one_char; + } + // Otherwise we have a single character, but it's > U+00FF + throw value_error("Character code point not in range(0x100)"); + } + } + + // UTF-16 is much easier: we can only have a surrogate pair for values above U+FFFF, thus a + // surrogate pair with total length 2 instantly indicates a range error (but not a "your + // string was too long" error). + else if (StringCaster::UTF_N == 16 && str_len == 2) { + one_char = static_cast(value[0]); + if (one_char >= 0xD800 && one_char < 0xE000) { + throw value_error("Character code point not in range(0x10000)"); + } + } + + if (str_len != 1) { + throw value_error("Expected a character, but multi-character string found"); + } + + one_char = value[0]; + return one_char; + } + + static constexpr auto name = const_name(PYBIND11_STRING_NAME); + template + using cast_op_type = pybind11::detail::cast_op_type<_T>; +}; + +// Base implementation for std::tuple and std::pair +template