diff --git a/doc/source/bit_generators/rdrand.rst b/doc/source/bit_generators/rdrand.rst index 05514fcb4..ec8b56af7 100644 --- a/doc/source/bit_generators/rdrand.rst +++ b/doc/source/bit_generators/rdrand.rst @@ -15,6 +15,7 @@ Seeding and State ~RDRAND.seed ~RDRAND.state + ~RDRAND.success Parallel generation =================== @@ -37,3 +38,9 @@ Testing :toctree: generated/ ~RDRAND.random_raw + ~RDRAND.checked_raw + +Custom Lock +=========== + +.. autoclass:: RaisingLock diff --git a/doc/source/change-log.rst b/doc/source/change-log.rst index f71b2fe59..e9531b49c 100644 --- a/doc/source/change-log.rst +++ b/doc/source/change-log.rst @@ -15,6 +15,16 @@ Change Log maintained until after NumPy 1.21 (or 2 releases after NumPy 1.19) for users who cannot update NumPy. +v1.19.2 +======= +- Corrected :class:`~randomgen.rdrand.RDRAND` to retry on failures with pause + between retries. Add a parameter ``retry`` which allows the number of retries + to be set. It defaults to the Intel recommended value of 10. Also sets an + exception when the number of retries has been exhausted (very unlikely). See + the :class:`~randomgen.rdrand.RDRAND` docstring with unique considerations + when using :class:`~randomgen.rdrand.RDRAND` that do not occur with deterministic + PRNGs. + v1.19.1 ======= - Added :class:`randomgen.romu.Romu` which is among the fastest available bit generators. diff --git a/doc/source/conf.py b/doc/source/conf.py index 7ac0dbe79..ab6058320 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -8,6 +8,8 @@ # -- Path setup -------------------------------------------------------------- +import sphinx_material + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. @@ -16,7 +18,6 @@ # import sys # sys.path.insert(0, os.path.abspath('.')) import randomgen -import sphinx_material # -- Project information ----------------------------------------------------- diff --git a/doc/source/spelling_wordlist.txt b/doc/source/spelling_wordlist.txt index 306a97c62..d683eee2c 100644 --- a/doc/source/spelling_wordlist.txt +++ b/doc/source/spelling_wordlist.txt @@ -272,3 +272,7 @@ weyl Romu ꭞ romu +unpickling +Skylake +intel +Intrinsics \ No newline at end of file diff --git a/randomgen/_testing.py b/randomgen/_testing.py index 931f00b24..f6300bcc4 100644 --- a/randomgen/_testing.py +++ b/randomgen/_testing.py @@ -45,9 +45,9 @@ def __str__(self): ) ) + from functools import wraps import re import warnings - from functools import wraps class suppress_warnings(object): """ diff --git a/randomgen/common.pyx b/randomgen/common.pyx index 7945b54fa..f1f97358c 100644 --- a/randomgen/common.pyx +++ b/randomgen/common.pyx @@ -119,7 +119,7 @@ cdef class BitGenerator: def random_raw(self, size=None, output=True): """ - random_raw(size=None) + random_raw(size=None, output=True) Return randoms as generated by the underlying BitGenerator @@ -135,7 +135,7 @@ cdef class BitGenerator: Returns ------- - out : uint or ndarray + out : {uint64, ndarray, None} Drawn samples. Notes diff --git a/randomgen/dsfmt.pxd b/randomgen/dsfmt.pxd index 89db59810..96852dff1 100644 --- a/randomgen/dsfmt.pxd +++ b/randomgen/dsfmt.pxd @@ -27,10 +27,10 @@ cdef extern from "src/dsfmt/dsfmt.h": ctypedef DSFMT_STATE_T dsfmt_state_t - double dsfmt_next_double(dsfmt_state_t *state) nogil - uint64_t dsfmt_next64(dsfmt_state_t *state) nogil - uint32_t dsfmt_next32(dsfmt_state_t *state) nogil - uint64_t dsfmt_next_raw(dsfmt_state_t *state) nogil + double dsfmt_next_double(dsfmt_state_t *state) nogil + uint64_t dsfmt_next64(dsfmt_state_t *state) nogil + uint32_t dsfmt_next32(dsfmt_state_t *state) nogil + uint64_t dsfmt_next_raw(dsfmt_state_t *state) nogil void dsfmt_init_gen_rand(dsfmt_t *dsfmt, uint32_t seed) void dsfmt_init_by_array(dsfmt_t *dsfmt, uint32_t init_key[], int key_length) diff --git a/randomgen/efiix64.pxd b/randomgen/efiix64.pxd index aca381dcf..d933328a6 100644 --- a/randomgen/efiix64.pxd +++ b/randomgen/efiix64.pxd @@ -14,8 +14,8 @@ cdef extern from "src/efiix64/efiix64.h": ctypedef EFIIX64_STATE_T efiix64_state_t - uint64_t efiix64_next64(efiix64_state_t *state) nogil - uint32_t efiix64_next32(efiix64_state_t *state) nogil + uint64_t efiix64_next64(efiix64_state_t *state) nogil + uint32_t efiix64_next32(efiix64_state_t *state) nogil void efiix64_seed(efiix64_state_t *state, uint64_t seed[4]) diff --git a/randomgen/entropy.pyx b/randomgen/entropy.pyx index 89c195f1f..c814bba22 100644 --- a/randomgen/entropy.pyx +++ b/randomgen/entropy.pyx @@ -8,7 +8,7 @@ __all__ = ["random_entropy", "seed_by_array"] np.import_array() cdef extern from "src/splitmix64/splitmix64.h": - cdef uint64_t splitmix64_next(uint64_t *state) nogil + cdef uint64_t splitmix64_next(uint64_t *state) nogil cdef extern from "src/entropy/entropy.h": cdef bint entropy_getbytes(void* dest, size_t size) diff --git a/randomgen/mt19937.pxd b/randomgen/mt19937.pxd index eb085df37..83f04906d 100644 --- a/randomgen/mt19937.pxd +++ b/randomgen/mt19937.pxd @@ -8,9 +8,9 @@ cdef extern from "src/mt19937/mt19937.h": ctypedef MT19937_STATE_T mt19937_state_t - uint64_t mt19937_next64(mt19937_state_t *state) nogil - uint32_t mt19937_next32(mt19937_state_t *state) nogil - double mt19937_next_double(mt19937_state_t *state) nogil + uint64_t mt19937_next64(mt19937_state_t *state) nogil + uint32_t mt19937_next32(mt19937_state_t *state) nogil + double mt19937_next_double(mt19937_state_t *state) nogil void mt19937_init_by_array(mt19937_state_t *state, uint32_t *init_key, int key_length) void mt19937_seed(mt19937_state_t *state, uint32_t seed) void mt19937_jump(mt19937_state_t *state) diff --git a/randomgen/mt64.pxd b/randomgen/mt64.pxd index 7d10d57fd..abc93c431 100644 --- a/randomgen/mt64.pxd +++ b/randomgen/mt64.pxd @@ -10,9 +10,9 @@ cdef extern from "src/mt64/mt64.h": ctypedef MT64_STATE_T mt64_state_t - uint64_t mt64_next64(mt64_state_t *state) nogil - uint32_t mt64_next32(mt64_state_t *state) nogil - double mt64_next_double(mt64_state_t *state) nogil + uint64_t mt64_next64(mt64_state_t *state) nogil + uint32_t mt64_next32(mt64_state_t *state) nogil + double mt64_next_double(mt64_state_t *state) nogil void mt64_init_by_array(mt64_state_t *state, uint64_t *init_key, int key_length) void mt64_seed(mt64_state_t *state, uint64_t seed) diff --git a/randomgen/pcg32.pxd b/randomgen/pcg32.pxd index a22b30718..068506e1d 100644 --- a/randomgen/pcg32.pxd +++ b/randomgen/pcg32.pxd @@ -13,9 +13,9 @@ cdef extern from "src/pcg32/pcg32.h": ctypedef PCG32_STATE_T pcg32_state_t - uint64_t pcg32_next64(pcg32_state_t *state) nogil - uint32_t pcg32_next32(pcg32_state_t *state) nogil - double pcg32_next_double(pcg32_state_t *state) nogil + uint64_t pcg32_next64(pcg32_state_t *state) nogil + uint32_t pcg32_next32(pcg32_state_t *state) nogil + double pcg32_next_double(pcg32_state_t *state) nogil void pcg32_jump(pcg32_state_t *state) void pcg32_advance_state(pcg32_state_t *state, uint64_t step) void pcg32_set_seed(pcg32_state_t *state, uint64_t seed, uint64_t inc) diff --git a/randomgen/pcg64.pxd b/randomgen/pcg64.pxd index ed3dc367c..185457de0 100644 --- a/randomgen/pcg64.pxd +++ b/randomgen/pcg64.pxd @@ -15,8 +15,8 @@ cdef extern from "src/pcg64/pcg64-v2.h": ctypedef PCG64_STATE_T pcg64_state_t - uint64_t pcg64_next64(pcg64_state_t *state) nogil - uint32_t pcg64_next32(pcg64_state_t *state) nogil + uint64_t pcg64_next64(pcg64_state_t *state) nogil + uint32_t pcg64_next32(pcg64_state_t *state) nogil uint64_t pcg64_cm_dxsm_next64(pcg64_state_t *state) nogil uint32_t pcg64_cm_dxsm_next32(pcg64_state_t *state) nogil diff --git a/randomgen/rdrand.pxd b/randomgen/rdrand.pxd index e69de29bb..4d8def688 100644 --- a/randomgen/rdrand.pxd +++ b/randomgen/rdrand.pxd @@ -0,0 +1,18 @@ +from randomgen.common cimport * + +DEF BUFFER_SIZE = 256 + +cdef extern from "src/rdrand/rdrand.h": + + struct s_rdrand_state: + uint64_t buffer[BUFFER_SIZE] + int buffer_loc + int status + int retries + uint64_t weyl_seq + + ctypedef s_rdrand_state rdrand_state + + int rdrand_fill_buffer(rdrand_state *state) nogil + int rdrand_next64(rdrand_state *state, uint64_t *val) nogil + int rdrand_capable() \ No newline at end of file diff --git a/randomgen/rdrand.pyx b/randomgen/rdrand.pyx index ead3016ae..34bed90cd 100644 --- a/randomgen/rdrand.pyx +++ b/randomgen/rdrand.pyx @@ -2,33 +2,106 @@ import numpy as np cimport numpy as np from randomgen.common cimport * -from randomgen.entropy import random_entropy, seed_by_array +from cpython cimport PyObject +from cpython.exc cimport PyErr_SetString, PyErr_Occurred, PyErr_Clear, PyErr_Print, PyErr_Fetch, PyErr_SetObject +cimport libc.stdint -__all__ = ["RDRAND"] - -cdef extern from "src/rdrand/rdrand.h": +np.import_array() - struct s_rdrand_state: - int status +__all__ = ["RDRAND"] - ctypedef s_rdrand_state rdrand_state +DEF BUFFER_SIZE = 256 - uint64_t rdrand_next64(rdrand_state* state) nogil - uint32_t rdrand_next32(rdrand_state* state) nogil - int rdrand_capable() +ERROR_MSG = """\ +Unable to get random values from RDRAND after {retries} retries. This can +happen if many process are accessing the hardware random number generator +simultaneously so that its capacity is being constantly exceeded. You can +increase the number of retries to slow down the generation on contested CPUs. +""" cdef uint64_t rdrand_uint64(void* st) nogil: - return rdrand_next64(st) + cdef PyObject *err + cdef rdrand_state *state + cdef int status, prev_status + cdef uint64_t val + state = st + status = 1 + if state.status == 1 and state.buffer_loc < BUFFER_SIZE: + val = state.buffer[state.buffer_loc] + state.buffer_loc += 1 + return val + elif state.status == 1: + # Only refill if good status + # This function will + status = rdrand_fill_buffer(state) + val = state.buffer[state.buffer_loc] + state.buffer_loc += 1 + if status == 0: + # Only raise on a status change + with gil: + err = PyErr_Occurred() + if err == NULL: + retries = state.retries + msg = ERROR_MSG.format(retries=retries).encode("utf8") + PyErr_SetString(RuntimeError, msg) + return val + cdef uint32_t rdrand_uint32(void *st) nogil: - return rdrand_next32(st) + # TODO: This is lazy + return rdrand_uint64(st) cdef double rdrand_double(void* st) nogil: - return uint64_to_double(rdrand_next64(st)) + return uint64_to_double(rdrand_uint64(st)) + + +cdef class RaisingLock: + """ + A Lock that wraps threading.Lock can can raise errors. + + Raises the last set exception that occurrect while the loc was held, + if any. It clears the error when the lock is acquired. + + Notes + ----- + This class has been specially designed for issues unique to RDRAND. + """ + cdef object lock + cdef PyObject *err + + def __init__(self): + self.lock = Lock() + self.err = NULL + + def acquire(self, blocking=True, timeout=-1): + PyErr_Clear() + return self.lock.acquire(blocking, timeout) + + def release(self): + cdef PyObject *typ + cdef PyObject *val + cdef PyObject *tb + + self.err = PyErr_Occurred() + if self.err != NULL: + try: + # Python operation causes error to be raised + print() + except Exception as exc: + self.release() + raise exc + self.lock.release() + + def __enter__(self): + self.acquire() + + def __exit__(self, type, value, traceback): + self.release() + cdef class RDRAND(BitGenerator): """ - RDRAND(seed=None) + RDRAND(seed=None, *, retries=10) Container for the hardware RDRAND random number generator. @@ -36,6 +109,15 @@ cdef class RDRAND(BitGenerator): ---------- seed : None Must be None. Raises if any other value is passed. + retries : int + The number of times to retry. On CPUs with many cores it is possible + for RDRAND to fail if heavily utilized. retries sets the number of + retries before a RuntimeError is raised. Each retry issues a pause + instruction which waits a CPU-specific number of cycles (140 on + Skylake [1]_). The default value of 10 is recommended by Intel ([2]_). + You can set any value up-to the maximum integer size on your platform + if you have issues with errors, although the practical maximum is less + than 100. See Notes for more on the error state. Attributes ---------- @@ -74,6 +156,42 @@ cdef class RDRAND(BitGenerator): >>> from randomgen import Generator, RDRAND >>> rg = [Generator(RDRAND()) for _ in range(10)] + **Exceptions** + + Bit generators are designed to run as quickly as possible to produce + deterministic but chaotic sequences. With the exception of RDRAND, all + other bit generators cannot fail (short of a massive CPU issue). RDRAND + can fail to produce a random value if many threads are all utilizing the + same random generator, and so it is necessary to check a flag to ensure + that the instruction has succeeded. When it does not exceed, an exception + should be raised. However, bit generators operate *without* the Python GIL + which means that they cannot directly raise. Instead, if an error is + detected when producing random values using RDRAND, the Python error flag + is set with a RuntimError. This error must then be checked for. In most + applications this happens automatically since the Lock attached to this + instance will check the error state when exiting and raise RuntimError. + + If you write custom code that uses lower-level function, e.g., the + PyCapsule, you will either need to check the status flag in the + state structure, or use PyErr_Occurred to see if an error occurred + during generation. + + To see the exception you will generatr, you can run this invalid code + + >>> from randomgen import RDRAND + >>> bitgen = RDRAND() + >>> state = bitgen.state + >>> state["retries"] = -1 # Ensure always fails + >>> bitgen.state = state + + The next command will always raise RuntimeError. + + >>> bitgen.random_raw() + + Note that ``random_raw`` has been customized for the needs to RDRAND + and does not rely on the Lock to raise. Instead it checks the status + directly and raises if the status is invalid. + **No Compatibility Guarantee** ``RDRAND`` is hardware dependent and not reproducible, and so there is no @@ -90,14 +208,36 @@ cdef class RDRAND(BitGenerator): >>> rg = Generator(RDRAND()) >>> rg.standard_normal() 0.123 # random + + References + ---------- + .. [1] Software.intel.com. 2020. Intel® Intrinsics Guide. [online] + Available at: + + [Accessed 10 July 2020]. + .. [2] Intel. 2020. Intel® Digital Random Number Generator (DRNG) Software Implementation. + [online] Available at: + + [Accessed 10 July 2020]. """ cdef rdrand_state rng_state - def __init__(self, seed=None): + def __init__(self, seed=None, *, int retries=10): + cdef int i + BitGenerator.__init__(self, seed, mode="sequence") + self.lock = RaisingLock() if not rdrand_capable(): raise RuntimeError("The RDRAND instruction is not available") # pragma: no cover self.rng_state.status = 1 + if retries < 0: + raise ValueError("retries must be a non-negative integer.") + self.rng_state.retries = retries + self.rng_state.weyl_seq = 0 + + self.rng_state.buffer_loc = BUFFER_SIZE + for i in range(BUFFER_SIZE): + self.rng_state.buffer[i] = libc.stdint.UINT64_MAX self.seed(seed) self._bitgen.state = &self.rng_state @@ -108,6 +248,37 @@ cdef class RDRAND(BitGenerator): def _seed_from_seq(self): pass + + @property + def success(self): + """ + Gets the flag indicating that all calls to RDRAND succeeded + + Returns + ------- + bool + True indicates success, false indicates failure + + Notes + ----- + Once status is set to 0, it remains 0 unless manually reset. + This happens to ensure that it is possible to manually verify + the status flag. + """ + return bool(self.rng_state.status) + + def _reset(self): + """ + Not part of the public API + + Resets RDRAND after a failure by setting status to 1 and + setting the buller_loc to BUFFER_SIZE so that a fresh set + of values is pulled. + """ + if self.rng_state.status == 0: + # Reset and ensure a new pull + self.rng_state.status = 1 + self.rng_state.buffer_loc = BUFFER_SIZE def seed(self, seed=None): """ @@ -126,6 +297,68 @@ cdef class RDRAND(BitGenerator): if seed is not None: raise TypeError("seed cannot be set and so must be None") + def random_raw(self, size=None, bint output=True): + """ + random_raw(size=None, output=True) + + Return randoms as generated by the underlying BitGenerator + + Parameters + ---------- + size : int or tuple of ints, optional + Output shape. If the given shape is, e.g., ``(m, n, k)``, then + ``m * n * k`` samples are drawn. Default is None, in which case a + single value is returned. + output : bool, optional + Output values. Used for performance testing since the generated + values are not returned. + + Returns + ------- + out : {uint64, ndarray, None} + Drawn samples. + + Raises + ------ + RuntimeError + Raised if the RDRAND instruction fails after retries. + """ + cdef np.ndarray randoms + cdef uint64_t *randoms_data + cdef uint64_t value + cdef Py_ssize_t i, n + cdef int status + + if not output: + n = 1 if size is None else size + status = self.rng_state.status + with self.lock, nogil: + for i in range(n): + status &= rdrand_next64(&self.rng_state, &value) + if status == 0: + raise RuntimeError(ERROR_MSG.format(retries=self.rng_state.retries)) + return + + if size is None: + with self.lock: + status = rdrand_next64(&self.rng_state, &value) + if status == 0: + raise RuntimeError(ERROR_MSG.format(retries=self.rng_state.retries)) + return value + + randoms = np.empty(size, np.uint64) + randoms_data = np.PyArray_DATA(randoms) + n = np.PyArray_SIZE(randoms) + + status = 1 + with self.lock, nogil: + for i in range(n): + status &= rdrand_next64(&self.rng_state, &randoms_data[i]) + if status == 0: + raise RuntimeError(ERROR_MSG.format(retries=self.rng_state.retries)) + + return randoms + def jumped(self, iter=1): """ jumped(iter=1) @@ -164,9 +397,25 @@ cdef class RDRAND(BitGenerator): state : dict Dictionary containing the information required to describe the state of the PRNG + + Notes + ----- + The values returned are the buffer that is used in the filling. This + is provided for testing and is never restored even when unpickling. """ + cdef uint64_t[::1] buffer + cdef int i + + buffer = np.empty(BUFFER_SIZE, dtype=np.uint64) + for i in range(BUFFER_SIZE): + buffer[i] = self.rng_state.buffer[i] + return {"bit_generator": type(self).__name__, - "status": self.rng_state.status} + "status": self.rng_state.status, + "retries": self.rng_state.retries, + "buffer_loc": self.rng_state.buffer_loc, + "buffer": np.asarray(buffer), + } @state.setter def state(self, value): @@ -176,3 +425,6 @@ cdef class RDRAND(BitGenerator): if bitgen != type(self).__name__: raise ValueError("state must be for a {0} " "PRNG".format(type(self).__name__)) + self.rng_state.retries = value["retries"] + +from threading import Lock diff --git a/randomgen/romu.pxd b/randomgen/romu.pxd index 820475015..77d3d557e 100644 --- a/randomgen/romu.pxd +++ b/randomgen/romu.pxd @@ -13,10 +13,10 @@ cdef extern from "src/romu/romu.h": ctypedef ROMU_STATE_T romu_state_t - uint64_t romuquad_next64(romu_state_t *state) nogil - uint32_t romuquad_next32(romu_state_t *state) nogil - uint64_t romutrio_next64(romu_state_t *state) nogil - uint32_t romutrio_next32(romu_state_t *state) nogil + uint64_t romuquad_next64(romu_state_t *state) nogil + uint32_t romuquad_next32(romu_state_t *state) nogil + uint64_t romutrio_next64(romu_state_t *state) nogil + uint32_t romutrio_next32(romu_state_t *state) nogil void romu_seed(romu_state_t *state, uint64_t w, uint64_t x, uint64_t y, uint64_t z, int quad) diff --git a/randomgen/seed_sequence.py b/randomgen/seed_sequence.py index b52cd3c2e..85675e966 100644 --- a/randomgen/seed_sequence.py +++ b/randomgen/seed_sequence.py @@ -1,24 +1,24 @@ try: from numpy.random._bit_generator import ( - SeedSequence, - SeedlessSeedSequence, ISeedSequence, ISpawnableSeedSequence, + SeedlessSeedSequence, + SeedSequence, ) except (ImportError, AttributeError): try: from numpy.random.bit_generator import ( - SeedSequence, - SeedlessSeedSequence, ISeedSequence, ISpawnableSeedSequence, + SeedlessSeedSequence, + SeedSequence, ) except (ImportError, AttributeError): from randomgen._seed_sequence import ( - SeedSequence, - SeedlessSeedSequence, ISeedSequence, ISpawnableSeedSequence, + SeedlessSeedSequence, + SeedSequence, ) __all__ = [ diff --git a/randomgen/sfc.pxd b/randomgen/sfc.pxd index 6e1188dde..fdd0f3462 100644 --- a/randomgen/sfc.pxd +++ b/randomgen/sfc.pxd @@ -14,8 +14,8 @@ cdef extern from "src/sfc/sfc.h": ctypedef SFC_STATE_T sfc_state_t - uint64_t sfc_next64(sfc_state_t *state) nogil - uint32_t sfc_next32(sfc_state_t *state) nogil + uint64_t sfc_next64(sfc_state_t *state) nogil + uint32_t sfc_next32(sfc_state_t *state) nogil void sfc_seed(sfc_state_t *state, uint64_t *seed, uint64_t w, uint64_t k) cdef class SFC64(BitGenerator): diff --git a/randomgen/sfmt.pxd b/randomgen/sfmt.pxd index a0215c584..6a7bb91ee 100644 --- a/randomgen/sfmt.pxd +++ b/randomgen/sfmt.pxd @@ -28,8 +28,8 @@ cdef extern from "src/sfmt/sfmt.h": ctypedef SFMT_STATE_T sfmt_state_t - uint64_t sfmt_next64(sfmt_state_t *state) nogil - uint32_t sfmt_next32(sfmt_state_t *state) nogil + uint64_t sfmt_next64(sfmt_state_t *state) nogil + uint32_t sfmt_next32(sfmt_state_t *state) nogil void sfmt_init_gen_rand(sfmt_t * sfmt, uint32_t seed) void sfmt_init_by_array(sfmt_t * sfmt, uint32_t *init_key, int key_length) diff --git a/randomgen/src/rdrand/rdrand.c b/randomgen/src/rdrand/rdrand.c index 0c82e1c97..9d9e3e2ed 100644 --- a/randomgen/src/rdrand/rdrand.c +++ b/randomgen/src/rdrand/rdrand.c @@ -3,11 +3,10 @@ #define RANDOMGEN_USE_RDRAND 30 -extern INLINE uint64_t rdrand_next64(rdrand_state* state); -extern INLINE uint32_t rdrand_next32(rdrand_state* state); +extern INLINE int rdrand_fill_buffer(rdrand_state *state); +extern INLINE int rdrand_next64(rdrand_state *state, uint64_t *val); -int rdrand_capable(void) -{ +int rdrand_capable(void) { #if defined(__RDRND__) && __RDRND__ int flags[32]; feature_flags(flags, RANDOMGEN_ECX); diff --git a/randomgen/src/rdrand/rdrand.h b/randomgen/src/rdrand/rdrand.h index 89fae6901..7d0567da8 100644 --- a/randomgen/src/rdrand/rdrand.h +++ b/randomgen/src/rdrand/rdrand.h @@ -4,39 +4,101 @@ #include "../common/randomgen_config.h" #include "../common/randomgen_immintrin.h" +#define BUFFER_SIZE 256 + typedef struct s_rdrand_state { + uint64_t buffer[BUFFER_SIZE]; + int buffer_loc; int status; + int retries; + uint64_t weyl_seq; } rdrand_state; +/* +next checks buffer, if available, fills +if not available, calls fill_buffer +fill_buffer will spin lock and pause +check status flag from fill buffer +if +*/ int rdrand_capable(void); - -static INLINE uint64_t rdrand_next64(rdrand_state* state){ +static INLINE int rdrand_fill_buffer(rdrand_state *state) { + /* + * You **must** check the returned value: + * 1 if success + * 0 if failure + */ + int status = 0, retries_cnt; + uint64_t val; + /* Assume success */ + for (int i = 0; i < BUFFER_SIZE; i++) { + status = 0; + retries_cnt = 0; + while ((status == 0) && (retries_cnt++ <= state->retries)) { #if defined(__RDRND__) && __RDRND__ - uint64_t val; #if defined(__x86_64__) || defined(_M_X64) - state->status &= _rdrand64_step((long long unsigned int *)&val); + status = _rdrand64_step((long long unsigned int *)&val); #else - uint32_t low, high; - state->status &= _rdrand32_step(&low); - state->status &= _rdrand32_step(&high); - val = ((uint64_t)high)<< 32 | low; + uint32_t low, high; + status = _rdrand32_step(&low); + status &= _rdrand32_step(&high); + val = ((uint64_t)high) << 32 | low; #endif - return val; #else - return UINT64_MAX; + /* Never called on platforms without RDRAND */ + return 0; #endif + if (status != 0) { + state->buffer[i] = val; + } else { +#if defined(__RDRND__) && __RDRND__ + _mm_pause(); +#endif + } + } + state->status &= status; + if (status == 0) { + /* This is dont to attempt to let rejection samplers exit */ + state->buffer[i] = state->weyl_seq += 11400714819323198485ULL; + } + } + /* Reset only on success */ + state->buffer_loc = 0; + return state->status; } -static INLINE uint32_t rdrand_next32(rdrand_state* state){ +static INLINE int rdrand_next64(rdrand_state *state, uint64_t *val) { + /* + * You **must** check the returned status + * 1 if success + * 0 if failure + */ + int status = 0, retries_cnt = 0; + while ((status == 0) && (retries_cnt++ <= state->retries)) { #if defined(__RDRND__) && __RDRND__ - uint32_t val; - state->status &= _rdrand32_step(&val); - return val; +#if defined(__x86_64__) || defined(_M_X64) + status = _rdrand64_step((long long unsigned int *)val); #else - return UINT32_MAX; + uint32_t low, high; + status = _rdrand32_step(&low); + status &= _rdrand32_step(&high); + val[0] = ((uint64_t)high) << 32 | low; +#endif +#else + /* Never called on platforms without RDRAND */ + state->status = 0; + return 0; +#endif + if (status == 0) { +#if defined(__RDRND__) && __RDRND__ + _mm_pause(); #endif + } + } + state->status &= status; + return state->status; } #endif /* _RANDOMDGEN__RDRAND_H_ */ \ No newline at end of file diff --git a/randomgen/tests/test_direct.py b/randomgen/tests/test_direct.py index f0d62d076..66a78c76b 100644 --- a/randomgen/tests/test_direct.py +++ b/randomgen/tests/test_direct.py @@ -1368,6 +1368,67 @@ def setup_class(cls): def setup_bitgenerator(self, seed, mode=None): return self.bit_generator(*seed) + def test_initialization(self): + bit_generator = self.bit_generator() + state = bit_generator.state + assert state["buffer_loc"] == 256 + assert state["buffer"].shape == (256,) + assert (state["buffer"] == np.iinfo(np.uint64).max).all() + assert state["retries"] == 10 + assert state["status"] == 1 + gen = Generator(bit_generator) + gen.integers(0, 2 ** 64, dtype=np.uint64, size=10) + state = bit_generator.state + # Incredibly conservative test + assert (state["buffer"] == np.iinfo(np.uint64).max).sum() < 10 + assert state["status"] == 1 + + bit_generator = RDRAND(retries=1000) + state = bit_generator.state + assert state["retries"] == 1000 + + def test_generator_raises(self): + bit_generator = self.bit_generator() + + state = bit_generator.state + state["retries"] = -1 + bit_generator.state = state + gen = Generator(bit_generator) + with pytest.raises(RuntimeError): + gen.integers(0, 2 ** 64, dtype=np.uint64, size=10) + assert bit_generator.state["status"] == 0 + bit_generator._reset() + state = bit_generator.state + assert state["status"] == 1 + assert state["buffer_loc"] == 256 + assert bit_generator.success is True + + def test_exception(self): + with pytest.raises(ValueError): + RDRAND(retries=-1) + + def test_runtimerror(self): + bit_generator = RDRAND() + state = bit_generator.state + state["retries"] = -1 + bit_generator.state = state + with pytest.raises(RuntimeError): + bit_generator.random_raw() + with pytest.raises(RuntimeError): + bit_generator.random_raw(10) + with pytest.raises(RuntimeError): + bit_generator.random_raw(output=False) + with pytest.raises(RuntimeError): + bit_generator.random_raw(10, output=False) + + def test_checked_raw(self): + bit_generator = RDRAND() + assert isinstance(bit_generator.random_raw(), int) + + def test_checked_raw_vector(self): + bit_generator = RDRAND() + assert isinstance(bit_generator.random_raw(10), np.ndarray) + def test_raw(self): bit_generator = self.bit_generator() raw = bit_generator.random_raw(1000) diff --git a/randomgen/tests/test_generator_117.py b/randomgen/tests/test_generator_117.py index d7e5e4660..4c561e6c4 100644 --- a/randomgen/tests/test_generator_117.py +++ b/randomgen/tests/test_generator_117.py @@ -8,8 +8,7 @@ from randomgen import Generator try: - from numpy.random import Generator as NPGenerator - from numpy.random import PCG64 + from numpy.random import PCG64, Generator as NPGenerator pcg = PCG64() initial_state = pcg.state diff --git a/randomgen/tests/test_lcg128mix_pcg64dxsm.py b/randomgen/tests/test_lcg128mix_pcg64dxsm.py index 0beeddc06..f3b6bd3a1 100644 --- a/randomgen/tests/test_lcg128mix_pcg64dxsm.py +++ b/randomgen/tests/test_lcg128mix_pcg64dxsm.py @@ -5,7 +5,7 @@ from randomgen.pcg64 import DEFAULT_DXSM_MULTIPLIER, DEFAULT_MULTIPLIER try: - from numba import types, cfunc + from numba import cfunc, types MISSING_NUMBA = False except ImportError: @@ -113,8 +113,8 @@ def lower(high, low): def test_ctypes(): import ctypes - import subprocess import os + import subprocess base = os.path.split(os.path.abspath(__file__))[0] diff --git a/randomgen/tests/test_randomstate_regression.py b/randomgen/tests/test_randomstate_regression.py index 8cf9d43db..84f86de19 100644 --- a/randomgen/tests/test_randomstate_regression.py +++ b/randomgen/tests/test_randomstate_regression.py @@ -5,7 +5,7 @@ from numpy.testing import assert_, assert_array_equal, assert_raises import pytest -import randomgen.mtrand as random +from randomgen import mtrand as random HAS_32BIT_CLONG = np.iinfo("l").max < 2 ** 32 diff --git a/randomgen/tests/test_seed_sequence.py b/randomgen/tests/test_seed_sequence.py index a05d4eb21..58447f20a 100644 --- a/randomgen/tests/test_seed_sequence.py +++ b/randomgen/tests/test_seed_sequence.py @@ -1,5 +1,5 @@ import numpy as np -from numpy.testing import assert_array_equal, assert_array_compare +from numpy.testing import assert_array_compare, assert_array_equal import pytest from randomgen._seed_sequence import SeedlessSeedSequence, SeedSequence diff --git a/randomgen/tests/test_stability.py b/randomgen/tests/test_stability.py index 5652d672f..aa99e7447 100644 --- a/randomgen/tests/test_stability.py +++ b/randomgen/tests/test_stability.py @@ -28,7 +28,7 @@ def configuration(request): key = request.param if key in EXECUTED: - return EXECUTED[key] + return key, EXECUTED[key] EXECUTED[key] = hash_configuration(final_configurations[key]) return key, EXECUTED[key] diff --git a/randomgen/tests/test_wrapper_numba.py b/randomgen/tests/test_wrapper_numba.py index a525d1bfa..040af5492 100644 --- a/randomgen/tests/test_wrapper_numba.py +++ b/randomgen/tests/test_wrapper_numba.py @@ -8,7 +8,7 @@ HAS_NUMBA = False try: - from numba import cfunc, types, carray, jit + from numba import carray, cfunc, jit, types HAS_NUMBA = True except ImportError: diff --git a/randomgen/xoroshiro128.pxd b/randomgen/xoroshiro128.pxd index 276253019..4c62946f3 100644 --- a/randomgen/xoroshiro128.pxd +++ b/randomgen/xoroshiro128.pxd @@ -10,12 +10,12 @@ cdef extern from "src/xoroshiro128/xoroshiro128.h": ctypedef XOROSHIRO128_STATE_T xoroshiro128_state_t - uint64_t xoroshiro128_next64(xoroshiro128_state_t *state) nogil - uint32_t xoroshiro128_next32(xoroshiro128_state_t *state) nogil + uint64_t xoroshiro128_next64(xoroshiro128_state_t *state) nogil + uint32_t xoroshiro128_next32(xoroshiro128_state_t *state) nogil void xoroshiro128_jump(xoroshiro128_state_t *state) - uint64_t xoroshiro128plusplus_next64(xoroshiro128_state_t *state) nogil - uint32_t xoroshiro128plusplus_next32(xoroshiro128_state_t *state) nogil + uint64_t xoroshiro128plusplus_next64(xoroshiro128_state_t *state) nogil + uint32_t xoroshiro128plusplus_next32(xoroshiro128_state_t *state) nogil void xoroshiro128plusplus_jump(xoroshiro128_state_t *state) diff --git a/randomgen/xorshift1024.pxd b/randomgen/xorshift1024.pxd index 389ac594b..c7a059906 100644 --- a/randomgen/xorshift1024.pxd +++ b/randomgen/xorshift1024.pxd @@ -11,8 +11,8 @@ cdef extern from "src/xorshift1024/xorshift1024.h": ctypedef XORSHIFT1024_STATE_T xorshift1024_state_t - uint64_t xorshift1024_next64(xorshift1024_state_t *state) nogil - uint32_t xorshift1024_next32(xorshift1024_state_t *state) nogil + uint64_t xorshift1024_next64(xorshift1024_state_t *state) nogil + uint32_t xorshift1024_next32(xorshift1024_state_t *state) nogil void xorshift1024_jump(xorshift1024_state_t *state) diff --git a/randomgen/xoshiro256.pxd b/randomgen/xoshiro256.pxd index 589b91075..de99043c6 100644 --- a/randomgen/xoshiro256.pxd +++ b/randomgen/xoshiro256.pxd @@ -10,8 +10,8 @@ cdef extern from "src/xoshiro256/xoshiro256.h": ctypedef XOSHIRO256_STATE_T xoshiro256_state_t - uint64_t xoshiro256_next64(xoshiro256_state_t *state) nogil - uint32_t xoshiro256_next32(xoshiro256_state_t *state) nogil + uint64_t xoshiro256_next64(xoshiro256_state_t *state) nogil + uint32_t xoshiro256_next32(xoshiro256_state_t *state) nogil void xoshiro256_jump(xoshiro256_state_t *state) diff --git a/randomgen/xoshiro512.pxd b/randomgen/xoshiro512.pxd index c7a79fa7f..01065f9b4 100644 --- a/randomgen/xoshiro512.pxd +++ b/randomgen/xoshiro512.pxd @@ -10,8 +10,8 @@ cdef extern from "src/xoshiro512/xoshiro512.h": ctypedef XOSHIRO512_STATE_T xoshiro512_state_t - uint64_t xoshiro512_next64(xoshiro512_state_t *state) nogil - uint32_t xoshiro512_next32(xoshiro512_state_t *state) nogil + uint64_t xoshiro512_next64(xoshiro512_state_t *state) nogil + uint32_t xoshiro512_next32(xoshiro512_state_t *state) nogil void xoshiro512_jump(xoshiro512_state_t *state) diff --git a/setup.py b/setup.py index 4e1af354e..bc9283855 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ import versioneer try: - import Cython.Tempita as tempita + from Cython import Tempita as tempita except ImportError: try: import tempita diff --git a/tools/prng-tester.py b/tools/prng-tester.py index 8433541ff..c39aae5ae 100644 --- a/tools/prng-tester.py +++ b/tools/prng-tester.py @@ -19,9 +19,10 @@ TEMPLATE, ) from joblib import Parallel, cpu_count, delayed -from randomgen import DSFMT, SFC64 from shared import get_logger, test_single +from randomgen import DSFMT, SFC64 + DEFAULT_STREAMS = (4, 8196) diff --git a/tools/test-seed-correlation.py b/tools/test-seed-correlation.py index 6a8153693..2b3ee2104 100644 --- a/tools/test-seed-correlation.py +++ b/tools/test-seed-correlation.py @@ -10,13 +10,13 @@ from multiprocessing import Manager import os -import jinja2 - from configuration import ALL_BIT_GENS, DSFMT_WRAPPER, OUTPUT, SPECIALS +import jinja2 from joblib import Parallel, cpu_count, delayed -from randomgen import DSFMT from shared import get_logger, test_single +from randomgen import DSFMT + with open("templates/seed-correlation.jinja") as tmpl: TEMPLATE = jinja2.Template(tmpl.read())