xxhash.h (280387B)
1 /* 2 * xxHash - Extremely Fast Hash algorithm 3 * Header File 4 * Copyright (C) 2012-2023 Yann Collet 5 * 6 * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions are 10 * met: 11 * 12 * * Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * * Redistributions in binary form must reproduce the above 15 * copyright notice, this list of conditions and the following disclaimer 16 * in the documentation and/or other materials provided with the 17 * distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 * You can contact the author at: 32 * - xxHash homepage: https://www.xxhash.com 33 * - xxHash source repository: https://github.com/Cyan4973/xxHash 34 */ 35 36 /*! 37 * @mainpage xxHash 38 * 39 * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed 40 * limits. 41 * 42 * It is proposed in four flavors, in three families: 43 * 1. @ref XXH32_family 44 * - Classic 32-bit hash function. Simple, compact, and runs on almost all 45 * 32-bit and 64-bit systems. 46 * 2. @ref XXH64_family 47 * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most 48 * 64-bit systems (but _not_ 32-bit systems). 49 * 3. @ref XXH3_family 50 * - Modern 64-bit and 128-bit hash function family which features improved 51 * strength and performance across the board, especially on smaller data. 52 * It benefits greatly from SIMD and 64-bit without requiring it. 53 * 54 * Benchmarks 55 * --- 56 * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04. 57 * The open source benchmark program is compiled with clang v10.0 using -O3 flag. 58 * 59 * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity | 60 * | -------------------- | ------- | ----: | ---------------: | ------------------: | 61 * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 | 62 * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 | 63 * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 | 64 * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 | 65 * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 | 66 * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 | 67 * | RAM sequential read | | N/A | 28.0 GB/s | N/A | 68 * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 | 69 * | City64 | | 64 | 22.0 GB/s | 76.6 | 70 * | T1ha2 | | 64 | 22.0 GB/s | 99.0 | 71 * | City128 | | 128 | 21.7 GB/s | 57.7 | 72 * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 | 73 * | XXH64() | | 64 | 19.4 GB/s | 71.0 | 74 * | SpookyHash | | 64 | 19.3 GB/s | 53.2 | 75 * | Mum | | 64 | 18.0 GB/s | 67.0 | 76 * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 | 77 * | XXH32() | | 32 | 9.7 GB/s | 71.9 | 78 * | City32 | | 32 | 9.1 GB/s | 66.0 | 79 * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 | 80 * | Murmur3 | | 32 | 3.9 GB/s | 56.1 | 81 * | SipHash* | | 64 | 3.0 GB/s | 43.2 | 82 * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 | 83 * | HighwayHash | | 64 | 1.4 GB/s | 6.0 | 84 * | FNV64 | | 64 | 1.2 GB/s | 62.7 | 85 * | Blake2* | | 256 | 1.1 GB/s | 5.1 | 86 * | SHA1* | | 160 | 0.8 GB/s | 5.6 | 87 * | MD5* | | 128 | 0.6 GB/s | 7.8 | 88 * @note 89 * - Hashes which require a specific ISA extension are noted. SSE2 is also noted, 90 * even though it is mandatory on x64. 91 * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic 92 * by modern standards. 93 * - Small data velocity is a rough average of algorithm's efficiency for small 94 * data. For more accurate information, see the wiki. 95 * - More benchmarks and strength tests are found on the wiki: 96 * https://github.com/Cyan4973/xxHash/wiki 97 * 98 * Usage 99 * ------ 100 * All xxHash variants use a similar API. Changing the algorithm is a trivial 101 * substitution. 102 * 103 * @pre 104 * For functions which take an input and length parameter, the following 105 * requirements are assumed: 106 * - The range from [`input`, `input + length`) is valid, readable memory. 107 * - The only exception is if the `length` is `0`, `input` may be `NULL`. 108 * - For C++, the objects must have the *TriviallyCopyable* property, as the 109 * functions access bytes directly as if it was an array of `unsigned char`. 110 * 111 * @anchor single_shot_example 112 * **Single Shot** 113 * 114 * These functions are stateless functions which hash a contiguous block of memory, 115 * immediately returning the result. They are the easiest and usually the fastest 116 * option. 117 * 118 * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits() 119 * 120 * @code{.c} 121 * #include <string.h> 122 * #include "xxhash.h" 123 * 124 * // Example for a function which hashes a null terminated string with XXH32(). 125 * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed) 126 * { 127 * // NULL pointers are only valid if the length is zero 128 * size_t length = (string == NULL) ? 0 : strlen(string); 129 * return XXH32(string, length, seed); 130 * } 131 * @endcode 132 * 133 * 134 * @anchor streaming_example 135 * **Streaming** 136 * 137 * These groups of functions allow incremental hashing of unknown size, even 138 * more than what would fit in a size_t. 139 * 140 * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset() 141 * 142 * @code{.c} 143 * #include <stdio.h> 144 * #include <assert.h> 145 * #include "xxhash.h" 146 * // Example for a function which hashes a FILE incrementally with XXH3_64bits(). 147 * XXH64_hash_t hashFile(FILE* f) 148 * { 149 * // Allocate a state struct. Do not just use malloc() or new. 150 * XXH3_state_t* state = XXH3_createState(); 151 * assert(state != NULL && "Out of memory!"); 152 * // Reset the state to start a new hashing session. 153 * XXH3_64bits_reset(state); 154 * char buffer[4096]; 155 * size_t count; 156 * // Read the file in chunks 157 * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) { 158 * // Run update() as many times as necessary to process the data 159 * XXH3_64bits_update(state, buffer, count); 160 * } 161 * // Retrieve the finalized hash. This will not change the state. 162 * XXH64_hash_t result = XXH3_64bits_digest(state); 163 * // Free the state. Do not use free(). 164 * XXH3_freeState(state); 165 * return result; 166 * } 167 * @endcode 168 * 169 * Streaming functions generate the xxHash value from an incremental input. 170 * This method is slower than single-call functions, due to state management. 171 * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. 172 * 173 * An XXH state must first be allocated using `XXH*_createState()`. 174 * 175 * Start a new hash by initializing the state with a seed using `XXH*_reset()`. 176 * 177 * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. 178 * 179 * The function returns an error code, with 0 meaning OK, and any other value 180 * meaning there is an error. 181 * 182 * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. 183 * This function returns the nn-bits hash as an int or long long. 184 * 185 * It's still possible to continue inserting input into the hash state after a 186 * digest, and generate new hash values later on by invoking `XXH*_digest()`. 187 * 188 * When done, release the state using `XXH*_freeState()`. 189 * 190 * 191 * @anchor canonical_representation_example 192 * **Canonical Representation** 193 * 194 * The default return values from XXH functions are unsigned 32, 64 and 128 bit 195 * integers. 196 * This the simplest and fastest format for further post-processing. 197 * 198 * However, this leaves open the question of what is the order on the byte level, 199 * since little and big endian conventions will store the same number differently. 200 * 201 * The canonical representation settles this issue by mandating big-endian 202 * convention, the same convention as human-readable numbers (large digits first). 203 * 204 * When writing hash values to storage, sending them over a network, or printing 205 * them, it's highly recommended to use the canonical representation to ensure 206 * portability across a wider range of systems, present and future. 207 * 208 * The following functions allow transformation of hash values to and from 209 * canonical format. 210 * 211 * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(), 212 * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(), 213 * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(), 214 * 215 * @code{.c} 216 * #include <stdio.h> 217 * #include "xxhash.h" 218 * 219 * // Example for a function which prints XXH32_hash_t in human readable format 220 * void printXxh32(XXH32_hash_t hash) 221 * { 222 * XXH32_canonical_t cano; 223 * XXH32_canonicalFromHash(&cano, hash); 224 * size_t i; 225 * for(i = 0; i < sizeof(cano.digest); ++i) { 226 * printf("%02x", cano.digest[i]); 227 * } 228 * printf("\n"); 229 * } 230 * 231 * // Example for a function which converts XXH32_canonical_t to XXH32_hash_t 232 * XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano) 233 * { 234 * XXH32_hash_t hash = XXH32_hashFromCanonical(&cano); 235 * return hash; 236 * } 237 * @endcode 238 * 239 * 240 * @file xxhash.h 241 * xxHash prototypes and implementation 242 */ 243 244 #if defined(__cplusplus) && !defined(XXH_NO_EXTERNC_GUARD) 245 extern "C" { 246 #endif 247 248 /* **************************** 249 * INLINE mode 250 ******************************/ 251 /*! 252 * @defgroup public Public API 253 * Contains details on the public xxHash functions. 254 * @{ 255 */ 256 #ifdef XXH_DOXYGEN 257 /*! 258 * @brief Gives access to internal state declaration, required for static allocation. 259 * 260 * Incompatible with dynamic linking, due to risks of ABI changes. 261 * 262 * Usage: 263 * @code{.c} 264 * #define XXH_STATIC_LINKING_ONLY 265 * #include "xxhash.h" 266 * @endcode 267 */ 268 # define XXH_STATIC_LINKING_ONLY 269 /* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */ 270 271 /*! 272 * @brief Gives access to internal definitions. 273 * 274 * Usage: 275 * @code{.c} 276 * #define XXH_STATIC_LINKING_ONLY 277 * #define XXH_IMPLEMENTATION 278 * #include "xxhash.h" 279 * @endcode 280 */ 281 # define XXH_IMPLEMENTATION 282 /* Do not undef XXH_IMPLEMENTATION for Doxygen */ 283 284 /*! 285 * @brief Exposes the implementation and marks all functions as `inline`. 286 * 287 * Use these build macros to inline xxhash into the target unit. 288 * Inlining improves performance on small inputs, especially when the length is 289 * expressed as a compile-time constant: 290 * 291 * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html 292 * 293 * It also keeps xxHash symbols private to the unit, so they are not exported. 294 * 295 * Usage: 296 * @code{.c} 297 * #define XXH_INLINE_ALL 298 * #include "xxhash.h" 299 * @endcode 300 * Do not compile and link xxhash.o as a separate object, as it is not useful. 301 */ 302 # define XXH_INLINE_ALL 303 # undef XXH_INLINE_ALL 304 /*! 305 * @brief Exposes the implementation without marking functions as inline. 306 */ 307 # define XXH_PRIVATE_API 308 # undef XXH_PRIVATE_API 309 /*! 310 * @brief Emulate a namespace by transparently prefixing all symbols. 311 * 312 * If you want to include _and expose_ xxHash functions from within your own 313 * library, but also want to avoid symbol collisions with other libraries which 314 * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix 315 * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE 316 * (therefore, avoid empty or numeric values). 317 * 318 * Note that no change is required within the calling program as long as it 319 * includes `xxhash.h`: Regular symbol names will be automatically translated 320 * by this header. 321 */ 322 # define XXH_NAMESPACE /* YOUR NAME HERE */ 323 # undef XXH_NAMESPACE 324 #endif 325 326 #define XXH_CAT(A,B) A##B 327 #define XXH_NAME2(A,B) XXH_CAT(A,B) 328 #define XXH_IPREF(Id) XXH_NAME2(XXH_NAMESPACE, Id) 329 330 #if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ 331 && !defined(XXH_INLINE_ALL_31684351384) 332 /* this section should be traversed only once */ 333 # define XXH_INLINE_ALL_31684351384 334 /* give access to the advanced API, required to compile implementations */ 335 # undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ 336 # define XXH_STATIC_LINKING_ONLY 337 /* make all functions private */ 338 # undef XXH_PUBLIC_API 339 # if defined(__GNUC__) 340 # define XXH_PUBLIC_API static __inline __attribute__((__unused__)) 341 # elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 342 # define XXH_PUBLIC_API static inline 343 # elif defined(_MSC_VER) 344 # define XXH_PUBLIC_API static __inline 345 # else 346 /* note: this version may generate warnings for unused static functions */ 347 # define XXH_PUBLIC_API static 348 # endif 349 350 /* 351 * This part deals with the special case where a unit wants to inline xxHash, 352 * but "xxhash.h" has previously been included without XXH_INLINE_ALL, 353 * such as part of some previously included *.h header file. 354 * Without further action, the new include would just be ignored, 355 * and functions would effectively _not_ be inlined (silent failure). 356 * The following macros solve this situation by prefixing all inlined names, 357 * avoiding naming collision with previous inclusions. 358 */ 359 /* Before that, we unconditionally #undef all symbols, 360 * in case they were already defined with XXH_NAMESPACE. 361 * They will then be redefined for XXH_INLINE_ALL 362 */ 363 # undef XXH_versionNumber 364 /* XXH32 */ 365 # undef XXH32 366 # undef XXH32_createState 367 # undef XXH32_freeState 368 # undef XXH32_reset 369 # undef XXH32_update 370 # undef XXH32_digest 371 # undef XXH32_copyState 372 # undef XXH32_canonicalFromHash 373 # undef XXH32_hashFromCanonical 374 /* XXH64 */ 375 # undef XXH64 376 # undef XXH64_createState 377 # undef XXH64_freeState 378 # undef XXH64_reset 379 # undef XXH64_update 380 # undef XXH64_digest 381 # undef XXH64_copyState 382 # undef XXH64_canonicalFromHash 383 # undef XXH64_hashFromCanonical 384 /* XXH3_64bits */ 385 # undef XXH3_64bits 386 # undef XXH3_64bits_withSecret 387 # undef XXH3_64bits_withSeed 388 # undef XXH3_64bits_withSecretandSeed 389 # undef XXH3_createState 390 # undef XXH3_freeState 391 # undef XXH3_copyState 392 # undef XXH3_64bits_reset 393 # undef XXH3_64bits_reset_withSeed 394 # undef XXH3_64bits_reset_withSecret 395 # undef XXH3_64bits_update 396 # undef XXH3_64bits_digest 397 # undef XXH3_generateSecret 398 /* XXH3_128bits */ 399 # undef XXH128 400 # undef XXH3_128bits 401 # undef XXH3_128bits_withSeed 402 # undef XXH3_128bits_withSecret 403 # undef XXH3_128bits_reset 404 # undef XXH3_128bits_reset_withSeed 405 # undef XXH3_128bits_reset_withSecret 406 # undef XXH3_128bits_reset_withSecretandSeed 407 # undef XXH3_128bits_update 408 # undef XXH3_128bits_digest 409 # undef XXH128_isEqual 410 # undef XXH128_cmp 411 # undef XXH128_canonicalFromHash 412 # undef XXH128_hashFromCanonical 413 /* Finally, free the namespace itself */ 414 # undef XXH_NAMESPACE 415 416 /* employ the namespace for XXH_INLINE_ALL */ 417 # define XXH_NAMESPACE XXH_INLINE_ 418 /* 419 * Some identifiers (enums, type names) are not symbols, 420 * but they must nonetheless be renamed to avoid redeclaration. 421 * Alternative solution: do not redeclare them. 422 * However, this requires some #ifdefs, and has a more dispersed impact. 423 * Meanwhile, renaming can be achieved in a single place. 424 */ 425 # define XXH_OK XXH_IPREF(XXH_OK) 426 # define XXH_ERROR XXH_IPREF(XXH_ERROR) 427 # define XXH_errorcode XXH_IPREF(XXH_errorcode) 428 # define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) 429 # define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) 430 # define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) 431 # define XXH32_state_s XXH_IPREF(XXH32_state_s) 432 # define XXH32_state_t XXH_IPREF(XXH32_state_t) 433 # define XXH64_state_s XXH_IPREF(XXH64_state_s) 434 # define XXH64_state_t XXH_IPREF(XXH64_state_t) 435 # define XXH3_state_s XXH_IPREF(XXH3_state_s) 436 # define XXH3_state_t XXH_IPREF(XXH3_state_t) 437 # define XXH128_hash_t XXH_IPREF(XXH128_hash_t) 438 /* Ensure the header is parsed again, even if it was previously included */ 439 # undef XXHASH_H_5627135585666179 440 # undef XXHASH_H_STATIC_13879238742 441 #endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ 442 443 /* **************************************************************** 444 * Stable API 445 *****************************************************************/ 446 #ifndef XXHASH_H_5627135585666179 447 #define XXHASH_H_5627135585666179 1 448 449 /*! @brief Marks a global symbol. */ 450 #if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) 451 # if defined(_WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) 452 # ifdef XXH_EXPORT 453 # define XXH_PUBLIC_API __declspec(dllexport) 454 # elif XXH_IMPORT 455 # define XXH_PUBLIC_API __declspec(dllimport) 456 # endif 457 # else 458 # define XXH_PUBLIC_API /* do nothing */ 459 # endif 460 #endif 461 462 #ifdef XXH_NAMESPACE 463 # define XXH_versionNumber XXH_IPREF(XXH_versionNumber) 464 /* XXH32 */ 465 # define XXH32 XXH_IPREF(XXH32) 466 # define XXH32_createState XXH_IPREF(XXH32_createState) 467 # define XXH32_freeState XXH_IPREF(XXH32_freeState) 468 # define XXH32_reset XXH_IPREF(XXH32_reset) 469 # define XXH32_update XXH_IPREF(XXH32_update) 470 # define XXH32_digest XXH_IPREF(XXH32_digest) 471 # define XXH32_copyState XXH_IPREF(XXH32_copyState) 472 # define XXH32_canonicalFromHash XXH_IPREF(XXH32_canonicalFromHash) 473 # define XXH32_hashFromCanonical XXH_IPREF(XXH32_hashFromCanonical) 474 /* XXH64 */ 475 # define XXH64 XXH_IPREF(XXH64) 476 # define XXH64_createState XXH_IPREF(XXH64_createState) 477 # define XXH64_freeState XXH_IPREF(XXH64_freeState) 478 # define XXH64_reset XXH_IPREF(XXH64_reset) 479 # define XXH64_update XXH_IPREF(XXH64_update) 480 # define XXH64_digest XXH_IPREF(XXH64_digest) 481 # define XXH64_copyState XXH_IPREF(XXH64_copyState) 482 # define XXH64_canonicalFromHash XXH_IPREF(XXH64_canonicalFromHash) 483 # define XXH64_hashFromCanonical XXH_IPREF(XXH64_hashFromCanonical) 484 /* XXH3_64bits */ 485 # define XXH3_64bits XXH_IPREF(XXH3_64bits) 486 # define XXH3_64bits_withSecret XXH_IPREF(XXH3_64bits_withSecret) 487 # define XXH3_64bits_withSeed XXH_IPREF(XXH3_64bits_withSeed) 488 # define XXH3_64bits_withSecretandSeed XXH_IPREF(XXH3_64bits_withSecretandSeed) 489 # define XXH3_createState XXH_IPREF(XXH3_createState) 490 # define XXH3_freeState XXH_IPREF(XXH3_freeState) 491 # define XXH3_copyState XXH_IPREF(XXH3_copyState) 492 # define XXH3_64bits_reset XXH_IPREF(XXH3_64bits_reset) 493 # define XXH3_64bits_reset_withSeed XXH_IPREF(XXH3_64bits_reset_withSeed) 494 # define XXH3_64bits_reset_withSecret XXH_IPREF(XXH3_64bits_reset_withSecret) 495 # define XXH3_64bits_reset_withSecretandSeed XXH_IPREF(XXH3_64bits_reset_withSecretandSeed) 496 # define XXH3_64bits_update XXH_IPREF(XXH3_64bits_update) 497 # define XXH3_64bits_digest XXH_IPREF(XXH3_64bits_digest) 498 # define XXH3_generateSecret XXH_IPREF(XXH3_generateSecret) 499 # define XXH3_generateSecret_fromSeed XXH_IPREF(XXH3_generateSecret_fromSeed) 500 /* XXH3_128bits */ 501 # define XXH128 XXH_IPREF(XXH128) 502 # define XXH3_128bits XXH_IPREF(XXH3_128bits) 503 # define XXH3_128bits_withSeed XXH_IPREF(XXH3_128bits_withSeed) 504 # define XXH3_128bits_withSecret XXH_IPREF(XXH3_128bits_withSecret) 505 # define XXH3_128bits_withSecretandSeed XXH_IPREF(XXH3_128bits_withSecretandSeed) 506 # define XXH3_128bits_reset XXH_IPREF(XXH3_128bits_reset) 507 # define XXH3_128bits_reset_withSeed XXH_IPREF(XXH3_128bits_reset_withSeed) 508 # define XXH3_128bits_reset_withSecret XXH_IPREF(XXH3_128bits_reset_withSecret) 509 # define XXH3_128bits_reset_withSecretandSeed XXH_IPREF(XXH3_128bits_reset_withSecretandSeed) 510 # define XXH3_128bits_update XXH_IPREF(XXH3_128bits_update) 511 # define XXH3_128bits_digest XXH_IPREF(XXH3_128bits_digest) 512 # define XXH128_isEqual XXH_IPREF(XXH128_isEqual) 513 # define XXH128_cmp XXH_IPREF(XXH128_cmp) 514 # define XXH128_canonicalFromHash XXH_IPREF(XXH128_canonicalFromHash) 515 # define XXH128_hashFromCanonical XXH_IPREF(XXH128_hashFromCanonical) 516 #endif 517 518 519 /* ************************************* 520 * Compiler specifics 521 ***************************************/ 522 523 /* specific declaration modes for Windows */ 524 #if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) 525 # if defined(_WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) 526 # ifdef XXH_EXPORT 527 # define XXH_PUBLIC_API __declspec(dllexport) 528 # elif XXH_IMPORT 529 # define XXH_PUBLIC_API __declspec(dllimport) 530 # endif 531 # else 532 # define XXH_PUBLIC_API /* do nothing */ 533 # endif 534 #endif 535 536 #if defined (__GNUC__) 537 # define XXH_CONSTF __attribute__((__const__)) 538 # define XXH_PUREF __attribute__((__pure__)) 539 # define XXH_MALLOCF __attribute__((__malloc__)) 540 #else 541 # define XXH_CONSTF /* disable */ 542 # define XXH_PUREF 543 # define XXH_MALLOCF 544 #endif 545 546 /* ************************************* 547 * Version 548 ***************************************/ 549 #define XXH_VERSION_MAJOR 0 550 #define XXH_VERSION_MINOR 8 551 #define XXH_VERSION_RELEASE 4 552 /*! @brief Version number, encoded as two digits each */ 553 #define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) 554 555 /*! 556 * @brief Obtains the xxHash version. 557 * 558 * This is mostly useful when xxHash is compiled as a shared library, 559 * since the returned value comes from the library, as opposed to header file. 560 * 561 * @return @ref XXH_VERSION_NUMBER of the invoked library. 562 */ 563 XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void); 564 565 566 /* **************************** 567 * Common basic types 568 ******************************/ 569 #include <stddef.h> /* size_t */ 570 /*! 571 * @brief Exit code for the streaming API. 572 */ 573 typedef enum { 574 XXH_OK = 0, /*!< OK */ 575 XXH_ERROR /*!< Error */ 576 } XXH_errorcode; 577 578 579 /*-********************************************************************** 580 * 32-bit hash 581 ************************************************************************/ 582 #if defined(XXH_DOXYGEN) /* Don't show <stdint.h> include */ 583 /*! 584 * @brief An unsigned 32-bit integer. 585 * 586 * Not necessarily defined to `uint32_t` but functionally equivalent. 587 */ 588 typedef uint32_t XXH32_hash_t; 589 590 #elif !defined (__VMS) \ 591 && (defined (__cplusplus) \ 592 || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) 593 # ifdef _AIX 594 # include <inttypes.h> 595 # else 596 # include <stdint.h> 597 # endif 598 typedef uint32_t XXH32_hash_t; 599 600 #else 601 # include <limits.h> 602 # if UINT_MAX == 0xFFFFFFFFUL 603 typedef unsigned int XXH32_hash_t; 604 # elif ULONG_MAX == 0xFFFFFFFFUL 605 typedef unsigned long XXH32_hash_t; 606 # else 607 # error "unsupported platform: need a 32-bit type" 608 # endif 609 #endif 610 611 /*! 612 * @} 613 * 614 * @defgroup XXH32_family XXH32 family 615 * @ingroup public 616 * Contains functions used in the classic 32-bit xxHash algorithm. 617 * 618 * @note 619 * XXH32 is useful for older platforms, with no or poor 64-bit performance. 620 * Note that the @ref XXH3_family provides competitive speed for both 32-bit 621 * and 64-bit systems, and offers true 64/128 bit hash results. 622 * 623 * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families 624 * @see @ref XXH32_impl for implementation details 625 * @{ 626 */ 627 628 /*! 629 * @brief Calculates the 32-bit hash of @p input using xxHash32. 630 * 631 * @param input The block of data to be hashed, at least @p length bytes in size. 632 * @param length The length of @p input, in bytes. 633 * @param seed The 32-bit seed to alter the hash's output predictably. 634 * 635 * @pre 636 * The memory between @p input and @p input + @p length must be valid, 637 * readable, contiguous memory. However, if @p length is `0`, @p input may be 638 * `NULL`. In C++, this also must be *TriviallyCopyable*. 639 * 640 * @return The calculated 32-bit xxHash32 value. 641 * 642 * @see @ref single_shot_example "Single Shot Example" for an example. 643 */ 644 XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); 645 646 #ifndef XXH_NO_STREAM 647 /*! 648 * @typedef struct XXH32_state_s XXH32_state_t 649 * @brief The opaque state struct for the XXH32 streaming API. 650 * 651 * @see XXH32_state_s for details. 652 * @see @ref streaming_example "Streaming Example" 653 */ 654 typedef struct XXH32_state_s XXH32_state_t; 655 656 /*! 657 * @brief Allocates an @ref XXH32_state_t. 658 * 659 * @return An allocated pointer of @ref XXH32_state_t on success. 660 * @return `NULL` on failure. 661 * 662 * @note Must be freed with XXH32_freeState(). 663 * 664 * @see @ref streaming_example "Streaming Example" 665 */ 666 XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void); 667 /*! 668 * @brief Frees an @ref XXH32_state_t. 669 * 670 * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). 671 * 672 * @return @ref XXH_OK. 673 * 674 * @note @p statePtr must be allocated with XXH32_createState(). 675 * 676 * @see @ref streaming_example "Streaming Example" 677 * 678 */ 679 XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); 680 /*! 681 * @brief Copies one @ref XXH32_state_t to another. 682 * 683 * @param dst_state The state to copy to. 684 * @param src_state The state to copy from. 685 * @pre 686 * @p dst_state and @p src_state must not be `NULL` and must not overlap. 687 */ 688 XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); 689 690 /*! 691 * @brief Resets an @ref XXH32_state_t to begin a new hash. 692 * 693 * @param statePtr The state struct to reset. 694 * @param seed The 32-bit seed to alter the hash result predictably. 695 * 696 * @pre 697 * @p statePtr must not be `NULL`. 698 * 699 * @return @ref XXH_OK on success. 700 * @return @ref XXH_ERROR on failure. 701 * 702 * @note This function resets and seeds a state. Call it before @ref XXH32_update(). 703 * 704 * @see @ref streaming_example "Streaming Example" 705 */ 706 XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); 707 708 /*! 709 * @brief Consumes a block of @p input to an @ref XXH32_state_t. 710 * 711 * @param statePtr The state struct to update. 712 * @param input The block of data to be hashed, at least @p length bytes in size. 713 * @param length The length of @p input, in bytes. 714 * 715 * @pre 716 * @p statePtr must not be `NULL`. 717 * @pre 718 * The memory between @p input and @p input + @p length must be valid, 719 * readable, contiguous memory. However, if @p length is `0`, @p input may be 720 * `NULL`. In C++, this also must be *TriviallyCopyable*. 721 * 722 * @return @ref XXH_OK on success. 723 * @return @ref XXH_ERROR on failure. 724 * 725 * @note Call this to incrementally consume blocks of data. 726 * 727 * @see @ref streaming_example "Streaming Example" 728 */ 729 XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); 730 731 /*! 732 * @brief Returns the calculated hash value from an @ref XXH32_state_t. 733 * 734 * @param statePtr The state struct to calculate the hash from. 735 * 736 * @pre 737 * @p statePtr must not be `NULL`. 738 * 739 * @return The calculated 32-bit xxHash32 value from that state. 740 * 741 * @note 742 * Calling XXH32_digest() will not affect @p statePtr, so you can update, 743 * digest, and update again. 744 * 745 * @see @ref streaming_example "Streaming Example" 746 */ 747 XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); 748 #endif /* !XXH_NO_STREAM */ 749 750 /******* Canonical representation *******/ 751 752 /*! 753 * @brief Canonical (big endian) representation of @ref XXH32_hash_t. 754 */ 755 typedef struct { 756 unsigned char digest[4]; /*!< Hash bytes, big endian */ 757 } XXH32_canonical_t; 758 759 /*! 760 * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. 761 * 762 * @param dst The @ref XXH32_canonical_t pointer to be stored to. 763 * @param hash The @ref XXH32_hash_t to be converted. 764 * 765 * @pre 766 * @p dst must not be `NULL`. 767 * 768 * @see @ref canonical_representation_example "Canonical Representation Example" 769 */ 770 XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); 771 772 /*! 773 * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. 774 * 775 * @param src The @ref XXH32_canonical_t to convert. 776 * 777 * @pre 778 * @p src must not be `NULL`. 779 * 780 * @return The converted hash. 781 * 782 * @see @ref canonical_representation_example "Canonical Representation Example" 783 */ 784 XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); 785 786 787 /*! @cond Doxygen ignores this part */ 788 #ifdef __has_attribute 789 # define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) 790 #else 791 # define XXH_HAS_ATTRIBUTE(x) 0 792 #endif 793 /*! @endcond */ 794 795 /*! @cond Doxygen ignores this part */ 796 /* C-language Attributes are added in C23. */ 797 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 202311L) && defined(__has_c_attribute) 798 # define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) 799 #else 800 # define XXH_HAS_C_ATTRIBUTE(x) 0 801 #endif 802 /*! @endcond */ 803 804 /*! @cond Doxygen ignores this part */ 805 #if defined(__cplusplus) && defined(__has_cpp_attribute) 806 # define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) 807 #else 808 # define XXH_HAS_CPP_ATTRIBUTE(x) 0 809 #endif 810 /*! @endcond */ 811 812 /*! @cond Doxygen ignores this part */ 813 /* 814 * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute 815 * introduced in CPP17 and C23. 816 * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough 817 * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough 818 */ 819 #if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough) 820 # define XXH_FALLTHROUGH [[fallthrough]] 821 #elif XXH_HAS_ATTRIBUTE(__fallthrough__) 822 # define XXH_FALLTHROUGH __attribute__ ((__fallthrough__)) 823 #else 824 # define XXH_FALLTHROUGH /* fallthrough */ 825 #endif 826 /*! @endcond */ 827 828 /*! @cond Doxygen ignores this part */ 829 /* 830 * Define XXH_NOESCAPE for annotated pointers in public API. 831 * https://clang.llvm.org/docs/AttributeReference.html#noescape 832 * As of writing this, only supported by clang. 833 */ 834 #if XXH_HAS_ATTRIBUTE(noescape) 835 # define XXH_NOESCAPE __attribute__((__noescape__)) 836 #else 837 # define XXH_NOESCAPE 838 #endif 839 /*! @endcond */ 840 841 842 /*! 843 * @} 844 * @ingroup public 845 * @{ 846 */ 847 848 #ifndef XXH_NO_LONG_LONG 849 /*-********************************************************************** 850 * 64-bit hash 851 ************************************************************************/ 852 #if defined(XXH_DOXYGEN) /* don't include <stdint.h> */ 853 /*! 854 * @brief An unsigned 64-bit integer. 855 * 856 * Not necessarily defined to `uint64_t` but functionally equivalent. 857 */ 858 typedef uint64_t XXH64_hash_t; 859 #elif !defined (__VMS) \ 860 && (defined (__cplusplus) \ 861 || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) 862 # ifdef _AIX 863 # include <inttypes.h> 864 # else 865 # include <stdint.h> 866 # endif 867 typedef uint64_t XXH64_hash_t; 868 #else 869 # include <limits.h> 870 # if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL 871 /* LP64 ABI says uint64_t is unsigned long */ 872 typedef unsigned long XXH64_hash_t; 873 # else 874 /* the following type must have a width of 64-bit */ 875 typedef unsigned long long XXH64_hash_t; 876 # endif 877 #endif 878 879 /*! 880 * @} 881 * 882 * @defgroup XXH64_family XXH64 family 883 * @ingroup public 884 * @{ 885 * Contains functions used in the classic 64-bit xxHash algorithm. 886 * 887 * @note 888 * XXH3 provides competitive speed for both 32-bit and 64-bit systems, 889 * and offers true 64/128 bit hash results. 890 * It provides better speed for systems with vector processing capabilities. 891 */ 892 893 /*! 894 * @brief Calculates the 64-bit hash of @p input using xxHash64. 895 * 896 * @param input The block of data to be hashed, at least @p length bytes in size. 897 * @param length The length of @p input, in bytes. 898 * @param seed The 64-bit seed to alter the hash's output predictably. 899 * 900 * @pre 901 * The memory between @p input and @p input + @p length must be valid, 902 * readable, contiguous memory. However, if @p length is `0`, @p input may be 903 * `NULL`. In C++, this also must be *TriviallyCopyable*. 904 * 905 * @return The calculated 64-bit xxHash64 value. 906 * 907 * @see @ref single_shot_example "Single Shot Example" for an example. 908 */ 909 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); 910 911 /******* Streaming *******/ 912 #ifndef XXH_NO_STREAM 913 /*! 914 * @brief The opaque state struct for the XXH64 streaming API. 915 * 916 * @see XXH64_state_s for details. 917 * @see @ref streaming_example "Streaming Example" 918 */ 919 typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ 920 921 /*! 922 * @brief Allocates an @ref XXH64_state_t. 923 * 924 * @return An allocated pointer of @ref XXH64_state_t on success. 925 * @return `NULL` on failure. 926 * 927 * @note Must be freed with XXH64_freeState(). 928 * 929 * @see @ref streaming_example "Streaming Example" 930 */ 931 XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void); 932 933 /*! 934 * @brief Frees an @ref XXH64_state_t. 935 * 936 * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState(). 937 * 938 * @return @ref XXH_OK. 939 * 940 * @note @p statePtr must be allocated with XXH64_createState(). 941 * 942 * @see @ref streaming_example "Streaming Example" 943 */ 944 XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); 945 946 /*! 947 * @brief Copies one @ref XXH64_state_t to another. 948 * 949 * @param dst_state The state to copy to. 950 * @param src_state The state to copy from. 951 * @pre 952 * @p dst_state and @p src_state must not be `NULL` and must not overlap. 953 */ 954 XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state); 955 956 /*! 957 * @brief Resets an @ref XXH64_state_t to begin a new hash. 958 * 959 * @param statePtr The state struct to reset. 960 * @param seed The 64-bit seed to alter the hash result predictably. 961 * 962 * @pre 963 * @p statePtr must not be `NULL`. 964 * 965 * @return @ref XXH_OK on success. 966 * @return @ref XXH_ERROR on failure. 967 * 968 * @note This function resets and seeds a state. Call it before @ref XXH64_update(). 969 * 970 * @see @ref streaming_example "Streaming Example" 971 */ 972 XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed); 973 974 /*! 975 * @brief Consumes a block of @p input to an @ref XXH64_state_t. 976 * 977 * @param statePtr The state struct to update. 978 * @param input The block of data to be hashed, at least @p length bytes in size. 979 * @param length The length of @p input, in bytes. 980 * 981 * @pre 982 * @p statePtr must not be `NULL`. 983 * @pre 984 * The memory between @p input and @p input + @p length must be valid, 985 * readable, contiguous memory. However, if @p length is `0`, @p input may be 986 * `NULL`. In C++, this also must be *TriviallyCopyable*. 987 * 988 * @return @ref XXH_OK on success. 989 * @return @ref XXH_ERROR on failure. 990 * 991 * @note Call this to incrementally consume blocks of data. 992 * 993 * @see @ref streaming_example "Streaming Example" 994 */ 995 XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); 996 997 /*! 998 * @brief Returns the calculated hash value from an @ref XXH64_state_t. 999 * 1000 * @param statePtr The state struct to calculate the hash from. 1001 * 1002 * @pre 1003 * @p statePtr must not be `NULL`. 1004 * 1005 * @return The calculated 64-bit xxHash64 value from that state. 1006 * 1007 * @note 1008 * Calling XXH64_digest() will not affect @p statePtr, so you can update, 1009 * digest, and update again. 1010 * 1011 * @see @ref streaming_example "Streaming Example" 1012 */ 1013 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr); 1014 #endif /* !XXH_NO_STREAM */ 1015 /******* Canonical representation *******/ 1016 1017 /*! 1018 * @brief Canonical (big endian) representation of @ref XXH64_hash_t. 1019 */ 1020 typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; 1021 1022 /*! 1023 * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t. 1024 * 1025 * @param dst The @ref XXH64_canonical_t pointer to be stored to. 1026 * @param hash The @ref XXH64_hash_t to be converted. 1027 * 1028 * @pre 1029 * @p dst must not be `NULL`. 1030 * 1031 * @see @ref canonical_representation_example "Canonical Representation Example" 1032 */ 1033 XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash); 1034 1035 /*! 1036 * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t. 1037 * 1038 * @param src The @ref XXH64_canonical_t to convert. 1039 * 1040 * @pre 1041 * @p src must not be `NULL`. 1042 * 1043 * @return The converted hash. 1044 * 1045 * @see @ref canonical_representation_example "Canonical Representation Example" 1046 */ 1047 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src); 1048 1049 #ifndef XXH_NO_XXH3 1050 1051 /*! 1052 * @} 1053 * ************************************************************************ 1054 * @defgroup XXH3_family XXH3 family 1055 * @ingroup public 1056 * @{ 1057 * 1058 * XXH3 is a more recent hash algorithm featuring: 1059 * - Improved speed for both small and large inputs 1060 * - True 64-bit and 128-bit outputs 1061 * - SIMD acceleration 1062 * - Improved 32-bit viability 1063 * 1064 * Speed analysis methodology is explained here: 1065 * 1066 * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html 1067 * 1068 * Compared to XXH64, expect XXH3 to run approximately 1069 * ~2x faster on large inputs and >3x faster on small ones, 1070 * exact differences vary depending on platform. 1071 * 1072 * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, 1073 * but does not require it. 1074 * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3 1075 * at competitive speeds, even without vector support. Further details are 1076 * explained in the implementation. 1077 * 1078 * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD 1079 * implementations for many common platforms: 1080 * - AVX512 1081 * - AVX2 1082 * - SSE2 1083 * - ARM NEON 1084 * - WebAssembly SIMD128 1085 * - POWER8 VSX 1086 * - s390x ZVector 1087 * This can be controlled via the @ref XXH_VECTOR macro, but it automatically 1088 * selects the best version according to predefined macros. For the x86 family, an 1089 * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c. 1090 * 1091 * XXH3 implementation is portable: 1092 * it has a generic C90 formulation that can be compiled on any platform, 1093 * all implementations generate exactly the same hash value on all platforms. 1094 * Starting from v0.8.0, it's also labelled "stable", meaning that 1095 * any future version will also generate the same hash value. 1096 * 1097 * XXH3 offers 2 variants, _64bits and _128bits. 1098 * 1099 * When only 64 bits are needed, prefer invoking the _64bits variant, as it 1100 * reduces the amount of mixing, resulting in faster speed on small inputs. 1101 * It's also generally simpler to manipulate a scalar return type than a struct. 1102 * 1103 * The API supports one-shot hashing, streaming mode, and custom secrets. 1104 */ 1105 1106 /*! 1107 * @ingroup tuning 1108 * @brief Possible values for @ref XXH_VECTOR. 1109 * 1110 * Unless set explicitly, determined automatically. 1111 */ 1112 # define XXH_SCALAR 0 /*!< Portable scalar version */ 1113 # define XXH_SSE2 1 /*!< SSE2 for Pentium 4, Opteron, all x86_64. */ 1114 # define XXH_AVX2 2 /*!< AVX2 for Haswell and Bulldozer */ 1115 # define XXH_AVX512 3 /*!< AVX512 for Skylake and Icelake */ 1116 # define XXH_NEON 4 /*!< NEON for most ARMv7-A, all AArch64, and WASM SIMD128 */ 1117 # define XXH_VSX 5 /*!< VSX and ZVector for POWER8/z13 (64-bit) */ 1118 # define XXH_SVE 6 /*!< SVE for some ARMv8-A and ARMv9-A */ 1119 # define XXH_LSX 7 /*!< LSX (128-bit SIMD) for LoongArch64 */ 1120 # define XXH_LASX 8 /*!< LASX (256-bit SIMD) for LoongArch64 */ 1121 # define XXH_RVV 9 /*!< RVV (RISC-V Vector) for RISC-V */ 1122 1123 /*-********************************************************************** 1124 * XXH3 64-bit variant 1125 ************************************************************************/ 1126 1127 /*! 1128 * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input. 1129 * 1130 * @param input The block of data to be hashed, at least @p length bytes in size. 1131 * @param length The length of @p input, in bytes. 1132 * 1133 * @pre 1134 * The memory between @p input and @p input + @p length must be valid, 1135 * readable, contiguous memory. However, if @p length is `0`, @p input may be 1136 * `NULL`. In C++, this also must be *TriviallyCopyable*. 1137 * 1138 * @return The calculated 64-bit XXH3 hash value. 1139 * 1140 * @note 1141 * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however 1142 * it may have slightly better performance due to constant propagation of the 1143 * defaults. 1144 * 1145 * @see 1146 * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants 1147 * @see @ref single_shot_example "Single Shot Example" for an example. 1148 */ 1149 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length); 1150 1151 /*! 1152 * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input. 1153 * 1154 * @param input The block of data to be hashed, at least @p length bytes in size. 1155 * @param length The length of @p input, in bytes. 1156 * @param seed The 64-bit seed to alter the hash result predictably. 1157 * 1158 * @pre 1159 * The memory between @p input and @p input + @p length must be valid, 1160 * readable, contiguous memory. However, if @p length is `0`, @p input may be 1161 * `NULL`. In C++, this also must be *TriviallyCopyable*. 1162 * 1163 * @return The calculated 64-bit XXH3 hash value. 1164 * 1165 * @note 1166 * seed == 0 produces the same results as @ref XXH3_64bits(). 1167 * 1168 * This variant generates a custom secret on the fly based on default secret 1169 * altered using the @p seed value. 1170 * 1171 * While this operation is decently fast, note that it's not completely free. 1172 * 1173 * @see @ref single_shot_example "Single Shot Example" for an example. 1174 */ 1175 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); 1176 1177 /*! 1178 * The bare minimum size for a custom secret. 1179 * 1180 * @see 1181 * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), 1182 * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). 1183 */ 1184 #define XXH3_SECRET_SIZE_MIN 136 1185 1186 /*! 1187 * @brief Calculates 64-bit variant of XXH3 with a custom "secret". 1188 * 1189 * @param data The block of data to be hashed, at least @p len bytes in size. 1190 * @param len The length of @p data, in bytes. 1191 * @param secret The secret data. 1192 * @param secretSize The length of @p secret, in bytes. 1193 * 1194 * @return The calculated 64-bit XXH3 hash value. 1195 * 1196 * @pre 1197 * The memory between @p data and @p data + @p len must be valid, 1198 * readable, contiguous memory. However, if @p length is `0`, @p data may be 1199 * `NULL`. In C++, this also must be *TriviallyCopyable*. 1200 * 1201 * It's possible to provide any blob of bytes as a "secret" to generate the hash. 1202 * This makes it more difficult for an external actor to prepare an intentional collision. 1203 * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). 1204 * However, the quality of the secret impacts the dispersion of the hash algorithm. 1205 * Therefore, the secret _must_ look like a bunch of random bytes. 1206 * Avoid "trivial" or structured data such as repeated sequences or a text document. 1207 * Whenever in doubt about the "randomness" of the blob of bytes, 1208 * consider employing @ref XXH3_generateSecret() instead (see below). 1209 * It will generate a proper high entropy secret derived from the blob of bytes. 1210 * Another advantage of using XXH3_generateSecret() is that 1211 * it guarantees that all bits within the initial blob of bytes 1212 * will impact every bit of the output. 1213 * This is not necessarily the case when using the blob of bytes directly 1214 * because, when hashing _small_ inputs, only a portion of the secret is employed. 1215 * 1216 * @see @ref single_shot_example "Single Shot Example" for an example. 1217 */ 1218 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); 1219 1220 1221 /******* Streaming *******/ 1222 #ifndef XXH_NO_STREAM 1223 /* 1224 * Streaming requires state maintenance. 1225 * This operation costs memory and CPU. 1226 * As a consequence, streaming is slower than one-shot hashing. 1227 * For better performance, prefer one-shot functions whenever applicable. 1228 */ 1229 1230 /*! 1231 * @brief The opaque state struct for the XXH3 streaming API. 1232 * 1233 * @see XXH3_state_s for details. 1234 * @see @ref streaming_example "Streaming Example" 1235 */ 1236 typedef struct XXH3_state_s XXH3_state_t; 1237 XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void); 1238 XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); 1239 1240 /*! 1241 * @brief Copies one @ref XXH3_state_t to another. 1242 * 1243 * @param dst_state The state to copy to. 1244 * @param src_state The state to copy from. 1245 * @pre 1246 * @p dst_state and @p src_state must not be `NULL` and must not overlap. 1247 */ 1248 XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state); 1249 1250 /*! 1251 * @brief Resets an @ref XXH3_state_t to begin a new hash. 1252 * 1253 * @param statePtr The state struct to reset. 1254 * 1255 * @pre 1256 * @p statePtr must not be `NULL`. 1257 * 1258 * @return @ref XXH_OK on success. 1259 * @return @ref XXH_ERROR on failure. 1260 * 1261 * @note 1262 * - This function resets `statePtr` and generate a secret with default parameters. 1263 * - Call this function before @ref XXH3_64bits_update(). 1264 * - Digest will be equivalent to `XXH3_64bits()`. 1265 * 1266 * @see @ref streaming_example "Streaming Example" 1267 * 1268 */ 1269 XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); 1270 1271 /*! 1272 * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. 1273 * 1274 * @param statePtr The state struct to reset. 1275 * @param seed The 64-bit seed to alter the hash result predictably. 1276 * 1277 * @pre 1278 * @p statePtr must not be `NULL`. 1279 * 1280 * @return @ref XXH_OK on success. 1281 * @return @ref XXH_ERROR on failure. 1282 * 1283 * @note 1284 * - This function resets `statePtr` and generate a secret from `seed`. 1285 * - Call this function before @ref XXH3_64bits_update(). 1286 * - Digest will be equivalent to `XXH3_64bits_withSeed()`. 1287 * 1288 * @see @ref streaming_example "Streaming Example" 1289 * 1290 */ 1291 XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); 1292 1293 /*! 1294 * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. 1295 * 1296 * @param statePtr The state struct to reset. 1297 * @param secret The secret data. 1298 * @param secretSize The length of @p secret, in bytes. 1299 * 1300 * @pre 1301 * @p statePtr must not be `NULL`. 1302 * 1303 * @return @ref XXH_OK on success. 1304 * @return @ref XXH_ERROR on failure. 1305 * 1306 * @note 1307 * `secret` is referenced, it _must outlive_ the hash streaming session. 1308 * 1309 * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, 1310 * and the quality of produced hash values depends on secret's entropy 1311 * (secret's content should look like a bunch of random bytes). 1312 * When in doubt about the randomness of a candidate `secret`, 1313 * consider employing `XXH3_generateSecret()` instead (see below). 1314 * 1315 * @see @ref streaming_example "Streaming Example" 1316 */ 1317 XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); 1318 1319 /*! 1320 * @brief Consumes a block of @p input to an @ref XXH3_state_t. 1321 * 1322 * @param statePtr The state struct to update. 1323 * @param input The block of data to be hashed, at least @p length bytes in size. 1324 * @param length The length of @p input, in bytes. 1325 * 1326 * @pre 1327 * @p statePtr must not be `NULL`. 1328 * @pre 1329 * The memory between @p input and @p input + @p length must be valid, 1330 * readable, contiguous memory. However, if @p length is `0`, @p input may be 1331 * `NULL`. In C++, this also must be *TriviallyCopyable*. 1332 * 1333 * @return @ref XXH_OK on success. 1334 * @return @ref XXH_ERROR on failure. 1335 * 1336 * @note Call this to incrementally consume blocks of data. 1337 * 1338 * @see @ref streaming_example "Streaming Example" 1339 */ 1340 XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); 1341 1342 /*! 1343 * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t. 1344 * 1345 * @param statePtr The state struct to calculate the hash from. 1346 * 1347 * @pre 1348 * @p statePtr must not be `NULL`. 1349 * 1350 * @return The calculated XXH3 64-bit hash value from that state. 1351 * 1352 * @note 1353 * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update, 1354 * digest, and update again. 1355 * 1356 * @see @ref streaming_example "Streaming Example" 1357 */ 1358 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); 1359 #endif /* !XXH_NO_STREAM */ 1360 1361 /* note : canonical representation of XXH3 is the same as XXH64 1362 * since they both produce XXH64_hash_t values */ 1363 1364 1365 /*-********************************************************************** 1366 * XXH3 128-bit variant 1367 ************************************************************************/ 1368 1369 /*! 1370 * @brief The return value from 128-bit hashes. 1371 * 1372 * Stored in little endian order, although the fields themselves are in native 1373 * endianness. 1374 */ 1375 typedef struct { 1376 XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ 1377 XXH64_hash_t high64; /*!< `value >> 64` */ 1378 } XXH128_hash_t; 1379 1380 /*! 1381 * @brief Calculates 128-bit unseeded variant of XXH3 of @p data. 1382 * 1383 * @param data The block of data to be hashed, at least @p length bytes in size. 1384 * @param len The length of @p data, in bytes. 1385 * 1386 * @return The calculated 128-bit variant of XXH3 value. 1387 * 1388 * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead 1389 * for shorter inputs. 1390 * 1391 * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however 1392 * it may have slightly better performance due to constant propagation of the 1393 * defaults. 1394 * 1395 * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants 1396 * @see @ref single_shot_example "Single Shot Example" for an example. 1397 */ 1398 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len); 1399 /*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. 1400 * 1401 * @param data The block of data to be hashed, at least @p length bytes in size. 1402 * @param len The length of @p data, in bytes. 1403 * @param seed The 64-bit seed to alter the hash result predictably. 1404 * 1405 * @return The calculated 128-bit variant of XXH3 value. 1406 * 1407 * @note 1408 * seed == 0 produces the same results as @ref XXH3_64bits(). 1409 * 1410 * This variant generates a custom secret on the fly based on default secret 1411 * altered using the @p seed value. 1412 * 1413 * While this operation is decently fast, note that it's not completely free. 1414 * 1415 * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants 1416 * @see @ref single_shot_example "Single Shot Example" for an example. 1417 */ 1418 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); 1419 /*! 1420 * @brief Calculates 128-bit variant of XXH3 with a custom "secret". 1421 * 1422 * @param data The block of data to be hashed, at least @p len bytes in size. 1423 * @param len The length of @p data, in bytes. 1424 * @param secret The secret data. 1425 * @param secretSize The length of @p secret, in bytes. 1426 * 1427 * @return The calculated 128-bit variant of XXH3 value. 1428 * 1429 * It's possible to provide any blob of bytes as a "secret" to generate the hash. 1430 * This makes it more difficult for an external actor to prepare an intentional collision. 1431 * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). 1432 * However, the quality of the secret impacts the dispersion of the hash algorithm. 1433 * Therefore, the secret _must_ look like a bunch of random bytes. 1434 * Avoid "trivial" or structured data such as repeated sequences or a text document. 1435 * Whenever in doubt about the "randomness" of the blob of bytes, 1436 * consider employing @ref XXH3_generateSecret() instead (see below). 1437 * It will generate a proper high entropy secret derived from the blob of bytes. 1438 * Another advantage of using XXH3_generateSecret() is that 1439 * it guarantees that all bits within the initial blob of bytes 1440 * will impact every bit of the output. 1441 * This is not necessarily the case when using the blob of bytes directly 1442 * because, when hashing _small_ inputs, only a portion of the secret is employed. 1443 * 1444 * @see @ref single_shot_example "Single Shot Example" for an example. 1445 */ 1446 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); 1447 1448 /******* Streaming *******/ 1449 #ifndef XXH_NO_STREAM 1450 /* 1451 * Streaming requires state maintenance. 1452 * This operation costs memory and CPU. 1453 * As a consequence, streaming is slower than one-shot hashing. 1454 * For better performance, prefer one-shot functions whenever applicable. 1455 * 1456 * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). 1457 * Use already declared XXH3_createState() and XXH3_freeState(). 1458 * 1459 * All reset and streaming functions have same meaning as their 64-bit counterpart. 1460 */ 1461 1462 /*! 1463 * @brief Resets an @ref XXH3_state_t to begin a new hash. 1464 * 1465 * @param statePtr The state struct to reset. 1466 * 1467 * @pre 1468 * @p statePtr must not be `NULL`. 1469 * 1470 * @return @ref XXH_OK on success. 1471 * @return @ref XXH_ERROR on failure. 1472 * 1473 * @note 1474 * - This function resets `statePtr` and generate a secret with default parameters. 1475 * - Call it before @ref XXH3_128bits_update(). 1476 * - Digest will be equivalent to `XXH3_128bits()`. 1477 * 1478 * @see @ref streaming_example "Streaming Example" 1479 */ 1480 XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); 1481 1482 /*! 1483 * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. 1484 * 1485 * @param statePtr The state struct to reset. 1486 * @param seed The 64-bit seed to alter the hash result predictably. 1487 * 1488 * @pre 1489 * @p statePtr must not be `NULL`. 1490 * 1491 * @return @ref XXH_OK on success. 1492 * @return @ref XXH_ERROR on failure. 1493 * 1494 * @note 1495 * - This function resets `statePtr` and generate a secret from `seed`. 1496 * - Call it before @ref XXH3_128bits_update(). 1497 * - Digest will be equivalent to `XXH3_128bits_withSeed()`. 1498 * 1499 * @see @ref streaming_example "Streaming Example" 1500 */ 1501 XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); 1502 /*! 1503 * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. 1504 * 1505 * @param statePtr The state struct to reset. 1506 * @param secret The secret data. 1507 * @param secretSize The length of @p secret, in bytes. 1508 * 1509 * @pre 1510 * @p statePtr must not be `NULL`. 1511 * 1512 * @return @ref XXH_OK on success. 1513 * @return @ref XXH_ERROR on failure. 1514 * 1515 * `secret` is referenced, it _must outlive_ the hash streaming session. 1516 * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, 1517 * and the quality of produced hash values depends on secret's entropy 1518 * (secret's content should look like a bunch of random bytes). 1519 * When in doubt about the randomness of a candidate `secret`, 1520 * consider employing `XXH3_generateSecret()` instead (see below). 1521 * 1522 * @see @ref streaming_example "Streaming Example" 1523 */ 1524 XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); 1525 1526 /*! 1527 * @brief Consumes a block of @p input to an @ref XXH3_state_t. 1528 * 1529 * Call this to incrementally consume blocks of data. 1530 * 1531 * @param statePtr The state struct to update. 1532 * @param input The block of data to be hashed, at least @p length bytes in size. 1533 * @param length The length of @p input, in bytes. 1534 * 1535 * @pre 1536 * @p statePtr must not be `NULL`. 1537 * 1538 * @return @ref XXH_OK on success. 1539 * @return @ref XXH_ERROR on failure. 1540 * 1541 * @note 1542 * The memory between @p input and @p input + @p length must be valid, 1543 * readable, contiguous memory. However, if @p length is `0`, @p input may be 1544 * `NULL`. In C++, this also must be *TriviallyCopyable*. 1545 * 1546 */ 1547 XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); 1548 1549 /*! 1550 * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t. 1551 * 1552 * @param statePtr The state struct to calculate the hash from. 1553 * 1554 * @pre 1555 * @p statePtr must not be `NULL`. 1556 * 1557 * @return The calculated XXH3 128-bit hash value from that state. 1558 * 1559 * @note 1560 * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update, 1561 * digest, and update again. 1562 * 1563 */ 1564 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); 1565 #endif /* !XXH_NO_STREAM */ 1566 1567 /* Following helper functions make it possible to compare XXH128_hast_t values. 1568 * Since XXH128_hash_t is a structure, this capability is not offered by the language. 1569 * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ 1570 1571 /*! 1572 * @brief Check equality of two XXH128_hash_t values 1573 * 1574 * @param h1 The 128-bit hash value. 1575 * @param h2 Another 128-bit hash value. 1576 * 1577 * @return `1` if `h1` and `h2` are equal. 1578 * @return `0` if they are not. 1579 */ 1580 XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); 1581 1582 /*! 1583 * @brief Compares two @ref XXH128_hash_t 1584 * 1585 * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. 1586 * 1587 * @param h128_1 Left-hand side value 1588 * @param h128_2 Right-hand side value 1589 * 1590 * @return >0 if @p h128_1 > @p h128_2 1591 * @return =0 if @p h128_1 == @p h128_2 1592 * @return <0 if @p h128_1 < @p h128_2 1593 */ 1594 XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2); 1595 1596 1597 /******* Canonical representation *******/ 1598 typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; 1599 1600 1601 /*! 1602 * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t. 1603 * 1604 * @param dst The @ref XXH128_canonical_t pointer to be stored to. 1605 * @param hash The @ref XXH128_hash_t to be converted. 1606 * 1607 * @pre 1608 * @p dst must not be `NULL`. 1609 * @see @ref canonical_representation_example "Canonical Representation Example" 1610 */ 1611 XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash); 1612 1613 /*! 1614 * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t. 1615 * 1616 * @param src The @ref XXH128_canonical_t to convert. 1617 * 1618 * @pre 1619 * @p src must not be `NULL`. 1620 * 1621 * @return The converted hash. 1622 * @see @ref canonical_representation_example "Canonical Representation Example" 1623 */ 1624 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src); 1625 1626 1627 #endif /* !XXH_NO_XXH3 */ 1628 #endif /* XXH_NO_LONG_LONG */ 1629 1630 /*! 1631 * @} 1632 */ 1633 #endif /* XXHASH_H_5627135585666179 */ 1634 1635 1636 1637 #if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) 1638 #define XXHASH_H_STATIC_13879238742 1639 /* **************************************************************************** 1640 * This section contains declarations which are not guaranteed to remain stable. 1641 * They may change in future versions, becoming incompatible with a different 1642 * version of the library. 1643 * These declarations should only be used with static linking. 1644 * Never use them in association with dynamic linking! 1645 ***************************************************************************** */ 1646 1647 /* 1648 * These definitions are only present to allow static allocation 1649 * of XXH states, on stack or in a struct, for example. 1650 * Never **ever** access their members directly. 1651 */ 1652 1653 /*! 1654 * @internal 1655 * @brief Structure for XXH32 streaming API. 1656 * 1657 * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, 1658 * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is 1659 * an opaque type. This allows fields to safely be changed. 1660 * 1661 * Typedef'd to @ref XXH32_state_t. 1662 * Do not access the members of this struct directly. 1663 * @see XXH64_state_s, XXH3_state_s 1664 */ 1665 struct XXH32_state_s { 1666 XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ 1667 XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ 1668 XXH32_hash_t acc[4]; /*!< Accumulator lanes */ 1669 unsigned char buffer[16]; /*!< Internal buffer for partial reads. */ 1670 XXH32_hash_t bufferedSize; /*!< Amount of data in @ref buffer */ 1671 XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ 1672 }; /* typedef'd to XXH32_state_t */ 1673 1674 1675 #ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ 1676 1677 /*! 1678 * @internal 1679 * @brief Structure for XXH64 streaming API. 1680 * 1681 * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, 1682 * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is 1683 * an opaque type. This allows fields to safely be changed. 1684 * 1685 * Typedef'd to @ref XXH64_state_t. 1686 * Do not access the members of this struct directly. 1687 * @see XXH32_state_s, XXH3_state_s 1688 */ 1689 struct XXH64_state_s { 1690 XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ 1691 XXH64_hash_t acc[4]; /*!< Accumulator lanes */ 1692 unsigned char buffer[32]; /*!< Internal buffer for partial reads.. */ 1693 XXH32_hash_t bufferedSize; /*!< Amount of data in @ref buffer */ 1694 XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ 1695 XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ 1696 }; /* typedef'd to XXH64_state_t */ 1697 1698 #ifndef XXH_NO_XXH3 1699 1700 #if defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ 1701 /* In C++ alignas() is a keyword */ 1702 # define XXH_ALIGN(n) alignas(n) 1703 #elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* > C11 */ 1704 # define XXH_ALIGN(n) _Alignas(n) 1705 #elif defined(__GNUC__) 1706 # define XXH_ALIGN(n) __attribute__ ((aligned(n))) 1707 #elif defined(_MSC_VER) 1708 # define XXH_ALIGN(n) __declspec(align(n)) 1709 #else 1710 # define XXH_ALIGN(n) /* disabled */ 1711 #endif 1712 1713 /* Old GCC versions only accept the attribute after the type in structures. */ 1714 #if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ 1715 && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ 1716 && defined(__GNUC__) 1717 # define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) 1718 #else 1719 # define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type 1720 #endif 1721 1722 /*! 1723 * @internal 1724 * @brief The size of the internal XXH3 buffer. 1725 * 1726 * This is the optimal update size for incremental hashing. 1727 * 1728 * @see XXH3_64b_update(), XXH3_128b_update(). 1729 */ 1730 #define XXH3_INTERNALBUFFER_SIZE 256 1731 1732 /*! 1733 * @def XXH3_SECRET_DEFAULT_SIZE 1734 * @brief Default Secret's size 1735 * 1736 * This is the size of internal XXH3_kSecret 1737 * and is needed by XXH3_generateSecret_fromSeed(). 1738 * 1739 * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. 1740 */ 1741 #define XXH3_SECRET_DEFAULT_SIZE 192 1742 1743 /*! 1744 * @internal 1745 * @brief Structure for XXH3 streaming API. 1746 * 1747 * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, 1748 * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. 1749 * Otherwise it is an opaque type. 1750 * Never use this definition in combination with dynamic library. 1751 * This allows fields to safely be changed in the future. 1752 * 1753 * @note ** This structure has a strict alignment requirement of 64 bytes!! ** 1754 * Do not allocate this with `malloc()` or `new`, 1755 * it will not be sufficiently aligned. 1756 * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. 1757 * 1758 * Typedef'd to @ref XXH3_state_t. 1759 * Do never access the members of this struct directly. 1760 * 1761 * @see XXH3_INITSTATE() for stack initialization. 1762 * @see XXH3_createState(), XXH3_freeState(). 1763 * @see XXH32_state_s, XXH64_state_s 1764 */ 1765 struct XXH3_state_s { 1766 XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); 1767 /*!< The 8 accumulators. See @ref XXH32_state_s::acc and @ref XXH64_state_s::acc */ 1768 XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); 1769 /*!< Used to store a custom secret generated from a seed. */ 1770 XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); 1771 /*!< The internal buffer. @see XXH32_state_s::mem32 */ 1772 XXH32_hash_t bufferedSize; 1773 /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ 1774 XXH32_hash_t useSeed; 1775 /*!< Reserved field. Needed for padding on 64-bit. */ 1776 size_t nbStripesSoFar; 1777 /*!< Number or stripes processed. */ 1778 XXH64_hash_t totalLen; 1779 /*!< Total length hashed. 64-bit even on 32-bit targets. */ 1780 size_t nbStripesPerBlock; 1781 /*!< Number of stripes per block. */ 1782 size_t secretLimit; 1783 /*!< Size of @ref customSecret or @ref extSecret */ 1784 XXH64_hash_t seed; 1785 /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ 1786 XXH64_hash_t reserved64; 1787 /*!< Reserved field. */ 1788 const unsigned char* extSecret; 1789 /*!< Reference to an external secret for the _withSecret variants, NULL 1790 * for other variants. */ 1791 /* note: there may be some padding at the end due to alignment on 64 bytes */ 1792 }; /* typedef'd to XXH3_state_t */ 1793 1794 #undef XXH_ALIGN_MEMBER 1795 1796 /*! 1797 * @brief Initializes a stack-allocated `XXH3_state_s`. 1798 * 1799 * When the @ref XXH3_state_t structure is merely emplaced on stack, 1800 * it should be initialized with XXH3_INITSTATE() or a memset() 1801 * in case its first reset uses XXH3_NNbits_reset_withSeed(). 1802 * This init can be omitted if the first reset uses default or _withSecret mode. 1803 * This operation isn't necessary when the state is created with XXH3_createState(). 1804 * Note that this doesn't prepare the state for a streaming operation, 1805 * it's still necessary to use XXH3_NNbits_reset*() afterwards. 1806 */ 1807 #define XXH3_INITSTATE(XXH3_state_ptr) \ 1808 do { \ 1809 XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \ 1810 tmp_xxh3_state_ptr->seed = 0; \ 1811 tmp_xxh3_state_ptr->extSecret = NULL; \ 1812 } while(0) 1813 1814 1815 /*! 1816 * @brief Calculates the 128-bit hash of @p data using XXH3. 1817 * 1818 * @param data The block of data to be hashed, at least @p len bytes in size. 1819 * @param len The length of @p data, in bytes. 1820 * @param seed The 64-bit seed to alter the hash's output predictably. 1821 * 1822 * @pre 1823 * The memory between @p data and @p data + @p len must be valid, 1824 * readable, contiguous memory. However, if @p len is `0`, @p data may be 1825 * `NULL`. In C++, this also must be *TriviallyCopyable*. 1826 * 1827 * @return The calculated 128-bit XXH3 value. 1828 * 1829 * @see @ref single_shot_example "Single Shot Example" for an example. 1830 */ 1831 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); 1832 1833 1834 /* === Experimental API === */ 1835 /* Symbols defined below must be considered tied to a specific library version. */ 1836 1837 /*! 1838 * @brief Derive a high-entropy secret from any user-defined content, named customSeed. 1839 * 1840 * @param secretBuffer A writable buffer for derived high-entropy secret data. 1841 * @param secretSize Size of secretBuffer, in bytes. Must be >= XXH3_SECRET_SIZE_MIN. 1842 * @param customSeed A user-defined content. 1843 * @param customSeedSize Size of customSeed, in bytes. 1844 * 1845 * @return @ref XXH_OK on success. 1846 * @return @ref XXH_ERROR on failure. 1847 * 1848 * The generated secret can be used in combination with `*_withSecret()` functions. 1849 * The `_withSecret()` variants are useful to provide a higher level of protection 1850 * than 64-bit seed, as it becomes much more difficult for an external actor to 1851 * guess how to impact the calculation logic. 1852 * 1853 * The function accepts as input a custom seed of any length and any content, 1854 * and derives from it a high-entropy secret of length @p secretSize into an 1855 * already allocated buffer @p secretBuffer. 1856 * 1857 * The generated secret can then be used with any `*_withSecret()` variant. 1858 * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(), 1859 * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret() 1860 * are part of this list. They all accept a `secret` parameter 1861 * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN) 1862 * _and_ feature very high entropy (consist of random-looking bytes). 1863 * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can 1864 * be employed to ensure proper quality. 1865 * 1866 * @p customSeed can be anything. It can have any size, even small ones, 1867 * and its content can be anything, even "poor entropy" sources such as a bunch 1868 * of zeroes. The resulting `secret` will nonetheless provide all required qualities. 1869 * 1870 * @pre 1871 * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN 1872 * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior. 1873 * 1874 * Example code: 1875 * @code{.c} 1876 * #include <stdio.h> 1877 * #include <stdlib.h> 1878 * #include <string.h> 1879 * #define XXH_STATIC_LINKING_ONLY // expose unstable API 1880 * #include "xxhash.h" 1881 * // Hashes argv[2] using the entropy from argv[1]. 1882 * int main(int argc, char* argv[]) 1883 * { 1884 * char secret[XXH3_SECRET_SIZE_MIN]; 1885 * if (argv != 3) { return 1; } 1886 * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1])); 1887 * XXH64_hash_t h = XXH3_64bits_withSecret( 1888 * argv[2], strlen(argv[2]), 1889 * secret, sizeof(secret) 1890 * ); 1891 * printf("%016llx\n", (unsigned long long) h); 1892 * } 1893 * @endcode 1894 */ 1895 XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize); 1896 1897 /*! 1898 * @brief Generate the same secret as the _withSeed() variants. 1899 * 1900 * @param secretBuffer A writable buffer of @ref XXH3_SECRET_DEFAULT_SIZE bytes 1901 * @param seed The 64-bit seed to alter the hash result predictably. 1902 * 1903 * The generated secret can be used in combination with 1904 *`*_withSecret()` and `_withSecretandSeed()` variants. 1905 * 1906 * Example C++ `std::string` hash class: 1907 * @code{.cpp} 1908 * #include <string> 1909 * #define XXH_STATIC_LINKING_ONLY // expose unstable API 1910 * #include "xxhash.h" 1911 * // Slow, seeds each time 1912 * class HashSlow { 1913 * XXH64_hash_t seed; 1914 * public: 1915 * HashSlow(XXH64_hash_t s) : seed{s} {} 1916 * size_t operator()(const std::string& x) const { 1917 * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)}; 1918 * } 1919 * }; 1920 * // Fast, caches the seeded secret for future uses. 1921 * class HashFast { 1922 * unsigned char secret[XXH3_SECRET_DEFAULT_SIZE]; 1923 * public: 1924 * HashFast(XXH64_hash_t s) { 1925 * XXH3_generateSecret_fromSeed(secret, seed); 1926 * } 1927 * size_t operator()(const std::string& x) const { 1928 * return size_t{ 1929 * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret)) 1930 * }; 1931 * } 1932 * }; 1933 * @endcode 1934 */ 1935 XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed); 1936 1937 /*! 1938 * @brief Maximum size of "short" key in bytes. 1939 */ 1940 #define XXH3_MIDSIZE_MAX 240 1941 1942 /*! 1943 * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data. 1944 * 1945 * @param data The block of data to be hashed, at least @p len bytes in size. 1946 * @param len The length of @p data, in bytes. 1947 * @param secret The secret data. 1948 * @param secretSize The length of @p secret, in bytes. 1949 * @param seed The 64-bit seed to alter the hash result predictably. 1950 * 1951 * These variants generate hash values using either: 1952 * - @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes) 1953 * - @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX). 1954 * 1955 * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. 1956 * `_withSeed()` has to generate the secret on the fly for "large" keys. 1957 * It's fast, but can be perceptible for "not so large" keys (< 1 KB). 1958 * `_withSecret()` has to generate the masks on the fly for "small" keys, 1959 * which requires more instructions than _withSeed() variants. 1960 * Therefore, _withSecretandSeed variant combines the best of both worlds. 1961 * 1962 * When @p secret has been generated by XXH3_generateSecret_fromSeed(), 1963 * this variant produces *exactly* the same results as `_withSeed()` variant, 1964 * hence offering only a pure speed benefit on "large" input, 1965 * by skipping the need to regenerate the secret for every large input. 1966 * 1967 * Another usage scenario is to hash the secret to a 64-bit hash value, 1968 * for example with XXH3_64bits(), which then becomes the seed, 1969 * and then employ both the seed and the secret in _withSecretandSeed(). 1970 * On top of speed, an added benefit is that each bit in the secret 1971 * has a 50% chance to swap each bit in the output, via its impact to the seed. 1972 * 1973 * This is not guaranteed when using the secret directly in "small data" scenarios, 1974 * because only portions of the secret are employed for small data. 1975 */ 1976 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t 1977 XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len, 1978 XXH_NOESCAPE const void* secret, size_t secretSize, 1979 XXH64_hash_t seed); 1980 1981 /*! 1982 * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. 1983 * 1984 * @param input The memory segment to be hashed, at least @p len bytes in size. 1985 * @param length The length of @p data, in bytes. 1986 * @param secret The secret used to alter hash result predictably. 1987 * @param secretSize The length of @p secret, in bytes (must be >= XXH3_SECRET_SIZE_MIN) 1988 * @param seed64 The 64-bit seed to alter the hash result predictably. 1989 * 1990 * @return @ref XXH_OK on success. 1991 * @return @ref XXH_ERROR on failure. 1992 * 1993 * @see XXH3_64bits_withSecretandSeed(): contract is the same. 1994 */ 1995 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t 1996 XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, 1997 XXH_NOESCAPE const void* secret, size_t secretSize, 1998 XXH64_hash_t seed64); 1999 2000 #ifndef XXH_NO_STREAM 2001 /*! 2002 * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. 2003 * 2004 * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). 2005 * @param secret The secret data. 2006 * @param secretSize The length of @p secret, in bytes. 2007 * @param seed64 The 64-bit seed to alter the hash result predictably. 2008 * 2009 * @return @ref XXH_OK on success. 2010 * @return @ref XXH_ERROR on failure. 2011 * 2012 * @see XXH3_64bits_withSecretandSeed(). Contract is identical. 2013 */ 2014 XXH_PUBLIC_API XXH_errorcode 2015 XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, 2016 XXH_NOESCAPE const void* secret, size_t secretSize, 2017 XXH64_hash_t seed64); 2018 2019 /*! 2020 * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. 2021 * 2022 * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). 2023 * @param secret The secret data. 2024 * @param secretSize The length of @p secret, in bytes. 2025 * @param seed64 The 64-bit seed to alter the hash result predictably. 2026 * 2027 * @return @ref XXH_OK on success. 2028 * @return @ref XXH_ERROR on failure. 2029 * 2030 * @see XXH3_64bits_withSecretandSeed(). Contract is identical. 2031 * 2032 * Note: there was a bug in an earlier version of this function (<= v0.8.2) 2033 * that would make it generate an incorrect hash value 2034 * when @p seed == 0 and @p length < XXH3_MIDSIZE_MAX 2035 * and @p secret is different from XXH3_generateSecret_fromSeed(). 2036 * As stated in the contract, the correct hash result must be 2037 * the same as XXH3_128bits_withSeed() when @p length <= XXH3_MIDSIZE_MAX. 2038 * Results generated by this older version are wrong, hence not comparable. 2039 */ 2040 XXH_PUBLIC_API XXH_errorcode 2041 XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, 2042 XXH_NOESCAPE const void* secret, size_t secretSize, 2043 XXH64_hash_t seed64); 2044 2045 #endif /* !XXH_NO_STREAM */ 2046 2047 #endif /* !XXH_NO_XXH3 */ 2048 #endif /* XXH_NO_LONG_LONG */ 2049 #if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) 2050 # define XXH_IMPLEMENTATION 2051 #endif 2052 2053 #endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ 2054 2055 #if defined (__cplusplus) && !defined(XXH_NO_EXTERNC_GUARD) 2056 } /* extern "C" */ 2057 #endif 2058 2059 2060 /* ======================================================================== */ 2061 /* ======================================================================== */ 2062 /* ======================================================================== */ 2063 2064 2065 /*-********************************************************************** 2066 * xxHash implementation 2067 *-********************************************************************** 2068 * xxHash's implementation used to be hosted inside xxhash.c. 2069 * 2070 * However, inlining requires implementation to be visible to the compiler, 2071 * hence be included alongside the header. 2072 * Previously, implementation was hosted inside xxhash.c, 2073 * which was then #included when inlining was activated. 2074 * This construction created issues with a few build and install systems, 2075 * as it required xxhash.c to be stored in /include directory. 2076 * 2077 * xxHash implementation is now directly integrated within xxhash.h. 2078 * As a consequence, xxhash.c is no longer needed in /include. 2079 * 2080 * xxhash.c is still available and is still useful. 2081 * In a "normal" setup, when xxhash is not inlined, 2082 * xxhash.h only exposes the prototypes and public symbols, 2083 * while xxhash.c can be built into an object file xxhash.o 2084 * which can then be linked into the final binary. 2085 ************************************************************************/ 2086 2087 #if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ 2088 || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) 2089 # define XXH_IMPLEM_13a8737387 2090 2091 /* ************************************* 2092 * Tuning parameters 2093 ***************************************/ 2094 2095 /*! 2096 * @defgroup tuning Tuning parameters 2097 * @{ 2098 * 2099 * Various macros to control xxHash's behavior. 2100 */ 2101 #ifdef XXH_DOXYGEN 2102 /*! 2103 * @brief Define this to disable 64-bit code. 2104 * 2105 * Useful if only using the @ref XXH32_family and you have a strict C90 compiler. 2106 */ 2107 # define XXH_NO_LONG_LONG 2108 # undef XXH_NO_LONG_LONG /* don't actually */ 2109 /*! 2110 * @brief Controls how unaligned memory is accessed. 2111 * 2112 * By default, access to unaligned memory is controlled by `memcpy()`, which is 2113 * safe and portable. 2114 * 2115 * Unfortunately, on some target/compiler combinations, the generated assembly 2116 * is sub-optimal. 2117 * 2118 * The below switch allow selection of a different access method 2119 * in the search for improved performance. 2120 * 2121 * @par Possible options: 2122 * 2123 * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` 2124 * @par 2125 * Use `memcpy()`. Safe and portable. Note that most modern compilers will 2126 * eliminate the function call and treat it as an unaligned access. 2127 * 2128 * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))` 2129 * @par 2130 * Depends on compiler extensions and is therefore not portable. 2131 * This method is safe _if_ your compiler supports it, 2132 * and *generally* as fast or faster than `memcpy`. 2133 * 2134 * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast 2135 * @par 2136 * Casts directly and dereferences. This method doesn't depend on the 2137 * compiler, but it violates the C standard as it directly dereferences an 2138 * unaligned pointer. It can generate buggy code on targets which do not 2139 * support unaligned memory accesses, but in some circumstances, it's the 2140 * only known way to get the most performance. 2141 * 2142 * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift 2143 * @par 2144 * Also portable. This can generate the best code on old compilers which don't 2145 * inline small `memcpy()` calls, and it might also be faster on big-endian 2146 * systems which lack a native byteswap instruction. However, some compilers 2147 * will emit literal byteshifts even if the target supports unaligned access. 2148 * 2149 * 2150 * @warning 2151 * Methods 1 and 2 rely on implementation-defined behavior. Use these with 2152 * care, as what works on one compiler/platform/optimization level may cause 2153 * another to read garbage data or even crash. 2154 * 2155 * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. 2156 * 2157 * Prefer these methods in priority order (0 > 3 > 1 > 2) 2158 */ 2159 # define XXH_FORCE_MEMORY_ACCESS 0 2160 2161 /*! 2162 * @def XXH_SIZE_OPT 2163 * @brief Controls how much xxHash optimizes for size. 2164 * 2165 * xxHash, when compiled, tends to result in a rather large binary size. This 2166 * is mostly due to heavy usage to forced inlining and constant folding of the 2167 * @ref XXH3_family to increase performance. 2168 * 2169 * However, some developers prefer size over speed. This option can 2170 * significantly reduce the size of the generated code. When using the `-Os` 2171 * or `-Oz` options on GCC or Clang, this is defined to 1 by default, 2172 * otherwise it is defined to 0. 2173 * 2174 * Most of these size optimizations can be controlled manually. 2175 * 2176 * This is a number from 0-2. 2177 * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed 2178 * comes first. 2179 * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more 2180 * conservative and disables hacks that increase code size. It implies the 2181 * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0, 2182 * and @ref XXH3_NEON_LANES == 8 if they are not already defined. 2183 * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible. 2184 * Performance may cry. For example, the single shot functions just use the 2185 * streaming API. 2186 */ 2187 # define XXH_SIZE_OPT 0 2188 2189 /*! 2190 * @def XXH_FORCE_ALIGN_CHECK 2191 * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() 2192 * and XXH64() only). 2193 * 2194 * This is an important performance trick for architectures without decent 2195 * unaligned memory access performance. 2196 * 2197 * It checks for input alignment, and when conditions are met, uses a "fast 2198 * path" employing direct 32-bit/64-bit reads, resulting in _dramatically 2199 * faster_ read speed. 2200 * 2201 * The check costs one initial branch per hash, which is generally negligible, 2202 * but not zero. 2203 * 2204 * Moreover, it's not useful to generate an additional code path if memory 2205 * access uses the same instruction for both aligned and unaligned 2206 * addresses (e.g. x86 and aarch64). 2207 * 2208 * In these cases, the alignment check can be removed by setting this macro to 0. 2209 * Then the code will always use unaligned memory access. 2210 * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips 2211 * which are platforms known to offer good unaligned memory accesses performance. 2212 * 2213 * It is also disabled by default when @ref XXH_SIZE_OPT >= 1. 2214 * 2215 * This option does not affect XXH3 (only XXH32 and XXH64). 2216 */ 2217 # define XXH_FORCE_ALIGN_CHECK 0 2218 2219 /*! 2220 * @def XXH_NO_INLINE_HINTS 2221 * @brief When non-zero, sets all functions to `static`. 2222 * 2223 * By default, xxHash tries to force the compiler to inline almost all internal 2224 * functions. 2225 * 2226 * This can usually improve performance due to reduced jumping and improved 2227 * constant folding, but significantly increases the size of the binary which 2228 * might not be favorable. 2229 * 2230 * Additionally, sometimes the forced inlining can be detrimental to performance, 2231 * depending on the architecture. 2232 * 2233 * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the 2234 * compiler full control on whether to inline or not. 2235 * 2236 * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if 2237 * @ref XXH_SIZE_OPT >= 1, this will automatically be defined. 2238 */ 2239 # define XXH_NO_INLINE_HINTS 0 2240 2241 /*! 2242 * @def XXH3_INLINE_SECRET 2243 * @brief Determines whether to inline the XXH3 withSecret code. 2244 * 2245 * When the secret size is known, the compiler can improve the performance 2246 * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret(). 2247 * 2248 * However, if the secret size is not known, it doesn't have any benefit. This 2249 * happens when xxHash is compiled into a global symbol. Therefore, if 2250 * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0. 2251 * 2252 * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers 2253 * that are *sometimes* force inline on -Og, and it is impossible to automatically 2254 * detect this optimization level. 2255 */ 2256 # define XXH3_INLINE_SECRET 0 2257 2258 /*! 2259 * @def XXH32_ENDJMP 2260 * @brief Whether to use a jump for `XXH32_finalize`. 2261 * 2262 * For performance, `XXH32_finalize` uses multiple branches in the finalizer. 2263 * This is generally preferable for performance, 2264 * but depending on exact architecture, a jmp may be preferable. 2265 * 2266 * This setting is only possibly making a difference for very small inputs. 2267 */ 2268 # define XXH32_ENDJMP 0 2269 2270 /*! 2271 * @internal 2272 * @brief Redefines old internal names. 2273 * 2274 * For compatibility with code that uses xxHash's internals before the names 2275 * were changed to improve namespacing. There is no other reason to use this. 2276 */ 2277 # define XXH_OLD_NAMES 2278 # undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ 2279 2280 /*! 2281 * @def XXH_NO_STREAM 2282 * @brief Disables the streaming API. 2283 * 2284 * When xxHash is not inlined and the streaming functions are not used, disabling 2285 * the streaming functions can improve code size significantly, especially with 2286 * the @ref XXH3_family which tends to make constant folded copies of itself. 2287 */ 2288 # define XXH_NO_STREAM 2289 # undef XXH_NO_STREAM /* don't actually */ 2290 #endif /* XXH_DOXYGEN */ 2291 /*! 2292 * @} 2293 */ 2294 2295 #ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ 2296 /* prefer __packed__ structures (method 1) for GCC 2297 * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy 2298 * which for some reason does unaligned loads. */ 2299 # if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED)) 2300 # define XXH_FORCE_MEMORY_ACCESS 1 2301 # endif 2302 #endif 2303 2304 #ifndef XXH_SIZE_OPT 2305 /* default to 1 for -Os or -Oz */ 2306 # if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__) 2307 # define XXH_SIZE_OPT 1 2308 # else 2309 # define XXH_SIZE_OPT 0 2310 # endif 2311 #endif 2312 2313 #ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ 2314 /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */ 2315 # if XXH_SIZE_OPT >= 1 || \ 2316 defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \ 2317 || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */ 2318 # define XXH_FORCE_ALIGN_CHECK 0 2319 # else 2320 # define XXH_FORCE_ALIGN_CHECK 1 2321 # endif 2322 #endif 2323 2324 #ifndef XXH_NO_INLINE_HINTS 2325 # if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */ 2326 # define XXH_NO_INLINE_HINTS 1 2327 # else 2328 # define XXH_NO_INLINE_HINTS 0 2329 # endif 2330 #endif 2331 2332 #ifndef XXH3_INLINE_SECRET 2333 # if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \ 2334 || !defined(XXH_INLINE_ALL) 2335 # define XXH3_INLINE_SECRET 0 2336 # else 2337 # define XXH3_INLINE_SECRET 1 2338 # endif 2339 #endif 2340 2341 #ifndef XXH32_ENDJMP 2342 /* generally preferable for performance */ 2343 # define XXH32_ENDJMP 0 2344 #endif 2345 2346 /*! 2347 * @defgroup impl Implementation 2348 * @{ 2349 */ 2350 2351 2352 /* ************************************* 2353 * Includes & Memory related functions 2354 ***************************************/ 2355 #if defined(XXH_NO_STREAM) 2356 /* nothing */ 2357 #elif defined(XXH_NO_STDLIB) 2358 2359 /* When requesting to disable any mention of stdlib, 2360 * the library loses the ability to invoked malloc / free. 2361 * In practice, it means that functions like `XXH*_createState()` 2362 * will always fail, and return NULL. 2363 * This flag is useful in situations where 2364 * xxhash.h is integrated into some kernel, embedded or limited environment 2365 * without access to dynamic allocation. 2366 */ 2367 2368 static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; } 2369 static void XXH_free(void* p) { (void)p; } 2370 2371 #else 2372 2373 /* 2374 * Modify the local functions below should you wish to use 2375 * different memory routines for malloc() and free() 2376 */ 2377 #include <stdlib.h> 2378 2379 /*! 2380 * @internal 2381 * @brief Modify this function to use a different routine than malloc(). 2382 */ 2383 static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); } 2384 2385 /*! 2386 * @internal 2387 * @brief Modify this function to use a different routine than free(). 2388 */ 2389 static void XXH_free(void* p) { free(p); } 2390 2391 #endif /* XXH_NO_STDLIB */ 2392 2393 #ifndef XXH_memcpy 2394 /*! 2395 * @internal 2396 * @brief XXH_memcpy() macro can be redirected at compile time 2397 */ 2398 # include <string.h> 2399 # define XXH_memcpy memcpy 2400 #endif 2401 2402 #ifndef XXH_memset 2403 /*! 2404 * @internal 2405 * @brief XXH_memset() macro can be redirected at compile time 2406 */ 2407 # include <string.h> 2408 # define XXH_memset memset 2409 #endif 2410 2411 #ifndef XXH_memcmp 2412 /*! 2413 * @internal 2414 * @brief XXH_memcmp() macro can be redirected at compile time 2415 * Note: only needed by XXH128. 2416 */ 2417 # include <string.h> 2418 # define XXH_memcmp memcmp 2419 #endif 2420 2421 2422 2423 #include <limits.h> /* ULLONG_MAX */ 2424 2425 2426 /* ************************************* 2427 * Compiler Specific Options 2428 ***************************************/ 2429 #ifdef _MSC_VER /* Visual Studio warning fix */ 2430 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 2431 #endif 2432 2433 #if XXH_NO_INLINE_HINTS /* disable inlining hints */ 2434 # if defined(__GNUC__) || defined(__clang__) 2435 # define XXH_FORCE_INLINE static __attribute__((__unused__)) 2436 # else 2437 # define XXH_FORCE_INLINE static 2438 # endif 2439 # define XXH_NO_INLINE static 2440 /* enable inlining hints */ 2441 #elif defined(__GNUC__) || defined(__clang__) 2442 # define XXH_FORCE_INLINE static __inline__ __attribute__((__always_inline__, __unused__)) 2443 # define XXH_NO_INLINE static __attribute__((__noinline__)) 2444 #elif defined(_MSC_VER) /* Visual Studio */ 2445 # define XXH_FORCE_INLINE static __forceinline 2446 # define XXH_NO_INLINE static __declspec(noinline) 2447 #elif defined (__cplusplus) \ 2448 || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ 2449 # define XXH_FORCE_INLINE static inline 2450 # define XXH_NO_INLINE static 2451 #else 2452 # define XXH_FORCE_INLINE static 2453 # define XXH_NO_INLINE static 2454 #endif 2455 2456 #if defined(XXH_INLINE_ALL) 2457 # define XXH_STATIC XXH_FORCE_INLINE 2458 #else 2459 # define XXH_STATIC static 2460 #endif 2461 2462 #if XXH3_INLINE_SECRET 2463 # define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE 2464 #else 2465 # define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE 2466 #endif 2467 2468 /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ 2469 #if ((defined(sun) || defined(__sun)) && defined(__cplusplus)) 2470 # define XXH_RESTRICT /* disable */ 2471 #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ 2472 # define XXH_RESTRICT restrict 2473 #elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \ 2474 || (defined (__clang__)) \ 2475 || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \ 2476 || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300)) 2477 /* 2478 * There are a LOT more compilers that recognize __restrict but this 2479 * covers the major ones. 2480 */ 2481 # define XXH_RESTRICT __restrict 2482 #else 2483 # define XXH_RESTRICT /* disable */ 2484 #endif 2485 2486 /* ************************************* 2487 * Debug 2488 ***************************************/ 2489 /*! 2490 * @ingroup tuning 2491 * @def XXH_DEBUGLEVEL 2492 * @brief Sets the debugging level. 2493 * 2494 * XXH_DEBUGLEVEL is expected to be defined externally, typically via the 2495 * compiler's command line options. The value must be a number. 2496 */ 2497 #ifndef XXH_DEBUGLEVEL 2498 # ifdef DEBUGLEVEL /* backwards compat */ 2499 # define XXH_DEBUGLEVEL DEBUGLEVEL 2500 # else 2501 # define XXH_DEBUGLEVEL 0 2502 # endif 2503 #endif 2504 2505 #if (XXH_DEBUGLEVEL>=1) 2506 # include <assert.h> /* note: can still be disabled with NDEBUG */ 2507 # define XXH_ASSERT(c) assert(c) 2508 #else 2509 # if defined(__INTEL_COMPILER) 2510 # define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c)) 2511 # else 2512 # define XXH_ASSERT(c) XXH_ASSUME(c) 2513 # endif 2514 #endif 2515 2516 /* note: use after variable declarations */ 2517 #ifndef XXH_STATIC_ASSERT 2518 # if defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ 2519 # define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) 2520 # elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ 2521 # define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0) 2522 # else 2523 # define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) 2524 # endif 2525 # define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) 2526 #endif 2527 2528 /*! 2529 * @internal 2530 * @def XXH_COMPILER_GUARD(var) 2531 * @brief Used to prevent unwanted optimizations for @p var. 2532 * 2533 * It uses an empty GCC inline assembly statement with a register constraint 2534 * which forces @p var into a general purpose register (eg eax, ebx, ecx 2535 * on x86) and marks it as modified. 2536 * 2537 * This is used in a few places to avoid unwanted autovectorization (e.g. 2538 * XXH32_round()). All vectorization we want is explicit via intrinsics, 2539 * and _usually_ isn't wanted elsewhere. 2540 * 2541 * We also use it to prevent unwanted constant folding for AArch64 in 2542 * XXH3_initCustomSecret_scalar(). 2543 */ 2544 #if defined(__GNUC__) || defined(__clang__) 2545 # define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var)) 2546 #else 2547 # define XXH_COMPILER_GUARD(var) ((void)0) 2548 #endif 2549 2550 /* Specifically for NEON vectors which use the "w" constraint, on 2551 * Clang. */ 2552 #if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__) 2553 # define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var)) 2554 #else 2555 # define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0) 2556 #endif 2557 2558 /* ************************************* 2559 * Basic Types 2560 ***************************************/ 2561 #if !defined (__VMS) \ 2562 && (defined (__cplusplus) \ 2563 || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) 2564 # ifdef _AIX 2565 # include <inttypes.h> 2566 # else 2567 # include <stdint.h> 2568 # endif 2569 typedef uint8_t xxh_u8; 2570 #else 2571 typedef unsigned char xxh_u8; 2572 #endif 2573 typedef XXH32_hash_t xxh_u32; 2574 2575 #ifdef XXH_OLD_NAMES 2576 # warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly" 2577 # define BYTE xxh_u8 2578 # define U8 xxh_u8 2579 # define U32 xxh_u32 2580 #endif 2581 2582 /* *** Memory access *** */ 2583 2584 /*! 2585 * @internal 2586 * @fn xxh_u32 XXH_read32(const void* ptr) 2587 * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. 2588 * 2589 * Affected by @ref XXH_FORCE_MEMORY_ACCESS. 2590 * 2591 * @param ptr The pointer to read from. 2592 * @return The 32-bit native endian integer from the bytes at @p ptr. 2593 */ 2594 2595 /*! 2596 * @internal 2597 * @fn xxh_u32 XXH_readLE32(const void* ptr) 2598 * @brief Reads an unaligned 32-bit little endian integer from @p ptr. 2599 * 2600 * Affected by @ref XXH_FORCE_MEMORY_ACCESS. 2601 * 2602 * @param ptr The pointer to read from. 2603 * @return The 32-bit little endian integer from the bytes at @p ptr. 2604 */ 2605 2606 /*! 2607 * @internal 2608 * @fn xxh_u32 XXH_readBE32(const void* ptr) 2609 * @brief Reads an unaligned 32-bit big endian integer from @p ptr. 2610 * 2611 * Affected by @ref XXH_FORCE_MEMORY_ACCESS. 2612 * 2613 * @param ptr The pointer to read from. 2614 * @return The 32-bit big endian integer from the bytes at @p ptr. 2615 */ 2616 2617 /*! 2618 * @internal 2619 * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) 2620 * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. 2621 * 2622 * Affected by @ref XXH_FORCE_MEMORY_ACCESS. 2623 * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is 2624 * always @ref XXH_alignment::XXH_unaligned. 2625 * 2626 * @param ptr The pointer to read from. 2627 * @param align Whether @p ptr is aligned. 2628 * @pre 2629 * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte 2630 * aligned. 2631 * @return The 32-bit little endian integer from the bytes at @p ptr. 2632 */ 2633 2634 #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) 2635 /* 2636 * Manual byteshift. Best for old compilers which don't inline memcpy. 2637 * We actually directly use XXH_readLE32 and XXH_readBE32. 2638 */ 2639 #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) 2640 2641 /* 2642 * Force direct memory access. Only works on CPU which support unaligned memory 2643 * access in hardware. 2644 */ 2645 static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } 2646 2647 #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) 2648 2649 /* 2650 * __attribute__((aligned(1))) is supported by gcc and clang. Originally the 2651 * documentation claimed that it only increased the alignment, but actually it 2652 * can decrease it on gcc, clang, and icc: 2653 * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, 2654 * https://gcc.godbolt.org/z/xYez1j67Y. 2655 */ 2656 #ifdef XXH_OLD_NAMES 2657 typedef union { xxh_u32 u32; } __attribute__((__packed__)) unalign; 2658 #endif 2659 static xxh_u32 XXH_read32(const void* ptr) 2660 { 2661 typedef __attribute__((__aligned__(1))) __attribute__((__may_alias__)) xxh_u32 xxh_unalign32; 2662 return *((const xxh_unalign32*)ptr); 2663 } 2664 2665 #else 2666 2667 /* 2668 * Portable and safe solution. Generally efficient. 2669 * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html 2670 */ 2671 static xxh_u32 XXH_read32(const void* memPtr) 2672 { 2673 xxh_u32 val; 2674 XXH_memcpy(&val, memPtr, sizeof(val)); 2675 return val; 2676 } 2677 2678 #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ 2679 2680 2681 /* *** Endianness *** */ 2682 2683 /*! 2684 * @ingroup tuning 2685 * @def XXH_CPU_LITTLE_ENDIAN 2686 * @brief Whether the target is little endian. 2687 * 2688 * Defined to 1 if the target is little endian, or 0 if it is big endian. 2689 * It can be defined externally, for example on the compiler command line. 2690 * 2691 * If it is not defined, 2692 * a runtime check (which is usually constant folded) is used instead. 2693 * 2694 * @note 2695 * This is not necessarily defined to an integer constant. 2696 * 2697 * @see XXH_isLittleEndian() for the runtime check. 2698 */ 2699 #ifndef XXH_CPU_LITTLE_ENDIAN 2700 /* 2701 * Try to detect endianness automatically, to avoid the nonstandard behavior 2702 * in `XXH_isLittleEndian()` 2703 */ 2704 # if defined(_WIN32) /* Windows is always little endian */ \ 2705 || defined(__LITTLE_ENDIAN__) \ 2706 || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) 2707 # define XXH_CPU_LITTLE_ENDIAN 1 2708 # elif defined(__BIG_ENDIAN__) \ 2709 || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) 2710 # define XXH_CPU_LITTLE_ENDIAN 0 2711 # else 2712 /*! 2713 * @internal 2714 * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. 2715 * 2716 * Most compilers will constant fold this. 2717 */ 2718 static int XXH_isLittleEndian(void) 2719 { 2720 /* 2721 * Portable and well-defined behavior. 2722 * Don't use static: it is detrimental to performance. 2723 */ 2724 const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; 2725 return one.c[0]; 2726 } 2727 # define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() 2728 # endif 2729 #endif 2730 2731 2732 2733 2734 /* **************************************** 2735 * Compiler-specific Functions and Macros 2736 ******************************************/ 2737 #define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) 2738 2739 #ifdef __has_builtin 2740 # define XXH_HAS_BUILTIN(x) __has_builtin(x) 2741 #else 2742 # define XXH_HAS_BUILTIN(x) 0 2743 #endif 2744 2745 2746 2747 /* 2748 * C23 and future versions have standard "unreachable()". 2749 * Once it has been implemented reliably we can add it as an 2750 * additional case: 2751 * 2752 * ``` 2753 * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 202311L) 2754 * # include <stddef.h> 2755 * # ifdef unreachable 2756 * # define XXH_UNREACHABLE() unreachable() 2757 * # endif 2758 * #endif 2759 * ``` 2760 * 2761 * Note C++23 also has std::unreachable() which can be detected 2762 * as follows: 2763 * ``` 2764 * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L) 2765 * # include <utility> 2766 * # define XXH_UNREACHABLE() std::unreachable() 2767 * #endif 2768 * ``` 2769 * NB: `__cpp_lib_unreachable` is defined in the `<version>` header. 2770 * We don't use that as including `<utility>` in `extern "C"` blocks 2771 * doesn't work on GCC12 2772 */ 2773 2774 #if XXH_HAS_BUILTIN(__builtin_unreachable) 2775 # define XXH_UNREACHABLE() __builtin_unreachable() 2776 2777 #elif defined(_MSC_VER) 2778 # define XXH_UNREACHABLE() __assume(0) 2779 2780 #else 2781 # define XXH_UNREACHABLE() 2782 #endif 2783 2784 #if XXH_HAS_BUILTIN(__builtin_assume) 2785 # define XXH_ASSUME(c) __builtin_assume(c) 2786 #else 2787 # define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); } 2788 #endif 2789 2790 /*! 2791 * @internal 2792 * @def XXH_rotl32(x,r) 2793 * @brief 32-bit rotate left. 2794 * 2795 * @param x The 32-bit integer to be rotated. 2796 * @param r The number of bits to rotate. 2797 * @pre 2798 * @p r > 0 && @p r < 32 2799 * @note 2800 * @p x and @p r may be evaluated multiple times. 2801 * @return The rotated result. 2802 */ 2803 #if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ 2804 && XXH_HAS_BUILTIN(__builtin_rotateleft64) 2805 # define XXH_rotl32 __builtin_rotateleft32 2806 # define XXH_rotl64 __builtin_rotateleft64 2807 #elif XXH_HAS_BUILTIN(__builtin_stdc_rotate_left) 2808 # define XXH_rotl32 __builtin_stdc_rotate_left 2809 # define XXH_rotl64 __builtin_stdc_rotate_left 2810 /* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ 2811 #elif defined(_MSC_VER) 2812 # define XXH_rotl32(x,r) _rotl(x,r) 2813 # define XXH_rotl64(x,r) _rotl64(x,r) 2814 #else 2815 # define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) 2816 # define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) 2817 #endif 2818 2819 /*! 2820 * @internal 2821 * @fn xxh_u32 XXH_swap32(xxh_u32 x) 2822 * @brief A 32-bit byteswap. 2823 * 2824 * @param x The 32-bit integer to byteswap. 2825 * @return @p x, byteswapped. 2826 */ 2827 #if defined(_MSC_VER) /* Visual Studio */ 2828 # define XXH_swap32 _byteswap_ulong 2829 #elif XXH_GCC_VERSION >= 403 2830 # define XXH_swap32 __builtin_bswap32 2831 #else 2832 static xxh_u32 XXH_swap32 (xxh_u32 x) 2833 { 2834 return ((x << 24) & 0xff000000 ) | 2835 ((x << 8) & 0x00ff0000 ) | 2836 ((x >> 8) & 0x0000ff00 ) | 2837 ((x >> 24) & 0x000000ff ); 2838 } 2839 #endif 2840 2841 2842 /* *************************** 2843 * Memory reads 2844 *****************************/ 2845 2846 /*! 2847 * @internal 2848 * @brief Enum to indicate whether a pointer is aligned. 2849 */ 2850 typedef enum { 2851 XXH_aligned, /*!< Aligned */ 2852 XXH_unaligned /*!< Possibly unaligned */ 2853 } XXH_alignment; 2854 2855 /* 2856 * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. 2857 * 2858 * This is ideal for older compilers which don't inline memcpy. 2859 */ 2860 #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) 2861 2862 XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) 2863 { 2864 const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; 2865 return bytePtr[0] 2866 | ((xxh_u32)bytePtr[1] << 8) 2867 | ((xxh_u32)bytePtr[2] << 16) 2868 | ((xxh_u32)bytePtr[3] << 24); 2869 } 2870 2871 XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) 2872 { 2873 const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; 2874 return bytePtr[3] 2875 | ((xxh_u32)bytePtr[2] << 8) 2876 | ((xxh_u32)bytePtr[1] << 16) 2877 | ((xxh_u32)bytePtr[0] << 24); 2878 } 2879 2880 #else 2881 XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) 2882 { 2883 return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); 2884 } 2885 2886 static xxh_u32 XXH_readBE32(const void* ptr) 2887 { 2888 return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); 2889 } 2890 #endif 2891 2892 XXH_FORCE_INLINE xxh_u32 2893 XXH_readLE32_align(const void* ptr, XXH_alignment align) 2894 { 2895 if (align==XXH_unaligned) { 2896 return XXH_readLE32(ptr); 2897 } else { 2898 return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); 2899 } 2900 } 2901 2902 2903 /* ************************************* 2904 * Misc 2905 ***************************************/ 2906 /*! @ingroup public */ 2907 XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } 2908 2909 2910 /* ******************************************************************* 2911 * 32-bit hash functions 2912 *********************************************************************/ 2913 /*! 2914 * @} 2915 * @defgroup XXH32_impl XXH32 implementation 2916 * @ingroup impl 2917 * 2918 * Details on the XXH32 implementation. 2919 * @{ 2920 */ 2921 /* #define instead of static const, to be used as initializers */ 2922 #define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ 2923 #define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ 2924 #define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ 2925 #define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ 2926 #define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ 2927 2928 #ifdef XXH_OLD_NAMES 2929 # define PRIME32_1 XXH_PRIME32_1 2930 # define PRIME32_2 XXH_PRIME32_2 2931 # define PRIME32_3 XXH_PRIME32_3 2932 # define PRIME32_4 XXH_PRIME32_4 2933 # define PRIME32_5 XXH_PRIME32_5 2934 #endif 2935 2936 /*! 2937 * @internal 2938 * @brief Normal stripe processing routine. 2939 * 2940 * This shuffles the bits so that any bit from @p input impacts several bits in 2941 * @p acc. 2942 * 2943 * @param acc The accumulator lane. 2944 * @param input The stripe of input to mix. 2945 * @return The mixed accumulator lane. 2946 */ 2947 static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) 2948 { 2949 acc += input * XXH_PRIME32_2; 2950 acc = XXH_rotl32(acc, 13); 2951 acc *= XXH_PRIME32_1; 2952 #if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) 2953 /* 2954 * UGLY HACK: 2955 * A compiler fence is used to prevent GCC and Clang from 2956 * autovectorizing the XXH32 loop (pragmas and attributes don't work for some 2957 * reason) without globally disabling SSE4.1. 2958 * 2959 * The reason we want to avoid vectorization is because despite working on 2960 * 4 integers at a time, there are multiple factors slowing XXH32 down on 2961 * SSE4: 2962 * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on 2963 * newer chips!) making it slightly slower to multiply four integers at 2964 * once compared to four integers independently. Even when pmulld was 2965 * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE 2966 * just to multiply unless doing a long operation. 2967 * 2968 * - Four instructions are required to rotate, 2969 * movqda tmp, v // not required with VEX encoding 2970 * pslld tmp, 13 // tmp <<= 13 2971 * psrld v, 19 // x >>= 19 2972 * por v, tmp // x |= tmp 2973 * compared to one for scalar: 2974 * roll v, 13 // reliably fast across the board 2975 * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason 2976 * 2977 * - Instruction level parallelism is actually more beneficial here because 2978 * the SIMD actually serializes this operation: While v1 is rotating, v2 2979 * can load data, while v3 can multiply. SSE forces them to operate 2980 * together. 2981 * 2982 * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing 2983 * the loop. NEON is only faster on the A53, and with the newer cores, it is less 2984 * than half the speed. 2985 * 2986 * Additionally, this is used on WASM SIMD128 because it JITs to the same 2987 * SIMD instructions and has the same issue. 2988 */ 2989 XXH_COMPILER_GUARD(acc); 2990 #endif 2991 return acc; 2992 } 2993 2994 /*! 2995 * @internal 2996 * @brief Mixes all bits to finalize the hash. 2997 * 2998 * The final mix ensures that all input bits have a chance to impact any bit in 2999 * the output digest, resulting in an unbiased distribution. 3000 * 3001 * @param hash The hash to avalanche. 3002 * @return The avalanched hash. 3003 */ 3004 static xxh_u32 XXH32_avalanche(xxh_u32 hash) 3005 { 3006 hash ^= hash >> 15; 3007 hash *= XXH_PRIME32_2; 3008 hash ^= hash >> 13; 3009 hash *= XXH_PRIME32_3; 3010 hash ^= hash >> 16; 3011 return hash; 3012 } 3013 3014 #define XXH_get32bits(p) XXH_readLE32_align(p, align) 3015 3016 /*! 3017 * @internal 3018 * @brief Sets up the initial accumulator state for XXH32(). 3019 */ 3020 XXH_FORCE_INLINE void 3021 XXH32_initAccs(xxh_u32 *acc, xxh_u32 seed) 3022 { 3023 XXH_ASSERT(acc != NULL); 3024 acc[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; 3025 acc[1] = seed + XXH_PRIME32_2; 3026 acc[2] = seed + 0; 3027 acc[3] = seed - XXH_PRIME32_1; 3028 } 3029 3030 /*! 3031 * @internal 3032 * @brief Consumes a block of data for XXH32(). 3033 * 3034 * @return the end input pointer. 3035 */ 3036 XXH_FORCE_INLINE const xxh_u8 * 3037 XXH32_consumeLong( 3038 xxh_u32 *XXH_RESTRICT acc, 3039 xxh_u8 const *XXH_RESTRICT input, 3040 size_t len, 3041 XXH_alignment align 3042 ) 3043 { 3044 const xxh_u8* const bEnd = input + len; 3045 const xxh_u8* const limit = bEnd - 15; 3046 XXH_ASSERT(acc != NULL); 3047 XXH_ASSERT(input != NULL); 3048 XXH_ASSERT(len >= 16); 3049 do { 3050 acc[0] = XXH32_round(acc[0], XXH_get32bits(input)); input += 4; 3051 acc[1] = XXH32_round(acc[1], XXH_get32bits(input)); input += 4; 3052 acc[2] = XXH32_round(acc[2], XXH_get32bits(input)); input += 4; 3053 acc[3] = XXH32_round(acc[3], XXH_get32bits(input)); input += 4; 3054 } while (input < limit); 3055 3056 return input; 3057 } 3058 3059 /*! 3060 * @internal 3061 * @brief Merges the accumulator lanes together for XXH32() 3062 */ 3063 XXH_FORCE_INLINE XXH_PUREF xxh_u32 3064 XXH32_mergeAccs(const xxh_u32 *acc) 3065 { 3066 XXH_ASSERT(acc != NULL); 3067 return XXH_rotl32(acc[0], 1) + XXH_rotl32(acc[1], 7) 3068 + XXH_rotl32(acc[2], 12) + XXH_rotl32(acc[3], 18); 3069 } 3070 3071 /*! 3072 * @internal 3073 * @brief Processes the last 0-15 bytes of @p ptr. 3074 * 3075 * There may be up to 15 bytes remaining to consume from the input. 3076 * This final stage will digest them to ensure that all input bytes are present 3077 * in the final mix. 3078 * 3079 * @param hash The hash to finalize. 3080 * @param ptr The pointer to the remaining input. 3081 * @param len The remaining length, modulo 16. 3082 * @param align Whether @p ptr is aligned. 3083 * @return The finalized hash. 3084 * @see XXH64_finalize(). 3085 */ 3086 static XXH_PUREF xxh_u32 3087 XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) 3088 { 3089 #define XXH_PROCESS1 do { \ 3090 hash += (*ptr++) * XXH_PRIME32_5; \ 3091 hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \ 3092 } while (0) 3093 3094 #define XXH_PROCESS4 do { \ 3095 hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \ 3096 ptr += 4; \ 3097 hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \ 3098 } while (0) 3099 3100 if (ptr==NULL) XXH_ASSERT(len == 0); 3101 3102 /* Compact rerolled version; generally faster */ 3103 if (!XXH32_ENDJMP) { 3104 len &= 15; 3105 while (len >= 4) { 3106 XXH_PROCESS4; 3107 len -= 4; 3108 } 3109 while (len > 0) { 3110 XXH_PROCESS1; 3111 --len; 3112 } 3113 return XXH32_avalanche(hash); 3114 } else { 3115 switch(len&15) /* or switch(bEnd - p) */ { 3116 case 12: XXH_PROCESS4; 3117 XXH_FALLTHROUGH; /* fallthrough */ 3118 case 8: XXH_PROCESS4; 3119 XXH_FALLTHROUGH; /* fallthrough */ 3120 case 4: XXH_PROCESS4; 3121 return XXH32_avalanche(hash); 3122 3123 case 13: XXH_PROCESS4; 3124 XXH_FALLTHROUGH; /* fallthrough */ 3125 case 9: XXH_PROCESS4; 3126 XXH_FALLTHROUGH; /* fallthrough */ 3127 case 5: XXH_PROCESS4; 3128 XXH_PROCESS1; 3129 return XXH32_avalanche(hash); 3130 3131 case 14: XXH_PROCESS4; 3132 XXH_FALLTHROUGH; /* fallthrough */ 3133 case 10: XXH_PROCESS4; 3134 XXH_FALLTHROUGH; /* fallthrough */ 3135 case 6: XXH_PROCESS4; 3136 XXH_PROCESS1; 3137 XXH_PROCESS1; 3138 return XXH32_avalanche(hash); 3139 3140 case 15: XXH_PROCESS4; 3141 XXH_FALLTHROUGH; /* fallthrough */ 3142 case 11: XXH_PROCESS4; 3143 XXH_FALLTHROUGH; /* fallthrough */ 3144 case 7: XXH_PROCESS4; 3145 XXH_FALLTHROUGH; /* fallthrough */ 3146 case 3: XXH_PROCESS1; 3147 XXH_FALLTHROUGH; /* fallthrough */ 3148 case 2: XXH_PROCESS1; 3149 XXH_FALLTHROUGH; /* fallthrough */ 3150 case 1: XXH_PROCESS1; 3151 XXH_FALLTHROUGH; /* fallthrough */ 3152 case 0: return XXH32_avalanche(hash); 3153 } 3154 XXH_ASSERT(0); 3155 return hash; /* reaching this point is deemed impossible */ 3156 } 3157 } 3158 3159 #ifdef XXH_OLD_NAMES 3160 # define PROCESS1 XXH_PROCESS1 3161 # define PROCESS4 XXH_PROCESS4 3162 #else 3163 # undef XXH_PROCESS1 3164 # undef XXH_PROCESS4 3165 #endif 3166 3167 /*! 3168 * @internal 3169 * @brief The implementation for @ref XXH32(). 3170 * 3171 * @param input , len , seed Directly passed from @ref XXH32(). 3172 * @param align Whether @p input is aligned. 3173 * @return The calculated hash. 3174 */ 3175 #if defined(_MSC_VER) 3176 # pragma warning(push) 3177 /* MSVC's LTCG reports valid branches eliminated for constant input lengths. */ 3178 # pragma warning(disable : 4702) /* C4702: unreachable code */ 3179 #endif 3180 XXH_FORCE_INLINE XXH_PUREF xxh_u32 3181 XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) 3182 { 3183 xxh_u32 h32; 3184 3185 if (input==NULL) XXH_ASSERT(len == 0); 3186 3187 if (len>=16) { 3188 xxh_u32 acc[4]; 3189 XXH32_initAccs(acc, seed); 3190 3191 input = XXH32_consumeLong(acc, input, len, align); 3192 3193 h32 = XXH32_mergeAccs(acc); 3194 } else { 3195 h32 = seed + XXH_PRIME32_5; 3196 } 3197 3198 h32 += (xxh_u32)len; 3199 3200 return XXH32_finalize(h32, input, len&15, align); 3201 } 3202 #if defined(_MSC_VER) 3203 # pragma warning(pop) 3204 #endif 3205 3206 /*! @ingroup XXH32_family */ 3207 XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) 3208 { 3209 #if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 3210 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ 3211 XXH32_state_t state; 3212 XXH32_reset(&state, seed); 3213 XXH32_update(&state, (const xxh_u8*)input, len); 3214 return XXH32_digest(&state); 3215 #else 3216 if (XXH_FORCE_ALIGN_CHECK) { 3217 if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ 3218 return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); 3219 } } 3220 3221 return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); 3222 #endif 3223 } 3224 3225 3226 3227 /******* Hash streaming *******/ 3228 #ifndef XXH_NO_STREAM 3229 /*! @ingroup XXH32_family */ 3230 XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) 3231 { 3232 return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); 3233 } 3234 /*! @ingroup XXH32_family */ 3235 XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) 3236 { 3237 XXH_free(statePtr); 3238 return XXH_OK; 3239 } 3240 3241 /*! @ingroup XXH32_family */ 3242 XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) 3243 { 3244 XXH_memcpy(dstState, srcState, sizeof(*dstState)); 3245 } 3246 3247 /*! @ingroup XXH32_family */ 3248 XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) 3249 { 3250 XXH_ASSERT(statePtr != NULL); 3251 XXH_memset(statePtr, 0, sizeof(*statePtr)); 3252 XXH32_initAccs(statePtr->acc, seed); 3253 return XXH_OK; 3254 } 3255 3256 3257 /*! @ingroup XXH32_family */ 3258 XXH_PUBLIC_API XXH_errorcode 3259 XXH32_update(XXH32_state_t* state, const void* input, size_t len) 3260 { 3261 if (input==NULL) { 3262 XXH_ASSERT(len == 0); 3263 return XXH_OK; 3264 } 3265 3266 state->total_len_32 += (XXH32_hash_t)len; 3267 state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); 3268 3269 XXH_ASSERT(state->bufferedSize < sizeof(state->buffer)); 3270 if (len < sizeof(state->buffer) - state->bufferedSize) { /* fill in tmp buffer */ 3271 XXH_memcpy(state->buffer + state->bufferedSize, input, len); 3272 state->bufferedSize += (XXH32_hash_t)len; 3273 return XXH_OK; 3274 } 3275 3276 { const xxh_u8* xinput = (const xxh_u8*)input; 3277 const xxh_u8* const bEnd = xinput + len; 3278 3279 if (state->bufferedSize) { /* non-empty buffer: complete first */ 3280 XXH_memcpy(state->buffer + state->bufferedSize, xinput, sizeof(state->buffer) - state->bufferedSize); 3281 xinput += sizeof(state->buffer) - state->bufferedSize; 3282 /* then process one round */ 3283 (void)XXH32_consumeLong(state->acc, state->buffer, sizeof(state->buffer), XXH_aligned); 3284 state->bufferedSize = 0; 3285 } 3286 3287 XXH_ASSERT(xinput <= bEnd); 3288 if ((size_t)(bEnd - xinput) >= sizeof(state->buffer)) { 3289 /* Process the remaining data */ 3290 xinput = XXH32_consumeLong(state->acc, xinput, (size_t)(bEnd - xinput), XXH_unaligned); 3291 } 3292 3293 if (xinput < bEnd) { 3294 /* Copy the leftover to the tmp buffer */ 3295 XXH_memcpy(state->buffer, xinput, (size_t)(bEnd-xinput)); 3296 state->bufferedSize = (unsigned)(bEnd-xinput); 3297 } 3298 } 3299 3300 return XXH_OK; 3301 } 3302 3303 3304 /*! @ingroup XXH32_family */ 3305 XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) 3306 { 3307 xxh_u32 h32; 3308 3309 if (state->large_len) { 3310 h32 = XXH32_mergeAccs(state->acc); 3311 } else { 3312 h32 = state->acc[2] /* == seed */ + XXH_PRIME32_5; 3313 } 3314 3315 h32 += state->total_len_32; 3316 3317 return XXH32_finalize(h32, state->buffer, state->bufferedSize, XXH_aligned); 3318 } 3319 #endif /* !XXH_NO_STREAM */ 3320 3321 /******* Canonical representation *******/ 3322 3323 /*! @ingroup XXH32_family */ 3324 XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) 3325 { 3326 XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); 3327 if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); 3328 XXH_memcpy(dst, &hash, sizeof(*dst)); 3329 } 3330 /*! @ingroup XXH32_family */ 3331 XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) 3332 { 3333 return XXH_readBE32(src); 3334 } 3335 3336 3337 #ifndef XXH_NO_LONG_LONG 3338 3339 /* ******************************************************************* 3340 * 64-bit hash functions 3341 *********************************************************************/ 3342 /*! 3343 * @} 3344 * @ingroup impl 3345 * @{ 3346 */ 3347 /******* Memory access *******/ 3348 3349 typedef XXH64_hash_t xxh_u64; 3350 3351 #ifdef XXH_OLD_NAMES 3352 # define U64 xxh_u64 3353 #endif 3354 3355 #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) 3356 /* 3357 * Manual byteshift. Best for old compilers which don't inline memcpy. 3358 * We actually directly use XXH_readLE64 and XXH_readBE64. 3359 */ 3360 #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) 3361 3362 /* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ 3363 static xxh_u64 XXH_read64(const void* memPtr) 3364 { 3365 return *(const xxh_u64*) memPtr; 3366 } 3367 3368 #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) 3369 3370 /* 3371 * __attribute__((aligned(1))) is supported by gcc and clang. Originally the 3372 * documentation claimed that it only increased the alignment, but actually it 3373 * can decrease it on gcc, clang, and icc: 3374 * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, 3375 * https://gcc.godbolt.org/z/xYez1j67Y. 3376 */ 3377 #ifdef XXH_OLD_NAMES 3378 typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((__packed__)) unalign64; 3379 #endif 3380 static xxh_u64 XXH_read64(const void* ptr) 3381 { 3382 typedef __attribute__((__aligned__(1))) __attribute__((__may_alias__)) xxh_u64 xxh_unalign64; 3383 return *((const xxh_unalign64*)ptr); 3384 } 3385 3386 #else 3387 3388 /* 3389 * Portable and safe solution. Generally efficient. 3390 * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html 3391 */ 3392 static xxh_u64 XXH_read64(const void* memPtr) 3393 { 3394 xxh_u64 val; 3395 XXH_memcpy(&val, memPtr, sizeof(val)); 3396 return val; 3397 } 3398 3399 #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ 3400 3401 #if defined(_MSC_VER) /* Visual Studio */ 3402 # define XXH_swap64 _byteswap_uint64 3403 #elif XXH_GCC_VERSION >= 403 3404 # define XXH_swap64 __builtin_bswap64 3405 #else 3406 static xxh_u64 XXH_swap64(xxh_u64 x) 3407 { 3408 return ((x << 56) & 0xff00000000000000ULL) | 3409 ((x << 40) & 0x00ff000000000000ULL) | 3410 ((x << 24) & 0x0000ff0000000000ULL) | 3411 ((x << 8) & 0x000000ff00000000ULL) | 3412 ((x >> 8) & 0x00000000ff000000ULL) | 3413 ((x >> 24) & 0x0000000000ff0000ULL) | 3414 ((x >> 40) & 0x000000000000ff00ULL) | 3415 ((x >> 56) & 0x00000000000000ffULL); 3416 } 3417 #endif 3418 3419 3420 /* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ 3421 #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) 3422 3423 XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) 3424 { 3425 const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; 3426 return bytePtr[0] 3427 | ((xxh_u64)bytePtr[1] << 8) 3428 | ((xxh_u64)bytePtr[2] << 16) 3429 | ((xxh_u64)bytePtr[3] << 24) 3430 | ((xxh_u64)bytePtr[4] << 32) 3431 | ((xxh_u64)bytePtr[5] << 40) 3432 | ((xxh_u64)bytePtr[6] << 48) 3433 | ((xxh_u64)bytePtr[7] << 56); 3434 } 3435 3436 XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) 3437 { 3438 const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; 3439 return bytePtr[7] 3440 | ((xxh_u64)bytePtr[6] << 8) 3441 | ((xxh_u64)bytePtr[5] << 16) 3442 | ((xxh_u64)bytePtr[4] << 24) 3443 | ((xxh_u64)bytePtr[3] << 32) 3444 | ((xxh_u64)bytePtr[2] << 40) 3445 | ((xxh_u64)bytePtr[1] << 48) 3446 | ((xxh_u64)bytePtr[0] << 56); 3447 } 3448 3449 #else 3450 XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) 3451 { 3452 return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); 3453 } 3454 3455 static xxh_u64 XXH_readBE64(const void* ptr) 3456 { 3457 return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); 3458 } 3459 #endif 3460 3461 XXH_FORCE_INLINE xxh_u64 3462 XXH_readLE64_align(const void* ptr, XXH_alignment align) 3463 { 3464 if (align==XXH_unaligned) 3465 return XXH_readLE64(ptr); 3466 else 3467 return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); 3468 } 3469 3470 3471 /******* xxh64 *******/ 3472 /*! 3473 * @} 3474 * @defgroup XXH64_impl XXH64 implementation 3475 * @ingroup impl 3476 * 3477 * Details on the XXH64 implementation. 3478 * @{ 3479 */ 3480 /* #define rather that static const, to be used as initializers */ 3481 #define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ 3482 #define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ 3483 #define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ 3484 #define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ 3485 #define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ 3486 3487 #ifdef XXH_OLD_NAMES 3488 # define PRIME64_1 XXH_PRIME64_1 3489 # define PRIME64_2 XXH_PRIME64_2 3490 # define PRIME64_3 XXH_PRIME64_3 3491 # define PRIME64_4 XXH_PRIME64_4 3492 # define PRIME64_5 XXH_PRIME64_5 3493 #endif 3494 3495 /*! @copydoc XXH32_round */ 3496 static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) 3497 { 3498 acc += input * XXH_PRIME64_2; 3499 acc = XXH_rotl64(acc, 31); 3500 acc *= XXH_PRIME64_1; 3501 #if (defined(__AVX512F__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) 3502 /* 3503 * DISABLE AUTOVECTORIZATION: 3504 * A compiler fence is used to prevent GCC and Clang from 3505 * autovectorizing the XXH64 loop (pragmas and attributes don't work for some 3506 * reason) without globally disabling AVX512. 3507 * 3508 * Autovectorization of XXH64 tends to be detrimental, 3509 * though the exact outcome may change depending on exact cpu and compiler version. 3510 * For information, it has been reported as detrimental for Skylake-X, 3511 * but possibly beneficial for Zen4. 3512 * 3513 * The default is to disable auto-vectorization, 3514 * but you can select to enable it instead using `XXH_ENABLE_AUTOVECTORIZE` build variable. 3515 */ 3516 XXH_COMPILER_GUARD(acc); 3517 #endif 3518 return acc; 3519 } 3520 3521 static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) 3522 { 3523 val = XXH64_round(0, val); 3524 acc ^= val; 3525 acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; 3526 return acc; 3527 } 3528 3529 /*! @copydoc XXH32_avalanche */ 3530 static xxh_u64 XXH64_avalanche(xxh_u64 hash) 3531 { 3532 hash ^= hash >> 33; 3533 hash *= XXH_PRIME64_2; 3534 hash ^= hash >> 29; 3535 hash *= XXH_PRIME64_3; 3536 hash ^= hash >> 32; 3537 return hash; 3538 } 3539 3540 3541 #define XXH_get64bits(p) XXH_readLE64_align(p, align) 3542 3543 /*! 3544 * @internal 3545 * @brief Sets up the initial accumulator state for XXH64(). 3546 */ 3547 XXH_FORCE_INLINE void 3548 XXH64_initAccs(xxh_u64 *acc, xxh_u64 seed) 3549 { 3550 XXH_ASSERT(acc != NULL); 3551 acc[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; 3552 acc[1] = seed + XXH_PRIME64_2; 3553 acc[2] = seed + 0; 3554 acc[3] = seed - XXH_PRIME64_1; 3555 } 3556 3557 /*! 3558 * @internal 3559 * @brief Consumes a block of data for XXH64(). 3560 * 3561 * @return the end input pointer. 3562 */ 3563 XXH_FORCE_INLINE const xxh_u8 * 3564 XXH64_consumeLong( 3565 xxh_u64 *XXH_RESTRICT acc, 3566 xxh_u8 const *XXH_RESTRICT input, 3567 size_t len, 3568 XXH_alignment align 3569 ) 3570 { 3571 const xxh_u8* const bEnd = input + len; 3572 const xxh_u8* const limit = bEnd - 31; 3573 XXH_ASSERT(acc != NULL); 3574 XXH_ASSERT(input != NULL); 3575 XXH_ASSERT(len >= 32); 3576 do { 3577 /* reroll on 32-bit */ 3578 if (sizeof(void *) < sizeof(xxh_u64)) { 3579 size_t i; 3580 for (i = 0; i < 4; i++) { 3581 acc[i] = XXH64_round(acc[i], XXH_get64bits(input)); 3582 input += 8; 3583 } 3584 } else { 3585 acc[0] = XXH64_round(acc[0], XXH_get64bits(input)); input += 8; 3586 acc[1] = XXH64_round(acc[1], XXH_get64bits(input)); input += 8; 3587 acc[2] = XXH64_round(acc[2], XXH_get64bits(input)); input += 8; 3588 acc[3] = XXH64_round(acc[3], XXH_get64bits(input)); input += 8; 3589 } 3590 } while (input < limit); 3591 3592 return input; 3593 } 3594 3595 /*! 3596 * @internal 3597 * @brief Merges the accumulator lanes together for XXH64() 3598 */ 3599 XXH_FORCE_INLINE XXH_PUREF xxh_u64 3600 XXH64_mergeAccs(const xxh_u64 *acc) 3601 { 3602 XXH_ASSERT(acc != NULL); 3603 { 3604 xxh_u64 h64 = XXH_rotl64(acc[0], 1) + XXH_rotl64(acc[1], 7) 3605 + XXH_rotl64(acc[2], 12) + XXH_rotl64(acc[3], 18); 3606 /* reroll on 32-bit */ 3607 if (sizeof(void *) < sizeof(xxh_u64)) { 3608 size_t i; 3609 for (i = 0; i < 4; i++) { 3610 h64 = XXH64_mergeRound(h64, acc[i]); 3611 } 3612 } else { 3613 h64 = XXH64_mergeRound(h64, acc[0]); 3614 h64 = XXH64_mergeRound(h64, acc[1]); 3615 h64 = XXH64_mergeRound(h64, acc[2]); 3616 h64 = XXH64_mergeRound(h64, acc[3]); 3617 } 3618 return h64; 3619 } 3620 } 3621 3622 /*! 3623 * @internal 3624 * @brief Processes the last 0-31 bytes of @p ptr. 3625 * 3626 * There may be up to 31 bytes remaining to consume from the input. 3627 * This final stage will digest them to ensure that all input bytes are present 3628 * in the final mix. 3629 * 3630 * @param hash The hash to finalize. 3631 * @param ptr The pointer to the remaining input. 3632 * @param len The remaining length, modulo 32. 3633 * @param align Whether @p ptr is aligned. 3634 * @return The finalized hash 3635 * @see XXH32_finalize(). 3636 */ 3637 XXH_STATIC XXH_PUREF xxh_u64 3638 XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) 3639 { 3640 if (ptr==NULL) XXH_ASSERT(len == 0); 3641 len &= 31; 3642 while (len >= 8) { 3643 xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); 3644 ptr += 8; 3645 hash ^= k1; 3646 hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4; 3647 len -= 8; 3648 } 3649 if (len >= 4) { 3650 hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; 3651 ptr += 4; 3652 hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; 3653 len -= 4; 3654 } 3655 while (len > 0) { 3656 hash ^= (*ptr++) * XXH_PRIME64_5; 3657 hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; 3658 --len; 3659 } 3660 return XXH64_avalanche(hash); 3661 } 3662 3663 #ifdef XXH_OLD_NAMES 3664 # define PROCESS1_64 XXH_PROCESS1_64 3665 # define PROCESS4_64 XXH_PROCESS4_64 3666 # define PROCESS8_64 XXH_PROCESS8_64 3667 #else 3668 # undef XXH_PROCESS1_64 3669 # undef XXH_PROCESS4_64 3670 # undef XXH_PROCESS8_64 3671 #endif 3672 3673 /*! 3674 * @internal 3675 * @brief The implementation for @ref XXH64(). 3676 * 3677 * @param input , len , seed Directly passed from @ref XXH64(). 3678 * @param align Whether @p input is aligned. 3679 * @return The calculated hash. 3680 */ 3681 XXH_FORCE_INLINE XXH_PUREF xxh_u64 3682 XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) 3683 { 3684 xxh_u64 h64; 3685 if (input==NULL) XXH_ASSERT(len == 0); 3686 3687 if (len>=32) { /* Process a large block of data */ 3688 xxh_u64 acc[4]; 3689 XXH64_initAccs(acc, seed); 3690 3691 input = XXH64_consumeLong(acc, input, len, align); 3692 3693 h64 = XXH64_mergeAccs(acc); 3694 } else { 3695 h64 = seed + XXH_PRIME64_5; 3696 } 3697 3698 h64 += (xxh_u64) len; 3699 3700 return XXH64_finalize(h64, input, len, align); 3701 } 3702 3703 3704 /*! @ingroup XXH64_family */ 3705 XXH_PUBLIC_API XXH64_hash_t XXH64 (XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) 3706 { 3707 #if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 3708 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ 3709 XXH64_state_t state; 3710 XXH64_reset(&state, seed); 3711 XXH64_update(&state, (const xxh_u8*)input, len); 3712 return XXH64_digest(&state); 3713 #else 3714 if (XXH_FORCE_ALIGN_CHECK) { 3715 if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ 3716 return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); 3717 } } 3718 3719 return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); 3720 3721 #endif 3722 } 3723 3724 /******* Hash Streaming *******/ 3725 #ifndef XXH_NO_STREAM 3726 /*! @ingroup XXH64_family*/ 3727 XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) 3728 { 3729 return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); 3730 } 3731 /*! @ingroup XXH64_family */ 3732 XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) 3733 { 3734 XXH_free(statePtr); 3735 return XXH_OK; 3736 } 3737 3738 /*! @ingroup XXH64_family */ 3739 XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState) 3740 { 3741 XXH_memcpy(dstState, srcState, sizeof(*dstState)); 3742 } 3743 3744 /*! @ingroup XXH64_family */ 3745 XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed) 3746 { 3747 XXH_ASSERT(statePtr != NULL); 3748 XXH_memset(statePtr, 0, sizeof(*statePtr)); 3749 XXH64_initAccs(statePtr->acc, seed); 3750 return XXH_OK; 3751 } 3752 3753 /*! @ingroup XXH64_family */ 3754 XXH_PUBLIC_API XXH_errorcode 3755 XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len) 3756 { 3757 if (input==NULL) { 3758 XXH_ASSERT(len == 0); 3759 return XXH_OK; 3760 } 3761 3762 state->total_len += len; 3763 3764 XXH_ASSERT(state->bufferedSize <= sizeof(state->buffer)); 3765 if (len < sizeof(state->buffer) - state->bufferedSize) { /* fill in tmp buffer */ 3766 XXH_memcpy(state->buffer + state->bufferedSize, input, len); 3767 state->bufferedSize += (XXH32_hash_t)len; 3768 return XXH_OK; 3769 } 3770 3771 { const xxh_u8* xinput = (const xxh_u8*)input; 3772 const xxh_u8* const bEnd = xinput + len; 3773 3774 if (state->bufferedSize) { /* non-empty buffer => complete first */ 3775 XXH_memcpy(state->buffer + state->bufferedSize, xinput, sizeof(state->buffer) - state->bufferedSize); 3776 xinput += sizeof(state->buffer) - state->bufferedSize; 3777 /* and process one round */ 3778 (void)XXH64_consumeLong(state->acc, state->buffer, sizeof(state->buffer), XXH_aligned); 3779 state->bufferedSize = 0; 3780 } 3781 3782 XXH_ASSERT(xinput <= bEnd); 3783 if ((size_t)(bEnd - xinput) >= sizeof(state->buffer)) { 3784 /* Process the remaining data */ 3785 xinput = XXH64_consumeLong(state->acc, xinput, (size_t)(bEnd - xinput), XXH_unaligned); 3786 } 3787 3788 if (xinput < bEnd) { 3789 /* Copy the leftover to the tmp buffer */ 3790 XXH_memcpy(state->buffer, xinput, (size_t)(bEnd-xinput)); 3791 state->bufferedSize = (unsigned)(bEnd-xinput); 3792 } 3793 } 3794 3795 return XXH_OK; 3796 } 3797 3798 3799 /*! @ingroup XXH64_family */ 3800 XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state) 3801 { 3802 xxh_u64 h64; 3803 3804 if (state->total_len >= 32) { 3805 h64 = XXH64_mergeAccs(state->acc); 3806 } else { 3807 h64 = state->acc[2] /*seed*/ + XXH_PRIME64_5; 3808 } 3809 3810 h64 += (xxh_u64) state->total_len; 3811 3812 return XXH64_finalize(h64, state->buffer, (size_t)state->total_len, XXH_aligned); 3813 } 3814 #endif /* !XXH_NO_STREAM */ 3815 3816 /******* Canonical representation *******/ 3817 3818 /*! @ingroup XXH64_family */ 3819 XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash) 3820 { 3821 XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); 3822 if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); 3823 XXH_memcpy(dst, &hash, sizeof(*dst)); 3824 } 3825 3826 /*! @ingroup XXH64_family */ 3827 XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src) 3828 { 3829 return XXH_readBE64(src); 3830 } 3831 3832 #ifndef XXH_NO_XXH3 3833 3834 /* ********************************************************************* 3835 * XXH3 3836 * New generation hash designed for speed on small keys and vectorization 3837 ************************************************************************ */ 3838 /*! 3839 * @} 3840 * @defgroup XXH3_impl XXH3 implementation 3841 * @ingroup impl 3842 * @{ 3843 */ 3844 3845 /* === Compiler specifics === */ 3846 3847 3848 #if (defined(__GNUC__) && (__GNUC__ >= 3)) \ 3849 || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ 3850 || defined(__clang__) 3851 # define XXH_likely(x) __builtin_expect(x, 1) 3852 # define XXH_unlikely(x) __builtin_expect(x, 0) 3853 #else 3854 # define XXH_likely(x) (x) 3855 # define XXH_unlikely(x) (x) 3856 #endif 3857 3858 #ifndef XXH_HAS_INCLUDE 3859 # ifdef __has_include 3860 /* 3861 * Not defined as XXH_HAS_INCLUDE(x) (function-like) because 3862 * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion) 3863 */ 3864 # define XXH_HAS_INCLUDE __has_include 3865 # else 3866 # define XXH_HAS_INCLUDE(x) 0 3867 # endif 3868 #endif 3869 3870 #if defined(__GNUC__) || defined(__clang__) 3871 # if defined(__ARM_FEATURE_SVE) 3872 # include <arm_sve.h> 3873 # endif 3874 # if defined(__ARM_NEON__) || defined(__ARM_NEON) \ 3875 || (defined(_M_ARM) && _M_ARM >= 7) \ 3876 || defined(_M_ARM64) || defined(_M_ARM64EC) \ 3877 || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* WASM SIMD128 via SIMDe */ 3878 # define inline __inline__ /* circumvent a clang bug */ 3879 # include <arm_neon.h> 3880 # undef inline 3881 # elif defined(__AVX2__) 3882 # include <immintrin.h> 3883 # elif defined(__SSE2__) 3884 # include <emmintrin.h> 3885 # elif defined(__loongarch_asx) 3886 # include <lasxintrin.h> 3887 # include <lsxintrin.h> 3888 # elif defined(__loongarch_sx) 3889 # include <lsxintrin.h> 3890 # elif defined(__riscv_vector) 3891 # include <riscv_vector.h> 3892 # endif 3893 #endif 3894 3895 #if defined(_MSC_VER) 3896 # include <intrin.h> 3897 #endif 3898 3899 /* 3900 * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while 3901 * remaining a true 64-bit/128-bit hash function. 3902 * 3903 * This is done by prioritizing a subset of 64-bit operations that can be 3904 * emulated without too many steps on the average 32-bit machine. 3905 * 3906 * For example, these two lines seem similar, and run equally fast on 64-bit: 3907 * 3908 * xxh_u64 x; 3909 * x ^= (x >> 47); // good 3910 * x ^= (x >> 13); // bad 3911 * 3912 * However, to a 32-bit machine, there is a major difference. 3913 * 3914 * x ^= (x >> 47) looks like this: 3915 * 3916 * x.lo ^= (x.hi >> (47 - 32)); 3917 * 3918 * while x ^= (x >> 13) looks like this: 3919 * 3920 * // note: funnel shifts are not usually cheap. 3921 * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); 3922 * x.hi ^= (x.hi >> 13); 3923 * 3924 * The first one is significantly faster than the second, simply because the 3925 * shift is larger than 32. This means: 3926 * - All the bits we need are in the upper 32 bits, so we can ignore the lower 3927 * 32 bits in the shift. 3928 * - The shift result will always fit in the lower 32 bits, and therefore, 3929 * we can ignore the upper 32 bits in the xor. 3930 * 3931 * Thanks to this optimization, XXH3 only requires these features to be efficient: 3932 * 3933 * - Usable unaligned access 3934 * - A 32-bit or 64-bit ALU 3935 * - If 32-bit, a decent ADC instruction 3936 * - A 32 or 64-bit multiply with a 64-bit result 3937 * - For the 128-bit variant, a decent byteswap helps short inputs. 3938 * 3939 * The first two are already required by XXH32, and almost all 32-bit and 64-bit 3940 * platforms which can run XXH32 can run XXH3 efficiently. 3941 * 3942 * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one 3943 * notable exception. 3944 * 3945 * First of all, Thumb-1 lacks support for the UMULL instruction which 3946 * performs the important long multiply. This means numerous __aeabi_lmul 3947 * calls. 3948 * 3949 * Second of all, the 8 functional registers are just not enough. 3950 * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need 3951 * Lo registers, and this shuffling results in thousands more MOVs than A32. 3952 * 3953 * A32 and T32 don't have this limitation. They can access all 14 registers, 3954 * do a 32->64 multiply with UMULL, and the flexible operand allowing free 3955 * shifts is helpful, too. 3956 * 3957 * Therefore, we do a quick sanity check. 3958 * 3959 * If compiling Thumb-1 for a target which supports ARM instructions, we will 3960 * emit a warning, as it is not a "sane" platform to compile for. 3961 * 3962 * Usually, if this happens, it is because of an accident and you probably need 3963 * to specify -march, as you likely meant to compile for a newer architecture. 3964 * 3965 * Credit: large sections of the vectorial and asm source code paths 3966 * have been contributed by @easyaspi314 3967 */ 3968 #if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) 3969 # warning "XXH3 is highly inefficient without ARM or Thumb-2." 3970 #endif 3971 3972 /* ========================================== 3973 * Vectorization detection 3974 * ========================================== */ 3975 3976 #ifdef XXH_DOXYGEN 3977 /*! 3978 * @ingroup tuning 3979 * @brief Overrides the vectorization implementation chosen for XXH3. 3980 * 3981 * Can be defined to 0 to disable SIMD, 3982 * or any other authorized value of @ref XXH_VECTOR. 3983 * 3984 * If this is not defined, it uses predefined macros to determine the best 3985 * implementation. 3986 */ 3987 # define XXH_VECTOR XXH_SCALAR 3988 /*! 3989 * @ingroup tuning 3990 * @brief Selects the minimum alignment for XXH3's accumulators. 3991 * 3992 * When using SIMD, this should match the alignment required for said vector 3993 * type, so, for example, 32 for AVX2. 3994 * 3995 * Default: Auto detected. 3996 */ 3997 # define XXH_ACC_ALIGN 8 3998 #endif 3999 4000 /* Actual definition */ 4001 #ifndef XXH_DOXYGEN 4002 #endif 4003 4004 #ifndef XXH_VECTOR /* can be defined on command line */ 4005 # if ( \ 4006 defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ 4007 || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ 4008 || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* wasm simd128 via SIMDe */ \ 4009 ) && ( \ 4010 defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ 4011 || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ 4012 ) 4013 # define XXH_VECTOR XXH_NEON 4014 # elif defined(__ARM_FEATURE_SVE) 4015 # define XXH_VECTOR XXH_SVE 4016 # elif defined(__AVX512F__) 4017 # define XXH_VECTOR XXH_AVX512 4018 # elif defined(__AVX2__) 4019 # define XXH_VECTOR XXH_AVX2 4020 # elif defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) 4021 # define XXH_VECTOR XXH_SSE2 4022 # elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ 4023 || (defined(__s390x__) && defined(__VEC__)) \ 4024 && defined(__GNUC__) /* TODO: IBM XL */ 4025 # define XXH_VECTOR XXH_VSX 4026 # elif defined(__loongarch_asx) 4027 # define XXH_VECTOR XXH_LASX 4028 # elif defined(__loongarch_sx) 4029 # define XXH_VECTOR XXH_LSX 4030 # elif defined(__riscv_vector) 4031 # define XXH_VECTOR XXH_RVV 4032 # else 4033 # define XXH_VECTOR XXH_SCALAR 4034 # endif 4035 #endif 4036 4037 /* __ARM_FEATURE_SVE is only supported by GCC & Clang. */ 4038 #if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE) 4039 # ifdef _MSC_VER 4040 # pragma warning(once : 4606) 4041 # else 4042 # warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead." 4043 # endif 4044 # undef XXH_VECTOR 4045 # define XXH_VECTOR XXH_SCALAR 4046 #endif 4047 4048 /* 4049 * Controls the alignment of the accumulator, 4050 * for compatibility with aligned vector loads, which are usually faster. 4051 */ 4052 #ifndef XXH_ACC_ALIGN 4053 # if defined(XXH_X86DISPATCH) 4054 # define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ 4055 # elif XXH_VECTOR == XXH_SCALAR /* scalar */ 4056 # define XXH_ACC_ALIGN 8 4057 # elif XXH_VECTOR == XXH_SSE2 /* sse2 */ 4058 # define XXH_ACC_ALIGN 16 4059 # elif XXH_VECTOR == XXH_AVX2 /* avx2 */ 4060 # define XXH_ACC_ALIGN 32 4061 # elif XXH_VECTOR == XXH_NEON /* neon */ 4062 # define XXH_ACC_ALIGN 16 4063 # elif XXH_VECTOR == XXH_VSX /* vsx */ 4064 # define XXH_ACC_ALIGN 16 4065 # elif XXH_VECTOR == XXH_AVX512 /* avx512 */ 4066 # define XXH_ACC_ALIGN 64 4067 # elif XXH_VECTOR == XXH_SVE /* sve */ 4068 # define XXH_ACC_ALIGN 64 4069 # elif XXH_VECTOR == XXH_LASX /* lasx */ 4070 # define XXH_ACC_ALIGN 64 4071 # elif XXH_VECTOR == XXH_LSX /* lsx */ 4072 # define XXH_ACC_ALIGN 64 4073 # elif XXH_VECTOR == XXH_RVV /* rvv */ 4074 # define XXH_ACC_ALIGN 64 /* could be 8, but 64 may be faster */ 4075 # endif 4076 #endif 4077 4078 #if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ 4079 || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 4080 # define XXH_SEC_ALIGN XXH_ACC_ALIGN 4081 #elif XXH_VECTOR == XXH_SVE 4082 # define XXH_SEC_ALIGN XXH_ACC_ALIGN 4083 #elif XXH_VECTOR == XXH_RVV 4084 # define XXH_SEC_ALIGN XXH_ACC_ALIGN 4085 #else 4086 # define XXH_SEC_ALIGN 8 4087 #endif 4088 4089 #if defined(__GNUC__) || defined(__clang__) 4090 # define XXH_ALIASING __attribute__((__may_alias__)) 4091 #else 4092 # define XXH_ALIASING /* nothing */ 4093 #endif 4094 4095 /* 4096 * UGLY HACK: 4097 * GCC usually generates the best code with -O3 for xxHash. 4098 * 4099 * However, when targeting AVX2, it is overzealous in its unrolling resulting 4100 * in code roughly 3/4 the speed of Clang. 4101 * 4102 * There are other issues, such as GCC splitting _mm256_loadu_si256 into 4103 * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which 4104 * only applies to Sandy and Ivy Bridge... which don't even support AVX2. 4105 * 4106 * That is why when compiling the AVX2 version, it is recommended to use either 4107 * -O2 -mavx2 -march=haswell 4108 * or 4109 * -O2 -mavx2 -mno-avx256-split-unaligned-load 4110 * for decent performance, or to use Clang instead. 4111 * 4112 * Fortunately, we can control the first one with a pragma that forces GCC into 4113 * -O2, but the other one we can't control without "failed to inline always 4114 * inline function due to target mismatch" warnings. 4115 */ 4116 #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ 4117 && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ 4118 && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ 4119 # pragma GCC push_options 4120 # pragma GCC optimize("-O2") 4121 #endif 4122 4123 #if XXH_VECTOR == XXH_NEON 4124 4125 /* 4126 * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3 4127 * optimizes out the entire hashLong loop because of the aliasing violation. 4128 * 4129 * However, GCC is also inefficient at load-store optimization with vld1q/vst1q, 4130 * so the only option is to mark it as aliasing. 4131 */ 4132 typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING; 4133 4134 /*! 4135 * @internal 4136 * @brief `vld1q_u64` but faster and alignment-safe. 4137 * 4138 * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only 4139 * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86). 4140 * 4141 * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it 4142 * prohibits load-store optimizations. Therefore, a direct dereference is used. 4143 * 4144 * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe 4145 * unaligned load. 4146 */ 4147 #if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) 4148 XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */ 4149 { 4150 return *(xxh_aliasing_uint64x2_t const *)ptr; 4151 } 4152 #else 4153 XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) 4154 { 4155 return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr)); 4156 } 4157 #endif 4158 4159 /*! 4160 * @internal 4161 * @brief `vmlal_u32` on low and high halves of a vector. 4162 * 4163 * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with 4164 * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32` 4165 * with `vmlal_u32`. 4166 */ 4167 #if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11 4168 XXH_FORCE_INLINE uint64x2_t 4169 XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) 4170 { 4171 /* Inline assembly is the only way */ 4172 __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs)); 4173 return acc; 4174 } 4175 XXH_FORCE_INLINE uint64x2_t 4176 XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) 4177 { 4178 /* This intrinsic works as expected */ 4179 return vmlal_high_u32(acc, lhs, rhs); 4180 } 4181 #else 4182 /* Portable intrinsic versions */ 4183 XXH_FORCE_INLINE uint64x2_t 4184 XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) 4185 { 4186 return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); 4187 } 4188 /*! @copydoc XXH_vmlal_low_u32 4189 * Assume the compiler converts this to vmlal_high_u32 on aarch64 */ 4190 XXH_FORCE_INLINE uint64x2_t 4191 XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) 4192 { 4193 return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); 4194 } 4195 #endif 4196 4197 /*! 4198 * @ingroup tuning 4199 * @brief Controls the NEON to scalar ratio for XXH3 4200 * 4201 * This can be set to 2, 4, 6, or 8. 4202 * 4203 * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used. 4204 * 4205 * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those 4206 * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU 4207 * bandwidth. 4208 * 4209 * This is even more noticeable on the more advanced cores like the Cortex-A76 which 4210 * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. 4211 * 4212 * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes 4213 * and 2 scalar lanes, which is chosen by default. 4214 * 4215 * This does not apply to Apple processors or 32-bit processors, which run better with 4216 * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes. 4217 * 4218 * This change benefits CPUs with large micro-op buffers without negatively affecting 4219 * most other CPUs: 4220 * 4221 * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | 4222 * |:----------------------|:--------------------|----------:|-----------:|------:| 4223 * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | 4224 * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | 4225 * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | 4226 * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% | 4227 * 4228 * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. 4229 * 4230 * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning 4231 * it effectively becomes worse 4. 4232 * 4233 * @see XXH3_accumulate_512_neon() 4234 */ 4235 # ifndef XXH3_NEON_LANES 4236 # if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ 4237 && !defined(__APPLE__) && XXH_SIZE_OPT <= 0 4238 # define XXH3_NEON_LANES 6 4239 # else 4240 # define XXH3_NEON_LANES XXH_ACC_NB 4241 # endif 4242 # endif 4243 #endif /* XXH_VECTOR == XXH_NEON */ 4244 4245 /* 4246 * VSX and Z Vector helpers. 4247 * 4248 * This is very messy, and any pull requests to clean this up are welcome. 4249 * 4250 * There are a lot of problems with supporting VSX and s390x, due to 4251 * inconsistent intrinsics, spotty coverage, and multiple endiannesses. 4252 */ 4253 #if XXH_VECTOR == XXH_VSX 4254 /* Annoyingly, these headers _may_ define three macros: `bool`, `vector`, 4255 * and `pixel`. This is a problem for obvious reasons. 4256 * 4257 * These keywords are unnecessary; the spec literally says they are 4258 * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd 4259 * after including the header. 4260 * 4261 * We use pragma push_macro/pop_macro to keep the namespace clean. */ 4262 # pragma push_macro("bool") 4263 # pragma push_macro("vector") 4264 # pragma push_macro("pixel") 4265 /* silence potential macro redefined warnings */ 4266 # undef bool 4267 # undef vector 4268 # undef pixel 4269 4270 # if defined(__s390x__) 4271 # include <s390intrin.h> 4272 # else 4273 # include <altivec.h> 4274 # endif 4275 4276 /* Restore the original macro values, if applicable. */ 4277 # pragma pop_macro("pixel") 4278 # pragma pop_macro("vector") 4279 # pragma pop_macro("bool") 4280 4281 typedef __vector unsigned long long xxh_u64x2; 4282 typedef __vector unsigned char xxh_u8x16; 4283 typedef __vector unsigned xxh_u32x4; 4284 4285 /* 4286 * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue. 4287 */ 4288 typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING; 4289 4290 # ifndef XXH_VSX_BE 4291 # if defined(__BIG_ENDIAN__) \ 4292 || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) 4293 # define XXH_VSX_BE 1 4294 # elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ 4295 # warning "-maltivec=be is not recommended. Please use native endianness." 4296 # define XXH_VSX_BE 1 4297 # else 4298 # define XXH_VSX_BE 0 4299 # endif 4300 # endif /* !defined(XXH_VSX_BE) */ 4301 4302 # if XXH_VSX_BE 4303 # if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) 4304 # define XXH_vec_revb vec_revb 4305 # else 4306 /*! 4307 * A polyfill for POWER9's vec_revb(). 4308 */ 4309 XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) 4310 { 4311 xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, 4312 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; 4313 return vec_perm(val, val, vByteSwap); 4314 } 4315 # endif 4316 # endif /* XXH_VSX_BE */ 4317 4318 /*! 4319 * Performs an unaligned vector load and byte swaps it on big endian. 4320 */ 4321 XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) 4322 { 4323 xxh_u64x2 ret; 4324 XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); 4325 # if XXH_VSX_BE 4326 ret = XXH_vec_revb(ret); 4327 # endif 4328 return ret; 4329 } 4330 4331 /* 4332 * vec_mulo and vec_mule are very problematic intrinsics on PowerPC 4333 * 4334 * These intrinsics weren't added until GCC 8, despite existing for a while, 4335 * and they are endian dependent. Also, their meaning swap depending on version. 4336 * */ 4337 # if defined(__s390x__) 4338 /* s390x is always big endian, no issue on this platform */ 4339 # define XXH_vec_mulo vec_mulo 4340 # define XXH_vec_mule vec_mule 4341 # elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__) 4342 /* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ 4343 /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */ 4344 # define XXH_vec_mulo __builtin_altivec_vmulouw 4345 # define XXH_vec_mule __builtin_altivec_vmuleuw 4346 # else 4347 /* gcc needs inline assembly */ 4348 /* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ 4349 XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) 4350 { 4351 xxh_u64x2 result; 4352 __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); 4353 return result; 4354 } 4355 XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) 4356 { 4357 xxh_u64x2 result; 4358 __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); 4359 return result; 4360 } 4361 # endif /* XXH_vec_mulo, XXH_vec_mule */ 4362 #endif /* XXH_VECTOR == XXH_VSX */ 4363 4364 #if XXH_VECTOR == XXH_SVE 4365 #define ACCRND(acc, offset) \ 4366 do { \ 4367 svuint64_t input_vec = svld1_u64(mask, xinput + offset); \ 4368 svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \ 4369 svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \ 4370 svuint64_t swapped = svtbl_u64(input_vec, kSwap); \ 4371 svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \ 4372 svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \ 4373 svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \ 4374 acc = svadd_u64_x(mask, acc, mul); \ 4375 } while (0) 4376 #endif /* XXH_VECTOR == XXH_SVE */ 4377 4378 /* prefetch 4379 * can be disabled, by declaring XXH_NO_PREFETCH build macro */ 4380 #if defined(XXH_NO_PREFETCH) 4381 # define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ 4382 #else 4383 # if XXH_SIZE_OPT >= 1 4384 # define XXH_PREFETCH(ptr) (void)(ptr) 4385 # elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ 4386 # include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ 4387 # define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) 4388 # elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) 4389 # define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) 4390 # else 4391 # define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ 4392 # endif 4393 #endif /* XXH_NO_PREFETCH */ 4394 4395 4396 /* ========================================== 4397 * XXH3 default settings 4398 * ========================================== */ 4399 4400 #define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ 4401 4402 #if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) 4403 # error "default keyset is not large enough" 4404 #endif 4405 4406 /*! 4407 * @internal 4408 * @def XXH3_kSecret 4409 * @brief Pseudorandom secret taken directly from FARSH. */ 4410 XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { 4411 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, 4412 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, 4413 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, 4414 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, 4415 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, 4416 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, 4417 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, 4418 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, 4419 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, 4420 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, 4421 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, 4422 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, 4423 }; 4424 4425 static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */ 4426 static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */ 4427 4428 #ifdef XXH_OLD_NAMES 4429 # define kSecret XXH3_kSecret 4430 #endif 4431 4432 #ifdef XXH_DOXYGEN 4433 /*! 4434 * @brief Calculates a 32-bit to 64-bit long multiply. 4435 * 4436 * Implemented as a macro. 4437 * 4438 * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't 4439 * need to (but it shouldn't need to anyways, it is about 7 instructions to do 4440 * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we 4441 * use that instead of the normal method. 4442 * 4443 * If you are compiling for platforms like Thumb-1 and don't have a better option, 4444 * you may also want to write your own long multiply routine here. 4445 * 4446 * @param x, y Numbers to be multiplied 4447 * @return 64-bit product of the low 32 bits of @p x and @p y. 4448 */ 4449 XXH_FORCE_INLINE xxh_u64 4450 XXH_mult32to64(xxh_u64 x, xxh_u64 y) 4451 { 4452 return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); 4453 } 4454 #elif defined(_MSC_VER) && defined(_M_IX86) 4455 # define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) 4456 #else 4457 /* 4458 * Downcast + upcast is usually better than masking on older compilers like 4459 * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. 4460 * 4461 * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands 4462 * and perform a full 64x64 multiply -- entirely redundant on 32-bit. 4463 */ 4464 # define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) 4465 #endif 4466 4467 /*! 4468 * @brief Calculates a 64->128-bit long multiply. 4469 * 4470 * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar 4471 * version. 4472 * 4473 * @param lhs , rhs The 64-bit integers to be multiplied 4474 * @return The 128-bit result represented in an @ref XXH128_hash_t. 4475 */ 4476 static XXH128_hash_t 4477 XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) 4478 { 4479 /* 4480 * GCC/Clang __uint128_t method. 4481 * 4482 * On most 64-bit targets, GCC and Clang define a __uint128_t type. 4483 * This is usually the best way as it usually uses a native long 64-bit 4484 * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. 4485 * 4486 * Usually. 4487 * 4488 * Despite being a 32-bit platform, Clang (and emscripten) define this type 4489 * despite not having the arithmetic for it. This results in a laggy 4490 * compiler builtin call which calculates a full 128-bit multiply. 4491 * In that case it is best to use the portable one. 4492 * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 4493 */ 4494 #if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ 4495 && defined(__SIZEOF_INT128__) \ 4496 || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) 4497 4498 __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; 4499 XXH128_hash_t r128; 4500 r128.low64 = (xxh_u64)(product); 4501 r128.high64 = (xxh_u64)(product >> 64); 4502 return r128; 4503 4504 /* 4505 * MSVC for x64's _umul128 method. 4506 * 4507 * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); 4508 * 4509 * This compiles to single operand MUL on x64. 4510 */ 4511 #elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) 4512 4513 #ifndef _MSC_VER 4514 # pragma intrinsic(_umul128) 4515 #endif 4516 xxh_u64 product_high; 4517 xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); 4518 XXH128_hash_t r128; 4519 r128.low64 = product_low; 4520 r128.high64 = product_high; 4521 return r128; 4522 4523 /* 4524 * MSVC for ARM64's __umulh method. 4525 * 4526 * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. 4527 */ 4528 #elif defined(_M_ARM64) || defined(_M_ARM64EC) 4529 4530 #ifndef _MSC_VER 4531 # pragma intrinsic(__umulh) 4532 #endif 4533 XXH128_hash_t r128; 4534 r128.low64 = lhs * rhs; 4535 r128.high64 = __umulh(lhs, rhs); 4536 return r128; 4537 4538 #else 4539 /* 4540 * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. 4541 * 4542 * This is a fast and simple grade school multiply, which is shown below 4543 * with base 10 arithmetic instead of base 0x100000000. 4544 * 4545 * 9 3 // D2 lhs = 93 4546 * x 7 5 // D2 rhs = 75 4547 * ---------- 4548 * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 4549 * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 4550 * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 4551 * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 4552 * --------- 4553 * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 4554 * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 4555 * --------- 4556 * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 4557 * 4558 * The reasons for adding the products like this are: 4559 * 1. It avoids manual carry tracking. Just like how 4560 * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. 4561 * This avoids a lot of complexity. 4562 * 4563 * 2. It hints for, and on Clang, compiles to, the powerful UMAAL 4564 * instruction available in ARM's Digital Signal Processing extension 4565 * in 32-bit ARMv6 and later, which is shown below: 4566 * 4567 * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) 4568 * { 4569 * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; 4570 * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); 4571 * *RdHi = (xxh_u32)(product >> 32); 4572 * } 4573 * 4574 * This instruction was designed for efficient long multiplication, and 4575 * allows this to be calculated in only 4 instructions at speeds 4576 * comparable to some 64-bit ALUs. 4577 * 4578 * 3. It isn't terrible on other platforms. Usually this will be a couple 4579 * of 32-bit ADD/ADCs. 4580 */ 4581 4582 /* First calculate all of the cross products. */ 4583 xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); 4584 xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); 4585 xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); 4586 xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); 4587 4588 /* Now add the products together. These will never overflow. */ 4589 xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; 4590 xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; 4591 xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); 4592 4593 XXH128_hash_t r128; 4594 r128.low64 = lower; 4595 r128.high64 = upper; 4596 return r128; 4597 #endif 4598 } 4599 4600 /*! 4601 * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. 4602 * 4603 * The reason for the separate function is to prevent passing too many structs 4604 * around by value. This will hopefully inline the multiply, but we don't force it. 4605 * 4606 * @param lhs , rhs The 64-bit integers to multiply 4607 * @return The low 64 bits of the product XOR'd by the high 64 bits. 4608 * @see XXH_mult64to128() 4609 */ 4610 static xxh_u64 4611 XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) 4612 { 4613 XXH128_hash_t product = XXH_mult64to128(lhs, rhs); 4614 return product.low64 ^ product.high64; 4615 } 4616 4617 /*! Seems to produce slightly better code on GCC for some reason. */ 4618 XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) 4619 { 4620 XXH_ASSERT(0 <= shift && shift < 64); 4621 return v64 ^ (v64 >> shift); 4622 } 4623 4624 /* 4625 * This is a fast avalanche stage, 4626 * suitable when input bits are already partially mixed 4627 */ 4628 static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) 4629 { 4630 h64 = XXH_xorshift64(h64, 37); 4631 h64 *= PRIME_MX1; 4632 h64 = XXH_xorshift64(h64, 32); 4633 return h64; 4634 } 4635 4636 /* 4637 * This is a stronger avalanche, 4638 * inspired by Pelle Evensen's rrmxmx 4639 * preferable when input has not been previously mixed 4640 */ 4641 static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) 4642 { 4643 /* this mix is inspired by Pelle Evensen's rrmxmx */ 4644 h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); 4645 h64 *= PRIME_MX2; 4646 h64 ^= (h64 >> 35) + len ; 4647 h64 *= PRIME_MX2; 4648 return XXH_xorshift64(h64, 28); 4649 } 4650 4651 4652 /* ========================================== 4653 * Short keys 4654 * ========================================== 4655 * One of the shortcomings of XXH32 and XXH64 was that their performance was 4656 * sub-optimal on short lengths. It used an iterative algorithm which strongly 4657 * favored lengths that were a multiple of 4 or 8. 4658 * 4659 * Instead of iterating over individual inputs, we use a set of single shot 4660 * functions which piece together a range of lengths and operate in constant time. 4661 * 4662 * Additionally, the number of multiplies has been significantly reduced. This 4663 * reduces latency, especially when emulating 64-bit multiplies on 32-bit. 4664 * 4665 * Depending on the platform, this may or may not be faster than XXH32, but it 4666 * is almost guaranteed to be faster than XXH64. 4667 */ 4668 4669 /* 4670 * At very short lengths, there isn't enough input to fully hide secrets, or use 4671 * the entire secret. 4672 * 4673 * There is also only a limited amount of mixing we can do before significantly 4674 * impacting performance. 4675 * 4676 * Therefore, we use different sections of the secret and always mix two secret 4677 * samples with an XOR. This should have no effect on performance on the 4678 * seedless or withSeed variants because everything _should_ be constant folded 4679 * by modern compilers. 4680 * 4681 * The XOR mixing hides individual parts of the secret and increases entropy. 4682 * 4683 * This adds an extra layer of strength for custom secrets. 4684 */ 4685 XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t 4686 XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) 4687 { 4688 XXH_ASSERT(input != NULL); 4689 XXH_ASSERT(1 <= len && len <= 3); 4690 XXH_ASSERT(secret != NULL); 4691 /* 4692 * len = 1: combined = { input[0], 0x01, input[0], input[0] } 4693 * len = 2: combined = { input[1], 0x02, input[0], input[1] } 4694 * len = 3: combined = { input[2], 0x03, input[0], input[1] } 4695 */ 4696 { xxh_u8 const c1 = input[0]; 4697 xxh_u8 const c2 = input[len >> 1]; 4698 xxh_u8 const c3 = input[len - 1]; 4699 xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) 4700 | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); 4701 xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; 4702 xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; 4703 return XXH64_avalanche(keyed); 4704 } 4705 } 4706 4707 XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t 4708 XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) 4709 { 4710 XXH_ASSERT(input != NULL); 4711 XXH_ASSERT(secret != NULL); 4712 XXH_ASSERT(4 <= len && len <= 8); 4713 seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; 4714 { xxh_u32 const input1 = XXH_readLE32(input); 4715 xxh_u32 const input2 = XXH_readLE32(input + len - 4); 4716 xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; 4717 xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); 4718 xxh_u64 const keyed = input64 ^ bitflip; 4719 return XXH3_rrmxmx(keyed, len); 4720 } 4721 } 4722 4723 XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t 4724 XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) 4725 { 4726 XXH_ASSERT(input != NULL); 4727 XXH_ASSERT(secret != NULL); 4728 XXH_ASSERT(9 <= len && len <= 16); 4729 { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; 4730 xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; 4731 xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; 4732 xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; 4733 xxh_u64 const acc = len 4734 + XXH_swap64(input_lo) + input_hi 4735 + XXH3_mul128_fold64(input_lo, input_hi); 4736 return XXH3_avalanche(acc); 4737 } 4738 } 4739 4740 XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t 4741 XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) 4742 { 4743 XXH_ASSERT(len <= 16); 4744 { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); 4745 if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); 4746 if (len) return XXH3_len_1to3_64b(input, len, secret, seed); 4747 return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); 4748 } 4749 } 4750 4751 /* 4752 * DISCLAIMER: the mid-size code path built on XXH3_mix16B() (lengths 17 to 240) 4753 * has known collision weaknesses. XXH3 is a non-cryptographic hash: collision 4754 * resistance is not one of its guarantees, and the seed (or custom secret) only 4755 * makes intentional collisions *harder*, it is not a security boundary. 4756 * These weaknesses are nonetheless documented here, so that users relying on a 4757 * seed for a low per-pair collision probability know what to expect. 4758 * 4759 * 1) XXH3_mul128_fold64() does not separate complemented multiplicands. 4760 * Modulo 2^128, and with s = a + b + 1 : 4761 * (~a) * (~b) == a*b + s - (s + 1) * 2^64 4762 * so the low half of the product gains `s` while the high half loses `s`. 4763 * Since carry chains are short, both halves happen to flip the same bits 4764 * fairly often, leaving their xor (the "fold") unchanged with probability 4765 * ~2^-27, instead of the ideal 2^-64. Complementing a single multiplicand 4766 * behaves the same way. 4767 * 4768 * Consequence for XXH3_64bits(): for any length in [32, 240], two messages 4769 * differing only by the complement of their first 8 (or first 16) bytes 4770 * collide with probability ~2^-27 over a uniformly random secret. The 4771 * _withSeed variant is also affected, but its collision probability depends 4772 * on the chosen pair because the 64-bit seed produces a structured secret. 4773 * A fixed pair has been measured to collide with probability ~2^-21 over 4774 * random seeds. This is an observed result, not a known upper bound. 4775 * 4776 * Consequence for XXH128(): XXH128_mix32B() mixes each 16-byte chunk a 4777 * second time as a raw 64-bit sum, which normally breaks the pattern above. 4778 * But that sum is *also* invariant under complementation when 4779 * w0 + w1 == 2^64 - 1, and under that additional constraint both output 4780 * halves collide simultaneously, again with probability ~2^-27 over a 4781 * uniformly random secret. The seeded variant is also affected, and its 4782 * exact probability can similarly depend on the chosen pair. 4783 * So the 128-bit variant is _not_ immune, contrarily to what was previously 4784 * believed, and its effective margin here (2^-27 vs. an ideal 2^-128) is 4785 * even thinner than the 64-bit one. 4786 * 4787 * Detailed analysis and measurements in 4788 * https://github.com/Cyan4973/xxHash/issues/1127 . 4789 * 4790 * 2) *seed-dependent* multicollisions, due to multiplication by zero: when one 4791 * of the two multiplicands below is zero, the whole 16-byte chunk stops 4792 * contributing to the hash, so all chunks sharing this property collide. 4793 * For random inputs, this is a 1 in 2^63 chance of 8 consecutive bytes 4794 * cancelling out, in a function invoked at most 16 times per hash, hence 4795 * essentially unreachable by accident. But it is trivially reachable by an 4796 * attacker who knows the secret (i.e. the unseeded variants). For 4797 * comparison, classic UMAC takes a 1 in 2^31 chance of 4 consecutive bytes 4798 * cancelling out the secret an arbitrary number of times (addressed for 4799 * long inputs in XXH3_accumulate_512). 4800 * 4801 * None of the above can be fixed without changing XXH3_mix16B(), which would 4802 * change every hash value in the [17, 240] range, and XXH3 output values are 4803 * now frozen. Therefore, if adversarial collision resistance matters for your 4804 * use case, use a cryptographic hash or a MAC instead. 4805 */ 4806 XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, 4807 const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) 4808 { 4809 #if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ 4810 && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ 4811 && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ 4812 /* 4813 * UGLY HACK: 4814 * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in 4815 * slower code. 4816 * 4817 * By forcing seed64 into a register, we disrupt the cost model and 4818 * cause it to scalarize. See `XXH32_round()` 4819 * 4820 * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, 4821 * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on 4822 * GCC 9.2, despite both emitting scalar code. 4823 * 4824 * GCC generates much better scalar code than Clang for the rest of XXH3, 4825 * which is why finding a more optimal codepath is an interest. 4826 */ 4827 XXH_COMPILER_GUARD(seed64); 4828 #endif 4829 { xxh_u64 const input_lo = XXH_readLE64(input); 4830 xxh_u64 const input_hi = XXH_readLE64(input+8); 4831 return XXH3_mul128_fold64( 4832 input_lo ^ (XXH_readLE64(secret) + seed64), 4833 input_hi ^ (XXH_readLE64(secret+8) - seed64) 4834 ); 4835 } 4836 } 4837 4838 /* For mid range keys, XXH3 uses a Mum-hash variant. */ 4839 XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t 4840 XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, 4841 const xxh_u8* XXH_RESTRICT secret, size_t secretSize, 4842 XXH64_hash_t seed) 4843 { 4844 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; 4845 XXH_ASSERT(16 < len && len <= 128); 4846 4847 { xxh_u64 acc = len * XXH_PRIME64_1; 4848 #if XXH_SIZE_OPT >= 1 4849 /* Smaller and cleaner, but slightly slower. */ 4850 unsigned int i = (unsigned int)(len - 1) / 32; 4851 do { 4852 acc += XXH3_mix16B(input+16 * i, secret+32*i, seed); 4853 acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed); 4854 } while (i-- != 0); 4855 #else 4856 if (len > 32) { 4857 if (len > 64) { 4858 if (len > 96) { 4859 acc += XXH3_mix16B(input+48, secret+96, seed); 4860 acc += XXH3_mix16B(input+len-64, secret+112, seed); 4861 } 4862 acc += XXH3_mix16B(input+32, secret+64, seed); 4863 acc += XXH3_mix16B(input+len-48, secret+80, seed); 4864 } 4865 acc += XXH3_mix16B(input+16, secret+32, seed); 4866 acc += XXH3_mix16B(input+len-32, secret+48, seed); 4867 } 4868 acc += XXH3_mix16B(input+0, secret+0, seed); 4869 acc += XXH3_mix16B(input+len-16, secret+16, seed); 4870 #endif 4871 return XXH3_avalanche(acc); 4872 } 4873 } 4874 4875 XXH_NO_INLINE XXH_PUREF XXH64_hash_t 4876 XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, 4877 const xxh_u8* XXH_RESTRICT secret, size_t secretSize, 4878 XXH64_hash_t seed) 4879 { 4880 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; 4881 XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); 4882 4883 #define XXH3_MIDSIZE_STARTOFFSET 3 4884 #define XXH3_MIDSIZE_LASTOFFSET 17 4885 4886 { xxh_u64 acc = len * XXH_PRIME64_1; 4887 xxh_u64 acc_end; 4888 unsigned int const nbRounds = (unsigned int)len / 16; 4889 unsigned int i; 4890 XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); 4891 for (i=0; i<8; i++) { 4892 acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); 4893 } 4894 /* last bytes */ 4895 acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); 4896 XXH_ASSERT(nbRounds >= 8); 4897 acc = XXH3_avalanche(acc); 4898 #if defined(__clang__) /* Clang */ \ 4899 && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ 4900 && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ 4901 /* 4902 * UGLY HACK: 4903 * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. 4904 * In everywhere else, it uses scalar code. 4905 * 4906 * For 64->128-bit multiplies, even if the NEON was 100% optimal, it 4907 * would still be slower than UMAAL (see XXH_mult64to128). 4908 * 4909 * Unfortunately, Clang doesn't handle the long multiplies properly and 4910 * converts them to the nonexistent "vmulq_u64" intrinsic, which is then 4911 * scalarized into an ugly mess of VMOV.32 instructions. 4912 * 4913 * This mess is difficult to avoid without turning autovectorization 4914 * off completely, but they are usually relatively minor and/or not 4915 * worth it to fix. 4916 * 4917 * This loop is the easiest to fix, as unlike XXH32, this pragma 4918 * _actually works_ because it is a loop vectorization instead of an 4919 * SLP vectorization. 4920 */ 4921 #pragma clang loop vectorize(disable) 4922 #endif 4923 for (i=8 ; i < nbRounds; i++) { 4924 /* 4925 * Prevents clang for unrolling the acc loop and interleaving with this one. 4926 */ 4927 XXH_COMPILER_GUARD(acc); 4928 acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); 4929 } 4930 return XXH3_avalanche(acc + acc_end); 4931 } 4932 } 4933 4934 4935 /* ======= Long Keys ======= */ 4936 4937 #define XXH_STRIPE_LEN 64 4938 #define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ 4939 #define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) 4940 4941 #ifdef XXH_OLD_NAMES 4942 # define STRIPE_LEN XXH_STRIPE_LEN 4943 # define ACC_NB XXH_ACC_NB 4944 #endif 4945 4946 #ifndef XXH_PREFETCH_DIST 4947 # ifdef __clang__ 4948 # define XXH_PREFETCH_DIST 320 4949 # else 4950 # if (XXH_VECTOR == XXH_AVX512) 4951 # define XXH_PREFETCH_DIST 512 4952 # else 4953 # define XXH_PREFETCH_DIST 384 4954 # endif 4955 # endif /* __clang__ */ 4956 #endif /* XXH_PREFETCH_DIST */ 4957 4958 /* 4959 * These macros are to generate an XXH3_accumulate() function. 4960 * The two arguments select the name suffix and target attribute. 4961 * 4962 * The name of this symbol is XXH3_accumulate_<name>() and it calls 4963 * XXH3_accumulate_512_<name>(). 4964 * 4965 * It may be useful to hand implement this function if the compiler fails to 4966 * optimize the inline function. 4967 */ 4968 #define XXH3_ACCUMULATE_TEMPLATE(name) \ 4969 void \ 4970 XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \ 4971 const xxh_u8* XXH_RESTRICT input, \ 4972 const xxh_u8* XXH_RESTRICT secret, \ 4973 size_t nbStripes) \ 4974 { \ 4975 size_t n; \ 4976 for (n = 0; n < nbStripes; n++ ) { \ 4977 const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \ 4978 XXH_PREFETCH(in + XXH_PREFETCH_DIST); \ 4979 XXH3_accumulate_512_##name( \ 4980 acc, \ 4981 in, \ 4982 secret + n*XXH_SECRET_CONSUME_RATE); \ 4983 } \ 4984 } 4985 4986 4987 XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) 4988 { 4989 if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); 4990 XXH_memcpy(dst, &v64, sizeof(v64)); 4991 } 4992 4993 /* Several intrinsic functions below are supposed to accept __int64 as argument, 4994 * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . 4995 * However, several environments do not define __int64 type, 4996 * requiring a workaround. 4997 */ 4998 #if !defined (__VMS) \ 4999 && (defined (__cplusplus) \ 5000 || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) 5001 typedef int64_t xxh_i64; 5002 #else 5003 /* the following type must have a width of 64-bit */ 5004 typedef long long xxh_i64; 5005 #endif 5006 5007 5008 /* 5009 * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. 5010 * 5011 * It is a hardened version of UMAC, based off of FARSH's implementation. 5012 * 5013 * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD 5014 * implementations, and it is ridiculously fast. 5015 * 5016 * We harden it by mixing the original input to the accumulators as well as the product. 5017 * 5018 * This means that in the (relatively likely) case of a multiply by zero, the 5019 * original input is preserved. 5020 * 5021 * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve 5022 * cross-pollination, as otherwise the upper and lower halves would be 5023 * essentially independent. 5024 * 5025 * This doesn't matter on 64-bit hashes since they all get merged together in 5026 * the end, so we skip the extra step. 5027 * 5028 * Both XXH3_64bits and XXH3_128bits use this subroutine. 5029 */ 5030 5031 #if (XXH_VECTOR == XXH_AVX512) \ 5032 || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) 5033 5034 #ifndef XXH_TARGET_AVX512 5035 # define XXH_TARGET_AVX512 /* disable attribute target */ 5036 #endif 5037 5038 XXH_FORCE_INLINE XXH_TARGET_AVX512 void 5039 XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, 5040 const void* XXH_RESTRICT input, 5041 const void* XXH_RESTRICT secret) 5042 { 5043 __m512i* const xacc = (__m512i *) acc; 5044 XXH_ASSERT((((size_t)acc) & 63) == 0); 5045 XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); 5046 5047 { 5048 /* data_vec = input[0]; */ 5049 __m512i const data_vec = _mm512_loadu_si512 (input); 5050 /* key_vec = secret[0]; */ 5051 __m512i const key_vec = _mm512_loadu_si512 (secret); 5052 /* data_key = data_vec ^ key_vec; */ 5053 __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); 5054 /* data_key_lo = data_key >> 32; */ 5055 __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32); 5056 /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ 5057 __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); 5058 /* xacc[0] += swap(data_vec); */ 5059 __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); 5060 __m512i const sum = _mm512_add_epi64(*xacc, data_swap); 5061 /* xacc[0] += product; */ 5062 *xacc = _mm512_add_epi64(product, sum); 5063 } 5064 } 5065 XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512) 5066 5067 /* 5068 * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. 5069 * 5070 * Multiplication isn't perfect, as explained by Google in HighwayHash: 5071 * 5072 * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to 5073 * // varying degrees. In descending order of goodness, bytes 5074 * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. 5075 * // As expected, the upper and lower bytes are much worse. 5076 * 5077 * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 5078 * 5079 * Since our algorithm uses a pseudorandom secret to add some variance into the 5080 * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. 5081 * 5082 * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid 5083 * extraction. 5084 * 5085 * Both XXH3_64bits and XXH3_128bits use this subroutine. 5086 */ 5087 5088 XXH_FORCE_INLINE XXH_TARGET_AVX512 void 5089 XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) 5090 { 5091 XXH_ASSERT((((size_t)acc) & 63) == 0); 5092 XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); 5093 { __m512i* const xacc = (__m512i*) acc; 5094 const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); 5095 5096 /* xacc[0] ^= (xacc[0] >> 47) */ 5097 __m512i const acc_vec = *xacc; 5098 __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); 5099 /* xacc[0] ^= secret; */ 5100 __m512i const key_vec = _mm512_loadu_si512 (secret); 5101 __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */); 5102 5103 /* xacc[0] *= XXH_PRIME32_1; */ 5104 __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32); 5105 __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); 5106 __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); 5107 *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); 5108 } 5109 } 5110 5111 XXH_FORCE_INLINE XXH_TARGET_AVX512 void 5112 XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) 5113 { 5114 XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); 5115 XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); 5116 XXH_ASSERT(((size_t)customSecret & 63) == 0); 5117 (void)(&XXH_writeLE64); 5118 { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); 5119 __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64); 5120 __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_setzero_si512(), seed_pos); 5121 5122 const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); 5123 __m512i* const dest = ( __m512i*) customSecret; 5124 int i; 5125 XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ 5126 XXH_ASSERT(((size_t)dest & 63) == 0); 5127 for (i=0; i < nbRounds; ++i) { 5128 dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed); 5129 } } 5130 } 5131 5132 #endif 5133 5134 #if (XXH_VECTOR == XXH_AVX2) \ 5135 || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) 5136 5137 #ifndef XXH_TARGET_AVX2 5138 # define XXH_TARGET_AVX2 /* disable attribute target */ 5139 #endif 5140 5141 XXH_FORCE_INLINE XXH_TARGET_AVX2 void 5142 XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, 5143 const void* XXH_RESTRICT input, 5144 const void* XXH_RESTRICT secret) 5145 { 5146 XXH_ASSERT((((size_t)acc) & 31) == 0); 5147 { __m256i* const xacc = (__m256i *) acc; 5148 /* Unaligned. This is mainly for pointer arithmetic, and because 5149 * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ 5150 const __m256i* const xinput = (const __m256i *) input; 5151 /* Unaligned. This is mainly for pointer arithmetic, and because 5152 * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ 5153 const __m256i* const xsecret = (const __m256i *) secret; 5154 5155 size_t i; 5156 for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { 5157 /* data_vec = xinput[i]; */ 5158 __m256i const data_vec = _mm256_loadu_si256 (xinput+i); 5159 /* key_vec = xsecret[i]; */ 5160 __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); 5161 /* data_key = data_vec ^ key_vec; */ 5162 __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); 5163 /* data_key_lo = data_key >> 32; */ 5164 __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32); 5165 /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ 5166 __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); 5167 /* xacc[i] += swap(data_vec); */ 5168 __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); 5169 __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); 5170 /* xacc[i] += product; */ 5171 xacc[i] = _mm256_add_epi64(product, sum); 5172 } } 5173 } 5174 XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2) 5175 5176 XXH_FORCE_INLINE XXH_TARGET_AVX2 void 5177 XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) 5178 { 5179 XXH_ASSERT((((size_t)acc) & 31) == 0); 5180 { __m256i* const xacc = (__m256i*) acc; 5181 /* Unaligned. This is mainly for pointer arithmetic, and because 5182 * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ 5183 const __m256i* const xsecret = (const __m256i *) secret; 5184 const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); 5185 5186 size_t i; 5187 for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { 5188 /* xacc[i] ^= (xacc[i] >> 47) */ 5189 __m256i const acc_vec = xacc[i]; 5190 __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); 5191 __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); 5192 /* xacc[i] ^= xsecret; */ 5193 __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); 5194 __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); 5195 5196 /* xacc[i] *= XXH_PRIME32_1; */ 5197 __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32); 5198 __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); 5199 __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); 5200 xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); 5201 } 5202 } 5203 } 5204 5205 XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) 5206 { 5207 XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); 5208 XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); 5209 XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); 5210 (void)(&XXH_writeLE64); 5211 XXH_PREFETCH(customSecret); 5212 { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); 5213 5214 const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); 5215 __m256i* dest = ( __m256i*) customSecret; 5216 5217 # if defined(__GNUC__) || defined(__clang__) 5218 /* 5219 * On GCC & Clang, marking 'dest' as modified will cause the compiler: 5220 * - do not extract the secret from sse registers in the internal loop 5221 * - use less common registers, and avoid pushing these reg into stack 5222 */ 5223 XXH_COMPILER_GUARD(dest); 5224 # endif 5225 XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ 5226 XXH_ASSERT(((size_t)dest & 31) == 0); 5227 5228 /* GCC -O2 need unroll loop manually */ 5229 dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed); 5230 dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed); 5231 dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed); 5232 dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed); 5233 dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed); 5234 dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed); 5235 } 5236 } 5237 5238 #endif 5239 5240 /* x86dispatch always generates SSE2 */ 5241 #if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) 5242 5243 #ifndef XXH_TARGET_SSE2 5244 # define XXH_TARGET_SSE2 /* disable attribute target */ 5245 #endif 5246 5247 XXH_FORCE_INLINE XXH_TARGET_SSE2 void 5248 XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, 5249 const void* XXH_RESTRICT input, 5250 const void* XXH_RESTRICT secret) 5251 { 5252 /* SSE2 is just a half-scale version of the AVX2 version. */ 5253 XXH_ASSERT((((size_t)acc) & 15) == 0); 5254 { __m128i* const xacc = (__m128i *) acc; 5255 /* Unaligned. This is mainly for pointer arithmetic, and because 5256 * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ 5257 const __m128i* const xinput = (const __m128i *) input; 5258 /* Unaligned. This is mainly for pointer arithmetic, and because 5259 * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ 5260 const __m128i* const xsecret = (const __m128i *) secret; 5261 5262 size_t i; 5263 for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { 5264 /* data_vec = xinput[i]; */ 5265 __m128i const data_vec = _mm_loadu_si128 (xinput+i); 5266 /* key_vec = xsecret[i]; */ 5267 __m128i const key_vec = _mm_loadu_si128 (xsecret+i); 5268 /* data_key = data_vec ^ key_vec; */ 5269 __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); 5270 /* data_key_lo = data_key >> 32; */ 5271 __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); 5272 /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ 5273 __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); 5274 /* xacc[i] += swap(data_vec); */ 5275 __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); 5276 __m128i const sum = _mm_add_epi64(xacc[i], data_swap); 5277 /* xacc[i] += product; */ 5278 xacc[i] = _mm_add_epi64(product, sum); 5279 } } 5280 } 5281 XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2) 5282 5283 XXH_FORCE_INLINE XXH_TARGET_SSE2 void 5284 XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) 5285 { 5286 XXH_ASSERT((((size_t)acc) & 15) == 0); 5287 { __m128i* const xacc = (__m128i*) acc; 5288 /* Unaligned. This is mainly for pointer arithmetic, and because 5289 * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ 5290 const __m128i* const xsecret = (const __m128i *) secret; 5291 const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); 5292 5293 size_t i; 5294 for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { 5295 /* xacc[i] ^= (xacc[i] >> 47) */ 5296 __m128i const acc_vec = xacc[i]; 5297 __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); 5298 __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); 5299 /* xacc[i] ^= xsecret[i]; */ 5300 __m128i const key_vec = _mm_loadu_si128 (xsecret+i); 5301 __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); 5302 5303 /* xacc[i] *= XXH_PRIME32_1; */ 5304 __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); 5305 __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); 5306 __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); 5307 xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); 5308 } 5309 } 5310 } 5311 5312 XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) 5313 { 5314 XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); 5315 (void)(&XXH_writeLE64); 5316 { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); 5317 5318 # if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER <= 1900 5319 /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 5320 * and some specific variants of 2015 may also lack it */ 5321 /* Cast to unsigned 64-bit first to avoid signed arithmetic issues */ 5322 xxh_u64 const seed64_unsigned = (xxh_u64)seed64; 5323 xxh_u64 const neg_seed64 = (xxh_u64)(0ULL - seed64_unsigned); 5324 __m128i const seed = _mm_set_epi32( 5325 (int)(neg_seed64 >> 32), /* high 32 bits of negated seed */ 5326 (int)(neg_seed64), /* low 32 bits of negated seed */ 5327 (int)(seed64_unsigned >> 32), /* high 32 bits of original seed */ 5328 (int)(seed64_unsigned) /* low 32 bits of original seed */ 5329 ); 5330 # else 5331 __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); 5332 # endif 5333 int i; 5334 5335 const void* const src16 = XXH3_kSecret; 5336 __m128i* dst16 = (__m128i*) customSecret; 5337 # if defined(__GNUC__) || defined(__clang__) 5338 /* 5339 * On GCC & Clang, marking 'dest' as modified will cause the compiler: 5340 * - do not extract the secret from sse registers in the internal loop 5341 * - use less common registers, and avoid pushing these reg into stack 5342 */ 5343 XXH_COMPILER_GUARD(dst16); 5344 # endif 5345 XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ 5346 XXH_ASSERT(((size_t)dst16 & 15) == 0); 5347 5348 for (i=0; i < nbRounds; ++i) { 5349 dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); 5350 } } 5351 } 5352 5353 #endif 5354 5355 #if (XXH_VECTOR == XXH_NEON) 5356 5357 /* forward declarations for the scalar routines */ 5358 XXH_FORCE_INLINE void 5359 XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, 5360 void const* XXH_RESTRICT secret, size_t lane); 5361 5362 XXH_FORCE_INLINE void 5363 XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, 5364 void const* XXH_RESTRICT secret, size_t lane); 5365 5366 /*! 5367 * @internal 5368 * @brief The bulk processing loop for NEON and WASM SIMD128. 5369 * 5370 * The NEON code path is actually partially scalar when running on AArch64. This 5371 * is to optimize the pipelining and can have up to 15% speedup depending on the 5372 * CPU, and it also mitigates some GCC codegen issues. 5373 * 5374 * @see XXH3_NEON_LANES for configuring this and details about this optimization. 5375 * 5376 * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit 5377 * integers instead of the other platforms which mask full 64-bit vectors, 5378 * so the setup is more complicated than just shifting right. 5379 * 5380 * Additionally, there is an optimization for 4 lanes at once noted below. 5381 * 5382 * Since, as stated, the most optimal amount of lanes for Cortexes is 6, 5383 * there needs to be *three* versions of the accumulate operation used 5384 * for the remaining 2 lanes. 5385 * 5386 * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap 5387 * nearly perfectly. 5388 */ 5389 5390 XXH_FORCE_INLINE void 5391 XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, 5392 const void* XXH_RESTRICT input, 5393 const void* XXH_RESTRICT secret) 5394 { 5395 XXH_ASSERT((((size_t)acc) & 15) == 0); 5396 XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); 5397 { /* GCC for darwin arm64 does not like aliasing here */ 5398 xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc; 5399 /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ 5400 uint8_t const* xinput = (const uint8_t *) input; 5401 uint8_t const* xsecret = (const uint8_t *) secret; 5402 5403 size_t i; 5404 #ifdef __wasm_simd128__ 5405 /* 5406 * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret 5407 * is constant propagated, which results in it converting it to this 5408 * inside the loop: 5409 * 5410 * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0) 5411 * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0) 5412 * ... 5413 * 5414 * This requires a full 32-bit address immediate (and therefore a 6 byte 5415 * instruction) as well as an add for each offset. 5416 * 5417 * Putting an asm guard prevents it from folding (at the cost of losing 5418 * the alignment hint), and uses the free offset in `v128.load` instead 5419 * of adding secret_offset each time which overall reduces code size by 5420 * about a kilobyte and improves performance. 5421 */ 5422 XXH_COMPILER_GUARD(xsecret); 5423 #endif 5424 /* Scalar lanes use the normal scalarRound routine */ 5425 for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { 5426 XXH3_scalarRound(acc, input, secret, i); 5427 } 5428 i = 0; 5429 /* 4 NEON lanes at a time. */ 5430 for (; i+1 < XXH3_NEON_LANES / 2; i+=2) { 5431 /* data_vec = xinput[i]; */ 5432 uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16)); 5433 uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16)); 5434 /* key_vec = xsecret[i]; */ 5435 uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16)); 5436 uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16)); 5437 /* data_swap = swap(data_vec) */ 5438 uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); 5439 uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); 5440 /* data_key = data_vec ^ key_vec; */ 5441 uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); 5442 uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); 5443 5444 /* 5445 * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a 5446 * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to 5447 * get one vector with the low 32 bits of each lane, and one vector 5448 * with the high 32 bits of each lane. 5449 * 5450 * The intrinsic returns a double vector because the original ARMv7-a 5451 * instruction modified both arguments in place. AArch64 and SIMD128 emit 5452 * two instructions from this intrinsic. 5453 * 5454 * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ] 5455 * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ] 5456 */ 5457 uint32x4x2_t unzipped = vuzpq_u32( 5458 vreinterpretq_u32_u64(data_key_1), 5459 vreinterpretq_u32_u64(data_key_2) 5460 ); 5461 /* data_key_lo = data_key & 0xFFFFFFFF */ 5462 uint32x4_t data_key_lo = unzipped.val[0]; 5463 /* data_key_hi = data_key >> 32 */ 5464 uint32x4_t data_key_hi = unzipped.val[1]; 5465 /* 5466 * Then, we can split the vectors horizontally and multiply which, as for most 5467 * widening intrinsics, have a variant that works on both high half vectors 5468 * for free on AArch64. A similar instruction is available on SIMD128. 5469 * 5470 * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi 5471 */ 5472 uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); 5473 uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); 5474 /* 5475 * Clang reorders 5476 * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s 5477 * c += a; // add acc.2d, acc.2d, swap.2d 5478 * to 5479 * c += a; // add acc.2d, acc.2d, swap.2d 5480 * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s 5481 * 5482 * While it would make sense in theory since the addition is faster, 5483 * for reasons likely related to umlal being limited to certain NEON 5484 * pipelines, this is worse. A compiler guard fixes this. 5485 */ 5486 XXH_COMPILER_GUARD_CLANG_NEON(sum_1); 5487 XXH_COMPILER_GUARD_CLANG_NEON(sum_2); 5488 /* xacc[i] = acc_vec + sum; */ 5489 xacc[i] = vaddq_u64(xacc[i], sum_1); 5490 xacc[i+1] = vaddq_u64(xacc[i+1], sum_2); 5491 } 5492 /* Operate on the remaining NEON lanes 2 at a time. */ 5493 for (; i < XXH3_NEON_LANES / 2; i++) { 5494 /* data_vec = xinput[i]; */ 5495 uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16)); 5496 /* key_vec = xsecret[i]; */ 5497 uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); 5498 /* acc_vec_2 = swap(data_vec) */ 5499 uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1); 5500 /* data_key = data_vec ^ key_vec; */ 5501 uint64x2_t data_key = veorq_u64(data_vec, key_vec); 5502 /* For two lanes, just use VMOVN and VSHRN. */ 5503 /* data_key_lo = data_key & 0xFFFFFFFF; */ 5504 uint32x2_t data_key_lo = vmovn_u64(data_key); 5505 /* data_key_hi = data_key >> 32; */ 5506 uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32); 5507 /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */ 5508 uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi); 5509 /* Same Clang workaround as before */ 5510 XXH_COMPILER_GUARD_CLANG_NEON(sum); 5511 /* xacc[i] = acc_vec + sum; */ 5512 xacc[i] = vaddq_u64 (xacc[i], sum); 5513 } 5514 } 5515 } 5516 XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon) 5517 5518 XXH_FORCE_INLINE void 5519 XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) 5520 { 5521 XXH_ASSERT((((size_t)acc) & 15) == 0); 5522 5523 { xxh_aliasing_uint64x2_t* xacc = (xxh_aliasing_uint64x2_t*) acc; 5524 uint8_t const* xsecret = (uint8_t const*) secret; 5525 5526 size_t i; 5527 /* WASM uses operator overloads and doesn't need these. */ 5528 #ifndef __wasm_simd128__ 5529 /* { prime32_1, prime32_1 } */ 5530 uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1); 5531 /* { 0, prime32_1, 0, prime32_1 } */ 5532 uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32)); 5533 #endif 5534 5535 /* AArch64 uses both scalar and neon at the same time */ 5536 for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { 5537 XXH3_scalarScrambleRound(acc, secret, i); 5538 } 5539 for (i=0; i < XXH3_NEON_LANES / 2; i++) { 5540 /* xacc[i] ^= (xacc[i] >> 47); */ 5541 uint64x2_t acc_vec = xacc[i]; 5542 uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); 5543 uint64x2_t data_vec = veorq_u64(acc_vec, shifted); 5544 5545 /* xacc[i] ^= xsecret[i]; */ 5546 uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); 5547 uint64x2_t data_key = veorq_u64(data_vec, key_vec); 5548 /* xacc[i] *= XXH_PRIME32_1 */ 5549 #ifdef __wasm_simd128__ 5550 /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */ 5551 xacc[i] = data_key * XXH_PRIME32_1; 5552 #else 5553 /* 5554 * Expanded version with portable NEON intrinsics 5555 * 5556 * lo(x) * lo(y) + (hi(x) * lo(y) << 32) 5557 * 5558 * prod_hi = hi(data_key) * lo(prime) << 32 5559 * 5560 * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector 5561 * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits 5562 * and avoid the shift. 5563 */ 5564 uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi); 5565 /* Extract low bits for vmlal_u32 */ 5566 uint32x2_t data_key_lo = vmovn_u64(data_key); 5567 /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */ 5568 xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo); 5569 #endif 5570 } 5571 } 5572 } 5573 #endif 5574 5575 #if (XXH_VECTOR == XXH_VSX) 5576 5577 XXH_FORCE_INLINE void 5578 XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, 5579 const void* XXH_RESTRICT input, 5580 const void* XXH_RESTRICT secret) 5581 { 5582 /* presumed aligned */ 5583 xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; 5584 xxh_u8 const* const xinput = (xxh_u8 const*) input; /* no alignment restriction */ 5585 xxh_u8 const* const xsecret = (xxh_u8 const*) secret; /* no alignment restriction */ 5586 xxh_u64x2 const v32 = { 32, 32 }; 5587 size_t i; 5588 for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { 5589 /* data_vec = xinput[i]; */ 5590 xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i); 5591 /* key_vec = xsecret[i]; */ 5592 xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); 5593 xxh_u64x2 const data_key = data_vec ^ key_vec; 5594 /* shuffled = (data_key << 32) | (data_key >> 32); */ 5595 xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); 5596 /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ 5597 xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); 5598 /* acc_vec = xacc[i]; */ 5599 xxh_u64x2 acc_vec = xacc[i]; 5600 acc_vec += product; 5601 5602 /* swap high and low halves */ 5603 #ifdef __s390x__ 5604 acc_vec += vec_permi(data_vec, data_vec, 2); 5605 #else 5606 acc_vec += vec_xxpermdi(data_vec, data_vec, 2); 5607 #endif 5608 xacc[i] = acc_vec; 5609 } 5610 } 5611 XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx) 5612 5613 XXH_FORCE_INLINE void 5614 XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) 5615 { 5616 XXH_ASSERT((((size_t)acc) & 15) == 0); 5617 5618 { xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; 5619 const xxh_u8* const xsecret = (const xxh_u8*) secret; 5620 /* constants */ 5621 xxh_u64x2 const v32 = { 32, 32 }; 5622 xxh_u64x2 const v47 = { 47, 47 }; 5623 xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; 5624 size_t i; 5625 for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { 5626 /* xacc[i] ^= (xacc[i] >> 47); */ 5627 xxh_u64x2 const acc_vec = xacc[i]; 5628 xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); 5629 5630 /* xacc[i] ^= xsecret[i]; */ 5631 xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); 5632 xxh_u64x2 const data_key = data_vec ^ key_vec; 5633 5634 /* xacc[i] *= XXH_PRIME32_1 */ 5635 /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ 5636 xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); 5637 /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ 5638 xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); 5639 xacc[i] = prod_odd + (prod_even << v32); 5640 } } 5641 } 5642 5643 #endif 5644 5645 #if (XXH_VECTOR == XXH_SVE) 5646 5647 XXH_FORCE_INLINE void 5648 XXH3_accumulate_512_sve( void* XXH_RESTRICT acc, 5649 const void* XXH_RESTRICT input, 5650 const void* XXH_RESTRICT secret) 5651 { 5652 uint64_t *xacc = (uint64_t *)acc; 5653 const uint64_t *xinput = (const uint64_t *)(const void *)input; 5654 const uint64_t *xsecret = (const uint64_t *)(const void *)secret; 5655 svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); 5656 uint64_t element_count = svcntd(); 5657 if (element_count >= 8) { 5658 svbool_t mask = svptrue_pat_b64(SV_VL8); 5659 svuint64_t vacc = svld1_u64(mask, xacc); 5660 ACCRND(vacc, 0); 5661 svst1_u64(mask, xacc, vacc); 5662 } else if (element_count == 2) { /* sve128 */ 5663 svbool_t mask = svptrue_pat_b64(SV_VL2); 5664 svuint64_t acc0 = svld1_u64(mask, xacc + 0); 5665 svuint64_t acc1 = svld1_u64(mask, xacc + 2); 5666 svuint64_t acc2 = svld1_u64(mask, xacc + 4); 5667 svuint64_t acc3 = svld1_u64(mask, xacc + 6); 5668 ACCRND(acc0, 0); 5669 ACCRND(acc1, 2); 5670 ACCRND(acc2, 4); 5671 ACCRND(acc3, 6); 5672 svst1_u64(mask, xacc + 0, acc0); 5673 svst1_u64(mask, xacc + 2, acc1); 5674 svst1_u64(mask, xacc + 4, acc2); 5675 svst1_u64(mask, xacc + 6, acc3); 5676 } else { 5677 svbool_t mask = svptrue_pat_b64(SV_VL4); 5678 svuint64_t acc0 = svld1_u64(mask, xacc + 0); 5679 svuint64_t acc1 = svld1_u64(mask, xacc + 4); 5680 ACCRND(acc0, 0); 5681 ACCRND(acc1, 4); 5682 svst1_u64(mask, xacc + 0, acc0); 5683 svst1_u64(mask, xacc + 4, acc1); 5684 } 5685 } 5686 5687 XXH_FORCE_INLINE void 5688 XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc, 5689 const xxh_u8* XXH_RESTRICT input, 5690 const xxh_u8* XXH_RESTRICT secret, 5691 size_t nbStripes) 5692 { 5693 if (nbStripes != 0) { 5694 uint64_t *xacc = (uint64_t *)acc; 5695 const uint64_t *xinput = (const uint64_t *)(const void *)input; 5696 const uint64_t *xsecret = (const uint64_t *)(const void *)secret; 5697 svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); 5698 uint64_t element_count = svcntd(); 5699 if (element_count >= 8) { 5700 svbool_t mask = svptrue_pat_b64(SV_VL8); 5701 svuint64_t vacc = svld1_u64(mask, xacc + 0); 5702 do { 5703 /* svprfd(svbool_t, void *, enum svfprop); */ 5704 svprfd(mask, xinput + 128, SV_PLDL1STRM); 5705 ACCRND(vacc, 0); 5706 xinput += 8; 5707 xsecret += 1; 5708 nbStripes--; 5709 } while (nbStripes != 0); 5710 5711 svst1_u64(mask, xacc + 0, vacc); 5712 } else if (element_count == 2) { /* sve128 */ 5713 svbool_t mask = svptrue_pat_b64(SV_VL2); 5714 svuint64_t acc0 = svld1_u64(mask, xacc + 0); 5715 svuint64_t acc1 = svld1_u64(mask, xacc + 2); 5716 svuint64_t acc2 = svld1_u64(mask, xacc + 4); 5717 svuint64_t acc3 = svld1_u64(mask, xacc + 6); 5718 do { 5719 svprfd(mask, xinput + 128, SV_PLDL1STRM); 5720 ACCRND(acc0, 0); 5721 ACCRND(acc1, 2); 5722 ACCRND(acc2, 4); 5723 ACCRND(acc3, 6); 5724 xinput += 8; 5725 xsecret += 1; 5726 nbStripes--; 5727 } while (nbStripes != 0); 5728 5729 svst1_u64(mask, xacc + 0, acc0); 5730 svst1_u64(mask, xacc + 2, acc1); 5731 svst1_u64(mask, xacc + 4, acc2); 5732 svst1_u64(mask, xacc + 6, acc3); 5733 } else { 5734 svbool_t mask = svptrue_pat_b64(SV_VL4); 5735 svuint64_t acc0 = svld1_u64(mask, xacc + 0); 5736 svuint64_t acc1 = svld1_u64(mask, xacc + 4); 5737 do { 5738 svprfd(mask, xinput + 128, SV_PLDL1STRM); 5739 ACCRND(acc0, 0); 5740 ACCRND(acc1, 4); 5741 xinput += 8; 5742 xsecret += 1; 5743 nbStripes--; 5744 } while (nbStripes != 0); 5745 5746 svst1_u64(mask, xacc + 0, acc0); 5747 svst1_u64(mask, xacc + 4, acc1); 5748 } 5749 } 5750 } 5751 5752 #endif 5753 5754 #if (XXH_VECTOR == XXH_LSX) 5755 #define _LSX_SHUFFLE(z, y, x, w) (((z) << 6) | ((y) << 4) | ((x) << 2) | (w)) 5756 5757 XXH_FORCE_INLINE void 5758 XXH3_accumulate_512_lsx( void* XXH_RESTRICT acc, 5759 const void* XXH_RESTRICT input, 5760 const void* XXH_RESTRICT secret) 5761 { 5762 XXH_ASSERT((((size_t)acc) & 15) == 0); 5763 { 5764 __m128i* const xacc = (__m128i *) acc; 5765 const __m128i* const xinput = (const __m128i *) input; 5766 const __m128i* const xsecret = (const __m128i *) secret; 5767 size_t i; 5768 5769 for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) { 5770 /* data_vec = xinput[i]; */ 5771 __m128i const data_vec = __lsx_vld(xinput + i, 0); 5772 /* key_vec = xsecret[i]; */ 5773 __m128i const key_vec = __lsx_vld(xsecret + i, 0); 5774 /* data_key = data_vec ^ key_vec; */ 5775 __m128i const data_key = __lsx_vxor_v(data_vec, key_vec); 5776 /* data_key_lo = data_key >> 32; */ 5777 __m128i const data_key_lo = __lsx_vsrli_d(data_key, 32); 5778 // __m128i const data_key_lo = __lsx_vsrli_d(data_key, 32); 5779 /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ 5780 __m128i const product = __lsx_vmulwev_d_wu(data_key, data_key_lo); 5781 /* xacc[i] += swap(data_vec); */ 5782 __m128i const data_swap = __lsx_vshuf4i_w(data_vec, _LSX_SHUFFLE(1, 0, 3, 2)); 5783 __m128i const sum = __lsx_vadd_d(xacc[i], data_swap); 5784 /* xacc[i] += product; */ 5785 xacc[i] = __lsx_vadd_d(product, sum); 5786 } 5787 } 5788 } 5789 XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(lsx) 5790 5791 XXH_FORCE_INLINE void 5792 XXH3_scrambleAcc_lsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) 5793 { 5794 XXH_ASSERT((((size_t)acc) & 15) == 0); 5795 { 5796 __m128i* const xacc = (__m128i*) acc; 5797 const __m128i* const xsecret = (const __m128i *) secret; 5798 const __m128i prime32 = __lsx_vreplgr2vr_d(XXH_PRIME32_1); 5799 size_t i; 5800 5801 for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) { 5802 /* xacc[i] ^= (xacc[i] >> 47) */ 5803 __m128i const acc_vec = xacc[i]; 5804 __m128i const shifted = __lsx_vsrli_d(acc_vec, 47); 5805 __m128i const data_vec = __lsx_vxor_v(acc_vec, shifted); 5806 /* xacc[i] ^= xsecret[i]; */ 5807 __m128i const key_vec = __lsx_vld(xsecret + i, 0); 5808 __m128i const data_key = __lsx_vxor_v(data_vec, key_vec); 5809 5810 /* xacc[i] *= XXH_PRIME32_1; */ 5811 xacc[i] = __lsx_vmul_d(data_key, prime32); 5812 } 5813 } 5814 } 5815 5816 #endif 5817 5818 #if (XXH_VECTOR == XXH_LASX) 5819 #define _LASX_SHUFFLE(z, y, x, w) (((z) << 6) | ((y) << 4) | ((x) << 2) | (w)) 5820 5821 XXH_FORCE_INLINE void 5822 XXH3_accumulate_512_lasx( void* XXH_RESTRICT acc, 5823 const void* XXH_RESTRICT input, 5824 const void* XXH_RESTRICT secret) 5825 { 5826 XXH_ASSERT((((size_t)acc) & 31) == 0); 5827 { 5828 size_t i; 5829 __m256i* const xacc = (__m256i *) acc; 5830 const __m256i* const xinput = (const __m256i *) input; 5831 const __m256i* const xsecret = (const __m256i *) secret; 5832 5833 for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m256i); i++) { 5834 /* data_vec = xinput[i]; */ 5835 __m256i const data_vec = __lasx_xvld(xinput + i, 0); 5836 /* key_vec = xsecret[i]; */ 5837 __m256i const key_vec = __lasx_xvld(xsecret + i, 0); 5838 /* data_key = data_vec ^ key_vec; */ 5839 __m256i const data_key = __lasx_xvxor_v(data_vec, key_vec); 5840 /* data_key_lo = data_key >> 32; */ 5841 __m256i const data_key_lo = __lasx_xvsrli_d(data_key, 32); 5842 // __m256i const data_key_lo = __lasx_xvsrli_d(data_key, 32); 5843 /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ 5844 __m256i const product = __lasx_xvmulwev_d_wu(data_key, data_key_lo); 5845 /* xacc[i] += swap(data_vec); */ 5846 __m256i const data_swap = __lasx_xvshuf4i_w(data_vec, _LASX_SHUFFLE(1, 0, 3, 2)); 5847 __m256i const sum = __lasx_xvadd_d(xacc[i], data_swap); 5848 /* xacc[i] += product; */ 5849 xacc[i] = __lasx_xvadd_d(product, sum); 5850 } 5851 } 5852 } 5853 XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(lasx) 5854 5855 XXH_FORCE_INLINE void 5856 XXH3_scrambleAcc_lasx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) 5857 { 5858 XXH_ASSERT((((size_t)acc) & 31) == 0); 5859 { 5860 __m256i* const xacc = (__m256i*) acc; 5861 const __m256i* const xsecret = (const __m256i *) secret; 5862 const __m256i prime32 = __lasx_xvreplgr2vr_d(XXH_PRIME32_1); 5863 size_t i; 5864 5865 for (i = 0; i < XXH_STRIPE_LEN / sizeof(__m256i); i++) { 5866 /* xacc[i] ^= (xacc[i] >> 47) */ 5867 __m256i const acc_vec = xacc[i]; 5868 __m256i const shifted = __lasx_xvsrli_d(acc_vec, 47); 5869 __m256i const data_vec = __lasx_xvxor_v(acc_vec, shifted); 5870 /* xacc[i] ^= xsecret[i]; */ 5871 __m256i const key_vec = __lasx_xvld(xsecret + i, 0); 5872 __m256i const data_key = __lasx_xvxor_v(data_vec, key_vec); 5873 5874 /* xacc[i] *= XXH_PRIME32_1; */ 5875 xacc[i] = __lasx_xvmul_d(data_key, prime32); 5876 } 5877 } 5878 } 5879 5880 #endif 5881 5882 #if (XXH_VECTOR == XXH_RVV) 5883 #define XXH_CONCAT2(X, Y) X ## Y 5884 #define XXH_CONCAT(X, Y) XXH_CONCAT2(X, Y) 5885 #if ((defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 13) || \ 5886 (defined(__clang__) && __clang_major__ < 16)) 5887 #define XXH_RVOP(op) op 5888 #define XXH_RVCAST(op) XXH_CONCAT(vreinterpret_v_, op) 5889 #else 5890 #define XXH_RVOP(op) XXH_CONCAT(__riscv_, op) 5891 #define XXH_RVCAST(op) XXH_CONCAT(__riscv_vreinterpret_v_, op) 5892 #endif 5893 XXH_FORCE_INLINE void 5894 XXH3_accumulate_512_rvv( void* XXH_RESTRICT acc, 5895 const void* XXH_RESTRICT input, 5896 const void* XXH_RESTRICT secret) 5897 { 5898 XXH_ASSERT((((size_t)acc) & 63) == 0); 5899 { 5900 // Try to set vector lenght to 512 bits. 5901 // If this length is unavailable, then maximum available will be used 5902 size_t vl = XXH_RVOP(vsetvl_e64m2)(8); 5903 5904 uint64_t* xacc = (uint64_t*) acc; 5905 const uint64_t* xinput = (const uint64_t*) input; 5906 const uint64_t* xsecret = (const uint64_t*) secret; 5907 static const uint64_t swap_mask[16] = {1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14}; 5908 vuint64m2_t xswap_mask = XXH_RVOP(vle64_v_u64m2)(swap_mask, vl); 5909 5910 size_t i; 5911 for (i = 0; i < XXH_STRIPE_LEN/8; i += vl) { 5912 /* data_vec = xinput[i]; */ 5913 vuint64m2_t data_vec = XXH_RVCAST(u8m2_u64m2)(XXH_RVOP(vle8_v_u8m2)((const uint8_t*)(xinput + i), vl * 8)); 5914 /* key_vec = xsecret[i]; */ 5915 vuint64m2_t key_vec = XXH_RVCAST(u8m2_u64m2)(XXH_RVOP(vle8_v_u8m2)((const uint8_t*)(xsecret + i), vl * 8)); 5916 /* acc_vec = xacc[i]; */ 5917 vuint64m2_t acc_vec = XXH_RVOP(vle64_v_u64m2)(xacc + i, vl); 5918 /* data_key = data_vec ^ key_vec; */ 5919 vuint64m2_t data_key = XXH_RVOP(vxor_vv_u64m2)(data_vec, key_vec, vl); 5920 /* data_key_hi = data_key >> 32; */ 5921 vuint64m2_t data_key_hi = XXH_RVOP(vsrl_vx_u64m2)(data_key, 32, vl); 5922 /* data_key_lo = data_key & 0xffffffff; */ 5923 vuint64m2_t data_key_lo = XXH_RVOP(vand_vx_u64m2)(data_key, 0xffffffff, vl); 5924 /* swap high and low halves */ 5925 vuint64m2_t data_swap = XXH_RVOP(vrgather_vv_u64m2)(data_vec, xswap_mask, vl); 5926 /* acc_vec += data_key_lo * data_key_hi; */ 5927 acc_vec = XXH_RVOP(vmacc_vv_u64m2)(acc_vec, data_key_lo, data_key_hi, vl); 5928 /* acc_vec += data_swap; */ 5929 acc_vec = XXH_RVOP(vadd_vv_u64m2)(acc_vec, data_swap, vl); 5930 /* xacc[i] = acc_vec; */ 5931 XXH_RVOP(vse64_v_u64m2)(xacc + i, acc_vec, vl); 5932 } 5933 } 5934 } 5935 5936 XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(rvv) 5937 5938 XXH_FORCE_INLINE void 5939 XXH3_scrambleAcc_rvv(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) 5940 { 5941 XXH_ASSERT((((size_t)acc) & 15) == 0); 5942 { 5943 size_t count = XXH_STRIPE_LEN/8; 5944 uint64_t* xacc = (uint64_t*)acc; 5945 const uint8_t* xsecret = (const uint8_t *)secret; 5946 size_t vl; 5947 for (; count > 0; count -= vl, xacc += vl, xsecret += vl*8) { 5948 vl = XXH_RVOP(vsetvl_e64m2)(count); 5949 { 5950 /* key_vec = xsecret[i]; */ 5951 vuint64m2_t key_vec = XXH_RVCAST(u8m2_u64m2)(XXH_RVOP(vle8_v_u8m2)(xsecret, vl*8)); 5952 /* acc_vec = xacc[i]; */ 5953 vuint64m2_t acc_vec = XXH_RVOP(vle64_v_u64m2)(xacc, vl); 5954 /* acc_vec ^= acc_vec >> 47; */ 5955 vuint64m2_t vsrl = XXH_RVOP(vsrl_vx_u64m2)(acc_vec, 47, vl); 5956 acc_vec = XXH_RVOP(vxor_vv_u64m2)(acc_vec, vsrl, vl); 5957 /* acc_vec ^= key_vec; */ 5958 acc_vec = XXH_RVOP(vxor_vv_u64m2)(acc_vec, key_vec, vl); 5959 /* acc_vec *= XXH_PRIME32_1; */ 5960 acc_vec = XXH_RVOP(vmul_vx_u64m2)(acc_vec, XXH_PRIME32_1, vl); 5961 /* xacc[i] *= acc_vec; */ 5962 XXH_RVOP(vse64_v_u64m2)(xacc, acc_vec, vl); 5963 } 5964 } 5965 } 5966 } 5967 5968 XXH_FORCE_INLINE void 5969 XXH3_initCustomSecret_rvv(void* XXH_RESTRICT customSecret, xxh_u64 seed64) 5970 { 5971 XXH_STATIC_ASSERT(XXH_SEC_ALIGN >= 8); 5972 XXH_ASSERT(((size_t)customSecret & 7) == 0); 5973 (void)(&XXH_writeLE64); 5974 { 5975 size_t count = XXH_SECRET_DEFAULT_SIZE/8; 5976 size_t vl; 5977 size_t VLMAX = XXH_RVOP(vsetvlmax_e64m2)(); 5978 int64_t* cSecret = (int64_t*)customSecret; 5979 const int64_t* kSecret = (const int64_t*)(const void*)XXH3_kSecret; 5980 5981 #if __riscv_v_intrinsic >= 1000000 5982 // ratified v1.0 intrinics version 5983 vbool32_t mneg = XXH_RVCAST(u8m1_b32)( 5984 XXH_RVOP(vmv_v_x_u8m1)(0xaa, XXH_RVOP(vsetvlmax_e8m1)())); 5985 #else 5986 // support pre-ratification intrinics, which lack mask to vector casts 5987 size_t vlmax = XXH_RVOP(vsetvlmax_e8m1)(); 5988 vbool32_t mneg = XXH_RVOP(vmseq_vx_u8mf4_b32)( 5989 XXH_RVOP(vand_vx_u8mf4)( 5990 XXH_RVOP(vid_v_u8mf4)(vlmax), 1, vlmax), 1, vlmax); 5991 #endif 5992 vint64m2_t seed = XXH_RVOP(vmv_v_x_i64m2)((int64_t)seed64, VLMAX); 5993 seed = XXH_RVOP(vneg_v_i64m2_mu)(mneg, seed, seed, VLMAX); 5994 5995 for (; count > 0; count -= vl, cSecret += vl, kSecret += vl) { 5996 /* make sure vl=VLMAX until last iteration */ 5997 vl = XXH_RVOP(vsetvl_e64m2)(count < VLMAX ? count : VLMAX); 5998 { 5999 vint64m2_t src = XXH_RVOP(vle64_v_i64m2)(kSecret, vl); 6000 vint64m2_t res = XXH_RVOP(vadd_vv_i64m2)(src, seed, vl); 6001 XXH_RVOP(vse64_v_i64m2)(cSecret, res, vl); 6002 } 6003 } 6004 } 6005 } 6006 #endif 6007 6008 6009 /* scalar variants - universal */ 6010 6011 #if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) 6012 /* 6013 * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they 6014 * emit an excess mask and a full 64-bit multiply-add (MADD X-form). 6015 * 6016 * While this might not seem like much, as AArch64 is a 64-bit architecture, only 6017 * big Cortex designs have a full 64-bit multiplier. 6018 * 6019 * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit 6020 * multiplies expand to 2-3 multiplies in microcode. This has a major penalty 6021 * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline. 6022 * 6023 * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does 6024 * not have this penalty and does the mask automatically. 6025 */ 6026 XXH_FORCE_INLINE xxh_u64 6027 XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) 6028 { 6029 xxh_u64 ret; 6030 /* note: %x = 64-bit register, %w = 32-bit register */ 6031 __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc)); 6032 return ret; 6033 } 6034 #else 6035 XXH_FORCE_INLINE xxh_u64 6036 XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) 6037 { 6038 return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc; 6039 } 6040 #endif 6041 6042 /*! 6043 * @internal 6044 * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). 6045 * 6046 * This is extracted to its own function because the NEON path uses a combination 6047 * of NEON and scalar. 6048 */ 6049 XXH_FORCE_INLINE void 6050 XXH3_scalarRound(void* XXH_RESTRICT acc, 6051 void const* XXH_RESTRICT input, 6052 void const* XXH_RESTRICT secret, 6053 size_t lane) 6054 { 6055 xxh_u64* xacc = (xxh_u64*) acc; 6056 xxh_u8 const* xinput = (xxh_u8 const*) input; 6057 xxh_u8 const* xsecret = (xxh_u8 const*) secret; 6058 XXH_ASSERT(lane < XXH_ACC_NB); 6059 XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); 6060 { 6061 xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); 6062 xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); 6063 xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ 6064 xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]); 6065 } 6066 } 6067 6068 /*! 6069 * @internal 6070 * @brief Processes a 64 byte block of data using the scalar path. 6071 */ 6072 XXH_FORCE_INLINE void 6073 XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, 6074 const void* XXH_RESTRICT input, 6075 const void* XXH_RESTRICT secret) 6076 { 6077 size_t i; 6078 /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */ 6079 #if defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 8 \ 6080 && (defined(__arm__) || defined(__thumb2__)) \ 6081 && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \ 6082 && XXH_SIZE_OPT <= 0 6083 # pragma GCC unroll 8 6084 #endif 6085 for (i=0; i < XXH_ACC_NB; i++) { 6086 XXH3_scalarRound(acc, input, secret, i); 6087 } 6088 } 6089 XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar) 6090 6091 /*! 6092 * @internal 6093 * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). 6094 * 6095 * This is extracted to its own function because the NEON path uses a combination 6096 * of NEON and scalar. 6097 */ 6098 XXH_FORCE_INLINE void 6099 XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, 6100 void const* XXH_RESTRICT secret, 6101 size_t lane) 6102 { 6103 xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ 6104 const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ 6105 XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); 6106 XXH_ASSERT(lane < XXH_ACC_NB); 6107 { 6108 xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); 6109 xxh_u64 acc64 = xacc[lane]; 6110 acc64 = XXH_xorshift64(acc64, 47); 6111 acc64 ^= key64; 6112 acc64 *= XXH_PRIME32_1; 6113 xacc[lane] = acc64; 6114 } 6115 } 6116 6117 /*! 6118 * @internal 6119 * @brief Scrambles the accumulators after a large chunk has been read 6120 */ 6121 XXH_FORCE_INLINE void 6122 XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) 6123 { 6124 size_t i; 6125 for (i=0; i < XXH_ACC_NB; i++) { 6126 XXH3_scalarScrambleRound(acc, secret, i); 6127 } 6128 } 6129 6130 XXH_FORCE_INLINE void 6131 XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) 6132 { 6133 /* 6134 * We need a separate pointer for the hack below, 6135 * which requires a non-const pointer. 6136 * Any decent compiler will optimize this out otherwise. 6137 */ 6138 const xxh_u8* kSecretPtr = XXH3_kSecret; 6139 XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); 6140 6141 #if defined(__GNUC__) && defined(__aarch64__) 6142 /* 6143 * UGLY HACK: 6144 * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are 6145 * placed sequentially, in order, at the top of the unrolled loop. 6146 * 6147 * While MOVK is great for generating constants (2 cycles for a 64-bit 6148 * constant compared to 4 cycles for LDR), it fights for bandwidth with 6149 * the arithmetic instructions. 6150 * 6151 * I L S 6152 * MOVK 6153 * MOVK 6154 * MOVK 6155 * MOVK 6156 * ADD 6157 * SUB STR 6158 * STR 6159 * By forcing loads from memory (as the asm line causes the compiler to assume 6160 * that XXH3_kSecretPtr has been changed), the pipelines are used more 6161 * efficiently: 6162 * I L S 6163 * LDR 6164 * ADD LDR 6165 * SUB STR 6166 * STR 6167 * 6168 * See XXH3_NEON_LANES for details on the pipeline. 6169 * 6170 * XXH3_64bits_withSeed, len == 256, Snapdragon 835 6171 * without hack: 2654.4 MB/s 6172 * with hack: 3202.9 MB/s 6173 */ 6174 XXH_COMPILER_GUARD(kSecretPtr); 6175 #endif 6176 { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; 6177 int i; 6178 for (i=0; i < nbRounds; i++) { 6179 /* 6180 * The asm hack causes the compiler to assume that kSecretPtr aliases with 6181 * customSecret, and on aarch64, this prevented LDP from merging two 6182 * loads together for free. Putting the loads together before the stores 6183 * properly generates LDP. 6184 */ 6185 xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; 6186 xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; 6187 XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); 6188 XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); 6189 } } 6190 } 6191 6192 6193 typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t); 6194 typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); 6195 typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); 6196 6197 6198 #if (XXH_VECTOR == XXH_AVX512) 6199 6200 #define XXH3_accumulate_512 XXH3_accumulate_512_avx512 6201 #define XXH3_accumulate XXH3_accumulate_avx512 6202 #define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 6203 #define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 6204 6205 #elif (XXH_VECTOR == XXH_AVX2) 6206 6207 #define XXH3_accumulate_512 XXH3_accumulate_512_avx2 6208 #define XXH3_accumulate XXH3_accumulate_avx2 6209 #define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 6210 #define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 6211 6212 #elif (XXH_VECTOR == XXH_SSE2) 6213 6214 #define XXH3_accumulate_512 XXH3_accumulate_512_sse2 6215 #define XXH3_accumulate XXH3_accumulate_sse2 6216 #define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 6217 #define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 6218 6219 #elif (XXH_VECTOR == XXH_NEON) 6220 6221 #define XXH3_accumulate_512 XXH3_accumulate_512_neon 6222 #define XXH3_accumulate XXH3_accumulate_neon 6223 #define XXH3_scrambleAcc XXH3_scrambleAcc_neon 6224 #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar 6225 6226 #elif (XXH_VECTOR == XXH_VSX) 6227 6228 #define XXH3_accumulate_512 XXH3_accumulate_512_vsx 6229 #define XXH3_accumulate XXH3_accumulate_vsx 6230 #define XXH3_scrambleAcc XXH3_scrambleAcc_vsx 6231 #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar 6232 6233 #elif (XXH_VECTOR == XXH_SVE) 6234 #define XXH3_accumulate_512 XXH3_accumulate_512_sve 6235 #define XXH3_accumulate XXH3_accumulate_sve 6236 #define XXH3_scrambleAcc XXH3_scrambleAcc_scalar 6237 #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar 6238 6239 #elif (XXH_VECTOR == XXH_LASX) 6240 #define XXH3_accumulate_512 XXH3_accumulate_512_lasx 6241 #define XXH3_accumulate XXH3_accumulate_lasx 6242 #define XXH3_scrambleAcc XXH3_scrambleAcc_lasx 6243 #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar 6244 6245 #elif (XXH_VECTOR == XXH_LSX) 6246 #define XXH3_accumulate_512 XXH3_accumulate_512_lsx 6247 #define XXH3_accumulate XXH3_accumulate_lsx 6248 #define XXH3_scrambleAcc XXH3_scrambleAcc_lsx 6249 #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar 6250 6251 #elif (XXH_VECTOR == XXH_RVV) 6252 #define XXH3_accumulate_512 XXH3_accumulate_512_rvv 6253 #define XXH3_accumulate XXH3_accumulate_rvv 6254 #define XXH3_scrambleAcc XXH3_scrambleAcc_rvv 6255 #define XXH3_initCustomSecret XXH3_initCustomSecret_rvv 6256 6257 #else /* scalar */ 6258 6259 #define XXH3_accumulate_512 XXH3_accumulate_512_scalar 6260 #define XXH3_accumulate XXH3_accumulate_scalar 6261 #define XXH3_scrambleAcc XXH3_scrambleAcc_scalar 6262 #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar 6263 6264 #endif 6265 6266 #if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */ 6267 # undef XXH3_initCustomSecret 6268 # define XXH3_initCustomSecret XXH3_initCustomSecret_scalar 6269 #endif 6270 6271 XXH_FORCE_INLINE void 6272 XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, 6273 const xxh_u8* XXH_RESTRICT input, size_t len, 6274 const xxh_u8* XXH_RESTRICT secret, size_t secretSize, 6275 XXH3_f_accumulate f_acc, 6276 XXH3_f_scrambleAcc f_scramble) 6277 { 6278 size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; 6279 size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; 6280 size_t const nb_blocks = (len - 1) / block_len; 6281 6282 size_t n; 6283 6284 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); 6285 6286 for (n = 0; n < nb_blocks; n++) { 6287 f_acc(acc, input + n*block_len, secret, nbStripesPerBlock); 6288 f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); 6289 } 6290 6291 /* last partial block */ 6292 XXH_ASSERT(len > XXH_STRIPE_LEN); 6293 { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; 6294 XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); 6295 f_acc(acc, input + nb_blocks*block_len, secret, nbStripes); 6296 6297 /* last stripe */ 6298 { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; 6299 #define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ 6300 XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); 6301 } } 6302 } 6303 6304 XXH_FORCE_INLINE xxh_u64 6305 XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) 6306 { 6307 return XXH3_mul128_fold64( 6308 acc[0] ^ XXH_readLE64(secret), 6309 acc[1] ^ XXH_readLE64(secret+8) ); 6310 } 6311 6312 static XXH_PUREF XXH64_hash_t 6313 XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) 6314 { 6315 xxh_u64 result64 = start; 6316 size_t i = 0; 6317 6318 for (i = 0; i < 4; i++) { 6319 result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); 6320 #if defined(__clang__) /* Clang */ \ 6321 && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ 6322 && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ 6323 && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ 6324 /* 6325 * UGLY HACK: 6326 * Prevent autovectorization on Clang ARMv7-a. Exact same problem as 6327 * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. 6328 * XXH3_64bits, len == 256, Snapdragon 835: 6329 * without hack: 2063.7 MB/s 6330 * with hack: 2560.7 MB/s 6331 */ 6332 XXH_COMPILER_GUARD(result64); 6333 #endif 6334 } 6335 6336 return XXH3_avalanche(result64); 6337 } 6338 6339 /* do not align on 8, so that the secret is different from the accumulator */ 6340 #define XXH_SECRET_MERGEACCS_START 11 6341 6342 static XXH_PUREF XXH64_hash_t 6343 XXH3_finalizeLong_64b(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 len) 6344 { 6345 return XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, len * XXH_PRIME64_1); 6346 } 6347 6348 #define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ 6349 XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } 6350 6351 XXH_FORCE_INLINE XXH64_hash_t 6352 XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, 6353 const void* XXH_RESTRICT secret, size_t secretSize, 6354 XXH3_f_accumulate f_acc, 6355 XXH3_f_scrambleAcc f_scramble) 6356 { 6357 XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; 6358 6359 XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble); 6360 6361 /* converge into final hash */ 6362 XXH_STATIC_ASSERT(sizeof(acc) == 64); 6363 XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); 6364 return XXH3_finalizeLong_64b(acc, (const xxh_u8*)secret, (xxh_u64)len); 6365 } 6366 6367 /* 6368 * It's important for performance to transmit secret's size (when it's static) 6369 * so that the compiler can properly optimize the vectorized loop. 6370 * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. 6371 * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE 6372 * breaks -Og, this is XXH_NO_INLINE. 6373 */ 6374 XXH3_WITH_SECRET_INLINE XXH64_hash_t 6375 XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, 6376 XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) 6377 { 6378 (void)seed64; 6379 return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc); 6380 } 6381 6382 /* 6383 * It's preferable for performance that XXH3_hashLong is not inlined, 6384 * as it results in a smaller function for small data, easier to the instruction cache. 6385 * Note that inside this no_inline function, we do inline the internal loop, 6386 * and provide a statically defined secret size to allow optimization of vector loop. 6387 */ 6388 XXH_NO_INLINE XXH_PUREF XXH64_hash_t 6389 XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, 6390 XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) 6391 { 6392 (void)seed64; (void)secret; (void)secretLen; 6393 return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); 6394 } 6395 6396 /* 6397 * XXH3_hashLong_64b_withSeed(): 6398 * Generate a custom key based on alteration of default XXH3_kSecret with the seed, 6399 * and then use this key for long mode hashing. 6400 * 6401 * This operation is decently fast but nonetheless costs a little bit of time. 6402 * Try to avoid it whenever possible (typically when seed==0). 6403 * 6404 * It's important for performance that XXH3_hashLong is not inlined. Not sure 6405 * why (uop cache maybe?), but the difference is large and easily measurable. 6406 */ 6407 XXH_FORCE_INLINE XXH64_hash_t 6408 XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, 6409 XXH64_hash_t seed, 6410 XXH3_f_accumulate f_acc, 6411 XXH3_f_scrambleAcc f_scramble, 6412 XXH3_f_initCustomSecret f_initSec) 6413 { 6414 #if XXH_SIZE_OPT <= 0 6415 if (seed == 0) 6416 return XXH3_hashLong_64b_internal(input, len, 6417 XXH3_kSecret, sizeof(XXH3_kSecret), 6418 f_acc, f_scramble); 6419 #endif 6420 { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; 6421 f_initSec(secret, seed); 6422 return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), 6423 f_acc, f_scramble); 6424 } 6425 } 6426 6427 /* 6428 * It's important for performance that XXH3_hashLong is not inlined. 6429 */ 6430 XXH_NO_INLINE XXH64_hash_t 6431 XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len, 6432 XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) 6433 { 6434 (void)secret; (void)secretLen; 6435 return XXH3_hashLong_64b_withSeed_internal(input, len, seed, 6436 XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); 6437 } 6438 6439 6440 typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, 6441 XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); 6442 6443 XXH_FORCE_INLINE XXH64_hash_t 6444 XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, 6445 XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, 6446 XXH3_hashLong64_f f_hashLong) 6447 { 6448 XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); 6449 /* 6450 * If an action is to be taken if `secretLen` condition is not respected, 6451 * it should be done here. 6452 * For now, it's a contract pre-condition. 6453 * Adding a check and a branch here would cost performance at every hash. 6454 * Also, note that function signature doesn't offer room to return an error. 6455 */ 6456 if (len <= 16) 6457 return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); 6458 if (len <= 128) 6459 return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); 6460 if (len <= XXH3_MIDSIZE_MAX) 6461 return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); 6462 return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); 6463 } 6464 6465 6466 /* === Public entry point === */ 6467 6468 /*! @ingroup XXH3_family */ 6469 XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) 6470 { 6471 return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); 6472 } 6473 6474 /*! @ingroup XXH3_family */ 6475 XXH_PUBLIC_API XXH64_hash_t 6476 XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize) 6477 { 6478 return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); 6479 } 6480 6481 /*! @ingroup XXH3_family */ 6482 XXH_PUBLIC_API XXH64_hash_t 6483 XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed) 6484 { 6485 return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); 6486 } 6487 6488 XXH_PUBLIC_API XXH64_hash_t 6489 XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) 6490 { 6491 if (length <= XXH3_MIDSIZE_MAX) 6492 return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); 6493 return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize); 6494 } 6495 6496 6497 /* === XXH3 streaming === */ 6498 #ifndef XXH_NO_STREAM 6499 /* 6500 * Malloc's a pointer that is always aligned to @align. 6501 * 6502 * This must be freed with `XXH_alignedFree()`. 6503 * 6504 * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte 6505 * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 6506 * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. 6507 * 6508 * This underalignment previously caused a rather obvious crash which went 6509 * completely unnoticed due to XXH3_createState() not actually being tested. 6510 * Credit to RedSpah for noticing this bug. 6511 * 6512 * The alignment is done manually: Functions like posix_memalign or _mm_malloc 6513 * are avoided: To maintain portability, we would have to write a fallback 6514 * like this anyways, and besides, testing for the existence of library 6515 * functions without relying on external build tools is impossible. 6516 * 6517 * The method is simple: Overallocate, manually align, and store the offset 6518 * to the original behind the returned pointer. 6519 * 6520 * Align must be a power of 2 and 8 <= align <= 128. 6521 */ 6522 static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align) 6523 { 6524 XXH_ASSERT(align <= 128 && align >= 8); /* range check */ 6525 XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ 6526 XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ 6527 { /* Overallocate to make room for manual realignment and an offset byte */ 6528 xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); 6529 if (base != NULL) { 6530 /* 6531 * Get the offset needed to align this pointer. 6532 * 6533 * Even if the returned pointer is aligned, there will always be 6534 * at least one byte to store the offset to the original pointer. 6535 */ 6536 size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ 6537 /* Add the offset for the now-aligned pointer */ 6538 xxh_u8* ptr = base + offset; 6539 6540 XXH_ASSERT((size_t)ptr % align == 0); 6541 6542 /* Store the offset immediately before the returned pointer. */ 6543 ptr[-1] = (xxh_u8)offset; 6544 return ptr; 6545 } 6546 return NULL; 6547 } 6548 } 6549 /* 6550 * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass 6551 * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. 6552 */ 6553 static void XXH_alignedFree(void* p) 6554 { 6555 if (p != NULL) { 6556 xxh_u8* ptr = (xxh_u8*)p; 6557 /* Get the offset byte we added in XXH_malloc. */ 6558 xxh_u8 offset = ptr[-1]; 6559 /* Free the original malloc'd pointer */ 6560 xxh_u8* base = ptr - offset; 6561 XXH_free(base); 6562 } 6563 } 6564 /*! @ingroup XXH3_family */ 6565 /*! 6566 * @brief Allocate an @ref XXH3_state_t. 6567 * 6568 * @return An allocated pointer of @ref XXH3_state_t on success. 6569 * @return `NULL` on failure. 6570 * 6571 * @note Must be freed with XXH3_freeState(). 6572 * 6573 * @see @ref streaming_example "Streaming Example" 6574 */ 6575 XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) 6576 { 6577 XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); 6578 if (state==NULL) return NULL; 6579 XXH3_INITSTATE(state); 6580 return state; 6581 } 6582 6583 /*! @ingroup XXH3_family */ 6584 /*! 6585 * @brief Frees an @ref XXH3_state_t. 6586 * 6587 * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). 6588 * 6589 * @return @ref XXH_OK. 6590 * 6591 * @note Must be allocated with XXH3_createState(). 6592 * 6593 * @see @ref streaming_example "Streaming Example" 6594 */ 6595 XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) 6596 { 6597 XXH_alignedFree(statePtr); 6598 return XXH_OK; 6599 } 6600 6601 /*! @ingroup XXH3_family */ 6602 XXH_PUBLIC_API void 6603 XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state) 6604 { 6605 XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); 6606 } 6607 6608 static void 6609 XXH3_reset_internal(XXH3_state_t* statePtr, 6610 XXH64_hash_t seed, 6611 const void* secret, size_t secretSize) 6612 { 6613 size_t const initStart = offsetof(XXH3_state_t, bufferedSize); 6614 size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; 6615 XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); 6616 XXH_ASSERT(statePtr != NULL); 6617 /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ 6618 XXH_memset((char*)statePtr + initStart, 0, initLength); 6619 statePtr->acc[0] = XXH_PRIME32_3; 6620 statePtr->acc[1] = XXH_PRIME64_1; 6621 statePtr->acc[2] = XXH_PRIME64_2; 6622 statePtr->acc[3] = XXH_PRIME64_3; 6623 statePtr->acc[4] = XXH_PRIME64_4; 6624 statePtr->acc[5] = XXH_PRIME32_2; 6625 statePtr->acc[6] = XXH_PRIME64_5; 6626 statePtr->acc[7] = XXH_PRIME32_1; 6627 statePtr->seed = seed; 6628 statePtr->useSeed = (seed != 0); 6629 statePtr->extSecret = (const unsigned char*)secret; 6630 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); 6631 statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; 6632 statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; 6633 } 6634 6635 /*! @ingroup XXH3_family */ 6636 XXH_PUBLIC_API XXH_errorcode 6637 XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) 6638 { 6639 if (statePtr == NULL) return XXH_ERROR; 6640 XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); 6641 return XXH_OK; 6642 } 6643 6644 /*! @ingroup XXH3_family */ 6645 XXH_PUBLIC_API XXH_errorcode 6646 XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) 6647 { 6648 if (statePtr == NULL) return XXH_ERROR; 6649 XXH3_reset_internal(statePtr, 0, secret, secretSize); 6650 if (secret == NULL) return XXH_ERROR; 6651 if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; 6652 return XXH_OK; 6653 } 6654 6655 /*! @ingroup XXH3_family */ 6656 XXH_PUBLIC_API XXH_errorcode 6657 XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) 6658 { 6659 if (statePtr == NULL) return XXH_ERROR; 6660 if (seed==0) return XXH3_64bits_reset(statePtr); 6661 if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) 6662 XXH3_initCustomSecret(statePtr->customSecret, seed); 6663 XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); 6664 return XXH_OK; 6665 } 6666 6667 /*! @ingroup XXH3_family */ 6668 XXH_PUBLIC_API XXH_errorcode 6669 XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64) 6670 { 6671 if (statePtr == NULL) return XXH_ERROR; 6672 if (secret == NULL) return XXH_ERROR; 6673 if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; 6674 XXH3_reset_internal(statePtr, seed64, secret, secretSize); 6675 statePtr->useSeed = 1; /* always, even if seed64==0 */ 6676 return XXH_OK; 6677 } 6678 6679 /*! 6680 * @internal 6681 * @brief Processes a large input for XXH3_update() and XXH3_digest_long(). 6682 * 6683 * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block. 6684 * 6685 * @param acc Pointer to the 8 accumulator lanes 6686 * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in the block* 6687 * @param nbStripesPerBlock Number of stripes in a block 6688 * @param input Input pointer 6689 * @param nbStripes Number of stripes to process 6690 * @param secret Secret pointer 6691 * @param secretLimit Offset of the last block in @p secret 6692 * @param f_acc Pointer to an XXH3_accumulate implementation 6693 * @param f_scramble Pointer to an XXH3_scrambleAcc implementation 6694 * @return Pointer past the end of @p input after processing 6695 */ 6696 XXH_FORCE_INLINE const xxh_u8 * 6697 XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, 6698 size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, 6699 const xxh_u8* XXH_RESTRICT input, size_t nbStripes, 6700 const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, 6701 XXH3_f_accumulate f_acc, 6702 XXH3_f_scrambleAcc f_scramble) 6703 { 6704 const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE; 6705 /* Process full blocks */ 6706 if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) { 6707 /* Process the initial partial block... */ 6708 size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr; 6709 6710 do { 6711 /* Accumulate and scramble */ 6712 f_acc(acc, input, initialSecret, nbStripesThisIter); 6713 f_scramble(acc, secret + secretLimit); 6714 input += nbStripesThisIter * XXH_STRIPE_LEN; 6715 nbStripes -= nbStripesThisIter; 6716 /* Then continue the loop with the full block size */ 6717 nbStripesThisIter = nbStripesPerBlock; 6718 initialSecret = secret; 6719 } while (nbStripes >= nbStripesPerBlock); 6720 *nbStripesSoFarPtr = 0; 6721 } 6722 /* Process a partial block */ 6723 if (nbStripes > 0) { 6724 f_acc(acc, input, initialSecret, nbStripes); 6725 input += nbStripes * XXH_STRIPE_LEN; 6726 *nbStripesSoFarPtr += nbStripes; 6727 } 6728 /* Return end pointer */ 6729 return input; 6730 } 6731 6732 #ifndef XXH3_STREAM_USE_STACK 6733 # if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */ 6734 # define XXH3_STREAM_USE_STACK 1 6735 # endif 6736 #endif 6737 /* This function accepts f_acc and f_scramble as function pointers, 6738 * making it possible to implement multiple variants with different acc & scramble stages. 6739 * This is notably useful to implement multiple vector variants with different intrinsics. 6740 */ 6741 XXH_FORCE_INLINE XXH_errorcode 6742 XXH3_update(XXH3_state_t* XXH_RESTRICT const state, 6743 const xxh_u8* XXH_RESTRICT input, size_t len, 6744 XXH3_f_accumulate f_acc, 6745 XXH3_f_scrambleAcc f_scramble) 6746 { 6747 if (input==NULL) { 6748 XXH_ASSERT(len == 0); 6749 return XXH_OK; 6750 } 6751 6752 XXH_ASSERT(state != NULL); 6753 state->totalLen += len; 6754 6755 /* small input : just fill in tmp buffer */ 6756 XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); 6757 if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { 6758 XXH_memcpy(state->buffer + state->bufferedSize, input, len); 6759 state->bufferedSize += (XXH32_hash_t)len; 6760 return XXH_OK; 6761 } 6762 6763 { const xxh_u8* const bEnd = input + len; 6764 const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; 6765 #if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 6766 /* For some reason, gcc and MSVC seem to suffer greatly 6767 * when operating accumulators directly into state. 6768 * Operating into stack space seems to enable proper optimization. 6769 * clang, on the other hand, doesn't seem to need this trick */ 6770 XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; 6771 XXH_memcpy(acc, state->acc, sizeof(acc)); 6772 #else 6773 xxh_u64* XXH_RESTRICT const acc = state->acc; 6774 #endif 6775 6776 /* total input is now > XXH3_INTERNALBUFFER_SIZE */ 6777 #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) 6778 XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ 6779 6780 /* 6781 * Internal buffer is partially filled (always, except at beginning) 6782 * Complete it, then consume it. 6783 */ 6784 if (state->bufferedSize) { 6785 size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; 6786 XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); 6787 input += loadSize; 6788 XXH3_consumeStripes(acc, 6789 &state->nbStripesSoFar, state->nbStripesPerBlock, 6790 state->buffer, XXH3_INTERNALBUFFER_STRIPES, 6791 secret, state->secretLimit, 6792 f_acc, f_scramble); 6793 state->bufferedSize = 0; 6794 } 6795 XXH_ASSERT(input < bEnd); 6796 if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { 6797 size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; 6798 input = XXH3_consumeStripes(acc, 6799 &state->nbStripesSoFar, state->nbStripesPerBlock, 6800 input, nbStripes, 6801 secret, state->secretLimit, 6802 f_acc, f_scramble); 6803 XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); 6804 6805 } 6806 /* Some remaining input (always) : buffer it */ 6807 XXH_ASSERT(input < bEnd); 6808 XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); 6809 XXH_ASSERT(state->bufferedSize == 0); 6810 XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); 6811 state->bufferedSize = (XXH32_hash_t)(bEnd-input); 6812 #if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 6813 /* save stack accumulators into state */ 6814 XXH_memcpy(state->acc, acc, sizeof(acc)); 6815 #endif 6816 } 6817 6818 return XXH_OK; 6819 } 6820 6821 /* 6822 * Both XXH3_64bits_update and XXH3_128bits_update use this routine. 6823 */ 6824 XXH_NO_INLINE XXH_errorcode 6825 XXH3_update_regular(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) 6826 { 6827 return XXH3_update(state, (const xxh_u8*)input, len, 6828 XXH3_accumulate, XXH3_scrambleAcc); 6829 } 6830 6831 /*! @ingroup XXH3_family */ 6832 XXH_PUBLIC_API XXH_errorcode 6833 XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) 6834 { 6835 return XXH3_update_regular(state, input, len); 6836 } 6837 6838 6839 XXH_FORCE_INLINE void 6840 XXH3_digest_long (XXH64_hash_t* acc, 6841 const XXH3_state_t* state, 6842 const unsigned char* secret) 6843 { 6844 xxh_u8 lastStripe[XXH_STRIPE_LEN]; 6845 const xxh_u8* lastStripePtr; 6846 6847 /* 6848 * Digest on a local copy. This way, the state remains unaltered, and it can 6849 * continue ingesting more input afterwards. 6850 */ 6851 XXH_memcpy(acc, state->acc, sizeof(state->acc)); 6852 if (state->bufferedSize >= XXH_STRIPE_LEN) { 6853 /* Consume remaining stripes then point to remaining data in buffer */ 6854 size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; 6855 size_t nbStripesSoFar = state->nbStripesSoFar; 6856 XXH3_consumeStripes(acc, 6857 &nbStripesSoFar, state->nbStripesPerBlock, 6858 state->buffer, nbStripes, 6859 secret, state->secretLimit, 6860 XXH3_accumulate, XXH3_scrambleAcc); 6861 lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN; 6862 } else { /* bufferedSize < XXH_STRIPE_LEN */ 6863 /* Copy to temp buffer */ 6864 size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; 6865 XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ 6866 XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); 6867 XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); 6868 lastStripePtr = lastStripe; 6869 } 6870 /* Last stripe */ 6871 XXH3_accumulate_512(acc, 6872 lastStripePtr, 6873 secret + state->secretLimit - XXH_SECRET_LASTACC_START); 6874 } 6875 6876 /*! @ingroup XXH3_family */ 6877 XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state) 6878 { 6879 const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; 6880 if (state->totalLen > XXH3_MIDSIZE_MAX) { 6881 XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; 6882 XXH3_digest_long(acc, state, secret); 6883 return XXH3_finalizeLong_64b(acc, secret, (xxh_u64)state->totalLen); 6884 } 6885 /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ 6886 if (state->useSeed) 6887 return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); 6888 return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), 6889 secret, state->secretLimit + XXH_STRIPE_LEN); 6890 } 6891 #endif /* !XXH_NO_STREAM */ 6892 6893 6894 /* ========================================== 6895 * XXH3 128 bits (a.k.a XXH128) 6896 * ========================================== 6897 * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, 6898 * even without counting the significantly larger output size. 6899 * 6900 * For example, extra steps are taken to mitigate the seed-dependent collisions 6901 * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). Note that these 6902 * steps only mitigate, they do not eliminate the issue: see the DISCLAIMER 6903 * above XXH3_mix16B() for the cases that get through. 6904 * 6905 * This strength naturally comes at the cost of some speed, especially on short 6906 * lengths. Note that longer hashes are about as fast as the 64-bit version 6907 * due to it using only a slight modification of the 64-bit loop. 6908 * 6909 * XXH128 is also more oriented towards 64-bit machines. It is still extremely 6910 * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). 6911 */ 6912 6913 XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t 6914 XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) 6915 { 6916 /* A doubled version of 1to3_64b with different constants. */ 6917 XXH_ASSERT(input != NULL); 6918 XXH_ASSERT(1 <= len && len <= 3); 6919 XXH_ASSERT(secret != NULL); 6920 /* 6921 * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } 6922 * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } 6923 * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } 6924 */ 6925 { xxh_u8 const c1 = input[0]; 6926 xxh_u8 const c2 = input[len >> 1]; 6927 xxh_u8 const c3 = input[len - 1]; 6928 xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) 6929 | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); 6930 xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); 6931 xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; 6932 xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; 6933 xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; 6934 xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; 6935 XXH128_hash_t h128; 6936 h128.low64 = XXH64_avalanche(keyed_lo); 6937 h128.high64 = XXH64_avalanche(keyed_hi); 6938 return h128; 6939 } 6940 } 6941 6942 XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t 6943 XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) 6944 { 6945 XXH_ASSERT(input != NULL); 6946 XXH_ASSERT(secret != NULL); 6947 XXH_ASSERT(4 <= len && len <= 8); 6948 seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; 6949 { xxh_u32 const input_lo = XXH_readLE32(input); 6950 xxh_u32 const input_hi = XXH_readLE32(input + len - 4); 6951 xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); 6952 xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; 6953 xxh_u64 const keyed = input_64 ^ bitflip; 6954 6955 /* Shift len to the left to ensure it is even, this avoids even multiplies. */ 6956 XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); 6957 6958 m128.high64 += (m128.low64 << 1); 6959 m128.low64 ^= (m128.high64 >> 3); 6960 6961 m128.low64 = XXH_xorshift64(m128.low64, 35); 6962 m128.low64 *= PRIME_MX2; 6963 m128.low64 = XXH_xorshift64(m128.low64, 28); 6964 m128.high64 = XXH3_avalanche(m128.high64); 6965 return m128; 6966 } 6967 } 6968 6969 XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t 6970 XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) 6971 { 6972 XXH_ASSERT(input != NULL); 6973 XXH_ASSERT(secret != NULL); 6974 XXH_ASSERT(9 <= len && len <= 16); 6975 { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; 6976 xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; 6977 xxh_u64 const input_lo = XXH_readLE64(input); 6978 xxh_u64 input_hi = XXH_readLE64(input + len - 8); 6979 XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); 6980 /* 6981 * Put len in the middle of m128 to ensure that the length gets mixed to 6982 * both the low and high bits in the 128x64 multiply below. 6983 */ 6984 m128.low64 += (xxh_u64)(len - 1) << 54; 6985 input_hi ^= bitfliph; 6986 /* 6987 * Add the high 32 bits of input_hi to the high 32 bits of m128, then 6988 * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to 6989 * the high 64 bits of m128. 6990 * 6991 * The best approach to this operation is different on 32-bit and 64-bit. 6992 */ 6993 if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ 6994 /* 6995 * 32-bit optimized version, which is more readable. 6996 * 6997 * On 32-bit, it removes an ADC and delays a dependency between the two 6998 * halves of m128.high64, but it generates an extra mask on 64-bit. 6999 */ 7000 m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); 7001 } else { 7002 /* 7003 * 64-bit optimized (albeit more confusing) version. 7004 * 7005 * Uses some properties of addition and multiplication to remove the mask: 7006 * 7007 * Let: 7008 * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) 7009 * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) 7010 * c = XXH_PRIME32_2 7011 * 7012 * a + (b * c) 7013 * Inverse Property: x + y - x == y 7014 * a + (b * (1 + c - 1)) 7015 * Distributive Property: x * (y + z) == (x * y) + (x * z) 7016 * a + (b * 1) + (b * (c - 1)) 7017 * Identity Property: x * 1 == x 7018 * a + b + (b * (c - 1)) 7019 * 7020 * Substitute a, b, and c: 7021 * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) 7022 * 7023 * Since input_hi.hi + input_hi.lo == input_hi, we get this: 7024 * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) 7025 */ 7026 m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); 7027 } 7028 /* m128 ^= XXH_swap64(m128 >> 64); */ 7029 m128.low64 ^= XXH_swap64(m128.high64); 7030 7031 { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ 7032 XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); 7033 h128.high64 += m128.high64 * XXH_PRIME64_2; 7034 7035 h128.low64 = XXH3_avalanche(h128.low64); 7036 h128.high64 = XXH3_avalanche(h128.high64); 7037 return h128; 7038 } } 7039 } 7040 7041 /* 7042 * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN 7043 */ 7044 XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t 7045 XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) 7046 { 7047 XXH_ASSERT(len <= 16); 7048 { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); 7049 if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); 7050 if (len) return XXH3_len_1to3_128b(input, len, secret, seed); 7051 { XXH128_hash_t h128; 7052 xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); 7053 xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); 7054 h128.low64 = XXH64_avalanche(seed ^ bitflipl); 7055 h128.high64 = XXH64_avalanche( seed ^ bitfliph); 7056 return h128; 7057 } } 7058 } 7059 7060 /* 7061 * A bit slower than XXH3_mix16B, but handles multiply by zero better. 7062 */ 7063 XXH_FORCE_INLINE XXH128_hash_t 7064 XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, 7065 const xxh_u8* secret, XXH64_hash_t seed) 7066 { 7067 acc.low64 += XXH3_mix16B (input_1, secret+0, seed); 7068 acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); 7069 acc.high64 += XXH3_mix16B (input_2, secret+16, seed); 7070 acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); 7071 return acc; 7072 } 7073 7074 7075 XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t 7076 XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, 7077 const xxh_u8* XXH_RESTRICT secret, size_t secretSize, 7078 XXH64_hash_t seed) 7079 { 7080 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; 7081 XXH_ASSERT(16 < len && len <= 128); 7082 7083 { XXH128_hash_t acc; 7084 acc.low64 = len * XXH_PRIME64_1; 7085 acc.high64 = 0; 7086 7087 #if XXH_SIZE_OPT >= 1 7088 { 7089 /* Smaller, but slightly slower. */ 7090 unsigned int i = (unsigned int)(len - 1) / 32; 7091 do { 7092 acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed); 7093 } while (i-- != 0); 7094 } 7095 #else 7096 if (len > 32) { 7097 if (len > 64) { 7098 if (len > 96) { 7099 acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); 7100 } 7101 acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); 7102 } 7103 acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); 7104 } 7105 acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); 7106 #endif 7107 { XXH128_hash_t h128; 7108 h128.low64 = acc.low64 + acc.high64; 7109 h128.high64 = (acc.low64 * XXH_PRIME64_1) 7110 + (acc.high64 * XXH_PRIME64_4) 7111 + ((len - seed) * XXH_PRIME64_2); 7112 h128.low64 = XXH3_avalanche(h128.low64); 7113 h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); 7114 return h128; 7115 } 7116 } 7117 } 7118 7119 XXH_NO_INLINE XXH_PUREF XXH128_hash_t 7120 XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, 7121 const xxh_u8* XXH_RESTRICT secret, size_t secretSize, 7122 XXH64_hash_t seed) 7123 { 7124 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; 7125 XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); 7126 7127 { XXH128_hash_t acc; 7128 unsigned i; 7129 acc.low64 = len * XXH_PRIME64_1; 7130 acc.high64 = 0; 7131 /* 7132 * We set as `i` as offset + 32. We do this so that unchanged 7133 * `len` can be used as upper bound. This reaches a sweet spot 7134 * where both x86 and aarch64 get simple agen and good codegen 7135 * for the loop. 7136 */ 7137 for (i = 32; i < 160; i += 32) { 7138 acc = XXH128_mix32B(acc, 7139 input + i - 32, 7140 input + i - 16, 7141 secret + i - 32, 7142 seed); 7143 } 7144 acc.low64 = XXH3_avalanche(acc.low64); 7145 acc.high64 = XXH3_avalanche(acc.high64); 7146 /* 7147 * NB: `i <= len` will duplicate the last 32-bytes if 7148 * len % 32 was zero. This is an unfortunate necessity to keep 7149 * the hash result stable. 7150 */ 7151 for (i=160; i <= len; i += 32) { 7152 acc = XXH128_mix32B(acc, 7153 input + i - 32, 7154 input + i - 16, 7155 secret + XXH3_MIDSIZE_STARTOFFSET + i - 160, 7156 seed); 7157 } 7158 /* last bytes */ 7159 acc = XXH128_mix32B(acc, 7160 input + len - 16, 7161 input + len - 32, 7162 secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, 7163 (XXH64_hash_t)0 - seed); 7164 7165 { XXH128_hash_t h128; 7166 h128.low64 = acc.low64 + acc.high64; 7167 h128.high64 = (acc.low64 * XXH_PRIME64_1) 7168 + (acc.high64 * XXH_PRIME64_4) 7169 + ((len - seed) * XXH_PRIME64_2); 7170 h128.low64 = XXH3_avalanche(h128.low64); 7171 h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); 7172 return h128; 7173 } 7174 } 7175 } 7176 7177 static XXH_PUREF XXH128_hash_t 7178 XXH3_finalizeLong_128b(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, size_t secretSize, xxh_u64 len) 7179 { 7180 XXH128_hash_t h128; 7181 h128.low64 = XXH3_finalizeLong_64b(acc, secret, len); 7182 h128.high64 = XXH3_mergeAccs(acc, secret + secretSize 7183 - XXH_STRIPE_LEN - XXH_SECRET_MERGEACCS_START, 7184 ~(len * XXH_PRIME64_2)); 7185 return h128; 7186 } 7187 7188 XXH_FORCE_INLINE XXH128_hash_t 7189 XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, 7190 const xxh_u8* XXH_RESTRICT secret, size_t secretSize, 7191 XXH3_f_accumulate f_acc, 7192 XXH3_f_scrambleAcc f_scramble) 7193 { 7194 XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; 7195 7196 XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble); 7197 7198 /* converge into final hash */ 7199 XXH_STATIC_ASSERT(sizeof(acc) == 64); 7200 XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); 7201 return XXH3_finalizeLong_128b(acc, secret, secretSize, (xxh_u64)len); 7202 } 7203 7204 /* 7205 * It's important for performance that XXH3_hashLong() is not inlined. 7206 */ 7207 XXH_NO_INLINE XXH_PUREF XXH128_hash_t 7208 XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, 7209 XXH64_hash_t seed64, 7210 const void* XXH_RESTRICT secret, size_t secretLen) 7211 { 7212 (void)seed64; (void)secret; (void)secretLen; 7213 return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), 7214 XXH3_accumulate, XXH3_scrambleAcc); 7215 } 7216 7217 /* 7218 * It's important for performance to pass @p secretLen (when it's static) 7219 * to the compiler, so that it can properly optimize the vectorized loop. 7220 * 7221 * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE 7222 * breaks -Og, this is XXH_NO_INLINE. 7223 */ 7224 XXH3_WITH_SECRET_INLINE XXH128_hash_t 7225 XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, 7226 XXH64_hash_t seed64, 7227 const void* XXH_RESTRICT secret, size_t secretLen) 7228 { 7229 (void)seed64; 7230 return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, 7231 XXH3_accumulate, XXH3_scrambleAcc); 7232 } 7233 7234 XXH_FORCE_INLINE XXH128_hash_t 7235 XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, 7236 XXH64_hash_t seed64, 7237 XXH3_f_accumulate f_acc, 7238 XXH3_f_scrambleAcc f_scramble, 7239 XXH3_f_initCustomSecret f_initSec) 7240 { 7241 if (seed64 == 0) 7242 return XXH3_hashLong_128b_internal(input, len, 7243 XXH3_kSecret, sizeof(XXH3_kSecret), 7244 f_acc, f_scramble); 7245 { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; 7246 f_initSec(secret, seed64); 7247 return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), 7248 f_acc, f_scramble); 7249 } 7250 } 7251 7252 /* 7253 * It's important for performance that XXH3_hashLong is not inlined. 7254 */ 7255 XXH_NO_INLINE XXH128_hash_t 7256 XXH3_hashLong_128b_withSeed(const void* input, size_t len, 7257 XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) 7258 { 7259 (void)secret; (void)secretLen; 7260 return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, 7261 XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); 7262 } 7263 7264 typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, 7265 XXH64_hash_t, const void* XXH_RESTRICT, size_t); 7266 7267 XXH_FORCE_INLINE XXH128_hash_t 7268 XXH3_128bits_internal(const void* input, size_t len, 7269 XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, 7270 XXH3_hashLong128_f f_hl128) 7271 { 7272 XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); 7273 /* 7274 * If an action is to be taken if `secret` conditions are not respected, 7275 * it should be done here. 7276 * For now, it's a contract pre-condition. 7277 * Adding a check and a branch here would cost performance at every hash. 7278 */ 7279 if (len <= 16) 7280 return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); 7281 if (len <= 128) 7282 return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); 7283 if (len <= XXH3_MIDSIZE_MAX) 7284 return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); 7285 return f_hl128(input, len, seed64, secret, secretLen); 7286 } 7287 7288 7289 /* === Public XXH128 API === */ 7290 7291 /*! @ingroup XXH3_family */ 7292 XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len) 7293 { 7294 return XXH3_128bits_internal(input, len, 0, 7295 XXH3_kSecret, sizeof(XXH3_kSecret), 7296 XXH3_hashLong_128b_default); 7297 } 7298 7299 /*! @ingroup XXH3_family */ 7300 XXH_PUBLIC_API XXH128_hash_t 7301 XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize) 7302 { 7303 return XXH3_128bits_internal(input, len, 0, 7304 (const xxh_u8*)secret, secretSize, 7305 XXH3_hashLong_128b_withSecret); 7306 } 7307 7308 /*! @ingroup XXH3_family */ 7309 XXH_PUBLIC_API XXH128_hash_t 7310 XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) 7311 { 7312 return XXH3_128bits_internal(input, len, seed, 7313 XXH3_kSecret, sizeof(XXH3_kSecret), 7314 XXH3_hashLong_128b_withSeed); 7315 } 7316 7317 /*! @ingroup XXH3_family */ 7318 XXH_PUBLIC_API XXH128_hash_t 7319 XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) 7320 { 7321 if (len <= XXH3_MIDSIZE_MAX) 7322 return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); 7323 return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); 7324 } 7325 7326 /*! @ingroup XXH3_family */ 7327 XXH_PUBLIC_API XXH128_hash_t 7328 XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) 7329 { 7330 return XXH3_128bits_withSeed(input, len, seed); 7331 } 7332 7333 7334 /* === XXH3 128-bit streaming === */ 7335 #ifndef XXH_NO_STREAM 7336 /* 7337 * All initialization and update functions are identical to 64-bit streaming variant. 7338 * The only difference is the finalization routine. 7339 */ 7340 7341 /*! @ingroup XXH3_family */ 7342 XXH_PUBLIC_API XXH_errorcode 7343 XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) 7344 { 7345 return XXH3_64bits_reset(statePtr); 7346 } 7347 7348 /*! @ingroup XXH3_family */ 7349 XXH_PUBLIC_API XXH_errorcode 7350 XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) 7351 { 7352 return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); 7353 } 7354 7355 /*! @ingroup XXH3_family */ 7356 XXH_PUBLIC_API XXH_errorcode 7357 XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) 7358 { 7359 return XXH3_64bits_reset_withSeed(statePtr, seed); 7360 } 7361 7362 /*! @ingroup XXH3_family */ 7363 XXH_PUBLIC_API XXH_errorcode 7364 XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) 7365 { 7366 return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); 7367 } 7368 7369 /*! @ingroup XXH3_family */ 7370 XXH_PUBLIC_API XXH_errorcode 7371 XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) 7372 { 7373 return XXH3_update_regular(state, input, len); 7374 } 7375 7376 /*! @ingroup XXH3_family */ 7377 XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state) 7378 { 7379 const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; 7380 if (state->totalLen > XXH3_MIDSIZE_MAX) { 7381 XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; 7382 XXH3_digest_long(acc, state, secret); 7383 XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); 7384 return XXH3_finalizeLong_128b(acc, secret, state->secretLimit + XXH_STRIPE_LEN, (xxh_u64)state->totalLen); 7385 } 7386 /* len <= XXH3_MIDSIZE_MAX : short code */ 7387 if (state->useSeed) 7388 return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); 7389 return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), 7390 secret, state->secretLimit + XXH_STRIPE_LEN); 7391 } 7392 #endif /* !XXH_NO_STREAM */ 7393 /* 128-bit utility functions */ 7394 7395 /* return : 1 is equal, 0 if different */ 7396 /*! @ingroup XXH3_family */ 7397 XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) 7398 { 7399 /* note : XXH128_hash_t is compact, it has no padding byte */ 7400 return !(XXH_memcmp(&h1, &h2, sizeof(h1))); 7401 } 7402 7403 /* This prototype is compatible with stdlib's qsort(). 7404 * @return : >0 if *h128_1 > *h128_2 7405 * <0 if *h128_1 < *h128_2 7406 * =0 if *h128_1 == *h128_2 */ 7407 /*! @ingroup XXH3_family */ 7408 XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2) 7409 { 7410 XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; 7411 XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; 7412 int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); 7413 /* note : bets that, in most cases, hash values are different */ 7414 if (hcmp) return hcmp; 7415 return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); 7416 } 7417 7418 7419 /*====== Canonical representation ======*/ 7420 /*! @ingroup XXH3_family */ 7421 XXH_PUBLIC_API void 7422 XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash) 7423 { 7424 XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); 7425 if (XXH_CPU_LITTLE_ENDIAN) { 7426 hash.high64 = XXH_swap64(hash.high64); 7427 hash.low64 = XXH_swap64(hash.low64); 7428 } 7429 XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); 7430 XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); 7431 } 7432 7433 /*! @ingroup XXH3_family */ 7434 XXH_PUBLIC_API XXH128_hash_t 7435 XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src) 7436 { 7437 XXH128_hash_t h; 7438 h.high64 = XXH_readBE64(src); 7439 h.low64 = XXH_readBE64(src->digest + 8); 7440 return h; 7441 } 7442 7443 7444 7445 /* ========================================== 7446 * Secret generators 7447 * ========================================== 7448 */ 7449 #define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) 7450 7451 XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) 7452 { 7453 XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); 7454 XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); 7455 } 7456 7457 /*! @ingroup XXH3_family */ 7458 XXH_PUBLIC_API XXH_errorcode 7459 XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize) 7460 { 7461 #if (XXH_DEBUGLEVEL >= 1) 7462 XXH_ASSERT(secretBuffer != NULL); 7463 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); 7464 #else 7465 /* production mode, assert() are disabled */ 7466 if (secretBuffer == NULL) return XXH_ERROR; 7467 if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; 7468 #endif 7469 7470 if (customSeedSize == 0) { 7471 customSeed = XXH3_kSecret; 7472 customSeedSize = XXH_SECRET_DEFAULT_SIZE; 7473 } 7474 #if (XXH_DEBUGLEVEL >= 1) 7475 XXH_ASSERT(customSeed != NULL); 7476 #else 7477 if (customSeed == NULL) return XXH_ERROR; 7478 #endif 7479 7480 /* Fill secretBuffer with a copy of customSeed - repeat as needed */ 7481 { size_t pos = 0; 7482 while (pos < secretSize) { 7483 size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); 7484 XXH_memcpy((char*)secretBuffer + pos, customSeed, toCopy); 7485 pos += toCopy; 7486 } } 7487 7488 { size_t const nbSeg16 = secretSize / 16; 7489 size_t n; 7490 XXH128_canonical_t scrambler; 7491 XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); 7492 for (n=0; n<nbSeg16; n++) { 7493 XXH128_hash_t const h128 = XXH128(&scrambler, sizeof(scrambler), n); 7494 XXH3_combine16((char*)secretBuffer + n*16, h128); 7495 } 7496 /* last segment */ 7497 XXH3_combine16((char*)secretBuffer + secretSize - 16, XXH128_hashFromCanonical(&scrambler)); 7498 } 7499 return XXH_OK; 7500 } 7501 7502 /*! @ingroup XXH3_family */ 7503 XXH_PUBLIC_API void 7504 XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed) 7505 { 7506 XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; 7507 XXH3_initCustomSecret(secret, seed); 7508 XXH_ASSERT(secretBuffer != NULL); 7509 XXH_memcpy(secretBuffer, secret, XXH_SECRET_DEFAULT_SIZE); 7510 } 7511 7512 7513 7514 /* Pop our optimization override from above */ 7515 #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ 7516 && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ 7517 && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ 7518 # pragma GCC pop_options 7519 #endif 7520 7521 #endif /* XXH_NO_LONG_LONG */ 7522 7523 #endif /* XXH_NO_XXH3 */ 7524 7525 /*! 7526 * @} 7527 */ 7528 #endif /* XXH_IMPLEMENTATION */